From c895d4c54f337691b3a01994c62d42e138d3f7bb Mon Sep 17 00:00:00 2001 From: qiuxin2012 Date: Thu, 28 Mar 2019 13:09:17 +0800 Subject: [PATCH 01/28] py inception --- .../examples/tensorflow/inception/__init__.py | 0 .../tensorflow/inception/bigdlinception.py | 364 +++++++++++++++++ .../tensorflow/inception/inception.py | 380 ++++++++++++++++++ pyzoo/zoo/feature/image/imagePreprocessing.py | 16 + .../zoo/pipeline/api/keras/engine/topology.py | 3 +- .../feature/python/PythonImageFeature.scala | 9 + .../pipeline/api/keras/models/Topology.scala | 39 +- .../api/keras/python/PythonZooKeras.scala | 10 + 8 files changed, 819 insertions(+), 2 deletions(-) create mode 100644 pyzoo/zoo/examples/tensorflow/inception/__init__.py create mode 100644 pyzoo/zoo/examples/tensorflow/inception/bigdlinception.py create mode 100644 pyzoo/zoo/examples/tensorflow/inception/inception.py diff --git a/pyzoo/zoo/examples/tensorflow/inception/__init__.py b/pyzoo/zoo/examples/tensorflow/inception/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/pyzoo/zoo/examples/tensorflow/inception/bigdlinception.py b/pyzoo/zoo/examples/tensorflow/inception/bigdlinception.py new file mode 100644 index 00000000000..6a7f56beb55 --- /dev/null +++ b/pyzoo/zoo/examples/tensorflow/inception/bigdlinception.py @@ -0,0 +1,364 @@ +from bigdl.nn.layer import * +from optparse import OptionParser +from bigdl.nn.criterion import * +from bigdl.nn.initialization_method import * +from bigdl.optim.optimizer import * +from bigdl.transform.vision.image import * +from math import ceil +from zoo.feature.common import * +from zoo.feature.image.imagePreprocessing import * +from zoo.feature.common import * + + + +def t(input_t): + if type(input_t) is list: + # insert into index 0 spot, such that the real data starts from index 1 + temp = [0] + temp.extend(input_t) + return dict(enumerate(temp)) + # if dictionary, return it back + return input_t + + +def inception_layer_v1(input_size, config, name_prefix=""): + concat = Concat(2) + conv1 = Sequential() + conv1.add(SpatialConvolution(input_size, config[1][1], 1, 1, 1, 1) + .set_init_method(weight_init_method=Xavier(),bias_init_method=Zeros()) + .set_name(name_prefix + "1x1")) + conv1.add(ReLU(True).set_name(name_prefix + "relu_1x1")) + concat.add(conv1) + conv3 = Sequential() + conv3.add(SpatialConvolution(input_size, config[2][1], 1, 1, 1, 1) + .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) + .set_name(name_prefix + "3x3_reduce")) + conv3.add(ReLU(True).set_name(name_prefix + "relu_3x3_reduce")) + conv3.add(SpatialConvolution(config[2][1], config[2][2], 3, 3, 1, 1, 1, 1) + .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) + .set_name(name_prefix + "3x3")) + conv3.add(ReLU(True).set_name(name_prefix + "relu_3x3")) + concat.add(conv3) + conv5 = Sequential() + conv5.add(SpatialConvolution(input_size, config[3][1], 1, 1, 1, 1) + .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) + .set_name(name_prefix + "5x5_reduce")) + conv5.add(ReLU(True).set_name(name_prefix + "relu_5x5_reduce")) + conv5.add(SpatialConvolution(config[3][1], config[3][2], 5, 5, 1, 1, 2, 2) + .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) + .set_name(name_prefix + "5x5")) + conv5.add(ReLU(True).set_name(name_prefix + "relu_5x5")) + concat.add(conv5) + pool = Sequential() + pool.add(SpatialMaxPooling(3, 3, 1, 1, 1, 1, to_ceil=True).set_name(name_prefix + "pool")) + pool.add(SpatialConvolution(input_size, config[4][1], 1, 1, 1, 1) + .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) + .set_name(name_prefix + "pool_proj")) + pool.add(ReLU(True).set_name(name_prefix + "relu_pool_proj")) + concat.add(pool).set_name(name_prefix + "output") + return concat + + +def inception_v1_no_aux_classifier(class_num, has_dropout=True): + model = Sequential() + model.add(SpatialConvolution(3, 64, 7, 7, 2, 2, 3, 3, 1, False) + .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) + .set_name("conv1/7x7_s2")) + model.add(ReLU(True).set_name("conv1/relu_7x7")) + model.add(SpatialMaxPooling(3, 3, 2, 2, to_ceil=True).set_name("pool1/3x3_s2")) + model.add(SpatialCrossMapLRN(5, 0.0001, 0.75).set_name("pool1/norm1")) + model.add(SpatialConvolution(64, 64, 1, 1, 1, 1) + .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) + .set_name("conv2/3x3_reduce")) + model.add(ReLU(True).set_name("conv2/relu_3x3_reduce")) + model.add(SpatialConvolution(64, 192, 3, 3, 1, 1, 1, 1) + .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) + .set_name("conv2/3x3")) + model.add(ReLU(True).set_name("conv2/relu_3x3")) + model.add(SpatialCrossMapLRN(5, 0.0001, 0.75).set_name("conv2/norm2")) + model.add(SpatialMaxPooling(3, 3, 2, 2, to_ceil=True).set_name("pool2/3x3_s2")) + model.add(inception_layer_v1(192, t([t([64]), t( + [96, 128]), t([16, 32]), t([32])]), "inception_3a/")) + model.add(inception_layer_v1(256, t([t([128]), t( + [128, 192]), t([32, 96]), t([64])]), "inception_3b/")) + model.add(SpatialMaxPooling(3, 3, 2, 2, to_ceil=True)) + model.add(inception_layer_v1(480, t([t([192]), t( + [96, 208]), t([16, 48]), t([64])]), "inception_4a/")) + model.add(inception_layer_v1(512, t([t([160]), t( + [112, 224]), t([24, 64]), t([64])]), "inception_4b/")) + model.add(inception_layer_v1(512, t([t([128]), t( + [128, 256]), t([24, 64]), t([64])]), "inception_4c/")) + model.add(inception_layer_v1(512, t([t([112]), t( + [144, 288]), t([32, 64]), t([64])]), "inception_4d/")) + model.add(inception_layer_v1(528, t([t([256]), t( + [160, 320]), t([32, 128]), t([128])]), "inception_4e/")) + model.add(SpatialMaxPooling(3, 3, 2, 2, to_ceil=True)) + model.add(inception_layer_v1(832, t([t([256]), t( + [160, 320]), t([32, 128]), t([128])]), "inception_5a/")) + model.add(inception_layer_v1(832, t([t([384]), t( + [192, 384]), t([48, 128]), t([128])]), "inception_5b/")) + model.add(SpatialAveragePooling(7, 7, 1, 1).set_name("pool5/7x7_s1")) + if has_dropout: + model.add(Dropout(0.4).set_name("pool5/drop_7x7_s1")) + model.add(View([1024], num_input_dims=3)) + model.add(Linear(1024, class_num) + .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) + .set_name("loss3/classifier")) + model.add(LogSoftMax().set_name("loss3/loss3")) + model.reset() + return model + + +def inception_v1(class_num, has_dropout=True): + feature1 = Sequential() + feature1.add(SpatialConvolution(3, 64, 7, 7, 2, 2, 3, 3, 1, False) + .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) + .set_name("conv1/7x7_s2")) + feature1.add(ReLU(True).set_name("conv1/relu_7x7")) + feature1.add( + SpatialMaxPooling(3, 3, 2, 2, to_ceil=True) + .set_name("pool1/3x3_s2")) + feature1.add(SpatialCrossMapLRN(5, 0.0001, 0.75) + .set_name("pool1/norm1")) + feature1.add(SpatialConvolution(64, 64, 1, 1, 1, 1) + .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) + .set_name("conv2/3x3_reduce")) + feature1.add(ReLU(True).set_name("conv2/relu_3x3_reduce")) + feature1.add(SpatialConvolution(64, 192, 3, 3, 1, 1, 1, 1) + .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) + .set_name("conv2/3x3")) + feature1.add(ReLU(True).set_name("conv2/relu_3x3")) + feature1.add(SpatialCrossMapLRN(5, 0.0001, 0.75).set_name("conv2/norm2")) + feature1.add( + SpatialMaxPooling(3, 3, 2, 2, to_ceil=True).set_name("pool2/3x3_s2")) + feature1.add(inception_layer_v1(192, t([ + t([64]), t([96, 128]), t([16, 32]), t([32])]), + "inception_3a/")) + feature1.add(inception_layer_v1(256, t([ + t([128]), t([128, 192]), t([32, 96]), t([64])]), + "inception_3b/")) + feature1.add( + SpatialMaxPooling(3, 3, 2, 2, to_ceil=True).set_name("pool3/3x3_s2")) + feature1.add(inception_layer_v1(480, t([ + t([192]), t([96, 208]), t([16, 48]), t([64])]), + "inception_4a/")) + + output1 = Sequential() + output1.add(SpatialAveragePooling( + 5, 5, 3, 3, ceil_mode=True).set_name("loss1/ave_pool")) + output1.add( + SpatialConvolution(512, 128, 1, 1, 1, 1).set_name("loss1/conv")) + output1.add(ReLU(True).set_name("loss1/relu_conv")) + output1.add(View([128 * 4 * 4, 3])) + output1.add(Linear(128 * 4 * 4, 1024).set_name("loss1/fc")) + output1.add(ReLU(True).set_name("loss1/relu_fc")) + if has_dropout: + output1.add(Dropout(0.7).set_name("loss1/drop_fc")) + output1.add(Linear(1024, class_num).set_name("loss1/classifier")) + output1.add(LogSoftMax().set_name("loss1/loss")) + + feature2 = Sequential() + feature2.add(inception_layer_v1(512, + t([t([160]), t([112, 224]), t([24, 64]), t([64])]), + "inception_4b/")) + feature2.add(inception_layer_v1(512, + t([t([128]), t([128, 256]), t([24, 64]), t([64])]), + "inception_4c/")) + feature2.add(inception_layer_v1(512, + t([t([112]), t([144, 288]), t([32, 64]), t([64])]), + "inception_4d/")) + + output2 = Sequential() + output2.add(SpatialAveragePooling(5, 5, 3, 3).set_name("loss2/ave_pool")) + output2.add( + SpatialConvolution(528, 128, 1, 1, 1, 1).set_name("loss2/conv")) + output2.add(ReLU(True).set_name("loss2/relu_conv")) + output2.add(View([128 * 4 * 4, 3])) + output2.add(Linear(128 * 4 * 4, 1024).set_name("loss2/fc")) + output2.add(ReLU(True).set_name("loss2/relu_fc")) + if has_dropout: + output2.add(Dropout(0.7).set_name("loss2/drop_fc")) + output2.add(Linear(1024, class_num).set_name("loss2/classifier")) + output2.add(LogSoftMax().set_name("loss2/loss")) + + output3 = Sequential() + output3.add(inception_layer_v1(528, + t([t([256]), t([160, 320]), t([32, 128]), t([128])]), + "inception_4e/")) + output3.add(SpatialMaxPooling(3, 3, 2, 2, to_ceil=True).set_name("pool4/3x3_s2")) + output3.add(inception_layer_v1(832, + t([t([256]), t([160, 320]), t([32, 128]), t([128])]), + "inception_5a/")) + output3.add(inception_layer_v1(832, + t([t([384]), t([192, 384]), t([48, 128]), t([128])]), + "inception_5b/")) + output3.add(SpatialAveragePooling(7, 7, 1, 1).set_name("pool5/7x7_s1")) + if has_dropout: + output3.add(Dropout(0.4).set_name("pool5/drop_7x7_s1")) + output3.add(View([1024, 3])) + output3.add(Linear(1024, class_num) + .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) + .set_name("loss3/classifier")) + output3.add(LogSoftMax().set_name("loss3/loss3")) + + split2 = Concat(2).set_name("split2") + split2.add(output3) + split2.add(output2) + + mainBranch = Sequential() + mainBranch.add(feature2) + mainBranch.add(split2) + + split1 = Concat(2).set_name("split1") + split1.add(mainBranch) + split1.add(output1) + + model = Sequential() + + model.add(feature1) + model.add(split1) + + model.reset() + return model + + +def get_inception_data(url, sc=None, data_type="train"): + path = os.path.join(url, data_type) + return SeqFileFolder.files_to_image_frame(url=path, sc=sc, class_num=1000) + + +def config_option_parser(): + parser = OptionParser() + parser.add_option("-f", "--folder", type=str, dest="folder", default="", + help="url of hdfs folder store the hadoop sequence files") + parser.add_option("--model", type=str, dest="model", default="", help="model snapshot location") + parser.add_option("--state", type=str, dest="state", default="", help="state snapshot location") + parser.add_option("--checkpoint", type=str, dest="checkpoint", default="", help="where to cache the model") + parser.add_option("-o", "--overwrite", action="store_true", dest="overwrite", default=False, + help="overwrite checkpoint files") + parser.add_option("-e", "--maxEpoch", type=int, dest="maxEpoch", default=0, help="epoch numbers") + parser.add_option("-i", "--maxIteration", type=int, dest="maxIteration", default=62000, help="iteration numbers") + parser.add_option("-l", "--learningRate", type=float, dest="learningRate", default=0.01, help="learning rate") + parser.add_option("--warmupEpoch", type=int, dest="warmupEpoch", default=0, help="warm up epoch numbers") + parser.add_option("--maxLr", type=float, dest="maxLr", default=0.0, help="max Lr after warm up") + parser.add_option("-b", "--batchSize", type=int, dest="batchSize", help="batch size") + parser.add_option("--classNum", type=int, dest="classNum", default=1000, help="class number") + parser.add_option("--weightDecay", type=float, dest="weightDecay", default=0.0001, help="weight decay") + parser.add_option("--checkpointIteration", type=int, dest="checkpointIteration", default=620, + help="checkpoint interval of iterations") + parser.add_option("--gradientMin", type=float, dest="gradientMin", default=0.0, help="min gradient clipping by") + parser.add_option("--gradientMax", type=float, dest="gradientMax", default=0.0, help="max gradient clipping by") + parser.add_option("--gradientL2NormThreshold", type=float, dest="gradientL2NormThreshold", default=0.0, help="gradient L2-Norm threshold") + + return parser + + +if __name__ == "__main__": + # parse options + parser = config_option_parser() + (options, args) = parser.parse_args(sys.argv) + if not options.learningRate: + parser.error("-l --learningRate is a mandatory opt") + if not options.batchSize: + parser.error("-b --batchSize is a mandatory opt") + + # init + sparkConf = create_spark_conf().setAppName("inception v1") + sc = get_spark_context(sparkConf) + redire_spark_logs() + show_bigdl_info_logs() + init_engine() + + image_size = 224 # create dataset + train_transformer = ChainedPreprocessing([ImagePixelBytesToMat(), + ImageRandomCrop(image_size, image_size), + ImageRandomPreprocessing(ImageHFlip(), 0.5), + ImageChannelNormalize(0.485, 0.456, 0.406, 0.229, 0.224, 0.225), + ImageMatToTensor(to_RGB=True), + ImageSetToSample(input_keys=["imageTensor"], target_keys=["label"]) + ]) + raw_train_data = get_inception_data(options.folder, sc, "train") + train_data = FeatureSet.image_frame(raw_train_data).transform(train_transformer).to_dataset() + + val_transformer = ChainedPreprocessing([ImagePixelBytesToMat(), + ImageCenterCrop(image_size, image_size), + ImageRandomPreprocessing(ImageHFlip(), 0.5), + ImageChannelNormalize(0.485, 0.456, 0.406, 0.229, 0.224, 0.225), + ImageMatToTensor(to_RGB=True), + ImageSetToSample(input_keys=["imageTensor"], target_keys=["label"]), + ]) + raw_val_data = get_inception_data(options.folder, sc, "val") + val_data = FeatureSet.image_frame(raw_val_data).transform(val_transformer).to_dataset() + + + # build model + if options.model != "": + # load model snapshot + inception_model = Model.load(options.model) + else: + inception_model = inception_v1_no_aux_classifier(options.classNum) + + # set optimization method + iterationPerEpoch = int(ceil(float(1281167) / options.batchSize)) + if options.maxEpoch: + maxIteration = iterationPerEpoch * options.maxEpoch + else: + maxIteration = options.maxIteration + warmup_iteration = options.warmupEpoch * iterationPerEpoch + if options.state != "": + # load state snapshot + optim = OptimMethod.load(options.state) + else: + if warmup_iteration == 0: + warmupDelta = 0.0 + else: + if options.maxLr: + maxlr = options.maxLr + else: + maxlr = options.learningRate + warmupDelta = (maxlr - options.learningRate)/warmup_iteration + polyIteration = maxIteration - warmup_iteration + lrSchedule = SequentialSchedule(iterationPerEpoch) + lrSchedule.add(Warmup(warmupDelta), warmup_iteration) + lrSchedule.add(Poly(0.5, polyIteration), polyIteration) + optim = SGD(learningrate=options.learningRate, learningrate_decay=0.0, weightdecay=options.weightDecay, + momentum=0.9, dampening=0.0, nesterov=False, + leaningrate_schedule=lrSchedule) + + # create triggers + if options.maxEpoch: + checkpoint_trigger = EveryEpoch() + test_trigger = EveryEpoch() + end_trigger = MaxEpoch(options.maxEpoch) + else: + checkpoint_trigger = SeveralIteration(options.checkpointIteration) + test_trigger = SeveralIteration(options.checkpointIteration) + end_trigger = MaxIteration(options.maxIteration) + + # Optimizer + optimizer = Optimizer.create( + model=inception_model, + training_set=train_data, + optim_method=optim, + criterion=ClassNLLCriterion(), + end_trigger=end_trigger, + batch_size=options.batchSize + ) + + if options.checkpoint: + optimizer.set_checkpoint(checkpoint_trigger, options.checkpoint, options.overwrite) + + if options.gradientMin and options.gradientMax: + optimizer.set_gradclip_const(options.gradientMin, options.gradientMax) + + if options.gradientL2NormThreshold: + optimizer.set_gradclip_l2norm(options.gradientL2NormThreshold) + + optimizer.set_validation(trigger=test_trigger, + val_rdd=val_data, + batch_size=options.batchSize, + val_method=[Top1Accuracy(), Top5Accuracy()]) + + trained_model = optimizer.optimize() + + sc.stop() diff --git a/pyzoo/zoo/examples/tensorflow/inception/inception.py b/pyzoo/zoo/examples/tensorflow/inception/inception.py new file mode 100644 index 00000000000..162be5dcea6 --- /dev/null +++ b/pyzoo/zoo/examples/tensorflow/inception/inception.py @@ -0,0 +1,380 @@ +from bigdl.nn.layer import Sequential, Concat, SpatialConvolution, ReLU, SpatialMaxPooling, SpatialCrossMapLRN, SpatialAveragePooling, Dropout, View, Linear, LogSoftMax +from optparse import OptionParser +from bigdl.nn.criterion import * +from bigdl.nn.initialization_method import * +from bigdl.optim.optimizer import * +from zoo.feature.image.imagePreprocessing import * +from zoo.feature.image.imageset import * +from zoo.feature.common import ChainedPreprocessing +from math import ceil +from zoo.common.nncontext import * +from zoo.feature.common import * +from zoo.feature.image.imagePreprocessing import * + +from zoo.common.nncontext import * +from zoo.feature.common import * +from zoo.feature.image.imagePreprocessing import * +from zoo.pipeline.api.keras.layers import Dense, Input, Flatten +from zoo.pipeline.api.net import * +from zoo.pipeline.api.keras.models import Model +from bigdl.optim.optimizer import * + + +def t(input_t): + if type(input_t) is list: + # insert into index 0 spot, such that the real data starts from index 1 + temp = [0] + temp.extend(input_t) + return dict(enumerate(temp)) + # if dictionary, return it back + return input_t + + +def inception_layer_v1(input_size, config, name_prefix=""): + concat = Concat(2) + conv1 = Sequential() + conv1.add(SpatialConvolution(input_size, config[1][1], 1, 1, 1, 1) + .set_init_method(weight_init_method=Xavier(),bias_init_method=Zeros()) + .set_name(name_prefix + "1x1")) + conv1.add(ReLU(True).set_name(name_prefix + "relu_1x1")) + concat.add(conv1) + conv3 = Sequential() + conv3.add(SpatialConvolution(input_size, config[2][1], 1, 1, 1, 1) + .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) + .set_name(name_prefix + "3x3_reduce")) + conv3.add(ReLU(True).set_name(name_prefix + "relu_3x3_reduce")) + conv3.add(SpatialConvolution(config[2][1], config[2][2], 3, 3, 1, 1, 1, 1) + .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) + .set_name(name_prefix + "3x3")) + conv3.add(ReLU(True).set_name(name_prefix + "relu_3x3")) + concat.add(conv3) + conv5 = Sequential() + conv5.add(SpatialConvolution(input_size, config[3][1], 1, 1, 1, 1) + .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) + .set_name(name_prefix + "5x5_reduce")) + conv5.add(ReLU(True).set_name(name_prefix + "relu_5x5_reduce")) + conv5.add(SpatialConvolution(config[3][1], config[3][2], 5, 5, 1, 1, 2, 2) + .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) + .set_name(name_prefix + "5x5")) + conv5.add(ReLU(True).set_name(name_prefix + "relu_5x5")) + concat.add(conv5) + pool = Sequential() + pool.add(SpatialMaxPooling(3, 3, 1, 1, 1, 1, to_ceil=True).set_name(name_prefix + "pool")) + pool.add(SpatialConvolution(input_size, config[4][1], 1, 1, 1, 1) + .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) + .set_name(name_prefix + "pool_proj")) + pool.add(ReLU(True).set_name(name_prefix + "relu_pool_proj")) + concat.add(pool).set_name(name_prefix + "output") + return concat + + +def inception_v1_no_aux_classifier(class_num, has_dropout=True): + model = Sequential() + model.add(SpatialConvolution(3, 64, 7, 7, 2, 2, 3, 3, 1, False) + .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) + .set_name("conv1/7x7_s2")) + model.add(ReLU(True).set_name("conv1/relu_7x7")) + model.add(SpatialMaxPooling(3, 3, 2, 2, to_ceil=True).set_name("pool1/3x3_s2")) + model.add(SpatialCrossMapLRN(5, 0.0001, 0.75).set_name("pool1/norm1")) + model.add(SpatialConvolution(64, 64, 1, 1, 1, 1) + .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) + .set_name("conv2/3x3_reduce")) + model.add(ReLU(True).set_name("conv2/relu_3x3_reduce")) + model.add(SpatialConvolution(64, 192, 3, 3, 1, 1, 1, 1) + .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) + .set_name("conv2/3x3")) + model.add(ReLU(True).set_name("conv2/relu_3x3")) + model.add(SpatialCrossMapLRN(5, 0.0001, 0.75).set_name("conv2/norm2")) + model.add(SpatialMaxPooling(3, 3, 2, 2, to_ceil=True).set_name("pool2/3x3_s2")) + model.add(inception_layer_v1(192, t([t([64]), t( + [96, 128]), t([16, 32]), t([32])]), "inception_3a/")) + model.add(inception_layer_v1(256, t([t([128]), t( + [128, 192]), t([32, 96]), t([64])]), "inception_3b/")) + model.add(SpatialMaxPooling(3, 3, 2, 2, to_ceil=True)) + model.add(inception_layer_v1(480, t([t([192]), t( + [96, 208]), t([16, 48]), t([64])]), "inception_4a/")) + model.add(inception_layer_v1(512, t([t([160]), t( + [112, 224]), t([24, 64]), t([64])]), "inception_4b/")) + model.add(inception_layer_v1(512, t([t([128]), t( + [128, 256]), t([24, 64]), t([64])]), "inception_4c/")) + model.add(inception_layer_v1(512, t([t([112]), t( + [144, 288]), t([32, 64]), t([64])]), "inception_4d/")) + model.add(inception_layer_v1(528, t([t([256]), t( + [160, 320]), t([32, 128]), t([128])]), "inception_4e/")) + model.add(SpatialMaxPooling(3, 3, 2, 2, to_ceil=True)) + model.add(inception_layer_v1(832, t([t([256]), t( + [160, 320]), t([32, 128]), t([128])]), "inception_5a/")) + model.add(inception_layer_v1(832, t([t([384]), t( + [192, 384]), t([48, 128]), t([128])]), "inception_5b/")) + model.add(SpatialAveragePooling(7, 7, 1, 1).set_name("pool5/7x7_s1")) + if has_dropout: + model.add(Dropout(0.4).set_name("pool5/drop_7x7_s1")) + model.add(View([1024], num_input_dims=3)) + model.add(Linear(1024, class_num) + .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) + .set_name("loss3/classifier")) + model.add(LogSoftMax().set_name("loss3/loss3")) + model.reset() + return model + + +def inception_v1(class_num, has_dropout=True): + feature1 = Sequential() + feature1.add(SpatialConvolution(3, 64, 7, 7, 2, 2, 3, 3, 1, False) + .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) + .set_name("conv1/7x7_s2")) + feature1.add(ReLU(True).set_name("conv1/relu_7x7")) + feature1.add( + SpatialMaxPooling(3, 3, 2, 2, to_ceil=True) + .set_name("pool1/3x3_s2")) + feature1.add(SpatialCrossMapLRN(5, 0.0001, 0.75) + .set_name("pool1/norm1")) + feature1.add(SpatialConvolution(64, 64, 1, 1, 1, 1) + .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) + .set_name("conv2/3x3_reduce")) + feature1.add(ReLU(True).set_name("conv2/relu_3x3_reduce")) + feature1.add(SpatialConvolution(64, 192, 3, 3, 1, 1, 1, 1) + .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) + .set_name("conv2/3x3")) + feature1.add(ReLU(True).set_name("conv2/relu_3x3")) + feature1.add(SpatialCrossMapLRN(5, 0.0001, 0.75).set_name("conv2/norm2")) + feature1.add( + SpatialMaxPooling(3, 3, 2, 2, to_ceil=True).set_name("pool2/3x3_s2")) + feature1.add(inception_layer_v1(192, t([ + t([64]), t([96, 128]), t([16, 32]), t([32])]), + "inception_3a/")) + feature1.add(inception_layer_v1(256, t([ + t([128]), t([128, 192]), t([32, 96]), t([64])]), + "inception_3b/")) + feature1.add( + SpatialMaxPooling(3, 3, 2, 2, to_ceil=True).set_name("pool3/3x3_s2")) + feature1.add(inception_layer_v1(480, t([ + t([192]), t([96, 208]), t([16, 48]), t([64])]), + "inception_4a/")) + + output1 = Sequential() + output1.add(SpatialAveragePooling( + 5, 5, 3, 3, ceil_mode=True).set_name("loss1/ave_pool")) + output1.add( + SpatialConvolution(512, 128, 1, 1, 1, 1).set_name("loss1/conv")) + output1.add(ReLU(True).set_name("loss1/relu_conv")) + output1.add(View([128 * 4 * 4, 3])) + output1.add(Linear(128 * 4 * 4, 1024).set_name("loss1/fc")) + output1.add(ReLU(True).set_name("loss1/relu_fc")) + if has_dropout: + output1.add(Dropout(0.7).set_name("loss1/drop_fc")) + output1.add(Linear(1024, class_num).set_name("loss1/classifier")) + output1.add(LogSoftMax().set_name("loss1/loss")) + + feature2 = Sequential() + feature2.add(inception_layer_v1(512, + t([t([160]), t([112, 224]), t([24, 64]), t([64])]), + "inception_4b/")) + feature2.add(inception_layer_v1(512, + t([t([128]), t([128, 256]), t([24, 64]), t([64])]), + "inception_4c/")) + feature2.add(inception_layer_v1(512, + t([t([112]), t([144, 288]), t([32, 64]), t([64])]), + "inception_4d/")) + + output2 = Sequential() + output2.add(SpatialAveragePooling(5, 5, 3, 3).set_name("loss2/ave_pool")) + output2.add( + SpatialConvolution(528, 128, 1, 1, 1, 1).set_name("loss2/conv")) + output2.add(ReLU(True).set_name("loss2/relu_conv")) + output2.add(View([128 * 4 * 4, 3])) + output2.add(Linear(128 * 4 * 4, 1024).set_name("loss2/fc")) + output2.add(ReLU(True).set_name("loss2/relu_fc")) + if has_dropout: + output2.add(Dropout(0.7).set_name("loss2/drop_fc")) + output2.add(Linear(1024, class_num).set_name("loss2/classifier")) + output2.add(LogSoftMax().set_name("loss2/loss")) + + output3 = Sequential() + output3.add(inception_layer_v1(528, + t([t([256]), t([160, 320]), t([32, 128]), t([128])]), + "inception_4e/")) + output3.add(SpatialMaxPooling(3, 3, 2, 2, to_ceil=True).set_name("pool4/3x3_s2")) + output3.add(inception_layer_v1(832, + t([t([256]), t([160, 320]), t([32, 128]), t([128])]), + "inception_5a/")) + output3.add(inception_layer_v1(832, + t([t([384]), t([192, 384]), t([48, 128]), t([128])]), + "inception_5b/")) + output3.add(SpatialAveragePooling(7, 7, 1, 1).set_name("pool5/7x7_s1")) + if has_dropout: + output3.add(Dropout(0.4).set_name("pool5/drop_7x7_s1")) + output3.add(View([1024, 3])) + output3.add(Linear(1024, class_num) + .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) + .set_name("loss3/classifier")) + output3.add(LogSoftMax().set_name("loss3/loss3")) + + split2 = Concat(2).set_name("split2") + split2.add(output3) + split2.add(output2) + + mainBranch = Sequential() + mainBranch.add(feature2) + mainBranch.add(split2) + + split1 = Concat(2).set_name("split1") + split1.add(mainBranch) + split1.add(output1) + + model = Sequential() + + model.add(feature1) + model.add(split1) + + model.reset() + return model + + +def get_inception_data(url, sc=None, data_type="train"): + path = os.path.join(url, data_type) + return SeqFileFolder.files_to_image_frame(url=path, sc=sc, class_num=1000) + + +def config_option_parser(): + parser = OptionParser() + parser.add_option("-f", "--folder", type=str, dest="folder", default="", + help="url of hdfs folder store the hadoop sequence files") + parser.add_option("--model", type=str, dest="model", default="", help="model snapshot location") + parser.add_option("--state", type=str, dest="state", default="", help="state snapshot location") + parser.add_option("--checkpoint", type=str, dest="checkpoint", default="", help="where to cache the model") + parser.add_option("-o", "--overwrite", action="store_true", dest="overwrite", default=False, + help="overwrite checkpoint files") + parser.add_option("-e", "--maxEpoch", type=int, dest="maxEpoch", default=0, help="epoch numbers") + parser.add_option("-i", "--maxIteration", type=int, dest="maxIteration", default=62000, help="iteration numbers") + parser.add_option("-l", "--learningRate", type=float, dest="learningRate", default=0.01, help="learning rate") + parser.add_option("--warmupEpoch", type=int, dest="warmupEpoch", default=0, help="warm up epoch numbers") + parser.add_option("--maxLr", type=float, dest="maxLr", default=0.0, help="max Lr after warm up") + parser.add_option("-b", "--batchSize", type=int, dest="batchSize", help="batch size") + parser.add_option("--classNum", type=int, dest="classNum", default=1000, help="class number") + parser.add_option("--weightDecay", type=float, dest="weightDecay", default=0.0001, help="weight decay") + parser.add_option("--checkpointIteration", type=int, dest="checkpointIteration", default=620, + help="checkpoint interval of iterations") + parser.add_option("--gradientMin", type=float, dest="gradientMin", default=0.0, help="min gradient clipping by") + parser.add_option("--gradientMax", type=float, dest="gradientMax", default=0.0, help="max gradient clipping by") + parser.add_option("--gradientL2NormThreshold", type=float, dest="gradientL2NormThreshold", default=0.0, help="gradient L2-Norm threshold") + + return parser + + +if __name__ == "__main__": + # parse options + parser = config_option_parser() + (options, args) = parser.parse_args(sys.argv) + if not options.learningRate: + parser.error("-l --learningRate is a mandatory opt") + if not options.batchSize: + parser.error("-b --batchSize is a mandatory opt") + + # init + sparkConf = create_spark_conf().setAppName("inception v1") + sc = get_spark_context(sparkConf) + redire_spark_logs() + show_bigdl_info_logs() + init_engine() + + image_size = 224 # create dataset + train_transformer = ChainedPreprocessing([ImagePixelBytesToMat(), + ImageRandomCrop(image_size, image_size), + ImageRandomPreprocessing(ImageHFlip(), 0.5), + ImageChannelNormalize(0.485, 0.456, 0.406, 0.229, 0.224, 0.225), + ImageMatToTensor(to_RGB=True), + ImageSetToSample(input_keys=["imageTensor"], target_keys=["label"]) + ]) + raw_train_data = get_inception_data(options.folder, sc, "train") + train_data = FeatureSet.image_frame(raw_train_data).transform(train_transformer).transform(ImageFeatureToSample()) + + val_transformer = ChainedPreprocessing([ImagePixelBytesToMat(), + ImageCenterCrop(image_size, image_size), + ImageRandomPreprocessing(ImageHFlip(), 0.5), + ImageChannelNormalize(0.485, 0.456, 0.406, 0.229, 0.224, 0.225), + ImageMatToTensor(to_RGB=True), + ImageSetToSample(input_keys=["imageTensor"], target_keys=["label"]), + ]) + raw_val_data = get_inception_data(options.folder, sc, "val") + val_data = FeatureSet.image_frame(raw_val_data).transform(val_transformer).transform(ImageFeatureToSample()) + + # build model + bigdl_model = inception_v1_no_aux_classifier(options.classNum) + + model_path = "/tmp/bigdl_inception-v1_imagenet_0.4.0.model" + bigdl_model.saveModel(model_path, over_write = True) + inception_zoo_model = Net.load_bigdl(model_path) + inputNode = Input(name="input", shape=(3, 224, 224)) + outputLayer = inception_zoo_model.to_keras()(inputNode) + model = Model(inputNode, outputLayer) + + + # set optimization method + iterationPerEpoch = int(ceil(float(1281167) / options.batchSize)) + if options.maxEpoch: + maxIteration = iterationPerEpoch * options.maxEpoch + else: + maxIteration = options.maxIteration + warmup_iteration = options.warmupEpoch * iterationPerEpoch + if options.state != "": + # load state snapshot + optim = OptimMethod.load(options.state) + else: + if warmup_iteration == 0: + warmupDelta = 0.0 + else: + if options.maxLr: + maxlr = options.maxLr + else: + maxlr = options.learningRate + warmupDelta = (maxlr - options.learningRate)/warmup_iteration + polyIteration = maxIteration - warmup_iteration + lrSchedule = SequentialSchedule(iterationPerEpoch) + lrSchedule.add(Warmup(warmupDelta), warmup_iteration) + lrSchedule.add(Poly(0.5, polyIteration), polyIteration) + optim = SGD(learningrate=options.learningRate, learningrate_decay=0.0, weightdecay=options.weightDecay, + momentum=0.9, dampening=0.0, nesterov=False, + leaningrate_schedule=lrSchedule) + + # create triggers + if options.maxEpoch: + checkpoint_trigger = EveryEpoch() + test_trigger = EveryEpoch() + end_trigger = MaxEpoch(options.maxEpoch) + else: + checkpoint_trigger = SeveralIteration(options.checkpointIteration) + test_trigger = SeveralIteration(options.checkpointIteration) + end_trigger = MaxIteration(options.maxIteration) + + # Optimizer + model.compile(optimizer=optim, + loss='categorical_crossentropy', + metrics=['accuracy', 'top5acc']) + model.fit(x = train_data, batch_size=options.batchSize, nb_epoch=70, validation_data=val_data) + # optimizer = Optimizer.create( + # model=inception_model, + # training_set=train_data, + # optim_method=optim, + # criterion=ClassNLLCriterion(), + # end_trigger=end_trigger, + # batch_size=options.batchSize + # ) + + # if options.checkpoint: + # optimizer.set_checkpoint(checkpoint_trigger, options.checkpoint, options.overwrite) + # + # if options.gradientMin and options.gradientMax: + # optimizer.set_gradclip_const(options.gradientMin, options.gradientMax) + # + # if options.gradientL2NormThreshold: + # optimizer.set_gradclip_l2norm(options.gradientL2NormThreshold) + + # optimizer.set_validation(trigger=test_trigger, + # val_rdd=val_data, + # batch_size=options.batchSize, + # val_method=[Top1Accuracy(), Top5Accuracy()]) + + # trained_model = optimizer.optimize() + + sc.stop() diff --git a/pyzoo/zoo/feature/image/imagePreprocessing.py b/pyzoo/zoo/feature/image/imagePreprocessing.py index c0762546807..58013db62fa 100644 --- a/pyzoo/zoo/feature/image/imagePreprocessing.py +++ b/pyzoo/zoo/feature/image/imagePreprocessing.py @@ -40,6 +40,14 @@ class ImageBytesToMat(ImagePreprocessing): def __init__(self, byte_key="bytes", image_codec=-1, bigdl_type="float"): super(ImageBytesToMat, self).__init__(bigdl_type, byte_key, image_codec) +class ImagePixelBytesToMat(ImagePreprocessing): + """ + Transform byte array(pixels in byte) to OpenCVMat + :param byte_key key that maps byte array + """ + def __init__(self, byte_key="bytes", bigdl_type="float"): + super(ImagePixelBytesToMat, self).__init__(bigdl_type, byte_key) + class ImageResize(ImagePreprocessing): """ @@ -339,6 +347,14 @@ def __init__(self, bigdl_type="float"): super(ImageFeatureToTensor, self).__init__(bigdl_type) +class ImageFeatureToSample(Preprocessing): + """ + a Transformer that get Sample from ImageFeature. + """ + def __init__(self, bigdl_type="float"): + super(ImageFeatureToSample, self).__init__(bigdl_type) + + class RowToImageFeature(Preprocessing): """ a Transformer that converts a Spark Row to a BigDL ImageFeature. diff --git a/pyzoo/zoo/pipeline/api/keras/engine/topology.py b/pyzoo/zoo/pipeline/api/keras/engine/topology.py index 6a3512343ae..cf2d4f1e346 100644 --- a/pyzoo/zoo/pipeline/api/keras/engine/topology.py +++ b/pyzoo/zoo/pipeline/api/keras/engine/topology.py @@ -17,6 +17,7 @@ import zoo.pipeline.api.autograd as autograd from zoo.feature.image import ImageSet from zoo.feature.text import TextSet +from zoo.feature.common import FeatureSet from zoo.pipeline.api.keras.base import ZooKerasLayer from zoo.pipeline.api.keras.utils import * from bigdl.nn.layer import Layer @@ -212,7 +213,7 @@ def fit(self, x, y=None, batch_size=32, nb_epoch=10, validation_data = to_sample_rdd(*validation_data) training_data = to_sample_rdd(x, y) elif (isinstance(x, RDD) or isinstance(x, ImageSet) or isinstance(x, TextSet))\ - and not y: + or isinstance(x, FeatureSet) and not y: training_data = x else: raise TypeError("Unsupported training data type: %s" % type(x)) diff --git a/zoo/src/main/scala/com/intel/analytics/zoo/feature/python/PythonImageFeature.scala b/zoo/src/main/scala/com/intel/analytics/zoo/feature/python/PythonImageFeature.scala index 3ffb90f2b0d..aa1fb860a70 100644 --- a/zoo/src/main/scala/com/intel/analytics/zoo/feature/python/PythonImageFeature.scala +++ b/zoo/src/main/scala/com/intel/analytics/zoo/feature/python/PythonImageFeature.scala @@ -220,6 +220,11 @@ class PythonImageFeature[T: ClassTag](implicit ev: TensorNumeric[T]) extends Pyt ImageBytesToMat(byteKey, imageCodec) } + def createImagePixelBytesToMat( + byteKey: String = ImageFeature.bytes): ImagePixelBytesToMat = { + ImagePixelBytesToMat(byteKey) + } + def createImageBrightness(deltaLow: Double, deltaHigh: Double): ImageBrightness = { ImageBrightness(deltaLow, deltaHigh) } @@ -228,6 +233,10 @@ class PythonImageFeature[T: ClassTag](implicit ev: TensorNumeric[T]) extends Pyt ImageFeatureToTensor() } + def createImageFeatureToSample(): ImageFeatureToSample[T] = { + ImageFeatureToSample() + } + def createImageChannelNormalizer( meanR: Double, meanG: Double, meanB: Double, stdR: Double = 1, stdG: Double = 1, stdB: Double = 1 diff --git a/zoo/src/main/scala/com/intel/analytics/zoo/pipeline/api/keras/models/Topology.scala b/zoo/src/main/scala/com/intel/analytics/zoo/pipeline/api/keras/models/Topology.scala index 70fc80f9f20..fc687cfe8cf 100644 --- a/zoo/src/main/scala/com/intel/analytics/zoo/pipeline/api/keras/models/Topology.scala +++ b/zoo/src/main/scala/com/intel/analytics/zoo/pipeline/api/keras/models/Topology.scala @@ -30,11 +30,12 @@ import com.intel.analytics.bigdl.optim.DistriOptimizer.Cache import com.intel.analytics.bigdl.optim._ import com.intel.analytics.bigdl.serialization.Bigdl.BigDLModule import com.intel.analytics.bigdl.tensor.TensorNumericMath.TensorNumeric +import com.intel.analytics.bigdl.transform.vision.image.ImageFeature import com.intel.analytics.bigdl.utils._ import com.intel.analytics.bigdl.utils.serializer.{DeserializeContext, ModuleData, ModuleSerializer, SerializeContext} import com.intel.analytics.bigdl.visualization.{TrainSummary, ValidationSummary} import com.intel.analytics.zoo.feature.{DistributedFeatureSet, FeatureSet} -import com.intel.analytics.zoo.feature.image.ImageSet +import com.intel.analytics.zoo.feature.image.{ImageFeatureToSample, ImageSet} import com.intel.analytics.zoo.feature.text._ import com.intel.analytics.zoo.pipeline.api.{Net, Predictable} import com.intel.analytics.zoo.pipeline.api.autograd.{Lambda, Variable} @@ -326,6 +327,27 @@ abstract class KerasNet[T](implicit val tag: ClassTag[T], implicit val ev: Tenso else null } + private def toDataSet[D: ClassTag]( + x: FeatureSet[D], batchSize: Int): DataSet[MiniBatch[T]] = { + if (x != null) { + implicitly[ClassTag[D]].runtimeClass match { + case t if t == classOf[Sample[T]] => + (x.asInstanceOf[FeatureSet[Sample[T]]] + -> SampleToMiniBatch[T](batchSize) + ).asInstanceOf[DataSet[MiniBatch[T]]] + case t if t == classOf[ImageFeature] => + (x.asInstanceOf[FeatureSet[ImageFeature]] + -> ImageFeatureToSample[T]() + -> SampleToMiniBatch[T](batchSize) + ).asInstanceOf[DataSet[MiniBatch[T]]] + case _ => + throw new IllegalArgumentException( + s"${implicitly[ClassTag[T]].runtimeClass} is not supported for now") + } + } + else null + } + /** * Train a model for a fixed number of epochs on a DataSet. * @@ -451,6 +473,21 @@ abstract class KerasNet[T](implicit val tag: ClassTag[T], implicit val ev: Tenso this.fit(x, batchSize, nbEpoch, null) } + def fit( + x: FeatureSet[Sample[T]], + batchSize: Int, + nbEpoch: Int)(implicit ev: TensorNumeric[T]): Unit = { + this.fit(toDataSet(x, batchSize), nbEpoch, null) + } + + def fit[D: ClassTag]( + x: FeatureSet[D], + batchSize: Int, + nbEpoch: Int, + validationData: FeatureSet[D])(implicit ev: TensorNumeric[T]): Unit = { + this.fit(toDataSet(x, batchSize), nbEpoch, toDataSet(validationData, batchSize)) + } + /** * Train a model for a fixed number of epochs on TextSet. * diff --git a/zoo/src/main/scala/com/intel/analytics/zoo/pipeline/api/keras/python/PythonZooKeras.scala b/zoo/src/main/scala/com/intel/analytics/zoo/pipeline/api/keras/python/PythonZooKeras.scala index 956836e0293..767f117b16b 100644 --- a/zoo/src/main/scala/com/intel/analytics/zoo/pipeline/api/keras/python/PythonZooKeras.scala +++ b/zoo/src/main/scala/com/intel/analytics/zoo/pipeline/api/keras/python/PythonZooKeras.scala @@ -41,6 +41,7 @@ import com.intel.analytics.zoo.pipeline.api.keras.objectives._ import com.intel.analytics.zoo.pipeline.api.keras.optimizers.{Adam, AdamWeightDecay} import org.apache.spark.api.java.JavaRDD import com.intel.analytics.zoo.common.PythonZoo +import com.intel.analytics.zoo.feature.FeatureSet import com.intel.analytics.zoo.feature.text.TextSet import com.intel.analytics.zoo.models.common.ZooModel import com.intel.analytics.zoo.models.seq2seq.{Bridge, RNNDecoder, RNNEncoder} @@ -123,6 +124,15 @@ class PythonZooKeras[T: ClassTag](implicit ev: TensorNumeric[T]) extends PythonZ module.fit(x, batchSize, nbEpoch, validationData) } + def zooFit[D: ClassTag]( + module: KerasNet[T], + x: FeatureSet[D], + batchSize: Int, + nbEpoch: Int, + validationData: FeatureSet[D]): Unit = { + module.fit(x, batchSize, nbEpoch, validationData) + } + def zooFit( module: KerasNet[T], xTrain: JList[JTensor], From 0fcf1fc70ec914e237ade8647d3760ca0025a96b Mon Sep 17 00:00:00 2001 From: qiuxin2012 Date: Mon, 29 Apr 2019 16:15:38 +0800 Subject: [PATCH 02/28] update inception --- .../tensorflow/inception/bigdlinception.py | 364 -------- .../tensorflow/inception/inception.py | 780 ++++++++++++------ 2 files changed, 535 insertions(+), 609 deletions(-) delete mode 100644 pyzoo/zoo/examples/tensorflow/inception/bigdlinception.py diff --git a/pyzoo/zoo/examples/tensorflow/inception/bigdlinception.py b/pyzoo/zoo/examples/tensorflow/inception/bigdlinception.py deleted file mode 100644 index 6a7f56beb55..00000000000 --- a/pyzoo/zoo/examples/tensorflow/inception/bigdlinception.py +++ /dev/null @@ -1,364 +0,0 @@ -from bigdl.nn.layer import * -from optparse import OptionParser -from bigdl.nn.criterion import * -from bigdl.nn.initialization_method import * -from bigdl.optim.optimizer import * -from bigdl.transform.vision.image import * -from math import ceil -from zoo.feature.common import * -from zoo.feature.image.imagePreprocessing import * -from zoo.feature.common import * - - - -def t(input_t): - if type(input_t) is list: - # insert into index 0 spot, such that the real data starts from index 1 - temp = [0] - temp.extend(input_t) - return dict(enumerate(temp)) - # if dictionary, return it back - return input_t - - -def inception_layer_v1(input_size, config, name_prefix=""): - concat = Concat(2) - conv1 = Sequential() - conv1.add(SpatialConvolution(input_size, config[1][1], 1, 1, 1, 1) - .set_init_method(weight_init_method=Xavier(),bias_init_method=Zeros()) - .set_name(name_prefix + "1x1")) - conv1.add(ReLU(True).set_name(name_prefix + "relu_1x1")) - concat.add(conv1) - conv3 = Sequential() - conv3.add(SpatialConvolution(input_size, config[2][1], 1, 1, 1, 1) - .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) - .set_name(name_prefix + "3x3_reduce")) - conv3.add(ReLU(True).set_name(name_prefix + "relu_3x3_reduce")) - conv3.add(SpatialConvolution(config[2][1], config[2][2], 3, 3, 1, 1, 1, 1) - .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) - .set_name(name_prefix + "3x3")) - conv3.add(ReLU(True).set_name(name_prefix + "relu_3x3")) - concat.add(conv3) - conv5 = Sequential() - conv5.add(SpatialConvolution(input_size, config[3][1], 1, 1, 1, 1) - .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) - .set_name(name_prefix + "5x5_reduce")) - conv5.add(ReLU(True).set_name(name_prefix + "relu_5x5_reduce")) - conv5.add(SpatialConvolution(config[3][1], config[3][2], 5, 5, 1, 1, 2, 2) - .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) - .set_name(name_prefix + "5x5")) - conv5.add(ReLU(True).set_name(name_prefix + "relu_5x5")) - concat.add(conv5) - pool = Sequential() - pool.add(SpatialMaxPooling(3, 3, 1, 1, 1, 1, to_ceil=True).set_name(name_prefix + "pool")) - pool.add(SpatialConvolution(input_size, config[4][1], 1, 1, 1, 1) - .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) - .set_name(name_prefix + "pool_proj")) - pool.add(ReLU(True).set_name(name_prefix + "relu_pool_proj")) - concat.add(pool).set_name(name_prefix + "output") - return concat - - -def inception_v1_no_aux_classifier(class_num, has_dropout=True): - model = Sequential() - model.add(SpatialConvolution(3, 64, 7, 7, 2, 2, 3, 3, 1, False) - .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) - .set_name("conv1/7x7_s2")) - model.add(ReLU(True).set_name("conv1/relu_7x7")) - model.add(SpatialMaxPooling(3, 3, 2, 2, to_ceil=True).set_name("pool1/3x3_s2")) - model.add(SpatialCrossMapLRN(5, 0.0001, 0.75).set_name("pool1/norm1")) - model.add(SpatialConvolution(64, 64, 1, 1, 1, 1) - .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) - .set_name("conv2/3x3_reduce")) - model.add(ReLU(True).set_name("conv2/relu_3x3_reduce")) - model.add(SpatialConvolution(64, 192, 3, 3, 1, 1, 1, 1) - .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) - .set_name("conv2/3x3")) - model.add(ReLU(True).set_name("conv2/relu_3x3")) - model.add(SpatialCrossMapLRN(5, 0.0001, 0.75).set_name("conv2/norm2")) - model.add(SpatialMaxPooling(3, 3, 2, 2, to_ceil=True).set_name("pool2/3x3_s2")) - model.add(inception_layer_v1(192, t([t([64]), t( - [96, 128]), t([16, 32]), t([32])]), "inception_3a/")) - model.add(inception_layer_v1(256, t([t([128]), t( - [128, 192]), t([32, 96]), t([64])]), "inception_3b/")) - model.add(SpatialMaxPooling(3, 3, 2, 2, to_ceil=True)) - model.add(inception_layer_v1(480, t([t([192]), t( - [96, 208]), t([16, 48]), t([64])]), "inception_4a/")) - model.add(inception_layer_v1(512, t([t([160]), t( - [112, 224]), t([24, 64]), t([64])]), "inception_4b/")) - model.add(inception_layer_v1(512, t([t([128]), t( - [128, 256]), t([24, 64]), t([64])]), "inception_4c/")) - model.add(inception_layer_v1(512, t([t([112]), t( - [144, 288]), t([32, 64]), t([64])]), "inception_4d/")) - model.add(inception_layer_v1(528, t([t([256]), t( - [160, 320]), t([32, 128]), t([128])]), "inception_4e/")) - model.add(SpatialMaxPooling(3, 3, 2, 2, to_ceil=True)) - model.add(inception_layer_v1(832, t([t([256]), t( - [160, 320]), t([32, 128]), t([128])]), "inception_5a/")) - model.add(inception_layer_v1(832, t([t([384]), t( - [192, 384]), t([48, 128]), t([128])]), "inception_5b/")) - model.add(SpatialAveragePooling(7, 7, 1, 1).set_name("pool5/7x7_s1")) - if has_dropout: - model.add(Dropout(0.4).set_name("pool5/drop_7x7_s1")) - model.add(View([1024], num_input_dims=3)) - model.add(Linear(1024, class_num) - .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) - .set_name("loss3/classifier")) - model.add(LogSoftMax().set_name("loss3/loss3")) - model.reset() - return model - - -def inception_v1(class_num, has_dropout=True): - feature1 = Sequential() - feature1.add(SpatialConvolution(3, 64, 7, 7, 2, 2, 3, 3, 1, False) - .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) - .set_name("conv1/7x7_s2")) - feature1.add(ReLU(True).set_name("conv1/relu_7x7")) - feature1.add( - SpatialMaxPooling(3, 3, 2, 2, to_ceil=True) - .set_name("pool1/3x3_s2")) - feature1.add(SpatialCrossMapLRN(5, 0.0001, 0.75) - .set_name("pool1/norm1")) - feature1.add(SpatialConvolution(64, 64, 1, 1, 1, 1) - .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) - .set_name("conv2/3x3_reduce")) - feature1.add(ReLU(True).set_name("conv2/relu_3x3_reduce")) - feature1.add(SpatialConvolution(64, 192, 3, 3, 1, 1, 1, 1) - .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) - .set_name("conv2/3x3")) - feature1.add(ReLU(True).set_name("conv2/relu_3x3")) - feature1.add(SpatialCrossMapLRN(5, 0.0001, 0.75).set_name("conv2/norm2")) - feature1.add( - SpatialMaxPooling(3, 3, 2, 2, to_ceil=True).set_name("pool2/3x3_s2")) - feature1.add(inception_layer_v1(192, t([ - t([64]), t([96, 128]), t([16, 32]), t([32])]), - "inception_3a/")) - feature1.add(inception_layer_v1(256, t([ - t([128]), t([128, 192]), t([32, 96]), t([64])]), - "inception_3b/")) - feature1.add( - SpatialMaxPooling(3, 3, 2, 2, to_ceil=True).set_name("pool3/3x3_s2")) - feature1.add(inception_layer_v1(480, t([ - t([192]), t([96, 208]), t([16, 48]), t([64])]), - "inception_4a/")) - - output1 = Sequential() - output1.add(SpatialAveragePooling( - 5, 5, 3, 3, ceil_mode=True).set_name("loss1/ave_pool")) - output1.add( - SpatialConvolution(512, 128, 1, 1, 1, 1).set_name("loss1/conv")) - output1.add(ReLU(True).set_name("loss1/relu_conv")) - output1.add(View([128 * 4 * 4, 3])) - output1.add(Linear(128 * 4 * 4, 1024).set_name("loss1/fc")) - output1.add(ReLU(True).set_name("loss1/relu_fc")) - if has_dropout: - output1.add(Dropout(0.7).set_name("loss1/drop_fc")) - output1.add(Linear(1024, class_num).set_name("loss1/classifier")) - output1.add(LogSoftMax().set_name("loss1/loss")) - - feature2 = Sequential() - feature2.add(inception_layer_v1(512, - t([t([160]), t([112, 224]), t([24, 64]), t([64])]), - "inception_4b/")) - feature2.add(inception_layer_v1(512, - t([t([128]), t([128, 256]), t([24, 64]), t([64])]), - "inception_4c/")) - feature2.add(inception_layer_v1(512, - t([t([112]), t([144, 288]), t([32, 64]), t([64])]), - "inception_4d/")) - - output2 = Sequential() - output2.add(SpatialAveragePooling(5, 5, 3, 3).set_name("loss2/ave_pool")) - output2.add( - SpatialConvolution(528, 128, 1, 1, 1, 1).set_name("loss2/conv")) - output2.add(ReLU(True).set_name("loss2/relu_conv")) - output2.add(View([128 * 4 * 4, 3])) - output2.add(Linear(128 * 4 * 4, 1024).set_name("loss2/fc")) - output2.add(ReLU(True).set_name("loss2/relu_fc")) - if has_dropout: - output2.add(Dropout(0.7).set_name("loss2/drop_fc")) - output2.add(Linear(1024, class_num).set_name("loss2/classifier")) - output2.add(LogSoftMax().set_name("loss2/loss")) - - output3 = Sequential() - output3.add(inception_layer_v1(528, - t([t([256]), t([160, 320]), t([32, 128]), t([128])]), - "inception_4e/")) - output3.add(SpatialMaxPooling(3, 3, 2, 2, to_ceil=True).set_name("pool4/3x3_s2")) - output3.add(inception_layer_v1(832, - t([t([256]), t([160, 320]), t([32, 128]), t([128])]), - "inception_5a/")) - output3.add(inception_layer_v1(832, - t([t([384]), t([192, 384]), t([48, 128]), t([128])]), - "inception_5b/")) - output3.add(SpatialAveragePooling(7, 7, 1, 1).set_name("pool5/7x7_s1")) - if has_dropout: - output3.add(Dropout(0.4).set_name("pool5/drop_7x7_s1")) - output3.add(View([1024, 3])) - output3.add(Linear(1024, class_num) - .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) - .set_name("loss3/classifier")) - output3.add(LogSoftMax().set_name("loss3/loss3")) - - split2 = Concat(2).set_name("split2") - split2.add(output3) - split2.add(output2) - - mainBranch = Sequential() - mainBranch.add(feature2) - mainBranch.add(split2) - - split1 = Concat(2).set_name("split1") - split1.add(mainBranch) - split1.add(output1) - - model = Sequential() - - model.add(feature1) - model.add(split1) - - model.reset() - return model - - -def get_inception_data(url, sc=None, data_type="train"): - path = os.path.join(url, data_type) - return SeqFileFolder.files_to_image_frame(url=path, sc=sc, class_num=1000) - - -def config_option_parser(): - parser = OptionParser() - parser.add_option("-f", "--folder", type=str, dest="folder", default="", - help="url of hdfs folder store the hadoop sequence files") - parser.add_option("--model", type=str, dest="model", default="", help="model snapshot location") - parser.add_option("--state", type=str, dest="state", default="", help="state snapshot location") - parser.add_option("--checkpoint", type=str, dest="checkpoint", default="", help="where to cache the model") - parser.add_option("-o", "--overwrite", action="store_true", dest="overwrite", default=False, - help="overwrite checkpoint files") - parser.add_option("-e", "--maxEpoch", type=int, dest="maxEpoch", default=0, help="epoch numbers") - parser.add_option("-i", "--maxIteration", type=int, dest="maxIteration", default=62000, help="iteration numbers") - parser.add_option("-l", "--learningRate", type=float, dest="learningRate", default=0.01, help="learning rate") - parser.add_option("--warmupEpoch", type=int, dest="warmupEpoch", default=0, help="warm up epoch numbers") - parser.add_option("--maxLr", type=float, dest="maxLr", default=0.0, help="max Lr after warm up") - parser.add_option("-b", "--batchSize", type=int, dest="batchSize", help="batch size") - parser.add_option("--classNum", type=int, dest="classNum", default=1000, help="class number") - parser.add_option("--weightDecay", type=float, dest="weightDecay", default=0.0001, help="weight decay") - parser.add_option("--checkpointIteration", type=int, dest="checkpointIteration", default=620, - help="checkpoint interval of iterations") - parser.add_option("--gradientMin", type=float, dest="gradientMin", default=0.0, help="min gradient clipping by") - parser.add_option("--gradientMax", type=float, dest="gradientMax", default=0.0, help="max gradient clipping by") - parser.add_option("--gradientL2NormThreshold", type=float, dest="gradientL2NormThreshold", default=0.0, help="gradient L2-Norm threshold") - - return parser - - -if __name__ == "__main__": - # parse options - parser = config_option_parser() - (options, args) = parser.parse_args(sys.argv) - if not options.learningRate: - parser.error("-l --learningRate is a mandatory opt") - if not options.batchSize: - parser.error("-b --batchSize is a mandatory opt") - - # init - sparkConf = create_spark_conf().setAppName("inception v1") - sc = get_spark_context(sparkConf) - redire_spark_logs() - show_bigdl_info_logs() - init_engine() - - image_size = 224 # create dataset - train_transformer = ChainedPreprocessing([ImagePixelBytesToMat(), - ImageRandomCrop(image_size, image_size), - ImageRandomPreprocessing(ImageHFlip(), 0.5), - ImageChannelNormalize(0.485, 0.456, 0.406, 0.229, 0.224, 0.225), - ImageMatToTensor(to_RGB=True), - ImageSetToSample(input_keys=["imageTensor"], target_keys=["label"]) - ]) - raw_train_data = get_inception_data(options.folder, sc, "train") - train_data = FeatureSet.image_frame(raw_train_data).transform(train_transformer).to_dataset() - - val_transformer = ChainedPreprocessing([ImagePixelBytesToMat(), - ImageCenterCrop(image_size, image_size), - ImageRandomPreprocessing(ImageHFlip(), 0.5), - ImageChannelNormalize(0.485, 0.456, 0.406, 0.229, 0.224, 0.225), - ImageMatToTensor(to_RGB=True), - ImageSetToSample(input_keys=["imageTensor"], target_keys=["label"]), - ]) - raw_val_data = get_inception_data(options.folder, sc, "val") - val_data = FeatureSet.image_frame(raw_val_data).transform(val_transformer).to_dataset() - - - # build model - if options.model != "": - # load model snapshot - inception_model = Model.load(options.model) - else: - inception_model = inception_v1_no_aux_classifier(options.classNum) - - # set optimization method - iterationPerEpoch = int(ceil(float(1281167) / options.batchSize)) - if options.maxEpoch: - maxIteration = iterationPerEpoch * options.maxEpoch - else: - maxIteration = options.maxIteration - warmup_iteration = options.warmupEpoch * iterationPerEpoch - if options.state != "": - # load state snapshot - optim = OptimMethod.load(options.state) - else: - if warmup_iteration == 0: - warmupDelta = 0.0 - else: - if options.maxLr: - maxlr = options.maxLr - else: - maxlr = options.learningRate - warmupDelta = (maxlr - options.learningRate)/warmup_iteration - polyIteration = maxIteration - warmup_iteration - lrSchedule = SequentialSchedule(iterationPerEpoch) - lrSchedule.add(Warmup(warmupDelta), warmup_iteration) - lrSchedule.add(Poly(0.5, polyIteration), polyIteration) - optim = SGD(learningrate=options.learningRate, learningrate_decay=0.0, weightdecay=options.weightDecay, - momentum=0.9, dampening=0.0, nesterov=False, - leaningrate_schedule=lrSchedule) - - # create triggers - if options.maxEpoch: - checkpoint_trigger = EveryEpoch() - test_trigger = EveryEpoch() - end_trigger = MaxEpoch(options.maxEpoch) - else: - checkpoint_trigger = SeveralIteration(options.checkpointIteration) - test_trigger = SeveralIteration(options.checkpointIteration) - end_trigger = MaxIteration(options.maxIteration) - - # Optimizer - optimizer = Optimizer.create( - model=inception_model, - training_set=train_data, - optim_method=optim, - criterion=ClassNLLCriterion(), - end_trigger=end_trigger, - batch_size=options.batchSize - ) - - if options.checkpoint: - optimizer.set_checkpoint(checkpoint_trigger, options.checkpoint, options.overwrite) - - if options.gradientMin and options.gradientMax: - optimizer.set_gradclip_const(options.gradientMin, options.gradientMax) - - if options.gradientL2NormThreshold: - optimizer.set_gradclip_l2norm(options.gradientL2NormThreshold) - - optimizer.set_validation(trigger=test_trigger, - val_rdd=val_data, - batch_size=options.batchSize, - val_method=[Top1Accuracy(), Top5Accuracy()]) - - trained_model = optimizer.optimize() - - sc.stop() diff --git a/pyzoo/zoo/examples/tensorflow/inception/inception.py b/pyzoo/zoo/examples/tensorflow/inception/inception.py index 162be5dcea6..16c3b43ffb2 100644 --- a/pyzoo/zoo/examples/tensorflow/inception/inception.py +++ b/pyzoo/zoo/examples/tensorflow/inception/inception.py @@ -1,23 +1,402 @@ -from bigdl.nn.layer import Sequential, Concat, SpatialConvolution, ReLU, SpatialMaxPooling, SpatialCrossMapLRN, SpatialAveragePooling, Dropout, View, Linear, LogSoftMax +#from bigdl.nn.layer import Sequential, Concat, SpatialConvolution, ReLU, SpatialMaxPooling, SpatialCrossMapLRN, SpatialAveragePooling, Dropout, View, Linear, LogSoftMax +#from optparse import OptionParser +#from bigdl.nn.criterion import * +#from bigdl.nn.initialization_method import * +#from bigdl.optim.optimizer import * +#from zoo.feature.image.imagePreprocessing import * +#from zoo.feature.image.imageset import * +#from zoo.feature.common import ChainedPreprocessing +#from math import ceil +#from zoo.common.nncontext import * +#from zoo.feature.common import * +#from zoo.feature.image.imagePreprocessing import * +# +#from zoo.common.nncontext import * +#from zoo.feature.common import * +#from zoo.feature.image.imagePreprocessing import * +#from zoo.pipeline.api.keras.layers import Dense, Input, Flatten +#from zoo.pipeline.api.net import * +#from zoo.pipeline.api.keras.models import Model +#from bigdl.optim.optimizer import * +# +# +#def t(input_t): +# if type(input_t) is list: +# # insert into index 0 spot, such that the real data starts from index 1 +# temp = [0] +# temp.extend(input_t) +# return dict(enumerate(temp)) +# # if dictionary, return it back +# return input_t +# +# +#def inception_layer_v1(input_size, config, name_prefix=""): +# concat = Concat(2) +# conv1 = Sequential() +# conv1.add(SpatialConvolution(input_size, config[1][1], 1, 1, 1, 1) +# .set_init_method(weight_init_method=Xavier(),bias_init_method=Zeros()) +# .set_name(name_prefix + "1x1")) +# conv1.add(ReLU(True).set_name(name_prefix + "relu_1x1")) +# concat.add(conv1) +# conv3 = Sequential() +# conv3.add(SpatialConvolution(input_size, config[2][1], 1, 1, 1, 1) +# .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) +# .set_name(name_prefix + "3x3_reduce")) +# conv3.add(ReLU(True).set_name(name_prefix + "relu_3x3_reduce")) +# conv3.add(SpatialConvolution(config[2][1], config[2][2], 3, 3, 1, 1, 1, 1) +# .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) +# .set_name(name_prefix + "3x3")) +# conv3.add(ReLU(True).set_name(name_prefix + "relu_3x3")) +# concat.add(conv3) +# conv5 = Sequential() +# conv5.add(SpatialConvolution(input_size, config[3][1], 1, 1, 1, 1) +# .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) +# .set_name(name_prefix + "5x5_reduce")) +# conv5.add(ReLU(True).set_name(name_prefix + "relu_5x5_reduce")) +# conv5.add(SpatialConvolution(config[3][1], config[3][2], 5, 5, 1, 1, 2, 2) +# .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) +# .set_name(name_prefix + "5x5")) +# conv5.add(ReLU(True).set_name(name_prefix + "relu_5x5")) +# concat.add(conv5) +# pool = Sequential() +# pool.add(SpatialMaxPooling(3, 3, 1, 1, 1, 1, to_ceil=True).set_name(name_prefix + "pool")) +# pool.add(SpatialConvolution(input_size, config[4][1], 1, 1, 1, 1) +# .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) +# .set_name(name_prefix + "pool_proj")) +# pool.add(ReLU(True).set_name(name_prefix + "relu_pool_proj")) +# concat.add(pool).set_name(name_prefix + "output") +# return concat +# +# +#def inception_v1_no_aux_classifier(class_num, has_dropout=True): +# model = Sequential() +# model.add(SpatialConvolution(3, 64, 7, 7, 2, 2, 3, 3, 1, False) +# .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) +# .set_name("conv1/7x7_s2")) +# model.add(ReLU(True).set_name("conv1/relu_7x7")) +# model.add(SpatialMaxPooling(3, 3, 2, 2, to_ceil=True).set_name("pool1/3x3_s2")) +# model.add(SpatialCrossMapLRN(5, 0.0001, 0.75).set_name("pool1/norm1")) +# model.add(SpatialConvolution(64, 64, 1, 1, 1, 1) +# .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) +# .set_name("conv2/3x3_reduce")) +# model.add(ReLU(True).set_name("conv2/relu_3x3_reduce")) +# model.add(SpatialConvolution(64, 192, 3, 3, 1, 1, 1, 1) +# .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) +# .set_name("conv2/3x3")) +# model.add(ReLU(True).set_name("conv2/relu_3x3")) +# model.add(SpatialCrossMapLRN(5, 0.0001, 0.75).set_name("conv2/norm2")) +# model.add(SpatialMaxPooling(3, 3, 2, 2, to_ceil=True).set_name("pool2/3x3_s2")) +# model.add(inception_layer_v1(192, t([t([64]), t( +# [96, 128]), t([16, 32]), t([32])]), "inception_3a/")) +# model.add(inception_layer_v1(256, t([t([128]), t( +# [128, 192]), t([32, 96]), t([64])]), "inception_3b/")) +# model.add(SpatialMaxPooling(3, 3, 2, 2, to_ceil=True)) +# model.add(inception_layer_v1(480, t([t([192]), t( +# [96, 208]), t([16, 48]), t([64])]), "inception_4a/")) +# model.add(inception_layer_v1(512, t([t([160]), t( +# [112, 224]), t([24, 64]), t([64])]), "inception_4b/")) +# model.add(inception_layer_v1(512, t([t([128]), t( +# [128, 256]), t([24, 64]), t([64])]), "inception_4c/")) +# model.add(inception_layer_v1(512, t([t([112]), t( +# [144, 288]), t([32, 64]), t([64])]), "inception_4d/")) +# model.add(inception_layer_v1(528, t([t([256]), t( +# [160, 320]), t([32, 128]), t([128])]), "inception_4e/")) +# model.add(SpatialMaxPooling(3, 3, 2, 2, to_ceil=True)) +# model.add(inception_layer_v1(832, t([t([256]), t( +# [160, 320]), t([32, 128]), t([128])]), "inception_5a/")) +# model.add(inception_layer_v1(832, t([t([384]), t( +# [192, 384]), t([48, 128]), t([128])]), "inception_5b/")) +# model.add(SpatialAveragePooling(7, 7, 1, 1).set_name("pool5/7x7_s1")) +# if has_dropout: +# model.add(Dropout(0.4).set_name("pool5/drop_7x7_s1")) +# model.add(View([1024], num_input_dims=3)) +# model.add(Linear(1024, class_num) +# .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) +# .set_name("loss3/classifier")) +# model.add(LogSoftMax().set_name("loss3/loss3")) +# model.reset() +# return model +# +# +#def inception_v1(class_num, has_dropout=True): +# feature1 = Sequential() +# feature1.add(SpatialConvolution(3, 64, 7, 7, 2, 2, 3, 3, 1, False) +# .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) +# .set_name("conv1/7x7_s2")) +# feature1.add(ReLU(True).set_name("conv1/relu_7x7")) +# feature1.add( +# SpatialMaxPooling(3, 3, 2, 2, to_ceil=True) +# .set_name("pool1/3x3_s2")) +# feature1.add(SpatialCrossMapLRN(5, 0.0001, 0.75) +# .set_name("pool1/norm1")) +# feature1.add(SpatialConvolution(64, 64, 1, 1, 1, 1) +# .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) +# .set_name("conv2/3x3_reduce")) +# feature1.add(ReLU(True).set_name("conv2/relu_3x3_reduce")) +# feature1.add(SpatialConvolution(64, 192, 3, 3, 1, 1, 1, 1) +# .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) +# .set_name("conv2/3x3")) +# feature1.add(ReLU(True).set_name("conv2/relu_3x3")) +# feature1.add(SpatialCrossMapLRN(5, 0.0001, 0.75).set_name("conv2/norm2")) +# feature1.add( +# SpatialMaxPooling(3, 3, 2, 2, to_ceil=True).set_name("pool2/3x3_s2")) +# feature1.add(inception_layer_v1(192, t([ +# t([64]), t([96, 128]), t([16, 32]), t([32])]), +# "inception_3a/")) +# feature1.add(inception_layer_v1(256, t([ +# t([128]), t([128, 192]), t([32, 96]), t([64])]), +# "inception_3b/")) +# feature1.add( +# SpatialMaxPooling(3, 3, 2, 2, to_ceil=True).set_name("pool3/3x3_s2")) +# feature1.add(inception_layer_v1(480, t([ +# t([192]), t([96, 208]), t([16, 48]), t([64])]), +# "inception_4a/")) +# +# output1 = Sequential() +# output1.add(SpatialAveragePooling( +# 5, 5, 3, 3, ceil_mode=True).set_name("loss1/ave_pool")) +# output1.add( +# SpatialConvolution(512, 128, 1, 1, 1, 1).set_name("loss1/conv")) +# output1.add(ReLU(True).set_name("loss1/relu_conv")) +# output1.add(View([128 * 4 * 4, 3])) +# output1.add(Linear(128 * 4 * 4, 1024).set_name("loss1/fc")) +# output1.add(ReLU(True).set_name("loss1/relu_fc")) +# if has_dropout: +# output1.add(Dropout(0.7).set_name("loss1/drop_fc")) +# output1.add(Linear(1024, class_num).set_name("loss1/classifier")) +# output1.add(LogSoftMax().set_name("loss1/loss")) +# +# feature2 = Sequential() +# feature2.add(inception_layer_v1(512, +# t([t([160]), t([112, 224]), t([24, 64]), t([64])]), +# "inception_4b/")) +# feature2.add(inception_layer_v1(512, +# t([t([128]), t([128, 256]), t([24, 64]), t([64])]), +# "inception_4c/")) +# feature2.add(inception_layer_v1(512, +# t([t([112]), t([144, 288]), t([32, 64]), t([64])]), +# "inception_4d/")) +# +# output2 = Sequential() +# output2.add(SpatialAveragePooling(5, 5, 3, 3).set_name("loss2/ave_pool")) +# output2.add( +# SpatialConvolution(528, 128, 1, 1, 1, 1).set_name("loss2/conv")) +# output2.add(ReLU(True).set_name("loss2/relu_conv")) +# output2.add(View([128 * 4 * 4, 3])) +# output2.add(Linear(128 * 4 * 4, 1024).set_name("loss2/fc")) +# output2.add(ReLU(True).set_name("loss2/relu_fc")) +# if has_dropout: +# output2.add(Dropout(0.7).set_name("loss2/drop_fc")) +# output2.add(Linear(1024, class_num).set_name("loss2/classifier")) +# output2.add(LogSoftMax().set_name("loss2/loss")) +# +# output3 = Sequential() +# output3.add(inception_layer_v1(528, +# t([t([256]), t([160, 320]), t([32, 128]), t([128])]), +# "inception_4e/")) +# output3.add(SpatialMaxPooling(3, 3, 2, 2, to_ceil=True).set_name("pool4/3x3_s2")) +# output3.add(inception_layer_v1(832, +# t([t([256]), t([160, 320]), t([32, 128]), t([128])]), +# "inception_5a/")) +# output3.add(inception_layer_v1(832, +# t([t([384]), t([192, 384]), t([48, 128]), t([128])]), +# "inception_5b/")) +# output3.add(SpatialAveragePooling(7, 7, 1, 1).set_name("pool5/7x7_s1")) +# if has_dropout: +# output3.add(Dropout(0.4).set_name("pool5/drop_7x7_s1")) +# output3.add(View([1024, 3])) +# output3.add(Linear(1024, class_num) +# .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) +# .set_name("loss3/classifier")) +# output3.add(LogSoftMax().set_name("loss3/loss3")) +# +# split2 = Concat(2).set_name("split2") +# split2.add(output3) +# split2.add(output2) +# +# mainBranch = Sequential() +# mainBranch.add(feature2) +# mainBranch.add(split2) +# +# split1 = Concat(2).set_name("split1") +# split1.add(mainBranch) +# split1.add(output1) +# +# model = Sequential() +# +# model.add(feature1) +# model.add(split1) +# +# model.reset() +# return model +# +# +#def get_inception_data(url, sc=None, data_type="train"): +# path = os.path.join(url, data_type) +# return SeqFileFolder.files_to_image_frame(url=path, sc=sc, class_num=1000) +# +# +#def config_option_parser(): +# parser = OptionParser() +# parser.add_option("-f", "--folder", type=str, dest="folder", default="", +# help="url of hdfs folder store the hadoop sequence files") +# parser.add_option("--model", type=str, dest="model", default="", help="model snapshot location") +# parser.add_option("--state", type=str, dest="state", default="", help="state snapshot location") +# parser.add_option("--checkpoint", type=str, dest="checkpoint", default="", help="where to cache the model") +# parser.add_option("-o", "--overwrite", action="store_true", dest="overwrite", default=False, +# help="overwrite checkpoint files") +# parser.add_option("-e", "--maxEpoch", type=int, dest="maxEpoch", default=0, help="epoch numbers") +# parser.add_option("-i", "--maxIteration", type=int, dest="maxIteration", default=62000, help="iteration numbers") +# parser.add_option("-l", "--learningRate", type=float, dest="learningRate", default=0.01, help="learning rate") +# parser.add_option("--warmupEpoch", type=int, dest="warmupEpoch", default=0, help="warm up epoch numbers") +# parser.add_option("--maxLr", type=float, dest="maxLr", default=0.0, help="max Lr after warm up") +# parser.add_option("-b", "--batchSize", type=int, dest="batchSize", help="batch size") +# parser.add_option("--classNum", type=int, dest="classNum", default=1000, help="class number") +# parser.add_option("--weightDecay", type=float, dest="weightDecay", default=0.0001, help="weight decay") +# parser.add_option("--checkpointIteration", type=int, dest="checkpointIteration", default=620, +# help="checkpoint interval of iterations") +# parser.add_option("--gradientMin", type=float, dest="gradientMin", default=0.0, help="min gradient clipping by") +# parser.add_option("--gradientMax", type=float, dest="gradientMax", default=0.0, help="max gradient clipping by") +# parser.add_option("--gradientL2NormThreshold", type=float, dest="gradientL2NormThreshold", default=0.0, help="gradient L2-Norm threshold") +# +# return parser +# +# +#if __name__ == "__main__": +# # parse options +# parser = config_option_parser() +# (options, args) = parser.parse_args(sys.argv) +# if not options.learningRate: +# parser.error("-l --learningRate is a mandatory opt") +# if not options.batchSize: +# parser.error("-b --batchSize is a mandatory opt") +# +# # init +# sparkConf = create_spark_conf().setAppName("inception v1") +# sc = get_spark_context(sparkConf) +# redire_spark_logs() +# show_bigdl_info_logs() +# init_engine() +# +# image_size = 224 # create dataset +# train_transformer = ChainedPreprocessing([ImagePixelBytesToMat(), +# ImageRandomCrop(image_size, image_size), +# ImageRandomPreprocessing(ImageHFlip(), 0.5), +# ImageChannelNormalize(0.485, 0.456, 0.406, 0.229, 0.224, 0.225), +# ImageMatToTensor(to_RGB=True), +# ImageSetToSample(input_keys=["imageTensor"], target_keys=["label"]) +# ]) +# raw_train_data = get_inception_data(options.folder, sc, "train") +# train_data = FeatureSet.image_frame(raw_train_data).transform(train_transformer).transform(ImageFeatureToSample()) +# +# val_transformer = ChainedPreprocessing([ImagePixelBytesToMat(), +# ImageCenterCrop(image_size, image_size), +# ImageRandomPreprocessing(ImageHFlip(), 0.5), +# ImageChannelNormalize(0.485, 0.456, 0.406, 0.229, 0.224, 0.225), +# ImageMatToTensor(to_RGB=True), +# ImageSetToSample(input_keys=["imageTensor"], target_keys=["label"]), +# ]) +# raw_val_data = get_inception_data(options.folder, sc, "val") +# val_data = FeatureSet.image_frame(raw_val_data).transform(val_transformer).transform(ImageFeatureToSample()) +# +# # build model +# bigdl_model = inception_v1_no_aux_classifier(options.classNum) +# +# model_path = "/tmp/bigdl_inception-v1_imagenet_0.4.0.model" +# bigdl_model.saveModel(model_path, over_write = True) +# inception_zoo_model = Net.load_bigdl(model_path) +# inputNode = Input(name="input", shape=(3, 224, 224)) +# outputLayer = inception_zoo_model.to_keras()(inputNode) +# model = Model(inputNode, outputLayer) +# +# +# # set optimization method +# iterationPerEpoch = int(ceil(float(1281167) / options.batchSize)) +# if options.maxEpoch: +# maxIteration = iterationPerEpoch * options.maxEpoch +# else: +# maxIteration = options.maxIteration +# warmup_iteration = options.warmupEpoch * iterationPerEpoch +# if options.state != "": +# # load state snapshot +# optim = OptimMethod.load(options.state) +# else: +# if warmup_iteration == 0: +# warmupDelta = 0.0 +# else: +# if options.maxLr: +# maxlr = options.maxLr +# else: +# maxlr = options.learningRate +# warmupDelta = (maxlr - options.learningRate)/warmup_iteration +# polyIteration = maxIteration - warmup_iteration +# lrSchedule = SequentialSchedule(iterationPerEpoch) +# lrSchedule.add(Warmup(warmupDelta), warmup_iteration) +# lrSchedule.add(Poly(0.5, polyIteration), polyIteration) +# optim = SGD(learningrate=options.learningRate, learningrate_decay=0.0, weightdecay=options.weightDecay, +# momentum=0.9, dampening=0.0, nesterov=False, +# leaningrate_schedule=lrSchedule) +# +# # create triggers +# if options.maxEpoch: +# checkpoint_trigger = EveryEpoch() +# test_trigger = EveryEpoch() +# end_trigger = MaxEpoch(options.maxEpoch) +# else: +# checkpoint_trigger = SeveralIteration(options.checkpointIteration) +# test_trigger = SeveralIteration(options.checkpointIteration) +# end_trigger = MaxIteration(options.maxIteration) +# +# # Optimizer +# model.compile(optimizer=optim, +# loss='categorical_crossentropy', +# metrics=['accuracy', 'top5acc']) +# model.fit(x = train_data, batch_size=options.batchSize, nb_epoch=70, validation_data=val_data) +# # optimizer = Optimizer.create( +# # model=inception_model, +# # training_set=train_data, +# # optim_method=optim, +# # criterion=ClassNLLCriterion(), +# # end_trigger=end_trigger, +# # batch_size=options.batchSize +# # ) +# +# # if options.checkpoint: +# # optimizer.set_checkpoint(checkpoint_trigger, options.checkpoint, options.overwrite) +# # +# # if options.gradientMin and options.gradientMax: +# # optimizer.set_gradclip_const(options.gradientMin, options.gradientMax) +# # +# # if options.gradientL2NormThreshold: +# # optimizer.set_gradclip_l2norm(options.gradientL2NormThreshold) +# +# # optimizer.set_validation(trigger=test_trigger, +# # val_rdd=val_data, +# # batch_size=options.batchSize, +# # val_method=[Top1Accuracy(), Top5Accuracy()]) +# +# # trained_model = optimizer.optimize() +# +# sc.stop() + from optparse import OptionParser -from bigdl.nn.criterion import * -from bigdl.nn.initialization_method import * -from bigdl.optim.optimizer import * -from zoo.feature.image.imagePreprocessing import * -from zoo.feature.image.imageset import * -from zoo.feature.common import ChainedPreprocessing from math import ceil -from zoo.common.nncontext import * -from zoo.feature.common import * -from zoo.feature.image.imagePreprocessing import * from zoo.common.nncontext import * -from zoo.feature.common import * -from zoo.feature.image.imagePreprocessing import * -from zoo.pipeline.api.keras.layers import Dense, Input, Flatten +from zoo.feature.image import * from zoo.pipeline.api.net import * -from zoo.pipeline.api.keras.models import Model +from zoo.pipeline.nnframes import * from bigdl.optim.optimizer import * +from bigdl.util.common import * + +# import the inception model +import tensorflow as tf + +#sys.path.append("/root/manfei/models/research/slim/") # add the slim library +sys.path.append("/home/xin/IdeaProjects/models/research/slim") # add the slim library +from nets import inception_v1 + +slim = tf.contrib.slim def t(input_t): @@ -30,207 +409,6 @@ def t(input_t): return input_t -def inception_layer_v1(input_size, config, name_prefix=""): - concat = Concat(2) - conv1 = Sequential() - conv1.add(SpatialConvolution(input_size, config[1][1], 1, 1, 1, 1) - .set_init_method(weight_init_method=Xavier(),bias_init_method=Zeros()) - .set_name(name_prefix + "1x1")) - conv1.add(ReLU(True).set_name(name_prefix + "relu_1x1")) - concat.add(conv1) - conv3 = Sequential() - conv3.add(SpatialConvolution(input_size, config[2][1], 1, 1, 1, 1) - .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) - .set_name(name_prefix + "3x3_reduce")) - conv3.add(ReLU(True).set_name(name_prefix + "relu_3x3_reduce")) - conv3.add(SpatialConvolution(config[2][1], config[2][2], 3, 3, 1, 1, 1, 1) - .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) - .set_name(name_prefix + "3x3")) - conv3.add(ReLU(True).set_name(name_prefix + "relu_3x3")) - concat.add(conv3) - conv5 = Sequential() - conv5.add(SpatialConvolution(input_size, config[3][1], 1, 1, 1, 1) - .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) - .set_name(name_prefix + "5x5_reduce")) - conv5.add(ReLU(True).set_name(name_prefix + "relu_5x5_reduce")) - conv5.add(SpatialConvolution(config[3][1], config[3][2], 5, 5, 1, 1, 2, 2) - .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) - .set_name(name_prefix + "5x5")) - conv5.add(ReLU(True).set_name(name_prefix + "relu_5x5")) - concat.add(conv5) - pool = Sequential() - pool.add(SpatialMaxPooling(3, 3, 1, 1, 1, 1, to_ceil=True).set_name(name_prefix + "pool")) - pool.add(SpatialConvolution(input_size, config[4][1], 1, 1, 1, 1) - .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) - .set_name(name_prefix + "pool_proj")) - pool.add(ReLU(True).set_name(name_prefix + "relu_pool_proj")) - concat.add(pool).set_name(name_prefix + "output") - return concat - - -def inception_v1_no_aux_classifier(class_num, has_dropout=True): - model = Sequential() - model.add(SpatialConvolution(3, 64, 7, 7, 2, 2, 3, 3, 1, False) - .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) - .set_name("conv1/7x7_s2")) - model.add(ReLU(True).set_name("conv1/relu_7x7")) - model.add(SpatialMaxPooling(3, 3, 2, 2, to_ceil=True).set_name("pool1/3x3_s2")) - model.add(SpatialCrossMapLRN(5, 0.0001, 0.75).set_name("pool1/norm1")) - model.add(SpatialConvolution(64, 64, 1, 1, 1, 1) - .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) - .set_name("conv2/3x3_reduce")) - model.add(ReLU(True).set_name("conv2/relu_3x3_reduce")) - model.add(SpatialConvolution(64, 192, 3, 3, 1, 1, 1, 1) - .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) - .set_name("conv2/3x3")) - model.add(ReLU(True).set_name("conv2/relu_3x3")) - model.add(SpatialCrossMapLRN(5, 0.0001, 0.75).set_name("conv2/norm2")) - model.add(SpatialMaxPooling(3, 3, 2, 2, to_ceil=True).set_name("pool2/3x3_s2")) - model.add(inception_layer_v1(192, t([t([64]), t( - [96, 128]), t([16, 32]), t([32])]), "inception_3a/")) - model.add(inception_layer_v1(256, t([t([128]), t( - [128, 192]), t([32, 96]), t([64])]), "inception_3b/")) - model.add(SpatialMaxPooling(3, 3, 2, 2, to_ceil=True)) - model.add(inception_layer_v1(480, t([t([192]), t( - [96, 208]), t([16, 48]), t([64])]), "inception_4a/")) - model.add(inception_layer_v1(512, t([t([160]), t( - [112, 224]), t([24, 64]), t([64])]), "inception_4b/")) - model.add(inception_layer_v1(512, t([t([128]), t( - [128, 256]), t([24, 64]), t([64])]), "inception_4c/")) - model.add(inception_layer_v1(512, t([t([112]), t( - [144, 288]), t([32, 64]), t([64])]), "inception_4d/")) - model.add(inception_layer_v1(528, t([t([256]), t( - [160, 320]), t([32, 128]), t([128])]), "inception_4e/")) - model.add(SpatialMaxPooling(3, 3, 2, 2, to_ceil=True)) - model.add(inception_layer_v1(832, t([t([256]), t( - [160, 320]), t([32, 128]), t([128])]), "inception_5a/")) - model.add(inception_layer_v1(832, t([t([384]), t( - [192, 384]), t([48, 128]), t([128])]), "inception_5b/")) - model.add(SpatialAveragePooling(7, 7, 1, 1).set_name("pool5/7x7_s1")) - if has_dropout: - model.add(Dropout(0.4).set_name("pool5/drop_7x7_s1")) - model.add(View([1024], num_input_dims=3)) - model.add(Linear(1024, class_num) - .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) - .set_name("loss3/classifier")) - model.add(LogSoftMax().set_name("loss3/loss3")) - model.reset() - return model - - -def inception_v1(class_num, has_dropout=True): - feature1 = Sequential() - feature1.add(SpatialConvolution(3, 64, 7, 7, 2, 2, 3, 3, 1, False) - .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) - .set_name("conv1/7x7_s2")) - feature1.add(ReLU(True).set_name("conv1/relu_7x7")) - feature1.add( - SpatialMaxPooling(3, 3, 2, 2, to_ceil=True) - .set_name("pool1/3x3_s2")) - feature1.add(SpatialCrossMapLRN(5, 0.0001, 0.75) - .set_name("pool1/norm1")) - feature1.add(SpatialConvolution(64, 64, 1, 1, 1, 1) - .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) - .set_name("conv2/3x3_reduce")) - feature1.add(ReLU(True).set_name("conv2/relu_3x3_reduce")) - feature1.add(SpatialConvolution(64, 192, 3, 3, 1, 1, 1, 1) - .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) - .set_name("conv2/3x3")) - feature1.add(ReLU(True).set_name("conv2/relu_3x3")) - feature1.add(SpatialCrossMapLRN(5, 0.0001, 0.75).set_name("conv2/norm2")) - feature1.add( - SpatialMaxPooling(3, 3, 2, 2, to_ceil=True).set_name("pool2/3x3_s2")) - feature1.add(inception_layer_v1(192, t([ - t([64]), t([96, 128]), t([16, 32]), t([32])]), - "inception_3a/")) - feature1.add(inception_layer_v1(256, t([ - t([128]), t([128, 192]), t([32, 96]), t([64])]), - "inception_3b/")) - feature1.add( - SpatialMaxPooling(3, 3, 2, 2, to_ceil=True).set_name("pool3/3x3_s2")) - feature1.add(inception_layer_v1(480, t([ - t([192]), t([96, 208]), t([16, 48]), t([64])]), - "inception_4a/")) - - output1 = Sequential() - output1.add(SpatialAveragePooling( - 5, 5, 3, 3, ceil_mode=True).set_name("loss1/ave_pool")) - output1.add( - SpatialConvolution(512, 128, 1, 1, 1, 1).set_name("loss1/conv")) - output1.add(ReLU(True).set_name("loss1/relu_conv")) - output1.add(View([128 * 4 * 4, 3])) - output1.add(Linear(128 * 4 * 4, 1024).set_name("loss1/fc")) - output1.add(ReLU(True).set_name("loss1/relu_fc")) - if has_dropout: - output1.add(Dropout(0.7).set_name("loss1/drop_fc")) - output1.add(Linear(1024, class_num).set_name("loss1/classifier")) - output1.add(LogSoftMax().set_name("loss1/loss")) - - feature2 = Sequential() - feature2.add(inception_layer_v1(512, - t([t([160]), t([112, 224]), t([24, 64]), t([64])]), - "inception_4b/")) - feature2.add(inception_layer_v1(512, - t([t([128]), t([128, 256]), t([24, 64]), t([64])]), - "inception_4c/")) - feature2.add(inception_layer_v1(512, - t([t([112]), t([144, 288]), t([32, 64]), t([64])]), - "inception_4d/")) - - output2 = Sequential() - output2.add(SpatialAveragePooling(5, 5, 3, 3).set_name("loss2/ave_pool")) - output2.add( - SpatialConvolution(528, 128, 1, 1, 1, 1).set_name("loss2/conv")) - output2.add(ReLU(True).set_name("loss2/relu_conv")) - output2.add(View([128 * 4 * 4, 3])) - output2.add(Linear(128 * 4 * 4, 1024).set_name("loss2/fc")) - output2.add(ReLU(True).set_name("loss2/relu_fc")) - if has_dropout: - output2.add(Dropout(0.7).set_name("loss2/drop_fc")) - output2.add(Linear(1024, class_num).set_name("loss2/classifier")) - output2.add(LogSoftMax().set_name("loss2/loss")) - - output3 = Sequential() - output3.add(inception_layer_v1(528, - t([t([256]), t([160, 320]), t([32, 128]), t([128])]), - "inception_4e/")) - output3.add(SpatialMaxPooling(3, 3, 2, 2, to_ceil=True).set_name("pool4/3x3_s2")) - output3.add(inception_layer_v1(832, - t([t([256]), t([160, 320]), t([32, 128]), t([128])]), - "inception_5a/")) - output3.add(inception_layer_v1(832, - t([t([384]), t([192, 384]), t([48, 128]), t([128])]), - "inception_5b/")) - output3.add(SpatialAveragePooling(7, 7, 1, 1).set_name("pool5/7x7_s1")) - if has_dropout: - output3.add(Dropout(0.4).set_name("pool5/drop_7x7_s1")) - output3.add(View([1024, 3])) - output3.add(Linear(1024, class_num) - .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) - .set_name("loss3/classifier")) - output3.add(LogSoftMax().set_name("loss3/loss3")) - - split2 = Concat(2).set_name("split2") - split2.add(output3) - split2.add(output2) - - mainBranch = Sequential() - mainBranch.add(feature2) - mainBranch.add(split2) - - split1 = Concat(2).set_name("split1") - split1.add(mainBranch) - split1.add(output1) - - model = Sequential() - - model.add(feature1) - model.add(split1) - - model.reset() - return model - - def get_inception_data(url, sc=None, data_type="train"): path = os.path.join(url, data_type) return SeqFileFolder.files_to_image_frame(url=path, sc=sc, class_num=1000) @@ -239,14 +417,14 @@ def get_inception_data(url, sc=None, data_type="train"): def config_option_parser(): parser = OptionParser() parser.add_option("-f", "--folder", type=str, dest="folder", default="", - help="url of hdfs folder store the hadoop sequence files") + help="url of hdf+s folder store the hadoop sequence files") parser.add_option("--model", type=str, dest="model", default="", help="model snapshot location") parser.add_option("--state", type=str, dest="state", default="", help="state snapshot location") parser.add_option("--checkpoint", type=str, dest="checkpoint", default="", help="where to cache the model") parser.add_option("-o", "--overwrite", action="store_true", dest="overwrite", default=False, help="overwrite checkpoint files") parser.add_option("-e", "--maxEpoch", type=int, dest="maxEpoch", default=0, help="epoch numbers") - parser.add_option("-i", "--maxIteration", type=int, dest="maxIteration", default=62000, help="iteration numbers") + parser.add_option("-i", "--maxIteration", type=int, dest="maxIteration", default=3100, help="iteration numbers") parser.add_option("-l", "--learningRate", type=float, dest="learningRate", default=0.01, help="learning rate") parser.add_option("--warmupEpoch", type=int, dest="warmupEpoch", default=0, help="warm up epoch numbers") parser.add_option("--maxLr", type=float, dest="maxLr", default=0.0, help="max Lr after warm up") @@ -257,58 +435,123 @@ def config_option_parser(): help="checkpoint interval of iterations") parser.add_option("--gradientMin", type=float, dest="gradientMin", default=0.0, help="min gradient clipping by") parser.add_option("--gradientMax", type=float, dest="gradientMax", default=0.0, help="max gradient clipping by") - parser.add_option("--gradientL2NormThreshold", type=float, dest="gradientL2NormThreshold", default=0.0, help="gradient L2-Norm threshold") + parser.add_option("--gradientL2NormThreshold", type=float, dest="gradientL2NormThreshold", default=0.0, + help="gradient L2-Norm threshold") return parser if __name__ == "__main__": # parse options + print("-----------set one option parser for user---------") parser = config_option_parser() + print("-----------got the settings from the user---------") (options, args) = parser.parse_args(sys.argv) + if not options.learningRate: parser.error("-l --learningRate is a mandatory opt") + print("-----------check if the learning rate has been set---------") if not options.batchSize: parser.error("-b --batchSize is a mandatory opt") + print("-----------check if the batchsize has been set---------") - # init - sparkConf = create_spark_conf().setAppName("inception v1") - sc = get_spark_context(sparkConf) - redire_spark_logs() - show_bigdl_info_logs() - init_engine() + ## init + sc = init_nncontext("inception v1") + print("-----------init the spark???---------") image_size = 224 # create dataset + print("-----------set the image size for this test, so we know that the image's size is 224---------") + ## got dataFrame + # transformer = ChainedPreprocessing( + # [RowToImageFeature(), ImageResize(256, 256), ImageCenterCrop(224, 224), + # ImageChannelNormalize(123.0, 117.0, 104.0), ImageMatToTensor(), ImageFeatureToTensor()]) train_transformer = ChainedPreprocessing([ImagePixelBytesToMat(), ImageRandomCrop(image_size, image_size), ImageRandomPreprocessing(ImageHFlip(), 0.5), ImageChannelNormalize(0.485, 0.456, 0.406, 0.229, 0.224, 0.225), - ImageMatToTensor(to_RGB=True), + ImageMatToTensor(format="NHWC", to_RGB=True), ImageSetToSample(input_keys=["imageTensor"], target_keys=["label"]) ]) raw_train_data = get_inception_data(options.folder, sc, "train") - train_data = FeatureSet.image_frame(raw_train_data).transform(train_transformer).transform(ImageFeatureToSample()) + train_data = FeatureSet.image_frame(raw_train_data).transform(train_transformer) val_transformer = ChainedPreprocessing([ImagePixelBytesToMat(), ImageCenterCrop(image_size, image_size), ImageRandomPreprocessing(ImageHFlip(), 0.5), ImageChannelNormalize(0.485, 0.456, 0.406, 0.229, 0.224, 0.225), - ImageMatToTensor(to_RGB=True), + ImageMatToTensor(format="NHWC", to_RGB=True), ImageSetToSample(input_keys=["imageTensor"], target_keys=["label"]), ]) raw_val_data = get_inception_data(options.folder, sc, "val") - val_data = FeatureSet.image_frame(raw_val_data).transform(val_transformer).transform(ImageFeatureToSample()) - - # build model - bigdl_model = inception_v1_no_aux_classifier(options.classNum) - - model_path = "/tmp/bigdl_inception-v1_imagenet_0.4.0.model" - bigdl_model.saveModel(model_path, over_write = True) - inception_zoo_model = Net.load_bigdl(model_path) - inputNode = Input(name="input", shape=(3, 224, 224)) - outputLayer = inception_zoo_model.to_keras()(inputNode) - model = Model(inputNode, outputLayer) - + val_data = FeatureSet.image_frame(raw_val_data).transform(val_transformer) + + #train_transformer = ChainedPreprocessing([ImagePixelBytesToMat(), + # ImageRandomCrop(image_size, image_size), + # ImageRandomPreprocessing(ImageHFlip(), 0.5), + # ImageChannelNormalize(0.485, 0.456, 0.406, 0.229, 0.224, 0.225), + # ImageMatToTensor(to_RGB=True), + # ImageSetToSample(input_keys=["imageTensor"], target_keys=["label"]) + # ]) + #raw_train_data = get_inception_data(options.folder, sc, "train") + #train_data = ImageSet.from_image_frame(raw_train_data).transform(train_transformer) + #print("-----------get the train data---------") + + #val_transformer = ChainedPreprocessing([ImagePixelBytesToMat(), + # ImageCenterCrop(image_size, image_size), + # ImageRandomPreprocessing(ImageHFlip(), 0.5), + # ImageChannelNormalize(0.485, 0.456, 0.406, 0.229, 0.224, 0.225), + # ImageMatToTensor(to_RGB=True), + # ImageSetToSample(input_keys=["imageTensor"], target_keys=["label"]) + # ]) + #raw_val_data = get_inception_data(options.folder, sc, "val") + #val_data = ImageSet.from_image_frame(raw_val_data).transform(val_transformer) + print("-----------get the validation data---------") + + dataset = TFDataset.from_feature_set(train_data, + features=(tf.float32, [224, 224, 3]), + labels=(tf.int32, [1]), + batch_size=options.batchSize, + validation_dataset=val_data) + + # construct the model from TFDataset + images, labels = dataset.tensors + + with slim.arg_scope(inception_v1.inception_v1_arg_scope()): + logits, end_points = inception_v1.inception_v1(images, num_classes=1000, is_training=True) + + loss = tf.reduce_mean(tf.losses.sparse_softmax_cross_entropy(logits=logits, labels=labels)) + + # model_path = "/root/manfei/trainedmodels/bigdl_inception-v1_imagenet_0.1.0.model" + # # bigdl_model.saveModel(model_path, over_write = True) + # inception_zoo_model = Net.load_bigdl(model_path) + # print("-----------loaded the model---------") + # # print("*****get type*****") + # # print(type(inception_zoo_model)) + # + # inputNode = Input(name="input", shape=(3, 224, 224)) + # outputLayer = inception_zoo_model(inputNode)# GraphNet tranfor to keras or we can directlly get the output node using the input + # #get loss form outputlayer and labels + # model = Model(inputNode, outputLayer)## whether it is a tensorflow model at this time and we do not need to save as one tensorflow model, but we need to save it as one tensorflow model before we can do some operation + # # or we just save the bigdl model from the loaded model/ + # # cause this model is keras model, but whether or not we need to create one tensorflow model? yes, we need, so that we cna achieve the tensorflow inference + # # print("-----------got the model---------") + # ######################### + # # from bigdl.nn.layer import * + # # from bigdl.optim.optimizer import * + # # from bigdl.util.common import * + # + # # # create a graph model + # # linear = Linear(10, 2)() + # # sigmoid = Sigmoid()(linear) + # # softmax = SoftMax()(sigmoid) + # # model = Model([linear], [softmax]) + # + # # save it to Tensorflow model file + # ### remember to transfor to nchw from nhwc + # model.save_tensorflow([("input", [3, 224, 224])], "/tmp/tensorflow_from_bigdl_inception-v1_imagenet_0.4.0_model.pb", data_format="nchw") + # print("-----------save the model as one tensorflow model type---------") + # + # ######################### # set optimization method iterationPerEpoch = int(ceil(float(1281167) / options.batchSize)) @@ -328,15 +571,17 @@ def config_option_parser(): maxlr = options.maxLr else: maxlr = options.learningRate - warmupDelta = (maxlr - options.learningRate)/warmup_iteration + warmupDelta = (maxlr - options.learningRate) / warmup_iteration polyIteration = maxIteration - warmup_iteration lrSchedule = SequentialSchedule(iterationPerEpoch) lrSchedule.add(Warmup(warmupDelta), warmup_iteration) lrSchedule.add(Poly(0.5, polyIteration), polyIteration) + ## can we use SGD in this way? optim = SGD(learningrate=options.learningRate, learningrate_decay=0.0, weightdecay=options.weightDecay, momentum=0.9, dampening=0.0, nesterov=False, leaningrate_schedule=lrSchedule) + ## where to find the same function # create triggers if options.maxEpoch: checkpoint_trigger = EveryEpoch() @@ -347,11 +592,56 @@ def config_option_parser(): test_trigger = SeveralIteration(options.checkpointIteration) end_trigger = MaxIteration(options.maxIteration) - # Optimizer - model.compile(optimizer=optim, - loss='categorical_crossentropy', - metrics=['accuracy', 'top5acc']) - model.fit(x = train_data, batch_size=options.batchSize, nb_epoch=70, validation_data=val_data) + ### set classifier + ### not sure if this can use + # classifier = NNClassifier(model, CrossEntropyCriterion()) \ + # .setBatchSize(options.batchSize).setMaxEpoch(70).setFeaturesCol("image") \ + # .setCachingSample(False).setOptimMethod(optim) + # pipeline = Pipeline(stages=[classifier]) + + # # Optimizer + # model.compile(optimizer=optim, + # loss='categorical_crossentropy', + # metrics=['accuracy', 'top5acc']) + # model.fit(x = train_data, batch_size=options.batchSize, nb_epoch=70, validation_data=val_data) + + # trainedModel = pipeline.fit(train_data) + # predictionDF = trainedModel.transform(val_data).cache() + # predictionDF.select("name", "label", "prediction").sort("label", ascending=False).show(10) + # predictionDF.select("name", "label", "prediction").show(10) + # correct = predictionDF.filter("label=prediction").count() + # overall = predictionDF.count() + # accuracy = correct * 1.0 / overall + # print("Test Error = %g " % (1.0 - accuracy)) + + optimizer = TFOptimizer(loss, optim, val_outputs=[logits], val_labels=[labels], val_method=Top1Accuracy()) + optimizer.set_train_summary(TrainSummary("/tmp/logs/inceptionV1", "inceptionV1")) + optimizer.set_val_summary(ValidationSummary("/tmp/logs/inceptionV1", "inceptionV1")) + optimizer.optimize(end_trigger=end_trigger) + + # ################################# + # # construct the model from TFDataset + # images, labels = dataset.tensors + # + # with slim.arg_scope(lenet.lenet_arg_scope()): + # logits, end_points = lenet.lenet(images, num_classes=10, is_training=True) + # + # loss = tf.reduce_mean(tf.losses.sparse_softmax_cross_entropy(logits=logits, labels=labels)) + # + # # create a optimizer + # optimizer = TFOptimizer(loss, Adam(1e-3), + # val_outputs=[logits], + # val_labels=[labels], + # val_method=Top1Accuracy()) + # optimizer.set_train_summary(TrainSummary("/tmp/az_lenet", "lenet")) + # optimizer.set_val_summary(ValidationSummary("/tmp/az_lenet", "lenet")) + # # kick off training + # optimizer.optimize(end_trigger=MaxEpoch(max_epoch)) + + saver = tf.train.Saver() + saver.save(optimizer.sess, "/root/manfei/model") + ################################# + # optimizer = Optimizer.create( # model=inception_model, # training_set=train_data, From d29ab252bf44e8a7dfa646f2dc7ab7195f5e703b Mon Sep 17 00:00:00 2001 From: qiuxin2012 Date: Wed, 8 May 2019 15:44:54 +0800 Subject: [PATCH 03/28] cleanup --- .../tensorflow/inception/inception.py | 516 +----------------- 1 file changed, 2 insertions(+), 514 deletions(-) diff --git a/pyzoo/zoo/examples/tensorflow/inception/inception.py b/pyzoo/zoo/examples/tensorflow/inception/inception.py index 16c3b43ffb2..e6866f686fe 100644 --- a/pyzoo/zoo/examples/tensorflow/inception/inception.py +++ b/pyzoo/zoo/examples/tensorflow/inception/inception.py @@ -1,384 +1,3 @@ -#from bigdl.nn.layer import Sequential, Concat, SpatialConvolution, ReLU, SpatialMaxPooling, SpatialCrossMapLRN, SpatialAveragePooling, Dropout, View, Linear, LogSoftMax -#from optparse import OptionParser -#from bigdl.nn.criterion import * -#from bigdl.nn.initialization_method import * -#from bigdl.optim.optimizer import * -#from zoo.feature.image.imagePreprocessing import * -#from zoo.feature.image.imageset import * -#from zoo.feature.common import ChainedPreprocessing -#from math import ceil -#from zoo.common.nncontext import * -#from zoo.feature.common import * -#from zoo.feature.image.imagePreprocessing import * -# -#from zoo.common.nncontext import * -#from zoo.feature.common import * -#from zoo.feature.image.imagePreprocessing import * -#from zoo.pipeline.api.keras.layers import Dense, Input, Flatten -#from zoo.pipeline.api.net import * -#from zoo.pipeline.api.keras.models import Model -#from bigdl.optim.optimizer import * -# -# -#def t(input_t): -# if type(input_t) is list: -# # insert into index 0 spot, such that the real data starts from index 1 -# temp = [0] -# temp.extend(input_t) -# return dict(enumerate(temp)) -# # if dictionary, return it back -# return input_t -# -# -#def inception_layer_v1(input_size, config, name_prefix=""): -# concat = Concat(2) -# conv1 = Sequential() -# conv1.add(SpatialConvolution(input_size, config[1][1], 1, 1, 1, 1) -# .set_init_method(weight_init_method=Xavier(),bias_init_method=Zeros()) -# .set_name(name_prefix + "1x1")) -# conv1.add(ReLU(True).set_name(name_prefix + "relu_1x1")) -# concat.add(conv1) -# conv3 = Sequential() -# conv3.add(SpatialConvolution(input_size, config[2][1], 1, 1, 1, 1) -# .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) -# .set_name(name_prefix + "3x3_reduce")) -# conv3.add(ReLU(True).set_name(name_prefix + "relu_3x3_reduce")) -# conv3.add(SpatialConvolution(config[2][1], config[2][2], 3, 3, 1, 1, 1, 1) -# .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) -# .set_name(name_prefix + "3x3")) -# conv3.add(ReLU(True).set_name(name_prefix + "relu_3x3")) -# concat.add(conv3) -# conv5 = Sequential() -# conv5.add(SpatialConvolution(input_size, config[3][1], 1, 1, 1, 1) -# .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) -# .set_name(name_prefix + "5x5_reduce")) -# conv5.add(ReLU(True).set_name(name_prefix + "relu_5x5_reduce")) -# conv5.add(SpatialConvolution(config[3][1], config[3][2], 5, 5, 1, 1, 2, 2) -# .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) -# .set_name(name_prefix + "5x5")) -# conv5.add(ReLU(True).set_name(name_prefix + "relu_5x5")) -# concat.add(conv5) -# pool = Sequential() -# pool.add(SpatialMaxPooling(3, 3, 1, 1, 1, 1, to_ceil=True).set_name(name_prefix + "pool")) -# pool.add(SpatialConvolution(input_size, config[4][1], 1, 1, 1, 1) -# .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) -# .set_name(name_prefix + "pool_proj")) -# pool.add(ReLU(True).set_name(name_prefix + "relu_pool_proj")) -# concat.add(pool).set_name(name_prefix + "output") -# return concat -# -# -#def inception_v1_no_aux_classifier(class_num, has_dropout=True): -# model = Sequential() -# model.add(SpatialConvolution(3, 64, 7, 7, 2, 2, 3, 3, 1, False) -# .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) -# .set_name("conv1/7x7_s2")) -# model.add(ReLU(True).set_name("conv1/relu_7x7")) -# model.add(SpatialMaxPooling(3, 3, 2, 2, to_ceil=True).set_name("pool1/3x3_s2")) -# model.add(SpatialCrossMapLRN(5, 0.0001, 0.75).set_name("pool1/norm1")) -# model.add(SpatialConvolution(64, 64, 1, 1, 1, 1) -# .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) -# .set_name("conv2/3x3_reduce")) -# model.add(ReLU(True).set_name("conv2/relu_3x3_reduce")) -# model.add(SpatialConvolution(64, 192, 3, 3, 1, 1, 1, 1) -# .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) -# .set_name("conv2/3x3")) -# model.add(ReLU(True).set_name("conv2/relu_3x3")) -# model.add(SpatialCrossMapLRN(5, 0.0001, 0.75).set_name("conv2/norm2")) -# model.add(SpatialMaxPooling(3, 3, 2, 2, to_ceil=True).set_name("pool2/3x3_s2")) -# model.add(inception_layer_v1(192, t([t([64]), t( -# [96, 128]), t([16, 32]), t([32])]), "inception_3a/")) -# model.add(inception_layer_v1(256, t([t([128]), t( -# [128, 192]), t([32, 96]), t([64])]), "inception_3b/")) -# model.add(SpatialMaxPooling(3, 3, 2, 2, to_ceil=True)) -# model.add(inception_layer_v1(480, t([t([192]), t( -# [96, 208]), t([16, 48]), t([64])]), "inception_4a/")) -# model.add(inception_layer_v1(512, t([t([160]), t( -# [112, 224]), t([24, 64]), t([64])]), "inception_4b/")) -# model.add(inception_layer_v1(512, t([t([128]), t( -# [128, 256]), t([24, 64]), t([64])]), "inception_4c/")) -# model.add(inception_layer_v1(512, t([t([112]), t( -# [144, 288]), t([32, 64]), t([64])]), "inception_4d/")) -# model.add(inception_layer_v1(528, t([t([256]), t( -# [160, 320]), t([32, 128]), t([128])]), "inception_4e/")) -# model.add(SpatialMaxPooling(3, 3, 2, 2, to_ceil=True)) -# model.add(inception_layer_v1(832, t([t([256]), t( -# [160, 320]), t([32, 128]), t([128])]), "inception_5a/")) -# model.add(inception_layer_v1(832, t([t([384]), t( -# [192, 384]), t([48, 128]), t([128])]), "inception_5b/")) -# model.add(SpatialAveragePooling(7, 7, 1, 1).set_name("pool5/7x7_s1")) -# if has_dropout: -# model.add(Dropout(0.4).set_name("pool5/drop_7x7_s1")) -# model.add(View([1024], num_input_dims=3)) -# model.add(Linear(1024, class_num) -# .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) -# .set_name("loss3/classifier")) -# model.add(LogSoftMax().set_name("loss3/loss3")) -# model.reset() -# return model -# -# -#def inception_v1(class_num, has_dropout=True): -# feature1 = Sequential() -# feature1.add(SpatialConvolution(3, 64, 7, 7, 2, 2, 3, 3, 1, False) -# .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) -# .set_name("conv1/7x7_s2")) -# feature1.add(ReLU(True).set_name("conv1/relu_7x7")) -# feature1.add( -# SpatialMaxPooling(3, 3, 2, 2, to_ceil=True) -# .set_name("pool1/3x3_s2")) -# feature1.add(SpatialCrossMapLRN(5, 0.0001, 0.75) -# .set_name("pool1/norm1")) -# feature1.add(SpatialConvolution(64, 64, 1, 1, 1, 1) -# .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) -# .set_name("conv2/3x3_reduce")) -# feature1.add(ReLU(True).set_name("conv2/relu_3x3_reduce")) -# feature1.add(SpatialConvolution(64, 192, 3, 3, 1, 1, 1, 1) -# .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) -# .set_name("conv2/3x3")) -# feature1.add(ReLU(True).set_name("conv2/relu_3x3")) -# feature1.add(SpatialCrossMapLRN(5, 0.0001, 0.75).set_name("conv2/norm2")) -# feature1.add( -# SpatialMaxPooling(3, 3, 2, 2, to_ceil=True).set_name("pool2/3x3_s2")) -# feature1.add(inception_layer_v1(192, t([ -# t([64]), t([96, 128]), t([16, 32]), t([32])]), -# "inception_3a/")) -# feature1.add(inception_layer_v1(256, t([ -# t([128]), t([128, 192]), t([32, 96]), t([64])]), -# "inception_3b/")) -# feature1.add( -# SpatialMaxPooling(3, 3, 2, 2, to_ceil=True).set_name("pool3/3x3_s2")) -# feature1.add(inception_layer_v1(480, t([ -# t([192]), t([96, 208]), t([16, 48]), t([64])]), -# "inception_4a/")) -# -# output1 = Sequential() -# output1.add(SpatialAveragePooling( -# 5, 5, 3, 3, ceil_mode=True).set_name("loss1/ave_pool")) -# output1.add( -# SpatialConvolution(512, 128, 1, 1, 1, 1).set_name("loss1/conv")) -# output1.add(ReLU(True).set_name("loss1/relu_conv")) -# output1.add(View([128 * 4 * 4, 3])) -# output1.add(Linear(128 * 4 * 4, 1024).set_name("loss1/fc")) -# output1.add(ReLU(True).set_name("loss1/relu_fc")) -# if has_dropout: -# output1.add(Dropout(0.7).set_name("loss1/drop_fc")) -# output1.add(Linear(1024, class_num).set_name("loss1/classifier")) -# output1.add(LogSoftMax().set_name("loss1/loss")) -# -# feature2 = Sequential() -# feature2.add(inception_layer_v1(512, -# t([t([160]), t([112, 224]), t([24, 64]), t([64])]), -# "inception_4b/")) -# feature2.add(inception_layer_v1(512, -# t([t([128]), t([128, 256]), t([24, 64]), t([64])]), -# "inception_4c/")) -# feature2.add(inception_layer_v1(512, -# t([t([112]), t([144, 288]), t([32, 64]), t([64])]), -# "inception_4d/")) -# -# output2 = Sequential() -# output2.add(SpatialAveragePooling(5, 5, 3, 3).set_name("loss2/ave_pool")) -# output2.add( -# SpatialConvolution(528, 128, 1, 1, 1, 1).set_name("loss2/conv")) -# output2.add(ReLU(True).set_name("loss2/relu_conv")) -# output2.add(View([128 * 4 * 4, 3])) -# output2.add(Linear(128 * 4 * 4, 1024).set_name("loss2/fc")) -# output2.add(ReLU(True).set_name("loss2/relu_fc")) -# if has_dropout: -# output2.add(Dropout(0.7).set_name("loss2/drop_fc")) -# output2.add(Linear(1024, class_num).set_name("loss2/classifier")) -# output2.add(LogSoftMax().set_name("loss2/loss")) -# -# output3 = Sequential() -# output3.add(inception_layer_v1(528, -# t([t([256]), t([160, 320]), t([32, 128]), t([128])]), -# "inception_4e/")) -# output3.add(SpatialMaxPooling(3, 3, 2, 2, to_ceil=True).set_name("pool4/3x3_s2")) -# output3.add(inception_layer_v1(832, -# t([t([256]), t([160, 320]), t([32, 128]), t([128])]), -# "inception_5a/")) -# output3.add(inception_layer_v1(832, -# t([t([384]), t([192, 384]), t([48, 128]), t([128])]), -# "inception_5b/")) -# output3.add(SpatialAveragePooling(7, 7, 1, 1).set_name("pool5/7x7_s1")) -# if has_dropout: -# output3.add(Dropout(0.4).set_name("pool5/drop_7x7_s1")) -# output3.add(View([1024, 3])) -# output3.add(Linear(1024, class_num) -# .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) -# .set_name("loss3/classifier")) -# output3.add(LogSoftMax().set_name("loss3/loss3")) -# -# split2 = Concat(2).set_name("split2") -# split2.add(output3) -# split2.add(output2) -# -# mainBranch = Sequential() -# mainBranch.add(feature2) -# mainBranch.add(split2) -# -# split1 = Concat(2).set_name("split1") -# split1.add(mainBranch) -# split1.add(output1) -# -# model = Sequential() -# -# model.add(feature1) -# model.add(split1) -# -# model.reset() -# return model -# -# -#def get_inception_data(url, sc=None, data_type="train"): -# path = os.path.join(url, data_type) -# return SeqFileFolder.files_to_image_frame(url=path, sc=sc, class_num=1000) -# -# -#def config_option_parser(): -# parser = OptionParser() -# parser.add_option("-f", "--folder", type=str, dest="folder", default="", -# help="url of hdfs folder store the hadoop sequence files") -# parser.add_option("--model", type=str, dest="model", default="", help="model snapshot location") -# parser.add_option("--state", type=str, dest="state", default="", help="state snapshot location") -# parser.add_option("--checkpoint", type=str, dest="checkpoint", default="", help="where to cache the model") -# parser.add_option("-o", "--overwrite", action="store_true", dest="overwrite", default=False, -# help="overwrite checkpoint files") -# parser.add_option("-e", "--maxEpoch", type=int, dest="maxEpoch", default=0, help="epoch numbers") -# parser.add_option("-i", "--maxIteration", type=int, dest="maxIteration", default=62000, help="iteration numbers") -# parser.add_option("-l", "--learningRate", type=float, dest="learningRate", default=0.01, help="learning rate") -# parser.add_option("--warmupEpoch", type=int, dest="warmupEpoch", default=0, help="warm up epoch numbers") -# parser.add_option("--maxLr", type=float, dest="maxLr", default=0.0, help="max Lr after warm up") -# parser.add_option("-b", "--batchSize", type=int, dest="batchSize", help="batch size") -# parser.add_option("--classNum", type=int, dest="classNum", default=1000, help="class number") -# parser.add_option("--weightDecay", type=float, dest="weightDecay", default=0.0001, help="weight decay") -# parser.add_option("--checkpointIteration", type=int, dest="checkpointIteration", default=620, -# help="checkpoint interval of iterations") -# parser.add_option("--gradientMin", type=float, dest="gradientMin", default=0.0, help="min gradient clipping by") -# parser.add_option("--gradientMax", type=float, dest="gradientMax", default=0.0, help="max gradient clipping by") -# parser.add_option("--gradientL2NormThreshold", type=float, dest="gradientL2NormThreshold", default=0.0, help="gradient L2-Norm threshold") -# -# return parser -# -# -#if __name__ == "__main__": -# # parse options -# parser = config_option_parser() -# (options, args) = parser.parse_args(sys.argv) -# if not options.learningRate: -# parser.error("-l --learningRate is a mandatory opt") -# if not options.batchSize: -# parser.error("-b --batchSize is a mandatory opt") -# -# # init -# sparkConf = create_spark_conf().setAppName("inception v1") -# sc = get_spark_context(sparkConf) -# redire_spark_logs() -# show_bigdl_info_logs() -# init_engine() -# -# image_size = 224 # create dataset -# train_transformer = ChainedPreprocessing([ImagePixelBytesToMat(), -# ImageRandomCrop(image_size, image_size), -# ImageRandomPreprocessing(ImageHFlip(), 0.5), -# ImageChannelNormalize(0.485, 0.456, 0.406, 0.229, 0.224, 0.225), -# ImageMatToTensor(to_RGB=True), -# ImageSetToSample(input_keys=["imageTensor"], target_keys=["label"]) -# ]) -# raw_train_data = get_inception_data(options.folder, sc, "train") -# train_data = FeatureSet.image_frame(raw_train_data).transform(train_transformer).transform(ImageFeatureToSample()) -# -# val_transformer = ChainedPreprocessing([ImagePixelBytesToMat(), -# ImageCenterCrop(image_size, image_size), -# ImageRandomPreprocessing(ImageHFlip(), 0.5), -# ImageChannelNormalize(0.485, 0.456, 0.406, 0.229, 0.224, 0.225), -# ImageMatToTensor(to_RGB=True), -# ImageSetToSample(input_keys=["imageTensor"], target_keys=["label"]), -# ]) -# raw_val_data = get_inception_data(options.folder, sc, "val") -# val_data = FeatureSet.image_frame(raw_val_data).transform(val_transformer).transform(ImageFeatureToSample()) -# -# # build model -# bigdl_model = inception_v1_no_aux_classifier(options.classNum) -# -# model_path = "/tmp/bigdl_inception-v1_imagenet_0.4.0.model" -# bigdl_model.saveModel(model_path, over_write = True) -# inception_zoo_model = Net.load_bigdl(model_path) -# inputNode = Input(name="input", shape=(3, 224, 224)) -# outputLayer = inception_zoo_model.to_keras()(inputNode) -# model = Model(inputNode, outputLayer) -# -# -# # set optimization method -# iterationPerEpoch = int(ceil(float(1281167) / options.batchSize)) -# if options.maxEpoch: -# maxIteration = iterationPerEpoch * options.maxEpoch -# else: -# maxIteration = options.maxIteration -# warmup_iteration = options.warmupEpoch * iterationPerEpoch -# if options.state != "": -# # load state snapshot -# optim = OptimMethod.load(options.state) -# else: -# if warmup_iteration == 0: -# warmupDelta = 0.0 -# else: -# if options.maxLr: -# maxlr = options.maxLr -# else: -# maxlr = options.learningRate -# warmupDelta = (maxlr - options.learningRate)/warmup_iteration -# polyIteration = maxIteration - warmup_iteration -# lrSchedule = SequentialSchedule(iterationPerEpoch) -# lrSchedule.add(Warmup(warmupDelta), warmup_iteration) -# lrSchedule.add(Poly(0.5, polyIteration), polyIteration) -# optim = SGD(learningrate=options.learningRate, learningrate_decay=0.0, weightdecay=options.weightDecay, -# momentum=0.9, dampening=0.0, nesterov=False, -# leaningrate_schedule=lrSchedule) -# -# # create triggers -# if options.maxEpoch: -# checkpoint_trigger = EveryEpoch() -# test_trigger = EveryEpoch() -# end_trigger = MaxEpoch(options.maxEpoch) -# else: -# checkpoint_trigger = SeveralIteration(options.checkpointIteration) -# test_trigger = SeveralIteration(options.checkpointIteration) -# end_trigger = MaxIteration(options.maxIteration) -# -# # Optimizer -# model.compile(optimizer=optim, -# loss='categorical_crossentropy', -# metrics=['accuracy', 'top5acc']) -# model.fit(x = train_data, batch_size=options.batchSize, nb_epoch=70, validation_data=val_data) -# # optimizer = Optimizer.create( -# # model=inception_model, -# # training_set=train_data, -# # optim_method=optim, -# # criterion=ClassNLLCriterion(), -# # end_trigger=end_trigger, -# # batch_size=options.batchSize -# # ) -# -# # if options.checkpoint: -# # optimizer.set_checkpoint(checkpoint_trigger, options.checkpoint, options.overwrite) -# # -# # if options.gradientMin and options.gradientMax: -# # optimizer.set_gradclip_const(options.gradientMin, options.gradientMax) -# # -# # if options.gradientL2NormThreshold: -# # optimizer.set_gradclip_l2norm(options.gradientL2NormThreshold) -# -# # optimizer.set_validation(trigger=test_trigger, -# # val_rdd=val_data, -# # batch_size=options.batchSize, -# # val_method=[Top1Accuracy(), Top5Accuracy()]) -# -# # trained_model = optimizer.optimize() -# -# sc.stop() - from optparse import OptionParser from math import ceil @@ -443,28 +62,18 @@ def config_option_parser(): if __name__ == "__main__": # parse options - print("-----------set one option parser for user---------") parser = config_option_parser() - print("-----------got the settings from the user---------") (options, args) = parser.parse_args(sys.argv) if not options.learningRate: parser.error("-l --learningRate is a mandatory opt") - print("-----------check if the learning rate has been set---------") if not options.batchSize: parser.error("-b --batchSize is a mandatory opt") - print("-----------check if the batchsize has been set---------") ## init sc = init_nncontext("inception v1") - print("-----------init the spark???---------") image_size = 224 # create dataset - print("-----------set the image size for this test, so we know that the image's size is 224---------") - ## got dataFrame - # transformer = ChainedPreprocessing( - # [RowToImageFeature(), ImageResize(256, 256), ImageCenterCrop(224, 224), - # ImageChannelNormalize(123.0, 117.0, 104.0), ImageMatToTensor(), ImageFeatureToTensor()]) train_transformer = ChainedPreprocessing([ImagePixelBytesToMat(), ImageRandomCrop(image_size, image_size), ImageRandomPreprocessing(ImageHFlip(), 0.5), @@ -485,28 +94,6 @@ def config_option_parser(): raw_val_data = get_inception_data(options.folder, sc, "val") val_data = FeatureSet.image_frame(raw_val_data).transform(val_transformer) - #train_transformer = ChainedPreprocessing([ImagePixelBytesToMat(), - # ImageRandomCrop(image_size, image_size), - # ImageRandomPreprocessing(ImageHFlip(), 0.5), - # ImageChannelNormalize(0.485, 0.456, 0.406, 0.229, 0.224, 0.225), - # ImageMatToTensor(to_RGB=True), - # ImageSetToSample(input_keys=["imageTensor"], target_keys=["label"]) - # ]) - #raw_train_data = get_inception_data(options.folder, sc, "train") - #train_data = ImageSet.from_image_frame(raw_train_data).transform(train_transformer) - #print("-----------get the train data---------") - - #val_transformer = ChainedPreprocessing([ImagePixelBytesToMat(), - # ImageCenterCrop(image_size, image_size), - # ImageRandomPreprocessing(ImageHFlip(), 0.5), - # ImageChannelNormalize(0.485, 0.456, 0.406, 0.229, 0.224, 0.225), - # ImageMatToTensor(to_RGB=True), - # ImageSetToSample(input_keys=["imageTensor"], target_keys=["label"]) - # ]) - #raw_val_data = get_inception_data(options.folder, sc, "val") - #val_data = ImageSet.from_image_frame(raw_val_data).transform(val_transformer) - print("-----------get the validation data---------") - dataset = TFDataset.from_feature_set(train_data, features=(tf.float32, [224, 224, 3]), labels=(tf.int32, [1]), @@ -518,40 +105,8 @@ def config_option_parser(): with slim.arg_scope(inception_v1.inception_v1_arg_scope()): logits, end_points = inception_v1.inception_v1(images, num_classes=1000, is_training=True) - - loss = tf.reduce_mean(tf.losses.sparse_softmax_cross_entropy(logits=logits, labels=labels)) - - # model_path = "/root/manfei/trainedmodels/bigdl_inception-v1_imagenet_0.1.0.model" - # # bigdl_model.saveModel(model_path, over_write = True) - # inception_zoo_model = Net.load_bigdl(model_path) - # print("-----------loaded the model---------") - # # print("*****get type*****") - # # print(type(inception_zoo_model)) - # - # inputNode = Input(name="input", shape=(3, 224, 224)) - # outputLayer = inception_zoo_model(inputNode)# GraphNet tranfor to keras or we can directlly get the output node using the input - # #get loss form outputlayer and labels - # model = Model(inputNode, outputLayer)## whether it is a tensorflow model at this time and we do not need to save as one tensorflow model, but we need to save it as one tensorflow model before we can do some operation - # # or we just save the bigdl model from the loaded model/ - # # cause this model is keras model, but whether or not we need to create one tensorflow model? yes, we need, so that we cna achieve the tensorflow inference - # # print("-----------got the model---------") - # ######################### - # # from bigdl.nn.layer import * - # # from bigdl.optim.optimizer import * - # # from bigdl.util.common import * - # - # # # create a graph model - # # linear = Linear(10, 2)() - # # sigmoid = Sigmoid()(linear) - # # softmax = SoftMax()(sigmoid) - # # model = Model([linear], [softmax]) - # - # # save it to Tensorflow model file - # ### remember to transfor to nchw from nhwc - # model.save_tensorflow([("input", [3, 224, 224])], "/tmp/tensorflow_from_bigdl_inception-v1_imagenet_0.4.0_model.pb", data_format="nchw") - # print("-----------save the model as one tensorflow model type---------") - # - # ######################### + # As sequence file's label is one-based, so labels need to subtract 1. + loss = tf.reduce_mean(tf.losses.sparse_softmax_cross_entropy(logits=logits, labels=labels - 1)) # set optimization method iterationPerEpoch = int(ceil(float(1281167) / options.batchSize)) @@ -592,79 +147,12 @@ def config_option_parser(): test_trigger = SeveralIteration(options.checkpointIteration) end_trigger = MaxIteration(options.maxIteration) - ### set classifier - ### not sure if this can use - # classifier = NNClassifier(model, CrossEntropyCriterion()) \ - # .setBatchSize(options.batchSize).setMaxEpoch(70).setFeaturesCol("image") \ - # .setCachingSample(False).setOptimMethod(optim) - # pipeline = Pipeline(stages=[classifier]) - - # # Optimizer - # model.compile(optimizer=optim, - # loss='categorical_crossentropy', - # metrics=['accuracy', 'top5acc']) - # model.fit(x = train_data, batch_size=options.batchSize, nb_epoch=70, validation_data=val_data) - - # trainedModel = pipeline.fit(train_data) - # predictionDF = trainedModel.transform(val_data).cache() - # predictionDF.select("name", "label", "prediction").sort("label", ascending=False).show(10) - # predictionDF.select("name", "label", "prediction").show(10) - # correct = predictionDF.filter("label=prediction").count() - # overall = predictionDF.count() - # accuracy = correct * 1.0 / overall - # print("Test Error = %g " % (1.0 - accuracy)) - optimizer = TFOptimizer(loss, optim, val_outputs=[logits], val_labels=[labels], val_method=Top1Accuracy()) optimizer.set_train_summary(TrainSummary("/tmp/logs/inceptionV1", "inceptionV1")) optimizer.set_val_summary(ValidationSummary("/tmp/logs/inceptionV1", "inceptionV1")) optimizer.optimize(end_trigger=end_trigger) - # ################################# - # # construct the model from TFDataset - # images, labels = dataset.tensors - # - # with slim.arg_scope(lenet.lenet_arg_scope()): - # logits, end_points = lenet.lenet(images, num_classes=10, is_training=True) - # - # loss = tf.reduce_mean(tf.losses.sparse_softmax_cross_entropy(logits=logits, labels=labels)) - # - # # create a optimizer - # optimizer = TFOptimizer(loss, Adam(1e-3), - # val_outputs=[logits], - # val_labels=[labels], - # val_method=Top1Accuracy()) - # optimizer.set_train_summary(TrainSummary("/tmp/az_lenet", "lenet")) - # optimizer.set_val_summary(ValidationSummary("/tmp/az_lenet", "lenet")) - # # kick off training - # optimizer.optimize(end_trigger=MaxEpoch(max_epoch)) - saver = tf.train.Saver() saver.save(optimizer.sess, "/root/manfei/model") - ################################# - - # optimizer = Optimizer.create( - # model=inception_model, - # training_set=train_data, - # optim_method=optim, - # criterion=ClassNLLCriterion(), - # end_trigger=end_trigger, - # batch_size=options.batchSize - # ) - - # if options.checkpoint: - # optimizer.set_checkpoint(checkpoint_trigger, options.checkpoint, options.overwrite) - # - # if options.gradientMin and options.gradientMax: - # optimizer.set_gradclip_const(options.gradientMin, options.gradientMax) - # - # if options.gradientL2NormThreshold: - # optimizer.set_gradclip_l2norm(options.gradientL2NormThreshold) - - # optimizer.set_validation(trigger=test_trigger, - # val_rdd=val_data, - # batch_size=options.batchSize, - # val_method=[Top1Accuracy(), Top5Accuracy()]) - - # trained_model = optimizer.optimize() sc.stop() From 2a209eafac790b62c80ecc3e754db48dcc020e08 Mon Sep 17 00:00:00 2001 From: qiuxin2012 Date: Fri, 10 May 2019 14:47:17 +0800 Subject: [PATCH 04/28] some fix --- pyzoo/zoo/examples/tensorflow/inception/inception.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pyzoo/zoo/examples/tensorflow/inception/inception.py b/pyzoo/zoo/examples/tensorflow/inception/inception.py index e6866f686fe..ac3bb2dfcdf 100644 --- a/pyzoo/zoo/examples/tensorflow/inception/inception.py +++ b/pyzoo/zoo/examples/tensorflow/inception/inception.py @@ -4,6 +4,7 @@ from zoo.common.nncontext import * from zoo.feature.image import * from zoo.pipeline.api.net import * +from zoo.pipeline.api.keras.metrics import * from zoo.pipeline.nnframes import * from bigdl.optim.optimizer import * from bigdl.util.common import * @@ -104,7 +105,10 @@ def config_option_parser(): images, labels = dataset.tensors with slim.arg_scope(inception_v1.inception_v1_arg_scope()): - logits, end_points = inception_v1.inception_v1(images, num_classes=1000, is_training=True) + logits, end_points = inception_v1.inception_v1(images, + dropout_keep_prob=0.6, + num_classes=1000, + is_training=True) # As sequence file's label is one-based, so labels need to subtract 1. loss = tf.reduce_mean(tf.losses.sparse_softmax_cross_entropy(logits=logits, labels=labels - 1)) @@ -147,7 +151,7 @@ def config_option_parser(): test_trigger = SeveralIteration(options.checkpointIteration) end_trigger = MaxIteration(options.maxIteration) - optimizer = TFOptimizer(loss, optim, val_outputs=[logits], val_labels=[labels], val_method=Top1Accuracy()) + optimizer = TFOptimizer(loss, optim, val_outputs=[logits], val_labels=[labels-1], val_method=Accuracy()) optimizer.set_train_summary(TrainSummary("/tmp/logs/inceptionV1", "inceptionV1")) optimizer.set_val_summary(ValidationSummary("/tmp/logs/inceptionV1", "inceptionV1")) optimizer.optimize(end_trigger=end_trigger) From 7243534181a6a81d5e0b053b6c14350463843b34 Mon Sep 17 00:00:00 2001 From: qiuxin2012 Date: Tue, 11 Jun 2019 14:01:12 +0800 Subject: [PATCH 05/28] add option --- pyzoo/zoo/examples/tensorflow/inception/inception.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyzoo/zoo/examples/tensorflow/inception/inception.py b/pyzoo/zoo/examples/tensorflow/inception/inception.py index ac3bb2dfcdf..e2fd1f3bf7c 100644 --- a/pyzoo/zoo/examples/tensorflow/inception/inception.py +++ b/pyzoo/zoo/examples/tensorflow/inception/inception.py @@ -57,6 +57,7 @@ def config_option_parser(): parser.add_option("--gradientMax", type=float, dest="gradientMax", default=0.0, help="max gradient clipping by") parser.add_option("--gradientL2NormThreshold", type=float, dest="gradientL2NormThreshold", default=0.0, help="gradient L2-Norm threshold") + parser.add_option("--memoryType", type=str, dest="memoryType", default="DRAM", help="memory storage type, DRAM or PMEM") return parser @@ -83,7 +84,7 @@ def config_option_parser(): ImageSetToSample(input_keys=["imageTensor"], target_keys=["label"]) ]) raw_train_data = get_inception_data(options.folder, sc, "train") - train_data = FeatureSet.image_frame(raw_train_data).transform(train_transformer) + train_data = FeatureSet.image_frame(raw_train_data, options.memoryType).transform(train_transformer) val_transformer = ChainedPreprocessing([ImagePixelBytesToMat(), ImageCenterCrop(image_size, image_size), From e4432bf8de85de69151e23bb84901eec28ff6894 Mon Sep 17 00:00:00 2001 From: YinaChen Date: Wed, 10 Jul 2019 16:33:35 +0800 Subject: [PATCH 06/28] Add checkpoint --- .../tensorflow/inception/inception.py | 22 ++++++++++++++----- pyzoo/zoo/pipeline/api/net/tf_optimizer.py | 10 +++++++-- 2 files changed, 24 insertions(+), 8 deletions(-) diff --git a/pyzoo/zoo/examples/tensorflow/inception/inception.py b/pyzoo/zoo/examples/tensorflow/inception/inception.py index e2fd1f3bf7c..db3bb5553aa 100644 --- a/pyzoo/zoo/examples/tensorflow/inception/inception.py +++ b/pyzoo/zoo/examples/tensorflow/inception/inception.py @@ -1,5 +1,6 @@ from optparse import OptionParser from math import ceil +from datetime import datetime from zoo.common.nncontext import * from zoo.feature.image import * @@ -13,7 +14,7 @@ import tensorflow as tf #sys.path.append("/root/manfei/models/research/slim/") # add the slim library -sys.path.append("/home/xin/IdeaProjects/models/research/slim") # add the slim library +sys.path.append("/opt/work/models/research/slim") # add the slim library from nets import inception_v1 slim = tf.contrib.slim @@ -137,7 +138,8 @@ def config_option_parser(): lrSchedule.add(Warmup(warmupDelta), warmup_iteration) lrSchedule.add(Poly(0.5, polyIteration), polyIteration) ## can we use SGD in this way? - optim = SGD(learningrate=options.learningRate, learningrate_decay=0.0, weightdecay=options.weightDecay, + optim = SGD(learningrate=options.learningRate, learningrate_decay=0.16, + weightdecay=options.weightDecay, momentum=0.9, dampening=0.0, nesterov=False, leaningrate_schedule=lrSchedule) @@ -152,10 +154,18 @@ def config_option_parser(): test_trigger = SeveralIteration(options.checkpointIteration) end_trigger = MaxIteration(options.maxIteration) - optimizer = TFOptimizer(loss, optim, val_outputs=[logits], val_labels=[labels-1], val_method=Accuracy()) - optimizer.set_train_summary(TrainSummary("/tmp/logs/inceptionV1", "inceptionV1")) - optimizer.set_val_summary(ValidationSummary("/tmp/logs/inceptionV1", "inceptionV1")) - optimizer.optimize(end_trigger=end_trigger) + timeStamp = datetime.now().strftime("%Y%m%d_%H%M%S") + + optimizer = TFOptimizer(loss, optim, val_outputs=[logits], val_labels=[labels-1], + val_method=[Accuracy(), Top5Accuracy()]) + train_summary = TrainSummary("/tmp/logs/inceptionV1", "inceptionV1" + timeStamp) + train_summary.set_summary_trigger("Parameters", SeveralIteration(10)) + optimizer.set_train_summary(train_summary) + optimizer.set_val_summary(ValidationSummary("/tmp/logs/inceptionV1", "inceptionV1" + timeStamp)) + if options.checkpoint != "": + optimizer.set_checkpoint(options.checkpoint + timeStamp, options.overwrite) + + optimizer.optimize(end_trigger=end_trigger, checkpoint_trigger=checkpoint_trigger) saver = tf.train.Saver() saver.save(optimizer.sess, "/root/manfei/model") diff --git a/pyzoo/zoo/pipeline/api/net/tf_optimizer.py b/pyzoo/zoo/pipeline/api/net/tf_optimizer.py index a4a5f39277f..8f409421993 100644 --- a/pyzoo/zoo/pipeline/api/net/tf_optimizer.py +++ b/pyzoo/zoo/pipeline/api/net/tf_optimizer.py @@ -426,6 +426,10 @@ def set_train_summary(self, summary): def set_val_summary(self, summary): self.optimizer.set_val_summary(summary) + def set_checkpoint(self, checkpointDir, overwrite): + self.checkPointDir = checkpointDir + self.isOverwrite = overwrite + def set_constant_gradient_clipping(self, min_value, max_value): """ Configure constant clipping settings. @@ -445,12 +449,14 @@ def set_gradient_clipping_by_l2_norm(self, clip_norm): """ self.optimizer.set_gradclip_l2norm(clip_norm) - def optimize(self, end_trigger=None): + def optimize(self, end_trigger=None, checkpoint_trigger=None): if end_trigger is None: end_trigger = MaxEpoch(1) self.optimizer.set_end_when(end_trigger) - + if self.checkPointDir and checkpoint_trigger: + self.optimizer.set_checkpoint(checkpoint_trigger, checkpoint_path=self.checkPointDir,\ + isOverWrite=self.isOverwrite) self.optimizer.optimize() variables = self.training_helper_layer.get_weights() From 93f2d5c8aece5141e8717a0721a40ccae6dadedd Mon Sep 17 00:00:00 2001 From: YinaChen Date: Wed, 31 Jul 2019 16:08:39 +0800 Subject: [PATCH 07/28] Add Estimator python api and test case --- pyzoo/test/zoo/pipeline/estimator/__init__.py | 15 +++ .../zoo/pipeline/estimator/test_estimator.py | 124 ++++++++++++++++++ pyzoo/zoo/__init__.py | 1 + pyzoo/zoo/feature/common.py | 16 ++- pyzoo/zoo/pipeline/estimator/__init__.py | 17 +++ pyzoo/zoo/pipeline/estimator/estimator.py | 49 +++++++ .../zoo/feature/python/PythonFeatureSet.scala | 9 +- .../estimator/python/PythonEstimator.scala | 124 ++++++++++++++++++ 8 files changed, 353 insertions(+), 2 deletions(-) create mode 100644 pyzoo/test/zoo/pipeline/estimator/__init__.py create mode 100644 pyzoo/test/zoo/pipeline/estimator/test_estimator.py create mode 100644 pyzoo/zoo/pipeline/estimator/__init__.py create mode 100644 pyzoo/zoo/pipeline/estimator/estimator.py create mode 100644 zoo/src/main/scala/com/intel/analytics/zoo/pipeline/estimator/python/PythonEstimator.scala diff --git a/pyzoo/test/zoo/pipeline/estimator/__init__.py b/pyzoo/test/zoo/pipeline/estimator/__init__.py new file mode 100644 index 00000000000..5976dc4df02 --- /dev/null +++ b/pyzoo/test/zoo/pipeline/estimator/__init__.py @@ -0,0 +1,15 @@ +# +# Copyright 2018 Analytics Zoo Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/pyzoo/test/zoo/pipeline/estimator/test_estimator.py b/pyzoo/test/zoo/pipeline/estimator/test_estimator.py new file mode 100644 index 00000000000..0a7563d7925 --- /dev/null +++ b/pyzoo/test/zoo/pipeline/estimator/test_estimator.py @@ -0,0 +1,124 @@ +# +# Copyright 2018 Analytics Zoo Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +import shutil +import errno +import string +from bigdl.nn.criterion import * +from bigdl.nn.layer import * +from bigdl.optim.optimizer import * +from numpy.testing import assert_allclose +from pyspark.ml import Pipeline +from pyspark.ml.feature import MinMaxScaler +from pyspark.sql.types import * + +from zoo.common.nncontext import * +from zoo.pipeline.estimator import * +from zoo.pipeline.api.keras.optimizers import Adam as KAdam +from zoo.feature.common import * +from zoo.feature.image import * +from zoo.util.tf import * + +from bigdl.nn.layer import * +from bigdl.nn.criterion import * +from bigdl.optim.optimizer import * +from test.zoo.pipeline.utils.test_utils import ZooTestCase +from zoo.feature.common import * +from zoo import init_nncontext, init_spark_conf + + +class TestEstimator(ZooTestCase): + + def setup_method(self, method): + """ setup any state tied to the execution of the given method in a + class. setup_method is invoked for every test method of a class. + """ + # sparkConf = init_spark_conf().setMaster("local[1]").setAppName("testEstimator") + # self.sc = init_nncontext(sparkConf) + # self.sqlContext = SQLContext(self.sc) + # assert(self.sc.appName == "testEstimator") + sparkConf = init_spark_conf().setMaster("local[4]").setAppName("test feature set") + self.sc = init_nncontext(sparkConf) + + + def teardown_method(self, method): + """ teardown any state that was previously setup with a setup_method + call. + """ + self.sc.stop() + + def test_estimator_train_imagefeature(self): + batch_size = 8 + epoch_num = 5 + images = [] + labels = [] + for i in range(0, 8): + features = np.random.uniform(0, 1, (200, 200, 3)) + label = np.array([2]) + images.append(features) + labels.append(label) + + image_frame = DistributedImageFrame(self.sc.parallelize(images), + self.sc.parallelize(labels)) + + transformer = Pipeline([BytesToMat(), Resize(256, 256), CenterCrop(224, 224), + ChannelNormalize(0.485, 0.456, 0.406, 0.229, 0.224, 0.225), + MatToTensor(), ImageFrameToSample(target_keys=['label'])]) + data_set = FeatureSet.image_frame(image_frame).transform(transformer) + + model = Sequential() + model.add(SpatialConvolution(3, 1, 5, 5)) + model.add(View([1 * 220 * 220])) + model.add(Linear(1 * 220 * 220, 20)) + model.add(LogSoftMax()) + optim_method = SGD(learningrate=0.01) + + estimator = Estimator(model, optim_method, "") + estimator.set_constant_gradient_clipping(0.1, 1.2) + estimator.train_imagefeature(train_set=data_set, criterion=ClassNLLCriterion(), + end_trigger=MaxEpoch(epoch_num), + checkpoint_trigger=EveryEpoch(), + validation_set=data_set, + validation_method=[Top1Accuracy()], batch_size=batch_size) + predict_result = model.predict_image(image_frame.transform(transformer)) + assert(predict_result.get_predict().count(), 8) + + # def test_estimator_train_rdd(self): + # dim = 2 + # data_len = 100 + # + # def gen_rand_sample(): + # features = np.random.uniform(0, 1, dim) + # label = np.array((2 * features).sum() + 0.4) + # return Sample.from_ndarray(features, label) + # + # data_set = FeatureSet.rdd(self.sc.parallelize(range(0, data_len)).map(lambda i: + # gen_rand_sample())) + # + # model = Sequential().set_name("mse") + # model.add(Linear(4, 4).set_name(("fc_1"))) + # model.add(Sigmoid()) + # model.add(Linear(4, 1).set_name("fc_2")) + # model.add(Sigmoid()) + # + # optim_method = SGD(learningrate=0.01) + # + # estimator = Estimator() + + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/pyzoo/zoo/__init__.py b/pyzoo/zoo/__init__.py index 2b9f8535c3d..60225f10248 100644 --- a/pyzoo/zoo/__init__.py +++ b/pyzoo/zoo/__init__.py @@ -29,6 +29,7 @@ JavaCreator.add_creator_class("com.intel.analytics.zoo.feature.python.PythonFeatureSet") JavaCreator.add_creator_class("com.intel.analytics.zoo.pipeline.api.net.python.PythonZooNet") JavaCreator.add_creator_class("com.intel.analytics.zoo.pipeline.inference.PythonInferenceModel") +JavaCreator.add_creator_class("com.intel.analytics.zoo.pipeline.estimator.python.PythonEstimator") for clz in creator_classes: JavaCreator.add_creator_class(clz) diff --git a/pyzoo/zoo/feature/common.py b/pyzoo/zoo/feature/common.py index c3ba2568ea3..f23950f34b7 100644 --- a/pyzoo/zoo/feature/common.py +++ b/pyzoo/zoo/feature/common.py @@ -233,7 +233,7 @@ def image_frame(cls, image_frame, memory_type="DRAM", bigdl_type="float"): def image_set(cls, imageset, memory_type="DRAM", bigdl_type="float"): """ Create FeatureSet from ImageFrame. - :param image_frame: ImageFrame + :param imageset: ImageFrame :param memory_type: string, DRAM or PMEM If it's DRAM, will cache dataset into dynamic random-access memory If it's PMEM, will cache dataset into Intel Optane DC Persistent Memory @@ -244,6 +244,20 @@ def image_set(cls, imageset, memory_type="DRAM", bigdl_type="float"): imageset.to_image_frame(), memory_type) return cls(jvalue=jvalue) + @classmethod + def sample_rdd(cls, rdd, memory_type="DRAM", bigdl_type="float"): + """ + Create FeatureSet from RDD[Sample]. + :param rdd: A RDD[Sample] + :param memory_type: string, DRAM or PMEM + If it's DRAM, will cache dataset into dynamic random-access memory + If it's PMEM, will cache dataset into Intel Optane DC Persistent Memory + :param bigdl_type:numeric type + :return: A feature set + """ + jvalue = callBigDlFunc(bigdl_type, "createSampleFeatureSetFromRDD", rdd, memory_type) + return cls(jvalue=jvalue) + @classmethod def rdd(cls, rdd, memory_type="DRAM", bigdl_type="float"): """ diff --git a/pyzoo/zoo/pipeline/estimator/__init__.py b/pyzoo/zoo/pipeline/estimator/__init__.py new file mode 100644 index 00000000000..2ee5758c0f0 --- /dev/null +++ b/pyzoo/zoo/pipeline/estimator/__init__.py @@ -0,0 +1,17 @@ +# +# Copyright 2018 Analytics Zoo Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from .estimator import * diff --git a/pyzoo/zoo/pipeline/estimator/estimator.py b/pyzoo/zoo/pipeline/estimator/estimator.py new file mode 100644 index 00000000000..6c649c3ff11 --- /dev/null +++ b/pyzoo/zoo/pipeline/estimator/estimator.py @@ -0,0 +1,49 @@ +# +# Copyright 2018 Analytics Zoo Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from bigdl.util.common import JavaValue, callBigDlFunc + + +class Estimator(JavaValue): + def __init__(self, model, optim_methods=None, model_dir=None, jvalue=None, bigdl_type="float"): + self.bigdl_type = bigdl_type + self.value = jvalue if jvalue else callBigDlFunc( + bigdl_type, self.jvm_class_constructor(), model, optim_methods, model_dir) + + def clear_gradient_clipping(self): + callBigDlFunc(self.bigdl_type, "clearGradientClipping") + + def set_constant_gradient_clipping(self, min, max): + callBigDlFunc(self.bigdl_type, "setConstantGradientClipping", self.value, min, max) + + def set_l2_norm_gradient_clipping(self, clip_norm): + callBigDlFunc(self.bigdl_type, "setGradientClippingByL2Norm", self.value, clip_norm) + + def train(self, train_set, criterion, end_trigger=None, checkpoint_trigger=None, + validation_set=None, validation_method=None, batch_size=32): + callBigDlFunc(self.bigdl_type, "estimatorTrain", self.value, train_set, + criterion, end_trigger, checkpoint_trigger, validation_set, + validation_method, batch_size) + + def train_imagefeature(self, train_set, criterion, end_trigger=None, checkpoint_trigger=None, + validation_set=None, validation_method=None, batch_size=32): + callBigDlFunc(self.bigdl_type, "estimatorTrainImageFeature", self.value, train_set, + criterion, end_trigger, checkpoint_trigger, validation_set, + validation_method, batch_size) + + def evaluate(self, validation_set, validation_method, batch_size=32): + callBigDlFunc(self.bigdl_type, "estimatorEvaluate", self.value, + validation_set, validation_method, batch_size) diff --git a/zoo/src/main/scala/com/intel/analytics/zoo/feature/python/PythonFeatureSet.scala b/zoo/src/main/scala/com/intel/analytics/zoo/feature/python/PythonFeatureSet.scala index b8832980505..65616ea89ab 100644 --- a/zoo/src/main/scala/com/intel/analytics/zoo/feature/python/PythonFeatureSet.scala +++ b/zoo/src/main/scala/com/intel/analytics/zoo/feature/python/PythonFeatureSet.scala @@ -17,11 +17,13 @@ package com.intel.analytics.zoo.feature.python import com.intel.analytics.bigdl.DataSet -import com.intel.analytics.bigdl.dataset.Transformer +import com.intel.analytics.bigdl.dataset.{SampleToMiniBatch, Transformer, Sample => JSample} +import com.intel.analytics.bigdl.python.api.Sample import com.intel.analytics.bigdl.tensor.TensorNumericMath.TensorNumeric import com.intel.analytics.bigdl.transform.vision.image._ import com.intel.analytics.zoo.common.PythonZoo import com.intel.analytics.zoo.feature.FeatureSet +import com.intel.analytics.zoo.feature.image.{ImageChannelNormalize, ImageFeatureToSample, ImageMatToTensor, ImagePixelBytesToMat, ImageRandomCrop, ImageSetToSample} import com.intel.analytics.zoo.feature.pmem.MemoryType import org.apache.spark.api.java.JavaRDD @@ -48,6 +50,11 @@ class PythonFeatureSet[T: ClassTag](implicit ev: TensorNumeric[T]) extends Pytho FeatureSet.rdd(data, MemoryType.fromString(memoryType)) } + def createSampleFeatureSetFromRDD(data: JavaRDD[Sample], memoryType:String) + : FeatureSet[JSample[T]] = { + FeatureSet.rdd(toJSample(data), MemoryType.fromString(memoryType)) + } + def transformFeatureSet(featureSet: FeatureSet[Any], transformer: Transformer[Any, Any]): FeatureSet[Any] = { featureSet -> transformer diff --git a/zoo/src/main/scala/com/intel/analytics/zoo/pipeline/estimator/python/PythonEstimator.scala b/zoo/src/main/scala/com/intel/analytics/zoo/pipeline/estimator/python/PythonEstimator.scala new file mode 100644 index 00000000000..23eb63c779f --- /dev/null +++ b/zoo/src/main/scala/com/intel/analytics/zoo/pipeline/estimator/python/PythonEstimator.scala @@ -0,0 +1,124 @@ +/* + * Copyright 2018 Analytics Zoo Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.intel.analytics.zoo.pipeline.estimator.python + +import java.util.{ArrayList => JArrayList, HashMap => JHashMap, List => JList, Map => JMap} + +import com.intel.analytics.bigdl.{Criterion, Module} +import com.intel.analytics.bigdl.dataset.{ByteRecord, MiniBatch, Sample, SampleToMiniBatch} +import com.intel.analytics.bigdl.optim.{OptimMethod, Trigger, ValidationMethod, ValidationResult} +import com.intel.analytics.bigdl.tensor.TensorNumericMath.TensorNumeric +import com.intel.analytics.bigdl.transform.vision.image.{ImageFeature, ImageFeatureToMiniBatch} +import com.intel.analytics.zoo.common.PythonZoo +import com.intel.analytics.zoo.feature.{DistributedFeatureSet, FeatureSet} +import com.intel.analytics.zoo.pipeline.estimator.Estimator + +import scala.reflect.ClassTag +import scala.collection.JavaConverters._ + +object PythonEstimator { + def ofFloat(): PythonEstimator[Float] = new PythonEstimator[Float]() + + def ofDouble(): PythonEstimator[Double] = new PythonEstimator[Double]() +} + +class PythonEstimator[T: ClassTag](implicit ev: TensorNumeric[T]) extends PythonZoo[T]{ + def createEstimator(model: Module[T], + optimMethod: OptimMethod[T], + modelDir: String): Estimator[T] = { + Estimator(model, optimMethod, modelDir) + } + + def createEstimator(model: Module[T], + optimMethods: Map[String, OptimMethod[T]], + modelDir: String): Estimator[T] = { + Estimator(model, optimMethods, modelDir) + } + + def estimatorEvaluate(estimator: Estimator[T], validationSet: FeatureSet[Sample[T]], + validationMethod: Array[ValidationMethod[T]], batchSize: Int + ): Map[ValidationMethod[T], ValidationResult] = { + val sample2batch = SampleToMiniBatch(batchSize) + val validationMiniBatch = validationSet -> sample2batch + estimator.evaluate(validationMiniBatch, validationMethod) + } + + def estimatorEvaluateImageFeature(estimator: Estimator[T], validationSet: FeatureSet[ImageFeature], + validationMethod: Array[ValidationMethod[T]], batchSize: Int + ): Map[ValidationMethod[T], ValidationResult] = { + val imageFeature2batch = ImageFeatureToMiniBatch(batchSize) + val validationMiniBatch = validationSet -> imageFeature2batch + estimator.evaluate(validationMiniBatch, validationMethod) + } + + def estimatorTrain(estimator: Estimator[T], trainSet: FeatureSet[Sample[T]], + criterion: Criterion[T], + endTrigger: Trigger = null, + checkPointTrigger: Trigger = null, + validationSet: FeatureSet[Sample[T]] = null, + validationMethod: Array[ValidationMethod[T]] = null, batchSize: Int) + : estimator.type = { + val sample2batch = SampleToMiniBatch(batchSize) + val trainMiniBatch = trainSet -> sample2batch + val validationMiniBatch = if (validationSet != null) { + validationSet -> sample2batch + } else { + null + } + + estimator.train(trainMiniBatch, criterion, + Some(endTrigger), Some(checkPointTrigger), + validationMiniBatch, validationMethod) + } + + def estimatorTrainImageFeature(estimator: Estimator[T], trainSet: FeatureSet[ImageFeature], + criterion: Criterion[T], + endTrigger: Trigger = null, + checkPointTrigger: Trigger = null, + validationSet: FeatureSet[ImageFeature] = null, + validationMethod: JList[ValidationMethod[T]] = null, batchSize: Int) + : estimator.type = { + val imageFeature2batch = ImageFeatureToMiniBatch(batchSize) + val trainMiniBatch = trainSet -> imageFeature2batch + val validationMiniBatch = if (validationSet != null) { + validationSet -> imageFeature2batch + } else { + null + } + val valMethods = if (validationMethod != null) { + validationMethod.asScala.toArray + } else { + null + } + + estimator.train(trainMiniBatch, criterion, + Some(endTrigger), Some(checkPointTrigger), + validationMiniBatch, valMethods) + } + + def clearGradientClipping(estimator: Estimator[T]): Unit = { + estimator.clearGradientClipping() + } + + def setConstantGradientClipping(estimator: Estimator[T], min: Double, max: Double): Unit = { + estimator.setConstantGradientClipping(min, max) + } + + def setGradientClippingByL2Norm(estimator: Estimator[T], clipNorm: Double): Unit = { + estimator.setGradientClippingByL2Norm(clipNorm) + } +} From 07fc1032cbb73f1209da3576c525c939d19ec1a5 Mon Sep 17 00:00:00 2001 From: YinaChen Date: Wed, 31 Jul 2019 16:53:37 +0800 Subject: [PATCH 08/28] Add bigdl inception v1 example[TODO:Test] --- .../zoo/pipeline/estimator/test_estimator.py | 14 -- pyzoo/zoo/examples/inception/__init__.py | 15 ++ pyzoo/zoo/examples/inception/inception.py | 149 ++++++++++++++++++ 3 files changed, 164 insertions(+), 14 deletions(-) create mode 100644 pyzoo/zoo/examples/inception/__init__.py create mode 100644 pyzoo/zoo/examples/inception/inception.py diff --git a/pyzoo/test/zoo/pipeline/estimator/test_estimator.py b/pyzoo/test/zoo/pipeline/estimator/test_estimator.py index 0a7563d7925..ae6d9659497 100644 --- a/pyzoo/test/zoo/pipeline/estimator/test_estimator.py +++ b/pyzoo/test/zoo/pipeline/estimator/test_estimator.py @@ -15,23 +15,9 @@ # import pytest -import shutil -import errno -import string -from bigdl.nn.criterion import * -from bigdl.nn.layer import * -from bigdl.optim.optimizer import * -from numpy.testing import assert_allclose from pyspark.ml import Pipeline -from pyspark.ml.feature import MinMaxScaler -from pyspark.sql.types import * -from zoo.common.nncontext import * from zoo.pipeline.estimator import * -from zoo.pipeline.api.keras.optimizers import Adam as KAdam -from zoo.feature.common import * -from zoo.feature.image import * -from zoo.util.tf import * from bigdl.nn.layer import * from bigdl.nn.criterion import * diff --git a/pyzoo/zoo/examples/inception/__init__.py b/pyzoo/zoo/examples/inception/__init__.py new file mode 100644 index 00000000000..5976dc4df02 --- /dev/null +++ b/pyzoo/zoo/examples/inception/__init__.py @@ -0,0 +1,15 @@ +# +# Copyright 2018 Analytics Zoo Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/pyzoo/zoo/examples/inception/inception.py b/pyzoo/zoo/examples/inception/inception.py new file mode 100644 index 00000000000..88312e1bd15 --- /dev/null +++ b/pyzoo/zoo/examples/inception/inception.py @@ -0,0 +1,149 @@ +from optparse import OptionParser +from math import ceil +from datetime import datetime + +from zoo.common.nncontext import * +from zoo.feature.image import * +from zoo.pipeline.nnframes import * +from bigdl.optim.optimizer import * +from bigdl.util.common import * +from bigdl.nn.layer import * +from bigdl.nn.criterion import * +from bigdl.models import inception +from zoo.pipeline.estimator import * + + +def config_option_parser(): + parser = OptionParser() + parser.add_option("-f", "--folder", type=str, dest="folder", default="", + help="url of hdf+s folder store the hadoop sequence files") + parser.add_option("--model", type=str, dest="model", default="", help="model snapshot location") + parser.add_option("--state", type=str, dest="state", default="", help="state snapshot location") + parser.add_option("--checkpoint", type=str, dest="checkpoint", default="", help="where to cache the model") + parser.add_option("-o", "--overwrite", action="store_true", dest="overwrite", default=False, + help="overwrite checkpoint files") + parser.add_option("-e", "--maxEpoch", type=int, dest="maxEpoch", default=0, help="epoch numbers") + parser.add_option("-i", "--maxIteration", type=int, dest="maxIteration", default=3100, help="iteration numbers") + parser.add_option("-l", "--learningRate", type=float, dest="learningRate", default=0.01, help="learning rate") + parser.add_option("--warmupEpoch", type=int, dest="warmupEpoch", default=0, help="warm up epoch numbers") + parser.add_option("--maxLr", type=float, dest="maxLr", default=0.0, help="max Lr after warm up") + parser.add_option("-b", "--batchSize", type=int, dest="batchSize", help="batch size") + parser.add_option("--classNum", type=int, dest="classNum", default=1000, help="class number") + parser.add_option("--weightDecay", type=float, dest="weightDecay", default=0.0001, help="weight decay") + parser.add_option("--checkpointIteration", type=int, dest="checkpointIteration", default=620, + help="checkpoint interval of iterations") + parser.add_option("--gradientMin", type=float, dest="gradientMin", default=0.0, help="min gradient clipping by") + parser.add_option("--gradientMax", type=float, dest="gradientMax", default=0.0, help="max gradient clipping by") + parser.add_option("--gradientL2NormThreshold", type=float, dest="gradientL2NormThreshold", default=0.0, + help="gradient L2-Norm threshold") + parser.add_option("--memoryType", type=str, dest="memoryType", default="DRAM", help="memory storage type, DRAM or PMEM") + + return parser + + +def get_inception_data(url, sc=None, data_type="train"): + path = os.path.join(url, data_type) + return SeqFileFolder.files_to_image_frame(url=path, sc=sc, class_num=1000) + + +if __name__ == "__main__": + # parse options + parser = config_option_parser() + (options, args) = parser.parse_args(sys.argv) + + if not options.learningRate: + parser.error("-l --learningRate is a mandatory opt") + if not options.batchSize: + parser.error("-b --batchSize is a mandatory opt") + + # init + sc = init_nncontext("inception v1") + + image_size = 224 # create dataset + train_transformer = ChainedPreprocessing([ImagePixelBytesToMat(), + ImageRandomCrop(image_size, image_size), + ImageRandomPreprocessing(ImageHFlip(), 0.5), + ImageChannelNormalize(123.0, 117.0, 104.0), + ImageMatToTensor(format="NHWC", to_RGB=True), + ImageSetToSample(input_keys=["imageTensor"], target_keys=["label"]) + ]) + + raw_train_data = get_inception_data(options.folder, sc, "train") + train_data = FeatureSet.image_frame(raw_train_data).transform(train_transformer) + + val_transformer = ChainedPreprocessing([ImagePixelBytesToMat(), + ImageRandomCrop(image_size, image_size), + ImageRandomPreprocessing(ImageHFlip(), 0.5), + ImageChannelNormalize(123.0, 117.0, 104.0), + ImageMatToTensor(format="NHWC", to_RGB=True), + ImageSetToSample(input_keys=["imageTensor"], target_keys=["label"]) + ]) + raw_val_data = get_inception_data(options.folder, sc, "val") + val_data = FeatureSet.image_frame(raw_val_data).transform(val_transformer) + + # build model + if options.model != "": + # load model snapshot + inception_model = Model.load(options.model) + else: + inception_model = inception.inception_v1_no_aux_classifier(options.classNum) + + # set optimization method + iterationPerEpoch = int(ceil(float(1281167) / options.batchSize)) + if options.maxEpoch: + maxIteration = iterationPerEpoch * options.maxEpoch + else: + maxIteration = options.maxIteration + warmup_iteration = options.warmupEpoch * iterationPerEpoch + if options.state != "": + # load state snapshot + optim = OptimMethod.load(options.state) + else: + if warmup_iteration == 0: + warmupDelta = 0.0 + else: + if options.maxLr: + maxlr = options.maxLr + else: + maxlr = options.learningRate + warmupDelta = (maxlr - options.learningRate)/warmup_iteration + polyIteration = maxIteration - warmup_iteration + lrSchedule = SequentialSchedule(iterationPerEpoch) + lrSchedule.add(Warmup(warmupDelta), warmup_iteration) + lrSchedule.add(Poly(0.5, maxIteration), polyIteration) + optim = SGD(learningrate=options.learningRate, learningrate_decay=0.0, weightdecay=options.weightDecay, + momentum=0.9, dampening=0.0, nesterov=False, + leaningrate_schedule=lrSchedule) + + # create triggers + if options.maxEpoch: + checkpoint_trigger = EveryEpoch() + test_trigger = EveryEpoch() + end_trigger = MaxEpoch(options.maxEpoch) + else: + checkpoint_trigger = SeveralIteration(options.checkpointIteration) + test_trigger = SeveralIteration(options.checkpointIteration) + end_trigger = MaxIteration(options.maxIteration) + + timeStamp = datetime.now().strftime("%Y%m%d_%H%M%S") + + # Estimator + estimator = Estimator(inception_model, optim_methods=optim, model_dir=options.checkpoint) + + if options.gradientMin and options.gradientMax: + estimator.set_constant_gradient_clipping(options.gradientMin, options.gradientMax) + + if options.gradientL2NormThreshold: + estimator.set_l2_norm_gradient_clipping(options.gradientL2NormThreshold) + + estimator.train_imagefeature(train_set=train_data, + criterion=ClassNLLCriterion(), + end_trigger=end_trigger, + checkpoint_trigger=checkpoint_trigger, + validation_set=val_data, + validation_method=[Top1Accuracy(), Top5Accuracy()], + batch_size=options.batchSize) + + inception_model.saveModel("/tmp/inception/model.bigdl", "/tmp/inception/model.bin", True) + + sc.stop() \ No newline at end of file From 18cfb6012033499beed27312c8c2258e14b27419 Mon Sep 17 00:00:00 2001 From: YinaChen Date: Fri, 2 Aug 2019 15:13:07 +0800 Subject: [PATCH 09/28] add inception model --- pyzoo/zoo/examples/inception/inception.py | 104 +++++++++++++++++++++- 1 file changed, 101 insertions(+), 3 deletions(-) diff --git a/pyzoo/zoo/examples/inception/inception.py b/pyzoo/zoo/examples/inception/inception.py index 88312e1bd15..cdad4c740a4 100644 --- a/pyzoo/zoo/examples/inception/inception.py +++ b/pyzoo/zoo/examples/inception/inception.py @@ -5,12 +5,12 @@ from zoo.common.nncontext import * from zoo.feature.image import * from zoo.pipeline.nnframes import * +from zoo.pipeline.estimator import * from bigdl.optim.optimizer import * from bigdl.util.common import * from bigdl.nn.layer import * from bigdl.nn.criterion import * -from bigdl.models import inception -from zoo.pipeline.estimator import * +from bigdl.nn.initialization_method import * def config_option_parser(): @@ -46,6 +46,104 @@ def get_inception_data(url, sc=None, data_type="train"): return SeqFileFolder.files_to_image_frame(url=path, sc=sc, class_num=1000) +def t(input_t): + if type(input_t) is list: + # insert into index 0 spot, such that the real data starts from index 1 + temp = [0] + temp.extend(input_t) + return dict(enumerate(temp)) + # if dictionary, return it back + return input_t + + +def inception_layer_v1(input_size, config, name_prefix=""): + concat = Concat(2) + conv1 = Sequential() + conv1.add(SpatialConvolution(input_size, config[1][1], 1, 1, 1, 1) + .set_init_method(weight_init_method=Xavier(),bias_init_method=ConstInitMethod(0.1)) + .set_name(name_prefix + "1x1")) + conv1.add(ReLU(True).set_name(name_prefix + "relu_1x1")) + concat.add(conv1) + conv3 = Sequential() + conv3.add(SpatialConvolution(input_size, config[2][1], 1, 1, 1, 1) + .set_init_method(weight_init_method=Xavier(), bias_init_method=ConstInitMethod(0.1)) + .set_name(name_prefix + "3x3_reduce")) + conv3.add(ReLU(True).set_name(name_prefix + "relu_3x3_reduce")) + conv3.add(SpatialConvolution(config[2][1], config[2][2], 3, 3, 1, 1, 1, 1) + .set_init_method(weight_init_method=Xavier(), bias_init_method=ConstInitMethod(0.1)) + .set_name(name_prefix + "3x3")) + conv3.add(ReLU(True).set_name(name_prefix + "relu_3x3")) + concat.add(conv3) + conv5 = Sequential() + conv5.add(SpatialConvolution(input_size, config[3][1], 1, 1, 1, 1) + .set_init_method(weight_init_method=Xavier(), bias_init_method=ConstInitMethod(0.1)) + .set_name(name_prefix + "5x5_reduce")) + conv5.add(ReLU(True).set_name(name_prefix + "relu_5x5_reduce")) + conv5.add(SpatialConvolution(config[3][1], config[3][2], 5, 5, 1, 1, 2, 2) + .set_init_method(weight_init_method=Xavier(), bias_init_method=ConstInitMethod(0.1)) + .set_name(name_prefix + "5x5")) + conv5.add(ReLU(True).set_name(name_prefix + "relu_5x5")) + concat.add(conv5) + pool = Sequential() + pool.add(SpatialMaxPooling(3, 3, 1, 1, 1, 1, to_ceil=True).set_name(name_prefix + "pool")) + pool.add(SpatialConvolution(input_size, config[4][1], 1, 1, 1, 1) + .set_init_method(weight_init_method=Xavier(), bias_init_method=ConstInitMethod(0.1)) + .set_name(name_prefix + "pool_proj")) + pool.add(ReLU(True).set_name(name_prefix + "relu_pool_proj")) + concat.add(pool).set_name(name_prefix + "output") + return concat + + +def inception_v1_no_aux_classifier(class_num, has_dropout=True): + model = Sequential() + model.add(SpatialConvolution(3, 64, 7, 7, 2, 2, 3, 3, 1, False) + .set_init_method(weight_init_method=Xavier(), bias_init_method=ConstInitMethod(0.1)) + .set_name("conv1/7x7_s2")) + model.add(ReLU(True).set_name("conv1/relu_7x7")) + model.add(SpatialMaxPooling(3, 3, 2, 2, to_ceil=True).set_name("pool1/3x3_s2")) + model.add(SpatialCrossMapLRN(5, 0.0001, 0.75).set_name("pool1/norm1")) + model.add(SpatialConvolution(64, 64, 1, 1, 1, 1) + .set_init_method(weight_init_method=Xavier(), bias_init_method=ConstInitMethod(0.1)) + .set_name("conv2/3x3_reduce")) + model.add(ReLU(True).set_name("conv2/relu_3x3_reduce")) + model.add(SpatialConvolution(64, 192, 3, 3, 1, 1, 1, 1) + .set_init_method(weight_init_method=Xavier(), bias_init_method=ConstInitMethod(0.1)) + .set_name("conv2/3x3")) + model.add(ReLU(True).set_name("conv2/relu_3x3")) + model.add(SpatialCrossMapLRN(5, 0.0001, 0.75).set_name("conv2/norm2")) + model.add(SpatialMaxPooling(3, 3, 2, 2, to_ceil=True).set_name("pool2/3x3_s2")) + model.add(inception_layer_v1(192, t([t([64]), t( + [96, 128]), t([16, 32]), t([32])]), "inception_3a/")) + model.add(inception_layer_v1(256, t([t([128]), t( + [128, 192]), t([32, 96]), t([64])]), "inception_3b/")) + model.add(SpatialMaxPooling(3, 3, 2, 2, to_ceil=True)) + model.add(inception_layer_v1(480, t([t([192]), t( + [96, 208]), t([16, 48]), t([64])]), "inception_4a/")) + model.add(inception_layer_v1(512, t([t([160]), t( + [112, 224]), t([24, 64]), t([64])]), "inception_4b/")) + model.add(inception_layer_v1(512, t([t([128]), t( + [128, 256]), t([24, 64]), t([64])]), "inception_4c/")) + model.add(inception_layer_v1(512, t([t([112]), t( + [144, 288]), t([32, 64]), t([64])]), "inception_4d/")) + model.add(inception_layer_v1(528, t([t([256]), t( + [160, 320]), t([32, 128]), t([128])]), "inception_4e/")) + model.add(SpatialMaxPooling(3, 3, 2, 2, to_ceil=True)) + model.add(inception_layer_v1(832, t([t([256]), t( + [160, 320]), t([32, 128]), t([128])]), "inception_5a/")) + model.add(inception_layer_v1(832, t([t([384]), t( + [192, 384]), t([48, 128]), t([128])]), "inception_5b/")) + model.add(SpatialAveragePooling(7, 7, 1, 1).set_name("pool5/7x7_s1")) + if has_dropout: + model.add(Dropout(0.4).set_name("pool5/drop_7x7_s1")) + model.add(View([1024], num_input_dims=3)) + model.add(Linear(1024, class_num) + .set_init_method(weight_init_method=Xavier(), bias_init_method=Zeros()) + .set_name("loss3/classifier")) + model.add(LogSoftMax().set_name("loss3/loss3")) + model.reset() + return model + + if __name__ == "__main__": # parse options parser = config_option_parser() @@ -86,7 +184,7 @@ def get_inception_data(url, sc=None, data_type="train"): # load model snapshot inception_model = Model.load(options.model) else: - inception_model = inception.inception_v1_no_aux_classifier(options.classNum) + inception_model = inception_v1_no_aux_classifier(options.classNum) # set optimization method iterationPerEpoch = int(ceil(float(1281167) / options.batchSize)) From ca7e3dde29b3e83e3be82d55291c6624d0624ed3 Mon Sep 17 00:00:00 2001 From: YinaChen Date: Fri, 2 Aug 2019 16:00:24 +0800 Subject: [PATCH 10/28] change preprocess --- pyzoo/zoo/examples/inception/inception.py | 43 +++++++++++++++-------- 1 file changed, 29 insertions(+), 14 deletions(-) diff --git a/pyzoo/zoo/examples/inception/inception.py b/pyzoo/zoo/examples/inception/inception.py index cdad4c740a4..a4adc88b4b0 100644 --- a/pyzoo/zoo/examples/inception/inception.py +++ b/pyzoo/zoo/examples/inception/inception.py @@ -158,24 +158,39 @@ def inception_v1_no_aux_classifier(class_num, has_dropout=True): sc = init_nncontext("inception v1") image_size = 224 # create dataset - train_transformer = ChainedPreprocessing([ImagePixelBytesToMat(), - ImageRandomCrop(image_size, image_size), - ImageRandomPreprocessing(ImageHFlip(), 0.5), - ImageChannelNormalize(123.0, 117.0, 104.0), - ImageMatToTensor(format="NHWC", to_RGB=True), - ImageSetToSample(input_keys=["imageTensor"], target_keys=["label"]) - ]) + # train_transformer = ChainedPreprocessing([ImagePixelBytesToMat(), + # ImageRandomCrop(image_size, image_size), + # ImageRandomPreprocessing(ImageHFlip(), 0.5), + # ImageChannelNormalize(123.0, 117.0, 104.0), + # ImageMatToTensor(format="NHWC", to_RGB=True), + # ImageSetToSample(input_keys=["imageTensor"], target_keys=["label"]) + # ]) + train_transformer = Pipeline([PixelBytesToMat(), + Resize(256, 256), + RandomCropper(image_size, image_size, True, "Random", 3), + ChannelNormalize(123.0, 117.0, 104.0), + MatToTensor(to_rgb=False), + ImageFrameToSample(input_keys=["imageTensor"], target_keys=["label"]) + ]) raw_train_data = get_inception_data(options.folder, sc, "train") train_data = FeatureSet.image_frame(raw_train_data).transform(train_transformer) - val_transformer = ChainedPreprocessing([ImagePixelBytesToMat(), - ImageRandomCrop(image_size, image_size), - ImageRandomPreprocessing(ImageHFlip(), 0.5), - ImageChannelNormalize(123.0, 117.0, 104.0), - ImageMatToTensor(format="NHWC", to_RGB=True), - ImageSetToSample(input_keys=["imageTensor"], target_keys=["label"]) - ]) + # val_transformer = ChainedPreprocessing([ImagePixelBytesToMat(), + # ImageRandomCrop(image_size, image_size), + # ImageRandomPreprocessing(ImageHFlip(), 0.5), + # ImageChannelNormalize(123.0, 117.0, 104.0), + # ImageMatToTensor(format="NHWC", to_RGB=True), + # ImageSetToSample(input_keys=["imageTensor"], target_keys=["label"]) + # ]) + + val_transformer = Pipeline([PixelBytesToMat(), + Resize(256, 256), + RandomCropper(image_size, image_size, False, "Center", 3), + ChannelNormalize(123.0, 117.0, 104.0), + MatToTensor(to_rgb=False), + ImageFrameToSample(input_keys=["imageTensor"], target_keys=["label"]) + ]) raw_val_data = get_inception_data(options.folder, sc, "val") val_data = FeatureSet.image_frame(raw_val_data).transform(val_transformer) From 850e4768b9d7a8966cdb2f634dd476531f97c6d6 Mon Sep 17 00:00:00 2001 From: YinaChen Date: Mon, 5 Aug 2019 15:55:35 +0800 Subject: [PATCH 11/28] test --- pyzoo/zoo/examples/inception/inception.py | 57 ++++++++++++----------- 1 file changed, 29 insertions(+), 28 deletions(-) diff --git a/pyzoo/zoo/examples/inception/inception.py b/pyzoo/zoo/examples/inception/inception.py index a4adc88b4b0..8834f6e0219 100644 --- a/pyzoo/zoo/examples/inception/inception.py +++ b/pyzoo/zoo/examples/inception/inception.py @@ -158,39 +158,40 @@ def inception_v1_no_aux_classifier(class_num, has_dropout=True): sc = init_nncontext("inception v1") image_size = 224 # create dataset - # train_transformer = ChainedPreprocessing([ImagePixelBytesToMat(), - # ImageRandomCrop(image_size, image_size), - # ImageRandomPreprocessing(ImageHFlip(), 0.5), - # ImageChannelNormalize(123.0, 117.0, 104.0), - # ImageMatToTensor(format="NHWC", to_RGB=True), - # ImageSetToSample(input_keys=["imageTensor"], target_keys=["label"]) - # ]) - train_transformer = Pipeline([PixelBytesToMat(), - Resize(256, 256), - RandomCropper(image_size, image_size, True, "Random", 3), - ChannelNormalize(123.0, 117.0, 104.0), - MatToTensor(to_rgb=False), - ImageFrameToSample(input_keys=["imageTensor"], target_keys=["label"]) - ]) + train_transformer = ChainedPreprocessing([ImagePixelBytesToMat(), + ImageResize(256, 256), + ImageRandomCrop(image_size, image_size), + ImageRandomPreprocessing(ImageHFlip(), 0.5), + ImageChannelNormalize(123.0, 117.0, 104.0), + ImageMatToTensor(format="NHWC", to_RGB=True), + ImageSetToSample(input_keys=["imageTensor"], target_keys=["label"]) + ]) + # train_transformer = Pipeline([PixelBytesToMat(), + # Resize(256, 256), + # RandomCropper(image_size, image_size, True, "Random", 3), + # ChannelNormalize(123.0, 117.0, 104.0), + # MatToTensor(to_rgb=False), + # ImageFrameToSample(input_keys=["imageTensor"], target_keys=["label"]) + # ]) raw_train_data = get_inception_data(options.folder, sc, "train") train_data = FeatureSet.image_frame(raw_train_data).transform(train_transformer) - # val_transformer = ChainedPreprocessing([ImagePixelBytesToMat(), - # ImageRandomCrop(image_size, image_size), - # ImageRandomPreprocessing(ImageHFlip(), 0.5), - # ImageChannelNormalize(123.0, 117.0, 104.0), - # ImageMatToTensor(format="NHWC", to_RGB=True), - # ImageSetToSample(input_keys=["imageTensor"], target_keys=["label"]) - # ]) + val_transformer = ChainedPreprocessing([ImagePixelBytesToMat(), + ImageRandomCrop(image_size, image_size), + ImageRandomPreprocessing(ImageHFlip(), 0.5), + ImageChannelNormalize(123.0, 117.0, 104.0), + ImageMatToTensor(format="NHWC", to_RGB=True), + ImageSetToSample(input_keys=["imageTensor"], target_keys=["label"]) + ]) - val_transformer = Pipeline([PixelBytesToMat(), - Resize(256, 256), - RandomCropper(image_size, image_size, False, "Center", 3), - ChannelNormalize(123.0, 117.0, 104.0), - MatToTensor(to_rgb=False), - ImageFrameToSample(input_keys=["imageTensor"], target_keys=["label"]) - ]) + # val_transformer = Pipeline([PixelBytesToMat(), + # Resize(256, 256), + # RandomCropper(image_size, image_size, False, "Center", 3), + # ChannelNormalize(123.0, 117.0, 104.0), + # MatToTensor(to_rgb=False), + # ImageFrameToSample(input_keys=["imageTensor"], target_keys=["label"]) + # ]) raw_val_data = get_inception_data(options.folder, sc, "val") val_data = FeatureSet.image_frame(raw_val_data).transform(val_transformer) From dc2fcad513fc2f66ce4bc0fe1946e41f1a42eaa9 Mon Sep 17 00:00:00 2001 From: YinaChen Date: Tue, 6 Aug 2019 09:48:51 +0800 Subject: [PATCH 12/28] Add comments --- .../zoo/pipeline/estimator/test_estimator.py | 46 +----------- pyzoo/zoo/pipeline/estimator/estimator.py | 72 +++++++++++++++++++ 2 files changed, 75 insertions(+), 43 deletions(-) diff --git a/pyzoo/test/zoo/pipeline/estimator/test_estimator.py b/pyzoo/test/zoo/pipeline/estimator/test_estimator.py index 0a7563d7925..c955b08391e 100644 --- a/pyzoo/test/zoo/pipeline/estimator/test_estimator.py +++ b/pyzoo/test/zoo/pipeline/estimator/test_estimator.py @@ -15,23 +15,8 @@ # import pytest -import shutil -import errno -import string -from bigdl.nn.criterion import * -from bigdl.nn.layer import * -from bigdl.optim.optimizer import * -from numpy.testing import assert_allclose from pyspark.ml import Pipeline -from pyspark.ml.feature import MinMaxScaler -from pyspark.sql.types import * - -from zoo.common.nncontext import * from zoo.pipeline.estimator import * -from zoo.pipeline.api.keras.optimizers import Adam as KAdam -from zoo.feature.common import * -from zoo.feature.image import * -from zoo.util.tf import * from bigdl.nn.layer import * from bigdl.nn.criterion import * @@ -47,13 +32,10 @@ def setup_method(self, method): """ setup any state tied to the execution of the given method in a class. setup_method is invoked for every test method of a class. """ - # sparkConf = init_spark_conf().setMaster("local[1]").setAppName("testEstimator") - # self.sc = init_nncontext(sparkConf) - # self.sqlContext = SQLContext(self.sc) - # assert(self.sc.appName == "testEstimator") - sparkConf = init_spark_conf().setMaster("local[4]").setAppName("test feature set") + sparkConf = init_spark_conf().setMaster("local[1]").setAppName("testEstimator") self.sc = init_nncontext(sparkConf) - + self.sqlContext = SQLContext(self.sc) + assert(self.sc.appName == "testEstimator") def teardown_method(self, method): """ teardown any state that was previously setup with a setup_method @@ -97,28 +79,6 @@ def test_estimator_train_imagefeature(self): predict_result = model.predict_image(image_frame.transform(transformer)) assert(predict_result.get_predict().count(), 8) - # def test_estimator_train_rdd(self): - # dim = 2 - # data_len = 100 - # - # def gen_rand_sample(): - # features = np.random.uniform(0, 1, dim) - # label = np.array((2 * features).sum() + 0.4) - # return Sample.from_ndarray(features, label) - # - # data_set = FeatureSet.rdd(self.sc.parallelize(range(0, data_len)).map(lambda i: - # gen_rand_sample())) - # - # model = Sequential().set_name("mse") - # model.add(Linear(4, 4).set_name(("fc_1"))) - # model.add(Sigmoid()) - # model.add(Linear(4, 1).set_name("fc_2")) - # model.add(Sigmoid()) - # - # optim_method = SGD(learningrate=0.01) - # - # estimator = Estimator() - if __name__ == "__main__": pytest.main([__file__]) diff --git a/pyzoo/zoo/pipeline/estimator/estimator.py b/pyzoo/zoo/pipeline/estimator/estimator.py index 6c649c3ff11..b1e6e3df774 100644 --- a/pyzoo/zoo/pipeline/estimator/estimator.py +++ b/pyzoo/zoo/pipeline/estimator/estimator.py @@ -18,32 +18,104 @@ class Estimator(JavaValue): + """ + Estimator class for training and evaluation BigDL models. + + Estimator wraps a model, and provide an uniform training, evaluation or prediction operation on + both local host and distributed spark environment. + """ def __init__(self, model, optim_methods=None, model_dir=None, jvalue=None, bigdl_type="float"): self.bigdl_type = bigdl_type self.value = jvalue if jvalue else callBigDlFunc( bigdl_type, self.jvm_class_constructor(), model, optim_methods, model_dir) def clear_gradient_clipping(self): + """ + Clear gradient clipping parameters. In this case, gradient clipping will not be applied. + In order to take effect, it needs to be called before fit. + :return: + """ callBigDlFunc(self.bigdl_type, "clearGradientClipping") def set_constant_gradient_clipping(self, min, max): + """ + Set constant gradient clipping during the training process. + In order to take effect, it needs to be called before fit. + :param min: The minimum value to clip by. + :param max: The maximum value to clip by. + :return: + """ callBigDlFunc(self.bigdl_type, "setConstantGradientClipping", self.value, min, max) def set_l2_norm_gradient_clipping(self, clip_norm): + """ + Clip gradient to a maximum L2-Norm during the training process. + In order to take effect, it needs to be called before fit. + :param clip_norm: Gradient L2-Norm threshold. + :return: + """ callBigDlFunc(self.bigdl_type, "setGradientClippingByL2Norm", self.value, clip_norm) def train(self, train_set, criterion, end_trigger=None, checkpoint_trigger=None, validation_set=None, validation_method=None, batch_size=32): + """ + Train model with provided trainSet and criterion. + The training will end until the endTrigger is triggered. + During the training, if checkPointTrigger is defined and triggered, + the model will be saved to modelDir. And if validationSet and validationMethod + is defined, the model will be evaluated at the checkpoint. + :param train_set: training FeatureSet, a FeatureSet[Sample[T]] + :param criterion: Loss function + :param end_trigger: When to finish the training + :param checkpoint_trigger: When to save a checkpoint and evaluate model. + :param validation_set: Validation FeatureSet, a FeatureSet[Sample[T]] + :param validation_method: Validation Methods. + :param batch_size: + :return: Estimator + """ callBigDlFunc(self.bigdl_type, "estimatorTrain", self.value, train_set, criterion, end_trigger, checkpoint_trigger, validation_set, validation_method, batch_size) def train_imagefeature(self, train_set, criterion, end_trigger=None, checkpoint_trigger=None, validation_set=None, validation_method=None, batch_size=32): + """ + Train model with provided imageFeature trainSet and criterion. + The training will end until the endTrigger is triggered. + During the training, if checkPointTrigger is defined and triggered, + the model will be saved to modelDir. And if validationSet and validationMethod + is defined, the model will be evaluated at the checkpoint. + :param train_set: training FeatureSet, a FeatureSet[ImageFeature] + :param criterion: Loss function + :param end_trigger: When to finish the training + :param checkpoint_trigger: When to save a checkpoint and evaluate model. + :param validation_set: Validation FeatureSet, a FeatureSet[Sample[T]] + :param validation_method: Validation Methods. + :param batch_size: Batch size + :return: + """ callBigDlFunc(self.bigdl_type, "estimatorTrainImageFeature", self.value, train_set, criterion, end_trigger, checkpoint_trigger, validation_set, validation_method, batch_size) def evaluate(self, validation_set, validation_method, batch_size=32): + """ + Evaluate the model on the validationSet with the validationMethods. + :param validation_set: validation FeatureSet, a FeatureSet[Sample[T]] + :param validation_method: validation methods + :param batch_size: batch size + :return: validation results + """ callBigDlFunc(self.bigdl_type, "estimatorEvaluate", self.value, validation_set, validation_method, batch_size) + + def evaluate_imagefeature(self, validation_set, validation_method, batch_size=32): + """ + Evaluate the model on the validationSet with the validationMethods. + :param validation_set: validation FeatureSet, a FeatureSet[Sample[T]] + :param validation_method: validation methods + :param batch_size: batch size + :return: validation results + """ + callBigDlFunc(self.bigdl_type, "estimatorEvaluateImageFeature", self.value, + validation_set, validation_method, batch_size) From 1300bffcaa02f49c5d99fa75cdbc849463e41944 Mon Sep 17 00:00:00 2001 From: YinaChen Date: Wed, 7 Aug 2019 14:29:18 +0800 Subject: [PATCH 13/28] bug fix --- pyzoo/zoo/examples/inception/inception.py | 4 +- .../examples/tensorflow/inception/__init__.py | 0 .../tensorflow/inception/inception.py | 173 ------------------ 3 files changed, 2 insertions(+), 175 deletions(-) delete mode 100644 pyzoo/zoo/examples/tensorflow/inception/__init__.py delete mode 100644 pyzoo/zoo/examples/tensorflow/inception/inception.py diff --git a/pyzoo/zoo/examples/inception/inception.py b/pyzoo/zoo/examples/inception/inception.py index 8834f6e0219..c7aea4b113c 100644 --- a/pyzoo/zoo/examples/inception/inception.py +++ b/pyzoo/zoo/examples/inception/inception.py @@ -163,7 +163,7 @@ def inception_v1_no_aux_classifier(class_num, has_dropout=True): ImageRandomCrop(image_size, image_size), ImageRandomPreprocessing(ImageHFlip(), 0.5), ImageChannelNormalize(123.0, 117.0, 104.0), - ImageMatToTensor(format="NHWC", to_RGB=True), + ImageMatToTensor(format="NCHW", to_RGB=False), ImageSetToSample(input_keys=["imageTensor"], target_keys=["label"]) ]) # train_transformer = Pipeline([PixelBytesToMat(), @@ -181,7 +181,7 @@ def inception_v1_no_aux_classifier(class_num, has_dropout=True): ImageRandomCrop(image_size, image_size), ImageRandomPreprocessing(ImageHFlip(), 0.5), ImageChannelNormalize(123.0, 117.0, 104.0), - ImageMatToTensor(format="NHWC", to_RGB=True), + ImageMatToTensor(format="NCHW", to_RGB=True), ImageSetToSample(input_keys=["imageTensor"], target_keys=["label"]) ]) diff --git a/pyzoo/zoo/examples/tensorflow/inception/__init__.py b/pyzoo/zoo/examples/tensorflow/inception/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/pyzoo/zoo/examples/tensorflow/inception/inception.py b/pyzoo/zoo/examples/tensorflow/inception/inception.py deleted file mode 100644 index db3bb5553aa..00000000000 --- a/pyzoo/zoo/examples/tensorflow/inception/inception.py +++ /dev/null @@ -1,173 +0,0 @@ -from optparse import OptionParser -from math import ceil -from datetime import datetime - -from zoo.common.nncontext import * -from zoo.feature.image import * -from zoo.pipeline.api.net import * -from zoo.pipeline.api.keras.metrics import * -from zoo.pipeline.nnframes import * -from bigdl.optim.optimizer import * -from bigdl.util.common import * - -# import the inception model -import tensorflow as tf - -#sys.path.append("/root/manfei/models/research/slim/") # add the slim library -sys.path.append("/opt/work/models/research/slim") # add the slim library -from nets import inception_v1 - -slim = tf.contrib.slim - - -def t(input_t): - if type(input_t) is list: - # insert into index 0 spot, such that the real data starts from index 1 - temp = [0] - temp.extend(input_t) - return dict(enumerate(temp)) - # if dictionary, return it back - return input_t - - -def get_inception_data(url, sc=None, data_type="train"): - path = os.path.join(url, data_type) - return SeqFileFolder.files_to_image_frame(url=path, sc=sc, class_num=1000) - - -def config_option_parser(): - parser = OptionParser() - parser.add_option("-f", "--folder", type=str, dest="folder", default="", - help="url of hdf+s folder store the hadoop sequence files") - parser.add_option("--model", type=str, dest="model", default="", help="model snapshot location") - parser.add_option("--state", type=str, dest="state", default="", help="state snapshot location") - parser.add_option("--checkpoint", type=str, dest="checkpoint", default="", help="where to cache the model") - parser.add_option("-o", "--overwrite", action="store_true", dest="overwrite", default=False, - help="overwrite checkpoint files") - parser.add_option("-e", "--maxEpoch", type=int, dest="maxEpoch", default=0, help="epoch numbers") - parser.add_option("-i", "--maxIteration", type=int, dest="maxIteration", default=3100, help="iteration numbers") - parser.add_option("-l", "--learningRate", type=float, dest="learningRate", default=0.01, help="learning rate") - parser.add_option("--warmupEpoch", type=int, dest="warmupEpoch", default=0, help="warm up epoch numbers") - parser.add_option("--maxLr", type=float, dest="maxLr", default=0.0, help="max Lr after warm up") - parser.add_option("-b", "--batchSize", type=int, dest="batchSize", help="batch size") - parser.add_option("--classNum", type=int, dest="classNum", default=1000, help="class number") - parser.add_option("--weightDecay", type=float, dest="weightDecay", default=0.0001, help="weight decay") - parser.add_option("--checkpointIteration", type=int, dest="checkpointIteration", default=620, - help="checkpoint interval of iterations") - parser.add_option("--gradientMin", type=float, dest="gradientMin", default=0.0, help="min gradient clipping by") - parser.add_option("--gradientMax", type=float, dest="gradientMax", default=0.0, help="max gradient clipping by") - parser.add_option("--gradientL2NormThreshold", type=float, dest="gradientL2NormThreshold", default=0.0, - help="gradient L2-Norm threshold") - parser.add_option("--memoryType", type=str, dest="memoryType", default="DRAM", help="memory storage type, DRAM or PMEM") - - return parser - - -if __name__ == "__main__": - # parse options - parser = config_option_parser() - (options, args) = parser.parse_args(sys.argv) - - if not options.learningRate: - parser.error("-l --learningRate is a mandatory opt") - if not options.batchSize: - parser.error("-b --batchSize is a mandatory opt") - - ## init - sc = init_nncontext("inception v1") - - image_size = 224 # create dataset - train_transformer = ChainedPreprocessing([ImagePixelBytesToMat(), - ImageRandomCrop(image_size, image_size), - ImageRandomPreprocessing(ImageHFlip(), 0.5), - ImageChannelNormalize(0.485, 0.456, 0.406, 0.229, 0.224, 0.225), - ImageMatToTensor(format="NHWC", to_RGB=True), - ImageSetToSample(input_keys=["imageTensor"], target_keys=["label"]) - ]) - raw_train_data = get_inception_data(options.folder, sc, "train") - train_data = FeatureSet.image_frame(raw_train_data, options.memoryType).transform(train_transformer) - - val_transformer = ChainedPreprocessing([ImagePixelBytesToMat(), - ImageCenterCrop(image_size, image_size), - ImageRandomPreprocessing(ImageHFlip(), 0.5), - ImageChannelNormalize(0.485, 0.456, 0.406, 0.229, 0.224, 0.225), - ImageMatToTensor(format="NHWC", to_RGB=True), - ImageSetToSample(input_keys=["imageTensor"], target_keys=["label"]), - ]) - raw_val_data = get_inception_data(options.folder, sc, "val") - val_data = FeatureSet.image_frame(raw_val_data).transform(val_transformer) - - dataset = TFDataset.from_feature_set(train_data, - features=(tf.float32, [224, 224, 3]), - labels=(tf.int32, [1]), - batch_size=options.batchSize, - validation_dataset=val_data) - - # construct the model from TFDataset - images, labels = dataset.tensors - - with slim.arg_scope(inception_v1.inception_v1_arg_scope()): - logits, end_points = inception_v1.inception_v1(images, - dropout_keep_prob=0.6, - num_classes=1000, - is_training=True) - # As sequence file's label is one-based, so labels need to subtract 1. - loss = tf.reduce_mean(tf.losses.sparse_softmax_cross_entropy(logits=logits, labels=labels - 1)) - - # set optimization method - iterationPerEpoch = int(ceil(float(1281167) / options.batchSize)) - if options.maxEpoch: - maxIteration = iterationPerEpoch * options.maxEpoch - else: - maxIteration = options.maxIteration - warmup_iteration = options.warmupEpoch * iterationPerEpoch - if options.state != "": - # load state snapshot - optim = OptimMethod.load(options.state) - else: - if warmup_iteration == 0: - warmupDelta = 0.0 - else: - if options.maxLr: - maxlr = options.maxLr - else: - maxlr = options.learningRate - warmupDelta = (maxlr - options.learningRate) / warmup_iteration - polyIteration = maxIteration - warmup_iteration - lrSchedule = SequentialSchedule(iterationPerEpoch) - lrSchedule.add(Warmup(warmupDelta), warmup_iteration) - lrSchedule.add(Poly(0.5, polyIteration), polyIteration) - ## can we use SGD in this way? - optim = SGD(learningrate=options.learningRate, learningrate_decay=0.16, - weightdecay=options.weightDecay, - momentum=0.9, dampening=0.0, nesterov=False, - leaningrate_schedule=lrSchedule) - - ## where to find the same function - # create triggers - if options.maxEpoch: - checkpoint_trigger = EveryEpoch() - test_trigger = EveryEpoch() - end_trigger = MaxEpoch(options.maxEpoch) - else: - checkpoint_trigger = SeveralIteration(options.checkpointIteration) - test_trigger = SeveralIteration(options.checkpointIteration) - end_trigger = MaxIteration(options.maxIteration) - - timeStamp = datetime.now().strftime("%Y%m%d_%H%M%S") - - optimizer = TFOptimizer(loss, optim, val_outputs=[logits], val_labels=[labels-1], - val_method=[Accuracy(), Top5Accuracy()]) - train_summary = TrainSummary("/tmp/logs/inceptionV1", "inceptionV1" + timeStamp) - train_summary.set_summary_trigger("Parameters", SeveralIteration(10)) - optimizer.set_train_summary(train_summary) - optimizer.set_val_summary(ValidationSummary("/tmp/logs/inceptionV1", "inceptionV1" + timeStamp)) - if options.checkpoint != "": - optimizer.set_checkpoint(options.checkpoint + timeStamp, options.overwrite) - - optimizer.optimize(end_trigger=end_trigger, checkpoint_trigger=checkpoint_trigger) - - saver = tf.train.Saver() - saver.save(optimizer.sess, "/root/manfei/model") - - sc.stop() From 5d775023ab581151cdbc2ef8108ec2b5f543c3b0 Mon Sep 17 00:00:00 2001 From: YinaChen Date: Mon, 12 Aug 2019 14:39:06 +0800 Subject: [PATCH 14/28] Change validation preprocess method --- pyzoo/zoo/examples/inception/inception.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pyzoo/zoo/examples/inception/inception.py b/pyzoo/zoo/examples/inception/inception.py index c7aea4b113c..9d7e7b4c8e9 100644 --- a/pyzoo/zoo/examples/inception/inception.py +++ b/pyzoo/zoo/examples/inception/inception.py @@ -178,8 +178,7 @@ def inception_v1_no_aux_classifier(class_num, has_dropout=True): train_data = FeatureSet.image_frame(raw_train_data).transform(train_transformer) val_transformer = ChainedPreprocessing([ImagePixelBytesToMat(), - ImageRandomCrop(image_size, image_size), - ImageRandomPreprocessing(ImageHFlip(), 0.5), + ImageCenterCrop(image_size, image_size), ImageChannelNormalize(123.0, 117.0, 104.0), ImageMatToTensor(format="NCHW", to_RGB=True), ImageSetToSample(input_keys=["imageTensor"], target_keys=["label"]) From f3e7e364271de51057d4d4b32c8ac650bb9a616d Mon Sep 17 00:00:00 2001 From: YinaChen Date: Mon, 12 Aug 2019 14:50:29 +0800 Subject: [PATCH 15/28] Bug fix --- pyzoo/zoo/examples/inception/inception.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyzoo/zoo/examples/inception/inception.py b/pyzoo/zoo/examples/inception/inception.py index 9d7e7b4c8e9..95480d9bb4d 100644 --- a/pyzoo/zoo/examples/inception/inception.py +++ b/pyzoo/zoo/examples/inception/inception.py @@ -180,7 +180,7 @@ def inception_v1_no_aux_classifier(class_num, has_dropout=True): val_transformer = ChainedPreprocessing([ImagePixelBytesToMat(), ImageCenterCrop(image_size, image_size), ImageChannelNormalize(123.0, 117.0, 104.0), - ImageMatToTensor(format="NCHW", to_RGB=True), + ImageMatToTensor(format="NCHW", to_RGB=False), ImageSetToSample(input_keys=["imageTensor"], target_keys=["label"]) ]) From 342ec82de2d8d5388200e1aac0baa047bfcbfca6 Mon Sep 17 00:00:00 2001 From: YinaChen Date: Thu, 22 Aug 2019 11:02:21 +0800 Subject: [PATCH 16/28] add imagemirror --- pyzoo/zoo/examples/inception/inception.py | 2 +- pyzoo/zoo/feature/image/imagePreprocessing.py | 6 ++++ .../zoo/feature/image/ImageMirror.scala | 32 +++++++++++++++++++ .../feature/python/PythonImageFeature.scala | 4 +++ 4 files changed, 43 insertions(+), 1 deletion(-) create mode 100644 zoo/src/main/scala/com/intel/analytics/zoo/feature/image/ImageMirror.scala diff --git a/pyzoo/zoo/examples/inception/inception.py b/pyzoo/zoo/examples/inception/inception.py index 95480d9bb4d..00f2a865c20 100644 --- a/pyzoo/zoo/examples/inception/inception.py +++ b/pyzoo/zoo/examples/inception/inception.py @@ -161,7 +161,7 @@ def inception_v1_no_aux_classifier(class_num, has_dropout=True): train_transformer = ChainedPreprocessing([ImagePixelBytesToMat(), ImageResize(256, 256), ImageRandomCrop(image_size, image_size), - ImageRandomPreprocessing(ImageHFlip(), 0.5), + ImageRandomPreprocessing(ImageMirror(), 0.5), ImageChannelNormalize(123.0, 117.0, 104.0), ImageMatToTensor(format="NCHW", to_RGB=False), ImageSetToSample(input_keys=["imageTensor"], target_keys=["label"]) diff --git a/pyzoo/zoo/feature/image/imagePreprocessing.py b/pyzoo/zoo/feature/image/imagePreprocessing.py index 58013db62fa..62e54d0b8fd 100644 --- a/pyzoo/zoo/feature/image/imagePreprocessing.py +++ b/pyzoo/zoo/feature/image/imagePreprocessing.py @@ -339,6 +339,12 @@ def __init__(self, bigdl_type="float"): super(ImageHFlip, self).__init__(bigdl_type) +class ImageMirror(ImagePreprocessing): + + def __init__(self, bigdl_type="float"): + super(ImageMirror, self).__init__(bigdl_type) + + class ImageFeatureToTensor(Preprocessing): """ a Transformer that convert ImageFeature to a Tensor. diff --git a/zoo/src/main/scala/com/intel/analytics/zoo/feature/image/ImageMirror.scala b/zoo/src/main/scala/com/intel/analytics/zoo/feature/image/ImageMirror.scala new file mode 100644 index 00000000000..69c823b9a6b --- /dev/null +++ b/zoo/src/main/scala/com/intel/analytics/zoo/feature/image/ImageMirror.scala @@ -0,0 +1,32 @@ +package com.intel.analytics.zoo.feature.image + +import com.intel.analytics.bigdl.transform.vision.image.opencv.OpenCVMat +import com.intel.analytics.bigdl.transform.vision.image.{FeatureTransformer, ImageFeature} +import org.opencv.core.Core + +class Mirror() extends FeatureTransformer { + override def transformMat(feature: ImageFeature): Unit = { + Mirror.transform(feature.opencvMat(), feature.opencvMat()) + } +} + +object Mirror { + def apply(): Mirror = new Mirror() + + def transform(input: OpenCVMat, output: OpenCVMat): OpenCVMat ={ + Core.flip(input, output, -1) + output + } +} + +class ImageMirror() extends ImageProcessing { + private val internalCrop = new Mirror + + override def apply(prev: Iterator[ImageFeature]): Iterator[ImageFeature] = { + internalCrop.apply(prev) + } +} + +object ImageMirror { + def apply(): ImageMirror = new ImageMirror() +} diff --git a/zoo/src/main/scala/com/intel/analytics/zoo/feature/python/PythonImageFeature.scala b/zoo/src/main/scala/com/intel/analytics/zoo/feature/python/PythonImageFeature.scala index aa1fb860a70..886c1087cab 100644 --- a/zoo/src/main/scala/com/intel/analytics/zoo/feature/python/PythonImageFeature.scala +++ b/zoo/src/main/scala/com/intel/analytics/zoo/feature/python/PythonImageFeature.scala @@ -356,6 +356,10 @@ class PythonImageFeature[T: ClassTag](implicit ev: TensorNumeric[T]) extends Pyt ImageHFlip() } + def createImageMirror(): ImageMirror = { + ImageMirror() + } + def createImageSetToSample(inputKeys: JList[String], targetKeys: JList[String], sampleKey: String): ImageSetToSample[T] = { From e671e13157df3a12c57d7bdc40c3fcb4b16a8227 Mon Sep 17 00:00:00 2001 From: YinaChen Date: Tue, 27 Aug 2019 15:00:55 +0800 Subject: [PATCH 17/28] add random cropper --- pyzoo/zoo/__init__.py | 2 +- pyzoo/zoo/examples/inception/inception.py | 18 +++++++++--------- pyzoo/zoo/feature/image/imagePreprocessing.py | 19 +++++++++++++++++++ .../feature/python/PythonImageFeature.scala | 12 ++++++++++++ 4 files changed, 41 insertions(+), 10 deletions(-) diff --git a/pyzoo/zoo/__init__.py b/pyzoo/zoo/__init__.py index 60225f10248..0874210a247 100644 --- a/pyzoo/zoo/__init__.py +++ b/pyzoo/zoo/__init__.py @@ -33,4 +33,4 @@ for clz in creator_classes: JavaCreator.add_creator_class(clz) -__version__ = "0.6.0.dev0" +__version__ = "0.6.0.dev14" diff --git a/pyzoo/zoo/examples/inception/inception.py b/pyzoo/zoo/examples/inception/inception.py index 00f2a865c20..5caa360ad74 100644 --- a/pyzoo/zoo/examples/inception/inception.py +++ b/pyzoo/zoo/examples/inception/inception.py @@ -160,19 +160,19 @@ def inception_v1_no_aux_classifier(class_num, has_dropout=True): image_size = 224 # create dataset train_transformer = ChainedPreprocessing([ImagePixelBytesToMat(), ImageResize(256, 256), - ImageRandomCrop(image_size, image_size), - ImageRandomPreprocessing(ImageMirror(), 0.5), + ImageRandomCropper(image_size, image_size, True, + "Random", 3), ImageChannelNormalize(123.0, 117.0, 104.0), ImageMatToTensor(format="NCHW", to_RGB=False), ImageSetToSample(input_keys=["imageTensor"], target_keys=["label"]) ]) - # train_transformer = Pipeline([PixelBytesToMat(), - # Resize(256, 256), - # RandomCropper(image_size, image_size, True, "Random", 3), - # ChannelNormalize(123.0, 117.0, 104.0), - # MatToTensor(to_rgb=False), - # ImageFrameToSample(input_keys=["imageTensor"], target_keys=["label"]) - # ]) + train_transformer = Pipeline([PixelBytesToMat(), + Resize(256, 256), + RandomCropper(image_size, image_size, True, "Random", 3), + ChannelNormalize(123.0, 117.0, 104.0), + MatToTensor(to_rgb=False), + ImageFrameToSample(input_keys=["imageTensor"], target_keys=["label"]) + ]) raw_train_data = get_inception_data(options.folder, sc, "train") train_data = FeatureSet.image_frame(raw_train_data).transform(train_transformer) diff --git a/pyzoo/zoo/feature/image/imagePreprocessing.py b/pyzoo/zoo/feature/image/imagePreprocessing.py index 62e54d0b8fd..7d7c54a23e8 100644 --- a/pyzoo/zoo/feature/image/imagePreprocessing.py +++ b/pyzoo/zoo/feature/image/imagePreprocessing.py @@ -249,6 +249,25 @@ class ImagePixelNormalize(ImagePreprocessing): def __init__(self, means, bigdl_type="float"): super(ImagePixelNormalize, self).__init__(bigdl_type, means) + # cropWidth: Int, cropHeight: Int, + # mirror: Boolean, cropperMethod: String, + # channels: Int) + + +class ImageRandomCropper(ImagePreprocessing): + """ + Random cropper on uniform distribution with fixed height & width + :param crop_w width cropped to + :param crop_h height cropped to + :param mirror whether mirror + :param cropper_method crop method + :param channels total channels + """ + + def __init__(self, crop_width, crop_height, mirror, cropper_method, channels, + bigdl_type="float"): + super(ImageRandomCropper, self).__init__(bigdl_type, crop_width, crop_height, mirror, + cropper_method, channels) class ImageRandomCrop(ImagePreprocessing): diff --git a/zoo/src/main/scala/com/intel/analytics/zoo/feature/python/PythonImageFeature.scala b/zoo/src/main/scala/com/intel/analytics/zoo/feature/python/PythonImageFeature.scala index 886c1087cab..67058dcddaf 100644 --- a/zoo/src/main/scala/com/intel/analytics/zoo/feature/python/PythonImageFeature.scala +++ b/zoo/src/main/scala/com/intel/analytics/zoo/feature/python/PythonImageFeature.scala @@ -19,6 +19,7 @@ package com.intel.analytics.zoo.feature.python import java.util import java.util.{List => JList} +import com.intel.analytics.bigdl.dataset.image.{CropCenter, CropRandom} import com.intel.analytics.bigdl.nn.abstractnn.DataFormat import com.intel.analytics.bigdl.python.api.JTensor import com.intel.analytics.bigdl.tensor.{Storage, Tensor} @@ -327,6 +328,17 @@ class PythonImageFeature[T: ClassTag](implicit ev: TensorNumeric[T]) extends Pyt ImageRandomPreprocessing(preprocessing, prob) } + def createImageRandomCropper(cropWidth: Int, cropHeight: Int, mirror: Boolean, + cropperMethod: String, channels: Int): ImageRandomCropper = { + if (cropperMethod == "Random") { + ImageRandomCropper(cropWidth, cropHeight, mirror, + CropRandom, channels) + } else { + ImageRandomCropper(cropWidth, cropHeight, mirror, + CropCenter, channels) + } + } + def createImageRandomCrop(cropWidth: Int, cropHeight: Int, isClip: Boolean): ImageRandomCrop = { ImageRandomCrop(cropWidth, cropHeight, isClip) } From 917edf061b6799837f45457cdf13a4bb5690f0b1 Mon Sep 17 00:00:00 2001 From: YinaChen Date: Tue, 27 Aug 2019 15:03:05 +0800 Subject: [PATCH 18/28] remove bigdl preprocessing --- pyzoo/zoo/examples/inception/inception.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/pyzoo/zoo/examples/inception/inception.py b/pyzoo/zoo/examples/inception/inception.py index 5caa360ad74..0de286af690 100644 --- a/pyzoo/zoo/examples/inception/inception.py +++ b/pyzoo/zoo/examples/inception/inception.py @@ -166,13 +166,13 @@ def inception_v1_no_aux_classifier(class_num, has_dropout=True): ImageMatToTensor(format="NCHW", to_RGB=False), ImageSetToSample(input_keys=["imageTensor"], target_keys=["label"]) ]) - train_transformer = Pipeline([PixelBytesToMat(), - Resize(256, 256), - RandomCropper(image_size, image_size, True, "Random", 3), - ChannelNormalize(123.0, 117.0, 104.0), - MatToTensor(to_rgb=False), - ImageFrameToSample(input_keys=["imageTensor"], target_keys=["label"]) - ]) + # train_transformer = Pipeline([PixelBytesToMat(), + # Resize(256, 256), + # RandomCropper(image_size, image_size, True, "Random", 3), + # ChannelNormalize(123.0, 117.0, 104.0), + # MatToTensor(to_rgb=False), + # ImageFrameToSample(input_keys=["imageTensor"], target_keys=["label"]) + # ]) raw_train_data = get_inception_data(options.folder, sc, "train") train_data = FeatureSet.image_frame(raw_train_data).transform(train_transformer) From e1c3ba1bc7499ba0d9a7a39b1414aa4af44815aa Mon Sep 17 00:00:00 2001 From: YinaChen Date: Fri, 30 Aug 2019 16:12:52 +0800 Subject: [PATCH 19/28] hflip --- pyzoo/zoo/examples/inception/inception.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyzoo/zoo/examples/inception/inception.py b/pyzoo/zoo/examples/inception/inception.py index 0de286af690..95480d9bb4d 100644 --- a/pyzoo/zoo/examples/inception/inception.py +++ b/pyzoo/zoo/examples/inception/inception.py @@ -160,8 +160,8 @@ def inception_v1_no_aux_classifier(class_num, has_dropout=True): image_size = 224 # create dataset train_transformer = ChainedPreprocessing([ImagePixelBytesToMat(), ImageResize(256, 256), - ImageRandomCropper(image_size, image_size, True, - "Random", 3), + ImageRandomCrop(image_size, image_size), + ImageRandomPreprocessing(ImageHFlip(), 0.5), ImageChannelNormalize(123.0, 117.0, 104.0), ImageMatToTensor(format="NCHW", to_RGB=False), ImageSetToSample(input_keys=["imageTensor"], target_keys=["label"]) From 25d79a075f297a5bdffbe055b7a2363eeac20da2 Mon Sep 17 00:00:00 2001 From: YinaChen Date: Fri, 30 Aug 2019 16:13:48 +0800 Subject: [PATCH 20/28] validation resize --- pyzoo/zoo/examples/inception/inception.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pyzoo/zoo/examples/inception/inception.py b/pyzoo/zoo/examples/inception/inception.py index 95480d9bb4d..f863134a001 100644 --- a/pyzoo/zoo/examples/inception/inception.py +++ b/pyzoo/zoo/examples/inception/inception.py @@ -178,6 +178,7 @@ def inception_v1_no_aux_classifier(class_num, has_dropout=True): train_data = FeatureSet.image_frame(raw_train_data).transform(train_transformer) val_transformer = ChainedPreprocessing([ImagePixelBytesToMat(), + ImageResize(256, 256), ImageCenterCrop(image_size, image_size), ImageChannelNormalize(123.0, 117.0, 104.0), ImageMatToTensor(format="NCHW", to_RGB=False), From fbbf24ce1dac2c2ac229952f3d6a46748984cdbf Mon Sep 17 00:00:00 2001 From: YinaChen Date: Mon, 2 Sep 2019 10:11:06 +0800 Subject: [PATCH 21/28] revert some changes --- pyzoo/zoo/examples/inception/inception.py | 14 ----------- pyzoo/zoo/feature/image/imagePreprocessing.py | 23 +++---------------- pyzoo/zoo/pipeline/api/net/tf_optimizer.py | 10 ++------ .../zoo/feature/image/ImageMirror.scala | 18 +++++++++++++++ .../feature/python/PythonImageFeature.scala | 12 ---------- 5 files changed, 23 insertions(+), 54 deletions(-) diff --git a/pyzoo/zoo/examples/inception/inception.py b/pyzoo/zoo/examples/inception/inception.py index f863134a001..f2e08bb3359 100644 --- a/pyzoo/zoo/examples/inception/inception.py +++ b/pyzoo/zoo/examples/inception/inception.py @@ -166,13 +166,6 @@ def inception_v1_no_aux_classifier(class_num, has_dropout=True): ImageMatToTensor(format="NCHW", to_RGB=False), ImageSetToSample(input_keys=["imageTensor"], target_keys=["label"]) ]) - # train_transformer = Pipeline([PixelBytesToMat(), - # Resize(256, 256), - # RandomCropper(image_size, image_size, True, "Random", 3), - # ChannelNormalize(123.0, 117.0, 104.0), - # MatToTensor(to_rgb=False), - # ImageFrameToSample(input_keys=["imageTensor"], target_keys=["label"]) - # ]) raw_train_data = get_inception_data(options.folder, sc, "train") train_data = FeatureSet.image_frame(raw_train_data).transform(train_transformer) @@ -185,13 +178,6 @@ def inception_v1_no_aux_classifier(class_num, has_dropout=True): ImageSetToSample(input_keys=["imageTensor"], target_keys=["label"]) ]) - # val_transformer = Pipeline([PixelBytesToMat(), - # Resize(256, 256), - # RandomCropper(image_size, image_size, False, "Center", 3), - # ChannelNormalize(123.0, 117.0, 104.0), - # MatToTensor(to_rgb=False), - # ImageFrameToSample(input_keys=["imageTensor"], target_keys=["label"]) - # ]) raw_val_data = get_inception_data(options.folder, sc, "val") val_data = FeatureSet.image_frame(raw_val_data).transform(val_transformer) diff --git a/pyzoo/zoo/feature/image/imagePreprocessing.py b/pyzoo/zoo/feature/image/imagePreprocessing.py index 7d7c54a23e8..38b07620ec8 100644 --- a/pyzoo/zoo/feature/image/imagePreprocessing.py +++ b/pyzoo/zoo/feature/image/imagePreprocessing.py @@ -249,25 +249,6 @@ class ImagePixelNormalize(ImagePreprocessing): def __init__(self, means, bigdl_type="float"): super(ImagePixelNormalize, self).__init__(bigdl_type, means) - # cropWidth: Int, cropHeight: Int, - # mirror: Boolean, cropperMethod: String, - # channels: Int) - - -class ImageRandomCropper(ImagePreprocessing): - """ - Random cropper on uniform distribution with fixed height & width - :param crop_w width cropped to - :param crop_h height cropped to - :param mirror whether mirror - :param cropper_method crop method - :param channels total channels - """ - - def __init__(self, crop_width, crop_height, mirror, cropper_method, channels, - bigdl_type="float"): - super(ImageRandomCropper, self).__init__(bigdl_type, crop_width, crop_height, mirror, - cropper_method, channels) class ImageRandomCrop(ImagePreprocessing): @@ -359,7 +340,9 @@ def __init__(self, bigdl_type="float"): class ImageMirror(ImagePreprocessing): - + """ + Flip the image horizontally and vertically + """ def __init__(self, bigdl_type="float"): super(ImageMirror, self).__init__(bigdl_type) diff --git a/pyzoo/zoo/pipeline/api/net/tf_optimizer.py b/pyzoo/zoo/pipeline/api/net/tf_optimizer.py index 34f5dadc861..42cf7aba309 100644 --- a/pyzoo/zoo/pipeline/api/net/tf_optimizer.py +++ b/pyzoo/zoo/pipeline/api/net/tf_optimizer.py @@ -427,10 +427,6 @@ def set_train_summary(self, summary): def set_val_summary(self, summary): self.optimizer.set_val_summary(summary) - def set_checkpoint(self, checkpointDir, overwrite): - self.checkPointDir = checkpointDir - self.isOverwrite = overwrite - def set_constant_gradient_clipping(self, min_value, max_value): """ Configure constant clipping settings. @@ -450,14 +446,12 @@ def set_gradient_clipping_by_l2_norm(self, clip_norm): """ self.optimizer.set_gradclip_l2norm(clip_norm) - def optimize(self, end_trigger=None, checkpoint_trigger=None): + def optimize(self, end_trigger=None): if end_trigger is None: end_trigger = MaxEpoch(1) self.optimizer.set_end_when(end_trigger) - if self.checkPointDir and checkpoint_trigger: - self.optimizer.set_checkpoint(checkpoint_trigger, checkpoint_path=self.checkPointDir,\ - isOverWrite=self.isOverwrite) + self.optimizer.optimize() variables = self.training_helper_layer.get_weights() diff --git a/zoo/src/main/scala/com/intel/analytics/zoo/feature/image/ImageMirror.scala b/zoo/src/main/scala/com/intel/analytics/zoo/feature/image/ImageMirror.scala index 69c823b9a6b..c82e93a6921 100644 --- a/zoo/src/main/scala/com/intel/analytics/zoo/feature/image/ImageMirror.scala +++ b/zoo/src/main/scala/com/intel/analytics/zoo/feature/image/ImageMirror.scala @@ -1,3 +1,18 @@ +/* + * Copyright 2018 Analytics Zoo Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.intel.analytics.zoo.feature.image import com.intel.analytics.bigdl.transform.vision.image.opencv.OpenCVMat @@ -19,6 +34,9 @@ object Mirror { } } +/** + * Flip the image horizontally and vertically + */ class ImageMirror() extends ImageProcessing { private val internalCrop = new Mirror diff --git a/zoo/src/main/scala/com/intel/analytics/zoo/feature/python/PythonImageFeature.scala b/zoo/src/main/scala/com/intel/analytics/zoo/feature/python/PythonImageFeature.scala index 67058dcddaf..886c1087cab 100644 --- a/zoo/src/main/scala/com/intel/analytics/zoo/feature/python/PythonImageFeature.scala +++ b/zoo/src/main/scala/com/intel/analytics/zoo/feature/python/PythonImageFeature.scala @@ -19,7 +19,6 @@ package com.intel.analytics.zoo.feature.python import java.util import java.util.{List => JList} -import com.intel.analytics.bigdl.dataset.image.{CropCenter, CropRandom} import com.intel.analytics.bigdl.nn.abstractnn.DataFormat import com.intel.analytics.bigdl.python.api.JTensor import com.intel.analytics.bigdl.tensor.{Storage, Tensor} @@ -328,17 +327,6 @@ class PythonImageFeature[T: ClassTag](implicit ev: TensorNumeric[T]) extends Pyt ImageRandomPreprocessing(preprocessing, prob) } - def createImageRandomCropper(cropWidth: Int, cropHeight: Int, mirror: Boolean, - cropperMethod: String, channels: Int): ImageRandomCropper = { - if (cropperMethod == "Random") { - ImageRandomCropper(cropWidth, cropHeight, mirror, - CropRandom, channels) - } else { - ImageRandomCropper(cropWidth, cropHeight, mirror, - CropCenter, channels) - } - } - def createImageRandomCrop(cropWidth: Int, cropHeight: Int, isClip: Boolean): ImageRandomCrop = { ImageRandomCrop(cropWidth, cropHeight, isClip) } From 94af40d4ee67c3f4da000f7f0817869c8acc5bbf Mon Sep 17 00:00:00 2001 From: YinaChen Date: Mon, 2 Sep 2019 10:18:39 +0800 Subject: [PATCH 22/28] fix --- .../zoo/pipeline/estimator/test_estimator.py | 31 ++----------------- .../zoo/feature/python/PythonFeatureSet.scala | 3 +- 2 files changed, 4 insertions(+), 30 deletions(-) diff --git a/pyzoo/test/zoo/pipeline/estimator/test_estimator.py b/pyzoo/test/zoo/pipeline/estimator/test_estimator.py index ae6d9659497..08652b10e1c 100644 --- a/pyzoo/test/zoo/pipeline/estimator/test_estimator.py +++ b/pyzoo/test/zoo/pipeline/estimator/test_estimator.py @@ -33,13 +33,10 @@ def setup_method(self, method): """ setup any state tied to the execution of the given method in a class. setup_method is invoked for every test method of a class. """ - # sparkConf = init_spark_conf().setMaster("local[1]").setAppName("testEstimator") - # self.sc = init_nncontext(sparkConf) - # self.sqlContext = SQLContext(self.sc) - # assert(self.sc.appName == "testEstimator") - sparkConf = init_spark_conf().setMaster("local[4]").setAppName("test feature set") + sparkConf = init_spark_conf().setMaster("local[1]").setAppName("testEstimator") self.sc = init_nncontext(sparkConf) - + self.sqlContext = SQLContext(self.sc) + assert(self.sc.appName == "testEstimator") def teardown_method(self, method): """ teardown any state that was previously setup with a setup_method @@ -83,28 +80,6 @@ def test_estimator_train_imagefeature(self): predict_result = model.predict_image(image_frame.transform(transformer)) assert(predict_result.get_predict().count(), 8) - # def test_estimator_train_rdd(self): - # dim = 2 - # data_len = 100 - # - # def gen_rand_sample(): - # features = np.random.uniform(0, 1, dim) - # label = np.array((2 * features).sum() + 0.4) - # return Sample.from_ndarray(features, label) - # - # data_set = FeatureSet.rdd(self.sc.parallelize(range(0, data_len)).map(lambda i: - # gen_rand_sample())) - # - # model = Sequential().set_name("mse") - # model.add(Linear(4, 4).set_name(("fc_1"))) - # model.add(Sigmoid()) - # model.add(Linear(4, 1).set_name("fc_2")) - # model.add(Sigmoid()) - # - # optim_method = SGD(learningrate=0.01) - # - # estimator = Estimator() - if __name__ == "__main__": pytest.main([__file__]) diff --git a/zoo/src/main/scala/com/intel/analytics/zoo/feature/python/PythonFeatureSet.scala b/zoo/src/main/scala/com/intel/analytics/zoo/feature/python/PythonFeatureSet.scala index 65616ea89ab..fb3e0fd39e9 100644 --- a/zoo/src/main/scala/com/intel/analytics/zoo/feature/python/PythonFeatureSet.scala +++ b/zoo/src/main/scala/com/intel/analytics/zoo/feature/python/PythonFeatureSet.scala @@ -17,13 +17,12 @@ package com.intel.analytics.zoo.feature.python import com.intel.analytics.bigdl.DataSet -import com.intel.analytics.bigdl.dataset.{SampleToMiniBatch, Transformer, Sample => JSample} +import com.intel.analytics.bigdl.dataset.{Transformer, Sample => JSample} import com.intel.analytics.bigdl.python.api.Sample import com.intel.analytics.bigdl.tensor.TensorNumericMath.TensorNumeric import com.intel.analytics.bigdl.transform.vision.image._ import com.intel.analytics.zoo.common.PythonZoo import com.intel.analytics.zoo.feature.FeatureSet -import com.intel.analytics.zoo.feature.image.{ImageChannelNormalize, ImageFeatureToSample, ImageMatToTensor, ImagePixelBytesToMat, ImageRandomCrop, ImageSetToSample} import com.intel.analytics.zoo.feature.pmem.MemoryType import org.apache.spark.api.java.JavaRDD From fc57b32c764e175de9c03adf73b73e3561282a48 Mon Sep 17 00:00:00 2001 From: YinaChen Date: Mon, 2 Sep 2019 15:44:18 +0800 Subject: [PATCH 23/28] Add readme --- pyzoo/zoo/examples/inception/README.md | 109 +++++++++++++++++++++++++ 1 file changed, 109 insertions(+) create mode 100644 pyzoo/zoo/examples/inception/README.md diff --git a/pyzoo/zoo/examples/inception/README.md b/pyzoo/zoo/examples/inception/README.md new file mode 100644 index 00000000000..dd9ae646120 --- /dev/null +++ b/pyzoo/zoo/examples/inception/README.md @@ -0,0 +1,109 @@ +# Inception Model on Imagenet +This example demonstrates how to use Analytics-zoo to train [Inception v1](https://arxiv +.org/abs/1409.4842) architecture on the [ImageNet](http://image-net.org/index) data. +## Get the JAR +You can build one by refer to the +[Build Page](https://analytics-zoo.github.io/master/#ScalaUserGuide/install/#download-analytics-zoo-source) from the source code. We +will release a pre-build package soon. + +## Prepare the data +You can download imagenet-2012 data from . + +After you download the files(**ILSVRC2012_img_train.tar** and **ILSVRC2012_img_val.tar**), +run the following commands to prepare the data. + +```bash +mkdir train +mv ILSVRC2012_img_train.tar train/ +cd train +tar -xvf ILSVRC2012_img_train.tar +rm -f ILSVRC2012_img_train.tar +find . -name "*.tar" | while read CLASS_NAME ; do mkdir -p "${CLASS_NAME%.tar}"; tar -xvf "${CLASS_NAME}" -C "${CLASS_NAME%.tar}"; done +rm *.tar +cd ../ +mkdir val +mv ILSVRC2012_img_val.tar val/ +cd val +tar -xvf ILSVRC2012_img_val.tar +cat classes.lst | while read CLASS_NAME; do mkdir -p ${CLASS_NAME}; done +cat img_class.lst | while read PARAM; do mv ${PARAM/ n[0-9]*/} ${PARAM/ILSVRC*JPEG /}; done +rm ILSVRC2012_img_val.tar +``` + +Now all the images belonging to the same category are moved to the same folder. + +This command will transform the images into hadoop sequence files, which are +more suitable for a distributed training. + +```bash +spark-submit --class com.intel.analytics.bigdl.models.utils.ImageNetSeqFileGenerator bigdl-VERSION-jar-with-dependencies.jar -f imagenet_folder -o output_folder -p cores_number +``` + +It will generate the hadoop sequence files in the output folder. + + + + +## Train the Model +* Spark standalone, example command +```bash +export SPARK_HOME=the root directory of Spark +export ANALYTICS_ZOO_HOME=the dist directory under the Analytics Zoo project + +${ANALYTICS_ZOO_HOME}/bin/spark-submit-with-zoo.sh \ +--master spark://xxx.xxx.xxx.xxx:xxxx \ +--executor-cores 32 \ +--num-executors 16 \ +--executor-memory 150G \ +--driver-memory 20G \ +--conf spark.network.timeout=10000000 pyzoo/zoo/examples/inception/inception.py \ +--batchSize 1024 \ +--learningRate 0.065 \ +--weightDecay 0.0002 \ +--checkpointIteration 1000 \ +-f hdfs://... \ +--checkpoint /models/inception \ +--maxIteration 90000 +``` + +* Spark yarn client mode, example command +``` +export SPARK_HOME=the root directory of Spark +export ANALYTICS_ZOO_HOME=the dist directory under the Analytics Zoo project + +${ANALYTICS_ZOO_HOME}/bin/spark-submit-with-zoo.sh \ +--master yarn \ +--deploy-mode client \ +--executor-cores 32 \ +--num-executors 16 \ +--executor-memory 150G \ +--driver-memory 20G \ +--conf spark.network.timeout=10000000 pyzoo/zoo/examples/inception/inception.py \ +--batchSize 1024 \ +--learningRate 0.065 \ +--weightDecay 0.0002 \ +--checkpointIteration 1000 \ +-f hdfs://... \ +--checkpoint /models/incepton \ +--maxIteration 90000 +``` + +In the above commands +* -f: where you put your ImageNet data, it should be a hdfs folder +* --checkpoint: Where you cache the model/train_state snapshot. You should input a folder and +make sure the folder is created when you run this example. The model snapshot will be named as +model.#iteration_number, and train state will be named as optimMethod.#iteration_number. Note that if +there are some files already exist in the folder, the old file will not be overwrite for the +safety of your model files. +* --batchSize: The mini-batch size. It is expected that the mini-batch size is a multiple of node_number * +core_number. In this example, node_number is 1 and the mini-batch size is suggested to be set to core_number * 4 +* --learningRate: inital learning rate. Note in this example, we use a Poly learning rate decay +policy. +* --weightDecay: weight decay. +* --checkpointIteration: the checkpoint interval in iteration. +* --maxLr: optional. Max learning rate after warm up. It has to be set together with warmupEpoch. +* --warmupEpoch: optional. Epoch numbers need to take to increase learning rate from learningRate to maxLR. +* --gradientL2NormThreshold: optional. Gradient L2-Norm threshold used for norm2 gradient clipping. +* --gradientMin: optional. Max gradient clipping by value, used in constant gradient clipping. +* --gradientMax: optional. Min gradient clipping by value, used in constant gradient clipping. +* --maxIteration: max iteration From 4b56df8b0f4acd171e17db95bbc048688e613a61 Mon Sep 17 00:00:00 2001 From: YinaChen Date: Tue, 3 Sep 2019 10:48:22 +0800 Subject: [PATCH 24/28] revert version --- pyzoo/zoo/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyzoo/zoo/__init__.py b/pyzoo/zoo/__init__.py index 0874210a247..60225f10248 100644 --- a/pyzoo/zoo/__init__.py +++ b/pyzoo/zoo/__init__.py @@ -33,4 +33,4 @@ for clz in creator_classes: JavaCreator.add_creator_class(clz) -__version__ = "0.6.0.dev14" +__version__ = "0.6.0.dev0" From c112efea9e9e47769ede39fe71e5dca44ac5152b Mon Sep 17 00:00:00 2001 From: YinaChen Date: Mon, 9 Sep 2019 13:28:28 +0800 Subject: [PATCH 25/28] Fix & add comments --- pyzoo/zoo/examples/inception/README.md | 2 +- pyzoo/zoo/feature/common.py | 10 +++++++++- pyzoo/zoo/feature/image/imagePreprocessing.py | 2 +- .../analytics/zoo/feature/image/ImageMirror.scala | 3 +++ 4 files changed, 14 insertions(+), 3 deletions(-) diff --git a/pyzoo/zoo/examples/inception/README.md b/pyzoo/zoo/examples/inception/README.md index dd9ae646120..66f16ea4be2 100644 --- a/pyzoo/zoo/examples/inception/README.md +++ b/pyzoo/zoo/examples/inception/README.md @@ -67,7 +67,7 @@ ${ANALYTICS_ZOO_HOME}/bin/spark-submit-with-zoo.sh \ ``` * Spark yarn client mode, example command -``` +```bash export SPARK_HOME=the root directory of Spark export ANALYTICS_ZOO_HOME=the dist directory under the Analytics Zoo project diff --git a/pyzoo/zoo/feature/common.py b/pyzoo/zoo/feature/common.py index 040023d0ac9..e25fce0b018 100644 --- a/pyzoo/zoo/feature/common.py +++ b/pyzoo/zoo/feature/common.py @@ -246,10 +246,14 @@ def image_frame(cls, image_frame, memory_type="DRAM", bigdl_type="float"): def image_set(cls, imageset, memory_type="DRAM", bigdl_type="float"): """ Create FeatureSet from ImageFrame. - :param imageset: ImageFrame + :param imageset: ImageSet :param memory_type: string, DRAM or PMEM If it's DRAM, will cache dataset into dynamic random-access memory If it's PMEM, will cache dataset into Intel Optane DC Persistent Memory + If it's a Int number n, will cache dataset into disk, and only hold 1/n + of the data into memory during the training. After going through the + 1/n, we will release the current cache, and load another 1/n into + memory. :param bigdl_type: numeric type :return: A feature set """ @@ -265,6 +269,10 @@ def sample_rdd(cls, rdd, memory_type="DRAM", bigdl_type="float"): :param memory_type: string, DRAM or PMEM If it's DRAM, will cache dataset into dynamic random-access memory If it's PMEM, will cache dataset into Intel Optane DC Persistent Memory + If it's a Int number n, will cache dataset into disk, and only hold 1/n + of the data into memory during the training. After going through the + 1/n, we will release the current cache, and load another 1/n into + memory. :param bigdl_type:numeric type :return: A feature set """ diff --git a/pyzoo/zoo/feature/image/imagePreprocessing.py b/pyzoo/zoo/feature/image/imagePreprocessing.py index 38b07620ec8..fcc8328dc33 100644 --- a/pyzoo/zoo/feature/image/imagePreprocessing.py +++ b/pyzoo/zoo/feature/image/imagePreprocessing.py @@ -357,7 +357,7 @@ def __init__(self, bigdl_type="float"): class ImageFeatureToSample(Preprocessing): """ - a Transformer that get Sample from ImageFeature. + A transformer that get Sample from ImageFeature. """ def __init__(self, bigdl_type="float"): super(ImageFeatureToSample, self).__init__(bigdl_type) diff --git a/zoo/src/main/scala/com/intel/analytics/zoo/feature/image/ImageMirror.scala b/zoo/src/main/scala/com/intel/analytics/zoo/feature/image/ImageMirror.scala index c82e93a6921..5c32b761e3e 100644 --- a/zoo/src/main/scala/com/intel/analytics/zoo/feature/image/ImageMirror.scala +++ b/zoo/src/main/scala/com/intel/analytics/zoo/feature/image/ImageMirror.scala @@ -19,6 +19,9 @@ import com.intel.analytics.bigdl.transform.vision.image.opencv.OpenCVMat import com.intel.analytics.bigdl.transform.vision.image.{FeatureTransformer, ImageFeature} import org.opencv.core.Core +/** + * Flip the image horizontally and vertically + */ class Mirror() extends FeatureTransformer { override def transformMat(feature: ImageFeature): Unit = { Mirror.transform(feature.opencvMat(), feature.opencvMat()) From a064fe472cbc86f51e77e34925421eb1e6289a62 Mon Sep 17 00:00:00 2001 From: YinaChen Date: Mon, 9 Sep 2019 14:16:56 +0800 Subject: [PATCH 26/28] Add license --- pyzoo/zoo/examples/inception/inception.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/pyzoo/zoo/examples/inception/inception.py b/pyzoo/zoo/examples/inception/inception.py index f2e08bb3359..5b292ff648e 100644 --- a/pyzoo/zoo/examples/inception/inception.py +++ b/pyzoo/zoo/examples/inception/inception.py @@ -1,3 +1,18 @@ +# +# Copyright 2018 Analytics Zoo Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# from optparse import OptionParser from math import ceil from datetime import datetime From 1906279b483668b110831cad994536ae98b83229 Mon Sep 17 00:00:00 2001 From: YinaChen Date: Mon, 9 Sep 2019 15:01:21 +0800 Subject: [PATCH 27/28] Code style fix --- .../zoo/pipeline/estimator/test_estimator.py | 9 ++-- pyzoo/zoo/examples/inception/inception.py | 44 ++++++++++++------- pyzoo/zoo/feature/image/imagePreprocessing.py | 1 + 3 files changed, 34 insertions(+), 20 deletions(-) diff --git a/pyzoo/test/zoo/pipeline/estimator/test_estimator.py b/pyzoo/test/zoo/pipeline/estimator/test_estimator.py index c955b08391e..24b9c6c7a12 100644 --- a/pyzoo/test/zoo/pipeline/estimator/test_estimator.py +++ b/pyzoo/test/zoo/pipeline/estimator/test_estimator.py @@ -72,10 +72,11 @@ def test_estimator_train_imagefeature(self): estimator = Estimator(model, optim_method, "") estimator.set_constant_gradient_clipping(0.1, 1.2) estimator.train_imagefeature(train_set=data_set, criterion=ClassNLLCriterion(), - end_trigger=MaxEpoch(epoch_num), - checkpoint_trigger=EveryEpoch(), - validation_set=data_set, - validation_method=[Top1Accuracy()], batch_size=batch_size) + end_trigger=MaxEpoch(epoch_num), + checkpoint_trigger=EveryEpoch(), + validation_set=data_set, + validation_method=[Top1Accuracy()], + batch_size=batch_size) predict_result = model.predict_image(image_frame.transform(transformer)) assert(predict_result.get_predict().count(), 8) diff --git a/pyzoo/zoo/examples/inception/inception.py b/pyzoo/zoo/examples/inception/inception.py index 5b292ff648e..f9b9cf3ea87 100644 --- a/pyzoo/zoo/examples/inception/inception.py +++ b/pyzoo/zoo/examples/inception/inception.py @@ -34,24 +34,33 @@ def config_option_parser(): help="url of hdf+s folder store the hadoop sequence files") parser.add_option("--model", type=str, dest="model", default="", help="model snapshot location") parser.add_option("--state", type=str, dest="state", default="", help="state snapshot location") - parser.add_option("--checkpoint", type=str, dest="checkpoint", default="", help="where to cache the model") + parser.add_option("--checkpoint", type=str, dest="checkpoint", default="", + help="where to cache the model") parser.add_option("-o", "--overwrite", action="store_true", dest="overwrite", default=False, help="overwrite checkpoint files") - parser.add_option("-e", "--maxEpoch", type=int, dest="maxEpoch", default=0, help="epoch numbers") - parser.add_option("-i", "--maxIteration", type=int, dest="maxIteration", default=3100, help="iteration numbers") - parser.add_option("-l", "--learningRate", type=float, dest="learningRate", default=0.01, help="learning rate") - parser.add_option("--warmupEpoch", type=int, dest="warmupEpoch", default=0, help="warm up epoch numbers") + parser.add_option("-e", "--maxEpoch", type=int, dest="maxEpoch", default=0, + help="epoch numbers") + parser.add_option("-i", "--maxIteration", type=int, dest="maxIteration", default=3100, + help="iteration numbers") + parser.add_option("-l", "--learningRate", type=float, dest="learningRate", default=0.01, + help="learning rate") + parser.add_option("--warmupEpoch", type=int, dest="warmupEpoch", default=0, + help="warm up epoch numbers") parser.add_option("--maxLr", type=float, dest="maxLr", default=0.0, help="max Lr after warm up") parser.add_option("-b", "--batchSize", type=int, dest="batchSize", help="batch size") parser.add_option("--classNum", type=int, dest="classNum", default=1000, help="class number") - parser.add_option("--weightDecay", type=float, dest="weightDecay", default=0.0001, help="weight decay") + parser.add_option("--weightDecay", type=float, dest="weightDecay", default=0.0001, + help="weight decay") parser.add_option("--checkpointIteration", type=int, dest="checkpointIteration", default=620, help="checkpoint interval of iterations") - parser.add_option("--gradientMin", type=float, dest="gradientMin", default=0.0, help="min gradient clipping by") - parser.add_option("--gradientMax", type=float, dest="gradientMax", default=0.0, help="max gradient clipping by") - parser.add_option("--gradientL2NormThreshold", type=float, dest="gradientL2NormThreshold", default=0.0, - help="gradient L2-Norm threshold") - parser.add_option("--memoryType", type=str, dest="memoryType", default="DRAM", help="memory storage type, DRAM or PMEM") + parser.add_option("--gradientMin", type=float, dest="gradientMin", default=0.0, + help="min gradient clipping by") + parser.add_option("--gradientMax", type=float, dest="gradientMax", default=0.0, + help="max gradient clipping by") + parser.add_option("--gradientL2NormThreshold", type=float, dest="gradientL2NormThreshold", + default=0.0, help="gradient L2-Norm threshold") + parser.add_option("--memoryType", type=str, dest="memoryType", default="DRAM", + help="memory storage type, DRAM or PMEM") return parser @@ -75,7 +84,7 @@ def inception_layer_v1(input_size, config, name_prefix=""): concat = Concat(2) conv1 = Sequential() conv1.add(SpatialConvolution(input_size, config[1][1], 1, 1, 1, 1) - .set_init_method(weight_init_method=Xavier(),bias_init_method=ConstInitMethod(0.1)) + .set_init_method(weight_init_method=Xavier(), bias_init_method=ConstInitMethod(0.1)) .set_name(name_prefix + "1x1")) conv1.add(ReLU(True).set_name(name_prefix + "relu_1x1")) concat.add(conv1) @@ -179,7 +188,8 @@ def inception_v1_no_aux_classifier(class_num, has_dropout=True): ImageRandomPreprocessing(ImageHFlip(), 0.5), ImageChannelNormalize(123.0, 117.0, 104.0), ImageMatToTensor(format="NCHW", to_RGB=False), - ImageSetToSample(input_keys=["imageTensor"], target_keys=["label"]) + ImageSetToSample(input_keys=["imageTensor"], + target_keys=["label"]) ]) raw_train_data = get_inception_data(options.folder, sc, "train") @@ -190,7 +200,8 @@ def inception_v1_no_aux_classifier(class_num, has_dropout=True): ImageCenterCrop(image_size, image_size), ImageChannelNormalize(123.0, 117.0, 104.0), ImageMatToTensor(format="NCHW", to_RGB=False), - ImageSetToSample(input_keys=["imageTensor"], target_keys=["label"]) + ImageSetToSample(input_keys=["imageTensor"], + target_keys=["label"]) ]) raw_val_data = get_inception_data(options.folder, sc, "val") @@ -226,7 +237,8 @@ def inception_v1_no_aux_classifier(class_num, has_dropout=True): lrSchedule = SequentialSchedule(iterationPerEpoch) lrSchedule.add(Warmup(warmupDelta), warmup_iteration) lrSchedule.add(Poly(0.5, maxIteration), polyIteration) - optim = SGD(learningrate=options.learningRate, learningrate_decay=0.0, weightdecay=options.weightDecay, + optim = SGD(learningrate=options.learningRate, learningrate_decay=0.0, + weightdecay=options.weightDecay, momentum=0.9, dampening=0.0, nesterov=False, leaningrate_schedule=lrSchedule) @@ -261,4 +273,4 @@ def inception_v1_no_aux_classifier(class_num, has_dropout=True): inception_model.saveModel("/tmp/inception/model.bigdl", "/tmp/inception/model.bin", True) - sc.stop() \ No newline at end of file + sc.stop() diff --git a/pyzoo/zoo/feature/image/imagePreprocessing.py b/pyzoo/zoo/feature/image/imagePreprocessing.py index fcc8328dc33..4dcf16c992c 100644 --- a/pyzoo/zoo/feature/image/imagePreprocessing.py +++ b/pyzoo/zoo/feature/image/imagePreprocessing.py @@ -40,6 +40,7 @@ class ImageBytesToMat(ImagePreprocessing): def __init__(self, byte_key="bytes", image_codec=-1, bigdl_type="float"): super(ImageBytesToMat, self).__init__(bigdl_type, byte_key, image_codec) + class ImagePixelBytesToMat(ImagePreprocessing): """ Transform byte array(pixels in byte) to OpenCVMat From 271ff9335aa4385aee54eac4a00d3f3758710f19 Mon Sep 17 00:00:00 2001 From: YinaChen Date: Mon, 9 Sep 2019 15:33:44 +0800 Subject: [PATCH 28/28] Scala style fix --- .../analytics/zoo/feature/image/ImageMirror.scala | 2 +- .../zoo/feature/python/PythonFeatureSet.scala | 2 +- .../pipeline/estimator/python/PythonEstimator.scala | 12 ++++++++---- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/zoo/src/main/scala/com/intel/analytics/zoo/feature/image/ImageMirror.scala b/zoo/src/main/scala/com/intel/analytics/zoo/feature/image/ImageMirror.scala index 5c32b761e3e..6cb7da56528 100644 --- a/zoo/src/main/scala/com/intel/analytics/zoo/feature/image/ImageMirror.scala +++ b/zoo/src/main/scala/com/intel/analytics/zoo/feature/image/ImageMirror.scala @@ -31,7 +31,7 @@ class Mirror() extends FeatureTransformer { object Mirror { def apply(): Mirror = new Mirror() - def transform(input: OpenCVMat, output: OpenCVMat): OpenCVMat ={ + def transform(input: OpenCVMat, output: OpenCVMat): OpenCVMat = { Core.flip(input, output, -1) output } diff --git a/zoo/src/main/scala/com/intel/analytics/zoo/feature/python/PythonFeatureSet.scala b/zoo/src/main/scala/com/intel/analytics/zoo/feature/python/PythonFeatureSet.scala index fb3e0fd39e9..e4e0cf9b6c8 100644 --- a/zoo/src/main/scala/com/intel/analytics/zoo/feature/python/PythonFeatureSet.scala +++ b/zoo/src/main/scala/com/intel/analytics/zoo/feature/python/PythonFeatureSet.scala @@ -49,7 +49,7 @@ class PythonFeatureSet[T: ClassTag](implicit ev: TensorNumeric[T]) extends Pytho FeatureSet.rdd(data, MemoryType.fromString(memoryType)) } - def createSampleFeatureSetFromRDD(data: JavaRDD[Sample], memoryType:String) + def createSampleFeatureSetFromRDD(data: JavaRDD[Sample], memoryType: String) : FeatureSet[JSample[T]] = { FeatureSet.rdd(toJSample(data), MemoryType.fromString(memoryType)) } diff --git a/zoo/src/main/scala/com/intel/analytics/zoo/pipeline/estimator/python/PythonEstimator.scala b/zoo/src/main/scala/com/intel/analytics/zoo/pipeline/estimator/python/PythonEstimator.scala index 23eb63c779f..9f3bf48a004 100644 --- a/zoo/src/main/scala/com/intel/analytics/zoo/pipeline/estimator/python/PythonEstimator.scala +++ b/zoo/src/main/scala/com/intel/analytics/zoo/pipeline/estimator/python/PythonEstimator.scala @@ -57,8 +57,10 @@ class PythonEstimator[T: ClassTag](implicit ev: TensorNumeric[T]) extends Python estimator.evaluate(validationMiniBatch, validationMethod) } - def estimatorEvaluateImageFeature(estimator: Estimator[T], validationSet: FeatureSet[ImageFeature], - validationMethod: Array[ValidationMethod[T]], batchSize: Int + def estimatorEvaluateImageFeature(estimator: Estimator[T], + validationSet: FeatureSet[ImageFeature], + validationMethod: Array[ValidationMethod[T]], + batchSize: Int ): Map[ValidationMethod[T], ValidationResult] = { val imageFeature2batch = ImageFeatureToMiniBatch(batchSize) val validationMiniBatch = validationSet -> imageFeature2batch @@ -85,12 +87,14 @@ class PythonEstimator[T: ClassTag](implicit ev: TensorNumeric[T]) extends Python validationMiniBatch, validationMethod) } - def estimatorTrainImageFeature(estimator: Estimator[T], trainSet: FeatureSet[ImageFeature], + def estimatorTrainImageFeature(estimator: Estimator[T], + trainSet: FeatureSet[ImageFeature], criterion: Criterion[T], endTrigger: Trigger = null, checkPointTrigger: Trigger = null, validationSet: FeatureSet[ImageFeature] = null, - validationMethod: JList[ValidationMethod[T]] = null, batchSize: Int) + validationMethod: JList[ValidationMethod[T]] = null, + batchSize: Int) : estimator.type = { val imageFeature2batch = ImageFeatureToMiniBatch(batchSize) val trainMiniBatch = trainSet -> imageFeature2batch