Unable to train segmentation network using VGG pretrained model

[carlo998]

Hi, I've been trying to retrain a model similar to Deeplab for segmentation. Instead of their model, I'm using the VGG-16 model for retraining. But while training, I see that after ~1000 iterations, the following accuracies:

I0303 02:39:56.538744 1635 solver.cpp:224] Train net output #0: accuracy = 0.694721 (pixel accuracy)
I0303 02:39:56.538760 1635 solver.cpp:224] Train net output #1: accuracy = 0.0833333 (overall classification accuracy)
I0303 02:39:56.538771 1635 solver.cpp:224] Train net output #2: accuracy = 0.461653 (iou)

The classification accuracy seems to be very less and the pixel accuracy does not seem to increase beyond this point. Can you please tell me if I'm doing something wrong ? Here is my prototxt.

name: "sud"

layers {
  name: "data"
  type: IMAGE_SEG_DATA
  top: "data"
  top: "label"
  image_data_param {
    root_folder: "/users/sudipto.b/VOCdevkit/VOC2012/"
    source: "list/train_aug.txt"
    label_type: PIXEL
    batch_size: 20
    shuffle: true
  }
  transform_param {
    mean_value: 104.008
    mean_value: 116.669
    mean_value: 122.675
    crop_size: 321
    mirror: true
  }
  include: { phase: TRAIN }
}

layers {
  bottom: "data"
  top: "conv1_1"
  name: "conv1_1"
  type: CONVOLUTION
  convolution_param {
    num_output: 64
    pad: 1
    kernel_size: 3
  }
  blobs_lr: 1
  blobs_lr: 2
}
layers {
  bottom: "conv1_1"
  top: "conv1_1"
  name: "relu1_1"
  type: RELU
}
layers {
  bottom: "conv1_1"
  top: "conv1_2"
  name: "conv1_2"
  type: CONVOLUTION
  convolution_param {
    num_output: 64
    pad: 1
    kernel_size: 3
  }
  blobs_lr: 1
  blobs_lr: 2
}
layers {
  bottom: "conv1_2"
  top: "conv1_2"
  name: "relu1_2"
  type: RELU
}
layers {
  bottom: "conv1_2"
  top: "pool1"
  name: "pool1"
  type: POOLING
  pooling_param {
    pool: MAX
    kernel_size: 2
    stride: 2
  }
}
layers {
  bottom: "pool1"
  top: "conv2_1"
  name: "conv2_1"
  type: CONVOLUTION
  convolution_param {
    num_output: 128
    pad: 1
    kernel_size: 3
  }
  blobs_lr: 1
  blobs_lr: 2
}
layers {
  bottom: "conv2_1"
  top: "conv2_1"
  name: "relu2_1"
  type: RELU
}
layers {
  bottom: "conv2_1"
  top: "conv2_2"
  name: "conv2_2"
  type: CONVOLUTION
  convolution_param {
    num_output: 128
    pad: 1
    kernel_size: 3
  }
  blobs_lr: 1
  blobs_lr: 2
}
layers {
  bottom: "conv2_2"
  top: "conv2_2"
  name: "relu2_2"
  type: RELU
}
layers {
  bottom: "conv2_2"
  top: "pool2"
  name: "pool2"
  type: POOLING
  pooling_param {
    pool: MAX
    kernel_size: 2
    stride: 2
  }
}
layers {
  bottom: "pool2"
  top: "conv3_1"
  name: "conv3_1"
  type: CONVOLUTION
  convolution_param {
    num_output: 256
    pad: 1
    kernel_size: 3
  }
  blobs_lr: 1
  blobs_lr: 2
}
layers {
  bottom: "conv3_1"
  top: "conv3_1"
  name: "relu3_1"
  type: RELU
}
layers {
  bottom: "conv3_1"
  top: "conv3_2"
  name: "conv3_2"
  type: CONVOLUTION
  convolution_param {
    num_output: 256
    pad: 1
    kernel_size: 3
  }
  blobs_lr: 1
  blobs_lr: 2
}
layers {
  bottom: "conv3_2"
  top: "conv3_2"
  name: "relu3_2"
  type: RELU
}
layers {
  bottom: "conv3_2"
  top: "conv3_3"
  name: "conv3_3"
  type: CONVOLUTION
  convolution_param {
    num_output: 256
    pad: 1
    kernel_size: 3
  }
  blobs_lr: 1
  blobs_lr: 2
}
layers {
  bottom: "conv3_3"
  top: "conv3_3"
  name: "relu3_3"
  type: RELU
}
layers {
  bottom: "conv3_3"
  top: "pool3"
  name: "pool3"
  type: POOLING
  pooling_param {
    pool: MAX
    kernel_size: 2
    stride: 2
  }
}
layers {
  bottom: "pool3"
  top: "conv4_1"
  name: "conv4_1"
  type: CONVOLUTION
  convolution_param {
    num_output: 512
    pad: 1
    kernel_size: 3
  }
  blobs_lr: 1
  blobs_lr: 2
}
layers {
  bottom: "conv4_1"
  top: "conv4_1"
  name: "relu4_1"
  type: RELU
}
layers {
  bottom: "conv4_1"
  top: "conv4_2"
  name: "conv4_2"
  type: CONVOLUTION
  convolution_param {
    num_output: 512
    pad: 1
    kernel_size: 3
  }
  blobs_lr: 1
  blobs_lr: 2
}
layers {
  bottom: "conv4_2"
  top: "conv4_2"
  name: "relu4_2"
  type: RELU
}
layers {
  bottom: "conv4_2"
  top: "conv4_3"
  name: "conv4_3"
  type: CONVOLUTION
  convolution_param {
    num_output: 512
    pad: 1
    kernel_size: 3
  }
  blobs_lr: 1
  blobs_lr: 2
}
layers {
  bottom: "conv4_3"
  top: "conv4_3"
  name: "relu4_3"
  type: RELU
}
layers {
  bottom: "conv4_3"
  top: "pool4-s"
  name: "pool4-s"
  type: POOLING
  pooling_param {
    pool: MAX
    kernel_size: 3
    pad: 1
    stride: 1
  }
}
layers {
  bottom: "pool4-s"
  top: "conv5_1-s"
  name: "conv5_1-s"
  type: CONVOLUTION
  blobs_lr: 1
  blobs_lr: 2
  weight_decay: 1
  weight_decay: 0
  convolution_param {
    num_output: 512
    pad: 1
    kernel_size: 3
    stride: 1
  }
}
layers {
  bottom: "conv5_1-s"
  top: "conv5_1-s"
  name: "relu5_1-s"
  type: RELU
}
layers {
  bottom: "conv5_1-s"
  top: "conv5_2-s"
  name: "conv5_2-s"
  type: CONVOLUTION
  blobs_lr: 1
  blobs_lr: 2
  weight_decay: 1
  weight_decay: 0
  convolution_param {
    num_output: 512
    pad: 1
    kernel_size: 3
    stride: 1
  }
}
layers {
  bottom: "conv5_2-s"
  top: "conv5_2-s"
  name: "relu5_2-s"
  type: RELU
}
layers {
  bottom: "conv5_2-s"
  top: "conv5_3-s"
  name: "conv5_3-s"
  type: CONVOLUTION
  blobs_lr: 1
  blobs_lr: 2
  weight_decay: 1
  weight_decay: 0
  convolution_param {
    num_output: 512
    pad: 1
    kernel_size: 3
    stride: 1
  }
}
layers {
  bottom: "conv5_3-s"
  top: "conv5_3-s"
  name: "relu5_3-s"
  type: RELU
}
layers {
  bottom: "conv5_3-s"
  top: "pool5-s"
  name: "pool5-s"
  type: POOLING
  pooling_param {
    pool: MAX
    #kernel_size: 2
    #stride: 2
    kernel_size: 3
    stride: 1
    pad: 1
  }
}

layers {
  bottom: "pool5-s"
  top: "pool5a-s"
  name: "pool5a-s"
  type: POOLING
  pooling_param {
    pool: AVE
    kernel_size: 3
    stride: 1
    pad: 1
  }
}

layers {
  bottom: "pool5a-s"
  top: "fc6-s"
  name: "fc6-s"
  type: CONVOLUTION
  blobs_lr: 1
  blobs_lr: 2
  weight_decay: 1
  weight_decay: 0
  convolution_param {
    num_output: 1024
    pad: 1
    kernel_size: 3
    stride: 1
  }
}
layers {
  bottom: "fc6-s"
  top: "fc6-s"
  name: "relu6-s"
  type: RELU
}

layers {
  bottom: "fc6-s"
  top: "fc7-s"
  name: "fc7-s"
  type: CONVOLUTION
  blobs_lr: 1
  blobs_lr: 2
  weight_decay: 1
  weight_decay: 0
  convolution_param {
    num_output: 1024
    kernel_size: 1
  }
}
layers {
  bottom: "fc7-s"
  top: "fc7-s"
  name: "relu7-s"
  type: RELU
}
layers {
  bottom: "fc7-s"
  top: "fc7-s"
  name: "drop7-s"
  type: DROPOUT
  dropout_param {
    dropout_ratio: 0.5
  }
}
layers {
  bottom: "fc7-s"
  top: "fc8_voc12"
  name: "fc8_voc12"
  type: CONVOLUTION
  blobs_lr: 10
  blobs_lr: 20
  weight_decay: 1
  weight_decay: 0
  convolution_param {
    num_output: 21
    kernel_size: 1
    weight_filler {
      type: "gaussian"
      std: 0.01
    }
    bias_filler {
      type: "constant"
      value: 0
    }
  }
}

layers {
  bottom: "label"
  top: "label_shrink"
  name: "label_shrink"
  type: INTERP
  interp_param {
    shrink_factor: 8
    pad_beg: 0
    pad_end: 0
  }
}
layers {
  name: "loss"
  type: SOFTMAX_LOSS
  bottom: "fc8_voc12"
  bottom: "label_shrink"
  softmaxloss_param {
    weight_source: "weights.txt"
    ignore_label: 255
  }
  include: { phase: TRAIN }
}
layers {
  name: "accuracy"
  type: SEG_ACCURACY
  bottom: "fc8_voc12"
  bottom: "label_shrink"
  top: "accuracy"
  seg_accuracy_param {
    ignore_label: 255
  } 

  include: { phase: TRAIN }
}

[seanbell]

Please ask for usage/modeling help on the mailing list.

From https://github.com/BVLC/caffe/blob/master/CONTRIBUTING.md:

Please do not post usage, installation, or modeling questions, or other requests for help to Issues.
Use the caffe-users list instead. This helps developers maintain a clear, uncluttered, and efficient view of the state of Caffe.