mxnet scripts updated, benchmark scripts updated. Details can be foun…

…d in README.md

README.md

@@ -41,3 +41,13 @@ To run benchmark test just execute
 
 ### Add new tools
 Follow the instructions in *tools/Readsme.md* preparing the running scripts and netowrk configurations. Note that training data should be put in **$HOME/data/** so that we can test new tools in our machines and update benchmarking results to our website.
+
+### Update log    
+May 24, 2017:   
+- Updated the scripts for mxnet, previous scripts are still preserved in tools/mxnet/mxnet0.7    
+- Updated benchmark.py and the format of config files. Now you only need to specify the device id (-1 for CPU; 0,1,2,3 for GPU) and device count (number of cores to use)  in the config file. An example called test.config can be found in configs/   
+- yaroslavvb helped improve the performance of CNTK CIFAR by keeping variables on CPU. Thank you!
+- tfboyd optimized the scripts for tensorflow by changing the image processing place to CPU and some other tweaks making tensorflow run faster then before. Thank you!   
+
+
+- 

batch-bencmarks-cpu-gpu20_mxnet.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+# The benchmarks of all toolkits 
+python benchmark.py -config ./configs/mxnet_bm2cpu1.config -post True
+python benchmark.py -config ./configs/mxnet_bm2cpu2.config -post True
+python benchmark.py -config ./configs/mxnet_bm2cpu4.config -post True
+python benchmark.py -config ./configs/mxnet_bm2cpu8.config -post True
+python benchmark.py -config ./configs/mxnet_bm2cpu16.config -post True
+python benchmark.py -config ./configs/mxnet_bm2cpu32.config -post True

benchmark.py

@@ -20,7 +20,7 @@
 flag = ''
 tools = ''
 cpu_name = ''
-device_name = ''
+gpu_name = ''
 cuda_driver = ''
 cudnn = ''
 cuda = ''
@@ -44,16 +44,14 @@
 				host_file = line.split(':')[1]
 			elif "cpu_name:" in line:
 				cpu_name = line.split(':')[1]
-			elif "device_name:" in line:
-				device_name = line.split(':')[1]
+			elif "gpu_name:" in line:
+				gpu_name = line.split(':')[1]
 			elif "cuda_driver:" in line:
 				cuda_driver = line.split(':')[1]
 			elif "cudnn:" in line:
 				cudnn = line.split(':')[1]
 			elif "cuda:" in line:
 				cuda = line.split(':')[1]
-			elif "cpu_count:" in line:
-				cpu_count = line.split(':')[1]
 		else:
 			if "}" in line:
 				config_experiments = False
@@ -64,7 +62,7 @@
 	print("Please add -config <path to your config file>")
 	sys.exit(0)
 
-post_flags = " -f " + flag + " -d " + device_name + " -c " + cpu_count + " -P " + cpu_name + " -A unknown" + " -r " + cuda_driver + " -C " + cuda + " -D " + cudnn
+post_flags = " -f " + flag + " -P " + cpu_name + " -A unknown" + " -r " + cuda_driver + " -C " + cuda + " -D " + cudnn
 
 if args.debug:
 	print "[DEBUG] Defalut post flags:" + str(post_flags)
@@ -88,13 +86,26 @@
 	for experiment in experiments:
 		os.chdir(work_dir)
 		exp_args = experiment.split(";")
+		device_name = ''
+		device_count = exp_args[3]
+		log_file = ''
+		if "-1" in exp_args[2]:
+			device_name = cpu_name
+			log_file = tool + "-" + exp_args[0] + "-" + exp_args[1] + "-" + device_name + "-c" + exp_args[3] + "-" +"b"+ exp_args[4] + "-" 
+		else:
+			device_name = gpu_name
+			log_file = tool + "-" + exp_args[0] + "-" + exp_args[1] + "-" + device_name + "-devId" + exp_args[2] + "-c" + exp_args[3] + "-" +"b"+ exp_args[4] + "-" 
 		print "\n-------Benchmarking " + tool + " " + exp_args[1] + "-------"
-		log_file = tool + "-" + exp_args[0] + "-" + exp_args[1] + "-" +"gpu"+ exp_args[2] + "-" + device_name + "-" +"b"+ exp_args[4] + "-" 
 		log_file += time.ctime()+ "-" + host_name + ".log"
 		log_file = log_file.replace(" ","_")
 		bm_script = "python " + tool + "bm.py" 
-		bm_script += " -netType "+exp_args[0]+" -log "+log_file+" -batchSize "+exp_args[4]+" -network "+exp_args[1]+" -lr "+exp_args[7] + " -cpuCount " + cpu_count
-		bm_script += " -devId " + exp_args[2] + " -numEpochs " + exp_args[5] + " -epochSize " + exp_args[6] + " -gpuCount " + exp_args[3]
+		bm_script += " -netType " + exp_args[0] + " -log "+log_file+" -batchSize "+exp_args[4]+" -network "+exp_args[1]+" -lr "+exp_args[7]
+		if "-1" in exp_args[2]:
+			bm_script += " -devId " + exp_args[2] + " -numEpochs " + exp_args[5] + " -epochSize " + exp_args[6] + " -cpuCount " + exp_args[3]
+			post_flags +=  " -c " + cpu_name 
+		else:
+			bm_script += " -devId " + exp_args[2] + " -numEpochs " + exp_args[5] + " -epochSize " + exp_args[6] + " -gpuCount " + exp_args[3]
+			post_flags +=  " -d " + gpu_name 
 		if host_file is not None and len(host_file) > 4:
 			bm_script += " -hostFile " + host_file
 		print bm_script
@@ -111,7 +122,7 @@
 			post_script = "python post_record.py " + post_flags
 			print post_script
 			print(subprocess.check_output(post_script, shell=True).strip().split('\n')[0])
-			post_flags = " -f " + flag + " -d " + device_name + " -c " + cpu_count + " -P " + cpu_name + " -A unknown" + " -r " + cuda_driver + " -C " + cuda + " -D " + cudnn
+			post_flags = " -f " + flag + " -d " + device_name + " -P " + cpu_name + " -A unknown" + " -r " + cuda_driver + " -C " + cuda + " -D " + cudnn
 			post_script = ''
 		else:
 			print "Result:"

configs/bm2cpu.config → configs/legacy/bm2cpu.config

@@ -2,7 +2,7 @@ flag:		syntheticSgbenchmark2 #Flag of current experiment
 tools:		caffe,cntk,mxnet,tensorflow,torch			#Tools to benchmark
 experiments: #<network type>; <network name>;  <device id>; <gpu count>;  <batch size>;  <number of epochs>;  <epoch size>; <learning rate>
 {
-		fc;		fcn5;		-1;	1;		1024;		4;		60000;		0.05
+		fc;		fcn5;		-1;		1;		1024;		4;		60000;		0.05
 		cnn;		alexnet;	-1;		1;		1024;		2;		50000;		0.01
 		cnn;		resnet;		-1;		1;		128;		2;		50000;		0.01
 #		rnn;		lstm;		-1;		1;		128;		2;		2048;		0.1

configs/legacy/mxnet_bm2cpu.config

@@ -0,0 +1,16 @@
+flag:		syntheticSgbenchmark2 #Flag of current experiment
+tools:		mxnet			#Tools to benchmark
+experiments: #<network type>; <network name>;  <device id>; <gpu count>;  <batch size>;  <number of epochs>;  <epoch size>; <learning rate>
+{
+#		fc;		fcn5;		-1;		1;		1024;		4;		60000;		0.05
+		cnn;		alexnet;	-1;		1;		1024;		2;		50000;		0.01
+#		cnn;		resnet;		-1;		1;		128;		2;		50000;		0.01
+#		rnn;		lstm;		-1;		1;		128;		2;		2048;		0.1
+}
+host_file:	None		#Path to host file or None
+cpu_name:	E5-2630v4	#CPU model
+device_name:	E5-2630v4		#GPU model
+cpu_count:	1		#CPU count for cpu parallel
+cuda:		8.0		#CUDA version
+cudnn:		5.1		#CUDNN version
+cuda_driver:	367.48		#CUDA driver version

configs/legacy/mxnet_bm2cpu1.config

@@ -0,0 +1,13 @@
+flag:		sgbenchmark6 #Flag of current experiment
+tools:		mxnet			#Tools to benchmark
+experiments: #<network type>; <network name>;  <device id>; <gpu count>;  <batch size>;  <number of epochs>;  <epoch size>; <learning rate>
+{
+		cnn;		alexnet;	-1;		1;		1024;		4;		50000;		0.01
+}
+host_file:	None		#Path to host file or None
+cpu_name:	E5-2630v4	#CPU model
+device_name:	E5-2630v4	#traning device model
+cpu_count:	1		#CPU count for cpu parallel
+cuda:		8.0		#CUDA version
+cudnn:		5.1		#CUDNN version
+cuda_driver:	367.48		#CUDA driver version

configs/legacy/mxnet_bm2cpu16.config

@@ -0,0 +1,13 @@
+flag:		sgbenchmark6 #Flag of current experiment
+tools:		mxnet			#Tools to benchmark
+experiments: #<network type>; <network name>;  <device id>; <gpu count>;  <batch size>;  <number of epochs>;  <epoch size>; <learning rate>
+{
+		cnn;		alexnet;	-1;		1;		1024;		4;		50000;		0.01
+}
+host_file:	None		#Path to host file or None
+cpu_name:	E5-2630v4	#CPU model
+device_name:	E5-2630v4		#GPU model
+cpu_count:	16		#CPU count for cpu parallel
+cuda:		8.0		#CUDA version
+cudnn:		5.1		#CUDNN version
+cuda_driver:	367.48		#CUDA driver version

configs/legacy/mxnet_bm2cpu2.config

@@ -0,0 +1,13 @@
+flag:		sgbenchmark6 #Flag of current experiment
+tools:		mxnet			#Tools to benchmark
+experiments: #<network type>; <network name>;  <device id>; <gpu count>;  <batch size>;  <number of epochs>;  <epoch size>; <learning rate>
+{
+		cnn;		alexnet;	-1;		1;		1024;		2;		50000;		0.01
+}
+host_file:	None		#Path to host file or None
+cpu_name:	E5-2630v4	#CPU model
+device_name:	E5-2630v4		#GPU model
+cpu_count:	2		#CPU count for cpu parallel
+cuda:		8.0		#CUDA version
+cudnn:		5.1		#CUDNN version
+cuda_driver:	367.48		#CUDA driver version

configs/legacy/mxnet_bm2cpu32.config

@@ -0,0 +1,13 @@
+flag:		sgbenchmark6 #Flag of current experiment
+tools:		mxnet			#Tools to benchmark
+experiments: #<network type>; <network name>;  <device id>; <gpu count>;  <batch size>;  <number of epochs>;  <epoch size>; <learning rate>
+{
+		cnn;		alexnet;	-1;		1;		1024;		2;		50000;		0.01
+}
+host_file:	None		#Path to host file or None
+cpu_name:	E5-2630v4	#CPU model
+device_name:	E5-2630v4		#GPU model
+cpu_count:	32		#CPU count for cpu parallel
+cuda:		8.0		#CUDA version
+cudnn:		5.1		#CUDNN version
+cuda_driver:	367.48		#CUDA driver version

configs/legacy/mxnet_bm2cpu4.config

@@ -0,0 +1,13 @@
+flag:		sgbenchmark6 #Flag of current experiment
+tools:		mxnet			#Tools to benchmark
+experiments: #<network type>; <network name>;  <device id>; <gpu count>;  <batch size>;  <number of epochs>;  <epoch size>; <learning rate>
+{
+		cnn;		alexnet;	-1;		1;		1024;		2;		50000;		0.01
+}
+host_file:	None		#Path to host file or None
+cpu_name:	E5-2630v4	#CPU model
+device_name:	E5-2630v4		#GPU model
+cpu_count:	4		#CPU count for cpu parallel
+cuda:		8.0		#CUDA version
+cudnn:		5.1		#CUDNN version
+cuda_driver:	367.48		#CUDA driver version

configs/legacy/mxnet_bm2cpu8.config

@@ -0,0 +1,13 @@
+flag:		sgbenchmark6 #Flag of current experiment
+tools:		mxnet			#Tools to benchmark
+experiments: #<network type>; <network name>;  <device id>; <gpu count>;  <batch size>;  <number of epochs>;  <epoch size>; <learning rate>
+{
+		cnn;		alexnet;	-1;		1;		1024;		2;		50000;		0.01
+}
+host_file:	None		#Path to host file or None
+cpu_name:	E5-2630v4	#CPU model
+device_name:	E5-2630v4		#GPU model
+cpu_count:	8		#CPU count for cpu parallel
+cuda:		8.0		#CUDA version
+cudnn:		5.1		#CUDNN version
+cuda_driver:	367.48		#CUDA driver version

configs/legacy/test.config

@@ -0,0 +1,30 @@
+flag:		testdpBenchmark		#Flag of current experiment
+tools:		tensorflow			#Tools to benchmark
+experiments: #<network type>; <network name>;  <device id>; <gpu count>;  <batch size>;  <number of epochs>;  <epoch size>; <Learning rate>
+{
+		fc;		fcn5;		0;		1;		4096;		2;		60000;		0.05
+#		fc;		fcn5;		0;		1;		2048;		2;		60000;		0.05
+#		fc;		fcn5;		0;		1;		1024;		2;		60000;		0.05
+#		fc;		fcn5;		0;		1;		512;	   	2;		60000;		0.05
+#		fc;		fcn5;		0;		1;		342;	   	2;		60000;		0.05
+		cnn;		alexnet;	0;		1;		1024;		2;		50000;		0.01
+#		cnn;		alexnet;	0;		1;		512;  	 	2;		50000;		0.01
+#		cnn;		alexnet;	0;		1;		256;	   	2;		50000;		0.01
+#		cnn;		alexnet;	0;		1;		128;	   	2;		50000;		0.01
+#		cnn;		alexnet;	0;		1;		86;   		2;		50000;		0.01
+		cnn;		resnet;		0;		1;		128;		2;		50000;		0.01
+#		cnn;		resnet;		0;		1;		64;		2;		50000;		0.01
+#		cnn;		resnet;		0;		1;		32;		2;		50000;		0.01
+#		cnn;		resnet;		0;		1;		16;		2;		50000;		0.01
+#		cnn;		resnet;		0;		1;		11;		2;		50000;		0.01
+		rnn;		lstm;		0;		1;		1024;		2;		50000;		0.1
+#		rnn;		lstm;		0;		1;		512;   		2;		50000;		0.1
+#		rnn;		lstm;		0;		1;		256;   		2;		50000;		0.1
+#		rnn;		lstm;		0;		1;		128;   		2;		50000;		0.1
+}
+host_file:	None		#Path to host file or None
+cpu_name:	E5-2630v3	#CPU model
+device_name:	K80		#GPU model
+cuda:		8.0		#CUDA version
+cudnn:		5.1		#CUDNN version
+cuda_driver:	367.48		#CUDA driver version

configs/test.config

@@ -1,30 +1,33 @@
 flag:		testdpBenchmark		#Flag of current experiment
-tools:		tensorflow			#Tools to benchmark
-experiments: #<network type>; <network name>;  <device id>; <gpu count>;  <batch size>;  <number of epochs>;  <epoch size>; <Learning rate>
+tools:		mxnet			#Tools to benchmark
+experiments: #<network type>; <network name>;  <device id>; <device count>;  <batch size>;  <number of epochs>;  <epoch size>; <Learning rate>
 {
 		fc;		fcn5;		0;		1;		4096;		2;		60000;		0.05
+		fc;		fcn5;		-1;		20;		4096;		2;		60000;		0.05
 #		fc;		fcn5;		0;		1;		2048;		2;		60000;		0.05
 #		fc;		fcn5;		0;		1;		1024;		2;		60000;		0.05
 #		fc;		fcn5;		0;		1;		512;	   	2;		60000;		0.05
 #		fc;		fcn5;		0;		1;		342;	   	2;		60000;		0.05
 		cnn;		alexnet;	0;		1;		1024;		2;		50000;		0.01
+		cnn;		alexnet;	-1;		20;		1024;		2;		50000;		0.01
 #		cnn;		alexnet;	0;		1;		512;  	 	2;		50000;		0.01
 #		cnn;		alexnet;	0;		1;		256;	   	2;		50000;		0.01
 #		cnn;		alexnet;	0;		1;		128;	   	2;		50000;		0.01
 #		cnn;		alexnet;	0;		1;		86;   		2;		50000;		0.01
 		cnn;		resnet;		0;		1;		128;		2;		50000;		0.01
+		cnn;		resnet;		-1;		20;		128;		2;		50000;		0.01
 #		cnn;		resnet;		0;		1;		64;		2;		50000;		0.01
 #		cnn;		resnet;		0;		1;		32;		2;		50000;		0.01
 #		cnn;		resnet;		0;		1;		16;		2;		50000;		0.01
 #		cnn;		resnet;		0;		1;		11;		2;		50000;		0.01
-		rnn;		lstm;		0;		1;		1024;		2;		50000;		0.1
-#		rnn;		lstm;		0;		1;		512;   		2;		50000;		0.1
-#		rnn;		lstm;		0;		1;		256;   		2;		50000;		0.1
-#		rnn;		lstm;		0;		1;		128;   		2;		50000;		0.1
+		rnn;		lstm;		0;		1;		1024;		2;		-1;		0.01
+#		rnn;		lstm;		0;		1;		512;   		2;		-1;		0.01
+#		rnn;		lstm;		0;		1;		256;   		2;		-1;		0.01
+#		rnn;		lstm;		0;		1;		128;   		2;		-1;		0.01
 }
 host_file:	None		#Path to host file or None
-cpu_name:	E5-2630v3	#CPU model
-device_name:	K80		#GPU model
+cpu_name:	E5-2630v4	#CPU model
+gpu_name:	K80		#device model
 cuda:		8.0		#CUDA version
 cudnn:		5.1		#CUDNN version
 cuda_driver:	367.48		#CUDA driver version

tools/cntk/cnn/resnet/resnet.cntk

@@ -1,7 +1,7 @@
 RootDir = "."
 
 ConfigDir = "$RootDir$"
-DataDir = "/home/comp/csshshi/data/cntk/cifar10"
+DataDir = "/home/comp/pengfeixu/data/cntk/cifar10"
 #DataDir = "/home/comp/pengfeixu/Data/cntk/cifar10"
 #DataDir = "/home/ipdps/Data/cntk/cifar10"
 OutputDir = "$RootDir$/Output"