Dumping of Per driver results intel-cloud#221

liyuanwa · Jan 22, 2015 · fcd3f30 · fcd3f30
1 parent 1f49173
commit fcd3f30
Show file tree

Hide file tree

Showing 12 changed files with 263 additions and 6 deletions.
diff --git a/...osbench-controller/src/com/intel/cosbench/controller/archiver/SimpleWorkloadArchiver.java b/...osbench-controller/src/com/intel/cosbench/controller/archiver/SimpleWorkloadArchiver.java
@@ -18,8 +18,11 @@
 package com.intel.cosbench.controller.archiver;
 
 import java.io.*;
+import java.util.List;
 import java.util.Scanner;
 
+import com.intel.cosbench.bench.Metrics;
+import com.intel.cosbench.bench.TaskReport;
 import com.intel.cosbench.config.*;
 import com.intel.cosbench.config.castor.CastorConfigTools;
 import com.intel.cosbench.exporter.*;
@@ -89,9 +92,28 @@ private void doArchive(WorkloadInfo info, File runDir) throws IOException {
         exportConfig(info.getWorkload(), runDir);
         exportLog(info, runDir);
         exportScriptsLog(info, runDir);
-        exportPerformanceMatrix(info);
+        exportPerformanceMatrix(info);    
+        exportTaskInfo(info,runDir);
     }
 
+    private void exportTaskInfo(WorkloadInfo info,File parent)throws IOException{
+		for(DriverInfo dInfo:info.getDriverInfos()){
+			File file = new File(parent, dInfo.getName() + ".csv");
+	    	Writer writer = new BufferedWriter(new FileWriter(file));
+	    	 TaskExporter exporter = Exporters.newTaskExporter(info,dInfo);
+	    	 try {
+	             exporter.export(writer);
+	         } finally {
+	             writer.close();
+	         }
+	         String name = dInfo.getName();
+	         String path = file.getAbsolutePath();
+	         String msg = "perf details of {} has been exported to {}";
+	         LOGGER.debug(msg, name, path);
+		}	    
+    }
+
+
     private static String getRunDirName(WorkloadInfo info) {
         String name = info.getId();
         name += '-' + info.getWorkload().getName();

diff --git a/dev/cosbench-controller/src/com/intel/cosbench/controller/model/StageContext.java b/dev/cosbench-controller/src/com/intel/cosbench/controller/model/StageContext.java
@@ -46,8 +46,18 @@ public class StageContext implements StageInfo {
     private volatile Report report = null; // will be merged from task reports
 
     private transient List<StageListener> listeners = new ArrayList<StageListener>();
+
+    private List<TaskReport> taskReports = new ArrayList<TaskReport>();
+
+    public List<TaskReport> getTaskReports() {
+		return taskReports;
+	}
+
+	public void setTaskReports(List<TaskReport> taskReports) {
+		this.taskReports = taskReports;
+	}
 
-    public StageContext() {
+	public StageContext() {
         /* empty */
     }
 
@@ -93,8 +103,14 @@ public Report mergeReport() {
         if (taskRegistry == null)
             return new Report();
         ReportMerger merger = new ReportMerger();
-        for (TaskContext task : taskRegistry)
-            merger.add(task.getReport());
+        for (TaskContext task : taskRegistry){
+        	TaskReport tReport=new TaskReport();
+        	tReport.setReport(task.getReport());
+        	tReport.setDriverName(task.getSchedule().getDriver().getName());
+        	tReport.setDriverUrl(task.getSchedule().getDriver().getUrl());
+        	taskReports.add(tReport);
+        	merger.add(task.getReport());
+        }
         return merger.merge();
     }
 

diff --git a/dev/cosbench-controller/src/com/intel/cosbench/controller/model/WorkloadContext.java b/dev/cosbench-controller/src/com/intel/cosbench/controller/model/WorkloadContext.java
@@ -22,11 +22,13 @@
 
 import com.intel.cosbench.bench.*;
 import com.intel.cosbench.config.*;
+import com.intel.cosbench.log.LogFactory;
 import com.intel.cosbench.log.Logger;
 import com.intel.cosbench.model.*;
 
 public class WorkloadContext implements WorkloadInfo {
 
+	private static final Logger LOGGER = LogFactory.getSystemLogger();
     private String id;
     private Date submitDate;
     private Date startDate;

diff --git a/dev/cosbench-controller/src/com/intel/cosbench/controller/repository/WorkloadList.java b/dev/cosbench-controller/src/com/intel/cosbench/controller/repository/WorkloadList.java
@@ -21,7 +21,8 @@
 
 import com.intel.cosbench.controller.model.WorkloadContext;
 
-interface WorkloadList {
+interface WorkloadList
+	{
 
     public int count();
 

diff --git a/dev/cosbench-core/src/com/intel/cosbench/bench/Metrics.java b/dev/cosbench-core/src/com/intel/cosbench/bench/Metrics.java
@@ -229,4 +229,15 @@ public static Metrics convert(Mark mark, long window) {
         return metrics;
     }
 
+	@Override
+	public String toString() {
+		return "Metrics [name=" + name + ", opType=" + opType + ", sampleType="
+				+ sampleType + ", opName=" + opName + ", opId=" + opId
+				+ ", sampleCount=" + sampleCount + ", totalSampleCount="
+				+ totalSampleCount + ", byteCount=" + byteCount
+				+ ", workerCount=" + workerCount + ", avgResTime=" + avgResTime
+				+ ", avgXferTime=" + avgXferTime + ", throughput=" + throughput
+				+ ", bandwidth=" + bandwidth + ", latency=" + latency
+				+ ", ratio=" + ratio + "]";
+	}
 }
diff --git a/dev/cosbench-core/src/com/intel/cosbench/bench/Report.java b/dev/cosbench-core/src/com/intel/cosbench/bench/Report.java
@@ -40,5 +40,6 @@ public Metrics getMetrics(String name) {
     public Metrics[] getAllMetrics() {
         return getAllItems().toArray(new Metrics[getSize()]);
     }
+
 
 }
diff --git a/dev/cosbench-core/src/com/intel/cosbench/bench/TaskReport.java b/dev/cosbench-core/src/com/intel/cosbench/bench/TaskReport.java
@@ -0,0 +1,31 @@
+package com.intel.cosbench.bench;
+/**
+ * The class is the data structure of task info
+ * 
+ * @author liyuan 
+ *
+ */
+public class TaskReport {
+	private String driverName;
+	private String driverUrl;
+	private Report report;
+	public String getDriverName() {
+		return driverName;
+	}
+	public void setDriverName(String driverName) {
+		this.driverName = driverName;
+	}
+	public String getDriverUrl() {
+		return driverUrl;
+	}
+	public void setDriverUrl(String driverUrl) {
+		this.driverUrl = driverUrl;
+	}
+	public Report getReport() {
+		return report;
+	}
+	public void setReport(Report report) {
+		this.report = report;
+	}
+
+}
diff --git a/dev/cosbench-core/src/com/intel/cosbench/exporter/AbstractTaskExporter.java b/dev/cosbench-core/src/com/intel/cosbench/exporter/AbstractTaskExporter.java
@@ -0,0 +1,62 @@
+package com.intel.cosbench.exporter;
+
+import java.io.IOException;
+import java.io.Writer;
+import java.util.ArrayList;
+import java.util.List;
+
+import com.intel.cosbench.bench.Snapshot;
+import com.intel.cosbench.bench.TaskReport;
+import com.intel.cosbench.model.DriverInfo;
+import com.intel.cosbench.model.StageInfo;
+import com.intel.cosbench.model.WorkloadInfo;
+
+public abstract class AbstractTaskExporter implements TaskExporter {
+
+    protected WorkloadInfo workloadInfo;
+    protected DriverInfo driverInfo;
+    protected List<TaskReport> driverTasks = new ArrayList<TaskReport>(); 
+
+	public WorkloadInfo getWorkloadInfo() {
+		return workloadInfo;
+	}
+
+	public void setWorkloadInfo(WorkloadInfo workloadInfo) {
+		this.workloadInfo = workloadInfo;
+	}
+
+	public DriverInfo getDriverInfo() {
+		return driverInfo;
+	}
+
+	public void setDriverInfo(DriverInfo driverInfo) {
+		this.driverInfo = driverInfo;
+	}
+
+
+	 @Override
+    public void export(Writer writer) throws IOException {
+		 //check every stage every task   
+		for(StageInfo sInfo:workloadInfo.getStageInfos()){
+			for(TaskReport tReport:sInfo.getTaskReports()){
+				if(tReport.getDriverUrl().equals(driverInfo.getUrl())){
+					driverTasks.add(tReport);
+				}
+			}
+		}
+        writeReport(writer);
+    }
+
+    private void writeReport(Writer writer) throws IOException {
+        writeHeader(writer);
+        writer.flush();
+        for (TaskReport tReport : driverTasks)
+            writeMetrics(writer, tReport);
+        writer.flush();
+    }
+
+    protected abstract void writeHeader(Writer writer) throws IOException;
+
+    protected abstract void writeMetrics(Writer writer, TaskReport tReport)
+            throws IOException;
+}
diff --git a/dev/cosbench-core/src/com/intel/cosbench/exporter/CSVTaskExporter.java b/dev/cosbench-core/src/com/intel/cosbench/exporter/CSVTaskExporter.java
@@ -0,0 +1,88 @@
+package com.intel.cosbench.exporter;
+
+import static com.intel.cosbench.exporter.Formats.NUM;
+import static com.intel.cosbench.exporter.Formats.RATIO;
+import static com.intel.cosbench.exporter.Formats.TIME;
+
+import java.io.IOException;
+import java.io.Writer;
+import java.util.Arrays;
+
+import org.apache.commons.lang.StringUtils;
+
+import com.intel.cosbench.bench.Metrics;
+import com.intel.cosbench.bench.Report;
+import com.intel.cosbench.bench.Snapshot;
+import com.intel.cosbench.bench.TaskReport;
+
+public class CSVTaskExporter extends AbstractTaskExporter{
+
+	public CSVTaskExporter() {
+        /* empty */
+    }
+    protected void writeHeader(Writer writer) throws IOException {
+        StringBuilder buffer = new StringBuilder();
+        char[] cs = new char[8];
+        buffer.append("Op-Type").append(',');
+        buffer.append("Sample-Type").append(',');
+        buffer.append("Op-Count").append(',');
+        buffer.append("Byte-Count").append(',');
+        buffer.append("Avg-ResTime").append(',');
+        buffer.append("Avg-ProcTime").append(',');
+        buffer.append("Throughput").append(',');
+        buffer.append("Bandwidth").append(',');
+        buffer.append("Succ-Ratio").append('\n');
+        writer.write(buffer.toString());
+    }
+
+    protected void writeMetrics(Writer writer,TaskReport tReport)throws IOException {
+        StringBuilder buffer = new StringBuilder();
+        Report report = tReport.getReport();
+        /*Operation Type*/
+        for(Metrics metrics:report)
+        	buffer.append(metrics.getOpType()).append(',');
+        /*sample Type*/
+        for(Metrics metrics:report)
+        	buffer.append(metrics.getSampleType()).append(',');
+        /* Operation Count */
+        for (Metrics metrics :report)
+            buffer.append(metrics.getSampleCount()).append(',');
+        /* Byte Count */
+        for (Metrics metrics : report)
+            buffer.append(metrics.getByteCount()).append(',');
+        /* Response Time */
+        for (Metrics metrics : report) {
+            double r = metrics.getAvgResTime();
+            if (r > 0)
+                buffer.append(NUM.format(r));
+            else
+                buffer.append("N/A");
+            buffer.append(',');
+        }
+        /* Transfer Time */
+        for (Metrics metrics : report) {
+            double pt = metrics.getAvgResTime() - metrics.getAvgXferTime();
+            if (pt > 0)
+                buffer.append(NUM.format(pt));
+            else
+                buffer.append("N/A");
+            buffer.append(',');
+        }
+        /* Throughput */
+        for (Metrics metrics : report)
+            buffer.append(NUM.format(metrics.getThroughput())).append(',');
+        /* Bandwidth */
+        for (Metrics metrics : report)
+            buffer.append(NUM.format(metrics.getBandwidth())).append(',');
+        /* Success Ratio */
+        for (Metrics metrics : report) {
+            double t = (double) metrics.getRatio();
+            if (t > 0)
+                buffer.append(RATIO.format(metrics.getRatio()));
+            else
+                buffer.append("N/A");
+            buffer.append('\n');
+        }
+        writer.write(buffer.toString());
+    }
+}
diff --git a/dev/cosbench-core/src/com/intel/cosbench/exporter/Exporters.java b/dev/cosbench-core/src/com/intel/cosbench/exporter/Exporters.java
@@ -17,15 +17,23 @@
 
 package com.intel.cosbench.exporter;
 
+import com.intel.cosbench.bench.TaskReport;
 import com.intel.cosbench.model.*;
 
 /**
  * This class is a wrapper to construct different exporters by providing static construction methods.
  * 
- * @author ywang19, qzheng7
+ * @author ywang19, qzheng7 
  *
  */
 public class Exporters {
+
+	public static TaskExporter newTaskExporter(WorkloadInfo workloadInfo,DriverInfo driverInfo){
+		AbstractTaskExporter exporter = new CSVTaskExporter();
+        exporter.setWorkloadInfo(workloadInfo);
+        exporter.setDriverInfo(driverInfo);
+        return exporter;
+	}
 
     public static RunExporter newRunExporter(WorkloadInfo workload) {
         AbstractRunExporter exporter = new CSVRunExporter();
@@ -68,5 +76,7 @@ public static MatrixExporter newMatrixExporter(WorkloadInfo workload) {
         exporter.setWorkload(workload);
         return exporter;
     }
+
+
 
 }
diff --git a/dev/cosbench-core/src/com/intel/cosbench/exporter/TaskExporter.java b/dev/cosbench-core/src/com/intel/cosbench/exporter/TaskExporter.java
@@ -0,0 +1,8 @@
+package com.intel.cosbench.exporter;
+
+import java.io.IOException;
+import java.io.Writer;
+
+public interface TaskExporter {
+	  public void export(Writer writer) throws IOException;
+}
diff --git a/dev/cosbench-core/src/com/intel/cosbench/model/StageInfo.java b/dev/cosbench-core/src/com/intel/cosbench/model/StageInfo.java
@@ -18,6 +18,7 @@
 package com.intel.cosbench.model;
 
 import java.util.Date;
+import java.util.List;
 import java.util.Set;
 
 import com.intel.cosbench.bench.*;
@@ -57,5 +58,9 @@ public interface StageInfo extends LifeCycle, Benchmark {
 	public void setReport(Report report);
 
 	public ListRegistry<Snapshot> getSnapshotRegistry();
+
+    public List<TaskReport> getTaskReports();
+
+	public void setTaskReports(List<TaskReport> taskReports);
 
 }