diff --git a/dev/cosbench-controller/src/com/intel/cosbench/controller/archiver/SimpleWorkloadArchiver.java b/dev/cosbench-controller/src/com/intel/cosbench/controller/archiver/SimpleWorkloadArchiver.java index 13f00367..4e9b45ee 100644 --- a/dev/cosbench-controller/src/com/intel/cosbench/controller/archiver/SimpleWorkloadArchiver.java +++ b/dev/cosbench-controller/src/com/intel/cosbench/controller/archiver/SimpleWorkloadArchiver.java @@ -18,8 +18,11 @@ package com.intel.cosbench.controller.archiver; import java.io.*; +import java.util.List; import java.util.Scanner; +import com.intel.cosbench.bench.Metrics; +import com.intel.cosbench.bench.TaskReport; import com.intel.cosbench.config.*; import com.intel.cosbench.config.castor.CastorConfigTools; import com.intel.cosbench.exporter.*; @@ -89,9 +92,28 @@ private void doArchive(WorkloadInfo info, File runDir) throws IOException { exportConfig(info.getWorkload(), runDir); exportLog(info, runDir); exportScriptsLog(info, runDir); - exportPerformanceMatrix(info); + exportPerformanceMatrix(info); + exportTaskInfo(info,runDir); } + private void exportTaskInfo(WorkloadInfo info,File parent)throws IOException{ + for(DriverInfo dInfo:info.getDriverInfos()){ + File file = new File(parent, dInfo.getName() + ".csv"); + Writer writer = new BufferedWriter(new FileWriter(file)); + TaskExporter exporter = Exporters.newTaskExporter(info,dInfo); + try { + exporter.export(writer); + } finally { + writer.close(); + } + String name = dInfo.getName(); + String path = file.getAbsolutePath(); + String msg = "perf details of {} has been exported to {}"; + LOGGER.debug(msg, name, path); + } + } + + private static String getRunDirName(WorkloadInfo info) { String name = info.getId(); name += '-' + info.getWorkload().getName(); diff --git a/dev/cosbench-controller/src/com/intel/cosbench/controller/model/StageContext.java b/dev/cosbench-controller/src/com/intel/cosbench/controller/model/StageContext.java index c081c7a4..2ae443a0 100644 --- a/dev/cosbench-controller/src/com/intel/cosbench/controller/model/StageContext.java +++ b/dev/cosbench-controller/src/com/intel/cosbench/controller/model/StageContext.java @@ -46,8 +46,18 @@ public class StageContext implements StageInfo { private volatile Report report = null; // will be merged from task reports private transient List listeners = new ArrayList(); + + private List taskReports = new ArrayList(); + + public List getTaskReports() { + return taskReports; + } + + public void setTaskReports(List taskReports) { + this.taskReports = taskReports; + } - public StageContext() { + public StageContext() { /* empty */ } @@ -93,8 +103,14 @@ public Report mergeReport() { if (taskRegistry == null) return new Report(); ReportMerger merger = new ReportMerger(); - for (TaskContext task : taskRegistry) - merger.add(task.getReport()); + for (TaskContext task : taskRegistry){ + TaskReport tReport=new TaskReport(); + tReport.setReport(task.getReport()); + tReport.setDriverName(task.getSchedule().getDriver().getName()); + tReport.setDriverUrl(task.getSchedule().getDriver().getUrl()); + taskReports.add(tReport); + merger.add(task.getReport()); + } return merger.merge(); } diff --git a/dev/cosbench-controller/src/com/intel/cosbench/controller/model/WorkloadContext.java b/dev/cosbench-controller/src/com/intel/cosbench/controller/model/WorkloadContext.java index 1debe4a5..4feb7cd7 100644 --- a/dev/cosbench-controller/src/com/intel/cosbench/controller/model/WorkloadContext.java +++ b/dev/cosbench-controller/src/com/intel/cosbench/controller/model/WorkloadContext.java @@ -22,11 +22,13 @@ import com.intel.cosbench.bench.*; import com.intel.cosbench.config.*; +import com.intel.cosbench.log.LogFactory; import com.intel.cosbench.log.Logger; import com.intel.cosbench.model.*; public class WorkloadContext implements WorkloadInfo { + private static final Logger LOGGER = LogFactory.getSystemLogger(); private String id; private Date submitDate; private Date startDate; diff --git a/dev/cosbench-controller/src/com/intel/cosbench/controller/repository/WorkloadList.java b/dev/cosbench-controller/src/com/intel/cosbench/controller/repository/WorkloadList.java index 0feee62e..1ab453ee 100644 --- a/dev/cosbench-controller/src/com/intel/cosbench/controller/repository/WorkloadList.java +++ b/dev/cosbench-controller/src/com/intel/cosbench/controller/repository/WorkloadList.java @@ -21,7 +21,8 @@ import com.intel.cosbench.controller.model.WorkloadContext; -interface WorkloadList { +interface WorkloadList + { public int count(); diff --git a/dev/cosbench-core/src/com/intel/cosbench/bench/Metrics.java b/dev/cosbench-core/src/com/intel/cosbench/bench/Metrics.java index b1fa5c96..1b2642ad 100644 --- a/dev/cosbench-core/src/com/intel/cosbench/bench/Metrics.java +++ b/dev/cosbench-core/src/com/intel/cosbench/bench/Metrics.java @@ -229,4 +229,15 @@ public static Metrics convert(Mark mark, long window) { return metrics; } + @Override + public String toString() { + return "Metrics [name=" + name + ", opType=" + opType + ", sampleType=" + + sampleType + ", opName=" + opName + ", opId=" + opId + + ", sampleCount=" + sampleCount + ", totalSampleCount=" + + totalSampleCount + ", byteCount=" + byteCount + + ", workerCount=" + workerCount + ", avgResTime=" + avgResTime + + ", avgXferTime=" + avgXferTime + ", throughput=" + throughput + + ", bandwidth=" + bandwidth + ", latency=" + latency + + ", ratio=" + ratio + "]"; + } } diff --git a/dev/cosbench-core/src/com/intel/cosbench/bench/Report.java b/dev/cosbench-core/src/com/intel/cosbench/bench/Report.java index 43b3196d..5b1e014f 100644 --- a/dev/cosbench-core/src/com/intel/cosbench/bench/Report.java +++ b/dev/cosbench-core/src/com/intel/cosbench/bench/Report.java @@ -40,5 +40,6 @@ public Metrics getMetrics(String name) { public Metrics[] getAllMetrics() { return getAllItems().toArray(new Metrics[getSize()]); } + } diff --git a/dev/cosbench-core/src/com/intel/cosbench/bench/TaskReport.java b/dev/cosbench-core/src/com/intel/cosbench/bench/TaskReport.java new file mode 100644 index 00000000..39a0b2bb --- /dev/null +++ b/dev/cosbench-core/src/com/intel/cosbench/bench/TaskReport.java @@ -0,0 +1,31 @@ +package com.intel.cosbench.bench; +/** + * The class is the data structure of task info + * + * @author liyuan + * + */ +public class TaskReport { + private String driverName; + private String driverUrl; + private Report report; + public String getDriverName() { + return driverName; + } + public void setDriverName(String driverName) { + this.driverName = driverName; + } + public String getDriverUrl() { + return driverUrl; + } + public void setDriverUrl(String driverUrl) { + this.driverUrl = driverUrl; + } + public Report getReport() { + return report; + } + public void setReport(Report report) { + this.report = report; + } + +} diff --git a/dev/cosbench-core/src/com/intel/cosbench/exporter/AbstractTaskExporter.java b/dev/cosbench-core/src/com/intel/cosbench/exporter/AbstractTaskExporter.java new file mode 100644 index 00000000..473c237e --- /dev/null +++ b/dev/cosbench-core/src/com/intel/cosbench/exporter/AbstractTaskExporter.java @@ -0,0 +1,62 @@ +package com.intel.cosbench.exporter; + +import java.io.IOException; +import java.io.Writer; +import java.util.ArrayList; +import java.util.List; + +import com.intel.cosbench.bench.Snapshot; +import com.intel.cosbench.bench.TaskReport; +import com.intel.cosbench.model.DriverInfo; +import com.intel.cosbench.model.StageInfo; +import com.intel.cosbench.model.WorkloadInfo; + +public abstract class AbstractTaskExporter implements TaskExporter { + + protected WorkloadInfo workloadInfo; + protected DriverInfo driverInfo; + protected List driverTasks = new ArrayList(); + + public WorkloadInfo getWorkloadInfo() { + return workloadInfo; + } + + public void setWorkloadInfo(WorkloadInfo workloadInfo) { + this.workloadInfo = workloadInfo; + } + + public DriverInfo getDriverInfo() { + return driverInfo; + } + + public void setDriverInfo(DriverInfo driverInfo) { + this.driverInfo = driverInfo; + } + + + @Override + public void export(Writer writer) throws IOException { + //check every stage every task + for(StageInfo sInfo:workloadInfo.getStageInfos()){ + for(TaskReport tReport:sInfo.getTaskReports()){ + if(tReport.getDriverUrl().equals(driverInfo.getUrl())){ + driverTasks.add(tReport); + } + } + } + writeReport(writer); + } + + private void writeReport(Writer writer) throws IOException { + writeHeader(writer); + writer.flush(); + for (TaskReport tReport : driverTasks) + writeMetrics(writer, tReport); + writer.flush(); + } + + protected abstract void writeHeader(Writer writer) throws IOException; + + protected abstract void writeMetrics(Writer writer, TaskReport tReport) + throws IOException; +} diff --git a/dev/cosbench-core/src/com/intel/cosbench/exporter/CSVTaskExporter.java b/dev/cosbench-core/src/com/intel/cosbench/exporter/CSVTaskExporter.java new file mode 100644 index 00000000..fb45e944 --- /dev/null +++ b/dev/cosbench-core/src/com/intel/cosbench/exporter/CSVTaskExporter.java @@ -0,0 +1,88 @@ +package com.intel.cosbench.exporter; + +import static com.intel.cosbench.exporter.Formats.NUM; +import static com.intel.cosbench.exporter.Formats.RATIO; +import static com.intel.cosbench.exporter.Formats.TIME; + +import java.io.IOException; +import java.io.Writer; +import java.util.Arrays; + +import org.apache.commons.lang.StringUtils; + +import com.intel.cosbench.bench.Metrics; +import com.intel.cosbench.bench.Report; +import com.intel.cosbench.bench.Snapshot; +import com.intel.cosbench.bench.TaskReport; + +public class CSVTaskExporter extends AbstractTaskExporter{ + + public CSVTaskExporter() { + /* empty */ + } + protected void writeHeader(Writer writer) throws IOException { + StringBuilder buffer = new StringBuilder(); + char[] cs = new char[8]; + buffer.append("Op-Type").append(','); + buffer.append("Sample-Type").append(','); + buffer.append("Op-Count").append(','); + buffer.append("Byte-Count").append(','); + buffer.append("Avg-ResTime").append(','); + buffer.append("Avg-ProcTime").append(','); + buffer.append("Throughput").append(','); + buffer.append("Bandwidth").append(','); + buffer.append("Succ-Ratio").append('\n'); + writer.write(buffer.toString()); + } + + protected void writeMetrics(Writer writer,TaskReport tReport)throws IOException { + StringBuilder buffer = new StringBuilder(); + Report report = tReport.getReport(); + /*Operation Type*/ + for(Metrics metrics:report) + buffer.append(metrics.getOpType()).append(','); + /*sample Type*/ + for(Metrics metrics:report) + buffer.append(metrics.getSampleType()).append(','); + /* Operation Count */ + for (Metrics metrics :report) + buffer.append(metrics.getSampleCount()).append(','); + /* Byte Count */ + for (Metrics metrics : report) + buffer.append(metrics.getByteCount()).append(','); + /* Response Time */ + for (Metrics metrics : report) { + double r = metrics.getAvgResTime(); + if (r > 0) + buffer.append(NUM.format(r)); + else + buffer.append("N/A"); + buffer.append(','); + } + /* Transfer Time */ + for (Metrics metrics : report) { + double pt = metrics.getAvgResTime() - metrics.getAvgXferTime(); + if (pt > 0) + buffer.append(NUM.format(pt)); + else + buffer.append("N/A"); + buffer.append(','); + } + /* Throughput */ + for (Metrics metrics : report) + buffer.append(NUM.format(metrics.getThroughput())).append(','); + /* Bandwidth */ + for (Metrics metrics : report) + buffer.append(NUM.format(metrics.getBandwidth())).append(','); + /* Success Ratio */ + for (Metrics metrics : report) { + double t = (double) metrics.getRatio(); + if (t > 0) + buffer.append(RATIO.format(metrics.getRatio())); + else + buffer.append("N/A"); + buffer.append('\n'); + } + writer.write(buffer.toString()); + } +} diff --git a/dev/cosbench-core/src/com/intel/cosbench/exporter/Exporters.java b/dev/cosbench-core/src/com/intel/cosbench/exporter/Exporters.java index 1aa86dce..455d7736 100644 --- a/dev/cosbench-core/src/com/intel/cosbench/exporter/Exporters.java +++ b/dev/cosbench-core/src/com/intel/cosbench/exporter/Exporters.java @@ -17,15 +17,23 @@ package com.intel.cosbench.exporter; +import com.intel.cosbench.bench.TaskReport; import com.intel.cosbench.model.*; /** * This class is a wrapper to construct different exporters by providing static construction methods. * - * @author ywang19, qzheng7 + * @author ywang19, qzheng7 * */ public class Exporters { + + public static TaskExporter newTaskExporter(WorkloadInfo workloadInfo,DriverInfo driverInfo){ + AbstractTaskExporter exporter = new CSVTaskExporter(); + exporter.setWorkloadInfo(workloadInfo); + exporter.setDriverInfo(driverInfo); + return exporter; + } public static RunExporter newRunExporter(WorkloadInfo workload) { AbstractRunExporter exporter = new CSVRunExporter(); @@ -68,5 +76,7 @@ public static MatrixExporter newMatrixExporter(WorkloadInfo workload) { exporter.setWorkload(workload); return exporter; } + + } diff --git a/dev/cosbench-core/src/com/intel/cosbench/exporter/TaskExporter.java b/dev/cosbench-core/src/com/intel/cosbench/exporter/TaskExporter.java new file mode 100644 index 00000000..b41c5050 --- /dev/null +++ b/dev/cosbench-core/src/com/intel/cosbench/exporter/TaskExporter.java @@ -0,0 +1,8 @@ +package com.intel.cosbench.exporter; + +import java.io.IOException; +import java.io.Writer; + +public interface TaskExporter { + public void export(Writer writer) throws IOException; +} diff --git a/dev/cosbench-core/src/com/intel/cosbench/model/StageInfo.java b/dev/cosbench-core/src/com/intel/cosbench/model/StageInfo.java index ad022bf7..15d718fc 100644 --- a/dev/cosbench-core/src/com/intel/cosbench/model/StageInfo.java +++ b/dev/cosbench-core/src/com/intel/cosbench/model/StageInfo.java @@ -18,6 +18,7 @@ package com.intel.cosbench.model; import java.util.Date; +import java.util.List; import java.util.Set; import com.intel.cosbench.bench.*; @@ -57,5 +58,9 @@ public interface StageInfo extends LifeCycle, Benchmark { public void setReport(Report report); public ListRegistry getSnapshotRegistry(); + + public List getTaskReports(); + + public void setTaskReports(List taskReports); }