Skip to content

Commit

Permalink
changes
Browse files Browse the repository at this point in the history
Signed-off-by: Bharathwaj G <[email protected]>
  • Loading branch information
bharath-techie committed Sep 27, 2023
1 parent e52c988 commit 37be142
Show file tree
Hide file tree
Showing 8 changed files with 35 additions and 72 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -214,7 +214,7 @@ public enum Metric {
FILE_CACHE_STATS("file_cache"),
TASK_CANCELLATION("task_cancellation"),
SEARCH_PIPELINE("search_pipeline"),
NODES_PERFORMANCE_STATS("nodes_performance_stats");
GLOBAL_PERFORMANCE_STATS("performance_stats");

private String metricName;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -125,7 +125,7 @@ protected NodeStats nodeOperation(NodeStatsRequest nodeStatsRequest) {
NodesStatsRequest.Metric.FILE_CACHE_STATS.containedIn(metrics),
NodesStatsRequest.Metric.TASK_CANCELLATION.containedIn(metrics),
NodesStatsRequest.Metric.SEARCH_PIPELINE.containedIn(metrics),
NodesStatsRequest.Metric.NODES_PERFORMANCE_STATS.containedIn(metrics)
NodesStatsRequest.Metric.GLOBAL_PERFORMANCE_STATS.containedIn(metrics)
);
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -228,7 +228,6 @@ private void monitorFSHealth() {
}
Files.delete(tempDataPath);
final long elapsedTime = currentTimeMillisSupplier.getAsLong() - executionStartTime;
logger.info("health check took {}", elapsedTime);
if (elapsedTime > slowPathLoggingThreshold.millis()) {
logger.warn(
"health check of [{}] took [{}ms] which is above the warn threshold of [{}]",
Expand Down
32 changes: 3 additions & 29 deletions server/src/main/java/org/opensearch/monitor/fs/FsInfo.java
Original file line number Diff line number Diff line change
Expand Up @@ -226,7 +226,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
* @opensearch.internal]
*/
public static class DeviceStats implements Writeable, ToXContentFragment {
private static final Logger logger = LogManager.getLogger(DeviceStats.class);

final int majorDeviceNumber;
final int minorDeviceNumber;
Expand Down Expand Up @@ -367,14 +366,11 @@ public long operations() {
public long readOperations() {
if (previousReadsCompleted == -1) return -1;

//logger.info("Current reads : {} , Previous reads : {}", currentReadsCompleted, previousReadsCompleted);

return (currentReadsCompleted - previousReadsCompleted);
}

public long writeOperations() {
if (previousWritesCompleted == -1) return -1;
//logger.info("Current writes : {} , Previous writes : {}", currentWritesCompleted, previousWritesCompleted);

return (currentWritesCompleted - previousWritesCompleted);
}
Expand Down Expand Up @@ -413,28 +409,6 @@ public long ioTimeInMillis() {
return (currentIOTime - previousIOTime);
}

public double getNewWriteLatency() {
//double readLatency = getReadTime() / readOperations();
double writeLatency = getWriteTime() / writeOperations();
return writeLatency;
}

public double getNewReadLatency() {
//double readLatency = getReadTime() / readOperations();
double readLatency = getReadTime() / readOperations();
return readLatency;
}

public double getReadTime() {
if(previousReadTime == -1.0) return -1.0;
return currentReadTime - previousReadTime;
}

public double getWriteTime() {
if(previousWriteTime == -1.0) return -1.0;
return currentWriteTime - previousWriteTime;
}

public long getCurrentIOTime() {
return this.currentIOTime;
}
Expand All @@ -459,7 +433,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
builder.field(IoStats.WRITE_OPERATIONS, writeOperations());
builder.field(IoStats.READ_KILOBYTES, readKilobytes());
builder.field(IoStats.WRITE_KILOBYTES, writeKilobytes());
builder.field(IoStats.IO_TIME_MS, ioTimeInMillis());
builder.field(IoStats.IO_TIME_MILLIS, ioTimeInMillis());
return builder;
}

Expand All @@ -477,7 +451,7 @@ public static class IoStats implements Writeable, ToXContentFragment {
private static final String WRITE_OPERATIONS = "write_operations";
private static final String READ_KILOBYTES = "read_kilobytes";
private static final String WRITE_KILOBYTES = "write_kilobytes";
private static final String IO_TIME_MS = "io_time_in_millis";
private static final String IO_TIME_MILLIS = "io_time_in_millis";

final DeviceStats[] devicesStats;
final long totalOperations;
Expand Down Expand Up @@ -585,7 +559,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
builder.field(WRITE_OPERATIONS, totalWriteOperations);
builder.field(READ_KILOBYTES, totalReadKilobytes);
builder.field(WRITE_KILOBYTES, totalWriteKilobytes);
builder.field(IO_TIME_MS, totalIOTimeInMillis);
builder.field(IO_TIME_MILLIS, totalIOTimeInMillis);
builder.endObject();
}
return builder;
Expand Down
4 changes: 0 additions & 4 deletions server/src/main/java/org/opensearch/monitor/fs/FsProbe.java
Original file line number Diff line number Diff line change
Expand Up @@ -125,8 +125,6 @@ final FsInfo.IoStats ioStats(final Set<Tuple<Integer, Integer>> devicesNumbers,
final long sectorsWritten = Long.parseLong(fields[9]);
final double readTime = Double.parseDouble(fields[6]);
final double writeTime = Double.parseDouble(fields[10]);
final double readLatency = readTime / readsCompleted;
final double writeLatency = writeTime / writesCompleted;
final long ioTime = Long.parseLong(fields[12]);
final FsInfo.DeviceStats deviceStats = new FsInfo.DeviceStats(
majorDeviceNumber,
Expand All @@ -139,8 +137,6 @@ final FsInfo.IoStats ioStats(final Set<Tuple<Integer, Integer>> devicesNumbers,
ioTime,
readTime,
writeTime,
readLatency,
writeLatency,
deviceMap.get(Tuple.tuple(majorDeviceNumber, minorDeviceNumber))
);
devicesStats.add(deviceStats);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ public Map<String, NodePerformanceStatistics> getNodeIdToNodePerfStatsMap() {

@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject("nodes_performance_stats");
builder.startObject("performance_stats");
for (String nodeId : nodePerfStats.keySet()) {
builder.startObject(nodeId);
NodePerformanceStatistics perfStats = nodePerfStats.get(nodeId);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,10 @@

import java.util.concurrent.atomic.AtomicReference;

/**
* This class calculates the average of various IO stats such as IOPS, throughput and IO use percent
* for a given window and polling interval
*/
public class AverageIOUsageTracker extends AbstractLifecycleComponent {

private static final Logger logger = LogManager.getLogger(AverageCpuUsageTracker.class);
Expand All @@ -42,8 +46,6 @@ public class AverageIOUsageTracker extends AbstractLifecycleComponent {
private final AtomicReference<DoubleMovingAverage> readLatencyObservations = new AtomicReference<>();
private final AtomicReference<DoubleMovingAverage> writeLatencyObservations = new AtomicReference<>();

private long runs = 1;

public AverageIOUsageTracker(
ThreadPool threadPool,
TimeValue pollingInterval,
Expand Down Expand Up @@ -120,15 +122,9 @@ public void setWindowDuration(TimeValue windowDuration) {
}

private void recordUsage(IoUsageFetcher.DiskStats usage) {
if(usage.getIoTime() == 0.0) {
runs++;
return;
} else {
runs = 1;
}
ioTimeObservations.get().record(usage.getIoTime());
readIopsObservations.get().record((long)usage.getReadOps());
readKbObservations.get().record(usage.getReadkb());
readKbObservations.get().record(usage.getReadThroughputInKB());
double readOps = usage.getReadOps() < 1 ? 1.0 : usage.getReadOps() * 1.0;
double writeOps = usage.getWriteOps() < 1 ? 1.0 : usage.getWriteOps() * 1.0;
double readTime = usage.getReadTime() < 1 ? 0.0 : usage.getReadTime();
Expand All @@ -137,7 +133,7 @@ private void recordUsage(IoUsageFetcher.DiskStats usage) {
double writeLatency = (writeTime/ writeOps);// * runs;
writeLatencyObservations.get().record(writeLatency);
readLatencyObservations.get().record(readLatency);
writeKbObservations.get().record(usage.getWritekb());
writeKbObservations.get().record(usage.getWriteThroughputInKB());
writeIopsObservations.get().record((long) usage.getWriteOps());
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,12 @@
import java.util.HashMap;
import java.util.Map;

public class IoUsageFetcher {
private static final Logger logger = LogManager.getLogger(AverageCpuUsageTracker.class);
/**
* This class calculates the delta of IO utilization data such as IOPS, throughput, IO use percent etc.
* from FS stats across all data disks
*/
class IoUsageFetcher {
private static final Logger logger = LogManager.getLogger(IoUsageFetcher.class);
private Map<String, DiskStats> previousIOTimeMap;
private FsService fsService;
public IoUsageFetcher(FsService fsService){
Expand All @@ -30,16 +34,17 @@ class DiskStats {
public double writeTime;
public double readOps;
public double writeOps;
public long readkb;
public long writekb;
public DiskStats(long ioTime, double readTime, double writeTime, double readOps, double writeOps, long readkb, long writekb) {
public long readThroughputInKB;
public long writeThroughputInKB;
public DiskStats(long ioTime, double readTime, double writeTime, double readOps, double writeOps,
long readThroughputInKB, long writeThroughputInKB) {
this.ioTime = ioTime;
this.readTime = readTime;
this.writeTime = writeTime;
this.readOps = readOps;
this.writeOps = writeOps;
this.readkb = readkb;
this.writekb = writekb;
this.readThroughputInKB = readThroughputInKB;
this.writeThroughputInKB = writeThroughputInKB;
}

public long getIoTime() {
Expand All @@ -54,8 +59,8 @@ public double getReadTime() {
return readTime;
}

public long getReadkb() {
return readkb;
public long getReadThroughputInKB() {
return readThroughputInKB;
}

public double getWriteOps() {
Expand All @@ -66,8 +71,8 @@ public double getWriteTime() {
return writeTime;
}

public long getWritekb() {
return writekb;
public long getWriteThroughputInKB() {
return writeThroughputInKB;
}
}
public DiskStats getDiskUtilizationStats() {
Expand All @@ -77,42 +82,35 @@ public DiskStats getDiskUtilizationStats() {
long writekb = 0;
double readTime = 0;
double writeTime = 0;
double readLatency = 0.0;
double writeLatency = 0.0;
double readOps = 0.0;
double writeOps = 0.0;
// For non linux machines, this will be null
if(this.fsService.stats().getIoStats() == null) {
return null;
}
// Sum the stats across all data volumes
for (FsInfo.DeviceStats devicesStat : this.fsService.stats().getIoStats().getDevicesStats()) {
if (previousIOTimeMap != null && previousIOTimeMap.containsKey(devicesStat.getDeviceName())){
//logger.info(this.fsService.stats().getTimestamp());
long ioSpentTime = devicesStat.getCurrentIOTime() - previousIOTimeMap.get(devicesStat.getDeviceName()).ioTime;
ioUsePercent = (ioSpentTime * 100) / (1000);
ioUsePercent = ioSpentTime / 10;
readOps += devicesStat.currentReadOperations() - previousIOTimeMap.get(devicesStat.getDeviceName()).readOps;
writeOps += devicesStat.currentWriteOpetations() - previousIOTimeMap.get(devicesStat.getDeviceName()).writeOps;
readkb += devicesStat.getCurrentReadKilobytes() - previousIOTimeMap.get(devicesStat.getDeviceName()).readkb;
writekb += devicesStat.getCurrentWriteKilobytes() - previousIOTimeMap.get(devicesStat.getDeviceName()).writekb;
readkb += devicesStat.getCurrentReadKilobytes() - previousIOTimeMap.get(devicesStat.getDeviceName()).readThroughputInKB;
writekb += devicesStat.getCurrentWriteKilobytes() - previousIOTimeMap.get(devicesStat.getDeviceName()).writeThroughputInKB;
readTime += devicesStat.getCurrentReadTime() - previousIOTimeMap.get(devicesStat.getDeviceName()).readTime;
writeTime += devicesStat.getCurrentWriteTime() - previousIOTimeMap.get(devicesStat.getDeviceName()).writeTime;
// Avoid dividing by fractions which will give false positives in results
if(readTime < 1) readTime = 1;
if(readOps < 1) readOps = 1;
if(writeOps < 1) writeOps = 1;
if(writeTime < 1) writeTime = 1;
readLatency += (readTime / readOps);
writeLatency += (writeTime / writeOps);
}
DiskStats ps = new DiskStats(devicesStat.getCurrentIOTime(), devicesStat.getCurrentReadTime(),
devicesStat.getCurrentWriteTime(), devicesStat.currentReadOperations(), devicesStat.currentWriteOpetations(),
devicesStat.getCurrentReadKilobytes(), devicesStat.getCurrentWriteKilobytes());
currentIOTimeMap.put(devicesStat.getDeviceName(), ps);
}
// logger.info("Read in MB : {} , Write in MB : {}", readkb/1000, writekb/1000);
// readLatency += (readOps / readTime) * 100;
// writeLatency += (writeOps / writeTime) * 100;
// logger.info("read ops : {} , writeops : {} , readtime: {} , writetime: {}", readOps, writeOps, readTime, writeTime);
// logger.info("Read latency : {} write latency : {}" , readLatency, writeLatency);
logger.info("IO use percent : {}", ioUsePercent);
logger.debug("IO use percent : {}", ioUsePercent);
previousIOTimeMap = currentIOTimeMap;

return new DiskStats(ioUsePercent, readTime, writeTime, readOps, writeOps, readkb, writekb);
Expand Down

0 comments on commit 37be142

Please sign in to comment.