Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Optimize resource utilization parser for linux #4744

Merged
merged 2 commits into from
Nov 22, 2023
Merged

Optimize resource utilization parser for linux #4744

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
aa529d7
Optimize resource utilization parser for linux
Nov 20, 2023
335d7d3
Change to remarks
Nov 21, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
86 changes: 39 additions & 47 deletions ...aries/Microsoft.Extensions.Diagnostics.ResourceMonitoring/Linux/LinuxUtilizationParser.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,15 +4,16 @@
using System;
using System.Diagnostics.CodeAnalysis;
using System.IO;
using Microsoft.Extensions.ObjectPool;
using Microsoft.Shared.Diagnostics;
using Microsoft.Shared.Pools;

namespace Microsoft.Extensions.Diagnostics.ResourceMonitoring.Linux;

/// <summary>
/// <remarks>
/// Parses Linux files to retrieve resource utilization data.
/// </summary>
/// This class is not thread safe.
/// When the same instance is called by multiple threads it may return corrupted data.
Comment on lines +14 to +15
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggest moving this as <remarks>. Intellisense shows those, making the discoverability easier

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Oh, it should be remarks. Thanks for spotting. :-)

/// </remarks>
internal sealed class LinuxUtilizationParser
{
/// <remarks>
Expand Down Expand Up @@ -77,21 +78,19 @@ internal sealed class LinuxUtilizationParser

private readonly IFileSystem _fileSystem;
private readonly long _userHz;
private readonly ObjectPool<BufferWriter<char>> _buffers;
private readonly BufferWriter<char> _buffer = new();

public LinuxUtilizationParser(IFileSystem fileSystem, IUserHz userHz)
{
_fileSystem = fileSystem;
_userHz = userHz.Value;
_buffers = BufferWriterPool.CreateBufferWriterPool<char>(maxCapacity: 64);
}

public long GetCgroupCpuUsageInNanoseconds()
{
var buffer = _buffers.Get();
_fileSystem.ReadAll(_cpuacctUsage, buffer);
_fileSystem.ReadAll(_cpuacctUsage, _buffer);

var usage = buffer.WrittenSpan;
var usage = _buffer.WrittenSpan;

_ = GetNextNumber(usage, out var nanoseconds);

Expand All @@ -100,7 +99,7 @@ public long GetCgroupCpuUsageInNanoseconds()
Throw.InvalidOperationException($"Could not get cpu usage from '{_cpuacctUsage}'. Expected positive number, but got '{new string(usage)}'.");
}

_buffers.Return(buffer);
_buffer.Reset();

return nanoseconds;
}
Expand All @@ -111,15 +110,14 @@ public long GetHostCpuUsageInNanoseconds()
const int NumberOfColumnsRepresentingCpuUsage = 8;
const int NanosecondsInSecond = 1_000_000_000;

var buffer = _buffers.Get();
_fileSystem.ReadFirstLine(_procStat, buffer);
_fileSystem.ReadFirstLine(_procStat, _buffer);

var stat = buffer.WrittenSpan;
var stat = _buffer.WrittenSpan;
var total = 0L;

if (!buffer.WrittenSpan.StartsWith(StartingTokens))
if (!_buffer.WrittenSpan.StartsWith(StartingTokens))
{
Throw.InvalidOperationException($"Expected proc/stat to start with '{StartingTokens}' but it was '{new string(buffer.WrittenSpan)}'.");
Throw.InvalidOperationException($"Expected proc/stat to start with '{StartingTokens}' but it was '{new string(_buffer.WrittenSpan)}'.");
}

stat = stat.Slice(StartingTokens.Length, stat.Length - StartingTokens.Length);
Expand All @@ -142,7 +140,7 @@ public long GetHostCpuUsageInNanoseconds()
stat = stat.Slice(next, stat.Length - next);
}

_buffers.Return(buffer);
_buffer.Reset();

return (long)(total / (double)_userHz * NanosecondsInSecond);
}
Expand All @@ -166,34 +164,29 @@ public ulong GetAvailableMemoryInBytes()
{
const long UnsetCgroupMemoryLimit = 9_223_372_036_854_771_712;

var buffer = _buffers.Get();
_fileSystem.ReadAll(_memoryLimitInBytes, buffer);
_fileSystem.ReadAll(_memoryLimitInBytes, _buffer);

var memoryBuffer = buffer.WrittenSpan;
var memoryBuffer = _buffer.WrittenSpan;
_ = GetNextNumber(memoryBuffer, out var maybeMemory);

if (maybeMemory == -1)
{
Throw.InvalidOperationException($"Could not parse '{_memoryLimitInBytes}' content. Expected to find available memory in bytes but got '{new string(memoryBuffer)}' instead.");
}

_buffers.Return(buffer);
_buffer.Reset();

if (maybeMemory == UnsetCgroupMemoryLimit)
{
return GetHostAvailableMemory();
}

return (ulong)maybeMemory;
return maybeMemory == UnsetCgroupMemoryLimit
? GetHostAvailableMemory()
: (ulong)maybeMemory;
}

public ulong GetMemoryUsageInBytes()
{
const string TotalInactiveFile = "total_inactive_file";

var buffer = _buffers.Get();
_fileSystem.ReadAll(_memoryStat, buffer);
var memoryFile = buffer.WrittenSpan;
_fileSystem.ReadAll(_memoryStat, _buffer);
var memoryFile = _buffer.WrittenSpan;

var index = memoryFile.IndexOf(TotalInactiveFile.AsSpan());

Expand All @@ -210,11 +203,11 @@ public ulong GetMemoryUsageInBytes()
Throw.InvalidOperationException($"The value of total_inactive_file found in '{_memoryStat}' is not a positive number: '{new string(inactiveMemorySlice)}'.");
}

buffer.Reset();
_buffer.Reset();

_fileSystem.ReadAll(_memoryUsageInBytes, buffer);
_fileSystem.ReadAll(_memoryUsageInBytes, _buffer);

var containerMemoryUsageFile = buffer.WrittenSpan;
var containerMemoryUsageFile = _buffer.WrittenSpan;
var next = GetNextNumber(containerMemoryUsageFile, out var containerMemoryUsage);

// this file format doesn't expect to contain anything after the number.
Expand All @@ -224,7 +217,7 @@ public ulong GetMemoryUsageInBytes()
$"We tried to read '{_memoryUsageInBytes}', and we expected to get a positive number but instead it was: '{new string(containerMemoryUsageFile)}'.");
}

_buffers.Return(buffer);
_buffer.Reset();

var memoryUsage = containerMemoryUsage - inactiveMemory;

Expand All @@ -243,9 +236,8 @@ public ulong GetHostAvailableMemory()
// The value we are interested in starts with this. We just want to make sure it is true.
const string MemTotal = "MemTotal:";

var buffer = _buffers.Get();
_fileSystem.ReadFirstLine(_memInfo, buffer);
var firstLine = buffer.WrittenSpan;
_fileSystem.ReadFirstLine(_memInfo, _buffer);
var firstLine = _buffer.WrittenSpan;

if (!firstLine.StartsWith(MemTotal))
{
Expand Down Expand Up @@ -279,7 +271,7 @@ public ulong GetHostAvailableMemory()
$"We tried to convert total memory usage value from '{_memInfo}' to bytes, but we've got a unit that we don't recognize: '{new string(unit)}'.")
};

_buffers.Return(buffer);
_buffer.Reset();

return u;
}
Expand All @@ -291,9 +283,8 @@ public ulong GetHostAvailableMemory()
/// </remarks>
public float GetHostCpuCount()
{
var buffer = _buffers.Get();
_fileSystem.ReadFirstLine(_cpuSetCpus, buffer);
var stats = buffer.WrittenSpan;
_fileSystem.ReadFirstLine(_cpuSetCpus, _buffer);
var stats = _buffer.WrittenSpan;

var start = stats.IndexOf("-", StringComparison.Ordinal);

Expand All @@ -313,7 +304,7 @@ public float GetHostCpuCount()
Throw.InvalidOperationException($"Could not parse '{_cpuSetCpus}'. Expected integer based range separated by dash (like 0-8) but got '{new string(stats)}'.");
}

_buffers.Return(buffer);
_buffer.Reset();

return endCpu - startCpu + 1;
}
Expand Down Expand Up @@ -354,13 +345,13 @@ private static int GetNextNumber(ReadOnlySpan<char> buffer, out long number)

private bool TryGetCpuUnitsFromCgroups(IFileSystem fileSystem, out float cpuUnits)
{
var buffer = _buffers.Get();
fileSystem.ReadFirstLine(_cpuCfsQuotaUs, buffer);
fileSystem.ReadFirstLine(_cpuCfsQuotaUs, _buffer);

var quotaBuffer = buffer.WrittenSpan;
var quotaBuffer = _buffer.WrittenSpan;

if (quotaBuffer.IsEmpty || (quotaBuffer.Length == 2 && quotaBuffer[0] == '-' && quotaBuffer[1] == '1'))
{
_buffer.Reset();
cpuUnits = -1;
return false;
}
Expand All @@ -372,13 +363,14 @@ private bool TryGetCpuUnitsFromCgroups(IFileSystem fileSystem, out float cpuUnit
Throw.InvalidOperationException($"Could not parse '{_cpuCfsQuotaUs}'. Expected an integer but got: '{new string(quotaBuffer)}'.");
}

buffer.Reset();
_buffer.Reset();

fileSystem.ReadFirstLine(_cpuCfsPeriodUs, buffer);
var periodBuffer = buffer.WrittenSpan;
fileSystem.ReadFirstLine(_cpuCfsPeriodUs, _buffer);
var periodBuffer = _buffer.WrittenSpan;

if (periodBuffer.IsEmpty || (periodBuffer.Length == 2 && periodBuffer[0] == '-' && periodBuffer[1] == '1'))
{
_buffer.Reset();
cpuUnits = -1;
return false;
}
Expand All @@ -390,7 +382,7 @@ private bool TryGetCpuUnitsFromCgroups(IFileSystem fileSystem, out float cpuUnit
Throw.InvalidOperationException($"Could not parse '{_cpuCfsPeriodUs}'. Expected to get an integer but got: '{new string(periodBuffer)}'.");
}

_buffers.Return(buffer);
_buffer.Reset();

cpuUnits = (float)quota / period;
return true;
Expand Down