Implement disk io metrics for linux

src/Libraries/Microsoft.Extensions.Diagnostics.ResourceMonitoring/Linux/Disk/DiskStats.cs

@@ -0,0 +1,36 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+namespace Microsoft.Extensions.Diagnostics.ResourceMonitoring.Linux.Disk;
+
+/// <summary>
+/// Represents one line of statistics from "/proc/diskstats"
+/// See https://www.kernel.org/doc/Documentation/ABI/testing/procfs-diskstats for details.
+/// </summary>
+internal sealed class DiskStats
+{
+    public int MajorNumber { get; set; }
+    public int MinorNumber { get; set; }
+    public string DeviceName { get; set; } = string.Empty;
+    public ulong ReadsCompleted { get; set; }
+    public ulong ReadsMerged { get; set; }
+    public ulong SectorsRead { get; set; }
+    public ulong TimeReadingMs { get; set; }
+    public ulong WritesCompleted { get; set; }
+    public ulong WritesMerged { get; set; }
+    public ulong SectorsWritten { get; set; }
+    public ulong TimeWritingMs { get; set; }
+    public ulong IoInProgress { get; set; }
+    public ulong TimeIoMs { get; set; }
+    public ulong WeightedTimeIoMs { get; set; }
+
+    // The following fields are available starting from kernel 4.18; if absent, remain 0
+    public ulong DiscardsCompleted { get; set; }
+    public ulong DiscardsMerged { get; set; }
+    public ulong SectorsDiscarded { get; set; }
+    public ulong TimeDiscardingMs { get; set; }
+
+    // The following fields are available starting from kernel 5.5; if absent, remain 0
+    public ulong FlushRequestsCompleted { get; set; }
+    public ulong TimeFlushingMs { get; set; }
+}

src/Libraries/Microsoft.Extensions.Diagnostics.ResourceMonitoring/Linux/Disk/DiskStatsReader.cs

@@ -0,0 +1,111 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+using System.Collections.Generic;
+using System.Diagnostics.CodeAnalysis;
+using System.Globalization;
+using System.IO;
+using Microsoft.Extensions.ObjectPool;
+using Microsoft.Shared.Pools;
+
+namespace Microsoft.Extensions.Diagnostics.ResourceMonitoring.Linux.Disk;
+
+/// <summary>
+/// Handles reading and parsing of Linux procfs-diskstats file(/proc/diskstats).
+/// </summary>
+internal sealed class DiskStatsReader(IFileSystem fileSystem) : IDiskStatsReader
+{
+    private static readonly FileInfo _diskStatsFile = new("/proc/diskstats");
+    private static readonly ObjectPool<BufferWriter<char>> _sharedBufferWriterPool = BufferWriterPool.CreateBufferWriterPool<char>();
+
+    /// <summary>
+    /// Reads and returns all disk statistics entries.
+    /// </summary>
+    /// <returns>List of <see cref="DiskStats"/>.</returns>
+    public List<DiskStats> ReadAll()
+    {
+        var diskStatsList = new List<DiskStats>();
+
+        using ReturnableBufferWriter<char> bufferWriter = new(_sharedBufferWriterPool);
+        using IEnumerator<ReadOnlyMemory<char>> enumerableLines = fileSystem.ReadAllByLines(_diskStatsFile, bufferWriter.Buffer).GetEnumerator();
+
+        while (enumerableLines.MoveNext())
+        {
+            string line = enumerableLines.Current.Trim().ToString();
+            if (string.IsNullOrWhiteSpace(line))
+            {
+                continue;
+            }
+
+            try
+            {
+                DiskStats stat = DiskStatsReader.ParseLine(line);
+                diskStatsList.Add(stat);
+            }
+#pragma warning disable CA1031
+            catch (Exception)
+#pragma warning restore CA1031
+            {
+                // ignore parsing errors
+            }
+        }
+
+        return diskStatsList;
+    }
+
+    /// <summary>
+    /// Parses one line of text into a DiskStats object.
+    /// </summary>
+    /// <param name="line">one line in "/proc/diskstats".</param>
+    /// <returns>parsed DiskStats object.</returns>
+    [SuppressMessage("Major Code Smell", "S109:Magic numbers should not be used", Justification = "These numbers represent fixed field indices in the Linux /proc/diskstats format")]
+    private static DiskStats ParseLine(string line)
+    {
+        // Split by any whitespace and remove empty entries
+#pragma warning disable EA0009
+        string[] parts = line.Split(Array.Empty<char>(), StringSplitOptions.RemoveEmptyEntries);
+#pragma warning restore EA0009
+
+        if (parts.Length < 14)
+        {
+            throw new FormatException($"Not enough fields: expected at least 14, got {parts.Length}");
+        }
+
+        // See https://www.kernel.org/doc/Documentation/ABI/testing/procfs-diskstats
+        var diskStats = new DiskStats
+        {
+            MajorNumber = int.Parse(parts[0], CultureInfo.InvariantCulture),
+            MinorNumber = int.Parse(parts[1], CultureInfo.InvariantCulture),
+            DeviceName = parts[2],
+            ReadsCompleted = ulong.Parse(parts[3], CultureInfo.InvariantCulture),
+            ReadsMerged = ulong.Parse(parts[4], CultureInfo.InvariantCulture),
+            SectorsRead = ulong.Parse(parts[5], CultureInfo.InvariantCulture),
+            TimeReadingMs = ulong.Parse(parts[6], CultureInfo.InvariantCulture),
+            WritesCompleted = ulong.Parse(parts[7], CultureInfo.InvariantCulture),
+            WritesMerged = ulong.Parse(parts[8], CultureInfo.InvariantCulture),
+            SectorsWritten = ulong.Parse(parts[9], CultureInfo.InvariantCulture),
+            TimeWritingMs = ulong.Parse(parts[10], CultureInfo.InvariantCulture),
+            IoInProgress = ulong.Parse(parts[11], CultureInfo.InvariantCulture),
+            TimeIoMs = ulong.Parse(parts[12], CultureInfo.InvariantCulture),
+            WeightedTimeIoMs = ulong.Parse(parts[13], CultureInfo.InvariantCulture)
+        };
+
+        // Parse additional fields if present
+        if (parts.Length >= 18)
+        {
+            diskStats.DiscardsCompleted = ulong.Parse(parts[14], CultureInfo.InvariantCulture);
+            diskStats.DiscardsMerged = ulong.Parse(parts[15], CultureInfo.InvariantCulture);
+            diskStats.SectorsDiscarded = ulong.Parse(parts[16], CultureInfo.InvariantCulture);
+            diskStats.TimeDiscardingMs = ulong.Parse(parts[17], CultureInfo.InvariantCulture);
+        }
+
+        if (parts.Length >= 20)
+        {
+            diskStats.FlushRequestsCompleted = ulong.Parse(parts[18], CultureInfo.InvariantCulture);
+            diskStats.TimeFlushingMs = ulong.Parse(parts[19], CultureInfo.InvariantCulture);
+        }
+
+        return diskStats;
+    }
+}

src/Libraries/Microsoft.Extensions.Diagnostics.ResourceMonitoring/Linux/Disk/IDiskStatsReader.cs

@@ -0,0 +1,18 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Collections.Generic;
+
+namespace Microsoft.Extensions.Diagnostics.ResourceMonitoring.Linux.Disk;
+
+/// <summary>
+/// An interface for reading disk statistics.
+/// </summary>
+internal interface IDiskStatsReader
+{
+    /// <summary>
+    /// Gets all the disk statistics from the system.
+    /// </summary>
+    /// <returns>List of <see cref="DiskStats"/> instances.</returns>
+    List<DiskStats> ReadAll();
+}

src/Libraries/Microsoft.Extensions.Diagnostics.ResourceMonitoring/Linux/Disk/LinuxDiskMetrics.cs

@@ -0,0 +1,184 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Diagnostics.Metrics;
+using System.Linq;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Logging.Abstractions;
+using Microsoft.Extensions.Options;
+using Microsoft.Shared.Instruments;
+
+namespace Microsoft.Extensions.Diagnostics.ResourceMonitoring.Linux.Disk;
+
+internal sealed class LinuxDiskMetrics
+{
+    // The kernel's block layer always reports counts in 512-byte "sectors" regardless of the underlying device's real block size
+    // https://docs.kernel.org/block/stat.html#read-sectors-write-sectors-discard-sectors
+    private const int LinuxDiskSectorSize = 512;
+    private const int MinimumDiskStatsRefreshIntervalInSeconds = 10;
+    private const string DeviceKey = "system.device";
+    private const string DirectionKey = "disk.io.direction";
+
+    private static readonly KeyValuePair<string, object?> _directionReadTag = new(DirectionKey, "read");
+    private static readonly KeyValuePair<string, object?> _directionWriteTag = new(DirectionKey, "write");
+    private readonly ILogger<LinuxDiskMetrics> _logger;
+    private readonly TimeProvider _timeProvider;
+    private readonly IDiskStatsReader _diskStatsReader;
+    private readonly object _lock = new();
+    private readonly Dictionary<string, DiskStats> _baselineDiskStatsDict = [];
+    private List<DiskStats> _diskStatsSnapshot = [];
+    private DateTimeOffset _lastRefreshTime = DateTimeOffset.MinValue;
+
+    public LinuxDiskMetrics(
+        ILogger<LinuxDiskMetrics>? logger,
+        IMeterFactory meterFactory,
+        IOptions<ResourceMonitoringOptions> options,
+        TimeProvider timeProvider,
+        IDiskStatsReader diskStatsReader)
+    {
+        _logger = logger ?? NullLogger<LinuxDiskMetrics>.Instance;
+        _timeProvider = timeProvider;
+        _diskStatsReader = diskStatsReader;
+        if (!options.Value.EnableDiskIoMetrics)
+        {
+            return;
+        }
+
+        // We need to read the disk stats once to get the baseline values
+        _baselineDiskStatsDict = GetAllDiskStats().ToDictionary(d => d.DeviceName);
+
+#pragma warning disable CA2000 // Dispose objects before losing scope
+        // We don't dispose the meter because IMeterFactory handles that
+        // It's a false-positive, see: https://github.com/dotnet/roslyn-analyzers/issues/6912.
+        // Related documentation: https://github.com/dotnet/docs/pull/37170
+        Meter meter = meterFactory.Create(ResourceUtilizationInstruments.MeterName);
+#pragma warning restore CA2000 // Dispose objects before losing scope
+
+        // The metric is aligned with
+        // https://opentelemetry.io/docs/specs/semconv/system/system-metrics/#metric-systemdiskio
+        _ = meter.CreateObservableCounter(
+            ResourceUtilizationInstruments.SystemDiskIo,
+            GetDiskIoMeasurements,
+            unit: "By",
+            description: "Disk bytes transferred");
+
+        // The metric is aligned with
+        // https://opentelemetry.io/docs/specs/semconv/system/system-metrics/#metric-systemdiskoperations
+        _ = meter.CreateObservableCounter(
+            ResourceUtilizationInstruments.SystemDiskOperations,
+            GetDiskOperationMeasurements,
+            unit: "{operation}",
+            description: "Disk operations");
+
+        // The metric is aligned with
+        // https://opentelemetry.io/docs/specs/semconv/system/system-metrics/#metric-systemdiskio_time
+        _ = meter.CreateObservableCounter(
+            ResourceUtilizationInstruments.SystemDiskIoTime,
+            GetDiskIoTimeMeasurements,
+            unit: "s",
+            description: "Time disk spent activated");
+    }
+
+    private IEnumerable<Measurement<long>> GetDiskIoMeasurements()
+    {
+        List<Measurement<long>> measurements = [];
+        List<DiskStats> diskStatsSnapshot = GetDiskStatsSnapshot();
+
+        foreach (DiskStats diskStats in diskStatsSnapshot)
+        {
+            if (!_baselineDiskStatsDict.TryGetValue(diskStats.DeviceName, out DiskStats? baselineDiskStats))
+            {
+                continue;
+            }
+
+            long readBytes = (long)(diskStats.SectorsRead - baselineDiskStats.SectorsRead) * LinuxDiskSectorSize;
+            long writeBytes = (long)(diskStats.SectorsWritten - baselineDiskStats.SectorsWritten) * LinuxDiskSectorSize;
+            measurements.Add(new Measurement<long>(readBytes, new TagList { _directionReadTag, new(DeviceKey, diskStats.DeviceName) }));
+            measurements.Add(new Measurement<long>(writeBytes, new TagList { _directionWriteTag, new(DeviceKey, diskStats.DeviceName) }));
+        }
+
+        return measurements;
+    }
+
+    private IEnumerable<Measurement<long>> GetDiskOperationMeasurements()
+    {
+        List<Measurement<long>> measurements = [];
+        List<DiskStats> diskStatsSnapshot = GetDiskStatsSnapshot();
+
+        foreach (DiskStats diskStats in diskStatsSnapshot)
+        {
+            if (!_baselineDiskStatsDict.TryGetValue(diskStats.DeviceName, out DiskStats? baselineDiskStats))
+            {
+                continue;
+            }
+
+            long readCount = (long)(diskStats.ReadsCompleted - baselineDiskStats.ReadsCompleted);
+            long writeCount = (long)(diskStats.WritesCompleted - baselineDiskStats.WritesCompleted);
+            measurements.Add(new Measurement<long>(readCount, new TagList { _directionReadTag, new(DeviceKey, diskStats.DeviceName) }));
+            measurements.Add(new Measurement<long>(writeCount, new TagList { _directionWriteTag, new(DeviceKey, diskStats.DeviceName) }));
+        }
+
+        return measurements;
+    }
+
+    private IEnumerable<Measurement<double>> GetDiskIoTimeMeasurements()
+    {
+        List<Measurement<double>> measurements = [];
+        List<DiskStats> diskStatsSnapshot = GetDiskStatsSnapshot();
+
+        foreach (DiskStats diskStats in diskStatsSnapshot)
+        {
+            if (!_baselineDiskStatsDict.TryGetValue(diskStats.DeviceName, out DiskStats? baselineDiskStats))
+            {
+                continue;
+            }
+
+            double ioTimeSeconds = (diskStats.TimeIoMs - baselineDiskStats.TimeIoMs) / 1000.0; // Convert to seconds
+            measurements.Add(new Measurement<double>(ioTimeSeconds, new TagList { new(DeviceKey, diskStats.DeviceName) }));
+        }
+
+        return measurements;
+    }
+
+    private List<DiskStats> GetDiskStatsSnapshot()
+    {
+        lock (_lock)
+        {
+            DateTimeOffset now = _timeProvider.GetUtcNow();
+            if (_diskStatsSnapshot.Count == 0 || (now - _lastRefreshTime).TotalSeconds > MinimumDiskStatsRefreshIntervalInSeconds)
+            {
+                _diskStatsSnapshot = GetAllDiskStats();
+                _lastRefreshTime = now;
+            }
+        }
+
+        return _diskStatsSnapshot;
+    }
+
+    private List<DiskStats> GetAllDiskStats()
+    {
+        try
+        {
+            List<DiskStats> diskStatsList = _diskStatsReader.ReadAll();
+
+            // We should not include ram, loop, or dm(device-mapper) devices in the disk stats, should we?
+            diskStatsList = diskStatsList
+                .Where(d => !d.DeviceName.StartsWith("ram", StringComparison.OrdinalIgnoreCase)
+                            && !d.DeviceName.StartsWith("loop", StringComparison.OrdinalIgnoreCase)
+                            && !d.DeviceName.StartsWith("dm-", StringComparison.OrdinalIgnoreCase))
+                .ToList();
+            return diskStatsList;
+        }
+#pragma warning disable CA1031
+        catch (Exception ex)
+#pragma warning restore CA1031
+        {
+            Log.HandleDiskStatsException(_logger, ex.Message);
+        }
+
+        return [];
+    }
+}

src/Libraries/Microsoft.Extensions.Diagnostics.ResourceMonitoring/Linux/Log.cs

@@ -56,4 +56,8 @@ public static partial void CounterMessage100(
     public static partial void CounterMessage110(
         ILogger logger,
         long counterValue);
+
+    [LoggerMessage(7, LogLevel.Warning,
+        "Error getting disk stats: Error={errorMessage}")]
+    public static partial void HandleDiskStatsException(ILogger logger, string errorMessage);
 }

src/Libraries/Microsoft.Extensions.Diagnostics.ResourceMonitoring/ResourceMonitoringServiceCollectionExtensions.cs

@@ -8,6 +8,7 @@
 using Microsoft.Extensions.Diagnostics.ResourceMonitoring;
 #if !NETFRAMEWORK
 using Microsoft.Extensions.Diagnostics.ResourceMonitoring.Linux;
+using Microsoft.Extensions.Diagnostics.ResourceMonitoring.Linux.Disk;
 using Microsoft.Extensions.Diagnostics.ResourceMonitoring.Linux.Network;
 
 #endif
@@ -129,14 +130,17 @@ private static ResourceMonitorBuilder AddLinuxProvider(this ResourceMonitorBuild
 
         builder.Services.TryAddActivatedSingleton<ISnapshotProvider, LinuxUtilizationProvider>();
 
+        builder.Services.TryAddSingleton(TimeProvider.System);
         builder.Services.TryAddSingleton<IFileSystem, OSFileSystem>();
         builder.Services.TryAddSingleton<IUserHz, UserHz>();
         builder.PickLinuxParser();
 
         _ = builder.Services
             .AddActivatedSingleton<LinuxNetworkUtilizationParser>()
             .AddActivatedSingleton<LinuxNetworkMetrics>()
-            .AddActivatedSingleton<ITcpStateInfoProvider, LinuxTcpStateInfo>();
+            .AddActivatedSingleton<ITcpStateInfoProvider, LinuxTcpStateInfo>()
+            .AddActivatedSingleton<IDiskStatsReader, DiskStatsReader>()
+            .AddActivatedSingleton<LinuxDiskMetrics>();
 
         return builder;
     }