Skip to content

Commit 906b6ff

Browse files
KSemenenkoKSemenenko
committed
Add automatic failure telemetry and registration helpers
1 parent fa908ff commit 906b6ff

20 files changed

Lines changed: 429 additions & 38 deletions

File tree

‎AGENTS.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,8 @@ If I tell you to remember something, you do the same, update
99

1010

1111
## Rules to follow
12+
- For failure logging, use Warning with failure details when there is no exception, and Error with the original stack trace when an exception is supplied.
13+
- For automatic Result failure telemetry, record the error and original exception immediately in the failure/catch path; never retain an exception reference in Result or Problem for later diagnostics.
1214
always check all test are passed.
1315
- Use TUnit with Microsoft.Testing.Platform for all .NET tests; do not add or retain xUnit packages, attributes, fixtures, configuration, or documentation.
1416
- Prefer static interface members for result/command factories to centralize shared overloads and avoid duplication across result-like types.

‎Directory.Build.props‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@
1313
<PropertyGroup>
1414
<Authors>ManagedCode</Authors>
1515
<Copyright>Copyright © 2021-$([System.DateTime]::Now.ToString(`yyyy`)) ManagedCode SAS</Copyright>
16-
<EmbedUntrackedSources>true</EmbedUntrackedSources>
16+
<EmbedUntrackedSources>false</EmbedUntrackedSources>
1717
<PublishRepositoryUrl>true</PublishRepositoryUrl>
1818
<IncludeSymbols>true</IncludeSymbols>
1919
<SymbolPackageFormat>snupkg</SymbolPackageFormat>
@@ -27,8 +27,8 @@
2727
<RepositoryUrl>https://github.com/managedcode/Communication</RepositoryUrl>
2828
<PackageProjectUrl>https://github.com/managedcode/Communication</PackageProjectUrl>
2929
<Product>Managed Code - Communication</Product>
30-
<Version>10.2.2</Version>
31-
<PackageVersion>10.2.2</PackageVersion>
30+
<Version>10.2.3</Version>
31+
<PackageVersion>10.2.3</PackageVersion>
3232

3333
</PropertyGroup>
3434
<PropertyGroup Condition="'$(GITHUB_ACTIONS)' == 'true'">

‎ManagedCode.Communication.AspNetCore/ManagedCode.Communication.AspNetCore.csproj‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22

33
<PropertyGroup>
44
<IsPackable>true</IsPackable>
5-
<DebugType>embedded</DebugType>
5+
<DebugType>portable</DebugType>
66
<OutputType>Library</OutputType>
77
</PropertyGroup>
88

‎ManagedCode.Communication.Extensions/ManagedCode.Communication.Extensions.csproj‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22

33
<PropertyGroup>
44
<IsPackable>true</IsPackable>
5-
<DebugType>embedded</DebugType>
5+
<DebugType>portable</DebugType>
66
</PropertyGroup>
77

88
<!--NuGet-->
@@ -15,6 +15,7 @@
1515

1616
<ItemGroup>
1717
<ProjectReference Include="..\ManagedCode.Communication\ManagedCode.Communication.csproj" />
18+
<PackageReference Include="OpenTelemetry.Extensions.Hosting" Version="1.18.0" />
1819
<PackageReference Include="Microsoft.Extensions.Http" Version="10.0.11" />
1920
</ItemGroup>
2021

‎ManagedCode.Communication.Extensions/Telemetry/CommunicationOpenTelemetryExtensions.cs‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
using System;
2+
using System.Threading;
3+
using System.Threading.Tasks;
4+
using ManagedCode.Communication.Logging;
5+
using ManagedCode.Communication.Telemetry;
6+
using Microsoft.Extensions.DependencyInjection;
7+
using Microsoft.Extensions.Hosting;
8+
using OpenTelemetry;
9+
using OpenTelemetry.Metrics;
10+
using OpenTelemetry.Trace;
11+
12+
namespace ManagedCode.Communication.Extensions.Telemetry;
13+
14+
/// <summary>Registers Communication signals with the application's OpenTelemetry providers.</summary>
15+
public static class CommunicationOpenTelemetryExtensions
16+
{
17+
/// <summary>Subscribes to Communication traces and metrics and connects failure logs at host startup.</summary>
18+
/// <remarks>Call in each service's ServiceDefaults. Configure exporters and OpenTelemetry logging in the application.</remarks>
19+
public static TBuilder AddCommunicationTelemetry<TBuilder>(this TBuilder builder) where TBuilder : IHostApplicationBuilder
20+
{
21+
ArgumentNullException.ThrowIfNull(builder);
22+
builder.Services.AddOpenTelemetry().WithCommunication();
23+
return builder;
24+
}
25+
26+
/// <summary>Adds Communication traces, metrics, and host logger registration without replacing existing providers.</summary>
27+
public static OpenTelemetryBuilder WithCommunication(this OpenTelemetryBuilder builder)
28+
{
29+
ArgumentNullException.ThrowIfNull(builder);
30+
builder.Services.AddHostedService<CommunicationTelemetryLoggerService>();
31+
return builder.WithTracing(tracing => tracing.AddCommunicationInstrumentation())
32+
.WithMetrics(metrics => metrics.AddCommunicationInstrumentation());
33+
}
34+
35+
/// <summary>Subscribes to Communication activities.</summary>
36+
public static TracerProviderBuilder AddCommunicationInstrumentation(this TracerProviderBuilder builder)
37+
{
38+
ArgumentNullException.ThrowIfNull(builder);
39+
return builder.AddSource(CommunicationTelemetry.SourceName);
40+
}
41+
42+
/// <summary>Subscribes to Communication meters.</summary>
43+
public static MeterProviderBuilder AddCommunicationInstrumentation(this MeterProviderBuilder builder)
44+
{
45+
ArgumentNullException.ThrowIfNull(builder);
46+
return builder.AddMeter(CommunicationTelemetry.SourceName);
47+
}
48+
}
49+
50+
internal sealed class CommunicationTelemetryLoggerService(IServiceProvider serviceProvider) : IHostedService
51+
{
52+
public Task StartAsync(CancellationToken cancellationToken)
53+
{
54+
CommunicationLogger.Configure(serviceProvider);
55+
return Task.CompletedTask;
56+
}
57+
58+
public Task StopAsync(CancellationToken cancellationToken) => Task.CompletedTask;
59+
}

‎ManagedCode.Communication.Orleans/ManagedCode.Communication.Orleans.csproj‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22

33
<PropertyGroup>
44
<IsPackable>true</IsPackable>
5-
<DebugType>embedded</DebugType>
5+
<DebugType>portable</DebugType>
66
<RootNamespace>ManagedCode.Communication</RootNamespace>
77
</PropertyGroup>
88

‎ManagedCode.Communication.Tests/Logging/LoggerRegistrationTests.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ public void AReportedFailureReachesARegisteredLogger()
3636
CommunicationDiagnostics.ReportFailure(logger, Problem.Create("payment_declined", "card refused", 402));
3737

3838
var entry = sink.Entries.ShouldHaveSingleItem();
39-
entry.Level.ShouldBe(LogLevel.Error);
39+
entry.Level.ShouldBe(LogLevel.Warning);
4040
entry.Message.ShouldContain("payment_declined");
4141
entry.Message.ShouldContain("card refused");
4242
entry.Message.ShouldContain("402");

‎ManagedCode.Communication.Tests/Telemetry/AutomaticFailureTelemetryTests.cs‎

Lines changed: 192 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,192 @@
1+
using System;
2+
using System.Collections.Generic;
3+
using System.Diagnostics;
4+
using System.Linq;
5+
using System.Text.Json;
6+
using System.Threading.Tasks;
7+
using ManagedCode.Communication.CollectionResultT;
8+
using ManagedCode.Communication.Extensions.Telemetry;
9+
using ManagedCode.Communication.Telemetry;
10+
using Microsoft.Extensions.DependencyInjection;
11+
using Microsoft.Extensions.Hosting;
12+
using Microsoft.Extensions.Logging;
13+
using OpenTelemetry;
14+
using OpenTelemetry.Logs;
15+
using OpenTelemetry.Metrics;
16+
using OpenTelemetry.Trace;
17+
using Shouldly;
18+
19+
namespace ManagedCode.Communication.Tests.Telemetry;
20+
21+
[NotInParallel]
22+
public sealed class AutomaticFailureTelemetryTests
23+
{
24+
private const string FailureTitle = "automatic-failure-test";
25+
private const string FailureDetail = "automatic failure detail";
26+
private const string ParentOperation = "automatic-failure-parent";
27+
private const string ExceptionEvent = "exception";
28+
private const string ExceptionStackTrace = "exception.stacktrace";
29+
private const string OriginalExceptionProperty = "OriginalException";
30+
private const string ParentActivityTag = "problem.title";
31+
private const string ValidationField = "email";
32+
33+
[Test]
34+
public async Task HostRegistration_ShouldExportAutomaticLogsTracesAndMetrics()
35+
{
36+
var spans = new List<Activity>();
37+
var logs = new List<(LogLevel Level, Exception? Exception, ActivityTraceId TraceId, ActivitySpanId SpanId, string? Message)>();
38+
long createdFailures = 0;
39+
var builder = Host.CreateApplicationBuilder();
40+
builder.Logging.ClearProviders();
41+
builder.Logging.AddOpenTelemetry(options =>
42+
{
43+
options.IncludeFormattedMessage = true;
44+
options.AddProcessor(
45+
new SimpleLogRecordExportProcessor(new CaptureExporter<LogRecord>(record =>
46+
{
47+
if (record.CategoryName == typeof(Result).FullName)
48+
{
49+
logs.Add((record.LogLevel, record.Exception, record.TraceId, record.SpanId, record.FormattedMessage));
50+
}
51+
})));
52+
});
53+
builder.AddCommunicationTelemetry().ShouldBeSameAs(builder);
54+
builder.AddCommunicationTelemetry(); // Repeated ServiceDefaults registration is harmless.
55+
builder.Services.AddOpenTelemetry()
56+
.WithTracing(tracing => tracing.AddProcessor(new SimpleActivityExportProcessor(
57+
new CaptureExporter<Activity>(spans.Add))))
58+
.WithMetrics(metrics => metrics.AddReader(new PeriodicExportingMetricReader(
59+
new CaptureExporter<Metric>(metric =>
60+
{
61+
if (metric.Name == CommunicationTelemetry.CreatedFailureCounterName)
62+
{
63+
foreach (ref readonly var point in metric.GetMetricPoints())
64+
{
65+
createdFailures += point.GetSumLong();
66+
}
67+
}
68+
}), exportIntervalMilliseconds: int.MaxValue)));
69+
using var host = builder.Build();
70+
await host.StartAsync();
71+
using var parent = CommunicationTelemetry.StartActivity(ParentOperation);
72+
parent.ShouldNotBeNull();
73+
Exception original;
74+
try
75+
{
76+
throw new InvalidOperationException(FailureDetail);
77+
}
78+
catch (InvalidOperationException exception)
79+
{
80+
original = exception;
81+
}
82+
83+
var result = Result<int>.Fail(original);
84+
Result.Fail(result.Problem!).IsFailed.ShouldBeTrue();
85+
CollectionResult<int>.Fail(result.Problem!).IsFailed.ShouldBeTrue();
86+
Result.Succeed().IsSuccess.ShouldBeTrue();
87+
Result<int>.Succeed(1).IsSuccess.ShouldBeTrue();
88+
logs.Count.ShouldBe(1);
89+
logs[0].Level.ShouldBe(LogLevel.Error);
90+
logs[0].Exception.ShouldBeSameAs(original);
91+
logs[0].TraceId.ShouldBe(parent.TraceId);
92+
parent.Status.ShouldBe(ActivityStatusCode.Unset);
93+
var failure = spans.Single(span => span.OperationName == CommunicationTelemetry.CreatedFailureActivityName);
94+
logs[0].SpanId.ShouldBe(failure.SpanId);
95+
failure.Status.ShouldBe(ActivityStatusCode.Error);
96+
failure.ParentSpanId.ShouldBe(parent.SpanId);
97+
failure.Events.Single(e => e.Name == ExceptionEvent).Tags
98+
.Single(tag => tag.Key == ExceptionStackTrace).Value.ShouldNotBeNull();
99+
using var json = JsonDocument.Parse(JsonSerializer.Serialize(result));
100+
json.RootElement.GetProperty(CommunicationJsonNames.Problem)
101+
.TryGetProperty(OriginalExceptionProperty, out _).ShouldBeFalse();
102+
JsonSerializer.Serialize(result).ShouldNotContain(original.StackTrace!);
103+
104+
Result.Fail(FailureTitle, FailureDetail).IsFailed.ShouldBeTrue();
105+
logs.Count.ShouldBe(2);
106+
logs[1].Level.ShouldBe(LogLevel.Warning);
107+
logs[1].Exception.ShouldBeNull();
108+
logs[1].Message.ShouldNotBeNull();
109+
logs[1].Message!.ShouldContain(FailureTitle);
110+
logs[1].Message!.ShouldContain(FailureDetail);
111+
112+
host.Services.GetRequiredService<MeterProvider>().ForceFlush().ShouldBeTrue();
113+
createdFailures.ShouldBe(2);
114+
await host.StopAsync();
115+
}
116+
117+
[Test]
118+
public void FailureFactories_ShouldTraceEveryShapeAndLeaveParentSuccessful()
119+
{
120+
var spans = new List<Activity>();
121+
using var listener = new ActivityListener
122+
{
123+
ShouldListenTo = source => source.Name == CommunicationTelemetry.SourceName,
124+
Sample = (ref ActivityCreationOptions<ActivityContext> _) => ActivitySamplingResult.AllDataAndRecorded,
125+
ActivityStopped = spans.Add
126+
};
127+
ActivitySource.AddActivityListener(listener);
128+
using var parent = CommunicationTelemetry.StartActivity(ParentOperation);
129+
Result.Fail(FailureTitle, FailureDetail).IsFailed.ShouldBeTrue();
130+
Result<int>.Fail(FailureTitle, FailureDetail).IsFailed.ShouldBeTrue();
131+
CollectionResult<int>.Fail(FailureTitle, FailureDetail).IsFailed.ShouldBeTrue();
132+
Result.FailValidation((ValidationField, FailureDetail)).IsInvalid.ShouldBeTrue();
133+
spans.Count.ShouldBe(4);
134+
spans.ShouldAllBe(span => span.Status == ActivityStatusCode.Error);
135+
spans.Count(span => Equals(span.GetTagItem(ParentActivityTag), FailureTitle)).ShouldBe(3);
136+
parent!.Status.ShouldBe(ActivityStatusCode.Unset);
137+
}
138+
139+
[Test]
140+
public async Task From_WhenDelegatesThrow_ShouldImmediatelyRecordStackTraces()
141+
{
142+
var spans = new List<Activity>();
143+
using var listener = new ActivityListener
144+
{
145+
ShouldListenTo = source => source.Name == CommunicationTelemetry.SourceName,
146+
Sample = (ref ActivityCreationOptions<ActivityContext> _) => ActivitySamplingResult.AllDataAndRecorded,
147+
ActivityStopped = spans.Add
148+
};
149+
ActivitySource.AddActivityListener(listener);
150+
151+
Result<int>.From((Func<int>)ThrowFailure).IsFailed.ShouldBeTrue();
152+
(await Result<int>.From((Func<Task<int>>)ThrowTaskFailure)).IsFailed.ShouldBeTrue();
153+
(await Result<int>.From((Func<ValueTask<int>>)ThrowValueTaskFailure)).IsFailed.ShouldBeTrue();
154+
155+
spans.Count.ShouldBe(3);
156+
foreach (var span in spans)
157+
{
158+
span.Status.ShouldBe(ActivityStatusCode.Error);
159+
var stack = span.Events.Single(e => e.Name == ExceptionEvent).Tags
160+
.Single(tag => tag.Key == ExceptionStackTrace).Value?.ToString();
161+
stack.ShouldNotBeNullOrWhiteSpace();
162+
stack.ShouldContain(nameof(ThrowFailure));
163+
}
164+
}
165+
166+
private static int ThrowFailure() => throw new InvalidOperationException(FailureDetail);
167+
168+
private static async Task<int> ThrowTaskFailure()
169+
{
170+
await Task.Yield();
171+
return ThrowFailure();
172+
}
173+
174+
private static async ValueTask<int> ThrowValueTaskFailure()
175+
{
176+
await Task.Yield();
177+
return ThrowFailure();
178+
}
179+
180+
private sealed class CaptureExporter<T>(Action<T> capture) : BaseExporter<T> where T : class
181+
{
182+
public override ExportResult Export(in Batch<T> batch)
183+
{
184+
foreach (var item in batch)
185+
{
186+
capture(item);
187+
}
188+
189+
return ExportResult.Success;
190+
}
191+
}
192+
}

‎ManagedCode.Communication/CollectionResultT/CollectionResult.cs‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,10 @@ private CollectionResult(bool isSuccess, T[]? collection, int pageNumber, int pa
3131
TotalItems = totalItems;
3232
TotalPages = pageSize > 0 ? (int)Math.Ceiling((double)totalItems / pageSize) : 0;
3333
Problem = problem;
34+
if (!isSuccess && problem is not null)
35+
{
36+
Telemetry.CommunicationDiagnostics.ReportCreatedFailure(problem);
37+
}
3438
}
3539

3640
internal static CollectionResult<T> CreateSuccess(T[]? collection, int pageNumber, int pageSize, int totalItems)

‎ManagedCode.Communication/Logging/ProblemLoggerCenter.cs‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -14,13 +14,15 @@ namespace ManagedCode.Communication.Logging;
1414
/// </remarks>
1515
public static partial class ProblemLoggerCenter
1616
{
17+
private const string ProblemMessage = "Operation failed: {ProblemTitle} ({StatusCode}) — {ProblemDetail}";
18+
1719
/// <summary>
18-
/// Logs a failure that has no originating exception.
20+
/// Logs a failure without an originating exception at Warning level.
1921
/// </summary>
2022
[LoggerMessage(
2123
EventId = 8001,
22-
Level = LogLevel.Error,
23-
Message = "Operation failed: {ProblemTitle} ({StatusCode}) — {ProblemDetail}")]
24+
Level = LogLevel.Warning,
25+
Message = ProblemMessage)]
2426
public static partial void LogProblem(
2527
ILogger logger,
2628
string? problemTitle,

0 commit comments

Comments
 (0)