From f90fd673dfbfded7b7b3eef8e59abbb95d939e1a Mon Sep 17 00:00:00 2001 From: "Vincent Bu (Centific Technologies Inc)" Date: Thu, 23 Apr 2026 14:18:21 +0800 Subject: [PATCH 1/3] add iterations section for end-2-end config and set for microbenchmarks when creating suites add json-trace map and implement AnalyzeForBenchmark Calculate comparison result by benchmark name, rename classes and adjust namespaces present list of microbenchmarkresults rename iteration section to iterations for Run.yaml Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> fix for microbencharmks comparison fix bugs intrduced in previous commit Add json-only comparison extract a shared helper for analyze command take trace type into consideration move MicrobenchmarkResult to GC.Infrastructure.Core.Analysis.Microbenchmarks namespace validate if run is null rename PauseDurationSeconds_SumWhereIsGen1 to PauseDurationMSec_SumWhereIsGen1 add properties for microbenchmarks comparison get value from StatsData if property is not found in GCTraceMetrics filtering to finite values check possible null references add console output when analyzing results assign value for PromotedMB_MeanWhereIsGen1 Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> redesign microbenchmarks result --- .../GC.Infrastructure/Configurations/Run.yaml | 4 + .../GC.Analysis.API/Statistics.cs | 24 ++ ...robenchmarkResults.cs => BdnJsonResult.cs} | 168 ++++------- .../Analysis/GCTraceMetricComparison.cs | 8 + .../Analysis/GCTraceMetricComparisonResult.cs | 100 +++++++ .../Analysis/GCTraceMetrics.cs | 147 ++++++++++ .../MicrobenchmarkComparisonResult.cs | 145 +++++++--- .../Microbenchmarks/MicrobenchmarkResult.cs | 88 ++++++ .../MicrobenchmarkResultComparison.cs | 264 ++++++++++++++++++ .../MicrobenchmarkResultsAnalyzer.cs | 178 ------------ .../Configurations/InputConfiguration.cs | 2 + .../Microbenchmarks.Configuration.cs | 8 +- .../Microbenchmarks/{Json => }/Json.cs | 10 +- .../Microbenchmarks/Json/JsonOutput.cs | 12 - .../Presentation/Microbenchmarks/Markdown.cs | 113 +++----- .../Microbenchmarks/Presentation.cs | 10 +- .../MicrobenchmarkAnalyzeCommand.cs | 48 +++- .../Microbenchmark/MicrobenchmarkCommand.cs | 27 +- .../RunCommand/BaseSuite/Microbenchmarks.yaml | 2 +- .../BaseSuite/MicrobenchmarksToRun.txt | 1 - .../Commands/RunCommand/CreateSuiteCommand.cs | 6 + 21 files changed, 927 insertions(+), 438 deletions(-) rename src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/{Microbenchmarks/MicrobenchmarkResults.cs => BdnJsonResult.cs} (67%) create mode 100644 src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/GCTraceMetricComparison.cs create mode 100644 src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/GCTraceMetricComparisonResult.cs create mode 100644 src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/GCTraceMetrics.cs create mode 100644 src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResult.cs create mode 100644 src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResultComparison.cs delete mode 100644 src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResultsAnalyzer.cs rename src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/{Json => }/Json.cs (50%) delete mode 100644 src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Json/JsonOutput.cs diff --git a/src/benchmarks/gc/GC.Infrastructure/Configurations/Run.yaml b/src/benchmarks/gc/GC.Infrastructure/Configurations/Run.yaml index 8dac83d30a2..17be7e9ab35 100644 --- a/src/benchmarks/gc/GC.Infrastructure/Configurations/Run.yaml +++ b/src/benchmarks/gc/GC.Infrastructure/Configurations/Run.yaml @@ -12,6 +12,10 @@ coreruns: environment_variables: DOTNET_GCName: clrgc.dll +iterations: + gcperfsim: 1 + microbenchmarks: 1 + trace_configuration_type: gc # Choose between: none, gc, verbose, cpu, cpu_managed, threadtime, join. # Optional fields: the contents of both the symbol_path and the source_path will be copied over to the output path. diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Analysis.API/Statistics.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Analysis.API/Statistics.cs index 85c8821a8ec..d0269d32326 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Analysis.API/Statistics.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Analysis.API/Statistics.cs @@ -31,5 +31,29 @@ public static double StandardDeviation(this IEnumerable doubleList) double sumOfDerivationAverage = sumOfDerivation / (doubleList.Count() - 1); return Math.Sqrt(sumOfDerivationAverage - (average * average)); } + + public static IEnumerable RemoveOutliers(IEnumerable collection) + { + if (!collection.Any()) + { + return Array.Empty(); + } + double[] validCollection = collection + .Where(x => !double.IsNaN(x) && !double.IsInfinity(x)) + .ToArray(); + // Calculate Q1 (25th percentile) and Q3 (75th percentile) + double q1 = GC.Analysis.API.Statistics.Percentile(validCollection, 0.25); + double q3 = GC.Analysis.API.Statistics.Percentile(validCollection, 0.75); + + // Calculate IQR (Interquartile Range) + double iqr = q3 - q1; + + // Calculate bounds: [Q1 - 1.5*IQR, Q3 + 1.5*IQR] + double lowerBound = q1 - 1.5 * iqr; + double upperBound = q3 + 1.5 * iqr; + + // Filter out outliers + return GoodLinq.Where(collection, x => x >= lowerBound && x <= upperBound); + } } } diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResults.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/BdnJsonResult.cs similarity index 67% rename from src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResults.cs rename to src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/BdnJsonResult.cs index 2af39abd257..e228ca6eff4 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResults.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/BdnJsonResult.cs @@ -1,103 +1,10 @@ -using GC.Analysis.API; -using GC.Infrastructure.Core.Configurations.Microbenchmarks; -using GC.Infrastructure.Core.Presentation.GCPerfSim; -using Newtonsoft.Json; - -namespace GC.Infrastructure.Core.Analysis +namespace GC.Infrastructure.Core.Analysis { - public sealed class MicrobenchmarkResult - { - public Statistics Statistics { get; set; } - - [JsonIgnore] - public GCProcessData? GCData { get; set; } - - public ResultItem ResultItem { get; set; } - - [JsonIgnore] - public CPUProcessData? CPUData { get; set; } - public Run Parent { get; set; } - public string MicrobenchmarkName { get; set; } - public Dictionary OtherMetrics { get; set; } = new(); - - private static readonly IReadOnlyDictionary> _customStatisticsCalculationMap = new Dictionary>(StringComparer.OrdinalIgnoreCase) - { - { "number of iterations", (Statistics stats) => stats.N }, - { "min", (Statistics stats) => stats.Min }, - { "max", (Statistics stats) => stats.Max }, - { "median", (Statistics stats) => stats.Median }, - { "q1", (Statistics stats) => stats.Q1 }, - { "q3", (Statistics stats) => stats.Q3 }, - { "variance", (Statistics stats) => stats.Variance }, - { "standard deviation", (Statistics stats) => stats.StandardDeviation }, - { "skewness", (Statistics stats) => stats.Skewness }, - { "kurtosis", (Statistics stats) => stats.Kurtosis }, - { "standard error", (Statistics stats) => stats.StandardError }, - { "standard error / mean", (Statistics stats) => stats.StandardError / stats.Mean }, - }; - - public static double? LookupStatisticsCalculation(string columnName, MicrobenchmarkResult result) - { - if (string.IsNullOrEmpty(columnName)) - { - return null; - } - - if (!_customStatisticsCalculationMap.TryGetValue(columnName, out var val)) - { - return null; - } - - else - { - return val.Invoke(result.Statistics); - } - } - } - - public sealed class Benchmark - { - public string DisplayInfo { get; set; } - public string Namespace { get; set; } - public string Type { get; set; } - public string Method { get; set; } - public string MethodTitle { get; set; } - public string Parameters { get; set; } - public string FullName { get; set; } - public Statistics Statistics { get; set; } - public Memory Memory { get; set; } - public List Measurements { get; set; } - public List Metrics { get; set; } - } - public sealed class ChronometerFrequency { public int Hertz { get; set; } } - public sealed class ConfidenceInterval - { - public int N { get; set; } - public double? Mean { get; set; } - public double? StandardError { get; set; } - public int? Level { get; set; } - public double? Margin { get; set; } - public double? Lower { get; set; } - public double? Upper { get; set; } - } - - public sealed class Descriptor - { - public string Id { get; set; } - public string DisplayName { get; set; } - public string Legend { get; set; } - public string NumberFormat { get; set; } - public int UnitType { get; set; } - public string Unit { get; set; } - public bool TheGreaterTheBetter { get; set; } - public int PriorityInCategory { get; set; } - } - public sealed class HostEnvironmentInfo { public string BenchmarkDotNetCaption { get; set; } @@ -117,6 +24,16 @@ public sealed class HostEnvironmentInfo public string HardwareTimerKind { get; set; } } + + public sealed class Memory + { + public int Gen0Collections { get; set; } + public int Gen1Collections { get; set; } + public int Gen2Collections { get; set; } + public int TotalOperations { get; set; } + public long BytesAllocatedPerOperation { get; set; } + } + public sealed class Measurement { public string IterationMode { get; set; } @@ -127,19 +44,27 @@ public sealed class Measurement public long Nanoseconds { get; set; } } - public sealed class Memory + public sealed class Descriptor { - public int Gen0Collections { get; set; } - public int Gen1Collections { get; set; } - public int Gen2Collections { get; set; } - public int TotalOperations { get; set; } - public long BytesAllocatedPerOperation { get; set; } + public string Id { get; set; } + public string DisplayName { get; set; } + public string Legend { get; set; } + public string NumberFormat { get; set; } + public int UnitType { get; set; } + public string Unit { get; set; } + public bool TheGreaterTheBetter { get; set; } + public int PriorityInCategory { get; set; } } - public sealed class Metric + public sealed class ConfidenceInterval { - public double Value { get; set; } - public Descriptor Descriptor { get; set; } + public int N { get; set; } + public double? Mean { get; set; } + public double? StandardError { get; set; } + public int? Level { get; set; } + public double? Margin { get; set; } + public double? Lower { get; set; } + public double? Upper { get; set; } } public sealed class Percentiles @@ -155,13 +80,6 @@ public sealed class Percentiles public double P100 { get; set; } } - public sealed class MicrobenchmarkResults - { - public string Title { get; set; } - public HostEnvironmentInfo HostEnvironmentInfo { get; set; } - public List Benchmarks { get; set; } - } - public sealed class Statistics { public List OriginalValues { get; set; } @@ -186,4 +104,32 @@ public sealed class Statistics public ConfidenceInterval? ConfidenceInterval { get; set; } public Percentiles Percentiles { get; set; } } -} \ No newline at end of file + + public sealed class Metric + { + public double Value { get; set; } + public Descriptor Descriptor { get; set; } + } + + public sealed class Benchmark + { + public string DisplayInfo { get; set; } + public string Namespace { get; set; } + public string Type { get; set; } + public string Method { get; set; } + public string MethodTitle { get; set; } + public string Parameters { get; set; } + public string FullName { get; set; } + public Statistics Statistics { get; set; } + public Memory Memory { get; set; } + public List Measurements { get; set; } + public List Metrics { get; set; } + } + + public sealed class BdnJsonResult + { + public string Title { get; set; } + public HostEnvironmentInfo HostEnvironmentInfo { get; set; } + public List Benchmarks { get; set; } + } +} diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/GCTraceMetricComparison.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/GCTraceMetricComparison.cs new file mode 100644 index 00000000000..3715fb5657a --- /dev/null +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/GCTraceMetricComparison.cs @@ -0,0 +1,8 @@ +namespace GC.Infrastructure.Core.Analysis +{ + public static class GCTraceMetricComparison + { + public static GCTraceMetricComparisonResult CompareGCTraceMetric(IEnumerable baselines, IEnumerable comparands,string nameOfMetric) + => new GCTraceMetricComparisonResult(baselines, comparands, nameOfMetric); + } +} diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/GCTraceMetricComparisonResult.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/GCTraceMetricComparisonResult.cs new file mode 100644 index 00000000000..deb14e44080 --- /dev/null +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/GCTraceMetricComparisonResult.cs @@ -0,0 +1,100 @@ +using API = GC.Analysis.API; +using Microsoft.Diagnostics.Tracing.Analysis.GC; +using System.Reflection; + +namespace GC.Infrastructure.Core.Analysis +{ + public sealed class GCTraceMetricComparisonResult + { + public GCTraceMetricComparisonResult(IEnumerable baselines, IEnumerable comparands, string metricName) + { + RunName = baselines.FirstOrDefault()?.RunName; + Key = $"{baselines.FirstOrDefault()?.ConfigurationName}_{RunName}"; + + MetricName = metricName; + PropertyInfo pInfo = typeof(GCTraceMetrics).GetProperty(metricName, BindingFlags.Instance | BindingFlags.Public); + + // Property found on the GCTraceMetrics. + if (pInfo != null) + { + OriginalBaselineMetricCollection = baselines.Select(baseline => (double)pInfo.GetValue(baseline)); + OriginalComparandMetricCollection = comparands.Select(comparand => (double)pInfo.GetValue(comparand)); + } + + // If property isn't found on the GCTraceMetrics, look in GCStats. + // TODO: Add the case where we look into the map. + else + { + pInfo = typeof(GCStats).GetProperty(metricName, BindingFlags.Instance | BindingFlags.Public); + if (pInfo == null) + { + FieldInfo fieldInfo = typeof(GCStats).GetField(metricName, BindingFlags.Instance | BindingFlags.Public); + if (fieldInfo == null) + { + // Out of luck! + OriginalBaselineMetricCollection = Array.Empty(); + OriginalComparandMetricCollection = Array.Empty(); + OutliersFreeBaselineMetricCollection = Array.Empty(); + OutliersFreeComparandMetricCollection = Array.Empty(); + AveragedBaselineMetric = double.NaN; + AveragedComparandMetric = double.NaN; + return; + } + + else + { + OriginalBaselineMetricCollection = baselines.Select(baseline => baseline.StatsData[fieldInfo.Name]); + OriginalComparandMetricCollection = comparands.Select(comparand => comparand.StatsData[fieldInfo.Name]); + } + } + + else + { + OriginalBaselineMetricCollection = baselines.Select(baseline => baseline.StatsData[pInfo.Name]); + OriginalComparandMetricCollection = comparands.Select(comparand => comparand.StatsData[pInfo.Name]); + } + } + + // Filter out outliers using IQR method + OutliersFreeBaselineMetricCollection = API.Statistics.RemoveOutliers(OriginalBaselineMetricCollection); + OutliersFreeComparandMetricCollection = API.Statistics.RemoveOutliers(OriginalComparandMetricCollection); + + // Calculate averaged metrics + AveragedBaselineMetric = API.GoodLinq.Average(OutliersFreeBaselineMetricCollection, r => r); + AveragedComparandMetric = API.GoodLinq.Average(OutliersFreeComparandMetricCollection, r => r); + } + + public string RunName { get; } + public string Key { get; } + public string MetricName { get; } + public IEnumerable OriginalBaselineMetricCollection { get; } + public IEnumerable OriginalComparandMetricCollection { get; } + public IEnumerable OutliersFreeBaselineMetricCollection { get; } + public IEnumerable OutliersFreeComparandMetricCollection { get; } + + public double AveragedBaselineMetric { get; } + public double AveragedComparandMetric { get; } + public double Delta => AveragedComparandMetric - AveragedBaselineMetric; + public double PercentageDelta + { + get + { + if (AveragedBaselineMetric == 0) + { + if (AveragedComparandMetric == 0) + { + return 0; + } + else + { + return double.NaN; + } + } + else + { + return Delta / AveragedBaselineMetric * 100.0; + } + } + } + } +} diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/GCTraceMetrics.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/GCTraceMetrics.cs new file mode 100644 index 00000000000..df14a2368f9 --- /dev/null +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/GCTraceMetrics.cs @@ -0,0 +1,147 @@ +using GC.Analysis.API; +using Microsoft.Diagnostics.Tracing.Parsers.Clr; + +namespace GC.Infrastructure.Core.Analysis +{ + public sealed class GCTraceMetrics + { + public static GCTraceMetrics GetNullItem(string runName, string corerun) => + new GCTraceMetrics(runName, corerun); + + private GCTraceMetrics(string runName, string corerun) + { + ConfigurationName = corerun; + RunName = runName; + PctTimePausedInGC = double.NaN; + FirstToLastGCSeconds = double.NaN; + HeapSizeBeforeMB_Mean = double.NaN; + HeapSizeAfter_Mean = double.NaN; + TotalCommittedInUse = double.NaN; + TotalBookkeepingCommitted = double.NaN; + TotalCommittedInGlobalDecommit = double.NaN; + TotalCommittedInFree = double.NaN; + TotalCommittedInGlobalFree = double.NaN; + PauseDurationMSec_95PWhereIsGen0 = double.NaN; + PauseDurationMSec_95PWhereIsGen1 = double.NaN; + PauseDurationMSec_95PWhereIsBackground = double.NaN; + PauseDurationMSec_MeanWhereIsBackground = double.NaN; + PauseDurationMSec_95PWhereIsBlockingGen2 = double.NaN; + PauseDurationMSec_MeanWhereIsBlockingGen2 = double.NaN; + CountIsBlockingGen2 = double.NaN; + PauseDurationMSec_SumWhereIsGen1 = double.NaN; + PauseDurationMSec_MeanWhereIsEphemeral = double.NaN; + PromotedMB_MeanWhereIsGen1 = double.NaN; + CountIsGen1 = double.NaN; + CountIsGen0 = double.NaN; + HeapCount = double.NaN; + PauseDurationMSec_Sum = double.NaN; + TotalAllocatedMB = double.NaN; + TotalNumberGCs = double.NaN; + Speed_MBPerMSec = double.NaN; + ExecutionTimeMSec = double.NaN; + } + + public GCTraceMetrics(GCProcessData processData, string runName, string configurationName) + { + RunName = runName; + ConfigurationName = configurationName; + ExecutionTimeMSec = processData.DurationMSec; + + PctTimePausedInGC = processData.Stats.GetGCPauseTimePercentage(); + FirstToLastGCSeconds = (processData.GCs.Last().StartRelativeMSec - processData.GCs.First().StartRelativeMSec) / 1000; + HeapSizeAfter_Mean = GoodLinq.Average(processData.GCs, (gc => gc.HeapSizeAfterMB)); + HeapSizeBeforeMB_Mean = GoodLinq.Average(processData.GCs, (gc => gc.HeapSizeBeforeMB)); + + TotalCommittedInUse = GoodLinq.Average(processData.GCs, (gc => gc.CommittedUsageBefore.TotalCommittedInUse)); + TotalBookkeepingCommitted = GoodLinq.Average(processData.GCs, (gc => gc.CommittedUsageBefore.TotalBookkeepingCommitted)); + TotalCommittedInGlobalDecommit = GoodLinq.Average(processData.GCs, (gc => gc.CommittedUsageBefore.TotalCommittedInGlobalDecommit)); + TotalCommittedInFree = GoodLinq.Average(processData.GCs, (gc => gc.CommittedUsageBefore.TotalCommittedInFree)); + TotalCommittedInGlobalFree = GoodLinq.Average(processData.GCs, (gc => gc.CommittedUsageBefore.TotalCommittedInGlobalFree)); + + var properties = processData.Stats.GetType().GetProperties(System.Reflection.BindingFlags.Public | System.Reflection.BindingFlags.Instance); + foreach (var property in properties) + { + if (property.PropertyType != typeof(double) || property.PropertyType != typeof(int)) + { + continue; + } + + string propertyName = property.Name; + double propertyValue = (double)(property.GetValue(processData.Stats) ?? double.NaN); + StatsData[propertyName] = propertyValue; + } + + var fields = processData.Stats.GetType().GetFields(System.Reflection.BindingFlags.Public | System.Reflection.BindingFlags.Instance); + foreach (var field in fields) + { + if (field.FieldType != typeof(double) || field.FieldType != typeof(int)) + { + continue; + } + + string name = field.Name; + double value = (double)(field.GetValue(processData.Stats) ?? double.NaN); + StatsData[name] = value; + } + + // 95P + PauseDurationMSec_95PWhereIsGen0 = GC.Analysis.API.Statistics.Percentile(GoodLinq.Select(GoodLinq.Where(processData.GCs, (gc => gc.Generation == 0)), (gc => gc.PauseDurationMSec)), 0.95); + PauseDurationMSec_95PWhereIsGen1 = GC.Analysis.API.Statistics.Percentile(GoodLinq.Select(GoodLinq.Where(processData.GCs, (gc => gc.Generation == 1)), (gc => gc.PauseDurationMSec)), 0.95); + + PauseDurationMSec_95PWhereIsBackground = GC.Analysis.API.Statistics.Percentile(GoodLinq.Select(GoodLinq.Where(processData.GCs, (gc => gc.Type == GCType.BackgroundGC)), (gc => gc.PauseDurationMSec)), 0.95); + PauseDurationMSec_MeanWhereIsBackground = GoodLinq.Average(GoodLinq.Select(GoodLinq.Where(processData.GCs, (gc => gc.Type == GCType.BackgroundGC)), (gc => gc.PauseDurationMSec)), (p => p)); + + PauseDurationMSec_95PWhereIsBlockingGen2 = GC.Analysis.API.Statistics.Percentile(GoodLinq.Select(GoodLinq.Where(processData.GCs, (gc => gc.Type != GCType.BackgroundGC && gc.Generation == 2)), (gc => gc.PauseDurationMSec)), 0.95); + PauseDurationMSec_MeanWhereIsBlockingGen2 = GoodLinq.Average(GoodLinq.Select(GoodLinq.Where(processData.GCs, (gc => gc.Type != GCType.BackgroundGC && gc.Generation == 2)), (gc => gc.PauseDurationMSec)), (p => p)); + + CountIsBlockingGen2 = processData.GCs.Count(gc => gc.Generation == 2 && gc.Type != GCType.BackgroundGC); + + HeapCount = processData.Stats.HeapCount; + TotalNumberGCs = processData.Stats.Count; + TotalAllocatedMB = processData.Stats.TotalAllocatedMB; + + Speed_MBPerMSec = processData.Stats.TotalPromotedMB / processData.Stats.TotalPauseTimeMSec; + + PauseDurationMSec_MeanWhereIsEphemeral = + GoodLinq.Average(GoodLinq.Where(processData.GCs, (gc => gc.Generation == 1 || gc.Generation == 0)), (gc => gc.PauseDurationMSec)); + PauseDurationMSec_SumWhereIsGen1 = + GoodLinq.Sum(GoodLinq.Where(processData.GCs, (gc => gc.Generation == 1)), (gc => gc.PauseDurationMSec)); + PromotedMB_MeanWhereIsGen1 = + GoodLinq.Average(GoodLinq.Where(processData.GCs, (gc => gc.Generation == 1)), (gc => gc.PromotedMB)); + PauseDurationMSec_Sum = GoodLinq.Sum(processData.GCs, (gc => gc.PauseDurationMSec)); + CountIsGen1 = GoodLinq.Where(processData.GCs, gc => gc.Generation == 1).Count; + CountIsGen0 = GoodLinq.Where(processData.GCs, gc => gc.Generation == 0).Count; + } + + public double PctTimePausedInGC { get; } + public double FirstToLastGCSeconds { get; } + public double HeapSizeBeforeMB_Mean { get; } + public double HeapSizeAfter_Mean { get; } + public double TotalCommittedInUse { get; set; } + public double TotalCommittedInGlobalDecommit { get; set; } + public double TotalCommittedInFree { get; set; } + public double TotalCommittedInGlobalFree { get; set; } + public double TotalBookkeepingCommitted { get; set; } + public double PauseDurationMSec_95PWhereIsGen0 { get; } + public double PauseDurationMSec_95PWhereIsGen1 { get; } + public double PauseDurationMSec_95PWhereIsBackground { get; } + public double PauseDurationMSec_MeanWhereIsBackground { get; } + public double PauseDurationMSec_95PWhereIsBlockingGen2 { get; } + public double PauseDurationMSec_MeanWhereIsBlockingGen2 { get; } + public double CountIsBlockingGen2 { get; } + public double PauseDurationMSec_SumWhereIsGen1 { get; } + public double PauseDurationMSec_MeanWhereIsEphemeral { get; } + public double PromotedMB_MeanWhereIsGen1 { get; } + public double CountIsGen1 { get; } + public double CountIsGen0 { get; } + public double HeapCount { get; } + public double PauseDurationMSec_Sum { get; } + public double TotalAllocatedMB { get; set; } + public double TotalNumberGCs { get; } + public double Speed_MBPerMSec { get; } + public string RunName { get; } + public string ConfigurationName { get; } + public double ExecutionTimeMSec { get; } + public Dictionary StatsData { get; } = new(); + } +} diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkComparisonResult.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkComparisonResult.cs index 60150f4d4c9..ade87c6d887 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkComparisonResult.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkComparisonResult.cs @@ -1,60 +1,131 @@ -using GC.Infrastructure.Core.Presentation.GCPerfSim; +using API = GC.Analysis.API; +using Microsoft.Diagnostics.Tracing.Parsers.Clr; namespace GC.Infrastructure.Core.Analysis.Microbenchmarks { // Per Microbenchmark result. public sealed class MicrobenchmarkComparisonResult { + public static readonly string[] RequiredMetrics = new string[] + { + "PctTimePausedInGC", + "ExecutionTimeMSec", + "PauseDurationMSec_MeanWhereIsEphemeral", + "PauseDurationMSec_MeanWhereIsBackground", + "PauseDurationMSec_MeanWhereIsBlockingGen2" + }; + public MicrobenchmarkComparisonResult() { } - public MicrobenchmarkComparisonResult(MicrobenchmarkResult baseline, MicrobenchmarkResult comparand) + public MicrobenchmarkComparisonResult(IEnumerable baselines, IEnumerable comparands, bool includeTraces = true) { - Baseline = baseline; - Comparand = comparand; - var result = new ResultItemComparison(baseline.ResultItem, comparand.ResultItem); ComparisonResults = new(); - ComparisonResults.Add(result.GetComparison("PctTimePausedInGC")); - ComparisonResults.Add(result.GetComparison("ExecutionTimeMSec")); - ComparisonResults.Add(result.GetComparison("PauseDurationMSec_MeanWhereIsEphemeral")); - ComparisonResults.Add(result.GetComparison("PauseDurationMSec_MeanWhereIsBackground")); - ComparisonResults.Add(result.GetComparison("PauseDurationMSec_MeanWhereIsBlockingGen2")); + if (includeTraces) + { + var baselineGCTraceMetricsCollection = baselines + .Where(baseline => baseline != null) + .Select(baseline => baseline.GCTraceMetrics) + .ToArray(); + + var comparandGCTraceMetricsCollection = comparands + .Where(comparand => comparand != null) + .Select(comparand => comparand.GCTraceMetrics) + .ToArray(); + + if (baselineGCTraceMetricsCollection.Length > 0 && comparandGCTraceMetricsCollection.Length > 0) + { + foreach (var metricName in RequiredMetrics) + { + ComparisonResults.Add( + GCTraceMetricComparison.CompareGCTraceMetric( + baselineGCTraceMetricsCollection!, comparandGCTraceMetricsCollection!, metricName)); + } + + } + } + + BaselineRunName = baselines?.FirstOrDefault()?.Parent?.Name; + ComparandRunName = comparands?.FirstOrDefault()?.Parent?.Name; + MicrobenchmarkName = baselines?.FirstOrDefault()?.MicrobenchmarkName; + + Baselines = baselines ?? new List(); + Comparands = comparands ?? new List(); } - public MicrobenchmarkResult Baseline { get; set; } - public MicrobenchmarkResult Comparand { get; set; } - public List ComparisonResults { get; set; } + public List ComparisonResults { get; set; } + public string BaselineRunName { get; } + public string ComparandRunName { get; } + public string ComparisonName => $"{ComparandRunName} vs {BaselineRunName}"; + public string MicrobenchmarkName { get; } + public IEnumerable Baselines { get; } + public IEnumerable Comparands { get; } - // TODO: Nullable double check. - public string BaselineRunName => Baseline?.Parent?.Name; - public string ComparandRunName => Comparand?.Parent?.Name; - public string MicrobenchmarkName => Baseline.MicrobenchmarkName; + public double[] OutliersFreeBaselineMeanValueCollection => + API.Statistics.RemoveOutliers(Baselines + .Select(baseline => baseline.Statistics?.Mean ?? double.NaN)) + .ToArray(); + public double[] OutliersFreeComparandMeanValueCollection => + API.Statistics.RemoveOutliers(Comparands + .Select(comparand => comparand.Statistics?.Mean ?? double.NaN)) + .ToArray(); - public double MeanDiff => (Comparand.Statistics?.Mean.Value - Baseline.Statistics?.Mean.Value) ?? double.NaN; - public double MeanDiffPerc => (MeanDiff / Baseline.Statistics?.Mean.Value) * 100 ?? double.NaN; + public double AveragedBaselineMeanValue => API.GoodLinq.Average(OutliersFreeBaselineMeanValueCollection, r => r); + public double AveragedComparandMeanValue => API.GoodLinq.Average(OutliersFreeComparandMeanValueCollection, r => r); - public double? GetDiffPercentFromOtherMetrics(string metric) - { - if (!Baseline.OtherMetrics.TryGetValue(metric, out var baselineMetric)) + public double MeanDiff => AveragedComparandMeanValue - AveragedBaselineMeanValue; + public double MeanDiffPerc{ + get { - return null; + if (AveragedBaselineMeanValue == 0) + { + if (AveragedComparandMeanValue == 0) + { + return 0; + } + else + { + return double.NaN; + } + } + return (MeanDiff / AveragedBaselineMeanValue) * 100; } + } - if (!baselineMetric.HasValue) - { - return null; - } + public Dictionary OriginalBaselineOtherMetrics { get; } = new(); + public Dictionary OriginalComparandOtherMetrics { get; } = new(); + public Dictionary OutliersFreeBaselineOtherMetrics => OriginalBaselineOtherMetrics + .Select(kvp => (kvp.Key, API.Statistics.RemoveOutliers(kvp.Value).ToArray())) + .ToDictionary(); + public Dictionary OutliersFreeComparandOtherMetrics => OriginalComparandOtherMetrics + .Select(kvp => (kvp.Key, API.Statistics.RemoveOutliers(kvp.Value).ToArray())) + .ToDictionary(); + public Dictionary AveragedBaselineOtherMetrics => OutliersFreeBaselineOtherMetrics + .Select(kvp => (kvp.Key, API.GoodLinq.Average(kvp.Value, v => v))) + .ToDictionary(); + public Dictionary AveragedComparandOtherMetrics => OutliersFreeComparandOtherMetrics + .Select(kvp => (kvp.Key, API.GoodLinq.Average(kvp.Value, v => v))) + .ToDictionary(); - if (!Comparand.OtherMetrics.TryGetValue(metric, out var comparandMetric)) - { - return null; - } + public Dictionary OtherMetricsDiff => OutliersFreeBaselineOtherMetrics + .Select(kvp => (kvp.Key, AveragedComparandOtherMetrics[kvp.Key] - AveragedBaselineOtherMetrics[kvp.Key])) + .ToDictionary(); - if (!comparandMetric.HasValue) + public Dictionary OtherMetricsDiffPerc => OutliersFreeBaselineOtherMetrics + .Select(kvp => { - return null; - } - - return (comparandMetric.Value - baselineMetric.Value) / baselineMetric.Value; - } + if (AveragedBaselineOtherMetrics[kvp.Key] == 0) + { + if (AveragedComparandOtherMetrics[kvp.Key] == 0) + { + return (kvp.Key, 0); + } + else + { + return (kvp.Key, double.NaN); + } + } + return (kvp.Key, OtherMetricsDiff[kvp.Key] / AveragedBaselineOtherMetrics[kvp.Key]); + }) + .ToDictionary(); } } diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResult.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResult.cs new file mode 100644 index 00000000000..d64cdf9566f --- /dev/null +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResult.cs @@ -0,0 +1,88 @@ +using GC.Infrastructure.Core.Configurations.Microbenchmarks; +using Microsoft.Diagnostics.Tracing.Parsers.Clr; +using API = GC.Analysis.API; + +namespace GC.Infrastructure.Core.Analysis.Microbenchmarks +{ + public sealed class MicrobenchmarkResult + { + public static readonly IReadOnlyDictionary> CustomStatisticsCalculationMap = new Dictionary>(StringComparer.OrdinalIgnoreCase) + { + { "number of iterations", (Statistics stats) => stats.N }, + { "min", (Statistics stats) => stats.Min }, + { "max", (Statistics stats) => stats.Max }, + { "median", (Statistics stats) => stats.Median }, + { "q1", (Statistics stats) => stats.Q1 }, + { "q3", (Statistics stats) => stats.Q3 }, + { "variance", (Statistics stats) => stats.Variance }, + { "standard deviation", (Statistics stats) => stats.StandardDeviation }, + { "skewness", (Statistics stats) => stats.Skewness }, + { "kurtosis", (Statistics stats) => stats.Kurtosis }, + { "standard error", (Statistics stats) => stats.StandardError }, + { "standard error / mean", (Statistics stats) => stats.StandardError / stats.Mean }, + }; + + public static readonly Dictionary> CustomAggregateCalculationMap = new Dictionary>(StringComparer.OrdinalIgnoreCase) + { + { "gc count", (gc) => gc.Stats.Count }, + { "non induced gc count", (gc) => gc.Stats.Count - gc.GCs.Count(g => g.Reason == GCReason.Induced)}, + { "induced gc count", (gc) => gc.GCs.Count(g => g.Reason == GCReason.Induced)}, + { "total allocated (mb)", (gc) => gc.Stats.TotalAllocatedMB }, + { "max size peak (mb)", (gc) => gc.Stats.MaxSizePeakMB }, + { "total pause time (msec)", (gc) => gc.Stats.TotalPauseTimeMSec }, + { "gc pause time %", (gc) => gc.Stats.GetGCPauseTimePercentage() }, + { "avg. heap size (mb)", (gc) => API.GoodLinq.Average(gc.GCs, g => g.HeapSizeBeforeMB) }, + { "avg. heap size after (mb)", (gc) => API.GoodLinq.Average(gc.GCs, g => g.HeapSizeAfterMB) }, + }; + + public MicrobenchmarkResult(string benchmarkFullName, + Run parent, + Benchmark benchmark, + API.GCProcessData? gcData = null, + GCTraceMetrics? gcTraceMetrics = null, + API.CPUProcessData? cpuData = null, + IEnumerable? additionalReportMetrics = null, + IEnumerable? columns = null, + IEnumerable? cpuColumns = null) + { + MicrobenchmarkName = benchmarkFullName; + Parent = parent; + Statistics = benchmark.Statistics; + GCTraceMetrics = gcTraceMetrics; + CPUData = cpuData; + + if (additionalReportMetrics != null) + { + OtherMetrics = benchmark.Metrics + .Where(metric => additionalReportMetrics.Contains(metric.Descriptor.Id)) + .ToDictionary(metric => metric.Descriptor.Id, metric => (double?)metric.Value); + } + + if (columns != null) + { + var customStatistics = columns + .Where(column => CustomStatisticsCalculationMap.Keys.Contains(column)) + .Select(column => (column, CustomStatisticsCalculationMap[column](benchmark.Statistics))) + .ToDictionary(); + + OtherMetrics = OtherMetrics.Concat(customStatistics).ToDictionary(); + + if (gcData != null) + { + var customGCData = columns + .Where(column => CustomAggregateCalculationMap.Keys.Contains(column)) + .Select(column => (column, (double?)CustomAggregateCalculationMap[column](gcData))) + .ToDictionary(); + + OtherMetrics = OtherMetrics.Concat(customGCData).ToDictionary(); + } + } + } + public string MicrobenchmarkName { get; set; } + public Run Parent { get; set; } + public Statistics Statistics { get; set; } + public GCTraceMetrics? GCTraceMetrics { get; set; } + public Dictionary OtherMetrics { get; set; } = new(); + public API.CPUProcessData? CPUData { get; set; } + } +} \ No newline at end of file diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResultComparison.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResultComparison.cs new file mode 100644 index 00000000000..746c9589521 --- /dev/null +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResultComparison.cs @@ -0,0 +1,264 @@ +using GC.Analysis.API; +using GC.Infrastructure.Core.Configurations.Microbenchmarks; +using Newtonsoft.Json; +using System.Collections.Concurrent; + +namespace GC.Infrastructure.Core.Analysis.Microbenchmarks +{ + public static class MicrobenchmarkResultComparison + { + private static readonly Dictionary _benchmarkNameToTraceFilePatternMap = new() + { + { "ByteMark.BenchBitOps", "ByteMark.BenchBitOps"}, + { "System.Collections.CtorGivenSize.Array(Size: 512)", "System.Collections.CtorGivenSize_String_.Array_size_512_"}, + { "System.Collections.Tests.Perf_BitArray.BitArrayByteArrayCtor(Size: 512)", "System.Collections.Tests.Perf_BitArray.BitArrayByteArrayCtor_size_512_"}, + { "System.IO.Tests.Perf_File.ReadAllBytes(size: 104857600)", "System.IO.Tests.Perf_File.ReadAllBytes_size_104857600_"}, + { "System.IO.Tests.Perf_File.ReadAllBytesAsync(size: 104857600)", "System.IO.Tests.Perf_File.ReadAllBytesAsync_size_104857600_"}, + { "System.Linq.Tests.Perf_Enumerable.ToArray(input: ICollection)", "System.Linq.Tests.Perf_Enumerable.ToArray_"}, + { "System.Linq.Tests.Perf_Enumerable.ToArray(input: IEnumerable)", "System.Linq.Tests.Perf_Enumerable.ToArray_"}, + { "System.Numerics.Tests.Perf_BigInteger.Add(arguments: 65536,65536 bits)", "System.Numerics.Tests.Perf_BigInteger.Add_arguments_65536_"}, + { "System.Numerics.Tests.Perf_BigInteger.Subtract(arguments: 65536,65536 bits)", "System.Numerics.Tests.Perf_BigInteger.Subtract_arguments_65536_"}, + { "System.Tests.Perf_GC.AllocateArray(length: 1000, pinned: False)", "System.Tests.Perf_GC_Byte_.AllocateArray_length_1000,_"}, + { "System.Tests.Perf_GC.AllocateArray(length: 1000, pinned: True)", "System.Tests.Perf_GC_Byte_.AllocateArray_length_1000,_"}, + { "System.Tests.Perf_GC.AllocateArray(length: 10000, pinned: False)", "System.Tests.Perf_GC_Byte_.AllocateArray_length_10000,_"}, + { "System.Tests.Perf_GC.AllocateArray(length: 10000, pinned: True)", "System.Tests.Perf_GC_Byte_.AllocateArray_length_10000,_"}, + { "System.Tests.Perf_GC.AllocateUninitializedArray(length: 1000, pinned: False)", "System.Tests.Perf_GC_Byte_.AllocateUninitializedArray_length_1000,_"}, + { "System.Tests.Perf_GC.AllocateUninitializedArray(length: 1000, pinned: True)", "System.Tests.Perf_GC_Byte_.AllocateUninitializedArray_length_1000,_"}, + { "System.Tests.Perf_GC.AllocateUninitializedArray(length: 10000, pinned: False)", "System.Tests.Perf_GC_Byte_.AllocateUninitializedArray_length_10000,_"}, + { "System.Tests.Perf_GC.AllocateUninitializedArray(length: 10000, pinned: True)", "System.Tests.Perf_GC_Byte_.AllocateUninitializedArray_length_10000,_"}, + { "System.Tests.Perf_GC.NewOperator_Array(length: 1000)", "System.Tests.Perf_GC_Byte_.NewOperator_Array_length_1000_"}, + { "System.Tests.Perf_GC.NewOperator_Array(length: 10000)", "System.Tests.Perf_GC_Byte_.NewOperator_Array_length_10000_"}, + { "System.Tests.Perf_GC.AllocateArray(length: 1000, pinned: False)", "System.Tests.Perf_GC_Char_.AllocateArray_length_1000,_"}, + { "System.Tests.Perf_GC.AllocateArray(length: 1000, pinned: True)", "System.Tests.Perf_GC_Char_.AllocateArray_length_1000,_"}, + { "System.Tests.Perf_GC.AllocateArray(length: 10000, pinned: False)", "System.Tests.Perf_GC_Char_.AllocateArray_length_10000,_"}, + { "System.Tests.Perf_GC.AllocateArray(length: 10000, pinned: True)", "System.Tests.Perf_GC_Char_.AllocateArray_length_10000,_"}, + { "System.Tests.Perf_GC.AllocateUninitializedArray(length: 1000, pinned: False)", "System.Tests.Perf_GC_Char_.AllocateUninitializedArray_length_1000,_"}, + { "System.Tests.Perf_GC.AllocateUninitializedArray(length: 1000, pinned: True)", "System.Tests.Perf_GC_Char_.AllocateUninitializedArray_length_1000,_"}, + { "System.Tests.Perf_GC.AllocateUninitializedArray(length: 10000, pinned: False)", "System.Tests.Perf_GC_Char_.AllocateUninitializedArray_length_10000,_"}, + { "System.Tests.Perf_GC.AllocateUninitializedArray(length: 10000, pinned: True)", "System.Tests.Perf_GC_Char_.AllocateUninitializedArray_length_10000,_"}, + { "System.Tests.Perf_GC.NewOperator_Array(length: 1000)", "System.Tests.Perf_GC_Char_.NewOperator_Array_length_1000_"}, + { "System.Tests.Perf_GC.NewOperator_Array(length: 10000)", "System.Tests.Perf_GC_Char_.NewOperator_Array_length_10000_"}, + }; + + private static readonly ConcurrentDictionary>> _benchmarkFullNameToJsonForRun = new(); + + public static ConcurrentDictionary> MapBenchmarkFullNameToJsonForRun(string outputPathForRun) + { + return _benchmarkFullNameToJsonForRun.GetOrAdd(outputPathForRun, path => + { + ConcurrentDictionary> benchmarkFullNameJsonMap = new(); + + string[] jsonFiles = Directory.GetFiles(outputPathForRun, "*full.json", SearchOption.AllDirectories); + + Parallel.ForEach(jsonFiles, (jsonFile) => + { + BdnJsonResult? results = JsonConvert.DeserializeObject(File.ReadAllText(jsonFile)); + string? fullName = results?.Benchmarks?.FirstOrDefault()?.FullName; + if (fullName != null) + { + benchmarkFullNameJsonMap.GetOrAdd(fullName, _ => new ConcurrentBag()).Add(jsonFile); + } + }); + + return benchmarkFullNameJsonMap; + }); + } + + public static ConcurrentDictionary MapJsonToTraceForSingleBenchmarkRun(string outputPathForRun, string benchmarkFullName) + { + ConcurrentDictionary jsonTraceMap = new(); + + var benchmarkFullNameJsonMap = MapBenchmarkFullNameToJsonForRun(outputPathForRun); + + string[] jsonFiles = benchmarkFullNameJsonMap.GetValueOrDefault(benchmarkFullName, new()).ToArray(); + + Parallel.ForEach(jsonFiles, (jsonFile) => + { + // placeholder + jsonTraceMap[jsonFile] = ""; + }); + + string[] sortedJsonFiles = jsonTraceMap.Keys + .OrderBy(jsonFile => Path.GetFileName(Path.GetDirectoryName(jsonFile))) + .ToArray(); + + if (!_benchmarkNameToTraceFilePatternMap.Keys.Contains(benchmarkFullName)) + { + throw new KeyNotFoundException("No trace file pattern found for benchmark: " + benchmarkFullName); + } + string traceFileNameTemplate = _benchmarkNameToTraceFilePatternMap[benchmarkFullName]; + + string[] sortedTraceFiles = Enumerable.Where(Directory.GetFiles(outputPathForRun, "*.etl.zip", SearchOption.TopDirectoryOnly), traceFile => + Path.GetFileName(traceFile).ToLower().Contains(traceFileNameTemplate.ToLower())) + .OrderBy(traceFile => traceFile) + .ToArray(); + + if (sortedJsonFiles.Length != sortedTraceFiles.Length) + { + throw new InvalidOperationException( + $"The number of JSON files ({sortedJsonFiles.Length}) does not match the number of trace files ({sortedTraceFiles.Length}) for benchmark: {benchmarkFullName}"); + } + + for (int idx = 0; idx < sortedJsonFiles.Length; idx++) + { + jsonTraceMap[sortedJsonFiles[idx]] = sortedTraceFiles[idx]; + } + + return jsonTraceMap; + } + + public static IReadOnlyDictionary> AnalyzeMicrobenchmarkResultsForSingleBenchmark(MicrobenchmarkConfiguration configuration, string benchmarkFullName, bool excludeTraces = false) + { + ConcurrentDictionary> runsToResults = new(); + + Parallel.ForEach(configuration.Runs, (run) => + { + string outputPathForRun = Path.Combine(configuration.Output.Path, run.Key); + run.Value.Name ??= run.Key; + + var benchmarkToJsonMapForRun = MapBenchmarkFullNameToJsonForRun(outputPathForRun); + var jsonFiles = benchmarkToJsonMapForRun.GetValueOrDefault(benchmarkFullName, new()); + + runsToResults[run.Value] = runsToResults.GetValueOrDefault(run.Value, new()); + + Parallel.ForEach(jsonFiles, jsonPath => + { + BdnJsonResult? results = JsonConvert.DeserializeObject(File.ReadAllText(jsonPath)); + + List? benchmarks = results?.Benchmarks; + + if (benchmarks == null) + { + return; + } + + foreach (var benchmark in benchmarks) + { + Statistics statistics = benchmark.Statistics; + + MicrobenchmarkResult? microbenchmarkResult = null; + if ((!excludeTraces) && configuration.TraceConfigurations.Type != "none") + { + var jsonTraceMap = MapJsonToTraceForSingleBenchmarkRun(outputPathForRun, benchmarkFullName); + string tracePath = jsonTraceMap.GetValueOrDefault(jsonPath, ""); + + using (var analyzer = AnalyzerManager.GetAnalyzer(tracePath)) + { + List allPertinentProcesses = analyzer.GetProcessGCData("dotnet"); + List corerunProcesses = analyzer.GetProcessGCData("corerun"); + allPertinentProcesses.AddRange(corerunProcesses); + + GCProcessData? benchmarkGCData = null; + foreach (var process in allPertinentProcesses) + { + string commandLine = process.CommandLine.Replace("\"", "").Replace("\\", ""); + string runCleaned = benchmark.FullName.Replace("\"", "").Replace("\\", ""); + if (commandLine.Contains(runCleaned) && commandLine.Contains("--benchmarkName")) + { + benchmarkGCData = process; + break; + } + } + if (benchmarkGCData != null) + { + int processID = benchmarkGCData.ProcessID; + + /* + TODO: THIS NEEDS TO BE ADDED BACK. + if (configuration.Output.cpu_columns != null && configuration.Output.cpu_columns.Count > 0) + { + // TODO: Add parameterize. + benchmark.Value.GCData.Parent.AddCPUAnalysis(yamlPath: @"C:\Users\musharm\source\repos\GC.Analysis.API\GC.Analysis.API\CPUAnalysis\DefaultMethods.yaml", + symbolLogFile: Path.Combine(configuration.Output.Path, run.Key, Guid.NewGuid() + ".txt"), + symbolPath: Path.Combine(configuration.Output.Path, run.Key)); + var d1 = benchmark.Value.GCData.Parent.CPUAnalyzer.GetCPUDataForProcessName("dotnet"); + d1.AddRange(benchmark.Value.GCData.Parent.CPUAnalyzer.GetCPUDataForProcessName("corerun")); + benchmark.Value.CPUData = d1.FirstOrDefault(p => p.ProcessID == processID); + } + */ + microbenchmarkResult = new(benchmarkFullName, + run.Value, + benchmark, + gcData: benchmarkGCData, + gcTraceMetrics: new GCTraceMetrics(benchmarkGCData, tracePath, benchmark.FullName), + additionalReportMetrics: configuration.Output.additional_report_metrics, + cpuColumns: configuration.Output.cpu_columns, + columns: configuration.Output.Columns); + } + } + System.GC.Collect(2); + } + else + { + microbenchmarkResult = new(benchmarkFullName, + run.Value, + benchmark, + additionalReportMetrics: configuration.Output.additional_report_metrics, + cpuColumns: configuration.Output.cpu_columns, + columns: configuration.Output.Columns); + } + runsToResults[run.Value].Add(microbenchmarkResult!); + } + }); + + }); + + return runsToResults; + } + + public static List CompareMicrobenchmarkResultForBenchmark(MicrobenchmarkConfiguration configuration, string benchmarkFullName, bool excludeTraces = false) + { + bool includeTraces = (!excludeTraces) && configuration.TraceConfigurations.Type != "none"; + IReadOnlyDictionary> runResults = AnalyzeMicrobenchmarkResultsForSingleBenchmark(configuration, benchmarkFullName, excludeTraces); + List comparisonResults = new(); + if (configuration.Output.run_comparisons != null) + { + foreach (var comparison in configuration.Output.run_comparisons) + { + string[] breakup = comparison.Split(",", StringSplitOptions.TrimEntries); + string baselineName = breakup[0]; + string runName = breakup[1]; + + var baselineRuns = GoodLinq.Where(runResults.Keys, r => r.Name == baselineName); + var comparandRuns = GoodLinq.Where(runResults.Keys, r => r.Name == runName); + + var baselineMicrobenchmarkResults = GoodLinq.Select(baselineRuns, b => runResults[b]).SelectMany(r => r); + var comparandMicrobenchmarkResults = GoodLinq.Select(comparandRuns, c => runResults[c]).SelectMany(r => r); + + comparisonResults.Add(new(baselineMicrobenchmarkResults, comparandMicrobenchmarkResults, includeTraces)); + } + } + + // Default case where the run comparisons aren't specified. + else + { + var baselineRuns = GoodLinq.Where(runResults.Keys, r => r.is_baseline); + var comparandRuns = GoodLinq.Where(runResults.Keys, r => !r.is_baseline); + + var baselineMicrobenchmarkResults = GoodLinq.Select(baselineRuns, b => runResults[b]).SelectMany(r => r); + var comparandMicrobenchmarkResults = GoodLinq.Select(comparandRuns, c => runResults[c]).SelectMany(r => r); + + comparisonResults.Add(new(baselineMicrobenchmarkResults, comparandMicrobenchmarkResults, includeTraces)); + } + + return comparisonResults; + } + + public static List GroupComparisonResultsByName(MicrobenchmarkConfiguration configuration, List comparisonResultForAllBenchmarks, bool excludeTraces = false) + { + List allComparisonResults = new(); + + comparisonResultForAllBenchmarks + .GroupBy(r => r.ComparisonName) + .ToList() + .ForEach(group => + { + string baselineName = group.FirstOrDefault()?.BaselineRunName ?? "Baseline"; + string runName = group.FirstOrDefault()?.ComparandRunName ?? "Comparand"; + allComparisonResults.Add(new(baselineName, runName, group.ToList())); + }); + + return allComparisonResults; + } + } +} \ No newline at end of file diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResultsAnalyzer.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResultsAnalyzer.cs deleted file mode 100644 index 4ad9158e74c..00000000000 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResultsAnalyzer.cs +++ /dev/null @@ -1,178 +0,0 @@ -using GC.Analysis.API; -using GC.Infrastructure.Core.Configurations.Microbenchmarks; -using Newtonsoft.Json; -using System.Collections.Concurrent; - -namespace GC.Infrastructure.Core.Analysis.Microbenchmarks -{ - public static class MicrobenchmarkResultsAnalyzer - { - public static IReadOnlyDictionary> Analyze(MicrobenchmarkConfiguration configuration, bool excludeTraces = false) - { - ConcurrentDictionary> runsToResults = new(); - - Parallel.ForEach(configuration.Runs, (run) => - { - string outputPathForRun = Path.Combine(configuration.Output.Path, run.Key); - run.Value.Name ??= run.Key; - - // Find the json path. - string[] jsonFiles = Directory.GetFiles(outputPathForRun, "*full.json", SearchOption.AllDirectories); - - // Retrieve benchmarks from all the JSON files. - Parallel.ForEach(jsonFiles, (jsonFile) => - { - MicrobenchmarkResults results = JsonConvert.DeserializeObject(File.ReadAllText(jsonFile)); - foreach (var benchmark in results?.Benchmarks) - { - string title = benchmark.FullName; - Statistics statistics = benchmark.Statistics; - - if (!runsToResults.TryGetValue(run.Value, out var perBenchmarkData)) - { - runsToResults[run.Value] = perBenchmarkData = new ConcurrentDictionary(); - } - - runsToResults[run.Value][title] = new MicrobenchmarkResult - { - Statistics = statistics, - Parent = run.Value, - MicrobenchmarkName = title, - }; - } - }); - - if (!excludeTraces) - { - Dictionary analyzers = AnalyzerManager.GetAllAnalyzers(outputPathForRun); - - foreach (var analyzer in analyzers) - { - List allPertinentProcesses = analyzer.Value.GetProcessGCData("dotnet"); - List corerunProcesses = analyzer.Value.GetProcessGCData("corerun"); - allPertinentProcesses.AddRange(corerunProcesses); - foreach (var benchmark in runsToResults[run.Value]) - { - GCProcessData? benchmarkGCData = null; - foreach (var process in allPertinentProcesses) - { - string commandLine = process.CommandLine.Replace("\"", "").Replace("\\", ""); - string runCleaned = benchmark.Key.Replace("\"", "").Replace("\\", ""); - if (commandLine.Contains(runCleaned) && commandLine.Contains("--benchmarkName")) - { - benchmarkGCData = process; - break; - } - } - - if (benchmarkGCData != null) - { - int processID = benchmarkGCData.ProcessID; - benchmark.Value.GCData = benchmarkGCData; - benchmark.Value.ResultItem = new Presentation.GCPerfSim.ResultItem(benchmarkGCData, analyzer.Key, benchmark.Key); - /* - TODO: THIS NEEDS TO BE ADDED BACK. - if (configuration.Output.cpu_columns != null && configuration.Output.cpu_columns.Count > 0) - { - // TODO: Add parameterize. - benchmark.Value.GCData.Parent.AddCPUAnalysis(yamlPath: @"C:\Users\musharm\source\repos\GC.Analysis.API\GC.Analysis.API\CPUAnalysis\DefaultMethods.yaml", - symbolLogFile: Path.Combine(configuration.Output.Path, run.Key, Guid.NewGuid() + ".txt"), - symbolPath: Path.Combine(configuration.Output.Path, run.Key)); - var d1 = benchmark.Value.GCData.Parent.CPUAnalyzer.GetCPUDataForProcessName("dotnet"); - d1.AddRange(benchmark.Value.GCData.Parent.CPUAnalyzer.GetCPUDataForProcessName("corerun")); - benchmark.Value.CPUData = d1.FirstOrDefault(p => p.ProcessID == processID); - } - */ - } - } - }; - } - }); - - return runsToResults; - } - - public static IReadOnlyList GetComparisons(MicrobenchmarkConfiguration configuration, bool excludeTraces = false) - { - IReadOnlyDictionary> runResults = Analyze(configuration, excludeTraces); - List comparisonResults = new(); - - if (configuration.Output.run_comparisons != null) - { - foreach (var comparison in configuration.Output.run_comparisons) - { - string[] breakup = comparison.Split(",", StringSplitOptions.TrimEntries); - string baselineName = breakup[0]; - string runName = breakup[1]; - - Run run = runResults.Keys.FirstOrDefault(k => string.CompareOrdinal(k.Name, runName) == 0); - Run baselineRun = runResults.Keys.FirstOrDefault(k => string.CompareOrdinal(k.Name, baselineName) == 0); - - List microbenchmarkResults = new(); - - // Go through all the microbenchmarks for the current run and find the corresponding runs in the baseline. - foreach (var r in runResults[run]) - { - string microbenchmarkName = r.Key; - if (runResults[baselineRun].TryGetValue(microbenchmarkName, out var m)) - { - MicrobenchmarkComparisonResult microbenchmarkResult = new(m, r.Value); - microbenchmarkResults.Add(microbenchmarkResult); - } - - else - { - // TODO: Log the fact that we haven't found a corresponding result in the baseline. - Console.WriteLine($"Microbenchmark: {microbenchmarkName} isn't found on the baseline: {baselineName} for run: {runName}"); - } - } - - // At this point of time, the lack thereof of either of the runs should be a non-issue. - comparisonResults.Add(new MicrobenchmarkComparisonResults(baselineName, runName, microbenchmarkResults)); - } - } - - // Default case where the run comparisons aren't specified. - else - { - string baselineName = configuration.Runs.FirstOrDefault(r => r.Value.is_baseline).Key; - KeyValuePair> baselineResult = baselineName != null ? runResults.First(r => r.Key.Name == baselineName) : runResults.First(); - - // For each run, we want to grab it and it's baseline and then do a per microbenchmark association. - foreach (var runResult in runResults) - { - Run run = runResult.Key; - string runName = run.Name; - - if (string.CompareOrdinal(runName, baselineName) == 0) - { - continue; - } - - List microbenchmarkResults = new(); - - // Go through all the microbenchmarks for the current run and find the corresponding runs in the baseline. - foreach (var r in runResult.Value) - { - string microbenchmarkName = r.Key; - if (baselineResult.Value.TryGetValue(microbenchmarkName, out var m)) - { - MicrobenchmarkComparisonResult microbenchmarkResult = new(m, r.Value); - microbenchmarkResults.Add(microbenchmarkResult); - } - - else - { - Console.WriteLine($"Microbenchmark: {microbenchmarkName} isn't found on the baseline: {baselineName} for run: {runName}"); - // TODO: Log the fact that we haven't found a corresponding result in the baseline. - } - } - - comparisonResults.Add(new MicrobenchmarkComparisonResults(baselineName, runName, microbenchmarkResults)); - } - } - - return comparisonResults; - } - } -} diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Configurations/InputConfiguration.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Configurations/InputConfiguration.cs index fb2546f9bb7..b0ca7613bc6 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Configurations/InputConfiguration.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Configurations/InputConfiguration.cs @@ -13,6 +13,8 @@ public sealed class InputConfiguration // public Dictionary? clrgcs { get; set; } // public string? debug_parameters { get; set; } + public Dictionary? iterations { get; set; } + public Dictionary? symbol_path { get; set; } public Dictionary? source_path { get; set; } } diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Configurations/Microbenchmarks.Configuration.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Configurations/Microbenchmarks.Configuration.cs index c44e02cd947..ab9ab01a911 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Configurations/Microbenchmarks.Configuration.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Configurations/Microbenchmarks.Configuration.cs @@ -24,7 +24,13 @@ public sealed class Run : RunBase public class Environment { public uint default_max_seconds { get; set; } = 300; - public uint iteration { get; set; } = 1; + public uint iterations { get; set; } = 1; + + [YamlMember(Alias = "iteration")] + public uint iteration + { + set => iterations = value; + } } public sealed class MicrobenchmarkConfigurations diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Json/Json.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Json.cs similarity index 50% rename from src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Json/Json.cs rename to src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Json.cs index 3d547c03a0c..9bf2736ef0a 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Json/Json.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Json.cs @@ -1,14 +1,16 @@ -using GC.Infrastructure.Core.Analysis.Microbenchmarks; +using GC.Analysis.API; +using GC.Infrastructure.Core.Analysis.Microbenchmarks; using GC.Infrastructure.Core.Configurations.Microbenchmarks; +using GC.Infrastructure.Core.Presentation.GCPerfSim; using Newtonsoft.Json; -namespace GC.Infrastructure.Core.Presentation.Microbenchmarks.Json +namespace GC.Infrastructure.Core.Presentation.Microbenchmarks { public static class Json { - public static void Generate(MicrobenchmarkConfiguration configuration, IReadOnlyList comparisonResults, string path) + public static void Generate(MicrobenchmarkConfiguration configuration, List comparisonResultsGroupedByName, string path) { - string json = JsonConvert.SerializeObject(comparisonResults); + string json = JsonConvert.SerializeObject(comparisonResultsGroupedByName); File.WriteAllText(path, json); } } diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Json/JsonOutput.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Json/JsonOutput.cs deleted file mode 100644 index 9840432b2ba..00000000000 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Json/JsonOutput.cs +++ /dev/null @@ -1,12 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; - -namespace GC.Infrastructure.Core.Presentation.Microbenchmarks.Json -{ - public sealed class JsonOutput - { - } -} diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Markdown.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Markdown.cs index 76dffd7f87d..73f25eab143 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Markdown.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Markdown.cs @@ -11,23 +11,23 @@ public static class Markdown private const string baseTableString = "| Benchmark Name | Baseline | Comparand | Baseline Mean Duration (MSec) | Comparand Mean Duration (MSec) | Δ Mean Duration (MSec) | Δ% Mean Duration |"; private const string baseTableRows = "| --- | --- | -- | --- | --- | --- | --- | "; - public static void GenerateTable(MicrobenchmarkConfiguration configuration, IReadOnlyList comparisonResults, Dictionary executionDetails, string path) + public static void GenerateTable(MicrobenchmarkConfiguration configuration, IReadOnlyList comparisonResultsCollection, Dictionary executionDetails, string path) { using (StreamWriter sw = new StreamWriter(path)) { // Create summary. sw.WriteLine("# Summary"); - string header = $"| Criteria | {string.Join("|", GoodLinq.Select(comparisonResults, s => $"[{s.BaselineName} {s.RunName}]({s.MarkdownIdentifier})"))}|"; + string header = $"| Criteria | {string.Join("|", GoodLinq.Select(comparisonResultsCollection, s => $"[{s.BaselineName} {s.RunName}]({s.MarkdownIdentifier})"))}|"; sw.WriteLine(header); - sw.WriteLine($"| ----- | {string.Join("|", Enumerable.Repeat(" ----- ", comparisonResults.Count))} |"); - sw.WriteLine($"| Large Regressions (>20%) | {GoodLinq.Sum(comparisonResults, s => s.LargeRegressions.Count())}|"); - sw.WriteLine($"| Regressions (5% - 20%) | {GoodLinq.Sum(comparisonResults, s => s.Regressions.Count())}|"); - sw.WriteLine($"| Stale Regressions (0% - 5%) | {GoodLinq.Sum(comparisonResults, s => s.StaleRegressions.Count())}|"); - sw.WriteLine($"| Stale Improvements (0% - 5%) | {GoodLinq.Sum(comparisonResults, s => s.StaleImprovements.Count())}|"); - sw.WriteLine($"| Improvements (5% - 20%) | {GoodLinq.Sum(comparisonResults, s => s.Improvements.Count())}|"); - sw.WriteLine($"| Large Improvements (>20%) | {GoodLinq.Sum(comparisonResults, s => s.LargeImprovements.Count())}|"); - sw.WriteLine($"| Total | {comparisonResults.Count} |"); + sw.WriteLine($"| ----- | {string.Join("|", Enumerable.Repeat(" ----- ", comparisonResultsCollection.Count))} |"); + sw.WriteLine($"| Large Regressions (>20%) | {GoodLinq.Sum(comparisonResultsCollection, s => s.LargeRegressions.Count())}|"); + sw.WriteLine($"| Regressions (5% - 20%) | {GoodLinq.Sum(comparisonResultsCollection, s => s.Regressions.Count())}|"); + sw.WriteLine($"| Stale Regressions (0% - 5%) | {GoodLinq.Sum(comparisonResultsCollection, s => s.StaleRegressions.Count())}|"); + sw.WriteLine($"| Stale Improvements (0% - 5%) | {GoodLinq.Sum(comparisonResultsCollection, s => s.StaleImprovements.Count())}|"); + sw.WriteLine($"| Improvements (5% - 20%) | {GoodLinq.Sum(comparisonResultsCollection, s => s.Improvements.Count())}|"); + sw.WriteLine($"| Large Improvements (>20%) | {GoodLinq.Sum(comparisonResultsCollection, s => s.LargeImprovements.Count())}|"); + sw.WriteLine($"| Total | {comparisonResultsCollection.Count} |"); sw.WriteLine("\n"); // Incomplete Tests. @@ -41,9 +41,9 @@ public static void GenerateTable(MicrobenchmarkConfiguration configuration, IRea sw.WriteLine("## Individual Results"); // Add details of Each Comparison. - foreach (var comparisonResult in comparisonResults) + foreach (var comparisonResults in comparisonResultsCollection) { - AddDetailsOfSingleComparison(sw, configuration, comparisonResult); + AddDetailsOfSingleComparison(sw, configuration, comparisonResults); sw.WriteLine("\n"); } } @@ -89,42 +89,36 @@ internal static void AddDetailsOfSingleComparison(this StreamWriter sw, Microben foreach (var metric in configuration.Output.additional_report_metrics) { sw.WriteLine($"## Comparison by {metric}"); - var ordered = comparisonResult.Comparisons.OrderByDescending(c => c.GetDiffPercentFromOtherMetrics(metric)); + var ordered = comparisonResult.Comparisons.OrderByDescending(c => c.OtherMetricsDiffPerc[metric]); // Large Regressions sw.WriteLine($"### Large Regressions (>20%): {comparisonResult.LargeRegressions.Count()} \n"); - sw.AddTableForSingleCriteria(configuration, GoodLinq.Where(ordered, o => o.GetDiffPercentFromOtherMetrics(metric) > 0.2)); + sw.AddTableForSingleCriteria(configuration, GoodLinq.Where(ordered, o => o.OtherMetricsDiffPerc[metric] > 0.2)); sw.WriteLine("\n"); // Large Improvements sw.WriteLine($"### Large Improvements (>20%): {comparisonResult.LargeImprovements.Count()} \n"); - var largeImprovements = GoodLinq.Where(ordered, o => o.GetDiffPercentFromOtherMetrics(metric) < -0.2); - largeImprovements.Reverse(); - sw.AddTableForSingleCriteria(configuration, largeImprovements); + sw.AddTableForSingleCriteria(configuration, GoodLinq.Where(ordered, o => o.OtherMetricsDiffPerc[metric] < -0.2)); sw.WriteLine("\n"); // Regressions sw.WriteLine($"### Regressions (5% - 20%): {comparisonResult.Regressions.Count()} \n"); - sw.AddTableForSingleCriteria(configuration, GoodLinq.Where(ordered, o => o.GetDiffPercentFromOtherMetrics(metric) > 0.05 && o.GetDiffPercentFromOtherMetrics(metric) < 0.2)); + sw.AddTableForSingleCriteria(configuration, GoodLinq.Where(ordered, o => o.OtherMetricsDiffPerc[metric] > 0.05 && o.OtherMetricsDiffPerc[metric] < 0.2)); sw.WriteLine("\n"); // Improvements sw.WriteLine($"### Improvements (5% - 20%): {comparisonResult.Improvements.Count()} \n"); - var improvements = GoodLinq.Where(ordered, o => o.GetDiffPercentFromOtherMetrics(metric) > 0.05 && o.GetDiffPercentFromOtherMetrics(metric) < 0.2); - improvements.Reverse(); - sw.AddTableForSingleCriteria(configuration, improvements); + sw.AddTableForSingleCriteria(configuration, GoodLinq.Where(ordered, o => o.OtherMetricsDiffPerc[metric] > 0.05 && o.OtherMetricsDiffPerc[metric] < 0.2)); sw.WriteLine("\n"); // Stale Regressions sw.WriteLine($"### Stale Regressions (Same or percent difference within 5% margin): {comparisonResult.StaleRegressions.Count()} \n"); - sw.AddTableForSingleCriteria(configuration, GoodLinq.Where(ordered, o => o.GetDiffPercentFromOtherMetrics(metric) < 0.05 && o.GetDiffPercentFromOtherMetrics(metric) >= 0.0)); + sw.AddTableForSingleCriteria(configuration, GoodLinq.Where(ordered, o => o.OtherMetricsDiffPerc[metric] < 0.05 && o.OtherMetricsDiffPerc[metric] >= 0.0)); sw.WriteLine("\n"); // Stale Improvements sw.WriteLine($"### Stale Improvements (Same or percent difference within 5% margin): {comparisonResult.StaleImprovements.Count()} \n"); - var staleImprovements = GoodLinq.Where(ordered, o => o.GetDiffPercentFromOtherMetrics(metric) > -0.05 && o.GetDiffPercentFromOtherMetrics(metric) < 0.0); - staleImprovements.Reverse(); - sw.AddTableForSingleCriteria(configuration, staleImprovements); + sw.AddTableForSingleCriteria(configuration, GoodLinq.Where(ordered, o => o.OtherMetricsDiffPerc[metric] > -0.05 && o.OtherMetricsDiffPerc[metric] < 0.0)); sw.WriteLine("\n"); } } @@ -161,24 +155,18 @@ internal static void AddTableForSingleCriteria(this StreamWriter sw, Microbenchm { try { - var baseRow = $"| {lr.MicrobenchmarkName} | {lr.BaselineRunName} | {lr.ComparandRunName} | {Math.Round(lr.Baseline.Statistics.Mean.Value, 2)} | {Math.Round(lr.Comparand.Statistics.Mean.Value, 2)} | {Math.Round(lr.MeanDiff, 2)}| {Math.Round(lr.MeanDiffPerc, 2)}|"; + string benchmarkName = lr.MicrobenchmarkName.Replace("<", "\\<").Replace(">", "\\>"); + var baseRow = $"| {benchmarkName} | {lr.BaselineRunName} | {lr.ComparandRunName} | {Math.Round(lr.AveragedBaselineMeanValue, 2)} | {Math.Round(lr.AveragedComparandMeanValue, 2)} | {Math.Round(lr.MeanDiff, 2)}| {Math.Round(lr.MeanDiffPerc, 2)}|"; if (configuration.Output.Columns != null) { foreach (var column in configuration.Output.Columns) { - if (!lr.Baseline.OtherMetrics.TryGetValue(column, out double? baselineValue)) - { - lr.Baseline.OtherMetrics[column] = baselineValue = API.GCProcessData.LookupAggregateCalculation(column, lr.Baseline.GCData) ?? MicrobenchmarkResult.LookupStatisticsCalculation(column, lr.Baseline); - } - string baselineResult = baselineValue.HasValue ? Math.Round(baselineValue.Value, 4).ToString() : string.Empty; + double? baselineValue = lr.AveragedBaselineOtherMetrics.GetValueOrDefault(column); + double? comparandValue = lr.AveragedComparandOtherMetrics.GetValueOrDefault(column); - if (!lr.Comparand.OtherMetrics.TryGetValue(column, out double? comparandValue)) - { - lr.Comparand.OtherMetrics[column] = comparandValue = API.GCProcessData.LookupAggregateCalculation(column, lr.Comparand.GCData) ?? MicrobenchmarkResult.LookupStatisticsCalculation(column, lr.Comparand); - } + string baselineResult = baselineValue.HasValue ? Math.Round(baselineValue.Value, 4).ToString() : string.Empty; string comparandResult = comparandValue.HasValue ? Math.Round(comparandValue.Value, 4).ToString() : string.Empty; - double? delta = baselineValue.HasValue && comparandValue.HasValue ? comparandValue.Value - baselineValue.Value : null; string deltaResult = delta.HasValue ? Math.Round(delta.Value, 4).ToString() : string.Empty; @@ -189,31 +177,31 @@ internal static void AddTableForSingleCriteria(this StreamWriter sw, Microbenchm } } - if (configuration.Output.cpu_columns != null) - { - foreach (var column in configuration.Output.cpu_columns) - { - if (!lr.Baseline.OtherMetrics.TryGetValue(column, out double? baselineValue)) - { - lr.Baseline.OtherMetrics[column] = baselineValue = lr.Baseline.CPUData?.GetIncCountForGCMethod(column) ?? null; - } - string baselineResult = baselineValue.HasValue ? Math.Round(baselineValue.Value, 2).ToString() : string.Empty; - - if (!lr.Comparand.OtherMetrics.TryGetValue(column, out double? comparandValue)) - { - lr.Comparand.OtherMetrics[column] = comparandValue = lr.Comparand.CPUData?.GetIncCountForGCMethod(column) ?? null; - } - string comparandResult = comparandValue.HasValue ? Math.Round(comparandValue.Value, 2).ToString() : string.Empty; + //if (configuration.Output.cpu_columns != null) + //{ + // foreach (var column in configuration.Output.cpu_columns) + // { + // if (!lr.Baseline.OtherMetrics.TryGetValue(column, out double? baselineValue)) + // { + // lr.Baseline.OtherMetrics[column] = baselineValue = lr.Baseline.CPUData?.GetIncCountForGCMethod(column) ?? null; + // } + // string baselineResult = baselineValue.HasValue ? Math.Round(baselineValue.Value, 2).ToString() : string.Empty; - double? delta = baselineValue.HasValue && comparandValue.HasValue ? comparandValue.Value - baselineValue.Value : null; - string deltaResult = delta.HasValue ? Math.Round(delta.Value, 2).ToString() : string.Empty; + // if (!lr.Comparand.OtherMetrics.TryGetValue(column, out double? comparandValue)) + // { + // lr.Comparand.OtherMetrics[column] = comparandValue = lr.Comparand.CPUData?.GetIncCountForGCMethod(column) ?? null; + // } + // string comparandResult = comparandValue.HasValue ? Math.Round(comparandValue.Value, 2).ToString() : string.Empty; - double? deltaPercent = delta.HasValue ? (delta / baselineValue.Value) * 100 : null; - string deltaPercentResult = deltaPercent.HasValue ? Math.Round(deltaPercent.Value, 2).ToString() : string.Empty; + // double? delta = baselineValue.HasValue && comparandValue.HasValue ? comparandValue.Value - baselineValue.Value : null; + // string deltaResult = delta.HasValue ? Math.Round(delta.Value, 2).ToString() : string.Empty; - baseRow += $"{baselineResult} | {comparandResult} | {deltaResult} | {deltaPercentResult} |"; - } - } + // double? deltaPercent = delta.HasValue ? (delta / baselineValue.Value) * 100 : null; + // string deltaPercentResult = deltaPercent.HasValue ? Math.Round(deltaPercent.Value, 2).ToString() : string.Empty; + + // baseRow += $"{baselineResult} | {comparandResult} | {deltaResult} | {deltaPercentResult} |"; + // } + //} sw.WriteLine(baseRow); } @@ -224,15 +212,6 @@ internal static void AddTableForSingleCriteria(this StreamWriter sw, Microbenchm Console.WriteLine(e.StackTrace); } } - - // Dispose all the Analyzers now that we are persisting the values. - foreach (var comparison in comparisons) - { - comparison.Baseline?.GCData?.Parent?.Dispose(); - comparison.Baseline?.CPUData?.Parent?.Analyzer?.Dispose(); - comparison.Comparand?.GCData?.Parent?.Dispose(); - comparison.Comparand?.CPUData?.Parent?.Analyzer?.Dispose(); - } } } } diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Presentation.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Presentation.cs index 93f4faea7f0..8d879a48ecd 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Presentation.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Presentation.cs @@ -1,31 +1,27 @@ using GC.Infrastructure.Core.Analysis; using GC.Infrastructure.Core.Analysis.Microbenchmarks; -using GC.Infrastructure.Core.Configurations; using GC.Infrastructure.Core.Configurations.Microbenchmarks; namespace GC.Infrastructure.Core.Presentation.Microbenchmarks { public static class Presentation { - public static IReadOnlyList Present(MicrobenchmarkConfiguration configuration, Dictionary executionDetails) + public static void Present(MicrobenchmarkConfiguration configuration, List comparisonResultsGroupedByName, Dictionary executionDetails) { - IReadOnlyList comparisonResults = MicrobenchmarkResultsAnalyzer.GetComparisons(configuration); foreach (var format in configuration.Output.Formats) { if (format == "markdown") { - Markdown.GenerateTable(configuration, comparisonResults, executionDetails, Path.Combine(configuration.Output.Path, "Results.md")); + Markdown.GenerateTable(configuration, comparisonResultsGroupedByName, executionDetails, Path.Combine(configuration.Output.Path, "Results.md")); continue; } if (format == "json") { - Json.Json.Generate(configuration, comparisonResults, Path.Combine(configuration.Output.Path, "Results.json")); + Json.Generate(configuration, comparisonResultsGroupedByName, Path.Combine(configuration.Output.Path, "Results.json")); continue; } } - - return comparisonResults; } } } diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/Microbenchmark/MicrobenchmarkAnalyzeCommand.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/Microbenchmark/MicrobenchmarkAnalyzeCommand.cs index 35c4e50a266..b07cde1f428 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/Microbenchmark/MicrobenchmarkAnalyzeCommand.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/Microbenchmark/MicrobenchmarkAnalyzeCommand.cs @@ -1,10 +1,11 @@ -using Spectre.Console.Cli; -using System.ComponentModel; -using System.Diagnostics.CodeAnalysis; -using GC.Infrastructure.Core.Analysis.Microbenchmarks; -using GC.Infrastructure.Core.Presentation.Microbenchmarks; +using GC.Infrastructure.Core.Analysis.Microbenchmarks; using GC.Infrastructure.Core.Configurations; using GC.Infrastructure.Core.Configurations.Microbenchmarks; +using GC.Infrastructure.Core.Presentation.Microbenchmarks; +using Spectre.Console; +using Spectre.Console.Cli; +using System.ComponentModel; +using System.Diagnostics.CodeAnalysis; namespace GC.Infrastructure.Commands.Microbenchmark { @@ -21,9 +22,42 @@ public override int Execute([NotNull] CommandContext context, [NotNull] Microben { ConfigurationChecker.VerifyFile(settings.ConfigurationPath, nameof(MicrobenchmarkAnalyzeCommand)); MicrobenchmarkConfiguration configuration = MicrobenchmarkConfigurationParser.Parse(settings.ConfigurationPath); - IReadOnlyList comparisonResults = MicrobenchmarkResultsAnalyzer.GetComparisons(configuration); - Presentation.Present(configuration, new()); // Execution details aren't available for the analysis-only mode. + + var comparisonResultsGroupedName = ExecuteAnalysis(configuration); + + Presentation.Present(configuration, comparisonResultsGroupedName, new()); // Execution details aren't available for the analysis-only mode. return 0; } + + public static List ExecuteAnalysis(MicrobenchmarkConfiguration configuration) + { + Run? run = configuration.Runs.Values.FirstOrDefault(); + if (run == null) + { + throw new InvalidOperationException("No runs found in the configuration."); + } + string outputPathForRun = Path.Combine(configuration.Output.Path, run.Name); + var benchmarkFullNameJsonMap = MicrobenchmarkResultComparison.MapBenchmarkFullNameToJsonForRun(outputPathForRun); + List comparisonResultForAllBenchmarks = new(); + + ParallelOptions options = new ParallelOptions + { + MaxDegreeOfParallelism = System.Environment.ProcessorCount + }; + + object _lock = new(); + + Parallel.ForEach(benchmarkFullNameJsonMap.Keys, options, benchmarkFullName => + { + List comparisonResultsForBenchmark = MicrobenchmarkResultComparison.CompareMicrobenchmarkResultForBenchmark(configuration, benchmarkFullName); + AnsiConsole.Markup($"[bold green] ({DateTime.Now}) Analysis For Microbenchmarks: {benchmarkFullName} completed. [/]\n"); + lock (_lock) + { + comparisonResultForAllBenchmarks.AddRange(comparisonResultsForBenchmark); + } + }); + + return MicrobenchmarkResultComparison.GroupComparisonResultsByName(configuration, comparisonResultForAllBenchmarks); + } } } diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/Microbenchmark/MicrobenchmarkCommand.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/Microbenchmark/MicrobenchmarkCommand.cs index ca063389ef5..74cb749e680 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/Microbenchmark/MicrobenchmarkCommand.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/Microbenchmark/MicrobenchmarkCommand.cs @@ -1,18 +1,19 @@ -using GC.Infrastructure.Core.Analysis.Microbenchmarks; +using GC.Analysis.API; using GC.Infrastructure.Core.Analysis; +using GC.Infrastructure.Core.Analysis.Microbenchmarks; using GC.Infrastructure.Core.CommandBuilders; -using GC.Infrastructure.Core.Configurations.Microbenchmarks; using GC.Infrastructure.Core.Configurations; +using GC.Infrastructure.Core.Configurations.Microbenchmarks; using GC.Infrastructure.Core.Presentation.Microbenchmarks; using GC.Infrastructure.Core.TraceCollection; using Newtonsoft.Json; -using Spectre.Console.Cli; using Spectre.Console; +using Spectre.Console.Cli; using System.ComponentModel; -using System.Diagnostics.CodeAnalysis; +using System.Configuration; using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; using System.Text; -using System.Configuration; namespace GC.Infrastructure.Commands.Microbenchmark { @@ -121,11 +122,11 @@ public static MicrobenchmarkOutputResults RunMicrobenchmarks(MicrobenchmarkConfi // Should only be one if it's a fresh run. string jsonFile = jsonFiles.First(); - MicrobenchmarkResults output = JsonConvert.DeserializeObject(File.ReadAllText(jsonFile)); + BdnJsonResult output = JsonConvert.DeserializeObject(File.ReadAllText(jsonFile)); // Assumption: A particular run, regardless of the parameters, will run ~the same vals. - IEnumerable operationsPerNanos = output.Benchmarks.First().Measurements.Where(m => m.IterationMode == "Workload" && m.IterationStage == "Actual") - .Select(m => m.Operations); + var operationsPerNanos = GoodLinq.Select(GoodLinq.Where(output.Benchmarks.First().Measurements, m => m.IterationMode == "Workload" && m.IterationStage == "Actual"), m => m.Operations); + // For now take the max but we will possibly be sacrificing duration for precision. invocationCountFromBaseline = operationsPerNanos.Max(); } @@ -149,9 +150,9 @@ public static MicrobenchmarkOutputResults RunMicrobenchmarks(MicrobenchmarkConfi (string, string) fileNameAndCommand = MicrobenchmarkCommandBuilder.Build(configuration, run, benchmark, invocationCountFromBaseline); run.Value.Name = run.Key; - for (int index = 0; index < configuration.Environment.iteration; index++) + for (int index = 0; index < configuration.Environment.iterations; index++) { - AnsiConsole.Markup($"[bold green] ({DateTime.Now}) Running Microbechmarks: {configuration.Name} - {run.Key} {benchmark} - iteration: {index} [/]\n"); + AnsiConsole.Markup($"[bold green] ({DateTime.Now}) Running Microbenchmarks: {configuration.Name} - {run.Key} {benchmark} - iteration: {index} [/]\n"); // Run The BDN process with the trace collector. using (Process bdnProcess = new()) { @@ -199,10 +200,12 @@ public static MicrobenchmarkOutputResults RunMicrobenchmarks(MicrobenchmarkConfi } } - IReadOnlyList results = Presentation.Present(configuration, executionDetails); + var comparisonResultsGroupedName = MicrobenchmarkAnalyzeCommand.ExecuteAnalysis(configuration); + + Presentation.Present(configuration, comparisonResultsGroupedName, executionDetails); // Execution details aren't available for the analysis-only mode. Directory.SetCurrentDirectory(currentDirectory); AnsiConsole.Markup($"[bold green] ({DateTime.Now}) Wrote Microbechmark Results to: {Markup.Escape(Path.Combine(configuration.Output.Path, "Results.md"))} [/]"); - return new MicrobenchmarkOutputResults(executionDetails, results); + return new MicrobenchmarkOutputResults(executionDetails, comparisonResultsGroupedName); } } } diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/RunCommand/BaseSuite/Microbenchmarks.yaml b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/RunCommand/BaseSuite/Microbenchmarks.yaml index ea719bad713..257f021c024 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/RunCommand/BaseSuite/Microbenchmarks.yaml +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/RunCommand/BaseSuite/Microbenchmarks.yaml @@ -6,7 +6,7 @@ microbenchmark_configurations: environment: default_max_seconds: 3000 - iteration: 1 + iterations: 1 # Configurations that involve capturing a trace. trace_configurations: diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/RunCommand/BaseSuite/MicrobenchmarksToRun.txt b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/RunCommand/BaseSuite/MicrobenchmarksToRun.txt index 1bef991eca5..6056a312aff 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/RunCommand/BaseSuite/MicrobenchmarksToRun.txt +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/RunCommand/BaseSuite/MicrobenchmarksToRun.txt @@ -11,7 +11,6 @@ "System.Tests.Perf_GC.NewOperator_Array(length: 10000)" | "System.Tests.Perf_GC.NewOperator_Array(length: 1000)" | "System.Tests.Perf_GC.NewOperator_Array(length: 10000)" | -"System.IO.Tests.Perf_File.ReadAllBytesAsync(size: 104857600)" | "System.Numerics.Tests.Perf_BigInteger.Subtract(arguments: 65536*" | "System.Collections.CtorGivenSize.Array(size: 512)" | "ByteMark.BenchBitOps" | diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/RunCommand/CreateSuiteCommand.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/RunCommand/CreateSuiteCommand.cs index 0cec453d0c3..96a6943019b 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/RunCommand/CreateSuiteCommand.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/RunCommand/CreateSuiteCommand.cs @@ -225,6 +225,12 @@ internal static MicrobenchmarkConfiguration CreateBaseMicrobenchmarkSuite(InputC }); } + // Set iterations if they exist. + if (inputConfiguration.iterations != null) + { + configuration.Environment.iterations = inputConfiguration.iterations.GetValueOrDefault("microbenchmarks", 1); + } + // The first run is always the baseline. configuration.Runs.First().Value.is_baseline = true; From 68e6f03178fa6f34f67057b9eb9956ef22e66928 Mon Sep 17 00:00:00 2001 From: "Vincent Bu (Centific Technologies Inc)" Date: Thu, 14 May 2026 15:08:10 +0800 Subject: [PATCH 2/3] redesign microbenchmarkresult MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit move presentation to analyze-command improve performance of analyzing stage includes int type properties check key existence and set parallelism degree to 2 * cpu_count check if metricName is a key of StatsData Filter out null GCTraceMetrics instances before calling CompareGCTraceMetric update initialization of OtherMetrics for MicrobenchmarkComparisonResult comment out cpu_columns related code remove unused imports "Microbechmark" (missing 'n') avoid breaking formatting sort improvements in ascending order Convert to ToDictionary(x => x.Item1, x => x.Item2) (or tuple names) for each projection provide key value pair projection sort in order by index take non-replayable enumerables into consideration check if TraceConfigurations is null skip null or empty MicrobenchmarkResult collection skip null or empty MicrobenchmarkResults collection remove Spectre markup tokens from output path Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> Check if first baseline/comparand is null Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> apply q1 and q3 on valid collection use run.Name instead of trace path check possible null value check if metric is in OtherMetricsDiffPerc Add a guard for an empty GC list check metric for both baseline and comparand formatting check keys existence don't drop bdn results even gcprocessdata are not found fix possible bugs fix bugs for addtional metrics comparison results presentation Disable NuGet package pruning to keep Microsoft.Extensions.* assemblies app-local (#5209) Starting with .NET 10, NuGet's package-pruning feature (NU1510) flags direct PackageReferences to packages that are now in-band in the Microsoft.AspNetCore.App shared framework (Microsoft.Extensions.Configuration, .DependencyInjection, .Caching.Memory, .Http, .Logging, etc.) as well as some that have moved into Microsoft.NETCore.App (System.Formats.Cbor). With TreatWarningsAsErrors this fails the build for net10.0+/net11.0+ TFMs. Rather than removing those PackageReferences and adding a FrameworkReference to Microsoft.AspNetCore.App, set RestoreEnablePackagePruning=false (and suppress NU1510). This keeps the assemblies app-local, which is required for BenchmarkDotNet's corerun toolchain: that toolchain points at a CoreRoot under shared/Microsoft.NETCore.App/... which does not contain Microsoft.AspNetCore.App or its assemblies, so an in-band/FrameworkReference approach would break perf runs that use --corerun against a runtime CoreRoot. Verified locally with benchmarks_ci.py -f net11.0 (build) and a smoke run of ActivatorUtilitiesBenchmark.* under --corerun. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Fix GC pipeline (#5210) * Fix GC pipeline The GC pipeline has been broken for some time, failing with `[error]No image label found to route agent pool Azure Pipelines` * Switch to NetCore-Public with ImageOverride Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * Temporary change to trigger the GC pipeline * Update gc-azure-pipelines.yml Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * Try to use the vmImage again * Add missing .NET 11 * Switch tests to core unit tests * Try to fix the test task by explicitly specifying .NET 8.0 framework * Fix indent tab and build config * Remove temporary change in .cs file --------- Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Re-enable Cobalt perf runs (#5212) The Cobalt machines are back online, so re-enable the cobaltMicro, cobaltSveMicro, and cobaltMicroR2RInterpreter jobs that were temporarily disabled. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Add explicit PackageReferences for transitive Microsoft.Extensions.* … (#5211) * Add explicit PackageReferences for transitive Microsoft.Extensions.* abstractions For .NET 10+, NuGet package pruning removes transitive Microsoft.Extensions.* abstractions (Logging.Abstractions, DependencyInjection.Abstractions, Options, Primitives) from project.assets.json because they are in-band in Microsoft.AspNetCore.App. Without them, the build fails with CS0246 for types like ILogger, IChangeToken, ObjectFactory, StringSegment, etc. Disabling pruning via RestoreEnablePackagePruning=false (PR #5209) only preserves DIRECT PackageReferences. Transitive prunable deps are still removed. Add them as explicit PackageReferences so they remain app-local for BenchmarkDotNet's corerun toolchain (which targets a Microsoft.NETCore.App CoreRoot that does not include the AspNetCore.App shared framework). Also re-enable Microsoft.Extensions.Primitives for net10+ (PR #5196 wrongly removed it assuming it was in NETCore.App; it is actually in AspNetCore.App). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Also add Microsoft.Extensions.Configuration.Abstractions explicit ref Same root cause: NuGet pruning removes this transitive dep on net10+, causing CS0246 for IConfiguration in ConfigurationBinderBenchmarks. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Also add Microsoft.Extensions.Caching.Abstractions explicit ref Same root cause: NuGet pruning removes this transitive dep on net10+, causing CS0012/CS0246 for ICacheEntry and IMemoryCache. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Shield SDK resolver from upstream global.json with paths entry Write an empty global.json at the workspace root (parent of the perf repo) before invoking dotnet-install. This stops the SDK resolver's upward walk from finding dotnet/runtime's global.json, which contains a 'paths' entry pointing at a '.dotnet' directory holding an older SDK than the one perf installs into tools/dotnet/x64. Without this shield, ci_setup.py's 'dotnet --info' resolves to the wrong SDK and propagates an outdated DOTNET_VERSION to Helix work items, breaking builds on test machines whose ref pack predates recent runtime API changes (e.g. the Sve API rename). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Strip sdk.paths instead of writing empty global.json Writing {} broke arcade's bootstrap because it requires tools.dotnet to be present in global.json. Instead, parse the upstream global.json and remove only the sdk.paths field, preserving all other keys. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Apply suggestions from code review Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --------- Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Exclude Perf_String.Split_Csv from WASM runs (#5214) The Split_Csv benchmark reads CSV test data files via File.ReadAllLines from AppContext.BaseDirectory. These files are not bundled into the WASM publish output, so CsvCorpus() throws DirectoryNotFoundException when BDN extracts arguments in the spawned WASM benchmark process. Add [BenchmarkCategory(Categories.NoWASM)] to skip the benchmark on WASM, matching the existing pattern used for other filesystem-dependent benchmarks in this repo. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Add test coverage for ResultsComparer and CLI helper paths (#5199) This expands automated coverage in the repo’s tooling-focused test surface without large production refactors. * Add coverage for ResultsComparer and CLI helpers * Fix Base classification in matrix comparisons * Run tooling tests in CI * Skip tooling tests on main * Limit tooling tests to public non-main runs * Fix brittle matrix comparison test * Restore culture after invoking ResultsComparer * Bump Newtonsoft.Json for tooling tests * Update src/tools/ResultsComparer.Tests/DataTests.cs --------- Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Rename MicroBenchmarks.Serializers.BinaryData to BinaryDataPayload (#5216) The serialization-test helper class `MicroBenchmarks.Serializers.BinaryData` shadowed the in-box `System.BinaryData` type (introduced in net6+ via `System.Memory.Data`, brought in transitively by Azure.Core). Inside `DataGenerator.cs` (namespace MicroBenchmarks.Serializers) the unqualified `typeof(BinaryData)` always bound to the local helper class. Inside `ReadJson.cs` / `WriteJson.cs` (namespace System.Text.Json.Serialization.Tests) C# enclosing-namespace lookup found `System.BinaryData` first, so the `[GenericTypeArguments(typeof(BinaryData))]` attribute resolved to a DIFFERENT type than the `Generate()` switch was checking. The mismatch was latent on main (where System.Memory.Data wasn't compile-visible to ReadJson.cs) but surfaced as `System.NotImplementedException` in `DataGenerator.Generate()` whenever the reference closure changed (e.g. retargeting BenchmarkDotNet.Extensions from netstandard2.0 to net8.0). Renaming the helper class to `BinaryDataPayload` removes the shadowing once and for all. No behavior change on main; future-proofs against reference-closure churn. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Bump urllib3 from 2.6.3 to 2.7.0 (#5218) Bumps [urllib3](https://github.com/urllib3/urllib3) from 2.6.3 to 2.7.0. - [Release notes](https://github.com/urllib3/urllib3/releases) - [Changelog](https://github.com/urllib3/urllib3/blob/main/CHANGES.rst) - [Commits](https://github.com/urllib3/urllib3/compare/2.6.3...2.7.0) --- updated-dependencies: - dependency-name: urllib3 dependency-version: 2.7.0 dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Bump cryptography from 46.0.3 to 46.0.7 (#5207) Bumps [cryptography](https://github.com/pyca/cryptography) from 46.0.3 to 46.0.7. - [Changelog](https://github.com/pyca/cryptography/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pyca/cryptography/compare/46.0.3...46.0.7) --- updated-dependencies: - dependency-name: cryptography dependency-version: 46.0.7 dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> [main] Update dependencies from dotnet/android (#5206) * Update dependencies from https://github.com/dotnet/android build 10.0.70-ci.main.26215.1+azdo.13844220 On relative base path root Microsoft.Android.Sdk.Windows From Version 36.1.55 -> To Version 36.1.59 * Update dependencies from https://github.com/dotnet/android build 10.0.70-ci.main.26216.1+azdo.13861683 On relative base path root Microsoft.Android.Sdk.Windows From Version 36.1.55 -> To Version 36.1.60 * Update dependencies from https://github.com/dotnet/android build 10.0.70-ci.main.26220.1+azdo.13883894 On relative base path root Microsoft.Android.Sdk.Windows From Version 36.1.55 -> To Version 36.1.61 * Update dependencies from https://github.com/dotnet/android build 10.0.70-ci.main.26228.1+azdo.13957946 On relative base path root Microsoft.Android.Sdk.Windows From Version 36.1.55 -> To Version 36.1.62 * Update dependencies from https://github.com/dotnet/android build 10.0.70-ci.main.26254.1+azdo.14005692 On relative base path root Microsoft.Android.Sdk.Windows From Version 36.1.55 -> To Version 36.1.63 * Update dependencies from https://github.com/dotnet/android build 10.0.70-ci.main.26254.1+azdo.14008404 On relative base path root Microsoft.Android.Sdk.Windows From Version 36.1.55 -> To Version 36.1.64 * Update dependencies from https://github.com/dotnet/android build 10.0.70-ci.main.26261.1+azdo.14066865 On relative base path root Microsoft.Android.Sdk.Windows From Version 36.1.55 -> To Version 36.1.65 --------- Co-authored-by: dotnet-maestro[bot] [main] Update dependencies from dotnet/dotnet (#5205) * Update dependencies from https://github.com/dotnet/dotnet build 20260414.3 On relative base path root Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.Helix.Sdk From Version 11.0.0-beta.26210.111 -> To Version 11.0.0-beta.26214.103 Microsoft.Extensions.Logging , Microsoft.NET.ILLink , Microsoft.NET.ILLink.Tasks , Microsoft.NET.Runtime.Emscripten.3.1.56.Node.win-x64 , Microsoft.NETCore.App.Ref , Microsoft.WindowsDesktop.App.Ref , System.Threading.Channels From Version 11.0.0-preview.4.26210.111 -> To Version 11.0.0-preview.4.26214.103 Microsoft.NET.Sdk From Version 11.0.100-preview.4.26210.111 -> To Version 11.0.100-preview.4.26214.103 * Update dependencies from https://github.com/dotnet/dotnet build 20260415.1 On relative base path root Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.Helix.Sdk From Version 11.0.0-beta.26210.111 -> To Version 11.0.0-beta.26215.101 Microsoft.Extensions.Logging , Microsoft.NET.ILLink , Microsoft.NET.ILLink.Tasks , Microsoft.NET.Runtime.Emscripten.3.1.56.Node.win-x64 , Microsoft.NETCore.App.Ref , Microsoft.WindowsDesktop.App.Ref , System.Threading.Channels From Version 11.0.0-preview.4.26210.111 -> To Version 11.0.0-preview.4.26215.101 Microsoft.NET.Sdk From Version 11.0.100-preview.4.26210.111 -> To Version 11.0.100-preview.4.26215.101 * Update dependencies from https://github.com/dotnet/dotnet build 20260415.8 On relative base path root Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.Helix.Sdk From Version 11.0.0-beta.26210.111 -> To Version 11.0.0-beta.26215.108 Microsoft.Extensions.Logging , Microsoft.NET.ILLink , Microsoft.NET.ILLink.Tasks , Microsoft.NET.Runtime.Emscripten.3.1.56.Node.win-x64 , Microsoft.NETCore.App.Ref , Microsoft.WindowsDesktop.App.Ref , System.Threading.Channels From Version 11.0.0-preview.4.26210.111 -> To Version 11.0.0-preview.4.26215.108 Microsoft.NET.Sdk From Version 11.0.100-preview.4.26210.111 -> To Version 11.0.100-preview.4.26215.108 * Update dependencies from https://github.com/dotnet/dotnet build 20260415.14 On relative base path root Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.Helix.Sdk From Version 11.0.0-beta.26210.111 -> To Version 11.0.0-beta.26215.114 Microsoft.Extensions.Logging , Microsoft.NET.ILLink , Microsoft.NET.ILLink.Tasks , Microsoft.NET.Runtime.Emscripten.3.1.56.Node.win-x64 , Microsoft.NETCore.App.Ref , Microsoft.WindowsDesktop.App.Ref , System.Threading.Channels From Version 11.0.0-preview.4.26210.111 -> To Version 11.0.0-preview.4.26215.114 Microsoft.NET.Sdk From Version 11.0.100-preview.4.26210.111 -> To Version 11.0.100-preview.4.26215.114 * Update dependencies from https://github.com/dotnet/dotnet build 20260415.21 On relative base path root Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.Helix.Sdk From Version 11.0.0-beta.26210.111 -> To Version 11.0.0-beta.26215.121 Microsoft.Extensions.Logging , Microsoft.NET.ILLink , Microsoft.NET.ILLink.Tasks , Microsoft.NET.Runtime.Emscripten.3.1.56.Node.win-x64 , Microsoft.NETCore.App.Ref , Microsoft.WindowsDesktop.App.Ref , System.Threading.Channels From Version 11.0.0-preview.4.26210.111 -> To Version 11.0.0-preview.4.26215.121 Microsoft.NET.Sdk From Version 11.0.100-preview.4.26210.111 -> To Version 11.0.100-preview.4.26215.121 * Update dependencies from https://github.com/dotnet/dotnet build 20260412.2 On relative base path root Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.Helix.Sdk From Version 11.0.0-beta.26210.111 -> To Version 11.0.0-beta.26212.102 Microsoft.Extensions.Logging , Microsoft.NET.ILLink , Microsoft.NET.ILLink.Tasks , Microsoft.NET.Runtime.Emscripten.3.1.56.Node.win-x64 , Microsoft.NETCore.App.Ref , Microsoft.WindowsDesktop.App.Ref , System.Threading.Channels From Version 11.0.0-preview.4.26210.111 -> To Version 11.0.0-preview.4.26212.102 Microsoft.NET.Sdk From Version 11.0.100-preview.4.26210.111 -> To Version 11.0.100-preview.4.26212.102 * Update dependencies from https://github.com/dotnet/dotnet build 20260411.2 On relative base path root Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.Helix.Sdk From Version 11.0.0-beta.26210.111 -> To Version 11.0.0-beta.26211.102 Microsoft.Extensions.Logging , Microsoft.NET.ILLink , Microsoft.NET.ILLink.Tasks , Microsoft.NET.Runtime.Emscripten.3.1.56.Node.win-x64 , Microsoft.NETCore.App.Ref , Microsoft.WindowsDesktop.App.Ref , System.Threading.Channels From Version 11.0.0-preview.4.26210.111 -> To Version 11.0.0-preview.4.26211.102 Microsoft.NET.Sdk From Version 11.0.100-preview.4.26210.111 -> To Version 11.0.100-preview.4.26211.102 * Update dependencies from https://github.com/dotnet/dotnet build 20260422.42 On relative base path root Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.Helix.Sdk From Version 11.0.0-beta.26210.111 -> To Version 11.0.0-beta.26222.142 Microsoft.Extensions.Logging , Microsoft.NET.ILLink , Microsoft.NET.ILLink.Tasks , Microsoft.NET.Runtime.Emscripten.3.1.56.Node.win-x64 , Microsoft.NETCore.App.Ref , Microsoft.WindowsDesktop.App.Ref , System.Threading.Channels From Version 11.0.0-preview.4.26210.111 -> To Version 11.0.0-preview.4.26222.142 Microsoft.NET.Sdk From Version 11.0.100-preview.4.26210.111 -> To Version 11.0.100-preview.4.26222.142 * Update dependencies from https://github.com/dotnet/dotnet build 20260423.12 On relative base path root Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.Helix.Sdk From Version 11.0.0-beta.26210.111 -> To Version 11.0.0-beta.26223.112 Microsoft.Extensions.Logging , Microsoft.NET.ILLink , Microsoft.NET.ILLink.Tasks , Microsoft.NET.Runtime.Emscripten.3.1.56.Node.win-x64 , Microsoft.NETCore.App.Ref , Microsoft.WindowsDesktop.App.Ref , System.Threading.Channels From Version 11.0.0-preview.4.26210.111 -> To Version 11.0.0-preview.4.26223.112 Microsoft.NET.Sdk From Version 11.0.100-preview.4.26210.111 -> To Version 11.0.100-preview.4.26223.112 * Update dependencies from https://github.com/dotnet/dotnet build 20260424.4 On relative base path root Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.Helix.Sdk From Version 11.0.0-beta.26210.111 -> To Version 11.0.0-beta.26224.104 Microsoft.Extensions.Logging , Microsoft.NET.ILLink , Microsoft.NET.ILLink.Tasks , Microsoft.NET.Runtime.Emscripten.3.1.56.Node.win-x64 , Microsoft.NETCore.App.Ref , Microsoft.WindowsDesktop.App.Ref , System.Threading.Channels From Version 11.0.0-preview.4.26210.111 -> To Version 11.0.0-preview.4.26224.104 Microsoft.NET.Sdk From Version 11.0.100-preview.4.26210.111 -> To Version 11.0.100-preview.4.26224.104 * Update dependencies from https://github.com/dotnet/dotnet build 20260424.23 On relative base path root Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.Helix.Sdk From Version 11.0.0-beta.26210.111 -> To Version 11.0.0-beta.26224.123 Microsoft.Extensions.Logging , Microsoft.NET.ILLink , Microsoft.NET.ILLink.Tasks , Microsoft.NET.Runtime.Emscripten.3.1.56.Node.win-x64 , Microsoft.NETCore.App.Ref , Microsoft.WindowsDesktop.App.Ref , System.Threading.Channels From Version 11.0.0-preview.4.26210.111 -> To Version 11.0.0-preview.5.26224.123 Microsoft.NET.Sdk From Version 11.0.100-preview.4.26210.111 -> To Version 11.0.100-preview.5.26224.123 * Update dependencies from https://github.com/dotnet/dotnet build 20260426.11 On relative base path root Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.Helix.Sdk From Version 11.0.0-beta.26210.111 -> To Version 11.0.0-beta.26226.111 Microsoft.Extensions.Logging , Microsoft.NET.ILLink , Microsoft.NET.ILLink.Tasks , Microsoft.NET.Runtime.Emscripten.3.1.56.Node.win-x64 , Microsoft.NETCore.App.Ref , Microsoft.WindowsDesktop.App.Ref , System.Threading.Channels From Version 11.0.0-preview.4.26210.111 -> To Version 11.0.0-preview.5.26226.111 Microsoft.NET.Sdk From Version 11.0.100-preview.4.26210.111 -> To Version 11.0.100-preview.5.26226.111 * Update dependencies from https://github.com/dotnet/dotnet build 20260427.4 On relative base path root Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.Helix.Sdk From Version 11.0.0-beta.26210.111 -> To Version 11.0.0-beta.26227.104 Microsoft.Extensions.Logging , Microsoft.NET.ILLink , Microsoft.NET.ILLink.Tasks , Microsoft.NET.Runtime.Emscripten.3.1.56.Node.win-x64 , Microsoft.NETCore.App.Ref , Microsoft.WindowsDesktop.App.Ref , System.Threading.Channels From Version 11.0.0-preview.4.26210.111 -> To Version 11.0.0-preview.5.26227.104 Microsoft.NET.Sdk From Version 11.0.100-preview.4.26210.111 -> To Version 11.0.100-preview.5.26227.104 * Update dependencies from https://github.com/dotnet/dotnet build 20260427.24 On relative base path root Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.Helix.Sdk From Version 11.0.0-beta.26210.111 -> To Version 11.0.0-beta.26227.124 Microsoft.Extensions.Logging , Microsoft.NET.ILLink , Microsoft.NET.ILLink.Tasks , Microsoft.NET.Runtime.Emscripten.3.1.56.Node.win-x64 , Microsoft.NETCore.App.Ref , Microsoft.WindowsDesktop.App.Ref , System.Threading.Channels From Version 11.0.0-preview.4.26210.111 -> To Version 11.0.0-preview.5.26227.124 Microsoft.NET.Sdk From Version 11.0.100-preview.4.26210.111 -> To Version 11.0.100-preview.5.26227.124 * Update dependencies from https://github.com/dotnet/dotnet build 20260427.31 On relative base path root Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.Helix.Sdk From Version 11.0.0-beta.26210.111 -> To Version 11.0.0-beta.26227.131 Microsoft.Extensions.Logging , Microsoft.NET.ILLink , Microsoft.NET.ILLink.Tasks , Microsoft.NET.Runtime.Emscripten.3.1.56.Node.win-x64 , Microsoft.NETCore.App.Ref , Microsoft.WindowsDesktop.App.Ref , System.Threading.Channels From Version 11.0.0-preview.4.26210.111 -> To Version 11.0.0-preview.5.26227.131 Microsoft.NET.Sdk From Version 11.0.100-preview.4.26210.111 -> To Version 11.0.100-preview.5.26227.131 * Update dependencies from https://github.com/dotnet/dotnet build 20260428.2 On relative base path root Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.Helix.Sdk From Version 11.0.0-beta.26210.111 -> To Version 11.0.0-beta.26228.102 Microsoft.Extensions.Logging , Microsoft.NET.ILLink , Microsoft.NET.ILLink.Tasks , Microsoft.NET.Runtime.Emscripten.3.1.56.Node.win-x64 , Microsoft.NETCore.App.Ref , Microsoft.WindowsDesktop.App.Ref , System.Threading.Channels From Version 11.0.0-preview.4.26210.111 -> To Version 11.0.0-preview.5.26228.102 Microsoft.NET.Sdk From Version 11.0.100-preview.4.26210.111 -> To Version 11.0.100-preview.5.26228.102 * Update dependencies from https://github.com/dotnet/dotnet build 20260428.23 On relative base path root Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.Helix.Sdk From Version 11.0.0-beta.26210.111 -> To Version 11.0.0-beta.26228.123 Microsoft.Extensions.Logging , Microsoft.NET.ILLink , Microsoft.NET.ILLink.Tasks , Microsoft.NET.Runtime.Emscripten.3.1.56.Node.win-x64 , Microsoft.NETCore.App.Ref , Microsoft.WindowsDesktop.App.Ref , System.Threading.Channels From Version 11.0.0-preview.4.26210.111 -> To Version 11.0.0-preview.5.26228.123 Microsoft.NET.Sdk From Version 11.0.100-preview.4.26210.111 -> To Version 11.0.100-preview.5.26228.123 * Update dependencies from https://github.com/dotnet/dotnet build 20260429.13 On relative base path root Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.Helix.Sdk From Version 11.0.0-beta.26210.111 -> To Version 11.0.0-beta.26229.113 Microsoft.Extensions.Logging , Microsoft.NET.ILLink , Microsoft.NET.ILLink.Tasks , Microsoft.NET.Runtime.Emscripten.3.1.56.Node.win-x64 , Microsoft.NETCore.App.Ref , Microsoft.WindowsDesktop.App.Ref , System.Threading.Channels From Version 11.0.0-preview.4.26210.111 -> To Version 11.0.0-preview.5.26229.113 Microsoft.NET.Sdk From Version 11.0.100-preview.4.26210.111 -> To Version 11.0.100-preview.5.26229.113 * Update dependencies from https://github.com/dotnet/dotnet build 20260430.1 On relative base path root Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.Helix.Sdk From Version 11.0.0-beta.26210.111 -> To Version 11.0.0-beta.26230.101 Microsoft.Extensions.Logging , Microsoft.NET.ILLink , Microsoft.NET.ILLink.Tasks , Microsoft.NET.Runtime.Emscripten.3.1.56.Node.win-x64 , Microsoft.NETCore.App.Ref , Microsoft.WindowsDesktop.App.Ref , System.Threading.Channels From Version 11.0.0-preview.4.26210.111 -> To Version 11.0.0-preview.5.26230.101 Microsoft.NET.Sdk From Version 11.0.100-preview.4.26210.111 -> To Version 11.0.100-preview.5.26230.101 * Update dependencies from https://github.com/dotnet/dotnet build 20260501.10 On relative base path root Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.Helix.Sdk From Version 11.0.0-beta.26210.111 -> To Version 11.0.0-beta.26251.110 Microsoft.Extensions.Logging , Microsoft.NET.ILLink , Microsoft.NET.ILLink.Tasks , Microsoft.NET.Runtime.Emscripten.3.1.56.Node.win-x64 , Microsoft.NETCore.App.Ref , Microsoft.WindowsDesktop.App.Ref , System.Threading.Channels From Version 11.0.0-preview.4.26210.111 -> To Version 11.0.0-preview.5.26251.110 Microsoft.NET.Sdk From Version 11.0.100-preview.4.26210.111 -> To Version 11.0.100-preview.5.26251.110 * Update dependencies from https://github.com/dotnet/dotnet build 20260501.12 On relative base path root Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.Helix.Sdk From Version 11.0.0-beta.26210.111 -> To Version 11.0.0-beta.26251.112 Microsoft.Extensions.Logging , Microsoft.NET.ILLink , Microsoft.NET.ILLink.Tasks , Microsoft.NET.Runtime.Emscripten.3.1.56.Node.win-x64 , Microsoft.NETCore.App.Ref , Microsoft.WindowsDesktop.App.Ref , System.Threading.Channels From Version 11.0.0-preview.4.26210.111 -> To Version 11.0.0-preview.5.26251.112 Microsoft.NET.Sdk From Version 11.0.100-preview.4.26210.111 -> To Version 11.0.100-preview.5.26251.112 * Update dependencies from https://github.com/dotnet/dotnet build 20260504.3 On relative base path root Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.Helix.Sdk From Version 11.0.0-beta.26210.111 -> To Version 11.0.0-beta.26254.103 Microsoft.Extensions.Logging , Microsoft.NET.ILLink , Microsoft.NET.ILLink.Tasks , Microsoft.NET.Runtime.Emscripten.3.1.56.Node.win-x64 , Microsoft.NETCore.App.Ref , Microsoft.WindowsDesktop.App.Ref , System.Threading.Channels From Version 11.0.0-preview.4.26210.111 -> To Version 11.0.0-preview.5.26254.103 Microsoft.NET.Sdk From Version 11.0.100-preview.4.26210.111 -> To Version 11.0.100-preview.5.26254.103 * Update dependencies from https://github.com/dotnet/dotnet build 20260504.12 On relative base path root Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.Helix.Sdk From Version 11.0.0-beta.26210.111 -> To Version 11.0.0-beta.26254.112 Microsoft.Extensions.Logging , Microsoft.NET.ILLink , Microsoft.NET.ILLink.Tasks , Microsoft.NET.Runtime.Emscripten.3.1.56.Node.win-x64 , Microsoft.NETCore.App.Ref , Microsoft.WindowsDesktop.App.Ref , System.Threading.Channels From Version 11.0.0-preview.4.26210.111 -> To Version 11.0.0-preview.5.26254.112 Microsoft.NET.Sdk From Version 11.0.100-preview.4.26210.111 -> To Version 11.0.100-preview.5.26254.112 * Update dependencies from https://github.com/dotnet/dotnet build 20260505.1 On relative base path root Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.Helix.Sdk From Version 11.0.0-beta.26210.111 -> To Version 11.0.0-beta.26255.101 Microsoft.Extensions.Logging , Microsoft.NET.ILLink , Microsoft.NET.ILLink.Tasks , Microsoft.NET.Runtime.Emscripten.3.1.56.Node.win-x64 , Microsoft.NETCore.App.Ref , Microsoft.WindowsDesktop.App.Ref , System.Threading.Channels From Version 11.0.0-preview.4.26210.111 -> To Version 11.0.0-preview.5.26255.101 Microsoft.NET.Sdk From Version 11.0.100-preview.4.26210.111 -> To Version 11.0.100-preview.5.26255.101 * Update dependencies from https://github.com/dotnet/dotnet build 20260505.6 On relative base path root Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.Helix.Sdk From Version 11.0.0-beta.26210.111 -> To Version 11.0.0-beta.26255.106 Microsoft.Extensions.Logging , Microsoft.NET.ILLink , Microsoft.NET.ILLink.Tasks , Microsoft.NET.Runtime.Emscripten.3.1.56.Node.win-x64 , Microsoft.NETCore.App.Ref , Microsoft.WindowsDesktop.App.Ref , System.Threading.Channels From Version 11.0.0-preview.4.26210.111 -> To Version 11.0.0-preview.5.26255.106 Microsoft.NET.Sdk From Version 11.0.100-preview.4.26210.111 -> To Version 11.0.100-preview.5.26255.106 * Update dependencies from https://github.com/dotnet/dotnet build 20260506.1 On relative base path root Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.Helix.Sdk From Version 11.0.0-beta.26210.111 -> To Version 11.0.0-beta.26256.101 Microsoft.Extensions.Logging , Microsoft.NET.ILLink , Microsoft.NET.ILLink.Tasks , Microsoft.NET.Runtime.Emscripten.3.1.56.Node.win-x64 , Microsoft.NETCore.App.Ref , Microsoft.WindowsDesktop.App.Ref , System.Threading.Channels From Version 11.0.0-preview.4.26210.111 -> To Version 11.0.0-preview.5.26256.101 Microsoft.NET.Sdk From Version 11.0.100-preview.4.26210.111 -> To Version 11.0.100-preview.5.26256.101 * Update dependencies from https://github.com/dotnet/dotnet build 20260506.5 On relative base path root Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.Helix.Sdk From Version 11.0.0-beta.26210.111 -> To Version 11.0.0-beta.26256.105 Microsoft.Extensions.Logging , Microsoft.NET.ILLink , Microsoft.NET.ILLink.Tasks , Microsoft.NET.Runtime.Emscripten.3.1.56.Node.win-x64 , Microsoft.NETCore.App.Ref , Microsoft.WindowsDesktop.App.Ref , System.Threading.Channels From Version 11.0.0-preview.4.26210.111 -> To Version 11.0.0-preview.5.26256.105 Microsoft.NET.Sdk From Version 11.0.100-preview.4.26210.111 -> To Version 11.0.100-preview.5.26256.105 * Update dependencies from https://github.com/dotnet/dotnet build 20260507.13 On relative base path root Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.Helix.Sdk From Version 11.0.0-beta.26210.111 -> To Version 11.0.0-beta.26257.113 Microsoft.Extensions.Logging , Microsoft.NET.ILLink , Microsoft.NET.ILLink.Tasks , Microsoft.NET.Runtime.Emscripten.3.1.56.Node.win-x64 , Microsoft.NETCore.App.Ref , Microsoft.WindowsDesktop.App.Ref , System.Threading.Channels From Version 11.0.0-preview.4.26210.111 -> To Version 11.0.0-preview.5.26257.113 Microsoft.NET.Sdk From Version 11.0.100-preview.4.26210.111 -> To Version 11.0.100-preview.5.26257.113 * Update dependencies from https://github.com/dotnet/dotnet build 20260508.7 On relative base path root Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.Helix.Sdk From Version 11.0.0-beta.26210.111 -> To Version 11.0.0-beta.26258.107 Microsoft.Extensions.Logging , Microsoft.NET.ILLink , Microsoft.NET.ILLink.Tasks , Microsoft.NET.Runtime.Emscripten.3.1.56.Node.win-x64 , Microsoft.NETCore.App.Ref , Microsoft.WindowsDesktop.App.Ref , System.Threading.Channels From Version 11.0.0-preview.4.26210.111 -> To Version 11.0.0-preview.5.26258.107 Microsoft.NET.Sdk From Version 11.0.100-preview.4.26210.111 -> To Version 11.0.100-preview.5.26258.107 * Update dependencies from https://github.com/dotnet/dotnet build 20260508.10 On relative base path root Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.Helix.Sdk From Version 11.0.0-beta.26210.111 -> To Version 11.0.0-beta.26258.110 Microsoft.Extensions.Logging , Microsoft.NET.ILLink , Microsoft.NET.ILLink.Tasks , Microsoft.NET.Runtime.Emscripten.3.1.56.Node.win-x64 , Microsoft.NETCore.App.Ref , Microsoft.WindowsDesktop.App.Ref , System.Threading.Channels From Version 11.0.0-preview.4.26210.111 -> To Version 11.0.0-preview.5.26258.110 Microsoft.NET.Sdk From Version 11.0.100-preview.4.26210.111 -> To Version 11.0.100-preview.5.26258.110 * Update dependencies from https://github.com/dotnet/dotnet build 20260511.1 On relative base path root Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.Helix.Sdk From Version 11.0.0-beta.26210.111 -> To Version 11.0.0-beta.26261.101 Microsoft.Extensions.Logging , Microsoft.NET.ILLink , Microsoft.NET.ILLink.Tasks , Microsoft.NET.Runtime.Emscripten.3.1.56.Node.win-x64 , Microsoft.NETCore.App.Ref , Microsoft.WindowsDesktop.App.Ref , System.Threading.Channels From Version 11.0.0-preview.4.26210.111 -> To Version 11.0.0-preview.5.26261.101 Microsoft.NET.Sdk From Version 11.0.100-preview.4.26210.111 -> To Version 11.0.100-preview.5.26261.101 --------- Co-authored-by: dotnet-maestro[bot] Co-authored-by: Parker Bibus Drop daily quality from release/10.0 and 10.0 channels (#5219) After .NET 10 GA, `-Quality daily` for these channels resolves to internal-only servicing SDK builds (e.g. 10.0.9) whose matching `Microsoft.NETCore.App.Runtime.` runtime pack has not yet shipped publicly on the dotnet10 / dotnet-public NuGet feeds. That breaks the CertHelper `--self-contained` publish in run_performance_job.py with NU1102. With no quality specified, dotnet-install hits the public `aka.ms/dotnet/10.0/` GA endpoint (currently SDK 10.0.203 / runtime 10.0.7), whose runtime packs are publicly available, so the publish step succeeds. main (net11) and nativeaot* channels are unchanged. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Update bdn 0.16 binarydata and async (#5217) * Enable runtime-async for net11.0+ * Update BenchmarkDotNet to 0.16.0-nightly.20260518.1249 Bumps BenchmarkDotNet to the latest master nightly, built from dotnet/BenchmarkDotNet@c7632225 ("Disassembly follow jump trampolines (#3136)") and pushed to the benchmark-dotnet-prerelease internal feed. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Migrate BenchmarkDotNet.Extensions to BDN 0.16.0 APIs BDN 0.16.0 ('Async Refactor', PR #2958) made breaking API changes that the harness still consumed: - ExporterBase.ExportToLog(Summary, ILogger) was removed; the new abstract ExportAsync uses an internal CancelableStreamWriter that subclasses can't satisfy. PerfLabExporter now implements IExporter directly. - IValidator.Validate returning IEnumerable was replaced with ValidateAsync returning IAsyncEnumerable. The four validators (UniqueArguments, TooManyTestCases, NoWasm, MandatoryCategory) use .ToAsyncEnumerable() (transitively from BDN's System.Linq.AsyncEnumerable dep), not async + yield return - the latter produces an AsyncIteratorMethodBuilder state machine that deadlocks with BDN's BenchmarkSynchronizationContext. Switched BenchmarkDotNet.Extensions from netstandard2.0 to net8.0. The netstandard2.0 target was only used to enable an opt-in net472 path, but PERFLAB_TARGET_FRAMEWORKS=net472 is no longer exercised in CI and the new BDN APIs would otherwise require polyfill packages. Removed the net472 package conditional from MicroBenchmarks.csproj and the net472/.NET Framework references from docs/benchmarkdotnet.md and the micro README. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Fix nullability errors surfaced by net8.0 retarget Switching BenchmarkDotNet.Extensions from netstandard2.0 to net8.0 brought in stricter nullable annotations from the BCL and BDN package, which combined with TreatWarningsAsErrors=true (set in src/Directory.Build.props) turned previously-hidden nullability warnings into build errors. - ValuesGenerator.Dictionary: add 'where TKey : notnull' constraint required by Dictionary<,>. - UniqueArgumentsValidator.BenchmarkArgumentsComparer.Equals: match the IEqualityComparer.Equals(T?, T?) interface signature; handle nulls. - TooManyTestCasesValidator.SkipValidation: parameter must be MemberInfo? since MemberInfo.DeclaringType returns Type?. - DiffableDisassemblyExporter: add null-forgiving (!) on reflection lookups whose return values are intentionally trusted by this type (it's a copy of internal BDN code that operates on known types). - PerfLabExporter: BuildJson can return null; use string?. Verified by running the exact CI commands locally (dotnet restore + build with --framework net11.0) using the SDK from global.json. Build succeeds with 0 errors. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Switch micro entrypoint to RunAsync to avoid discovery-time deadlock BDN 0.16's sync entrypoints (BenchmarkSwitcher.Run / BenchmarkRunner.Run) install BenchmarkDotNetSynchronizationContext (a single-threaded message pump) before benchmark discovery. Discovery executes [ParamsSource] and [ArgumentsSource] callbacks; some perf-repo callbacks do sync-over-async (notably SslStreamTests.GetTls13Support, which calls HandshakeAsync(...) .GetAwaiter().GetResult()). Sync-over-async deadlocks on the single-threaded SyncCtx because the awaited continuation is queued back to a pump that the caller is blocking. Switch Program.Main to async Task and await BenchmarkSwitcher.RunAsync. The async entrypoint never installs BenchmarkDotNetSynchronizationContext on the caller, so discovery runs on the default context and sync-over-async in source callbacks no longer deadlocks. Real benchmark execution still gets the SyncCtx semantics it needs because BDN installs it inside the per-benchmark execute path, not on the entrypoint thread. This is the supported BDN-recommended fix for the discovery-deadlock symptom; no BDN code change is required. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Gate runtime-async behind 'runtimeasync' experiment PR #5195 enabled the runtime-async language feature unconditionally for net11.0+ via ``. To avoid affecting baseline measurement runs, gate it behind the existing experiment infrastructure so it only takes effect in the dedicated experiment lane. - src/Directory.Build.targets: only set `runtime-async=on` when the `EnableRuntimeAsync` MSBuild property is `true` (in addition to the existing TFM check). - scripts/ci_setup.py: when `--experiment-name=runtimeasync` is passed, emit `EnableRuntimeAsync=true` as an env var so MSBuild picks it up as a property (matches the pattern used by the `jitoptrepeat` experiment). Verified locally: BDN dry runs against the BinaryDataPayload tests succeed both with and without the env var (18/18 benchmarks each). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --------- Co-authored-by: Andy Gocke Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> remove empty constructor and unused imports avoid printing dup command line args --- NuGet.config | 5 +- docs/benchmarkdotnet.md | 14 +- eng/Version.Details.xml | 44 +-- eng/Versions.props | 12 +- eng/common/AGENTS.md | 5 + eng/common/core-templates/job/job.yml | 5 +- eng/common/core-templates/job/onelocbuild.yml | 3 + .../job/publish-build-assets.yml | 12 +- eng/common/core-templates/job/renovate.yml | 2 +- eng/common/core-templates/jobs/jobs.yml | 5 + .../post-build/common-variables.yml | 2 - .../core-templates/post-build/post-build.yml | 134 ++++--- .../post-build/setup-maestro-vars.yml | 5 +- .../steps/component-governance.yml | 16 - .../core-templates/steps/generate-sbom.yml | 60 +-- .../core-templates/steps/publish-logs.yml | 10 +- .../core-templates/steps/source-build.yml | 2 +- eng/common/cross/toolchain.cmake | 12 +- eng/common/darc-init.ps1 | 8 +- eng/common/darc-init.sh | 4 +- eng/common/dotnet-install.ps1 | 6 +- eng/common/dotnet-install.sh | 6 +- eng/common/generate-sbom-prep.ps1 | 29 -- eng/common/generate-sbom-prep.sh | 39 -- eng/common/post-build/redact-logs.ps1 | 4 +- .../post-build/sourcelink-validation.ps1 | 327 ---------------- eng/common/sdk-task.ps1 | 15 +- eng/common/template-guidance.md | 2 - eng/common/templates-official/job/job.yml | 57 +-- .../steps/component-governance.yml | 7 - .../steps/publish-pipeline-artifacts.yml | 4 +- eng/common/templates/job/job.yml | 53 +-- .../templates/steps/component-governance.yml | 7 - eng/common/tools.ps1 | 141 +++---- eng/common/tools.sh | 66 ++-- eng/pipelines/runtime-perf-jobs.yml | 7 +- eng/pipelines/sdk-perf-jobs.yml | 37 ++ gc-azure-pipelines.yml | 49 ++- global.json | 8 +- requirements.txt | 4 +- scripts/channel_map.py | 6 +- scripts/ci_setup.py | 5 + scripts/dotnet.py | 31 ++ src/Directory.Build.props | 10 +- src/Directory.Build.targets | 11 + .../GC.Analysis.API/Statistics.cs | 17 +- .../Analysis/BdnJsonResult.cs | 44 +-- .../Analysis/GCTraceMetricComparisonResult.cs | 17 +- .../Analysis/GCTraceMetrics.cs | 12 +- .../MicrobenchmarkComparisonResult.cs | 68 +++- .../MicrobenchmarkComparisonResults.cs | 2 +- .../Microbenchmarks/MicrobenchmarkResult.cs | 16 +- .../MicrobenchmarkResultComparison.cs | 351 +++++++++++------- .../Presentation/MarkdownReportBuilder.cs | 8 +- .../Presentation/Microbenchmarks/Markdown.cs | 102 +++-- .../Microbenchmarks/Presentation.cs | 27 -- .../MicrobenchmarkAnalyzeCommand.cs | 55 +-- .../Microbenchmark/MicrobenchmarkCommand.cs | 6 +- src/benchmarks/micro/MicroBenchmarks.csproj | 28 +- src/benchmarks/micro/Program.cs | 16 +- src/benchmarks/micro/README.md | 2 +- .../micro/Serializers/DataGenerator.cs | 16 +- .../libraries/System.Runtime/Perf.String.cs | 1 + .../System.Text.Json/Serializer/ReadJson.cs | 2 +- .../System.Text.Json/Serializer/WriteJson.cs | 2 +- .../BenchmarkDotNet.Extensions.csproj | 8 +- .../CommandLineOptions.cs | 4 +- .../DiffableDisassemblyExporter.cs | 14 +- .../MandatoryCategoryValidator.cs | 5 +- .../NoWasmValidator.cs | 5 +- .../PerfLabExporter.cs | 63 +++- .../TooManyTestCasesValidator.cs | 7 +- .../UniqueArgumentsValidator.cs | 12 +- .../ValuesGenerator.cs | 2 +- .../BenchmarkDotNet.Extensions.Tests.csproj | 4 +- .../CommandLineOptionsTests.cs | 84 +++++ .../PartitionFilterTests.cs | 7 +- .../UniqueArgumentsValidatorTests.cs | 3 +- .../UniqueValuesGeneratorTests.cs | 4 +- src/tools/Reporting/Directory.Packages.props | 2 +- .../ConsoleOutputCollection.cs | 8 + src/tools/ResultsComparer.Tests/DataTests.cs | 161 ++++++++ .../ResultsComparer.Tests/HelperTests.cs | 137 +++++++ .../ResultsComparer.Tests/ProgramTests.cs | 158 ++++++++ .../ResultsComparer.Tests.csproj | 13 + .../ResultsComparerTestData.cs | 52 +++ src/tools/ResultsComparer.Tests/StatsTests.cs | 81 ++++ src/tools/ResultsComparer/Data.cs | 15 +- .../ResultsComparer/Directory.Packages.props | 2 +- .../ResultsComparer/MultipleInputsComparer.cs | 8 +- src/tools/ResultsComparer/Program.cs | 4 +- .../Properties/AssemblyInfo.cs | 3 + src/tools/ResultsComparer/ResultsComparer.sln | 29 ++ src/tools/ResultsComparer/Stats.cs | 2 +- .../ResultsComparer/TwoInputsComparer.cs | 10 +- .../Startup.Tests/StartupTests.cs | 23 +- 96 files changed, 1804 insertions(+), 1209 deletions(-) create mode 100644 eng/common/AGENTS.md delete mode 100644 eng/common/core-templates/steps/component-governance.yml delete mode 100644 eng/common/generate-sbom-prep.ps1 delete mode 100644 eng/common/generate-sbom-prep.sh delete mode 100644 eng/common/post-build/sourcelink-validation.ps1 delete mode 100644 eng/common/templates-official/steps/component-governance.yml delete mode 100644 eng/common/templates/steps/component-governance.yml create mode 100644 src/Directory.Build.targets delete mode 100644 src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Presentation.cs create mode 100644 src/tools/ResultsComparer.Tests/ConsoleOutputCollection.cs create mode 100644 src/tools/ResultsComparer.Tests/DataTests.cs create mode 100644 src/tools/ResultsComparer.Tests/HelperTests.cs create mode 100644 src/tools/ResultsComparer.Tests/ProgramTests.cs create mode 100644 src/tools/ResultsComparer.Tests/ResultsComparer.Tests.csproj create mode 100644 src/tools/ResultsComparer.Tests/ResultsComparerTestData.cs create mode 100644 src/tools/ResultsComparer.Tests/StatsTests.cs create mode 100644 src/tools/ResultsComparer/Properties/AssemblyInfo.cs diff --git a/NuGet.config b/NuGet.config index 2c233f9c47f..84541553c4b 100644 --- a/NuGet.config +++ b/NuGet.config @@ -7,13 +7,10 @@ - - + - - diff --git a/docs/benchmarkdotnet.md b/docs/benchmarkdotnet.md index f8c9e222849..be429f57ad7 100644 --- a/docs/benchmarkdotnet.md +++ b/docs/benchmarkdotnet.md @@ -289,7 +289,7 @@ M00_L00: The `--runtimes` or just `-r` allows you to run the benchmarks for **multiple Runtimes**. -Available options are: Mono, wasmnet70, CoreRT, net462, net47, net471, net472, netcoreapp3.1, net6.0, net7.0, net8.0, and net9.0. +Available options are: Mono, wasmnet70, CoreRT, netcoreapp3.1, net6.0, net7.0, net8.0, and net9.0. Example: run the benchmarks for .NET 7.0 and 8.0: @@ -361,18 +361,6 @@ dotnet run -c Release -f net9.0 --cli "C:\Projects\performance\.dotnet\dotnet.ex This is very useful when you want to compare different builds of .NET. -### Private CLR Build - -It's possible to benchmark a private build of .NET Runtime. You just need to pass the value of `COMPLUS_Version` to BenchmarkDotNet. You can do that by either using `--clrVersion $theVersion` as an argument or `Job.ShortRun.With(new ClrRuntime(version: "$theVersion"))` in the code. - -So if you made a change in CLR and want to measure the difference, you can run the benchmarks with: - -```cmd -dotnet run -c Release -f net48 -- --clrVersion $theVersion -``` - -More info can be found in [BenchmarkDotNet issue #706](https://github.com/dotnet/BenchmarkDotNet/issues/706). - ### Private CoreRT Build To run benchmarks with private CoreRT build you need to provide the `IlcPath`. Example: diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 6edc23c1cef..c345e47db95 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -1,39 +1,39 @@ - + https://github.com/dotnet/dotnet - 5bed4499b04cbbaec57ac4209ae993acca3648cc + 547c640d5626b2976499cb3433abc741a63d67c5 - + https://github.com/dotnet/dotnet - 5bed4499b04cbbaec57ac4209ae993acca3648cc + 547c640d5626b2976499cb3433abc741a63d67c5 - + https://github.com/dotnet/dotnet - 5bed4499b04cbbaec57ac4209ae993acca3648cc + 547c640d5626b2976499cb3433abc741a63d67c5 - + https://github.com/dotnet/dotnet - 5bed4499b04cbbaec57ac4209ae993acca3648cc + 547c640d5626b2976499cb3433abc741a63d67c5 - + https://github.com/dotnet/dotnet - 5bed4499b04cbbaec57ac4209ae993acca3648cc + 547c640d5626b2976499cb3433abc741a63d67c5 - + https://github.com/dotnet/dotnet - 5bed4499b04cbbaec57ac4209ae993acca3648cc + 547c640d5626b2976499cb3433abc741a63d67c5 - + https://github.com/dotnet/dotnet - 5bed4499b04cbbaec57ac4209ae993acca3648cc + 547c640d5626b2976499cb3433abc741a63d67c5 - + https://github.com/dotnet/dotnet - 5bed4499b04cbbaec57ac4209ae993acca3648cc + 547c640d5626b2976499cb3433abc741a63d67c5 https://github.com/dotnet/xharness @@ -55,17 +55,17 @@ 6e563dcf3cbf4853316eb4724e49ec92caeabb07 https://github.com/dotnet/maui - - 5bed4499b04cbbaec57ac4209ae993acca3648cc + + 547c640d5626b2976499cb3433abc741a63d67c5 https://github.com/dotnet/dotnet - - 5bed4499b04cbbaec57ac4209ae993acca3648cc + + 547c640d5626b2976499cb3433abc741a63d67c5 https://github.com/dotnet/dotnet - + https://github.com/dotnet/android - 9f89a27a70ce6828cffc2e66c967dd047b98656a + 6b255ec42cef9f9f757400f6edec7af239cacd58 4c9d1b112c16716c2479e054e9ad4db8b5b8c70c diff --git a/eng/Versions.props b/eng/Versions.props index c8cece31b71..07ade91729c 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -7,12 +7,12 @@ - 11.0.0-preview.4.26210.111 - 11.0.0-preview.4.26210.111 - 11.0.0-preview.4.26210.111 - 11.0.0-preview.4.26210.111 - 0.16.0-nightly.20260320.467 - 11.0.0-preview.4.26210.111 + 11.0.0-preview.5.26261.101 + 11.0.0-preview.5.26261.101 + 11.0.0-preview.5.26261.101 + 11.0.0-preview.5.26261.101 + 0.16.0-nightly.20260518.1249 + 11.0.0-preview.5.26261.101 11.0.0-prerelease.26204.1 diff --git a/eng/common/AGENTS.md b/eng/common/AGENTS.md new file mode 100644 index 00000000000..a5ed8f72926 --- /dev/null +++ b/eng/common/AGENTS.md @@ -0,0 +1,5 @@ +# `eng/common` + +Files under `eng/common` come from [Arcade](https://github.com/dotnet/arcade). +Edits in `eng/common` will be overwritten by automation unless the changes are made directly in the Arcade repository. +For more information, see the [Arcade documentation](https://github.com/dotnet/arcade/tree/main/Documentation). diff --git a/eng/common/core-templates/job/job.yml b/eng/common/core-templates/job/job.yml index 748c4f07a64..66c7988f222 100644 --- a/eng/common/core-templates/job/job.yml +++ b/eng/common/core-templates/job/job.yml @@ -26,12 +26,12 @@ parameters: enablePublishBuildArtifacts: false enablePublishBuildAssets: false enablePublishTestResults: false + enablePublishing: false enableBuildRetry: false mergeTestResults: false testRunTitle: '' testResultsFormat: '' name: '' - componentGovernanceSteps: [] preSteps: [] artifactPublishSteps: [] runAsPublic: false @@ -152,9 +152,6 @@ jobs: - ${{ each step in parameters.steps }}: - ${{ step }} - - ${{ each step in parameters.componentGovernanceSteps }}: - - ${{ step }} - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - template: /eng/common/core-templates/steps/cleanup-microbuild.yml parameters: diff --git a/eng/common/core-templates/job/onelocbuild.yml b/eng/common/core-templates/job/onelocbuild.yml index eefed3b667a..86ea9f63504 100644 --- a/eng/common/core-templates/job/onelocbuild.yml +++ b/eng/common/core-templates/job/onelocbuild.yml @@ -22,6 +22,7 @@ parameters: GitHubOrg: dotnet MirrorRepo: '' MirrorBranch: main + xLocCustomPowerShellScript: '' condition: '' JobNameSuffix: '' is1ESPipeline: '' @@ -97,6 +98,8 @@ jobs: gitHubOrganization: ${{ parameters.GitHubOrg }} mirrorRepo: ${{ parameters.MirrorRepo }} mirrorBranch: ${{ parameters.MirrorBranch }} + ${{ if ne(parameters.xLocCustomPowerShellScript, '') }}: + xLocCustomPowerShellScript: ${{ parameters.xLocCustomPowerShellScript }} condition: ${{ parameters.condition }} # Copy the locProject.json to the root of the Loc directory, then publish a pipeline artifact diff --git a/eng/common/core-templates/job/publish-build-assets.yml b/eng/common/core-templates/job/publish-build-assets.yml index 9d7490518c4..700f7711465 100644 --- a/eng/common/core-templates/job/publish-build-assets.yml +++ b/eng/common/core-templates/job/publish-build-assets.yml @@ -172,17 +172,18 @@ jobs: targetPath: '$(Build.ArtifactStagingDirectory)/MergedManifest.xml' artifactName: AssetManifests displayName: 'Publish Merged Manifest' - retryCountOnTaskFailure: 10 # for any logs being locked - sbomEnabled: false # we don't need SBOM for logs + retryCountOnTaskFailure: 10 # for any files being locked + isProduction: false # just metadata for publishing - - template: /eng/common/core-templates/steps/publish-build-artifacts.yml + - template: /eng/common/core-templates/steps/publish-pipeline-artifacts.yml parameters: is1ESPipeline: ${{ parameters.is1ESPipeline }} args: displayName: Publish ReleaseConfigs Artifact - pathToPublish: '$(Build.StagingDirectory)/ReleaseConfigs' - publishLocation: Container + targetPath: '$(Build.StagingDirectory)/ReleaseConfigs' artifactName: ReleaseConfigs + retryCountOnTaskFailure: 10 # for any files being locked + isProduction: false # just metadata for publishing - ${{ if or(eq(parameters.publishAssetsImmediately, 'true'), eq(parameters.isAssetlessBuild, 'true')) }}: - template: /eng/common/core-templates/post-build/setup-maestro-vars.yml @@ -218,4 +219,5 @@ jobs: - template: /eng/common/core-templates/steps/publish-logs.yml parameters: is1ESPipeline: ${{ parameters.is1ESPipeline }} + StageLabel: 'BuildAssetRegistry' JobLabel: 'Publish_Artifacts_Logs' diff --git a/eng/common/core-templates/job/renovate.yml b/eng/common/core-templates/job/renovate.yml index ab233539b5d..ff86c80b468 100644 --- a/eng/common/core-templates/job/renovate.yml +++ b/eng/common/core-templates/job/renovate.yml @@ -135,7 +135,7 @@ jobs: condition: succeededOrFailed() targetPath: $(Build.ArtifactStagingDirectory) artifactName: $(Agent.JobName)_Logs_Attempt$(System.JobAttempt) - sbomEnabled: false + isProduction: false # logs are non-production artifacts steps: - checkout: self diff --git a/eng/common/core-templates/jobs/jobs.yml b/eng/common/core-templates/jobs/jobs.yml index 01ada747665..cc8cce45278 100644 --- a/eng/common/core-templates/jobs/jobs.yml +++ b/eng/common/core-templates/jobs/jobs.yml @@ -43,6 +43,10 @@ parameters: artifacts: {} is1ESPipeline: '' + + # Publishing version w/default. + publishingVersion: 3 + repositoryAlias: self officialBuildId: '' @@ -102,6 +106,7 @@ jobs: parameters: is1ESPipeline: ${{ parameters.is1ESPipeline }} continueOnError: ${{ parameters.continueOnError }} + publishingVersion: ${{ parameters.publishingVersion }} dependsOn: - ${{ if ne(parameters.publishBuildAssetsDependsOn, '') }}: - ${{ each job in parameters.publishBuildAssetsDependsOn }}: diff --git a/eng/common/core-templates/post-build/common-variables.yml b/eng/common/core-templates/post-build/common-variables.yml index d5627a994ae..db298ae16ba 100644 --- a/eng/common/core-templates/post-build/common-variables.yml +++ b/eng/common/core-templates/post-build/common-variables.yml @@ -11,8 +11,6 @@ variables: - name: MaestroApiVersion value: "2020-02-20" - - name: SourceLinkCLIVersion - value: 3.0.0 - name: SymbolToolVersion value: 1.0.1 - name: BinlogToolVersion diff --git a/eng/common/core-templates/post-build/post-build.yml b/eng/common/core-templates/post-build/post-build.yml index 0994189969f..8aa86e30491 100644 --- a/eng/common/core-templates/post-build/post-build.yml +++ b/eng/common/core-templates/post-build/post-build.yml @@ -9,6 +9,7 @@ parameters: default: 3 values: - 3 + - 4 - name: BARBuildId displayName: BAR Build Id @@ -130,16 +131,30 @@ stages: PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} is1ESPipeline: ${{ parameters.is1ESPipeline }} - - task: DownloadBuildArtifacts@0 - displayName: Download Package Artifacts - inputs: - buildType: specific - buildVersionToDownload: specific - project: $(AzDOProjectName) - pipeline: $(AzDOPipelineId) - buildId: $(AzDOBuildId) - artifactName: PackageArtifacts - checkDownloadedFiles: true + - ${{ if ne(parameters.publishingInfraVersion, 4) }}: + - task: DownloadBuildArtifacts@0 + displayName: Download Package Artifacts + inputs: + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) + artifactName: PackageArtifacts + checkDownloadedFiles: true + - ${{ if eq(parameters.publishingInfraVersion, 4) }}: + - task: DownloadPipelineArtifact@2 + displayName: Download Pipeline Artifacts (V4) + inputs: + itemPattern: '*/packages/**/*.nupkg' + targetPath: '$(Build.ArtifactStagingDirectory)/PipelineArtifactsDownload' + - task: CopyFiles@2 + displayName: Flatten packages to PackageArtifacts + inputs: + SourceFolder: '$(Build.ArtifactStagingDirectory)/PipelineArtifactsDownload' + Contents: '**/*.nupkg' + TargetFolder: '$(Build.ArtifactStagingDirectory)/PackageArtifacts' + flattenFolders: true - task: PowerShell@2 displayName: Validate @@ -173,16 +188,30 @@ stages: PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} is1ESPipeline: ${{ parameters.is1ESPipeline }} - - task: DownloadBuildArtifacts@0 - displayName: Download Package Artifacts - inputs: - buildType: specific - buildVersionToDownload: specific - project: $(AzDOProjectName) - pipeline: $(AzDOPipelineId) - buildId: $(AzDOBuildId) - artifactName: PackageArtifacts - checkDownloadedFiles: true + - ${{ if ne(parameters.publishingInfraVersion, 4) }}: + - task: DownloadBuildArtifacts@0 + displayName: Download Package Artifacts + inputs: + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) + artifactName: PackageArtifacts + checkDownloadedFiles: true + - ${{ if eq(parameters.publishingInfraVersion, 4) }}: + - task: DownloadPipelineArtifact@2 + displayName: Download Pipeline Artifacts (V4) + inputs: + itemPattern: '*/packages/**/*.nupkg' + targetPath: '$(Build.ArtifactStagingDirectory)/PipelineArtifactsDownload' + - task: CopyFiles@2 + displayName: Flatten packages to PackageArtifacts + inputs: + SourceFolder: '$(Build.ArtifactStagingDirectory)/PipelineArtifactsDownload' + Contents: '**/*.nupkg' + TargetFolder: '$(Build.ArtifactStagingDirectory)/PackageArtifacts' + flattenFolders: true # This is necessary whenever we want to publish/restore to an AzDO private feed # Since sdk-task.ps1 tries to restore packages we need to do this authentication here @@ -196,7 +225,7 @@ stages: displayName: Validate inputs: filePath: eng\common\sdk-task.ps1 - arguments: -task SigningValidation -restore + arguments: -task SigningValidation -restore -msbuildEngine dotnet /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts' /p:SignCheckExclusionsFile='$(System.DefaultWorkingDirectory)/eng/SignCheckExclusionsFile.txt' ${{ parameters.signingValidationAdditionalParameters }} @@ -208,53 +237,20 @@ stages: JobLabel: 'Signing' BinlogToolVersion: $(BinlogToolVersion) - - job: - displayName: SourceLink Validation - condition: eq( ${{ parameters.enableSourceLinkValidation }}, 'true') - pool: - # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) - ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: - name: AzurePipelines-EO - image: 1ESPT-Windows2025 - demands: Cmd - os: windows - # If it's not devdiv, it's dnceng - ${{ else }}: - ${{ if eq(parameters.is1ESPipeline, true) }}: - name: $(DncEngInternalBuildPool) - image: windows.vs2026.amd64 - os: windows - ${{ else }}: - name: $(DncEngInternalBuildPool) - demands: ImageOverride -equals windows.vs2026.amd64 - steps: - - template: /eng/common/core-templates/post-build/setup-maestro-vars.yml - parameters: - BARBuildId: ${{ parameters.BARBuildId }} - PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} - is1ESPipeline: ${{ parameters.is1ESPipeline }} - - - task: DownloadBuildArtifacts@0 - displayName: Download Blob Artifacts - inputs: - buildType: specific - buildVersionToDownload: specific - project: $(AzDOProjectName) - pipeline: $(AzDOPipelineId) - buildId: $(AzDOBuildId) - artifactName: BlobArtifacts - checkDownloadedFiles: true - - - task: PowerShell@2 - displayName: Validate - inputs: - filePath: $(System.DefaultWorkingDirectory)/eng/common/post-build/sourcelink-validation.ps1 - arguments: -InputPath $(Build.ArtifactStagingDirectory)/BlobArtifacts/ - -ExtractPath $(Agent.BuildDirectory)/Extract/ - -GHRepoName $(Build.Repository.Name) - -GHCommit $(Build.SourceVersion) - -SourcelinkCliVersion $(SourceLinkCLIVersion) - continueOnError: true + # SourceLink validation has been removed — the underlying CLI tool + # (targeting netcoreapp2.1) has not functioned for years. + # The enableSourceLinkValidation parameter is kept but ignored so + # existing pipelines that pass it are not broken. + # See https://github.com/dotnet/arcade/issues/16647 + - ${{ if eq(parameters.enableSourceLinkValidation, 'true') }}: + - job: + displayName: 'SourceLink Validation Removed - please remove enableSourceLinkValidation from your pipeline' + pool: server + steps: + - task: Delay@1 + displayName: 'Warning: SourceLink validation removed (see https://github.com/dotnet/arcade/issues/16647)' + inputs: + delayForMinutes: '0' - ${{ if ne(parameters.publishAssetsImmediately, 'true') }}: - stage: publish_using_darc @@ -317,7 +313,7 @@ stages: scriptPath: $(System.DefaultWorkingDirectory)/eng/common/post-build/publish-using-darc.ps1 arguments: > -BuildId $(BARBuildId) - -PublishingInfraVersion ${{ parameters.publishingInfraVersion }} + -PublishingInfraVersion 3 -AzdoToken '$(System.AccessToken)' -WaitPublishingFinish true -RequireDefaultChannels ${{ parameters.requireDefaultChannels }} diff --git a/eng/common/core-templates/post-build/setup-maestro-vars.yml b/eng/common/core-templates/post-build/setup-maestro-vars.yml index a7abd58c4bb..6dfa99ec5e3 100644 --- a/eng/common/core-templates/post-build/setup-maestro-vars.yml +++ b/eng/common/core-templates/post-build/setup-maestro-vars.yml @@ -8,12 +8,11 @@ steps: - 'Illegal entry point, is1ESPipeline is not defined. Repository yaml should not directly reference templates in core-templates folder.': error - ${{ if eq(coalesce(parameters.PromoteToChannelIds, 0), 0) }}: - - task: DownloadBuildArtifacts@0 + - task: DownloadPipelineArtifact@2 displayName: Download Release Configs inputs: - buildType: current artifactName: ReleaseConfigs - checkDownloadedFiles: true + targetPath: '$(Build.StagingDirectory)/ReleaseConfigs' - task: AzureCLI@2 name: setReleaseVars diff --git a/eng/common/core-templates/steps/component-governance.yml b/eng/common/core-templates/steps/component-governance.yml deleted file mode 100644 index cf0649aa956..00000000000 --- a/eng/common/core-templates/steps/component-governance.yml +++ /dev/null @@ -1,16 +0,0 @@ -parameters: - disableComponentGovernance: false - componentGovernanceIgnoreDirectories: '' - is1ESPipeline: false - displayName: 'Component Detection' - -steps: -- ${{ if eq(parameters.disableComponentGovernance, 'true') }}: - - script: echo "##vso[task.setvariable variable=skipComponentGovernanceDetection]true" - displayName: Set skipComponentGovernanceDetection variable -- ${{ if ne(parameters.disableComponentGovernance, 'true') }}: - - task: ComponentGovernanceComponentDetection@0 - continueOnError: true - displayName: ${{ parameters.displayName }} - inputs: - ignoreDirectories: ${{ parameters.componentGovernanceIgnoreDirectories }} diff --git a/eng/common/core-templates/steps/generate-sbom.yml b/eng/common/core-templates/steps/generate-sbom.yml index 003f7eae0fa..aad0a8aeda3 100644 --- a/eng/common/core-templates/steps/generate-sbom.yml +++ b/eng/common/core-templates/steps/generate-sbom.yml @@ -1,54 +1,14 @@ -# BuildDropPath - The root folder of the drop directory for which the manifest file will be generated. -# PackageName - The name of the package this SBOM represents. -# PackageVersion - The version of the package this SBOM represents. -# ManifestDirPath - The path of the directory where the generated manifest files will be placed -# IgnoreDirectories - Directories to ignore for SBOM generation. This will be passed through to the CG component detector. - parameters: - PackageVersion: 11.0.0 - BuildDropPath: '$(System.DefaultWorkingDirectory)/artifacts' - PackageName: '.NET' - ManifestDirPath: $(Build.ArtifactStagingDirectory)/sbom - IgnoreDirectories: '' - sbomContinueOnError: true - is1ESPipeline: false - # disable publishArtifacts if some other step is publishing the artifacts (like job.yml). - publishArtifacts: true + PackageVersion: unused + BuildDropPath: unused + PackageName: unused + ManifestDirPath: unused + IgnoreDirectories: unused + sbomContinueOnError: unused + is1ESPipeline: unused + publishArtifacts: unused steps: -- task: PowerShell@2 - displayName: Prep for SBOM generation in (Non-linux) - condition: or(eq(variables['Agent.Os'], 'Windows_NT'), eq(variables['Agent.Os'], 'Darwin')) - inputs: - filePath: ./eng/common/generate-sbom-prep.ps1 - arguments: ${{parameters.manifestDirPath}} - -# Chmodding is a workaround for https://github.com/dotnet/arcade/issues/8461 - script: | - chmod +x ./eng/common/generate-sbom-prep.sh - ./eng/common/generate-sbom-prep.sh ${{parameters.manifestDirPath}} - displayName: Prep for SBOM generation in (Linux) - condition: eq(variables['Agent.Os'], 'Linux') - continueOnError: ${{ parameters.sbomContinueOnError }} - -- task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 - displayName: 'Generate SBOM manifest' - continueOnError: ${{ parameters.sbomContinueOnError }} - inputs: - PackageName: ${{ parameters.packageName }} - BuildDropPath: ${{ parameters.buildDropPath }} - PackageVersion: ${{ parameters.packageVersion }} - ManifestDirPath: ${{ parameters.manifestDirPath }}/$(ARTIFACT_NAME) - ${{ if ne(parameters.IgnoreDirectories, '') }}: - AdditionalComponentDetectorArgs: '--IgnoreDirectories ${{ parameters.IgnoreDirectories }}' - -- ${{ if eq(parameters.publishArtifacts, 'true')}}: - - template: /eng/common/core-templates/steps/publish-pipeline-artifacts.yml - parameters: - is1ESPipeline: ${{ parameters.is1ESPipeline }} - args: - displayName: Publish SBOM manifest - continueOnError: ${{parameters.sbomContinueOnError}} - targetPath: '${{ parameters.manifestDirPath }}' - artifactName: $(ARTIFACT_NAME) - + echo "##vso[task.logissue type=warning]Including generate-sbom.yml is deprecated, SBOM generation is handled 1ES PT now. Remove this include." + displayName: Issue generate-sbom.yml deprecation warning diff --git a/eng/common/core-templates/steps/publish-logs.yml b/eng/common/core-templates/steps/publish-logs.yml index a9ea99ba6aa..84a1922c73f 100644 --- a/eng/common/core-templates/steps/publish-logs.yml +++ b/eng/common/core-templates/steps/publish-logs.yml @@ -50,13 +50,15 @@ steps: TargetFolder: '$(Build.ArtifactStagingDirectory)/PostBuildLogs' condition: always() -- template: /eng/common/core-templates/steps/publish-build-artifacts.yml +- template: /eng/common/core-templates/steps/publish-pipeline-artifacts.yml parameters: is1ESPipeline: ${{ parameters.is1ESPipeline }} args: displayName: Publish Logs - pathToPublish: '$(Build.ArtifactStagingDirectory)/PostBuildLogs' - publishLocation: Container - artifactName: PostBuildLogs + targetPath: '$(Build.ArtifactStagingDirectory)/PostBuildLogs' + artifactName: PostBuildLogs_${{ parameters.StageLabel }}_${{ parameters.JobLabel }}_Attempt$(System.JobAttempt) continueOnError: true condition: always() + retryCountOnTaskFailure: 10 # for any files being locked + isProduction: false # logs are non-production artifacts + diff --git a/eng/common/core-templates/steps/source-build.yml b/eng/common/core-templates/steps/source-build.yml index acf16ed3496..b75f59c428d 100644 --- a/eng/common/core-templates/steps/source-build.yml +++ b/eng/common/core-templates/steps/source-build.yml @@ -62,4 +62,4 @@ steps: artifactName: BuildLogs_SourceBuild_${{ parameters.platform.name }}_Attempt$(System.JobAttempt) continueOnError: true condition: succeededOrFailed() - sbomEnabled: false # we don't need SBOM for logs + isProduction: false # logs are non-production artifacts diff --git a/eng/common/cross/toolchain.cmake b/eng/common/cross/toolchain.cmake index ff2dfdb4a5b..99d6dfe82dd 100644 --- a/eng/common/cross/toolchain.cmake +++ b/eng/common/cross/toolchain.cmake @@ -225,13 +225,19 @@ elseif(ILLUMOS) locate_toolchain_exec(g++ CMAKE_CXX_COMPILER) elseif(HAIKU) set(CMAKE_SYSROOT "${CROSS_ROOTFS}") - set(CMAKE_PROGRAM_PATH "${CMAKE_PROGRAM_PATH};${CROSS_ROOTFS}/cross-tools-x86_64/bin") set(CMAKE_SYSTEM_PREFIX_PATH "${CROSS_ROOTFS}") set(CMAKE_C_STANDARD_LIBRARIES "${CMAKE_C_STANDARD_LIBRARIES} -lssp") set(CMAKE_CXX_STANDARD_LIBRARIES "${CMAKE_CXX_STANDARD_LIBRARIES} -lssp") - locate_toolchain_exec(gcc CMAKE_C_COMPILER) - locate_toolchain_exec(g++ CMAKE_CXX_COMPILER) + if ($ENV{CCC_CC} MATCHES ".*gcc.*") + set(CMAKE_PROGRAM_PATH "${CMAKE_PROGRAM_PATH};${CROSS_ROOTFS}/cross-tools-x86_64/bin") + locate_toolchain_exec(gcc CMAKE_C_COMPILER) + locate_toolchain_exec(g++ CMAKE_CXX_COMPILER) + else() + set(CMAKE_C_COMPILER_EXTERNAL_TOOLCHAIN "${CROSS_ROOTFS}/cross-tools-x86_64") + set(CMAKE_CXX_COMPILER_EXTERNAL_TOOLCHAIN "${CROSS_ROOTFS}/cross-tools-x86_64") + set(CMAKE_ASM_COMPILER_EXTERNAL_TOOLCHAIN "${CROSS_ROOTFS}/cross-tools-x86_64") + endif() # let CMake set up the correct search paths include(Platform/Haiku) diff --git a/eng/common/darc-init.ps1 b/eng/common/darc-init.ps1 index e3374310563..a5be41db690 100644 --- a/eng/common/darc-init.ps1 +++ b/eng/common/darc-init.ps1 @@ -29,11 +29,11 @@ function InstallDarcCli ($darcVersion, $toolpath) { Write-Host "Installing Darc CLI version $darcVersion..." Write-Host 'You may need to restart your command window if this is the first dotnet tool you have installed.' if (-not $toolpath) { - Write-Host "'$dotnet' tool install $darcCliPackageName --version $darcVersion --add-source '$arcadeServicesSource' -v $verbosity -g" - & "$dotnet" tool install $darcCliPackageName --version $darcVersion --add-source "$arcadeServicesSource" -v $verbosity -g + Write-Host "'$dotnet' tool install $darcCliPackageName --version $darcVersion --source '$arcadeServicesSource' -v $verbosity -g" + & "$dotnet" tool install $darcCliPackageName --version $darcVersion --source "$arcadeServicesSource" -v $verbosity -g }else { - Write-Host "'$dotnet' tool install $darcCliPackageName --version $darcVersion --add-source '$arcadeServicesSource' -v $verbosity --tool-path '$toolpath'" - & "$dotnet" tool install $darcCliPackageName --version $darcVersion --add-source "$arcadeServicesSource" -v $verbosity --tool-path "$toolpath" + Write-Host "'$dotnet' tool install $darcCliPackageName --version $darcVersion --source '$arcadeServicesSource' -v $verbosity --tool-path '$toolpath'" + & "$dotnet" tool install $darcCliPackageName --version $darcVersion --source "$arcadeServicesSource" -v $verbosity --tool-path "$toolpath" } } diff --git a/eng/common/darc-init.sh b/eng/common/darc-init.sh index 9f5ad6b763b..b56d40e5706 100755 --- a/eng/common/darc-init.sh +++ b/eng/common/darc-init.sh @@ -73,9 +73,9 @@ function InstallDarcCli { echo "Installing Darc CLI version $darcVersion..." echo "You may need to restart your command shell if this is the first dotnet tool you have installed." if [ -z "$toolpath" ]; then - echo $($dotnet_root/dotnet tool install $darc_cli_package_name --version $darcVersion --add-source "$arcadeServicesSource" -v $verbosity -g) + echo $($dotnet_root/dotnet tool install $darc_cli_package_name --version $darcVersion --source "$arcadeServicesSource" -v $verbosity -g) else - echo $($dotnet_root/dotnet tool install $darc_cli_package_name --version $darcVersion --add-source "$arcadeServicesSource" -v $verbosity --tool-path "$toolpath") + echo $($dotnet_root/dotnet tool install $darc_cli_package_name --version $darcVersion --source "$arcadeServicesSource" -v $verbosity --tool-path "$toolpath") fi } diff --git a/eng/common/dotnet-install.ps1 b/eng/common/dotnet-install.ps1 index 811f0f717f7..50ae6273768 100644 --- a/eng/common/dotnet-install.ps1 +++ b/eng/common/dotnet-install.ps1 @@ -10,7 +10,11 @@ Param( . $PSScriptRoot\tools.ps1 -$dotnetRoot = Join-Path $RepoRoot '.dotnet' +if (-not [string]::IsNullOrEmpty($env:DOTNET_GLOBAL_INSTALL_DIR)) { + $dotnetRoot = $env:DOTNET_GLOBAL_INSTALL_DIR +} else { + $dotnetRoot = Join-Path $RepoRoot '.dotnet' +} $installdir = $dotnetRoot try { diff --git a/eng/common/dotnet-install.sh b/eng/common/dotnet-install.sh index 61f302bb677..1cb3f5abac2 100755 --- a/eng/common/dotnet-install.sh +++ b/eng/common/dotnet-install.sh @@ -80,7 +80,11 @@ case $cpuname in ;; esac -dotnetRoot="${repo_root}.dotnet" +if [[ -n "${DOTNET_GLOBAL_INSTALL_DIR:-}" ]]; then + dotnetRoot="$DOTNET_GLOBAL_INSTALL_DIR" +else + dotnetRoot="${repo_root}.dotnet" +fi if [[ $architecture != "" ]] && [[ $architecture != $buildarch ]]; then dotnetRoot="$dotnetRoot/$architecture" fi diff --git a/eng/common/generate-sbom-prep.ps1 b/eng/common/generate-sbom-prep.ps1 deleted file mode 100644 index a0c7d792a76..00000000000 --- a/eng/common/generate-sbom-prep.ps1 +++ /dev/null @@ -1,29 +0,0 @@ -Param( - [Parameter(Mandatory=$true)][string] $ManifestDirPath # Manifest directory where sbom will be placed -) - -. $PSScriptRoot\pipeline-logging-functions.ps1 - -# Normally - we'd listen to the manifest path given, but 1ES templates will overwrite if this level gets uploaded directly -# with their own overwriting ours. So we create it as a sub directory of the requested manifest path. -$ArtifactName = "${env:SYSTEM_STAGENAME}_${env:AGENT_JOBNAME}_SBOM" -$SafeArtifactName = $ArtifactName -replace '["/:<>\\|?@*"() ]', '_' -$SbomGenerationDir = Join-Path $ManifestDirPath $SafeArtifactName - -Write-Host "Artifact name before : $ArtifactName" -Write-Host "Artifact name after : $SafeArtifactName" - -Write-Host "Creating dir $ManifestDirPath" - -# create directory for sbom manifest to be placed -if (!(Test-Path -path $SbomGenerationDir)) -{ - New-Item -ItemType Directory -path $SbomGenerationDir - Write-Host "Successfully created directory $SbomGenerationDir" -} -else{ - Write-PipelineTelemetryError -category 'Build' "Unable to create sbom folder." -} - -Write-Host "Updating artifact name" -Write-Host "##vso[task.setvariable variable=ARTIFACT_NAME]$SafeArtifactName" diff --git a/eng/common/generate-sbom-prep.sh b/eng/common/generate-sbom-prep.sh deleted file mode 100644 index b8ecca72bbf..00000000000 --- a/eng/common/generate-sbom-prep.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env bash - -source="${BASH_SOURCE[0]}" - -# resolve $SOURCE until the file is no longer a symlink -while [[ -h $source ]]; do - scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" - source="$(readlink "$source")" - - # if $source was a relative symlink, we need to resolve it relative to the path where the - # symlink file was located - [[ $source != /* ]] && source="$scriptroot/$source" -done -scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" -. $scriptroot/pipeline-logging-functions.sh - - -# replace all special characters with _, some builds use special characters like : in Agent.Jobname, that is not a permissible name while uploading artifacts. -artifact_name=$SYSTEM_STAGENAME"_"$AGENT_JOBNAME"_SBOM" -safe_artifact_name="${artifact_name//["/:<>\\|?@*$" ]/_}" -manifest_dir=$1 - -# Normally - we'd listen to the manifest path given, but 1ES templates will overwrite if this level gets uploaded directly -# with their own overwriting ours. So we create it as a sub directory of the requested manifest path. -sbom_generation_dir="$manifest_dir/$safe_artifact_name" - -if [ ! -d "$sbom_generation_dir" ] ; then - mkdir -p "$sbom_generation_dir" - echo "Sbom directory created." $sbom_generation_dir -else - Write-PipelineTelemetryError -category 'Build' "Unable to create sbom folder." -fi - -echo "Artifact name before : "$artifact_name -echo "Artifact name after : "$safe_artifact_name -export ARTIFACT_NAME=$safe_artifact_name -echo "##vso[task.setvariable variable=ARTIFACT_NAME]$safe_artifact_name" - -exit 0 diff --git a/eng/common/post-build/redact-logs.ps1 b/eng/common/post-build/redact-logs.ps1 index fc0218a013d..672f4e2652e 100644 --- a/eng/common/post-build/redact-logs.ps1 +++ b/eng/common/post-build/redact-logs.ps1 @@ -49,8 +49,8 @@ try { Write-Host "Installing Binlog redactor CLI..." Write-Host "'$dotnet' new tool-manifest" & "$dotnet" new tool-manifest - Write-Host "'$dotnet' tool install $packageName --local --add-source '$PackageFeed' -v $verbosity --version $BinlogToolVersion" - & "$dotnet" tool install $packageName --local --add-source "$PackageFeed" -v $verbosity --version $BinlogToolVersion + Write-Host "'$dotnet' tool install $packageName --local --source '$PackageFeed' -v $verbosity --version $BinlogToolVersion" + & "$dotnet" tool install $packageName --local --source "$PackageFeed" -v $verbosity --version $BinlogToolVersion if (Test-Path $TokensFilePath) { Write-Host "Adding additional sensitive data for redaction from file: " $TokensFilePath diff --git a/eng/common/post-build/sourcelink-validation.ps1 b/eng/common/post-build/sourcelink-validation.ps1 deleted file mode 100644 index 1976ef70fb8..00000000000 --- a/eng/common/post-build/sourcelink-validation.ps1 +++ /dev/null @@ -1,327 +0,0 @@ -param( - [Parameter(Mandatory=$true)][string] $InputPath, # Full path to directory where Symbols.NuGet packages to be checked are stored - [Parameter(Mandatory=$true)][string] $ExtractPath, # Full path to directory where the packages will be extracted during validation - [Parameter(Mandatory=$false)][string] $GHRepoName, # GitHub name of the repo including the Org. E.g., dotnet/arcade - [Parameter(Mandatory=$false)][string] $GHCommit, # GitHub commit SHA used to build the packages - [Parameter(Mandatory=$true)][string] $SourcelinkCliVersion # Version of SourceLink CLI to use -) - -$ErrorActionPreference = 'Stop' -Set-StrictMode -Version 2.0 - -# `tools.ps1` checks $ci to perform some actions. Since the post-build -# scripts don't necessarily execute in the same agent that run the -# build.ps1/sh script this variable isn't automatically set. -$ci = $true -$disableConfigureToolsetImport = $true -. $PSScriptRoot\..\tools.ps1 - -# Cache/HashMap (File -> Exist flag) used to consult whether a file exist -# in the repository at a specific commit point. This is populated by inserting -# all files present in the repo at a specific commit point. -$global:RepoFiles = @{} - -# Maximum number of jobs to run in parallel -$MaxParallelJobs = 16 - -$MaxRetries = 5 -$RetryWaitTimeInSeconds = 30 - -# Wait time between check for system load -$SecondsBetweenLoadChecks = 10 - -if (!$InputPath -or !(Test-Path $InputPath)){ - Write-Host "No files to validate." - ExitWithExitCode 0 -} - -$ValidatePackage = { - param( - [string] $PackagePath # Full path to a Symbols.NuGet package - ) - - . $using:PSScriptRoot\..\tools.ps1 - - # Ensure input file exist - if (!(Test-Path $PackagePath)) { - Write-Host "Input file does not exist: $PackagePath" - return [pscustomobject]@{ - result = 1 - packagePath = $PackagePath - } - } - - # Extensions for which we'll look for SourceLink information - # For now we'll only care about Portable & Embedded PDBs - $RelevantExtensions = @('.dll', '.exe', '.pdb') - - Write-Host -NoNewLine 'Validating ' ([System.IO.Path]::GetFileName($PackagePath)) '...' - - $PackageId = [System.IO.Path]::GetFileNameWithoutExtension($PackagePath) - $ExtractPath = Join-Path -Path $using:ExtractPath -ChildPath $PackageId - $FailedFiles = 0 - - Add-Type -AssemblyName System.IO.Compression.FileSystem - - [System.IO.Directory]::CreateDirectory($ExtractPath) | Out-Null - - try { - $zip = [System.IO.Compression.ZipFile]::OpenRead($PackagePath) - - $zip.Entries | - Where-Object {$RelevantExtensions -contains [System.IO.Path]::GetExtension($_.Name)} | - ForEach-Object { - $FileName = $_.FullName - $Extension = [System.IO.Path]::GetExtension($_.Name) - $FakeName = -Join((New-Guid), $Extension) - $TargetFile = Join-Path -Path $ExtractPath -ChildPath $FakeName - - # We ignore resource DLLs - if ($FileName.EndsWith('.resources.dll')) { - return [pscustomobject]@{ - result = 0 - packagePath = $PackagePath - } - } - - [System.IO.Compression.ZipFileExtensions]::ExtractToFile($_, $TargetFile, $true) - - $ValidateFile = { - param( - [string] $FullPath, # Full path to the module that has to be checked - [string] $RealPath, - [ref] $FailedFiles - ) - - $sourcelinkExe = "$env:USERPROFILE\.dotnet\tools" - $sourcelinkExe = Resolve-Path "$sourcelinkExe\sourcelink.exe" - $SourceLinkInfos = & $sourcelinkExe print-urls $FullPath | Out-String - - if ($LASTEXITCODE -eq 0 -and -not ([string]::IsNullOrEmpty($SourceLinkInfos))) { - $NumFailedLinks = 0 - - # We only care about Http addresses - $Matches = (Select-String '(http[s]?)(:\/\/)([^\s,]+)' -Input $SourceLinkInfos -AllMatches).Matches - - if ($Matches.Count -ne 0) { - $Matches.Value | - ForEach-Object { - $Link = $_ - $CommitUrl = "https://raw.githubusercontent.com/${using:GHRepoName}/${using:GHCommit}/" - - $FilePath = $Link.Replace($CommitUrl, "") - $Status = 200 - $Cache = $using:RepoFiles - - $attempts = 0 - - while ($attempts -lt $using:MaxRetries) { - if ( !($Cache.ContainsKey($FilePath)) ) { - try { - $Uri = $Link -as [System.URI] - - if ($Link -match "submodules") { - # Skip submodule links until sourcelink properly handles submodules - $Status = 200 - } - elseif ($Uri.AbsoluteURI -ne $null -and ($Uri.Host -match 'github' -or $Uri.Host -match 'githubusercontent')) { - # Only GitHub links are valid - $Status = (Invoke-WebRequest -Uri $Link -UseBasicParsing -Method HEAD -TimeoutSec 5).StatusCode - } - else { - # If it's not a github link, we want to break out of the loop and not retry. - $Status = 0 - $attempts = $using:MaxRetries - } - } - catch { - Write-Host $_ - $Status = 0 - } - } - - if ($Status -ne 200) { - $attempts++ - - if ($attempts -lt $using:MaxRetries) - { - $attemptsLeft = $using:MaxRetries - $attempts - Write-Warning "Download failed, $attemptsLeft attempts remaining, will retry in $using:RetryWaitTimeInSeconds seconds" - Start-Sleep -Seconds $using:RetryWaitTimeInSeconds - } - else { - if ($NumFailedLinks -eq 0) { - if ($FailedFiles.Value -eq 0) { - Write-Host - } - - Write-Host "`tFile $RealPath has broken links:" - } - - Write-Host "`t`tFailed to retrieve $Link" - - $NumFailedLinks++ - } - } - else { - break - } - } - } - } - - if ($NumFailedLinks -ne 0) { - $FailedFiles.value++ - $global:LASTEXITCODE = 1 - } - } - } - - &$ValidateFile $TargetFile $FileName ([ref]$FailedFiles) - } - } - catch { - Write-Host $_ - } - finally { - $zip.Dispose() - } - - if ($FailedFiles -eq 0) { - Write-Host 'Passed.' - return [pscustomobject]@{ - result = 0 - packagePath = $PackagePath - } - } - else { - Write-PipelineTelemetryError -Category 'SourceLink' -Message "$PackagePath has broken SourceLink links." - return [pscustomobject]@{ - result = 1 - packagePath = $PackagePath - } - } -} - -function CheckJobResult( - $result, - $packagePath, - [ref]$ValidationFailures, - [switch]$logErrors) { - if ($result -ne '0') { - if ($logErrors) { - Write-PipelineTelemetryError -Category 'SourceLink' -Message "$packagePath has broken SourceLink links." - } - $ValidationFailures.Value++ - } -} - -function ValidateSourceLinkLinks { - if ($GHRepoName -ne '' -and !($GHRepoName -Match '^[^\s\/]+/[^\s\/]+$')) { - if (!($GHRepoName -Match '^[^\s-]+-[^\s]+$')) { - Write-PipelineTelemetryError -Category 'SourceLink' -Message "GHRepoName should be in the format / or -. '$GHRepoName'" - ExitWithExitCode 1 - } - else { - $GHRepoName = $GHRepoName -replace '^([^\s-]+)-([^\s]+)$', '$1/$2'; - } - } - - if ($GHCommit -ne '' -and !($GHCommit -Match '^[0-9a-fA-F]{40}$')) { - Write-PipelineTelemetryError -Category 'SourceLink' -Message "GHCommit should be a 40 chars hexadecimal string. '$GHCommit'" - ExitWithExitCode 1 - } - - if ($GHRepoName -ne '' -and $GHCommit -ne '') { - $RepoTreeURL = -Join('http://api.github.com/repos/', $GHRepoName, '/git/trees/', $GHCommit, '?recursive=1') - $CodeExtensions = @('.cs', '.vb', '.fs', '.fsi', '.fsx', '.fsscript') - - try { - # Retrieve the list of files in the repo at that particular commit point and store them in the RepoFiles hash - $Data = Invoke-WebRequest $RepoTreeURL -UseBasicParsing | ConvertFrom-Json | Select-Object -ExpandProperty tree - - foreach ($file in $Data) { - $Extension = [System.IO.Path]::GetExtension($file.path) - - if ($CodeExtensions.Contains($Extension)) { - $RepoFiles[$file.path] = 1 - } - } - } - catch { - Write-Host "Problems downloading the list of files from the repo. Url used: $RepoTreeURL . Execution will proceed without caching." - } - } - elseif ($GHRepoName -ne '' -or $GHCommit -ne '') { - Write-Host 'For using the http caching mechanism both GHRepoName and GHCommit should be informed.' - } - - if (Test-Path $ExtractPath) { - Remove-Item $ExtractPath -Force -Recurse -ErrorAction SilentlyContinue - } - - $ValidationFailures = 0 - - # Process each NuGet package in parallel - Get-ChildItem "$InputPath\*.symbols.nupkg" | - ForEach-Object { - Write-Host "Starting $($_.FullName)" - Start-Job -ScriptBlock $ValidatePackage -ArgumentList $_.FullName | Out-Null - $NumJobs = @(Get-Job -State 'Running').Count - - while ($NumJobs -ge $MaxParallelJobs) { - Write-Host "There are $NumJobs validation jobs running right now. Waiting $SecondsBetweenLoadChecks seconds to check again." - sleep $SecondsBetweenLoadChecks - $NumJobs = @(Get-Job -State 'Running').Count - } - - foreach ($Job in @(Get-Job -State 'Completed')) { - $jobResult = Wait-Job -Id $Job.Id | Receive-Job - CheckJobResult $jobResult.result $jobResult.packagePath ([ref]$ValidationFailures) -LogErrors - Remove-Job -Id $Job.Id - } - } - - foreach ($Job in @(Get-Job)) { - $jobResult = Wait-Job -Id $Job.Id | Receive-Job - CheckJobResult $jobResult.result $jobResult.packagePath ([ref]$ValidationFailures) - Remove-Job -Id $Job.Id - } - if ($ValidationFailures -gt 0) { - Write-PipelineTelemetryError -Category 'SourceLink' -Message "$ValidationFailures package(s) failed validation." - ExitWithExitCode 1 - } -} - -function InstallSourcelinkCli { - $sourcelinkCliPackageName = 'sourcelink' - - $dotnetRoot = InitializeDotNetCli -install:$true - $dotnet = "$dotnetRoot\dotnet.exe" - $toolList = & "$dotnet" tool list --global - - if (($toolList -like "*$sourcelinkCliPackageName*") -and ($toolList -like "*$sourcelinkCliVersion*")) { - Write-Host "SourceLink CLI version $sourcelinkCliVersion is already installed." - } - else { - Write-Host "Installing SourceLink CLI version $sourcelinkCliVersion..." - Write-Host 'You may need to restart your command window if this is the first dotnet tool you have installed.' - & "$dotnet" tool install $sourcelinkCliPackageName --version $sourcelinkCliVersion --verbosity "minimal" --global - } -} - -try { - InstallSourcelinkCli - - foreach ($Job in @(Get-Job)) { - Remove-Job -Id $Job.Id - } - - ValidateSourceLinkLinks -} -catch { - Write-Host $_.Exception - Write-Host $_.ScriptStackTrace - Write-PipelineTelemetryError -Category 'SourceLink' -Message $_ - ExitWithExitCode 1 -} diff --git a/eng/common/sdk-task.ps1 b/eng/common/sdk-task.ps1 index 4017ff15ebf..68119de603e 100644 --- a/eng/common/sdk-task.ps1 +++ b/eng/common/sdk-task.ps1 @@ -66,20 +66,7 @@ try { if( $msbuildEngine -eq "vs") { # Ensure desktop MSBuild is available for sdk tasks. - if( -not ($GlobalJson.tools.PSObject.Properties.Name -contains "vs" )) { - $GlobalJson.tools | Add-Member -Name "vs" -Value (ConvertFrom-Json "{ `"version`": `"16.5`" }") -MemberType NoteProperty - } - if( -not ($GlobalJson.tools.PSObject.Properties.Name -match "xcopy-msbuild" )) { - $GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "18.0.0" -MemberType NoteProperty - } - if ($GlobalJson.tools."xcopy-msbuild".Trim() -ine "none") { - $xcopyMSBuildToolsFolder = InitializeXCopyMSBuild $GlobalJson.tools."xcopy-msbuild" -install $true - } - if ($xcopyMSBuildToolsFolder -eq $null) { - throw 'Unable to get xcopy downloadable version of msbuild' - } - - $global:_MSBuildExe = "$($xcopyMSBuildToolsFolder)\MSBuild\Current\Bin\MSBuild.exe" + $global:_MSBuildExe = InitializeVisualStudioMSBuild } $taskProject = GetSdkTaskProject $task diff --git a/eng/common/template-guidance.md b/eng/common/template-guidance.md index cdc62e72b07..f772aa3d78f 100644 --- a/eng/common/template-guidance.md +++ b/eng/common/template-guidance.md @@ -81,7 +81,6 @@ eng\common\ publish-build-artifacts.yml (logic) publish-pipeline-artifacts.yml (logic) component-governance.yml (shim) - generate-sbom.yml (shim) publish-logs.yml (shim) retain-build.yml (shim) send-to-helix.yml (shim) @@ -104,7 +103,6 @@ eng\common\ setup-maestro-vars.yml (logic) steps\ component-governance.yml (logic) - generate-sbom.yml (logic) publish-build-artifacts.yml (redirect) publish-logs.yml (logic) publish-pipeline-artifacts.yml (redirect) diff --git a/eng/common/templates-official/job/job.yml b/eng/common/templates-official/job/job.yml index f70224eaa45..d68e9fbc265 100644 --- a/eng/common/templates-official/job/job.yml +++ b/eng/common/templates-official/job/job.yml @@ -1,24 +1,15 @@ parameters: -# Sbom related params - enableSbom: true runAsPublic: false - PackageVersion: 9.0.0 - BuildDropPath: '$(System.DefaultWorkingDirectory)/artifacts' +# Sbom related params, unused now and can eventually be removed + enableSbom: unused + PackageVersion: unused + BuildDropPath: unused jobs: - template: /eng/common/core-templates/job/job.yml parameters: is1ESPipeline: true - componentGovernanceSteps: - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.enableSbom, 'true')) }}: - - template: /eng/common/templates/steps/generate-sbom.yml - parameters: - PackageVersion: ${{ parameters.packageVersion }} - BuildDropPath: ${{ parameters.buildDropPath }} - ManifestDirPath: $(Build.ArtifactStagingDirectory)/sbom - publishArtifacts: false - # publish artifacts # for 1ES managed templates, use the templateContext.output to handle multiple outputs. templateContext: @@ -26,12 +17,19 @@ jobs: outputs: - ${{ if ne(parameters.artifacts.publish, '') }}: - ${{ if and(ne(parameters.artifacts.publish.artifacts, 'false'), ne(parameters.artifacts.publish.artifacts, '')) }}: - - output: buildArtifacts + - output: pipelineArtifact displayName: Publish pipeline artifacts - PathtoPublish: '$(Build.ArtifactStagingDirectory)/artifacts' - ArtifactName: ${{ coalesce(parameters.artifacts.publish.artifacts.name , 'Artifacts_$(Agent.Os)_$(_BuildConfig)') }} - condition: always() - retryCountOnTaskFailure: 10 # for any logs being locked + targetPath: '$(Build.ArtifactStagingDirectory)/artifacts' + artifactName: ${{ coalesce(parameters.artifacts.publish.artifacts.name , 'Artifacts_$(Agent.Os)_$(_BuildConfig)') }} + condition: succeeded() + retryCountOnTaskFailure: 10 # for any files being locked + continueOnError: true + - output: pipelineArtifact + displayName: Publish pipeline artifacts + targetPath: '$(Build.ArtifactStagingDirectory)/artifacts' + artifactName: ${{ coalesce(parameters.artifacts.publish.artifacts.name , 'Artifacts_$(Agent.Os)_$(_BuildConfig)') }}_Attempt$(System.JobAttempt) + condition: not(succeeded()) + retryCountOnTaskFailure: 10 # for any files being locked continueOnError: true - ${{ if and(ne(parameters.artifacts.publish.logs, 'false'), ne(parameters.artifacts.publish.logs, '')) }}: - output: pipelineArtifact @@ -40,8 +38,8 @@ jobs: displayName: 'Publish logs' continueOnError: true condition: always() - retryCountOnTaskFailure: 10 # for any logs being locked - sbomEnabled: false # we don't need SBOM for logs + retryCountOnTaskFailure: 10 # for any files being locked + isProduction: false # logs are non-production artifacts - ${{ if eq(parameters.enablePublishBuildArtifacts, true) }}: - output: pipelineArtifact @@ -50,7 +48,8 @@ jobs: artifactName: ${{ coalesce(parameters.enablePublishBuildArtifacts.artifactName, '$(Agent.Os)_$(Agent.JobName)_Attempt$(System.JobAttempt)' ) }} continueOnError: true condition: always() - sbomEnabled: false # we don't need SBOM for logs + retryCountOnTaskFailure: 10 # for any files being locked + isProduction: false # logs are non-production artifacts - ${{ if eq(parameters.enableBuildRetry, 'true') }}: - output: pipelineArtifact @@ -58,14 +57,20 @@ jobs: artifactName: 'BuildConfiguration' displayName: 'Publish build retry configuration' continueOnError: true - sbomEnabled: false # we don't need SBOM for BuildConfiguration + retryCountOnTaskFailure: 10 # for any files being locked + isProduction: false # BuildConfiguration is a non-production artifact - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.enableSbom, 'true')) }}: + # V4 publishing: automatically publish staged artifacts as a pipeline artifact. + # The artifact name matches the SDK's FutureArtifactName ($(System.PhaseName)_Artifacts), + # which is encoded in the asset manifest for downstream publishing to discover. + # Jobs can opt in by setting enablePublishing: true. + - ${{ if and(eq(parameters.publishingVersion, 4), eq(parameters.enablePublishing, 'true')) }}: - output: pipelineArtifact - displayName: Publish SBOM manifest + displayName: 'Publish V4 pipeline artifacts' + targetPath: '$(Build.ArtifactStagingDirectory)/artifacts' + artifactName: '$(System.PhaseName)_Artifacts' continueOnError: true - targetPath: $(Build.ArtifactStagingDirectory)/sbom - artifactName: $(ARTIFACT_NAME) + retryCountOnTaskFailure: 10 # for any files being locked # add any outputs provided via root yaml - ${{ if ne(parameters.templateContext.outputs, '') }}: diff --git a/eng/common/templates-official/steps/component-governance.yml b/eng/common/templates-official/steps/component-governance.yml deleted file mode 100644 index 30bb3985ca2..00000000000 --- a/eng/common/templates-official/steps/component-governance.yml +++ /dev/null @@ -1,7 +0,0 @@ -steps: -- template: /eng/common/core-templates/steps/component-governance.yml - parameters: - is1ESPipeline: true - - ${{ each parameter in parameters }}: - ${{ parameter.key }}: ${{ parameter.value }} diff --git a/eng/common/templates-official/steps/publish-pipeline-artifacts.yml b/eng/common/templates-official/steps/publish-pipeline-artifacts.yml index 172f9f0fdc9..9e5981365e5 100644 --- a/eng/common/templates-official/steps/publish-pipeline-artifacts.yml +++ b/eng/common/templates-official/steps/publish-pipeline-artifacts.yml @@ -24,5 +24,7 @@ steps: artifactName: ${{ parameters.args.artifactName }} ${{ if parameters.args.properties }}: properties: ${{ parameters.args.properties }} - ${{ if parameters.args.sbomEnabled }}: + ${{ if ne(parameters.args.sbomEnabled, '') }}: sbomEnabled: ${{ parameters.args.sbomEnabled }} + ${{ if ne(parameters.args.isProduction, '') }}: + isProduction: ${{ parameters.args.isProduction }} diff --git a/eng/common/templates/job/job.yml b/eng/common/templates/job/job.yml index 7f1b5d97d1a..5e261f34db4 100644 --- a/eng/common/templates/job/job.yml +++ b/eng/common/templates/job/job.yml @@ -1,12 +1,12 @@ parameters: enablePublishBuildArtifacts: false - disableComponentGovernance: '' - componentGovernanceIgnoreDirectories: '' -# Sbom related params - enableSbom: true runAsPublic: false - PackageVersion: 9.0.0 - BuildDropPath: '$(System.DefaultWorkingDirectory)/artifacts' +# CG related params, unused now and can eventually be removed + disableComponentGovernance: unused +# Sbom related params, unused now and can eventually be removed + enableSbom: unused + PackageVersion: unused + BuildDropPath: unused jobs: - template: /eng/common/core-templates/job/job.yml @@ -21,32 +21,34 @@ jobs: - ${{ each step in parameters.steps }}: - ${{ step }} - componentGovernanceSteps: - - template: /eng/common/templates/steps/component-governance.yml - parameters: - ${{ if eq(parameters.disableComponentGovernance, '') }}: - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.runAsPublic, 'false'), or(startsWith(variables['Build.SourceBranch'], 'refs/heads/release/'), startsWith(variables['Build.SourceBranch'], 'refs/heads/dotnet/'), startsWith(variables['Build.SourceBranch'], 'refs/heads/microsoft/'), eq(variables['Build.SourceBranch'], 'refs/heads/main'))) }}: - disableComponentGovernance: false - ${{ else }}: - disableComponentGovernance: true - ${{ else }}: - disableComponentGovernance: ${{ parameters.disableComponentGovernance }} - componentGovernanceIgnoreDirectories: ${{ parameters.componentGovernanceIgnoreDirectories }} + # we don't run CG in public + - ${{ if eq(variables['System.TeamProject'], 'public') }}: + - script: echo "##vso[task.setvariable variable=skipComponentGovernanceDetection]true" + displayName: Set skipComponentGovernanceDetection variable artifactPublishSteps: - ${{ if ne(parameters.artifacts.publish, '') }}: - ${{ if and(ne(parameters.artifacts.publish.artifacts, 'false'), ne(parameters.artifacts.publish.artifacts, '')) }}: - - template: /eng/common/core-templates/steps/publish-build-artifacts.yml + - template: /eng/common/core-templates/steps/publish-pipeline-artifacts.yml parameters: is1ESPipeline: false args: displayName: Publish pipeline artifacts - pathToPublish: '$(Build.ArtifactStagingDirectory)/artifacts' - publishLocation: Container + targetPath: '$(Build.ArtifactStagingDirectory)/artifacts' artifactName: ${{ coalesce(parameters.artifacts.publish.artifacts.name , 'Artifacts_$(Agent.Os)_$(_BuildConfig)') }} continueOnError: true - condition: always() - retryCountOnTaskFailure: 10 # for any logs being locked + condition: succeeded() + retryCountOnTaskFailure: 10 # for any files being locked + - template: /eng/common/core-templates/steps/publish-pipeline-artifacts.yml + parameters: + is1ESPipeline: false + args: + displayName: Publish pipeline artifacts + targetPath: '$(Build.ArtifactStagingDirectory)/artifacts' + artifactName: ${{ coalesce(parameters.artifacts.publish.artifacts.name , 'Artifacts_$(Agent.Os)_$(_BuildConfig)') }}_Attempt$(System.JobAttempt) + continueOnError: true + condition: not(succeeded()) + retryCountOnTaskFailure: 10 # for any files being locked - ${{ if and(ne(parameters.artifacts.publish.logs, 'false'), ne(parameters.artifacts.publish.logs, '')) }}: - template: /eng/common/core-templates/steps/publish-pipeline-artifacts.yml parameters: @@ -57,8 +59,7 @@ jobs: displayName: 'Publish logs' continueOnError: true condition: always() - retryCountOnTaskFailure: 10 # for any logs being locked - sbomEnabled: false # we don't need SBOM for logs + retryCountOnTaskFailure: 10 # for any files being locked - ${{ if ne(parameters.enablePublishBuildArtifacts, 'false') }}: - template: /eng/common/core-templates/steps/publish-pipeline-artifacts.yml @@ -70,7 +71,7 @@ jobs: artifactName: ${{ coalesce(parameters.enablePublishBuildArtifacts.artifactName, '$(Agent.Os)_$(Agent.JobName)_Attempt$(System.JobAttempt)' ) }} continueOnError: true condition: always() - sbomEnabled: false + retryCountOnTaskFailure: 10 # for any files being locked - ${{ if eq(parameters.enableBuildRetry, 'true') }}: - template: /eng/common/core-templates/steps/publish-pipeline-artifacts.yml @@ -81,4 +82,4 @@ jobs: artifactName: 'BuildConfiguration' displayName: 'Publish build retry configuration' continueOnError: true - sbomEnabled: false # we don't need SBOM for BuildConfiguration + retryCountOnTaskFailure: 10 # for any files being locked diff --git a/eng/common/templates/steps/component-governance.yml b/eng/common/templates/steps/component-governance.yml deleted file mode 100644 index c12a5f8d21d..00000000000 --- a/eng/common/templates/steps/component-governance.yml +++ /dev/null @@ -1,7 +0,0 @@ -steps: -- template: /eng/common/core-templates/steps/component-governance.yml - parameters: - is1ESPipeline: false - - ${{ each parameter in parameters }}: - ${{ parameter.key }}: ${{ parameter.value }} diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index 6710ffb884b..0e281df8cae 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -168,6 +168,12 @@ function InitializeDotNetCli([bool]$install, [bool]$createSdkLocationFile) { $env:DOTNET_CLI_TELEMETRY_OPTOUT=1 } + # Keep repo builds isolated from machine-installed SDK state and workload advertising. + # This avoids preview SDK builds picking up mismatched workloads on CI images. + $env:DOTNET_MULTILEVEL_LOOKUP = '0' + $env:DOTNET_SKIP_FIRST_TIME_EXPERIENCE = '1' + $env:DOTNET_CLI_WORKLOAD_UPDATE_NOTIFY_DISABLE = '1' + # Find the first path on %PATH% that contains the dotnet.exe if ($useInstalledDotNetCli -and (-not $globalJsonHasRuntimes) -and ($env:DOTNET_INSTALL_DIR -eq $null)) { $dotnetExecutable = GetExecutableFileName 'dotnet' @@ -185,7 +191,11 @@ function InitializeDotNetCli([bool]$install, [bool]$createSdkLocationFile) { if ((-not $globalJsonHasRuntimes) -and (-not [string]::IsNullOrEmpty($env:DOTNET_INSTALL_DIR)) -and (Test-Path(Join-Path $env:DOTNET_INSTALL_DIR "sdk\$dotnetSdkVersion"))) { $dotnetRoot = $env:DOTNET_INSTALL_DIR } else { - $dotnetRoot = Join-Path $RepoRoot '.dotnet' + if (-not [string]::IsNullOrEmpty($env:DOTNET_GLOBAL_INSTALL_DIR)) { + $dotnetRoot = $env:DOTNET_GLOBAL_INSTALL_DIR + } else { + $dotnetRoot = Join-Path $RepoRoot '.dotnet' + } if (-not (Test-Path(Join-Path $dotnetRoot "sdk\$dotnetSdkVersion"))) { if ($install) { @@ -226,6 +236,9 @@ function InitializeDotNetCli([bool]$install, [bool]$createSdkLocationFile) { Write-PipelinePrependPath -Path $dotnetRoot Write-PipelineSetVariable -Name 'DOTNET_NOLOGO' -Value '1' + Write-PipelineSetVariable -Name 'DOTNET_MULTILEVEL_LOOKUP' -Value '0' + Write-PipelineSetVariable -Name 'DOTNET_SKIP_FIRST_TIME_EXPERIENCE' -Value '1' + Write-PipelineSetVariable -Name 'DOTNET_CLI_WORKLOAD_UPDATE_NOTIFY_DISABLE' -Value '1' return $global:_DotNetInstallDir = $dotnetRoot } @@ -375,12 +388,11 @@ function InstallDotNet([string] $dotnetRoot, # # 1. MSBuild from an active VS command prompt # 2. MSBuild from a compatible VS installation -# 3. MSBuild from the xcopy tool package # # Returns full path to msbuild.exe. # Throws on failure. # -function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements = $null) { +function InitializeVisualStudioMSBuild([object]$vsRequirements = $null) { if (-not (IsWindowsPlatform)) { throw "Cannot initialize Visual Studio on non-Windows" } @@ -390,13 +402,7 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements = } # Minimum VS version to require. - $vsMinVersionReqdStr = '17.7' - $vsMinVersionReqd = [Version]::new($vsMinVersionReqdStr) - - # If the version of msbuild is going to be xcopied, - # use this version. Version matches a package here: - # https://dev.azure.com/dnceng/public/_artifacts/feed/dotnet-eng/NuGet/Microsoft.DotNet.Arcade.MSBuild.Xcopy/versions/18.0.0 - $defaultXCopyMSBuildVersion = '18.0.0' + $vsMinVersionReqdStr = '18.0' if (!$vsRequirements) { if (Get-Member -InputObject $GlobalJson.tools -Name 'vs') { @@ -426,46 +432,16 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements = } } - # Locate Visual Studio installation or download x-copy msbuild. + # Locate Visual Studio installation. $vsInfo = LocateVisualStudio $vsRequirements - if ($vsInfo -ne $null -and $env:ForceUseXCopyMSBuild -eq $null) { + if ($vsInfo -ne $null) { # Ensure vsInstallDir has a trailing slash $vsInstallDir = Join-Path $vsInfo.installationPath "\" $vsMajorVersion = $vsInfo.installationVersion.Split('.')[0] InitializeVisualStudioEnvironmentVariables $vsInstallDir $vsMajorVersion } else { - if (Get-Member -InputObject $GlobalJson.tools -Name 'xcopy-msbuild') { - $xcopyMSBuildVersion = $GlobalJson.tools.'xcopy-msbuild' - $vsMajorVersion = $xcopyMSBuildVersion.Split('.')[0] - } else { - #if vs version provided in global.json is incompatible (too low) then use the default version for xcopy msbuild download - if($vsMinVersion -lt $vsMinVersionReqd){ - Write-Host "Using xcopy-msbuild version of $defaultXCopyMSBuildVersion since VS version $vsMinVersionStr provided in global.json is not compatible" - $xcopyMSBuildVersion = $defaultXCopyMSBuildVersion - $vsMajorVersion = $xcopyMSBuildVersion.Split('.')[0] - } - else{ - # If the VS version IS compatible, look for an xcopy msbuild package - # with a version matching VS. - # Note: If this version does not exist, then an explicit version of xcopy msbuild - # can be specified in global.json. This will be required for pre-release versions of msbuild. - $vsMajorVersion = $vsMinVersion.Major - $vsMinorVersion = $vsMinVersion.Minor - $xcopyMSBuildVersion = "$vsMajorVersion.$vsMinorVersion.0" - } - } - - $vsInstallDir = $null - if ($xcopyMSBuildVersion.Trim() -ine "none") { - $vsInstallDir = InitializeXCopyMSBuild $xcopyMSBuildVersion $install - if ($vsInstallDir -eq $null) { - throw "Could not xcopy msbuild. Please check that package 'Microsoft.DotNet.Arcade.MSBuild.Xcopy @ $xcopyMSBuildVersion' exists on feed 'dotnet-eng'." - } - } - if ($vsInstallDir -eq $null) { - throw 'Unable to find Visual Studio that has required version and components installed' - } + throw 'Unable to find Visual Studio that has required version and components installed' } $msbuildVersionDir = if ([int]$vsMajorVersion -lt 16) { "$vsMajorVersion.0" } else { "Current" } @@ -492,38 +468,6 @@ function InitializeVisualStudioEnvironmentVariables([string] $vsInstallDir, [str } } -function InstallXCopyMSBuild([string]$packageVersion) { - return InitializeXCopyMSBuild $packageVersion -install $true -} - -function InitializeXCopyMSBuild([string]$packageVersion, [bool]$install) { - $packageName = 'Microsoft.DotNet.Arcade.MSBuild.Xcopy' - $packageDir = Join-Path $ToolsDir "msbuild\$packageVersion" - $packagePath = Join-Path $packageDir "$packageName.$packageVersion.nupkg" - - if (!(Test-Path $packageDir)) { - if (!$install) { - return $null - } - - Create-Directory $packageDir - - Write-Host "Downloading $packageName $packageVersion" - $ProgressPreference = 'SilentlyContinue' # Don't display the console progress UI - it's a huge perf hit - Retry({ - Invoke-WebRequest "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/flat2/$packageName/$packageVersion/$packageName.$packageVersion.nupkg" -UseBasicParsing -OutFile $packagePath - }) - - if (!(Test-Path $packagePath)) { - Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "See https://dev.azure.com/dnceng/internal/_wiki/wikis/DNCEng%20Services%20Wiki/1074/Updating-Microsoft.DotNet.Arcade.MSBuild.Xcopy-WAS-RoslynTools.MSBuild-(xcopy-msbuild)-generation?anchor=troubleshooting for help troubleshooting issues with XCopy MSBuild" - throw - } - Unzip $packagePath $packageDir - } - - return Join-Path $packageDir 'tools' -} - # # Locates Visual Studio instance that meets the minimal requirements specified by tools.vs object in global.json. # @@ -632,7 +576,7 @@ function InitializeBuildTool() { $buildTool = @{ Path = $dotnetPath; Command = 'msbuild'; Tool = 'dotnet'; Framework = 'net' } } elseif ($msbuildEngine -eq "vs") { try { - $msbuildPath = InitializeVisualStudioMSBuild -install:$restore + $msbuildPath = InitializeVisualStudioMSBuild } catch { Write-PipelineTelemetryError -Category 'InitializeToolset' -Message $_ ExitWithExitCode 1 @@ -684,11 +628,7 @@ function GetSdkTaskProject([string]$taskName) { if (Test-Path $proj) { return $proj } - # TODO: Remove this fallback once all supported versions use the new layout. - $legacyProj = Join-Path $toolsetDir "SdkTasks\$taskName.proj" - if (Test-Path $legacyProj) { - return $legacyProj - } + throw "Unable to find $taskName.proj in toolset at: $toolsetDir" } @@ -745,32 +685,33 @@ function InitializeToolset() { ExitWithExitCode 1 } - $downloadArgs = @("package", "download", "Microsoft.DotNet.Arcade.Sdk@$toolsetVersion", "--prerelease", "--output", "$nugetCache") - if ($env:NUGET_CONFIG) { + $downloadArgs = @("package", "download", "Microsoft.DotNet.Arcade.Sdk@$toolsetVersion", "--verbosity", "minimal", "--prerelease", "--output", "$nugetCache") + $nugetConfig = $env:NUGET_CONFIG + if (-not $nugetConfig) { + # Search for any variation of nuget.config in the RepoRoot + $configFile = Get-ChildItem -Path $RepoRoot -File | Where-Object { $_.Name -ieq "nuget.config" } | Select-Object -First 1 + + if ($configFile) { + $nugetConfig = $configFile.FullName + } + } + + if ($nugetConfig) { $downloadArgs += "--configfile" - $downloadArgs += $env:NUGET_CONFIG + $downloadArgs += $nugetConfig } DotNet @downloadArgs $packageDir = Join-Path $nugetCache (Join-Path 'microsoft.dotnet.arcade.sdk' $toolsetVersion) $packageToolsetDir = Join-Path $packageDir 'toolset' - $packageToolsDir = Join-Path $packageDir 'tools' - # TODO: Remove the tools/ check once all supported versions have the toolset folder. - if (!(Test-Path $packageToolsetDir) -and !(Test-Path $packageToolsDir)) { + if (!(Test-Path $packageToolsetDir)) { Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Arcade SDK package does not contain a toolset or tools folder: $packageDir" ExitWithExitCode 3 } New-Item -ItemType Directory -Path $toolsetToolsDir -Force | Out-Null - - # Copy toolset if present at the package root (new layout), otherwise fall back to tools - if (Test-Path $packageToolsetDir) { - Copy-Item -Path "$packageToolsetDir\*" -Destination $toolsetToolsDir -Recurse -Force - } else { - # TODO: Remove this fallback once all supported versions have the toolset folder. - Copy-Item -Path "$packageToolsDir\*" -Destination $toolsetToolsDir -Recurse -Force - } + Copy-Item -Path "$packageToolsetDir\*" -Destination $toolsetToolsDir -Recurse -Force if (Test-Path $buildProjPath) { $toolsetBuildProj = $buildProjPath @@ -897,6 +838,10 @@ function MSBuild-Core() { $cmdArgs = "$($buildTool.Command) /m /nologo /clp:Summary /v:$verbosity /nr:$nodeReuse /p:ContinuousIntegrationBuild=$ci" + if ($ci -and $buildTool.Tool -eq 'dotnet') { + $cmdArgs += ' /p:MSBuildEnableWorkloadResolver=false' + } + # Add -mt flag for MSBuild multithreaded mode if enabled via environment variable if ($env:MSBUILD_MT_ENABLED -eq "1") { $cmdArgs += ' -mt' @@ -1007,6 +952,12 @@ Create-Directory $ToolsetDir Create-Directory $TempDir Create-Directory $LogDir +# Direct MSBuild crash diagnostics (MSB4166 failure.txt files) to a known location +# under artifacts/log so they are captured as build artifacts in CI. +if (-not $env:MSBUILDDEBUGPATH) { + $env:MSBUILDDEBUGPATH = Join-Path $LogDir 'MsbuildDebugLogs' +} + Write-PipelineSetVariable -Name 'Artifacts' -Value $ArtifactsDir Write-PipelineSetVariable -Name 'Artifacts.Toolset' -Value $ToolsetDir Write-PipelineSetVariable -Name 'Artifacts.Log' -Value $LogDir diff --git a/eng/common/tools.sh b/eng/common/tools.sh index d2339eb21d5..5ff37cfb700 100755 --- a/eng/common/tools.sh +++ b/eng/common/tools.sh @@ -126,6 +126,12 @@ function InitializeDotNetCli { export DOTNET_CLI_TELEMETRY_OPTOUT=1 fi + # Keep repo builds isolated from machine-installed SDK state and workload advertising. + # This avoids preview SDK builds picking up mismatched workloads on CI images. + export DOTNET_MULTILEVEL_LOOKUP=0 + export DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1 + export DOTNET_CLI_WORKLOAD_UPDATE_NOTIFY_DISABLE=1 + # LTTNG is the logging infrastructure used by Core CLR. Need this variable set # so it doesn't output warnings to the console. export LTTNG_HOME="$HOME" @@ -148,7 +154,11 @@ function InitializeDotNetCli { if [[ $global_json_has_runtimes == false && -n "${DOTNET_INSTALL_DIR:-}" && -d "$DOTNET_INSTALL_DIR/sdk/$dotnet_sdk_version" ]]; then dotnet_root="$DOTNET_INSTALL_DIR" else - dotnet_root="${repo_root}.dotnet" + if [[ -n "${DOTNET_GLOBAL_INSTALL_DIR:-}" ]]; then + dotnet_root="$DOTNET_GLOBAL_INSTALL_DIR" + else + dotnet_root="${repo_root}.dotnet" + fi export DOTNET_INSTALL_DIR="$dotnet_root" @@ -167,6 +177,9 @@ function InitializeDotNetCli { Write-PipelinePrependPath -path "$dotnet_root" Write-PipelineSetVariable -name "DOTNET_NOLOGO" -value "1" + Write-PipelineSetVariable -name "DOTNET_MULTILEVEL_LOOKUP" -value "0" + Write-PipelineSetVariable -name "DOTNET_SKIP_FIRST_TIME_EXPERIENCE" -value "1" + Write-PipelineSetVariable -name "DOTNET_CLI_WORKLOAD_UPDATE_NOTIFY_DISABLE" -value "1" # return value _InitializeDotNetCli="$dotnet_root" @@ -426,29 +439,32 @@ function InitializeToolset { ExitWithExitCode 2 fi - local download_args=("package" "download" "Microsoft.DotNet.Arcade.Sdk@$toolset_version" "--prerelease" "--output" "$_GetNuGetPackageCachePath") - if [[ -n "${NUGET_CONFIG:-}" ]]; then - download_args+=("--configfile" "$NUGET_CONFIG") + local download_args=("package" "download" "Microsoft.DotNet.Arcade.Sdk@$toolset_version" "--verbosity" "minimal" "--prerelease" "--output" "$_GetNuGetPackageCachePath") + local nuget_config="${NUGET_CONFIG:-}" + if [[ -z "$nuget_config" ]]; then + # Search for any variation of nuget.config in the RepoRoot + local found_config + found_config=$(find "$repo_root" -maxdepth 1 -type f -iname "nuget.config" -print -quit) + + if [[ -n "$found_config" ]]; then + nuget_config="$found_config" + fi + fi + + if [[ -n "$nuget_config" ]]; then + download_args+=("--configfile" "$nuget_config") fi DotNet "${download_args[@]}" local package_dir="$_GetNuGetPackageCachePath/microsoft.dotnet.arcade.sdk/$toolset_version" - # TODO: Remove the tools/ check once all supported versions have the toolset folder. - if [[ ! -d "$package_dir/toolset" && ! -d "$package_dir/tools" ]]; then - Write-PipelineTelemetryError -category 'InitializeToolset' "Arcade SDK package does not contain a toolset or tools folder: $package_dir" + if [[ ! -d "$package_dir/toolset" ]]; then + Write-PipelineTelemetryError -category 'InitializeToolset' "Arcade SDK package does not contain a toolset folder: $package_dir" ExitWithExitCode 3 fi mkdir -p "$toolset_tools_dir" - - # Copy toolset if present at the package root (new layout), otherwise fall back to tools - if [[ -d "$package_dir/toolset" ]]; then - cp -r "$package_dir/toolset/." "$toolset_tools_dir" - else - # TODO: Remove this fallback once all supported versions have the toolset folder. - cp -r "$package_dir/tools/." "$toolset_tools_dir" - fi + cp -r "$package_dir/toolset/." "$toolset_tools_dir" if [[ -a "$toolset_tools_dir/Build.proj" ]]; then toolset_build_proj="$toolset_tools_dir/Build.proj" @@ -575,7 +591,12 @@ function MSBuild-Core { warnnotaserror_switch="/warnnotaserror:$warn_not_as_error /p:AdditionalWarningsNotAsErrors=$warn_not_as_error" fi - RunBuildTool "$_InitializeBuildToolCommand" /m /nologo /clp:Summary /v:$verbosity /nr:$node_reuse $warnaserror_switch $mt_switch $warnnotaserror_switch /p:TreatWarningsAsErrors=$warn_as_error /p:ContinuousIntegrationBuild=$ci "$@" + local workload_resolver_switch="" + if [[ "$ci" == true && -n "${_InitializeBuildToolCommand:-}" ]]; then + workload_resolver_switch="/p:MSBuildEnableWorkloadResolver=false" + fi + + RunBuildTool "$_InitializeBuildToolCommand" /m /nologo /clp:Summary /v:$verbosity /nr:$node_reuse $warnaserror_switch $mt_switch $warnnotaserror_switch $workload_resolver_switch /p:TreatWarningsAsErrors=$warn_as_error /p:ContinuousIntegrationBuild=$ci "$@" } function GetDarc { @@ -600,12 +621,7 @@ function GetSdkTaskProject { echo "$proj" return fi - # TODO: Remove this fallback once all supported versions use the new layout. - local legacyProj="$toolsetDir/SdkTasks/$taskName.proj" - if [[ -a "$legacyProj" ]]; then - echo "$legacyProj" - return - fi + Write-PipelineTelemetryError -category 'Build' "Unable to find $taskName.proj in toolset at: $toolsetDir" ExitWithExitCode 3 } @@ -645,6 +661,12 @@ mkdir -p "$toolset_dir" mkdir -p "$temp_dir" mkdir -p "$log_dir" +# Direct MSBuild crash diagnostics (MSB4166 failure.txt files) to a known location +# under artifacts/log so they are captured as build artifacts in CI. +if [[ -z "${MSBUILDDEBUGPATH:-}" ]]; then + export MSBUILDDEBUGPATH="$log_dir/MsbuildDebugLogs" +fi + Write-PipelineSetVariable -name "Artifacts" -value "$artifacts_dir" Write-PipelineSetVariable -name "Artifacts.Toolset" -value "$toolset_dir" Write-PipelineSetVariable -name "Artifacts.Log" -value "$log_dir" diff --git a/eng/pipelines/runtime-perf-jobs.yml b/eng/pipelines/runtime-perf-jobs.yml index a37eef06f18..3b9493cc0ee 100644 --- a/eng/pipelines/runtime-perf-jobs.yml +++ b/eng/pipelines/runtime-perf-jobs.yml @@ -56,23 +56,22 @@ parameters: type: object default: enabled: true - # Temporarily disabling cobalt jobs because the cobalt machines are offline - name: cobaltMicro type: object default: - enabled: false + enabled: true configs: - linux_arm64 - name: cobaltSveMicro type: object default: - enabled: false + enabled: true configs: - linux_arm64 - name: cobaltMicroR2RInterpreter type: object default: - enabled: false + enabled: true configs: - linux_arm64 - name: androidCoreclrR2r diff --git a/eng/pipelines/sdk-perf-jobs.yml b/eng/pipelines/sdk-perf-jobs.yml index 4c3a62d1a72..c3c87fca015 100644 --- a/eng/pipelines/sdk-perf-jobs.yml +++ b/eng/pipelines/sdk-perf-jobs.yml @@ -17,6 +17,43 @@ jobs: # Public correctness jobs ###################################################### +- ${{ if parameters.runPublicJobs }}: + - job: Tooling_Tests_Windows + displayName: Tooling Tests (Windows) + timeoutInMinutes: 30 + condition: and(succeeded(), ne(variables['Build.SourceBranch'], 'refs/heads/main')) + pool: + vmImage: windows-2022 + + steps: + - task: UseDotNet@2 + displayName: Install .NET 8.0 + inputs: + version: 8.0.x + + - task: UseDotNet@2 + displayName: Install .NET 10.0 + inputs: + version: 10.0.x + includePreviewVersions: true + + - task: UseDotNet@2 + displayName: Install .NET 11.0 + inputs: + version: 11.0.x + includePreviewVersions: true + + - pwsh: | + $ErrorActionPreference = 'Stop' + $PSNativeCommandUseErrorActionPreference = $true + + dotnet test src\tools\ResultsComparer.Tests\ResultsComparer.Tests.csproj --configuration Release --framework net11.0 --nologo --verbosity minimal + dotnet test src\tools\Reporting\Reporting.Tests\Reporting.Tests.csproj --configuration Release --framework net11.0 --nologo --verbosity minimal + dotnet test src\tools\CertHelperTests\CertRotatorTests.csproj --configuration Release --framework net10.0 --nologo --verbosity minimal + dotnet test src\tests\harness\BenchmarkDotNet.Extensions.Tests\BenchmarkDotNet.Extensions.Tests.csproj --configuration Release --framework net11.0 -p:PERFLAB_TARGET_FRAMEWORKS=net10.0 --nologo --verbosity minimal + dotnet test src\tools\ScenarioMeasurement\Startup.Tests\Startup.Tests.csproj --configuration Release --nologo --verbosity minimal + displayName: Run tooling tests + - ${{ if parameters.runPublicJobs }}: # Scenario benchmarks diff --git a/gc-azure-pipelines.yml b/gc-azure-pipelines.yml index cf6b208a8ee..1d1fd31fb48 100644 --- a/gc-azure-pipelines.yml +++ b/gc-azure-pipelines.yml @@ -1,5 +1,5 @@ -# Project: src\benchmarks\gc\GC.Infrastructure\GC.Infrastructure.NotebookTests\GC.Infrastructure.NotebookTests.csproj. -# Output: artifacts\bin\GC.Infrastructure.NotebookTests\Debug\net8.0\GC.Infrastructure.NotebookTests.dll +# Project: src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core.UnitTests/GC.Infrastructure.Core.UnitTests.csproj. +# Output: artifacts\bin\GC.Infrastructure.Core.UnitTests\Debug\net8.0\GC.Infrastructure.Core.UnitTests.dll resources: containers: @@ -26,15 +26,15 @@ pr: jobs: # TODO: Add back once we have validated windows. - # - job: GCNotebookValidation_Ubuntu + # - job: GCInfrastructureValidation_Ubuntu # pool: # vmImage: ubuntu-latest # container: ubuntu_x64_build_container # TODO: Add. - - job: GCNotebookValidation_Windows + - job: GCInfrastructureValidation_Windows pool: - vmImage: windows-2019 + vmImage: windows-2025 steps: # Install dotnet. We temporarily need both .NET 8 (for building and running the projects) and .NET 9 (for installing deps). @@ -52,10 +52,17 @@ jobs: inputs: version: 10.0.x includePreviewVersions: true + - task: UseDotNet@2 + displayName: Install .NET 11.0 + inputs: + version: 11.0.x + includePreviewVersions: true - script: dotnet tool restore - script: dotnet tool install --global dotnet-repl - - script: dotnet build src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.NotebookTests/GC.Infrastructure.NotebookTests.csproj --configuration Debug --framework net8.0 - - script: dotnet build src/benchmarks/gc/GC.Infrastructure/GC.Analysis.API/GC.Analysis.API.csproj --configuration Release --framework net8.0 + - script: dotnet build src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/GC.Infrastructure.Core.csproj --configuration Debug --framework net8.0 + - script: dotnet build src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core.UnitTests/GC.Infrastructure.Core.UnitTests.csproj --configuration Debug --framework net8.0 + - script: dotnet build src/benchmarks/gc/GC.Infrastructure/GC.Analysis.API/GC.Analysis.API.csproj --configuration Debug --framework net8.0 + - script: dotnet build src/benchmarks/gc/GC.Infrastructure/GC.Analysis.API.UnitTests/GC.Analysis.API.UnitTests.csproj --configuration Debug --framework net8.0 # Run tests. Template installed from: https://learn.microsoft.com/en-us/azure/devops/pipelines/tasks/reference/vstest-v3?view=azure-pipelines - task: VSTest@3 @@ -63,16 +70,36 @@ jobs: # -------------- # # Test selection # # -------------- # - testAssemblyVer2: GC.Infrastructure.NotebookTests.dll - searchFolder: '$(System.DefaultWorkingDirectory)/artifacts/bin/GC.Infrastructure.NotebookTests/Debug/net8.0' + testAssemblyVer2: GC.Infrastructure.Core.UnitTests.dll + searchFolder: '$(System.DefaultWorkingDirectory)/artifacts/bin/GC.Infrastructure.Core.UnitTests/Debug/net8.0' + otherConsoleOptions: '--Framework:net8.0' # Uncomment to save results in the future. # resultsFolder: '$(Agent.TempDirectory)\TestResults' # string. Test results folder. Default: $(Agent.TempDirectory)\TestResults. # ----------------- # # Reporting options # # ----------------- # - testRunTitle: 'GC Notebook Test' + testRunTitle: 'GC Infrastructure Core Unit Test' # TODO: Double Check. platform: 'Windows' ## string. Build platform. # TODO: Double Check. - configuration: 'Test Notebooks' # string. Build configuration. \ No newline at end of file + configuration: 'Test GC Infrastructure Core' # string. Build configuration. + - task: VSTest@3 + inputs: + # -------------- # + # Test selection # + # -------------- # + testAssemblyVer2: GC.Analysis.API.UnitTests.dll + searchFolder: '$(System.DefaultWorkingDirectory)/artifacts/bin/GC.Analysis.API.UnitTests/Debug/net8.0' + otherConsoleOptions: '--Framework:net8.0' + # Uncomment to save results in the future. + # resultsFolder: '$(Agent.TempDirectory)\TestResults' # string. Test results folder. Default: $(Agent.TempDirectory)\TestResults. + + # ----------------- # + # Reporting options # + # ----------------- # + testRunTitle: 'GC Analysis API Unit Test' + # TODO: Double Check. + platform: 'Windows' ## string. Build platform. + # TODO: Double Check. + configuration: 'Test GC Analysis API' # string. Build configuration. diff --git a/global.json b/global.json index 79f2b114c10..148bd94594e 100644 --- a/global.json +++ b/global.json @@ -1,15 +1,15 @@ { "sdk": { - "version": "11.0.100-preview.3.26170.106", + "version": "11.0.100-preview.4.26210.111", "allowPrerelease": true, "rollForward": "major" }, "tools": { - "dotnet": "11.0.100-preview.3.26170.106" + "dotnet": "11.0.100-preview.4.26210.111" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "11.0.0-beta.26210.111", - "Microsoft.DotNet.Helix.Sdk": "11.0.0-beta.26210.111" + "Microsoft.DotNet.Arcade.Sdk": "11.0.0-beta.26261.101", + "Microsoft.DotNet.Helix.Sdk": "11.0.0-beta.26261.101" }, "native-tools": { "python3": "3.7.1" diff --git a/requirements.txt b/requirements.txt index 41c1983ebf9..4b121a529d9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,9 +1,9 @@ -cryptography==46.0.3 +cryptography==46.0.7 azure.storage.blob==12.13.0 azure.storage.queue==12.4.0 azure.identity==1.16.1 gitpython<=3.1.41 -urllib3==2.6.3 +urllib3==2.7.0 opentelemetry-api==1.23.0 opentelemetry-sdk==1.23.0 six==1.17.0 \ No newline at end of file diff --git a/scripts/channel_map.py b/scripts/channel_map.py index 0ca011576d5..118ad072022 100644 --- a/scripts/channel_map.py +++ b/scripts/channel_map.py @@ -9,13 +9,11 @@ class ChannelMap(): }, '10.0': { 'tfm': 'net10.0', - 'branch': '10.0', - 'quality': 'daily' + 'branch': '10.0' }, 'release/10.0': { 'tfm': 'net10.0', - 'branch': '10.0', - 'quality': 'daily' + 'branch': '10.0' }, 'nativeaot10.0': { 'tfm': 'nativeaot10.0', diff --git a/scripts/ci_setup.py b/scripts/ci_setup.py index e622a9fe69b..07c0a95733f 100644 --- a/scripts/ci_setup.py +++ b/scripts/ci_setup.py @@ -424,6 +424,11 @@ def main(args: CiSetupArgs): if args.experiment_name == "jitoptrepeat": experiment_config = variable_format % ('DOTNET_JitOptRepeat', '*') + if args.experiment_name == "runtimeasync": + # Surfaced to MSBuild as the $(EnableRuntimeAsync) property; gates the + # runtime-async Features flag in src/Directory.Build.targets. + experiment_config = variable_format % ('EnableRuntimeAsync', 'true') + output = '' with push_dir(get_repo_root_path()): diff --git a/scripts/dotnet.py b/scripts/dotnet.py index 72f006bac80..fc57103c5f1 100755 --- a/scripts/dotnet.py +++ b/scripts/dotnet.py @@ -5,6 +5,7 @@ """ import re +import json import datetime from argparse import ArgumentParser, ArgumentTypeError from glob import iglob @@ -862,6 +863,36 @@ def install( common_cmdline_args += ['-AzureFeed', azure_feed_url] common_cmdline_args += ['-FeedCredential', internal_build_key] + # Shield subsequent `dotnet` invocations (e.g. `dotnet --info` in ci_setup.py) + # from picking up an unrelated repo's global.json `paths` entry during the + # SDK resolver's upward walk. In particular, dotnet/runtime's global.json + # contains a `paths` entry that points the resolver at a `.dotnet` directory + # installed by arcade -- which can hold an SDK older than the one the perf + # scripts just installed into install_dir. We rewrite the upstream + # global.json with the `sdk.paths` field removed, preserving all other + # keys (notably `tools.dotnet`, which arcade's bootstrap requires to be + # present), letting the perf-installed SDK win via standard `$host$` + # resolution while keeping unrelated tooling functional. + workspace_root = path.abspath(path.join(get_repo_root_path(), '..')) + shield_global_json = path.join(workspace_root, 'global.json') + try: + if path.exists(shield_global_json): + with open(shield_global_json, 'r') as f: + shield_data = json.load(f) + sdk_section = shield_data.get('sdk') + if isinstance(sdk_section, dict) and 'paths' in sdk_section: + removed_paths = sdk_section.pop('paths') + with open(shield_global_json, 'w') as f: + json.dump(shield_data, f, indent=2) + f.write('\n') + getLogger().info( + "Stripped sdk.paths=%s from %s to shield SDK resolver " + "from an unrelated repo's global.json", + removed_paths, shield_global_json) + except (OSError, ValueError) as ex: + getLogger().warning( + "Could not shield global.json at %s: %s", shield_global_json, ex) + # Install Runtime/SDKs if versions: for version in versions: diff --git a/src/Directory.Build.props b/src/Directory.Build.props index 96647e856eb..b168fcecc65 100644 --- a/src/Directory.Build.props +++ b/src/Directory.Build.props @@ -5,27 +5,27 @@ false - + latest - + False - + $(NoWarn);NU1507 $(NoWarn);NETSDK1138 $(NoWarn);CS9057 True 4 - + True false false - + false diff --git a/src/Directory.Build.targets b/src/Directory.Build.targets new file mode 100644 index 00000000000..63d3b91f501 --- /dev/null +++ b/src/Directory.Build.targets @@ -0,0 +1,11 @@ + + + + + + $(Features);runtime-async=on + + \ No newline at end of file diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Analysis.API/Statistics.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Analysis.API/Statistics.cs index d0269d32326..5f0309b21dd 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Analysis.API/Statistics.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Analysis.API/Statistics.cs @@ -34,13 +34,22 @@ public static double StandardDeviation(this IEnumerable doubleList) public static IEnumerable RemoveOutliers(IEnumerable collection) { + List validCollection = new(); if (!collection.Any()) { return Array.Empty(); } - double[] validCollection = collection - .Where(x => !double.IsNaN(x) && !double.IsInfinity(x)) - .ToArray(); + foreach (double value in collection) + { + if (!double.IsNaN(value) && !double.IsInfinity(value)) + { + validCollection.Add(value); + } + } + if (validCollection.Count == 0) + { + return Array.Empty(); + } // Calculate Q1 (25th percentile) and Q3 (75th percentile) double q1 = GC.Analysis.API.Statistics.Percentile(validCollection, 0.25); double q3 = GC.Analysis.API.Statistics.Percentile(validCollection, 0.75); @@ -53,7 +62,7 @@ public static IEnumerable RemoveOutliers(IEnumerable collection) double upperBound = q3 + 1.5 * iqr; // Filter out outliers - return GoodLinq.Where(collection, x => x >= lowerBound && x <= upperBound); + return GoodLinq.Where(validCollection, x => x >= lowerBound && x <= upperBound); } } } diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/BdnJsonResult.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/BdnJsonResult.cs index e228ca6eff4..a64f11000a4 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/BdnJsonResult.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/BdnJsonResult.cs @@ -59,12 +59,12 @@ public sealed class Descriptor public sealed class ConfidenceInterval { public int N { get; set; } - public double? Mean { get; set; } - public double? StandardError { get; set; } - public int? Level { get; set; } - public double? Margin { get; set; } - public double? Lower { get; set; } - public double? Upper { get; set; } + public double Mean { get; set; } + public double StandardError { get; set; } + public int Level { get; set; } + public double Margin { get; set; } + public double Lower { get; set; } + public double Upper { get; set; } } public sealed class Percentiles @@ -84,23 +84,23 @@ public sealed class Statistics { public List OriginalValues { get; set; } public int N { get; set; } - public double? Min { get; set; } - public double? LowerFence { get; set; } - public double? Q1 { get; set; } - public double? Median { get; set; } - public double? Mean { get; set; } - public double? Q3 { get; set; } - public double? UpperFence { get; set; } - public double? Max { get; set; } - public double? InterquartileRange { get; set; } - public List LowerOutliers { get; set; } + public double Min { get; set; } + public double LowerFence { get; set; } + public double Q1 { get; set; } + public double Median { get; set; } + public double Mean { get; set; } + public double Q3 { get; set; } + public double UpperFence { get; set; } + public double Max { get; set; } + public double InterquartileRange { get; set; } + public List LowerOutliers { get; set; } public List UpperOutliers { get; set; } - public List AllOutliers { get; set; } - public double? StandardError { get; set; } - public double? Variance { get; set; } - public double? StandardDeviation { get; set; } - public double? Skewness { get; set; } - public double? Kurtosis { get; set; } + public List AllOutliers { get; set; } + public double StandardError { get; set; } + public double Variance { get; set; } + public double StandardDeviation { get; set; } + public double Skewness { get; set; } + public double Kurtosis { get; set; } public ConfidenceInterval? ConfidenceInterval { get; set; } public Percentiles Percentiles { get; set; } } diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/GCTraceMetricComparisonResult.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/GCTraceMetricComparisonResult.cs index deb14e44080..9f749a6a0c7 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/GCTraceMetricComparisonResult.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/GCTraceMetricComparisonResult.cs @@ -43,15 +43,24 @@ public GCTraceMetricComparisonResult(IEnumerable baselines, IEnu else { - OriginalBaselineMetricCollection = baselines.Select(baseline => baseline.StatsData[fieldInfo.Name]); - OriginalComparandMetricCollection = comparands.Select(comparand => comparand.StatsData[fieldInfo.Name]); + OriginalBaselineMetricCollection = baselines + .Where(baseline => baseline.StatsData.ContainsKey(fieldInfo.Name)) + .Select(baseline => baseline.StatsData[fieldInfo.Name]); + + OriginalComparandMetricCollection = comparands + .Where(comparand => comparand.StatsData.ContainsKey(fieldInfo.Name)) + .Select(comparand => comparand.StatsData[fieldInfo.Name]); } } else { - OriginalBaselineMetricCollection = baselines.Select(baseline => baseline.StatsData[pInfo.Name]); - OriginalComparandMetricCollection = comparands.Select(comparand => comparand.StatsData[pInfo.Name]); + OriginalBaselineMetricCollection = baselines + .Where(baseline => baseline.StatsData.ContainsKey(pInfo.Name)) + .Select(baseline => baseline.StatsData[pInfo.Name]); + OriginalComparandMetricCollection = comparands + .Where(comparand => comparand.StatsData.ContainsKey(pInfo.Name)) + .Select(comparand => comparand.StatsData[pInfo.Name]); } } diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/GCTraceMetrics.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/GCTraceMetrics.cs index df14a2368f9..d88777bc010 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/GCTraceMetrics.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/GCTraceMetrics.cs @@ -61,27 +61,29 @@ public GCTraceMetrics(GCProcessData processData, string runName, string configur var properties = processData.Stats.GetType().GetProperties(System.Reflection.BindingFlags.Public | System.Reflection.BindingFlags.Instance); foreach (var property in properties) { - if (property.PropertyType != typeof(double) || property.PropertyType != typeof(int)) + if (property.PropertyType != typeof(double) && property.PropertyType != typeof(int)) { continue; } string propertyName = property.Name; - double propertyValue = (double)(property.GetValue(processData.Stats) ?? double.NaN); + object? value = property.GetValue(processData.Stats); + double propertyValue = value != null ? Convert.ToDouble(value) : double.NaN; StatsData[propertyName] = propertyValue; } var fields = processData.Stats.GetType().GetFields(System.Reflection.BindingFlags.Public | System.Reflection.BindingFlags.Instance); foreach (var field in fields) { - if (field.FieldType != typeof(double) || field.FieldType != typeof(int)) + if (field.FieldType != typeof(double) && field.FieldType != typeof(int)) { continue; } string name = field.Name; - double value = (double)(field.GetValue(processData.Stats) ?? double.NaN); - StatsData[name] = value; + object? value = field.GetValue(processData.Stats); + double doubleValue = value != null ? Convert.ToDouble(value) : double.NaN; + StatsData[name] = doubleValue; } // 95P diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkComparisonResult.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkComparisonResult.cs index ade87c6d887..760a042e6ff 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkComparisonResult.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkComparisonResult.cs @@ -1,5 +1,4 @@ using API = GC.Analysis.API; -using Microsoft.Diagnostics.Tracing.Parsers.Clr; namespace GC.Infrastructure.Core.Analysis.Microbenchmarks { @@ -15,8 +14,6 @@ public sealed class MicrobenchmarkComparisonResult "PauseDurationMSec_MeanWhereIsBlockingGen2" }; - public MicrobenchmarkComparisonResult() { } - public MicrobenchmarkComparisonResult(IEnumerable baselines, IEnumerable comparands, bool includeTraces = true) { ComparisonResults = new(); @@ -24,11 +21,13 @@ public MicrobenchmarkComparisonResult(IEnumerable baseline { var baselineGCTraceMetricsCollection = baselines .Where(baseline => baseline != null) + .Where(baseline => baseline.GCTraceMetrics != null) .Select(baseline => baseline.GCTraceMetrics) .ToArray(); var comparandGCTraceMetricsCollection = comparands .Where(comparand => comparand != null) + .Where(comparand => comparand.GCTraceMetrics != null) .Select(comparand => comparand.GCTraceMetrics) .ToArray(); @@ -44,12 +43,27 @@ public MicrobenchmarkComparisonResult(IEnumerable baseline } } - BaselineRunName = baselines?.FirstOrDefault()?.Parent?.Name; - ComparandRunName = comparands?.FirstOrDefault()?.Parent?.Name; - MicrobenchmarkName = baselines?.FirstOrDefault()?.MicrobenchmarkName; + var firstBaseline = baselines?.FirstOrDefault(); + var firstComparand = comparands?.FirstOrDefault(); + + BaselineRunName = firstBaseline?.Parent?.Name ?? string.Empty; + ComparandRunName = firstComparand?.Parent?.Name ?? string.Empty; + MicrobenchmarkName = firstBaseline?.MicrobenchmarkName ?? string.Empty; Baselines = baselines ?? new List(); Comparands = comparands ?? new List(); + + OriginalBaselineOtherMetrics = Baselines + .Select(baseline => baseline.OtherMetrics) + .SelectMany(kvp => kvp) + .GroupBy(kvp => kvp.Key) + .ToDictionary(g => g.Key, g => g.Select(kvp => kvp.Value).ToArray()); + + OriginalComparandOtherMetrics = Comparands + .Select(comparand => comparand.OtherMetrics) + .SelectMany(kvp => kvp) + .GroupBy(kvp => kvp.Key) + .ToDictionary(g => g.Key, g => g.Select(kvp => kvp.Value).ToArray()); } public List ComparisonResults { get; set; } @@ -95,24 +109,35 @@ public double MeanDiffPerc{ public Dictionary OriginalComparandOtherMetrics { get; } = new(); public Dictionary OutliersFreeBaselineOtherMetrics => OriginalBaselineOtherMetrics .Select(kvp => (kvp.Key, API.Statistics.RemoveOutliers(kvp.Value).ToArray())) - .ToDictionary(); + .ToDictionary(x => x.Item1, x => x.Item2); public Dictionary OutliersFreeComparandOtherMetrics => OriginalComparandOtherMetrics .Select(kvp => (kvp.Key, API.Statistics.RemoveOutliers(kvp.Value).ToArray())) - .ToDictionary(); + .ToDictionary(x => x.Item1, x => x.Item2); public Dictionary AveragedBaselineOtherMetrics => OutliersFreeBaselineOtherMetrics .Select(kvp => (kvp.Key, API.GoodLinq.Average(kvp.Value, v => v))) - .ToDictionary(); + .ToDictionary(x => x.Item1, x => x.Item2); public Dictionary AveragedComparandOtherMetrics => OutliersFreeComparandOtherMetrics .Select(kvp => (kvp.Key, API.GoodLinq.Average(kvp.Value, v => v))) - .ToDictionary(); + .ToDictionary(x => x.Item1, x => x.Item2); - public Dictionary OtherMetricsDiff => OutliersFreeBaselineOtherMetrics - .Select(kvp => (kvp.Key, AveragedComparandOtherMetrics[kvp.Key] - AveragedBaselineOtherMetrics[kvp.Key])) - .ToDictionary(); + public Dictionary OtherMetricsDiff => AveragedBaselineOtherMetrics + .Select(kvp => + { + if (AveragedComparandOtherMetrics.ContainsKey(kvp.Key)) + { + return (kvp.Key, AveragedComparandOtherMetrics[kvp.Key] - AveragedBaselineOtherMetrics[kvp.Key]); + } + return (kvp.Key, double.NaN); + }) + .ToDictionary(x => x.Item1, x => x.Item2); - public Dictionary OtherMetricsDiffPerc => OutliersFreeBaselineOtherMetrics + public Dictionary OtherMetricsDiffPerc => AveragedBaselineOtherMetrics .Select(kvp => { + if (!AveragedComparandOtherMetrics.ContainsKey(kvp.Key)) + { + return (kvp.Key, double.NaN); + } if (AveragedBaselineOtherMetrics[kvp.Key] == 0) { if (AveragedComparandOtherMetrics[kvp.Key] == 0) @@ -124,8 +149,19 @@ public double MeanDiffPerc{ return (kvp.Key, double.NaN); } } - return (kvp.Key, OtherMetricsDiff[kvp.Key] / AveragedBaselineOtherMetrics[kvp.Key]); + + if (!OtherMetricsDiff.ContainsKey(kvp.Key)) + { + return (kvp.Key, double.NaN); + } + + if (OtherMetricsDiff[kvp.Key] == double.NaN) + { + return (kvp.Key, double.NaN); + } + + return (kvp.Key, 100 * OtherMetricsDiff[kvp.Key] / AveragedBaselineOtherMetrics[kvp.Key]); }) - .ToDictionary(); + .ToDictionary(x => x.Item1, x => x.Item2); } } diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkComparisonResults.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkComparisonResults.cs index 79d2f8eccee..4a20b557394 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkComparisonResults.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkComparisonResults.cs @@ -21,7 +21,7 @@ public MicrobenchmarkComparisonResults(string baselineName, string runName, IEnu public IEnumerable LargeImprovements => Ordered.Where(o => o.MeanDiffPerc < -20).OrderBy(g => g.MeanDiffPerc); public IEnumerable Regressions => Ordered.Where(o => o.MeanDiffPerc < 20 && o.MeanDiffPerc > 5); public IEnumerable Improvements => Ordered.Where(o => o.MeanDiffPerc > -20 && o.MeanDiffPerc < -5).OrderBy(g => g.MeanDiffPerc); - public IEnumerable StaleRegressions => Ordered.Where((o => o.MeanDiffPerc > 0 && o.MeanDiffPerc < 5)).OrderByDescending(g => g.MeanDiffPerc); + public IEnumerable StaleRegressions => Ordered.Where((o => o.MeanDiffPerc > 0 && o.MeanDiffPerc < 5)); public IEnumerable StaleImprovements => Ordered.Where((o => o.MeanDiffPerc < 0 && o.MeanDiffPerc > -5)).OrderBy(g => g.MeanDiffPerc); } } diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResult.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResult.cs index d64cdf9566f..f5fb0f898b3 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResult.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResult.cs @@ -6,7 +6,7 @@ namespace GC.Infrastructure.Core.Analysis.Microbenchmarks { public sealed class MicrobenchmarkResult { - public static readonly IReadOnlyDictionary> CustomStatisticsCalculationMap = new Dictionary>(StringComparer.OrdinalIgnoreCase) + public static readonly IReadOnlyDictionary> CustomStatisticsCalculationMap = new Dictionary>(StringComparer.OrdinalIgnoreCase) { { "number of iterations", (Statistics stats) => stats.N }, { "min", (Statistics stats) => stats.Min }, @@ -55,7 +55,7 @@ public MicrobenchmarkResult(string benchmarkFullName, { OtherMetrics = benchmark.Metrics .Where(metric => additionalReportMetrics.Contains(metric.Descriptor.Id)) - .ToDictionary(metric => metric.Descriptor.Id, metric => (double?)metric.Value); + .ToDictionary(metric => metric.Descriptor.Id, metric => metric.Value); } if (columns != null) @@ -63,18 +63,18 @@ public MicrobenchmarkResult(string benchmarkFullName, var customStatistics = columns .Where(column => CustomStatisticsCalculationMap.Keys.Contains(column)) .Select(column => (column, CustomStatisticsCalculationMap[column](benchmark.Statistics))) - .ToDictionary(); + .ToDictionary(x => x.column, x => x.Item2); - OtherMetrics = OtherMetrics.Concat(customStatistics).ToDictionary(); + OtherMetrics = OtherMetrics.Concat(customStatistics).ToDictionary(x => x.Key, x => x.Value); if (gcData != null) { var customGCData = columns .Where(column => CustomAggregateCalculationMap.Keys.Contains(column)) - .Select(column => (column, (double?)CustomAggregateCalculationMap[column](gcData))) - .ToDictionary(); + .Select(column => (column, CustomAggregateCalculationMap[column](gcData))) + .ToDictionary(x => x.column, x => x.Item2); - OtherMetrics = OtherMetrics.Concat(customGCData).ToDictionary(); + OtherMetrics = OtherMetrics.Concat(customGCData).ToDictionary(x => x.Key, x => x.Value); } } } @@ -82,7 +82,7 @@ public MicrobenchmarkResult(string benchmarkFullName, public Run Parent { get; set; } public Statistics Statistics { get; set; } public GCTraceMetrics? GCTraceMetrics { get; set; } - public Dictionary OtherMetrics { get; set; } = new(); + public Dictionary OtherMetrics { get; set; } = new(); public API.CPUProcessData? CPUData { get; set; } } } \ No newline at end of file diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResultComparison.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResultComparison.cs index 746c9589521..061a68db2eb 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResultComparison.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResultComparison.cs @@ -2,11 +2,13 @@ using GC.Infrastructure.Core.Configurations.Microbenchmarks; using Newtonsoft.Json; using System.Collections.Concurrent; +using System.Text.RegularExpressions; namespace GC.Infrastructure.Core.Analysis.Microbenchmarks { public static class MicrobenchmarkResultComparison { + private static readonly int _CPUCount = System.Environment.ProcessorCount; private static readonly Dictionary _benchmarkNameToTraceFilePatternMap = new() { { "ByteMark.BenchBitOps", "ByteMark.BenchBitOps"}, @@ -40,206 +42,285 @@ public static class MicrobenchmarkResultComparison { "System.Tests.Perf_GC.NewOperator_Array(length: 10000)", "System.Tests.Perf_GC_Char_.NewOperator_Array_length_10000_"}, }; - private static readonly ConcurrentDictionary>> _benchmarkFullNameToJsonForRun = new(); - - public static ConcurrentDictionary> MapBenchmarkFullNameToJsonForRun(string outputPathForRun) + public static ConcurrentBag> LoadBdnJsonResults(MicrobenchmarkConfiguration configuration) { - return _benchmarkFullNameToJsonForRun.GetOrAdd(outputPathForRun, path => + ConcurrentBag> bdnJsonResults = new(); + Parallel.ForEach(configuration.Runs, (run) => { - ConcurrentDictionary> benchmarkFullNameJsonMap = new(); - + string outputPathForRun = Path.Combine(configuration.Output.Path, run.Key); string[] jsonFiles = Directory.GetFiles(outputPathForRun, "*full.json", SearchOption.AllDirectories); - - Parallel.ForEach(jsonFiles, (jsonFile) => + Parallel.ForEach(jsonFiles, jsonPath => { - BdnJsonResult? results = JsonConvert.DeserializeObject(File.ReadAllText(jsonFile)); - string? fullName = results?.Benchmarks?.FirstOrDefault()?.FullName; - if (fullName != null) + run.Value.Name ??= run.Key; + BdnJsonResult? results = JsonConvert.DeserializeObject(File.ReadAllText(jsonPath)); + if (results != null) { - benchmarkFullNameJsonMap.GetOrAdd(fullName, _ => new ConcurrentBag()).Add(jsonFile); + bdnJsonResults.Add(new(run.Value, results, jsonPath)); } }); - - return benchmarkFullNameJsonMap; }); + return bdnJsonResults; } - public static ConcurrentDictionary MapJsonToTraceForSingleBenchmarkRun(string outputPathForRun, string benchmarkFullName) + // TODO: We should specify relationship between json files and trace files before running benchmarks instead of relying on file name patterns. + // This will make the mapping more robust and less prone to errors due to file naming. + public static Dictionary MapJsonToTrace(string outputPath, ConcurrentBag> bdnJsonResults) { - ConcurrentDictionary jsonTraceMap = new(); - - var benchmarkFullNameJsonMap = MapBenchmarkFullNameToJsonForRun(outputPathForRun); - - string[] jsonFiles = benchmarkFullNameJsonMap.GetValueOrDefault(benchmarkFullName, new()).ToArray(); - - Parallel.ForEach(jsonFiles, (jsonFile) => + Dictionary jsonToTrace = new(); + foreach (var groupForRun in bdnJsonResults.GroupBy(t => t.Item1)) { - // placeholder - jsonTraceMap[jsonFile] = ""; - }); - - string[] sortedJsonFiles = jsonTraceMap.Keys - .OrderBy(jsonFile => Path.GetFileName(Path.GetDirectoryName(jsonFile))) - .ToArray(); + var run = groupForRun.Key; - if (!_benchmarkNameToTraceFilePatternMap.Keys.Contains(benchmarkFullName)) - { - throw new KeyNotFoundException("No trace file pattern found for benchmark: " + benchmarkFullName); - } - string traceFileNameTemplate = _benchmarkNameToTraceFilePatternMap[benchmarkFullName]; + // GroupBy(t => t.Item2.Benchmarks.First().FullName) is not a bug: + // In single *full.json, multiple benchmarks stands for multiple input parameter combinations for the same benchmark + // If trace collection is enabled, process data for all those parameter combinations will be in the same trace file + foreach (var g in groupForRun.GroupBy(t => t.Item2.Benchmarks.First().FullName)) + { + var benchmarkName = g.Key; + var sortedJsonFiles = GoodLinq.Select(g, t => t.Item3) + .OrderBy(jsonFile => Path.GetFileName(Path.GetDirectoryName(jsonFile))) + .ToArray(); - string[] sortedTraceFiles = Enumerable.Where(Directory.GetFiles(outputPathForRun, "*.etl.zip", SearchOption.TopDirectoryOnly), traceFile => - Path.GetFileName(traceFile).ToLower().Contains(traceFileNameTemplate.ToLower())) - .OrderBy(traceFile => traceFile) - .ToArray(); + if (!_benchmarkNameToTraceFilePatternMap.ContainsKey(benchmarkName)) + { + throw new InvalidOperationException($"Benchmark name {benchmarkName} does not have a corresponding trace file pattern in the map."); + } + var traceFileNameTemplate = _benchmarkNameToTraceFilePatternMap[benchmarkName]; + string outputPathForRun = Path.Combine(outputPath, run.Name!); + var sortedTraceFiles = Directory.GetFiles(outputPathForRun, $"{traceFileNameTemplate}*.etl.zip", SearchOption.TopDirectoryOnly) + .OrderBy(traceFile => + { + var match = Regex.Match(Path.GetFileName(traceFile), @"_(\d+)\.etl\.zip$"); + return match.Success ? int.Parse(match.Groups[1].Value) : 0; + }) + .ToArray(); - if (sortedJsonFiles.Length != sortedTraceFiles.Length) - { - throw new InvalidOperationException( - $"The number of JSON files ({sortedJsonFiles.Length}) does not match the number of trace files ({sortedTraceFiles.Length}) for benchmark: {benchmarkFullName}"); - } + if (sortedJsonFiles.Length != sortedTraceFiles.Length) + { + throw new InvalidOperationException( + $"The number of JSON files ({sortedJsonFiles.Length}) does not match the number of trace files ({sortedTraceFiles.Length}) for benchmark: {benchmarkName}"); + } - for (int idx = 0; idx < sortedJsonFiles.Length; idx++) - { - jsonTraceMap[sortedJsonFiles[idx]] = sortedTraceFiles[idx]; + for (int i = 0; i < sortedJsonFiles.Length; i++) + { + jsonToTrace[sortedJsonFiles[i]] = sortedTraceFiles[i]; + } + } } - return jsonTraceMap; + return jsonToTrace; } - public static IReadOnlyDictionary> AnalyzeMicrobenchmarkResultsForSingleBenchmark(MicrobenchmarkConfiguration configuration, string benchmarkFullName, bool excludeTraces = false) + public static ConcurrentBag + AnalyzeMicrobenchmarkResults(MicrobenchmarkConfiguration configuration, + ConcurrentBag> bdnJsonResults, + bool excludeTraces = false) { - ConcurrentDictionary> runsToResults = new(); + ConcurrentBag microbenchmarkResults = new(); - Parallel.ForEach(configuration.Runs, (run) => + Dictionary jsonToTraceMap = new(); + if ((!excludeTraces) && (configuration.TraceConfigurations?.Type ?? "none") != "none") { - string outputPathForRun = Path.Combine(configuration.Output.Path, run.Key); - run.Value.Name ??= run.Key; + jsonToTraceMap = MapJsonToTrace(configuration.Output.Path, bdnJsonResults); + } + + ParallelOptions options = new() + { + MaxDegreeOfParallelism = _CPUCount * 2 + }; - var benchmarkToJsonMapForRun = MapBenchmarkFullNameToJsonForRun(outputPathForRun); - var jsonFiles = benchmarkToJsonMapForRun.GetValueOrDefault(benchmarkFullName, new()); + int count = 0; + object _lock = new(); - runsToResults[run.Value] = runsToResults.GetValueOrDefault(run.Value, new()); + Parallel.ForEach(bdnJsonResults, options, t => + { + var run = t.Item1; + var bdnJsonResult = t.Item2; + var jsonPath = t.Item3; - Parallel.ForEach(jsonFiles, jsonPath => + List? benchmarks = bdnJsonResult?.Benchmarks; + + if (benchmarks == null) { - BdnJsonResult? results = JsonConvert.DeserializeObject(File.ReadAllText(jsonPath)); + return; + } - List? benchmarks = results?.Benchmarks; + if ((!excludeTraces) && configuration.TraceConfigurations?.Type != "none") + { + string outputPathForRun = Path.Combine(configuration.Output.Path, run.Name!); - if (benchmarks == null) + if (!jsonToTraceMap.TryGetValue(jsonPath, out string? tracePath) || string.IsNullOrWhiteSpace(tracePath)) { - return; + throw new InvalidOperationException($"Trace collection is enabled, but no trace path mapping was found for benchmark result '{jsonPath}'."); } - - foreach (var benchmark in benchmarks) + using (var analyzer = AnalyzerManager.GetAnalyzer(tracePath)) { - Statistics statistics = benchmark.Statistics; + List allPertinentProcesses = analyzer.GetProcessGCData("dotnet"); + List corerunProcesses = analyzer.GetProcessGCData("corerun"); + allPertinentProcesses.AddRange(corerunProcesses); - MicrobenchmarkResult? microbenchmarkResult = null; - if ((!excludeTraces) && configuration.TraceConfigurations.Type != "none") + foreach (var benchmark in benchmarks) { - var jsonTraceMap = MapJsonToTraceForSingleBenchmarkRun(outputPathForRun, benchmarkFullName); - string tracePath = jsonTraceMap.GetValueOrDefault(jsonPath, ""); + Statistics statistics = benchmark.Statistics; + var benchmarkFullName = benchmark.FullName; - using (var analyzer = AnalyzerManager.GetAnalyzer(tracePath)) + MicrobenchmarkResult? microbenchmarkResult = null; + GCProcessData? benchmarkGCData = null; + foreach (var process in allPertinentProcesses) + { + string commandLine = process.CommandLine.Replace("\"", "").Replace("\\", ""); + string runCleaned = benchmark.FullName.Replace("\"", "").Replace("\\", ""); + if (commandLine.Contains(runCleaned) && commandLine.Contains("--benchmarkName")) + { + benchmarkGCData = process; + break; + } + } + if (benchmarkGCData != null) { - List allPertinentProcesses = analyzer.GetProcessGCData("dotnet"); - List corerunProcesses = analyzer.GetProcessGCData("corerun"); - allPertinentProcesses.AddRange(corerunProcesses); + int processID = benchmarkGCData.ProcessID; - GCProcessData? benchmarkGCData = null; - foreach (var process in allPertinentProcesses) + /* + TODO: THIS NEEDS TO BE ADDED BACK. + if (configuration.Output.cpu_columns != null && configuration.Output.cpu_columns.Count > 0) { - string commandLine = process.CommandLine.Replace("\"", "").Replace("\\", ""); - string runCleaned = benchmark.FullName.Replace("\"", "").Replace("\\", ""); - if (commandLine.Contains(runCleaned) && commandLine.Contains("--benchmarkName")) - { - benchmarkGCData = process; - break; - } + // TODO: Add parameterize. + benchmark.Value.GCData.Parent.AddCPUAnalysis(yamlPath: @"C:\Users\musharm\source\repos\GC.Analysis.API\GC.Analysis.API\CPUAnalysis\DefaultMethods.yaml", + symbolLogFile: Path.Combine(configuration.Output.Path, run.Key, Guid.NewGuid() + ".txt"), + symbolPath: Path.Combine(configuration.Output.Path, run.Key)); + var d1 = benchmark.Value.GCData.Parent.CPUAnalyzer.GetCPUDataForProcessName("dotnet"); + d1.AddRange(benchmark.Value.GCData.Parent.CPUAnalyzer.GetCPUDataForProcessName("corerun")); + benchmark.Value.CPUData = d1.FirstOrDefault(p => p.ProcessID == processID); } - if (benchmarkGCData != null) + */ + + if (benchmarkGCData.GCs.Count > 0) { - int processID = benchmarkGCData.ProcessID; - - /* - TODO: THIS NEEDS TO BE ADDED BACK. - if (configuration.Output.cpu_columns != null && configuration.Output.cpu_columns.Count > 0) - { - // TODO: Add parameterize. - benchmark.Value.GCData.Parent.AddCPUAnalysis(yamlPath: @"C:\Users\musharm\source\repos\GC.Analysis.API\GC.Analysis.API\CPUAnalysis\DefaultMethods.yaml", - symbolLogFile: Path.Combine(configuration.Output.Path, run.Key, Guid.NewGuid() + ".txt"), - symbolPath: Path.Combine(configuration.Output.Path, run.Key)); - var d1 = benchmark.Value.GCData.Parent.CPUAnalyzer.GetCPUDataForProcessName("dotnet"); - d1.AddRange(benchmark.Value.GCData.Parent.CPUAnalyzer.GetCPUDataForProcessName("corerun")); - benchmark.Value.CPUData = d1.FirstOrDefault(p => p.ProcessID == processID); - } - */ microbenchmarkResult = new(benchmarkFullName, - run.Value, + run, benchmark, gcData: benchmarkGCData, - gcTraceMetrics: new GCTraceMetrics(benchmarkGCData, tracePath, benchmark.FullName), + gcTraceMetrics: new GCTraceMetrics(benchmarkGCData, run.Name!, benchmark.FullName), + additionalReportMetrics: configuration.Output.additional_report_metrics, + cpuColumns: configuration.Output.cpu_columns, + columns: configuration.Output.Columns); + } + else + { + microbenchmarkResult = new(benchmarkFullName, + run, + benchmark, additionalReportMetrics: configuration.Output.additional_report_metrics, cpuColumns: configuration.Output.cpu_columns, columns: configuration.Output.Columns); } } - System.GC.Collect(2); - } - else - { - microbenchmarkResult = new(benchmarkFullName, - run.Value, - benchmark, - additionalReportMetrics: configuration.Output.additional_report_metrics, - cpuColumns: configuration.Output.cpu_columns, - columns: configuration.Output.Columns); - } - runsToResults[run.Value].Add(microbenchmarkResult!); + else + { + microbenchmarkResult = new(benchmarkFullName, + run, + benchmark, + additionalReportMetrics: configuration.Output.additional_report_metrics, + cpuColumns: configuration.Output.cpu_columns, + columns: configuration.Output.Columns); + } + + microbenchmarkResults.Add(microbenchmarkResult!); + } } - }); + } + else + { + foreach (var benchmark in benchmarks) + { + Statistics statistics = benchmark.Statistics; + var benchmarkFullName = benchmark.FullName; + + MicrobenchmarkResult? microbenchmarkResult = null; + microbenchmarkResult = new(benchmarkFullName, + run, + benchmark, + additionalReportMetrics: configuration.Output.additional_report_metrics, + cpuColumns: configuration.Output.cpu_columns, + columns: configuration.Output.Columns); + microbenchmarkResults.Add(microbenchmarkResult!); + } + } + lock (_lock) + { + count = count + 1; + Console.Write($"\r{count}/{bdnJsonResults.Count} BDN results analyzed."); + } }); - return runsToResults; + Console.WriteLine(); + return microbenchmarkResults; } - public static List CompareMicrobenchmarkResultForBenchmark(MicrobenchmarkConfiguration configuration, string benchmarkFullName, bool excludeTraces = false) + public static List CompareMicrobenchmarkResults(MicrobenchmarkConfiguration configuration, IEnumerable microbenchmarkResults, bool excludeTraces = false) { - bool includeTraces = (!excludeTraces) && configuration.TraceConfigurations.Type != "none"; - IReadOnlyDictionary> runResults = AnalyzeMicrobenchmarkResultsForSingleBenchmark(configuration, benchmarkFullName, excludeTraces); + bool includeTraces = (!excludeTraces) && (configuration.TraceConfigurations.Type != "none"); + var microbenchmarkResultsGroupedByBenchmarkName = microbenchmarkResults + .GroupBy(microbenchmarkResult => microbenchmarkResult.MicrobenchmarkName); + List comparisonResults = new(); - if (configuration.Output.run_comparisons != null) + object _lock = new(); + ParallelOptions options = new() { - foreach (var comparison in configuration.Output.run_comparisons) + MaxDegreeOfParallelism = _CPUCount + }; + Parallel.ForEach(microbenchmarkResultsGroupedByBenchmarkName, options, microbenchmarkResultsGroup => + { + if (configuration.Output.run_comparisons != null) { - string[] breakup = comparison.Split(",", StringSplitOptions.TrimEntries); - string baselineName = breakup[0]; - string runName = breakup[1]; + foreach (var comparison in configuration.Output.run_comparisons) + { + string[] breakup = comparison.Split(",", StringSplitOptions.TrimEntries); + string baselineName = breakup[0]; + string runName = breakup[1]; + + var baselineMicrobenchmarkResults = GoodLinq.Where(microbenchmarkResultsGroup, r => r.Parent.Name == baselineName); + var comparandMicrobenchmarkResults = GoodLinq.Where(microbenchmarkResultsGroup, r => r.Parent.Name == runName); - var baselineRuns = GoodLinq.Where(runResults.Keys, r => r.Name == baselineName); - var comparandRuns = GoodLinq.Where(runResults.Keys, r => r.Name == runName); + if (baselineMicrobenchmarkResults == null || comparandMicrobenchmarkResults ==null) + { + continue; + } - var baselineMicrobenchmarkResults = GoodLinq.Select(baselineRuns, b => runResults[b]).SelectMany(r => r); - var comparandMicrobenchmarkResults = GoodLinq.Select(comparandRuns, c => runResults[c]).SelectMany(r => r); + if (baselineMicrobenchmarkResults.Count == 0 || comparandMicrobenchmarkResults.Count == 0) + { + continue; + } - comparisonResults.Add(new(baselineMicrobenchmarkResults, comparandMicrobenchmarkResults, includeTraces)); + lock (_lock) + { + comparisonResults.Add(new(baselineMicrobenchmarkResults, comparandMicrobenchmarkResults, includeTraces)); + } + } } - } - // Default case where the run comparisons aren't specified. - else - { - var baselineRuns = GoodLinq.Where(runResults.Keys, r => r.is_baseline); - var comparandRuns = GoodLinq.Where(runResults.Keys, r => !r.is_baseline); + // Default case where the run comparisons aren't specified. + else + { + var baselineMicrobenchmarkResults = GoodLinq.Where(microbenchmarkResultsGroup, r => r.Parent.is_baseline); + var comparandMicrobenchmarkResults = GoodLinq.Where(microbenchmarkResultsGroup, r => !r.Parent.is_baseline); - var baselineMicrobenchmarkResults = GoodLinq.Select(baselineRuns, b => runResults[b]).SelectMany(r => r); - var comparandMicrobenchmarkResults = GoodLinq.Select(comparandRuns, c => runResults[c]).SelectMany(r => r); + if (baselineMicrobenchmarkResults == null || comparandMicrobenchmarkResults == null) + { + return; + } - comparisonResults.Add(new(baselineMicrobenchmarkResults, comparandMicrobenchmarkResults, includeTraces)); - } + if (baselineMicrobenchmarkResults.Count == 0 || comparandMicrobenchmarkResults.Count == 0) + { + return; + } + lock (_lock) + { + comparisonResults.Add(new(baselineMicrobenchmarkResults, comparandMicrobenchmarkResults, includeTraces)); + } + } + }); return comparisonResults; } diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/MarkdownReportBuilder.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/MarkdownReportBuilder.cs index ce0415c70e0..61cfd5dcadf 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/MarkdownReportBuilder.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/MarkdownReportBuilder.cs @@ -103,10 +103,12 @@ public static void AddReproSection(this StreamWriter sw, Dictionary kvp.Value.CommandlineArgs); + foreach (var kvp in distinctCommandlineArgs) { - sw.WriteLine($"### {p.Key}"); - sw.WriteLine($"```{p.Value.CommandlineArgs}```\n"); + sw.WriteLine($"### {kvp.Key}"); + sw.WriteLine($"```{kvp.Value.CommandlineArgs}```\n"); } sw.WriteLine(); diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Markdown.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Markdown.cs index 73f25eab143..fcc488c050a 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Markdown.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Markdown.cs @@ -1,16 +1,12 @@ -using API = GC.Analysis.API; -using GC.Infrastructure.Core.Analysis; +using GC.Infrastructure.Core.Analysis; using GC.Infrastructure.Core.Analysis.Microbenchmarks; -using GC.Analysis.API; using GC.Infrastructure.Core.Configurations.Microbenchmarks; +using API = GC.Analysis.API; namespace GC.Infrastructure.Core.Presentation.Microbenchmarks { public static class Markdown { - private const string baseTableString = "| Benchmark Name | Baseline | Comparand | Baseline Mean Duration (MSec) | Comparand Mean Duration (MSec) | Δ Mean Duration (MSec) | Δ% Mean Duration |"; - private const string baseTableRows = "| --- | --- | -- | --- | --- | --- | --- | "; - public static void GenerateTable(MicrobenchmarkConfiguration configuration, IReadOnlyList comparisonResultsCollection, Dictionary executionDetails, string path) { using (StreamWriter sw = new StreamWriter(path)) @@ -18,15 +14,15 @@ public static void GenerateTable(MicrobenchmarkConfiguration configuration, IRea // Create summary. sw.WriteLine("# Summary"); - string header = $"| Criteria | {string.Join("|", GoodLinq.Select(comparisonResultsCollection, s => $"[{s.BaselineName} {s.RunName}]({s.MarkdownIdentifier})"))}|"; + string header = $"| Criteria | {string.Join("|", API.GoodLinq.Select(comparisonResultsCollection, s => $"[{s.BaselineName} {s.RunName}]({s.MarkdownIdentifier})"))}|"; sw.WriteLine(header); sw.WriteLine($"| ----- | {string.Join("|", Enumerable.Repeat(" ----- ", comparisonResultsCollection.Count))} |"); - sw.WriteLine($"| Large Regressions (>20%) | {GoodLinq.Sum(comparisonResultsCollection, s => s.LargeRegressions.Count())}|"); - sw.WriteLine($"| Regressions (5% - 20%) | {GoodLinq.Sum(comparisonResultsCollection, s => s.Regressions.Count())}|"); - sw.WriteLine($"| Stale Regressions (0% - 5%) | {GoodLinq.Sum(comparisonResultsCollection, s => s.StaleRegressions.Count())}|"); - sw.WriteLine($"| Stale Improvements (0% - 5%) | {GoodLinq.Sum(comparisonResultsCollection, s => s.StaleImprovements.Count())}|"); - sw.WriteLine($"| Improvements (5% - 20%) | {GoodLinq.Sum(comparisonResultsCollection, s => s.Improvements.Count())}|"); - sw.WriteLine($"| Large Improvements (>20%) | {GoodLinq.Sum(comparisonResultsCollection, s => s.LargeImprovements.Count())}|"); + sw.WriteLine($"| Large Regressions (>20%) | {API.GoodLinq.Sum(comparisonResultsCollection, s => s.LargeRegressions.Count())}|"); + sw.WriteLine($"| Regressions (5% - 20%) | {API.GoodLinq.Sum(comparisonResultsCollection, s => s.Regressions.Count())}|"); + sw.WriteLine($"| Stale Regressions (0% - 5%) | {API.GoodLinq.Sum(comparisonResultsCollection, s => s.StaleRegressions.Count())}|"); + sw.WriteLine($"| Stale Improvements (0% - 5%) | {API.GoodLinq.Sum(comparisonResultsCollection, s => s.StaleImprovements.Count())}|"); + sw.WriteLine($"| Improvements (5% - 20%) | {API.GoodLinq.Sum(comparisonResultsCollection, s => s.Improvements.Count())}|"); + sw.WriteLine($"| Large Improvements (>20%) | {API.GoodLinq.Sum(comparisonResultsCollection, s => s.LargeImprovements.Count())}|"); sw.WriteLine($"| Total | {comparisonResultsCollection.Count} |"); sw.WriteLine("\n"); @@ -89,46 +85,65 @@ internal static void AddDetailsOfSingleComparison(this StreamWriter sw, Microben foreach (var metric in configuration.Output.additional_report_metrics) { sw.WriteLine($"## Comparison by {metric}"); - var ordered = comparisonResult.Comparisons.OrderByDescending(c => c.OtherMetricsDiffPerc[metric]); + var ordered = comparisonResult.Comparisons + .Where(c => c.OtherMetricsDiffPerc.ContainsKey(metric)) + .OrderByDescending(c => c.OtherMetricsDiffPerc[metric]); // Large Regressions - sw.WriteLine($"### Large Regressions (>20%): {comparisonResult.LargeRegressions.Count()} \n"); - sw.AddTableForSingleCriteria(configuration, GoodLinq.Where(ordered, o => o.OtherMetricsDiffPerc[metric] > 0.2)); + var largeRegression = API.GoodLinq.Where(ordered, o => o.OtherMetricsDiffPerc[metric] >= 20); + sw.WriteLine($"### Large Regressions (>20%): {largeRegression.Count()} \n"); + sw.AddTableForSingleCriteria(configuration, largeRegression, metric); sw.WriteLine("\n"); // Large Improvements - sw.WriteLine($"### Large Improvements (>20%): {comparisonResult.LargeImprovements.Count()} \n"); - sw.AddTableForSingleCriteria(configuration, GoodLinq.Where(ordered, o => o.OtherMetricsDiffPerc[metric] < -0.2)); + var largeImprovements = API.GoodLinq.Where(ordered, o => o.OtherMetricsDiffPerc[metric] <= -20); + largeImprovements.Reverse(); + sw.WriteLine($"### Large Improvements (>20%): {largeImprovements.Count()} \n"); + sw.AddTableForSingleCriteria(configuration, largeImprovements, metric); sw.WriteLine("\n"); // Regressions - sw.WriteLine($"### Regressions (5% - 20%): {comparisonResult.Regressions.Count()} \n"); - sw.AddTableForSingleCriteria(configuration, GoodLinq.Where(ordered, o => o.OtherMetricsDiffPerc[metric] > 0.05 && o.OtherMetricsDiffPerc[metric] < 0.2)); + var regressions = API.GoodLinq.Where(ordered, o => o.OtherMetricsDiffPerc[metric] >= 5 && o.OtherMetricsDiffPerc[metric] < 20); + sw.WriteLine($"### Regressions (5% - 20%): {regressions.Count()} \n"); + sw.AddTableForSingleCriteria(configuration, regressions, metric); sw.WriteLine("\n"); // Improvements - sw.WriteLine($"### Improvements (5% - 20%): {comparisonResult.Improvements.Count()} \n"); - sw.AddTableForSingleCriteria(configuration, GoodLinq.Where(ordered, o => o.OtherMetricsDiffPerc[metric] > 0.05 && o.OtherMetricsDiffPerc[metric] < 0.2)); + var improvements = API.GoodLinq.Where(ordered, o => o.OtherMetricsDiffPerc[metric] <= -5 && o.OtherMetricsDiffPerc[metric] > -20); + improvements.Reverse(); + sw.WriteLine($"### Improvements (5% - 20%): {improvements.Count()} \n"); + sw.AddTableForSingleCriteria(configuration, improvements, metric); sw.WriteLine("\n"); // Stale Regressions - sw.WriteLine($"### Stale Regressions (Same or percent difference within 5% margin): {comparisonResult.StaleRegressions.Count()} \n"); - sw.AddTableForSingleCriteria(configuration, GoodLinq.Where(ordered, o => o.OtherMetricsDiffPerc[metric] < 0.05 && o.OtherMetricsDiffPerc[metric] >= 0.0)); + var staleRegressions = API.GoodLinq.Where(ordered, o => o.OtherMetricsDiffPerc[metric] >= 0.0 && o.OtherMetricsDiffPerc[metric] < 5); + sw.WriteLine($"### Stale Regressions (Same or percent difference within 5% margin): {staleRegressions.Count()} \n"); + sw.AddTableForSingleCriteria(configuration, staleRegressions, metric); sw.WriteLine("\n"); // Stale Improvements - sw.WriteLine($"### Stale Improvements (Same or percent difference within 5% margin): {comparisonResult.StaleImprovements.Count()} \n"); - sw.AddTableForSingleCriteria(configuration, GoodLinq.Where(ordered, o => o.OtherMetricsDiffPerc[metric] > -0.05 && o.OtherMetricsDiffPerc[metric] < 0.0)); + var staleImprovements = API.GoodLinq.Where(ordered, o => o.OtherMetricsDiffPerc[metric] > -5 && o.OtherMetricsDiffPerc[metric] <= 0.0); + staleImprovements.Reverse(); + sw.WriteLine($"### Stale Improvements (Same or percent difference within 5% margin): {staleImprovements.Count()} \n"); + sw.AddTableForSingleCriteria(configuration, staleImprovements, metric); sw.WriteLine("\n"); } } } - internal static void AddTableForSingleCriteria(this StreamWriter sw, MicrobenchmarkConfiguration configuration, IEnumerable comparisons) + internal static void AddTableForSingleCriteria(this StreamWriter sw, MicrobenchmarkConfiguration configuration, IEnumerable comparisons, string? metricName = null) { // Check if all comparisons have traces. - string tableHeader0 = baseTableString; - string tableHeader1 = baseTableRows; + string tableHeader0 = ""; + if (!string.IsNullOrEmpty(metricName)) + { + tableHeader0 = $"| Benchmark Name | Baseline | Comparand | Baseline {metricName} | Comparand {metricName} | Δ {metricName} | Δ% {metricName} |"; + } + else + { + tableHeader0 = "| Benchmark Name | Baseline | Comparand | Baseline Mean Duration (MSec) | Comparand Mean Duration (MSec) | Δ Mean Duration (MSec) | Δ% Mean Duration |"; + } + string tableHeader1 = "| --- | --- | -- | --- | --- | --- | --- | "; if (configuration.Output.Columns != null) { @@ -139,14 +154,15 @@ internal static void AddTableForSingleCriteria(this StreamWriter sw, Microbenchm } } - if (configuration.Output.cpu_columns != null) - { - foreach (var column in configuration.Output.cpu_columns) - { - tableHeader0 += $"Baseline {column} | Comparand {column} | Δ {column} | Δ% {column} |"; - tableHeader1 += "--- | --- | --- | --- |"; - } - } + // TODO: Add CPU columns if needed in the future. + //if (configuration.Output.cpu_columns != null) + //{ + // foreach (var column in configuration.Output.cpu_columns) + // { + // tableHeader0 += $"Baseline {column} | Comparand {column} | Δ {column} | Δ% {column} |"; + // tableHeader1 += "--- | --- | --- | --- |"; + // } + //} sw.WriteLine(tableHeader0); sw.WriteLine(tableHeader1); @@ -156,7 +172,16 @@ internal static void AddTableForSingleCriteria(this StreamWriter sw, Microbenchm try { string benchmarkName = lr.MicrobenchmarkName.Replace("<", "\\<").Replace(">", "\\>"); - var baseRow = $"| {benchmarkName} | {lr.BaselineRunName} | {lr.ComparandRunName} | {Math.Round(lr.AveragedBaselineMeanValue, 2)} | {Math.Round(lr.AveragedComparandMeanValue, 2)} | {Math.Round(lr.MeanDiff, 2)}| {Math.Round(lr.MeanDiffPerc, 2)}|"; + string baseRow = ""; + + if (!String.IsNullOrEmpty(metricName)) + { + baseRow = $"| {benchmarkName} | {lr.BaselineRunName} | {lr.ComparandRunName} | {Math.Round(lr.AveragedBaselineOtherMetrics[metricName], 2)} | {Math.Round(lr.AveragedComparandOtherMetrics[metricName], 2)} | {Math.Round(lr.OtherMetricsDiff[metricName], 2)}| {Math.Round(lr.OtherMetricsDiffPerc[metricName], 2)}|"; + } + else + { + baseRow = $"| {benchmarkName} | {lr.BaselineRunName} | {lr.ComparandRunName} | {Math.Round(lr.AveragedBaselineMeanValue, 2)} | {Math.Round(lr.AveragedComparandMeanValue, 2)} | {Math.Round(lr.MeanDiff, 2)}| {Math.Round(lr.MeanDiffPerc, 2)}|"; + } if (configuration.Output.Columns != null) { @@ -177,6 +202,7 @@ internal static void AddTableForSingleCriteria(this StreamWriter sw, Microbenchm } } + // TODO: Add CPU columns if needed in the future. //if (configuration.Output.cpu_columns != null) //{ // foreach (var column in configuration.Output.cpu_columns) diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Presentation.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Presentation.cs deleted file mode 100644 index 8d879a48ecd..00000000000 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Presentation.cs +++ /dev/null @@ -1,27 +0,0 @@ -using GC.Infrastructure.Core.Analysis; -using GC.Infrastructure.Core.Analysis.Microbenchmarks; -using GC.Infrastructure.Core.Configurations.Microbenchmarks; - -namespace GC.Infrastructure.Core.Presentation.Microbenchmarks -{ - public static class Presentation - { - public static void Present(MicrobenchmarkConfiguration configuration, List comparisonResultsGroupedByName, Dictionary executionDetails) - { - foreach (var format in configuration.Output.Formats) - { - if (format == "markdown") - { - Markdown.GenerateTable(configuration, comparisonResultsGroupedByName, executionDetails, Path.Combine(configuration.Output.Path, "Results.md")); - continue; - } - - if (format == "json") - { - Json.Generate(configuration, comparisonResultsGroupedByName, Path.Combine(configuration.Output.Path, "Results.json")); - continue; - } - } - } - } -} diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/Microbenchmark/MicrobenchmarkAnalyzeCommand.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/Microbenchmark/MicrobenchmarkAnalyzeCommand.cs index b07cde1f428..b179894d524 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/Microbenchmark/MicrobenchmarkAnalyzeCommand.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/Microbenchmark/MicrobenchmarkAnalyzeCommand.cs @@ -1,4 +1,5 @@ -using GC.Infrastructure.Core.Analysis.Microbenchmarks; +using GC.Infrastructure.Core.Analysis; +using GC.Infrastructure.Core.Analysis.Microbenchmarks; using GC.Infrastructure.Core.Configurations; using GC.Infrastructure.Core.Configurations.Microbenchmarks; using GC.Infrastructure.Core.Presentation.Microbenchmarks; @@ -23,41 +24,45 @@ public override int Execute([NotNull] CommandContext context, [NotNull] Microben ConfigurationChecker.VerifyFile(settings.ConfigurationPath, nameof(MicrobenchmarkAnalyzeCommand)); MicrobenchmarkConfiguration configuration = MicrobenchmarkConfigurationParser.Parse(settings.ConfigurationPath); - var comparisonResultsGroupedName = ExecuteAnalysis(configuration); + var comparisonResultsGroupedByName = ExecuteAnalysis(configuration); - Presentation.Present(configuration, comparisonResultsGroupedName, new()); // Execution details aren't available for the analysis-only mode. + Present(configuration, comparisonResultsGroupedByName, new()); // Execution details aren't available for the analysis-only mode. return 0; } public static List ExecuteAnalysis(MicrobenchmarkConfiguration configuration) { - Run? run = configuration.Runs.Values.FirstOrDefault(); - if (run == null) - { - throw new InvalidOperationException("No runs found in the configuration."); - } - string outputPathForRun = Path.Combine(configuration.Output.Path, run.Name); - var benchmarkFullNameJsonMap = MicrobenchmarkResultComparison.MapBenchmarkFullNameToJsonForRun(outputPathForRun); - List comparisonResultForAllBenchmarks = new(); - - ParallelOptions options = new ParallelOptions - { - MaxDegreeOfParallelism = System.Environment.ProcessorCount - }; - - object _lock = new(); + var bdnJsonResults = MicrobenchmarkResultComparison.LoadBdnJsonResults(configuration); + AnsiConsole.MarkupLine($"[bold green] ({DateTime.Now}) {bdnJsonResults.Count} BDN results loaded.[/]"); + var microbenchmarkResults = MicrobenchmarkResultComparison.AnalyzeMicrobenchmarkResults(configuration, bdnJsonResults); + AnsiConsole.MarkupLine($"[bold green] ({DateTime.Now}) Analysis completed.[/]"); + var comparisonResults = MicrobenchmarkResultComparison.CompareMicrobenchmarkResults(configuration, microbenchmarkResults); + + return MicrobenchmarkResultComparison.GroupComparisonResultsByName(configuration, comparisonResults); + } - Parallel.ForEach(benchmarkFullNameJsonMap.Keys, options, benchmarkFullName => + public static void Present(MicrobenchmarkConfiguration configuration, + List comparisonResultsGroupedByName, + Dictionary executionDetails) + { + foreach (var format in configuration.Output.Formats) { - List comparisonResultsForBenchmark = MicrobenchmarkResultComparison.CompareMicrobenchmarkResultForBenchmark(configuration, benchmarkFullName); - AnsiConsole.Markup($"[bold green] ({DateTime.Now}) Analysis For Microbenchmarks: {benchmarkFullName} completed. [/]\n"); - lock (_lock) + if (format == "markdown") { - comparisonResultForAllBenchmarks.AddRange(comparisonResultsForBenchmark); + string outputPath = Path.Combine(configuration.Output.Path, "Results.md"); + Markdown.GenerateTable(configuration, comparisonResultsGroupedByName, executionDetails, outputPath); + AnsiConsole.MarkupLine($"[bold green] ({DateTime.Now}) Results written to {Markup.Escape(outputPath)}.[/]"); + continue; } - }); - return MicrobenchmarkResultComparison.GroupComparisonResultsByName(configuration, comparisonResultForAllBenchmarks); + if (format == "json") + { + string outputPath = Path.Combine(configuration.Output.Path, "Results.json"); + Json.Generate(configuration, comparisonResultsGroupedByName, outputPath); + AnsiConsole.MarkupLine($"[bold green] ({DateTime.Now}) Results written to {Markup.Escape(outputPath)}.[/]"); + continue; + } + } } } } diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/Microbenchmark/MicrobenchmarkCommand.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/Microbenchmark/MicrobenchmarkCommand.cs index 74cb749e680..e411bbd2f65 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/Microbenchmark/MicrobenchmarkCommand.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/Microbenchmark/MicrobenchmarkCommand.cs @@ -4,13 +4,11 @@ using GC.Infrastructure.Core.CommandBuilders; using GC.Infrastructure.Core.Configurations; using GC.Infrastructure.Core.Configurations.Microbenchmarks; -using GC.Infrastructure.Core.Presentation.Microbenchmarks; using GC.Infrastructure.Core.TraceCollection; using Newtonsoft.Json; using Spectre.Console; using Spectre.Console.Cli; using System.ComponentModel; -using System.Configuration; using System.Diagnostics; using System.Diagnostics.CodeAnalysis; using System.Text; @@ -202,9 +200,9 @@ public static MicrobenchmarkOutputResults RunMicrobenchmarks(MicrobenchmarkConfi var comparisonResultsGroupedName = MicrobenchmarkAnalyzeCommand.ExecuteAnalysis(configuration); - Presentation.Present(configuration, comparisonResultsGroupedName, executionDetails); // Execution details aren't available for the analysis-only mode. + MicrobenchmarkAnalyzeCommand.Present(configuration, comparisonResultsGroupedName, executionDetails); // Execution details aren't available for the analysis-only mode. Directory.SetCurrentDirectory(currentDirectory); - AnsiConsole.Markup($"[bold green] ({DateTime.Now}) Wrote Microbechmark Results to: {Markup.Escape(Path.Combine(configuration.Output.Path, "Results.md"))} [/]"); + AnsiConsole.Markup($"[bold green] ({DateTime.Now}) Wrote Microbenchmark Results to: {Markup.Escape(Path.Combine(configuration.Output.Path, "Results.md"))} [/]"); return new MicrobenchmarkOutputResults(executionDetails, comparisonResultsGroupedName); } } diff --git a/src/benchmarks/micro/MicroBenchmarks.csproj b/src/benchmarks/micro/MicroBenchmarks.csproj index cf9ab26266b..d7b14d9336f 100644 --- a/src/benchmarks/micro/MicroBenchmarks.csproj +++ b/src/benchmarks/micro/MicroBenchmarks.csproj @@ -16,6 +16,13 @@ $(NoWarn);SYSLIB0011 $(NoWarn);SYSLIB5003 + + false + $(NoWarn);NU1510 Exe AnyCPU portable @@ -94,7 +101,7 @@ - + @@ -102,13 +109,18 @@ - - - - - - - + + + + + + + diff --git a/src/benchmarks/micro/Program.cs b/src/benchmarks/micro/Program.cs index 121d9dd9b98..8d2a590cae9 100644 --- a/src/benchmarks/micro/Program.cs +++ b/src/benchmarks/micro/Program.cs @@ -5,6 +5,7 @@ using System; using System.Collections.Generic; using System.Collections.Immutable; +using System.Threading.Tasks; using BenchmarkDotNet.Running; using System.IO; using BenchmarkDotNet.Extensions; @@ -14,7 +15,7 @@ namespace MicroBenchmarks { class Program { - static int Main(string[] args) + static async Task Main(string[] args) { var argsList = new List(args); int? partitionCount; @@ -40,9 +41,14 @@ static int Main(string[] args) return 1; } - return BenchmarkSwitcher + // Use RunAsync (not Run) so BDN does not install its single-threaded + // BenchmarkDotNetSynchronizationContext on the entrypoint thread. The sync + // entrypoint installs that context before benchmark discovery, which + // deadlocks any sync-over-async work performed by [ParamsSource]/[ArgumentsSource] + // callbacks (e.g. SslStreamTests.GetTls13Support). + var summaries = await BenchmarkSwitcher .FromAssembly(typeof(Program).Assembly) - .Run(argsList.ToArray(), + .RunAsync(argsList.ToArray(), RecommendedConfig.Create( artifactsPath: new DirectoryInfo(Path.Combine(AppContext.BaseDirectory, "BenchmarkDotNet.Artifacts")), mandatoryCategories: ImmutableHashSet.Create([Categories.Libraries, Categories.Runtime, Categories.ThirdParty, Categories.Sve]), @@ -52,7 +58,9 @@ static int Main(string[] args) categoryExclusionFilterValue: categoryExclusionFilterValue, getDiffableDisasm: getDiffableDisasm) .AddValidator(new NoWasmValidator(Categories.NoWASM))) - .ToExitCode(); + .ConfigureAwait(false); + + return summaries.ToExitCode(); } } } \ No newline at end of file diff --git a/src/benchmarks/micro/README.md b/src/benchmarks/micro/README.md index 590edec71af..d19ed650f5f 100644 --- a/src/benchmarks/micro/README.md +++ b/src/benchmarks/micro/README.md @@ -12,7 +12,7 @@ To learn more about designing benchmarks, please read [Microbenchmark Design Gui ## Quick Start -The first thing that you need to choose is the Target Framework. Available options are: `netcoreapp3.1|net6.0|net7.0|net8.0|net9.0|net10.0|net11.0|net472`. You can specify the target framework using `-f|--framework` argument. For the sake of simplicity, all examples below use `net11.0` as the target framework. +The first thing that you need to choose is the Target Framework. Available options are: `net8.0|net9.0|net10.0|net11.0`. You can specify the target framework using `-f|--framework` argument. For the sake of simplicity, all examples below use `net11.0` as the target framework. The following commands are run from the `src/benchmarks/micro` directory. diff --git a/src/benchmarks/micro/Serializers/DataGenerator.cs b/src/benchmarks/micro/Serializers/DataGenerator.cs index b78f5d9330b..55bb71917a1 100644 --- a/src/benchmarks/micro/Serializers/DataGenerator.cs +++ b/src/benchmarks/micro/Serializers/DataGenerator.cs @@ -32,7 +32,7 @@ internal static T Generate() return (T)(object)CreateIndexViewModel(); if (typeof(T) == typeof(MyEventsListerViewModel)) return (T)(object)CreateMyEventsListerViewModel(); - if (typeof(T) == typeof(BinaryData)) + if (typeof(T) == typeof(BinaryDataPayload)) return (T)(object)CreateBinaryData(1024); if (typeof(T) == typeof(CollectionsOfPrimitives)) return (T)(object)CreateCollectionsOfPrimitives(1024); // 1024 values was copied from CoreFX benchmarks @@ -163,8 +163,8 @@ private static MyEventsListerItem CreateMyEventsListerItem() }, 4).ToList() }; - private static BinaryData CreateBinaryData(int size) - => new BinaryData + private static BinaryDataPayload CreateBinaryData(int size) + => new BinaryDataPayload { ByteArray = CreateByteArray(size) }; @@ -383,7 +383,13 @@ public string FormattedDate [Serializable] [ProtoContract] [MessagePackObject] - public class BinaryData + // Renamed from BinaryData to avoid name collision with System.BinaryData + // (introduced in net6+ via System.Memory.Data, transitive via Azure.Core). + // The collision caused [GenericTypeArguments(typeof(BinaryData))] to resolve + // to System.BinaryData in some compile contexts (e.g. when the harness targets + // net8.0 instead of netstandard2.0), bypassing the data generator and producing + // NotImplementedException at runtime. + public class BinaryDataPayload { [ProtoMember(1)] [Key(0)] public byte[] ByteArray { get; set; } } @@ -496,7 +502,7 @@ public enum SystemTextJsonSerializationMode [JsonSerializable(typeof(Location))] [JsonSerializable(typeof(IndexViewModel))] [JsonSerializable(typeof(MyEventsListerViewModel))] - [JsonSerializable(typeof(BinaryData))] + [JsonSerializable(typeof(BinaryDataPayload))] [JsonSerializable(typeof(CollectionsOfPrimitives))] [JsonSerializable(typeof(XmlElement))] [JsonSerializable(typeof(SimpleStructWithProperties))] diff --git a/src/benchmarks/micro/libraries/System.Runtime/Perf.String.cs b/src/benchmarks/micro/libraries/System.Runtime/Perf.String.cs index 4622f970fd7..357052a8d39 100644 --- a/src/benchmarks/micro/libraries/System.Runtime/Perf.String.cs +++ b/src/benchmarks/micro/libraries/System.Runtime/Perf.String.cs @@ -321,6 +321,7 @@ public static string[] ReadInputFile(string name) => File.ReadAllLines(Path.Combine(AppContext.BaseDirectory, "libraries", "System.Runtime", "TestData", name)); [Benchmark] + [BenchmarkCategory(Categories.NoWASM)] // CSV files in TestData are not available in WASM filesystem [ArgumentsSource(nameof(CsvCorpus))] public string[] Split_Csv(string testName, string[] lines) { diff --git a/src/benchmarks/micro/libraries/System.Text.Json/Serializer/ReadJson.cs b/src/benchmarks/micro/libraries/System.Text.Json/Serializer/ReadJson.cs index 10fe6cc0778..49ec76814a1 100644 --- a/src/benchmarks/micro/libraries/System.Text.Json/Serializer/ReadJson.cs +++ b/src/benchmarks/micro/libraries/System.Text.Json/Serializer/ReadJson.cs @@ -18,7 +18,7 @@ namespace System.Text.Json.Serialization.Tests [GenericTypeArguments(typeof(Location))] [GenericTypeArguments(typeof(IndexViewModel))] [GenericTypeArguments(typeof(MyEventsListerViewModel))] - [GenericTypeArguments(typeof(BinaryData))] + [GenericTypeArguments(typeof(BinaryDataPayload))] [GenericTypeArguments(typeof(Dictionary))] [GenericTypeArguments(typeof(ImmutableDictionary))] [GenericTypeArguments(typeof(ImmutableSortedDictionary))] diff --git a/src/benchmarks/micro/libraries/System.Text.Json/Serializer/WriteJson.cs b/src/benchmarks/micro/libraries/System.Text.Json/Serializer/WriteJson.cs index 1eb8b8a57f8..f00a4da8b26 100644 --- a/src/benchmarks/micro/libraries/System.Text.Json/Serializer/WriteJson.cs +++ b/src/benchmarks/micro/libraries/System.Text.Json/Serializer/WriteJson.cs @@ -20,7 +20,7 @@ namespace System.Text.Json.Serialization.Tests [GenericTypeArguments(typeof(Location))] [GenericTypeArguments(typeof(IndexViewModel))] [GenericTypeArguments(typeof(MyEventsListerViewModel))] - [GenericTypeArguments(typeof(BinaryData))] + [GenericTypeArguments(typeof(BinaryDataPayload))] [GenericTypeArguments(typeof(Dictionary))] [GenericTypeArguments(typeof(ImmutableDictionary))] [GenericTypeArguments(typeof(ImmutableSortedDictionary))] diff --git a/src/harness/BenchmarkDotNet.Extensions/BenchmarkDotNet.Extensions.csproj b/src/harness/BenchmarkDotNet.Extensions/BenchmarkDotNet.Extensions.csproj index ef8782db2aa..a54dc819171 100644 --- a/src/harness/BenchmarkDotNet.Extensions/BenchmarkDotNet.Extensions.csproj +++ b/src/harness/BenchmarkDotNet.Extensions/BenchmarkDotNet.Extensions.csproj @@ -1,7 +1,11 @@  Library - netstandard2.0 + + net8.0 enable true @@ -13,7 +17,7 @@ - + diff --git a/src/harness/BenchmarkDotNet.Extensions/CommandLineOptions.cs b/src/harness/BenchmarkDotNet.Extensions/CommandLineOptions.cs index 3c8b343fc57..b76dd07dc29 100644 --- a/src/harness/BenchmarkDotNet.Extensions/CommandLineOptions.cs +++ b/src/harness/BenchmarkDotNet.Extensions/CommandLineOptions.cs @@ -35,7 +35,7 @@ public static List ParseAndRemoveStringsParameter(List argsList, int parameterIndex = argsList.IndexOf(parameter); parameterValue = new List(); - if (parameterIndex + 1 < argsList.Count) + if (parameterIndex != -1 && parameterIndex + 1 < argsList.Count) { while (parameterIndex + 1 < argsList.Count && !argsList[parameterIndex + 1].StartsWith("-")) { @@ -94,4 +94,4 @@ public static void ValidatePartitionParameters(int? count, int? index) } } } -} \ No newline at end of file +} diff --git a/src/harness/BenchmarkDotNet.Extensions/DiffableDisassemblyExporter.cs b/src/harness/BenchmarkDotNet.Extensions/DiffableDisassemblyExporter.cs index 44f81dd29e9..3124e82c546 100644 --- a/src/harness/BenchmarkDotNet.Extensions/DiffableDisassemblyExporter.cs +++ b/src/harness/BenchmarkDotNet.Extensions/DiffableDisassemblyExporter.cs @@ -61,26 +61,26 @@ internal static string BuildDisassemblyString(DisassemblyResult disassemblyResul private static Func GetElementGetter(string name) { - var type = typeof(DisassemblyDiagnoser).Assembly.GetType("BenchmarkDotNet.Disassemblers.Exporters.DisassemblyPrettifier"); + var type = typeof(DisassemblyDiagnoser).Assembly.GetType("BenchmarkDotNet.Disassemblers.Exporters.DisassemblyPrettifier")!; - type = type.GetNestedType("Element", BindingFlags.Instance | BindingFlags.NonPublic); + type = type.GetNestedType("Element", BindingFlags.Instance | BindingFlags.NonPublic)!; - var property = type.GetProperty(name, BindingFlags.Instance | BindingFlags.NonPublic); + var property = type.GetProperty(name, BindingFlags.Instance | BindingFlags.NonPublic)!; - var method = property.GetGetMethod(nonPublic: true); + var method = property.GetGetMethod(nonPublic: true)!; var generic = typeof(Func<,>).MakeGenericType(type, typeof(T)); var @delegate = method.CreateDelegate(generic); - return (obj) => (T)@delegate.DynamicInvoke(obj); // cast to (Func) throws + return (obj) => (T)@delegate.DynamicInvoke(obj)!; // cast to (Func) throws } private static Func> GetPrettifyMethod() { - var type = typeof(DisassemblyDiagnoser).Assembly.GetType("BenchmarkDotNet.Disassemblers.Exporters.DisassemblyPrettifier"); + var type = typeof(DisassemblyDiagnoser).Assembly.GetType("BenchmarkDotNet.Disassemblers.Exporters.DisassemblyPrettifier")!; - var method = type.GetMethod("Prettify", BindingFlags.Static | BindingFlags.NonPublic); + var method = type.GetMethod("Prettify", BindingFlags.Static | BindingFlags.NonPublic)!; var @delegate = method.CreateDelegate(typeof(Func>)); diff --git a/src/harness/BenchmarkDotNet.Extensions/MandatoryCategoryValidator.cs b/src/harness/BenchmarkDotNet.Extensions/MandatoryCategoryValidator.cs index 7b3b2d38f3b..280e43653d9 100644 --- a/src/harness/BenchmarkDotNet.Extensions/MandatoryCategoryValidator.cs +++ b/src/harness/BenchmarkDotNet.Extensions/MandatoryCategoryValidator.cs @@ -21,7 +21,7 @@ public class MandatoryCategoryValidator : IValidator public MandatoryCategoryValidator(ImmutableHashSet categories) => _mandatoryCategories = categories; - public IEnumerable Validate(ValidationParameters validationParameters) + public IAsyncEnumerable ValidateAsync(ValidationParameters validationParameters) => validationParameters.Benchmarks .Where(benchmark => !benchmark.Descriptor.Categories.Any(category => _mandatoryCategories.Contains(category))) .Select(benchmark => benchmark.Descriptor.GetFilterName()) @@ -30,6 +30,7 @@ public IEnumerable Validate(ValidationParameters validationPara new ValidationError( isCritical: TreatsWarningsAsErrors, $"{benchmarkId} does not belong to one of the mandatory categories: {string.Join(", ", _mandatoryCategories)}. Use [BenchmarkCategory(Categories.$)]") - ); + ) + .ToAsyncEnumerable(); } } \ No newline at end of file diff --git a/src/harness/BenchmarkDotNet.Extensions/NoWasmValidator.cs b/src/harness/BenchmarkDotNet.Extensions/NoWasmValidator.cs index 5b2cfec63ca..68fe73a4645 100644 --- a/src/harness/BenchmarkDotNet.Extensions/NoWasmValidator.cs +++ b/src/harness/BenchmarkDotNet.Extensions/NoWasmValidator.cs @@ -23,7 +23,7 @@ public class NoWasmValidator : IValidator public NoWasmValidator(string noWasmCategory) => _noWasmCategory = noWasmCategory; - public IEnumerable Validate(ValidationParameters validationParameters) + public IAsyncEnumerable ValidateAsync(ValidationParameters validationParameters) => validationParameters.Benchmarks .Where(benchmark => IsAsyncMethod(benchmark.Descriptor.WorkloadMethod) && !benchmark.Descriptor.Categories.Any(category => category.Equals(_noWasmCategory, StringComparison.Ordinal))) .Select(benchmark => benchmark.Descriptor.GetFilterName()) @@ -32,7 +32,8 @@ public IEnumerable Validate(ValidationParameters validationPara new ValidationError( isCritical: TreatsWarningsAsErrors, $"{benchmarkId} returns an awaitable object and has no: {_noWasmCategory} category applied. Use [BenchmarkCategory(Categories.NoWASM)]") - ); + ) + .ToAsyncEnumerable(); private bool IsAsyncMethod(MethodInfo workloadMethod) { diff --git a/src/harness/BenchmarkDotNet.Extensions/PerfLabExporter.cs b/src/harness/BenchmarkDotNet.Extensions/PerfLabExporter.cs index c1ed4268e0f..d308afb27a8 100644 --- a/src/harness/BenchmarkDotNet.Extensions/PerfLabExporter.cs +++ b/src/harness/BenchmarkDotNet.Extensions/PerfLabExporter.cs @@ -4,24 +4,67 @@ using BenchmarkDotNet.Diagnosers; using BenchmarkDotNet.Exporters; +using BenchmarkDotNet.Helpers; using BenchmarkDotNet.Loggers; using BenchmarkDotNet.Reports; using Reporting; using System; +using System.IO; using System.Linq; +using System.Threading; +using System.Threading.Tasks; namespace BenchmarkDotNet.Extensions { - public class PerfLabExporter : ExporterBase + // Implements IExporter directly (not ExporterBase) because PerfLabExporter writes + // a file with a custom name pattern ("{type}-perf-lab-report.json") via + // File.WriteAllTextAsync and manages the file lifecycle itself, rather than having + // ExporterBase open and hand us a writer for a default-named file. + public class PerfLabExporter : IExporter { - protected override string FileExtension => "json"; - protected override string FileCaption => "perf-lab-report"; + private const string FileExtension = "json"; + private const string FileCaption = "perf-lab-report"; - public PerfLabExporter() + public string Name => nameof(PerfLabExporter); + + public async ValueTask ExportAsync(Summary summary, ILogger logger, CancellationToken cancellationToken) + { + string? jsonOutput = BuildJson(summary); + if (jsonOutput is null) + return; + + string filePath = GetArtifactFullName(summary); + if (File.Exists(filePath)) + { + try + { + File.Delete(filePath); + } + catch (IOException) + { + string uniqueString = DateTime.Now.ToString("yyyyMMdd-HHmmss"); + string altPath = $"{Path.Combine(summary.ResultsDirectoryPath, GetFileName(summary))}-{FileCaption}-{uniqueString}.{FileExtension}"; + logger.WriteLineError($"Could not overwrite file {filePath}. Exporting to {altPath}"); + filePath = altPath; + } + } + + await File.WriteAllTextAsync(filePath, jsonOutput, cancellationToken).ConfigureAwait(false); + logger.WriteLineInfo($" {filePath}"); + } + + private string GetArtifactFullName(Summary summary) + => $"{Path.Combine(summary.ResultsDirectoryPath, GetFileName(summary))}-{FileCaption}.{FileExtension}"; + + private static string GetFileName(Summary summary) { + var targets = summary.BenchmarksCases.Select(b => b.Descriptor.Type).Distinct().ToArray(); + if (targets.Length == 1) + return FolderNameHelper.ToFolderName(targets.Single()); + return summary.Title; } - public override void ExportToLog(Summary summary, ILogger logger) + private static string? BuildJson(Summary summary) { var reporter = new Reporter(); @@ -49,7 +92,7 @@ public override void ExportToLog(Summary summary, ILogger logger) var test = new Test(); test.Name = FullNameProvider.GetBenchmarkName(report.BenchmarkCase); test.Categories = report.BenchmarkCase.Descriptor.Categories; - + if (hasCriticalErrors) { test.AdditionalData["criticalErrors"] = "true"; @@ -58,7 +101,7 @@ public override void ExportToLog(Summary summary, ILogger logger) var results = from result in report.AllMeasurements where result.IterationMode == Engines.IterationMode.Workload && result.IterationStage == Engines.IterationStage.Result orderby result.LaunchIndex, result.IterationIndex - select new { result.Nanoseconds, result.Operations}; + select new { result.Nanoseconds, result.Operations }; var overheadResults = from result in report.AllMeasurements where result.IsOverhead() && result.IterationStage != Engines.IterationStage.Jitting @@ -104,7 +147,7 @@ where result.IsOverhead() && result.IterationStage != Engines.IterationStage.Jit HigherIsBetter = true, MetricName = "Count", Results = (from result in results - select (double)result.Operations).ToList() + select (double)result.Operations).ToList() }); foreach (var metric in report.Metrics.Keys) @@ -130,9 +173,7 @@ where result.IsOverhead() && result.IterationStage != Engines.IterationStage.Jit reporter.AddTest(test); } - var jsonOutput = reporter.GetJson(); - if (jsonOutput is not null) - logger.WriteLine(jsonOutput); + return reporter.GetJson(); } } } diff --git a/src/harness/BenchmarkDotNet.Extensions/TooManyTestCasesValidator.cs b/src/harness/BenchmarkDotNet.Extensions/TooManyTestCasesValidator.cs index aaf8a62317a..9df1b3d99cb 100644 --- a/src/harness/BenchmarkDotNet.Extensions/TooManyTestCasesValidator.cs +++ b/src/harness/BenchmarkDotNet.Extensions/TooManyTestCasesValidator.cs @@ -19,7 +19,7 @@ public class TooManyTestCasesValidator : IValidator public bool TreatsWarningsAsErrors => true; - public IEnumerable Validate(ValidationParameters validationParameters) + public IAsyncEnumerable ValidateAsync(ValidationParameters validationParameters) { var byDescriptor = validationParameters.Benchmarks .Where(benchmark => !SkipValidation(benchmark.Descriptor.WorkloadMethod)) @@ -29,10 +29,11 @@ public IEnumerable Validate(ValidationParameters validationPara new ValidationError( isCritical: true, message: $"{group.Key.Descriptor.Type.Name}.{group.Key.Descriptor.WorkloadMethod.Name} has {group.Count()} test cases. It MUST NOT have more than {Limit} test cases. We don't have inifinite amount of time to run all the benchmarks!!", - benchmarkCase: group.First())); + benchmarkCase: group.First())) + .ToAsyncEnumerable(); } - private static bool SkipValidation(MemberInfo member) + private static bool SkipValidation(MemberInfo? member) { while (member != null) { diff --git a/src/harness/BenchmarkDotNet.Extensions/UniqueArgumentsValidator.cs b/src/harness/BenchmarkDotNet.Extensions/UniqueArgumentsValidator.cs index 532e9b003f5..e3903769a69 100644 --- a/src/harness/BenchmarkDotNet.Extensions/UniqueArgumentsValidator.cs +++ b/src/harness/BenchmarkDotNet.Extensions/UniqueArgumentsValidator.cs @@ -14,7 +14,9 @@ public class UniqueArgumentsValidator : IValidator { public bool TreatsWarningsAsErrors => true; - public IEnumerable Validate(ValidationParameters validationParameters) + // Use ToAsyncEnumerable() (not async + yield return) to avoid the AsyncIteratorMethodBuilder + // state machine deadlocking with BDN's BenchmarkSynchronizationContext (matches BDN's own validators). + public IAsyncEnumerable ValidateAsync(ValidationParameters validationParameters) => validationParameters.Benchmarks .Where(benchmark => benchmark.HasArguments || benchmark.HasParameters) .GroupBy(benchmark => (benchmark.Descriptor.Type, benchmark.Descriptor.WorkloadMethod, benchmark.Job)) @@ -25,12 +27,16 @@ public IEnumerable Validate(ValidationParameters validationPara return numberOfTestCases != numberOfUniqueTestCases; }) - .Select(duplicate => new ValidationError(true, $"Benchmark Arguments should be unique, {duplicate.Key.Type}.{duplicate.Key.WorkloadMethod} has duplicate arguments.", duplicate.First())); + .Select(duplicate => new ValidationError(true, $"Benchmark Arguments should be unique, {duplicate.Key.Type}.{duplicate.Key.WorkloadMethod} has duplicate arguments.", duplicate.First())) + .ToAsyncEnumerable(); private class BenchmarkArgumentsComparer : IEqualityComparer { - public bool Equals(BenchmarkCase x, BenchmarkCase y) + public bool Equals(BenchmarkCase? x, BenchmarkCase? y) { + if (x is null || y is null) + return ReferenceEquals(x, y); + if (FullNameProvider.GetBenchmarkName(x).Equals(FullNameProvider.GetBenchmarkName(y), System.StringComparison.Ordinal)) return true; diff --git a/src/harness/BenchmarkDotNet.Extensions/ValuesGenerator.cs b/src/harness/BenchmarkDotNet.Extensions/ValuesGenerator.cs index 107c298e93f..e36efdf9747 100644 --- a/src/harness/BenchmarkDotNet.Extensions/ValuesGenerator.cs +++ b/src/harness/BenchmarkDotNet.Extensions/ValuesGenerator.cs @@ -130,7 +130,7 @@ public static byte[] ArrayBase64EncodingBytes(int count) /// the stored values are randomly generated. /// GenerateValue is used to generate a random value in the appropriate range for both the key and value /// - public static Dictionary Dictionary(int count) + public static Dictionary Dictionary(int count) where TKey : notnull { if (count > 2 && typeof(TKey) == typeof(bool)) throw new ArgumentOutOfRangeException("count", "Cannot exceed 2 for Dictionary"); diff --git a/src/tests/harness/BenchmarkDotNet.Extensions.Tests/BenchmarkDotNet.Extensions.Tests.csproj b/src/tests/harness/BenchmarkDotNet.Extensions.Tests/BenchmarkDotNet.Extensions.Tests.csproj index 796d014d367..24a2526f6ec 100644 --- a/src/tests/harness/BenchmarkDotNet.Extensions.Tests/BenchmarkDotNet.Extensions.Tests.csproj +++ b/src/tests/harness/BenchmarkDotNet.Extensions.Tests/BenchmarkDotNet.Extensions.Tests.csproj @@ -7,7 +7,9 @@ - + diff --git a/src/tests/harness/BenchmarkDotNet.Extensions.Tests/CommandLineOptionsTests.cs b/src/tests/harness/BenchmarkDotNet.Extensions.Tests/CommandLineOptionsTests.cs index 4fbe6a7a10e..48e53620bcf 100644 --- a/src/tests/harness/BenchmarkDotNet.Extensions.Tests/CommandLineOptionsTests.cs +++ b/src/tests/harness/BenchmarkDotNet.Extensions.Tests/CommandLineOptionsTests.cs @@ -128,5 +128,89 @@ public void PartitionIndexValueGreaterThanCount() Assert.Throws(() => CommandLineOptions.ValidatePartitionParameters(count, index)); } + + [Fact] + public void ParseAndRemoveStringsParameterCollectsValuesUntilNextFlag() + { + List argsList = new List { + "--exclusion-filter", + "System.*", + "Microsoft.*", + "--partition-count", + "4" + }; + + var remaining = CommandLineOptions.ParseAndRemoveStringsParameter(argsList, "--exclusion-filter", out List filters); + + Assert.Equal(new[] { "System.*", "Microsoft.*" }, filters); + Assert.Equal(new[] { "--partition-count", "4" }, remaining); + } + + [Fact] + public void ParseAndRemoveBooleanParameterRemovesSwitchWhenPresent() + { + List argsList = new List { + "--wasm", + "--filter", + "*" + }; + + CommandLineOptions.ParseAndRemoveBooleanParameter(argsList, "--wasm", out bool enabled); + + Assert.True(enabled); + Assert.Equal(new[] { "--filter", "*" }, argsList); + } + + [Fact] + public void ParseAndRemoveStringsParameterLeavesArgsUntouchedWhenSwitchIsMissing() + { + List argsList = new List { + "literal-value", + "--filter", + "*" + }; + + var remaining = CommandLineOptions.ParseAndRemoveStringsParameter(argsList, "--exclusion-filter", out List filters); + + Assert.Empty(filters); + Assert.Equal(new[] { "literal-value", "--filter", "*" }, remaining); + } + + [Fact] + public void ParseAndRemoveBooleanParameterReturnsFalseWhenSwitchIsMissing() + { + List argsList = new List { + "--filter", + "*" + }; + + CommandLineOptions.ParseAndRemoveBooleanParameter(argsList, "--wasm", out bool enabled); + + Assert.False(enabled); + Assert.Equal(new[] { "--filter", "*" }, argsList); + } + + [Theory] + [InlineData("--partition-count")] + [InlineData("--partition-index")] + public void ParseAndRemoveIntParameterThrowsWhenValueIsMissing(string parameter) + { + List argsList = new List { parameter }; + + Assert.Throws(() => CommandLineOptions.ParseAndRemoveIntParameter(argsList, parameter, out int? _)); + } + + [Theory] + [InlineData("--partition-count", "abc")] + [InlineData("--partition-index", "3.14")] + public void ParseAndRemoveIntParameterThrowsWhenValueIsNotAnInteger(string parameter, string value) + { + List argsList = new List { + parameter, + value + }; + + Assert.Throws(() => CommandLineOptions.ParseAndRemoveIntParameter(argsList, parameter, out int? _)); + } } } diff --git a/src/tests/harness/BenchmarkDotNet.Extensions.Tests/PartitionFilterTests.cs b/src/tests/harness/BenchmarkDotNet.Extensions.Tests/PartitionFilterTests.cs index 9319d50c607..f2a551c3bf7 100644 --- a/src/tests/harness/BenchmarkDotNet.Extensions.Tests/PartitionFilterTests.cs +++ b/src/tests/harness/BenchmarkDotNet.Extensions.Tests/PartitionFilterTests.cs @@ -63,8 +63,9 @@ public void NoBenchmarksAreOmitted_RealData() IConfig recommendedConfig = RecommendedConfig.Create( artifactsPath: new DirectoryInfo(Path.Combine(Path.GetDirectoryName(typeof(PartitionFilterTests).Assembly.Location)!, "BenchmarkDotNet.Artifacts")), mandatoryCategories: ImmutableHashSet.Create(Categories.Libraries, Categories.Runtime, Categories.ThirdParty)); - (bool isSuccess, IConfig parsedConfig, var _) = ConfigParser.Parse(new string[] { "--filter", "*" }, nullLogger, recommendedConfig); + (bool isSuccess, IConfig? parsedConfig, var _) = ConfigParser.Parse(new string[] { "--filter", "*" }, nullLogger, recommendedConfig); Assert.True(isSuccess); + Assert.NotNull(parsedConfig); Assembly microbenchmarksAssembly = typeof(Categories).Assembly; (bool allTypesValid, IReadOnlyList runnable) = Running.TypeFilter.GetTypesWithRunnableBenchmarks( @@ -73,7 +74,7 @@ public void NoBenchmarksAreOmitted_RealData() nullLogger); Assert.True(allTypesValid); - BenchmarkRunInfo[] allBenchmarks = GetAllBenchmarks(parsedConfig, runnable); + BenchmarkRunInfo[] allBenchmarks = GetAllBenchmarks(parsedConfig!, runnable); Dictionary idToPartitionIndex = new (); for (int i = 0; i < 10; i++) @@ -86,7 +87,7 @@ public void NoBenchmarksAreOmitted_RealData() { PartitionFilter filter = new(PartitionCount, partitionIndex); - foreach (BenchmarkCase benchmark in GetAllBenchmarks(parsedConfig, runnable).SelectMany(benchmark => benchmark.BenchmarksCases)) + foreach (BenchmarkCase benchmark in GetAllBenchmarks(parsedConfig!, runnable).SelectMany(benchmark => benchmark.BenchmarksCases)) { if (filter.Predicate(benchmark)) { diff --git a/src/tests/harness/BenchmarkDotNet.Extensions.Tests/UniqueArgumentsValidatorTests.cs b/src/tests/harness/BenchmarkDotNet.Extensions.Tests/UniqueArgumentsValidatorTests.cs index fdabaa78db0..5715ffcf676 100644 --- a/src/tests/harness/BenchmarkDotNet.Extensions.Tests/UniqueArgumentsValidatorTests.cs +++ b/src/tests/harness/BenchmarkDotNet.Extensions.Tests/UniqueArgumentsValidatorTests.cs @@ -6,6 +6,7 @@ using System.Collections.Generic; using System.Linq; using System.Reflection; +using System.Threading.Tasks; using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Extensions; using BenchmarkDotNet.Running; @@ -26,7 +27,7 @@ public void DuplicatedArgumentsAreDetected(Type typeWithBenchmarks, bool shouldR var benchmarksForType = BenchmarkConverter.TypeToBenchmarks(typeWithBenchmarks); var validationParameters = new ValidationParameters(benchmarksForType.BenchmarksCases, benchmarksForType.Config); - var validationErrors = new UniqueArgumentsValidator().Validate(validationParameters); + var validationErrors = new UniqueArgumentsValidator().ValidateAsync(validationParameters).ToBlockingEnumerable(); if (shouldReportError) Assert.NotEmpty(validationErrors); diff --git a/src/tests/harness/BenchmarkDotNet.Extensions.Tests/UniqueValuesGeneratorTests.cs b/src/tests/harness/BenchmarkDotNet.Extensions.Tests/UniqueValuesGeneratorTests.cs index ff3b8f0f6e0..58a6124938f 100644 --- a/src/tests/harness/BenchmarkDotNet.Extensions.Tests/UniqueValuesGeneratorTests.cs +++ b/src/tests/harness/BenchmarkDotNet.Extensions.Tests/UniqueValuesGeneratorTests.cs @@ -151,14 +151,14 @@ private static void SupportsNonDefaultValue() Assert.NotEqual(default, value); } - private static void SupportsDictionary(int count) + private static void SupportsDictionary(int count) where TKey : notnull { var dictionary = ValuesGenerator.Dictionary(count); Assert.NotNull(dictionary); Assert.Equal(count, dictionary.Count); } - private static void Supports(int count = 10) + private static void Supports(int count = 10) where T : notnull { SupportsArray(count); SupportsNonDefaultValue(); diff --git a/src/tools/Reporting/Directory.Packages.props b/src/tools/Reporting/Directory.Packages.props index ab6543dc86b..0fde0cf995d 100644 --- a/src/tools/Reporting/Directory.Packages.props +++ b/src/tools/Reporting/Directory.Packages.props @@ -4,7 +4,7 @@ true - + diff --git a/src/tools/ResultsComparer.Tests/ConsoleOutputCollection.cs b/src/tools/ResultsComparer.Tests/ConsoleOutputCollection.cs new file mode 100644 index 00000000000..8f3c5aa69b6 --- /dev/null +++ b/src/tools/ResultsComparer.Tests/ConsoleOutputCollection.cs @@ -0,0 +1,8 @@ +using Xunit; + +namespace ResultsComparer.Tests; + +[CollectionDefinition("Console output", DisableParallelization = true)] +public sealed class ConsoleOutputCollection +{ +} diff --git a/src/tools/ResultsComparer.Tests/DataTests.cs b/src/tools/ResultsComparer.Tests/DataTests.cs new file mode 100644 index 00000000000..a94592f139f --- /dev/null +++ b/src/tools/ResultsComparer.Tests/DataTests.cs @@ -0,0 +1,161 @@ +using System.Formats.Tar; +using System.IO; +using System.IO.Compression; +using System.Text; +using Xunit; + +namespace ResultsComparer.Tests; + +public class DataTests +{ + [Fact] + public void DecompressExtractsJsonFilesFromNestedZipArchives() + { + var tempDir = Directory.CreateTempSubdirectory(); + try + { + var outerZipPath = Path.Combine(tempDir.FullName, "results.zip"); + var outputDirectory = new DirectoryInfo(Path.Combine(tempDir.FullName, "output")); + outputDirectory.Create(); + + var innerZipBytes = CreateInnerZip(("net10.0/SampleBenchmark.full.json", ResultsComparerTestData.CreateBdnJson())); + + using (var fileStream = File.Create(outerZipPath)) + using (var archive = new ZipArchive(fileStream, ZipArchiveMode.Create)) + { + var entry = archive.CreateEntry("Performance-Runs/net10.0/testuser/results.zip"); + using var entryStream = entry.Open(); + entryStream.Write(innerZipBytes, 0, innerZipBytes.Length); + } + + global::ResultsComparer.Data.Decompress(new FileInfo(outerZipPath), outputDirectory); + + var extractedFiles = Directory.GetFiles(outputDirectory.FullName, "*.full.json", SearchOption.AllDirectories); + var extractedFile = Assert.Single(extractedFiles); + var directoryName = Path.GetFileName(Path.GetDirectoryName(extractedFile)); + + Assert.Contains("testuser", directoryName, System.StringComparison.OrdinalIgnoreCase); + Assert.Contains("net10.0", directoryName, System.StringComparison.OrdinalIgnoreCase); + Assert.Contains("SampleBenchmark", File.ReadAllText(extractedFile)); + } + finally + { + tempDir.Delete(recursive: true); + } + } + + [Fact] + public void DecompressExtractsJsonFilesFromTarGzArchives() + { + var tempDir = Directory.CreateTempSubdirectory(); + try + { + var outerZipPath = Path.Combine(tempDir.FullName, "results.zip"); + var outputDirectory = new DirectoryInfo(Path.Combine(tempDir.FullName, "output")); + outputDirectory.Create(); + + var tarGzBytes = CreateTarGzArchive( + ("payload/SampleBenchmark.full.json", ResultsComparerTestData.CreateBdnJson()), + ("payload/README.md", "ignored")); + + using (var fileStream = File.Create(outerZipPath)) + using (var archive = new ZipArchive(fileStream, ZipArchiveMode.Create)) + { + var entry = archive.CreateEntry("Performance-Runs/nativeaot10.0/testuser/arm64_win10-nativeaot10.0.tar.gz"); + using var entryStream = entry.Open(); + entryStream.Write(tarGzBytes, 0, tarGzBytes.Length); + } + + global::ResultsComparer.Data.Decompress(new FileInfo(outerZipPath), outputDirectory); + + var extractedFiles = Directory.GetFiles(outputDirectory.FullName, "*.full.json", SearchOption.AllDirectories); + var extractedFile = Assert.Single(extractedFiles); + var directoryName = Path.GetFileName(Path.GetDirectoryName(extractedFile)); + + Assert.Contains("nativeaot10.0", directoryName, System.StringComparison.OrdinalIgnoreCase); + } + finally + { + tempDir.Delete(recursive: true); + } + } + + [Fact] + public void DecompressPrefersNewestBenchmarkDotNetVersionWhenDuplicatesExist() + { + var tempDir = Directory.CreateTempSubdirectory(); + try + { + var outerZipPath = Path.Combine(tempDir.FullName, "results.zip"); + var outputDirectory = new DirectoryInfo(Path.Combine(tempDir.FullName, "output")); + outputDirectory.Create(); + + var innerZipBytes = CreateInnerZip( + ("net10.0/SampleBenchmark-a.full.json", ResultsComparerTestData.CreateBdnJson(benchmarkDotNetVersion: "0.13.9")), + ("net10.0/SampleBenchmark-b.full.json", ResultsComparerTestData.CreateBdnJson(benchmarkDotNetVersion: "0.13.10"))); + + using (var fileStream = File.Create(outerZipPath)) + using (var archive = new ZipArchive(fileStream, ZipArchiveMode.Create)) + { + var entry = archive.CreateEntry("Performance-Runs/net10.0/testuser/results.zip"); + using var entryStream = entry.Open(); + entryStream.Write(innerZipBytes, 0, innerZipBytes.Length); + } + + global::ResultsComparer.Data.Decompress(new FileInfo(outerZipPath), outputDirectory); + + var extractedFile = Assert.Single(Directory.GetFiles(outputDirectory.FullName, "*.full.json", SearchOption.AllDirectories)); + var json = File.ReadAllText(extractedFile); + + Assert.Contains("\"BenchmarkDotNetVersion\": \"0.13.10\"", json); + } + finally + { + tempDir.Delete(recursive: true); + } + } + + private static byte[] CreateInnerZip(params (string EntryName, string Content)[] entries) + { + using var stream = new MemoryStream(); + using (var archive = new ZipArchive(stream, ZipArchiveMode.Create, leaveOpen: true)) + { + foreach (var (entryName, content) in entries) + { + var entry = archive.CreateEntry(entryName); + using var writer = new StreamWriter(entry.Open()); + writer.Write(content); + } + } + + return stream.ToArray(); + } + + private static byte[] CreateTarGzArchive(params (string EntryName, string Content)[] entries) + { + using var tarStream = new MemoryStream(); + using (var tarWriter = new TarWriter(tarStream, leaveOpen: true)) + { + foreach (var (entryName, content) in entries) + { + using var dataStream = new MemoryStream(Encoding.UTF8.GetBytes(content)); + var tarEntry = new UstarTarEntry(TarEntryType.RegularFile, entryName) + { + DataStream = dataStream + }; + + tarWriter.WriteEntry(tarEntry); + } + } + + tarStream.Position = 0; + + using var gzipStream = new MemoryStream(); + using (var compressor = new GZipStream(gzipStream, CompressionLevel.SmallestSize, leaveOpen: true)) + { + tarStream.CopyTo(compressor); + } + + return gzipStream.ToArray(); + } +} diff --git a/src/tools/ResultsComparer.Tests/HelperTests.cs b/src/tools/ResultsComparer.Tests/HelperTests.cs new file mode 100644 index 00000000000..b59b713fa29 --- /dev/null +++ b/src/tools/ResultsComparer.Tests/HelperTests.cs @@ -0,0 +1,137 @@ +using System.IO; +using System.Text; +using DataTransferContracts; +using Xunit; + +namespace ResultsComparer.Tests; + +public class HelperTests +{ + [Fact] + public void GetFilesToParseReturnsAllFullJsonFilesFromDirectory() + { + var tempDir = Directory.CreateTempSubdirectory(); + try + { + var nestedDirectory = Directory.CreateDirectory(Path.Combine(tempDir.FullName, "nested")); + var expectedA = Path.Combine(tempDir.FullName, "first.full.json"); + var expectedB = Path.Combine(nestedDirectory.FullName, "second.full.json"); + + File.WriteAllText(expectedA, "{}"); + File.WriteAllText(expectedB, "{}"); + File.WriteAllText(Path.Combine(tempDir.FullName, "ignored.json"), "{}"); + + var result = global::ResultsComparer.Helper.GetFilesToParse(tempDir.FullName); + + Assert.Equal(2, result.Length); + Assert.Contains(expectedA, result); + Assert.Contains(expectedB, result); + } + finally + { + tempDir.Delete(recursive: true); + } + } + + [Fact] + public void GetFilesToParseThrowsForMissingFullJsonPath() + { + var tempDir = Directory.CreateTempSubdirectory(); + try + { + var missingPath = Path.Combine(tempDir.FullName, "missing.full.json"); + + Assert.Throws(() => global::ResultsComparer.Helper.GetFilesToParse(missingPath)); + } + finally + { + tempDir.Delete(recursive: true); + } + } + + [Fact] + public void ReadFromStreamDeserializesBenchmarkResults() + { + var json = ResultsComparerTestData.CreateBdnJson(originalValues: [1.0, 1.1, 1.2], median: 1.1); + + using var stream = new MemoryStream(Encoding.UTF8.GetBytes(json)); + + BdnResult result = global::ResultsComparer.Helper.ReadFromStream(stream); + + Assert.Equal("SampleBenchmark-20240504-182513", result.Title); + Assert.Equal("X64", result.HostEnvironmentInfo.Architecture); + Assert.Single(result.Benchmarks); + Assert.Equal("Demo.Namespace.SampleBenchmark", result.Benchmarks[0].FullName); + } + + [Fact] + public void ReadFromFileDeserializesBenchmarkResults() + { + var tempDir = Directory.CreateTempSubdirectory(); + try + { + var filePath = Path.Combine(tempDir.FullName, "sample.full.json"); + File.WriteAllText(filePath, ResultsComparerTestData.CreateBdnJson()); + + BdnResult result = global::ResultsComparer.Helper.ReadFromFile(filePath); + + Assert.Equal("SampleBenchmark-20240504-182513", result.Title); + Assert.Single(result.Benchmarks); + } + finally + { + tempDir.Delete(recursive: true); + } + } + + [Fact] + public void GetFilesToParseReturnsExplicitFilePath() + { + var tempDir = Directory.CreateTempSubdirectory(); + try + { + var filePath = Path.Combine(tempDir.FullName, "custom-name.json"); + File.WriteAllText(filePath, ResultsComparerTestData.CreateBdnJson()); + + var result = global::ResultsComparer.Helper.GetFilesToParse(filePath); + + Assert.Equal([filePath], result); + } + finally + { + tempDir.Delete(recursive: true); + } + } + + [Fact] + public void GetModalInfoReturnsNullForSmallSampleSets() + { + var benchmark = new Benchmark + { + Statistics = new Statistics + { + N = 3, + OriginalValues = [1.0, 1.1, 1.2] + } + }; + + Assert.Null(global::ResultsComparer.Helper.GetModalInfo(benchmark)); + } + + [Fact] + public void GetModalInfoDetectsMultiClusterData() + { + var benchmark = new Benchmark + { + Statistics = new Statistics + { + N = 16, + OriginalValues = [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0] + } + }; + + var modality = global::ResultsComparer.Helper.GetModalInfo(benchmark); + + Assert.Equal("bimodal", modality); + } +} diff --git a/src/tools/ResultsComparer.Tests/ProgramTests.cs b/src/tools/ResultsComparer.Tests/ProgramTests.cs new file mode 100644 index 00000000000..786f05209fc --- /dev/null +++ b/src/tools/ResultsComparer.Tests/ProgramTests.cs @@ -0,0 +1,158 @@ +using System; +using System.Globalization; +using System.IO; +using Xunit; + +namespace ResultsComparer.Tests; + +[Collection("Console output")] +public class ProgramTests +{ + [Fact] + public void MainReportsInvalidThreshold() + { + var output = InvokeProgram(["--base", "base.json", "--diff", "diff.json", "--threshold", "not-a-threshold"]); + + Assert.Contains("Invalid Threshold", output); + } + + [Fact] + public void MainReportsMissingMatrixInputDirectory() + { + var missingDirectory = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString("N")); + + var output = InvokeProgram(["matrix", "--input", missingDirectory, "--base", "base", "--diff", "diff", "--threshold", "5%"]); + + Assert.Contains("does NOT exist", output); + } + + [Fact] + public void MainPrintsNoDifferencesForEquivalentInputs() + { + var tempDir = Directory.CreateTempSubdirectory(); + try + { + var baseFile = Path.Combine(tempDir.FullName, "base.full.json"); + var diffFile = Path.Combine(tempDir.FullName, "diff.full.json"); + var json = ResultsComparerTestData.CreateBdnJson(); + + File.WriteAllText(baseFile, json); + File.WriteAllText(diffFile, json); + + var output = InvokeProgram(["--base", baseFile, "--diff", diffFile, "--threshold", "5%"]); + + Assert.Contains("No differences found", output); + } + finally + { + tempDir.Delete(recursive: true); + } + } + + [Fact] + public void MatrixCommandPrintsLegendAndBenchmarkTable() + { + var tempDir = Directory.CreateTempSubdirectory(); + try + { + var inputDirectory = Directory.CreateDirectory(Path.Combine(tempDir.FullName, "input")); + var baseDirectory = Directory.CreateDirectory(Path.Combine(inputDirectory.FullName, "run-base")); + var diffDirectory = Directory.CreateDirectory(Path.Combine(inputDirectory.FullName, "run-diff")); + + File.WriteAllText( + Path.Combine(baseDirectory.FullName, "SampleBenchmark.full.json"), + ResultsComparerTestData.CreateBdnJson( + originalValues: [100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111], + median: 106, + allocatedBytes: 10)); + File.WriteAllText( + Path.Combine(diffDirectory.FullName, "SampleBenchmark.full.json"), + ResultsComparerTestData.CreateBdnJson( + originalValues: [200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211], + median: 206, + allocatedBytes: 30)); + + var output = InvokeProgram(["matrix", "--input", inputDirectory.FullName, "--base", "base", "--diff", "diff", "--threshold", "5%", "--ratio-only"]); + + Assert.Contains("# Legend", output); + Assert.Contains("Demo.Namespace.SampleBenchmark", output); + Assert.Contains("Slower", output); + } + finally + { + tempDir.Delete(recursive: true); + } + } + + [Fact] + public void MatrixCommandTreatsEquivalentInputsAsSameNotNoise() + { + var tempDir = Directory.CreateTempSubdirectory(); + try + { + var inputDirectory = Directory.CreateDirectory(Path.Combine(tempDir.FullName, "input")); + var baseDirectory = Directory.CreateDirectory(Path.Combine(inputDirectory.FullName, "run-base")); + var diffDirectory = Directory.CreateDirectory(Path.Combine(inputDirectory.FullName, "run-diff")); + var identicalJson = ResultsComparerTestData.CreateBdnJson(); + + File.WriteAllText(Path.Combine(baseDirectory.FullName, "SampleBenchmark.full.json"), identicalJson); + File.WriteAllText(Path.Combine(diffDirectory.FullName, "SampleBenchmark.full.json"), identicalJson); + + var output = InvokeProgram(["matrix", "--input", inputDirectory.FullName, "--base", "base", "--diff", "diff", "--threshold", "5%", "--ratio-only"]); + var lines = output.Split(Environment.NewLine, StringSplitOptions.RemoveEmptyEntries); + + Assert.Contains(lines, line => line.TrimStart().StartsWith("| Same", StringComparison.Ordinal)); + Assert.DoesNotContain(lines, line => line.TrimStart().StartsWith("| Noise", StringComparison.Ordinal)); + } + finally + { + tempDir.Delete(recursive: true); + } + } + + [Fact] + public void InvokeProgramRestoresCurrentCulture() + { + var originalCulture = CultureInfo.CurrentCulture; + var originalUICulture = CultureInfo.CurrentUICulture; + + try + { + var expectedCulture = new CultureInfo("fr-FR"); + var expectedUICulture = new CultureInfo("de-DE"); + + CultureInfo.CurrentCulture = expectedCulture; + CultureInfo.CurrentUICulture = expectedUICulture; + + _ = InvokeProgram(["--base", "base.json", "--diff", "diff.json", "--threshold", "not-a-threshold"]); + + Assert.Equal(expectedCulture.Name, CultureInfo.CurrentCulture.Name); + Assert.Equal(expectedUICulture.Name, CultureInfo.CurrentUICulture.Name); + } + finally + { + CultureInfo.CurrentCulture = originalCulture; + CultureInfo.CurrentUICulture = originalUICulture; + } + } + + private static string InvokeProgram(string[] args) + { + using var writer = new StringWriter(); + var originalOut = Console.Out; + var originalCulture = CultureInfo.CurrentCulture; + var originalUICulture = CultureInfo.CurrentUICulture; + Console.SetOut(writer); + try + { + Program.Main(args); + return writer.ToString(); + } + finally + { + CultureInfo.CurrentCulture = originalCulture; + CultureInfo.CurrentUICulture = originalUICulture; + Console.SetOut(originalOut); + } + } +} diff --git a/src/tools/ResultsComparer.Tests/ResultsComparer.Tests.csproj b/src/tools/ResultsComparer.Tests/ResultsComparer.Tests.csproj new file mode 100644 index 00000000000..5611c64bee9 --- /dev/null +++ b/src/tools/ResultsComparer.Tests/ResultsComparer.Tests.csproj @@ -0,0 +1,13 @@ + + + net11.0 + enable + enable + false + true + + + + + + diff --git a/src/tools/ResultsComparer.Tests/ResultsComparerTestData.cs b/src/tools/ResultsComparer.Tests/ResultsComparerTestData.cs new file mode 100644 index 00000000000..323f66a9455 --- /dev/null +++ b/src/tools/ResultsComparer.Tests/ResultsComparerTestData.cs @@ -0,0 +1,52 @@ +using System.Globalization; +using System.Linq; + +namespace ResultsComparer.Tests; + +internal static class ResultsComparerTestData +{ + internal static string CreateBdnJson( + string title = "SampleBenchmark-20240504-182513", + string fullName = "Demo.Namespace.SampleBenchmark", + string? benchmarkNamespace = "Demo.Namespace", + double[]? originalValues = null, + double? median = null, + long? allocatedBytes = 42, + string benchmarkDotNetVersion = "0.13.10", + string osVersion = "Windows 11 (10.0.26100)", + string processorName = "Intel Core i7-8700", + string architecture = "X64") + { + originalValues ??= [100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111]; + median ??= originalValues.OrderBy(v => v).ElementAt(originalValues.Length / 2); + string values = string.Join(", ", originalValues.Select(v => v.ToString(CultureInfo.InvariantCulture))); + string namespaceField = benchmarkNamespace is null ? "null" : $"\"{benchmarkNamespace}\""; + string memoryField = allocatedBytes is null + ? "\"Memory\": {}" + : $"\"Memory\": {{ \"BytesAllocatedPerOperation\": {allocatedBytes.Value.ToString(CultureInfo.InvariantCulture)} }}"; + + return $$""" + { + "Title": "{{title}}", + "HostEnvironmentInfo": { + "BenchmarkDotNetVersion": "{{benchmarkDotNetVersion}}", + "OsVersion": "{{osVersion}}", + "ProcessorName": "{{processorName}}", + "Architecture": "{{architecture}}" + }, + "Benchmarks": [ + { + "Namespace": {{namespaceField}}, + "FullName": "{{fullName}}", + "Statistics": { + "OriginalValues": [{{values}}], + "N": {{originalValues.Length}}, + "Median": {{median.Value.ToString(CultureInfo.InvariantCulture)}} + }, + {{memoryField}} + } + ] + } + """; + } +} diff --git a/src/tools/ResultsComparer.Tests/StatsTests.cs b/src/tools/ResultsComparer.Tests/StatsTests.cs new file mode 100644 index 00000000000..a8b8cfeb38e --- /dev/null +++ b/src/tools/ResultsComparer.Tests/StatsTests.cs @@ -0,0 +1,81 @@ +using System; +using System.IO; +using DataTransferContracts; +using Perfolizer.Mathematics.SignificanceTesting; +using Xunit; + +namespace ResultsComparer.Tests; + +[Collection("Console output")] +public class StatsTests +{ + [Fact] + public void GetSimplifiedOSNameRemovesParentheticalSuffix() + { + Assert.Equal("Windows 11", global::ResultsComparer.Stats.GetSimplifiedOSName("Windows 11 (10.0.26100)")); + } + + [Fact] + public void PrintAggregatesTotalsAndEmitsSectionsOnce() + { + var stats = new global::ResultsComparer.Stats(); + var environment = new HostEnvironmentInfo + { + Architecture = "X64", + OsVersion = "Windows 11 (10.0.26100)", + ProcessorName = "Intel Core i7-8700" + }; + var benchmark = new Benchmark + { + Namespace = "Demo.Namespace", + Statistics = new Statistics + { + OriginalValues = new[] { 1.0, 1.1, 1.2 }, + N = 3, + Median = 1.1 + }, + Memory = new Memory() + }; + + stats.Record(EquivalenceTestConclusion.Same, environment, benchmark); + stats.Record(EquivalenceTestConclusion.Faster, environment, benchmark); + stats.Record(EquivalenceTestConclusion.Slower, environment, benchmark); + stats.Record(EquivalenceTestConclusion.Unknown, environment, benchmark); + stats.Record(global::ResultsComparer.Stats.Noise, environment, benchmark); + + using var writer = new StringWriter(); + var originalOut = Console.Out; + Console.SetOut(writer); + try + { + stats.Print(); + var firstOutput = writer.ToString(); + + Assert.Contains("## Statistics", firstOutput); + Assert.Contains("Total: 5", firstOutput); + Assert.Contains("## Statistics per Architecture", firstOutput); + Assert.Contains("## Statistics per Operating System", firstOutput); + Assert.Contains("## Statistics per Namespace", firstOutput); + Assert.Contains("Demo.Namespace", firstOutput); + + writer.GetStringBuilder().Clear(); + stats.Print(); + + Assert.Equal(string.Empty, writer.ToString()); + } + finally + { + Console.SetOut(originalOut); + } + } + + [Fact] + public void RecordThrowsForUnsupportedConclusion() + { + var stats = new global::ResultsComparer.Stats(); + var environment = new HostEnvironmentInfo { Architecture = "X64", OsVersion = "Windows 11 (10.0.26100)" }; + var benchmark = new Benchmark { Statistics = new Statistics { OriginalValues = [1.0], N = 1, Median = 1.0 }, Memory = new Memory() }; + + Assert.Throws(() => stats.Record((EquivalenceTestConclusion)999, environment, benchmark)); + } +} diff --git a/src/tools/ResultsComparer/Data.cs b/src/tools/ResultsComparer/Data.cs index 3a61ad01a03..a663f1e8f63 100644 --- a/src/tools/ResultsComparer/Data.cs +++ b/src/tools/ResultsComparer/Data.cs @@ -130,6 +130,9 @@ static Version GetVersion(BdnResult bdnResult) private static string GetMoniker(string key) { + if (string.IsNullOrEmpty(key)) + return null; + if (key.Contains("net6")) // some files are net6.0, some are missing the dot (net60) return "net6.0"; if (key.Contains("nativeaot6")) @@ -158,14 +161,14 @@ private static string GetMoniker(string key) return "nativeaot9.0-preview" + key[key.IndexOf("nativeaot9.0-preview") + "nativeaot9.0-preview".Length]; if (key.Contains("net9.0")) return "net9.0"; - if (key.StartsWith("net10.0")) + if (key.Contains("net10.0")) return "net10.0"; - if (key.StartsWith("nativeaot10.0")) - return key; - if (key.StartsWith("net11.0")) + if (key.Contains("nativeaot10.0")) + return "nativeaot10.0"; + if (key.Contains("net11.0")) return "net11.0"; - if (key.StartsWith("nativeaot11.0")) - return key; + if (key.Contains("nativeaot11.0")) + return "nativeaot11.0"; return null; } diff --git a/src/tools/ResultsComparer/Directory.Packages.props b/src/tools/ResultsComparer/Directory.Packages.props index 28ef11f8a62..d910ce303ad 100644 --- a/src/tools/ResultsComparer/Directory.Packages.props +++ b/src/tools/ResultsComparer/Directory.Packages.props @@ -5,7 +5,7 @@ - + diff --git a/src/tools/ResultsComparer/MultipleInputsComparer.cs b/src/tools/ResultsComparer/MultipleInputsComparer.cs index 986a879e2b8..ba3ee7656ed 100644 --- a/src/tools/ResultsComparer/MultipleInputsComparer.cs +++ b/src/tools/ResultsComparer/MultipleInputsComparer.cs @@ -141,13 +141,15 @@ static long GetMetricValue(Benchmark result) var baseValues = info.baseResult.Statistics.OriginalValues; var diffValues = info.diffResult.Statistics.OriginalValues; - var userTresholdResult = StatisticalTestHelper.CalculateTost(MannWhitneyTest.Instance, baseValues, diffValues, args.StatisticalTestThreshold); + var userThresholdResult = StatisticalTestHelper.CalculateTost(MannWhitneyTest.Instance, baseValues, diffValues, args.StatisticalTestThreshold); var noiseResult = StatisticalTestHelper.CalculateTost(MannWhitneyTest.Instance, baseValues, diffValues, args.NoiseThreshold); + var userConclusion = userThresholdResult.Conclusion == EquivalenceTestConclusion.Base ? EquivalenceTestConclusion.Same : userThresholdResult.Conclusion; + var noiseConclusion = noiseResult.Conclusion == EquivalenceTestConclusion.Base ? EquivalenceTestConclusion.Same : noiseResult.Conclusion; // filter noise (0.20 ns vs 0.25ns is 25% difference) - var conclusion = userTresholdResult.Conclusion != EquivalenceTestConclusion.Same && noiseResult.Conclusion == EquivalenceTestConclusion.Same + var conclusion = userConclusion != EquivalenceTestConclusion.Same && noiseConclusion == EquivalenceTestConclusion.Same ? Stats.Noise - : userTresholdResult.Conclusion == EquivalenceTestConclusion.Base ? EquivalenceTestConclusion.Same : userTresholdResult.Conclusion; + : userConclusion; stats.Record(conclusion, info.baseEnv, info.baseResult); diff --git a/src/tools/ResultsComparer/Program.cs b/src/tools/ResultsComparer/Program.cs index 8bb8d1294cf..92620cb3470 100644 --- a/src/tools/ResultsComparer/Program.cs +++ b/src/tools/ResultsComparer/Program.cs @@ -172,7 +172,9 @@ private static bool TryGetPaths(DirectoryInfo input, string basePattern, string } private static Regex[] GetFilters(string[] filters) - => filters.Select(pattern => new Regex(WildcardToRegex(pattern), RegexOptions.IgnoreCase | RegexOptions.CultureInvariant)).ToArray(); + => (filters ?? Array.Empty()) + .Select(pattern => new Regex(WildcardToRegex(pattern), RegexOptions.IgnoreCase | RegexOptions.CultureInvariant)) + .ToArray(); // https://stackoverflow.com/a/6907849/5852046 not perfect but should work for all we need private static string WildcardToRegex(string pattern) => $"^{Regex.Escape(pattern).Replace(@"\*", ".*").Replace(@"\?", ".")}$"; diff --git a/src/tools/ResultsComparer/Properties/AssemblyInfo.cs b/src/tools/ResultsComparer/Properties/AssemblyInfo.cs new file mode 100644 index 00000000000..11fac6ab430 --- /dev/null +++ b/src/tools/ResultsComparer/Properties/AssemblyInfo.cs @@ -0,0 +1,3 @@ +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("ResultsComparer.Tests")] diff --git a/src/tools/ResultsComparer/ResultsComparer.sln b/src/tools/ResultsComparer/ResultsComparer.sln index 951a4d0fb5d..5985914bae2 100644 --- a/src/tools/ResultsComparer/ResultsComparer.sln +++ b/src/tools/ResultsComparer/ResultsComparer.sln @@ -2,15 +2,44 @@ Microsoft Visual Studio Solution File, Format Version 12.00 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ResultsComparer", "ResultsComparer.csproj", "{00859394-44F8-466B-8624-41578CA94009}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ResultsComparer.Tests", "..\ResultsComparer.Tests\ResultsComparer.Tests.csproj", "{F1CA7160-F99B-484F-975E-04C4638BCFAF}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU + Debug|x64 = Debug|x64 + Debug|x86 = Debug|x86 Release|Any CPU = Release|Any CPU + Release|x64 = Release|x64 + Release|x86 = Release|x86 EndGlobalSection GlobalSection(ProjectConfigurationPlatforms) = postSolution {00859394-44F8-466B-8624-41578CA94009}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {00859394-44F8-466B-8624-41578CA94009}.Debug|Any CPU.Build.0 = Debug|Any CPU + {00859394-44F8-466B-8624-41578CA94009}.Debug|x64.ActiveCfg = Debug|Any CPU + {00859394-44F8-466B-8624-41578CA94009}.Debug|x64.Build.0 = Debug|Any CPU + {00859394-44F8-466B-8624-41578CA94009}.Debug|x86.ActiveCfg = Debug|Any CPU + {00859394-44F8-466B-8624-41578CA94009}.Debug|x86.Build.0 = Debug|Any CPU {00859394-44F8-466B-8624-41578CA94009}.Release|Any CPU.ActiveCfg = Release|Any CPU {00859394-44F8-466B-8624-41578CA94009}.Release|Any CPU.Build.0 = Release|Any CPU + {00859394-44F8-466B-8624-41578CA94009}.Release|x64.ActiveCfg = Release|Any CPU + {00859394-44F8-466B-8624-41578CA94009}.Release|x64.Build.0 = Release|Any CPU + {00859394-44F8-466B-8624-41578CA94009}.Release|x86.ActiveCfg = Release|Any CPU + {00859394-44F8-466B-8624-41578CA94009}.Release|x86.Build.0 = Release|Any CPU + {F1CA7160-F99B-484F-975E-04C4638BCFAF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {F1CA7160-F99B-484F-975E-04C4638BCFAF}.Debug|Any CPU.Build.0 = Debug|Any CPU + {F1CA7160-F99B-484F-975E-04C4638BCFAF}.Debug|x64.ActiveCfg = Debug|Any CPU + {F1CA7160-F99B-484F-975E-04C4638BCFAF}.Debug|x64.Build.0 = Debug|Any CPU + {F1CA7160-F99B-484F-975E-04C4638BCFAF}.Debug|x86.ActiveCfg = Debug|Any CPU + {F1CA7160-F99B-484F-975E-04C4638BCFAF}.Debug|x86.Build.0 = Debug|Any CPU + {F1CA7160-F99B-484F-975E-04C4638BCFAF}.Release|Any CPU.ActiveCfg = Release|Any CPU + {F1CA7160-F99B-484F-975E-04C4638BCFAF}.Release|Any CPU.Build.0 = Release|Any CPU + {F1CA7160-F99B-484F-975E-04C4638BCFAF}.Release|x64.ActiveCfg = Release|Any CPU + {F1CA7160-F99B-484F-975E-04C4638BCFAF}.Release|x64.Build.0 = Release|Any CPU + {F1CA7160-F99B-484F-975E-04C4638BCFAF}.Release|x86.ActiveCfg = Release|Any CPU + {F1CA7160-F99B-484F-975E-04C4638BCFAF}.Release|x86.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE EndGlobalSection EndGlobal diff --git a/src/tools/ResultsComparer/Stats.cs b/src/tools/ResultsComparer/Stats.cs index e230c0478a4..4e0a472c33f 100644 --- a/src/tools/ResultsComparer/Stats.cs +++ b/src/tools/ResultsComparer/Stats.cs @@ -77,7 +77,7 @@ static void Print(Dictionary dictionary, string name) } } - internal static string GetSimplifiedOSName(string text) => text.Split('(')[0]; + internal static string GetSimplifiedOSName(string text) => text.Split('(')[0].TrimEnd(); private class PerConclusion { diff --git a/src/tools/ResultsComparer/TwoInputsComparer.cs b/src/tools/ResultsComparer/TwoInputsComparer.cs index 89209aefb58..a929e937d27 100644 --- a/src/tools/ResultsComparer/TwoInputsComparer.cs +++ b/src/tools/ResultsComparer/TwoInputsComparer.cs @@ -35,15 +35,17 @@ internal static void Compare(TwoInputsOptions args) var baseValues = baseResult.Statistics.OriginalValues.ToArray(); var diffValues = diffResult.Statistics.OriginalValues.ToArray(); - var userTresholdResult = StatisticalTestHelper.CalculateTost(MannWhitneyTest.Instance, baseValues, diffValues, args.StatisticalTestThreshold); - if (userTresholdResult.Conclusion == EquivalenceTestConclusion.Same) + var userThresholdResult = StatisticalTestHelper.CalculateTost(MannWhitneyTest.Instance, baseValues, diffValues, args.StatisticalTestThreshold); + if (userThresholdResult.Conclusion == EquivalenceTestConclusion.Same + || userThresholdResult.Conclusion == EquivalenceTestConclusion.Base) continue; var noiseResult = StatisticalTestHelper.CalculateTost(MannWhitneyTest.Instance, baseValues, diffValues, args.NoiseThreshold); - if (noiseResult.Conclusion == EquivalenceTestConclusion.Same) + if (noiseResult.Conclusion == EquivalenceTestConclusion.Same + || noiseResult.Conclusion == EquivalenceTestConclusion.Base) continue; - yield return (id, baseResult, diffResult, userTresholdResult.Conclusion); + yield return (id, baseResult, diffResult, userThresholdResult.Conclusion); } } diff --git a/src/tools/ScenarioMeasurement/Startup.Tests/StartupTests.cs b/src/tools/ScenarioMeasurement/Startup.Tests/StartupTests.cs index 83ceb5c79c0..e94ceaa3b0d 100644 --- a/src/tools/ScenarioMeasurement/Startup.Tests/StartupTests.cs +++ b/src/tools/ScenarioMeasurement/Startup.Tests/StartupTests.cs @@ -3,7 +3,9 @@ using System; using System.Collections.Generic; using System.IO; +using System.Security.Principal; using System.Threading; +using System.Runtime.Versioning; using Xunit; @@ -131,10 +133,27 @@ public sealed class WindowsOnly : FactAttribute { public WindowsOnly() { - if (Environment.OSVersion.Platform != PlatformID.Win32NT) + if (!OperatingSystem.IsWindows()) { Skip = "Skip on non-windows platform"; } + else if (!IsRunningAsAdministrator()) + { + Skip = "Requires administrator privileges to start ETW sessions"; + } + } + + [SupportedOSPlatform("windows")] + private static bool IsRunningAsAdministrator() + { + if (!OperatingSystem.IsWindows()) + { + return false; + } + + using WindowsIdentity identity = WindowsIdentity.GetCurrent(); + WindowsPrincipal principal = new(identity); + return principal.IsInRole(WindowsBuiltInRole.Administrator); } } @@ -142,7 +161,7 @@ public sealed class LinuxOnly : FactAttribute { public LinuxOnly() { - if(Environment.OSVersion.Platform != PlatformID.Unix) + if (!OperatingSystem.IsLinux()) { Skip = "Skip on non-linux platform"; } From a6a5a3df27bcb4e82f62a5ded7727dcaedc57e36 Mon Sep 17 00:00:00 2001 From: "Vincent Bu (Centific Technologies Inc)" Date: Thu, 21 May 2026 09:30:17 +0800 Subject: [PATCH 3/3] fix bugs for createsuites and runsuites remove dup console output in presentation stage remove dup lines from markdown table fix bug for run command --- .../Microbenchmarks/MicrobenchmarkComparisonResults.cs | 10 +++++----- .../Presentation/Microbenchmarks/Markdown.cs | 10 +++++----- .../Commands/Microbenchmark/MicrobenchmarkCommand.cs | 1 - .../Commands/RunCommand/CreateSuiteCommand.cs | 4 ++-- .../Commands/RunCommand/RunCommand.cs | 2 +- .../Commands/RunCommand/RunSuiteCommand.cs | 8 ++++---- 6 files changed, 17 insertions(+), 18 deletions(-) diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkComparisonResults.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkComparisonResults.cs index 4a20b557394..cbba03262ec 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkComparisonResults.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkComparisonResults.cs @@ -17,11 +17,11 @@ public MicrobenchmarkComparisonResults(string baselineName, string runName, IEnu public IEnumerable Comparisons { get; } public IEnumerable Ordered => Comparisons.Where(o => !double.IsNaN(o.MeanDiff)).OrderByDescending(m => m.MeanDiffPerc); - public IEnumerable LargeRegressions => Ordered.Where(o => o.MeanDiffPerc > 20); - public IEnumerable LargeImprovements => Ordered.Where(o => o.MeanDiffPerc < -20).OrderBy(g => g.MeanDiffPerc); - public IEnumerable Regressions => Ordered.Where(o => o.MeanDiffPerc < 20 && o.MeanDiffPerc > 5); - public IEnumerable Improvements => Ordered.Where(o => o.MeanDiffPerc > -20 && o.MeanDiffPerc < -5).OrderBy(g => g.MeanDiffPerc); + public IEnumerable LargeRegressions => Ordered.Where(o => o.MeanDiffPerc >= 20); + public IEnumerable LargeImprovements => Ordered.Where(o => o.MeanDiffPerc <= -20).OrderBy(g => g.MeanDiffPerc); + public IEnumerable Regressions => Ordered.Where(o => o.MeanDiffPerc < 20 && o.MeanDiffPerc >= 5); + public IEnumerable Improvements => Ordered.Where(o => o.MeanDiffPerc > -20 && o.MeanDiffPerc <= -5).OrderBy(g => g.MeanDiffPerc); public IEnumerable StaleRegressions => Ordered.Where((o => o.MeanDiffPerc > 0 && o.MeanDiffPerc < 5)); - public IEnumerable StaleImprovements => Ordered.Where((o => o.MeanDiffPerc < 0 && o.MeanDiffPerc > -5)).OrderBy(g => g.MeanDiffPerc); + public IEnumerable StaleImprovements => Ordered.Where((o => o.MeanDiffPerc <= 0 && o.MeanDiffPerc > -5)).OrderBy(g => g.MeanDiffPerc); } } diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Markdown.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Markdown.cs index fcc488c050a..7636f46f314 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Markdown.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Markdown.cs @@ -51,12 +51,12 @@ internal static void AddDetailsOfSingleComparison(this StreamWriter sw, Microben sw.WriteLine("\n"); // Large Regressions - sw.WriteLine($"### Large Regressions (>20%): {comparisonResult.LargeRegressions.Count()} \n"); + sw.WriteLine($"### Large Regressions (>=20%): {comparisonResult.LargeRegressions.Count()} \n"); sw.AddTableForSingleCriteria(configuration, comparisonResult.LargeRegressions); sw.WriteLine("\n"); // Large Improvements - sw.WriteLine($"### Large Improvements (>20%): {comparisonResult.LargeImprovements.Count()} \n"); + sw.WriteLine($"### Large Improvements (>=20%): {comparisonResult.LargeImprovements.Count()} \n"); sw.AddTableForSingleCriteria(configuration, comparisonResult.LargeImprovements); sw.WriteLine("\n"); @@ -71,7 +71,7 @@ internal static void AddDetailsOfSingleComparison(this StreamWriter sw, Microben sw.WriteLine("\n"); // Stale Regressions - sw.WriteLine($"### Stale Regressions (Same or percent difference within 5% margin): {comparisonResult.StaleRegressions.Count()} \n"); + sw.WriteLine($"### Stale Regressions (Percent difference within 5% margin): {comparisonResult.StaleRegressions.Count()} \n"); sw.AddTableForSingleCriteria(configuration, comparisonResult.StaleRegressions); sw.WriteLine("\n"); @@ -116,8 +116,8 @@ internal static void AddDetailsOfSingleComparison(this StreamWriter sw, Microben sw.WriteLine("\n"); // Stale Regressions - var staleRegressions = API.GoodLinq.Where(ordered, o => o.OtherMetricsDiffPerc[metric] >= 0.0 && o.OtherMetricsDiffPerc[metric] < 5); - sw.WriteLine($"### Stale Regressions (Same or percent difference within 5% margin): {staleRegressions.Count()} \n"); + var staleRegressions = API.GoodLinq.Where(ordered, o => o.OtherMetricsDiffPerc[metric] > 0.0 && o.OtherMetricsDiffPerc[metric] < 5); + sw.WriteLine($"### Stale Regressions (Percent difference within 5% margin): {staleRegressions.Count()} \n"); sw.AddTableForSingleCriteria(configuration, staleRegressions, metric); sw.WriteLine("\n"); diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/Microbenchmark/MicrobenchmarkCommand.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/Microbenchmark/MicrobenchmarkCommand.cs index e411bbd2f65..4de521e5cc8 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/Microbenchmark/MicrobenchmarkCommand.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/Microbenchmark/MicrobenchmarkCommand.cs @@ -202,7 +202,6 @@ public static MicrobenchmarkOutputResults RunMicrobenchmarks(MicrobenchmarkConfi MicrobenchmarkAnalyzeCommand.Present(configuration, comparisonResultsGroupedName, executionDetails); // Execution details aren't available for the analysis-only mode. Directory.SetCurrentDirectory(currentDirectory); - AnsiConsole.Markup($"[bold green] ({DateTime.Now}) Wrote Microbenchmark Results to: {Markup.Escape(Path.Combine(configuration.Output.Path, "Results.md"))} [/]"); return new MicrobenchmarkOutputResults(executionDetails, comparisonResultsGroupedName); } } diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/RunCommand/CreateSuiteCommand.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/RunCommand/CreateSuiteCommand.cs index 96a6943019b..213385812ea 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/RunCommand/CreateSuiteCommand.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/RunCommand/CreateSuiteCommand.cs @@ -91,7 +91,7 @@ public static Dictionary CreateSuites(InputConfiguration configu configurationMap["GCPerfSim"] = gcPerfSimBase; string microbenchmarkBase = CreateMicrobenchmarkSuite(configuration, suitePath); - configurationMap["Microbenchmark"] = microbenchmarkBase; + configurationMap["Microbenchmarks"] = microbenchmarkBase; string aspnetBenchmarkBase = CreateASPNetBenchmarkSuite(configuration, suitePath); configurationMap["ASPNetBenchmarks"] = aspnetBenchmarkBase; @@ -158,7 +158,7 @@ internal static string CreateASPNetBenchmarkSuite(InputConfiguration inputConfig internal static string CreateMicrobenchmarkSuite(InputConfiguration inputConfiguration, string suitePath) { - string microbenchmarkSuitePath = Path.Combine(suitePath, "Microbenchmark"); + string microbenchmarkSuitePath = Path.Combine(suitePath, "Microbenchmarks"); Core.Utilities.TryCreateDirectory(microbenchmarkSuitePath); diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/RunCommand/RunCommand.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/RunCommand/RunCommand.cs index 0cd4fc33087..1483aa011d1 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/RunCommand/RunCommand.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/RunCommand/RunCommand.cs @@ -138,7 +138,7 @@ public override int Execute([NotNull] CommandContext context, [NotNull] RunComma AnsiConsole.WriteLine(); AnsiConsole.Write(new Rule("Running Microbenchmarks")); AnsiConsole.WriteLine(); - string microbenchmarkBase = configurationMap["Microbenchmark"]; + string microbenchmarkBase = configurationMap["Microbenchmarks"]; string[] microbenchmarkConfigurations = Directory.GetFiles(microbenchmarkBase, "*.yaml"); HashSet uniqueMicrobenchmarks = new(); diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/RunCommand/RunSuiteCommand.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/RunCommand/RunSuiteCommand.cs index 2fc52eb00ff..baac3f6ffca 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/RunCommand/RunSuiteCommand.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/RunCommand/RunSuiteCommand.cs @@ -22,7 +22,7 @@ public sealed class RunSuiteCommandSettings : CommandSettings public override int Execute([NotNull] CommandContext context, [NotNull] RunSuiteCommandSettings settings) { - if (!string.IsNullOrEmpty(settings.SuiteBasePath) || !Directory.Exists(settings.SuiteBasePath)) + if (string.IsNullOrEmpty(settings.SuiteBasePath) || !Directory.Exists(settings.SuiteBasePath)) { throw new ArgumentNullException($"{nameof(RunSuiteCommandSettings)}: {nameof(settings.SuiteBasePath)} was either null or the directory doesn't exists."); } @@ -50,7 +50,7 @@ public static void RunSuite(Dictionary configuration) catch (Exception e) { - AnsiConsole.Write($"[red] GCPerfSim Configuration: {c} failed with {e.Message} [/]"); + AnsiConsole.MarkupLine($"[red] GCPerfSim Configuration: {c} failed with {e.Message} [/]"); } } @@ -66,7 +66,7 @@ public static void RunSuite(Dictionary configuration) catch (Exception e) { - AnsiConsole.Write($"[red] Microbenchmark Configuration: {c} failed with {e.Message} [/]"); + AnsiConsole.MarkupLine($"[red] Microbenchmark Configuration: {c} failed with {e.Message} [/]"); } } @@ -83,7 +83,7 @@ public static void RunSuite(Dictionary configuration) catch (Exception e) { - AnsiConsole.Write($"[red] ASPNet Configuration: {c} failed with {e.Message} [/]"); + AnsiConsole.MarkupLine($"[red] ASPNet Configuration: {c} failed with {e.Message} [/]"); } } }