feat(cv): detection/OCR test families, evaluation metrics, and train/clone/predict fixes

.github/test-shards.yml

@@ -659,7 +659,7 @@ shard:
     framework: net10.0
     filter: >-
       FullyQualifiedName~ModelFamilyTests.Generated&
-      (FullyQualifiedName~Generated.Tem|FullyQualifiedName~Generated.Tex|FullyQualifiedName~Generated.Tho|FullyQualifiedName~Generated.Thr|FullyQualifiedName~Generated.TiD|FullyQualifiedName~Generated.TiM|FullyQualifiedName~Generated.Tim|FullyQualifiedName~Generated.Tin|FullyQualifiedName~Generated.Tit|FullyQualifiedName~Generated.Too|FullyQualifiedName~Generated.Tor|FullyQualifiedName~Generated.Tra|FullyQualifiedName~Generated.Tri)
+      (FullyQualifiedName~Generated.Tem|FullyQualifiedName~Generated.Tex|FullyQualifiedName~Generated.Tho|FullyQualifiedName~Generated.Thr|FullyQualifiedName~Generated.TiD|FullyQualifiedName~Generated.TiM|FullyQualifiedName~Generated.Tim|FullyQualifiedName~Generated.Tin|FullyQualifiedName~Generated.Tit|FullyQualifiedName~Generated.Too|FullyQualifiedName~Generated.Tor|FullyQualifiedName~Generated.Tra|FullyQualifiedName~Generated.Tri|FullyQualifiedName~Generated.TrO)
   - name: ModelFamily - Generated Layers U
     project: tests/AiDotNet.Tests/AiDotNetTests.csproj
     framework: net10.0

review-tests/Pr2154.APRangeBenchmark/Pr2154.APRangeBenchmark.csproj

@@ -0,0 +1,13 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net10.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+    <IsPackable>false</IsPackable>
+    <ReviewedSourceRoot Condition="'$(ReviewedSourceRoot)' == ''">$(MSBuildThisFileDirectory)../../src</ReviewedSourceRoot>
+  </PropertyGroup>
+  <ItemGroup>
+    <ProjectReference Include="$(ReviewedSourceRoot)/AiDotNet.csproj" />
+  </ItemGroup>
+</Project>

review-tests/Pr2154.APRangeBenchmark/Program.cs

@@ -0,0 +1,147 @@
+using System.Diagnostics;
+using System.Runtime.CompilerServices;
+using System.Security.Cryptography;
+using System.Text.Json;
+using AiDotNet.Augmentation.Image;
+using AiDotNet.ComputerVision.Detection.ObjectDetection;
+using AiDotNet.Metrics;
+using AiDotNet.Tensors.Engines;
+
+// A CPU metric microbenchmark, not a detector/GPU pipeline benchmark. Identical source, inputs,
+// warmups and iteration counts are used with each separately retained production assembly.
+const int seed = 2154;
+const int imageCount = 12;
+const int classCount = 4;
+const int boxesPerClass = 24;
+#if !AP_WORKLOAD_COUNTER
+const int measuredRuns = 9;
+#endif
+var random = new Random(seed);
+var predictions = new List<IReadOnlyList<Detection<double>>>();
+var truth = new List<IReadOnlyList<Detection<double>>>();
+for (int image = 0; image < imageCount; image++)
+{
+    var actual = new List<Detection<double>>();
+    var predicted = new List<Detection<double>>();
+    for (int classId = 0; classId < classCount; classId++)
+    {
+        for (int box = 0; box < boxesPerClass; box++)
+        {
+            double x = box % 6 * 12;
+            double y = box / 6 * 12;
+            actual.Add(new Detection<double>(new BoundingBox<double>(x, y, x + 10, y + 10), classId, 1));
+            for (int duplicate = 0; duplicate < 3; duplicate++)
+            {
+                double width = 4 + random.NextDouble() * 6;
+                double height = 4 + random.NextDouble() * 6;
+                predicted.Add(new Detection<double>(new BoundingBox<double>(x, y, x + width, y + height), classId, random.NextDouble()));
+            }
+        }
+    }
+    truth.Add(actual);
+    predictions.Add(predicted);
+}
+
+string assembly = typeof(ObjectDetectionMetrics<double>).Assembly.Location;
+using var assemblyStream = File.OpenRead(assembly);
+string assemblyHash = Convert.ToHexString(SHA256.HashData(assemblyStream));
+var metrics = new ObjectDetectionMetrics<double>();
+var cases = new[]
+{
+    (MetricWorkload.SingleMeanAveragePrecision, 1),
+    (MetricWorkload.FullPrecisionRecallCurve, 1),
+    (MetricWorkload.ThresholdRange, 1),
+    (MetricWorkload.ThresholdRange, 10),
+    (MetricWorkload.ThresholdRange, 32),
+    (MetricWorkload.ThresholdRange, 33),
+    (MetricWorkload.ThresholdRange, 65)
+};
+foreach (var (workload, thresholds) in cases)
+{
+    double step = thresholds == 10 ? 0.05 : 1.0 / 128;
+    double maximum = 0.5 + (thresholds - 1) * step;
+    double Score()
+    {
+        switch (workload)
+        {
+            case MetricWorkload.SingleMeanAveragePrecision:
+                return metrics.MeanAveragePrecision(predictions, truth);
+            case MetricWorkload.FullPrecisionRecallCurve:
+                var curve = metrics.PrecisionRecallCurve(predictions, truth, 0);
+                double checksum = 0;
+                for (int point = 0; point < curve.Precision.Length; point++)
+                    checksum += curve.Precision[point] + curve.Recall[point];
+                return checksum;
+            case MetricWorkload.ThresholdRange:
+                return metrics.MeanAveragePrecisionRange(predictions, truth, 0.5, maximum, step);
+            default:
+                throw new ArgumentOutOfRangeException(nameof(workload));
+        }
+    }
+#if AP_WORKLOAD_COUNTER
+    WorkloadProbe.IoUCalls = 0;
+    double result = Score();
+    Console.WriteLine(JsonSerializer.Serialize(new
+    {
+        instrumentedAssemblyHash = assemblyHash,
+        workload = workload.ToString(),
+        thresholds,
+        minimumIoU = 0.5,
+        maximumIoU = maximum,
+        iouStep = step,
+        score = result,
+        iouCalls = WorkloadProbe.IoUCalls
+    }));
+#else
+    double expected = Score();
+    var warmupTimer = Stopwatch.StartNew();
+    do
+    {
+        if (Score() != expected) throw new InvalidOperationException("Warmup changed the deterministic metric score.");
+    } while (warmupTimer.Elapsed < TimeSpan.FromSeconds(1));
+
+    var times = new double[measuredRuns];
+    var allocations = new long[measuredRuns];
+    for (int iteration = 0; iteration < measuredRuns; iteration++)
+    {
+        long allocated = GC.GetAllocatedBytesForCurrentThread();
+        long start = Stopwatch.GetTimestamp();
+        double actual = Score();
+        times[iteration] = Stopwatch.GetElapsedTime(start).TotalMilliseconds;
+        allocations[iteration] = GC.GetAllocatedBytesForCurrentThread() - allocated;
+        if (actual != expected) throw new InvalidOperationException("Measured run changed the deterministic metric score.");
+    }
+    Array.Sort(times);
+    Array.Sort(allocations);
+    Console.WriteLine(JsonSerializer.Serialize(new
+    {
+        assemblyHash,
+        seed,
+        imageCount,
+        classCount,
+        boxesPerClass,
+        predictions = imageCount * classCount * boxesPerClass * 3,
+        groundTruth = imageCount * classCount * boxesPerClass,
+        workload = workload.ToString(),
+        thresholds,
+        minimumIoU = 0.5,
+        maximumIoU = maximum,
+        iouStep = step,
+        measuredRuns,
+        score = expected,
+        medianMilliseconds = times[measuredRuns / 2],
+        minimumMilliseconds = times[0],
+        medianAllocatedBytes = allocations[measuredRuns / 2]
+    }));
+#endif
+}
+
+internal enum MetricWorkload { SingleMeanAveragePrecision, FullPrecisionRecallCurve, ThresholdRange }
+
+internal static class BenchmarkEnvironment
+{
+    // Select CPU before Main's workload is touched. The engine's static initialization may
+    // probe GPUs first; that startup is outside the warmup and measurement intervals.
+    [ModuleInitializer]
+    internal static void Initialize() => AiDotNetEngine.ResetToCpu();
+}

review-tests/Pr2154.APRangeWorkload/Pr2154.APRangeWorkload.csproj

@@ -0,0 +1,29 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net10.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+    <IsPackable>false</IsPackable>
+    <DefineConstants>$(DefineConstants);AP_WORKLOAD_COUNTER</DefineConstants>
+    <!-- The local instrumented metrics source intentionally wins over the imported production
+         type. Production BoundingBox/Detection/numerics remain the actual referenced library. -->
+    <NoWarn>$(NoWarn);CS0436</NoWarn>
+    <ReviewedDependencyDirectory Condition="'$(ReviewedDependencyDirectory)' == ''">$(MSBuildThisFileDirectory)../Pr2154.APRangeBenchmark/bin/Release/net10.0</ReviewedDependencyDirectory>
+  </PropertyGroup>
+  <ItemGroup>
+    <Reference Include="AiDotNet">
+      <HintPath>$(ReviewedDependencyDirectory)/AiDotNet.dll</HintPath>
+      <Private>false</Private>
+    </Reference>
+    <Reference Include="AiDotNet.Tensors">
+      <HintPath>$(ReviewedDependencyDirectory)/AiDotNet.Tensors.dll</HintPath>
+      <Private>false</Private>
+    </Reference>
+    <Compile Include="../Pr2154.APRangeBenchmark/Program.cs" Link="Program.cs" />
+    <Compile Include="$(MetricSource)" Link="InstrumentedObjectDetectionMetrics.cs" />
+  </ItemGroup>
+  <Target Name="RequireInstrumentedSource" BeforeTargets="CoreCompile">
+    <Error Condition="'$(MetricSource)' == '' or !Exists('$(MetricSource)')" Text="Supply the actual metrics source with only IoU calls instrumented, via -p:MetricSource=absolute-path." />
+  </Target>
+</Project>

review-tests/Pr2154.APRangeWorkload/WorkloadProbe.cs

@@ -0,0 +1,17 @@
+global using AiDotNet.Tensors.Interfaces;
+global using AiDotNet.Tensors.Helpers;
+
+using AiDotNet.Augmentation.Image;
+
+// Only compiled into the isolated counter executable. The timed production assembly has no
+// counter, callback, subclassed geometry, or numeric-provider mutation.
+internal static class WorkloadProbe
+{
+    internal static long IoUCalls { get; set; }
+
+    internal static double CountIoU<T>(BoundingBox<T> prediction, BoundingBox<T> candidate) where T : struct
+    {
+        IoUCalls++;
+        return prediction.IoU(candidate);
+    }
+}

review-tests/Pr2154.ComputerVision/Pr2154.ComputerVision.csproj

@@ -0,0 +1,42 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <TargetFrameworks>net471;net8.0;net10.0</TargetFrameworks>
+    <LangVersion>latest</LangVersion>
+    <AssemblyName>AiDotNetTests</AssemblyName>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+    <IsTestProject>true</IsTestProject>
+    <IsPackable>false</IsPackable>
+    <EnableDefaultCompileItems>false</EnableDefaultCompileItems>
+    <ReviewedSourceRoot Condition="'$(ReviewedSourceRoot)' == ''">$(MSBuildThisFileDirectory)../../src</ReviewedSourceRoot>
+  </PropertyGroup>
+  <ItemGroup>
+    <ProjectReference Include="$(ReviewedSourceRoot)/AiDotNet.csproj" />
+    <ProjectReference Include="$(ReviewedSourceRoot)/AiDotNet.Generators/AiDotNet.Generators.csproj" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" />
+    <PackageReference Include="xunit" />
+    <PackageReference Include="xunit.runner.visualstudio" />
+    <PackageReference Include="Microsoft.CodeAnalysis.CSharp" />
+    <Compile Include="../../tests/AiDotNet.Tests/ModuleInitializer.cs" />
+    <Compile Include="../../tests/AiDotNet.Tests/GlobalUsings.cs" />
+    <Compile Include="../../tests/AiDotNet.Tests/Helpers/LicenseTestSupport.cs" />
+    <None Include="../../tests/AiDotNet.Tests/xunit.runner.json" Link="xunit.runner.json" CopyToOutputDirectory="PreserveNewest" />
+    <Compile Include="../../tests/AiDotNet.Tests/ModelFamilyTests/Base/DetectionModelTestBase.cs" />
+    <Compile Include="../../tests/AiDotNet.Tests/ModelFamilyTests/Base/ObjectDetectionTestBase.cs" />
+    <Compile Include="../../tests/AiDotNet.Tests/ModelFamilyTests/Base/ObjectDetectionPositiveFixture.cs" />
+    <Compile Include="../../tests/AiDotNet.Tests/ModelFamilyTests/Base/OCRTestBase.cs" />
+    <Compile Include="../../tests/AiDotNet.Tests/ModelFamilyTests/Base/TextDetectionTestBase.cs" />
+    <Compile Include="../../tests/AiDotNet.Tests/ModelFamilyTests/Base/ModelTestHelpers.cs" />
+    <Compile Include="../../tests/AiDotNet.Tests/Generators/CvNullableComponentReviewTests.cs" />
+    <Compile Include="../../tests/AiDotNet.Tests/UnitTests/ComputerVision/CvReviewRegressionTests.cs" />
+    <Compile Include="../../tests/AiDotNet.Tests/UnitTests/ComputerVision/CvInputBoundaryReviewTests.cs" />
+    <Compile Include="../../tests/AiDotNet.Tests/UnitTests/ComputerVision/CvTensorOpsEquivalenceTests.cs" />
+    <Compile Include="../../tests/AiDotNet.Tests/UnitTests/ComputerVision/FpnRoIPoolerTests.cs" />
+    <Compile Include="../../tests/AiDotNet.Tests/UnitTests/ComputerVision/NeckTopDownPathwayTests.cs" />
+    <Compile Include="../../tests/AiDotNet.Tests/UnitTests/ComputerVision/TrOCRIncrementalDecodingTests.cs" />
+    <Compile Include="../../tests/AiDotNet.Tests/UnitTests/Metrics/ObjectDetectionMetricsTests.cs" />
+    <Compile Include="../../tests/AiDotNet.Tests/UnitTests/Metrics/ObjectDetectionRangeCacheReviewTests.cs" />
+    <Compile Include="../../tests/AiDotNet.Tests/UnitTests/Metrics/TextDetectionMetricsTests.cs" />
+    <Compile Include="../../tests/AiDotNet.Tests/UnitTests/Metrics/TextRecognitionMetricsTests.cs" />
+  </ItemGroup>
+</Project>