diff --git a/.github/test-shards.yml b/.github/test-shards.yml index c8ebdea359..9a5169f3ef 100644 --- a/.github/test-shards.yml +++ b/.github/test-shards.yml @@ -659,7 +659,7 @@ shard: framework: net10.0 filter: >- FullyQualifiedName~ModelFamilyTests.Generated& - (FullyQualifiedName~Generated.Tem|FullyQualifiedName~Generated.Tex|FullyQualifiedName~Generated.Tho|FullyQualifiedName~Generated.Thr|FullyQualifiedName~Generated.TiD|FullyQualifiedName~Generated.TiM|FullyQualifiedName~Generated.Tim|FullyQualifiedName~Generated.Tin|FullyQualifiedName~Generated.Tit|FullyQualifiedName~Generated.Too|FullyQualifiedName~Generated.Tor|FullyQualifiedName~Generated.Tra|FullyQualifiedName~Generated.Tri) + (FullyQualifiedName~Generated.Tem|FullyQualifiedName~Generated.Tex|FullyQualifiedName~Generated.Tho|FullyQualifiedName~Generated.Thr|FullyQualifiedName~Generated.TiD|FullyQualifiedName~Generated.TiM|FullyQualifiedName~Generated.Tim|FullyQualifiedName~Generated.Tin|FullyQualifiedName~Generated.Tit|FullyQualifiedName~Generated.Too|FullyQualifiedName~Generated.Tor|FullyQualifiedName~Generated.Tra|FullyQualifiedName~Generated.Tri|FullyQualifiedName~Generated.TrO) - name: ModelFamily - Generated Layers U project: tests/AiDotNet.Tests/AiDotNetTests.csproj framework: net10.0 diff --git a/review-tests/Pr2154.APRangeBenchmark/Pr2154.APRangeBenchmark.csproj b/review-tests/Pr2154.APRangeBenchmark/Pr2154.APRangeBenchmark.csproj new file mode 100644 index 0000000000..a8fcfee1cd --- /dev/null +++ b/review-tests/Pr2154.APRangeBenchmark/Pr2154.APRangeBenchmark.csproj @@ -0,0 +1,13 @@ + + + Exe + net10.0 + enable + enable + false + $(MSBuildThisFileDirectory)../../src + + + + + diff --git a/review-tests/Pr2154.APRangeBenchmark/Program.cs b/review-tests/Pr2154.APRangeBenchmark/Program.cs new file mode 100644 index 0000000000..18b0ea8c0d --- /dev/null +++ b/review-tests/Pr2154.APRangeBenchmark/Program.cs @@ -0,0 +1,147 @@ +using System.Diagnostics; +using System.Runtime.CompilerServices; +using System.Security.Cryptography; +using System.Text.Json; +using AiDotNet.Augmentation.Image; +using AiDotNet.ComputerVision.Detection.ObjectDetection; +using AiDotNet.Metrics; +using AiDotNet.Tensors.Engines; + +// A CPU metric microbenchmark, not a detector/GPU pipeline benchmark. Identical source, inputs, +// warmups and iteration counts are used with each separately retained production assembly. +const int seed = 2154; +const int imageCount = 12; +const int classCount = 4; +const int boxesPerClass = 24; +#if !AP_WORKLOAD_COUNTER +const int measuredRuns = 9; +#endif +var random = new Random(seed); +var predictions = new List>>(); +var truth = new List>>(); +for (int image = 0; image < imageCount; image++) +{ + var actual = new List>(); + var predicted = new List>(); + for (int classId = 0; classId < classCount; classId++) + { + for (int box = 0; box < boxesPerClass; box++) + { + double x = box % 6 * 12; + double y = box / 6 * 12; + actual.Add(new Detection(new BoundingBox(x, y, x + 10, y + 10), classId, 1)); + for (int duplicate = 0; duplicate < 3; duplicate++) + { + double width = 4 + random.NextDouble() * 6; + double height = 4 + random.NextDouble() * 6; + predicted.Add(new Detection(new BoundingBox(x, y, x + width, y + height), classId, random.NextDouble())); + } + } + } + truth.Add(actual); + predictions.Add(predicted); +} + +string assembly = typeof(ObjectDetectionMetrics).Assembly.Location; +using var assemblyStream = File.OpenRead(assembly); +string assemblyHash = Convert.ToHexString(SHA256.HashData(assemblyStream)); +var metrics = new ObjectDetectionMetrics(); +var cases = new[] +{ + (MetricWorkload.SingleMeanAveragePrecision, 1), + (MetricWorkload.FullPrecisionRecallCurve, 1), + (MetricWorkload.ThresholdRange, 1), + (MetricWorkload.ThresholdRange, 10), + (MetricWorkload.ThresholdRange, 32), + (MetricWorkload.ThresholdRange, 33), + (MetricWorkload.ThresholdRange, 65) +}; +foreach (var (workload, thresholds) in cases) +{ + double step = thresholds == 10 ? 0.05 : 1.0 / 128; + double maximum = 0.5 + (thresholds - 1) * step; + double Score() + { + switch (workload) + { + case MetricWorkload.SingleMeanAveragePrecision: + return metrics.MeanAveragePrecision(predictions, truth); + case MetricWorkload.FullPrecisionRecallCurve: + var curve = metrics.PrecisionRecallCurve(predictions, truth, 0); + double checksum = 0; + for (int point = 0; point < curve.Precision.Length; point++) + checksum += curve.Precision[point] + curve.Recall[point]; + return checksum; + case MetricWorkload.ThresholdRange: + return metrics.MeanAveragePrecisionRange(predictions, truth, 0.5, maximum, step); + default: + throw new ArgumentOutOfRangeException(nameof(workload)); + } + } +#if AP_WORKLOAD_COUNTER + WorkloadProbe.IoUCalls = 0; + double result = Score(); + Console.WriteLine(JsonSerializer.Serialize(new + { + instrumentedAssemblyHash = assemblyHash, + workload = workload.ToString(), + thresholds, + minimumIoU = 0.5, + maximumIoU = maximum, + iouStep = step, + score = result, + iouCalls = WorkloadProbe.IoUCalls + })); +#else + double expected = Score(); + var warmupTimer = Stopwatch.StartNew(); + do + { + if (Score() != expected) throw new InvalidOperationException("Warmup changed the deterministic metric score."); + } while (warmupTimer.Elapsed < TimeSpan.FromSeconds(1)); + + var times = new double[measuredRuns]; + var allocations = new long[measuredRuns]; + for (int iteration = 0; iteration < measuredRuns; iteration++) + { + long allocated = GC.GetAllocatedBytesForCurrentThread(); + long start = Stopwatch.GetTimestamp(); + double actual = Score(); + times[iteration] = Stopwatch.GetElapsedTime(start).TotalMilliseconds; + allocations[iteration] = GC.GetAllocatedBytesForCurrentThread() - allocated; + if (actual != expected) throw new InvalidOperationException("Measured run changed the deterministic metric score."); + } + Array.Sort(times); + Array.Sort(allocations); + Console.WriteLine(JsonSerializer.Serialize(new + { + assemblyHash, + seed, + imageCount, + classCount, + boxesPerClass, + predictions = imageCount * classCount * boxesPerClass * 3, + groundTruth = imageCount * classCount * boxesPerClass, + workload = workload.ToString(), + thresholds, + minimumIoU = 0.5, + maximumIoU = maximum, + iouStep = step, + measuredRuns, + score = expected, + medianMilliseconds = times[measuredRuns / 2], + minimumMilliseconds = times[0], + medianAllocatedBytes = allocations[measuredRuns / 2] + })); +#endif +} + +internal enum MetricWorkload { SingleMeanAveragePrecision, FullPrecisionRecallCurve, ThresholdRange } + +internal static class BenchmarkEnvironment +{ + // Select CPU before Main's workload is touched. The engine's static initialization may + // probe GPUs first; that startup is outside the warmup and measurement intervals. + [ModuleInitializer] + internal static void Initialize() => AiDotNetEngine.ResetToCpu(); +} diff --git a/review-tests/Pr2154.APRangeWorkload/Pr2154.APRangeWorkload.csproj b/review-tests/Pr2154.APRangeWorkload/Pr2154.APRangeWorkload.csproj new file mode 100644 index 0000000000..e0ea57cc24 --- /dev/null +++ b/review-tests/Pr2154.APRangeWorkload/Pr2154.APRangeWorkload.csproj @@ -0,0 +1,29 @@ + + + Exe + net10.0 + enable + enable + false + $(DefineConstants);AP_WORKLOAD_COUNTER + + $(NoWarn);CS0436 + $(MSBuildThisFileDirectory)../Pr2154.APRangeBenchmark/bin/Release/net10.0 + + + + $(ReviewedDependencyDirectory)/AiDotNet.dll + false + + + $(ReviewedDependencyDirectory)/AiDotNet.Tensors.dll + false + + + + + + + + diff --git a/review-tests/Pr2154.APRangeWorkload/WorkloadProbe.cs b/review-tests/Pr2154.APRangeWorkload/WorkloadProbe.cs new file mode 100644 index 0000000000..ed9ac93631 --- /dev/null +++ b/review-tests/Pr2154.APRangeWorkload/WorkloadProbe.cs @@ -0,0 +1,17 @@ +global using AiDotNet.Tensors.Interfaces; +global using AiDotNet.Tensors.Helpers; + +using AiDotNet.Augmentation.Image; + +// Only compiled into the isolated counter executable. The timed production assembly has no +// counter, callback, subclassed geometry, or numeric-provider mutation. +internal static class WorkloadProbe +{ + internal static long IoUCalls { get; set; } + + internal static double CountIoU(BoundingBox prediction, BoundingBox candidate) where T : struct + { + IoUCalls++; + return prediction.IoU(candidate); + } +} diff --git a/review-tests/Pr2154.ComputerVision/Pr2154.ComputerVision.csproj b/review-tests/Pr2154.ComputerVision/Pr2154.ComputerVision.csproj new file mode 100644 index 0000000000..032f8c29bf --- /dev/null +++ b/review-tests/Pr2154.ComputerVision/Pr2154.ComputerVision.csproj @@ -0,0 +1,42 @@ + + + net471;net8.0;net10.0 + latest + AiDotNetTests + enable + enable + true + false + false + $(MSBuildThisFileDirectory)../../src + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/review-tests/Pr2154.ComputerVision/README.md b/review-tests/Pr2154.ComputerVision/README.md new file mode 100644 index 0000000000..5793975f28 --- /dev/null +++ b/review-tests/Pr2154.ComputerVision/README.md @@ -0,0 +1,179 @@ +# PR #2154 bounded review validation + +The current AP-cache follow-up and 271-case inventory are documented in [AP_CACHE_PROOF.md](AP_CACHE_PROOF.md). The historical commands below explicitly retain their original net10.0 fixture scopes; the focused project now also supports net8.0 and net471. + +This project compiles the real AiDotNet library and generator and source-links the changed regression tests, the relevant existing numerical/metrics tests, and all four edited detection/OCR model-family bases. It does not stub production contracts or replace generated family fixtures. + +The baseline is PR head `ebf7a1c9891af791e8715fb4bc5c74c1b270c34a` (base `1c8647e293ff9f5180a071a8e42f16dc90849102`). The reviewed changes are a local follow-up, not a claim that the whole PR is merge-ready. The corrected exhaustive inventory contained **36 threads, 34 unresolved**; thread pagination and every per-thread comments connection reported `hasNextPage: false`. The earlier 33/31 inventory was incomplete, not evidence that three threads had been resolved. + +## Reproduction + +Run from the follow-up worktree using PowerShell and the installed .NET 10 SDK. Enter the path to your own clean baseline worktree when prompted; the guard rejects a missing path, a different commit, or uncommitted files before starting a build. These commands reproduce the recorded 175-case inventory, excluding the later text-input cases documented separately below. This focused project uses the repository's real `ModuleInitializer.cs`, licensing test support, `GlobalUsings.cs`, and `xunit.runner.json`; no numerical tolerance is relaxed. CPU selection is test-only. Runtime preprocessing still uses the selected tensor engine, with no CPU-only production switch. + +```powershell +$baselineRoot = Read-Host 'Path to the clean PR #2154 baseline worktree' +if ([string]::IsNullOrWhiteSpace($baselineRoot)) { + throw 'A baseline worktree path is required.' +} +$baselineRoot = (Resolve-Path -LiteralPath $baselineRoot -ErrorAction Stop).ProviderPath +$expectedBaselineHead = 'ebf7a1c9891af791e8715fb4bc5c74c1b270c34a' +$baselineHead = git -C $baselineRoot rev-parse --verify HEAD +if ($LASTEXITCODE -ne 0 -or $baselineHead -ne $expectedBaselineHead) { + throw "The baseline must be checked out at exactly $expectedBaselineHead." +} +$baselineChanges = @(git -C $baselineRoot status --porcelain) +if ($LASTEXITCODE -ne 0 -or $baselineChanges.Count -ne 0) { + throw 'The baseline worktree must have no tracked or untracked changes.' +} +$baselineSourceRoot = Join-Path $baselineRoot 'src' +$baselineProject = Join-Path $baselineSourceRoot 'AiDotNet.csproj' +if (-not (Test-Path -LiteralPath $baselineProject -PathType Leaf)) { + throw 'The baseline worktree does not contain src/AiDotNet.csproj.' +} +$env:AIDOTNET_FORCE_CPU='1' +dotnet build $baselineProject -c Release -f net10.0 -p:GeneratePackageOnBuild=false +if ($LASTEXITCODE -ne 0) { throw 'The baseline build failed; do not test a stale DLL.' } +dotnet test review-tests/Pr2154.ComputerVision/Pr2154.ComputerVision.csproj -c Release -f net10.0 "-p:ReviewedSourceRoot=$baselineSourceRoot" -p:BuildProjectReferences=false -p:GeneratePackageOnBuild=false --filter 'FullyQualifiedName!~Pyramid_RejectsOverflowingStrideWithoutEnteringLegacyShiftLoop&FullyQualifiedName!~CvInputBoundaryReviewTests.TextDetector_&FullyQualifiedName!~ObjectDetectionRangeCacheReviewTests' --logger 'trx;LogFileName=pr2154-boundary-full-baseline.trx' --results-directory artifacts/pr2154-review --verbosity quiet '-clp:ErrorsOnly' '-flp:logfile=pr2154-boundary-full-baseline.log;verbosity=normal' + +dotnet test review-tests/Pr2154.ComputerVision/Pr2154.ComputerVision.csproj -c Release -f net10.0 -p:GeneratePackageOnBuild=false --filter 'FullyQualifiedName!~CvInputBoundaryReviewTests.TextDetector_&FullyQualifiedName!~ObjectDetectionRangeCacheReviewTests' --logger 'trx;LogFileName=pr2154-boundary-full-after.trx' --results-directory artifacts/pr2154-review --verbosity quiet '-clp:ErrorsOnly' '-flp:logfile=pr2154-boundary-full-after.log;verbosity=normal' +``` + +The baseline source lives in a separate, clean, detached worktree at the exact head above. `ReviewedSourceRoot` changes only the real library/generator project references; both runs compile the same final test sources. Do not run the commands concurrently: the focused project's output directory is intentionally shared. Within the recorded 175-case inventory, the baseline excludes only two stride values above `2^30`: the legacy signed left-shift loop does not terminate for them. These tests are not skipped in source or CI and execute in the unfiltered follow-up run. + +## Text-input follow-up evidence (historical 208-case inventory) + +Comments **3991259128** and **3991259119** add one shared text-detector input validator and portable reproduction inputs. The text-detector fix is at the shared base, not in concrete detectors or generated leaf tests. Both consuming paths validate a snapshot of the publicly mutable `InputSize` array before indexing, resizing, or allocating a deferred input. The already-resolved serialization path does not consume the option and still returns without another forward pass. + +The 33 added cases cover prediction, preprocessing and initial serialization with null external bindings, empty/short/long arrays, zero/negative dimensions, valid `1x1`/`2x3` inputs, normalization and input nonmutation, repeated serialization, and in-place dimension mutation after a real prediction. The **before** library is the unchanged review head `f74c1a6d5c80b197f22ec2d2c4f76895c5ff5762`; both runs compile the same final test sources. To reproduce this newer baseline, use that exact commit as `$expectedBaselineHead` in the guard above and replace the historical filters with `--filter 'FullyQualifiedName!~ObjectDetectionRangeCacheReviewTests'`. That head already contains the stride-overflow fix, so no case in this historical 208-case inventory needs exclusion; the filter only removes the subsequently added AP-cache fixture. + +| Run | Passed | Failed | Skipped | TRX under `artifacts/pr2154-review` | +| --- | ---: | ---: | ---: | --- | +| Unchanged `f74c1a6d5c` DLL, 33 new cases | 7 | 26 | 0 | `pr2154-text-boundary-before.trx` | +| Unchanged `f74c1a6d5c` DLL, unfiltered suite | 182 | 26 | 0 | `pr2154-text-boundary-full-before.trx` | +| Shared validator, unfiltered suite | 208 | 0 | 0 | `pr2154-text-boundary-full-after.trx` | +| Fresh-process no-build repeat | 208 | 0 | 0 | `pr2154-text-boundary-full-after-repeat.trx` | +| Primary reviewer, independent no-build replay | 208 | 0 | 0 | `pr2154-text-boundary-root-independent.trx` | + +All original 175 controls passed before and after. Before the fix, null/short arrays produced `NullReferenceException`/`IndexOutOfRangeException`, negative dimensions reached an `OverflowException`, and long arrays were accepted. These are 26 failing cases for one missing shared validation boundary, not 26 distinct defects. The new guard consistently reports `ArgumentException` with `ParamName == "InputSize"` before forwarding. The selected-engine resize/multiply path and strict pixel/gradient controls are unchanged; these CPU runs are not physical-GPU proof. + +The final core build completed with **0 errors, 2,842 warnings**, in 4m29s. The unfiltered before/after runs took five seconds each; the fresh-process repeat took four seconds. The loaded focused-project DLL hashes were captured before subsequent builds could replace them: + +| Artifact | Before SHA-256 | After SHA-256 | +| --- | --- | --- | +| `AiDotNet.dll` | `52D186AFC8B5484E5A131D39CC060C2E27833F11A7A0FFB1E51BFD08353F2C7F` | `B254FAD5B62ABC52973F4634AD50B30343EF4A64E34A83F4B454280008556FD3` | +| `AiDotNetTests.dll` | `F45C6701F6837599D9BE1A69346FA4F65E6B049826CAF68B2F04B5AD7D5132A7` | `9E36E347EC2C93DC22640B8B142FEDC31B0DC0B7EEAA2FC36918A551A8D0DD02` | + +`AiDotNet.Tensors.dll` remained `EB681AE60F23B03CF08E0BF3AB70A372673927ACD87A428C74536D424846D5E7`. The documented PowerShell block parsed without errors; its guard accepted the actual clean `ebf7a1c989` baseline and rejected empty input, a missing path, and the wrong-head review worktree. No build was launched by those guard-only checks. + +The historical scoped reproduction commands, after building the current core, are: + +```powershell +dotnet test review-tests/Pr2154.ComputerVision/Pr2154.ComputerVision.csproj -c Release -f net10.0 -p:BuildProjectReferences=false -p:GeneratePackageOnBuild=false --filter 'FullyQualifiedName!~ObjectDetectionRangeCacheReviewTests' --logger 'trx;LogFileName=pr2154-text-boundary-full-after.trx' --results-directory artifacts/pr2154-review --verbosity quiet '-clp:ErrorsOnly' +dotnet test review-tests/Pr2154.ComputerVision/Pr2154.ComputerVision.csproj -c Release -f net10.0 --no-build --no-restore --filter 'FullyQualifiedName!~ObjectDetectionRangeCacheReviewTests' --logger 'trx;LogFileName=pr2154-text-boundary-full-after-repeat.trx' --results-directory artifacts/pr2154-review --verbosity quiet +``` + +## Extended boundary evidence (recorded 175-case inventory) + +This recorded suite includes the primary reviewer's independent non-integer `3x5 -> 2x2` pixel/gradient oracle and 42 separate input/stride boundary cases. The exact commands are above. + +| Run | Passed | Failed | Skipped | Not selected | TRX under `artifacts/pr2154-review` | +| --- | ---: | ---: | ---: | ---: | --- | +| Exact baseline DLL, current test sources | 123 | 50 | 0 | 2 | `pr2154-boundary-full-baseline.trx` | +| Follow-up, unfiltered | 175 | 0 | 0 | 0 | `pr2154-boundary-full-after.trx` | +| Follow-up, fresh-process no-build repeat | 175 | 0 | 0 | 0 | `pr2154-boundary-full-after-repeat.trx` | +| Primary reviewer's independent final replay | 175 | 0 | 0 | 0 | `pr2154-expanded-root-independent.trx` | + +The baseline's 50 failures comprise the earlier 17, the independent resize oracle, and 32 boundary-contract cases. Some invalid-stride cases previously rejected the value only inside `Log2` with a singular `stride` parameter error; the new boundary consistently rejects the caller's `strides` configuration before assignment. These counts are cases, not distinct defects. Eight new valid/fast-path controls already passed on the baseline; all 123 baseline-passing controls remain green. The two unselected baseline cases are `int.MaxValue` and `2^30 + 1`, whose legacy loop cannot terminate; their unfiltered after results are passing, not claimed before executions. + +That source build reported **0 errors, 2,775 warnings**, 3m43s. Test durations were eight seconds before, five seconds after, and four seconds for the repeat. The test sources and numerical tolerances were identical across the baseline and follow-up compilations. The new guard preserves the already-resolved serialization return, and valid stride boundaries include both `1` and the largest positive signed-int power of two (`2^30`). + +| Artifact | Baseline SHA-256 | Follow-up SHA-256 | +| --- | --- | --- | +| Loaded `AiDotNet.dll` | `9114068D81C131953BBA5087943181725C0046348E61DEFA3CDAA433E136C2DD` | `52D186AFC8B5484E5A131D39CC060C2E27833F11A7A0FFB1E51BFD08353F2C7F` | +| `AiDotNetTests.dll` | `5FB685DE4C67610C78BE4374FFE27BC25059CF11E9F81EAF9CA02ED8C8298221` | `DFC40A44A69B4B74705C7A1A984364393FFBEDE9BF8E8370E8188689A09A09C3` | +| Boundary TRX | `A6479CF7DCF80A27668818FBC4D69C62787D039914AD8BF77B1BC6A6A05BE48E` | `B98C784F9149C8D62FA4427B39510FB3358AEFB01E9BC34A15DB12AE9D1E5B3F` | + +The loaded tensor dependency is unchanged (`AiDotNet.Tensors` 0.130.3; SHA-256 `EB681AE60F23B03CF08E0BF3AB70A372673927ACD87A428C74536D424846D5E7`). The repeat used `dotnet test` with `--no-build --no-restore` and the repeat TRX name, so it did not rebuild or change the tested library. + +## Initial corrected-harness evidence (132 cases, historical) + +Both initial corrected-harness runs used the same test sources and the published `AiDotNet.Tensors` package `0.130.3`. SDK: `10.0.401`, Windows x64, `net10.0`. + +| Actual source | Passed | Failed | Skipped | Source build | TRX under `artifacts/pr2154-review` | +| --- | ---: | ---: | ---: | --- | --- | +| Exact PR baseline | 115 | 17 | 0 | 0 errors, 2,842 warnings; 3m48s | `pr2154-review-final-harness-baseline.trx` | +| Follow-up source | 132 | 0 | 0 | 0 errors, 2,775 warnings; 3m53s | `pr2154-review-final-harness-after.trx` | + +Each test run reported a five-second test duration, separate from source compilation. These are 17 failing regression **cases**, not 17 distinct root causes. All 115 baseline-passing controls remained green after the changes. The 132 cases comprise 19 new CV regressions/controls, four actual nullable-generator/runtime cases, and 109 existing numerical, FPN/PANet, TrOCR-decoding and metric cases. The four edited abstract family bases compile but are not counted as executed generated model families. + +Concrete observations pinned by the before/after tests include: + +- CTC/attention with a two-character budget: baseline emits `abc`; follow-up emits `ab`. Leading CTC blanks/repeats and attention EOS are also checked. +- Multi-image empty RoIs: baseline throws `ArgumentNullException` while concatenating no tensors; follow-up returns shape `[0,3,2,2]`. +- Concurrent warm-up: baseline produced seven duplicate-key exceptions and one silent empty-registry success; follow-up initializes once and reports the real empty-registry error to all eight callers. +- Text prediction: baseline skips the configured spatial resize and gives input gradient `1`; follow-up uses the expected shape and gradient `1/255`, with unchanged input pixels. +- Tensor-list/adapter chunk IDs: baseline layout `weights` does not match `weights/0.0` and `weights/1.0`; follow-up IDs and live storage identity agree. Flat snapshots are not advertised as writable model storage, and mutating them does not change the model. +- The ready-training control passes both before and after: weight `2 -> 1.6 -> 1.28`, losses `4` then `2.56`, and exactly three forward calls (one warm-up plus two gradient steps). + +SHA-256 hashes were read from the **loaded focused-project output**, before the next run replaced it: + +| Artifact | Baseline SHA-256 | Follow-up SHA-256 | +| --- | --- | --- | +| `AiDotNet.dll` | `9114068D81C131953BBA5087943181725C0046348E61DEFA3CDAA433E136C2DD` | `0C7E3196109DE670DE47C7F7DC6828D0E6700A3C06F01327176C40375EB8A23F` | +| `AiDotNetTests.dll` | `DA3FA766CDA03FB24B63890FB50BB1254A64F0D74F6C526361C0B12FA63A6FA5` | `CE99EC7801BC612958C846ED1E596511C058191BAD1BEA0EAE464CD6A5E7D39C` | +| Final TRX | `7CAB5D99F44A52D8527E11DDDFD0DD43BC637A9232E7975FB4FF6EB97C100961` | `2772A2227F0C706520558DC0E7BCD549D1C25CCF45095B8810EC9AAD96A6930B` | + +The `AiDotNet.Tensors.dll` hash is identical in both runs: `EB681AE60F23B03CF08E0BF3AB70A372673927ACD87A428C74536D424846D5E7`. Earlier diagnostic runs remain in `artifacts/pr2154-review`, but are not the final before/after claim. + +A fresh-process no-build repeat also passed **132/132**, with no skipped tests, in four seconds: `pr2154-review-final-harness-after-repeat.trx`. This was the initial 132-case inventory, before the independent resize oracle and boundary cases below. `git diff --check` passed; the added C# lines and new C# files contain no null-forgiving operators. + +The primary reviewer independently inspected the production/test diff and repeated the +same final binary in another fresh process: **132 passed, zero failed, zero skipped**, +five seconds (`pr2154-review-root-independent.trx`). The loaded `AiDotNet.dll` +SHA-256 matches the follow-up hash above. This replay does not expand the tested scope +to the complete generated model families or GPU execution. + +The corrected initialization intermediate run, `pr2154-review-exact-initializer-before-fallback.trx`, passed 129 tests and failed 2 (0 skipped). Both failures were copied parameter payloads incorrectly marked `IsWritableInPlace`, after the separate chunk-ID fix. Those failures directly justify the two shared fallback metadata corrections. + +### Diagnostic mistakes explicitly excluded from defect counts + +- The first isolated harness omitted the repository's CPU initialization. That produced float-sized discrepancies in strict double numerical tests; importing the real initializer/configuration made those controls pass without source or tolerance changes. `AIDOTNET_FORCE_CPU` alone was not an equivalent setup. +- The original nullable-generator probe used the global namespace. The generator emitted an invalid namespace for that unrelated configuration. The final probe uses an explicit namespace and executes real generated component adapters in four enabled/disabled and optional/required cases. The existing global-namespace generator limitation is not fixed here. +- An initial text-output oracle expected a flattened tensor even though the existing single-output contract preserves rank. The final oracle uses `[1,3,2,2]`. +- An intermediate CRNN cleanup removed `_sequenceFeatureDim` before its weight-file header consumers were checked. The resulting three `CS0103` errors were introduced during this work, not baseline errors. The field and its value `512` are preserved; its persistence role is documented. Unused duplicate scratch state was removed instead. + +## Review mapping + +IDs below are GitHub review-comment database IDs; the corresponding full thread IDs were retained in the review inventory. The mapping distinguishes completed fixes from the explicitly pending items below; it is not a claim that every review item is complete. + +| Comment IDs | Scoped disposition and evidence | +| --- | --- | +| 3985472465 | Both base decoders honor the emitted-character budget. CTC blanks and repeated timesteps do not consume it; zero-budget, blank/repeat and EOS controls are included. | +| 3985472468, 3990067342 | Empty live trainable discovery throws with model identity. A typed shared warm-up gate prevents duplicate initialization, retries failed initialization, and preserves the ready-model fast path. Tests include concurrent first calls, failure retry and two real gradient steps with exact expected weights. | +| 3990067371 | Tensor-list layouts and live chunks use identical stable IDs. Accessor/collection passthrough requires compatible layout metadata; fallback payloads are explicitly not writable model storage. Tests cover live identity, both adapters, flat-only sources and snapshot nonmutation. | +| 3990067051 | Empty RoIs preserve `[0,C,outH,outW]` for single- and multi-image inputs without concatenating an empty list. | +| 3990067177 | Text prediction uses the same asymmetric resize and normalization as detection. The implementation uses tensor-engine operations and retains the input gradient. Exact pixel, input-nonmutation and gradient tests are included. | +| 3990067093 | All three CV base copy-preparation paths replay batch one without modifying the source shape; text and OCR runtime probes cover the shared behavior. | +| 3990067103 | A shared object-detector base guard validates exactly two positive input dimensions before deferred probing or preprocessing. Tests cover null external binding, empty/short/long arrays, nonpositive dimensions, valid 1x1/2x3 inputs and the already-resolved fast path. | +| 3991259128 | The shared text-detector base now applies the same exact-two-positive-dimensions contract to prediction/preprocessing and deferred serialization. The 33 new cases reproduce 26 failures on `f74c1a6d5c` and pass with the private validator; all 175 earlier controls remain green. | +| 3991259119 | Reproduction prompts for the caller's baseline root and validates its exact recorded commit, clean Git state and real project path before building. Both commands use that resolved input rather than an author-specific path. Guard-only positive and negative controls are recorded above. | +| 3990067121 | Shared FPN assignment validates nonempty, positive power-of-two, contiguous doubling strides before taking logarithms. The integer logarithm shifts its value down, so it cannot wrap a left-shift count indefinitely. Tests cover invalid assignment/pooling inputs and valid/invalid signed-int boundaries. | +| 3990067140 | Do not internalize `RPN`: it was already public at merge base `1c8647e293ff9f5180a071a8e42f16dc90849102`, so that recommendation would break an existing public type. Its explicit parameter members already delegate to the shared internal `DelegatingCvParameterModule`; the forwarding does not duplicate the implementation. | +| 3990067157 | Both YOLO head decoders hoist `scaleX`/`scaleY` once per level, using the existing feature dimensions and preserving the arithmetic. Source inspection and real library compilation verify this cleanup; no dedicated decode-speed benchmark is claimed. | +| 3985472484 | Corpus CER/WER return NaN for nonzero edits over zero reference length, consistent with the existing per-sample contract; zero-edit and ordinary-reference controls remain. | +| 3985472480 | Polygon conversion is internal. Existing text-detection metric tests compile and execute against the actual implementation. | +| 3985472488, 3985472504, 3985472513 | Shared family bases verify clone mutation, batch box coordinates and exact source image dimensions. Their real sources compile in this harness; complete generated model families have not been rerun here. | +| 3990067400, 3990067419 | Input dependence always reaches an assertion, including length differences; NMS invariants honor the fixture's typed/overridable threshold properties. No generated leaf tests were edited. | +| 3990067450, 3990067474 | FPN pooling uses an independent exact level oracle; PANet adds the meaningful level-zero pathway case. Existing numerical/pathway tests execute unchanged except these stronger test inputs/oracles. | +| 3990067066, 3990067079, 3990067167, 3990067215 | Stale XML parameter/summary tags and unreferenced GELU helpers are removed. Actual source compilation and repository-wide caller search validate the cleanup; no public detector API is substituted. | +| 3990067246, 3990067275, 3990067306, 3990067323 | Remove unreachable grayscale branching, unused duplicate CRNN scratch state, unused private shape arguments/locals and dead helper code. CRNN's persisted feature dimension remains. Existing TrOCR incremental/full-decoder parity controls execute. | +| 3990067038 | No readiness weakening: actual Roslyn/generator/runtime tests show explicit `?` remains optional under either nullable context, while unannotated components remain required and raise `ParameterLayoutNotReadyException`. | +| 3990067227 | The current concrete TrOCR `Train` override uses teacher-forced logits and cross-entropy, not the base inference `NoGrad` path. This is source-backed rejection of that specific premise, not proof that every real-model training case is correct. | + +## Explicitly pending / not claimed + +- **3985472460:** proper detector-specific annotation/loss training remains an architectural gap. Concatenating all head outputs prevents dropping heads, but is not proof that generic MSE is a correct detector loss. +- **3985472491 and 3985472507:** deterministic non-empty object/text detection fixtures still need a shared base/generator design. Random-image tests may produce no detections; compiling stronger invariants does not prove their nonvacuity. +- **3985472478:** caching threshold-independent AP data remains a performance follow-up. Per-threshold greedy matching must remain independent; this batch does not replace it with a shared match set. +- This is focused local Windows `net10.0` validation, not the complete CI workflow, other target-framework builds, GPU execution or a full generated model-family sweep. The existing project emits many warnings. GitHub readiness/merge decisions must retain these limitations and the pending architecture work. +- No package-version change or merge-ready claim is part of this batch. The PR remains draft while the explicitly pending work is incomplete. diff --git a/review-tests/Pr2154.DetectionBoundaries/Pr2154.DetectionBoundaries.csproj b/review-tests/Pr2154.DetectionBoundaries/Pr2154.DetectionBoundaries.csproj new file mode 100644 index 0000000000..3a84b6a3c8 --- /dev/null +++ b/review-tests/Pr2154.DetectionBoundaries/Pr2154.DetectionBoundaries.csproj @@ -0,0 +1,40 @@ + + + net471;net8.0;net10.0 + AiDotNetTests + latest + enable + enable + true + false + false + false + false + <_GetChildProjectCopyToOutputDirectoryItems>false + true + true + true + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/review-tests/Pr2154.DetectionBoundaries/README.md b/review-tests/Pr2154.DetectionBoundaries/README.md new file mode 100644 index 0000000000..e32a20dc88 --- /dev/null +++ b/review-tests/Pr2154.DetectionBoundaries/README.md @@ -0,0 +1,142 @@ +# PR #2154: detection metric and geometry boundaries + +This batch addresses review comments `3994110796`, `3994110814`, and +`3994110819`. It changes the shared metric implementation and shared text-test +invariant, not generated leaf tests or detector numerical forwards. + +## Contracts and regression controls + +Single-threshold AP, mean AP, and precision/recall now reject non-finite IoU +thresholds and thresholds outside `[0,1]`, including inputs with no ground-truth +classes. Null outer lists and mismatched image counts retain their earlier +exception precedence. Invalid thresholds must be rejected before enumerating +either inner detection list; finite endpoints 0 and 1 remain valid. As before, +zero-overlap boxes do not become matches merely because the threshold is zero. + +The range API replaces the fixed `1e-9` quotient bias with a correction scaled +to binary64 roundoff. An endpoint mathematically on the grid can be recovered +from division roundoff. If reconstructing that final grid point slightly exceeds +the caller's maximum, it is clamped only within the corresponding arithmetic +tolerance. An off-grid maximum is not appended. The existing 32-threshold batches, +independent match claims, stable ranking, lazy IoU access, and ordered averaging +are unchanged. + +The grid controls distinguish `max=0.9999999995, step=1` from a genuine endpoint, +cover an interior near-grid maximum, decimal `0.1..0.3` by `0.1`, the COCO grid, +an off-grid maximum, and a single threshold with `double.Epsilon` step. Existing +controls also require overflow rejection before enumeration for both +`step=1/Int32.MaxValue` and `step=double.Epsilon` over `[0,1]`, where the raw +quotient is infinity. A representable `Int32.MaxValue` threshold count with no +classes still takes the empty result path without allocating matching state. + +The shared random-input text invariant now checks all four coordinates for both +NaN and infinity before its unchanged positive-width/height assertions. Four +coordinates times three non-finite values are tested directly. Finite negative +coordinates remain legal for an unclipped EAST box; no image-boundary condition +was added to that contract. The 18 existing generated positive text cases are +included unchanged. + +## Failure-before evidence + +The first run used the actual production core at +`d8183f1d0e8e8f552b828f33efc459f64318d1ab`, with SHA-256 +`69BCCF0B1B5AD45EB4050C3D83D352980BB7F0F8EC4D3153EBEBB5469A4B9B8D`. +For direct testing, the old text helper's visibility was widened from private to +internal, but its assertion body was unchanged. The same 172 cases produced +120 passes, 52 failures, and zero skips: + +- 45 invalid single-threshold cases failed their required argument-validation + contract across three APIs and populated/empty inputs. +- Three range cases failed: the two near-grid maxima admitted a forbidden extra + threshold, and decimal `0.1..0.3` reconstructed a final threshold above `0.3`. +- Four text cases exposed accepted infinities: negative infinity in left/top, + positive infinity in right/bottom. Other non-finite combinations were already + rejected by the old NaN or positive-area conditions. + +The other 120 controls passed. The baseline report is +`artifacts/pr2154-review/pr2154-boundaries-first-before.trx`; its test assembly +hash is `B52FB918A966E1A11C9D367CF09E428D96CC6363DB81A2513B0E484AF2963AD1`. +This is actual-library CPU evidence, not a stub or a copied metric implementation. + +## Corrected-code results + +The 172 cases comprise 15 existing metric cases, 63 existing range/cache cases, +18 generated positive-text cases, 62 new threshold-boundary cases, and 14 new +shared text-geometry cases. No test or assertion was removed to obtain green. + +| Actual run | Passed | Failed | Skipped | Report in `artifacts/pr2154-review` | +| --- | ---: | ---: | ---: | --- | +| Historical core and historical text assertions, .NET 10 | 120 | 52 | 0 | `pr2154-boundaries-first-before.trx` | +| Historical core, corrected text assertions only, .NET 10 | 124 | 48 | 0 | `pr2154-boundaries-text-only-before-core.trx` | +| Corrected core and text assertions, .NET 10 | 172 | 0 | 0 | `pr2154-boundaries-final-net10.trx` | +| Corrected core and text assertions, .NET 8 | 172 | 0 | 0 | `pr2154-boundaries-final-net8.trx` | +| Corrected core and text assertions, .NET Framework 4.7.1 | 172 | 0 | 0 | `pr2154-boundaries-final-net471.trx` | +| Independent parent-agent replay, .NET 10 | 172 | 0 | 0 | `pr2154-boundaries-root-independent.trx` | + +All three actual core builds succeeded: net10/net8/net471 had zero errors and +2,775/2,775/2,777 warnings respectively, taking 3m40s/4m12s/3m40s. All three +focused test builds had zero errors and zero warnings. The final test executions +took 14/14/20 seconds. The independent net10 replay checked the exact core/test +hashes and passed in 16 seconds. + +| Artifact | SHA-256 | +| --- | --- | +| Corrected .NET 10 core | `BB9F13F390C9EB9056891A0BE91E17312851F1095E99F95B512976BF04565E8F` | +| Corrected .NET 10 test assembly | `C230662175B48E3295FD814E0CA59988666FB779050B0BE3D8237D54A9AEB15A` | +| Corrected .NET 8 core | `1EC7D7A5409FD0360239D5FAE3B54B8EAED3251CAA552E23EC6C4EA5BAEAC9AE` | +| Corrected .NET 8 test assembly | `05B40C3CEED00287AEB826B7C0402DE04241B7FDB391816128D8C2D2BB7810CE` | +| Corrected .NET Framework 4.7.1 core | `4FAED56C5CB86ED460CCAB3371918600F91E2ACCB973E45CE851FE47C5A43831` | +| Corrected .NET Framework 4.7.1 test assembly | `F4D7E6337E4D976356376BD7A34CCD3A429B3D7886D7FAFEA10A417E32567D82` | +| Unchanged actual generator | `1AF4448E70ED82A2248EDC7077225B7925ED9F69E78CE642331D71BC5587141F` | + +## Focused runner integration + +The preceding object-positive fixture added a helper referenced by +`ObjectDetectionTestBase`. Two older focused projects linked the base explicitly +but omitted that helper. Both failed with the same two `CS0103` diagnostics. +The source includes are corrected in `Pr2154.ComputerVision` and +`Pr2154.DetectionParameters`; all six project/framework compile configurations +then succeeded. A repository-wide scan of project/props/targets source files, +including paths outside `review-tests`, found only these two omissions. The +other matching runners already included their required helpers. + +These checks used `dotnet msbuild -t:Compile`, so they compiled the current source +into `obj` without replacing the historical test binaries in `bin`. SHA-256 +checks confirmed all six frozen binaries were unchanged. Logs are named +`artifacts/pr2154-review/pr2154-focused-links--.log`. +The six test hashes and all three earlier positive-object core hashes were +checked again after the compatibility builds and remained unchanged. +This source-list integration check is not a claim that the full main test +project or every model family was executed. + +```powershell +foreach ($project in @('ComputerVision', 'DetectionParameters')) { + foreach ($framework in @('net10.0', 'net8.0', 'net471')) { + dotnet msbuild "review-tests/Pr2154.$project/Pr2154.$project.csproj" -t:Compile -p:TargetFramework=$framework -p:Configuration=Release -p:BuildProjectReferences=false -p:CopyLocalRuntimeTargetAssets=false -p:CopyLocalLockFileAssemblies=false -p:_GetChildProjectCopyToOutputDirectoryItems=false -m:1 -nodeReuse:false -nologo -v:quiet -clp:ErrorsOnly + if ($LASTEXITCODE -ne 0) { throw "Focused source compilation failed for $project / $framework." } + } +} +``` + +## Reproduction + +Run from the repository root. The small runner includes the actual module +initializer, license helper, trace helper, and xUnit configuration. It reuses +actual built core assemblies and the existing CPU native closure; all-RID asset +copying and child Content propagation are disabled. On .NET Framework it uses +the existing managed-only dependency closure target. + +```powershell +$framework = 'net10.0' # Also exercise net8.0 and net471. +dotnet restore review-tests/Pr2154.DetectionBoundaries/Pr2154.DetectionBoundaries.csproj -p:NuGetAudit=false +dotnet build src/AiDotNet.Generators/AiDotNet.Generators.csproj -c Release --no-restore -m:1 -nodeReuse:false +dotnet build src/AiDotNet.csproj -f $framework -c Release --no-restore -m:1 -nodeReuse:false -p:BuildProjectReferences=false -p:GeneratePackageOnBuild=false -p:CopyLocalRuntimeTargetAssets=false -p:CopyLocalLockFileAssemblies=false -p:_GetChildProjectCopyToOutputDirectoryItems=false +dotnet build review-tests/Pr2154.DetectionBoundaries/Pr2154.DetectionBoundaries.csproj -f $framework -c Release --no-restore -m:1 -nodeReuse:false -p:BuildProjectReferences=false -p:GeneratePackageOnBuild=false +$cpuNativeDirectory = (Resolve-Path "review-tests/Pr2154.DetectionParameters/bin/Release/$framework").Path +$env:PATH = $cpuNativeDirectory + ';' + $env:PATH +dotnet test review-tests/Pr2154.DetectionBoundaries/Pr2154.DetectionBoundaries.csproj -f $framework -c Release --no-build --no-restore --logger 'trx;LogFileName=boundaries-replay.trx' --results-directory artifacts/pr2154-review +``` + +The semantic detector-training review remains separate open work. These +boundary checks are not GPU proof, trained detection accuracy, or full-repository +CI proof, and do not by themselves make this draft PR ready to merge. diff --git a/review-tests/Pr2154.DetectionParameters/Pr2154.DetectionParameters.csproj b/review-tests/Pr2154.DetectionParameters/Pr2154.DetectionParameters.csproj new file mode 100644 index 0000000000..c23add19e1 --- /dev/null +++ b/review-tests/Pr2154.DetectionParameters/Pr2154.DetectionParameters.csproj @@ -0,0 +1,49 @@ + + + net471;net8.0;net10.0 + AiDotNetTests + latest + enable + enable + true + false + false + false + + <_GetChildProjectCopyToOutputDirectoryItems>false + true + true + true + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/review-tests/Pr2154.DetectionParameters/README.md b/review-tests/Pr2154.DetectionParameters/README.md new file mode 100644 index 0000000000..437914025e --- /dev/null +++ b/review-tests/Pr2154.DetectionParameters/README.md @@ -0,0 +1,179 @@ +# PR #2154: live parameter-layout and metadata proof + +This batch fixes shared parameter exposure needed by the detector review. It does +**not** claim that positive detection fixtures or task-specific detector training +are complete. The separate AP-cache proof remains in +[`../Pr2154.ComputerVision/AP_CACHE_PROOF.md`](../Pr2154.ComputerVision/AP_CACHE_PROOF.md). + +## Root defects and contracts + +The five shared backbone adapters exposed actual parameter chunks but not their +underlying layouts. The registry correctly refused to treat an unverified chunk +source as writable and used a detached fallback. `CvParameterModule` and RPN had +the same missing layout contract. They now delegate the real layer layout; the +registry's safety gate and its negative controls are unchanged. + +Shared module layout and chunks use the same canonical indexed child IDs. Tests +check tensor identity, role, flat order, normalized offsets, zero-sized own +slots, null children, deferred shapes, and every propagated layout descriptor. +Accessor and collection registration both expose actual tape weights: a real +gradient update must change the real forward and reduce an independently defined +sum objective. No replacement forward or synthetic optimizer is used. + +Exposing the real MHA layout uncovered a second defect: querying optional child +structure called its value initializer and allocated projection weights. The +base still initializes unknown child structures. Only a conservative generator +proof can omit this step for an exact runtime layer type; derived types retain +the original path. Nullable fields alone are not evidence that children are +ready, and collections, inherited/unknown structural paths, owner escapes, +callbacks, setters, user-defined operators, and unknown external contracts +retain initialization. + +The proof assumes the explicitly resolved numeric, tensor, engine, and +initialization APIs honor their value-only contracts. An initialization strategy +that secretly captures its owner to create child layers violates that contract. +This is not general whole-program side-effect analysis. Metadata tests use a +rejecting initialization strategy to prove that weight initialization is not +invoked at all, not merely that it leaves the same values behind. + +## Failure-first evidence + +TRXs and logs are retained locally in `artifacts/pr2154-review/`; they are not +committed binaries. Counts below are TRX executed/passed/failed counts, not just +process exit codes. The original adapter baseline is the frozen AP-complete +core (commit `7ccc544f94694dd41c6a73350473f0c397bac023`, core SHA256 +`2D5355184C81886DA018076141A8A1030F9E5C49527C8DDB145B75253399388C`). + +| Snapshot / control | Evidence file | Passed / executed | +| --- | --- | ---: | +| Original adapter regression cohort | `pr2154-adapter-live-before.trx` | 20 / 50 (30 failed) | +| Expanded original adapter cohort | `pr2154-adapter-live-expanded-before.trx` | 20 / 56 (36 failed) | +| Layout delegation before canonical child-ID and metadata fixes | `pr2154-adapter-live-expanded-after.trx` | 51 / 56 (5 failed) | +| Canonical IDs, before actual MHA initializer eligibility | `pr2154-adapter-live-intermediate.trx` | 57 / 59 (2 failed) | +| Actual MHA eligibility, before final primitive-contract tightening | `pr2154-adapter-cv310-intermediate.trx` | 310 / 310 | +| Adversarial setter / owner-alias controls before correction | `pr2154-metadata-generator-adversarial-red.trx` | 12 / 19 (7 failed) | +| Reassigned-null callback controls before correction | `pr2154-metadata-generator-callback-red.trx` | 20 / 23 (3 failed) | +| Constructor / user-operator controls before correction | `pr2154-metadata-generator-operator-red.trx` | 22 / 32 (10 failed) | +| Implicit callback controls before correction | `pr2154-metadata-generator-implicit-effects-red.trx` | 38 / 41 (3 failed) | +| Non-primitive `SpecialType` controls before correction | `pr2154-metadata-generator-specialtype-red.trx` | 41 / 44 (3 failed) | +| All final policy cases against the exact unchanged baseline generator | `pr2154-metadata-generator-final44-baseline.trx` | 36 / 44 (8 failed) | +| Final 44 generator policy controls | `pr2154-metadata-generator-specialtype-after.trx` | 44 / 44 | + +The intermediate real-core SHA256 was +`69BCCF0B1B5AD45EB4050C3D83D352980BB7F0F8EC4D3153EBEBB5469A4B9B8D`. +That earlier runtime result proved actual MHA eligibility but was not the final +generator-policy snapshot. The final reviewed-source rebuild and replays below +produced the same net10 core hash: the additional conservative rejects do not +change emitted code for these actual layer types. The source-level adversarial +controls, not an assumed change to the core DLL hash, prove those rejects. + +## Final reviewed-source validation + +All three actual core builds completed with zero errors. Build logs are +`pr2154-layout-reviewed-core-net10.log` (2775 warnings, 2m52s), +`pr2154-layout-reviewed-core-net8.log` (2775 warnings, 2m57s), and +`pr2154-layout-reviewed-core-net471.log` (2777 warnings, 2m59s). These are core +compatibility builds plus bounded actual-source test runners, not a claim that +the entire main test assembly or every model-family shard ran. + +| Target | Runtime TRX | Passed / executed / skipped | +| --- | --- | ---: | +| net10.0 | `pr2154-layout-reviewed-net10.trx` | 310 / 310 / 0 | +| net8.0 | `pr2154-layout-reviewed-net8.trx` | 310 / 310 / 0 | +| net471 | `pr2154-layout-reviewed-net471.trx` | 310 / 310 / 0 | + +The actual core and copied runner DLL hashes agree for each target: + +- net10.0: `69BCCF0B1B5AD45EB4050C3D83D352980BB7F0F8EC4D3153EBEBB5469A4B9B8D` +- net8.0: `97F5D10424D8B75698A144E248797B18A149F08B376727361A37E609E82B27BA` +- net471: `B6E19F6F44C9F6E43FFA2A9A29B14B9999B9F76F86669F273644027CA6BEB64B` + +### Harness negative controls + +The initial isolated generator harness omitted the real test initializer and +failed two existing semantic-compilation tests against both the old and new +generator. With the actual `ModuleInitializer.cs`, licensing helper, and core +dependency graph, the unchanged baseline passes 59/59 +(`pr2154-metadata-generator-initialized-baseline.trx`). No semantic assertion was +weakened. That unchanged generator DLL has SHA256 +`D71ABAE9452767B828CEFB6379D25A2A78E5F38016DD36F3F11A8A3B64D73CB0`; +the reviewed generator DLL has SHA256 +`488763248BE3B050EB1EACB8BAE44CBAECAE80FCC67797C6A436B83D22993A53`. +The final generator then passes all 59 existing plus 44 new cases: + +| Target | Evidence file | Passed / executed / skipped | +| --- | --- | ---: | +| net10.0 | `pr2154-generator-reviewed-restored-net10.trx` | 103 / 103 / 0 | +| net8.0 | `pr2154-generator-reviewed-net8.0.trx` | 103 / 103 / 0 | +| net471 | `pr2154-generator-reviewed-net471.trx` | 103 / 103 / 0 | + +After the last historical baseline control, the net10 runner was rebuilt with +the reviewed generator; its copied generator hash was checked and all 103 cases +were rerun. No runner is left pointing at the baseline generator. + +An earlier net471 launch exited zero but discovered no tests because managed +xUnit dependencies were missing; it is **not** a passing result. The corrected +runner copies the SDK-resolved managed runtime assemblies on .NET Framework, +which cannot use `.runtimeconfig.dev.json` NuGet probing. It does not copy every +platform's native runtime assets. The repository's actual CPU initializer and +xUnit configuration remain in both runners. + +## Focused reproduction + +Run from the repository root. Restore the two runner projects once. Build the +generator and actual core for the chosen target before using `--no-build` tests; +`BuildProjectReferences=false` deliberately prevents a hidden large rebuild. +The detection runner contains 310 unique cases: the existing 271 CV/AP cases +and 39 new shared-adapter cases. The generator runner contains 103 cases. + +```powershell +dotnet restore review-tests/Pr2154.DetectionParameters/Pr2154.DetectionParameters.csproj +dotnet restore review-tests/Pr2154.LayerStructureGenerator/Pr2154.LayerStructureGenerator.csproj +dotnet build src/AiDotNet.Generators/AiDotNet.Generators.csproj -c Release --no-restore -m:2 -nodeReuse:false + +$reviewTarget = 'net10.0' # Repeat with net8.0 and net471. +dotnet build src/AiDotNet.csproj -f $reviewTarget -c Release --no-restore -m:2 -nodeReuse:false -p:BuildProjectReferences=false -p:GeneratePackageOnBuild=false -p:CopyLocalLockFileAssemblies=false +dotnet build review-tests/Pr2154.DetectionParameters/Pr2154.DetectionParameters.csproj -f $reviewTarget -c Release --no-restore -m:2 -nodeReuse:false -p:BuildProjectReferences=false -p:GeneratePackageOnBuild=false +dotnet build review-tests/Pr2154.LayerStructureGenerator/Pr2154.LayerStructureGenerator.csproj -f $reviewTarget -c Release --no-restore -m:2 -nodeReuse:false -p:BuildProjectReferences=false -p:GeneratePackageOnBuild=false + +dotnet test review-tests/Pr2154.DetectionParameters/Pr2154.DetectionParameters.csproj -f $reviewTarget -c Release --no-build --no-restore --logger "trx;LogFileName=pr2154-layout-$reviewTarget.trx" --results-directory artifacts/pr2154-review +dotnet test review-tests/Pr2154.LayerStructureGenerator/Pr2154.LayerStructureGenerator.csproj -f $reviewTarget -c Release --no-build --no-restore --logger "trx;LogFileName=pr2154-generator-$reviewTarget.trx" --results-directory artifacts/pr2154-review +``` + +These bounded runners suppress transitive platform-native Content copies. Supply +the ordinary CPU-native dependencies through an existing validated CPU runtime +directory on the native library search path. Do not copy another runner's +managed dependency directory wholesale: the generator runner deliberately uses +its own resolved Roslyn 4.14.0 package version. On Windows, an existing CPU-native +directory can be prepended to `PATH` for this shell. This is CPU correctness +proof, not a physical-GPU performance claim. + +Verify the expected executed counts as well as the exit status: + +```powershell +$reviewCases = @{ + "pr2154-layout-$reviewTarget.trx" = 310 + "pr2154-generator-$reviewTarget.trx" = 103 +} +foreach ($reviewCase in $reviewCases.GetEnumerator()) { + [xml] $reviewResult = Get-Content -LiteralPath (Join-Path artifacts/pr2154-review $reviewCase.Key) + $reviewCounters = $reviewResult.TestRun.ResultSummary.Counters + if ([int]$reviewCounters.total -ne $reviewCase.Value -or + [int]$reviewCounters.executed -ne $reviewCase.Value -or + [int]$reviewCounters.passed -ne $reviewCase.Value -or + [int]$reviewCounters.failed -ne 0 -or + [int]$reviewCounters.notExecuted -ne 0) { + throw "Unexpected test result counts: $($reviewCase.Key)" + } +} +``` + +## Remaining review scope + +Diagnostic runs demonstrate that CRAFT, DBNet, EAST, and all nine object +detectors now expose live trainable chunks. Controlled actual text heads produce +nondegenerate output. These probes are not yet shared positive fixture tests, +and tied object scores do not prove sorting or NMS. Those reviews remain open. +Likewise, preserving raw-output regression training does not implement typed +detection assignment/classification/box losses; that is a separate unfinished +review item. diff --git a/review-tests/Pr2154.LayerStructureGenerator/Pr2154.LayerStructureGenerator.csproj b/review-tests/Pr2154.LayerStructureGenerator/Pr2154.LayerStructureGenerator.csproj new file mode 100644 index 0000000000..ccfaca47de --- /dev/null +++ b/review-tests/Pr2154.LayerStructureGenerator/Pr2154.LayerStructureGenerator.csproj @@ -0,0 +1,35 @@ + + + net471;net8.0;net10.0 + AiDotNetTests + latest + enable + enable + true + false + false + false + + <_GetChildProjectCopyToOutputDirectoryItems>false + true + true + true + ../../src/AiDotNet.Generators/bin/Release/netstandard2.0/AiDotNet.Generators.dll + + + + + + + + + + + + + + + + + diff --git a/review-tests/Pr2154.ManagedTestClosure.targets b/review-tests/Pr2154.ManagedTestClosure.targets new file mode 100644 index 0000000000..264e314c25 --- /dev/null +++ b/review-tests/Pr2154.ManagedTestClosure.targets @@ -0,0 +1,11 @@ + + + + + + diff --git a/review-tests/Pr2154.PositiveDetections/Pr2154.PositiveDetections.csproj b/review-tests/Pr2154.PositiveDetections/Pr2154.PositiveDetections.csproj new file mode 100644 index 0000000000..fae662f205 --- /dev/null +++ b/review-tests/Pr2154.PositiveDetections/Pr2154.PositiveDetections.csproj @@ -0,0 +1,36 @@ + + + net471;net8.0;net10.0 + AiDotNetTests + latest + enable + enable + true + false + false + false + <_GetChildProjectCopyToOutputDirectoryItems>false + true + true + true + $(MSBuildThisFileDirectory)../../src/AiDotNet.Generators/bin/Release/netstandard2.0/AiDotNet.Generators.dll + + + + + + + + + + + + + + + + + + + + diff --git a/review-tests/Pr2154.PositiveDetections/README.md b/review-tests/Pr2154.PositiveDetections/README.md new file mode 100644 index 0000000000..e72ef5529e --- /dev/null +++ b/review-tests/Pr2154.PositiveDetections/README.md @@ -0,0 +1,132 @@ +# PR #2154: generated positive text-detector proof + +This batch addresses review comment `3985472507`. It adds one shared positive +invariant and a typed factory emitted by the actual test scaffold generator for +CRAFT, DBNet, and EAST. No generated leaf test is edited by hand. Production +models, forward methods, decoding, and thresholds are unchanged. + +## What is proved + +The shared fixture draws a deterministic 64-by-64 `HI` bitmap without a font or +external image dependency. A separate generated factory supplies the bounded +Nano profile; ordinary generated factories retain their original defaults. +After a real forward initializes lazy shapes, every trainable chunk must be +writable in place. The fixture controls those actual weights and calls the +production `Predict` and `Detect` paths, without replacing `Forward`. + +| Detector | Controlled positive result at threshold 0.05 | +| --- | --- | +| CRAFT | One region, confidence 0.5, box `(0, 0, 60, 60)` | +| DBNet | One region, confidence 0.5, box `(0, 0, 63, 63)` | +| EAST | 64 eligible score cells and known RBOX distances; real IoU-0.2 suppression leaves eight regions, first box `(-20, -12, 28, 20)` | +| EAST, separated geometry | The same live head changed to distances 0.25 produces 64 nonoverlapping 4-by-4 boxes at exact grid coordinates | + +All positive regions must have a nonempty finite polygon enclosing area, a +positive finite box containing that polygon, the expected confidence, and the +correct source-image dimensions. The EAST raw-output assertions cover all 384 +values of the documented flattened `[1, 384]` public `Predict` result: 64 scores +and five geometry channels. With the same model and image, raising the detection +threshold to 0.75 must reject every known 0.5-confidence region. + +The exact shared geometry assertions also run through the original random-input +factories. Their existing empty-result and box-only behavior is preserved; the +new positive invariant separately rejects empty results and empty polygons. + +This is a **controlled numerical-pipeline and decoder proof**, not evidence of +trained text-recognition accuracy or image-to-label generalization. Zeroing the +weights intentionally makes the known scores analytically predictable. It is +CPU validation, not a physical-GPU or full-repository shard result. The distinct +object-detector positive-fixture and semantic-training findings remain open. + +## Actual before/after evidence + +The focused suite has 18 cases: three generated-factory contracts, three real +generated positive fixtures, three unchanged default-profile geometry replays, +one valid-result oracle control, and eight malformed/empty-result controls. +Invalid controls cover empty regions, empty/degenerate/nonfinite polygons, +inverted/nonfinite boxes, invalid confidence, and a box not containing its +polygon. The nonfinite-box and containment controls adjust the separately +expected first box so another exact-box comparison cannot mask the relevant +missing guard. + +| Run | Passed | Failed | Skipped | +| --- | ---: | ---: | ---: | +| Final 18-case suite, prior generator from `1d8961633155aefd881d7f0f025b0a4f3e6ca625`, .NET 10 | 9 | 9 | 0 | +| Current generator restored, .NET 10 | 18 | 0 | 0 | +| Current generator, .NET 8 | 18 | 0 | 0 | +| Current generator, .NET Framework 4.7.1 | 18 | 0 | 0 | + +All nine before failures identify the absent positive factory: three syntax +assertions and six generated-source compilations reporting only the missing +abstract factory implementation. These are regression-guard failures for the +new generator/base contract, not nine claimed production detector defects. +The nine assertion controls still pass against that prior generator. + +Final result files in `artifacts/pr2154-review/`: + +- `pr2154-positive-text-complete-before.trx` +- `pr2154-positive-text-complete-restored-net10.trx` +- `pr2154-positive-text-complete-net8.0.trx` +- `pr2154-positive-text-net471-final.trx` +- `pr2154-positive-root-independent18.trx` — independent reviewer replay: + 18 passed, zero failed/skipped, .NET 10, 13 seconds. + +All three focused project builds completed with zero warnings and zero errors. +An earlier EAST oracle mistakenly expected a four-dimensional public result; +the actual documented flattened contract was checked and corrected before the +final runs. Earlier harness failures concerning synchronous timeout support or +missing trace-helper references are not included as before/after proof. + +The generator's normal metadata discovery runs against real production model +types. This small discovery compilation omits the repository's unrelated manual +test census, so its global coverage/name-collision diagnostics are printed, not +claimed as whole-project validation. The test requires exactly the three real +hint identities and compiles **every selected generated source** against the +actual shared test base and actual detector assemblies with zero emit errors. +It then invokes the generated classes. No mock detector/base or handwritten +replacement leaf supplies the proof. + +SHA-256 identities: + +| Assembly | SHA-256 | +| --- | --- | +| Prior generator | `488763248BE3B050EB1EACB8BAE44CBAECAE80FCC67797C6A436B83D22993A53` | +| Current generator | `7F99F0DA179EB2C184AEAFCA88BEBD9146DF8C0C16437D72E1DFC76A6D3A80F2` | +| Actual unchanged .NET 10 core | `69BCCF0B1B5AD45EB4050C3D83D352980BB7F0F8EC4D3153EBEBB5469A4B9B8D` | +| Actual unchanged .NET 8 core | `97F5D10424D8B75698A144E248797B18A149F08B376727361A37E609E82B27BA` | +| Actual unchanged .NET Framework core | `B6E19F6F44C9F6E43FFA2A9A29B14B9999B9F76F86669F273644027CA6BEB64B` | + +These are the previously validated shared-layout core binaries; this batch +changes only the scaffold generator and tests. The frozen AP and shared-layout +review runner outputs were not rebuilt or overwritten. + +## Reproduce from the repository root + +First build the current generator and actual Release core for each framework, +or reuse the hash-identified core outputs above. The preceding +`Pr2154.DetectionParameters` proof documents the existing validated CPU native +closure. This runner reuses that closure rather than copying every native RID. +It includes the repository's real module initializer, licensing support, +generated-test trace helper, and xUnit configuration. The imported managed-only +closure target supplies .NET Framework dependencies without a native-tree copy. + +```powershell +$project = 'review-tests/Pr2154.PositiveDetections/Pr2154.PositiveDetections.csproj' +$env:PATH = (Resolve-Path -LiteralPath 'review-tests/Pr2154.DetectionParameters/bin/Release/net10.0').Path + ';' + $env:PATH +dotnet restore $project +dotnet build src/AiDotNet.Generators/AiDotNet.Generators.csproj -c Release --no-restore -p:GeneratePackageOnBuild=false +foreach ($framework in @('net10.0', 'net8.0', 'net471')) { + dotnet build $project -f $framework -c Release --no-restore -m:2 -nodeReuse:false -p:BuildProjectReferences=false -p:GeneratePackageOnBuild=false + if ($LASTEXITCODE -ne 0) { throw "Build failed for $framework" } + dotnet test $project -f $framework -c Release --no-build --no-restore --logger "trx;LogFileName=positive-text-$framework.trx" --results-directory artifacts/pr2154-review + if ($LASTEXITCODE -ne 0) { throw "Tests failed for $framework" } +} +``` + +To reproduce the isolated prior-generator control, supply its exact DLL through +`-p:GeneratorAssemblyPath=` on the focused build. Verify its +SHA-256 against the table before running the same 18 tests; expect nine failures +and nine passes, not a green baseline. Omitting that property on the subsequent +build restores the current generator; verify its copied hash and rerun green. +The property changes only this runner's reference and does not modify either +generator binary or the actual core. diff --git a/review-tests/Pr2154.PositiveObjects/Pr2154.PositiveObjects.csproj b/review-tests/Pr2154.PositiveObjects/Pr2154.PositiveObjects.csproj new file mode 100644 index 0000000000..9e4dbe28d4 --- /dev/null +++ b/review-tests/Pr2154.PositiveObjects/Pr2154.PositiveObjects.csproj @@ -0,0 +1,38 @@ + + + net471;net8.0;net10.0 + AiDotNetTests + latest + enable + enable + true + false + false + false + false + <_GetChildProjectCopyToOutputDirectoryItems>false + true + true + true + $(MSBuildThisFileDirectory)../../src/AiDotNet.Generators/bin/Release/netstandard2.0/AiDotNet.Generators.dll + + + + + + + + + + + + + + + + + + + + + diff --git a/review-tests/Pr2154.PositiveObjects/README.md b/review-tests/Pr2154.PositiveObjects/README.md new file mode 100644 index 0000000000..31ed8e67c2 --- /dev/null +++ b/review-tests/Pr2154.PositiveObjects/README.md @@ -0,0 +1,122 @@ +# PR #2154: controlled positive object detections + +This batch addresses review `3985472491` / `PRRT_kwDOKSXUF86hUmqh`. +It adds a shared positive invariant and a typed factory emitted by the real test +scaffold generator. No generated leaf test or production numerical forward was +edited or substituted. Existing random-input tests continue to permit empty +results and keep their geometry, ordering, and NMS assertions. + +## What was proved + +The runner discovers the actual nine model types through production metadata, +runs the real generator, selects their exact nine generated source identities, +compiles all nine against the actual shared base classes, and invokes them. +The positive factory uses an explicit Nano/64x64/two-class profile. Ordinary +generated factories retain their previous defaults. + +The fixture initializes each real model, requires live writable trainable chunks, +then configures actual weights. `Predict` receives normalized 0/1 pixels; +`Detect` receives the equivalent 0/255 image, matching its public normalization +contract. Each same initialized model is checked at NMS thresholds 1 and 0.45, +then must reject its known scores at confidence 0.99. + +| Actual model/profile | Candidates at NMS 1 | Result at requested NMS 0.45 | Independent checks | +| --- | ---: | ---: | --- | +| YOLOv8/v9/v11 | 84 | 1 | Zero 16-bin DFL logits imply distance 7.5; all clipped boxes are `[0,0,64,64]`. Three live classification biases produce 64 scores of 0.5, 16 of 0.75, and 4 of 0.875. | +| YOLOv10 default | 84 | 84 | Its documented NMS-free default is preserved; the candidates remain confidence-ranked. | +| YOLOv10 with explicit `useNmsFree: false` | 84 | 1 | The actual supported NMS mode suppresses duplicates without replacing the numerical forward. | +| DETR / RT-DETR / DINO | 2 | 1 | Two distinct live query directions pass through actual decoder normalization. An independent scalar normalization/softmax oracle includes the background class. Zero box logits imply exact `[16,16,48,48]` boxes. Scores are approximately 0.952574 and 0.658553. | +| Faster R-CNN / Cascade R-CNN | 275 | 39 | A real spatial channel passes through backbone, FPN, ROIAlign, and classifier. Raw heads establish scores/proposals; independent softmax and greedy IoU/NMS check decoded detections and ordering. | + +For the R-CNN profiles, zero RPN score/delta heads yield fixed proposals and zero +ROI regression heads preserve proposal geometry. The highest-score proposal is +independently pinned to the stride-4, ratio-1/2 anchor centered at `(62,46)`: +`[62-16*sqrt(2), 46-16/sqrt(2), 64, 46+16/sqrt(2)]`. R-CNN's oracle derives other +scores and proposals from actual raw model outputs; it is **not** a separately +implemented reference backbone or evidence of trained recognition accuracy. + +Non-vacuity guards require at least two same-class candidates, genuinely distinct +scores, and an overlap that requires suppression. Independent negative controls +reject empty/missing results, reversed/tied scores, wrong classes/geometry, +non-finite values, wrong image dimensions, ignored NMS, lower-score winners, and +over-suppression. Three additional real-model mutants corrupt only `Detect` +(empty/reverse/ignore NMS), never `Forward`. Each must fail the shared invariant; +`MutationApplied` proves the failure occurred after reaching that corruption, +not at an earlier constructor, live-state, or raw-oracle precondition. + +## Failure-before / success-after evidence + +The same final 49-case test source was compiled against the previous generator +from `7e1cb86a4ba9a5d4a02569d870d28d26a4a5d0de`, then against the new generator. +Both used the same unchanged actual .NET 10 production DLL. The old generator +misses nine positive factories: nine syntax assertions fail and eighteen runtime +cases fail compilation with the exact nine `CS0534` missing-factory diagnostics. +The other 22 controls pass. Those failures are missing coverage infrastructure, +not fabricated claims that the old production models returned empty results. + +| Run | Passed | Failed | Skipped | Report in `artifacts/pr2154-review` | +| --- | ---: | ---: | ---: | --- | +| Final-source historical generator, .NET 10 | 22 | 27 | 0 | `pr2154-positive-object-final-before49.trx` | +| Final .NET 10 | 49 | 0 | 0 | `pr2154-positive-object-final-net10.trx` | +| Final .NET 8 | 49 | 0 | 0 | `pr2154-positive-object-final-net8.trx` | +| Final .NET Framework 4.7.1 | 49 | 0 | 0 | `pr2154-positive-object-final-net471.trx` | +| Independent parent-agent replay, .NET 10 | 49 | 0 | 0 | `pr2154-positive-object-root-independent49.trx` | + +The 49 cases are nine factory contracts, nine actual positive invariants, nine +ordinary generated-fixture replays (each invokes three existing invariants), +19 oracle/precondition controls, and three live-model negative controls. +Generator build: zero errors, 67 analyzer warnings. Focused runner builds: +zero errors on all three frameworks (0/2/0 warnings on net10/net8/net471). +No production core rebuild was required for these test/generator-only changes. +The independent replay verified the exact final test-DLL hash and completed in +31 seconds after a separate full source review of the shared fixture and controls. + +SHA-256 identities: + +| Artifact | SHA-256 | +| --- | --- | +| Historical generator | `7F99F0DA179EB2C184AEAFCA88BEBD9146DF8C0C16437D72E1DFC76A6D3A80F2` | +| Final generator | `1AF4448E70ED82A2248EDC7077225B7925ED9F69E78CE642331D71BC5587141F` | +| Unchanged net10 core | `69BCCF0B1B5AD45EB4050C3D83D352980BB7F0F8EC4D3153EBEBB5469A4B9B8D` | +| Unchanged net8 core | `97F5D10424D8B75698A144E248797B18A149F08B376727361A37E609E82B27BA` | +| Unchanged net471 core | `B6E19F6F44C9F6E43FFA2A9A29B14B9999B9F76F86669F273644027CA6BEB64B` | +| Final net10 test DLL | `6A5E039CB1106415509DDD8F74F38D3EFBDD9602175E07834CF96A4E8EF72AD8` | +| Final net8 test DLL | `B7A103D563DBBF33C38D7D79B8E23C6D639DF128460CFCA2740B13566B4512D5` | +| Final net471 test DLL | `0943A30D97872B3780B261AF566AFF1E8E1FF795A35BC9D11867875A001B424A` | + +## Reproduction + +Run from the repository root after the actual production core has been built for +the selected framework. This small runner reuses that output and the installed +CPU native closure; it does not copy all RID/native assets. Its settings explicitly +disable `CopyLocalRuntimeTargetAssets`, `CopyLocalLockFileAssemblies`, and child +project Content propagation. The actual module initializer, license helper, +generated trace helper, and xUnit configuration are included. .NET Framework uses +the existing managed-only test dependency closure target. + +```powershell +dotnet restore review-tests/Pr2154.PositiveObjects/Pr2154.PositiveObjects.csproj -p:NuGetAudit=false +dotnet build src/AiDotNet.Generators/AiDotNet.Generators.csproj -c Release --no-restore -m:1 -nodeReuse:false +$framework = 'net10.0' # also validated: net8.0 and net471 +dotnet build review-tests/Pr2154.PositiveObjects/Pr2154.PositiveObjects.csproj -f $framework -c Release --no-restore -m:1 -nodeReuse:false -p:BuildProjectReferences=false -p:GeneratePackageOnBuild=false +$cpuNativeDirectory = (Resolve-Path "review-tests/Pr2154.DetectionParameters/bin/Release/$framework").Path +$env:PATH = $cpuNativeDirectory + ';' + $env:PATH +dotnet test review-tests/Pr2154.PositiveObjects/Pr2154.PositiveObjects.csproj -f $framework -c Release --no-build --no-restore --logger 'trx;LogFileName=positive-object-replay.trx' --results-directory artifacts/pr2154-review +``` + +For an isolated historical comparison, pass the path of a generator built from +the exact historical revision above as `GeneratorAssemblyPath`. Verify it before +using it; the retained previous text-fixture output is the local evidence source: + +```powershell +$historicalGenerator = (Resolve-Path 'review-tests/Pr2154.PositiveDetections/bin/Release/net10.0/AiDotNet.Generators.dll').Path +if ((Get-FileHash -LiteralPath $historicalGenerator).Hash -ne '7F99F0DA179EB2C184AEAFCA88BEBD9146DF8C0C16437D72E1DFC76A6D3A80F2') { throw 'Historical generator hash mismatch.' } +dotnet build review-tests/Pr2154.PositiveObjects/Pr2154.PositiveObjects.csproj -f net10.0 -c Release --no-restore -m:1 -nodeReuse:false -p:BuildProjectReferences=false -p:GeneratorAssemblyPath=$historicalGenerator +# Run the same 49 tests; expected outcome: 22 passed, 27 failed, no skips. +# Rebuild without GeneratorAssemblyPath to restore the current generator afterward. +``` + +This is focused local CPU proof, not a full detector-family/all-repository CI run, +GPU validation, a trained accuracy benchmark, or semantic detector-training proof. +The production training review and newly reported metric/text-boundary findings +remain separate open work; this batch alone does not make the PR ready to merge. diff --git a/src/AiDotNet.Generators/LayerStructureInitializationAnalysis.cs b/src/AiDotNet.Generators/LayerStructureInitializationAnalysis.cs new file mode 100644 index 0000000000..6c73857fb7 --- /dev/null +++ b/src/AiDotNet.Generators/LayerStructureInitializationAnalysis.cs @@ -0,0 +1,355 @@ +using System.Collections.Generic; +using System.Linq; +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Microsoft.CodeAnalysis.Operations; + +namespace AiDotNet.Generators; + +/// +/// Proves the narrow case where a layer's initializer cannot construct its optional children. +/// Unknown paths retain ordinary initialization; this is not whole-program side-effect analysis. +/// +internal static class LayerStructureInitializationAnalysis +{ + internal static bool IsChildIndependent( + Compilation compilation, INamedTypeSymbol owner, IReadOnlyList children) + { + var layerBase = compilation.GetTypeByMetadataName("AiDotNet.NeuralNetworks.Layers.LayerBase`1"); + if (layerBase is null || !SymbolEqualityComparer.Default.Equals(owner.BaseType?.OriginalDefinition, layerBase)) + return false; + if (children.Count == 0 || children.Any(child => child.DeclaredAccessibility != Accessibility.Private + || child.NullableAnnotation != NullableAnnotation.Annotated)) + return false; + var initializer = owner.GetMembers("EnsureInitialized").OfType() + .SingleOrDefault(method => !method.IsStatic && method.Parameters.Length == 0); + if (initializer is null) return false; + var proof = new Proof(compilation, owner, children); + foreach (var constructor in owner.InstanceConstructors.Where(constructor => !constructor.IsImplicitlyDeclared)) + if (!proof.VisitMethod(constructor)) return false; + return proof.VisitMethod(initializer); + } + + private sealed class Proof : OperationWalker + { + private const int MaximumMethods = 128; + private readonly Compilation _compilation; + private readonly HashSet _children; + private readonly HashSet _ownerTypes = new(SymbolEqualityComparer.Default); + private readonly HashSet _valueContracts = new(SymbolEqualityComparer.Default); + private readonly HashSet _exceptionContracts = new(SymbolEqualityComparer.Default); + private readonly IPropertySymbol? _memberName; + private readonly INamedTypeSymbol? _runtimeType; + private readonly Dictionary> _visited = new(SymbolEqualityComparer.Default); + private IMethodSymbol? _currentMethod; + private ulong _knownNullParameters; + private int _methodContexts; + private bool _independent = true; + + internal Proof(Compilation compilation, INamedTypeSymbol owner, IEnumerable children) + { + _compilation = compilation; + _memberName = compilation.GetTypeByMetadataName("System.Reflection.MemberInfo")? + .GetMembers(nameof(System.Reflection.MemberInfo.Name)).OfType().SingleOrDefault(); + _runtimeType = compilation.GetTypeByMetadataName("System.Type"); + _children = new HashSet(children.Select(child => child.OriginalDefinition), SymbolEqualityComparer.Default); + for (var type = owner; type is not null && type.SpecialType != SpecialType.System_Object; type = type.BaseType) + _ownerTypes.Add(type.OriginalDefinition); + foreach (var contract in owner.AllInterfaces) _ownerTypes.Add(contract.OriginalDefinition); + + // Resolve contracts to symbols, never match a model/type name at runtime. These APIs + // operate on numeric values, shapes, tensor storage or value-only initialization. An + // arbitrary external receiver is NOT assumed to be independent of the layer graph. + foreach (string metadataName in new[] + { + "AiDotNet.Tensors.LinearAlgebra.Tensor`1", "AiDotNet.Tensors.LinearAlgebra.TensorBase`1", + "AiDotNet.Tensors.LinearAlgebra.Vector`1", + "AiDotNet.Tensors.LinearAlgebra.Matrix`1", "AiDotNet.Tensors.LinearAlgebra.TensorShape", + "AiDotNet.Tensors.LinearAlgebra.WeightRegistry", "AiDotNet.Tensors.Interfaces.INumericOperations`1", + "AiDotNet.Tensors.Interfaces.IVectorizedOperations`1", "AiDotNet.Tensors.Engines.IEngine", + "AiDotNet.Tensors.Engines.AiDotNetEngine", "AiDotNet.Tensors.Helpers.SimdRandom", + "AiDotNet.Tensors.Helpers.MathHelper", "AiDotNet.Helpers.MathHelper", + "AiDotNet.Initialization.IInitializationStrategy`1", "System.Math", + "System.Threading.Interlocked", "System.Runtime.CompilerServices.Unsafe", + "System.MemoryExtensions", "System.Buffers.ArrayPool`1", "System.Collections.Generic.List`1", + "System.Span`1", "System.ReadOnlySpan`1", "System.Memory`1", "System.ReadOnlyMemory`1", + "System.Nullable`1", "System.Array" + }) + { + if (compilation.GetTypeByMetadataName(metadataName) is { } contract) + _valueContracts.Add(contract.OriginalDefinition); + } + foreach (string metadataName in new[] + { + "System.ArgumentException", "System.ArgumentNullException", "System.ArgumentOutOfRangeException", + "System.InvalidOperationException", "System.NotSupportedException", "System.OverflowException" + }) + if (compilation.GetTypeByMetadataName(metadataName) is { } exception) + _exceptionContracts.Add(exception); + } + + internal bool VisitMethod(IMethodSymbol method, ulong knownNullParameters = 0) + { + method = method.OriginalDefinition; + if (!_independent) return false; + if (_visited.TryGetValue(method, out var contexts) && contexts.Contains(knownNullParameters)) return true; + if (_methodContexts++ >= MaximumMethods || method.IsAbstract || method.IsExtern || method.Parameters.Length > 64) + return _independent = false; + if (contexts is null) _visited.Add(method, contexts = new HashSet()); + contexts.Add(knownNullParameters); + + // GetType cannot construct children and has no source body in the compilation. + if (method.ContainingType.SpecialType == SpecialType.System_Object + && method.Name == nameof(object.GetType) && method.Parameters.Length == 0) + return true; + + if (method.DeclaringSyntaxReferences.Length != 1) return _independent = false; + var syntax = method.DeclaringSyntaxReferences[0].GetSyntax(); + SyntaxNode? body = syntax switch + { + MethodDeclarationSyntax declaration => (SyntaxNode?)declaration.Body ?? declaration.ExpressionBody?.Expression, + ConstructorDeclarationSyntax constructor => (SyntaxNode?)constructor.Body ?? constructor.ExpressionBody?.Expression, + AccessorDeclarationSyntax accessor => (SyntaxNode?)accessor.Body ?? accessor.ExpressionBody?.Expression, + PropertyDeclarationSyntax property => property.ExpressionBody?.Expression, + ArrowExpressionClauseSyntax arrow => arrow.Expression, + _ => null + }; + // An auto-property's compiler-generated accessor only accesses its backing field. + if (body is null && syntax is AccessorDeclarationSyntax { Body: null, ExpressionBody: null } accessorSyntax + && accessorSyntax.Parent?.Parent is PropertyDeclarationSyntax { ExpressionBody: null }) + return true; + if (body is null) return _independent = false; + var operation = _compilation.GetSemanticModel(body.SyntaxTree).GetOperation(body); + if (operation is null) return _independent = false; + var previousMethod = _currentMethod; + ulong previousNullParameters = _knownNullParameters; + _currentMethod = method; + _knownNullParameters = knownNullParameters; + try { Visit(operation); } + finally + { + _currentMethod = previousMethod; + _knownNullParameters = previousNullParameters; + } + return _independent; + } + + public override void Visit(IOperation? operation) + { + if (operation is null || !_independent) return; + // A null argument is a call-site fact, not general dataflow. Any write or ref escape + // invalidates this proof instead of incorrectly pruning a later conditional callback. + if (operation is IAssignmentOperation assignment && ContainsKnownNullParameter(assignment.Target) + || operation is ISimpleAssignmentOperation { IsRef: true } + || operation is IVariableDeclaratorOperation { Symbol.RefKind: not RefKind.None } + || operation is IArgumentOperation { Parameter.RefKind: not RefKind.None } argument + && ContainsKnownNullParameter(argument.Value)) + _independent = false; + + // Operators and conversions can execute arbitrary user code without an invocation node. + // Only the same resolved value contracts may supply those methods; unknown/dynamic + // operations and implicit disposal remain outside this narrowly bounded proof. + IMethodSymbol? operatorMethod = operation switch + { + IBinaryOperation binary => binary.OperatorMethod, + IUnaryOperation unary => unary.OperatorMethod, + IConversionOperation conversion => conversion.OperatorMethod, + IIncrementOrDecrementOperation increment => increment.OperatorMethod, + ICompoundAssignmentOperation compound => compound.OperatorMethod, + _ => null + }; + if (operatorMethod is not null && !IsValueContract(operatorMethod.ContainingType) + && !IsRuntimeTypeEquality(operation, operatorMethod)) _independent = false; + if (operation is ICompoundAssignmentOperation compoundAssignment + && (!IsValueConversion(compoundAssignment.InConversion) || !IsValueConversion(compoundAssignment.OutConversion))) + _independent = false; + if (operation.Type?.TypeKind == TypeKind.Dynamic + || operation is IDynamicObjectCreationOperation or IDynamicIndexerAccessOperation + or IDynamicMemberReferenceOperation or ITypeParameterObjectCreationOperation + or IUsingOperation or IUsingDeclarationOperation or IAwaitOperation + or IForEachLoopOperation or IEventAssignmentOperation or IEventReferenceOperation + or ISpreadOperation or IInterpolatedStringHandlerCreationOperation) + _independent = false; + if (operation is ICollectionExpressionOperation { Type: not IArrayTypeSymbol }) _independent = false; + if (operation is IInterpolationOperation interpolation && !IsPrimitiveFormatting(interpolation.Expression.Type)) + _independent = false; + if (_independent) base.Visit(operation); + } + + private static bool IsPrimitiveFormatting(ITypeSymbol? type) => + type is not null && (type.TypeKind == TypeKind.Enum || IsPrimitiveValueType(type.SpecialType)); + + private static bool IsPrimitiveValueType(SpecialType type) => type is + SpecialType.System_Boolean or SpecialType.System_Char or SpecialType.System_String + or SpecialType.System_SByte or SpecialType.System_Byte or SpecialType.System_Int16 + or SpecialType.System_UInt16 or SpecialType.System_Int32 or SpecialType.System_UInt32 + or SpecialType.System_Int64 or SpecialType.System_UInt64 or SpecialType.System_IntPtr + or SpecialType.System_UIntPtr or SpecialType.System_Single or SpecialType.System_Double + or SpecialType.System_Decimal or SpecialType.System_Void; + + private bool IsValueConversion(CommonConversion conversion) => + conversion.MethodSymbol is null || IsValueContract(conversion.MethodSymbol.ContainingType); + + private bool IsRuntimeTypeEquality(IOperation operation, IMethodSymbol method) => + SymbolEqualityComparer.Default.Equals(method.ContainingType, _runtimeType) + && operation is IBinaryOperation + { + OperatorKind: BinaryOperatorKind.Equals or BinaryOperatorKind.NotEquals, + LeftOperand: ITypeOfOperation, + RightOperand: ITypeOfOperation + }; + + private bool ContainsKnownNullParameter(IOperation operation) => + operation is IParameterReferenceOperation && IsKnownNull(operation) + || operation.ChildOperations.Any(ContainsKnownNullParameter); + + public override void VisitObjectCreation(IObjectCreationOperation operation) + { + if (operation.Constructor is not { } constructor + || !IsValueContract(constructor.ContainingType) && !_exceptionContracts.Contains(constructor.ContainingType) + || !ArgumentsAreValues(operation.Arguments)) + _independent = false; + base.VisitObjectCreation(operation); + } + + public override void VisitFieldReference(IFieldReferenceOperation operation) + { + // Even reading a child is rejected: it could be aliased into a later mutation. + if (_children.Contains(operation.Field.OriginalDefinition)) _independent = false; + base.VisitFieldReference(operation); + } + + public override void VisitInvocation(IInvocationOperation operation) + { + var method = operation.TargetMethod; + if (method.MethodKind is MethodKind.DelegateInvoke or MethodKind.LocalFunction) + { + _independent = false; + return; + } + if (operation.Instance is IInstanceReferenceOperation { ReferenceKind: InstanceReferenceKind.ContainingTypeInstance }) + { + bool explicitBase = operation.Syntax is InvocationExpressionSyntax + { Expression: MemberAccessExpressionSyntax { Expression: BaseExpressionSyntax } }; + if (!explicitBase && !method.IsSealed && (method.IsVirtual || method.IsOverride || method.IsAbstract)) + _independent = false; + else + VisitMethod(method, NullArguments(operation)); + } + else if (method.IsStatic && _ownerTypes.Contains(method.ContainingType.OriginalDefinition)) + VisitMethod(method, NullArguments(operation)); + else if (method.ContainingType.SpecialType == SpecialType.System_Object + && method.IsStatic && method.Name == nameof(object.ReferenceEquals)) + { + // Object identity is a framework intrinsic, not a callback into either operand. + } + else if (!IsValueContract(method.ContainingType) || !ArgumentsAreValues(operation.Arguments)) + _independent = false; + // Calls on external tensors/strategies operate only on their typed arguments. A strategy + // secretly capturing the owner to build its graph is not a supported structure contract. + // Owner/child/delegate arguments are rejected by the ordinary descendant walk below. + base.VisitInvocation(operation); + } + + public override void VisitPropertyReference(IPropertyReferenceOperation operation) + { + if (operation.Instance is IInstanceReferenceOperation { ReferenceKind: InstanceReferenceKind.ContainingTypeInstance } + || operation.Property.IsStatic && _ownerTypes.Contains(operation.Property.ContainingType.OriginalDefinition)) + { + var property = operation.Property; + if (!property.IsSealed && (property.IsVirtual || property.IsOverride || property.IsAbstract)) + _independent = false; + else + { + if (property.GetMethod is { } getter) VisitMethod(getter); + // Inspect both accessors even for a read: ++, deconstruction and ref patterns + // must never hide a structural setter behind a non-assignment parent node. + if (property.SetMethod is { } setter) VisitMethod(setter); + } + } + else if (!IsRuntimeTypeName(operation) && !IsValueContract(operation.Property.ContainingType)) + _independent = false; + base.VisitPropertyReference(operation); + } + + private bool IsRuntimeTypeName(IPropertyReferenceOperation operation) => + SymbolEqualityComparer.Default.Equals(operation.Property.OriginalDefinition, _memberName) + && operation.Instance is IInvocationOperation + { + TargetMethod.ContainingType.SpecialType: SpecialType.System_Object, + TargetMethod.Name: nameof(object.GetType), + Arguments.Length: 0 + }; + + private bool IsValueContract(ITypeSymbol? type) + { + if (type is null || type.TypeKind == TypeKind.Error) return false; + if (_ownerTypes.Contains(type.OriginalDefinition)) return false; + if (type is IArrayTypeSymbol array) return IsValueContract(array.ElementType); + if (type.TypeKind is TypeKind.Enum or TypeKind.TypeParameter) return true; + if (IsPrimitiveValueType(type.SpecialType)) return true; + return type is INamedTypeSymbol named && _valueContracts.Contains(named.OriginalDefinition) + && named.TypeArguments.All(IsValueContract); + } + + private bool ArgumentsAreValues(IEnumerable arguments) + { + foreach (var argument in arguments) + { + if (IsKnownNull(argument.Value)) continue; + IOperation value = argument.Value; + while (value is IConversionOperation conversion) value = conversion.Operand; + if (!IsValueContract(value.Type)) return false; + } + return true; + } + + private ulong NullArguments(IInvocationOperation operation) + { + ulong mask = 0; + foreach (var argument in operation.Arguments) + if (argument.Parameter is { Ordinal: < 64 } parameter && IsKnownNull(argument.Value)) + mask |= 1UL << parameter.Ordinal; + return mask; + } + + private bool IsKnownNull(IOperation operation) + { + if (operation.ConstantValue is { HasValue: true, Value: null }) return true; + if (operation is IConversionOperation conversion) return IsKnownNull(conversion.Operand); + return operation is IParameterReferenceOperation reference && reference.Parameter.Ordinal < 64 + && SymbolEqualityComparer.Default.Equals(reference.Parameter.ContainingSymbol.OriginalDefinition, _currentMethod) + && (_knownNullParameters & (1UL << reference.Parameter.Ordinal)) != 0; + } + + public override void VisitConditionalAccess(IConditionalAccessOperation operation) + { + // The shared tensor allocator's optional callback is null at this call site. Track + // that explicit/default constant per method context, never suppress unknown callbacks. + if (IsKnownNull(operation.Operation)) Visit(operation.Operation); + else base.VisitConditionalAccess(operation); + } + + public override void VisitInstanceReference(IInstanceReferenceOperation operation) + { + if (operation.ReferenceKind == InstanceReferenceKind.ContainingTypeInstance) + { + bool receiver = operation.Parent switch + { + IInvocationOperation call => call.Instance == operation, + IFieldReferenceOperation field => field.Instance == operation, + IPropertyReferenceOperation property => property.Instance == operation, + _ => false + }; + if (!receiver) _independent = false; + } + base.VisitInstanceReference(operation); + } + + public override void VisitAnonymousFunction(IAnonymousFunctionOperation operation) => _independent = false; + public override void VisitDelegateCreation(IDelegateCreationOperation operation) => _independent = false; + public override void VisitDynamicInvocation(IDynamicInvocationOperation operation) => _independent = false; + public override void VisitFunctionPointerInvocation(IFunctionPointerInvocationOperation operation) => _independent = false; + public override void VisitInvalid(IInvalidOperation operation) => _independent = false; + } +} diff --git a/src/AiDotNet.Generators/ModelParameterGenerator.cs b/src/AiDotNet.Generators/ModelParameterGenerator.cs index fc9a46b8f8..8202d21ec4 100644 --- a/src/AiDotNet.Generators/ModelParameterGenerator.cs +++ b/src/AiDotNet.Generators/ModelParameterGenerator.cs @@ -375,8 +375,18 @@ and not ParameterMemberSemanticModel.Kind.External var kind = ComponentKindFor(memberType, elem, isDeclaredSlot: member.IsAbstract); if (kind == "one") { + // A nullable-annotated component may legitimately be absent (a detector + // without a neck). A non-optional accessor reports a null component as + // ShapeDeferred with no count, which makes the WHOLE model's layout + // unresolved and every parameter read throw -- the same regression the + // "adapt" branch below documents for absent conditioners. Mark it optional + // so absence is the resolved, parameter-free fact it is; a present + // component is unaffected. + bool absentIsResolved = memberType.NullableAnnotation == NullableAnnotation.Annotated; components.Add((member.Name, - $"new ComponentAccessorParameterSource<{elem}>(() => {member.Name})", + absentIsResolved + ? $"new ComponentAccessorParameterSource<{elem}>(() => {member.Name}, optional: true)" + : $"new ComponentAccessorParameterSource<{elem}>(() => {member.Name})", RoleExpression(classification.Kind), AvailabilityExpression(member, classification.Kind))); continue; diff --git a/src/AiDotNet.Generators/TestScaffoldGenerator.cs b/src/AiDotNet.Generators/TestScaffoldGenerator.cs index 1ae8f195ec..f560744c30 100644 --- a/src/AiDotNet.Generators/TestScaffoldGenerator.cs +++ b/src/AiDotNet.Generators/TestScaffoldGenerator.cs @@ -1,4 +1,4 @@ -using System.Collections.Generic; +using System.Collections.Generic; using System.Collections.Immutable; using System.Linq; using System.Text; @@ -470,6 +470,12 @@ public class TestScaffoldGenerator : IIncrementalGenerator // at full paper scale. The model defaults stay 16B and fully user-customizable; only the default-gate test defers. // KimiVLThinking is the same 16B MoE backbone with a reasoning head — same OOM profile, same deferral. "KimiVL", "KimiVLThinking", + // CascadeRCNN (Cai & Vasconcelos 2018) at paper scale: 1000 proposals through three 12544x1024 stages on + // ResNet-50/FPN, every stage on the tape. Measured alone on the 26-test class: 38.6 GB in double (killed + // the ~16 GB shard-C runner). The bin-major RoIAlign cut it to 27.2 GB; float cut the double run to + // 20.7 GB but broke Detect_ControlledPositiveHead; under a 12 GB GC hard limit two tests still exhaust + // the heap. The live set exceeds the runner, so it runs at paper scale in the nightly heavy lane. + "CascadeRCNN", // MiniGPTv2 (Chen et al. 2023): LLaMA-2-backbone VLM. Already in Fp32, but the LLaMA-2 decoder weights // are ~28 GB even at fp32, so the live J-M run OOMs it (NamedLayerActivations). Float is insufficient // for a 7B-class backbone; defer to the nightly HeavyTimeout lane. Paper defaults (LLaMA-2 scale) intact. @@ -3057,7 +3063,8 @@ private static void Execute( bool canConstruct = (model.HasParameterlessConstructor || model.HasArchitectureOnlyConstructor - || model.HasVectorOnlyConstructor) && + || model.HasVectorOnlyConstructor + || model.HasOptionsOnlyConstructor) && IsCompatibleWithFamily(model, family.Value); // Don't emit a runtime-throwing NotImplementedException stub @@ -3081,9 +3088,11 @@ private static void Execute( // together is what made this class of gap unreadable in the first place. bool hasCtor = model.HasParameterlessConstructor || model.HasArchitectureOnlyConstructor - || model.HasVectorOnlyConstructor; + || model.HasVectorOnlyConstructor + || model.HasOptionsOnlyConstructor; string reason = !hasCtor - ? "it has no supported parameterless, architecture-only, or vector-only constructor, so the " + ? "it has no supported parameterless, architecture-only, vector-only, or options-only " + + "constructor, so the " + "generated fixture has no way to build it" : $"it resolves to test family {family.Value}, whose fixture requires an " + $"interface this type does not implement (see IsCompatibleWithFamily); the " @@ -3148,6 +3157,34 @@ private static void Execute( /// /// Processes a single model type symbol, extracting metadata and checking for test coverage. /// + /// + /// Whether a type can be instantiated with new T(): it has a public constructor taking + /// no arguments, or one whose parameters all have defaults, or it declares none at all and so + /// carries the implicit public parameterless constructor. + /// + private static bool IsConstructibleWithNoArguments(INamedTypeSymbol type) + { + if (type.IsAbstract || type.IsStatic) + return false; + + foreach (var ctor in type.InstanceConstructors) + { + if (ctor.DeclaredAccessibility != Accessibility.Public) + continue; + + bool callable = true; + foreach (var p in ctor.Parameters) + { + if (!p.HasExplicitDefaultValue) { callable = false; break; } + } + + if (callable) + return true; + } + + return false; + } + private static void ProcessModelSymbol( INamedTypeSymbol modelClass, INamedTypeSymbol? domainAttrSymbol, @@ -3317,6 +3354,10 @@ private static void ProcessModelSymbol( bool extendsMultiLabel = false, extendsFinancialNLP = false; bool extendsRiskModel = false, extendsPortfolioOptimizer = false; bool extendsTransformerNER = false, extendsSpanBasedNER = false, extendsSequenceLabelingNER = false; + // Computer-vision detection / OCR. These derive from ModelBase, Tensor> + // rather than NeuralNetworkBase, so without their own families they fell through to + // NeuralNetwork and were rejected for not implementing INeuralNetworkModel. + bool extendsObjectDetector = false, extendsTextDetector = false, extendsOcr = false; var baseType = modelClass.BaseType; while (baseType is not null) @@ -3386,6 +3427,12 @@ private static void ProcessModelSymbol( extendsDocumentNN = true; else if (baseName.StartsWith("VisionLanguageModelBase", System.StringComparison.Ordinal)) extendsVisionLanguage = true; + else if (baseName.StartsWith("ObjectDetectorBase", System.StringComparison.Ordinal)) + extendsObjectDetector = true; + else if (baseName.StartsWith("TextDetectorBase", System.StringComparison.Ordinal)) + extendsTextDetector = true; + else if (baseName.StartsWith("OCRBase", System.StringComparison.Ordinal)) + extendsOcr = true; else if (baseName.StartsWith("SegmentationModelBase", System.StringComparison.Ordinal) || baseName.EndsWith("SegmentationBase", System.StringComparison.Ordinal)) extendsSegmentation = true; @@ -3438,7 +3485,9 @@ private static void ProcessModelSymbol( bool hasParameterlessCtor = false; bool hasArchitectureOnlyCtor = false; bool hasVectorOnlyCtor = false; + bool hasOptionsOnlyCtor = false; string? architectureParamTypeName = null; + string? optionsOnlyParamTypeName = null; foreach (var ctor in modelClass.InstanceConstructors) { if (ctor.DeclaredAccessibility != Accessibility.Public) @@ -3490,6 +3539,35 @@ private static void ProcessModelSymbol( hasVectorOnlyCtor = true; } + // Options-only: the model's single required argument is its own options object, and + // that object can itself be built with no arguments (#2137). The detection-family + // models are the bulk of this: YOLOv8/v10/v11, DETR, RTDETR, DINO and CascadeRCNN all + // take ObjectDetectionOptions and nothing else, and that type declares no + // constructor at all, so `new YOLOv8(new ObjectDetectionOptions())` + // already compiles. The same holds for TextDetectionOptions and OCROptions. + // + // Both halves are required. The name check keeps this to types that are genuinely a + // model's configuration bag rather than any default-constructible dependency, and the + // constructor check is what makes the emitted expression compile -- a name ending in + // "Options" proves nothing on its own. + if (!firstParam.HasExplicitDefaultValue + && restOptional + && firstParam.Type is INamedTypeSymbol optionsType + && StripBacktick(optionsType.Name).EndsWith("Options", System.StringComparison.Ordinal) + && IsConstructibleWithNoArguments(optionsType)) + { + hasOptionsOnlyCtor = true; + string optionsTypeName = optionsType.ToDisplayString(); + if (optionsType.IsGenericType) + { + var unbound = optionsType.ConstructedFrom.ToDisplayString(); + int tick = unbound.IndexOf('<'); + if (tick > 0) unbound = unbound.Substring(0, tick); + optionsTypeName = unbound + ""; + } + optionsOnlyParamTypeName = optionsTypeName; + } + // Check if the first parameter type IS exactly NeuralNetworkArchitecture. // Derived types (CodeSynthesisArchitecture, etc.) have incompatible constructors // and need manual test classes — they stay as NotImplementedException. @@ -3553,6 +3631,12 @@ private static void ProcessModelSymbol( HasParameterlessConstructor = hasParameterlessCtor, HasArchitectureOnlyConstructor = hasArchitectureOnlyCtor, HasVectorOnlyConstructor = hasVectorOnlyCtor, + HasOptionsOnlyConstructor = hasOptionsOnlyCtor, + ImplementsDetectionTraining = domainAttrSymbol?.ContainingAssembly.GetTypeByMetadataName( + "AiDotNet.Interfaces.IDetectionTrainingModel`1") is INamedTypeSymbol detectionTrainingInterface + && modelClass.AllInterfaces.Any(iface => SymbolEqualityComparer.Default.Equals( + iface.OriginalDefinition, detectionTrainingInterface)), + OptionsOnlyParamTypeName = optionsOnlyParamTypeName, InheritsFromExcludedBase = InheritsFromAnyExcludedBase(modelClass), RequestsFloatScaffold = HasFloatScaffoldAttribute(modelClass), ArchitectureParamTypeName = architectureParamTypeName, @@ -3563,6 +3647,9 @@ private static void ProcessModelSymbol( ExtendsDocumentNeuralNetworkBase = extendsDocumentNN, ExtendsVisionLanguageModelBase = extendsVisionLanguage, ExtendsSegmentationModelBase = extendsSegmentation, + ExtendsObjectDetectorBase = extendsObjectDetector, + ExtendsTextDetectorBase = extendsTextDetector, + ExtendsOCRBase = extendsOcr, ExtendsVideoNeuralNetworkBase = extendsVideoNN, ExtendsTtsModelBase = extendsTts, ExtendsFinancialModelBase = extendsFinancial, @@ -4005,6 +4092,20 @@ private static bool ImplementsIFullModel(INamedTypeSymbol type) if (model.ExtendsVisionLanguageModelBase) return TestFamily.VisionLanguage; + // Priority 10a: Object detection (YOLO, DETR/DINO/RT-DETR, Faster/Cascade R-CNN). + // Checked ahead of Segmentation because instance-segmentation detectors carry masks on + // their detections but are still detectors: their invariant set is the box/NMS one. + if (model.ExtendsObjectDetectorBase) + return TestFamily.ObjectDetection; + + // Priority 10b: Text detection (CRAFT, DBNet, EAST). + if (model.ExtendsTextDetectorBase) + return TestFamily.TextDetection; + + // Priority 10c: Text recognition / OCR (CRNN, TrOCR). + if (model.ExtendsOCRBase) + return TestFamily.OCR; + // Priority 11: Segmentation if (model.ExtendsSegmentationModelBase) return TestFamily.Segmentation; @@ -11067,6 +11168,28 @@ private static void EmitGeneratedTestClass( "taskType: AiDotNet.Enums.NeuralNetworkTaskType.Regression, " + "inputHeight: 32, inputWidth: 32, inputDepth: 3, outputSize: 3))"; } + else if (model.HasOptionsOnlyConstructor + && model.TypeParameterCount == 1 + && model.OptionsOnlyParamTypeName is not null) + { + // The model's one required argument is its own options object, and that object is + // constructible with no arguments, so the fixture builds the model from it (#2137). + // The detection families additionally pin InputSize to the fixture's 64x64 image: + // Detect resizes every image to InputSize, and at the 640x640 default a CPU fixture + // spends minutes per call - and DINO/RT-DETR run dense attention over every pyramid + // token, which does not fit at all (#2171). Only the working resolution changes; + // architecture and widths stay at their defaults. The OCR family likewise pins the + // decoding budget: an untrained autoregressive recognizer (TrOCR) almost never emits + // its end token, so every Predict runs to MaxSequenceLength (100 by default) - about + // six seconds per call on a CPU fixture. Sixteen steps exercise the same decoder. + bool pinInputSize = family == TestFamily.ObjectDetection || family == TestFamily.TextDetection; + bool pinDecodeLength = family == TestFamily.OCR; + constructorExpr = pinInputSize + ? $"new {typeName}(new {model.OptionsOnlyParamTypeName} {{ InputSize = new[] {{ 64, 64 }} }})" + : pinDecodeLength + ? $"new {typeName}(new {model.OptionsOnlyParamTypeName} {{ MaxSequenceLength = 16 }})" + : $"new {typeName}(new {model.OptionsOnlyParamTypeName}())"; + } else if (model.HasVectorOnlyConstructor && model.TypeParameterCount == 1) { // A coefficient-backed regression model is only meaningful when its coefficient width @@ -11787,7 +11910,18 @@ private static void EmitGeneratedTestClass( bool isVisionModel = (model.Domains.Contains(1) || model.Domains.Contains(11)) && !model.ExtendsForecastingModelBase; bool isAudioModel = model.Domains.Contains(3); // Audio=3 (was incorrectly 4) - if (model.ClassName == "StableVideoSR") + if (IsTensorModelFamily(family)) + { + // Detection and OCR fixtures declare InputShape only -- see IsTensorModelFamily. The + // OCR base already defaults to a wide, short text crop, so only the detection families + // need a shape here; both stay a multiple of 32 so the feature-pyramid strides divide + // evenly. + if (family != TestFamily.OCR) + { + sb.AppendLine(" protected override int[] InputShape => new[] { 1, 3, 64, 64 };"); + } + } + else if (model.ClassName == "StableVideoSR") { // Keep this in lockstep with the bounded four-level constructor above. An 8x8 input is // the minimum geometry that still traverses every spatial and four-frame temporal stage, @@ -15072,7 +15206,12 @@ model.ClassName is "Bark" or "BarkModel" or "FishSpeech" or "Llasa" or "MegaTTS3 // paper-scale — single-forward tests (DifferentInputs / Clone / Metadata) run at full fidelity. // Emitted after the InputShape chain so it applies regardless of family branch; the set is // disjoint from every other iteration override above, so it cannot double-emit. - if (HeavyTrainingTimeoutClassNames.Contains(model.ClassName)) + // The detection / OCR bases declare none of the properties this block overrides, and the + // set is keyed by SIMPLE class name -- CRAFT, DBNet, EAST, CRNN and TrOCR each name TWO + // distinct models (one under ComputerVision, one under Document/OCR), so an entry added for + // the Document namesake fires on the ComputerVision one too. Skip the block for these + // families rather than widening their bases with knobs they have no invariant for. + if (!IsTensorModelFamily(family) && HeavyTrainingTimeoutClassNames.Contains(model.ClassName)) { // Training_ShouldReduceLoss runs TrainingIterations*3 steps; a deep model's Adam moments // overshoot for the first few steps (Mask2Former: 5.07 -> 7.76 over 3 steps) then descend, @@ -15408,6 +15547,36 @@ model.ClassName is "Mamba2LanguageModel" or "Zamba2LanguageModel" or "RemoteCLIP { sb.AppendLine(factoryBody); } + if (family == TestFamily.TextDetection && model.HasOptionsOnlyConstructor) + { + // The shared positive invariant supplies an explicit bounded profile and controls + // real head weights. Keep the ordinary fixture/default architecture unchanged. + sb.AppendLine(); + sb.AppendLine(" protected override AiDotNet.ComputerVision.Detection.TextDetection.TextDetectorBase CreatePositiveTextDetector("); + sb.AppendLine(" AiDotNet.ComputerVision.Detection.TextDetection.TextDetectionOptions options)"); + sb.AppendLine($" => new {typeName}(options);"); + } + if (family == TestFamily.ObjectDetection && model.HasOptionsOnlyConstructor) + { + // Positive object fixtures use actual model heads with an explicit bounded profile; + // the existing random/default fixture remains responsible for empty-safe invariants. + sb.AppendLine(); + sb.AppendLine(" protected override AiDotNet.ComputerVision.Detection.ObjectDetection.ObjectDetectorBase CreatePositiveObjectDetector("); + sb.AppendLine(" AiDotNet.Models.Options.ObjectDetectionOptions options)"); + sb.AppendLine($" => new {typeName}(options);"); + } + if (family == TestFamily.ObjectDetection && model.ImplementsDetectionTraining) + { + // Emit only for the actual typed capability. Unsupported detector families do not + // inherit a returning/no-op test that would falsely count semantic training as covered. + sb.AppendLine(); + sb.AppendLine(" [Xunit.Fact(Timeout = 180000)]"); + sb.AppendLine(" public async System.Threading.Tasks.Task TrainDetections_ShouldUseSemanticTargetsAndUpdateBothHeads()"); + sb.AppendLine(" {"); + sb.AppendLine(" await System.Threading.Tasks.Task.Yield();"); + sb.AppendLine(" VerifySemanticDetectionTraining();"); + sb.AppendLine(" }"); + } if (model.HasVectorOnlyConstructor) { string featureWidthConstructor = constructorExpr @@ -15612,6 +15781,24 @@ private static string DropDuplicateOverrides(string source) /// Verifies that the model's actual interfaces are compatible with the resolved test family. /// Prevents generating code that won't compile (e.g., casting to wrong interface). /// + /// + /// True for the Tensor -> Tensor computer-vision families, whose fixtures derive from + /// DetectionModelTestBase rather than NeuralNetworkModelTestBase. + /// + /// + /// Those bases deliberately declare a much smaller surface: no OutputShape (a detector's + /// raw head output has no shape contract worth asserting -- the meaningful contract is the + /// decoded DetectionResult, which the family base tests directly) and none of the + /// many-iteration convergence knobs (MoreDataShortIterations, + /// MemorizationTaskLossThreshold and friends). Emitting an override for a member + /// the base does not declare is CS0115, so every emission site that assumes the neural-network + /// base has to consult this first. + /// + private static bool IsTensorModelFamily(TestFamily family) + => family == TestFamily.ObjectDetection + || family == TestFamily.TextDetection + || family == TestFamily.OCR; + private static bool IsCompatibleWithFamily(ModelTestInfo model, TestFamily family) { switch (family) @@ -15659,6 +15846,14 @@ private static bool IsCompatibleWithFamily(ModelTestInfo model, TestFamily famil case TestFamily.GaussianProcess: return model.ImplementsGaussianProcess; + // Detection and OCR families are Tensor -> Tensor IFullModel, NOT INeuralNetworkModel: + // they derive from ModelBase and hold their layers as discrete fields rather than a + // layer collection, so they expose no Layers/GetArchitecture surface to test against. + case TestFamily.ObjectDetection: + case TestFamily.TextDetection: + case TestFamily.OCR: + return model.UsesTensorInput; + // Matrix/Vector families require IFullModel, Vector> case TestFamily.Regression: case TestFamily.NonLinearRegression: @@ -17848,6 +18043,21 @@ private class ModelTestInfo /// public bool HasVectorOnlyConstructor { get; set; } + /// + /// The model's only required constructor argument is its own options object, and that + /// object is itself constructible with no arguments (#2137). + /// + public bool HasOptionsOnlyConstructor { get; set; } + + /// Implements the framework's resolved semantic detection-training interface. + public bool ImplementsDetectionTraining { get; set; } + + /// + /// The options type to instantiate for , already + /// closed over double when generic. + /// + public string? OptionsOnlyParamTypeName { get; set; } + /// /// The fully-qualified display name of the architecture parameter type (e.g., /// "AiDotNet.ProgramSynthesis.Models.CodeSynthesisArchitecture<double>"). @@ -17879,6 +18089,15 @@ private class ModelTestInfo public bool ExtendsDocumentNeuralNetworkBase { get; set; } public bool ExtendsVisionLanguageModelBase { get; set; } public bool ExtendsSegmentationModelBase { get; set; } + + /// True when the model derives from ObjectDetectorBase<T>. + public bool ExtendsObjectDetectorBase { get; set; } + + /// True when the model derives from TextDetectorBase<T>. + public bool ExtendsTextDetectorBase { get; set; } + + /// True when the model derives from OCRBase<T>. + public bool ExtendsOCRBase { get; set; } public bool ExtendsVideoNeuralNetworkBase { get; set; } public bool ExtendsLatentDiffusionModelBase { get; set; } public bool ExtendsTtsModelBase { get; set; } @@ -17977,6 +18196,9 @@ private enum TestFamily Classification, ProbabilisticClassifier, Clustering, + ObjectDetection, + TextDetection, + OCR, NeuralNetwork } @@ -18725,6 +18947,9 @@ private static string GetBaseClassName(TestFamily family) case TestFamily.DocumentNN: return "DocumentNNModelTestBase"; case TestFamily.VisionLanguage: return "VisionLanguageTestBase"; case TestFamily.Segmentation: return "SegmentationTestBase"; + case TestFamily.ObjectDetection: return "ObjectDetectionTestBase"; + case TestFamily.TextDetection: return "TextDetectionTestBase"; + case TestFamily.OCR: return "OCRTestBase"; case TestFamily.VideoNN: return "VideoNNModelTestBase"; case TestFamily.TTS: return "TTSModelTestBase"; case TestFamily.Financial: return "FinancialModelTestBase"; @@ -18856,6 +19081,10 @@ private static string GetReturnTypeCode(TestFamily family) case TestFamily.SequenceLabelingNER: case TestFamily.NeuralNetwork: return "INeuralNetworkModel"; + case TestFamily.ObjectDetection: + case TestFamily.TextDetection: + case TestFamily.OCR: + return "IFullModel, Tensor>"; case TestFamily.ReinforcementLearning: return "IFullModel, Vector>"; case TestFamily.MultiLabelClassifier: diff --git a/src/AiDotNet.Generators/TrainableParameterGenerator.cs b/src/AiDotNet.Generators/TrainableParameterGenerator.cs index a12b1c69d2..beec021d18 100644 --- a/src/AiDotNet.Generators/TrainableParameterGenerator.cs +++ b/src/AiDotNet.Generators/TrainableParameterGenerator.cs @@ -644,7 +644,7 @@ or ParameterMemberSemanticModel.Kind.Scratch // Generate the partial class source var unguardableAxes = new List(); var source = GenerateSource( - classSymbol, paramFields, gradientFields, subLayerFields, bufferFields, + compilation, classSymbol, paramFields, gradientFields, subLayerFields, bufferFields, useRuntimeParameterRegistry, useConventionalTensorEnumerator, emitParameterFreeContract, suppressGeneratedParameterAccessors, unguardableAxes); @@ -676,6 +676,7 @@ private static bool IsIdentifierOrMemberNamed(ExpressionSyntax expression, strin }; private static string GenerateSource( + Compilation compilation, INamedTypeSymbol classSymbol, List paramFields, Dictionary gradientFields, @@ -1561,6 +1562,20 @@ void EmitFieldAssign(ParameterFieldInfo pf, string indexExpr, string idxLabel) sb.AppendLine(" /// Auto-generated: this layer owns child-module structure."); sb.AppendLine(" [global::System.CodeDom.Compiler.GeneratedCode(\"AiDotNet.Generators.TrainableParameterGenerator\", \"1.0.0\")]"); sb.AppendLine(" protected override bool HasDeclaredSubLayerStructure => true;"); + var optionalChildren = subLayerFields + .Select(child => classSymbol.GetMembers(child.Name).OfType().SingleOrDefault()) + .ToArray(); + bool childIndependentInitializer = subLayerFields.All(child => !child.IsCollection && child.InputShape is null) + && optionalChildren.All(child => child is not null) + && LayerStructureInitializationAnalysis.IsChildIndependent( + compilation, classSymbol, optionalChildren.OfType().ToArray()); + if (childIndependentInitializer) + { + sb.AppendLine(); + sb.AppendLine(" /// Only this exact initializer is proven independent of child structure."); + sb.AppendLine(" [global::System.CodeDom.Compiler.GeneratedCode(\"AiDotNet.Generators.TrainableParameterGenerator\", \"1.0.0\")]"); + sb.AppendLine($" protected override bool NeedsDeclaredSubLayerInitialization => GetType() != typeof({classSymbol.ToDisplayString(SymbolDisplayFormat.FullyQualifiedFormat)});"); + } sb.AppendLine(); sb.AppendLine(" private bool _subLayersRegistered;"); sb.AppendLine(); diff --git a/src/AiModelBuilder.Detection.cs b/src/AiModelBuilder.Detection.cs new file mode 100644 index 0000000000..c236d6d844 --- /dev/null +++ b/src/AiModelBuilder.Detection.cs @@ -0,0 +1,37 @@ +using AiDotNet.ComputerVision.Detection; + +namespace AiDotNet; + +/// Explicit semantic detection training through the fluent model-builder facade. +public static class DetectionBuilderExtensions +{ + /// Runs one detection-task update on the configured capable model. + /// + /// Unlike raw tensor Train, this API deliberately selects the configured model's assignment and + /// classification/box objective. Unsupported model families fail explicitly; there is no MSE fallback. + /// Input images must already have the preprocessing expected by the model's Predict method. + /// + public static IAiModelBuilder, Tensor> TrainDetections( + this IAiModelBuilder, Tensor> builder, + Tensor input, DetectionTrainingBatch targets) + { + if (builder is null) throw new ArgumentNullException(nameof(builder)); + if (input is null) throw new ArgumentNullException(nameof(input)); + if (targets is null) throw new ArgumentNullException(nameof(targets)); + if (builder is not AiModelBuilder, Tensor> facade) + throw new NotSupportedException("This operation requires the AiModelBuilder facade."); + if (facade.ConfiguredModel is null) + throw new InvalidOperationException("Configure a detection model before training it."); + if (facade.ConfiguredModel is not IDetectionTrainingModel model) + throw new NotSupportedException("The configured model does not implement semantic detection training."); + model.TrainDetections(input, targets); + return builder; + } + + /// Explicitly adapts the COCO loader's normalized, zero-padded xywh labels before training. + /// This conversion is never selected by guessing the shape passed to raw Train. + public static IAiModelBuilder, Tensor> TrainCocoDetections( + this IAiModelBuilder, Tensor> builder, + Tensor input, Tensor paddedCocoLabels) => + builder.TrainDetections(input, DetectionTrainingBatch.FromPaddedCoco(paddedCocoLabels)); +} diff --git a/src/ComputerVision/CvParameterModule.cs b/src/ComputerVision/CvParameterModule.cs new file mode 100644 index 0000000000..831cc57d78 --- /dev/null +++ b/src/ComputerVision/CvParameterModule.cs @@ -0,0 +1,252 @@ +using AiDotNet.Models.Parameters; + +namespace AiDotNet.ComputerVision; + +/// +/// Base for the hand-rolled building blocks of the computer-vision detection and OCR models - an +/// encoder layer, a cross-attention block, a detection head - that own weights but are neither a +/// LayerBase nor a ModelBase. +/// +/// +/// +/// The parameter generator registers a model field only when its type is a parameter source, a layer +/// or layer collection, or follows the EnumerateLayers() convention. A field typed as a plain +/// helper class matched none of those, so everything behind it was invisible: missing from +/// GetParameters(), from Serialize, from the rebuild-and-reload DeepCopy, and from +/// training. For DETR that was the entire encoder and decoder. +/// +/// +/// Deriving from this class makes a block a parameter source in its own right, so the generator picks +/// it up, and the block exposes its weights as LIVE chunks - the very tensor instances its forward pass +/// reads. That matters for training: the autodiff tape keys gradients by reference, so an optimizer +/// can only update a weight if it is handed that exact instance. +/// +/// +/// A derived block declares, in a fixed order, the child components it owns +/// () and any raw weight tensors it holds directly +/// (). Every child must itself expose live chunks; a child that can +/// only produce a copy is rejected at first use, because silently training a copy would leave the real +/// weight untouched. +/// +/// +/// The numeric type of the weights. +internal abstract class CvParameterModule : IParameterSource, IParameterChunkSource, IParameterLayoutSource +{ + /// + /// The child components this block owns, in a fixed order. Null entries (an optional component + /// the configuration did not build) are skipped. + /// + protected abstract IEnumerable?> ParameterChildren(); + + /// + /// Raw weight tensors this block holds directly (a learnable query embedding, a norm's scale and + /// shift), in a fixed order. They are exposed and restored in place. + /// + protected virtual IEnumerable> OwnParameterTensors() => Array.Empty>(); + + /// + /// + /// Follows the same own-then-child order and relative IDs as the value/chunk surfaces. Child + /// schemas are queried directly: counting or describing the module must not run a forward pass + /// or materialize a lazy child merely to infer its shape from values. + /// + public IReadOnlyList GetParameterLayout() + { + var slots = new List(); + int own = 0; + foreach (var tensor in OwnParameterTensors()) + { + slots.Add(new ParameterSlotDescriptor( + $"w{own}", ParameterSlotRole.Trainable, + tensor.Length == 0 ? ParameterReadiness.ParameterFree : ParameterReadiness.Materialized, + tensor.Length, shape: tensor.Shape.ToArray(), elementType: typeof(T).FullName)); + own++; + } + + int index = 0; + foreach (var child in Children()) + { + IReadOnlyList childSlots = child switch + { + IParameterLayoutSource layout => layout.GetParameterLayout(), + IParameterManifestProvider manifest => manifest.ParameterLayout.Slots, + _ => throw new InvalidOperationException( + $"{GetType().Name} child #{index} ({child.GetType().Name}) must expose a parameter " + + "layout alongside its live chunks so the registry can validate the same state.") + }; + string prefix = ParameterStableId.IndexSegment(index); + foreach (var slot in childSlots) + { + string id = slot.StableId == "$" ? prefix : prefix + "/" + slot.StableId; + slots.Add(new ParameterSlotDescriptor( + id, slot.Role, slot.Readiness, slot.ParameterCount, + shape: slot.Shape, elementType: slot.ElementType, + updatePolicy: slot.UpdatePolicy, persistence: slot.Persistence, + ownership: slot.Ownership, availability: slot.Availability, + materializedParameterCount: slot.MaterializedParameterCount)); + } + index++; + } + return slots; + } + + /// + public long ParameterCount + { + get + { + long total = 0; + foreach (var tensor in OwnParameterTensors()) + { + total += tensor.Length; + } + + foreach (var child in Children()) + { + total += child.ParameterCount; + } + + return total; + } + } + + /// + public Vector GetParameters() + { + var result = new Vector(checked((int)ParameterCount)); + int offset = 0; + foreach (var tensor in OwnParameterTensors()) + { + for (int i = 0; i < tensor.Length; i++) + { + result[offset++] = tensor[i]; + } + } + + foreach (var child in Children()) + { + var values = child.GetParameters(); + for (int i = 0; i < values.Length; i++) + { + result[offset++] = values[i]; + } + } + + return result; + } + + /// + public void SetParameters(Vector parameters) + { + if (parameters is null) + { + throw new ArgumentNullException(nameof(parameters)); + } + + long expected = ParameterCount; + if (parameters.Length != expected) + { + throw new ArgumentException( + $"{GetType().Name} expects {expected} parameter values but received {parameters.Length}.", + nameof(parameters)); + } + + int offset = 0; + foreach (var tensor in OwnParameterTensors()) + { + // Written through, never replaced: the forward pass keeps reading this instance. + for (int i = 0; i < tensor.Length; i++) + { + tensor[i] = parameters[offset++]; + } + } + + foreach (var child in Children()) + { + int count = checked((int)child.ParameterCount); + var slice = new Vector(count); + for (int i = 0; i < count; i++) + { + slice[i] = parameters[offset++]; + } + + child.SetParameters(slice); + } + } + + /// + public IEnumerable> GetParameterStateChunks() + { + int own = 0; + foreach (var tensor in OwnParameterTensors()) + { + if (tensor.Length > 0) + { + yield return new ParameterChunk($"w{own}", ParameterSlotRole.Trainable, tensor); + } + + own++; + } + + int index = 0; + foreach (var child in Children()) + { + if (child is not IParameterChunkSource chunked) + { + throw new InvalidOperationException( + $"{GetType().Name} child #{index} ({child.GetType().Name}) exposes parameters only as a " + + "copy. It must implement IParameterChunkSource so training updates the live weight."); + } + + string prefix = ParameterStableId.IndexSegment(index); + foreach (var chunk in chunked.GetParameterStateChunks()) + { + string id = chunk.StableId == "$" ? prefix : prefix + "/" + chunk.StableId; + yield return new ParameterChunk(id, chunk.Role, chunk.Tensor, chunk.SourceTensor, chunk.IsWritableInPlace); + } + + index++; + } + } + + private IEnumerable> Children() + { + foreach (var child in ParameterChildren()) + { + if (child is not null) + { + yield return child; + } + } + } +} + +/// +/// A whose children and own tensors are supplied by delegates. +/// +/// +/// For public building blocks (the region proposal network, for one) that cannot derive from the +/// internal : they hold one of these and forward +/// and to it. +/// +/// The numeric type of the weights. +internal sealed class DelegatingCvParameterModule : CvParameterModule +{ + private readonly Func?>> _children; + private readonly Func>>? _own; + + /// Creates a module over the given children and, optionally, raw tensors. + public DelegatingCvParameterModule( + Func?>> children, + Func>>? own = null) + { + _children = children ?? throw new ArgumentNullException(nameof(children)); + _own = own; + } + + /// + protected override IEnumerable?> ParameterChildren() => _children(); + + /// + protected override IEnumerable> OwnParameterTensors() => _own?.Invoke() ?? Array.Empty>(); +} diff --git a/src/ComputerVision/CvTensorOps.cs b/src/ComputerVision/CvTensorOps.cs new file mode 100644 index 0000000000..4e96aa8bb6 --- /dev/null +++ b/src/ComputerVision/CvTensorOps.cs @@ -0,0 +1,710 @@ +using AiDotNet.Tensors.Engines; +using AiDotNet.Tensors.Helpers; +using AiDotNet.Tensors.Interfaces; +using AiDotNet.Tensors.LinearAlgebra; + +namespace AiDotNet.ComputerVision; + +/// +/// Tape-visible tensor operations shared by the computer-vision detection and OCR models. +/// +/// +/// +/// These models were written with hand-rolled scalar loops: each helper read elements out one at a +/// time and wrote a freshly allocated tensor. Arithmetically fine, but every such loop severs the +/// autodiff tape, so any trainable layer upstream of it silently received no gradient. +/// +/// +/// Every operation here is composed from engine primitives that record themselves on the tape, and +/// each reproduces the EXACT semantics of the loop it replaces - including the non-standard ones. +/// uses the asymmetric source mapping +/// src = dst * in / out, not the half-pixel convention of a generic interpolate, and +/// keeps the partial edge window rather than dropping it. Swapping in a +/// convenient engine op with a different convention would have shifted every feature map. +/// +/// +/// Resampling and padding are expressed as index selection with precomputed integer indices. A +/// selection is an exact copy, so a remap built from it matches the scalar loop bit for bit, and its +/// backward pass is a scatter-add, which is the correct gradient of a gather. +/// +/// +/// The numeric type of the tensors. +internal static class CvTensorOps +{ + private static IEngine Engine => AiDotNetEngine.Current; + + private static INumericOperations NumOps => MathHelper.GetNumericOperations(); + + /// + /// Nearest-neighbour resize of an NCHW tensor using src = min(dst * in / out, in - 1) + /// on each spatial axis (integer division). This is the convention the neck and head + /// ResizeToMatch helpers used. + /// + public static Tensor ResizeNearest(Tensor x, int targetH, int targetW) + { + int srcH = x.Shape[2]; + int srcW = x.Shape[3]; + if (srcH == targetH && srcW == targetW) + { + return x; + } + + var rows = new int[targetH]; + for (int h = 0; h < targetH; h++) + { + rows[h] = Math.Min((int)((long)h * srcH / targetH), srcH - 1); + } + + var cols = new int[targetW]; + for (int w = 0; w < targetW; w++) + { + cols[w] = Math.Min((int)((long)w * srcW / targetW), srcW - 1); + } + + return Select(Select(x, rows, 2), cols, 3); + } + + /// + /// Nearest-neighbour 2x upsample (each source pixel becomes a 2x2 block). + /// + public static Tensor Upsample2xNearest(Tensor x) + => ResizeNearest(x, x.Shape[2] * 2, x.Shape[3] * 2); + + /// + /// Bilinear resize of an NCHW tensor with the ASYMMETRIC source mapping + /// src = dst * in / out (no half-pixel offset), clamping the upper neighbour to the last + /// row or column. Separable: interpolate along width, then along height, which evaluates + /// wy0*(wx0*v00 + wx1*v01) + wy1*(wx0*v10 + wx1*v11) in the same order as the loop it + /// replaces. + /// + public static Tensor ResizeBilinearAsymmetric(Tensor x, int targetH, int targetW) + { + int srcH = x.Shape[2]; + int srcW = x.Shape[3]; + + var (x0, x1, wx0, wx1) = BilinearTaps(srcW, targetW); + var (y0, y1, wy0, wy1) = BilinearTaps(srcH, targetH); + + // Along width: [N, C, srcH, targetW]. + var left = Select(x, x0, 3); + var right = Select(x, x1, 3); + var alongW = Engine.TensorAdd( + Engine.TensorMultiply(left, Broadcast(wx0, Dims(left), 3)), + Engine.TensorMultiply(right, Broadcast(wx1, Dims(right), 3))); + + // Along height: [N, C, targetH, targetW]. + var top = Select(alongW, y0, 2); + var bottom = Select(alongW, y1, 2); + return Engine.TensorAdd( + Engine.TensorMultiply(Broadcast(wy0, Dims(top), 2), top), + Engine.TensorMultiply(Broadcast(wy1, Dims(bottom), 2), bottom)); + } + + /// + /// 2x2, stride-2 max pooling in CEIL mode: an odd-sized input keeps its partial last window, + /// whose maximum is taken over the in-bounds cells only. + /// + /// + /// Implemented by replicating the last row and column when the extent is odd, then pooling in + /// floor mode. A replicated cell duplicates a value already inside the same window, so it can + /// never change that window's maximum - which is exactly "max over the in-bounds cells". + /// + public static Tensor MaxPool2x2Ceil(Tensor x) + { + int h = x.Shape[2]; + int w = x.Shape[3]; + var padded = x; + if (h % 2 == 1) + { + padded = Select(padded, ClampedRange(0, h + 1, h), 2); + } + + if (w % 2 == 1) + { + padded = Select(padded, ClampedRange(0, w + 1, w), 3); + } + + return Engine.MaxPool2DWithIndices(padded, new[] { 2, 2 }, new[] { 2, 2 }, out _); + } + + /// + /// Max pooling with window and stride both equal to (kernelH, kernelW), in FLOOR mode: + /// trailing rows or columns that do not fill a whole window are dropped. + /// + public static Tensor MaxPoolFloor(Tensor x, int kernelH, int kernelW) + => Engine.MaxPool2DWithIndices(x, new[] { kernelH, kernelW }, new[] { kernelH, kernelW }, out _); + + /// + /// Stride-1 "same" max pooling with a square window of odd size : + /// the output has the input's spatial size and each window's maximum is taken over the cells that + /// fall inside the image (out-of-bounds positions are ignored, not treated as zero). + /// + /// + /// Out-of-bounds positions are filled by clamping their index to the nearest edge. The clamped + /// cell always lies inside the same window, so it duplicates an in-bounds candidate and cannot + /// change the maximum. + /// + public static Tensor MaxPoolSame(Tensor x, int kernelSize) + => MaxPoolPadded(x, kernelSize, 1, kernelSize / 2); + + /// + /// Max pooling with a square window, a stride and symmetric padding, where padded positions are + /// ignored rather than treated as zero - PyTorch's nn.MaxPool2d(kernel, stride, padding) + /// in floor mode. + /// + /// + /// Out-of-bounds positions are filled by clamping their index to the nearest edge. While + /// is smaller than (which PyTorch itself + /// requires), every window reaches at least one in-bounds cell on the side it overhangs, and the + /// clamped cell IS that edge cell - a duplicate candidate that cannot change the maximum. + /// + public static Tensor MaxPoolPadded(Tensor x, int kernelSize, int stride, int padding) + { + if (padding < 0 || padding >= kernelSize) + { + throw new ArgumentOutOfRangeException(nameof(padding), padding, + $"Padding must lie in [0, kernelSize) = [0, {kernelSize})."); + } + + int h = x.Shape[2]; + int w = x.Shape[3]; + int outH = (h + 2 * padding - kernelSize) / stride + 1; + int outW = (w + 2 * padding - kernelSize) / stride + 1; + + // Only the cells some window reads: from -padding to the last window's far edge. + int endH = (outH - 1) * stride - padding + kernelSize; + int endW = (outW - 1) * stride - padding + kernelSize; + var padded = x; + if (padding > 0 || endH != h) + { + padded = Select(padded, ClampedRange(-padding, endH, h), 2); + } + + if (padding > 0 || endW != w) + { + padded = Select(padded, ClampedRange(-padding, endW, w), 3); + } + + return Engine.MaxPool2DWithIndices(padded, new[] { kernelSize, kernelSize }, new[] { stride, stride }, out _); + } + + /// + /// Normalises each channel of an NCHW tensor with the statistics of the CURRENT batch + /// (biased variance, no affine parameters): (x - mean_c) / sqrt(var_c + eps). + /// + public static Tensor BatchStatisticsNorm(Tensor x, double epsilon) + { + var axes = new[] { 0, 2, 3 }; + var mean = Engine.ReduceMean(x, axes, true); + var centered = Engine.TensorSubtract(x, Engine.TensorBroadcastTo(mean, Dims(x))); + var variance = Engine.ReduceMean(Engine.TensorMultiply(centered, centered), axes, true); + var std = Engine.TensorSqrt(Engine.TensorAddScalar(variance, NumOps.FromDouble(epsilon))); + return Engine.TensorDivide(centered, Engine.TensorBroadcastTo(std, Dims(x))); + } + + /// + /// Concatenates NCHW tensors along the channel axis. + /// + public static Tensor ConcatChannels(Tensor first, Tensor second) + => Engine.TensorConcatenate(new[] { first, second }, 1); + + /// + /// Flattens the spatial axes of an NCHW tensor into a token sequence [N, H*W, C], in + /// row-major spatial order. + /// + public static Tensor FlattenSpatial(Tensor x) + { + int n = x.Shape[0], c = x.Shape[1], h = x.Shape[2], w = x.Shape[3]; + return Engine.Reshape(Engine.TensorPermute(x, new[] { 0, 2, 3, 1 }), new[] { n, h * w, c }); + } + + /// + /// Inverse of : [N, H*W, C] back to [N, C, H, W]. + /// + public static Tensor UnflattenSpatial(Tensor tokens, int height, int width) + { + int n = tokens.Shape[0], c = tokens.Shape[2]; + return Engine.TensorPermute(Engine.Reshape(tokens, new[] { n, height, width, c }), new[] { 0, 3, 1, 2 }); + } + + /// + /// Layer normalisation over the last axis with learnable scale and shift: + /// gamma * (x - mean) / sqrt(var + eps) + beta, biased variance. + /// + public static Tensor LayerNormLastAxis(Tensor x, Tensor gamma, Tensor beta, double epsilon) + => Engine.LayerNorm(x, gamma, beta, epsilon, out _, out _); + + /// + /// Multi-head scaled dot-product attention over [N, L, D] sequences. Heads are the + /// contiguous D / numHeads slices of the model dimension, softmax runs over the keys, + /// and the heads are concatenated back in order. No projections: the caller applies those. + /// + /// Queries [N, Lq, D]. + /// Keys [N, Lk, D]. + /// Values [N, Lk, D]. + /// Number of heads; must divide D. + /// Score scale, normally 1 / sqrt(D / numHeads). + /// When true, query i attends only to keys j <= i. + /// Optional additive bias on the scaled scores, broadcastable to + /// [N, H, Lq, Lk] - for example . + /// The attended values [N, Lq, D]. + public static Tensor MultiHeadAttention( + Tensor query, Tensor key, Tensor value, int numHeads, double scale, bool causal = false, + Tensor? scoreBias = null) + { + int n = query.Shape[0], lq = query.Shape[1], d = query.Shape[2], lk = key.Shape[1]; + int headDim = d / numHeads; + + var q = SplitHeads(query, numHeads, headDim); + var k = SplitHeads(key, numHeads, headDim); + var v = SplitHeads(value, numHeads, headDim); + + var scores = Engine.TensorMultiplyScalar( + Engine.TensorMatMul(q, Engine.TensorPermute(k, new[] { 0, 1, 3, 2 })), NumOps.FromDouble(scale)); + + if (scoreBias is not null) + { + scores = Engine.TensorAdd(scores, Engine.TensorBroadcastTo(scoreBias, Dims(scores))); + } + + if (causal) + { + // Additive mask: a large negative score gives the masked key an attention weight that + // underflows to exactly zero after the softmax, matching a loop that skips j > i. + var mask = new Tensor(new[] { 1, 1, lq, lk }); + var blocked = NumOps.FromDouble(-1e30); + for (int i = 0; i < lq; i++) + { + for (int j = i + 1; j < lk; j++) + { + mask[(i * lk) + j] = blocked; + } + } + + scores = Engine.TensorAdd(scores, Engine.TensorBroadcastTo(mask, Dims(scores))); + } + + var weights = Engine.Softmax(scores, -1); + var attended = Engine.TensorMatMul(weights, v); // [N, H, Lq, hd] + return Engine.Reshape(Engine.TensorPermute(attended, new[] { 0, 2, 1, 3 }), new[] { n, lq, d }); + } + + private static Tensor SplitHeads(Tensor x, int numHeads, int headDim) + { + int n = x.Shape[0], l = x.Shape[1]; + return Engine.TensorPermute(Engine.Reshape(x, new[] { n, l, numHeads, headDim }), new[] { 0, 2, 1, 3 }); + } + + /// + /// Concatenates every head's raw output into one tensor. When all outputs share a leading + /// (per-image) dimension N, each is flattened to [N, -1] and the result is + /// [N, total]; when they do not - a two-stage detector's per-RoI heads beside its per-image + /// RPN maps - everything is flattened into [1, total]. A single output is returned unchanged. + /// + public static Tensor ConcatenateOutputs(IReadOnlyList> outputs) + { + if (outputs.Count == 0) + { + return new Tensor(new[] { 1, 0 }); + } + + if (outputs.Count == 1) + { + return outputs[0]; + } + + bool sharedLeading = true; + for (int i = 1; i < outputs.Count && sharedLeading; i++) + { + sharedLeading = outputs[i].Shape[0] == outputs[0].Shape[0] && outputs[i].Shape[0] > 0; + } + + var flat = new Tensor[outputs.Count]; + for (int i = 0; i < outputs.Count; i++) + { + int leading = sharedLeading ? outputs[i].Shape[0] : 1; + flat[i] = Engine.Reshape(outputs[i], new[] { leading, outputs[i].Length / leading }); + } + + return Engine.TensorConcatenate(flat, 1); + } + + /// + /// Builds a Swin relative-position bias [1, H, L, L] from a learnable table + /// [R, H] and an index map index[i, j] into its rows. The lookup is a gather, so the + /// table receives gradients. + /// + public static Tensor RelativePositionBias(Tensor table, int[,] index) + { + int lq = index.GetLength(0), lk = index.GetLength(1), heads = table.Shape[1]; + var flat = new int[lq * lk]; + for (int i = 0; i < lq; i++) + { + for (int j = 0; j < lk; j++) + { + flat[(i * lk) + j] = index[i, j]; + } + } + + var gathered = Select(table, flat, 0); // [L*L, H] + return Engine.Reshape(Engine.TensorPermute(gathered, new[] { 1, 0 }), new[] { 1, heads, lq, lk }); + } + + /// + /// Rolls a [N, H, W, C] map by along both spatial axes with + /// wrap-around: out[i, j] = x[(i - shift) mod H, (j - shift) mod W] (Swin's cyclic shift). + /// + public static Tensor CyclicShift(Tensor x, int shift) + => Engine.TensorRoll(x, new[] { shift, shift }, new[] { 1, 2 }); + + /// + /// Partitions a [N, H, W, C] map into non-overlapping + /// squares, zero-padding the bottom and right edges up to a multiple of the window size. + /// Returns [N * nH * nW, windowSize^2, C] with windows in row-major order per image and + /// tokens in row-major order per window. + /// + public static (Tensor Windows, int WindowsH, int WindowsW) WindowPartition(Tensor x, int windowSize) + { + int n = x.Shape[0], h = x.Shape[1], w = x.Shape[2], c = x.Shape[3]; + int padH = (windowSize - (h % windowSize)) % windowSize; + int padW = (windowSize - (w % windowSize)) % windowSize; + + var padded = ZeroPadBottomRight(x, padH, padW); + + int wh = (h + padH) / windowSize, ww = (w + padW) / windowSize; + var blocks = Engine.Reshape(padded, new[] { n, wh, windowSize, ww, windowSize, c }); + var ordered = Engine.TensorPermute(blocks, new[] { 0, 1, 3, 2, 4, 5 }); // [N, wh, ww, ws, ws, C] + return (Engine.Reshape(ordered, new[] { n * wh * ww, windowSize * windowSize, c }), wh, ww); + } + + /// + /// Inverse of : reassembles windows into a [N, H, W, C] map + /// and drops the padding. + /// + public static Tensor WindowReverse( + Tensor windows, int windowsH, int windowsW, int batch, int height, int width, int windowSize) + { + int c = windows.Shape[2]; + var blocks = Engine.Reshape(windows, new[] { batch, windowsH, windowsW, windowSize, windowSize, c }); + var ordered = Engine.TensorPermute(blocks, new[] { 0, 1, 3, 2, 4, 5 }); // [N, wh, ws, ww, ws, C] + var full = Engine.Reshape(ordered, new[] { batch, windowsH * windowSize, windowsW * windowSize, c }); + if (full.Shape[1] == height && full.Shape[2] == width) + { + return full; + } + + return Engine.TensorSlice(full, new[] { 0, 0, 0, 0 }, new[] { batch, height, width, c }); + } + + /// + /// Applies a row-wise layer (a linear map, typically) independently to every position of a + /// [..., features] tensor by folding the leading axes into one batch axis and unfolding the + /// result. Replaces the copy-one-row, forward, copy-back loops, which were slow and severed the tape. + /// + public static Tensor Tokenwise(Tensor x, Func, Tensor> rowwise) + { + int rank = x.Shape.Length; + if (rank <= 2) + { + return rowwise(x); + } + + int rows = 1; + for (int d = 0; d < rank - 1; d++) + { + rows *= x.Shape[d]; + } + + var result = rowwise(Engine.Reshape(x, new[] { rows, x.Shape[rank - 1] })); + var outShape = new int[rank]; + for (int d = 0; d < rank - 1; d++) + { + outShape[d] = x.Shape[d]; + } + + outShape[rank - 1] = result.Shape[1]; + return Engine.Reshape(result, outShape); + } + + /// + /// Swin patch merging on a [N, H, W, C] map: zero-pads odd sides to even, then + /// concatenates each 2x2 quad's tokens along channels in the order (r0,c0), (r0,c1), (r1,c0), + /// (r1,c1), giving [N, (H/2)*(W/2), 4C] with quads in row-major order. + /// + public static Tensor PatchMerge2x2(Tensor x) + { + int n = x.Shape[0], h = x.Shape[1], w = x.Shape[2], c = x.Shape[3]; + var padded = ZeroPadBottomRight(x, h & 1, w & 1); + int newH = (h + (h & 1)) / 2, newW = (w + (w & 1)) / 2; + var quads = Engine.Reshape(padded, new[] { n, newH, 2, newW, 2, c }); + var ordered = Engine.TensorPermute(quads, new[] { 0, 1, 3, 2, 4, 5 }); // [N, newH, newW, 2, 2, C] + return Engine.Reshape(ordered, new[] { n, newH * newW, 4 * c }); + } + + /// + /// Zero-pads a [N, H, W, C] map with rows at the bottom and + /// columns at the right, by concatenating constant zero blocks (which the + /// tape treats as constants, so the gradient passes straight through to the original cells). + /// + public static Tensor ZeroPadBottomRight(Tensor x, int padH, int padW) + { + int n = x.Shape[0], h = x.Shape[1], w = x.Shape[2], c = x.Shape[3]; + var padded = x; + if (padH > 0) + { + padded = Engine.TensorConcatenate(new[] { padded, new Tensor(new[] { n, padH, w, c }) }, 1); + } + + if (padW > 0) + { + padded = Engine.TensorConcatenate(new[] { padded, new Tensor(new[] { n, h + padH, padW, c }) }, 2); + } + + return padded; + } + + /// + /// RoIAlign: pools each region of interest into an outputSize x outputSize grid by + /// averaging samplingRatio^2 bilinear samples per bin, over an NCHW feature map. + /// + /// + /// + /// The boxes are treated as constants - as in standard RoIAlign, the gradient flows into the + /// FEATURES, not the box coordinates - so the whole operation is a fixed sparse linear map of the + /// feature map. It is built as one gather of the four bilinear corners of every sample, a + /// multiply by the precomputed corner weights (each already divided by the bin's in-bounds sample + /// count), and a sum. Samples that fall outside the map contribute nothing and are not counted; a + /// bin with no in-bounds samples is zero. + /// + /// + /// Feature map [N, C, H, W]. + /// Per-RoI (x1, y1, x2, y2) in image coordinates, length 4 * R. + /// Per-RoI image index into , length R. + /// Image-to-feature-map scale. + /// Pooled grid side. + /// Samples per bin side. + /// Pooled features [R, C, outputSize, outputSize]. + public static Tensor RoIAlign( + Tensor features, double[] boxes, int[] batchIndices, double spatialScale, int outputSize, int samplingRatio) + { + int n = features.Shape[0], c = features.Shape[1], h = features.Shape[2], w = features.Shape[3]; + int rois = batchIndices.Length; + int side = outputSize * samplingRatio; + + if (rois == 0) + { + return new Tensor(new[] { 0, c, outputSize, outputSize }); + } + + // Bilinear sampling through the engine's GridSample (align_corners = false, zero padding; + // NCHW in and out - the IEngine summary says NHWC, but the engine reads [N, C, H, W]): one grid point per sample, so the op stores [rois * side * side, C] values and + // its backward is the engine's native GridSample gradient. The earlier formulation gathered + // all four bilinear taps of every sample as separate rows and broadcast a weight per row - + // three [rois * bins * 4 * samples, C] tensors, several GB per call at a thousand proposals. + // + // Exactness against the per-sample loop: a sample with y in [h - 1, h) read its upper tap + // clamped to row h - 1, i.e. the edge value; clamping the coordinate to h - 1 reproduces that + // under zero padding. A sample outside [0, h) x [0, w) contributes nothing and is excluded + // from its bin's average; it is pointed at a valid pixel and given weight zero. + var gridValues = new T[rois * side * side * 2]; + var maskValues = new T[rois * side * side]; + var zero = NumOps.Zero; + for (int r = 0; r < rois; r++) + { + double x1 = boxes[(4 * r) + 0] * spatialScale, y1 = boxes[(4 * r) + 1] * spatialScale; + double x2 = boxes[(4 * r) + 2] * spatialScale, y2 = boxes[(4 * r) + 3] * spatialScale; + double binW = (x2 - x1) / outputSize, binH = (y2 - y1) / outputSize; + + for (int ph = 0; ph < outputSize; ph++) + { + for (int pw = 0; pw < outputSize; pw++) + { + double startY = y1 + (ph * binH), startX = x1 + (pw * binW); + int count = 0; + for (int iy = 0; iy < samplingRatio; iy++) + { + for (int ix = 0; ix < samplingRatio; ix++) + { + double y = startY + ((iy + 0.5) * binH / samplingRatio); + double x = startX + ((ix + 0.5) * binW / samplingRatio); + if (y >= 0 && y < h && x >= 0 && x < w) + { + count++; + } + } + } + + for (int iy = 0; iy < samplingRatio; iy++) + { + for (int ix = 0; ix < samplingRatio; ix++) + { + double y = startY + ((iy + 0.5) * binH / samplingRatio); + double x = startX + ((ix + 0.5) * binW / samplingRatio); + // Bin-major order: each bin's samplingRatio^2 points are contiguous, so the pooled bin + // is one [1, s*s] x [s*s, C] product below rather than a mask broadcast to every channel. + int point = ((((((r * outputSize) + ph) * outputSize) + pw) * samplingRatio) + iy) * samplingRatio + ix; + bool valid = y >= 0 && y < h && x >= 0 && x < w; + double sy = valid ? Math.Min(y, h - 1) : 0; + double sx = valid ? Math.Min(x, w - 1) : 0; + + // Pixel coordinate p to normalised g under align_corners = false. + gridValues[(2 * point) + 0] = NumOps.FromDouble(((2 * sx) + 1) / w - 1); + gridValues[(2 * point) + 1] = NumOps.FromDouble(((2 * sy) + 1) / h - 1); + maskValues[point] = valid ? NumOps.FromDouble(1.0 / count) : zero; + } + } + } + } + } + + var grid = new Tensor(new[] { rois * side, side, 2 }, new Vector(gridValues)); + var sampled = SampleByBatch(features, grid, batchIndices, side); // [rois * side, side, C] + // The bin average as a batched product. Broadcasting the weights to every channel and multiplying + // materialised two more tensors the size of the samples, and the tape held all three, per stage: about + // 400 MB each in double at a thousand proposals, which is what killed CascadeRCNN's CI runner. + int bins = rois * outputSize * outputSize, perBin = samplingRatio * samplingRatio; + var weights = new Tensor(new[] { bins, 1, perBin }, new Vector(maskValues)); + var pooled = Engine.BatchMatMul(weights, Engine.Reshape(sampled, new[] { bins, perBin, c })); // [bins, 1, C] + return Engine.TensorPermute(Engine.Reshape(pooled, new[] { rois, outputSize, outputSize, c }), new[] { 0, 3, 1, 2 }); + } + + /// + /// Bilinearly samples one image [1, C, H, W] at a grid [1, rows, cols, 2], returning + /// [rows, cols, C]. + /// + private static Tensor SampleImage(Tensor image, Tensor grid, int rows, int cols, int channels) + { + var sampled = Engine.GridSample(image, grid); // [1, C, rows, cols] + return Engine.TensorPermute(Engine.Reshape(sampled, new[] { channels, rows, cols }), new[] { 1, 2, 0 }); + } + + /// + /// Samples each RoI's grid rows (side rows per RoI) from the image its batch index names. + /// + private static Tensor SampleByBatch(Tensor features, Tensor grid, int[] batchIndices, int side) + { + int n = features.Shape[0], c = features.Shape[1]; + int rois = batchIndices.Length; + if (n == 1) + { + return SampleImage(features, Engine.Reshape(grid, new[] { 1, rois * side, side, 2 }), rois * side, side, c); + } + + var parts = new List>(); + var order = new List(); + for (int b = 0; b < n; b++) + { + var members = new List(); + for (int r = 0; r < rois; r++) + { + if (batchIndices[r] == b) + { + members.Add(r); + } + } + + if (members.Count == 0) + { + continue; + } + + var rows = new int[members.Count * side]; + for (int m = 0; m < members.Count; m++) + { + for (int k = 0; k < side; k++) + { + rows[(m * side) + k] = (members[m] * side) + k; + } + } + + var image = Engine.TensorNarrow(features, 0, b, 1); + var imageGrid = Engine.Reshape(Select(grid, rows, 0), new[] { 1, rows.Length, side, 2 }); + parts.Add(SampleImage(image, imageGrid, rows.Length, side, c)); + order.AddRange(members); + } + + var stacked = parts.Count == 1 ? parts[0] : Engine.TensorConcatenate(parts.ToArray(), 0); + + // stacked holds each RoI's side rows in `order`; put them back in RoI order. + var positionOf = new int[rois]; + for (int k = 0; k < order.Count; k++) + { + positionOf[order[k]] = k; + } + + var back = new int[rois * side]; + for (int r = 0; r < rois; r++) + { + for (int k = 0; k < side; k++) + { + back[(r * side) + k] = (positionOf[r] * side) + k; + } + } + + return Select(stacked, back, 0); + } + + /// + /// Gathers slices of along at the given indices. + /// + public static Tensor Select(Tensor x, int[] indices, int axis) + => Engine.TensorGather(x, new Tensor(new[] { indices.Length }, new Vector(indices)), axis); + + /// Indices start .. end-1 clamped into [0, extent-1]. + private static int[] ClampedRange(int start, int end, int extent) + { + var result = new int[end - start]; + for (int i = 0; i < result.Length; i++) + { + result[i] = Math.Min(Math.Max(start + i, 0), extent - 1); + } + + return result; + } + + private static (int[] Lo, int[] Hi, T[] WeightLo, T[] WeightHi) BilinearTaps(int src, int dst) + { + var lo = new int[dst]; + var hi = new int[dst]; + var wLo = new T[dst]; + var wHi = new T[dst]; + for (int i = 0; i < dst; i++) + { + double s = (double)i / dst * src; + int i0 = (int)Math.Floor(s); + lo[i] = i0; + hi[i] = Math.Min(i0 + 1, src - 1); + double frac = s - i0; + wHi[i] = NumOps.FromDouble(frac); + wLo[i] = NumOps.FromDouble(1.0 - frac); + } + + return (lo, hi, wLo, wHi); + } + + /// + /// Expands a per-position weight vector along to the full target shape. + /// + private static Tensor Broadcast(T[] weights, int[] dims, int axis) + { + var viewShape = new int[dims.Length]; + for (int d = 0; d < dims.Length; d++) + { + viewShape[d] = d == axis ? weights.Length : 1; + } + + var view = new Tensor(viewShape, new Vector(weights)); + return Engine.TensorBroadcastTo(view, dims); + } + + private static int[] Dims(Tensor t) + { + var dims = new int[t.Shape.Length]; + for (int i = 0; i < dims.Length; i++) + { + dims[i] = t.Shape[i]; + } + + return dims; + } +} diff --git a/src/ComputerVision/Detection/Backbones/BackboneLayerShims.cs b/src/ComputerVision/Detection/Backbones/BackboneLayerShims.cs index 151376a6bc..ef7016a577 100644 --- a/src/ComputerVision/Detection/Backbones/BackboneLayerShims.cs +++ b/src/ComputerVision/Detection/Backbones/BackboneLayerShims.cs @@ -1,4 +1,5 @@ using System.IO; +using AiDotNet.Models.Parameters; using AiDotNet.NeuralNetworks.Layers; using AiDotNet.Tensors; using AiDotNet.Tensors.Helpers; @@ -15,7 +16,7 @@ namespace AiDotNet.ComputerVision.Detection.Backbones; /// written against the pre-lazy parallel-Conv2D contract. Post-#1209 it is a 30-line /// adapter, not a parallel implementation. /// -internal class Conv2D +internal class Conv2D : IParameterSource, IParameterChunkSource, IParameterLayoutSource { private readonly ConvolutionalLayer _layer; private readonly int _inChannels; @@ -79,6 +80,55 @@ public Tensor Forward(Tensor input) public long GetParameterCount() => _layer.ParameterCount; + /// + /// Describes the same underlying state as the live chunks without initializing lazy weights. + public IReadOnlyList GetParameterLayout() => _layer.GetParameterLayout(); + + // The shim implements IParameterSource by delegating to the layer it wraps. Without this + // the wrapped weights were invisible to ModelBase's parameter registry: a detection or OCR + // model built from these shims reported only its backbone and neck from GetParameters(), so + // every head weight was missing from the flat parameter vector -- and therefore from + // Serialize/Deserialize, and therefore from the rebuild-and-reload DeepCopy, which handed + // back a copy whose head had been re-initialised from scratch. + // + // The wrapped layer is lazy: it resolves its input depth on first Forward(). Before that + // resolution it honestly reports zero parameters rather than throwing, so registration at + // construction is safe and the count fills in once shapes are known. + /// + public long ParameterCount => _layer.IsShapeResolved ? _layer.ParameterCount : 0L; + + /// + public Vector GetParameters() => + _layer.IsShapeResolved ? _layer.GetParameters() : new Vector(0); + + /// + public void SetParameters(Vector parameters) + { + if (parameters.Length == 0) + { + return; + } + + _layer.SetParameters(parameters); + } + + /// + /// + /// Forwards the wrapped layer's own chunks, which are the live tensors its forward pass reads, so + /// a trainer handed these chunks updates the real weights. Before the lazy layer has resolved its + /// shape it owns nothing yet and yields nothing. + /// + public IEnumerable> GetParameterStateChunks() + { + if (!_layer.IsShapeResolved) + { + return Array.Empty>(); + } + + return ((AiDotNet.Models.Parameters.IParameterChunkSource)_layer).GetParameterStateChunks(); + } + + public void WriteParameters(BinaryWriter writer) => BackboneSerialization.WriteLayerParameters(writer, _layer); @@ -118,7 +168,7 @@ public Tensor Bias } /// Thin adapter around for legacy detection-head call sites. -internal class Dense +internal class Dense : IParameterSource, IParameterChunkSource, IParameterLayoutSource { private readonly DenseLayer _layer; private readonly int _inDim; @@ -137,6 +187,38 @@ public Dense(int inDim, int outDim) _layer = new DenseLayer(outDim, (Interfaces.IActivationFunction?)null); } + /// + /// Applies this linear layer independently to every position of a sequence: input + /// [..., inDim], output [..., outDim]. + /// + /// + /// The detection and OCR blocks used to do this one position at a time - copy a row into a fresh + /// [1, inDim] tensor, run Forward, copy the result back - which is both slow and invisible to + /// the autodiff tape. Folding the leading axes into one batch dimension gives the same per-row result + /// (a linear map treats rows independently) through engine reshapes the tape records. + /// + public Tensor ForwardTokens(Tensor input) + { + int rank = input.Shape.Length; + if (rank <= 2) + { + return Forward(input); + } + + var engine = AiDotNet.Tensors.Engines.AiDotNetEngine.Current; + int rows = 1; + var outShape = new int[rank]; + for (int d = 0; d < rank - 1; d++) + { + rows *= input.Shape[d]; + outShape[d] = input.Shape[d]; + } + + outShape[rank - 1] = _outDim; + var flat = engine.Reshape(input, new[] { rows, input.Shape[rank - 1] }); + return engine.Reshape(Forward(flat), outShape); + } + public Tensor Forward(Tensor input) { // Validate runtime input feature size against the shim's @@ -163,6 +245,54 @@ public Tensor Forward(Tensor input) public long GetParameterCount() => _layer.ParameterCount; + /// + public IReadOnlyList GetParameterLayout() => _layer.GetParameterLayout(); + + // The shim implements IParameterSource by delegating to the layer it wraps. Without this + // the wrapped weights were invisible to ModelBase's parameter registry: a detection or OCR + // model built from these shims reported only its backbone and neck from GetParameters(), so + // every head weight was missing from the flat parameter vector -- and therefore from + // Serialize/Deserialize, and therefore from the rebuild-and-reload DeepCopy, which handed + // back a copy whose head had been re-initialised from scratch. + // + // The wrapped layer is lazy: it resolves its input depth on first Forward(). Before that + // resolution it honestly reports zero parameters rather than throwing, so registration at + // construction is safe and the count fills in once shapes are known. + /// + public long ParameterCount => _layer.IsShapeResolved ? _layer.ParameterCount : 0L; + + /// + public Vector GetParameters() => + _layer.IsShapeResolved ? _layer.GetParameters() : new Vector(0); + + /// + public void SetParameters(Vector parameters) + { + if (parameters.Length == 0) + { + return; + } + + _layer.SetParameters(parameters); + } + + /// + /// + /// Forwards the wrapped layer's own chunks, which are the live tensors its forward pass reads, so + /// a trainer handed these chunks updates the real weights. Before the lazy layer has resolved its + /// shape it owns nothing yet and yields nothing. + /// + public IEnumerable> GetParameterStateChunks() + { + if (!_layer.IsShapeResolved) + { + return Array.Empty>(); + } + + return ((AiDotNet.Models.Parameters.IParameterChunkSource)_layer).GetParameterStateChunks(); + } + + public void WriteParameters(BinaryWriter writer) => BackboneSerialization.WriteLayerParameters(writer, _layer); @@ -206,7 +336,7 @@ public Tensor Bias } /// Thin adapter around . -internal class MultiHeadSelfAttention +internal class MultiHeadSelfAttention : IParameterSource, IParameterChunkSource, IParameterLayoutSource { private readonly MultiHeadAttentionLayer _layer; private readonly int _dim; @@ -231,9 +361,217 @@ public MultiHeadSelfAttention(int dim, int numHeads) public long GetParameterCount() => _layer.ParameterCount; + /// + public IReadOnlyList GetParameterLayout() => _layer.GetParameterLayout(); + + // The shim implements IParameterSource by delegating to the layer it wraps. Without this + // the wrapped weights were invisible to ModelBase's parameter registry: a detection or OCR + // model built from these shims reported only its backbone and neck from GetParameters(), so + // every head weight was missing from the flat parameter vector -- and therefore from + // Serialize/Deserialize, and therefore from the rebuild-and-reload DeepCopy, which handed + // back a copy whose head had been re-initialised from scratch. + // + // The wrapped layer is lazy: it resolves its input depth on first Forward(). Before that + // resolution it honestly reports zero parameters rather than throwing, so registration at + // construction is safe and the count fills in once shapes are known. + /// + public long ParameterCount => _layer.IsShapeResolved ? _layer.ParameterCount : 0L; + + /// + public Vector GetParameters() => + _layer.IsShapeResolved ? _layer.GetParameters() : new Vector(0); + + /// + public void SetParameters(Vector parameters) + { + if (parameters.Length == 0) + { + return; + } + + _layer.SetParameters(parameters); + } + + /// + /// + /// Forwards the wrapped layer's own chunks, which are the live tensors its forward pass reads, so + /// a trainer handed these chunks updates the real weights. Before the lazy layer has resolved its + /// shape it owns nothing yet and yields nothing. + /// + public IEnumerable> GetParameterStateChunks() + { + if (!_layer.IsShapeResolved) + { + return Array.Empty>(); + } + + return ((AiDotNet.Models.Parameters.IParameterChunkSource)_layer).GetParameterStateChunks(); + } + + public void WriteParameters(BinaryWriter writer) => BackboneSerialization.WriteLayerParameters(writer, _layer); public void ReadParameters(BinaryReader reader) => BackboneSerialization.ReadLayerParameters(reader, _layer); } + +/// +/// Adapter around for detection heads: 2-D batch +/// normalisation over the channel axis of an NCHW tensor, with learnable scale and shift and running +/// statistics for inference. +/// +/// +/// Batch statistics are used while the owning model is in training mode and the running statistics +/// otherwise, so the owner must forward . The running statistics are not +/// trainable, but they are part of the model and are saved and restored with it. +/// +internal class BatchNorm2D : IParameterSource, IParameterChunkSource, IParameterLayoutSource +{ + private readonly BatchNormalizationLayer _layer; + private readonly int _channels; + + public BatchNorm2D(int channels) + { + if (channels <= 0) throw new ArgumentOutOfRangeException(nameof(channels)); + _channels = channels; + _layer = new BatchNormalizationLayer(channels); + } + + public Tensor Forward(Tensor input) + { + if (input.Shape.Length != 4 || input.Shape[1] != _channels) + { + throw new ArgumentException( + $"BatchNorm2D expects NCHW input with {_channels} channels; got [{string.Join(",", input.Shape)}].", + nameof(input)); + } + + return _layer.Forward(input); + } + + public void SetTrainingMode(bool training) => _layer.SetTrainingMode(training); + + public long GetParameterCount() => _layer.ParameterCount; + + /// + public IReadOnlyList GetParameterLayout() => _layer.GetParameterLayout(); + + /// + public long ParameterCount => _layer.ParameterCount; + + /// + public Vector GetParameters() => _layer.GetParameters(); + + /// + public void SetParameters(Vector parameters) + { + if (parameters.Length == 0) + { + return; + } + + _layer.SetParameters(parameters); + } + + /// + public IEnumerable> GetParameterStateChunks() + => ((AiDotNet.Models.Parameters.IParameterChunkSource)_layer).GetParameterStateChunks(); + + public void WriteParameters(BinaryWriter writer) + { + BackboneSerialization.WriteLayerParameters(writer, _layer); + var ops = MathHelper.GetNumericOperations(); + foreach (var statistic in new[] { _layer.GetRunningMean(), _layer.GetRunningVariance() }) + { + writer.Write(statistic.Length); + for (int i = 0; i < statistic.Length; i++) + { + writer.Write(ops.ToDouble(statistic[i])); + } + } + } + + public void ReadParameters(BinaryReader reader) + { + BackboneSerialization.ReadLayerParameters(reader, _layer); + var ops = MathHelper.GetNumericOperations(); + foreach (var statistic in new[] { _layer.GetRunningMean(), _layer.GetRunningVariance() }) + { + int length = reader.ReadInt32(); + if (length != statistic.Length) + { + throw new InvalidDataException( + $"BatchNorm2D running statistic has {length} values on the wire; the layer has {statistic.Length}."); + } + + for (int i = 0; i < length; i++) + { + statistic[i] = ops.FromDouble(reader.ReadDouble()); + } + } + } +} + +/// +/// Adapter around for detection heads: a transposed 2-D +/// convolution with no activation (the layer's own default is ReLU, so identity is passed explicitly). +/// +internal class ConvTranspose2D : IParameterSource, IParameterChunkSource, IParameterLayoutSource +{ + private readonly DeconvolutionalLayer _layer; + private readonly int _inChannels; + + public ConvTranspose2D(int inChannels, int outChannels, int kernelSize, int stride) + { + if (inChannels <= 0) throw new ArgumentOutOfRangeException(nameof(inChannels)); + if (outChannels <= 0) throw new ArgumentOutOfRangeException(nameof(outChannels)); + _inChannels = inChannels; + _layer = new DeconvolutionalLayer(outChannels, kernelSize, stride, padding: 0, + activationFunction: new AiDotNet.ActivationFunctions.IdentityActivation()); + } + + public Tensor Forward(Tensor input) + { + if (input.Shape.Length != 4 || input.Shape[1] != _inChannels) + { + throw new ArgumentException( + $"ConvTranspose2D expects NCHW input with {_inChannels} channels; got [{string.Join(",", input.Shape)}].", + nameof(input)); + } + + return _layer.Forward(input); + } + + public long GetParameterCount() => _layer.IsShapeResolved ? _layer.ParameterCount : 0L; + + /// + public IReadOnlyList GetParameterLayout() => _layer.GetParameterLayout(); + + /// + public long ParameterCount => GetParameterCount(); + + /// + public Vector GetParameters() => _layer.IsShapeResolved ? _layer.GetParameters() : new Vector(0); + + /// + public void SetParameters(Vector parameters) + { + if (parameters.Length == 0) + { + return; + } + + _layer.SetParameters(parameters); + } + + /// + public IEnumerable> GetParameterStateChunks() + => _layer.IsShapeResolved + ? ((AiDotNet.Models.Parameters.IParameterChunkSource)_layer).GetParameterStateChunks() + : Array.Empty>(); + + public void WriteParameters(BinaryWriter writer) => BackboneSerialization.WriteLayerParameters(writer, _layer); + + public void ReadParameters(BinaryReader reader) => BackboneSerialization.ReadLayerParameters(reader, _layer); +} diff --git a/src/ComputerVision/Detection/Backbones/BackboneOps.cs b/src/ComputerVision/Detection/Backbones/BackboneOps.cs index 248f28651d..d16eef82ea 100644 --- a/src/ComputerVision/Detection/Backbones/BackboneOps.cs +++ b/src/ComputerVision/Detection/Backbones/BackboneOps.cs @@ -4,47 +4,12 @@ namespace AiDotNet.ComputerVision.Detection.Backbones; /// -/// Shared CPU-side tensor primitives reused by every detection backbone -/// (ResNet stem ReLU + MaxPool, EfficientNet swish, etc.). Replaces the -/// duplicated nested loops that lived in each backbone before -/// BackboneBase was deleted. +/// Shared tensor primitives reused by the detection backbones. Every op here must go through the +/// engine so the gradient tape records it; the ResNet stem's max pool, which used to live here as +/// an element loop, is now . /// internal static class BackboneOps { - private static readonly INumericOperations Ops = MathHelper.GetNumericOperations(); - - public static Tensor MaxPool2D(Tensor x, int kernelSize, int stride, int padding) - { - int batch = x.Shape[0]; - int channels = x.Shape[1]; - int height = x.Shape[2]; - int width = x.Shape[3]; - int outH = (height + 2 * padding - kernelSize) / stride + 1; - int outW = (width + 2 * padding - kernelSize) / stride + 1; - var output = new Tensor(new[] { batch, channels, outH, outW }); - - for (int n = 0; n < batch; n++) - for (int c = 0; c < channels; c++) - for (int oh = 0; oh < outH; oh++) - for (int ow = 0; ow < outW; ow++) - { - double maxVal = double.NegativeInfinity; - for (int kh = 0; kh < kernelSize; kh++) - for (int kw = 0; kw < kernelSize; kw++) - { - int ih = oh * stride - padding + kh; - int iw = ow * stride - padding + kw; - if (ih >= 0 && ih < height && iw >= 0 && iw < width) - { - double v = Ops.ToDouble(x[n, c, ih, iw]); - if (v > maxVal) maxVal = v; - } - } - output[n, c, oh, ow] = Ops.FromDouble(maxVal == double.NegativeInfinity ? 0 : maxVal); - } - return output; - } - /// /// Element-wise residual addition (a + b in-place into a fresh tensor of a's shape). /// Validates BOTH length and rank-by-rank shape so a same-element-count but @@ -63,10 +28,9 @@ public static Tensor AddResidual(Tensor a, Tensor b) $"BackboneOps.AddResidual shape mismatch at axis {axis}: " + $"[{string.Join(",", a._shape)}] vs [{string.Join(",", b._shape)}]."); } - var result = new Tensor(a._shape); - for (int i = 0; i < a.Length; i++) - result[i] = Ops.Add(a[i], b[i]); - return result; + // Engine add, not an element loop: a residual skip that drops to scalars severs the gradient + // for every layer before it, which in a deep backbone is nearly all of them. + return AiDotNetEngine.Current.TensorAdd(a, b); } // ApplyReLU / ApplySiLU / ApplySwish removed — backbones (ResNet, CSPDarknet, diff --git a/src/ComputerVision/Detection/Backbones/EfficientNet.cs b/src/ComputerVision/Detection/Backbones/EfficientNet.cs index fac29a1646..24b9c27304 100644 --- a/src/ComputerVision/Detection/Backbones/EfficientNet.cs +++ b/src/ComputerVision/Detection/Backbones/EfficientNet.cs @@ -1,3 +1,4 @@ +using AiDotNet.Tensors.Engines; using System.IO; using AiDotNet.ActivationFunctions; using AiDotNet.Attributes; @@ -410,19 +411,12 @@ public Tensor Forward(Tensor input) int height = input.Shape[2]; int width = input.Shape[3]; - // Global average pool → [batch, channels] - var squeezed = new Tensor(new[] { batch, channels }); - for (int n = 0; n < batch; n++) - { - for (int c = 0; c < channels; c++) - { - double sum = 0; - for (int h = 0; h < height; h++) - for (int w = 0; w < width; w++) - sum += _numOps.ToDouble(input[n, c, h, w]); - squeezed[n, c] = _numOps.FromDouble(sum / (height * width)); - } - } + var engine = AiDotNetEngine.Current; + + // Global average pool -> [batch, channels]. Engine ops throughout: the pooled loop and the + // per-channel rescale loop below each severed the tape, so neither the SE block nor anything + // upstream of it in the MBConv block trained. + var squeezed = engine.ReduceMean(input, new[] { 2, 3 }, false); var excited = _fc1.Forward(squeezed); excited = _activation.Activate(excited); @@ -432,19 +426,11 @@ public Tensor Forward(Tensor input) // be in [0,1] to act as a multiplicative attention mask. excited = ApplySigmoid(excited); - var output = new Tensor(input._shape); - for (int n = 0; n < batch; n++) - { - for (int c = 0; c < channels; c++) - { - T scale = excited[n, c]; - for (int h = 0; h < height; h++) - for (int w = 0; w < width; w++) - output[n, c, h, w] = _numOps.Multiply(input[n, c, h, w], scale); - } - } - - return output; + // Per-channel gate: broadcast [batch, channels] over the spatial axes. + var gate = engine.TensorBroadcastTo( + engine.Reshape(excited, new[] { batch, channels, 1, 1 }), + new[] { batch, channels, height, width }); + return engine.TensorMultiply(input, gate); } public long GetParameterCount() => _fc1.ParameterCount + _fc2.ParameterCount; @@ -463,14 +449,14 @@ public void ReadParameters(BinaryReader reader) // ApplySwish moved to BackboneOps.ApplySwish — was duplicated 3 times in this file. - private Tensor ApplySigmoid(Tensor x) - { - var result = new Tensor(x._shape); - for (int i = 0; i < x.Length; i++) - { - double val = _numOps.ToDouble(x[i]); - result[i] = _numOps.FromDouble(1.0 / (1.0 + Math.Exp(-val))); - } - return result; - } + /// + /// Elementwise Sigmoid, delegated to the engine. + /// + /// + /// This was a scalar loop that read each element out to double and wrote a fresh + /// tensor. Arithmetically identical, but it severed the autodiff tape: the gradient chain + /// stopped here, so every trainable layer UPSTREAM of this call received no gradient and + /// silently never trained. The engine op records itself on the tape. + /// + private Tensor ApplySigmoid(Tensor x) => AiDotNetEngine.Current.Sigmoid(x); } diff --git a/src/ComputerVision/Detection/Backbones/ResNet.cs b/src/ComputerVision/Detection/Backbones/ResNet.cs index 42b5a2446a..575b4c7a8a 100644 --- a/src/ComputerVision/Detection/Backbones/ResNet.cs +++ b/src/ComputerVision/Detection/Backbones/ResNet.cs @@ -148,7 +148,7 @@ public List> ExtractFeatures(Tensor input) var features = new List>(); var x = _conv1.Forward(input); x = _activation.Activate(x); - x = BackboneOps.MaxPool2D(x, kernelSize: 3, stride: 2, padding: 1); + x = CvTensorOps.MaxPoolPadded(x, kernelSize: 3, stride: 2, padding: 1); for (int i = 0; i < _stages.Count; i++) { x = _stages[i].Forward(x); @@ -183,7 +183,7 @@ public override Dictionary> GetNamedLayerActivations(Tensor var activations = new Dictionary>(); var x = _conv1.Forward(input); x = _activation.Activate(x); - x = BackboneOps.MaxPool2D(x, kernelSize: 3, stride: 2, padding: 1); + x = CvTensorOps.MaxPoolPadded(x, kernelSize: 3, stride: 2, padding: 1); activations["Stem"] = x.Clone(); for (int i = 0; i < _stages.Count; i++) { diff --git a/src/ComputerVision/Detection/Backbones/SwinTransformer.cs b/src/ComputerVision/Detection/Backbones/SwinTransformer.cs index a2c02a6a26..fe977052df 100644 --- a/src/ComputerVision/Detection/Backbones/SwinTransformer.cs +++ b/src/ComputerVision/Detection/Backbones/SwinTransformer.cs @@ -181,16 +181,7 @@ private Tensor ReshapeToFeatureMap(Tensor x, int height, int width, int st nameof(height)); } - var featureMap = new Tensor(new[] { batch, dim, height, width }); - for (int n = 0; n < batch; n++) - for (int h = 0; h < height; h++) - for (int w = 0; w < width; w++) - { - int seqIdx = h * width + w; - for (int c = 0; c < dim; c++) - featureMap[n, c, h, w] = x[n, seqIdx, c]; - } - return featureMap; + return CvTensorOps.UnflattenSpatial(x, height, width); } // Most-square factor pair (H >= W) of seqLen, or (0, 0) if seqLen <= 0. @@ -296,6 +287,28 @@ public override void Train(Tensor input, Tensor expectedOutput) => public override IFullModel, Tensor> WithParameters(Vector parameters) => throw new NotSupportedException( $"{GetType().Name}: WithParameters(Vector) is unsupported on backbones."); + + /// + /// Registers the Swin weights that are not layers: every block's two layer norms and its + /// relative-position bias table. + /// + /// + /// The generated registration follows the EnumerateLayers() convention, which yields only + /// LayerBase instances, so these tensors were outside the parameter registry - never saved, + /// cloned, or trained. They are exposed as live tensors so a tape-based step updates the real ones. + /// + protected override void RegisterComponents() + { + base.RegisterComponents(); + // A method group, not a lambda over _stages: the registration scan links every member named in + // this call to the registered source, and _stages would then be classified as trainable state + // the state registry must persist -- which it cannot for List> (ADN0062), and must + // not, since this source and the generated layer registration already own every tensor in it. + RegisterParameterComponent(new AiDotNet.Models.Parameters.TensorListParameterSource(StageParameterTensors)); + } + + private List> StageParameterTensors() + => _stages.SelectMany(stage => stage.ExtraParameterTensors()).ToList(); } /// @@ -335,26 +348,7 @@ public PatchEmbeddingBlock(int patchSize, int embedDim) _proj = new ConvolutionalLayer(outputDepth: embedDim, kernelSize: patchSize, stride: patchSize, padding: 0); } - public Tensor Forward(Tensor input) - { - var x = _proj.Forward(input); - int batch = x.Shape[0]; - int channels = x.Shape[1]; - int height = x.Shape[2]; - int width = x.Shape[3]; - int numPatches = height * width; - - var sequence = new Tensor(new[] { batch, numPatches, channels }); - for (int n = 0; n < batch; n++) - for (int h = 0; h < height; h++) - for (int w = 0; w < width; w++) - { - int seqIdx = h * width + w; - for (int c = 0; c < channels; c++) - sequence[n, seqIdx, c] = x[n, c, h, w]; - } - return sequence; - } + public Tensor Forward(Tensor input) => CvTensorOps.FlattenSpatial(_proj.Forward(input)); public long GetParameterCount() => _proj.ParameterCount; @@ -470,6 +464,14 @@ public void ReadParameters(BinaryReader reader) foreach (var block in _blocks) block.ReadParameters(reader); if (_patchMerge is not null) _patchMerge.ReadParameters(reader); } + + /// Every block's , in order. + internal IEnumerable> ExtraParameterTensors() + { + foreach (var block in _blocks) + foreach (var tensor in block.ExtraParameterTensors()) + yield return tensor; + } } /// @@ -625,249 +627,40 @@ private Tensor WindowAttention(Tensor x, int h, int w) } private Tensor ReshapeToSpatial(Tensor x, int batch, int h, int w, int c) - { - var spatial = new Tensor(new[] { batch, h, w, c }); - for (int b = 0; b < batch; b++) - for (int i = 0; i < h; i++) - for (int j = 0; j < w; j++) - { - int seqIdx = i * w + j; - for (int d = 0; d < c; d++) spatial[b, i, j, d] = x[b, seqIdx, d]; - } - return spatial; - } + => AiDotNetEngine.Current.Reshape(x, new[] { batch, h, w, c }); private Tensor ReshapeToSequence(Tensor spatial) - { - int batch = spatial.Shape[0]; - int h = spatial.Shape[1]; - int w = spatial.Shape[2]; - int c = spatial.Shape[3]; - var seq = new Tensor(new[] { batch, h * w, c }); - for (int b = 0; b < batch; b++) - for (int i = 0; i < h; i++) - for (int j = 0; j < w; j++) - { - int seqIdx = i * w + j; - for (int d = 0; d < c; d++) seq[b, seqIdx, d] = spatial[b, i, j, d]; - } - return seq; - } - - private Tensor CyclicShift(Tensor x, int shift) - { - int batch = x.Shape[0]; - int h = x.Shape[1]; - int w = x.Shape[2]; - int c = x.Shape[3]; - var shifted = new Tensor(x._shape); - for (int b = 0; b < batch; b++) - for (int i = 0; i < h; i++) - for (int j = 0; j < w; j++) - { - int srcI = (i - shift % h + h) % h; - int srcJ = (j - shift % w + w) % w; - for (int d = 0; d < c; d++) shifted[b, i, j, d] = x[b, srcI, srcJ, d]; - } - return shifted; - } + => AiDotNetEngine.Current.Reshape( + spatial, new[] { spatial.Shape[0], spatial.Shape[1] * spatial.Shape[2], spatial.Shape[3] }); - private (Tensor windows, int numWindowsH, int numWindowsW) WindowPartition(Tensor x) - { - int batch = x.Shape[0]; - int h = x.Shape[1]; - int w = x.Shape[2]; - int c = x.Shape[3]; + private Tensor CyclicShift(Tensor x, int shift) => CvTensorOps.CyclicShift(x, shift); - int padH = (_windowSize - h % _windowSize) % _windowSize; - int padW = (_windowSize - w % _windowSize) % _windowSize; - int paddedH = h + padH; - int paddedW = w + padW; - - Tensor padded; - if (padH > 0 || padW > 0) - { - padded = new Tensor(new[] { batch, paddedH, paddedW, c }); - for (int b = 0; b < batch; b++) - for (int i = 0; i < paddedH; i++) - for (int j = 0; j < paddedW; j++) - for (int d = 0; d < c; d++) - padded[b, i, j, d] = (i < h && j < w) ? x[b, i, j, d] : _numOps.FromDouble(0.0); - } - else - { - padded = x; - paddedH = h; - paddedW = w; - } - - int numWindowsH = paddedH / _windowSize; - int numWindowsW = paddedW / _windowSize; - int numWindows = numWindowsH * numWindowsW; - int windowArea = _windowSize * _windowSize; - - var windows = new Tensor(new[] { batch * numWindows, windowArea, c }); - for (int b = 0; b < batch; b++) - for (int wh = 0; wh < numWindowsH; wh++) - for (int ww = 0; ww < numWindowsW; ww++) - { - int windowIdx = b * numWindows + wh * numWindowsW + ww; - int startH = wh * _windowSize; - int startW = ww * _windowSize; - for (int i = 0; i < _windowSize; i++) - for (int j = 0; j < _windowSize; j++) - { - int tokenIdx = i * _windowSize + j; - for (int d = 0; d < c; d++) - windows[windowIdx, tokenIdx, d] = padded[b, startH + i, startW + j, d]; - } - } - return (windows, numWindowsH, numWindowsW); - } + private (Tensor Windows, int NumWindowsH, int NumWindowsW) WindowPartition(Tensor x) + => CvTensorOps.WindowPartition(x, _windowSize); private Tensor WindowReverse(Tensor windows, int numWindowsH, int numWindowsW, int batch, int h, int w) - { - int numWindows = numWindowsH * numWindowsW; - int c = windows.Shape[2]; - - var spatial = new Tensor(new[] { batch, h, w, c }); - for (int b = 0; b < batch; b++) - for (int wh = 0; wh < numWindowsH; wh++) - for (int ww = 0; ww < numWindowsW; ww++) - { - int windowIdx = b * numWindows + wh * numWindowsW + ww; - int startH = wh * _windowSize; - int startW = ww * _windowSize; - for (int i = 0; i < _windowSize; i++) - for (int j = 0; j < _windowSize; j++) - { - int outH = startH + i; - int outW = startW + j; - if (outH < h && outW < w) - { - int tokenIdx = i * _windowSize + j; - for (int d = 0; d < c; d++) - spatial[b, outH, outW, d] = windows[windowIdx, tokenIdx, d]; - } - } - } - return spatial; - } + => CvTensorOps.WindowReverse(windows, numWindowsH, numWindowsW, batch, h, w, _windowSize); private Tensor WindowedSelfAttention(Tensor windows) { - int numWindows = windows.Shape[0]; - int windowArea = windows.Shape[1]; + var engine = AiDotNetEngine.Current; int c = windows.Shape[2]; - var qkv = new Tensor(new[] { numWindows, windowArea, 3 * c }); - for (int wIdx = 0; wIdx < numWindows; wIdx++) - { - for (int t = 0; t < windowArea; t++) - { - var tokenIn = new Tensor(new[] { 1, c }); - for (int d = 0; d < c; d++) tokenIn[0, d] = windows[wIdx, t, d]; - var tokenQkv = _qkvProj.Forward(tokenIn); - for (int d = 0; d < 3 * c; d++) qkv[wIdx, t, d] = tokenQkv[0, d]; - } - } + // One fused projection, then split into Q, K and V along the feature axis. + var qkv = CvTensorOps.Tokenwise(windows, _qkvProj.Forward); + var q = engine.TensorNarrow(qkv, 2, 0, c); + var k = engine.TensorNarrow(qkv, 2, c, c); + var v = engine.TensorNarrow(qkv, 2, 2 * c, c); - var output = new Tensor(new[] { numWindows, windowArea, c }); - for (int wIdx = 0; wIdx < numWindows; wIdx++) - { - var attnScores = new double[_numHeads, windowArea, windowArea]; - for (int head = 0; head < _numHeads; head++) - { - int headOffset = head * _headDim; - for (int i = 0; i < windowArea; i++) - for (int j = 0; j < windowArea; j++) - { - double score = 0; - for (int d = 0; d < _headDim; d++) - { - double q = _numOps.ToDouble(qkv[wIdx, i, headOffset + d]); - double k = _numOps.ToDouble(qkv[wIdx, j, c + headOffset + d]); - score += q * k; - } - score *= _scale; - int biasIdx = _relativePositionIndex[i, j]; - score += _numOps.ToDouble(_relativePositionBiasTable[biasIdx, head]); - attnScores[head, i, j] = score; - } - } - - var attnProbs = new double[_numHeads, windowArea, windowArea]; - for (int head = 0; head < _numHeads; head++) - { - for (int i = 0; i < windowArea; i++) - { - double maxScore = double.NegativeInfinity; - for (int j = 0; j < windowArea; j++) - if (attnScores[head, i, j] > maxScore) maxScore = attnScores[head, i, j]; - - double sumExp = 0; - for (int j = 0; j < windowArea; j++) - { - attnProbs[head, i, j] = Math.Exp(attnScores[head, i, j] - maxScore); - sumExp += attnProbs[head, i, j]; - } - for (int j = 0; j < windowArea; j++) attnProbs[head, i, j] /= sumExp; - } - } - - var attnOut = new double[windowArea, c]; - for (int head = 0; head < _numHeads; head++) - { - int headOffset = head * _headDim; - int vOffset = 2 * c + headOffset; - for (int i = 0; i < windowArea; i++) - for (int d = 0; d < _headDim; d++) - { - double val = 0; - for (int j = 0; j < windowArea; j++) - val += attnProbs[head, i, j] * _numOps.ToDouble(qkv[wIdx, j, vOffset + d]); - attnOut[i, headOffset + d] = val; - } - } - - for (int t = 0; t < windowArea; t++) - { - var tokenIn = new Tensor(new[] { 1, c }); - for (int d = 0; d < c; d++) tokenIn[0, d] = _numOps.FromDouble(attnOut[t, d]); - var tokenOut = _outProj.Forward(tokenIn); - for (int d = 0; d < c; d++) output[wIdx, t, d] = tokenOut[0, d]; - } - } - return output; + // The learnable relative-position bias table is gathered by the fixed index map, so it now + // receives a gradient; the scalar loop read it out as doubles and it never trained. + var bias = CvTensorOps.RelativePositionBias(_relativePositionBiasTable, _relativePositionIndex); + var attended = CvTensorOps.MultiHeadAttention(q, k, v, _numHeads, _scale, scoreBias: bias); + return CvTensorOps.Tokenwise(attended, _outProj.Forward); } private Tensor ApplyMLP(Tensor x) - { - int batch = x.Shape[0]; - int seqLen = x.Shape[1]; - var result = new Tensor(x._shape); - - for (int b = 0; b < batch; b++) - { - for (int s = 0; s < seqLen; s++) - { - var tokenIn = new Tensor(new[] { 1, _dim }); - for (int d = 0; d < _dim; d++) tokenIn[0, d] = x[b, s, d]; - - var hidden = _mlpFc1.Forward(tokenIn); - for (int d = 0; d < hidden.Shape[1]; d++) - { - double val = _numOps.ToDouble(hidden[0, d]); - double gelu = 0.5 * val * (1 + Math.Tanh(Math.Sqrt(2 / Math.PI) * (val + 0.044715 * val * val * val))); - hidden[0, d] = _numOps.FromDouble(gelu); - } - - var tokenOut = _mlpFc2.Forward(hidden); - for (int d = 0; d < _dim; d++) result[b, s, d] = tokenOut[0, d]; - } - } - return result; - } + => CvTensorOps.Tokenwise(x, rows => _mlpFc2.Forward(AiDotNetEngine.Current.GELU(_mlpFc1.Forward(rows)))); private Tensor AddTensors(Tensor a, Tensor b) => AiDotNetEngine.Current.TensorAdd(a, b); @@ -916,6 +709,17 @@ public void ReadParameters(BinaryReader reader) BackboneSerialization.ReadLayerParameters(reader, _mlpFc1); BackboneSerialization.ReadLayerParameters(reader, _mlpFc2); } + + /// + /// Weights this block owns outside its layers: both norms' scale + /// and shift, and the relative-position bias table. Live tensors, in a fixed order. + /// + internal IEnumerable> ExtraParameterTensors() + { + foreach (var tensor in _norm1.ParameterTensors()) yield return tensor; + foreach (var tensor in _norm2.ParameterTensors()) yield return tensor; + yield return _relativePositionBiasTable; + } } /// @@ -943,39 +747,7 @@ public SwinLayerNorm(int dim, double eps = 1e-6) } } - public Tensor Forward(Tensor x) - { - int batch = x.Shape[0]; - int seqLen = x.Shape[1]; - int dim = x.Shape[2]; - var result = new Tensor(x._shape); - - for (int b = 0; b < batch; b++) - for (int s = 0; s < seqLen; s++) - { - double mean = 0; - for (int d = 0; d < dim; d++) mean += _numOps.ToDouble(x[b, s, d]); - mean /= dim; - - double variance = 0; - for (int d = 0; d < dim; d++) - { - double diff = _numOps.ToDouble(x[b, s, d]) - mean; - variance += diff * diff; - } - variance /= dim; - - double std = Math.Sqrt(variance + _eps); - for (int d = 0; d < dim; d++) - { - double normalized = (_numOps.ToDouble(x[b, s, d]) - mean) / std; - double gamma = _numOps.ToDouble(_gamma[d]); - double beta = _numOps.ToDouble(_beta[d]); - result[b, s, d] = _numOps.FromDouble(gamma * normalized + beta); - } - } - return result; - } + public Tensor Forward(Tensor x) => CvTensorOps.LayerNormLastAxis(x, _gamma, _beta, _eps); public long GetParameterCount() => 2 * _dim; @@ -994,6 +766,13 @@ public void ReadParameters(BinaryReader reader) for (int i = 0; i < _dim; i++) _gamma[i] = _numOps.FromDouble(reader.ReadDouble()); for (int i = 0; i < _dim; i++) _beta[i] = _numOps.FromDouble(reader.ReadDouble()); } + + /// The learnable scale and shift, as live tensors. + internal IEnumerable> ParameterTensors() + { + yield return _gamma; + yield return _beta; + } } /// @@ -1049,50 +828,10 @@ public Tensor Forward(Tensor input, int? inputHeight, int? inputWidth) $"Cannot infer spatial dimensions from sequence length {seqLen} for patch merging."); } - // Pad odd H/W up to the next even size (zeros), so the 2×2 merge always has full quads. - int hPad = h + (h & 1); - int wPad = w + (w & 1); - Tensor src = input; - if (hPad != h || wPad != w) - { - var padded = new Tensor(new[] { batch, hPad * wPad, dim }); - for (int n = 0; n < batch; n++) - for (int i = 0; i < h; i++) - for (int j = 0; j < w; j++) - { - int srcIdx = i * w + j; - int dstIdx = i * wPad + j; - for (int d = 0; d < dim; d++) - padded[n, dstIdx, d] = input[n, srcIdx, d]; - } - src = padded; - } - - int newH = hPad / 2; - int newW = wPad / 2; - int newSeqLen = newH * newW; - var merged = new Tensor(new[] { batch, newSeqLen, dim * 4 }); - - for (int n = 0; n < batch; n++) - { - for (int i = 0; i < newH; i++) - for (int j = 0; j < newW; j++) - { - int newIdx = i * newW + j; - int idx0 = (2 * i) * wPad + (2 * j); - int idx1 = (2 * i) * wPad + (2 * j + 1); - int idx2 = (2 * i + 1) * wPad + (2 * j); - int idx3 = (2 * i + 1) * wPad + (2 * j + 1); - for (int d = 0; d < dim; d++) - { - merged[n, newIdx, d] = src[n, idx0, d]; - merged[n, newIdx, dim + d] = src[n, idx1, d]; - merged[n, newIdx, 2 * dim + d] = src[n, idx2, d]; - merged[n, newIdx, 3 * dim + d] = src[n, idx3, d]; - } - } - } - + // Zero-pad odd H/W to even and gather each 2x2 quad into one token (engine ops, so the tape + // survives the merge and the stages before it keep receiving gradient). + var spatial = AiDotNetEngine.Current.Reshape(input, new[] { batch, h, w, dim }); + var merged = CvTensorOps.PatchMerge2x2(spatial); return _reduction.Forward(merged); } diff --git a/src/ComputerVision/Detection/DetectionTrainingBatch.cs b/src/ComputerVision/Detection/DetectionTrainingBatch.cs new file mode 100644 index 0000000000..a01accf67d --- /dev/null +++ b/src/ComputerVision/Detection/DetectionTrainingBatch.cs @@ -0,0 +1,120 @@ +namespace AiDotNet.ComputerVision.Detection; + +/// Owns an immutable, unpadded target list for every image in a training batch. +/// The detector's numeric type. +public sealed class DetectionTrainingBatch +{ + private readonly IReadOnlyList>[] _images; + + /// Copies the supplied lists. Empty images are valid; an empty batch is not. + public DetectionTrainingBatch(IEnumerable>> images) + { + if (images is null) throw new ArgumentNullException(nameof(images)); + var owned = new List>>(); + int total = 0; + foreach (var image in images) + { + if (image is null) throw new ArgumentException("Each image must have a target list, even when empty.", nameof(images)); + var targets = image.ToArray(); + if (targets.Any(target => target is null)) + throw new ArgumentException("Target lists cannot contain null entries.", nameof(images)); + total = checked(total + targets.Length); + owned.Add(Array.AsReadOnly(targets)); + } + if (owned.Count == 0) throw new ArgumentException("A training batch must contain at least one image.", nameof(images)); + _images = owned.ToArray(); + TargetCount = total; + } + + /// Gets the number of images, including images with no objects. + public int ImageCount => _images.Length; + /// Gets the total foreground target count across all images. + public int TargetCount { get; } + /// Gets the immutable targets for one image. + public IReadOnlyList> this[int imageIndex] => _images[imageIndex]; + + /// + /// Converts the COCO loader's [batch, objects, 5] normalized top-left xywh labels. + /// A completely zero row is padding; all other rows must describe a valid foreground box. + /// + public static DetectionTrainingBatch FromPaddedCoco(Tensor labels) + { + ValidatePaddedShape(labels, exactWidth: true); + var ops = MathHelper.GetNumericOperations(); + var images = new List>[labels.Shape[0]]; + for (int image = 0; image < images.Length; image++) + { + var targets = new List>(); + for (int item = 0; item < labels.Shape[1]; item++) + { + bool padding = true; + for (int coordinate = 0; coordinate < 5; coordinate++) + padding &= ops.Equals(labels[image, item, coordinate], ops.Zero); + if (padding) continue; + int label = ReadClass(labels[image, item, 0], nameof(labels)); + targets.Add(DetectionTrainingTarget.FromNormalizedXywh(label, + labels[image, item, 1], labels[image, item, 2], labels[image, item, 3], labels[image, item, 4])); + } + images[image] = targets; + } + return new DetectionTrainingBatch(images); + } + + internal static DetectionTrainingBatch FromPaddedDetr(Tensor labels) + { + ValidatePaddedShape(labels, exactWidth: false); + var ops = MathHelper.GetNumericOperations(); + var images = new List>[labels.Shape[0]]; + for (int image = 0; image < images.Length; image++) + { + var targets = new List>(); + bool sawPadding = false; + for (int item = 0; item < labels.Shape[1]; item++) + { + if (ops.Equals(labels[image, item, 0], ops.FromDouble(-1))) + { + sawPadding = true; + continue; + } + if (sawPadding) + throw new ArgumentException("DETR targets cannot follow a -1 padding row.", nameof(labels)); + int label = ReadClass(labels[image, item, 0], nameof(labels)); + targets.Add(new DetectionTrainingTarget(label, + labels[image, item, 1], labels[image, item, 2], labels[image, item, 3], labels[image, item, 4])); + } + images[image] = targets; + } + return new DetectionTrainingBatch(images); + } + + internal void ValidateForModel(int imageCount, int foregroundClasses, int queries) + { + if (ImageCount != imageCount) + throw new ArgumentException("The target batch must contain one list per input image.", "targets"); + foreach (var image in _images) + { + if (image.Count > queries) + throw new ArgumentException("An image has more targets than detection queries; targets cannot be silently discarded.", "targets"); + foreach (var target in image) + if (target.ClassId >= foregroundClasses) + throw new ArgumentException("Every target class must be a foreground class supported by the model.", "targets"); + } + } + + private static int ReadClass(T value, string parameterName) + { + double label = MathHelper.GetNumericOperations().ToDouble(value); + if (double.IsNaN(label) || double.IsInfinity(label) || label < 0 || label > int.MaxValue || label != Math.Truncate(label)) + throw new ArgumentException("Target classes must be finite nonnegative integers.", parameterName); + return (int)label; + } + + private static void ValidatePaddedShape(Tensor labels, bool exactWidth) + { + if (labels is null) throw new ArgumentNullException(nameof(labels)); + if (labels.Rank != 3 || labels.Shape[0] <= 0 || labels.Shape[2] < 5) + throw new ArgumentException("Padded labels must have shape [positive batch, objects, at least 5].", nameof(labels)); + if (exactWidth && labels.Shape[2] != 5) + throw new ArgumentException("COCO labels must have exactly five values per row.", nameof(labels)); + } +} diff --git a/src/ComputerVision/Detection/DetectionTrainingTarget.cs b/src/ComputerVision/Detection/DetectionTrainingTarget.cs new file mode 100644 index 0000000000..ac88e50b15 --- /dev/null +++ b/src/ComputerVision/Detection/DetectionTrainingTarget.cs @@ -0,0 +1,70 @@ +namespace AiDotNet.ComputerVision.Detection; + +/// A foreground label and a normalized center-format box used to train a detector. +/// The detector's numeric type. +/// +/// Coordinates are center-x, center-y, width and height, each relative to its own image dimension. +/// Width and height must be positive. This type does not clip boxes or infer a coordinate format. +/// +public sealed class DetectionTrainingTarget +{ + /// Creates an immutable target with normalized center-format coordinates. + public DetectionTrainingTarget(int classId, T centerX, T centerY, T width, T height) + { + if (classId < 0) + throw new ArgumentOutOfRangeException(nameof(classId), "A target must have a nonnegative foreground class."); + ValidateCoordinate(centerX, nameof(centerX), positive: false); + ValidateCoordinate(centerY, nameof(centerY), positive: false); + ValidateCoordinate(width, nameof(width), positive: true); + ValidateCoordinate(height, nameof(height), positive: true); + ClassId = classId; + CenterX = centerX; + CenterY = centerY; + Width = width; + Height = height; + } + + /// Gets the zero-based foreground class; the no-object class is never a target. + public int ClassId { get; } + /// Gets the horizontal center divided by image width. + public T CenterX { get; } + /// Gets the vertical center divided by image height. + public T CenterY { get; } + /// Gets box width divided by image width. + public T Width { get; } + /// Gets box height divided by image height. + public T Height { get; } + + /// Converts a pixel-space top-left xywh box without assuming a square image. + public static DetectionTrainingTarget FromPixelXywh( + int classId, T x, T y, T width, T height, int imageWidth, int imageHeight) + { + if (imageWidth <= 0) throw new ArgumentOutOfRangeException(nameof(imageWidth)); + if (imageHeight <= 0) throw new ArgumentOutOfRangeException(nameof(imageHeight)); + var ops = MathHelper.GetNumericOperations(); + return FromNormalizedXywh(classId, + ops.Divide(x, ops.FromDouble(imageWidth)), + ops.Divide(y, ops.FromDouble(imageHeight)), + ops.Divide(width, ops.FromDouble(imageWidth)), + ops.Divide(height, ops.FromDouble(imageHeight))); + } + + internal static DetectionTrainingTarget FromNormalizedXywh(int classId, T x, T y, T width, T height) + { + ValidateCoordinate(x, nameof(x), positive: false); + ValidateCoordinate(y, nameof(y), positive: false); + var ops = MathHelper.GetNumericOperations(); + T half = ops.FromDouble(0.5); + return new DetectionTrainingTarget(classId, + ops.Add(x, ops.Multiply(width, half)), ops.Add(y, ops.Multiply(height, half)), width, height); + } + + private static void ValidateCoordinate(T value, string parameterName, bool positive) + { + double coordinate = MathHelper.GetNumericOperations().ToDouble(value); + if (double.IsNaN(coordinate) || double.IsInfinity(coordinate)) + throw new ArgumentOutOfRangeException(parameterName, "Box coordinates must be finite."); + if (coordinate < 0 || coordinate > 1 || (positive && coordinate == 0)) + throw new ArgumentOutOfRangeException(parameterName, "Center coordinates must be in [0, 1] and extents in (0, 1]."); + } +} diff --git a/src/ComputerVision/Detection/Losses/DETRSetLoss.cs b/src/ComputerVision/Detection/Losses/DETRSetLoss.cs index 4ea87d9827..0b8c7fc10d 100644 --- a/src/ComputerVision/Detection/Losses/DETRSetLoss.cs +++ b/src/ComputerVision/Detection/Losses/DETRSetLoss.cs @@ -1,580 +1,503 @@ using AiDotNet.Augmentation.Image; using AiDotNet.ComputerVision.Detection.PostProcessing; +using AiDotNet.Enums; using AiDotNet.LossFunctions; -using AiDotNet.Tensors; +using AiDotNet.Solvers.Assignment; +using AiDotNet.Tensors.Engines.Autodiff; namespace AiDotNet.ComputerVision.Detection.Losses; -/// -/// DETR Set Prediction Loss with Hungarian Matching for end-to-end object detection. -/// +/// DETR-family set prediction loss with exact Hungarian assignment. /// The numeric type used for calculations. /// -/// For Beginners: Unlike traditional detectors that use anchors and NMS, -/// DETR treats detection as a set prediction problem. It uses Hungarian matching to -/// find the optimal assignment between predicted and ground truth boxes, then computes -/// loss on the matched pairs. -/// -/// The loss has three components: -/// - Classification loss: Cross-entropy for class predictions -/// - Box loss: L1 loss for box coordinates -/// - GIoU loss: For better box regression +/// +/// Foreground queries are assigned with a weighted sum of a classification cost, center-format L1 +/// distance and negative GIoU. The matched boxes receive L1 and GIoU losses normalized by the +/// total foreground target count across the local batch. Every query receives classification +/// supervision, including unmatched queries and empty images. +/// +/// +/// Three classification forms are supported (): +/// DETR's softmax cross-entropy with a down-weighted no-object class (Carion et al. 2020); the +/// sigmoid focal loss of DINO (Zhang et al. 2022), normalized by the target count; and RT-DETR's +/// IoU-aware varifocal loss (Zhao et al. 2023; Zhang et al. 2021), whose matched class target is the +/// IoU of the matched predicted box. Sigmoid heads are matched with the focal classification cost +/// of Deformable DETR's reference matcher. +/// +/// +/// Only the supplied final prediction heads are supervised; intermediate decoder, query-selection +/// and denoising losses are not fabricated. More targets than queries in an image are rejected +/// rather than silently dropped. /// -/// -/// Reference: Carion et al., "End-to-End Object Detection with Transformers", ECCV 2020 /// public class DETRSetLoss : LossFunctionBase { - private readonly NMS _nms; - private readonly double _classWeight; - private readonly double _boxL1Weight; - private readonly double _boxGIoUWeight; + private readonly NMS _nms = new(); + private readonly DetrSetLossOptions _options; private readonly int _numClasses; - /// - /// Creates a new DETR set loss instance. - /// - /// Number of object classes (including no-object class). - /// Weight for classification loss. - /// Weight for L1 box loss. - /// Weight for GIoU box loss. - public DETRSetLoss( - int numClasses = 91, - double classWeight = 1.0, - double boxL1Weight = 5.0, - double boxGIoUWeight = 2.0) : base() + /// Creates a DETR objective with the standard final-head loss weights. + /// Number of output classes, including the final no-object class. + /// Nonnegative classification and matching cost weight. + /// Nonnegative center-format L1 loss and matching cost weight. + /// Nonnegative GIoU loss and matching cost weight. + public DETRSetLoss(int numClasses = 91, double classWeight = 1.0, + double boxL1Weight = 5.0, double boxGIoUWeight = 2.0) + : this(numClasses, SoftmaxOptions(classWeight, boxL1Weight, boxGIoUWeight)) { - _nms = new NMS(); + } + + /// Creates a DETR-family objective with explicit classification form and weights. + /// + /// Width of the class head: foreground classes plus the final no-object class for softmax + /// cross-entropy, or foreground classes only for the sigmoid focal and varifocal forms. + /// + /// Classification form, matching costs and loss weights; copied on construction. + public DETRSetLoss(int numClasses, DetrSetLossOptions options) + { + if (options is null) throw new ArgumentNullException(nameof(options)); + _options = options.Snapshot(); + int minimumClasses = UsesNoObjectClass ? 2 : 1; + if (numClasses < minimumClasses) throw new ArgumentOutOfRangeException(nameof(numClasses)); _numClasses = numClasses; - _classWeight = classWeight; - _boxL1Weight = boxL1Weight; - _boxGIoUWeight = boxGIoUWeight; } - /// - /// Calculates the DETR loss using flattened vectors (simplified for interface compatibility). - /// - /// - /// For DETR, the tensor-based CalculateLoss overload is preferred as it preserves - /// the structured format needed for Hungarian matching. This vector-based method - /// provides a basic L1 loss between predicted and actual values. - /// + /// The classification form this objective trains. + public SetPredictionClassificationLoss ClassificationLoss => _options.ClassificationLoss; + + private bool UsesNoObjectClass => _options.ClassificationLoss == SetPredictionClassificationLoss.SoftmaxCrossEntropy; + + private int ForegroundClasses => UsesNoObjectClass ? _numClasses - 1 : _numClasses; + + /// Calculates the documented element-wise MAE compatibility objective. + /// This vector overload does not describe detection targets or perform matching. public override T CalculateLoss(Vector predicted, Vector actual) { ValidateVectorLengths(predicted, actual); - - // Simple L1 loss for vector interface compatibility - double totalLoss = 0; + double total = 0; for (int i = 0; i < predicted.Length; i++) - { - double diff = NumOps.ToDouble(predicted[i]) - NumOps.ToDouble(actual[i]); - totalLoss += Math.Abs(diff); - } - - return NumOps.FromDouble(totalLoss / predicted.Length); + total += Math.Abs(NumOps.ToDouble(predicted[i]) - NumOps.ToDouble(actual[i])); + return NumOps.FromDouble(total / predicted.Length); } - /// - /// Calculates the DETR set loss. - /// - /// Predicted tensor containing class logits and boxes. - /// Target tensor containing ground truth. - /// Combined loss value. + /// Evaluates structured predictions against -1-padded DETR targets. /// - /// Expected shapes: - /// - predicted: [batch, num_queries, num_classes + 4] (logits + boxes) - /// - targets: [batch, max_objects, 1 + 4] (class + boxes, padded) + /// Predictions are [batch, queries, classes + 4], with logits followed by normalized cxcywh. + /// Targets are [batch, objects, at least 5], containing class and normalized cxcywh. + /// A -1 class starts padding. COCO loader xywh targets must be converted explicitly. /// public T CalculateLoss(Tensor predicted, Tensor targets) { - int batch = predicted.Shape[0]; - int numQueries = predicted.Shape[1]; - - double totalLoss = 0; - int validBatches = 0; - - for (int b = 0; b < batch; b++) - { - // Extract predictions and targets for this batch - var predBoxes = ExtractPredictedBoxes(predicted, b, numQueries); - var predLogits = ExtractPredictedLogits(predicted, b, numQueries, _numClasses); - var gtBoxes = ExtractGroundTruthBoxes(targets, b); - var gtClasses = ExtractGroundTruthClasses(targets, b); - - if (gtBoxes.Count == 0) continue; - - // Perform Hungarian matching - var (predIndices, gtIndices) = HungarianMatch(predBoxes, predLogits, gtBoxes, gtClasses); - - // Calculate losses for matched pairs - double classLoss = CalculateClassificationLoss(predLogits, gtClasses, predIndices, gtIndices, numQueries); - double boxL1Loss = CalculateBoxL1Loss(predBoxes, gtBoxes, predIndices, gtIndices); - double boxGIoULoss = CalculateBoxGIoULoss(predBoxes, gtBoxes, predIndices, gtIndices); - - totalLoss += _classWeight * classLoss + _boxL1Weight * boxL1Loss + _boxGIoUWeight * boxGIoULoss; - validBatches++; - } - - double meanLoss = validBatches > 0 ? totalLoss / validBatches : 0; - return NumOps.FromDouble(meanLoss); + using var noGrad = new NoGradScope(); + ValidateStructuredLayout(predicted, targets); + using var objective = ComputeStructuredLoss(predicted, targets); + return objective[0]; } - /// - /// Performs Hungarian matching between predictions and ground truth. - /// - /// Predicted bounding boxes. - /// Predicted class logits. - /// Ground truth bounding boxes. - /// Ground truth class labels. - /// Matched indices (prediction indices, ground truth indices). - private (int[] PredIndices, int[] GtIndices) HungarianMatch( - List> predBoxes, - double[,] predLogits, - List> gtBoxes, - List gtClasses) + /// Evaluates the actual class and normalized cxcywh heads against typed targets. + public T CalculateLoss(Tensor classLogits, Tensor boxes, DetectionTrainingBatch targets) { - int numPred = predBoxes.Count; - int numGt = gtBoxes.Count; - - // Build cost matrix [numPred x numGt] - var costMatrix = new double[numPred, numGt]; - - for (int i = 0; i < numPred; i++) - { - for (int j = 0; j < numGt; j++) - { - // Classification cost: negative log probability for the correct class - double classCost = ComputeClassCost(predLogits, i, gtClasses[j]); - - // Box L1 cost - double l1Cost = ComputeL1Cost(predBoxes[i], gtBoxes[j]); - - // Box GIoU cost (negative because we minimize) - double giouCost = 1.0 - _nms.ComputeGIoU(predBoxes[i], gtBoxes[j]); - - // Combined cost (weighted) - costMatrix[i, j] = classCost + 5.0 * l1Cost + 2.0 * giouCost; - } - } - - // Solve the assignment optimally with the Hungarian algorithm. - // - // DETR (Carion et al., 2020) defines its loss through the OPTIMAL bipartite matching between - // predictions and ground-truth boxes — the permutation minimizing the total matching cost. - // This previously used a greedy approximation (sort all pairs by cost, take them in order), - // which is not the same matching: committing early to a locally cheap pair can force an - // expensive one later, and the gap is unbounded. Because the matching decides which - // prediction is supervised by which target, a different matching produces a different loss - // and therefore a different trained model, so the approximation was a deviation from the - // paper rather than an implementation detail. - // - // Ground truth indexes the ROWS so that every ground-truth box is matched (there are always - // at least as many object queries as boxes in DETR); surplus predictions stay unmatched and - // are supervised as "no object" by the caller. - var cost = new Matrix(numGt, numPred); - for (int j = 0; j < numGt; j++) - { - for (int i = 0; i < numPred; i++) cost[j, i] = costMatrix[i, j]; - } - - var assignment = new AiDotNet.Solvers.Assignment.LinearAssignmentSolver().Solve(cost); - - var predIndices = new List(numGt); - var gtIndices = new List(numGt); - for (int j = 0; j < numGt; j++) - { - int predictionIndex = assignment[j]; - if (predictionIndex < 0) continue; - - predIndices.Add(predictionIndex); - gtIndices.Add(j); - } - - return (predIndices.ToArray(), gtIndices.ToArray()); + using var noGrad = new NoGradScope(); + using var objective = ComputeTapeLoss(classLogits, boxes, targets); + return objective[0]; } - /// - /// Mean absolute error built from engine ops, used when the inputs are not in DETR's structured - /// prediction layout so that the tape loss agrees with the vector - /// overload and remains differentiable. - /// - private Tensor ComputeMeanAbsoluteErrorTapeLoss(Tensor predicted, Tensor target) + /// + /// + /// Structured tensor inputs use the same objective as the typed-head overload. Identical + /// non-structured shapes retain the documented element-wise MAE compatibility objective. + /// + public override Tensor ComputeTapeLoss(Tensor predicted, Tensor target) { - var aligned = EnsureTargetMatchesPredicted(predicted, target); - var difference = Engine.TensorSubtract(predicted, aligned); - var magnitude = Engine.TensorAbs(difference); - - var allAxes = Enumerable.Range(0, magnitude.Shape.Length).ToArray(); - var total = Engine.ReduceSum(magnitude, allAxes, keepDims: false); + if (predicted is null) throw new ArgumentNullException(nameof(predicted)); + if (target is null) throw new ArgumentNullException(nameof(target)); + if (HasStructuredLayout(predicted, target)) + return ComputeStructuredLoss(predicted, target); + + bool sameShape = predicted.Rank == target.Rank; + for (int axis = 0; axis < predicted.Rank && sameShape; axis++) + sameShape = predicted.Shape[axis] == target.Shape[axis]; + if (sameShape && !IsStructuredPrediction(predicted)) + { + var difference = Engine.TensorAbs(Engine.TensorSubtract(predicted, target)); + return Engine.TensorMultiplyScalar(Engine.ReduceSum(difference, null), + NumOps.FromDouble(1.0 / Math.Max(1, predicted.Length))); + } - return Engine.TensorMultiplyScalar( - total, NumOps.FromDouble(1.0 / Math.Max(1, predicted.Length))); + ValidateStructuredLayout(predicted, target); + throw new InvalidOperationException("Structured layout validation did not reject an incompatible shape."); } - /// - /// Computes classification cost for Hungarian matching. - /// - private double ComputeClassCost(double[,] predLogits, int predIdx, int gtClass) + /// Builds differentiable final-head classification, L1 and GIoU losses after discrete assignment. + /// Raw class logits [batch, queries, class-head width]. + /// Sigmoid box predictions [batch, queries, 4] in normalized cxcywh. + /// Immutable, unpadded foreground targets; empty images are valid. + /// A scalar connected to both prediction heads on the active gradient tape. + /// + /// Assignment, and the varifocal IoU targets and weights, use detached host values and the exact + /// shared Hungarian solver. Loss and gradient calculations use engine operations and retain the + /// active CPU/GPU backend. Input tensors are borrowed, never mutated or disposed. The returned + /// scalar belongs to the active tensor/tape lifetime and must be consumed before that lifetime ends. + /// + public Tensor ComputeTapeLoss(Tensor classLogits, Tensor boxes, DetectionTrainingBatch targets) { - // Softmax over classes - double maxLogit = double.NegativeInfinity; - int numClasses = predLogits.GetLength(1); - for (int c = 0; c < numClasses; c++) + ValidateHeads(classLogits, boxes, targets); + int batch = classLogits.Shape[0]; + int queries = classLogits.Shape[1]; + + // Materialize detached host snapshots once for discrete matching, not once per pair. + // These reads do not replace either live head in the differentiable objective below. + var logitsData = classLogits.ToArray(); + var boxData = boxes.ToArray(); + ValidateFinitePredictions(logitsData, boxData); + var assignments = Match(logitsData, boxData, targets, queries); + + var classification = UsesNoObjectClass + ? SoftmaxClassification(classLogits, assignments, targets) + : SigmoidClassification(classLogits, logitsData, boxData, assignments, targets); + + if (targets.TargetCount == 0) { - maxLogit = Math.Max(maxLogit, predLogits[predIdx, c]); + // Background classification is still nonzero. Connect the box head with an exact zero derivative. + var zeroBoxes = Engine.TensorMultiplyScalar(Engine.ReduceSum(boxes, null), NumOps.Zero); + return Engine.TensorAdd(classification, zeroBoxes); } - double sumExp = 0; - for (int c = 0; c < numClasses; c++) + var matchedRows = new int[targets.TargetCount]; + var targetBoxes = new T[checked(targets.TargetCount * 4)]; + int matched = 0; + for (int image = 0; image < batch; image++) { - sumExp += Math.Exp(predLogits[predIdx, c] - maxLogit); + for (int targetIndex = 0; targetIndex < targets[image].Count; targetIndex++) + { + matchedRows[matched] = image * queries + assignments[image][targetIndex]; + WriteBox(targetBoxes, matched * 4, targets[image][targetIndex]); + matched++; + } } - double logProb = predLogits[predIdx, gtClass] - maxLogit - Math.Log(sumExp); - return -logProb; // Negative log probability + var flatBoxes = Engine.Reshape(boxes, new[] { checked(batch * queries), 4 }); + var matchedBoxes = CvTensorOps.Select(flatBoxes, matchedRows, 0); + var actualBoxes = new Tensor(targetBoxes, new[] { matched, 4 }); + var l1Sum = Engine.ReduceSum(Engine.TensorAbs(Engine.TensorSubtract(matchedBoxes, actualBoxes)), null); + var giouSum = Engine.ReduceSum( + Engine.TensorGIoULoss(ToCorners(matchedBoxes), ToCorners(actualBoxes)), null); + var weightedL1 = Engine.TensorMultiplyScalar(l1Sum, NumOps.FromDouble(_options.L1LossWeight / targets.TargetCount)); + var weightedGIoU = Engine.TensorMultiplyScalar(giouSum, NumOps.FromDouble(_options.GIoULossWeight / targets.TargetCount)); + return Engine.TensorAdd(classification, Engine.TensorAdd(weightedL1, weightedGIoU)); } - /// - /// Computes L1 cost between two boxes. - /// - private double ComputeL1Cost(BoundingBox pred, BoundingBox gt) + private Tensor SoftmaxClassification(Tensor classLogits, int[][] assignments, DetectionTrainingBatch targets) { - var (px1, py1, px2, py2) = pred.ToXYXY(); - var (gx1, gy1, gx2, gy2) = gt.ToXYXY(); + int batch = classLogits.Shape[0]; + int queries = classLogits.Shape[1]; + var selectedEntries = new int[checked(batch * queries)]; + var weights = new T[selectedEntries.Length]; + double denominator = 0; + for (int image = 0; image < batch; image++) + { + var assignedClasses = new int[queries]; + for (int query = 0; query < queries; query++) assignedClasses[query] = _numClasses - 1; + for (int targetIndex = 0; targetIndex < targets[image].Count; targetIndex++) + assignedClasses[assignments[image][targetIndex]] = targets[image][targetIndex].ClassId; + for (int query = 0; query < queries; query++) + { + int label = assignedClasses[query]; + double weight = label == _numClasses - 1 ? _options.NoObjectWeight : 1; + int row = image * queries + query; + selectedEntries[row] = row * _numClasses + label; + weights[row] = NumOps.FromDouble(weight); + denominator += weight; + } + } - return Math.Abs(px1 - gx1) + Math.Abs(py1 - gy1) + - Math.Abs(px2 - gx2) + Math.Abs(py2 - gy2); + // Gather only each query's assigned log-probability. A dense one-hot product would multiply + // the -infinity log-probability of an extreme finite unselected logit by zero and yield NaN. + var logProbabilities = Engine.TensorLogSoftmax(classLogits, axis: 2); + var flatLogProbabilities = Engine.Reshape(logProbabilities, new[] { checked(batch * queries * _numClasses) }); + var selected = CvTensorOps.Select(flatLogProbabilities, selectedEntries, 0); + var negativeLogLikelihood = Engine.TensorNegate(Engine.ReduceSum( + Engine.TensorMultiply(selected, new Tensor(weights, new[] { weights.Length })), null)); + // An all-background batch with no-object weight zero has nothing to classify. + double scale = denominator > 0 ? _options.ClassLossWeight / denominator : 0; + return Engine.TensorMultiplyScalar(negativeLogLikelihood, NumOps.FromDouble(scale)); } - /// - /// Calculates classification loss for matched pairs. - /// - private double CalculateClassificationLoss( - double[,] predLogits, - List gtClasses, - int[] predIndices, - int[] gtIndices, - int numQueries) + private Tensor SigmoidClassification(Tensor classLogits, T[] logitsData, T[] boxData, + int[][] assignments, DetectionTrainingBatch targets) { - int numClasses = predLogits.GetLength(1); - double loss = 0; - - // Loss for matched pairs - for (int i = 0; i < predIndices.Length; i++) + int batch = classLogits.Shape[0]; + int queries = classLogits.Shape[1]; + int length = checked(batch * queries * _numClasses); + var positive = new bool[length]; + var soft = new double[length]; + for (int image = 0; image < batch; image++) { - int predIdx = predIndices[i]; - int gtIdx = gtIndices[i]; - int gtClass = gtClasses[gtIdx]; - - loss += ComputeClassCost(predLogits, predIdx, gtClass); + for (int targetIndex = 0; targetIndex < targets[image].Count; targetIndex++) + { + var target = targets[image][targetIndex]; + int query = assignments[image][targetIndex]; + int index = (image * queries + query) * _numClasses + target.ClassId; + positive[index] = true; + soft[index] = _options.ClassificationLoss == SetPredictionClassificationLoss.VariFocal + ? _nms.ComputeIoU(PredictedBox(boxData, image, queries, query), TargetBox(target)) + : 1.0; + } } - // Loss for unmatched predictions (should predict no-object class) - int noObjectClass = numClasses - 1; - for (int p = 0; p < numQueries; p++) + // Deformable DETR, DINO and RT-DETR sum the per-element loss over queries and classes and + // divide by the number of target boxes (at least one). + double scale = _options.ClassLossWeight / Math.Max(1, targets.TargetCount); + var shape = classLogits.Shape.ToArray(); + var targetTensor = new Tensor(soft.Select(value => NumOps.FromDouble(value)).ToArray(), shape); + var complement = new Tensor(soft.Select(value => NumOps.FromDouble(1 - value)).ToArray(), shape); + + // log(p) = -softplus(-x) and log(1 - p) = -softplus(x) avoid evaluating log(sigmoid(x)). + var logProbability = Engine.TensorNegate(Engine.Softplus(Engine.TensorNegate(classLogits))); + var logComplement = Engine.TensorNegate(Engine.Softplus(classLogits)); + var crossEntropy = Engine.TensorNegate(Engine.TensorAdd( + Engine.TensorMultiply(targetTensor, logProbability), + Engine.TensorMultiply(complement, logComplement))); + + Tensor perElement; + if (_options.ClassificationLoss == SetPredictionClassificationLoss.SigmoidFocal) { - if (!predIndices.Contains(p)) + // FL = -alpha_t (1 - p_t)^gamma log(p_t) with binary targets (Lin et al. 2017); the + // modulating factor stays on the tape, as in the reference sigmoid_focal_loss. + double alpha = _options.FocalAlpha; + var alphaT = new Tensor(positive.Select(isPositive => NumOps.FromDouble(isPositive ? alpha : 1 - alpha)).ToArray(), shape); + perElement = Engine.TensorMultiply(alphaT, crossEntropy); + if (_options.FocalGamma > 0) { - // Lower weight for no-object class - loss += 0.1 * ComputeClassCost(predLogits, p, noObjectClass); + // 1 - p_t = p + t - 2 p t for a binary target t. + var signs = new Tensor(positive.Select(isPositive => NumOps.FromDouble(isPositive ? -1 : 1)).ToArray(), shape); + var oneMinusPt = Engine.TensorAdd(Engine.TensorMultiply(Engine.Sigmoid(classLogits), signs), targetTensor); + perElement = Engine.TensorMultiply(perElement, + Engine.TensorPower(oneMinusPt, NumOps.FromDouble(_options.FocalGamma))); } } - - return loss / numQueries; - } - - /// - /// Calculates L1 box loss for matched pairs. - /// - private double CalculateBoxL1Loss( - List> predBoxes, - List> gtBoxes, - int[] predIndices, - int[] gtIndices) - { - if (predIndices.Length == 0) return 0; - - double loss = 0; - for (int i = 0; i < predIndices.Length; i++) + else { - loss += ComputeL1Cost(predBoxes[predIndices[i]], gtBoxes[gtIndices[i]]); + // VFL(p, q) = -q (q log p + (1 - q) log(1 - p)) for the matched class and + // -alpha p^gamma log(1 - p) otherwise (Zhang et al. 2021). The weights use the detached + // score, as the RT-DETR and VarifocalNet reference implementations do. + var weights = new T[length]; + for (int index = 0; index < length; index++) + { + double probability = Logistic(NumOps.ToDouble(logitsData[index])); + weights[index] = NumOps.FromDouble(positive[index] + ? soft[index] + : _options.FocalAlpha * Math.Pow(probability, _options.FocalGamma)); + } + perElement = Engine.TensorMultiply(new Tensor(weights, shape), crossEntropy); } - return loss / predIndices.Length; + return Engine.TensorMultiplyScalar(Engine.ReduceSum(perElement, null), NumOps.FromDouble(scale)); } - /// - /// Calculates GIoU box loss for matched pairs. - /// - private double CalculateBoxGIoULoss( - List> predBoxes, - List> gtBoxes, - int[] predIndices, - int[] gtIndices) + private Tensor ComputeStructuredLoss(Tensor predicted, Tensor targets) { - if (predIndices.Length == 0) return 0; - - double loss = 0; - for (int i = 0; i < predIndices.Length; i++) - { - double giou = _nms.ComputeGIoU(predBoxes[predIndices[i]], gtBoxes[gtIndices[i]]); - loss += 1.0 - giou; - } - - return loss / predIndices.Length; + int batch = predicted.Shape[0]; + int queries = predicted.Shape[1]; + var typedTargets = DetectionTrainingBatch.FromPaddedDetr(targets); + // Validate before allocating loss intermediates, including before slicing the predictions. + typedTargets.ValidateForModel(batch, ForegroundClasses, queries); + var logits = Engine.TensorSlice(predicted, new[] { 0, 0, 0 }, new[] { batch, queries, _numClasses }); + var boxes = Engine.TensorSlice(predicted, new[] { 0, 0, _numClasses }, new[] { batch, queries, 4 }); + return ComputeTapeLoss(logits, boxes, typedTargets); } - /// - /// Extracts predicted boxes from the combined tensor. - /// - private List> ExtractPredictedBoxes(Tensor predicted, int batch, int numQueries) + private int[][] Match(T[] logits, T[] boxes, DetectionTrainingBatch targets, int queries) { - var boxes = new List>(); - int boxOffset = _numClasses; // Boxes come after class logits - - for (int i = 0; i < numQueries; i++) + var result = new int[targets.ImageCount][]; + var solver = new LinearAssignmentSolver(); + for (int image = 0; image < targets.ImageCount; image++) { - boxes.Add(new BoundingBox( - predicted[batch, i, boxOffset], - predicted[batch, i, boxOffset + 1], - predicted[batch, i, boxOffset + 2], - predicted[batch, i, boxOffset + 3], - BoundingBoxFormat.CXCYWH)); // DETR uses center format - } + var imageTargets = targets[image]; + if (imageTargets.Count == 0) + { + result[image] = Array.Empty(); + continue; + } - return boxes; + var classCosts = UsesNoObjectClass + ? SoftmaxClassCosts(logits, image, queries) + : FocalClassCosts(logits, image, queries); + var predictedBoxes = new BoundingBox[queries]; + for (int query = 0; query < queries; query++) + predictedBoxes[query] = PredictedBox(boxes, image, queries, query); + var costs = new Matrix(imageTargets.Count, queries); + for (int targetIndex = 0; targetIndex < imageTargets.Count; targetIndex++) + { + var target = imageTargets[targetIndex]; + var actual = TargetBox(target); + for (int query = 0; query < queries; query++) + { + int offset = (image * queries + query) * 4; + double l1 = Math.Abs(NumOps.ToDouble(boxes[offset]) - NumOps.ToDouble(target.CenterX)) + + Math.Abs(NumOps.ToDouble(boxes[offset + 1]) - NumOps.ToDouble(target.CenterY)) + + Math.Abs(NumOps.ToDouble(boxes[offset + 2]) - NumOps.ToDouble(target.Width)) + + Math.Abs(NumOps.ToDouble(boxes[offset + 3]) - NumOps.ToDouble(target.Height)); + costs[targetIndex, query] = _options.ClassCostWeight * classCosts[query * _numClasses + target.ClassId] + + _options.L1CostWeight * l1 - _options.GIoUCostWeight * _nms.ComputeGIoU(predictedBoxes[query], actual); + } + } + var assignment = solver.Solve(costs); + var rows = new int[imageTargets.Count]; + for (int targetIndex = 0; targetIndex < rows.Length; targetIndex++) + { + int query = assignment[targetIndex]; + if (query < 0 || query >= queries) + throw new InvalidOperationException("The assignment solver did not match every validated target."); + rows[targetIndex] = query; + } + result[image] = rows; + } + return result; } - /// - /// Extracts predicted logits from the combined tensor. - /// - private double[,] ExtractPredictedLogits(Tensor predicted, int batch, int numQueries, int numClasses) + /// DETR's class cost: the negative softmax probability of the target class. + private double[] SoftmaxClassCosts(T[] logits, int image, int queries) { - var logits = new double[numQueries, numClasses]; - - for (int i = 0; i < numQueries; i++) + var costs = new double[checked(queries * _numClasses)]; + for (int query = 0; query < queries; query++) { - for (int c = 0; c < numClasses; c++) + int offset = (image * queries + query) * _numClasses; + double maximum = double.NegativeInfinity; + for (int label = 0; label < _numClasses; label++) + maximum = Math.Max(maximum, NumOps.ToDouble(logits[offset + label])); + double sum = 0; + for (int label = 0; label < _numClasses; label++) { - logits[i, c] = NumOps.ToDouble(predicted[batch, i, c]); + double value = Math.Exp(NumOps.ToDouble(logits[offset + label]) - maximum); + costs[query * _numClasses + label] = value; + sum += value; } + for (int label = 0; label < _numClasses; label++) + costs[query * _numClasses + label] = -costs[query * _numClasses + label] / sum; } - - return logits; + return costs; } /// - /// Extracts ground truth boxes from the targets tensor. + /// Deformable DETR's focal class cost: alpha (1 - p)^gamma (-log p) - (1 - alpha) p^gamma (-log(1 - p)). /// - private List> ExtractGroundTruthBoxes(Tensor targets, int batch) + private double[] FocalClassCosts(T[] logits, int image, int queries) { - var boxes = new List>(); - int maxObjects = targets.Shape[1]; - - for (int i = 0; i < maxObjects; i++) + double alpha = _options.MatchingFocalAlpha; + double gamma = _options.MatchingFocalGamma; + var costs = new double[checked(queries * _numClasses)]; + for (int index = 0; index < costs.Length; index++) { - // Class is first, check if valid (not padding) - int classId = (int)NumOps.ToDouble(targets[batch, i, 0]); - if (classId < 0) break; // Padding marker - - boxes.Add(new BoundingBox( - targets[batch, i, 1], - targets[batch, i, 2], - targets[batch, i, 3], - targets[batch, i, 4], - BoundingBoxFormat.CXCYWH)); + double logit = NumOps.ToDouble(logits[image * queries * _numClasses + index]); + double probability = Logistic(logit); + double positive = alpha * Math.Pow(1 - probability, gamma) * Softplus(-logit); + double negative = (1 - alpha) * Math.Pow(probability, gamma) * Softplus(logit); + costs[index] = positive - negative; } - - return boxes; + return costs; } - /// - /// Extracts ground truth class labels from the targets tensor. - /// - private List ExtractGroundTruthClasses(Tensor targets, int batch) - { - var classes = new List(); - int maxObjects = targets.Shape[1]; + private static double Logistic(double x) => x >= 0 ? 1 / (1 + Math.Exp(-x)) : Math.Exp(x) / (1 + Math.Exp(x)); - for (int i = 0; i < maxObjects; i++) - { - int classId = (int)NumOps.ToDouble(targets[batch, i, 0]); - if (classId < 0) break; // Padding marker - - classes.Add(classId); - } + private static double Softplus(double x) => x > 0 ? x + Math.Log(1 + Math.Exp(-x)) : Math.Log(1 + Math.Exp(x)); - return classes; - } - - /// - /// - /// DETR set loss with Hungarian matching: - /// 1. Hungarian matching is discrete (run under non-tape path on detached data) - /// 2. Once matching is determined, compute differentiable losses on matched pairs - /// using engine ops so gradients flow through predicted boxes/logits - /// - /// Expected shapes: - /// - predicted: [batch, num_queries, num_classes + 4] - /// - target: [batch, max_objects, 1 + 4] (class_id + x1,y1,x2,y2, padded with -1 class) - /// - public override Tensor ComputeTapeLoss(Tensor predicted, Tensor target) + private static BoundingBox PredictedBox(T[] boxes, int image, int queries, int query) { - // Inputs that are not in DETR's structured layout cannot be Hungarian-matched — there are - // no boxes or class logits to match. Previously this path fell through to the "no matched - // pairs" branch below and returned a FRESHLY CONSTRUCTED zero tensor, which is not attached - // to the gradient tape: the loss reported a non-zero value through CalculateLoss while its - // gradient was identically zero, so a model training against it would never move. Fall back - // to the same mean-absolute-error that the vector CalculateLoss overload documents itself as - // computing, built from engine ops so the tape can differentiate it. - bool hasDetrLayout = - predicted.Shape.Length == 3 && - predicted.Shape[2] == _numClasses + 4 && - target.Shape.Length == 3 && - target.Shape[0] == predicted.Shape[0] && - target.Shape[2] >= 5; - - if (!hasDetrLayout) - { - bool sameElementwiseShape = predicted.Shape.Length == target.Shape.Length; - for (int axis = 0; axis < predicted.Shape.Length && sameElementwiseShape; axis++) - { - sameElementwiseShape = predicted.Shape[axis] == target.Shape[axis]; - } + int offset = (image * queries + query) * 4; + return new BoundingBox(boxes[offset], boxes[offset + 1], boxes[offset + 2], boxes[offset + 3], BoundingBoxFormat.CXCYWH); + } - if (sameElementwiseShape) - { - return ComputeMeanAbsoluteErrorTapeLoss(predicted, target); - } + private static BoundingBox TargetBox(DetectionTrainingTarget target) => + new(target.CenterX, target.CenterY, target.Width, target.Height, BoundingBoxFormat.CXCYWH); - throw new ArgumentException( - $"DETR tape loss requires predicted shape [batch, queries, {_numClasses + 4}] and " + - "target shape [the same batch, objects, at least 5], or identical shapes for the " + - $"element-wise fallback. Received predicted [{string.Join(", ", predicted.Shape)}] " + - $"and target [{string.Join(", ", target.Shape)}].", - nameof(target)); - } + private Tensor ToCorners(Tensor boxes) + { + int count = boxes.Shape[0]; + var centers = Engine.TensorSlice(boxes, new[] { 0, 0 }, new[] { count, 2 }); + var halfExtents = Engine.TensorMultiplyScalar( + Engine.TensorSlice(boxes, new[] { 0, 2 }, new[] { count, 2 }), NumOps.FromDouble(0.5)); + return Engine.TensorConcatenate( + new[] { Engine.TensorSubtract(centers, halfExtents), Engine.TensorAdd(centers, halfExtents) }, 1); + } - int batch = predicted.Shape[0]; - int numQueries = predicted.Shape[1]; - int predDim = predicted.Shape[2]; // num_classes + 4 + private void ValidateHeads(Tensor logits, Tensor boxes, DetectionTrainingBatch targets) + { + if (logits is null) throw new ArgumentNullException(nameof(logits)); + if (boxes is null) throw new ArgumentNullException(nameof(boxes)); + if (targets is null) throw new ArgumentNullException(nameof(targets)); + if (logits.Rank != 3 || logits.Shape[0] <= 0 || logits.Shape[1] <= 0 || logits.Shape[2] != _numClasses) + throw new ArgumentException("Class logits must be [positive batch, positive queries, configured classes].", nameof(logits)); + if (boxes.Rank != 3 || boxes.Shape[0] != logits.Shape[0] || boxes.Shape[1] != logits.Shape[1] || boxes.Shape[2] != 4) + throw new ArgumentException("Boxes must match the logit batch and query dimensions, with four cxcywh coordinates.", nameof(boxes)); + targets.ValidateForModel(logits.Shape[0], ForegroundClasses, logits.Shape[1]); + } - // Step 1: Run Hungarian matching on detached data (discrete, not differentiable) - // Extract CPU data for matching computation - var matchedPredBoxIndices = new List<(int batch, int predIdx, int gtIdx)>(); - for (int b = 0; b < batch; b++) + private void ValidateFinitePredictions(T[] logits, T[] boxes) + { + foreach (T value in logits) { - var predBoxes = ExtractPredictedBoxes(predicted, b, numQueries); - var predLogits = ExtractPredictedLogits(predicted, b, numQueries, _numClasses); - var gtBoxes = ExtractGroundTruthBoxes(target, b); - var gtClasses = ExtractGroundTruthClasses(target, b); - - if (gtBoxes.Count == 0) continue; - - var (predIndices, gtIndices) = HungarianMatch(predBoxes, predLogits, gtBoxes, gtClasses); - for (int m = 0; m < predIndices.Length; m++) - matchedPredBoxIndices.Add((b, predIndices[m], gtIndices[m])); + double number = NumOps.ToDouble(value); + if (double.IsNaN(number) || double.IsInfinity(number)) + throw new ArgumentException("Class logits must be finite.", nameof(logits)); } - - if (matchedPredBoxIndices.Count == 0) + foreach (T value in boxes) { - // An image with no ground-truth objects contributes no matching loss, but the result - // must still be attached to the tape: a detached constant would make the whole batch's - // gradient vanish rather than merely contributing nothing to it. Scaling the prediction - // by zero keeps the graph connected and the value at zero. - var scaled = Engine.TensorMultiplyScalar(predicted, NumOps.Zero); - var allAxes = Enumerable.Range(0, scaled.Shape.Length).ToArray(); - return Engine.ReduceSum(scaled, allAxes, keepDims: false); + double number = NumOps.ToDouble(value); + if (double.IsNaN(number) || double.IsInfinity(number) || number < 0 || number > 1) + throw new ArgumentException("Predicted sigmoid cxcywh coordinates must be finite and in [0, 1].", nameof(boxes)); } + } - // Step 2: Compute differentiable losses on matched pairs using engine ops - // For each matched pair, slice the predicted tensor to get tape-tracked boxes/logits - var numOps = NumOps; - T totalClassLoss = numOps.Zero; - T totalBoxL1Loss = numOps.Zero; - - // Build tape-tracked matched prediction tensors via gather from predicted. - // Use engine ops so gradients flow back to the original predicted tensor. - int numMatched = matchedPredBoxIndices.Count; - int boxOffset = _numClasses; // boxes start after class logits - - // Gather matched boxes and logits from predicted using index tensors - // We build flat index arrays then gather slices via engine scatter/gather - var matchedTargData = new T[numMatched * 4]; - var matchedPredBoxSlices = new List>(numMatched); - var matchedPredLogitSlices = new List>(numMatched); - var matchedGtClasses = new int[numMatched]; - - for (int m = 0; m < numMatched; m++) - { - var (b, pi, gi) = matchedPredBoxIndices[m]; - - // Slice predicted[b, pi, :] for this query (tape-tracked) - var predBatch = Engine.TensorSliceAxis(predicted, 0, b); // shape [numQueries, predDim] - var predQuery = Engine.TensorSliceAxis(predBatch, 0, pi); // shape [predDim] - - // Slice box coordinates: [boxOffset:boxOffset+4] - var predBox = Engine.TensorSlice(predQuery, new[] { boxOffset }, new[] { 4 }); // shape [4] - matchedPredBoxSlices.Add(predBox); - - // Slice class logits: [0:numClasses] - var predLogits = Engine.TensorSlice(predQuery, new[] { 0 }, new[] { _numClasses }); // shape [numClasses] - matchedPredLogitSlices.Add(predLogits); - - // Extract target box (non-differentiable target) - for (int c = 0; c < 4; c++) - matchedTargData[m * 4 + c] = target[b, gi, 1 + c]; - - matchedGtClasses[m] = (int)numOps.ToDouble(target[b, gi, 0]); - } + private bool IsStructuredPrediction(Tensor predicted) => + predicted.Rank == 3 && predicted.Shape[2] == _numClasses + 4; - // Stack matched boxes: [numMatched, 4] - var matchedPredBoxes = Engine.TensorStack(matchedPredBoxSlices.ToArray(), axis: 0); - var matchedTarg = new Tensor(matchedTargData, new[] { numMatched, 4 }); + private bool HasStructuredLayout(Tensor predicted, Tensor target) + { + if (!IsStructuredPrediction(predicted) || predicted.Shape[0] <= 0 || predicted.Shape[1] <= 0) + return false; + return target.Rank == 3 && target.Shape[0] == predicted.Shape[0] && target.Shape[2] >= 5; + } - // L1 box loss via engine ops (tape-tracked through predicted) - var boxDiff = Engine.TensorSubtract(matchedPredBoxes, matchedTarg); - var boxAbsDiff = Engine.TensorAbs(boxDiff); - var boxAxes = Enumerable.Range(0, boxAbsDiff.Shape.Length).ToArray(); - var l1Loss = Engine.ReduceMean(boxAbsDiff, boxAxes, keepDims: false); + private void ValidateStructuredLayout(Tensor predicted, Tensor target) + { + if (predicted is null) throw new ArgumentNullException(nameof(predicted)); + if (target is null) throw new ArgumentNullException(nameof(target)); + if (!HasStructuredLayout(predicted, target)) + throw new ArgumentException( + $"DETR loss requires predicted [batch, queries, {_numClasses + 4}] and target [the same batch, objects, at least 5]. " + + $"Received predicted [{string.Join(", ", predicted.Shape)}] and target [{string.Join(", ", target.Shape)}].", + nameof(target)); + } - // Classification loss via engine ops (tape-tracked) - // Compute per-match cross entropy through engine softmax + nll - var classLossTerms = new List>(numMatched); - for (int m = 0; m < numMatched; m++) - { - int gtClass = matchedGtClasses[m]; - if (gtClass >= 0 && gtClass < _numClasses) - { - // LogSoftmax through engine (tape-tracked) - var logits = matchedPredLogitSlices[m]; - var logSoftmax = Engine.TensorLogSoftmax(logits, axis: 0); - - // NLL: -logSoftmax[gtClass] - var targetOneHot = new Tensor(new int[] { _numClasses }); - targetOneHot[gtClass] = numOps.One; - var nll = Engine.TensorMultiply(logSoftmax, targetOneHot); - var negNll = Engine.TensorNegate(Engine.ReduceSum(nll, new[] { 0 }, keepDims: false)); - classLossTerms.Add(negNll); - } - } + private static void WriteBox(T[] destination, int offset, DetectionTrainingTarget target) + { + destination[offset] = target.CenterX; + destination[offset + 1] = target.CenterY; + destination[offset + 2] = target.Width; + destination[offset + 3] = target.Height; + } - Tensor classLoss; - if (classLossTerms.Count > 0) - { - var classStack = Engine.TensorStack(classLossTerms.ToArray(), axis: 0); - classLoss = Engine.ReduceMean(classStack, new[] { 0 }, keepDims: false); - } - else + private static DetrSetLossOptions SoftmaxOptions(double classWeight, double boxL1Weight, double boxGIoUWeight) + { + ValidateWeight(classWeight, nameof(classWeight)); + ValidateWeight(boxL1Weight, nameof(boxL1Weight)); + ValidateWeight(boxGIoUWeight, nameof(boxGIoUWeight)); + return new DetrSetLossOptions { - classLoss = new Tensor(new T[] { numOps.Zero }, new[] { 1 }); - classLoss = Engine.ReduceSum(classLoss, new[] { 0 }, keepDims: false); - } - - // Composite: class_weight * CE + box_l1_weight * L1 - var weightedClass = Engine.TensorMultiplyScalar(classLoss, numOps.FromDouble(_classWeight)); - var weightedL1 = Engine.TensorMultiplyScalar(l1Loss, numOps.FromDouble(_boxL1Weight)); + ClassificationLoss = SetPredictionClassificationLoss.SoftmaxCrossEntropy, + ClassLossWeight = classWeight, + ClassCostWeight = classWeight, + L1LossWeight = boxL1Weight, + L1CostWeight = boxL1Weight, + GIoULossWeight = boxGIoUWeight, + GIoUCostWeight = boxGIoUWeight + }; + } - return Engine.TensorAdd(weightedClass, weightedL1); + private static void ValidateWeight(double weight, string parameterName) + { + if (double.IsNaN(weight) || double.IsInfinity(weight) || weight < 0) + throw new ArgumentOutOfRangeException(parameterName, "Loss weights must be finite and nonnegative."); } } diff --git a/src/ComputerVision/Detection/Losses/DetrSetLossOptions.cs b/src/ComputerVision/Detection/Losses/DetrSetLossOptions.cs new file mode 100644 index 0000000000..92aabf5b1c --- /dev/null +++ b/src/ComputerVision/Detection/Losses/DetrSetLossOptions.cs @@ -0,0 +1,126 @@ +using AiDotNet.Enums; + +namespace AiDotNet.ComputerVision.Detection.Losses; + +/// Weights and classification form for a DETR-family set prediction loss. +/// +/// +/// Matching costs and loss weights are separate, because the published recipes differ between them: +/// DINO (Zhang et al. 2022, Table 8) and RT-DETR (Zhao et al. 2023, Table A) both match with class +/// cost 2, L1 cost 5 and GIoU cost 2, but train with class weight 1, L1 weight 5 and GIoU weight 2. +/// The factory methods return each paper's defaults; every value can be overridden. +/// +/// For Beginners: Training a set detector happens in two steps. First each real object +/// is paired with one prediction using the matching costs. Then the paired predictions are pushed +/// toward their objects using the loss weights. These options control both steps. +/// +public sealed class DetrSetLossOptions +{ + /// How candidate class scores are trained. + /// For Beginners: Must match how the detector's class head is decoded: + /// softmax heads include a no-object class, sigmoid heads do not. + public SetPredictionClassificationLoss ClassificationLoss { get; set; } = SetPredictionClassificationLoss.SoftmaxCrossEntropy; + + /// Weight of the classification loss. + /// For Beginners: Larger values make correct class labels matter more. + public double ClassLossWeight { get; set; } = 1.0; + + /// Weight of the matched center-format L1 box loss. + /// For Beginners: Larger values push box coordinates harder. + public double L1LossWeight { get; set; } = 5.0; + + /// Weight of the matched generalized IoU box loss. + /// For Beginners: Larger values push box overlap harder. + public double GIoULossWeight { get; set; } = 2.0; + + /// Weight of the classification term in the matching cost. + /// For Beginners: Larger values pair objects with confident predictions. + public double ClassCostWeight { get; set; } = 1.0; + + /// Weight of the L1 box distance in the matching cost. + /// For Beginners: Larger values pair objects with nearby predictions. + public double L1CostWeight { get; set; } = 5.0; + + /// Weight of the negative generalized IoU in the matching cost. + /// For Beginners: Larger values pair objects with overlapping predictions. + public double GIoUCostWeight { get; set; } = 2.0; + + /// Relative weight of the no-object class in softmax cross-entropy (DETR uses 0.1). + /// For Beginners: Most predictions are empty, so this keeps them from + /// dominating training. Only used by . + public double NoObjectWeight { get; set; } = 0.1; + + /// The alpha of the focal or varifocal classification loss. + /// For Beginners: Balances object and background examples. Focal loss + /// uses 0.25 (Lin et al. 2017); varifocal loss in RT-DETR uses 0.75. + public double FocalAlpha { get; set; } = 0.25; + + /// The gamma (focusing exponent) of the focal or varifocal classification loss. + /// For Beginners: Larger values ignore easy examples more (2 in both papers). + public double FocalGamma { get; set; } = 2.0; + + /// The alpha of the focal classification matching cost used with sigmoid heads. + /// For Beginners: Only affects which prediction is paired with each object. + public double MatchingFocalAlpha { get; set; } = 0.25; + + /// The gamma of the focal classification matching cost used with sigmoid heads. + /// For Beginners: Only affects which prediction is paired with each object. + public double MatchingFocalGamma { get; set; } = 2.0; + + /// DETR defaults: softmax cross-entropy, no-object weight 0.1, costs and weights 1/5/2. + /// For Beginners: Use with detectors whose class head has a no-object class. + public static DetrSetLossOptions ForDetr() => new(); + + /// DINO defaults: sigmoid focal loss (alpha 0.25, gamma 2), costs 2/5/2, weights 1/5/2. + /// For Beginners: The published DINO recipe, Table 8 of Zhang et al. 2022. + public static DetrSetLossOptions ForDino() => new() + { + ClassificationLoss = SetPredictionClassificationLoss.SigmoidFocal, + ClassCostWeight = 2.0 + }; + + /// RT-DETR defaults: varifocal loss (alpha 0.75, gamma 2), costs 2/5/2, weights 1/5/2. + /// For Beginners: The published RT-DETR recipe, Table A of Zhao et al. 2023. + public static DetrSetLossOptions ForRtDetr() => new() + { + ClassificationLoss = SetPredictionClassificationLoss.VariFocal, + ClassCostWeight = 2.0, + FocalAlpha = 0.75 + }; + + internal DetrSetLossOptions Snapshot() + { + var copy = (DetrSetLossOptions)MemberwiseClone(); + copy.Validate(); + return copy; + } + + internal void Validate() + { + if (!Enum.IsDefined(typeof(SetPredictionClassificationLoss), ClassificationLoss)) + throw new ArgumentOutOfRangeException(nameof(ClassificationLoss)); + RequireNonnegative(ClassLossWeight, nameof(ClassLossWeight)); + RequireNonnegative(L1LossWeight, nameof(L1LossWeight)); + RequireNonnegative(GIoULossWeight, nameof(GIoULossWeight)); + RequireNonnegative(ClassCostWeight, nameof(ClassCostWeight)); + RequireNonnegative(L1CostWeight, nameof(L1CostWeight)); + RequireNonnegative(GIoUCostWeight, nameof(GIoUCostWeight)); + RequireNonnegative(NoObjectWeight, nameof(NoObjectWeight)); + RequireNonnegative(FocalGamma, nameof(FocalGamma)); + RequireNonnegative(MatchingFocalGamma, nameof(MatchingFocalGamma)); + RequireProbability(FocalAlpha, nameof(FocalAlpha)); + RequireProbability(MatchingFocalAlpha, nameof(MatchingFocalAlpha)); + } + + private static void RequireNonnegative(double value, string name) + { + if (double.IsNaN(value) || double.IsInfinity(value) || value < 0) + throw new ArgumentOutOfRangeException(name, "Loss weights and exponents must be finite and nonnegative."); + } + + private static void RequireProbability(double value, string name) + { + if (double.IsNaN(value) || value < 0 || value > 1) + throw new ArgumentOutOfRangeException(name, "Focal alpha must be in [0, 1]."); + } +} diff --git a/src/ComputerVision/Detection/Losses/TaskAlignedDetectionLoss.cs b/src/ComputerVision/Detection/Losses/TaskAlignedDetectionLoss.cs new file mode 100644 index 0000000000..4f826b2c31 --- /dev/null +++ b/src/ComputerVision/Detection/Losses/TaskAlignedDetectionLoss.cs @@ -0,0 +1,388 @@ +using AiDotNet.Tensors.Engines; +using AiDotNet.Tensors.Engines.Autodiff; + +namespace AiDotNet.ComputerVision.Detection.Losses; + +/// Task-aligned assignment with BCE, CIoU and distribution focal losses for anchor-free YOLO heads. +/// The numeric type used for calculations. +/// +/// +/// Assignment (Feng et al. 2021, TOOD): an anchor point is a candidate for an object when it lies inside +/// the object's box. Candidates are ranked by t = s^alpha * IoU^beta, where s is the predicted score of +/// the object's class and IoU is between the anchor's predicted box and the object. The top-k candidates +/// become positives; an anchor selected by several objects keeps the one with the highest IoU. Each +/// positive's classification target is t normalized so that, per object, the largest target equals +/// the largest IoU among that object's positives. Assignment uses detached predictions. +/// +/// +/// Losses: BCE of every class logit against those targets; CIoU of each positive's decoded box; and the +/// distribution focal loss of Li et al. (2020), DFL = -((y_{i+1} - y) log S_i + (y - y_i) log S_{i+1}), +/// averaged over the four box sides. Box and DFL terms are weighted per positive by its target. All +/// three sums are divided by the total target mass (at least 1) and scaled by the configured gains. +/// Boxes and DFL distances are measured in units of each level's stride, as the head predicts them. +/// +/// For Beginners: This is the training objective of YOLOv8-style detectors. It picks the grid +/// cells best placed to detect each object, teaches their class scores to reflect how well they detect +/// it, and pulls their predicted box edges toward the real ones. +/// +public sealed class TaskAlignedDetectionLoss +{ + private const double InsideEpsilon = 1e-9; + private const double MetricEpsilon = 1e-9; + private static readonly INumericOperations NumOps = AiDotNet.Tensors.Helpers.MathHelper.GetNumericOperations(); + private readonly TaskAlignedLossOptions _options; + private readonly int _numClasses; + private readonly int _regMax; + + /// Creates the objective for a head with the given class count and distribution bins. + /// Foreground classes scored by independent sigmoids. + /// Distribution bins per box side (16 in YOLOv8-family heads). + /// Assignment exponents, top-k and loss gains; copied on construction. + public TaskAlignedDetectionLoss(int numClasses, int regMax, TaskAlignedLossOptions options) + { + if (numClasses < 1) throw new ArgumentOutOfRangeException(nameof(numClasses)); + if (regMax < 2) throw new ArgumentOutOfRangeException(nameof(regMax), "A distribution needs at least two bins."); + if (options is null) throw new ArgumentNullException(nameof(options)); + _options = options.Snapshot(); + _numClasses = numClasses; + _regMax = regMax; + } + + /// Anchors selected per object by the one-to-many assignment. + public int TopK => _options.TopK; + + /// Anchors selected per object by a one-to-one head. + public int OneToOneTopK => _options.OneToOneTopK; + + /// Evaluates the objective without recording gradients. + public T CalculateLoss(IReadOnlyList> classLevels, IReadOnlyList> distributionLevels, + IReadOnlyList strides, int imageHeight, int imageWidth, DetectionTrainingBatch targets, int topK) + { + using var noGrad = new NoGradScope(); + using var objective = ComputeTapeLoss(classLevels, distributionLevels, strides, imageHeight, imageWidth, targets, topK); + return objective[0]; + } + + /// Builds the differentiable objective over the live head outputs. + /// Raw class logits per level, [batch, classes, height, width]. + /// Raw box-distribution logits per level, [batch, 4 * regMax, height, width]. + /// Input pixels per feature cell for each level. + /// Height in pixels of the network input the targets are normalized against. + /// Width in pixels of the network input the targets are normalized against. + /// Normalized center-format targets, one list per image; empty lists are valid. + /// Anchors selected per object: or . + /// A scalar connected to every class and distribution level on the active tape. + public Tensor ComputeTapeLoss(IReadOnlyList> classLevels, IReadOnlyList> distributionLevels, + IReadOnlyList strides, int imageHeight, int imageWidth, DetectionTrainingBatch targets, int topK) + { + Validate(classLevels, distributionLevels, strides, imageHeight, imageWidth, targets, topK); + var engine = AiDotNetEngine.Current; + int levels = classLevels.Count; + int batch = classLevels[0].Shape[0]; + var cells = new int[levels]; + var widths = new int[levels]; + var levelStart = new int[levels + 1]; + for (int level = 0; level < levels; level++) + { + widths[level] = classLevels[level].Shape[3]; + cells[level] = classLevels[level].Shape[2] * widths[level]; + levelStart[level + 1] = levelStart[level] + cells[level]; + } + int anchors = levelStart[levels]; + + var classData = new T[levels][]; + var distributionData = new T[levels][]; + for (int level = 0; level < levels; level++) + { + classData[level] = classLevels[level].ToArray(); + distributionData[level] = distributionLevels[level].ToArray(); + } + + var positives = new List(); + double targetMass = 0; + for (int image = 0; image < batch; image++) + targetMass += Assign(image, targets[image], classData, distributionData, strides, cells, widths, levelStart, + imageHeight, imageWidth, topK, positives); + double normalizer = Math.Max(1.0, targetMass); + + // Classification: BCE-with-logits, softplus(x) - t x, over every class logit of every anchor. + Tensor? classification = null; + var byLevel = positives.GroupBy(positive => positive.Level).ToDictionary(group => group.Key, group => group.ToArray()); + for (int level = 0; level < levels; level++) + { + var targetValues = new T[classLevels[level].Length]; + if (byLevel.TryGetValue(level, out var levelPositives)) + foreach (var positive in levelPositives) + targetValues[(positive.Image * _numClasses + positive.ClassId) * cells[level] + positive.Cell] = NumOps.FromDouble(positive.Target); + var logits = classLevels[level]; + var targetTensor = new Tensor(targetValues, logits.Shape.ToArray()); + var bce = engine.ReduceSum(engine.TensorSubtract(engine.Softplus(logits), engine.TensorMultiply(targetTensor, logits)), null); + classification = classification is null ? bce : engine.TensorAdd(classification, bce); + } + var loss = engine.TensorMultiplyScalar(classification ?? throw new InvalidOperationException("No class levels were supplied."), + NumOps.FromDouble(_options.ClassGain / normalizer)); + + if (positives.Count == 0) + { + // Background classification is still trained. Connect each distribution level with an exact zero derivative. + foreach (var distribution in distributionLevels) + loss = engine.TensorAdd(loss, engine.TensorMultiplyScalar(engine.ReduceSum(distribution, null), NumOps.Zero)); + return loss; + } + + var ordered = Enumerable.Range(0, levels).Where(byLevel.ContainsKey).SelectMany(level => byLevel[level]).ToArray(); + int count = ordered.Length; + int bins = _regMax; + var gathered = new List>(); + foreach (int level in Enumerable.Range(0, levels).Where(byLevel.ContainsKey)) + { + var levelPositives = byLevel[level]; + var indices = new int[checked(levelPositives.Length * 4 * bins)]; + int write = 0; + foreach (var positive in levelPositives) + for (int side = 0; side < 4; side++) + for (int bin = 0; bin < bins; bin++) + indices[write++] = (positive.Image * 4 * bins + side * bins + bin) * cells[level] + positive.Cell; + var flat = engine.Reshape(distributionLevels[level], new[] { distributionLevels[level].Length }); + gathered.Add(CvTensorOps.Select(flat, indices, 0)); + } + var rows = engine.Reshape(gathered.Count == 1 ? gathered[0] : engine.TensorConcatenate(gathered.ToArray(), 0), + new[] { count * 4, bins }); + + // Decoded distances: the expectation of each side's softmax distribution, in stride units. + var binValues = new T[count * 4 * bins]; + for (int row = 0; row < count * 4; row++) + for (int bin = 0; bin < bins; bin++) + binValues[row * bins + bin] = NumOps.FromDouble(bin); + var distances = engine.Reshape(engine.ReduceSum( + engine.TensorMultiply(engine.TensorSoftmax(rows, 1), new Tensor(binValues, new[] { count * 4, bins })), + new[] { 1 }, false), new[] { count, 4 }); + + var anchorX = new T[count]; + var anchorY = new T[count]; + var goldBoxes = new T[count * 4]; + var weights = new T[count]; + var dflIndices = new int[count * 8]; + var dflWeights = new T[count * 8]; + for (int index = 0; index < count; index++) + { + var positive = ordered[index]; + double stride = strides[positive.Level]; + double gridX = positive.AnchorX / stride; + double gridY = positive.AnchorY / stride; + anchorX[index] = NumOps.FromDouble(gridX); + anchorY[index] = NumOps.FromDouble(gridY); + var gold = new[] { positive.Gold[0] / stride, positive.Gold[1] / stride, positive.Gold[2] / stride, positive.Gold[3] / stride }; + for (int coordinate = 0; coordinate < 4; coordinate++) goldBoxes[index * 4 + coordinate] = NumOps.FromDouble(gold[coordinate]); + weights[index] = NumOps.FromDouble(positive.Target); + + var sides = new[] { gridX - gold[0], gridY - gold[1], gold[2] - gridX, gold[3] - gridY }; + for (int side = 0; side < 4; side++) + { + double distance = Math.Min(Math.Max(sides[side], 0), bins - 1 - 0.01); + int lower = (int)Math.Floor(distance); + int row = index * 4 + side; + double sideWeight = positive.Target / 4; + dflIndices[row * 2] = row * bins + lower; + dflIndices[row * 2 + 1] = row * bins + lower + 1; + dflWeights[row * 2] = NumOps.FromDouble((lower + 1 - distance) * sideWeight); + dflWeights[row * 2 + 1] = NumOps.FromDouble((distance - lower) * sideWeight); + } + } + + var column = new[] { count, 1 }; + var x = new Tensor(anchorX, column); + var y = new Tensor(anchorY, column); + var predicted = engine.TensorConcatenate(new[] + { + engine.TensorSubtract(x, engine.TensorNarrow(distances, 1, 0, 1)), + engine.TensorSubtract(y, engine.TensorNarrow(distances, 1, 1, 1)), + engine.TensorAdd(x, engine.TensorNarrow(distances, 1, 2, 1)), + engine.TensorAdd(y, engine.TensorNarrow(distances, 1, 3, 1)) + }, 1); + var ciou = engine.TensorCIoULoss(predicted, new Tensor(goldBoxes, new[] { count, 4 })); + var box = engine.ReduceSum(engine.TensorMultiply(engine.Reshape(ciou, new[] { count }), new Tensor(weights, new[] { count })), null); + loss = engine.TensorAdd(loss, engine.TensorMultiplyScalar(box, NumOps.FromDouble(_options.BoxGain / normalizer))); + + // Gather only the two neighbouring bins, so an extreme finite logit elsewhere cannot turn 0 * -inf into NaN. + var logProbabilities = engine.Reshape(engine.TensorLogSoftmax(rows, 1), new[] { count * 4 * bins }); + var dfl = engine.TensorNegate(engine.ReduceSum(engine.TensorMultiply( + CvTensorOps.Select(logProbabilities, dflIndices, 0), new Tensor(dflWeights, new[] { dflWeights.Length })), null)); + loss = engine.TensorAdd(loss, engine.TensorMultiplyScalar(dfl, NumOps.FromDouble(_options.DflGain / normalizer))); + + // A level can hold no positives (always true for most levels under top-1). Its distribution logits + // still belong to the objective, with an exact zero derivative, so every head output has a gradient. + foreach (int level in Enumerable.Range(0, levels).Where(level => !byLevel.ContainsKey(level))) + loss = engine.TensorAdd(loss, engine.TensorMultiplyScalar(engine.ReduceSum(distributionLevels[level], null), NumOps.Zero)); + return loss; + } + + private double Assign(int image, IReadOnlyList> objects, T[][] classData, T[][] distributionData, + IReadOnlyList strides, int[] cells, int[] widths, int[] levelStart, int imageHeight, int imageWidth, int topK, + List positives) + { + if (objects.Count == 0) return 0; + int anchors = levelStart[levelStart.Length - 1]; + var anchorX = new double[anchors]; + var anchorY = new double[anchors]; + var predicted = new double[anchors, 4]; + var anchorLevel = new int[anchors]; + for (int level = 0; level < cells.Length; level++) + { + int stride = strides[level]; + for (int cell = 0; cell < cells[level]; cell++) + { + int anchor = levelStart[level] + cell; + anchorLevel[anchor] = level; + // Row-major grid: the integer quotient is the row, the remainder the column. + int row = cell / widths[level]; + int column = cell % widths[level]; + anchorX[anchor] = (column + 0.5) * stride; + anchorY[anchor] = (row + 0.5) * stride; + for (int side = 0; side < 4; side++) + { + double expectation = ExpectedBin(distributionData[level], image, side, cells[level], cell) * stride; + predicted[anchor, side] = side < 2 + ? (side == 0 ? anchorX[anchor] : anchorY[anchor]) - expectation + : (side == 2 ? anchorX[anchor] : anchorY[anchor]) + expectation; + } + } + } + + var gold = new double[objects.Count][]; + var assigned = new int[anchors]; + var assignedIoU = new double[anchors]; + var assignedMetric = new double[anchors]; + for (int anchor = 0; anchor < anchors; anchor++) assigned[anchor] = -1; + for (int index = 0; index < objects.Count; index++) + { + var target = objects[index]; + double cx = NumOps.ToDouble(target.CenterX) * imageWidth; + double cy = NumOps.ToDouble(target.CenterY) * imageHeight; + double halfWidth = NumOps.ToDouble(target.Width) * imageWidth / 2; + double halfHeight = NumOps.ToDouble(target.Height) * imageHeight / 2; + gold[index] = new[] { cx - halfWidth, cy - halfHeight, cx + halfWidth, cy + halfHeight }; + + var candidates = new List<(int Anchor, double Metric, double IoU)>(); + for (int anchor = 0; anchor < anchors; anchor++) + { + double inside = Math.Min(Math.Min(anchorX[anchor] - gold[index][0], anchorY[anchor] - gold[index][1]), + Math.Min(gold[index][2] - anchorX[anchor], gold[index][3] - anchorY[anchor])); + if (inside <= InsideEpsilon) continue; + int level = anchorLevel[anchor]; + int cell = anchor - levelStart[level]; + double score = Logistic(NumOps.ToDouble(classData[level][(image * _numClasses + target.ClassId) * cells[level] + cell])); + double iou = IoU(predicted, anchor, gold[index]); + candidates.Add((anchor, Math.Pow(score, _options.Alpha) * Math.Pow(iou, _options.Beta), iou)); + } + foreach (var candidate in candidates.OrderByDescending(item => item.Metric).ThenBy(item => item.Anchor).Take(topK)) + { + if (assigned[candidate.Anchor] >= 0 && candidate.IoU <= assignedIoU[candidate.Anchor]) continue; + assigned[candidate.Anchor] = index; + assignedIoU[candidate.Anchor] = candidate.IoU; + assignedMetric[candidate.Anchor] = candidate.Metric; + } + } + + var maximumMetric = new double[objects.Count]; + var maximumIoU = new double[objects.Count]; + for (int anchor = 0; anchor < anchors; anchor++) + { + int index = assigned[anchor]; + if (index < 0) continue; + maximumMetric[index] = Math.Max(maximumMetric[index], assignedMetric[anchor]); + maximumIoU[index] = Math.Max(maximumIoU[index], assignedIoU[anchor]); + } + + double mass = 0; + for (int anchor = 0; anchor < anchors; anchor++) + { + int index = assigned[anchor]; + if (index < 0) continue; + double normalized = assignedMetric[anchor] * maximumIoU[index] / (maximumMetric[index] + MetricEpsilon); + int level = anchorLevel[anchor]; + positives.Add(new Positive(image, level, anchor - levelStart[level], objects[index].ClassId, + anchorX[anchor], anchorY[anchor], gold[index], normalized)); + mass += normalized; + } + return mass; + } + + private double ExpectedBin(T[] distribution, int image, int side, int cells, int cell) + { + double maximum = double.NegativeInfinity; + for (int bin = 0; bin < _regMax; bin++) + maximum = Math.Max(maximum, NumOps.ToDouble(distribution[(image * 4 * _regMax + side * _regMax + bin) * cells + cell])); + double total = 0; + double weighted = 0; + for (int bin = 0; bin < _regMax; bin++) + { + double value = Math.Exp(NumOps.ToDouble(distribution[(image * 4 * _regMax + side * _regMax + bin) * cells + cell]) - maximum); + total += value; + weighted += value * bin; + } + return weighted / total; + } + + private static double IoU(double[,] predicted, int anchor, double[] gold) + { + double width = Math.Max(0, Math.Min(predicted[anchor, 2], gold[2]) - Math.Max(predicted[anchor, 0], gold[0])); + double height = Math.Max(0, Math.Min(predicted[anchor, 3], gold[3]) - Math.Max(predicted[anchor, 1], gold[1])); + double intersection = width * height; + double union = (predicted[anchor, 2] - predicted[anchor, 0]) * (predicted[anchor, 3] - predicted[anchor, 1]) + + (gold[2] - gold[0]) * (gold[3] - gold[1]) - intersection; + return union > 0 ? Math.Max(0, intersection / union) : 0; + } + + private static double Logistic(double x) => x >= 0 ? 1 / (1 + Math.Exp(-x)) : Math.Exp(x) / (1 + Math.Exp(x)); + + private void Validate(IReadOnlyList> classLevels, IReadOnlyList> distributionLevels, + IReadOnlyList strides, int imageHeight, int imageWidth, DetectionTrainingBatch targets, int topK) + { + if (classLevels is null) throw new ArgumentNullException(nameof(classLevels)); + if (distributionLevels is null) throw new ArgumentNullException(nameof(distributionLevels)); + if (strides is null) throw new ArgumentNullException(nameof(strides)); + if (targets is null) throw new ArgumentNullException(nameof(targets)); + if (classLevels.Count == 0 || distributionLevels.Count != classLevels.Count || strides.Count != classLevels.Count) + throw new ArgumentException("Class levels, distribution levels and strides must be nonempty and of equal count.", nameof(classLevels)); + if (imageHeight <= 0 || imageWidth <= 0) throw new ArgumentOutOfRangeException(nameof(imageHeight), "Image dimensions must be positive."); + if (topK < 1) throw new ArgumentOutOfRangeException(nameof(topK)); + int batch = classLevels[0].Rank == 4 ? classLevels[0].Shape[0] : 0; + for (int level = 0; level < classLevels.Count; level++) + { + var logits = classLevels[level]; + var distribution = distributionLevels[level]; + if (logits.Rank != 4 || logits.Shape[0] != batch || batch <= 0 || logits.Shape[1] != _numClasses || logits.Shape[2] <= 0 || logits.Shape[3] <= 0) + throw new ArgumentException($"Class level {level} must be [batch, {_numClasses}, height, width].", nameof(classLevels)); + if (distribution.Rank != 4 || distribution.Shape[0] != batch || distribution.Shape[1] != 4 * _regMax + || distribution.Shape[2] != logits.Shape[2] || distribution.Shape[3] != logits.Shape[3]) + throw new ArgumentException($"Distribution level {level} must be [batch, {4 * _regMax}, height, width] matching its class level.", nameof(distributionLevels)); + if (strides[level] <= 0) throw new ArgumentOutOfRangeException(nameof(strides), "Strides must be positive."); + } + targets.ValidateForModel(batch, _numClasses, int.MaxValue); + } + + private sealed class Positive + { + internal Positive(int image, int level, int cell, int classId, double anchorX, double anchorY, double[] gold, double target) + { + Image = image; + Level = level; + Cell = cell; + ClassId = classId; + AnchorX = anchorX; + AnchorY = anchorY; + Gold = gold; + Target = target; + } + + internal int Image { get; } + internal int Level { get; } + internal int Cell { get; } + internal int ClassId { get; } + internal double AnchorX { get; } + internal double AnchorY { get; } + internal double[] Gold { get; } + internal double Target { get; } + } +} diff --git a/src/ComputerVision/Detection/Losses/TaskAlignedLossOptions.cs b/src/ComputerVision/Detection/Losses/TaskAlignedLossOptions.cs new file mode 100644 index 0000000000..8c44b38b71 --- /dev/null +++ b/src/ComputerVision/Detection/Losses/TaskAlignedLossOptions.cs @@ -0,0 +1,72 @@ +namespace AiDotNet.ComputerVision.Detection.Losses; + +/// Assignment and loss settings for task-aligned anchor-free YOLO training. +/// +/// +/// Defaults follow the published YOLO recipes. Task-aligned assignment ranks anchors inside each +/// object by t = s^alpha * IoU^beta (Feng et al. 2021, TOOD), with alpha 0.5 and beta 6 as in YOLOv8 and +/// both YOLOv10 heads (Wang et al. 2024, Sec. 3.1). Losses are BCE classification against the +/// normalized alignment target, CIoU box regression and distribution focal loss (Li et al. 2020), +/// weighted by the box/class/DFL gains 7.5/0.5/1.5 (YOLOv9 Table 1; YOLOv10 Table 14). YOLOv10's +/// one-to-one head uses top-1 selection. +/// +/// For Beginners: A YOLO model predicts a box and class scores at every grid cell. These +/// settings decide which cells are responsible for each real object and how strongly each part of the +/// prediction is corrected. +/// +public sealed class TaskAlignedLossOptions +{ + /// Anchors selected per object by the one-to-many assignment. + /// For Beginners: How many grid cells learn from each object. The YOLOv8 family + /// uses 10; TOOD used 13. + public int TopK { get; set; } = 10; + + /// Anchors selected per object by YOLOv10's one-to-one head. + /// For Beginners: YOLOv10 trains its inference head with exactly one cell per + /// object, which is what lets it skip non-maximum suppression. + public int OneToOneTopK { get; set; } = 1; + + /// Exponent of the classification score in the alignment metric. + /// For Beginners: Larger values favor cells that are already confident. + public double Alpha { get; set; } = 0.5; + + /// Exponent of the IoU in the alignment metric. + /// For Beginners: Larger values favor cells whose box already overlaps well. + public double Beta { get; set; } = 6.0; + + /// Weight of the CIoU box loss. + /// For Beginners: Larger values push box overlap harder. + public double BoxGain { get; set; } = 7.5; + + /// Weight of the BCE classification loss. + /// For Beginners: Larger values push class scores harder. + public double ClassGain { get; set; } = 0.5; + + /// Weight of the distribution focal loss. + /// For Beginners: Larger values sharpen the predicted box-edge distributions. + public double DflGain { get; set; } = 1.5; + + internal TaskAlignedLossOptions Snapshot() + { + var copy = (TaskAlignedLossOptions)MemberwiseClone(); + copy.Validate(); + return copy; + } + + internal void Validate() + { + if (TopK < 1) throw new ArgumentOutOfRangeException(nameof(TopK), "At least one anchor must be selected per object."); + if (OneToOneTopK < 1) throw new ArgumentOutOfRangeException(nameof(OneToOneTopK), "At least one anchor must be selected per object."); + RequireNonnegative(Alpha, nameof(Alpha)); + RequireNonnegative(Beta, nameof(Beta)); + RequireNonnegative(BoxGain, nameof(BoxGain)); + RequireNonnegative(ClassGain, nameof(ClassGain)); + RequireNonnegative(DflGain, nameof(DflGain)); + } + + private static void RequireNonnegative(double value, string name) + { + if (double.IsNaN(value) || double.IsInfinity(value) || value < 0) + throw new ArgumentOutOfRangeException(name, "Exponents and gains must be finite and nonnegative."); + } +} diff --git a/src/ComputerVision/Detection/Losses/TwoStageDetectionLoss.cs b/src/ComputerVision/Detection/Losses/TwoStageDetectionLoss.cs new file mode 100644 index 0000000000..bd6c2b4483 --- /dev/null +++ b/src/ComputerVision/Detection/Losses/TwoStageDetectionLoss.cs @@ -0,0 +1,314 @@ +using AiDotNet.Augmentation.Image; +using AiDotNet.Tensors.Engines; + +namespace AiDotNet.ComputerVision.Detection.Losses; + +/// Region proposal and region-of-interest losses for Faster R-CNN and Cascade R-CNN. +/// The numeric type used for calculations. +/// +/// +/// Region proposal network (Ren et al. 2015, Eq. 1): anchors are labeled by IoU with the objects (positive above the +/// positive threshold or as an object's best anchor, negative below the negative threshold, otherwise ignored), +/// a balanced sample is drawn, the two-way object/background cross-entropy is averaged over the sample, and the +/// smooth-L1 regression of positive anchors is weighted by lambda and divided by the number of anchor locations. +/// +/// +/// Region-of-interest head (Girshick 2015, Eq. 1-3): proposals with IoU of at least the stage threshold take their +/// object's class, proposals with IoU in [low, threshold) are background (class 0), a sample with a bounded +/// foreground fraction is drawn, cross-entropy is averaged over it, and foreground boxes regress the deltas of +/// their own class with smooth-L1, also averaged over the sample. Cascade R-CNN applies this per stage with rising +/// thresholds to the boxes that stage actually receives (Cai and Vasconcelos 2018, Eq. 8). +/// +/// +/// Box deltas use the R-CNN parameterization the detectors here decode: t_x = (g_x - p_x) / p_w, +/// t_y = (g_y - p_y) / p_h, t_w = log(g_w / p_w), t_h = log(g_h / p_h), with center coordinates. Assignment and +/// sampling use detached host values; the losses are built from engine operations on the live heads. +/// +/// For Beginners: the first loss teaches the detector where objects might be; the second teaches it what +/// each proposed region contains and how to tighten its box. +/// +public sealed class TwoStageDetectionLoss +{ + private static readonly INumericOperations NumOps = AiDotNet.Tensors.Helpers.MathHelper.GetNumericOperations(); + private readonly TwoStageDetectionLossOptions _options; + private readonly int _foregroundClasses; + + /// Creates the objective for a detector with the given foreground classes and detection stages. + /// Object classes; the heads add a background class at index 0. + /// Detection stages (1 for Faster R-CNN). + /// Sampling thresholds and weights; copied on construction. + public TwoStageDetectionLoss(int foregroundClasses, int stages, TwoStageDetectionLossOptions options) + { + if (foregroundClasses < 1) throw new ArgumentOutOfRangeException(nameof(foregroundClasses)); + if (stages < 1) throw new ArgumentOutOfRangeException(nameof(stages)); + if (options is null) throw new ArgumentNullException(nameof(options)); + _options = options.Snapshot(stages); + _foregroundClasses = foregroundClasses; + } + + /// The copied sampling thresholds and weights. + internal TwoStageDetectionLossOptions Options => _options; + + /// Builds the region proposal loss for one image. + /// Raw two-way logits [anchors, 2]; index 1 is "object". + /// Raw box deltas [anchors, 4]. + /// Anchors in input pixels (corner format), aligned with the logits. + /// Object boxes in input pixels, as (x1, y1, x2, y2). + /// Anchor shapes per feature position, for the paper's N_reg. + /// The source of the sampling draws. + public Tensor ComputeProposalLoss(Tensor objectness, Tensor deltas, IReadOnlyList> anchors, + IReadOnlyList gold, int anchorsPerLocation, Random random) + { + if (objectness is null) throw new ArgumentNullException(nameof(objectness)); + if (deltas is null) throw new ArgumentNullException(nameof(deltas)); + if (anchors is null) throw new ArgumentNullException(nameof(anchors)); + if (gold is null) throw new ArgumentNullException(nameof(gold)); + if (random is null) throw new ArgumentNullException(nameof(random)); + int count = anchors.Count; + if (objectness.Rank != 2 || objectness.Shape[0] != count || objectness.Shape[1] != 2) + throw new ArgumentException("Objectness must be [anchors, 2].", nameof(objectness)); + if (deltas.Rank != 2 || deltas.Shape[0] != count || deltas.Shape[1] != 4) + throw new ArgumentException("Deltas must be [anchors, 4].", nameof(deltas)); + if (anchorsPerLocation < 1) throw new ArgumentOutOfRangeException(nameof(anchorsPerLocation)); + + var anchorBoxes = anchors.Select(anchor => new[] + { + NumOps.ToDouble(anchor.X1), NumOps.ToDouble(anchor.Y1), NumOps.ToDouble(anchor.X2), NumOps.ToDouble(anchor.Y2) + }).ToArray(); + var labels = new int[count]; // -1 ignored, 0 background, 1 object + var match = new int[count]; + for (int a = 0; a < count; a++) labels[a] = -1; + if (gold.Count == 0) + { + for (int a = 0; a < count; a++) labels[a] = 0; + } + else + { + var bestForGold = new double[gold.Count]; + for (int a = 0; a < count; a++) + { + double best = -1; + for (int g = 0; g < gold.Count; g++) + { + double iou = IoU(anchorBoxes[a], gold[g]); + if (iou > best) { best = iou; match[a] = g; } + bestForGold[g] = Math.Max(bestForGold[g], iou); + } + if (best < _options.RpnNegativeIoU) labels[a] = 0; + if (best > _options.RpnPositiveIoU) labels[a] = 1; + } + // Ren et al.: the anchor(s) with the highest IoU for each object are positive even below the threshold. + for (int a = 0; a < count; a++) + for (int g = 0; g < gold.Count; g++) + if (bestForGold[g] > 0 && IoU(anchorBoxes[a], gold[g]) == bestForGold[g]) + { + labels[a] = 1; + match[a] = g; + } + } + + var positives = Sample(Enumerable.Range(0, count).Where(a => labels[a] == 1).ToList(), + (int)(_options.RpnBatchSizePerImage * _options.RpnPositiveFraction), random); + var negatives = Sample(Enumerable.Range(0, count).Where(a => labels[a] == 0).ToList(), + _options.RpnBatchSizePerImage - positives.Count, random); + var engine = AiDotNetEngine.Current; + int sampled = positives.Count + negatives.Count; + if (sampled == 0) + return engine.TensorAdd(ZeroConnected(objectness), ZeroConnected(deltas)); + + var rows = positives.Concat(negatives).ToArray(); + var classes = positives.Select(_ => 1).Concat(negatives.Select(_ => 0)).ToArray(); + var classification = engine.TensorMultiplyScalar(CrossEntropySum(objectness, rows, classes, 2), + NumOps.FromDouble(1.0 / sampled)); + if (positives.Count == 0) + return engine.TensorAdd(classification, ZeroConnected(deltas)); + + var targets = new T[positives.Count * 4]; + for (int i = 0; i < positives.Count; i++) + WriteDeltas(targets, i * 4, anchorBoxes[positives[i]], gold[match[positives[i]]]); + double locations = Math.Max(1.0, count / (double)anchorsPerLocation); + var regression = engine.TensorMultiplyScalar( + SmoothL1Sum(CvTensorOps.Select(deltas, positives.ToArray(), 0), new Tensor(targets, new[] { positives.Count, 4 })), + NumOps.FromDouble(_options.RpnRegressionWeight / locations)); + return engine.TensorAdd(classification, regression); + } + + /// Builds one detection stage's region-of-interest loss for one image. + /// Raw logits [proposals, foreground classes + 1]; index 0 is background. + /// Class-specific deltas [proposals, (classes + 1) * 4]. + /// The boxes this stage received [proposals, 4] in input pixels (corner format). + /// Object boxes in input pixels, as (x1, y1, x2, y2). + /// Each object's foreground class, aligned with . + /// Zero-based stage index selecting the IoU threshold and weight. + /// The source of the sampling draws. + public Tensor ComputeStageLoss(Tensor classLogits, Tensor boxDeltas, Tensor proposals, + IReadOnlyList gold, IReadOnlyList goldClasses, int stage, Random random) + { + if (classLogits is null) throw new ArgumentNullException(nameof(classLogits)); + if (boxDeltas is null) throw new ArgumentNullException(nameof(boxDeltas)); + if (proposals is null) throw new ArgumentNullException(nameof(proposals)); + if (gold is null) throw new ArgumentNullException(nameof(gold)); + if (goldClasses is null || goldClasses.Count != gold.Count) + throw new ArgumentException("Each object needs a class.", nameof(goldClasses)); + if (random is null) throw new ArgumentNullException(nameof(random)); + if (stage < 0 || stage >= _options.StageForegroundIoU.Length) + throw new ArgumentOutOfRangeException(nameof(stage)); + int width = _foregroundClasses + 1; + int count = proposals.Rank == 2 ? proposals.Shape[0] : -1; + if (count < 0 || proposals.Shape[1] != 4) + throw new ArgumentException("Proposals must be [proposals, 4].", nameof(proposals)); + if (classLogits.Rank != 2 || classLogits.Shape[0] != count || classLogits.Shape[1] != width) + throw new ArgumentException($"Class logits must be [proposals, {width}].", nameof(classLogits)); + if (boxDeltas.Rank != 2 || boxDeltas.Shape[0] != count || boxDeltas.Shape[1] != width * 4) + throw new ArgumentException($"Box deltas must be [proposals, {width * 4}].", nameof(boxDeltas)); + foreach (int goldClass in goldClasses) + if (goldClass < 0 || goldClass >= _foregroundClasses) + throw new ArgumentException("Every object class must be a foreground class of the detector.", nameof(goldClasses)); + + var engine = AiDotNetEngine.Current; + var proposalValues = proposals.ToArray(); + var boxes = new double[count][]; + var bestIoU = new double[count]; + var match = new int[count]; + for (int r = 0; r < count; r++) + { + boxes[r] = new[] + { + NumOps.ToDouble(proposalValues[r * 4]), NumOps.ToDouble(proposalValues[r * 4 + 1]), + NumOps.ToDouble(proposalValues[r * 4 + 2]), NumOps.ToDouble(proposalValues[r * 4 + 3]) + }; + for (int g = 0; g < gold.Count; g++) + { + double iou = IoU(boxes[r], gold[g]); + if (iou > bestIoU[r]) { bestIoU[r] = iou; match[r] = g; } + } + } + + double threshold = _options.StageForegroundIoU[stage]; + var foreground = Sample(Enumerable.Range(0, count).Where(r => gold.Count > 0 && bestIoU[r] >= threshold).ToList(), + (int)(_options.RoiBatchSizePerImage * _options.RoiForegroundFraction), random); + var background = Sample(Enumerable.Range(0, count) + .Where(r => (gold.Count == 0 || bestIoU[r] < threshold) && bestIoU[r] >= _options.RoiBackgroundIoULow).ToList(), + _options.RoiBatchSizePerImage - foreground.Count, random); + int sampled = foreground.Count + background.Count; + if (sampled == 0) + return engine.TensorAdd(ZeroConnected(classLogits), ZeroConnected(boxDeltas)); + + var rows = foreground.Concat(background).ToArray(); + var classes = foreground.Select(r => goldClasses[match[r]] + 1).Concat(background.Select(_ => 0)).ToArray(); + var classification = engine.TensorMultiplyScalar(CrossEntropySum(classLogits, rows, classes, width), + NumOps.FromDouble(1.0 / sampled)); + Tensor loss = classification; + if (foreground.Count > 0) + { + var deltaIndices = new int[foreground.Count * 4]; + var targets = new T[foreground.Count * 4]; + for (int i = 0; i < foreground.Count; i++) + { + int r = foreground[i]; + int column = (goldClasses[match[r]] + 1) * 4; + for (int k = 0; k < 4; k++) deltaIndices[i * 4 + k] = r * width * 4 + column + k; + WriteDeltas(targets, i * 4, boxes[r], gold[match[r]]); + } + var flat = engine.Reshape(boxDeltas, new[] { boxDeltas.Length }); + var predicted = engine.Reshape(CvTensorOps.Select(flat, deltaIndices, 0), new[] { foreground.Count, 4 }); + var regression = engine.TensorMultiplyScalar( + SmoothL1Sum(predicted, new Tensor(targets, new[] { foreground.Count, 4 })), + NumOps.FromDouble(_options.RoiRegressionWeight / sampled)); + loss = engine.TensorAdd(loss, regression); + } + else + { + loss = engine.TensorAdd(loss, ZeroConnected(boxDeltas)); + } + return engine.TensorMultiplyScalar(loss, NumOps.FromDouble(_options.StageLossWeights[stage])); + } + + /// R-CNN box deltas of relative to . + internal static double[] EncodeDeltas(double[] reference, double[] gold) + { + double pw = reference[2] - reference[0]; + double ph = reference[3] - reference[1]; + double gw = gold[2] - gold[0]; + double gh = gold[3] - gold[1]; + if (pw <= 0 || ph <= 0 || gw <= 0 || gh <= 0) + throw new ArgumentException("Boxes used for regression targets must have positive width and height."); + return new[] + { + (gold[0] + gw / 2 - (reference[0] + pw / 2)) / pw, + (gold[1] + gh / 2 - (reference[1] + ph / 2)) / ph, + Math.Log(gw / pw), + Math.Log(gh / ph) + }; + } + + private static void WriteDeltas(T[] destination, int offset, double[] reference, double[] gold) + { + var delta = EncodeDeltas(reference, gold); + for (int k = 0; k < 4; k++) destination[offset + k] = NumOps.FromDouble(delta[k]); + } + + /// Sum over selected rows of -log softmax(logits)[class], gathered to avoid 0 * -inf. + private static Tensor CrossEntropySum(Tensor logits, int[] rows, int[] classes, int width) + { + var engine = AiDotNetEngine.Current; + var logProbabilities = engine.Reshape(engine.TensorLogSoftmax(logits, 1), new[] { logits.Length }); + var entries = new int[rows.Length]; + for (int i = 0; i < rows.Length; i++) entries[i] = rows[i] * width + classes[i]; + return engine.TensorNegate(engine.ReduceSum(CvTensorOps.Select(logProbabilities, entries, 0), null)); + } + + /// Sum of smooth-L1 (Girshick 2015, Eq. 3): 0.5 x^2 where |x| < 1, otherwise |x| - 0.5. + /// + /// The branch is chosen per element from detached values and applied as a constant mask, so the derivative is + /// x inside the quadratic region and sign(x) outside, using only differentiable elementwise operations. + /// + private static Tensor SmoothL1Sum(Tensor predicted, Tensor target) + { + var engine = AiDotNetEngine.Current; + var difference = engine.TensorSubtract(predicted, target); + var values = difference.ToArray(); + var quadraticMask = new T[values.Length]; + var linearMask = new T[values.Length]; + for (int i = 0; i < values.Length; i++) + { + bool quadratic = Math.Abs(NumOps.ToDouble(values[i])) < 1; + quadraticMask[i] = quadratic ? NumOps.One : NumOps.Zero; + linearMask[i] = quadratic ? NumOps.Zero : NumOps.One; + } + var shape = difference.Shape.ToArray(); + var quadraticPart = engine.TensorMultiply(new Tensor(quadraticMask, shape), + engine.TensorMultiplyScalar(engine.TensorMultiply(difference, difference), NumOps.FromDouble(0.5))); + var linearPart = engine.TensorMultiply(new Tensor(linearMask, shape), + engine.TensorAddScalar(engine.TensorAbs(difference), NumOps.FromDouble(-0.5))); + return engine.ReduceSum(engine.TensorAdd(quadraticPart, linearPart), null); + } + + private static Tensor ZeroConnected(Tensor tensor) + { + var engine = AiDotNetEngine.Current; + return engine.TensorMultiplyScalar(engine.ReduceSum(tensor, null), NumOps.Zero); + } + + private static List Sample(List candidates, int limit, Random random) + { + if (limit <= 0) return new List(); + if (candidates.Count <= limit) return candidates; + // Partial Fisher-Yates: an unbiased sample without replacement. + for (int i = 0; i < limit; i++) + { + int j = i + random.Next(candidates.Count - i); + (candidates[i], candidates[j]) = (candidates[j], candidates[i]); + } + return candidates.GetRange(0, limit); + } + + private static double IoU(double[] a, double[] b) + { + double width = Math.Max(0, Math.Min(a[2], b[2]) - Math.Max(a[0], b[0])); + double height = Math.Max(0, Math.Min(a[3], b[3]) - Math.Max(a[1], b[1])); + double intersection = width * height; + double union = (a[2] - a[0]) * (a[3] - a[1]) + (b[2] - b[0]) * (b[3] - b[1]) - intersection; + return union > 0 ? intersection / union : 0; + } +} diff --git a/src/ComputerVision/Detection/Losses/TwoStageDetectionLossOptions.cs b/src/ComputerVision/Detection/Losses/TwoStageDetectionLossOptions.cs new file mode 100644 index 0000000000..635ded3b4c --- /dev/null +++ b/src/ComputerVision/Detection/Losses/TwoStageDetectionLossOptions.cs @@ -0,0 +1,110 @@ +namespace AiDotNet.ComputerVision.Detection.Losses; + +/// Sampling and weighting for training two-stage (R-CNN family) detectors. +/// +/// +/// Defaults follow the published recipes. Region proposal network (Ren et al. 2015, Sec. 3.1.2-3.1.3): an anchor +/// is positive when its IoU with an object exceeds 0.7 or it is an object's best anchor, negative below 0.3; +/// 256 anchors per image are sampled with up to half positive; the classification term is averaged over the +/// sample and the regression term is weighted by lambda = 10 and normalized by the number of anchor locations. +/// Region-of-interest head (Girshick 2015, Sec. 2.3): 64 RoIs per image, 25% from proposals with IoU of at least +/// 0.5, the rest from IoU in [0.1, 0.5), with smooth-L1 box regression weighted 1. Cascade R-CNN (Cai and +/// Vasconcelos 2018, Sec. 5.1) trains three stages at IoU thresholds 0.5, 0.6 and 0.7, each with that loss. +/// +/// For Beginners: a two-stage detector first proposes regions that might hold objects, then classifies +/// and refines them. These settings decide which proposals count as objects or background while training, how +/// many of each are used, and how strongly box positions are corrected. +/// +public sealed class TwoStageDetectionLossOptions +{ + /// IoU above which an anchor is a positive proposal example. + /// For Beginners: how closely an anchor must overlap an object to count as one. + public double RpnPositiveIoU { get; set; } = 0.7; + + /// IoU below which an anchor is a negative (background) proposal example. + /// For Beginners: anchors between the two thresholds are ignored while training. + public double RpnNegativeIoU { get; set; } = 0.3; + + /// Anchors sampled per image for the proposal loss. + /// For Beginners: using a fixed, balanced sample stops background anchors dominating. + public int RpnBatchSizePerImage { get; set; } = 256; + + /// Largest fraction of the anchor sample that may be positive. + /// For Beginners: 0.5 means at most one object anchor per background anchor. + public double RpnPositiveFraction { get; set; } = 0.5; + + /// Weight lambda of the proposal box-regression term. + /// For Beginners: balances box correction against object-versus-background scoring. + public double RpnRegressionWeight { get; set; } = 10.0; + + /// Regions of interest sampled per image for the detection-head loss. + /// For Beginners: how many proposals per image teach the final classifier. + public int RoiBatchSizePerImage { get; set; } = 64; + + /// Largest fraction of the RoI sample drawn from foreground proposals. + /// For Beginners: 0.25 keeps three background examples per object example. + public double RoiForegroundFraction { get; set; } = 0.25; + + /// Lower bound of the background IoU interval [low, foreground threshold). + /// For Beginners: proposals overlapping nothing at all are skipped as too easy. + public double RoiBackgroundIoULow { get; set; } = 0.1; + + /// Weight of the detection-head box-regression term. + /// For Beginners: balances box correction against class scoring in the final head. + public double RoiRegressionWeight { get; set; } = 1.0; + + /// Foreground IoU threshold of each detection stage; Faster R-CNN uses the first. + /// For Beginners: later cascade stages demand tighter boxes before calling them objects. + public double[] StageForegroundIoU { get; set; } = { 0.5, 0.6, 0.7 }; + + /// Weight of each detection stage's loss in the total objective. + /// For Beginners: Cascade R-CNN sums its stages equally by default. + public double[] StageLossWeights { get; set; } = { 1.0, 1.0, 1.0 }; + + internal TwoStageDetectionLossOptions Snapshot(int stages) + { + var copy = (TwoStageDetectionLossOptions)MemberwiseClone(); + copy.StageForegroundIoU = (double[])(StageForegroundIoU ?? throw new ArgumentNullException(nameof(StageForegroundIoU))).Clone(); + copy.StageLossWeights = (double[])(StageLossWeights ?? throw new ArgumentNullException(nameof(StageLossWeights))).Clone(); + copy.Validate(stages); + return copy; + } + + private void Validate(int stages) + { + RequireProbability(RpnPositiveIoU, nameof(RpnPositiveIoU)); + RequireProbability(RpnNegativeIoU, nameof(RpnNegativeIoU)); + if (RpnNegativeIoU > RpnPositiveIoU) + throw new ArgumentOutOfRangeException(nameof(RpnNegativeIoU), "The negative IoU threshold cannot exceed the positive one."); + if (RpnBatchSizePerImage < 1) throw new ArgumentOutOfRangeException(nameof(RpnBatchSizePerImage)); + RequireProbability(RpnPositiveFraction, nameof(RpnPositiveFraction)); + RequireNonnegative(RpnRegressionWeight, nameof(RpnRegressionWeight)); + if (RoiBatchSizePerImage < 1) throw new ArgumentOutOfRangeException(nameof(RoiBatchSizePerImage)); + RequireProbability(RoiForegroundFraction, nameof(RoiForegroundFraction)); + RequireProbability(RoiBackgroundIoULow, nameof(RoiBackgroundIoULow)); + RequireNonnegative(RoiRegressionWeight, nameof(RoiRegressionWeight)); + if (StageForegroundIoU.Length < stages) + throw new ArgumentException($"StageForegroundIoU needs one threshold for each of the {stages} detection stages.", nameof(StageForegroundIoU)); + if (StageLossWeights.Length < stages) + throw new ArgumentException($"StageLossWeights needs one weight for each of the {stages} detection stages.", nameof(StageLossWeights)); + for (int stage = 0; stage < stages; stage++) + { + RequireProbability(StageForegroundIoU[stage], nameof(StageForegroundIoU)); + if (RoiBackgroundIoULow > StageForegroundIoU[stage]) + throw new ArgumentOutOfRangeException(nameof(RoiBackgroundIoULow), "The background interval must end at or above its lower bound."); + RequireNonnegative(StageLossWeights[stage], nameof(StageLossWeights)); + } + } + + private static void RequireProbability(double value, string name) + { + if (double.IsNaN(value) || value < 0 || value > 1) + throw new ArgumentOutOfRangeException(name, "Thresholds and fractions must be in [0, 1]."); + } + + private static void RequireNonnegative(double value, string name) + { + if (double.IsNaN(value) || double.IsInfinity(value) || value < 0) + throw new ArgumentOutOfRangeException(name, "Loss weights must be finite and nonnegative."); + } +} diff --git a/src/ComputerVision/Detection/Necks/BiFPN.cs b/src/ComputerVision/Detection/Necks/BiFPN.cs index f8a2691710..509879c43c 100644 --- a/src/ComputerVision/Detection/Necks/BiFPN.cs +++ b/src/ComputerVision/Detection/Necks/BiFPN.cs @@ -53,7 +53,11 @@ protected override void RegisterComponents() () => _topDownConvWeights, () => _topDownConvBiases, () => _bottomUpConvWeights, - () => _bottomUpConvBiases)); + () => _bottomUpConvBiases, + // The learnable fast-normalized-fusion weights. They sit in nested lists, and used to be + // left out of this declaration entirely - so they were never saved, cloned or trained. + () => _topDownFusionWeights.SelectMany(level => level).ToList(), + () => _bottomUpFusionWeights.SelectMany(level => level).ToList())); private readonly int _outputChannels; private readonly int[] _inputChannels; private readonly int _numLevels; @@ -298,36 +302,46 @@ private Tensor FastNormalizedFusion(List> inputs, List> w throw new ArgumentException("Number of inputs must match number of weights"); } - // Calculate normalized weights using ReLU - var normalizedWeights = new double[weights.Count]; - double weightSum = _epsilon; + // Fast normalized fusion (EfficientDet): out = sum_i relu(w_i) / (eps + sum_j relu(w_j)) * x_i. + // The fusion weights are LEARNABLE; this used to read them out as doubles, so they received + // no gradient and never moved from their initial values. Every step is an engine op now. + var relu = new Tensor[weights.Count]; + relu[0] = Engine.ReLU(weights[0]); + var denominator = Engine.TensorAddScalar(relu[0], NumOps.FromDouble(_epsilon)); + for (int i = 1; i < weights.Count; i++) + { + relu[i] = Engine.ReLU(weights[i]); + denominator = Engine.TensorAdd(denominator, relu[i]); + } - for (int i = 0; i < weights.Count; i++) + var dims = new int[inputs[0].Shape.Length]; + for (int d = 0; d < dims.Length; d++) { - double w = Math.Max(0, NumOps.ToDouble(weights[i][0])); // ReLU - normalizedWeights[i] = w; - weightSum += w; + dims[d] = inputs[0].Shape[d]; } - // Normalize - for (int i = 0; i < normalizedWeights.Length; i++) + Tensor? result = null; + for (int i = 0; i < inputs.Count; i++) { - normalizedWeights[i] /= weightSum; + var coefficient = Engine.TensorDivide(relu[i], denominator); + var scaled = Engine.TensorMultiply( + inputs[i], + Engine.TensorBroadcastTo(Engine.Reshape(coefficient, OnesShape(dims.Length)), dims)); + result = result is null ? scaled : Engine.TensorAdd(result, scaled); } - // Weighted sum - var result = new Tensor(inputs[0].Shape.ToArray()); - for (int i = 0; i < result.Length; i++) + return result ?? throw new ArgumentException("At least one input is required.", nameof(inputs)); + } + + private static int[] OnesShape(int rank) + { + var shape = new int[rank]; + for (int d = 0; d < rank; d++) { - double sum = 0; - for (int j = 0; j < inputs.Count; j++) - { - sum += normalizedWeights[j] * NumOps.ToDouble(inputs[j][i]); - } - result[i] = NumOps.FromDouble(sum); + shape[d] = 1; } - return result; + return shape; } /// @@ -507,46 +521,19 @@ private void ReadTensor(BinaryReader reader, Tensor tensor) } private Tensor ResizeToMatch(Tensor source, Tensor target) - { - int batch = source.Shape[0]; - int channels = source.Shape[1]; - int targetH = target.Shape[2]; - int targetW = target.Shape[3]; - int sourceH = source.Shape[2]; - int sourceW = source.Shape[3]; - - var result = new Tensor(new[] { batch, channels, targetH, targetW }); - - for (int n = 0; n < batch; n++) - { - for (int c = 0; c < channels; c++) - { - for (int h = 0; h < targetH; h++) - { - for (int w = 0; w < targetW; w++) - { - int srcH = Math.Min(h * sourceH / targetH, sourceH - 1); - int srcW = Math.Min(w * sourceW / targetW, sourceW - 1); - result[n, c, h, w] = source[n, c, srcH, srcW]; - } - } - } - } - - return result; - } + // Nearest neighbour, src = min(dst * in / out, in - 1), through tape-visible index gathers. + => CvTensorOps.ResizeNearest(source, target.Shape[2], target.Shape[3]); - private Tensor ApplySwish(Tensor x) - { - var result = new Tensor(x._shape); - for (int i = 0; i < x.Length; i++) - { - double val = NumOps.ToDouble(x[i]); - double swish = val * (1.0 / (1.0 + Math.Exp(-val))); - result[i] = NumOps.FromDouble(swish); - } - return result; - } + /// + /// Elementwise Swish, delegated to the engine. + /// + /// + /// This was a scalar loop that read each element out to double and wrote a fresh + /// tensor. Arithmetically identical, but it severed the autodiff tape: the gradient chain + /// stopped here, so every trainable layer UPSTREAM of this call received no gradient and + /// silently never trained. The engine op records itself on the tape. + /// + private Tensor ApplySwish(Tensor x) => Engine.Swish(x); /// /// Copies every element from into in diff --git a/src/ComputerVision/Detection/Necks/FPN.cs b/src/ComputerVision/Detection/Necks/FPN.cs index c9f8ede689..1b3c56d1b8 100644 --- a/src/ComputerVision/Detection/Necks/FPN.cs +++ b/src/ComputerVision/Detection/Necks/FPN.cs @@ -143,15 +143,20 @@ public override List> Forward(List> features) // Top-down pathway with lateral connections // Start from the deepest level (smallest spatial resolution) + Tensor? deeperMerged = null; for (int i = _numLevels - 1; i >= 0; i--) { Tensor current = lateralFeatures[i]; - // Add upsampled feature from deeper level (if not the deepest) - if (i < _numLevels - 1) + // Top-down input is the MERGED map of the next deeper level (M_{i+1} in Lin et al. 2017), + // before its smoothing conv. This used to read outputFeatures[^1] - but the list is built with + // Insert(0, ...), so [^1] is the DEEPEST level, not the next one: every level took its + // top-down signal from the coarsest map, the second-deepest level fed nothing, and a + // detector reading one pyramid level (Faster/Cascade R-CNN use P3) left the other levels' + // convs without any gradient. + if (deeperMerged is not null) { - // Get the output from the next deeper level and upsample - var upsampled = Upsample2x(outputFeatures[^1]); + var upsampled = Upsample2x(deeperMerged); // Resize if dimensions don't match exactly (due to odd sizes) if (upsampled.Shape[2] != current.Shape[2] || upsampled.Shape[3] != current.Shape[3]) @@ -162,6 +167,8 @@ public override List> Forward(List> features) current = Add(current, upsampled); } + deeperMerged = current; + // Apply output convolution var output = Conv1x1(current, _outputWeights[i], _outputBiases[i]); output = ApplyReLU(output); @@ -278,46 +285,19 @@ private void ReadTensor(BinaryReader reader, Tensor tensor) } private Tensor ResizeToMatch(Tensor source, Tensor target) - { - int batch = source.Shape[0]; - int channels = source.Shape[1]; - int targetH = target.Shape[2]; - int targetW = target.Shape[3]; - int sourceH = source.Shape[2]; - int sourceW = source.Shape[3]; - - var result = new Tensor(new[] { batch, channels, targetH, targetW }); - - for (int n = 0; n < batch; n++) - { - for (int c = 0; c < channels; c++) - { - for (int h = 0; h < targetH; h++) - { - for (int w = 0; w < targetW; w++) - { - // Nearest neighbor interpolation - int srcH = Math.Min(h * sourceH / targetH, sourceH - 1); - int srcW = Math.Min(w * sourceW / targetW, sourceW - 1); - result[n, c, h, w] = source[n, c, srcH, srcW]; - } - } - } - } + // Nearest neighbour, src = min(dst * in / out, in - 1), through tape-visible index gathers. + => CvTensorOps.ResizeNearest(source, target.Shape[2], target.Shape[3]); - return result; - } - - private Tensor ApplyReLU(Tensor x) - { - var result = new Tensor(x._shape); - for (int i = 0; i < x.Length; i++) - { - double val = NumOps.ToDouble(x[i]); - result[i] = NumOps.FromDouble(Math.Max(0, val)); - } - return result; - } + /// + /// Elementwise ReLU, delegated to the engine. + /// + /// + /// This was a scalar loop that read each element out to double and wrote a fresh + /// tensor. Arithmetically identical, but it severed the autodiff tape: the gradient chain + /// stopped here, so every trainable layer UPSTREAM of this call received no gradient and + /// silently never trained. The engine op records itself on the tape. + /// + private Tensor ApplyReLU(Tensor x) => Engine.ReLU(x); /// /// Copies every element from into in diff --git a/src/ComputerVision/Detection/Necks/NeckBase.cs b/src/ComputerVision/Detection/Necks/NeckBase.cs index 765cc2aca3..10ca4b41c2 100644 --- a/src/ComputerVision/Detection/Necks/NeckBase.cs +++ b/src/ComputerVision/Detection/Necks/NeckBase.cs @@ -140,33 +140,7 @@ protected void ValidateFeatures(List> features, int[] expectedInputCha /// /// Input feature map. /// Upsampled feature map. - protected Tensor Upsample2x(Tensor input) - { - int batch = input.Shape[0]; - int channels = input.Shape[1]; - int height = input.Shape[2]; - int width = input.Shape[3]; - - var output = new Tensor(new[] { batch, channels, height * 2, width * 2 }); - - for (int b = 0; b < batch; b++) - { - for (int c = 0; c < channels; c++) - { - for (int h = 0; h < height * 2; h++) - { - for (int w = 0; w < width * 2; w++) - { - int srcH = h / 2; - int srcW = w / 2; - output[b, c, h, w] = input[b, c, srcH, srcW]; - } - } - } - } - - return output; - } + protected Tensor Upsample2x(Tensor input) => CvTensorOps.Upsample2xNearest(input); /// /// Downsample a feature map by a factor of 2 using max pooling. @@ -174,59 +148,10 @@ protected Tensor Upsample2x(Tensor input) /// Input feature map. /// Downsampled feature map. protected Tensor Downsample2x(Tensor input) - { - int batch = input.Shape[0]; - int channels = input.Shape[1]; - int height = input.Shape[2]; - int width = input.Shape[3]; - - // Use ceiling division so a 5x5 input produces a 3x3 output (matching the - // dynamic-spatial pyramid alignment used elsewhere). Floor division would - // silently drop the last row/column for odd-sized features and break - // multi-scale detection heads at non-power-of-two input sizes. - int outHeight = (height + 1) / 2; - int outWidth = (width + 1) / 2; - - var output = new Tensor(new[] { batch, channels, outHeight, outWidth }); - - for (int b = 0; b < batch; b++) - { - for (int c = 0; c < channels; c++) - { - for (int h = 0; h < outHeight; h++) - { - for (int w = 0; w < outWidth; w++) - { - int srcRow = h * 2; - int srcCol = w * 2; - // Max pooling 2x2 with bounds-checked sampling: the right/bottom - // edge of an odd-sized window covers fewer than 4 source cells, - // so we take the max only over the in-bounds entries. - T maxVal = input[b, c, srcRow, srcCol]; - if (srcCol + 1 < width) - { - T v = input[b, c, srcRow, srcCol + 1]; - if (NumOps.GreaterThan(v, maxVal)) maxVal = v; - } - if (srcRow + 1 < height) - { - T v = input[b, c, srcRow + 1, srcCol]; - if (NumOps.GreaterThan(v, maxVal)) maxVal = v; - } - if (srcRow + 1 < height && srcCol + 1 < width) - { - T v = input[b, c, srcRow + 1, srcCol + 1]; - if (NumOps.GreaterThan(v, maxVal)) maxVal = v; - } - - output[b, c, h, w] = maxVal; - } - } - } - } - - return output; - } + // 2x2 max pooling in CEIL mode, so a 5x5 input produces a 3x3 output (matching the + // dynamic-spatial pyramid alignment used elsewhere) and the partial right/bottom window takes + // its max over the in-bounds cells only. + => CvTensorOps.MaxPool2x2Ceil(input); /// /// Applies a 1x1 convolution to change the number of channels. @@ -243,54 +168,26 @@ protected Tensor Conv1x1(Tensor input, Tensor weights, Tensor? bias int width = input.Shape[3]; int outChannels = weights.Shape[0]; - // 1x1 conv = matmul: reshape [B,C_in,H,W] -> [B*H*W, C_in] @ W^T -> [B*H*W, C_out] - // Transpose input from NCHW to NHWC: [B, C_in, H, W] -> permute to get [B*H*W, C_in] - int spatialSize = height * width; - var inputFlat = new Tensor(new[] { batch * spatialSize, inChannels }); - for (int b = 0; b < batch; b++) - { - for (int h = 0; h < height; h++) - { - for (int w = 0; w < width; w++) - { - int spatialIdx = b * spatialSize + h * width + w; - for (int ic = 0; ic < inChannels; ic++) - { - inputFlat[spatialIdx, ic] = input[b, ic, h, w]; - } - } - } - } + // A 1x1 convolution is a matmul over the channel axis: NCHW -> [B*H*W, C_in] @ W^T -> NCHW. + // Every reshape and transpose here is an ENGINE op. Tensor.Reshape and .Transpose bypass + // the autodiff tape, so using them on the input severed the gradient to the backbone, and + // using them on the WEIGHTS meant the neck's own weights never received a gradient either. + var inputFlat = Engine.Reshape( + Engine.TensorPermute(input, new[] { 0, 2, 3, 1 }), + new[] { batch * height * width, inChannels }); - // MatMul: [B*H*W, C_in] @ [C_out, C_in]^T = [B*H*W, C_out] - var weightsT = weights.Transpose(new[] { 1, 0 }); - var outputFlat = Engine.TensorMatMul(inputFlat, weightsT); + var outputFlat = Engine.TensorMatMul(inputFlat, Engine.TensorPermute(weights, new[] { 1, 0 })); - // Add bias if present if (bias is not null) { - var biasBroadcast = bias.Reshape(1, outChannels); - outputFlat = Engine.TensorAdd(outputFlat, biasBroadcast); + outputFlat = Engine.TensorAdd( + outputFlat, + Engine.TensorBroadcastTo(Engine.Reshape(bias, new[] { 1, outChannels }), new[] { batch * height * width, outChannels })); } - // Reshape back to NCHW: [B*H*W, C_out] -> [B, C_out, H, W] - var output = new Tensor(new[] { batch, outChannels, height, width }); - for (int b = 0; b < batch; b++) - { - for (int h = 0; h < height; h++) - { - for (int w = 0; w < width; w++) - { - int spatialIdx = b * spatialSize + h * width + w; - for (int oc = 0; oc < outChannels; oc++) - { - output[b, oc, h, w] = outputFlat[spatialIdx, oc]; - } - } - } - } - - return output; + return Engine.TensorPermute( + Engine.Reshape(outputFlat, new[] { batch, height, width, outChannels }), + new[] { 0, 3, 1, 2 }); } /// @@ -306,12 +203,10 @@ protected Tensor Add(Tensor a, Tensor b) throw new ArgumentException("Feature maps must have the same shape for addition"); } - var output = new Tensor(a._shape); - for (int i = 0; i < a.Length; i++) - { - output[i] = NumOps.Add(a[i], b[i]); - } - return output; + // Engine op rather than a scalar loop so the tape records the addition: FPN's top-down + // pathway adds the upsampled higher level into the lateral one, and a severed add there + // cuts every level below it out of the gradient. + return Engine.TensorAdd(a, b); } #region ModelBase Overrides diff --git a/src/ComputerVision/Detection/Necks/PANet.cs b/src/ComputerVision/Detection/Necks/PANet.cs index ba968c0d9e..92139e0df4 100644 --- a/src/ComputerVision/Detection/Necks/PANet.cs +++ b/src/ComputerVision/Detection/Necks/PANet.cs @@ -182,13 +182,20 @@ public override List> Forward(List> features) } // Top-down fusion + Tensor? deeperMerged = null; for (int i = _numLevels - 1; i >= 0; i--) { Tensor current = lateralFeatures[i]; - if (i < _numLevels - 1) + // Top-down input is the MERGED map of the next deeper level (M_{i+1} in Lin et al. 2017), + // before its smoothing conv. This used to read topDownFeatures[^1] - but the list is built with + // Insert(0, ...), so [^1] is the DEEPEST level, not the next one: every level took its + // top-down signal from the coarsest map, the second-deepest level fed nothing, and a + // detector reading one pyramid level (Faster/Cascade R-CNN use P3) left the other levels' + // convs without any gradient. + if (deeperMerged is not null) { - var upsampled = Upsample2x(topDownFeatures[^1]); + var upsampled = Upsample2x(deeperMerged); if (upsampled.Shape[2] != current.Shape[2] || upsampled.Shape[3] != current.Shape[3]) { upsampled = ResizeToMatch(upsampled, current); @@ -196,6 +203,8 @@ public override List> Forward(List> features) current = Add(current, upsampled); } + deeperMerged = current; + var output = Conv1x1(current, _topDownWeights[i], _topDownBiases[i]); output = ApplyReLU(output); topDownFeatures.Insert(0, output); @@ -373,45 +382,19 @@ private void ReadTensor(BinaryReader reader, Tensor tensor) } private Tensor ResizeToMatch(Tensor source, Tensor target) - { - int batch = source.Shape[0]; - int channels = source.Shape[1]; - int targetH = target.Shape[2]; - int targetW = target.Shape[3]; - int sourceH = source.Shape[2]; - int sourceW = source.Shape[3]; - - var result = new Tensor(new[] { batch, channels, targetH, targetW }); - - for (int n = 0; n < batch; n++) - { - for (int c = 0; c < channels; c++) - { - for (int h = 0; h < targetH; h++) - { - for (int w = 0; w < targetW; w++) - { - int srcH = Math.Min(h * sourceH / targetH, sourceH - 1); - int srcW = Math.Min(w * sourceW / targetW, sourceW - 1); - result[n, c, h, w] = source[n, c, srcH, srcW]; - } - } - } - } + // Nearest neighbour, src = min(dst * in / out, in - 1), through tape-visible index gathers. + => CvTensorOps.ResizeNearest(source, target.Shape[2], target.Shape[3]); - return result; - } - - private Tensor ApplyReLU(Tensor x) - { - var result = new Tensor(x._shape); - for (int i = 0; i < x.Length; i++) - { - double val = NumOps.ToDouble(x[i]); - result[i] = NumOps.FromDouble(Math.Max(0, val)); - } - return result; - } + /// + /// Elementwise ReLU, delegated to the engine. + /// + /// + /// This was a scalar loop that read each element out to double and wrote a fresh + /// tensor. Arithmetically identical, but it severed the autodiff tape: the gradient chain + /// stopped here, so every trainable layer UPSTREAM of this call received no gradient and + /// silently never trained. The engine op records itself on the tape. + /// + private Tensor ApplyReLU(Tensor x) => Engine.ReLU(x); /// /// Copies every element from into in diff --git a/src/ComputerVision/Detection/ObjectDetection/DETR/DETR.cs b/src/ComputerVision/Detection/ObjectDetection/DETR/DETR.cs index 71176c2a86..58773df5c6 100644 --- a/src/ComputerVision/Detection/ObjectDetection/DETR/DETR.cs +++ b/src/ComputerVision/Detection/ObjectDetection/DETR/DETR.cs @@ -3,6 +3,7 @@ using System.Threading.Tasks; using AiDotNet.Augmentation.Image; using AiDotNet.ComputerVision.Detection.Backbones; +using AiDotNet.ComputerVision.Detection.Losses; using AiDotNet.ComputerVision.Detection.PostProcessing; using AiDotNet.Attributes; using AiDotNet.Enums; @@ -41,13 +42,16 @@ namespace AiDotNet.ComputerVision.Detection.ObjectDetection.DETR; "https://arxiv.org/abs/2005.12872", Year = 2020, Authors = "Nicolas Carion, Francisco Massa, Gabriel Synnaeve, Nicolas Usunier, Alexander Kirillov, Sergey Zagoruyko")] -public class DETR : ObjectDetectorBase +public partial class DETR : ObjectDetectorBase, IDetectionTrainingModel { private readonly DETREncoder _encoder; private readonly DETRDecoder _decoder; private readonly Conv2D _inputProj; private readonly int _hiddenDim; private readonly NMS _nms; + private readonly DETRSetLoss _detectionLoss; + private readonly int _trainingClassCount; + private readonly int _trainingQueryCount; /// public override string Name => $"DETR-{Options.Size}"; @@ -60,6 +64,12 @@ public DETR(ObjectDetectionOptions options) : base(options) { var (hiddenDim, numHeads, numEncoderLayers, numDecoderLayers, numQueries) = GetSizeConfig(options.Size); _hiddenDim = hiddenDim; + _trainingClassCount = options.NumClasses; + _trainingQueryCount = numQueries; + var lossOptions = options.SetPredictionLoss ?? DetrSetLossOptions.ForDetr(); + if (lossOptions.ClassificationLoss != SetPredictionClassificationLoss.SoftmaxCrossEntropy) + throw new ArgumentException("DETR's class head is a softmax with a no-object class; use a softmax cross-entropy set loss.", nameof(options)); + _detectionLoss = new DETRSetLoss(checked(options.NumClasses + 1), lossOptions); // Initialize backbone (ResNet-50 by default) Backbone = new ResNet(ResNetVariant.ResNet50); @@ -90,6 +100,33 @@ public DETR(ObjectDetectionOptions options) : base(options) _ => (256, 8, 6, 6, 100) }; + /// Trains the final DETR heads with exact assignment, no-object CE, L1 and GIoU. + /// + /// Inputs are model-ready NCHW tensors, as for Predict; this method does not implicitly resize + /// or normalize them. Targets use normalized center-format boxes. An image with more targets + /// than this model has queries is rejected before initialization or update. Intermediate decoder + /// outputs are not exposed by this architecture, so no auxiliary decoder objective is claimed. + /// + public void TrainDetections(Tensor input, DetectionTrainingBatch targets) + { + if (input is null) throw new ArgumentNullException(nameof(input)); + if (targets is null) throw new ArgumentNullException(nameof(targets)); + if (input.Rank != 4 || input.Shape[0] <= 0 || input.Shape[1] != 3 || input.Shape[2] <= 0 || input.Shape[3] <= 0) + throw new ArgumentException("DETR training requires a nonempty NCHW three-channel image batch.", nameof(input)); + targets.ValidateForModel(input.Shape[0], _trainingClassCount, _trainingQueryCount); + TrainWithTargets(input, targets, ComputeDetectionLoss); + } + + private Tensor ComputeDetectionLoss(List> heads, DetectionTrainingBatch targets) + { + if (heads.Count != 2) + throw new InvalidOperationException("DETR training requires the actual final class and box heads."); + // Forward/Predict intentionally expose raw box logits; DecodeOutputs applies sigmoid for + // inference. Apply that same transformation on the tape for semantic training only, keeping + // both the normalized-box loss contract and raw-output regression API unchanged. + return _detectionLoss.ComputeTapeLoss(heads[0], Engine.Sigmoid(heads[1]), targets); + } + /// public override DetectionResult Detect(Tensor image, double confidenceThreshold, double nmsThreshold) { @@ -187,7 +224,7 @@ protected override List> PostProcess( // Note: DETR is designed to not need NMS, but we apply it for safety // with a very high IoU threshold - var nmsResults = _nms.Apply(candidateDetections, Math.Max(0.9, nmsThreshold)); + var nmsResults = _nms.Apply(candidateDetections, EffectiveNmsThreshold(nmsThreshold)); // Limit to max detections if (nmsResults.Count > Options.MaxDetections) @@ -287,33 +324,7 @@ public override void SaveWeights(string path) _decoder.WriteParameters(writer); } - private Tensor FlattenForTransformer(Tensor x) - { - int batch = x.Shape[0]; - int channels = x.Shape[1]; - int height = x.Shape[2]; - int width = x.Shape[3]; - int seqLen = height * width; - - var result = new Tensor(new[] { batch, seqLen, channels }); - - for (int b = 0; b < batch; b++) - { - for (int h = 0; h < height; h++) - { - for (int w = 0; w < width; w++) - { - int seqIdx = h * width + w; - for (int c = 0; c < channels; c++) - { - result[b, seqIdx, c] = x[b, c, h, w]; - } - } - } - } - - return result; - } + private Tensor FlattenForTransformer(Tensor x) => CvTensorOps.FlattenSpatial(x); private Tensor GeneratePositionalEncoding(int[] shape) { @@ -338,12 +349,19 @@ private Tensor GeneratePositionalEncoding(int[] shape) return encoding; } + + /// + /// + /// One query per object means duplicates are rare, so NMS runs only as a safety net at IoU 0.9 + /// (or the requested value, if that is higher) instead of the caller's threshold. + /// + public override double EffectiveNmsThreshold(double requested) => Math.Max(0.9, requested); } /// /// Transformer encoder for DETR. /// -internal class DETREncoder +internal class DETREncoder : CvParameterModule { private readonly INumericOperations _numOps; private readonly int _hiddenDim; @@ -372,19 +390,7 @@ public DETREncoder(int hiddenDim, int numHeads, int numLayers) public Tensor Forward(Tensor x, Tensor posEncoding) { - // Create a copy of input to avoid mutating the original tensor - var output = new Tensor(x._shape); - for (int i = 0; i < x.Length; i++) - { - output[i] = x[i]; - } - - // Add positional encoding - for (int i = 0; i < x.Length; i++) - { - output[i] = _numOps.Add(output[i], posEncoding[i]); - } - + var output = AiDotNetEngine.Current.TensorAdd(x, posEncoding); foreach (var layer in _layers) { output = layer.Forward(output); @@ -438,12 +444,18 @@ public void ReadParameters(BinaryReader reader) layer.ReadParameters(reader); } } + + /// + protected override IEnumerable?> ParameterChildren() + { + foreach (var child in _layers) yield return child; + } } /// /// Single encoder layer in DETR. /// -internal class EncoderLayer +internal class EncoderLayer : CvParameterModule { private readonly INumericOperations _numOps; private readonly MultiHeadSelfAttention _selfAttn; @@ -531,53 +543,21 @@ public void ReadParameters(BinaryReader reader) } private Tensor ApplyFFN(Tensor x) - { - int batch = x.Shape[0]; - int seqLen = x.Shape[1]; - int ffnDim = _ffn1.OutputSize; - - var result = new Tensor(x._shape); - - for (int b = 0; b < batch; b++) - { - for (int s = 0; s < seqLen; s++) - { - var feat = new Tensor(new[] { 1, _hiddenDim }); - for (int d = 0; d < _hiddenDim; d++) - { - feat[0, d] = x[b, s, d]; - } - - // FFN1 with GELU - var h = _ffn1.Forward(feat); - for (int d = 0; d < ffnDim; d++) - { - double val = _numOps.ToDouble(h[0, d]); - h[0, d] = _numOps.FromDouble(GELU(val)); - } - - // FFN2 - var output = _ffn2.Forward(h); - - for (int d = 0; d < _hiddenDim; d++) - { - result[b, s, d] = output[0, d]; - } - } - } - - return result; - } + => _ffn2.ForwardTokens(AiDotNetEngine.Current.GELU(_ffn1.ForwardTokens(x))); private Tensor AddTensors(Tensor a, Tensor b) { return AiDotNetEngine.Current.TensorAdd(a, b); } - private static double GELU(double x) + /// + protected override IEnumerable?> ParameterChildren() { - double c = Math.Sqrt(2.0 / Math.PI); - return 0.5 * x * (1.0 + Math.Tanh(c * (x + 0.044715 * x * x * x))); + yield return _selfAttn; + yield return _ffn1; + yield return _ffn2; + yield return _norm1; + yield return _norm2; } } @@ -585,7 +565,7 @@ private static double GELU(double x) /// Layer normalization with learnable affine parameters (gamma and beta). /// /// The numeric type used for calculations. -internal class LayerNorm +internal class LayerNorm : CvParameterModule { private readonly INumericOperations _numOps; private readonly int _hiddenDim; @@ -620,49 +600,7 @@ public LayerNorm(int hiddenDim, double eps = 1e-6) } } - public Tensor Forward(Tensor x) - { - int batch = x.Shape[0]; - int seqLen = x.Shape[1]; - int hiddenDim = x.Shape[2]; - - var result = new Tensor(x._shape); - - for (int b = 0; b < batch; b++) - { - for (int s = 0; s < seqLen; s++) - { - // Compute mean - double mean = 0; - for (int d = 0; d < hiddenDim; d++) - { - mean += _numOps.ToDouble(x[b, s, d]); - } - mean /= hiddenDim; - - // Compute variance - double variance = 0; - for (int d = 0; d < hiddenDim; d++) - { - double diff = _numOps.ToDouble(x[b, s, d]) - mean; - variance += diff * diff; - } - variance /= hiddenDim; - - // Normalize and apply affine transformation: gamma * (x - mean) / std + beta - double std = Math.Sqrt(variance + _eps); - for (int d = 0; d < hiddenDim; d++) - { - double normalized = (_numOps.ToDouble(x[b, s, d]) - mean) / std; - double gamma = _numOps.ToDouble(_gamma[d]); - double beta = _numOps.ToDouble(_beta[d]); - result[b, s, d] = _numOps.FromDouble(gamma * normalized + beta); - } - } - } - - return result; - } + public Tensor Forward(Tensor x) => CvTensorOps.LayerNormLastAxis(x, _gamma, _beta, _eps); public long GetParameterCount() { @@ -720,5 +658,15 @@ public void ReadParameters(BinaryReader reader) _beta[i] = _numOps.FromDouble(reader.ReadDouble()); } } + + /// + protected override IEnumerable?> ParameterChildren() => Array.Empty?>(); + + /// + protected override IEnumerable> OwnParameterTensors() + { + yield return _gamma; + yield return _beta; + } } diff --git a/src/ComputerVision/Detection/ObjectDetection/DETR/DETRDecoder.cs b/src/ComputerVision/Detection/ObjectDetection/DETR/DETRDecoder.cs index 8da82ceab5..cc777c5f7c 100644 --- a/src/ComputerVision/Detection/ObjectDetection/DETR/DETRDecoder.cs +++ b/src/ComputerVision/Detection/ObjectDetection/DETR/DETRDecoder.cs @@ -21,7 +21,7 @@ namespace AiDotNet.ComputerVision.Detection.ObjectDetection.DETR; /// - FFN (feed-forward network) for each query /// /// -internal partial class DETRDecoder +internal partial class DETRDecoder : CvParameterModule { private readonly INumericOperations _numOps; private readonly int _numLayers; @@ -282,102 +282,40 @@ private Tensor InitializeQueryEmbeddings(int numQueries, int hiddenDim) private Tensor ExpandQueriesForBatch(Tensor queries, int batch) { + // Broadcast rather than copy: the learnable query embeddings must stay on the tape. int numQueries = queries.Shape[0]; int hiddenDim = queries.Shape[1]; - - var expanded = new Tensor(new[] { batch, numQueries, hiddenDim }); - - for (int b = 0; b < batch; b++) - { - for (int q = 0; q < numQueries; q++) - { - for (int d = 0; d < hiddenDim; d++) - { - expanded[b, q, d] = queries[q, d]; - } - } - } - - return expanded; + return AiDotNetEngine.Current.TensorBroadcastTo(AiDotNetEngine.Current.Reshape(queries, new[] { 1, numQueries, hiddenDim }), new[] { batch, numQueries, hiddenDim }); } - private Tensor ApplyClassHead(Tensor output) - { - int batch = output.Shape[0]; - int numQueries = output.Shape[1]; - int hiddenDim = output.Shape[2]; - int numClasses = _classHead.OutputSize; - - var result = new Tensor(new[] { batch, numQueries, numClasses }); - - for (int b = 0; b < batch; b++) - { - for (int q = 0; q < numQueries; q++) - { - // Extract query features - var queryFeat = new Tensor(new[] { 1, hiddenDim }); - for (int d = 0; d < hiddenDim; d++) - { - queryFeat[0, d] = output[b, q, d]; - } - - // Apply class head - var classOut = _classHead.Forward(queryFeat); + private Tensor ApplyClassHead(Tensor output) => _classHead.ForwardTokens(output); - // Copy to result - for (int c = 0; c < numClasses; c++) - { - result[b, q, c] = classOut[0, c]; - } - } - } + private Tensor ApplyBoxHead(Tensor output) => _boxHead.ForwardTokens(output); - return result; + private static double Sigmoid(double x) + { + return 1.0 / (1.0 + Math.Exp(-x)); } - private Tensor ApplyBoxHead(Tensor output) + /// + protected override IEnumerable?> ParameterChildren() { - int batch = output.Shape[0]; - int numQueries = output.Shape[1]; - int hiddenDim = output.Shape[2]; - - var result = new Tensor(new[] { batch, numQueries, 4 }); - - for (int b = 0; b < batch; b++) - { - for (int q = 0; q < numQueries; q++) - { - // Extract query features - var queryFeat = new Tensor(new[] { 1, hiddenDim }); - for (int d = 0; d < hiddenDim; d++) - { - queryFeat[0, d] = output[b, q, d]; - } - - // Apply box head - var boxOut = _boxHead.Forward(queryFeat); - - // Copy to result - for (int i = 0; i < 4; i++) - { - result[b, q, i] = boxOut[0, i]; - } - } - } - - return result; + foreach (var child in _layers) yield return child; + yield return _classHead; + yield return _boxHead; } - private static double Sigmoid(double x) + /// + protected override IEnumerable> OwnParameterTensors() { - return 1.0 / (1.0 + Math.Exp(-x)); + yield return _queryEmbed; } } /// /// Single decoder layer in DETR. /// -internal class DecoderLayer +internal class DecoderLayer : CvParameterModule { private readonly INumericOperations _numOps; private readonly int _hiddenDim; @@ -479,63 +417,30 @@ public void ReadParameters(BinaryReader reader) } private Tensor ApplyFFN(Tensor x) - { - int batch = x.Shape[0]; - int seqLen = x.Shape[1]; - int hiddenDim = x.Shape[2]; - int ffnDim = _ffn1.OutputSize; - - var result = new Tensor(x._shape); - - for (int b = 0; b < batch; b++) - { - for (int s = 0; s < seqLen; s++) - { - // Extract features - var feat = new Tensor(new[] { 1, hiddenDim }); - for (int d = 0; d < hiddenDim; d++) - { - feat[0, d] = x[b, s, d]; - } - - // FFN1 with GELU - var h = _ffn1.Forward(feat); - for (int d = 0; d < ffnDim; d++) - { - double val = _numOps.ToDouble(h[0, d]); - h[0, d] = _numOps.FromDouble(GELU(val)); - } - - // FFN2 - var output = _ffn2.Forward(h); - - // Copy to result - for (int d = 0; d < hiddenDim; d++) - { - result[b, s, d] = output[0, d]; - } - } - } - - return result; - } + => _ffn2.ForwardTokens(AiDotNetEngine.Current.GELU(_ffn1.ForwardTokens(x))); private Tensor AddTensors(Tensor a, Tensor b) { return AiDotNetEngine.Current.TensorAdd(a, b); } - private static double GELU(double x) + + /// + protected override IEnumerable?> ParameterChildren() { - // Approximate GELU: 0.5 * x * (1 + tanh(sqrt(2/pi) * (x + 0.044715 * x^3))) - double c = Math.Sqrt(2.0 / Math.PI); - return 0.5 * x * (1.0 + Math.Tanh(c * (x + 0.044715 * x * x * x))); + yield return _selfAttn; + yield return _crossAttn; + yield return _ffn1; + yield return _ffn2; + yield return _norm1; + yield return _norm2; + yield return _norm3; } } /// /// Multi-head cross-attention for DETR decoder. /// -internal class MultiHeadCrossAttention +internal class MultiHeadCrossAttention : CvParameterModule { private readonly INumericOperations _numOps; private readonly int _hiddenDim; @@ -563,33 +468,15 @@ public MultiHeadCrossAttention(int hiddenDim, int numHeads) public Tensor Forward(Tensor queries, Tensor memory, Tensor? posEncoding) { - int batch = queries.Shape[0]; - int queryLen = queries.Shape[1]; - int memoryLen = memory.Shape[1]; + // Keys see the positional encoding; values do not (DETR convention). + var memoryWithPos = posEncoding is not null ? AiDotNetEngine.Current.TensorAdd(memory, posEncoding) : memory; - // Add positional encoding to memory if provided - var memoryWithPos = memory; - if (posEncoding is not null) - { - memoryWithPos = new Tensor(memory._shape); - for (int i = 0; i < memory.Length; i++) - { - memoryWithPos[i] = _numOps.Add(memory[i], posEncoding[i]); - } - } - - // Project queries, keys, values - var q = ProjectSequence(queries, _queryProj); - var k = ProjectSequence(memoryWithPos, _keyProj); - var v = ProjectSequence(memory, _valueProj); - - // Compute attention - var attnOutput = ComputeAttention(q, k, v, batch, queryLen, memoryLen); + var q = _queryProj.ForwardTokens(queries); + var k = _keyProj.ForwardTokens(memoryWithPos); + var v = _valueProj.ForwardTokens(memory); - // Project output - var output = ProjectSequence(attnOutput, _outputProj); - - return output; + var attended = CvTensorOps.MultiHeadAttention(q, k, v, _numHeads, _scale); + return _outputProj.ForwardTokens(attended); } public long GetParameterCount() @@ -634,102 +521,12 @@ public void ReadParameters(BinaryReader reader) _outputProj.ReadParameters(reader); } - private Tensor ProjectSequence(Tensor x, Dense proj) - { - int batch = x.Shape[0]; - int seqLen = x.Shape[1]; - int dim = x.Shape[2]; - int outDim = proj.OutputSize; - - var result = new Tensor(new[] { batch, seqLen, outDim }); - - for (int b = 0; b < batch; b++) - { - for (int s = 0; s < seqLen; s++) - { - var feat = new Tensor(new[] { 1, dim }); - for (int d = 0; d < dim; d++) - { - feat[0, d] = x[b, s, d]; - } - - var projected = proj.Forward(feat); - - for (int d = 0; d < outDim; d++) - { - result[b, s, d] = projected[0, d]; - } - } - } - - return result; - } - - private Tensor ComputeAttention(Tensor q, Tensor k, Tensor v, int batch, int queryLen, int keyLen) + /// + protected override IEnumerable?> ParameterChildren() { - // Simplified attention computation for each head - var output = new Tensor(new[] { batch, queryLen, _hiddenDim }); - - for (int b = 0; b < batch; b++) - { - for (int h = 0; h < _numHeads; h++) - { - int headOffset = h * _headDim; - - // Compute attention scores for this head - var scores = new double[queryLen, keyLen]; - for (int i = 0; i < queryLen; i++) - { - for (int j = 0; j < keyLen; j++) - { - double score = 0; - for (int d = 0; d < _headDim; d++) - { - score += _numOps.ToDouble(q[b, i, headOffset + d]) * - _numOps.ToDouble(k[b, j, headOffset + d]); - } - scores[i, j] = score * _scale; - } - } - - // Softmax over keys - for (int i = 0; i < queryLen; i++) - { - double maxScore = double.NegativeInfinity; - for (int j = 0; j < keyLen; j++) - { - maxScore = Math.Max(maxScore, scores[i, j]); - } - - double sumExp = 0; - for (int j = 0; j < keyLen; j++) - { - scores[i, j] = Math.Exp(scores[i, j] - maxScore); - sumExp += scores[i, j]; - } - - for (int j = 0; j < keyLen; j++) - { - scores[i, j] /= sumExp; - } - } - - // Apply attention to values - for (int i = 0; i < queryLen; i++) - { - for (int d = 0; d < _headDim; d++) - { - double value = 0; - for (int j = 0; j < keyLen; j++) - { - value += scores[i, j] * _numOps.ToDouble(v[b, j, headOffset + d]); - } - output[b, i, headOffset + d] = _numOps.FromDouble(value); - } - } - } - } - - return output; + yield return _queryProj; + yield return _keyProj; + yield return _valueProj; + yield return _outputProj; } } diff --git a/src/ComputerVision/Detection/ObjectDetection/DETR/DETRHelpers.cs b/src/ComputerVision/Detection/ObjectDetection/DETR/DETRHelpers.cs index a039080ea3..01d8bac465 100644 --- a/src/ComputerVision/Detection/ObjectDetection/DETR/DETRHelpers.cs +++ b/src/ComputerVision/Detection/ObjectDetection/DETR/DETRHelpers.cs @@ -29,8 +29,6 @@ public static (Tensor flattened, int[] levelStarts, int[][] spatialShapes) Fl } int batch = features[0].Shape[0]; - - // Validate consistent batch size across all features for (int i = 1; i < features.Count; i++) { if (features[i].Shape[0] != batch) @@ -41,91 +39,46 @@ public static (Tensor flattened, int[] levelStarts, int[][] spatialShapes) Fl } } + var engine = AiDotNetEngine.Current; int totalTokens = 0; var spatialShapes = new int[features.Count][]; var levelStarts = new int[features.Count]; - + var levels = new Tensor[features.Count]; for (int i = 0; i < features.Count; i++) { + int c = features[i].Shape[1]; int h = features[i].Shape[2]; int w = features[i].Shape[3]; spatialShapes[i] = new[] { h, w }; levelStarts[i] = totalTokens; totalTokens += h * w; - } - - var flattened = new Tensor(new[] { batch, totalTokens, hiddenDim }); - int offset = 0; - for (int level = 0; level < features.Count; level++) - { - var feat = features[level]; - int c = feat.Shape[1]; - int h = feat.Shape[2]; - int w = feat.Shape[3]; - - for (int b = 0; b < batch; b++) + // [B, C, H, W] -> [B, H*W, C], then fit the channel axis to hiddenDim: the first + // min(C, hiddenDim) channels are kept and any shortfall is zero-filled. Engine ops, so the + // backbone and neck below this point stay on the gradient tape. + var tokens = CvTensorOps.FlattenSpatial(features[i]); + if (c > hiddenDim) { - for (int y = 0; y < h; y++) - { - for (int x = 0; x < w; x++) - { - int tokenIdx = offset + y * w + x; - for (int d = 0; d < c && d < hiddenDim; d++) - { - flattened[b, tokenIdx, d] = feat[b, d, y, x]; - } - } - } + tokens = engine.TensorNarrow(tokens, 2, 0, hiddenDim); } - offset += h * w; + else if (c < hiddenDim) + { + tokens = engine.TensorConcatenate(new[] { tokens, new Tensor(new[] { batch, h * w, hiddenDim - c }) }, 2); + } + + levels[i] = tokens; } + var flattened = levels.Length == 1 ? levels[0] : engine.TensorConcatenate(levels, 1); return (flattened, levelStarts, spatialShapes); } - /// - /// Computes the GELU activation function. - /// - /// Input value. - /// GELU activation output. - public static double GELU(double x) - { - // Approximate GELU: 0.5 * x * (1 + tanh(sqrt(2/pi) * (x + 0.044715 * x^3))) - double c = Math.Sqrt(2.0 / Math.PI); - return 0.5 * x * (1.0 + Math.Tanh(c * (x + 0.044715 * x * x * x))); - } - /// /// Adds two tensors element-wise. /// /// The numeric type. /// First tensor. /// Second tensor. - /// Numeric operations provider. /// Element-wise sum of the tensors. - public static Tensor AddTensors(Tensor a, Tensor b, INumericOperations numOps) - { - if (a is null) - { - throw new ArgumentNullException(nameof(a)); - } - if (b is null) - { - throw new ArgumentNullException(nameof(b)); - } - if (a.Shape.Length != b.Shape.Length || !a._shape.SequenceEqual(b._shape)) - { - throw new ArgumentException( - $"Tensors must have the same shape. a.Shape=[{string.Join(",", a._shape)}], b.Shape=[{string.Join(",", b._shape)}].", - nameof(b)); - } - - var result = new Tensor(a._shape); - for (int i = 0; i < a.Length; i++) - { - result[i] = numOps.Add(a[i], b[i]); - } - return result; - } + public static Tensor AddTensors(Tensor a, Tensor b) => AiDotNetEngine.Current.TensorAdd(a, b); } diff --git a/src/ComputerVision/Detection/ObjectDetection/DETR/DINO.cs b/src/ComputerVision/Detection/ObjectDetection/DETR/DINO.cs index 842c371abd..b362dbf6ff 100644 --- a/src/ComputerVision/Detection/ObjectDetection/DETR/DINO.cs +++ b/src/ComputerVision/Detection/ObjectDetection/DETR/DINO.cs @@ -40,7 +40,7 @@ namespace AiDotNet.ComputerVision.Detection.ObjectDetection.DETR; "https://arxiv.org/abs/2203.03605", Year = 2023, Authors = "Hao Zhang, Feng Li, Shilong Liu, Lei Zhang, Hang Su, Jun Zhu, Lionel M. Ni, Heung-Yeung Shum")] -public class DINO : ObjectDetectorBase +public partial class DINO : ObjectDetectorBase, IDetectionTrainingModel { private readonly DINOEncoder _encoder; private readonly DINODecoder _decoder; @@ -48,6 +48,8 @@ public class DINO : ObjectDetectorBase private readonly int _hiddenDim; private readonly int _numQueries; private readonly NMS _nms; + private readonly AiDotNet.ComputerVision.Detection.Losses.DETRSetLoss _detectionLoss; + private readonly int _trainingClassCount; /// public override string Name => $"DINO-{Options.Size}"; @@ -75,6 +77,12 @@ public DINO(ObjectDetectionOptions options) : base(options) // DINO decoder with contrastive denoising _decoder = new DINODecoder(hiddenDim, numHeads, numDecoderLayers, numQueries, options.NumClasses); + var lossOptions = options.SetPredictionLoss ?? AiDotNet.ComputerVision.Detection.Losses.DetrSetLossOptions.ForDino(); + if (lossOptions.ClassificationLoss == SetPredictionClassificationLoss.SoftmaxCrossEntropy) + throw new ArgumentException("DINO's class head has independent sigmoid classes and no no-object class; use a sigmoid focal or varifocal set loss.", nameof(options)); + _trainingClassCount = options.NumClasses; + _detectionLoss = new AiDotNet.ComputerVision.Detection.Losses.DETRSetLoss(options.NumClasses, lossOptions); + _nms = new NMS(); } @@ -88,6 +96,38 @@ public DINO(ObjectDetectionOptions options) : base(options) _ => (256, 8, 6, 6, 300) }; + /// Trains the final DINO heads with exact assignment, sigmoid focal loss, L1 and GIoU. + /// + /// + /// Uses DINO's published recipe by default (Zhang et al. 2022, Table 8): focal loss with alpha 0.25 + /// and gamma 2, matching costs 2/5/2 and loss weights 1/5/2 for class/L1/GIoU. Override it with + /// . + /// + /// + /// Inputs are model-ready NCHW tensors, as for Predict. Targets use normalized center-format boxes. + /// An image with more targets than queries is rejected before any update. This architecture + /// exposes only its final decoder heads, so the paper's per-layer auxiliary, query-selection and + /// contrastive denoising losses are not claimed. + /// + /// + public void TrainDetections(Tensor input, DetectionTrainingBatch targets) + { + if (input is null) throw new ArgumentNullException(nameof(input)); + if (targets is null) throw new ArgumentNullException(nameof(targets)); + if (input.Rank != 4 || input.Shape[0] <= 0 || input.Shape[1] != 3 || input.Shape[2] <= 0 || input.Shape[3] <= 0) + throw new ArgumentException("DINO training requires a nonempty NCHW three-channel image batch.", nameof(input)); + targets.ValidateForModel(input.Shape[0], _trainingClassCount, _numQueries); + TrainWithTargets(input, targets, ComputeDetectionLoss); + } + + private Tensor ComputeDetectionLoss(List> heads, DetectionTrainingBatch targets) + { + if (heads.Count != 2) + throw new InvalidOperationException("DINO training requires the actual final class and box heads."); + // Forward exposes raw box logits and DecodeOutputs applies sigmoid; apply it on the tape here. + return _detectionLoss.ComputeTapeLoss(heads[0], Engine.Sigmoid(heads[1]), targets); + } + /// public override DetectionResult Detect(Tensor image, double confidenceThreshold, double nmsThreshold) { @@ -276,35 +316,7 @@ public override void SaveWeights(string path) return DETRHelpers.FlattenMultiScale(features, _hiddenDim); } - private Tensor ProjectFeatures(Tensor features) - { - // Apply linear projection using Dense layer - int batch = features.Shape[0]; - int seqLen = features.Shape[1]; - - var result = new Tensor(features._shape); - - for (int b = 0; b < batch; b++) - { - for (int s = 0; s < seqLen; s++) - { - var feat = new Tensor(new[] { 1, _hiddenDim }); - for (int d = 0; d < _hiddenDim; d++) - { - feat[0, d] = features[b, s, d]; - } - - // Apply projection - var projected = _inputProj.Forward(feat); - for (int d = 0; d < _hiddenDim; d++) - { - result[b, s, d] = projected[0, d]; - } - } - } - - return result; - } + private Tensor ProjectFeatures(Tensor features) => _inputProj.ForwardTokens(features); private Tensor GenerateMultiScalePositionalEncoding(int[] shape, int[][] spatialShapes, int[] levelStarts) { @@ -361,7 +373,7 @@ private Tensor GenerateMultiScalePositionalEncoding(int[] shape, int[][] spat /// /// DINO encoder with deformable attention. /// -internal class DINOEncoder +internal class DINOEncoder : CvParameterModule { private readonly INumericOperations _numOps; private readonly int _hiddenDim; @@ -387,12 +399,7 @@ public DINOEncoder(int hiddenDim, int numHeads, int numLayers, int numLevels) public Tensor Forward(Tensor x, Tensor posEncoding, int[][] spatialShapes, int[] levelStarts) { - var output = new Tensor(x._shape); - for (int i = 0; i < x.Length; i++) - { - output[i] = _numOps.Add(x[i], posEncoding[i]); - } - + var output = AiDotNetEngine.Current.TensorAdd(x, posEncoding); foreach (var layer in _layers) { output = layer.Forward(output, spatialShapes, levelStarts); @@ -450,12 +457,18 @@ public void ReadParameters(BinaryReader reader) layer.ReadParameters(reader); } } + + /// + protected override IEnumerable?> ParameterChildren() + { + foreach (var child in _layers) yield return child; + } } /// /// Single DINO encoder layer with deformable attention. /// -internal class DINOEncoderLayer +internal class DINOEncoderLayer : CvParameterModule { private readonly INumericOperations _numOps; private readonly MultiHeadSelfAttention _selfAttn; @@ -536,58 +549,28 @@ public void ReadParameters(BinaryReader reader) } private Tensor ApplyFFN(Tensor x) - { - int batch = x.Shape[0]; - int seqLen = x.Shape[1]; - int ffnDim = _ffn1.OutputSize; - - var result = new Tensor(x._shape); - - for (int b = 0; b < batch; b++) - { - for (int s = 0; s < seqLen; s++) - { - var feat = new Tensor(new[] { 1, _hiddenDim }); - for (int d = 0; d < _hiddenDim; d++) - { - feat[0, d] = x[b, s, d]; - } - - var h = _ffn1.Forward(feat); - for (int d = 0; d < ffnDim; d++) - { - double val = _numOps.ToDouble(h[0, d]); - h[0, d] = _numOps.FromDouble(GELU(val)); - } - - var output = _ffn2.Forward(h); - - for (int d = 0; d < _hiddenDim; d++) - { - result[b, s, d] = output[0, d]; - } - } - } - - return result; - } + => _ffn2.ForwardTokens(AiDotNetEngine.Current.GELU(_ffn1.ForwardTokens(x))); private Tensor AddTensors(Tensor a, Tensor b) { return AiDotNetEngine.Current.TensorAdd(a, b); } - private static double GELU(double x) + /// + protected override IEnumerable?> ParameterChildren() { - double c = Math.Sqrt(2.0 / Math.PI); - return 0.5 * x * (1.0 + Math.Tanh(c * (x + 0.044715 * x * x * x))); + yield return _selfAttn; + yield return _ffn1; + yield return _ffn2; + yield return _norm1; + yield return _norm2; } } /// /// DINO decoder with contrastive denoising and mixed query selection. /// -internal class DINODecoder +internal class DINODecoder : CvParameterModule { private readonly INumericOperations _numOps; private readonly int _numLayers; @@ -618,7 +601,7 @@ public DINODecoder(int hiddenDim, int numHeads, int numLayers, int numQueries, i _contentQueries = InitializeQueries(numQueries, hiddenDim); _positionQueries = InitializeQueries(numQueries, hiddenDim); - _classHead = new Dense(hiddenDim, numClasses + 1); + _classHead = new Dense(hiddenDim, numClasses); // Sigmoid classes; no no-object column. _boxHead = new Dense(hiddenDim, 4); } @@ -668,28 +651,13 @@ public DINODecoder(int hiddenDim, int numHeads, int numLayers, int numQueries, i { for (int q = 0; q < numQueries; q++) { - // Softmax over classes - double maxLogit = double.NegativeInfinity; - for (int c = 0; c < numClasses; c++) - { - double logit = _numOps.ToDouble(classLogits[b, q, c]); - maxLogit = Math.Max(maxLogit, logit); - } - - var probs = new double[numClasses]; - double sumExp = 0; - for (int c = 0; c < numClasses; c++) - { - double logit = _numOps.ToDouble(classLogits[b, q, c]); - probs[c] = Math.Exp(logit - maxLogit); - sumExp += probs[c]; - } - + // DINO classifies each query with independent per-class sigmoids trained by focal + // loss (Zhang et al. 2022); there is no no-object column. double maxScore = 0; int maxClassId = 0; - for (int c = 0; c < numClasses - 1; c++) + for (int c = 0; c < numClasses; c++) { - double prob = probs[c] / sumExp; + double prob = Sigmoid(_numOps.ToDouble(classLogits[b, q, c])); if (prob > maxScore) { maxScore = prob; @@ -829,56 +797,31 @@ private Tensor InitializeQueries(int numQueries, int hiddenDim) private Tensor CombineQueries(int batch) { + // content + position, broadcast over the batch. Both query tensors are learnable. int numQueries = _contentQueries.Shape[0]; + var combined = AiDotNetEngine.Current.TensorAdd(_contentQueries, _positionQueries); + return AiDotNetEngine.Current.TensorBroadcastTo(AiDotNetEngine.Current.Reshape(combined, new[] { 1, numQueries, _hiddenDim }), new[] { batch, numQueries, _hiddenDim }); + } - var combined = new Tensor(new[] { batch, numQueries, _hiddenDim }); - - for (int b = 0; b < batch; b++) - { - for (int q = 0; q < numQueries; q++) - { - for (int d = 0; d < _hiddenDim; d++) - { - combined[b, q, d] = _numOps.Add(_contentQueries[q, d], _positionQueries[q, d]); - } - } - } + private Tensor ApplyHead(Tensor output, Dense head) => head.ForwardTokens(output); - return combined; + private static double Sigmoid(double x) + { + return 1.0 / (1.0 + Math.Exp(-x)); } - private Tensor ApplyHead(Tensor output, Dense head) + /// + protected override IEnumerable?> ParameterChildren() { - int batch = output.Shape[0]; - int numQueries = output.Shape[1]; - int outDim = head.OutputSize; - - var result = new Tensor(new[] { batch, numQueries, outDim }); - - for (int b = 0; b < batch; b++) - { - for (int q = 0; q < numQueries; q++) - { - var feat = new Tensor(new[] { 1, _hiddenDim }); - for (int d = 0; d < _hiddenDim; d++) - { - feat[0, d] = output[b, q, d]; - } - - var headOut = head.Forward(feat); - - for (int i = 0; i < outDim; i++) - { - result[b, q, i] = headOut[0, i]; - } - } - } - - return result; + foreach (var child in _layers) yield return child; + yield return _classHead; + yield return _boxHead; } - private static double Sigmoid(double x) + /// + protected override IEnumerable> OwnParameterTensors() { - return 1.0 / (1.0 + Math.Exp(-x)); + yield return _contentQueries; + yield return _positionQueries; } } diff --git a/src/ComputerVision/Detection/ObjectDetection/DETR/RTDETR.cs b/src/ComputerVision/Detection/ObjectDetection/DETR/RTDETR.cs index 2d90405b15..0da809a8d2 100644 --- a/src/ComputerVision/Detection/ObjectDetection/DETR/RTDETR.cs +++ b/src/ComputerVision/Detection/ObjectDetection/DETR/RTDETR.cs @@ -40,13 +40,15 @@ namespace AiDotNet.ComputerVision.Detection.ObjectDetection.DETR; "https://arxiv.org/abs/2304.08069", Year = 2024, Authors = "Yian Zhao, Wenyu Lv, Shangliang Xu, Jinman Wei, Guanzhong Wang, Qingqing Dang, Yi Liu, Jie Chen")] -public class RTDETR : ObjectDetectorBase +public partial class RTDETR : ObjectDetectorBase, IDetectionTrainingModel { private readonly RTDETREncoder _encoder; private readonly RTDETRDecoder _decoder; private readonly int _hiddenDim; private readonly int _numQueries; private readonly NMS _nms; + private readonly AiDotNet.ComputerVision.Detection.Losses.DETRSetLoss _detectionLoss; + private readonly int _trainingClassCount; /// public override string Name => $"RT-DETR-{Options.Size}"; @@ -73,6 +75,12 @@ public RTDETR(ObjectDetectionOptions options) : base(options) // Efficient decoder _decoder = new RTDETRDecoder(hiddenDim, numHeads, numDecoderLayers, numQueries, options.NumClasses); + var lossOptions = options.SetPredictionLoss ?? AiDotNet.ComputerVision.Detection.Losses.DetrSetLossOptions.ForRtDetr(); + if (lossOptions.ClassificationLoss == SetPredictionClassificationLoss.SoftmaxCrossEntropy) + throw new ArgumentException("RT-DETR's class head has independent sigmoid classes and no no-object class; use a varifocal or sigmoid focal set loss.", nameof(options)); + _trainingClassCount = options.NumClasses; + _detectionLoss = new AiDotNet.ComputerVision.Detection.Losses.DETRSetLoss(options.NumClasses, lossOptions); + _nms = new NMS(); } @@ -86,6 +94,38 @@ public RTDETR(ObjectDetectionOptions options) : base(options) _ => (256, 8, 1, 4, 300) }; + /// Trains the final RT-DETR heads with exact assignment, varifocal loss, L1 and GIoU. + /// + /// + /// Uses RT-DETR's published recipe by default (Zhao et al. 2023, Table A): varifocal loss with alpha + /// 0.75 and gamma 2, matching costs 2/5/2 and loss weights 1/5/2 for class/L1/GIoU. Override it + /// with . + /// + /// + /// Inputs are model-ready NCHW tensors, as for Predict. Targets use normalized center-format boxes. + /// An image with more targets than queries is rejected before any update. This architecture + /// exposes only its final decoder heads, so per-layer auxiliary and encoder query-selection + /// losses are not claimed. + /// + /// + public void TrainDetections(Tensor input, DetectionTrainingBatch targets) + { + if (input is null) throw new ArgumentNullException(nameof(input)); + if (targets is null) throw new ArgumentNullException(nameof(targets)); + if (input.Rank != 4 || input.Shape[0] <= 0 || input.Shape[1] != 3 || input.Shape[2] <= 0 || input.Shape[3] <= 0) + throw new ArgumentException("RT-DETR training requires a nonempty NCHW three-channel image batch.", nameof(input)); + targets.ValidateForModel(input.Shape[0], _trainingClassCount, _numQueries); + TrainWithTargets(input, targets, ComputeDetectionLoss); + } + + private Tensor ComputeDetectionLoss(List> heads, DetectionTrainingBatch targets) + { + if (heads.Count != 2) + throw new InvalidOperationException("RT-DETR training requires the actual final class and box heads."); + // Forward exposes raw box logits and DecodeOutputs applies sigmoid; apply it on the tape here. + return _detectionLoss.ComputeTapeLoss(heads[0], Engine.Sigmoid(heads[1]), targets); + } + /// public override DetectionResult Detect(Tensor image, double confidenceThreshold, double nmsThreshold) { @@ -169,7 +209,7 @@ protected override List> PostProcess( } // RT-DETR is designed to be NMS-free, but apply with high threshold for safety - var nmsResults = _nms.Apply(candidateDetections, Math.Max(0.9, nmsThreshold)); + var nmsResults = _nms.Apply(candidateDetections, EffectiveNmsThreshold(nmsThreshold)); if (nmsResults.Count > Options.MaxDetections) { @@ -277,12 +317,19 @@ public override void SaveWeights(string path) { return DETRHelpers.FlattenMultiScale(features, _hiddenDim); } + + /// + /// + /// One query per object means duplicates are rare, so NMS runs only as a safety net at IoU 0.9 + /// (or the requested value, if that is higher) instead of the caller's threshold. + /// + public override double EffectiveNmsThreshold(double requested) => Math.Max(0.9, requested); } /// /// RT-DETR hybrid encoder with intra-scale and cross-scale attention. /// -internal class RTDETREncoder +internal class RTDETREncoder : CvParameterModule { private readonly INumericOperations _numOps; private readonly int _hiddenDim; @@ -381,12 +428,19 @@ public void ReadParameters(BinaryReader reader) _crossScaleModule.ReadParameters(reader); } + + /// + protected override IEnumerable?> ParameterChildren() + { + foreach (var child in _intrascaleLayers) yield return child; + yield return _crossScaleModule; + } } /// /// RT-DETR intra-scale encoder layer. /// -internal class RTDETREncoderLayer +internal class RTDETREncoderLayer : CvParameterModule { private readonly INumericOperations _numOps; private readonly MultiHeadSelfAttention _selfAttn; @@ -467,58 +521,28 @@ public void ReadParameters(BinaryReader reader) } private Tensor ApplyFFN(Tensor x) - { - int batch = x.Shape[0]; - int seqLen = x.Shape[1]; - int ffnDim = _ffn1.OutputSize; - - var result = new Tensor(x._shape); - - for (int b = 0; b < batch; b++) - { - for (int s = 0; s < seqLen; s++) - { - var feat = new Tensor(new[] { 1, _hiddenDim }); - for (int d = 0; d < _hiddenDim; d++) - { - feat[0, d] = x[b, s, d]; - } - - var h = _ffn1.Forward(feat); - for (int d = 0; d < ffnDim; d++) - { - double val = _numOps.ToDouble(h[0, d]); - h[0, d] = _numOps.FromDouble(GELU(val)); - } - - var output = _ffn2.Forward(h); - - for (int d = 0; d < _hiddenDim; d++) - { - result[b, s, d] = output[0, d]; - } - } - } - - return result; - } + => _ffn2.ForwardTokens(AiDotNetEngine.Current.GELU(_ffn1.ForwardTokens(x))); private Tensor AddTensors(Tensor a, Tensor b) { return AiDotNetEngine.Current.TensorAdd(a, b); } - private static double GELU(double x) + /// + protected override IEnumerable?> ParameterChildren() { - double c = Math.Sqrt(2.0 / Math.PI); - return 0.5 * x * (1.0 + Math.Tanh(c * (x + 0.044715 * x * x * x))); + yield return _selfAttn; + yield return _ffn1; + yield return _ffn2; + yield return _norm1; + yield return _norm2; } } /// /// Cross-scale feature fusion module for RT-DETR. /// -internal class CrossScaleModule +internal class CrossScaleModule : CvParameterModule { private readonly INumericOperations _numOps; private readonly int _hiddenDim; @@ -540,77 +564,33 @@ public CrossScaleModule(int hiddenDim, int numLevels) public Tensor Forward(Tensor x, int[][] spatialShapes, int[] levelStarts) { + var engine = AiDotNetEngine.Current; int batch = x.Shape[0]; - int totalTokens = x.Shape[1]; - - // Compute global representation for each level - var levelRepresentations = new List>(); + // Summarise each level by its token mean, concatenate the summaries, and add each level's + // fused projection back onto that level's tokens. + var levelTokens = new Tensor[_numLevels]; + var summaries = new Tensor[_numLevels]; for (int level = 0; level < _numLevels; level++) { - int start = levelStarts[level]; int numTokens = spatialShapes[level][0] * spatialShapes[level][1]; - - var levelRep = new Tensor(new[] { batch, _hiddenDim }); - - for (int b = 0; b < batch; b++) - { - for (int d = 0; d < _hiddenDim; d++) - { - double sum = 0; - for (int t = 0; t < numTokens; t++) - { - sum += _numOps.ToDouble(x[b, start + t, d]); - } - levelRep[b, d] = _numOps.FromDouble(sum / numTokens); - } - } - - levelRepresentations.Add(levelRep); - } - - // Concatenate level representations - var concat = new Tensor(new[] { batch, _hiddenDim * _numLevels }); - for (int b = 0; b < batch; b++) - { - int offset = 0; - for (int level = 0; level < _numLevels; level++) - { - for (int d = 0; d < _hiddenDim; d++) - { - concat[b, offset + d] = levelRepresentations[level][b, d]; - } - offset += _hiddenDim; - } + levelTokens[level] = engine.TensorNarrow(x, 1, levelStarts[level], numTokens); + summaries[level] = engine.ReduceMean(levelTokens[level], new[] { 1 }, false); // [B, D] } - // Fuse and add back to each level - var result = new Tensor(x._shape); - for (int i = 0; i < x.Length; i++) - { - result[i] = x[i]; - } + var concat = _numLevels == 1 ? summaries[0] : engine.TensorConcatenate(summaries, 1); // [B, D*L] + var updated = new Tensor[_numLevels]; for (int level = 0; level < _numLevels; level++) { - int start = levelStarts[level]; - int numTokens = spatialShapes[level][0] * spatialShapes[level][1]; - - for (int b = 0; b < batch; b++) - { - var fused = _fusionLayers[level].Forward(ExtractRow(concat, b)); - - for (int t = 0; t < numTokens; t++) - { - for (int d = 0; d < _hiddenDim; d++) - { - result[b, start + t, d] = _numOps.Add(result[b, start + t, d], fused[0, d]); - } - } - } + int numTokens = levelTokens[level].Shape[1]; + var fused = _fusionLayers[level].Forward(concat); // [B, D] + var broadcast = engine.TensorBroadcastTo( + engine.Reshape(fused, new[] { batch, 1, _hiddenDim }), new[] { batch, numTokens, _hiddenDim }); + updated[level] = engine.TensorAdd(levelTokens[level], broadcast); } - return result; + return _numLevels == 1 ? updated[0] : engine.TensorConcatenate(updated, 1); } public long GetParameterCount() @@ -657,22 +637,17 @@ public void ReadParameters(BinaryReader reader) } } - private Tensor ExtractRow(Tensor x, int row) + /// + protected override IEnumerable?> ParameterChildren() { - int cols = x.Shape[1]; - var result = new Tensor(new[] { 1, cols }); - for (int c = 0; c < cols; c++) - { - result[0, c] = x[row, c]; - } - return result; + foreach (var child in _fusionLayers) yield return child; } } /// /// RT-DETR decoder with uncertainty-minimal query selection. /// -internal class RTDETRDecoder +internal class RTDETRDecoder : CvParameterModule { private readonly INumericOperations _numOps; private readonly int _hiddenDim; @@ -699,7 +674,7 @@ public RTDETRDecoder(int hiddenDim, int numHeads, int numLayers, int numQueries, } _queryEmbed = InitializeQueries(numQueries, hiddenDim); - _classHead = new Dense(hiddenDim, numClasses + 1); + _classHead = new Dense(hiddenDim, numClasses); // Sigmoid classes; no no-object column. _boxHead = new Dense(hiddenDim, 4); } @@ -745,28 +720,13 @@ public RTDETRDecoder(int hiddenDim, int numHeads, int numLayers, int numQueries, for (int q = 0; q < numQueries; q++) { - // Softmax over classes - double maxLogit = double.NegativeInfinity; - for (int c = 0; c < numClasses; c++) - { - double logit = _numOps.ToDouble(classLogits[b, q, c]); - maxLogit = Math.Max(maxLogit, logit); - } - - var probs = new double[numClasses]; - double sumExp = 0; - for (int c = 0; c < numClasses; c++) - { - double logit = _numOps.ToDouble(classLogits[b, q, c]); - probs[c] = Math.Exp(logit - maxLogit); - sumExp += probs[c]; - } - + // RT-DETR scores each class with an independent sigmoid trained toward the box IoU + // (varifocal loss, Zhao et al. 2023); there is no no-object column. double maxScore = 0; int maxClassId = 0; - for (int c = 0; c < numClasses - 1; c++) + for (int c = 0; c < numClasses; c++) { - double prob = probs[c] / sumExp; + double prob = Sigmoid(_numOps.ToDouble(classLogits[b, q, c])); if (prob > maxScore) { maxScore = prob; @@ -889,59 +849,26 @@ private Tensor InitializeQueries(int numQueries, int hiddenDim) } private Tensor SelectQueries(Tensor memory, int batch) - { - // TODO: Implement uncertainty-minimal query selection as described in RT-DETR paper. - // Current simplified implementation uses fixed learnable queries. - // Full implementation should compute uncertainty scores from encoder output - // and select top-K positions with minimal uncertainty. - var queries = new Tensor(new[] { batch, _numQueries, _hiddenDim }); + => AiDotNetEngine.Current.TensorBroadcastTo(AiDotNetEngine.Current.Reshape(_queryEmbed, new[] { 1, _numQueries, _hiddenDim }), new[] { batch, _numQueries, _hiddenDim }); - for (int b = 0; b < batch; b++) - { - for (int q = 0; q < _numQueries; q++) - { - for (int d = 0; d < _hiddenDim; d++) - { - queries[b, q, d] = _queryEmbed[q, d]; - } - } - } + private Tensor ApplyHead(Tensor output, Dense head) => head.ForwardTokens(output); - return queries; + private static double Sigmoid(double x) + { + return 1.0 / (1.0 + Math.Exp(-x)); } - private Tensor ApplyHead(Tensor output, Dense head) + /// + protected override IEnumerable?> ParameterChildren() { - int batch = output.Shape[0]; - int numQueries = output.Shape[1]; - int outDim = head.OutputSize; - - var result = new Tensor(new[] { batch, numQueries, outDim }); - - for (int b = 0; b < batch; b++) - { - for (int q = 0; q < numQueries; q++) - { - var feat = new Tensor(new[] { 1, _hiddenDim }); - for (int d = 0; d < _hiddenDim; d++) - { - feat[0, d] = output[b, q, d]; - } - - var headOut = head.Forward(feat); - - for (int i = 0; i < outDim; i++) - { - result[b, q, i] = headOut[0, i]; - } - } - } - - return result; + foreach (var child in _layers) yield return child; + yield return _classHead; + yield return _boxHead; } - private static double Sigmoid(double x) + /// + protected override IEnumerable> OwnParameterTensors() { - return 1.0 / (1.0 + Math.Exp(-x)); + yield return _queryEmbed; } } diff --git a/src/ComputerVision/Detection/ObjectDetection/ObjectDetectorBase.cs b/src/ComputerVision/Detection/ObjectDetection/ObjectDetectorBase.cs index 417b14d2ea..b23c145915 100644 --- a/src/ComputerVision/Detection/ObjectDetection/ObjectDetectorBase.cs +++ b/src/ComputerVision/Detection/ObjectDetection/ObjectDetectorBase.cs @@ -53,6 +53,10 @@ public abstract partial class ObjectDetectorBase : ModelBase, Te /// Gets the backbone network, throwing if not initialized. /// /// Thrown when backbone has not been initialized. + // An accessor over the Backbone property, not separate storage. Without the alias the generator + // registered BOTH, so these weights were counted twice in the flat parameter vector, and for a + // detector without a neck (DETR) reading parameters threw from the accessor's null check. + [AiDotNet.Attributes.ParameterAlias(nameof(Backbone))] protected IDetectionBackbone EnsureBackbone => Backbone ?? throw new InvalidOperationException( $"{GetType().Name}: Backbone not initialized. Ensure the model is properly constructed."); @@ -61,6 +65,10 @@ public abstract partial class ObjectDetectorBase : ModelBase, Te /// Gets the neck module, throwing if not initialized. /// /// Thrown when neck has not been initialized. + // An accessor over the Neck property, not separate storage. Without the alias the generator + // registered BOTH, so these weights were counted twice in the flat parameter vector, and for a + // detector without a neck (DETR) reading parameters threw from the accessor's null check. + [AiDotNet.Attributes.ParameterAlias(nameof(Neck))] protected NeckBase EnsureNeck => Neck ?? throw new InvalidOperationException( $"{GetType().Name}: Neck not initialized. Ensure the model is properly constructed."); @@ -85,6 +93,30 @@ public abstract partial class ObjectDetectorBase : ModelBase, Te /// public string[] ClassNames { get; protected set; } + /// + /// Gets the number of object classes this detector was configured for. + /// + /// + /// Every the detector emits indexes into a label set of + /// this size, so callers need it to interpret the output. + /// + public int NumClasses => Options.NumClasses; + + /// + /// Gets the maximum number of detections kept for a single image after non-maximum suppression. + /// + public int MaxDetections => Options.MaxDetections; + + /// + /// Gets the default minimum confidence a detection needs to be reported. + /// + public double ConfidenceThreshold => Options.ConfidenceThreshold; + + /// + /// Gets the default IoU threshold used by non-maximum suppression. + /// + public double NmsThreshold => Options.NmsThreshold; + /// /// Name of this detector architecture. /// @@ -161,8 +193,11 @@ public virtual BatchDetectionResult DetectBatch( int imageHeight = images.Shape[2]; int imageWidth = images.Shape[3]; - // Perform batch forward pass (single GPU call for all images) - var batchOutputs = Forward(images); + // Preprocess exactly as Detect does, then one forward pass for the whole batch. This used to + // feed the RAW images straight to Forward, so a batch and a single Detect of the same image + // ran the network on different inputs and disagreed; PostProcess then also mapped + // coordinates from a frame the network never saw. + var batchOutputs = Forward(Preprocess(images)); // Post-process outputs for each image in the batch for (int i = 0; i < batchSize; i++) @@ -330,10 +365,16 @@ public virtual long GetParameterCount() /// Raw image tensor. /// Preprocessed tensor ready for the network. protected virtual Tensor Preprocess(Tensor image) + { + var prepared = PreprocessCore(image); + NoteResolvedInput(prepared); + return prepared; + } + + private Tensor PreprocessCore(Tensor image) { // Default preprocessing: resize to input size and normalize - int targetHeight = Options.InputSize[0]; - int targetWidth = Options.InputSize[1]; + var (targetHeight, targetWidth) = GetValidatedInputSize(); // Resize if needed var resized = ResizeImage(image, targetHeight, targetWidth); @@ -344,6 +385,30 @@ protected virtual Tensor Preprocess(Tensor image) return normalized; } + private (int Height, int Width) GetValidatedInputSize() + { + // InputSize is publicly mutable, so validate at each consuming boundary rather than + // only at construction. Return the dimensions, not the caller-owned array. + var inputSize = Options.InputSize; + if (inputSize is null || inputSize.Length != 2) + { + throw new ArgumentException( + "InputSize must contain exactly two positive dimensions [height, width].", + nameof(Options.InputSize)); + } + + int height = inputSize[0]; + int width = inputSize[1]; + if (height <= 0 || width <= 0) + { + throw new ArgumentException( + "InputSize must contain exactly two positive dimensions [height, width].", + nameof(Options.InputSize)); + } + + return (height, width); + } + /// /// Resizes an image tensor to the specified dimensions. /// @@ -477,16 +542,73 @@ protected static string[] GetCocoClassNames() /// /// Predicts by running the forward pass and returning raw network outputs concatenated. /// + /// + /// Each output is flattened per image to [batch, -1] and the results are concatenated, so + /// the prediction carries every head: all YOLO pyramid levels, both DETR's class logits and its + /// boxes. It used to return outputs[0] alone despite this summary, so a detector trained + /// against never trained any head but the first. A single-output model is + /// unchanged. + /// public override Tensor Predict(Tensor input) { - var outputs = Forward(input); - return outputs.Count > 0 ? outputs[0] : new Tensor(new[] { 1, 0 }); + NoteResolvedInput(input); + return CvTensorOps.ConcatenateOutputs(Forward(input)); } /// - /// Training object detectors requires specialized loss. Override in subclasses. + /// Gets the step size used by . /// - public override void Train(Tensor input, Tensor expectedOutput) { } + /// + /// Detection losses are large early in training, so this is deliberately conservative. + /// Override it to match a paper recipe. + /// + protected virtual double TrainingLearningRate => 0.001; + + /// + /// Runs one training step against the model's public prediction. + /// + /// The training image. + /// The desired output, shaped like . + /// + /// + /// This used to be an empty method whose comment said "override in subclasses" -- and no + /// subclass ever did, so every detector in the library silently ignored training and left + /// its weights at their initial values. + /// + /// + /// The step records the forward pass on a gradient tape, takes mean squared error against + /// , and applies a stochastic-gradient update to every + /// trainable tensor reachable from this model. A detector-specific loss (assignment plus + /// box regression plus classification) is the right objective for a full training recipe and + /// is not implied by a tensor's shape. This overload is raw-output regression, not semantic + /// detection training. Models implementing expose + /// a separate typed target API for their detection objective. + /// + /// + public override void Train(Tensor input, Tensor expectedOutput) + { + if (input is null) + { + throw new ArgumentNullException(nameof(input)); + } + + if (expectedOutput is null) + { + throw new ArgumentNullException(nameof(expectedOutput)); + } + + bool wasTraining = IsTrainingMode; + SetTrainingMode(true); + try + { + RecordTrainingLoss(TensorModelTrainer.Step( + this, input, expectedOutput, NumOps.FromDouble(TrainingLearningRate), Predict)); + } + finally + { + SetTrainingMode(wasTraining); + } + } /// public override ILossFunction DefaultLossFunction => new MeanSquaredErrorLoss(); @@ -499,9 +621,162 @@ public override IFullModel, Tensor> WithParameters(Vector par return copy; } - /// - public override IFullModel, Tensor> DeepCopy() - => (ObjectDetectorBase)MemberwiseClone(); + // DeepCopy is deliberately NOT overridden here. It used to return MemberwiseClone(), which + // is a SHALLOW copy: the clone shared every layer, backbone and neck reference with the + // original, so fine-tuning a clone silently rewrote the source model's weights. ModelBase + // rebuilds the model from its recorded constructor and reloads state through + // Serialize/Deserialize, giving the copy its own storage -- the same reasoning already + // recorded on NeckBase. #endregion + + /// + /// The shape of the first input this model's forward pass ran on. Its lazily-shaped layers sized + /// their weights from it, so replaying it on a rebuilt copy reproduces the same parameter + /// topology. Scratch: never persisted, and rebuilt copies record their own. + /// + [AiDotNet.Attributes.Scratch] + private int[]? _resolvedInputShape; + + /// Trains structured heads with typed targets through the shared single-update path. + /// The derived model validates its task targets before calling this method. + protected void TrainWithTargets(Tensor input, TTarget targets, + Func>, TTarget, Tensor> loss) where TTarget : class + => TrainWithTargets(input, targets, Forward, loss); + + /// Trains heads produced by a training-specific forward, such as auxiliary heads inference drops. + /// The derived model validates its task targets before calling this method. + protected void TrainWithTargets(Tensor input, TTarget targets, + Func, List>> forward, Func>, TTarget, Tensor> loss) where TTarget : class + { + if (input is null) throw new ArgumentNullException(nameof(input)); + if (targets is null) throw new ArgumentNullException(nameof(targets)); + if (forward is null) throw new ArgumentNullException(nameof(forward)); + if (loss is null) throw new ArgumentNullException(nameof(loss)); + NoteResolvedInput(input); + bool wasTraining = IsTrainingMode; + SetTrainingMode(true); + try + { + RecordTrainingLoss(TensorModelTrainer.StepWithTargets( + this, input, targets, NumOps.FromDouble(TrainingLearningRate), forward, loss)); + } + finally + { + SetTrainingMode(wasTraining); + } + } + + /// Records the input shape on the first forward pass. + private void NoteResolvedInput(Tensor input) + { + if (_resolvedInputShape is not null || input is null) + { + return; + } + + var shape = new int[input.Shape.Length]; + for (int i = 0; i < shape.Length; i++) + { + shape[i] = input.Shape[i]; + } + + _resolvedInputShape = shape; + } + + /// + /// + /// Runs the copy once on a zero input of the shape this model has already processed, so its + /// lazily-shaped layers (the convolutions behind the Conv2D adapter, the backbone's lazy layers) + /// size their weights exactly as this model's did before its state is loaded into them. + /// + protected override void PrepareCopyForStateRestore(ModelBase, Tensor> copy) + { + if (_resolvedInputShape is not null && copy is ObjectDetectorBase rebuilt) + { + var shape = (int[])_resolvedInputShape.Clone(); + shape[0] = 1; + rebuilt.Predict(new Tensor(shape)); + } + } + + /// + /// Scale factors from the network-input frame (the + /// that resizes to) to the source image's frame. + /// + /// + /// Boxes decode in network-input coordinates. Clipping them to the source image's size without + /// this mapping produced inverted boxes (x1 beyond x2) whenever the source image was smaller than + /// the input size, and silently dropped the boxes a two-stage detector's degenerate-box check + /// then rejected. + /// + protected (double ScaleX, double ScaleY) InputToImageScale(int imageWidth, int imageHeight) + => (imageWidth / (double)Options.InputSize[1], imageHeight / (double)Options.InputSize[0]); + + /// + /// Gets the IoU threshold non-maximum suppression actually applies for a requested threshold. + /// + /// The threshold passed to Detect. + /// The requested threshold, unless the model deliberately suppresses less aggressively. + /// + /// Set-prediction detectors (DETR, RT-DETR) are trained so that each object gets one query, and + /// apply NMS only as a safety net at a high threshold rather than at the caller's value. That + /// used to happen silently inside their post-processing; it is now declared here so callers can + /// see it. + /// + public virtual double EffectiveNmsThreshold(double requested) => requested; + + /// + /// Gets the number of channels in the images this model reads. + /// + /// RGB unless a model overrides it; every backbone here is built for three channels. + protected virtual int InputChannels => 3; + + /// + /// Gives a model that has never run a concrete parameter topology, so its state can be captured. + /// + /// + /// Several layers size their weights on their first forward pass. Until then the model reports + /// its parameters as shape-deferred, which is correct for a parameter query but made + /// - and therefore Clone - throw on a freshly constructed model. + /// Running the network once on a zero image of the configured input size resolves exactly the + /// shapes the first real image would, because every image is resized to that size first. + /// + private void ResolveDeferredParameters() + { + if (_resolvedInputShape is not null) + { + return; + } + + var (height, width) = GetValidatedInputSize(); + Predict(new Tensor(new[] { 1, InputChannels, height, width })); + } + + /// + /// Resolves shape-deferred layers first; see . + public override byte[] Serialize() + { + ResolveDeferredParameters(); + return base.Serialize(); + } + + /// + /// The loss of the most recent call, measured before its update. + /// + [AiDotNet.Attributes.Scratch] + private T _lastTrainingLoss = MathHelper.GetNumericOperations().Zero; + + /// + /// Gets the loss of the most recent call, measured on that call's input before + /// its update (zero before the first call). + /// + /// The training objective's value: mean squared error, or the model's own loss where it + /// has one. + /// Same contract as INeuralNetwork<T>.GetLastLoss. + public T GetLastLoss() => _lastTrainingLoss; + + /// Records the loss a training step reported. + /// The step's loss. + protected void RecordTrainingLoss(T loss) => _lastTrainingLoss = loss; } diff --git a/src/ComputerVision/Detection/ObjectDetection/RCNN/CascadeRCNN.cs b/src/ComputerVision/Detection/ObjectDetection/RCNN/CascadeRCNN.cs index 5b4f1bbeaf..cdaea9113c 100644 --- a/src/ComputerVision/Detection/ObjectDetection/RCNN/CascadeRCNN.cs +++ b/src/ComputerVision/Detection/ObjectDetection/RCNN/CascadeRCNN.cs @@ -1,3 +1,4 @@ +using AiDotNet.Tensors.Engines; using System.IO; using AiDotNet.Augmentation.Image; using AiDotNet.ComputerVision.Detection.Backbones; @@ -39,8 +40,12 @@ namespace AiDotNet.ComputerVision.Detection.ObjectDetection.RCNN; "https://arxiv.org/abs/1712.00726", Year = 2018, Authors = "Zhaowei Cai, Nuno Vasconcelos")] -public class CascadeRCNN : ObjectDetectorBase +public partial class CascadeRCNN : ObjectDetectorBase, IDetectionTrainingModel { + private readonly AiDotNet.ComputerVision.Detection.Losses.TwoStageDetectionLoss _detectionLoss; + + [AiDotNet.Attributes.Scratch] + private Random? _trainingRandom; private readonly RPN _rpn; private readonly RoIAlign _roiAlign; private readonly List> _stages; @@ -84,6 +89,8 @@ public CascadeRCNN(ObjectDetectionOptions options, int numStages = 3) : base( } _nms = new NMS(); + _detectionLoss = new AiDotNet.ComputerVision.Detection.Losses.TwoStageDetectionLoss(options.NumClasses, numStages, + options.TwoStageLoss ?? new AiDotNet.ComputerVision.Detection.Losses.TwoStageDetectionLossOptions()); } private static (int hiddenDim, int roiOutputSize) GetSizeConfig(ModelSize size) => size switch @@ -119,6 +126,104 @@ public override DetectionResult Detect(Tensor image, double confidenceThre /// protected override List> Forward(Tensor input) + { + var stages = ForwardStages(input, null, out _, out var objectness, out var bboxDeltas); + if (stages is null) + { + return new List> + { + new Tensor(new[] { 0, Options.NumClasses + 1 }), + new Tensor(new[] { 0, (Options.NumClasses + 1) * 4 }), + new Tensor(new[] { 0, 4 }), + objectness, + bboxDeltas + }; + } + + // PostProcess reads the first three entries: the last stage's logits, deltas and the boxes that stage + // received. The earlier stages' outputs and the RPN's follow, so every head reaches a training objective. + var last = stages[stages.Count - 1]; + var outputs = new List> { last.ClassLogits, last.BoxDeltas, last.Boxes }; + for (int stage = 0; stage < stages.Count - 1; stage++) + { + outputs.Add(stages[stage].ClassLogits); + outputs.Add(stages[stage].BoxDeltas); + } + outputs.Add(objectness); + outputs.Add(bboxDeltas); + return outputs; + } + + /// Trains the proposal network and every cascade stage with their published objectives. + /// + /// + /// One update sums the region proposal loss (Ren et al. 2015) and, for each stage t, the region-of-interest loss + /// L_cls + [y_t >= 1] L_loc on the boxes that stage actually received, labeled at that stage's IoU threshold + /// (Cai and Vasconcelos 2018, Eq. 8; thresholds 0.5, 0.6, 0.7). As in the reference implementation, the object + /// boxes join the first stage's proposals, and each later stage resamples the previous stage's regressed boxes. + /// Override the sampling and weights with . + /// + /// + /// Inputs are model-ready NCHW tensors, as for Predict, and targets are normalized against that input size. The + /// stages classify the proposals of a single image per forward pass, so each step takes one image. + /// + /// + public void TrainDetections(Tensor input, DetectionTrainingBatch targets) + { + if (input is null) throw new ArgumentNullException(nameof(input)); + if (targets is null) throw new ArgumentNullException(nameof(targets)); + if (input.Rank != 4 || input.Shape[1] != 3 || input.Shape[2] <= 0 || input.Shape[3] <= 0) + throw new ArgumentException("Cascade R-CNN training requires a three-channel NCHW image batch.", nameof(input)); + if (input.Shape[0] != 1) + throw new ArgumentException("Cascade R-CNN stages classify one image's proposals per forward pass; train one image per step.", nameof(input)); + targets.ValidateForModel(1, Options.NumClasses, int.MaxValue); + int height = input.Shape[2]; + int width = input.Shape[3]; + var gold = targets[0].Select(target => TwoStageTargets.PixelCorners(target, width, height)).ToList(); + var goldClasses = targets[0].Select(target => target.ClassId).ToList(); + var random = _trainingRandom ??= Options.RandomSeed is int seed + ? AiDotNet.Tensors.Helpers.RandomHelper.CreateSeededRandom(seed) + : AiDotNet.Tensors.Helpers.RandomHelper.CreateSecureRandom(); + List> anchors = new(); + TrainWithTargets(input, targets, + image => ForwardForTraining(image, gold, out anchors), + (heads, batch) => + { + var loss = _detectionLoss.ComputeProposalLoss(TwoStageTargets.FirstImage(heads[heads.Count - 2]), + TwoStageTargets.FirstImage(heads[heads.Count - 1]), anchors, gold, _rpn.AnchorsPerLocation, random); + int stages = (heads.Count - 2) / 3; + for (int stage = 0; stage < stages; stage++) + loss = Engine.TensorAdd(loss, _detectionLoss.ComputeStageLoss( + heads[3 * stage], heads[3 * stage + 1], heads[3 * stage + 2], gold, goldClasses, stage, random)); + return loss; + }); + } + + /// Each stage's logits, deltas and received boxes in order, then the RPN's objectness and deltas. + private List> ForwardForTraining(Tensor input, IReadOnlyList gold, out List> anchors) + { + var stages = ForwardStages(input, gold, out anchors, out var objectness, out var bboxDeltas); + var outputs = new List>(); + if (stages is not null) + { + foreach (var stage in stages) + { + outputs.Add(stage.ClassLogits); + outputs.Add(stage.BoxDeltas); + outputs.Add(stage.Boxes); + } + } + outputs.Add(objectness); + outputs.Add(bboxDeltas); + return outputs; + } + + /// + /// Runs the backbone, proposal network and every cascade stage, optionally adding boxes to the first stage's + /// proposals. Returns null when no stage receives a box. + /// + private List? ForwardStages(Tensor input, IReadOnlyList? extraProposals, + out List> anchors, out Tensor objectness, out Tensor bboxDeltas) { int imageHeight = input.Shape[2]; int imageWidth = input.Shape[3]; @@ -126,73 +231,66 @@ protected override List> Forward(Tensor input) // Extract backbone features var backboneFeatures = EnsureBackbone.ExtractFeatures(input); - // Apply FPN neck + // Apply FPN neck to get multi-scale features var fpnFeatures = EnsureNeck.Forward(backboneFeatures); - // Use P4 level for RPN - var rpnFeatures = fpnFeatures.Count > 1 ? fpnFeatures[1] : fpnFeatures[0]; - - // Stage 1: Region Proposal Network - var (objectness, bboxDeltas, anchors) = _rpn.Forward(rpnFeatures); + // Cascade R-CNN (Cai & Vasconcelos 2018) on an FPN (Lin et al. 2017; detectron2, torchvision): the shared + // RPN head runs on every level P2-P5 plus P6 (P5 subsampled by 2), each with its own anchor size, and each + // RoI is pooled from the level matching its size. + var rpnLevels = new List>(fpnFeatures) { CvTensorOps.MaxPoolPadded(fpnFeatures[^1], 1, 2, 0) }; + var (rpnObjectness, rpnDeltas, levelAnchors, levelAnchorCounts) = _rpn.ForwardLevels(rpnLevels); + objectness = rpnObjectness; + bboxDeltas = rpnDeltas; + anchors = levelAnchors; - // Generate initial proposals + // Generate initial proposals: top 1000 per level, NMS within each level, best 1000 overall. var initialProposals = _rpn.GenerateProposals( - objectness, bboxDeltas, anchors, + rpnObjectness, rpnDeltas, levelAnchors, imageHeight, imageWidth, - preNmsTopK: 2000, + preNmsTopK: 1000, postNmsTopK: 1000, - nmsThreshold: 0.7); - - if (initialProposals.Count == 0 || initialProposals[0].boxes.Shape[0] == 0) - { - return new List> - { - new Tensor(new[] { 0, Options.NumClasses + 1 }), - new Tensor(new[] { 0, (Options.NumClasses + 1) * 4 }), - new Tensor(new[] { 0, 4 }) - }; - } + nmsThreshold: 0.7, + levelAnchorCounts: levelAnchorCounts); - // Get P4 features for RoI Align - var p4Features = fpnFeatures.Count > 1 ? fpnFeatures[1] : fpnFeatures[0]; - double spatialScale = 1.0 / 16.0; + var currentBoxes = initialProposals.Count == 0 ? new Tensor(new[] { 0, 4 }) : initialProposals[0].boxes; + if (extraProposals is { Count: > 0 }) + currentBoxes = TwoStageTargets.AppendBoxes(currentBoxes, extraProposals); + if (currentBoxes.Shape[0] == 0) + return null; - // Current boxes to refine - var currentBoxes = initialProposals[0].boxes; - Tensor? classLogits = null; - Tensor? boxDeltas = null; - - // Cascade through stages + var stages = new List(_numStages); for (int stageIdx = 0; stageIdx < _numStages; stageIdx++) { - // Extract RoI features for current boxes - var roiFeatures = _roiAlign.Forward(p4Features, currentBoxes, spatialScale); - - // Flatten RoI features + // Extract RoI features for current boxes, each from its size-matched pyramid level (the level can + // change between stages as refinement resizes the boxes) + var roiFeatures = FpnRoIPooler.Pool(_roiAlign, fpnFeatures, EnsureBackbone.Strides, currentBoxes); var flattenedFeatures = FlattenRoIFeatures(roiFeatures); - - // Run cascade stage - var stage = _stages[stageIdx]; - (classLogits, boxDeltas) = stage.Forward(flattenedFeatures); - + var (classLogits, boxDeltas) = _stages[stageIdx].Forward(flattenedFeatures); if (boxDeltas is null) - { throw new InvalidOperationException("Cascade stage did not produce box deltas."); - } + stages.Add(new CascadeStageOutput(classLogits, boxDeltas, currentBoxes)); - // Refine boxes for next stage (except for last stage) + // Refine boxes for the next stage. The boxes are constants to RoIAlign, so refinement carries no + // gradient; each stage trains through its own logits and deltas above. if (stageIdx < _numStages - 1) - { currentBoxes = RefineBoxes(currentBoxes, boxDeltas, imageWidth, imageHeight); - } } + return stages; + } - if (classLogits is null || boxDeltas is null) + /// One cascade stage's raw heads and the boxes it classified. + private sealed class CascadeStageOutput + { + internal CascadeStageOutput(Tensor classLogits, Tensor boxDeltas, Tensor boxes) { - throw new InvalidOperationException("Cascade RCNN requires at least one stage to produce outputs."); + ClassLogits = classLogits; + BoxDeltas = boxDeltas; + Boxes = boxes; } - return new List> { classLogits, boxDeltas, currentBoxes }; + internal Tensor ClassLogits { get; } + internal Tensor BoxDeltas { get; } + internal Tensor Boxes { get; } } /// @@ -276,10 +374,13 @@ protected override List> PostProcess( double predW = pw * Math.Exp(Math.Min(dw, 4.0)); double predH = ph * Math.Exp(Math.Min(dh, 4.0)); - double x1 = Math.Max(0, predCx - predW / 2); - double y1 = Math.Max(0, predCy - predH / 2); - double x2 = Math.Min(imageWidth, predCx + predW / 2); - double y2 = Math.Min(imageHeight, predCy + predH / 2); + // Decoded in network-input coordinates; map to the source image before clipping. + // (RefineBoxes does NOT do this: it works on proposals in the input frame on purpose.) + var (scaleX, scaleY) = InputToImageScale(imageWidth, imageHeight); + double x1 = Math.Max(0, (predCx - predW / 2) * scaleX); + double y1 = Math.Max(0, (predCy - predH / 2) * scaleY); + double x2 = Math.Min(imageWidth, (predCx + predW / 2) * scaleX); + double y2 = Math.Min(imageHeight, (predCy + predH / 2) * scaleY); if (x2 <= x1 || y2 <= y1) continue; @@ -402,84 +503,65 @@ public override void SaveWeights(string path) } private Tensor FlattenRoIFeatures(Tensor roiFeatures) - { - int numRois = roiFeatures.Shape[0]; - int channels = roiFeatures.Shape[1]; - int h = roiFeatures.Shape[2]; - int w = roiFeatures.Shape[3]; - int flattenedSize = channels * h * w; - - var result = new Tensor(new[] { numRois, flattenedSize }); - - for (int roi = 0; roi < numRois; roi++) - { - int idx = 0; - for (int c = 0; c < channels; c++) - { - for (int y = 0; y < h; y++) - { - for (int x = 0; x < w; x++) - { - result[roi, idx++] = roiFeatures[roi, c, y, x]; - } - } - } - } - - return result; - } + => AiDotNetEngine.Current.Reshape( + roiFeatures, new[] { roiFeatures.Shape[0], roiFeatures.Shape[1] * roiFeatures.Shape[2] * roiFeatures.Shape[3] }); - private Tensor RefineBoxes(Tensor boxes, Tensor deltas, int imageWidth, int imageHeight) + /// + /// Applies one stage's box deltas to its input boxes, giving the next stage's boxes. + /// + /// Input boxes [N, 4] as (x1, y1, x2, y2) in network-input coordinates. + /// The stage's regression output [N, 4 * numClasses]; the first + /// foreground class's (dx, dy, dw, dh) are applied. + /// Right clip bound. + /// Bottom clip bound. + /// The refined boxes [N, 4], still in network-input coordinates. + /// + /// Engine ops over whole columns rather than a per-box scalar loop. The refined boxes only tell + /// the next stage's RoIAlign WHERE to sample, and RoIAlign treats box coordinates as data, so no + /// gradient flows back through them - the same "detached proposals" rule as Cai and Vasconcelos + /// (2018) and detectron2's cascade head. The deltas themselves still reach the loss through the + /// stage outputs returns. + /// + internal static Tensor RefineBoxes(Tensor boxes, Tensor deltas, int imageWidth, int imageHeight) { - int numBoxes = boxes.Shape[0]; - int numClasses = deltas.Shape[1] / 4; - - var refinedBoxes = new Tensor(new[] { numBoxes, 4 }); - - for (int i = 0; i < numBoxes; i++) - { - double px1 = NumOps.ToDouble(boxes[i, 0]); - double py1 = NumOps.ToDouble(boxes[i, 1]); - double px2 = NumOps.ToDouble(boxes[i, 2]); - double py2 = NumOps.ToDouble(boxes[i, 3]); - - double pw = px2 - px1; - double ph = py2 - py1; - double pcx = px1 + pw / 2; - double pcy = py1 + ph / 2; - - // Use class-agnostic refinement (average across all classes) - // or use the most likely class - here we use first non-background class - int deltaOffset = 4; // Skip background class - double dx = NumOps.ToDouble(deltas[i, deltaOffset]); - double dy = NumOps.ToDouble(deltas[i, deltaOffset + 1]); - double dw = NumOps.ToDouble(deltas[i, deltaOffset + 2]); - double dh = NumOps.ToDouble(deltas[i, deltaOffset + 3]); - - double predCx = pcx + dx * pw; - double predCy = pcy + dy * ph; - double predW = pw * Math.Exp(Math.Min(dw, 4.0)); - double predH = ph * Math.Exp(Math.Min(dh, 4.0)); - - double x1 = Math.Max(0, predCx - predW / 2); - double y1 = Math.Max(0, predCy - predH / 2); - double x2 = Math.Min(imageWidth, predCx + predW / 2); - double y2 = Math.Min(imageHeight, predCy + predH / 2); - - refinedBoxes[i, 0] = NumOps.FromDouble(x1); - refinedBoxes[i, 1] = NumOps.FromDouble(y1); - refinedBoxes[i, 2] = NumOps.FromDouble(x2); - refinedBoxes[i, 3] = NumOps.FromDouble(y2); - } - - return refinedBoxes; + var engine = AiDotNetEngine.Current; + var ops = MathHelper.GetNumericOperations(); + Tensor Column(Tensor source, int index) => engine.TensorNarrow(source, 1, index, 1); + var half = ops.FromDouble(0.5); + var unbounded = ops.FromDouble(double.MinValue); + + var px1 = Column(boxes, 0); + var py1 = Column(boxes, 1); + var pw = engine.TensorSubtract(Column(boxes, 2), px1); + var ph = engine.TensorSubtract(Column(boxes, 3), py1); + var pcx = engine.TensorAdd(px1, engine.TensorMultiplyScalar(pw, half)); + var pcy = engine.TensorAdd(py1, engine.TensorMultiplyScalar(ph, half)); + + // Deltas of the first foreground class (columns 4..7; class 0 is background). The scale + // deltas are capped at 4 before exponentiating, as in the per-box version this replaces. + const int deltaOffset = 4; + var predCx = engine.TensorAdd(pcx, engine.TensorMultiply(Column(deltas, deltaOffset), pw)); + var predCy = engine.TensorAdd(pcy, engine.TensorMultiply(Column(deltas, deltaOffset + 1), ph)); + var cap = ops.FromDouble(4.0); + var predW = engine.TensorMultiply(pw, engine.TensorExp(engine.TensorClamp(Column(deltas, deltaOffset + 2), unbounded, cap))); + var predH = engine.TensorMultiply(ph, engine.TensorExp(engine.TensorClamp(Column(deltas, deltaOffset + 3), unbounded, cap))); + var halfW = engine.TensorMultiplyScalar(predW, half); + var halfH = engine.TensorMultiplyScalar(predH, half); + + // Clip each edge on its own side only: x1/y1 at zero, x2/y2 at the image extent. + var x1 = engine.TensorClampMin(engine.TensorSubtract(predCx, halfW), ops.Zero); + var y1 = engine.TensorClampMin(engine.TensorSubtract(predCy, halfH), ops.Zero); + var x2 = engine.TensorClamp(engine.TensorAdd(predCx, halfW), unbounded, ops.FromDouble(imageWidth)); + var y2 = engine.TensorClamp(engine.TensorAdd(predCy, halfH), unbounded, ops.FromDouble(imageHeight)); + + return engine.TensorConcatenate(new[] { x1, y1, x2, y2 }, 1); } } /// /// A single stage in the Cascade R-CNN pipeline. /// -internal class CascadeStage +internal class CascadeStage : CvParameterModule { private readonly INumericOperations _numOps; private readonly Dense _fc1; @@ -559,14 +641,23 @@ public void ReadParameters(BinaryReader reader) _regHead.ReadParameters(reader); } - private Tensor ApplyReLU(Tensor x) + /// + /// Elementwise ReLU, delegated to the engine. + /// + /// + /// This was a scalar loop that read each element out to double and wrote a fresh + /// tensor. Arithmetically identical, but it severed the autodiff tape: the gradient chain + /// stopped here, so every trainable layer UPSTREAM of this call received no gradient and + /// silently never trained. The engine op records itself on the tape. + /// + private Tensor ApplyReLU(Tensor x) => AiDotNetEngine.Current.ReLU(x); + + /// + protected override IEnumerable?> ParameterChildren() { - var result = new Tensor(x._shape); - for (int i = 0; i < x.Length; i++) - { - double val = _numOps.ToDouble(x[i]); - result[i] = _numOps.FromDouble(Math.Max(0, val)); - } - return result; + yield return _fc1; + yield return _fc2; + yield return _clsHead; + yield return _regHead; } } diff --git a/src/ComputerVision/Detection/ObjectDetection/RCNN/FasterRCNN.cs b/src/ComputerVision/Detection/ObjectDetection/RCNN/FasterRCNN.cs index 4dbfc4bba9..9de98630d1 100644 --- a/src/ComputerVision/Detection/ObjectDetection/RCNN/FasterRCNN.cs +++ b/src/ComputerVision/Detection/ObjectDetection/RCNN/FasterRCNN.cs @@ -41,8 +41,12 @@ namespace AiDotNet.ComputerVision.Detection.ObjectDetection.RCNN; "https://arxiv.org/abs/1506.01497", Year = 2015, Authors = "Shaoqing Ren, Kaiming He, Ross Girshick, Jian Sun")] -public class FasterRCNN : ObjectDetectorBase +public partial class FasterRCNN : ObjectDetectorBase, IDetectionTrainingModel { + private readonly AiDotNet.ComputerVision.Detection.Losses.TwoStageDetectionLoss _detectionLoss; + + [AiDotNet.Attributes.Scratch] + private Random? _trainingRandom; private readonly RPN _rpn; private readonly RoIAlign _roiAlign; private readonly Dense _fcClassifier; @@ -99,6 +103,8 @@ public FasterRCNN(ObjectDetectionOptions options) : base(options) _fcBoxRegressor = new Dense(roiFeatureSize, (options.NumClasses + 1) * 4); _nms = new NMS(); + _detectionLoss = new AiDotNet.ComputerVision.Detection.Losses.TwoStageDetectionLoss(options.NumClasses, 1, + options.TwoStageLoss ?? new AiDotNet.ComputerVision.Detection.Losses.TwoStageDetectionLossOptions()); } private static (int hiddenDim, int roiOutputSize) GetSizeConfig(ModelSize size) => size switch @@ -133,7 +139,50 @@ public override DetectionResult Detect(Tensor image, double confidenceThre } /// - protected override List> Forward(Tensor input) + protected override List> Forward(Tensor input) => ForwardDetection(input, null, out _); + + /// Trains the proposal network and the detection head with the Faster R-CNN objectives. + /// + /// + /// One update sums the region proposal loss (Ren et al. 2015: IoU above 0.7 or best anchor positive, below 0.3 + /// negative, 256 anchors at up to 1:1, lambda = 10 over the anchor locations) and the region-of-interest loss + /// (Girshick 2015: 64 RoIs with 25% foreground at IoU of at least 0.5, background in [0.1, 0.5), smooth-L1 + /// regression). As in the reference implementation, the object boxes are added to the proposals the head + /// learns from. Override the settings with . + /// + /// + /// Inputs are model-ready NCHW tensors, as for Predict, and targets are normalized against that input size. The + /// detection head here classifies the proposals of a single image per forward pass, so each step takes one image. + /// + /// + public void TrainDetections(Tensor input, DetectionTrainingBatch targets) + { + if (input is null) throw new ArgumentNullException(nameof(input)); + if (targets is null) throw new ArgumentNullException(nameof(targets)); + if (input.Rank != 4 || input.Shape[1] != 3 || input.Shape[2] <= 0 || input.Shape[3] <= 0) + throw new ArgumentException("Faster R-CNN training requires a three-channel NCHW image batch.", nameof(input)); + if (input.Shape[0] != 1) + throw new ArgumentException("Faster R-CNN's detection head classifies one image's proposals per forward pass; train one image per step.", nameof(input)); + targets.ValidateForModel(1, Options.NumClasses, int.MaxValue); + int height = input.Shape[2]; + int width = input.Shape[3]; + var gold = targets[0].Select(target => TwoStageTargets.PixelCorners(target, width, height)).ToList(); + var goldClasses = targets[0].Select(target => target.ClassId).ToList(); + var random = _trainingRandom ??= Options.RandomSeed is int seed + ? AiDotNet.Tensors.Helpers.RandomHelper.CreateSeededRandom(seed) + : AiDotNet.Tensors.Helpers.RandomHelper.CreateSecureRandom(); + List> anchors = new(); + TrainWithTargets(input, targets, + image => ForwardDetection(image, gold, out anchors), + (heads, batch) => Engine.TensorAdd( + _detectionLoss.ComputeProposalLoss(TwoStageTargets.FirstImage(heads[3]), TwoStageTargets.FirstImage(heads[4]), + anchors, gold, _rpn.AnchorsPerLocation, random), + _detectionLoss.ComputeStageLoss(heads[0], heads[1], heads[2], gold, goldClasses, 0, random))); + } + + /// The detection forward, optionally adding boxes to the proposals the detection head classifies. + private List> ForwardDetection(Tensor input, IReadOnlyList? extraProposals, + out List> anchors) { int imageHeight = input.Shape[2]; int imageWidth = input.Shape[3]; @@ -144,39 +193,41 @@ protected override List> Forward(Tensor input) // Apply FPN neck to get multi-scale features var fpnFeatures = EnsureNeck.Forward(backboneFeatures); - // Use P4 level for RPN (good balance of resolution and receptive field) - var rpnFeatures = fpnFeatures.Count > 1 ? fpnFeatures[1] : fpnFeatures[0]; - - // Stage 1: Region Proposal Network - var (objectness, bboxDeltas, anchors) = _rpn.Forward(rpnFeatures); + // Faster R-CNN with FPN (Lin et al. 2017; detectron2, torchvision): the shared RPN head runs on + // every level P2-P5 plus P6 (P5 subsampled by 2), each with its own anchor size, and each RoI + // is pooled from the level matching its size. + var rpnLevels = new List>(fpnFeatures) { CvTensorOps.MaxPoolPadded(fpnFeatures[^1], 1, 2, 0) }; + var (objectness, bboxDeltas, levelAnchors, levelAnchorCounts) = _rpn.ForwardLevels(rpnLevels); + anchors = levelAnchors; - // Generate proposals + // Generate proposals: top 1000 per level, NMS within each level, best 1000 overall. var proposals = _rpn.GenerateProposals( - objectness, bboxDeltas, anchors, + objectness, bboxDeltas, levelAnchors, imageHeight, imageWidth, - preNmsTopK: 2000, + preNmsTopK: 1000, postNmsTopK: 1000, - nmsThreshold: 0.7); + nmsThreshold: 0.7, + levelAnchorCounts: levelAnchorCounts); - if (proposals.Count == 0 || proposals[0].boxes.Shape[0] == 0) + var proposalBoxes = proposals.Count == 0 ? new Tensor(new[] { 0, 4 }) : proposals[0].boxes; + if (extraProposals is { Count: > 0 }) + proposalBoxes = TwoStageTargets.AppendBoxes(proposalBoxes, extraProposals); + + if (proposalBoxes.Shape[0] == 0) { // No proposals, return empty result return new List> { new Tensor(new[] { 0, Options.NumClasses + 1 }), new Tensor(new[] { 0, (Options.NumClasses + 1) * 4 }), - new Tensor(new[] { 0, 4 }) + new Tensor(new[] { 0, 4 }), + objectness, + bboxDeltas }; } - var proposalBoxes = proposals[0].boxes; - - // Stage 2: RoI feature extraction and classification - // Use P4 features for RoI Align - var p4Features = fpnFeatures.Count > 1 ? fpnFeatures[1] : fpnFeatures[0]; - double spatialScale = 1.0 / 16.0; // P4 is typically 1/16 resolution - - var roiFeatures = _roiAlign.Forward(p4Features, proposalBoxes, spatialScale); + // Stage 2: RoI feature extraction from the size-matched pyramid level, then classification + var roiFeatures = FpnRoIPooler.Pool(_roiAlign, fpnFeatures, EnsureBackbone.Strides, proposalBoxes); // Flatten RoI features: [num_rois, channels, H, W] -> [num_rois, channels*H*W] var flattenedFeatures = FlattenRoIFeatures(roiFeatures); @@ -185,7 +236,10 @@ protected override List> Forward(Tensor input) var classLogits = _fcClassifier.Forward(flattenedFeatures); var boxDeltas = _fcBoxRegressor.Forward(flattenedFeatures); - return new List> { classLogits, boxDeltas, proposalBoxes }; + // The RPN's raw objectness and box deltas are outputs too: proposal selection is a + // non-differentiable top-k, so without them nothing would train the RPN. PostProcess reads + // only the first three entries. + return new List> { classLogits, boxDeltas, proposalBoxes, objectness, bboxDeltas }; } /// @@ -271,10 +325,12 @@ protected override List> PostProcess( double predH = ph * Math.Exp(Math.Min(dh, 4.0)); // Convert to (x1, y1, x2, y2) and clip - double x1 = Math.Max(0, predCx - predW / 2); - double y1 = Math.Max(0, predCy - predH / 2); - double x2 = Math.Min(imageWidth, predCx + predW / 2); - double y2 = Math.Min(imageHeight, predCy + predH / 2); + // Decoded in network-input coordinates; map to the source image before clipping. + var (scaleX, scaleY) = InputToImageScale(imageWidth, imageHeight); + double x1 = Math.Max(0, (predCx - predW / 2) * scaleX); + double y1 = Math.Max(0, (predCy - predH / 2) * scaleY); + double x2 = Math.Min(imageWidth, (predCx + predW / 2) * scaleX); + double y2 = Math.Min(imageHeight, (predCy + predH / 2) * scaleY); if (x2 <= x1 || y2 <= y1) continue; @@ -386,30 +442,6 @@ public override void SaveWeights(string path) } private Tensor FlattenRoIFeatures(Tensor roiFeatures) - { - int numRois = roiFeatures.Shape[0]; - int channels = roiFeatures.Shape[1]; - int h = roiFeatures.Shape[2]; - int w = roiFeatures.Shape[3]; - int flattenedSize = channels * h * w; - - var result = new Tensor(new[] { numRois, flattenedSize }); - - for (int roi = 0; roi < numRois; roi++) - { - int idx = 0; - for (int c = 0; c < channels; c++) - { - for (int y = 0; y < h; y++) - { - for (int x = 0; x < w; x++) - { - result[roi, idx++] = roiFeatures[roi, c, y, x]; - } - } - } - } - - return result; - } + => AiDotNetEngine.Current.Reshape( + roiFeatures, new[] { roiFeatures.Shape[0], roiFeatures.Shape[1] * roiFeatures.Shape[2] * roiFeatures.Shape[3] }); } diff --git a/src/ComputerVision/Detection/ObjectDetection/RCNN/FpnRoIPooler.cs b/src/ComputerVision/Detection/ObjectDetection/RCNN/FpnRoIPooler.cs new file mode 100644 index 0000000000..0f1ac99016 --- /dev/null +++ b/src/ComputerVision/Detection/ObjectDetection/RCNN/FpnRoIPooler.cs @@ -0,0 +1,162 @@ +using AiDotNet.Tensors.Engines; + +namespace AiDotNet.ComputerVision.Detection.ObjectDetection.RCNN; + +/// +/// Pools each region of interest from the feature-pyramid level that matches its size. +/// +/// +/// +/// Feature Pyramid Networks (Lin et al. 2017, eq. 1) assign a box of width w and height +/// h (in input pixels) to level k = floor(k0 + log2(sqrt(w h) / 224)) with +/// k0 = 4, clamped to the available levels: small boxes read the fine, high-resolution maps +/// and large boxes the coarse ones. This is the rule detectron2 and torchvision implement +/// (canonical box size 224 at canonical level 4). +/// +/// +/// The boxes are sampling coordinates, constant to the gradient as in standard RoIAlign; the pooled +/// features stay on the gradient tape, and the per-level results are put back in the caller's box +/// order with an engine gather. +/// +/// +/// The numeric type. +internal static class FpnRoIPooler +{ + private const double CanonicalBoxSize = 224.0; + private const int CanonicalLevel = 4; + + /// + /// Assigns each box to a pyramid level. + /// + /// Boxes [N, 4] as (x1, y1, x2, y2) in input pixels. + /// The stride of each available level, finest first (for example 4, 8, 16, 32). + /// For each box, the index into of its level. + internal static int[] AssignLevels(Tensor boxes, IReadOnlyList strides) + { + ValidateStrides(strides); + var ops = MathHelper.GetNumericOperations(); + int minLevel = Log2(strides[0]); + int maxLevel = Log2(strides[strides.Count - 1]); + var assignment = new int[boxes.Shape[0]]; + for (int i = 0; i < assignment.Length; i++) + { + double w = ops.ToDouble(boxes[i, 2]) - ops.ToDouble(boxes[i, 0]); + double h = ops.ToDouble(boxes[i, 3]) - ops.ToDouble(boxes[i, 1]); + double size = Math.Sqrt(Math.Max(w, 0) * Math.Max(h, 0)); + + // + 1e-8 as in detectron2, so a degenerate box maps to the finest level rather than -inf. + int level = (int)Math.Floor(CanonicalLevel + Math.Log(size / CanonicalBoxSize + 1e-8, 2)); + assignment[i] = Math.Min(Math.Max(level, minLevel), maxLevel) - minLevel; + } + + return assignment; + } + + /// + /// Pools every box from its assigned level. + /// + /// The RoIAlign operator (output size and sampling ratio). + /// Pyramid feature maps, finest first, one per stride. + /// The stride of each level. + /// Boxes [N, 4] in input pixels. + /// Pooled features [N, channels, outputSize, outputSize] in the order of . + public static Tensor Pool(RoIAlign align, IReadOnlyList> levels, IReadOnlyList strides, Tensor boxes) + { + if (strides is null) throw new ArgumentNullException(nameof(strides)); + if (levels.Count != strides.Count) + { + throw new ArgumentException( + $"{levels.Count} pyramid levels but {strides.Count} strides; they must correspond one to one.", + nameof(strides)); + } + + var ops = MathHelper.GetNumericOperations(); + var assignment = AssignLevels(boxes, strides); + + var parts = new List>(); + var order = new List(assignment.Length); + for (int level = 0; level < levels.Count; level++) + { + var members = new List(); + for (int i = 0; i < assignment.Length; i++) + { + if (assignment[i] == level) + { + members.Add(i); + } + } + + if (members.Count == 0) + { + continue; + } + + var subset = new Tensor(new[] { members.Count, 4 }); + for (int m = 0; m < members.Count; m++) + { + for (int c = 0; c < 4; c++) + { + subset[m, c] = ops.FromDouble(ops.ToDouble(boxes[members[m], c])); + } + } + + parts.Add(align.Forward(levels[level], subset, 1.0 / strides[level])); + order.AddRange(members); + } + + var pooled = parts.Count == 1 ? parts[0] : AiDotNetEngine.Current.TensorConcatenate(parts.ToArray(), 0); + + // pooled row r holds box order[r]; gather it back so row i holds box i. + var positionOf = new int[order.Count]; + bool identity = true; + for (int r = 0; r < order.Count; r++) + { + positionOf[order[r]] = r; + identity &= order[r] == r; + } + + return identity ? pooled : CvTensorOps.Select(pooled, positionOf, 0); + } + + private static void ValidateStrides(IReadOnlyList strides) + { + if (strides is null) throw new ArgumentNullException(nameof(strides)); + if (strides.Count == 0) + { + throw new ArgumentException("At least one pyramid stride is required.", nameof(strides)); + } + + int previous = 0; + for (int i = 0; i < strides.Count; i++) + { + int stride = strides[i]; + if (stride <= 0 || (stride & (stride - 1)) != 0) + { + throw new ArgumentException( + $"Pyramid strides must be positive powers of two; got {stride}.", nameof(strides)); + } + + if (i > 0 && stride != 2L * previous) + { + throw new ArgumentException( + "Each pyramid stride must be exactly double the previous stride.", nameof(strides)); + } + + previous = stride; + } + } + + private static int Log2(int stride) + { + // Shift the value down, rather than shifting 1 past the signed-int boundary. + // The latter wraps its shift count and can loop forever on a large invalid stride. + int level = 0; + while (stride > 1) + { + stride >>= 1; + level++; + } + + return level; + } +} diff --git a/src/ComputerVision/Detection/ObjectDetection/RCNN/RPN.cs b/src/ComputerVision/Detection/ObjectDetection/RCNN/RPN.cs index 216c268662..7e07e21b6b 100644 --- a/src/ComputerVision/Detection/ObjectDetection/RCNN/RPN.cs +++ b/src/ComputerVision/Detection/ObjectDetection/RCNN/RPN.cs @@ -1,3 +1,4 @@ +using AiDotNet.Tensors.Engines; using System.IO; using AiDotNet.Augmentation.Image; using AiDotNet.ComputerVision.Detection.Anchors; @@ -25,7 +26,7 @@ namespace AiDotNet.ComputerVision.Detection.ObjectDetection.RCNN; /// Reference: Ren et al., "Faster R-CNN: Towards Real-Time Object Detection with /// Region Proposal Networks", NeurIPS 2015 /// -public class RPN +public class RPN : IParameterSource, AiDotNet.Models.Parameters.IParameterChunkSource, AiDotNet.Models.Parameters.IParameterLayoutSource { private readonly INumericOperations _numOps; private readonly Conv2D _conv; @@ -36,6 +37,8 @@ public class RPN private readonly int _numAnchors; private readonly int _featureStride; private readonly double _baseAnchorSize; + private readonly int[] _levelStrides; + private readonly double[] _levelBaseSizes; /// /// Gets the anchor generator used by this RPN. @@ -89,37 +92,93 @@ public RPN(int inChannels, int hiddenDim = 256, int[]? anchorSizes = null, doubl } _featureStride = strides[level]; _baseAnchorSize = baseSizes[level]; + _levelStrides = strides; + _levelBaseSizes = baseSizes; } + /// + /// Gets the number of pyramid levels the RPN has anchors for (one per anchor size). + /// + public int LevelCount => _levelStrides.Length; + + /// Anchor shapes laid out at every feature position (aspect ratios times scales). + internal int AnchorsPerLocation => _numAnchors; + /// /// Forward pass through the RPN. /// /// Feature map from backbone [batch, channels, height, width]. /// Tuple of (objectness logits, bbox deltas, anchors as list of BoundingBox). public (Tensor objectness, Tensor bboxDeltas, List> anchors) Forward(Tensor features) + { + var (objectness, bboxDeltas) = Head(features); + + // Generate anchors for this feature map size using configured stride and base size + var anchors = _anchorGenerator.GenerateAnchorsForLevel( + features.Shape[2], features.Shape[3], stride: _featureStride, baseSize: _baseAnchorSize); + + return (objectness, bboxDeltas, anchors); + } + + /// + /// Runs the shared RPN head over every level of a feature pyramid. + /// + /// Pyramid levels, finest first (P2, P3, ...), one per anchor size. + /// + /// Objectness [batch, totalAnchors, 2] and deltas [batch, totalAnchors, 4] with the + /// levels concatenated finest first, the matching anchors, and how many anchors each level has. + /// + /// + /// The FPN form of the RPN (Lin et al. 2017; detectron2, torchvision): ONE head, shared across + /// levels, with anchors of a single size per level - level i uses the i-th anchor size at + /// the i-th stride. Each level's anchors are laid out at that level's own stride. + /// + public (Tensor objectness, Tensor bboxDeltas, List> anchors, int[] levelAnchorCounts) ForwardLevels( + IReadOnlyList> levels) + { + if (levels is null || levels.Count == 0) + { + throw new ArgumentException("At least one pyramid level is required.", nameof(levels)); + } + + if (levels.Count > _levelStrides.Length) + { + throw new ArgumentException( + $"The RPN has anchors for {_levelStrides.Length} levels but received {levels.Count}.", nameof(levels)); + } + + var objectness = new Tensor[levels.Count]; + var deltas = new Tensor[levels.Count]; + var anchors = new List>(); + var counts = new int[levels.Count]; + for (int l = 0; l < levels.Count; l++) + { + (objectness[l], deltas[l]) = Head(levels[l]); + var levelAnchors = _anchorGenerator.GenerateAnchorsForLevel( + levels[l].Shape[2], levels[l].Shape[3], stride: _levelStrides[l], baseSize: _levelBaseSizes[l]); + anchors.AddRange(levelAnchors); + counts[l] = levelAnchors.Count; + } + + var engine = AiDotNetEngine.Current; + return levels.Count == 1 + ? (objectness[0], deltas[0], anchors, counts) + : (engine.TensorConcatenate(objectness, 1), engine.TensorConcatenate(deltas, 1), anchors, counts); + } + + private (Tensor Objectness, Tensor Deltas) Head(Tensor features) { int batch = features.Shape[0]; int height = features.Shape[2]; int width = features.Shape[3]; // Shared convolution with ReLU - var x = _conv.Forward(features); - x = ApplyReLU(x); - - // Get objectness scores - var objectness = _clsHead.Forward(x); - // Reshape: [B, numAnchors*2, H, W] -> [B, H*W*numAnchors, 2] - objectness = ReshapeRPNOutput(objectness, batch, height, width, 2); + var x = ApplyReLU(_conv.Forward(features)); - // Get bbox deltas - var bboxDeltas = _regHead.Forward(x); - // Reshape: [B, numAnchors*4, H, W] -> [B, H*W*numAnchors, 4] - bboxDeltas = ReshapeRPNOutput(bboxDeltas, batch, height, width, 4); - - // Generate anchors for this feature map size using configured stride and base size - var anchors = _anchorGenerator.GenerateAnchorsForLevel(height, width, stride: _featureStride, baseSize: _baseAnchorSize); - - return (objectness, bboxDeltas, anchors); + // [B, numAnchors*2, H, W] -> [B, H*W*numAnchors, 2] and [B, numAnchors*4, H, W] -> [B, H*W*numAnchors, 4] + var objectness = ReshapeRPNOutput(_clsHead.Forward(x), batch, height, width, 2); + var bboxDeltas = ReshapeRPNOutput(_regHead.Forward(x), batch, height, width, 4); + return (objectness, bboxDeltas); } /// @@ -133,6 +192,13 @@ public RPN(int inChannels, int hiddenDim = 256, int[]? anchorSizes = null, doubl /// Maximum proposals before NMS. /// Maximum proposals after NMS. /// IoU threshold for NMS. + /// + /// Anchors per pyramid level, as returned by . When given, the top + /// are taken and NMS is applied WITHIN each level, then the best + /// are kept across levels - the FPN proposal rule, which stops the + /// many fine-level anchors from crowding out the coarse levels. When null, all anchors form one + /// group. + /// /// Proposal boxes [num_proposals, 4] as (x1, y1, x2, y2). public List<(Tensor boxes, Tensor scores)> GenerateProposals( Tensor objectness, @@ -142,7 +208,8 @@ public RPN(int inChannels, int hiddenDim = 256, int[]? anchorSizes = null, doubl int imageWidth, int preNmsTopK = 2000, int postNmsTopK = 1000, - double nmsThreshold = 0.7) + double nmsThreshold = 0.7, + int[]? levelAnchorCounts = null) { int batch = objectness.Shape[0]; int objectnessAnchors = objectness.Shape[1]; @@ -175,55 +242,73 @@ public RPN(int inChannels, int hiddenDim = 256, int[]? anchorSizes = null, doubl scores[i] = Math.Exp(obj - maxVal) / sumExp; } - // Get top-k proposals before NMS - var indices = Enumerable.Range(0, numAnchors) - .OrderByDescending(i => scores[i]) - .Take(preNmsTopK) - .ToList(); + var groups = levelAnchorCounts ?? new[] { numAnchors }; + if (groups.Sum() != numAnchors) + { + throw new ArgumentException( + $"Level anchor counts sum to {groups.Sum()}, but there are {numAnchors} anchors.", + nameof(levelAnchorCounts)); + } - // Decode boxes - var decodedBoxes = new List<(double x1, double y1, double x2, double y2, double score, int idx)>(); - foreach (int i in indices) + var kept = new List<(double x1, double y1, double x2, double y2, double score)>(); + int groupStart = 0; + foreach (int groupSize in groups) { - // Get anchor - BoundingBox stores (x1, y1, x2, y2) in XYXY format - var anchor = anchors[i]; - double ax1 = _numOps.ToDouble(anchor.X1); - double ay1 = _numOps.ToDouble(anchor.Y1); - double ax2 = _numOps.ToDouble(anchor.X2); - double ay2 = _numOps.ToDouble(anchor.Y2); - double aw = ax2 - ax1; - double ah = ay2 - ay1; - - // Get deltas - double dx = _numOps.ToDouble(bboxDeltas[b, i, 0]); - double dy = _numOps.ToDouble(bboxDeltas[b, i, 1]); - double dw = _numOps.ToDouble(bboxDeltas[b, i, 2]); - double dh = _numOps.ToDouble(bboxDeltas[b, i, 3]); - - // Anchor center - double cx = ax1 + aw / 2; - double cy = ay1 + ah / 2; - - // Apply deltas (standard bbox encoding) - double predCx = cx + dx * aw; - double predCy = cy + dy * ah; - double predW = aw * Math.Exp(Math.Min(dw, 4.0)); // Clip to prevent explosion - double predH = ah * Math.Exp(Math.Min(dh, 4.0)); - - // Convert to (x1, y1, x2, y2) - double x1 = Math.Max(0, predCx - predW / 2); - double y1 = Math.Max(0, predCy - predH / 2); - double x2 = Math.Min(imageWidth, predCx + predW / 2); - double y2 = Math.Min(imageHeight, predCy + predH / 2); - - if (x2 > x1 && y2 > y1) + int start = groupStart; + groupStart += groupSize; + + // Get top-k proposals before NMS + var indices = Enumerable.Range(start, groupSize) + .OrderByDescending(i => scores[i]) + .Take(preNmsTopK) + .ToList(); + + // Decode boxes + var decodedBoxes = new List<(double x1, double y1, double x2, double y2, double score, int idx)>(); + foreach (int i in indices) { - decodedBoxes.Add((x1, y1, x2, y2, scores[i], i)); + // Get anchor - BoundingBox stores (x1, y1, x2, y2) in XYXY format + var anchor = anchors[i]; + double ax1 = _numOps.ToDouble(anchor.X1); + double ay1 = _numOps.ToDouble(anchor.Y1); + double ax2 = _numOps.ToDouble(anchor.X2); + double ay2 = _numOps.ToDouble(anchor.Y2); + double aw = ax2 - ax1; + double ah = ay2 - ay1; + + // Get deltas + double dx = _numOps.ToDouble(bboxDeltas[b, i, 0]); + double dy = _numOps.ToDouble(bboxDeltas[b, i, 1]); + double dw = _numOps.ToDouble(bboxDeltas[b, i, 2]); + double dh = _numOps.ToDouble(bboxDeltas[b, i, 3]); + + // Anchor center + double cx = ax1 + aw / 2; + double cy = ay1 + ah / 2; + + // Apply deltas (standard bbox encoding) + double predCx = cx + dx * aw; + double predCy = cy + dy * ah; + double predW = aw * Math.Exp(Math.Min(dw, 4.0)); // Clip to prevent explosion + double predH = ah * Math.Exp(Math.Min(dh, 4.0)); + + // Convert to (x1, y1, x2, y2) + double x1 = Math.Max(0, predCx - predW / 2); + double y1 = Math.Max(0, predCy - predH / 2); + double x2 = Math.Min(imageWidth, predCx + predW / 2); + double y2 = Math.Min(imageHeight, predCy + predH / 2); + + if (x2 > x1 && y2 > y1) + { + decodedBoxes.Add((x1, y1, x2, y2, scores[i], i)); + } } + + // Apply NMS + kept.AddRange(ApplyNMS(decodedBoxes, nmsThreshold, postNmsTopK)); } - // Apply NMS - var nmsBoxes = ApplyNMS(decodedBoxes, nmsThreshold, postNmsTopK); + var nmsBoxes = kept.OrderByDescending(box => box.score).Take(postNmsTopK).ToList(); // Convert to tensors int numProposals = nmsBoxes.Count; @@ -290,7 +375,7 @@ public void ReadParameters(BinaryReader reader) _regHead.ReadParameters(reader); } - private Tensor ReshapeRPNOutput(Tensor x, int batch, int height, int width, int outputDim) + internal static Tensor ReshapeRPNOutput(Tensor x, int batch, int height, int width, int outputDim) { int channelDim = x.Shape[1]; @@ -302,42 +387,25 @@ private Tensor ReshapeRPNOutput(Tensor x, int batch, int height, int width $"Expected channel dimension to be numAnchors * {outputDim}."); } + // [B, A*D, H, W] -> [B, A, D, H, W] -> [B, H, W, A, D] -> [B, H*W*A, D], as engine ops so the + // RPN heads stay on the gradient tape. int numAnchors = channelDim / outputDim; - var result = new Tensor(new[] { batch, height * width * numAnchors, outputDim }); - - for (int b = 0; b < batch; b++) - { - int idx = 0; - for (int h = 0; h < height; h++) - { - for (int w = 0; w < width; w++) - { - for (int a = 0; a < numAnchors; a++) - { - for (int d = 0; d < outputDim; d++) - { - int channelIdx = a * outputDim + d; - result[b, idx, d] = x[b, channelIdx, h, w]; - } - idx++; - } - } - } - } - - return result; + var engine = AiDotNetEngine.Current; + var split = engine.Reshape(x, new[] { batch, numAnchors, outputDim, height, width }); + var ordered = engine.TensorPermute(split, new[] { 0, 3, 4, 1, 2 }); + return engine.Reshape(ordered, new[] { batch, height * width * numAnchors, outputDim }); } - private Tensor ApplyReLU(Tensor x) - { - var result = new Tensor(x._shape); - for (int i = 0; i < x.Length; i++) - { - double val = _numOps.ToDouble(x[i]); - result[i] = _numOps.FromDouble(Math.Max(0, val)); - } - return result; - } + /// + /// Elementwise ReLU, delegated to the engine. + /// + /// + /// This was a scalar loop that read each element out to double and wrote a fresh + /// tensor. Arithmetically identical, but it severed the autodiff tape: the gradient chain + /// stopped here, so every trainable layer UPSTREAM of this call received no gradient and + /// silently never trained. The engine op records itself on the tape. + /// + private Tensor ApplyReLU(Tensor x) => AiDotNetEngine.Current.ReLU(x); private List<(double x1, double y1, double x2, double y2, double score)> ApplyNMS( List<(double x1, double y1, double x2, double y2, double score, int idx)> boxes, @@ -391,6 +459,31 @@ private double ComputeIoU( return union > 0 ? intersect / union : 0; } + + // The shared convolution and both heads, registered as live chunks. RPN is public, so it forwards + // the parameter interfaces to an internal module instead of deriving from one. Before this the + // generator could not see anything inside the RPN at all. + private DelegatingCvParameterModule? _parameters; + + private DelegatingCvParameterModule Parameters + => _parameters ??= new DelegatingCvParameterModule(() => new IParameterSource?[] { _conv, _clsHead, _regHead }); + + /// + long IParameterSource.ParameterCount => Parameters.ParameterCount; + + /// + IReadOnlyList AiDotNet.Models.Parameters.IParameterLayoutSource.GetParameterLayout() + => Parameters.GetParameterLayout(); + + /// + Vector IParameterSource.GetParameters() => Parameters.GetParameters(); + + /// + void IParameterSource.SetParameters(Vector parameters) => Parameters.SetParameters(parameters); + + /// + IEnumerable> AiDotNet.Models.Parameters.IParameterChunkSource.GetParameterStateChunks() + => Parameters.GetParameterStateChunks(); } /// @@ -432,88 +525,28 @@ public RoIAlign(int outputSize = 7, int samplingRatio = 2) public Tensor Forward(Tensor features, Tensor rois, double spatialScale = 1.0 / 16.0, int[]? batchIndices = null) { int batchSize = features.Shape[0]; - int channels = features.Shape[1]; - int featureH = features.Shape[2]; - int featureW = features.Shape[3]; int numRois = rois.Shape[0]; - var output = new Tensor(new[] { numRois, channels, _outputSize, _outputSize }); + // The boxes are constants to the gradient (as in standard RoIAlign), so they are read out once; + // the pooling itself is a tape-visible gather over the feature map. + var boxes = new double[numRois * 4]; + for (int i = 0; i < boxes.Length; i++) + { + boxes[i] = _numOps.ToDouble(rois[i]); + } + // A supplied batch index is clamped at BOTH ends: an index below zero reached RoIAlign unchanged + // and read before the first image, while only the upper bound was ever guarded. + var indices = new int[numRois]; for (int roiIdx = 0; roiIdx < numRois; roiIdx++) { - // Get batch index for this RoI (default to 0 if not provided) - int batchIdx = batchIndices is not null && roiIdx < batchIndices.Length - ? Math.Min(batchIndices[roiIdx], batchSize - 1) + indices[roiIdx] = batchIndices is not null && roiIdx < batchIndices.Length + ? Math.Max(0, Math.Min(batchIndices[roiIdx], batchSize - 1)) : 0; - - // Scale RoI to feature map coordinates - double x1 = _numOps.ToDouble(rois[roiIdx, 0]) * spatialScale; - double y1 = _numOps.ToDouble(rois[roiIdx, 1]) * spatialScale; - double x2 = _numOps.ToDouble(rois[roiIdx, 2]) * spatialScale; - double y2 = _numOps.ToDouble(rois[roiIdx, 3]) * spatialScale; - - double roiW = x2 - x1; - double roiH = y2 - y1; - - double binW = roiW / _outputSize; - double binH = roiH / _outputSize; - - for (int c = 0; c < channels; c++) - { - for (int ph = 0; ph < _outputSize; ph++) - { - for (int pw = 0; pw < _outputSize; pw++) - { - // Compute bin boundaries - double binStartY = y1 + ph * binH; - double binStartX = x1 + pw * binW; - - double sum = 0; - int count = 0; - - // Sample points within the bin - for (int iy = 0; iy < _samplingRatio; iy++) - { - for (int ix = 0; ix < _samplingRatio; ix++) - { - double y = binStartY + (iy + 0.5) * binH / _samplingRatio; - double x = binStartX + (ix + 0.5) * binW / _samplingRatio; - - // Bilinear interpolation - if (y >= 0 && y < featureH && x >= 0 && x < featureW) - { - sum += BilinearInterpolate(features, batchIdx, c, y, x, featureH, featureW); - count++; - } - } - } - - output[roiIdx, c, ph, pw] = _numOps.FromDouble(count > 0 ? sum / count : 0); - } - } - } } - return output; + return CvTensorOps.RoIAlign(features, boxes, indices, spatialScale, _outputSize, _samplingRatio); } - private double BilinearInterpolate(Tensor features, int batch, int channel, double y, double x, int height, int width) - { - int y0 = (int)Math.Floor(y); - int x0 = (int)Math.Floor(x); - int y1 = Math.Min(y0 + 1, height - 1); - int x1 = Math.Min(x0 + 1, width - 1); - - double wy1 = y - y0; - double wy0 = 1.0 - wy1; - double wx1 = x - x0; - double wx0 = 1.0 - wx1; - - double v00 = _numOps.ToDouble(features[batch, channel, y0, x0]); - double v01 = _numOps.ToDouble(features[batch, channel, y0, x1]); - double v10 = _numOps.ToDouble(features[batch, channel, y1, x0]); - double v11 = _numOps.ToDouble(features[batch, channel, y1, x1]); - - return wy0 * (wx0 * v00 + wx1 * v01) + wy1 * (wx0 * v10 + wx1 * v11); - } + } diff --git a/src/ComputerVision/Detection/ObjectDetection/RCNN/TwoStageTargets.cs b/src/ComputerVision/Detection/ObjectDetection/RCNN/TwoStageTargets.cs new file mode 100644 index 0000000000..6b8fc96dab --- /dev/null +++ b/src/ComputerVision/Detection/ObjectDetection/RCNN/TwoStageTargets.cs @@ -0,0 +1,38 @@ +namespace AiDotNet.ComputerVision.Detection.ObjectDetection.RCNN; + +/// Shared target conversion for training the two-stage detectors. +internal static class TwoStageTargets +{ + /// A normalized center-format target as corner coordinates in input pixels. + internal static double[] PixelCorners(DetectionTrainingTarget target, int width, int height) + { + var ops = AiDotNet.Tensors.Helpers.MathHelper.GetNumericOperations(); + double cx = ops.ToDouble(target.CenterX) * width; + double cy = ops.ToDouble(target.CenterY) * height; + double halfWidth = ops.ToDouble(target.Width) * width / 2; + double halfHeight = ops.ToDouble(target.Height) * height / 2; + return new[] { cx - halfWidth, cy - halfHeight, cx + halfWidth, cy + halfHeight }; + } + + /// The first image's rows of a [batch, rows, width] head output, as [rows, width]. + internal static Tensor FirstImage(Tensor head) + { + if (head.Rank != 3 || head.Shape[0] != 1) + throw new InvalidOperationException("Two-stage training expects the proposal outputs of exactly one image."); + return AiDotNet.Tensors.Engines.AiDotNetEngine.Current.Reshape(head, new[] { head.Shape[1], head.Shape[2] }); + } + + /// Appends constant corner boxes to detached proposal boxes [proposals, 4]. + internal static Tensor AppendBoxes(Tensor proposals, IReadOnlyList boxes) + { + var ops = AiDotNet.Tensors.Helpers.MathHelper.GetNumericOperations(); + int existing = proposals.Shape[0]; + var combined = new Tensor(new[] { existing + boxes.Count, 4 }); + var source = proposals.ToArray(); + for (int i = 0; i < source.Length; i++) combined[i] = source[i]; + for (int b = 0; b < boxes.Count; b++) + for (int k = 0; k < 4; k++) + combined[(existing + b) * 4 + k] = ops.FromDouble(boxes[b][k]); + return combined; + } +} diff --git a/src/ComputerVision/Detection/ObjectDetection/YOLO/YOLOHead.cs b/src/ComputerVision/Detection/ObjectDetection/YOLO/YOLOHead.cs index 74169a5984..c5a6796b3d 100644 --- a/src/ComputerVision/Detection/ObjectDetection/YOLO/YOLOHead.cs +++ b/src/ComputerVision/Detection/ObjectDetection/YOLO/YOLOHead.cs @@ -1,3 +1,4 @@ +using AiDotNet.Tensors.Engines; using System.IO; using AiDotNet.ComputerVision.Detection.Backbones; using AiDotNet.Tensors; @@ -17,7 +18,7 @@ namespace AiDotNet.ComputerVision.Detection.ObjectDetection.YOLO; /// Each output tensor has shape [batch, num_anchors * (5 + num_classes), height, width] /// where 5 = (x, y, w, h, objectness). /// -internal class YOLOHead +internal class YOLOHead : CvParameterModule { private readonly INumericOperations _numOps; private readonly int _numClasses; @@ -153,6 +154,8 @@ public List> Forward(List> features) int batch = output.Shape[0]; int featH = output.Shape[2]; int featW = output.Shape[3]; + double scaleX = imageWidth / ((double)featW * stride); + double scaleY = imageHeight / ((double)featH * stride); for (int b = 0; b < batch; b++) { @@ -195,10 +198,15 @@ public List> Forward(List> features) double bh = Math.Exp(MathHelper.Clamp(th, -88.0, 88.0)) * stride; // Convert to xyxy format - float x1 = (float)Math.Max(0, cx - bw / 2); - float y1 = (float)Math.Max(0, cy - bh / 2); - float x2 = (float)Math.Min(imageWidth, cx + bw / 2); - float y2 = (float)Math.Min(imageHeight, cy + bh / 2); + // Map from the network-input frame to the source image before clipping. + float x1 = (float)Math.Max(0, (cx - bw / 2) * scaleX); + float y1 = (float)Math.Max(0, (cy - bh / 2) * scaleY); + float x2 = (float)Math.Min(imageWidth, (cx + bw / 2) * scaleX); + float y2 = (float)Math.Min(imageHeight, (cy + bh / 2) * scaleY); + if (x2 <= x1 || y2 <= y1) + { + continue; + } // Add to this batch's collections batchBoxes[b].AddRange(new[] { x1, y1, x2, y2 }); @@ -306,6 +314,12 @@ private static double Sigmoid(double x) { return 1.0 / (1.0 + Math.Exp(-x)); } + + /// + protected override IEnumerable?> ParameterChildren() + { + foreach (var child in _convLayers) yield return child; + } } /// @@ -317,7 +331,7 @@ private static double Sigmoid(double x) /// YOLOv8+ uses an anchor-free approach where the network directly predicts box sizes /// relative to each grid cell. This simplifies the architecture and often improves accuracy. /// -internal class YOLOv8Head +internal class YOLOv8Head : CvParameterModule { private readonly INumericOperations _numOps; private readonly int _numClasses; @@ -334,6 +348,9 @@ internal class YOLOv8Head /// Input channels for each feature level. /// Number of detection classes. /// Maximum value for regression distribution (default 16). + /// Distribution bins predicted per box side. + internal int RegMax => _regMax; + public YOLOv8Head(int[] inputChannels, int numClasses, int regMax = 16) { _numOps = Tensors.Helpers.MathHelper.GetNumericOperations(); @@ -478,6 +495,8 @@ public YOLOv8Head(int[] inputChannels, int numClasses, int regMax = 16) int batch = clsOutput.Shape[0]; int featH = clsOutput.Shape[2]; int featW = clsOutput.Shape[3]; + double scaleX = imageWidth / ((double)featW * stride); + double scaleY = imageHeight / ((double)featH * stride); for (int b = 0; b < batch; b++) { @@ -509,10 +528,17 @@ public YOLOv8Head(int[] inputChannels, int numClasses, int regMax = 16) double cx = (w + 0.5) * stride; double cy = (h + 0.5) * stride; - float x1 = (float)Math.Max(0, cx - left * stride); - float y1 = (float)Math.Max(0, cy - top * stride); - float x2 = (float)Math.Min(imageWidth, cx + right * stride); - float y2 = (float)Math.Min(imageHeight, cy + bottom * stride); + // Decoded in network-input coordinates (the feature grid times its stride); + // map to the source image before clipping, or a source image smaller than the + // input size yields inverted boxes. + float x1 = (float)Math.Max(0, (cx - left * stride) * scaleX); + float y1 = (float)Math.Max(0, (cy - top * stride) * scaleY); + float x2 = (float)Math.Min(imageWidth, (cx + right * stride) * scaleX); + float y2 = (float)Math.Min(imageHeight, (cy + bottom * stride) * scaleY); + if (x2 <= x1 || y2 <= y1) + { + continue; // Entirely outside the image once mapped. + } // Add to this batch's collections batchBoxes[b].AddRange(new[] { x1, y1, x2, y2 }); @@ -677,26 +703,28 @@ public void ReadParameters(BinaryReader reader) } } - private Tensor ApplySiLU(Tensor x) - { - var result = new Tensor(x._shape); - for (int i = 0; i < x.Length; i++) - { - double val = _numOps.ToDouble(x[i]); - // Numerically stable SiLU: x * sigmoid(x) - // For large positive x: sigmoid(x) ≈ 1, so SiLU ≈ x - // For large negative x: sigmoid(x) ≈ 0, so SiLU ≈ 0 - // Clamp to prevent overflow in exp(-val) when val is very negative - double clampedVal = MathHelper.Clamp(val, -88.0, 88.0); - double sigmoid = 1.0 / (1.0 + Math.Exp(-clampedVal)); - double silu = val * sigmoid; - result[i] = _numOps.FromDouble(silu); - } - return result; - } + /// + /// Elementwise Swish, delegated to the engine. + /// + /// + /// This was a scalar loop that read each element out to double and wrote a fresh + /// tensor. Arithmetically identical, but it severed the autodiff tape: the gradient chain + /// stopped here, so every trainable layer UPSTREAM of this call received no gradient and + /// silently never trained. The engine op records itself on the tape. + /// + private Tensor ApplySiLU(Tensor x) => AiDotNetEngine.Current.Swish(x); private static double Sigmoid(double x) { return 1.0 / (1.0 + Math.Exp(-x)); } + + /// + protected override IEnumerable?> ParameterChildren() + { + foreach (var child in _clsConvs) yield return child; + foreach (var child in _regConvs) yield return child; + foreach (var child in _clsHeads) yield return child; + foreach (var child in _regHeads) yield return child; + } } diff --git a/src/ComputerVision/Detection/ObjectDetection/YOLO/YOLOv10.cs b/src/ComputerVision/Detection/ObjectDetection/YOLO/YOLOv10.cs index c9a8b463bd..e66924b744 100644 --- a/src/ComputerVision/Detection/ObjectDetection/YOLO/YOLOv10.cs +++ b/src/ComputerVision/Detection/ObjectDetection/YOLO/YOLOv10.cs @@ -38,10 +38,14 @@ namespace AiDotNet.ComputerVision.Detection.ObjectDetection.YOLO; "https://arxiv.org/abs/2405.14458", Year = 2024, Authors = "Ao Wang, Hui Chen, Lihao Liu, Kai Chen, Zijia Lin, Jungong Han, Guiguang Ding")] -public class YOLOv10 : ObjectDetectorBase +public partial class YOLOv10 : ObjectDetectorBase, IDetectionTrainingModel { - private readonly YOLOv8Head _head; - private readonly YOLOv8Head? _auxHead; // Auxiliary head for training + private readonly YOLOv8Head _head; // One-to-one head: the only head used at inference. + private readonly YOLOv8Head _auxHead; // One-to-many head: trained jointly, dropped at inference. + private readonly AiDotNet.ComputerVision.Detection.Losses.TaskAlignedDetectionLoss _detectionLoss; + + [AiDotNet.Attributes.Scratch] + private bool _auxHeadShapesResolved; private readonly int[] _strides; private readonly bool _useNmsFree; private readonly NMS _nms; @@ -69,13 +73,14 @@ public YOLOv10(ObjectDetectionOptions options, bool useNmsFree = true) : base var neckChannels = Enumerable.Repeat(Neck.OutputChannels, Neck.NumLevels).ToArray(); _head = new YOLOv8Head(neckChannels, options.NumClasses); - // Auxiliary head for training (one-to-many assignment) - if (IsTrainingMode) - { - _auxHead = new YOLOv8Head(neckChannels, options.NumClasses); - } + // One-to-many head (Wang et al. 2024, dual label assignments): it supplies the rich supervision + // during training and is discarded at inference. It must exist whenever the model can be trained; + // it used to be built only when training mode was already on at construction, which it never is. + _auxHead = new YOLOv8Head(neckChannels, options.NumClasses); _strides = Backbone.Strides.ToArray(); + _detectionLoss = new AiDotNet.ComputerVision.Detection.Losses.TaskAlignedDetectionLoss(options.NumClasses, + _head.RegMax, options.TaskAlignedLoss ?? new AiDotNet.ComputerVision.Detection.Losses.TaskAlignedLossOptions()); _nms = new NMS(); } @@ -89,6 +94,65 @@ public YOLOv10(ObjectDetectionOptions options, bool useNmsFree = true) : base _ => (0.67, 0.75) }; + /// Trains both heads with YOLOv10's consistent dual assignments. + /// + /// + /// The one-to-many head uses task-aligned top-k assignment and the one-to-one head uses top-1 selection, + /// both with the same metric exponents (alpha 0.5, beta 6), so the one-to-one head is supervised + /// consistently with the one-to-many head (Wang et al. 2024, Sec. 3.1). Each head is assigned from its own + /// predictions and trained with BCE, CIoU and distribution focal loss (gains 7.5/0.5/1.5, Table 14); the + /// two losses are summed. Override the settings with . + /// + /// Inputs are model-ready NCHW tensors, as for Predict, and targets are normalized against that input size. + /// + public void TrainDetections(Tensor input, DetectionTrainingBatch targets) + { + YoloDetectionTraining.Validate(input, targets, Options.NumClasses, "YOLOv10"); + int height = input.Shape[2]; + int width = input.Shape[3]; + int levels = _strides.Length; + TrainWithTargets(input, targets, ForwardTrainingHeads, (heads, batch) => Engine.TensorAdd( + YoloDetectionTraining.HeadLoss(_detectionLoss, heads, 0, levels, _strides, height, width, batch, _detectionLoss.OneToOneTopK), + YoloDetectionTraining.HeadLoss(_detectionLoss, heads, 2 * levels, levels, _strides, height, width, batch, _detectionLoss.TopK))); + } + + /// + /// Regresses both heads onto a raw output-shaped target and sums the two losses. + /// + /// + /// The base path fits only Predict's output, which is the one-to-one head, so the one-to-many head's + /// registered weights never received a gradient. Both heads emit the same layout, and the paper supervises + /// them jointly with summed losses (Wang et al. 2024, Sec. 3.1), as does. + /// + public override void Train(Tensor input, Tensor expectedOutput) + { + if (input is null) throw new ArgumentNullException(nameof(input)); + if (expectedOutput is null) throw new ArgumentNullException(nameof(expectedOutput)); + TrainWithTargets(input, expectedOutput, ForwardTrainingHeads, (heads, target) => + { + int half = heads.Count / 2; + return Engine.TensorAdd( + TensorModelTrainer.MeanSquaredError(CvTensorOps.ConcatenateOutputs(heads.GetRange(0, half)), target), + TensorModelTrainer.MeanSquaredError(CvTensorOps.ConcatenateOutputs(heads.GetRange(half, half)), target)); + }); + } + /// + /// Runs the shared backbone and neck once and returns the one-to-one head's class and distribution levels, + /// followed by the one-to-many head's. + /// + internal List> ForwardTrainingHeads(Tensor input) + { + var neckFeatures = EnsureNeck.Forward(EnsureBackbone.ExtractFeatures(input)); + var (oneToOneClasses, oneToOneDistributions) = _head.Forward(neckFeatures); + var (oneToManyClasses, oneToManyDistributions) = _auxHead.Forward(neckFeatures); + var outputs = new List>(); + outputs.AddRange(oneToOneClasses); + outputs.AddRange(oneToOneDistributions); + outputs.AddRange(oneToManyClasses); + outputs.AddRange(oneToManyDistributions); + return outputs; + } + /// public override DetectionResult Detect(Tensor image, double confidenceThreshold, double nmsThreshold) { @@ -122,6 +186,15 @@ protected override List> Forward(Tensor input) // Main detection head var (clsOutputs, regOutputs) = _head.Forward(neckFeatures); + if (!_auxHeadShapesResolved) + { + // The one-to-many head runs only in training, but its lazily sized convolutions must exist + // whenever the parameters are enumerated, saved or cloned. Size them on the first forward, as + // the one-to-one head is sized; inference does not use these outputs. + _ = _auxHead.Forward(neckFeatures); + _auxHeadShapesResolved = true; + } + var outputs = new List>(); outputs.AddRange(clsOutputs); outputs.AddRange(regOutputs); @@ -229,12 +302,7 @@ private List> SelectTopKPerClass(List> detections, int /// protected override long GetHeadParameterCount() { - long count = _head.GetParameterCount(); - if (_auxHead is not null) - { - count += _auxHead.GetParameterCount(); - } - return count; + return _head.GetParameterCount() + _auxHead.GetParameterCount(); } /// @@ -276,7 +344,7 @@ public override Task LoadWeightsAsync(string pathOrUrl, CancellationToken cancel // Read auxiliary head parameters if present bool hasAuxHead = reader.ReadBoolean(); - if (hasAuxHead && _auxHead is not null) + if (hasAuxHead) { _auxHead.ReadParameters(reader); } @@ -307,10 +375,15 @@ public override void SaveWeights(string path) _head.WriteParameters(writer); // Write auxiliary head parameters if present - writer.Write(_auxHead is not null); - if (_auxHead is not null) - { - _auxHead.WriteParameters(writer); - } + writer.Write(true); + _auxHead.WriteParameters(writer); } + + /// + /// + /// In NMS-free mode (the default) the one-to-one head is trained to emit one box per object and + /// detections are selected top-K per class with no suppression at all - an IoU threshold of 1, + /// whatever the caller requests. With useNmsFree: false the requested threshold applies. + /// + public override double EffectiveNmsThreshold(double requested) => _useNmsFree ? 1.0 : requested; } diff --git a/src/ComputerVision/Detection/ObjectDetection/YOLO/YOLOv11.cs b/src/ComputerVision/Detection/ObjectDetection/YOLO/YOLOv11.cs index c5a568f93b..5b765b61f3 100644 --- a/src/ComputerVision/Detection/ObjectDetection/YOLO/YOLOv11.cs +++ b/src/ComputerVision/Detection/ObjectDetection/YOLO/YOLOv11.cs @@ -1,4 +1,5 @@ -using System.IO; +using AiDotNet.Tensors.Engines; +using System.IO; using AiDotNet.Augmentation.Image; using AiDotNet.ComputerVision.Detection.Backbones; using AiDotNet.ComputerVision.Detection.Necks; @@ -39,8 +40,9 @@ namespace AiDotNet.ComputerVision.Detection.ObjectDetection.YOLO; "https://github.com/ultralytics/ultralytics", Year = 2024, Authors = "Glenn Jocher, Jing Qiu")] -public class YOLOv11 : ObjectDetectorBase +public partial class YOLOv11 : ObjectDetectorBase, IDetectionTrainingModel { + private readonly AiDotNet.ComputerVision.Detection.Losses.TaskAlignedDetectionLoss _detectionLoss; private readonly YOLOv8Head _head; private readonly int[] _strides; private readonly List> _attentionBlocks; @@ -79,6 +81,8 @@ public YOLOv11(ObjectDetectionOptions options) : base(options) _head = new YOLOv8Head(neckChannels, options.NumClasses); _strides = Backbone.Strides.ToArray(); + _detectionLoss = new AiDotNet.ComputerVision.Detection.Losses.TaskAlignedDetectionLoss(options.NumClasses, + _head.RegMax, options.TaskAlignedLoss ?? new AiDotNet.ComputerVision.Detection.Losses.TaskAlignedLossOptions()); _nms = new NMS(); } @@ -92,6 +96,21 @@ public YOLOv11(ObjectDetectionOptions options) : base(options) _ => (0.67, 0.75) }; + /// Trains the head with task-aligned assignment, BCE classification, CIoU and distribution focal loss. + /// + /// Uses the YOLOv8-family objective (alpha 0.5, beta 6, top-10; box/class/DFL gains 7.5/0.5/1.5); override it + /// with . Inputs are model-ready NCHW tensors, as for + /// Predict, and targets are normalized against that input size. + /// + public void TrainDetections(Tensor input, DetectionTrainingBatch targets) + { + YoloDetectionTraining.Validate(input, targets, Options.NumClasses, "YOLOv11"); + int height = input.Shape[2]; + int width = input.Shape[3]; + TrainWithTargets(input, targets, (heads, batch) => YoloDetectionTraining.HeadLoss( + _detectionLoss, heads, 0, _strides.Length, _strides, height, width, batch, _detectionLoss.TopK)); + } + /// public override DetectionResult Detect(Tensor image, double confidenceThreshold, double nmsThreshold) { @@ -316,7 +335,7 @@ public override void SaveWeights(string path) /// /// Spatial Pyramid Pooling Fast (SPPF) block. /// -internal class SPPFBlock +internal class SPPFBlock : CvParameterModule { private readonly INumericOperations _numOps; private readonly Conv2D _conv1; @@ -391,68 +410,37 @@ public void ReadParameters(BinaryReader reader) } private Tensor MaxPool(Tensor x, int kernelSize) - { - int padding = kernelSize / 2; - int batch = x.Shape[0]; - int channels = x.Shape[1]; - int height = x.Shape[2]; - int width = x.Shape[3]; - - var output = new Tensor(x._shape); - - for (int n = 0; n < batch; n++) - { - for (int c = 0; c < channels; c++) - { - for (int h = 0; h < height; h++) - { - for (int w = 0; w < width; w++) - { - double maxVal = double.NegativeInfinity; - for (int kh = 0; kh < kernelSize; kh++) - { - for (int kw = 0; kw < kernelSize; kw++) - { - int ih = h - padding + kh; - int iw = w - padding + kw; - if (ih >= 0 && ih < height && iw >= 0 && iw < width) - { - double val = _numOps.ToDouble(x[n, c, ih, iw]); - maxVal = Math.Max(maxVal, val); - } - } - } - output[n, c, h, w] = _numOps.FromDouble(maxVal == double.NegativeInfinity ? 0 : maxVal); - } - } - } - } - - return output; - } + // Stride-1 "same" max pooling that ignores out-of-bounds cells (SPPF), tape-visible. + => CvTensorOps.MaxPoolSame(x, kernelSize); private Tensor ConcatenateChannels(params Tensor[] tensors) { return AiDotNetEngine.Current.TensorConcatenate(tensors, axis: 1); } - private Tensor ApplySiLU(Tensor x) + /// + /// Elementwise Swish, delegated to the engine. + /// + /// + /// This was a scalar loop that read each element out to double and wrote a fresh + /// tensor. Arithmetically identical, but it severed the autodiff tape: the gradient chain + /// stopped here, so every trainable layer UPSTREAM of this call received no gradient and + /// silently never trained. The engine op records itself on the tape. + /// + private Tensor ApplySiLU(Tensor x) => AiDotNetEngine.Current.Swish(x); + + /// + protected override IEnumerable?> ParameterChildren() { - var result = new Tensor(x._shape); - for (int i = 0; i < x.Length; i++) - { - double val = _numOps.ToDouble(x[i]); - double silu = val * (1.0 / (1.0 + Math.Exp(-val))); - result[i] = _numOps.FromDouble(silu); - } - return result; + yield return _conv1; + yield return _conv2; } } /// /// Lightweight attention block for feature enhancement. /// -internal class AttentionBlock +internal class AttentionBlock : CvParameterModule { private readonly INumericOperations _numOps; private readonly Conv2D _query; @@ -476,67 +464,31 @@ public AttentionBlock(int channels) public Tensor Forward(Tensor input) { + var engine = AiDotNetEngine.Current; int batch = input.Shape[0]; int channels = input.Shape[1]; int height = input.Shape[2]; int width = input.Shape[3]; int spatialSize = height * width; - // Compute Q, K, V var q = _query.Forward(input); var k = _key.Forward(input); var v = _value.Forward(input); - // Reshape and compute attention - // For simplicity, compute spatial attention per batch - var output = new Tensor(input._shape); - - for (int n = 0; n < batch; n++) - { - // Global average for channel attention (simplified) - var channelWeights = new double[channels]; - double sumWeights = 0; - - for (int c = 0; c < channels; c++) - { - double qSum = 0, kSum = 0; - for (int h = 0; h < height; h++) - { - for (int w = 0; w < width; w++) - { - qSum += _numOps.ToDouble(q[n, c, h, w]); - kSum += _numOps.ToDouble(k[n, c, h, w]); - } - } - double attn = Math.Exp(qSum * kSum * _scale / spatialSize); - channelWeights[c] = attn; - sumWeights += attn; - } - - // Normalize and apply - for (int c = 0; c < channels; c++) - { - double weight = channelWeights[c] / sumWeights; - for (int h = 0; h < height; h++) - { - for (int w = 0; w < width; w++) - { - double vVal = _numOps.ToDouble(v[n, c, h, w]); - output[n, c, h, w] = _numOps.FromDouble(vVal * weight); - } - } - } - } - - // Project and add residual - var projected = _proj.Forward(output); - - for (int i = 0; i < projected.Length; i++) - { - projected[i] = _numOps.Add(projected[i], input[i]); - } - - return projected; + // Channel attention: logit_c = sum_hw(Q_c) * sum_hw(K_c) * scale / (H*W), softmax over channels, + // then each channel of V is scaled by its weight. The softmax is max-shifted (the loop it + // replaces exponentiated unshifted logits, which overflowed to NaN on large activations). + var qSum = engine.ReduceSum(q, new[] { 2, 3 }, false); // [B, C] + var kSum = engine.ReduceSum(k, new[] { 2, 3 }, false); + var logits = engine.TensorMultiplyScalar( + engine.TensorMultiply(qSum, kSum), _numOps.FromDouble(_scale / spatialSize)); + var weights = engine.Softmax(logits, -1); + var gate = engine.TensorBroadcastTo( + engine.Reshape(weights, new[] { batch, channels, 1, 1 }), new[] { batch, channels, height, width }); + + // Project, then add the residual with an engine op (the old in-place indexer write severed + // the tape for everything upstream of this block). + return engine.TensorAdd(_proj.Forward(engine.TensorMultiply(v, gate)), input); } public long GetParameterCount() @@ -567,4 +519,13 @@ public void ReadParameters(BinaryReader reader) _value.ReadParameters(reader); _proj.ReadParameters(reader); } + + /// + protected override IEnumerable?> ParameterChildren() + { + yield return _query; + yield return _key; + yield return _value; + yield return _proj; + } } diff --git a/src/ComputerVision/Detection/ObjectDetection/YOLO/YOLOv8.cs b/src/ComputerVision/Detection/ObjectDetection/YOLO/YOLOv8.cs index 04261a0358..1d4365fc09 100644 --- a/src/ComputerVision/Detection/ObjectDetection/YOLO/YOLOv8.cs +++ b/src/ComputerVision/Detection/ObjectDetection/YOLO/YOLOv8.cs @@ -38,8 +38,9 @@ namespace AiDotNet.ComputerVision.Detection.ObjectDetection.YOLO; "https://github.com/ultralytics/ultralytics", Year = 2023, Authors = "Glenn Jocher, Ayush Chaurasia, Jing Qiu")] -public class YOLOv8 : ObjectDetectorBase +public partial class YOLOv8 : ObjectDetectorBase, IDetectionTrainingModel { + private readonly AiDotNet.ComputerVision.Detection.Losses.TaskAlignedDetectionLoss _detectionLoss; private readonly YOLOv8Head _head; private readonly int[] _strides; private readonly NMS _nms; @@ -67,6 +68,8 @@ public YOLOv8(ObjectDetectionOptions options) : base(options) _head = new YOLOv8Head(neckChannels, options.NumClasses); _strides = Backbone.Strides.ToArray(); + _detectionLoss = new AiDotNet.ComputerVision.Detection.Losses.TaskAlignedDetectionLoss(options.NumClasses, + _head.RegMax, options.TaskAlignedLoss ?? new AiDotNet.ComputerVision.Detection.Losses.TaskAlignedLossOptions()); _nms = new NMS(); } @@ -83,6 +86,21 @@ public YOLOv8(ObjectDetectionOptions options) : base(options) _ => (0.67, 0.75) }; + /// Trains the head with task-aligned assignment, BCE classification, CIoU and distribution focal loss. + /// + /// Defaults follow the YOLOv8 recipe cited by YOLOv10 (alpha 0.5, beta 6, top-10) with box/class/DFL gains + /// 7.5/0.5/1.5; override them with . Inputs are + /// model-ready NCHW tensors, as for Predict, and targets are normalized against that input size. + /// + public void TrainDetections(Tensor input, DetectionTrainingBatch targets) + { + YoloDetectionTraining.Validate(input, targets, Options.NumClasses, "YOLOv8"); + int height = input.Shape[2]; + int width = input.Shape[3]; + TrainWithTargets(input, targets, (heads, batch) => YoloDetectionTraining.HeadLoss( + _detectionLoss, heads, 0, _strides.Length, _strides, height, width, batch, _detectionLoss.TopK)); + } + /// public override DetectionResult Detect(Tensor image, double confidenceThreshold, double nmsThreshold) { diff --git a/src/ComputerVision/Detection/ObjectDetection/YOLO/YOLOv9.cs b/src/ComputerVision/Detection/ObjectDetection/YOLO/YOLOv9.cs index 3fe3e10f3c..537af55cb8 100644 --- a/src/ComputerVision/Detection/ObjectDetection/YOLO/YOLOv9.cs +++ b/src/ComputerVision/Detection/ObjectDetection/YOLO/YOLOv9.cs @@ -39,8 +39,9 @@ namespace AiDotNet.ComputerVision.Detection.ObjectDetection.YOLO; "https://arxiv.org/abs/2402.13616", Year = 2024, Authors = "Chien-Yao Wang, I-Hau Yeh, Hong-Yuan Mark Liao")] -public class YOLOv9 : ObjectDetectorBase +public partial class YOLOv9 : ObjectDetectorBase, IDetectionTrainingModel { + private readonly AiDotNet.ComputerVision.Detection.Losses.TaskAlignedDetectionLoss _detectionLoss; private readonly YOLOv8Head _head; private readonly int[] _strides; private readonly List> _gelanBlocks; @@ -99,6 +100,8 @@ public YOLOv9(ObjectDetectionOptions options) : base(options) _head = new YOLOv8Head(neckChannels, options.NumClasses); _strides = Backbone.Strides.ToArray(); + _detectionLoss = new AiDotNet.ComputerVision.Detection.Losses.TaskAlignedDetectionLoss(options.NumClasses, + _head.RegMax, options.TaskAlignedLoss ?? new AiDotNet.ComputerVision.Detection.Losses.TaskAlignedLossOptions()); _nms = new NMS(); } @@ -112,6 +115,22 @@ public YOLOv9(ObjectDetectionOptions options) : base(options) _ => (0.75, 0.75) }; + /// Trains the head with task-aligned assignment, BCE classification, CIoU and distribution focal loss. + /// + /// Box/class/DFL gains default to 7.5/0.5/1.5 (YOLOv9 Table 1) with task-aligned assignment (alpha 0.5, + /// beta 6, top-10); override them with . This + /// architecture has no auxiliary reversible branch, so PGI's auxiliary loss is not claimed. Inputs are + /// model-ready NCHW tensors, as for Predict, and targets are normalized against that input size. + /// + public void TrainDetections(Tensor input, DetectionTrainingBatch targets) + { + YoloDetectionTraining.Validate(input, targets, Options.NumClasses, "YOLOv9"); + int height = input.Shape[2]; + int width = input.Shape[3]; + TrainWithTargets(input, targets, (heads, batch) => YoloDetectionTraining.HeadLoss( + _detectionLoss, heads, 0, _strides.Length, _strides, height, width, batch, _detectionLoss.TopK)); + } + /// public override DetectionResult Detect(Tensor image, double confidenceThreshold, double nmsThreshold) { @@ -227,17 +246,16 @@ protected override long GetHeadParameterCount() return count; } - private Tensor ApplySiLU(Tensor x) - { - var result = new Tensor(x._shape); - for (int i = 0; i < x.Length; i++) - { - double val = NumOps.ToDouble(x[i]); - double silu = val * (1.0 / (1.0 + Math.Exp(-val))); - result[i] = NumOps.FromDouble(silu); - } - return result; - } + /// + /// Elementwise Swish, delegated to the engine. + /// + /// + /// This was a scalar loop that read each element out to double and wrote a fresh + /// tensor. Arithmetically identical, but it severed the autodiff tape: the gradient chain + /// stopped here, so every trainable layer UPSTREAM of this call received no gradient and + /// silently never trained. The engine op records itself on the tape. + /// + private Tensor ApplySiLU(Tensor x) => Engine.Swish(x); private Tensor AddTensors(Tensor a, Tensor b) { diff --git a/src/ComputerVision/Detection/ObjectDetection/YOLO/YoloDetectionTraining.cs b/src/ComputerVision/Detection/ObjectDetection/YOLO/YoloDetectionTraining.cs new file mode 100644 index 0000000000..defe751d09 --- /dev/null +++ b/src/ComputerVision/Detection/ObjectDetection/YOLO/YoloDetectionTraining.cs @@ -0,0 +1,30 @@ +using AiDotNet.ComputerVision.Detection.Losses; + +namespace AiDotNet.ComputerVision.Detection.ObjectDetection.YOLO; + +/// Shared input checks and head splitting for task-aligned YOLO detection training. +internal static class YoloDetectionTraining +{ + /// Rejects inputs and targets the model cannot train on, before any forward pass or update. + internal static void Validate(Tensor input, DetectionTrainingBatch targets, int numClasses, string family) + { + if (input is null) throw new ArgumentNullException(nameof(input)); + if (targets is null) throw new ArgumentNullException(nameof(targets)); + if (input.Rank != 4 || input.Shape[0] <= 0 || input.Shape[1] != 3 || input.Shape[2] <= 0 || input.Shape[3] <= 0) + throw new ArgumentException($"{family} training requires a nonempty NCHW three-channel image batch.", nameof(input)); + targets.ValidateForModel(input.Shape[0], numClasses, int.MaxValue); + } + + /// + /// The loss of one YOLOv8-style head whose class levels start at in + /// , followed by its distribution levels. + /// + internal static Tensor HeadLoss(TaskAlignedDetectionLoss loss, List> heads, int offset, int levels, + int[] strides, int imageHeight, int imageWidth, DetectionTrainingBatch targets, int topK) + { + if (levels <= 0 || levels != strides.Length || heads.Count < offset + 2 * levels) + throw new InvalidOperationException("YOLO training requires one class and one distribution output per pyramid level."); + return loss.ComputeTapeLoss(heads.GetRange(offset, levels), heads.GetRange(offset + levels, levels), + strides, imageHeight, imageWidth, targets, topK); + } +} diff --git a/src/ComputerVision/Detection/TextDetection/CRAFT.cs b/src/ComputerVision/Detection/TextDetection/CRAFT.cs index 6126cc15b7..1d8f9e128c 100644 --- a/src/ComputerVision/Detection/TextDetection/CRAFT.cs +++ b/src/ComputerVision/Detection/TextDetection/CRAFT.cs @@ -36,7 +36,7 @@ namespace AiDotNet.ComputerVision.Detection.TextDetection; "https://arxiv.org/abs/1904.01941", Year = 2019, Authors = "Youngmin Baek, Bado Lee, Dongyoon Han, Sangdoo Yun, Hwalsuk Lee")] -public class CRAFT : TextDetectorBase +public partial class CRAFT : TextDetectorBase { private readonly Conv2D _upConv1; private readonly Conv2D _upConv2; @@ -60,11 +60,16 @@ public CRAFT(TextDetectionOptions options) : base(options) Backbone = new ResNet(ResNetVariant.ResNet50); // Upsampling convolutions for feature fusion - int backboneChannels = Backbone.OutputChannels[^1]; - _upConv1 = new Conv2D(backboneChannels, _hiddenDim, kernelSize: 3, padding: 1); - _upConv2 = new Conv2D(_hiddenDim * 2, _hiddenDim, kernelSize: 3, padding: 1); - _upConv3 = new Conv2D(_hiddenDim * 2, _hiddenDim, kernelSize: 3, padding: 1); - _upConv4 = new Conv2D(_hiddenDim * 2, _hiddenDim, kernelSize: 3, padding: 1); + var stageChannels = Backbone.OutputChannels; + _upConv1 = new Conv2D(stageChannels[^1], _hiddenDim, kernelSize: 3, padding: 1); + // Each merge conv receives the upsampled decoder map CONCATENATED with a raw backbone + // stage, so its input width is the decoder width plus that stage's channel count. These were + // declared as twice the decoder width, which matches no backbone stage, so the first merge + // threw on a channel mismatch (e.g. ResNet-50's C4: 256 + 1024 = 1280 channels into a conv + // built for 512) and the model could not run a forward pass at all. + _upConv2 = new Conv2D(_hiddenDim + stageChannels[^2], _hiddenDim, kernelSize: 3, padding: 1); + _upConv3 = new Conv2D(_hiddenDim + stageChannels[^3], _hiddenDim, kernelSize: 3, padding: 1); + _upConv4 = new Conv2D(_hiddenDim + stageChannels[^4], _hiddenDim, kernelSize: 3, padding: 1); // Prediction heads: region score and affinity score _regionHead = new Conv2D(_hiddenDim, 1, kernelSize: 1); @@ -318,116 +323,36 @@ public override void SaveWeights(string path) _affinityHead.WriteParameters(writer); } - private Tensor ApplyReLU(Tensor x) - { - var result = new Tensor(x._shape); - for (int i = 0; i < x.Length; i++) - { - double val = NumOps.ToDouble(x[i]); - result[i] = NumOps.FromDouble(Math.Max(0, val)); - } - return result; - } + /// + /// Elementwise ReLU, delegated to the engine. + /// + /// + /// This was a scalar loop that read each element out to double and wrote a fresh + /// tensor. Arithmetically identical, but it severed the autodiff tape: the gradient chain + /// stopped here, so every trainable layer UPSTREAM of this call received no gradient and + /// silently never trained. The engine op records itself on the tape. + /// + private Tensor ApplyReLU(Tensor x) => Engine.ReLU(x); - private Tensor ApplySigmoid(Tensor x) - { - var result = new Tensor(x._shape); - for (int i = 0; i < x.Length; i++) - { - double val = NumOps.ToDouble(x[i]); - result[i] = NumOps.FromDouble(1.0 / (1.0 + Math.Exp(-val))); - } - return result; - } + /// + /// Elementwise Sigmoid, delegated to the engine. + /// + /// + /// This was a scalar loop that read each element out to double and wrote a fresh + /// tensor. Arithmetically identical, but it severed the autodiff tape: the gradient chain + /// stopped here, so every trainable layer UPSTREAM of this call received no gradient and + /// silently never trained. The engine op records itself on the tape. + /// + private Tensor ApplySigmoid(Tensor x) => Engine.Sigmoid(x); private Tensor UpsampleAndConcat(Tensor x, Tensor skip) - { - int batch = x.Shape[0]; - int xChannels = x.Shape[1]; - int skipChannels = skip.Shape[1]; - int targetH = skip.Shape[2]; - int targetW = skip.Shape[3]; - - // Upsample x to match skip spatial dimensions - var upsampled = BilinearUpsample(x, targetH, targetW); - - // Concatenate along channel dimension - var result = new Tensor(new[] { batch, xChannels + skipChannels, targetH, targetW }); - - for (int b = 0; b < batch; b++) - { - // Copy upsampled - for (int c = 0; c < xChannels; c++) - { - for (int h = 0; h < targetH; h++) - { - for (int w = 0; w < targetW; w++) - { - result[b, c, h, w] = upsampled[b, c, h, w]; - } - } - } - - // Copy skip - for (int c = 0; c < skipChannels; c++) - { - for (int h = 0; h < targetH; h++) - { - for (int w = 0; w < targetW; w++) - { - result[b, xChannels + c, h, w] = skip[b, c, h, w]; - } - } - } - } - - return result; - } + // Upsample to the skip connection's resolution, then stack along channels. Tape-visible, so + // the decoder's gradient reaches the backbone through every skip. + => CvTensorOps.ConcatChannels(BilinearUpsample(x, skip.Shape[2], skip.Shape[3]), skip); private Tensor BilinearUpsample(Tensor x, int targetH, int targetW) - { - int batch = x.Shape[0]; - int channels = x.Shape[1]; - int srcH = x.Shape[2]; - int srcW = x.Shape[3]; - - var result = new Tensor(new[] { batch, channels, targetH, targetW }); - - for (int b = 0; b < batch; b++) - { - for (int c = 0; c < channels; c++) - { - for (int h = 0; h < targetH; h++) - { - for (int w = 0; w < targetW; w++) - { - double srcY = (double)h / targetH * srcH; - double srcX = (double)w / targetW * srcW; - - int y0 = (int)Math.Floor(srcY); - int x0 = (int)Math.Floor(srcX); - int y1 = Math.Min(y0 + 1, srcH - 1); - int x1 = Math.Min(x0 + 1, srcW - 1); - - double wy1 = srcY - y0; - double wy0 = 1.0 - wy1; - double wx1 = srcX - x0; - double wx0 = 1.0 - wx1; - - double v00 = NumOps.ToDouble(x[b, c, y0, x0]); - double v01 = NumOps.ToDouble(x[b, c, y0, x1]); - double v10 = NumOps.ToDouble(x[b, c, y1, x0]); - double v11 = NumOps.ToDouble(x[b, c, y1, x1]); - - double val = wy0 * (wx0 * v00 + wx1 * v01) + wy1 * (wx0 * v10 + wx1 * v11); - result[b, c, h, w] = NumOps.FromDouble(val); - } - } - } - } - - return result; - } + // Asymmetric bilinear (src = dst * in / out, no half-pixel offset), as the loop it replaces. + => CvTensorOps.ResizeBilinearAsymmetric(x, targetH, targetW); private List> FindConnectedComponents(bool[,] mask, int height, int width) { diff --git a/src/ComputerVision/Detection/TextDetection/DBNet.cs b/src/ComputerVision/Detection/TextDetection/DBNet.cs index 91d3a794e4..94644c0c8a 100644 --- a/src/ComputerVision/Detection/TextDetection/DBNet.cs +++ b/src/ComputerVision/Detection/TextDetection/DBNet.cs @@ -26,6 +26,12 @@ namespace AiDotNet.ComputerVision.Detection.TextDetection; /// - Works well for both regular and irregular text shapes /// /// +/// Architecture, as in the paper and its reference implementation: a ResNet backbone; a feature +/// pyramid whose four levels are reduced to a common width by 1x1 lateral convolutions, merged top-down, +/// smoothed by 3x3 convolutions to a quarter of that width and upsampled to 1/4 resolution and +/// concatenated; then two identical heads - convolution, batch norm, ReLU, then two stride-2 transposed +/// convolutions back to full resolution - predicting the probability map and the threshold map. +/// /// Reference: Liao et al., "Real-time Scene Text Detection with Differentiable /// Binarization", AAAI 2020 /// @@ -38,14 +44,11 @@ namespace AiDotNet.ComputerVision.Detection.TextDetection; "https://arxiv.org/abs/1911.08947", Year = 2020, Authors = "Minghui Liao, Zhaoyi Wan, Cong Yao, Kai Chen, Xiang Bai")] -public class DBNet : TextDetectorBase +public partial class DBNet : TextDetectorBase { - private readonly Conv2D _inConv; - private readonly Conv2D _upConv1; - private readonly Conv2D _upConv2; - private readonly Conv2D _upConv3; - private readonly Conv2D _probHead; - private readonly Conv2D _threshHead; + private readonly DbFeaturePyramid _pyramid; + private readonly DbHead _probabilityHead; + private readonly DbHead _thresholdHead; private readonly int _hiddenDim; private readonly double _k; @@ -65,18 +68,22 @@ public DBNet(TextDetectionOptions options, double k = 50.0) : base(options) // ResNet backbone Backbone = new ResNet(ResNetVariant.ResNet50); - // Feature pyramid for multi-scale fusion - int backboneChannels = Backbone.OutputChannels[^1]; - _inConv = new Conv2D(backboneChannels, _hiddenDim, kernelSize: 1); - _upConv1 = new Conv2D(_hiddenDim * 2, _hiddenDim, kernelSize: 3, padding: 1); - _upConv2 = new Conv2D(_hiddenDim * 2, _hiddenDim, kernelSize: 3, padding: 1); - _upConv3 = new Conv2D(_hiddenDim, _hiddenDim / 2, kernelSize: 3, padding: 1); + // Feature pyramid (the paper's FPN, mmocr FPNC): the fused map has _hiddenDim channels at 1/4 + // resolution - 256 at the default size, as in the paper. + _pyramid = new DbFeaturePyramid(Backbone.OutputChannels, _hiddenDim); - // Probability map head (text probability) - _probHead = new Conv2D(_hiddenDim / 2, 1, kernelSize: 1); + // Probability and threshold heads, each back to full input resolution. + _probabilityHead = new DbHead(_hiddenDim); + _thresholdHead = new DbHead(_hiddenDim); + } - // Threshold map head (adaptive threshold) - _threshHead = new Conv2D(_hiddenDim / 2, 1, kernelSize: 1); + /// + /// Also switches the heads' batch-norm layers. + public override void SetTrainingMode(bool training) + { + base.SetTrainingMode(training); + _probabilityHead.SetTrainingMode(training); + _thresholdHead.SetTrainingMode(training); } private static int GetHiddenDim(ModelSize size) => size switch @@ -119,39 +126,13 @@ public override TextDetectionResult Detect(Tensor image, double confidence /// protected override List> Forward(Tensor input) { - // Extract multi-scale backbone features - var features = EnsureBackbone.ExtractFeatures(input); - - // Feature pyramid fusion - var x = _inConv.Forward(features[^1]); - x = ApplyReLU(x); + var fused = _pyramid.Forward(EnsureBackbone.ExtractFeatures(input)); - if (features.Count > 1) - { - x = UpsampleAndConcat(x, features[^2]); - x = _upConv1.Forward(x); - x = ApplyReLU(x); - } - - if (features.Count > 2) - { - x = UpsampleAndConcat(x, features[^3]); - x = _upConv2.Forward(x); - x = ApplyReLU(x); - } - - x = _upConv3.Forward(x); - x = ApplyReLU(x); - - // Predict probability and threshold maps - var probMap = _probHead.Forward(x); - probMap = ApplySigmoid(probMap); - - var threshMap = _threshHead.Forward(x); - threshMap = ApplySigmoid(threshMap); + var probMap = _probabilityHead.Forward(fused); + var threshMap = _thresholdHead.Forward(fused); // Apply differentiable binarization: DB = 1 / (1 + exp(-k * (P - T))) - var binaryMap = ApplyDifferentiableBinarization(probMap, threshMap); + var binaryMap = ApplyDifferentiableBinarization(probMap, threshMap, _k); return new List> { probMap, threshMap, binaryMap }; } @@ -246,33 +227,19 @@ protected override List> PostProcess( return regions; } - private Tensor ApplyDifferentiableBinarization(Tensor prob, Tensor thresh) + internal static Tensor ApplyDifferentiableBinarization(Tensor prob, Tensor thresh, double k) { - var result = new Tensor(prob._shape); - - for (int i = 0; i < prob.Length; i++) - { - double p = NumOps.ToDouble(prob[i]); - double t = NumOps.ToDouble(thresh[i]); - - // DB formula: 1 / (1 + exp(-k * (P - T))) - double db = 1.0 / (1.0 + Math.Exp(-_k * (p - t))); - result[i] = NumOps.FromDouble(db); - } - - return result; + // DB (Liao et al. 2020): B = 1 / (1 + exp(-k (P - T))). Engine ops, so the binarization step - + // the whole point of DBNet - passes gradient to both the probability and threshold heads. + var engine = AiDotNetEngine.Current; + var scaled = engine.TensorMultiplyScalar( + engine.TensorSubtract(prob, thresh), MathHelper.GetNumericOperations().FromDouble(k)); + return engine.Sigmoid(scaled); } /// protected override long GetHeadParameterCount() - { - return _inConv.GetParameterCount() + - _upConv1.GetParameterCount() + - _upConv2.GetParameterCount() + - _upConv3.GetParameterCount() + - _probHead.GetParameterCount() + - _threshHead.GetParameterCount(); - } + => _pyramid.ParameterCount + _probabilityHead.ParameterCount + _thresholdHead.ParameterCount; /// public override async Task LoadWeightsAsync(string pathOrUrl, CancellationToken cancellationToken = default) @@ -301,9 +268,12 @@ public override async Task LoadWeightsAsync(string pathOrUrl, CancellationToken } int version = reader.ReadInt32(); - if (version != 1) + if (version != 2) { - throw new InvalidDataException($"Unsupported DBNet model version: {version}"); + throw new InvalidDataException( + $"Unsupported DBNet model version: {version}. Version 2 is the paper architecture (feature " + + "pyramid with two upsampling heads); version 1 files hold the earlier concatenation decoder, " + + "whose layout no longer exists and cannot be loaded into it."); } string name = reader.ReadString(); @@ -332,12 +302,9 @@ public override async Task LoadWeightsAsync(string pathOrUrl, CancellationToken // Read component weights EnsureBackbone.ReadParameters(reader); - _inConv.ReadParameters(reader); - _upConv1.ReadParameters(reader); - _upConv2.ReadParameters(reader); - _upConv3.ReadParameters(reader); - _probHead.ReadParameters(reader); - _threshHead.ReadParameters(reader); + _pyramid.ReadParameters(reader); + _probabilityHead.ReadParameters(reader); + _thresholdHead.ReadParameters(reader); } /// @@ -348,125 +315,16 @@ public override void SaveWeights(string path) // Write header writer.Write(0x44424E54); // "DBNT" in ASCII - writer.Write(1); // Version 1 + writer.Write(2); // Version 2: feature pyramid + upsampling heads writer.Write(Name); writer.Write(_hiddenDim); writer.Write(_k); // Write component weights EnsureBackbone.WriteParameters(writer); - _inConv.WriteParameters(writer); - _upConv1.WriteParameters(writer); - _upConv2.WriteParameters(writer); - _upConv3.WriteParameters(writer); - _probHead.WriteParameters(writer); - _threshHead.WriteParameters(writer); - } - - private Tensor ApplyReLU(Tensor x) - { - var result = new Tensor(x._shape); - for (int i = 0; i < x.Length; i++) - { - double val = NumOps.ToDouble(x[i]); - result[i] = NumOps.FromDouble(Math.Max(0, val)); - } - return result; - } - - private Tensor ApplySigmoid(Tensor x) - { - var result = new Tensor(x._shape); - for (int i = 0; i < x.Length; i++) - { - double val = NumOps.ToDouble(x[i]); - result[i] = NumOps.FromDouble(1.0 / (1.0 + Math.Exp(-val))); - } - return result; - } - - private Tensor UpsampleAndConcat(Tensor x, Tensor skip) - { - int batch = x.Shape[0]; - int xChannels = x.Shape[1]; - int skipChannels = skip.Shape[1]; - int targetH = skip.Shape[2]; - int targetW = skip.Shape[3]; - - var upsampled = BilinearUpsample(x, targetH, targetW); - var result = new Tensor(new[] { batch, xChannels + skipChannels, targetH, targetW }); - - for (int b = 0; b < batch; b++) - { - for (int c = 0; c < xChannels; c++) - { - for (int h = 0; h < targetH; h++) - { - for (int w = 0; w < targetW; w++) - { - result[b, c, h, w] = upsampled[b, c, h, w]; - } - } - } - - for (int c = 0; c < skipChannels; c++) - { - for (int h = 0; h < targetH; h++) - { - for (int w = 0; w < targetW; w++) - { - result[b, xChannels + c, h, w] = skip[b, c, h, w]; - } - } - } - } - - return result; - } - - private Tensor BilinearUpsample(Tensor x, int targetH, int targetW) - { - int batch = x.Shape[0]; - int channels = x.Shape[1]; - int srcH = x.Shape[2]; - int srcW = x.Shape[3]; - - var result = new Tensor(new[] { batch, channels, targetH, targetW }); - - for (int b = 0; b < batch; b++) - { - for (int c = 0; c < channels; c++) - { - for (int h = 0; h < targetH; h++) - { - for (int w = 0; w < targetW; w++) - { - double srcY = (double)h / targetH * srcH; - double srcX = (double)w / targetW * srcW; - - int y0 = (int)Math.Floor(srcY); - int x0 = (int)Math.Floor(srcX); - int y1 = Math.Min(y0 + 1, srcH - 1); - int x1 = Math.Min(x0 + 1, srcW - 1); - - double wy1 = srcY - y0; - double wy0 = 1.0 - wy1; - double wx1 = srcX - x0; - double wx0 = 1.0 - wx1; - - double v00 = NumOps.ToDouble(x[b, c, y0, x0]); - double v01 = NumOps.ToDouble(x[b, c, y0, x1]); - double v10 = NumOps.ToDouble(x[b, c, y1, x0]); - double v11 = NumOps.ToDouble(x[b, c, y1, x1]); - - double val = wy0 * (wx0 * v00 + wx1 * v01) + wy1 * (wx0 * v10 + wx1 * v11); - result[b, c, h, w] = NumOps.FromDouble(val); - } - } - } - } - - return result; + _pyramid.WriteParameters(writer); + _probabilityHead.WriteParameters(writer); + _thresholdHead.WriteParameters(writer); } private List> FindConnectedComponents(bool[,] mask, int height, int width) @@ -618,3 +476,133 @@ private double ComputeBoxIoU(BoundingBox a, BoundingBox b) return union > 0 ? intersect / union : 0; } } + +/// +/// DBNet's feature pyramid (Liao et al. 2020; mmocr FPNC): 1x1 lateral convolutions to a common +/// width, a top-down merge, 3x3 smoothing to a quarter of that width per level, and all levels +/// upsampled to the finest level's resolution and concatenated. +/// +internal sealed class DbFeaturePyramid : CvParameterModule +{ + private readonly Conv2D[] _lateral; + private readonly Conv2D[] _smooth; + + public DbFeaturePyramid(IReadOnlyList stageChannels, int width) + { + if (width % 4 != 0) + { + throw new ArgumentException($"The pyramid width must be divisible by 4; got {width}.", nameof(width)); + } + + _lateral = stageChannels.Select(channels => new Conv2D(channels, width, kernelSize: 1)).ToArray(); + _smooth = stageChannels.Select(_ => new Conv2D(width, width / 4, kernelSize: 3, padding: 1)).ToArray(); + } + + public Tensor Forward(List> features) + { + if (features.Count != _lateral.Length) + { + throw new ArgumentException( + $"Expected {_lateral.Length} backbone stages, got {features.Count}.", nameof(features)); + } + + var engine = AiDotNetEngine.Current; + int levels = features.Count; + var merged = new Tensor[levels]; + for (int i = levels - 1; i >= 0; i--) + { + var lateral = _lateral[i].Forward(features[i]); + merged[i] = i == levels - 1 + ? lateral + : engine.TensorAdd(lateral, CvTensorOps.ResizeNearest(merged[i + 1], lateral.Shape[2], lateral.Shape[3])); + } + + int height = merged[0].Shape[2], width = merged[0].Shape[3]; + var smoothed = new Tensor[levels]; + for (int i = 0; i < levels; i++) + { + // Deepest level first, as the reference implementation concatenates them. + var level = _smooth[levels - 1 - i].Forward(merged[levels - 1 - i]); + smoothed[i] = i == levels - 1 ? level : CvTensorOps.ResizeNearest(level, height, width); + } + + return engine.TensorConcatenate(smoothed, 1); + } + + protected override IEnumerable?> ParameterChildren() => _lateral.Concat(_smooth); + + public void WriteParameters(BinaryWriter writer) + { + foreach (var conv in _lateral.Concat(_smooth)) + { + conv.WriteParameters(writer); + } + } + + public void ReadParameters(BinaryReader reader) + { + foreach (var conv in _lateral.Concat(_smooth)) + { + conv.ReadParameters(reader); + } + } +} + +/// +/// One DBNet prediction head (probability or threshold): 3x3 convolution to a quarter of the width, +/// batch norm and ReLU, a stride-2 transposed convolution with batch norm and ReLU, and a stride-2 +/// transposed convolution to one channel, then a sigmoid - from 1/4 resolution back to full. +/// +internal sealed class DbHead : CvParameterModule +{ + private readonly Conv2D _conv; + private readonly BatchNorm2D _norm1; + private readonly ConvTranspose2D _up1; + private readonly BatchNorm2D _norm2; + private readonly ConvTranspose2D _up2; + + public DbHead(int width) + { + int inner = width / 4; + _conv = new Conv2D(width, inner, kernelSize: 3, padding: 1); + _norm1 = new BatchNorm2D(inner); + _up1 = new ConvTranspose2D(inner, inner, kernelSize: 2, stride: 2); + _norm2 = new BatchNorm2D(inner); + _up2 = new ConvTranspose2D(inner, 1, kernelSize: 2, stride: 2); + } + + public Tensor Forward(Tensor x) + { + var engine = AiDotNetEngine.Current; + var h = engine.ReLU(_norm1.Forward(_conv.Forward(x))); + h = engine.ReLU(_norm2.Forward(_up1.Forward(h))); + return engine.Sigmoid(_up2.Forward(h)); + } + + public void SetTrainingMode(bool training) + { + _norm1.SetTrainingMode(training); + _norm2.SetTrainingMode(training); + } + + protected override IEnumerable?> ParameterChildren() + => new IParameterSource?[] { _conv, _norm1, _up1, _norm2, _up2 }; + + public void WriteParameters(BinaryWriter writer) + { + _conv.WriteParameters(writer); + _norm1.WriteParameters(writer); + _up1.WriteParameters(writer); + _norm2.WriteParameters(writer); + _up2.WriteParameters(writer); + } + + public void ReadParameters(BinaryReader reader) + { + _conv.ReadParameters(reader); + _norm1.ReadParameters(reader); + _up1.ReadParameters(reader); + _norm2.ReadParameters(reader); + _up2.ReadParameters(reader); + } +} diff --git a/src/ComputerVision/Detection/TextDetection/EAST.cs b/src/ComputerVision/Detection/TextDetection/EAST.cs index 4a6b184641..60affd18e5 100644 --- a/src/ComputerVision/Detection/TextDetection/EAST.cs +++ b/src/ComputerVision/Detection/TextDetection/EAST.cs @@ -37,7 +37,7 @@ namespace AiDotNet.ComputerVision.Detection.TextDetection; "https://arxiv.org/abs/1704.03155", Year = 2017, Authors = "Xinyu Zhou, Cong Yao, He Wen, Yuzhi Wang, Shuchang Zhou, Weiran He, Jiajun Liang")] -public class EAST : TextDetectorBase +public partial class EAST : TextDetectorBase { private readonly Conv2D _mergeConv1; private readonly Conv2D _mergeConv2; @@ -65,10 +65,15 @@ public EAST(TextDetectionOptions options, bool useRotatedBoxes = true) : base Backbone = new ResNet(ResNetVariant.ResNet50); // Feature merging branch (U-Net style) - int backboneChannels = Backbone.OutputChannels[^1]; - _mergeConv1 = new Conv2D(backboneChannels, _hiddenDim, kernelSize: 1); - _mergeConv2 = new Conv2D(_hiddenDim * 2, _hiddenDim, kernelSize: 3, padding: 1); - _mergeConv3 = new Conv2D(_hiddenDim * 2, _hiddenDim, kernelSize: 3, padding: 1); + var stageChannels = Backbone.OutputChannels; + _mergeConv1 = new Conv2D(stageChannels[^1], _hiddenDim, kernelSize: 1); + // Each merge conv receives the upsampled decoder map CONCATENATED with a raw backbone + // stage, so its input width is the decoder width plus that stage's channel count. These were + // declared as twice the decoder width, which matches no backbone stage, so the first merge + // threw on a channel mismatch (e.g. ResNet-50's C4: 256 + 1024 = 1280 channels into a conv + // built for 512) and the model could not run a forward pass at all. + _mergeConv2 = new Conv2D(_hiddenDim + stageChannels[^2], _hiddenDim, kernelSize: 3, padding: 1); + _mergeConv3 = new Conv2D(_hiddenDim + stageChannels[^3], _hiddenDim, kernelSize: 3, padding: 1); _mergeConv4 = new Conv2D(_hiddenDim, _hiddenDim / 2, kernelSize: 3, padding: 1); // Output heads @@ -124,24 +129,24 @@ protected override List> Forward(Tensor input) // Feature merging (U-Net style) var x = _mergeConv1.Forward(features[^1]); - x = ApplyBatchNormReLU(x); + x = ApplyMergeActivation(x); if (features.Count > 1) { x = UpsampleAndConcat(x, features[^2]); x = _mergeConv2.Forward(x); - x = ApplyBatchNormReLU(x); + x = ApplyMergeActivation(x); } if (features.Count > 2) { x = UpsampleAndConcat(x, features[^3]); x = _mergeConv3.Forward(x); - x = ApplyBatchNormReLU(x); + x = ApplyMergeActivation(x); } x = _mergeConv4.Forward(x); - x = ApplyBatchNormReLU(x); + x = ApplyMergeActivation(x); // Predict score and geometry var score = _scoreHead.Forward(x); @@ -370,111 +375,31 @@ public override void SaveWeights(string path) _geometryHead.WriteParameters(writer); } - private Tensor ApplyBatchNormReLU(Tensor x) - { - var result = new Tensor(x._shape); - for (int i = 0; i < x.Length; i++) - { - double val = NumOps.ToDouble(x[i]); - result[i] = NumOps.FromDouble(Math.Max(0, val)); - } - return result; - } + /// + /// ReLU. (This was named ApplyBatchNormReLU, but it never normalised anything: EAST's merge branch + /// here has no batch-norm parameters, so the name described a step the model does not take.) + /// + private Tensor ApplyMergeActivation(Tensor x) => Engine.ReLU(x); - private Tensor ApplySigmoid(Tensor x) - { - var result = new Tensor(x._shape); - for (int i = 0; i < x.Length; i++) - { - double val = NumOps.ToDouble(x[i]); - result[i] = NumOps.FromDouble(1.0 / (1.0 + Math.Exp(-val))); - } - return result; - } + /// + /// Elementwise Sigmoid, delegated to the engine. + /// + /// + /// This was a scalar loop that read each element out to double and wrote a fresh + /// tensor. Arithmetically identical, but it severed the autodiff tape: the gradient chain + /// stopped here, so every trainable layer UPSTREAM of this call received no gradient and + /// silently never trained. The engine op records itself on the tape. + /// + private Tensor ApplySigmoid(Tensor x) => Engine.Sigmoid(x); private Tensor UpsampleAndConcat(Tensor x, Tensor skip) - { - int batch = x.Shape[0]; - int xChannels = x.Shape[1]; - int skipChannels = skip.Shape[1]; - int targetH = skip.Shape[2]; - int targetW = skip.Shape[3]; - - var upsampled = BilinearUpsample(x, targetH, targetW); - var result = new Tensor(new[] { batch, xChannels + skipChannels, targetH, targetW }); - - for (int b = 0; b < batch; b++) - { - for (int c = 0; c < xChannels; c++) - { - for (int h = 0; h < targetH; h++) - { - for (int w = 0; w < targetW; w++) - { - result[b, c, h, w] = upsampled[b, c, h, w]; - } - } - } - - for (int c = 0; c < skipChannels; c++) - { - for (int h = 0; h < targetH; h++) - { - for (int w = 0; w < targetW; w++) - { - result[b, xChannels + c, h, w] = skip[b, c, h, w]; - } - } - } - } - - return result; - } + // Upsample to the skip connection's resolution, then stack along channels. Tape-visible, so + // the decoder's gradient reaches the backbone through every skip. + => CvTensorOps.ConcatChannels(BilinearUpsample(x, skip.Shape[2], skip.Shape[3]), skip); private Tensor BilinearUpsample(Tensor x, int targetH, int targetW) - { - int batch = x.Shape[0]; - int channels = x.Shape[1]; - int srcH = x.Shape[2]; - int srcW = x.Shape[3]; - - var result = new Tensor(new[] { batch, channels, targetH, targetW }); - - for (int b = 0; b < batch; b++) - { - for (int c = 0; c < channels; c++) - { - for (int h = 0; h < targetH; h++) - { - for (int w = 0; w < targetW; w++) - { - double srcY = (double)h / targetH * srcH; - double srcX = (double)w / targetW * srcW; - - int y0 = (int)Math.Floor(srcY); - int x0 = (int)Math.Floor(srcX); - int y1 = Math.Min(y0 + 1, srcH - 1); - int x1 = Math.Min(x0 + 1, srcW - 1); - - double wy1 = srcY - y0; - double wy0 = 1.0 - wy1; - double wx1 = srcX - x0; - double wx0 = 1.0 - wx1; - - double v00 = NumOps.ToDouble(x[b, c, y0, x0]); - double v01 = NumOps.ToDouble(x[b, c, y0, x1]); - double v10 = NumOps.ToDouble(x[b, c, y1, x0]); - double v11 = NumOps.ToDouble(x[b, c, y1, x1]); - - double val = wy0 * (wx0 * v00 + wx1 * v01) + wy1 * (wx0 * v10 + wx1 * v11); - result[b, c, h, w] = NumOps.FromDouble(val); - } - } - } - } - - return result; - } + // Asymmetric bilinear (src = dst * in / out, no half-pixel offset), as the loop it replaces. + => CvTensorOps.ResizeBilinearAsymmetric(x, targetH, targetW); private List> ApplyTextNMS(List> regions, double iouThreshold) { diff --git a/src/ComputerVision/Detection/TextDetection/TextDetectorBase.cs b/src/ComputerVision/Detection/TextDetection/TextDetectorBase.cs index 63cbe4a102..6d15a640fa 100644 --- a/src/ComputerVision/Detection/TextDetection/TextDetectorBase.cs +++ b/src/ComputerVision/Detection/TextDetection/TextDetectorBase.cs @@ -219,6 +219,10 @@ public abstract partial class TextDetectorBase : ModelBase, Tens /// Gets the backbone network, throwing if not initialized. /// /// Thrown when backbone has not been initialized. + // An accessor over the Backbone field, not separate storage. Without the alias the generator + // registered BOTH, so these weights were counted twice in the flat parameter vector, and for a + // detector without a neck (DETR) reading parameters threw from the accessor's null check. + [AiDotNet.Attributes.ParameterAlias(nameof(Backbone))] protected IDetectionBackbone EnsureBackbone => Backbone ?? throw new InvalidOperationException( $"{GetType().Name}: Backbone not initialized. Ensure the model is properly constructed."); @@ -229,8 +233,16 @@ public abstract partial class TextDetectorBase : ModelBase, Tens public abstract string Name { get; } /// - /// Creates a new text detector. + /// Gets the maximum number of text regions kept for a single image. /// + public int MaxDetections => Options.MaxDetections; + + /// + /// Gets the default minimum confidence a text region needs to be reported. + /// + public double ConfidenceThreshold => NumOps.ToDouble(Options.ConfidenceThreshold); + + /// Creates a new text detector. protected TextDetectorBase(TextDetectionOptions options) { Options = options; @@ -253,57 +265,44 @@ protected TextDetectorBase(TextDetectionOptions options) /// protected virtual Tensor Preprocess(Tensor image) { - // Standard preprocessing: resize to input size, normalize - int targetH = Options.InputSize[0]; - int targetW = Options.InputSize[1]; + var prepared = PreprocessCore(image); + NoteResolvedInput(prepared); + return prepared; + } - int batch = image.Shape[0]; - int channels = image.Shape[1]; - int height = image.Shape[2]; - int width = image.Shape[3]; + private Tensor PreprocessCore(Tensor image) + { + var (height, width) = GetValidatedInputSize(); + // Keep the original asymmetric pixel mapping, but execute through the selected engine so + // prediction/training share inference's pixel domain without forcing GPU data onto the CPU + // or cutting the gradient path to an upstream image-producing model. + var resized = CvTensorOps.ResizeBilinearAsymmetric( + image, height, width); + return Engine.TensorMultiplyScalar(resized, NumOps.FromDouble(1.0 / 255.0)); + } - // Create resized output - var output = new Tensor(new[] { batch, channels, targetH, targetW }); + private (int Height, int Width) GetValidatedInputSize() + { + // InputSize is publicly mutable: validate at each consuming boundary and return the + // validated values rather than reading the caller-owned array again after validation. + var inputSize = Options.InputSize; + if (inputSize is null || inputSize.Length != 2) + { + throw new ArgumentException( + "InputSize must contain exactly two positive dimensions [height, width].", + nameof(Options.InputSize)); + } - // Bilinear interpolation resize - for (int b = 0; b < batch; b++) + int height = inputSize[0]; + int width = inputSize[1]; + if (height <= 0 || width <= 0) { - for (int c = 0; c < channels; c++) - { - for (int h = 0; h < targetH; h++) - { - for (int w = 0; w < targetW; w++) - { - double srcY = (double)h / targetH * height; - double srcX = (double)w / targetW * width; - - int y0 = (int)Math.Floor(srcY); - int x0 = (int)Math.Floor(srcX); - int y1 = Math.Min(y0 + 1, height - 1); - int x1 = Math.Min(x0 + 1, width - 1); - - double wy1 = srcY - y0; - double wy0 = 1.0 - wy1; - double wx1 = srcX - x0; - double wx0 = 1.0 - wx1; - - double v00 = NumOps.ToDouble(image[b, c, y0, x0]); - double v01 = NumOps.ToDouble(image[b, c, y0, x1]); - double v10 = NumOps.ToDouble(image[b, c, y1, x0]); - double v11 = NumOps.ToDouble(image[b, c, y1, x1]); - - double val = wy0 * (wx0 * v00 + wx1 * v01) + wy1 * (wx0 * v10 + wx1 * v11); - - // Normalize to [0, 1] - val /= 255.0; - - output[b, c, h, w] = NumOps.FromDouble(val); - } - } - } + throw new ArgumentException( + "InputSize must contain exactly two positive dimensions [height, width].", + nameof(Options.InputSize)); } - return output; + return (height, width); } /// @@ -428,12 +427,64 @@ private double PerpendicularDistance( #region ModelBase Overrides /// - /// Predicts by returning the preprocessed input (text detection is done via Detect method). + /// Predicts by running the forward pass and returning the primary output map. /// - public override Tensor Predict(Tensor input) => Preprocess(input); + /// + /// This used to return Preprocess(input) -- the resized, normalised INPUT IMAGE -- so + /// the model reported its own input back as a prediction. Nothing downstream could tell, + /// because the returned tensor has a plausible shape. It now runs the network, matching + /// ObjectDetectorBase.Predict: every output map is flattened per image and concatenated, + /// so a model with a probability map and a threshold map (DBNet) or a score and a geometry map + /// (EAST) exposes both, and training against the prediction reaches both heads. + /// + public override Tensor Predict(Tensor input) + { + return CvTensorOps.ConcatenateOutputs(Forward(Preprocess(input))); + } /// - public override void Train(Tensor input, Tensor expectedOutput) { } + /// + /// Gets the step size used by . + /// + /// + /// Detection losses are large early in training, so this is deliberately conservative. + /// Override it to match a paper recipe. + /// + protected virtual double TrainingLearningRate => 0.001; + + /// + /// Runs one training step against the model's public prediction. + /// + /// The training image. + /// The desired output, shaped like . + /// + /// Previously an empty method, so text detectors ignored training entirely. See + /// ObjectDetectorBase.Train for the mechanism and its limits. + /// + public override void Train(Tensor input, Tensor expectedOutput) + { + if (input is null) + { + throw new ArgumentNullException(nameof(input)); + } + + if (expectedOutput is null) + { + throw new ArgumentNullException(nameof(expectedOutput)); + } + + bool wasTraining = IsTrainingMode; + SetTrainingMode(true); + try + { + RecordTrainingLoss(TensorModelTrainer.Step( + this, input, expectedOutput, NumOps.FromDouble(TrainingLearningRate), Predict)); + } + finally + { + SetTrainingMode(wasTraining); + } + } /// public override ILossFunction DefaultLossFunction => new MeanSquaredErrorLoss(); @@ -447,8 +498,125 @@ public override IFullModel, Tensor> WithParameters(Vector par } /// - public override IFullModel, Tensor> DeepCopy() - => (TextDetectorBase)MemberwiseClone(); + // See the note on ObjectDetectorBase: MemberwiseClone gave a shallow copy that shared + // weights with the original. ModelBase's rebuild-and-reload DeepCopy is correct here. #endregion + + /// + /// The shape of the first input this model's forward pass ran on. Its lazily-shaped layers sized + /// their weights from it, so replaying it on a rebuilt copy reproduces the same parameter + /// topology. Scratch: never persisted, and rebuilt copies record their own. + /// + [AiDotNet.Attributes.Scratch] + private int[]? _resolvedInputShape; + + /// Records the input shape on the first forward pass. + private void NoteResolvedInput(Tensor input) + { + if (_resolvedInputShape is not null || input is null) + { + return; + } + + var shape = new int[input.Shape.Length]; + for (int i = 0; i < shape.Length; i++) + { + shape[i] = input.Shape[i]; + } + + _resolvedInputShape = shape; + } + + /// + /// + /// Runs the copy once on a zero input of the shape this model has already processed, so its + /// lazily-shaped layers (the convolutions behind the Conv2D adapter, the backbone's lazy layers) + /// size their weights exactly as this model's did before its state is loaded into them. + /// + protected override void PrepareCopyForStateRestore(ModelBase, Tensor> copy) + { + if (_resolvedInputShape is not null && copy is TextDetectorBase rebuilt) + { + var shape = (int[])_resolvedInputShape.Clone(); + shape[0] = 1; + rebuilt.Predict(new Tensor(shape)); + } + } + + /// + /// Gets the number of channels in the images this model reads. + /// + /// RGB unless a model overrides it; every backbone here is built for three channels. + protected virtual int InputChannels => 3; + + /// + /// Gives a model that has never run a concrete parameter topology, so its state can be captured. + /// + /// + /// Several layers size their weights on their first forward pass. Until then the model reports + /// its parameters as shape-deferred, which is correct for a parameter query but made + /// - and therefore Clone - throw on a freshly constructed model. + /// Running the network once on a zero image of the configured input size resolves exactly the + /// shapes the first real image would, because every image is resized to that size first. + /// + private void ResolveDeferredParameters() + { + if (_resolvedInputShape is not null) + { + return; + } + + var (height, width) = GetValidatedInputSize(); + Predict(new Tensor(new[] { 1, InputChannels, height, width })); + } + + /// + /// Resolves shape-deferred layers first; see . + public override byte[] Serialize() + { + ResolveDeferredParameters(); + return base.Serialize(); + } + + /// + /// The loss of the most recent call, measured before its update. + /// + [AiDotNet.Attributes.Scratch] + private T _lastTrainingLoss = MathHelper.GetNumericOperations().Zero; + + /// + /// Gets the loss of the most recent call, measured on that call's input before + /// its update (zero before the first call). + /// + /// The training objective's value: mean squared error, or the model's own loss where it + /// has one. + /// Same contract as INeuralNetwork<T>.GetLastLoss. + public T GetLastLoss() => _lastTrainingLoss; + + /// Records the loss a training step reported. + /// The step's loss. + protected void RecordTrainingLoss(T loss) => _lastTrainingLoss = loss; + + /// + /// Whether the model is in training mode. + /// + protected bool IsTrainingMode; + + /// + /// Sets the model to training or inference mode. + /// + /// True for training mode, false for inference. + /// + /// Batch normalisation depends on it: batch statistics (and running-statistic updates) while + /// training, running statistics at inference. The text detectors had no such switch, so their + /// backbone's batch-norm layers never left inference mode, even inside - + /// unlike the object detectors, which have always switched theirs. Override to forward the mode to + /// head modules that depend on it, calling the base. + /// + public virtual void SetTrainingMode(bool training) + { + IsTrainingMode = training; + Backbone?.SetTrainingMode(training); + } } diff --git a/src/ComputerVision/OCR/OCRBase.cs b/src/ComputerVision/OCR/OCRBase.cs index e0f85140b7..b36737aa59 100644 --- a/src/ComputerVision/OCR/OCRBase.cs +++ b/src/ComputerVision/OCR/OCRBase.cs @@ -215,6 +215,21 @@ public abstract class OCRBase : ModelBase, Tensor> /// protected readonly Dictionary CharToIndex; + /// + /// Gets the set of characters this model can emit. + /// + /// + /// Recognition decodes class indices into characters from this set, so a caller needs it to + /// know what the model is capable of reading -- and every character in the recognised text + /// must come from it. + /// + public string CharacterSet => Options.CharacterSet ?? DefaultCharacterSet; + + /// + /// Gets the maximum number of characters the decoder will emit for one text region. + /// + public int MaxSequenceLength => Options.MaxSequenceLength; + /// /// Index to character mapping. /// @@ -270,6 +285,13 @@ protected OCRBase(OCROptions options) /// Preprocesses a text crop for recognition. /// protected virtual Tensor PreprocessCrop(Tensor crop) + { + var prepared = PreprocessCropCore(crop); + NoteResolvedInput(prepared); + return prepared; + } + + private Tensor PreprocessCropCore(Tensor crop) { int targetH = Options.RecognitionHeight; int srcH = crop.Shape[2]; @@ -303,7 +325,9 @@ protected string DecodeCTC(Tensor logits) var result = new List(); int prevIndex = 0; - for (int t = 0; t < seqLen; t++) + // The budget is emitted characters, not timesteps: CTC blanks and repeats consume + // sequence positions without consuming the caller's character budget. + for (int t = 0; t < seqLen && result.Count < Options.MaxSequenceLength; t++) { // Find argmax int maxIdx = 0; @@ -344,7 +368,7 @@ protected string DecodeAttention(Tensor logits, int endTokenId) var result = new List(); - for (int t = 0; t < seqLen; t++) + for (int t = 0; t < seqLen && result.Count < Options.MaxSequenceLength; t++) { int maxIdx = 0; double maxVal = double.NegativeInfinity; @@ -490,35 +514,63 @@ protected Tensor ResizeBilinear(Tensor input, int targetH, int targetW) #region ModelBase Overrides /// - /// Runs OCR and returns region info as a tensor [numRegions, 6]. - /// Columns: confidence, textLength, x1, y1, x2, y2. + /// Returns the model's raw, differentiable recognition output (see ). /// + /// + /// Use to read text. used to run + /// and pack its decoded regions into a [regions, 6] tensor of + /// confidence, text length and box - a decoded summary that has no gradient, so nothing trained + /// against it could ever learn. It now returns the network output that + /// fits, matching the detection bases. + /// public override Tensor Predict(Tensor input) { - var result = Recognize(input); - int regions = result.TextRegions.Count; - if (regions == 0) - return new Tensor([0, 6]); + NoteResolvedInput(input); + return ForwardLogits(input); + } + + /// + /// Runs the differentiable recognition forward pass on an image and returns its raw output: + /// per-timestep character logits for a CTC recognizer, the encoder output and first decoding step + /// for an encoder-decoder recognizer. Every trainable weight must be reachable from it. + /// + /// The image or cropped text line, NCHW. + /// The raw recognition output that fits. + protected abstract Tensor ForwardLogits(Tensor image); + + /// + /// Gets the step size used by . Override it to match a paper recipe. + /// + protected virtual double TrainingLearningRate => 0.001; + + /// + /// + /// Runs one training step against the model's raw recognition output. + /// + /// The training image. + /// The desired output, shaped like . + /// + /// This was an empty method, so CRNN and TrOCR ignored training entirely. The step records + /// on a gradient tape, takes mean squared error against + /// and updates every live trainable weight. A recognition loss + /// (CTC, or teacher-forced cross-entropy on target text) is the right objective for a full + /// training recipe and belongs in an override; this base step is what makes the models trainable. + /// + public override void Train(Tensor input, Tensor expectedOutput) + { + if (input is null) + { + throw new ArgumentNullException(nameof(input)); + } - var output = new Tensor([regions, 6]); - for (int i = 0; i < regions; i++) + if (expectedOutput is null) { - var region = result.TextRegions[i]; - output[i, 0] = region.Confidence; - output[i, 1] = NumOps.FromDouble(region.Text.Length); - if (region.Box is not null) - { - output[i, 2] = region.Box.X1; - output[i, 3] = region.Box.Y1; - output[i, 4] = region.Box.X2; - output[i, 5] = region.Box.Y2; - } + throw new ArgumentNullException(nameof(expectedOutput)); } - return output; - } - /// - public override void Train(Tensor input, Tensor expectedOutput) { } + RecordTrainingLoss(TensorModelTrainer.Step( + this, input, expectedOutput, NumOps.FromDouble(TrainingLearningRate), ForwardLogits)); + } /// public override ILossFunction DefaultLossFunction => new MeanSquaredErrorLoss(); @@ -532,8 +584,102 @@ public override IFullModel, Tensor> WithParameters(Vector par } /// - public override IFullModel, Tensor> DeepCopy() - => (OCRBase)MemberwiseClone(); + // See the note on ObjectDetectorBase: MemberwiseClone gave a shallow copy that shared + // weights with the original. ModelBase's rebuild-and-reload DeepCopy is correct here. #endregion + + /// + /// The shape of the first input this model's forward pass ran on. Its lazily-shaped layers sized + /// their weights from it, so replaying it on a rebuilt copy reproduces the same parameter + /// topology. Scratch: never persisted, and rebuilt copies record their own. + /// + [AiDotNet.Attributes.Scratch] + private int[]? _resolvedInputShape; + + /// Records the input shape on the first forward pass. + private void NoteResolvedInput(Tensor input) + { + if (_resolvedInputShape is not null || input is null) + { + return; + } + + var shape = new int[input.Shape.Length]; + for (int i = 0; i < shape.Length; i++) + { + shape[i] = input.Shape[i]; + } + + _resolvedInputShape = shape; + } + + /// + /// + /// Runs the copy once on a zero input of the shape this model has already processed, so its + /// lazily-shaped layers (the convolutions behind the Conv2D adapter, the backbone's lazy layers) + /// size their weights exactly as this model's did before its state is loaded into them. + /// + protected override void PrepareCopyForStateRestore(ModelBase, Tensor> copy) + { + if (_resolvedInputShape is not null && copy is OCRBase rebuilt) + { + var shape = (int[])_resolvedInputShape.Clone(); + shape[0] = 1; + rebuilt.Predict(new Tensor(shape)); + } + } + + /// + /// Gets the number of channels in the images this model reads. + /// + /// RGB unless a model overrides it; every backbone here is built for three channels. + protected virtual int InputChannels => 3; + + /// + /// Gives a model that has never run a concrete parameter topology, so its state can be captured. + /// + /// + /// Several layers size their weights on their first forward pass. Until then the model reports + /// its parameters as shape-deferred, which is correct for a parameter query but made + /// - and therefore Clone - throw on a freshly constructed model. + /// Running the network once on a zero image of the configured recognition height and maximum width resolves exactly the + /// shapes the first real image would, because every image is resized to that size first. + /// + private void ResolveDeferredParameters() + { + if (_resolvedInputShape is not null) + { + return; + } + + Predict(new Tensor(new[] { 1, InputChannels, Options.RecognitionHeight, Options.MaxRecognitionWidth })); + } + + /// + /// Resolves shape-deferred layers first; see . + public override byte[] Serialize() + { + ResolveDeferredParameters(); + return base.Serialize(); + } + + /// + /// The loss of the most recent call, measured before its update. + /// + [AiDotNet.Attributes.Scratch] + private T _lastTrainingLoss = MathHelper.GetNumericOperations().Zero; + + /// + /// Gets the loss of the most recent call, measured on that call's input before + /// its update (zero before the first call). + /// + /// The training objective's value: mean squared error, or the model's own loss where it + /// has one. + /// Same contract as INeuralNetwork<T>.GetLastLoss. + public T GetLastLoss() => _lastTrainingLoss; + + /// Records the loss a training step reported. + /// The step's loss. + protected void RecordTrainingLoss(T loss) => _lastTrainingLoss = loss; } diff --git a/src/ComputerVision/OCR/Recognition/CRNN.cs b/src/ComputerVision/OCR/Recognition/CRNN.cs index 7c1a8e9188..35a309f765 100644 --- a/src/ComputerVision/OCR/Recognition/CRNN.cs +++ b/src/ComputerVision/OCR/Recognition/CRNN.cs @@ -56,26 +56,9 @@ public partial class CRNN : OCRBase private readonly Dense _outputLayer; private readonly int _hiddenDim; + // Part of the native weight-file configuration, even though lazy LSTMs infer their input shape. private readonly int _sequenceFeatureDim; - // LSTM state tracking - [Scratch] - private Tensor? _lstm1FwHidden; - [Scratch] - private Tensor? _lstm1FwCell; - [Scratch] - private Tensor? _lstm1BwHidden; - [Scratch] - private Tensor? _lstm1BwCell; - [Scratch] - private Tensor? _lstm2FwHidden; - [Scratch] - private Tensor? _lstm2FwCell; - [Scratch] - private Tensor? _lstm2BwHidden; - [Scratch] - private Tensor? _lstm2BwCell; - /// public override string Name => "CRNN"; @@ -85,6 +68,7 @@ public partial class CRNN : OCRBase public CRNN(OCROptions options) : base(options) { _hiddenDim = 256; + _sequenceFeatureDim = 512; // CNN backbone for feature extraction (VGG-style architecture) // Stage 1 @@ -102,44 +86,20 @@ public CRNN(OCROptions options) : base(options) // Stage 4 _conv7 = new Conv2D(512, 512, kernelSize: 2, padding: 0); - // After conv layers, assuming input height 32, the feature map height becomes 1 - // Width is preserved (roughly input_width / 4 due to pooling) - // Feature dimension = 512 channels * 1 height = 512 - _sequenceFeatureDim = 512; - // Bidirectional LSTM Layer 1 // Input: [batch, seqLen, 512], Output: [batch, seqLen, 256] - int[] inputShape1 = new[] { 1, _sequenceFeatureDim }; // [batch, features] for single timestep IActivationFunction tanhActivation = new TanhActivation(); _lstm1Forward = new LSTMLayer( _hiddenDim, tanhActivation); _lstm1Backward = new LSTMLayer( _hiddenDim, tanhActivation); // Bidirectional LSTM Layer 2 // Input: [batch, seqLen, 512 (256*2)], Output: [batch, seqLen, 256] - int[] inputShape2 = new[] { 1, _hiddenDim * 2 }; _lstm2Forward = new LSTMLayer( _hiddenDim, tanhActivation); _lstm2Backward = new LSTMLayer( _hiddenDim, tanhActivation); // Output layer to vocabulary (512 = 256*2 from bidirectional) _outputLayer = new Dense(_hiddenDim * 2, VocabularySize); - // Initialize LSTM states - ResetLSTMStates(1); - } - - /// - /// Resets the LSTM hidden and cell states. - /// - private void ResetLSTMStates(int batchSize) - { - _lstm1FwHidden = new Tensor(new[] { batchSize, _hiddenDim }); - _lstm1FwCell = new Tensor(new[] { batchSize, _hiddenDim }); - _lstm1BwHidden = new Tensor(new[] { batchSize, _hiddenDim }); - _lstm1BwCell = new Tensor(new[] { batchSize, _hiddenDim }); - _lstm2FwHidden = new Tensor(new[] { batchSize, _hiddenDim }); - _lstm2FwCell = new Tensor(new[] { batchSize, _hiddenDim }); - _lstm2BwHidden = new Tensor(new[] { batchSize, _hiddenDim }); - _lstm2BwCell = new Tensor(new[] { batchSize, _hiddenDim }); } /// @@ -175,15 +135,23 @@ public override OCRResult Recognize(Tensor image) /// public override (string text, T confidence) RecognizeText(Tensor croppedImage) { - int batch = croppedImage.Shape[0]; + var probs = ApplySoftmax(ComputeLogits(croppedImage)); + string text = DecodeCTC(probs); + T confidence = ComputeConfidence(probs, text); + return (text, confidence); + } - // Reset LSTM states for new sequence - ResetLSTMStates(batch); + /// + /// Per-timestep character logits [batch, width, vocabulary], before the softmax. + protected override Tensor ForwardLogits(Tensor image) => ComputeLogits(PreprocessCrop(image)); - // Convert to grayscale if needed + /// + /// CNN backbone, bidirectional LSTM and output projection: the differentiable part of CRNN. + /// + private Tensor ComputeLogits(Tensor croppedImage) + { var grayImage = ConvertToGrayscale(croppedImage); - // Forward pass through CNN backbone var x = _conv1.Forward(grayImage); x = ApplyReLU(x); x = MaxPool2D(x, 2, 2); @@ -211,23 +179,9 @@ public override (string text, T confidence) RecognizeText(Tensor croppedImage x = _conv7.Forward(x); x = ApplyReLU(x); - // Squeeze height dimension and transpose to (batch, width, channels) var seqFeatures = SqueezeAndPermute(x); - - // Bidirectional LSTM processing - var lstmOut = ApplyBidirectionalLSTM(seqFeatures, batch); - - // Output projection - var logits = ApplyOutputLayer(lstmOut); - - // Apply softmax for probabilities - var probs = ApplySoftmax(logits); - - // CTC decoding - string text = DecodeCTC(probs); - T confidence = ComputeConfidence(probs, text); - - return (text, confidence); + var lstmOut = ApplyBidirectionalLSTM(seqFeatures); + return ApplyOutputLayer(lstmOut); } /// @@ -235,151 +189,67 @@ public override (string text, T confidence) RecognizeText(Tensor croppedImage /// private Tensor ConvertToGrayscale(Tensor image) { - int batch = image.Shape[0]; int channels = image.Shape[1]; - int height = image.Shape[2]; - int width = image.Shape[3]; - if (channels == 1) { return image; } - var gray = new Tensor(new[] { batch, 1, height, width }); - - for (int b = 0; b < batch; b++) - { - for (int h = 0; h < height; h++) - { - for (int w = 0; w < width; w++) - { - // Standard grayscale conversion: 0.299*R + 0.587*G + 0.114*B - double r = NumOps.ToDouble(image[b, 0, h, w]); - double g = channels > 1 ? NumOps.ToDouble(image[b, 1, h, w]) : r; - double bl = channels > 2 ? NumOps.ToDouble(image[b, 2, h, w]) : r; - - double grayVal = 0.299 * r + 0.587 * g + 0.114 * bl; - gray[b, 0, h, w] = NumOps.FromDouble(grayVal); - } - } - } - - return gray; + // gray = 0.299 R + 0.587 G + 0.114 B; a missing blue channel uses red. + var engine = AiDotNetEngine.Current; + var r = engine.TensorNarrow(image, 1, 0, 1); + var g = engine.TensorNarrow(image, 1, 1, 1); + var b = channels > 2 ? engine.TensorNarrow(image, 1, 2, 1) : r; + return engine.TensorAdd( + engine.TensorAdd(engine.TensorMultiplyScalar(r, NumOps.FromDouble(0.299)), engine.TensorMultiplyScalar(g, NumOps.FromDouble(0.587))), + engine.TensorMultiplyScalar(b, NumOps.FromDouble(0.114))); } /// /// Applies bidirectional LSTM using proper LSTMLayer cells. /// - private Tensor ApplyBidirectionalLSTM(Tensor x, int batch) + private Tensor ApplyBidirectionalLSTM(Tensor x) { - // x: [batch, seq_len, features] - int seqLen = x.Shape[1]; - int features = x.Shape[2]; - - // First bidirectional layer - var fw1Outputs = new Tensor(new[] { batch, seqLen, _hiddenDim }); - var bw1Outputs = new Tensor(new[] { batch, seqLen, _hiddenDim }); - - // Forward direction - _lstm1Forward.ResetState(); - for (int t = 0; t < seqLen; t++) - { - var input = ExtractTimestep(x, t, batch, features); - var output = _lstm1Forward.Forward(input); - StoreTimestep(fw1Outputs, output, t, batch, _hiddenDim); - } - - // Backward direction - _lstm1Backward.ResetState(); - for (int t = seqLen - 1; t >= 0; t--) - { - var input = ExtractTimestep(x, t, batch, features); - var output = _lstm1Backward.Forward(input); - StoreTimestep(bw1Outputs, output, t, batch, _hiddenDim); - } - - // Concatenate forward and backward outputs - var concat1 = ConcatenateBidirectional(fw1Outputs, bw1Outputs, batch, seqLen, _hiddenDim); - - // Second bidirectional layer - var fw2Outputs = new Tensor(new[] { batch, seqLen, _hiddenDim }); - var bw2Outputs = new Tensor(new[] { batch, seqLen, _hiddenDim }); - - // Forward direction - _lstm2Forward.ResetState(); - for (int t = 0; t < seqLen; t++) - { - var input = ExtractTimestep(concat1, t, batch, _hiddenDim * 2); - var output = _lstm2Forward.Forward(input); - StoreTimestep(fw2Outputs, output, t, batch, _hiddenDim); - } - - // Backward direction - _lstm2Backward.ResetState(); - for (int t = seqLen - 1; t >= 0; t--) - { - var input = ExtractTimestep(concat1, t, batch, _hiddenDim * 2); - var output = _lstm2Backward.Forward(input); - StoreTimestep(bw2Outputs, output, t, batch, _hiddenDim); - } - - // Final concatenation - return ConcatenateBidirectional(fw2Outputs, bw2Outputs, batch, seqLen, _hiddenDim); + var layer1 = ConcatenateBidirectional( + RunDirection(_lstm1Forward, x, reverse: false), RunDirection(_lstm1Backward, x, reverse: true)); + return ConcatenateBidirectional( + RunDirection(_lstm2Forward, layer1, reverse: false), RunDirection(_lstm2Backward, layer1, reverse: true)); } /// - /// Extracts a single timestep from the sequence tensor. + /// Runs one LSTM direction over the whole sequence [batch, seqLen, features] and returns + /// its outputs [batch, seqLen, hidden] in the original time order. /// - private Tensor ExtractTimestep(Tensor x, int t, int batch, int features) + /// + /// + /// The sequence goes to the layer in ONE call, which carries the hidden and cell state from step + /// to step inside it. This used to feed the layer one timestep at a time as a + /// [batch, features] tensor - which the layer reads as a [timeSteps, features] + /// sequence of batch one, starting from zero state on every call. Each "step" was therefore an + /// independent one-step LSTM: nothing was carried across time, the recurrent weights and the + /// forget gate multiplied zero state and never received a gradient (24 of the model's 64 + /// trainable tensors), and a batch larger than one was misread as time. + /// + /// + /// The backward direction reverses time with an engine gather before and after the layer, so the + /// flip stays on the gradient tape. + /// + /// + private Tensor RunDirection(LSTMLayer lstm, Tensor x, bool reverse) { - var timestep = new Tensor(new[] { batch, features }); - - for (int b = 0; b < batch; b++) - { - for (int f = 0; f < features; f++) - { - timestep[b, f] = x[b, t, f]; - } - } + int seqLen = x.Shape[1]; + var reversed = reverse ? Enumerable.Range(0, seqLen).Reverse().ToArray() : null; - return timestep; - } - - /// - /// Stores LSTM output into the sequence tensor at a specific timestep. - /// - private void StoreTimestep(Tensor output, Tensor lstmOut, int t, int batch, int hiddenDim) - { - for (int b = 0; b < batch; b++) - { - for (int h = 0; h < hiddenDim; h++) - { - output[b, t, h] = lstmOut[b, h]; - } - } + lstm.ResetState(); + var output = lstm.Forward(reversed is null ? x : CvTensorOps.Select(x, reversed, 1)); + return reversed is null ? output : CvTensorOps.Select(output, reversed, 1); } /// /// Concatenates forward and backward LSTM outputs. /// - private Tensor ConcatenateBidirectional(Tensor forward, Tensor backward, int batch, int seqLen, int hiddenDim) - { - var concat = new Tensor(new[] { batch, seqLen, hiddenDim * 2 }); - - for (int b = 0; b < batch; b++) - { - for (int t = 0; t < seqLen; t++) - { - for (int h = 0; h < hiddenDim; h++) - { - concat[b, t, h] = forward[b, t, h]; - concat[b, t, hiddenDim + h] = backward[b, t, h]; - } - } - } - - return concat; - } + private Tensor ConcatenateBidirectional(Tensor forward, Tensor backward) + => AiDotNetEngine.Current.TensorConcatenate(new[] { forward, backward }, 2); /// /// Applies softmax normalization across the vocabulary dimension. @@ -393,56 +263,10 @@ private Tensor ApplySoftmax(Tensor logits) /// Applies simple batch normalization. /// private Tensor ApplyBatchNorm(Tensor x) - { - int batch = x.Shape[0]; - int channels = x.Shape[1]; - int height = x.Shape[2]; - int width = x.Shape[3]; - - var result = new Tensor(x._shape); - double epsilon = 1e-5; - - for (int c = 0; c < channels; c++) - { - // Compute mean and variance for this channel - double sum = 0; - double sumSq = 0; - int count = batch * height * width; - - for (int b = 0; b < batch; b++) - { - for (int h = 0; h < height; h++) - { - for (int w = 0; w < width; w++) - { - double val = NumOps.ToDouble(x[b, c, h, w]); - sum += val; - sumSq += val * val; - } - } - } - - double mean = sum / count; - double variance = (sumSq / count) - (mean * mean); - double stdDev = Math.Sqrt(variance + epsilon); - - // Normalize - for (int b = 0; b < batch; b++) - { - for (int h = 0; h < height; h++) - { - for (int w = 0; w < width; w++) - { - double val = NumOps.ToDouble(x[b, c, h, w]); - double normalized = (val - mean) / stdDev; - result[b, c, h, w] = NumOps.FromDouble(normalized); - } - } - } - } - - return result; - } + // Normalises with the CURRENT batch's statistics (biased variance, no affine parameters), + // exactly as the loop it replaces. Note that this makes one image's output depend on what else + // is in its batch; it is preserved here and not silently changed. + => CvTensorOps.BatchStatisticsNorm(x, 1e-5); /// public override long GetParameterCount() @@ -737,119 +561,175 @@ private void LoadWeightsFromFile(string path) _outputLayer.ReadParameters(reader); } - private Tensor ApplyReLU(Tensor x) + /// + /// Elementwise ReLU, delegated to the engine. + /// + /// + /// This was a scalar loop that read each element out to double and wrote a fresh + /// tensor. Arithmetically identical, but it severed the autodiff tape: the gradient chain + /// stopped here, so every trainable layer UPSTREAM of this call received no gradient and + /// silently never trained. The engine op records itself on the tape. + /// + private Tensor ApplyReLU(Tensor x) => Engine.ReLU(x); + + private Tensor MaxPool2D(Tensor x, int kernelH, int kernelW) + => CvTensorOps.MaxPoolFloor(x, kernelH, kernelW); + + private Tensor SqueezeAndPermute(Tensor x) + { + // [batch, channels, height, width] -> [batch, width, channels * height], channel-major. + int batch = x.Shape[0], channels = x.Shape[1], height = x.Shape[2], width = x.Shape[3]; + return AiDotNetEngine.Current.Reshape(AiDotNetEngine.Current.TensorPermute(x, new[] { 0, 3, 1, 2 }), new[] { batch, width, channels * height }); + } + + private Tensor ApplyOutputLayer(Tensor x) => _outputLayer.ForwardTokens(x); + + /// + /// Runs one training step with the CTC loss. + /// + /// The text-line image. + /// + /// The target text as label ids [batch, length] (0 is the blank and is treated as padding), + /// or as per-column scores [batch, columns, vocabulary] - such as 's + /// output shape - whose greedy CTC decoding (most likely class per column, repeats merged, blanks + /// dropped) is the label sequence. + /// + /// + /// Connectionist temporal classification (Graves et al. 2006) is how CRNN is trained in the paper + /// (Shi et al. 2016) and every reference implementation: the loss sums over every alignment of the + /// label sequence to the image columns, so no per-column targets are needed. Reduced as PyTorch's + /// default does: each sequence's loss divided by its label length, then averaged over the batch. + /// + public override void Train(Tensor input, Tensor expectedOutput) { - var result = new Tensor(x._shape); - for (int i = 0; i < x.Length; i++) + if (input is null) { - double val = NumOps.ToDouble(x[i]); - result[i] = NumOps.FromDouble(Math.Max(0, val)); + throw new ArgumentNullException(nameof(input)); } - return result; + + if (expectedOutput is null) + { + throw new ArgumentNullException(nameof(expectedOutput)); + } + + var labels = CtcLabelsFrom(expectedOutput); + var ctc = new CTCLoss(VocabularySize, blankIndex: 0); + RecordTrainingLoss(TensorModelTrainer.Step( + this, input, EncodeCtcTargets(labels), NumOps.FromDouble(TrainingLearningRate), ForwardLogits, + (logits, encoded) => MeanCtcLoss(ctc, logits, encoded, labels))); } - private Tensor MaxPool2D(Tensor x, int kernelH, int kernelW) + private Tensor MeanCtcLoss(CTCLoss ctc, Tensor logits, Tensor encodedTargets, int[][] labels) { - int batch = x.Shape[0]; - int channels = x.Shape[1]; - int height = x.Shape[2]; - int width = x.Shape[3]; + int columns = logits.Shape[1]; + for (int b = 0; b < labels.Length; b++) + { + // CTC needs a column per label plus a blank between each pair of equal neighbours. + int required = labels[b].Length + labels[b].Where((label, i) => i > 0 && labels[b][i - 1] == label).Count(); + if (required > columns) + { + throw new ArgumentException( + $"Label sequence {b} needs at least {required} columns for CTC, but the recognizer produces {columns}."); + } + } - int outH = height / kernelH; - int outW = width / kernelW; + var perSequence = ctc.ComputeTapeLoss(Engine.TensorLogSoftmax(logits, axis: -1), encodedTargets); // [batch] + var weights = new Tensor(new[] { labels.Length }); + for (int b = 0; b < labels.Length; b++) + { + weights[b] = NumOps.FromDouble(1.0 / ((double)Math.Max(1, labels[b].Length) * labels.Length)); + } - var result = new Tensor(new[] { batch, channels, outH, outW }); + return Engine.ReduceSum(Engine.TensorMultiply(perSequence, weights), null); + } - for (int b = 0; b < batch; b++) + /// + /// Reads CTC label sequences from label ids [batch, length] or scores [batch, columns, vocabulary]. + /// + private int[][] CtcLabelsFrom(Tensor target) + { + if (target.Rank == 3 && target.Shape[2] == VocabularySize) { - for (int c = 0; c < channels; c++) + int batch = target.Shape[0], columns = target.Shape[1]; + var labels = new int[batch][]; + for (int b = 0; b < batch; b++) { - for (int h = 0; h < outH; h++) + var sequence = new List(); + int previous = 0; + for (int t = 0; t < columns; t++) { - for (int w = 0; w < outW; w++) + int best = 0; + double bestValue = double.NegativeInfinity; + for (int v = 0; v < VocabularySize; v++) { - double maxVal = double.NegativeInfinity; - - for (int kh = 0; kh < kernelH; kh++) + double value = NumOps.ToDouble(target[(((b * columns) + t) * VocabularySize) + v]); + if (value > bestValue) { - for (int kw = 0; kw < kernelW; kw++) - { - int srcH = h * kernelH + kh; - int srcW = w * kernelW + kw; - - if (srcH < height && srcW < width) - { - maxVal = Math.Max(maxVal, NumOps.ToDouble(x[b, c, srcH, srcW])); - } - } + bestValue = value; + best = v; } - - result[b, c, h, w] = NumOps.FromDouble(maxVal); } - } - } - } - return result; - } + if (best != 0 && best != previous) + { + sequence.Add(best); + } - private Tensor SqueezeAndPermute(Tensor x) - { - // x: [batch, channels, height, width] - // Output: [batch, width, channels*height] - int batch = x.Shape[0]; - int channels = x.Shape[1]; - int height = x.Shape[2]; - int width = x.Shape[3]; + previous = best; + } - int featureDim = channels * height; + labels[b] = sequence.ToArray(); + } - var result = new Tensor(new[] { batch, width, featureDim }); + return labels; + } - for (int b = 0; b < batch; b++) + if (target.Rank == 2) { - for (int w = 0; w < width; w++) + int batch = target.Shape[0], length = target.Shape[1]; + var labels = new int[batch][]; + for (int b = 0; b < batch; b++) { - int idx = 0; - for (int c = 0; c < channels; c++) + var sequence = new List(); + for (int t = 0; t < length; t++) { - for (int h = 0; h < height; h++) + double id = Math.Round(NumOps.ToDouble(target[(b * length) + t])); + if (id < 0 || id >= VocabularySize) { - result[b, w, idx++] = x[b, c, h, w]; + throw new ArgumentException( + $"Label {id} at [{b}, {t}] is outside the vocabulary [0, {VocabularySize}).", nameof(target)); + } + + if (id != 0) + { + sequence.Add((int)id); } } + + labels[b] = sequence.ToArray(); } + + return labels; } - return result; + throw new ArgumentException( + $"CRNN training targets are label ids [batch, length] or scores [batch, columns, {VocabularySize}]; " + + $"got [{string.Join(", ", target.Shape.ToArray())}].", nameof(target)); } - private Tensor ApplyOutputLayer(Tensor x) + /// + /// Encodes label sequences in 's layout: + /// [batch, length0, labels0..., length1, labels1..., ...]. + /// + private Tensor EncodeCtcTargets(int[][] labels) { - int batch = x.Shape[0]; - int seqLen = x.Shape[1]; - int features = x.Shape[2]; - - var result = new Tensor(new[] { batch, seqLen, VocabularySize }); - - for (int b = 0; b < batch; b++) + var values = new List { NumOps.FromDouble(labels.Length) }; + foreach (var sequence in labels) { - for (int t = 0; t < seqLen; t++) - { - var feat = new Tensor(new[] { 1, features }); - for (int f = 0; f < features; f++) - { - feat[0, f] = x[b, t, f]; - } - - var output = _outputLayer.Forward(feat); - for (int v = 0; v < VocabularySize; v++) - { - result[b, t, v] = output[0, v]; - } - } + values.Add(NumOps.FromDouble(sequence.Length)); + values.AddRange(sequence.Select(label => NumOps.FromDouble(label))); } - return result; + return new Tensor(new[] { values.Count }, new Vector(values.ToArray())); } } diff --git a/src/ComputerVision/OCR/Recognition/TrOCR.cs b/src/ComputerVision/OCR/Recognition/TrOCR.cs index d968c4b8c0..110d585d4d 100644 --- a/src/ComputerVision/OCR/Recognition/TrOCR.cs +++ b/src/ComputerVision/OCR/Recognition/TrOCR.cs @@ -1,4 +1,5 @@ -using System.IO; +using AiDotNet.Tensors.Engines.Autodiff; +using System.IO; using AiDotNet.ComputerVision.Detection.Backbones; using AiDotNet.ComputerVision.Weights; using AiDotNet.Attributes; @@ -38,7 +39,7 @@ namespace AiDotNet.ComputerVision.OCR.Recognition; "https://arxiv.org/abs/2109.10282", Year = 2023, Authors = "Minghao Li, Tengchao Lv, Jingye Chen, Lei Cui, Yijuan Lu, Dinei Florencio, Cha Zhang, Zhoujun Li, Furu Wei")] -public class TrOCR : OCRBase +public partial class TrOCR : OCRBase { private readonly Conv2D _patchEmbed; private readonly TrOCREncoderLayer[] _encoderLayers; @@ -133,193 +134,208 @@ public override (string text, T confidence) RecognizeText(Tensor croppedImage // Decode text autoregressively var (text, confidence) = DecodeText(encoderOutput); - return (text, confidence); } + /// + /// + /// The encoder output concatenated with the logits of the first decoding step (the decoder run + /// on the start token, cross-attending to the encoder). Autoregressive decoding has no single + /// fixed-shape output, and the first step is deterministic and reaches every decoder weight - the + /// token embedding, both attentions, the FFN, the norms and the output projection - where the + /// encoder output alone (the convention of the Document/OCR TrOCR) would leave the whole decoder + /// untrained. + /// + /// + /// + /// + /// Greedy autoregressive generation - the standard TrOCR inference (Li et al. 2021; Hugging Face + /// generate): encode the image, then decode one token at a time from the start token, + /// feeding each step's most likely token back in, until every sequence has produced the end token + /// or is reached. Returns the logits of every step, + /// [batch, steps, vocabulary]; a sequence that finished early keeps receiving the end token. + /// + /// + /// Generation is not differentiated (it is no_grad in every reference implementation): + /// uses teacher forcing instead. + /// + /// + protected override Tensor ForwardLogits(Tensor image) + => Generate(EncodeImage(PreprocessCrop(image))).Logits; + private Tensor EncodeImage(Tensor image) { - // Patch embedding - var patches = _patchEmbed.Forward(image); + // Patch embedding, flattened to a token sequence, plus positional encoding. + var x = AddPositionalEncoding(CvTensorOps.FlattenSpatial(_patchEmbed.Forward(image))); + for (int l = 0; l < _numLayers; l++) + { + x = ApplyEncoderLayer(x, l); + } - // Flatten patches: [batch, channels, h, w] -> [batch, seq_len, hidden_dim] - int batch = patches.Shape[0]; - int channels = patches.Shape[1]; - int h = patches.Shape[2]; - int w = patches.Shape[3]; - int seqLen = h * w; + return x; + } - var x = new Tensor(new[] { batch, seqLen, channels }); + private (string text, T confidence) DecodeText(Tensor encoderOutput) + { + var (_, tokens, confidences) = Generate(encoderOutput); - for (int b = 0; b < batch; b++) + // Convert tokens to text (the generated tokens exclude the start and end tokens) + var textChars = new List(); + foreach (int tokenId in tokens[0]) { - int idx = 0; - for (int ph = 0; ph < h; ph++) + if (tokenId > 0 && tokenId < VocabularySize && IndexToChar.TryGetValue(tokenId, out char ch)) { - for (int pw = 0; pw < w; pw++) - { - for (int c = 0; c < channels; c++) - { - x[b, idx, c] = patches[b, c, ph, pw]; - } - idx++; - } + textChars.Add(ch); } } - // Add positional encoding - x = AddPositionalEncoding(x); - - // Apply encoder layers - for (int l = 0; l < _numLayers; l++) - { - x = ApplyEncoderLayer(x, l); - } + string text = new string(textChars.ToArray()); + double avgConf = confidences[0].Count > 0 ? confidences[0].Average() : 0; - return x; + return (text, NumOps.FromDouble(avgConf)); } - private (string text, T confidence) DecodeText(Tensor encoderOutput) + /// + /// Greedy generation with a key/value cache. + /// + /// The encoder output [batch, patches, hidden]. + /// + /// The logits of every step [batch, steps, vocabulary], and per sequence the generated + /// tokens (without the start and end tokens) and the probability of each. + /// + private (Tensor Logits, List[] Tokens, List[] Confidences) Generate(Tensor encoderOutput) { + using var noGrad = new NoGradScope(); int batch = encoderOutput.Shape[0]; - int maxLen = Options.MaxSequenceLength; - - var tokens = new List { _startTokenId }; - var confidences = new List(); + int vocab = VocabularySize + 2; + int maxSteps = Math.Max(1, Options.MaxSequenceLength - 1); - // Autoregressive decoding - for (int step = 0; step < maxLen - 1; step++) + var caches = new TrOCRLayerCache[_numLayers]; + for (int l = 0; l < _numLayers; l++) { - // Create decoder input from current tokens - var decoderInput = CreateDecoderInput(tokens); - - // Apply decoder - var decoderOutput = ApplyDecoder(decoderInput, encoderOutput); + caches[l] = new TrOCRLayerCache(); + } - // Get output for last position - int lastPos = tokens.Count - 1; - var logits = new double[VocabularySize + 2]; + var tokens = new List[batch]; + var confidences = new List[batch]; + var finished = new bool[batch]; + var current = new int[batch]; + for (int b = 0; b < batch; b++) + { + tokens[b] = new List(); + confidences[b] = new List(); + current[b] = _startTokenId; + } - for (int v = 0; v < VocabularySize + 2; v++) + var steps = new List>(); + for (int step = 0; step < maxSteps; step++) + { + var x = EmbedTokens(current.Select(t => new[] { t }).ToArray(), step); + for (int l = 0; l < _numLayers; l++) { - logits[v] = NumOps.ToDouble(decoderOutput[0, lastPos, v]); + x = _decoderLayers[l].ForwardStep(x, encoderOutput, caches[l]); } - // Apply softmax and get best token - double maxLogit = logits.Max(); - double sumExp = 0; - for (int v = 0; v < logits.Length; v++) - { - logits[v] = Math.Exp(logits[v] - maxLogit); - sumExp += logits[v]; - } + var logits = _outputProjection.ForwardTokens(x); // [batch, 1, vocab] + steps.Add(logits); - int bestToken = 0; - double bestProb = 0; - for (int v = 0; v < logits.Length; v++) + bool allFinished = true; + for (int b = 0; b < batch; b++) { - double prob = logits[v] / sumExp; - if (prob > bestProb) + if (finished[b]) { - bestProb = prob; - bestToken = v; + current[b] = _endTokenId; + continue; } - } - // Stop if end token - if (bestToken == _endTokenId) - break; + // Softmax over this step's logits; the first most likely token wins ties. + double max = double.NegativeInfinity; + for (int v = 0; v < vocab; v++) + { + max = Math.Max(max, NumOps.ToDouble(logits[(b * vocab) + v])); + } - tokens.Add(bestToken); - confidences.Add(bestProb); - } + double sum = 0; + int best = 0; + double bestValue = double.NegativeInfinity; + for (int v = 0; v < vocab; v++) + { + double value = NumOps.ToDouble(logits[(b * vocab) + v]); + sum += Math.Exp(value - max); + if (value > bestValue) + { + bestValue = value; + best = v; + } + } - // Convert tokens to text - var textChars = new List(); - for (int i = 1; i < tokens.Count; i++) // Skip start token - { - int tokenId = tokens[i]; - if (tokenId > 0 && tokenId < VocabularySize && IndexToChar.TryGetValue(tokenId, out char ch)) + if (best == _endTokenId) + { + finished[b] = true; + current[b] = _endTokenId; + continue; + } + + tokens[b].Add(best); + confidences[b].Add(Math.Exp(bestValue - max) / sum); + current[b] = best; + allFinished = false; + } + + if (allFinished) { - textChars.Add(ch); + break; } } - string text = new string(textChars.ToArray()); - double avgConf = confidences.Count > 0 ? confidences.Average() : 0; - - return (text, NumOps.FromDouble(avgConf)); + var all = steps.Count == 1 ? steps[0] : Engine.TensorConcatenate(steps.ToArray(), 1); + return (all, tokens, confidences); } - private Tensor CreateDecoderInput(List tokens) + /// + /// Embeds token ids [batch][length] as [batch, length, hidden]: the learned token + /// embedding plus the sinusoidal position encoding, positions starting at . + /// + private Tensor EmbedTokens(int[][] tokens, int startPosition) { - int seqLen = tokens.Count; + int batch = tokens.Length; + int seqLen = tokens[0].Length; int vocabSize = VocabularySize + 2; // +2 for start/end tokens - // Create one-hot representation for embedding lookup - var oneHot = new Tensor(new[] { 1, seqLen, vocabSize }); - for (int t = 0; t < seqLen; t++) - { - int tokenId = MathHelper.Clamp(tokens[t], 0, vocabSize - 1); - oneHot[0, t, tokenId] = NumOps.FromDouble(1.0); - } - - // Apply learned token embedding projection - var embedded = new Tensor(new[] { 1, seqLen, _hiddenDim }); - for (int t = 0; t < seqLen; t++) + // One-hot token ids through the learned embedding projection, then positional encoding. + var oneHot = new Tensor(new[] { batch, seqLen, vocabSize }); + for (int b = 0; b < batch; b++) { - // Extract single token one-hot vector - var tokenOneHot = new Tensor(new[] { 1, vocabSize }); - for (int v = 0; v < vocabSize; v++) - { - tokenOneHot[0, v] = oneHot[0, t, v]; - } - - // Apply embedding projection - var tokenEmb = _tokenEmbedding.Forward(tokenOneHot); - - // Copy to output - for (int h = 0; h < _hiddenDim; h++) + for (int t = 0; t < seqLen; t++) { - embedded[0, t, h] = tokenEmb[0, h]; + int tokenId = MathHelper.Clamp(tokens[b][t], 0, vocabSize - 1); + oneHot[(((b * seqLen) + t) * vocabSize) + tokenId] = NumOps.FromDouble(1.0); } } - // Add positional encoding - critical for transformer to understand token positions - // Uses sinusoidal positional encoding matching the encoder's positional encoding - var embeddedWithPos = AddPositionalEncoding(embedded); - - return embeddedWithPos; + return AddPositionalEncoding(_tokenEmbedding.ForwardTokens(oneHot), startPosition); } - private Tensor AddPositionalEncoding(Tensor x) + private Tensor AddPositionalEncoding(Tensor x, int startPosition = 0) { int batch = x.Shape[0]; int seqLen = x.Shape[1]; int hiddenDim = x.Shape[2]; - var result = new Tensor(x._shape); - - for (int b = 0; b < batch; b++) + // The sinusoidal table is a constant; only the ADD must stay on the tape. + var table = new Tensor(new[] { 1, seqLen, hiddenDim }); + for (int pos = 0; pos < seqLen; pos++) { - for (int pos = 0; pos < seqLen; pos++) + for (int i = 0; i < hiddenDim; i++) { - for (int i = 0; i < hiddenDim; i++) - { - // Use explicit floor division to get the pair index (0,1->0, 2,3->1, etc.) - int pairIndex = i / 2; - double exponent = (2.0 * pairIndex) / hiddenDim; - double angle = pos / Math.Pow(10000.0, exponent); - double pe = (i % 2 == 0) ? Math.Sin(angle) : Math.Cos(angle); - - result[b, pos, i] = NumOps.FromDouble( - NumOps.ToDouble(x[b, pos, i]) + pe - ); - } + int pairIndex = i / 2; + double exponent = (2.0 * pairIndex) / hiddenDim; + double angle = (pos + startPosition) / Math.Pow(10000.0, exponent); + table[(pos * hiddenDim) + i] = NumOps.FromDouble((i % 2 == 0) ? Math.Sin(angle) : Math.Cos(angle)); } } - return result; + return Engine.TensorAdd(x, Engine.TensorBroadcastTo(table, new[] { batch, seqLen, hiddenDim })); } private Tensor ApplyEncoderLayer(Tensor x, int layerIdx) @@ -330,45 +346,13 @@ private Tensor ApplyEncoderLayer(Tensor x, int layerIdx) private Tensor ApplyDecoder(Tensor decoderInput, Tensor encoderOutput) { - int batch = decoderInput.Shape[0]; - int seqLen = decoderInput.Shape[1]; - var x = decoderInput; - - // Apply proper transformer decoder layers with self-attention and cross-attention for (int l = 0; l < _numLayers; l++) { x = _decoderLayers[l].Forward(x, encoderOutput); } - // Output projection - var logits = new Tensor(new[] { batch, seqLen, VocabularySize + 2 }); - - for (int b = 0; b < batch; b++) - { - for (int t = 0; t < seqLen; t++) - { - var feat = new Tensor(new[] { 1, _hiddenDim }); - for (int h = 0; h < _hiddenDim; h++) - { - feat[0, h] = x[b, t, h]; - } - - var output = _outputProjection.Forward(feat); - for (int v = 0; v < VocabularySize + 2; v++) - { - logits[b, t, v] = output[0, v]; - } - } - } - - return logits; - } - - private static double GELU(double x) - { - double c = Math.Sqrt(2.0 / Math.PI); - return 0.5 * x * (1.0 + Math.Tanh(c * (x + 0.044715 * x * x * x))); + return _outputProjection.ForwardTokens(x); } /// @@ -619,13 +603,171 @@ private void LoadWeightsFromFile(string path) _outputProjection.ReadParameters(reader); } + + /// + /// Runs one teacher-forced training step with cross-entropy. + /// + /// The text-line image. + /// + /// The target text as token ids [batch, length], or as scores [batch, length, vocabulary] + /// (such as 's output shape), whose most likely token at each position is the + /// label. Positions after the first end token are padding and are ignored. + /// + /// + /// The standard TrOCR recipe (Li et al. 2021; Hugging Face VisionEncoderDecoderModel): the + /// decoder reads the labels shifted right behind the start token in ONE parallel causal pass, and + /// the loss is the cross-entropy of each position's prediction of the next label. Every decoder + /// weight is on the gradient path - including the self-attention query and key projections, which + /// a single-step decode could never train (one key makes the attention weight exactly 1). + /// + public override void Train(Tensor input, Tensor expectedOutput) + { + if (input is null) + { + throw new ArgumentNullException(nameof(input)); + } + + if (expectedOutput is null) + { + throw new ArgumentNullException(nameof(expectedOutput)); + } + + var labels = LabelsFrom(expectedOutput); + var targets = LabelTargets(labels); + RecordTrainingLoss(TensorModelTrainer.Step( + this, input, targets, NumOps.FromDouble(TrainingLearningRate), + image => TeacherForcedLogits(image, labels), + CrossEntropy)); + } + + /// + /// Decoder logits [batch, length, vocabulary] under teacher forcing: position t reads the + /// start token followed by labels 0..t-1. + /// + private Tensor TeacherForcedLogits(Tensor image, int[][] labels) + { + var encoderOutput = EncodeImage(PreprocessCrop(image)); + if (labels.Length != encoderOutput.Shape[0]) + { + throw new ArgumentException( + $"The target has {labels.Length} sequences but the input has {encoderOutput.Shape[0]} images."); + } + + var shifted = labels.Select(row => new[] { _startTokenId }.Concat(row.Take(row.Length - 1)).ToArray()).ToArray(); + return ApplyDecoder(EmbedTokens(shifted, 0), encoderOutput); + } + + /// + /// Reads label ids from token ids [batch, length] or scores [batch, length, vocabulary]. + /// + private int[][] LabelsFrom(Tensor target) + { + int vocab = VocabularySize + 2; + if (target.Rank == 3 && target.Shape[2] == vocab) + { + int batch = target.Shape[0], length = target.Shape[1]; + var labels = new int[batch][]; + for (int b = 0; b < batch; b++) + { + labels[b] = new int[length]; + for (int t = 0; t < length; t++) + { + int best = 0; + double bestValue = double.NegativeInfinity; + for (int v = 0; v < vocab; v++) + { + double value = NumOps.ToDouble(target[(((b * length) + t) * vocab) + v]); + if (value > bestValue) + { + bestValue = value; + best = v; + } + } + + labels[b][t] = best; + } + } + + return labels; + } + + if (target.Rank == 2) + { + int batch = target.Shape[0], length = target.Shape[1]; + var labels = new int[batch][]; + for (int b = 0; b < batch; b++) + { + labels[b] = new int[length]; + for (int t = 0; t < length; t++) + { + double id = Math.Round(NumOps.ToDouble(target[(b * length) + t])); + if (id < 0 || id >= vocab) + { + throw new ArgumentException( + $"Label {id} at [{b}, {t}] is outside the vocabulary [0, {vocab}).", nameof(target)); + } + + labels[b][t] = (int)id; + } + } + + return labels; + } + + throw new ArgumentException( + $"TrOCR training targets are token ids [batch, length] or scores [batch, length, {vocab}]; " + + $"got [{string.Join(", ", target.Shape.ToArray())}].", nameof(target)); + } + + /// + /// One-hot targets [batch, length, vocabulary]; positions after the first end token are + /// all zero, so they drop out of the loss. + /// + private Tensor LabelTargets(int[][] labels) + { + int vocab = VocabularySize + 2; + int batch = labels.Length, length = labels[0].Length; + var targets = new Tensor(new[] { batch, length, vocab }); + for (int b = 0; b < batch; b++) + { + for (int t = 0; t < length; t++) + { + targets[(((b * length) + t) * vocab) + labels[b][t]] = NumOps.One; + if (labels[b][t] == _endTokenId) + { + break; + } + } + } + + return targets; + } + + /// + /// Mean cross-entropy over the labelled positions: -sum(target * log_softmax(logits)) / count. + /// + private static Tensor CrossEntropy(Tensor logits, Tensor oneHotTargets) + { + var engine = AiDotNetEngine.Current; + var ops = MathHelper.GetNumericOperations(); + + double labelled = 0; + for (int i = 0; i < oneHotTargets.Length; i++) + { + labelled += ops.ToDouble(oneHotTargets[i]); + } + + var logProbabilities = engine.TensorLogSoftmax(logits, axis: -1); + var picked = engine.ReduceSum(engine.TensorMultiply(logProbabilities, oneHotTargets), null); + return engine.TensorMultiplyScalar(picked, ops.FromDouble(-1.0 / Math.Max(1.0, labelled))); + } } /// /// Transformer encoder layer with proper multi-head self-attention for TrOCR. /// /// The numeric type used for calculations. -internal class TrOCREncoderLayer +internal class TrOCREncoderLayer : CvParameterModule { private readonly INumericOperations _numOps; private readonly int _hiddenDim; @@ -680,32 +822,29 @@ public TrOCREncoderLayer(int hiddenDim, int numHeads) public Tensor Forward(Tensor x) { - int batch = x.Shape[0]; - int seqLen = x.Shape[1]; - // Self-attention with proper scaled dot-product attention - var attnOut = ApplySelfAttention(x, batch, seqLen); + var attnOut = ApplySelfAttention(x); // Add residual & LayerNorm with learnable parameters - var residual1 = AddTensors(x, attnOut, batch, seqLen); + var residual1 = AddTensors(x, attnOut); var x1 = _norm1.Forward(residual1); // FFN - var ffnOut = ApplyFFN(x1, batch, seqLen); + var ffnOut = ApplyFFN(x1); // Add residual & LayerNorm with learnable parameters - var residual2 = AddTensors(x1, ffnOut, batch, seqLen); + var residual2 = AddTensors(x1, ffnOut); var output = _norm2.Forward(residual2); return output; } - private Tensor AddTensors(Tensor a, Tensor b, int batch, int seqLen) + private Tensor AddTensors(Tensor a, Tensor b) { return AiDotNetEngine.Current.TensorAdd(a, b); } - private Tensor ApplySelfAttention(Tensor x, int batch, int seqLen) + private Tensor ApplySelfAttention(Tensor x) { // Project Q, K, V var q = ProjectSequence(x, _queryProj); @@ -713,150 +852,21 @@ private Tensor ApplySelfAttention(Tensor x, int batch, int seqLen) var v = ProjectSequence(x, _valueProj); // Compute multi-head attention - var attnOutput = ComputeMultiHeadAttention(q, k, v, batch, seqLen, seqLen); + var attnOutput = ComputeMultiHeadAttention(q, k, v); // Output projection return ProjectSequence(attnOutput, _outputProj); } - private Tensor ComputeMultiHeadAttention(Tensor q, Tensor k, Tensor v, - int batch, int queryLen, int keyLen) - { - var output = new Tensor(new[] { batch, queryLen, _hiddenDim }); + private Tensor ComputeMultiHeadAttention(Tensor q, Tensor k, Tensor v) + => CvTensorOps.MultiHeadAttention(q, k, v, _numHeads, _scale); - for (int b = 0; b < batch; b++) - { - for (int h = 0; h < _numHeads; h++) - { - int headOffset = h * _headDim; + private Tensor ProjectSequence(Tensor x, Dense proj) => proj.ForwardTokens(x); - // Compute attention scores: Q * K^T / sqrt(d_k) - var scores = new double[queryLen, keyLen]; - for (int i = 0; i < queryLen; i++) - { - for (int j = 0; j < keyLen; j++) - { - double score = 0; - for (int d = 0; d < _headDim; d++) - { - score += _numOps.ToDouble(q[b, i, headOffset + d]) * - _numOps.ToDouble(k[b, j, headOffset + d]); - } - scores[i, j] = score * _scale; - } - } + private Tensor ApplyFFN(Tensor x) + => _ffn2.ForwardTokens(AiDotNetEngine.Current.GELU(_ffn1.ForwardTokens(x))); - // Softmax over key dimension - for (int i = 0; i < queryLen; i++) - { - double maxScore = double.NegativeInfinity; - for (int j = 0; j < keyLen; j++) - { - maxScore = Math.Max(maxScore, scores[i, j]); - } - double sumExp = 0; - for (int j = 0; j < keyLen; j++) - { - scores[i, j] = Math.Exp(scores[i, j] - maxScore); - sumExp += scores[i, j]; - } - - for (int j = 0; j < keyLen; j++) - { - scores[i, j] /= sumExp; - } - } - - // Apply attention weights to values - for (int i = 0; i < queryLen; i++) - { - for (int d = 0; d < _headDim; d++) - { - double value = 0; - for (int j = 0; j < keyLen; j++) - { - value += scores[i, j] * _numOps.ToDouble(v[b, j, headOffset + d]); - } - output[b, i, headOffset + d] = _numOps.FromDouble(value); - } - } - } - } - - return output; - } - - private Tensor ProjectSequence(Tensor x, Dense proj) - { - int batch = x.Shape[0]; - int seqLen = x.Shape[1]; - int dim = x.Shape[2]; - int outDim = proj.OutputSize; - - var result = new Tensor(new[] { batch, seqLen, outDim }); - - for (int b = 0; b < batch; b++) - { - for (int s = 0; s < seqLen; s++) - { - var feat = new Tensor(new[] { 1, dim }); - for (int d = 0; d < dim; d++) - { - feat[0, d] = x[b, s, d]; - } - - var projected = proj.Forward(feat); - for (int d = 0; d < outDim; d++) - { - result[b, s, d] = projected[0, d]; - } - } - } - - return result; - } - - private Tensor ApplyFFN(Tensor x, int batch, int seqLen) - { - int ffnDim = _ffn1.OutputSize; - var result = new Tensor(x._shape); - - for (int b = 0; b < batch; b++) - { - for (int s = 0; s < seqLen; s++) - { - var feat = new Tensor(new[] { 1, _hiddenDim }); - for (int d = 0; d < _hiddenDim; d++) - { - feat[0, d] = x[b, s, d]; - } - - // FFN1 with GELU - var h = _ffn1.Forward(feat); - for (int d = 0; d < ffnDim; d++) - { - double val = _numOps.ToDouble(h[0, d]); - h[0, d] = _numOps.FromDouble(GELU(val)); - } - - // FFN2 - var output = _ffn2.Forward(h); - for (int d = 0; d < _hiddenDim; d++) - { - result[b, s, d] = output[0, d]; - } - } - } - - return result; - } - - private static double GELU(double x) - { - double c = Math.Sqrt(2.0 / Math.PI); - return 0.5 * x * (1.0 + Math.Tanh(c * (x + 0.044715 * x * x * x))); - } public long GetParameterCount() { @@ -903,13 +913,41 @@ public void ReadParameters(BinaryReader reader) _norm1.ReadParameters(reader); _norm2.ReadParameters(reader); } + + /// + protected override IEnumerable?> ParameterChildren() + { + yield return _queryProj; + yield return _keyProj; + yield return _valueProj; + yield return _outputProj; + yield return _ffn1; + yield return _ffn2; + yield return _norm1; + yield return _norm2; + } +} + +/// +/// Incremental-decoding state of one : the self-attention keys and +/// values of every token decoded so far, and the cross-attention keys and values of the encoder output. +/// +internal sealed class TrOCRLayerCache +{ + public Tensor? SelfKeys { get; set; } + + public Tensor? SelfValues { get; set; } + + public Tensor? CrossKeys { get; set; } + + public Tensor? CrossValues { get; set; } } /// /// Transformer decoder layer with proper multi-head self-attention and cross-attention for TrOCR. /// /// The numeric type used for calculations. -internal class TrOCRDecoderLayer +internal class TrOCRDecoderLayer : CvParameterModule { private readonly INumericOperations _numOps; private readonly int _hiddenDim; @@ -991,34 +1029,30 @@ public TrOCRDecoderLayer(int hiddenDim, int numHeads) public Tensor Forward(Tensor x, Tensor encoderOutput) { - int batch = x.Shape[0]; - int seqLen = x.Shape[1]; - int encoderLen = encoderOutput.Shape[1]; - // Masked self-attention (causal mask for autoregressive decoding) - var selfAttnOut = ApplyCausalSelfAttention(x, batch, seqLen); - var residual1 = AddTensors(x, selfAttnOut, batch, seqLen); + var selfAttnOut = ApplyCausalSelfAttention(x); + var residual1 = AddTensors(x, selfAttnOut); var x1 = _norm1.Forward(residual1); // Cross-attention to encoder output - var crossAttnOut = ApplyCrossAttention(x1, encoderOutput, batch, seqLen, encoderLen); - var residual2 = AddTensors(x1, crossAttnOut, batch, seqLen); + var crossAttnOut = ApplyCrossAttention(x1, encoderOutput); + var residual2 = AddTensors(x1, crossAttnOut); var x2 = _norm2.Forward(residual2); // FFN - var ffnOut = ApplyFFN(x2, batch, seqLen); - var residual3 = AddTensors(x2, ffnOut, batch, seqLen); + var ffnOut = ApplyFFN(x2); + var residual3 = AddTensors(x2, ffnOut); var output = _norm3.Forward(residual3); return output; } - private Tensor AddTensors(Tensor a, Tensor b, int batch, int seqLen) + private Tensor AddTensors(Tensor a, Tensor b) { return AiDotNetEngine.Current.TensorAdd(a, b); } - private Tensor ApplyCausalSelfAttention(Tensor x, int batch, int seqLen) + private Tensor ApplyCausalSelfAttention(Tensor x) { // Project Q, K, V var q = ProjectSequence(x, _selfQueryProj); @@ -1026,95 +1060,16 @@ private Tensor ApplyCausalSelfAttention(Tensor x, int batch, int seqLen) var v = ProjectSequence(x, _selfValueProj); // Compute masked attention (causal mask) - var attnOutput = ComputeCausalAttention(q, k, v, batch, seqLen); + var attnOutput = ComputeCausalAttention(q, k, v); // Output projection return ProjectSequence(attnOutput, _selfOutputProj); } - private Tensor ComputeCausalAttention(Tensor q, Tensor k, Tensor v, int batch, int seqLen) - { - var output = new Tensor(new[] { batch, seqLen, _hiddenDim }); - - for (int b = 0; b < batch; b++) - { - for (int h = 0; h < _numHeads; h++) - { - int headOffset = h * _headDim; - - // Compute attention scores with causal mask - var scores = new double[seqLen, seqLen]; - for (int i = 0; i < seqLen; i++) - { - for (int j = 0; j < seqLen; j++) - { - if (j > i) - { - // Future tokens are masked (set to -inf before softmax) - scores[i, j] = double.NegativeInfinity; - } - else - { - double score = 0; - for (int d = 0; d < _headDim; d++) - { - score += _numOps.ToDouble(q[b, i, headOffset + d]) * - _numOps.ToDouble(k[b, j, headOffset + d]); - } - scores[i, j] = score * _scale; - } - } - } - - // Softmax - for (int i = 0; i < seqLen; i++) - { - double maxScore = double.NegativeInfinity; - for (int j = 0; j <= i; j++) // Only look at non-masked positions - { - maxScore = Math.Max(maxScore, scores[i, j]); - } - - double sumExp = 0; - for (int j = 0; j < seqLen; j++) - { - if (j <= i) - { - scores[i, j] = Math.Exp(scores[i, j] - maxScore); - sumExp += scores[i, j]; - } - else - { - scores[i, j] = 0; // Masked out - } - } + private Tensor ComputeCausalAttention(Tensor q, Tensor k, Tensor v) + => CvTensorOps.MultiHeadAttention(q, k, v, _numHeads, _scale, causal: true); - for (int j = 0; j <= i; j++) - { - scores[i, j] /= sumExp; - } - } - - // Apply attention to values - for (int i = 0; i < seqLen; i++) - { - for (int d = 0; d < _headDim; d++) - { - double value = 0; - for (int j = 0; j <= i; j++) - { - value += scores[i, j] * _numOps.ToDouble(v[b, j, headOffset + d]); - } - output[b, i, headOffset + d] = _numOps.FromDouble(value); - } - } - } - } - - return output; - } - - private Tensor ApplyCrossAttention(Tensor x, Tensor encoderOutput, int batch, int seqLen, int encoderLen) + private Tensor ApplyCrossAttention(Tensor x, Tensor encoderOutput) { // Query from decoder, Key/Value from encoder var q = ProjectSequence(x, _crossQueryProj); @@ -1122,150 +1077,21 @@ private Tensor ApplyCrossAttention(Tensor x, Tensor encoderOutput, int var v = ProjectSequence(encoderOutput, _crossValueProj); // Compute cross-attention (no mask needed) - var attnOutput = ComputeCrossAttention(q, k, v, batch, seqLen, encoderLen); + var attnOutput = ComputeCrossAttention(q, k, v); // Output projection return ProjectSequence(attnOutput, _crossOutputProj); } - private Tensor ComputeCrossAttention(Tensor q, Tensor k, Tensor v, - int batch, int queryLen, int keyLen) - { - var output = new Tensor(new[] { batch, queryLen, _hiddenDim }); - - for (int b = 0; b < batch; b++) - { - for (int h = 0; h < _numHeads; h++) - { - int headOffset = h * _headDim; - - // Compute attention scores - var scores = new double[queryLen, keyLen]; - for (int i = 0; i < queryLen; i++) - { - for (int j = 0; j < keyLen; j++) - { - double score = 0; - for (int d = 0; d < _headDim; d++) - { - score += _numOps.ToDouble(q[b, i, headOffset + d]) * - _numOps.ToDouble(k[b, j, headOffset + d]); - } - scores[i, j] = score * _scale; - } - } - - // Softmax - for (int i = 0; i < queryLen; i++) - { - double maxScore = double.NegativeInfinity; - for (int j = 0; j < keyLen; j++) - { - maxScore = Math.Max(maxScore, scores[i, j]); - } - - double sumExp = 0; - for (int j = 0; j < keyLen; j++) - { - scores[i, j] = Math.Exp(scores[i, j] - maxScore); - sumExp += scores[i, j]; - } - - for (int j = 0; j < keyLen; j++) - { - scores[i, j] /= sumExp; - } - } - - // Apply attention to values - for (int i = 0; i < queryLen; i++) - { - for (int d = 0; d < _headDim; d++) - { - double value = 0; - for (int j = 0; j < keyLen; j++) - { - value += scores[i, j] * _numOps.ToDouble(v[b, j, headOffset + d]); - } - output[b, i, headOffset + d] = _numOps.FromDouble(value); - } - } - } - } - - return output; - } - - private Tensor ProjectSequence(Tensor x, Dense proj) - { - int batch = x.Shape[0]; - int seqLen = x.Shape[1]; - int dim = x.Shape[2]; - int outDim = proj.OutputSize; - - var result = new Tensor(new[] { batch, seqLen, outDim }); - - for (int b = 0; b < batch; b++) - { - for (int s = 0; s < seqLen; s++) - { - var feat = new Tensor(new[] { 1, dim }); - for (int d = 0; d < dim; d++) - { - feat[0, d] = x[b, s, d]; - } - - var projected = proj.Forward(feat); - for (int d = 0; d < outDim; d++) - { - result[b, s, d] = projected[0, d]; - } - } - } - - return result; - } - - private Tensor ApplyFFN(Tensor x, int batch, int seqLen) - { - int ffnDim = _ffn1.OutputSize; - var result = new Tensor(x._shape); - - for (int b = 0; b < batch; b++) - { - for (int s = 0; s < seqLen; s++) - { - var feat = new Tensor(new[] { 1, _hiddenDim }); - for (int d = 0; d < _hiddenDim; d++) - { - feat[0, d] = x[b, s, d]; - } + private Tensor ComputeCrossAttention(Tensor q, Tensor k, Tensor v) + => CvTensorOps.MultiHeadAttention(q, k, v, _numHeads, _scale); - // FFN1 with GELU - var h = _ffn1.Forward(feat); - for (int d = 0; d < ffnDim; d++) - { - double val = _numOps.ToDouble(h[0, d]); - h[0, d] = _numOps.FromDouble(GELU(val)); - } + private Tensor ProjectSequence(Tensor x, Dense proj) => proj.ForwardTokens(x); - // FFN2 - var output = _ffn2.Forward(h); - for (int d = 0; d < _hiddenDim; d++) - { - result[b, s, d] = output[0, d]; - } - } - } + private Tensor ApplyFFN(Tensor x) + => _ffn2.ForwardTokens(AiDotNetEngine.Current.GELU(_ffn1.ForwardTokens(x))); - return result; - } - private static double GELU(double x) - { - double c = Math.Sqrt(2.0 / Math.PI); - return 0.5 * x * (1.0 + Math.Tanh(c * (x + 0.044715 * x * x * x))); - } public long GetParameterCount() { @@ -1327,13 +1153,70 @@ public void ReadParameters(BinaryReader reader) _norm2.ReadParameters(reader); _norm3.ReadParameters(reader); } + + /// + protected override IEnumerable?> ParameterChildren() + { + yield return _selfQueryProj; + yield return _selfKeyProj; + yield return _selfValueProj; + yield return _selfOutputProj; + yield return _crossQueryProj; + yield return _crossKeyProj; + yield return _crossValueProj; + yield return _crossOutputProj; + yield return _ffn1; + yield return _ffn2; + yield return _norm1; + yield return _norm2; + yield return _norm3; + } + + /// + /// Runs this layer for ONE new decoder token, reusing the keys and values of every earlier token. + /// + /// The new token's hidden state [batch, 1, hidden]. + /// The encoder output [batch, patches, hidden]. + /// This layer's cache; the new token's keys and values are appended to it. + /// The new token's output [batch, 1, hidden]. + /// + /// Incremental decoding with a key/value cache - the standard generate path (Hugging Face + /// use_cache). Because self-attention is causal, the output for the newest token equals + /// the last position of over the whole prefix; the cache just avoids + /// recomputing the earlier positions, making each step O(prefix) instead of O(prefix squared). + /// The encoder's cross-attention keys and values are computed once, on the first step. + /// + public Tensor ForwardStep(Tensor x, Tensor encoderOutput, TrOCRLayerCache cache) + { + var engine = AiDotNetEngine.Current; + + var q = ProjectSequence(x, _selfQueryProj); + var k = ProjectSequence(x, _selfKeyProj); + var v = ProjectSequence(x, _selfValueProj); + cache.SelfKeys = cache.SelfKeys is null ? k : engine.TensorConcatenate(new[] { cache.SelfKeys, k }, 1); + cache.SelfValues = cache.SelfValues is null ? v : engine.TensorConcatenate(new[] { cache.SelfValues, v }, 1); + + // The newest token may attend to every cached token, so no mask is needed. + var selfAttn = ProjectSequence( + CvTensorOps.MultiHeadAttention(q, cache.SelfKeys, cache.SelfValues, _numHeads, _scale), _selfOutputProj); + var x1 = _norm1.Forward(engine.TensorAdd(x, selfAttn)); + + cache.CrossKeys ??= ProjectSequence(encoderOutput, _crossKeyProj); + cache.CrossValues ??= ProjectSequence(encoderOutput, _crossValueProj); + var crossAttn = ProjectSequence( + CvTensorOps.MultiHeadAttention(ProjectSequence(x1, _crossQueryProj), cache.CrossKeys, cache.CrossValues, _numHeads, _scale), + _crossOutputProj); + var x2 = _norm2.Forward(engine.TensorAdd(x1, crossAttn)); + + return _norm3.Forward(engine.TensorAdd(x2, ApplyFFN(x2))); + } } /// /// Layer normalization with learnable affine parameters for TrOCR. /// /// The numeric type used for calculations. -internal class TrOCRLayerNorm +internal class TrOCRLayerNorm : CvParameterModule { private readonly INumericOperations _numOps; private readonly int _hiddenDim; @@ -1368,49 +1251,7 @@ public TrOCRLayerNorm(int hiddenDim, double eps = 1e-6) } } - public Tensor Forward(Tensor x) - { - int batch = x.Shape[0]; - int seqLen = x.Shape[1]; - int hiddenDim = x.Shape[2]; - - var result = new Tensor(x._shape); - - for (int b = 0; b < batch; b++) - { - for (int s = 0; s < seqLen; s++) - { - // Compute mean - double mean = 0; - for (int d = 0; d < hiddenDim; d++) - { - mean += _numOps.ToDouble(x[b, s, d]); - } - mean /= hiddenDim; - - // Compute variance - double variance = 0; - for (int d = 0; d < hiddenDim; d++) - { - double diff = _numOps.ToDouble(x[b, s, d]) - mean; - variance += diff * diff; - } - variance /= hiddenDim; - - // Normalize and apply affine transformation - double std = Math.Sqrt(variance + _eps); - for (int d = 0; d < hiddenDim; d++) - { - double normalized = (_numOps.ToDouble(x[b, s, d]) - mean) / std; - double gamma = _numOps.ToDouble(_gamma[d]); - double beta = _numOps.ToDouble(_beta[d]); - result[b, s, d] = _numOps.FromDouble(gamma * normalized + beta); - } - } - } - - return result; - } + public Tensor Forward(Tensor x) => CvTensorOps.LayerNormLastAxis(x, _gamma, _beta, _eps); public long GetParameterCount() { @@ -1446,4 +1287,14 @@ public void ReadParameters(BinaryReader reader) _beta[i] = _numOps.FromDouble(reader.ReadDouble()); } } + + /// + protected override IEnumerable?> ParameterChildren() => Array.Empty?>(); + + /// + protected override IEnumerable> OwnParameterTensors() + { + yield return _gamma; + yield return _beta; + } } diff --git a/src/ComputerVision/Segmentation/InstanceSegmentation/MaskRCNN.cs b/src/ComputerVision/Segmentation/InstanceSegmentation/MaskRCNN.cs index 2dd1e61854..8e6262e485 100644 --- a/src/ComputerVision/Segmentation/InstanceSegmentation/MaskRCNN.cs +++ b/src/ComputerVision/Segmentation/InstanceSegmentation/MaskRCNN.cs @@ -1,3 +1,4 @@ +using AiDotNet.Tensors.Engines; using System.IO; using AiDotNet.Attributes; using AiDotNet.Augmentation.Image; @@ -360,18 +361,16 @@ private Tensor Flatten(Tensor input) return output; } - private Tensor ApplyReLU(Tensor input) - { - var output = new Tensor(input._shape); - - for (int i = 0; i < input.Length; i++) - { - double val = NumOps.ToDouble(input[i]); - output[i] = NumOps.FromDouble(Math.Max(0, val)); - } - - return output; - } + /// + /// Elementwise ReLU, delegated to the engine. + /// + /// + /// This was a scalar loop that read each element out to double and wrote a fresh + /// tensor. Arithmetically identical, but it severed the autodiff tape: the gradient chain + /// stopped here, so every trainable layer UPSTREAM of this call received no gradient and + /// silently never trained. The engine op records itself on the tape. + /// + private Tensor ApplyReLU(Tensor input) => AiDotNetEngine.Current.ReLU(input); private (int classId, T confidence) GetPrediction(Tensor logits) { diff --git a/src/ComputerVision/TensorModelTrainer.cs b/src/ComputerVision/TensorModelTrainer.cs new file mode 100644 index 0000000000..afb6e5b581 --- /dev/null +++ b/src/ComputerVision/TensorModelTrainer.cs @@ -0,0 +1,208 @@ +using System.Runtime.CompilerServices; +using AiDotNet.Models; +using AiDotNet.Models.Parameters; +using AiDotNet.Tensors.Engines.Autodiff; + +namespace AiDotNet.ComputerVision; + +/// +/// Tape-based training for the computer-vision models that are built on +/// ModelBase<T, Tensor<T>, Tensor<T>> rather than NeuralNetworkBase - +/// the object detectors, text detectors and OCR recognizers. +/// +/// +/// +/// The weights a step updates come from the model's parameter registry: every LIVE chunk with the +/// role, i.e. the exact tensor instances the forward pass +/// reads. That makes the registry the single source of truth for training, GetParameters(), +/// serialization and cloning. A weight the registry cannot see is therefore not silently skipped by +/// one surface and handled by another; it is missing from all of them, which is what the family +/// conformance audit checks for. +/// +/// +/// A chunk that is only a COPY of a weight (not writable in place) is excluded: the autodiff tape +/// keys gradients by tensor reference, so updating a copy would change nothing the model uses. +/// +/// +/// The numeric type the model is expressed in. +internal static class TensorModelTrainer +{ + /// + /// Models whose lazy layers have already resolved their shapes, so the warm-up forward is paid + /// once per model rather than on every training step. + /// + private static readonly ConditionalWeakTable Warmed = new(); + + private sealed class WarmupState + { + public bool Complete; + } + + /// + /// Runs one tape-based training step: forward under a gradient tape, the loss (mean squared + /// error unless the model supplies its own), then a stochastic-gradient update of every live + /// trainable tensor. + /// + /// The model being trained. + /// The training input. + /// The desired output, shaped like the model prediction. + /// Step size for the parameter update. + /// + /// The model's differentiable forward pass. It must be built from engine operations so the tape + /// records it - a forward that drops to scalar loops severs the chain and the parameters upstream + /// of the break receive no gradient. + /// + /// + /// The training objective as loss(predicted, target), a one-element tensor built from engine + /// operations. Null means mean squared error. A model whose paper trains it with another objective + /// (TrOCR: cross-entropy under teacher forcing) passes it here. + /// + /// The loss value for this step, measured before the update. + public static T Step( + ModelBase, Tensor> model, + Tensor input, + Tensor target, + T learningRate, + Func, Tensor> forward, + Func, Tensor, Tensor>? loss = null) + => StepWithTargets(model, input, target, learningRate, forward, loss ?? MeanSquaredError); + + /// + /// Runs the same single update for structured heads and typed task targets, without flattening + /// away their meaning. Forward outputs and the loss are consumed inside the tape/arena lifetime. + /// + public static T StepWithTargets( + ModelBase, Tensor> model, + Tensor input, + TTarget target, + T learningRate, + Func, TPrediction> forward, + Func> loss) + { + var numOps = MathHelper.GetNumericOperations(); + + // Resolve lazy layer shapes BEFORE reading the registry. The convolutions behind the Conv2D + // adapter infer their input depth on first Forward and own no parameters until then, so the + // registry would report none and the step would silently do nothing. No tape is active here, + // so this records nothing. + var warmup = Warmed.GetValue(model, static _ => new WarmupState()); + if (!System.Threading.Volatile.Read(ref warmup.Complete)) + { + // GetValue may invoke competing factories; its returned state is the one shared by + // every caller. Serialize initialization, not the whole training step. A failed forward + // leaves Complete false so a later call retries instead of caching the exception. + lock (warmup) + { + if (!warmup.Complete) + { + forward(input); + System.Threading.Volatile.Write(ref warmup.Complete, true); + } + } + } + + var parameters = LiveTrainableTensors(model); + if (parameters.Length == 0) + { + throw new InvalidOperationException( + $"No live trainable tensors were discovered for model '{model.GetType().FullName}'."); + } + + var engine = AiDotNetEngine.Current; + using (var tape = new GradientTape()) + { + var predicted = forward(input); + var objective = loss(predicted, target); + var gradients = tape.ComputeGradients(objective, parameters); + + // The update runs INSIDE the tape's scope. Disposing the outermost tape rewinds the + // active TensorArena (the per-step recycling of AiDotNet #1804), and the gradients and + // the loss live in that arena: consumed after the dispose, their storage is already + // being reissued to the update's own temporaries. Every model trained inside an arena + // then applied a mix of its gradients and unrelated scratch - and threw only when a + // reissued buffer happened to have a different shape (a [256, 1024] weight receiving + // [1024, 256]). The no-grad scope keeps the update itself off the tape. + using (new NoGradScope()) + { + foreach (var parameter in parameters) + { + if (gradients.TryGetValue(parameter, out var gradient)) + { + if (!SameShape(parameter, gradient)) + { + throw new InvalidOperationException( + $"{model.GetType().Name}: the gradient for a trainable tensor of shape " + + $"[{string.Join(", ", parameter._shape)}] has shape [{string.Join(", ", gradient._shape)}]. " + + "The forward pass must use this tensor exactly as registered - a reshaped copy or " + + "a view created outside the engine records the wrong tensor on the tape."); + } + + engine.TensorSubtractInPlace(parameter, engine.TensorMultiplyScalar(gradient, learningRate)); + } + } + + return objective.Length > 0 ? objective[0] : numOps.Zero; + } + } + } + + private static bool SameShape(Tensor a, Tensor b) + { + if (a._shape.Length != b._shape.Length) + { + return false; + } + + for (int i = 0; i < a._shape.Length; i++) + { + if (a._shape[i] != b._shape[i]) + { + return false; + } + } + + return true; + } + + /// + /// The distinct live tensors the registry marks trainable, in registry order. + /// + public static Tensor[] LiveTrainableTensors(ModelBase, Tensor> model) + { + var seen = new HashSet>(TensorReferenceComparer.Instance); + var result = new List>(); + foreach (var chunk in model.GetParameterStateChunks()) + { + if (chunk.Role == ParameterSlotRole.Trainable && chunk.IsWritableInPlace && seen.Add(chunk.Tensor)) + { + result.Add(chunk.Tensor); + } + } + + return result.ToArray(); + } + + /// + /// Mean squared error built from engine operations so the gradient tape can differentiate it. + /// + internal static Tensor MeanSquaredError(Tensor predicted, Tensor target) + { + var engine = AiDotNetEngine.Current; + var numOps = MathHelper.GetNumericOperations(); + + var difference = engine.TensorSubtract(predicted, target); + var squared = engine.TensorMultiply(difference, difference); + return engine.TensorMultiplyScalar( + engine.ReduceSum(squared, null), + numOps.FromDouble(1.0 / Math.Max(1, squared.Length))); + } + + private sealed class TensorReferenceComparer : IEqualityComparer> + { + public static readonly TensorReferenceComparer Instance = new(); + + public bool Equals(Tensor? x, Tensor? y) => ReferenceEquals(x, y); + + public int GetHashCode(Tensor obj) => RuntimeHelpers.GetHashCode(obj); + } +} diff --git a/src/Enums/SetPredictionClassificationLoss.cs b/src/Enums/SetPredictionClassificationLoss.cs new file mode 100644 index 0000000000..c548043db1 --- /dev/null +++ b/src/Enums/SetPredictionClassificationLoss.cs @@ -0,0 +1,28 @@ +namespace AiDotNet.Enums; + +/// The classification objective used by a DETR-family set prediction loss. +/// +/// For Beginners: A DETR-style detector predicts a fixed set of candidate objects. After +/// each ground-truth object is matched to one candidate, this setting chooses how the class scores of +/// every candidate are trained. +/// +public enum SetPredictionClassificationLoss +{ + /// + /// Softmax cross-entropy over the foreground classes plus a final no-object class, with the + /// no-object class down-weighted (Carion et al. 2020, DETR). + /// + SoftmaxCrossEntropy, + + /// + /// Per-class sigmoid focal loss with no no-object class (Lin et al. 2017), as used by Deformable + /// DETR and DINO (Zhang et al. 2022). + /// + SigmoidFocal, + + /// + /// IoU-aware varifocal loss (Zhang et al. 2021): the matched class is trained toward the IoU of + /// its predicted box, as used by RT-DETR (Zhao et al. 2023). + /// + VariFocal +} diff --git a/src/Interfaces/IDetectionTrainingModel.cs b/src/Interfaces/IDetectionTrainingModel.cs new file mode 100644 index 0000000000..0d4a8e432b --- /dev/null +++ b/src/Interfaces/IDetectionTrainingModel.cs @@ -0,0 +1,18 @@ +using AiDotNet.ComputerVision.Detection; + +namespace AiDotNet.Interfaces; + +/// A model that implements its detection task's assignment and classification/box loss. +/// The detector's numeric type. +/// +/// This capability is separate from raw-output tensor regression. Implementing it promises a real +/// family-specific detection objective, not a generic MSE fallback or an inferred tensor format. +/// +public interface IDetectionTrainingModel +{ + /// Runs one semantic detection training step. + /// Model-ready NCHW image batch, preprocessed like the model's Predict input. + /// One immutable foreground target list per image; empty lists are valid. + /// Inputs are borrowed. Models must reject unsupported target cardinality before updating. + void TrainDetections(Tensor input, DetectionTrainingBatch targets); +} diff --git a/src/Metrics/ObjectDetectionMetrics.cs b/src/Metrics/ObjectDetectionMetrics.cs new file mode 100644 index 0000000000..db66f62dad --- /dev/null +++ b/src/Metrics/ObjectDetectionMetrics.cs @@ -0,0 +1,633 @@ +using AiDotNet.Augmentation.Image; +using AiDotNet.ComputerVision.Detection.ObjectDetection; +using AiDotNet.Helpers; + +namespace AiDotNet.Metrics; + +/// +/// COCO-style evaluation metrics for object detection: Average Precision (AP), +/// mean Average Precision (mAP) and the underlying precision-recall curve. +/// +/// +/// +/// These are the metrics the object-detection literature reports. A detector emits a set of +/// detections per image, each carrying a box, a class and a confidence score. Evaluation matches +/// those predictions against ground-truth detections and summarises the quality of the ranking. +/// Ground truth is expressed with the same type, of which only +/// and are read - its confidence +/// is ignored. +/// +/// The matching rule (COCO / Pascal VOC): +/// predictions for one class are sorted by confidence, highest first. Each prediction is +/// matched greedily to the highest-IoU ground-truth box of the same class in the same image +/// that has not already been claimed. A match with IoU at or above the threshold is a true +/// positive; anything else is a false positive. Ground-truth boxes that end up unmatched are +/// false negatives. This one-to-one, highest-confidence-wins rule is what stops a detector +/// from inflating its score by emitting many overlapping boxes for the same object. +/// +/// Interpolation: +/// AP is the area under the precision-recall curve. COCO computes it by sampling the curve at +/// 101 evenly spaced recall levels (0.00, 0.01, ... 1.00) and, at each level, taking the highest +/// precision observed at that recall or beyond. That highest-precision-at-or-beyond step removes +/// the small downward wiggles the raw curve has, which would otherwise make the score depend on +/// ties in the confidence ordering. +/// +/// Which number to report: +/// +/// at 0.5 is Pascal VOC mAP@0.5 - the lenient, +/// widely quoted number. +/// is COCO mAP@[.50:.95], the primary +/// COCO metric: mAP averaged over ten IoU thresholds from 0.50 to 0.95. It rewards precise +/// localisation, so it is always lower than mAP@0.5. +/// +/// +/// For Beginners: IoU (intersection over union) measures how much a predicted box +/// overlaps a real one: 1.0 means identical, 0.0 means no overlap at all. An IoU threshold of 0.5 +/// says a prediction counts as correct if it covers at least half the union of itself and the +/// real box. Average Precision then rolls the whole precision/recall trade-off into a single +/// number between 0 and 1, where higher is better. Reporting mAP@[.50:.95] rather than mAP@0.5 +/// is the stricter, modern convention because it also asks the box to be tightly placed, not +/// merely in roughly the right spot. +/// +/// +/// +/// // One image: a class-0 ground-truth box and a detection that overlaps it. +/// var groundTruthPerImage = new List<IReadOnlyList<AiDotNet.ComputerVision.Detection.ObjectDetection.Detection<double>>> +/// { +/// new[] { new AiDotNet.ComputerVision.Detection.ObjectDetection.Detection<double>( +/// new AiDotNet.Augmentation.Image.BoundingBox<double>(0, 0, 10, 10), classId: 0, confidence: 1.0) } +/// }; +/// var predicted = new List<IReadOnlyList<AiDotNet.ComputerVision.Detection.ObjectDetection.Detection<double>>> +/// { +/// new[] { new AiDotNet.ComputerVision.Detection.ObjectDetection.Detection<double>( +/// new AiDotNet.Augmentation.Image.BoundingBox<double>(1, 1, 10, 10), classId: 0, confidence: 0.9) } +/// }; +/// +/// var metrics = new ObjectDetectionMetrics<double>(); +/// double cocoMap = metrics.MeanAveragePrecisionRange(predicted, groundTruthPerImage); +/// double vocMap = metrics.MeanAveragePrecision(predicted, groundTruthPerImage, 0.5); +/// +/// +/// +/// The numeric type the detections are expressed in. +public class ObjectDetectionMetrics where T : struct +{ + /// + /// Number of recall points COCO samples the precision-recall curve at (0.00 to 1.00 inclusive). + /// + private const int RecallSampleCount = 101; + + // Bound per-threshold claims and AP points even for very densely sampled ranges. COCO's + // ten thresholds fit in one batch; larger ranges reuse class preparation across batches. + private const int ThresholdBatchSize = 32; + + /// + /// The numeric operations provider for type . + /// + private readonly INumericOperations _numOps; + + /// + /// Initializes a new instance of the class. + /// + public ObjectDetectionMetrics() + { + _numOps = MathHelper.GetNumericOperations(); + } + + /// + /// Computes Average Precision for a single class at one IoU threshold. + /// + /// Predicted detections, one list per image. Confidence drives the ranking. + /// Ground-truth detections, one list per image, aligned with + /// . Only box and class are read. + /// The class to score. Detections of other classes are ignored. + /// Minimum IoU for a prediction to count as a true positive. + /// AP in [0, 1], or when the class has no ground-truth boxes + /// (an undefined score, which excludes from its average). + /// A required argument is null. + /// The two lists describe a different number of images. + /// is not finite or is outside [0, 1]. + public double AveragePrecision( + IReadOnlyList>> predictions, + IReadOnlyList>> groundTruth, + int classIndex, + double iouThreshold = 0.5) + { + var curve = ComputeCurve(predictions, groundTruth, classIndex, iouThreshold, out int groundTruthCount); + if (groundTruthCount == 0) + { + return double.NaN; + } + + return InterpolatedAveragePrecision(curve.Precision, curve.Recall, curve.Precision.Length); + } + + /// + /// Computes mean Average Precision at one IoU threshold: the mean of the per-class + /// over every class that has at least one ground-truth box. + /// + /// Predicted detections, one list per image. + /// Ground-truth detections, one list per image. + /// Minimum IoU for a prediction to count as a true positive. 0.5 is Pascal VOC mAP@0.5. + /// mAP in [0, 1], or 0 when the ground truth contains no detections at all. + /// A required argument is null. + /// The two lists describe a different number of images. + /// is not finite or is outside [0, 1]. + public double MeanAveragePrecision( + IReadOnlyList>> predictions, + IReadOnlyList>> groundTruth, + double iouThreshold = 0.5) + { + ValidateAligned(predictions, groundTruth); + ValidateIoUThreshold(iouThreshold); + + var classes = GetGroundTruthClasses(groundTruth); + + if (classes.Count == 0) + { + return 0.0; + } + + double sum = 0.0; + int counted = 0; + foreach (int classIndex in classes) + { + double ap = AveragePrecision(predictions, groundTruth, classIndex, iouThreshold); + if (!double.IsNaN(ap)) + { + sum += ap; + counted++; + } + } + + return counted > 0 ? sum / counted : 0.0; + } + + /// + /// Computes COCO mAP@[.50:.95]: averaged over a range of + /// IoU thresholds. This is the primary COCO detection metric. + /// + /// + /// Ground-truth lists and stable confidence rankings are prepared once per class. Each batch + /// of at most 32 thresholds has independent greedy matches and shares a lazily computed IoU + /// row for the current prediction. Thus COCO's ten thresholds compute each needed IoU once; + /// ranges spanning several batches may compute it once per batch. No all-pairs IoU matrix or + /// range-sized collection of matching states is allocated. AP retains only true-positive + /// points, bounding per-batch state by the number of ground-truth boxes, not false positives. + /// An endpoint numerically on the grid is included using a scale-aware floating-point + /// tolerance. If its reconstructed value overshoots the maximum only by arithmetic + /// roundoff, that final threshold is capped at the requested maximum; off-grid maxima + /// are not appended as additional thresholds. + /// + /// Predicted detections, one list per image. + /// Ground-truth detections, one list per image. + /// First IoU threshold. COCO uses 0.50. + /// Last IoU threshold, inclusive. COCO uses 0.95. + /// Spacing between thresholds. COCO uses 0.05, giving ten thresholds. + /// mAP averaged across the thresholds, in [0, 1]. + /// is not finite and positive, + /// the range is non-finite, empty or outside [0, 1], or its threshold count exceeds . + public double MeanAveragePrecisionRange( + IReadOnlyList>> predictions, + IReadOnlyList>> groundTruth, + double minIoU = 0.5, + double maxIoU = 0.95, + double step = 0.05) + { + if (double.IsNaN(step) || double.IsInfinity(step) || step <= 0.0) + { + throw new ArgumentOutOfRangeException(nameof(step), step, "IoU step must be finite and positive."); + } + + if (!IsUnitInterval(minIoU) || !IsUnitInterval(maxIoU) || minIoU > maxIoU) + { + throw new ArgumentOutOfRangeException( + nameof(minIoU), $"IoU range [{minIoU}, {maxIoU}] must be non-empty and within [0, 1]."); + } + + // Derive the count first rather than accumulating threshold += step, so floating-point + // drift cannot silently drop or duplicate the final threshold. + var (thresholdCount, lastThreshold) = GetThresholdGrid(minIoU, maxIoU, step); + ValidateAligned(predictions, groundTruth); + var preparedClasses = GetGroundTruthClasses(groundTruth) + .Select(classIndex => PrepareClass(predictions, groundTruth, classIndex)).ToArray(); + int counted = preparedClasses.Count(prepared => prepared.GroundTruthCount > 0); + if (counted == 0) + { + return 0.0; + } + + double sum = 0.0; + int firstThreshold = 0; + while (firstThreshold < thresholdCount) + { + int batchCount = Math.Min(ThresholdBatchSize, thresholdCount - firstThreshold); + var thresholds = new double[batchCount]; + var classSums = new double[batchCount]; + for (int i = 0; i < batchCount; i++) + { + int index = firstThreshold + i; + thresholds[i] = index == thresholdCount - 1 ? lastThreshold : minIoU + (index * step); + } + + foreach (var prepared in preparedClasses) + { + if (prepared.GroundTruthCount == 0) + { + continue; + } + + var scores = ComputeAveragePrecisionBatch(prepared, thresholds); + for (int i = 0; i < batchCount; i++) + { + classSums[i] += scores[i]; + } + } + + // Preserve the original order of both sums: sorted classes within each threshold, + // then increasing thresholds. Reordering these averages changes floating-point bits. + for (int i = 0; i < batchCount; i++) + { + sum += classSums[i] / counted; + } + + firstThreshold += batchCount; + } + + return sum / thresholdCount; + } + + // Both comparisons are false for NaN; infinities also fall outside this finite interval. + private static bool IsUnitInterval(double value) => value >= 0.0 && value <= 1.0; + + private static void ValidateIoUThreshold(double iouThreshold) + { + if (!IsUnitInterval(iouThreshold)) + { + throw new ArgumentOutOfRangeException(nameof(iouThreshold), iouThreshold, + "IoU threshold must be finite and within [0, 1]."); + } + } + + private static (int Count, double Last) GetThresholdGrid(double minimum, double maximum, double step) + { + // double.Epsilon is the smallest subnormal, not the machine rounding epsilon. + const double machineEpsilon = 2.2204460492503131e-16; + double rawLastIndex = (maximum - minimum) / step; + double nearestInteger = Math.Round(rawLastIndex); + double quotientTolerance = 16 * machineEpsilon * Math.Max(1, Math.Abs(rawLastIndex)); + bool endpointOnGrid = Math.Abs(rawLastIndex - nearestInteger) <= quotientTolerance; + double lastIndex = endpointOnGrid ? nearestInteger : Math.Floor(rawLastIndex); + if (lastIndex >= int.MaxValue) + { + throw new ArgumentOutOfRangeException(nameof(step), step, + "IoU step produces more thresholds than an Int32 count can represent."); + } + + double distance = lastIndex * step; + double lastThreshold = minimum + distance; + if (lastThreshold > maximum) + { + double endpointTolerance = 32 * machineEpsilon * Math.Max(Math.Abs(maximum), Math.Abs(minimum) + Math.Abs(distance)); + if (endpointOnGrid && lastThreshold - maximum <= endpointTolerance) + { + // For example, .1 + 2*.1 rounds above .3. Never pass that larger value + // to the matcher, and never manufacture an endpoint for an off-grid range. + lastThreshold = maximum; + } + else + { + // A genuine overshoot is outside the requested range, not a reason to clamp + // a new off-grid threshold into it. Only an included last index can overshoot. + lastIndex--; + lastThreshold = minimum + lastIndex * step; + } + } + return ((int)lastIndex + 1, lastThreshold); + } + + /// + /// Computes the raw (uninterpolated) precision-recall curve for one class, in descending + /// confidence order. Point i is the precision and recall achieved when the top + /// i + 1 predictions are accepted. + /// + /// Predicted detections, one list per image. + /// Ground-truth detections, one list per image. + /// The class to score. + /// Minimum IoU for a prediction to count as a true positive. + /// Parallel precision and recall arrays. Both are empty when the class has no predictions. + /// A required argument is null. + /// The two lists describe a different number of images. + /// is not finite or is outside [0, 1]. + public (double[] Precision, double[] Recall) PrecisionRecallCurve( + IReadOnlyList>> predictions, + IReadOnlyList>> groundTruth, + int classIndex, + double iouThreshold = 0.5) + => ComputeCurve(predictions, groundTruth, classIndex, iouThreshold, out _); + + private (double[] Precision, double[] Recall) ComputeCurve( + IReadOnlyList>> predictions, + IReadOnlyList>> groundTruth, + int classIndex, + double iouThreshold, + out int groundTruthCount) + { + ValidateAligned(predictions, groundTruth); + ValidateIoUThreshold(iouThreshold); + + var prepared = PrepareClass(predictions, groundTruth, classIndex); + groundTruthCount = prepared.GroundTruthCount; + var claimed = new bool[groundTruthCount]; + var precision = new double[prepared.RankOrder.Length]; + var recall = new double[precision.Length]; + int truePositives = 0; + + for (int rank = 0; rank < prepared.RankOrder.Length; rank++) + { + var (imageIndex, box) = prepared.Predictions[prepared.RankOrder[rank]]; + var candidates = prepared.TruthByImage[imageIndex]; + int offset = prepared.TruthOffsets[imageIndex]; + + double bestIoU = 0.0; + int bestCandidate = -1; + for (int c = 0; c < candidates.Count; c++) + { + if (claimed[offset + c]) + { + continue; + } + + double iou = box.IoU(candidates[c]); + if (!double.IsNaN(iou) && (bestCandidate < 0 || iou > bestIoU)) + { + bestIoU = iou; + bestCandidate = c; + } + } + + if (bestCandidate >= 0 && bestIoU >= iouThreshold) + { + claimed[offset + bestCandidate] = true; + truePositives++; + } + + precision[rank] = truePositives / (double)(rank + 1); + recall[rank] = groundTruthCount > 0 ? truePositives / (double)groundTruthCount : 0.0; + } + + return (precision, recall); + } + + private PreparedClass PrepareClass( + IReadOnlyList>> predictions, + IReadOnlyList>> groundTruth, + int classIndex) + { + // Keep original per-image candidate order, including equal-IoU tie precedence. Geometry + // is deliberately not read here: an already-claimed candidate must remain unused. + var truthByImage = new List>[groundTruth.Count]; + var truthOffsets = new int[groundTruth.Count]; + int groundTruthCount = 0; + int maxTruthPerImage = 0; + for (int i = 0; i < groundTruth.Count; i++) + { + var kept = new List>(); + var image = groundTruth[i]; + if (image is not null) + { + foreach (var detection in image) + { + if (detection is not null && detection.ClassId == classIndex && detection.Box is not null) + { + kept.Add(detection.Box); + } + } + } + + truthByImage[i] = kept; + truthOffsets[i] = groundTruthCount; + groundTruthCount += kept.Count; + maxTruthPerImage = Math.Max(maxTruthPerImage, kept.Count); + } + + // Every prediction of this class across all images, ranked by confidence. OrderByDescending + // is a stable sort, so equal-confidence predictions keep their original order and the curve + // is reproducible run to run. + var ranked = new List<(int ImageIndex, BoundingBox Box)>(); + var scores = new List(); + for (int i = 0; i < predictions.Count; i++) + { + var image = predictions[i]; + if (image is null) + { + continue; + } + + foreach (var detection in image) + { + if (detection is not null && detection.ClassId == classIndex && detection.Box is not null) + { + ranked.Add((i, detection.Box)); + scores.Add(_numOps.ToDouble(detection.Confidence)); + } + } + } + + var order = Enumerable.Range(0, ranked.Count).OrderByDescending(i => scores[i]).ToArray(); + + return new PreparedClass(truthByImage, truthOffsets, ranked, order, groundTruthCount, maxTruthPerImage); + } + + private static double[] ComputeAveragePrecisionBatch(PreparedClass prepared, double[] thresholds) + { + int maxPoints = Math.Min(prepared.RankOrder.Length, prepared.GroundTruthCount); + var scores = new double[thresholds.Length]; + if (maxPoints == 0) + { + return scores; + } + + var claimed = new bool[thresholds.Length][]; + var precision = new double[thresholds.Length][]; + var recall = new double[thresholds.Length][]; + var truePositives = new int[thresholds.Length]; + for (int threshold = 0; threshold < thresholds.Length; threshold++) + { + claimed[threshold] = new bool[prepared.GroundTruthCount]; + precision[threshold] = new double[maxPoints]; + recall[threshold] = new double[maxPoints]; + } + + // One lazily populated IoU row, not a predictions-by-ground-truth matrix. Rank stamps + // distinguish uncomputed entries from every possible IoU value, including zero and NaN. + var iouRow = new double[prepared.MaxTruthPerImage]; + var rowRanks = new int[prepared.MaxTruthPerImage]; + for (int rank = 0; rank < prepared.RankOrder.Length; rank++) + { + var (imageIndex, box) = prepared.Predictions[prepared.RankOrder[rank]]; + var candidates = prepared.TruthByImage[imageIndex]; + int offset = prepared.TruthOffsets[imageIndex]; + for (int threshold = 0; threshold < thresholds.Length; threshold++) + { + var candidateClaimed = claimed[threshold]; + // The first candidate is always eligible, so an IoU of exactly 0 can still match at the + // inclusive threshold 0 ('at or above'). Starting at 0 with a strict '>' made that endpoint + // unreachable. NaN (a degenerate box) is never picked, or it would block every later candidate. + double bestIoU = 0.0; + int bestCandidate = -1; + for (int c = 0; c < candidates.Count; c++) + { + if (candidateClaimed[offset + c]) + { + continue; + } + + if (rowRanks[c] != rank + 1) + { + iouRow[c] = box.IoU(candidates[c]); + rowRanks[c] = rank + 1; + } + + double iou = iouRow[c]; + if (!double.IsNaN(iou) && (bestCandidate < 0 || iou > bestIoU)) + { + bestIoU = iou; + bestCandidate = c; + } + } + + if (bestCandidate >= 0 && bestIoU >= thresholds[threshold]) + { + candidateClaimed[offset + bestCandidate] = true; + int point = truePositives[threshold]++; + precision[threshold][point] = truePositives[threshold] / (double)(rank + 1); + recall[threshold][point] = truePositives[threshold] / (double)prepared.GroundTruthCount; + } + } + } + + // A false positive cannot improve precision at unchanged recall; its preceding true + // positive dominates it. Initial false positives are zero, and no true positives means + // AP zero. Keeping only TP points therefore preserves the exact 101-sample envelope. + for (int threshold = 0; threshold < thresholds.Length; threshold++) + { + scores[threshold] = InterpolatedAveragePrecision( + precision[threshold], recall[threshold], truePositives[threshold]); + } + + return scores; + } + + private static SortedSet GetGroundTruthClasses(IReadOnlyList>> groundTruth) + { + // Classes absent from ground truth have undefined recall and are not averaged in. + var classes = new SortedSet(); + foreach (var image in groundTruth) + { + if (image is null) + { + continue; + } + + foreach (var detection in image) + { + if (detection is not null) + { + classes.Add(detection.ClassId); + } + } + } + + return classes; + } + + private sealed class PreparedClass + { + public List>[] TruthByImage { get; } + public int[] TruthOffsets { get; } + public List<(int ImageIndex, BoundingBox Box)> Predictions { get; } + public int[] RankOrder { get; } + public int GroundTruthCount { get; } + public int MaxTruthPerImage { get; } + + public PreparedClass(List>[] truthByImage, int[] truthOffsets, + List<(int ImageIndex, BoundingBox Box)> predictions, int[] rankOrder, + int groundTruthCount, int maxTruthPerImage) + { + TruthByImage = truthByImage; + TruthOffsets = truthOffsets; + Predictions = predictions; + RankOrder = rankOrder; + GroundTruthCount = groundTruthCount; + MaxTruthPerImage = maxTruthPerImage; + } + } + + /// + /// Area under the precision-recall curve using COCO 101-point interpolation: at each of 101 + /// evenly spaced recall levels, take the highest precision attained at that recall or beyond, + /// then average those 101 values. + /// + private static double InterpolatedAveragePrecision(double[] precision, double[] recall, int pointCount) + { + if (pointCount == 0) + { + return 0.0; + } + + // Sweep right-to-left so envelope[i] is the best precision achievable at recall >= recall[i]. + var envelope = new double[pointCount]; + double running = 0.0; + for (int i = pointCount - 1; i >= 0; i--) + { + running = Math.Max(running, precision[i]); + envelope[i] = running; + } + + double sum = 0.0; + int cursor = 0; + for (int s = 0; s < RecallSampleCount; s++) + { + double target = s / (double)(RecallSampleCount - 1); + + // recall is non-decreasing along the ranking, so the cursor only ever moves forward. + while (cursor < pointCount && recall[cursor] < target) + { + cursor++; + } + + if (cursor >= pointCount) + { + break; // No prediction reaches this recall; the remaining samples contribute 0. + } + + sum += envelope[cursor]; + } + + return sum / RecallSampleCount; + } + + private static void ValidateAligned( + IReadOnlyList>> predictions, + IReadOnlyList>> groundTruth) + { + if (predictions is null) + { + throw new ArgumentNullException(nameof(predictions)); + } + + if (groundTruth is null) + { + throw new ArgumentNullException(nameof(groundTruth)); + } + + if (predictions.Count != groundTruth.Count) + { + throw new ArgumentException( + $"Predictions cover {predictions.Count} images but ground truth covers {groundTruth.Count}. " + + "Both lists must be indexed by the same image order.", + nameof(predictions)); + } + } +} diff --git a/src/Metrics/TextDetectionMetrics.cs b/src/Metrics/TextDetectionMetrics.cs new file mode 100644 index 0000000000..4732b1c907 --- /dev/null +++ b/src/Metrics/TextDetectionMetrics.cs @@ -0,0 +1,404 @@ +using AiDotNet.ComputerVision.Detection.TextDetection; +using AiDotNet.Helpers; + +namespace AiDotNet.Metrics; + +/// +/// ICDAR-style evaluation metrics for text detection: polygon IoU, and the precision, recall and +/// H-mean triple that the ICDAR Robust Reading competitions report. +/// +/// +/// +/// Text detectors localise words or lines as quadrilaterals or polygons rather than axis-aligned +/// boxes, because printed and scene text is frequently rotated or curved. Evaluation therefore +/// works on polygon overlap, not box overlap. +/// +/// The ICDAR 2015 IoU protocol. +/// Within each image, predictions are considered in descending confidence order and matched +/// one-to-one against ground-truth regions: a prediction claims the unmatched ground-truth region +/// it overlaps most, provided that overlap reaches the IoU threshold (0.5 by convention). Then +/// precision is matched / predicted, recall is matched / ground-truth, and H-mean is their +/// harmonic mean. H-mean is the number competitions rank on. +/// +/// Convexity. +/// computes the intersection by Sutherland-Hodgman clipping, which is +/// exact when the second polygon is convex. Detector output for word- and line-level text is +/// quadrilateral or near-convex, which is the case the ICDAR protocol is defined over. For a +/// strongly concave polygon (a curved-text detector emitting a banana-shaped region) the +/// intersection can be over-estimated, so treat such scores as approximate. +/// +/// For Beginners: Precision asks "of the regions the model reported, how many were +/// real text?" Recall asks "of the real text, how much did the model find?" A model can score +/// perfectly on one by sacrificing the other - report everything for perfect recall, report only +/// the single most obvious word for perfect precision. H-mean combines them so that a model has to +/// do well at both: it is close to the smaller of the two, so one bad number drags it down. +/// +/// +/// +/// // One image: a ground-truth text region and a predicted region that overlaps it. +/// var groundTruthRegionsPerImage = new List<IReadOnlyList<AiDotNet.ComputerVision.Detection.TextDetection.TextRegion<double>>> +/// { +/// new[] { new AiDotNet.ComputerVision.Detection.TextDetection.TextRegion<double>( +/// new AiDotNet.Augmentation.Image.BoundingBox<double>(0, 0, 40, 12), 1.0) } +/// }; +/// var predictedRegionsPerImage = new List<IReadOnlyList<AiDotNet.ComputerVision.Detection.TextDetection.TextRegion<double>>> +/// { +/// new[] { new AiDotNet.ComputerVision.Detection.TextDetection.TextRegion<double>( +/// new AiDotNet.Augmentation.Image.BoundingBox<double>(1, 0, 40, 12), 0.9) } +/// }; +/// +/// var metrics = new TextDetectionMetrics<double>(); +/// var (precision, recall, hmean) = metrics.Evaluate(predictedRegionsPerImage, groundTruthRegionsPerImage); +/// +/// +/// +/// The numeric type the detected regions are expressed in. +public class TextDetectionMetrics where T : struct +{ + /// + /// Coordinates closer together than this are treated as coincident when intersecting edges. + /// + private const double GeometricTolerance = 1e-12; + + /// + /// The numeric operations provider for type . + /// + private readonly INumericOperations _numOps; + + /// + /// Initializes a new instance of the class. + /// + public TextDetectionMetrics() + { + _numOps = MathHelper.GetNumericOperations(); + } + + /// + /// Computes the area of a simple polygon using the shoelace formula. + /// + /// The polygon vertices, in order. Fewer than three vertices enclose no area. + /// The absolute area, so the result does not depend on winding direction. + public static double PolygonArea(IReadOnlyList<(double X, double Y)> polygon) + => Math.Abs(SignedArea(polygon)); + + /// + /// Computes intersection-over-union between two polygons. + /// + /// The first polygon vertices, in order. + /// The second polygon vertices, in order. This one is used as the clipping + /// polygon, so the result is exact when it is convex (see the class remarks). + /// IoU in [0, 1]. Returns 0 when either polygon is degenerate or they do not overlap. + /// A required argument is null. + public static double PolygonIoU( + IReadOnlyList<(double X, double Y)> first, + IReadOnlyList<(double X, double Y)> second) + { + if (first is null) + { + throw new ArgumentNullException(nameof(first)); + } + + if (second is null) + { + throw new ArgumentNullException(nameof(second)); + } + + double areaFirst = PolygonArea(first); + double areaSecond = PolygonArea(second); + if (areaFirst <= 0.0 || areaSecond <= 0.0) + { + return 0.0; + } + + // Sutherland-Hodgman requires both polygons wound the same way and the clip polygon + // counter-clockwise, so the left-of-edge test means inside. + var subject = EnsureCounterClockwise(first); + var clip = EnsureCounterClockwise(second); + + double intersection = PolygonArea(ClipToConvex(subject, clip)); + double union = areaFirst + areaSecond - intersection; + + return union > 0.0 ? intersection / union : 0.0; + } + + /// + /// Evaluates detected text regions against ground truth using the ICDAR IoU protocol. + /// + /// Detected regions, one list per image. Confidence drives the matching + /// order within each image. + /// Ground-truth regions, one list per image, aligned with + /// . + /// Minimum polygon IoU for a match. ICDAR uses 0.5. + /// Precision, recall and their harmonic mean, each in [0, 1]. Precision is 1 when nothing + /// was predicted, recall is 1 when there is nothing to find, and H-mean is 0 when precision and + /// recall are both 0. + /// A required argument is null. + /// The two lists describe a different number of images. + public (double Precision, double Recall, double HMean) Evaluate( + IReadOnlyList>> predictions, + IReadOnlyList>> groundTruth, + double iouThreshold = 0.5) + { + if (predictions is null) + { + throw new ArgumentNullException(nameof(predictions)); + } + + if (groundTruth is null) + { + throw new ArgumentNullException(nameof(groundTruth)); + } + + if (predictions.Count != groundTruth.Count) + { + throw new ArgumentException( + $"Predictions cover {predictions.Count} images but ground truth covers {groundTruth.Count}. " + + "Both lists must be indexed by the same image order.", + nameof(predictions)); + } + + int matched = 0; + int predictedCount = 0; + int truthCount = 0; + + for (int i = 0; i < predictions.Count; i++) + { + var truthPolygons = ToPolygons(groundTruth[i]); + var claimed = new bool[truthPolygons.Count]; + truthCount += truthPolygons.Count; + + // Highest confidence first, so when two predictions both cover a region the better one + // claims it. OrderByDescending is stable, keeping equal-confidence order reproducible. + var ordered = OrderByConfidenceDescending(predictions[i]); + predictedCount += ordered.Count; + + foreach (var region in ordered) + { + var polygon = ToPolygon(region); + if (polygon.Count < 3) + { + continue; // Degenerate prediction: counted against precision, can match nothing. + } + + double bestIoU = 0.0; + int bestCandidate = -1; + for (int c = 0; c < truthPolygons.Count; c++) + { + if (claimed[c]) + { + continue; + } + + double iou = PolygonIoU(polygon, truthPolygons[c]); + if (iou > bestIoU) + { + bestIoU = iou; + bestCandidate = c; + } + } + + if (bestCandidate >= 0 && bestIoU >= iouThreshold) + { + claimed[bestCandidate] = true; + matched++; + } + } + } + + double precision = predictedCount > 0 ? matched / (double)predictedCount : 1.0; + double recall = truthCount > 0 ? matched / (double)truthCount : 1.0; + double hmean = (precision + recall) > 0.0 + ? 2.0 * precision * recall / (precision + recall) + : 0.0; + + return (precision, recall, hmean); + } + + /// + /// Converts a detected region to a double-precision polygon, falling back to the corners of its + /// bounding box when no polygon was supplied. + /// + /// The region to convert. + /// The polygon vertices, or an empty list when the region carries neither polygon nor box. + internal List<(double X, double Y)> ToPolygon(TextRegion region) + { + var polygon = new List<(double X, double Y)>(); + if (region is null) + { + return polygon; + } + + if (region.Polygon is not null && region.Polygon.Count >= 3) + { + foreach (var vertex in region.Polygon) + { + polygon.Add((_numOps.ToDouble(vertex.X), _numOps.ToDouble(vertex.Y))); + } + + return polygon; + } + + if (region.Box is not null) + { + var (xMin, yMin, xMax, yMax) = region.Box.ToXYXY(); + polygon.Add((xMin, yMin)); + polygon.Add((xMax, yMin)); + polygon.Add((xMax, yMax)); + polygon.Add((xMin, yMax)); + } + + return polygon; + } + + private List> ToPolygons(IReadOnlyList>? regions) + { + var polygons = new List>(); + if (regions is null) + { + return polygons; + } + + foreach (var region in regions) + { + var polygon = ToPolygon(region); + if (polygon.Count >= 3) + { + polygons.Add(polygon); + } + } + + return polygons; + } + + private List> OrderByConfidenceDescending(IReadOnlyList>? regions) + { + var kept = new List>(); + if (regions is null) + { + return kept; + } + + foreach (var region in regions) + { + if (region is not null) + { + kept.Add(region); + } + } + + return kept.OrderByDescending(r => _numOps.ToDouble(r.Confidence)).ToList(); + } + + private static double SignedArea(IReadOnlyList<(double X, double Y)> polygon) + { + if (polygon is null || polygon.Count < 3) + { + return 0.0; + } + + double sum = 0.0; + for (int i = 0; i < polygon.Count; i++) + { + var current = polygon[i]; + var next = polygon[(i + 1) % polygon.Count]; + sum += (current.X * next.Y) - (next.X * current.Y); + } + + return sum / 2.0; + } + + private static List<(double X, double Y)> EnsureCounterClockwise(IReadOnlyList<(double X, double Y)> polygon) + { + var ordered = new List<(double X, double Y)>(polygon); + if (SignedArea(ordered) < 0.0) + { + ordered.Reverse(); + } + + return ordered; + } + + /// + /// Sutherland-Hodgman polygon clipping: successively clips the subject polygon against each + /// directed edge of the (convex, counter-clockwise) clip polygon. + /// + private static List<(double X, double Y)> ClipToConvex( + List<(double X, double Y)> subject, + List<(double X, double Y)> clip) + { + var output = new List<(double X, double Y)>(subject); + + for (int edge = 0; edge < clip.Count && output.Count > 0; edge++) + { + var edgeStart = clip[edge]; + var edgeEnd = clip[(edge + 1) % clip.Count]; + + var input = output; + output = new List<(double X, double Y)>(input.Count + 2); + + for (int i = 0; i < input.Count; i++) + { + var current = input[i]; + var previous = input[(i + input.Count - 1) % input.Count]; + + bool currentInside = IsLeftOfOrOn(edgeStart, edgeEnd, current); + bool previousInside = IsLeftOfOrOn(edgeStart, edgeEnd, previous); + + if (currentInside) + { + if (!previousInside) + { + output.Add(LineIntersection(previous, current, edgeStart, edgeEnd)); + } + + output.Add(current); + } + else if (previousInside) + { + output.Add(LineIntersection(previous, current, edgeStart, edgeEnd)); + } + } + } + + return output; + } + + /// + /// True when lies to the left of, or on, the directed edge + /// to . For a counter-clockwise polygon + /// that is the inside half-plane. + /// + private static bool IsLeftOfOrOn( + (double X, double Y) edgeStart, + (double X, double Y) edgeEnd, + (double X, double Y) point) + => (((edgeEnd.X - edgeStart.X) * (point.Y - edgeStart.Y)) + - ((edgeEnd.Y - edgeStart.Y) * (point.X - edgeStart.X))) >= 0.0; + + private static (double X, double Y) LineIntersection( + (double X, double Y) firstStart, + (double X, double Y) firstEnd, + (double X, double Y) secondStart, + (double X, double Y) secondEnd) + { + double firstCross = (firstStart.X * firstEnd.Y) - (firstStart.Y * firstEnd.X); + double secondCross = (secondStart.X * secondEnd.Y) - (secondStart.Y * secondEnd.X); + + double firstDx = firstStart.X - firstEnd.X; + double firstDy = firstStart.Y - firstEnd.Y; + double secondDx = secondStart.X - secondEnd.X; + double secondDy = secondStart.Y - secondEnd.Y; + + double denominator = (firstDx * secondDy) - (firstDy * secondDx); + if (Math.Abs(denominator) < GeometricTolerance) + { + // Parallel or coincident edges: the crossing is degenerate, so fall back to the endpoint + // that the caller was about to emit anyway. + return firstEnd; + } + + double x = ((firstCross * secondDx) - (firstDx * secondCross)) / denominator; + double y = ((firstCross * secondDy) - (firstDy * secondCross)) / denominator; + return (x, y); + } +} diff --git a/src/Metrics/TextRecognitionMetrics.cs b/src/Metrics/TextRecognitionMetrics.cs new file mode 100644 index 0000000000..a696c19964 --- /dev/null +++ b/src/Metrics/TextRecognitionMetrics.cs @@ -0,0 +1,413 @@ +using System.Text; + +namespace AiDotNet.Metrics; + +/// +/// Evaluation metrics for text recognition (OCR): Character Error Rate, Word Error Rate, +/// normalized edit distance and exact-match accuracy. +/// +/// +/// +/// Recognition output is a string, so quality is measured by how many edits turn the prediction +/// into the reference. All the metrics here are built on Levenshtein distance - the smallest +/// number of single-character insertions, deletions and substitutions that transform one string +/// into another. +/// +/// Corpus versus sentence averaging. +/// The overloads taking a whole list are corpus-level: they sum the edit distances and +/// divide by the summed reference length. That is the convention used by the ICDAR Robust Reading +/// competitions and by speech recognition. Averaging the per-sample rates instead would let a +/// single short reference dominate, so prefer the corpus overloads when reporting a benchmark +/// number. +/// +/// Which number to report: +/// +/// - +/// the standard number for line- and page-level OCR. Lower is better; 0 is perfect. +/// - +/// the same idea over whitespace-separated tokens, for document OCR. +/// - +/// ICDAR 1-NED. Higher is better; 1 is perfect. This is the one scene-text papers quote. +/// - +/// word-level accuracy for cropped-word benchmarks (IIIT5K, SVT, IC13, IC15). The field convention +/// is case-insensitive and alphanumeric-only, which is this method default. +/// +/// +/// For Beginners: Error rates answer "what fraction of the text did the model get +/// wrong?" A CER of 0.05 means about 5 characters in every 100 needed fixing. Note that an error +/// rate can exceed 1.0: if the model emits far more text than the reference contains, the number +/// of edits can be larger than the reference length. Accuracy metrics run the other way - higher +/// is better - so always check which direction a reported number points. +/// +/// +/// +/// string[] references = { "hello world", "aidotnet" }; +/// string[] predictions = { "hell0 world", "aidotnet" }; +/// double cer = TextRecognitionMetrics.CharacterErrorRate(references, predictions); // 1 edit / 19 chars +/// double acc = TextRecognitionMetrics.ExactMatchAccuracy(references, predictions); // 0.5 +/// +/// +/// +public static class TextRecognitionMetrics +{ + /// + /// Computes the Levenshtein edit distance between two strings: the minimum number of + /// single-character insertions, deletions and substitutions needed to turn + /// into . + /// + /// The first string. Null is treated as empty. + /// The second string. Null is treated as empty. + /// The edit distance, always at least the difference in lengths. + public static int LevenshteinDistance(string? source, string? target) + { + string a = source ?? string.Empty; + string b = target ?? string.Empty; + + if (a.Length == 0) + { + return b.Length; + } + + if (b.Length == 0) + { + return a.Length; + } + + // Two rolling rows rather than the full matrix: the recurrence only ever reads the previous + // row, so a page of text costs O(min(n, m)) memory instead of O(n * m). + var previous = new int[b.Length + 1]; + var current = new int[b.Length + 1]; + + for (int j = 0; j <= b.Length; j++) + { + previous[j] = j; + } + + for (int i = 1; i <= a.Length; i++) + { + current[0] = i; + for (int j = 1; j <= b.Length; j++) + { + int substitution = previous[j - 1] + (a[i - 1] == b[j - 1] ? 0 : 1); + int deletion = previous[j] + 1; + int insertion = current[j - 1] + 1; + current[j] = Math.Min(substitution, Math.Min(deletion, insertion)); + } + + (previous, current) = (current, previous); + } + + return previous[b.Length]; + } + + /// + /// Computes the Levenshtein edit distance between two token sequences, used by + /// . + /// + /// The first token sequence. Null is treated as empty. + /// The second token sequence. Null is treated as empty. + /// The edit distance in tokens. + public static int TokenEditDistance(IReadOnlyList? source, IReadOnlyList? target) + { + int n = source is null ? 0 : source.Count; + int m = target is null ? 0 : target.Count; + + if (n == 0) + { + return m; + } + + if (m == 0) + { + return n; + } + + var previous = new int[m + 1]; + var current = new int[m + 1]; + + for (int j = 0; j <= m; j++) + { + previous[j] = j; + } + + for (int i = 1; i <= n; i++) + { + current[0] = i; + for (int j = 1; j <= m; j++) + { + bool equal = string.Equals(source![i - 1], target![j - 1], StringComparison.Ordinal); + int substitution = previous[j - 1] + (equal ? 0 : 1); + int deletion = previous[j] + 1; + int insertion = current[j - 1] + 1; + current[j] = Math.Min(substitution, Math.Min(deletion, insertion)); + } + + (previous, current) = (current, previous); + } + + return previous[m]; + } + + /// + /// Computes the Character Error Rate for one prediction: edit distance divided by the + /// reference length. + /// + /// The ground-truth text. + /// The recognised text. + /// CER, 0 when the strings match. Can exceed 1 when the hypothesis is much longer + /// than the reference. Returns 0 for an empty reference matched by an empty hypothesis, and + /// for an empty reference with a non-empty hypothesis, where the rate + /// is undefined. + public static double CharacterErrorRate(string? reference, string? hypothesis) + { + string reference1 = reference ?? string.Empty; + string hypothesis1 = hypothesis ?? string.Empty; + + if (reference1.Length == 0) + { + return hypothesis1.Length == 0 ? 0.0 : double.NaN; + } + + return LevenshteinDistance(reference1, hypothesis1) / (double)reference1.Length; + } + + /// + /// Computes the corpus-level Character Error Rate: the summed edit distance over the summed + /// reference length. This is the standard way to report CER over a dataset. + /// + /// The ground-truth texts. + /// The recognised texts, aligned with . + /// CER over the whole corpus. With no reference characters, returns 0 if no edits + /// are needed, otherwise because the rate is undefined. + /// A required argument is null. + /// The lists have different lengths. + public static double CharacterErrorRate(IReadOnlyList references, IReadOnlyList hypotheses) + { + ValidateAligned(references, hypotheses); + + long distance = 0; + long length = 0; + for (int i = 0; i < references.Count; i++) + { + string reference = references[i] ?? string.Empty; + distance += LevenshteinDistance(reference, hypotheses[i]); + length += reference.Length; + } + + return length > 0 ? distance / (double)length : distance == 0 ? 0.0 : double.NaN; + } + + /// + /// Computes the Word Error Rate for one prediction: token-level edit distance divided by the + /// reference token count. Tokens are whitespace-separated. + /// + /// The ground-truth text. + /// The recognised text. + /// WER, 0 when the token sequences match. Returns when the + /// reference has no tokens but the hypothesis does. + public static double WordErrorRate(string? reference, string? hypothesis) + { + var referenceTokens = Tokenize(reference); + var hypothesisTokens = Tokenize(hypothesis); + + if (referenceTokens.Count == 0) + { + return hypothesisTokens.Count == 0 ? 0.0 : double.NaN; + } + + return TokenEditDistance(referenceTokens, hypothesisTokens) / (double)referenceTokens.Count; + } + + /// + /// Computes the corpus-level Word Error Rate: the summed token edit distance over the summed + /// reference token count. + /// + /// The ground-truth texts. + /// The recognised texts, aligned with . + /// WER over the whole corpus. With no reference tokens, returns 0 if no edits + /// are needed, otherwise because the rate is undefined. + /// A required argument is null. + /// The lists have different lengths. + public static double WordErrorRate(IReadOnlyList references, IReadOnlyList hypotheses) + { + ValidateAligned(references, hypotheses); + + long distance = 0; + long count = 0; + for (int i = 0; i < references.Count; i++) + { + var referenceTokens = Tokenize(references[i]); + distance += TokenEditDistance(referenceTokens, Tokenize(hypotheses[i])); + count += referenceTokens.Count; + } + + return count > 0 ? distance / (double)count : distance == 0 ? 0.0 : double.NaN; + } + + /// + /// Computes ICDAR normalized edit distance (1-NED) for one prediction: + /// 1 - distance / max(referenceLength, hypothesisLength). + /// + /// The ground-truth text. + /// The recognised text. + /// A similarity in [0, 1] where 1 is an exact match. Two empty strings score 1. + /// + /// Unlike this is bounded above by 1 and is a + /// similarity rather than an error, because it divides by the longer of the two strings. That + /// is what makes it safe to average across samples of very different lengths. + /// + public static double NormalizedEditDistance(string? reference, string? hypothesis) + { + string reference1 = reference ?? string.Empty; + string hypothesis1 = hypothesis ?? string.Empty; + + int longest = Math.Max(reference1.Length, hypothesis1.Length); + if (longest == 0) + { + return 1.0; + } + + return 1.0 - (LevenshteinDistance(reference1, hypothesis1) / (double)longest); + } + + /// + /// Computes mean ICDAR 1-NED over a dataset: the average of the per-sample + /// . + /// + /// The ground-truth texts. + /// The recognised texts, aligned with . + /// Mean 1-NED in [0, 1], or 1 for an empty dataset. + /// A required argument is null. + /// The lists have different lengths. + public static double NormalizedEditDistance(IReadOnlyList references, IReadOnlyList hypotheses) + { + ValidateAligned(references, hypotheses); + + if (references.Count == 0) + { + return 1.0; + } + + double sum = 0.0; + for (int i = 0; i < references.Count; i++) + { + sum += NormalizedEditDistance(references[i], hypotheses[i]); + } + + return sum / references.Count; + } + + /// + /// Computes exact-match accuracy: the fraction of predictions that equal their reference after + /// normalization. + /// + /// The ground-truth texts. + /// The recognised texts, aligned with . + /// When false (the default) both strings are lower-cased first. + /// When true (the default) every character that is not a letter or + /// digit is stripped first. + /// Accuracy in [0, 1], or 1 for an empty dataset. + /// + /// The defaults reproduce the scene-text benchmark protocol (IIIT5K, SVT, IC13, IC15), which + /// scores case-insensitively over the 36-character alphanumeric set. Pass + /// as true and as false to + /// score raw strings instead. + /// + /// A required argument is null. + /// The lists have different lengths. + public static double ExactMatchAccuracy( + IReadOnlyList references, + IReadOnlyList hypotheses, + bool caseSensitive = false, + bool alphanumericOnly = true) + { + ValidateAligned(references, hypotheses); + + if (references.Count == 0) + { + return 1.0; + } + + int matched = 0; + for (int i = 0; i < references.Count; i++) + { + string reference = Normalize(references[i], caseSensitive, alphanumericOnly); + string hypothesis = Normalize(hypotheses[i], caseSensitive, alphanumericOnly); + if (string.Equals(reference, hypothesis, StringComparison.Ordinal)) + { + matched++; + } + } + + return matched / (double)references.Count; + } + + /// + /// Applies the benchmark normalization used by . + /// + /// The text to normalize. Null is treated as empty. + /// When false the text is lower-cased. + /// When true non-alphanumeric characters are removed. + /// The normalized text. + public static string Normalize(string? value, bool caseSensitive = false, bool alphanumericOnly = true) + { + string text = value ?? string.Empty; + if (!caseSensitive) + { + text = text.ToLowerInvariant(); + } + + if (!alphanumericOnly) + { + return text; + } + + var builder = new StringBuilder(text.Length); + foreach (char c in text) + { + if (char.IsLetterOrDigit(c)) + { + builder.Append(c); + } + } + + return builder.ToString(); + } + + private static List Tokenize(string? value) + { + var tokens = new List(); + if (string.IsNullOrEmpty(value)) + { + return tokens; + } + + foreach (string token in value!.Split((char[]?)null, StringSplitOptions.RemoveEmptyEntries)) + { + tokens.Add(token); + } + + return tokens; + } + + private static void ValidateAligned(IReadOnlyList references, IReadOnlyList hypotheses) + { + if (references is null) + { + throw new ArgumentNullException(nameof(references)); + } + + if (hypotheses is null) + { + throw new ArgumentNullException(nameof(hypotheses)); + } + + if (references.Count != hypotheses.Count) + { + throw new ArgumentException( + $"There are {references.Count} references but {hypotheses.Count} hypotheses. " + + "Both lists must be indexed by the same sample order.", + nameof(references)); + } + } +} diff --git a/src/Models/ModelBase.cs b/src/Models/ModelBase.cs index 9dfdfec49b..4086a233b8 100644 --- a/src/Models/ModelBase.cs +++ b/src/Models/ModelBase.cs @@ -311,7 +311,17 @@ public virtual bool SupportsParameterInitialization /// without another per-model override. /// public virtual IEnumerable> GetParameterStateChunks() - => _parameterRegistry.GetParameterStateChunks(); + { + // Components are registered lazily, on first access through Components. Every other + // parameter surface - GetParameters, SetParameters, ParameterCount, ParameterLayout - goes + // through it; this one went straight to the registry, so a caller that enumerated chunks + // BEFORE anything else had touched the parameters saw an empty registry and got no chunks + // at all, while GetParameters on the same model kept working. A tape-based trainer or a + // chunk-based optimizer enumerates chunks first. Not an iterator on purpose: registration + // must happen at the call, not whenever the sequence is first enumerated. + _ = Components; + return _parameterRegistry.GetParameterStateChunks(); + } /// public virtual IEnumerable> GetParameterChunks() @@ -347,6 +357,7 @@ public virtual IFullModel DeepCopy() { byte[] state = Serialize(); var copy = (ModelBase)AiDotNet.Models.CloneEngine.CopyConfiguration(this); + PrepareCopyForStateRestore(copy); AiDotNet.Models.CloneEngine.PrepareParameterTopology( this, copy, @@ -358,6 +369,22 @@ public virtual IFullModel DeepCopy() } } + /// + /// Called by after the copy has been rebuilt from its recorded + /// constructor and before this model's state is loaded into it. + /// + /// The freshly rebuilt copy. + /// + /// A model built from lazily-shaped layers - layers that size their weights on their first + /// forward pass - has, once used, more parameters than the freshly rebuilt copy, so loading the + /// state fails on a parameter-count mismatch. Override this to bring the copy to the same + /// parameter topology first, typically by running it once on an input of the shape this model + /// has already seen. The default does nothing. + /// + protected virtual void PrepareCopyForStateRestore(ModelBase copy) + { + } + /// public virtual IFullModel Clone() => DeepCopy(); diff --git a/src/Models/Options/ObjectDetectionOptions.cs b/src/Models/Options/ObjectDetectionOptions.cs index 24c8029a38..455ed4864e 100644 --- a/src/Models/Options/ObjectDetectionOptions.cs +++ b/src/Models/Options/ObjectDetectionOptions.cs @@ -130,6 +130,40 @@ public class ObjectDetectionOptions : ModelOptions /// Random seed for reproducibility. /// public int? RandomSeed { get; set; } = 42; + + /// + /// Set prediction loss used by TrainDetections on DETR-family detectors, or null for the + /// detector's published recipe (DETR, DINO or RT-DETR). + /// + /// + /// For Beginners: Leave this empty to train with the settings from the model's paper. + /// Set it to change the matching costs, loss weights or focal parameters. The classification form + /// must match the detector's class head: softmax for DETR, sigmoid focal or varifocal for DINO and + /// RT-DETR. + /// + public AiDotNet.ComputerVision.Detection.Losses.DetrSetLossOptions? SetPredictionLoss { get; set; } + + /// + /// Task-aligned assignment and loss used by TrainDetections on anchor-free YOLO detectors + /// (YOLOv8, YOLOv9, YOLOv10, YOLOv11), or null for the published defaults. + /// + /// + /// For Beginners: Leave this empty to train with the published YOLO settings. Set it to + /// change how many grid cells learn from each object or how strongly boxes, classes and box-edge + /// distributions are corrected. + /// + public AiDotNet.ComputerVision.Detection.Losses.TaskAlignedLossOptions? TaskAlignedLoss { get; set; } + + /// + /// Proposal and region-of-interest sampling and loss weights used by TrainDetections on Faster R-CNN and + /// Cascade R-CNN, or null for the published defaults. + /// + /// + /// For Beginners: Leave this empty to train with the settings from the Faster R-CNN, Fast R-CNN and + /// Cascade R-CNN papers. Set it to change which proposals count as objects or background during training, how + /// many are sampled, or how strongly boxes are corrected. + /// + public AiDotNet.ComputerVision.Detection.Losses.TwoStageDetectionLossOptions? TwoStageLoss { get; set; } } /// diff --git a/src/Models/Parameters/ModelParameterSources.cs b/src/Models/Parameters/ModelParameterSources.cs index 97f2521eff..07805a72b7 100644 --- a/src/Models/Parameters/ModelParameterSources.cs +++ b/src/Models/Parameters/ModelParameterSources.cs @@ -427,15 +427,36 @@ public void SetParameters(Vector parameters) /// lists are given and, within each list, in index order. /// /// +/// /// For models that hold weights as bare List<Tensor<T>> rather than layers -- a /// feature-pyramid neck keeps a lateral weight and bias per level, and an output pair per level. /// The tensors are written THROUGH, never replaced, so a restore reaches the same instances the /// forward pass reads. +/// +/// +/// The lists are also exposed as LIVE chunks, one per tensor. Without that the registry could only +/// hand an optimizer a flat copy of these weights, and a tape-based training step - which keys +/// gradients by tensor reference - had nothing it could update in place, so every neck weight stayed +/// at its initial value. +/// /// -public sealed class TensorListParameterSource : IParameterSource +public sealed class TensorListParameterSource : IParameterSource, IParameterChunkSource, IParameterLayoutSource { private readonly Func>>[] _lists; + /// + public IReadOnlyList GetParameterLayout() + { + var slots = new List(); + foreach (var chunk in GetParameterStateChunks()) + { + slots.Add(new ParameterSlotDescriptor( + chunk.StableId, chunk.Role, ParameterReadiness.Materialized, chunk.Tensor.Length, + shape: chunk.Tensor.Shape.ToArray(), elementType: typeof(T).FullName)); + } + return slots; + } + /// Creates a source over the given tensor lists, in order. public TensorListParameterSource(params Func>>[] lists) { @@ -493,4 +514,20 @@ public void SetParameters(Vector parameters) for (int i = 0; i < t.Length; i++) t[i] = parameters[idx++]; } } + + /// + public IEnumerable> GetParameterStateChunks() + { + for (int list = 0; list < _lists.Length; list++) + { + var items = _lists[list](); + if (items is null) continue; + for (int i = 0; i < items.Count; i++) + { + var tensor = items[i]; + if (tensor is null || tensor.Length == 0) continue; + yield return new ParameterChunk($"{list}.{i}", ParameterSlotRole.Trainable, tensor); + } + } + } } diff --git a/src/Models/Parameters/ParameterComponentRegistry.cs b/src/Models/Parameters/ParameterComponentRegistry.cs index 8ef6dba464..c4cee5b6de 100644 --- a/src/Models/Parameters/ParameterComponentRegistry.cs +++ b/src/Models/Parameters/ParameterComponentRegistry.cs @@ -351,10 +351,11 @@ public IEnumerable> GetParameterStateChunks() int expected = checked((int)item.ParameterCount!.Value); if (expected == 0) continue; - if (source is IParameterChunkSource chunkSource) + var liveChunks = LiveChunksOf(source); + if (liveChunks is not null) { int actual = 0; - foreach (var chunk in chunkSource.GetParameterStateChunks()) + foreach (var chunk in liveChunks) { if (chunk is null || chunk.Tensor.Length == 0) continue; actual = checked(actual + chunk.Tensor.Length); @@ -364,7 +365,7 @@ public IEnumerable> GetParameterStateChunks() string localId = chunk.StableId == "$" ? entry.StableId : entry.StableId + "/" + chunk.StableId; - yield return new ParameterChunk(localId, role, chunk.Tensor, chunk.SourceTensor); + yield return new ParameterChunk(localId, role, chunk.Tensor, chunk.SourceTensor, chunk.IsWritableInPlace); } if (actual != expected) throw new ParameterContractViolationException( @@ -401,7 +402,8 @@ public IEnumerable> GetParameterStateChunks() : entry.StableId + "/" + slot.StableId; var role = entry.Role == ParameterSlotRole.Trainable ? slot.Role : entry.Role; yield return new ParameterChunk( - localId, role, new Tensor(new[] { count }, values)); + localId, role, new Tensor(new[] { count }, values), + sourceTensor: null, writableInPlace: false); offset += count; } if (offset != flat.Length) @@ -415,7 +417,8 @@ public IEnumerable> GetParameterStateChunks() // above supplies the model's real backing tensor; this fallback is the explicit, // immutable-payload style used by scalar/tree/classical sources. yield return new ParameterChunk(entry.StableId, entry.Role, - new Tensor(new[] { flat.Length }, flat)); + new Tensor(new[] { flat.Length }, flat), + sourceTensor: null, writableInPlace: false); } } @@ -1062,4 +1065,70 @@ private static int SkipLeadingZeros(string value, int start, int end) while (i < end - 1 && value[i] == '0') i++; return i; } + + /// + /// The live chunks of a registered source, seeing through the generated component adapters; or + /// null when the source can only be read as a flat copy. + /// + /// + /// + /// The parameter generator registers a component member through + /// (one component) or + /// (a collection). Neither adapter is a chunk + /// source, so every component registered that way used to be enumerated as a detached COPY, + /// even when the component itself exposed live, zero-copy chunks - a layer, a network, a + /// computer-vision building block. A tape-based training step keys gradients by tensor + /// reference and can only update live tensors, so everything behind those adapters was + /// silently untrainable through the registry. + /// + /// + /// The adapters are only seen through when the component (or every collection member) is itself + /// a chunk source; anything else keeps the per-slot copy path unchanged. Stable ids follow the + /// adapters' own layout scheme - an accessor passes its component's ids through, a collection + /// prefixes each member's with index=NNNNNNNN - so the chunk ids match the layout either way. + /// + /// + private static IEnumerable>? LiveChunksOf(IParameterSource source) + { + switch (source) + { + case IParameterChunkSource chunked: + return chunked.GetParameterStateChunks(); + + case ComponentAccessorParameterSource accessor: + return accessor.Current is IParameterChunkSource component + && component is IParameterLayoutSource or IParameterManifestProvider + ? component.GetParameterStateChunks() + : null; + + case ComponentCollectionParameterSource collection: + var members = collection.Current.ToList(); + foreach (var member in members) + { + if (member is not IParameterChunkSource + || member is not (IParameterLayoutSource or IParameterManifestProvider)) + { + return null; + } + } + + return CollectionChunks(members); + + default: + return null; + } + } + + private static IEnumerable> CollectionChunks(List> members) + { + for (int index = 0; index < members.Count; index++) + { + string prefix = $"index={index:D8}"; + foreach (var chunk in ((IParameterChunkSource)members[index]).GetParameterStateChunks()) + { + string id = chunk.StableId == "$" ? prefix : prefix + "/" + chunk.StableId; + yield return new ParameterChunk(id, chunk.Role, chunk.Tensor, chunk.SourceTensor, chunk.IsWritableInPlace); + } + } + } } diff --git a/src/NeuralNetworks/Layers/LayerBase.cs b/src/NeuralNetworks/Layers/LayerBase.cs index 5ac9e7098c..105e5d2240 100644 --- a/src/NeuralNetworks/Layers/LayerBase.cs +++ b/src/NeuralNetworks/Layers/LayerBase.cs @@ -1499,6 +1499,13 @@ protected static void ResolveAndMaterialize(LayerBase? child, int[] inputShap /// protected virtual bool HasDeclaredSubLayerStructure => false; + /// + /// Whether the lazy initializer may construct declared child modules. The generator may skip + /// that initializer for an exact runtime type only after proving its optional children are + /// untouched by every reachable owner call. Unknown and inherited paths remain conservative. + /// + protected virtual bool NeedsDeclaredSubLayerInitialization => true; + /// /// Wraps dimensions as a without copying them. /// @@ -1922,7 +1929,7 @@ private bool TryGetDeclaredParameterCount(out long count, out bool materialized) /// private void EnsureDeclaredSubLayerStructure() { - if (!HasDeclaredSubLayerStructure) return; + if (!HasDeclaredSubLayerStructure || !NeedsDeclaredSubLayerInitialization) return; if (!IsShapeResolved && !ParametersAreConstructionSized) return; bool wasResolvingShapesOnly = IsResolvingShapesOnly; diff --git a/tests/AiDotNet.Tests/AiDotNetTests.csproj b/tests/AiDotNet.Tests/AiDotNetTests.csproj index b5440537d4..78d94150ce 100644 --- a/tests/AiDotNet.Tests/AiDotNetTests.csproj +++ b/tests/AiDotNet.Tests/AiDotNetTests.csproj @@ -92,6 +92,7 @@ intentionally does not expose generator types as a compile reference. --> +