First version of experiment framework

.github/workflows/fp-arena-ci.yml

@@ -53,7 +53,7 @@ jobs:
         # FP-Arena without deps so it uses the DaCe just installed (any main),
         # rather than re-pulling pinned main from git.
         pip install --no-deps ./fp-arena
-        pip install pytest
+        pip install pytest scipy tqdm
 
     - name: Run FP-Arena tests
       run: |

.gitignore

@@ -22,3 +22,7 @@ out.sdfg
 .idea/
 *.swp
 .DS_Store
+
+
+#Experiment results database
+*.db

corpus/__init__.py

@@ -0,0 +1,2 @@
+# Copyright 2019-2026 ETH Zurich and the FP-Arena authors. All rights reserved.
+"""A corpus of benchmark kernels written in the DaCe Python frontend."""

examples/heat3d.py → corpus/heat3d.py

@@ -1,20 +1,18 @@
 # Copyright 2019-2026 ETH Zurich and the FP-Arena authors. All rights reserved.
-"""Run the heat3d stencil in reduced precision."""
+"""
+The heat3d stencil kernel (Polybench), a 3D Jacobi-style heat-diffusion step.
+"""
 
 import dace as dc
-import numpy as np
-
-from fp_arena.transformations.change_and_propagate_fp_types import (
-    change_and_propagate_fp_types,
-)
 
+# Default problem size: an N x N x N grid advanced over TSTEPS time steps.
 GRID_N, TSTEPS = 40, 20
 
 N = dc.symbol("N", dtype=dc.int64)
 
 
 @dc.program
-def kernel(TSTEPS: dc.int64, A: dc.float64[N, N, N], B: dc.float64[N, N, N]):
+def heat3d_kernel(TSTEPS: dc.int64, A: dc.float64[N, N, N], B: dc.float64[N, N, N]):
     for t in range(1, TSTEPS):
         B[1:-1, 1:-1, 1:-1] = (
             0.125 * (A[2:, 1:-1, 1:-1] - 2.0 * A[1:-1, 1:-1, 1:-1] + A[:-2, 1:-1, 1:-1])
@@ -32,22 +30,3 @@ def kernel(TSTEPS: dc.int64, A: dc.float64[N, N, N], B: dc.float64[N, N, N]):
             * (B[1:-1, 1:-1, 2:] - 2.0 * B[1:-1, 1:-1, 1:-1] + B[1:-1, 1:-1, :-2])
             + B[1:-1, 1:-1, 1:-1]
         )
-
-
-def main():
-    sdfg = kernel.to_sdfg(simplify=True)
-
-    change_and_propagate_fp_types(
-        sdfg,
-        {"A": dc.float16, "B": dc.float16},
-        constant_type=dc.float16,
-    )
-
-    rng = np.random.default_rng(0)
-    A = rng.uniform(0, 100, (GRID_N,) * 3)
-    B = rng.uniform(0, 100, (GRID_N,) * 3)
-    sdfg(TSTEPS=TSTEPS, A=A, B=B, N=GRID_N)
-
-
-if __name__ == "__main__":
-    main()

examples/experiment_error.py

@@ -0,0 +1,55 @@
+# Copyright 2019-2026 ETH Zurich and the FP-Arena authors. All rights reserved.
+"""Measure the heat3d stencil's error in reduced precision against an fp64 reference."""
+
+from scipy import stats
+
+from corpus.heat3d import GRID_N, TSTEPS, heat3d_kernel
+from fp_arena.experiment import (
+    ErrorAnalysisConfig,
+    ExperimentConfig,
+    ResultStore,
+    run_error,
+)
+
+N_SAMPLES = 3
+
+
+def main():
+    experiment = ExperimentConfig(
+        name="heat3d",
+        program=heat3d_kernel.to_sdfg(simplify=True),
+        symbols={"N": GRID_N},
+        scalar_args={"TSTEPS": TSTEPS},
+        target="cpu",
+        inputs={
+            "A": stats.uniform(0, 100),
+            "B": stats.uniform(0, 100),
+        },
+    )
+
+    store = ResultStore("heat3d_results.db")
+    results = run_error(
+        ErrorAnalysisConfig(
+            experiment,
+            precisions=[{"A": "fp32", "B": "fp32"}, {"A": "fp16", "B": "fp16"}],
+            reference="fp64",
+            n_samples=N_SAMPLES,
+        ),
+        store=store,
+    )
+
+    for r in results:
+        label = " ".join(f"{k}={v}" for k, v in r.precision.items())
+        for arr in sorted(r.errors):
+            e = r.errors[arr]
+            print(
+                f"{label:>16} {arr}: rel_mean {e.rel_mean:.3e}  "
+                f"rel_max {e.rel_max:.3e}  linf {e.linf:.3e}"
+            )
+
+    print(f"{len(store.query(experiment='heat3d', kind='error'))} rows in the database")
+    store.close()
+
+
+if __name__ == "__main__":
+    main()

examples/experiment_performance.py

@@ -0,0 +1,54 @@
+# Copyright 2019-2026 ETH Zurich and the FP-Arena authors. All rights reserved.
+"""Time the heat3d stencil across precision points with the experiment framework."""
+
+import statistics
+
+from scipy import stats
+
+from corpus.heat3d import GRID_N, TSTEPS, heat3d_kernel
+from fp_arena.experiment import (
+    ExperimentConfig,
+    PerformanceAnalysisConfig,
+    ResultStore,
+    run_performance,
+)
+
+
+def main():
+    experiment = ExperimentConfig(
+        name="heat3d",
+        program=heat3d_kernel.to_sdfg(simplify=True),
+        symbols={"N": GRID_N},
+        scalar_args={"TSTEPS": TSTEPS},
+        target="cpu",
+        inputs={
+            "A": stats.uniform(0, 100),
+            "B": stats.uniform(0, 100),
+        },
+    )
+
+    store = ResultStore("heat3d_results.db")
+    results = run_performance(
+        PerformanceAnalysisConfig(
+            experiment,
+            precisions=[
+                {},  # the unmodified fp64 program
+                {"A": "fp32", "B": "fp32"},
+                {"A": "fp16", "B": "fp16"},
+            ],
+            n_warmup=1,
+            n_reps=5,
+        ),
+        store=store,
+    )
+
+    for r in results:
+        label = " ".join(f"{k}={v}" for k, v in r.precision.items()) or "fp64 baseline"
+        print(
+            f"{label:>16}: total {statistics.median(r.total_times):8.3f} ms  "
+            f"kernel {statistics.median(r.kernel_times):8.3f} ms"
+        )
+
+
+if __name__ == "__main__":
+    main()

examples/experiment_perturbation.py

@@ -0,0 +1,50 @@
+# Copyright 2019-2026 ETH Zurich and the FP-Arena authors. All rights reserved.
+"""Measure the heat3d stencil's sensitivity to input noise with the experiment framework."""
+
+from scipy import stats
+
+from corpus.heat3d import GRID_N, TSTEPS, heat3d_kernel
+from fp_arena.experiment import (
+    ExperimentConfig,
+    Noise,
+    PerturbationAnalysisConfig,
+    ResultStore,
+    run_perturbation,
+)
+
+
+def main():
+    experiment = ExperimentConfig(
+        name="heat3d",
+        program=heat3d_kernel.to_sdfg(simplify=True),
+        symbols={"N": GRID_N},
+        scalar_args={"TSTEPS": TSTEPS},
+        target="cpu",
+        inputs={
+            "A": stats.uniform(0, 100),
+            "B": stats.uniform(0, 100),
+        },
+    )
+
+    noise = Noise(relative=1e-3, relative_dist=stats.uniform(-1.0, 2.0))
+    store = ResultStore("heat3d_results.db")
+    results = run_perturbation(
+        PerturbationAnalysisConfig(
+            experiment,
+            noise={"A": noise, "B": noise},
+            precisions=[{}],  # the unmodified fp64 program
+        ),
+        store=store,
+    )
+
+    for r in results:
+        for arr in sorted(r.errors):
+            e = r.errors[arr]
+            print(
+                f"perturbed {r.perturbed} -> {arr}: rel_mean {e.rel_mean:.3e}  "
+                f"rel_max {e.rel_max:.3e}"
+            )
+
+
+if __name__ == "__main__":
+    main()

examples/reduced_precision.py

@@ -0,0 +1,29 @@
+# Copyright 2019-2026 ETH Zurich and the FP-Arena authors. All rights reserved.
+"""Run the heat3d stencil in reduced precision."""
+
+import dace as dc
+import numpy as np
+
+from corpus.heat3d import GRID_N, TSTEPS, heat3d_kernel
+from fp_arena.transformations.change_and_propagate_fp_types import (
+    change_and_propagate_fp_types,
+)
+
+
+def main():
+    sdfg = heat3d_kernel.to_sdfg(simplify=True)
+
+    change_and_propagate_fp_types(
+        sdfg,
+        {"A": dc.float16, "B": dc.float16},
+        constant_type=dc.float16,
+    )
+
+    rng = np.random.default_rng(0)
+    A = rng.uniform(0, 100, (GRID_N,) * 3)
+    B = rng.uniform(0, 100, (GRID_N,) * 3)
+    sdfg(TSTEPS=TSTEPS, A=A, B=B, N=GRID_N)
+
+
+if __name__ == "__main__":
+    main()

fp_arena/experiment/__init__.py

@@ -0,0 +1,41 @@
+# Copyright 2019-2026 ETH Zurich and the FP-Arena authors. All rights reserved.
+"""
+FP-Arena experiment framework: precision performance and error analysis.
+"""
+
+from fp_arena.experiment.config import (
+    ErrorAnalysisConfig,
+    ExperimentConfig,
+    PerformanceAnalysisConfig,
+    PerturbationAnalysisConfig,
+    PrecisionMap,
+)
+from fp_arena.experiment.results import (
+    ErrorResult,
+    ErrorStats,
+    PerfResult,
+    PerturbationResult,
+)
+from fp_arena.experiment.inputs import Noise
+from fp_arena.experiment import registry
+from fp_arena.experiment.runner import run_error, run_performance, run_perturbation
+from fp_arena.experiment.store import ResultStore, StoredResult
+
+__all__ = [
+    "ExperimentConfig",
+    "PerformanceAnalysisConfig",
+    "ErrorAnalysisConfig",
+    "PerturbationAnalysisConfig",
+    "PrecisionMap",
+    "PerfResult",
+    "ErrorResult",
+    "ErrorStats",
+    "PerturbationResult",
+    "Noise",
+    "run_performance",
+    "run_error",
+    "run_perturbation",
+    "registry",
+    "ResultStore",
+    "StoredResult",
+]

fp_arena/experiment/config.py

@@ -0,0 +1,87 @@
+# Copyright 2019-2026 ETH Zurich and the FP-Arena authors. All rights reserved.
+"""
+Experiment config.
+"""
+
+from dataclasses import dataclass, field
+from typing import Any, Dict, FrozenSet, List, Optional, Union
+
+import dace
+
+from fp_arena.experiment.inputs import DistributionLike, Noise
+
+#: ``{array_name: precision_key}`` -- pins a subset of arrays to a target format.
+PrecisionMap = Dict[str, str]
+
+
+@dataclass
+class ExperimentConfig:
+    """
+    The program under test plus everything needed to run it -- shared by all analyses.
+
+    :param name: identifier used to group results in the database.
+    :param program: the ``dace.SDFG`` under test
+    :param inputs: per-array input distributions for the *read* arrays.
+    :param promotion_rules: promotion rules for resolving precision conflicts (e.g. ``{frozenset({fp16, fp32}): fp32}``); ``None`` uses ``DEFAULT_PROMOTION_RULES``.
+    :param symbols: values for the SDFG's free symbols (e.g. ``{"N": 100}``);
+    :param scalar_args: values for non-array scalar arguments.
+    :param target: ``"cpu"`` (default) or ``"gpu"``;
+    :param seed: base RNG seed for inputs and noise, shared across analyses.
+    :param gpu_block_size: GPU thread-block size ``[x, y, z]`` (x = contiguous dim) set on every GPU_Device map; ``None`` uses DaCe's default.
+    """
+
+    name: str
+    program: dace.SDFG
+    inputs: Dict[str, DistributionLike] = field(default_factory=dict)
+    promotion_rules: Optional[
+        Dict[FrozenSet[dace.dtypes.typeclass], dace.dtypes.typeclass]
+    ] = None
+    symbols: Dict[str, int] = field(default_factory=dict)
+    scalar_args: Dict[str, Any] = field(default_factory=dict)
+    target: str = "cpu"
+    seed: int = 0
+    gpu_block_size: Optional[List[int]] = None
+
+
+@dataclass
+class PerformanceAnalysisConfig:
+    """
+    Measure wall-clock runtime across precision points.
+    """
+
+    experiment: ExperimentConfig
+    precisions: List[PrecisionMap]
+    noise: Dict[str, Noise] = field(default_factory=dict)
+    n_warmup: int = 1
+    n_reps: int = 10
+
+
+@dataclass
+class ErrorAnalysisConfig:
+    """
+    Measure per-array error of each precision point against a high-precision
+    reference, aggregated over ``n_samples`` input realisations.
+    ``reference`` is a single key (e.g. ``"mpfr128"``, ``"fp64"``) or a per-array ``{name: key}`` map.
+    """
+
+    experiment: ExperimentConfig
+    precisions: List[PrecisionMap]
+    noise: Dict[str, Noise] = field(default_factory=dict)
+    reference: Union[str, Dict[str, str]] = "fp64"
+    n_samples: int = 1
+
+
+@dataclass
+class PerturbationAnalysisConfig:
+    """
+    Measure input sensitivity: perturb one input array at a time and compare
+    each written array against the clean run at the same precision point,
+    aggregated over ``n_samples`` input realisations.
+    ``noise`` names the inputs to perturb (each analysed separately);
+    ``precisions`` lists the points to analyse at (default: the unmodified program).
+    """
+
+    experiment: ExperimentConfig
+    noise: Dict[str, Noise]
+    precisions: List[PrecisionMap] = field(default_factory=lambda: [{}])
+    n_samples: int = 1