test(sweep): Sourcify corpus sweep harness for the Slang frontend

.github/workflows/sourcify-sweep.yaml

@@ -0,0 +1,205 @@
+name: Sourcify Sweep
+
+# Runs `solx-dev test sourcify` over the pinned Sourcify corpus with the slang
+# build of the commit, one shard per job, and reports the outcome table plus
+# the ranked failure census. Runs on every push to main, and on PR pushes while
+# the PR carries the `ci:sourcify-sweep` label (adding the label starts a run).
+# Report-only: frontend failures never fail the run, only harness breakage does.
+#
+# The corpus release and both compilers are pinned in solx-dev/sourcify-tests.toml.
+
+on:
+  pull_request:
+    types: [opened, synchronize, reopened, labeled]
+  push:
+    branches:
+      - main
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  # Pushes group by SHA: GitHub keeps one pending run per group, so a
+  # shared main group would drop queued pushes.
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
+defaults:
+  run:
+    shell: bash -ex {0}
+
+env:
+  SHARD_COUNT: 24
+
+jobs:
+
+  label-check:
+    if: >-
+      github.event_name != 'pull_request'
+      || (contains(github.event.pull_request.labels.*.name, 'ci:sourcify-sweep')
+        && (github.event.action != 'labeled' || github.event.label.name == 'ci:sourcify-sweep')
+        && !github.event.pull_request.head.repo.fork)
+    runs-on: ubuntu-24.04
+    steps:
+      - run: 'true'
+
+  cooldown-check:
+    name: Cargo cooldown check
+    runs-on: ubuntu-24.04
+    needs: label-check
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          ref: ${{ github.event.pull_request.head.sha || github.sha }}
+          persist-credentials: false
+      - uses: ./.github/actions/cooldown-check
+
+  sweep:
+    name: Sweep shard ${{ matrix.shard }}
+    needs: [label-check, cooldown-check]
+    permissions:
+      contents: read
+      packages: read
+    runs-on: ubuntu-24.04
+    container:
+      image: ghcr.io/nomicfoundation/solx-ci-runner@sha256:cd5a37f2630fdf1898ddb2ca11f8c2ecc4d572c60011fe923b27b1625f92ba15
+      # Same host bind-mounts as test.yaml's cargo-checks job so the
+      # free-disk-space-linux action can reclaim room for the LLVM tree.
+      volumes:
+        - /usr/share/dotnet:/mnt/free-disk-space/dotnet
+        - /usr/local/lib/android:/mnt/free-disk-space/android
+        - /opt/ghc:/mnt/free-disk-space/ghc
+        - /usr/local/.ghcup:/mnt/free-disk-space/ghcup
+        - /opt/hostedtoolcache/CodeQL:/mnt/free-disk-space/codeql
+    # Warm caches: 12-18 min per shard, build included. A cold LLVM build
+    # (solx-llvm bump on the PR) adds well over an hour.
+    timeout-minutes: 180
+    env:
+      CARGO_INCREMENTAL: "0"
+      SHARD: ${{ matrix.shard }}
+    strategy:
+      fail-fast: false
+      matrix:
+        # Must stay in sync with SHARD_COUNT above.
+        shard: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          ref: ${{ github.event.pull_request.head.sha || github.sha }}
+          submodules: true
+          persist-credentials: false
+
+      - name: Free disk space (Linux)
+        uses: ./.github/actions/free-disk-space-linux
+
+      # Standard toolchain configuration (RelWithDebInfo + assertions + MLIR):
+      # the LLVM cache that cache-warmup.yaml keeps warm on hosted runners.
+      - name: Build toolchain
+        uses: ./.github/actions/build-toolchain
+
+      - name: Cache cargo artifacts
+        uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
+        with:
+          prefix-key: sourcify-sweep-v1
+          cache-on-failure: true
+          # Every shard builds the same artifacts; one saver is enough.
+          save-if: ${{ matrix.shard == 0 }}
+
+      - name: Build slang and solx-dev
+        run: |
+          ${SFW_PREFIX:-} cargo fetch --locked
+          ${SFW_PREFIX:-} cargo build --frozen --release --bin slang --bin solx-dev
+          ./target/release/slang --version
+
+      # The sweep needs only the two binaries; the LLVM tree and the rest of
+      # the target dir go so the corpus slice and 4 concurrent compiler
+      # processes have room.
+      - name: Free disk space (remove build artifacts)
+        run: |
+          df -h .
+          mv target/release/slang target/release/solx-dev "${RUNNER_TEMP}/"
+          rm -rf solx-llvm target-llvm target
+          mkdir -p target/release
+          mv "${RUNNER_TEMP}/slang" "${RUNNER_TEMP}/solx-dev" target/release/
+          df -h .
+
+      - name: Run the sweep shard
+        run: |
+          ./target/release/solx-dev test sourcify \
+            --corpus-dir "${RUNNER_TEMP}/corpus" \
+            --output-path "${RUNNER_TEMP}/results-shard-${SHARD}.jsonl" \
+            --shard-count "${SHARD_COUNT}" \
+            --shard-index "${SHARD}" \
+            --threads 4 \
+            --memory-limit-mib 6144
+
+      - name: Upload shard results
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: sourcify-sweep-results-shard-${{ matrix.shard }}
+          path: ${{ runner.temp }}/results-shard-${{ matrix.shard }}.jsonl
+          if-no-files-found: error
+
+      - name: Upload solx-dev
+        if: matrix.shard == 0
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: sourcify-sweep-solx-dev
+          path: target/release/solx-dev
+          if-no-files-found: error
+
+  summarize:
+    name: Summarize
+    needs: sweep
+    if: ${{ !cancelled() && needs.sweep.result != 'skipped' }}
+    permissions:
+      contents: read
+      pull-requests: write
+    runs-on: ubuntu-24.04
+    timeout-minutes: 20
+    steps:
+      - name: Download solx-dev
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          name: sourcify-sweep-solx-dev
+          path: ${{ runner.temp }}/bin
+
+      - name: Download shard results
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          pattern: sourcify-sweep-results-shard-*
+          path: ${{ runner.temp }}/shards
+          merge-multiple: true
+
+      - name: Combine shards and report
+        run: |
+          shards=$(find "${RUNNER_TEMP}/shards" -name 'results-shard-*.jsonl' | wc -l)
+          if [ "${shards}" -ne "${SHARD_COUNT}" ]; then
+            echo "Expected ${SHARD_COUNT} shard result files, found ${shards}." >&2
+            exit 1
+          fi
+          cat "${RUNNER_TEMP}"/shards/results-shard-*.jsonl > "${RUNNER_TEMP}/results.jsonl"
+          # Artifacts drop the executable bit.
+          chmod +x "${RUNNER_TEMP}/bin/solx-dev"
+          "${RUNNER_TEMP}/bin/solx-dev" test sourcify-report \
+            --results-paths "${RUNNER_TEMP}/results.jsonl" \
+            --output-path "${RUNNER_TEMP}/report.md" \
+            --title "Sourcify sweep (slang vs latest solx release)"
+          cat "${RUNNER_TEMP}/report.md" >> "${GITHUB_STEP_SUMMARY}"
+
+      - name: Post report comment
+        if: github.event_name == 'pull_request'
+        uses: mshick/add-pr-comment@ec328af66588ab8f77cdeb2c264f14aba45bbf59 # v3.12
+        with:
+          message-id: 'sourcify-sweep-report'
+          message-path: ${{ runner.temp }}/report.md
+
+      - name: Upload combined results
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: sourcify-sweep-results
+          path: |
+            ${{ runner.temp }}/results.jsonl
+            ${{ runner.temp }}/report.md

docs/src/developer-guide/01-testing.md

@@ -90,6 +90,32 @@ cargo build --release --bin solx-dev
 
 The test configurations list projects that are cloned and tested automatically. See [foundry-tests.toml](https://github.com/NomicFoundation/solx/blob/main/solx-dev/foundry-tests.toml) and [hardhat-tests.toml](https://github.com/NomicFoundation/solx/blob/main/solx-dev/hardhat-tests.toml) for the full list of tested projects.
 
+## Sourcify Corpus
+
+The Sourcify sweep compiles every verified Sourcify contract for solc 0.8.34 and 0.8.36 with `evmVersion` at least `cancun` (119,117 contracts) through `slang --standard-json`, and re-runs each failure with the latest solx release, so a `slang` gap is told apart from input neither compiles:
+
+```shell
+cargo build --release --bin slang --bin solx-dev
+
+# The 10 committed fixtures
+./target/release/solx-dev test sourcify --corpus-path tests/sourcify-sweep/fixtures
+./target/release/solx-dev test sourcify-report --results-paths temp-sourcify-reports/results.jsonl
+
+# One shard of the pinned corpus, as a CI job runs it
+./target/release/solx-dev test sourcify --shard-count 24 --shard-index 0 --memory-limit-mib 6144
+```
+
+[sourcify-tests.toml](https://github.com/NomicFoundation/solx/blob/main/solx-dev/sourcify-tests.toml) pins the corpus release and names the two compilers. Each contract becomes one standard JSON input with its sources inline, `evmVersion`, `libraries`, `remappings` and `viaIR` passed through verbatim, and bytecode as the only output. Every version pragma is rewritten to `pragma solidity >=<--solidity-version>;`, so both compilers accept the same input.
+
+| Outcome | Meaning |
+|---|---|
+| `ok` | No error diagnostics and bytecode for a contract of the target source |
+| `slang-fail` | `slang` fails and the latest solx release compiles |
+| `both-fail` | Both fail: a corpus artifact or a documented compiler limitation |
+| `timeout` | `slang` exceeded `--timeout` |
+
+The report ranks failures by normalized diagnostic, with example contracts. CI runs the sweep on every push to `main` and on pull requests labelled `ci:sourcify-sweep`, where the report becomes a pull request comment.
+
 ## Test Collection
 
 This section describes the format of test files used by **solx-tester**.

solx-compiler-downloader/src/lib.rs

@@ -191,7 +191,7 @@ impl Downloader {
     ///
     /// Downloads a file from a URL to a destination path, with resume support and retries.
     ///
-    fn download_file(&self, url: &str, destination: impl AsRef<Path>) -> anyhow::Result<()> {
+    pub fn download_file(&self, url: &str, destination: impl AsRef<Path>) -> anyhow::Result<()> {
         std::fs::create_dir_all(destination.as_ref().parent().expect("Always exists"))?;
 
         let destination = destination.as_ref();

solx-dev/Cargo.toml

@@ -31,3 +31,6 @@ sedregex = "0.2"
 solx-compiler-downloader = { path = "../solx-compiler-downloader" }
 solx-benchmark-converter = { path = "../solx-benchmark-converter" }
 solx-utils = { path = "../solx-utils" }
+
+[target.'cfg(unix)'.dependencies]
+libc = "0.2"

solx-dev/sourcify-tests.toml

@@ -0,0 +1,32 @@
+# Sourcify sweep configuration for `solx-dev test sourcify`.
+#
+# Corpus:
+#   [corpus]
+#   url = "https://..."             # Release asset unpacking to ./corpus.json + ./contracts/<chain>_<address>.json
+#   sha256 = "..."                  # SHA-256 of the release asset
+#
+# Compilers, as in hardhat-tests.toml: the candidate compiles every contract,
+# the reference re-runs the candidate's failures.
+#   [compilers.<id>]
+#   name = "slang"                  # Display name
+#   path = "./path/to/compiler"     # Path to compiler executable
+#   solidity_version = "0.8.36"     # Override --solidity-version for this compiler (optional)
+#   is_correctness_reference = true/false
+#   is_correctness_candidate = true/false
+
+# Every verified Sourcify contract for solc 0.8.34 and 0.8.36 with evmVersion >= cancun.
+[corpus]
+url = "https://github.com/nebasuke/sourcify-corpora/releases/download/sourcify-corpus-solc-0.8.34-0.8.36-20260920/sourcify-corpus-solc-0.8.34-0.8.36-20260920.tar.xz"
+sha256 = "0642e21776808674fb3fc50d352fc666dd7e8d062a322de51d44ab288fd0eb2e"
+
+[compilers.slang]
+name = "slang"
+path = "./target/release/slang"
+description = "slang built from the current commit"
+is_correctness_candidate = true
+
+[compilers.solx-latest]
+name = "solx-latest"
+path = "./temp-compilers/solx-latest"
+description = "solx downloaded from the latest release"
+is_correctness_reference = true

solx-dev/src/arguments/test/mod.rs

@@ -5,12 +5,15 @@
 pub mod foundry;
 pub mod hardhat;
 pub mod solx_tester;
+pub mod sourcify;
 
 use clap::Subcommand;
 
 pub use self::foundry::Foundry;
 pub use self::hardhat::Hardhat;
 pub use self::solx_tester::SolxTester;
+pub use self::sourcify::Sourcify;
+pub use self::sourcify::SourcifyReport;
 
 ///
 /// `solx` test arguments.
@@ -23,4 +26,8 @@ pub enum Test {
     Foundry(Foundry),
     /// Run integration tests with solx-tester.
     SolxTester(SolxTester),
+    /// Compile a Sourcify corpus and record the outcomes.
+    Sourcify(Sourcify),
+    /// Render the Sourcify sweep report from recorded outcomes.
+    SourcifyReport(SourcifyReport),
 }