Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 42 additions & 0 deletions .github/workflows/bench-checks.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
name: Benchmark checks

on:
pull_request:
paths:
- 'bench/**'
Comment thread
yslpn marked this conversation as resolved.
Resolved
- '.github/workflows/*.yml'
- '.github/actions/environment/**'
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
- 'pnpm-workspace.yaml'
- 'pnpm-lock.yaml'

permissions:
contents: read

jobs:
bench_prettier:
name: Run Prettier in bench
runs-on: ubuntu-latest
steps:
- name: Checkout repository
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- name: Setup environment
uses: ./.github/actions/environment
- name: Prettier check
run: pnpm format.check
working-directory: bench

bench_eslint:
name: Run ESLint in bench
runs-on: ubuntu-latest
steps:
- name: Checkout repository
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- name: Setup environment
uses: ./.github/actions/environment
- name: ESLint check
run: pnpm lint
working-directory: bench
38 changes: 38 additions & 0 deletions .github/workflows/bench.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
name: Benchmark

on:
pull_request:
paths:
- 'library/**'
- 'bench/**'
- '.github/workflows/bench.yml'
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
- '.github/actions/environment/**'

permissions:
contents: read

jobs:
bench:
name: Run benchmarks
runs-on: ubuntu-latest
steps:
- name: Checkout pull request base
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
ref: ${{ github.event.pull_request.base.sha }}
- name: Setup environment
uses: ./.github/actions/environment
- name: Benchmark pull request base
run: pnpm exec vitest bench --run representative --outputJson=/tmp/valibot-baseline.json
working-directory: bench
- name: Checkout pull request
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
ref: ${{ github.event.pull_request.head.sha }}
- name: Install pull request dependencies
run: pnpm install
- name: Compare pull request benchmarks
run: pnpm exec vitest bench --run representative --compare=/tmp/valibot-baseline.json
working-directory: bench
1 change: 1 addition & 0 deletions bench/.gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
/baseline.json
55 changes: 55 additions & 0 deletions bench/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
# Benchmarking

Benchmarks measure performance. Test correctness in the usual `library/src/`
tests.

## Structure

Benchmarks are grouped by workload purpose:

- `src/benchmarks/focused/` contains focused workloads. Use these to investigate a regression in a
particular scenario.
- `src/benchmarks/representative/` contains realistic combined workloads such
as `product`. It intentionally exercises nested schemas, arrays, pipes, and
multiple API operations, so its result is an overall signal rather than a
measurement attributable to one feature.

The file name identifies the workload and the directory identifies its
purpose. Keep schemas and inputs at the top of each benchmark file, outside
benchmark callbacks. Keep suite and benchmark names stable so Vitest can match
comparison results.

## Running

Run all benchmarks, selected workloads, or selected suites:

```sh
cd bench
pnpm bench
pnpm bench object
pnpm bench array
pnpm bench representative/product
pnpm bench --testNamePattern='safeParse'
pnpm bench url --testNamePattern='safeParse.*invalid'
```

Vitest matches file filters against benchmark paths, so a short name is usually
enough. Combine file and `--testNamePattern` filters as needed.

## Baselines

Create a local baseline from any branch, commit, or checkout. Use the same
filters when comparing it:

```sh
cd bench
pnpm bench:baseline url --testNamePattern='invalid'
pnpm bench:compare url --testNamePattern='invalid'
```

The baseline is saved to the ignored local file `baseline.json`. Recreate it
Comment thread
yslpn marked this conversation as resolved.
Resolved
after moving or renaming benchmark files.

CI runs the `representative` benchmark comparison automatically for pull
requests that change `library/` as a required check. Focused workloads remain
available for local investigation.
39 changes: 39 additions & 0 deletions bench/eslint.config.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
import eslint from '@eslint/js';
import importPlugin from 'eslint-plugin-import';
import tseslint from 'typescript-eslint';

export default tseslint.config(
{
ignores: ['eslint.config.js', 'vitest.config.ts'],
},
eslint.configs.recommended,
tseslint.configs.strict,
tseslint.configs.stylistic,
{
files: ['src/**/*.ts'],
extends: [importPlugin.flatConfigs.recommended],
languageOptions: {
parserOptions: {
project: './tsconfig.json',
tsconfigRootDir: import.meta.dirname,
},
},
settings: {
'import/parsers': {
'@typescript-eslint/parser': ['.ts'],
},
'import/resolver': {
typescript: {
project: './tsconfig.json',
},
},
},
rules: {
'@typescript-eslint/consistent-type-definitions': 'error',
'@typescript-eslint/consistent-type-imports': 'warn',
'import/extensions': ['error', 'always'],
'import/no-cycle': 'error',
'no-duplicate-imports': 'off',
},
}
);
24 changes: 24 additions & 0 deletions bench/package.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
{
"name": "@valibot/bench",
"private": true,
"type": "module",
"scripts": {
"bench": "vitest bench --run",

@ysknsid25 ysknsid25 Sep 6, 2026 •

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Valibot originally uses https://github.com/open-circle/schema-benchmarks for benchmarking, so wouldn't using Vitest's bench feature skew all the reference points?

The point is that the figures shown in the PR do not strictly match those published in the blog or documentation.

Also, Vitest benchmark is still experimental at the moment, so in that sense, it might be better to avoid it.

So before adding benchmark measurements to the CI, it might be better to first modify https://github.com/open-circle/schema-benchmarks so that it can be used within the CI environment 👀

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

so wouldn't using Vitest's bench feature skew all the reference points?

What do you mean?

The point is that the figures shown in the PR do not strictly match those published in the blog or documentation.

Why should they match?

Also, Vitest benchmark is still experimental at the moment, so in that sense, it might be better to avoid it.

No problem, this doesn't affect the runtime or the published library. There are no risks. Furthermore, in version 5, the Vitest API will be fully stabilized, with minor breaking changes described in the migration guide. https://vitest.dev/blog/vitest-5.html

So before adding benchmark measurements to the CI, it might be better to first modify https://github.com/open-circle/schema-benchmarks so that it can be used within the CI environment 👀

We discussed this with @EskiMojo14 and @fabian-hiller on Discord: schema-benchmarks can only measure published package versions. Furthermore, its goal is to benchmark various libraries that have different capabilities and are not interchangeable. Developers need a way to easily track function performance during development, even if other libraries lack similar validation features.

A key objective is also to make it easier to evaluate performance locally when testing new implementations and alternative approaches.

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

so wouldn't using Vitest's bench feature skew all the reference points?
The point is that the figures shown in the PR do not strictly match those published in the blog or documentation.

if we measure Valibot benchmarks using a different method than before, the benchmark figures won't strictly match those presented in the blog post below, and I am concerned this might give users the impression that we are benchmarking in an ambiguous manner.

https://valibot.dev/blog/why-migrate-to-valibot/#:~:text=This%20is%20one%20of%20Valibot%27s%20biggest%20strengths.%20On%20Schema%20Benchmarks%2C%20Valibot%20currently%20needs%20only%201.91%20kB%20gzipped%20whereas%20other%20schema%20libraries%20such%20as%20Zod%20v4%20require%2016.57%20kB.%20On%20the%20corresponding%20initialization%20benchmark%2C%20the%20same%20schema%20is%20initialized%20in%2054%20%CE%BCs%20and%20therefore%2016x%20faster%20than%20Zod%20v4

A key objective is also to make it easier to evaluate performance locally when testing new implementations and alternative approaches.

I see. So, if we were to add a method for local verification, is there any reason why adding a way to verify using schema-benchmarks wouldn't work? (Instead of using Vitest bench.)

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could we add a local verification mode to schema-benchmarks instead of using Vitest's bench?

Technically, yes, but it would require changing schema-benchmarks in two important ways.

First, schema-benchmarks is currently designed around a shared cross-library suite and published package versions. Local verification would require a separate mode for loading and benchmarking the unpublished source from the current PR.

Second, the shared suite intentionally focuses on scenarios that are comparable across libraries. Our internal suite also needs to cover Valibot-specific validations, transformations, and implementation alternatives that other libraries may not support. Adding those cases to the shared suite would either make them unavailable for other libraries or move schema-benchmarks away from its original purpose.

So I am not saying that a local schema-benchmarks mode is impossible. I am saying that it would be a separate feature with a different purpose, and it would not replace the need for an internal Valibot regression suite.

schema-benchmarks should remain the source for public cross-library comparisons, while this private suite is for testing unpublished Valibot code and tracking regressions during development. The goal of this PR is to provide that internal feedback loop without changing the public benchmark methodology.

"bench:baseline": "vitest bench --run --outputJson=baseline.json",
"bench:compare": "vitest bench --run --compare=baseline.json",
"lint": "eslint \"src/**/*.ts*\" && tsc --noEmit",
"format": "prettier --write ./src",
"format.check": "prettier --check ./src"
},
"devDependencies": {
"@eslint/js": "^9.39.1",
"eslint": "^9.39.1",
"eslint-import-resolver-typescript": "^4.4.4",
"eslint-plugin-import": "^2.32.0",
"prettier": "^3.6.2",
"typescript": "^5.9.3",
"typescript-eslint": "^8.47.0",
"vite": "^7.2.4",
"vitest": "4.1.8"

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

4.1.8 is old. Latest is v5.
It seems best to use the latest 5.x version.
FYI: https://vitest.dev/guide/benchmarking

This approach allows you to include benchmarks—by naming files something like <name>.bench.ts—without needing to separate them into a different package.

Furthermore, it is a good practice to establish a benchmark baseline (using writeResult)—perhaps on a dedicated fork branch—and then compare against it during each CI run.

To facilitate this, I’ve created a Composite Action called vitest-bench-compare. If you run it in dry-run mode, it won’t cause the CI to fail; instead, the benchmark results will simply appear in the GitHub Actions execution logs.

I have implemented Vitest v5 benchmarking along with vitest-bench-compare in eslint-plugin-react, so feel free to take a look as a reference.

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks for the information; I’ll definitely make use of it, but I’ll only apply changes if I get some feedback or a signal from Fabian. I’m wary of spending time on a PR that might never get merged.

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I get some feedback or a signal from Fabian

Sure! Please don’t take this as a demand—just think of it as a bit of advice!

}
}
36 changes: 36 additions & 0 deletions bench/src/benchmarks/focused/array.bench.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
import { bench, describe } from 'vitest';
import * as v from '../../../../library/src/index.ts';

const schema = v.array(
v.object({
id: v.number(),
title: v.string(),
tags: v.array(v.string()),
})
);

const validInput = Array.from({ length: 100 }, (_, index) => ({
id: index,
title: `Item ${index}`,
tags: ['schema', 'validation'],
}));

const invalidInput = validInput.map((item, index) =>
index === 0
? { ...item, id: 'zero' }
: index === 99
? { ...item, tags: ['schema', 99] }
: item
);

describe('safeParse', () => {
bench('valid input', () => {
v.safeParse(schema, validInput);
});
bench('invalid input (all errors)', () => {
v.safeParse(schema, invalidInput);
});
bench('invalid input (abort early)', () => {
v.safeParse(schema, invalidInput, { abortEarly: true });
});
});
26 changes: 26 additions & 0 deletions bench/src/benchmarks/focused/enum.bench.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
import { bench, describe } from 'vitest';
import * as v from '../../../../library/src/index.ts';

enum Status {
Draft = 'draft',
Review = 'review',
Published = 'published',
Archived = 'archived',
}

const schema = v.enum(Status);
const validFirstInput = Status.Draft;
const validLastInput = Status.Archived;
const invalidInput = 'deleted';

describe('safeParse', () => {
bench('valid input (first option)', () => {
v.safeParse(schema, validFirstInput);
});
bench('valid input (last option)', () => {
v.safeParse(schema, validLastInput);
});
bench('invalid input', () => {
v.safeParse(schema, invalidInput);
});
});
34 changes: 34 additions & 0 deletions bench/src/benchmarks/focused/object.bench.ts

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

(Sorry, I don't have any concrete ideas myself, but...)

I wonder if this is the right way to set up the benchmark?

From the perspective of the PR's purpose, we need to be able to detect cases where a change introduces an implementation that could degrade performance. However, if—for instance—an ISBN action were added, wouldn't we fail to notice any resulting performance degradation or improvement unless we also updated this benchmark code to account for it?

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

For now, we have a basic reference case, and that suffices. New methods can be added later—for instance, during optimization—but they shouldn't be automatically included in CI checks. A reviewer or developer can run a specific benchmark locally or share the results.

In principle, we could run all the benchmarks, but that should be done infrequently—such as when releasing a new version.

We can consider this later; it’s not a blocker.

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I see. This is just out of pure curiosity, but do you have a concrete exit strategy?
For instance, what are the conditions for scrapping this change, and what are the conditions for introducing the next one?
My main concern is simply that I’d hate for it to just linger on in a half-finished state.

Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
import { bench, describe } from 'vitest';
import * as v from '../../../../library/src/index.ts';

const fieldCount = 100;

const entries = Object.fromEntries(
Array.from({ length: fieldCount }, (_, index) => [
`field_${index}`,
v.number(),
])
);

const validInput = Object.fromEntries(
Array.from({ length: fieldCount }, (_, index) => [`field_${index}`, index])
);

const schema = v.object(entries);
const invalidInput = {
...validInput,
field_0: 'zero',
field_99: 'ninety-nine',
};

describe('safeParse', () => {
bench('valid input', () => {
v.safeParse(schema, validInput);
});
bench('invalid input (all errors)', () => {
v.safeParse(schema, invalidInput);
});
bench('invalid input (abort early)', () => {
v.safeParse(schema, invalidInput, { abortEarly: true });
});
});
19 changes: 19 additions & 0 deletions bench/src/benchmarks/focused/picklist.bench.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
import { bench, describe } from 'vitest';
import * as v from '../../../../library/src/index.ts';

const schema = v.picklist(['draft', 'review', 'published', 'archived']);
const validFirstInput = 'draft';
const validLastInput = 'archived';
const invalidInput = 'deleted';

describe('safeParse', () => {
bench('valid input (first option)', () => {
v.safeParse(schema, validFirstInput);
});
bench('valid input (last option)', () => {
v.safeParse(schema, validLastInput);
});
bench('invalid input', () => {
v.safeParse(schema, invalidInput);
});
});
24 changes: 24 additions & 0 deletions bench/src/benchmarks/focused/union.bench.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
import { bench, describe } from 'vitest';
import * as v from '../../../../library/src/index.ts';

const schema = v.union([
v.object({ type: v.literal('user'), id: v.number() }),
v.object({ type: v.literal('product'), sku: v.string() }),
v.object({ type: v.literal('order'), total: v.number() }),
]);

const validFirstInput = { type: 'user', id: 1 };
const validLastInput = { type: 'order', total: 99 };
const invalidInput = { type: 'unknown', value: true };

describe('safeParse', () => {
bench('valid input (first option)', () => {
v.safeParse(schema, validFirstInput);
});
bench('valid input (last option)', () => {
v.safeParse(schema, validLastInput);
});
bench('invalid input (all errors)', () => {
v.safeParse(schema, invalidInput);
});
});
15 changes: 15 additions & 0 deletions bench/src/benchmarks/focused/url.bench.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
import { bench, describe } from 'vitest';
import * as v from '../../../../library/src/index.ts';

const schema = v.pipe(v.string(), v.url());
const validInput = 'https://valibot.dev/blog/valibot-v1.4-release-notes/';
const invalidInput = 'invalid-url';

describe('safeParse', () => {
bench('valid input', () => {
v.safeParse(schema, validInput);
});
bench('invalid input (all errors)', () => {
v.safeParse(schema, invalidInput);
});
});
24 changes: 24 additions & 0 deletions bench/src/benchmarks/focused/variant.bench.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
import { bench, describe } from 'vitest';
import * as v from '../../../../library/src/index.ts';

const schema = v.variant('type', [
v.object({ type: v.literal('user'), id: v.number() }),
v.object({ type: v.literal('product'), sku: v.string() }),
v.object({ type: v.literal('order'), total: v.number() }),
]);

const validInput = { type: 'product', sku: 'SKU-1' };
const invalidDiscriminatorInput = { type: 'unknown', value: true };
const invalidSelectedOptionInput = { type: 'product', sku: 123 };

describe('safeParse', () => {
bench('valid input', () => {
v.safeParse(schema, validInput);
});
bench('invalid discriminator', () => {
v.safeParse(schema, invalidDiscriminatorInput);
});
bench('invalid selected option', () => {
v.safeParse(schema, invalidSelectedOptionInput);
});
});
Loading
Loading