Repository navigation
perf: add benchmark suite and CI comparison #1609
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
f57da29
13c6f85
00f4a11
9ededf3
2c82d7d
462a030
a548146
fd4572e
9315ebf
272fa59
151b9d2
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,42 @@ | ||
| name: Benchmark checks | ||
|
|
||
| on: | ||
| pull_request: | ||
| paths: | ||
| - 'bench/**' | ||
| - '.github/workflows/*.yml' | ||
| - '.github/actions/environment/**' | ||
|
coderabbitai[bot] marked this conversation as resolved.
Resolved
|
||
| - 'pnpm-workspace.yaml' | ||
| - 'pnpm-lock.yaml' | ||
|
|
||
| permissions: | ||
| contents: read | ||
|
|
||
| jobs: | ||
| bench_prettier: | ||
| name: Run Prettier in bench | ||
| runs-on: ubuntu-latest | ||
| steps: | ||
| - name: Checkout repository | ||
| uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | ||
| with: | ||
| persist-credentials: false | ||
| - name: Setup environment | ||
| uses: ./.github/actions/environment | ||
| - name: Prettier check | ||
| run: pnpm format.check | ||
| working-directory: bench | ||
|
|
||
| bench_eslint: | ||
| name: Run ESLint in bench | ||
| runs-on: ubuntu-latest | ||
| steps: | ||
| - name: Checkout repository | ||
| uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | ||
| with: | ||
| persist-credentials: false | ||
| - name: Setup environment | ||
| uses: ./.github/actions/environment | ||
| - name: ESLint check | ||
| run: pnpm lint | ||
| working-directory: bench | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,38 @@ | ||
| name: Benchmark | ||
|
|
||
| on: | ||
| pull_request: | ||
| paths: | ||
| - 'library/**' | ||
| - 'bench/**' | ||
| - '.github/workflows/bench.yml' | ||
|
coderabbitai[bot] marked this conversation as resolved.
Resolved
|
||
| - '.github/actions/environment/**' | ||
|
|
||
| permissions: | ||
| contents: read | ||
|
|
||
| jobs: | ||
| bench: | ||
| name: Run benchmarks | ||
| runs-on: ubuntu-latest | ||
| steps: | ||
| - name: Checkout pull request base | ||
| uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | ||
| with: | ||
| persist-credentials: false | ||
| ref: ${{ github.event.pull_request.base.sha }} | ||
| - name: Setup environment | ||
| uses: ./.github/actions/environment | ||
| - name: Benchmark pull request base | ||
| run: pnpm exec vitest bench --run representative --outputJson=/tmp/valibot-baseline.json | ||
| working-directory: bench | ||
| - name: Checkout pull request | ||
| uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | ||
| with: | ||
| persist-credentials: false | ||
| ref: ${{ github.event.pull_request.head.sha }} | ||
| - name: Install pull request dependencies | ||
| run: pnpm install | ||
| - name: Compare pull request benchmarks | ||
| run: pnpm exec vitest bench --run representative --compare=/tmp/valibot-baseline.json | ||
| working-directory: bench | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1 @@ | ||
| /baseline.json |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,55 @@ | ||
| # Benchmarking | ||
|
|
||
| Benchmarks measure performance. Test correctness in the usual `library/src/` | ||
| tests. | ||
|
|
||
| ## Structure | ||
|
|
||
| Benchmarks are grouped by workload purpose: | ||
|
|
||
| - `src/benchmarks/focused/` contains focused workloads. Use these to investigate a regression in a | ||
| particular scenario. | ||
| - `src/benchmarks/representative/` contains realistic combined workloads such | ||
| as `product`. It intentionally exercises nested schemas, arrays, pipes, and | ||
| multiple API operations, so its result is an overall signal rather than a | ||
| measurement attributable to one feature. | ||
|
|
||
| The file name identifies the workload and the directory identifies its | ||
| purpose. Keep schemas and inputs at the top of each benchmark file, outside | ||
| benchmark callbacks. Keep suite and benchmark names stable so Vitest can match | ||
| comparison results. | ||
|
|
||
| ## Running | ||
|
|
||
| Run all benchmarks, selected workloads, or selected suites: | ||
|
|
||
| ```sh | ||
| cd bench | ||
| pnpm bench | ||
| pnpm bench object | ||
| pnpm bench array | ||
| pnpm bench representative/product | ||
| pnpm bench --testNamePattern='safeParse' | ||
| pnpm bench url --testNamePattern='safeParse.*invalid' | ||
| ``` | ||
|
|
||
| Vitest matches file filters against benchmark paths, so a short name is usually | ||
| enough. Combine file and `--testNamePattern` filters as needed. | ||
|
|
||
| ## Baselines | ||
|
|
||
| Create a local baseline from any branch, commit, or checkout. Use the same | ||
| filters when comparing it: | ||
|
|
||
| ```sh | ||
| cd bench | ||
| pnpm bench:baseline url --testNamePattern='invalid' | ||
| pnpm bench:compare url --testNamePattern='invalid' | ||
| ``` | ||
|
|
||
| The baseline is saved to the ignored local file `baseline.json`. Recreate it | ||
|
yslpn marked this conversation as resolved.
Resolved
|
||
| after moving or renaming benchmark files. | ||
|
|
||
| CI runs the `representative` benchmark comparison automatically for pull | ||
| requests that change `library/` as a required check. Focused workloads remain | ||
| available for local investigation. | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,39 @@ | ||
| import eslint from '@eslint/js'; | ||
| import importPlugin from 'eslint-plugin-import'; | ||
| import tseslint from 'typescript-eslint'; | ||
|
|
||
| export default tseslint.config( | ||
| { | ||
| ignores: ['eslint.config.js', 'vitest.config.ts'], | ||
| }, | ||
| eslint.configs.recommended, | ||
| tseslint.configs.strict, | ||
| tseslint.configs.stylistic, | ||
| { | ||
| files: ['src/**/*.ts'], | ||
| extends: [importPlugin.flatConfigs.recommended], | ||
| languageOptions: { | ||
| parserOptions: { | ||
| project: './tsconfig.json', | ||
| tsconfigRootDir: import.meta.dirname, | ||
| }, | ||
| }, | ||
| settings: { | ||
| 'import/parsers': { | ||
| '@typescript-eslint/parser': ['.ts'], | ||
| }, | ||
| 'import/resolver': { | ||
| typescript: { | ||
| project: './tsconfig.json', | ||
| }, | ||
| }, | ||
| }, | ||
| rules: { | ||
| '@typescript-eslint/consistent-type-definitions': 'error', | ||
| '@typescript-eslint/consistent-type-imports': 'warn', | ||
| 'import/extensions': ['error', 'always'], | ||
| 'import/no-cycle': 'error', | ||
| 'no-duplicate-imports': 'off', | ||
| }, | ||
| } | ||
| ); |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,24 @@ | ||
| { | ||
| "name": "@valibot/bench", | ||
| "private": true, | ||
| "type": "module", | ||
| "scripts": { | ||
| "bench": "vitest bench --run", | ||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Valibot originally uses https://github.com/open-circle/schema-benchmarks for benchmarking, so wouldn't using Vitest's The point is that the figures shown in the PR do not strictly match those published in the blog or documentation. Also, Vitest benchmark is still experimental at the moment, so in that sense, it might be better to avoid it. So before adding benchmark measurements to the CI, it might be better to first modify https://github.com/open-circle/schema-benchmarks so that it can be used within the CI environment 👀
Member
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
What do you mean?
Why should they match?
No problem, this doesn't affect the runtime or the published library. There are no risks. Furthermore, in version 5, the Vitest API will be fully stabilized, with minor breaking changes described in the migration guide. https://vitest.dev/blog/vitest-5.html
We discussed this with @EskiMojo14 and @fabian-hiller on Discord: A key objective is also to make it easier to evaluate performance locally when testing new implementations and alternative approaches.
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
if we measure Valibot benchmarks using a different method than before, the benchmark figures won't strictly match those presented in the blog post below, and I am concerned this might give users the impression that we are benchmarking in an ambiguous manner.
I see. So, if we were to add a method for local verification, is there any reason why adding a way to verify using
Member
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Technically, yes, but it would require changing First, Second, the shared suite intentionally focuses on scenarios that are comparable across libraries. Our internal suite also needs to cover Valibot-specific validations, transformations, and implementation alternatives that other libraries may not support. Adding those cases to the shared suite would either make them unavailable for other libraries or move So I am not saying that a local
|
||
| "bench:baseline": "vitest bench --run --outputJson=baseline.json", | ||
| "bench:compare": "vitest bench --run --compare=baseline.json", | ||
| "lint": "eslint \"src/**/*.ts*\" && tsc --noEmit", | ||
| "format": "prettier --write ./src", | ||
| "format.check": "prettier --check ./src" | ||
| }, | ||
| "devDependencies": { | ||
| "@eslint/js": "^9.39.1", | ||
| "eslint": "^9.39.1", | ||
| "eslint-import-resolver-typescript": "^4.4.4", | ||
| "eslint-plugin-import": "^2.32.0", | ||
| "prettier": "^3.6.2", | ||
| "typescript": "^5.9.3", | ||
| "typescript-eslint": "^8.47.0", | ||
| "vite": "^7.2.4", | ||
| "vitest": "4.1.8" | ||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
This approach allows you to include benchmarks—by naming files something like Furthermore, it is a good practice to establish a benchmark baseline (using To facilitate this, I’ve created a Composite Action called vitest-bench-compare. If you run it in I have implemented Vitest v5 benchmarking along with
Member
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Thanks for the information; I’ll definitely make use of it, but I’ll only apply changes if I get some feedback or a signal from Fabian. I’m wary of spending time on a PR that might never get merged.
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Sure! Please don’t take this as a demand—just think of it as a bit of advice! |
||
| } | ||
| } | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,36 @@ | ||
| import { bench, describe } from 'vitest'; | ||
| import * as v from '../../../../library/src/index.ts'; | ||
|
|
||
| const schema = v.array( | ||
| v.object({ | ||
| id: v.number(), | ||
| title: v.string(), | ||
| tags: v.array(v.string()), | ||
| }) | ||
| ); | ||
|
|
||
| const validInput = Array.from({ length: 100 }, (_, index) => ({ | ||
| id: index, | ||
| title: `Item ${index}`, | ||
| tags: ['schema', 'validation'], | ||
| })); | ||
|
|
||
| const invalidInput = validInput.map((item, index) => | ||
| index === 0 | ||
| ? { ...item, id: 'zero' } | ||
| : index === 99 | ||
| ? { ...item, tags: ['schema', 99] } | ||
| : item | ||
| ); | ||
|
|
||
| describe('safeParse', () => { | ||
| bench('valid input', () => { | ||
| v.safeParse(schema, validInput); | ||
| }); | ||
| bench('invalid input (all errors)', () => { | ||
| v.safeParse(schema, invalidInput); | ||
| }); | ||
| bench('invalid input (abort early)', () => { | ||
| v.safeParse(schema, invalidInput, { abortEarly: true }); | ||
| }); | ||
| }); |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,26 @@ | ||
| import { bench, describe } from 'vitest'; | ||
| import * as v from '../../../../library/src/index.ts'; | ||
|
|
||
| enum Status { | ||
| Draft = 'draft', | ||
| Review = 'review', | ||
| Published = 'published', | ||
| Archived = 'archived', | ||
| } | ||
|
|
||
| const schema = v.enum(Status); | ||
| const validFirstInput = Status.Draft; | ||
| const validLastInput = Status.Archived; | ||
| const invalidInput = 'deleted'; | ||
|
|
||
| describe('safeParse', () => { | ||
| bench('valid input (first option)', () => { | ||
| v.safeParse(schema, validFirstInput); | ||
| }); | ||
| bench('valid input (last option)', () => { | ||
| v.safeParse(schema, validLastInput); | ||
| }); | ||
| bench('invalid input', () => { | ||
| v.safeParse(schema, invalidInput); | ||
| }); | ||
| }); |
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. (Sorry, I don't have any concrete ideas myself, but...) I wonder if this is the right way to set up the benchmark? From the perspective of the PR's purpose, we need to be able to detect cases where a change introduces an implementation that could degrade performance. However, if—for instance—an
Member
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. For now, we have a basic reference case, and that suffices. New methods can be added later—for instance, during optimization—but they shouldn't be automatically included in CI checks. A reviewer or developer can run a specific benchmark locally or share the results. In principle, we could run all the benchmarks, but that should be done infrequently—such as when releasing a new version. We can consider this later; it’s not a blocker.
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I see. This is just out of pure curiosity, but do you have a concrete exit strategy? |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,34 @@ | ||
| import { bench, describe } from 'vitest'; | ||
| import * as v from '../../../../library/src/index.ts'; | ||
|
|
||
| const fieldCount = 100; | ||
|
|
||
| const entries = Object.fromEntries( | ||
| Array.from({ length: fieldCount }, (_, index) => [ | ||
| `field_${index}`, | ||
| v.number(), | ||
| ]) | ||
| ); | ||
|
|
||
| const validInput = Object.fromEntries( | ||
| Array.from({ length: fieldCount }, (_, index) => [`field_${index}`, index]) | ||
| ); | ||
|
|
||
| const schema = v.object(entries); | ||
| const invalidInput = { | ||
| ...validInput, | ||
| field_0: 'zero', | ||
| field_99: 'ninety-nine', | ||
| }; | ||
|
|
||
| describe('safeParse', () => { | ||
| bench('valid input', () => { | ||
| v.safeParse(schema, validInput); | ||
| }); | ||
| bench('invalid input (all errors)', () => { | ||
| v.safeParse(schema, invalidInput); | ||
| }); | ||
| bench('invalid input (abort early)', () => { | ||
| v.safeParse(schema, invalidInput, { abortEarly: true }); | ||
| }); | ||
| }); |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,19 @@ | ||
| import { bench, describe } from 'vitest'; | ||
| import * as v from '../../../../library/src/index.ts'; | ||
|
|
||
| const schema = v.picklist(['draft', 'review', 'published', 'archived']); | ||
| const validFirstInput = 'draft'; | ||
| const validLastInput = 'archived'; | ||
| const invalidInput = 'deleted'; | ||
|
|
||
| describe('safeParse', () => { | ||
| bench('valid input (first option)', () => { | ||
| v.safeParse(schema, validFirstInput); | ||
| }); | ||
| bench('valid input (last option)', () => { | ||
| v.safeParse(schema, validLastInput); | ||
| }); | ||
| bench('invalid input', () => { | ||
| v.safeParse(schema, invalidInput); | ||
| }); | ||
| }); |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,24 @@ | ||
| import { bench, describe } from 'vitest'; | ||
| import * as v from '../../../../library/src/index.ts'; | ||
|
|
||
| const schema = v.union([ | ||
| v.object({ type: v.literal('user'), id: v.number() }), | ||
| v.object({ type: v.literal('product'), sku: v.string() }), | ||
| v.object({ type: v.literal('order'), total: v.number() }), | ||
| ]); | ||
|
|
||
| const validFirstInput = { type: 'user', id: 1 }; | ||
| const validLastInput = { type: 'order', total: 99 }; | ||
| const invalidInput = { type: 'unknown', value: true }; | ||
|
|
||
| describe('safeParse', () => { | ||
| bench('valid input (first option)', () => { | ||
| v.safeParse(schema, validFirstInput); | ||
| }); | ||
| bench('valid input (last option)', () => { | ||
| v.safeParse(schema, validLastInput); | ||
| }); | ||
| bench('invalid input (all errors)', () => { | ||
| v.safeParse(schema, invalidInput); | ||
| }); | ||
| }); |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,15 @@ | ||
| import { bench, describe } from 'vitest'; | ||
| import * as v from '../../../../library/src/index.ts'; | ||
|
|
||
| const schema = v.pipe(v.string(), v.url()); | ||
| const validInput = 'https://valibot.dev/blog/valibot-v1.4-release-notes/'; | ||
| const invalidInput = 'invalid-url'; | ||
|
|
||
| describe('safeParse', () => { | ||
| bench('valid input', () => { | ||
| v.safeParse(schema, validInput); | ||
| }); | ||
| bench('invalid input (all errors)', () => { | ||
| v.safeParse(schema, invalidInput); | ||
| }); | ||
| }); |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,24 @@ | ||
| import { bench, describe } from 'vitest'; | ||
| import * as v from '../../../../library/src/index.ts'; | ||
|
|
||
| const schema = v.variant('type', [ | ||
| v.object({ type: v.literal('user'), id: v.number() }), | ||
| v.object({ type: v.literal('product'), sku: v.string() }), | ||
| v.object({ type: v.literal('order'), total: v.number() }), | ||
| ]); | ||
|
|
||
| const validInput = { type: 'product', sku: 'SKU-1' }; | ||
| const invalidDiscriminatorInput = { type: 'unknown', value: true }; | ||
| const invalidSelectedOptionInput = { type: 'product', sku: 123 }; | ||
|
|
||
| describe('safeParse', () => { | ||
| bench('valid input', () => { | ||
| v.safeParse(schema, validInput); | ||
| }); | ||
| bench('invalid discriminator', () => { | ||
| v.safeParse(schema, invalidDiscriminatorInput); | ||
| }); | ||
| bench('invalid selected option', () => { | ||
| v.safeParse(schema, invalidSelectedOptionInput); | ||
| }); | ||
| }); |
Uh oh!
There was an error while loading. Please reload this page.