diff --git a/package.json b/package.json index 8fc8a7e..c5b6977 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "muya", - "version": "2.4.4", + "version": "2.4.5", "author": "samuel.gjabel@gmail.com", "repository": "https://github.com/samuelgjabel/muya", "main": "cjs/index.js", diff --git a/packages/core/scheduler.ts b/packages/core/scheduler.ts index dc44c67..d4ae99b 100644 --- a/packages/core/scheduler.ts +++ b/packages/core/scheduler.ts @@ -10,7 +10,7 @@ interface GlobalSchedulerItem { export interface SchedulerOptions { readonly onResolveItem?: (item: T) => void - readonly onScheduleDone: () => void | Promise + readonly onScheduleDone: (values?: unknown[]) => void | Promise } /** @@ -59,6 +59,7 @@ export function createScheduler() { } const effectedListeners = new Set() + const valuesMap = new Map() for (const value of batches) { if (listeners.has(value.id)) { effectedListeners.add(value.id) @@ -66,6 +67,10 @@ export function createScheduler() { if (onResolveItem) { onResolveItem(value.value) } + if (!valuesMap.has(value.id)) { + valuesMap.set(value.id, []) + } + valuesMap.get(value.id)!.push(value.value) } batches.delete(value) } @@ -76,7 +81,8 @@ export function createScheduler() { } for (const id of effectedListeners) { - listeners.get(id)?.onScheduleDone() + const values = valuesMap.get(id) + listeners.get(id)?.onScheduleDone(values) } } diff --git a/packages/core/sqlite/__tests__/use-slite-count.test.tsx b/packages/core/sqlite/__tests__/use-slite-count.test.tsx new file mode 100644 index 0000000..4f0a3f6 --- /dev/null +++ b/packages/core/sqlite/__tests__/use-slite-count.test.tsx @@ -0,0 +1,96 @@ +import { act, renderHook } from '@testing-library/react-hooks' +import { createSqliteState } from '../create-sqlite' +import { useSqliteCount } from '../use-sqlite-count' +import { bunMemoryBackend } from '../table/bun-backend' +import { waitFor } from '@testing-library/react' + +const backend = bunMemoryBackend() +interface Person { + id: string + name: string + age: number +} + +describe('useSqliteCount', () => { + it('should count items in the table', async () => { + const sql = createSqliteState({ backend, tableName: 'CountTest1', key: 'id' }) + const { result } = renderHook(() => useSqliteCount(sql)) + + await waitFor(() => { + expect(result.current).toBe(0) + }) + + act(() => { + sql.set({ id: '1', name: 'Alice', age: 30 }) + sql.set({ id: '2', name: 'Bob', age: 25 }) + }) + + await waitFor(() => { + expect(result.current).toBe(2) + }) + + act(() => { + sql.delete('1') + }) + + await waitFor(() => { + expect(result.current).toBe(1) + }) + }) + + it('should support filtering with where clause', async () => { + const sql = createSqliteState({ backend, tableName: 'CountTest2', key: 'id' }) + await sql.batchSet([ + { id: '1', name: 'Alice', age: 30 }, + { id: '2', name: 'Bob', age: 25 }, + { id: '3', name: 'Carol', age: 40 }, + ]) + + const { result } = renderHook(() => useSqliteCount(sql, { where: { age: { gt: 30 } } })) + + await waitFor(() => { + expect(result.current).toBe(1) + }) + + act(() => { + sql.set({ id: '4', name: 'Dave', age: 35 }) + }) + + await waitFor(() => { + expect(result.current).toBe(2) + }) + + act(() => { + sql.delete('3') + }) + + await waitFor(() => { + expect(result.current).toBe(1) + }) + }) + + it('should react to dependency changes', async () => { + const sql = createSqliteState({ backend, tableName: 'CountTest3', key: 'id' }) + await sql.batchSet([ + { id: '1', name: 'Alice', age: 30 }, + { id: '2', name: 'Bob', age: 25 }, + { id: '3', name: 'Carol', age: 40 }, + ]) + + const { result, rerender } = renderHook(({ minAge }) => useSqliteCount(sql, { where: { age: { gt: minAge } } }, [minAge]), { + initialProps: { minAge: 20 }, + }) + + await waitFor(() => { + expect(result.current).toBe(3) + }) + + act(() => { + rerender({ minAge: 30 }) + }) + + await waitFor(() => { + expect(result.current).toBe(1) + }) + }) +}) diff --git a/packages/core/sqlite/__tests__/use-sqlite.test.tsx b/packages/core/sqlite/__tests__/use-sqlite.test.tsx index 56dbe9e..b19fbc7 100644 --- a/packages/core/sqlite/__tests__/use-sqlite.test.tsx +++ b/packages/core/sqlite/__tests__/use-sqlite.test.tsx @@ -5,7 +5,7 @@ import { useSqliteValue } from '../use-sqlite' import { waitFor } from '@testing-library/react' import { bunMemoryBackend } from '../table/bun-backend' import { StrictMode, Suspense, useState } from 'react' -import { DEFAULT_STEP_SIZE } from '../table/table' +import { DEFAULT_PAGE_SIZE } from '../table/table' const backend = bunMemoryBackend() interface Person { @@ -28,7 +28,7 @@ describe('use-sqlite-state', () => { const { result, rerender } = renderHook( () => { reRenders++ - const aha = useSqliteValue(sql, {}, []) + const aha = useSqliteValue(sql) return aha }, { wrapper: Wrapper }, @@ -67,7 +67,7 @@ describe('use-sqlite-state', () => { sql.set({ id: '2', name: 'Bob', age: 25 }) }) await waitFor(() => { - expect(result.current[0].length).toBe(2) + expect(result.current[0]?.length).toBe(2) expect(reRenders).toBe(6) }) @@ -76,7 +76,7 @@ describe('use-sqlite-state', () => { }) await waitFor(() => { expect(reRenders).toBe(7) - expect(result.current[0].length).toBe(2) + expect(result.current[0]?.length).toBe(2) }) }) @@ -95,7 +95,8 @@ describe('use-sqlite-state', () => { }) await waitFor(() => { - expect(result.current[0][0].map((p) => p.name)).toEqual(['Bob', 'Alice', 'Carol']) + const names = result.current?.[0][0]?.map((p) => p.name) + expect(names).toEqual(['Bob', 'Alice', 'Carol']) expect(reRenders).toBe(2) }) @@ -104,7 +105,8 @@ describe('use-sqlite-state', () => { result.current[1](29) }) await waitFor(() => { - expect(result.current[0][0].map((p) => p.name)).toEqual(['Alice', 'Carol']) + const names = result.current?.[0][0]?.map((p) => p.name) + expect(names).toEqual(['Alice', 'Carol']) expect(reRenders).toBe(4) }) }) @@ -125,13 +127,13 @@ describe('use-sqlite-state', () => { { initialProps: { like: '%Ali%' } }, ) await waitFor(() => { - expect(result.current[0].map((p) => p.name)).toEqual(['Alice', 'Alicia']) + expect(result.current?.[0]?.map((p) => p.name)).toEqual(['Alice', 'Alicia']) }) act(() => { rerender({ like: '%Bob%' }) }) await waitFor(() => { - expect(result.current[0].map((p) => p.name)).toEqual(['Bob']) + expect(result.current?.[0]?.map((p) => p.name)).toEqual(['Bob']) }) expect(reRenders).toBeGreaterThanOrEqual(2) }) @@ -148,19 +150,19 @@ describe('use-sqlite-state', () => { { initialProps: { order: 'asc' as 'asc' | 'desc', limit: 2 } }, ) await waitFor(() => { - expect(result.current[0].map((p) => p.name)).toEqual(['Bob', 'Alice']) + expect(result.current?.[0]?.map((p) => p.name)).toEqual(['Bob', 'Alice']) }) act(() => { rerender({ order: 'desc', limit: 2 }) }) await waitFor(() => { - expect(result.current[0].map((p) => p.name)).toEqual(['Carol', 'Alice']) + expect(result.current?.[0]?.map((p) => p.name)).toEqual(['Carol', 'Alice']) }) act(() => { rerender({ order: 'desc', limit: 1 }) }) await waitFor(() => { - expect(result.current[0].map((p) => p.name)).toEqual(['Carol']) + expect(result.current?.[0]?.map((p) => p.name)).toEqual(['Carol']) }) }) @@ -173,13 +175,11 @@ describe('use-sqlite-state', () => { const { result } = renderHook(() => useSqliteValue(sql, {}, [])) // actions.next and actions.refresh should be functions await waitFor(() => { - expect(typeof result.current[1].next).toBe('function') + expect(typeof result.current[1].nextPage).toBe('function') expect(typeof result.current[1].reset).toBe('function') - expect(result.current[1].reset()).resolves.toBeUndefined() - expect(result.current[1].next()).resolves.toBeFalsy() }) }) - it('should handle thousands of records', async () => { + it('should handle thousands of records Here', async () => { const sql = createSqliteState({ backend, tableName: 'State6Hook', key: 'id' }) const people: Person[] = [] const ITEMS_COUNT = 1000 @@ -189,16 +189,16 @@ describe('use-sqlite-state', () => { await sql.batchSet(people) const { result } = renderHook(() => useSqliteValue(sql, {}, [])) await waitFor(() => { - expect(result.current[0].length).toBe(DEFAULT_STEP_SIZE) + expect(result.current?.[0]?.length ?? 0).toBe(DEFAULT_PAGE_SIZE) }) - // loop until we have all ITEMS_COUNT items - for (let index = 0; index < ITEMS_COUNT / DEFAULT_STEP_SIZE; index++) { + // // loop until we have all ITEMS_COUNT items + for (let index = 0; index < ITEMS_COUNT / DEFAULT_PAGE_SIZE; index++) { act(() => { - result.current[1].next() + result.current[1].nextPage() }) await waitFor(() => { - expect(result.current[0].length).toBe(Math.min(DEFAULT_STEP_SIZE * (index + 2), ITEMS_COUNT)) + expect(result.current?.[0]?.length).toBe(Math.min(DEFAULT_PAGE_SIZE * (index + 2), ITEMS_COUNT)) }) } @@ -206,7 +206,7 @@ describe('use-sqlite-state', () => { result.current[1].reset() }) await waitFor(() => { - expect(result.current[0].length).toBe(DEFAULT_STEP_SIZE) + expect(result.current?.[0]?.length).toBe(DEFAULT_PAGE_SIZE) }) }) @@ -214,7 +214,7 @@ describe('use-sqlite-state', () => { const sql = createSqliteState({ backend, tableName: 'State6Hook', key: 'id' }) const people: Person[] = [] const ITEMS_COUNT = 10_000 - const stepSize = 5000 + const pageSize = 500 for (let index = 1; index <= ITEMS_COUNT; index++) { people.push({ id: index.toString(), name: `Person${index}`, age: 20 + (index % 50) }) } @@ -222,30 +222,30 @@ describe('use-sqlite-state', () => { let reRenders = 0 const { result } = renderHook(() => { reRenders++ - return useSqliteValue(sql, { stepSize }, []) + return useSqliteValue(sql, { pageSize }, []) }) await waitFor(() => { expect(reRenders).toBe(2) - expect(result.current[0].length).toBe(stepSize) + expect(result.current?.[0]?.length).toBe(pageSize) }) act(() => { - while (!result.current[1].next()) { - /* empty */ + for (let index = 0; index < (ITEMS_COUNT - pageSize) / pageSize; index++) { + result.current[1].nextPage() } }) await waitFor(() => { - expect(reRenders).toBe(3) - expect(result.current[0].length).toBe(ITEMS_COUNT) + expect(reRenders).toBe(21) + expect(result.current?.[0]?.length).toBe(ITEMS_COUNT) }) act(() => { result.current[1].reset() }) await waitFor(() => { - expect(reRenders).toBe(4) - expect(result.current[0].length).toBe(stepSize) + expect(reRenders).toBe(22) + expect(result.current?.[0]?.length).toBe(pageSize) }) }) it('should change ordering', async () => { @@ -259,13 +259,13 @@ describe('use-sqlite-state', () => { initialProps: { order: 'asc' as 'asc' | 'desc' }, }) await waitFor(() => { - expect(result.current[0][0].age).toBe(20) + expect(result.current?.[0]?.[0]?.age).toBe(20) }) act(() => { rerender({ order: 'desc' }) }) await waitFor(() => { - expect(result.current[0][0].age).toBe(69) + expect(result.current?.[0]?.[0]?.age).toBe(69) }) }) @@ -306,7 +306,7 @@ describe('use-sqlite-state', () => { }) await waitFor(() => { expect(reRenders).toBe(1) - expect(result1.current[0].length).toBe(0) + expect(result1.current?.[0]?.length).toBe(undefined) }) const people: Person[] = [] @@ -316,12 +316,12 @@ describe('use-sqlite-state', () => { await sql.batchSet(people) await waitFor(() => { expect(reRenders).toBe(3) - expect(result1.current[0].length).toBe(50) + expect(result1.current?.[0]?.length).toBe(50) }) const { result: result2 } = renderHook(() => useSqliteValue(sql, {}, [])) await waitFor(() => { - expect(result2.current[0].length).toBe(50) + expect(result2.current?.[0]?.length).toBe(50) }) }) @@ -341,8 +341,8 @@ describe('use-sqlite-state', () => { }) await waitFor(() => { - expect(reRenders).toBe(1) - expect(result.current[0].length).toBe(0) + expect(reRenders).toBe(2) + expect(result.current?.[0]?.length).toBe(0) }) act(() => { @@ -387,7 +387,7 @@ describe('use-sqlite-state', () => { await waitFor(() => { expect(reRenders).toBe(2) - expect(result.current[0].length).toBe(1) + expect(result.current?.[0]?.length).toBe(1) }) act(() => { @@ -395,15 +395,98 @@ describe('use-sqlite-state', () => { }) await waitFor(() => { expect(reRenders).toBe(3) - expect(result.current[0].length).toBe(2) + expect(result.current?.[0]?.length).toBe(2) }) act(() => { result.current[1].reset() }) await waitFor(() => { - expect(result.current[0].length).toBe(2) + expect(result.current?.[0]?.length).toBe(2) expect(reRenders).toBe(4) }) }) + + it('should handle no items in the database', async () => { + const sql = createSqliteState({ backend, tableName: 'EmptyState', key: 'id' }) + const { result } = renderHook(() => useSqliteValue(sql, {}, [])) + + await waitFor(() => { + expect(result.current[0]).toEqual([]) + }) + }) + + it('should handle fewer items than page size', async () => { + const sql = createSqliteState({ backend, tableName: 'FewItemsState', key: 'id' }) + await sql.batchSet([ + { id: '1', name: 'Alice', age: 30 }, + { id: '2', name: 'Bob', age: 25 }, + ]) + + const { result } = renderHook(() => useSqliteValue(sql, {}, [])) + + await waitFor(() => { + expect(result.current[0]).toEqual([ + { id: '1', name: 'Alice', age: 30 }, + { id: '2', name: 'Bob', age: 25 }, + ]) + }) + }) + + it('should handle exactly page size items', async () => { + const sql = createSqliteState({ backend, tableName: 'ExactPageSizeState', key: 'id' }) + const items = Array.from({ length: DEFAULT_PAGE_SIZE }, (_, index) => ({ + id: `${index + 1}`, + name: `Person${index + 1}`, + age: 20 + (index % 50), + })) + await sql.batchSet(items) + + const { result } = renderHook(() => useSqliteValue(sql, {}, [])) + + await waitFor(() => { + expect(result.current[0]?.length).toBe(DEFAULT_PAGE_SIZE) + }) + }) + + it('should have thousands items, and update in middle check', async () => { + let reRenders = 0 + const sql = createSqliteState({ backend, tableName: 'ManyItemsState', key: 'id' }) + const ITEMS_COUNT = 1000 + const people: Person[] = [] + for (let index = 1; index <= ITEMS_COUNT; index++) { + people.push({ id: index.toString(), name: `Person${index}`, age: 20 + (index % 50) }) + } + await sql.batchSet(people) + + const { result } = renderHook(() => { + reRenders++ + return useSqliteValue(sql, { pageSize: 100 }, []) + }) + + await waitFor(() => { + expect(result.current[0]?.length).toBe(100) + expect(reRenders).toBe(2) + }) + act(() => { + for (let index = 0; index < (ITEMS_COUNT - 100) / 100; index++) { + result.current[1].nextPage() + } + }) + await waitFor(() => { + expect(result.current[0]?.length).toBe(ITEMS_COUNT) + expect(reRenders).toBe(11) + }) + + act(() => { + sql.set({ id: '500', name: 'UpdatedPerson500', age: 99 }) + }) + + await waitFor(() => { + const updated = result.current[0]?.find((p) => p.id === '500') + expect(updated).toEqual({ id: '500', name: 'UpdatedPerson500', age: 99 }) + expect(reRenders).toBe(12) + expect(result.current[0]?.length).toBe(ITEMS_COUNT) + }) + }) }) diff --git a/packages/core/sqlite/create-sqlite.ts b/packages/core/sqlite/create-sqlite.ts index 1cc2ba4..4ff642e 100644 --- a/packages/core/sqlite/create-sqlite.ts +++ b/packages/core/sqlite/create-sqlite.ts @@ -1,26 +1,21 @@ -/* eslint-disable sonarjs/redundant-type-aliases */ import { STATE_SCHEDULER } from '../create' import { getId } from '../utils/id' -import { shallow } from '../utils/shallow' -import { selectSql, type CreateState } from './select-sql' import type { Backend } from './table' -import { createTable, DEFAULT_STEP_SIZE } from './table/table' +import { createTable } from './table/table' import type { DbOptions, DocType, Key, MutationResult, SearchOptions, Table } from './table/table.types' import type { Where } from './table/where' -type SearchId = string - export interface CreateSqliteOptions extends Omit, 'backend'> { readonly backend: Backend | Promise } -export interface SyncTable { - // readonly registerSearch: (searchId: SearchId, options: SearchOptions) => () => void - readonly updateSearchOptions: (searchId: SearchId, options: SearchOptions) => void - readonly subscribe: (searchId: SearchId, componentId: string, listener: () => void) => () => void - readonly getSnapshot: (searchId: SearchId) => Document[] - readonly refresh: (searchId: SearchId) => Promise +export interface MutationItems { + mutations?: MutationResult[] + removedAll?: boolean +} +export interface SyncTable { + readonly subscribe: (listener: (mutation: MutationItems) => void) => () => void readonly set: (document: Document) => Promise readonly batchSet: (documents: Document[]) => Promise readonly get: (key: Key, selector?: (document: Document) => Selected) => Promise @@ -29,21 +24,7 @@ export interface SyncTable { readonly search: (options?: SearchOptions) => AsyncIterableIterator readonly count: (options?: { where?: Where }) => Promise readonly deleteBy: (where: Where) => Promise - readonly destroy: () => void - readonly next: (searchId: SearchId) => Promise - readonly clear: (searchId: SearchId) => void - readonly load: (searchId: SearchId) => void - - readonly select: ( - compute: (...args: Params) => SearchOptions, - ) => CreateState -} - -interface CachedItem { - items: Document[] - keys: Set - options?: SearchOptions - wasInitialized: boolean + readonly clear: () => void } /** @@ -52,17 +33,6 @@ interface CachedItem { * @returns A SyncTable instance with methods to interact with the underlying Table and manage reactive searches */ export function createSqliteState(options: CreateSqliteOptions): SyncTable { - const id = getId() - - /** - * Get a unique schedule ID for a search ID - * @param searchId The search ID - * @returns The unique schedule ID - */ - function getScheduleId(searchId: SearchId) { - return `state-${id}-search-${searchId}` - } - let cachedTable: Table | undefined /** * Get or create the underlying table @@ -77,213 +47,74 @@ export function createSqliteState(options: CreateSqlit return cachedTable } - interface NextResult { - document: Document - key: Key - } - // const emitter = createEmitter>() - const cachedData = new Map>() - const listeners = new Map void>>() - const iterators = new Map>() - - interface GetNextBase { - readonly newItems?: Document[] - readonly isOk: boolean - readonly isDone?: boolean - } - - interface GetNextTrue extends GetNextBase { - readonly isOk: true - readonly newItems: Document[] - } - interface GetNextFalse extends GetNextBase { - readonly isOk: false - } - type GetNext = GetNextTrue | GetNextFalse - /** - * Next step in the iterator - * @param searchId The search ID - * @param data The data items to process - * @param shouldReset Whether to reset the items - * @returns boolean indicating if new items were added - */ - async function getNext(searchId: SearchId, data: CachedItem, shouldReset: boolean): Promise { - const iterator = iterators.get(searchId) - const { options: nextOptions = {} } = data - const { stepSize = DEFAULT_STEP_SIZE } = nextOptions - if (!iterator) return { isOk: false } - const newItems: Document[] = [] - if (shouldReset) { - data.keys.clear() - } - - for (let index = 0; index < stepSize; index++) { - const result = await iterator.next() - if (result.done) { - iterators.delete(searchId) - break - } - - if (!data.keys.has(String(result.value.key))) { - newItems.push(result.value.document) - data.keys.add(String(result.value.key)) - } - } - - if (newItems.length === 0) return { isOk: true, newItems: shouldReset ? [] : data.items, isDone: true } - if (shallow(data.items, newItems)) return { isOk: false } - if (shouldReset) return { isOk: true, newItems } - return { newItems: [...data.items, ...newItems], isOk: true } - } - - /** - * Notify listeners of up dates - * @param searchId The search ID to notify - */ - function notifyListeners(searchId: SearchId) { - const searchListeners = listeners.get(searchId) - if (searchListeners) { - for (const [, listener] of searchListeners) { - listener() + const id = getId() + STATE_SCHEDULER.add(id, { + onScheduleDone(unknownItems) { + if (!unknownItems) { + return } - } - } - - /** - * Refresh the cache for a search ID - * @param searchId The search ID to refresh - */ - async function refreshCache(searchId: SearchId) { - const table = await getTable() - const data = cachedData.get(searchId) - if (!data) return - const { options: refreshOptions } = data - const iterator = table.search({ ...refreshOptions, select: (document, { rowId, key }) => ({ document, rowId, key }) }) - iterators.set(searchId, iterator) - const { isOk, newItems } = await getNext(searchId, data, true) - if (isOk) { - data.items = newItems - } - } - /** - * Refresh the data and notify listeners - * @param searchId The search ID to refresh - */ - async function refresh(searchId: SearchId) { - await refreshCache(searchId) - notifyListeners(searchId) - } - - /** - * Handle changes to the data - * @param mutationResult The mutation result - * @returns A set of search IDs that need to be updated - */ - function getChangedKeys(mutationResult: MutationResult) { - const { key, op } = mutationResult - // find all cached data with key - const searchIds = new Set() - for (const [searchId, { keys }] of cachedData) { - switch (op) { - case 'delete': - case 'update': { - if (keys.has(String(key))) { - searchIds.add(searchId) - } - break + const items = unknownItems as MutationItems[] + const merged: MutationItems = {} + for (const item of items) { + if (item.removedAll) { + merged.removedAll = true } - case 'insert': { - // we do not know about the key - searchIds.add(searchId) - break + if (item.mutations) { + if (!merged.mutations) { + merged.mutations = [] + } + merged.mutations.push(...item.mutations) } } - } - return searchIds - } - - /** - * Handle multiple changes - * @param mutationResults The array of mutation results - */ - async function handleChanges(mutationResults: MutationResult[]) { - const updateSearchIds = new Set() - for (const mutationResult of mutationResults) { - const searchIds = getChangedKeys(mutationResult) - for (const searchId of searchIds) { - updateSearchIds.add(searchId) + for (const listener of listeners) { + listener(merged) } - } - - for (const searchId of updateSearchIds) { - const scheduleId = getScheduleId(searchId) - STATE_SCHEDULER.schedule(scheduleId, {}) - } - } - - const clearSchedulers = new Set<() => void>() + }, + }) /** - * Register data for a search ID - * @param searchId The search ID - * @param registerDataOptions Optional search options - * @returns The data items for the search ID + * Notify all subscribers of changes + * @param item The mutation items to notify subscribers about */ - function registerData(searchId: SearchId, registerDataOptions?: SearchOptions) { - if (!cachedData.has(searchId)) { - const cachedItem: CachedItem = { - items: [], - options: registerDataOptions, - keys: new Set(), - wasInitialized: false, - } - cachedData.set(searchId, cachedItem) - } - const cachedItem = cachedData.get(searchId)! - - if (registerDataOptions) { - cachedItem.options = registerDataOptions - } - return cachedItem + function handleChanges(item: MutationItems) { + STATE_SCHEDULER.schedule(id, item) } + const listeners = new Set<(mutation: MutationItems) => void>() + const state: SyncTable = { - load(searchId: SearchId) { - const cachedItem = cachedData.get(searchId) - if (!cachedItem) return - if (!cachedItem.wasInitialized) { - cachedItem.wasInitialized = true - const scheduleId = getScheduleId(searchId) - STATE_SCHEDULER.schedule(scheduleId, { searchId }) - } + subscribe(listener) { + listeners.add(listener) + return () => listeners.delete(listener) }, - clear(searchId: SearchId) { - cachedData.delete(searchId) + clear() { + cachedTable?.clear() + handleChanges({ removedAll: true }) }, async set(document) { const table = await getTable() const changes = await table.set(document) - await handleChanges([changes]) + handleChanges({ mutations: [changes] }) return changes }, async batchSet(documents) { const table = await getTable() const changes = await table.batchSet(documents) - await handleChanges(changes) + handleChanges({ mutations: changes }) return changes }, async delete(key) { const table = await getTable() const changes = await table.delete(key) if (changes) { - await handleChanges([changes]) + handleChanges({ mutations: [changes] }) } return changes }, async deleteBy(where) { const table = await getTable() const changes = await table.deleteBy(where) - await handleChanges(changes) + handleChanges({ mutations: changes }) return changes }, async get(key, selector) { @@ -300,65 +131,6 @@ export function createSqliteState(options: CreateSqlit const table = await getTable() return await table.count(countOptions) }, - - updateSearchOptions(searchId, updateSearchOptions) { - const data = registerData(searchId, updateSearchOptions) - data.options = updateSearchOptions - const scheduleId = getScheduleId(searchId) - STATE_SCHEDULER.schedule(scheduleId, { searchId }) - }, - - subscribe(searchId, componentId, listener) { - const scheduleId = getScheduleId(searchId) - const clearScheduler = STATE_SCHEDULER.add(scheduleId, { - onScheduleDone() { - refresh(searchId) - }, - }) - // console.log('Subscribing to searchId:', searchId) - this.load(searchId) - clearSchedulers.add(clearScheduler) - - if (!listeners.has(searchId)) { - listeners.set(searchId, new Map()) - } - const searchListeners = listeners.get(searchId)! - searchListeners.set(componentId, listener) - - return () => { - searchListeners.delete(componentId) - if (searchListeners.size === 0) { - listeners.delete(searchId) - } - clearScheduler() - } - }, - getSnapshot(searchId) { - const data = registerData(searchId) - return data.items - }, - refresh, - destroy() { - for (const clear of clearSchedulers) clear() - cachedData.clear() - listeners.clear() - }, - async next(searchId) { - const data = cachedData.get(searchId) - if (data) { - const hasNext = await getNext(searchId, data, false) - if (hasNext.isOk) { - data.items = hasNext.newItems - notifyListeners(searchId) - } - return hasNext.isDone ?? false - } - return false - }, - - select(compute) { - return selectSql(state, compute) - }, } return state diff --git a/packages/core/sqlite/index.ts b/packages/core/sqlite/index.ts index df2298d..7cd6d0c 100644 --- a/packages/core/sqlite/index.ts +++ b/packages/core/sqlite/index.ts @@ -1,4 +1,4 @@ export * from './create-sqlite' export * from './table' -export * from './select-sql' +export * from './use-sqlite-count' export * from './use-sqlite' diff --git a/packages/core/sqlite/select-sql.ts b/packages/core/sqlite/select-sql.ts deleted file mode 100644 index 3fa10c3..0000000 --- a/packages/core/sqlite/select-sql.ts +++ /dev/null @@ -1,65 +0,0 @@ -import { createState } from '../create-state' -import type { GetState } from '../types' -import type { SyncTable } from './create-sqlite' -import type { DocType, DotPath } from './table/table.types' -import type { Where } from './table/where' - -export type CreateState = (...params: Params) => GetState - -export interface SqlSeachOptions { - readonly sortBy?: DotPath - readonly order?: 'asc' | 'desc' - readonly limit?: number - readonly offset?: number - readonly where?: Where - readonly stepSize?: number -} - -let stateId = 0 -/** - * Generate a unique state ID - * @returns A unique state ID - */ -function getStateId() { - stateId++ - return `${stateId.toString(36)}-sql` -} - -/** - * Create a state that derives its value from a SyncTable using a compute function - * @param state The SyncTable to derive from - * @param compute A function that takes parameters and returns SqlSeachOptions to filter the SyncTable - * @returns A function that takes parameters and returns a GetState of the derived documents - */ -export function selectSql( - state: SyncTable, - compute: (...args: Params) => SqlSeachOptions, -): CreateState { - const { subscribe, updateSearchOptions } = state - const componentId = getStateId() - const result: CreateState = (...params) => { - const searchId = getStateId() - const destroy = subscribe(searchId, componentId, () => { - getState.emitter.emit() - }) - - const options = compute(...params) - const getState = createState({ - destroy() { - destroy() - getState.emitter.clear() - getState.cache.current = undefined - }, - get() { - return state.getSnapshot(searchId) - }, - getSnapshot() { - return state.getSnapshot(searchId) - }, - }) - updateSearchOptions(searchId, options) - - return getState - } - return result -} diff --git a/packages/core/sqlite/table/table.ts b/packages/core/sqlite/table/table.ts index 9d6c5ca..62ed851 100644 --- a/packages/core/sqlite/table/table.ts +++ b/packages/core/sqlite/table/table.ts @@ -9,7 +9,7 @@ import type { Where } from './where' import { getWhereQuery } from './where' const DELETE_IN_CHUNK = 500 -export const DEFAULT_STEP_SIZE = 100 +export const DEFAULT_PAGE_SIZE = 100 /** * Convert a dot-separated path to a JSON path @@ -239,7 +239,7 @@ export async function createTable(options: DbOptions document as unknown as Selected, - stepSize = DEFAULT_STEP_SIZE, + pageSize = DEFAULT_PAGE_SIZE, } = options const whereSql = getWhereQuery(where, tableName) @@ -256,7 +256,7 @@ export async function createTable(options: DbOptions>(query) diff --git a/packages/core/sqlite/table/table.types.ts b/packages/core/sqlite/table/table.types.ts index 9163fcb..497b60e 100644 --- a/packages/core/sqlite/table/table.types.ts +++ b/packages/core/sqlite/table/table.types.ts @@ -1,4 +1,3 @@ -import type { SqlSeachOptions } from '../select-sql' import type { Backend } from './backend' import type { FtsTokenizerOptions } from './tokenizer' import type { Where } from './where' @@ -7,6 +6,15 @@ import type { Where } from './where' export type DocType = { [key: string]: any } export type KeyTypeAvailable = 'string' | 'number' +export interface SqlSeachOptions { + readonly sortBy?: DotPath + readonly order?: 'asc' | 'desc' + readonly limit?: number + readonly offset?: number + readonly where?: Where + readonly pageSize?: number +} + // Expand all nested keys into dot-paths export type DotPrefix = T extends '' ? '' : `.${T}` diff --git a/packages/core/sqlite/use-sqlite-count.ts b/packages/core/sqlite/use-sqlite-count.ts new file mode 100644 index 0000000..333c5d4 --- /dev/null +++ b/packages/core/sqlite/use-sqlite-count.ts @@ -0,0 +1,69 @@ +import { useCallback, useEffect, useLayoutEffect, useReducer, useRef, type DependencyList } from 'react' +import type { SyncTable } from './create-sqlite' +import type { DocType } from './table/table.types' +import type { Where } from './table/where' + +/** + * A React hook to count the number of items in a SyncTable reactively. + * It updates the count when items are inserted or deleted, but ignores updates. + * Supports filtering the count using a `where` clause. + * @param state The SyncTable instance to observe. + * @param options Optional filtering options. + * @param options.where A `where` clause to filter the count. + * @param deps Dependency list to control when to re-run the effect. + * @returns The current count of items in the table. + */ +export function useSqliteCount( + state: SyncTable, + options: { where?: Where } = {}, + deps: DependencyList = [], +): number { + const countRef = useRef(0) + const [, rerender] = useReducer((c: number) => c + 1, 0) + + const updateCount = useCallback(async () => { + const newCount = await state.count(options) + countRef.current = newCount + rerender() + // eslint-disable-next-line react-hooks/exhaustive-deps + }, deps) + + useEffect(() => { + updateCount() + // eslint-disable-next-line react-hooks/exhaustive-deps + }, deps) + + useLayoutEffect(() => { + const unsubscribe = state.subscribe((item) => { + const { mutations, removedAll } = item + if (removedAll) { + countRef.current = 0 + rerender() + return + } + if (!mutations) { + return + } + + let shouldUpdate = false + for (const mutation of mutations) { + const { op } = mutation + if (op === 'insert' || op === 'delete') { + shouldUpdate = true + break + } + } + + if (shouldUpdate) { + updateCount() + } + }) + + return () => { + unsubscribe() + } + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [state]) + + return countRef.current +} diff --git a/packages/core/sqlite/use-sqlite.ts b/packages/core/sqlite/use-sqlite.ts index 3814d29..42271c6 100644 --- a/packages/core/sqlite/use-sqlite.ts +++ b/packages/core/sqlite/use-sqlite.ts @@ -1,21 +1,22 @@ -import { useCallback, useDebugValue, useEffect, useId, useLayoutEffect, useMemo, type DependencyList } from 'react' +/* eslint-disable sonarjs/cognitive-complexity */ +import { useCallback, useLayoutEffect, useReducer, useRef, type DependencyList } from 'react' import type { SyncTable } from './create-sqlite' -import type { DocType } from './table/table.types' -import { isError, isPromise } from '../utils/is' -import { useSyncExternalStoreWithSelector } from 'use-sync-external-store/shim/with-selector' -import type { SqlSeachOptions } from './select-sql' +import type { DocType, Key, SqlSeachOptions } from './table/table.types' +import { DEFAULT_PAGE_SIZE } from './table' +const MAX_ITERATIONS = 10_000 export interface SqLiteActions { /** * Load the next page of results and return if isDone to show more results. * @returns isDone: boolean */ - readonly next: () => Promise + readonly nextPage: () => Promise /** * Reset the pagination and load the first page of results. * @returns void */ readonly reset: () => Promise + readonly keysIndex: Map } export interface UseSearchOptions extends SqlSeachOptions { @@ -26,102 +27,160 @@ export interface UseSearchOptions } /** - * Generate a cache key based on the search options to uniquely identify the query - * @param options The search options to generate the key from - * @returns A string representing the unique cache key for the given search options - */ -function generateCacheKey(options: UseSearchOptions): string { - const { limit, offset, order, sortBy, where, stepSize, select } = options - let key = '' - if (limit !== undefined) { - key += `l${limit}` - } - if (offset !== undefined) { - key += `o${offset}` - } - if (order !== undefined) { - key += `r${order}` - } - if (sortBy !== undefined) { - key += `s${sortBy}` - } - if (where !== undefined) { - key += `w${JSON.stringify(where)}` - } - if (stepSize !== undefined) { - key += `t${stepSize}` - } - if (select !== undefined) { - key += `f${select.toString()}` - } - return key -} - -/** - * React hook to subscribe to a SyncTable and get its current snapshot, with optional search options and selector for derived state - * @param state The SyncTable to subscribe to - * @param options Optional search options to filter and sort the documents - * @param deps Dependency list to control when to update the search options - * @returns A tuple containing the current array of documents (or selected documents) and an object with actions to interact with the SyncTable - * @throws If the value is a Promise or an Error, it will be thrown to be handled by an error boundary or suspense + * A React hook to perform paginated searches on a SyncTable and reactively update the results. + * It supports pagination, resetting the search, and selecting specific fields from the documents. + * @param state The SyncTable instance to perform searches on. + * @param options Options to customize the search behavior, including pagination size and selection function. + * @param deps Dependency list to control when to re-run the search and reset the iterator. + * @returns A tuple containing the current list of results and an object with actions to manage pagination and resetting. */ export function useSqliteValue( state: SyncTable, options: UseSearchOptions = {}, deps: DependencyList = [], -): [undefined extends Selected ? Document[] : Selected[], SqLiteActions] { - const { select } = options +): [(undefined extends Selected ? Document[] : Selected[]) | undefined, SqLiteActions] { + const { select, pageSize = DEFAULT_PAGE_SIZE } = options - const searchId = useMemo(() => generateCacheKey({ ...options, select: undefined }), [options]) - const componentId = useId() + const itemsRef = useRef() + const [, rerender] = useReducer((c: number) => c + 1, 0) + const keysIndex = useRef(new Map()) + const iteratorRef = useRef>() - useLayoutEffect(() => { - state.updateSearchOptions(searchId, { ...options, select: undefined }) + const updateIterator = useCallback(() => { + // eslint-disable-next-line sonarjs/no-unused-vars + const { select: _ignore, ...resetOptions } = options + iteratorRef.current = state.search({ select: (doc, meta) => ({ doc, meta }), ...resetOptions }) // eslint-disable-next-line react-hooks/exhaustive-deps - }, deps) + }, [state, ...deps]) - useEffect(() => { - // state.load(searchId) - return () => { - state.clear(searchId) + const reset = useCallback(() => { + itemsRef.current = [] + keysIndex.current.clear() + updateIterator() + }, [updateIterator]) + + const fillNextPage = useCallback(async (shouldReset: boolean) => { + if (itemsRef.current === undefined) { + itemsRef.current = [] + } + if (shouldReset === true) { + reset() + } + + const { current: iterator } = iteratorRef + if (!iterator) { + return true + } + let isDone = false + for (let index = 0; index < pageSize; index++) { + const result = await iterator.next() + if (result.done) { + iteratorRef.current = undefined + isDone = true + break + } + if (keysIndex.current.has(result.value.meta.key)) { + // eslint-disable-next-line sonarjs/updated-loop-counter + index += -1 + continue + } + itemsRef.current.push(select ? select(result.value.doc) : (result.value.doc as unknown as Selected)) + keysIndex.current.set(result.value.meta.key, itemsRef.current.length - 1) } + return isDone // eslint-disable-next-line react-hooks/exhaustive-deps }, []) - const selector = useCallback( - (documents: Document[]) => { - // eslint-disable-next-line unicorn/no-array-callback-reference - return select ? documents.map(select) : (documents as unknown as Selected[]) - }, - [select], - ) - - const subscribe = useCallback( - (onStorageChange: () => void) => { - return state.subscribe(searchId, componentId, onStorageChange) - }, - [state, searchId, componentId], - ) - - const getSnapshot = useCallback(() => { - return state.getSnapshot(searchId) - }, [state, searchId]) - - const value = useSyncExternalStoreWithSelector(subscribe, getSnapshot, getSnapshot, selector) - - useDebugValue(value) - if (isPromise(value)) { - throw value - } - if (isError(value)) { - throw value - } - - const actions = useMemo((): SqLiteActions => { - return { - next: () => state.next(searchId), - reset: () => state.refresh(searchId), + const nextPage = useCallback(async () => { + const isDone = await fillNextPage(false) + rerender() + return isDone + }, [fillNextPage]) + + useLayoutEffect(() => { + const unsubscribe = state.subscribe(async (item) => { + const { mutations, removedAll } = item + if (removedAll) { + reset() + } + if (!mutations) { + return + } + + const oldLength = itemsRef.current?.length ?? 0 + let newLength = oldLength + let hasUpdate = false + for (const mutation of mutations) { + const { key, op } = mutation + switch (op) { + case 'insert': { + newLength += 1 + break + } + case 'delete': { + if (itemsRef.current && keysIndex.current.has(key)) { + const index = keysIndex.current.get(key) + if (index === undefined) break + itemsRef.current.splice(index, 1) + keysIndex.current.delete(key) + hasUpdate = true + } + break + } + case 'update': { + if (itemsRef.current && keysIndex.current.has(key)) { + const index = keysIndex.current.get(key) + if (index === undefined) break + itemsRef.current[index] = (await state.get(key, select)) as Selected + hasUpdate = true + } + break + } + } + } + + const isLengthChanged = oldLength !== newLength + const isChanged = isLengthChanged || hasUpdate + if (!isChanged) return + if (isLengthChanged) { + await fillNextPage(true) + + // here we ensure that if the length changed, we fill the next page + + let iterations = 0 + while ((itemsRef.current?.length ?? 0) < newLength && iterations < MAX_ITERATIONS) { + await fillNextPage(false) + iterations++ + } + if (iterations === MAX_ITERATIONS) { + // Optionally log a warning to help with debugging + // eslint-disable-next-line no-console + console.warn('Reached maximum iterations in fillNextPage loop. Possible duplicate or data issue.') + } + } + rerender() + }) + return () => { + unsubscribe() } - }, [searchId, state]) - return [value as undefined extends Selected ? Document[] : Selected[], actions] + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [state]) + + useLayoutEffect(() => { + updateIterator() + itemsRef.current = undefined + keysIndex.current.clear() + nextPage() + // eslint-disable-next-line react-hooks/exhaustive-deps + }, deps) + + const resetCb = useCallback(async () => { + reset() + await nextPage() + }, [nextPage, reset]) + + return [itemsRef.current, { nextPage, reset: resetCb, keysIndex: keysIndex.current }] as [ + (undefined extends Selected ? Document[] : Selected[]) | undefined, + SqLiteActions, + ] }