From 559cfc9f4565c9ee69949bef22493f391b43d90f Mon Sep 17 00:00:00 2001 From: jstet Date: Wed, 19 Aug 2026 10:05:01 +0200 Subject: [PATCH 1/7] chore: make the quality gates pass again The committed tree failed several `npm run validate` gates: - prettier: seven src files were committed unformatted - markdownlint: ARCHITECTURE.md had list-spacing and trailing-newline errors - knip: `@registry/*` (a docs vite alias, not a package) was reported as an unlisted dependency, and the entry/project config belonged under a `workspaces` block now that the repo has one Also drop `docker-compose` and `ruff` from knip's ignoreBinaries (neither is invoked anywhere) and tighten the lint gate to `--max-warnings 0`, which the tree already satisfies. --- ARCHITECTURE.md | 4 +- knip.json | 16 ++++-- package.json | 2 +- src/ddi/codebook.ts | 14 ++--- src/pipelines/lstsv2ddi/toVariables.ts | 21 ++++++-- src/pipelines/lstsv2xlsform/emParser.ts | 15 ++---- src/pipelines/lstsv2xlsform/toXlsform.ts | 8 ++- src/pipelines/xlsform2ddi/variables.ts | 13 +++-- src/pipelines/xlsform2lstsv/groupProcessor.ts | 6 ++- .../xlsform2lstsv/xpathTranspiler.ts | 52 ++++++++++++------- 10 files changed, 92 insertions(+), 59 deletions(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index bca8f0e..fffcc46 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -58,6 +58,7 @@ Python package (run `uv run codegen` or `python -m codegen`) that validates the ### Tests (`tests/`) Everything that isn't a TypeScript unit test, split by subject: + - `tests/validation/` — pytest: committed DDI snapshots vs XSD + CDL Schematron, XLSForm fixtures vs the pyxform oracle - `tests/live/` — pytest, docker: real survey engines — import, respond, read the stored data back - `tests/fixtures/surveys/` — hand-authored multi-question surveys shared by both @@ -85,6 +86,7 @@ Both DDI pipelines converge on the same emitter: they produce `Variable[]` (`src Deliberate design decision. **DDI is the terminus of the pipeline graph** — it describes a *dataset*, not an *instrument*, so it does not carry the information a survey needs to run. The canonical `Variable` (`src/ddi/types.ts`) is what survives an emit. Everything that makes a form behave is absent: + - **no `relevant`** — DDI Codebook 2.5 has no machine-readable expression syntax at all, so skip logic is dropped on the way in - **no `constraint`** — same reason - **no `required`, `default`, `hint`, per-question `appearance`, `calculation`** @@ -113,4 +115,4 @@ Compare `lstsv2xlsform`, which *is* implemented: a LimeSurvey structure TSV carr 1. Edit `registry/schema.jsonld` with field definitions 2. Update `consumedBy` arrays to indicate which outputs use each field 3. Run `uv run codegen` to validate and regenerate artifacts -4. Run tests to ensure nothing broke \ No newline at end of file +4. Run tests to ensure nothing broke diff --git a/knip.json b/knip.json index 9db96e8..c3239b2 100644 --- a/knip.json +++ b/knip.json @@ -1,15 +1,21 @@ { "$schema": "https://unpkg.com/knip@6/schema.json", - "entry": ["src/generateFixtures.ts"], - "project": ["src/**/*.ts"], - "ignore": ["src/generated/**", "src/**/*.test.ts", "tests/**"], - "ignoreBinaries": ["docker-compose", "uv", "ruff"], - "ignoreDependencies": [], + "ignoreBinaries": ["uv"], "rules": { "exports": "off", "types": "off", "nsExports": "off", "nsTypes": "off", "enumMembers": "off" + }, + "workspaces": { + ".": { + "entry": ["src/generateFixtures.ts"], + "project": ["src/**/*.ts"], + "ignore": ["src/generated/**"] + }, + "docs": { + "ignoreDependencies": ["@registry/.*"] + } } } diff --git a/package.json b/package.json index 41ac288..dbcea97 100644 --- a/package.json +++ b/package.json @@ -35,7 +35,7 @@ "test:all": "npm run test && npm run test:py && npm run test:live", "bless": "bash scripts/bless.sh", "fixtures:generate": "npm run build && node dist/generateFixtures.js", - "lint": "eslint src --max-warnings 12", + "lint": "eslint src --max-warnings 0", "lint:fix": "eslint src --fix", "format": "prettier --write src/", "format:check": "prettier --check src/", diff --git a/src/ddi/codebook.ts b/src/ddi/codebook.ts index 8924ff5..fcd38cc 100644 --- a/src/ddi/codebook.ts +++ b/src/ddi/codebook.ts @@ -403,7 +403,8 @@ function addVars( for (const [groupName, members] of gridGroups) { const groupLabel = getGroupLabel(dataVars, groupName); members.forEach((v, i) => { - const pre = i === 0 ? combineNote(notePreqtxt, v.name, groupLabel) : groupLabel; + const pre = + i === 0 ? combineNote(notePreqtxt, v.name, groupLabel) : groupLabel; addVarElement(dataDscr, { varId: makeVarId(v.name), name: v.name, @@ -417,7 +418,8 @@ function addVars( for (const [smName, smVar] of multiRespGroups) { smVar.choices.forEach((choice, i) => { - const stem = i === 0 ? combineNote(notePreqtxt, smName, smVar.label) : smVar.label; + const stem = + i === 0 ? combineNote(notePreqtxt, smName, smVar.label) : smVar.label; addBinaryVar( dataDscr, makeVarId(`${smName}_${choice.name}`), @@ -481,13 +483,7 @@ export function buildDdiCodebook( const dataDscr = root.child('dataDscr'); const buckets = splitDataVars(dataVars); addVarGroups(dataDscr, dataVars, buckets, buckets.otherPatterns); - addVars( - dataDscr, - dataVars, - inlinePreqtxt, - buckets, - buckets.otherPatterns, - ); + addVars(dataDscr, dataVars, inlinePreqtxt, buckets, buckets.otherPatterns); return root; } diff --git a/src/pipelines/lstsv2ddi/toVariables.ts b/src/pipelines/lstsv2ddi/toVariables.ts index 3447826..eee8a3d 100644 --- a/src/pipelines/lstsv2ddi/toVariables.ts +++ b/src/pipelines/lstsv2ddi/toVariables.ts @@ -246,7 +246,13 @@ function processQuestionRow( const label = cell(row, 'text'); const cdlVocab = vocabFromCssClass(cell(row, 'cssclass')); - const variable = buildQuestionVar(lsType, name, label, cdlVocab, currentGroup); + const variable = buildQuestionVar( + lsType, + name, + label, + cdlVocab, + currentGroup, + ); if (cell(row, 'other') === 'Y' && isOtherEligibleSelect(variable)) { otherSelects.add(variable.name); @@ -254,7 +260,11 @@ function processQuestionRow( renameOtherCompanion(variable, otherSelects); // from_file selects (vocab set) inline options in the TSV that DDI drops. - return { kind: 'question', variable, currentVar: variable.vocab ? null : variable }; + return { + kind: 'question', + variable, + currentVar: variable.vocab ? null : variable, + }; } /** Materialise `array` into one `select_one` Variable per subquestion. */ @@ -349,7 +359,12 @@ export function lstsvToVariables(rows: Row[]): Variable[] { } if (cls === 'Q') { - const outcome = processQuestionRow(row, currentGroup, otherSelects, flushArray); + const outcome = processQuestionRow( + row, + currentGroup, + otherSelects, + flushArray, + ); if (outcome.kind === 'array') { array = outcome.array; continue; diff --git a/src/pipelines/lstsv2xlsform/emParser.ts b/src/pipelines/lstsv2xlsform/emParser.ts index c557d46..387c54c 100644 --- a/src/pipelines/lstsv2xlsform/emParser.ts +++ b/src/pipelines/lstsv2xlsform/emParser.ts @@ -74,30 +74,21 @@ function scanString( } /** Numeric literal (digits + at most one dot) starting at `i`. */ -function scanNumber( - src: string, - i: number, -): { token: Token; next: number } { +function scanNumber(src: string, i: number): { token: Token; next: number } { let j = i; while (j < src.length && /[0-9.]/.test(src[j])) j++; return { token: { type: 'num', value: src.slice(i, j) }, next: j }; } /** Identifier (`[A-Za-z_][A-Za-z0-9_.]*`) — possibly `.NAOK` suffixed. */ -function scanIdent( - src: string, - i: number, -): { token: Token; next: number } { +function scanIdent(src: string, i: number): { token: Token; next: number } { let j = i; while (j < src.length && /[A-Za-z0-9_.]/.test(src[j])) j++; return { token: { type: 'ident', value: src.slice(i, j) }, next: j }; } /** Two-char operator if present, else single-char. Unknowns throw. */ -function scanOperator( - src: string, - i: number, -): { token: Token; next: number } { +function scanOperator(src: string, i: number): { token: Token; next: number } { const two = src.slice(i, i + 2); if (MULTI_CHAR_OPS.includes(two)) { return { token: { type: 'op', value: two }, next: i + 2 }; diff --git a/src/pipelines/lstsv2xlsform/toXlsform.ts b/src/pipelines/lstsv2xlsform/toXlsform.ts index d689676..358e90b 100644 --- a/src/pipelines/lstsv2xlsform/toXlsform.ts +++ b/src/pipelines/lstsv2xlsform/toXlsform.ts @@ -381,7 +381,9 @@ function collectSelectMultiples( ): Array<{ name: string; codes: string[] }> { const out: Array<{ name: string; codes: string[] }> = []; for (const bucket of buckets) { - const baseRows = bucket.rows.filter((r) => cell(r, 'language') === baseLanguage); + const baseRows = bucket.rows.filter( + (r) => cell(r, 'language') === baseLanguage, + ); const { items, choicesByName } = readLogicalQuestions(baseRows, languages); for (const item of items) { if ( @@ -580,7 +582,9 @@ function composeTypeWithList( const vocab = vocabFromCssClass(item.cssclass); if (vocab) { const fromFile = - base === 'select_one' ? 'select_one_from_file' : 'select_multiple_from_file'; + base === 'select_one' + ? 'select_one_from_file' + : 'select_multiple_from_file'; return { type: `${fromFile} ${vocab}.csv`, emittedChoices: false }; } if (base === 'select_one' || base === 'select_multiple') { diff --git a/src/pipelines/xlsform2ddi/variables.ts b/src/pipelines/xlsform2ddi/variables.ts index ce6eb88..e8da2da 100644 --- a/src/pipelines/xlsform2ddi/variables.ts +++ b/src/pipelines/xlsform2ddi/variables.ts @@ -217,9 +217,10 @@ function closeGroup(state: ExtractState): void { } /** Inner-most enclosing group's stored label/appearance (defaults if outside any group). */ -function currentGroupMeta( - state: ExtractState, -): { label: string; appearance: string } { +function currentGroupMeta(state: ExtractState): { + label: string; + appearance: string; +} { const cur = state.groupStack[state.groupStack.length - 1] ?? ''; return state.groupMeta[cur] ?? { label: '', appearance: '' }; } @@ -257,7 +258,7 @@ function pushQuestionRow( const group = state.groupStack.join('/'); const gm = currentGroupMeta(state); - const baseChoices = listName ? state.choicesByList[listName] ?? [] : []; + const baseChoices = listName ? (state.choicesByList[listName] ?? []) : []; const choices = expandedChoices(baseChoices, stdType, rawType, state.lang); state.variables.push({ @@ -326,9 +327,7 @@ export function extractVariables( } else if (kind === 'skip') { continue; } else if ( - NO_DATA_APPEARANCES.has( - toStr(row['appearance']).trim().toLowerCase(), - ) + NO_DATA_APPEARANCES.has(toStr(row['appearance']).trim().toLowerCase()) ) { continue; } else { diff --git a/src/pipelines/xlsform2lstsv/groupProcessor.ts b/src/pipelines/xlsform2lstsv/groupProcessor.ts index 4901a32..3595f53 100644 --- a/src/pipelines/xlsform2lstsv/groupProcessor.ts +++ b/src/pipelines/xlsform2lstsv/groupProcessor.ts @@ -97,7 +97,11 @@ export class GroupProcessor { } continue; } - if (!type || stack.length === 0 || SKIP_TYPES.includes(extractBaseType(row))) + if ( + !type || + stack.length === 0 || + SKIP_TYPES.includes(extractBaseType(row)) + ) continue; const info = groupInfo.get(stack[stack.length - 1]); if (!info) continue; diff --git a/src/pipelines/xlsform2lstsv/xpathTranspiler.ts b/src/pipelines/xlsform2lstsv/xpathTranspiler.ts index 1b47614..f0f5910 100644 --- a/src/pipelines/xlsform2lstsv/xpathTranspiler.ts +++ b/src/pipelines/xlsform2lstsv/xpathTranspiler.ts @@ -67,10 +67,11 @@ function sanitizeName(name: string): string { } /** All positional args joined with `, ` (XPath variadic → EM list). */ -function joinArgs(args: unknown[] | undefined, ctx?: TranspilerContext): string { - return (args ?? []) - .map((a) => transpile(a as XPathNode, ctx)) - .join(', '); +function joinArgs( + args: unknown[] | undefined, + ctx?: TranspilerContext, +): string { + return (args ?? []).map((a) => transpile(a as XPathNode, ctx)).join(', '); } /** Pass-through: every arg becomes `prefix(argN)`. */ @@ -103,9 +104,7 @@ const FUNCTION_HANDLERS: Record< // variadic count: (args, ctx) => wrapArgs('count', args, ctx), concat: (args, ctx) => - (args ?? []) - .map((a) => transpile(a as XPathNode, ctx)) - .join(' + ') || '', + (args ?? []).map((a) => transpile(a as XPathNode, ctx)).join(' + ') || '', regex: (args, ctx) => wrapArgs('regexMatch', args, ctx), // 2-arg contains: (args, ctx) => @@ -152,7 +151,10 @@ function rewriteWithAnswerLookup( } /** String-valued function calls that need custom logic beyond a single Map entry. */ -function transpileSelected(args: unknown[] | undefined, ctx?: TranspilerContext): string { +function transpileSelected( + args: unknown[] | undefined, + ctx?: TranspilerContext, +): string { if (args?.length !== 2) throw new Error('selected() needs 2 arguments'); const fieldArg = args[0] as XPathNode; const valueArg = args[1] as XPathNode; @@ -170,17 +172,20 @@ function transpileSubstring( args: unknown[] | undefined, ctx?: TranspilerContext, ): string { - if (!args || args.length < 2) throw new Error('substring() needs ≥2 arguments'); + if (!args || args.length < 2) + throw new Error('substring() needs ≥2 arguments'); const stringArg = transpile(args[0] as XPathNode, ctx); const startArg = transpile(args[1] as XPathNode, ctx); - const lengthArg = - args.length > 2 ? transpile(args[2] as XPathNode, ctx) : ''; + const lengthArg = args.length > 2 ? transpile(args[2] as XPathNode, ctx) : ''; return `substr(${stringArg}, ${startArg}${lengthArg ? ', ' + lengthArg : ''})`; } /** `selected(${field}, 'value')` and `substring(...)` need custom logic; dispatch * any other known function via the static table. */ -function transpileFunctionCall(node: XPathNode, ctx?: TranspilerContext): string { +function transpileFunctionCall( + node: XPathNode, + ctx?: TranspilerContext, +): string { const id = node.id!; const args = node.args; if (id === 'selected') return transpileSelected(args, ctx); @@ -238,11 +243,16 @@ function transpileBinaryOp(node: XPathNode, ctx?: TranspilerContext): string { } /** Variable references become the sanitized, possibly truncated field name. */ -function transpileVariableRef(node: XPathNode, ctx?: TranspilerContext): string { +function transpileVariableRef( + node: XPathNode, + ctx?: TranspilerContext, +): string { const step = node.steps![0]; if (!step.name) return 'self'; const fieldName = sanitizeName(step.name); - return ctx?.getTruncatedFieldName ? ctx.getTruncatedFieldName(fieldName) : fieldName; + return ctx?.getTruncatedFieldName + ? ctx.getTruncatedFieldName(fieldName) + : fieldName; } /** Literal (string / numeric) values. `valueDisplay` carries the original quoted form. */ @@ -279,7 +289,8 @@ function transpile(node: XPathNode, ctx?: TranspilerContext): string { if (!node) return ''; if (node.id) return transpileFunctionCall(node, ctx); if (node.type) return transpileBinaryOp(node, ctx); - if (node.steps && node.steps.length > 0) return transpileVariableRef(node, ctx); + if (node.steps && node.steps.length > 0) + return transpileVariableRef(node, ctx); if (node.value !== undefined) return transpileLiteral(node); throw new Error(`Unsupported node structure: ${JSON.stringify(node)}`); } @@ -339,7 +350,8 @@ const LOGICAL_OPERATORS = ['>=', '<=', '>', '<', '=', '!=', 'and', 'or']; function firstArgLooksLogical(firstArg: string): boolean { return LOGICAL_OPERATORS.some( (op) => - firstArg.includes(op) && !(firstArg.includes('[') && firstArg.includes(']')), + firstArg.includes(op) && + !(firstArg.includes('[') && firstArg.includes(']')), ); } @@ -358,9 +370,13 @@ function firstArgLooksLikePattern(firstArg: string): boolean { } /** Apply the `regexMatch(, )` reconstruction rule to two parsed args. */ -function reconstructRegexMatch(firstArg: string, secondArg: string): string | null { +function reconstructRegexMatch( + firstArg: string, + secondArg: string, +): string | null { if (firstArgLooksLogical(firstArg)) return firstArg.replace(/^"|"$/g, ''); - if (!secondArgIsFieldRef(secondArg) || !firstArgLooksLikePattern(firstArg)) return null; + if (!secondArgIsFieldRef(secondArg) || !firstArgLooksLikePattern(firstArg)) + return null; const processedFieldArg = secondArg.replace(/\./g, 'self'); const processedPatternArg = firstArg .replace(/^"|"$/g, "'") From e9820fcddb68e46261d514fa6865af648f47cc87 Mon Sep 17 00:00:00 2001 From: jstet Date: Wed, 19 Aug 2026 10:05:16 +0200 Subject: [PATCH 2/7] refactor(codegen): emit markdownlint-clean skill references The two files under skills/cdl-survey-types/references/ are codegen output, so their markdownlint errors could only be fixed at the source: - consecutive per-type warnings were emitted as separate blockquotes separated by a blank line (MD028); they are now one `>`-joined blockquote - the names/choice-codes list had no blank line above it (MD032) - blocks each carry a trailing newline and are then joined with another, which left double blank lines at the seams (MD012); a new `_tidy()` normalises the whole document once instead of making every emitter track blank lines Splitting the two emitters keeps them under the C901 complexity limit that `ruff check` enforces: `_question_types_md` now delegates to `_variants_by_base`, `_constraint_lines` and `_type_section`, and `_syntax_md` walks a (rule, section-builder) table instead of one long chain of `if` blocks. Regenerated output is byte-identical apart from the whitespace fixes above. --- codegen/emit_skill.py | 407 ++++++++++-------- .../references/question-types.md | 3 +- .../references/xlsform-syntax.md | 4 +- 3 files changed, 229 insertions(+), 185 deletions(-) diff --git a/codegen/emit_skill.py b/codegen/emit_skill.py index 31a3545..f739c11 100644 --- a/codegen/emit_skill.py +++ b/codegen/emit_skill.py @@ -18,6 +18,7 @@ """ import json +import re from pathlib import Path from typing import Any @@ -29,6 +30,16 @@ ) +def _tidy(md: str) -> str: + """Collapse runs of blank lines and end with exactly one newline. + + The emitters below build markdown by appending blocks that each carry their own + trailing newline, which leaves double blank lines at the seams (markdownlint + MD012). Normalising once here keeps every emitter free of blank-line bookkeeping. + """ + return re.sub(r"\n{3,}", "\n\n", md).rstrip("\n") + "\n" + + def _load_example(base_dir: Path, entry: dict[str, Any]) -> dict[str, Any] | None: """Read an entity's xlsform.json worked example, if it has one on disk.""" ex = entry.get("examplePath") or (f"{entry['exampleDir']}/xlsform.json" if entry.get("exampleDir") else None) @@ -117,82 +128,222 @@ def _skill_md(registry: dict[str, Any]) -> str: ) -def _question_types_md(registry: dict[str, Any], base_dir: Path) -> str: - qtypes = sorted( - (d for d in registry.values() if d.get("@type") == "QuestionType"), - key=lambda d: d["xlsform"]["typeString"], - ) - variants_by_base: dict[str, list[dict]] = {} +def _variants_by_base(registry: dict[str, Any]) -> dict[str, list[dict]]: + """Variant entries grouped by the `@id` of the base type they narrow.""" + grouped: dict[str, list[dict]] = {} for d in registry.values(): if d.get("@type") != "QuestionTypeVariant": continue base = d.get("skos:broader", {}) base_id = base.get("@id", "") if isinstance(base, dict) else "" - variants_by_base.setdefault(base_id, []).append(d) + grouped.setdefault(base_id, []).append(d) + return grouped - out = [_DO_NOT_EDIT, "# Question Types (detail)\n"] - for d in qtypes: - ts = d["xlsform"]["typeString"] - out.append(f"## `{ts}` — {d.get('skos:prefLabel', ts)}\n") - out.append(f"**Use when:** {d.get('useWhen', '—')}\n") +def _constraint_lines(constraints: dict[str, Any]) -> list[str]: + """Constraints paragraph + warning blockquote for one type.""" + out: list[str] = [] + cl = [] + if "maxNameLength" in constraints: + cl.append( + f"variable `name` \u2264 {constraints['maxNameLength']} chars, `{constraints.get('namePattern', '')}`" + ) + if "maxChoiceCodeLength" in constraints: + cl.append( + f"choice code \u2264 {constraints['maxChoiceCodeLength']} chars, " + f"`{constraints.get('choiceCodePattern', '')}`" + ) + if cl: + out.append("**Constraints:** " + "; ".join(cl) + "\n") + warnings = constraints.get("warnings", []) + if warnings: + # One blockquote, `>`-joined: separate quotes with a blank line between them + # would be two blockquotes (markdownlint MD028). + out.append("\n>\n".join(f"> \u26a0\ufe0f {w}" for w in warnings) + "\n") + return out + + +def _type_section(d: dict[str, Any], variants: list[dict], base_dir: Path) -> list[str]: + """The `## ` block: header, use-when, aliases, constraints, variants, examples.""" + ts = d["xlsform"]["typeString"] + out = [ + f"## `{ts}` \u2014 {d.get('skos:prefLabel', ts)}\n", + f"**Use when:** {d.get('useWhen', '\u2014')}\n", + ] + + aliases = d["xlsform"].get("aliases") + if aliases: + out.append(f"**Accepted aliases:** {', '.join(f'`{a}`' for a in aliases)}\n") + if d["xlsform"].get("requiresListName"): + out.append( + "**Requires a choice list:** write the type cell as " + f"`{ts} ` and define the options on the `choices` sheet under " + "that `list_name`.\n" + ) - aliases = d["xlsform"].get("aliases") - if aliases: - out.append(f"**Accepted aliases:** {', '.join(f'`{a}`' for a in aliases)}\n") - if d["xlsform"].get("requiresListName"): + out += _constraint_lines(d.get("constraints", {})) + + vs = sorted(variants, key=lambda x: x["@id"]) + if vs: + out.append("**Variants:**\n") + for v in vs: + out.append( + f"- `{v['@id'].split(':', 1)[1]}` \u2014 {v.get('skos:prefLabel', '')}: {v.get('useWhen', '\u2014')}" + ) + out.append("") + + example = _load_example(base_dir, d) + if example: + out.append("**Example:**\n") + out.append(_fmt_example(example) + "\n") + + # Worked examples for this type's variants (e.g. the _other companion row). + for v in vs: + vex = _load_example(base_dir, v) + if vex: + out.append(f"**Example \u2014 `{v['@id'].split(':', 1)[1]}`:**\n") + out.append(_fmt_example(vex) + "\n") + + return out + + +def _question_types_md(registry: dict[str, Any], base_dir: Path) -> str: + qtypes = sorted( + (d for d in registry.values() if d.get("@type") == "QuestionType"), + key=lambda d: d["xlsform"]["typeString"], + ) + variants_by_base = _variants_by_base(registry) + + out = [_DO_NOT_EDIT, "# Question Types (detail)\n"] + for d in qtypes: + out += _type_section(d, variants_by_base.get(d["@id"], []), base_dir) + + return _tidy("\n".join(out)) + + +def _naming_section(san: dict) -> list[str]: + n = san.get("name", {}) + cc = san.get("choiceCode", {}) + return [ + "## Names & choice codes\n", + "Author names/codes in English `snake_case`. Downstream (LimeSurvey) " + f"sanitization strips `{n.get('stripCharsRegex', '[_-]')}` " + f"to the pattern `{n.get('pattern', '')}` and truncates, so keep the " + "alphanumeric stem short and unambiguous:\n", + f"- **Variable `name`:** \u2264 {n.get('maxLength', '?')} chars after stripping. " + "Must be unique \u2014 duplicates get a numeric suffix.", + f"- **Choice `name` (code):** \u2264 {cc.get('maxLength', '?')} chars after stripping " + "\u2014 longer codes are truncated in LimeSurvey, so short codes (or plain integers, " + "as in the examples) are safest.", + "- Labels/hints carry the human text; keep names/codes machine-stable.\n", + ] + + +def _allowlist_section(unreg: dict) -> list[str]: + meta_rows = ", ".join(f"`{r}`" for r in unreg.get("metadataRowTypes", [])) + return [ + "## Row types are an allowlist\n", + f"{unreg.get('unknownTypeBehavior', '')}\n\n" + f"Metadata rows ({meta_rows}) are accepted and carried but produce no authored " + "question. Every other `type` must be one of the registered question / structural " + "types.\n", + ] + + +def _other_section(other: dict) -> list[str]: + code = other.get("choiceCode", "other") + return [ + "## `or_other` pattern (open \u201cOther\u201d field)\n", + f"Applies to: {', '.join(f'`{t}`' for t in other.get('appliesTo', []))}. " + f"Add a choice with code `{code}`, then a companion " + f"row of type `{other.get('companionType', 'text')}` named " + f"`{other.get('companionSuffix', '_other')}` shown only when the " + "\u201cother\u201d choice is picked \u2014 set its `relevant` to `${} = " + f"'{code}'`. See the `select_one_other` example.\n", + ] + + +def _external_list_section(ext: dict, vocabs: list[dict]) -> list[str]: + # Example filename comes from the registry, not a hardcoded vocabulary: + # vocabularies are open-ended (convention:externalCodeList's + # vocabularyDeclaration), so the skill doc must not pin one. + example_file = vocabs[0].get("xlsformFilename", ".csv") if vocabs else ".csv" + out = [ + "## Long lists from a file (`select_*_from_file`)\n", + f"Type pattern: `{ext.get('xlsformTypePattern', '')}` \u2014 e.g. " + f"`select_one_from_file {example_file}`. Use for long, standardised controlled " + "vocabularies instead of hundreds of inline choices. Registered vocabularies:\n", + ] + if vocabs: + out.append("| File | Vocabulary | Standard |") + out.append("|---|---|---|") + for v in vocabs: out.append( - "**Requires a choice list:** write the type cell as " - f"`{ts} ` and define the options on the `choices` sheet under " - "that `list_name`.\n" + f"| `{v.get('xlsformFilename', '')}` | {v.get('skos:prefLabel', '')} | {v.get('standard', '\u2014')} |" ) + out.append("") + return out + + +def _logic_section(logic: dict) -> list[str]: + out = [ + "## Skip logic & validation (`relevant`, `constraint`)\n", + "Written in the XLSForm XPath subset. `${name}` references another answer; " + "`.` (in `constraint`) refers to this answer; `selected(${q}, 'code')` tests a " + "choice. `constraint_message` holds the plain-text error.\n", + ] + fns = logic.get("supportedXPathFunctions", []) + if fns: + out.append( + "**Only these functions are supported** (anything else is rejected):\n\n" + + ", ".join(f"`{f}`" for f in fns) + + "\n" + ) + return out + + +def _appearances_section(appearances: list[dict]) -> list[str]: + out = ["## Appearances (`appearance` column)\n", "| `appearance` | Meaning | Valid on |", "|---|---|---|"] + for a in appearances: + valid = ", ".join(f"`{t}`" for t in a.get("validForTypes", [])) + out.append(f"| `{a['xlsform']['appearanceString']}` | {a.get('skos:prefLabel', '')} | {valid} |") + out.append("") + return out + + +def _settings_section(settings: dict) -> list[str]: + out = ["## `settings` sheet\n", "| Key | Required |", "|---|---|"] + for f in settings.get("fields", []): + out.append(f"| `{f.get('xlsformKey', '')}` | {'yes' if f.get('required') else 'no'} |") + out.append("") + magic = settings.get("magicNames", []) + if magic: + out.append("**Conventional magic rows:**\n") + for m in magic: + out.append(f"- {m.get('xlsformRow', '')} \u2192 {m.get('note', '')}") + out.append("") + return out + - c = d.get("constraints", {}) - if c: - cl = [] - if "maxNameLength" in c: - cl.append(f"variable `name` ≤ {c['maxNameLength']} chars, `{c.get('namePattern', '')}`") - if "maxChoiceCodeLength" in c: - cl.append(f"choice code ≤ {c['maxChoiceCodeLength']} chars, `{c.get('choiceCodePattern', '')}`") - if cl: - out.append("**Constraints:** " + "; ".join(cl) + "\n") - for w in c.get("warnings", []): - out.append(f"> ⚠️ {w}\n") - - vs = variants_by_base.get(d["@id"], []) - if vs: - out.append("**Variants:**\n") - for v in sorted(vs, key=lambda x: x["@id"]): - out.append(f"- `{v['@id'].split(':', 1)[1]}` — {v.get('skos:prefLabel', '')}: {v.get('useWhen', '—')}") - out.append("") - - example = _load_example(base_dir, d) - if example: - out.append("**Example:**\n") - out.append(_fmt_example(example) + "\n") - - # Worked examples for this type's variants (e.g. the _other companion row). - for v in sorted(vs, key=lambda x: x["@id"]): - vex = _load_example(base_dir, v) - if vex: - out.append(f"**Example — `{v['@id'].split(':', 1)[1]}`:**\n") - out.append(_fmt_example(vex) + "\n") - - return "\n".join(out) + "\n" +_LANGUAGE_SECTION = [ + "## Multi-language\n", + "Add per-language columns `label::`, `hint::`, etc., where `` " + "is an IETF BCP-47 tag (`de`, `en`, `fr-BE`). Set `default_language` on the " + "`settings` sheet.\n", +] + +_CHOICES_SECTION = [ + "## Choices sheet\n", + "`select_one` / `select_multiple` rows reference a `list_name`. Define options on " + "the `choices` sheet with columns `list_name`, `name` (code), `label`. Every " + "`list_name` used on the survey sheet must exist on the choices sheet.\n", +] def _syntax_md(registry: dict[str, Any]) -> str: def conv(cid: str) -> dict: return registry.get(cid, {}).get("rule", {}) - san = conv("convention:sanitization") - other = conv("convention:other") - ext = conv("convention:externalCodeList") - logic = conv("convention:logicMapping") - lang = conv("convention:languageTagging") - settings = conv("convention:surveySettings") - unreg = conv("convention:unregisteredRows") - appearances = sorted( (d for d in registry.values() if d.get("@type") == "Appearance"), key=lambda d: d["xlsform"]["appearanceString"], @@ -202,129 +353,25 @@ def conv(cid: str) -> dict: key=lambda d: d.get("xlsformFilename", ""), ) - out = [_DO_NOT_EDIT, "# Allowed XLSForm Syntax\n"] + # (rule, section builder) in document order; a missing rule drops its section. + sections: list[tuple[Any, Any]] = [ + (conv("convention:unregisteredRows"), _allowlist_section), + (conv("convention:sanitization"), _naming_section), + (True, lambda _: _CHOICES_SECTION), + (conv("convention:other"), _other_section), + (conv("convention:externalCodeList"), lambda ext: _external_list_section(ext, vocabs)), + (conv("convention:logicMapping"), _logic_section), + (appearances, lambda _: _appearances_section(appearances)), + (conv("convention:languageTagging"), lambda _: _LANGUAGE_SECTION), + (conv("convention:surveySettings"), _settings_section), + ] - # Allowlist note - if unreg: - meta_rows = ", ".join(f"`{r}`" for r in unreg.get("metadataRowTypes", [])) - out.append("## Row types are an allowlist\n") - out.append( - f"{unreg.get('unknownTypeBehavior', '')}\n\n" - f"Metadata rows ({meta_rows}) are accepted and carried but produce no authored " - "question. Every other `type` must be one of the registered question / structural " - "types.\n" - ) - - # Naming & choice codes - if san: - n = san.get("name", {}) - cc = san.get("choiceCode", {}) - out.append("## Names & choice codes\n") - out.append( - "Author names/codes in English `snake_case`. Downstream (LimeSurvey) " - f"sanitization strips `{san.get('name', {}).get('stripCharsRegex', '[_-]')}` " - f"to the pattern `{n.get('pattern', '')}` and truncates, so keep the " - "alphanumeric stem short and unambiguous:\n" - f"- **Variable `name`:** ≤ {n.get('maxLength', '?')} chars after stripping. " - "Must be unique — duplicates get a numeric suffix.\n" - f"- **Choice `name` (code):** ≤ {cc.get('maxLength', '?')} chars after stripping " - "— longer codes are truncated in LimeSurvey, so short codes (or plain integers, " - "as in the examples) are safest.\n" - "- Labels/hints carry the human text; keep names/codes machine-stable.\n" - ) - - # Choice sheet - out.append("## Choices sheet\n") - out.append( - "`select_one` / `select_multiple` rows reference a `list_name`. Define options on " - "the `choices` sheet with columns `list_name`, `name` (code), `label`. Every " - "`list_name` used on the survey sheet must exist on the choices sheet.\n" - ) - - # or_other - if other: - out.append("## `or_other` pattern (open “Other” field)\n") - out.append( - f"Applies to: {', '.join(f'`{t}`' for t in other.get('appliesTo', []))}. " - f"Add a choice with code `{other.get('choiceCode', 'other')}`, then a companion " - f"row of type `{other.get('companionType', 'text')}` named " - f"`{other.get('companionSuffix', '_other')}` shown only when the " - "“other” choice is picked — set its `relevant` to `${} = " - f"'{other.get('choiceCode', 'other')}'`. See the `select_one_other` example.\n" - ) - - # External code list / long list - if ext: - out.append("## Long lists from a file (`select_*_from_file`)\n") - # Example filename comes from the registry, not a hardcoded vocabulary: - # vocabularies are open-ended (convention:externalCodeList's - # vocabularyDeclaration), so the skill doc must not pin one. - example_file = vocabs[0].get("xlsformFilename", ".csv") if vocabs else ".csv" - out.append( - f"Type pattern: `{ext.get('xlsformTypePattern', '')}` — e.g. " - f"`select_one_from_file {example_file}`. Use for long, standardised controlled " - "vocabularies instead of hundreds of inline choices. Registered vocabularies:\n" - ) - if vocabs: - out.append("\n| File | Vocabulary | Standard |") - out.append("|---|---|---|") - for v in vocabs: - out.append( - f"| `{v.get('xlsformFilename', '')}` | {v.get('skos:prefLabel', '')} | {v.get('standard', '—')} |" - ) - out.append("") - - # Logic - if logic: - out.append("## Skip logic & validation (`relevant`, `constraint`)\n") - out.append( - "Written in the XLSForm XPath subset. `${name}` references another answer; " - "`.` (in `constraint`) refers to this answer; `selected(${q}, 'code')` tests a " - "choice. `constraint_message` holds the plain-text error.\n" - ) - fns = logic.get("supportedXPathFunctions", []) - if fns: - out.append( - "\n**Only these functions are supported** (anything else is rejected):\n\n" - + ", ".join(f"`{f}`" for f in fns) - + "\n" - ) - - # Appearances - if appearances: - out.append("## Appearances (`appearance` column)\n") - out.append("| `appearance` | Meaning | Valid on |") - out.append("|---|---|---|") - for a in appearances: - valid = ", ".join(f"`{t}`" for t in a.get("validForTypes", [])) - out.append(f"| `{a['xlsform']['appearanceString']}` | {a.get('skos:prefLabel', '')} | {valid} |") - out.append("") - - # Language - if lang: - out.append("## Multi-language\n") - out.append( - "Add per-language columns `label::`, `hint::`, etc., where `` " - "is an IETF BCP-47 tag (`de`, `en`, `fr-BE`). Set `default_language` on the " - "`settings` sheet.\n" - ) + out = [_DO_NOT_EDIT, "# Allowed XLSForm Syntax\n"] + for rule, build in sections: + if rule: + out += build(rule) - # Settings - if settings: - out.append("## `settings` sheet\n") - out.append("| Key | Required |") - out.append("|---|---|") - for f in settings.get("fields", []): - out.append(f"| `{f.get('xlsformKey', '')}` | {'yes' if f.get('required') else 'no'} |") - out.append("") - magic = settings.get("magicNames", []) - if magic: - out.append("**Conventional magic rows:**\n") - for m in magic: - out.append(f"- {m.get('xlsformRow', '')} → {m.get('note', '')}") - out.append("") - - return "\n".join(out) + "\n" + return _tidy("\n".join(out)) def generate_skill(registry: dict[str, Any], base_dir: Path, skill_root: Path) -> int: diff --git a/skills/cdl-survey-types/references/question-types.md b/skills/cdl-survey-types/references/question-types.md index 7b9e6b1..81a9826 100644 --- a/skills/cdl-survey-types/references/question-types.md +++ b/skills/cdl-survey-types/references/question-types.md @@ -37,7 +37,7 @@ **Constraints:** variable `name` ≤ 20 chars, `^[a-zA-Z0-9]+$`; choice code ≤ 5 chars, `^[a-zA-Z0-9]+$` > ⚠️ Choice codes > 5 chars will be truncated in LimeSurvey - +> > ⚠️ Avoid choice codes with identical 5-char prefixes **Variants:** @@ -89,4 +89,3 @@ **Use when:** When capturing a time of day without an associated date (e.g. preferred appointment time). **Constraints:** variable `name` ≤ 20 chars, `^[a-zA-Z0-9]+$` - diff --git a/skills/cdl-survey-types/references/xlsform-syntax.md b/skills/cdl-survey-types/references/xlsform-syntax.md index b1cdb30..b987cdc 100644 --- a/skills/cdl-survey-types/references/xlsform-syntax.md +++ b/skills/cdl-survey-types/references/xlsform-syntax.md @@ -11,6 +11,7 @@ Metadata rows (`start`, `end`, `today`, `deviceid`, `username`, `hidden`, `audit ## Names & choice codes Author names/codes in English `snake_case`. Downstream (LimeSurvey) sanitization strips `[_-]` to the pattern `^[a-zA-Z0-9]+$` and truncates, so keep the alphanumeric stem short and unambiguous: + - **Variable `name`:** ≤ 20 chars after stripping. Must be unique — duplicates get a numeric suffix. - **Choice `name` (code):** ≤ 5 chars after stripping — longer codes are truncated in LimeSurvey, so short codes (or plain integers, as in the examples) are safest. - Labels/hints carry the human text; keep names/codes machine-stable. @@ -27,7 +28,6 @@ Applies to: `select_one`, `select_multiple`. Add a choice with code `other`, the Type pattern: `^select_(one|multiple)_from_file (?P[^ ]+\.csv)$` — e.g. `select_one_from_file iso_3166_1.csv`. Use for long, standardised controlled vocabularies instead of hundreds of inline choices. Registered vocabularies: - | File | Vocabulary | Standard | |---|---|---| | `iso_3166_1.csv` | ISO 3166-1 country codes (alpha-2) | ISO 3166-1 | @@ -36,7 +36,6 @@ Type pattern: `^select_(one|multiple)_from_file (?P[^ ]+\.csv)$` — e Written in the XLSForm XPath subset. `${name}` references another answer; `.` (in `constraint`) refers to this answer; `selected(${q}, 'code')` tests a choice. `constraint_message` holds the plain-text error. - **Only these functions are supported** (anything else is rejected): `and`, `or`, `not`, `if`, `selected`, `contains`, `starts-with`, `ends-with`, `regex`, `string`, `number`, `concat`, `substring`, `string-length`, `normalize-space`, `count`, `sum`, `round`, `floor`, `ceiling`, `div`, `mod`, `today`, `now` @@ -71,4 +70,3 @@ Add per-language columns `label::`, `hint::`, etc., where `` i - type=note + name=welcome → Body text from label promoted to survey-level intro. Convention only; opt-in via tool config. - type=note + name=end → End-of-survey thank-you. LS-only. - From 4481ce99329fb2a7cf94768c1c8fc52f8c1d87bb Mon Sep 17 00:00:00 2001 From: jstet Date: Wed, 19 Aug 2026 10:05:28 +0200 Subject: [PATCH 3/7] fix(tests): point the pyxform validation at the moved xlsx fixtures 7a892c3 restructured registry/entities into definition/fixtures/generated and moved each entity's xlsform.xlsx into generated/, but test_xlsform_pyxform.py still looked for it at the exampleDir root. The parametrisation collapsed to zero cases, so its guard test failed and none of the 14 fixtures were actually validated. `generated/` is gitignored codegen output, so the guard now skips (rather than fails) when no generated/ directory exists at all, and still fails if the directories are there but hold no xlsx. Also extract helpers from test_testB_question_order_in_groups to clear its C901 complexity error: dict-or-object field access, the gid/question groupings, the per-group order assertion, and the ordered-titles lookup were all inline. --- tests/live/limesurvey/test_xlsx_surveys.py | 154 +++++++++------------ tests/validation/test_xlsform_pyxform.py | 11 +- 2 files changed, 73 insertions(+), 92 deletions(-) diff --git a/tests/live/limesurvey/test_xlsx_surveys.py b/tests/live/limesurvey/test_xlsx_surveys.py index 2cf3634..778114f 100644 --- a/tests/live/limesurvey/test_xlsx_surveys.py +++ b/tests/live/limesurvey/test_xlsx_surveys.py @@ -3,6 +3,7 @@ Tests TSV generation from real-world XLSX files and validates structure. """ +from itertools import pairwise from pathlib import Path import pytest @@ -315,6 +316,55 @@ def test_testB_import(limesurvey_client: Client, generated_files_dir: Path): cleanup_survey(limesurvey_client, survey_id) +def _field(obj, key, default=None): + """Read a field from a citric result that may be a dict or an object.""" + return obj.get(key, default) if isinstance(obj, dict) else getattr(obj, key, default) + + +def _gid_to_name(groups) -> dict[int, str]: + return {int(_field(g, "gid")): _field(g, "group_name") for g in groups} + + +def _questions_by_group(questions) -> dict[int, list[dict]]: + """Parent questions (subquestions dropped) grouped by gid, each as {title, order}.""" + by_group: dict[int, list[dict]] = {} + for q in questions: + parent_qid = _field(q, "parent_qid", 0) + if parent_qid and int(parent_qid) != 0: + continue # skip subquestions + gid = int(_field(q, "gid")) + by_group.setdefault(gid, []).append({"title": _field(q, "title"), "order": int(_field(q, "question_order"))}) + return by_group + + +def _assert_strictly_increasing_orders(group_name: str, qs: list[dict]) -> list[int]: + """question_order in one group must be unique and strictly increasing.""" + sorted_qs = sorted(qs, key=lambda x: x["order"]) + orders = [q["order"] for q in sorted_qs] + titles = [q["title"] for q in sorted_qs] + pairs = list(zip(titles, orders, strict=False)) + + # Orders must be unique (no duplicates from counter reset bug) + assert len(set(orders)) == len(orders), f"Group '{group_name}' has duplicate question_order values: {pairs}" + for i in range(1, len(orders)): + assert orders[i] > orders[i - 1], f"Group '{group_name}' has non-increasing question_order: {pairs}" + return orders + + +def _ordered_titles(gid_to_name: dict[int, str], by_group: dict[int, list[dict]], group_name: str) -> list[str]: + """Titles of one named group in question_order, or [] if the group is absent.""" + for gid, gname in gid_to_name.items(): + if gname == group_name: + return [q["title"] for q in sorted(by_group.get(gid, []), key=lambda x: x["order"])] + return [] + + +def _assert_title_sequence(titles: list[str], expected: list[str]) -> None: + """`expected` titles must appear in that relative order within `titles`.""" + for earlier, later in pairwise(expected): + assert titles.index(earlier) < titles.index(later), f"{earlier} should come before {later}, got: {titles}" + + def test_testB_question_order_in_groups(limesurvey_client: Client, generated_files_dir: Path): """Verify that questions within each group have correct, incrementing order after import. @@ -330,104 +380,32 @@ def test_testB_question_order_in_groups(limesurvey_client: Client, generated_fil survey_id = import_survey_from_tsv(limesurvey_client, tsv_path, "testB Question Order Test") try: - questions = limesurvey_client.list_questions(survey_id) - groups = limesurvey_client.list_groups(survey_id) - - # Build group_id -> group_name mapping - gid_to_name = {} - for g in groups: - gid = g.get("gid") if isinstance(g, dict) else g.gid - gname = g.get("group_name") if isinstance(g, dict) else g.group_name - gid_to_name[int(gid)] = gname + gid_to_name = _gid_to_name(limesurvey_client.list_groups(survey_id)) + questions_by_group = _questions_by_group(limesurvey_client.list_questions(survey_id)) - # Build group_id -> sorted questions mapping (parent questions only) - questions_by_group = {} - for q in questions: - parent_qid = q.get("parent_qid") if isinstance(q, dict) else getattr(q, "parent_qid", 0) - if parent_qid and int(parent_qid) != 0: - continue # skip subquestions - - gid = int(q.get("gid") if isinstance(q, dict) else q.gid) - title = q.get("title") if isinstance(q, dict) else q.title - order = int(q.get("question_order") if isinstance(q, dict) else q.question_order) - - if gid not in questions_by_group: - questions_by_group[gid] = [] - questions_by_group[gid].append({"title": title, "order": order}) - - # For every group, verify question_order values are unique and incrementing for gid, qs in questions_by_group.items(): - sorted_qs = sorted(qs, key=lambda x: x["order"]) - orders = [q["order"] for q in sorted_qs] - titles = [q["title"] for q in sorted_qs] group_name = gid_to_name.get(gid, f"gid={gid}") - - # Orders must be unique (no duplicates from counter reset bug) - assert len(set(orders)) == len(orders), ( - f"Group '{group_name}' has duplicate question_order values: {list(zip(titles, orders, strict=False))}" - ) - - # Orders must be strictly increasing - for i in range(1, len(orders)): - assert orders[i] > orders[i - 1], ( - f"Group '{group_name}' has non-increasing question_order: {list(zip(titles, orders, strict=False))}" - ) - + orders = _assert_strictly_increasing_orders(group_name, qs) print(f"✓ Group '{group_name}': {len(qs)} questions with correct order {orders}") - # Verify specific expected ordering within key groups - # Group "groupgi4rv46": Hallo → Disclaimer (only notes, project questions are in auto-group) - g1_qs = None - for gid, gname in gid_to_name.items(): - if gname == "groupgi4rv46": - g1_qs = sorted(questions_by_group.get(gid, []), key=lambda x: x["order"]) - break - - if g1_qs: - g1_titles = [q["title"] for q in g1_qs] - assert g1_titles.index("Hallo") < g1_titles.index("Disclaimer"), ( - f"Hallo should come before Disclaimer, got: {g1_titles}" - ) + # Verify specific expected ordering within key groups. + # Group "groupgi4rv46": only notes — project questions land in an auto-group. + g1_titles = _ordered_titles(gid_to_name, questions_by_group, "groupgi4rv46") + if g1_titles: + _assert_title_sequence(g1_titles, ["Hallo", "Disclaimer"]) # projectid should NOT be in this group (it's in an auto-generated group) assert "projectid" not in g1_titles, f"projectid should not be in groupgi4rv46, got: {g1_titles}" - # Auto-generated group "G1": projectid → projectroleprojectal → projectroleprojectbe → projectroleprojectga - auto_g1_qs = None - for gid, gname in gid_to_name.items(): - if gname == "G1": - auto_g1_qs = sorted(questions_by_group.get(gid, []), key=lambda x: x["order"]) - break - - if auto_g1_qs: - auto_titles = [q["title"] for q in auto_g1_qs] - assert auto_titles.index("projectid") < auto_titles.index("projectroleprojectal"), ( - f"projectid should come before projectroleprojectal, got: {auto_titles}" - ) - assert auto_titles.index("projectroleprojectal") < auto_titles.index("projectroleprojectbe"), ( - f"projectroleprojectal should come before projectroleprojectbe, got: {auto_titles}" - ) - assert auto_titles.index("projectroleprojectbe") < auto_titles.index("projectroleprojectga"), ( - f"projectroleprojectbe should come before projectroleprojectga, got: {auto_titles}" + auto_titles = _ordered_titles(gid_to_name, questions_by_group, "G1") + if auto_titles: + _assert_title_sequence( + auto_titles, + ["projectid", "projectroleprojectal", "projectroleprojectbe", "projectroleprojectga"], ) - # Group "demographics": firstname → lastname → emailaddress → gender → genderselfidentifica - demo_qs = None - for gid, gname in gid_to_name.items(): - if gname == "demographics": - demo_qs = sorted(questions_by_group.get(gid, []), key=lambda x: x["order"]) - break - - if demo_qs: - demo_titles = [q["title"] for q in demo_qs] - assert demo_titles.index("firstname") < demo_titles.index("lastname"), ( - f"firstname should come before lastname, got: {demo_titles}" - ) - assert demo_titles.index("lastname") < demo_titles.index("emailaddress"), ( - f"lastname should come before emailaddress, got: {demo_titles}" - ) - assert demo_titles.index("emailaddress") < demo_titles.index("gender"), ( - f"emailaddress should come before gender, got: {demo_titles}" - ) + demo_titles = _ordered_titles(gid_to_name, questions_by_group, "demographics") + if demo_titles: + _assert_title_sequence(demo_titles, ["firstname", "lastname", "emailaddress", "gender"]) # consentprivacypolicy is now in its own auto-generated group assert "consentprivacypolicy" not in demo_titles, ( f"consentprivacypolicy should not be in demographics, got: {demo_titles}" diff --git a/tests/validation/test_xlsform_pyxform.py b/tests/validation/test_xlsform_pyxform.py index b8206a5..66f3866 100644 --- a/tests/validation/test_xlsform_pyxform.py +++ b/tests/validation/test_xlsform_pyxform.py @@ -24,18 +24,21 @@ pyxform = pytest.importorskip("pyxform", reason="pyxform not installed (dev dependency group)") from pyxform.xls2xform import convert # noqa: E402 (after importorskip) -# Only entities that ship an xlsx fixture. -_ENTITIES = [(e["@id"], e) for e in examples() if (example_dir(e) / "xlsform.xlsx").exists()] +# codegen renders each entity's xlsx into /generated/ (gitignored). +_XLSX = "generated/xlsform.xlsx" +_ENTITIES = [(e["@id"], e) for e in examples() if (example_dir(e) / _XLSX).exists()] @pytest.mark.parametrize("entity_id,entity", _ENTITIES, ids=[eid for eid, _ in _ENTITIES]) def test_xlsform_fixture_is_valid_xlsform(entity_id: str, entity: dict) -> None: """pyxform must accept the fixture and emit a non-empty XForm.""" - xlsx = example_dir(entity) / "xlsform.xlsx" + xlsx = example_dir(entity) / _XLSX result = convert(str(xlsx)) assert result.xform and result.xform.strip(), f"pyxform produced empty XForm for {entity_id} ({xlsx})" def test_at_least_one_fixture_checked() -> None: """Guard against the parametrization silently collapsing to zero cases.""" - assert _ENTITIES, "No xlsform.xlsx fixtures discovered under registry/entities/" + if not any((example_dir(e) / "generated").is_dir() for e in examples()): + pytest.skip("no generated/ artifacts present -- run `python -m codegen` first") + assert _ENTITIES, f"No {_XLSX} fixtures discovered under registry/entities/" From 9ec80b39e6827ae416f0f34c62b3d40000ef3dae Mon Sep 17 00:00:00 2001 From: jstet Date: Wed, 19 Aug 2026 10:20:29 +0200 Subject: [PATCH 4/7] docs: handover plan for moving formtransform-app onto this library MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit formtransform-app still runs two unrelated conversion engines: the old xlsform2lstsv npm package for the TSV path, and Pyodide plus the survey2ddi Python wheel for the DDI paths. This library covers all of that except the response-data CSV, so the app can drop both engines for a single dependency. The document records the verified API surface the app needs, the one gap that blocks deleting Pyodide (buildDdiXml uses `submissions` for caseQnty only — there is no TS equivalent of survey2ddi's build_data_csv), and links the sequenced issues filed in CorrelAid/formtransform-app (#4-#10). It also carries the scope notice the app's site is missing: the site presents FormTransform as an XLSForm converter, while the library deliberately rejects anything outside the registered CDL subset. Stating that on the page turns a confusing error into an expected one. --- HANDOVER_FORMTRANSFORM_APP.md | 148 ++++++++++++++++++++++++++++++++++ README.md | 2 + 2 files changed, 150 insertions(+) create mode 100644 HANDOVER_FORMTRANSFORM_APP.md diff --git a/HANDOVER_FORMTRANSFORM_APP.md b/HANDOVER_FORMTRANSFORM_APP.md new file mode 100644 index 0000000..1175ed3 --- /dev/null +++ b/HANDOVER_FORMTRANSFORM_APP.md @@ -0,0 +1,148 @@ +# Handover: making formtransform-app run entirely on this library + +Audience: whoever (person or agent) picks up +[`CorrelAid/formtransform-app`](https://github.com/CorrelAid/formtransform-app) +next. This document is the plan; the work is split into issues in that repo, +linked below. Each issue is written to stand on its own — an agent should be able +to act on one without reading the others. + +## Goal + +The app is a SvelteKit static site (bun, `adapter-static`, 100% client-side) that +today converts surveys with **two unrelated engines**: + +| Path in the app | Engine today | Where it should end up | +|---|---|---| +| XLSForm → LimeSurvey TSV | `xlsform2lstsv@0.3.0` (old npm package) | `@correlaid/formtransform` | +| Kobo → DDI, metadata only | Pyodide + `survey2ddi` wheel (CPython in WASM) | `@correlaid/formtransform` | +| Kobo → DDI, full (with responses CSV) | Pyodide + `survey2ddi` wheel | blocked — see [gap](#the-one-real-gap) | +| LimeSurvey → DDI | Pyodide, backend only, unreachable in the UI | decide: implement or delete | + +End state: one dependency, `@correlaid/formtransform`, installed from +`github:CorrelAid/formtransform`; no Pyodide, no Python wheel, no `xlsform2lstsv`. +Every type decision the app makes then comes from this repo's `registry/`, which +is the point of the consolidation. + +## What this library gives the app + +Verified against `src/index.ts` at the time of writing. All importable from the +package root: + +- `XLSFormParser.convertXLSDataToTSV(data: Buffer | ArrayBuffer, config?: Partial): Promise` + — XLSForm bytes → LimeSurvey structure TSV. **Same class, method and signature + as the old `xlsform2lstsv` package**, so the app's TSV tab is a one-line import + swap. +- `XLSLoader.parseXLSData(data, { skipValidation? }): { surveyData, choicesData, settingsData, … }` + — parse a workbook into row arrays. Validates against the supported XLSForm + subset by default and throws on violations. +- `buildDdiXml(surveyRows, choices, options?): string` — XLSForm rows → + DDI-Codebook 2.5 XML. `choices` takes the flat `choicesData` array directly. + Options: `assetName`, `settings`, `submissions`, `datasetFilename`, `prodDate`. +- `lstsvToDdiXml(tsv: string, options?): string` — LimeSurvey structure TSV → + DDI XML. **This did not exist when the migration was first sketched; it does + now**, which is what makes the dormant LimeSurvey → DDI tab decidable. +- `lstsvToXlsform(...)`, `validateLstsvSubset(...)`, `ConfigManager`, + `defaultConfig` — available, not currently needed by the app. + +Browser-safety: the Node-only modules (`src/cli.ts`, `src/fileChoices.ts`, the +only places that touch `node:fs`) are deliberately **not** re-exported from +`src/index.ts`. The browser entry pulls in only `xlsx`, `js-xpath` and `marked`. +If a bundler ever reports `node:fs` from this package, that is a regression here, +not something for the app to shim. + +Config parity: the app's five conversion toggles (`convertWelcomeNote`, +`convertEndNote`, `convertOtherPattern`, `convertMarkdown`, `hideNoAnswer`) all +exist in `ConversionConfig` with the same names. The library additionally offers +`hideQuestionTips` (default `true`), which the app does not expose yet. + +## The one real gap + +`buildDdiXml` emits DDI **metadata**. Given `submissions` it uses only their +*count*, for ``: + +```ts +/** Response records — only their count (`caseQnty`) is used. */ +submissions?: unknown[]; +``` + +There is no TypeScript equivalent of the Python `survey2ddi_core.data.build_data_csv`, +which remaps raw response columns onto DDI variable names. So the app's Kobo → DDI +**full mode** (XLSForm + responses CSV → XML **and** `data.csv`) cannot leave +Pyodide, and Pyodide cannot be deleted, until this library grows a response-data +emitter. That is tracked from the app side in issue #9, which asks for an upstream +issue against this repo. + +Everything else the app does is already covered. + +## The issues + +Do them in this order. #7 is independent of the rest. + +| # | Issue | When | +|---|---|---| +| 1 | [#4 Add `@correlaid/formtransform` and rename the app package](https://github.com/CorrelAid/formtransform-app/issues/4) | first — everything else depends on it | +| 2 | [#5 TSV tab: swap `xlsform2lstsv` for the library](https://github.com/CorrelAid/formtransform-app/issues/5) | after #4 | +| 3 | [#6 Kobo → DDI metadata mode: replace Pyodide with `buildDdiXml`](https://github.com/CorrelAid/formtransform-app/issues/6) | after #4, parallel with #5 | +| 4 | [#8 LimeSurvey → DDI: wire the dead tab to `lstsvToDdiXml`, or remove it](https://github.com/CorrelAid/formtransform-app/issues/8) | after #4 | +| 5 | [#9 Gap: no response-data CSV emitter (blocks dropping Pyodide)](https://github.com/CorrelAid/formtransform-app/issues/9) | after #6; tracking only | +| 6 | [#10 Remove Pyodide and the `survey2ddi` wheel](https://github.com/CorrelAid/formtransform-app/issues/10) | last, and only once #5, #6, #8, #9 are all done | +| — | [#7 State on the site that this is not a general-purpose converter](https://github.com/CorrelAid/formtransform-app/issues/7) | any time | + +Two things worth knowing before starting #4: + +- The library ships no `dist/` in git; its `prepare` script (`husky || true; npm run build`) + compiles it at install time. The first `bun install` after adding the dependency + must be checked for `node_modules/@correlaid/formtransform/dist/index.js` — the + app deploys through nixpacks with `bun install --frozen-lockfile`, so a git + dependency that fails to build locally will fail the deploy too. +- The app's `package.json` is still named `formtransform`, which shadows the + library. #4 renames it to `formtransform-app`. + +## The scope notice (#7) + +Independent of the migration, and the reason it is on this list: **this is not a +general-purpose converter, and the site does not say so.** + +The library validates strictly and *rejects* rather than approximates — +unregistered question types (`image`, `audio`, `geopoint`, …), unregistered +appearances, identifiers over the sanitisation limits, nesting deeper than three +levels, selects without an explicit `list_name`, LimeSurvey reserved words. See +[Supported XLSForm Subset](README.md#supported-xlsform-subset). Someone arriving +with an arbitrary XLSForm will hit an error and read it as a bug; saying so up +front turns that error into an expected outcome. + +Two surfaces need it, and #7 carries the drafted EN + DE copy for both: + +1. **In the app** — a callout on `/` between the `

` and the tab bar, strings + in `src/lib/i18n.ts`. It has to live in the app's own i18n, not in the fetched + CDL content: the `cdl-content` Vite plugin silently falls back to an empty + string when its GitHub fetch fails (no token, rate limit), and a disclaimer + that can vanish is not a disclaimer. +2. **On the CDL site** — `CorrelAid/cdl-wp-eins`, at + `src/content/snippets/formtransform/{en,de}.html`, fetched at build time. That + snippet is also factually stale: it still credits `xlsform2lstsv` and mentions + only the LimeSurvey TSV conversion. + +## Hard rules for whoever does the work + +- **Do not modify this repo (`CorrelAid/formtransform`) from the app side.** Missing + capability → open an issue here. +- **Do not reimplement conversion logic in the app.** The app is a file picker, a + progress spinner and a download button. Every type decision belongs to the + registry. +- **Keep the TSV output byte-identical.** It feeds a real LimeSurvey import. The + diff in issue #5 is the acceptance test, and a difference is a finding to + report, not something to paper over. +- **Do not delete a working feature to reach a green grep.** Full-mode Kobo → DDI + stays on Pyodide until this library can replace it. +- **Keep the library's validation on.** `skipValidation` exists for callers that + have already validated; an app that passes it turns a clear rejection into + silently wrong output. + +## Related documents + +- [`README.md`](README.md) — what this repo ships and to whom +- [`ARCHITECTURE.md`](ARCHITECTURE.md) — registry → codegen → artifacts, and why + DDI is the terminus of the pipeline graph +- `src/pipelines/lstsv2xlsform/README.md` — the documented losses on the reverse + path, relevant if the app ever exposes it diff --git a/README.md b/README.md index 7b9c9ba..b4152b9 100644 --- a/README.md +++ b/README.md @@ -142,6 +142,8 @@ npm run bless ## Documentation - [Architecture](ARCHITECTURE.md) — Technical architecture and internal structure +- [formtransform-app handover](HANDOVER_FORMTRANSFORM_APP.md) — plan for moving the + frontend app onto this library (issue-by-issue) - [Claude Code Integration](CLAUDE.md) — Claude-specific features and skills - [Pipeline Documentation](src/pipelines/README.md) — Transformation pipeline details - [Test Documentation](tests/README.md) — Test structure and running tests From 82e5602d2a94466361ff3d8518133e7fdd9c4d76 Mon Sep 17 00:00:00 2001 From: jstet Date: Wed, 19 Aug 2026 10:32:09 +0200 Subject: [PATCH 5/7] docs: handover plan for aligning qwac and formulaid with the registry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Neither repo runs a competing converter, so unlike formtransform-app this is a drift problem rather than a dependency one: both carry hand-maintained copies of knowledge the registry owns. qwac renders qwacback data — whose types come from here — using labels derived by string surgery on the type name, and encodes the type list a second time in its preview components, with nothing to catch a registry type that has no component. formulaid's skill teaches ~1000 lines of general XLSForm specification, well beyond what the pipeline accepts, and its app builds workbooks with xlsx and never validates them. The document records what the package can hand over today, the one thing it cannot (no labelled type catalogue: prefLabel/useWhen live in registry/, which "files": ["dist"] keeps out of consumers' node_modules), and links the sequenced issues filed in CorrelAid/qwac (#9-#12) and CorrelAid/formulaid (#9-#12). It also flags the three details that went stale in qwacback's own untracked brief. --- HANDOVER_QWAC_FORMULAID.md | 168 +++++++++++++++++++++++++++++++++++++ README.md | 2 + 2 files changed, 170 insertions(+) create mode 100644 HANDOVER_QWAC_FORMULAID.md diff --git a/HANDOVER_QWAC_FORMULAID.md b/HANDOVER_QWAC_FORMULAID.md new file mode 100644 index 0000000..aaf93aa --- /dev/null +++ b/HANDOVER_QWAC_FORMULAID.md @@ -0,0 +1,168 @@ +# Handover: aligning qwac and formulaid with this registry + +Audience: whoever (person or agent) picks up +[`CorrelAid/qwac`](https://github.com/CorrelAid/qwac) or +[`CorrelAid/formulaid`](https://github.com/CorrelAid/formulaid) next. This +document is the plan; the work is split into issues in those repos, linked below. +Each issue stands on its own — an agent should be able to act on one without +reading the others. + +Companion document: +[`HANDOVER_FORMTRANSFORM_APP.md`](HANDOVER_FORMTRANSFORM_APP.md), which covers the +frontend converter app. The situation there was a *dependency* problem (two +conversion engines, one library to replace them). Here it is a *drift* problem: +neither repo runs a competing converter, but both carry hand-maintained copies of +knowledge this registry owns. + +## Which repo consumes what + +| Repo | Relationship to this registry today | What should change | +|---|---|---| +| **qwac** (SvelteKit SPA, question-bank browser) | No dependency at all. Renders qwacback data, whose types come from here, using its own hardcoded type vocabulary | Import the type catalogue instead of hardcoding it; explain what its DDI validation actually checks | +| **formulaid** (SvelteKit app + `generating-xlsforms` Claude skill) | None in code. The skill teaches XLSForm from `umfragen.civic-data.de` llm.txt; the app builds workbooks with `xlsx` and never validates them | Bundle the generated `cdl-survey-types` sub-skill; validate generated forms with the library | +| **qwacback** (Go/PocketBase API) | Consumes the schematron-worker image and the DDI validation assets | Already has its own brief — see [qwacback](#qwacback-already-briefed) | + +## The shared failure mode + +This registry is an **allowlist**. Unregistered question types, unregistered +appearances, over-long identifiers, nesting deeper than three levels and selects +without an explicit `list_name` are *rejected*, not approximated — see +[Supported XLSForm Subset](README.md#supported-xlsform-subset). + +Both repos currently describe the survey world in their own words: + +- qwac derives human labels from type strings by text surgery + (`AnswerTypeTag.svelte` strips `_other` / `_long_list`, replaces underscores, + title-cases), so `select_one` shows as "Select One" rather than the registry's + own `skos:prefLabel`. Its `question-types/*.svelte` preview components encode + the type list a second time, with nothing to catch a registry type that has no + component. +- formulaid's skill ships ~1000 lines of general XLSForm specification — + `calculation`, "Anhang – Laden großer CSV-Dateien", "Weitere Antwortformate", + "Nicht empfohlene Antwortformate" — which describes far more than the pipeline + accepts. A model reading it will confidently produce forms that fail + conversion, and the failure surfaces in a different repo. Its app also hardcodes + `QuestionType = 'select_one' | 'select_multiple' | 'text' | 'integer' | 'decimal' | 'date' | 'note'` + and never checks its output. + +Neither is a bug today. Both become wrong the moment a question type is added, +renamed or archived here. + +## What this repo can and cannot hand over + +Available now, from the package root (`github:CorrelAid/formtransform`): + +- `XLSLoader.parseXLSData(data, { skipValidation? })` — parse a workbook; + validates against the supported subset by default and throws. +- `XLSValidator` + the `SubsetViolation` type — validate and get findings back + instead of an exception. This is what formulaid needs for a repair loop. +- `XLSFormParser.convertXLSDataToTSV`, `buildDdiXml`, `lstsvToDdiXml`, + `lstsvToXlsform` — the four supported directions. +- `TYPE_MAPPINGS` — per-type mapping facts: `kind`, `limeSurveyType`, + `supported`, `requiresListName`, `answerClass`, `dateFormat`. +- `skills/cdl-survey-types/` — the generated sub-skill (`SKILL.md` plus + `references/question-types.md` and `references/xlsform-syntax.md`), portable to + any agent runtime. Not part of the npm-format package; fetched from the repo at + a pinned tag. + +**Not available, and this is the constraint that shapes both plans:** there is no +labelled, machine-readable type catalogue. `skos:prefLabel`, `useWhen` and the +variant → base relation exist in `registry/` and are rendered into the generated +skill and the docs site, but nothing exports them as data — and the published +package sets `"files": ["dist"]`, so `registry/` never reaches a consumer's +`node_modules`. `src/generated/Appearances.ts` (including the `carriesData` flag a +preview UI wants) is generated but not re-exported from `src/index.ts`. + +Both tracking issues below ask for the same thing: an addition to +`codegen/emit_ts.py` emitting a `QUESTION_TYPES` record with labels, exported from +the package root. It is derived data — every field is already in the registry — so +it is an emitter change, not a registry change. **One upstream issue can serve +both repos**; whoever files first should link the other. + +## The issues + +### qwac + +| # | Issue | When | +|---|---|---| +| 1 | [#9 Centralise question-type knowledge in one module](https://github.com/CorrelAid/qwac/issues/9) | first; no new dependency | +| 2 | [#10 Import the type catalogue from `@correlaid/formtransform`](https://github.com/CorrelAid/qwac/issues/10) | after #9 **and** after the upstream catalogue ships | +| 3 | [#11 Upstream gaps: labelled catalogue, `APPEARANCES` export](https://github.com/CorrelAid/qwac/issues/11) | now; tracking only, blocks #10 | +| — | [#12 Upload page: show why DDI validation failed](https://github.com/CorrelAid/qwac/issues/12) | any time; relates to existing #8 | + +Issue #9 is deliberately doable with no dependency: it collapses the scattered type +knowledge into one `src/lib/questionTypes.ts` whose contents #10 then swaps for +the imported catalogue in a single edit. The labels for #9 are copied from +`skills/cdl-survey-types/references/question-types.md` in this repo — the `##` +headings carry the registry's own `prefLabel`. + +Issue #12 is the qwac counterpart of the app's scope notice. Its upload page POSTs DDI +XML to qwacback's `/api/validate`, which validates against the DDI 2.5 XSDs **and** +the CDL Schematron rules generated here — so a rejection can mean "not valid DDI" +or "valid DDI that is not CDL-shaped", and the page currently says neither. +`src/lib/validation.ts`'s `safeErrorMessage()` throws the detail away on purpose, +which is the right default everywhere except the screen whose entire job is +explaining what is wrong with a file. + +### formulaid + +| # | Issue | When | +|---|---|---| +| 1 | [#9 Bundle the generated `cdl-survey-types` sub-skill into the skill build](https://github.com/CorrelAid/formulaid/issues/9) | first | +| 2 | [#10 Strip the generic XLSForm spec from the skill references](https://github.com/CorrelAid/formulaid/issues/10) | after #9 — never before | +| 3 | [#11 Validate generated workbooks with the library before download](https://github.com/CorrelAid/formulaid/issues/11) | any time; app-side, independent of the skill work | +| — | [#12 Upstream gaps: labelled catalogue, sub-skill release asset](https://github.com/CorrelAid/formulaid/issues/12) | now; tracking only | + +Order matters between #9 and #10: #10 removes the reference content that #9 +replaces, and doing it the other way round leaves the skill with no type +reference at all for however long the gap lasts. + +`scripts/build_skill.sh` already fetches remote content (llm.txt, qwacback +demographics) into `references/`, so vendoring the sub-skill from a pinned tag +fits the existing shape. The one hard requirement: a failed fetch must fail the +build. A skill shipped without its type reference is worse than a stale one, +because the model silently falls back on generic XLSForm knowledge. + +Issue #11 implements existing formulaid issue +[#4](https://github.com/CorrelAid/formulaid/issues/4) ("Let xlsform be validated +and try again with error messages"), which asks whether to add validation to +qwacback or "use some existing package". The package is this library, it runs in +the browser, and it needs no service. + +## qwacback (already briefed) + +`CorrelAid/qwacback` has its own migration brief, currently untracked at +`REGISTRY_SYNC.md` in that working copy. Its plan still holds — delete the +vendored Go converter, the vendored XSDs/Schematron and the Java worker source; +consume `@correlaid/formtransform` plus `ghcr.io/correlaid/schematron-worker` +pinned to one version — but it was written before this repo was renamed and +restructured, so three details in it are stale: + +- the repo is `CorrelAid/formtransform`, not `CorrelAid/survey-type-registry` +- example fixtures live at `registry/entities//fixtures/xlsform.json`, not + `registry/types//examples//xlsform.json` +- `lstsv2ddi` and `lstsv2xlsform` now exist, which the brief predates + +Worth filing as issues in that repo the same way, once someone picks it up. + +## Hard rules for whoever does the work + +- **Do not modify this repo from a consumer.** Missing capability → open an issue + here. Both tracking issues (#11 in qwac, #12 in formulaid) exist for exactly + that. +- **Do not vendor `registry/`.** A copied JSON-LD graph is drift with extra + steps. If the data is not exported, the fix is an emitter change here. +- **Do not hand-edit generated files.** Everything under + `skills/cdl-survey-types/` and `src/generated/` carries a "DO NOT EDIT" header + and is overwritten by the next `uv run codegen`. +- **Do not reimplement subset rules in a consumer.** Call `XLSValidator`. If a + rule looks wrong, that is a finding for this repo. +- **Pin to tags, not branches.** A registry change should never alter a deployed + UI or a shipped skill without a commit in the consuming repo. + +## Related documents + +- [`README.md`](README.md) — what this repo ships and to whom +- [`ARCHITECTURE.md`](ARCHITECTURE.md) — registry → codegen → artifacts +- [`HANDOVER_FORMTRANSFORM_APP.md`](HANDOVER_FORMTRANSFORM_APP.md) — the frontend + converter app's migration diff --git a/README.md b/README.md index b4152b9..71fcecd 100644 --- a/README.md +++ b/README.md @@ -144,6 +144,8 @@ npm run bless - [Architecture](ARCHITECTURE.md) — Technical architecture and internal structure - [formtransform-app handover](HANDOVER_FORMTRANSFORM_APP.md) — plan for moving the frontend app onto this library (issue-by-issue) +- [qwac + formulaid handover](HANDOVER_QWAC_FORMULAID.md) — plan for aligning the + question-bank browser and the survey generator with this registry - [Claude Code Integration](CLAUDE.md) — Claude-specific features and skills - [Pipeline Documentation](src/pipelines/README.md) — Transformation pipeline details - [Test Documentation](tests/README.md) — Test structure and running tests From 5c6184f80ead50d245bebfc2749befa5054a7190 Mon Sep 17 00:00:00 2001 From: jstet Date: Wed, 19 Aug 2026 10:39:40 +0200 Subject: [PATCH 6/7] fix: stop importing generated JSON at runtime, and ship it in dist MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes #1. Every module that needed the generated conventions imported them as `generated/conventions.json` with an import attribute. That made the published package depend on two things outside its control: tsc copying the JSON into `dist/` (it only does so as a side effect of some .ts file importing it), and the consumer's bundler understanding `with { type: 'json' }`. A consumer installing from GitHub reported `dist/generated/conventions.json` missing and the bundle failing to resolve it. codegen now emits `src/generated/conventions.ts` alongside the JSON — same payload, same inferred types — and the library imports that. tsc always emits it and every bundler resolves it, so neither failure mode is reachable any more. The JSON stays as the artifact for non-TypeScript consumers, and `npm run build` copies it into `dist/generated/` after tsc so it is present in an installed package regardless of what imports it. Verified: clean `npm run build` produces dist/generated/{conventions.js, conventions.json}, no import attributes remain in dist, and `bun build dist/index.js --target=browser` bundles 55 modules without error. --- ARCHITECTURE.md | 2 +- codegen/cli.py | 13 +- codegen/emit_ts.py | 43 ++- package.json | 2 +- scripts/copy-generated-json.mjs | 30 +++ src/generated/conventions.ts | 327 +++++++++++++++++++++++ src/pipelines/lstsv2ddi/toVariables.ts | 2 +- src/pipelines/xlsform2ddi/variables.ts | 2 +- src/pipelines/xlsform2lstsv/constants.ts | 2 +- src/utils/helpers.ts | 2 +- src/xlsform/sanitize.ts | 2 +- src/xlsform/validate.ts | 2 +- 12 files changed, 415 insertions(+), 14 deletions(-) create mode 100644 scripts/copy-generated-json.mjs create mode 100644 src/generated/conventions.ts diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index fffcc46..978fcfc 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -49,7 +49,7 @@ The TypeScript library (`@correlaid/formtransform`), split into **format modules Python package (run `uv run codegen` or `python -m codegen`) that validates the registry, then emits generated artifacts: -- `src/generated/` — `TypeMappings.ts`, `DdiMappings.ts`, `Appearances.ts`, `conventions.json` (consumed by the library) +- `src/generated/` — `TypeMappings.ts`, `DdiMappings.ts`, `Appearances.ts`, `conventions.ts` (consumed by the library) plus `conventions.json`, the same payload for non-TypeScript consumers. The library imports the `.ts` twin, never the JSON: a runtime JSON import only reaches `dist/` if tsc copies it, and import attributes are not understood by every consumer's bundler. `npm run build` copies the JSON to `dist/generated/` after `tsc`. - `registry/entities//` — `docs.md` + `xlsform.xlsx` per entity - `skills/cdl-survey-types/` — the generated sub-skill - `ddi-validation/ddi_custom_rules.sch` — the CDL Schematron rules diff --git a/codegen/cli.py b/codegen/cli.py index 35bb89a..569952c 100644 --- a/codegen/cli.py +++ b/codegen/cli.py @@ -5,7 +5,13 @@ from .emit_docs import generate_type_docs from .emit_skill import generate_skill -from .emit_ts import generate_appearances, generate_conventions, generate_typescript, generate_typescript_ddi +from .emit_ts import ( + generate_appearances, + generate_conventions, + generate_conventions_ts, + generate_typescript, + generate_typescript_ddi, +) from .examples import build_example_artifacts from .loader import load_registry from .schematron import generate_schematron @@ -51,7 +57,10 @@ def main(argv: list[str] | None = None) -> None: print(" ✅ src/generated/Appearances.ts (transformer)") generate_conventions(registry, ts_out / "conventions.json") - print(" ✅ src/generated/conventions.json (transformer)") + print(" ✅ src/generated/conventions.json (artifact for non-TS consumers)") + + generate_conventions_ts(registry, ts_out / "conventions.ts") + print(" ✅ src/generated/conventions.ts (transformer)") generate_schematron(registry, base_dir / "ddi-validation" / "schematron" / "ddi_custom_rules.sch") print(" ✅ ddi-validation/schematron/ddi_custom_rules.sch (DDI validation rules)") diff --git a/codegen/emit_ts.py b/codegen/emit_ts.py index f02fb52..794061a 100644 --- a/codegen/emit_ts.py +++ b/codegen/emit_ts.py @@ -278,8 +278,8 @@ def expand_types(types: list[str]) -> list[str]: output.write_text("\n".join(lines)) -def generate_conventions(registry: dict[str, Any], output: Path): - """Emit global conventions + column map + composites as JSON.""" +def _conventions_payload(registry: dict[str, Any]) -> dict[str, Any]: + """Global conventions + column map + composites, in emit order.""" conv = {} col_map = [] composites = [] @@ -309,10 +309,45 @@ def generate_conventions(registry: dict[str, Any], output: Path): # live in separate conventions/*.jsonld files loaded by glob). conv = {k: conv[k] for k in sorted(conv)} composites.sort(key=lambda c: c["id"]) - out = { + return { "$comment": "GENERATED - DO NOT EDIT. Source: registry/root.jsonld", "conventions": conv, "xlsformColumnMap": col_map, "composites": composites, } - output.write_text(json.dumps(out, indent=2, ensure_ascii=False) + "\n") + + +def generate_conventions(registry: dict[str, Any], output: Path): + """Emit global conventions + column map + composites as JSON. + + The JSON is the artifact for non-TypeScript consumers; the library itself + imports the `.ts` twin written by {@link generate_conventions_ts}. + """ + payload = _conventions_payload(registry) + output.write_text(json.dumps(payload, indent=2, ensure_ascii=False) + "\n") + + +def generate_conventions_ts(registry: dict[str, Any], output: Path): + """Emit the same payload as a TypeScript module. + + The library must not `import ... with { type: 'json' }` at runtime: whether + the JSON reaches `dist/` depends on tsc copying it, and whether a consumer's + bundler understands import attributes varies by bundler version. A plain + module sidesteps both — tsc always emits it, every bundler resolves it, and + the inferred types are identical to the JSON import's. + """ + payload = json.dumps(_conventions_payload(registry), indent=2, ensure_ascii=False) + output.write_text( + "/**\n" + " * GENERATED CODE - DO NOT EDIT\n" + " * Source: registry/root.jsonld\n" + " * Generated by: codegen\n" + " *\n" + " * Same payload as `conventions.json`, emitted as a module so the published\n" + " * library carries no runtime JSON import. Edit the registry, not this file.\n" + " */\n" + "\n" + f"const conventions = {payload};\n" + "\n" + "export default conventions;\n" + ) diff --git a/package.json b/package.json index dbcea97..6e495bd 100644 --- a/package.json +++ b/package.json @@ -23,7 +23,7 @@ "docs" ], "scripts": { - "build": "tsc", + "build": "tsc && node scripts/copy-generated-json.mjs", "clean": "rm -rf dist", "test": "vitest run", "test:watch": "vitest", diff --git a/scripts/copy-generated-json.mjs b/scripts/copy-generated-json.mjs new file mode 100644 index 0000000..4ffc450 --- /dev/null +++ b/scripts/copy-generated-json.mjs @@ -0,0 +1,30 @@ +// Copy src/generated/*.json into dist/generated/ after tsc. +// +// tsc only emits a .json file into outDir when some .ts file imports it, and +// the library deliberately imports `generated/conventions.ts` instead (see the +// note in that file). Without this step `dist/generated/` would hold no JSON at +// all, so a consumer that reads the artifact from the installed package — rather +// than through the library's API — finds nothing there. +// +// Runs as part of `npm run build`, which is what the `prepare` script executes +// when the package is installed from GitHub. + +import { copyFileSync, mkdirSync, readdirSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const root = dirname(dirname(fileURLToPath(import.meta.url))); +const src = join(root, 'src', 'generated'); +const dest = join(root, 'dist', 'generated'); + +const files = readdirSync(src).filter((f) => f.endsWith('.json')); +if (files.length === 0) { + console.error(`No JSON to copy from ${src} — run \`uv run codegen\` first.`); + process.exit(1); +} + +mkdirSync(dest, { recursive: true }); +for (const file of files) { + copyFileSync(join(src, file), join(dest, file)); +} +console.log(`copied ${files.length} generated JSON file(s) to dist/generated/`); diff --git a/src/generated/conventions.ts b/src/generated/conventions.ts new file mode 100644 index 0000000..8b9bd99 --- /dev/null +++ b/src/generated/conventions.ts @@ -0,0 +1,327 @@ +/** + * GENERATED CODE - DO NOT EDIT + * Source: registry/root.jsonld + * Generated by: codegen + * + * Same payload as `conventions.json`, emitted as a module so the published + * library carries no runtime JSON import. Edit the registry, not this file. + */ + +const conventions = { + "$comment": "GENERATED - DO NOT EDIT. Source: registry/root.jsonld", + "conventions": { + "externalCodeList": { + "xlsformTypePattern": "^select_(one|multiple)_from_file (?P[^ ]+\\.csv)$", + "ddiVocabFromFilename": "stripExtension", + "ddiEmission": "concept[@vocab=''] in place of inline catgry; catgry MUST NOT be emitted", + "limesurveyEmission": "base select (L/M) with the vocabulary's options inlined as A/SQ rows + cssclass='cdlvocab-' carrying provenance (a registered LimeSurvey question attribute, so it survives import)", + "structuralNote": "select_multiple_from_file emits a flat with concept/@vocab, NOT a varGrp[@type='multipleResp'] with binary children. This differs from select_multiple.", + "appliesTo": [ + "select_one_from_file", + "select_multiple_from_file" + ], + "vocabularyCsvHeader": "code,label", + "vocabularyDeclaration": "Vocabularies are open-ended: this convention defines only how one is declared and handled, never which ones exist. To add one, put its options in registry/vocab/.csv with a 'code,label' header and add a '@type': 'Vocabulary' node to the graph with xlsformFilename/ddiVocab/vocabURI/standard/skos:prefLabel. Every consumer discovers vocabularies by filtering '@type' == 'Vocabulary', so no code, convention or emitter needs editing. vocab:iso_3166_1 is one example, not a special case." + }, + "languageTagging": { + "xlsformColumnPattern": "^(?Plabel|hint|constraint_message|guidance_hint|media::image|media::audio|media::video)::(?P.+)$", + "xlsformLanguageList": { + "setting": "settings.default_language", + "fallback": "first language encountered in label columns", + "note": "Each language code SHOULD be a valid IETF BCP 47 tag (e.g. 'de', 'en', 'fr-BE'). Tools may warn on non-BCP47 codes." + }, + "lsTsvHandling": { + "column": "language", + "emission": "one row per language per question — same name/type, different language column + text/help cells", + "baseLanguage": "first emitted; subsequent languages reference same question name" + }, + "ddiHandling": { + "attribute": "xml:lang", + "appliesToElements": [ + "txt", + "qstnLit", + "labl", + "preQTxt", + "postQTxt" + ], + "emission": "one child element per language with xml:lang attr" + } + }, + "logicMapping": { + "description": "XLSForm expresses survey logic in an XPath subset; LimeSurvey uses ExpressionScript (EM). The transformation transpiles between them (implemented in src/pipelines/xlsform2lstsv/xpathTranspiler.ts). DDI Codebook has no machine-readable expression syntax — logic fields are dropped when emitting DDI and tools MUST report this as loss.", + "fields": [ + { + "concept": "relevance (conditional display)", + "xlsformColumn": "relevant", + "xlsformSyntax": "XPath subset", + "lsTsvColumn": "relevance", + "lsSyntax": "ExpressionScript (EM)", + "ddi": { + "closestElement": "var/universe", + "note": "DDI universe describes the subpopulation as prose, not as machine syntax. Currently NOT emitted — relevance is dropped in DDI output.", + "lossy": true + } + }, + { + "concept": "validation (answer constraint)", + "xlsformColumn": "constraint", + "xlsformSyntax": "XPath subset, `.` refers to the answer", + "lsTsvColumn": "em_validation_q", + "lsSyntax": "ExpressionScript (EM)", + "ddi": { + "closestElement": "var/valrng", + "note": "DDI valrng can express simple numeric ranges only. Currently NOT emitted — constraints are dropped in DDI output.", + "lossy": true + } + }, + { + "concept": "validation message", + "xlsformColumn": "constraint_message", + "xlsformSyntax": "plain text (translatable)", + "lsTsvColumn": "validation", + "lsSyntax": "plain text", + "ddi": { + "closestElement": null, + "note": "No DDI counterpart; dropped.", + "lossy": true + } + } + ], + "syntaxMappings": [ + { + "xlsform": "${name}", + "ls": "{name} in labels; name / name.NAOK in expressions", + "note": "Variable reference; name is sanitized per convention:sanitization." + }, + { + "xlsform": "selected(${f}, 'v')", + "ls": "select_one: (f == \"v\") — select_multiple: f_v.NAOK == \"Y\"", + "note": "select_multiple expands to per-choice binary subquestions, so membership becomes a check on the choice column." + }, + { + "xlsform": "regex(., '...')", + "ls": "regexMatch(...)" + } + ], + "supportedXPathFunctions": [ + "and", + "or", + "not", + "if", + "selected", + "contains", + "starts-with", + "ends-with", + "regex", + "string", + "number", + "concat", + "substring", + "string-length", + "normalize-space", + "count", + "sum", + "round", + "floor", + "ceiling", + "div", + "mod", + "today", + "now" + ], + "unsupportedFunctionBehavior": "error — the transpiler throws on any XPath function outside supportedXPathFunctions" + }, + "other": { + "companionSuffix": "_other", + "companionType": "text", + "relevanceTemplate": "${{${name}}} = 'other'", + "choiceCode": "other", + "appliesTo": [ + "select_one", + "select_multiple" + ], + "labels": { + "en": "Other", + "de": "Sonstiges" + } + }, + "sanitization": { + "name": { + "maxLength": 20, + "stripCharsRegex": "[_-]", + "pattern": "^[a-zA-Z0-9]+$", + "dedupStrategy": "numeric-suffix" + }, + "choiceCode": { + "maxLength": 5, + "stripCharsRegex": "[_-]", + "pattern": "^[a-zA-Z0-9]+$" + } + }, + "surveySettings": { + "fields": [ + { + "xlsformKey": "form_title", + "lsTsvRow": "SL surveyls_title", + "ddiPath": "stdyDscr/citation/titlStmt/titl", + "required": true + }, + { + "xlsformKey": "form_id", + "lsTsvRow": null, + "ddiPath": "fileDscr/@ID", + "required": true + }, + { + "xlsformKey": "default_language", + "lsTsvCell": "language column on every row", + "ddiPath": "fallback xml:lang on var/qstn elements", + "required": false + }, + { + "xlsformKey": "version", + "lsTsvRow": null, + "ddiPath": "codeBook/@version (DDI version, not survey version — record survey version elsewhere)", + "required": false + }, + { + "xlsformKey": "instance_name", + "lsTsvRow": null, + "ddiPath": "fileDscr/fileTxt/fileName", + "required": false + } + ], + "magicNames": [ + { + "xlsformRow": "type=note + name=welcome", + "lsTsvRow": "SL surveyls_welcometext", + "ddiPath": "stdyDscr/stdyInfo/abstract (or notes)", + "note": "Body text from label promoted to survey-level intro. Convention only; opt-in via tool config." + }, + { + "xlsformRow": "type=note + name=end", + "lsTsvRow": "SL surveyls_endtext", + "ddiPath": "—", + "note": "End-of-survey thank-you. LS-only." + } + ] + }, + "unregisteredRows": { + "metadataRowTypes": [ + "start", + "end", + "today", + "deviceid", + "username", + "hidden", + "audit" + ], + "metadataBehavior": "skip silently — device/session metadata rows carry no authored content", + "unknownTypeBehavior": "error — the registry is an allowlist; any survey-sheet type that is neither registered nor a metadata row MUST abort the transformation with a clear message" + } + }, + "xlsformColumnMap": [ + { + "xlsform": "required", + "lsTsv": "mandatory", + "transform": "yes→Y, no→N or empty" + }, + { + "xlsform": "constraint", + "lsTsv": "em_validation_q", + "transform": "XPath → LimeSurvey EM expression — see convention:logicMapping" + }, + { + "xlsform": "constraint_message", + "lsTsv": "validation", + "transform": "passthrough — see convention:logicMapping" + }, + { + "xlsform": "default", + "lsTsv": "default", + "transform": "passthrough" + }, + { + "xlsform": "relevant", + "lsTsv": "relevance", + "transform": "XPath → LimeSurvey EM expression — see convention:logicMapping" + }, + { + "xlsform": "label", + "lsTsv": "text", + "transform": "Markdown → HTML; ${var} → {name}" + }, + { + "xlsform": "hint", + "lsTsv": "help", + "transform": "Markdown → HTML; ${var} → {name}" + }, + { + "xlsform": "name", + "lsTsv": "name", + "transform": "sanitize per convention:sanitization" + } + ], + "composites": [ + { + "id": "grid", + "label": "Grid / Matrix Group", + "broader": [ + { + "@id": "type:begin_group" + }, + { + "@id": "type:select_one" + } + ], + "trigger": { + "xlsformPattern": "begin_group row with appearance=table-list (or name contains 'grid', or label contains 'matrix')", + "applyTo": "begin_group + all rows until matching end_group" + }, + "input": { + "rows": [ + { + "role": "header", + "match": "type=begin_group, appearance=table-list", + "cardinality": 1 + }, + { + "role": "item", + "match": "type=select_one — all items MUST share the same listname", + "cardinality": "1..*" + }, + { + "role": "closer", + "match": "type=end_group", + "cardinality": 1 + } + ] + }, + "output": { + "ddi": { + "container": "varGrp[@type='grid']", + "containerAttrs": { + "name": "", + "var": "space-separated member IDs" + }, + "memberVars": "one per item row. Each var carries the full set repeated (responseDomainType='category')." + }, + "lsTsv": { + "headerRow": "F-type question (matrix header) with answerClass=A; answer codes from shared list", + "itemRows": "SQ-class rows, one per item", + "note": "LS represents the matrix as a single question with sub-questions; categories not re-emitted per row" + } + }, + "schematronPatterns": [ + "essentials", + "logic" + ], + "notes": [ + "If `appearance=table-list` is absent, the group is flattened — children become top-level vars in DDI, group is lost.", + "Items SHOULD share the same `listname` so the matrix is rectangular." + ] + } + ] +}; + +export default conventions; diff --git a/src/pipelines/lstsv2ddi/toVariables.ts b/src/pipelines/lstsv2ddi/toVariables.ts index eee8a3d..92d70eb 100644 --- a/src/pipelines/lstsv2ddi/toVariables.ts +++ b/src/pipelines/lstsv2ddi/toVariables.ts @@ -16,7 +16,7 @@ * - `F` (array) → grid group of `select_one` variables */ -import conventions from '../../generated/conventions.json' with { type: 'json' }; +import conventions from '../../generated/conventions.js'; import { Choice, Variable } from '../../ddi/types.js'; import { APPEARANCES } from '../../generated/Appearances.js'; diff --git a/src/pipelines/xlsform2ddi/variables.ts b/src/pipelines/xlsform2ddi/variables.ts index e8da2da..ec10b29 100644 --- a/src/pipelines/xlsform2ddi/variables.ts +++ b/src/pipelines/xlsform2ddi/variables.ts @@ -7,7 +7,7 @@ */ import { APPEARANCES } from '../../generated/Appearances.js'; -import conventions from '../../generated/conventions.json' with { type: 'json' }; +import conventions from '../../generated/conventions.js'; import { TYPE_MAP, NON_DDI_EMITTABLE_TYPES, diff --git a/src/pipelines/xlsform2lstsv/constants.ts b/src/pipelines/xlsform2lstsv/constants.ts index cf4dfe3..d677829 100644 --- a/src/pipelines/xlsform2lstsv/constants.ts +++ b/src/pipelines/xlsform2lstsv/constants.ts @@ -1,5 +1,5 @@ import { TYPE_MAPPINGS } from './typeMapper.js'; -import conventions from '../../generated/conventions.json' with { type: 'json' }; +import conventions from '../../generated/conventions.js'; // Derived from registry convention:unregisteredRows — device/session metadata // rows are silently skipped; any other unregistered type is an error. diff --git a/src/utils/helpers.ts b/src/utils/helpers.ts index ef373a6..4fae0ab 100644 --- a/src/utils/helpers.ts +++ b/src/utils/helpers.ts @@ -1,4 +1,4 @@ -import conventions from '../generated/conventions.json' with { type: 'json' }; +import conventions from '../generated/conventions.js'; const NAME_RULES = conventions.conventions.sanitization.name; const NAME_STRIP_REGEX = new RegExp(NAME_RULES.stripCharsRegex, 'g'); diff --git a/src/xlsform/sanitize.ts b/src/xlsform/sanitize.ts index 5e6a15b..d1fcb4b 100644 --- a/src/xlsform/sanitize.ts +++ b/src/xlsform/sanitize.ts @@ -1,4 +1,4 @@ -import conventions from '../generated/conventions.json' with { type: 'json' }; +import conventions from '../generated/conventions.js'; import { sanitizeFieldName } from '../utils/helpers.js'; const NAME_RULES = conventions.conventions.sanitization.name; diff --git a/src/xlsform/validate.ts b/src/xlsform/validate.ts index 126ef77..37f3fce 100644 --- a/src/xlsform/validate.ts +++ b/src/xlsform/validate.ts @@ -1,4 +1,4 @@ -import conventions from '../generated/conventions.json' with { type: 'json' }; +import conventions from '../generated/conventions.js'; import { APPEARANCES } from '../generated/Appearances.js'; import { TYPE_MAPPINGS } from '../generated/TypeMappings.js'; From 58c85bf97bddca4beafa2371bd19cf511397745a Mon Sep 17 00:00:00 2001 From: jstet Date: Wed, 19 Aug 2026 11:20:14 +0200 Subject: [PATCH 7/7] docs: split the qwac/formulaid handover into one document per repo The combined document made a reader of either repo skim past half of it, and the two plans have little in common beyond one shared upstream gap: qwac needs a labelled type catalogue for its UI, formulaid needs it for a type union. HANDOVER_QWAC.md now carries the label/preview-component drift, the DDI upload page's silence about what it validates, and the qwacback notes (Go-fmt-serialised DDI fields, structured validation findings, the stale details in that repo's own brief). HANDOVER_FORMULAID.md carries the skill's over-broad XLSForm reference, the unvalidated workbook output, and the cdl-survey-types bundling contract. Each document repeats the allowlist framing and the missing-catalogue constraint so it stands alone, and cross-links the other. The eight issues in the two repos were edited to point at their own document. --- HANDOVER_FORMULAID.md | 136 ++++++++++++++++++++++++++++++ HANDOVER_QWAC.md | 144 +++++++++++++++++++++++++++++++ HANDOVER_QWAC_FORMULAID.md | 168 ------------------------------------- README.md | 6 +- 4 files changed, 284 insertions(+), 170 deletions(-) create mode 100644 HANDOVER_FORMULAID.md create mode 100644 HANDOVER_QWAC.md delete mode 100644 HANDOVER_QWAC_FORMULAID.md diff --git a/HANDOVER_FORMULAID.md b/HANDOVER_FORMULAID.md new file mode 100644 index 0000000..304010c --- /dev/null +++ b/HANDOVER_FORMULAID.md @@ -0,0 +1,136 @@ +# Handover: aligning formulaid with this registry + +Audience: whoever (person or agent) picks up +[`CorrelAid/formulaid`](https://github.com/CorrelAid/formulaid) next — the +SvelteKit app plus the `generating-xlsforms` Claude skill that generate XLSForm +questionnaires. This document is the plan; the work is split into issues in that +repo, linked below. Each issue stands on its own — an agent should be able to act +on one without reading the others. + +Companion documents: [`HANDOVER_QWAC.md`](HANDOVER_QWAC.md) (the question-bank +browser, which shares the upstream gap described below) and +[`HANDOVER_FORMTRANSFORM_APP.md`](HANDOVER_FORMTRANSFORM_APP.md) (the converter +app, a dependency migration rather than a drift problem). + +## The problem + +formulaid produces the input to the CDL survey pipeline, and it does so without +consulting the registry that defines what the pipeline accepts. Two independent +halves, both drifting: + +**The skill teaches more XLSForm than exists here.** +`scripts/build_skill.sh` fetches `umfragen.civic-data.de/llm-phases12-xlsform.txt` +into `references/survey-methodology.md` (~1630 lines). Roughly: + +| Lines | Content | Verdict | +|---|---|---| +| ~35–247 | Studiendesign, Forschungsfragen, Messtheorie, Stichprobenauswahl, Datenschutz, Operationalisierung | Keep — the real value, no upstream equivalent | +| ~248–1243 | `# XLSForm-Dokumentation`: Fragetypen, hints, constraint, relevant, **Berechnung** (`calculation`), Gruppierung, Mehrsprachigkeit, Darstellung, settings sheet, "Anhang – Laden großer CSV-Dateien" | The general xlsform.org spec — this is the problem | +| ~1244–1549 | `# Antworttypen`, including "Weitere Antwortformate" and "Nicht empfohlene Antwortformate" | Overlaps the registry's catalogue and contradicts its allowlist | + +This registry is an **allowlist**: unregistered question types, unregistered +appearances, over-long identifiers, nesting deeper than three levels and selects +without an explicit `list_name` are *rejected*, not approximated — see +[Supported XLSForm Subset](README.md#supported-xlsform-subset). A model reading +the full specification will confidently emit forms that fail conversion, and the +failure surfaces in a different repo, far from its cause. + +**The app never checks its own output.** +`src/lib/agents/xlsform_generator.ts` builds the workbook with `xlsx` +(`XLSX.write(wb, { type: 'buffer', bookType: 'xlsx' })`) and hands it straight to +the download button. `src/lib/agents/types.ts` hardcodes the vocabulary: + +```ts +export type QuestionType = 'select_one' | 'select_multiple' | 'text' | 'integer' | 'decimal' | 'date' | 'note'; +``` + +— a hand-picked subset of the registry's list, with nothing keeping it aligned. + +## What this repo can and cannot hand over + +Available now from the package root (`github:CorrelAid/formtransform`): + +- `XLSValidator` and the `SubsetViolation` type — validate a parsed workbook and + get findings back instead of an exception. This is the one formulaid wants: the + point is to *repair and report*, not to abort. +- `XLSLoader.parseXLSData(data, { skipValidation? })` — parse a workbook into row + arrays; validates and throws by default. +- `XLSFormParser.convertXLSDataToTSV`, `buildDdiXml`, `lstsvToDdiXml`, + `lstsvToXlsform` — the four supported directions, if the app ever wants to offer + LimeSurvey TSV or DDI alongside the `.xlsx`. +- `TYPE_MAPPINGS` — per-type mapping facts (`kind`, `limeSurveyType`, + `supported`, `requiresListName`, `answerClass`, `dateFormat`). Filtered to + `kind === 'question' && supported`, this is a usable stand-in for the + hand-written `QuestionType` union today. +- `skills/cdl-survey-types/` — the generated sub-skill: `SKILL.md` plus + `references/question-types.md` and `references/xlsform-syntax.md`, regenerated + by `uv run codegen`, portable to any agent runtime that reads the format. Its + emitter docstring names formulaid's `generating-xlsforms` as the intended + parent: the parent owns the workflow (intake, qwac search, delivery), the + sub-skill owns "which type, and how to write it". Not part of the npm-format + package — fetched from this repo at a pinned tag. + +**Not available:** a labelled, machine-readable type catalogue. +`skos:prefLabel`, `useWhen` and the variant → base relation live in `registry/`, +and the package sets `"files": ["dist"]`, so `registry/` never reaches a +consumer's `node_modules`. That is why issue #11 derives its type union from +mapping facts instead of labels, and why #12 tracks the upstream ask. +`CorrelAid/qwac` needs the same catalogue, so **one upstream issue can serve both +repos** — whoever files first should link the other. + +## The issues + +| # | Issue | When | +|---|---|---| +| 1 | [#9 Bundle the generated `cdl-survey-types` sub-skill into the skill build](https://github.com/CorrelAid/formulaid/issues/9) | first | +| 2 | [#10 Strip the generic XLSForm spec from the skill references](https://github.com/CorrelAid/formulaid/issues/10) | after #9 — never before | +| 3 | [#11 Validate generated workbooks with the library before download](https://github.com/CorrelAid/formulaid/issues/11) | any time; app-side, independent of the skill work | +| — | [#12 Upstream gaps: labelled catalogue, sub-skill release asset](https://github.com/CorrelAid/formulaid/issues/12) | now; tracking only | + +Order matters between #9 and #10: #10 removes the reference content that #9 +replaces, and doing it the other way round leaves the skill with no type +reference at all for however long the gap lasts. + +`scripts/build_skill.sh` already fetches remote content (llm.txt, qwacback +demographics) into `references/`, so vendoring the sub-skill from a pinned tag +fits the shape that exists. One hard requirement: **a failed fetch must fail the +build.** A skill shipped without its type reference is worse than a stale one, +because the model silently falls back on generic XLSForm knowledge — the exact +failure mode this work exists to remove. Remember that `skills/xlsform.zip` is +what the README tells users to upload, so a rebuild that is not committed changes +nothing for them. + +Issue #11 implements existing formulaid issue +[#4](https://github.com/CorrelAid/formulaid/issues/4) ("Let xlsform be validated +and try again with error messages"), which asks whether to add validation to +qwacback or "use some existing package". The package is this library; it runs in +the browser, needs no service and no network hop, and it is the *same* check the +downstream converters apply — so a form that passes it is a form that converts. + +## Hard rules for whoever does the work + +- **Do not modify this repo from formulaid.** Missing capability → open an issue + here (that is what #12 is for). +- **Do not hand-edit the vendored `cdl-survey-types/` files.** They carry a + "GENERATED — DO NOT EDIT" header; a local edit is overwritten on the next build + and diverges from what the converters enforce in the meantime. +- **Do not delete the methodology content.** The German survey-methodology + material is this skill's actual value-add and exists nowhere upstream. Only the + XLSForm *specification* sections go. +- **Do not reimplement subset rules in the app.** Call `XLSValidator`. If a rule + looks wrong or missing, that is a finding for this repo. +- **Never ship an invalid form silently.** Cap the repair attempts, then deliver + with the remaining findings shown — a partially convertible form plus an honest + warning beats both a spinner that never resolves and a clean-looking download + that fails later. +- **Pin to tags, not branches.** A registry change should never alter a shipped + skill or a deployed app without a commit in formulaid. + +## Related documents + +- [`README.md`](README.md) — what this repo ships and to whom +- [`ARCHITECTURE.md`](ARCHITECTURE.md) — registry → codegen → artifacts +- `codegen/emit_skill.py` — the emitter that writes `skills/cdl-survey-types/`, + and the contract it assumes with the parent skill +- [`HANDOVER_QWAC.md`](HANDOVER_QWAC.md) — the question-bank browser's plan, + sharing the catalogue gap diff --git a/HANDOVER_QWAC.md b/HANDOVER_QWAC.md new file mode 100644 index 0000000..40c24a2 --- /dev/null +++ b/HANDOVER_QWAC.md @@ -0,0 +1,144 @@ +# Handover: aligning qwac with this registry + +Audience: whoever (person or agent) picks up +[`CorrelAid/qwac`](https://github.com/CorrelAid/qwac) next — the SvelteKit SPA +that browses the question bank. This document is the plan; the work is split into +issues in that repo, linked below. Each issue stands on its own — an agent should +be able to act on one without reading the others. + +Companion documents: +[`HANDOVER_FORMULAID.md`](HANDOVER_FORMULAID.md) (the survey generator, which has +the same upstream gap) and +[`HANDOVER_FORMTRANSFORM_APP.md`](HANDOVER_FORMTRANSFORM_APP.md) (the converter +app, a dependency migration rather than a drift problem). + +## The problem + +qwac has **no dependency on this repo at all**, yet it renders qwacback data whose +question types come from here, and it validates DDI uploads against rules +generated here. It describes the survey world in its own words, and those words +are correct only until a question type is added, renamed or archived in +`registry/`. + +Two concrete symptoms: + +- **Labels are computed, not looked up.** + `src/lib/components/AnswerTypeTag.svelte` strips `_other` / `_long_list`, + replaces underscores with spaces and title-cases the rest. So `select_one` + renders as "Select One" instead of the registry's own `skos:prefLabel`, and any + type whose name does not survive that transformation renders wrongly. +- **The type list exists twice.** `src/lib/components/question-types/*.svelte` + holds one preview component per type (`TextInput`, `IntegerInput`, + `DecimalInput`, `DateInput`, `DateTimeInput`, `TimeInput`, `SelectOneInput`, + `SelectMultipleInput`, `LongListInput`), and whatever dispatches between them + encodes the type set a second time. Nothing catches a registry type that has no + component — it silently renders as a bare tag. + +Separately, the DDI upload page is opaque about what it checks. It POSTs to +qwacback's `/api/validate`, which validates against the DDI-Codebook 2.5 XSDs +**and** the CDL Schematron rules generated from this registry +(`ddi-validation/ddi_custom_rules.sch`), both baked into the +`ghcr.io/correlaid/schematron-worker` image. A rejection therefore means either +"not valid DDI 2.5" or "valid DDI that is not CDL-shaped" — very different +messages for a user, and the page currently gives neither. +`src/lib/validation.ts`'s `safeErrorMessage()` discards backend detail by design, +which is the right default everywhere except the one screen whose whole job is +explaining what is wrong with a file. + +This registry is an **allowlist**: unregistered question types, unregistered +appearances, over-long identifiers, nesting deeper than three levels and selects +without an explicit `list_name` are *rejected*, not approximated — see +[Supported XLSForm Subset](README.md#supported-xlsform-subset). + +## What this repo can and cannot hand over + +Available now from the package root (`github:CorrelAid/formtransform`): + +- `TYPE_MAPPINGS` — per-type mapping facts: `kind`, `limeSurveyType`, + `supported`, `requiresListName`, `answerClass`, `dateFormat`. +- The four conversion directions (`XLSFormParser.convertXLSDataToTSV`, + `buildDdiXml`, `lstsvToDdiXml`, `lstsvToXlsform`) — qwac needs none of them + today, but `lstsvToDdiXml` is worth knowing about if the app ever grows an + export path. +- `skills/cdl-survey-types/references/question-types.md` in this repo — not part + of the package, but its `##` headings carry the registry's `skos:prefLabel` for + every type, which is where the labels for issue #9 come from. + +**Not available, and this is what shapes the plan:** there is no labelled, +machine-readable type catalogue. `skos:prefLabel`, `useWhen` and the variant → +base relation live in `registry/`, and the package sets `"files": ["dist"]`, so +`registry/` never reaches a consumer's `node_modules`. +`src/generated/Appearances.ts` — including the `carriesData` flag a preview UI +wants, false for matrix headers — is generated but not re-exported from +`src/index.ts`. + +Both gaps are an addition to `codegen/emit_ts.py` plus an export line: derived +data, not a registry change. `CorrelAid/formulaid` needs the same catalogue, so +**one upstream issue can serve both repos** — whoever files first should link the +other. + +## The issues + +| # | Issue | When | +|---|---|---| +| 1 | [#9 Centralise question-type knowledge in one module](https://github.com/CorrelAid/qwac/issues/9) | first; no new dependency | +| 2 | [#10 Import the type catalogue from `@correlaid/formtransform`](https://github.com/CorrelAid/qwac/issues/10) | after #9 **and** after the upstream catalogue ships | +| 3 | [#11 Upstream gaps: labelled catalogue, `APPEARANCES` export](https://github.com/CorrelAid/qwac/issues/11) | now; tracking only, blocks #10 | +| — | [#12 Upload page: show why DDI validation failed](https://github.com/CorrelAid/qwac/issues/12) | any time; relates to existing #8 | + +Issue #9 is deliberately doable with no dependency at all: it collapses the +scattered type knowledge into one `src/lib/questionTypes.ts`, whose contents #10 +then swaps for the imported catalogue in a single edit. Splitting it this way +means the useful half of the work is not blocked on an upstream release. + +Issue #10 also adds the payoff test — when the registry gains a data-carrying +question type that has no preview component, the suite fails and the app gets +told, instead of quietly rendering a bare tag. + +## qwacback, while you are here + +Two things worth filing against +[`CorrelAid/qwacback`](https://github.com/CorrelAid/qwacback) rather than fixing +in qwac: + +- **`src/lib/ddi.ts` should not exist.** It is a ~100-line parser for Go + `fmt.Sprintf("%v", map)` output — `map[#text:value -attr:value]` — that + qwacback stores in PocketBase text fields. The app is reverse-engineering Go's + debug format to render DDI content. If the API returned JSON (or the DDI XML + itself), `ddi.ts` and its test could be deleted outright. +- **Structured validation findings.** If `/api/validate` collapses XSD and + Schematron failures into one opaque blob, issue #12 cannot do its job properly; + ask for findings that distinguish the two. + +qwacback also has its own registry-consumption brief, currently untracked at +`REGISTRY_SYNC.md` in that working copy — delete the vendored Go converter, the +vendored XSDs/Schematron and the Java worker source; consume +`@correlaid/formtransform` plus `ghcr.io/correlaid/schematron-worker` pinned to +one version. The plan still holds, but three details in it went stale: the repo +is `CorrelAid/formtransform` (not `survey-type-registry`), example fixtures live +at `registry/entities//fixtures/xlsform.json` (not +`registry/types//examples//xlsform.json`), and `lstsv2ddi` / +`lstsv2xlsform` now exist, which the brief predates. + +## Hard rules for whoever does the work + +- **Do not modify this repo from qwac.** Missing capability → open an issue here + (that is what #11 is for). +- **Do not vendor `registry/`.** A copied JSON-LD graph is drift with extra + steps. If the data is not exported, the fix is an emitter change here. +- **Do not hand-edit generated files.** Everything under `src/generated/` and + `skills/cdl-survey-types/` carries a "DO NOT EDIT" header and is overwritten by + the next `uv run codegen`. +- **Pin to tags, not branches.** A registry change should never alter a deployed + UI without a commit in qwac. +- **Unknown types must still render.** qwacback can hold data the app has not + been taught about; degrade to a plain tag rather than throwing. + +## Related documents + +- [`README.md`](README.md) — what this repo ships and to whom +- [`ARCHITECTURE.md`](ARCHITECTURE.md) — registry → codegen → artifacts +- [`HANDOVER_FORMULAID.md`](HANDOVER_FORMULAID.md) — the survey generator's plan, + sharing the catalogue gap +- [`HANDOVER_FORMTRANSFORM_APP.md`](HANDOVER_FORMTRANSFORM_APP.md) — the converter + app's migration diff --git a/HANDOVER_QWAC_FORMULAID.md b/HANDOVER_QWAC_FORMULAID.md deleted file mode 100644 index aaf93aa..0000000 --- a/HANDOVER_QWAC_FORMULAID.md +++ /dev/null @@ -1,168 +0,0 @@ -# Handover: aligning qwac and formulaid with this registry - -Audience: whoever (person or agent) picks up -[`CorrelAid/qwac`](https://github.com/CorrelAid/qwac) or -[`CorrelAid/formulaid`](https://github.com/CorrelAid/formulaid) next. This -document is the plan; the work is split into issues in those repos, linked below. -Each issue stands on its own — an agent should be able to act on one without -reading the others. - -Companion document: -[`HANDOVER_FORMTRANSFORM_APP.md`](HANDOVER_FORMTRANSFORM_APP.md), which covers the -frontend converter app. The situation there was a *dependency* problem (two -conversion engines, one library to replace them). Here it is a *drift* problem: -neither repo runs a competing converter, but both carry hand-maintained copies of -knowledge this registry owns. - -## Which repo consumes what - -| Repo | Relationship to this registry today | What should change | -|---|---|---| -| **qwac** (SvelteKit SPA, question-bank browser) | No dependency at all. Renders qwacback data, whose types come from here, using its own hardcoded type vocabulary | Import the type catalogue instead of hardcoding it; explain what its DDI validation actually checks | -| **formulaid** (SvelteKit app + `generating-xlsforms` Claude skill) | None in code. The skill teaches XLSForm from `umfragen.civic-data.de` llm.txt; the app builds workbooks with `xlsx` and never validates them | Bundle the generated `cdl-survey-types` sub-skill; validate generated forms with the library | -| **qwacback** (Go/PocketBase API) | Consumes the schematron-worker image and the DDI validation assets | Already has its own brief — see [qwacback](#qwacback-already-briefed) | - -## The shared failure mode - -This registry is an **allowlist**. Unregistered question types, unregistered -appearances, over-long identifiers, nesting deeper than three levels and selects -without an explicit `list_name` are *rejected*, not approximated — see -[Supported XLSForm Subset](README.md#supported-xlsform-subset). - -Both repos currently describe the survey world in their own words: - -- qwac derives human labels from type strings by text surgery - (`AnswerTypeTag.svelte` strips `_other` / `_long_list`, replaces underscores, - title-cases), so `select_one` shows as "Select One" rather than the registry's - own `skos:prefLabel`. Its `question-types/*.svelte` preview components encode - the type list a second time, with nothing to catch a registry type that has no - component. -- formulaid's skill ships ~1000 lines of general XLSForm specification — - `calculation`, "Anhang – Laden großer CSV-Dateien", "Weitere Antwortformate", - "Nicht empfohlene Antwortformate" — which describes far more than the pipeline - accepts. A model reading it will confidently produce forms that fail - conversion, and the failure surfaces in a different repo. Its app also hardcodes - `QuestionType = 'select_one' | 'select_multiple' | 'text' | 'integer' | 'decimal' | 'date' | 'note'` - and never checks its output. - -Neither is a bug today. Both become wrong the moment a question type is added, -renamed or archived here. - -## What this repo can and cannot hand over - -Available now, from the package root (`github:CorrelAid/formtransform`): - -- `XLSLoader.parseXLSData(data, { skipValidation? })` — parse a workbook; - validates against the supported subset by default and throws. -- `XLSValidator` + the `SubsetViolation` type — validate and get findings back - instead of an exception. This is what formulaid needs for a repair loop. -- `XLSFormParser.convertXLSDataToTSV`, `buildDdiXml`, `lstsvToDdiXml`, - `lstsvToXlsform` — the four supported directions. -- `TYPE_MAPPINGS` — per-type mapping facts: `kind`, `limeSurveyType`, - `supported`, `requiresListName`, `answerClass`, `dateFormat`. -- `skills/cdl-survey-types/` — the generated sub-skill (`SKILL.md` plus - `references/question-types.md` and `references/xlsform-syntax.md`), portable to - any agent runtime. Not part of the npm-format package; fetched from the repo at - a pinned tag. - -**Not available, and this is the constraint that shapes both plans:** there is no -labelled, machine-readable type catalogue. `skos:prefLabel`, `useWhen` and the -variant → base relation exist in `registry/` and are rendered into the generated -skill and the docs site, but nothing exports them as data — and the published -package sets `"files": ["dist"]`, so `registry/` never reaches a consumer's -`node_modules`. `src/generated/Appearances.ts` (including the `carriesData` flag a -preview UI wants) is generated but not re-exported from `src/index.ts`. - -Both tracking issues below ask for the same thing: an addition to -`codegen/emit_ts.py` emitting a `QUESTION_TYPES` record with labels, exported from -the package root. It is derived data — every field is already in the registry — so -it is an emitter change, not a registry change. **One upstream issue can serve -both repos**; whoever files first should link the other. - -## The issues - -### qwac - -| # | Issue | When | -|---|---|---| -| 1 | [#9 Centralise question-type knowledge in one module](https://github.com/CorrelAid/qwac/issues/9) | first; no new dependency | -| 2 | [#10 Import the type catalogue from `@correlaid/formtransform`](https://github.com/CorrelAid/qwac/issues/10) | after #9 **and** after the upstream catalogue ships | -| 3 | [#11 Upstream gaps: labelled catalogue, `APPEARANCES` export](https://github.com/CorrelAid/qwac/issues/11) | now; tracking only, blocks #10 | -| — | [#12 Upload page: show why DDI validation failed](https://github.com/CorrelAid/qwac/issues/12) | any time; relates to existing #8 | - -Issue #9 is deliberately doable with no dependency: it collapses the scattered type -knowledge into one `src/lib/questionTypes.ts` whose contents #10 then swaps for -the imported catalogue in a single edit. The labels for #9 are copied from -`skills/cdl-survey-types/references/question-types.md` in this repo — the `##` -headings carry the registry's own `prefLabel`. - -Issue #12 is the qwac counterpart of the app's scope notice. Its upload page POSTs DDI -XML to qwacback's `/api/validate`, which validates against the DDI 2.5 XSDs **and** -the CDL Schematron rules generated here — so a rejection can mean "not valid DDI" -or "valid DDI that is not CDL-shaped", and the page currently says neither. -`src/lib/validation.ts`'s `safeErrorMessage()` throws the detail away on purpose, -which is the right default everywhere except the screen whose entire job is -explaining what is wrong with a file. - -### formulaid - -| # | Issue | When | -|---|---|---| -| 1 | [#9 Bundle the generated `cdl-survey-types` sub-skill into the skill build](https://github.com/CorrelAid/formulaid/issues/9) | first | -| 2 | [#10 Strip the generic XLSForm spec from the skill references](https://github.com/CorrelAid/formulaid/issues/10) | after #9 — never before | -| 3 | [#11 Validate generated workbooks with the library before download](https://github.com/CorrelAid/formulaid/issues/11) | any time; app-side, independent of the skill work | -| — | [#12 Upstream gaps: labelled catalogue, sub-skill release asset](https://github.com/CorrelAid/formulaid/issues/12) | now; tracking only | - -Order matters between #9 and #10: #10 removes the reference content that #9 -replaces, and doing it the other way round leaves the skill with no type -reference at all for however long the gap lasts. - -`scripts/build_skill.sh` already fetches remote content (llm.txt, qwacback -demographics) into `references/`, so vendoring the sub-skill from a pinned tag -fits the existing shape. The one hard requirement: a failed fetch must fail the -build. A skill shipped without its type reference is worse than a stale one, -because the model silently falls back on generic XLSForm knowledge. - -Issue #11 implements existing formulaid issue -[#4](https://github.com/CorrelAid/formulaid/issues/4) ("Let xlsform be validated -and try again with error messages"), which asks whether to add validation to -qwacback or "use some existing package". The package is this library, it runs in -the browser, and it needs no service. - -## qwacback (already briefed) - -`CorrelAid/qwacback` has its own migration brief, currently untracked at -`REGISTRY_SYNC.md` in that working copy. Its plan still holds — delete the -vendored Go converter, the vendored XSDs/Schematron and the Java worker source; -consume `@correlaid/formtransform` plus `ghcr.io/correlaid/schematron-worker` -pinned to one version — but it was written before this repo was renamed and -restructured, so three details in it are stale: - -- the repo is `CorrelAid/formtransform`, not `CorrelAid/survey-type-registry` -- example fixtures live at `registry/entities//fixtures/xlsform.json`, not - `registry/types//examples//xlsform.json` -- `lstsv2ddi` and `lstsv2xlsform` now exist, which the brief predates - -Worth filing as issues in that repo the same way, once someone picks it up. - -## Hard rules for whoever does the work - -- **Do not modify this repo from a consumer.** Missing capability → open an issue - here. Both tracking issues (#11 in qwac, #12 in formulaid) exist for exactly - that. -- **Do not vendor `registry/`.** A copied JSON-LD graph is drift with extra - steps. If the data is not exported, the fix is an emitter change here. -- **Do not hand-edit generated files.** Everything under - `skills/cdl-survey-types/` and `src/generated/` carries a "DO NOT EDIT" header - and is overwritten by the next `uv run codegen`. -- **Do not reimplement subset rules in a consumer.** Call `XLSValidator`. If a - rule looks wrong, that is a finding for this repo. -- **Pin to tags, not branches.** A registry change should never alter a deployed - UI or a shipped skill without a commit in the consuming repo. - -## Related documents - -- [`README.md`](README.md) — what this repo ships and to whom -- [`ARCHITECTURE.md`](ARCHITECTURE.md) — registry → codegen → artifacts -- [`HANDOVER_FORMTRANSFORM_APP.md`](HANDOVER_FORMTRANSFORM_APP.md) — the frontend - converter app's migration diff --git a/README.md b/README.md index 71fcecd..4a95342 100644 --- a/README.md +++ b/README.md @@ -144,8 +144,10 @@ npm run bless - [Architecture](ARCHITECTURE.md) — Technical architecture and internal structure - [formtransform-app handover](HANDOVER_FORMTRANSFORM_APP.md) — plan for moving the frontend app onto this library (issue-by-issue) -- [qwac + formulaid handover](HANDOVER_QWAC_FORMULAID.md) — plan for aligning the - question-bank browser and the survey generator with this registry +- [qwac handover](HANDOVER_QWAC.md) — plan for aligning the question-bank browser + with this registry +- [formulaid handover](HANDOVER_FORMULAID.md) — plan for aligning the survey + generator and its Claude skill with this registry - [Claude Code Integration](CLAUDE.md) — Claude-specific features and skills - [Pipeline Documentation](src/pipelines/README.md) — Transformation pipeline details - [Test Documentation](tests/README.md) — Test structure and running tests