From bf59a57ea3730a4221e0333b7787ad5d47fb8591 Mon Sep 17 00:00:00 2001 From: jstet Date: Wed, 19 Aug 2026 12:24:20 +0200 Subject: [PATCH] feat(xlsform2ddi): emit the DDI response-data CSV MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `buildDdiXml` describes a dataset it could not produce: given `submissions` it used their count for `` and a filename in ``, but nothing here wrote the data file itself. Consumers had to keep a Python runtime around for `survey2ddi_core.data.build_data_csv` — the app boots Pyodide and the `survey2ddi` wheel for that one call. Adds `pipelines/xlsform2ddi/data.ts`: - `buildDataCsv(variables, submissions)` — RFC 4180 CSV (CRLF, minimal quoting), header row of DDI variable names, submissions in input order. - `getDdiColumnNames(variables)` / `remapSubmissionsToDdi(variables, rows)` for callers that write the file themselves. The contract is that column order equals `` order, so schema and data align positionally. The column plan therefore walks the same buckets `dataDscr` does — grid members, `select_multiple` binaries, `_other` patterns, standalone vars — which required exporting `splitDataVars` and its types from `ddi/codebook.ts`. A `select_multiple` expands to one `0`/`1` column per choice, `_other` multi patterns drop the `other` binary in favour of the text column (matching `emitOtherPatternVars`), and `note` variables get no column. Note this diverges from the Python `get_canonical_columns`, which documents XML order but emits input order — its CSV header does not match its own XML once a grid, `select_multiple` or `_other` pattern reorders the buckets. Cell values are byte-identical to the Python emitter; only the column order is fixed here. Response rows are keyed by bare question name or by the slash-joined group path (`group/name`) that Kobo's CSV export uses. Both are accepted, bare name wins, so no `data_key` field is needed on `Variable`. Also exports `extractVariables` / `choicesByListFromRows` / `normalizeChoices` from the package root — building a `Variable[]` for `buildDataCsv` was otherwise impossible from outside the library. Closes #5 Co-Authored-By: Claude Opus 5 --- README.md | 18 ++ src/ddi/codebook.ts | 6 +- src/index.ts | 11 + src/pipelines/README.md | 18 ++ src/pipelines/xlsform2ddi/data.ts | 188 +++++++++++++++ src/pipelines/xlsform2ddi/index.ts | 7 + .../unit/pipelines/xlsform2ddi/data.test.ts | 228 ++++++++++++++++++ 7 files changed, 473 insertions(+), 3 deletions(-) create mode 100644 src/pipelines/xlsform2ddi/data.ts create mode 100644 tests/ts/unit/pipelines/xlsform2ddi/data.test.ts diff --git a/README.md b/README.md index f558944..3f9f54b 100644 --- a/README.md +++ b/README.md @@ -44,6 +44,24 @@ const converter = new XLSFormToTSVConverter(); const tsv = await converter.convert(survey, choices, settings); ``` +A DDI codebook and the data file it describes come from the same variable list, +so the CSV headers match the XML `` elements one-to-one: + +```typescript +import { + buildDdiXml, + buildDataCsv, + extractVariables, + choicesByListFromRows, +} from '@correlaid/formtransform'; + +const xml = buildDdiXml(survey, choices, { settings: settings[0], submissions }); +const csv = buildDataCsv( + extractVariables(survey, choicesByListFromRows(choices)), + submissions, +); +``` + ### Command Line ```bash diff --git a/src/ddi/codebook.ts b/src/ddi/codebook.ts index fcd38cc..e42142c 100644 --- a/src/ddi/codebook.ts +++ b/src/ddi/codebook.ts @@ -127,7 +127,7 @@ function addBinaryVar( return varEl; } -interface OtherPattern { +export interface OtherPattern { base: Variable; otherVar: Variable; isMulti: boolean; @@ -260,7 +260,7 @@ export interface BuildDdiOptions { } /** Returned by {@link splitDataVars}: every data var bucketed by its emit role. */ -interface DataVarBuckets { +export interface DataVarBuckets { otherPatterns: Map; gridGroups: Map; multiRespGroups: Map; @@ -268,7 +268,7 @@ interface DataVarBuckets { } /** Sort the flat data vars into the four emit roles `dataDscr` walks through. */ -function splitDataVars(dataVars: Variable[]): DataVarBuckets { +export function splitDataVars(dataVars: Variable[]): DataVarBuckets { const otherPatterns = detectOtherPatterns(dataVars); const baseNamesInOther = new Set( [...otherPatterns.values()].map((p) => p.base.name), diff --git a/src/index.ts b/src/index.ts index 76b5cbd..ed10228 100644 --- a/src/index.ts +++ b/src/index.ts @@ -32,6 +32,17 @@ export { } from './pipelines/xlsform2lstsv/typeMapper.js'; export { buildDdiXml } from './pipelines/xlsform2ddi/index.js'; +export { + extractVariables, + choicesByListFromRows, + normalizeChoices, +} from './pipelines/xlsform2ddi/variables.js'; +export { + buildDataCsv, + getDdiColumnNames, + remapSubmissionsToDdi, +} from './pipelines/xlsform2ddi/data.js'; +export type { Submission } from './pipelines/xlsform2ddi/data.js'; export { lstsvToDdiXml, diff --git a/src/pipelines/README.md b/src/pipelines/README.md index aea774a..e0c1dc9 100644 --- a/src/pipelines/README.md +++ b/src/pipelines/README.md @@ -15,6 +15,24 @@ Both DDI pipelines converge on the same emitter: they produce `Variable[]` (`src/ddi/types.ts`) and hand it to `buildDdiCodebook`. The variable model is the hub, not any one format. +## The DDI data file + +`xlsform2ddi/data.ts` emits the response-data CSV that the codebook describes +(`buildDataCsv`, plus `getDdiColumnNames` / `remapSubmissionsToDdi` for callers +writing the file themselves). It is schema-side-agnostic in the same way the XML +emitter is: it takes `Variable[]` and raw response rows, so either DDI pipeline +can feed it. + +Its one hard contract is **column order equals `` order**. The +column plan walks the same buckets `dataDscr` does — grid-group members, +`select_multiple` binaries, `_other` patterns, standalone variables — so every +header matches a `` in the XML, position for position, and schema↔data +alignment stays a zip rather than a lookup. A `select_multiple` expands to one +`0`/`1` column per choice; `note` variables get no column at all. + +Response rows are keyed by bare question name or by the slash-joined group path +(`group/name`) Kobo's CSV export uses — both are accepted, bare name wins. + ## Why there is no `ddi2xlsform` or `ddi2lstsv` Deliberate, not a gap. **DDI is the terminus of the pipeline graph** — it diff --git a/src/pipelines/xlsform2ddi/data.ts b/src/pipelines/xlsform2ddi/data.ts new file mode 100644 index 0000000..ceea5ba --- /dev/null +++ b/src/pipelines/xlsform2ddi/data.ts @@ -0,0 +1,188 @@ +/** + * Response-data CSV emitter, kept in lock-step with the DDI XML schema. + * + * `ddi/codebook.ts` expands every `select_multiple` question into N binary + * `` elements (one per choice, named `_`). The CSV + * mirrors that expansion column for column, so every header matches a + * `` in the XML — in the same order the emitter writes them. + * + * Source-agnostic: any adapter producing `Variable[]` plus raw response rows + * can use it. `select_multiple` values are expected as space-joined choice + * codes (what Kobo and the LimeSurvey adapters both produce). + */ + +import { splitDataVars } from '../../ddi/codebook.js'; +import type { DataVarBuckets, OtherPattern } from '../../ddi/codebook.js'; +import { classifyNotes } from '../../ddi/notes.js'; +import type { Variable } from '../../ddi/types.js'; + +/** One raw response record, keyed by question name or `group/name` path. */ +export type Submission = Record; + +/** + * One CSV column: either the variable's own value (`single`) or one binary + * `0`/`1` membership flag of a `select_multiple` choice (`binary`). + */ +interface Column { + name: string; + variable: Variable; + /** Choice code this column flags; `''` for a `single` column. */ + choice: string; +} + +function single(variable: Variable, name = variable.name): Column { + return { name, variable, choice: '' }; +} + +function binary(variable: Variable, choice: string): Column { + return { name: `${variable.name}_${choice}`, variable, choice }; +} + +/** Columns for one `_other` pattern, mirroring `emitOtherPatternVars`. */ +function otherPatternColumns(p: OtherPattern): Column[] { + const cols: Column[] = p.isMulti + ? p.base.choices + .filter((c) => c.name !== 'other') + .map((c) => binary(p.base, c.name)) + : [single(p.base)]; + // The `_other` free text is always its own column. + cols.push(single(p.otherVar)); + return cols; +} + +/** + * Ordered column plan for a variable list. + * + * The walk order is the one `addVars` in `ddi/codebook.ts` uses: grid-group + * members, then `select_multiple` binaries, then `_other` patterns, then + * standalone variables. `note` variables carry no data and are skipped. + */ +function columnPlan(variables: Variable[]): Column[] { + const { dataVars } = classifyNotes(variables); + const buckets: DataVarBuckets = splitDataVars(dataVars); + const { gridGroups, multiRespGroups, otherPatterns, standaloneVars } = + buckets; + + const cols: Column[] = []; + for (const members of gridGroups.values()) { + cols.push(...members.map((v) => single(v))); + } + for (const smVar of multiRespGroups.values()) { + cols.push(...smVar.choices.map((c) => binary(smVar, c.name))); + } + for (const p of otherPatterns.values()) { + cols.push(...otherPatternColumns(p)); + } + cols.push(...standaloneVars.map((v) => single(v))); + return cols; +} + +/** + * Read a variable's raw value from a submission. + * + * Adapters key rows either by bare question name (Kobo's `select_multiple` + * flat export, LimeSurvey) or by the grouped path Kobo's CSV export uses + * (`group/name`, nested groups slash-joined). Both are accepted; the bare + * name wins when a row carries both. + */ +function readCell(row: Submission, v: Variable): unknown { + if (v.name in row) return row[v.name]; + if (v.group) { + const path = `${v.group}/${v.name}`; + if (path in row) return row[path]; + } + return ''; +} + +/** + * Stringify a cell. `null`/`undefined` become `''` (never `"None"`), and so do + * objects/arrays — a data column holds one scalar per respondent. + */ +function cellText(raw: unknown): string { + if (typeof raw === 'string') return raw; + if (typeof raw === 'number' || typeof raw === 'boolean') return String(raw); + return ''; +} + +/** Selected choice codes of a space-joined `select_multiple` value. */ +function selectedCodes(raw: unknown): Set { + return new Set(cellText(raw).split(/\s+/).filter(Boolean)); +} + +/** + * DDI variable names in the order `buildDdiXml` emits ``. + * + * `select_multiple` expands to one `_` column per choice; every + * other variable contributes a single column equal to `variable.name`. + */ +export function getDdiColumnNames(variables: Variable[]): string[] { + return columnPlan(variables).map((c) => c.name); +} + +/** + * Re-key raw submissions onto DDI variable names. + * + * Each output row has exactly one entry per {@link getDdiColumnNames} column, + * in that order. `select_multiple` values (space-joined choice codes) expand + * into per-choice `"0"` / `"1"` columns; everything else becomes one string. + */ +export function remapSubmissionsToDdi( + variables: Variable[], + submissions: Submission[], +): Record[] { + return rowsFromPlan(columnPlan(variables), submissions); +} + +/** Shared body of {@link remapSubmissionsToDdi} over an existing plan. */ +function rowsFromPlan( + cols: Column[], + submissions: Submission[], +): Record[] { + return submissions.map((row) => { + const out: Record = {}; + const multiCache = new Map>(); + for (const col of cols) { + if (col.choice) { + let selected = multiCache.get(col.variable.name); + if (!selected) { + selected = selectedCodes(readCell(row, col.variable)); + multiCache.set(col.variable.name, selected); + } + out[col.name] = selected.has(col.choice) ? '1' : '0'; + } else { + out[col.name] = cellText(readCell(row, col.variable)); + } + } + return out; + }); +} + +/** RFC 4180 minimal quoting: quote only on delimiter, quote, or line break. */ +function quoteField(value: string): string { + if (!/[",\r\n]/.test(value)) return value; + return `"${value.replace(/"/g, '""')}"`; +} + +function csvLine(fields: string[]): string { + return `${fields.map(quoteField).join(',')}\r\n`; +} + +/** + * Build the response-data CSV for a DDI codebook. + * + * RFC 4180: CRLF line endings, minimal quoting, header row of DDI variable + * names in XML order followed by the submissions in input order. Callers own + * validation — a malformed submission row is emitted as read. + */ +export function buildDataCsv( + variables: Variable[], + submissions: Submission[], +): string { + const cols = columnPlan(variables); + const names = cols.map((c) => c.name); + const rows = rowsFromPlan(cols, submissions); + return ( + csvLine(names) + + rows.map((row) => csvLine(names.map((n) => row[n]))).join('') + ); +} diff --git a/src/pipelines/xlsform2ddi/index.ts b/src/pipelines/xlsform2ddi/index.ts index c0d8deb..5dc7157 100644 --- a/src/pipelines/xlsform2ddi/index.ts +++ b/src/pipelines/xlsform2ddi/index.ts @@ -35,6 +35,13 @@ export function buildDdiXml( return buildDdiCodebook(variables, options).toDocument(); } +export { + buildDataCsv, + getDdiColumnNames, + remapSubmissionsToDdi, +} from './data.js'; +export type { Submission } from './data.js'; + export { extractVariables, choicesByListFromRows, diff --git a/tests/ts/unit/pipelines/xlsform2ddi/data.test.ts b/tests/ts/unit/pipelines/xlsform2ddi/data.test.ts new file mode 100644 index 0000000..26065e3 --- /dev/null +++ b/tests/ts/unit/pipelines/xlsform2ddi/data.test.ts @@ -0,0 +1,228 @@ +/** Response-data CSV emitter — column plan, binary expansion, RFC 4180 output. */ +import { describe, test, expect } from 'vitest'; + +import { + buildDataCsv, + getDdiColumnNames, + remapSubmissionsToDdi, +} from '../../../../../src/pipelines/xlsform2ddi/data.js'; +import { + buildDdiXml, + extractVariables, + choicesByListFromRows, +} from '../../../../../src/pipelines/xlsform2ddi/index.js'; +import type { Variable } from '../../../../../src/ddi/types.js'; + +type Row = Record; + +function v( + partial: Partial & { name: string; type: string }, +): Variable { + return { + label: '', + group: '', + groupLabel: '', + groupAppearance: '', + listName: '', + vocab: '', + choices: [], + ...partial, + }; +} + +/** `` values in document order. */ +function xmlVarNames(xml: string): string[] { + return [...xml.matchAll(/ m[1]); +} + +function variablesOf(survey: Row[], choices: Row[]): Variable[] { + return extractVariables(survey, choicesByListFromRows(choices)); +} + +const multiVar = v({ + name: 'colors', + type: 'select_multiple', + label: 'Colors', + listName: 'colors', + choices: [ + { name: 'red', label: 'Red' }, + { name: 'blue', label: 'Blue' }, + ], +}); + +describe('getDdiColumnNames', () => { + test('one column per non-categorical variable, in order', () => { + const cols = getDdiColumnNames([ + v({ name: 'q1', type: 'text' }), + v({ name: 'q2', type: 'integer' }), + ]); + expect(cols).toEqual(['q1', 'q2']); + }); + + test('expands select_multiple into one binary column per choice', () => { + expect(getDdiColumnNames([multiVar])).toEqual([ + 'colors_red', + 'colors_blue', + ]); + }); + + test('skips note variables — they carry no data', () => { + const cols = getDdiColumnNames([ + v({ name: 'intro', type: 'note', label: 'Hello' }), + v({ name: 'q1', type: 'text' }), + ]); + expect(cols).toEqual(['q1']); + }); + + test('matches the DDI XML order for a mixed survey', () => { + const survey: Row[] = [ + { type: 'note', name: 'intro', label: 'Welcome' }, + { type: 'text', name: 'q_free', label: 'Free text' }, + { type: 'select_multiple colors', name: 'colors', label: 'Colors' }, + { + type: 'begin_group', + name: 'grid', + label: 'Grid', + appearance: 'field-list', + }, + { type: 'select_one likert', name: 'g1', label: 'G1' }, + { type: 'select_one likert', name: 'g2', label: 'G2' }, + { type: 'end_group', name: 'grid' }, + { type: 'select_one colors or_other', name: 'fav', label: 'Favourite' }, + { type: 'integer', name: 'age', label: 'Age' }, + ]; + const choices: Row[] = [ + { list_name: 'colors', name: 'red', label: 'Red' }, + { list_name: 'colors', name: 'blue', label: 'Blue' }, + { list_name: 'likert', name: '1', label: 'Low' }, + { list_name: 'likert', name: '2', label: 'High' }, + ]; + const variables = variablesOf(survey, choices); + const xml = buildDdiXml(survey, choices); + expect(getDdiColumnNames(variables)).toEqual(xmlVarNames(xml)); + }); +}); + +describe('remapSubmissionsToDdi', () => { + test('binary columns are "1" for selected choices, "0" otherwise', () => { + const rows = remapSubmissionsToDdi([multiVar], [{ colors: 'blue' }]); + expect(rows).toEqual([{ colors_red: '0', colors_blue: '1' }]); + }); + + test('splits space-joined select_multiple values', () => { + const rows = remapSubmissionsToDdi([multiVar], [{ colors: 'red blue' }]); + expect(rows[0]).toEqual({ colors_red: '1', colors_blue: '1' }); + }); + + test('an empty or missing select_multiple selects nothing', () => { + const rows = remapSubmissionsToDdi([multiVar], [{ colors: '' }, {}]); + expect(rows[0]).toEqual({ colors_red: '0', colors_blue: '0' }); + expect(rows[1]).toEqual({ colors_red: '0', colors_blue: '0' }); + }); + + test('null and undefined become empty strings, never "None"', () => { + const vars = [ + v({ name: 'q1', type: 'text' }), + v({ name: 'q2', type: 'text' }), + ]; + const rows = remapSubmissionsToDdi(vars, [{ q1: null, q2: undefined }]); + expect(rows[0]).toEqual({ q1: '', q2: '' }); + }); + + test('stringifies numbers and booleans', () => { + const vars = [ + v({ name: 'n', type: 'integer' }), + v({ name: 'b', type: 'text' }), + ]; + const rows = remapSubmissionsToDdi(vars, [{ n: 42, b: true }]); + expect(rows[0]).toEqual({ n: '42', b: 'true' }); + }); + + test('falls back to the group/name path key', () => { + const vars = [v({ name: 'q1', type: 'text', group: 'sec/sub' })]; + const rows = remapSubmissionsToDdi(vars, [{ 'sec/sub/q1': 'from path' }]); + expect(rows[0]).toEqual({ q1: 'from path' }); + }); + + test('the bare name wins when a row carries both keys', () => { + const vars = [v({ name: 'q1', type: 'text', group: 'sec' })]; + const rows = remapSubmissionsToDdi(vars, [ + { q1: 'bare', 'sec/q1': 'path' }, + ]); + expect(rows[0]).toEqual({ q1: 'bare' }); + }); + + test('preserves submission order', () => { + const vars = [v({ name: 'q1', type: 'text' })]; + const rows = remapSubmissionsToDdi(vars, [{ q1: 'a' }, { q1: 'b' }]); + expect(rows.map((r) => r.q1)).toEqual(['a', 'b']); + }); +}); + +describe('buildDataCsv', () => { + test('emits a CRLF-terminated header row and one row per submission', () => { + const vars = [ + v({ name: 'q1', type: 'text' }), + v({ name: 'q2', type: 'integer' }), + ]; + const csv = buildDataCsv(vars, [ + { q1: 'a', q2: 1 }, + { q1: 'b', q2: 2 }, + ]); + expect(csv).toBe('q1,q2\r\na,1\r\nb,2\r\n'); + }); + + test('emits only the header when there are no submissions', () => { + expect(buildDataCsv([v({ name: 'q1', type: 'text' })], [])).toBe('q1\r\n'); + }); + + test('quotes only fields containing a delimiter, quote, or line break', () => { + const vars = [ + v({ name: 'plain', type: 'text' }), + v({ name: 'comma', type: 'text' }), + v({ name: 'quote', type: 'text' }), + v({ name: 'newline', type: 'text' }), + ]; + const csv = buildDataCsv(vars, [ + { + plain: 'no quoting needed', + comma: 'a,b', + quote: 'say "hi"', + newline: 'line1\nline2', + }, + ]); + expect(csv).toBe( + 'plain,comma,quote,newline\r\n' + + 'no quoting needed,"a,b","say ""hi""","line1\nline2"\r\n', + ); + }); + + test('uses CRLF exclusively for row separation', () => { + const csv = buildDataCsv([v({ name: 'q1', type: 'text' })], [{ q1: 'x' }]); + expect(csv.replace(/\r\n/g, '')).not.toContain('\n'); + }); + + test('expands select_multiple to binary columns in the header', () => { + const csv = buildDataCsv([multiVar], [{ colors: 'red' }]); + expect(csv).toBe('colors_red,colors_blue\r\n1,0\r\n'); + }); + + test('an _other multi pattern drops the other binary and keeps the text column', () => { + const survey: Row[] = [ + { type: 'select_multiple colors', name: 'colors', label: 'Colors' }, + { type: 'text', name: 'colors_other', label: 'Other' }, + ]; + const choices: Row[] = [ + { list_name: 'colors', name: 'red', label: 'Red' }, + { list_name: 'colors', name: 'other', label: 'Other' }, + ]; + const variables = variablesOf(survey, choices); + const csv = buildDataCsv(variables, [ + { colors: 'red other', colors_other: 'teal' }, + ]); + expect(csv).toBe('colors_red,colors_other\r\n1,teal\r\n'); + expect(getDdiColumnNames(variables)).toEqual( + xmlVarNames(buildDdiXml(survey, choices)), + ); + }); +});