diff --git a/tools/dictionary-parser/ParseWikidata b/tools/dictionary-parser/ParseWikidata index f19c1c44a..6dfb9e833 100755 --- a/tools/dictionary-parser/ParseWikidata +++ b/tools/dictionary-parser/ParseWikidata @@ -23,7 +23,7 @@ if [[ $# -eq 2 && $1 == "--all" ]]; then if [[ "$LAST_LINE" =~ $COMMAND_OPTS_PREFIX ]]; then OPTIONS=`echo $LAST_LINE | sed "s/$COMMAND_OPTS_PREFIX//"` echo Generating $LANGUAGE from $LEXEME_FILE - $RUN_COMMAND $OPTIONS --inflections $LEXICAL_DIR/inflectional_${LANGUAGE}.xml --dictionary $LEXICAL_DIR/dictionary_${LANGUAGE}.lst $LEXEME_FILE + $RUN_COMMAND $OPTIONS --inflections $LEXICAL_DIR/inflectional_${LANGUAGE}.xml --dictionary $LEXICAL_DIR/dictionary_${LANGUAGE}.lst --wikidata $LEXEME_FILE else echo skipping $LANGUAGE fi diff --git a/tools/dictionary-parser/README.md b/tools/dictionary-parser/README.md index 736bda902..20091ead9 100644 --- a/tools/dictionary-parser/README.md +++ b/tools/dictionary-parser/README.md @@ -20,5 +20,27 @@ These tools generate files that describes the grammatical properties of words fr ./ParseWikidata \ --inflections ../../resources/org/unicode/inflection/dictionary/inflectional_XX.xml \ --dictionary ../../resources/org/unicode/inflection/dictionary/dictionary_XX.lst \ - + --wikidata ``` + +## Usage for OASIS DMLex and Multi-Source Inputs + +`ParseWikidata` also supports [OASIS DMLex v1.0](https://docs.oasis-open.org/lexidma/dmlex/v1.0/dmlex-v1.0.html) JSON files via `--dmlex`, either standalone (bypassing Wikidata completely) or combined with `--wikidata`: + +- **Standalone DMLex (single or multiple files)**: +``` + ./ParseWikidata --language \ + --inflections inflectional_.xml \ + --dictionary dictionary_.lst \ + --dmlex [ ...] +``` +- **Multi-Source (Wikidata + DMLex)**: +``` + ./ParseWikidata \ + --inflections inflectional_.xml \ + --dictionary dictionary_.lst \ + --dmlex [ ...] \ + --wikidata +``` + - Multiple `--wikidata` and `--dmlex` inputs are processed in command-line argument order, merging grammemes and inflection patterns across sources (prioritizing earlier sources' inflection patterns when rarity and pattern frequency counts are equal). + diff --git a/tools/dictionary-parser/src/main/java/org/unicode/wikidata/DMLexEntry.java b/tools/dictionary-parser/src/main/java/org/unicode/wikidata/DMLexEntry.java new file mode 100644 index 000000000..45d15d63e --- /dev/null +++ b/tools/dictionary-parser/src/main/java/org/unicode/wikidata/DMLexEntry.java @@ -0,0 +1,106 @@ +/* + * Copyright 2025-2026 Unicode Incorporated and others. All rights reserved. + */ +package org.unicode.wikidata; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; + +import java.util.List; + +/** + * A lexical entry in OASIS DMLex v1.0 JSON format (spec §5.2, {@code dmlex.schema.json}). + */ +@JsonIgnoreProperties({ + "homographNumber", + "senses", + "placeholderMarkers", + "etymologies" +}) +public class DMLexEntry { + /** + * Root {@code lexicographicResource} object in OASIS DMLex v1.0 JSON format. + */ + @JsonIgnoreProperties({ + "title", + "uri", + "translationLanguages", + "definitionTypeTags", + "labelTypeTags", + "sourceIdentityTags", + "transcriptionSchemeTags", + "relations", + "relationTypes", + "etymonLanguages", + "etymonTypes" + }) + public static class Lexicon { + public String langCode; + public List entries; + public List partOfSpeechTags; + public List inflectedFormTags; + public List labelTags; + } + + /** + * A {@code partOfSpeechTag} controlled value in OASIS DMLex v1.0 JSON format. + */ + @JsonIgnoreProperties({ + "description", + "for" + }) + public static class PartOfSpeechTag { + public String tag; + public List sameAs; + } + + /** + * An {@code inflectedFormTag} controlled value in OASIS DMLex v1.0 JSON format. + */ + @JsonIgnoreProperties({ + "description", + "for" + }) + public static class InflectedFormTag { + public String tag; + public List sameAs; + } + + /** + * A {@code labelTag} controlled value in OASIS DMLex v1.0 JSON format. + */ + @JsonIgnoreProperties({ + "typeTag", + "description", + "for" + }) + public static class LabelTag { + public String tag; + public List sameAs; + } + + /** + * A {@code pronunciation} object in OASIS DMLex v1.0 JSON format. + */ + @JsonIgnoreProperties({ + "soundFile", + "labels" + }) + public static class Pronunciation { + public List transcriptions; + } + + /** + * A {@code transcription} object in OASIS DMLex v1.0 JSON format. + */ + public static class Transcription { + public String text; + public String scheme; + } + + public String id; + public String headword; + public List partsOfSpeech; + public List labels; + public List pronunciations; + public List inflectedForms; +} diff --git a/tools/dictionary-parser/src/main/java/org/unicode/wikidata/DMLexInflectedForm.java b/tools/dictionary-parser/src/main/java/org/unicode/wikidata/DMLexInflectedForm.java new file mode 100644 index 000000000..04b1cf206 --- /dev/null +++ b/tools/dictionary-parser/src/main/java/org/unicode/wikidata/DMLexInflectedForm.java @@ -0,0 +1,16 @@ +/* + * Copyright 2025-2026 Unicode Incorporated and others. All rights reserved. + */ +package org.unicode.wikidata; + +import java.util.List; + +/** + * An {@code inflectedForm} of a lexical entry in OASIS DMLex v1.0 JSON format (spec §5.2, {@code dmlex.schema.json}). + */ +public class DMLexInflectedForm { + public String tag; + public String text; + public List labels; + public List pronunciations; +} diff --git a/tools/dictionary-parser/src/main/java/org/unicode/wikidata/DMLexParser.java b/tools/dictionary-parser/src/main/java/org/unicode/wikidata/DMLexParser.java new file mode 100644 index 000000000..ec72a5fec --- /dev/null +++ b/tools/dictionary-parser/src/main/java/org/unicode/wikidata/DMLexParser.java @@ -0,0 +1,407 @@ +/* + * Copyright 2025-2026 Unicode Incorporated and others. All rights reserved. + */ +package org.unicode.wikidata; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.exc.MismatchedInputException; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeSet; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import javax.annotation.Nullable; + +import static org.unicode.wikidata.Grammar.Gender; +import static org.unicode.wikidata.Grammar.Ignorable; +import static org.unicode.wikidata.Grammar.PartOfSpeech; + +/** + * Parses OASIS DMLex v1.0 JSON lexicographic resources (spec §5.2, {@code dmlex.schema.json}), + * resolves tags via the Controlled Values module ({@code sameAs} Wikidata entity URIs), and feeds + * normalized {@link ParseWikidata.Lemma} entries into {@link ParseWikidata}. + */ +final class DMLexParser { + /** + * DMLex Controlled Values map tags to external inventories using full URIs in {@code sameAs} + * (e.g. {@code "http://www.wikidata.org/entity/Q1084"}), whereas {@link Grammar#TYPEMAP} is + * keyed by bare Wikidata Q-IDs (e.g. {@code "Q1084"}). This pattern extracts the trailing + * {@code Q} entity ID from a Wikidata entity URI. + */ + private static final Pattern WIKIDATA_ENTITY_PATTERN = Pattern.compile("entity/(Q\\d+)$"); + + private final ParseWikidata parseWikidata; + + DMLexParser(ParseWikidata parseWikidata) { + this.parseWikidata = parseWikidata; + } + + void parseDMLexStream( + ObjectMapper objectMapper, + JsonParser parser, + String sourceFilename) throws IOException { + DMLexEntry.Lexicon lexicon; + try { + lexicon = objectMapper + .readerFor(DMLexEntry.Lexicon.class) + .with(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES) + .readValue(parser); + } catch (MismatchedInputException e) { + throw MismatchedInputException.from( + parser, + e.getTargetType(), + sourceFilename + ": " + e.getOriginalMessage()); + } + if (lexicon == null || lexicon.langCode == null || lexicon.langCode.isBlank() || lexicon.entries == null) { + throw MismatchedInputException.from( + parser, + DMLexEntry.Lexicon.class, + "Invalid DMLex file (missing required 'langCode' or 'entries'): " + sourceFilename); + } + Map> posTagSameAsMap = new HashMap<>(); + if (lexicon.partOfSpeechTags != null) { + for (DMLexEntry.PartOfSpeechTag posTag : lexicon.partOfSpeechTags) { + if (posTag == null || posTag.tag == null || posTag.tag.isBlank()) { + throw new IllegalArgumentException(sourceFilename + ": Missing tag in partOfSpeechTags"); + } + if (posTagSameAsMap.containsKey(posTag.tag)) { + throw new IllegalArgumentException( + sourceFilename + ": Duplicate partOfSpeechTag '" + posTag.tag + "'"); + } + posTagSameAsMap.put(posTag.tag, posTag.sameAs); + } + } + Map> formTagSameAsMap = new HashMap<>(); + if (lexicon.inflectedFormTags != null) { + for (DMLexEntry.InflectedFormTag formTag : lexicon.inflectedFormTags) { + if (formTag == null || formTag.tag == null || formTag.tag.isBlank()) { + throw new IllegalArgumentException(sourceFilename + ": Missing tag in inflectedFormTags"); + } + if (formTagSameAsMap.containsKey(formTag.tag)) { + throw new IllegalArgumentException( + sourceFilename + ": Duplicate inflectedFormTag '" + formTag.tag + "'"); + } + formTagSameAsMap.put(formTag.tag, formTag.sameAs); + } + } + Map> labelTagSameAsMap = new HashMap<>(); + if (lexicon.labelTags != null) { + for (DMLexEntry.LabelTag labelTag : lexicon.labelTags) { + if (labelTag == null || labelTag.tag == null || labelTag.tag.isBlank()) { + throw new IllegalArgumentException(sourceFilename + ": Missing tag in labelTags"); + } + if (labelTagSameAsMap.containsKey(labelTag.tag)) { + throw new IllegalArgumentException( + sourceFilename + ": Duplicate labelTag '" + labelTag.tag + "'"); + } + labelTagSameAsMap.put(labelTag.tag, labelTag.sameAs); + } + } + Map>> resolvedPosTagCache = new HashMap<>(); + Map>> resolvedFormTagCache = new HashMap<>(); + Map>> resolvedLabelTagCache = new HashMap<>(); + ParseWikidata.Lemma lemma = new ParseWikidata.Lemma(); + for (DMLexEntry entry : lexicon.entries) { + analyzeDMLexEntry( + lemma, + entry, + lexicon.langCode, + sourceFilename, + posTagSameAsMap, + formTagSameAsMap, + labelTagSameAsMap, + resolvedPosTagCache, + resolvedFormTagCache, + resolvedLabelTagCache); + } + for (var posEntry : posTagSameAsMap.entrySet()) { + resolvedPosTagCache.computeIfAbsent( + posEntry.getKey(), + k -> resolveSameAsGrammemes( + posEntry.getValue(), sourceFilename, "", k, + "partOfSpeechTag", k)); + } + for (var formEntry : formTagSameAsMap.entrySet()) { + resolvedFormTagCache.computeIfAbsent( + formEntry.getKey(), + k -> resolveSameAsGrammemes( + formEntry.getValue(), sourceFilename, "", k, + "inflectedFormTag", k)); + } + for (var labelEntry : labelTagSameAsMap.entrySet()) { + resolvedLabelTagCache.computeIfAbsent( + labelEntry.getKey(), + k -> resolveSameAsGrammemes( + labelEntry.getValue(), sourceFilename, "", k, + "labelTag", k)); + } + } + + private void analyzeDMLexEntry( + ParseWikidata.Lemma lemma, + DMLexEntry entry, + String lexiconLangCode, + String sourceFilename, + Map> posTagSameAsMap, + Map> formTagSameAsMap, + Map> labelTagSameAsMap, + Map>> resolvedPosTagCache, + Map>> resolvedFormTagCache, + Map>> resolvedLabelTagCache) { + if (entry == null) { + throw new IllegalArgumentException(sourceFilename + ": Null DMLex entry"); + } + String entryId = (entry.id != null && !entry.id.isBlank()) ? entry.id : ""; + if (entry.headword == null || entry.headword.isBlank()) { + throw new IllegalArgumentException( + sourceFilename + " (" + entryId + "): Missing headword for DMLex entry"); + } + if (!LexemesJsonDeserializer.isContained(lexiconLangCode)) { + return; + } + if (parseWikidata.omitLemmas.contains(entryId)) { + return; + } + lemma.reset(); + parseWikidata.documentState.lemmaCount++; + lemma.value = entry.headword; + + int qVariantIdx = lexiconLangCode.indexOf(ParseWikidata.VARIANT_SEPARATOR); + if (qVariantIdx >= 0) { + var additionalCategory = lexiconLangCode.substring(qVariantIdx + ParseWikidata.VARIANT_SEPARATOR.length()); + var variant = Grammar.getMappedGrammemes(additionalCategory); + if (variant == null) { + throw new IllegalArgumentException( + sourceFilename + " (" + entryId + " / " + lemma.value + "): " + additionalCategory + + " is not a known grammeme for the language variant"); + } + if (variant.contains(Ignorable.IGNORABLE_INFLECTION)) { + return; + } + lemma.grammemes.addAll(variant); + } + + int resolvedPosCount = 0; + if (entry.partsOfSpeech != null) { + for (String posTag : entry.partsOfSpeech) { + if (posTag == null || !posTagSameAsMap.containsKey(posTag)) { + throw new IllegalArgumentException( + sourceFilename + " (" + entryId + " / " + lemma.value + "): Undefined partOfSpeech tag '" + + posTag + "'"); + } + Set> mappedPos = resolvedPosTagCache.computeIfAbsent( + posTag, + k -> resolveSameAsGrammemes( + posTagSameAsMap.get(k), sourceFilename, entryId, lemma.value, "partOfSpeechTag", k)); + for (Enum g : mappedPos) { + if (g instanceof PartOfSpeech) { + resolvedPosCount++; + } + } + lemma.grammemes.addAll(mappedPos); + } + } + if (parseWikidata.rareLemmas.contains(entryId)) { + lemma.grammemes.add(Grammar.Usage.RARE); + } + if (entry.labels != null) { + for (String labelTag : entry.labels) { + if (labelTag == null || !labelTagSameAsMap.containsKey(labelTag)) { + throw new IllegalArgumentException( + sourceFilename + " (" + entryId + " / " + lemma.value + "): Undefined label tag '" + + labelTag + "'"); + } + Set> mappedLabel = resolvedLabelTagCache.computeIfAbsent( + labelTag, + k -> resolveSameAsGrammemes( + labelTagSameAsMap.get(k), sourceFilename, entryId, lemma.value, "labelTag", k)); + for (Enum g : mappedLabel) { + if (g instanceof PartOfSpeech) { + resolvedPosCount++; + } + } + lemma.grammemes.addAll(mappedLabel); + } + } + if (entry.pronunciations != null && !entry.pronunciations.isEmpty()) { + List ipaTexts = extractIpaTranscriptions( + entry.pronunciations, sourceFilename, entryId, lemma.value); + if (parseWikidata.parserOptions.addSound && !ipaTexts.isEmpty()) { + parseWikidata.addSound("P898", ipaTexts, lemma.grammemes, entryId, lemma.value); + } + } + + if (lemma.grammemes.contains(Ignorable.IGNORABLE_LEMMA) + || lemma.grammemes.contains(Ignorable.IGNORABLE_INFLECTION)) { + parseWikidata.documentState.unusableLemmaCount++; + return; + } + lemma.grammemes.remove(Ignorable.IGNORABLE_PROPERTY); + + int posCount = Math.max(resolvedPosCount, parseWikidata.countGrammemeType(lemma.grammemes, PartOfSpeech.class)); + if (parseWikidata.countGrammemeType(lemma.grammemes, PartOfSpeech.class) != 1 || resolvedPosCount != 1) { + throw new IllegalArgumentException( + sourceFilename + " (" + entryId + " / " + lemma.value + + "): Expected exactly 1 part of speech, found " + posCount); + } + + TreeSet> genderlessLemmaGrammemes = null; + if (parseWikidata.countGrammemeType(lemma.grammemes, Gender.class) > 0) { + genderlessLemmaGrammemes = new TreeSet<>(lemma.grammemes); + parseWikidata.removeGrammemeType(genderlessLemmaGrammemes, Gender.class); + } + + List forms = entry.inflectedForms; + boolean synthesizedDefaultForm = false; + if (forms == null || forms.isEmpty()) { + DMLexInflectedForm defaultForm = new DMLexInflectedForm(); + defaultForm.text = entry.headword; + forms = List.of(defaultForm); + synthesizedDefaultForm = true; + } else { + boolean hasHeadwordForm = false; + for (DMLexInflectedForm form : forms) { + if (form != null && entry.headword.equals(form.text)) { + hasHeadwordForm = true; + break; + } + } + if (!hasHeadwordForm) { + throw new IllegalArgumentException( + sourceFilename + " (" + entryId + " / " + lemma.value + + "): inflectedForms must include the headword '" + entry.headword + "'"); + } + } + + for (DMLexInflectedForm form : forms) { + if (form == null || form.text == null || form.text.isBlank()) { + throw new IllegalArgumentException( + sourceFilename + " (" + entryId + " / " + lemma.value + "): Missing inflectedForm text"); + } + Inflection currentInflection = new Inflection(form.text); + if (!synthesizedDefaultForm || form.tag != null) { + if (form.tag == null || form.tag.isBlank()) { + throw new IllegalArgumentException( + sourceFilename + " (" + entryId + " / " + lemma.value + + "): Missing inflectedForm tag for form '" + form.text + "'"); + } + if (!formTagSameAsMap.containsKey(form.tag)) { + throw new IllegalArgumentException( + sourceFilename + " (" + entryId + " / " + lemma.value + + "): Undefined inflectedForm tag '" + form.tag + "'"); + } + Set> mappedFormTag = resolvedFormTagCache.computeIfAbsent( + form.tag, + k -> resolveSameAsGrammemes( + formTagSameAsMap.get(k), sourceFilename, entryId, lemma.value, "inflectedFormTag", k)); + currentInflection.grammemeSet.addAll(mappedFormTag); + } + if (form.labels != null) { + for (String labelTag : form.labels) { + if (labelTag == null || !labelTagSameAsMap.containsKey(labelTag)) { + throw new IllegalArgumentException( + sourceFilename + " (" + entryId + " / " + lemma.value + + "): Undefined label tag '" + labelTag + "'"); + } + Set> mappedLabel = resolvedLabelTagCache.computeIfAbsent( + labelTag, + k -> resolveSameAsGrammemes( + labelTagSameAsMap.get(k), sourceFilename, entryId, lemma.value, "labelTag", k)); + currentInflection.grammemeSet.addAll(mappedLabel); + } + } + if (form.pronunciations != null && !form.pronunciations.isEmpty()) { + List ipaTexts = extractIpaTranscriptions( + form.pronunciations, sourceFilename, entryId, lemma.value); + if (parseWikidata.parserOptions.addSound && !ipaTexts.isEmpty()) { + parseWikidata.addSound("P898", ipaTexts, currentInflection.grammemeSet, entryId, lemma.value); + } + } + if (!parseWikidata.finalizeAndAddInflection(lemma, genderlessLemmaGrammemes, currentInflection)) { + return; + } + } + + if (!parseWikidata.finalizeLemmaAfterForms(lemma)) { + return; + } + parseWikidata.analyzeLemma(lemma); + } + + private static Set> resolveSameAsGrammemes( + List sameAsUris, + String sourceFilename, + String entryId, + String headword, + String tagType, + String tag) { + if (sameAsUris == null || sameAsUris.isEmpty()) { + throw new IllegalArgumentException( + sourceFilename + " (" + entryId + " / " + headword + "): Missing or empty sameAs mapping for " + + tagType + " '" + tag + "'"); + } + Set> resolved = new HashSet<>(); + for (String uri : sameAsUris) { + if (uri == null || uri.isBlank()) { + throw new IllegalArgumentException( + sourceFilename + " (" + entryId + " / " + headword + "): Empty sameAs URI in " + tagType + " '" + tag + "'"); + } + Matcher matcher = WIKIDATA_ENTITY_PATTERN.matcher(uri.trim()); + Set> mapped = matcher.find() ? Grammar.getMappedGrammemes(matcher.group(1)) : null; + if (mapped == null) { + throw new IllegalArgumentException( + sourceFilename + " (" + entryId + " / " + headword + "): Unrecognized sameAs URI '" + uri + + "' for " + tagType + " '" + tag + "'"); + } + resolved.addAll(mapped); + } + return resolved; + } + + private static boolean isIpaScheme(@Nullable String scheme) { + if (scheme == null || scheme.isBlank()) { + return true; + } + for (String subtag : scheme.trim().split("[-_]")) { + if ("fonipa".equalsIgnoreCase(subtag)) { + return true; + } + } + return false; + } + + private static List extractIpaTranscriptions( + List pronunciations, + String sourceFilename, + String entryId, + String headword) { + List ipaTexts = new ArrayList<>(); + for (DMLexEntry.Pronunciation pron : pronunciations) { + if (pron == null) { + throw new IllegalArgumentException( + sourceFilename + " (" + entryId + " / " + headword + "): Null pronunciation object"); + } + if (pron.transcriptions == null) { + continue; + } + for (DMLexEntry.Transcription transcription : pron.transcriptions) { + if (transcription == null || transcription.text == null || transcription.text.isBlank()) { + throw new IllegalArgumentException( + sourceFilename + " (" + entryId + " / " + headword + "): Missing transcription text"); + } + if (isIpaScheme(transcription.scheme)) { + ipaTexts.add(transcription.text); + } + } + } + return ipaTexts; + } +} diff --git a/tools/dictionary-parser/src/main/java/org/unicode/wikidata/DocumentState.java b/tools/dictionary-parser/src/main/java/org/unicode/wikidata/DocumentState.java index 3078fe619..804f685ae 100644 --- a/tools/dictionary-parser/src/main/java/org/unicode/wikidata/DocumentState.java +++ b/tools/dictionary-parser/src/main/java/org/unicode/wikidata/DocumentState.java @@ -95,8 +95,8 @@ void printDocument(ParserOptions parserOptions, long startTime) throws FileNotFo int dictionarySize = dictionary.size(); StringBuilder source = new StringBuilder(); Pattern anythingSlash = Pattern.compile(".*/"); - for (String sourceFilename : parserOptions.sourceFilenames) { - source.append(anythingSlash.matcher(sourceFilename).replaceAll("")).append(" "); + for (ParserOptions.SourceInput sourceInput : parserOptions.sourceInputs) { + source.append(anythingSlash.matcher(sourceInput.filename()).replaceAll("")).append(" "); } lexicalDictionaryStream.println("=============================================="); lexicalDictionaryStream.printf("%30s %7s%n", "Source:", source); diff --git a/tools/dictionary-parser/src/main/java/org/unicode/wikidata/Grammar.java b/tools/dictionary-parser/src/main/java/org/unicode/wikidata/Grammar.java index a8ee68787..f3ca6a62d 100644 --- a/tools/dictionary-parser/src/main/java/org/unicode/wikidata/Grammar.java +++ b/tools/dictionary-parser/src/main/java/org/unicode/wikidata/Grammar.java @@ -1178,6 +1178,9 @@ public String toString() { } static Set> getMappedGrammemes(String grammeme) { + if (grammeme == null) { + throw new IllegalArgumentException("Grammeme cannot be null"); + } return TYPEMAP.get(grammeme); } } diff --git a/tools/dictionary-parser/src/main/java/org/unicode/wikidata/ParseWikidata.java b/tools/dictionary-parser/src/main/java/org/unicode/wikidata/ParseWikidata.java index 33c038bab..48d636853 100644 --- a/tools/dictionary-parser/src/main/java/org/unicode/wikidata/ParseWikidata.java +++ b/tools/dictionary-parser/src/main/java/org/unicode/wikidata/ParseWikidata.java @@ -18,6 +18,7 @@ import com.fasterxml.jackson.core.JsonToken; import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.exc.MismatchedInputException; import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream; import org.apache.commons.lang3.StringUtils; @@ -71,17 +72,18 @@ void reset() { inflections.clear(); } - private Lemma() { + Lemma() { } } - private final ParserOptions parserOptions; - private final DocumentState documentState = new DocumentState(); - private final TreeSet rareLemmas = new TreeSet<>(); - private final TreeSet omitLemmas = new TreeSet<>(); + final ParserOptions parserOptions; + final DocumentState documentState = new DocumentState(); + final TreeSet rareLemmas = new TreeSet<>(); + final TreeSet omitLemmas = new TreeSet<>(); private final Map> mergeMap = new HashMap<>(); private final TreeSet deferredLexemes = new TreeSet<>(); private final Map deferredLemmaMap = new HashMap<>(); + private final Set seenDeferredLexemes = new HashSet<>(); ParseWikidata(ParserOptions parserOptions) { this.parserOptions = parserOptions; @@ -126,6 +128,70 @@ private Lemma() { static final String VARIANT_SEPARATOR = "-x-"; + /** + * Finalizes a single surface-form inflection with lemma grammeme propagation, ignorable filtering, + * rarity extraction, and {@code --expand-grammemes} expansion. Shared by both Wikidata and DMLex paths. + * + * @return {@code false} if the entire lemma must be aborted due to {@code IGNORABLE_LEMMA}; {@code true} otherwise. + */ + boolean finalizeAndAddInflection( + Lemma lemma, + @Nullable TreeSet> genderlessLemmaGrammemes, + Inflection currentInflection) { + if (genderlessLemmaGrammemes == null || countGrammemeType(currentInflection.grammemeSet, Gender.class) == 0) { + currentInflection.grammemeSet.addAll(lemma.grammemes); + } else { + currentInflection.grammemeSet.addAll(genderlessLemmaGrammemes); + } + if (currentInflection.grammemeSet.contains(Ignorable.IGNORABLE_LEMMA)) { + documentState.unusableLemmaCount++; + return false; + } + if (currentInflection.grammemeSet.contains(Ignorable.IGNORABLE_INFLECTION)) { + documentState.unusableSurfaceFormCount++; + return true; + } + currentInflection.rareUsage = currentInflection.grammemeSet.contains(Grammar.Usage.RARE); + if (currentInflection.rareUsage) { + currentInflection.grammemeSet.remove(Grammar.Usage.RARE); + } + currentInflection.grammemeSet.remove(Ignorable.IGNORABLE_PROPERTY); + var grammemeExpansion = parserOptions.expandGramemes != null + ? parserOptions.expandGramemes.get(currentInflection.grammemeSet) + : null; + if (grammemeExpansion == null) { + lemma.inflections.add(currentInflection); + } else { + for (var grammemeSet : grammemeExpansion) { + var expandedInflection = new Inflection(currentInflection.inflection, + currentInflection.rareUsage); + expandedInflection.grammemeSet.addAll(currentInflection.grammemeSet); + expandedInflection.grammemeSet.addAll(grammemeSet); + lemma.inflections.add(expandedInflection); + } + } + return true; + } + + /** + * Performs post-form lemma cleanup shared by Wikidata and DMLex: counts incoming surface forms, + * extracts lemma-level {@link Grammar.Usage#RARE}, rejects lemmas with no usable surface forms, + * and strips {@link Gender} from {@code lemma.grammemes}. + */ + boolean finalizeLemmaAfterForms(Lemma lemma) { + documentState.incomingSurfaceForm += lemma.inflections.size(); + lemma.isRare = lemma.grammemes.contains(Grammar.Usage.RARE); + if (lemma.isRare) { + lemma.grammemes.remove(Grammar.Usage.RARE); + } + if (lemma.inflections.isEmpty()) { + documentState.unusableLemmaCount++; + return false; + } + removeGrammemeType(lemma.grammemes, Gender.class); + return true; + } + private void analyzeLexeme(int lineNumber, Lexeme lexeme) { if (omitLemmas.contains(lexeme.id)) { // We really don't want this junk. @@ -133,6 +199,7 @@ private void analyzeLexeme(int lineNumber, Lexeme lexeme) { } Lemma lemma = new Lemma(); Set> partOfSpeechSet = null; + boolean addedToDeferredInThisLexeme = false; for (var lemmaEntry : lexeme.lemmas.entrySet()) { var currentLemmaLanguage = lemmaEntry.getKey(); lemma.reset(); @@ -204,80 +271,61 @@ private void analyzeLexeme(int lineNumber, Lexeme lexeme) { } } convertGrammemes(form, currentInflection, lexeme.id, lemma.value); - if (genderlessLemmaGrammemes == null || countGrammemeType(currentInflection.grammemeSet, Gender.class) == 0) { - currentInflection.grammemeSet.addAll(lemma.grammemes); - } - else { - currentInflection.grammemeSet.addAll(genderlessLemmaGrammemes); - } - if (currentInflection.grammemeSet.contains(Ignorable.IGNORABLE_LEMMA)) { - documentState.unusableLemmaCount++; + if (!finalizeAndAddInflection(lemma, genderlessLemmaGrammemes, currentInflection)) { return; } - if (currentInflection.grammemeSet.contains(Ignorable.IGNORABLE_INFLECTION)) { - documentState.unusableSurfaceFormCount++; - continue; - } - currentInflection.rareUsage = currentInflection.grammemeSet.contains(Grammar.Usage.RARE); - if (currentInflection.rareUsage) { - currentInflection.grammemeSet.remove(Grammar.Usage.RARE); - } - currentInflection.grammemeSet.remove(Ignorable.IGNORABLE_PROPERTY); - var grammemeExpansion = parserOptions.expandGramemes != null - ? parserOptions.expandGramemes.get(currentInflection.grammemeSet) - : null; - if (grammemeExpansion == null) { - lemma.inflections.add(currentInflection); - } else { - for (var grammemeSet : grammemeExpansion) { - var expandedInflection = new Inflection(currentInflection.inflection, - currentInflection.rareUsage); - expandedInflection.grammemeSet.addAll(currentInflection.grammemeSet); - expandedInflection.grammemeSet.addAll(grammemeSet); - lemma.inflections.add(expandedInflection); - } - } } - documentState.incomingSurfaceForm += lemma.inflections.size(); - lemma.isRare = lemma.grammemes.contains(Grammar.Usage.RARE); - if (lemma.isRare) { - lemma.grammemes.remove(Grammar.Usage.RARE); - } - if (lemma.inflections.isEmpty()) { - documentState.unusableLemmaCount++; + if (!finalizeLemmaAfterForms(lemma)) { return; } - removeGrammemeType(lemma.grammemes, Gender.class); if (deferredLexemes.contains(lexeme.id)) { - if (deferredLemmaMap.put(lexeme.id, lemma) != null) { + seenDeferredLexemes.add(lexeme.id); + if (addedToDeferredInThisLexeme) { + Lemma existing = deferredLemmaMap.get(lexeme.id); + existing.inflections.addAll(lemma.inflections); + existing.isRare |= lemma.isRare; + } else if (deferredLemmaMap.put(lexeme.id, lemma) != null) { throw new IllegalStateException("Duplicate lexeme " + lexeme.id); + } else { + addedToDeferredInThisLexeme = true; + lemma = new Lemma(); } - } - else { + } else { analyzeLemma(lemma); } } } - // Method to process and merge lexemes private void processAndMergeLexemes() { for (Map.Entry> entry : mergeMap.entrySet()) { - Lemma lemma = deferredLemmaMap.computeIfAbsent(entry.getKey(), key -> { - throw new IllegalArgumentException(key + ": id not found"); - }); + Lemma lemma = deferredLemmaMap.remove(entry.getKey()); for (var value : entry.getValue()) { - Lemma lemmaToMerge = deferredLemmaMap.computeIfAbsent(value, key -> { - throw new IllegalArgumentException(key + ": id not found"); - }); - lemma.inflections.addAll(lemmaToMerge.inflections); - lemma.isRare |= lemmaToMerge.isRare; - // The lemma grammemes should already have the relevant POS data. + Lemma lemmaToMerge = deferredLemmaMap.remove(value); + if (lemmaToMerge != null) { + if (lemma == null) { + lemma = lemmaToMerge; + } else { + lemma.inflections.addAll(lemmaToMerge.inflections); + lemma.isRare |= lemmaToMerge.isRare; + } + } + } + if (lemma != null) { + analyzeLemma(lemma); + } + } + deferredLemmaMap.clear(); + } + + private void validateDeferredLexemes() { + for (String expectedId : deferredLexemes) { + if (!seenDeferredLexemes.contains(expectedId)) { + throw new IllegalArgumentException(expectedId + ": id not found"); } - analyzeLemma(lemma); } } - private void removeGrammemeType(TreeSet> grammemes, Class grammemeType) { + void removeGrammemeType(TreeSet> grammemes, Class grammemeType) { var iter = grammemes.iterator(); while (iter.hasNext()) { var grammeme = iter.next(); @@ -287,7 +335,7 @@ private void removeGrammemeType(TreeSet> grammemes, Class grammemeTyp } } - private int countGrammemeType(TreeSet> grammemes, Class grammemeType) { + int countGrammemeType(TreeSet> grammemes, Class grammemeType) { int count = 0; var iter = grammemes.iterator(); while (iter.hasNext()) { @@ -357,7 +405,7 @@ private boolean extractImportantProperties(Map> claims, Tre return conflicts; } - private void addSound(String property, List claims, TreeSet> grammemeSet, String id, String lemma) { + void addSound(String property, List claims, TreeSet> grammemeSet, String id, String lemma) { var dataForClaim = parserOptions.claimsToSound.get(property); if (dataForClaim != null && !dataForClaim.isEmpty()) { for (var soundMatcher : dataForClaim.entrySet()) { @@ -598,7 +646,7 @@ private void mergeAdditionalGrammemes() { } } - private void analyzeLemma(Lemma lemma) { + void analyzeLemma(Lemma lemma) { analyzeInflections(lemma, lemma.inflections); if (parserOptions.includeLemmasWithoutWords) { @@ -611,6 +659,40 @@ public void writeInflectionPatterns(long startTime) throws FileNotFoundException documentState.printDocument(parserOptions, startTime); } + private boolean parseWikidataStream( + ObjectMapper objectMapper, + JsonParser parser, + String sourceFilename) throws IOException { + JsonToken currToken = parser.nextToken(); + if (currToken == null) { + return false; + } + if (currToken != JsonToken.START_ARRAY) { + throw MismatchedInputException.from( + parser, + Lexeme[].class, + sourceFilename + ": Expected JSON array of Wikidata lexemes"); + } + while ((currToken = parser.nextToken()) != JsonToken.START_OBJECT && currToken != JsonToken.END_ARRAY && currToken != null) { + // Find the first object in the array. + } + if (currToken == JsonToken.START_OBJECT) { + do { + Lexeme lexeme = objectMapper.readValue(parser, Lexeme.class); + try { + analyzeLexeme(parser.currentLocation().getLineNr(), lexeme); + } catch (IllegalArgumentException e) { + documentState.unusableLemmaCount++; + System.err.println( + "Line " + parser.currentLocation().getLineNr() + ": " + e.getMessage()); + } + } while ((currToken = parser.nextToken()) != JsonToken.END_ARRAY && currToken != null); + processAndMergeLexemes(); + return true; + } + return false; + } + public static void main(String[] args) throws Exception { ParserOptions parserOptions = new ParserOptions(args); long startTime = System.currentTimeMillis(); @@ -618,6 +700,7 @@ public static void main(String[] args) throws Exception { .configure(JsonParser.Feature.ALLOW_UNQUOTED_FIELD_NAMES, true) .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); var lexParser = new ParseWikidata(parserOptions); + var dmLexParser = new DMLexParser(lexParser); LexemesJsonDeserializer.setLanguage(parserOptions.locales); LexemesJsonDeserializer.setExcludedLanguages(parserOptions.excludedLanguages); @@ -625,7 +708,9 @@ public static void main(String[] args) throws Exception { // being interpreted as malformed UTF-8. // We use a large buffer because we're reading a large file, and we're // frequently reading file data. - for (String sourceFilename : parserOptions.sourceFilenames) { + boolean hasNonEmptyWikidataSource = false; + for (ParserOptions.SourceInput sourceInput : parserOptions.sourceInputs) { + String sourceFilename = sourceInput.filename(); try (InputStream fileInputStream = new FileInputStream(sourceFilename)) { InputStream inputStream = fileInputStream; if (sourceFilename.endsWith(".bz2")) { @@ -633,26 +718,17 @@ public static void main(String[] args) throws Exception { inputStream = new BZip2CompressorInputStream(new BufferedInputStream(inputStream, 32768)); } try (JsonParser parser = objectMapper.createParser(inputStream)) { - JsonToken currToken; - while ((currToken = parser.nextToken()) != JsonToken.START_OBJECT && currToken != null) { - // Find the first object in the array. - } - if (currToken == JsonToken.START_OBJECT) { - do { - Lexeme lexeme = objectMapper.readValue(parser, Lexeme.class); - try { - lexParser.analyzeLexeme(parser.currentLocation().getLineNr(), lexeme); - } catch (IllegalArgumentException e) { - lexParser.documentState.unusableLemmaCount++; - System.err.println( - "Line " + parser.currentLocation().getLineNr() + ": " + e.getMessage()); - } - } while (parser.nextToken() != JsonToken.END_ARRAY); - lexParser.processAndMergeLexemes(); + if (sourceInput.format() == ParserOptions.SourceFormat.DMLEX) { + dmLexParser.parseDMLexStream(objectMapper, parser, sourceFilename); + } else { + hasNonEmptyWikidataSource |= lexParser.parseWikidataStream(objectMapper, parser, sourceFilename); } } } } + if (hasNonEmptyWikidataSource) { + lexParser.validateDeferredLexemes(); + } if (!lexParser.parserOptions.additionalGrammemesDict.isEmpty()) { lexParser.mergeAdditionalGrammemes(); diff --git a/tools/dictionary-parser/src/main/java/org/unicode/wikidata/ParserOptions.java b/tools/dictionary-parser/src/main/java/org/unicode/wikidata/ParserOptions.java index 98768b47b..5f9c52032 100644 --- a/tools/dictionary-parser/src/main/java/org/unicode/wikidata/ParserOptions.java +++ b/tools/dictionary-parser/src/main/java/org/unicode/wikidata/ParserOptions.java @@ -31,7 +31,16 @@ * The options to extract the data from the data source. */ final class ParserOptions { + enum SourceFormat { + WIKIDATA, + DMLEX + } + + record SourceInput(String filename, SourceFormat format) {} + private static final char COLON_SEPARATOR = ':'; + static final String WIKIDATA_FILE = "--wikidata"; + static final String DMLEX_FILE = "--dmlex"; static final String INFLECTIONS_FILE = "--inflections"; static final String DICTIONARY_FILE = "--dictionary"; static final String MAP_GRAMMEME = "--map-grammeme"; @@ -55,7 +64,7 @@ final class ParserOptions { TreeMap> additionalGrammemesDict; TreeMap> claimsToSound; - ArrayList sourceFilenames; + ArrayList sourceInputs; String inflectionalFilename = ParserDefaults.DEFAULT_INFLECTION_FILE_NAME; String lexicalDictionaryFilename = ParserDefaults.DEFAULT_DICTIONARY_FILE_NAME; ArrayList locales = new ArrayList<>(List.of(Locale.ENGLISH.getLanguage())); @@ -63,8 +72,10 @@ final class ParserOptions { List optionsUsedToInvoke = new ArrayList<>(); private static void printUsage() { - System.err.println("Usage: ParseLexicon [OPTIONS] [ ...]"); + System.err.println("Usage: ParseLexicon [OPTIONS] (--wikidata [ ...] | --dmlex [ ...]) ..."); System.err.println("\nOPTIONS"); + System.err.println(WIKIDATA_FILE + " [ ...]\tOne or more Wikidata lexeme JSON/bz2 files (space- or comma-separated)"); + System.err.println(DMLEX_FILE + " [ ...]\tOne or more OASIS DMLex JSON files (space- or comma-separated)"); System.err.println(INFLECTIONS_FILE + " \tthe file for the inflectional patterns to be generated, default: inflectional.xml"); System.err.println(DICTIONARY_FILE + " \tthe file for the lexical dictionary to be generated, default: dictionary.lst"); System.err.println(ADD_EXTRA_GRAMMEMES + " \tFile containing words with the extra grammemes to be added, provide path relative to tools/dictionary-parser/src/main/resources/org/unicode/wikidata/ (only to be used for a temporary grammeme addition)"); @@ -83,12 +94,20 @@ private static void printUsage() { ParserOptions(String[] args) throws IOException { posToBeInflected = EnumSet.of(Grammar.PartOfSpeech.NOUN); additionalGrammemesDict = new TreeMap<>(); - sourceFilenames = new ArrayList<>(); + sourceInputs = new ArrayList<>(); claimsToSound = new TreeMap<>(); for (int i = 0; i < args.length; i++) { String arg = args[i]; - if (ParserOptions.INFLECTIONS_FILE.equals(arg)) { + if (arg == null) { + printUsage(); + throw new IllegalArgumentException("Null argument at index " + i); + } + if (ParserOptions.WIKIDATA_FILE.equals(arg) || arg.startsWith(ParserOptions.WIKIDATA_FILE + "=")) { + i = consumeSourceFiles(args, i, ParserOptions.WIKIDATA_FILE, SourceFormat.WIKIDATA); + } else if (ParserOptions.DMLEX_FILE.equals(arg) || arg.startsWith(ParserOptions.DMLEX_FILE + "=")) { + i = consumeSourceFiles(args, i, ParserOptions.DMLEX_FILE, SourceFormat.DMLEX); + } else if (ParserOptions.INFLECTIONS_FILE.equals(arg)) { inflectionalFilename = args[++i]; } else if (ParserOptions.DICTIONARY_FILE.equals(arg)) { lexicalDictionaryFilename = args[++i]; @@ -141,7 +160,7 @@ private static void printUsage() { posToBeInflected.clear(); for (String pos : inflectionTypes.split(",")) { - posToBeInflected.add(Grammar.PartOfSpeech.valueOf(pos.toUpperCase().replace('-', '_'))); + posToBeInflected.add(Grammar.PartOfSpeech.valueOf(pos.toUpperCase(Locale.ROOT).replace('-', '_'))); } optionsUsedToInvoke.add(ParserOptions.INFLECTION_TYPES); @@ -202,15 +221,40 @@ private static void printUsage() { optionsUsedToInvoke.add(ParserOptions.ADD_SOUND); optionsUsedToInvoke.add(soundGrammemeTypes); } else { - sourceFilenames.add(arg); + printUsage(); + throw new IllegalArgumentException("Unknown option: " + arg); } } addSound = !claimsToSound.isEmpty(); - if (sourceFilenames.isEmpty()) { + if (sourceInputs.isEmpty()) { + printUsage(); + throw new IllegalArgumentException("At least one input file must be specified via --wikidata or --dmlex"); + } + } + + private int consumeSourceFiles(String[] args, int i, String flagPrefix, SourceFormat format) { + int startCount = sourceInputs.size(); + if (args[i].startsWith(flagPrefix + "=")) { + addSourceFiles(args[i].substring(flagPrefix.length() + 1), format); + } + while (i + 1 < args.length && args[i + 1] != null && !args[i + 1].startsWith("--")) { + addSourceFiles(args[++i], format); + } + if (sourceInputs.size() == startCount) { printUsage(); - throw new IllegalArgumentException(); + throw new IllegalArgumentException(flagPrefix + " requires at least one file argument"); + } + return i; + } + + private void addSourceFiles(String rawArg, SourceFormat format) { + for (String part : rawArg.split(",")) { + String trimmed = part.trim(); + if (!trimmed.isEmpty()) { + sourceInputs.add(new SourceInput(trimmed, format)); + } } } @@ -222,8 +266,7 @@ void setIgnoreProperty(String[] grammemes, Grammar.Ignorable ignorable) { } else { for (Map.Entry>> entry : TYPEMAP.entrySet()) { for (var grammemeEnum : entry.getValue()) { - String name = grammemeEnum.name(); - if (name.equalsIgnoreCase(grammeme)) { + if (grammemeEnum.toString().equalsIgnoreCase(grammeme)) { if (entry.getValue().size() == 1) { entry.setValue(ignorableSet); } else { diff --git a/tools/dictionary-parser/src/test/java/org/unicode/wikidata/ParseWikidataTest.java b/tools/dictionary-parser/src/test/java/org/unicode/wikidata/ParseWikidataTest.java index cf057c899..204e657a2 100644 --- a/tools/dictionary-parser/src/test/java/org/unicode/wikidata/ParseWikidataTest.java +++ b/tools/dictionary-parser/src/test/java/org/unicode/wikidata/ParseWikidataTest.java @@ -7,8 +7,18 @@ import java.lang.invoke.MethodHandles; import java.nio.charset.StandardCharsets; import java.nio.file.Files; +import java.nio.file.Path; import java.nio.file.Paths; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; import java.util.Objects; +import java.util.Set; +import java.util.TreeSet; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; import com.fasterxml.jackson.core.JsonParseException; import com.fasterxml.jackson.databind.exc.MismatchedInputException; import org.junit.jupiter.api.Assertions; @@ -18,13 +28,20 @@ class ParseWikidataTest { private static final String lexiconSource = Objects .requireNonNull(MethodHandles.lookup().lookupClass().getClassLoader().getResource("sourceLexicon.json")).getFile(); private static final String baseDir = lexiconSource.substring(0, lexiconSource.lastIndexOf("/")); - private static final String inflectionalFile = baseDir+"/LexiconInflectOut.xml"; - private static final String dictionaryFile = baseDir+"/LexiconInflectOut.lst"; - private static final String expectedOutputFile = baseDir+ "/lexiconCorrectOut.txt"; - private static final String fatalErrorFile = baseDir+ "/lexiconFatalError.json"; - private static final String missingGrammemeErrorFile = baseDir+ "/lexiconMissingGrammemeError.json"; - private static final String caseInsensitiveGrammemeFile = baseDir+ "/lexiconcaseInsenstiveGrammeme.json"; + private static final String inflectionalFile = baseDir + "/LexiconInflectOut.xml"; + private static final String dictionaryFile = baseDir + "/LexiconInflectOut.lst"; + private static final String expectedOutputFile = baseDir + "/lexiconCorrectOut.txt"; + private static final String fatalErrorFile = baseDir + "/lexiconFatalError.json"; + private static final String missingGrammemeErrorFile = baseDir + "/lexiconMissingGrammemeError.json"; + private static final String caseInsensitiveGrammemeFile = baseDir + "/lexiconcaseInsenstiveGrammeme.json"; + private static final String dmlexLexiconFile = baseDir + "/dmlexLexicon.json"; + private static final String dmlexOverlayFile = baseDir + "/dmlexOverlay.json"; + private static final String dmlexPatchFile = baseDir + "/dmlexPatch.json"; + private static final String dmlexExampleA28File = baseDir + "/dmlexExampleA28.json"; + private static final String dmlexInvalidSchemaFile = baseDir + "/dmlexInvalidSchema.json"; private static final String LINE_END = "=============================================="; + private static final Pattern XML_PATTERN_NAME = Pattern.compile(" parseDictionaryEntries(String output) { + return Arrays.stream(output.split("\n")) + .takeWhile(line -> !line.equals(LINE_END)) + .filter(line -> line.contains(":")) + .collect(Collectors.toMap( + line -> line.substring(0, line.indexOf(':')), + line -> line.substring(line.indexOf(':') + 1).trim())); + } + + private Set assertAllXmlPatternsReferencedInDictionary(String dictionaryOutput) throws Exception { + String xml = Files.readString(Paths.get(inflectionalFile), StandardCharsets.UTF_8); + Set xmlPatternIds = new TreeSet<>(); + Matcher xmlMatcher = XML_PATTERN_NAME.matcher(xml); + while (xmlMatcher.find()) { + Assertions.assertTrue(xmlPatternIds.add(xmlMatcher.group(1)), + "Duplicate pattern name in inflectional XML: " + xmlMatcher.group(1)); + } + Set dictPatternIds = new TreeSet<>(); + for (String line : dictionaryOutput.split("\n")) { + if (line.equals(LINE_END)) { + break; + } + Matcher dictMatcher = DICT_INFLECTION_ID.matcher(line); + while (dictMatcher.find()) { + dictPatternIds.add(dictMatcher.group(1)); + } + } + Assertions.assertEquals(dictPatternIds, xmlPatternIds, + "Every in inflectional.xml must be referenced as inflection=X in dictionary.lst"); + return xmlPatternIds; + } + @Test public void missingInputFile() { - Assertions.assertThrows(Exception.class,() -> ParseWikidata.main(new String[1])); + Assertions.assertThrows(Exception.class, () -> ParseWikidata.main(new String[1])); } @Test @@ -55,32 +104,1091 @@ public void enumTest() { Assertions.assertEquals("spelling", fieldValue); } + @Test + public void nullMappedGrammemeThrowsTest() { + Assertions.assertThrows(IllegalArgumentException.class, () -> Grammar.getMappedGrammemes(null)); + } + @Test public void fatalErrorHandlerTest() { // Malformed JSON syntax cannot be parsed by the Wikidata reader. - String[] args = {"--inflections", inflectionalFile, "--dictionary", dictionaryFile, fatalErrorFile}; - Assertions.assertThrows(JsonParseException.class,() -> ParseWikidata.main(args)); + String[] args = {"--inflections", inflectionalFile, "--dictionary", dictionaryFile, "--wikidata", fatalErrorFile}; + Assertions.assertThrows(JsonParseException.class, () -> ParseWikidata.main(args)); } @Test public void missingGrammemeTest() { // The grammatical features are not in the expected array form, so the reader rejects the input. - String[] args = {"--inflections", inflectionalFile, "--dictionary", dictionaryFile, missingGrammemeErrorFile}; - Assertions.assertThrows(MismatchedInputException.class,() -> ParseWikidata.main(args)); + String[] args = {"--inflections", inflectionalFile, "--dictionary", dictionaryFile, "--wikidata", missingGrammemeErrorFile}; + Assertions.assertThrows(MismatchedInputException.class, () -> ParseWikidata.main(args)); } @Test public void caseInsensitiveGrammemeTest() { - String[] args = {"--inflections", inflectionalFile, "--dictionary", dictionaryFile, caseInsensitiveGrammemeFile}; + String[] args = {"--inflections", inflectionalFile, "--dictionary", dictionaryFile, "--wikidata", caseInsensitiveGrammemeFile}; Assertions.assertDoesNotThrow(() -> ParseWikidata.main(args)); } @Test public void lexiconParserTest() throws Exception { String[] args = {"--inflection-types", "noun,adjective,proper-noun", "--add-sound", "vowel-start", - "--inflections", inflectionalFile, "--dictionary", dictionaryFile, lexiconSource}; + "--inflections", inflectionalFile, "--dictionary", dictionaryFile, "--wikidata", lexiconSource}; String actual = getParserOutput(args); String expected = Files.readString(Paths.get(expectedOutputFile), StandardCharsets.UTF_8); compareOutputs(actual, expected); } + + @Test + public void standaloneDMLexParserTest() throws Exception { + String[] args = { + "--inflection-types", "noun,proper-noun", + "--add-sound", "vowel-start", + "--inflections", inflectionalFile, + "--dictionary", dictionaryFile, + "--dmlex", dmlexLexiconFile + }; + String actual = getParserOutput(args); + Map entries = parseDictionaryEntries(actual); + + // cat and bat share pattern 1 (count=2, most frequent) + Assertions.assertEquals("singular masculine noun inflection=1", entries.get("cat")); + Assertions.assertEquals("plural masculine noun inflection=1", entries.get("cats")); + Assertions.assertEquals("singular masculine noun inflection=1", entries.get("bat")); + Assertions.assertEquals("plural masculine noun inflection=1", entries.get("bats")); + + // apple has IPA pronunciation matching vowel-start + Assertions.assertEquals("singular vowel-start noun inflection=2", entries.get("apple")); + Assertions.assertEquals("plural vowel-start noun inflection=2", entries.get("apples")); + + // London uses proper-noun + Assertions.assertEquals("singular proper-noun inflection=3", entries.get("London")); + + // wow has no inflectedForms array, synthesized default form + Assertions.assertEquals("interjection", entries.get("wow")); + + // Verify inflectional XML was generated + String xml = Files.readString(Paths.get(inflectionalFile), StandardCharsets.UTF_8); + Assertions.assertTrue(xml.contains("")); + Assertions.assertTrue(xml.contains("")); + } + + @Test + public void dmlexSpecExampleA28Test() throws Exception { + // With default --language en, the Irish ("ga") lexicon is skipped + String[] enArgs = { + "--inflection-types", "noun", + "--inflections", inflectionalFile, + "--dictionary", dictionaryFile, + "--dmlex", dmlexExampleA28File + }; + Assertions.assertTrue(parseDictionaryEntries(getParserOutput(enArgs)).isEmpty()); + + String[] args = { + "--language", "ga", + "--inflection-types", "noun", + "--inflections", inflectionalFile, + "--dictionary", dictionaryFile, + "--dmlex", dmlexExampleA28File + }; + String actual = getParserOutput(args); + Map entries = parseDictionaryEntries(actual); + + Assertions.assertEquals("singular masculine noun", entries.get("folúsghlantóir")); + Assertions.assertEquals("singular genitive masculine noun", entries.get("folúsghlantóra")); + Assertions.assertEquals("plural masculine noun", entries.get("folúsghlantóirí")); + } + + @Test + public void multiSourceWikidataAndDMLexMergeTest() throws Exception { + String[] args = { + "--inflection-types", "noun,adjective,proper-noun", + "--add-sound", "vowel-start", + "--inflections", inflectionalFile, + "--dictionary", dictionaryFile, + "--wikidata", lexiconSource, + "--dmlex", dmlexOverlayFile + }; + String actual = getParserOutput(args); + Map entries = parseDictionaryEntries(actual); + + // 1. New lemma "book" from DMLex is added + Assertions.assertTrue(entries.containsKey("book")); + Assertions.assertTrue(entries.get("book").startsWith("singular noun inflection=")); + Assertions.assertTrue(entries.containsKey("book's")); + Assertions.assertTrue(entries.containsKey("books")); + Assertions.assertTrue(entries.containsKey("books'")); + + // 2. "hey" merges Wikidata (interjection) and DMLex (singular noun) + String heyEntry = entries.get("hey"); + Assertions.assertNotNull(heyEntry); + Assertions.assertTrue(heyEntry.contains("noun"), "Expected noun in: " + heyEntry); + Assertions.assertTrue(heyEntry.contains("interjection"), "Expected interjection in: " + heyEntry); + Assertions.assertTrue(entries.containsKey("heys")); + + // 3. "ciel" and "cieux" merge Wikidata (masculine) and DMLex (feminine) + String cielEntry = entries.get("ciel"); + Assertions.assertNotNull(cielEntry); + Assertions.assertTrue(cielEntry.contains("masculine"), "Expected masculine in: " + cielEntry); + Assertions.assertTrue(cielEntry.contains("feminine"), "Expected feminine in: " + cielEntry); + String[] cielParts = cielEntry.split("inflection="); + Assertions.assertEquals(3, cielParts.length, "Expected 2 inflection patterns in: " + cielEntry); + String firstPatternId = cielParts[1].trim(); + String secondPatternId = cielParts[2].trim(); + + // "ciels" only exists in Wikidata; Wikidata's pattern has count=3 vs count=1 in dmlexOverlayFile, so it comes first + String cielsEntry = entries.get("ciels"); + Assertions.assertNotNull(cielsEntry); + String wikidataPatternId = cielsEntry.split("inflection=")[1].trim(); + Assertions.assertEquals(wikidataPatternId, firstPatternId, + "Expected higher-count Wikidata pattern to come first in: " + cielEntry); + Assertions.assertNotEquals(firstPatternId, secondPatternId); + + String cieuxEntry = entries.get("cieux"); + Assertions.assertNotNull(cieuxEntry); + Assertions.assertTrue(cieuxEntry.contains("masculine"), "Expected masculine in: " + cieuxEntry); + Assertions.assertTrue(cieuxEntry.contains("feminine"), "Expected feminine in: " + cieuxEntry); + assertAllXmlPatternsReferencedInDictionary(actual); + } + + @Test + public void multiSourceArgumentOrderDefinesEqualCountPatternOrderTest() throws Exception { + // Verify that: + // 1. When rarity and pattern word counts are equal ("when all other things are equal"), + // the CLI argument order of --wikidata and --dmlex defines the inflection pattern priority order + // on ambiguous surface forms (first definition comes first). + // 2. When pattern word counts differ (e.g. "ciel" in lexiconSource has count=3 vs count=1 in dmlexOverlayFile), + // the higher-count pattern takes precedence regardless of CLI argument order. + // 3. When rarity differs (a rare lemma in the earlier source vs a non-rare lemma in the later source, + // both with count=1), the non-rare pattern still precedes the rare pattern. + Path tempWikidata = Files.createTempFile(Paths.get(baseDir), "wikidata-order-", ".json"); + Path tempDmlex = Files.createTempFile(Paths.get(baseDir), "dmlex-order-", ".json"); + Path tempRareDmlex = Files.createTempFile(Paths.get(baseDir), "dmlex-rare-order-", ".json"); + try { + Files.writeString(tempWikidata, """ + [ + { + "type": "lexeme", + "id": "L100", + "lemmas": { "en": { "language": "en", "value": "index" } }, + "lexicalCategory": "Q1084", + "claims": {}, + "forms": [ + { + "id": "L100-F1", + "representations": { "en": { "language": "en", "value": "index" } }, + "grammaticalFeatures": ["Q110786"], + "claims": {} + }, + { + "id": "L100-F2", + "representations": { "en": { "language": "en", "value": "indexes" } }, + "grammaticalFeatures": ["Q146786"], + "claims": {} + } + ], + "senses": [] + } + ] + """, StandardCharsets.UTF_8); + Files.writeString(tempDmlex, """ + { + "langCode": "en", + "entries": [ + { + "id": "dmlex-index", + "headword": "index", + "partsOfSpeech": ["n"], + "inflectedForms": [ + { "tag": "sg", "text": "index" }, + { "tag": "pl", "text": "indices" } + ] + } + ], + "partOfSpeechTags": [ + { "tag": "n", "sameAs": ["http://www.wikidata.org/entity/Q1084"] } + ], + "inflectedFormTags": [ + { "tag": "sg", "sameAs": ["http://www.wikidata.org/entity/Q110786"] }, + { "tag": "pl", "sameAs": ["http://www.wikidata.org/entity/Q146786"] } + ] + } + """, StandardCharsets.UTF_8); + + // Case 1a: Equal counts (count=1 each), --wikidata before --dmlex -> Wikidata pattern first + String[] wikidataFirstArgs = { + "--inflection-types", "noun", + "--inflections", inflectionalFile, + "--dictionary", dictionaryFile, + "--wikidata", tempWikidata.toString(), + "--dmlex", tempDmlex.toString() + }; + Map wdFirstEntries = parseDictionaryEntries(getParserOutput(wikidataFirstArgs)); + String wdPatternId1 = wdFirstEntries.get("indexes").split("inflection=")[1].trim(); + String dmPatternId1 = wdFirstEntries.get("indices").split("inflection=")[1].trim(); + Assertions.assertNotEquals(wdPatternId1, dmPatternId1); + String[] wdFirstIndexParts = wdFirstEntries.get("index").split("inflection="); + Assertions.assertEquals(3, wdFirstIndexParts.length); + Assertions.assertEquals(wdPatternId1, wdFirstIndexParts[1].trim(), + "Expected --wikidata pattern first when --wikidata precedes --dmlex"); + Assertions.assertEquals(dmPatternId1, wdFirstIndexParts[2].trim()); + + // Case 1b: Equal counts (count=1 each), --dmlex before --wikidata -> DMLex pattern first + String[] dmlexFirstArgs = { + "--inflection-types", "noun", + "--inflections", inflectionalFile, + "--dictionary", dictionaryFile, + "--dmlex", tempDmlex.toString(), + "--wikidata", tempWikidata.toString() + }; + String dmFirstOutput = getParserOutput(dmlexFirstArgs); + Map dmFirstEntries = parseDictionaryEntries(dmFirstOutput); + String wdPatternId2 = dmFirstEntries.get("indexes").split("inflection=")[1].trim(); + String dmPatternId2 = dmFirstEntries.get("indices").split("inflection=")[1].trim(); + Assertions.assertNotEquals(wdPatternId2, dmPatternId2); + String[] dmFirstIndexParts = dmFirstEntries.get("index").split("inflection="); + Assertions.assertEquals(3, dmFirstIndexParts.length); + Assertions.assertEquals(dmPatternId2, dmFirstIndexParts[1].trim(), + "Expected --dmlex pattern first when --dmlex precedes --wikidata"); + Assertions.assertEquals(wdPatternId2, dmFirstIndexParts[2].trim()); + assertAllXmlPatternsReferencedInDictionary(dmFirstOutput); + + // Case 2: Unequal counts ("ciel" in lexiconSource has count=3 vs count=1 in dmlexOverlayFile) -> + // higher-count Wikidata pattern still wins even when --dmlex precedes --wikidata + String[] unequalCountDmlexFirstArgs = { + "--inflection-types", "noun,adjective,proper-noun", + "--add-sound", "vowel-start", + "--inflections", inflectionalFile, + "--dictionary", dictionaryFile, + "--dmlex", dmlexOverlayFile, + "--wikidata", lexiconSource + }; + Map unequalEntries = parseDictionaryEntries(getParserOutput(unequalCountDmlexFirstArgs)); + String higherCountWikidataPatternId = unequalEntries.get("ciels").split("inflection=")[1].trim(); + String[] unequalCielParts = unequalEntries.get("ciel").split("inflection="); + Assertions.assertEquals(3, unequalCielParts.length); + Assertions.assertEquals(higherCountWikidataPatternId, unequalCielParts[1].trim(), + "Expected higher-count pattern to precede lower-count pattern even when --dmlex came first"); + + // Case 3: Unequal rarity (rare DMLex lemma first vs non-rare Wikidata lemma second, both count=1) -> + // non-rare Wikidata pattern precedes rare DMLex pattern despite CLI order + Files.writeString(tempRareDmlex, """ + { + "langCode": "en", + "entries": [ + { + "id": "dmlex-index-rare", + "headword": "index", + "partsOfSpeech": ["n"], + "labels": ["rare"], + "inflectedForms": [ + { "tag": "sg", "text": "index" }, + { "tag": "pl", "text": "indices" } + ] + } + ], + "partOfSpeechTags": [ + { "tag": "n", "sameAs": ["http://www.wikidata.org/entity/Q1084"] } + ], + "labelTags": [ + { "tag": "rare", "sameAs": ["http://www.wikidata.org/entity/Q55094451"] } + ], + "inflectedFormTags": [ + { "tag": "sg", "sameAs": ["http://www.wikidata.org/entity/Q110786"] }, + { "tag": "pl", "sameAs": ["http://www.wikidata.org/entity/Q146786"] } + ] + } + """, StandardCharsets.UTF_8); + String[] rareDmlexFirstArgs = { + "--inflection-types", "noun", + "--inflections", inflectionalFile, + "--dictionary", dictionaryFile, + "--dmlex", tempRareDmlex.toString(), + "--wikidata", tempWikidata.toString() + }; + Map rareFirstEntries = parseDictionaryEntries(getParserOutput(rareDmlexFirstArgs)); + String nonRareWikidataPatternId = rareFirstEntries.get("indexes").split("inflection=")[1].trim(); + String rareDmlexPatternId = rareFirstEntries.get("indices").split("inflection=")[1].trim(); + String[] rareFirstIndexParts = rareFirstEntries.get("index").split("inflection="); + Assertions.assertEquals(3, rareFirstIndexParts.length); + Assertions.assertEquals(nonRareWikidataPatternId, rareFirstIndexParts[1].trim(), + "Expected non-rare pattern to precede rare pattern even when rare source came first"); + Assertions.assertEquals(rareDmlexPatternId, rareFirstIndexParts[2].trim()); + } finally { + Files.deleteIfExists(tempWikidata); + Files.deleteIfExists(tempDmlex); + Files.deleteIfExists(tempRareDmlex); + } + } + + @Test + public void multiSourceSameFileHomographMergeTest() throws Exception { + // Verify that multiple entries for the same surface form ("Élie" as both proper-noun and verb) + // in a DMLex file merge cleanly with Wikidata's entry and preserve all referenced XML patterns. + Path tempDmlex = Files.createTempFile(Paths.get(baseDir), "dmlex-homograph-", ".json"); + try { + String json = """ + { + "langCode": "en", + "entries": [ + { + "id": "elie-proper-noun", + "headword": "Élie", + "partsOfSpeech": ["propn-fem"], + "pronunciations": [ + { + "transcriptions": [ + { "text": "e.li", "scheme": "en-fonipa" } + ] + } + ], + "inflectedForms": [ + { + "tag": "sg", + "text": "Élie" + } + ] + }, + { + "id": "elie-verb", + "headword": "Élie", + "partsOfSpeech": ["v"] + } + ], + "partOfSpeechTags": [ + { + "tag": "propn-fem", + "sameAs": [ + "http://www.wikidata.org/entity/Q147276", + "http://www.wikidata.org/entity/Q1775415" + ] + }, + { + "tag": "v", + "sameAs": ["http://www.wikidata.org/entity/Q24905"] + } + ], + "inflectedFormTags": [ + { + "tag": "sg", + "sameAs": ["http://www.wikidata.org/entity/Q110786"] + } + ] + } + """; + Files.writeString(tempDmlex, json, StandardCharsets.UTF_8); + + String[] mergeArgs = { + "--inflection-types", "noun,adjective,proper-noun", + "--add-sound", "vowel-start", + "--inflections", inflectionalFile, + "--dictionary", dictionaryFile, + "--wikidata", lexiconSource, + "--dmlex=" + tempDmlex + }; + String mergeOutput = getParserOutput(mergeArgs); + Map mergeEntries = parseDictionaryEntries(mergeOutput); + String elieMerge = mergeEntries.get("Élie"); + Assertions.assertNotNull(elieMerge); + Assertions.assertTrue( + elieMerge.startsWith("singular plural masculine feminine vowel-start noun proper-noun verb inflection="), + "Unexpected merge entry: " + elieMerge); + Assertions.assertEquals(2, elieMerge.split("inflection=").length - 1); + Set mergeXmlPatternIds = assertAllXmlPatternsReferencedInDictionary(mergeOutput); + Assertions.assertEquals(8, mergeXmlPatternIds.size()); + } finally { + Files.deleteIfExists(tempDmlex); + } + } + + @Test + public void multiFileDMLexSpaceAndCommaSeparatedTest() throws Exception { + String[] spaceArgs = { + "--inflection-types", "noun,proper-noun", + "--inflections", inflectionalFile, + "--dictionary", dictionaryFile, + "--dmlex", dmlexLexiconFile, dmlexPatchFile + }; + String spaceOutput = getParserOutput(spaceArgs); + + String[] commaArgs = { + "--inflection-types", "noun,proper-noun", + "--inflections", inflectionalFile, + "--dictionary", dictionaryFile, + "--dmlex", dmlexLexiconFile + "," + dmlexPatchFile + }; + String commaOutput = getParserOutput(commaArgs); + + compareOutputs(spaceOutput, commaOutput); + + Map entries = parseDictionaryEntries(spaceOutput); + // "cat" merges dmlexLexicon.json (masculine, plural "cats") and dmlexPatch.json (feminine, plural "kitties") + Assertions.assertTrue(entries.get("cat").contains("singular masculine feminine noun")); + Assertions.assertEquals(2, entries.get("cat").split("inflection=").length - 1); + Assertions.assertTrue(entries.containsKey("kitties")); + Assertions.assertTrue(entries.containsKey("cats")); + + // "bats" merges lemma "bat" (masculine noun) and lemma "batte" (feminine noun) + Assertions.assertTrue(entries.get("bats").contains("plural masculine feminine noun")); + + // "wow" merges dmlexLexicon.json (interjection) and dmlexPatch.json (infinitive verb) + Assertions.assertEquals("infinitive interjection verb", entries.get("wow")); + Assertions.assertEquals("past verb", entries.get("wowed")); + assertAllXmlPatternsReferencedInDictionary(spaceOutput); + } + + @Test + public void invalidDMLexSchemaAndUnknownOptionTest() throws Exception { + // 1. JSON object missing required "langCode" / "entries" + String[] invalidObjArgs = { + "--inflections", inflectionalFile, + "--dictionary", dictionaryFile, + "--dmlex", dmlexInvalidSchemaFile + }; + Assertions.assertThrows(MismatchedInputException.class, () -> ParseWikidata.main(invalidObjArgs)); + + // 2. Passing Wikidata JSON array to --dmlex or DMLex JSON object to --wikidata + String[] wikidataAsDmlexArgs = { + "--inflections", inflectionalFile, + "--dictionary", dictionaryFile, + "--dmlex", lexiconSource + }; + Assertions.assertThrows(MismatchedInputException.class, () -> ParseWikidata.main(wikidataAsDmlexArgs)); + String[] dmlexAsWikidataArgs = { + "--inflections", inflectionalFile, + "--dictionary", dictionaryFile, + "--wikidata", dmlexLexiconFile + }; + Assertions.assertThrows(MismatchedInputException.class, () -> ParseWikidata.main(dmlexAsWikidataArgs)); + + // 3. Bare positional file without --wikidata or --dmlex is rejected + String[] bareFileArgs = { + "--inflections", inflectionalFile, + "--dictionary", dictionaryFile, + lexiconSource + }; + Assertions.assertThrows(IllegalArgumentException.class, () -> ParseWikidata.main(bareFileArgs)); + + // 4. --dmlex with no file argument is rejected + String[] emptyDmlexArgs = { + "--inflections", inflectionalFile, + "--dictionary", dictionaryFile, + "--dmlex" + }; + Assertions.assertThrows(IllegalArgumentException.class, () -> ParseWikidata.main(emptyDmlexArgs)); + + // 5. Strict validation: unknown property, missing/multiple POS, undefined tag, unresolvable sameAs, + // and missing headword in inflectedForms must fail loudly with file name and entry id. + Path tempInvalid = Files.createTempFile(Paths.get(baseDir), "dmlex-strict-invalid-", ".json"); + try { + // 5a. Unknown JSON property on entry ("partOfSpeech" instead of "partsOfSpeech") + Files.writeString(tempInvalid, """ + { + "langCode": "en", + "entries": [ + { "id": "bad-prop", "headword": "cat", "partOfSpeech": ["n"] } + ] + } + """, StandardCharsets.UTF_8); + String[] badArgs = { + "--inflections", inflectionalFile, + "--dictionary", dictionaryFile, + "--dmlex", tempInvalid.toString() + }; + MismatchedInputException badPropEx = Assertions.assertThrows( + MismatchedInputException.class, () -> ParseWikidata.main(badArgs)); + Assertions.assertTrue(badPropEx.getMessage().contains(tempInvalid.toString())); + + // 5b. Entry with no part of speech + Files.writeString(tempInvalid, """ + { + "langCode": "en", + "entries": [ + { "id": "no-pos", "headword": "cat" } + ] + } + """, StandardCharsets.UTF_8); + IllegalArgumentException noPosEx = Assertions.assertThrows( + IllegalArgumentException.class, () -> ParseWikidata.main(badArgs)); + Assertions.assertTrue(noPosEx.getMessage().contains(tempInvalid.toString())); + Assertions.assertTrue(noPosEx.getMessage().contains("no-pos")); + + // 5c. Entry with multiple parts of speech + Files.writeString(tempInvalid, """ + { + "langCode": "en", + "entries": [ + { "id": "multi-pos", "headword": "Czech", "partsOfSpeech": ["n", "adj"] } + ], + "partOfSpeechTags": [ + { "tag": "n", "sameAs": ["http://www.wikidata.org/entity/Q1084"] }, + { "tag": "adj", "sameAs": ["http://www.wikidata.org/entity/Q34698"] } + ] + } + """, StandardCharsets.UTF_8); + IllegalArgumentException multiPosEx = Assertions.assertThrows( + IllegalArgumentException.class, () -> ParseWikidata.main(badArgs)); + Assertions.assertTrue(multiPosEx.getMessage().contains(tempInvalid.toString())); + Assertions.assertTrue(multiPosEx.getMessage().contains("multi-pos")); + + // 5d. Undefined tag in partsOfSpeech + Files.writeString(tempInvalid, """ + { + "langCode": "en", + "entries": [ + { "id": "undef-tag", "headword": "cat", "partsOfSpeech": ["n-missing"] } + ] + } + """, StandardCharsets.UTF_8); + IllegalArgumentException undefTagEx = Assertions.assertThrows( + IllegalArgumentException.class, () -> ParseWikidata.main(badArgs)); + Assertions.assertTrue(undefTagEx.getMessage().contains(tempInvalid.toString())); + Assertions.assertTrue(undefTagEx.getMessage().contains("undef-tag")); + + // 5e. Unresolvable sameAs URI + Files.writeString(tempInvalid, """ + { + "langCode": "en", + "entries": [ + { "id": "bad-uri", "headword": "cat", "partsOfSpeech": ["n"] } + ], + "partOfSpeechTags": [ + { "tag": "n", "sameAs": ["http://www.wikidata.org/entity/Q999999999"] } + ] + } + """, StandardCharsets.UTF_8); + IllegalArgumentException badUriEx = Assertions.assertThrows( + IllegalArgumentException.class, () -> ParseWikidata.main(badArgs)); + Assertions.assertTrue(badUriEx.getMessage().contains(tempInvalid.toString())); + Assertions.assertTrue(badUriEx.getMessage().contains("bad-uri")); + + // 5f. Non-empty inflectedForms missing the headword form (unmodified A.28 without headword form) + Files.writeString(tempInvalid, """ + { + "langCode": "ga", + "entries": [ + { + "id": "folúsghlantóir-n", + "headword": "folúsghlantóir", + "partsOfSpeech": ["n-masc"], + "inflectedForms": [ + { "tag": "sg-gen", "text": "folúsghlantóra" }, + { "tag": "pl", "text": "folúsghlantóirí" } + ] + } + ], + "partOfSpeechTags": [ + { + "tag": "n-masc", + "sameAs": [ + "http://www.wikidata.org/entity/Q1084", + "http://www.wikidata.org/entity/Q499327" + ] + } + ], + "inflectedFormTags": [ + { + "tag": "sg-gen", + "sameAs": [ + "http://www.wikidata.org/entity/Q110786", + "http://www.wikidata.org/entity/Q146233" + ] + }, + { + "tag": "pl", + "sameAs": ["http://www.wikidata.org/entity/Q146786"] + } + ] + } + """, StandardCharsets.UTF_8); + String[] gaBadArgs = { + "--language", "ga", + "--inflections", inflectionalFile, + "--dictionary", dictionaryFile, + "--dmlex", tempInvalid.toString() + }; + IllegalArgumentException missingHwEx = Assertions.assertThrows( + IllegalArgumentException.class, () -> ParseWikidata.main(gaBadArgs)); + Assertions.assertTrue(missingHwEx.getMessage().contains(tempInvalid.toString())); + Assertions.assertTrue(missingHwEx.getMessage().contains("folúsghlantóir-n")); + } finally { + Files.deleteIfExists(tempInvalid); + } + } + + @Test + public void dmlexCliOptionsCompatibilityTest() throws Exception { + Path tempDmlex = Files.createTempFile(Paths.get(baseDir), "dmlex-opts-", ".json"); + Path tempExcludedDmlex = Files.createTempFile(Paths.get(baseDir), "dmlex-opts-md-", ".json"); + Map>> savedTypeMap = new HashMap<>(Grammar.TYPEMAP.size()); + for (var entry : Grammar.TYPEMAP.entrySet()) { + savedTypeMap.put(entry.getKey(), new HashSet<>(entry.getValue())); + } + try { + String json = """ + { + "langCode": "ro", + "entries": [ + { + "id": "ro-1", + "headword": "casă", + "partsOfSpeech": ["n-fem"], + "labels": ["cnt", "dial"], + "inflectedForms": [ + { "tag": "sg-nom-acc", "text": "casă" }, + { "tag": "pl-nom-acc", "text": "case" }, + { "tag": "sg-short", "text": "casă-short" }, + { "tag": "sg", "text": "casă-archaic", "labels": ["arch"] } + ] + } + ], + "partOfSpeechTags": [ + { + "tag": "n-fem", + "sameAs": [ + "http://www.wikidata.org/entity/Q1084", + "http://www.wikidata.org/entity/Q1775415" + ] + } + ], + "labelTags": [ + { "tag": "cnt", "sameAs": ["http://www.wikidata.org/entity/Q1520033"] }, + { "tag": "dial", "sameAs": ["http://www.wikidata.org/entity/Q99990001"] }, + { "tag": "arch", "sameAs": ["http://www.wikidata.org/entity/Q99990002"] } + ], + "inflectedFormTags": [ + { + "tag": "sg", + "sameAs": ["http://www.wikidata.org/entity/Q110786"] + }, + { + "tag": "sg-nom-acc", + "sameAs": [ + "http://www.wikidata.org/entity/Q110786", + "http://www.wikidata.org/entity/Q99990003" + ] + }, + { + "tag": "pl-nom-acc", + "sameAs": [ + "http://www.wikidata.org/entity/Q146786", + "http://www.wikidata.org/entity/Q99990003" + ] + }, + { + "tag": "sg-short", + "sameAs": [ + "http://www.wikidata.org/entity/Q110786", + "http://www.wikidata.org/entity/Q96406487" + ] + } + ] + } + """; + String excludedJson = """ + { + "langCode": "ro-md", + "entries": [ + { + "id": "ro-md-1", + "headword": "moldovanWord", + "partsOfSpeech": ["n"], + "inflectedForms": [ + { "tag": "sg", "text": "moldovanWord" } + ] + } + ], + "partOfSpeechTags": [ + { "tag": "n", "sameAs": ["http://www.wikidata.org/entity/Q1084"] } + ], + "inflectedFormTags": [ + { "tag": "sg", "sameAs": ["http://www.wikidata.org/entity/Q110786"] } + ] + } + """; + Files.writeString(tempDmlex, json, StandardCharsets.UTF_8); + Files.writeString(tempExcludedDmlex, excludedJson, StandardCharsets.UTF_8); + + String[] args = { + "--language", "ro", + "--exclude-language", "ro-md", + "--inflection-types", "noun,proper-noun", + "--map-grammeme", "Q99990003,nominative,accusative", + "--expand-grammemes", "singular,nominative,accusative,feminine,noun:indefinite", + "--ignore-property", "countable,Q99990001", + "--ignore-entries-with-grammemes", "short-form,Q99990002", + "--inflections", inflectionalFile, + "--dictionary", dictionaryFile, + "--dmlex", tempDmlex.toString(), tempExcludedDmlex.toString() + }; + String actual = getParserOutput(args); + Map entries = parseDictionaryEntries(actual); + + // ro-md lexicon excluded via --exclude-language + Assertions.assertFalse(entries.containsKey("moldovanWord")); + // short-form and Q99990002 ("arch") surface forms ignored via --ignore-entries-with-grammemes + Assertions.assertFalse(entries.containsKey("casă-short")); + Assertions.assertFalse(entries.containsKey("casă-archaic")); + // countable and Q99990001 ("dial") ignored via --ignore-property; + // Q99990003 mapped to nominative,accusative; singular expanded with indefinite via --expand-grammemes + Assertions.assertEquals("singular accusative nominative indefinite feminine noun", entries.get("casă")); + Assertions.assertEquals("plural accusative nominative feminine noun", entries.get("case")); + } finally { + Grammar.TYPEMAP.clear(); + Grammar.TYPEMAP.putAll(savedTypeMap); + Files.deleteIfExists(tempDmlex); + Files.deleteIfExists(tempExcludedDmlex); + } + } + + @Test + public void dmlexEntriesBeforeLangCodeImplicitSchemeAndInterleavedSourcesTest() throws Exception { + // Tests: + // 1. "entries" appearing before "langCode" in the root DMLex JSON object. + // 2. Implicit transcription scheme (omitted "scheme") treated as IPA, while non-fonipa scheme is ignored. + // 3. Interleaved sources (--dmlex --wikidata --dmlex ) merging across all files. + Path tempDmlex = Files.createTempFile(Paths.get(baseDir), "dmlex-interleaved-", ".json"); + try { + String json = """ + { + "entries": [ + { + "id": "dmlex-apple", + "headword": "apple", + "partsOfSpeech": ["n"], + "labels": ["nonhuman"], + "pronunciations": [ + { + "labels": ["US"], + "transcriptions": [ + { "text": "ˈæp.əl" } + ] + } + ], + "inflectedForms": [ + { + "tag": "sg", + "text": "apple" + }, + { + "tag": "pl", + "text": "apples" + } + ] + }, + { + "id": "dmlex-ciel-early", + "headword": "ciel", + "partsOfSpeech": ["n-fem"], + "pronunciations": [ + { + "labels": ["en-US"], + "transcriptions": [ + { "text": "sjɛl", "scheme": "fr-fonipa" }, + { "text": "ael", "scheme": "fr-Latn" } + ] + } + ], + "inflectedForms": [ + { "tag": "sg", "text": "ciel" }, + { "tag": "pl", "text": "cieux" }, + { "tag": "pl", "text": "ciel-dmlex-only" } + ] + }, + { + "id": "dmlex-intj-early", + "headword": "uh-oh", + "partsOfSpeech": ["intj"], + "inflectedForms": [ + { "tag": "sg", "text": "uh-oh" } + ] + } + ], + "partOfSpeechTags": [ + { "tag": "n", "sameAs": ["http://www.wikidata.org/entity/Q1084"] }, + { + "tag": "n-fem", + "sameAs": [ + "http://www.wikidata.org/entity/Q1084", + "http://www.wikidata.org/entity/Q1775415" + ] + }, + { "tag": "intj", "sameAs": ["http://www.wikidata.org/entity/Q83034"] } + ], + "labelTags": [ + { "tag": "nonhuman", "sameAs": ["http://www.wikidata.org/entity/Q67372837"] } + ], + "inflectedFormTags": [ + { "tag": "sg", "sameAs": ["http://www.wikidata.org/entity/Q110786"] }, + { "tag": "pl", "sameAs": ["http://www.wikidata.org/entity/Q146786"] } + ], + "langCode": "en" + } + """; + Files.writeString(tempDmlex, json, StandardCharsets.UTF_8); + + Path tempDmlex2 = Files.createTempFile(Paths.get(baseDir), "dmlex-intj-late-", ".json"); + try { + String json2 = """ + { + "langCode": "en", + "entries": [ + { + "id": "dmlex-intj-late", + "headword": "uh-oh", + "partsOfSpeech": ["intj"], + "inflectedForms": [ + { "tag": "pl", "text": "uh-oh" } + ] + } + ], + "partOfSpeechTags": [ + { "tag": "intj", "sameAs": ["http://www.wikidata.org/entity/Q83034"] } + ], + "inflectedFormTags": [ + { "tag": "pl", "sameAs": ["http://www.wikidata.org/entity/Q146786"] } + ] + } + """; + Files.writeString(tempDmlex2, json2, StandardCharsets.UTF_8); + + String[] args = { + "--inflection-types", "noun,adjective,proper-noun", + "--add-sound", "vowel-start", + "--inflections", inflectionalFile, + "--dictionary", dictionaryFile, + "--dmlex", tempDmlex.toString(), + "--wikidata", lexiconSource, + "--dmlex", tempDmlex2.toString() + }; + String actual = getParserOutput(args); + Map entries = parseDictionaryEntries(actual); + + // "apple" from DMLex is parsed with implicit IPA scheme and "entries" before "langCode" + Assertions.assertTrue(entries.get("apple").startsWith("singular nonhuman vowel-start noun inflection=")); + Assertions.assertTrue(entries.get("apples").startsWith("plural nonhuman vowel-start noun inflection=")); + + // Both DMLex (feminine) and Wikidata (masculine) grammemes are merged on "ciel" and "cieux" + Assertions.assertTrue(entries.get("ciel").contains("masculine")); + Assertions.assertTrue(entries.get("ciel").contains("feminine")); + Assertions.assertTrue(entries.containsKey("ciels")); + Assertions.assertTrue(entries.get("cieux").contains("masculine")); + Assertions.assertTrue(entries.get("cieux").contains("feminine")); + // "ciel-dmlex-only" was not in Wikidata, and its non-IPA transcription ("fr-Latn": "ael") + // must not be treated as a vowel-start IPA transcription. + Assertions.assertTrue(entries.get("ciel-dmlex-only").startsWith("plural feminine noun inflection=")); + Assertions.assertFalse(entries.get("ciel-dmlex-only").contains("vowel-start")); + + // Non-inflected POS surface form "uh-oh" merges tempDmlex (singular) and tempDmlex2 (plural) + Assertions.assertEquals("singular plural interjection", entries.get("uh-oh")); + assertAllXmlPatternsReferencedInDictionary(actual); + } finally { + Files.deleteIfExists(tempDmlex2); + } + } finally { + Files.deleteIfExists(tempDmlex); + } + } + + @Test + public void deferredLexemeMultiVariantAndSupplementalMergeTest() throws Exception { + // filter_fr.properties defines L27261=L1566399 (organisateur = organisatrice). + // Tests: + // 1. Primary Wikidata file with multi-variant L27261 ("fr" and "fr-x-Q56352306") + // and L1566399 merges both variants and both partner lexemes without throwing "Duplicate lexeme". + // 2. Supplemental Wikidata file containing only L27261 (without L1566399) + // does not throw "L1566399: id not found" and merges with the primary file (with primary pattern first). + // 3. Duplicate top-level Lexeme objects with the same deferred ID in one file still throw IllegalStateException. + Path primaryWikidata = Files.createTempFile(Paths.get(baseDir), "wikidata-fr-primary-", ".json"); + Path supplementalWikidata = Files.createTempFile(Paths.get(baseDir), "wikidata-fr-supp-", ".json"); + Path duplicateWikidata = Files.createTempFile(Paths.get(baseDir), "wikidata-fr-dup-", ".json"); + try { + String primaryJson = """ + [ + { + "type": "lexeme", + "id": "L27261", + "lemmas": { + "fr": { "language": "fr", "value": "organisateur" }, + "fr-x-Q56352306": { "language": "fr-x-Q56352306", "value": "organisateur" } + }, + "lexicalCategory": "Q1084", + "claims": {}, + "forms": [ + { + "id": "L27261-F1", + "representations": { + "fr": { "language": "fr", "value": "organisateur" } + }, + "grammaticalFeatures": ["Q110786", "Q499327"], + "claims": {} + }, + { + "id": "L27261-F2", + "representations": { + "fr": { "language": "fr", "value": "organisateurs-old" } + }, + "grammaticalFeatures": ["Q146786", "Q499327"], + "claims": {} + } + ], + "senses": [] + }, + { + "type": "lexeme", + "id": "L1566399", + "lemmas": { + "fr": { "language": "fr", "value": "organisatrice" } + }, + "lexicalCategory": "Q1084", + "claims": {}, + "forms": [ + { + "id": "L1566399-F1", + "representations": { + "fr": { "language": "fr", "value": "organisatrice" } + }, + "grammaticalFeatures": ["Q110786", "Q1775415"], + "claims": {} + } + ], + "senses": [] + } + ] + """; + String supplementalJson = """ + [ + { + "type": "lexeme", + "id": "L27261", + "lemmas": { + "fr": { "language": "fr", "value": "organisateur" } + }, + "lexicalCategory": "Q1084", + "claims": {}, + "forms": [ + { + "id": "L27261-F1", + "representations": { + "fr": { "language": "fr", "value": "organisateur" } + }, + "grammaticalFeatures": ["Q110786", "Q499327"], + "claims": {} + }, + { + "id": "L27261-F2", + "representations": { + "fr": { "language": "fr", "value": "organisateurs" } + }, + "grammaticalFeatures": ["Q146786", "Q499327"], + "claims": {} + } + ], + "senses": [] + } + ] + """; + String duplicateJson = """ + [ + { + "type": "lexeme", + "id": "L27261", + "lemmas": { "fr": { "language": "fr", "value": "organisateur" } }, + "lexicalCategory": "Q1084", + "claims": {}, + "forms": [ + { + "id": "L27261-F1", + "representations": { "fr": { "language": "fr", "value": "organisateur" } }, + "grammaticalFeatures": ["Q110786"], + "claims": {} + } + ], + "senses": [] + }, + { + "type": "lexeme", + "id": "L27261", + "lemmas": { "fr": { "language": "fr", "value": "organisateur" } }, + "lexicalCategory": "Q1084", + "claims": {}, + "forms": [ + { + "id": "L27261-F1", + "representations": { "fr": { "language": "fr", "value": "organisateur" } }, + "grammaticalFeatures": ["Q110786"], + "claims": {} + } + ], + "senses": [] + } + ] + """; + Files.writeString(primaryWikidata, primaryJson, StandardCharsets.UTF_8); + Files.writeString(supplementalWikidata, supplementalJson, StandardCharsets.UTF_8); + Files.writeString(duplicateWikidata, duplicateJson, StandardCharsets.UTF_8); + + String[] args = { + "--language", "fr", + "--inflection-types", "noun", + "--inflections", inflectionalFile, + "--dictionary", dictionaryFile, + "--wikidata", primaryWikidata.toString(), supplementalWikidata.toString() + }; + String actual = getParserOutput(args); + Map entries = parseDictionaryEntries(actual); + + Assertions.assertTrue(entries.containsKey("organisateur")); + Assertions.assertTrue(entries.containsKey("organisateurs-old")); + Assertions.assertTrue(entries.containsKey("organisateurs")); + Assertions.assertTrue(entries.containsKey("organisatrice")); + // "organisateur" merges patterns from both Wikidata files, with primaryWikidata's pattern first + String primaryPatternId = entries.get("organisateurs-old").split("inflection=")[1].trim(); + String supplementalPatternId = entries.get("organisateurs").split("inflection=")[1].trim(); + String[] organisateurParts = entries.get("organisateur").split("inflection="); + Assertions.assertEquals(3, organisateurParts.length); + Assertions.assertEquals(primaryPatternId, organisateurParts[1].trim()); + Assertions.assertEquals(supplementalPatternId, organisateurParts[2].trim()); + assertAllXmlPatternsReferencedInDictionary(actual); + + // Also verify passing the partial supplemental Wikidata file BEFORE the primary Wikidata file + // does not fail with "L1566399: id not found" and places supplementalWikidata's pattern first. + String[] suppFirstArgs = { + "--language", "fr", + "--inflection-types", "noun", + "--inflections", inflectionalFile, + "--dictionary", dictionaryFile, + "--wikidata", supplementalWikidata.toString(), primaryWikidata.toString() + }; + String suppFirstOutput = getParserOutput(suppFirstArgs); + Map suppFirstEntries = parseDictionaryEntries(suppFirstOutput); + String suppFirstPrimaryId = suppFirstEntries.get("organisateurs-old").split("inflection=")[1].trim(); + String suppFirstSupplementalId = suppFirstEntries.get("organisateurs").split("inflection=")[1].trim(); + String[] suppFirstOrganisateurParts = suppFirstEntries.get("organisateur").split("inflection="); + Assertions.assertEquals(3, suppFirstOrganisateurParts.length); + Assertions.assertEquals(suppFirstSupplementalId, suppFirstOrganisateurParts[1].trim()); + Assertions.assertEquals(suppFirstPrimaryId, suppFirstOrganisateurParts[2].trim()); + assertAllXmlPatternsReferencedInDictionary(suppFirstOutput); + + // Missing deferred partner across ALL --wikidata inputs still throws IllegalArgumentException + String[] missingPartnerArgs = { + "--language", "fr", + "--inflections", inflectionalFile, + "--dictionary", dictionaryFile, + "--wikidata", supplementalWikidata.toString() + }; + Assertions.assertThrows(IllegalArgumentException.class, () -> ParseWikidata.main(missingPartnerArgs)); + + String[] dupArgs = { + "--language", "fr", + "--inflections", inflectionalFile, + "--dictionary", dictionaryFile, + "--wikidata", duplicateWikidata.toString() + }; + Assertions.assertThrows(IllegalStateException.class, () -> ParseWikidata.main(dupArgs)); + } finally { + Files.deleteIfExists(primaryWikidata); + Files.deleteIfExists(supplementalWikidata); + Files.deleteIfExists(duplicateWikidata); + } + } } diff --git a/tools/dictionary-parser/src/test/resources/dmlexExampleA28.json b/tools/dictionary-parser/src/test/resources/dmlexExampleA28.json new file mode 100644 index 000000000..6273d6523 --- /dev/null +++ b/tools/dictionary-parser/src/test/resources/dmlexExampleA28.json @@ -0,0 +1,67 @@ +{ + "uri": "http://example.com", + "langCode": "ga", + "entries": [ + { + "id": "folúsghlantóir-n", + "headword": "folúsghlantóir", + "partsOfSpeech": ["n-masc"], + "inflectedForms": [ + { + "tag": "sg", + "text": "folúsghlantóir" + }, + { + "tag": "sg-gen", + "text": "folúsghlantóra" + }, + { + "tag": "pl", + "text": "folúsghlantóirí" + } + ], + "senses": [] + } + ], + "partOfSpeechTags": [ + { + "tag": "n-masc", + "description": "masculine noun", + "sameAs": [ + "http://www.wikidata.org/entity/Q1084", + "http://www.wikidata.org/entity/Q499327" + ] + }, + { + "tag": "n-fem", + "description": "feminine noun", + "sameAs": [ + "http://www.wikidata.org/entity/Q1084", + "http://www.wikidata.org/entity/Q1775415" + ] + } + ], + "inflectedFormTags": [ + { + "tag": "sg", + "description": "singular", + "for": "n-masc n-fem", + "sameAs": ["http://www.wikidata.org/entity/Q110786"] + }, + { + "tag": "sg-gen", + "description": "singular genitive", + "for": "n-masc n-fem", + "sameAs": [ + "http://www.wikidata.org/entity/Q110786", + "http://www.wikidata.org/entity/Q146233" + ] + }, + { + "tag": "pl", + "description": "plural", + "for": "n-masc n-fem", + "sameAs": ["http://www.wikidata.org/entity/Q146786"] + } + ] +} diff --git a/tools/dictionary-parser/src/test/resources/dmlexInvalidSchema.json b/tools/dictionary-parser/src/test/resources/dmlexInvalidSchema.json new file mode 100644 index 000000000..ec1f4c8f9 --- /dev/null +++ b/tools/dictionary-parser/src/test/resources/dmlexInvalidSchema.json @@ -0,0 +1,3 @@ +{ + "unexpectedField": "not a valid DMLex document" +} diff --git a/tools/dictionary-parser/src/test/resources/dmlexLexicon.json b/tools/dictionary-parser/src/test/resources/dmlexLexicon.json new file mode 100644 index 000000000..e80196e80 --- /dev/null +++ b/tools/dictionary-parser/src/test/resources/dmlexLexicon.json @@ -0,0 +1,120 @@ +{ + "uri": "http://example.org/lexicon/en", + "langCode": "en", + "entries": [ + { + "id": "dmlex-1", + "headword": "cat", + "partsOfSpeech": ["n-masc"], + "inflectedForms": [ + { + "tag": "sg", + "text": "cat" + }, + { + "tag": "pl", + "text": "cats" + } + ] + }, + { + "id": "dmlex-2", + "headword": "bat", + "partsOfSpeech": ["n-masc"], + "inflectedForms": [ + { + "tag": "sg", + "text": "bat" + }, + { + "tag": "pl", + "text": "bats" + } + ] + }, + { + "id": "dmlex-3", + "headword": "apple", + "partsOfSpeech": ["n"], + "pronunciations": [ + { + "transcriptions": [ + { + "text": "ˈæp.əl", + "scheme": "en-fonipa" + }, + { + "text": "bæd", + "scheme": "en-fonxsamp" + } + ] + } + ], + "inflectedForms": [ + { + "tag": "sg", + "text": "apple" + }, + { + "tag": "pl", + "text": "apples" + } + ] + }, + { + "id": "dmlex-4", + "headword": "wow", + "partsOfSpeech": ["intj"] + }, + { + "id": "dmlex-5", + "headword": "London", + "partsOfSpeech": ["propn"], + "inflectedForms": [ + { + "tag": "sg", + "text": "London" + } + ] + } + ], + "partOfSpeechTags": [ + { + "tag": "n", + "description": "noun", + "sameAs": ["http://www.wikidata.org/entity/Q1084"] + }, + { + "tag": "n-masc", + "description": "masculine noun", + "sameAs": [ + "http://www.wikidata.org/entity/Q1084", + "http://www.wikidata.org/entity/Q499327" + ] + }, + { + "tag": "intj", + "description": "interjection", + "sameAs": ["http://www.wikidata.org/entity/Q83034"] + }, + { + "tag": "propn", + "description": "proper noun", + "sameAs": ["http://www.wikidata.org/entity/Q147276"] + } + ], + "inflectedFormTags": [ + { + "tag": "sg", + "description": "singular", + "for": "n n-masc propn", + "sameAs": ["http://www.wikidata.org/entity/Q110786"] + }, + { + "tag": "pl", + "description": "plural", + "for": "n n-masc", + "sameAs": ["http://www.wikidata.org/entity/Q146786"] + } + ] +} diff --git a/tools/dictionary-parser/src/test/resources/dmlexOverlay.json b/tools/dictionary-parser/src/test/resources/dmlexOverlay.json new file mode 100644 index 000000000..fd6ae1d05 --- /dev/null +++ b/tools/dictionary-parser/src/test/resources/dmlexOverlay.json @@ -0,0 +1,106 @@ +{ + "uri": "http://example.org/lexicon/en-overlay", + "langCode": "en", + "entries": [ + { + "id": "overlay-new", + "headword": "book", + "partsOfSpeech": ["n"], + "inflectedForms": [ + { + "tag": "sg", + "text": "book" + }, + { + "tag": "sg-gen", + "text": "book's" + }, + { + "tag": "pl", + "text": "books" + }, + { + "tag": "pl-gen", + "text": "books'" + } + ] + }, + { + "id": "overlay-diff-pos", + "headword": "hey", + "partsOfSpeech": ["n"], + "inflectedForms": [ + { + "tag": "sg", + "text": "hey" + }, + { + "tag": "pl", + "text": "heys" + } + ] + }, + { + "id": "overlay-same-pos-conflict", + "headword": "ciel", + "partsOfSpeech": ["n-fem"], + "inflectedForms": [ + { + "tag": "sg", + "text": "ciel" + }, + { + "tag": "pl", + "text": "cieux" + } + ] + } + ], + "partOfSpeechTags": [ + { + "tag": "n", + "description": "noun", + "sameAs": ["http://www.wikidata.org/entity/Q1084"] + }, + { + "tag": "n-fem", + "description": "feminine noun", + "sameAs": [ + "http://www.wikidata.org/entity/Q1084", + "http://www.wikidata.org/entity/Q1775415" + ] + } + ], + "inflectedFormTags": [ + { + "tag": "sg", + "description": "singular", + "for": "n n-fem", + "sameAs": ["http://www.wikidata.org/entity/Q110786"] + }, + { + "tag": "sg-gen", + "description": "singular genitive", + "for": "n", + "sameAs": [ + "http://www.wikidata.org/entity/Q110786", + "http://www.wikidata.org/entity/Q146233" + ] + }, + { + "tag": "pl", + "description": "plural", + "for": "n n-fem", + "sameAs": ["http://www.wikidata.org/entity/Q146786"] + }, + { + "tag": "pl-gen", + "description": "plural genitive", + "for": "n", + "sameAs": [ + "http://www.wikidata.org/entity/Q146786", + "http://www.wikidata.org/entity/Q146233" + ] + } + ] +} diff --git a/tools/dictionary-parser/src/test/resources/dmlexPatch.json b/tools/dictionary-parser/src/test/resources/dmlexPatch.json new file mode 100644 index 000000000..66293c561 --- /dev/null +++ b/tools/dictionary-parser/src/test/resources/dmlexPatch.json @@ -0,0 +1,92 @@ +{ + "uri": "http://example.org/lexicon/en-patch", + "langCode": "en", + "entries": [ + { + "id": "patch-1", + "headword": "cat", + "partsOfSpeech": ["n-fem"], + "inflectedForms": [ + { + "tag": "sg", + "text": "cat" + }, + { + "tag": "pl", + "text": "kitties" + } + ] + }, + { + "id": "patch-2", + "headword": "wow", + "partsOfSpeech": ["v"], + "inflectedForms": [ + { + "tag": "inf", + "text": "wow" + }, + { + "tag": "pst", + "text": "wowed" + } + ] + }, + { + "id": "patch-3", + "headword": "batte", + "partsOfSpeech": ["n-fem"], + "inflectedForms": [ + { + "tag": "sg", + "text": "batte" + }, + { + "tag": "pl", + "text": "bats" + } + ] + } + ], + "partOfSpeechTags": [ + { + "tag": "n-fem", + "description": "feminine noun", + "sameAs": [ + "http://www.wikidata.org/entity/Q1084", + "http://www.wikidata.org/entity/Q1775415" + ] + }, + { + "tag": "v", + "description": "verb", + "sameAs": ["http://www.wikidata.org/entity/Q24905"] + } + ], + "inflectedFormTags": [ + { + "tag": "sg", + "description": "singular", + "for": "n-fem", + "sameAs": ["http://www.wikidata.org/entity/Q110786"] + }, + { + "tag": "pl", + "description": "plural", + "for": "n-fem", + "sameAs": ["http://www.wikidata.org/entity/Q146786"] + }, + { + "tag": "inf", + "description": "infinitive", + "for": "v", + "sameAs": ["http://www.wikidata.org/entity/Q179230"] + }, + { + "tag": "pst", + "description": "past", + "for": "v", + "sameAs": ["http://www.wikidata.org/entity/Q1994301"] + } + ] +}