Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion tools/dictionary-parser/ParseWikidata
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ if [[ $# -eq 2 && $1 == "--all" ]]; then
if [[ "$LAST_LINE" =~ $COMMAND_OPTS_PREFIX ]]; then
OPTIONS=`echo $LAST_LINE | sed "s/$COMMAND_OPTS_PREFIX//"`
echo Generating $LANGUAGE from $LEXEME_FILE
$RUN_COMMAND $OPTIONS --inflections $LEXICAL_DIR/inflectional_${LANGUAGE}.xml --dictionary $LEXICAL_DIR/dictionary_${LANGUAGE}.lst $LEXEME_FILE
$RUN_COMMAND $OPTIONS --inflections $LEXICAL_DIR/inflectional_${LANGUAGE}.xml --dictionary $LEXICAL_DIR/dictionary_${LANGUAGE}.lst --wikidata $LEXEME_FILE
else
echo skipping $LANGUAGE
fi
Expand Down
24 changes: 23 additions & 1 deletion tools/dictionary-parser/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -20,5 +20,27 @@ These tools generate files that describes the grammatical properties of words fr
./ParseWikidata <THE_OPTIONS_FROM_STEP_3> \
--inflections ../../resources/org/unicode/inflection/dictionary/inflectional_XX.xml \
--dictionary ../../resources/org/unicode/inflection/dictionary/dictionary_XX.lst \
<wikidata-NNNNNNNN-lexemes.json[.bz2]>
--wikidata <wikidata-NNNNNNNN-lexemes.json[.bz2]>
```

## Usage for OASIS DMLex and Multi-Source Inputs

`ParseWikidata` also supports [OASIS DMLex v1.0](https://docs.oasis-open.org/lexidma/dmlex/v1.0/dmlex-v1.0.html) JSON files via `--dmlex`, either standalone (bypassing Wikidata completely) or combined with `--wikidata`:

- **Standalone DMLex (single or multiple files)**:
```
./ParseWikidata --language <lang> \
--inflections inflectional_<lang>.xml \
--dictionary dictionary_<lang>.lst \
--dmlex <custom1.json> [<custom2.json> ...]
```
- **Multi-Source (Wikidata + DMLex)**:
```
./ParseWikidata <OPTIONS> \
--inflections inflectional_<lang>.xml \
--dictionary dictionary_<lang>.lst \
--dmlex <supplement1.json> [<supplement2.json> ...] \
--wikidata <wikidata-lexemes.json[.bz2]>
```
- Multiple `--wikidata` and `--dmlex` inputs are processed in command-line argument order, merging grammemes and inflection patterns across sources (prioritizing earlier sources' inflection patterns when rarity and pattern frequency counts are equal).

Original file line number Diff line number Diff line change
@@ -0,0 +1,106 @@
/*
* Copyright 2025-2026 Unicode Incorporated and others. All rights reserved.
*/
package org.unicode.wikidata;

import com.fasterxml.jackson.annotation.JsonIgnoreProperties;

import java.util.List;

/**
* A lexical entry in OASIS DMLex v1.0 JSON format (spec §5.2, {@code dmlex.schema.json}).
*/
@JsonIgnoreProperties({
"homographNumber",
"senses",
"placeholderMarkers",
"etymologies"
})
public class DMLexEntry {
/**
* Root {@code lexicographicResource} object in OASIS DMLex v1.0 JSON format.
*/
@JsonIgnoreProperties({
"title",
"uri",
"translationLanguages",
"definitionTypeTags",
"labelTypeTags",
"sourceIdentityTags",
"transcriptionSchemeTags",
"relations",
"relationTypes",
"etymonLanguages",
"etymonTypes"
})
public static class Lexicon {
public String langCode;
public List<DMLexEntry> entries;
public List<PartOfSpeechTag> partOfSpeechTags;
public List<InflectedFormTag> inflectedFormTags;
public List<LabelTag> labelTags;
}

/**
* A {@code partOfSpeechTag} controlled value in OASIS DMLex v1.0 JSON format.
*/
@JsonIgnoreProperties({
"description",
"for"
})
public static class PartOfSpeechTag {
public String tag;
public List<String> sameAs;
}

/**
* An {@code inflectedFormTag} controlled value in OASIS DMLex v1.0 JSON format.
*/
@JsonIgnoreProperties({
"description",
"for"
})
public static class InflectedFormTag {
public String tag;
public List<String> sameAs;
}

/**
* A {@code labelTag} controlled value in OASIS DMLex v1.0 JSON format.
*/
@JsonIgnoreProperties({
"typeTag",
"description",
"for"
})
public static class LabelTag {
public String tag;
public List<String> sameAs;
}

/**
* A {@code pronunciation} object in OASIS DMLex v1.0 JSON format.
*/
@JsonIgnoreProperties({
"soundFile",
"labels"
})
public static class Pronunciation {
public List<Transcription> transcriptions;
}

/**
* A {@code transcription} object in OASIS DMLex v1.0 JSON format.
*/
public static class Transcription {
public String text;
public String scheme;
}

public String id;
public String headword;
public List<String> partsOfSpeech;
public List<String> labels;
public List<Pronunciation> pronunciations;
public List<DMLexInflectedForm> inflectedForms;
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
/*
* Copyright 2025-2026 Unicode Incorporated and others. All rights reserved.
*/
package org.unicode.wikidata;

import java.util.List;

/**
* An {@code inflectedForm} of a lexical entry in OASIS DMLex v1.0 JSON format (spec §5.2, {@code dmlex.schema.json}).
*/
public class DMLexInflectedForm {
public String tag;
public String text;
public List<String> labels;
public List<DMLexEntry.Pronunciation> pronunciations;
}
Loading
Loading