Skip to content

Commit 346cf89

Browse files
authored
Inflection-217 Support modifying form representations in wikidata_modify.py (#218)
1 parent 9e4f625 commit 346cf89

1 file changed

Lines changed: 107 additions & 24 deletions

File tree

‎tools/wikidata/tools/wikidata_modify.py‎

Lines changed: 107 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -22,14 +22,16 @@
2222
"lexemeId": "L1493026",
2323
"modifications": [
2424
{ "formId": "L1493026-F9", "add": ["Q65048529"] },
25-
{ "form": "ai-udviklingen", "add": ["Q65048529"], "remove": [] }
25+
{ "form": "ai-udviklingen", "add": ["Q65048529"], "remove": [] },
26+
{ "formId": "L1590644-F13", "representations": { "sr": "кошћу", "sr-cyrl": "кошћу", "sr-latn": "košću" } }
2627
]
2728
}
2829
]
2930
3031
A form is identified either by "formId" (preferred, unambiguous) or by "form"
3132
(the surface string, matched against the representation in the requested
32-
language). "add" and "remove" hold raw feature Q-ids.
33+
language). "add" and "remove" hold raw feature Q-ids, and "representations"
34+
holds a mapping of representation language codes to replacement surface strings.
3335
3436
Run with --self-check to exercise the pure logic without any network access.
3537
"""
@@ -52,7 +54,7 @@ def _import_tfsl():
5254
Deferred so that pure-logic paths (validation, --self-check) work without a
5355
tfsl checkout or network access.
5456
"""
55-
sys.path.append(os.path.join(os.path.dirname(__file__), "..", "..", "..", "tfsl"))
57+
sys.path.append(os.path.join(os.path.dirname(__file__), "..", "..", "..", "..", "tfsl"))
5658
import tfsl
5759

5860
return tfsl
@@ -92,11 +94,19 @@ def validate_modifications(data):
9294
raise ValueError(f"'form' must be a string in:\n{mod}")
9395
add = mod.get("add", [])
9496
remove = mod.get("remove", [])
97+
reps = mod.get("representations", {})
9598
if not isinstance(add, list) or not isinstance(remove, list):
9699
raise ValueError(f"'add' and 'remove' must be arrays in:\n{mod}")
97-
if not add and not remove:
100+
if not isinstance(reps, dict) or any(
101+
not isinstance(k, str) or not isinstance(v, str) or not v
102+
for k, v in reps.items()
103+
):
98104
raise ValueError(
99-
f"Modification must have a non-empty 'add' or 'remove':\n{mod}"
105+
f"'representations' must be a dict of non-empty strings in:\n{mod}"
106+
)
107+
if not add and not remove and not reps:
108+
raise ValueError(
109+
f"Modification must have a non-empty 'add', 'remove', or 'representations':\n{mod}"
100110
)
101111
for qid in add + remove:
102112
if not isinstance(qid, str) or not QID_RE.match(qid):
@@ -180,42 +190,87 @@ def locate_form(lexeme, mod, lang):
180190
def apply_to_lexeme(lexeme, mods, lang):
181191
"""Applies modifications to an editable lexeme in place.
182192
183-
Mutates each target form's feature set directly, preserving the form id so
184-
the eventual push edits the existing form rather than adding a new one.
193+
Mutates each target form's feature set and/or representations directly,
194+
preserving the form id so the eventual push edits the existing form rather
195+
than adding a new one.
185196
186197
Args:
187198
lexeme: An editable tfsl Lexeme.
188199
mods (list): Modifications for this lexeme.
189200
lang (str): Language code for surface-string matching.
190201
191202
Returns:
192-
List of (form_id, surface, before_features, after_features) tuples.
203+
List of (form_id, surface, before_features, after_features, rep_diffs) tuples.
193204
"""
205+
tfsl = _import_tfsl()
194206
changes = []
195207
for mod in mods:
196208
form = locate_form(lexeme, mod, lang)
197209
before = set(form.features)
198-
new_features, warnings = compute_new_features(
199-
before, mod.get("add", []), mod.get("remove", [])
200-
)
210+
add = mod.get("add", [])
211+
remove = mod.get("remove", [])
212+
reps = mod.get("representations", {})
213+
if add or remove:
214+
new_features, warnings = compute_new_features(before, add, remove)
215+
else:
216+
new_features, warnings = before, []
201217
surface = _form_rep_value(form, lang)
202218
for warning in warnings:
203219
print(f" WARNING ({getattr(form, 'id', '?')} '{surface}'): {warning}")
204-
form.features.clear()
205-
form.features.update(new_features)
220+
if add or remove:
221+
form.features.clear()
222+
form.features.update(new_features)
223+
rep_diffs = {}
224+
if reps:
225+
before_reps = form.__jsonout__().get("representations", {})
226+
for rep_lang, new_val in reps.items():
227+
old_val = before_reps.get(rep_lang, {}).get("value")
228+
if old_val == new_val:
229+
print(
230+
f" WARNING ({getattr(form, 'id', '?')} '{surface}'): representation[{rep_lang}] '{new_val}' already matches (no-op)"
231+
)
232+
rep_diffs[rep_lang] = (old_val, new_val)
233+
lang_obj = tfsl.languages.get_first_lang(rep_lang)
234+
form.representations = form.representations + (new_val @ lang_obj)
206235
changes.append(
207-
(getattr(form, "id", None), surface, sorted(before), sorted(new_features))
236+
(
237+
getattr(form, "id", None),
238+
surface,
239+
sorted(before),
240+
sorted(new_features),
241+
rep_diffs,
242+
)
208243
)
209244
return changes
210245

211246

212247
def _print_changes(lexeme_id, changes):
213-
"""Prints a per-form before -> after feature diff."""
248+
"""Prints a per-form before -> after feature/representation diff."""
214249
print(f"{lexeme_id}:")
215-
for form_id, surface, before, after in changes:
250+
for change in changes:
251+
form_id, surface, before, after = change[:4]
252+
rep_diffs = change[4] if len(change) > 4 else {}
216253
print(f" {form_id} '{surface}'")
217-
print(f" before: {before}")
218-
print(f" after: {after}")
254+
if before != after:
255+
print(f" features before: {before}")
256+
print(f" features after: {after}")
257+
for rep_lang, (old_val, new_val) in rep_diffs.items():
258+
print(f" rep[{rep_lang}]: '{old_val}' -> '{new_val}'")
259+
260+
261+
def build_form_edit_payload(form):
262+
"""Builds the ``data`` payload for a single ``wbleditformelements`` request.
263+
264+
Unlike ``WikibaseSession.push`` (a full-entity ``wbeditentity``),
265+
``wbleditformelements`` updates a single form's representations and
266+
grammaticalFeatures without re-serializing the lexeme's statements or senses
267+
(avoiding ``param-illegal`` errors on lexemes with existing statements/senses).
268+
"""
269+
form_json = form.__jsonout__()
270+
return {
271+
"representations": form_json["representations"],
272+
"grammaticalFeatures": form_json["grammaticalFeatures"],
273+
}
219274

220275

221276
def modify_data(username, password, lang, input_file, test_only, delay_ms):
@@ -243,9 +298,32 @@ def modify_data(username, password, lang, input_file, test_only, delay_ms):
243298
if test_only:
244299
_print_changes(lexeme_id, changes)
245300
else:
246-
print(f"Pushing {lexeme_id} ({len(changes)} forms modified).")
247-
session.push(lexeme)
248-
time.sleep(delay_ms / 1000)
301+
print(f"Updating {lexeme_id} ({len(changes)} forms modified):")
302+
forms_by_id = {getattr(f, "id", None): f for f in lexeme.forms}
303+
for change in changes:
304+
form_id = change[0]
305+
form = forms_by_id[form_id]
306+
payload = build_form_edit_payload(form)
307+
params = {
308+
"action": "wbleditformelements",
309+
"formId": form_id,
310+
"data": json.dumps(payload),
311+
"token": "__AUTO__",
312+
"format": "json",
313+
}
314+
while True:
315+
try:
316+
session.post(params)
317+
print(f" Updated {form_id}")
318+
break
319+
except PermissionError as err:
320+
if "ratelimited" in str(err):
321+
wait_sec = max(delay_ms / 1000, 10.0)
322+
print(f" Rate limited on {form_id}, sleeping {wait_sec}s and retrying...")
323+
time.sleep(wait_sec)
324+
else:
325+
raise
326+
time.sleep(delay_ms / 1000)
249327

250328

251329
def self_check():
@@ -271,7 +349,10 @@ def check(name, condition):
271349
ok = [
272350
{
273351
"lexemeId": "L1",
274-
"modifications": [{"formId": "L1-F2", "add": ["Q65048529"]}],
352+
"modifications": [
353+
{"formId": "L1-F2", "add": ["Q65048529"]},
354+
{"formId": "L1-F3", "representations": {"sr": "кошћу", "sr-latn": "košću"}},
355+
],
275356
}
276357
]
277358
try:
@@ -289,6 +370,8 @@ def check(name, condition):
289370
[{"lexemeId": "L1", "modifications": [{"formId": "L1-F2", "form": "x", "add": ["Q1"]}]}],
290371
[{"lexemeId": "L1", "modifications": [{"formId": "L1-F2"}]}],
291372
[{"lexemeId": "L1", "modifications": [{"formId": "L1-F2", "add": ["bad"]}]}],
373+
[{"lexemeId": "L1", "modifications": [{"formId": "L1-F2", "representations": {"sr": ""}}]}],
374+
[{"lexemeId": "L1", "modifications": [{"formId": "L1-F2", "representations": ["bad"]}]}],
292375
]
293376
rejected = 0
294377
for case in bad_cases:
@@ -316,8 +399,8 @@ def check(name, condition):
316399
parser.add_argument(
317400
"--delay",
318401
type=int,
319-
default=1000,
320-
help="Milliseconds between pushes for rate limiting.",
402+
default=5000,
403+
help="Milliseconds between pushes for rate limiting (default: 5000).",
321404
)
322405
parser.add_argument(
323406
"--self-check",

0 commit comments

Comments
 (0)