Repository navigation
Expand file tree
/
Copy pathtest_interaction_check.py
More file actions
317 lines (274 loc) · 16.6 KB
/
Copy pathtest_interaction_check.py
File metadata and controls
317 lines (274 loc) · 16.6 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
import json
import contextlib
import io
import os
from pathlib import Path
import subprocess
import sys
import tempfile
import unittest
from unittest import mock
import xml.etree.ElementTree as ET
from interaction_check import (CheckError, LIMIT, MAX_DEPTH, differences, replay, strict_json,
validate, main, write_report)
def scenario():
return {"schema": 1, "name": "minimal", "steps": [
{"event": {"text": "Україна e\u0301 👩💻"}, "expect": {"text": "Україна e\u0301 👩💻"}}
]}
ECHO = '''import json, sys
for line in sys.stdin:
r = json.loads(line)
assert "expect" not in r
print(json.dumps({"protocol": 1, "id": r["id"], "state": r["event"]}))
'''
class Tests(unittest.TestCase):
def adapter(self, code):
return [sys.executable, "-X", "utf8", "-c", code]
def test_real_subprocess_and_unicode_roundtrip(self):
self.assertTrue(replay(scenario(), self.adapter(ECHO))["passed"])
def test_unicode_line_separators_inside_json_are_not_record_boundaries(self):
case = scenario()
for separator in ["\u0085", "\u2028", "\u2029"]:
value = f"first{separator}second"
case["steps"][0] = {"event": {"text": value}, "expect": {"text": value}}
raw_unicode_echo = ECHO.replace('r["event"]})', 'r["event"]}, ensure_ascii=False)')
with self.subTest(separator=repr(separator)):
self.assertTrue(replay(case, self.adapter(raw_unicode_echo))["passed"])
def test_jsonl_accepts_lf_crlf_and_optional_final_newline(self):
line = json.dumps({"protocol": 1, "id": 1, "state": scenario()["steps"][0]["event"]})
for ending in ["", "\n", "\r\n"]:
with self.subTest(ending=repr(ending)):
code = f"import sys; sys.stdout.buffer.write({(line + ending).encode()!r})"
self.assertTrue(replay(scenario(), self.adapter(code))["passed"])
with self.assertRaisesRegex(CheckError, "Expected 1 replies, got 2"):
replay(scenario(), self.adapter(ECHO + "print()"))
def test_partial_nested_objects_and_exact_array_length_and_order(self):
self.assertEqual(differences({"a": {"b": 1}}, {"a": {"b": 1, "c": 2}}), [])
self.assertEqual(differences([{"b": 1}], [{"b": 1, "c": 2}]), [])
self.assertTrue(differences([1, 2], [2, 1]))
self.assertTrue(differences([1], [1, 2]))
self.assertTrue(differences([True], [1]))
def test_missing_null_is_not_present_null(self):
self.assertTrue(differences({"a": None}, {}))
self.assertEqual(differences({"a": None}, {"a": None}), [])
def test_bool_is_not_number(self):
self.assertTrue(differences(True, 1))
self.assertTrue(differences(0, False))
def test_failure_points_to_step_and_field(self):
case = scenario()
case["steps"][0]["expect"]["text"] = "wrong"
with self.assertRaisesRegex(CheckError, r"Step 1 .*state.text"):
replay(case, self.adapter(ECHO))
def test_exit_failure_is_not_hidden_by_matching_output(self):
with self.assertRaisesRegex(CheckError, "exited with 7"):
replay(scenario(), self.adapter(ECHO + "raise SystemExit(7)"))
def test_timeout_terminates_stalled_adapter(self):
with self.assertRaisesRegex(CheckError, "timed out"):
replay(scenario(), self.adapter("import time; time.sleep(5)"), 0.05)
def test_output_limit_stops_adapter_before_timeout(self):
for stream in ["stdout", "stderr"]:
code = (f"import sys, time; sys.{stream}.buffer.write(b'x' * {LIMIT + 1}); "
f"sys.{stream}.buffer.flush(); time.sleep(10)")
with self.subTest(stream=stream), self.assertRaisesRegex(CheckError, "output exceeds"):
replay(scenario(), self.adapter(code), timeout=2)
def test_output_limit_is_checked_after_fast_exit_too(self):
for stream in ["stdout", "stderr"]:
code = f"import sys; sys.{stream}.buffer.write(b'x' * {LIMIT + 1})"
with self.subTest(stream=stream), self.assertRaisesRegex(CheckError, "output exceeds"):
replay(scenario(), self.adapter(code))
def test_output_at_limit_is_accepted(self):
for stream in ["stdout", "stderr"]:
if stream == "stdout":
code = ("import json, sys; r = json.loads(sys.stdin.readline()); "
"data = json.dumps({'protocol': 1, 'id': r['id'], 'state': r['event']}).encode(); "
f"sys.stdout.buffer.write(data.ljust({LIMIT - 1}, b' ') + b'\\n')")
else:
code = ECHO + f"sys.stderr.write('x' * {LIMIT})"
with self.subTest(stream=stream):
result = replay(scenario(), self.adapter(code))
self.assertTrue(result["passed"])
if stream == "stderr":
self.assertEqual(len(result["stderr"]), LIMIT)
def test_large_input_does_not_block_timeout_if_adapter_never_reads(self):
case = scenario()
case["steps"][0]["event"]["padding"] = "x" * (LIMIT // 2)
with self.assertRaisesRegex(CheckError, "timed out"):
replay(case, self.adapter("import time; time.sleep(10)"), timeout=0.1)
def test_timeout_keeps_partial_diagnostic_output(self):
code = "import sys, time; print('before stall', file=sys.stderr, flush=True); time.sleep(10)"
with self.assertRaisesRegex(CheckError, "timed out") as error:
replay(scenario(), self.adapter(code), timeout=2)
self.assertIn("before stall", error.exception.stderr)
def test_failed_assertion_and_protocol_keep_adapter_diagnostics(self):
for code in [ECHO, "print('not json')"]:
case = scenario()
case["steps"][0]["expect"]["text"] = "wrong"
code = "import sys; print('useful diagnostic', file=sys.stderr)\n" + code
with self.subTest(code=code), self.assertRaises(CheckError) as error:
replay(case, self.adapter(code))
self.assertIn("useful diagnostic", getattr(error.exception, "stderr", ""))
def test_failed_reports_include_adapter_stderr(self):
with tempfile.TemporaryDirectory() as root:
root = Path(root)
case, report, junit = root / "case.json", root / "report.json", root / "junit.xml"
value = scenario()
value["steps"][0]["expect"]["text"] = "wrong"
case.write_text(json.dumps(value), encoding="utf-8")
code = "import sys; print('failure details', file=sys.stderr)\n" + ECHO
with contextlib.redirect_stdout(io.StringIO()):
status = main(["--report", str(report), "--junit", str(junit),
str(case), "--", *self.adapter(code)])
self.assertEqual(status, 1)
result = json.loads(report.read_text(encoding="utf-8"))["results"][0]
self.assertIn("failure details", result.get("stderr", ""))
self.assertIn("failure details", ET.parse(junit).find(".//system-err").text)
def test_wrong_id_is_rejected(self):
with self.assertRaisesRegex(CheckError, "mismatched reply id"):
replay(scenario(), self.adapter(ECHO.replace('r["id"]', '999')))
def test_invalid_and_missing_output(self):
for code in ["print('not json')", "pass", "print('{}\\n{}')"]:
with self.subTest(code=code), self.assertRaises(CheckError):
replay(scenario(), self.adapter(code))
def test_non_finite_json_is_rejected(self):
for text in ['NaN', '{"v": Infinity}', '[-Infinity]', '{"v": 1e999}']:
with self.subTest(text=text), self.assertRaises(CheckError):
strict_json(text)
def test_duplicate_json_keys_are_rejected(self):
with self.assertRaisesRegex(CheckError, "Duplicate JSON key"):
strict_json('{"schema": 1, "schema": 2}')
def test_json_nesting_limit_and_parser_recursion_fail_cleanly(self):
strict_json("[" * MAX_DEPTH + "0" + "]" * MAX_DEPTH)
for depth in [MAX_DEPTH + 1, 20000]:
with self.subTest(depth=depth), self.assertRaises(CheckError):
strict_json("[" * depth + "0" + "]" * depth)
def test_deep_adapter_response_identifies_step(self):
response = '{"protocol":1,"id":1,"state":{"deep":' + "[" * MAX_DEPTH + "0" + "]" * MAX_DEPTH + '}}'
with self.assertRaisesRegex(CheckError, r"Step 1: JSON nesting"):
replay(scenario(), self.adapter(f"print({response!r})"))
def test_python_api_rejects_cycles_and_non_json_values(self):
cyclic = []
cyclic.append(cyclic)
for value in [cyclic, float("nan"), (1, 2), {1: "not a string key"}, "\ud800"]:
case = scenario()
case["steps"][0]["event"]["extra"] = value
with self.subTest(value_type=type(value).__name__), self.assertRaises(CheckError):
replay(case, self.adapter(ECHO))
def test_unpaired_surrogates_are_rejected_in_keys_and_values(self):
for value in [r'"\ud800"', r'{"\udfff": 1}']:
with self.subTest(value=value), self.assertRaisesRegex(CheckError, "Unicode"):
strict_json(value)
def test_invalid_duplicate_key_still_produces_a_cli_failure_report(self):
with tempfile.TemporaryDirectory() as root:
case, report = Path(root) / "case.json", Path(root) / "report.json"
case.write_text(r'{"\ud800": 1, "\ud800": 2}', encoding="utf-8")
completed = subprocess.run(
[sys.executable, "-X", "utf8", str(Path(__file__).with_name("interaction_check.py")),
"--report", str(report), str(case), "--", *self.adapter(ECHO)],
capture_output=True, text=True, encoding="utf-8", timeout=10, check=False)
self.assertEqual(completed.returncode, 1)
self.assertNotIn("Traceback", completed.stderr)
result = json.loads(report.read_text(encoding="utf-8"))["results"][0]
self.assertFalse(result["passed"])
self.assertIn("Duplicate JSON key", result["error"])
def test_empty_or_ambiguous_scenarios_rejected(self):
cases = [dict(scenario(), schema=True), dict(scenario(), steps=[]),
dict(scenario(), steps=[{"event": {}, "expect": {}}]),
dict(scenario(), steps=[{"event": {}, "expect": {"a": 1}, "exepct": {}}])]
for case in cases:
with self.subTest(case=case), self.assertRaises(CheckError):
validate(case)
def test_non_finite_timeout(self):
for timeout in [0, -1, float("inf"), float("nan")]:
with self.subTest(timeout=timeout), self.assertRaises(CheckError):
replay(scenario(), self.adapter(ECHO), timeout)
def test_report_and_failure_exit_status(self):
with tempfile.TemporaryDirectory() as root:
case = Path(root) / "scenario.json"
report = Path(root) / "report.json"
case.write_text(json.dumps(scenario()), encoding="utf-8")
with contextlib.redirect_stdout(io.StringIO()):
status = main(["--report", str(report), str(case), "--", *self.adapter("pass")])
self.assertEqual(status, 1)
self.assertFalse(json.loads(report.read_text())["results"][0]["passed"])
def test_json_and_junit_reports_keep_names_sources_and_failures(self):
with tempfile.TemporaryDirectory() as root:
root = Path(root)
cases = [root / name for name in ["pass.json", "fail.json", "broken.json"]]
first = dict(scenario(), name='Україна <preview> & "reset"')
second = dict(scenario(), name="wrong expectation")
second["steps"][0]["expect"]["text"] = "wrong"
cases[0].write_text(json.dumps(first), encoding="utf-8")
cases[1].write_text(json.dumps(second), encoding="utf-8")
cases[2].write_text("{broken", encoding="utf-8")
report, junit = root / "report.json", root / "junit.xml"
code = "import sys; sys.stderr.write('diagnostic\\x01 <&>')\n" + ECHO
with contextlib.redirect_stdout(io.StringIO()):
status = main(["--report", str(report), "--junit", str(junit),
*map(str, cases), "--", *self.adapter(code)])
self.assertEqual(status, 1)
results = json.loads(report.read_text(encoding="utf-8"))["results"]
self.assertEqual([r["name"] for r in results], [first["name"], second["name"], "broken.json"])
self.assertEqual([r["source"] for r in results], list(map(str, cases)))
self.assertEqual([r["passed"] for r in results], [True, False, False])
suite = ET.parse(junit).getroot()
self.assertEqual(suite.attrib["tests"], "3")
self.assertEqual(suite.attrib["failures"], "2")
tests = suite.findall("testcase")
self.assertEqual(tests[0].attrib["name"], first["name"])
self.assertEqual(tests[0].find("system-err").text, "diagnostic\ufffd <&>")
self.assertIsNone(tests[0].find("failure"))
self.assertIn("state.text", tests[1].find("failure").text)
self.assertIsNotNone(tests[2].find("failure"))
def test_successful_junit_report_and_replacing_existing_report(self):
with tempfile.TemporaryDirectory() as root:
case, report = Path(root) / "case.json", Path(root) / "report.xml"
case.write_text(json.dumps(scenario()), encoding="utf-8")
report.write_text("old report", encoding="utf-8")
with contextlib.redirect_stdout(io.StringIO()):
status = main(["--junit", str(report), str(case), "--", *self.adapter(ECHO)])
self.assertEqual(status, 0)
self.assertEqual(ET.parse(report).getroot().attrib["failures"], "0")
self.assertEqual(set(Path(root).iterdir()), {case, report})
def test_report_write_errors_fail_without_tracebacks_and_other_report_is_written(self):
with tempfile.TemporaryDirectory() as root:
case, good_report = Path(root) / "case.json", Path(root) / "junit.xml"
case.write_text(json.dumps(scenario()), encoding="utf-8")
for bad_report in [Path(root), Path(root) / "missing" / "report.json"]:
stderr = io.StringIO()
with contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(stderr):
status = main(["--report", str(bad_report), "--junit", str(good_report),
str(case), "--", *self.adapter(ECHO)])
self.assertEqual(status, 1)
self.assertIn("ERROR writing report", stderr.getvalue())
self.assertNotIn("Traceback", stderr.getvalue())
self.assertEqual(ET.parse(good_report).getroot().attrib["failures"], "0")
self.assertEqual(set(Path(root).iterdir()), {case, good_report})
def test_report_path_aliases_are_rejected_before_running_adapter(self):
with tempfile.TemporaryDirectory() as root:
root = Path(root)
case, hardlink, report = root / "case.json", root / "hardlink.json", root / "out.json"
original = json.dumps(scenario())
case.write_text(original, encoding="utf-8")
os.link(case, hardlink)
for options in [["--report", str(case)], ["--junit", str(hardlink)],
["--report", str(report), "--junit", str(report)]]:
with self.subTest(options=options), mock.patch("interaction_check.subprocess.Popen") as run:
with contextlib.redirect_stderr(io.StringIO()), self.assertRaises(SystemExit) as error:
main([*options, str(case), "--", *self.adapter(ECHO)])
self.assertEqual(error.exception.code, 2)
run.assert_not_called()
self.assertEqual(case.read_text(encoding="utf-8"), original)
def test_failed_atomic_replace_preserves_old_report_and_removes_temporary_file(self):
with tempfile.TemporaryDirectory() as root:
report = Path(root) / "report.json"
report.write_text("old", encoding="utf-8")
with mock.patch("interaction_check.os.replace", side_effect=OSError("disk error")):
with self.assertRaises(OSError):
write_report(report, "new")
self.assertEqual(report.read_text(encoding="utf-8"), "old")
self.assertEqual(list(Path(root).iterdir()), [report])
def test_stdout_protocol_is_separate_from_stderr(self):
result = replay(scenario(), self.adapter("import sys; print('diagnostic', file=sys.stderr)\n" + ECHO))
self.assertIn("diagnostic", result["stderr"])
if __name__ == "__main__":
unittest.main()