Repository navigation
Expand file tree
/
Copy pathgenerate_rules.py
More file actions
252 lines (222 loc) · 10.3 KB
/
Copy pathgenerate_rules.py
File metadata and controls
252 lines (222 loc) · 10.3 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
"""Render the generated rule files in ``rules/local`` from ``rules/templates``.
A template is a YARA rule file with ``{{name}}`` placeholders. Each name is a fragment in
``rules/templates/fragments.toml``: a word list (an array, joined with ``|``) or a sub-pattern
(a string), either of which may hold placeholders of its own. ``{{lazy name}}`` renders the
fragment with every quantifier made non-greedy, for a string whose other quantifiers are lazy:
YARA-X rejects a pattern that mixes the two. So the separator between words, each word list,
and the documentary exemption's copy of the grammar are each written once.
uv run python scripts/generate_rules.py # rewrite rules/local from the templates
uv run python scripts/generate_rules.py --check # exit 1 when a generated file is stale
``docs/verification/lint_rules.py`` runs the check, so CI fails a hand edit to a generated file
and a template or word-list edit that was not regenerated. A change to a fragment is a rule
change: it reaches every string that names the fragment, and it has to pass the checks in
``docs/research/detection-backlog.md`` like any other.
"""
from __future__ import annotations
import argparse
import re
import tomllib
from pathlib import Path
REPO = Path(__file__).resolve().parents[1]
TEMPLATES = REPO / "rules/templates"
LOCAL = REPO / "rules/local"
FRAGMENTS = "fragments.toml"
SUFFIX = ".yar.tmpl"
MARKER = "// Generated by scripts/generate_rules.py"
_PLACEHOLDER = re.compile(r"\{\{[ \t]*(lazy[ \t]+)?([a-z][a-z0-9_]*)[ \t]*\}\}")
_BOUNDS = re.compile(r"\{[0-9]+,[0-9]*\}")
class TemplateError(ValueError):
"""A template or fragment the generator will not render."""
def lazy(pattern: str) -> str:
"""``pattern`` with every greedy quantifier (``?``, ``*``, ``+``, ``{m,n}``) made lazy.
Escapes, character classes, and a group's flags (``(?i:``) are copied as written. A
quantifier that is already lazy or possessive, and a repetition with no lazy reading such
as ``{3}``, are refused: nothing reviewed uses them, so one is a mistake until decided.
"""
out: list[str] = []
index, size = 0, len(pattern)
while index < size:
char = pattern[index]
if char == "\\":
if index + 1 == size:
raise TemplateError(f"pattern ends inside an escape: {pattern!r}")
out.append(pattern[index : index + 2])
index += 2
continue
if char == "[":
end = _class_end(pattern, index)
out.append(pattern[index:end])
index = end
continue
if pattern.startswith("(?", index):
end = index + 2
while end < size and pattern[end] not in ":)":
end += 1
out.append(pattern[index:end])
index = end
continue
bounds = _BOUNDS.match(pattern, index) if char == "{" else None
if bounds is not None:
token = bounds.group()
elif char in "?*+":
token = char
elif char == "{":
raise TemplateError(f"no lazy form for the repetition at {pattern[index:]!r}")
else:
out.append(char)
index += 1
continue
index += len(token)
if index < size and pattern[index] in "?+":
raise TemplateError(f"{token + pattern[index]!r} is already lazy or possessive")
out.append(token + "?")
return "".join(out)
def _class_end(pattern: str, start: int) -> int:
"""The index after the ``]`` that closes the character class opening at ``start``."""
index = start + 1
if pattern.startswith("^", index):
index += 1
if pattern.startswith("]", index):
index += 1
while index < len(pattern):
if pattern[index] == "\\":
index += 2
elif pattern[index] == "]":
return index + 1
else:
index += 1
raise TemplateError(f"unclosed character class in {pattern[start:]!r}")
class Fragments:
"""The named word lists and sub-patterns of one ``fragments.toml``."""
def __init__(self, path: Path) -> None:
self.path = path
self.used: set[str] = set()
self._text: dict[str, str] = {}
try:
raw = tomllib.loads(path.read_bytes().decode("utf-8"))
except (OSError, UnicodeDecodeError, tomllib.TOMLDecodeError) as exc:
raise TemplateError(f"{path.name}: {exc}") from exc
for name, value in raw.items():
if isinstance(value, str):
self._text[name] = value
elif isinstance(value, list) and value and all(isinstance(e, str) for e in value):
if "" in value:
raise TemplateError(f"{path.name}: {name} has an empty entry")
self._text[name] = "|".join(value)
else:
raise TemplateError(f"{path.name}: {name} is not a string or a list of strings")
def expand(self, text: str, source: str, chain: tuple[str, ...] = ()) -> str:
"""``text`` with every placeholder replaced, fragments inside fragments included."""
def replace(match: re.Match[str]) -> str:
name = match.group(2)
if name not in self._text:
raise TemplateError(f"{source}: no fragment named {name}")
if name in chain:
loop = " > ".join((*chain, name))
raise TemplateError(f"{self.path.name}: circular reference {loop}")
self.used.add(name)
body = self.expand(self._text[name], f"{self.path.name}: {name}", (*chain, name))
if not match.group(1):
return body
try:
return lazy(body)
except TemplateError as exc:
raise TemplateError(f"{source}: {match.group()}: {exc}") from exc
result = _PLACEHOLDER.sub(replace, text)
if "{{" in result or "}}" in result:
raise TemplateError(f"{source}: a placeholder is not written as {{{{name}}}}")
return result
def unused(self) -> list[str]:
return sorted(set(self._text) - self.used)
def render_all(templates: Path = TEMPLATES) -> dict[str, bytes]:
"""Each generated file's name and exact bytes, from the templates under ``templates``."""
fragments = Fragments(templates / FRAGMENTS)
rendered: dict[str, bytes] = {}
for template in sorted(templates.glob(f"*{SUFFIX}")):
name = template.name.removesuffix(".tmpl")
try:
text = template.read_bytes().decode("utf-8")
except UnicodeDecodeError as exc:
raise TemplateError(f"{template.name}: {exc}") from exc
header = (
f"{MARKER}; lint_rules.py fails a hand edit. Edit\n"
f"// rules/templates/{template.name} or\n"
f"// rules/templates/{FRAGMENTS}, then run the script.\n"
)
rendered[name] = (header + fragments.expand(text, template.name)).encode("utf-8")
if not rendered:
raise TemplateError(f"no {SUFFIX} template under {templates}")
if unused := fragments.unused():
raise TemplateError(f"{FRAGMENTS}: no template uses {', '.join(unused)}")
return rendered
def stale(templates: Path = TEMPLATES, local: Path = LOCAL) -> list[str]:
"""One line for each generated file that is missing, hand-edited, or left without a template."""
self_check()
rendered = render_all(templates)
problems: list[str] = []
for name, expected in rendered.items():
target = local / name
if not target.is_file():
problems.append(f"{name}: missing; run scripts/generate_rules.py")
continue
actual = target.read_bytes()
if actual != expected:
pairs = zip(actual.split(b"\n"), expected.split(b"\n"), strict=False)
line = next((n for n, (a, b) in enumerate(pairs, 1) if a != b), 0)
where = f"line {line}" if line else "its length"
problems.append(
f"{name}: {where} differs from what rules/templates renders; edit the "
"template or fragments.toml and run scripts/generate_rules.py"
)
for path in sorted(local.iterdir()):
if path.suffix in {".yar", ".yara"} and path.name not in rendered:
if path.read_bytes().startswith(MARKER.encode()):
problems.append(f"{path.name}: marked as generated but has no template")
return problems
def self_check() -> None:
"""Fail unless ``lazy`` gives the reviewed spellings and refuses what it cannot decide."""
spellings = {
r"([ \t]+|[ \t]*\r?\n[ \t]*)": r"([ \t]+?|[ \t]*?\r??\n[ \t]*?)",
r"(?i:you are now)\b[\s\S]{0,96}\bDAN\b": r"(?i:you are now)\b[\s\S]{0,96}?\bDAN\b",
r"a\?b[?*+{0,2}]c\+": r"a\?b[?*+{0,2}]c\+",
r"x(y|z{2,})?": r"x(y|z{2,}?)??",
}
for pattern, expected in spellings.items():
if (found := lazy(pattern)) != expected:
raise TemplateError(f"lazy({pattern!r}) gave {found!r}, not {expected!r}")
for refused in (r"a+?", r"a*+", r"a{3}", r"[ab", "a\\"):
try:
lazy(refused)
except TemplateError:
continue
raise TemplateError(f"lazy({refused!r}) was not refused")
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
parser.add_argument(
"--check", action="store_true", help="report stale generated files; write nothing"
)
parser.add_argument("--templates", type=Path, default=TEMPLATES, help="templates directory")
parser.add_argument("--output", type=Path, default=LOCAL, help="directory to write or check")
args = parser.parse_args()
try:
if args.check:
problems = stale(args.templates, args.output)
for problem in problems:
print(problem)
if not problems:
print(f"generated rule files are current ({len(render_all(args.templates))})")
return 1 if problems else 0
self_check()
rendered = render_all(args.templates)
except TemplateError as exc:
print(f"refusing: {exc}")
return 1
for name, data in rendered.items():
target = args.output / name
changed = not target.is_file() or target.read_bytes() != data
if changed:
target.write_bytes(data)
print(f"{'wrote' if changed else 'unchanged'} {target}")
return 0
if __name__ == "__main__":
raise SystemExit(main())