Skip to content

[v2] fix(lexer): lex /// lines separately and merge them in the tok… #2255

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
teofr merged 2 commits into from
Nov 2, 2026
Merged

[v2] fix(lexer): lex /// lines separately and merge them in the tok… #2255

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
a1b80e8
[v2] fix(lexer): lex `///` lines separately and merge them in the tok…
teofr Oct 8, 2026
c8e0e5b
Don't include the line break on single line `///` comments
teofr Oct 30, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
144 changes: 24 additions & 120 deletions crates/solidity-v2/inputs/language/src/definition.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -311,46 +311,19 @@ language_v2_macros::compile!(Language(
items = [
Trivia(
name = PragmaWhitespace,
scanner = OneOrMore(Fragment(PragmaSpaceOrTab))
),
Fragment(
name = PragmaSpaceOrTab,
scanner = Choice([Atom(" "), Atom("\t")])
scanner = OneOrMore(Choice([Atom(" "), Atom("\t")]))
),
Trivia(
name = PragmaEndOfLine,
scanner = Fragment(PragmaLineBreak)
),
Fragment(
name = PragmaLineBreak,
scanner = Choice([Atom("\n"), Sequence([Atom("\r"), Optional(Atom("\n"))])])
),
Fragment(
name = PragmaTextAfterSlashes,
// The text of a `//` or `///` line after its leading slashes, so it can't start with `/`
scanner = Sequence([Not(['/', '\r', '\n']), ZeroOrMore(Not(['\r', '\n']))])
scanner = Choice([
Atom("\n"),
Sequence([Atom("\r"), Optional(Atom("\n"))])
])
),
Trivia(
name = PragmaSingleLineComment,
category = RegularComment,
// Match consecutive `//` lines that aren't `///` ones, each line keeps its line break, except for the last line of the file.
scanner = Sequence([
Atom("//"),
Optional(Fragment(PragmaSingleLineRegularCommentText)),
Optional(Sequence([
Fragment(PragmaLineBreak),
ZeroOrMore(Sequence([ZeroOrMore(Fragment(PragmaSpaceOrTab)), Atom("//"), Optional(Fragment(PragmaSingleLineRegularCommentText)), Fragment(PragmaLineBreak)])),
Optional(Sequence([ZeroOrMore(Fragment(PragmaSpaceOrTab)), Atom("//"), Fragment(PragmaSingleLineRegularCommentText)]))
]))
])
),
Fragment(
name = PragmaSingleLineRegularCommentText,
// The text of a `//` line, which can't start with a single `/`, as that would make it a `///` one
scanner = Choice([
Fragment(PragmaTextAfterSlashes),
Sequence([Atom("//"), ZeroOrMore(Not(['\r', '\n']))])
])
scanner = Sequence([Atom("//"), ZeroOrMore(Not(['\r', '\n']))])
),
Trivia(
name = PragmaMultiLineComment,
Expand All @@ -371,15 +344,10 @@ language_v2_macros::compile!(Language(
Trivia(
name = PragmaSingleLineNatSpecComment,
category = NatSpecComment,
// Match consecutive `///` lines, each line keeps its line break, except for the last line of the file.
// `////` is a regular comment
scanner = Sequence([
Atom("///"),
Optional(Fragment(PragmaTextAfterSlashes)),
Optional(Sequence([
Fragment(PragmaLineBreak),
ZeroOrMore(Sequence([ZeroOrMore(Fragment(PragmaSpaceOrTab)), Atom("///"), Optional(Fragment(PragmaTextAfterSlashes)), Fragment(PragmaLineBreak)])),
Optional(Sequence([ZeroOrMore(Fragment(PragmaSpaceOrTab)), Atom("///"), Fragment(PragmaTextAfterSlashes)]))
]))
Optional(Sequence([Not(['/', '\r', '\n']), ZeroOrMore(Not(['\r', '\n']))]))
])
),
Trivia(
Expand Down Expand Up @@ -1756,46 +1724,19 @@ language_v2_macros::compile!(Language(
items = [
Trivia(
name = Whitespace,
scanner = OneOrMore(Fragment(SpaceOrTab))
),
Fragment(
name = SpaceOrTab,
scanner = Choice([Atom(" "), Atom("\t")])
scanner = OneOrMore(Choice([Atom(" "), Atom("\t")]))
),
Trivia(
name = EndOfLine,
scanner = Fragment(LineBreak)
),
Fragment(
name = LineBreak,
scanner = Choice([Atom("\n"), Sequence([Atom("\r"), Optional(Atom("\n"))])])
),
Fragment(
name = TextAfterSlashes,
// The text of a `//` or `///` line after its leading slashes, so it can't start with `/`
scanner = Sequence([Not(['/', '\r', '\n']), ZeroOrMore(Not(['\r', '\n']))])
scanner = Choice([
Atom("\n"),
Sequence([Atom("\r"), Optional(Atom("\n"))])
])
),
Trivia(
name = SingleLineComment,
category = RegularComment,
// Match consecutive `//` lines that aren't `///` ones, each line keeps its line break, except for the last line of the file.
scanner = Sequence([
Atom("//"),
Optional(Fragment(SingleLineRegularCommentText)),
Optional(Sequence([
Fragment(LineBreak),
ZeroOrMore(Sequence([ZeroOrMore(Fragment(SpaceOrTab)), Atom("//"), Optional(Fragment(SingleLineRegularCommentText)), Fragment(LineBreak)])),
Optional(Sequence([ZeroOrMore(Fragment(SpaceOrTab)), Atom("//"), Fragment(SingleLineRegularCommentText)]))
]))
])
),
Fragment(
name = SingleLineRegularCommentText,
// The text of a `//` line, which can't start with a single `/`, as that would make it a `///` one
scanner = Choice([
Fragment(TextAfterSlashes),
Sequence([Atom("//"), ZeroOrMore(Not(['\r', '\n']))])
])
scanner = Sequence([Atom("//"), ZeroOrMore(Not(['\r', '\n']))])
),
Trivia(
name = MultiLineComment,
Expand All @@ -1816,15 +1757,10 @@ language_v2_macros::compile!(Language(
Trivia(
name = SingleLineNatSpecComment,
category = NatSpecComment,
// Match consecutive `///` lines, each line keeps its line break, except for the last line of the file.
// `////` is a regular comment
scanner = Sequence([
Atom("///"),
Optional(Fragment(TextAfterSlashes)),
Optional(Sequence([
Fragment(LineBreak),
ZeroOrMore(Sequence([ZeroOrMore(Fragment(SpaceOrTab)), Atom("///"), Optional(Fragment(TextAfterSlashes)), Fragment(LineBreak)])),
Optional(Sequence([ZeroOrMore(Fragment(SpaceOrTab)), Atom("///"), Fragment(TextAfterSlashes)]))
]))
Optional(Sequence([Not(['/', '\r', '\n']), ZeroOrMore(Not(['\r', '\n']))]))
])
),
Trivia(
Expand Down Expand Up @@ -4809,46 +4745,19 @@ IdentifierPathTailElement: (Period, IdentifierPathElement) = {
items = [
Trivia(
name = YulWhitespace,
scanner = OneOrMore(Fragment(YulSpaceOrTab))
),
Fragment(
name = YulSpaceOrTab,
scanner = Choice([Atom(" "), Atom("\t")])
scanner = OneOrMore(Choice([Atom(" "), Atom("\t")]))
),
Trivia(
name = YulEndOfLine,
scanner = Fragment(YulLineBreak)
),
Fragment(
name = YulLineBreak,
scanner = Choice([Atom("\n"), Sequence([Atom("\r"), Optional(Atom("\n"))])])
),
Fragment(
name = YulTextAfterSlashes,
// The text of a `//` or `///` line after its leading slashes, so it can't start with `/`
scanner = Sequence([Not(['/', '\r', '\n']), ZeroOrMore(Not(['\r', '\n']))])
scanner = Choice([
Atom("\n"),
Sequence([Atom("\r"), Optional(Atom("\n"))])
])
),
Trivia(
name = YulSingleLineComment,
category = RegularComment,
// Match consecutive `//` lines that aren't `///` ones, each line keeps its line break, except for the last line of the file.
scanner = Sequence([
Atom("//"),
Optional(Fragment(YulSingleLineRegularCommentText)),
Optional(Sequence([
Fragment(YulLineBreak),
ZeroOrMore(Sequence([ZeroOrMore(Fragment(YulSpaceOrTab)), Atom("//"), Optional(Fragment(YulSingleLineRegularCommentText)), Fragment(YulLineBreak)])),
Optional(Sequence([ZeroOrMore(Fragment(YulSpaceOrTab)), Atom("//"), Fragment(YulSingleLineRegularCommentText)]))
]))
])
),
Fragment(
name = YulSingleLineRegularCommentText,
// The text of a `//` line, which can't start with a single `/`, as that would make it a `///` one
scanner = Choice([
Fragment(YulTextAfterSlashes),
Sequence([Atom("//"), ZeroOrMore(Not(['\r', '\n']))])
])
scanner = Sequence([Atom("//"), ZeroOrMore(Not(['\r', '\n']))])
),
Trivia(
name = YulMultiLineComment,
Expand All @@ -4869,15 +4778,10 @@ IdentifierPathTailElement: (Period, IdentifierPathElement) = {
Trivia(
name = YulSingleLineNatSpecComment,
category = NatSpecComment,
// Match consecutive `///` lines, each line keeps its line break, except for the last line of the file.
// `////` is a regular comment
scanner = Sequence([
Atom("///"),
Optional(Fragment(YulTextAfterSlashes)),
Optional(Sequence([
Fragment(YulLineBreak),
ZeroOrMore(Sequence([ZeroOrMore(Fragment(YulSpaceOrTab)), Atom("///"), Optional(Fragment(YulTextAfterSlashes)), Fragment(YulLineBreak)])),
Optional(Sequence([ZeroOrMore(Fragment(YulSpaceOrTab)), Atom("///"), Fragment(YulTextAfterSlashes)]))
]))
Optional(Sequence([Not(['/', '\r', '\n']), ZeroOrMore(Not(['\r', '\n']))]))
])
),
Trivia(
Expand Down
2 changes: 1 addition & 1 deletion crates/solidity-v2/outputs/cargo/cst/src/structured_cst/natspec/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,6 @@ use std::ops::Range;
pub struct NatSpec {
/// Range of the comment in the source.
///
/// A `///` comment includes the line break after its last line, unless it ends the source.
/// A `///` comment ends with its last line, without the line break after it.
pub range: Range<usize>,
}
24 changes: 6 additions & 18 deletions crates/solidity-v2/outputs/cargo/ir/src/ir/builder/assembly/memory_safety.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,14 +62,9 @@ impl<S: Source> CstToIrBuilder<'_, S> {
return false;
};

let text = self.source.text(natspec.range.clone());
let range = natspec.range.clone();

// A `///` comment includes the line break after its last line: leave it out of the
// reported range
let range =
natspec.range.start..natspec.range.start + text.trim_end_matches(['\r', '\n']).len();

if is_memory_safe_marker(text) {
if is_memory_safe_marker(self.source.text(range.clone())) {
// We only warn about deprecation from 0.8.31, following solc
if self.language_version >= LanguageVersion::V0_8_31 {
self.report_range(range, DeprecatedAssemblyNatSpec);
Expand All @@ -85,12 +80,6 @@ impl<S: Source> CstToIrBuilder<'_, S> {
/// Whether the text of a `NatSpec` comment marks the assembly statement it documents as memory
/// safe: only a single `/// @solidity memory-safe-assembly` line does.
fn is_memory_safe_marker(comment: &str) -> bool {
// A `///` comment includes the line break after its last line
let comment = comment
.strip_suffix("\r\n")
.or_else(|| comment.strip_suffix(['\n', '\r']))
.unwrap_or(comment);

comment.strip_prefix("///").is_some_and(|text| {
// Strict on purpose: only spaces and tabs may surround the marker. The line
// break check is redundant with the comparison, but spells out the single line rule
Expand All @@ -105,11 +94,10 @@ mod tests {

#[test]
fn marker() {
// Spaces and tabs around the marker don't matter, nor does the line break after it
// Spaces and tabs around the marker don't matter
for comment in [
"/// @solidity memory-safe-assembly",
"///\t @solidity memory-safe-assembly \t\n",
"/// @solidity memory-safe-assembly\r\n",
"///\t @solidity memory-safe-assembly \t",
] {
assert!(is_memory_safe_marker(comment), "{comment:?}");
}
Expand All @@ -118,9 +106,9 @@ mod tests {
#[test]
fn anything_else() {
for comment in [
"///\n",
"///",
"/** @solidity memory-safe-assembly */",
"/// @notice Writes to scratch space\n/// @solidity memory-safe-assembly\n",
"/// @notice Writes to scratch space\n/// @solidity memory-safe-assembly",
"/// @solidity memory-safe-assembly",
// Other whitespace looks the same, but it isn't the expected whitespace
"/// @solidity\u{a0}memory-safe-assembly",
Expand Down
Loading
Loading