diff --git a/.gitignore b/.gitignore index 61ddbe1..817f22b 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,5 @@ composer.phar composer.lock /vendor/ /coverage/ -/var/ \ No newline at end of file +/var/ +.phpunit.result.cache \ No newline at end of file diff --git a/docs/20__Container_Files/10_Syntax.md b/docs/20__Container_Files/10_Syntax.md index 336e49d..ce38da2 100644 --- a/docs/20__Container_Files/10_Syntax.md +++ b/docs/20__Container_Files/10_Syntax.md @@ -45,6 +45,24 @@ Beloved or Hated emojis will also work just fine. :snails: '🐌🐌🐌' ``` +Repeated spaces and tabs are always collapsed to a single space, and this also applies inside a string. This keeps container files tidy regardless of how they're indented, but it means you can't rely on whitespace to format text you store, e.g. a multi-line example embedded in a description. + +``` +:example: "a b" # becomes "a b" +``` + +If you need to preserve formatting exactly as written, use a heredoc block instead. This is an explicit opt-out of the whitespace collapsing described above. + +``` +:example: <<extractHeredocs($code, $rawStrings); + + // there is never a need for tabs or multiple whitespaces // so we remove them before assigning the code - $this->code = trim(preg_replace("/[ \t]+/", ' ', $code) ?? ''); + $code = trim(preg_replace("/[ \t]+/", ' ', $code) ?? ''); + + // restore the raw string bodies now that everything around them has + // already been collapsed, so the placeholders themselves were never + // touched by the collapse above. + $this->code = $rawStrings ? strtr($code, $rawStrings) : $code; // we need to know the codes length $this->length = strlen($this->code); @@ -128,6 +140,41 @@ public function __construct(string $code, ?string $filename = null) } } + /** + * Replace heredoc blocks (<< $rawStrings Filled with placeholder => restored single-quoted string body. + * + * @return string + */ + protected function extractHeredocs(string $code, array &$rawStrings) : string + { + $pattern = '/<<<([A-Za-z_][A-Za-z0-9_]*)\r?\n(.*?)\r?\n\1(?=\r?\n|$)/s'; + + return preg_replace_callback($pattern, function (array $matches) use (&$rawStrings) : string { + $placeholder = "\x00RAW" . count($rawStrings) . "\x00"; + + // the restored body will be re-scanned as a regular single quoted + // string, so any single quote it contains must be escaped to not + // be mistaken for the closing quote. + $rawStrings[$placeholder] = str_replace("'", "\\'", $matches[2]); + + // the placeholder collapses the multi-line heredoc (its opening `<<assertNotFalse($pos, "marker \"$marker\" not found in test source"); + return substr_count(substr($code, 0, $pos), "\n") + 1; + } + + /** + * @param array $tokens + */ + private function findParameterToken(array $tokens, string $name) : ?T + { + foreach ($tokens as $token) { + if ($token->getType() === T::TOKEN_PARAMETER && $token->getValue() === $name) { + return $token; + } + } + return null; + } + + /** + * @param array $tokens + */ + private function findFirstOfType(array $tokens, int $type) : ?T + { + foreach ($tokens as $token) { + if ($token->getType() === $type) { + return $token; + } + } + return null; + } + public function testConstruct() { $lexer = new ContainerLexer('test'); @@ -67,6 +105,175 @@ public function testMultilineStrings() CODE, [T::TOKEN_PARAMETER, T::TOKEN_ASSIGN, T::TOKEN_SPACE, T::TOKEN_STRING]); } + public function testHeredocStrings() + { + // regular multiline strings collapse repeated whitespace / indentation + $string = $this->tokensFromCode(<<<'CODE' + "line one + line two + line three" + CODE)[0]; + $this->assertEquals("line one\n line two\nline three", $string->getValue()); + + // a heredoc block opts out of that collapsing + $string = $this->tokensFromCode(<<<'CODE' + <<assertEquals("line one\n line two\nline three", $string->getValue()); + + // a heredoc is still just a single string token + $this->assertTokenTypes(<<<'CODE' + <<tokensFromCode(<<<'CODE' + <<assertEquals("it's \"fine\".", $string->getValue()); + + // a line that merely starts with the tag does not close the block, + // only an exact match on its own line does + $string = $this->tokensFromCode(<<<'CODE' + <<assertEquals("EOTAG\nline two", $string->getValue()); + + // whitespace outside of a heredoc block is still collapsed as usual + $tokens = $this->tokensFromCode(<<<'CODE' + a <<assertTokenTypesArray($tokens, [ + T::TOKEN_IDENTIFIER, + T::TOKEN_SPACE, + T::TOKEN_STRING, + T::TOKEN_LINE, + T::TOKEN_SPACE, + T::TOKEN_IDENTIFIER, + ]); + $this->assertEquals('raw text', $tokens[2]->getValue()); + } + + public function testHeredocPreservesLineNumbers() + { + // a heredoc spans multiple source lines; tokens after it must keep their + // real line numbers so lexer/parser error messages point at the right line. + // + // line 1: :before: 'a' + // line 2: :doc: <<tokensFromCode(":before: 'a'\n:doc: <<getType() === T::TOKEN_PARAMETER && $token->getValue() === $name) { + return $token; + } + } + return null; + }; + + $this->assertNotNull($findParameter(':before')); + $this->assertNotNull($findParameter(':after')); + + // the parameter before the heredoc is unaffected + $this->assertEquals(1, $findParameter(':before')->getLine()); + + // the parameter after the heredoc must still report its real source line (6), + // i.e. the heredoc extraction must not "eat" the lines it spanned + $this->assertEquals(6, $findParameter(':after')->getLine()); + } + + public function testHeredocLineNumbersAcrossMultipleHeredocs() + { + // two heredocs (reusing the same tag name) in a single file; every token + // after each one must keep accumulating the real line count rather than + // resetting or double-counting across the two placeholder substitutions. + $code = ":before: 'a'\n:x: <<tokensFromCode($code); + + $before = $this->findParameterToken($tokens, ':before'); + $mid = $this->findParameterToken($tokens, ':mid'); + $after = $this->findParameterToken($tokens, ':after'); + + $this->assertNotNull($before); + $this->assertNotNull($mid); + $this->assertNotNull($after); + + $this->assertEquals($this->expectedLineOf($code, ':before'), $before->getLine()); + $this->assertEquals($this->expectedLineOf($code, ':mid'), $mid->getLine()); + $this->assertEquals($this->expectedLineOf($code, ':after'), $after->getLine()); + } + + public function testHeredocLineNumbersWithCrlf() + { + // Windows-style line endings throughout a heredoc block; every "\r\n" + // must still count as exactly one line for line-number tracking. + $code = ":before: 'a'\r\n:doc: <<tokensFromCode($code); + + $before = $this->findParameterToken($tokens, ':before'); + $after = $this->findParameterToken($tokens, ':after'); + + $this->assertNotNull($before); + $this->assertNotNull($after); + + $this->assertEquals($this->expectedLineOf($code, ':before'), $before->getLine()); + $this->assertEquals($this->expectedLineOf($code, ':after'), $after->getLine()); + } + + public function testHeredocWithEmptyBody() + { + // a heredoc whose entire body is a single blank line - the minimal / + // degenerate case for both the extraction regex and the "lost newlines" + // line-number bookkeeping. + $code = ":doc: <<tokensFromCode($code); + $string = $this->findFirstOfType($tokens, T::TOKEN_STRING); + $after = $this->findParameterToken($tokens, ':after'); + + $this->assertNotNull($string); + $this->assertEquals('', $string->getValue()); + + $this->assertNotNull($after); + $this->assertEquals($this->expectedLineOf($code, ':after'), $after->getLine()); + } + + public function testHeredocRequiresClosingTagOnOwnLine() + { + // the closing tag must be preceded by its own newline (i.e. sit alone on + // its own line); a heredoc with no blank body line before an adjacent + // closing tag is therefore not recognised as a heredoc at all, and the + // literal "<" falls through to the normal tokenizer and fails predictably + // rather than silently producing a corrupted token stream. + // + // this is positioned starting on line 2 (not line 1) to avoid a separate, + // pre-existing, unrelated off-by-one in ContainerLexerException's line + // reporting for errors on the very first source line. + $this->expectException(\ClanCats\Container\Exceptions\ContainerLexerException::class); + $this->tokensFromCode(":before: 'a'\n<<assertTokenTypes("-1", [T::TOKEN_NUMBER]); diff --git a/tests/ContainerParser/Parser/ScopeParserTest.php b/tests/ContainerParser/Parser/ScopeParserTest.php index 1190acc..acbb600 100644 --- a/tests/ContainerParser/Parser/ScopeParserTest.php +++ b/tests/ContainerParser/Parser/ScopeParserTest.php @@ -63,12 +63,33 @@ public function testInvalidOverrideKeyword() $this->scopeNodeFromCode('override 42'); // actually i want this in the feature } - public function testUnexpectedToken() + public function testUnexpectedToken() { $this->expectException(\ClanCats\Container\Exceptions\ContainerParserException::class); $this->scopeNodeFromCode(":test: 42\n42"); // actually i want this in the feature } + public function testParserExceptionReportsCorrectLineAfterHeredoc() + { + // the point of the heredoc line-counting fix is that error messages built + // from Token::getLine() stay accurate after a heredoc block; exercise the + // real exception message here, not just the token accessor. + $code = ":doc: <<assertNotFalse($badTokenPos); + $expectedLine = substr_count(substr($code, 0, $badTokenPos), "\n") + 1; + + try { + $this->scopeNodeFromCode($code); + $this->fail('Expected a ContainerParserException to be thrown.'); + } catch (\ClanCats\Container\Exceptions\ContainerParserException $e) { + $this->assertStringContainsString('given at line ' . $expectedLine, $e->getMessage()); + } + } + public function testParseImport() { $scopeNode = $this->scopeNodeFromCode('import foo/bar');