Add heredoc like string support for preserving exact formatting

.gitignore

@@ -2,4 +2,5 @@ composer.phar
 composer.lock
 /vendor/
 /coverage/
-/var/
+/var/
+.phpunit.result.cache

docs/20__Container_Files/10_Syntax.md

@@ -45,6 +45,24 @@ Beloved or Hated emojis will also work just fine.
 :snails: '🐌🐌🐌'
 ```
 
+Repeated spaces and tabs are always collapsed to a single space, and this also applies inside a string. This keeps container files tidy regardless of how they're indented, but it means you can't rely on whitespace to format text you store, e.g. a multi-line example embedded in a description.
+
+```
+:example: "a  b" # becomes "a b"
+```
+
+If you need to preserve formatting exactly as written, use a heredoc block instead. This is an explicit opt-out of the whitespace collapsing described above.
+
+```
+:example: <<<EOT
+    {
+        "foo": "bar"
+    }
+EOT
+```
+
+A heredoc starts with `<<<` followed by a tag name, and ends with a line containing only the matching tag. The tag can be any combination of letters, digits and underscores, and everything in between is kept exactly as written, including indentation and repeated spaces. Unlike PHP's heredoc syntax there is no automatic indentation stripping, the closing tag must be flush against the start of its line.
+
 ### Booleans and Null
 
 There is not much to say about them:

src/ContainerParser/ContainerLexer.php

@@ -115,9 +115,21 @@ class ContainerLexer
      */
     public function __construct(string $code, ?string $filename = null)
     {
-        // there is never a need for tabs or multiple whitespaces 
+        // heredoc blocks (<<<TAG ... TAG) are an explicit opt-out of the
+        // whitespace collapsing below. Extract them first so their content
+        // survives exactly as written, e.g. for a formatted multi-line
+        // example embedded in a hint string.
+        $rawStrings = [];
+        $code = $this->extractHeredocs($code, $rawStrings);
+
+        // there is never a need for tabs or multiple whitespaces
         // so we remove them before assigning the code
-        $this->code = trim(preg_replace("/[ \t]+/", ' ', $code) ?? '');
+        $code = trim(preg_replace("/[ \t]+/", ' ', $code) ?? '');
+
+        // restore the raw string bodies now that everything around them has
+        // already been collapsed, so the placeholders themselves were never
+        // touched by the collapse above.
+        $this->code = $rawStrings ? strtr($code, $rawStrings) : $code;
 
         // we need to know the codes length
         $this->length = strlen($this->code);
@@ -128,6 +140,41 @@ public function __construct(string $code, ?string $filename = null)
         }
     }
 
+    /**
+     * Replace heredoc blocks (<<<TAG ... TAG) with placeholders so their
+     * exact formatting survives the whitespace collapse in the constructor.
+     *
+     * The closing tag must be alone on its own line, there is no automatic
+     * indentation stripping like PHP's flexible heredoc syntax.
+     *
+     * @param string                 $code
+     * @param array<string, string>  $rawStrings Filled with placeholder => restored single-quoted string body.
+     *
+     * @return string
+     */
+    protected function extractHeredocs(string $code, array &$rawStrings) : string
+    {
+        $pattern = '/<<<([A-Za-z_][A-Za-z0-9_]*)\r?\n(.*?)\r?\n\1(?=\r?\n|$)/s';
+
+        return preg_replace_callback($pattern, function (array $matches) use (&$rawStrings) : string {
+            $placeholder = "\x00RAW" . count($rawStrings) . "\x00";
+
+            // the restored body will be re-scanned as a regular single quoted
+            // string, so any single quote it contains must be escaped to not
+            // be mistaken for the closing quote.
+            $rawStrings[$placeholder] = str_replace("'", "\\'", $matches[2]);
+
+            // the placeholder collapses the multi-line heredoc (its opening `<<<TAG`
+            // line and closing tag line) onto fewer lines than it occupied in the
+            // source. re-emit the "lost" newlines after the string so line numbers of
+            // the following tokens - and therefore lexer/parser error messages - stay
+            // accurate.
+            $lostNewlines = substr_count($matches[0], "\n") - substr_count($matches[2], "\n");
+
+            return "'" . $placeholder . "'" . str_repeat("\n", $lostNewlines);
+        }, $code) ?? $code;
+    }
+
     /**
      * Get the current code
      *

tests/ContainerParser/ContainerLexerTest.php

@@ -10,6 +10,44 @@
 
 class ContainerLexerTest extends LexerTestCase
 {
+    /**
+     * Computes the 1-indexed source line on which the given marker text starts.
+     * Used so expected line numbers are derived programmatically instead of by
+     * hand-counting, which is error prone for anything but the shortest snippets.
+     */
+    private function expectedLineOf(string $code, string $marker) : int
+    {
+        $pos = strpos($code, $marker);
+        $this->assertNotFalse($pos, "marker \"$marker\" not found in test source");
+        return substr_count(substr($code, 0, $pos), "\n") + 1;
+    }
+
+    /**
+     * @param array<T> $tokens
+     */
+    private function findParameterToken(array $tokens, string $name) : ?T
+    {
+        foreach ($tokens as $token) {
+            if ($token->getType() === T::TOKEN_PARAMETER && $token->getValue() === $name) {
+                return $token;
+            }
+        }
+        return null;
+    }
+
+    /**
+     * @param array<T> $tokens
+     */
+    private function findFirstOfType(array $tokens, int $type) : ?T
+    {
+        foreach ($tokens as $token) {
+            if ($token->getType() === $type) {
+                return $token;
+            }
+        }
+        return null;
+    }
+
     public function testConstruct()
     {
         $lexer = new ContainerLexer('test');
@@ -67,6 +105,175 @@ public function testMultilineStrings()
         CODE, [T::TOKEN_PARAMETER, T::TOKEN_ASSIGN, T::TOKEN_SPACE, T::TOKEN_STRING]);
     }
 
+    public function testHeredocStrings()
+    {
+        // regular multiline strings collapse repeated whitespace / indentation
+        $string = $this->tokensFromCode(<<<'CODE'
+        "line one
+            line two
+        line three"
+        CODE)[0];
+        $this->assertEquals("line one\n line two\nline three", $string->getValue());
+
+        // a heredoc block opts out of that collapsing
+        $string = $this->tokensFromCode(<<<'CODE'
+        <<<EOT
+        line one
+            line two
+        line three
+        EOT
+        CODE)[0];
+        $this->assertEquals("line one\n    line two\nline three", $string->getValue());
+
+        // a heredoc is still just a single string token
+        $this->assertTokenTypes(<<<'CODE'
+        <<<EOT
+        hello
+        EOT
+        CODE, [T::TOKEN_STRING]);
+
+        // single quotes inside a heredoc don't terminate it early
+        $string = $this->tokensFromCode(<<<'CODE'
+        <<<EOT
+        it's "fine".
+        EOT
+        CODE)[0];
+        $this->assertEquals("it's \"fine\".", $string->getValue());
+
+        // a line that merely starts with the tag does not close the block,
+        // only an exact match on its own line does
+        $string = $this->tokensFromCode(<<<'CODE'
+        <<<EOT
+        EOTAG
+        line two
+        EOT
+        CODE)[0];
+        $this->assertEquals("EOTAG\nline two", $string->getValue());
+
+        // whitespace outside of a heredoc block is still collapsed as usual
+        $tokens = $this->tokensFromCode(<<<'CODE'
+        a    <<<EOT
+        raw   text
+        EOT
+           b
+        CODE);
+        $this->assertTokenTypesArray($tokens, [
+            T::TOKEN_IDENTIFIER,
+            T::TOKEN_SPACE,
+            T::TOKEN_STRING,
+            T::TOKEN_LINE,
+            T::TOKEN_SPACE,
+            T::TOKEN_IDENTIFIER,
+        ]);
+        $this->assertEquals('raw   text', $tokens[2]->getValue());
+    }
+
+    public function testHeredocPreservesLineNumbers()
+    {
+        // a heredoc spans multiple source lines; tokens after it must keep their
+        // real line numbers so lexer/parser error messages point at the right line.
+        //
+        //   line 1: :before: 'a'
+        //   line 2: :doc: <<<EOT
+        //   line 3: hello
+        //   line 4: world
+        //   line 5: EOT
+        //   line 6: :after: 'b'
+        $tokens = $this->tokensFromCode(":before: 'a'\n:doc: <<<EOT\nhello\nworld\nEOT\n:after: 'b'");
+
+        $findParameter = function (string $name) use ($tokens) {
+            foreach ($tokens as $token) {
+                if ($token->getType() === T::TOKEN_PARAMETER && $token->getValue() === $name) {
+                    return $token;
+                }
+            }
+            return null;
+        };
+
+        $this->assertNotNull($findParameter(':before'));
+        $this->assertNotNull($findParameter(':after'));
+
+        // the parameter before the heredoc is unaffected
+        $this->assertEquals(1, $findParameter(':before')->getLine());
+
+        // the parameter after the heredoc must still report its real source line (6),
+        // i.e. the heredoc extraction must not "eat" the lines it spanned
+        $this->assertEquals(6, $findParameter(':after')->getLine());
+    }
+
+    public function testHeredocLineNumbersAcrossMultipleHeredocs()
+    {
+        // two heredocs (reusing the same tag name) in a single file; every token
+        // after each one must keep accumulating the real line count rather than
+        // resetting or double-counting across the two placeholder substitutions.
+        $code = ":before: 'a'\n:x: <<<EOT\none\nEOT\n:mid: 'm'\n:y: <<<EOT\nAAA\nBBB\nEOT\n:after: 'z'";
+
+        $tokens = $this->tokensFromCode($code);
+
+        $before = $this->findParameterToken($tokens, ':before');
+        $mid = $this->findParameterToken($tokens, ':mid');
+        $after = $this->findParameterToken($tokens, ':after');
+
+        $this->assertNotNull($before);
+        $this->assertNotNull($mid);
+        $this->assertNotNull($after);
+
+        $this->assertEquals($this->expectedLineOf($code, ':before'), $before->getLine());
+        $this->assertEquals($this->expectedLineOf($code, ':mid'), $mid->getLine());
+        $this->assertEquals($this->expectedLineOf($code, ':after'), $after->getLine());
+    }
+
+    public function testHeredocLineNumbersWithCrlf()
+    {
+        // Windows-style line endings throughout a heredoc block; every "\r\n"
+        // must still count as exactly one line for line-number tracking.
+        $code = ":before: 'a'\r\n:doc: <<<EOT\r\nhello\r\nworld\r\nEOT\r\n:after: 'b'";
+
+        $tokens = $this->tokensFromCode($code);
+
+        $before = $this->findParameterToken($tokens, ':before');
+        $after = $this->findParameterToken($tokens, ':after');
+
+        $this->assertNotNull($before);
+        $this->assertNotNull($after);
+
+        $this->assertEquals($this->expectedLineOf($code, ':before'), $before->getLine());
+        $this->assertEquals($this->expectedLineOf($code, ':after'), $after->getLine());
+    }
+
+    public function testHeredocWithEmptyBody()
+    {
+        // a heredoc whose entire body is a single blank line - the minimal /
+        // degenerate case for both the extraction regex and the "lost newlines"
+        // line-number bookkeeping.
+        $code = ":doc: <<<EOT\n\nEOT\n:after: 'b'";
+
+        $tokens = $this->tokensFromCode($code);
+        $string = $this->findFirstOfType($tokens, T::TOKEN_STRING);
+        $after = $this->findParameterToken($tokens, ':after');
+
+        $this->assertNotNull($string);
+        $this->assertEquals('', $string->getValue());
+
+        $this->assertNotNull($after);
+        $this->assertEquals($this->expectedLineOf($code, ':after'), $after->getLine());
+    }
+
+    public function testHeredocRequiresClosingTagOnOwnLine()
+    {
+        // the closing tag must be preceded by its own newline (i.e. sit alone on
+        // its own line); a heredoc with no blank body line before an adjacent
+        // closing tag is therefore not recognised as a heredoc at all, and the
+        // literal "<" falls through to the normal tokenizer and fails predictably
+        // rather than silently producing a corrupted token stream.
+        //
+        // this is positioned starting on line 2 (not line 1) to avoid a separate,
+        // pre-existing, unrelated off-by-one in ContainerLexerException's line
+        // reporting for errors on the very first source line.
+        $this->expectException(\ClanCats\Container\Exceptions\ContainerLexerException::class);
+        $this->tokensFromCode(":before: 'a'\n<<<EOT\nEOT");
+    }
+
     public function testScalarNumber()
     {
         $this->assertTokenTypes("-1", [T::TOKEN_NUMBER]);

tests/ContainerParser/Parser/ScopeParserTest.php

@@ -63,12 +63,33 @@ public function testInvalidOverrideKeyword()
         $this->scopeNodeFromCode('override 42'); // actually i want this in the feature
     }
 
-     public function testUnexpectedToken() 
+     public function testUnexpectedToken()
     {
         $this->expectException(\ClanCats\Container\Exceptions\ContainerParserException::class);
         $this->scopeNodeFromCode(":test: 42\n42"); // actually i want this in the feature
     }
 
+    public function testParserExceptionReportsCorrectLineAfterHeredoc()
+    {
+        // the point of the heredoc line-counting fix is that error messages built
+        // from Token::getLine() stay accurate after a heredoc block; exercise the
+        // real exception message here, not just the token accessor.
+        $code = ":doc: <<<EOT\nhello\nworld\nEOT\n:test: 42\n42";
+
+        // the offending token is the trailing, standalone "42" - i.e. the *last*
+        // occurrence of "42" in the source (the first one is part of ":test: 42").
+        $badTokenPos = strrpos($code, '42');
+        $this->assertNotFalse($badTokenPos);
+        $expectedLine = substr_count(substr($code, 0, $badTokenPos), "\n") + 1;
+
+        try {
+            $this->scopeNodeFromCode($code);
+            $this->fail('Expected a ContainerParserException to be thrown.');
+        } catch (\ClanCats\Container\Exceptions\ContainerParserException $e) {
+            $this->assertStringContainsString('given at line ' . $expectedLine, $e->getMessage());
+        }
+    }
+
     public function testParseImport()
     {
         $scopeNode = $this->scopeNodeFromCode('import foo/bar');