fix(internal): XML (TwiML) serialization drops characters outside XML 1.0 in all SDK generators

generators/csharp/base/src/asIs/XmlUtils.Template.cs

@@ -1,6 +1,7 @@
 using global::System.Globalization;
 using global::System.IO;
 using global::System.Runtime.CompilerServices;
+using global::System.Text;
 using global::System.Text.Json;
 using global::System.Xml;
 using global::System.Xml.Linq;
@@ -39,7 +40,7 @@ internal static XElement CreateElement(string name, string? @namespace, string?
 
     internal static string Serialize(XElement element, bool xmlDeclaration = false, bool indent = false)
     {
-        var xml = element.ToString(indent ? SaveOptions.None : SaveOptions.DisableFormatting);
+        var xml = WithoutInvalidXmlChars(element).ToString(indent ? SaveOptions.None : SaveOptions.DisableFormatting);
         var siblings = element.Annotation<SiblingComments>();
         if (siblings != null)
         {
@@ -48,6 +49,52 @@ internal static string Serialize(XElement element, bool xmlDeclaration = false,
         return xmlDeclaration ? XmlDeclaration + xml : xml;
     }
 
+    /// <summary>
+    /// Returns a copy of <paramref name="element"/> with characters outside the XML 1.0 <c>Char</c> production
+    /// (control characters, lone surrogates) removed from text, attributes and comments; <see cref="XElement.ToString()"/>
+    /// throws on them and no XML parser accepts them.
+    /// </summary>
+    private static XElement WithoutInvalidXmlChars(XElement element)
+    {
+        var copy = new XElement(element);
+        foreach (var text in copy.DescendantNodesAndSelf().OfType<XText>())
+        {
+            text.Value = StripInvalidXmlChars(text.Value);
+        }
+        foreach (var comment in copy.DescendantNodesAndSelf().OfType<XComment>())
+        {
+            comment.Value = StripInvalidXmlChars(comment.Value);
+        }
+        foreach (var attribute in copy.DescendantsAndSelf().Attributes())
+        {
+            attribute.Value = StripInvalidXmlChars(attribute.Value);
+        }
+        return copy;
+    }
+
+    internal static string StripInvalidXmlChars(string value)
+    {
+        StringBuilder? builder = null;
+        for (var i = 0; i < value.Length; i++)
+        {
+            var c = value[i];
+            if (XmlConvert.IsXmlChar(c))
+            {
+                builder?.Append(c);
+                continue;
+            }
+            if (char.IsHighSurrogate(c) && i + 1 < value.Length && char.IsLowSurrogate(value[i + 1]))
+            {
+                builder?.Append(c).Append(value[i + 1]);
+                i++;
+                continue;
+            }
+            // Lone surrogates, control characters and the noncharacters U+FFFE/U+FFFF are dropped.
+            builder ??= new StringBuilder(value, 0, i, value.Length);
+        }
+        return builder?.ToString() ?? value;
+    }
+
     /// <summary>
     /// Comments rendered immediately before and after an element (<see cref="XmlComment.Before"/> /
     /// <see cref="XmlComment.After"/>), attached to the rendered <see cref="XElement"/> as an annotation
@@ -84,7 +131,8 @@ internal object[] Surround(XElement element)
             return nodes.ToArray();
         }
 
-        private static string Render(XComment comment) => "<!--" + comment.Value + "-->";
+        private static string Render(XComment comment) =>
+            "<!--" + StripInvalidXmlChars(comment.Value) + "-->";
 
         internal string Wrap(string xml)
         {

generators/csharp/base/src/asIs/test/Xml/XmlElementTests.Template.cs

@@ -22,6 +22,23 @@ public void ToXml_WritesAttributesTextAndChildren()
         );
     }
 
+    [Test]
+    public void ToXml_DropsCharactersOutsideXml10()
+    {
+        var element = new XmlElement("Say", "a\0b\u001fc\td").SetAttribute("voice", "man\u0001");
+
+        Assert.That(element.ToXml(), Is.EqualTo("<Say voice=\"man\">abc\td</Say>"));
+    }
+
+    [Test]
+    public void ToXml_KeepsSupplementaryCharactersAndDropsLoneSurrogates()
+    {
+        var element = new XmlElement("Say", "\U0001F600 \uD83D!").SetAttribute("voice", "\U0001F600\uDE00");
+        element.Content.Add(XmlComment.Before("c\u0001\U0001F600"));
+
+        Assert.That(element.ToXml(), Is.EqualTo("<!--c\U0001F600--><Say voice=\"\U0001F600\">\U0001F600 !</Say>"));
+    }
+
     [Test]
     public void ToXml_WritesNamespaceAndPrefix()
     {

generators/csharp/sdk/changes/unreleased/fix-xml-strip-invalid-chars.yml

@@ -0,0 +1,5 @@
+# yaml-language-server: $schema=../../../../../fern-changes-yml.schema.json
+
+- summary: |
+    XML (TwiML) `ToXml()` drops characters outside the XML 1.0 `Char` production (e.g. `\u0001`) from text, attributes and comments instead of throwing `ArgumentException`.
+  type: fix

generators/go/internal/generator/sdk/core/xml.go

@@ -84,6 +84,7 @@ func (c XmlComment) write(buffer *bytes.Buffer) {
 // within a comment and a trailing "-", and either would otherwise end the
 // comment early and turn the rest into markup.
 func xmlCommentText(text string) string {
+	text = stripInvalidXmlChars(text)
 	var out strings.Builder
 	for i := 0; i < len(text); i++ {
 		out.WriteByte(text[i])
@@ -319,9 +320,29 @@ func copyDeclarations(declared map[string]string) map[string]string {
 	return result
 }
 
+// xmlEscape writes value with markup characters escaped and characters outside
+// the XML 1.0 Char production (control characters) dropped, since no XML
+// parser accepts them.
 func xmlEscape(buffer *bytes.Buffer, value string) {
 	// EscapeText only fails on writer errors, which bytes.Buffer never returns.
-	_ = xml.EscapeText(buffer, []byte(value))
+	_ = xml.EscapeText(buffer, []byte(stripInvalidXmlChars(value)))
+}
+
+// stripInvalidXmlChars drops characters outside the XML 1.0 Char production.
+func stripInvalidXmlChars(value string) string {
+	return strings.Map(func(r rune) rune {
+		if isXmlChar(r) {
+			return r
+		}
+		return -1
+	}, value)
+}
+
+func isXmlChar(r rune) bool {
+	return r == 0x9 || r == 0xA || r == 0xD ||
+		(r >= 0x20 && r <= 0xD7FF) ||
+		(r >= 0xE000 && r <= 0xFFFD) ||
+		(r >= 0x10000 && r <= 0x10FFFF)
 }
 
 // ParseXml parses an XML document into an XmlElement tree. Processing

generators/go/internal/generator/sdk/core/xml_test.go

@@ -72,6 +72,12 @@ func TestParseXmlErrors(t *testing.T) {
 	assert.ErrorContains(t, err, "multiple root")
 }
 
+func TestXmlElementDropsInvalidChars(t *testing.T) {
+	element := NewXmlElement("Say").SetAttribute("voice", "man\x01").SetText("a\x00b\x1fc d")
+	assert.Equal(t, "<Say voice=\"man\">abc d</Say>", element.ToXml())
+	assert.Equal(t, "<R><!--c\U0001F600--></R>", NewXmlElement("R").AddComment("c\x01\U0001F600").ToXml())
+}
+
 func TestXmlScalars(t *testing.T) {
 	for _, value := range []string{"true", "1"} {
 		parsed, err := ParseXmlBool(value)

generators/go/sdk/changes/unreleased/fix-xml-strip-invalid-chars.yml

@@ -0,0 +1,5 @@
+# yaml-language-server: $schema=../../../../../fern-changes-yml.schema.json
+
+- summary: |
+    XML (TwiML) serialization drops characters outside the XML 1.0 `Char` production (e.g. `\x01`) from text and attribute values (previously replaced with U+FFFD).
+  type: fix

generators/java/generator-utils/src/main/resources/XmlNode.java

@@ -96,7 +96,7 @@ public static String renderComment(String text) {
      * and either would otherwise end the comment early and turn the rest into markup.
      */
     public static String commentText(String text) {
-        String safe = text.replaceAll("-(?=-)", "- ");
+        String safe = XmlWriter.stripInvalidXmlChars(text).replaceAll("-(?=-)", "- ");
         return safe.endsWith("-") ? safe + " " : safe;
     }
 

generators/java/generator-utils/src/main/resources/XmlWriter.java

@@ -219,11 +219,19 @@ private static String scalarToString(Object value, String separator) {
         return String.valueOf(value);
     }
 
+    /**
+     * Escapes markup characters and drops characters outside the XML 1.0 {@code Char} production (control
+     * characters, lone surrogates), which no XML parser accepts.
+     */
     public static String escape(String value) {
         StringBuilder escaped = new StringBuilder(value.length());
-        for (int i = 0; i < value.length(); i++) {
-            char c = value.charAt(i);
-            switch (c) {
+        for (int i = 0; i < value.length(); ) {
+            int codePoint = value.codePointAt(i);
+            i += Character.charCount(codePoint);
+            if (!isXmlChar(codePoint)) {
+                continue;
+            }
+            switch (codePoint) {
                 case '&':
                     escaped.append("&amp;");
                     break;
@@ -237,9 +245,31 @@ public static String escape(String value) {
                     escaped.append("&quot;");
                     break;
                 default:
-                    escaped.append(c);
+                    escaped.appendCodePoint(codePoint);
             }
         }
         return escaped.toString();
     }
+
+    /** Drops characters outside the XML 1.0 {@code Char} production without escaping markup. */
+    public static String stripInvalidXmlChars(String value) {
+        StringBuilder result = new StringBuilder(value.length());
+        for (int i = 0; i < value.length(); ) {
+            int codePoint = value.codePointAt(i);
+            i += Character.charCount(codePoint);
+            if (isXmlChar(codePoint)) {
+                result.appendCodePoint(codePoint);
+            }
+        }
+        return result.toString();
+    }
+
+    private static boolean isXmlChar(int codePoint) {
+        return codePoint == 0x9
+                || codePoint == 0xA
+                || codePoint == 0xD
+                || (codePoint >= 0x20 && codePoint <= 0xD7FF)
+                || (codePoint >= 0xE000 && codePoint <= 0xFFFD)
+                || (codePoint >= 0x10000 && codePoint <= 0x10FFFF);
+    }
 }

generators/java/sdk/changes/unreleased/fix-xml-strip-invalid-chars.yml

@@ -0,0 +1,5 @@
+# yaml-language-server: $schema=../../../../../fern-changes-yml.schema.json
+
+- summary: |
+    XML (TwiML) serialization drops characters outside the XML 1.0 `Char` production (e.g. `\u0001`) from text and attribute values instead of emitting XML that no parser accepts.
+  type: fix

generators/php/base/src/asIs/Xml/XmlElementTest.Template.php

@@ -31,6 +31,17 @@ public function testSerializesAttributesTextAndChildren(): void
         $this->assertStringStartsWith('<?xml version="1.0" encoding="UTF-8"?>', $element->toXml(xmlDeclaration: true));
     }
 
+    public function testDropsCharactersOutsideXml10(): void
+    {
+        $element = new XmlElement('Say', "a\x00b\x1fc\td", ['voice' => "man\x01"]);
+        $this->assertSame("<Say voice=\"man\">abc\td</Say>", $element->toXml());
+
+        $element = new XmlElement('Say', "\u{1F600}", namespace: "urn:a\x01", prefix: 'tw');
+        $element->namespaceDeclarations['x'] = "urn:b\x01";
+        $element->addChild(XmlComment::before("c\x01"));
+        $this->assertSame('<!--c--><tw:Say xmlns:tw="urn:a" xmlns:x="urn:b">' . "\u{1F600}" . '</tw:Say>', $element->toXml());
+    }
+
     public function testSerializesNamespacesAndPrefixes(): void
     {
         $element = new XmlElement('Dial', '+15551234567', namespace: 'https://www.twilio.com/twiml', prefix: 'tw');

generators/php/base/src/asIs/Xml/XmlUtils.Template.php

@@ -54,7 +54,7 @@ private static function toDom(DOMDocument $document, DOMDocument|DOMElement $par
         $qualifiedName = $element->prefix !== null && $element->prefix !== ''
             ? "{$element->prefix}:{$element->name}"
             : $element->name;
-        $namespace = $element->namespace;
+        $namespace = $element->namespace === null ? null : self::stripInvalidXmlChars($element->namespace);
         $parentElement = $parent instanceof DOMElement ? $parent : null;
         if ($namespace === null && $element->prefix !== null && $parentElement !== null) {
             $namespace = $parentElement->lookupNamespaceURI($element->prefix);
@@ -70,6 +70,7 @@ private static function toDom(DOMDocument $document, DOMDocument|DOMElement $par
         $parent->appendChild($node);
 
         foreach ($element->namespaceDeclarations as $prefix => $uri) {
+            $uri = self::stripInvalidXmlChars($uri);
             $inherited = $parentElement?->lookupNamespaceURI($prefix === '' ? null : $prefix);
             if ($inherited !== $uri) {
                 $node->setAttributeNS(self::XMLNS_NAMESPACE_URI, $prefix === '' ? 'xmlns' : "xmlns:$prefix", $uri);
@@ -86,14 +87,14 @@ private static function toDom(DOMDocument $document, DOMDocument|DOMElement $par
             }
         }
         if ($element->text !== null) {
-            $node->appendChild($document->createTextNode($element->text));
+            $node->appendChild($document->createTextNode(self::stripInvalidXmlChars($element->text)));
         }
         foreach ($element->children as $child) {
             if ($child instanceof XmlText) {
-                $node->appendChild($document->createTextNode($child->text));
+                $node->appendChild($document->createTextNode(self::stripInvalidXmlChars($child->text)));
             } elseif ($child instanceof XmlComment) {
                 if ($child->placement === XmlComment::PLACEMENT_INSIDE) {
-                    $node->appendChild($document->createComment($child->xmlText()));
+                    $node->appendChild($document->createComment(self::stripInvalidXmlChars($child->xmlText())));
                 }
             } else {
                 self::toDom($document, $node, $child->toXmlElement());
@@ -114,13 +115,22 @@ private static function appendSiblingComments(
     ): void {
         foreach ($element->children as $child) {
             if ($child instanceof XmlComment && $child->placement === $placement) {
-                $parent->appendChild($document->createComment($child->xmlText()));
+                $parent->appendChild($document->createComment(self::stripInvalidXmlChars($child->xmlText())));
             }
         }
     }
 
+    /**
+     * Removes characters outside the XML 1.0 `Char` production (control characters), which no XML parser accepts.
+     */
+    private static function stripInvalidXmlChars(string $value): string
+    {
+        return preg_replace('/[^\x{9}\x{A}\x{D}\x{20}-\x{D7FF}\x{E000}-\x{FFFD}\x{10000}-\x{10FFFF}]/u', '', $value) ?? $value;
+    }
+
     private static function setDomAttribute(DOMElement $node, string $name, string $value): void
     {
+        $value = self::stripInvalidXmlChars($value);
         $colon = strpos($name, ':');
         if ($colon === false) {
             $node->setAttribute($name, $value);

generators/php/sdk/changes/unreleased/fix-xml-strip-invalid-chars.yml

@@ -0,0 +1,5 @@
+# yaml-language-server: $schema=../../../../../fern-changes-yml.schema.json
+
+- summary: |
+    XML (TwiML) serialization drops characters outside the XML 1.0 `Char` production (e.g. `\x01`) from text and attribute values, consistently across SDK languages.
+  type: fix

generators/python/core_utilities/shared/xml_utilities.py

@@ -20,7 +20,8 @@
 )
 from xml.dom import minidom
 from xml.parsers.expat import ExpatError
-from xml.sax.saxutils import escape, quoteattr
+from xml.sax.saxutils import escape as _sax_escape
+from xml.sax.saxutils import quoteattr as _sax_quoteattr
 
 import pydantic
 from .pydantic_utilities import IS_PYDANTIC_V2
@@ -73,7 +74,7 @@ class XmlChild:
 def xml_comment_text(text: str) -> str:
     """Makes text safe to place inside a comment: XML forbids `--` within a comment and a trailing `-`,
     and either would otherwise end the comment early and turn the rest into markup."""
-    safe = re.sub(r"-(?=-)", "- ", text)
+    safe = re.sub(r"-(?=-)", "- ", _INVALID_XML_CHARS.sub("", text))
     return f"{safe} " if safe.endswith("-") else safe
 
 
@@ -96,6 +97,18 @@ def __repr__(self) -> str:
         return f"XmlComment({self.text!r})"
 
 
+# Characters outside the XML 1.0 `Char` production (control characters, lone surrogates), which no XML parser accepts.
+_INVALID_XML_CHARS = re.compile("[^\u0009\u000a\u000d\u0020-\ud7ff\ue000-\ufffd\U00010000-\U0010ffff]")
+
+
+def escape(value: str) -> str:
+    return _sax_escape(_INVALID_XML_CHARS.sub("", value))
+
+
+def quoteattr(value: str) -> str:
+    return _sax_quoteattr(_INVALID_XML_CHARS.sub("", value))
+
+
 def serialize_xml_element(
     *,
     name: str,

generators/python/sdk/changes/unreleased/fix-xml-strip-invalid-chars.yml

@@ -0,0 +1,5 @@
+# yaml-language-server: $schema=../../../../../fern-changes-yml.schema.json
+
+- summary: |
+    XML (TwiML) serialization drops characters outside the XML 1.0 `Char` production (e.g. `\x01`) from text and attribute values instead of emitting XML that no parser accepts.
+  type: fix

generators/python/tests/utils/test_xml_utilities.py

@@ -61,6 +61,13 @@ def test_empty_element_is_self_closing() -> None:
     assert serialize_xml_element(name="Hangup") == "<Hangup />"
 
 
+def test_characters_outside_xml_1_0_are_dropped() -> None:
+    xml = serialize_xml_element(
+        name="Say", attributes=[XmlAttribute(name="voice", value="man\x01")], text="a\x00b\x1fc\td"
+    )
+    assert xml == '<Say voice="man">abc\td</Say>'
+
+
 def test_attributes_skip_none_and_render_scalars() -> None:
     xml = serialize_xml_element(
         name="Say",

generators/ruby-v2/base/src/asIs/internal/xml/element.Template.rb

@@ -79,7 +79,7 @@ def to_s
         # and either would otherwise end the comment early and turn the rest into markup, so both are spaced out.
         # @return [String]
         def xml_text
-          safe = value.gsub(/-(?=-)/, "- ")
+          safe = value.gsub(Utils::INVALID_XML_CHARS, "").gsub(/-(?=-)/, "- ")
           safe.end_with?("-") ? "#{safe} " : safe
         end