FIX: using replacement char for all chars outside the Basic Multiling…

…ual Plane
Oldes · Oct 26, 2023 · 3a72825 · 3a72825
1 parent 3eb0b1b
commit 3a72825
Show file tree

Hide file tree

Showing 2 changed files with 6 additions and 0 deletions.
diff --git a/src/core/s-unicode.c b/src/core/s-unicode.c
@@ -446,6 +446,7 @@ Boolean isLegalUTF8Sequence(const UTF8 *source, const UTF8 *sourceEnd) {
 	}
 	*str = src;
 	if (state != UTF8_ACCEPT) return 0; //UNI_REPLACEMENT_CHAR;
+	if (codepoint >= UNI_MAX_BMP) return UNI_REPLACEMENT_CHAR;
 	return codepoint;
 #else
 	const UTF8 *source = *str;

diff --git a/src/tests/units/make-test.r3 b/src/tests/units/make-test.r3
@@ -737,6 +737,11 @@ Rebol [
 		--assert "make bitset! #{FF}"  = try [to string! quote #[bitset! #{FF}] ] ; bitset!
 		--assert "make image! [1x1 #{FFFFFF}]"  = try [to string! quote #[image! 1x1 #{FFFFFF}] ] ; image!
 		--assert "integer! percent!"  = try [to string! quote #[typeset! [integer! percent!]] ] ; typeset!
+
+	--test-- "to string! with chars outside the BMP"
+		;; current Rebol is able to use only 16bit Unicode..
+		--assert #{EFBFBD} = to binary! to string! #{F09F989A}
+		--assert #{EFBFBD} = to binary! to string! #{F09F989C}
 ===end-group===
 
 ===start-group=== "make/to tag"