mb_convert_encoding will not auto-detect input string as UUEncode, Ba…

…se64, QPrint

In a2bc57e, mb_detect_encoding was modified to ensure it would never
return 'UUENCODE', 'QPrint', or other non-encodings as the "detected
text encoding". Before mb_detect_encoding was enhanced so that it could
detect any supported text encoding, those were never returned, and they
are not desired. Actually, we want to eventually remove them completely
from mbstring, since PHP already contains other implementations of
UUEncode, QPrint, Base64, and HTML entities.

For more clarity on why we need to suppress UUEncode, etc. from being
detected by mb_detect_encoding, the existing UUEncode implementation
in mbstring *never* treats any input as erroneous. It just accepts
everything. This means that it would *always* be treated as a valid
choice by mb_detect_encoding, and would be returned in many, many cases
where the input is obviously not UUEncoded.

It turns out that the form of mb_convert_encoding where the user passes
multiple candidate encodings (and mbstring auto-detects which one to
use) was also affected by the same issue. Apply the same fix.

ext/mbstring/mbstring.c

@@ -2491,6 +2491,23 @@ MBSTRING_API HashTable *php_mb_convert_encoding_recursive(HashTable *input, cons
 }
 /* }}} */
 
+static void remove_non_encodings_from_elist(const mbfl_encoding **elist, size_t *size)
+{
+	/* mbstring supports some 'text encodings' which aren't really text encodings
+	 * at all, but really 'byte encodings', like Base64, QPrint, and so on.
+	 * These should never be returned by `mb_detect_encoding`. */
+	int shift = 0;
+	for (int i = 0; i < *size; i++) {
+		const mbfl_encoding *encoding = elist[i];
+		if (encoding->no_encoding <= mbfl_no_encoding_charset_min) {
+			shift++; /* Remove this encoding from the list */
+		} else if (shift) {
+			elist[i - shift] = encoding;
+		}
+	}
+	*size -= shift;
+}
+
 /* {{{ Returns converted string in desired encoding */
 PHP_FUNCTION(mb_convert_encoding)
 {
@@ -2531,6 +2548,10 @@ PHP_FUNCTION(mb_convert_encoding)
 		free_from_encodings = 0;
 	}
 
+	if (num_from_encodings > 1) {
+		remove_non_encodings_from_elist(from_encodings, &num_from_encodings);
+	}
+
 	if (!num_from_encodings) {
 		efree(ZEND_VOIDP(from_encodings));
 		zend_argument_value_error(3, "must specify at least one encoding");
@@ -2664,23 +2685,6 @@ PHP_FUNCTION(mb_strtolower)
 }
 /* }}} */
 
-static void remove_non_encodings_from_elist(const mbfl_encoding **elist, size_t *size)
-{
-	/* mbstring supports some 'text encodings' which aren't really text encodings
-	 * at all, but really 'byte encodings', like Base64, QPrint, and so on.
-	 * These should never be returned by `mb_detect_encoding`. */
-	int shift = 0;
-	for (int i = 0; i < *size; i++) {
-		const mbfl_encoding *encoding = elist[i];
-		if (encoding->no_encoding <= mbfl_no_encoding_charset_min) {
-			shift++; /* Remove this encoding from the list */
-		} else if (shift) {
-			elist[i - shift] = encoding;
-		}
-	}
-	*size -= shift;
-}
-
 /* {{{ Encodings of the given string is returned (as a string) */
 PHP_FUNCTION(mb_detect_encoding)
 {

ext/mbstring/tests/mb_convert_encoding.phpt

@@ -9,33 +9,24 @@ mbstring.language=Japanese
 <?php
 // TODO: Add more tests
 
-// SJIS string (BASE64 encoded)
 $sjis = base64_decode('k/qWe4zqg2WDTINYg2eCxYK3gUIwMTIzNIJUglWCVoJXgliBQg==');
-// JIS string (BASE64 encoded)
 $jis = base64_decode('GyRCRnxLXDhsJUYlLSU5JUgkRyQ5ISMbKEIwMTIzNBskQiM1IzYjNyM4IzkhIxsoQg==');
-// EUC-JP string
-$euc_jp = '日本語テキストです。01234５６７８９。';
+$euc_jp = "\xC6\xFC\xCB\xDC\xB8\xEC\xA5\xC6\xA5\xAD\xA5\xB9\xA5\xC8\xA4\xC7\xA4\xB9\xA1\xA301234\xA3\xB5\xA3\xB6\xA3\xB7\xA3\xB8\xA3\xB9\xA1\xA3";
 
 // Test with single "form encoding"
-// Note: For some reason it complains, results are different. Not researched.
 echo "== BASIC TEST ==\n";
-$s = $sjis;
-$s = bin2hex(mb_convert_encoding($s, 'EUC-JP', 'SJIS'));
+$s = bin2hex(mb_convert_encoding($sjis, 'EUC-JP', 'SJIS'));
 print("EUC-JP: $s\n"); // EUC-JP
 
-$s = $jis;
-$s = bin2hex(mb_convert_encoding($s, 'EUC-JP', 'JIS'));
+$s = bin2hex(mb_convert_encoding($jis, 'EUC-JP', 'JIS'));
 print("EUC-JP: $s\n"); // EUC-JP
 
-$s = $euc_jp;
-$s = mb_convert_encoding($s, 'SJIS', 'EUC-JP');
+$s = mb_convert_encoding($euc_jp, 'SJIS', 'EUC-JP');
 print("SJIS: ".base64_encode($s)."\n"); // SJIS
 
-$s = $euc_jp;
-$s = mb_convert_encoding($s, 'JIS', 'EUC-JP');
+$s = mb_convert_encoding($euc_jp, 'JIS', 'EUC-JP');
 print("JIS: ".base64_encode($s)."\n"); // JIS
 
-
 // Using Encoding List Array
 echo "== STRING ENCODING LIST ==\n";
 
@@ -52,11 +43,10 @@ $s = $euc_jp;
 $s = mb_convert_encoding($s, 'JIS', $a);
 print("JIS: ".base64_encode($s)."\n"); // JIS
 
-
 // Using Encoding List Array
 echo "== ARRAY ENCODING LIST ==\n";
 
-$a = array(0=>'JIS', 1=>'UTF-8', 2=>'EUC-JP', 3=>'SJIS');
+$a = ['JIS', 'UTF-8', 'EUC-JP', 'SJIS'];
 $s = $jis;
 $s = bin2hex(mb_convert_encoding($s, 'EUC-JP', $a));
 print("EUC-JP: $s\n"); // EUC-JP
@@ -69,6 +59,8 @@ $s = $euc_jp;
 $s = mb_convert_encoding($s, 'JIS', $a);
 print("JIS: ".base64_encode($s)."\n"); // JIS
 
+// Regression test for bug #81676
+echo "UTF-8: " . mb_convert_encoding('test', 'UTF-8', mb_list_encodings()), "\n";
 
 // Using Detect Order
 echo "== DETECT ORDER ==\n";
@@ -117,6 +109,7 @@ JIS: GyRCRnxLXDhsJUYlLSU5JUgkRyQ5ISMbKEIwMTIzNBskQiM1IzYjNyM4IzkhIxsoQg==
 EUC-JP: c6fccbdcb8eca5c6a5ada5b9a5c8a4c7a4b9a1a33031323334a3b5a3b6a3b7a3b8a3b9a1a3
 SJIS: k/qWe4zqg2WDTINYg2eCxYK3gUIwMTIzNIJUglWCVoJXgliBQg==
 JIS: GyRCRnxLXDhsJUYlLSU5JUgkRyQ5ISMbKEIwMTIzNBskQiM1IzYjNyM4IzkhIxsoQg==
+UTF-8: test
 == DETECT ORDER ==
 EUC-JP: c6fccbdcb8eca5c6a5ada5b9a5c8a4c7a4b9a1a33031323334a3b5a3b6a3b7a3b8a3b9a1a3
 SJIS: k/qWe4zqg2WDTINYg2eCxYK3gUIwMTIzNIJUglWCVoJXgliBQg==