Skip to content

Commit 63792a2

Browse files
ArtUkrainskiyArtUkrainskiy
committed
ext/standard: Add php_next_utf8_char_inline()
json_encode(), utf8_decode() and the xml extension decode one character at a time with php_next_utf8_char(), and every call went through the generic charset decoder of html.c. The new static inline function in html.h decodes valid UTF-8 at the call site and leaves every invalid sequence to php_next_utf8_char(), which is unchanged.
1 parent bd6c637 commit 63792a2

7 files changed

Lines changed: 280 additions & 3 deletions

File tree

‎UPGRADING‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -106,5 +106,9 @@ PHP 8.7 UPGRADE NOTES
106106
. Improved performance of pack() for large strings using the a, A and Z
107107
formats.
108108

109+
- JSON:
110+
. Improved performance of json_encode() for strings with non-ASCII
111+
characters.
112+
109113
- MBString:
110114
. Improved performance of mb_strlen for UTF-8 strings.

‎UPGRADING.INTERNALS‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,9 @@ PHP 8.7 INTERNALS UPGRADE NOTES
7575
- Standard:
7676
. php_explode() now takes a HashTable* instead of a zval* to store the
7777
resulting parts. It already expected the zval* to be IS_ARRAY.
78+
. Added php_next_utf8_char_inline() to ext/standard/html.h. It has the same
79+
contract as php_next_utf8_char() and decodes valid UTF-8 at the call
80+
site.
7881

7982
========================
8083
4. OpCode changes

‎ext/json/json_encoder.c‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -405,7 +405,7 @@ zend_result php_json_escape_string(
405405
us = (unsigned char)s[0];
406406
if (UNEXPECTED(us >= 0x80)) {
407407
zend_result status;
408-
us = php_next_utf8_char((unsigned char *)s, len, &pos, &status);
408+
us = php_next_utf8_char_inline((const unsigned char *)s, len, &pos, &status);
409409

410410
/* check whether UTF8 character is correct */
411411
if (UNEXPECTED(status != SUCCESS)) {

‎ext/json/tests/json_encode_utf8_sequences.phpt‎

Lines changed: 217 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,217 @@
1+
--TEST--
2+
json_encode() with valid and invalid UTF-8 sequences around the two-, three- and four-byte boundaries
3+
--FILE--
4+
<?php
5+
$cases = [
6+
'C2 80, smallest two-byte' => "\xC2\x80",
7+
'DF BF, largest two-byte' => "\xDF\xBF",
8+
'C0 AF, overlong' => "\xC0\xAF",
9+
'C1 BF, overlong' => "\xC1\xBF",
10+
'80, lone continuation byte' => "\x80",
11+
'C2 at the end' => "a\xC2",
12+
'C2 then ASCII' => "\xC2a",
13+
'C2 then lead byte' => "\xC2\xC3\xA9",
14+
'E0 A0 80, smallest three-byte' => "\xE0\xA0\x80",
15+
'E0 9F BF, overlong' => "\xE0\x9F\xBF",
16+
'ED 9F BF, before surrogates' => "\xED\x9F\xBF",
17+
'ED A0 80, surrogate' => "\xED\xA0\x80",
18+
'ED BF BF, surrogate' => "\xED\xBF\xBF",
19+
'ED A0 at the end' => "a\xED\xA0",
20+
'EE 80 80, after surrogates' => "\xEE\x80\x80",
21+
'EF BF BF, largest three-byte' => "\xEF\xBF\xBF",
22+
'E2 82 at the end' => "a\xE2\x82",
23+
'E2 82 then ASCII' => "\xE2\x82A",
24+
'E2 then ASCII' => "\xE2ab",
25+
'E2 80 A8, U+2028' => "\xE2\x80\xA8",
26+
'F0 90 80 80, smallest four-byte' => "\xF0\x90\x80\x80",
27+
'F4 8F BF BF, largest four-byte' => "\xF4\x8F\xBF\xBF",
28+
'F0 8F BF BF, overlong' => "\xF0\x8F\xBF\xBF",
29+
'F4 90 80 80, above U+10FFFF' => "\xF4\x90\x80\x80",
30+
'F5 80 80 80' => "\xF5\x80\x80\x80",
31+
'F0 9F 98 at the end' => "a\xF0\x9F\x98",
32+
'F0 9F 98 then ASCII' => "\xF0\x9F\x98a",
33+
'F0 9F 98 then lead byte' => "\xF0\x9F\x98\xF0\x9F\x98\x80",
34+
'four-byte between ASCII' => "x\xF0\x9F\x98\x80y",
35+
'two-byte after ASCII and at the end' => "abc\xD0\xB6",
36+
'quote after two-byte' => "\xD0\xB6\"\xD0\xB6/",
37+
'three-byte between ASCII' => "x\xE6\xBC\xA2y",
38+
];
39+
foreach ($cases as $name => $str) {
40+
echo "$name:\n";
41+
var_dump(json_encode($str));
42+
var_dump(json_encode($str, JSON_INVALID_UTF8_IGNORE));
43+
var_dump(json_encode($str, JSON_INVALID_UTF8_SUBSTITUTE));
44+
var_dump(bin2hex(json_encode($str, JSON_UNESCAPED_UNICODE | JSON_INVALID_UTF8_SUBSTITUTE)));
45+
}
46+
echo "U+2028 with JSON_UNESCAPED_LINE_TERMINATORS:\n";
47+
var_dump(bin2hex(json_encode("a\xE2\x80\xA8b", JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_LINE_TERMINATORS)));
48+
echo "JSON_PARTIAL_OUTPUT_ON_ERROR:\n";
49+
var_dump(json_encode(["ok\xD0\xB6", "bad\xC0\xAF", "\xE2\x82"], JSON_PARTIAL_OUTPUT_ON_ERROR));
50+
var_dump(json_last_error_msg());
51+
?>
52+
--EXPECT--
53+
C2 80, smallest two-byte:
54+
string(8) ""\u0080""
55+
string(8) ""\u0080""
56+
string(8) ""\u0080""
57+
string(8) "22c28022"
58+
DF BF, largest two-byte:
59+
string(8) ""\u07ff""
60+
string(8) ""\u07ff""
61+
string(8) ""\u07ff""
62+
string(8) "22dfbf22"
63+
C0 AF, overlong:
64+
bool(false)
65+
string(2) """"
66+
string(14) ""\ufffd\ufffd""
67+
string(16) "22efbfbdefbfbd22"
68+
C1 BF, overlong:
69+
bool(false)
70+
string(2) """"
71+
string(14) ""\ufffd\ufffd""
72+
string(16) "22efbfbdefbfbd22"
73+
80, lone continuation byte:
74+
bool(false)
75+
string(2) """"
76+
string(8) ""\ufffd""
77+
string(10) "22efbfbd22"
78+
C2 at the end:
79+
bool(false)
80+
string(3) ""a""
81+
string(9) ""a\ufffd""
82+
string(12) "2261efbfbd22"
83+
C2 then ASCII:
84+
bool(false)
85+
string(3) ""a""
86+
string(9) ""\ufffda""
87+
string(12) "22efbfbd6122"
88+
C2 then lead byte:
89+
bool(false)
90+
string(8) ""\u00e9""
91+
string(14) ""\ufffd\u00e9""
92+
string(14) "22efbfbdc3a922"
93+
E0 A0 80, smallest three-byte:
94+
string(8) ""\u0800""
95+
string(8) ""\u0800""
96+
string(8) ""\u0800""
97+
string(10) "22e0a08022"
98+
E0 9F BF, overlong:
99+
bool(false)
100+
string(2) """"
101+
string(8) ""\ufffd""
102+
string(10) "22efbfbd22"
103+
ED 9F BF, before surrogates:
104+
string(8) ""\ud7ff""
105+
string(8) ""\ud7ff""
106+
string(8) ""\ud7ff""
107+
string(10) "22ed9fbf22"
108+
ED A0 80, surrogate:
109+
bool(false)
110+
string(2) """"
111+
string(8) ""\ufffd""
112+
string(10) "22efbfbd22"
113+
ED BF BF, surrogate:
114+
bool(false)
115+
string(2) """"
116+
string(8) ""\ufffd""
117+
string(10) "22efbfbd22"
118+
ED A0 at the end:
119+
bool(false)
120+
string(3) ""a""
121+
string(9) ""a\ufffd""
122+
string(12) "2261efbfbd22"
123+
EE 80 80, after surrogates:
124+
string(8) ""\ue000""
125+
string(8) ""\ue000""
126+
string(8) ""\ue000""
127+
string(10) "22ee808022"
128+
EF BF BF, largest three-byte:
129+
string(8) ""\uffff""
130+
string(8) ""\uffff""
131+
string(8) ""\uffff""
132+
string(10) "22efbfbf22"
133+
E2 82 at the end:
134+
bool(false)
135+
string(3) ""a""
136+
string(9) ""a\ufffd""
137+
string(12) "2261efbfbd22"
138+
E2 82 then ASCII:
139+
bool(false)
140+
string(3) ""A""
141+
string(9) ""\ufffdA""
142+
string(12) "22efbfbd4122"
143+
E2 then ASCII:
144+
bool(false)
145+
string(4) ""ab""
146+
string(10) ""\ufffdab""
147+
string(14) "22efbfbd616222"
148+
E2 80 A8, U+2028:
149+
string(8) ""\u2028""
150+
string(8) ""\u2028""
151+
string(8) ""\u2028""
152+
string(16) "225c753230323822"
153+
F0 90 80 80, smallest four-byte:
154+
string(14) ""\ud800\udc00""
155+
string(14) ""\ud800\udc00""
156+
string(14) ""\ud800\udc00""
157+
string(12) "22f090808022"
158+
F4 8F BF BF, largest four-byte:
159+
string(14) ""\udbff\udfff""
160+
string(14) ""\udbff\udfff""
161+
string(14) ""\udbff\udfff""
162+
string(12) "22f48fbfbf22"
163+
F0 8F BF BF, overlong:
164+
bool(false)
165+
string(2) """"
166+
string(8) ""\ufffd""
167+
string(10) "22efbfbd22"
168+
F4 90 80 80, above U+10FFFF:
169+
bool(false)
170+
string(2) """"
171+
string(8) ""\ufffd""
172+
string(10) "22efbfbd22"
173+
F5 80 80 80:
174+
bool(false)
175+
string(2) """"
176+
string(26) ""\ufffd\ufffd\ufffd\ufffd""
177+
string(28) "22efbfbdefbfbdefbfbdefbfbd22"
178+
F0 9F 98 at the end:
179+
bool(false)
180+
string(3) ""a""
181+
string(9) ""a\ufffd""
182+
string(12) "2261efbfbd22"
183+
F0 9F 98 then ASCII:
184+
bool(false)
185+
string(3) ""a""
186+
string(9) ""\ufffda""
187+
string(12) "22efbfbd6122"
188+
F0 9F 98 then lead byte:
189+
bool(false)
190+
string(14) ""\ud83d\ude00""
191+
string(20) ""\ufffd\ud83d\ude00""
192+
string(18) "22efbfbdf09f988022"
193+
four-byte between ASCII:
194+
string(16) ""x\ud83d\ude00y""
195+
string(16) ""x\ud83d\ude00y""
196+
string(16) ""x\ud83d\ude00y""
197+
string(16) "2278f09f98807922"
198+
two-byte after ASCII and at the end:
199+
string(11) ""abc\u0436""
200+
string(11) ""abc\u0436""
201+
string(11) ""abc\u0436""
202+
string(14) "22616263d0b622"
203+
quote after two-byte:
204+
string(18) ""\u0436\"\u0436\/""
205+
string(18) ""\u0436\"\u0436\/""
206+
string(18) ""\u0436\"\u0436\/""
207+
string(20) "22d0b65c22d0b65c2f22"
208+
three-byte between ASCII:
209+
string(10) ""x\u6f22y""
210+
string(10) ""x\u6f22y""
211+
string(10) ""x\u6f22y""
212+
string(14) "2278e6bca27922"
213+
U+2028 with JSON_UNESCAPED_LINE_TERMINATORS:
214+
string(14) "2261e280a86222"
215+
JSON_PARTIAL_OUTPUT_ON_ERROR:
216+
string(22) "["ok\u0436",null,null]"
217+
string(56) "Malformed UTF-8 characters, possibly incorrectly encoded"

‎ext/standard/html.h‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,4 +47,57 @@ PHPAPI zend_string *php_escape_html_entities_ex(const unsigned char *old, size_t
4747
PHPAPI zend_string *php_unescape_html_entities(zend_string *str, int all, int flags, const char *hint_charset);
4848
PHPAPI unsigned int php_next_utf8_char(const unsigned char *str, size_t str_len, size_t *cursor, zend_result *status);
4949

50+
/* Same contract as php_next_utf8_char(). Decodes valid UTF-8 inline; invalid input goes
51+
* to php_next_utf8_char(). */
52+
static zend_always_inline unsigned int php_next_utf8_char_inline(
53+
const unsigned char *str, size_t str_len, size_t *cursor, zend_result *status)
54+
{
55+
size_t pos = *cursor;
56+
size_t avail;
57+
58+
ZEND_ASSERT(pos <= str_len);
59+
avail = str_len - pos;
60+
if (EXPECTED(avail >= 1)) {
61+
unsigned int c = str[pos];
62+
63+
if (c < 0x80) {
64+
*cursor = pos + 1;
65+
*status = SUCCESS;
66+
return c;
67+
}
68+
if (EXPECTED(c < 0xe0)) {
69+
/* C2..DF + continuation byte: always a valid shortest form */
70+
if (c >= 0xc2 && avail >= 2 && (str[pos + 1] & 0xc0) == 0x80) {
71+
*cursor = pos + 2;
72+
*status = SUCCESS;
73+
return ((c & 0x1f) << 6) | (str[pos + 1] & 0x3f);
74+
}
75+
} else if (c < 0xf0) {
76+
if (avail >= 3 && (str[pos + 1] & 0xc0) == 0x80 && (str[pos + 2] & 0xc0) == 0x80) {
77+
unsigned int cp = ((c & 0x0f) << 12) | ((str[pos + 1] & 0x3f) << 6) | (str[pos + 2] & 0x3f);
78+
/* neither overlong nor a surrogate */
79+
if (cp >= 0x800 && (cp < 0xd800 || cp > 0xdfff)) {
80+
*cursor = pos + 3;
81+
*status = SUCCESS;
82+
return cp;
83+
}
84+
}
85+
} else if (c < 0xf5) {
86+
if (avail >= 4 && (str[pos + 1] & 0xc0) == 0x80 && (str[pos + 2] & 0xc0) == 0x80
87+
&& (str[pos + 3] & 0xc0) == 0x80) {
88+
unsigned int cp = ((c & 0x07) << 18) | ((str[pos + 1] & 0x3f) << 12)
89+
| ((str[pos + 2] & 0x3f) << 6) | (str[pos + 3] & 0x3f);
90+
/* neither overlong nor above U+10FFFF */
91+
if (cp >= 0x10000 && cp <= 0x10ffff) {
92+
*cursor = pos + 4;
93+
*status = SUCCESS;
94+
return cp;
95+
}
96+
}
97+
}
98+
}
99+
100+
return php_next_utf8_char(str, str_len, cursor, status);
101+
}
102+
50103
#endif /* HTML_H */

‎ext/standard/string.c‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6431,7 +6431,7 @@ static zend_string *php_utf8_decode(const char *s, size_t len)
64316431
ZSTR_LEN(str) = 0;
64326432
while (pos < len) {
64336433
zend_result status = FAILURE;
6434-
c = php_next_utf8_char((const unsigned char*)s, (size_t) len, &pos, &status);
6434+
c = php_next_utf8_char_inline((const unsigned char*)s, (size_t) len, &pos, &status);
64356435

64366436
/* The lower 256 codepoints of Unicode are identical to Latin-1,
64376437
* so we don't need to do any mapping here beyond replacing non-Latin-1

‎ext/xml/xml.c‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -513,7 +513,7 @@ static zend_string *xml_utf8_decode(const XML_Char *s, size_t len, const XML_Cha
513513
ZSTR_LEN(str) = 0;
514514
while (pos < len) {
515515
zend_result status = FAILURE;
516-
c = php_next_utf8_char((const unsigned char*)s, len, &pos, &status);
516+
c = php_next_utf8_char_inline((const unsigned char*)s, len, &pos, &status);
517517

518518
if (status == FAILURE || c > 0xFFU) {
519519
c = '?';

0 commit comments

Comments
 (0)