From 3e4a1821c46d7767055079758fa5601d02570c89 Mon Sep 17 00:00:00 2001 From: barry Date: Thu, 23 Jul 2026 20:39:09 -0500 Subject: [PATCH 01/19] Comments: Encode emoji for utf8mb3 comment columns in wp_insert_comment() and wp_update_comment(). wp_insert_post() already encodes emoji as HTML entities before saving to a utf8/utf8mb3 column, via wp_encode_emoji(). wp_insert_comment() and wp_update_comment() had no equivalent handling. Extracts the shared logic into a new private helper, _wp_encode_emoji_in_fields( $data, $table, $fields ), in wp-includes/formatting.php, used by wp_insert_post() (refactor, no behavior change) and now also by wp_insert_comment() and wp_update_comment() for the comment_author and comment_content fields. Includes PHPUnit coverage for the new helper directly, plus deterministic (pre_get_col_charset-filter-forced) tests for wp_insert_post(), wp_insert_comment(), and wp_update_comment(), so the utf8mb3 code path is exercised in CI regardless of the test database's actual charset. Props barry. Fixes #65700. --- src/wp-includes/comment.php | 4 + src/wp-includes/formatting.php | 36 ++++++++ src/wp-includes/post.php | 13 +-- tests/phpunit/tests/comment.php | 108 +++++++++++++++++++++++ tests/phpunit/tests/formatting/emoji.php | 81 +++++++++++++++++ tests/phpunit/tests/post.php | 40 +++++++++ 6 files changed, 270 insertions(+), 12 deletions(-) diff --git a/src/wp-includes/comment.php b/src/wp-includes/comment.php index b3738c24ec9df..8447fdd7907ae 100644 --- a/src/wp-includes/comment.php +++ b/src/wp-includes/comment.php @@ -2162,6 +2162,8 @@ function wp_insert_comment( $commentdata ) { 'user_id' ); + $compacted = _wp_encode_emoji_in_fields( $compacted, $wpdb->comments, array( 'comment_author', 'comment_content' ) ); + if ( ! $wpdb->insert( $wpdb->comments, $compacted ) ) { return false; } @@ -2759,6 +2761,8 @@ function wp_update_comment( $commentarr, $wp_error = false ) { $data = wp_array_slice_assoc( $data, $keys ); + $data = _wp_encode_emoji_in_fields( $data, $wpdb->comments, array( 'comment_author', 'comment_content' ) ); + $result = $wpdb->update( $wpdb->comments, $data, array( 'comment_ID' => $comment_id ) ); if ( false === $result ) { diff --git a/src/wp-includes/formatting.php b/src/wp-includes/formatting.php index 74a28109b6536..58737946b4e21 100644 --- a/src/wp-includes/formatting.php +++ b/src/wp-includes/formatting.php @@ -6116,6 +6116,42 @@ function wp_encode_emoji( $content ) { return $content; } +/** + * Encodes emoji in the given fields of a data array whose corresponding + * database columns cannot natively store 4-byte UTF-8 characters. + * + * Used by wp_insert_post(), wp_insert_comment(), and wp_update_comment() so + * that emoji (and other 4-byte characters) are stored as HTML entities on + * `utf8`/`utf8mb3` columns instead of causing truncated or failed writes. + * + * @access private + * @since 7.1.0 + * + * @global wpdb $wpdb WordPress database abstraction object. + * + * @param array $data Array of field name => value pairs to check, keyed the + * same way as the corresponding database columns. + * @param string $table Name of the database table the fields belong to. + * @param array $fields Names of the fields within $data that may contain emoji. + * @return array $data with emoji encoded in the given fields, where needed. + */ +function _wp_encode_emoji_in_fields( $data, $table, $fields ) { + global $wpdb; + + foreach ( $fields as $field ) { + if ( isset( $data[ $field ] ) ) { + $charset = $wpdb->get_col_charset( $table, $field ); + + // The 'utf8' character set is a deprecated alias of 'utf8mb3'. See . + if ( 'utf8' === $charset || 'utf8mb3' === $charset ) { + $data[ $field ] = wp_encode_emoji( $data[ $field ] ); + } + } + } + + return $data; +} + /** * Converts emoji to a static img element. * diff --git a/src/wp-includes/post.php b/src/wp-includes/post.php index 3813176140bb4..a2b3a7f9afad1 100644 --- a/src/wp-includes/post.php +++ b/src/wp-includes/post.php @@ -4917,18 +4917,7 @@ function wp_insert_post( $postarr, $wp_error = false, $fire_after_hooks = true ) 'guid' ); - $emoji_fields = array( 'post_title', 'post_content', 'post_excerpt' ); - - foreach ( $emoji_fields as $emoji_field ) { - if ( isset( $data[ $emoji_field ] ) ) { - $charset = $wpdb->get_col_charset( $wpdb->posts, $emoji_field ); - - // The 'utf8' character set is a deprecated alias of 'utf8mb3'. See . - if ( 'utf8' === $charset || 'utf8mb3' === $charset ) { - $data[ $emoji_field ] = wp_encode_emoji( $data[ $emoji_field ] ); - } - } - } + $data = _wp_encode_emoji_in_fields( $data, $wpdb->posts, array( 'post_title', 'post_content', 'post_excerpt' ) ); if ( 'attachment' === $post_type ) { /** diff --git a/tests/phpunit/tests/comment.php b/tests/phpunit/tests/comment.php index 8c7e8d684bc59..1d927a1101040 100644 --- a/tests/phpunit/tests/comment.php +++ b/tests/phpunit/tests/comment.php @@ -2045,4 +2045,112 @@ public function test_get_comment_should_only_treat_numeric_values_as_comment_ids $this->assertNull( get_comment( $comment_id . 'abc' ), 'Expected a malformed numeric string not to be cast to a comment ID.' ); $this->assertNull( get_comment( true ), 'Expected true not to be cast to comment ID 1.' ); // @phpstan-ignore argument.type (Intentionally passing an invalid value.) } + + /** + * `wp_insert_post()` already encodes emoji as HTML entities before saving to a + * utf8/utf8mb3 column (see #31242). This forces the same code path for comments + * via the `pre_get_col_charset` filter, so it runs deterministically regardless + * of the test database's actual charset. + * + * @ticket 65700 + * + * @covers ::wp_insert_comment + * @covers ::_wp_encode_emoji_in_fields + */ + public function test_wp_insert_comment_encodes_emoji_for_utf8mb3_columns(): void { + add_filter( 'pre_get_col_charset', array( $this, 'filter_pre_get_col_charset_utf8mb3_for_comment_fields' ), 10, 3 ); + + $comment_id = wp_insert_comment( + array( + 'comment_post_ID' => self::$post_id, + 'comment_author' => "foo\xf0\x9f\x98\x88bar", + 'comment_content' => "foo\xf0\x9f\x98\x8ebaz", + ) + ); + + remove_filter( 'pre_get_col_charset', array( $this, 'filter_pre_get_col_charset_utf8mb3_for_comment_fields' ), 10 ); + + $this->assertIsInt( $comment_id, 'Expected wp_insert_comment() to return the new comment ID.' ); + + $comment = get_comment( $comment_id ); + + $this->assertSame( 'foo😈bar', $comment->comment_author ); + $this->assertSame( 'foo😎baz', $comment->comment_content ); + } + + /** + * Control for test_wp_insert_comment_encodes_emoji_for_utf8mb3_columns(): on a + * utf8mb4 column, emoji should be stored as-is rather than as HTML entities. + * + * @ticket 65700 + * + * @covers ::wp_insert_comment + */ + public function test_wp_insert_comment_does_not_encode_emoji_for_utf8mb4_columns(): void { + add_filter( 'pre_get_col_charset', array( $this, 'filter_pre_get_col_charset_utf8mb4_for_comment_fields' ), 10, 3 ); + + $comment_id = wp_insert_comment( + array( + 'comment_post_ID' => self::$post_id, + 'comment_author' => "foo\xf0\x9f\x98\x88bar", + 'comment_content' => "foo\xf0\x9f\x98\x8ebaz", + ) + ); + + remove_filter( 'pre_get_col_charset', array( $this, 'filter_pre_get_col_charset_utf8mb4_for_comment_fields' ), 10 ); + + $comment = get_comment( $comment_id ); + + $this->assertSame( "foo\xf0\x9f\x98\x88bar", $comment->comment_author ); + $this->assertSame( "foo\xf0\x9f\x98\x8ebaz", $comment->comment_content ); + } + + /** + * @ticket 65700 + * + * @covers ::wp_update_comment + * @covers ::_wp_encode_emoji_in_fields + */ + public function test_wp_update_comment_encodes_emoji_for_utf8mb3_columns(): void { + $comment_id = self::factory()->comment->create( array( 'comment_post_ID' => self::$post_id ) ); + + add_filter( 'pre_get_col_charset', array( $this, 'filter_pre_get_col_charset_utf8mb3_for_comment_fields' ), 10, 3 ); + + $result = wp_update_comment( + array( + 'comment_ID' => $comment_id, + 'comment_author' => "foo\xf0\x9f\x98\x88bar", + 'comment_content' => "foo\xf0\x9f\x98\x8ebaz", + ) + ); + + remove_filter( 'pre_get_col_charset', array( $this, 'filter_pre_get_col_charset_utf8mb3_for_comment_fields' ), 10 ); + + $this->assertSame( 1, $result ); + + $comment = get_comment( $comment_id ); + + $this->assertSame( 'foo😈bar', $comment->comment_author ); + $this->assertSame( 'foo😎baz', $comment->comment_content ); + } + + public function filter_pre_get_col_charset_utf8mb3_for_comment_fields( $charset, $table, $column ) { + global $wpdb; + + if ( $wpdb->comments === $table && in_array( $column, array( 'comment_author', 'comment_content' ), true ) ) { + return 'utf8mb3'; + } + + return $charset; + } + + public function filter_pre_get_col_charset_utf8mb4_for_comment_fields( $charset, $table, $column ) { + global $wpdb; + + if ( $wpdb->comments === $table && in_array( $column, array( 'comment_author', 'comment_content' ), true ) ) { + return 'utf8mb4'; + } + + return $charset; + } } diff --git a/tests/phpunit/tests/formatting/emoji.php b/tests/phpunit/tests/formatting/emoji.php index d885c49fd7289..75901004e726a 100644 --- a/tests/phpunit/tests/formatting/emoji.php +++ b/tests/phpunit/tests/formatting/emoji.php @@ -376,4 +376,85 @@ public function data_wp_staticize_emoji() { public function test_wp_staticize_emoji( $emoji, $expected ) { $this->assertSame( $expected, wp_staticize_emoji( $emoji ) ); } + + /** + * @ticket 65700 + * + * @covers ::_wp_encode_emoji_in_fields + */ + public function test_wp_encode_emoji_in_fields_only_touches_requested_utf8mb3_fields(): void { + $data = array( + 'legacy_field' => '๐Ÿ™‚', + 'modern_field' => '๐Ÿ™‚', + 'ignored_field' => '๐Ÿ™‚', + ); + + add_filter( 'pre_get_col_charset', array( $this, 'filter_pre_get_col_charset_for_encode_emoji_in_fields_test' ), 10, 3 ); + + $result = _wp_encode_emoji_in_fields( $data, 'some_table', array( 'legacy_field', 'modern_field' ) ); + + remove_filter( 'pre_get_col_charset', array( $this, 'filter_pre_get_col_charset_for_encode_emoji_in_fields_test' ), 10 ); + + $this->assertSame( + '🙂', + $result['legacy_field'], + 'Expected emoji to be encoded for a field on a utf8/utf8mb3 column.' + ); + $this->assertSame( + '๐Ÿ™‚', + $result['modern_field'], + 'Expected emoji to be left alone for a field on a utf8mb4 column.' + ); + $this->assertSame( + '๐Ÿ™‚', + $result['ignored_field'], + 'Expected a field not passed in $fields to be left alone entirely.' + ); + } + + /** + * @ticket 65700 + * + * @covers ::_wp_encode_emoji_in_fields + */ + public function test_wp_encode_emoji_in_fields_recognizes_the_utf8_alias(): void { + $data = array( 'legacy_field' => '๐Ÿ™‚' ); + + add_filter( 'pre_get_col_charset', array( $this, 'filter_pre_get_col_charset_utf8_alias_for_encode_emoji_in_fields_test' ) ); + + $result = _wp_encode_emoji_in_fields( $data, 'some_table', array( 'legacy_field' ) ); + + remove_filter( 'pre_get_col_charset', array( $this, 'filter_pre_get_col_charset_utf8_alias_for_encode_emoji_in_fields_test' ) ); + + $this->assertSame( + '🙂', + $result['legacy_field'], + 'Expected emoji to be encoded for a field reporting the deprecated utf8 charset alias.' + ); + } + + /** + * @ticket 65700 + * + * @covers ::_wp_encode_emoji_in_fields + */ + public function test_wp_encode_emoji_in_fields_ignores_fields_missing_from_data(): void { + $data = array( 'present_field' => '๐Ÿ™‚' ); + + $result = _wp_encode_emoji_in_fields( $data, 'some_table', array( 'present_field', 'absent_field' ) ); + + $this->assertArrayNotHasKey( + 'absent_field', + $result, + 'Expected a field missing from $data to not be added by _wp_encode_emoji_in_fields().' + ); + } + + public function filter_pre_get_col_charset_for_encode_emoji_in_fields_test( $charset, $table, $column ) { + return 'legacy_field' === $column ? 'utf8mb3' : 'utf8mb4'; + } + + public function filter_pre_get_col_charset_utf8_alias_for_encode_emoji_in_fields_test() { + return 'utf8'; + } } diff --git a/tests/phpunit/tests/post.php b/tests/phpunit/tests/post.php index e609fa0d3003d..1d2834fd243ab 100644 --- a/tests/phpunit/tests/post.php +++ b/tests/phpunit/tests/post.php @@ -395,6 +395,46 @@ public function test_utf8mb3_post_saves_with_emoji() { } } + /** + * Forces the utf8mb3 code path via the `pre_get_col_charset` filter, rather + * than relying on the test database actually using that character set, so + * this runs deterministically regardless of the environment's DB charset. + * + * @ticket 65700 + * + * @covers ::wp_insert_post + * @covers ::_wp_encode_emoji_in_fields + */ + public function test_wp_insert_post_encodes_emoji_for_utf8mb3_columns_via_filter(): void { + add_filter( 'pre_get_col_charset', array( $this, 'filter_pre_get_col_charset_utf8mb3_for_post_fields' ), 10, 3 ); + + $post_id = self::factory()->post->create( + array( + 'post_title' => "foo\xf0\x9f\x98\x88bar", + 'post_content' => "foo\xf0\x9f\x98\x8ebaz", + 'post_excerpt' => "foo\xf0\x9f\x98\x90bat", + ) + ); + + remove_filter( 'pre_get_col_charset', array( $this, 'filter_pre_get_col_charset_utf8mb3_for_post_fields' ), 10 ); + + $post = get_post( $post_id ); + + $this->assertSame( 'foo😈bar', $post->post_title ); + $this->assertSame( 'foo😎baz', $post->post_content ); + $this->assertSame( 'foo😐bat', $post->post_excerpt ); + } + + public function filter_pre_get_col_charset_utf8mb3_for_post_fields( $charset, $table, $column ) { + global $wpdb; + + if ( $wpdb->posts === $table && in_array( $column, array( 'post_title', 'post_content', 'post_excerpt' ), true ) ) { + return 'utf8mb3'; + } + + return $charset; + } + /** * If a sticky post is updated via `wp_update_post()` by a user * without the `publish_posts` capability, it should stay sticky. From 4d26752533c73a52089798cbeb5d376a7077fe0a Mon Sep 17 00:00:00 2001 From: barry Date: Thu, 23 Jul 2026 22:44:06 -0500 Subject: [PATCH 02/19] Fix test_wp_encode_emoji_in_fields_ignores_fields_missing_from_data() hitting a real DB query. The test passed a fake 'some_table' table name to _wp_encode_emoji_in_fields() without short-circuiting wpdb::get_col_charset() via the pre_get_col_charset filter, unlike the other tests in this file. This caused a real 'SHOW FULL COLUMNS FROM some_table' query against a table that doesn't exist, failing with a WordPress database error instead of testing the intended behavior (that a field missing from $data is left alone). See #65700. --- tests/phpunit/tests/formatting/emoji.php | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/phpunit/tests/formatting/emoji.php b/tests/phpunit/tests/formatting/emoji.php index 75901004e726a..81c2d5cdd1b14 100644 --- a/tests/phpunit/tests/formatting/emoji.php +++ b/tests/phpunit/tests/formatting/emoji.php @@ -441,8 +441,14 @@ public function test_wp_encode_emoji_in_fields_recognizes_the_utf8_alias(): void public function test_wp_encode_emoji_in_fields_ignores_fields_missing_from_data(): void { $data = array( 'present_field' => '๐Ÿ™‚' ); + // Short-circuits wpdb::get_col_charset() so this doesn't hit a real, + // nonexistent 'some_table' table in the test database. + add_filter( 'pre_get_col_charset', array( $this, 'filter_pre_get_col_charset_utf8mb4_for_encode_emoji_in_fields_test' ) ); + $result = _wp_encode_emoji_in_fields( $data, 'some_table', array( 'present_field', 'absent_field' ) ); + remove_filter( 'pre_get_col_charset', array( $this, 'filter_pre_get_col_charset_utf8mb4_for_encode_emoji_in_fields_test' ) ); + $this->assertArrayNotHasKey( 'absent_field', $result, @@ -457,4 +463,8 @@ public function filter_pre_get_col_charset_for_encode_emoji_in_fields_test( $cha public function filter_pre_get_col_charset_utf8_alias_for_encode_emoji_in_fields_test() { return 'utf8'; } + + public function filter_pre_get_col_charset_utf8mb4_for_encode_emoji_in_fields_test() { + return 'utf8mb4'; + } } From a8076627e13c047e7cbad6dfec5b11403e19bdda Mon Sep 17 00:00:00 2001 From: Jon Surrell Date: Thu, 19 Nov 2026 12:41:13 +0400 Subject: [PATCH 03/19] Update @since for _wp_encode_emoji_in_fields() to 7.2.0. --- src/wp-includes/formatting.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/wp-includes/formatting.php b/src/wp-includes/formatting.php index 13a9375fe788a..63807130d7b01 100644 --- a/src/wp-includes/formatting.php +++ b/src/wp-includes/formatting.php @@ -6202,7 +6202,7 @@ function wp_encode_emoji( $content ) { * `utf8`/`utf8mb3` columns instead of causing truncated or failed writes. * * @access private - * @since 7.1.0 + * @since 7.2.0 * * @global wpdb $wpdb WordPress database abstraction object. * From c1054946fcddf3bf6ecbe788607338c934fcab1b Mon Sep 17 00:00:00 2001 From: Jon Surrell Date: Thu, 19 Nov 2026 12:41:13 +0400 Subject: [PATCH 04/19] Add @ignore to _wp_encode_emoji_in_fields() docblock. --- src/wp-includes/formatting.php | 1 + 1 file changed, 1 insertion(+) diff --git a/src/wp-includes/formatting.php b/src/wp-includes/formatting.php index 63807130d7b01..f43e1f4c2680c 100644 --- a/src/wp-includes/formatting.php +++ b/src/wp-includes/formatting.php @@ -6201,6 +6201,7 @@ function wp_encode_emoji( $content ) { * that emoji (and other 4-byte characters) are stored as HTML entities on * `utf8`/`utf8mb3` columns instead of causing truncated or failed writes. * + * @ignore * @access private * @since 7.2.0 * From 29eb7122d4ad8efc92778a87cbd0b74366225a62 Mon Sep 17 00:00:00 2001 From: Jon Surrell Date: Thu, 19 Nov 2026 15:33:26 +0400 Subject: [PATCH 05/19] Compare emoji in comment lookups as stored. --- src/wp-includes/comment.php | 10 +++++--- tests/phpunit/tests/comment.php | 44 +++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 3 deletions(-) diff --git a/src/wp-includes/comment.php b/src/wp-includes/comment.php index 44498b69195a5..5e2b582a6de5c 100644 --- a/src/wp-includes/comment.php +++ b/src/wp-includes/comment.php @@ -145,6 +145,8 @@ function check_comment( $author, $email, $url, $comment, $user_ip, $user_agent, ) ); } else { + $stored = _wp_encode_emoji_in_fields( array( 'comment_author' => $author ), $wpdb->comments, array( 'comment_author' ) ); + // expected_slashed ($author, $email) $ok_to_comment = $wpdb->get_var( $wpdb->prepare( @@ -154,7 +156,7 @@ function check_comment( $author, $email, $url, $comment, $user_ip, $user_agent, AND comment_author_email = %s AND comment_approved = '1' LIMIT 1", - $author, + $stored['comment_author'], $email ) ); @@ -768,6 +770,8 @@ function sanitize_comment_cookies() { function wp_allow_comment( $commentdata, $wp_error = false ) { global $wpdb; + $stored = _wp_encode_emoji_in_fields( $commentdata, $wpdb->comments, array( 'comment_author', 'comment_content' ) ); + /* * Simple duplicate check. * expected_slashed ($comment_post_ID, $comment_author, $comment_author_email, $comment_content) @@ -776,7 +780,7 @@ function wp_allow_comment( $commentdata, $wp_error = false ) { "SELECT comment_ID FROM $wpdb->comments WHERE comment_post_ID = %d AND comment_parent = %s AND comment_approved != 'trash' AND ( comment_author = %s ", wp_unslash( $commentdata['comment_post_ID'] ), wp_unslash( $commentdata['comment_parent'] ), - wp_unslash( $commentdata['comment_author'] ) + wp_unslash( $stored['comment_author'] ) ); if ( $commentdata['comment_author_email'] ) { $dupe .= $wpdb->prepare( @@ -786,7 +790,7 @@ function wp_allow_comment( $commentdata, $wp_error = false ) { } $dupe .= $wpdb->prepare( ') AND comment_content = %s LIMIT 1', - wp_unslash( $commentdata['comment_content'] ) + wp_unslash( $stored['comment_content'] ) ); $dupe_id = $wpdb->get_var( $dupe ); diff --git a/tests/phpunit/tests/comment.php b/tests/phpunit/tests/comment.php index a463f9ea0dc7c..00f7e3bdf407f 100644 --- a/tests/phpunit/tests/comment.php +++ b/tests/phpunit/tests/comment.php @@ -2648,6 +2648,50 @@ public function test_wp_update_comment_encodes_emoji_for_utf8mb3_columns(): void $this->assertSame( 'foo😎baz', $comment->comment_content ); } + /** + * @ticket 65700 + * + * @covers ::wp_allow_comment + */ + public function test_duplicate_comment_with_emoji_is_rejected_on_utf8mb3(): void { + add_filter( 'pre_get_col_charset', array( $this, 'filter_pre_get_col_charset_utf8mb3_for_comment_fields' ), 10, 3 ); + + $submission = array( + 'comment_post_ID' => self::$post_id, + 'author' => 'Bob ๐Ÿ˜€', + 'email' => 'bob@example.org', + 'comment' => 'Hello ๐Ÿ˜€', + ); + $this->assertInstanceOf( WP_Comment::class, wp_handle_comment_submission( $submission ) ); + + $result = wp_handle_comment_submission( $submission ); + $this->assertWPError( $result ); + $this->assertSame( 'comment_duplicate', $result->get_error_code() ); + } + + /** + * @ticket 65700 + * + * @covers ::check_comment + */ + public function test_previously_approved_author_with_emoji_is_recognized_on_utf8mb3(): void { + add_filter( 'pre_get_col_charset', array( $this, 'filter_pre_get_col_charset_utf8mb3_for_comment_fields' ), 10, 3 ); + update_option( 'comment_previously_approved', '1' ); + + $author = 'Bob ๐Ÿ˜€'; + $email = 'bob@example.org'; + self::factory()->comment->create( + array( + 'comment_post_ID' => self::$post_id, + 'comment_author' => $author, + 'comment_author_email' => $email, + 'comment_approved' => 1, + ) + ); + + $this->assertTrue( check_comment( $author, $email, '', 'Another comment.', '', '', 'comment' ) ); + } + public function filter_pre_get_col_charset_utf8mb3_for_comment_fields( $charset, $table, $column ) { global $wpdb; From a1cea187af1f6dd1bc31cbfe1deb1a848d28992a Mon Sep 17 00:00:00 2001 From: Jon Surrell Date: Thu, 19 Nov 2026 16:41:51 +0400 Subject: [PATCH 06/19] Rename and document the four-byte character field helper. Document that the helper encodes recognized emoji and add a TODO for all four-byte UTF-8 characters. --- src/wp-includes/comment.php | 8 ++--- src/wp-includes/formatting.php | 25 ++++++++-------- src/wp-includes/post.php | 2 +- tests/phpunit/tests/comment.php | 4 +-- tests/phpunit/tests/formatting/emoji.php | 38 ++++++++++++------------ tests/phpunit/tests/post.php | 2 +- 6 files changed, 39 insertions(+), 40 deletions(-) diff --git a/src/wp-includes/comment.php b/src/wp-includes/comment.php index 5e2b582a6de5c..276405607fd58 100644 --- a/src/wp-includes/comment.php +++ b/src/wp-includes/comment.php @@ -145,7 +145,7 @@ function check_comment( $author, $email, $url, $comment, $user_ip, $user_agent, ) ); } else { - $stored = _wp_encode_emoji_in_fields( array( 'comment_author' => $author ), $wpdb->comments, array( 'comment_author' ) ); + $stored = _wp_encode_4byte_chars_in_fields( array( 'comment_author' => $author ), $wpdb->comments, array( 'comment_author' ) ); // expected_slashed ($author, $email) $ok_to_comment = $wpdb->get_var( @@ -770,7 +770,7 @@ function sanitize_comment_cookies() { function wp_allow_comment( $commentdata, $wp_error = false ) { global $wpdb; - $stored = _wp_encode_emoji_in_fields( $commentdata, $wpdb->comments, array( 'comment_author', 'comment_content' ) ); + $stored = _wp_encode_4byte_chars_in_fields( $commentdata, $wpdb->comments, array( 'comment_author', 'comment_content' ) ); /* * Simple duplicate check. @@ -2309,7 +2309,7 @@ function wp_insert_comment( $commentdata ) { 'user_id' ); - $compacted = _wp_encode_emoji_in_fields( $compacted, $wpdb->comments, array( 'comment_author', 'comment_content' ) ); + $compacted = _wp_encode_4byte_chars_in_fields( $compacted, $wpdb->comments, array( 'comment_author', 'comment_content' ) ); if ( ! $wpdb->insert( $wpdb->comments, $compacted ) ) { return false; @@ -3099,7 +3099,7 @@ function wp_update_comment( $commentarr, $wp_error = false ) { $data = wp_array_slice_assoc( $data, $keys ); - $data = _wp_encode_emoji_in_fields( $data, $wpdb->comments, array( 'comment_author', 'comment_content' ) ); + $data = _wp_encode_4byte_chars_in_fields( $data, $wpdb->comments, array( 'comment_author', 'comment_content' ) ); $result = $wpdb->update( $wpdb->comments, $data, array( 'comment_ID' => $comment_id ) ); diff --git a/src/wp-includes/formatting.php b/src/wp-includes/formatting.php index f43e1f4c2680c..077dd90d7f628 100644 --- a/src/wp-includes/formatting.php +++ b/src/wp-includes/formatting.php @@ -6194,26 +6194,25 @@ function wp_encode_emoji( $content ) { } /** - * Encodes emoji in the given fields of a data array whose corresponding - * database columns cannot natively store 4-byte UTF-8 characters. + * Replaces recognized emoji with hexadecimal HTML numeric character references in fields + * whose database columns use utf8 or utf8mb3. * - * Used by wp_insert_post(), wp_insert_comment(), and wp_update_comment() so - * that emoji (and other 4-byte characters) are stored as HTML entities on - * `utf8`/`utf8mb3` columns instead of causing truncated or failed writes. + * Use for fields where consumers interpret HTML character references. + * + * @todo Encode all four-byte UTF-8 characters. * - * @ignore - * @access private * @since 7.2.0 + * @access private + * @ignore * * @global wpdb $wpdb WordPress database abstraction object. * - * @param array $data Array of field name => value pairs to check, keyed the - * same way as the corresponding database columns. - * @param string $table Name of the database table the fields belong to. - * @param array $fields Names of the fields within $data that may contain emoji. - * @return array $data with emoji encoded in the given fields, where needed. + * @param array $data Field values keyed by column name. + * @param string $table Database table name. + * @param string[] $fields Names of the fields to check. + * @return array Field values with recognized emoji encoded where needed. */ -function _wp_encode_emoji_in_fields( $data, $table, $fields ) { +function _wp_encode_4byte_chars_in_fields( $data, $table, $fields ) { global $wpdb; foreach ( $fields as $field ) { diff --git a/src/wp-includes/post.php b/src/wp-includes/post.php index d1cab1a8c4793..53903e220b295 100644 --- a/src/wp-includes/post.php +++ b/src/wp-includes/post.php @@ -5174,7 +5174,7 @@ function wp_insert_post( $postarr, $wp_error = false, $fire_after_hooks = true ) 'guid' ); - $data = _wp_encode_emoji_in_fields( $data, $wpdb->posts, array( 'post_title', 'post_content', 'post_excerpt' ) ); + $data = _wp_encode_4byte_chars_in_fields( $data, $wpdb->posts, array( 'post_title', 'post_content', 'post_excerpt' ) ); if ( 'attachment' === $post_type ) { /** diff --git a/tests/phpunit/tests/comment.php b/tests/phpunit/tests/comment.php index 00f7e3bdf407f..757613c902f98 100644 --- a/tests/phpunit/tests/comment.php +++ b/tests/phpunit/tests/comment.php @@ -2569,7 +2569,7 @@ public function test_get_comment_should_only_treat_numeric_values_as_comment_ids * @ticket 65700 * * @covers ::wp_insert_comment - * @covers ::_wp_encode_emoji_in_fields + * @covers ::_wp_encode_4byte_chars_in_fields */ public function test_wp_insert_comment_encodes_emoji_for_utf8mb3_columns(): void { add_filter( 'pre_get_col_charset', array( $this, 'filter_pre_get_col_charset_utf8mb3_for_comment_fields' ), 10, 3 ); @@ -2623,7 +2623,7 @@ public function test_wp_insert_comment_does_not_encode_emoji_for_utf8mb4_columns * @ticket 65700 * * @covers ::wp_update_comment - * @covers ::_wp_encode_emoji_in_fields + * @covers ::_wp_encode_4byte_chars_in_fields */ public function test_wp_update_comment_encodes_emoji_for_utf8mb3_columns(): void { $comment_id = self::factory()->comment->create( array( 'comment_post_ID' => self::$post_id ) ); diff --git a/tests/phpunit/tests/formatting/emoji.php b/tests/phpunit/tests/formatting/emoji.php index 7f637cd5caac7..0fd653854f584 100644 --- a/tests/phpunit/tests/formatting/emoji.php +++ b/tests/phpunit/tests/formatting/emoji.php @@ -380,20 +380,20 @@ public function test_wp_staticize_emoji( $emoji, $expected ) { /** * @ticket 65700 * - * @covers ::_wp_encode_emoji_in_fields + * @covers ::_wp_encode_4byte_chars_in_fields */ - public function test_wp_encode_emoji_in_fields_only_touches_requested_utf8mb3_fields(): void { + public function test_wp_encode_4byte_chars_in_fields_only_touches_requested_utf8mb3_fields(): void { $data = array( 'legacy_field' => '๐Ÿ™‚', 'modern_field' => '๐Ÿ™‚', 'ignored_field' => '๐Ÿ™‚', ); - add_filter( 'pre_get_col_charset', array( $this, 'filter_pre_get_col_charset_for_encode_emoji_in_fields_test' ), 10, 3 ); + add_filter( 'pre_get_col_charset', array( $this, 'filter_pre_get_col_charset_for_encode_4byte_chars_in_fields_test' ), 10, 3 ); - $result = _wp_encode_emoji_in_fields( $data, 'some_table', array( 'legacy_field', 'modern_field' ) ); + $result = _wp_encode_4byte_chars_in_fields( $data, 'some_table', array( 'legacy_field', 'modern_field' ) ); - remove_filter( 'pre_get_col_charset', array( $this, 'filter_pre_get_col_charset_for_encode_emoji_in_fields_test' ), 10 ); + remove_filter( 'pre_get_col_charset', array( $this, 'filter_pre_get_col_charset_for_encode_4byte_chars_in_fields_test' ), 10 ); $this->assertSame( '🙂', @@ -415,16 +415,16 @@ public function test_wp_encode_emoji_in_fields_only_touches_requested_utf8mb3_fi /** * @ticket 65700 * - * @covers ::_wp_encode_emoji_in_fields + * @covers ::_wp_encode_4byte_chars_in_fields */ - public function test_wp_encode_emoji_in_fields_recognizes_the_utf8_alias(): void { + public function test_wp_encode_4byte_chars_in_fields_recognizes_the_utf8_alias(): void { $data = array( 'legacy_field' => '๐Ÿ™‚' ); - add_filter( 'pre_get_col_charset', array( $this, 'filter_pre_get_col_charset_utf8_alias_for_encode_emoji_in_fields_test' ) ); + add_filter( 'pre_get_col_charset', array( $this, 'filter_pre_get_col_charset_utf8_alias_for_encode_4byte_chars_in_fields_test' ) ); - $result = _wp_encode_emoji_in_fields( $data, 'some_table', array( 'legacy_field' ) ); + $result = _wp_encode_4byte_chars_in_fields( $data, 'some_table', array( 'legacy_field' ) ); - remove_filter( 'pre_get_col_charset', array( $this, 'filter_pre_get_col_charset_utf8_alias_for_encode_emoji_in_fields_test' ) ); + remove_filter( 'pre_get_col_charset', array( $this, 'filter_pre_get_col_charset_utf8_alias_for_encode_4byte_chars_in_fields_test' ) ); $this->assertSame( '🙂', @@ -436,35 +436,35 @@ public function test_wp_encode_emoji_in_fields_recognizes_the_utf8_alias(): void /** * @ticket 65700 * - * @covers ::_wp_encode_emoji_in_fields + * @covers ::_wp_encode_4byte_chars_in_fields */ - public function test_wp_encode_emoji_in_fields_ignores_fields_missing_from_data(): void { + public function test_wp_encode_4byte_chars_in_fields_ignores_fields_missing_from_data(): void { $data = array( 'present_field' => '๐Ÿ™‚' ); // Short-circuits wpdb::get_col_charset() so this doesn't hit a real, // nonexistent 'some_table' table in the test database. - add_filter( 'pre_get_col_charset', array( $this, 'filter_pre_get_col_charset_utf8mb4_for_encode_emoji_in_fields_test' ) ); + add_filter( 'pre_get_col_charset', array( $this, 'filter_pre_get_col_charset_utf8mb4_for_encode_4byte_chars_in_fields_test' ) ); - $result = _wp_encode_emoji_in_fields( $data, 'some_table', array( 'present_field', 'absent_field' ) ); + $result = _wp_encode_4byte_chars_in_fields( $data, 'some_table', array( 'present_field', 'absent_field' ) ); - remove_filter( 'pre_get_col_charset', array( $this, 'filter_pre_get_col_charset_utf8mb4_for_encode_emoji_in_fields_test' ) ); + remove_filter( 'pre_get_col_charset', array( $this, 'filter_pre_get_col_charset_utf8mb4_for_encode_4byte_chars_in_fields_test' ) ); $this->assertArrayNotHasKey( 'absent_field', $result, - 'Expected a field missing from $data to not be added by _wp_encode_emoji_in_fields().' + 'Expected a field missing from $data to not be added by _wp_encode_4byte_chars_in_fields().' ); } - public function filter_pre_get_col_charset_for_encode_emoji_in_fields_test( $charset, $table, $column ) { + public function filter_pre_get_col_charset_for_encode_4byte_chars_in_fields_test( $charset, $table, $column ) { return 'legacy_field' === $column ? 'utf8mb3' : 'utf8mb4'; } - public function filter_pre_get_col_charset_utf8_alias_for_encode_emoji_in_fields_test() { + public function filter_pre_get_col_charset_utf8_alias_for_encode_4byte_chars_in_fields_test() { return 'utf8'; } - public function filter_pre_get_col_charset_utf8mb4_for_encode_emoji_in_fields_test() { + public function filter_pre_get_col_charset_utf8mb4_for_encode_4byte_chars_in_fields_test() { return 'utf8mb4'; } } diff --git a/tests/phpunit/tests/post.php b/tests/phpunit/tests/post.php index 9e807080cd727..23b2cc23e1ab3 100644 --- a/tests/phpunit/tests/post.php +++ b/tests/phpunit/tests/post.php @@ -404,7 +404,7 @@ public function test_utf8mb3_post_saves_with_emoji() { * @ticket 65700 * * @covers ::wp_insert_post - * @covers ::_wp_encode_emoji_in_fields + * @covers ::_wp_encode_4byte_chars_in_fields */ public function test_wp_insert_post_encodes_emoji_for_utf8mb3_columns_via_filter(): void { add_filter( 'pre_get_col_charset', array( $this, 'filter_pre_get_col_charset_utf8mb3_for_post_fields' ), 10, 3 ); From ff855fdc6d431a97ae78391acd2c9c5a3e9a2ef2 Mon Sep 17 00:00:00 2001 From: Jon Surrell Date: Thu, 19 Nov 2026 17:04:24 +0400 Subject: [PATCH 07/19] Encode four-byte UTF-8 characters in utf8mb3 post and comment fields. Use lowercase hexadecimal character references for all four-byte UTF-8 characters. This preserves supplementary characters outside the emoji list and leaves characters that fit in utf8mb3 intact. Match valid sequences even when the field also contains malformed bytes. --- src/wp-includes/formatting.php | 47 +++++++++++++++++++----- tests/phpunit/tests/formatting/emoji.php | 16 ++++---- tests/phpunit/tests/post.php | 6 +-- 3 files changed, 48 insertions(+), 21 deletions(-) diff --git a/src/wp-includes/formatting.php b/src/wp-includes/formatting.php index 077dd90d7f628..596477b336380 100644 --- a/src/wp-includes/formatting.php +++ b/src/wp-includes/formatting.php @@ -6194,12 +6194,12 @@ function wp_encode_emoji( $content ) { } /** - * Replaces recognized emoji with hexadecimal HTML numeric character references in fields - * whose database columns use utf8 or utf8mb3. + * Replaces four-byte UTF-8 characters with HTML character references in fields whose database + * columns use `utf8` or `utf8mb3`. * * Use for fields where consumers interpret HTML character references. * - * @todo Encode all four-byte UTF-8 characters. + * @link https://dev.mysql.com/doc/refman/8.4/en/charset-unicode-utf8.html * * @since 7.2.0 * @access private @@ -6210,19 +6210,46 @@ function wp_encode_emoji( $content ) { * @param array $data Field values keyed by column name. * @param string $table Database table name. * @param string[] $fields Names of the fields to check. - * @return array Field values with recognized emoji encoded where needed. + * @return array Field values with four-byte characters encoded where needed. */ function _wp_encode_4byte_chars_in_fields( $data, $table, $fields ) { global $wpdb; foreach ( $fields as $field ) { - if ( isset( $data[ $field ] ) ) { - $charset = $wpdb->get_col_charset( $table, $field ); + if ( ! isset( $data[ $field ] ) || ! is_string( $data[ $field ] ) ) { + continue; + } - // The 'utf8' character set is a deprecated alias of 'utf8mb3'. See . - if ( 'utf8' === $charset || 'utf8mb3' === $charset ) { - $data[ $field ] = wp_encode_emoji( $data[ $field ] ); - } + $charset = $wpdb->get_col_charset( $table, $field ); + + /* + * Note that the `utf8` character set is a deprecated alias of `utf8mb3`. + * + * Table 3-7 of the Unicode Standard gives these ranges for four-byte UTF-8 sequences. + * + * โ•ญโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ฌโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ฌโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ฌโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ฌโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฎ + * โ”‚ Code Points โ”‚ First Byte โ”‚ Second Byte โ”‚ Third Byte โ”‚ Fourth Byte โ”‚ + * โ”œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ผโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ผโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ผโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ผโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ค + * โ”‚ U+10000..U+3FFFF โ”‚ F0 โ”‚ 90..BF โ”‚ 80..BF โ”‚ 80..BF โ”‚ + * โ”‚ U+40000..U+FFFFF โ”‚ F1..F3 โ”‚ 80..BF โ”‚ 80..BF โ”‚ 80..BF โ”‚ + * โ”‚ U+100000..U+10FFFF โ”‚ F4 โ”‚ 80..8F โ”‚ 80..BF โ”‚ 80..BF โ”‚ + * โ•ฐโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ดโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ดโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ดโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ดโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฏ + * + * @link https://www.unicode.org/versions/Unicode16.0.0/core-spec/chapter-3/#G27506 + */ + if ( 'utf8' === $charset || 'utf8mb3' === $charset ) { + $data[ $field ] = preg_replace_callback( + '/ + # Match bytes so malformed UTF-8 elsewhere in the field does not prevent a match. + \xF0 [\x90-\xBF] [\x80-\xBF] [\x80-\xBF] # U+10000..U+3FFFF + | [\xF1-\xF3] [\x80-\xBF] [\x80-\xBF] [\x80-\xBF] # U+40000..U+FFFFF + | \xF4 [\x80-\x8F] [\x80-\xBF] [\x80-\xBF] # U+100000..U+10FFFF + /x', + static function ( $match ) { + return sprintf( '&#x%x;', mb_ord( $match[0], 'UTF-8' ) ); + }, + $data[ $field ] + ) ?? $data[ $field ]; } } diff --git a/tests/phpunit/tests/formatting/emoji.php b/tests/phpunit/tests/formatting/emoji.php index 0fd653854f584..52120a2d0174d 100644 --- a/tests/phpunit/tests/formatting/emoji.php +++ b/tests/phpunit/tests/formatting/emoji.php @@ -384,9 +384,9 @@ public function test_wp_staticize_emoji( $emoji, $expected ) { */ public function test_wp_encode_4byte_chars_in_fields_only_touches_requested_utf8mb3_fields(): void { $data = array( - 'legacy_field' => '๐Ÿ™‚', - 'modern_field' => '๐Ÿ™‚', - 'ignored_field' => '๐Ÿ™‚', + 'legacy_field' => 'โ„ข๐Ÿ™‚๐ ฎท', + 'modern_field' => 'โ„ข๐Ÿ™‚๐ ฎท', + 'ignored_field' => 'โ„ข๐Ÿ™‚๐ ฎท', ); add_filter( 'pre_get_col_charset', array( $this, 'filter_pre_get_col_charset_for_encode_4byte_chars_in_fields_test' ), 10, 3 ); @@ -396,17 +396,17 @@ public function test_wp_encode_4byte_chars_in_fields_only_touches_requested_utf8 remove_filter( 'pre_get_col_charset', array( $this, 'filter_pre_get_col_charset_for_encode_4byte_chars_in_fields_test' ), 10 ); $this->assertSame( - '🙂', + 'โ„ข🙂𠮷', $result['legacy_field'], - 'Expected emoji to be encoded for a field on a utf8/utf8mb3 column.' + 'Expected four-byte characters to be encoded for a field on a utf8/utf8mb3 column.' ); $this->assertSame( - '๐Ÿ™‚', + 'โ„ข๐Ÿ™‚๐ ฎท', $result['modern_field'], - 'Expected emoji to be left alone for a field on a utf8mb4 column.' + 'Expected four-byte characters to be left alone for a field on a utf8mb4 column.' ); $this->assertSame( - '๐Ÿ™‚', + 'โ„ข๐Ÿ™‚๐ ฎท', $result['ignored_field'], 'Expected a field not passed in $fields to be left alone entirely.' ); diff --git a/tests/phpunit/tests/post.php b/tests/phpunit/tests/post.php index 23b2cc23e1ab3..c7204c8a03872 100644 --- a/tests/phpunit/tests/post.php +++ b/tests/phpunit/tests/post.php @@ -406,12 +406,12 @@ public function test_utf8mb3_post_saves_with_emoji() { * @covers ::wp_insert_post * @covers ::_wp_encode_4byte_chars_in_fields */ - public function test_wp_insert_post_encodes_emoji_for_utf8mb3_columns_via_filter(): void { + public function test_wp_insert_post_encodes_four_byte_characters_for_utf8mb3_columns_via_filter(): void { add_filter( 'pre_get_col_charset', array( $this, 'filter_pre_get_col_charset_utf8mb3_for_post_fields' ), 10, 3 ); $post_id = self::factory()->post->create( array( - 'post_title' => "foo\xf0\x9f\x98\x88bar", + 'post_title' => 'Acmeโ„ข ๐ ฎท', 'post_content' => "foo\xf0\x9f\x98\x8ebaz", 'post_excerpt' => "foo\xf0\x9f\x98\x90bat", ) @@ -421,7 +421,7 @@ public function test_wp_insert_post_encodes_emoji_for_utf8mb3_columns_via_filter $post = get_post( $post_id ); - $this->assertSame( 'foo😈bar', $post->post_title ); + $this->assertSame( 'Acmeโ„ข 𠮷', $post->post_title ); $this->assertSame( 'foo😎baz', $post->post_content ); $this->assertSame( 'foo😐bat', $post->post_excerpt ); } From 0ae0b3fe38edcae8340bf43954fb5a0e3414bcf2 Mon Sep 17 00:00:00 2001 From: Jon Surrell Date: Thu, 19 Nov 2026 17:10:51 +0400 Subject: [PATCH 08/19] Assert comment insertion succeeds in utf8mb4 control test. --- tests/phpunit/tests/comment.php | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/phpunit/tests/comment.php b/tests/phpunit/tests/comment.php index 757613c902f98..d72f1396f6cdd 100644 --- a/tests/phpunit/tests/comment.php +++ b/tests/phpunit/tests/comment.php @@ -2613,6 +2613,8 @@ public function test_wp_insert_comment_does_not_encode_emoji_for_utf8mb4_columns remove_filter( 'pre_get_col_charset', array( $this, 'filter_pre_get_col_charset_utf8mb4_for_comment_fields' ), 10 ); + $this->assertIsInt( $comment_id, 'Expected wp_insert_comment() to return the new comment ID.' ); + $comment = get_comment( $comment_id ); $this->assertSame( "foo\xf0\x9f\x98\x88bar", $comment->comment_author ); From fd913e7b455c166c74c8103687489d2ab8d87bd9 Mon Sep 17 00:00:00 2001 From: Jon Surrell Date: Thu, 19 Nov 2026 17:24:55 +0400 Subject: [PATCH 09/19] Compare comment authors as stored in comment_exists(). --- src/wp-admin/includes/comment.php | 4 +++- tests/phpunit/tests/comment.php | 21 +++++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/src/wp-admin/includes/comment.php b/src/wp-admin/includes/comment.php index 5f46be6a5cdfa..746c5da4412c7 100644 --- a/src/wp-admin/includes/comment.php +++ b/src/wp-admin/includes/comment.php @@ -31,11 +31,13 @@ function comment_exists( $comment_author, $comment_date, $timezone = 'blog' ) { $date_field = 'comment_date_gmt'; } + $stored = _wp_encode_4byte_chars_in_fields( array( 'comment_author' => stripslashes( $comment_author ) ), $wpdb->comments, array( 'comment_author' ) ); + return $wpdb->get_var( $wpdb->prepare( "SELECT comment_post_ID FROM $wpdb->comments WHERE comment_author = %s AND $date_field = %s", - stripslashes( $comment_author ), + $stored['comment_author'], stripslashes( $comment_date ) ) ); diff --git a/tests/phpunit/tests/comment.php b/tests/phpunit/tests/comment.php index d72f1396f6cdd..1cc05b1380deb 100644 --- a/tests/phpunit/tests/comment.php +++ b/tests/phpunit/tests/comment.php @@ -2694,6 +2694,27 @@ public function test_previously_approved_author_with_emoji_is_recognized_on_utf8 $this->assertTrue( check_comment( $author, $email, '', 'Another comment.', '', '', 'comment' ) ); } + /** + * @ticket 65700 + * + * @covers ::comment_exists + */ + public function test_comment_exists_finds_author_with_emoji_on_utf8mb3(): void { + add_filter( 'pre_get_col_charset', array( $this, 'filter_pre_get_col_charset_utf8mb3_for_comment_fields' ), 10, 3 ); + + $author = "O'Brien ๐Ÿ˜€"; + $date = '2014-05-06 12:00:00'; + wp_insert_comment( + array( + 'comment_post_ID' => self::$post_id, + 'comment_author' => wp_slash( $author ), + 'comment_date' => $date, + ) + ); + + $this->assertSame( (string) self::$post_id, comment_exists( wp_slash( $author ), $date ) ); + } + public function filter_pre_get_col_charset_utf8mb3_for_comment_fields( $charset, $table, $column ) { global $wpdb; From 4141e89592da486e8765b07fee7e52b322d2f3c5 Mon Sep 17 00:00:00 2001 From: Jon Surrell Date: Thu, 19 Nov 2026 17:26:49 +0400 Subject: [PATCH 10/19] Preserve encodable comment fields when retrying an insert. Check whether the encoded field fits before stripping the submitted text. This lets the retry clean an unrelated field while passing the original characters to moderation filters. Fields whose stored form needs cleanup still use the existing retry behavior. --- src/wp-includes/comment.php | 15 +++++++++++-- tests/phpunit/tests/comment.php | 38 +++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 2 deletions(-) diff --git a/src/wp-includes/comment.php b/src/wp-includes/comment.php index 276405607fd58..65cf8f79f3b06 100644 --- a/src/wp-includes/comment.php +++ b/src/wp-includes/comment.php @@ -2568,11 +2568,22 @@ function wp_new_comment( $commentdata, $wp_error = false ) { if ( ! $comment_id ) { $fields = array( 'comment_author', 'comment_author_email', 'comment_author_url', 'comment_content' ); + $stored = _wp_encode_4byte_chars_in_fields( $commentdata, $wpdb->comments, array( 'comment_author', 'comment_content' ) ); foreach ( $fields as $field ) { - if ( isset( $commentdata[ $field ] ) ) { - $commentdata[ $field ] = $wpdb->strip_invalid_text_for_column( $wpdb->comments, $field, $commentdata[ $field ] ); + if ( ! isset( $commentdata[ $field ] ) ) { + continue; + } + + // Keep the submitted characters for comment filters when their stored form fits the column. + if ( $stored[ $field ] !== $commentdata[ $field ] ) { + $value = wp_unslash( $stored[ $field ] ); + if ( $value === $wpdb->strip_invalid_text_for_column( $wpdb->comments, $field, $value ) ) { + continue; + } } + + $commentdata[ $field ] = $wpdb->strip_invalid_text_for_column( $wpdb->comments, $field, $commentdata[ $field ] ); } $commentdata = wp_filter_comment( $commentdata ); diff --git a/tests/phpunit/tests/comment.php b/tests/phpunit/tests/comment.php index 1cc05b1380deb..6530924aac283 100644 --- a/tests/phpunit/tests/comment.php +++ b/tests/phpunit/tests/comment.php @@ -2715,6 +2715,44 @@ public function test_comment_exists_finds_author_with_emoji_on_utf8mb3(): void { $this->assertSame( (string) self::$post_id, comment_exists( wp_slash( $author ), $date ) ); } + /** + * @ticket 65700 + * + * @covers ::wp_new_comment + */ + public function test_wp_new_comment_preserves_emoji_when_another_field_needs_cleanup(): void { + add_filter( 'pre_get_col_charset', array( $this, 'filter_pre_get_col_charset_utf8mb3_for_comment_fields' ), 10, 3 ); + + $commentdata = array( + 'comment_post_ID' => self::$post_id, + 'comment_author' => 'Ann ๐Ÿ˜€', + 'comment_author_email' => 'ann@example.org', + 'comment_author_url' => 'https://example.org/' . str_repeat( 'a', 181 ), + 'comment_content' => 'Hello ๐Ÿ˜Ž', + ); + + $approved_data = null; + add_filter( + 'pre_comment_approved', + static function ( $approved, $data ) use ( &$approved_data ) { + $approved_data = $data; + return $approved; + }, + 10, + 2 + ); + + $comment_id = wp_new_comment( $commentdata, true ); + $this->assertIsInt( $comment_id ); + $comment = get_comment( $comment_id ); + + $this->assertSame( substr( $commentdata['comment_author_url'], 0, 200 ), $comment->comment_author_url ); + $this->assertSame( 'Ann 😀', $comment->comment_author ); + $this->assertSame( 'Hello 😎', $comment->comment_content ); + $this->assertSame( $commentdata['comment_author'], $approved_data['comment_author'] ); + $this->assertSame( $commentdata['comment_content'], $approved_data['comment_content'] ); + } + public function filter_pre_get_col_charset_utf8mb3_for_comment_fields( $charset, $table, $column ) { global $wpdb; From 032e0e90d85eb87c95aecb74d2379a90ec587ee0 Mon Sep 17 00:00:00 2001 From: Jon Surrell Date: Thu, 19 Nov 2026 17:39:17 +0400 Subject: [PATCH 11/19] Render emoji sequences that mix literal characters and character references. Encode literal emoji before checking for hexadecimal character references. In utf8mb3 posts, a family can contain four-byte characters stored as references and three-byte joiners stored literally. Render the family as one image. A separate reference in utf8mb4 content also permits literal emoji to render as images. This change accompanies the four-byte storage encoder in ff855fdc6d. If the PR uses wp_encode_emoji() for storage again, both changes can be reverted together. --- src/wp-includes/formatting.php | 18 +++++++----------- tests/phpunit/tests/formatting/emoji.php | 12 ++++++++++++ tests/phpunit/tests/post.php | 20 ++++++++++++++++++++ 3 files changed, 39 insertions(+), 11 deletions(-) diff --git a/src/wp-includes/formatting.php b/src/wp-includes/formatting.php index 596477b336380..28cdb475c482e 100644 --- a/src/wp-includes/formatting.php +++ b/src/wp-includes/formatting.php @@ -6265,18 +6265,14 @@ static function ( $match ) { * @return string The encoded content. */ function wp_staticize_emoji( $text ) { - if ( ! str_contains( $text, '&#x' ) ) { - if ( ( function_exists( 'mb_check_encoding' ) && mb_check_encoding( $text, 'ASCII' ) ) || ! preg_match( '/[^\x00-\x7F]/', $text ) ) { - // The text doesn't contain anything that might be emoji, so we can return early. - return $text; - } else { - $encoded_text = wp_encode_emoji( $text ); - if ( $encoded_text === $text ) { - return $encoded_text; - } + $is_ascii = function_exists( 'mb_check_encoding' ) ? mb_check_encoding( $text, 'ASCII' ) : ! preg_match( '/[^\x00-\x7F]/', $text ); + if ( ! $is_ascii ) { + // An emoji sequence can contain both literal characters and character references. + $text = wp_encode_emoji( $text ); + } - $text = $encoded_text; - } + if ( ! str_contains( $text, '&#x' ) ) { + return $text; } $emoji = _wp_emoji_list( 'entities' ); diff --git a/tests/phpunit/tests/formatting/emoji.php b/tests/phpunit/tests/formatting/emoji.php index 52120a2d0174d..7081a862e9c62 100644 --- a/tests/phpunit/tests/formatting/emoji.php +++ b/tests/phpunit/tests/formatting/emoji.php @@ -347,6 +347,18 @@ public static function data_wp_staticize_emoji() { '๐Ÿ™‚', '๐Ÿ™‚', ), + 'four-byte emoji beside a character reference' => array( + '๐Ÿ˜€ A', + '๐Ÿ˜€ A', + ), + 'three-byte emoji beside a character reference' => array( + 'โŒš A', + 'โŒš A', + ), + 'family with literal joiners between character references' => array( + "👨\u{200D}👩\u{200D}👧", + '๐Ÿ‘จโ€๐Ÿ‘ฉโ€๐Ÿ‘ง', + ), array( // Skin tone, gender, ZWJ, emoji selector. '๐Ÿ‘ฎ๐Ÿผโ€โ™€๏ธ', diff --git a/tests/phpunit/tests/post.php b/tests/phpunit/tests/post.php index c7204c8a03872..fe254b15dc3cc 100644 --- a/tests/phpunit/tests/post.php +++ b/tests/phpunit/tests/post.php @@ -426,6 +426,26 @@ public function test_wp_insert_post_encodes_four_byte_characters_for_utf8mb3_col $this->assertSame( 'foo😐bat', $post->post_excerpt ); } + /** + * @ticket 65700 + * + * @covers ::wp_insert_post + * @covers ::wp_staticize_emoji + */ + public function test_staticize_emoji_in_post_content_stored_as_utf8mb3(): void { + $content = '๐Ÿ‘จโ€๐Ÿ‘ฉโ€๐Ÿ‘ง โค๏ธ'; + + add_filter( 'pre_get_col_charset', array( $this, 'filter_pre_get_col_charset_utf8mb3_for_post_fields' ), 10, 3 ); + $post_id = wp_insert_post( array( 'post_content' => $content ), true ); + remove_filter( 'pre_get_col_charset', array( $this, 'filter_pre_get_col_charset_utf8mb3_for_post_fields' ), 10 ); + + $this->assertNotWPError( $post_id ); + $stored = get_post( $post_id )->post_content; + + $this->assertSame( "👨\u{200D}👩\u{200D}👧 โค๏ธ", $stored ); + $this->assertSame( wp_staticize_emoji( $content ), wp_staticize_emoji( $stored ) ); + } + public function filter_pre_get_col_charset_utf8mb3_for_post_fields( $charset, $table, $column ) { global $wpdb; From ce022c12dbd7bb9fcbcc974d5eff0b9e61293731 Mon Sep 17 00:00:00 2001 From: Jon Surrell Date: Thu, 19 Nov 2026 19:48:04 +0400 Subject: [PATCH 12/19] Validate the stored length of comment authors and content. Keep the submitted field limits and check the encoded value against the column capacity. Reject a filtered value that exceeds the column before the insert retry can strip four-byte characters. --- src/wp-includes/comment.php | 63 ++++++++++++++++++++++++---- tests/phpunit/tests/comment.php | 73 +++++++++++++++++++++++++++++++++ 2 files changed, 129 insertions(+), 7 deletions(-) diff --git a/src/wp-includes/comment.php b/src/wp-includes/comment.php index 65cf8f79f3b06..fba0f74169970 100644 --- a/src/wp-includes/comment.php +++ b/src/wp-includes/comment.php @@ -1326,15 +1326,18 @@ function wp_get_comment_fields_max_lengths() { } /** - * Compares the lengths of comment data against the maximum character limits. + * Checks submitted comment field lengths and the storage limits for the author and content. * * @since 4.7.0 * - * @param array $comment_data Array of arguments for inserting a comment. + * @global wpdb $wpdb WordPress database abstraction object. + * @param array $comment_data Array of unslashed arguments for inserting a comment. * @return WP_Error|true WP_Error when a comment field exceeds the limit, * otherwise true. */ function wp_check_comment_data_max_lengths( $comment_data ) { + global $wpdb; + $max_lengths = wp_get_comment_fields_max_lengths(); if ( isset( $comment_data['comment_author'] ) && mb_strlen( $comment_data['comment_author'], '8bit' ) > $max_lengths['comment_author'] ) { @@ -1353,6 +1356,46 @@ function wp_check_comment_data_max_lengths( $comment_data ) { return new WP_Error( 'comment_content_column_length', __( 'Error: Your comment is too long.' ), 200 ); } + $stored = _wp_encode_4byte_chars_in_fields( $comment_data, $wpdb->comments, array( 'comment_author', 'comment_content' ) ); + + return _wp_check_comment_data_storage_lengths( $stored ); +} + +/** + * Checks the column lengths of the author and content prepared for storage. + * + * @since 7.2.0 + * @access private + * @ignore + * @global wpdb $wpdb WordPress database abstraction object. + * @param array $stored Array of unslashed comment data prepared for storage. + * @return WP_Error|true WP_Error when the author or content exceeds its column length, otherwise true. + */ +function _wp_check_comment_data_storage_lengths( $stored ) { + global $wpdb; + + foreach ( array( 'comment_author', 'comment_content' ) as $field ) { + if ( ! isset( $stored[ $field ] ) || ! is_string( $stored[ $field ] ) ) { + continue; + } + + $col_length = $wpdb->get_col_length( $wpdb->comments, $field ); + if ( ! is_array( $col_length ) ) { + continue; + } + + $length = 'byte' === $col_length['type'] ? strlen( $stored[ $field ] ) : mb_strlen( $stored[ $field ], 'UTF-8' ); + if ( $length <= $col_length['length'] ) { + continue; + } + + if ( 'comment_author' === $field ) { + return new WP_Error( 'comment_author_column_length', __( 'Error: Your name is too long.' ), 200 ); + } + + return new WP_Error( 'comment_content_column_length', __( 'Error: Your comment is too long.' ), 200 ); + } + return true; } @@ -2567,17 +2610,23 @@ function wp_new_comment( $commentdata, $wp_error = false ) { $comment_id = wp_insert_comment( $commentdata ); if ( ! $comment_id ) { - $fields = array( 'comment_author', 'comment_author_email', 'comment_author_url', 'comment_content' ); - $stored = _wp_encode_4byte_chars_in_fields( $commentdata, $wpdb->comments, array( 'comment_author', 'comment_content' ) ); + $fields = array( 'comment_author', 'comment_author_email', 'comment_author_url', 'comment_content' ); + $unslashed = wp_unslash( $commentdata ); + $stored = _wp_encode_4byte_chars_in_fields( $unslashed, $wpdb->comments, array( 'comment_author', 'comment_content' ) ); + + $check_storage_lengths = _wp_check_comment_data_storage_lengths( $stored ); + if ( is_wp_error( $check_storage_lengths ) ) { + return $wp_error ? $check_storage_lengths : false; + } foreach ( $fields as $field ) { if ( ! isset( $commentdata[ $field ] ) ) { continue; } - // Keep the submitted characters for comment filters when their stored form fits the column. - if ( $stored[ $field ] !== $commentdata[ $field ] ) { - $value = wp_unslash( $stored[ $field ] ); + // Keep the original characters for comment filters when the encoded value is valid. + if ( $stored[ $field ] !== $unslashed[ $field ] ) { + $value = $stored[ $field ]; if ( $value === $wpdb->strip_invalid_text_for_column( $wpdb->comments, $field, $value ) ) { continue; } diff --git a/tests/phpunit/tests/comment.php b/tests/phpunit/tests/comment.php index 6530924aac283..70112d65aa849 100644 --- a/tests/phpunit/tests/comment.php +++ b/tests/phpunit/tests/comment.php @@ -2592,6 +2592,79 @@ public function test_wp_insert_comment_encodes_emoji_for_utf8mb3_columns(): void $this->assertSame( 'foo😎baz', $comment->comment_content ); } + /** + * @ticket 65700 + * @covers ::wp_handle_comment_submission + * @covers ::wp_check_comment_data_max_lengths + * + * @dataProvider data_comment_encoded_length + * @param string $input Submission field name. + * @param string $column Comment table column name. + * @param string $prefix ASCII prefix before the four-byte characters. + * @param int $count Number of four-byte characters. + * @param bool $should_insert Whether the encoded value fits in the column. + */ + public function test_wp_handle_comment_submission_checks_encoded_length( $input, $column, $prefix, $count, $should_insert ): void { + add_filter( 'pre_get_col_charset', array( $this, 'filter_pre_get_col_charset_utf8mb3_for_comment_fields' ), 10, 3 ); + + $submission = array( + 'comment_post_ID' => self::$post_id, + 'author' => 'Comment author', + 'email' => 'author@example.org', + 'comment' => 'Comment content.', + ); + + $submission[ $input ] = $prefix . str_repeat( '๐Ÿ˜€', $count ); + $before = get_comments( array( 'count' => true ) ); + $result = wp_handle_comment_submission( $submission ); + + if ( $should_insert ) { + $this->assertInstanceOf( WP_Comment::class, $result ); + $this->assertSame( $prefix . str_repeat( '😀', $count ), $result->$column ); + } else { + $this->assertSame( $before, get_comments( array( 'count' => true ) ), 'The overlong comment should not be inserted.' ); + $this->assertWPError( $result ); + $this->assertSame( $column . '_column_length', $result->get_error_code() ); + } + } + + public function data_comment_encoded_length() { + return array( + '255-byte author' => array( 'author', 'comment_author', 'abc', 28, true ), + '256-byte author' => array( 'author', 'comment_author', 'abcd', 28, false ), + '65,535-byte content' => array( 'comment', 'comment_content', 'abcdef', 7281, true ), + '65,536-byte content' => array( 'comment', 'comment_content', 'abcdefg', 7281, false ), + ); + } + + /** + * @ticket 65700 + * @covers ::wp_new_comment + */ + public function test_wp_new_comment_rejects_author_whose_encoded_value_exceeds_column_after_filtering(): void { + add_filter( 'pre_get_col_charset', array( $this, 'filter_pre_get_col_charset_utf8mb3_for_comment_fields' ), 10, 3 ); + add_filter( + 'pre_comment_author_name', + static function ( $author ) { + return 'd' . $author; + } + ); + + $before = get_comments( array( 'count' => true ) ); + $result = wp_handle_comment_submission( + array( + 'comment_post_ID' => self::$post_id, + 'author' => 'abc' . str_repeat( '๐Ÿ˜€', 28 ), + 'email' => 'author@example.org', + 'comment' => 'Comment with an overlong filtered author name.', + ) + ); + + $this->assertSame( $before, get_comments( array( 'count' => true ) ), 'The overlong comment should not be inserted.' ); + $this->assertWPError( $result ); + $this->assertSame( 'comment_author_column_length', $result->get_error_code() ); + } + /** * Control for test_wp_insert_comment_encodes_emoji_for_utf8mb3_columns(): on a * utf8mb4 column, emoji should be stored as-is rather than as HTML entities. From 77d634c5489052c11b6cd202f3cb16c48d028321 Mon Sep 17 00:00:00 2001 From: Jon Surrell Date: Thu, 19 Nov 2026 19:48:21 +0400 Subject: [PATCH 13/19] Revert "Render emoji sequences that mix literal characters and character references." This reverts commit 032e0e90d85eb87c95aecb74d2379a90ec587ee0. --- src/wp-includes/formatting.php | 18 +++++++++++------- tests/phpunit/tests/formatting/emoji.php | 12 ------------ tests/phpunit/tests/post.php | 20 -------------------- 3 files changed, 11 insertions(+), 39 deletions(-) diff --git a/src/wp-includes/formatting.php b/src/wp-includes/formatting.php index 28cdb475c482e..596477b336380 100644 --- a/src/wp-includes/formatting.php +++ b/src/wp-includes/formatting.php @@ -6265,14 +6265,18 @@ static function ( $match ) { * @return string The encoded content. */ function wp_staticize_emoji( $text ) { - $is_ascii = function_exists( 'mb_check_encoding' ) ? mb_check_encoding( $text, 'ASCII' ) : ! preg_match( '/[^\x00-\x7F]/', $text ); - if ( ! $is_ascii ) { - // An emoji sequence can contain both literal characters and character references. - $text = wp_encode_emoji( $text ); - } - if ( ! str_contains( $text, '&#x' ) ) { - return $text; + if ( ( function_exists( 'mb_check_encoding' ) && mb_check_encoding( $text, 'ASCII' ) ) || ! preg_match( '/[^\x00-\x7F]/', $text ) ) { + // The text doesn't contain anything that might be emoji, so we can return early. + return $text; + } else { + $encoded_text = wp_encode_emoji( $text ); + if ( $encoded_text === $text ) { + return $encoded_text; + } + + $text = $encoded_text; + } } $emoji = _wp_emoji_list( 'entities' ); diff --git a/tests/phpunit/tests/formatting/emoji.php b/tests/phpunit/tests/formatting/emoji.php index 7081a862e9c62..52120a2d0174d 100644 --- a/tests/phpunit/tests/formatting/emoji.php +++ b/tests/phpunit/tests/formatting/emoji.php @@ -347,18 +347,6 @@ public static function data_wp_staticize_emoji() { '๐Ÿ™‚', '๐Ÿ™‚', ), - 'four-byte emoji beside a character reference' => array( - '๐Ÿ˜€ A', - '๐Ÿ˜€ A', - ), - 'three-byte emoji beside a character reference' => array( - 'โŒš A', - 'โŒš A', - ), - 'family with literal joiners between character references' => array( - "👨\u{200D}👩\u{200D}👧", - '๐Ÿ‘จโ€๐Ÿ‘ฉโ€๐Ÿ‘ง', - ), array( // Skin tone, gender, ZWJ, emoji selector. '๐Ÿ‘ฎ๐Ÿผโ€โ™€๏ธ', diff --git a/tests/phpunit/tests/post.php b/tests/phpunit/tests/post.php index fe254b15dc3cc..c7204c8a03872 100644 --- a/tests/phpunit/tests/post.php +++ b/tests/phpunit/tests/post.php @@ -426,26 +426,6 @@ public function test_wp_insert_post_encodes_four_byte_characters_for_utf8mb3_col $this->assertSame( 'foo😐bat', $post->post_excerpt ); } - /** - * @ticket 65700 - * - * @covers ::wp_insert_post - * @covers ::wp_staticize_emoji - */ - public function test_staticize_emoji_in_post_content_stored_as_utf8mb3(): void { - $content = '๐Ÿ‘จโ€๐Ÿ‘ฉโ€๐Ÿ‘ง โค๏ธ'; - - add_filter( 'pre_get_col_charset', array( $this, 'filter_pre_get_col_charset_utf8mb3_for_post_fields' ), 10, 3 ); - $post_id = wp_insert_post( array( 'post_content' => $content ), true ); - remove_filter( 'pre_get_col_charset', array( $this, 'filter_pre_get_col_charset_utf8mb3_for_post_fields' ), 10 ); - - $this->assertNotWPError( $post_id ); - $stored = get_post( $post_id )->post_content; - - $this->assertSame( "👨\u{200D}👩\u{200D}👧 โค๏ธ", $stored ); - $this->assertSame( wp_staticize_emoji( $content ), wp_staticize_emoji( $stored ) ); - } - public function filter_pre_get_col_charset_utf8mb3_for_post_fields( $charset, $table, $column ) { global $wpdb; From 9aacc96a706a2f8f11c9658b1f261e041b6ea18f Mon Sep 17 00:00:00 2001 From: Jon Surrell Date: Thu, 19 Nov 2026 19:48:21 +0400 Subject: [PATCH 14/19] Restore post emoji encoding to the PR base. Keep the existing wp_insert_post() encoding for titles, content, and excerpts. Remove the tests added for the broader post encoding. --- src/wp-includes/post.php | 13 +++++++++++- tests/phpunit/tests/post.php | 40 ------------------------------------ 2 files changed, 12 insertions(+), 41 deletions(-) diff --git a/src/wp-includes/post.php b/src/wp-includes/post.php index 53903e220b295..3511fbfc81d7a 100644 --- a/src/wp-includes/post.php +++ b/src/wp-includes/post.php @@ -5174,7 +5174,18 @@ function wp_insert_post( $postarr, $wp_error = false, $fire_after_hooks = true ) 'guid' ); - $data = _wp_encode_4byte_chars_in_fields( $data, $wpdb->posts, array( 'post_title', 'post_content', 'post_excerpt' ) ); + $emoji_fields = array( 'post_title', 'post_content', 'post_excerpt' ); + + foreach ( $emoji_fields as $emoji_field ) { + if ( isset( $data[ $emoji_field ] ) ) { + $charset = $wpdb->get_col_charset( $wpdb->posts, $emoji_field ); + + // The 'utf8' character set is a deprecated alias of 'utf8mb3'. See . + if ( 'utf8' === $charset || 'utf8mb3' === $charset ) { + $data[ $emoji_field ] = wp_encode_emoji( $data[ $emoji_field ] ); + } + } + } if ( 'attachment' === $post_type ) { /** diff --git a/tests/phpunit/tests/post.php b/tests/phpunit/tests/post.php index c7204c8a03872..fcbe1941de287 100644 --- a/tests/phpunit/tests/post.php +++ b/tests/phpunit/tests/post.php @@ -396,46 +396,6 @@ public function test_utf8mb3_post_saves_with_emoji() { } } - /** - * Forces the utf8mb3 code path via the `pre_get_col_charset` filter, rather - * than relying on the test database actually using that character set, so - * this runs deterministically regardless of the environment's DB charset. - * - * @ticket 65700 - * - * @covers ::wp_insert_post - * @covers ::_wp_encode_4byte_chars_in_fields - */ - public function test_wp_insert_post_encodes_four_byte_characters_for_utf8mb3_columns_via_filter(): void { - add_filter( 'pre_get_col_charset', array( $this, 'filter_pre_get_col_charset_utf8mb3_for_post_fields' ), 10, 3 ); - - $post_id = self::factory()->post->create( - array( - 'post_title' => 'Acmeโ„ข ๐ ฎท', - 'post_content' => "foo\xf0\x9f\x98\x8ebaz", - 'post_excerpt' => "foo\xf0\x9f\x98\x90bat", - ) - ); - - remove_filter( 'pre_get_col_charset', array( $this, 'filter_pre_get_col_charset_utf8mb3_for_post_fields' ), 10 ); - - $post = get_post( $post_id ); - - $this->assertSame( 'Acmeโ„ข 𠮷', $post->post_title ); - $this->assertSame( 'foo😎baz', $post->post_content ); - $this->assertSame( 'foo😐bat', $post->post_excerpt ); - } - - public function filter_pre_get_col_charset_utf8mb3_for_post_fields( $charset, $table, $column ) { - global $wpdb; - - if ( $wpdb->posts === $table && in_array( $column, array( 'post_title', 'post_content', 'post_excerpt' ), true ) ) { - return 'utf8mb3'; - } - - return $charset; - } - /** * If a sticky post is updated via `wp_update_post()` by a user * without the `publish_posts` capability, it should stay sticky. From ea6887226f2cfc5f9b5b6bbac546f25bc27f78d4 Mon Sep 17 00:00:00 2001 From: Jon Surrell Date: Thu, 19 Nov 2026 19:48:59 +0400 Subject: [PATCH 15/19] Require utf8mb4 columns for the comment round-trip test. Use the actual column charsets so the test can only run where both comment fields can store four-byte characters. --- tests/phpunit/tests/comment.php | 23 ++++++++--------------- 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/tests/phpunit/tests/comment.php b/tests/phpunit/tests/comment.php index 70112d65aa849..53eea0d9994b0 100644 --- a/tests/phpunit/tests/comment.php +++ b/tests/phpunit/tests/comment.php @@ -2666,15 +2666,20 @@ static function ( $author ) { } /** - * Control for test_wp_insert_comment_encodes_emoji_for_utf8mb3_columns(): on a - * utf8mb4 column, emoji should be stored as-is rather than as HTML entities. + * Verifies that four-byte characters round-trip on utf8mb4 columns. * * @ticket 65700 * * @covers ::wp_insert_comment */ public function test_wp_insert_comment_does_not_encode_emoji_for_utf8mb4_columns(): void { - add_filter( 'pre_get_col_charset', array( $this, 'filter_pre_get_col_charset_utf8mb4_for_comment_fields' ), 10, 3 ); + global $wpdb; + + if ( 'utf8mb4' !== $wpdb->get_col_charset( $wpdb->comments, 'comment_author' ) + || 'utf8mb4' !== $wpdb->get_col_charset( $wpdb->comments, 'comment_content' ) + ) { + $this->markTestSkipped( 'This test requires utf8mb4 comment author and content columns.' ); + } $comment_id = wp_insert_comment( array( @@ -2684,8 +2689,6 @@ public function test_wp_insert_comment_does_not_encode_emoji_for_utf8mb4_columns ) ); - remove_filter( 'pre_get_col_charset', array( $this, 'filter_pre_get_col_charset_utf8mb4_for_comment_fields' ), 10 ); - $this->assertIsInt( $comment_id, 'Expected wp_insert_comment() to return the new comment ID.' ); $comment = get_comment( $comment_id ); @@ -2836,16 +2839,6 @@ public function filter_pre_get_col_charset_utf8mb3_for_comment_fields( $charset, return $charset; } - public function filter_pre_get_col_charset_utf8mb4_for_comment_fields( $charset, $table, $column ) { - global $wpdb; - - if ( $wpdb->comments === $table && in_array( $column, array( 'comment_author', 'comment_content' ), true ) ) { - return 'utf8mb4'; - } - - return $charset; - } - /** * Queries made by metadata observers must see the newly added metadata. * From a231c932e86a31a337e5daf62882b82366062878 Mon Sep 17 00:00:00 2001 From: Jon Surrell Date: Thu, 19 Nov 2026 19:50:04 +0400 Subject: [PATCH 16/19] Preserve comment retry cleanup for unencoded fields. Apply the storage length check in the insert retry only to author or content changed by four-byte character encoding. The retry still truncates overlong plain text and URLs. --- src/wp-includes/comment.php | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/wp-includes/comment.php b/src/wp-includes/comment.php index fba0f74169970..913f59bb2eae4 100644 --- a/src/wp-includes/comment.php +++ b/src/wp-includes/comment.php @@ -2614,19 +2614,19 @@ function wp_new_comment( $commentdata, $wp_error = false ) { $unslashed = wp_unslash( $commentdata ); $stored = _wp_encode_4byte_chars_in_fields( $unslashed, $wpdb->comments, array( 'comment_author', 'comment_content' ) ); - $check_storage_lengths = _wp_check_comment_data_storage_lengths( $stored ); - if ( is_wp_error( $check_storage_lengths ) ) { - return $wp_error ? $check_storage_lengths : false; - } - foreach ( $fields as $field ) { if ( ! isset( $commentdata[ $field ] ) ) { continue; } - // Keep the original characters for comment filters when the encoded value is valid. if ( $stored[ $field ] !== $unslashed[ $field ] ) { - $value = $stored[ $field ]; + $value = $stored[ $field ]; + $check_storage_lengths = _wp_check_comment_data_storage_lengths( array( $field => $value ) ); + if ( is_wp_error( $check_storage_lengths ) ) { + return $wp_error ? $check_storage_lengths : false; + } + + // Keep the original characters for comment filters when the encoded value is valid. if ( $value === $wpdb->strip_invalid_text_for_column( $wpdb->comments, $field, $value ) ) { continue; } From cd3cb51362181d39af4763be95a2625b15af9382 Mon Sep 17 00:00:00 2001 From: Jon Surrell Date: Thu, 19 Nov 2026 21:07:43 +0400 Subject: [PATCH 17/19] Apply comment field limits to the encoded values. Prepare the author and content before the existing length checks. Use the same validator for encoded fields during the insert retry so the configured limits and field-specific errors apply in both places. --- src/wp-includes/comment.php | 55 +++++---------------------------- tests/phpunit/tests/comment.php | 16 +++++----- 2 files changed, 16 insertions(+), 55 deletions(-) diff --git a/src/wp-includes/comment.php b/src/wp-includes/comment.php index 913f59bb2eae4..dda5cc56ae362 100644 --- a/src/wp-includes/comment.php +++ b/src/wp-includes/comment.php @@ -1326,7 +1326,7 @@ function wp_get_comment_fields_max_lengths() { } /** - * Checks submitted comment field lengths and the storage limits for the author and content. + * Checks comment field lengths after preparing the author and content for storage. * * @since 4.7.0 * @@ -1338,7 +1338,8 @@ function wp_get_comment_fields_max_lengths() { function wp_check_comment_data_max_lengths( $comment_data ) { global $wpdb; - $max_lengths = wp_get_comment_fields_max_lengths(); + $comment_data = _wp_encode_4byte_chars_in_fields( $comment_data, $wpdb->comments, array( 'comment_author', 'comment_content' ) ); + $max_lengths = wp_get_comment_fields_max_lengths(); if ( isset( $comment_data['comment_author'] ) && mb_strlen( $comment_data['comment_author'], '8bit' ) > $max_lengths['comment_author'] ) { return new WP_Error( 'comment_author_column_length', __( 'Error: Your name is too long.' ), 200 ); @@ -1356,46 +1357,6 @@ function wp_check_comment_data_max_lengths( $comment_data ) { return new WP_Error( 'comment_content_column_length', __( 'Error: Your comment is too long.' ), 200 ); } - $stored = _wp_encode_4byte_chars_in_fields( $comment_data, $wpdb->comments, array( 'comment_author', 'comment_content' ) ); - - return _wp_check_comment_data_storage_lengths( $stored ); -} - -/** - * Checks the column lengths of the author and content prepared for storage. - * - * @since 7.2.0 - * @access private - * @ignore - * @global wpdb $wpdb WordPress database abstraction object. - * @param array $stored Array of unslashed comment data prepared for storage. - * @return WP_Error|true WP_Error when the author or content exceeds its column length, otherwise true. - */ -function _wp_check_comment_data_storage_lengths( $stored ) { - global $wpdb; - - foreach ( array( 'comment_author', 'comment_content' ) as $field ) { - if ( ! isset( $stored[ $field ] ) || ! is_string( $stored[ $field ] ) ) { - continue; - } - - $col_length = $wpdb->get_col_length( $wpdb->comments, $field ); - if ( ! is_array( $col_length ) ) { - continue; - } - - $length = 'byte' === $col_length['type'] ? strlen( $stored[ $field ] ) : mb_strlen( $stored[ $field ], 'UTF-8' ); - if ( $length <= $col_length['length'] ) { - continue; - } - - if ( 'comment_author' === $field ) { - return new WP_Error( 'comment_author_column_length', __( 'Error: Your name is too long.' ), 200 ); - } - - return new WP_Error( 'comment_content_column_length', __( 'Error: Your comment is too long.' ), 200 ); - } - return true; } @@ -2620,14 +2581,14 @@ function wp_new_comment( $commentdata, $wp_error = false ) { } if ( $stored[ $field ] !== $unslashed[ $field ] ) { - $value = $stored[ $field ]; - $check_storage_lengths = _wp_check_comment_data_storage_lengths( array( $field => $value ) ); - if ( is_wp_error( $check_storage_lengths ) ) { - return $wp_error ? $check_storage_lengths : false; + $value = $stored[ $field ]; + $check_lengths = wp_check_comment_data_max_lengths( array( $field => $value ) ); + if ( is_wp_error( $check_lengths ) ) { + return $wp_error ? $check_lengths : false; } // Keep the original characters for comment filters when the encoded value is valid. - if ( $value === $wpdb->strip_invalid_text_for_column( $wpdb->comments, $field, $value ) ) { + if ( wp_is_valid_utf8( $value ) ) { continue; } } diff --git a/tests/phpunit/tests/comment.php b/tests/phpunit/tests/comment.php index 53eea0d9994b0..12d5dd6f093e9 100644 --- a/tests/phpunit/tests/comment.php +++ b/tests/phpunit/tests/comment.php @@ -2602,7 +2602,7 @@ public function test_wp_insert_comment_encodes_emoji_for_utf8mb3_columns(): void * @param string $column Comment table column name. * @param string $prefix ASCII prefix before the four-byte characters. * @param int $count Number of four-byte characters. - * @param bool $should_insert Whether the encoded value fits in the column. + * @param bool $should_insert Whether the encoded value fits the comment field limit. */ public function test_wp_handle_comment_submission_checks_encoded_length( $input, $column, $prefix, $count, $should_insert ): void { add_filter( 'pre_get_col_charset', array( $this, 'filter_pre_get_col_charset_utf8mb3_for_comment_fields' ), 10, 3 ); @@ -2630,10 +2630,10 @@ public function test_wp_handle_comment_submission_checks_encoded_length( $input, public function data_comment_encoded_length() { return array( - '255-byte author' => array( 'author', 'comment_author', 'abc', 28, true ), - '256-byte author' => array( 'author', 'comment_author', 'abcd', 28, false ), - '65,535-byte content' => array( 'comment', 'comment_content', 'abcdef', 7281, true ), - '65,536-byte content' => array( 'comment', 'comment_content', 'abcdefg', 7281, false ), + '245-byte author' => array( 'author', 'comment_author', 'ab', 27, true ), + '246-byte author' => array( 'author', 'comment_author', 'abc', 27, false ), + '65,525-byte content' => array( 'comment', 'comment_content', 'abcde', 7280, true ), + '65,526-byte content' => array( 'comment', 'comment_content', 'abcdef', 7280, false ), ); } @@ -2641,12 +2641,12 @@ public function data_comment_encoded_length() { * @ticket 65700 * @covers ::wp_new_comment */ - public function test_wp_new_comment_rejects_author_whose_encoded_value_exceeds_column_after_filtering(): void { + public function test_wp_new_comment_rejects_overlong_encoded_author_on_retry(): void { add_filter( 'pre_get_col_charset', array( $this, 'filter_pre_get_col_charset_utf8mb3_for_comment_fields' ), 10, 3 ); add_filter( 'pre_comment_author_name', static function ( $author ) { - return 'd' . $author; + return 'Comment author ' . $author; } ); @@ -2654,7 +2654,7 @@ static function ( $author ) { $result = wp_handle_comment_submission( array( 'comment_post_ID' => self::$post_id, - 'author' => 'abc' . str_repeat( '๐Ÿ˜€', 28 ), + 'author' => 'ab' . str_repeat( '๐Ÿ˜€', 27 ), 'email' => 'author@example.org', 'comment' => 'Comment with an overlong filtered author name.', ) From 15c0c4469e92bd74ee69dc2b4c8f0fb3ea6fcb49 Mon Sep 17 00:00:00 2001 From: Dennis Snell Date: Fri, 20 Nov 2026 16:50:09 -0600 Subject: [PATCH 18/19] Separate HTML transformation; use mb_encode_numericentity() --- src/wp-admin/includes/comment.php | 2 +- src/wp-includes/comment.php | 12 +-- src/wp-includes/formatting.php | 96 ++++++++++++++++-------- tests/phpunit/tests/comment.php | 4 +- tests/phpunit/tests/formatting/emoji.php | 12 +-- 5 files changed, 79 insertions(+), 47 deletions(-) diff --git a/src/wp-admin/includes/comment.php b/src/wp-admin/includes/comment.php index 746c5da4412c7..1d22980a80425 100644 --- a/src/wp-admin/includes/comment.php +++ b/src/wp-admin/includes/comment.php @@ -31,7 +31,7 @@ function comment_exists( $comment_author, $comment_date, $timezone = 'blog' ) { $date_field = 'comment_date_gmt'; } - $stored = _wp_encode_4byte_chars_in_fields( array( 'comment_author' => stripslashes( $comment_author ) ), $wpdb->comments, array( 'comment_author' ) ); + $stored = _wp_encode_field_values_for_database( array( 'comment_author' => stripslashes( $comment_author ) ), $wpdb->comments, array( 'comment_author' ) ); return $wpdb->get_var( $wpdb->prepare( diff --git a/src/wp-includes/comment.php b/src/wp-includes/comment.php index dda5cc56ae362..898dcd114ab8b 100644 --- a/src/wp-includes/comment.php +++ b/src/wp-includes/comment.php @@ -145,7 +145,7 @@ function check_comment( $author, $email, $url, $comment, $user_ip, $user_agent, ) ); } else { - $stored = _wp_encode_4byte_chars_in_fields( array( 'comment_author' => $author ), $wpdb->comments, array( 'comment_author' ) ); + $stored = _wp_encode_field_values_for_database( array( 'comment_author' => $author ), $wpdb->comments, array( 'comment_author' ) ); // expected_slashed ($author, $email) $ok_to_comment = $wpdb->get_var( @@ -770,7 +770,7 @@ function sanitize_comment_cookies() { function wp_allow_comment( $commentdata, $wp_error = false ) { global $wpdb; - $stored = _wp_encode_4byte_chars_in_fields( $commentdata, $wpdb->comments, array( 'comment_author', 'comment_content' ) ); + $stored = _wp_encode_field_values_for_database( $commentdata, $wpdb->comments, array( 'comment_author', 'comment_content' ) ); /* * Simple duplicate check. @@ -1338,7 +1338,7 @@ function wp_get_comment_fields_max_lengths() { function wp_check_comment_data_max_lengths( $comment_data ) { global $wpdb; - $comment_data = _wp_encode_4byte_chars_in_fields( $comment_data, $wpdb->comments, array( 'comment_author', 'comment_content' ) ); + $comment_data = _wp_encode_field_values_for_database( $comment_data, $wpdb->comments, array( 'comment_author', 'comment_content' ) ); $max_lengths = wp_get_comment_fields_max_lengths(); if ( isset( $comment_data['comment_author'] ) && mb_strlen( $comment_data['comment_author'], '8bit' ) > $max_lengths['comment_author'] ) { @@ -2313,7 +2313,7 @@ function wp_insert_comment( $commentdata ) { 'user_id' ); - $compacted = _wp_encode_4byte_chars_in_fields( $compacted, $wpdb->comments, array( 'comment_author', 'comment_content' ) ); + $compacted = _wp_encode_field_values_for_database( $compacted, $wpdb->comments, array( 'comment_author', 'comment_content' ) ); if ( ! $wpdb->insert( $wpdb->comments, $compacted ) ) { return false; @@ -2573,7 +2573,7 @@ function wp_new_comment( $commentdata, $wp_error = false ) { if ( ! $comment_id ) { $fields = array( 'comment_author', 'comment_author_email', 'comment_author_url', 'comment_content' ); $unslashed = wp_unslash( $commentdata ); - $stored = _wp_encode_4byte_chars_in_fields( $unslashed, $wpdb->comments, array( 'comment_author', 'comment_content' ) ); + $stored = _wp_encode_field_values_for_database( $unslashed, $wpdb->comments, array( 'comment_author', 'comment_content' ) ); foreach ( $fields as $field ) { if ( ! isset( $commentdata[ $field ] ) ) { @@ -3120,7 +3120,7 @@ function wp_update_comment( $commentarr, $wp_error = false ) { $data = wp_array_slice_assoc( $data, $keys ); - $data = _wp_encode_4byte_chars_in_fields( $data, $wpdb->comments, array( 'comment_author', 'comment_content' ) ); + $data = _wp_encode_field_values_for_database( $data, $wpdb->comments, array( 'comment_author', 'comment_content' ) ); $result = $wpdb->update( $wpdb->comments, $data, array( 'comment_ID' => $comment_id ) ); diff --git a/src/wp-includes/formatting.php b/src/wp-includes/formatting.php index 596477b336380..89b43b0605a19 100644 --- a/src/wp-includes/formatting.php +++ b/src/wp-includes/formatting.php @@ -6193,6 +6193,67 @@ function wp_encode_emoji( $content ) { return $content; } +/** + * Escapes a UTF-8-encoded HTML document for a given MySQL charset, replacing + * unrepresentable characters with their character-reference equivalents. + * + * Because WordPress only supports charsets compatible in the ASCII range, + * this function will return a common-demoninator for unrecognized encodings + * as the ASCII-safe escaping of the input HTML. + * + * In the future this function may perform conversion differently in certain + * cases, as not all content may be safely transformed; specifically, there + * is no representation in HTML for tag names, attribute names, comment + * content, using bytes that arenโ€™t valid in the document encoding. + * + * Example: + * + * '🅰' === _wp_escape_html_for_mysql_charset( 'latin1', '๐Ÿ…ฐ' ); + * '🅰' === _wp_escape_html_for_mysql_charset( 'utf8mb3', '๐Ÿ…ฐ' ); + * '๐Ÿ…ฐ' === _wp_escape_html_for_mysql_charset( 'utf8mb4', '๐Ÿ…ฐ' ); + * + * Because invalid byte spans are not generally safe for database storage, this + * function will replace those spans with an appropriate substitution character. + * + * Example: + * + * '๏ฟฝ' === _wp_escape_html_for_mysql_charset( 'utf8mb4', "\xC0" ); + * '�' === _wp_escape_html_for_mysql_charset( 'utf8mb3', "\xC0" ); + * + * Future improvement would scan through the HTML using the HTML API and make + * semantic substitutions where appropriate. For example, JavaScript, JSON, + * and JSON attributes in block deloimiters should use JavaScript Unicode + * escaping, CSS should use CSS Unicode escaping, CDATA sections should be + * split apart and escaped, while other syntax tokens will not be able to + * be properly escaped. + * + * @since 7.2.0 + * @access private + * @ignore + * + * @param string $mysql_charset Convert into this encoding, according to MySQL charset names. + * @param string $html_in_utf8 Convert this HTML. + * @return string Transformed input safe for storing in a MySQL column of the given charset. + */ +function _wp_escape_html_for_mysql_charset( string $mysql_charset, string $html_in_utf8 ): string { + if ( 'utf8mb4' === $mysql_charset ) { + return wp_scrub_utf8( $html_in_utf8 ); + } + + $is_utf8mb3 = 'utf8' === $mysql_charset || 'utf8mb3' === $mysql_charset; + $substitute_character = $is_utf8mb3 ? 0xFFFD : 0x3F; + $replacement_range = $is_utf8mb3 + ? array( 0x10000, 0x10FFFF, 0, 0x1FFFFF ) // Replace four-byte UTF-8 code points. + : array( 0x80, 0x10FFFF, 0, 0x1FFFFF ); // Replace all multi-byte UTF-8 code points. + + $previous_substitute_character = mb_substitute_character(); + mb_substitute_character( $substitute_character ); + $transformed = mb_encode_numericentity( $html_in_utf8, $replacement_range, 'UTF-8', true ); + mb_substitute_character( $previous_substitute_character ); + + return $transformed; +} + /** * Replaces four-byte UTF-8 characters with HTML character references in fields whose database * columns use `utf8` or `utf8mb3`. @@ -6212,7 +6273,7 @@ function wp_encode_emoji( $content ) { * @param string[] $fields Names of the fields to check. * @return array Field values with four-byte characters encoded where needed. */ -function _wp_encode_4byte_chars_in_fields( $data, $table, $fields ) { +function _wp_encode_field_values_for_database( $data, $table, $fields ) { global $wpdb; foreach ( $fields as $field ) { @@ -6220,37 +6281,8 @@ function _wp_encode_4byte_chars_in_fields( $data, $table, $fields ) { continue; } - $charset = $wpdb->get_col_charset( $table, $field ); - - /* - * Note that the `utf8` character set is a deprecated alias of `utf8mb3`. - * - * Table 3-7 of the Unicode Standard gives these ranges for four-byte UTF-8 sequences. - * - * โ•ญโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ฌโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ฌโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ฌโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ฌโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฎ - * โ”‚ Code Points โ”‚ First Byte โ”‚ Second Byte โ”‚ Third Byte โ”‚ Fourth Byte โ”‚ - * โ”œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ผโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ผโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ผโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ผโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ค - * โ”‚ U+10000..U+3FFFF โ”‚ F0 โ”‚ 90..BF โ”‚ 80..BF โ”‚ 80..BF โ”‚ - * โ”‚ U+40000..U+FFFFF โ”‚ F1..F3 โ”‚ 80..BF โ”‚ 80..BF โ”‚ 80..BF โ”‚ - * โ”‚ U+100000..U+10FFFF โ”‚ F4 โ”‚ 80..8F โ”‚ 80..BF โ”‚ 80..BF โ”‚ - * โ•ฐโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ดโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ดโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ดโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ดโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฏ - * - * @link https://www.unicode.org/versions/Unicode16.0.0/core-spec/chapter-3/#G27506 - */ - if ( 'utf8' === $charset || 'utf8mb3' === $charset ) { - $data[ $field ] = preg_replace_callback( - '/ - # Match bytes so malformed UTF-8 elsewhere in the field does not prevent a match. - \xF0 [\x90-\xBF] [\x80-\xBF] [\x80-\xBF] # U+10000..U+3FFFF - | [\xF1-\xF3] [\x80-\xBF] [\x80-\xBF] [\x80-\xBF] # U+40000..U+FFFFF - | \xF4 [\x80-\x8F] [\x80-\xBF] [\x80-\xBF] # U+100000..U+10FFFF - /x', - static function ( $match ) { - return sprintf( '&#x%x;', mb_ord( $match[0], 'UTF-8' ) ); - }, - $data[ $field ] - ) ?? $data[ $field ]; - } + $charset = $wpdb->get_col_charset( $table, $field ); + $data[ $field ] = _wp_escape_html_for_mysql_charset( $charset, $data[ $field ] ); } return $data; diff --git a/tests/phpunit/tests/comment.php b/tests/phpunit/tests/comment.php index 12d5dd6f093e9..64d18da58036a 100644 --- a/tests/phpunit/tests/comment.php +++ b/tests/phpunit/tests/comment.php @@ -2569,7 +2569,7 @@ public function test_get_comment_should_only_treat_numeric_values_as_comment_ids * @ticket 65700 * * @covers ::wp_insert_comment - * @covers ::_wp_encode_4byte_chars_in_fields + * @covers ::_wp_encode_field_values_for_database */ public function test_wp_insert_comment_encodes_emoji_for_utf8mb3_columns(): void { add_filter( 'pre_get_col_charset', array( $this, 'filter_pre_get_col_charset_utf8mb3_for_comment_fields' ), 10, 3 ); @@ -2701,7 +2701,7 @@ public function test_wp_insert_comment_does_not_encode_emoji_for_utf8mb4_columns * @ticket 65700 * * @covers ::wp_update_comment - * @covers ::_wp_encode_4byte_chars_in_fields + * @covers ::_wp_encode_field_values_for_database */ public function test_wp_update_comment_encodes_emoji_for_utf8mb3_columns(): void { $comment_id = self::factory()->comment->create( array( 'comment_post_ID' => self::$post_id ) ); diff --git a/tests/phpunit/tests/formatting/emoji.php b/tests/phpunit/tests/formatting/emoji.php index 52120a2d0174d..bb10d04c67b46 100644 --- a/tests/phpunit/tests/formatting/emoji.php +++ b/tests/phpunit/tests/formatting/emoji.php @@ -380,7 +380,7 @@ public function test_wp_staticize_emoji( $emoji, $expected ) { /** * @ticket 65700 * - * @covers ::_wp_encode_4byte_chars_in_fields + * @covers ::_wp_encode_field_values_for_database */ public function test_wp_encode_4byte_chars_in_fields_only_touches_requested_utf8mb3_fields(): void { $data = array( @@ -391,7 +391,7 @@ public function test_wp_encode_4byte_chars_in_fields_only_touches_requested_utf8 add_filter( 'pre_get_col_charset', array( $this, 'filter_pre_get_col_charset_for_encode_4byte_chars_in_fields_test' ), 10, 3 ); - $result = _wp_encode_4byte_chars_in_fields( $data, 'some_table', array( 'legacy_field', 'modern_field' ) ); + $result = _wp_encode_field_values_for_database( $data, 'some_table', array( 'legacy_field', 'modern_field' ) ); remove_filter( 'pre_get_col_charset', array( $this, 'filter_pre_get_col_charset_for_encode_4byte_chars_in_fields_test' ), 10 ); @@ -415,14 +415,14 @@ public function test_wp_encode_4byte_chars_in_fields_only_touches_requested_utf8 /** * @ticket 65700 * - * @covers ::_wp_encode_4byte_chars_in_fields + * @covers ::_wp_encode_field_values_for_database */ public function test_wp_encode_4byte_chars_in_fields_recognizes_the_utf8_alias(): void { $data = array( 'legacy_field' => '๐Ÿ™‚' ); add_filter( 'pre_get_col_charset', array( $this, 'filter_pre_get_col_charset_utf8_alias_for_encode_4byte_chars_in_fields_test' ) ); - $result = _wp_encode_4byte_chars_in_fields( $data, 'some_table', array( 'legacy_field' ) ); + $result = _wp_encode_field_values_for_database( $data, 'some_table', array( 'legacy_field' ) ); remove_filter( 'pre_get_col_charset', array( $this, 'filter_pre_get_col_charset_utf8_alias_for_encode_4byte_chars_in_fields_test' ) ); @@ -436,7 +436,7 @@ public function test_wp_encode_4byte_chars_in_fields_recognizes_the_utf8_alias() /** * @ticket 65700 * - * @covers ::_wp_encode_4byte_chars_in_fields + * @covers ::_wp_encode_field_values_for_database */ public function test_wp_encode_4byte_chars_in_fields_ignores_fields_missing_from_data(): void { $data = array( 'present_field' => '๐Ÿ™‚' ); @@ -445,7 +445,7 @@ public function test_wp_encode_4byte_chars_in_fields_ignores_fields_missing_from // nonexistent 'some_table' table in the test database. add_filter( 'pre_get_col_charset', array( $this, 'filter_pre_get_col_charset_utf8mb4_for_encode_4byte_chars_in_fields_test' ) ); - $result = _wp_encode_4byte_chars_in_fields( $data, 'some_table', array( 'present_field', 'absent_field' ) ); + $result = _wp_encode_field_values_for_database( $data, 'some_table', array( 'present_field', 'absent_field' ) ); remove_filter( 'pre_get_col_charset', array( $this, 'filter_pre_get_col_charset_utf8mb4_for_encode_4byte_chars_in_fields_test' ) ); From 58eb8c89ab41451a2fdf7d6cddbcaadb0e04e95c Mon Sep 17 00:00:00 2001 From: Dennis Snell Date: Fri, 20 Nov 2026 21:04:44 -0600 Subject: [PATCH 19/19] Polyfill mb_encode_numericentity() and mb_substitute_character() --- phpcs.xml.dist | 5 + src/wp-includes/compat-utf8.php | 158 ++++++++++++++++++++++++++++++++ src/wp-includes/compat.php | 91 ++++++++++++++++++ 3 files changed, 254 insertions(+) diff --git a/phpcs.xml.dist b/phpcs.xml.dist index 9f15c0c3670d6..7d4105ca3df98 100644 --- a/phpcs.xml.dist +++ b/phpcs.xml.dist @@ -286,6 +286,11 @@ /wp-tests-config-sample\.php + + + /src/wp-includes/compat.php + + /tests/phpunit/tests/* diff --git a/src/wp-includes/compat-utf8.php b/src/wp-includes/compat-utf8.php index 326786fe608f2..15f15a095e510 100644 --- a/src/wp-includes/compat-utf8.php +++ b/src/wp-includes/compat-utf8.php @@ -312,6 +312,164 @@ function _wp_scrub_utf8_fallback( string $bytes ): string { return $scrubbed; } +/** + * Polyfill for `mb_encode_numericentity()` operating on UTF-8 inputs. + * + * @since 7.2.0 + * @access private + * @internal + * + * @param string $bytes The string to be encoded. + * @param int[] $map map is array specifies code area to convert. + * @param bool|null $hex Optional. Whether the returned entity reference should be in + * hexadecimal notation (otherwise it is in decimal notation). + * @return string The converted string. + */ +function _wp_encode_utf8_numericentity( $bytes, $map, $hex = false ) { + $output = ''; + $at = 0; + $was_at = 0; + $end = strlen( $bytes ); + $fffd = $hex ? '�' : '�'; + $map_end = count( $map ); + + if ( 0 !== ( $map_end % 4 ) ) { + if ( class_exists( 'ValueError' ) ) { + throw new ValueError( 'mb_encode_numericentity(): Argument #2 ($map) must have a multiple of 4 elements' ); + } else { + throw new Error( 'mb_encode_numericentity(): Argument #2 ($map) must have a multiple of 4 elements' ); + } + } + + while ( $at < $end ) { + $next_at = $at; + $invalid_length = 0; + $count = _wp_scan_utf8( $bytes, $next_at, $invalid_length, null, 1 ); + $char_length = $next_at - $at; + $next_char = substr( $bytes, $at, $char_length ); + $code_point = 0 < $count ? mb_ord( $next_char, 'UTF-8' ) : null; + + // This is not the most efficient algorithm; install the mbstring extension! + $needs_remapping = false; + $offset = 0; + $mask = 0; + if ( 0 < $count ) { + for ( $i = 0; $i < $map_end; $i += 4 ) { + if ( $code_point >= $map[ $i ] && $code_point <= $map[ $i + 1 ] ) { + $needs_remapping = true; + $offset = $map[ $i + 2 ]; + $mask = $map[ $i + 3 ]; + break; + } + } + } + + if ( ! $needs_remapping && 0 === $invalid_length ) { + $at = $next_at; + continue; + } + + if ( isset( $code_point ) && $needs_remapping ) { + $code_point = ( $code_point + $offset ) & $mask; + $digits = $hex ? strtoupper( dechex( $code_point ) ) : (string) $code_point; + $output .= substr( $bytes, $was_at, $at - $was_at ); + $output .= $hex ? "&#x{$digits};" : "&#{$digits};"; + } else { + $output .= substr( $bytes, $was_at, $at + $char_length - $was_at ); + } + + if ( $invalid_length > 0 ) { + $output .= $fffd; + } + + $at = $next_at + $invalid_length; + $was_at = $at; + } + + if ( $was_at < $end ) { + $output .= substr( $bytes, $was_at ); + } + + return $output; +} + +/** + * Polyfill for `mb_encode_numericentity()` operating on UTF-8 inputs when the goal + * is to encode based on byte length of code point spans. + * + * @since 7.2.0 + * @access private + * @internal + * + * @param int $byte_length Encode code points taking this many bytes or more to represent. + * @param string $bytes The string to be encoded. + * @param bool|null $hex Optional. Whether the returned entity reference should be in + * hexadecimal notation (otherwise it is in decimal notation). + * @return string The converted string. + */ +function _wp_encode_utf8_numericentity_bytelength( $byte_length, $bytes, $hex = false ) { + $output = ''; + $at = 0; + $was_at = 0; + $end = strlen( $bytes ); + $fffd = $byte_length > 3 ? "\u{FFFD}" : ( $hex ? '�' : '�' ); + + while ( $at < $end ) { + /* + * Quickly skip past US-ASCII bytes, all of which are valid UTF-8. + * + * This optimization step improves the speed from 10x to 100x + * depending on whether the JIT has optimized the function. + */ + $ascii_byte_count = strspn( + $bytes, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f" . + "\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f" . + " !\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\x7f", + $at, + ); + + $at += $ascii_byte_count; + if ( $at >= $end ) { + break; + } + + $next_at = $at; + $invalid_length = 0; + $count = _wp_scan_utf8( $bytes, $next_at, $invalid_length, null, 1 ); + $char_length = $next_at - $at; + $next_char = 0 < $count ? substr( $bytes, $at, $char_length ) : null; + + // The character is within bounds and is followed by no invalid bytes. + if ( $char_length < $byte_length && 0 === $invalid_length ) { + $at = $next_at; + continue; + } + + if ( isset( $next_char ) && $char_length >= $byte_length ) { + $code_point = mb_ord( $next_char, 'UTF-8' ); + $output .= substr( $bytes, $was_at, $at - $was_at ); + $digits = $hex ? strtoupper( dechex( $code_point ) ) : (string) $code_point; + $output .= $hex ? "&#x{$digits};" : "&#{$digits};"; + } else { + $output .= substr( $bytes, $was_at, $at + $char_length - $was_at ); + } + + if ( $invalid_length > 0 ) { + $output .= $fffd; + } + + $at = $next_at + $invalid_length; + $was_at = $at; + } + + if ( $was_at < $end ) { + $output .= substr( $bytes, $was_at ); + } + + return $output; +} + /** * Returns how many code points are found in the given UTF-8 string. * diff --git a/src/wp-includes/compat.php b/src/wp-includes/compat.php index 0aac0481c5d23..7b8905d059402 100644 --- a/src/wp-includes/compat.php +++ b/src/wp-includes/compat.php @@ -378,6 +378,97 @@ function _mb_strlen( $str, $encoding = null ) { : strlen( $str ); } +if ( ! function_exists( 'mb_encode_numericentity' ) ) : + /** + * Internal compat function to mimic mb_encode_numericentity(). + * + * Only supports UTF-8 inputs. All other input encodings will be treated as UTF-8. + * + * @since 7.2.0 + * @ignore + * + * @param string $string The string to be encoded. + * @param int[] $map map is array specifies code area to convert. + * @param string|null $encoding Optional. The encoding parameter is the character encoding. If it is + * omitted or null, the internal character encoding value will be used. + * @param bool|null $hex Optional. Whether the returned entity reference should be in + * hexadecimal notation (otherwise it is in decimal notation). + * @return string The converted string. + */ + function mb_encode_numericentity( $string, $map, $encoding = null, $hex = false ) { + if ( isset( $encoding ) && 'UTF-8' !== $encoding ) { + wp_trigger_error( + __FUNCTION__, + 'Only UTF-8 is supported in the wp_encode_numericentity() polyfill; treating as UTF-8.', + E_USER_WARNING + ); + } + + if ( array( 0x10000, 0x10FFFF, 0, 0x1FFFFF ) === $map ) { + return _wp_encode_utf8_numericentity_bytelength( 4, $string, $hex ); + } + + if ( array( 0x80, 0x10FFFF, 0, 0x1FFFFF ) === $map ) { + return _wp_encode_utf8_numericentity_bytelength( 2, $string, $hex ); + } + + return _wp_encode_utf8_numericentity( $string, $map, $hex ); + } +endif; + +if ( ! function_exists( 'mb_substitute_character' ) ) : + /** + * Internal compat function to mimic mb_substitute_character(). + * + * Effectively does nothing, because while this function stores the + * substitute character, it affects no other functions. + * + * @since 7.2.0 + * @ignore + * + * @param string|int|null $substitute_character Specify the Unicode value as an int, or as one of the + * following strings: none, long, entity. + * @return int|string|true If substitute_character is set, it returns true for success, + * otherwise returns false. If substitute_character is not set, + * it returns the current setting. + */ + function mb_substitute_character( $substitute_character = null ) { + static $character = 0x3F; + + if ( ! isset( $substitute_character ) ) { + return $character; + } + + if ( + is_string( $substitute_character ) && + in_array( strtolower( $substitute_character ), array( 'none', 'long', 'entity' ), true ) + ) { + $character = strtolower( $substitute_character ); + return true; + } + + if ( ( is_float( $substitute_character ) && ! is_nan( $substitute_character ) ) || is_bool( $substitute_character ) ) { + $substitute_character = (int) $substitute_character; + } + + if ( + is_int( $substitute_character ) && + 0 <= $substitute_character && + ( $substitute_character < 0xD800 || $substitute_character > 0xDFFF ) && + $substitute_character <= 0x10FFFF + ) { + $character = $substitute_character; + return true; + } + + if ( class_exists( 'ValueError' ) ) { + throw new ValueError( 'Argument #1 ($substitute_character) is not a valid codepoint' ); + } else { + throw new Error( 'Argument #1 ($substitute_character) is not a valid codepoint' ); + } + } +endif; + if ( ! function_exists( 'utf8_encode' ) ) : if ( extension_loaded( 'mbstring' ) ) : /**