Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
3e4a182
Comments: Encode emoji for utf8mb3 comment columns in wp_insert_comme…
Jul 24, 2026
4d26752
Fix test_wp_encode_emoji_in_fields_ignores_fields_missing_from_data()…
Jul 24, 2026
fb1a193
Merge branch 'trunk' into 65700-encode-emoji-comments
bazza Jul 24, 2026
ba90c43
Merge remote-tracking branch 'upstream/trunk' into 65700-encode-emoji…
sirreal Nov 19, 2026
a807662
Update @since for _wp_encode_emoji_in_fields() to 7.2.0.
sirreal Nov 19, 2026
c105494
Add @ignore to _wp_encode_emoji_in_fields() docblock.
sirreal Nov 19, 2026
29eb712
Compare emoji in comment lookups as stored.
sirreal Nov 19, 2026
a1cea18
Rename and document the four-byte character field helper.
sirreal Nov 19, 2026
ff855fd
Encode four-byte UTF-8 characters in utf8mb3 post and comment fields.
sirreal Nov 19, 2026
0ae0b3f
Assert comment insertion succeeds in utf8mb4 control test.
sirreal Nov 19, 2026
fd913e7
Compare comment authors as stored in comment_exists().
sirreal Nov 19, 2026
4141e89
Preserve encodable comment fields when retrying an insert.
sirreal Nov 19, 2026
032e0e9
Render emoji sequences that mix literal characters and character refe…
sirreal Nov 19, 2026
ce022c1
Validate the stored length of comment authors and content.
sirreal Nov 19, 2026
77d634c
Revert "Render emoji sequences that mix literal characters and charac…
sirreal Nov 19, 2026
9aacc96
Restore post emoji encoding to the PR base.
sirreal Nov 19, 2026
ea68872
Require utf8mb4 columns for the comment round-trip test.
sirreal Nov 19, 2026
a231c93
Preserve comment retry cleanup for unencoded fields.
sirreal Nov 19, 2026
cd3cb51
Apply comment field limits to the encoded values.
sirreal Nov 19, 2026
15c0c44
Separate HTML transformation; use mb_encode_numericentity()
dmsnell Nov 20, 2026
58eb8c8
Polyfill mb_encode_numericentity() and mb_substitute_character()
dmsnell Nov 21, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions phpcs.xml.dist
Original file line number Diff line number Diff line change
Expand Up @@ -286,6 +286,11 @@
<exclude-pattern>/wp-tests-config-sample\.php</exclude-pattern>
</rule>

<rule ref="Universal.NamingConventions.NoReservedKeywordParameterNames.stringFound">
<!-- Compatibility polyfills should retain the names of the functions they polyfill. -->
<exclude-pattern>/src/wp-includes/compat.php</exclude-pattern>
</rule>

<rule ref="PEAR.NamingConventions.ValidClassName.Invalid">
<exclude-pattern>/tests/phpunit/tests/*</exclude-pattern>
<!-- Exclude some old classes that cannot be renamed, as it would break back compat. -->
Expand Down
4 changes: 3 additions & 1 deletion src/wp-admin/includes/comment.php
Original file line number Diff line number Diff line change
Expand Up @@ -31,11 +31,13 @@ function comment_exists( $comment_author, $comment_date, $timezone = 'blog' ) {
$date_field = 'comment_date_gmt';
}

$stored = _wp_encode_field_values_for_database( array( 'comment_author' => stripslashes( $comment_author ) ), $wpdb->comments, array( 'comment_author' ) );

return $wpdb->get_var(
$wpdb->prepare(
"SELECT comment_post_ID FROM $wpdb->comments
WHERE comment_author = %s AND $date_field = %s",
stripslashes( $comment_author ),
$stored['comment_author'],

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

going forward this will help, but it’s worth pointing out that this abandons a lot of existing data. there is no general easy solution for this in the database without tradeoffs.

  • we could retrieve all comments matching the date field, transform them one at a time, and pick a winning match
  • migrate data in the database
  • create a new column with the transformed values

none of these are great, and abandoning the existing data is probably fair enough given that the current state of things is that they don’t work.

stripslashes( $comment_date )
)
);
Expand Down
47 changes: 38 additions & 9 deletions src/wp-includes/comment.php
Original file line number Diff line number Diff line change
Expand Up @@ -145,6 +145,8 @@ function check_comment( $author, $email, $url, $comment, $user_ip, $user_agent,
)
);
} else {
$stored = _wp_encode_field_values_for_database( array( 'comment_author' => $author ), $wpdb->comments, array( 'comment_author' ) );

// expected_slashed ($author, $email)
$ok_to_comment = $wpdb->get_var(
$wpdb->prepare(
Expand All @@ -154,7 +156,7 @@ function check_comment( $author, $email, $url, $comment, $user_ip, $user_agent,
AND comment_author_email = %s
AND comment_approved = '1'
LIMIT 1",
$author,
$stored['comment_author'],
$email
)
);
Expand Down Expand Up @@ -768,6 +770,8 @@ function sanitize_comment_cookies() {
function wp_allow_comment( $commentdata, $wp_error = false ) {
global $wpdb;

$stored = _wp_encode_field_values_for_database( $commentdata, $wpdb->comments, array( 'comment_author', 'comment_content' ) );

/*
* Simple duplicate check.
* expected_slashed ($comment_post_ID, $comment_author, $comment_author_email, $comment_content)
Expand All @@ -776,7 +780,7 @@ function wp_allow_comment( $commentdata, $wp_error = false ) {
"SELECT comment_ID FROM $wpdb->comments WHERE comment_post_ID = %d AND comment_parent = %s AND comment_approved != 'trash' AND ( comment_author = %s ",
wp_unslash( $commentdata['comment_post_ID'] ),
wp_unslash( $commentdata['comment_parent'] ),
wp_unslash( $commentdata['comment_author'] )
wp_unslash( $stored['comment_author'] )
);
if ( $commentdata['comment_author_email'] ) {
$dupe .= $wpdb->prepare(
Expand All @@ -786,7 +790,7 @@ function wp_allow_comment( $commentdata, $wp_error = false ) {
}
$dupe .= $wpdb->prepare(
') AND comment_content = %s LIMIT 1',
wp_unslash( $commentdata['comment_content'] )
wp_unslash( $stored['comment_content'] )
);

$dupe_id = $wpdb->get_var( $dupe );
Expand Down Expand Up @@ -1322,16 +1326,20 @@ function wp_get_comment_fields_max_lengths() {
}

/**
* Compares the lengths of comment data against the maximum character limits.
* Checks comment field lengths after preparing the author and content for storage.
*
* @since 4.7.0
*
* @param array $comment_data Array of arguments for inserting a comment.
* @global wpdb $wpdb WordPress database abstraction object.
* @param array $comment_data Array of unslashed arguments for inserting a comment.
* @return WP_Error|true WP_Error when a comment field exceeds the limit,
* otherwise true.
*/
function wp_check_comment_data_max_lengths( $comment_data ) {
$max_lengths = wp_get_comment_fields_max_lengths();
global $wpdb;

$comment_data = _wp_encode_field_values_for_database( $comment_data, $wpdb->comments, array( 'comment_author', 'comment_content' ) );
$max_lengths = wp_get_comment_fields_max_lengths();

if ( isset( $comment_data['comment_author'] ) && mb_strlen( $comment_data['comment_author'], '8bit' ) > $max_lengths['comment_author'] ) {
return new WP_Error( 'comment_author_column_length', __( '<strong>Error:</strong> Your name is too long.' ), 200 );
Expand Down Expand Up @@ -2305,6 +2313,8 @@ function wp_insert_comment( $commentdata ) {
'user_id'
);

$compacted = _wp_encode_field_values_for_database( $compacted, $wpdb->comments, array( 'comment_author', 'comment_content' ) );

if ( ! $wpdb->insert( $wpdb->comments, $compacted ) ) {
return false;
}
Expand Down Expand Up @@ -2561,12 +2571,29 @@ function wp_new_comment( $commentdata, $wp_error = false ) {
$comment_id = wp_insert_comment( $commentdata );

if ( ! $comment_id ) {
$fields = array( 'comment_author', 'comment_author_email', 'comment_author_url', 'comment_content' );
$fields = array( 'comment_author', 'comment_author_email', 'comment_author_url', 'comment_content' );
$unslashed = wp_unslash( $commentdata );
$stored = _wp_encode_field_values_for_database( $unslashed, $wpdb->comments, array( 'comment_author', 'comment_content' ) );

foreach ( $fields as $field ) {
if ( isset( $commentdata[ $field ] ) ) {
$commentdata[ $field ] = $wpdb->strip_invalid_text_for_column( $wpdb->comments, $field, $commentdata[ $field ] );
if ( ! isset( $commentdata[ $field ] ) ) {
continue;
}

if ( $stored[ $field ] !== $unslashed[ $field ] ) {
$value = $stored[ $field ];
$check_lengths = wp_check_comment_data_max_lengths( array( $field => $value ) );
if ( is_wp_error( $check_lengths ) ) {
return $wp_error ? $check_lengths : false;
}

// Keep the original characters for comment filters when the encoded value is valid.
if ( wp_is_valid_utf8( $value ) ) {
continue;
}
}

$commentdata[ $field ] = $wpdb->strip_invalid_text_for_column( $wpdb->comments, $field, $commentdata[ $field ] );
}

$commentdata = wp_filter_comment( $commentdata );
Expand Down Expand Up @@ -3093,6 +3120,8 @@ function wp_update_comment( $commentarr, $wp_error = false ) {

$data = wp_array_slice_assoc( $data, $keys );

$data = _wp_encode_field_values_for_database( $data, $wpdb->comments, array( 'comment_author', 'comment_content' ) );

$result = $wpdb->update( $wpdb->comments, $data, array( 'comment_ID' => $comment_id ) );

if ( false === $result ) {
Expand Down
158 changes: 158 additions & 0 deletions src/wp-includes/compat-utf8.php
Original file line number Diff line number Diff line change
Expand Up @@ -312,6 +312,164 @@ function _wp_scrub_utf8_fallback( string $bytes ): string {
return $scrubbed;
}

/**
* Polyfill for `mb_encode_numericentity()` operating on UTF-8 inputs.
*
* @since 7.2.0
* @access private
* @internal
*
* @param string $bytes The string to be encoded.
* @param int[] $map map is array specifies code area to convert.
* @param bool|null $hex Optional. Whether the returned entity reference should be in
* hexadecimal notation (otherwise it is in decimal notation).
* @return string The converted string.
*/
function _wp_encode_utf8_numericentity( $bytes, $map, $hex = false ) {
$output = '';
$at = 0;
$was_at = 0;
$end = strlen( $bytes );
$fffd = $hex ? '&#xFFFD;' : '&#65533;';
$map_end = count( $map );

if ( 0 !== ( $map_end % 4 ) ) {
if ( class_exists( 'ValueError' ) ) {
throw new ValueError( 'mb_encode_numericentity(): Argument #2 ($map) must have a multiple of 4 elements' );
} else {
throw new Error( 'mb_encode_numericentity(): Argument #2 ($map) must have a multiple of 4 elements' );
}
}

while ( $at < $end ) {
$next_at = $at;
$invalid_length = 0;
$count = _wp_scan_utf8( $bytes, $next_at, $invalid_length, null, 1 );
$char_length = $next_at - $at;
$next_char = substr( $bytes, $at, $char_length );
$code_point = 0 < $count ? mb_ord( $next_char, 'UTF-8' ) : null;

// This is not the most efficient algorithm; install the mbstring extension!
$needs_remapping = false;
$offset = 0;
$mask = 0;
if ( 0 < $count ) {
for ( $i = 0; $i < $map_end; $i += 4 ) {
if ( $code_point >= $map[ $i ] && $code_point <= $map[ $i + 1 ] ) {
$needs_remapping = true;
$offset = $map[ $i + 2 ];
$mask = $map[ $i + 3 ];
break;
}
}
}

if ( ! $needs_remapping && 0 === $invalid_length ) {
$at = $next_at;
continue;
}

if ( isset( $code_point ) && $needs_remapping ) {
$code_point = ( $code_point + $offset ) & $mask;
$digits = $hex ? strtoupper( dechex( $code_point ) ) : (string) $code_point;
$output .= substr( $bytes, $was_at, $at - $was_at );
$output .= $hex ? "&#x{$digits};" : "&#{$digits};";
} else {
$output .= substr( $bytes, $was_at, $at + $char_length - $was_at );
}

if ( $invalid_length > 0 ) {
$output .= $fffd;
}

$at = $next_at + $invalid_length;
$was_at = $at;
}

if ( $was_at < $end ) {
$output .= substr( $bytes, $was_at );
}

return $output;
}

/**
* Polyfill for `mb_encode_numericentity()` operating on UTF-8 inputs when the goal
* is to encode based on byte length of code point spans.
*
* @since 7.2.0
* @access private
* @internal
*
* @param int $byte_length Encode code points taking this many bytes or more to represent.
* @param string $bytes The string to be encoded.
* @param bool|null $hex Optional. Whether the returned entity reference should be in
* hexadecimal notation (otherwise it is in decimal notation).
* @return string The converted string.
*/
function _wp_encode_utf8_numericentity_bytelength( $byte_length, $bytes, $hex = false ) {
$output = '';
$at = 0;
$was_at = 0;
$end = strlen( $bytes );
$fffd = $byte_length > 3 ? "\u{FFFD}" : ( $hex ? '&#xFFFD;' : '&#65533;' );

while ( $at < $end ) {
/*
* Quickly skip past US-ASCII bytes, all of which are valid UTF-8.
*
* This optimization step improves the speed from 10x to 100x
* depending on whether the JIT has optimized the function.
*/
$ascii_byte_count = strspn(
$bytes,
"\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f" .
"\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f" .
" !\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\x7f",
$at,
);

$at += $ascii_byte_count;
if ( $at >= $end ) {
break;
}

$next_at = $at;
$invalid_length = 0;
$count = _wp_scan_utf8( $bytes, $next_at, $invalid_length, null, 1 );
$char_length = $next_at - $at;
$next_char = 0 < $count ? substr( $bytes, $at, $char_length ) : null;

// The character is within bounds and is followed by no invalid bytes.
if ( $char_length < $byte_length && 0 === $invalid_length ) {
$at = $next_at;
continue;
}

if ( isset( $next_char ) && $char_length >= $byte_length ) {
$code_point = mb_ord( $next_char, 'UTF-8' );
$output .= substr( $bytes, $was_at, $at - $was_at );
$digits = $hex ? strtoupper( dechex( $code_point ) ) : (string) $code_point;
$output .= $hex ? "&#x{$digits};" : "&#{$digits};";
} else {
$output .= substr( $bytes, $was_at, $at + $char_length - $was_at );
}

if ( $invalid_length > 0 ) {
$output .= $fffd;
}

$at = $next_at + $invalid_length;
$was_at = $at;
}

if ( $was_at < $end ) {
$output .= substr( $bytes, $was_at );
}

return $output;
}

/**
* Returns how many code points are found in the given UTF-8 string.
*
Expand Down
Loading
Loading