Skip to content

Better handle combined html_safe and ascii_only #1145

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
byroot merged 1 commit into from
Nov 14, 2026
+222 −183
Merged

Better handle combined html_safe and ascii_only #1145

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
208 changes: 110 additions & 98 deletions ext/json/ext/generator/generator.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -104,8 +104,11 @@ NORETURN(static void) raise_generator_error(VALUE invalid_object, const char *fm

// 0 - single byte char that don't need to be escaped.
// (x | 8) - char that needs to be escaped.
static const unsigned char CHAR_LENGTH_MASK = 7;
static const unsigned char ESCAPE_MASK = 8;
#define CHAR_LENGTH_MASK 7
#define ESCAPE_MASK 8
#define HTML_ESCAPE_MASK 16
#define ESCAPE_LEN_1 9 // ESCAPE_MASK | 1
#define HTML_ESCAPE_LEN_1 25 // HTML_ESCAPE_MASK | ESCAPE_LEN_1

typedef struct _search_state {
const char *ptr;
Expand Down Expand Up @@ -155,6 +158,84 @@ static const unsigned char escape_table_basic[256] = {
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
};

static const unsigned char script_safe_escape_table[256] = {
// ASCII Control Characters
9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
// ASCII Characters
0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // '"'
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,25, 0, 0, 0, // '<'
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, // '\\'
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
// Continuation byte
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
// First byte of a 2-byte code point
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
// First byte of a 3-byte code point
3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
//First byte of a 4+ byte code point
4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 9, 9,
};

static const unsigned char ascii_only_escape_table[256] = {
// ASCII Control Characters
9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
// ASCII Characters
0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // '"'
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, // '\\'
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
// Continuation byte
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
// First byte of a 2-byte code point
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
// First byte of a 3-byte code point
3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
//First byte of a 4+ byte code point
4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 9, 9,
};

static inline bool unsafe_html_tag_p(const char *ptr, const char *end)
{
size_t remaining = end - ptr;

if (remaining < 4) return false;
if (memcmp(ptr, "<!--", 4) == 0) return true;

union { uint64_t num; char bytes[8]; } chunk;
if (remaining >= 8) {
memcpy(chunk.bytes, ptr, 8);
}
else if (remaining == 7) {
chunk.num = 0;
memcpy(chunk.bytes, ptr, 7);
}
else {
return false;
}

chunk.num |= 0x2020202020202020;

if (memcmp(chunk.bytes, "</script", 8) == 0) return true;

chunk.bytes[7] = 0;

return memcmp(chunk.bytes, "<script\0", 8) == 0;
}

static inline unsigned char search_escape_basic(search_state *search)
{
while (search->ptr < search->end) {
Expand Down Expand Up @@ -269,16 +350,18 @@ static inline void escape_UTF8_char(search_state *search, unsigned char ch_len)
}
break;
}
case 3: {
if (search->ptr[2] & 1) {
fbuffer_append(search->buffer, "\\u2029", 6);
case HTML_ESCAPE_LEN_1: {
if (unsafe_html_tag_p(search->ptr, search->end)) {
search_flush(search);
fbuffer_append(search->buffer, "\\u003c", 6);
} else {
fbuffer_append(search->buffer, "\\u2028", 6);
search->ptr++;
return; // no escape needed, resume search without updating cursor.
}
break;
}
}
search->cursor = (search->ptr += ch_len);
search->cursor = (search->ptr += (ch_len & CHAR_LENGTH_MASK));
}

#ifdef HAVE_SIMD
Expand Down Expand Up @@ -511,59 +594,6 @@ ALWAYS_INLINE(static) TARGET_SSE2 unsigned char search_escape_basic_sse2(search_

#endif /* HAVE_SIMD */

static const unsigned char script_safe_escape_table[256] = {
// ASCII Control Characters
9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
// ASCII Characters
0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // '"'
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,25, 0, 0, 0, // '<'
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, // '\\'
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
// Continuation byte
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
// First byte of a 2-byte code point
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
// First byte of a 3-byte code point
3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
//First byte of a 4+ byte code point
4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 9, 9,
};

static inline bool unsafe_html_tag_p(const char *ptr, const char *end)
{
size_t remaining = end - ptr;

if (remaining < 4) return false;
if (memcmp(ptr, "<!--", 4) == 0) return true;

union { uint64_t num; char bytes[8]; } chunk;
if (remaining >= 8) {
memcpy(chunk.bytes, ptr, 8);
}
else if (remaining == 7) {
chunk.num = 0;
memcpy(chunk.bytes, ptr, 7);
}
else {
return false;
}

chunk.num |= 0x2020202020202020;

if (memcmp(chunk.bytes, "</script", 8) == 0) return true;

chunk.bytes[7] = 0;

return memcmp(chunk.bytes, "<script\0", 8) == 0;
}

static inline unsigned char search_script_safe_escape(search_state *search)
{
while (search->ptr < search->end) {
Expand All @@ -572,15 +602,9 @@ static inline unsigned char search_script_safe_escape(search_state *search)

if (RB_UNLIKELY(ch_len)) {
if (ch_len & ESCAPE_MASK) {
if (RB_UNLIKELY(ch_len == 25)) {
if (!unsafe_html_tag_p(search->ptr, search->end)) {
search->ptr += 1;
continue;
}
}
search_flush(search);
return ch_len & CHAR_LENGTH_MASK;
} else {
return ch_len;
}
else {
search->ptr += ch_len;
}
} else {
Expand All @@ -599,40 +623,14 @@ static void convert_UTF8_to_script_safe_JSON(search_state *search)
}
}

static const unsigned char ascii_only_escape_table[256] = {
// ASCII Control Characters
9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
// ASCII Characters
0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // '"'
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, // '\\'
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
// Continuation byte
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
// First byte of a 2-byte code point
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
// First byte of a 3-byte code point
3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
//First byte of a 4+ byte code point
4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 9, 9,
};

static inline unsigned char search_ascii_only_escape(search_state *search, const unsigned char escape_table[256])
{
while (search->ptr < search->end) {
unsigned char ch = (unsigned char)*search->ptr;
unsigned char ch_len = escape_table[ch];

if (RB_UNLIKELY(ch_len)) {
search_flush(search);
return ch_len & CHAR_LENGTH_MASK;
return ch_len;
} else {
search->ptr++;
}
Expand All @@ -645,7 +643,9 @@ static inline void full_escape_UTF8_char(search_state *search, unsigned char ch_
{
const unsigned char ch = (unsigned char)*search->ptr;
switch (ch_len) {
case 1: {
case ESCAPE_LEN_1: {
search_flush(search);

switch (ch) {
case '"': fbuffer_append(search->buffer, "\\\"", 2); break;
case '\\': fbuffer_append(search->buffer, "\\\\", 2); break;
Expand All @@ -666,7 +666,19 @@ static inline void full_escape_UTF8_char(search_state *search, unsigned char ch_
}
break;
}
case HTML_ESCAPE_LEN_1: {
if (unsafe_html_tag_p(search->ptr, search->end)) {
search_flush(search);
fbuffer_append(search->buffer, "\\u003c", 6);
} else {
search->ptr++;
return; // no escape needed, resume search without updating cursor.
}
break;
}
default: {
search_flush(search);

const char *hexdig = "0123456789abcdef";
char scratch[12] = { '\\', 'u', 0, 0, 0, 0, '\\', 'u' };

Expand Down Expand Up @@ -716,7 +728,7 @@ static inline void full_escape_UTF8_char(search_state *search, unsigned char ch_
break;
}
}
search->cursor = (search->ptr += ch_len);
search->cursor = (search->ptr += (ch_len & CHAR_LENGTH_MASK));
}

static void convert_UTF8_to_ASCII_only_JSON(search_state *search, const unsigned char escape_table[256])
Expand Down
47 changes: 23 additions & 24 deletions java/src/json/ext/StringEncoder.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -113,8 +113,7 @@ class StringEncoder extends ByteListTranscoder {
};

private static final ByteBuffer HTML_COMMENT = ByteBuffer.wrap("<!--".getBytes(StandardCharsets.US_ASCII));
private static final ByteBuffer HTML_SCRIPT_OPEN = ByteBuffer.wrap("<script".getBytes(StandardCharsets.US_ASCII));
private static final ByteBuffer HTML_SCRIPT_CLOSE = ByteBuffer.wrap("</script".getBytes(StandardCharsets.US_ASCII));
protected static final byte[] ESCAPED_LT = "\\u003c".getBytes(StandardCharsets.US_ASCII);

protected final byte[] escapeTable;

Expand Down Expand Up @@ -274,7 +273,7 @@ void encodeBasic(ByteList src) throws IOException {
}
}

private boolean isUnsafeScriptTag(byte[] ptrBytes, int pos, int len) {
protected boolean isUnsafeScriptTag(byte[] ptrBytes, int pos, int len) {
int remaining = len - pos;

if ((remaining >= 4 && ByteBuffer.wrap(ptrBytes, pos, 4).equals(HTML_COMMENT))) {
Expand Down Expand Up @@ -323,31 +322,27 @@ void encode(ByteList src) throws IOException {
int ch_len = escapeTable[ch];
/* JSON encoding */

if (ch_len > 0) {
switch (ch_len) {
case 9: {
switch (ch_len) {
case 0:
pos++;
break;
case 9: {
beg = pos = flushPos(pos, beg, ptrBytes, ptr, 1);
escapeAscii(ch, scratch, hexdig);
break;
}
case 25: {
if (isUnsafeScriptTag(ptrBytes, pos, len)) {
beg = pos = flushPos(pos, beg, ptrBytes, ptr, 1);
escapeAscii(ch, scratch, hexdig);
break;
} else {
pos++; // nothing to escape, resume search.
}
case 25: {
int remaining = len - pos;
if (remaining > 8) remaining = 8;

if (isUnsafeScriptTag(ptrBytes, pos, len)) {
beg = pos = flushPos(pos, beg, ptrBytes, ptr, 1);
escapeAscii(ch, scratch, hexdig);
break;
}
ch_len = 1;
// fallthrough
}
default:
pos += ch_len;
break;
break;
}
} else {
pos++;
default:
pos += ch_len;
break;
}
}

Expand Down Expand Up @@ -394,6 +389,10 @@ protected void append(byte[] origin, int start, int length) throws IOException {
out.write(origin, start, length);
}

protected void append(byte[] origin) throws IOException {
append(origin, 0, origin.length);
}

@Override
protected RaiseException invalidUtf8(ThreadContext context) {
return Utils.newException(context, Utils.M_GENERATOR_ERROR, "source sequence is illegal/malformed utf-8");
Expand Down
Loading
Loading