From 60e126c4fbda36e6149652374308cbebf8a58bf7 Mon Sep 17 00:00:00 2001 From: Richard Davison Date: Thu, 3 Apr 2025 14:52:18 +0200 Subject: [PATCH 1/3] Fix broken surrogate pairs --- core/src/value/string.rs | 117 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 115 insertions(+), 2 deletions(-) diff --git a/core/src/value/string.rs b/core/src/value/string.rs index 5fff424ea..b4614a0e0 100644 --- a/core/src/value/string.rs +++ b/core/src/value/string.rs @@ -10,6 +10,7 @@ impl<'js> String<'js> { /// Convert the JavaScript string to a Rust string. pub fn to_string(&self) -> Result { let mut len = mem::MaybeUninit::uninit(); + let ptr = unsafe { qjs::JS_ToCStringLen(self.0.ctx.as_ptr(), len.as_mut_ptr(), self.0.as_js_value()) }; @@ -20,9 +21,108 @@ impl<'js> String<'js> { } let len = unsafe { len.assume_init() }; let bytes: &[u8] = unsafe { slice::from_raw_parts(ptr as _, len as _) }; - let result = str::from_utf8(bytes).map(|s| s.into()); + + let string = Self::replace_invalid_utf8_and_utf16(bytes); + unsafe { qjs::JS_FreeCString(self.0.ctx.as_ptr(), ptr) }; - Ok(result?) + Ok(string) + } + + fn replace_invalid_utf8_and_utf16(bytes: &[u8]) -> StdString { + let mut result = StdString::with_capacity(bytes.len()); + let mut i = 0; + + while i < bytes.len() { + let current = bytes[i]; + match current { + // ASCII (1-byte) + 0x00..=0x7F => { + result.push(current as char); + i += 1; + } + // 2-byte UTF-8 sequence + 0xC0..=0xDF => { + if i + 1 < bytes.len() { + let next = bytes[i + 1]; + if (next & 0xC0) == 0x80 { + let code_point = ((current as u32 & 0x1F) << 6) | (next as u32 & 0x3F); + if let Some(c) = char::from_u32(code_point) { + result.push(c); + } else { + result.push('�'); + } + i += 2; + } else { + result.push('�'); + i += 1; + } + } else { + result.push('�'); + i += 1; + } + } + // 3-byte UTF-8 sequence + 0xE0..=0xEF => { + if i + 2 < bytes.len() { + let next1 = bytes[i + 1]; + let next2 = bytes[i + 2]; + if (next1 & 0xC0) == 0x80 && (next2 & 0xC0) == 0x80 { + let code_point = ((current as u32 & 0x0F) << 12) + | ((next1 as u32 & 0x3F) << 6) + | (next2 as u32 & 0x3F); + if let Some(c) = char::from_u32(code_point) { + result.push(c); + } else { + result.push('�'); + } + i += 3; + } else { + result.push('�'); + i += 1; + } + } else { + result.push('�'); + i += 1; + } + } + // 4-byte UTF-8 sequence + 0xF0..=0xF7 => { + if i + 3 < bytes.len() { + let next1 = bytes[i + 1]; + let next2 = bytes[i + 2]; + let next3 = bytes[i + 3]; + if (next1 & 0xC0) == 0x80 + && (next2 & 0xC0) == 0x80 + && (next3 & 0xC0) == 0x80 + { + let code_point = ((current as u32 & 0x07) << 18) + | ((next1 as u32 & 0x3F) << 12) + | ((next2 as u32 & 0x3F) << 6) + | (next3 as u32 & 0x3F); + if let Some(c) = char::from_u32(code_point) { + result.push(c); + } else { + result.push('�'); + } + i += 4; + } else { + result.push('�'); + i += 1; + } + } else { + result.push('�'); + i += 1; + } + } + // Invalid starting byte + _ => { + result.push('�'); + i += 1; + } + } + } + + result } /// Convert the Javascript string to a Javascript C string. @@ -142,4 +242,17 @@ mod test { assert_eq!(text, "foobar".to_string()); }); } + + #[test] + fn utf8_sliced_string() { + test_with(|ctx| { + let string = String::from_str(ctx.clone(), "🌍🌎🌏").unwrap(); + + assert_eq!(string.to_string().unwrap(), "🌍🌎🌏".to_string()); + + let func: Function = ctx.eval("x => x.slice(1)").unwrap(); + let text: StdString = (string,).apply(&func).unwrap(); + assert_eq!(text, "�🌎🌏".to_string()); + }); + } } From 5ba6ceb3839c1478fbed4c20b7ca80a79009f1fc Mon Sep 17 00:00:00 2001 From: Richard Davison Date: Thu, 3 Apr 2025 15:03:31 +0200 Subject: [PATCH 2/3] Add to_lossy_method --- core/src/value/string.rs | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/core/src/value/string.rs b/core/src/value/string.rs index b4614a0e0..41751b85f 100644 --- a/core/src/value/string.rs +++ b/core/src/value/string.rs @@ -9,8 +9,23 @@ pub struct String<'js>(pub(crate) Value<'js>); impl<'js> String<'js> { /// Convert the JavaScript string to a Rust string. pub fn to_string(&self) -> Result { - let mut len = mem::MaybeUninit::uninit(); + let (ptr, len) = self.get_ptr_len()?; + let bytes: &[u8] = unsafe { slice::from_raw_parts(ptr as _, len as _) }; + let result = str::from_utf8(bytes).map(|s| s.into()); + unsafe { qjs::JS_FreeCString(self.0.ctx.as_ptr(), ptr) }; + Ok(result?) + } + pub fn to_string_lossy(&self) -> Result { + let (ptr, len) = self.get_ptr_len()?; + let bytes: &[u8] = unsafe { slice::from_raw_parts(ptr as _, len as _) }; + let string = Self::replace_invalid_utf8_and_utf16(bytes); + unsafe { qjs::JS_FreeCString(self.0.ctx.as_ptr(), ptr) }; + Ok(string) + } + + fn get_ptr_len(&self) -> Result<(*const i8, usize)> { + let mut len = mem::MaybeUninit::uninit(); let ptr = unsafe { qjs::JS_ToCStringLen(self.0.ctx.as_ptr(), len.as_mut_ptr(), self.0.as_js_value()) }; @@ -20,12 +35,7 @@ impl<'js> String<'js> { return Err(Error::Unknown); } let len = unsafe { len.assume_init() }; - let bytes: &[u8] = unsafe { slice::from_raw_parts(ptr as _, len as _) }; - - let string = Self::replace_invalid_utf8_and_utf16(bytes); - - unsafe { qjs::JS_FreeCString(self.0.ctx.as_ptr(), ptr) }; - Ok(string) + Ok((ptr, len)) } fn replace_invalid_utf8_and_utf16(bytes: &[u8]) -> StdString { From 4b7b4cc4ff1584f84e816596e5531c53aef14668 Mon Sep 17 00:00:00 2001 From: Richard Davison Date: Thu, 3 Apr 2025 15:07:41 +0200 Subject: [PATCH 3/3] Fix test --- core/src/value/string.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/core/src/value/string.rs b/core/src/value/string.rs index 41751b85f..eda2d598c 100644 --- a/core/src/value/string.rs +++ b/core/src/value/string.rs @@ -259,9 +259,12 @@ mod test { let string = String::from_str(ctx.clone(), "🌍🌎🌏").unwrap(); assert_eq!(string.to_string().unwrap(), "🌍🌎🌏".to_string()); + assert_eq!(string.to_string_lossy().unwrap(), "🌍🌎🌏".to_string()); let func: Function = ctx.eval("x => x.slice(1)").unwrap(); - let text: StdString = (string,).apply(&func).unwrap(); + let text: String = (string,).apply(&func).unwrap(); + let text = text.to_string_lossy().unwrap(); + assert_eq!(text, "�🌎🌏".to_string()); }); }