From 8106f1941825fcbd1b66d70c14a2a68c493e5b10 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 26 Aug 2026 04:02:10 +0900 Subject: [PATCH 01/11] Fix %-formatting of bytes precision, width and %c errors `format_bytes` subtracted the byte length from the field width without saturating, so `b'%1s' % (b'abcde',)` aborted with a capacity overflow. A precision written as a bare dot now truncates to nothing, and `%r`/`%a` on bytes now honour the precision and width like the other conversions. `%c` with a bytes, bytearray or str argument that is not one unit long now names the argument by its length instead of by its type. Assisted-by: Claude Code:claude-opus-5 --- crates/common/src/cformat.rs | 15 ++++++++------- crates/vm/src/cformat.rs | 24 ++++++++++++++++++------ 2 files changed, 26 insertions(+), 13 deletions(-) diff --git a/crates/common/src/cformat.rs b/crates/common/src/cformat.rs index f018dd32f93..6a37718cb21 100644 --- a/crates/common/src/cformat.rs +++ b/crates/common/src/cformat.rs @@ -426,15 +426,16 @@ impl CFormatSpec { #[must_use] pub fn format_bytes(&self, bytes: &[u8]) -> Vec { - let bytes = if let Some(CFormatPrecision::Quantity(CFormatQuantity::Amount(precision))) = - self.precision - { - &bytes[..cmp::min(bytes.len(), precision)] - } else { - bytes + let bytes = match self.precision { + Some(CFormatPrecision::Quantity(CFormatQuantity::Amount(precision))) => { + &bytes[..cmp::min(bytes.len(), precision)] + } + // A precision written as a bare dot truncates to nothing. + Some(CFormatPrecision::Dot) => &bytes[..0], + _ => bytes, }; if let Some(CFormatQuantity::Amount(width)) = self.min_field_width { - let fill = cmp::max(0, width - bytes.len()); + let fill = width.saturating_sub(bytes.len()); let mut v = Vec::with_capacity(bytes.len() + fill); if self.flags.contains(CConversionFlags::LEFT_ADJUST) { v.extend_from_slice(bytes); diff --git a/crates/vm/src/cformat.rs b/crates/vm/src/cformat.rs index 00147103851..daab91f878b 100644 --- a/crates/vm/src/cformat.rs +++ b/crates/vm/src/cformat.rs @@ -35,8 +35,7 @@ fn spec_format_bytes( // Unlike strings, %r and %a are identical for bytes: the behaviour corresponds to // %a for strings (not %r) CFormatType::String(CFormatConversion::Repr | CFormatConversion::Ascii) => { - let b = builtins::ascii(obj, vm)?.as_bytes().to_vec(); - Ok(b) + Ok(spec.format_bytes(builtins::ascii(obj, vm)?.as_bytes())) } // %b and %s are equivalent for bytes formatting. // Mirrors CPython's format_obj() in bytesobject.c @@ -148,9 +147,17 @@ fn spec_format_bytes( } else if let Some(int_result) = obj.try_index_opt(vm) { int_result? } else { + // A bytes-like argument that is not one byte long is named by + // its length rather than by its type. + let what = if let Some(b) = obj.downcast_ref::() { + format!("a bytes object of length {}", b.len()) + } else if let Some(ba) = obj.downcast_ref::() { + format!("a bytearray object of length {}", ba.borrow_buf().len()) + } else { + obj.class().name().to_string() + }; return Err(vm.new_type_error(format!( - "%c requires an integer in range(256) or a single byte, not {}", - obj.class().name() + "%c requires an integer in range(256) or a single byte, not {what}" ))); }; let ch = int @@ -248,9 +255,14 @@ fn spec_format_string( } else if let Some(int_result) = obj.try_index_opt(vm) { int_result? } else { + // A string argument that is not one character long is named by + // its length rather than by its type. + let what = match obj.downcast_ref::() { + Some(s) => format!("a string of length {}", s.char_len()), + None => obj.class().name().to_string(), + }; return Err(vm.new_type_error(format!( - "%c requires an int or a unicode character, not {}", - obj.class().name() + "%c requires an int or a unicode character, not {what}" ))); }; let ch = int From 057637bffb18a736d71e51fbd0e2c260f4fb4d50 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 26 Aug 2026 04:02:25 +0900 Subject: [PATCH 02/11] Fix format spec handling of grouping, bool, 'c' and complex A grouping option next to an unrecognised presentation type now reports "Cannot specify ',' with 'a'." instead of falling through to the type check. `format(True, spec)` now validates the spec the way an int does. The 'c' type now pads and aligns its character, and an integer outside the C long range raises OverflowError rather than reporting a code point out of range. A complex formatted with the 'n'/'g' types or with a precision and no type now goes through the general format without forcing a fraction. Assisted-by: Claude Code:claude-opus-5 --- crates/common/src/format.rs | 64 ++++++++++++++++++------------------- crates/vm/src/format.rs | 3 ++ 2 files changed, 35 insertions(+), 32 deletions(-) diff --git a/crates/common/src/format.rs b/crates/common/src/format.rs index 0308721975a..6bbbdf2d04e 100644 --- a/crates/common/src/format.rs +++ b/crates/common/src/format.rs @@ -491,14 +491,18 @@ impl FormatSpec { | FormatType::Binary | FormatType::Octal | FormatType::Hex(_) - | FormatType::Number(_), + | FormatType::Number(_) + | FormatType::Unknown(_), ) => { let ch = char::from(format_type); Err(FormatSpecError::UnspecifiedFormat(',', ch)) } ( Some(FormatGrouping::Underscore), - FormatType::String | FormatType::Character | FormatType::Number(_), + FormatType::String + | FormatType::Character + | FormatType::Number(_) + | FormatType::Unknown(_), ) => { let ch = char::from(format_type); Err(FormatSpecError::UnspecifiedFormat('_', ch)) @@ -859,6 +863,7 @@ impl FormatSpec { } pub fn format_bool(&self, input: bool) -> Result { + self.validate_format(FormatType::Decimal)?; let x = u8::from(input); match &self.format_type { Some( @@ -1061,11 +1066,16 @@ impl FormatSpec { match (self.sign, self.alternate_form) { (Some(_), _) => Err(FormatSpecError::NotAllowed("Sign")), (_, true) => Err(FormatSpecError::NotAllowed("Alternate form (#)")), - _ => match num.to_u32() { - Some(n) if n <= 0x10ffff => { - Ok(core::char::from_u32(n).unwrap().to_string()) + _ => match num + .to_i64() + .filter(|code| core::ffi::c_long::try_from(*code).is_ok()) + { + None => Err(FormatSpecError::IntTooLargeForCLong), + Some(n @ 0..=0x10ffff) => { + let ch = core::char::from_u32(n as u32).unwrap().to_string(); + return Ok(self.format_sign_and_align(&ch, "", FormatAlign::Right)); } - Some(_) | None => Err(FormatSpecError::CodeNotInRange), + Some(_) => Err(FormatSpecError::CodeNotInRange), }, } } @@ -1200,6 +1210,17 @@ impl FormatSpec { self.validate_format(FormatType::FixedPoint(Case::Lower))?; let precision = self.precision.unwrap_or(6); let magnitude = num.abs(); + // A zero precision means one significant digit, and a part of a + // complex keeps no fraction of its own the way a bare float does. + let general = |case| { + float::format_general( + if precision == 0 { 1 } else { precision }, + magnitude, + case, + self.alternate_form, + false, + ) + }; let magnitude_str = match &self.format_type { Some( FormatType::Decimal @@ -1221,16 +1242,7 @@ impl FormatSpec { *case, self.alternate_form, )), - Some(FormatType::GeneralFormat(case) | FormatType::Number(case)) => { - let precision = if precision == 0 { 1 } else { precision }; - Ok(float::format_general( - precision, - magnitude, - *case, - self.alternate_form, - false, - )) - } + Some(FormatType::GeneralFormat(case) | FormatType::Number(case)) => Ok(general(*case)), Some(FormatType::Exponent(case)) => Ok(float::format_exponent( precision, magnitude, @@ -1240,22 +1252,9 @@ impl FormatSpec { None => match magnitude { magnitude if magnitude.is_nan() => Ok("nan".to_owned()), magnitude if magnitude.is_infinite() => Ok("inf".to_owned()), - _ => match self.precision { - Some(precision) => Ok(float::format_general( - precision, - magnitude, - Case::Lower, - self.alternate_form, - true, - )), - None => { - if magnitude.fract() == 0.0 { - Ok(magnitude.trunc().to_string()) - } else { - Ok(magnitude.to_string()) - } - } - }, + _ if self.precision.is_some() => Ok(general(Case::Lower)), + magnitude if magnitude.fract() == 0.0 => Ok(magnitude.trunc().to_string()), + magnitude => Ok(magnitude.to_string()), }, }?; let magnitude_str = match &self.grouping_option { @@ -1367,6 +1366,7 @@ pub enum FormatSpecError { NotAllowed(&'static str), UnableToConvert, CodeNotInRange, + IntTooLargeForCLong, ZeroPadding, AlignmentFlag, NegativeZeroCoercionNotAllowed(&'static str), diff --git a/crates/vm/src/format.rs b/crates/vm/src/format.rs index de2391dcde5..9a54df3b4cb 100644 --- a/crates/vm/src/format.rs +++ b/crates/vm/src/format.rs @@ -71,6 +71,9 @@ impl IntoPyException for FormatSpecError { } Self::UnableToConvert => vm.new_value_error("Unable to convert int to float"), Self::CodeNotInRange => vm.new_overflow_error("%c arg not in range(0x110000)"), + Self::IntTooLargeForCLong => { + vm.new_overflow_error("Python int too large to convert to C long") + } Self::ZeroPadding => { vm.new_value_error("Zero padding is not allowed in complex format specifier") } From 1d7a3317c4e80c9b1075168b8e3c300ecd7dffeb Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 26 Aug 2026 04:02:27 +0900 Subject: [PATCH 03/11] Round floats at a negative ndigits from the exact decimal digits Dividing by a power of ten rounded twice: once when the quotient was stored and again at the tie, so e.g. `round(2.5e2, -2)` came out as 300.0. The digits now come from the exact decimal expansion of the value. Assisted-by: Claude Code:claude-opus-5 --- crates/common/src/float_ops.rs | 63 +++++++++++++++++++++++++--------- 1 file changed, 47 insertions(+), 16 deletions(-) diff --git a/crates/common/src/float_ops.rs b/crates/common/src/float_ops.rs index 4f117e80c46..ac4e6b0bfcb 100644 --- a/crates/common/src/float_ops.rs +++ b/crates/common/src/float_ops.rs @@ -255,22 +255,7 @@ pub fn round_float_digits(x: f64, ndigits: i32) -> Option { let s = format!("{:.*}", ndigits as usize, x); s.parse().ok()? } else { - // ndigits < 0: divide-then-round avoids the phantom-tie problem - // because dividing typical inputs by 10**|ndigits| produces genuine - // half-integer ties rather than synthesizing them. - let pow1 = 10.0f64.powi(-ndigits); - let y = x / pow1; - let z = y.round(); - #[expect( - clippy::float_cmp, - reason = "exact half-tie detection for banker's rounding correction" - )] - let z = if (y - z).abs() == 0.5 { - 2.0 * (y / 2.0).round() - } else { - z - }; - z * pow1 + round_at_power_of_ten(x, (-ndigits) as usize)? }; if !result.is_finite() { @@ -279,6 +264,52 @@ pub fn round_float_digits(x: f64, ndigits: i32) -> Option { Some(result) } +/// Round `x` half-to-even at the `10.pow(place)` digit, the way +/// `_Py_dg_dtoa` in mode 3 does for a negative `ndigits`. +/// +/// The digits come from the exact decimal expansion rather than from a divide +/// by `10.pow(place)`, which rounds twice: once when the quotient is stored +/// and again at the tie. +fn round_at_power_of_ten(x: f64, place: usize) -> Option { + // `trunc` is exact, so formatting it writes the integer digits themselves. + let digits = format!("{:.0}", x.trunc().abs()); + let has_fraction = x.fract() != 0.0; + + let padded = format!("{digits:0>width$}", width = place + 1); + let (kept, dropped) = padded.split_at(padded.len() - place); + let mut kept: Vec = kept.bytes().collect(); + + // The dropped digits read as a fraction of the rounding place: "5" then + // zeros is the tie, which the fraction below them, if any, tips over. + let round_up = match dropped.as_bytes().split_first() { + None => false, + Some((&first, rest)) => { + first > b'5' + || (first == b'5' + && (rest.iter().any(|&digit| digit != b'0') + || has_fraction + || kept.last().is_some_and(|digit| (digit - b'0') % 2 == 1))) + } + }; + + if round_up { + // The carry stops at the first digit that does not wrap; if none of + // them stops it, the number has grown a digit. + let carried = kept.iter_mut().rev().all(|digit| { + *digit = if *digit == b'9' { b'0' } else { *digit + 1 }; + *digit == b'0' + }); + if carried { + kept.insert(0, b'1'); + } + } + + let mut rounded = String::from_utf8(kept).ok()?; + rounded.extend(core::iter::repeat_n('0', place)); + let magnitude: f64 = rounded.parse().ok()?; + Some(magnitude.copysign(x)) +} + /// Error from [`from_hex`], mapping to the exception the caller should raise. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum HexFloatError { From ff47c3fca431cc7fd6d18ea11fea5d7000c03078 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 26 Aug 2026 04:02:28 +0900 Subject: [PATCH 04/11] Report the requested base in int literal errors and reject a bare sign An invalid literal parsed with base 0 named the base that the prefix turned out to mean rather than 0. A string of nothing but a sign was taken for the old octal form and returned 0 instead of raising. Assisted-by: Claude Code:claude-opus-5 --- crates/common/src/int.rs | 48 ++++++++++++++++++++++++++++++---------- 1 file changed, 36 insertions(+), 12 deletions(-) diff --git a/crates/common/src/int.rs b/crates/common/src/int.rs index 0a8f054bf1a..06dc46c4098 100644 --- a/crates/common/src/int.rs +++ b/crates/common/src/int.rs @@ -48,6 +48,9 @@ pub fn bytes_to_int( if base != 0 && !(2..=36).contains(&base) { return Err(BytesToIntError::InvalidBase); } + // A rejected literal names the base that was asked for, not the one a + // base of 0 turned out to mean. + let requested_base = base; let mut buf = buf.trim_ascii(); @@ -55,7 +58,11 @@ pub fn bytes_to_int( let sign = match buf.first() { Some(b'+') => Some(Sign::Plus), Some(b'-') => Some(Sign::Minus), - None => return Err(BytesToIntError::InvalidLiteral { base }), + None => { + return Err(BytesToIntError::InvalidLiteral { + base: requested_base, + }); + } _ => None, }; @@ -67,6 +74,9 @@ pub fn bytes_to_int( if base == 0 { match (buf.first(), buf.get(1)) { (Some(v), _) if *v != b'0' => base = 10, + // A sign on its own leaves nothing to read, which is not the + // old octal form either. + (None, _) => base = 10, (_, Some(b'x' | b'X')) => base = 16, (_, Some(b'o' | b'O')) => base = 8, (_, Some(b'b' | b'B')) => base = 2, @@ -82,7 +92,9 @@ pub fn bytes_to_int( if let [_first, others @ .., last] = buf { let is_zero = *last == b'0' && others.iter().all(|&c| c == b'0' || c == b'_'); if !is_zero { - return Err(BytesToIntError::InvalidLiteral { base }); + return Err(BytesToIntError::InvalidLiteral { + base: requested_base, + }); } } return Ok(BigInt::zero()); @@ -104,13 +116,15 @@ pub fn bytes_to_int( } // Reject empty strings - let mut prev = *buf - .first() - .ok_or(BytesToIntError::InvalidLiteral { base })?; + let mut prev = *buf.first().ok_or(BytesToIntError::InvalidLiteral { + base: requested_base, + })?; // Leading underscore not allowed if prev == b'_' || !prev.is_ascii_alphanumeric() { - return Err(BytesToIntError::InvalidLiteral { base }); + return Err(BytesToIntError::InvalidLiteral { + base: requested_base, + }); } // Verify all characters are digits and underscores @@ -119,12 +133,16 @@ pub fn bytes_to_int( if cur == b'_' { // Double underscore not allowed if prev == b'_' { - return Err(BytesToIntError::InvalidLiteral { base }); + return Err(BytesToIntError::InvalidLiteral { + base: requested_base, + }); } } else if cur.is_ascii_alphanumeric() { digits += 1; } else { - return Err(BytesToIntError::InvalidLiteral { base }); + return Err(BytesToIntError::InvalidLiteral { + base: requested_base, + }); } prev = cur; @@ -132,7 +150,9 @@ pub fn bytes_to_int( // Trailing underscore not allowed if prev == b'_' { - return Err(BytesToIntError::InvalidLiteral { base }); + return Err(BytesToIntError::InvalidLiteral { + base: requested_base, + }); } if digit_limit > 0 && !base.is_power_of_two() && digits > digit_limit { @@ -142,7 +162,9 @@ pub fn bytes_to_int( }); } - let uint = BigUint::parse_bytes(buf, base).ok_or(BytesToIntError::InvalidLiteral { base })?; + let uint = BigUint::parse_bytes(buf, base).ok_or(BytesToIntError::InvalidLiteral { + base: requested_base, + })?; Ok(BigInt::from_biguint(sign.unwrap_or(Sign::Plus), uint)) } @@ -180,9 +202,11 @@ mod tests { #[test] fn bytes_to_int_invalid_literal() { for ((buf, base), expected) in [ - (("09_99", 0), BytesToIntError::InvalidLiteral { base: 10 }), - (("0_", 0), BytesToIntError::InvalidLiteral { base: 10 }), + (("09_99", 0), BytesToIntError::InvalidLiteral { base: 0 }), + (("0_", 0), BytesToIntError::InvalidLiteral { base: 0 }), (("0_", 2), BytesToIntError::InvalidLiteral { base: 2 }), + (("-", 0), BytesToIntError::InvalidLiteral { base: 0 }), + (("+", 0), BytesToIntError::InvalidLiteral { base: 0 }), ] { assert_eq!( bytes_to_int(buf.as_bytes(), base, DIGIT_LIMIT), From 09c0d333a8291da6056c295513e214683253fb83 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 26 Aug 2026 04:02:39 +0900 Subject: [PATCH 05/11] Split, strip and expand tabs over the Py_UNICODE_ISSPACE set `str.split`, `rsplit`, `strip`, `lstrip` and `rstrip` with no separator used Rust's whitespace predicate, which differs from the 29 code points `Py_UNICODE_ISSPACE` names: U+001C-U+001F and U+0085 were skipped and U+200B was treated as a separator. `expandtabs` went through `str`, so it dropped the lone surrogates of a WTF-8 string; it now works on `Wtf8` throughout. Indexing a str out of range now says "string index out of range". Assisted-by: Claude Code:claude-opus-5 --- crates/common/src/str.rs | 58 +++++++++++++++---------------- crates/vm/src/builtins/str.rs | 64 +++++++++++++++++++---------------- 2 files changed, 64 insertions(+), 58 deletions(-) diff --git a/crates/common/src/str.rs b/crates/common/src/str.rs index 3e972dc7bbe..ec9be305899 100644 --- a/crates/common/src/str.rs +++ b/crates/common/src/str.rs @@ -742,35 +742,31 @@ pub mod levenshtein { /// Replace all tabs in a string with spaces, using the given tab size. #[must_use] -pub fn expandtabs(input: &str, tab_size: usize) -> String { +pub fn expandtabs(input: &Wtf8, tab_size: usize) -> Wtf8Buf { // A tab size of zero, which is also where a negative one lands, leaves no // column for a tab to advance to: the tabs come out and nothing else moves. // Going through the arithmetic anyway subtracts the current column from a // tab stop of zero and underflows on the first tab, so the width asked for // next is `usize::MAX`. The bytes version of this already returns here. if tab_size == 0 { - return input.chars().filter(|ch| *ch != '\t').collect(); + return input.code_points().filter(|ch| *ch != '\t').collect(); } let tab_stop = tab_size; - let mut expanded_str = String::with_capacity(input.len()); + let mut expanded_str = Wtf8Buf::with_capacity(input.len()); let mut tab_size = tab_stop; let mut col_count = 0usize; - for ch in input.chars() { - match ch { - '\t' => { - let num_spaces = tab_size - col_count; - col_count += num_spaces; - let expand = " ".repeat(num_spaces); - expanded_str.push_str(&expand); - } - '\r' | '\n' => { - expanded_str.push(ch); + for ch in input.code_points() { + if ch == '\t' { + let num_spaces = tab_size - col_count; + col_count += num_spaces; + expanded_str.push_str(&" ".repeat(num_spaces)); + } else { + expanded_str.push(ch); + if ch == '\r' || ch == '\n' { col_count = 0; tab_size = 0; - } - _ => { - expanded_str.push(ch); + } else { col_count += 1; } } @@ -915,27 +911,31 @@ mod tests { assert_eq!(get_chars(s, 3..7), "๐Ÿ˜„๐Ÿ˜๐Ÿ˜†๐Ÿ˜…"); } + fn expandtabs(input: &str, tab_size: usize) -> Wtf8Buf { + super::expandtabs(Wtf8::new(input), tab_size) + } + #[test] fn expandtabs_with_zero_tab_size_drops_tabs() { // A tab that follows a character used to subtract that column from a // tab stop of zero, so the width of the run of spaces came out as // `usize::MAX` and the allocation aborted the process. - assert_eq!(expandtabs("a\tb", 0), "ab"); - assert_eq!(expandtabs("ab\tcd\tef", 0), "abcdef"); - assert_eq!(expandtabs("a\nb\tc", 0), "a\nbc"); - assert_eq!(expandtabs("รก\tb", 0), "รกb"); - assert_eq!(expandtabs("\ta", 0), "a"); - assert_eq!(expandtabs("\t", 0), ""); - assert_eq!(expandtabs("", 0), ""); - assert_eq!(expandtabs("no tabs", 0), "no tabs"); + assert_eq!(expandtabs("a\tb", 0), Wtf8Buf::from("ab")); + assert_eq!(expandtabs("ab\tcd\tef", 0), Wtf8Buf::from("abcdef")); + assert_eq!(expandtabs("a\nb\tc", 0), Wtf8Buf::from("a\nbc")); + assert_eq!(expandtabs("รก\tb", 0), Wtf8Buf::from("รกb")); + assert_eq!(expandtabs("\ta", 0), Wtf8Buf::from("a")); + assert_eq!(expandtabs("\t", 0), Wtf8Buf::from("")); + assert_eq!(expandtabs("", 0), Wtf8Buf::from("")); + assert_eq!(expandtabs("no tabs", 0), Wtf8Buf::from("no tabs")); } #[test] fn expandtabs_with_a_real_tab_size_is_unchanged() { - assert_eq!(expandtabs("a\tb", 8), "a b"); - assert_eq!(expandtabs("a\tb", 1), "a b"); - assert_eq!(expandtabs("abcd\te", 4), "abcd e"); - assert_eq!(expandtabs("a\nb\tc", 4), "a\nb c"); - assert_eq!(expandtabs("\ta", 4), " a"); + assert_eq!(expandtabs("a\tb", 8), Wtf8Buf::from("a b")); + assert_eq!(expandtabs("a\tb", 1), Wtf8Buf::from("a b")); + assert_eq!(expandtabs("abcd\te", 4), Wtf8Buf::from("abcd e")); + assert_eq!(expandtabs("a\nb\tc", 4), Wtf8Buf::from("a\nb c")); + assert_eq!(expandtabs("\ta", 4), Wtf8Buf::from(" a")); } } diff --git a/crates/vm/src/builtins/str.rs b/crates/vm/src/builtins/str.rs index dc5a1daf4b9..7a4d8e5fb41 100644 --- a/crates/vm/src/builtins/str.rs +++ b/crates/vm/src/builtins/str.rs @@ -830,8 +830,8 @@ impl PyStr { .collect() }, |v, n, vm| { - v.as_bytes().py_split_whitespace(n, |s| { - unsafe { AsciiStr::from_ascii_unchecked(s) }.to_pyobject(vm) + v.as_str().py_split_whitespace(n, |s| { + unsafe { AsciiStr::from_ascii_unchecked(s.as_bytes()) }.to_pyobject(vm) }) }, ), @@ -883,21 +883,24 @@ impl PyStr { .trim_matches(|c| memchr::memchr(c as _, chars.as_bytes()).is_some()); unsafe { AsciiStr::from_ascii_unchecked(s.as_bytes()) } }, - |s| s.trim(), + |s| { + let s = s.as_str().trim_matches(unicode::classify::is_space); + unsafe { AsciiStr::from_ascii_unchecked(s.as_bytes()) } + }, ) .into(), PyKindStr::Utf8(s) => s .py_strip( chars, |s, chars| s.trim_matches(|c| chars.contains(c)), - |s| s.trim(), + |s| s.trim_matches(unicode::classify::is_space), ) .into(), PyKindStr::Wtf8(w) => w .py_strip( chars, |s, chars| s.trim_matches(|c| chars.code_points().contains(&c)), - |s| s.trim(), + |s| s.trim_matches(|c: CodePoint| c.is_char_and(unicode::classify::is_space)), ) .into(), } @@ -913,7 +916,7 @@ impl PyStr { let stripped = s.py_strip( chars, |s, chars| s.trim_start_matches(|c| chars.contains_code_point(c)), - |s| s.trim_start(), + |s| s.trim_start_matches(|c: CodePoint| c.is_char_and(unicode::classify::is_space)), ); if s == stripped { zelf @@ -932,7 +935,7 @@ impl PyStr { let stripped = s.py_strip( chars, |s, chars| s.trim_end_matches(|c| chars.contains_code_point(c)), - |s| s.trim_end(), + |s| s.trim_end_matches(|c: CodePoint| c.is_char_and(unicode::classify::is_space)), ); if s == stripped { zelf @@ -1383,12 +1386,8 @@ impl PyStr { } #[pymethod] - fn expandtabs(&self, args: anystr::ExpandTabsArgs, vm: &VirtualMachine) -> PyResult { - // TODO: support WTF-8 - Ok(rustpython_common::str::expandtabs( - self.try_as_utf8(vm)?.as_str(), - args.tabsize(), - )) + fn expandtabs(&self, args: anystr::ExpandTabsArgs) -> Wtf8Buf { + rustpython_common::str::expandtabs(self.as_wtf8(), args.tabsize()) } #[pymethod] @@ -1890,6 +1889,13 @@ impl SliceableSequenceOp for PyStr { self.data.nth_char(index) } + fn getitem_by_index(&self, vm: &VirtualMachine, index: isize) -> PyResult { + let pos = self + .wrap_index(index) + .ok_or_else(|| vm.new_index_error("string index out of range"))?; + Ok(self.do_get(pos)) + } + fn do_slice(&self, range: Range) -> Self::Sliced { if let PyKindStr::Ascii(s) = self.as_str_kind() { return s[range].into(); @@ -2276,16 +2282,16 @@ impl AnyStr for str { let mut splits = Vec::new(); let mut last_offset = 0; let mut count = maxsplit; - for (offset, _) in self.match_indices(|c: char| c.is_ascii_whitespace() || c == '\x0b') { + for (offset, separator) in self.match_indices(unicode::classify::is_space) { if last_offset == offset { - last_offset += 1; + last_offset += separator.len(); continue; } if count == 0 { break; } splits.push(convert(&self[last_offset..offset])); - last_offset = offset + 1; + last_offset = offset + separator.len(); count -= 1; } if last_offset != self.len() { @@ -2302,15 +2308,15 @@ impl AnyStr for str { let mut splits = Vec::new(); let mut last_offset = self.len(); let mut count = maxsplit; - for (offset, _) in self.rmatch_indices(|c: char| c.is_ascii_whitespace() || c == '\x0b') { - if last_offset == offset + 1 { - last_offset -= 1; + for (offset, separator) in self.rmatch_indices(unicode::classify::is_space) { + if last_offset == offset + separator.len() { + last_offset = offset; continue; } if count == 0 { break; } - splits.push(convert(&self[offset + 1..last_offset])); + splits.push(convert(&self[offset + separator.len()..last_offset])); last_offset = offset; count -= 1; } @@ -2395,19 +2401,19 @@ impl AnyStr for Wtf8 { let mut splits = Vec::new(); let mut last_offset = 0; let mut count = maxsplit; - for (offset, _) in self + for (offset, separator) in self .code_point_indices() - .filter(|(_, c)| c.is_char_and(|c| c.is_ascii_whitespace() || c == '\x0b')) + .filter(|(_, c)| c.is_char_and(unicode::classify::is_space)) { if last_offset == offset { - last_offset += 1; + last_offset += separator.len_wtf8(); continue; } if count == 0 { break; } splits.push(convert(&self[last_offset..offset])); - last_offset = offset + 1; + last_offset = offset + separator.len_wtf8(); count -= 1; } if last_offset != self.len() { @@ -2424,19 +2430,19 @@ impl AnyStr for Wtf8 { let mut splits = Vec::new(); let mut last_offset = self.len(); let mut count = maxsplit; - for (offset, _) in self + for (offset, separator) in self .code_point_indices() .rev() - .filter(|(_, c)| c.is_char_and(|c| c.is_ascii_whitespace() || c == '\x0b')) + .filter(|(_, c)| c.is_char_and(unicode::classify::is_space)) { - if last_offset == offset + 1 { - last_offset -= 1; + if last_offset == offset + separator.len_wtf8() { + last_offset = offset; continue; } if count == 0 { break; } - splits.push(convert(&self[offset + 1..last_offset])); + splits.push(convert(&self[offset + separator.len_wtf8()..last_offset])); last_offset = offset; count -= 1; } From c2e15b053fd953d762aad05690a306277445659b Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 26 Aug 2026 04:02:41 +0900 Subject: [PATCH 06/11] Read Unicode numeric values as exact rationals MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `DerivedNumericValues.txt` field 1 carries a rounded decimal, so `unicodedata.numeric('โ…')` and its neighbours came back short of the value CPython reports. The generator now reads the rational in field 3, and takes the 3.2 values from field 8 of `UnicodeData-3.2.0.txt`. Assisted-by: Claude Code:claude-opus-5 --- crates/unicode/build.rs | 49 ++++++++++++++++++++++++++++++++------ crates/unicode/src/data.rs | 4 ++-- 2 files changed, 44 insertions(+), 9 deletions(-) diff --git a/crates/unicode/build.rs b/crates/unicode/build.rs index 8960c3872f6..92a771bafc4 100644 --- a/crates/unicode/build.rs +++ b/crates/unicode/build.rs @@ -274,6 +274,21 @@ fn generate_membership_3_2() { write!(writer, "{membership:?};").unwrap(); } +/// Read a numeric value written either on its own or as `numerator/denominator`. +fn parse_numeric_value(text: &str) -> f64 { + let text = text.trim(); + let (numerator, denominator) = text.split_once('/').unwrap_or((text, "1")); + let numerator: f64 = numerator + .trim() + .parse() + .expect("Unicode data contains valid properties"); + let denominator: f64 = denominator + .trim() + .parse() + .expect("Unicode data contains valid properties"); + numerator / denominator +} + fn generate_numeric_value() { let path = PathBuf::from(env::var("OUT_DIR").unwrap()) .join("generated") @@ -284,6 +299,26 @@ fn generate_numeric_value() { let ucd32 = Path::new(env!("CARGO_MANIFEST_DIR")) .join("unicode") .join("ucd32"); + // `DerivedNumericValues` writes the value rounded to a few digits, so the + // fraction `UnicodeData` field 8 spells out is what the value comes from. + // A character `UnicodeData` leaves out takes its value from Unihan, where + // the rounded field is a whole number and loses nothing. + let mut ucd32_fractions = BTreeMap::new(); + for line in BufReader::new(File::open(ucd32.join("UnicodeData-3.2.0.txt")).unwrap()) + .lines() + .map(Result::unwrap) + { + let mut fields = line.split(';'); + let Some(code) = fields.next() else { continue }; + let Some(numeric) = fields.nth(7) else { + continue; + }; + if !numeric.is_empty() { + let code = u32::from_str_radix(code.trim(), 16).unwrap(); + ucd32_fractions.insert(code, parse_numeric_value(numeric)); + } + } + let mut ucd32_diffs = BTreeMap::new(); let numeric_32 = BufReader::new(File::open(ucd32.join("DerivedNumericValues-3.2.0.txt")).unwrap()); @@ -292,9 +327,10 @@ fn generate_numeric_value() { NonZeroUsize::new(1).unwrap(), &mut io::empty(), |start, end, value, _| { - let value: f64 = value - .parse() - .expect("Unicode data contains valid properties"); + let value = ucd32_fractions + .get(&start) + .copied() + .unwrap_or_else(|| parse_numeric_value(value)); ucd32_diffs.insert((start, end), value); Option::<()>::None }, @@ -310,12 +346,11 @@ fn generate_numeric_value() { "DerivedNumericValues.txt", "NUMERIC_VALUES", "(u32, u32, f64)", - NonZeroUsize::new(1).unwrap(), + // Field 3 holds the fraction; field 1 rounds it to a few digits. + NonZeroUsize::new(3).unwrap(), &mut writer, |start, end, value, _| { - let value: f64 = value - .parse() - .expect("Unicode data contains valid properties"); + let value = parse_numeric_value(value); if ucd32_diffs .get(&(start, end)) diff --git a/crates/unicode/src/data.rs b/crates/unicode/src/data.rs index 3b17143290f..15e76bb171c 100644 --- a/crates/unicode/src/data.rs +++ b/crates/unicode/src/data.rs @@ -374,8 +374,8 @@ mod tests { assert_eq!(character_name('โ˜ƒ').as_deref(), Some("SNOWMAN")); assert_eq!(ucd.decimal(cp('เฅซ')), Some(5)); assert_eq!(ucd.digit(cp('ยฒ')), Some(2)); - let third = ucd.numeric(cp('โ…“')).unwrap(); - assert!((third - 1.0 / 3.0).abs() < 1e-6, "got {third}"); + assert_eq!(ucd.numeric(cp('โ…“')), Some(1.0 / 3.0)); + assert_eq!(ucd.numeric(cp('โ…')), Some(1.0 / 7.0)); } #[test] From 1ed87250cada636bb2cb282e4ce4d33e5e1a450b Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 26 Aug 2026 04:02:51 +0900 Subject: [PATCH 07/11] Return a mappingproxy from Pattern.groupindex `pattern_groupindex` wraps the group index in a dict proxy, and hands back a plain dict only when the pattern has no named group. Assisted-by: Claude Code:claude-opus-5 --- Lib/test/test_re.py | 1 - crates/vm/src/stdlib/_sre.rs | 14 ++++++++++---- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_re.py b/Lib/test/test_re.py index 1cfddb8e19c..502b3659e86 100644 --- a/Lib/test/test_re.py +++ b/Lib/test/test_re.py @@ -772,7 +772,6 @@ def test_repeat_minmax(self): self.checkPatternError(r'x{2,1}', 'min repeat greater than max repeat', 2) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_getattr(self): self.assertEqual(re.compile("(?i)(a)(b)").pattern, "(?i)(a)(b)") self.assertEqual(re.compile("(?i)(a)(b)").flags, re.I | re.U) diff --git a/crates/vm/src/stdlib/_sre.rs b/crates/vm/src/stdlib/_sre.rs index 4575901d3e8..f4f4485ead8 100644 --- a/crates/vm/src/stdlib/_sre.rs +++ b/crates/vm/src/stdlib/_sre.rs @@ -6,8 +6,8 @@ mod _sre { Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, VirtualMachine, atomic_func, builtins::{ - PyCallableIterator, PyDictRef, PyGenericAlias, PyInt, PyList, PyListRef, PyStr, - PyStrRef, PyTuple, PyTupleRef, PyTypeRef, + PyCallableIterator, PyDictRef, PyGenericAlias, PyInt, PyList, PyListRef, + PyMappingProxy, PyStr, PyStrRef, PyTuple, PyTupleRef, PyTypeRef, }, common::wtf8::{Wtf8, Wtf8Buf, wtf8_concat}, common::{ascii, hash::PyHash}, @@ -569,8 +569,14 @@ mod _sre { } #[pygetset] - fn groupindex(&self) -> PyDictRef { - self.groupindex.clone() + fn groupindex(&self, vm: &VirtualMachine) -> PyObjectRef { + // A pattern with no named group hands back a plain dict. + if self.groupindex.is_empty() { + return vm.ctx.new_dict().into(); + } + PyMappingProxy::from(self.groupindex.clone()) + .into_ref(&vm.ctx) + .into() } #[pygetset] From cb9fe1825712d0e938c73972399a285d7de2b481 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 26 Aug 2026 04:02:53 +0900 Subject: [PATCH 08/11] Fix the utf-16/32/7 codecs' error handling The utf-32 decoder dropped a code point above U+10FFFF instead of calling the error handler, and both the utf-16 and utf-32 handlers were told the encoding was "utf-16"/"utf-32" whichever byte order the data was in. The encoders now take a bytes replacement from the handler as it stands, rejecting one that does not fill whole code units, rather than iterating over it as text; a str replacement from the utf-16 encoder now splits an astral code point into a surrogate pair. The utf-7 decoder names its encoding "utf7", as its codec does. Assisted-by: Claude Code:claude-opus-5 --- Lib/_pycodecs.py | 132 ++++++++++++++++++-------------- Lib/test/test_codeccallbacks.py | 4 - 2 files changed, 74 insertions(+), 62 deletions(-) diff --git a/Lib/_pycodecs.py b/Lib/_pycodecs.py index 2c043f0ec76..b6065fa0130 100644 --- a/Lib/_pycodecs.py +++ b/Lib/_pycodecs.py @@ -479,6 +479,24 @@ def STORECHAR32(ch, byteorder): return [b3, b2, b1, b0] +def _encoded_replacement(replacement, store, unit_bytes, encoding, s, pos): + """The bytes an error handler's replacement adds to the output. + + A str is encoded the way the rest of the input is. A bytes object is taken + as it stands, and only when it fills whole code units. + """ + if isinstance(replacement, bytes): + if len(replacement) % unit_bytes: + raise UnicodeEncodeError( + encoding, s, pos, pos + 1, "surrogates not allowed" + ) + return list(replacement) + encoded = [] + for c in replacement: + encoded += store(ord(c)) + return encoded + + def PyUnicode_EncodeUTF32(s, size, errors, byteorder="little"): """Encode a Unicode string to UTF-32.""" p = [] @@ -494,21 +512,23 @@ def PyUnicode_EncodeUTF32(s, size, errors, byteorder="little"): elif byteorder == "big": bom = "big" + # Only the mode that writes a BOM leaves the order out of its name. + encoding = "utf-32" if byteorder == "native" else "utf-32-" + bom[0] + "e" + + def store(cp): + return STORECHAR32(cp, bom) + pos = 0 while pos < len(s): ch = ord(s[pos]) - if 0xD800 <= ch <= 0xDFFF: - if errors == "surrogatepass": - p += STORECHAR32(ch, bom) - pos += 1 - else: - res, pos = unicode_call_errorhandler( - errors, "utf-32", "surrogates not allowed", s, pos, pos + 1, False - ) - for c in res: - p += STORECHAR32(ord(c), bom) + if 0xD800 <= ch <= 0xDFFF and errors != "surrogatepass": + start = pos + res, pos = unicode_call_errorhandler( + errors, encoding, "surrogates not allowed", s, pos, pos + 1, False + ) + p += _encoded_replacement(res, store, 4, encoding, s, start) else: - p += STORECHAR32(ch, bom) + p += store(ch) pos += 1 return p @@ -541,6 +561,7 @@ def PyUnicode_DecodeUTF32Stateful(data, size, errors, byteorder="little", final= result = [] pos = 0 aligned_size = (size // 4) * 4 + encoding = "utf-32-le" if byteorder == "little" else "utf-32-be" while pos + 3 < aligned_size: if byteorder == "little": @@ -560,18 +581,16 @@ def PyUnicode_DecodeUTF32Stateful(data, size, errors, byteorder="little", final= # Validate code point if ch > 0x10FFFF: - if errors == "strict": - raise UnicodeDecodeError( - "utf-32", - bytes(data), - pos, - pos + 4, - "codepoint not in range(0x110000)", - ) - elif errors == "replace": - result.append("\ufffd") - # 'ignore' - skip this character - pos += 4 + res, pos = unicode_call_errorhandler( + errors, + encoding, + "code point not in range(0x110000)", + data, + pos, + pos + 4, + True, + ) + result.append(res) elif 0xD800 <= ch <= 0xDFFF: if errors == "surrogatepass": result.append(chr(ch)) @@ -579,7 +598,7 @@ def PyUnicode_DecodeUTF32Stateful(data, size, errors, byteorder="little", final= else: msg = "code point in surrogate code point range(0xd800, 0xe000)" res, pos = unicode_call_errorhandler( - errors, "utf-32", msg, data, pos, pos + 4, True + errors, encoding, msg, data, pos, pos + 4, True ) result.append(res) else: @@ -590,7 +609,7 @@ def PyUnicode_DecodeUTF32Stateful(data, size, errors, byteorder="little", final= if pos < size: if final: res, pos = unicode_call_errorhandler( - errors, "utf-32", "truncated data", data, pos, size, True + errors, encoding, "truncated data", data, pos, size, True ) if res: result.append(res) @@ -915,7 +934,7 @@ def PyUnicode_DecodeUTF7(s, size, errors, final=False): i += 1 errmsg = "partial character in shift sequence" out, i = unicode_call_errorhandler( - errors, "utf-7", errmsg, s, startinpos, i + errors, "utf7", errmsg, s, startinpos, i ) p.append(out) continue @@ -924,7 +943,7 @@ def PyUnicode_DecodeUTF7(s, size, errors, final=False): i += 1 errmsg = "non-zero padding bits in shift sequence" out, i = unicode_call_errorhandler( - errors, "utf-7", errmsg, s, startinpos, i + errors, "utf7", errmsg, s, startinpos, i ) p.append(out) continue @@ -943,7 +962,7 @@ def PyUnicode_DecodeUTF7(s, size, errors, final=False): i += 1 errmsg = "ill-formed sequence" out, i = unicode_call_errorhandler( - errors, "utf-7", errmsg, s, startinpos, i + errors, "utf7", errmsg, s, startinpos, i ) p.append(out) else: @@ -960,7 +979,7 @@ def PyUnicode_DecodeUTF7(s, size, errors, final=False): i += 1 errmsg = "unexpected special character" out, i = unicode_call_errorhandler( - errors, "utf-7", errmsg, s, startinpos, i + errors, "utf7", errmsg, s, startinpos, i ) p.append(out) @@ -971,7 +990,7 @@ def PyUnicode_DecodeUTF7(s, size, errors, final=False): if surrogate or base64bits >= 6 or (base64bits > 0 and base64buffer != 0): errmsg = "unterminated shift sequence" out, i = unicode_call_errorhandler( - errors, "utf-7", errmsg, s, startinpos, size + errors, "utf7", errmsg, s, startinpos, size ) p.append(out) @@ -1219,13 +1238,16 @@ def PyUnicode_DecodeUTF16Stateful(s, size, errors, byteorder="native", final=Tru ihi = 0 ilo = 1 + # The order the data turned out to be in is the one an error names. + encoding = "utf-16-be" if ihi == 0 else "utf-16-le" + while q < len(s): # /* remaining bytes at the end? (size should be even) */ if len(s) - q < 2: if not final: break res, q = unicode_call_errorhandler( - errors, "utf-16", "truncated data", s, q, len(s), True + errors, encoding, "truncated data", s, q, len(s), True ) p.append(res) break @@ -1253,7 +1275,7 @@ def PyUnicode_DecodeUTF16Stateful(s, size, errors, byteorder="native", final=Tru q += 2 continue res, q = unicode_call_errorhandler( - errors, "utf-16", "illegal UTF-16 surrogate", s, q, q + 2, True + errors, encoding, "illegal UTF-16 surrogate", s, q, q + 2, True ) p.append(res) else: @@ -1265,7 +1287,7 @@ def PyUnicode_DecodeUTF16Stateful(s, size, errors, byteorder="native", final=Tru q += 2 continue res, q = unicode_call_errorhandler( - errors, "utf-16", "unexpected end of data", s, q, len(s), True + errors, encoding, "unexpected end of data", s, q, len(s), True ) p.append(res) else: @@ -1275,7 +1297,7 @@ def PyUnicode_DecodeUTF16Stateful(s, size, errors, byteorder="native", final=Tru q += 2 continue res, q = unicode_call_errorhandler( - errors, "utf-16", "illegal encoding", s, q, q + 2, True + errors, encoding, "illegal encoding", s, q, q + 2, True ) p.append(res) @@ -1309,34 +1331,28 @@ def PyUnicode_EncodeUTF16(s, size, errors, byteorder="little"): elif byteorder == "big": bom = "big" + # Only the mode that writes a BOM leaves the order out of its name. + encoding = "utf-16" if byteorder == "native" else "utf-16-" + bom[0] + "e" + + def store(cp): + if cp >= 0x10000: + cp -= 0x10000 + return STORECHAR(0xD800 | (cp >> 10), bom) + STORECHAR( + 0xDC00 | (cp & 0x3FF), bom + ) + return STORECHAR(cp, bom) + pos = 0 while pos < len(s): ch = ord(s[pos]) - if 0xD800 <= ch <= 0xDFFF: - if errors == "surrogatepass": - p += STORECHAR(ch, bom) - pos += 1 - else: - res, pos = unicode_call_errorhandler( - errors, "utf-16", "surrogates not allowed", s, pos, pos + 1, False - ) - for c in res: - cp = ord(c) - cp2 = 0 - if cp >= 0x10000: - cp2 = 0xDC00 | ((cp - 0x10000) & 0x3FF) - cp = 0xD800 | ((cp - 0x10000) >> 10) - p += STORECHAR(cp, bom) - if cp2: - p += STORECHAR(cp2, bom) + if 0xD800 <= ch <= 0xDFFF and errors != "surrogatepass": + start = pos + res, pos = unicode_call_errorhandler( + errors, encoding, "surrogates not allowed", s, pos, pos + 1, False + ) + p += _encoded_replacement(res, store, 2, encoding, s, start) else: - ch2 = 0 - if ch >= 0x10000: - ch2 = 0xDC00 | ((ch - 0x10000) & 0x3FF) - ch = 0xD800 | ((ch - 0x10000) >> 10) - p += STORECHAR(ch, bom) - if ch2: - p += STORECHAR(ch2, bom) + p += store(ch) pos += 1 return p diff --git a/Lib/test/test_codeccallbacks.py b/Lib/test/test_codeccallbacks.py index 28ddf0a63b0..f269da36a15 100644 --- a/Lib/test/test_codeccallbacks.py +++ b/Lib/test/test_codeccallbacks.py @@ -854,8 +854,6 @@ def unencrepl(exc): self.assertEqual(exc.end, 2) self.assertEqual(exc.object, input) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_encode_bytes_replacement(self): def handle(exc): if isinstance(exc, UnicodeEncodeError): @@ -878,8 +876,6 @@ def handle(exc): res = input.encode(enc, "test.replacing") self.assertEqual(res, "[".encode(enc) + repl + "]".encode(enc)) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_encode_odd_bytes_replacement(self): def handle(exc): if isinstance(exc, UnicodeEncodeError): From 4b548281c3a5f28e276dca60a2bb9bfed0db4f84 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 26 Aug 2026 04:03:48 +0900 Subject: [PATCH 09/11] Rework complex multiplication, division and power Division now divides by whichever part of the divisor is larger, the way `_Py_c_quot` does, and both it and multiplication recover the infinities that the plain formulas turn into nan, following C11 Annex G. An operand that is a real number is now multiplied and divided component-wise, which keeps the sign of a zero and the other part of an infinity; a real numerator has its own quotient so that its zero keeps a sign too. An integer exponent no greater than 100 in magnitude is now reached by repeated multiplication rather than through the polar form, so e.g. `(1+2j) ** 2` is exact. A zero exponent gives 1, and an exponent with no imaginary part no longer reaches for a logarithm that turns an infinite magnitude into a nan. A result that came out infinite raises OverflowError. `complex() arg is a malformed string` is replaced by `could not convert string to complex: ...` when the string carries a misplaced underscore, `complex division by zero` by `division by zero`, and the message for zero raised to a negative or complex power no longer names the base. Assisted-by: Claude Code:claude-opus-5 --- Lib/test/test_complex.py | 2 - crates/vm/src/builtins/complex.rs | 288 +++++++++++++++++++++++++++--- 2 files changed, 259 insertions(+), 31 deletions(-) diff --git a/Lib/test/test_complex.py b/Lib/test/test_complex.py index 3ed6bfd9cb7..6dc27ff07a3 100644 --- a/Lib/test/test_complex.py +++ b/Lib/test/test_complex.py @@ -92,7 +92,6 @@ def check_div(self, x, y): if y: self.assertClose(z / y, x) - @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: floats nan and inf are not identical def test_truediv(self): simple_real = [float(i) for i in range(-5, 6)] simple_complex = [complex(x, y) for x in simple_real for y in simple_real] @@ -289,7 +288,6 @@ def test_sub(self): self.assertRaises(TypeError, operator.sub, 1j, None) self.assertRaises(TypeError, operator.sub, None, 1j) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_mul(self): self.assertEqual(1j * int(20), complex(0, 20)) self.assertEqual(1j * int(-1), complex(0, -1)) diff --git a/crates/vm/src/builtins/complex.rs b/crates/vm/src/builtins/complex.rs index 7dcbedf7e17..e67c3c1f9a6 100644 --- a/crates/vm/src/builtins/complex.rs +++ b/crates/vm/src/builtins/complex.rs @@ -144,12 +144,178 @@ fn to_op_complex(value: &PyObject, vm: &VirtualMachine) -> PyResult f64 { + if part.is_infinite() { 1.0 } else { 0.0f64 }.copysign(part) +} + +fn tamed(value: Complex64) -> Complex64 { + Complex64::new(signed_unit(value.re), signed_unit(value.im)) +} + +fn nans_to_zero(value: Complex64) -> Complex64 { + let zeroed = |part: f64| { + if part.is_nan() { + 0.0f64.copysign(part) + } else { + part + } + }; + Complex64::new(zeroed(value.re), zeroed(value.im)) +} + +/// Multiply, recovering the infinities that the plain formula turns into nan, +/// the way C11 Annex G.5.1 does. +fn prod(a: Complex64, b: Complex64) -> Complex64 { + let r = a * b; + if !(r.re.is_nan() && r.im.is_nan()) { + return r; + } + + // "Box" an infinite operand into a signed one and turn the other's nans + // into signed zeros, so that the infinity survives a second pass. + let (mut a, mut b) = (a, b); + let a_infinite = a.re.is_infinite() || a.im.is_infinite(); + if a_infinite { + a = tamed(a); + b = nans_to_zero(b); + } + let b_infinite = b.re.is_infinite() || b.im.is_infinite(); + if b_infinite { + b = tamed(b); + a = nans_to_zero(a); + } + // An infinity that overflow lost, rather than one an operand carried. + let overflowed = !a_infinite + && !b_infinite + && ((a.re * b.re).is_infinite() + || (a.im * b.im).is_infinite() + || (a.re * b.im).is_infinite() + || (a.im * b.re).is_infinite()); + if overflowed { + a = nans_to_zero(a); + b = nans_to_zero(b); + } + + if !(a_infinite || b_infinite || overflowed) { + return r; + } + Complex64::new( + f64::INFINITY * (a.re * b.re - a.im * b.im), + f64::INFINITY * (a.re * b.im + a.im * b.re), + ) +} + +/// Divide a real by a complex. Written out rather than routed through `quot` +/// with a zero imaginary part, which would lose the sign of a zero. +fn rc_quot(a: f64, b: Complex64) -> Option { + let abs_re = b.re.abs(); + let abs_im = b.im.abs(); + + let r = if abs_re >= abs_im { + if abs_re == 0.0 { + return None; + } + let ratio = b.im / b.re; + let denom = b.re + b.im * ratio; + Complex64::new(a / denom, -a * ratio / denom) + } else if abs_im >= abs_re { + let ratio = b.re / b.im; + let denom = b.re * ratio + b.im; + Complex64::new(a * ratio / denom, -a / denom) + } else { + // One part of the divisor is a nan, so neither comparison held. + Complex64::new(f64::NAN, f64::NAN) + }; + + // A quotient that came out as nan recovers the way `recovered_quot` does, + // except that a real numerator has no imaginary term to carry a sign. + if r.re.is_nan() && r.im.is_nan() && (b.re.is_infinite() || b.im.is_infinite()) && a.is_finite() + { + let Complex64 { re: x, im: y } = tamed(b); + return Some(Complex64::new(0.0 * (a * x), 0.0 * -(a * y))); + } + + Some(r) +} + +/// Divide by whichever part of the divisor is larger, so that the ratio the +/// other part is scaled by cannot overflow. `None` stands for a divisor of +/// zero. +fn quot(a: Complex64, b: Complex64) -> Option { + let abs_re = b.re.abs(); + let abs_im = b.im.abs(); + + let r = if abs_re >= abs_im { + if abs_re == 0.0 { + return None; + } + let ratio = b.im / b.re; + let denom = b.re + b.im * ratio; + Complex64::new((a.re + a.im * ratio) / denom, (a.im - a.re * ratio) / denom) + } else if abs_im >= abs_re { + let ratio = b.re / b.im; + let denom = b.re * ratio + b.im; + Complex64::new((a.re * ratio + a.im) / denom, (a.im * ratio - a.re) / denom) + } else { + // One part of the divisor is a nan, so neither comparison held. + Complex64::new(f64::NAN, f64::NAN) + }; + + Some(recovered_quot(r, a, b)) +} + +/// Recover the infinities and zeros a quotient came out of as nan, the way +/// C11 Annex G.5.2 does. +fn recovered_quot(r: Complex64, a: Complex64, b: Complex64) -> Complex64 { + if !(r.re.is_nan() && r.im.is_nan()) { + return r; + } + + if (a.re.is_infinite() || a.im.is_infinite()) && b.re.is_finite() && b.im.is_finite() { + let Complex64 { re: x, im: y } = tamed(a); + Complex64::new( + f64::INFINITY * (x * b.re + y * b.im), + f64::INFINITY * (y * b.re - x * b.im), + ) + } else if (b.re.is_infinite() || b.im.is_infinite()) && a.re.is_finite() && a.im.is_finite() { + let Complex64 { re: x, im: y } = tamed(b); + Complex64::new(0.0 * (a.re * x + a.im * y), 0.0 * (a.im * x - a.re * y)) + } else { + r + } +} + fn inner_div(v1: Complex64, v2: Complex64, vm: &VirtualMachine) -> PyResult { - if v2.is_zero() { - return Err(vm.new_zero_division_error("complex division by zero")); + quot(v1, v2).ok_or_else(|| vm.new_zero_division_error("division by zero")) +} + +/// Raise to a non-negative integer power by repeated squaring. +fn pow_unsigned(x: Complex64, n: u32) -> Complex64 { + let mut r = ONE; + let mut p = x; + let mut mask = 1u32; + while mask > 0 && n >= mask { + if n & mask != 0 { + r = prod(r, p); + } + mask <<= 1; + p = prod(p, p); } + r +} - Ok(v1.fdiv(v2)) +/// Raise to an integer power. `None` stands for zero raised to a negative one. +fn powi(x: Complex64, n: i32) -> Option { + if n > 0 { + Some(pow_unsigned(x, n as u32)) + } else { + quot(ONE, pow_unsigned(x, n.unsigned_abs())) + } } pub(crate) fn complex_pow( @@ -157,36 +323,63 @@ pub(crate) fn complex_pow( v2: Complex64, vm: &VirtualMachine, ) -> PyResult { - if v1.is_zero() { - return if v2.re < 0.0 || v2.im != 0.0 { - let msg = format!("{v1} cannot be raised to a negative or complex power"); - Err(vm.new_zero_division_error(msg)) - } else if v2.is_zero() { - Ok(Complex64::new(1.0, 0.0)) - } else { - Ok(Complex64::new(0.0, 0.0)) - }; - } - - let ans = powc(v1, v2); - if ans.is_infinite() && !(v1.is_infinite() || v2.is_infinite()) { - Err(vm.new_overflow_error("complex exponentiation")) + // A small integer power is reached by multiplying, which stays exact + // where going through the polar form would not. + let exponent = v2.re as i32; + let result = if v2.im == 0.0 && v2.re == f64::from(exponent) && v2.re.abs() <= 100.0 { + powi(v1, exponent) + } else if v1.is_zero() && (v2.im != 0.0 || v2.re < 0.0) { + None } else { - Ok(ans) + Some(powc(v1, v2)) + }; + + let Some(result) = result else { + return Err(vm.new_zero_division_error("zero to a negative or complex power")); + }; + if result.re.is_infinite() || result.im.is_infinite() { + return Err(vm.new_overflow_error("complex exponentiation")); } + Ok(result) } -// num-complex changed their powc() implementation in 0.4.4, making it incompatible -// with what the regression tests expect. this is that old formula. +/// Raise to a power through the polar form. fn powc(a: Complex64, exp: Complex64) -> Complex64 { - let (r, theta) = a.to_polar(); - if r.is_zero() { - return Complex64::new(r, r); + if exp.is_zero() { + return ONE; } - Complex64::from_polar( - r.powf(exp.re) * (-exp.im * theta).exp(), - exp.re * theta + exp.im * r.ln(), - ) + if a.is_zero() { + return Complex64::new(0.0, 0.0); + } + + let magnitude = a.norm(); + let angle = a.arg(); + let mut len = magnitude.powf(exp.re); + let mut phase = angle * exp.re; + // An exponent with no imaginary part leaves these alone, and reaching for + // them anyway would turn an infinite magnitude into a nan. + if exp.im != 0.0 { + len *= (-angle * exp.im).exp(); + phase += exp.im * magnitude.ln(); + } + Complex64::new(len * phase.cos(), len * phase.sin()) +} + +/// Whether an underscore sits where a numeric literal does not allow one: +/// they only ever join two digits. +fn has_misplaced_underscore(bytes: &[u8]) -> bool { + let mut prev = b'\0'; + for &byte in bytes { + if byte == b'_' { + if !prev.is_ascii_digit() { + return true; + } + } else if prev == b'_' && !byte.is_ascii_digit() { + return true; + } + prev = byte; + } + prev == b'_' } impl Constructor for PyComplex { @@ -220,6 +413,12 @@ impl Constructor for PyComplex { "complex() can't take second arg if first is a string", )); } + if has_misplaced_underscore(s.as_wtf8().as_bytes()) { + let repr = val.repr(vm)?; + return Err(vm.new_value_error(format!( + "could not convert string to complex: {repr}" + ))); + } let (re, im) = rustpython_literal::complex::parse_str( &crate::protocol::numeric_literal_from_str(s), ) @@ -476,7 +675,20 @@ impl AsNumber for PyComplex { vm, ) }), - multiply: Some(|a, b, vm| PyComplex::number_op(a, b, |a, b, _vm| a * b, vm)), + multiply: Some(|a, b, vm| { + PyComplex::complex_real_binop( + a, + b, + prod, + |a_complex, b_real| { + Complex64::new(a_complex.re * b_real, a_complex.im * b_real) + }, + |a_real, b_complex| { + Complex64::new(a_real * b_complex.re, a_real * b_complex.im) + }, + vm, + ) + }), power: Some(|a, b, c, vm| { if vm.is_none(c) { PyComplex::number_op(a, b, complex_pow, vm) @@ -501,7 +713,25 @@ impl AsNumber for PyComplex { result.to_pyresult(vm) }), boolean: Some(|number, _vm| Ok(!PyComplex::number_downcast(number).value.is_zero())), - true_divide: Some(|a, b, vm| PyComplex::number_op(a, b, inner_div, vm)), + true_divide: Some(|a, b, vm| { + PyComplex::complex_real_binop( + a, + b, + |a, b| inner_div(a, b, vm), + |a_complex, b_real| { + if b_real == 0.0 { + Err(vm.new_zero_division_error("division by zero")) + } else { + Ok(Complex64::new(a_complex.re / b_real, a_complex.im / b_real)) + } + }, + |a_real, b_complex| { + rc_quot(a_real, b_complex) + .ok_or_else(|| vm.new_zero_division_error("division by zero")) + }, + vm, + ) + }), ..PyNumberMethods::NOT_IMPLEMENTED }; &AS_NUMBER From f1b0f68aea85057d3cbda74697015a450b63ee9e Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 26 Aug 2026 04:03:59 +0900 Subject: [PATCH 10/11] Intern the string constants that read as names A code object's string constants were interned by length, so a short string that is not a name was interned and a long name was not. They are now picked out the way `all_name_chars` does. The one-character latin-1 strings are interned as well, so that every route to one of them lands on the same object. Assisted-by: Claude Code:claude-opus-5 --- crates/vm/src/builtins/code.rs | 13 +++++++++++-- crates/vm/src/vm/context.rs | 12 ++++++++++-- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/crates/vm/src/builtins/code.rs b/crates/vm/src/builtins/code.rs index 63c73fe4e5b..ef2720dd9aa 100644 --- a/crates/vm/src/builtins/code.rs +++ b/crates/vm/src/builtins/code.rs @@ -272,6 +272,15 @@ impl<'a> AsBag for &'a Context { #[derive(Clone, Copy)] pub struct PyObjBag<'a>(pub &'a Context); +/// Whether a string constant reads as a name. Those are the ones interned, +/// the way `all_name_chars` picks them out. +fn is_name_chars(value: &crate::common::wtf8::Wtf8) -> bool { + value + .as_bytes() + .iter() + .all(|&b| b.is_ascii_alphanumeric() || b == b'_') +} + impl ConstantBag for PyObjBag<'_> { type Constant = Literal; @@ -281,7 +290,7 @@ impl ConstantBag for PyObjBag<'_> { BorrowedConstant::Integer { value } => ctx.new_bigint(value).into(), BorrowedConstant::Float { value } => ctx.new_float(value).into(), BorrowedConstant::Complex { value } => ctx.new_complex(value).into(), - BorrowedConstant::Str { value } if value.len() <= 20 => { + BorrowedConstant::Str { value } if is_name_chars(value) => { ctx.intern_str(value).to_object() } BorrowedConstant::Str { value } => ctx.new_str(value).into(), @@ -356,7 +365,7 @@ impl ConstantBag for PyVmBag<'_> { BorrowedConstant::Integer { value } => ctx.new_bigint(value).into(), BorrowedConstant::Float { value } => ctx.new_float(value).into(), BorrowedConstant::Complex { value } => ctx.new_complex(value).into(), - BorrowedConstant::Str { value } if value.len() <= 20 => { + BorrowedConstant::Str { value } if is_name_chars(value) => { ctx.intern_str(value).to_object() } BorrowedConstant::Str { value } => ctx.new_str(value).into(), diff --git a/crates/vm/src/vm/context.rs b/crates/vm/src/vm/context.rs index 5deaffb3f6a..a5a113f2505 100644 --- a/crates/vm/src/vm/context.rs +++ b/crates/vm/src/vm/context.rs @@ -330,8 +330,17 @@ impl Context { }) .collect(); + let string_pool = StringPool::default(); + + // The one-character latin-1 strings are interned, so that every route + // to one of them lands on the same object. let latin1_char_cache = (u8::MIN..=u8::MAX) - .map(|b| create_object(PyStr::from(char::from(b)), types.str_type)) + .map(|b| { + let s = unsafe { + string_pool.intern(char::from(b).to_string(), types.str_type.to_owned()) + }; + s.to_owned() + }) .collect::>>(); let ascii_char_cache = latin1_char_cache[..128].to_vec(); @@ -349,7 +358,6 @@ impl Context { None, ); - let string_pool = StringPool::default(); let names = unsafe { ConstName::new(&string_pool, types.str_type) }; let slot_new_wrapper = PyMethodDef::new_const( From a5fbe531d95aaf1610cc9d28688911406a09a8aa Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 26 Aug 2026 04:04:02 +0900 Subject: [PATCH 11/11] Honour the marshal version and separate its EOF errors Versions below 2 now write a float or complex as its digits under TYPE_FLOAT/TYPE_COMPLEX, versions from 4 on write a tuple of fewer than 256 items under TYPE_SMALL_TUPLE and a long ascii string under TYPE_ASCII, and a string is marked interned only when it is. `marshal.loads` now tells apart the three ways its data can run out: "EOF read where object expected" where an object was to start, "EOF read where not expected" where a single byte was, and "marshal data too short" where a run of bytes was cut short. Assisted-by: Claude Code:claude-opus-5 --- crates/compiler-core/src/marshal.rs | 31 +++++++++--- crates/vm/src/stdlib/marshal.rs | 74 ++++++++++++++++++++++++----- 2 files changed, 86 insertions(+), 19 deletions(-) diff --git a/crates/compiler-core/src/marshal.rs b/crates/compiler-core/src/marshal.rs index d73b51f45b4..acc45e75c6f 100644 --- a/crates/compiler-core/src/marshal.rs +++ b/crates/compiler-core/src/marshal.rs @@ -11,6 +11,10 @@ pub const FORMAT_VERSION: u32 = 5; pub enum MarshalError { /// Unexpected End Of File Eof, + /// Unexpected End Of File where an object was to start + EofObject, + /// A run of bytes that the data ends in the middle of + DataTooShort, /// Invalid Bytecode InvalidBytecode, /// Invalid utf8 in string @@ -33,6 +37,8 @@ impl core::fmt::Display for MarshalError { fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { match self { Self::Eof => f.write_str("unexpected end of data"), + Self::EofObject => f.write_str("unexpected end of data where an object was expected"), + Self::DataTooShort => f.write_str("data ends in the middle of a value"), Self::InvalidBytecode => f.write_str("invalid bytecode"), Self::InvalidUtf8 => f.write_str("invalid utf8"), Self::InvalidLocation => f.write_str("invalid source location"), @@ -144,7 +150,9 @@ pub trait Read { } fn read_u8(&mut self) -> Result { - Ok(u8::from_le_bytes(*self.read_array()?)) + // A single byte is not a run that got cut short: nothing came at all. + let byte = self.read_array().map_err(|_| MarshalError::Eof)?; + Ok(u8::from_le_bytes(*byte)) } fn read_u16(&mut self) -> Result { @@ -181,7 +189,9 @@ impl Read for &[u8] { } fn read_array(&mut self) -> Result<&[u8; N]> { - let (chunk, rest) = self.split_first_chunk::().ok_or(MarshalError::Eof)?; + let (chunk, rest) = self + .split_first_chunk::() + .ok_or(MarshalError::DataTooShort)?; *self = rest; Ok(chunk) } @@ -189,7 +199,8 @@ impl Read for &[u8] { impl<'a> ReadBorrowed<'a> for &'a [u8] { fn read_slice_borrow(&mut self, n: u32) -> Result<&'a [u8]> { - self.split_off(..n as usize).ok_or(MarshalError::Eof) + self.split_off(..n as usize) + .ok_or(MarshalError::DataTooShort) } } @@ -201,7 +212,7 @@ pub struct Cursor { impl> Read for Cursor { fn read_slice(&mut self, n: u32) -> Result<&[u8]> { let data = &self.data.as_ref()[self.position..]; - let slice = data.get(..n as usize).ok_or(MarshalError::Eof)?; + let slice = data.get(..n as usize).ok_or(MarshalError::DataTooShort)?; self.position += n as usize; Ok(slice) } @@ -847,6 +858,12 @@ pub fn deserialize_value(rdr: &mut R, bag: Bag) -> Res deserialize_value_depth(rdr, bag, MAX_MARSHAL_STACK_DEPTH, &mut refs) } +/// Read the byte an object starts with. Running out of data here names the +/// object that never came, rather than a field that was cut short. +fn read_object_type(rdr: &mut R) -> Result { + rdr.read_u8().map_err(|_| MarshalError::EofObject) +} + fn deserialize_value_depth( rdr: &mut R, bag: Bag, @@ -856,7 +873,7 @@ fn deserialize_value_depth( if depth == 0 { return Err(MarshalError::InvalidBytecode); } - let raw = rdr.read_u8()?; + let raw = read_object_type(rdr)?; deserialize_value_after_header(rdr, bag, depth, refs, raw) } @@ -1187,7 +1204,7 @@ fn deserialize_value_typed( { refs[index] = Some(dict.clone()); loop { - let raw = rdr.read_u8()?; + let raw = read_object_type(rdr)?; if raw & !FLAG_REF == b'0' { break; } @@ -1199,7 +1216,7 @@ fn deserialize_value_typed( } else { let mut pairs = Vec::new(); loop { - let raw = rdr.read_u8()?; + let raw = read_object_type(rdr)?; if raw & !FLAG_REF == b'0' { break; } diff --git a/crates/vm/src/stdlib/marshal.rs b/crates/vm/src/stdlib/marshal.rs index 08a8f589a77..8c3c05bda9e 100644 --- a/crates/vm/src/stdlib/marshal.rs +++ b/crates/vm/src/stdlib/marshal.rs @@ -196,6 +196,28 @@ mod decl { ) } + /// Write a float the way the pre-binary formats carry one: seventeen + /// significant digits, prefixed with their length. + fn write_float_str(buf: &mut Vec, value: f64) { + use marshal::Write; + let digits = rustpython_literal::float::format_general( + 17, + value.abs(), + rustpython_literal::format::Case::Lower, + false, + false, + ); + // A nan carries no sign of its own. + let sign = if value.is_sign_negative() && !value.is_nan() { + "-" + } else { + "" + }; + buf.write_u8((sign.len() + digits.len()) as u8); + buf.write_slice(sign.as_bytes()); + buf.write_slice(digits.as_bytes()); + } + fn write_object_depth( buf: &mut Vec, obj: &PyObjectRef, @@ -290,19 +312,37 @@ mod decl { } } } else if let Some(f) = obj.downcast_ref::() { - buf.write_u8(b'g'); - buf.write_u64(f.to_f64().to_bits()); + if version > 1 { + buf.write_u8(b'g'); + buf.write_u64(f.to_f64().to_bits()); + } else { + buf.write_u8(b'f'); + write_float_str(buf, f.to_f64()); + } } else if let Some(c) = obj.downcast_ref::() { - buf.write_u8(b'y'); let cv = c.to_complex64(); - buf.write_u64(cv.re.to_bits()); - buf.write_u64(cv.im.to_bits()); + if version > 1 { + buf.write_u8(b'y'); + buf.write_u64(cv.re.to_bits()); + buf.write_u64(cv.im.to_bits()); + } else { + buf.write_u8(b'x'); + write_float_str(buf, cv.re); + write_float_str(buf, cv.im); + } } else if let Some(s) = obj.downcast_ref::() { let bytes = s.as_wtf8().as_bytes(); - let interned = version >= 3; - if bytes.len() < 256 && bytes.is_ascii() { - buf.write_u8(if interned { b'Z' } else { b'z' }); - buf.write_u8(bytes.len() as u8); + // Only the formats that carry back-references tell an interned + // string apart, and only those from 4 on have the ascii-only forms. + let interned = version >= 3 && obj.is_interned(); + if version >= 4 && bytes.is_ascii() { + if bytes.len() <= 255 { + buf.write_u8(if interned { b'Z' } else { b'z' }); + buf.write_u8(bytes.len() as u8); + } else { + buf.write_u8(if interned { b'A' } else { b'a' }); + buf.write_u32(bytes.len() as u32); + } } else { buf.write_u8(if interned { b't' } else { b'u' }); buf.write_u32(bytes.len() as u32); @@ -319,8 +359,14 @@ mod decl { buf.write_u32(data.len() as u32); buf.write_slice(&data); } else if let Some(t) = obj.downcast_ref::() { - buf.write_u8(b'('); - buf.write_u32(t.len() as u32); + // From 4 on a short tuple carries its length in a single byte. + if version >= 4 && t.len() < 256 { + buf.write_u8(b')'); + buf.write_u8(t.len() as u8); + } else { + buf.write_u8(b'('); + buf.write_u32(t.len() as u32); + } for elem in t.as_slice() { write_object_depth(buf, elem, refs, version, allow_code, vm, depth - 1)?; } @@ -683,7 +729,11 @@ mod decl { match marshal::deserialize_value(rdr, PyMarshalBag::new(vm, &pending_error, allow_code)) { Ok(value) => Ok(value), Err(error) => Err(pending_error.into_inner().unwrap_or_else(|| match error { - marshal::MarshalError::Eof => vm.new_eof_error("marshal data too short"), + marshal::MarshalError::Eof => vm.new_eof_error("EOF read where not expected"), + marshal::MarshalError::EofObject => { + vm.new_eof_error("EOF read where object expected") + } + marshal::MarshalError::DataTooShort => vm.new_eof_error("marshal data too short"), error @ marshal::MarshalError::NullObject => vm.new_type_error(error.to_string()), error @ (marshal::MarshalError::BadSize(_) | marshal::MarshalError::UnknownType