fix(serde): correct Value end offset with utf8_lossy and invalid UTF-8

src/serde/de.rs

@@ -372,14 +372,32 @@ impl<'de, R: Reader<'de>> Deserializer<R> {
             let json = self.parser.read.as_u8_slice();
 
             // get n to check trailing characters in later
-            let n = if cfg.utf8_lossy && self.parser.read.next_invalid_utf8() != usize::MAX {
+            if cfg.utf8_lossy && self.parser.read.next_invalid_utf8() != usize::MAX {
                 // repr the invalid utf8, not need to care about the invalid UTF8 char in non-string
                 // parts, it will cause errors when parsing.
-                val.parse_with_padding(String::from_utf8_lossy(json).as_bytes(), cfg)?
+                let lossy = String::from_utf8_lossy(json);
+                let n = val.parse_with_padding(lossy.as_bytes(), cfg)?;
+                // `n` is an offset into the lossy copy, where every invalid sequence became the
+                // 3-byte U+FFFD, so it is not an offset into `json`. What follows the value is
+                // the same in both (whitespace or trailing characters; invalid UTF-8 there is a
+                // trailing-character error either way), so count it from the end.
+                // `n` past the end means the parser ran into the padding (unterminated input):
+                // keep it past the end of `json` by the same amount, so that is still reported.
+                let consumed = if n >= lossy.len() {
+                    json.len() + (n - lossy.len())
+                } else {
+                    json.len() - (lossy.len() - n)
+                };
+                self.parser.read.eat(consumed);
+                // the invalid UTF-8 inside the value has been replaced: look for it only in the
+                // part that is still to be read (if any; past the end is reported as EOF)
+                if consumed <= json.len() {
+                    self.parser.read.check_invalid_utf8();
+                }
             } else {
-                val.parse_with_padding(json, cfg)?
-            };
-            self.parser.read.eat(n);
+                let n = val.parse_with_padding(json, cfg)?;
+                self.parser.read.eat(n);
+            }
         } else {
             let shared = unsafe {
                 if self.shared.is_none() {

src/serde/mod.rs

@@ -765,6 +765,42 @@ mod test {
         }
     }
 
+    #[test]
+    fn test_utf8_lossy_value_trailing() {
+        // Into `Value`, the lossy path parses a copy where each invalid sequence became the 3-byte
+        // U+FFFD; its end offset was used as an offset into the original input. So valid input
+        // failed in `end()`, and trailing characters after the value were skipped.
+        let parse = |json: &[u8]| -> Result<Value> {
+            let mut de = Deserializer::from_slice(json).utf8_lossy();
+            let value: Value = de.deserialize()?;
+            de.end()?;
+            Ok(value)
+        };
+        for json in [
+            &b"[\"a\xa0b\"]"[..],
+            b"[\"a\xa0b\"]\n",
+            b"{\"k\":\"\xc3\"}",
+            b"\"\xff\"",
+        ] {
+            parse(json).unwrap_or_else(|e| panic!("{json:?}: {e}"));
+        }
+        assert_eq!(
+            parse(b"[\"a\xa0b\"]").unwrap(),
+            crate::json!(["a\u{fffd}b"])
+        );
+        for json in [
+            &b"[\"a\xa0b\"]xx"[..],
+            b"[\"\xa0\xa0\xa0\"]  [1]",
+            b"\"\xa0\"1",
+        ] {
+            let err = parse(json).expect_err(&format!("{json:?} has trailing characters"));
+            assert!(err.to_string().contains("trailing"), "{json:?}: {err}");
+        }
+        // unterminated input is still an error
+        assert!(parse(b"[\"a\xa0").is_err());
+        assert!(parse(b"\"abc\xfa").is_err());
+    }
+
     #[test]
     fn test_utf8_lossy_surrogate_backtrack() {
         // high surrogate + non-\u content → FFFD + literal chars