fix: strip BOM in Response::text_with_charset (seanmonstar#1898)

The byte order mark (BOM) is now stripped from utf-8 encoded response bodies when calling `Response::text` and `Response::text_with_charset`. This should prevent surprising behaviour when trying to use the returned String. Closes seanmonstar#1897
kvinwang · Jul 6, 2023 · 3abcc7c · 3abcc7c
1 parent 89df5d3
commit 3abcc7c
Showing 1 changed file with 5 additions and 9 deletions.
diff --git a/src/async_impl/response.rs b/src/async_impl/response.rs
@@ -1,4 +1,3 @@
-use std::borrow::Cow;
 use std::fmt;
 use std::net::SocketAddr;
 use std::pin::Pin;
@@ -130,6 +129,8 @@ impl Response {
     /// Encoding is determined from the `charset` parameter of `Content-Type` header,
     /// and defaults to `utf-8` if not presented.
     ///
+    /// Note that the BOM is stripped from the returned String.
+    ///
     /// # Example
     ///
     /// ```
@@ -155,6 +156,8 @@ impl Response {
     /// `charset` parameter of `Content-Type` header is still prioritized. For more information
     /// about the possible encoding name, please go to [`encoding_rs`] docs.
     ///
+    /// Note that the BOM is stripped from the returned String.
+    ///
     /// [`encoding_rs`]: https://docs.rs/encoding_rs/0.8/encoding_rs/#relationship-with-windows-code-pages
     ///
     /// # Example
@@ -185,14 +188,7 @@ impl Response {
         let full = self.bytes().await?;
 
         let (text, _, _) = encoding.decode(&full);
-        if let Cow::Owned(s) = text {
-            return Ok(s);
-        }
-        unsafe {
-            // decoding returned Cow::Borrowed, meaning these bytes
-            // are already valid utf8
-            Ok(String::from_utf8_unchecked(full.to_vec()))
-        }
+        Ok(text.into_owned())
     }
 
     /// Try to deserialize the response body as JSON.