Auto merge of #27808 - SimonSapin:utf16decoder, r=alexcrichton

* Rename `Utf16Items` to `Utf16Decoder`. "Items" is meaningless.
* Generalize it to any `u16` iterator, not just `[u16].iter()`
* Make it yield `Result` instead of a custom `Utf16Item` enum that was isomorphic to `Result`. This enable using the `FromIterator for Result` impl.
* Replace `Utf16Item::to_char_lossy` with a `Utf16Decoder::lossy` iterator adaptor.

This is a [breaking change], but only for users of the unstable `rustc_unicode` crate.

I’d like this functionality to be stabilized and re-exported in `std` eventually, as the "low-level equivalent" of `String::from_utf16` and `String::from_utf16_lossy` like #27784 is the low-level equivalent of #27714.

CC @aturon, @alexcrichton

Author: bors

src/liballoc/arc.rs

@@ -73,6 +73,7 @@ use boxed::Box;
 
 use core::sync::atomic;
 use core::sync::atomic::Ordering::{Relaxed, Release, Acquire, SeqCst};
+use core::borrow;
 use core::fmt;
 use core::cmp::Ordering;
 use core::mem::{align_of_val, size_of_val};
@@ -1109,3 +1110,7 @@ mod tests {
         assert!(y.upgrade().is_none());
     }
 }
+
+impl<T: ?Sized> borrow::Borrow<T> for Arc<T> {
+    fn borrow(&self) -> &T { &**self }
+}

src/liballoc/boxed.rs

@@ -57,6 +57,7 @@ use heap;
 use raw_vec::RawVec;
 
 use core::any::Any;
+use core::borrow;
 use core::cmp::Ordering;
 use core::fmt;
 use core::hash::{self, Hash};
@@ -562,3 +563,10 @@ impl<T: Clone> Clone for Box<[T]> {
     }
 }
 
+impl<T: ?Sized> borrow::Borrow<T> for Box<T> {
+    fn borrow(&self) -> &T { &**self }
+}
+
+impl<T: ?Sized> borrow::BorrowMut<T> for Box<T> {
+    fn borrow_mut(&mut self) -> &mut T { &mut **self }
+}

src/liballoc/rc.rs

@@ -158,6 +158,7 @@ use boxed::Box;
 #[cfg(test)]
 use std::boxed::Box;
 
+use core::borrow;
 use core::cell::Cell;
 use core::cmp::Ordering;
 use core::fmt;
@@ -1091,3 +1092,7 @@ mod tests {
         assert_eq!(foo, foo.clone());
     }
 }
+
+impl<T: ?Sized> borrow::Borrow<T> for Rc<T> {
+    fn borrow(&self) -> &T { &**self }
+}

src/libcollections/borrow.rs

@@ -21,119 +21,10 @@ use core::ops::Deref;
 use core::option::Option;
 
 use fmt;
-use alloc::{boxed, rc, arc};
 
 use self::Cow::*;
 
-/// A trait for borrowing data.
-///
-/// In general, there may be several ways to "borrow" a piece of data.  The
-/// typical ways of borrowing a type `T` are `&T` (a shared borrow) and `&mut T`
-/// (a mutable borrow). But types like `Vec<T>` provide additional kinds of
-/// borrows: the borrowed slices `&[T]` and `&mut [T]`.
-///
-/// When writing generic code, it is often desirable to abstract over all ways
-/// of borrowing data from a given type. That is the role of the `Borrow`
-/// trait: if `T: Borrow<U>`, then `&U` can be borrowed from `&T`.  A given
-/// type can be borrowed as multiple different types. In particular, `Vec<T>:
-/// Borrow<Vec<T>>` and `Vec<T>: Borrow<[T]>`.
-///
-/// If you are implementing `Borrow` and both `Self` and `Borrowed` implement
-/// `Hash`, `Eq`, and/or `Ord`, they must produce the same result.
-///
-/// `Borrow` is very similar to, but different than, `AsRef`. See
-/// [the book][book] for more.
-///
-/// [book]: ../../book/borrow-and-asref.html
-#[stable(feature = "rust1", since = "1.0.0")]
-pub trait Borrow<Borrowed: ?Sized> {
-    /// Immutably borrows from an owned value.
-    ///
-    /// # Examples
-    ///
-    /// ```
-    /// use std::borrow::Borrow;
-    ///
-    /// fn check<T: Borrow<str>>(s: T) {
-    ///     assert_eq!("Hello", s.borrow());
-    /// }
-    ///
-    /// let s = "Hello".to_string();
-    ///
-    /// check(s);
-    ///
-    /// let s = "Hello";
-    ///
-    /// check(s);
-    /// ```
-    #[stable(feature = "rust1", since = "1.0.0")]
-    fn borrow(&self) -> &Borrowed;
-}
-
-/// A trait for mutably borrowing data.
-///
-/// Similar to `Borrow`, but for mutable borrows.
-#[stable(feature = "rust1", since = "1.0.0")]
-pub trait BorrowMut<Borrowed: ?Sized> : Borrow<Borrowed> {
-    /// Mutably borrows from an owned value.
-    ///
-    /// # Examples
-    ///
-    /// ```
-    /// use std::borrow::BorrowMut;
-    ///
-    /// fn check<T: BorrowMut<[i32]>>(mut v: T) {
-    ///     assert_eq!(&mut [1, 2, 3], v.borrow_mut());
-    /// }
-    ///
-    /// let v = vec![1, 2, 3];
-    ///
-    /// check(v);
-    /// ```
-    #[stable(feature = "rust1", since = "1.0.0")]
-    fn borrow_mut(&mut self) -> &mut Borrowed;
-}
-
-#[stable(feature = "rust1", since = "1.0.0")]
-impl<T: ?Sized> Borrow<T> for T {
-    fn borrow(&self) -> &T { self }
-}
-
-#[stable(feature = "rust1", since = "1.0.0")]
-impl<T: ?Sized> BorrowMut<T> for T {
-    fn borrow_mut(&mut self) -> &mut T { self }
-}
-
-#[stable(feature = "rust1", since = "1.0.0")]
-impl<'a, T: ?Sized> Borrow<T> for &'a T {
-    fn borrow(&self) -> &T { &**self }
-}
-
-#[stable(feature = "rust1", since = "1.0.0")]
-impl<'a, T: ?Sized> Borrow<T> for &'a mut T {
-    fn borrow(&self) -> &T { &**self }
-}
-
-#[stable(feature = "rust1", since = "1.0.0")]
-impl<'a, T: ?Sized> BorrowMut<T> for &'a mut T {
-    fn borrow_mut(&mut self) -> &mut T { &mut **self }
-}
-
-impl<T: ?Sized> Borrow<T> for boxed::Box<T> {
-    fn borrow(&self) -> &T { &**self }
-}
-
-impl<T: ?Sized> BorrowMut<T> for boxed::Box<T> {
-    fn borrow_mut(&mut self) -> &mut T { &mut **self }
-}
-
-impl<T: ?Sized> Borrow<T> for rc::Rc<T> {
-    fn borrow(&self) -> &T { &**self }
-}
-
-impl<T: ?Sized> Borrow<T> for arc::Arc<T> {
-    fn borrow(&self) -> &T { &**self }
-}
+pub use core::borrow::{Borrow, BorrowMut};
 
 #[stable(feature = "rust1", since = "1.0.0")]
 impl<'a, B: ?Sized> Borrow<B> for Cow<'a, B> where B: ToOwned, <B as ToOwned>::Owned: 'a {

src/libcollections/lib.rs

@@ -56,6 +56,7 @@
 #![feature(unicode)]
 #![feature(unique)]
 #![feature(unsafe_no_drop_flag, filling_drop)]
+#![feature(decode_utf16)]
 #![feature(utf8_error)]
 #![cfg_attr(test, feature(rand, test))]
 

src/libcollections/string.rs

@@ -20,8 +20,8 @@ use core::ops::{self, Deref, Add, Index};
 use core::ptr;
 use core::slice;
 use core::str::pattern::Pattern;
+use rustc_unicode::char::{decode_utf16, REPLACEMENT_CHARACTER};
 use rustc_unicode::str as unicode_str;
-use rustc_unicode::str::Utf16Item;
 
 use borrow::{Cow, IntoCow};
 use range::RangeArgument;
@@ -267,14 +267,7 @@ impl String {
     /// ```
     #[stable(feature = "rust1", since = "1.0.0")]
     pub fn from_utf16(v: &[u16]) -> Result<String, FromUtf16Error> {
-        let mut s = String::with_capacity(v.len());
-        for c in unicode_str::utf16_items(v) {
-            match c {
-                Utf16Item::ScalarValue(c) => s.push(c),
-                Utf16Item::LoneSurrogate(_) => return Err(FromUtf16Error(())),
-            }
-        }
-        Ok(s)
+        decode_utf16(v.iter().cloned()).collect::<Result<_, _>>().map_err(|_| FromUtf16Error(()))
     }
 
     /// Decode a UTF-16 encoded vector `v` into a string, replacing
@@ -294,7 +287,7 @@ impl String {
     #[inline]
     #[stable(feature = "rust1", since = "1.0.0")]
     pub fn from_utf16_lossy(v: &[u16]) -> String {
-        unicode_str::utf16_items(v).map(|c| c.to_char_lossy()).collect()
+        decode_utf16(v.iter().cloned()).map(|r| r.unwrap_or(REPLACEMENT_CHARACTER)).collect()
     }
 
     /// Creates a new `String` from a length, capacity, and pointer.

src/libcore/borrow.rs

@@ -0,0 +1,109 @@
+// Copyright 2014 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+//! A module for working with borrowed data.
+
+#![stable(feature = "rust1", since = "1.0.0")]
+
+use marker::Sized;
+
+/// A trait for borrowing data.
+///
+/// In general, there may be several ways to "borrow" a piece of data.  The
+/// typical ways of borrowing a type `T` are `&T` (a shared borrow) and `&mut T`
+/// (a mutable borrow). But types like `Vec<T>` provide additional kinds of
+/// borrows: the borrowed slices `&[T]` and `&mut [T]`.
+///
+/// When writing generic code, it is often desirable to abstract over all ways
+/// of borrowing data from a given type. That is the role of the `Borrow`
+/// trait: if `T: Borrow<U>`, then `&U` can be borrowed from `&T`.  A given
+/// type can be borrowed as multiple different types. In particular, `Vec<T>:
+/// Borrow<Vec<T>>` and `Vec<T>: Borrow<[T]>`.
+///
+/// If you are implementing `Borrow` and both `Self` and `Borrowed` implement
+/// `Hash`, `Eq`, and/or `Ord`, they must produce the same result.
+///
+/// `Borrow` is very similar to, but different than, `AsRef`. See
+/// [the book][book] for more.
+///
+/// [book]: ../../book/borrow-and-asref.html
+#[stable(feature = "rust1", since = "1.0.0")]
+pub trait Borrow<Borrowed: ?Sized> {
+    /// Immutably borrows from an owned value.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// use std::borrow::Borrow;
+    ///
+    /// fn check<T: Borrow<str>>(s: T) {
+    ///     assert_eq!("Hello", s.borrow());
+    /// }
+    ///
+    /// let s = "Hello".to_string();
+    ///
+    /// check(s);
+    ///
+    /// let s = "Hello";
+    ///
+    /// check(s);
+    /// ```
+    #[stable(feature = "rust1", since = "1.0.0")]
+    fn borrow(&self) -> &Borrowed;
+}
+
+/// A trait for mutably borrowing data.
+///
+/// Similar to `Borrow`, but for mutable borrows.
+#[stable(feature = "rust1", since = "1.0.0")]
+pub trait BorrowMut<Borrowed: ?Sized> : Borrow<Borrowed> {
+    /// Mutably borrows from an owned value.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// use std::borrow::BorrowMut;
+    ///
+    /// fn check<T: BorrowMut<[i32]>>(mut v: T) {
+    ///     assert_eq!(&mut [1, 2, 3], v.borrow_mut());
+    /// }
+    ///
+    /// let v = vec![1, 2, 3];
+    ///
+    /// check(v);
+    /// ```
+    #[stable(feature = "rust1", since = "1.0.0")]
+    fn borrow_mut(&mut self) -> &mut Borrowed;
+}
+
+#[stable(feature = "rust1", since = "1.0.0")]
+impl<T: ?Sized> Borrow<T> for T {
+    fn borrow(&self) -> &T { self }
+}
+
+#[stable(feature = "rust1", since = "1.0.0")]
+impl<T: ?Sized> BorrowMut<T> for T {
+    fn borrow_mut(&mut self) -> &mut T { self }
+}
+
+#[stable(feature = "rust1", since = "1.0.0")]
+impl<'a, T: ?Sized> Borrow<T> for &'a T {
+    fn borrow(&self) -> &T { &**self }
+}
+
+#[stable(feature = "rust1", since = "1.0.0")]
+impl<'a, T: ?Sized> Borrow<T> for &'a mut T {
+    fn borrow(&self) -> &T { &**self }
+}
+
+#[stable(feature = "rust1", since = "1.0.0")]
+impl<'a, T: ?Sized> BorrowMut<T> for &'a mut T {
+    fn borrow_mut(&mut self) -> &mut T { &mut **self }
+}

src/libcore/lib.rs

@@ -139,6 +139,7 @@ pub mod cmp;
 pub mod clone;
 pub mod default;
 pub mod convert;
+pub mod borrow;
 
 /* Core types and methods on primitives */
 

src/libcoretest/char.rs

@@ -207,3 +207,12 @@ fn test_len_utf16() {
     assert!('\u{a66e}'.len_utf16() == 1);
     assert!('\u{1f4a9}'.len_utf16() == 2);
 }
+
+#[test]
+fn test_decode_utf16() {
+    fn check(s: &[u16], expected: &[Result<char, u16>]) {
+        assert_eq!(::std::char::decode_utf16(s.iter().cloned()).collect::<Vec<_>>(), expected);
+    }
+    check(&[0xD800, 0x41, 0x42], &[Err(0xD800), Ok('A'), Ok('B')]);
+    check(&[0xD800, 0], &[Err(0xD800), Ok('\0')]);
+}

src/libcoretest/lib.rs

@@ -19,6 +19,7 @@
 #![feature(float_from_str_radix)]
 #![feature(flt2dec)]
 #![feature(dec2flt)]
+#![feature(decode_utf16)]
 #![feature(fmt_radix)]
 #![feature(iter_arith)]
 #![feature(iter_arith)]