feature(SourceId): use stable hash from rustc-stable-hash (#14917)

### What does this PR try to resolve?

This helps `-Ztrim-paths` build a stable cross-platform path for the
registry and git sources. Sources files then can be found from the same
path when debugging.

It also helps cache registry index all at once for all platforms,
for example the use case in
#14795
(despite they should use `cargo vendor` instead IMO).

Some caveats:

* Newer cargo will need to re-download files for global caches
  (index files, git/registry sources).
  The old cache is still kept and used when running with older cargoes.
* Absolute paths on windows iarenot really covered by the
"cross-platform" hash,
  because path prefix components like `C:` are always there.
  That means hashes of some sources kind,
  like local registry and local path,
  are not going to be real cross-platform stable.

#### Security concern

There might be hash collisions if you have two registries under the same
domain. This won't happen to crates.io, as the infra would have to
intentionally put another registry on index.crates.io to collide.
We don't consider this is an actual threat model, so we are not going to
use any cryptographically secure hash algorithm like BLAKE3.

At least, the current unstable SipHash isn't in a better situation.
We might switch to a cryptographic secure one when needed.

See also
<#13171 (comment)>

### How should we test and review this PR?

We have an FCP in
<#14795 (comment)>.

This PR implements the proposal,
The path-length concern in
<#14795 (comment)>
is automatically addressed
because we don't need cryptographically secure hash for now.

### Additional information

See more information and benchmark results in
<#14116>.

Author: epage

Cargo.lock

@@ -82,6 +82,7 @@ rand = "0.8.5"
 regex = "1.10.5"
 rusqlite = { version = "0.32.0", features = ["bundled"] }
 rustc-hash = "2.0.0"
+rustc-stable-hash = "0.1.1"
 rustfix = { version = "0.9.0", path = "crates/rustfix" }
 same-file = "1.0.6"
 schemars = "0.8.21"
@@ -194,6 +195,7 @@ rand.workspace = true
 regex.workspace = true
 rusqlite.workspace = true
 rustc-hash.workspace = true
+rustc-stable-hash.workspace = true
 rustfix.workspace = true
 same-file.workspace = true
 semver.workspace = true

Cargo.toml

@@ -718,8 +718,8 @@ fn compute_metadata(
         }
     }
 
-    let c_metadata = UnitHash(c_metadata_hasher.finish());
-    let c_extra_filename = UnitHash(c_extra_filename_hasher.finish());
+    let c_metadata = UnitHash(Hasher::finish(&c_metadata_hasher));
+    let c_extra_filename = UnitHash(Hasher::finish(&c_extra_filename_hasher));
     let unit_id = c_extra_filename;
 
     let c_extra_filename = use_extra_filename.then_some(c_extra_filename);

src/cargo/core/compiler/build_runner/compilation_files.rs

@@ -195,6 +195,6 @@ impl CompileTarget {
                 self.name.hash(&mut hasher);
             }
         }
-        hasher.finish()
+        Hasher::finish(&hasher)
     }
 }

src/cargo/core/compiler/compile_kind.rs

@@ -1561,7 +1561,7 @@ fn calculate_normal(
         local: Mutex::new(local),
         memoized_hash: Mutex::new(None),
         metadata,
-        config: config.finish(),
+        config: Hasher::finish(&config),
         compile_kind,
         rustflags: extra_flags,
         fs_status: FsStatus::Stale,

src/cargo/core/compiler/fingerprint/mod.rs

@@ -782,70 +782,89 @@ mod tests {
     // Otherwise please just leave a comment in your PR as to why the hash value is
     // changing and why the old value can't be easily preserved.
     //
-    // The hash value depends on endianness and bit-width, so we only run this test on
-    // little-endian 64-bit CPUs (such as x86-64 and ARM64) where it matches the
-    // well-known value.
+    // The hash value should be stable across platforms, and doesn't depend on
+    // endianness and bit-width. One caveat is that absolute paths on Windows
+    // are inherently different than on Unix-like platforms. Unless we omit or
+    // strip the prefix components (e.g. `C:`), there is not way to have a true
+    // cross-platform stable hash for absolute paths.
     #[test]
-    #[cfg(all(target_endian = "little", target_pointer_width = "64"))]
-    fn test_cratesio_hash() {
-        let gctx = GlobalContext::default().unwrap();
-        let crates_io = SourceId::crates_io(&gctx).unwrap();
-        assert_eq!(crate::util::hex::short_hash(&crates_io), "1ecc6299db9ec823");
-    }
-
-    // See the comment in `test_cratesio_hash`.
-    //
-    // Only test on non-Windows as paths on Windows will get different hashes.
-    #[test]
-    #[cfg(all(target_endian = "little", target_pointer_width = "64", not(windows)))]
     fn test_stable_hash() {
         use std::hash::Hasher;
         use std::path::Path;
 
+        use crate::util::StableHasher;
+
+        #[cfg(not(windows))]
+        let ws_root = Path::new("/tmp/ws");
+        #[cfg(windows)]
+        let ws_root = Path::new(r"C:\\tmp\ws");
+
         let gen_hash = |source_id: SourceId| {
-            let mut hasher = std::collections::hash_map::DefaultHasher::new();
-            source_id.stable_hash(Path::new("/tmp/ws"), &mut hasher);
-            hasher.finish()
+            let mut hasher = StableHasher::new();
+            source_id.stable_hash(ws_root, &mut hasher);
+            Hasher::finish(&hasher)
         };
 
+        let source_id = SourceId::crates_io(&GlobalContext::default().unwrap()).unwrap();
+        assert_eq!(gen_hash(source_id), 7062945687441624357);
+        assert_eq!(crate::util::hex::short_hash(&source_id), "25cdd57fae9f0462");
+
         let url = "https://my-crates.io".into_url().unwrap();
         let source_id = SourceId::for_registry(&url).unwrap();
-        assert_eq!(gen_hash(source_id), 18108075011063494626);
-        assert_eq!(crate::util::hex::short_hash(&source_id), "fb60813d6cb8df79");
+        assert_eq!(gen_hash(source_id), 8310250053664888498);
+        assert_eq!(crate::util::hex::short_hash(&source_id), "b2d65deb64f05373");
 
         let url = "https://your-crates.io".into_url().unwrap();
         let source_id = SourceId::for_alt_registry(&url, "alt").unwrap();
-        assert_eq!(gen_hash(source_id), 12862859764592646184);
-        assert_eq!(crate::util::hex::short_hash(&source_id), "09c10fd0cbd74bce");
+        assert_eq!(gen_hash(source_id), 14149534903000258933);
+        assert_eq!(crate::util::hex::short_hash(&source_id), "755952de063f5dc4");
 
         let url = "sparse+https://my-crates.io".into_url().unwrap();
         let source_id = SourceId::for_registry(&url).unwrap();
-        assert_eq!(gen_hash(source_id), 8763561830438022424);
-        assert_eq!(crate::util::hex::short_hash(&source_id), "d1ea0d96f6f759b5");
+        assert_eq!(gen_hash(source_id), 16249512552851930162);
+        assert_eq!(crate::util::hex::short_hash(&source_id), "327cfdbd92dd81e1");
 
         let url = "sparse+https://your-crates.io".into_url().unwrap();
         let source_id = SourceId::for_alt_registry(&url, "alt").unwrap();
-        assert_eq!(gen_hash(source_id), 5159702466575482972);
-        assert_eq!(crate::util::hex::short_hash(&source_id), "135d23074253cb78");
+        assert_eq!(gen_hash(source_id), 6156697384053352292);
+        assert_eq!(crate::util::hex::short_hash(&source_id), "64a713b6a6fb7055");
 
         let url = "file:///tmp/ws/crate".into_url().unwrap();
         let source_id = SourceId::for_git(&url, GitReference::DefaultBranch).unwrap();
-        assert_eq!(gen_hash(source_id), 15332537265078583985);
-        assert_eq!(crate::util::hex::short_hash(&source_id), "73a808694abda756");
-
-        let path = Path::new("/tmp/ws/crate");
+        assert_eq!(gen_hash(source_id), 473480029881867801);
+        assert_eq!(crate::util::hex::short_hash(&source_id), "199e591d94239206");
 
+        let path = &ws_root.join("crate");
         let source_id = SourceId::for_local_registry(path).unwrap();
-        assert_eq!(gen_hash(source_id), 18446533307730842837);
-        assert_eq!(crate::util::hex::short_hash(&source_id), "52a84cc73f6fd48b");
+        #[cfg(not(windows))]
+        {
+            assert_eq!(gen_hash(source_id), 11515846423845066584);
+            assert_eq!(crate::util::hex::short_hash(&source_id), "58d73c154f81d09f");
+        }
+        #[cfg(windows)]
+        {
+            assert_eq!(gen_hash(source_id), 6146331155906064276);
+            assert_eq!(crate::util::hex::short_hash(&source_id), "946fb2239f274c55");
+        }
 
         let source_id = SourceId::for_path(path).unwrap();
-        assert_eq!(gen_hash(source_id), 8764714075439899829);
-        assert_eq!(crate::util::hex::short_hash(&source_id), "e1ddd48578620fc1");
+        assert_eq!(gen_hash(source_id), 215644081443634269);
+        #[cfg(not(windows))]
+        assert_eq!(crate::util::hex::short_hash(&source_id), "64bace89c92b101f");
+        #[cfg(windows)]
+        assert_eq!(crate::util::hex::short_hash(&source_id), "01e1e6c391813fb6");
 
         let source_id = SourceId::for_directory(path).unwrap();
-        assert_eq!(gen_hash(source_id), 17459999773908528552);
-        assert_eq!(crate::util::hex::short_hash(&source_id), "6568fe2c2fab5bfe");
+        #[cfg(not(windows))]
+        {
+            assert_eq!(gen_hash(source_id), 6127590343904940368);
+            assert_eq!(crate::util::hex::short_hash(&source_id), "505191d1f3920955");
+        }
+        #[cfg(windows)]
+        {
+            assert_eq!(gen_hash(source_id), 10423446877655960172);
+            assert_eq!(crate::util::hex::short_hash(&source_id), "6c8ad69db585a790");
+        }
     }
 
     #[test]

src/cargo/core/source_id.rs

@@ -657,7 +657,7 @@ fn traverse_and_share(
         .collect();
     // Here, we have recursively traversed this unit's dependencies, and hashed them: we can
     // finalize the dep hash.
-    let new_dep_hash = dep_hash.finish();
+    let new_dep_hash = Hasher::finish(&dep_hash);
 
     // This is the key part of the sharing process: if the unit is a runtime dependency, whose
     // target is the same as the host, we canonicalize the compile kind to `CompileKind::Host`.

src/cargo/ops/cargo_compile/mod.rs

@@ -1,25 +1,6 @@
-//! Implementation of a hasher that produces the same values across releases.
+//! A hasher that produces the same values across releases and platforms.
 //!
 //! The hasher should be fast and have a low chance of collisions (but is not
 //! sufficient for cryptographic purposes).
-#![allow(deprecated)]
 
-use std::hash::{Hasher, SipHasher};
-
-#[derive(Clone)]
-pub struct StableHasher(SipHasher);
-
-impl StableHasher {
-    pub fn new() -> StableHasher {
-        StableHasher(SipHasher::new())
-    }
-}
-
-impl Hasher for StableHasher {
-    fn finish(&self) -> u64 {
-        self.0.finish()
-    }
-    fn write(&mut self, bytes: &[u8]) {
-        self.0.write(bytes)
-    }
-}
+pub use rustc_stable_hash::StableSipHasher128 as StableHasher;

src/cargo/util/hasher.rs

@@ -10,7 +10,7 @@ pub fn to_hex(num: u64) -> String {
 pub fn hash_u64<H: Hash>(hashable: H) -> u64 {
     let mut hasher = StableHasher::new();
     hashable.hash(&mut hasher);
-    hasher.finish()
+    Hasher::finish(&hasher)
 }
 
 pub fn hash_u64_file(mut file: &File) -> std::io::Result<u64> {
@@ -23,7 +23,7 @@ pub fn hash_u64_file(mut file: &File) -> std::io::Result<u64> {
         }
         hasher.write(&buf[..n]);
     }
-    Ok(hasher.finish())
+    Ok(Hasher::finish(&hasher))
 }
 
 pub fn short_hash<H: Hash>(hashable: &H) -> String {

src/cargo/util/hex.rs

@@ -381,7 +381,7 @@ fn rustc_fingerprint(
         _ => (),
     }
 
-    Ok(hasher.finish())
+    Ok(Hasher::finish(&hasher))
 }
 
 fn process_fingerprint(cmd: &ProcessBuilder, extra_fingerprint: u64) -> u64 {
@@ -391,5 +391,5 @@ fn process_fingerprint(cmd: &ProcessBuilder, extra_fingerprint: u64) -> u64 {
     let mut env = cmd.get_envs().iter().collect::<Vec<_>>();
     env.sort_unstable();
     env.hash(&mut hasher);
-    hasher.finish()
+    Hasher::finish(&hasher)
 }

src/cargo/util/rustc.rs

@@ -2004,7 +2004,16 @@ fn compatible_with_older_cargo() {
     assert_eq!(get_registry_names("src"), ["middle-1.0.0", "new-1.0.0"]);
     assert_eq!(
         get_registry_names("cache"),
-        ["middle-1.0.0.crate", "new-1.0.0.crate", "old-1.0.0.crate"]
+        // Duplicate crates from two different cache location
+        // because we're changing how SourceId is hashed.
+        // This change should be reverted once rust-lang/cargo#14917 lands.
+        [
+            "middle-1.0.0.crate",
+            "middle-1.0.0.crate",
+            "new-1.0.0.crate",
+            "new-1.0.0.crate",
+            "old-1.0.0.crate"
+        ]
     );
 
     // T-0 months: Current version, make sure it can read data from stable,
@@ -2027,7 +2036,10 @@ fn compatible_with_older_cargo() {
     assert_eq!(get_registry_names("src"), ["new-1.0.0"]);
     assert_eq!(
         get_registry_names("cache"),
-        ["middle-1.0.0.crate", "new-1.0.0.crate"]
+        // Duplicate crates from two different cache location
+        // because we're changing how SourceId is hashed.
+        // This change should be reverted once rust-lang/cargo#14917 lands.
+        ["middle-1.0.0.crate", "new-1.0.0.crate", "new-1.0.0.crate"]
     );
 }