Add OsSystem support to mdtests (#16518)

## Summary This PR introduces a new mdtest option `system` that can either be `in-memory` or `os` where `in-memory` is the default. The motivation for supporting `os` is so that we can write OS/system specific tests with mdtests. Specifically, I want to write mdtests for the module resolver, testing that module resolution is case sensitive. ## Test Plan I tested that the case-sensitive module resolver test start failing when setting `system = "os"`
astral-sh · Mar 6, 2025 · ce0018c · ce0018c
1 parent 48f906e
commit ce0018c
Show file tree

Hide file tree

Showing 31 changed files with 519 additions and 207 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/crates/red_knot_project/src/files.rs b/crates/red_knot_project/src/files.rs
@@ -255,7 +255,7 @@ mod tests {
     use crate::files::Index;
     use crate::ProjectMetadata;
     use ruff_db::files::system_path_to_file;
-    use ruff_db::system::{DbWithTestSystem, SystemPathBuf};
+    use ruff_db::system::{DbWithWritableSystem as _, SystemPathBuf};
     use ruff_python_ast::name::Name;
 
     #[test]

diff --git a/crates/red_knot_project/src/lib.rs b/crates/red_knot_project/src/lib.rs
@@ -528,7 +528,7 @@ mod tests {
     use ruff_db::diagnostic::OldDiagnosticTrait;
     use ruff_db::files::system_path_to_file;
     use ruff_db::source::source_text;
-    use ruff_db::system::{DbWithTestSystem, SystemPath, SystemPathBuf};
+    use ruff_db::system::{DbWithTestSystem, DbWithWritableSystem as _, SystemPath, SystemPathBuf};
     use ruff_db::testing::assert_function_query_was_not_run;
     use ruff_python_ast::name::Name;
 

diff --git a/crates/red_knot_project/src/metadata.rs b/crates/red_knot_project/src/metadata.rs
@@ -321,7 +321,7 @@ mod tests {
 
         system
             .memory_file_system()
-            .write_files([(root.join("foo.py"), ""), (root.join("bar.py"), "")])
+            .write_files_all([(root.join("foo.py"), ""), (root.join("bar.py"), "")])
             .context("Failed to write files")?;
 
         let project =
@@ -349,7 +349,7 @@ mod tests {
 
         system
             .memory_file_system()
-            .write_files([
+            .write_files_all([
                 (
                     root.join("pyproject.toml"),
                     r#"
@@ -393,7 +393,7 @@ mod tests {
 
         system
             .memory_file_system()
-            .write_files([
+            .write_files_all([
                 (
                     root.join("pyproject.toml"),
                     r#"
@@ -432,7 +432,7 @@ expected `.`, `]`
 
         system
             .memory_file_system()
-            .write_files([
+            .write_files_all([
                 (
                     root.join("pyproject.toml"),
                     r#"
@@ -482,7 +482,7 @@ expected `.`, `]`
 
         system
             .memory_file_system()
-            .write_files([
+            .write_files_all([
                 (
                     root.join("pyproject.toml"),
                     r#"
@@ -532,7 +532,7 @@ expected `.`, `]`
 
         system
             .memory_file_system()
-            .write_files([
+            .write_files_all([
                 (
                     root.join("pyproject.toml"),
                     r#"
@@ -572,7 +572,7 @@ expected `.`, `]`
 
         system
             .memory_file_system()
-            .write_files([
+            .write_files_all([
                 (
                     root.join("pyproject.toml"),
                     r#"
@@ -623,7 +623,7 @@ expected `.`, `]`
 
         system
             .memory_file_system()
-            .write_files([
+            .write_files_all([
                 (
                     root.join("pyproject.toml"),
                     r#"
@@ -673,7 +673,7 @@ expected `.`, `]`
 
         system
             .memory_file_system()
-            .write_file(
+            .write_file_all(
                 root.join("pyproject.toml"),
                 r#"
                 [project]
@@ -703,7 +703,7 @@ expected `.`, `]`
 
         system
             .memory_file_system()
-            .write_file(
+            .write_file_all(
                 root.join("pyproject.toml"),
                 r#"
                 [project]
@@ -735,7 +735,7 @@ expected `.`, `]`
 
         system
             .memory_file_system()
-            .write_file(
+            .write_file_all(
                 root.join("pyproject.toml"),
                 r#"
                 [project]
@@ -765,7 +765,7 @@ expected `.`, `]`
 
         system
             .memory_file_system()
-            .write_file(
+            .write_file_all(
                 root.join("pyproject.toml"),
                 r#"
                 [project]
@@ -795,7 +795,7 @@ expected `.`, `]`
 
         system
             .memory_file_system()
-            .write_file(
+            .write_file_all(
                 root.join("pyproject.toml"),
                 r#"
                 [project]
@@ -828,7 +828,7 @@ expected `.`, `]`
 
         system
             .memory_file_system()
-            .write_file(
+            .write_file_all(
                 root.join("pyproject.toml"),
                 r#"
                 [project]
@@ -861,7 +861,7 @@ expected `.`, `]`
 
         system
             .memory_file_system()
-            .write_file(
+            .write_file_all(
                 root.join("pyproject.toml"),
                 r#"
                 [project]
@@ -886,7 +886,7 @@ expected `.`, `]`
 
         system
             .memory_file_system()
-            .write_file(
+            .write_file_all(
                 root.join("pyproject.toml"),
                 r#"
                 [project]
@@ -911,7 +911,7 @@ expected `.`, `]`
 
         system
             .memory_file_system()
-            .write_file(
+            .write_file_all(
                 root.join("pyproject.toml"),
                 r#"
                 [project]

diff --git a/crates/red_knot_project/tests/check.rs b/crates/red_knot_project/tests/check.rs
@@ -117,7 +117,7 @@ fn run_corpus_tests(pattern: &str) -> anyhow::Result<()> {
         let code = std::fs::read_to_string(source)?;
 
         let mut check_with_file_name = |path: &SystemPath| {
-            memory_fs.write_file(path, &code).unwrap();
+            memory_fs.write_file_all(path, &code).unwrap();
             File::sync_path(&mut db, path);
 
             // this test is only asserting that we can pull every expression type without a panic

diff --git a/crates/red_knot_python_semantic/resources/mdtest/import/case_sensitive.md b/crates/red_knot_python_semantic/resources/mdtest/import/case_sensitive.md
@@ -1,6 +1,11 @@
 # Case Sensitive Imports
 
-TODO: This test should use the real file system instead of the memory file system.
+```toml
+# TODO: This test should use the real file system instead of the memory file system.
+# but we can't change the file system yet because the tests would then start failing for 
+# case-insensitive file systems.
+#system = "os"
+```
 
 Python's import system is case-sensitive even on case-insensitive file system. This means, importing
 a module `a` should fail if the file in the search paths is named `A.py`. See

diff --git a/crates/red_knot_python_semantic/src/db.rs b/crates/red_knot_python_semantic/src/db.rs
@@ -25,7 +25,9 @@ pub(crate) mod tests {
     use crate::lint::{LintRegistry, RuleSelection};
     use anyhow::Context;
     use ruff_db::files::{File, Files};
-    use ruff_db::system::{DbWithTestSystem, System, SystemPathBuf, TestSystem};
+    use ruff_db::system::{
+        DbWithTestSystem, DbWithWritableSystem as _, System, SystemPathBuf, TestSystem,
+    };
     use ruff_db::vendored::VendoredFileSystem;
     use ruff_db::{Db as SourceDb, Upcast};
     use ruff_python_ast::PythonVersion;

diff --git a/crates/red_knot_python_semantic/src/module_resolver/resolver.rs b/crates/red_knot_python_semantic/src/module_resolver/resolver.rs
@@ -720,7 +720,7 @@ impl<'db> ResolverContext<'db> {
 #[cfg(test)]
 mod tests {
     use ruff_db::files::{system_path_to_file, File, FilePath};
-    use ruff_db::system::DbWithTestSystem;
+    use ruff_db::system::{DbWithTestSystem as _, DbWithWritableSystem as _};
     use ruff_db::testing::{
         assert_const_function_query_was_not_run, assert_function_query_was_not_run,
     };

diff --git a/crates/red_knot_python_semantic/src/module_resolver/testing.rs b/crates/red_knot_python_semantic/src/module_resolver/testing.rs
@@ -1,4 +1,6 @@
-use ruff_db::system::{DbWithTestSystem, SystemPath, SystemPathBuf};
+use ruff_db::system::{
+    DbWithTestSystem as _, DbWithWritableSystem as _, SystemPath, SystemPathBuf,
+};
 use ruff_db::vendored::VendoredPathBuf;
 use ruff_python_ast::PythonVersion;
 

diff --git a/crates/red_knot_python_semantic/src/semantic_index.rs b/crates/red_knot_python_semantic/src/semantic_index.rs
@@ -409,7 +409,7 @@ impl FusedIterator for ChildrenIter<'_> {}
 mod tests {
     use ruff_db::files::{system_path_to_file, File};
     use ruff_db::parsed::parsed_module;
-    use ruff_db::system::DbWithTestSystem;
+    use ruff_db::system::DbWithWritableSystem as _;
     use ruff_python_ast as ast;
     use ruff_text_size::{Ranged, TextRange};
 
@@ -440,7 +440,7 @@ mod tests {
         file: File,
     }
 
-    fn test_case(content: impl ToString) -> TestCase {
+    fn test_case(content: impl AsRef<str>) -> TestCase {
         let mut db = TestDb::new();
         db.write_file("test.py", content).unwrap();
 

diff --git a/crates/red_knot_python_semantic/src/site_packages.rs b/crates/red_knot_python_semantic/src/site_packages.rs
@@ -545,7 +545,7 @@ mod tests {
                         system_install_sys_prefix.join(&unix_site_packages);
                     (system_home_path, system_exe_path, system_site_packages_path)
                 };
-            memory_fs.write_file(system_exe_path, "").unwrap();
+            memory_fs.write_file_all(system_exe_path, "").unwrap();
             memory_fs
                 .create_directory_all(&system_site_packages_path)
                 .unwrap();
@@ -562,7 +562,7 @@ mod tests {
                     venv_sys_prefix.join(&unix_site_packages),
                 )
             };
-            memory_fs.write_file(&venv_exe, "").unwrap();
+            memory_fs.write_file_all(&venv_exe, "").unwrap();
             memory_fs.create_directory_all(&site_packages_path).unwrap();
 
             let pyvenv_cfg_path = venv_sys_prefix.join("pyvenv.cfg");
@@ -576,7 +576,7 @@ mod tests {
                 pyvenv_cfg_contents.push_str("include-system-site-packages = TRuE\n");
             }
             memory_fs
-                .write_file(pyvenv_cfg_path, &pyvenv_cfg_contents)
+                .write_file_all(pyvenv_cfg_path, &pyvenv_cfg_contents)
                 .unwrap();
 
             venv_sys_prefix
@@ -740,7 +740,7 @@ mod tests {
         let system = TestSystem::default();
         system
             .memory_file_system()
-            .write_file("/.venv", "")
+            .write_file_all("/.venv", "")
             .unwrap();
         assert!(matches!(
             VirtualEnvironment::new("/.venv", &system),
@@ -767,7 +767,7 @@ mod tests {
         let memory_fs = system.memory_file_system();
         let pyvenv_cfg_path = SystemPathBuf::from("/.venv/pyvenv.cfg");
         memory_fs
-            .write_file(&pyvenv_cfg_path, "home = bar = /.venv/bin")
+            .write_file_all(&pyvenv_cfg_path, "home = bar = /.venv/bin")
             .unwrap();
         let venv_result = VirtualEnvironment::new("/.venv", &system);
         assert!(matches!(
@@ -785,7 +785,9 @@ mod tests {
         let system = TestSystem::default();
         let memory_fs = system.memory_file_system();
         let pyvenv_cfg_path = SystemPathBuf::from("/.venv/pyvenv.cfg");
-        memory_fs.write_file(&pyvenv_cfg_path, "home =").unwrap();
+        memory_fs
+            .write_file_all(&pyvenv_cfg_path, "home =")
+            .unwrap();
         let venv_result = VirtualEnvironment::new("/.venv", &system);
         assert!(matches!(
             venv_result,
@@ -803,7 +805,7 @@ mod tests {
         let memory_fs = system.memory_file_system();
         let pyvenv_cfg_path = SystemPathBuf::from("/.venv/pyvenv.cfg");
         memory_fs
-            .write_file(&pyvenv_cfg_path, "= whatever")
+            .write_file_all(&pyvenv_cfg_path, "= whatever")
             .unwrap();
         let venv_result = VirtualEnvironment::new("/.venv", &system);
         assert!(matches!(
@@ -821,7 +823,7 @@ mod tests {
         let system = TestSystem::default();
         let memory_fs = system.memory_file_system();
         let pyvenv_cfg_path = SystemPathBuf::from("/.venv/pyvenv.cfg");
-        memory_fs.write_file(&pyvenv_cfg_path, "").unwrap();
+        memory_fs.write_file_all(&pyvenv_cfg_path, "").unwrap();
         let venv_result = VirtualEnvironment::new("/.venv", &system);
         assert!(matches!(
             venv_result,
@@ -839,7 +841,7 @@ mod tests {
         let memory_fs = system.memory_file_system();
         let pyvenv_cfg_path = SystemPathBuf::from("/.venv/pyvenv.cfg");
         memory_fs
-            .write_file(&pyvenv_cfg_path, "home = foo")
+            .write_file_all(&pyvenv_cfg_path, "home = foo")
             .unwrap();
         let venv_result = VirtualEnvironment::new("/.venv", &system);
         assert!(matches!(

diff --git a/crates/red_knot_python_semantic/src/types.rs b/crates/red_knot_python_semantic/src/types.rs
@@ -4350,7 +4350,7 @@ pub(crate) mod tests {
     };
     use ruff_db::files::system_path_to_file;
     use ruff_db::parsed::parsed_module;
-    use ruff_db::system::DbWithTestSystem;
+    use ruff_db::system::DbWithWritableSystem as _;
     use ruff_db::testing::assert_function_query_was_not_run;
     use ruff_python_ast::PythonVersion;
     use strum::IntoEnumIterator;

diff --git a/crates/red_knot_python_semantic/src/types/infer.rs b/crates/red_knot_python_semantic/src/types/infer.rs
@@ -6551,7 +6551,7 @@ mod tests {
     use crate::symbol::global_symbol;
     use crate::types::check_types;
     use ruff_db::files::{system_path_to_file, File};
-    use ruff_db::system::DbWithTestSystem;
+    use ruff_db::system::DbWithWritableSystem as _;
     use ruff_db::testing::{assert_function_query_was_not_run, assert_function_query_was_run};
 
     use super::*;

diff --git a/crates/red_knot_python_semantic/src/types/signatures.rs b/crates/red_knot_python_semantic/src/types/signatures.rs
@@ -348,7 +348,7 @@ mod tests {
     use crate::db::tests::{setup_db, TestDb};
     use crate::symbol::global_symbol;
     use crate::types::{FunctionType, KnownClass};
-    use ruff_db::system::DbWithTestSystem;
+    use ruff_db::system::DbWithWritableSystem as _;
 
     #[track_caller]
     fn get_function_f<'db>(db: &'db TestDb, file: &'static str) -> FunctionType<'db> {

diff --git a/crates/red_knot_test/Cargo.toml b/crates/red_knot_test/Cargo.toml
@@ -15,6 +15,7 @@ red_knot_python_semantic = { workspace = true, features = ["serde"] }
 red_knot_vendored = { workspace = true }
 ruff_db = { workspace = true, features = ["testing"] }
 ruff_index = { workspace = true }
+ruff_notebook = { workspace = true }
 ruff_python_trivia = { workspace = true }
 ruff_source_file = { workspace = true }
 ruff_text_size = { workspace = true }
@@ -30,6 +31,7 @@ rustc-hash = { workspace = true }
 salsa = { workspace = true }
 smallvec = { workspace = true }
 serde = { workspace = true }
+tempfile = { workspace = true }
 toml = { workspace = true }
 thiserror = { workspace = true }
 

diff --git a/crates/red_knot_test/src/assertion.rs b/crates/red_knot_test/src/assertion.rs
@@ -490,12 +490,12 @@ pub(crate) enum ErrorAssertionParseError<'a> {
 mod tests {
     use super::*;
     use ruff_db::files::system_path_to_file;
-    use ruff_db::system::{DbWithTestSystem, SystemPathBuf};
+    use ruff_db::system::DbWithWritableSystem as _;
     use ruff_python_trivia::textwrap::dedent;
     use ruff_source_file::OneIndexed;
 
     fn get_assertions(source: &str) -> InlineFileAssertions {
-        let mut db = crate::db::Db::setup(SystemPathBuf::from("/src"));
+        let mut db = Db::setup();
         db.write_file("/src/test.py", source).unwrap();
         let file = system_path_to_file(&db, "/src/test.py").unwrap();
         InlineFileAssertions::from_file(&db, file)