From ce0018c3cb22e39c12d0b88c83953ee215e72552 Mon Sep 17 00:00:00 2001 From: Micha Reiser Date: Thu, 6 Mar 2025 09:41:40 +0000 Subject: [PATCH] Add `OsSystem` support to mdtests (#16518) ## Summary This PR introduces a new mdtest option `system` that can either be `in-memory` or `os` where `in-memory` is the default. The motivation for supporting `os` is so that we can write OS/system specific tests with mdtests. Specifically, I want to write mdtests for the module resolver, testing that module resolution is case sensitive. ## Test Plan I tested that the case-sensitive module resolver test start failing when setting `system = "os"` --- Cargo.lock | 2 + crates/red_knot_project/src/files.rs | 2 +- crates/red_knot_project/src/lib.rs | 2 +- crates/red_knot_project/src/metadata.rs | 34 +-- crates/red_knot_project/tests/check.rs | 2 +- .../resources/mdtest/import/case_sensitive.md | 7 +- crates/red_knot_python_semantic/src/db.rs | 4 +- .../src/module_resolver/resolver.rs | 2 +- .../src/module_resolver/testing.rs | 4 +- .../src/semantic_index.rs | 4 +- .../src/site_packages.rs | 20 +- crates/red_knot_python_semantic/src/types.rs | 2 +- .../src/types/infer.rs | 2 +- .../src/types/signatures.rs | 2 +- crates/red_knot_test/Cargo.toml | 2 + crates/red_knot_test/src/assertion.rs | 4 +- crates/red_knot_test/src/config.rs | 23 +- crates/red_knot_test/src/db.rs | 234 +++++++++++++++--- crates/red_knot_test/src/diagnostic.rs | 4 +- crates/red_knot_test/src/lib.rs | 81 +++--- crates/red_knot_test/src/matcher.rs | 4 +- crates/red_knot_wasm/src/lib.rs | 2 +- crates/ruff_benchmark/benches/red_knot.rs | 4 +- crates/ruff_db/src/files.rs | 2 +- crates/ruff_db/src/parsed.rs | 8 +- crates/ruff_db/src/source.rs | 10 +- crates/ruff_db/src/system.rs | 11 +- crates/ruff_db/src/system/memory_fs.rs | 88 ++++--- crates/ruff_db/src/system/os.rs | 12 +- crates/ruff_db/src/system/test.rs | 144 +++++++---- .../red_knot_check_invalid_syntax.rs | 4 +- 31 files changed, 519 insertions(+), 207 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d79795d463cbe..5a8b235671ab3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2541,6 +2541,7 @@ dependencies = [ "regex", "ruff_db", "ruff_index", + "ruff_notebook", "ruff_python_ast", "ruff_python_trivia", "ruff_source_file", @@ -2549,6 +2550,7 @@ dependencies = [ "salsa", "serde", "smallvec", + "tempfile", "thiserror 2.0.11", "toml", ] diff --git a/crates/red_knot_project/src/files.rs b/crates/red_knot_project/src/files.rs index 6935577ba21a5..182e20c03a18e 100644 --- a/crates/red_knot_project/src/files.rs +++ b/crates/red_knot_project/src/files.rs @@ -255,7 +255,7 @@ mod tests { use crate::files::Index; use crate::ProjectMetadata; use ruff_db::files::system_path_to_file; - use ruff_db::system::{DbWithTestSystem, SystemPathBuf}; + use ruff_db::system::{DbWithWritableSystem as _, SystemPathBuf}; use ruff_python_ast::name::Name; #[test] diff --git a/crates/red_knot_project/src/lib.rs b/crates/red_knot_project/src/lib.rs index 1cc4c7d850a3a..580b000486a02 100644 --- a/crates/red_knot_project/src/lib.rs +++ b/crates/red_knot_project/src/lib.rs @@ -528,7 +528,7 @@ mod tests { use ruff_db::diagnostic::OldDiagnosticTrait; use ruff_db::files::system_path_to_file; use ruff_db::source::source_text; - use ruff_db::system::{DbWithTestSystem, SystemPath, SystemPathBuf}; + use ruff_db::system::{DbWithTestSystem, DbWithWritableSystem as _, SystemPath, SystemPathBuf}; use ruff_db::testing::assert_function_query_was_not_run; use ruff_python_ast::name::Name; diff --git a/crates/red_knot_project/src/metadata.rs b/crates/red_knot_project/src/metadata.rs index d6e174059e0d8..76bb10c612868 100644 --- a/crates/red_knot_project/src/metadata.rs +++ b/crates/red_knot_project/src/metadata.rs @@ -321,7 +321,7 @@ mod tests { system .memory_file_system() - .write_files([(root.join("foo.py"), ""), (root.join("bar.py"), "")]) + .write_files_all([(root.join("foo.py"), ""), (root.join("bar.py"), "")]) .context("Failed to write files")?; let project = @@ -349,7 +349,7 @@ mod tests { system .memory_file_system() - .write_files([ + .write_files_all([ ( root.join("pyproject.toml"), r#" @@ -393,7 +393,7 @@ mod tests { system .memory_file_system() - .write_files([ + .write_files_all([ ( root.join("pyproject.toml"), r#" @@ -432,7 +432,7 @@ expected `.`, `]` system .memory_file_system() - .write_files([ + .write_files_all([ ( root.join("pyproject.toml"), r#" @@ -482,7 +482,7 @@ expected `.`, `]` system .memory_file_system() - .write_files([ + .write_files_all([ ( root.join("pyproject.toml"), r#" @@ -532,7 +532,7 @@ expected `.`, `]` system .memory_file_system() - .write_files([ + .write_files_all([ ( root.join("pyproject.toml"), r#" @@ -572,7 +572,7 @@ expected `.`, `]` system .memory_file_system() - .write_files([ + .write_files_all([ ( root.join("pyproject.toml"), r#" @@ -623,7 +623,7 @@ expected `.`, `]` system .memory_file_system() - .write_files([ + .write_files_all([ ( root.join("pyproject.toml"), r#" @@ -673,7 +673,7 @@ expected `.`, `]` system .memory_file_system() - .write_file( + .write_file_all( root.join("pyproject.toml"), r#" [project] @@ -703,7 +703,7 @@ expected `.`, `]` system .memory_file_system() - .write_file( + .write_file_all( root.join("pyproject.toml"), r#" [project] @@ -735,7 +735,7 @@ expected `.`, `]` system .memory_file_system() - .write_file( + .write_file_all( root.join("pyproject.toml"), r#" [project] @@ -765,7 +765,7 @@ expected `.`, `]` system .memory_file_system() - .write_file( + .write_file_all( root.join("pyproject.toml"), r#" [project] @@ -795,7 +795,7 @@ expected `.`, `]` system .memory_file_system() - .write_file( + .write_file_all( root.join("pyproject.toml"), r#" [project] @@ -828,7 +828,7 @@ expected `.`, `]` system .memory_file_system() - .write_file( + .write_file_all( root.join("pyproject.toml"), r#" [project] @@ -861,7 +861,7 @@ expected `.`, `]` system .memory_file_system() - .write_file( + .write_file_all( root.join("pyproject.toml"), r#" [project] @@ -886,7 +886,7 @@ expected `.`, `]` system .memory_file_system() - .write_file( + .write_file_all( root.join("pyproject.toml"), r#" [project] @@ -911,7 +911,7 @@ expected `.`, `]` system .memory_file_system() - .write_file( + .write_file_all( root.join("pyproject.toml"), r#" [project] diff --git a/crates/red_knot_project/tests/check.rs b/crates/red_knot_project/tests/check.rs index 5e94a960004a9..bf95eefe4c8f7 100644 --- a/crates/red_knot_project/tests/check.rs +++ b/crates/red_knot_project/tests/check.rs @@ -117,7 +117,7 @@ fn run_corpus_tests(pattern: &str) -> anyhow::Result<()> { let code = std::fs::read_to_string(source)?; let mut check_with_file_name = |path: &SystemPath| { - memory_fs.write_file(path, &code).unwrap(); + memory_fs.write_file_all(path, &code).unwrap(); File::sync_path(&mut db, path); // this test is only asserting that we can pull every expression type without a panic diff --git a/crates/red_knot_python_semantic/resources/mdtest/import/case_sensitive.md b/crates/red_knot_python_semantic/resources/mdtest/import/case_sensitive.md index 5ce8c1a11911f..568aa7c57cdf1 100644 --- a/crates/red_knot_python_semantic/resources/mdtest/import/case_sensitive.md +++ b/crates/red_knot_python_semantic/resources/mdtest/import/case_sensitive.md @@ -1,6 +1,11 @@ # Case Sensitive Imports -TODO: This test should use the real file system instead of the memory file system. +```toml +# TODO: This test should use the real file system instead of the memory file system. +# but we can't change the file system yet because the tests would then start failing for +# case-insensitive file systems. +#system = "os" +``` Python's import system is case-sensitive even on case-insensitive file system. This means, importing a module `a` should fail if the file in the search paths is named `A.py`. See diff --git a/crates/red_knot_python_semantic/src/db.rs b/crates/red_knot_python_semantic/src/db.rs index d84a3648ec0f2..477ff9a2f3d9b 100644 --- a/crates/red_knot_python_semantic/src/db.rs +++ b/crates/red_knot_python_semantic/src/db.rs @@ -25,7 +25,9 @@ pub(crate) mod tests { use crate::lint::{LintRegistry, RuleSelection}; use anyhow::Context; use ruff_db::files::{File, Files}; - use ruff_db::system::{DbWithTestSystem, System, SystemPathBuf, TestSystem}; + use ruff_db::system::{ + DbWithTestSystem, DbWithWritableSystem as _, System, SystemPathBuf, TestSystem, + }; use ruff_db::vendored::VendoredFileSystem; use ruff_db::{Db as SourceDb, Upcast}; use ruff_python_ast::PythonVersion; diff --git a/crates/red_knot_python_semantic/src/module_resolver/resolver.rs b/crates/red_knot_python_semantic/src/module_resolver/resolver.rs index 276640de801b1..51fd4fb4bea8e 100644 --- a/crates/red_knot_python_semantic/src/module_resolver/resolver.rs +++ b/crates/red_knot_python_semantic/src/module_resolver/resolver.rs @@ -720,7 +720,7 @@ impl<'db> ResolverContext<'db> { #[cfg(test)] mod tests { use ruff_db::files::{system_path_to_file, File, FilePath}; - use ruff_db::system::DbWithTestSystem; + use ruff_db::system::{DbWithTestSystem as _, DbWithWritableSystem as _}; use ruff_db::testing::{ assert_const_function_query_was_not_run, assert_function_query_was_not_run, }; diff --git a/crates/red_knot_python_semantic/src/module_resolver/testing.rs b/crates/red_knot_python_semantic/src/module_resolver/testing.rs index 71b18a3ab15c7..75e0dfdc233b5 100644 --- a/crates/red_knot_python_semantic/src/module_resolver/testing.rs +++ b/crates/red_knot_python_semantic/src/module_resolver/testing.rs @@ -1,4 +1,6 @@ -use ruff_db::system::{DbWithTestSystem, SystemPath, SystemPathBuf}; +use ruff_db::system::{ + DbWithTestSystem as _, DbWithWritableSystem as _, SystemPath, SystemPathBuf, +}; use ruff_db::vendored::VendoredPathBuf; use ruff_python_ast::PythonVersion; diff --git a/crates/red_knot_python_semantic/src/semantic_index.rs b/crates/red_knot_python_semantic/src/semantic_index.rs index b003d142ea55f..dd103425eb3ab 100644 --- a/crates/red_knot_python_semantic/src/semantic_index.rs +++ b/crates/red_knot_python_semantic/src/semantic_index.rs @@ -409,7 +409,7 @@ impl FusedIterator for ChildrenIter<'_> {} mod tests { use ruff_db::files::{system_path_to_file, File}; use ruff_db::parsed::parsed_module; - use ruff_db::system::DbWithTestSystem; + use ruff_db::system::DbWithWritableSystem as _; use ruff_python_ast as ast; use ruff_text_size::{Ranged, TextRange}; @@ -440,7 +440,7 @@ mod tests { file: File, } - fn test_case(content: impl ToString) -> TestCase { + fn test_case(content: impl AsRef) -> TestCase { let mut db = TestDb::new(); db.write_file("test.py", content).unwrap(); diff --git a/crates/red_knot_python_semantic/src/site_packages.rs b/crates/red_knot_python_semantic/src/site_packages.rs index 3fc5719b27a85..f2812aa70d57a 100644 --- a/crates/red_knot_python_semantic/src/site_packages.rs +++ b/crates/red_knot_python_semantic/src/site_packages.rs @@ -545,7 +545,7 @@ mod tests { system_install_sys_prefix.join(&unix_site_packages); (system_home_path, system_exe_path, system_site_packages_path) }; - memory_fs.write_file(system_exe_path, "").unwrap(); + memory_fs.write_file_all(system_exe_path, "").unwrap(); memory_fs .create_directory_all(&system_site_packages_path) .unwrap(); @@ -562,7 +562,7 @@ mod tests { venv_sys_prefix.join(&unix_site_packages), ) }; - memory_fs.write_file(&venv_exe, "").unwrap(); + memory_fs.write_file_all(&venv_exe, "").unwrap(); memory_fs.create_directory_all(&site_packages_path).unwrap(); let pyvenv_cfg_path = venv_sys_prefix.join("pyvenv.cfg"); @@ -576,7 +576,7 @@ mod tests { pyvenv_cfg_contents.push_str("include-system-site-packages = TRuE\n"); } memory_fs - .write_file(pyvenv_cfg_path, &pyvenv_cfg_contents) + .write_file_all(pyvenv_cfg_path, &pyvenv_cfg_contents) .unwrap(); venv_sys_prefix @@ -740,7 +740,7 @@ mod tests { let system = TestSystem::default(); system .memory_file_system() - .write_file("/.venv", "") + .write_file_all("/.venv", "") .unwrap(); assert!(matches!( VirtualEnvironment::new("/.venv", &system), @@ -767,7 +767,7 @@ mod tests { let memory_fs = system.memory_file_system(); let pyvenv_cfg_path = SystemPathBuf::from("/.venv/pyvenv.cfg"); memory_fs - .write_file(&pyvenv_cfg_path, "home = bar = /.venv/bin") + .write_file_all(&pyvenv_cfg_path, "home = bar = /.venv/bin") .unwrap(); let venv_result = VirtualEnvironment::new("/.venv", &system); assert!(matches!( @@ -785,7 +785,9 @@ mod tests { let system = TestSystem::default(); let memory_fs = system.memory_file_system(); let pyvenv_cfg_path = SystemPathBuf::from("/.venv/pyvenv.cfg"); - memory_fs.write_file(&pyvenv_cfg_path, "home =").unwrap(); + memory_fs + .write_file_all(&pyvenv_cfg_path, "home =") + .unwrap(); let venv_result = VirtualEnvironment::new("/.venv", &system); assert!(matches!( venv_result, @@ -803,7 +805,7 @@ mod tests { let memory_fs = system.memory_file_system(); let pyvenv_cfg_path = SystemPathBuf::from("/.venv/pyvenv.cfg"); memory_fs - .write_file(&pyvenv_cfg_path, "= whatever") + .write_file_all(&pyvenv_cfg_path, "= whatever") .unwrap(); let venv_result = VirtualEnvironment::new("/.venv", &system); assert!(matches!( @@ -821,7 +823,7 @@ mod tests { let system = TestSystem::default(); let memory_fs = system.memory_file_system(); let pyvenv_cfg_path = SystemPathBuf::from("/.venv/pyvenv.cfg"); - memory_fs.write_file(&pyvenv_cfg_path, "").unwrap(); + memory_fs.write_file_all(&pyvenv_cfg_path, "").unwrap(); let venv_result = VirtualEnvironment::new("/.venv", &system); assert!(matches!( venv_result, @@ -839,7 +841,7 @@ mod tests { let memory_fs = system.memory_file_system(); let pyvenv_cfg_path = SystemPathBuf::from("/.venv/pyvenv.cfg"); memory_fs - .write_file(&pyvenv_cfg_path, "home = foo") + .write_file_all(&pyvenv_cfg_path, "home = foo") .unwrap(); let venv_result = VirtualEnvironment::new("/.venv", &system); assert!(matches!( diff --git a/crates/red_knot_python_semantic/src/types.rs b/crates/red_knot_python_semantic/src/types.rs index 879ee52619636..6e75548ce22c6 100644 --- a/crates/red_knot_python_semantic/src/types.rs +++ b/crates/red_knot_python_semantic/src/types.rs @@ -4350,7 +4350,7 @@ pub(crate) mod tests { }; use ruff_db::files::system_path_to_file; use ruff_db::parsed::parsed_module; - use ruff_db::system::DbWithTestSystem; + use ruff_db::system::DbWithWritableSystem as _; use ruff_db::testing::assert_function_query_was_not_run; use ruff_python_ast::PythonVersion; use strum::IntoEnumIterator; diff --git a/crates/red_knot_python_semantic/src/types/infer.rs b/crates/red_knot_python_semantic/src/types/infer.rs index 90e1cf3a0811d..bf8e4c1d23a0f 100644 --- a/crates/red_knot_python_semantic/src/types/infer.rs +++ b/crates/red_knot_python_semantic/src/types/infer.rs @@ -6551,7 +6551,7 @@ mod tests { use crate::symbol::global_symbol; use crate::types::check_types; use ruff_db::files::{system_path_to_file, File}; - use ruff_db::system::DbWithTestSystem; + use ruff_db::system::DbWithWritableSystem as _; use ruff_db::testing::{assert_function_query_was_not_run, assert_function_query_was_run}; use super::*; diff --git a/crates/red_knot_python_semantic/src/types/signatures.rs b/crates/red_knot_python_semantic/src/types/signatures.rs index 067c2c57948d2..17c4f57c73ab3 100644 --- a/crates/red_knot_python_semantic/src/types/signatures.rs +++ b/crates/red_knot_python_semantic/src/types/signatures.rs @@ -348,7 +348,7 @@ mod tests { use crate::db::tests::{setup_db, TestDb}; use crate::symbol::global_symbol; use crate::types::{FunctionType, KnownClass}; - use ruff_db::system::DbWithTestSystem; + use ruff_db::system::DbWithWritableSystem as _; #[track_caller] fn get_function_f<'db>(db: &'db TestDb, file: &'static str) -> FunctionType<'db> { diff --git a/crates/red_knot_test/Cargo.toml b/crates/red_knot_test/Cargo.toml index df1e807671c2e..fee0d05141ead 100644 --- a/crates/red_knot_test/Cargo.toml +++ b/crates/red_knot_test/Cargo.toml @@ -15,6 +15,7 @@ red_knot_python_semantic = { workspace = true, features = ["serde"] } red_knot_vendored = { workspace = true } ruff_db = { workspace = true, features = ["testing"] } ruff_index = { workspace = true } +ruff_notebook = { workspace = true } ruff_python_trivia = { workspace = true } ruff_source_file = { workspace = true } ruff_text_size = { workspace = true } @@ -30,6 +31,7 @@ rustc-hash = { workspace = true } salsa = { workspace = true } smallvec = { workspace = true } serde = { workspace = true } +tempfile = { workspace = true } toml = { workspace = true } thiserror = { workspace = true } diff --git a/crates/red_knot_test/src/assertion.rs b/crates/red_knot_test/src/assertion.rs index cb0c7993a3d58..8c4f3fbfea1d9 100644 --- a/crates/red_knot_test/src/assertion.rs +++ b/crates/red_knot_test/src/assertion.rs @@ -490,12 +490,12 @@ pub(crate) enum ErrorAssertionParseError<'a> { mod tests { use super::*; use ruff_db::files::system_path_to_file; - use ruff_db::system::{DbWithTestSystem, SystemPathBuf}; + use ruff_db::system::DbWithWritableSystem as _; use ruff_python_trivia::textwrap::dedent; use ruff_source_file::OneIndexed; fn get_assertions(source: &str) -> InlineFileAssertions { - let mut db = crate::db::Db::setup(SystemPathBuf::from("/src")); + let mut db = Db::setup(); db.write_file("/src/test.py", source).unwrap(); let file = system_path_to_file(&db, "/src/test.py").unwrap(); InlineFileAssertions::from_file(&db, file) diff --git a/crates/red_knot_test/src/config.rs b/crates/red_knot_test/src/config.rs index 842e108f4e5cb..173453ed189c5 100644 --- a/crates/red_knot_test/src/config.rs +++ b/crates/red_knot_test/src/config.rs @@ -12,7 +12,7 @@ use anyhow::Context; use red_knot_python_semantic::PythonPlatform; use ruff_db::system::{SystemPath, SystemPathBuf}; use ruff_python_ast::PythonVersion; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; #[derive(Deserialize, Debug, Default, Clone)] #[serde(rename_all = "kebab-case", deny_unknown_fields)] @@ -20,6 +20,11 @@ pub(crate) struct MarkdownTestConfig { pub(crate) environment: Option, pub(crate) log: Option, + + /// The [`ruff_db::system::System`] to use for tests. + /// + /// Defaults to the case-sensitive [`ruff_db::system::InMemorySystem`]. + pub(crate) system: Option, } impl MarkdownTestConfig { @@ -74,3 +79,19 @@ pub(crate) enum Log { /// Enable logging and only show filters that match the given [env-filter](https://docs.rs/tracing-subscriber/latest/tracing_subscriber/filter/struct.EnvFilter.html) Filter(String), } + +/// The system to use for tests. +#[derive(Copy, Clone, Debug, PartialEq, Eq, Deserialize, Serialize, Default)] +#[serde(rename_all = "kebab-case")] +pub(crate) enum SystemKind { + /// Use an in-memory system with a case sensitive file system.. + /// + /// This is recommended for all tests because it's fast. + #[default] + InMemory, + + /// Use the os system. + /// + /// This system should only be used when testing system or OS specific behavior. + Os, +} diff --git a/crates/red_knot_test/src/db.rs b/crates/red_knot_test/src/db.rs index 673511b150590..aaa770fc546c6 100644 --- a/crates/red_knot_test/src/db.rs +++ b/crates/red_knot_test/src/db.rs @@ -1,69 +1,53 @@ -use std::sync::Arc; - +use camino::{Utf8Component, Utf8PathBuf}; use red_knot_python_semantic::lint::{LintRegistry, RuleSelection}; -use red_knot_python_semantic::{ - default_lint_registry, Db as SemanticDb, Program, ProgramSettings, PythonPlatform, - SearchPathSettings, -}; +use red_knot_python_semantic::{default_lint_registry, Db as SemanticDb}; use ruff_db::files::{File, Files}; -use ruff_db::system::{DbWithTestSystem, System, SystemPath, SystemPathBuf, TestSystem}; +use ruff_db::system::{ + DbWithWritableSystem, InMemorySystem, OsSystem, System, SystemPath, SystemPathBuf, + WritableSystem, +}; use ruff_db::vendored::VendoredFileSystem; use ruff_db::{Db as SourceDb, Upcast}; -use ruff_python_ast::PythonVersion; +use ruff_notebook::{Notebook, NotebookError}; +use std::borrow::Cow; +use std::sync::Arc; +use tempfile::TempDir; #[salsa::db] #[derive(Clone)] pub(crate) struct Db { - project_root: SystemPathBuf, storage: salsa::Storage, files: Files, - system: TestSystem, + system: MdtestSystem, vendored: VendoredFileSystem, rule_selection: Arc, } impl Db { - pub(crate) fn setup(project_root: SystemPathBuf) -> Self { + pub(crate) fn setup() -> Self { let rule_selection = RuleSelection::from_registry(default_lint_registry()); - let db = Self { - project_root, + Self { + system: MdtestSystem::in_memory(), storage: salsa::Storage::default(), - system: TestSystem::default(), vendored: red_knot_vendored::file_system().clone(), files: Files::default(), rule_selection: Arc::new(rule_selection), - }; - - db.memory_file_system() - .create_directory_all(&db.project_root) - .unwrap(); - - Program::from_settings( - &db, - ProgramSettings { - python_version: PythonVersion::default(), - python_platform: PythonPlatform::default(), - search_paths: SearchPathSettings::new(vec![db.project_root.clone()]), - }, - ) - .expect("Invalid search path settings"); - - db + } } - pub(crate) fn project_root(&self) -> &SystemPath { - &self.project_root + pub(crate) fn use_os_system_with_temp_dir(&mut self, cwd: SystemPathBuf, temp_dir: TempDir) { + self.system.with_os(cwd, temp_dir); + Files::sync_all(self); } -} -impl DbWithTestSystem for Db { - fn test_system(&self) -> &TestSystem { - &self.system + pub(crate) fn use_in_memory_system(&mut self) { + self.system.with_in_memory(); + Files::sync_all(self); } - fn test_system_mut(&mut self) -> &mut TestSystem { - &mut self.system + pub(crate) fn create_directory_all(&self, path: &SystemPath) -> ruff_db::system::Result<()> { + self.system.create_directory_all(path) } } @@ -110,3 +94,175 @@ impl SemanticDb for Db { impl salsa::Database for Db { fn salsa_event(&self, _event: &dyn Fn() -> salsa::Event) {} } + +impl DbWithWritableSystem for Db { + type System = MdtestSystem; + fn writable_system(&self) -> &Self::System { + &self.system + } +} + +#[derive(Debug, Clone)] +pub(crate) struct MdtestSystem(Arc); + +#[derive(Debug)] +enum MdtestSystemInner { + InMemory(InMemorySystem), + Os { + os_system: OsSystem, + _temp_dir: TempDir, + }, +} + +impl MdtestSystem { + fn in_memory() -> Self { + Self(Arc::new(MdtestSystemInner::InMemory( + InMemorySystem::default(), + ))) + } + + fn as_system(&self) -> &dyn WritableSystem { + match &*self.0 { + MdtestSystemInner::InMemory(system) => system, + MdtestSystemInner::Os { os_system, .. } => os_system, + } + } + + fn with_os(&mut self, cwd: SystemPathBuf, temp_dir: TempDir) { + self.0 = Arc::new(MdtestSystemInner::Os { + os_system: OsSystem::new(cwd), + _temp_dir: temp_dir, + }); + } + + fn with_in_memory(&mut self) { + if let MdtestSystemInner::InMemory(in_memory) = &*self.0 { + in_memory.fs().remove_all(); + } else { + self.0 = Arc::new(MdtestSystemInner::InMemory(InMemorySystem::default())); + } + } + + fn normalize_path<'a>(&self, path: &'a SystemPath) -> Cow<'a, SystemPath> { + match &*self.0 { + MdtestSystemInner::InMemory(_) => Cow::Borrowed(path), + MdtestSystemInner::Os { os_system, .. } => { + // Make all paths relative to the current directory + // to avoid writing or reading from outside the temp directory. + let without_root: Utf8PathBuf = path + .components() + .skip_while(|component| { + matches!( + component, + Utf8Component::RootDir | Utf8Component::Prefix(..) + ) + }) + .collect(); + Cow::Owned(os_system.current_directory().join(&without_root)) + } + } + } +} + +impl System for MdtestSystem { + fn path_metadata( + &self, + path: &SystemPath, + ) -> ruff_db::system::Result { + self.as_system().path_metadata(&self.normalize_path(path)) + } + + fn canonicalize_path(&self, path: &SystemPath) -> ruff_db::system::Result { + let canonicalized = self + .as_system() + .canonicalize_path(&self.normalize_path(path))?; + + if let MdtestSystemInner::Os { os_system, .. } = &*self.0 { + // Make the path relative to the current directory + Ok(canonicalized + .strip_prefix(os_system.current_directory()) + .unwrap() + .to_owned()) + } else { + Ok(canonicalized) + } + } + + fn read_to_string(&self, path: &SystemPath) -> ruff_db::system::Result { + self.as_system().read_to_string(&self.normalize_path(path)) + } + + fn read_to_notebook(&self, path: &SystemPath) -> Result { + self.as_system() + .read_to_notebook(&self.normalize_path(path)) + } + + fn read_virtual_path_to_string( + &self, + path: &ruff_db::system::SystemVirtualPath, + ) -> ruff_db::system::Result { + self.as_system().read_virtual_path_to_string(path) + } + + fn read_virtual_path_to_notebook( + &self, + path: &ruff_db::system::SystemVirtualPath, + ) -> Result { + self.as_system().read_virtual_path_to_notebook(path) + } + + fn current_directory(&self) -> &SystemPath { + self.as_system().current_directory() + } + + fn user_config_directory(&self) -> Option { + self.as_system().user_config_directory() + } + + fn read_directory<'a>( + &'a self, + path: &SystemPath, + ) -> ruff_db::system::Result< + Box> + 'a>, + > { + self.as_system().read_directory(&self.normalize_path(path)) + } + + fn walk_directory( + &self, + path: &SystemPath, + ) -> ruff_db::system::walk_directory::WalkDirectoryBuilder { + self.as_system().walk_directory(&self.normalize_path(path)) + } + + fn glob( + &self, + pattern: &str, + ) -> Result< + Box>>, + ruff_db::system::PatternError, + > { + self.as_system() + .glob(self.normalize_path(SystemPath::new(pattern)).as_str()) + } + + fn as_any(&self) -> &dyn std::any::Any { + self + } + + fn as_any_mut(&mut self) -> &mut dyn std::any::Any { + self + } +} + +impl WritableSystem for MdtestSystem { + fn write_file(&self, path: &SystemPath, content: &str) -> ruff_db::system::Result<()> { + self.as_system() + .write_file(&self.normalize_path(path), content) + } + + fn create_directory_all(&self, path: &SystemPath) -> ruff_db::system::Result<()> { + self.as_system() + .create_directory_all(&self.normalize_path(path)) + } +} diff --git a/crates/red_knot_test/src/diagnostic.rs b/crates/red_knot_test/src/diagnostic.rs index b233edd26b0eb..db3c2a77d2592 100644 --- a/crates/red_knot_test/src/diagnostic.rs +++ b/crates/red_knot_test/src/diagnostic.rs @@ -148,14 +148,14 @@ mod tests { use ruff_db::diagnostic::{DiagnosticId, LintName, Severity, Span}; use ruff_db::files::{system_path_to_file, File}; use ruff_db::source::line_index; - use ruff_db::system::{DbWithTestSystem, SystemPathBuf}; + use ruff_db::system::DbWithWritableSystem as _; use ruff_source_file::OneIndexed; use ruff_text_size::{TextRange, TextSize}; use std::borrow::Cow; #[test] fn sort_and_group() { - let mut db = Db::setup(SystemPathBuf::from("/src")); + let mut db = Db::setup(); db.write_file("/src/test.py", "one\ntwo\n").unwrap(); let file = system_path_to_file(&db, "/src/test.py").unwrap(); let lines = line_index(&db, file); diff --git a/crates/red_knot_test/src/lib.rs b/crates/red_knot_test/src/lib.rs index ef5e40953df9a..e649626056e35 100644 --- a/crates/red_knot_test/src/lib.rs +++ b/crates/red_knot_test/src/lib.rs @@ -2,14 +2,15 @@ use crate::config::Log; use crate::parser::{BacktickOffsets, EmbeddedFileSourceMap}; use camino::Utf8Path; use colored::Colorize; +use config::SystemKind; use parser as test_parser; use red_knot_python_semantic::types::check_types; use red_knot_python_semantic::{Program, ProgramSettings, PythonPath, SearchPathSettings}; use ruff_db::diagnostic::{DisplayDiagnosticConfig, OldDiagnosticTrait, OldParseDiagnostic}; -use ruff_db::files::{system_path_to_file, File, Files}; +use ruff_db::files::{system_path_to_file, File}; use ruff_db::panic::catch_unwind; use ruff_db::parsed::parsed_module; -use ruff_db::system::{DbWithTestSystem, SystemPath, SystemPathBuf}; +use ruff_db::system::{DbWithWritableSystem as _, SystemPath, SystemPathBuf}; use ruff_db::testing::{setup_logging, setup_logging_with_filter}; use ruff_source_file::{LineIndex, OneIndexed}; use std::fmt::Write; @@ -42,7 +43,7 @@ pub fn run( } }; - let mut db = db::Db::setup(SystemPathBuf::from("/src")); + let mut db = db::Db::setup(); let filter = std::env::var(MDTEST_TEST_FILTER).ok(); let mut any_failures = false; @@ -56,10 +57,6 @@ pub fn run( Log::Filter(filter) => setup_logging_with_filter(filter), }); - // Remove all files so that the db is in a "fresh" state. - db.memory_file_system().remove_all(); - Files::sync_all(&mut db); - if let Err(failures) = run_test(&mut db, relative_fixture_path, snapshot_path, &test) { any_failures = true; println!("\n{}\n", test.name().bold().underline()); @@ -104,9 +101,30 @@ fn run_test( snapshot_path: &Utf8Path, test: &parser::MarkdownTest, ) -> Result<(), Failures> { - let project_root = db.project_root().to_path_buf(); - let src_path = SystemPathBuf::from("/src"); - let custom_typeshed_path = test.configuration().typeshed().map(SystemPath::to_path_buf); + // Initialize the system and remove all files and directories to reset the system to a clean state. + match test.configuration().system.unwrap_or_default() { + SystemKind::InMemory => { + db.use_in_memory_system(); + } + SystemKind::Os => { + let dir = tempfile::TempDir::new().expect("Creating a temporary directory to succeed"); + let root_path = dir + .path() + .canonicalize() + .expect("Canonicalizing to succeed"); + let root_path = SystemPathBuf::from_path_buf(root_path) + .expect("Temp directory to be a valid UTF8 path"); + + db.use_os_system_with_temp_dir(root_path, dir); + } + } + + let project_root = SystemPathBuf::from("/src"); + db.create_directory_all(&project_root) + .expect("Creating the project root to succeed"); + + let src_path = project_root.clone(); + let custom_typeshed_path = test.configuration().typeshed(); let mut typeshed_files = vec![]; let mut has_custom_versions_file = false; @@ -124,7 +142,7 @@ fn run_test( let full_path = embedded.full_path(&project_root); - if let Some(ref typeshed_path) = custom_typeshed_path { + if let Some(typeshed_path) = custom_typeshed_path { if let Ok(relative_path) = full_path.strip_prefix(typeshed_path.join("stdlib")) { if relative_path.as_str() == "VERSIONS" { has_custom_versions_file = true; @@ -151,7 +169,7 @@ fn run_test( .collect(); // Create a custom typeshed `VERSIONS` file if none was provided. - if let Some(ref typeshed_path) = custom_typeshed_path { + if let Some(typeshed_path) = custom_typeshed_path { if !has_custom_versions_file { let versions_file = typeshed_path.join("stdlib/VERSIONS"); let contents = typeshed_files @@ -170,25 +188,26 @@ fn run_test( } } - Program::get(db) - .update_from_settings( - db, - ProgramSettings { - python_version: test.configuration().python_version().unwrap_or_default(), - python_platform: test.configuration().python_platform().unwrap_or_default(), - search_paths: SearchPathSettings { - src_roots: vec![src_path], - extra_paths: test - .configuration() - .extra_paths() - .unwrap_or_default() - .to_vec(), - custom_typeshed: custom_typeshed_path, - python_path: PythonPath::KnownSitePackages(vec![]), - }, - }, - ) - .expect("Failed to update Program settings in TestDb"); + let settings = ProgramSettings { + python_version: test.configuration().python_version().unwrap_or_default(), + python_platform: test.configuration().python_platform().unwrap_or_default(), + search_paths: SearchPathSettings { + src_roots: vec![src_path], + extra_paths: test + .configuration() + .extra_paths() + .unwrap_or_default() + .to_vec(), + custom_typeshed: custom_typeshed_path.map(SystemPath::to_path_buf), + python_path: PythonPath::KnownSitePackages(vec![]), + }, + }; + + match Program::try_get(db) { + Some(program) => program.update_from_settings(db, settings), + None => Program::from_settings(db, settings).map(|_| ()), + } + .expect("Failed to update Program settings in TestDb"); // When snapshot testing is enabled, this is populated with // all diagnostics. Otherwise it remains empty. diff --git a/crates/red_knot_test/src/matcher.rs b/crates/red_knot_test/src/matcher.rs index acaf75ead644a..c38efefc4e792 100644 --- a/crates/red_knot_test/src/matcher.rs +++ b/crates/red_knot_test/src/matcher.rs @@ -349,7 +349,7 @@ mod tests { use super::FailuresByLine; use ruff_db::diagnostic::{DiagnosticId, OldDiagnosticTrait, Severity, Span}; use ruff_db::files::{system_path_to_file, File}; - use ruff_db::system::{DbWithTestSystem, SystemPathBuf}; + use ruff_db::system::DbWithWritableSystem as _; use ruff_python_trivia::textwrap::dedent; use ruff_source_file::OneIndexed; use ruff_text_size::TextRange; @@ -413,7 +413,7 @@ mod tests { ) -> Result<(), FailuresByLine> { colored::control::set_override(false); - let mut db = crate::db::Db::setup(SystemPathBuf::from("/src")); + let mut db = crate::db::Db::setup(); db.write_file("/src/test.py", source).unwrap(); let file = system_path_to_file(&db, "/src/test.py").unwrap(); diff --git a/crates/red_knot_wasm/src/lib.rs b/crates/red_knot_wasm/src/lib.rs index 3112d3d718629..3edac448e4910 100644 --- a/crates/red_knot_wasm/src/lib.rs +++ b/crates/red_knot_wasm/src/lib.rs @@ -64,7 +64,7 @@ impl Workspace { pub fn open_file(&mut self, path: &str, contents: &str) -> Result { self.system .fs - .write_file(path, contents) + .write_file_all(path, contents) .map_err(into_error)?; let file = system_path_to_file(&self.db, path).expect("File to exist"); diff --git a/crates/ruff_benchmark/benches/red_knot.rs b/crates/ruff_benchmark/benches/red_knot.rs index 06d77c827cc93..4e45a0dcb6f55 100644 --- a/crates/ruff_benchmark/benches/red_knot.rs +++ b/crates/ruff_benchmark/benches/red_knot.rs @@ -106,7 +106,7 @@ fn setup_case() -> Case { let system = TestSystem::default(); let fs = system.memory_file_system().clone(); - fs.write_files( + fs.write_files_all( TOMLLIB_FILES .iter() .map(|file| (tomllib_path(file), file.code().to_string())), @@ -173,7 +173,7 @@ fn benchmark_incremental(criterion: &mut Criterion) { assert_diagnostics(&case.db, &result); case.fs - .write_file( + .write_file_all( &case.re_path, format!("{}\n# A comment\n", source_text(&case.db, case.re).as_str()), ) diff --git a/crates/ruff_db/src/files.rs b/crates/ruff_db/src/files.rs index d4354957aa523..46f23fe2e4ff6 100644 --- a/crates/ruff_db/src/files.rs +++ b/crates/ruff_db/src/files.rs @@ -496,7 +496,7 @@ impl std::error::Error for FileError {} mod tests { use crate::file_revision::FileRevision; use crate::files::{system_path_to_file, vendored_path_to_file, FileError}; - use crate::system::DbWithTestSystem; + use crate::system::DbWithWritableSystem as _; use crate::tests::TestDb; use crate::vendored::VendoredFileSystemBuilder; use zip::CompressionMethod; diff --git a/crates/ruff_db/src/parsed.rs b/crates/ruff_db/src/parsed.rs index 9fcfe1d5b70fa..4f0971f5b7a8a 100644 --- a/crates/ruff_db/src/parsed.rs +++ b/crates/ruff_db/src/parsed.rs @@ -85,7 +85,9 @@ impl Eq for ParsedModule {} mod tests { use crate::files::{system_path_to_file, vendored_path_to_file}; use crate::parsed::parsed_module; - use crate::system::{DbWithTestSystem, SystemPath, SystemVirtualPath}; + use crate::system::{ + DbWithTestSystem, DbWithWritableSystem as _, SystemPath, SystemVirtualPath, + }; use crate::tests::TestDb; use crate::vendored::{VendoredFileSystemBuilder, VendoredPath}; use crate::Db; @@ -96,7 +98,7 @@ mod tests { let mut db = TestDb::new(); let path = "test.py"; - db.write_file(path, "x = 10".to_string())?; + db.write_file(path, "x = 10")?; let file = system_path_to_file(&db, path).unwrap(); @@ -112,7 +114,7 @@ mod tests { let mut db = TestDb::new(); let path = SystemPath::new("test.ipynb"); - db.write_file(path, "%timeit a = b".to_string())?; + db.write_file(path, "%timeit a = b")?; let file = system_path_to_file(&db, path).unwrap(); diff --git a/crates/ruff_db/src/source.rs b/crates/ruff_db/src/source.rs index 115b274c7a991..a50cf5ed62793 100644 --- a/crates/ruff_db/src/source.rs +++ b/crates/ruff_db/src/source.rs @@ -176,7 +176,7 @@ mod tests { use crate::files::system_path_to_file; use crate::source::{line_index, source_text}; - use crate::system::{DbWithTestSystem, SystemPath}; + use crate::system::{DbWithWritableSystem as _, SystemPath}; use crate::tests::TestDb; #[test] @@ -184,13 +184,13 @@ mod tests { let mut db = TestDb::new(); let path = SystemPath::new("test.py"); - db.write_file(path, "x = 10".to_string())?; + db.write_file(path, "x = 10")?; let file = system_path_to_file(&db, path).unwrap(); assert_eq!(source_text(&db, file).as_str(), "x = 10"); - db.write_file(path, "x = 20".to_string()).unwrap(); + db.write_file(path, "x = 20").unwrap(); assert_eq!(source_text(&db, file).as_str(), "x = 20"); @@ -202,7 +202,7 @@ mod tests { let mut db = TestDb::new(); let path = SystemPath::new("test.py"); - db.write_file(path, "x = 10".to_string())?; + db.write_file(path, "x = 10")?; let file = system_path_to_file(&db, path).unwrap(); @@ -228,7 +228,7 @@ mod tests { let mut db = TestDb::new(); let path = SystemPath::new("test.py"); - db.write_file(path, "x = 10\ny = 20".to_string())?; + db.write_file(path, "x = 10\ny = 20")?; let file = system_path_to_file(&db, path).unwrap(); let index = line_index(&db, file); diff --git a/crates/ruff_db/src/system.rs b/crates/ruff_db/src/system.rs index 14946e8e59f76..6d9ed9c42b7ca 100644 --- a/crates/ruff_db/src/system.rs +++ b/crates/ruff_db/src/system.rs @@ -12,7 +12,7 @@ use std::error::Error; use std::fmt::Debug; use std::path::{Path, PathBuf}; use std::{fmt, io}; -pub use test::{DbWithTestSystem, InMemorySystem, TestSystem}; +pub use test::{DbWithTestSystem, DbWithWritableSystem, InMemorySystem, TestSystem}; use walk_directory::WalkDirectoryBuilder; use crate::file_revision::FileRevision; @@ -161,6 +161,15 @@ pub trait System: Debug { fn as_any_mut(&mut self) -> &mut dyn std::any::Any; } +/// System trait for non-readonly systems. +pub trait WritableSystem: System { + /// Writes the given content to the file at the given path. + fn write_file(&self, path: &SystemPath, content: &str) -> Result<()>; + + /// Creates a directory at `path` as well as any intermediate directories. + fn create_directory_all(&self, path: &SystemPath) -> Result<()>; +} + #[derive(Clone, Debug, Eq, PartialEq)] pub struct Metadata { revision: FileRevision, diff --git a/crates/ruff_db/src/system/memory_fs.rs b/crates/ruff_db/src/system/memory_fs.rs index 0c70c09c486e6..44d1eafc95d29 100644 --- a/crates/ruff_db/src/system/memory_fs.rs +++ b/crates/ruff_db/src/system/memory_fs.rs @@ -153,18 +153,33 @@ impl MemoryFileSystem { virtual_files.contains_key(&path.to_path_buf()) } + /// Stores a new file in the file system. + /// + /// The operation overrides the content for an existing file with the same normalized `path`. + pub fn write_file(&self, path: impl AsRef, content: impl ToString) -> Result<()> { + let mut by_path = self.inner.by_path.write().unwrap(); + + let normalized = self.normalize_path(path.as_ref()); + + let file = get_or_create_file(&mut by_path, &normalized)?; + file.content = content.to_string(); + file.last_modified = now(); + + Ok(()) + } + /// Writes the files to the file system. /// /// The operation overrides existing files with the same normalized path. /// /// Enclosing directories are automatically created if they don't exist. - pub fn write_files(&self, files: impl IntoIterator) -> Result<()> + pub fn write_files_all(&self, files: impl IntoIterator) -> Result<()> where P: AsRef, C: ToString, { for (path, content) in files { - self.write_file(path.as_ref(), content.to_string())?; + self.write_file_all(path.as_ref(), content.to_string())?; } Ok(()) @@ -175,16 +190,18 @@ impl MemoryFileSystem { /// The operation overrides the content for an existing file with the same normalized `path`. /// /// Enclosing directories are automatically created if they don't exist. - pub fn write_file(&self, path: impl AsRef, content: impl ToString) -> Result<()> { - let mut by_path = self.inner.by_path.write().unwrap(); - - let normalized = self.normalize_path(path.as_ref()); + pub fn write_file_all( + &self, + path: impl AsRef, + content: impl ToString, + ) -> Result<()> { + let path = path.as_ref(); - let file = get_or_create_file(&mut by_path, &normalized)?; - file.content = content.to_string(); - file.last_modified = now(); + if let Some(parent) = path.parent() { + self.create_directory_all(parent)?; + } - Ok(()) + self.write_file(path, content) } /// Stores a new virtual file in the file system. @@ -486,7 +503,11 @@ fn get_or_create_file<'a>( normalized: &Utf8Path, ) -> Result<&'a mut File> { if let Some(parent) = normalized.parent() { - create_dir_all(paths, parent)?; + let parent_entry = paths.get(parent).ok_or_else(not_found)?; + + if parent_entry.is_file() { + return Err(not_a_directory()); + } } let entry = paths.entry(normalized.to_path_buf()).or_insert_with(|| { @@ -719,7 +740,7 @@ mod tests { P: AsRef, { let fs = MemoryFileSystem::new(); - fs.write_files(files.into_iter().map(|path| (path, ""))) + fs.write_files_all(files.into_iter().map(|path| (path, ""))) .unwrap(); fs @@ -822,29 +843,25 @@ mod tests { } #[test] - fn write_file_fails_if_a_component_is_a_file() { - let fs = with_files(["a/b.py"]); + fn write_file_fails_if_a_parent_directory_is_missing() { + let fs = with_files(["c.py"]); let error = fs - .write_file(SystemPath::new("a/b.py/c"), "content".to_string()) + .write_file(SystemPath::new("a/b.py"), "content".to_string()) .unwrap_err(); - assert_eq!(error.kind(), ErrorKind::Other); + assert_eq!(error.kind(), ErrorKind::NotFound); } #[test] - fn write_file_fails_if_path_points_to_a_directory() -> Result<()> { - let fs = MemoryFileSystem::new(); - - fs.create_directory_all("a")?; + fn write_file_all_fails_if_a_component_is_a_file() { + let fs = with_files(["a/b.py"]); let error = fs - .write_file(SystemPath::new("a"), "content".to_string()) + .write_file_all(SystemPath::new("a/b.py/c"), "content".to_string()) .unwrap_err(); assert_eq!(error.kind(), ErrorKind::Other); - - Ok(()) } #[test] @@ -864,7 +881,7 @@ mod tests { let fs = MemoryFileSystem::new(); let path = SystemPath::new("a.py"); - fs.write_file(path, "Test content".to_string())?; + fs.write_file_all(path, "Test content".to_string())?; assert_eq!(fs.read_to_string(path)?, "Test content"); @@ -895,6 +912,21 @@ mod tests { Ok(()) } + #[test] + fn write_file_fails_if_path_points_to_a_directory() -> Result<()> { + let fs = MemoryFileSystem::new(); + + fs.create_directory_all("a")?; + + let error = fs + .write_file(SystemPath::new("a"), "content".to_string()) + .unwrap_err(); + + assert_eq!(error.kind(), ErrorKind::Other); + + Ok(()) + } + #[test] fn read_fails_if_virtual_path_doesnt_exit() { let fs = MemoryFileSystem::new(); @@ -1046,7 +1078,7 @@ mod tests { let root = SystemPath::new("/src"); let system = MemoryFileSystem::with_current_directory(root); - system.write_files([ + system.write_files_all([ (root.join("foo.py"), "print('foo')"), (root.join("a/bar.py"), "print('bar')"), (root.join("a/baz.py"), "print('baz')"), @@ -1105,7 +1137,7 @@ mod tests { let root = SystemPath::new("/src"); let system = MemoryFileSystem::with_current_directory(root); - system.write_files([ + system.write_files_all([ (root.join("foo.py"), "print('foo')"), (root.join("a/bar.py"), "print('bar')"), (root.join("a/.baz.py"), "print('baz')"), @@ -1151,7 +1183,7 @@ mod tests { let root = SystemPath::new("/src"); let system = MemoryFileSystem::with_current_directory(root); - system.write_file(root.join("foo.py"), "print('foo')")?; + system.write_file_all(root.join("foo.py"), "print('foo')")?; let writer = DirectoryEntryToString::new(root.to_path_buf()); @@ -1181,7 +1213,7 @@ mod tests { let root = SystemPath::new("/src"); let fs = MemoryFileSystem::with_current_directory(root); - fs.write_files([ + fs.write_files_all([ (root.join("foo.py"), "print('foo')"), (root.join("a/bar.py"), "print('bar')"), (root.join("a/.baz.py"), "print('baz')"), diff --git a/crates/ruff_db/src/system/os.rs b/crates/ruff_db/src/system/os.rs index d1703f0952e29..95a5fe9bbcd6e 100644 --- a/crates/ruff_db/src/system/os.rs +++ b/crates/ruff_db/src/system/os.rs @@ -7,7 +7,7 @@ use ruff_notebook::{Notebook, NotebookError}; use crate::system::{ DirectoryEntry, FileType, GlobError, GlobErrorKind, Metadata, Result, System, SystemPath, - SystemPathBuf, SystemVirtualPath, + SystemPathBuf, SystemVirtualPath, WritableSystem, }; use super::walk_directory::{ @@ -191,6 +191,16 @@ impl System for OsSystem { } } +impl WritableSystem for OsSystem { + fn write_file(&self, path: &SystemPath, content: &str) -> Result<()> { + std::fs::write(path.as_std_path(), content) + } + + fn create_directory_all(&self, path: &SystemPath) -> Result<()> { + std::fs::create_dir_all(path.as_std_path()) + } +} + #[derive(Debug)] struct OsDirectoryWalker; diff --git a/crates/ruff_db/src/system/test.rs b/crates/ruff_db/src/system/test.rs index fa199fb97ea0f..c171420f1d2a4 100644 --- a/crates/ruff_db/src/system/test.rs +++ b/crates/ruff_db/src/system/test.rs @@ -1,6 +1,5 @@ use glob::PatternError; use ruff_notebook::{Notebook, NotebookError}; -use ruff_python_trivia::textwrap; use std::panic::RefUnwindSafe; use std::sync::{Arc, Mutex}; @@ -12,6 +11,7 @@ use crate::system::{ use crate::Db; use super::walk_directory::WalkDirectoryBuilder; +use super::WritableSystem; /// System implementation intended for testing. /// @@ -22,10 +22,16 @@ use super::walk_directory::WalkDirectoryBuilder; /// Don't use this system for production code. It's intended for testing only. #[derive(Debug, Clone)] pub struct TestSystem { - inner: Arc, + inner: Arc, } impl TestSystem { + pub fn new(inner: impl WritableSystem + RefUnwindSafe + Send + Sync + 'static) -> Self { + Self { + inner: Arc::new(inner), + } + } + /// Returns the [`InMemorySystem`]. /// /// ## Panics @@ -50,12 +56,12 @@ impl TestSystem { fn use_system(&mut self, system: S) where - S: System + Send + Sync + RefUnwindSafe + 'static, + S: WritableSystem + Send + Sync + RefUnwindSafe + 'static, { self.inner = Arc::new(system); } - pub fn system(&self) -> &dyn System { + pub fn system(&self) -> &dyn WritableSystem { &*self.inner } } @@ -134,72 +140,64 @@ impl Default for TestSystem { } } -/// Extension trait for databases that use [`TestSystem`]. +impl WritableSystem for TestSystem { + fn write_file(&self, path: &SystemPath, content: &str) -> Result<()> { + self.system().write_file(path, content) + } + + fn create_directory_all(&self, path: &SystemPath) -> Result<()> { + self.system().create_directory_all(path) + } +} + +/// Extension trait for databases that use a [`WritableSystem`]. /// /// Provides various helper function that ease testing. -pub trait DbWithTestSystem: Db + Sized { - fn test_system(&self) -> &TestSystem; +pub trait DbWithWritableSystem: Db + Sized { + type System: WritableSystem; - fn test_system_mut(&mut self) -> &mut TestSystem; + fn writable_system(&self) -> &Self::System; /// Writes the content of the given file and notifies the Db about the change. - /// - /// ## Panics - /// If the db isn't using the [`InMemorySystem`]. - fn write_file(&mut self, path: impl AsRef, content: impl ToString) -> Result<()> { + fn write_file(&mut self, path: impl AsRef, content: impl AsRef) -> Result<()> { let path = path.as_ref(); + match self.writable_system().write_file(path, content.as_ref()) { + Ok(()) => { + File::sync_path(self, path); + Ok(()) + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + if let Some(parent) = path.parent() { + self.writable_system().create_directory_all(parent)?; - let memory_fs = self.test_system().memory_file_system(); - - let sync_ancestors = path - .parent() - .is_some_and(|parent| !memory_fs.exists(parent)); - let result = memory_fs.write_file(path, content); + for ancestor in parent.ancestors() { + File::sync_path(self, ancestor); + } - if result.is_ok() { - File::sync_path(self, path); + self.writable_system().write_file(path, content.as_ref())?; + File::sync_path(self, path); - // Sync the ancestor paths if the path's parent - // directory didn't exist before. - if sync_ancestors { - for ancestor in path.ancestors() { - File::sync_path(self, ancestor); + Ok(()) + } else { + Err(error) } } + err => err, } - - result - } - - /// Writes the content of the given virtual file. - /// - /// ## Panics - /// If the db isn't using the [`InMemorySystem`]. - fn write_virtual_file(&mut self, path: impl AsRef, content: impl ToString) { - let path = path.as_ref(); - self.test_system() - .memory_file_system() - .write_virtual_file(path, content); } /// Writes auto-dedented text to a file. - /// - /// ## Panics - /// If the db isn't using the [`InMemorySystem`]. - fn write_dedented(&mut self, path: &str, content: &str) -> crate::system::Result<()> { - self.write_file(path, textwrap::dedent(content))?; + fn write_dedented(&mut self, path: &str, content: &str) -> Result<()> { + self.write_file(path, ruff_python_trivia::textwrap::dedent(content))?; Ok(()) } /// Writes the content of the given files and notifies the Db about the change. - /// - /// ## Panics - /// If the db isn't using the [`InMemorySystem`]. - fn write_files(&mut self, files: I) -> crate::system::Result<()> + fn write_files(&mut self, files: I) -> Result<()> where I: IntoIterator, P: AsRef, - C: ToString, + C: AsRef, { for (path, content) in files { self.write_file(path, content)?; @@ -207,6 +205,26 @@ pub trait DbWithTestSystem: Db + Sized { Ok(()) } +} + +/// Extension trait for databases that use [`TestSystem`]. +/// +/// Provides various helper function that ease testing. +pub trait DbWithTestSystem: Db + Sized { + fn test_system(&self) -> &TestSystem; + + fn test_system_mut(&mut self) -> &mut TestSystem; + + /// Writes the content of the given virtual file. + /// + /// ## Panics + /// If the db isn't using the [`InMemorySystem`]. + fn write_virtual_file(&mut self, path: impl AsRef, content: impl ToString) { + let path = path.as_ref(); + self.test_system() + .memory_file_system() + .write_virtual_file(path, content); + } /// Uses the given system instead of the testing system. /// @@ -215,7 +233,7 @@ pub trait DbWithTestSystem: Db + Sized { /// Note that any files written to the memory file system won't be copied over. fn use_system(&mut self, os: S) where - S: System + Send + Sync + RefUnwindSafe + 'static, + S: WritableSystem + Send + Sync + RefUnwindSafe + 'static, { self.test_system_mut().use_system(os); } @@ -229,6 +247,17 @@ pub trait DbWithTestSystem: Db + Sized { } } +impl DbWithWritableSystem for T +where + T: DbWithTestSystem, +{ + type System = TestSystem; + + fn writable_system(&self) -> &Self::System { + self.test_system() + } +} + #[derive(Default, Debug)] pub struct InMemorySystem { user_config_directory: Mutex>, @@ -236,6 +265,13 @@ pub struct InMemorySystem { } impl InMemorySystem { + pub fn new(cwd: SystemPathBuf) -> Self { + Self { + user_config_directory: Mutex::new(None), + memory_fs: MemoryFileSystem::with_current_directory(cwd), + } + } + pub fn fs(&self) -> &MemoryFileSystem { &self.memory_fs } @@ -314,3 +350,13 @@ impl System for InMemorySystem { self } } + +impl WritableSystem for InMemorySystem { + fn write_file(&self, path: &SystemPath, content: &str) -> Result<()> { + self.memory_fs.write_file(path, content) + } + + fn create_directory_all(&self, path: &SystemPath) -> Result<()> { + self.memory_fs.create_directory_all(path) + } +} diff --git a/fuzz/fuzz_targets/red_knot_check_invalid_syntax.rs b/fuzz/fuzz_targets/red_knot_check_invalid_syntax.rs index b98aeb0bb2583..1da7da48dd3c0 100644 --- a/fuzz/fuzz_targets/red_knot_check_invalid_syntax.rs +++ b/fuzz/fuzz_targets/red_knot_check_invalid_syntax.rs @@ -14,7 +14,9 @@ use red_knot_python_semantic::{ PythonPlatform, SearchPathSettings, }; use ruff_db::files::{system_path_to_file, File, Files}; -use ruff_db::system::{DbWithTestSystem, System, SystemPathBuf, TestSystem}; +use ruff_db::system::{ + DbWithTestSystem, DbWithWritableSystem as _, System, SystemPathBuf, TestSystem, +}; use ruff_db::vendored::VendoredFileSystem; use ruff_db::{Db as SourceDb, Upcast}; use ruff_python_ast::PythonVersion;