Skip to content

Minor: Improve error handling in sqllogictest runner #8544

New issue

Have a question about this project? # for a free GitHub account to open an issue and contact its maintainers and the community.

#

By clicking “#”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? # to your account

Merged
merged 1 commit into from
Dec 14, 2023
Merged

Minor: Improve error handling in sqllogictest runner #8544

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
53 changes: 34 additions & 19 deletions datafusion/sqllogictest/bin/sqllogictests.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ use futures::stream::StreamExt;
use log::info;
use sqllogictest::strict_column_validator;

use datafusion_common::{exec_err, DataFusionError, Result};
use datafusion_common::{exec_datafusion_err, exec_err, DataFusionError, Result};

const TEST_DIRECTORY: &str = "test_files/";
const PG_COMPAT_FILE_PREFIX: &str = "pg_compat_";
Expand Down Expand Up @@ -84,7 +84,7 @@ async fn run_tests() -> Result<()> {
// Doing so is safe because each slt file runs with its own
// `SessionContext` and should not have side effects (like
// modifying shared state like `/tmp/`)
let errors: Vec<_> = futures::stream::iter(read_test_files(&options))
let errors: Vec<_> = futures::stream::iter(read_test_files(&options)?)
.map(|test_file| {
tokio::task::spawn(async move {
println!("Running {:?}", test_file.relative_path);
Expand Down Expand Up @@ -247,30 +247,45 @@ impl TestFile {
}
}

fn read_test_files<'a>(options: &'a Options) -> Box<dyn Iterator<Item = TestFile> + 'a> {
Box::new(
read_dir_recursive(TEST_DIRECTORY)
fn read_test_files<'a>(
options: &'a Options,
) -> Result<Box<dyn Iterator<Item = TestFile> + 'a>> {
Ok(Box::new(
read_dir_recursive(TEST_DIRECTORY)?
.into_iter()
.map(TestFile::new)
.filter(|f| options.check_test_file(&f.relative_path))
.filter(|f| f.is_slt_file())
.filter(|f| f.check_tpch(options))
.filter(|f| options.check_pg_compat_file(f.path.as_path())),
)
))
}

fn read_dir_recursive<P: AsRef<Path>>(path: P) -> Box<dyn Iterator<Item = PathBuf>> {
Box::new(
std::fs::read_dir(path)
.expect("Readable directory")
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The point of this PR is to remove the expect in this line and the line below

While I am sure we could retain the iterator nature of the previous implementation, given we are talking about less than 30 files I think buffering them in a Vec is perfectly acceptable and makes the error handling easier

.map(|path| path.expect("Readable entry").path())
.flat_map(|path| {
if path.is_dir() {
read_dir_recursive(path)
} else {
Box::new(std::iter::once(path))
}
}),
)
fn read_dir_recursive<P: AsRef<Path>>(path: P) -> Result<Vec<PathBuf>> {
let mut dst = vec![];
read_dir_recursive_impl(&mut dst, path.as_ref())?;
Ok(dst)
}

/// Append all paths recursively to dst
fn read_dir_recursive_impl(dst: &mut Vec<PathBuf>, path: &Path) -> Result<()> {
let entries = std::fs::read_dir(path)
.map_err(|e| exec_datafusion_err!("Error reading directory {path:?}: {e}"))?;
for entry in entries {
let path = entry
.map_err(|e| {
exec_datafusion_err!("Error reading entry in directory {path:?}: {e}")
})?
.path();

if path.is_dir() {
read_dir_recursive_impl(dst, &path)?;
} else {
dst.push(path);
}
}

Ok(())
}

/// Parsed command line options
Expand Down