From 9f60b673b8a41dbd6b6ee5e77c34e5d9b750fd7b Mon Sep 17 00:00:00 2001 From: Sergey Kvachonok Date: Wed, 23 Mar 2022 15:18:42 +0300 Subject: [PATCH] pyo3-build-config: Make `lib_dir` optional in `CrossCompileConfig` Change the `CrossCompileConfig` structure definition and make the public `lib_dir` field optional to support more flexible cross-compilation configuration in the future. FIXME: This change breaks the public `pyo3-build-config` crate API. Update the sysconfigdata handling functions to fall through when `lib_dir` is not set. WIP: Add `unwrap()` stubs to the main cross compile switch. --- pyo3-build-config/src/impl_.rs | 172 ++++++++++++++++++++++----------- 1 file changed, 117 insertions(+), 55 deletions(-) diff --git a/pyo3-build-config/src/impl_.rs b/pyo3-build-config/src/impl_.rs index 48b4464a698..20cc996d3d3 100644 --- a/pyo3-build-config/src/impl_.rs +++ b/pyo3-build-config/src/impl_.rs @@ -758,7 +758,7 @@ impl TargetInfo { #[derive(Debug, PartialEq)] pub struct CrossCompileConfig { /// The directory containing the Python library to link against. - pub lib_dir: PathBuf, + pub lib_dir: Option, /// The version of the Python library to link against. version: Option, @@ -768,18 +768,39 @@ pub struct CrossCompileConfig { } impl CrossCompileConfig { - fn from_env_vars(env_vars: CrossCompileEnvVars, target_info: TargetInfo) -> Result { - Ok(CrossCompileConfig { - lib_dir: env_vars - .pyo3_cross_lib_dir - .as_ref() - .ok_or( - "The PYO3_CROSS_LIB_DIR environment variable must be set when cross-compiling", - )? - .into(), - target_info, - version: env_vars.parse_version()?, - }) + /// Creates a new cross compile config struct from PyO3 environment variables + /// and the build environment when cross compilation mode is detected. + /// + /// Returns `None` when not cross compiling. + fn try_from_env_vars( + env_vars: CrossCompileEnvVars, + host: &str, + target_info: TargetInfo, + ) -> Result> { + let maybe_config = if env_vars.any() || target_info.is_cross_compiling_from(host) { + let lib_dir = env_vars.lib_dir_path()?; + let version = env_vars.parse_version()?; + + Some(CrossCompileConfig { + lib_dir, + target_info, + version, + }) + } else { + None + }; + + Ok(maybe_config) + } + + /// Converts `lib_dir` member field to an UTF-8 string. + /// + /// The conversion can not fail because `PYO3_CROSS_LIB_DIR` variable + /// is ensured to be valid UTF-8 bytes. + fn lib_dir_string(&self) -> Option { + self.lib_dir + .as_ref() + .map(|s| s.to_str().unwrap().to_owned()) } } @@ -796,6 +817,22 @@ impl CrossCompileEnvVars { || self.pyo3_cross_python_version.is_some() } + /// Converts the stored `PYO3_CROSS_LIB_DIR` variable value (if any) + /// into a `PathBuf` instance. + /// + /// Ensures that the path is a valid UTF-8 string. + fn lib_dir_path(&self) -> Result> { + let lib_dir = self.pyo3_cross_lib_dir.as_ref().map(PathBuf::from); + + if let Some(dir) = lib_dir.as_ref() { + if dir.to_str().is_none() { + bail!("PYO3_CROSS_LIB_DIR is not valid UTF-8"); + } + } + + Ok(lib_dir) + } + fn parse_version(&self) -> Result> { let version = self .pyo3_cross_python_version @@ -827,9 +864,9 @@ pub(crate) fn cross_compile_env_vars() -> CrossCompileEnvVars { /// This function relies on PyO3 cross-compiling environment variables: /// /// * `PYO3_CROSS`: If present, forces PyO3 to configure as a cross-compilation. -/// * `PYO3_CROSS_LIB_DIR`: Must be set to the directory containing the target's libpython DSO and -/// the associated `_sysconfigdata*.py` file for Unix-like targets, or the Python DLL import -/// libraries for the Windows target. +/// * `PYO3_CROSS_LIB_DIR`: If present, must be set to the directory containing +/// the target's libpython DSO and the associated `_sysconfigdata*.py` file for +/// Unix-like targets, or the Python DLL import libraries for the Windows target. /// * `PYO3_CROSS_PYTHON_VERSION`: Major and minor version (e.g. 3.9) of the target Python /// installation. This variable is only needed if PyO3 cannnot determine the version to target /// from `abi3-py3*` features, or if there are multiple versions of Python present in @@ -845,11 +882,20 @@ pub fn cross_compiling( let env_vars = cross_compile_env_vars(); let target_info = TargetInfo::from_triple(target_arch, target_vendor, target_os, None); - if !env_vars.any() && !target_info.is_cross_compiling_from(host) { - return Ok(None); - } + CrossCompileConfig::try_from_env_vars(env_vars, host, target_info) +} - CrossCompileConfig::from_env_vars(env_vars, target_info).map(Some) +/// Detect whether we are cross compiling from Cargo and `PYO3_CROSS_*` environment +/// variables and return an assembled `CrossCompileConfig` if so. +/// +/// This must be called from PyO3's build script, because it relies on environment +/// variables such as `CARGO_CFG_TARGET_OS` which aren't available at any other time. +pub fn cross_compiling_from_cargo_env() -> Result> { + let env_vars = cross_compile_env_vars(); + let host = cargo_env_var("HOST").ok_or("expected HOST env var")?; + let target_info = TargetInfo::from_cargo_env()?; + + CrossCompileConfig::try_from_env_vars(env_vars, &host, target_info) } #[allow(non_camel_case_types)] @@ -1065,13 +1111,22 @@ fn ends_with(entry: &DirEntry, pat: &str) -> bool { name.to_string_lossy().ends_with(pat) } -fn find_sysconfigdata(cross: &CrossCompileConfig) -> Result { +/// Finds the sysconfigdata file when the target Python library directory is set. +/// +/// Returns `None` if the library directory is not available, and a runtime error +/// when no or multiple sysconfigdata files are found. +fn find_sysconfigdata(cross: &CrossCompileConfig) -> Result> { let mut sysconfig_paths = find_all_sysconfigdata(cross); if sysconfig_paths.is_empty() { - bail!( - "Could not find either libpython.so or _sysconfigdata*.py in {}", - cross.lib_dir.display() - ); + if let Some(lib_dir) = cross.lib_dir.as_ref() { + bail!( + "Could not find either libpython.so or _sysconfigdata*.py in {}", + lib_dir.display() + ); + } else { + // Continue with the default configuration when PYO3_CROSS_LIB_DIR is not set. + return Ok(None); + } } else if sysconfig_paths.len() > 1 { let mut error_msg = String::from( "Detected multiple possible Python versions. Please set either the \ @@ -1085,7 +1140,7 @@ fn find_sysconfigdata(cross: &CrossCompileConfig) -> Result { bail!("{}\n", error_msg); } - Ok(sysconfig_paths.remove(0)) + Ok(Some(sysconfig_paths.remove(0))) } /// Finds `_sysconfigdata*.py` files for detected Python interpreters. @@ -1123,8 +1178,16 @@ fn find_sysconfigdata(cross: &CrossCompileConfig) -> Result { /// ``` /// /// [1]: https://github.com/python/cpython/blob/3.5/Lib/sysconfig.py#L389 +/// +/// Returns an empty vector when the target Python library directory +/// is not set via `PYO3_CROSS_LIB_DIR`. pub fn find_all_sysconfigdata(cross: &CrossCompileConfig) -> Vec { - let sysconfig_paths = search_lib_dir(&cross.lib_dir, cross); + let sysconfig_paths = if let Some(lib_dir) = cross.lib_dir.as_ref() { + search_lib_dir(lib_dir, cross) + } else { + return Vec::new(); + }; + let sysconfig_name = env_var("_PYTHON_SYSCONFIGDATA_NAME"); let mut sysconfig_paths = sysconfig_paths .iter() @@ -1224,11 +1287,19 @@ fn search_lib_dir(path: impl AsRef, cross: &CrossCompileConfig) -> Vec Result { - let sysconfigdata_path = find_sysconfigdata(&cross_compile_config)?; - InterpreterConfig::from_sysconfigdata(&parse_sysconfigdata(sysconfigdata_path)?) + cross_compile_config: &CrossCompileConfig, +) -> Result> { + if let Some(path) = find_sysconfigdata(cross_compile_config)? { + let data = parse_sysconfigdata(path)?; + let config = InterpreterConfig::from_sysconfigdata(&data)?; + + Ok(Some(config)) + } else { + Ok(None) + } } fn windows_hardcoded_cross_compile( @@ -1251,7 +1322,7 @@ fn windows_hardcoded_cross_compile( abi3, cross_compile_config.target_info.is_windows_mingw(), )), - lib_dir: cross_compile_config.lib_dir.to_str().map(String::from), + lib_dir: cross_compile_config.lib_dir_string(), executable: None, pointer_width: None, build_flags: BuildFlags::default(), @@ -1265,11 +1336,15 @@ fn load_cross_compile_config( ) -> Result { match cargo_env_var("CARGO_CFG_TARGET_FAMILY") { // Configure for unix platforms using the sysconfigdata file - Some(os) if os == "unix" => cross_compile_from_sysconfigdata(cross_compile_config), + Some(os) if os == "unix" => cross_compile_from_sysconfigdata(&cross_compile_config) + .transpose() + .unwrap(), // Use hardcoded interpreter config when targeting Windows Some(os) if os == "windows" => windows_hardcoded_cross_compile(cross_compile_config), // sysconfigdata works fine on wasm/wasi - Some(os) if os == "wasm" => cross_compile_from_sysconfigdata(cross_compile_config), + Some(os) if os == "wasm" => cross_compile_from_sysconfigdata(&cross_compile_config) + .transpose() + .unwrap(), // Waiting for users to tell us what they expect on their target platform Some(os) => bail!( "Unknown target OS family for cross-compilation: {:?}.\n\ @@ -1279,7 +1354,9 @@ fn load_cross_compile_config( os ), // Unknown os family - try to do something useful - None => cross_compile_from_sysconfigdata(cross_compile_config), + None => cross_compile_from_sysconfigdata(&cross_compile_config) + .transpose() + .unwrap(), } } @@ -1436,26 +1513,11 @@ pub fn find_interpreter() -> Result { /// This must be called from PyO3's build script, because it relies on environment variables such as /// CARGO_CFG_TARGET_OS which aren't available at any other time. pub fn make_cross_compile_config() -> Result> { - let env_vars = cross_compile_env_vars(); - - let host = cargo_env_var("HOST").ok_or("expected HOST env var")?; - let target = cargo_env_var("TARGET").ok_or("expected TARGET env var")?; - - let target_info = TargetInfo::from_cargo_env()?; - - let interpreter_config = if env_vars.any() { - let cross_config = CrossCompileConfig::from_env_vars(env_vars, target_info)?; + let interpreter_config = if let Some(cross_config) = cross_compiling_from_cargo_env()? { let mut interpreter_config = load_cross_compile_config(cross_config)?; interpreter_config.fixup_for_abi3_version(get_abi3_version())?; Some(interpreter_config) } else { - ensure!( - host == target || !target_info.is_cross_compiling_from(&host), - "PyO3 detected compile host {host} and build target {target}, but none of PYO3_CROSS, PYO3_CROSS_LIB_DIR \ - or PYO3_CROSS_PYTHON_VERSION environment variables are set.", - host=host, - target=target, - ); None }; @@ -1721,7 +1783,7 @@ mod tests { #[test] fn windows_hardcoded_cross_compile() { let cross_config = CrossCompileConfig { - lib_dir: "C:\\some\\path".into(), + lib_dir: Some("C:\\some\\path".into()), version: Some(PythonVersion { major: 3, minor: 7 }), target_info: TargetInfo::from_triple("x86", "pc", "windows", Some("msvc")), }; @@ -1934,15 +1996,15 @@ mod tests { }; let cross = CrossCompileConfig { - lib_dir: lib_dir.into(), + lib_dir: Some(lib_dir.into()), version: Some(interpreter_config.version), target_info: TargetInfo::from_triple("x86_64", "unknown", "linux", Some("gnu")), }; let sysconfigdata_path = match find_sysconfigdata(&cross) { - Ok(path) => path, + Ok(Some(path)) => path, // Couldn't find a matching sysconfigdata; never mind! - Err(_) => return, + _ => return, }; let sysconfigdata = super::parse_sysconfigdata(sysconfigdata_path).unwrap(); let parsed_config = InterpreterConfig::from_sysconfigdata(&sysconfigdata).unwrap();