chore: clean up runtime code for python setup

README_processing_engine.md

@@ -154,25 +154,32 @@ $ tar -C /here --strip-components=1 -zxvf /path/to/build/influxdb3-<VERSION>_lin
 $ /here/influxdb3 serve ...
 $ /here/influxdb3 query ...
 
-# with the processing engine without an activated venv
+# with processing engine (/path/to/plugins/.venv created automatically)
 $ mkdir /path/to/plugins
 $ /here/influxdb3 serve --plugin-dir /path/to/plugins ...        # server
 $ /here/influxdb3 create database foo                            # client
 $ /here/influxdb3 test schedule_plugin -d foo testme.py          # client
+... <plugins can use whatever is in /path/to/plugins/.venv> ...
+$ /here/influxdb3 install package bar                            # client
+... <plugins can now 'import bar'> ...
+$ /here/influxdb3 test schedule_plugin -d foo testme.py          # client
 
-# create a venv
-$ /here/python/bin/python3 -m venv /path/to/venv
-$ source /path/to/venv/bin/activate
-(venv)$ pip install requests
+# with processsing engine and alternate venv
+$ /here/python/bin/python3 -m venv /path/to/other-venv
+$ source /path/to/other-venv/bin/activate
+(venv)$ python3 -m pip install foo
+(venv)$ python3 -m pip freeze > /path/to/other-venv/requirements.txt
 ...
 (venv)$ deactivate
 
 # start server in the venv
-$ source /path/to/venv/bin/activate                              # server
-(venv)$ /here/influxdb3 serve --plugin-dir /path/to/plugins ...  # server
-... <plugins can now 'import requests'> ...
+$ /here/influxdb3 serve --plugin-dir /path/to/plugins --virtual-env-location /path/to/other-venv
+... <plugins can now use whatever is in /path/to/other-venv> ...
 
-$ /here/influxdb3 test schedule_plugin -d foo test-requests.py   # client
+$ /here/influxdb3 test schedule_plugin -d foo testme.py          # client
+$ /here/influxdb3 install package bar                            # client
+... <plugins can now 'import bar' in /path/to/other-venv> ...
+$ /here/influxdb3 test schedule_plugin -d foo testme.py          # client
 ```
 
 ### Local development with python-build-standalone
@@ -290,8 +297,14 @@ the `PBS_DATE` and `PBS_VERSION` environment variables in
 astral-sh creates new builds for CPython minor releases as they become
 available from upstream Python. Updating the official builds to pull in a new
 minor release is straightforward, but processes for verifying builds of
-InfluxDB with the new `python-build-standalone` minor release are TBD.
+InfluxDB with the new `python-build-standalone` minor release are
+[TBD](https://github.com/influxdata/influxdb/issues/26045).
 
+References:
+ * https://www.python.org/dev/security/
+ * https://mail.python.org/mailman3/lists/security-announce.python.org/
+ * https://mail.python.org/archives/list/security-announce@python.org/latest
+ * https://github.com/astral-sh/python-build-standalone/releases
 
 ### How is python-build-standalone licensed?
 
@@ -436,7 +449,8 @@ to use the InfluxDB processing engine. This is achieved by:
 
 ### There is no `pip.exe` on Windows. Why?
 
-From [upstream](https://github.com/astral-sh/python-build-standalone/blob/main/docs/quirks.rst#no-pipexe-on-windows):
+Historical `python-build-standalone` didn't ship `pip.exe` on Windows. From
+[upstream](https://github.com/astral-sh/python-build-standalone/blob/main/docs/quirks.rst#no-pipexe-on-windows):
 "The Windows distributions have pip installed however no `Scripts/pip.exe`,
 `Scripts/pip3.exe`, and `Scripts/pipX.Y.exe` files are provided because the way
 these executables are built isn't portable. (It might be possible to change how
@@ -446,6 +460,8 @@ To use pip, run `python.exe -m pip`. (It is generally a best practice to invoke
 pip via `python -m pip` on all platforms so you can be explicit about the
 python executable that pip uses.)"
 
+While newer `python-build-standalone` releases seem to have addressed this, the
+recommendation to call with `python -m pip` still holds true.
 
 ### What limitations are there?
 

influxdb3/src/commands/serve.rs

@@ -23,6 +23,8 @@ use influxdb3_processing_engine::environment::{
     DisabledManager, PipManager, PythonEnvironmentManager, UVManager,
 };
 use influxdb3_processing_engine::plugins::ProcessingEngineEnvironmentManager;
+#[cfg(feature = "system-py")]
+use influxdb3_processing_engine::virtualenv::find_python;
 use influxdb3_server::{
     CommonServerState,
     auth::AllOrNothingAuthorizer,
@@ -49,10 +51,7 @@ use panic_logging::SendPanicsToTracing;
 use parquet_file::storage::{ParquetStorage, StorageId};
 use std::process::Command;
 use std::{env, num::NonZeroUsize, sync::Arc, time::Duration};
-use std::{
-    path::{Path, PathBuf},
-    str::FromStr,
-};
+use std::{path::Path, str::FromStr};
 use thiserror::Error;
 use tokio::net::TcpListener;
 use tokio::time::Instant;
@@ -665,31 +664,18 @@ pub(crate) fn setup_processing_engine_env_manager(
 
 fn determine_package_manager() -> Arc<dyn PythonEnvironmentManager> {
     // Check for pip (highest preference)
-    // XXX: put this somewhere common
-    let python_exe_bn = if cfg!(windows) {
-        "python.exe"
-    } else {
-        "python3"
-    };
-    let python_exe = if let Ok(v) = env::var("PYTHONHOME") {
-        // honor PYTHONHOME (set earlier for python standalone). python build
-        // standalone has bin/python3 on OSX/Linux and python.exe on Windows
-        let mut path = PathBuf::from(v);
-        if !cfg!(windows) {
-            path.push("bin");
-        }
-        path.push(python_exe_bn);
-        path
-    } else {
-        PathBuf::from(python_exe_bn)
-    };
-
-    if let Ok(output) = Command::new(python_exe)
-        .args(["-m", "pip", "--version"])
-        .output()
+    #[cfg(feature = "system-py")]
     {
-        if output.status.success() {
-            return Arc::new(PipManager);
+        let python_exe = find_python();
+        debug!("Running: {} -m pip --version", python_exe.display());
+
+        if let Ok(output) = Command::new(&python_exe)
+            .args(["-m", "pip", "--version"])
+            .output()
+        {
+            if output.status.success() {
+                return Arc::new(PipManager);
+            }
         }
     }
 

influxdb3/src/main.rs

@@ -14,11 +14,6 @@ use dotenvy::dotenv;
 use influxdb3_clap_blocks::tokio::TokioIoConfig;
 use influxdb3_process::VERSION_STRING;
 use observability_deps::tracing::warn;
-#[cfg(feature = "system-py")]
-use std::{
-    env,
-    path::{Path, PathBuf},
-};
 use trogging::{
     TroggingGuard,
     cli::LoggingConfigBuilderExt,
@@ -125,9 +120,6 @@ fn main() -> Result<(), std::io::Error> {
     #[cfg(unix)]
     install_crash_handler(); // attempt to render a useful stacktrace to stderr
 
-    #[cfg(feature = "system-py")]
-    set_pythonhome();
-
     // load all environment variables from .env before doing anything
     load_dotenv();
 
@@ -311,38 +303,3 @@ fn init_logs_and_tracing(
     let subscriber = Registry::default().with(layers);
     trogging::install_global(subscriber)
 }
-
-// XXX: this should be somewhere more appropriate
-#[cfg(feature = "system-py")]
-fn set_pythonhome() {
-    // This would ideally be detected by pyo3, but it isn't
-    match env::var("PYTHONHOME") {
-        Ok(_) => {}
-        Err(env::VarError::NotPresent) => {
-            let exe_path = env::current_exe().unwrap();
-            let exe_dir = exe_path.parent().unwrap();
-
-            let pythonhome: PathBuf = if cfg!(target_os = "linux")
-                && (exe_dir == Path::new("/usr/bin") || exe_dir == Path::new("/usr/local/bin"))
-            {
-                // Official Linux builds may be in /usr or /usr/local
-                // XXX: handle this for local build and install (eg DESTDIR)
-                let parent_dir = exe_dir.parent().unwrap();
-                parent_dir.join("lib/influxdb3/python")
-            } else {
-                exe_dir.join("python")
-            };
-
-            if pythonhome.is_dir() {
-                unsafe { env::set_var("PYTHONHOME", pythonhome.to_str().unwrap()) };
-                //println!("Set PYTHONHOME to '{}'", env::var("PYTHONHOME").unwrap());
-            } else {
-                // TODO: use logger
-                eprintln!("Could not find python installation. May need to set PYTHONHOME");
-            }
-        }
-        Err(e) => {
-            eprintln!("Failed to retrieve PYTHONHOME: {e}");
-        }
-    };
-}

influxdb3_processing_engine/src/environment.rs

@@ -1,7 +1,6 @@
 use crate::environment::PluginEnvironmentError::PluginEnvironmentDisabled;
 #[cfg(feature = "system-py")]
-use crate::virtualenv::{VenvError, initialize_venv};
-use std::env;
+use crate::virtualenv::{VenvError, find_python, initialize_venv};
 use std::fmt::Debug;
 use std::path::{Path, PathBuf};
 use std::process::Command;
@@ -87,36 +86,6 @@ impl PythonEnvironmentManager for UVManager {
     }
 }
 
-// XXX: put this somewhere common
-fn find_python() -> PathBuf {
-    let python_exe_bn = if cfg!(windows) {
-        "python.exe"
-    } else {
-        "python3"
-    };
-    if let Ok(v) = env::var("VIRTUAL_ENV") {
-        let mut path = PathBuf::from(v);
-        if cfg!(windows) {
-            path.push("Scripts");
-        } else {
-            path.push("bin");
-        }
-        path.push(python_exe_bn);
-        path
-    } else if let Ok(v) = env::var("PYTHONHOME") {
-        // honor PYTHONHOME (set earlier for python standalone). python build
-        // standalone has bin/python3 on OSX/Linx and python.exe on Windows
-        let mut path = PathBuf::from(v);
-        if !cfg!(windows) {
-            path.push("bin");
-        }
-        path.push(python_exe_bn);
-        path
-    } else {
-        PathBuf::from(python_exe_bn)
-    }
-}
-
 impl PythonEnvironmentManager for PipManager {
     fn init_pyenv(
         &self,
@@ -128,6 +97,7 @@ impl PythonEnvironmentManager for PipManager {
             None => &plugin_dir.join(".venv"),
         };
 
+        #[cfg(feature = "system-py")]
         if !is_valid_venv(venv_path) {
             let python_exe = find_python();
             Command::new(python_exe)
@@ -143,25 +113,31 @@ impl PythonEnvironmentManager for PipManager {
     }
 
     fn install_packages(&self, packages: Vec<String>) -> Result<(), PluginEnvironmentError> {
-        let python_exe = find_python();
-        Command::new(python_exe)
-            .arg("-m")
-            .arg("pip")
-            .arg("install")
-            .args(&packages)
-            .output()?;
+        #[cfg(feature = "system-py")]
+        {
+            let python_exe = find_python();
+            Command::new(python_exe)
+                .arg("-m")
+                .arg("pip")
+                .arg("install")
+                .args(&packages)
+                .output()?;
+        }
         Ok(())
     }
     fn install_requirements(
         &self,
         requirements_path: String,
     ) -> Result<(), PluginEnvironmentError> {
-        let python_exe = find_python();
-        Command::new(python_exe)
-            .arg("-m")
-            .arg("pip")
-            .args(["install", "-r", &requirements_path])
-            .output()?;
+        #[cfg(feature = "system-py")]
+        {
+            let python_exe = find_python();
+            Command::new(python_exe)
+                .arg("-m")
+                .arg("pip")
+                .args(["install", "-r", &requirements_path])
+                .output()?;
+        }
         Ok(())
     }
 }