OpenVINO support

BUILDING.md

@@ -57,3 +57,36 @@ brew install cmake
 ```
 
 CMake can also be installed from https://cmake.org/download/ but `cmake` binary needs to be in your PATH.
+
+# OpenVINO support
+
+## Development Tools
+OpenVINO support requires the OpenVINO Development Tools to be installed. You can find
+instructions for installing the OpenVINO Development Tools here:
+https://docs.openvino.ai/2023.0/openvino_docs_install_guides_install_dev_tools.html#for-c-developers
+
+On Arch Linux, you can install the OpenVINO Development Tools with the following command:
+```
+paru -S openvino
+```
+This build may take a significant amount of time, but can save massive headaches later on.
+
+## Building
+First, the `openvino` feature must be enabled in your Cargo.toml.
+
+Next, you must set the `OpenVINO_DIR` environment variable to the path where CMake can find
+`OpenVINOConfig.cmake`.
+This is usually in the `cmake` directory of the OpenVINO installation.
+
+If you used the AUR package to install OpenVINO, the location of this file is `/opt/intel/openvino/runtime/cmake`.
+
+```
+export OpenVINO_DIR=/opt/intel/openvino/runtime/cmake
+```
+
+Finally, you can build whisper-rs as normal.
+
+## Tested platforms
+- Arch Linux
+
+If you have successfully built whisper-rs with OpenVINO on another platform, please open a PR to document it here!

Cargo.toml

@@ -33,6 +33,7 @@ openblas = ["whisper-rs-sys/openblas"]
 metal = ["whisper-rs-sys/metal", "_gpu"]
 vulkan = ["whisper-rs-sys/vulkan", "_gpu"]
 _gpu = []
+openvino = ["whisper-rs-sys/openvino"]
 test-with-tiny-model = []
 whisper-cpp-log = ["dep:log"]
 whisper-cpp-tracing = ["dep:tracing"]

README.md

@@ -80,7 +80,9 @@ All disabled by default unless otherwise specified.
 
 ## Building
 
-See [BUILDING.md](BUILDING.md) for instructions for building whisper-rs on Windows and OSX M1. Linux builds should just
+See [BUILDING.md](BUILDING.md) for instructions for building whisper-rs on Windows and OSX M1,
+or with OpenVINO on any OS.
+Besides OpenVINO, Linux builds should just
 work out of the box.
 
 ## Troubleshooting

src/whisper_ctx.rs

@@ -70,6 +70,8 @@ impl WhisperInnerContext {
         }
     }
 
+    // we don't implement `whisper_init()` here since i have zero clue what `whisper_model_loader` does
+
     /// Convert the provided text into tokens.
     ///
     /// # Arguments

src/whisper_state.rs

@@ -30,6 +30,50 @@ impl WhisperState {
         Self { ctx, ptr }
     }
 
+    /// Using this context, enable use of OpenVINO for encoder inference.
+    ///
+    /// # Arguments
+    /// * `model_path`: An optional path to the OpenVINO encoder IR model.
+    /// If set to `None`,
+    /// the path will be generated from the ggml model path
+    /// that was passed in to whisper_init_from_file.
+    /// For example, if the model path was "/path/to/ggml-base.en.bin",
+    /// then the OpenVINO IR model path will be assumed as "/path/to/ggml-base.en-encoder-openvino.xml".
+    ///
+    /// * `device`: The OpenVINO device to use for inference (e.g. "CPU", "GPU")
+    ///
+    /// * `cache_dir`: Optional cache directory that can speed up init time,
+    /// especially for GPU, by caching compiled 'blobs' there.
+    /// Set to nullptr if not used.
+    ///
+    /// # Returns
+    /// `true` on success, `false` if OpenVINO was not enabled at compile time
+    /// (enable the `openvino` feature flag in your Cargo.toml).
+    ///
+    /// # C++ equivalent
+    /// `int whisper_ctx_init_openvino_encoder(struct whisper_context * ctx, const char * model_path, const char * device, const char * cache_dir);`
+    #[cfg(feature = "openvino")]
+    pub fn init_openvino_encoder(
+        &mut self,
+        model_path: Option<&str>,
+        device: &str,
+        cache_dir: Option<&str>,
+    ) -> bool {
+        let model_path = model_path.map(|s| CString::new(s).unwrap());
+        let device = CString::new(device).unwrap();
+        let cache_dir = cache_dir.map(|s| CString::new(s).unwrap());
+        let ret = unsafe {
+            whisper_rs_sys::whisper_ctx_init_openvino_encoder_with_state(
+                self.ctx.ctx,
+                self.ptr,
+                model_path.map(|s| s.as_ptr()).unwrap_or(std::ptr::null()),
+                device.as_ptr(),
+                cache_dir.map(|s| s.as_ptr()).unwrap_or(std::ptr::null()),
+            )
+        };
+        ret != 0
+    }
+
     /// Convert raw PCM audio (floating point 32 bit) to log mel spectrogram.
     /// The resulting spectrogram is stored in the context transparently.
     ///

sys/Cargo.toml

@@ -61,6 +61,9 @@ metal = []
 vulkan = []
 force-debug = []
 openmp = []
+openvino = []
+
+[dependencies]
 
 [build-dependencies]
 cmake = "0.1"

sys/build.rs

@@ -9,6 +9,25 @@ use std::io::{BufRead, BufReader};
 use std::path::PathBuf;
 
 fn main() {
+    // Fail-fast test for OpenVINO
+    #[cfg(feature = "openvino")]
+    {
+        let openvino_dir = env::var("OpenVINO_DIR")
+            .unwrap_or_else(|_| String::from("/opt/intel/openvino/runtime/cmake/"));
+        // see if we can find OpenVINOConfig.cmake
+        let openvino_config_path = PathBuf::from(&openvino_dir).join("OpenVINOConfig.cmake");
+        if !openvino_config_path.exists() {
+            panic!(
+                "Couldn't find OpenVINOConfig.cmake in OpenVINO_DIR. Please set it to the path where `OpenVINOConfig.cmake` can be found.\n\
+                On Arch Linux, if you installed the AUR package, this path is `/opt/intel/openvino/runtime/cmake/`.\n\
+                Note the `/cmake/` at the end of the path."
+            );
+        }
+
+        // exists so be sure to reexport it
+        unsafe { env::set_var("OpenVINO_DIR", openvino_dir) }
+    }
+
     let target = env::var("TARGET").unwrap();
     // Link C++ standard library
     if let Some(cpp_stdlib) = get_cpp_link_stdlib(&target) {
@@ -182,6 +201,8 @@ fn main() {
             config.define("AMDGPU_TARGETS", gpu_targets);
         }
     }
+    #[cfg(feature = "openvino")]
+    config.define("WHISPER_OPENVINO", "1");
 
     if cfg!(feature = "vulkan") {
         config.define("GGML_VULKAN", "ON");