diff --git a/CMakeLists.txt b/CMakeLists.txt
index 24c3561f1a..bb5cc8c414 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -263,7 +263,9 @@ include(simple-sentencepiece)
 set(ONNXRUNTIME_DIR ${onnxruntime_SOURCE_DIR})
 message(STATUS "ONNXRUNTIME_DIR: ${ONNXRUNTIME_DIR}")
 
-if(SHERPA_ONNX_ENABLE_PORTAUDIO)
+if(SHERPA_ONNX_ENABLE_PORTAUDIO AND SHERPA_ONNX_ENABLE_BINARY)
+  # portaudio is used only in building demo binaries and the sherpa-onnx-core
+  # library does not depend on it.
   include(portaudio)
 endif()