diff --git a/paddlenlp/ops/faster_transformer/src/CMakeLists.txt b/paddlenlp/ops/faster_transformer/src/CMakeLists.txt
index 46588a22151a..46d292a5ca84 100644
--- a/paddlenlp/ops/faster_transformer/src/CMakeLists.txt
+++ b/paddlenlp/ops/faster_transformer/src/CMakeLists.txt
@@ -272,21 +272,22 @@ else(ON_INFER)
     )
   endif()
 
-  if(EXISTS ${PY_LIB_DIR}/../fluid/core_avx.so)
+  if(EXISTS ${PY_LIB_DIR}/../fluid/libpaddle.so)
     set(lib_link
-      -l:core_avx.so
+      -lpaddle
     )
-    link_directories(
-      ${PY_LIB_DIR}/../fluid/
+  elseif(EXISTS ${PY_LIB_DIR}/../fluid/core_avx.so)
+    set(lib_link
+      -l:core_avx.so
     )
   else()
     set(lib_link
       -l:core_noavx.so
     )
-    link_directories(
-      ${PY_LIB_DIR}/../fluid/
-    )
   endif()
+  link_directories(
+    ${PY_LIB_DIR}/../fluid/
+  )
 
   set(ft_lib_link
     -ldecoder -ldecoding -ltopk -lcuda_int8_kernels -lcuda_kernels -lonline_softmax_beamsearch -ltransformer_kernels -lattention_kernels -lencoder -lnccl_utils -lnvtx_utils