Fix runtime usage problem in linux

CMakeLists.txt

@@ -117,11 +117,11 @@ if(ENABLE_TRT_BACKEND)
   list(APPEND DEPEND_LIBS ${TRT_INFER_LIB} ${TRT_ONNX_LIB} ${TRT_CAFFE_LIB} ${TRT_PLUGIN_LIB})
 
   # copy tensorrt libraries to third lib
-  if(EXISTS "${CMAKE_CURRENT_BINARY_DIR}/third_libs/install/tensorrt")
-    file(REMOVE_RECURSE "${CMAKE_CURRENT_BINARY_DIR}/third_libs/install/tensorrt/lib")
-  endif()
-  file(MAKE_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/third_libs/install/tensorrt/lib")
-  file(COPY ${TRT_INFER_LIB} ${TRT_ONNX_LIB} ${TRT_CAFFE_LIB} ${TRT_PLUGIN_LIB} DESTINATION "${CMAKE_CURRENT_BINARY_DIR}/third_libs/install/tensorrt/lib" FOLLOW_SYMLINK_CHAIN)
+#  if(EXISTS "${CMAKE_CURRENT_BINARY_DIR}/third_libs/install/tensorrt")
+#    file(REMOVE_RECURSE "${CMAKE_CURRENT_BINARY_DIR}/third_libs/install/tensorrt/lib")
+#  endif()
+#  file(MAKE_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/third_libs/install/tensorrt/lib")
+#  file(COPY ${TRT_INFER_LIB} ${TRT_ONNX_LIB} ${TRT_CAFFE_LIB} ${TRT_PLUGIN_LIB} DESTINATION "${CMAKE_CURRENT_BINARY_DIR}/third_libs/install/tensorrt/lib" FOLLOW_SYMLINK_CHAIN)
 endif()
 
 if(ENABLE_VISION)
@@ -278,4 +278,5 @@ if(BUILD_FASTDEPLOY_PYTHON)
                                    ${EXTRA_FLAGS})
     target_compile_options(fastdeploy_main PRIVATE $<$<NOT:$<CONFIG:Debug>>:/MT> $<$<CONFIG:Debug>:/MTd>)
   endif()
+
 endif(BUILD_FASTDEPLOY_PYTHON)

fastdeploy/fastdeploy_runtime.cc

@@ -75,8 +75,10 @@ bool ModelFormatCheck(const std::string& model_file,
 bool Runtime::Init(const RuntimeOption& _option) {
   option = _option;
   if (option.backend == Backend::ORT) {
+    FDASSERT(option.device == Device::CPU || option.device == Device::GPU, "Backend::TRT only supports Device::CPU/Device::GPU.");
     CreateOrtBackend();
   } else if (option.backend == Backend::TRT) {
+    FDASSERT(option.device == Device::GPU, "Backend::TRT only supports Device::GPU.");
     CreateTrtBackend();
   } else {
     FDERROR << "Runtime only support Backend::ORT/Backend::TRT as backend now."

fastdeploy/fastdeploy_runtime.py

@@ -53,3 +53,29 @@ def initialized(self):
         if self._model is None:
             return False
         return self._model.initialized()
+
+
+class FastDeployRuntime:
+    def __init__(self, runtime_option):
+        self._runtime = C.Runtime();
+        assert self._runtime.init(runtime_option), "Initialize FastDeployRuntime Failed!"
+
+    def infer(self, data):
+        assert isinstance(data, dict), "The input data should be type of dict."
+        return self._runtime.infer(data)
+
+    def num_inputs(self):
+        return self._runtime.num_inputs();
+
+    def num_outputs(self):
+        return self._runtime.num_outputs();
+
+    def get_input_info(self, index):
+        assert isinstance(index, int), "The input parameter index should be type of int."
+        assert index < self.num_inputs(), "The input parameter index:{} should less than number of inputs:{}.".format(index, self.num_inputs)
+        return self._runtime.get_input_info(index)
+
+    def get_output_info(self, index):
+        assert isinstance(index, int), "The input parameter index should be type of int."
+        assert index < self.num_outputs(), "The input parameter index:{} should less than number of outputs:{}.".format(index, self.num_outputs)
+        return self._runtime.get_output_info(index)

fastdeploy/pybind/fastdeploy_runtime.cc

@@ -40,12 +40,15 @@ void BindRuntime(pybind11::module& m) {
       .def_readwrite("trt_max_batch_size", &RuntimeOption::trt_max_batch_size)
       .def_readwrite("trt_max_workspace_size",
                      &RuntimeOption::trt_max_workspace_size);
+
+  pybind11::class_<TensorInfo>(m, "TensorInfo")
+      .def_readwrite("name", &TensorInfo::name)
+      .def_readwrite("shape", &TensorInfo::shape)
+      .def_readwrite("dtype", &TensorInfo::dtype);
+
   pybind11::class_<Runtime>(m, "Runtime")
-      .def(pybind11::init([](RuntimeOption& option) {
-        Runtime* runtime = new Runtime();
-        runtime->Init(option);
-        return runtime;
-      }))
+      .def(pybind11::init())
+      .def("init", &Runtime::Init)
       .def("infer", [](Runtime& self,
                        std::map<std::string, pybind11::array>& data) {
         std::vector<FDTensor> inputs(data.size());
@@ -75,7 +78,12 @@ void BindRuntime(pybind11::module& m) {
                  outputs[i].Numel() * FDDataTypeSize(outputs[i].dtype));
         }
         return results;
-      });
+      })
+     .def("num_inputs", &Runtime::NumInputs)
+     .def("num_outputs", &Runtime::NumOutputs)
+     .def("get_input_info", &Runtime::GetInputInfo)
+     .def("get_output_info", &Runtime::GetOutputInfo)
+     .def_readonly("option", &Runtime::option);
 
   pybind11::enum_<Backend>(m, "Backend", pybind11::arithmetic(),
                            "Backend for inference.")
@@ -103,11 +111,6 @@ void BindRuntime(pybind11::module& m) {
       .value("FP64", FDDataType::FP64)
       .value("UINT8", FDDataType::UINT8);
 
-  pybind11::class_<TensorInfo>(m, "TensorInfo")
-      .def_readwrite("name", &TensorInfo::name)
-      .def_readwrite("shape", &TensorInfo::shape)
-      .def_readwrite("dtype", &TensorInfo::dtype);
-
   m.def("get_available_backends", []() { return GetAvailableBackends(); });
 }
 

setup.py

@@ -324,11 +324,18 @@ def run(self):
     shutil.copy("ThirdPartyNotices.txt", "fastdeploy")
     shutil.copy("LICENSE", "fastdeploy")
     depend_libs = list()
+
+    # modify the search path of libraries
+    command = "patchelf --set-rpath '$ORIGIN/libs/' .setuptools-cmake-build/fastdeploy_main.cpython-36m-x86_64-linux-gnu.so"
+    # The sw_64 not suppot patchelf, so we just disable that.
+    if platform.machine() != 'sw_64' and platform.machine() != 'mips64':
+        assert os.system(command) == 0, "patch fastdeploy_main.cpython-36m-x86_64-linux-gnu.so failed, the command: {}".format(command)
+
     for f in os.listdir(".setuptools-cmake-build"):
         if not os.path.isfile(os.path.join(".setuptools-cmake-build", f)):
             continue
         if f.count("libfastdeploy") > 0:
-            depend_libs.append(os.path.join(".setuptools-cmake-build", f))
+            shutil.copy(os.path.join(".setuptools-cmake-build", f), "fastdeploy/libs")
     for dirname in os.listdir(".setuptools-cmake-build/third_libs/install"):
         for lib in os.listdir(
                 os.path.join(".setuptools-cmake-build/third_libs/install",