enhance: [2.5]compress jsonstatslog path only save filename (#39979)

enhance: compress jsonstatslog path only save filename issue: #36995 pr: #38039 Signed-off-by: Xianhui.Lin <xianhui.lin@zilliz.com>
milvus-io · Feb 19, 2025 · 04175d8 · 04175d8
1 parent 6357673
commit 04175d8
Show file tree

Hide file tree

Showing 7 changed files with 17 additions and 11 deletions.
diff --git a/configs/milvus.yaml b/configs/milvus.yaml
@@ -416,7 +416,7 @@ queryNode:
       buildParallelRate: 0.5 # the ratio of building interim index parallel matched with cpu num
     multipleChunkedEnable: true # Enable multiple chunked search
     knowhereScoreConsistency: false # Enable knowhere strong consistency score computation logic
-    jsonIndexMemoryBudgetInTantivy: 64 # the memory budget for the JSON index In Tantivy
+    jsonIndexMemoryBudgetInTantivy: 16 # the memory budget for the JSON index In Tantivy
     jsonIndexCommitInterval: 200 # the commit interval for the JSON index to commit
   loadMemoryUsageFactor: 1 # The multiply factor of calculating the memory usage while loading segments
   enableDisk: false # enable querynode load disk index, and search on disk index

diff --git a/internal/core/src/common/Consts.h b/internal/core/src/common/Consts.h
@@ -79,6 +79,6 @@ const int64_t DEFAULT_HYBRID_INDEX_BITMAP_CARDINALITY_LIMIT = 100;
 
 const size_t MARISA_NULL_KEY_ID = -1;
 
-const int64_t DEFAULT_JSON_INDEX_MEMORY_BUDGET = 67108864;  // bytes, 64MB
+const int64_t DEFAULT_JSON_INDEX_MEMORY_BUDGET = 16777216;  // bytes, 16MB
 const int64_t DEFAULT_JSON_INDEX_COMMIT_INTERVAL = 200;
 const bool DEFAULT_JSON_INDEX_ENABLED = true;
diff --git a/internal/core/src/exec/expression/Expr.cpp b/internal/core/src/exec/expression/Expr.cpp
@@ -152,8 +152,6 @@ CompileExpression(const expr::TypedExprPtr& expr,
                   const std::unordered_set<std::string>& flatten_candidates,
                   bool enable_constant_folding) {
     ExprPtr result;
-    std::cout << "CompileExpression context" << context->get_consistency_level()
-              << std::endl;
     auto compiled_inputs = CompileInputs(expr, context, flatten_candidates);
 
     auto GetTypes = [](const std::vector<ExprPtr>& exprs) {

diff --git a/internal/core/src/index/JsonKeyInvertedIndex.cpp b/internal/core/src/index/JsonKeyInvertedIndex.cpp
@@ -222,7 +222,8 @@ JsonKeyInvertedIndex::Upload(const Config& config) {
     index_files.reserve(remote_paths_to_size.size() +
                         remote_mem_path_to_size.size());
     for (auto& file : remote_paths_to_size) {
-        index_files.emplace_back(file.first, file.second);
+        index_files.emplace_back(disk_file_manager_->GetFileName(file.first),
+                                 file.second);
     }
     for (auto& file : remote_mem_path_to_size) {
         index_files.emplace_back(file.first, file.second);
@@ -240,6 +241,15 @@ JsonKeyInvertedIndex::Load(milvus::tracer::TraceContext ctx,
     AssertInfo(index_files.has_value(),
                "index file paths is empty when load json key index");
 
+    for (auto& index_file : index_files.value()) {
+        boost::filesystem::path p(index_file);
+        if (!p.has_parent_path()) {
+            auto remote_prefix =
+                disk_file_manager_->GetRemoteJsonKeyLogPrefix();
+            index_file = remote_prefix + "/" + index_file;
+        }
+    }
+
     disk_file_manager_->CacheJsonKeyIndexToDisk(index_files.value());
     AssertInfo(
         tantivy_index_exist(path_.c_str()), "index not exist: {}", path_);

diff --git a/internal/core/src/query/ExecPlanNodeVisitor.cpp b/internal/core/src/query/ExecPlanNodeVisitor.cpp
@@ -83,8 +83,6 @@ ExecPlanNodeVisitor::ExecuteTask(
               plan.plan_node_->ToString(),
               query_context->get_active_count(),
               query_context->get_query_timestamp());
-    std::cout << "ExecuteTask query_context"
-              << query_context->get_consistency_level() << std::endl;
     auto task =
         milvus::exec::Task::Create(DEFAULT_TASK_ID, plan, 0, query_context);
     int64_t processed_num = 0;

diff --git a/internal/core/src/storage/DiskFileManagerImpl.h b/internal/core/src/storage/DiskFileManagerImpl.h
@@ -133,15 +133,15 @@ class DiskFileManagerImpl : public FileManagerImpl {
         return added_total_file_size_;
     }
 
+    std::string
+    GetFileName(const std::string& localfile);
+
  private:
     int64_t
     GetIndexBuildId() {
         return index_meta_.build_id;
     }
 
-    std::string
-    GetFileName(const std::string& localfile);
-
     std::string
     GetRemoteIndexPath(const std::string& file_name, int64_t slice_num) const;
 

diff --git a/pkg/util/paramtable/component_param.go b/pkg/util/paramtable/component_param.go
@@ -3285,7 +3285,7 @@ user-task-polling:
 	p.JSONIndexMemoryBudgetInTantivy = ParamItem{
 		Key:          "queryNode.segcore.jsonIndexMemoryBudgetInTantivy",
 		Version:      "2.5.0",
-		DefaultValue: "64",
+		DefaultValue: "16",
 		Doc:          "the memory budget for the JSON index In Tantivy",
 		Export:       true,
 	}