[backport] fix: disable nanosecond timestamp when only EnableTimestam…

…pNanosecond is enabled (#1528) (#1552)
alibaba · Jun 20, 2024 · 218dc71 · 218dc71
1 parent 43df145
commit 218dc71
Show file tree

Hide file tree

Showing 10 changed files with 58 additions and 32 deletions.
diff --git a/core/common/TimeUtil.cpp b/core/common/TimeUtil.cpp
@@ -302,10 +302,11 @@ void SetLogTime(sls_logs::Log* log, time_t second) {
     log->set_time(second);
 }
 
-void SetLogTimeWithNano(sls_logs::Log* log, time_t second, long nanosecond) {
-    // Usage: set nanosecond first, and then discard at LogProcess@ProcessBufferLegacy
+void SetLogTimeWithNano(sls_logs::Log* log, time_t second, std::optional<uint32_t> nanosecond) {
     log->set_time(second);
-    log->set_time_ns(nanosecond);
+    if (nanosecond) {
+        log->set_time_ns(nanosecond.value());
+    }
 }
 
 LogtailTime GetCurrentLogtailTime() {

diff --git a/core/common/TimeUtil.h b/core/common/TimeUtil.h
@@ -16,6 +16,7 @@
 
 #pragma once
 #include <ctime>
+#include <optional>
 #include <string>
 #include <thread>
 
@@ -83,7 +84,7 @@ uint64_t GetPreciseTimestampFromLogtailTime(LogtailTime logTime, const PreciseTi
 
 void SetLogTime(sls_logs::Log* log, time_t second);
 
-void SetLogTimeWithNano(sls_logs::Log* log, time_t second, long nanosecond);
+void SetLogTimeWithNano(sls_logs::Log* log, time_t second, std::optional<uint32_t> nanosecond);
 
 LogtailTime GetCurrentLogtailTime();
 

diff --git a/core/models/LogEvent.cpp b/core/models/LogEvent.cpp
@@ -131,7 +131,9 @@ Json::Value LogEvent::ToJson() const {
     Json::Value root;
     root["type"] = static_cast<int>(GetType());
     root["timestamp"] = GetTimestamp();
-    root["timestampNanosecond"] = GetTimestampNanosecond();
+    if (GetTimestampNanosecond()) {
+        root["timestampNanosecond"] = static_cast<int32_t>(GetTimestampNanosecond().value());
+    }
     if (!Empty()) {
         Json::Value contents;
         for (const auto& content : *this) {

diff --git a/core/models/MetricEvent.cpp b/core/models/MetricEvent.cpp
@@ -31,7 +31,9 @@ Json::Value MetricEvent::ToJson() const {
     Json::Value root;
     root["type"] = static_cast<int>(GetType());
     root["timestamp"] = GetTimestamp();
-    root["timestampNanosecond"] = GetTimestampNanosecond();
+    if (GetTimestampNanosecond()) {
+        root["timestampNanosecond"] = static_cast<int32_t>(GetTimestampNanosecond().value());
+    }
     return root;
 }
 

diff --git a/core/models/PipelineEvent.h b/core/models/PipelineEvent.h
@@ -22,6 +22,7 @@
 #include <ctime>
 #include <map>
 #include <memory>
+#include <optional>
 #include <string>
 
 #include "models/StringView.h"
@@ -38,12 +39,16 @@ class PipelineEvent {
     virtual ~PipelineEvent() = default;
 
     Type GetType() const { return mType; }
-    time_t GetTimestamp() const { return timestamp; }
-    long GetTimestampNanosecond() const { return timestampNanosecond; }
-    void SetTimestamp(time_t t) { timestamp = t; }
-    void SetTimestamp(time_t t, long ns) {
-        timestamp = t;
-        timestampNanosecond = ns; // Only nanosecond part
+    time_t GetTimestamp() const { return mTimestamp; }
+    std::optional<uint32_t> GetTimestampNanosecond() const { return mTimestampNanosecond; }
+    void SetTimestamp(time_t t) { mTimestamp = t; }
+    void SetTimestamp(time_t t, uint32_t ns) {
+        mTimestamp = t;
+        mTimestampNanosecond = ns; // Only nanosecond part
+    }
+    void SetTimestamp(time_t t, std::optional<uint32_t> ns) {
+        mTimestamp = t;
+        mTimestampNanosecond = ns; // Only nanosecond part
     }
     void ResetPipelineEventGroup(PipelineEventGroup* ptr) { mPipelineEventGroupPtr = ptr; }
     std::shared_ptr<SourceBuffer>& GetSourceBuffer();
@@ -61,8 +66,8 @@ class PipelineEvent {
     PipelineEvent(Type type, PipelineEventGroup* ptr);
 
     Type mType = Type::NONE;
-    time_t timestamp = 0;
-    long timestampNanosecond = 0;
+    time_t mTimestamp = 0;
+    std::optional<uint32_t> mTimestampNanosecond;
     PipelineEventGroup* mPipelineEventGroupPtr = nullptr;
 };
 

diff --git a/core/models/SpanEvent.cpp b/core/models/SpanEvent.cpp
@@ -31,7 +31,9 @@ Json::Value SpanEvent::ToJson() const {
     Json::Value root;
     root["type"] = static_cast<int>(GetType());
     root["timestamp"] = GetTimestamp();
-    root["timestampNanosecond"] = GetTimestampNanosecond();
+    if (GetTimestampNanosecond()) {
+        root["timestampNanosecond"] = static_cast<int32_t>(GetTimestampNanosecond().value());
+    }
     return root;
 }
 

diff --git a/core/spl/PipelineEventGroupInput.cpp b/core/spl/PipelineEventGroupInput.cpp
@@ -56,7 +56,7 @@ void PipelineEventGroupInput::getTimeColumns(std::vector<uint32_t>& times,
     for (const auto &event : mLogGroup->GetEvents()) {
         const LogEvent& sourceEvent = event.Cast<LogEvent>();
         times.emplace_back(sourceEvent.GetTimestamp());
-        timeNanos.emplace_back(sourceEvent.GetTimestampNanosecond());
+        timeNanos.emplace_back(sourceEvent.GetTimestampNanosecond() ? sourceEvent.GetTimestampNanosecond().value() : 0);
     }
 }
 

diff --git a/core/unittest/processor/ProcessorParseApsaraNativeUnittest.cpp b/core/unittest/processor/ProcessorParseApsaraNativeUnittest.cpp
@@ -492,15 +492,13 @@ void ProcessorParseApsaraNativeUnittest::TestMultipleLines() {
                     "__raw__": "[2023-09-04 13:15"
                 },
                 "timestamp": 12345678901,
-                "timestampNanosecond": 0,
                 "type": 1
             },
             {
                 "contents": {
                     "__raw__": ":50.1]\t[ERROR]\t[1]\t/ilogtail/AppConfigBase.cpp:1\t\tAppConfigBase AppConfigBase:1"
                 },
                 "timestamp": 12345678901,
-                "timestampNanosecond": 0,
                 "type": 1
             },
             {
@@ -864,7 +862,6 @@ void ProcessorParseApsaraNativeUnittest::TestProcessKeyOverwritten() {
                     "rawLog": "value1"
                 },
                 "timestamp": 12345678901,
-                "timestampNanosecond": 0,
                 "type": 1
             }
         ]
@@ -940,7 +937,6 @@ void ProcessorParseApsaraNativeUnittest::TestUploadRawLog() {
                     "rawLog": "value1"
                 },
                 "timestamp": 12345678901,
-                "timestampNanosecond": 0,
                 "type": 1
             }
         ]
@@ -1051,7 +1047,6 @@ void ProcessorParseApsaraNativeUnittest::TestProcessEventKeepUnmatch() {
                     "rawLog" : "value1"
                 },
                 "timestamp" : 12345678901,
-                "timestampNanosecond": 0,
                 "type" : 1
             },
             {
@@ -1060,7 +1055,6 @@ void ProcessorParseApsaraNativeUnittest::TestProcessEventKeepUnmatch() {
                     "rawLog" : "value1"
                 },
                 "timestamp" : 12345678901,
-                "timestampNanosecond": 0,
                 "type" : 1
             },
             {
@@ -1069,7 +1063,6 @@ void ProcessorParseApsaraNativeUnittest::TestProcessEventKeepUnmatch() {
                     "rawLog" : "value1"
                 },
                 "timestamp" : 12345678901,
-                "timestampNanosecond": 0,
                 "type" : 1
             },
             {
@@ -1078,7 +1071,6 @@ void ProcessorParseApsaraNativeUnittest::TestProcessEventKeepUnmatch() {
                     "rawLog" : "value1"
                 },
                 "timestamp" : 12345678901,
-                "timestampNanosecond": 0,
                 "type" : 1
             },
             {
@@ -1087,7 +1079,6 @@ void ProcessorParseApsaraNativeUnittest::TestProcessEventKeepUnmatch() {
                     "rawLog" : "value1"
                 },
                 "timestamp" : 12345678901,
-                "timestampNanosecond": 0,
                 "type" : 1
             }
         ]
@@ -1293,7 +1284,6 @@ void ProcessorParseApsaraNativeUnittest::TestProcessEventMicrosecondUnmatch() {
                     "rawLog": "[2023-09-04 13:18:04"
                 },
                 "timestamp": 12345678901,
-                "timestampNanosecond": 0,
                 "type": 1
             }
         ]