From 218dc716a9f3ea022cc8b6cfeb6fe10377119e11 Mon Sep 17 00:00:00 2001 From: henryzhx8 Date: Thu, 20 Jun 2024 08:47:42 +0800 Subject: [PATCH] [backport] fix: disable nanosecond timestamp when only EnableTimestampNanosecond is enabled (#1528) (#1552) --- core/common/TimeUtil.cpp | 7 +++-- core/common/TimeUtil.h | 3 +- core/models/LogEvent.cpp | 4 ++- core/models/MetricEvent.cpp | 4 ++- core/models/PipelineEvent.h | 21 +++++++++----- core/models/SpanEvent.cpp | 4 ++- core/spl/PipelineEventGroupInput.cpp | 2 +- .../ProcessorParseApsaraNativeUnittest.cpp | 10 ------- .../ProcessorParseDelimiterNativeUnittest.cpp | 29 +++++++++++++++++++ .../ProcessorParseRegexNativeUnittest.cpp | 6 ---- 10 files changed, 58 insertions(+), 32 deletions(-) diff --git a/core/common/TimeUtil.cpp b/core/common/TimeUtil.cpp index cc2dc4bf24..10754cf742 100644 --- a/core/common/TimeUtil.cpp +++ b/core/common/TimeUtil.cpp @@ -302,10 +302,11 @@ void SetLogTime(sls_logs::Log* log, time_t second) { log->set_time(second); } -void SetLogTimeWithNano(sls_logs::Log* log, time_t second, long nanosecond) { - // Usage: set nanosecond first, and then discard at LogProcess@ProcessBufferLegacy +void SetLogTimeWithNano(sls_logs::Log* log, time_t second, std::optional nanosecond) { log->set_time(second); - log->set_time_ns(nanosecond); + if (nanosecond) { + log->set_time_ns(nanosecond.value()); + } } LogtailTime GetCurrentLogtailTime() { diff --git a/core/common/TimeUtil.h b/core/common/TimeUtil.h index 59a6f3ccfc..24fcb81d2c 100644 --- a/core/common/TimeUtil.h +++ b/core/common/TimeUtil.h @@ -16,6 +16,7 @@ #pragma once #include +#include #include #include @@ -83,7 +84,7 @@ uint64_t GetPreciseTimestampFromLogtailTime(LogtailTime logTime, const PreciseTi void SetLogTime(sls_logs::Log* log, time_t second); -void SetLogTimeWithNano(sls_logs::Log* log, time_t second, long nanosecond); +void SetLogTimeWithNano(sls_logs::Log* log, time_t second, std::optional nanosecond); LogtailTime GetCurrentLogtailTime(); diff --git a/core/models/LogEvent.cpp b/core/models/LogEvent.cpp index dc10b34be1..e9b3106bd2 100644 --- a/core/models/LogEvent.cpp +++ b/core/models/LogEvent.cpp @@ -131,7 +131,9 @@ Json::Value LogEvent::ToJson() const { Json::Value root; root["type"] = static_cast(GetType()); root["timestamp"] = GetTimestamp(); - root["timestampNanosecond"] = GetTimestampNanosecond(); + if (GetTimestampNanosecond()) { + root["timestampNanosecond"] = static_cast(GetTimestampNanosecond().value()); + } if (!Empty()) { Json::Value contents; for (const auto& content : *this) { diff --git a/core/models/MetricEvent.cpp b/core/models/MetricEvent.cpp index 402882688b..35160b525b 100644 --- a/core/models/MetricEvent.cpp +++ b/core/models/MetricEvent.cpp @@ -31,7 +31,9 @@ Json::Value MetricEvent::ToJson() const { Json::Value root; root["type"] = static_cast(GetType()); root["timestamp"] = GetTimestamp(); - root["timestampNanosecond"] = GetTimestampNanosecond(); + if (GetTimestampNanosecond()) { + root["timestampNanosecond"] = static_cast(GetTimestampNanosecond().value()); + } return root; } diff --git a/core/models/PipelineEvent.h b/core/models/PipelineEvent.h index 255ae1357a..044b58fd28 100644 --- a/core/models/PipelineEvent.h +++ b/core/models/PipelineEvent.h @@ -22,6 +22,7 @@ #include #include #include +#include #include #include "models/StringView.h" @@ -38,12 +39,16 @@ class PipelineEvent { virtual ~PipelineEvent() = default; Type GetType() const { return mType; } - time_t GetTimestamp() const { return timestamp; } - long GetTimestampNanosecond() const { return timestampNanosecond; } - void SetTimestamp(time_t t) { timestamp = t; } - void SetTimestamp(time_t t, long ns) { - timestamp = t; - timestampNanosecond = ns; // Only nanosecond part + time_t GetTimestamp() const { return mTimestamp; } + std::optional GetTimestampNanosecond() const { return mTimestampNanosecond; } + void SetTimestamp(time_t t) { mTimestamp = t; } + void SetTimestamp(time_t t, uint32_t ns) { + mTimestamp = t; + mTimestampNanosecond = ns; // Only nanosecond part + } + void SetTimestamp(time_t t, std::optional ns) { + mTimestamp = t; + mTimestampNanosecond = ns; // Only nanosecond part } void ResetPipelineEventGroup(PipelineEventGroup* ptr) { mPipelineEventGroupPtr = ptr; } std::shared_ptr& GetSourceBuffer(); @@ -61,8 +66,8 @@ class PipelineEvent { PipelineEvent(Type type, PipelineEventGroup* ptr); Type mType = Type::NONE; - time_t timestamp = 0; - long timestampNanosecond = 0; + time_t mTimestamp = 0; + std::optional mTimestampNanosecond; PipelineEventGroup* mPipelineEventGroupPtr = nullptr; }; diff --git a/core/models/SpanEvent.cpp b/core/models/SpanEvent.cpp index 108a181b50..ed72e692e1 100644 --- a/core/models/SpanEvent.cpp +++ b/core/models/SpanEvent.cpp @@ -31,7 +31,9 @@ Json::Value SpanEvent::ToJson() const { Json::Value root; root["type"] = static_cast(GetType()); root["timestamp"] = GetTimestamp(); - root["timestampNanosecond"] = GetTimestampNanosecond(); + if (GetTimestampNanosecond()) { + root["timestampNanosecond"] = static_cast(GetTimestampNanosecond().value()); + } return root; } diff --git a/core/spl/PipelineEventGroupInput.cpp b/core/spl/PipelineEventGroupInput.cpp index 334e0fe2fd..fce5248ee5 100644 --- a/core/spl/PipelineEventGroupInput.cpp +++ b/core/spl/PipelineEventGroupInput.cpp @@ -56,7 +56,7 @@ void PipelineEventGroupInput::getTimeColumns(std::vector& times, for (const auto &event : mLogGroup->GetEvents()) { const LogEvent& sourceEvent = event.Cast(); times.emplace_back(sourceEvent.GetTimestamp()); - timeNanos.emplace_back(sourceEvent.GetTimestampNanosecond()); + timeNanos.emplace_back(sourceEvent.GetTimestampNanosecond() ? sourceEvent.GetTimestampNanosecond().value() : 0); } } diff --git a/core/unittest/processor/ProcessorParseApsaraNativeUnittest.cpp b/core/unittest/processor/ProcessorParseApsaraNativeUnittest.cpp index b0f83ab094..b0905cb195 100644 --- a/core/unittest/processor/ProcessorParseApsaraNativeUnittest.cpp +++ b/core/unittest/processor/ProcessorParseApsaraNativeUnittest.cpp @@ -492,7 +492,6 @@ void ProcessorParseApsaraNativeUnittest::TestMultipleLines() { "__raw__": "[2023-09-04 13:15" }, "timestamp": 12345678901, - "timestampNanosecond": 0, "type": 1 }, { @@ -500,7 +499,6 @@ void ProcessorParseApsaraNativeUnittest::TestMultipleLines() { "__raw__": ":50.1]\t[ERROR]\t[1]\t/ilogtail/AppConfigBase.cpp:1\t\tAppConfigBase AppConfigBase:1" }, "timestamp": 12345678901, - "timestampNanosecond": 0, "type": 1 }, { @@ -864,7 +862,6 @@ void ProcessorParseApsaraNativeUnittest::TestProcessKeyOverwritten() { "rawLog": "value1" }, "timestamp": 12345678901, - "timestampNanosecond": 0, "type": 1 } ] @@ -940,7 +937,6 @@ void ProcessorParseApsaraNativeUnittest::TestUploadRawLog() { "rawLog": "value1" }, "timestamp": 12345678901, - "timestampNanosecond": 0, "type": 1 } ] @@ -1051,7 +1047,6 @@ void ProcessorParseApsaraNativeUnittest::TestProcessEventKeepUnmatch() { "rawLog" : "value1" }, "timestamp" : 12345678901, - "timestampNanosecond": 0, "type" : 1 }, { @@ -1060,7 +1055,6 @@ void ProcessorParseApsaraNativeUnittest::TestProcessEventKeepUnmatch() { "rawLog" : "value1" }, "timestamp" : 12345678901, - "timestampNanosecond": 0, "type" : 1 }, { @@ -1069,7 +1063,6 @@ void ProcessorParseApsaraNativeUnittest::TestProcessEventKeepUnmatch() { "rawLog" : "value1" }, "timestamp" : 12345678901, - "timestampNanosecond": 0, "type" : 1 }, { @@ -1078,7 +1071,6 @@ void ProcessorParseApsaraNativeUnittest::TestProcessEventKeepUnmatch() { "rawLog" : "value1" }, "timestamp" : 12345678901, - "timestampNanosecond": 0, "type" : 1 }, { @@ -1087,7 +1079,6 @@ void ProcessorParseApsaraNativeUnittest::TestProcessEventKeepUnmatch() { "rawLog" : "value1" }, "timestamp" : 12345678901, - "timestampNanosecond": 0, "type" : 1 } ] @@ -1293,7 +1284,6 @@ void ProcessorParseApsaraNativeUnittest::TestProcessEventMicrosecondUnmatch() { "rawLog": "[2023-09-04 13:18:04" }, "timestamp": 12345678901, - "timestampNanosecond": 0, "type": 1 } ] diff --git a/core/unittest/processor/ProcessorParseDelimiterNativeUnittest.cpp b/core/unittest/processor/ProcessorParseDelimiterNativeUnittest.cpp index 10463509aa..8f6bfbee5d 100644 --- a/core/unittest/processor/ProcessorParseDelimiterNativeUnittest.cpp +++ b/core/unittest/processor/ProcessorParseDelimiterNativeUnittest.cpp @@ -86,6 +86,7 @@ void ProcessorParseDelimiterNativeUnittest::TestAllowingShortenedFields() { "content" : "2013-10-31 21:03:49,POST,'PutData?Category=YunOsAccountOpLog',0.024" }, "timestamp" : 12345678901, + "timestampNanosecond": 0, "type" : 1 }, { @@ -94,6 +95,7 @@ void ProcessorParseDelimiterNativeUnittest::TestAllowingShortenedFields() { "content" : "value1" }, "timestamp" : 12345678901, + "timestampNanosecond": 0, "type" : 1 } ] @@ -146,6 +148,7 @@ void ProcessorParseDelimiterNativeUnittest::TestAllowingShortenedFields() { "__file_offset__": 0 }, "timestamp" : 12345678901, + "timestampNanosecond": 0, "type" : 1 } ] @@ -270,6 +273,7 @@ void ProcessorParseDelimiterNativeUnittest::TestAllowingShortenedFields() { "__file_offset__": 0 }, "timestamp" : 12345678901, + "timestampNanosecond": 0, "type" : 1 } ] @@ -417,6 +421,7 @@ void ProcessorParseDelimiterNativeUnittest::TestExtend() { "__file_offset__": 0 }, "timestamp" : 12345678901, + "timestampNanosecond": 0, "type" : 1 } ] @@ -545,6 +550,7 @@ void ProcessorParseDelimiterNativeUnittest::TestExtend() { "__file_offset__": 0 }, "timestamp" : 12345678901, + "timestampNanosecond": 0, "type" : 1 } ] @@ -676,6 +682,7 @@ void ProcessorParseDelimiterNativeUnittest::TestMultipleLines() { "__file_offset__": 0 }, "timestamp" : 12345678901, + "timestampNanosecond": 0, "type" : 1 } ] @@ -800,6 +807,7 @@ void ProcessorParseDelimiterNativeUnittest::TestMultipleLines() { "__file_offset__": 0 }, "timestamp" : 12345678901, + "timestampNanosecond": 0, "type" : 1 } ] @@ -923,6 +931,7 @@ void ProcessorParseDelimiterNativeUnittest::TestMultipleLines() { "__file_offset__": 0 }, "timestamp" : 12345678901, + "timestampNanosecond": 0, "type" : 1 } ] @@ -1065,6 +1074,7 @@ void ProcessorParseDelimiterNativeUnittest::TestProcessWholeLine() { "content" : "2013-10-31 21:03:49,POST,PutData?Category=YunOsAccountOpLog,0.024" }, "timestamp" : 12345678901, + "timestampNanosecond": 0, "type" : 1 }, { @@ -1073,6 +1083,7 @@ void ProcessorParseDelimiterNativeUnittest::TestProcessWholeLine() { "content" : "2013-10-31 21:04:49,POST,PutData?Category=YunOsAccountOpLog,0.024" }, "timestamp" : 12345678901, + "timestampNanosecond": 0, "type" : 1 } ] @@ -1133,6 +1144,7 @@ void ProcessorParseDelimiterNativeUnittest::TestProcessQuote() { "__file_offset__": 0 }, "timestamp" : 12345678901, + "timestampNanosecond": 0, "type" : 1 } ] @@ -1285,6 +1297,7 @@ void ProcessorParseDelimiterNativeUnittest::TestProcessQuote() { "content" : "2013-10-31 21:03:49,POST,'PutData?Category=YunOsAccountOpLog',0.024" }, "timestamp" : 12345678901, + "timestampNanosecond": 0, "type" : 1 }, { @@ -1293,6 +1306,7 @@ void ProcessorParseDelimiterNativeUnittest::TestProcessQuote() { "content" : "2013-10-31 21:03:49,POST,'PutData?Category=YunOsAccountOpLog,0.024" }, "timestamp" : 12345678901, + "timestampNanosecond": 0, "type" : 1 }, { @@ -1301,6 +1315,7 @@ void ProcessorParseDelimiterNativeUnittest::TestProcessQuote() { "content" : "2013-10-31 21:03:49,POST,'PutData?Category=YunOs'AccountOpLog',0.024" }, "timestamp" : 12345678901, + "timestampNanosecond": 0, "type" : 1 } ] @@ -1381,6 +1396,7 @@ void ProcessorParseDelimiterNativeUnittest::TestProcessKeyOverwritten() { "content" : "2013-10-31 21:03:49,POST,'PutData?Category=YunOsAccountOpLog',0.024" }, "timestamp" : 12345678901, + "timestampNanosecond": 0, "type" : 1 }, { @@ -1389,6 +1405,7 @@ void ProcessorParseDelimiterNativeUnittest::TestProcessKeyOverwritten() { "content" : "value1" }, "timestamp" : 12345678901, + "timestampNanosecond": 0, "type" : 1 } ] @@ -1460,6 +1477,7 @@ void ProcessorParseDelimiterNativeUnittest::TestUploadRawLog() { "content" : "2013-10-31 21:03:49,POST,'PutData?Category=YunOsAccountOpLog',0.024" }, "timestamp" : 12345678901, + "timestampNanosecond": 0, "type" : 1 }, { @@ -1468,6 +1486,7 @@ void ProcessorParseDelimiterNativeUnittest::TestUploadRawLog() { "content" : "value1" }, "timestamp" : 12345678901, + "timestampNanosecond": 0, "type" : 1 } ] @@ -1565,6 +1584,7 @@ void ProcessorParseDelimiterNativeUnittest::TestProcessEventKeepUnmatch() { "content" : "value1" }, "timestamp" : 12345678901, + "timestampNanosecond": 0, "type" : 1 }, { @@ -1573,6 +1593,7 @@ void ProcessorParseDelimiterNativeUnittest::TestProcessEventKeepUnmatch() { "content" : "value1" }, "timestamp" : 12345678901, + "timestampNanosecond": 0, "type" : 1 }, { @@ -1581,6 +1602,7 @@ void ProcessorParseDelimiterNativeUnittest::TestProcessEventKeepUnmatch() { "content" : "value1" }, "timestamp" : 12345678901, + "timestampNanosecond": 0, "type" : 1 }, { @@ -1589,6 +1611,7 @@ void ProcessorParseDelimiterNativeUnittest::TestProcessEventKeepUnmatch() { "content" : "value1" }, "timestamp" : 12345678901, + "timestampNanosecond": 0, "type" : 1 }, { @@ -1597,6 +1620,7 @@ void ProcessorParseDelimiterNativeUnittest::TestProcessEventKeepUnmatch() { "content" : "value1" }, "timestamp" : 12345678901, + "timestampNanosecond": 0, "type" : 1 } ] @@ -1705,6 +1729,7 @@ void ProcessorParseDelimiterNativeUnittest::TestProcessEventDiscardUnmatch() { "content" : "value1" }, "timestamp" : 12345678901, + "timestampNanosecond": 0, "type" : 1 }, { @@ -1713,6 +1738,7 @@ void ProcessorParseDelimiterNativeUnittest::TestProcessEventDiscardUnmatch() { "content" : "value1" }, "timestamp" : 12345678901, + "timestampNanosecond": 0, "type" : 1 }, { @@ -1721,6 +1747,7 @@ void ProcessorParseDelimiterNativeUnittest::TestProcessEventDiscardUnmatch() { "content" : "value1" }, "timestamp" : 12345678901, + "timestampNanosecond": 0, "type" : 1 }, { @@ -1729,6 +1756,7 @@ void ProcessorParseDelimiterNativeUnittest::TestProcessEventDiscardUnmatch() { "content" : "value1" }, "timestamp" : 12345678901, + "timestampNanosecond": 0, "type" : 1 }, { @@ -1737,6 +1765,7 @@ void ProcessorParseDelimiterNativeUnittest::TestProcessEventDiscardUnmatch() { "content" : "value1" }, "timestamp" : 12345678901, + "timestampNanosecond": 0, "type" : 1 } ] diff --git a/core/unittest/processor/ProcessorParseRegexNativeUnittest.cpp b/core/unittest/processor/ProcessorParseRegexNativeUnittest.cpp index 3e5f7cf438..60573f4caa 100644 --- a/core/unittest/processor/ProcessorParseRegexNativeUnittest.cpp +++ b/core/unittest/processor/ProcessorParseRegexNativeUnittest.cpp @@ -210,7 +210,6 @@ void ProcessorParseRegexNativeUnittest::TestProcessRegex() { "rawLog" : "value1\tvalue2" }, "timestamp" : 12345678901, - "timestampNanosecond" : 0, "type" : 1 }, { @@ -221,7 +220,6 @@ void ProcessorParseRegexNativeUnittest::TestProcessRegex() { "rawLog" : "value3\tvalue4" }, "timestamp" : 12345678901, - "timestampNanosecond" : 0, "type" : 1 } ] @@ -288,7 +286,6 @@ void ProcessorParseRegexNativeUnittest::TestProcessRegexRaw() { "rawLog" : "value1" }, "timestamp" : 12345678901, - "timestampNanosecond" : 0, "type" : 1 }, { @@ -298,7 +295,6 @@ void ProcessorParseRegexNativeUnittest::TestProcessRegexRaw() { "rawLog" : "value3" }, "timestamp" : 12345678901, - "timestampNanosecond" : 0, "type" : 1 } ] @@ -366,7 +362,6 @@ void ProcessorParseRegexNativeUnittest::TestProcessRegexContent() { "rawLog" : "value1\tvalue2" }, "timestamp" : 12345678901, - "timestampNanosecond" : 0, "type" : 1 }, { @@ -377,7 +372,6 @@ void ProcessorParseRegexNativeUnittest::TestProcessRegexContent() { "rawLog" : "value3\tvalue4" }, "timestamp" : 12345678901, - "timestampNanosecond" : 0, "type" : 1 } ]