Skip to content
New issue

Have a question about this project? # for a free GitHub account to open an issue and contact its maintainers and the community.

#

By clicking “#”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? # to your account

chore: Add example for containerd multiline CS-1457 #272

Merged
merged 1 commit into from
Nov 22, 2024
Merged

chore: Add example for containerd multiline CS-1457 #272

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions examples/stack/logs/multi_line/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,3 +12,13 @@ helm install --namespace=observe observe-stack observe/stack \
--set observe.token.value="dsabcdefghijk:qrstuvwzyz" \
-f ./multiline-values.yaml
```

## Deploy containerd runtime collection to combine lines above 16k
`containerd` logs are split into 16KB chunks due to the design of `containerd`’s logging mechanism, the `containerd-multiline.yaml` will combine the multiline log files.

```
helm install --namespace=observe observe-stack observe/stack \
--set global.observe.collectionEndpoint="https://12345678.observeinc.com/v1/http" \
--set observe.token.value="dsabcdefghijk:qrstuvwzyz" \
-f ./containerd-multiline.yaml
```
210 changes: 210 additions & 0 deletions examples/stack/logs/multi_line/containerd-multiline.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,210 @@
global:
observe:
collectionEndpoint: ""

observe:
token:
value: ""

events:
resources:
limits:
cpu: 50m
memory: 256Mi
requests:
cpu: 50m
memory: 256Mi

logs:
fluent-bit:
resources:
limits:
cpu: 100m
memory: 128Mi
requests:
cpu: 100m
memory: 128Mi
config:
buffer_chunk_size: 32k
buffer_max_size: 256k
dns_mode: UDP
dns_resolver: LEGACY
flush: 2
grace: 10
hc_errors_count: 5
hc_retry_failure_count: 5
hc_period: 10
log_level: warning
mem_buf_limit: 10MB
node_log_include_path: /var/log/kube-apiserver-audit.log
node_log_exclude_path: nomatch
read_from_head: "true"
ignore_older: "2d"
refresh_interval: 2
retry_limit: 5
rotate_wait: 5
storage_metrics: "off"
grep_match_tag: "nothing"
grep_exclude: "nomatch ^$"
inotify_watcher: "true"
workers: 2
connect_timeout: 10
keepalive: "on"
keepalive_idle_timeout: 30
keepalive_max_recycle: 2000
max_worker_connections: 25

service: |
[SERVICE]
Flush {{.Values.config.flush}}
Grace {{.Values.config.grace}}
Daemon Off
Log_Level {{.Values.config.log_level}}
Parsers_File custom_parsers.conf
HTTP_Server On
HTTP_Listen 0.0.0.0
HTTP_PORT 2020
Health_Check On
HC_Errors_Count {{.Values.config.hc_errors_count}}
HC_Retry_Failure_Count {{.Values.config.hc_retry_failure_count}}
HC_Period {{.Values.config.hc_period}}
dns.mode {{.Values.config.dns_mode}}
dns.resolver {{.Values.config.dns_resolver}}
storage.metrics {{.Values.config.storage_metrics}}

inputs: |
[INPUT]
Name tail
Tag k8slogs
Alias k8slogs
Path /var/log/containers/*.log
Path_Key filename
DB /var/log/flb_kube_${NAMESPACE}.db
Skip_Long_Lines On
Multiline.Parser multilinecontainerd
Read_From_Head {{.Values.config.read_from_head}}
Mem_Buf_Limit {{.Values.config.mem_buf_limit}}
Buffer_Chunk_Size {{.Values.config.buffer_chunk_size}}
Buffer_Max_Size {{.Values.config.buffer_max_size}}
Rotate_Wait {{.Values.config.rotate_wait}}
Refresh_Interval {{.Values.config.refresh_interval}}
Inotify_Watcher {{.Values.config.inotify_watcher}}

[INPUT]
Name tail
Tag k8snode
Alias k8snode
Path {{.Values.config.node_log_include_path}}
Exclude_Path {{.Values.config.node_log_exclude_path}}
Path_Key filename
DB /var/log/flb_node_${NAMESPACE}.db
Skip_Long_Lines On
Read_From_Head {{.Values.config.read_from_head}}
Ignore_Older {{.Values.config.ignore_older}}
Mem_Buf_Limit {{.Values.config.mem_buf_limit}}
Buffer_Chunk_Size {{.Values.config.buffer_chunk_size}}
Buffer_Max_Size {{.Values.config.buffer_max_size}}
Rotate_Wait {{.Values.config.rotate_wait}}
Inotify_Watcher {{.Values.config.inotify_watcher}}

filters: |
[FILTER]
Name lua
Match k8slogs
Call remove_timestamp_flags_and_eol
Code function remove_timestamp_flags_and_eol(tag, timestamp, record) if record["log"] then record["log"] = string.gsub(record["log"], "%s?%d+-%d+-%d+T%d+:%d+:%d+.%d+Z stdout%sF%s", ""); record["log"] = string.gsub(record["log"], "[\n\r]+$", ""); end return 2, timestamp, record end

[FILTER]
Name record_modifier
Alias add_nodename
Match *
Record nodeName ${NODE}

[FILTER]
Name parser
Alias parse_filename
Match k8slogs
Key_Name filename
Reserve_Data True
Parser kube-custom

[FILTER]
Name record_modifier
Alias filter_docker
Match k8slogs
Whitelist_key containerId
Whitelist_key containerName
Whitelist_key log
Whitelist_key podName
Whitelist_key namespace
Whitelist_key nodeName

[FILTER]
Name grep
Alias exclude
Match {{.Values.config.grep_match_tag}}
Exclude {{.Values.config.grep_exclude}}

outputs: |
# uncomment below for additional debugging
# [OUTPUT]
# Name stdout
# Match k8slogs

[OUTPUT]
Name http
Match k8slogs*
Alias k8slogs
Host {{ include "observe.collectorHost" . }}
Port {{ include "observe.collectorPort" . }}
TLS {{ include "observe.useTLS" . }}
URI /v1/http/kubernetes/logs?clusterUid=${OBSERVE_CLUSTER}
Format msgpack
Header X-Observe-Decoder fluent
Header Authorization Bearer ${OBSERVE_TOKEN}
Compress gzip
Retry_Limit {{.Values.config.retry_limit}}
Workers {{.Values.config.workers}}
net.connect_timeout {{.Values.config.connect_timeout}}
net.keepalive {{.Values.config.keepalive}}
net.keepalive_idle_timeout {{.Values.config.keepalive_idle_timeout}}
net.keepalive_max_recycle {{.Values.config.keepalive_max_recycle}}
net.max_worker_connections {{.Values.config.max_worker_connections}}

[OUTPUT]
Name http
Match k8snode*
Alias k8snode
Host {{ include "observe.collectorHost" . }}
Port {{ include "observe.collectorPort" . }}
TLS {{ include "observe.useTLS" . }}
URI /v1/http/kubernetes/node?clusterUid=${OBSERVE_CLUSTER}
Format msgpack
Header X-Observe-Decoder fluent
Header Authorization Bearer ${OBSERVE_TOKEN}
Compress gzip
Retry_Limit {{.Values.config.retry_limit}}
Workers {{.Values.config.workers}}
net.connect_timeout {{.Values.config.connect_timeout}}
net.keepalive {{.Values.config.keepalive}}
net.keepalive_idle_timeout {{.Values.config.keepalive_idle_timeout}}
net.keepalive_max_recycle {{.Values.config.keepalive_max_recycle}}
net.max_worker_connections {{.Values.config.max_worker_connections}}

{{- include "observe.includeExtraFiles" . }}

customParsers: |
[PARSER]
Name kube-custom
Format regex
Regex (?<podName>[a-z0-9](?:[-a-z0-9]*[a-z0-9])?(?:\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)_(?<namespace>[^_]+)_(?<containerName>.+)-(?<containerId>[a-f0-9]{64})\.log$

[MULTILINE_PARSER]
name multilinecontainerd
type regex
flush_timeout 1000
rule "start_state" "/^\S+ stdout P.*$/" "cont"
rule "cont" "/^\S+ stdout F.*$/" "cont"

metrics:
enabled: false
Loading