Configuration guide of structured logging for Kyuubi server

LICENSE-binary

@@ -262,6 +262,7 @@ org.eclipse.jetty:jetty-proxy
 org.apache.logging.log4j:log4j-1.2-api
 org.apache.logging.log4j:log4j-api
 org.apache.logging.log4j:log4j-core
+org.apache.logging.log4j:log4j-layout-template-json
 org.apache.logging.log4j:log4j-slf4j-impl
 org.yaml:snakeyaml
 io.dropwizard.metrics:metrics-core

conf/log4j2.xml.template

@@ -16,7 +16,7 @@
   ~ limitations under the License.
   -->
 
-<!-- Provide log4j2.xml.template to fix `ERROR Filters contains invalid attributes "onMatch", "onMismatch"`, see KYUUBI-2247 -->
+<!-- Provide log4j2.xml.template to fix `ERROR Filters contains invalid attributes "onMatch", "onMismatch"`, see KYUUBI #2247 -->
 <!-- Extra logging related to initialization of Log4j.
  Set to debug or trace if log4j initialization is failing. -->
 <Configuration status="INFO">
@@ -57,6 +57,17 @@
             </Policies>
             <DefaultRolloverStrategy max="10"/>
         </RollingFile>
+        <!-- Kafka appender with Elastic Common Schema(ECS) JSON template layout
+        <Kafka name="kafka" topic="ecs-json-logs" syncSend="false">
+            <JsonTemplateLayout>
+                <EventTemplateAdditionalField key="app" value="kyuubi"/>
+                <EventTemplateAdditionalField key="cluster" value="kyuubi-cluster"/>
+                <EventTemplateAdditionalField key="host" value="${hostName}"/>
+            </JsonTemplateLayout>
+            <Property name="bootstrap.servers" value="kafka-1:9092,kafka-2:9092,kafka-3:9092"/>
+            <Property name="compression.type" value="gzip"/>
+        </Kafka>
+        -->
     </Appenders>
     <Loggers>
         <Root level="INFO">

dev/dependencyList

@@ -126,6 +126,7 @@ kubernetes-model-storageclass/6.13.1//kubernetes-model-storageclass-6.13.1.jar
 log4j-1.2-api/2.24.2//log4j-1.2-api-2.24.2.jar
 log4j-api/2.24.2//log4j-api-2.24.2.jar
 log4j-core/2.24.2//log4j-core-2.24.2.jar
+log4j-layout-template-json/2.24.2//log4j-layout-template-json-2.24.2.jar
 log4j-slf4j-impl/2.24.2//log4j-slf4j-impl-2.24.2.jar
 logging-interceptor/3.12.12//logging-interceptor-3.12.12.jar
 metrics-core/4.2.26//metrics-core-4.2.26.jar

docs/monitor/logging.md

@@ -130,6 +130,55 @@ For example, we can disable the console appender and enable the file appender li
 
 Then everything goes to `log/dummy.log`.
 
+#### Sending Structured Logs to Kafka
+
+The Log4j2 has a built-in [KafkaAppender](https://logging.apache.org/log4j/2.x/manual/appenders/message-queue.html#KafkaAppender)
+which allows sending log messages to an Apache Kafka topic with a few configurations, and it also provides a built-in
+[JSON Template Layout](https://logging.apache.org/log4j/2.x/manual/json-template-layout.html) that supports encoding
+`LogEvents` to structured JSON messages according to the structure described by the provided template.
+
+For example, we can configure the Kyuubi server to send the structured logs to Kafka `ecs-json-logs` topic,
+
+```xml
+<Configuration status="INFO">
+  <Appenders>
+    <Kafka name="kafka" topic="ecs-json-logs" syncSend="false">
+      <JsonTemplateLayout>
+        <EventTemplateAdditionalField key="app" value="kyuubi"/>
+        <EventTemplateAdditionalField key="cluster" value="kyuubi-cluster"/>
+        <EventTemplateAdditionalField key="host" value="${hostName}"/>
+      </JsonTemplateLayout>
+      <Property name="bootstrap.servers" value="kafka-1:9092,kafka-2:9092,kafka-3:9092"/>
+      <Property name="compression.type" value="gzip"/>
+    </Kafka>
+  </Appenders>
+  <Loggers>
+    <Root level="INFO">
+      <AppenderRef ref="kafka"/>
+    </Root>
+  </Loggers>
+</Configuration>
+```
+
+And each structured log message looks like,
+
+```json
+{
+	"@timestamp": "2024-12-24T18:53:01.030Z",
+	"ecs.version": "1.2.0",
+	"log.level": "INFO",
+	"message": "Service[KyuubiServer] is started.",
+	"process.thread.name": "main",
+	"log.logger": "org.apache.kyuubi.server.KyuubiServer",
+	"app": "kyuubi",
+	"cluster": "kyuubi-cluster",
+	"host": "hadoop-master1.orb.local"
+}
+```
+
+Note: this feature may require additional jars to work. Please read the Log4j2 docs and ensure those jars are
+on the Kyuubi server's classpath before enabling it.
+
 ## Logs of Spark SQL Engine
 
 Spark SQL Engine is one type of Kyuubi Engines and also a typical Spark application.

kyuubi-assembly/pom.xml

@@ -119,5 +119,10 @@
             <groupId>org.apache.logging.log4j</groupId>
             <artifactId>log4j-1.2-api</artifactId>
         </dependency>
+
+        <dependency>
+            <groupId>org.apache.logging.log4j</groupId>
+            <artifactId>log4j-layout-template-json</artifactId>
+        </dependency>
     </dependencies>
 </project>

pom.xml

@@ -663,6 +663,11 @@
                 <artifactId>log4j-1.2-api</artifactId>
                 <version>${log4j.version}</version>
             </dependency>
+            <dependency>
+                <groupId>org.apache.logging.log4j</groupId>
+                <artifactId>log4j-layout-template-json</artifactId>
+                <version>${log4j.version}</version>
+            </dependency>
 
             <dependency>
                 <groupId>io.dropwizard.metrics</groupId>