1、采集方案

Spring通过logback产生日志, 然后通过logback的一个TCP Appender产生结构化的Json日志数据, 并直接将该日志数据发送到Logstash监听的TCP端口, Logstash收到后再发送到Elasticsearch

在这里插入图片描述

2、实战

2.1、SpringBoot和Logback配置

2.1.1 核心依赖

implementation "net.logstash.logback:logstash-logback-encoder:6.2"

compileOnly "org.projectlombok:lombok"
annotationProcessor "org.projectlombok:lombok"
复制代码

2.1.2 application.yml

spring:
  profiles:
    active: ${SPRING_ACTIVE:dev}

  application:
    name: elk-server

custom:
  logstash_uri: 127.0.0.1:4660
  appender_ref: Tcp_Logstash
复制代码

2.1.3 logback-spring.xml

Logstash appender的详细配置说明见此链接

<?xml version="1.0" encoding="UTF-8"?>
<configuration>
    <!-- 彩色日志依赖的渲染类 -->
    <conversionRule conversionWord="clr" converterClass="org.springframework.boot.logging.logback.ColorConverter"/>
    <conversionRule conversionWord="wex" converterClass="org.springframework.boot.logging.logback.WhitespaceThrowableProxyConverter"/>
    <conversionRule conversionWord="wEx" converterClass="org.springframework.boot.logging.logback.ExtendedWhitespaceThrowableProxyConverter"/>

    <!-- 自定义属性   -->
    <property name="CONSOLE_LOG_PATTERN"
              value="[%clr(%X{traceId:-}){yellow}] %clr(%d{${LOG_DATEFORMAT_PATTERN:-yyyy-MM-dd HH:mm:ss.SSS}}){faint} %clr(${LOG_LEVEL_PATTERN:-%5p}) %clr(${PID:- }){magenta} %clr(---){faint} %clr([%15.15t]){faint} %clr(%-40.40logger{39}){cyan} %clr(:){faint} %m%n${LOG_EXCEPTION_CONVERSION_WORD:-%wEx}"/>

    <!-- 获取spring上下文属性 -->
    <springProperty scope="local" name="App_Name" source="spring.application.name"
                    defaultValue="demo-server"/>
    <springProperty scope="local" name="Appender_Ref" source="custom.appender_ref"
                    defaultValue="Console"/>
    <springProperty scope="local" name="Logstash_Uri" source="custom.logstash_uri"/>

    <!-- 控制台 appender  -->
    <appender name="Console" class="ch.qos.logback.core.ConsoleAppender">
        <encoder>
            <pattern>${CONSOLE_LOG_PATTERN}</pattern>
            <charset>UTF-8</charset>
        </encoder>
    </appender>

 
    <!-- Logstash appender   -->
     <appender name="Tcp_Logstash" class="net.logstash.logback.appender.LogstashTcpSocketAppender">
         <keepAliveDuration>5 minutes</keepAliveDuration>
         <!--    logstash地址     -->
         <destination>${Logstash_Uri}</destination>
         <!--  encoder必须配置   -->
         <encoder  charset="UTF-8" class="net.logstash.logback.encoder.LogstashEncoder">
             <!--自定义时间戳格式， 默认是yyyy-MM-dd'T'HH:mm:ss.SSS<-->
             <timestampPattern>yyyy-MM-dd HH:mm:ss.SSS</timestampPattern>
             <!-- 设置默认字段及其名字 -->
             <fieldNames>
                 <timestamp>time</timestamp>
                 <message>msg</message>
                 <logger>class</logger>
                 <!--设置为[ignore]可以忽略此标准字段-->
                 <version>[ignore]</version>
                 <thread>[ignore]</thread>
                 <levelValue>[ignore]</levelValue>
             </fieldNames>
             <!--  设置mdc字段名  -->
             <includeMdcKeyName>traceId</includeMdcKeyName>
             <!--    自定义字段  -->
             <customFields>{"appName": "${App_Name}"}</customFields>
             <!--   设置logger name的长度   -->
             <shortenedLoggerNameLength>40</shortenedLoggerNameLength>
         </encoder>
     </appender>

    <!--控制框架输出日志-->
    <logger name="org.slf4j" level="INFO"/>
    <logger name="springfox" level="INFO"/>
    <logger name="io.swagger" level="INFO"/>
    <logger name="org.springframework" level="INFO"/>
    <logger name="org.hibernate.validator" level="INFO"/>

    <!--  定义测试环境日志级别  -->
    <springProfile name="dev">
        <root level="INFO">
            <appender-ref ref="Console" />
        </root>
    </springProfile>

    <!--  定义线上环境日志级别  -->
    <springProfile name="prod">
        <root level="INFO">
            <appender-ref ref="${Appender_Ref}" />
            <appender-ref ref="Console"/>
        </root>
    </springProfile>

</configuration>
复制代码

2.2 Logstash 配置

1、创建一个conf文件: vim tcp_to_es.conf

// 监听一个4660的TCP端口接收数据
input {
  tcp {
    host => "localhost"
    port => 4660
    mode => "server"
    tags => ["burukeyou"]
    codec => json_lines
  }
}

output {
  // 输出到ES
  elasticsearch {
    // 输出的es地址
    hosts => ["http://localhost:9200"]
    // 输出到的索引, 如果不存在会自动创建名为"log-'appName'-'当日日期'"的索引
    index => "log-%{[appName]}-%{+YYYY.MM.dd}"
    #user => "elastic"
    #password => "changeme"
  }
  
  // 以rubydebug格式输出到Logstash的控制台
  stdout { codec => rubydebug }
}
复制代码

2、启动logstash

启动后会默认再ES中创建一个 logstash-* 的索引模版

bin/logstash -f ../config/tcp_to_es.conf
复制代码

3、启动SpringBoot并打印日志

@Slf4j
@Component
public class LogCreateListener implements ApplicationListener<ApplicationStartedEvent> {

    @Override
    public void onApplicationEvent(ApplicationStartedEvent event) {
        for (int i = 0; i < 10; i++) {
            log.info("日志信息: {}", i);
        }
    }
}
复制代码