运维

日志收集器Logstash与Filebeat部署配置指南

本文详细介绍了Logstash作为独立日志收集器和Filebeat日志传输器的两种部署方案，包含Docker容器化部署、Kubernetes集群部署配置模板以及多行日志解析等核心功能实现。

English
这篇文章已发布 744 天，部分内容可能已过时。如有疑问，可在评论区留言。

日志格式

1

2024-01-29 16:11:11.189 |INFO | 1.1.1.1|2345 | com.smart.service.receive.impl.ReceiveServiceImpl:903 | 能力>总共04步 | 6df2f14fca4b40f6be89b9ef19382c42adasfasf

logstash

docker方式部署 

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13

[root@master logstash]# cat docker-compose.yaml

version: '3'
services:
  logstash:
    image: docker.elastic.co/logstash/logstash:8.12.0
    container_name: logstash
    volumes:
      - ./conf/logstash.yml:/usr/share/logstash/config/logstash.yml
      - ./conf/conf.d:/usr/share/logstash/config/conf.d/
      - ./logs:/opt
    ports:
      - 5044:5044

配置文件

logstash.yml

1
2
3

http.host: "0.0.0.0"
xpack.monitoring.elasticsearch.hosts: [ "http://192.168.142.106:9200" ]
path.config: /usr/share/logstash/config/conf.d/*.conf

collect.conf

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53

input {
    file {
       type => "info_log"
       path => "/opt/kaikai.log"
       discover_interval => 10 # 监听间隔
       start_position => "end"
       # sincedb_path => "/usr/share/logstash/sincedb_kaikai"
       #start_position => "beginning"
       codec => multiline {
           pattern => "^%{TIMESTAMP_ISO8601}"
           negate => true
           what => "previous"
       }
    }

    file {
      type => "error_log"
      path => "/opt/error.log"
      discover_interval => 10
      start_position => "beginning"
      codec => multiline {
          pattern => "^%{TIMESTAMP_ISO8601}"
          negate => true
          what => "previous"
      }
    }
}

filter {

    grok {
      match => { "[log][file][path]" => "/(?<logfilename>[^/]+)\.log$" }  # 获取文件名logfilename
    }
    grok {
        match => { "message" => "%{DATA:time}\|%{DATA:level}\|%{DATA:ip}\|%{DATA:pid}\|%{DATA:source}\|%{GREEDYDATA:content}"}
    }
    if "_grokparsefailure" in [tags] {
        mutate {
            add_field => { "content" => "%{message}" }
            add_field => { "level" => "ERROR" }
        }
    }

}

output {
    stdout { codec => rubydebug }
    elasticsearch {
        hosts => ["192.168.142.106:9200"]
        index => "%{logfilename}-%{+YYYY-MM-dd}"   # 以文件名为索引
    }

}

k8s部署

作为filebeat接收收集器日志处理 logstash.yaml

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107

apiVersion: v1
kind: ConfigMap
metadata:
  name: log-file-config
data:
  logstash.yml: |
    http.host: "0.0.0.0"
    xpack.monitoring.elasticsearch.hosts: [ "http://192.168.142.106:9200" ]
    #xpack.monitoring.elasticsearch.hosts: [ "http://192.168.142.106:9200" ]
    path.config: /usr/share/logstash/config/conf.d/*.conf
  collect.conf: |
    input {
        beats {
            port => 5044
        }
    }

    filter {

        grok {
            match => { "[log][file][path]" => ["/(?<logfilename>[^/]+)\.log$"] }
        }
        grok {
            match => { "message" => "%{DATA:time}\|%{DATA:level}\|%{DATA:ip}\|%{DATA:pid}\|%{DATA:source}\|%{GREEDYDATA:content}" }
        }
        if "_grokparsefailure" in [tags] {
            mutate {
                add_field => { "content" => "%{message}" }
                add_field => { "level" => "ERROR" }
            }
        }


    }

    output {
        stdout { codec => rubydebug }
        elasticsearch {
            hosts => ["192.168.142.106:9200"]
            index => "%{logfilename}-%{+YYYY-MM-dd}"
        }


    }
---
kind: Deployment
apiVersion: apps/v1
metadata:
  name: logstash
  labels:
    app: logstash
spec:
  replicas: 4
  selector:
    matchLabels:
      app: logstash
  template:
    metadata:
      labels:
        app: logstash
      annotations:
        appName: logstash
        appType: java
    spec:
      containers:

        - name: logstash-logging
          image: registry.cn-beijing.aliyuncs.com/kaikai136/logstash:8.12.0
          volumeMounts:
          - name: logstash-config
            mountPath: /usr/share/logstash/config/logstash.yml
            subPath: logstash.yml
          - name: logstash-config
            mountPath: /usr/share/logstash/config/conf.d/collect.conf
            subPath: collect.conf

      volumes:
        - name: logstash-config
          configMap:
            name: log-file-config
            items:
              - key: logstash.yml
                path: logstash.yml
              - key: collect.conf
                path: collect.conf

      imagePullSecrets:
        - name: my-harbor


---
apiVersion: v1
kind: Service
metadata:
  name: logstash-svc
  labels:
    app: logstash-svc
spec:
  ports:
  - port: 5044
    targetPort: 5044
    protocol: TCP
    name: http
    nodePort: 32467
  type: NodePort
  selector:
    app: logstash

filebeat

收集器测试

filebeat.yaml

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123

apiVersion: v1
kind: ConfigMap
metadata:
  name: filebeat-config
data:
  filebeat.yml: |
    filebeat.inputs:
      - type: log
        enabled: true
        paths:
          - /logs/*_info.log
        scan_frequency: 1s  # 将扫描频率设置为1秒
        harvester_buffer_size: 32768  # 增加harvester的缓冲区大小
        backoff_factor: 2
        ignore_older: 24h  # 忽略超过24小时未修改的文件
        close_inactive: 5m  # 关闭5分钟内无活动的harvester
        clean_inactive: 72h  # 清理超过72小时无活动的harvester
        close_removed: true  # 当文件被删除时关闭harvester
        clean_removed: true  # 清理被删除的harvester
        close_eof: true  # 当文件达到EOF时关闭harvester
        multiline.pattern: '^[0-9]{4}'  # 匹配多行日志
        multiline.negate: true
        multiline.match: after
        var.convert_timezone: true  # 转换时区
        encoding: UTF-8  # 设置编码
        fields:
          wisentIp: 0.0.0.0  # 添加自定义字段
          log_type: info_log
      - type: log
        enabled: true
        paths:
          - /logs/*_error.log
        scan_frequency: 1s  # 将扫描频率设置为1秒
        harvester_buffer_size: 32768  # 增加harvester的缓冲区大小
        backoff_factor: 2
        ignore_older: 24h  # 忽略超过24小时未修改的文件
        close_inactive: 5m  # 关闭5分钟内无活动的harvester
        clean_inactive: 72h  # 清理超过72小时无活动的harvester
        close_removed: true  # 当文件被删除时关闭harvester
        clean_removed: true  # 清理被删除的harvester
        close_eof: true  # 当文件达到EOF时关闭harvester
        multiline.pattern: '^[0-9]{4}'  # 匹配多行日志
        multiline.negate: true
        multiline.match: after
        var.convert_timezone: true  # 转换时区
        encoding: UTF-8  # 设置编码
        fields:
          wisentIp: 0.0.0.0  # 添加自定义字段
          log_type: error_log



    queue.mem:
      events: 4096  # 内存队列大小
      flush.min_events: 2048  # 最小刷新事件数
      flush.timeout: 1s  # 刷新超时

    #queue.disk:
    #  max_size: 1024mb  # 最大磁盘使用空间
    #  segment_size: 10mb  # 每个段的大小
    #  max_retries: 3  # 最大重试次数
    logging.level: debug
    filebeat.shutdown_timeout: 30s  # 在关闭Filebeat时，确保有足够的时间处理完当前的事件
    throttle: 5s  # 设置Filebeat在被节流之前等待的时间


    logging.level: info  # 设置日志级别为info以获得详细的运行信息
    logging.to_files: true
    logging.files:
      path:  /usr/share/filebeat/logs
      name: filebeat
      keepfiles: 7
      permissions: 0644

    output.logstash:
      hosts: ["logstash-svc.default.svc.cluster.local:5044"]
---

kind: Deployment
apiVersion: apps/v1
metadata:
  name: filebeat
  labels:
    app: filebeat
spec:
  replicas: 1
  selector:
    matchLabels:
      app: filebeat
  template:
    metadata:
      labels:
        app: filebeat
      annotations:
        appName: filebeat
        appType: java
    spec:
      containers:

        - name: filebeat-logging
          image: registry.cn-beijing.aliyuncs.com/kaikai136/filebeat:8.12.0
          volumeMounts:
          - name: filebeat-config
            mountPath: /usr/share/filebeat/filebeat.yml
            subPath: filebeat.yml
          - name: myhostpath
            mountPath: /logs

      volumes:
        - name: filebeat-config
          configMap:
            name: filebeat-config
            items:
              - key: filebeat.yml
                path: filebeat.yml
        - name: myhostpath
          hostPath:
            path: /opt/kaikai/file-logstash/filebeat_log
            type: DirectoryOrCreate


      imagePullSecrets:
        - name: my-harbor
© 2019 - 2026 Kang's Blog
面朝大海，春暖花开。
使用 Hugo 构建
主题 Stack 由 Jimmy 设计