Kubernetes配置 on Kang's Blog

日志收集器Logstash与Filebeat部署配置指南

Sat, 09 Mar 2024 00:00:00 +0000

日志格式

`1`	`2024-01-29 16:11:11.189 \|INFO \| 1.1.1.1\|2345 \| com.smart.service.receive.impl.ReceiveServiceImpl:903 \| 能力>总共04步 \| 6df2f14fca4b40f6be89b9ef19382c42adasfasf`

logstash

docker方式部署

[root@master logstash]# cat docker-compose.yaml

version: '3'
services:
  logstash:
    image: docker.elastic.co/logstash/logstash:8.12.0
    container_name: logstash
    volumes:
      - ./conf/logstash.yml:/usr/share/logstash/config/logstash.yml
      - ./conf/conf.d:/usr/share/logstash/config/conf.d/
      - ./logs:/opt
    ports:
      - 5044:5044

配置文件

logstash.yml

1
2
3

http.host: "0.0.0.0"
xpack.monitoring.elasticsearch.hosts: [ "http://192.168.142.106:9200" ]
path.config: /usr/share/logstash/config/conf.d/*.conf

collect.conf

input {
    file {
       type => "info_log"
       path => "/opt/kaikai.log"
       discover_interval => 10 # 监听间隔
       start_position => "end"
       # sincedb_path => "/usr/share/logstash/sincedb_kaikai"
       #start_position => "beginning"
       codec => multiline {
           pattern => "^%{TIMESTAMP_ISO8601}"
           negate => true
           what => "previous"
       }
    }

    file {
      type => "error_log"
      path => "/opt/error.log"
      discover_interval => 10
      start_position => "beginning"
      codec => multiline {
          pattern => "^%{TIMESTAMP_ISO8601}"
          negate => true
          what => "previous"
      }
    }
}

filter {

    grok {
      match => { "[log][file][path]" => "/(?<logfilename>[^/]+)\.log$" }  # 获取文件名logfilename
    }
    grok {
        match => { "message" => "%{DATA:time}\|%{DATA:level}\|%{DATA:ip}\|%{DATA:pid}\|%{DATA:source}\|%{GREEDYDATA:content}"}
    }
    if "_grokparsefailure" in [tags] {
        mutate {
            add_field => { "content" => "%{message}" }
            add_field => { "level" => "ERROR" }
        }
    }

}

output {
    stdout { codec => rubydebug }
    elasticsearch {
        hosts => ["192.168.142.106:9200"]
        index => "%{logfilename}-%{+YYYY-MM-dd}"   # 以文件名为索引
    }

}

k8s部署

作为filebeat接收收集器日志处理 logstash.yaml

apiVersion: v1
kind: ConfigMap
metadata:
  name: log-file-config
data:
  logstash.yml: |
    http.host: "0.0.0.0"
    xpack.monitoring.elasticsearch.hosts: [ "http://192.168.142.106:9200" ]
    #xpack.monitoring.elasticsearch.hosts: [ "http://192.168.142.106:9200" ]
    path.config: /usr/share/logstash/config/conf.d/*.conf
  collect.conf: |
    input {
        beats {
            port => 5044
        }
    }

    filter {

        grok {
            match => { "[log][file][path]" => ["/(?<logfilename>[^/]+)\.log$"] }
        }
        grok {
            match => { "message" => "%{DATA:time}\|%{DATA:level}\|%{DATA:ip}\|%{DATA:pid}\|%{DATA:source}\|%{GREEDYDATA:content}" }
        }
        if "_grokparsefailure" in [tags] {
            mutate {
                add_field => { "content" => "%{message}" }
                add_field => { "level" => "ERROR" }
            }
        }


    }

    output {
        stdout { codec => rubydebug }
        elasticsearch {
            hosts => ["192.168.142.106:9200"]
            index => "%{logfilename}-%{+YYYY-MM-dd}"
        }


    }
---
kind: Deployment
apiVersion: apps/v1
metadata:
  name: logstash
  labels:
    app: logstash
spec:
  replicas: 4
  selector:
    matchLabels:
      app: logstash
  template:
    metadata:
      labels:
        app: logstash
      annotations:
        appName: logstash
        appType: java
    spec:
      containers:

        - name: logstash-logging
          image: registry.cn-beijing.aliyuncs.com/kaikai136/logstash:8.12.0
          volumeMounts:
          - name: logstash-config
            mountPath: /usr/share/logstash/config/logstash.yml
            subPath: logstash.yml
          - name: logstash-config
            mountPath: /usr/share/logstash/config/conf.d/collect.conf
            subPath: collect.conf

      volumes:
        - name: logstash-config
          configMap:
            name: log-file-config
            items:
              - key: logstash.yml
                path: logstash.yml
              - key: collect.conf
                path: collect.conf

      imagePullSecrets:
        - name: my-harbor


---
apiVersion: v1
kind: Service
metadata:
  name: logstash-svc
  labels:
    app: logstash-svc
spec:
  ports:
  - port: 5044
    targetPort: 5044
    protocol: TCP
    name: http
    nodePort: 32467
  type: NodePort
  selector:
    app: logstash

filebeat

收集器测试

filebeat.yaml

apiVersion: v1
kind: ConfigMap
metadata:
  name: filebeat-config
data:
  filebeat.yml: |
    filebeat.inputs:
      - type: log
        enabled: true
        paths:
          - /logs/*_info.log
        scan_frequency: 1s  # 将扫描频率设置为1秒
        harvester_buffer_size: 32768  # 增加harvester的缓冲区大小
        backoff_factor: 2
        ignore_older: 24h  # 忽略超过24小时未修改的文件
        close_inactive: 5m  # 关闭5分钟内无活动的harvester
        clean_inactive: 72h  # 清理超过72小时无活动的harvester
        close_removed: true  # 当文件被删除时关闭harvester
        clean_removed: true  # 清理被删除的harvester
        close_eof: true  # 当文件达到EOF时关闭harvester
        multiline.pattern: '^[0-9]{4}'  # 匹配多行日志
        multiline.negate: true
        multiline.match: after
        var.convert_timezone: true  # 转换时区
        encoding: UTF-8  # 设置编码
        fields:
          wisentIp: 0.0.0.0  # 添加自定义字段
          log_type: info_log
      - type: log
        enabled: true
        paths:
          - /logs/*_error.log
        scan_frequency: 1s  # 将扫描频率设置为1秒
        harvester_buffer_size: 32768  # 增加harvester的缓冲区大小
        backoff_factor: 2
        ignore_older: 24h  # 忽略超过24小时未修改的文件
        close_inactive: 5m  # 关闭5分钟内无活动的harvester
        clean_inactive: 72h  # 清理超过72小时无活动的harvester
        close_removed: true  # 当文件被删除时关闭harvester
        clean_removed: true  # 清理被删除的harvester
        close_eof: true  # 当文件达到EOF时关闭harvester
        multiline.pattern: '^[0-9]{4}'  # 匹配多行日志
        multiline.negate: true
        multiline.match: after
        var.convert_timezone: true  # 转换时区
        encoding: UTF-8  # 设置编码
        fields:
          wisentIp: 0.0.0.0  # 添加自定义字段
          log_type: error_log



    queue.mem:
      events: 4096  # 内存队列大小
      flush.min_events: 2048  # 最小刷新事件数
      flush.timeout: 1s  # 刷新超时

    #queue.disk:
    #  max_size: 1024mb  # 最大磁盘使用空间
    #  segment_size: 10mb  # 每个段的大小
    #  max_retries: 3  # 最大重试次数
    logging.level: debug
    filebeat.shutdown_timeout: 30s  # 在关闭Filebeat时，确保有足够的时间处理完当前的事件
    throttle: 5s  # 设置Filebeat在被节流之前等待的时间


    logging.level: info  # 设置日志级别为info以获得详细的运行信息
    logging.to_files: true
    logging.files:
      path:  /usr/share/filebeat/logs
      name: filebeat
      keepfiles: 7
      permissions: 0644

    output.logstash:
      hosts: ["logstash-svc.default.svc.cluster.local:5044"]
---

kind: Deployment
apiVersion: apps/v1
metadata:
  name: filebeat
  labels:
    app: filebeat
spec:
  replicas: 1
  selector:
    matchLabels:
      app: filebeat
  template:
    metadata:
      labels:
        app: filebeat
      annotations:
        appName: filebeat
        appType: java
    spec:
      containers:

        - name: filebeat-logging
          image: registry.cn-beijing.aliyuncs.com/kaikai136/filebeat:8.12.0
          volumeMounts:
          - name: filebeat-config
            mountPath: /usr/share/filebeat/filebeat.yml
            subPath: filebeat.yml
          - name: myhostpath
            mountPath: /logs

      volumes:
        - name: filebeat-config
          configMap:
            name: filebeat-config
            items:
              - key: filebeat.yml
                path: filebeat.yml
        - name: myhostpath
          hostPath:
            path: /opt/kaikai/file-logstash/filebeat_log
            type: DirectoryOrCreate


      imagePullSecrets:
        - name: my-harbor

FRP 反向代理服务部署完整指南

Sun, 09 Jul 2023 00:00:00 +0000

frp 是什么？

frp 是一款高性能的反向代理应用，专注于内网穿透。它支持多种协议，包括 TCP、UDP、HTTP、HTTPS 等，并且具备 P2P 通信功能。使用 frp，您可以安全、便捷地将内网服务暴露到公网，通过拥有公网 IP 的节点进行中转。

为什么选择 frp？

通过在具有公网 IP 的节点上部署 frp 服务端，您可以轻松地将内网服务穿透到公网，并享受以下专业特性：

多种协议支持：客户端服务端通信支持 TCP、QUIC、KCP 和 Websocket 等多种协议。
TCP 连接流式复用：在单个连接上承载多个请求，减少连接建立时间，降低请求延迟。
代理组间的负载均衡。
端口复用：多个服务可以通过同一个服务端端口暴露。
P2P 通信：流量不必经过服务器中转，充分利用带宽资源。
客户端插件：提供多个原生支持的客户端插件，如静态文件查看、HTTPS/HTTP 协议转换、HTTP、SOCKS5 代理等，以便满足各种需求。
服务端插件系统：高度可扩展的服务端插件系统，便于根据自身需求进行功能扩展。
用户友好的 UI 页面：提供服务端和客户端的用户界面，使配置和监控变得更加方便。

frps

frp 的服务端，需要有公网 IP，开通12345端口。

docker-compose.yaml

  services:
    frps:
      image: fatedier/frps:v0.61.0
      container_name: frps
      restart: always
      working_dir: /etc/frps/
      command: ["-c", "./config.toml"]
      volumes:
        - /etc/localtime:/etc/localtime:ro
        - ./:/etc/frps/
      network_mode: host

config.toml

  bindAddr = "0.0.0.0"
  bindPort = 12345
  
  vhostHTTPPort = 12380
  vhostHTTPSPort = 12343
  
  webServer.addr = "0.0.0.0"
  webServer.port = 12375
  webServer.user = "admin"
  webServer.password = "******"
  
  enablePrometheus = true
  
  log.to = "./logs/frps.log"
  log.level = "debug"
  log.maxDays = 7
  log.disablePrintColor = false
  
  auth.method = "token"
  auth.token = "******"
  
  custom404Page = "./404.html"

404.html

  <!DOCTYPE html>
  <html lang="en">
  <head>
      <meta charset="UTF-8">
      <meta name="viewport" content="width=device-width, initial-scale=1.0">
      <title>Not Found</title>
      <style>
          body {
              font-family: Arial, sans-serif;
              text-align: center;
              margin-top: 100px;
          }
          h1 {
              font-size: 50px;
              margin-bottom: 10px;
          }
          p {
              font-size: 20px;
              margin-top: 0;
          }
      </style>
  </head>
  <body>
  
      <h1>404</h1>
      <p>Page not found</p>
      
  </body>
  </html>

frpc

docker 部署

docker-compose.yaml

  services:
    net-tunnel:
      image: "harbor.example.com/devops/kube-net-tunnel:standalone"
      container_name: "net-tunnel"
      restart: "always"
      network_mode: "bridge"
      working_dir: /etc/net-tunnel/
      volumes:
        - "/etc/localtime:/etc/localtime:ro"
        - "./:/etc/net-tunnel/"
      command: ["-c", "/etc/net-tunnel/config.toml"]

config.toml

  user = "test"
  
  serverAddr = "*.*.*.*"
  serverPort = 12345
  
  auth.method = "token"
  auth.token = "******"
  
  log.to = "./logs/frpc.log"
  log.level = "debug"
  log.maxDays = 3
  log.disablePrintColor = false
  
  webServer.addr = "127.0.0.1"
  webServer.port = 12346
  webServer.user = "admin"
  webServer.password = "******"
  
  [[proxies]]
  name = "admin_ui"
  type = "tcp"
  localIP = "127.0.0.1"
  localPort = 12346
  remotePort = 12440
  
  [[proxies]]
  name = "ssh"
  type = "tcp"
  localIP = "172.17.0.1" # Windows or MacOS: "host.docker.internal"
  localPort = 22
  remotePort = 12441

K8s 部署

kube-net-tunnel.yaml

  apiVersion: v1
  kind: ConfigMap
  metadata:
    name: kube-net-tunnel-config
    namespace: kube-system
  data:
    config.toml: |
      user="test"
      serverAddr = "*.*.*.*"
      serverPort = 12345
  
      auth.method = "token"
      auth.token = "******"
  
      log.to = "console"
      log.level = "debug"
      log.maxDays = 3
      log.disablePrintColor = false
  
      webServer.addr = "127.0.0.1"
      webServer.port = 7400
      webServer.user = "admin"
      webServer.password = "******"
  
      [[proxies]]
      name = "admin"
      type = "tcp"
      localIP = "127.0.0.1"
      localPort = 7400
      remotePort = 12431
  
      [[proxies]]
      name = "ssh"
      type = "tcp"
      localIP = "10.19.31.7"
      localPort = 22
      remotePort = 12432
  
  
  ---
  apiVersion: apps/v1
  kind: Deployment
  metadata:
    name: kube-net-tunnel
    namespace: kube-system
  spec:
    replicas: 1
    selector:
      matchLabels:
        app: kube-net-tunnel
    template:
      metadata:
        labels:
          app: kube-net-tunnel
      spec:
        containers:
          - name: kube-net-tunnel
            image: harbor.example.com/devops/kube-net-tunnel:standalone
            imagePullPolicy: IfNotPresent
            args: ["-c", "/etc/kube-net-tunnel/config.toml"]
            volumeMounts:
              - name: config-volume
                mountPath: /etc/kube-net-tunnel/config.toml
                subPath: config.toml
        volumes:
          - name: config-volume
            configMap:
              defaultMode: 0755
              name: kube-net-tunnel-config
              items:
                - key: config.toml
                  path: config.toml