Kubernetes Configuration on Kang's Blog

Log Collector Configuration Guide for Logstash and Filebeat

Sat, 09 Mar 2024 00:00:00 +0000

Log Format

`1`	`2024-01-29 16:11:11.189 \|INFO \| 1.1.1.1\|2345 \| com.smart.service.receive.impl.ReceiveServiceImpl:903 \| Capability>Total 04 steps \| 6df2f14fca4b40f6be89b9ef19382c42adasfasf`

Logstash Configuration Example

docker deployment

[root@master logstash]# cat docker-compose.yaml

version: '3'
services:
  logstash:
    image: docker.elastic.co/logstash/logstash:8.12.0
    container_name: logstash
    volumes:
      - ./conf/logstash.yml:/usr/share/logstash/config/logstash.yml
      - ./conf/conf.d:/usr/share/logstash/config/conf.d/
      - ./logs:/opt
    ports:
      - 5044:5044

config files

logstash.yml

1
2
3

http.host: "0.0.0.0"
xpack.monitoring.elasticsearch.hosts: [ "http://192.168.142.106:9200" ]
path.config: /usr/share/logstash/config/conf.d/*.conf

input {
    file {
       type => "info_log"
       path => "/opt/kaikai.log"
       discover_interval => 10 # listen interval
       start_position => "end"
       # sincedb_path => "/usr/share/logstash/sincedb_kaikai"
       #start_position => "beginning"
       codec => multiline {
           pattern => "^%{TIMESTAMP_ISO8601}"
           negate => true
           what => "previous"
       }
    }

    file {
      type => "error_log"
      path => "/opt/error.log"
      discover_interval => 10
      start_position => "beginning"
      codec => multiline {
          pattern => "^%{TIMESTAMP_ISO8601}"
          negate => true
          what => "previous"
      }
    }
}

filter {

    grok {
      match => { "[log][file][path]" => "/(?<logfilename>[^/]+)\.log$" }  # get file name logfilename
    }
    grok {
        match => { "message" => "%{DATA:time}\|%{DATA:level}\|%{DATA:ip}\|%{DATA:pid}\|%{DATA:source}\|%{GREEDYDATA:content}"}
    }
    if "_grokparsefailure" in [tags] {
        mutate {
            add_field => { "content" => "%{message}" }
            add_field => { "level" => "ERROR" }
        }
    }

}

output {
    stdout { codec => rubydebug }
    elasticsearch {
        hosts => ["192.168.142.106:9200"]
        index => "%{logfilename}-%{+YYYY-MM-dd}"   # index by file name
    }

}

k8s deployment

logstash.yaml

apiVersion: v1
kind: ConfigMap
metadata:
  name: log-file-config
data:
  logstash.yml: |
    http.host: "0.0.0.0"
    xpack.monitoring.elasticsearch.hosts: [ "http://192.168.142.106:9200" ]
    #xpack.monitoring.elasticsearch.hosts: [ "http://192.168.142.106:9200" ]
    path.config: /usr/share/logstash/config/conf.d/*.conf
  collect.conf: |
    input {
        beats {
            port => 5044
        }
    }

    filter {

        grok {
            match => { "[log][file][path]" => ["/(?<logfilename>[^/]+)\.log$"] }
        }
        grok {
            match => { "message" => "%{DATA:time}\|%{DATA:level}\|%{DATA:ip}\|%{DATA:pid}\|%{DATA:source}\|%{GREEDYDATA:content}" }
        }
        if "_grokparsefailure" in [tags] {
            mutate {
                add_field => { "content" => "%{message}" }
                add_field => { "level" => "ERROR" }
            }
        }


    }

    output {
        stdout { codec => rubydebug }
        elasticsearch {
            hosts => ["192.168.142.106:9200"]
            index => "%{logfilename}-%{+YYYY-MM-dd}"
        }


    }
---
kind: Deployment
apiVersion: apps/v1
metadata:
  name: logstash
  labels:
    app: logstash
spec:
  replicas: 4
  selector:
    matchLabels:
      app: logstash
  template:
    metadata:
      labels:
        app: logstash
      annotations:
        appName: logstash
        appType: java
    spec:
      containers:

        - name: logstash-logging
          image: registry.cn-beijing.aliyuncs.com/kaikai136/logstash:8.12.0
          volumeMounts:
          - name: logstash-config
            mountPath: /usr/share/logstash/config/logstash.yml
            subPath: logstash.yml
          - name: logstash-config
            mountPath: /usr/share/logstash/config/conf.d/collect.conf
            subPath: collect.conf

      volumes:
        - name: logstash-config
          configMap:
            name: log-file-config
            items:
              - key: logstash.yml
                path: logstash.yml
              - key: collect.conf
                path: collect.conf

      imagePullSecrets:
        - name: my-harbor


---
apiVersion: v1
kind: Service
metadata:
  name: logstash-svc
  labels:
    app: logstash-svc
spec:
  ports:
  - port: 5044
    targetPort: 5044
    protocol: TCP
    name: http
    nodePort: 32467
  type: NodePort
  selector:
    app: logstash

filebeat

collector test

filebeat.yaml

apiVersion: v1
kind: ConfigMap
metadata:
  name: filebeat-config
data:
  filebeat.yml: |
    filebeat.inputs:
      - type: log
        enabled: true
        paths:
          - /logs/*_info.log
        scan_frequency: 1s  # set scan frequency to 1 second
        harvester_buffer_size: 32768  # increase harvester buffer size
        backoff_factor: 2
        ignore_older: 24h  # ignore files older than 24 hours
        close_inactive: 5m  # close harvester inactive for 5 minutes
        clean_inactive: 72h  # clean inactive harvester after 72 hours
        close_removed: true  # close harvester when file is removed
        clean_removed: true  # clean removed harvester
        close_eof: true  # close harvester when file reaches EOF
        multiline.pattern: '^[0-9]{4}'  # match multiline logs
        multiline.negate: true
        multiline.match: after
        var.convert_timezone: true  # convert timezone
        encoding: UTF-8  # set encoding
        fields:
          wisentIp: 0.0.0.0  # add custom field
          log_type: info_log
      - type: log
        enabled: true
        paths:
          - /logs/*_error.log
        scan_frequency: 1s  # set scan frequency to 1 second
        harvester_buffer_size: 32768  # increase harvester buffer size
        backoff_factor: 2
        ignore_older: 24h  # ignore files older than 24 hours
        close_inactive: 5m  # close harvester inactive for 5 minutes
        clean_inactive: 72h  # clean inactive harvester after 72 hours
        close_removed: true  # close harvester when file is removed
        clean_removed: true  # clean removed harvester
        close_eof: true  # close harvester when file reaches EOF
        multiline.pattern: '^[0-9]{4}'  # match multiline logs
        multiline.negate: true
        multiline.match: after
        var.convert_timezone: true  # convert timezone
        encoding: UTF-8  # set encoding
        fields:
          wisentIp: 0.0.0.0  # add custom field
          log_type: error_log



    queue.mem:
      events: 4096  # memory queue size
      flush.min_events: 2048  # minimum flush events
      flush.timeout: 1s  # flush timeout

    #queue.disk:
    #  max_size: 1024mb  # maximum disk usage
    #  segment_size: 10mb  # size of each segment
    #  max_retries: 3  # maximum retries
    logging.level: debug
    filebeat.shutdown_timeout: 30s  # ensure enough time to process current events when shutting down Filebeat
    throttle: 5s  # set the time Filebeat waits before being throttled


    logging.level: info  # set logging level to info for detailed run information
    logging.to_files: true
    logging.files:
      path:  /usr/share/filebeat/logs
      name: filebeat
      keepfiles: 7
      permissions: 0644

    output.logstash:
      hosts: ["logstash-svc.default.svc.cluster.local:5044"]
---

kind: Deployment
apiVersion: apps/v1
metadata:
  name: filebeat
  labels:
    app: filebeat
spec:
  replicas: 1
  selector:
    matchLabels:
      app: filebeat
  template:
    metadata:
      labels:
        app: filebeat
      annotations:
        appName: filebeat
        appType: java
    spec:
      containers:

        - name: filebeat-logging
          image: registry.cn-beijing.aliyuncs.com/kaikai136/filebeat:8.12.0
          volumeMounts:
          - name: filebeat-config
            mountPath: /usr/share/filebeat/filebeat.yml
            subPath: filebeat.yml
          - name: myhostpath
            mountPath: /logs

      volumes:
        - name: filebeat-config
          configMap:
            name: filebeat-config
            items:
              - key: filebeat.yml
                path: filebeat.yml
        - name: myhostpath
          hostPath:
            path: /opt/kaikai/file-logstash/filebeat_log
            type: DirectoryOrCreate


      imagePullSecrets:
        - name: my-harbor

Complete Guide to FRP Reverse Proxy Deployment

Sun, 09 Jul 2023 00:00:00 +0000

What is frp?

frp is a high-performance reverse proxy application specialized in intranet penetration. It supports multiple protocols including TCP, UDP, HTTP, HTTPS, and features P2P communication capabilities. Using frp, you can securely and conveniently expose internal network services to the public network through relay nodes with public IP addresses.

Why choose frp?

By deploying an frp server on a node with a public IP, you can easily penetrate internal services to the public network while enjoying these professional features:

Multi-protocol Support: Client-server communication supports TCP, QUIC, KCP, Websocket, and other protocols.
TCP Connection Stream Multiplexing: Carry multiple requests over a single connection to reduce connection setup time and lower request latency.
Load Balancing between proxy groups.
Port Multiplexing: Multiple services can be exposed through the same server port.
P2P Communication: Traffic bypasses server relay, maximizing bandwidth utilization.
Client Plugins: Provides natively supported client plugins for static file viewing, HTTPS/HTTP protocol conversion, HTTP/SOCKS5 proxies, and more.
Server Plugin System: Highly extensible plugin system for custom functional expansion.
User-Friendly UI: Server and client interfaces for simplified configuration and monitoring.

frps

FRPS is the server part of frp, which needs a public IP and opens port 12345.

docker-compose.yaml

  services:
    frps:
      image: fatedier/frps:v0.61.0
      container_name: frps
      restart: always
      working_dir: /etc/frps/
      command: ["-c", "./config.toml"]
      volumes:
        - /etc/localtime:/etc/localtime:ro
        - ./:/etc/frps/
      network_mode: host

config.toml

  bindAddr = "0.0.0.0"
  bindPort = 12345
  
  vhostHTTPPort = 12380
  vhostHTTPSPort = 12343
  
  webServer.addr = "0.0.0.0"
  webServer.port = 12375
  webServer.user = "admin"
  webServer.password = "******"
  
  enablePrometheus = true
  
  log.to = "./logs/frps.log"
  log.level = "debug"
  log.maxDays = 7
  log.disablePrintColor = false
  
  auth.method = "token"
  auth.token = "******"
  
  custom404Page = "./404.html"

404.html

  <!DOCTYPE html>
  <html lang="en">
  <head>
      <meta charset="UTF-8">
      <meta name="viewport" content="width=device-width, initial-scale=1.0">
      <title>Not Found</title>
      <style>
          body {
              font-family: Arial, sans-serif;
              text-align: center;
              margin-top: 100px;
          }
          h1 {
              font-size: 50px;
              margin-bottom: 10px;
          }
          p {
              font-size: 20px;
              margin-top: 0;
          }
      </style>
  </head>
  <body>
  
      <h1>404</h1>
      <p>Page not found</p>
      
  </body>
  </html>

frpc

docker deployment

docker-compose.yaml

  services:
    net-tunnel:
      image: "harbor.example.com/devops/kube-net-tunnel:standalone"
      container_name: "net-tunnel"
      restart: "always"
      network_mode: "bridge"
      working_dir: /etc/net-tunnel/
      volumes:
        - "/etc/localtime:/etc/localtime:ro"
        - "./:/etc/net-tunnel/"
      command: ["-c", "/etc/net-tunnel/config.toml"]

config.toml

  user = "test"
  
  serverAddr = "*.*.*.*"
  serverPort = 12345
  
  auth.method = "token"
  auth.token = "******"
  
  log.to = "./logs/frpc.log"
  log.level = "debug"
  log.maxDays = 3
  log.disablePrintColor = false
  
  webServer.addr = "127.0.0.1"
  webServer.port = 12346
  webServer.user = "admin"
  webServer.password = "******"
  
  [[proxies]]
  name = "admin_ui"
  type = "tcp"
  localIP = "127.0.0.1"
  localPort = 12346
  remotePort = 12440
  
  [[proxies]]
  name = "ssh"
  type = "tcp"
  localIP = "172.17.0.1" # Windows or MacOS: "host.docker.internal"
  localPort = 22
  remotePort = 12441

K8s deployment

kube-net-tunnel.yaml

apiVersion: v1
kind: ConfigMap
metadata:
  name: kube-net-tunnel-config
  namespace: kube-system
data:
  config.toml: |
    user="test"
    serverAddr = "*.*.*.*"
    serverPort = 12345

    auth.method = "token"
    auth.token = "******"

    log.to = "console"
    log.level = "debug"
    log.maxDays = 3
    log.disablePrintColor = false

    webServer.addr = "127.0.0.1"
    webServer.port = 7400
    webServer.user = "admin"
    webServer.password = "******"

    [[proxies]]
    name = "admin"
    type = "tcp"
    localIP = "127.0.0.1"
    localPort = 7400
    remotePort = 12431

    [[proxies]]
    name = "ssh"
    type = "tcp"
    localIP = "10.19.31.7"
    localPort = 22
    remotePort = 12432


---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: kube-net-tunnel
  namespace: kube-system
spec:
  replicas: 1
  selector:
    matchLabels:
      app: kube-net-tunnel
  template:
    metadata:
      labels:
        app: kube-net-tunnel
    spec:
      containers:
        - name: kube-net-tunnel
          image: harbor.example.com/devops/kube-net-tunnel:standalone
          imagePullPolicy: IfNotPresent
          args: ["-c", "/etc/kube-net-tunnel/config.toml"]
          volumeMounts:
            - name: config-volume
              mountPath: /etc/kube-net-tunnel/config.toml
              subPath: config.toml
      volumes:
        - name: config-volume
          configMap:
            defaultMode: 0755
            name: kube-net-tunnel-config
            items:
              - key: config.toml
                path: config.toml