logstash 配置记录

logstash 常用配置记录

读取 kafka 写文件, kafka --> logstash --> file

input {
 kafka {
 bootstrap_servers => ["192.168.1.11:9092"]
 auto_offset_reset => "earliest"
 consumer_threads => 5
 security_protocol => "PLAINTEXT"
 topics_pattern => "log_18((3[1-9])|([456][0-9]))"
 group_id => "log_local_82_0801_3"
 max_poll_records => "100"
 connections_max_idle_ms => 180000
 codec => json
 }
}

filter {
 grok {

   match => {
   "source" => ["Server(?<server>[0-9]+)\\([a-zA-Z_]+)\\([a-zA-Z_]+)\\Year_(?<year>[0-9]+)\\Month_(?<month>[0-9]+)\\(?<day>[a-zA-Z0-9\-_]+)\\(?<filename>[a-zA-Z0-9\-_]+\.log$)"]
   }
 }

# mutate {
# windows 路径 \ 替换成 linux 路径 /
# convert => { "\\" => "/" }
# }

}

output {

#stdout {
 # codec => rubydebug
 #}

 file {
  path => "/data/logs/%{year}%{month}%{day}/%{pid}/server%{server}/%{filename}"
  codec => line {format => "%{message}"}

 }

}