1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33

docker pull centos
docker image ls


REPOSITORY          TAG                 IMAGE ID            CREATED             SIZE
centos              latest              e934aafc2206        2 weeks ago         199MB



#运行docker
#此处如果没有/bin/bash的话，生成的container立马就停止了
#端口映射什么的后面再弄
docker run -d -i <image-id> /bin/bash



#查看容器
docker container ls

CONTAINER ID        IMAGE               COMMAND             CREATED             STATUS              PORTS               NAMES
27b10f5015be        e934aafc2206        "/bin/bash"         About an hour ago   Up About an hour                        ecstatic_boyd



#进入docker
docke exec -it <container-id> /bin/bash
#当然，你也可以运行SSHD，通过端口映射，连接到docker内

#[root@27b10f5015be /]#


#在docker中安装各类需要的软件了
#可能需要设置一下/etc/resolv.conf

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35

#从运行的容器中重构镜像
#docker commit -m "centos7+elk" <container-id> user/repo:tag
docker commit -m 'centos7+elk' 27b10f5015be zhang21/centos7:elk



#查看新镜像
docker image ls

REPOSITORY          TAG                 IMAGE ID            CREATED             SIZE
zhang21/centos7     elk                 0b22d93f7353        16 minutes ago      1.04GB
centos              latest              e934aafc2206        2 weeks ago         199MB


#运行新镜像
docker run -id -p 80:80 9200:9200 <image-id> /bin/bash

#此处遇到一个错，因为docker的网络是通过iptables来转发的，因此主机上不能关闭firewalld，不能无法启动容器

#进入新容器
docker exec -it <container-id> /bin/bash

#此处无法使用systemctl，原因已写到前面
#Failed to get D-Bus connection: Operation not permitted


#获得systemd权限启动
docker run -id --privileged -p 80:80 <image-id> /usr/sbin/init

#进入
docker exec -it <container-id> /bin/bash

#启动Nginx
systemctl start nginx

1
2
3
4
5
6
7
8

docker login -u zhang21

#上传镜像到我的Hub
docker push zhang21/centos7:elk


#拉取镜像
docker pull zhang21/centos7:elk

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22

#安装Java
yum install java-1.8.0-openjdk-headless-1.8.0.161-0.b14.el7_4.x86_64 -y



#编写repo
vim /etc/yum.repo.d/elk.repo


[elasticsearch-6.x]
name=Elasticsearch repository for 6.x packages
baseurl=https://artifacts.elastic.co/packages/6.x/yum
gpgcheck=1
gpgkey=https://artifacts.elastic.co/GPG-KEY-elasticsearch
enabled=1
autorefresh=1
type=rpm-md



#安装
yum install -y elasticsearch logstash kibana filebeat

 1
 2
 3
 4
 5
 6
 7
 8
 9
10

vim /etc/profile


export PATH=$PATH:/usr/share/elasticsearch/bin:/usr/share/kibana/bin:/usr/share/logstash/bin:/usr/share/elasticsearch/bin/x-pack:/usr/share/filebeat/bin



#执行
source /etc/profile

1
2
3
4
5
6

elasticsearch-plugin list
elasticsearch-plugin remove x-pack

kibana-plugin remove x-pack

logstash-plugin remove x-pack

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30

#xpack.graph.enabled
#xpack.ml.enabled
#xpack.monitoring.enabled
#xpack.reporting.enabled
#xpack.security.enabled
#xpack.watcher.enabled


#在Elasticsearch和kibana中禁用验证后，不用在logstash中输入，否则会报错。
xpack.security.enabled: false



#启用验证
#具体可参考官方文档

#在logstash.yml中配置xpack.monitoring
xpack.monitoring.enabled: true
#xpack.monitoring.elasticsearch.url: "http://127.0.0.1:9200"
#xpack.monitoring.elasticsearch.username: logstash_system
#xpack.monitoring.elasticsearch.password: logstash


#在Filebeat中添加monitoring
xpack.monitoring:
  enabled: true
  #elasticsearch:
        #url: "http://localhost:9200"
		#usernaem: "elastic"
		#password: "elastic"

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62

#Elastcisearch安装X-Pack
elasticsearch-plugin install x-pack


#启动
#9200, 9300端口
#elasticsearch不能使用root启动，所以我把elastic用户修改为/bin/bash
su elasticsearch
elasticsearch -d
#elasticsearch



#生成默认用户密码，此密码针对elastic和kibana用户
#/usr/share/elasticsearch/bin/x-pack
#将此加入PATH
setup-passwords auto

#或手动输入密码
setup-passwords interactive
elastic
#elastic
kibana
#kibana
logstash_system
#logstash




#Kibana安装X-Pack
kibana-plugin install x-pack


#对kibana.yml添加用户和密码
#此密码是前面默认生成的
vim /etc/kibana/kibana.yml`

elasticsearch.username: "elastic"
elasticsearch.password:  "elastic"
#修改监听地址
server.host: "0.0.0.0"

logging.dest: /var/log/kibana/kibana.log
#kibana日志默认是stdout
#修改为/var/log/kibana/kibana.log
mkdir /var/log/kibana


#启动kibana
#5601端口
#kibana可用root启动
kibana
#或
systemctl start kibana


#Logstash安装X-Pack
logstash-plugin install x-pack

1
2
3

#最便捷
systemctl start elasticsearch logstash filebeat metricbeat heartbeat packetbeat auditbeat

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22

#Elasticsearch
su elasticsearch
#elasticsearch，查看输出
elasticsearch -d
#kill -15 pid && elasticsearch -d


#Kibana
kibana&
systemctl start kibana
#kill -15 pid && kibana&


#Logstash
#logstash -f xxx.conf
systemctl start logstash


#Filebeat
#filebeat -e -c filebeat.yml，查看输出信息
systemctl start filebeat

1
2
3
4
5
6
7
8

vim /etc/security/limits.conf


* soft nofile 655350
* hard nofile 655350


ulimit -a

1
2
3
4
5
6

$ip:9200
#此处访问是需要用户名和密码的
#使用前面X-Pack生成的默认用户名和密码

elastic elastic
#登录之后便可看到node，cluster相关信息

1
2
3

#5601端口
http://0.0.0.0:5601

1
2
3
4
5
6
7
8
9

yum install -y filebeat


#开启X-Pack monitor
#默认关闭
vim /etc/filebeat/filebeat.yml

xpack.monitoring.enabled: true

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13

#最小
-Xms
#最大
-Xmx


vim /etc/elasticsearch/jvm.options

-Xms4g
-Xmx4g


#其它如此

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39

#安装Nginx

vim /etc/yum.repo.d/nginx.repo


[nginx]
name=nginx repo
baseurl=http://nginx.org/packages/centos/$releasever/$basearch/
gpgcheck=0
enabled=1


yum install -y nginx  nginx-mod-stream




#配置
vim /etc/nginx/conf.d/kibana.conf


#可把IP换成kibana相应的域名
#再将域名解析到此IP
server {

    listen 80;
    server_name 172.16.129.150;

#Kibana
    location / {
        proxy_pass http://127.0.0.1:5601;
        proxy_set_header Upgrade $http_upgrade;
        proxy_set_header Connection 'upgrade';
        proxy_set_header Host $host;
        proxy_cache_bypass $http_upgrade;
    }

}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12

sudo cat /var/log/audit/audit.log | grep nginx | grep denied

#后来判断是SELinux的问题
getenforce
setenforce 0


#修改SELinux
vim /etc/selinux/config

SELINUX=disabled

1
2
3
4
5
6

logstash -e 'input { stdin { } } output { stdout {} }'

#等待启动，输入hello world
#Logstash将时间戳和主机名添加到message
#2018-04-13T08:17:51.702Z zhang22 helloworld

1
2
3
4

cd /usr/share/logstash/bin

ln -s /etc/logstash ./config

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19

vim /etc/filebeat/filebeat.yml


filebeat.prospectors:
- type: log
#需要处理的日志的路径，如Nginx
  paths:
  	- /var/log/nginx/*.log

output.logstash:
  hosts: ["localhost:5044"]



#运行Filebeat
Filebeat -e -c filebeat.yml -d "publish"


#Filebeat将会尝试连接到5044端口，在Logstash以一个活动的beats plugin开始前，不会有任何应答。

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56

cd /etc/logstash/conf.d
vim ./first-pipeline.conf

input {

}

#filter部分可选
filter {

}

output {

}




#实例
input {
    beats {
    	port => "5044"
    }
}

output {
	stdout {
    	codec => dubydebug
    }
}




#验证配置
logstash -f first-pipe.conf --config.tst_and_exit


#消息
2018-04-17T14:15:46.187+0800    ERROR   pipeline/output.go:74   Failed to connect: dial tcp [::1]:5044: getsockopt: connection refused
2018-04-17T14:15:46.607+0800    INFO    log/harvester.go:241    File is inactive: /var/log/nginx/access.log. Closing because close_inactive of 5m0s reached.
2018-04-17T14:15:46.607+0800    INFO    log/harvester.go:241    File is inactive: /var/log/nginx/error.log. Closing because close_inactive of 5m0s reached.
2018-04-17T14:15:46.923+0800    INFO    log/harvester.go:241    File is inactive: /var/log/secure-20180409. Closing because close_inactive of 5m0s reached.
2018-04-17T14:15:51.096+0800    INFO    log/harvester.go:241    File is inactive: /var/log/secure-20180401. Closing because close_inactive of 5m0s reached.
2018-04-17T14:15:52.687+0800    INFO    log/harvester.go:241    File is inactive: /var/log/secure-20180415. Closing because close_inactive of 5m0s reached.


#启动Logstash
#修改配置后可动态载入
logstash -f first-pipe.conf --config.reload.automatic


#消息
2018-04-17T14:18:41.542+0800    INFO    [monitoring]    log/log.go:124  Non-zero metrics in the last 30s        {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":150,"time":159},"total":{"ticks":450,"time":468,"value":450},"user":{"ticks":300,"time":309}},"info":{"ephemeral_id":"84cbf5cd-dfff-4391-9631-2b8e77329696","uptime":{"ms":480009}},"memstats":{"gc_next":11030992,"memory_alloc":6588088,"memory_total":40882600}},"filebeat":{"harvester":{"open_files":5,"running":8}},"libbeat":{"config":{"module":{"running":2}},"pipeline":{"clients":8,"events":{"active":4118}}},"registrar":{"states":{"current":10}},"system":{"load":{"1":4.86,"15":4.41,"5":4.53,"norm":{"1":2.43,"15":2.205,"5":2.265}}}}}}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25

vim first-pipline.conf


input {
	beats {
    	port => "5044"
    }
}

filter {
	grok {
    	match => {
        	"message" => "%{COMBINEDAPACHELOG}"
        }
    }
}

output {
	stdout {
    	codec => rubydebug
    }
}


#启动查看效果

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22

vim first-pipeline.conf


input {
    beats {
        port => "5044"
    }
}
 filter {
    grok {
        match => { "message" => "%{COMBINEDAPACHELOG}"}
    }
    geoip {
        source => "clientip"
    }
}
output {
    stdout { codec => rubydebug }
}


#重启服务

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23

vim first-pipeline.conf


input {
    beats {
        port => "5044"
    }
}
 filter {
    grok {
        match => { "message" => "%{COMBINEDAPACHELOG}"}
    }
    geoip {
        source => "clientip"
    }
}
output {
    elasticsearch {
        hosts => [ "localhost:9200" ]
    }
}

#重启服务

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16

curl -XGET 'localhost:9200/logstash-$DATE/_search?pretty&q=response=200'

#索引名称使用的日期基于UTC，而不是Logstash正在运行的timezone


#查看可用索引列表
curl 'localhost:9200/_cat/indices?v'


health status index                           uuid                   pri rep docs.count docs.deleted store.size pri.store.size
yellow open   logstash-2018.04.13             dRW2veUgS2ObZmP3lepqsQ   5   1        154            0    266.6kb        266.6kb



curl -XGET 'localhost:9200/logstash-2018.04.13/_search?pretty&q=response=200'

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49

#配置Filebeat发送Log Line到Logstash
vim /etc/filebeat/filebeat.yml

filebeat.prospectors:
- type: log
  paths:
    - /var/log/*.log
  fields:
    type: syslog

output.logstash:
  hosts: ["localhost:5044"]


########################


cd /etc/logstash/conf.d

vim 2nd-pipeline.conf


input {
	twitter {
        consumer_key => "enter_your_consumer_key_here"
        consumer_secret => "enter_your_secret_here"
        keywords => ["cloud"]
        oauth_token => "enter_your_access_token_here"
        oauth_token_secret => "enter_your_access_token_secret_here"
    }
	beats {
    	prot => "5044"
    }
}

output {
	elasticsearch {
    	hosts => ["hosts1:port1", "host2:port2"...]
    }
    file {
    	path => "/path/to/target/file"
    }
}


#重启服务
#测试，Replace $DATE with the current date, in YYYY.MM.DD format.
curl -XGET 'localhost:9200/logstash-$DATE/_search?pretty&q=fields.type:syslog'

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16

input {
  beats {
    port => 5044
  }
}

output {
  elasticsearch {
    hosts => "localhost:9200"
	#hosts => ["hosts1", "hosts2", ...]
    manage_template => false
    index => "%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}"
    document_type => "%{[@metadata][type]}"
  }
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17

input {
  elasticsearch {
    hosts => "es.production.mysite.org"
    index => "mydata-2018.09.*"
    query => '{ "query": { "query_string": { "query": "*" } } }'
    size => 500
    scroll => "5m"
    docinfo => true
  }
}
output {
  elasticsearch {
    index => "copy-of-production.%{[@metadata][_index]}"
    document_type => "%{[@metadata][_type]}"
    document_id => "%{[@metadata][_id]}"
  }
}

1
2
3
4
5
6

input {
  exec {
    command => "ls"
    interval => 30
  }
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11

input {
  sqlite {
    path => "/tmp/example.db"
    type => weblogs
  }
}
output {
  stdout {
    debug => true
  }
}

1
2
3
4
5
6
7
8

input {
  syslog {
    port => 12345
    codec => cef
    syslog_field => "syslog"
    grok_pattern => "<%{POSINT:priority}>%{SYSLOGTIMESTAMP:timestamp} CUSTOM GROK HERE"
  }
}

1
2
3
4
5
6

input {
  tcp {
    port => 12345
    codec => json
  }
}

1
2
3
4
5
6
7

output {
  if [type] == "abuse" {
    exec {
      command => "iptables -A INPUT -s %{clientip} -j DROP"
    }
  }
}

1
2
3
4
5
6

output {
 file {
   path => ...
   codec => line { format => "custom format: %{message}"}
 }
}

1
2
3
4
5
6

output {
  kafka {
    codec => json
    topic_id => "mytopic"
  }
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11

output {
  redmine {
    url => "http://redmineserver.tld"
    token => 'token'
    project_id => 200
    tracker_id => 1
    status_id => 3
    priority_id => 2
    subject => "Error ... detected"
  }
}

1
2
3

output {
  stdout { codec => json }
}

1
2
3
4

 INFO - 12345 - TASK_START - start
 INFO - 12345 - SQL - sqlQuery1 - 12
 INFO - 12345 - SQL - sqlQuery2 - 34
 INFO - 12345 - TASK_END - end

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31

 filter {
   grok {
     match => [ "message", "%{LOGLEVEL:loglevel} - %{NOTSPACE:taskid} - %{NOTSPACE:logger} - %{WORD:label}( - %{INT:duration:int})?" ]
   }

   if [logger] == "TASK_START" {
     aggregate {
       task_id => "%{taskid}"
       code => "map['sql_duration'] = 0"
       map_action => "create"
     }
   }

   if [logger] == "SQL" {
     aggregate {
       task_id => "%{taskid}"
       code => "map['sql_duration'] += event.get('duration')"
       map_action => "update"
     }
   }

   if [logger] == "TASK_END" {
     aggregate {
       task_id => "%{taskid}"
       code => "event.set('sql_duration', map['sql_duration'])"
       map_action => "update"
       end_of_task => true
       timeout => 120
     }
   }
 }

1

logstash-plugin install logstash-filter-alter

1
2
3
4
5

match => [ "filed-name", "MMM dd yyyy HH:mm:ss",
          "MMM  d yyyy HH:mm:ss", "ISO8601" ]

嵌套字段表示
[foo][bar]

1
2
3

field: %{a}
delimiter: -
%{a} - %{b} - %{c}

1
2
3
4
5
6
7
8

filter {
  dissect {
    convert_datatype => {
      cpu => "float"
      code => "int"
    }
  }
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10

filter {
  dissect {
    mapping => {
      # using an actual line break
      "message" => '"%{field1}" "%{field2}"
 "%{description}"'
      "description" => "%{field3} %{field4} %{field5}"
    }
  }
}

1
2
3

filter {
  grok { match => { "message" => [ "Duration: %{NUMBER:duration}", "Speed: %{NUMBER:speed}" ] } }
}

1
2
3
4
5
6

filter {
  grok {
    match => { "message" => "%{SYSLOGBASE} %{DATA:message}" }
    overwrite => [ "message" ]
  }
}

1

patterns_dir => ["/opt/logstash/patterns", "/opt/logstash/extra_patterns"]

1

%{NUMBER:duration} %{IP:client}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16

日志格式
55.3.244.1 GET /index.html 15824 0.043


grok pattern
  grok {
    match => { "message" => "%{IP:client} %{WORD:method} %{URIPATHPARAM:request} %{NUMBER:bytes} %{NUMBER:duration}" }
  }


grok filter之后的格式
client: 55.3.244.1
method: GET
request: /index.html
bytes: 15824
duration: 0.043

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14

mkdir /etc/logstash/patterns

vim nginx


NGINX_ACCESS %{IPORHOST:clientip} (?:-|(%{WORD}.%{WORD})) %{USER:ident} \[%{HTTPDATE:timestamp}\] "(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})" %{NUMBER:response} (?:%{NUMBER:bytes}|-) %{QS:referrer} %{QS:agent} %{QS:forwarder}




grok {
  patterns_dir => "/etc/logstash/patterns"
  match => { "message" => %{NGINX_ACCESS}}
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96

USERNAME [a-zA-Z0-9._-]+
USER %{USERNAME}
EMAILLOCALPART [a-zA-Z][a-zA-Z0-9_.+-=:]+
EMAILADDRESS %{EMAILLOCALPART}@%{HOSTNAME}
INT (?:[+-]?(?:[0-9]+))
BASE10NUM (?<![0-9.+-])(?>[+-]?(?:(?:[0-9]+(?:\.[0-9]+)?)|(?:\.[0-9]+)))
NUMBER (?:%{BASE10NUM})
BASE16NUM (?<![0-9A-Fa-f])(?:[+-]?(?:0x)?(?:[0-9A-Fa-f]+))
BASE16FLOAT \b(?<![0-9A-Fa-f.])(?:[+-]?(?:0x)?(?:(?:[0-9A-Fa-f]+(?:\.[0-9A-Fa-f]*)?)|(?:\.[0-9A-Fa-f]+)))\b

POSINT \b(?:[1-9][0-9]*)\b
NONNEGINT \b(?:[0-9]+)\b
WORD \b\w+\b
NOTSPACE \S+
SPACE \s*
DATA .*?
GREEDYDATA .*
QUOTEDSTRING (?>(?<!\\)(?>"(?>\\.|[^\\"]+)+"|""|(?>'(?>\\.|[^\\']+)+')|''|(?>`(?>\\.|[^\\`]+)+`)|``))
UUID [A-Fa-f0-9]{8}-(?:[A-Fa-f0-9]{4}-){3}[A-Fa-f0-9]{12}
# URN, allowing use of RFC 2141 section 2.3 reserved characters
URN urn:[0-9A-Za-z][0-9A-Za-z-]{0,31}:(?:%[0-9a-fA-F]{2}|[0-9A-Za-z()+,.:=@;$_!*'/?#-])+

# Networking
MAC (?:%{CISCOMAC}|%{WINDOWSMAC}|%{COMMONMAC})
CISCOMAC (?:(?:[A-Fa-f0-9]{4}\.){2}[A-Fa-f0-9]{4})
WINDOWSMAC (?:(?:[A-Fa-f0-9]{2}-){5}[A-Fa-f0-9]{2})
COMMONMAC (?:(?:[A-Fa-f0-9]{2}:){5}[A-Fa-f0-9]{2})
IPV6 ((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:)))(%.+)?
IPV4 (?<![0-9])(?:(?:[0-1]?[0-9]{1,2}|2[0-4][0-9]|25[0-5])[.](?:[0-1]?[0-9]{1,2}|2[0-4][0-9]|25[0-5])[.](?:[0-1]?[0-9]{1,2}|2[0-4][0-9]|25[0-5])[.](?:[0-1]?[0-9]{1,2}|2[0-4][0-9]|25[0-5]))(?![0-9])
IP (?:%{IPV6}|%{IPV4})
HOSTNAME \b(?:[0-9A-Za-z][0-9A-Za-z-]{0,62})(?:\.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*(\.?|\b)
IPORHOST (?:%{IP}|%{HOSTNAME})
HOSTPORT %{IPORHOST}:%{POSINT}

# paths
PATH (?:%{UNIXPATH}|%{WINPATH})
UNIXPATH (/([\w_%!$@:.,+~-]+|\\.)*)+
TTY (?:/dev/(pts|tty([pq])?)(\w+)?/?(?:[0-9]+))
WINPATH (?>[A-Za-z]+:|\\)(?:\\[^\\?*]*)+
URIPROTO [A-Za-z]([A-Za-z0-9+\-.]+)+
URIHOST %{IPORHOST}(?::%{POSINT:port})?
# uripath comes loosely from RFC1738, but mostly from what Firefox
# doesn't turn into %XX
URIPATH (?:/[A-Za-z0-9$.+!*'(){},~:;=@#%&_\-]*)+
#URIPARAM \?(?:[A-Za-z0-9]+(?:=(?:[^&]*))?(?:&(?:[A-Za-z0-9]+(?:=(?:[^&]*))?)?)*)?
URIPARAM \?[A-Za-z0-9$.+!*'|(){},~@#%&/=:;_?\-\[\]<>]*
URIPATHPARAM %{URIPATH}(?:%{URIPARAM})?
URI %{URIPROTO}://(?:%{USER}(?::[^@]*)?@)?(?:%{URIHOST})?(?:%{URIPATHPARAM})?

# Months: January, Feb, 3, 03, 12, December
MONTH \b(?:[Jj]an(?:uary|uar)?|[Ff]eb(?:ruary|ruar)?|[Mm](?:a|ä)?r(?:ch|z)?|[Aa]pr(?:il)?|[Mm]a(?:y|i)?|[Jj]un(?:e|i)?|[Jj]ul(?:y)?|[Aa]ug(?:ust)?|[Ss]ep(?:tember)?|[Oo](?:c|k)?t(?:ober)?|[Nn]ov(?:ember)?|[Dd]e(?:c|z)(?:ember)?)\b
MONTHNUM (?:0?[1-9]|1[0-2])
MONTHNUM2 (?:0[1-9]|1[0-2])
MONTHDAY (?:(?:0[1-9])|(?:[12][0-9])|(?:3[01])|[1-9])

# Days: Monday, Tue, Thu, etc...
DAY (?:Mon(?:day)?|Tue(?:sday)?|Wed(?:nesday)?|Thu(?:rsday)?|Fri(?:day)?|Sat(?:urday)?|Sun(?:day)?)

# Years?
YEAR (?>\d\d){1,2}
HOUR (?:2[0123]|[01]?[0-9])
MINUTE (?:[0-5][0-9])
# '60' is a leap second in most time standards and thus is valid.
SECOND (?:(?:[0-5]?[0-9]|60)(?:[:.,][0-9]+)?)
TIME (?!<[0-9])%{HOUR}:%{MINUTE}(?::%{SECOND})(?![0-9])
# datestamp is YYYY/MM/DD-HH:MM:SS.UUUU (or something like it)
DATE_US %{MONTHNUM}[/-]%{MONTHDAY}[/-]%{YEAR}
DATE_EU %{MONTHDAY}[./-]%{MONTHNUM}[./-]%{YEAR}
ISO8601_TIMEZONE (?:Z|[+-]%{HOUR}(?::?%{MINUTE}))
ISO8601_SECOND (?:%{SECOND}|60)
TIMESTAMP_ISO8601 %{YEAR}-%{MONTHNUM}-%{MONTHDAY}[T ]%{HOUR}:?%{MINUTE}(?::?%{SECOND})?%{ISO8601_TIMEZONE}?
DATE %{DATE_US}|%{DATE_EU}
DATESTAMP %{DATE}[- ]%{TIME}
TZ (?:[APMCE][SD]T|UTC)
DATESTAMP_RFC822 %{DAY} %{MONTH} %{MONTHDAY} %{YEAR} %{TIME} %{TZ}
DATESTAMP_RFC2822 %{DAY}, %{MONTHDAY} %{MONTH} %{YEAR} %{TIME} %{ISO8601_TIMEZONE}
DATESTAMP_OTHER %{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{TZ} %{YEAR}
DATESTAMP_EVENTLOG %{YEAR}%{MONTHNUM2}%{MONTHDAY}%{HOUR}%{MINUTE}%{SECOND}

# Syslog Dates: Month Day HH:MM:SS
SYSLOGTIMESTAMP %{MONTH} +%{MONTHDAY} %{TIME}
PROG [\x21-\x5a\x5c\x5e-\x7e]+
SYSLOGPROG %{PROG:program}(?:\[%{POSINT:pid}\])?
SYSLOGHOST %{IPORHOST}
SYSLOGFACILITY <%{NONNEGINT:facility}.%{NONNEGINT:priority}>
HTTPDATE %{MONTHDAY}/%{MONTH}/%{YEAR}:%{TIME} %{INT}

# Shortcuts
QS %{QUOTEDSTRING}

# Log formats
SYSLOGBASE %{SYSLOGTIMESTAMP:timestamp} (?:%{SYSLOGFACILITY} )?%{SYSLOGHOST:logsource} %{SYSLOGPROG}:

# Log Levels
LOGLEVEL ([Aa]lert|ALERT|[Tt]race|TRACE|[Dd]ebug|DEBUG|[Nn]otice|NOTICE|[Ii]nfo|INFO|[Ww]arn?(?:ing)?|WARN?(?:ING)?|[Ee]rr?(?:or)?|ERR?(?:OR)?|[Cc]rit?(?:ical)?|CRIT?(?:ICAL)?|[Ff]atal|FATAL|[Ss]evere|SEVERE|EMERG(?:ENCY)?|[Ee]merg(?:ency)?)

1
2
3
4
5

filter {
  json {
    source => "message"
  }
}

1
2
3
4
5

filter {
  json {
    target => "doc"
  }
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12

#before
ip=1.2.3.4 error=REFUSED


filter {
  kv {}
}


#after
ip: 1.2.3.4
error: REFUSED

1
2
3
4
5
6
7
8

#栗子
name=zhang21&age=25&[email protected]

filter {
  kv {
    field_split => "&"
  }
}

1
2
3
4
5
6
7
8
9

#栗子
k1=v1:k2=v2:::k3=v3::k4=v4

filter {
  kv {
    field_split_pattern => ":+"
  }
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13

#栗子
one=(o n e) two=[t w o] three=<t h r e e>

filter {
  kv {
    include_brackets => tree
  }
}

#after
one: o n e
two: t w o
three: t h r e e

1
2
3
4
5
6
7
8

#计算每种http响应吗

filter {
  metrics {
    meter => [ "http_%{response}" ]
    add_tag => "metric"
  }
}

1
2
3
4
5
6
7

filter {
  mutate {
    gsub => [
	  "field1", "value", "replacement string",
	]
  }
}

1
2
3
4
5

filter {
  mutate {
    merge => { "dest_field" => "added_field"}
  }
}

1
2
3
4
5
6
7

filter {
  mutate {
    replace => {
	  "message" => "%{source_host}: My new message"
	}
  }
}

1
2
3
4
5
6
7
8

filter {
  xml {
    namespaces => {
      "xsl" => "http://www.w3.org/1999/XSL/Transform"
      "xhtml" => "http://www.w3.org/1999/xhtml"
    }
  }
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20

output {
  if [path] == "/var/nginx/access.log" {
    elasticsearch {
	  hosts =>
	  user =>
	  password =>
	  index => "nginx-access-%{+YYYY.MM.dd}"
	}
  }
  else if [path] == "/var/nginx/error.log" {
    elasticsearch {
	  hosts =>
	  user =>
	  password =>
	  index => "nginx-error-%{+YYYY.MM.dd}"
	}
  }
  else {
  }
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19

vim /etc/filebeat/filebeat.yml


filebeat.prospectors:
 -type: log
   enabled: true
   paths:
     - /var/log/*.log

output.elasticsearch:
  hosts: [ "ip:9200" ]
  #username
  #password

setup.kibana:
  host: "localhost:5601"
  #username
  #password

1
2
3
4
5
6
7

vim /etc/filebeat/filebeat.yml

output.logstash:
  hosts: [ "127.0.0.1:5044" ]

#logstash需要配置监听beats

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12

vim /etc/filebeat/filebeat.yml


setup.template.name: "template-name"
setup.template.fields: "/path/xxx/xxx.yml"

#强制覆盖已存在模板
setup.template.overwrite: true

#关闭自动载入模板
setup.template.enabled: false

1
2
3
4
5

output.elasticsearch.index: "customname-%{[version]}-%{+yyyy.MM.dd}"
setup.template.name: "customname"
setup.template.pattern: "customname-*"
setup.dashboards.index: "customname-*"

1
2

filebeat setup --template

1
2

curl -XDELETE 'http://localhost:9200/filebeat-*'

1
2
3
4
5

systemctl start filebeat

#前台启动并查看相关信息
filebeat -e -c filebeat.yml

1
2
3

elasticsearch-plugin install ingest-geoip
elasticsearch-plugin install ingest-user-agent

1
2
3
4
5
6
7
8
9

#启用模块
filebeat modules enable nginx system

#配置path
cd /etc/filebeat/modules.d

vim nginx.yml
vim system.yml

1
2
3

#重新载入索引模板
filebeat setup

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57

vim /etc/filebeat/filebeat.yml


#找到output


#redis
output.redis:
  hosts: "localhost"
  port: 6379
  key: "filebeat"
  #自定义key-name
  #password:
  #db:
  #data_type: 'list'



#logstash
output.logstash:
  hosts: [ "localhost:5044" ]



#Elasticsearch
elasticsearch.output:
  hosts: [ "localhost:9200" ]
  #username:
  #name:


==================

#redis对应的pipeline
vim /etc/logstash/conf.d/redis-pipeline.conf


input {
  redis {
    data_type => "list"
    key => "filebeat"
    host => "localhost"
    port => 6379
	#password =>
	#db =>
  }
}
#filter{ }
output {
  elasticsearch {
  	hosts => [ "localhost:9200" ]
	#user
	#password
	index => "filebeat-%{+YYYY.MM.dd}"
  }
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24

vim /etc/filebeat/filebeat.yml

# Optional index name. The default is "filebeat" plus date
# and generates [filebeat-]YYYY.MM.DD keys.
# In case you modify this pattern you must update setup.template.name and setup.template.pattern accordingly.
#index: "filebeat-%{[beat.version]}-%{+yyyy.MM.dd}"
#写到事件中的索引名，默认 "filebeat-%{[beat.version]}-%{+yyyy.MM.dd}"
#如果更改此设置，还需要配置setup.template.name和setup.template.pattern选项
#如果使用的是预先构建的kibana dashboard，还需要配置setup.dashboards.index选项

#定义索引
output.elasticsearch:
  hosts: ["10.0.1.8:9002", "10.0.1.7:9002", "10.0.1.9:9002"]
  loadbalance: true
  username: "elastic"
  password: xxx
  index: "filebeat-publish-%{+yyyy.MM.dd}"


#添加这几项
setup.template.name: "filebeat"
setup.template.pattern: "filebeat-*"
setup.template.fields: "fields.yml"
setup.template.overwrite: false

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14

#modules.d
filebeat modules list
filebeat modules enable nginx
#filebeat modules disable nginx


#filebeat命令
./filebeat -e --modules nginx


#filebeat.yml
filebeat.modules:
- module: nginx
- module: system

1
2
3
4
5
6
7
8

- module: nginx
  access:
    var.path: ["/var/log/nginx/access.log*"]


#or
filebeat -M "nginx.access.var.paths=[/var/log/access.log*]"
filebeat --modules nginx -M "nginx.access.var.paths=[/var/log/nginx/access.log*]" -M "nginx.error.var.paths=[/var/log/nginx/error.log*]"

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11

- module: nginx
  access:
    input:
	  close_eof: true


#or
filebeat -M "nginx.access.input.close_eof=true"

filebeat --modules nginx -M "nginx.access.input.close_eof=true"

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12

yesterday=`/bin/date +%Y%m%d --date='-1days'`
today=`/bin/date +%Y%m%d`

/bin/sed -i "s/service_err_${yesterday}/service_err_${today}/" /etc/filebeat/filebeat.yml
/bin/filebeat test config

if [ $? -eq 0 ] ;then
    /bin/systemctl restart filebeat
else
   exit 0
fi

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28

#启用/禁用inputs
enabled


#增加tags字段
tags


#向输出添加其他信息
fields

filebeat.inputs:
- type: log
  fields:
    author: zhang21


#自定义字段存储为输出文档中的顶级字段，而不是在字段子字典下分组。如果与filebeat冲突，则会覆盖源字段
fields_under_root


#应用于inputs的处理器列表
#已被弃用
processors


#为input生成的事件设置ingest node pipeline id
pipeline

1
2
3
4
5

filebeat.inputs:
- type: log
  paths:
    - /var/log/messages
    - /var/log/*.log

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11

filebeat.inputs:
- type: log
  paths:
    - /var/log/1.log
    - /var/log/2.log
- type: log
  paths:
    - "/var/log/appache/*"
  fields:
    apache: true
  fields_under_root: true

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124

paths
#将读取的基于全局路径的列表


recursive_glob.enabled
#true允许扩展为递归模式


encoding
#读取数据的文件编码


exclude_lines
#正则表达式列表，用于匹配你希望filebeat排除的行

filebeat.inputs:
- type: log
  ...
  exclude_lines: ['^debug']


include_lines
#正则表达式列表，用于匹配你希望filebeat包含的行。
#如果`exclude_lines`和`include_lines`都定义了，filebeat首先执行`include_lines`，之后才执行`exclude_lines`。

filebeat.inputs:
- type: log
  ...
  include_lines: ['^ERR', '^WARN']


harvester_buffer_size
#每个收集器在获取文件时使用的buffer大小，默认 16 384Byte。


max_bytes
#单日志消息可以具有的最大字节数。默认 10MB


json
#此选项使filebeat解码日志结构为json消息。filebeat逐行处理日志，因此每一行要有json对象才有效。

json.keys_under_root
json.overwrite_keys
json.add_error_key
json.message_key
json.ignore_decoding_error


mutiline
#控制filebeat如果处理跨越多行的日志消息。


exclude_files
#正则表达式列表，用于匹配你希望filebeat忽略的文件。默认无。

filebeat.inputs:
- type: log
  ...
  exclude_files: ['\.gz$']


ignore_older
#如果启用此选项，filebeat将忽略在指定的事件跨度之前修改的所有文件。

close_*
#用于在某个标准或时间后关闭收集器。


close_inactive
#如果文件尚未在指定的时间内收获，则filebeat将关闭文件句柄。


close_renamed
#filebeat会在重命名文件时关闭文件处理程序，请注意日志轮询。


close_removed
#删除文件后，filebeat会关闭收集器。


close_eof
#一旦到达文件末尾，filebeat就会关闭文件。


clean_*
#用于清理注册表文件中的状态条目。

clean_inactive
#filebeat在指定的不活动事件段过去后删除文件的状态。


clean_removed
#如果在最后一个已知名称下无法在磁盘上找到文件，则filebeat会清除注册表中的文件。


scan_frequency
#filebeat检查指定路径文件的频率。官方不建议将此值设置为小于1s。默认 10s。


tail_files
#filebeat开始在每个文件的末尾而不是开头读取新文件。默认 false


symlinks
#允许filebeat收集符号链接，它读取符号链接的原始文件。
由于此选项可能会导致数据丢失，默认 disabled


backoff
#指定filebeat如何积极地抓取打开的文件以进行更新。


max_backoff
#在到达eof后再次检查文件之间filebeat等待的最长时间。


backoff_factor
#指定等待时间增加的速度。


harvester_limit
#限制一个input并行启动的收集器数量。

1
2

filebeat.inputs:
- type: stdin

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12

encoding

exclude_lines
include_lines

harvester_buffer_size

max_bytes

json

multiline

1
2
3
4

filebeat.inputs:
- type: udp
  max_message_size: 10KB
  host: "localhost:5678"

1
2
3
4
5
6

#通过udp接收的最大消息大小，默认 10KB
max_message_size


#udp host
host

1
2
3
4

filebeat.inputs:
- type: tcp
    max_message_size: 10MB
	host: "localhost:5679"

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14

max_message_size
#通过tcp接收的最大消息大小， 默认 10MB


#host and tcp port
host


#指定用于拆分事件的字符，默认 \n
line_delimiter


#关闭连接前不活动的秒数， 默认 300s
timeout

1
2
3
4
5
6
7
8

filebeat.inputs:
- type: docke
  containers:
    path: "/var/lib/docker/containers"
	stream: "all"
	ids:
      - 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx'
#必须填写容器ID

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61

container.ids


#默认 /var/lib/docker/containers
container.path


#从指定stream读取: all/stdout/stderr，默认 all
container.stream


encoding

exclude_line

include_line

harvester_buffer_size

max_bytes

json

multiline

exclude_files

ignore_older

close_*

close_inactive

close_renamed

close_removed

close_eof

close_timeout

clean_*

clean_inactive

clean_removed

sacn_frequency

tail_files

symlinks

backoff

max_backoff

backoff_factor

harvester_limit

1
2
3
4
5
6

vim /etc/rsyslog.d/filebeat.conf

*.* @127.0.0.1:5678

#重启服务
systemctl restart rsyslog

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15

filebeat.inputs:
- type: syslog
  protocol.udp:
    host: "localhost:5678"
	max_message_size: 100KB


#定义索引
setup.template.name: "filebeat"
setup.template.pattern: "filebeat-*"
setup.template.fields: "fields.yml"
setup.template.overwrite: false
out.elastisearch：
  hosts: ["localhost:9200"]
  index: "syslog-%{+yyyy.MM.dd}"

1
2
3
4

netstat -nltup | grep 5678

udp        0      0 127.0.0.1:5678          0.0.0.0:*                           12434/filebeat

1
2
3
4
5
6
7
8

output.elasticsearch:
  hosts: ["https://localhost:9200"]
  username: "filebeat_internal"
  password: "YOUR_PASSWORD"
  index: "filebeat-%{[beat.version]}-%{+yyyy.MM.dd}"
  #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"]
  #ssl.certificate: "/etc/pki/client/cert.pem"
  #ssl.key: "/etc/pki/client/cert.key"

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104

#启用/禁用output，默认 true
enabled


hosts
#["hsot1:port1", "host2:port2", "host3:port3"]
username
#建议为filebeat创建一个专门的用户用于发送事件，而不是使用es的用户
password

compression_level
#gzip压缩等级, 0-9，默认 0


worker
#每个配置主机向es发布事件的worker数，默认 1


parameters
#http 参数字典


protocol
#网络协议, http/https


path
#http api调用前面的http路径前缀


headers
#定义headers


proxy_url
#代理的url


index
#写到事件中的索引名，默认 "filebeat-%{[beat.version]}-%{+yyyy.MM.dd}"
#如果更改此设置，还需要配置setup.template.name和setup.template.pattern选项
#如果使用的是预先构建的kibana dashboard，还需要配置setup.dashboards.index选项


indices
#支持条件的索引选择器规则数组，基于格式字符串的字段访问和名称映射。
indices.index: 要使用的索引格式字符串
indices.mapping： 映射
indices.default： 如果映射找不到匹配项的默认字符串值
indices.when： 成功的条件才执行当前规则

output.elasticsearch:
  hosts: ["http://localhost:9200"]
  index: "logs-%{[beat.version]}-%{+yyyy.MM.dd}"
  indices:
    - index: "critical-%{[beat.version]}-%{+yyyy.MM.dd}"
      when.contains:
        message: "CRITICAL"
    - index: "error-%{[beat.version]}-%{+yyyy.MM.dd}"
      when.contains:
        message: "ERR"


pipeline
#与indices类似，管道选择器配置数组

filebeat.inputs:
- type: log
  paths: ["/var/log/app/normal/*.log"]
  fields:
    type: "normal"
- type: log
  paths: ["/var/log/app/critical/*.log"]
  fields:
    type: "critical"

output.elasticsearch:
  hosts: ["http://localhost:9200"]
  index: "filebeat-%{[beat.version]}-%{+yyyy.MM.dd}"
  pipelines:
    - pipeline: critical_pipeline
      when.equals:
        fields.type: "critical"
    - pipeline: normal_pipeline
      when.equals:
        fields.type: "normal"


max_retries

bulk_max_size
#单个es批量挨批索引请求中要批量处理的最大事件数，默认 50

backoff.init
#在网络错误之后尝试重连到es之前等待的秒数，默认 1s

backoff.max
#在网络错误后尝试连接到es之前等待的最大秒数，默认 60s

timeout
#超时时间

ssl

1
2
3
4
5
6
7
8

output.redis:
  hosts: "localhost"
  port: 6379
  key: "filebeat"
  #自定义key-name
  #password:
  #db:
  #data_type: 'list'

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47

#启用/禁用output
enabled


hosts
port
#可将端口写在hosts里，默认6379
username
password
db
key
datatype
#默认 list
codec


keys
keys.key
keys.mapping
keys.default
keys.when

output.redis:
  hosts: ["localhost"]
  key: "default_list"
  keys:
    - key: "info_list"   # send to info_list if `message` field contains INFO
      when.contains:
        message: "INFO"
    - key: "debug_list"  # send to debug_list if `message` field contains DEBUG
      when.contains:
        message: "DEBUG"
    - key: "%{[fields.list]}"
      mapping:
        "http": "frontend_list"
        "nginx": "frontend_list"
        "mysql": "backend_list"

loadbalance
#如果配置了多个主机，则输出插件会将已发布的事件负载均衡到所有redis主机上

timeout
max_retries
bulk_max_size
ssl
proxy_url
proxy_use_local_resolver

1
2
3

output.file:
  path: "/tmp/filebeat"
  filename: filebeat

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17

enabled

path

filename

rotate_every_kb
#默认 10 240KB

number_of_files
#路径下要保存的最大文件数

permissions
#创建的文件权限， 默认 0600

codec

1
2

output.console:
  pretty: true

1
2
3
4
5
6
7
8
9

pretty
#美化输出， 默认 false

codec

enabled

bulk_max_size

1
2
3

output.logstash:
  hosts: ["localhost:5044", "localhost:5045"]
  loadbalance: true

1
2
3
4
5
6
7
8

{
	"line_id": INT,
    "play_name": "String",
    "speech_number": INT,
    "line_number": "String",
    "speaker": "String",
    "text_entry": "String"
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13

{
	"account_number": INT,
    "balance": INT,
    "firstname": "String",
    "lastname": "String",
    "age": INT,
    "gender": "M or F",
    "address": "String",
    "employer": "String",
    "email": "String",
    "city": "String",
    "state": "String"
}

1
2
3
4
5

{
    "memory": INT,
    "geo.coordinates": "geo_point",
    "@timestamp": "date"
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13

PUT /shakespeare
{
 "mappings": {
  "doc": {
   "properties": {
    "speaker": {"type": "keyword"},
    "play_name": {"type": "keyword"},
    "line_id": {"type": "integer"},
    "speech_number": {"type": "integer"}
   }
  }
 }
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16

PUT /logstash-2015.05.18
{
  "mappings": {
    "log": {
      "properties": {
        "geo": {
          "properties": {
            "coordinates": {
              "type": "geo_point"
            }
          }
        }
      }
    }
  }
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16

PUT /logstash-2015.05.19
{
  "mappings": {
    "log": {
      "properties": {
        "geo": {
          "properties": {
            "coordinates": {
              "type": "geo_point"
            }
          }
        }
      }
    }
  }
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16

PUT /logstash-2015.05.20
{
  "mappings": {
    "log": {
      "properties": {
        "geo": {
          "properties": {
            "coordinates": {
              "type": "geo_point"
            }
          }
        }
      }
    }
  }
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10

#这些命令要花一些时间

curl -H 'Content-Type: application/x-ndjson' -XPOST 'localhost:9200/bank/account/_bulk?pretty' --data-binary @accounts.json
curl -H 'Content-Type: application/x-ndjson' -XPOST 'localhost:9200/shakespeare/doc/_bulk?pretty' --data-binary @shakespeare_6.0.json
curl -H 'Content-Type: application/x-ndjson' -XPOST 'localhost:9200/_bulk?pretty' --data-binary @logs.jsonl


#验证
#在Kibana中的DevTools中运行
GET /_cat/indices?v

 1
 2
 3
 4
 5
 6
 7
 8
 9
10

#Kibana
GET /_cat/health?v


#cmd
curl -X GET "localhost:9200/_cat/health?v" -u elastic


epoch      timestamp cluster    status node.total node.data shards pri relo init unassign pending_tasks max_task_wait_time active_shards_percent
1525330981 15:03:01  docker-elk yellow          1         1     32  32    0    0        6             0                  -                 84.2%

1
2
3
4
5

GET /_cat/nodes?v


ip        heap.percent ram.percent cpu load_1m load_5m load_15m node.role master name
127.0.0.1           47          74  93    3.18    3.13     2.90 mdi       *      LGrAIE5

1
2
3
4
5
6
7
8
9

GET /_cat/indicies?v


health status index                             uuid                   pri rep docs.count docs.deleted store.size pri.store.size
green  open   .monitoring-kibana-6-2018.04.27   bsKsurh7TKaCsnekwHs3yg   1   0        870            0    328.1kb        328.1kb
green  open   .watcher-history-7-2018.04.28     zuq3rjI8S0OSS7vcZl7kSQ   1   0        954            0      1.4mb          1.4mb
green  open   .kibana                           8t_7lqq4TFSfelA7phgv5g   1   0        142           18    191.8kb        191.8kb
green  open   .monitoring-es-6-2018.04.28       vtUSjqaITT28CMHArpfNoA   1   0      20436            0      9.6mb          9.6mb
yellow open   filebeat-6.2.4-2018.05.03         sK3lIvMXS8GoRbWYCjdgzg   3   1        568            0    348.6kb        348.6kb

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14

#pretty漂亮JSON显示
PUT /customer?pretty

#或
curl -X PUT "localhost:9200/zhang" -u elastic:elastic


GET /_cat/indices?v


#pri主分片，rep副本
health status index                             uuid                   pri rep docs.count docs.deleted store.size pri.store.size
yellow open   customer                          WQ3qEnPQRW6FpVIHYVJ7yA   5   1          0            0      1.1kb          1.1kb
yellow open   zhang                             nkOUPOWERsS1PT_wEui67g   5   1          0            0      1.1kb          1.1kb

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43

PUT /customer/_doc/1?pretty
{
  "name": "John Doe"
}

#或
curl -X PUT -u elastic:elastic "localhost:9200/customer/_doc/1?pretty" -H 'Content-Type: application/json' -d'
{
  "name": "John Doe"
}'


{
  "_index" : "customer",
  "_type" : "_doc",
  "_id" : "1",
  "_version" : 1,
  "result" : "created",
  "_shards" : {
    "total" : 2,
    "successful" : 1,
    "failed" : 0
  },
  "_seq_no" : 0,
  "_primary_term" : 1
}



GET /customer/_doc/1?pretty

{
  "_index": "customer",
  "_type": "_doc",
  "_id": "1",
  "_version": 1,
  "found": true,
  "_source": {
    "name": "John Doe"
  }
}

#name:John Doe _id:1 _type:_doc _index:customer _score:1

1
2
3
4
5
6
7
8
9

DELETE /customer?pretty

curl -X DELETE "localhost:9200/customer?pretty" -u elastic:elastic


{
  "acknowledged": true
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22

PUT /customer/_doc/1?pretty
{
  "name": "John Doe"
}


#如果我修改此处文档信息，则Elasticsearch会替换之前的文档
PUT /customer/_doc/1?pretty
{
  "name": "Zhang"
}

#name:Zhang _id:1 _type:_doc _index:customer _score:1


#或者新增一个文档
PUT /customer/_doc/2?pretty
{
  "name": "Zhang"
}

#name:Zhang _id:2 _type:_doc _index:customer _score:1

1
2
3
4
5
6

POST /customer/_doc?pretty
{
  "name": "Zhang"
}

#name:Zhang _id:76xJJWMBddhqcmsO07A_ _type:_doc _index:customer _score:1

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20

POST /customer/_doc/1/_update?pretty
{
  "doc": { "name": "Jane Doe" }
}


#继续更新
POST /customer/_doc/1/_update?pretty
{
  "doc": { "name": "Jane Doe", "age": 20}
}



#简单脚本
#ctx._source指即将更新的当前源文档
POST /customer/_doc/1/_update?pretty
{
  "script": "ctx._source.age += 5"
}

1
2

DELETE /customer/_doc/2?pretty

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12

POST /customer/_doc/_bulk?pretty
{ "index": { "_id": "1" } }
{ "name": "John Doe" }
{ "index": { "_id": "2" } }
{ "name": "Jane Doe" }


#更新
POST /customer/_doc/_bulk?pretty
{"update": { "_id": "1" } }
{ "doc": { "name": "John Doe becomes Jane Doe" } }
{ "delete": { "_id": "2" } }

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14

{
    "account_number": 0,
    "balance": 16623,
    "firstname": "Bradshaw",
    "lastname": "Mckenzie",
    "age": 29,
    "gender": "F",
    "address": "244 Columbus Place",
    "employer": "Euron",
    "email": "[email protected]",
    "city": "Hobucken",
    "state": "CO"
}

1
2
3
4
5
6

curl -H "Content-Type: application/json" -u elastic:elastic -XPOST "localhost:9200/bank/_doc/_bulk?pretty&refresh" --data-binary "@accounts.json"
curl "localhost:9200/_cat/indices?v"


health status index                             uuid                   pri rep docs.count docs.deleted store.size pri.store.size
yellow open   bank                              PGSvNwQwQIOhMDr1nmXIuw   5   1       1000            0    474.7kb        474.7kb

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37