packetbeat 收集es http日志
1. 介绍¶
通过packetbeat收集日志es的http协议日志，通过console输出结果
2. 下载安装¶
cd /usr/local/src/elk/
wget https://artifacts.elastic.co/downloads/beats/packetbeat/packetbeat-5.6.16-linux-x86_64.tar.gz
tar xf packetbeat-5.6.16-linux-x86_64.tar.gz -C /opt/
ln -sf /opt/packetbeat-5.6.16-linux-x86_64 /opt/packetbeat
cd /opt/packetbeat
mkdir conf && cd conf
cat>>es.yml<<EOF
packetbeat.interfaces.device: any
packetbeat.protocols.http:
ports: [9200]
send_request: true
include_body_for: ["application/json", "x-www-form-urlencoded"]
output.console:
pretty: true
EOF
/opt/packetbeat/packetbeat -e -c /opt/packetbeat/conf/es.yml -strict.perms=false
详细过程
root@master:~# cd /usr/local/src/elk/
root@master:/usr/local/src/elk# wget https://artifacts.elastic.co/downloads/beats/packetbeat/packetbea
t-5.6.16-linux-x86_64.tar.gz--2019-10-09 16:42:37-- https://artifacts.elastic.co/downloads/beats/packetbeat/packetbeat-5.6.16-lin
ux-x86_64.tar.gzResolving artifacts.elastic.co (artifacts.elastic.co)... 151.101.110.222, 2a04:4e42:36::734
Connecting to artifacts.elastic.co (artifacts.elastic.co)|151.101.110.222|:443... connected.
HTTP request sent, awaiting response... 200 OK
Length: 9844232 (9.4M) [application/x-gzip]
Saving to: ‘packetbeat-5.6.16-linux-x86_64.tar.gz’
packetbeat-5.6.16-linux-x 100%[===================================>] 9.39M 71.9KB/s in 82s
2019-10-09 16:44:00 (117 KB/s) - ‘packetbeat-5.6.16-linux-x86_64.tar.gz’ saved [9844232/9844232]
root@master:/usr/local/src/elk# tar xf packetbeat-5.6.16-linux-x86_64.tar.gz -C /opt/
root@master:/usr/local/src/elk# ln -sf /opt/packetbeat-5.6.16-linux-x86_64 /opt/packetbeat
root@master:/usr/local/src/elk# cd /opt/packetbeat
root@master:/opt/packetbeat# mkdir conf && cd conf
root@master:/opt/packetbeat/conf# cat>>es.yml<<EOF
> packetbeat.interfaces.device: any
> packetbeat.protocols.http:
> ports: [9200]
> send_request: true
> include_body_for: ["application/json", "x-www-form-urlencoded"]
> output.console:
> pretty: true
> EOF
root@master:/opt/packetbeat/conf# /opt/packetbeat/packetbeat -e -c /opt/packetbeat/conf/es.yml -strict.perms=false
2019/10/09 08:47:12.247767 beat.go:297: INFO Home path: [/opt/packetbeat] Config path: [/opt/packetbea
t] Data path: [/opt/packetbeat/data] Logs path: [/opt/packetbeat/logs]2019/10/09 08:47:12.247792 beat.go:192: INFO Setup Beat: packetbeat; Version: 5.6.16
2019/10/09 08:47:12.247818 outputs.go:108: INFO Activated console as output plugin.
2019/10/09 08:47:12.247855 publish.go:300: INFO Publisher name: master
2019/10/09 08:47:12.247861 metrics.go:23: INFO Metrics logging every 30s
2019/10/09 08:47:12.247972 async.go:63: INFO Flush Interval set to: 1s
2019/10/09 08:47:12.247979 async.go:64: INFO Max Bulk Size set to: 2048
2019/10/09 08:47:12.248028 procs.go:79: INFO Process matching disabled
2019/10/09 08:47:12.248119 protos.go:89: INFO registered protocol plugin: dns
2019/10/09 08:47:12.248125 protos.go:89: INFO registered protocol plugin: nfs
2019/10/09 08:47:12.248127 protos.go:89: INFO registered protocol plugin: pgsql
2019/10/09 08:47:12.248130 protos.go:89: INFO registered protocol plugin: redis
2019/10/09 08:47:12.248133 protos.go:89: INFO registered protocol plugin: thrift
2019/10/09 08:47:12.248135 protos.go:89: INFO registered protocol plugin: amqp
2019/10/09 08:47:12.248138 protos.go:89: INFO registered protocol plugin: cassandra
2019/10/09 08:47:12.248142 protos.go:89: INFO registered protocol plugin: http
2019/10/09 08:47:12.248147 protos.go:89: INFO registered protocol plugin: memcache
2019/10/09 08:47:12.248152 protos.go:89: INFO registered protocol plugin: mongodb
2019/10/09 08:47:12.248155 protos.go:89: INFO registered protocol plugin: mysql
2019/10/09 08:47:12.261885 beat.go:233: INFO packetbeat start running.
{
"@timestamp": "2019-10-09T08:47:12.718Z",
"beat": {
"hostname": "master",
"name": "master",
"version": "5.6.16"
},
"bytes_in": 81,
"bytes_out": 411,
"client_ip": "192.168.2.20",
"client_port": 56548,
"client_proc": "",
"client_server": "",
"direction": "in",
"http": {
"request": {
"headers": {
"content-length": 0
},
"params": ""
},
"response": {
"body": "HTTP/1.1 200 OK\r\ncontent-type: application/json; charset=UTF-8\r\ncontent-length: 350
\r\n\r\n{\"nodes\":{\"Q99k3iNRRouuMqNZie6K6Q\":{\"ip\":\"192.168.2.20\",\"version\":\"5.6.16\",\"http\":{\"publish_address\":\"192.168.2.20:9200\"}},\"8AwLte6OTDqLuALshSi-Vw\":{\"ip\":\"192.168.2.22\",\"version\":\"5.6.16\",\"http\":{\"publish_address\":\"192.168.2.22:9200\"}},\"", "code": 200,
"headers": {
"content-length": 324,
"content-type": "application/json; charset=UTF-8"
},
"phrase": "OK"
}
},
"ip": "192.168.2.20",
"method": "HEAD",
"path": "/",
"port": 9200,
"proc": "",
"query": "HEAD /",
"request": "HEAD / HTTP/1.1\r\nHost: master:9200\r\nContent-Length: 0\r\nConnection: keep-alive\r\n\
r\n", "responsetime": 0,
"server": "",
"status": "OK",
"type": "http"
}
{
"@timestamp": "2019-10-09T08:47:12.720Z",
"beat": {
"hostname": "master",
"name": "master",
"version": "5.6.16"
},
"bytes_in": 158,
"bytes_out": 205,
"client_ip": "192.168.2.20",
"client_port": 56548,
"client_proc": "",
"client_server": "",
"direction": "in",
"http": {
"request": {
"headers": {
"content-length": 0
},
"params": "filter_path=nodes.%2A.version%2Cnodes.%2A.http.publish_address%2Cnodes.%2A.ip"
},
"response": {
"body": "{\"docs\":[{\"_index\":\".kibana\",\"_type\":\"config\",\"_id\":\"5.6.16\",\"_version\"
:1,\"found\":true,\"_source\":{\"buildNum\":15690}}]}", "code": 200,
"headers": {
"content-length": 118,
"content-type": "application/json; charset=UTF-8"
},
"phrase": "OK"
}
},
"ip": "192.168.2.20",
"method": "GET",
"path": "/_nodes",
"port": 9200,
"proc": "",
"query": "GET /_nodes",
"request": "GET /_nodes?filter_path=nodes.
省略