Logstash 收集Nginx日志
1. 下载安装¶
cd /usr/local/src/elk/
wget https://artifacts.elastic.co/downloads/logstash/logstash-5.6.16.tar.gz
tar xf logstash-5.6.16.tar.gz -C /opt/
ln -sf /opt/logstash-5.6.16 /opt/logstash
mkdir /opt/logstash/extra
cd /opt/logstash/extra
cat>>nginx_logstash.conf<<EOF
input {
stdin { }
}
filter {
grok {
match => {
"message" => '%{IPORHOST:remote_ip} - %{DATA:user_name} \[%{HTTPDATE:time}\] "%{WORD:request_act
ion} %{DATA:request} HTTP/%{NUMBER:http_version}" %{NUMBER:response} %{NUMBER:bytes} "%{DATA:referrer}" "%{DATA:agent}"' }
}
date {
match => [ "time", "dd/MMM/YYYY:HH:mm:ss Z" ]
locale => en
}
geoip {
source => "remote_ip"
target => "geoip"
}
useragent {
source => "agent"
target => "user_agent"
}
}
output {
stdout {
codec => rubydebug
}
}
EOF
head -n2 /var/log/nginx_logs|/opt/logstash/bin/logstash -f /opt/logstash/extra/nginx_logstash.conf
详细过程
root@master:~# cd /usr/local/src/elk/
root@master:/usr/local/src/elk# wget https://artifacts.elastic.co/downloads/logstash/logstash-5.6.16.tar.gz
root@master:/usr/local/src/elk# tar xf logstash-5.6.16.tar.gz -C /opt/
root@master:/usr/local/src/elk# ln -sf /opt/logstash-5.6.16 /opt/logstash
root@master:~# mkdir /opt/logstash/extra
root@master:/opt/logstash/extra# egrep -v '#|^$' nginx_logstash.conf
input {
stdin { }
}
filter {
grok {
match => {
"message" => '%{IPORHOST:remote_ip} - %{DATA:user_name} \[%{HTTPDATE:time}\] "%{WORD:request_act
ion} %{DATA:request} HTTP/%{NUMBER:http_version}" %{NUMBER:response} %{NUMBER:bytes} "%{DATA:referrer}" "%{DATA:agent}"' }
}
date {
match => [ "time", "dd/MMM/YYYY:HH:mm:ss Z" ]
locale => en
}
geoip {
source => "remote_ip"
target => "geoip"
}
useragent {
source => "agent"
target => "user_agent"
}
}
output {
stdout {
codec => rubydebug
}
}
root@master:/var/log# head -n2 nginx_logs
93.180.71.3 - - [17/May/2015:08:05:32 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.21)"
93.180.71.3 - - [17/May/2015:08:05:23 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.21)"
root@master:/var/log# head -n2 /var/log/nginx_logs|/opt/logstash/bin/logstash -f /opt/logstash/extra/nginx_logstash.conf
/opt/logstash/vendor/bundle/jruby/1.9/gems/manticore-0.6.4-java/lib/manticore/client.rb:109 warning: already initialized constant DEFAULT_MAX_POOL_SIZE
/opt/logstash/vendor/bundle/jruby/1.9/gems/manticore-0.6.4-java/lib/manticore/client.rb:111 warning: already initialized constant DEFAULT_REQUEST_TIMEOUT
/opt/logstash/vendor/bundle/jruby/1.9/gems/manticore-0.6.4-java/lib/manticore/client.rb:112 warning: already initialized constant DEFAULT_SOCKET_TIMEOUT
/opt/logstash/vendor/bundle/jruby/1.9/gems/manticore-0.6.4-java/lib/manticore/client.rb:113 warning: already initialized constant DEFAULT_CONNECT_TIMEOUT
/opt/logstash/vendor/bundle/jruby/1.9/gems/manticore-0.6.4-java/lib/manticore/client.rb:114 warning: already initialized constant DEFAULT_MAX_REDIRECTS
/opt/logstash/vendor/bundle/jruby/1.9/gems/manticore-0.6.4-java/lib/manticore/client.rb:115 warning: already initialized constant DEFAULT_EXPECT_CONTINUE
/opt/logstash/vendor/bundle/jruby/1.9/gems/manticore-0.6.4-java/lib/manticore/client.rb:116 warning: already initialized constant DEFAULT_STALE_CHECK
/opt/logstash/vendor/bundle/jruby/1.9/gems/manticore-0.6.4-java/lib/manticore/client.rb:592 warning: already initialized constant ISO_8859_1
/opt/logstash/vendor/bundle/jruby/1.9/gems/manticore-0.6.4-java/lib/manticore/client.rb:644 warning: already initialized constant KEY_EXTRACTION_REGEXP
Sending Logstash's logs to /opt/logstash/logs which is now configured via log4j2.properties
[2019-10-09T17:59:34,685][INFO ][logstash.modules.scaffold] Initializing module {:module_name=>"fb_apache", :directory=>"/opt/logstash/modules/fb_apache/configuration"}
[2019-10-09T17:59:34,688][INFO ][logstash.modules.scaffold] Initializing module {:module_name=>"netflow", :directory=>"/opt/logstash/modules/netflow/configuration"}
[2019-10-09T17:59:34,690][INFO ][logstash.setting.writabledirectory] Creating directory {:setting=>"path.queue", :path=>"/opt/logstash/data/queue"}
[2019-10-09T17:59:34,691][INFO ][logstash.setting.writabledirectory] Creating directory {:setting=>"path.dead_letter_queue", :path=>"/opt/logstash/data/dead_letter_queue"}
[2019-10-09T17:59:34,705][INFO ][logstash.agent ] No persistent UUID file found. Generating new UUID {:uuid=>"66202436-14a5-46b1-8a4b-42cde92ebae6", :path=>"/opt/logstash/data/uui
d"}[2019-10-09T17:59:35,047][INFO ][logstash.filters.geoip ] Using geoip database {:path=>"/opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-filter-geoip-4.3.1-java/vendor/GeoLite2-City.mm
db"}[2019-10-09T17:59:35,173][INFO ][logstash.pipeline ] Starting pipeline {"id"=>"main", "pipeline.workers"=>8, "pipeline.batch.size"=>125, "pipeline.batch.delay"=>5, "pipeline.max_infl
ight"=>1000}[2019-10-09T17:59:35,182][INFO ][logstash.pipeline ] Pipeline main started
[2019-10-09T17:59:35,234][INFO ][logstash.agent ] Successfully started Logstash API endpoint {:port=>9600}
{
"request" => "/downloads/product_1",
"request_action" => "GET",
"agent" => "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.21)",
"geoip" => {
"timezone" => "Europe/Amsterdam",
"ip" => "93.180.71.3",
"latitude" => 52.3824,
"country_name" => "Netherlands",
"country_code2" => "NL",
"continent_code" => "EU",
"country_code3" => "NL",
"location" => {
"lon" => 4.8995,
"lat" => 52.3824
},
"longitude" => 4.8995
},
"user_name" => "-",
"http_version" => "1.1",
"message" => "93.180.71.3 - - [17/May/2015:08:05:32 +0000] \"GET /downloads/product_1 HTTP/1.1\" 304 0 \"-\" \"Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.21)\"",
"referrer" => "-",
"@timestamp" => 2015-05-17T08:05:32.000Z,
"remote_ip" => "93.180.71.3",
"response" => "304",
"bytes" => "0",
"@version" => "1",
"host" => "master",
"time" => "17/May/2015:08:05:32 +0000",
"user_agent" => {
"os" => "Debian",
"major" => "1",
"minor" => "3",
"build" => "",
"name" => "Debian APT-HTTP",
"os_name" => "Debian",
"device" => "Other"
}
}
{
"request" => "/downloads/product_1",
"request_action" => "GET",
"agent" => "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.21)",
"geoip" => {
"timezone" => "Europe/Amsterdam",
"ip" => "93.180.71.3",
"latitude" => 52.3824,
"country_name" => "Netherlands",
"country_code2" => "NL",
"continent_code" => "EU",
"country_code3" => "NL",
"location" => {
"lon" => 4.8995,
"lat" => 52.3824
},
"longitude" => 4.8995
},
"user_name" => "-",
"http_version" => "1.1",
"message" => "93.180.71.3 - - [17/May/2015:08:05:23 +0000] \"GET /downloads/product_1 HTTP/1.1\" 304 0 \"-\" \"Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.21)\"",
"referrer" => "-",
"@timestamp" => 2015-05-17T08:05:23.000Z,
"remote_ip" => "93.180.71.3",
"response" => "304",
"bytes" => "0",
"@version" => "1",
"host" => "master",
"time" => "17/May/2015:08:05:23 +0000",
"user_agent" => {
"os" => "Debian",
"major" => "1",
"minor" => "3",
"build" => "",
"name" => "Debian APT-HTTP",
"os_name" => "Debian",
"device" => "Other"
}
}
[2019-10-09T17:59:38,205][WARN ][logstash.agent ] stopping pipeline {:id=>"main"}