elk-7-logstash6.5.1添加kafka消息队列

安装

[root@operation nginx]# yum install -y nginx

改成json格式[vip]

[root@operation nginx]# cat nginx.conf

http {
log_format json '{"@timestamp":"$time_iso8601",'
'"host":"$server_addr",'
'"clientip":"$remote_addr",'
'"size":$body_bytes_sent,'
'"responsetime":$request_time,'
'"upstreamtime":"$upstream_response_time",'
'"upstreamhost":"$upstream_addr",'
'"http_host":"$host",'
'"url":"$uri",'
'"referer":"$http_referer",'
'"agent":"$http_user_agent",'
'"status":"$status"}';


# access_log /var/log/nginx/access.log main;
access_log /var/log/nginx/access.log json;

启动

[root@operation nginx]# /usr/sbin/nginx

配置

在应用断安装logstash并在根目录创建conf.d目录

[root@localhost conf.d]# pwd
/opt/logstash-6.5.1/conf.d

查看配置

[root@operation conf.d]# cat nginx.conf 
input{
    file {
        type=>"nginx_access"
        path => "/var/log/nginx/access.log"
        start_position=>"beginning"
        codec => "json"
    }
}
output { 
kafka { 
bootstrap_servers=>"192.168.1.58:9092,192.168.1.59:9092,192.168.1.60:9092" 
topic_id=>"1217"  ###这个主题名称会在kafka自动创建
compression_type=>"snappy" ###压缩类型
}
}

检查配置文件是否错误

[root@operation conf.d]# /opt/logstash/bin/logstash -t -f ./nginx.conf 
Sending Logstash logs to /opt/logstash/logs which is now configured via log4j2.properties
[2018-12-17T11:46:32,571][WARN ][logstash.config.source.multilocal] Ignoring the 'pipelines.yml' file because modules or command line options are specified
Configuration OK
[2018-12-17T11:46:34,169][INFO ][logstash.runner          ] Using config.test_and_exit mode. Config Validation Result: OK. Exiting Logstash
您在 /var/spool/mail/root 中有邮件

启动

[root@localhost conf.d]# nohup /opt/logstash-6.5.1/bin/logstash -f ./nginx.conf &
[1] 21419
[root@localhost conf.d]# nohup: 忽略输入并把输出追加到"nohup.out"

[root@localhost conf.d]# ps aux|grep logstash_cli.conf
root 21419 355 8.3 3714276 649324 pts/0 Sl 17:44 0:39 /bin/java -Xms1g -Xmx1g -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:CMSInitiatingOccupancyFraction=75 -XX:+UseCMSInitiatingOccupancyOnly -Djava.awt.headless=true -Dfile.encoding=UTF-8 -Djruby.compile.invokedynamic=true -Djruby.jit.threshold=0 -XX:+HeapDumpOnOutOfMemoryError -Djava.security.egd=file:/dev/urandom -cp /opt/logstash-6.5.1/logstash-core/lib/jars/animal-sniffer-annotations-1.14.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/commons-codec-1.11.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/commons-compiler-3.0.8.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/error_prone_annotations-2.0.18.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/google-java-format-1.1.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/gradle-license-report-0.7.1.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/guava-22.0.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/j2objc-annotations-1.1.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/jackson-annotations-2.9.5.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/jackson-core-2.9.5.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/jackson-databind-2.9.5.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/jackson-dataformat-cbor-2.9.5.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/janino-3.0.8.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/jruby-complete-9.1.13.0.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/jsr305-1.3.9.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/log4j-api-2.9.1.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/log4j-core-2.9.1.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/log4j-slf4j-impl-2.9.1.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/logstash-core.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/org.eclipse.core.commands-3.6.0.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/org.eclipse.core.contenttype-3.4.100.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/org.eclipse.core.expressions-3.4.300.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/org.eclipse.core.filesystem-1.3.100.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/org.eclipse.core.jobs-3.5.100.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/org.eclipse.core.resources-3.7.100.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/org.eclipse.core.runtime-3.7.0.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/org.eclipse.equinox.app-1.3.100.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/org.eclipse.equinox.common-3.6.0.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/org.eclipse.equinox.preferences-3.4.1.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/org.eclipse.equinox.registry-3.5.101.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/org.eclipse.jdt.core-3.10.0.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/org.eclipse.osgi-3.7.1.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/org.eclipse.text-3.5.101.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/slf4j-api-1.7.25.jar org.logstash.Logstash -f ./logstash_cli.conf
root 21449 0.0 0.0 112664 984 pts/0 R+ 17:44 0:00 grep --color=auto logstash_cli.conf

使用命令在kafka里面查看

[root@kafka kafka]# bin/kafka-console-consumer.sh --bootstrap-server 192.168.1.59:9092 --topic 1217 --from-beginning 
2018-12-07T09:46:05.908Z localhost.localdomain 192.168.1.157 - - [07/Dec/2018:17:46:04 +0800] "GET /favicon.ico HTTP/1.1" 404 3650 "http://192.168.1.214/" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36" "-"
2018-12-07T09:46:06.969Z localhost.localdomain 192.168.1.157 - - [07/Dec/2018:17:46:06 +0800] "GET / HTTP/1.1" 304 0 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36" "-"
2018-12-07T09:46:06.970Z localhost.localdomain 192.168.1.157 - - [07/Dec/2018:17:46:06 +0800] "GET / HTTP/1.1" 304 0 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36" "-"
2018-12-07T09:46:06.971Z localhost.localdomain 192.168.1.157 - - [07/Dec/2018:17:46:06 +0800] "GET / HTTP/1.1" 304 0 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36" "-"
2018-12-07T09:46:06.971Z localhost.localdomain 192.168.1.157 - - [07/Dec/2018:17:46:06 +0800] "GET / HTTP/1.1" 304 0 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36" "-"
2018-12-07T09:46:06.971Z localhost.localdomain 192.168.1.157 - - [07/Dec/2018:17:46:06 +0800] "GET / HTTP/1.1" 304 0 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36" "-"
2018-12-07T09:47:52.051Z localhost.localdomain 192.168.1.157 - - [07/Dec/2018:17:47:51 +0800] "GET / HTTP/1.1" 304 0 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36" "-"
2018-12-07T09:47:52.052Z localhost.localdomain 192.168.1.157 - - [07/Dec/2018:17:47:51 +0800] "GET / HTTP/1.1" 304 0 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36" "-"
2018-12-07T09:47:53.054Z localhost.localdomain 192.168.1.157 - - [07/Dec/2018:17:47:52 +0800] "GET / HTTP/1.1" 304 0 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36" "-"
2018-12-07T09:47:53.054Z localhost.localdomain 192.168.1.157 - - [07/Dec/2018:17:47:52 +0800] "GET / HTTP/1.1" 304 0 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36" "-"
2018-12-07T09:47:53.054Z localhost.localdomain 192.168.1.157 - - [07/Dec/2018:17:47:52 +0800] "GET / HTTP/1.1" 304 0 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36" "-"
2018-12-07T09:50:14.149Z localhost.localdomain 192.168.1.157 - - [07/Dec/2018:17:50:14 +0800] "GET / HTTP/1.1" 304 0 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36" "-"
2018-12-07T09:50:15.151Z localhost.localdomain 192.168.1.157 - - [07/Dec/2018:17:50:14 +0800] "GET / HTTP/1.1" 304 0 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36" "-"
2018-12-07T09:50:15.151Z localhost.localdomain 192.168.1.157 - - [07/Dec/2018:17:50:14 +0800] "GET / HTTP/1.1" 304 0 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36" "-"

创建从kafka传消息到es

[root@localhost config]# cat nginx.conf 
input{
 kafka{
 bootstrap_servers => ["192.168.1.58:9092,192.168.1.59:9092,192.168.1.60:9092"]
 group_id=>"httpgroup"
 topics=>"1217"
 consumer_threads=>3
 decorate_events=>true
 codec=>"json"
 }
}
 output{
 elasticsearch{
 hosts=>["192.168.1.209:9200","192.168.1.210:9200","192.168.1.211:9200"]
 index=>"httpss"
 codec=>"json"
 } 
}

检查配置文件

[root@localhost conf.d]# /opt/logstash-6.5.1/bin/logstash -t -f ./nginx.conf 
Sending Logstash logs to /opt/logstash-6.5.1/logs which is now configured via log4j2.properties
[2018-12-07T19:54:42,408][WARN ][logstash.config.source.multilocal] Ignoring the 'pipelines.yml' file because modules or command line options are specified
Configuration OK
[2018-12-07T19:54:43,856][INFO ][logstash.runner ] Using config.test_and_exit mode. Config Validation Result: OK. Exiting Logstash

启动

[root@localhost conf.d]# nohup /opt/logstash-6.5.1/bin/logstash -f ./nginx.conf &
[2] 21835
[root@localhost conf.d]# nohup: 忽略输入并把输出追加到"nohup.out"

[root@localhost conf.d]# ps aux|grep logstash
root 21419 1.4 10.5 4744396 817284 pts/0 Sl 17:44 1:52 /bin/java -Xms1g -Xmx1g -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:CMSInitiatingOccupancyFraction=75 -XX:+UseCMSInitiatingOccupancyOnly -Djava.awt.headless=true -Dfile.encoding=UTF-8 -Djruby.compile.invokedynamic=true -Djruby.jit.threshold=0 -XX:+HeapDumpOnOutOfMemoryError -Djava.security.egd=file:/dev/urandom -cp /opt/logstash-6.5.1/logstash-core/lib/jars/animal-sniffer-annotations-1.14.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/commons-codec-1.11.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/commons-compiler-3.0.8.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/error_prone_annotations-2.0.18.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/google-java-format-1.1.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/gradle-license-report-0.7.1.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/guava-22.0.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/j2objc-annotations-1.1.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/jackson-annotations-2.9.5.jar:/opt/logstas-6.5.1/logstash-core/lib/jars/jackson-core-2.9.5.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/jackson-databind-2.9.5.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/jackson-dataformat-cbor-2.9.5.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/janino-3.0.8.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/jruby-complete-9.1.13.0.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/jsr305-1.3.9.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/log4j-api-2.9.1.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/log4j-core-2.9.1.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/log4j-slf4j-impl-2.9.1.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/logstash-core.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/org.eclipse.core.commands-3.6.0.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/org.eclipse.core.contenttype-3.4.100.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/org.eclipse.core.expressions-3.4.300.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/org.eclipse.core.filesystem-1.3.100.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/org.eclipse.core.jobs-3.5.100.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/org.eclipse.core.resources-3.7.100.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/org.eclipse.core.runtime-3.7.0.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/org.eclipse.equinox.app-1.3.100.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/org.eclipse.equinox.common-3.6.0.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/org.eclipse.equinox.preferences-3.4.1.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/org.eclipse.equinox.registry-3.5.101.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/org.eclipse.jdt.core-3.10.0.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/org.eclipse.osgi-3.7.1.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/org.eclipse.text-3.5.101.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/slf4j-api-1.7.25.jar org.logstash.Logstash -f ./logstash_cli.conf
root 21835 355 8.4 3714200 652128 pts/0 Sl 19:56 0:39 /bin/java -Xms1g -Xmx1g -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:CMSInitiatingOccupancyFraction=75 -XX:+UseCMSInitiatingOccupancyOnly -Djava.awt.headless=true -Dfile.encoding=UTF-8 -Djruby.compile.invokedynamic=true -Djruby.jit.threshold=0 -XX:+HeapDumpOnOutOfMemoryError -Djava.security.egd=file:/dev/urandom -cp /opt/logstash-6.5.1/logstash-core/lib/jars/animal-sniffer-annotations-1.14.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/commons-codec-1.11.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/commons-compiler-3.0.8.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/error_prone_annotations-2.0.18.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/google-java-format-1.1.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/gradle-license-report-0.7.1.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/guava-22.0.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/j2objc-annotations-1.1.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/jackson-annotations-2.9.5.jar:/opt/logstas-6.5.1/logstash-core/lib/jars/jackson-core-2.9.5.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/jackson-databind-2.9.5.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/jackson-dataformat-cbor-2.9.5.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/janino-3.0.8.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/jruby-complete-9.1.13.0.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/jsr305-1.3.9.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/log4j-api-2.9.1.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/log4j-core-2.9.1.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/log4j-slf4j-impl-2.9.1.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/logstash-core.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/org.eclipse.core.commands-3.6.0.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/org.eclipse.core.contenttype-3.4.100.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/org.eclipse.core.expressions-3.4.300.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/org.eclipse.core.filesystem-1.3.100.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/org.eclipse.core.jobs-3.5.100.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/org.eclipse.core.resources-3.7.100.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/org.eclipse.core.runtime-3.7.0.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/org.eclipse.equinox.app-1.3.100.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/org.eclipse.equinox.common-3.6.0.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/org.eclipse.equinox.preferences-3.4.1.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/org.eclipse.equinox.registry-3.5.101.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/org.eclipse.jdt.core-3.10.0.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/org.eclipse.osgi-3.7.1.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/org.eclipse.text-3.5.101.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/slf4j-api-1.7.25.jar org.logstash.Logstash -f ./server.nginx.conf
root 21865 0.0 0.0 112664 972 pts/0 R+ 19:57 0:00 grep --color=auto logstash

在es查看

</a[/vip]>