安装
[root@operation nginx]# yum install -y nginx
改成json格式[vip]
[root@operation nginx]# cat nginx.conf http { log_format json '{"@timestamp":"$time_iso8601",' '"host":"$server_addr",' '"clientip":"$remote_addr",' '"size":$body_bytes_sent,' '"responsetime":$request_time,' '"upstreamtime":"$upstream_response_time",' '"upstreamhost":"$upstream_addr",' '"http_host":"$host",' '"url":"$uri",' '"referer":"$http_referer",' '"agent":"$http_user_agent",' '"status":"$status"}'; # access_log /var/log/nginx/access.log main; access_log /var/log/nginx/access.log json;
启动
[root@operation nginx]# /usr/sbin/nginx
配置
在应用断安装logstash并在根目录创建conf.d目录
[root@localhost conf.d]# pwd /opt/logstash-6.5.1/conf.d
查看配置
[root@operation conf.d]# cat nginx.conf input{ file { type=>"nginx_access" path => "/var/log/nginx/access.log" start_position=>"beginning" codec => "json" } } output { kafka { bootstrap_servers=>"192.168.1.58:9092,192.168.1.59:9092,192.168.1.60:9092" topic_id=>"1217" ###这个主题名称会在kafka自动创建 compression_type=>"snappy" ###压缩类型 } }
检查配置文件是否错误
[root@operation conf.d]# /opt/logstash/bin/logstash -t -f ./nginx.conf Sending Logstash logs to /opt/logstash/logs which is now configured via log4j2.properties [2018-12-17T11:46:32,571][WARN ][logstash.config.source.multilocal] Ignoring the 'pipelines.yml' file because modules or command line options are specified Configuration OK [2018-12-17T11:46:34,169][INFO ][logstash.runner ] Using config.test_and_exit mode. Config Validation Result: OK. Exiting Logstash 您在 /var/spool/mail/root 中有邮件
启动
[root@localhost conf.d]# nohup /opt/logstash-6.5.1/bin/logstash -f ./nginx.conf & [1] 21419 [root@localhost conf.d]# nohup: 忽略输入并把输出追加到"nohup.out" [root@localhost conf.d]# ps aux|grep logstash_cli.conf root 21419 355 8.3 3714276 649324 pts/0 Sl 17:44 0:39 /bin/java -Xms1g -Xmx1g -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:CMSInitiatingOccupancyFraction=75 -XX:+UseCMSInitiatingOccupancyOnly -Djava.awt.headless=true -Dfile.encoding=UTF-8 -Djruby.compile.invokedynamic=true -Djruby.jit.threshold=0 -XX:+HeapDumpOnOutOfMemoryError -Djava.security.egd=file:/dev/urandom -cp /opt/logstash-6.5.1/logstash-core/lib/jars/animal-sniffer-annotations-1.14.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/commons-codec-1.11.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/commons-compiler-3.0.8.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/error_prone_annotations-2.0.18.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/google-java-format-1.1.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/gradle-license-report-0.7.1.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/guava-22.0.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/j2objc-annotations-1.1.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/jackson-annotations-2.9.5.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/jackson-core-2.9.5.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/jackson-databind-2.9.5.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/jackson-dataformat-cbor-2.9.5.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/janino-3.0.8.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/jruby-complete-9.1.13.0.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/jsr305-1.3.9.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/log4j-api-2.9.1.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/log4j-core-2.9.1.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/log4j-slf4j-impl-2.9.1.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/logstash-core.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/org.eclipse.core.commands-3.6.0.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/org.eclipse.core.contenttype-3.4.100.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/org.eclipse.core.expressions-3.4.300.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/org.eclipse.core.filesystem-1.3.100.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/org.eclipse.core.jobs-3.5.100.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/org.eclipse.core.resources-3.7.100.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/org.eclipse.core.runtime-3.7.0.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/org.eclipse.equinox.app-1.3.100.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/org.eclipse.equinox.common-3.6.0.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/org.eclipse.equinox.preferences-3.4.1.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/org.eclipse.equinox.registry-3.5.101.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/org.eclipse.jdt.core-3.10.0.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/org.eclipse.osgi-3.7.1.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/org.eclipse.text-3.5.101.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/slf4j-api-1.7.25.jar org.logstash.Logstash -f ./logstash_cli.conf root 21449 0.0 0.0 112664 984 pts/0 R+ 17:44 0:00 grep --color=auto logstash_cli.conf
使用命令在kafka里面查看
[root@kafka kafka]# bin/kafka-console-consumer.sh --bootstrap-server 192.168.1.59:9092 --topic 1217 --from-beginning 2018-12-07T09:46:05.908Z localhost.localdomain 192.168.1.157 - - [07/Dec/2018:17:46:04 +0800] "GET /favicon.ico HTTP/1.1" 404 3650 "http://192.168.1.214/" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36" "-" 2018-12-07T09:46:06.969Z localhost.localdomain 192.168.1.157 - - [07/Dec/2018:17:46:06 +0800] "GET / HTTP/1.1" 304 0 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36" "-" 2018-12-07T09:46:06.970Z localhost.localdomain 192.168.1.157 - - [07/Dec/2018:17:46:06 +0800] "GET / HTTP/1.1" 304 0 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36" "-" 2018-12-07T09:46:06.971Z localhost.localdomain 192.168.1.157 - - [07/Dec/2018:17:46:06 +0800] "GET / HTTP/1.1" 304 0 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36" "-" 2018-12-07T09:46:06.971Z localhost.localdomain 192.168.1.157 - - [07/Dec/2018:17:46:06 +0800] "GET / HTTP/1.1" 304 0 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36" "-" 2018-12-07T09:46:06.971Z localhost.localdomain 192.168.1.157 - - [07/Dec/2018:17:46:06 +0800] "GET / HTTP/1.1" 304 0 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36" "-" 2018-12-07T09:47:52.051Z localhost.localdomain 192.168.1.157 - - [07/Dec/2018:17:47:51 +0800] "GET / HTTP/1.1" 304 0 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36" "-" 2018-12-07T09:47:52.052Z localhost.localdomain 192.168.1.157 - - [07/Dec/2018:17:47:51 +0800] "GET / HTTP/1.1" 304 0 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36" "-" 2018-12-07T09:47:53.054Z localhost.localdomain 192.168.1.157 - - [07/Dec/2018:17:47:52 +0800] "GET / HTTP/1.1" 304 0 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36" "-" 2018-12-07T09:47:53.054Z localhost.localdomain 192.168.1.157 - - [07/Dec/2018:17:47:52 +0800] "GET / HTTP/1.1" 304 0 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36" "-" 2018-12-07T09:47:53.054Z localhost.localdomain 192.168.1.157 - - [07/Dec/2018:17:47:52 +0800] "GET / HTTP/1.1" 304 0 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36" "-" 2018-12-07T09:50:14.149Z localhost.localdomain 192.168.1.157 - - [07/Dec/2018:17:50:14 +0800] "GET / HTTP/1.1" 304 0 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36" "-" 2018-12-07T09:50:15.151Z localhost.localdomain 192.168.1.157 - - [07/Dec/2018:17:50:14 +0800] "GET / HTTP/1.1" 304 0 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36" "-" 2018-12-07T09:50:15.151Z localhost.localdomain 192.168.1.157 - - [07/Dec/2018:17:50:14 +0800] "GET / HTTP/1.1" 304 0 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36" "-"
创建从kafka传消息到es
[root@localhost config]# cat nginx.conf input{ kafka{ bootstrap_servers => ["192.168.1.58:9092,192.168.1.59:9092,192.168.1.60:9092"] group_id=>"httpgroup" topics=>"1217" consumer_threads=>3 decorate_events=>true codec=>"json" } } output{ elasticsearch{ hosts=>["192.168.1.209:9200","192.168.1.210:9200","192.168.1.211:9200"] index=>"httpss" codec=>"json" } }
检查配置文件
[root@localhost conf.d]# /opt/logstash-6.5.1/bin/logstash -t -f ./nginx.conf Sending Logstash logs to /opt/logstash-6.5.1/logs which is now configured via log4j2.properties [2018-12-07T19:54:42,408][WARN ][logstash.config.source.multilocal] Ignoring the 'pipelines.yml' file because modules or command line options are specified Configuration OK [2018-12-07T19:54:43,856][INFO ][logstash.runner ] Using config.test_and_exit mode. Config Validation Result: OK. Exiting Logstash
启动
[root@localhost conf.d]# nohup /opt/logstash-6.5.1/bin/logstash -f ./nginx.conf & [2] 21835 [root@localhost conf.d]# nohup: 忽略输入并把输出追加到"nohup.out" [root@localhost conf.d]# ps aux|grep logstash root 21419 1.4 10.5 4744396 817284 pts/0 Sl 17:44 1:52 /bin/java -Xms1g -Xmx1g -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:CMSInitiatingOccupancyFraction=75 -XX:+UseCMSInitiatingOccupancyOnly -Djava.awt.headless=true -Dfile.encoding=UTF-8 -Djruby.compile.invokedynamic=true -Djruby.jit.threshold=0 -XX:+HeapDumpOnOutOfMemoryError -Djava.security.egd=file:/dev/urandom -cp /opt/logstash-6.5.1/logstash-core/lib/jars/animal-sniffer-annotations-1.14.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/commons-codec-1.11.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/commons-compiler-3.0.8.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/error_prone_annotations-2.0.18.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/google-java-format-1.1.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/gradle-license-report-0.7.1.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/guava-22.0.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/j2objc-annotations-1.1.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/jackson-annotations-2.9.5.jar:/opt/logstas-6.5.1/logstash-core/lib/jars/jackson-core-2.9.5.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/jackson-databind-2.9.5.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/jackson-dataformat-cbor-2.9.5.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/janino-3.0.8.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/jruby-complete-9.1.13.0.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/jsr305-1.3.9.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/log4j-api-2.9.1.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/log4j-core-2.9.1.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/log4j-slf4j-impl-2.9.1.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/logstash-core.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/org.eclipse.core.commands-3.6.0.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/org.eclipse.core.contenttype-3.4.100.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/org.eclipse.core.expressions-3.4.300.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/org.eclipse.core.filesystem-1.3.100.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/org.eclipse.core.jobs-3.5.100.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/org.eclipse.core.resources-3.7.100.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/org.eclipse.core.runtime-3.7.0.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/org.eclipse.equinox.app-1.3.100.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/org.eclipse.equinox.common-3.6.0.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/org.eclipse.equinox.preferences-3.4.1.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/org.eclipse.equinox.registry-3.5.101.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/org.eclipse.jdt.core-3.10.0.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/org.eclipse.osgi-3.7.1.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/org.eclipse.text-3.5.101.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/slf4j-api-1.7.25.jar org.logstash.Logstash -f ./logstash_cli.conf root 21835 355 8.4 3714200 652128 pts/0 Sl 19:56 0:39 /bin/java -Xms1g -Xmx1g -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:CMSInitiatingOccupancyFraction=75 -XX:+UseCMSInitiatingOccupancyOnly -Djava.awt.headless=true -Dfile.encoding=UTF-8 -Djruby.compile.invokedynamic=true -Djruby.jit.threshold=0 -XX:+HeapDumpOnOutOfMemoryError -Djava.security.egd=file:/dev/urandom -cp /opt/logstash-6.5.1/logstash-core/lib/jars/animal-sniffer-annotations-1.14.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/commons-codec-1.11.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/commons-compiler-3.0.8.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/error_prone_annotations-2.0.18.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/google-java-format-1.1.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/gradle-license-report-0.7.1.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/guava-22.0.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/j2objc-annotations-1.1.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/jackson-annotations-2.9.5.jar:/opt/logstas-6.5.1/logstash-core/lib/jars/jackson-core-2.9.5.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/jackson-databind-2.9.5.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/jackson-dataformat-cbor-2.9.5.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/janino-3.0.8.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/jruby-complete-9.1.13.0.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/jsr305-1.3.9.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/log4j-api-2.9.1.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/log4j-core-2.9.1.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/log4j-slf4j-impl-2.9.1.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/logstash-core.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/org.eclipse.core.commands-3.6.0.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/org.eclipse.core.contenttype-3.4.100.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/org.eclipse.core.expressions-3.4.300.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/org.eclipse.core.filesystem-1.3.100.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/org.eclipse.core.jobs-3.5.100.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/org.eclipse.core.resources-3.7.100.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/org.eclipse.core.runtime-3.7.0.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/org.eclipse.equinox.app-1.3.100.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/org.eclipse.equinox.common-3.6.0.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/org.eclipse.equinox.preferences-3.4.1.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/org.eclipse.equinox.registry-3.5.101.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/org.eclipse.jdt.core-3.10.0.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/org.eclipse.osgi-3.7.1.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/org.eclipse.text-3.5.101.jar:/opt/logstash-6.5.1/logstash-core/lib/jars/slf4j-api-1.7.25.jar org.logstash.Logstash -f ./server.nginx.conf root 21865 0.0 0.0 112664 972 pts/0 R+ 19:57 0:00 grep --color=auto logstash
在es查看
继续阅读
- 我的QQ
- QQ扫一扫
-
- 我的头条
- 头条扫一扫
-
评论