一個日志服務架構主要包括3個部分：日志采集agent，日志存儲，及日志瀏覽

本日志服務方案采用logstash+elasticsearch+kibana的組合搭建，其中logstash負責日志的采集和入庫，elasticsearch負責日志的存儲和索引，kubana負責日志的搜索和前端展示

在實際部署過程中，為保證可用和節省資源，主要注意以下幾點：

1.logstash支持很多socket遠程input和ouput插件，但為不影響主干業務流，盡量采用日志文件異步讀出寫入es的方式，如log4j支持socketappend，但需要修改既有應用的配置，而且一旦server端出錯，應用日志沒有本地文件，容易丟失

2.logstash agent可以直接寫入es，但為減小es的壓力，采用redis和logstash

indexer的較色來延長消息傳遞流程，降低各環節的壓力

3.logstash基于java實現，為節約系統資源，不論agent還是indexer，單臺物理機只配置一個文件，采用一個守護進程啟動服務

cat

/usr/local/logserver/logstash/conf/allinone_indexer.conf

input

{

redis

{

host =>

"10.241.223.112"

type =>

"log4j-account"

data_type =>

"list"

key =>

"log4j-account"

format =>

"json_event"

}

redis

{

host =>

"10.241.223.112"

type =>

"log4j-activity"

data_type =>

"list"

key =>

"log4j-activity"

format =>

"json_event"

}

redis

{

host =>

"10.241.223.112"

type =>

"log4j-databag"

data_type =>

"list"

key =>

"log4j-databag"

format =>

"json_event"

}

redis

{

host =>

"10.241.223.112"

type =>

"log4j-document"

data_type =>

"list"

key =>

"log4j-document"

format =>

"json_event"

}

redis

{

host =>

"10.241.223.112"

type =>

"log4j-job"

data_type =>

"list"

key =>

"log4j-job"

format =>

"json_event"

}

redis

{

host =>

"10.241.223.112"

type =>

"log4j-monitor"

data_type =>

"list"

key =>

"log4j-monitor"

format =>

"json_event"

}

redis

{

host =>

"10.241.223.112"

type =>

"log4j-notification"

data_type =>

"list"

key =>

"log4j-notification"

format =>

"json_event"

}

redis

{

host =>

"10.241.223.112"

type =>

"log4j-qto"

data_type =>

"list"

key =>

"log4j-qto"

format =>

"json_event"

}

redis

{

host =>

"10.241.223.112"

type =>

"log4j-search"

data_type =>

"list"

key =>

"log4j-search"

format =>

"json_event"

}

redis

{

host =>

"10.241.223.112"

type =>

"log4j-social"

data_type =>

"list"

key =>

"log4j-social"

format =>

"json_event"

}

redis

{

host =>

"10.241.223.112"

type =>

"log4j-storage"

data_type =>

"list"

key =>

"log4j-storage"

format =>

"json_event"

}

redis

{

host =>

"10.241.223.112"

type =>

"tomcat-account"

data_type =>

"list"

key =>

"tomcat-account"

format =>

"json_event"

}

redis

{

host =>

"10.241.223.112"

type =>

"tomcat-databag"

data_type =>

"list"

key =>

"tomcat-databag"

format =>

"json_event"

}

redis

{

host =>

"10.241.223.112"

type =>

"tomcat-monitor"

data_type =>

"list"

key =>

"tomcat-monitor"

format =>

"json_event"

}

}

filter {

multiline

{

type =>

"log4j-account"

pattern =>

"^\s"

what =>

"previous"

}

multiline

{

type =>

"log4j-activity"

pattern =>

"^\s"

what =>

"previous"

}

multiline

{

type =>

"log4j-databag"

pattern =>

"^\s"

what =>

"previous"

}

multiline

{

type =>

"log4j-document"

pattern =>

"^\s"

what =>

"previous"

}

multiline

{

type =>

"log4j-job"

pattern =>

"^\s"

what =>

"previous"

}

multiline

{

type =>

"log4j-monitor"

pattern =>

"^\s"

what =>

"previous"

}

multiline

{

type =>

"log4j-notification"

pattern =>

"^\s"

what =>

"previous"

}

multiline

{

type =>

"log4j-qto"

pattern =>

"^\s"

what =>

"previous"

}

multiline

{

type =>

"log4j-search"

pattern =>

"^\s"

what =>

"previous"

}

multiline

{

type =>

"log4j-social"

pattern =>

"^\s"

what =>

"previous"

}

multiline

{

type =>

"log4j-storage"

pattern =>

"^\s"

what =>

"previous"

}

multiline

{

type =>

"tomcat-account"

pattern =>

"^\s"

what =>

"previous"

}

multiline

{

type =>

"tomcat-databag"

pattern =>

"^\s"

what =>

"previous"

}

multiline

{

type =>

"tomcat-monitor"

pattern =>

"^\s"

what =>

"previous"

}

}

output {

elasticsearch {

host

=> "10.241.223.112"

cluster

=> "logstashelasticsearch"

port

=> 9300

}

}

cat

/usr/local/logserver/logstash/conf/nginx_agent.conf

input {

file {

type => "nginx"

path => ["/data/logs/nginx/*.log"

]

}

}

output {

redis {

host => "10.241.223.112"

data_type => "list"

key => "nginx"

type => "nginx"

}

}

cat

/usr/local/logserver/logstash/startup.sh

#!/bin/bash

workdir=$(cd $(dirname $0);pwd)

#echo $workdir

usage(){

cat <

This script start, stop and retart logstash agent and indexer

of apps

Usage: $(basename $0) [ agent | indexer | all ] app1 app2 app3

...

EOF1

}

[ $# -lt 2 ] && usage

&& exit

start_mode=$1

for appname in "$@";do

case "$appname" in

"agent" )

continue

;;

"indexer" )

continue

;;

"all" )

continue

;;

*)

if [ "$start_mode" =

"all" ]; then

java

-jar $workdir/logstash.jar agent -f

$workdir/conf/"$appname"_indexer.conf &

java

-jar $workdir/logstash.jar agent -f

$workdir/conf/"$appname"_agent.conf &

else

java

-jar $workdir/logstash.jar agent -f

$workdir/conf/"$appname"_"$start_mode".conf &

fi

;;

esac

echo

done

cat /usr/local/logserver/logstash/indexer_startup.sh

#!/bin/bash

#/usr/local/logserver/logstash/startup.sh indexer account activity

databag

/usr/local/logserver/logstash/startup.sh indexer allinone

總結

以上是生活随笔為你收集整理的java+log日志服务器_Logserver日志服务器结构的全部內容，希望文章能夠幫你解決所遇到的問題。

如果覺得生活随笔網站內容還不錯，歡迎將生活随笔推薦給好友。