Graylog+Elasticsearch+MongoDB+Nginx+Rsyslog日志集中收集解决方案

前言

Graylog官网：https://www.graylog.org/
Graylog docs： https://www.graylog.org/
Graylog download：https://www.graylog.org/

实践操作

环境说明

[root@linux-node01 ~]# cat /etc/redhat-release
CentOS Linux release 7.4.1708 (Core)
[root@linux-node01 ~]# uname -r
3.10.0-693.el7.x86_64
[root@linux-node01 ~]# uname -m
x86_64

MongoDB安装

# yum install -y mongodb-*
# grep -vE "^$|^#" /etc/mongod.conf
bind_ip = 127.0.0.1
port = 27017
auth = true
# systemctl enable mongod
# systemctl start mongod
# netstat -tunlp|grep mongod
tcp        0      0 127.0.0.1:27017         0.0.0.0:*               LISTEN      3120/mongod
# mongo
MongoDB shell version: 2.6.12
connecting to: test
Welcome to the MongoDB shell.
For interactive help, type "help".
For more comprehensive documentation, see
        http://docs.mongodb.org/
Questions? Try the support group
        http://groups.google.com/group/mongodb-user
Server has startup warnings:
2018-04-22T12:08:56.915+0800 [initandlisten]
2018-04-22T12:08:56.915+0800 [initandlisten] ** WARNING: Readahead for /var/lib/mongodb is set to 4096KB
2018-04-22T12:08:56.915+0800 [initandlisten] **          We suggest setting it to 256KB (512 sectors) or less
2018-04-22T12:08:56.915+0800 [initandlisten] **          http://dochub.mongodb.org/core/readahead
> use graylog2
switched to db graylog2
> db.addUser("login","123456")
WARNING: The 'addUser' shell helper is DEPRECATED. Please use 'createUser' instead
Successfully added user: { "user" : "login", "roles" : [ "dbOwner" ] }

JDK安装

Graylog及Elasticsearch都需要java环境

JDK下载地址,下载jdk-8u171-linux-x64.rpm

# rpm -ivh jdk-8u171-linux-x64.rpm
# vim /etc/profile
# JAVA bin PATH setup
export JAVA_HOME=/usr/java/jdk1.8.0_171/
export CLASSPATH=.:$JAVA_HOME/jre/lib/rt.jar:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar
export PATH=$PATH:$JAVA_HOME/bin
# java -version
java version "1.8.0_171"
Java(TM) SE Runtime Environment (build 1.8.0_171-b11)
Java HotSpot(TM) 64-Bit Server VM (build 25.171-b11, mixed mode)

Elasticsearch安装

官网地址：https://www.elastic.co/products/elasticsearch

# wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-6.2.4.rpm
# rpm -ivh elasticsearch-6.2.4.rpm
# grep -vE "^$|^#" /etc/elasticsearch/elasticsearch.yml
cluster.name: graylog
# systemctl daemon-reload
# systemctl enable elasticsearch.service
# systemctl start elasticsearch.service
#  curl -X GET http://localhost:9200
{
  "name" : "elcTqyk",
  "cluster_name" : "graylog",
  "cluster_uuid" : "NJjhhbAyToOw_6by88wG8A",
  "version" : {
    "number" : "6.2.4",
    "build_hash" : "ccec39f",
    "build_date" : "2018-04-12T20:37:28.497551Z",
    "build_snapshot" : false,
    "lucene_version" : "7.2.1",
    "minimum_wire_compatibility_version" : "5.6.0",
    "minimum_index_compatibility_version" : "5.0.0"
  },
  "tagline" : "You Know, for Search"
}

```/
## Graylog安装
> 参照官网RPM安装方式，[地址](http://docs.graylog.org/en/latest/pages/installation/operating_system_packages.html)

``` bash
# yum install -y pwgen
# pwgen -N 1 -s 96
b6lQupldhq7OdH7qac7jg0ulFHgYoVqAhwDZSpmzpLwFXTWJE3o9GqLFYA4Aac2DuXlPy91VradEgQXYvVfIcWbpT2gexAUb
# yum install -y perl-Digest-SHA
# echo -n 123456 | shasum -a 256
8d969eef6ecad3c29a3a629280e686cf0c3f5d5a86aff3ca12020c923adc6c92  -

# rpm -Uvh https://packages.graylog2.org/repo/packages/graylog-2.4-repository_latest.rpm
# yum install -y graylog-server
# grep -vE "^$|^#" /etc/graylog/server/server.conf
is_master = true
node_id_file = /etc/graylog/server/node-id
root_username = admin
root_password_sha2 = 8d969eef6ecad3c29a3a629280e686cf0c3f5d5a86aff3ca12020c923adc6c92
plugin_dir = /usr/share/graylog-server/plugin
rest_listen_uri = http://0.0.0.0:9000/api/
rest_transport_uri = http://192.168.200.100:9000/api/
web_listen_uri = http://0.0.0.0:9000/
elasticsearch_hosts = http://127.0.0.1:9200
rotation_strategy = count
elasticsearch_max_docs_per_index = 20000000
elasticsearch_max_number_of_indices = 20
retention_strategy = delete
elasticsearch_shards = 4
elasticsearch_replicas = 0
elasticsearch_index_prefix = graylog
allow_leading_wildcard_searches = false
allow_highlighting = false
elasticsearch_analyzer = standard
output_batch_size = 500
output_flush_interval = 1
output_fault_count_threshold = 5
output_fault_penalty_seconds = 30
processbuffer_processors = 5
outputbuffer_processors = 3
processor_wait_strategy = blocking
ring_size = 65536
inputbuffer_ring_size = 65536
inputbuffer_processors = 2
inputbuffer_wait_strategy = blocking
message_journal_enabled = true
message_journal_dir = /var/lib/graylog-server/journal
lb_recognition_period_seconds = 3
mongodb_uri = mongodb://login:123456@localhost:27017/graylog2
mongodb_max_connections = 1000
mongodb_threads_allowed_to_block_multiplier = 5
content_packs_dir = /usr/share/graylog-server/contentpacks
content_packs_auto_load = grok-patterns.json
proxied_requests_thread_pool_size = 32

# systemctl start graylog-server
# systemctl enable graylog-server