안녕하세요. 개발자 모도리입니다.
서비스 운영 중 로그 관리가 필요하여 예전에 구축했었던 ELK를 이용한 로그 관제 시스템을 다시 구성해 봤습니다.
sudo add-apt-repository ppa:webupd8team/java
sudo apt update
sudo apt install oracle-java8-installer
sudo apt install oracle-java8-set-default
java -version
wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-6.6.2.deb
sudo dpkg -i elasticsearch-6.6.2.deb
sudo systemctl daemon-reload
sudo systemctl enable elasticsearch.service
sudo systemctl start elasticsearch.service
sudo systemctl status elasticsearch.service
● elasticsearch.service - Elasticsearch
Loaded: loaded (/usr/lib/systemd/system/elasticsearch.service; enabled; vendor preset: enabled)
Active: active (running) since Mon 2019-03-25 11:58:03 UTC; 13min ago
sudo systemctl disable elasticsearch.service
sudo systemctl stop elasticsearch.service
curl -X GET "localhost:9200/"
{
"name" : "xQub8IM",
"cluster_name" : "elasticsearch",
"cluster_uuid" : "7RLVGGnxR6qHbV9mYqA_mg",
"version" : {
"number" : "6.6.2",
"build_flavor" : "default",
"build_type" : "deb",
"build_hash" : "3bd3e59",
"build_date" : "2019-03-06T15:16:26.864148Z",
"build_snapshot" : false,
"lucene_version" : "7.6.0",
"minimum_wire_compatibility_version" : "5.6.0",
"minimum_index_compatibility_version" : "5.0.0"
},
"tagline" : "You Know, for Search"
}
wget https://artifacts.elastic.co/downloads/kibana/kibana-6.6.2-amd64.deb
sudo dpkg -i kibana-6.6.2-amd64.deb
sudo systemctl enable kibana.service
sudo systemctl start kibana.service
sudo systemctl status kibana.service
● kibana.service - Kibana
Loaded: loaded (/etc/systemd/system/kibana.service; enabled; vendor preset: enabled)
Active: active (running) since Mon 2019-03-25 11:58:03 UTC; 13min ago
curl -v [Log 수집 서버 IP]:5601
* Rebuilt URL to: localhost:5601/
* Trying 127.0.0.1...
* TCP_NODELAY set
* Connected to localhost (127.0.0.1) port 5601 (#0)
> GET / HTTP/1.1
> Host: localhost:5601
> User-Agent: curl/7.58.0
> Accept: */*
>
< HTTP/1.1 302 Found
< location: /app/kibana
< kbn-name: kibana
< kbn-xpack-sig: 292ada877125f67092b9a6a4b59b08ca
< content-type: text/html; charset=utf-8
< cache-control: no-cache
< content-length: 0
< connection: close
< Date: Tue, 26 Mar 2019 02:11:00 GMT
<
* Closing connection 0
wget https://artifacts.elastic.co/downloads/logstash/logstash-6.6.2.deb
sudo dpkg -i logstash-6.6.2.deb
sudo systemctl enable logstash.service
sudo systemctl start logstash.service
sudo systemctl status logstash.service
● logstash.service - logstash
Loaded: loaded (/etc/systemd/system/logstash.service; enabled; vendor preset: enabled)
Active: active (running) since Mon 2019-03-25 12:10:22 UTC; 17s ago
wget https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-6.6.2-x86_64.rpm
sudo rpm -vi filebeat-6.6.2-x86_64.rpm
sudo systemctl enable filebeat.service
sudo systemctl start filebeat.service
sudo systemctl status filebeat.service
● filebeat.service - Filebeat sends log files to Logstash or directly to Elasticsearch.
Loaded: loaded (/usr/lib/systemd/system/filebeat.service; enabled; vendor preset: disabled)
Active: active (running) since 월 2019-03-25 21:41:23 KST; 51s ago
sudo vi /etc/elasticsearch/elasticsearch.yml
network.host : 0.0.0.0
sudo systemctl restart elasticsearch.service
sudo vi /etc/kibana/kibana.yml
server.host : "0.0.0.0"
elasticsearch.url : "http://elasticsearch_server_address:9200"
sudo systemctl restart kibana.service
sudo vi /etc/logstash/conf.d/kstarlive-web.conf
input {
beats {
port => 5044
}
}
filter {
if [fields][log_type] == "nginx_access" {
grok {
match => { "message" => ["%{IPORHOST:[nginx][access][remote_ip]} - %{DATA:[nginx][access][user_name]} \[%{HTTPDATE:[nginx][access][time]}\] \"%{WORD:[nginx][access][method]} %{DATA:[nginx][access][url]} HTTP/%{NUMBER:[nginx][access][http_version]}\" %{NUMBER:[nginx][access][response_code]} %{NUMBER:[nginx][access][body_sent][bytes]} \"%{DATA:[nginx][access][referrer]}\"\"%{DATA:[nginx][access][agent]}\""] }
remove_field => "message"
}
mutate {
add_field => { "read_timestamp" => "%{@timestamp}" }
}
date {
match => [ "[nginx][access][time]", "dd/MMM/YYYY:H:m:s Z" ]
remove_field => "[nginx][access][time]"
}
useragent {
source => "[nginx][access][agent]"
target => "[nginx][access][user_agent]"
remove_field => "[nginx][access][agent]"
}
geoip {
source => "[nginx][access][remote_ip]"
target => "[nginx][access][geoip]"
}
}
}
output {
if([fields][log_type] == "nginx_access") {
elasticsearch {
hosts => ["localhost:9200"]
index => "access-log-%{+YYYY.MM.dd}"
}
}
else if([fields][log_type] == "nginx_error") {
elasticsearch {
hosts => ["localhost:9200"]
index => "error-log-%{+YYYY.MM.dd}"
}
}
else if([fields][log_type] == "laravel") {
elasticsearch {
hosts => ["localhost:9200"]
index => "laravel-log-%{+YYYY.MM.dd}"
}
}
}
sudo systemctl restart logstash.service
sudo filebeat modules enable nginx
sudo filebeat modules list
sudo filebeat setup -e
sudo vi /etc/filebeat/modules.d/nginx.yml
- module: nginx
# Access logs
access:
enabled: true
input:
fields:
server_name: dev-web
log_type: nginx_access
# Set custom paths for the log files. If left empty,
# Filebeat will choose the paths depending on your OS.
var.paths: ["/home/ec2-user/kstarlive_web/storage/logs/nginx/access.log"]
# Error logs
error:
enabled: true
input:
fields:
server_name: dev-web
log_type: nginx_error
# Set custom paths for the log files. If left empty,
# Filebeat will choose the paths depending on your OS.
var.paths: ["/home/ec2-user/kstarlive_web/storage/logs/nginx/error.log"]
sudo vi /etc/filebeat/filebeat.yml
#=========================== Filebeat inputs =============================
filebeat.inputs:
- type: log
enabled: true
paths:
- /home/ec2-user/kstarlive_web/www/storage/logs/*.log
fields:
server_name: dev-web
log_type: laravel
#-------------------------- Elasticsearch output ------------------------------
#output.elasticsearch:
#----------------------------- Logstash output --------------------------------
output.logstash:
# The Logstash hosts
hosts: ["10.0.1.45:5044"]
sudo systemctl restart filebeat.service
[Kibana가 설치 된 서버 주소]:5601 를 입력하여 Kibana에 접속한다.Management에 들어가서 Kibana > Index Patterns 선택Create index pattern을 누르면 Define index pattern 라는 입력창이 나온다.access-log-* 과 같은 형식으로 입력하면 access log만 모아서 볼 수 있다.@timestamp를 선택한다.Unrecognized VM option 'UseParNewGC'
Error: Could not create the Java Virtual Machine.
Error: A fatal exception has occurred. Program will exit.
chmod: cannot access '/etc/default/logstash': No such file or directory
github에서는 java 11 지원하도록 수정했다고 하는데, 배포판에서는 아직 지원 안하는 듯 https://github.com/elastic/logstash/issues/9316
JAVA8 설치 권장
Java HotSpot(TM) 64-Bit Server VM warning: INFO: os::commit_memory(0x00000000ca660000, 899284992, 0) failed; error='Cannot allocate memory' (errno=12)
[2019-03-26T04:35:00,614][INFO ][logstash.outputs.elasticsearch] retrying failed action with response code: 403 ({"type"=>"cluster_block_exception", "reason"=>"blocked by: [FORBIDDEN/12/index read-only / allow delete (api)];"})
curl -X PUT "localhost:9200/*/_settings" -H 'Content-Type: application/json' -d'{"index.blocks.read_only_allow_delete": null}'