Installation and configuration of the ELK Stack (Elasticsearch, Logstash, Kibana)

  • warning: Illegal string offset 'files' in /data/web/1/000/027/003/273448/htdocs/panticz.de/modules/upload/upload.module on line 281.
  • warning: preg_replace(): The /e modifier is no longer supported, use preg_replace_callback instead in /data/web/1/000/027/003/273448/htdocs/panticz.de/includes/unicode.inc on line 349.
  • warning: preg_replace(): The /e modifier is no longer supported, use preg_replace_callback instead in /data/web/1/000/027/003/273448/htdocs/panticz.de/includes/unicode.inc on line 349.
  • warning: preg_replace(): The /e modifier is no longer supported, use preg_replace_callback instead in /data/web/1/000/027/003/273448/htdocs/panticz.de/includes/unicode.inc on line 349.
  • warning: preg_replace(): The /e modifier is no longer supported, use preg_replace_callback instead in /data/web/1/000/027/003/273448/htdocs/panticz.de/includes/unicode.inc on line 349.
  • warning: preg_replace(): The /e modifier is no longer supported, use preg_replace_callback instead in /data/web/1/000/027/003/273448/htdocs/panticz.de/includes/unicode.inc on line 349.
  • warning: preg_replace(): The /e modifier is no longer supported, use preg_replace_callback instead in /data/web/1/000/027/003/273448/htdocs/panticz.de/includes/unicode.inc on line 349.
  • warning: preg_replace(): The /e modifier is no longer supported, use preg_replace_callback instead in /data/web/1/000/027/003/273448/htdocs/panticz.de/includes/unicode.inc on line 349.
  • warning: preg_replace(): The /e modifier is no longer supported, use preg_replace_callback instead in /data/web/1/000/027/003/273448/htdocs/panticz.de/includes/unicode.inc on line 349.
  • warning: preg_replace(): The /e modifier is no longer supported, use preg_replace_callback instead in /data/web/1/000/027/003/273448/htdocs/panticz.de/includes/unicode.inc on line 349.
  • warning: preg_replace(): The /e modifier is no longer supported, use preg_replace_callback instead in /data/web/1/000/027/003/273448/htdocs/panticz.de/includes/unicode.inc on line 349.
  • warning: preg_replace(): The /e modifier is no longer supported, use preg_replace_callback instead in /data/web/1/000/027/003/273448/htdocs/panticz.de/includes/unicode.inc on line 349.
  • warning: preg_replace(): The /e modifier is no longer supported, use preg_replace_callback instead in /data/web/1/000/027/003/273448/htdocs/panticz.de/includes/unicode.inc on line 349.
  • warning: preg_replace(): The /e modifier is no longer supported, use preg_replace_callback instead in /data/web/1/000/027/003/273448/htdocs/panticz.de/includes/unicode.inc on line 349.
  • warning: preg_replace(): The /e modifier is no longer supported, use preg_replace_callback instead in /data/web/1/000/027/003/273448/htdocs/panticz.de/includes/unicode.inc on line 349.
  • warning: preg_replace(): The /e modifier is no longer supported, use preg_replace_callback instead in /data/web/1/000/027/003/273448/htdocs/panticz.de/includes/unicode.inc on line 349.
  • warning: preg_replace(): The /e modifier is no longer supported, use preg_replace_callback instead in /data/web/1/000/027/003/273448/htdocs/panticz.de/includes/unicode.inc on line 349.
  • warning: preg_replace(): The /e modifier is no longer supported, use preg_replace_callback instead in /data/web/1/000/027/003/273448/htdocs/panticz.de/includes/unicode.inc on line 349.

# Overview and download homepage

#
# Prerequirements (Elasticsearch and Logstash are Java packages so please install Java JRE first)
#
# Install Java JRE package on Debian
apt-get install -y openjre-7-jre

#
# Elasticsearch (distributed restful search and analytics)
#
# Install Elasticsearch package on Debian
wget -P /tmp
dpkg -i /tmp/elasticsearch-1.3.2.deb

# Enable Elasticsearch daemon
update-rc.d elasticsearch defaults 95 10

# Start Elasticsearch manually
/etc/init.d/elasticsearch start

#
# Logstash (manage events and logs)
#
# Install Logstash package on Debian
wget -P /tmp
dpkg -i /tmp/logstash_1.4.2-1-2c0f5a1_all.deb

# Optional: Install Logstash contrib package (plug-ins contributed by the community and not supported by Elasticsearch)
wget -P /tmp
dpkg -i /tmp/logstash-contrib_1.4.2-1-efd53ef_all.deb

# Enable Logstash daemon by default
update-rc.d logstash defaults 96 10

# Start Logstash manually
/etc/init.d/elasticsearch start

#
# Kibana (webinterface to visualize ElasticSearch data)
#
# Kibana is already included in the Logstash Debian package.
# URL:
#
# Optinal: There is also a stand-alone archive avaiable with can by installed on a different webserver:

# Enable Kibana webservie by default
update-rc.d logstash-web defaults 97 10

# Start Kibana manually
/etc/init.d/logstash-web start

# Optional: configure the Elasticsearch server FQHN
Open config.js and edit the "elasticsearch" parameter to the fully qualified hostname of your Elasticsearch server

# Logstash config for apache.log

cat < /etc/logstash/conf.d/logstash.conf
/etc/logstash/conf.d/apache_access_log.conf
input {
file {
path => "/var/log/apache2/access.log"
start_position => "beginning"
# sincedb_path => "/dev/null" # dont track the position of monitored log files
}
}

filter {
grok {
pattern => "%{IP:remote_ip} - - \[%{HTTPDATE:time}\] \"(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})\" %{NUMBER:response} (?:%{NUMBER:bytes:int}|-) %{HOST:host} \"(?[^\"]*)\" \"(?[^\"]*)\" \"(?[^\"]*)\" %{NUMBER:duration:int} microsec"
}
}

output {
# DEBUG: output to console
# stdout {
# codec => rubydebug
# }

elasticsearch {
host => localhost
}
}
EOF

# get total index size

# show config
curl | python -m json.tool

# delete old logs
apt install elasticsearch-curator
curl -XDELETE
curl -XDELETE