ELK日志系统搭建(草稿)

[root@luckly data]# cat logstash/logstash_pipeline.conf 
input {
  file {
    path => "/access.log"
    start_position => "beginning"
    sincedb_path => "/dev/null"
    codec => "json"
    type => "nginx_access"
  }
}

filter {
  if [type] == "nginx_access" {
    grok {
      match => { "message" => "%{IPORHOST:client_ip} - %{DATA:user_name} \[%{HTTPDATE:timestamp}\] \"%{WORD:method} %{DATA:request} HTTP/%{NUMBER:http_version}\" %{NUMBER:response_code} %{NUMBER:bytes_sent} \"%{DATA:referrer}\" \"%{DATA:user_agent}\"" }
    }
    date {
      match => [ "timestamp", "dd/MMM/YYYY:H:m:s Z" ]
      remove_field => ["timestamp"]
    }
  }
}

output {
  elasticsearch {
    hosts  => ["http://10.0.0.16:9200"]
    index  => "nginx_access_logs"
  }
}

[root@luckly data]# cat logstash/logstash_pipeline.conf.bak 
input {
    tcp {
      mode => "server"
      host => "0.0.0.0"
      port => 4560
      codec => json_lines
      type => "applicationName"
    }
}
output {
    elasticsearch {
      hosts  => ["http://10.0.0.16:9200"]
      index  => "%{[applicationName]}"
      codec  => "json"
    }
}

[root@luckly data]# cat >docker-compose.yaml<EOF
version: '3.7'
services:
  elasticsearch:
    image: elasticsearch:7.14.2
    container_name: elasticsearch
    restart: always
    ports:
      - "9200:9200"
      - "9300:9300"
    environment:
      - discovery.type=single-node
      - http.cors.enabled=true
      - http.cors.allow-origin="*"
      - TZ=Asia/Shanghai
      - bootstrap.memory_lock=true
      - ES_JAVA_OPTS=-Xms1024m -Xmx1024m
    volumes:
      - /data/elasticsearch/data:/usr/share/elasticsearch/data
    ulimits:
      memlock:
        soft: -1
        hard: -1

  kibana:
    image: kibana:7.14.2
    container_name: kibana1
    restart: always
    links:
      - elasticsearch
    ports:
      - "5601:5601"
    environment:
      - ELASTICSEARCH_HOSTS=http://10.0.0.16:9200
      - I18N_LOCALE=zh-CN
    depends_on:
      - elasticsearch

  es_head:
    image: ruanjf/elasticsearch-head
    container_name: es_head
    restart: always
    ports:
      - "9100:9100"  
    depends_on:
      - elasticsearch
   
  logstash:
    image: logstash:7.14.2
    container_name: logstash
    restart: always
    ports:
      - "5044:5044"
      - "9600:9600"
      - "4560:4560"
    depends_on: 
      - elasticsearch
    volumes:
      - /data/logstash/logstash_pipeline.conf:/usr/share/logstash/pipeline/logstash.conf
      - /var/log/nginx/access.log:/access.log
      - /data/logstash/data:/usr/share/logstash/data
EOF

[root@luckly ~]# mkdir -p /data/{elasticsearch,logstash}/data

[root@luckly ~]# tree /data/
/data/
|-- docker-compose.yaml
|-- elasticsearch
|   `-- data
`-- logstash
    |-- data
    |-- logstash_pipeline.conf
    `-- logstash_pipeline.conf.bak

4 directories, 3 files