diff --git a/docker-compose.efk.yml b/docker-compose.efk.yml new file mode 100644 index 0000000000000000000000000000000000000000..6ee6324a8a72b1814b3bf893378011335f242b32 --- /dev/null +++ b/docker-compose.efk.yml @@ -0,0 +1,133 @@ +version: "3.6" + +services: + whoami: + image: tutum/hello-world + networks: + - net + ports: + - "8888:80" + logging: + driver: "fluentd"# Logging Driver + options: + tag: tutum # TAG + deploy: + restart_policy: + condition: on-failure + delay: 20s + max_attempts: 3 + window: 120s + mode: replicated + replicas: 1 + placement: + constraints: [node.role == manager] + update_config: + delay: 2s + + vizualizer: + image: dockersamples/visualizer + volumes: + - /var/run/docker.sock:/var/run/docker.sock + ports: + - "8080:8080" + networks: + - net + logging: + driver: "fluentd" + options: + tag: visualizer #TAG + deploy: + restart_policy: + condition: on-failure + delay: 20s + max_attempts: 3 + window: 120s + mode: replicated # one container per manager node + replicas: 1 + update_config: + delay: 2s + placement: + constraints: [node.role == manager] + + + fluentd: + image: fluentd-bm:1 + volumes: + - ./fluentd/conf:/fluentd/etc + ports: + - "24224:24224" + - "24224:24224/udp" + networks: + - net + deploy: + restart_policy: + condition: on-failure + delay: 20s + max_attempts: 3 + window: 120s + mode: replicated + replicas: 1 + placement: + constraints: [node.role == manager] + update_config: + delay: 2s + + elasticsearch: + image: elasticsearch:7.9.0 + ports: + - "9200:9200" + networks: + - net + environment: + - bootstrap.memory_lock=true + - "ES_JAVA_OPTS=-Xms512m -Xmx512m" + - "discovery.type=single-node" + logging: + driver: "json-file" + options: + max-size: 10M + max-file: 1 + deploy: + restart_policy: + condition: on-failure + delay: 20s + max_attempts: 3 + window: 120s + mode: replicated + replicas: 1 + placement: + constraints: [node.role == manager] + update_config: + delay: 2s + resources: + limits: + memory: 1000M + volumes: + - ./esdata:/usr/share/elasticsearch/data + + kibana: + image: kibana:7.9.0 + ports: + - "5601:5601" + networks: + - net + logging: + driver: "json-file" + options: + max-size: 10M + max-file: 1 + deploy: + restart_policy: + condition: on-failure + delay: 20s + max_attempts: 3 + window: 120s + mode: replicated + replicas: 1 + placement: + constraints: [node.role == manager] + update_config: + delay: 2s + +networks: + net: diff --git a/docker-compose.vhr18.yml b/docker-compose.vhr18.yml index 8b808a9d9a4b2b08ea574f9fafb21e41e565c097..4260afe387cb66c146d1d329fe8730fda0b78674 100644 --- a/docker-compose.vhr18.yml +++ b/docker-compose.vhr18.yml @@ -12,6 +12,8 @@ services: deploy: placement: constraints: [node.role == manager] + logging: + driver: "fluentd" networks: - intnet redis: @@ -21,6 +23,8 @@ services: deploy: placement: constraints: [node.role == manager] + logging: + driver: "fluentd" networks: - intnet renderer: @@ -51,6 +55,8 @@ services: target: /init-db.sh deploy: replicas: 1 + logging: + driver: "fluentd" networks: - intnet command: @@ -74,6 +80,10 @@ services: WAIT_TIMEOUT: 300 # wait up to 5 minutes deploy: replicas: 1 + logging: + driver: "fluentd" + options: + tag: "docker.apache2" networks: - intnet command: @@ -96,6 +106,8 @@ services: WAIT_SERVICES: "redis:6379 database:5432" deploy: replicas: 0 + logging: + driver: "fluentd" networks: - intnet command: @@ -112,6 +124,8 @@ services: WAIT_SERVICES: "redis:6379" deploy: replicas: 1 + logging: + driver: "fluentd" networks: - intnet command: @@ -143,6 +157,8 @@ services: target: /init-db.sh deploy: replicas: 1 + logging: + driver: "fluentd" networks: - intnet command: @@ -151,6 +167,49 @@ services: image: registry.gitlab.eox.at/esa/prism/vs/pvs_client:latest deploy: replicas: 1 + fluentd: + image: fluentd-bm:2 + volumes: + - ./fluentd/conf:/fluentd/etc + ports: + - "24224:24224" + - "24224:24224/udp" + networks: + - intnet + deploy: + replicas: 1 + placement: + constraints: [node.role == manager] + elasticsearch: + image: elasticsearch:7.9.0 + ports: + - "9200:9200" + networks: + - intnet + environment: + - bootstrap.memory_lock=true + - "ES_JAVA_OPTS=-Xms512m -Xmx512m" + - "discovery.type=single-node" + deploy: + replicas: 1 + placement: + constraints: [node.role == manager] + resources: + limits: + memory: 1000M + volumes: + - ./esdata:/usr/share/elasticsearch/data + kibana: + image: kibana:7.9.0 + ports: + - "5601:5601" + networks: + - intnet + deploy: + mode: replicated + replicas: 1 + placement: + constraints: [node.role == manager] configs: init-db: file: ./config/vhr18_init-db.sh diff --git a/fluentd/Dockerfile b/fluentd/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..174646e7a3571d552b92c62243d97f9b46b1888a --- /dev/null +++ b/fluentd/Dockerfile @@ -0,0 +1,5 @@ +FROM fluent/fluentd:v1.11.2-debian-1.0 +USER root +RUN gem install fluent-plugin-elasticsearch \ + && gem install fluent-plugin-rewrite-tag-filter +USER fluent diff --git a/fluentd/conf/fluent.conf b/fluentd/conf/fluent.conf new file mode 100644 index 0000000000000000000000000000000000000000..ac25f7ef09c9d6ae629321f6efc6321a587facda --- /dev/null +++ b/fluentd/conf/fluent.conf @@ -0,0 +1,60 @@ +<system> + log_level debug +</system> + +<source> + @type forward + port 24224 + bind 0.0.0.0 +</source> + +<match docker.apache2> + @type rewrite_tag_filter + <rule> + key source + pattern /^(.*)$/ + tag $1.${tag} + </rule> +</match> + +<filter stdout.docker.apache2> + @type parser + key_name log + <parse> + @type regexp + # this is basically apache2 but also has the request time at the end as extra field + expression /^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")? (?<request_serve_duration>[^ ]*)$/ + time_format %d/%b/%Y:%H:%M:%S %z + types request_serve_duration:integer + </parse> +</filter> + +<filter YYY.stderr.docker.apache2> + @type parser + key_name log + <parse> + @type apache_error + </parse> +</filter> + + +<match *.**> + @type copy + + <store> + @type elasticsearch + host elasticsearch + port 9200 + logstash_format true + logstash_prefix fluentd + logstash_dateformat %Y%m%d + include_tag_key true + type_name access_log + tag_key @log_name + flush_interval 1s + </store> + + <store> + @type stdout + </store> +</match>