Add basic efk stack for vhr18

docker-compose.efk.yml

+version: "3.6"
+
+services:
+  whoami:
+    image: tutum/hello-world
+    networks:
+      - net
+    ports:
+      - "8888:80"
+    logging:
+      driver: "fluentd"# Logging Driver
+      options:
+        tag: tutum    # TAG 
+    deploy:
+      restart_policy:
+           condition: on-failure
+           delay: 20s
+           max_attempts: 3
+           window: 120s
+      mode: replicated
+      replicas: 1
+      placement:
+        constraints: [node.role == manager]
+      update_config:
+        delay: 2s
+ 
+  vizualizer:
+      image: dockersamples/visualizer
+      volumes:
+         - /var/run/docker.sock:/var/run/docker.sock
+      ports:
+        - "8080:8080"
+      networks:
+        - net
+      logging:
+        driver: "fluentd"
+        options:
+         tag: visualizer   #TAG 
+      deploy:
+          restart_policy:
+             condition: on-failure
+             delay: 20s
+             max_attempts: 3
+             window: 120s
+          mode: replicated # one container per manager node
+          replicas: 1
+          update_config:
+            delay: 2s
+          placement:
+             constraints: [node.role == manager]
+ 
+        
+  fluentd:
+    image: fluentd-bm:1
+    volumes:
+      - ./fluentd/conf:/fluentd/etc
+    ports:
+      - "24224:24224"
+      - "24224:24224/udp"
+    networks:
+      - net
+    deploy:
+      restart_policy:
+           condition: on-failure
+           delay: 20s
+           max_attempts: 3
+           window: 120s
+      mode: replicated
+      replicas: 1
+      placement:
+        constraints: [node.role == manager]
+      update_config:
+        delay: 2s
+ 
+  elasticsearch:
+    image: elasticsearch:7.9.0
+    ports:
+      - "9200:9200"
+    networks:
+      - net
+    environment:
+      - bootstrap.memory_lock=true
+      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
+      - "discovery.type=single-node"
+    logging:
+        driver: "json-file"
+        options:
+          max-size: 10M
+          max-file: 1  
+    deploy:
+      restart_policy:
+        condition: on-failure
+        delay: 20s
+        max_attempts: 3
+        window: 120s
+      mode: replicated
+      replicas: 1
+      placement:
+        constraints: [node.role == manager]
+      update_config:
+        delay: 2s
+      resources:
+        limits:
+          memory: 1000M
+    volumes:
+      - ./esdata:/usr/share/elasticsearch/data    
+      
+  kibana:
+    image: kibana:7.9.0
+    ports:
+      - "5601:5601"
+    networks:
+      - net
+    logging:
+        driver: "json-file"
+        options:
+           max-size: 10M
+           max-file: 1        
+    deploy:
+      restart_policy:
+        condition: on-failure
+        delay: 20s
+        max_attempts: 3
+        window: 120s
+      mode: replicated
+      replicas: 1
+      placement:
+        constraints: [node.role == manager]
+      update_config:
+        delay: 2s  
+ 
+networks:
+  net:

docker-compose.vhr18.yml

@@ -12,6 +12,8 @@ services:
     deploy:
       placement:
         constraints: [node.role == manager]
+    logging:
+      driver: "fluentd"
     networks:
       - intnet
   redis:
@@ -21,6 +23,8 @@ services:
     deploy:
       placement:
         constraints: [node.role == manager]
+    logging:
+      driver: "fluentd"
     networks:
       - intnet
   renderer:
@@ -51,6 +55,8 @@ services:
         target: /init-db.sh
     deploy:
       replicas: 1
+    logging:
+      driver: "fluentd"
     networks:
       - intnet
     command:
@@ -74,6 +80,10 @@ services:
       WAIT_TIMEOUT: 300  # wait up to 5 minutes
     deploy:
       replicas: 1
+    logging:
+      driver: "fluentd"
+      options:
+        tag: "docker.apache2"
     networks:
       - intnet
     command:
@@ -96,6 +106,8 @@ services:
       WAIT_SERVICES: "redis:6379 database:5432"
     deploy:
       replicas: 0
+    logging:
+      driver: "fluentd"
     networks:
       - intnet
     command:
@@ -112,6 +124,8 @@ services:
       WAIT_SERVICES: "redis:6379"
     deploy:
       replicas: 1
+    logging:
+      driver: "fluentd"
     networks:
       - intnet
     command:
@@ -143,6 +157,8 @@ services:
         target: /init-db.sh
     deploy:
       replicas: 1
+    logging:
+      driver: "fluentd"
     networks:
       - intnet
     command:
@@ -151,6 +167,49 @@ services:
     image: registry.gitlab.eox.at/esa/prism/vs/pvs_client:latest
     deploy:
       replicas: 1
+  fluentd:
+    image: fluentd-bm:2
+    volumes:
+      - ./fluentd/conf:/fluentd/etc
+    ports:
+      - "24224:24224"
+      - "24224:24224/udp"
+    networks:
+      - intnet
+    deploy:
+      replicas: 1
+      placement:
+        constraints: [node.role == manager]
+  elasticsearch:
+    image: elasticsearch:7.9.0
+    ports:
+      - "9200:9200"
+    networks:
+      - intnet
+    environment:
+      - bootstrap.memory_lock=true
+      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
+      - "discovery.type=single-node"
+    deploy:
+      replicas: 1
+      placement:
+        constraints: [node.role == manager]
+      resources:
+        limits:
+          memory: 1000M
+    volumes:
+      - ./esdata:/usr/share/elasticsearch/data
+  kibana:
+    image: kibana:7.9.0
+    ports:
+      - "5601:5601"
+    networks:
+      - intnet
+    deploy:
+      mode: replicated
+      replicas: 1
+      placement:
+        constraints: [node.role == manager]
 configs:
   init-db:
     file: ./config/vhr18_init-db.sh

fluentd/Dockerfile

+FROM fluent/fluentd:v1.11.2-debian-1.0
+USER root
+RUN gem install fluent-plugin-elasticsearch \
+ && gem install fluent-plugin-rewrite-tag-filter
+USER fluent

fluentd/conf/fluent.conf

+<system>
+  log_level debug
+</system>
+
+<source>
+  @type forward
+  port 24224
+  bind 0.0.0.0
+</source>
+
+<match docker.apache2>
+  @type rewrite_tag_filter
+  <rule>
+    key source
+    pattern /^(.*)$/
+    tag $1.${tag}
+  </rule>
+</match>
+
+<filter stdout.docker.apache2>
+  @type parser
+  key_name log
+  <parse>
+   @type regexp
+   # this is basically apache2 but also has the request time at the end as extra field
+   expression /^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")? (?<request_serve_duration>[^ ]*)$/
+   time_format %d/%b/%Y:%H:%M:%S %z
+   types request_serve_duration:integer
+  </parse>
+</filter>
+
+<filter YYY.stderr.docker.apache2>
+  @type parser
+  key_name log
+  <parse>
+   @type apache_error
+  </parse>
+</filter>
+
+
+<match *.**>
+  @type copy
+
+  <store>
+    @type elasticsearch
+    host elasticsearch
+    port 9200
+    logstash_format true
+    logstash_prefix fluentd
+    logstash_dateformat %Y%m%d
+    include_tag_key true
+    type_name access_log
+    tag_key @log_name
+    flush_interval 1s
+  </store>
+
+  <store>
+    @type stdout
+  </store>
+</match>