EOX GitLab Instance

Skip to content
Snippets Groups Projects
Commit eb863d9a authored by Bernhard Mallinger's avatar Bernhard Mallinger
Browse files

Merge branch '9-logging-setup' into 'master' 

Resolve "Integrate ELK stack with Fluentd"

Closes #9

See merge request !17
parents 451e056a bde49a1f
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
Showing
with 263 additions and 3 deletions
.gitlab-ci.yml
+ 10
− 0
View file @ eb863d9a
......@@ -29,6 +29,10 @@ build-master:
- IMAGE_4="$CI_REGISTRY_IMAGE/pvs_cache"
- docker pull "$IMAGE_4":latest || true
- docker build --cache-from "$IMAGE_4":latest -t "$IMAGE_4":latest -t "$IMAGE_4":$VERSION_4 cache/
- VERSION_5=`grep 'version="*"' fluentd/Dockerfile | cut -d '"' -f2`
- IMAGE_5="$CI_REGISTRY_IMAGE/fluentd"
- docker pull "$IMAGE_5":latest || true
- docker build --cache-from "$IMAGE_5":latest -t "$IMAGE_5":latest -t "$IMAGE_5":$VERSION_5 fluentd/
- chmod +x gitlab_test.sh
- ./gitlab_test.sh
- docker push "$IMAGE_1":$VERSION_1
......@@ -39,6 +43,8 @@ build-master:
- docker push "$IMAGE_3":latest
- docker push "$IMAGE_4":$VERSION_4
- docker push "$IMAGE_4":latest
- docker push "$IMAGE_5":$VERSION_5
- docker push "$IMAGE_5":latest
only:
- master
......@@ -66,6 +72,10 @@ build:
- docker pull "$IMAGE":latest || true
- docker build --cache-from "$IMAGE":latest -t "$IMAGE":$CI_COMMIT_REF_SLUG -t "$IMAGE":$CI_COMMIT_REF_NAME cache/
- docker tag "$IMAGE:$CI_COMMIT_REF_NAME" "$IMAGE:latest"
- IMAGE="$CI_REGISTRY_IMAGE/fluentd"
- docker pull "$IMAGE":latest || true
- docker build --cache-from "$IMAGE":latest -t "$IMAGE":$CI_COMMIT_REF_SLUG -t "$IMAGE":$CI_COMMIT_REF_NAME fluentd/
- docker tag "$IMAGE:$CI_COMMIT_REF_NAME" "$IMAGE:latest"
- chmod +x gitlab_test.sh
- ./gitlab_test.sh
except:
......
README.md
+ 10
− 0
View file @ eb863d9a
......@@ -209,6 +209,16 @@ docker volume rm vhr18-pvs_cache-db
docker volume rm vhr18-pvs_instance-data
```
### Setup logging
To access the logs, navigate to http://localhost:5601 . Ignore all of the fancy enterprise capabilities and select Kibana > Discover in the hamburger menu.
On first run, you need to define an index pattern to select the data source for kibana in elastic search.
Since we only have fluentd, you can just use `*` as index pattern.
Select `@timestamp` as time field
([see also](https://www.elastic.co/guide/en/kibana/current/tutorial-define-index.html)).
# Documentation
## Installation
......
cache/run-httpd.sh
+ 2
− 2
View file @ eb863d9a
#!/bin/bash -e
/configure.sh
/configure.sh >&2
echo "Running Apache server"
echo "Running Apache server" >&2
rm -rf /run/apache2/* /var/run/apache2/* /tmp/apache2*
exec /usr/sbin/apache2ctl -D FOREGROUND
docker-compose.dem.dev.yml
+ 12
− 0
View file @ eb863d9a
......@@ -9,6 +9,8 @@ services:
- type: bind
source: ./data/
target: /data/
logging:
driver: "fluentd"
client:
ports:
- "80:80"
......@@ -19,6 +21,8 @@ services:
- type: bind
source: ./data/
target: /data/
logging:
driver: "fluentd"
renderer:
ports:
- "81:80"
......@@ -27,6 +31,8 @@ services:
- type: bind
source: ./data/
target: /data/
logging:
driver: "fluentd"
registrar:
volumes:
- type: bind
......@@ -35,6 +41,8 @@ services:
- type: bind
source: ./core/
target: /core/
logging:
driver: "fluentd"
cache:
ports:
- "83:80"
......@@ -45,6 +53,8 @@ services:
configs:
- source: mapcache-dev
target: /mapcache-template.xml
logging:
driver: "fluentd"
preprocessor:
volumes:
- type: tmpfs
......@@ -55,6 +65,8 @@ services:
- type: bind
source: ./data/
target: /data/
logging:
driver: "fluentd"
networks:
extnet:
name: dem-extnet
......
docker-compose.emg.dev.yml
+ 12
− 0
View file @ eb863d9a
......@@ -9,6 +9,8 @@ services:
- type: bind
source: ./data/
target: /data/
logging:
driver: "fluentd"
client:
ports:
- "80:80"
......@@ -19,6 +21,8 @@ services:
- type: bind
source: ./data/
target: /data/
logging:
driver: "fluentd"
renderer:
ports:
- "81:80"
......@@ -27,6 +31,8 @@ services:
- type: bind
source: ./data/
target: /data/
logging:
driver: "fluentd"
registrar:
volumes:
- type: bind
......@@ -35,6 +41,8 @@ services:
- type: bind
source: ./core/
target: /core/
logging:
driver: "fluentd"
cache:
ports:
- "83:80"
......@@ -45,6 +53,8 @@ services:
configs:
- source: mapcache-dev
target: /mapcache-template.xml
logging:
driver: "fluentd"
preprocessor:
volumes:
- type: tmpfs
......@@ -55,6 +65,8 @@ services:
- type: bind
source: ./data/
target: /data/
logging:
driver: "fluentd"
networks:
extnet:
name: emg-extnet
......
docker-compose.logging.dev.yml 0 → 100644
+ 18
− 0
View file @ eb863d9a
version: "3.6"
services:
elasticsearch:
ports:
- "9200:9200"
environment:
- bootstrap.memory_lock=true
- "ES_JAVA_OPTS=-Xms256m -Xmx256m"
- "discovery.type=single-node"
deploy:
resources:
limits:
memory: 500M
kibana:
ports:
- "5601:5601"
logging:
driver: "fluentd"
docker-compose.logging.ops.yml 0 → 100644
+ 31
− 0
View file @ eb863d9a
version: "3.6"
services:
elasticsearch:
environment:
- bootstrap.memory_lock=true
- "ES_JAVA_OPTS=-Xms512m -Xmx512m"
- "discovery.type=single-node"
ulimits:
memlock:
soft: -1
hard: -1
nofile:
soft: 65536
hard: 65536
deploy:
placement:
constraints: [node.role == manager]
resources:
limits:
memory: 1000M
kibana:
deploy:
placement:
constraints: [node.role == manager]
labels:
# router for basic auth based access (https)
- "traefik.http.routers.logging.rule=Host(`kibana.prism.eox.at`)"
- "traefik.http.routers.logging.middlewares=apiauth@file,compress@file"
- "traefik.http.routers.logging.tls=true"
- "traefik.http.routers.logging.tls.certresolver=default"
- "traefik.http.routers.logging.entrypoints=https"
docker-compose.logging.yml 0 → 100644
+ 48
− 0
View file @ eb863d9a
version: "3.6"
services:
fluentd:
image: registry.gitlab.eox.at/esa/prism/vs/fluentd:latest
configs:
- source: fluentd-conf
target: /fluentd/etc/fluent.conf
ports:
- "24224:24224"
- "24224:24224/udp"
networks:
- logging
deploy:
replicas: 1
logging:
# fluentd can't start logging to itself
driver: "json-file"
elasticsearch:
image: elasticsearch:7.9.0
networks:
- logging
environment:
- bootstrap.memory_lock=true
- "ES_JAVA_OPTS=-Xms256m -Xmx256m"
- "discovery.type=single-node"
deploy:
replicas: 1
resources:
limits:
memory: 500M
volumes:
- es-data:/usr/share/elasticsearch/data
logging:
# there are startup issues if ES tried to log to fluentd
driver: "json-file"
kibana:
image: kibana:7.9.0
networks:
- logging
deploy:
replicas: 1
configs:
fluentd-conf:
file: ./fluentd/conf/fluent.conf
volumes:
es-data:
networks:
logging:
docker-compose.vhr18.dev.yml
+ 12
− 0
View file @ eb863d9a
......@@ -9,6 +9,8 @@ services:
- type: bind
source: ./data/
target: /data/
logging:
driver: "fluentd"
client:
ports:
- "80:80"
......@@ -19,6 +21,8 @@ services:
- type: bind
source: ./data/
target: /data/
logging:
driver: "fluentd"
renderer:
ports:
- "81:80"
......@@ -27,6 +31,8 @@ services:
- type: bind
source: ./data/
target: /data/
logging:
driver: "fluentd"
registrar:
volumes:
- type: bind
......@@ -35,6 +41,8 @@ services:
- type: bind
source: ./core/
target: /core/
logging:
driver: "fluentd"
cache:
ports:
- "83:80"
......@@ -45,6 +53,8 @@ services:
configs:
- source: mapcache-dev
target: /mapcache-template.xml
logging:
driver: "fluentd"
preprocessor:
volumes:
- type: tmpfs
......@@ -52,6 +62,8 @@ services:
- type: bind
source: ./preprocessor/
target: /preprocessor/
logging:
driver: "fluentd"
networks:
extnet:
name: vhr18-extnet
......
docker-compose.vhr18.yml
+ 18
− 0
View file @ eb863d9a
......@@ -12,6 +12,8 @@ services:
deploy:
placement:
constraints: [node.role == manager]
logging:
driver: "fluentd"
networks:
- intnet
redis:
......@@ -21,6 +23,8 @@ services:
deploy:
placement:
constraints: [node.role == manager]
logging:
driver: "fluentd"
networks:
- intnet
renderer:
......@@ -51,6 +55,8 @@ services:
target: /init-db.sh
deploy:
replicas: 1
logging:
driver: "fluentd"
networks:
- intnet
command:
......@@ -74,6 +80,10 @@ services:
WAIT_TIMEOUT: 300 # wait up to 5 minutes
deploy:
replicas: 1
logging:
driver: "fluentd"
options:
tag: "docker.apache2"
networks:
- intnet
command:
......@@ -96,6 +106,8 @@ services:
WAIT_SERVICES: "redis:6379 database:5432"
deploy:
replicas: 0
logging:
driver: "fluentd"
networks:
- intnet
command:
......@@ -112,6 +124,8 @@ services:
WAIT_SERVICES: "redis:6379"
deploy:
replicas: 1
logging:
driver: "fluentd"
networks:
- intnet
command:
......@@ -143,6 +157,8 @@ services:
target: /init-db.sh
deploy:
replicas: 1
logging:
driver: "fluentd"
networks:
- intnet
command:
......@@ -151,6 +167,8 @@ services:
image: registry.gitlab.eox.at/esa/prism/vs/pvs_client:latest
deploy:
replicas: 1
logging:
driver: "fluentd"
configs:
init-db:
file: ./config/vhr18_init-db.sh
......
fluentd/Dockerfile 0 → 100644
+ 40
− 0
View file @ eb863d9a
#------------------------------------------------------------------------------
#
# Project: prism view server
# Authors: Stephan Meissl <stephan.meissl@eox.at>
#
#------------------------------------------------------------------------------
# Copyright (C) 2018 EOX IT Services GmbH <https://eox.at>
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to
# deal in the Software without restriction, including without limitation the
# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
# sell copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies of this Software or works derived from this Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
# IN THE SOFTWARE.
#-----------------------------------------------------------------------------
FROM fluent/fluentd:v1.11.2-debian-1.0
MAINTAINER EOX
LABEL name="prism view server cache" \
vendor="EOX IT Services GmbH <https://eox.at>" \
license="MIT Copyright (C) 2019 EOX IT Services GmbH <https://eox.at>" \
type="prism view server fluentd" \
version="0.0.1-dev"
USER root
RUN gem install fluent-plugin-elasticsearch \
&& gem install fluent-plugin-rewrite-tag-filter
USER fluent
fluentd/conf/fluent.conf 0 → 100644
+ 49
− 0
View file @ eb863d9a
<system>
log_level debug
</system>
<source>
@type forward
port 24224
bind 0.0.0.0
</source>
<match docker.apache2>
@type rewrite_tag_filter
<rule>
key source
pattern /^(.*)$/
tag $1.${tag}
</rule>
</match>
<filter stdout.docker.apache2>
@type parser
key_name log
reserve_data true
<parse>
@type regexp
# this is basically apache2 but also has the request time at the end as extra field
expression /^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")? (?<request_serve_duration>[^ ]*)$/
time_format %d/%b/%Y:%H:%M:%S %z
types request_serve_duration:integer
</parse>
</filter>
<match *.**>
@type copy
<store>
@type elasticsearch
host elasticsearch
port 9200
logstash_format true
logstash_prefix fluentd
logstash_dateformat %Y%m%d
include_tag_key true
type_name access_log
tag_key @log_name
flush_interval 1s
</store>
</match>
gitlab_test.sh
+ 1
− 1
View file @ eb863d9a
......@@ -4,7 +4,7 @@ chmod +x env_setup.sh wait_for_container.sh
mkdir data
docker swarm init
docker network create -d overlay emg-extnet
docker stack deploy -c docker-compose.emg.yml -c docker-compose.emg.dev.yml emg-pvs
docker stack deploy -c docker-compose.emg.yml -c docker-compose.emg.dev.yml -c docker-compose.logging.yml emg-pvs
apk update && apk add bash postgresql-dev gcc python3-dev musl-dev py-pip gdal
pip3 install -r requirements.txt
./wait_for_container.sh
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment