EOX GitLab Instance

Commit 0d06bcce authored by Nikola Jankovic's avatar Nikola Jankovic 💻
Browse files

added helm chart

added bump config
optimized docker image
updated gitlab ci
added config sample
parent 8bc1d017
Pipeline #17540 passed with stage
in 15 seconds
[bumpversion]
current_version = 2.0.0-alpha.2
commit = True
tag = True
parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(\-(?P<release>[a-z]+)\.(?P<build>\d+))?
serialize =
{major}.{minor}.{patch}-{release}.{build}
{major}.{minor}.{patch}
tag_name = release-{new_version}
[bumpversion:part:release]
optional_value = final
first_value = alpha
values =
alpha
beta
rc
final
[bumpversion:file:.bumpversion.cfg]
search = current_version = {current_version}
[bumpversion:file:Dockerfile]
search = version="{current_version}"
replace = version="{new_version}"
[bumpversion:file:setup.py]
search = version="{current_version}"
replace = version="{new_version}"
[bumpversion:glob:chart/Chart.yaml]
search = appVersion: "{current_version}"
replace = appVersion: "{new_version}"
chart/
tests/
.gitignore
.gitlab-ci.yml
Dockerfile
.venv/
.git/
requirements-test.txt
......@@ -137,3 +137,4 @@ dmypy.json
bin
include
pyvenv.cfg
.vscode/
stages:
- test
- deploy
- chart
variables:
DOCKER_TLS_CERTDIR: ""
......@@ -9,32 +10,42 @@ test:
image: python:3.8-slim
stage: test
script:
- python3 setup.py install
- pip3 install -r requirements-test.txt
- pip3 install .
- pytest
deploy:
stage: deploy
deploy_latest:
image: docker:20.10.8
services:
- docker:20.10.8-dind
before_script:
- docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY
stage: deploy
script:
- docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" "$CI_REGISTRY"
- docker build -t $CI_REGISTRY_IMAGE .
- docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME
only:
- docker push $CI_REGISTRY_IMAGE:latest
except:
- tags
- branches
deploy-latest:
stage: deploy
deploy:
image: docker:20.10.8
services:
- docker:20.10.8-dind
before_script:
- docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY
stage: deploy
script:
- docker build -t $CI_REGISTRY_IMAGE .
- docker push $CI_REGISTRY_IMAGE:latest
- docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" "$CI_REGISTRY"
- docker build --cache-from $CI_REGISTRY_IMAGE:latest -t $CI_REGISTRY_IMAGE:$CI_COMMIT_TAG .
- docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_TAG
only:
- main
- tags
helm:
image: devth/helm:v3.6.3
stage: chart
script:
# push Helm chart
- "upload_filename=$(helm package ./chart | sed 's/^Successfully packaged chart and saved it to: \\(.*\\)/\\1/')"
- curl -u $HELM_CHART_REPOSITORY_CREDENTIALS -v -X POST https://charts-public.hub.eox.at/api/charts --data-binary "@${upload_filename}"
only:
- tags
- master
......@@ -25,7 +25,7 @@
# IN THE SOFTWARE.
#-----------------------------------------------------------------------------
FROM python:3.8
FROM python:3.8-slim
LABEL name="view server harvester" \
vendor="EOX IT Services GmbH <https://eox.at>" \
......@@ -34,10 +34,13 @@ LABEL name="view server harvester" \
WORKDIR /opt/harvester
ADD . .
COPY requirements.txt .
RUN pip3 install --no-cache-dir -r requirements.txt
RUN pip3 install .
COPY . .
CMD run-harvester.sh
RUN python3 setup.py install
CMD ["./run-harvester.sh"]
LABEL version="2.0.0-alpha.2"
# harvester
Service used to harvest data from various endpoints
# Patterns to ignore when building packages.
# This supports shell glob matching, relative path matching, and
# negation (prefixed with !). Only one pattern per line.
.DS_Store
# Common VCS dirs
.git/
.gitignore
.bzr/
.bzrignore
.hg/
.hgignore
.svn/
# Common backup files
*.swp
*.bak
*.tmp
*.orig
*~
# Various IDEs
.project
.idea/
*.tmproj
.vscode/
apiVersion: v2
name: vs-harvester
description: A Helm chart for the View server harvester
# A chart can be either an 'application' or a 'library' chart.
#
# Application charts are a collection of templates that can be packaged into versioned archives
# to be deployed.
#
# Library charts provide useful utilities or functions for the chart developer. They're included as
# a dependency of application charts to inject those utilities and functions into the rendering
# pipeline. Library charts do not define any templates and therefore cannot be deployed.
type: application
# This is the chart version. This version number should be incremented each time you make changes
# to the chart and its templates, including the app version.
# Versions are expected to follow Semantic Versioning (https://semver.org/)
version: 0.1.0
# This is the version number of the application being deployed. This version number should be
# incremented each time you make changes to the application. Versions are not expected to
# follow Semantic Versioning. They should reflect the version the application is using.
# It is recommended to use it with quotes.
appVersion: "2.0.0-alpha.2"
{{- toYaml .Values.config | nindent 2 -}}
{{/*
Expand the name of the chart.
*/}}
{{- define "vs-harvester.name" -}}
{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
{{- end }}
{{/*
Create a default fully qualified app name.
We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
If release name contains chart name it will be used as a full name.
*/}}
{{- define "vs-harvester.fullname" -}}
{{- if .Values.fullnameOverride }}
{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
{{- else }}
{{- $name := default .Chart.Name .Values.nameOverride }}
{{- if contains $name .Release.Name }}
{{- .Release.Name | trunc 63 | trimSuffix "-" }}
{{- else }}
{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
{{- end }}
{{- end }}
{{- end }}
{{/*
Create chart name and version as used by the chart label.
*/}}
{{- define "vs-harvester.chart" -}}
{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
{{- end }}
{{/*
Common labels
*/}}
{{- define "vs-harvester.labels" -}}
helm.sh/chart: {{ include "vs-harvester.chart" . }}
{{ include "vs-harvester.selectorLabels" . }}
{{- if .Chart.AppVersion }}
app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
{{- end }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
{{- end }}
{{/*
Selector labels
*/}}
{{- define "vs-harvester.selectorLabels" -}}
app.kubernetes.io/name: {{ include "vs-harvester.name" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
{{- end }}
{{/*
Create the name of the service account to use
*/}}
{{- define "vs-harvester.serviceAccountName" -}}
{{- if .Values.serviceAccount.create }}
{{- default (include "vs-harvester.fullname" .) .Values.serviceAccount.name }}
{{- else }}
{{- default "default" .Values.serviceAccount.name }}
{{- end }}
{{- end }}
apiVersion: v1
kind: ConfigMap
metadata:
name: {{ include "vs-harvester.fullname" . }}-config
data:
{{ (tpl (.Files.Glob "files/harvester-config.yaml").AsConfig . ) | nindent 2}}
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ include "vs-harvester.fullname" . }}
labels:
{{- include "vs-harvester.labels" . | nindent 4 }}
app.kubernetes.io/service: harvester
spec:
replicas: {{ .Values.replicaCount }}
selector:
matchLabels:
{{- include "vs-harvester.selectorLabels" . | nindent 6 }}
app.kubernetes.io/service: harvester
strategy:
rollingUpdate:
maxUnavailable: 0
type: RollingUpdate
template:
metadata:
annotations:
prometheus.io/scrape: "false"
labels:
{{- include "vs-harvester.selectorLabels" . | nindent 8 }}
app.kubernetes.io/service: harvester
spec:
{{- with .Values.imagePullSecrets }}
imagePullSecrets:
{{- toYaml . | nindent 8 }}
{{- end }}
containers:
- name: {{ .Chart.Name }}
image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default ( print "release-" .Chart.AppVersion ) }}"
imagePullPolicy: {{ .Values.image.pullPolicy | default "IfNotPresent" }}
resources:
{{- toYaml .Values.resources | nindent 12 }}
args:
- /opt/harvester/run-harvester.sh
env:
{{- range $key, $value := .Values.global.env }}
- name: {{ $key }}
value: {{ $value | quote }}
{{- end }}
- name: REDIS_HOST
value: {{ .Release.Name }}-redis-master
volumeMounts:
- mountPath: /config.yaml
name: harvester-config
subPath: harvester-config.yaml
{{- range $volumes := .Values.volumeMounts }}
- {{- toYaml $volumes | nindent 16 }}
{{- end }}
{{- with .Values.affinity }}
affinity:
{{- toYaml . | nindent 8 }}
{{- end }}
volumes:
- configMap:
items:
- key: harvester-config.yaml
path: harvester-config.yaml
name: {{ include "vs-harvester.fullname" . }}-config
name: harvester-config
{{- range $volumes := .Values.volumes }}
- {{- toYaml $volumes | nindent 12 }}
{{- end }}
replicaCount: 1
image:
repository: registry.gitlab.eox.at/vs/harvester
pullPolicy: IfNotPresent
tag: ""
imagePullSecrets: []
config: {}
resources: {}
affinity: {}
volumes: []
volumeMounts: []
global:
env:
REDIS_PORT: "6379"
REDIS_HARVESTER_QUEUE_KEY: harvester_queue
harvesters:
- name: S2L2A_Element84
endpoint:
url: https://earth-search.aws.element84.com/v0/
type: STACAPI # STACAPI, STACCatalog, OpenSearch, FTP, S3, Swift, OGCAPI, OADS
query:
time:
begin: 2021-08-01
end: 2021-08-31
property: datetime # datetime, updated, created ###if supported by stacapi###
collection: sentinel-s2-l2a-cogs
bbox: 14.9,47.7,16.4,48.7
filter:
and:
- during:
- property: updated
- - P5D
- !now
queue: register # register, ingest, delete, update, preprocess
File mode changed from 100644 to 100755
......@@ -11,24 +11,17 @@ setup(
description="harvester for PVS",
long_description=long_description,
long_description_content_type="text/markdown",
url="https://gitlab.eox.at/esa/prism/vs/-/tree/master/harvester",
url="https://gitlab.eox.at/vs/harvester",
packages=find_packages(),
classifiers=[
"Programming Language :: Python :: 3",
"License :: OSI Approved :: MIT License",
"Operating System :: OS Independent",
],
python_requires='>=3.6',
python_requires=">=3.6",
entry_points={
"console_scripts": [
"harvester = harvester.cli:cli",
],
},
install_requires=[
'boto3',
'botocore',
'pystac',
'pygeofilter',
'redis',
],
)
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment