EOX GitLab Instance

Commit 5c243427 authored by Lubomir Doležal's avatar Lubomir Doležal
Browse files

allow to specify gdal_config_options in preprocessor per product type

parent 65d73da0
......@@ -15,7 +15,7 @@ from .steps import (
georeference_step, extract_subdataset_step, calc_step, stack_bands_step, output_step
from .steps.browse_report import browse_georeference
from .util import workdir, Timer, get_size_in_bytes
from .util import workdir, Timer, get_size_in_bytes, apply_gdal_config_options, set_gdal_options
from .exceptions import ExistsAtUploadError
......@@ -77,6 +77,8 @@ def flatten(l):
def preprocess_internal(preprocess_config, previous_step='unpack'):
force_refresh = False
# apply specific gdal config options
original_config = apply_gdal_config_options(preprocess_config)
# make processing steps
for step in ['custom_preprocessor', 'subdatasets', 'georeference', 'calc', 'stack_bands', 'output', 'custom_postprocessor']:
step_config = preprocess_config.get(step)
......@@ -108,6 +110,9 @@ def preprocess_internal(preprocess_config, previous_step='unpack'):
previous_step = step
# put back original configuration for further steps
if not os.path.isdir('upload') or force_refresh:
......@@ -118,8 +123,6 @@ def preprocess_internal(preprocess_config, previous_step='unpack'):
copy_files(previous_step, 'upload', move=preprocess_config.get('move_files', False))
def preprocess_file(config: dict, file_path: os.PathLike, use_dir: os.PathLike=None):
""" Runs the preprocessing of a single file.
......@@ -5,6 +5,8 @@ import logging
from ..util import replace_ext, gdal, get_all_data_files
logger = logging.getLogger(__name__)
def extract_subdataset_step(source_dir: os.PathLike, target_dir: os.PathLike, preprocessor_config: dict, subdataset_types: Dict[str, str]=None, data_file_globs: List[str]=[]):
filenames = get_all_data_files(source_dir, preprocessor_config, data_file_globs)
......@@ -107,3 +107,23 @@ def get_all_data_files(source_dir, preprocessor_config, data_file_globs=[]):
# get only unique files to compensate for possibly bad glob yielding doubles, keeping order
file_paths_filt = list(dict.fromkeys(file_paths_filt))
return file_paths_filt
def apply_gdal_config_options(preprocessor_config):
""" Applies config specific gdal configuration options for a given preprocessing step
Returning original values to allow switching them back after preprocessing done.
original_gdal_config_options = {}
for config_option in preprocessor_config.get('gdal_config_options', []):
key, _, val = config_option.partition('=')
orig_val = gdal.GetConfigOption(key)
gdal.SetConfigOption(key, val)
original_gdal_config_options[key] = orig_val
return original_gdal_config_options
def set_gdal_options(config_options):
""" Sets a key, value dictionary of config options to gdal
for key, value in config_options.items():
gdal.SetConfigOption(key, value)
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment