EOX GitLab Instance

Skip to content
Snippets Groups Projects
Commit 707eb35d authored by Fabian Schindler's avatar Fabian Schindler
Browse files

Merge branch 'preprocessor-modularization' of gitlab.eox.at:esa/prism/vs into...

Merge branch 'preprocessor-modularization' of gitlab.eox.at:esa/prism/vs into preprocessor-modularization
parents 661e94a4 8d62363f
No related branches found
No related tags found
No related merge requests found
...@@ -145,7 +145,7 @@ def preprocess_file(config: dict, file_path: os.PathLike, use_dir: os.PathLike=N ...@@ -145,7 +145,7 @@ def preprocess_file(config: dict, file_path: os.PathLike, use_dir: os.PathLike=N
# get a concrete configuration for the type, filled with the defaults # get a concrete configuration for the type, filled with the defaults
default_config = dict(config['preprocessing'].get('defaults', {})) default_config = dict(config['preprocessing'].get('defaults', {}))
preprocess_config = dict(config['preprocessing']['types'].get(product_type, {})) preprocess_config = dict(config['preprocessing']['types'].get(product_type, {}))
preprocess_config.update(default_config) default_config.update(preprocess_config)
logger.debug('Using preprocessing config %s' % pformat(preprocess_config)) logger.debug('Using preprocessing config %s' % pformat(preprocess_config))
...@@ -199,10 +199,10 @@ def preprocess_file(config: dict, file_path: os.PathLike, use_dir: os.PathLike=N ...@@ -199,10 +199,10 @@ def preprocess_file(config: dict, file_path: os.PathLike, use_dir: os.PathLike=N
logger.info('Running preprocessing step %s' % step) logger.info('Running preprocessing step %s' % step)
os.mkdir(step) os.mkdir(step)
preprpocessor = STEP_FUNCTIONS[step] preprocessor = STEP_FUNCTIONS[step]
with Timer() as step_timer: with Timer() as step_timer:
preprpocessor(previous_step, step, **step_config) preprocessor(previous_step, step, **step_config)
logger.info( logger.info(
'Finished preprocessing step %s after %f seconds.' 'Finished preprocessing step %s after %f seconds.'
......
...@@ -26,7 +26,7 @@ def calc_step(source_dir: os.PathLike, target_dir: os.PathLike, formulas: List[d ...@@ -26,7 +26,7 @@ def calc_step(source_dir: os.PathLike, target_dir: os.PathLike, formulas: List[d
if isfile(target_filename): if isfile(target_filename):
logger.warn('Calc output filename %s already exists' % target_filename) logger.warn('Calc output filename %s already exists' % target_filename)
calc_formula(source_dir, item['inputs'], target_filename, item['formula'], item.get('data_type', 'Float32')) calc_formula(source_dir, item['inputs'], target_filename, item['formula'], item.get('data_type', 'Float32'), item.get('nodata_value', None))
# take all original files with from the last step # take all original files with from the last step
for filename in glob('%s/*' % source_dir): for filename in glob('%s/*' % source_dir):
...@@ -44,15 +44,15 @@ def calc_formula(source_dir: os.PathLike, inputs: List[dict], target_filename: o ...@@ -44,15 +44,15 @@ def calc_formula(source_dir: os.PathLike, inputs: List[dict], target_filename: o
"--type", data_type, "--type", data_type,
] ]
for name, locator in inputs: for name in inputs:
# select first # select first
filename = glob(join(source_dir, locator['glob']))[0] filename = glob(join(source_dir, inputs[name]['glob']))[0]
cmd.extend([ cmd.extend([
"-%s" % name, filename, "-%s" % name, filename,
"--%s_band=%d" % locator.get('band', 1), "--%s_band=%d" % (name, inputs[name].get('band', 1)),
]) ])
if nodata_value is not None: if nodata_value is not None:
cmd.extend("--NoDataValue=%f" % nodata_value) cmd.append("--NoDataValue=%f" % nodata_value)
subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
...@@ -15,8 +15,7 @@ def output_step(source_dir: os.PathLike, target_dir: os.PathLike, options: dict= ...@@ -15,8 +15,7 @@ def output_step(source_dir: os.PathLike, target_dir: os.PathLike, options: dict=
driver = gdal.GetDriverByName(frmt) driver = gdal.GetDriverByName(frmt)
if not driver: if not driver:
raise ValueError('Unsupported driver %s' % frmt) raise ValueError('Unsupported driver %s' % frmt)
extension = driver.GetMetadata()['DMD_EXTENSIONS'].split(' ')[0] extension = driver.GetMetadata().get('DMD_EXTENSIONS', 'tif').split(' ')[0]
# warp each individual file # warp each individual file
warped_files = [] warped_files = []
for filename in glob(join(source_dir, '*.tif')): for filename in glob(join(source_dir, '*.tif')):
......
...@@ -12,8 +12,7 @@ from ..util import replace_ext ...@@ -12,8 +12,7 @@ from ..util import replace_ext
def stack_bands_step(source_dir: os.PathLike, target_dir: os.PathLike, group_by: str=None, sort_by: str=None, order: List[str]=None): def stack_bands_step(source_dir: os.PathLike, target_dir: os.PathLike, group_by: str=None, sort_by: str=None, order: List[str]=None):
""" Stack bands of the individual images """ Stack bands of the individual images
""" """
filenames = glob(join(source_dir, '*/*.tif'), recursive=True) filenames = glob(join(source_dir, '*.tif'), recursive=True)
# check if we have a group_by regex. If yes, use the first # check if we have a group_by regex. If yes, use the first
# re-group to group by. # re-group to group by.
# Fallback is basename of file as groupname # Fallback is basename of file as groupname
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment