Merge branch 'preprocessor-modularization' of gitlab.eox.at:esa/prism/vs into...

Merge branch 'preprocessor-modularization' of gitlab.eox.at:esa/prism/vs into preprocessor-modularization

preprocessor/preprocessor/preprocess.py

@@ -145,7 +145,7 @@ def preprocess_file(config: dict, file_path: os.PathLike, use_dir: os.PathLike=N
         # get a concrete configuration for the type, filled with the defaults
         default_config = dict(config['preprocessing'].get('defaults', {}))
         preprocess_config = dict(config['preprocessing']['types'].get(product_type, {}))
-        preprocess_config.update(default_config)
+        default_config.update(preprocess_config)
 
         logger.debug('Using preprocessing config %s' % pformat(preprocess_config))
 
@@ -199,10 +199,10 @@ def preprocess_file(config: dict, file_path: os.PathLike, use_dir: os.PathLike=N
 
                 logger.info('Running preprocessing step %s' % step)
                 os.mkdir(step)
-                preprpocessor = STEP_FUNCTIONS[step]
+                preprocessor = STEP_FUNCTIONS[step]
 
                 with Timer() as step_timer:
-                    preprpocessor(previous_step, step, **step_config)
+                    preprocessor(previous_step, step, **step_config)
 
                 logger.info(
                     'Finished preprocessing step %s after %f seconds.'

preprocessor/preprocessor/steps/calc.py

@@ -26,7 +26,7 @@ def calc_step(source_dir: os.PathLike, target_dir: os.PathLike, formulas: List[d
         if isfile(target_filename):
             logger.warn('Calc output filename %s already exists' % target_filename)
 
-        calc_formula(source_dir, item['inputs'], target_filename, item['formula'], item.get('data_type', 'Float32'))
+        calc_formula(source_dir, item['inputs'], target_filename, item['formula'], item.get('data_type', 'Float32'), item.get('nodata_value', None))
 
     # take all original files with from the last step
     for filename in glob('%s/*' % source_dir):
@@ -44,15 +44,15 @@ def calc_formula(source_dir: os.PathLike, inputs: List[dict], target_filename: o
         "--type", data_type,
     ]
 
-    for name, locator in inputs:
+    for name in inputs:
         # select first
-        filename = glob(join(source_dir, locator['glob']))[0]
+        filename = glob(join(source_dir, inputs[name]['glob']))[0]
         cmd.extend([
             "-%s" % name, filename,
-            "--%s_band=%d" % locator.get('band', 1),
+            "--%s_band=%d" % (name, inputs[name].get('band', 1)),
         ])
 
     if nodata_value is not None:
-        cmd.extend("--NoDataValue=%f" % nodata_value)
+        cmd.append("--NoDataValue=%f" % nodata_value)
 
     subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)

preprocessor/preprocessor/steps/output.py

@@ -15,8 +15,7 @@ def output_step(source_dir: os.PathLike, target_dir: os.PathLike, options: dict=
     driver = gdal.GetDriverByName(frmt)
     if not driver:
         raise ValueError('Unsupported driver %s' % frmt)
-    extension = driver.GetMetadata()['DMD_EXTENSIONS'].split(' ')[0]
-
+    extension = driver.GetMetadata().get('DMD_EXTENSIONS', 'tif').split(' ')[0]
     # warp each individual file
     warped_files = []
     for filename in glob(join(source_dir, '*.tif')):

preprocessor/preprocessor/steps/stack.py

@@ -12,8 +12,7 @@ from ..util import replace_ext
 def stack_bands_step(source_dir: os.PathLike, target_dir: os.PathLike, group_by: str=None, sort_by: str=None, order: List[str]=None):
     """ Stack bands of the individual images
     """
-    filenames = glob(join(source_dir, '*/*.tif'), recursive=True)
-
+    filenames = glob(join(source_dir, '*.tif'), recursive=True)
     # check if we have a group_by regex. If yes, use the first
     # re-group to group by.
     # Fallback is basename of file as groupname