remove need for separate glob in subdataset

config/emg_preprocessor-config.yml

@@ -231,6 +231,29 @@ preprocessing:
     DM02:
       data_file_globs:
         - "*.tif"
+    CS00:
+      data_file_globs:
+        - "*.h5"
+        - "*.tif"
+      subdatasets:
+        subdataset_types:
+          '//S01/SBI': 'S01_SBI'
+          '//S02/SBI': 'S02_SBI'
+          '//S03/SBI': 'S03_SBI'
+          '//S04/SBI': 'S04_SBI'
+          '//S05/SBI': 'S05_SBI'
+          '//S06/SBI': 'S06_SBI'
+          '//S01/MBI': 'S01_MBI'
+          '//S02/MBI': 'S02_MBI'
+          '//S03/MBI': 'S03_MBI'
+          '//S04/MBI': 'S04_MBI'
+          '//S05/MBI': 'S05_MBI'
+          '//S06/MBI': 'S06_MBI'
+      georeference:
+        - type: corners
+          corner_names: ["S01_SBI_Bottom_Left_Geodetic_Coordinates", "S01_SBI_Bottom_Right_Geodetic_Coordinates", "S01_SBI_Top_Left_Geodetic_Coordinates", "S01_SBI_Top_Right_Geodetic_Coordinates"]
+          orbit_direction_name: Orbit_Direction
+          force_north_up: false
 # this configuration is still a stub - not all product types are done 
 # https://gitlab.eox.at/esa/prism/vs/-/issues/56
 # https://gitlab.eox.at/esa/prism/vs/-/issues/23

documentation/operator-guide/configuration.rst

@@ -343,10 +343,6 @@ preprocessing
 
       What subdatasets to extract and how to name them.
 
-      data_file_glob
-
-        A file glob pattern to select files to extract from.
-
       subdataset_types
 
         Mapping of subdataset identifier to output filename postfix for

preprocessor/preprocessor/config-schema.yaml

@@ -107,9 +107,6 @@ definitions:
       description: The definition of the subdataset extraction step.
       type: object
       properties:
-        data_file_glob:
-          description: The data file selector.
-          type: string
         subdataset_types:
           description: Mapping of subdataset identifier to output filename postfix for subdatasets to be extracted for each data file.
           type: object

preprocessor/preprocessor/steps/georeference.py

@@ -125,10 +125,6 @@ def corner_georef(input_filename: os.PathLike, target_filename: os.PathLike, cor
     orbit_direction = ds.GetMetadata()[orbit_direction_name].lower()
     metadata = ds.GetRasterBand(1).GetMetadata()
 
-    # from pprint import pprint
-
-    # pprint (metadata)
-    # pprint(ds.GetMetadata())
     bl, br, tl, tr = [
         [float(num) for num in metadata[corner_name].split()]
         for corner_name in corner_names

preprocessor/preprocessor/steps/subdataset.py

 import os
-from os.path import join, splitext, basename, dirname
+from os.path import join, splitext, basename, dirname, isdir
 from glob import glob
 from typing import Dict
 
 from ..util import replace_ext, gdal
 
 
-def extract_subdataset_step(source_dir: os.PathLike, target_dir: os.PathLike, preprocessor_config: dict, data_file_glob: str, subdataset_types: Dict[str, str]=None):
-    datafiles = glob(join(source_dir, data_file_glob))
-    if not datafiles:
+def extract_subdataset_step(source_dir: os.PathLike, target_dir: os.PathLike, preprocessor_config: dict, subdataset_types: Dict[str, str]=None):
+    filenames = []
+    for dataglob in preprocessor_config.get('data_file_globs', '*'):
+        for p in [path for path in glob(join(source_dir, '**', dataglob), recursive=True) if not isdir(path)]:
+            filenames.append(p)
+    if len(filenames) == 0:
         raise Exception('No datafiles were matched by the provided glob')
 
     for filename in datafiles: