Restructuring into subpackages

Improved on processing steps

preprocessor/preprocessor/output.py

-from os.path import join, basename
-from glob import glob
-
-from osgeo import gdal
-
-
-def output_step(source_dir, target_dir, crs, driver, format_options):
-    for filename in glob(join(source_dir, '*')):
-        target_filename = join(target_dir, basename(filename))
-        gdal.Warp(target_filename, filename, options=gdal.WarpOptions(
-            dstSRS=crs,
-            format=driver,
-            creationOptions=format_options,
-        ))
-

preprocessor/preprocessor/steps/output.py

+import os
+from os.path import join, basename
+from glob import glob
+from uuid import uuid4
+
+from osgeo import gdal
+
+from .util import replace_ext
+
+
+def output_step(source_dir: os.PathLike, target_dir: os.PathLike, options: dict=None):
+    # find out the driver to get the extension
+    options = options if options is not None else {}
+    frmt = options.get('format', 'GTiff')
+    driver = gdal.GetDriverByName(frmt)
+    if not driver:
+        raise ValueError('Unsupported driver %s' % frmt)
+    extension = driver.GetMetadata()['DMD_EXTENSIONS'].split(' ')[0]
+
+    # warp each individual file
+    warped_files = []
+    for filename in glob(join(source_dir, '*')):
+        target_filename = join(target_dir, replace_ext(basename(filename), extension))
+        gdal.Warp(target_filename, filename, options=gdal.WarpOptions(
+            **options
+        ))
+        warped_files.append(target_filename)
+
+    if len(warped_files) > 1:
+        tmp_filename = join(target_dir, '%s.%s' % (uuid4().hex, extension))
+        gdal.Warp(tmp_filename, warped_files, options=gdal.WarpOptions(
+            **options
+        ))
+
+        # delete old files and rename the combined file to the first filename
+        for filename in warped_files:
+            os.unlink(filename)
+        os.rename(tmp_filename, warped_files[0])

preprocessor/preprocessor/stack.py → preprocessor/preprocessor/steps/stack.py

@@ -6,12 +6,10 @@ from glob import glob
 from typing import List
 from osgeo import gdal
 
+from ..util import replace_ext
 
-def replace_ext(filename: os.PathLike, new_ext: str) -> os.PathLike:
-    return splitext(filename)[0] + ('' if new_ext.startswith('.') else '.') + new_ext
 
-
-def stack_bands_step(source_dir, target_dir, group_by: str=None, sort_by: str=None, order: List[str]=None):
+def stack_bands_step(source_dir: os.PathLike, target_dir: os.PathLike, group_by: str=None, sort_by: str=None, order: List[str]=None):
     """ Stack bands of the individual images
     """
     filenames = glob(join(source_dir, '*/*.tif'), recursive=True)
@@ -46,10 +44,5 @@ def stack_bands_step(source_dir, target_dir, group_by: str=None, sort_by: str=No
                 )
 
         # build a VRT to stack bands for each group
-        vrt_filename = replace_ext(join(source_dir, groupname), '.vrt')
+        vrt_filename = replace_ext(join(target_dir, groupname), '.vrt')
         gdal.BuildVRT(vrt_filename, group, separate=True)
-
-        # translate to TIFF
-        # TODO: necessary? maybe just keep a VRT instead
-        out_filename = replace_ext(join(target_dir, groupname), '.tif')
-        gdal.Translate(out_filename, vrt_filename, format='GTiff')

preprocessor/preprocessor/subdataset.py → preprocessor/preprocessor/steps/subdataset.py

@@ -2,11 +2,10 @@ import os
 from os.path import join, splitext, basename
 from glob import glob
 from typing import Set
-from osgeo import gdal
 
+from osgeo import gdal
 
-def replace_ext(filename: os.PathLike, new_ext: str) -> os.PathLike:
-    return splitext(filename)[0] + '' if new_ext.startswith('.') else '.' + new_ext
+from ..util import replace_ext
 
 
 def extract_subdataset_step(source_dir: os.PathLike, target_dir: os.PathLike, data_file_glob: str, subdataset_types: Set[str]=None):

preprocessor/preprocessor/transfer/__init__.py

+from .util import get_downloader, get_uploader
\ No newline at end of file

preprocessor/preprocessor/util.py

+import os
+from os.path import splitext
+
+
+def replace_ext(filename: os.PathLike, new_ext: str) -> os.PathLike:
+    return splitext(filename)[0] + ('' if new_ext.startswith('.') else '.') + new_ext