diff --git a/preprocessor/preprocessor/preprocess.py b/preprocessor/preprocessor/preprocess.py index 7f9eab88f986ce2043fa745bc6f33548cd647ba1..17a0490844d27870b86874e5d53ae04f8508ed7b 100644 --- a/preprocessor/preprocessor/preprocess.py +++ b/preprocessor/preprocessor/preprocess.py @@ -223,12 +223,14 @@ def preprocess_file(config: dict, file_path: os.PathLike, use_dir: os.PathLike=N uploader = get_uploader( target_config['type'], target_config.get('args'), target_config.get('kwargs') ) - - upload_filenames = [ - os.path.join(dirpath, filename) - for dirpath, _, filenames in os.walk('upload') - for filename in filenames - ] + paths_for_upload = ['upload', 'extra'] + upload_filenames = [] + for path_to_upload in paths_for_upload: + upload_filenames.extend([ + os.path.join(dirpath, filename) + for dirpath, _, filenames in os.walk(path_to_upload) + for filename in filenames + ]) # send all files in the upload directory to the target storage logger.info( @@ -285,10 +287,11 @@ def preprocess_browse(config: dict, browse_type: str, browse_report: dict, brows # TODO: check if allowed and download from there raise NotImplementedError - os.mkdir('unpack') + if not os.path.isdir('unpack'): + os.mkdir('unpack') logger.info('Applying browse georeference to browse %s' % filename) - browse_georeference('download', 'unpack', browse_type, browse) + browse_georeference('download', 'unpack', browse_report, browse) # fetch the product type from the browse_type