do not put item to failure-set when skip due to already present

core/registrar/daemon.py

@@ -27,8 +27,9 @@ def run_daemon(config, replace, host, port, listen_queue, progress_set, failure_
             item = register_file(config, value, replace)
             client.sadd(success_set, item.identifier)
             client.srem(progress_set, value)
-
         except Exception as e:
+            if 'is already registered' not in "%s" % e:
+                # do not add to failure if skipped due to already registered
+                client.sadd(failure_set, value)
             client.srem(progress_set, value)
-            client.sadd(failure_set, value)
             logger.exception(e)

preprocessor/preprocessor/daemon.py

@@ -3,7 +3,7 @@ import logging
 import json
 
 from .preprocess import preprocess_file, preprocess_browse
-
+from .exceptions import ExistsAtUploadError
 
 logger = logging.getLogger(__name__)
 
@@ -30,6 +30,11 @@ def run_daemon(config, host, port, listen_queue, listen_md_queue, write_queue, p
                 file_paths.append(file_path)
                 client.sadd(success_set, value)
                 client.srem(progress_set, value)
+            except ExistsAtUploadError as e:
+                # do not add file already present to failure set
+                client.srem(progress_set, value)
+                logger.exception(e)
+                continue
             except Exception as e:
                 client.sadd(failure_set, value)
                 client.srem(progress_set, value)
@@ -43,6 +48,10 @@ def run_daemon(config, host, port, listen_queue, listen_md_queue, write_queue, p
                     filename, file_path = preprocess_browse(config, browse_type, browse_report_data, browse)
                     file_paths.append(file_path)
                     client.sadd(success_set, browse['filename'])
+                except ExistsAtUploadError as e:
+                    # do not add file already present to failure set
+                    logger.exception(e)
+                    continue
                 except Exception as e:
                     client.sadd(failure_set, browse['filename'])
                     logger.exception(e)

preprocessor/preprocessor/exceptions.py

+class ExistsAtUploadError(Exception):
+    pass

preprocessor/preprocessor/preprocess.py

@@ -16,6 +16,7 @@ from .steps import (
 )
 from .steps.browse_report import browse_georeference
 from .util import workdir, Timer, get_size_in_bytes
+from .exceptions import ExistsAtUploadError
 
 logging.basicConfig()
 
@@ -131,7 +132,7 @@ def preprocess_file(config: dict, file_path: os.PathLike, use_dir: os.PathLike=N
                 target_config['type'], target_config.get('args'), target_config.get('kwargs')
             )
             if uploader.product_exists(file_path):
-                raise Exception('Target.replace configuration is not set to true and objects already exist in target %s.' % file_path)
+                raise ExistsAtUploadError('Target.replace configuration is not set to true and objects already exist in target %s.' % file_path)
             else:
                 logger.debug('Product does not yet exist on target')
         # check if we can reuse a previous download