Merge branch 'staging' into shib-configs-update

ingestor/filedaemon.py

 #!/usr/bin/env python
-#------------------------------------------------------------------------------
+# ------------------------------------------------------------------------------
 #
 # Project: prism view server
 # Authors: Fabian Schindler <fabian.schindler@eox.at>
 #
-#------------------------------------------------------------------------------
+# ------------------------------------------------------------------------------
 # Copyright (C) 2020 EOX IT Services GmbH <https://eox.at>
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
@@ -24,7 +24,7 @@
 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 # IN THE SOFTWARE.
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 import os
 import logging
@@ -80,11 +80,12 @@ client = redis.Redis(
 
 watchmanager = pyinotify.WatchManager()
 
+
 class EventHandler(pyinotify.ProcessEvent):
     def process_IN_CLOSE_WRITE(self, event):
         logger.info(f'Parsing browse file: {event.pathname}')
 
-        try: 
+        try:
             with open(event.pathname) as f:
                 browse_report = parse_browse_report(f)
             logger.debug(browse_report)
@@ -95,15 +96,16 @@ class EventHandler(pyinotify.ProcessEvent):
                 queue_content = json.dumps(
                     browse_report, default=converter
                 )
-         
+
             client.lpush(queue_name, queue_content)
-            
+
             save_mount_report(event.pathname, True)
         except Exception as e:
 
-            save_mount_report(event.pathname, False) 
+            save_mount_report(event.pathname, False)
             logger.exception(e)
 
+
 handler = EventHandler()
 notifier = pyinotify.Notifier(watchmanager, handler)
 

ingestor/ingestor/util.py

-#------------------------------------------------------------------------------
+# ------------------------------------------------------------------------------
 #
 # Project: prism view server
 # Authors: Fabian Schindler <fabian.schindler@eox.at>
 #
-#------------------------------------------------------------------------------
+# ------------------------------------------------------------------------------
 # Copyright (C) 2020 EOX IT Services GmbH <https://eox.at>
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
@@ -23,9 +23,8 @@
 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 # IN THE SOFTWARE.
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
-import json
 import os
 import shutil
 from datetime import datetime
@@ -41,20 +40,26 @@ def converter(o):
     if isinstance(o, datetime):
         return o.isoformat()
 
+
 def save_mount_report(data, success: bool):
     if success:
         save_dir = os.environ['INGESTOR_SUCCESS_DIR']
-    else: 
+    else:
         save_dir = os.environ['INGESTOR_FAIL_DIR']
-    shutil.move(data, save_dir)
+
+    shutil.copy(data, save_dir)
+    os.remove(data)
+
 
 def save_endpoint_report(filename: str, data, success: bool):
     if success:
         save_dir = os.environ['INGESTOR_SUCCESS_DIR']
-    else: 
+    else:
         save_dir = os.environ['INGESTOR_FAIL_DIR']
-    with open(os.path.join(save_dir, '%s.xml' % filename), "w") as outfile: 
-        outfile.write(data) 
+
+    with open(os.path.join(save_dir, '%s.xml' % filename), "w") as outfile:
+        outfile.write(data)
+
 
 def browse_name(report):
     return '_'.join(browse["browse_identifier"] for browse in report["browses"])