diff --git a/ingestor/filedaemon.py b/ingestor/filedaemon.py index 8c066c4ccb3c34ea9ddb4a77bc446d070d3edad9..0d077ecbde7384e393b67a6503012340fa758b52 100644 --- a/ingestor/filedaemon.py +++ b/ingestor/filedaemon.py @@ -1,10 +1,10 @@ #!/usr/bin/env python -#------------------------------------------------------------------------------ +# ------------------------------------------------------------------------------ # # Project: prism view server # Authors: Fabian Schindler <fabian.schindler@eox.at> # -#------------------------------------------------------------------------------ +# ------------------------------------------------------------------------------ # Copyright (C) 2020 EOX IT Services GmbH <https://eox.at> # # Permission is hereby granted, free of charge, to any person obtaining a copy @@ -24,7 +24,7 @@ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. -#----------------------------------------------------------------------------- +# ----------------------------------------------------------------------------- import os import logging @@ -80,11 +80,12 @@ client = redis.Redis( watchmanager = pyinotify.WatchManager() + class EventHandler(pyinotify.ProcessEvent): def process_IN_CLOSE_WRITE(self, event): logger.info(f'Parsing browse file: {event.pathname}') - try: + try: with open(event.pathname) as f: browse_report = parse_browse_report(f) logger.debug(browse_report) @@ -95,15 +96,16 @@ class EventHandler(pyinotify.ProcessEvent): queue_content = json.dumps( browse_report, default=converter ) - + client.lpush(queue_name, queue_content) - + save_mount_report(event.pathname, True) except Exception as e: - save_mount_report(event.pathname, False) + save_mount_report(event.pathname, False) logger.exception(e) + handler = EventHandler() notifier = pyinotify.Notifier(watchmanager, handler) diff --git a/ingestor/ingestor/util.py b/ingestor/ingestor/util.py index 43c5f1b81a1321641cd2eaf84fdb926500d56cb4..bb92a6befdd86cd8ea90657e51bab0fec78ba1f9 100644 --- a/ingestor/ingestor/util.py +++ b/ingestor/ingestor/util.py @@ -1,9 +1,9 @@ -#------------------------------------------------------------------------------ +# ------------------------------------------------------------------------------ # # Project: prism view server # Authors: Fabian Schindler <fabian.schindler@eox.at> # -#------------------------------------------------------------------------------ +# ------------------------------------------------------------------------------ # Copyright (C) 2020 EOX IT Services GmbH <https://eox.at> # # Permission is hereby granted, free of charge, to any person obtaining a copy @@ -23,9 +23,8 @@ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. -#----------------------------------------------------------------------------- +# ----------------------------------------------------------------------------- -import json import os import shutil from datetime import datetime @@ -41,20 +40,26 @@ def converter(o): if isinstance(o, datetime): return o.isoformat() + def save_mount_report(data, success: bool): if success: save_dir = os.environ['INGESTOR_SUCCESS_DIR'] - else: + else: save_dir = os.environ['INGESTOR_FAIL_DIR'] - shutil.move(data, save_dir) + + shutil.copy(data, save_dir) + os.remove(data) + def save_endpoint_report(filename: str, data, success: bool): if success: save_dir = os.environ['INGESTOR_SUCCESS_DIR'] - else: + else: save_dir = os.environ['INGESTOR_FAIL_DIR'] - with open(os.path.join(save_dir, '%s.xml' % filename), "w") as outfile: - outfile.write(data) + + with open(os.path.join(save_dir, '%s.xml' % filename), "w") as outfile: + outfile.write(data) + def browse_name(report): return '_'.join(browse["browse_identifier"] for browse in report["browses"])