#!/usr/bin/env python #------------------------------------------------------------------------------ # # Project: prism view server # Authors: Fabian Schindler <fabian.schindler@eox.at> # #------------------------------------------------------------------------------ # Copyright (C) 2020 EOX IT Services GmbH <https://eox.at> # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to # deal in the Software without restriction, including without limitation the # rights to use, copy, modify, merge, publish, distribute, sublicense, and/or # sell copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be included in # all copies of this Software or works derived from this Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. #----------------------------------------------------------------------------- import os import logging import logging.config import json import pyinotify import redis from ingestor.browse_report import parse_browse_report from ingestor.util import converter, save_mount_report logger = logging.getLogger(__name__) logging.config.dictConfig({ 'version': 1, 'disable_existing_loggers': False, 'formatters': { 'simple': { 'format': '%(levelname)s: %(message)s', }, 'verbose': { 'format': '[%(asctime)s][%(module)s] %(levelname)s: %(message)s', } }, 'handlers': { 'console': { 'level': 'DEBUG', 'class': 'logging.StreamHandler', 'formatter': 'verbose', } }, 'loggers': { '': { 'handlers': ['console'], 'level': 'DEBUG', 'propagate': False, } } }) queue_name = os.environ['REDIS_PREPROCESS_MD_QUEUE_KEY'] watch_dir = os.environ['INOTIFY_WATCH_DIR'] client = redis.Redis( host=os.environ['REDIS_HOST'], port=int(os.environ.get('REDIS_PORT', '6379')), charset='utf-8', decode_responses=True, ) watchmanager = pyinotify.WatchManager() class EventHandler(pyinotify.ProcessEvent): def process_IN_CLOSE_WRITE(self, event): logger.info(f'Parsing browse file: {event.pathname}') with open(event.pathname) as f: browse_report = parse_browse_report(f) logger.debug(browse_report) if isinstance(browse_report, str): queue_content = browse_report else: queue_content = json.dumps( browse_report, default=converter ) try: client.lpush(queue_name, queue_content) save_mount_report(event.pathname, True) except Exception as e: save_mount_report(event.pathname, False) logger.exception(e) handler = EventHandler() notifier = pyinotify.Notifier(watchmanager, handler) wdd = watchmanager.add_watch(watch_dir, pyinotify.IN_CLOSE_WRITE, rec=True) notifier.loop()