import logging import json import redis from .registrar import register_file logger = logging.getLogger(__name__) def run_daemon(config, replace, host, port, listen_queue, progress_set, failure_set, success_set): """ Run the registrar daemon, listening on a redis queue for files to be registered. """ # initialize the queue client client = redis.Redis( host=host, port=port, charset="utf-8", decode_responses=True ) logger.debug("waiting for redis queue '%s'..." % listen_queue) while True: # fetch an item from the queue to be registered _, value = client.brpop(listen_queue) client.sadd(progress_set, value) # start the registration on that file try: item = register_file(config, value, replace) client.sadd(success_set, item.identifier) client.srem(progress_set, value) except Exception as e: if 'is already registered' not in "%s" % e: # do not add to failure if skipped due to already registered client.sadd(failure_set, value) client.srem(progress_set, value) logger.exception(e)