From 7ff11021c53c2cc4589de137746e6d9f0c0f4592 Mon Sep 17 00:00:00 2001
From: Fabian Schindler <fabian.schindler.strauss@gmail.com>
Date: Wed, 26 Aug 2020 14:09:56 +0200
Subject: [PATCH] Fixes for prepocessing daemon
---
preprocessor/preprocessor/cli.py | 15 +++++++++++++--
preprocessor/preprocessor/preprocess.py | 10 ++++++----
2 files changed, 19 insertions(+), 6 deletions(-)
diff --git a/preprocessor/preprocessor/cli.py b/preprocessor/preprocessor/cli.py
index db8896fd..5fa8e005 100644
--- a/preprocessor/preprocessor/cli.py
+++ b/preprocessor/preprocessor/cli.py
@@ -6,6 +6,7 @@ import yaml
import jsonschema
from .preprocess import preprocess_file
+from .daemon import run_daemon
def setup_logging(debug=False):
@@ -44,8 +45,18 @@ def cli():
@cli.command(help='Run the preprocess daemon, attaching to a Redis queue')
-def daemon(ctx):
- click.echo(ctx['CONFIG'])
+@click.option('--config-file', type=click.File('r'))
+@click.option('--use-dir', type=str) # TODO: check dir
+@click.option('--validate/--no-validate', default=False)
+@click.option('--host', type=str)
+@click.option('--port', type=int)
+@click.option('--listen-queue', type=str)
+@click.option('--write-queue', type=str)
+def daemon(config_file=None, use_dir=None, validate=False, host=None, port=None, listen_queue=None, write_queue=None):
+ config = yaml.load(config_file)
+ if validate:
+ validate_config(config)
+ run_daemon(config, host, port, listen_queue, write_queue)
@cli.command(help='Run a single, one-off preprocessing')
diff --git a/preprocessor/preprocessor/preprocess.py b/preprocessor/preprocessor/preprocess.py
index 2c167570..f7f3cad2 100644
--- a/preprocessor/preprocessor/preprocess.py
+++ b/preprocessor/preprocessor/preprocess.py
@@ -246,7 +246,9 @@ def preprocess_file(config: dict, file_path: os.PathLike, use_dir: os.PathLike=N
% (upload_timer.elapsed)
)
- logger.info(
- 'Finished preprocessing of %s after %f seconds.'
- % (file_path, preprocess_timer.elapsed)
- )
+ logger.info(
+ 'Finished preprocessing of %s after %f seconds.'
+ % (file_path, preprocess_timer.elapsed)
+ )
+
+ return upload_filenames
--
GitLab