Better splitting app/parsing. Adding pushing to redis queue

ingestor/app.py

@@ -29,13 +29,16 @@
 import os
 import logging
 import logging.config
+import json
+from datetime import datetime
 
-from flask import Flask, request, Response, jsonify
+from flask import Flask, request, Response
+import redis
 
 from ingestor.browse_report import parse_browse_report
 
 
-app = Flask(__name__, static_url_path='')
+application = Flask(__name__)
 
 logger = logging.getLogger(__name__)
 
@@ -66,9 +69,29 @@ logging.config.dictConfig({
 })
 
 
-@app.route('/')
+def converter(o):
+    if isinstance(o, datetime):
+        return o.isoformat()
+
+client = redis.Redis(
+    host=os.environ['REDIS_HOST'],
+    port=int(os.environ.get('REDIS_PORT', '6379')),
+    charset="utf-8",
+    decode_responses=True,
+)
+
+queue_name = os.environ['REDIS_QUEUE_KEY']
+
+@application.route('/', methods=['POST'])
 def ingest():
     try:
+        request.get_data()
+        browse_report = parse_browse_report(request.data)
+        logger.debug(browse_report)
+        client.lpush(queue_name, json.dumps(
+            browse_report, default=converter
+        ))
         return Response(status=202)
-    except:
-        return Response(status=400)
+
+    except Exception as e:
+        return Response(str(e), status=400)

ingestor/ingestor/browse_report.py

@@ -25,6 +25,8 @@
 # IN THE SOFTWARE.
 #-----------------------------------------------------------------------------
 
+import io
+
 from lxml import etree
 import dateutil.parser
 
@@ -62,6 +64,9 @@ def parse_browse_report(input_file):
     """
         :returns: list of browses
     """
+    if isinstance(input_file, bytes):
+        input_file = io.BytesIO(input_file)
+
     try:
         tree = etree.parse(input_file)
     except etree.XMLSyntaxError as e:

ingestor/ingestor/test_browse_report.py

+#------------------------------------------------------------------------------
+#
+# Project: prism view server
+# Authors: Fabian Schindler <fabian.schindler@eox.at>
+#
+#------------------------------------------------------------------------------
+# Copyright (C) 2020 EOX IT Services GmbH <https://eox.at>
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to
+# deal in the Software without restriction, including without limitation the
+# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+# sell copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies of this Software or works derived from this Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+# IN THE SOFTWARE.
+#-----------------------------------------------------------------------------
+
+
 from os.path import dirname, join
 from datetime import datetime
 from dateutil import tz
 
-
-
 from ingestor.browse_report import parse_browse_report
 
 

ingestor/ingestor/util.py

+#------------------------------------------------------------------------------
+#
+# Project: prism view server
+# Authors: Fabian Schindler <fabian.schindler@eox.at>
+#
+#------------------------------------------------------------------------------
+# Copyright (C) 2020 EOX IT Services GmbH <https://eox.at>
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to
+# deal in the Software without restriction, including without limitation the
+# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+# sell copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies of this Software or works derived from this Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+# IN THE SOFTWARE.
+#-----------------------------------------------------------------------------
+
+from datetime import datetime
+
+
 def pairwise(iterable):
     "s -> (s0,s1), (s2,s3), (s4, s5), ..."
     a = iter(iterable)
     return zip(a, a)
+
+
+def converter(o):
+    if isinstance(o, datetime):
+        return o.isoformat()