registrar get up to date with DEM updates in vhr_image_2018 reg

core/registrar.py

@@ -36,6 +36,7 @@ import textwrap
 import logging
 import traceback
 import redis
+import subprocess
 
 import lxml.etree
 from swiftclient.service import SwiftService
@@ -43,6 +44,7 @@ from swiftclient.service import SwiftService
 import django
 from django.db import transaction
 from django.contrib.gis.geos import GEOSGeometry
+from osgeo import gdal
 
 path = os.path.join(os.getenv('INSTALL_DIR', "/var/www/pvs"), "pvs_instance")
 if path not in sys.path:
@@ -52,6 +54,7 @@ os.environ.setdefault("DJANGO_SETTINGS_MODULE", "pvs_instance.settings")
 django.setup()
 
 from eoxserver.backends import access
+from eoxserver.contrib import vsi
 from eoxserver.backends import models as backends
 from eoxserver.resources.coverages import models
 from eoxserver.resources.coverages.registration.product import (
@@ -87,6 +90,16 @@ def setup_logging(verbosity):
     # finished logging setup
 
 
+def set_gdal_swift_auth():
+    # parsing command line output of swift auth
+    auth_keys = subprocess.check_output(["swift", "auth"]).split("\n")
+    storage_url = auth_keys[0].split("OS_STORAGE_URL=")[1]
+    auth_token = auth_keys[1].split("OS_AUTH_TOKEN=")[1]
+    # setting gdal config
+    gdal.SetConfigOption("SWIFT_STORAGE_URL", storage_url)
+    gdal.SetConfigOption("SWIFT_AUTH_TOKEN", auth_token)
+
+
 def add_mask(product):
     metadata_item = product.metadata_items.all()[0]
     with access.vsi_open(metadata_item) as f:
@@ -141,13 +154,49 @@ def get_product_type_and_level(metadata_item):
     return product_type_name, level
 
 
+def get_product_collection(metadata_file):
+    # in case collection needs to be determined from metadata
+    try:
+        if metadata_file.startswith("/vsiswift"):
+            set_gdal_swift_auth()
+        with vsi.open(metadata_file, "r") as f:
+            tree = lxml.etree.parse(f)
+        root = tree.getroot()
+        xp = '//gml:metaDataProperty/gsc:EarthObservationMetaData/eop:parentIdentifier/text()'
+        product_type_name = tree.xpath(xp, namespaces=root.nsmap)
+        extracted = product_type_name[0].split('/')[0]
+        return extracted
+    except Exception as e:
+        logger.debug(
+            'Failed to determine product collection for metadata file %s, error was %s'
+            % (metadata_file, e)
+        )
+
+
+def get_product_type_from_band_count(product_type_name, file_path):
+    # get raster band count via gdal
+    logger.debug("Opening file using GDAL: %s" % file_path)
+    if file_path.startswith("/vsiswift"):
+        set_gdal_swift_auth()
+    src_ds = gdal.Open(file_path)
+    if src_ds is None:
+        raise RegistrationError("Band check: failed to open dataset: %s " % file_path)
+    # try to fetch product model with _bandcount
+    product_type_name_upd = "%s_%s" % (product_type_name, src_ds.RasterCount)
+    try:
+        product_type_model = models.ProductType.objects.get(name=product_type_name_upd)
+        return product_type_model
+    except models.ProductType.DoesNotExist:
+        raise RegistrationError("Product Type: '%s' was not found" % product_type_name_upd)
+
+
 class RegistrationError(Exception):
     pass
 
 
 @transaction.atomic
 def registrar(
-    collection,
+    collection_stack,
     objects_prefix, upload_container=None, replace=False, client=None, registered_set_key=None
 ):
     logger.info("Starting registration of product '%s'." % objects_prefix)
@@ -187,38 +236,72 @@ def registrar(
             "Product with objects prefix '%s' has missing content."
             % objects_prefix
         )
+    logger.debug("Found objects '%s' and '%s'." % (data_package, metadata_package))
 
     storage = backends.Storage.objects.get(name=upload_container)
     metadata_item = models.MetaDataItem(storage=storage, location=metadata_package)
 
     product_type, level = get_product_type_and_level(metadata_item)
+    if collection_stack == 'DEM':
+        # special for DEM files, collection name === product_type
+        gdal_metadata_file_path = "/vsiswift/%s/%s" % (upload_container, metadata_package)
+        product_type = get_product_collection(gdal_metadata_file_path)
     logger.debug("Registering product")
+    product_type_name = "%s_Product_%s" % (collection_stack, product_type)
+
+    try:
+        # first find product type by name from path
+        product_type_model = models.ProductType.objects.get(name=product_type_name)
+    except models.ProductType.DoesNotExist:
+        # if not found, maybe there are more product types with _bandcount suffix
+        gdal_file_path = "/vsiswift/%s/%s" % (upload_container, data_package)
+        product_type_model = get_product_type_from_band_count(product_type_name, gdal_file_path)
+        product_type_name = product_type_model.name
+    coverage_type_names = product_type_model.allowed_coverage_types.all()
+    if len(coverage_type_names) > 1:
+        logger.warning("More available 'CoverageType' found, selecting the first one.")
+    coverage_type_name = coverage_type_names[0].name
+
     product, replaced = ProductRegistrator().register(
         metadata_locations=[[upload_container,
                              metadata_package, ], ],
-        type_name="%s_Product_%s" % (collection, product_type),
+        type_name=product_type_name,
         replace=replace,
         extended_metadata=True,
         mask_locations=None,
         package_path=None,
+        simplify_footprint_tolerance=0.0001,  # ~10meters
         overrides={},
     )
+    if product.footprint.empty:
+        product.delete()
+        raise RegistrationError("No footprint was extracted. full product: %s" % product)
 
     collection = models.Collection.objects.get(
-        identifier=collection
+        identifier=collection_stack
     )
-    logger.debug("Inserting product into collection")
+    logger.debug("Inserting product into collection %s" % collection_stack)
     models.collection_insert_eo_object(collection, product)
 
+    if collection_stack == "DEM":
+        # also insert it to its own collection
+        collection_own = models.Collection.objects.get(
+            identifier="%s_%s" % (collection, product_type)
+        )
+        logger.debug("Inserting product to collection %s_%s" % (collection, product_type))
+        models.collection_insert_eo_object(collection_own, product)
+
     if level == 'Level_1':
         collection_level_1 = models.Collection.objects.get(
             identifier="%s_Level_1" % collection
         )
+        logger.debug("Inserting product to collection %s_Level_1" % collection)
         models.collection_insert_eo_object(collection_level_1, product)
     elif level == 'Level_3':
         collection_level_3 = models.Collection.objects.get(
             identifier="%s_Level_3" % collection
         )
+        logger.debug("Inserting product to collection %s_Level_3" % collection)
         models.collection_insert_eo_object(collection_level_3, product)
 
     logger.debug("Registering coverage")
@@ -226,7 +309,7 @@ def registrar(
         data_locations=[[upload_container, data_package, ], ],
         metadata_locations=[[upload_container,
                              metadata_package, ], ],
-        coverage_type_name="RGBNir",
+        coverage_type_name=coverage_type_name,
         overrides={
             "identifier": "%s__coverage" % product.identifier,
             "footprint": None,