emg_preprocessor-config.yml

source:
  type: swift
  kwargs:
    username: !env '${OS_USERNAME_DOWNLOAD}'
    password: !env '${OS_PASSWORD_DOWNLOAD}'
    tenant_name: !env '${OS_TENANT_NAME_DOWNLOAD}'
    tenant_id: !env '${OS_TENANT_ID_DOWNLOAD}'
    region_name: !env '${OS_REGION_NAME_DOWNLOAD}'
    auth_url: !env '${OS_AUTH_URL_DOWNLOAD}'
    auth_version: !env '${ST_AUTH_VERSION_DOWNLOAD}'
    user_domain_name: !env '${OS_USER_DOMAIN_NAME_DOWNLOAD}'
target:
  type: swift
  replace: false
  kwargs:
    username: !env '${OS_USERNAME}'
    password: !env '${OS_PASSWORD}'
    tenant_name: !env '${OS_TENANT_NAME}'
    tenant_id: !env '${OS_TENANT_ID}'
    region_name: !env '${OS_REGION_NAME}'
    auth_version: !env '${ST_AUTH_VERSION}'
    auth_url: !env '${OS_AUTH_URL}'
    user_domain_name: !env '${OS_USER_DOMAIN_NAME}'
    container: !env '${UPLOAD_CONTAINER}'
workdir: /tmp
keep_temp: false
metadata_glob: "*GSC*.xml"
type_extractor:
  xpath:
    - /gsc:report/gsc:opt_metadata/gml:using/eop:EarthObservationEquipment/eop:platform/eop:Platform/eop:shortName/text()
    - /gsc:report/gsc:sar_metadata/gml:using/eop:EarthObservationEquipment/eop:platform/eop:Platform/eop:shortName/text()
level_extractor:
  # xpath can also be a list of xpaths to be tried one after another
  xpath: substring-after(substring-after(/gsc:report/gsc:opt_metadata/gml:metaDataProperty/gsc:EarthObservationMetaData/eop:parentIdentifier/text(), '/'), '/')
preprocessing:
  defaults:
    move_files: true
    nested: true
    data_file_globs:
      - '*.tif'
      - '*.jp2'
      - '*.h5'
    output:
      options:
        format: COG
        dstSRS: 'EPSG:4326'
        dstNodata: 0
        multithread: True
        warpMemoryLimit: 3000
        creationOptions:
          - BLOCKSIZE=512
          - COMPRESS=DEFLATE
          - NUM_THREADS=8
          - BIGTIFF=YES
          - OVERVIEWS=AUTO
  types:
    KS03:
      data_file_globs:
        - "*.tif"
      additional_file_globs:
        - "*.rpc*"
      stack_bands:
        group_by: ".*/(.*)_P..tif"
        sort_by: ".*_P(R|G|B|N).tif"
        order:
          - R
          - G
          - B
          - N
    RS02:
      data_file_globs:
        - "*imagery_*.tif"
      georeference:
        geotransforms:
          - type: gcp
          - type: no_op
      calc:
        formulas:
          # complex b1/b2 to b1 uint16 for all 4 polarizations
          - inputs:
              A:
                glob: '*HH.tif'
                band: 1
              B:
                glob: '*HH.tif'
                band: 2
            formula: sqrt(A.astype(float)*A.astype(float)+B.astype(float)*B.astype(float))
            output_postfix: _proc
            nodata_value: 0
            data_type: UInt16
          - inputs:
              A:
                glob: '*HV.tif'
                band: 1
              B:
                glob: '*HV.tif'
                band: 2
            formula: sqrt(A.astype(float)*A.astype(float)+B.astype(float)*B.astype(float))
            output_postfix: _proc
            nodata_value: 0
            data_type: UInt16
          - inputs:
              A:
                glob: '*VH.tif'
                band: 1
              B:
                glob: '*VH.tif'
                band: 2
            formula: sqrt(A.astype(float)*A.astype(float)+B.astype(float)*B.astype(float))
            output_postfix: _proc
            nodata_value: 0
            data_type: UInt16
          - inputs:
              A:
                glob: '*VV.tif'
                band: 1
              B:
                glob: '*VV.tif'
                band: 2
            formula: sqrt(A.astype(float)*A.astype(float)+B.astype(float)*B.astype(float))
            output_postfix: _proc
            nodata_value: 0
            data_type: UInt16
      stack_bands:
        sort_by: ".*_(HH|HV|VH|VV)_proc.tif"
        order:
          - HH
          - HV
          - VH
          - VV
    SK00:
      data_file_globs:
        - "*pansharpened_clip.tif"
        - "*_pansharpened.tif"
    SP06:
      data_file_globs:
        # throw away Panchromatic *_P_*
        - "*IMG_*_PMS_*.JP2"
        - "*IMG_*_PMS_*.tif"
        - "*IMG_*_MS_*.JP2"
        - "*IMG_*_MS_*.tif"
      additional_file_globs:
        - "*RPC_*"
        - "*DIM_*"
      georeference:
        geotransforms:
          - type: rpc
          - type: no_op
    SP07:
      data_file_globs:
        - "*IMG_*_PMS_*.JP2"
        - "*IMG_*_PMS_*.tif"
        - "*IMG_*_MS_*.JP2"
        - "*IMG_*_MS_*.tif"
      additional_file_globs:
        - "*RPC_*"
        - "*DIM_*"
      georeference:
        geotransforms:
          # first try RPC, if not present, go on
          - type: rpc
          - type: no_op
    PH1A:
      data_file_globs:
        - "*IMG_*_PMS_*.JP2"
        - "*IMG_*_PMS_*.tif"
        - "*IMG_*_MS_*.JP2"
        - "*IMG_*_MS_*.tif"
      additional_file_globs:
        - "*RPC_*"
        - "*DIM_*"
      georeference:
        geotransforms:
          - type: rpc
          - type: no_op
    PH1B:
      data_file_globs:
        - "*IMG_*_PMS_*.JP2"
        - "*IMG_*_PMS_*.tif"
        - "*IMG_*_MS_*.JP2"
        - "*IMG_*_MS_*.tif"
      additional_file_globs:
        - "*RPC_*"
        - "*DIM_*"
      georeference:
        geotransforms:
          - type: rpc
          - type: no_op
    PL00:
      data_file_globs:
        - "*MS.tif"
        - "*MS_clip.tif"
    SW03:
      data_file_globs:
        - "*.tif"
    GY01:
      # throw away Panchromatic *-P3D*
      data_file_globs:
        - "*-M3D*.TIF"
        - "*-M3D*.tif"
        - "*-S3D*.TIF"
        - "*-S3D*.tif"
        - "*-M2A*.TIF"
        - "*-M2A*.tif"
        - "*-S2A*.TIF"
        - "*-S2A*.tif"
    EW03:
      data_file_globs:
        - "*-M3D*.TIF"
        - "*-M3D*.tif"
        - "*-S3D*.TIF"
        - "*-S3D*.tif"
        - "*-M2A*.TIF"
        - "*-M2A*.tif"
        - "*-S2A*.TIF"
        - "*-S2A*.tif"
    EW02:
      data_file_globs:
        - "*-M3D*.TIF"
        - "*-M3D*.tif"
        - "*-S3D*.TIF"
        - "*-S3D*.tif"
        - "*-M2A*.TIF"
        - "*-M2A*.tif"
        - "*-S2A*.TIF"
        - "*-S2A*.tif"
    EW01:
      data_file_globs:
        - "*.TIF"
        - "*.tif"
    DM02:
      data_file_globs:
        - "*.tif"
# this configuration is still a stub - not all product types are done 
# https://gitlab.eox.at/esa/prism/vs/-/issues/56
# https://gitlab.eox.at/esa/prism/vs/-/issues/23