From 20bb7a270dbdc8c4c090339a6a37c3ae4d600024 Mon Sep 17 00:00:00 2001 From: Lubomir Bucek Date: Tue, 24 Nov 2020 15:34:40 +0100 Subject: [PATCH 1/9] redo emg type_bands prod types, now more cov types per prod type --- config/emg_init-db.sh | 458 +++++------------------- config/emg_preprocessor-config.yml | 4 +- config/emg_registrar-config.yml | 124 ++----- core/rgbnir_definition.json | 133 ++++--- testing/testing_preprocessor_config.yml | 6 +- 5 files changed, 213 insertions(+), 512 deletions(-) diff --git a/config/emg_init-db.sh b/config/emg_init-db.sh index 739119a1..4fe5909e 100644 --- a/config/emg_init-db.sh +++ b/config/emg_init-db.sh @@ -93,10 +93,11 @@ if python3 manage.py id check "${COLLECTION}"; then --red-nodata 0 \ --green-nodata 0 \ --blue-nodata 0 - # EQ02_3 - python3 manage.py producttype create "${COLLECTION}"_Product_EQ02_3 --traceback \ + # EQ02 + python3 manage.py producttype create "${COLLECTION}"_Product_EQ02 --traceback \ + --coverage-type "BGRNir" \ --coverage-type "RGB" - python3 manage.py browsetype create "${COLLECTION}"_Product_EQ02_3 --traceback \ + python3 manage.py browsetype create "${COLLECTION}"_Product_EQ02 --traceback \ --red "red" \ --green "green" \ --blue "blue" \ @@ -106,7 +107,7 @@ if python3 manage.py id check "${COLLECTION}"; then --red-nodata 0 \ --green-nodata 0 \ --blue-nodata 0 - python3 manage.py browsetype create "${COLLECTION}"_Product_EQ02_3 "TRUE_COLOR" --traceback \ + python3 manage.py browsetype create "${COLLECTION}"_Product_EQ02 "TRUE_COLOR" --traceback \ --red "red" \ --green "green" \ --blue "blue" \ @@ -116,42 +117,9 @@ if python3 manage.py id check "${COLLECTION}"; then --red-nodata 0 \ --green-nodata 0 \ --blue-nodata 0 - python3 manage.py browsetype create "${COLLECTION}"_Product_EQ02_3 "FALSE_COLOR" --traceback \ - --red "green" \ - --green "red" \ - --blue "blue" \ - --red-range 0 600 \ - --green-range 0 500 \ - --blue-range 0 500 \ - --red-nodata 0 \ - --green-nodata 0 \ - --blue-nodata 0 - # EQ02_4 - python3 manage.py producttype create "${COLLECTION}"_Product_EQ02_4 --traceback \ - --coverage-type "RGBNir" - python3 manage.py browsetype create "${COLLECTION}"_Product_EQ02_4 --traceback \ - --red "blue" \ - --green "green" \ - --blue "red" \ - --red-range 0 500 \ - --green-range 0 600 \ - --blue-range 0 500 \ - --red-nodata 0 \ - --green-nodata 0 \ - --blue-nodata 0 - python3 manage.py browsetype create "${COLLECTION}"_Product_EQ02_4 "TRUE_COLOR" --traceback \ - --red "blue" \ - --green "green" \ - --blue "red" \ - --red-range 0 500 \ - --green-range 0 600 \ - --blue-range 0 500 \ - --red-nodata 0 \ - --green-nodata 0 \ - --blue-nodata 0 - python3 manage.py browsetype create "${COLLECTION}"_Product_EQ02_4 "FALSE_COLOR" --traceback \ + python3 manage.py browsetype create "${COLLECTION}"_Product_EQ02 "FALSE_COLOR" --traceback \ --red "nir" \ - --green "blue" \ + --green "red" \ --blue "green" \ --red-range 0 700 \ --green-range 0 500 \ @@ -159,8 +127,8 @@ if python3 manage.py id check "${COLLECTION}"; then --red-nodata 0 \ --green-nodata 0 \ --blue-nodata 0 - python3 manage.py browsetype create "${COLLECTION}"_Product_EQ02_4 "NDVI" --traceback \ - --grey "(nir-blue)/(nir+blue)" --grey-range -1 1 + python3 manage.py browsetype create "${COLLECTION}"_Product_EQ02 "NDVI" --traceback \ + --grey "(nir-red)/(nir+red)" --grey-range -1 1 # EW01 python3 manage.py producttype create "${COLLECTION}"_Product_EW01 --traceback \ --coverage-type "grayscale" @@ -172,238 +140,106 @@ if python3 manage.py id check "${COLLECTION}"; then --red "gray" \ --red-range 0 1500 \ --red-nodata 0 - # EW02_3 - python3 manage.py producttype create "${COLLECTION}"_Product_EW02_3 --traceback \ - --coverage-type "RGB" - python3 manage.py browsetype create "${COLLECTION}"_Product_EW02_3 --traceback \ + # EW02 + python3 manage.py producttype create "${COLLECTION}"_Product_EW02 --traceback \ + --coverage-type "BGRNir" \ + --coverage-type "RGB" \ + --coverage-type "CBGYRReNirNir2" + python3 manage.py browsetype create "${COLLECTION}"_Product_EW02 --traceback \ --red "red" \ --green "green" \ --blue "blue" \ - --red-range 0 750 \ - --green-range 0 1200 \ - --blue-range 0 800 \ + --red-range 100 700 \ + --green-range 100 600 \ + --blue-range 100 700 \ --red-nodata 0 \ --green-nodata 0 \ --blue-nodata 0 - python3 manage.py browsetype create "${COLLECTION}"_Product_EW02_3 "TRUE_COLOR" --traceback \ + python3 manage.py browsetype create "${COLLECTION}"_Product_EW02 "TRUE_COLOR" --traceback \ --red "red" \ --green "green" \ --blue "blue" \ - --red-range 0 750 \ - --green-range 0 1200 \ - --blue-range 0 800 \ - --red-nodata 0 \ - --green-nodata 0 \ - --blue-nodata 0 - python3 manage.py browsetype create "${COLLECTION}"_Product_EW02_3 "FALSE_COLOR" --traceback \ - --red "green" \ - --green "red" \ - --blue "blue" \ - --red-range 0 1200 \ - --green-range 0 750 \ - --blue-range 0 800 \ - --red-nodata 0 \ - --green-nodata 0 \ - --blue-nodata 0 - # EW02_4 - python3 manage.py producttype create "${COLLECTION}"_Product_EW02_4 --traceback \ - --coverage-type "RGBNir" - python3 manage.py browsetype create "${COLLECTION}"_Product_EW02_4 --traceback \ - --red "blue" \ - --green "green" \ - --blue "red" \ - --red-range 0 500 \ - --green-range 0 650 \ - --blue-range 0 500 \ - --red-nodata 0 \ - --green-nodata 0 \ - --blue-nodata 0 - python3 manage.py browsetype create "${COLLECTION}"_Product_EW02_4 "TRUE_COLOR" --traceback \ - --red "blue" \ - --green "green" \ - --blue "red" \ - --red-range 0 500 \ - --green-range 0 650 \ - --blue-range 0 500 \ + --red-range 100 700 \ + --green-range 100 600 \ + --blue-range 100 700 \ --red-nodata 0 \ --green-nodata 0 \ --blue-nodata 0 - python3 manage.py browsetype create "${COLLECTION}"_Product_EW02_4 "FALSE_COLOR" --traceback \ + python3 manage.py browsetype create "${COLLECTION}"_Product_EW02 "FALSE_COLOR" --traceback \ --red "nir" \ - --green "blue" \ + --green "red" \ --blue "green" \ - --red-range 0 700 \ - --green-range 0 500 \ - --blue-range 0 650 \ + --red-range 100 700 \ + --green-range 100 600 \ + --blue-range 100 700 \ --red-nodata 0 \ --green-nodata 0 \ --blue-nodata 0 - python3 manage.py browsetype create "${COLLECTION}"_Product_EW02_4 "NDVI" --traceback \ + python3 manage.py browsetype create "${COLLECTION}"_Product_EW02 "NDVI" --traceback \ --grey "(nir-blue)/(nir+blue)" --grey-range -1 1 - # EW02_8 - python3 manage.py producttype create "${COLLECTION}"_Product_EW02_8 --traceback \ + # EW03 + python3 manage.py producttype create "${COLLECTION}"_Product_EW03 --traceback \ + --coverage-type "BGRNir" \ + --coverage-type "RGB" \ --coverage-type "CBGYRReNirNir2" - python3 manage.py browsetype create "${COLLECTION}"_Product_EW02_8 --traceback \ - --red "red" \ - --green "green" \ - --blue "blue" \ - --red-range 150 750 \ - --green-range 250 800 \ - --blue-range 200 600 \ - --red-nodata 0 \ - --green-nodata 0 \ - --blue-nodata 0 - python3 manage.py browsetype create "${COLLECTION}"_Product_EW02_8 "TRUE_COLOR" --traceback \ + python3 manage.py browsetype create "${COLLECTION}"_Product_EW03 --traceback \ --red "red" \ --green "green" \ --blue "blue" \ - --red-range 150 750 \ - --green-range 250 800 \ - --blue-range 200 600 \ - --red-nodata 0 \ - --green-nodata 0 \ - --blue-nodata 0 - python3 manage.py browsetype create "${COLLECTION}"_Product_EW02_8 "FALSE_COLOR" --traceback \ - --red "nir_2" \ - --green "red" \ - --blue "green" \ - --red-range 250 1000 \ - --green-range 250 800 \ - --blue-range 200 650 \ + --red-range 100 700 \ + --green-range 100 600 \ + --blue-range 100 700 \ --red-nodata 0 \ --green-nodata 0 \ --blue-nodata 0 - python3 manage.py browsetype create "${COLLECTION}"_Product_EW02_8 "NDVI" --traceback \ - --grey "(nir_2-red)/(nir_2+red)" --grey-range -1 1 - # EW03_3 - python3 manage.py producttype create "${COLLECTION}"_Product_EW03_3 --traceback \ - --coverage-type "RGB" - python3 manage.py browsetype create "${COLLECTION}"_Product_EW03_3 --traceback \ + python3 manage.py browsetype create "${COLLECTION}"_Product_EW03 "TRUE_COLOR" --traceback \ --red "red" \ --green "green" \ --blue "blue" \ - --red-range 0 750 \ - --green-range 0 1200 \ - --blue-range 0 800 \ + --red-range 100 700 \ + --green-range 100 600 \ + --blue-range 100 700 \ --red-nodata 0 \ --green-nodata 0 \ --blue-nodata 0 - python3 manage.py browsetype create "${COLLECTION}"_Product_EW03_3 "TRUE_COLOR" --traceback \ - --red "red" \ - --green "green" \ - --blue "blue" \ - --red-range 0 750 \ - --green-range 0 1200 \ - --blue-range 0 800 \ - --red-nodata 0 \ - --green-nodata 0 \ - --blue-nodata 0 - python3 manage.py browsetype create "${COLLECTION}"_Product_EW03_3 "FALSE_COLOR" --traceback \ - --red "green" \ - --green "red" \ - --blue "blue" \ - --red-range 0 1200 \ - --green-range 0 750 \ - --blue-range 0 800 \ - --red-nodata 0 \ - --green-nodata 0 \ - --blue-nodata 0 - # EW03_4 - python3 manage.py producttype create "${COLLECTION}"_Product_EW03_4 --traceback \ - --coverage-type "RGBNir" - python3 manage.py browsetype create "${COLLECTION}"_Product_EW03_4 --traceback \ - --red "blue" \ - --green "green" \ - --blue "red" \ - --red-range 0 500 \ - --green-range 0 650 \ - --blue-range 0 500 \ - --red-nodata 0 \ - --green-nodata 0 \ - --blue-nodata 0 - python3 manage.py browsetype create "${COLLECTION}"_Product_EW03_4 "TRUE_COLOR" --traceback \ - --red "blue" \ - --green "green" \ - --blue "red" \ - --red-range 0 500 \ - --green-range 0 650 \ - --blue-range 0 500 \ - --red-nodata 0 \ - --green-nodata 0 \ - --blue-nodata 0 - python3 manage.py browsetype create "${COLLECTION}"_Product_EW03_4 "FALSE_COLOR" --traceback \ + python3 manage.py browsetype create "${COLLECTION}"_Product_EW03 "FALSE_COLOR" --traceback \ --red "nir" \ - --green "blue" \ + --green "red" \ --blue "green" \ - --red-range 0 700 \ - --green-range 0 500 \ - --blue-range 0 650 \ + --red-range 100 700 \ + --green-range 100 600 \ + --blue-range 100 700 \ --red-nodata 0 \ --green-nodata 0 \ --blue-nodata 0 - python3 manage.py browsetype create "${COLLECTION}"_Product_EW03_4 "NDVI" --traceback \ + python3 manage.py browsetype create "${COLLECTION}"_Product_EW03 "NDVI" --traceback \ --grey "(nir-blue)/(nir+blue)" --grey-range -1 1 - # EW03_8 - python3 manage.py producttype create "${COLLECTION}"_Product_EW03_8 --traceback \ - --coverage-type "CBGYRReNirNir2" - python3 manage.py browsetype create "${COLLECTION}"_Product_EW03_8 --traceback \ - --red "red" \ - --green "green" \ - --blue "blue" \ - --red-range 150 750 \ - --green-range 250 800 \ - --blue-range 200 600 \ - --red-nodata 0 \ - --green-nodata 0 \ - --blue-nodata 0 - python3 manage.py browsetype create "${COLLECTION}"_Product_EW03_8 "TRUE_COLOR" --traceback \ + python3 manage.py producttype create "${COLLECTION}"_Product_GE01 --traceback \ + --coverage-type "BGRNir" \ + --coverage-type "RGB" + python3 manage.py browsetype create "${COLLECTION}"_Product_GE01 --traceback \ --red "red" \ --green "green" \ --blue "blue" \ - --red-range 150 750 \ - --green-range 250 800 \ - --blue-range 200 600 \ - --red-nodata 0 \ - --green-nodata 0 \ - --blue-nodata 0 - python3 manage.py browsetype create "${COLLECTION}"_Product_EW03_8 "FALSE_COLOR" --traceback \ - --red "nir_2" \ - --green "red" \ - --blue "green" \ - --red-range 250 1000 \ - --green-range 250 800 \ - --blue-range 200 650 \ - --red-nodata 0 \ - --green-nodata 0 \ - --blue-nodata 0 - python3 manage.py browsetype create "${COLLECTION}"_Product_EW03_8 "NDVI" --traceback \ - --grey "(nir_2-red)/(nir_2+red)" --grey-range -1 1 - # GE01_4 - python3 manage.py producttype create "${COLLECTION}"_Product_GE01_4 --traceback \ - --coverage-type "RGBNir" - python3 manage.py browsetype create "${COLLECTION}"_Product_GE01_4 --traceback \ - --red "blue" \ - --green "green" \ - --blue "red" \ --red-range 0 600 \ --green-range 0 700 \ --blue-range 0 800 \ --red-nodata 0 \ --green-nodata 0 \ --blue-nodata 0 - python3 manage.py browsetype create "${COLLECTION}"_Product_GE01_4 "TRUE_COLOR" --traceback \ - --red "blue" \ + python3 manage.py browsetype create "${COLLECTION}"_Product_GE01 "TRUE_COLOR" --traceback \ + --red "red" \ --green "green" \ - --blue "red" \ + --blue "blue" \ --red-range 0 600 \ --green-range 0 700 \ --blue-range 0 800 \ --red-nodata 0 \ --green-nodata 0 \ --blue-nodata 0 - python3 manage.py browsetype create "${COLLECTION}"_Product_GE01_4 "FALSE_COLOR" --traceback \ + python3 manage.py browsetype create "${COLLECTION}"_Product_GE01 "FALSE_COLOR" --traceback \ --red "nir" \ - --green "blue" \ + --green "red" \ --blue "green" \ --red-range 0 1000 \ --green-range 0 600 \ @@ -411,42 +247,9 @@ if python3 manage.py id check "${COLLECTION}"; then --red-nodata 0 \ --green-nodata 0 \ --blue-nodata 0 - python3 manage.py browsetype create "${COLLECTION}"_Product_GE01_4 "NDVI" --traceback \ - --grey "(nir-blue)/(nir+blue)" --grey-range -1 1 - # GE01_3 - python3 manage.py producttype create "${COLLECTION}"_Product_GE01_3 --traceback \ - --coverage-type "RGB" - python3 manage.py browsetype create "${COLLECTION}"_Product_GE01_3 --traceback \ - --red "red" \ - --green "green" \ - --blue "blue" \ - --red-range 1000 2048 \ - --green-range 1000 2048 \ - --blue-range 1000 2048 \ - --red-nodata 0 \ - --green-nodata 0 \ - --blue-nodata 0 - python3 manage.py browsetype create "${COLLECTION}"_Product_GE01_3 "TRUE_COLOR" --traceback \ - --red "red" \ - --green "green" \ - --blue "blue" \ - --red-range 1000 1900 \ - --green-range 1000 1900 \ - --blue-range 1000 1900 \ - --red-nodata 0 \ - --green-nodata 0 \ - --blue-nodata 0 - python3 manage.py browsetype create "${COLLECTION}"_Product_GE01_3 "FALSE_COLOR" --traceback \ - --red "green" \ - --green "red" \ - --blue "blue" \ - --red-range 1000 1900 \ - --green-range 1000 1900 \ - --blue-range 1000 1900 \ - --red-nodata 0 \ - --green-nodata 0 \ - --blue-nodata 0 - # GE01_1 + python3 manage.py browsetype create "${COLLECTION}"_Product_GE01 "NDVI" --traceback \ + --grey "(nir-red)/(nir+red)" --grey-range -1 1 + # GE01_1 - believe that this cant be fixed anyhow because of rendering grayscale vs rgb python3 manage.py producttype create "${COLLECTION}"_Product_GE01_1 --traceback \ --coverage-type "grayscale" python3 manage.py browsetype create "${COLLECTION}"_Product_GE01_1 --traceback \ @@ -459,39 +262,40 @@ if python3 manage.py id check "${COLLECTION}"; then --red-nodata 0 # GY01 python3 manage.py producttype create "${COLLECTION}"_Product_GY01 --traceback \ - --coverage-type "RGBNir" + --coverage-type "BGRNir" \ + --coverage-type "RGB" python3 manage.py browsetype create "${COLLECTION}"_Product_GY01 --traceback \ - --red "blue" \ + --red "red" \ --green "green" \ - --blue "red" \ - --red-range 50 450 \ + --blue "blue" \ + --red-range 50 600 \ --green-range 50 800 \ - --blue-range 50 600 \ + --blue-range 50 450 \ --red-nodata 0 \ --green-nodata 0 \ --blue-nodata 0 python3 manage.py browsetype create "${COLLECTION}"_Product_GY01 "TRUE_COLOR" --traceback \ - --red "blue" \ + --red "red" \ --green "green" \ - --blue "red" \ - --red-range 50 450 \ + --blue "blue" \ + --red-range 50 600 \ --green-range 50 800 \ - --blue-range 50 600 \ + --blue-range 50 450 \ --red-nodata 0 \ --green-nodata 0 \ --blue-nodata 0 python3 manage.py browsetype create "${COLLECTION}"_Product_GY01 "FALSE_COLOR" --traceback \ --red "nir" \ - --green "blue" \ + --green "red" \ --blue "green" \ --red-range 50 1000 \ - --green-range 50 450 \ + --green-range 50 600 \ --blue-range 50 800 \ --red-nodata 0 \ --green-nodata 0 \ --blue-nodata 0 python3 manage.py browsetype create "${COLLECTION}"_Product_GY01 "NDVI" --traceback \ - --grey "(nir-blue)/(nir+blue)" --grey-range -1 1 + --grey "(nir-red)/(nir+red)" --grey-range -1 1 # IK02 python3 manage.py producttype create "${COLLECTION}"_Product_IK02 --traceback \ --coverage-type "RGBNir" @@ -562,51 +366,20 @@ if python3 manage.py id check "${COLLECTION}"; then --blue-nodata 0 python3 manage.py browsetype create "${COLLECTION}"_Product_RE00 "NDVI" --traceback \ --grey "(nir-red)/(nir+red)" --grey-range -1 1 - # RS02_2 - python3 manage.py producttype create "${COLLECTION}"_Product_RS02_2 --traceback \ - --coverage-type "sar_hh_gray" - python3 manage.py browsetype create "${COLLECTION}"_Product_RS02_2 --traceback \ - --red "gray" \ - --red-range 1 65535 \ - --red-nodata 0 - python3 manage.py browsetype create "${COLLECTION}"_Product_RS02_2 "TRUE_COLOR" --traceback \ - --red "gray" \ - --red-range 1 65535 \ - --red-nodata 0 - # RS02_3 - python3 manage.py producttype create "${COLLECTION}"_Product_RS02_3 --traceback \ - --coverage-type "sar_hh_vv_gray" - python3 manage.py browsetype create "${COLLECTION}"_Product_RS02_3 --traceback \ + # RS02 + python3 manage.py producttype create "${COLLECTION}"_Product_RS02 --traceback \ + --coverage-type "sar_hh_gray" \ + --coverage-type "sar_hh_vv_gray" \ + --coverage-type "sar_hh_hv_vh_vv_gray" + + python3 manage.py browsetype create "${COLLECTION}"_Product_RS02 --traceback \ --red "gray" \ --red-range 1 65535 \ --red-nodata 0 - python3 manage.py browsetype create "${COLLECTION}"_Product_RS02_3 "TRUE_COLOR" --traceback \ + python3 manage.py browsetype create "${COLLECTION}"_Product_RS02 "TRUE_COLOR" --traceback \ --red "gray" \ --red-range 1 65535 \ --red-nodata 0 - # RS02_7 - python3 manage.py producttype create "${COLLECTION}"_Product_RS02_7 --traceback \ - --coverage-type "sar_hh_hv_vh_vv_rgb" - python3 manage.py browsetype create "${COLLECTION}"_Product_RS02_7 --traceback \ - --red "red" \ - --red-range 1 65535 \ - --red-nodata 0 - python3 manage.py browsetype create "${COLLECTION}"_Product_RS02_7 "TRUE_COLOR" --traceback \ - --red "red" \ - --red-range 1 65535 \ - --red-nodata 0 - python3 manage.py browsetype create "${COLLECTION}"_Product_RS02_7 "FALSE_COLOR" --traceback \ - --red "red" \ - --green "green" \ - --blue "blue" \ - --red-range 1 65535 \ - --green-range 1 65535 \ - --blue-range 1 65535 \ - --red-nodata 0 \ - --green-nodata 0 \ - --blue-nodata 0 - python3 manage.py browsetype create "${COLLECTION}"_Product_RS02_7 "NDVI" --traceback \ - --grey "(vh-vv)/(vh+vv)" --grey-range -1 1 # SP04 python3 manage.py producttype create "${COLLECTION}"_Product_SP04 --traceback \ --coverage-type "RGBNirByte" @@ -666,50 +439,18 @@ if python3 manage.py id check "${COLLECTION}"; then --green-nodata 0 \ --blue-nodata 0 # TX01_2 - python3 manage.py producttype create "${COLLECTION}"_Product_TX01_2 --traceback \ - --coverage-type "sar_hh_gray" - python3 manage.py browsetype create "${COLLECTION}"_Product_TX01_2 --traceback \ - --red "gray" \ - --red-range 1 65535 \ - --red-nodata 0 - python3 manage.py browsetype create "${COLLECTION}"_Product_TX01_2 "TRUE_COLOR" --traceback \ - --red "gray" \ - --red-range 1 65535 \ - --red-nodata 0 - # TX01_3 - python3 manage.py producttype create "${COLLECTION}"_Product_TX01_3 --traceback \ - --coverage-type "sar_hh_vv_gray" - python3 manage.py browsetype create "${COLLECTION}"_Product_TX01_3 --traceback \ + python3 manage.py producttype create "${COLLECTION}"_Product_TX01 --traceback \ + --coverage-type "sar_hh_gray" \ + --coverage-type "sar_hh_vv_gray" \ + --coverage-type "sar_hh_hv_vh_vv_gray" + python3 manage.py browsetype create "${COLLECTION}"_Product_TX01 --traceback \ --red "gray" \ --red-range 1 65535 \ --red-nodata 0 - python3 manage.py browsetype create "${COLLECTION}"_Product_TX01_3 "TRUE_COLOR" --traceback \ + python3 manage.py browsetype create "${COLLECTION}"_Product_TX01 "TRUE_COLOR" --traceback \ --red "gray" \ --red-range 1 65535 \ --red-nodata 0 - # TX01_7 - python3 manage.py producttype create "${COLLECTION}"_Product_TX01_7 --traceback \ - --coverage-type "sar_hh_hv_vh_vv_rgb" - python3 manage.py browsetype create "${COLLECTION}"_Product_TX01_7 --traceback \ - --red "red" \ - --red-range 1 65535 \ - --red-nodata 0 - python3 manage.py browsetype create "${COLLECTION}"_Product_TX01_7 "TRUE_COLOR" --traceback \ - --red "red" \ - --red-range 1 65535 \ - --red-nodata 0 - python3 manage.py browsetype create "${COLLECTION}"_Product_TX01_7 "FALSE_COLOR" --traceback \ - --red "red" \ - --green "green" \ - --blue "blue" \ - --red-range 1 65535 \ - --green-range 1 65535 \ - --blue-range 1 65535 \ - --red-nodata 0 \ - --green-nodata 0 \ - --blue-nodata 0 - python3 manage.py browsetype create "${COLLECTION}"_Product_TX01_7 "NDVI" --traceback \ - --grey "(vh-vv)/(vh+vv)" --grey-range -1 1 # DM02 python3 manage.py producttype create "${COLLECTION}"_Product_DM02 --traceback \ --coverage-type "RGBNir" @@ -885,7 +626,6 @@ if python3 manage.py id check "${COLLECTION}"; then --blue-nodata 0 python3 manage.py browsetype create "${COLLECTION}"_Product_PH1A "NDVI" --traceback \ --grey "(nir-red)/(nir+red)" --grey-range -1 1 - # PH1B python3 manage.py producttype create "${COLLECTION}"_Product_PH1B --traceback \ --coverage-type "RGBNir" @@ -921,6 +661,7 @@ if python3 manage.py id check "${COLLECTION}"; then --blue-nodata 0 python3 manage.py browsetype create "${COLLECTION}"_Product_PH1B "NDVI" --traceback \ --grey "(nir-red)/(nir+red)" --grey-range -1 1 + # SK00 python3 manage.py producttype create "${COLLECTION}"_Product_SK00 --traceback \ --coverage-type "RGBNir" python3 manage.py browsetype create "${COLLECTION}"_Product_SK00 --traceback \ @@ -955,6 +696,7 @@ if python3 manage.py id check "${COLLECTION}"; then --blue-nodata 0 python3 manage.py browsetype create "${COLLECTION}"_Product_SK00 "NDVI" --traceback \ --grey "(nir-red)/(nir+red)" --grey-range -1 1 + # SW03 python3 manage.py producttype create "${COLLECTION}"_Product_SW03 --traceback \ --coverage-type "RGBNir" python3 manage.py browsetype create "${COLLECTION}"_Product_SW03 --traceback \ @@ -989,7 +731,7 @@ if python3 manage.py id check "${COLLECTION}"; then --blue-nodata 0 python3 manage.py browsetype create "${COLLECTION}"_Product_SW03 "NDVI" --traceback \ --grey "(nir-red)/(nir+red)" --grey-range -1 1 - # bands go in order b,g,r,nir and I did not change them in preprocessing + # PL00 bands go in order b,g,r,nir python3 manage.py producttype create "${COLLECTION}"_Product_PL00 --traceback \ --coverage-type "RGBNir" python3 manage.py browsetype create "${COLLECTION}"_Product_PL00 --traceback \ @@ -1034,7 +776,7 @@ if python3 manage.py id check "${COLLECTION}"; then --coverage-type "BGRReNir" \ --coverage-type "sar_hh_gray" \ --coverage-type "sar_hh_vv_gray" \ - --coverage-type "sar_hh_hv_vh_vv_rgb" \ + --coverage-type "sar_hh_hv_vh_vv_gray" \ --product-type "${COLLECTION}"_Product_PH1A \ --product-type "${COLLECTION}"_Product_PH1B \ --product-type "${COLLECTION}"_Product_DM02 \ @@ -1045,26 +787,16 @@ if python3 manage.py id check "${COLLECTION}"; then --product-type "${COLLECTION}"_Product_CS03 \ --product-type "${COLLECTION}"_Product_CS04 \ --product-type "${COLLECTION}"_Product_EW01 \ - --product-type "${COLLECTION}"_Product_EW02_3 \ - --product-type "${COLLECTION}"_Product_EW02_4 \ - --product-type "${COLLECTION}"_Product_EW02_8 \ - --product-type "${COLLECTION}"_Product_EW03_3 \ - --product-type "${COLLECTION}"_Product_EW03_4 \ - --product-type "${COLLECTION}"_Product_EW03_8 \ - --product-type "${COLLECTION}"_Product_EQ02_3 \ - --product-type "${COLLECTION}"_Product_EQ02_4 \ - --product-type "${COLLECTION}"_Product_GE01_4 \ - --product-type "${COLLECTION}"_Product_GE01_3 \ + --product-type "${COLLECTION}"_Product_EW02 \ + --product-type "${COLLECTION}"_Product_EW03 \ + --product-type "${COLLECTION}"_Product_EQ02 \ + --product-type "${COLLECTION}"_Product_GE01 \ --product-type "${COLLECTION}"_Product_GE01_1 \ --product-type "${COLLECTION}"_Product_GY01 \ - --product-type "${COLLECTION}"_Product_TX01_2 \ - --product-type "${COLLECTION}"_Product_TX01_3 \ - --product-type "${COLLECTION}"_Product_TX01_7 \ + --product-type "${COLLECTION}"_Product_TX01 \ --product-type "${COLLECTION}"_Product_IK02 \ --product-type "${COLLECTION}"_Product_RE00 \ - --product-type "${COLLECTION}"_Product_RS02_2 \ - --product-type "${COLLECTION}"_Product_RS02_3 \ - --product-type "${COLLECTION}"_Product_RS02_7 \ + --product-type "${COLLECTION}"_Product_RS02 \ --product-type "${COLLECTION}"_Product_SP04 \ --product-type "${COLLECTION}"_Product_SP05 \ --product-type "${COLLECTION}"_Product_SP06 \ diff --git a/config/emg_preprocessor-config.yml b/config/emg_preprocessor-config.yml index 3ed46327..fa4a3bf0 100644 --- a/config/emg_preprocessor-config.yml +++ b/config/emg_preprocessor-config.yml @@ -63,10 +63,10 @@ preprocessing: group_by: ".*/(.*)_P..tif" sort_by: ".*_P(R|G|B|N).tif" order: - - R - - G - B + - G - N + - R RS02: data_file_globs: - "*imagery_*.tif" diff --git a/config/emg_registrar-config.yml b/config/emg_registrar-config.yml index dd7acf4b..1c5c7466 100644 --- a/config/emg_registrar-config.yml +++ b/config/emg_registrar-config.yml @@ -72,20 +72,15 @@ backends: - !env '${COLLECTION}' coverages: DM02: RGBNir - EQ02_3: + EQ02: ~: - product_type_name: !env '${COLLECTION}_Product_EQ02_3' + product_type_name: !env '${COLLECTION}_Product_EQ02' collections: - !env '${COLLECTION}' coverages: - EQ02_3: RGB - EQ02_4: - ~: - product_type_name: !env '${COLLECTION}_Product_EQ02_4' - collections: - - !env '${COLLECTION}' - coverages: - EQ02_4: RGBNir + EQ02: + - RGB + - BGRNir EW01: ~: product_type_name: !env '${COLLECTION}_Product_EW01' @@ -93,63 +88,36 @@ backends: - !env '${COLLECTION}' coverages: EW01: grayscale - EW02_3: - ~: - product_type_name: !env '${COLLECTION}_Product_EW02_3' - collections: - - !env '${COLLECTION}' - coverages: - EW02_3: RGB - EW02_4: - ~: - product_type_name: !env '${COLLECTION}_Product_EW02_4' - collections: - - !env '${COLLECTION}' - coverages: - EW02_4: RGBNir - EW02_8: - ~: - product_type_name: !env '${COLLECTION}_Product_EW02_8' - collections: - - !env '${COLLECTION}' - coverages: - EW02_8: CBGYRReNirNir2 - EW03_3: - ~: - product_type_name: !env '${COLLECTION}_Product_EW03_3' - collections: - - !env '${COLLECTION}' - coverages: - EW03_3: RGB - EW03_4: + EW02: ~: - product_type_name: !env '${COLLECTION}_Product_EW03_4' + product_type_name: !env '${COLLECTION}_Product_EW02' collections: - !env '${COLLECTION}' coverages: - EW03_4: RGBNir - EW03_8: + EW02: + - RGB + - BGRNir + - CBGYRReNirNir2 + EW03: ~: - product_type_name: !env '${COLLECTION}_Product_EW03_8' + product_type_name: !env '${COLLECTION}_Product_EW03' collections: - !env '${COLLECTION}' coverages: - EW03_8: CBGYRReNirNir2 - GE01_4: + EW03: + - RGB + - BGRNir + - CBGYRReNirNir2 + GE01: ~: - product_type_name: !env '${COLLECTION}_Product_GE01_4' + product_type_name: !env '${COLLECTION}_Product_GE01' collections: - !env '${COLLECTION}' coverages: - GE01_4: RGBNir - GE01_3: - ~: - product_type_name: !env '${COLLECTION}_Product_GE01_3' - collections: - - !env '${COLLECTION}' - coverages: - GE01_3: RGB - GE01_1: + GE01: + - RGB + - BGRNir + GE01_1: # I think that because of browse types this can not be fixed ~: product_type_name: !env '${COLLECTION}_Product_GE01_1' collections: @@ -198,27 +166,16 @@ backends: - !env '${COLLECTION}' coverages: RE00: BGRReNir - RS02_2: - ~: - product_type_name: !env '${COLLECTION}_Product_RS02_2' - collections: - - !env '${COLLECTION}' - coverages: - RS02_2: sar_hh_gray - RS02_3: - ~: - product_type_name: !env '${COLLECTION}_Product_RS02_3' - collections: - - !env '${COLLECTION}' - coverages: - RS02_3: sar_hh_vv_gray - RS02_7: + RS02: ~: - product_type_name: !env '${COLLECTION}_Product_RS02_7' + product_type_name: !env '${COLLECTION}_Product_RS02' collections: - !env '${COLLECTION}' coverages: - RS02_7: sar_hh_hv_vh_vv_rgb + RS02: + - sar_hh_gray + - sar_hh_vv_gray + - sar_hh_hv_vh_vv_gray SP04: ~: product_type_name: !env '${COLLECTION}_Product_SP04' @@ -268,27 +225,16 @@ backends: - !env '${COLLECTION}' coverages: PL00: RGBNir - TX01_2: - ~: - product_type_name: !env '${COLLECTION}_Product_TX01_2' - collections: - - !env '${COLLECTION}' - coverages: - TX01_2: sar_hh_gray - TX01_3: - ~: - product_type_name: !env '${COLLECTION}_Product_TX01_3' - collections: - - !env '${COLLECTION}' - coverages: - TX01_3: sar_hh_vv_gray - TX01_7: + TX01: ~: - product_type_name: !env '${COLLECTION}_Product_TX01_7' + product_type_name: !env '${COLLECTION}_Product_TX01' collections: - !env '${COLLECTION}' coverages: - TX01_7: sar_hh_hv_vh_vv_rgb + TX01: + - sar_hh_gray + - sar_hh_vv_gray + - sar_hh_hv_vh_vv_gray post_handlers: - path: registrar.post_handlers.ReportingPostHandler diff --git a/core/rgbnir_definition.json b/core/rgbnir_definition.json index 6dc31cdc..3834f9fc 100644 --- a/core/rgbnir_definition.json +++ b/core/rgbnir_definition.json @@ -466,64 +466,10 @@ "allowed_value_ranges": [ [0, 65535] ] - }, - { - "definition": "http://www.opengis.net/def/property/OGC/0/Radiance", - "description": "Red Channel", - "gdal_interpretation": "RedBand", - "identifier": "red", - "name": "red", - "nil_values": [ - { - "reason": "http://www.opengis.net/def/nil/OGC/0/unknown", - "value": 0 - } - ], - "uom": "W.m-2.Sr-1", - "significant_figures": 5, - "allowed_value_ranges": [ - [0, 65535] - ] - }, - { - "definition": "http://www.opengis.net/def/property/OGC/0/Radiance", - "description": "Green Channel", - "gdal_interpretation": "GreenBand", - "identifier": "green", - "name": "green", - "nil_values": [ - { - "reason": "http://www.opengis.net/def/nil/OGC/0/unknown", - "value": 0 - } - ], - "uom": "W.m-2.Sr-1", - "significant_figures": 5, - "allowed_value_ranges": [ - [0, 65535] - ] - }, - { - "definition": "http://www.opengis.net/def/property/OGC/0/Radiance", - "description": "Blue Channel", - "gdal_interpretation": "BlueBand", - "identifier": "blue", - "name": "blue", - "nil_values": [ - { - "reason": "http://www.opengis.net/def/nil/OGC/0/unknown", - "value": 0 - } - ], - "uom": "W.m-2.Sr-1", - "significant_figures": 5, - "allowed_value_ranges": [ - [0, 65535] - ] } ], "data_type": "Uint16", - "name": "sar_hh_hv_vh_vv_rgb" + "name": "sar_hh_hv_vh_vv" },{ "bands": [ { @@ -814,4 +760,81 @@ ], "data_type": "Float32", "name": "float32_grayscale" +},{ + "bands": [ + { + "definition": "http://www.opengis.net/def/property/OGC/0/Radiance", + "description": "Blue Channel", + "gdal_interpretation": "BlueBand", + "identifier": "blue", + "name": "blue", + "nil_values": [ + { + "reason": "http://www.opengis.net/def/nil/OGC/0/unknown", + "value": 0 + } + ], + "uom": "W.m-2.Sr-1", + "significant_figures": 5, + "allowed_value_ranges": [ + [0, 65535] + ] + }, + { + "definition": "http://www.opengis.net/def/property/OGC/0/Radiance", + "description": "Green Channel", + "gdal_interpretation": "GreenBand", + "identifier": "green", + "name": "green", + "nil_values": [ + { + "reason": "http://www.opengis.net/def/nil/OGC/0/unknown", + "value": 0 + } + ], + "uom": "W.m-2.Sr-1", + "significant_figures": 5, + "allowed_value_ranges": [ + [0, 65535] + ] + }, + { + "definition": "http://www.opengis.net/def/property/OGC/0/Radiance", + "description": "Red Channel", + "gdal_interpretation": "RedBand", + "identifier": "red", + "name": "red", + "nil_values": [ + { + "reason": "http://www.opengis.net/def/nil/OGC/0/unknown", + "value": 0 + } + ], + "uom": "W.m-2.Sr-1", + "significant_figures": 5, + "allowed_value_ranges": [ + [0, 65535] + ] + }, + { + "definition": "http://www.opengis.net/def/property/OGC/0/Radiance", + "description": "Nir Channel", + "gdal_interpretation": "NirBand", + "identifier": "nir", + "name": "nir", + "nil_values": [ + { + "reason": "http://www.opengis.net/def/nil/OGC/0/unknown", + "value": 0 + } + ], + "uom": "W.m-2.Sr-1", + "significant_figures": 5, + "allowed_value_ranges": [ + [0, 65535] + ] + } + ], + "data_type": "Uint16", + "name": "BGRNir" }] diff --git a/testing/testing_preprocessor_config.yml b/testing/testing_preprocessor_config.yml index 5edd267e..827c7b06 100644 --- a/testing/testing_preprocessor_config.yml +++ b/testing/testing_preprocessor_config.yml @@ -61,11 +61,11 @@ preprocessing: stack_bands: group_by: ".*/(.*)_P..tif" sort_by: ".*_P(R|G|B|N).tif" - order: - - R - - G + order: # to match browse type in init_db - B + - G - N + - R MER_RR__2P: georeference: geotransforms: -- GitLab From 89b6412791170b95fb0180999a673aa5a65d2cfd Mon Sep 17 00:00:00 2001 From: Lubomir Bucek Date: Tue, 24 Nov 2020 15:52:03 +0100 Subject: [PATCH 2/9] fix covtype name --- core/rgbnir_definition.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/rgbnir_definition.json b/core/rgbnir_definition.json index 3834f9fc..dcd7cba8 100644 --- a/core/rgbnir_definition.json +++ b/core/rgbnir_definition.json @@ -469,7 +469,7 @@ } ], "data_type": "Uint16", - "name": "sar_hh_hv_vh_vv" + "name": "sar_hh_hv_vh_vv_gray" },{ "bands": [ { -- GitLab From e1e1702d7a6d55e4812ea6c1432e33fad6bd823c Mon Sep 17 00:00:00 2001 From: Lubomir Bucek Date: Tue, 24 Nov 2020 22:13:11 +0100 Subject: [PATCH 3/9] add gray part to 5 band covtyp on hhhvvhvv sars --- core/rgbnir_definition.json | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/core/rgbnir_definition.json b/core/rgbnir_definition.json index dcd7cba8..ad8be6b6 100644 --- a/core/rgbnir_definition.json +++ b/core/rgbnir_definition.json @@ -466,6 +466,24 @@ "allowed_value_ranges": [ [0, 65535] ] + }, + { + "definition": "http://www.opengis.net/def/property/OGC/0/Radiance", + "description": "Gray Channel", + "gdal_interpretation": "GrayBand", + "identifier": "gray", + "name": "gray", + "nil_values": [ + { + "reason": "http://www.opengis.net/def/nil/OGC/0/unknown", + "value": 0 + } + ], + "uom": "W.m-2.Sr-1", + "significant_figures": 5, + "allowed_value_ranges": [ + [0, 65535] + ] } ], "data_type": "Uint16", -- GitLab From 81a1d2a7b511512f62bd8d57bef128e4f7e48695 Mon Sep 17 00:00:00 2001 From: Lubomir Bucek Date: Wed, 25 Nov 2020 09:34:04 +0100 Subject: [PATCH 4/9] remove need for separate glob in subdataset --- config/emg_preprocessor-config.yml | 23 +++++++++++++++++++ .../operator-guide/configuration.rst | 4 ---- preprocessor/preprocessor/config-schema.yaml | 3 --- .../preprocessor/steps/georeference.py | 4 ---- preprocessor/preprocessor/steps/subdataset.py | 11 +++++---- 5 files changed, 30 insertions(+), 15 deletions(-) diff --git a/config/emg_preprocessor-config.yml b/config/emg_preprocessor-config.yml index fa4a3bf0..261d3374 100644 --- a/config/emg_preprocessor-config.yml +++ b/config/emg_preprocessor-config.yml @@ -231,6 +231,29 @@ preprocessing: DM02: data_file_globs: - "*.tif" + CS00: + data_file_globs: + - "*.h5" + - "*.tif" + subdatasets: + subdataset_types: + '//S01/SBI': 'S01_SBI' + '//S02/SBI': 'S02_SBI' + '//S03/SBI': 'S03_SBI' + '//S04/SBI': 'S04_SBI' + '//S05/SBI': 'S05_SBI' + '//S06/SBI': 'S06_SBI' + '//S01/MBI': 'S01_MBI' + '//S02/MBI': 'S02_MBI' + '//S03/MBI': 'S03_MBI' + '//S04/MBI': 'S04_MBI' + '//S05/MBI': 'S05_MBI' + '//S06/MBI': 'S06_MBI' + georeference: + - type: corners + corner_names: ["S01_SBI_Bottom_Left_Geodetic_Coordinates", "S01_SBI_Bottom_Right_Geodetic_Coordinates", "S01_SBI_Top_Left_Geodetic_Coordinates", "S01_SBI_Top_Right_Geodetic_Coordinates"] + orbit_direction_name: Orbit_Direction + force_north_up: false # this configuration is still a stub - not all product types are done # https://gitlab.eox.at/esa/prism/vs/-/issues/56 # https://gitlab.eox.at/esa/prism/vs/-/issues/23 diff --git a/documentation/operator-guide/configuration.rst b/documentation/operator-guide/configuration.rst index de3e0084..7565e8f3 100644 --- a/documentation/operator-guide/configuration.rst +++ b/documentation/operator-guide/configuration.rst @@ -343,10 +343,6 @@ preprocessing What subdatasets to extract and how to name them. - data_file_glob - - A file glob pattern to select files to extract from. - subdataset_types Mapping of subdataset identifier to output filename postfix for diff --git a/preprocessor/preprocessor/config-schema.yaml b/preprocessor/preprocessor/config-schema.yaml index 10bb58f5..472f2ce0 100644 --- a/preprocessor/preprocessor/config-schema.yaml +++ b/preprocessor/preprocessor/config-schema.yaml @@ -107,9 +107,6 @@ definitions: description: The definition of the subdataset extraction step. type: object properties: - data_file_glob: - description: The data file selector. - type: string subdataset_types: description: Mapping of subdataset identifier to output filename postfix for subdatasets to be extracted for each data file. type: object diff --git a/preprocessor/preprocessor/steps/georeference.py b/preprocessor/preprocessor/steps/georeference.py index 80a8b432..9896ad76 100644 --- a/preprocessor/preprocessor/steps/georeference.py +++ b/preprocessor/preprocessor/steps/georeference.py @@ -125,10 +125,6 @@ def corner_georef(input_filename: os.PathLike, target_filename: os.PathLike, cor orbit_direction = ds.GetMetadata()[orbit_direction_name].lower() metadata = ds.GetRasterBand(1).GetMetadata() - # from pprint import pprint - - # pprint (metadata) - # pprint(ds.GetMetadata()) bl, br, tl, tr = [ [float(num) for num in metadata[corner_name].split()] for corner_name in corner_names diff --git a/preprocessor/preprocessor/steps/subdataset.py b/preprocessor/preprocessor/steps/subdataset.py index 6438d277..6fe05bcb 100644 --- a/preprocessor/preprocessor/steps/subdataset.py +++ b/preprocessor/preprocessor/steps/subdataset.py @@ -1,14 +1,17 @@ import os -from os.path import join, splitext, basename, dirname +from os.path import join, splitext, basename, dirname, isdir from glob import glob from typing import Dict from ..util import replace_ext, gdal -def extract_subdataset_step(source_dir: os.PathLike, target_dir: os.PathLike, preprocessor_config: dict, data_file_glob: str, subdataset_types: Dict[str, str]=None): - datafiles = glob(join(source_dir, data_file_glob)) - if not datafiles: +def extract_subdataset_step(source_dir: os.PathLike, target_dir: os.PathLike, preprocessor_config: dict, subdataset_types: Dict[str, str]=None): + filenames = [] + for dataglob in preprocessor_config.get('data_file_globs', '*'): + for p in [path for path in glob(join(source_dir, '**', dataglob), recursive=True) if not isdir(path)]: + filenames.append(p) + if len(filenames) == 0: raise Exception('No datafiles were matched by the provided glob') for filename in datafiles: -- GitLab From 461a9f1fd07fa08243e1d314204e1a1081c8a6b3 Mon Sep 17 00:00:00 2001 From: Lubomir Bucek Date: Wed, 25 Nov 2020 19:29:38 +0100 Subject: [PATCH 5/9] add fix geotransform function from old preprocessor and adapt it for jp2 driver open & close --- core/registrar/source.py | 2 +- preprocessor/preprocessor/config-schema.yaml | 2 +- .../preprocessor/steps/georeference.py | 53 +++++++++++++++++++ 3 files changed, 55 insertions(+), 2 deletions(-) diff --git a/core/registrar/source.py b/core/registrar/source.py index ce043662..53b56ee5 100644 --- a/core/registrar/source.py +++ b/core/registrar/source.py @@ -67,7 +67,7 @@ class SwiftSource(Source): def get_container_and_path(self, path: str): container = self.container - if container is None: + if container is None or container == '': parts = (path[1:] if path.startswith('/') else path).split('/') container, path = parts[0], '/'.join(parts[1:]) diff --git a/preprocessor/preprocessor/config-schema.yaml b/preprocessor/preprocessor/config-schema.yaml index 472f2ce0..f1c7138e 100644 --- a/preprocessor/preprocessor/config-schema.yaml +++ b/preprocessor/preprocessor/config-schema.yaml @@ -125,7 +125,7 @@ definitions: type: description: The type of georeferencing to apply. type: string - enum: [gcp, rpc, corner, world, no_op] # TODO: more + enum: [gcp, rpc, corner, world, no_op, fix_geotrans] # TODO: more options: description: Additional options for the georeferencing. Depends on the type of georeferencing. type: object diff --git a/preprocessor/preprocessor/steps/georeference.py b/preprocessor/preprocessor/steps/georeference.py index 9896ad76..afadc7bf 100644 --- a/preprocessor/preprocessor/steps/georeference.py +++ b/preprocessor/preprocessor/steps/georeference.py @@ -26,6 +26,8 @@ def georeference_step(source_dir: os.PathLike, target_dir: os.PathLike, preproce georef_func = corner_georef elif type_name == 'no_op': georef_func = no_op + elif type_name == 'fix_geotrans': + georef_func = fix_geotrans else: raise Exception('Invalid georeference type %s' % type_name) try: @@ -166,6 +168,57 @@ def no_op(input_filename: os.PathLike, target_filename: os.PathLike): shutil.move(input_filename, target_filename) +def fix_geotrans(input_filename: os.PathLike, target_filename: os.PathLike, warp_options: dict=None): + # assumes already existing geotransform and if xres=0,yres=0, fixes it + try: + ds = gdal.Open(input_filename, gdal.GA_Update) + except RuntimeError: + logger.warn('Can not open file by GDAL %s' % (input_filename)) + return + ds = correct_geo_transform(ds) + if ds.GetDriver().ShortName == 'JP2OpenJPEG': + # workaround for rewriting jp2 files and exception thrown by "USE_SRC_CODESTREAM=YES specified, but no codestream found" + # then needs .vrt configured in data_file_globs too + target_filename = replace_ext(target_filename, '.vrt'), + gdal.Warp( + target_filename, + ds, + **(warp_options or {}), + ) + + +def correct_geo_transform(src_ds): + # input - gdal dataset + # sets new geotransform if necessary by creating control points of a raster with switched height and width + # returns - gdal dataset + ulx, xres, xskew, uly, yskew, yres = src_ds.GetGeoTransform() + logger.debug("Testing for malformed geotransform") + # test geotransform if necessary to shift + if xres == 0.0 and yres == 0.0: + logger.info("Malformed geotransform xres,yres=0 detected, correcting.") + # malformed image, compute xres and yres switched in geotransform + lrx = ulx + (src_ds.RasterXSize * xskew) + lry = uly + (src_ds.RasterYSize * yskew) + # [ulx, lrx, lry, uly] - bounds = lon_min, lon_max, lat_min, lat_max + fp = [[0, src_ds.RasterXSize, src_ds.RasterXSize, 0], [0, 0, src_ds.RasterYSize, src_ds.RasterYSize]] + tp = [[ulx, lrx, lrx, ulx], [lry, lry, uly, uly]] + pix = list(zip(fp[0], fp[1])) + coor = list(zip(tp[0], tp[1])) + # compute the gdal.GCP parameters + gcps = [] + for index, txt in enumerate(pix): + gcps.append(gdal.GCP()) + gcps[index].GCPPixel = pix[index][0] + gcps[index].GCPLine = src_ds.RasterYSize - int(pix[index][1]) + gcps[index].GCPX = coor[index][0] + gcps[index].GCPY = coor[index][1] + # get correct geotransform from gcps + geotransform_new = gdal.GCPsToGeoTransform(gcps) + # overwrite geotransform with new + src_ds.SetGeoTransform(geotransform_new) + return src_ds + + def gcps_from_borders(size: Tuple[float, float], coords: List[Tuple[float, float]], orbit_direction: str, force_north_up: bool=False): x_size, y_size = size # expects coordinates in dict(.*border_left.*:[lat,lon],...) -- GitLab From 5a1281126e8f8edb7794983e6ccef9f2189b07bb Mon Sep 17 00:00:00 2001 From: Lubomir Bucek Date: Wed, 25 Nov 2020 19:31:43 +0100 Subject: [PATCH 6/9] add fix_geotrans to PH1A,PH1B preprocessor config --- config/emg_preprocessor-config.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/config/emg_preprocessor-config.yml b/config/emg_preprocessor-config.yml index 261d3374..c12bb450 100644 --- a/config/emg_preprocessor-config.yml +++ b/config/emg_preprocessor-config.yml @@ -166,12 +166,14 @@ preprocessing: - "*IMG_*_PMS_*.tif" - "*IMG_*_MS_*.JP2" - "*IMG_*_MS_*.tif" + - "*.vrt" additional_file_globs: - "*RPC_*" - "*DIM_*" georeference: geotransforms: - type: rpc + - type: fix_geotrans - type: no_op PH1B: data_file_globs: @@ -179,12 +181,14 @@ preprocessing: - "*IMG_*_PMS_*.tif" - "*IMG_*_MS_*.JP2" - "*IMG_*_MS_*.tif" + - "*.vrt" additional_file_globs: - "*RPC_*" - "*DIM_*" georeference: geotransforms: - type: rpc + - type: fix_geotrans - type: no_op PL00: data_file_globs: -- GitLab From 32f8ab9316fdfe0c699e81f4f4b3591a60deeb87 Mon Sep 17 00:00:00 2001 From: Lubomir Bucek Date: Thu, 26 Nov 2020 14:29:52 +0100 Subject: [PATCH 7/9] registering to coverage type based on band count --- .bumpversion.cfg | 2 +- core/registrar/backend.py | 29 +++++++++++++++++++++++++---- core/setup.py | 2 +- 3 files changed, 27 insertions(+), 6 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 801f789a..4ad935d8 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -28,7 +28,7 @@ replace = version="{new_version}" search = version="{current_version}" replace = version="{new_version}" -[bumpversion:glob:registrar/setup.py] +[bumpversion:glob:core/setup.py] search = version="{current_version}" replace = version="{new_version}" diff --git a/core/registrar/backend.py b/core/registrar/backend.py index 056017e0..a51d3dee 100644 --- a/core/registrar/backend.py +++ b/core/registrar/backend.py @@ -7,7 +7,9 @@ import json import django from django.db import transaction +from django.db.models import Count from django.contrib.gis.geos import GEOSGeometry, Polygon +from osgeo import gdal from .exceptions import RegistrationError from .context import Context @@ -129,14 +131,16 @@ class EOxServerBackend(Backend): from eoxserver.resources.coverages.registration.registrators.gdal import GDALRegistrator # get the mapping for this particular item - type_mapping = self.mapping[item.product_type] + try: + type_mapping = self.mapping[item.product_type] + except KeyError: + raise RegistrationError(f'Could not find mapping for {item.product_type} in configuration') mapping = type_mapping.get(item.product_level) or type_mapping.get(None) if not mapping: raise RegistrationError(f'Could not get mapping for {item.product_type} {item.product_level}') _, metadata_file = source.get_container_and_path(item.metadata_files[0]) - # metadata_file = '/'.join(item.metadata_files[0].split('/')[1:]) storage = self._get_storage_from_source(source, item.path) try: @@ -180,9 +184,26 @@ class EOxServerBackend(Backend): storage + [source.get_container_and_path(raster_item)[1]] for raster_item in (raster_items if isinstance(raster_items, list) else [raster_items]) ] - + if isinstance(coverage_type_name, list): + # determine which one to use based on number of bands + gdal_path_prefix = '' + if isinstance(source, SwiftSource): + gdal_path_prefix = '/vsiswift/' + elif isinstance(source, S3Source): + gdal_path_prefix = '/vsis3/' + raster_item_to_check = f'{gdal_path_prefix}{raster_items[0][0]}/{raster_items[0][1]}' + ds = gdal.Open(raster_item_to_check) + logger.debug("Raster has number of bands: '%s'" % ds.RasterCount) + # find covtype that has correct number of bands + try: + coverage_model = models.CoverageType.objects \ + .filter(name__in=coverage_type_name) \ + .annotate(field_count=Count('field_types')) \ + .get(field_count=ds.RasterCount) + except models.CoverageType.DoesNotExist: + raise RegistrationError(f'Coverage Band Count is different than in any of configured CoverageTypes') + coverage_type_name = coverage_model.name logger.info(f"Registering coverage{'s' if len(raster_items) > 1 else ''} {raster_items} as {coverage_type_name}") - report = GDALRegistrator().register( data_locations=raster_items, metadata_locations=[storage + [metadata_file]], diff --git a/core/setup.py b/core/setup.py index 93112025..223d0ed5 100644 --- a/core/setup.py +++ b/core/setup.py @@ -6,7 +6,7 @@ long_description = "" setup( name="registrar", # Replace with your own username - version="1.0.0-rc.2", + version="1.0.0", author="", author_email="", description="registrar for PVS", -- GitLab From 85fbaa17218eb720b879f4cfcb4910ed93028b60 Mon Sep 17 00:00:00 2001 From: Lubomir Bucek Date: Fri, 27 Nov 2020 17:32:42 +0100 Subject: [PATCH 8/9] remove `-` from storage name in swift, update configs --- config/emg_init-db.sh | 26 +++++++------------------- config/emg_registrar-config.yml | 11 +++-------- core/registrar/backend.py | 11 +++-------- 3 files changed, 13 insertions(+), 35 deletions(-) diff --git a/config/emg_init-db.sh b/config/emg_init-db.sh index 4fe5909e..8b5cbc8d 100644 --- a/config/emg_init-db.sh +++ b/config/emg_init-db.sh @@ -249,17 +249,6 @@ if python3 manage.py id check "${COLLECTION}"; then --blue-nodata 0 python3 manage.py browsetype create "${COLLECTION}"_Product_GE01 "NDVI" --traceback \ --grey "(nir-red)/(nir+red)" --grey-range -1 1 - # GE01_1 - believe that this cant be fixed anyhow because of rendering grayscale vs rgb - python3 manage.py producttype create "${COLLECTION}"_Product_GE01_1 --traceback \ - --coverage-type "grayscale" - python3 manage.py browsetype create "${COLLECTION}"_Product_GE01_1 --traceback \ - --red "gray" \ - --red-range 0 2000 \ - --red-nodata 0 - python3 manage.py browsetype create "${COLLECTION}"_Product_GE01_1 "TRUE_COLOR" --traceback \ - --red "gray" \ - --red-range 0 2000 \ - --red-nodata 0 # GY01 python3 manage.py producttype create "${COLLECTION}"_Product_GY01 --traceback \ --coverage-type "BGRNir" \ @@ -269,8 +258,8 @@ if python3 manage.py id check "${COLLECTION}"; then --green "green" \ --blue "blue" \ --red-range 50 600 \ - --green-range 50 800 \ - --blue-range 50 450 \ + --green-range 50 600 \ + --blue-range 50 600 \ --red-nodata 0 \ --green-nodata 0 \ --blue-nodata 0 @@ -279,8 +268,8 @@ if python3 manage.py id check "${COLLECTION}"; then --green "green" \ --blue "blue" \ --red-range 50 600 \ - --green-range 50 800 \ - --blue-range 50 450 \ + --green-range 50 600 \ + --blue-range 50 600 \ --red-nodata 0 \ --green-nodata 0 \ --blue-nodata 0 @@ -288,9 +277,9 @@ if python3 manage.py id check "${COLLECTION}"; then --red "nir" \ --green "red" \ --blue "green" \ - --red-range 50 1000 \ + --red-range 50 800 \ --green-range 50 600 \ - --blue-range 50 800 \ + --blue-range 50 600 \ --red-nodata 0 \ --green-nodata 0 \ --blue-nodata 0 @@ -438,7 +427,7 @@ if python3 manage.py id check "${COLLECTION}"; then --red-nodata 0 \ --green-nodata 0 \ --blue-nodata 0 - # TX01_2 + # TX01 python3 manage.py producttype create "${COLLECTION}"_Product_TX01 --traceback \ --coverage-type "sar_hh_gray" \ --coverage-type "sar_hh_vv_gray" \ @@ -791,7 +780,6 @@ if python3 manage.py id check "${COLLECTION}"; then --product-type "${COLLECTION}"_Product_EW03 \ --product-type "${COLLECTION}"_Product_EQ02 \ --product-type "${COLLECTION}"_Product_GE01 \ - --product-type "${COLLECTION}"_Product_GE01_1 \ --product-type "${COLLECTION}"_Product_GY01 \ --product-type "${COLLECTION}"_Product_TX01 \ --product-type "${COLLECTION}"_Product_IK02 \ diff --git a/config/emg_registrar-config.yml b/config/emg_registrar-config.yml index 1c5c7466..faa8c512 100644 --- a/config/emg_registrar-config.yml +++ b/config/emg_registrar-config.yml @@ -117,20 +117,15 @@ backends: GE01: - RGB - BGRNir - GE01_1: # I think that because of browse types this can not be fixed - ~: - product_type_name: !env '${COLLECTION}_Product_GE01_1' - collections: - - !env '${COLLECTION}' - coverages: - GE01_1: grayscale GY01: ~: product_type_name: !env '${COLLECTION}_Product_GY01' collections: - !env '${COLLECTION}' coverages: - GY01: RGBNir + GY01: + - RGB + - BGRNir IK02: ~: product_type_name: !env '${COLLECTION}_Product_IK02' diff --git a/core/registrar/backend.py b/core/registrar/backend.py index a51d3dee..b070498f 100644 --- a/core/registrar/backend.py +++ b/core/registrar/backend.py @@ -107,7 +107,7 @@ class EOxServerBackend(Backend): container, _ = source.get_container_and_path(path) storage, created_storage = backends.Storage.objects.get_or_create( - name=source.name if source.container else f'{source.name}-{container}', + name=source.name if source.container else f'{source.name}{container}', url=container, storage_type='swift', storage_auth=storage_auth, @@ -186,13 +186,8 @@ class EOxServerBackend(Backend): ] if isinstance(coverage_type_name, list): # determine which one to use based on number of bands - gdal_path_prefix = '' - if isinstance(source, SwiftSource): - gdal_path_prefix = '/vsiswift/' - elif isinstance(source, S3Source): - gdal_path_prefix = '/vsis3/' - raster_item_to_check = f'{gdal_path_prefix}{raster_items[0][0]}/{raster_items[0][1]}' - ds = gdal.Open(raster_item_to_check) + _, vsi_path = source.get_vsi_env_and_path(item.raster_files.get(raster_identifier)[0]) + ds = gdal.Open(vsi_path) logger.debug("Raster has number of bands: '%s'" % ds.RasterCount) # find covtype that has correct number of bands try: -- GitLab From af447f88f47cddccef338eca00565f09299ff130 Mon Sep 17 00:00:00 2001 From: Lubomir Bucek Date: Fri, 27 Nov 2020 18:23:16 +0100 Subject: [PATCH 9/9] remove cs00 from preprocessor config --- config/emg_preprocessor-config.yml | 23 ----------------------- 1 file changed, 23 deletions(-) diff --git a/config/emg_preprocessor-config.yml b/config/emg_preprocessor-config.yml index c12bb450..9bb78a30 100644 --- a/config/emg_preprocessor-config.yml +++ b/config/emg_preprocessor-config.yml @@ -235,29 +235,6 @@ preprocessing: DM02: data_file_globs: - "*.tif" - CS00: - data_file_globs: - - "*.h5" - - "*.tif" - subdatasets: - subdataset_types: - '//S01/SBI': 'S01_SBI' - '//S02/SBI': 'S02_SBI' - '//S03/SBI': 'S03_SBI' - '//S04/SBI': 'S04_SBI' - '//S05/SBI': 'S05_SBI' - '//S06/SBI': 'S06_SBI' - '//S01/MBI': 'S01_MBI' - '//S02/MBI': 'S02_MBI' - '//S03/MBI': 'S03_MBI' - '//S04/MBI': 'S04_MBI' - '//S05/MBI': 'S05_MBI' - '//S06/MBI': 'S06_MBI' - georeference: - - type: corners - corner_names: ["S01_SBI_Bottom_Left_Geodetic_Coordinates", "S01_SBI_Bottom_Right_Geodetic_Coordinates", "S01_SBI_Top_Left_Geodetic_Coordinates", "S01_SBI_Top_Right_Geodetic_Coordinates"] - orbit_direction_name: Orbit_Direction - force_north_up: false # this configuration is still a stub - not all product types are done # https://gitlab.eox.at/esa/prism/vs/-/issues/56 # https://gitlab.eox.at/esa/prism/vs/-/issues/23 -- GitLab