diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 801f789aad5bf3bec30fb8b3beb635bf96e5169f..4ad935d8368aa8e6d3d22bfe3936dd63475eed9f 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -28,7 +28,7 @@ replace = version="{new_version}" search = version="{current_version}" replace = version="{new_version}" -[bumpversion:glob:registrar/setup.py] +[bumpversion:glob:core/setup.py] search = version="{current_version}" replace = version="{new_version}" diff --git a/config/emg_init-db.sh b/config/emg_init-db.sh index 739119a119713ef3b190df2a67ca3cb1c54ecfe8..8b5cbc8d9295c2e950265d0b8bf160f5f2c80b60 100644 --- a/config/emg_init-db.sh +++ b/config/emg_init-db.sh @@ -93,10 +93,11 @@ if python3 manage.py id check "${COLLECTION}"; then --red-nodata 0 \ --green-nodata 0 \ --blue-nodata 0 - # EQ02_3 - python3 manage.py producttype create "${COLLECTION}"_Product_EQ02_3 --traceback \ + # EQ02 + python3 manage.py producttype create "${COLLECTION}"_Product_EQ02 --traceback \ + --coverage-type "BGRNir" \ --coverage-type "RGB" - python3 manage.py browsetype create "${COLLECTION}"_Product_EQ02_3 --traceback \ + python3 manage.py browsetype create "${COLLECTION}"_Product_EQ02 --traceback \ --red "red" \ --green "green" \ --blue "blue" \ @@ -106,7 +107,7 @@ if python3 manage.py id check "${COLLECTION}"; then --red-nodata 0 \ --green-nodata 0 \ --blue-nodata 0 - python3 manage.py browsetype create "${COLLECTION}"_Product_EQ02_3 "TRUE_COLOR" --traceback \ + python3 manage.py browsetype create "${COLLECTION}"_Product_EQ02 "TRUE_COLOR" --traceback \ --red "red" \ --green "green" \ --blue "blue" \ @@ -116,42 +117,9 @@ if python3 manage.py id check "${COLLECTION}"; then --red-nodata 0 \ --green-nodata 0 \ --blue-nodata 0 - python3 manage.py browsetype create "${COLLECTION}"_Product_EQ02_3 "FALSE_COLOR" --traceback \ - --red "green" \ - --green "red" \ - --blue "blue" \ - --red-range 0 600 \ - --green-range 0 500 \ - --blue-range 0 500 \ - --red-nodata 0 \ - --green-nodata 0 \ - --blue-nodata 0 - # EQ02_4 - python3 manage.py producttype create "${COLLECTION}"_Product_EQ02_4 --traceback \ - --coverage-type "RGBNir" - python3 manage.py browsetype create "${COLLECTION}"_Product_EQ02_4 --traceback \ - --red "blue" \ - --green "green" \ - --blue "red" \ - --red-range 0 500 \ - --green-range 0 600 \ - --blue-range 0 500 \ - --red-nodata 0 \ - --green-nodata 0 \ - --blue-nodata 0 - python3 manage.py browsetype create "${COLLECTION}"_Product_EQ02_4 "TRUE_COLOR" --traceback \ - --red "blue" \ - --green "green" \ - --blue "red" \ - --red-range 0 500 \ - --green-range 0 600 \ - --blue-range 0 500 \ - --red-nodata 0 \ - --green-nodata 0 \ - --blue-nodata 0 - python3 manage.py browsetype create "${COLLECTION}"_Product_EQ02_4 "FALSE_COLOR" --traceback \ + python3 manage.py browsetype create "${COLLECTION}"_Product_EQ02 "FALSE_COLOR" --traceback \ --red "nir" \ - --green "blue" \ + --green "red" \ --blue "green" \ --red-range 0 700 \ --green-range 0 500 \ @@ -159,8 +127,8 @@ if python3 manage.py id check "${COLLECTION}"; then --red-nodata 0 \ --green-nodata 0 \ --blue-nodata 0 - python3 manage.py browsetype create "${COLLECTION}"_Product_EQ02_4 "NDVI" --traceback \ - --grey "(nir-blue)/(nir+blue)" --grey-range -1 1 + python3 manage.py browsetype create "${COLLECTION}"_Product_EQ02 "NDVI" --traceback \ + --grey "(nir-red)/(nir+red)" --grey-range -1 1 # EW01 python3 manage.py producttype create "${COLLECTION}"_Product_EW01 --traceback \ --coverage-type "grayscale" @@ -172,238 +140,106 @@ if python3 manage.py id check "${COLLECTION}"; then --red "gray" \ --red-range 0 1500 \ --red-nodata 0 - # EW02_3 - python3 manage.py producttype create "${COLLECTION}"_Product_EW02_3 --traceback \ - --coverage-type "RGB" - python3 manage.py browsetype create "${COLLECTION}"_Product_EW02_3 --traceback \ + # EW02 + python3 manage.py producttype create "${COLLECTION}"_Product_EW02 --traceback \ + --coverage-type "BGRNir" \ + --coverage-type "RGB" \ + --coverage-type "CBGYRReNirNir2" + python3 manage.py browsetype create "${COLLECTION}"_Product_EW02 --traceback \ --red "red" \ --green "green" \ --blue "blue" \ - --red-range 0 750 \ - --green-range 0 1200 \ - --blue-range 0 800 \ + --red-range 100 700 \ + --green-range 100 600 \ + --blue-range 100 700 \ --red-nodata 0 \ --green-nodata 0 \ --blue-nodata 0 - python3 manage.py browsetype create "${COLLECTION}"_Product_EW02_3 "TRUE_COLOR" --traceback \ + python3 manage.py browsetype create "${COLLECTION}"_Product_EW02 "TRUE_COLOR" --traceback \ --red "red" \ --green "green" \ --blue "blue" \ - --red-range 0 750 \ - --green-range 0 1200 \ - --blue-range 0 800 \ - --red-nodata 0 \ - --green-nodata 0 \ - --blue-nodata 0 - python3 manage.py browsetype create "${COLLECTION}"_Product_EW02_3 "FALSE_COLOR" --traceback \ - --red "green" \ - --green "red" \ - --blue "blue" \ - --red-range 0 1200 \ - --green-range 0 750 \ - --blue-range 0 800 \ + --red-range 100 700 \ + --green-range 100 600 \ + --blue-range 100 700 \ --red-nodata 0 \ --green-nodata 0 \ --blue-nodata 0 - # EW02_4 - python3 manage.py producttype create "${COLLECTION}"_Product_EW02_4 --traceback \ - --coverage-type "RGBNir" - python3 manage.py browsetype create "${COLLECTION}"_Product_EW02_4 --traceback \ - --red "blue" \ - --green "green" \ - --blue "red" \ - --red-range 0 500 \ - --green-range 0 650 \ - --blue-range 0 500 \ - --red-nodata 0 \ - --green-nodata 0 \ - --blue-nodata 0 - python3 manage.py browsetype create "${COLLECTION}"_Product_EW02_4 "TRUE_COLOR" --traceback \ - --red "blue" \ - --green "green" \ - --blue "red" \ - --red-range 0 500 \ - --green-range 0 650 \ - --blue-range 0 500 \ - --red-nodata 0 \ - --green-nodata 0 \ - --blue-nodata 0 - python3 manage.py browsetype create "${COLLECTION}"_Product_EW02_4 "FALSE_COLOR" --traceback \ + python3 manage.py browsetype create "${COLLECTION}"_Product_EW02 "FALSE_COLOR" --traceback \ --red "nir" \ - --green "blue" \ + --green "red" \ --blue "green" \ - --red-range 0 700 \ - --green-range 0 500 \ - --blue-range 0 650 \ + --red-range 100 700 \ + --green-range 100 600 \ + --blue-range 100 700 \ --red-nodata 0 \ --green-nodata 0 \ --blue-nodata 0 - python3 manage.py browsetype create "${COLLECTION}"_Product_EW02_4 "NDVI" --traceback \ + python3 manage.py browsetype create "${COLLECTION}"_Product_EW02 "NDVI" --traceback \ --grey "(nir-blue)/(nir+blue)" --grey-range -1 1 - # EW02_8 - python3 manage.py producttype create "${COLLECTION}"_Product_EW02_8 --traceback \ + # EW03 + python3 manage.py producttype create "${COLLECTION}"_Product_EW03 --traceback \ + --coverage-type "BGRNir" \ + --coverage-type "RGB" \ --coverage-type "CBGYRReNirNir2" - python3 manage.py browsetype create "${COLLECTION}"_Product_EW02_8 --traceback \ - --red "red" \ - --green "green" \ - --blue "blue" \ - --red-range 150 750 \ - --green-range 250 800 \ - --blue-range 200 600 \ - --red-nodata 0 \ - --green-nodata 0 \ - --blue-nodata 0 - python3 manage.py browsetype create "${COLLECTION}"_Product_EW02_8 "TRUE_COLOR" --traceback \ + python3 manage.py browsetype create "${COLLECTION}"_Product_EW03 --traceback \ --red "red" \ --green "green" \ --blue "blue" \ - --red-range 150 750 \ - --green-range 250 800 \ - --blue-range 200 600 \ + --red-range 100 700 \ + --green-range 100 600 \ + --blue-range 100 700 \ --red-nodata 0 \ --green-nodata 0 \ --blue-nodata 0 - python3 manage.py browsetype create "${COLLECTION}"_Product_EW02_8 "FALSE_COLOR" --traceback \ - --red "nir_2" \ - --green "red" \ - --blue "green" \ - --red-range 250 1000 \ - --green-range 250 800 \ - --blue-range 200 650 \ - --red-nodata 0 \ - --green-nodata 0 \ - --blue-nodata 0 - python3 manage.py browsetype create "${COLLECTION}"_Product_EW02_8 "NDVI" --traceback \ - --grey "(nir_2-red)/(nir_2+red)" --grey-range -1 1 - # EW03_3 - python3 manage.py producttype create "${COLLECTION}"_Product_EW03_3 --traceback \ - --coverage-type "RGB" - python3 manage.py browsetype create "${COLLECTION}"_Product_EW03_3 --traceback \ - --red "red" \ - --green "green" \ - --blue "blue" \ - --red-range 0 750 \ - --green-range 0 1200 \ - --blue-range 0 800 \ - --red-nodata 0 \ - --green-nodata 0 \ - --blue-nodata 0 - python3 manage.py browsetype create "${COLLECTION}"_Product_EW03_3 "TRUE_COLOR" --traceback \ + python3 manage.py browsetype create "${COLLECTION}"_Product_EW03 "TRUE_COLOR" --traceback \ --red "red" \ --green "green" \ --blue "blue" \ - --red-range 0 750 \ - --green-range 0 1200 \ - --blue-range 0 800 \ - --red-nodata 0 \ - --green-nodata 0 \ - --blue-nodata 0 - python3 manage.py browsetype create "${COLLECTION}"_Product_EW03_3 "FALSE_COLOR" --traceback \ - --red "green" \ - --green "red" \ - --blue "blue" \ - --red-range 0 1200 \ - --green-range 0 750 \ - --blue-range 0 800 \ - --red-nodata 0 \ - --green-nodata 0 \ - --blue-nodata 0 - # EW03_4 - python3 manage.py producttype create "${COLLECTION}"_Product_EW03_4 --traceback \ - --coverage-type "RGBNir" - python3 manage.py browsetype create "${COLLECTION}"_Product_EW03_4 --traceback \ - --red "blue" \ - --green "green" \ - --blue "red" \ - --red-range 0 500 \ - --green-range 0 650 \ - --blue-range 0 500 \ - --red-nodata 0 \ - --green-nodata 0 \ - --blue-nodata 0 - python3 manage.py browsetype create "${COLLECTION}"_Product_EW03_4 "TRUE_COLOR" --traceback \ - --red "blue" \ - --green "green" \ - --blue "red" \ - --red-range 0 500 \ - --green-range 0 650 \ - --blue-range 0 500 \ + --red-range 100 700 \ + --green-range 100 600 \ + --blue-range 100 700 \ --red-nodata 0 \ --green-nodata 0 \ --blue-nodata 0 - python3 manage.py browsetype create "${COLLECTION}"_Product_EW03_4 "FALSE_COLOR" --traceback \ + python3 manage.py browsetype create "${COLLECTION}"_Product_EW03 "FALSE_COLOR" --traceback \ --red "nir" \ - --green "blue" \ + --green "red" \ --blue "green" \ - --red-range 0 700 \ - --green-range 0 500 \ - --blue-range 0 650 \ + --red-range 100 700 \ + --green-range 100 600 \ + --blue-range 100 700 \ --red-nodata 0 \ --green-nodata 0 \ --blue-nodata 0 - python3 manage.py browsetype create "${COLLECTION}"_Product_EW03_4 "NDVI" --traceback \ + python3 manage.py browsetype create "${COLLECTION}"_Product_EW03 "NDVI" --traceback \ --grey "(nir-blue)/(nir+blue)" --grey-range -1 1 - # EW03_8 - python3 manage.py producttype create "${COLLECTION}"_Product_EW03_8 --traceback \ - --coverage-type "CBGYRReNirNir2" - python3 manage.py browsetype create "${COLLECTION}"_Product_EW03_8 --traceback \ - --red "red" \ - --green "green" \ - --blue "blue" \ - --red-range 150 750 \ - --green-range 250 800 \ - --blue-range 200 600 \ - --red-nodata 0 \ - --green-nodata 0 \ - --blue-nodata 0 - python3 manage.py browsetype create "${COLLECTION}"_Product_EW03_8 "TRUE_COLOR" --traceback \ + python3 manage.py producttype create "${COLLECTION}"_Product_GE01 --traceback \ + --coverage-type "BGRNir" \ + --coverage-type "RGB" + python3 manage.py browsetype create "${COLLECTION}"_Product_GE01 --traceback \ --red "red" \ --green "green" \ --blue "blue" \ - --red-range 150 750 \ - --green-range 250 800 \ - --blue-range 200 600 \ - --red-nodata 0 \ - --green-nodata 0 \ - --blue-nodata 0 - python3 manage.py browsetype create "${COLLECTION}"_Product_EW03_8 "FALSE_COLOR" --traceback \ - --red "nir_2" \ - --green "red" \ - --blue "green" \ - --red-range 250 1000 \ - --green-range 250 800 \ - --blue-range 200 650 \ - --red-nodata 0 \ - --green-nodata 0 \ - --blue-nodata 0 - python3 manage.py browsetype create "${COLLECTION}"_Product_EW03_8 "NDVI" --traceback \ - --grey "(nir_2-red)/(nir_2+red)" --grey-range -1 1 - # GE01_4 - python3 manage.py producttype create "${COLLECTION}"_Product_GE01_4 --traceback \ - --coverage-type "RGBNir" - python3 manage.py browsetype create "${COLLECTION}"_Product_GE01_4 --traceback \ - --red "blue" \ - --green "green" \ - --blue "red" \ --red-range 0 600 \ --green-range 0 700 \ --blue-range 0 800 \ --red-nodata 0 \ --green-nodata 0 \ --blue-nodata 0 - python3 manage.py browsetype create "${COLLECTION}"_Product_GE01_4 "TRUE_COLOR" --traceback \ - --red "blue" \ + python3 manage.py browsetype create "${COLLECTION}"_Product_GE01 "TRUE_COLOR" --traceback \ + --red "red" \ --green "green" \ - --blue "red" \ + --blue "blue" \ --red-range 0 600 \ --green-range 0 700 \ --blue-range 0 800 \ --red-nodata 0 \ --green-nodata 0 \ --blue-nodata 0 - python3 manage.py browsetype create "${COLLECTION}"_Product_GE01_4 "FALSE_COLOR" --traceback \ + python3 manage.py browsetype create "${COLLECTION}"_Product_GE01 "FALSE_COLOR" --traceback \ --red "nir" \ - --green "blue" \ + --green "red" \ --blue "green" \ --red-range 0 1000 \ --green-range 0 600 \ @@ -411,87 +247,44 @@ if python3 manage.py id check "${COLLECTION}"; then --red-nodata 0 \ --green-nodata 0 \ --blue-nodata 0 - python3 manage.py browsetype create "${COLLECTION}"_Product_GE01_4 "NDVI" --traceback \ - --grey "(nir-blue)/(nir+blue)" --grey-range -1 1 - # GE01_3 - python3 manage.py producttype create "${COLLECTION}"_Product_GE01_3 --traceback \ - --coverage-type "RGB" - python3 manage.py browsetype create "${COLLECTION}"_Product_GE01_3 --traceback \ - --red "red" \ - --green "green" \ - --blue "blue" \ - --red-range 1000 2048 \ - --green-range 1000 2048 \ - --blue-range 1000 2048 \ - --red-nodata 0 \ - --green-nodata 0 \ - --blue-nodata 0 - python3 manage.py browsetype create "${COLLECTION}"_Product_GE01_3 "TRUE_COLOR" --traceback \ - --red "red" \ - --green "green" \ - --blue "blue" \ - --red-range 1000 1900 \ - --green-range 1000 1900 \ - --blue-range 1000 1900 \ - --red-nodata 0 \ - --green-nodata 0 \ - --blue-nodata 0 - python3 manage.py browsetype create "${COLLECTION}"_Product_GE01_3 "FALSE_COLOR" --traceback \ - --red "green" \ - --green "red" \ - --blue "blue" \ - --red-range 1000 1900 \ - --green-range 1000 1900 \ - --blue-range 1000 1900 \ - --red-nodata 0 \ - --green-nodata 0 \ - --blue-nodata 0 - # GE01_1 - python3 manage.py producttype create "${COLLECTION}"_Product_GE01_1 --traceback \ - --coverage-type "grayscale" - python3 manage.py browsetype create "${COLLECTION}"_Product_GE01_1 --traceback \ - --red "gray" \ - --red-range 0 2000 \ - --red-nodata 0 - python3 manage.py browsetype create "${COLLECTION}"_Product_GE01_1 "TRUE_COLOR" --traceback \ - --red "gray" \ - --red-range 0 2000 \ - --red-nodata 0 + python3 manage.py browsetype create "${COLLECTION}"_Product_GE01 "NDVI" --traceback \ + --grey "(nir-red)/(nir+red)" --grey-range -1 1 # GY01 python3 manage.py producttype create "${COLLECTION}"_Product_GY01 --traceback \ - --coverage-type "RGBNir" + --coverage-type "BGRNir" \ + --coverage-type "RGB" python3 manage.py browsetype create "${COLLECTION}"_Product_GY01 --traceback \ - --red "blue" \ + --red "red" \ --green "green" \ - --blue "red" \ - --red-range 50 450 \ - --green-range 50 800 \ + --blue "blue" \ + --red-range 50 600 \ + --green-range 50 600 \ --blue-range 50 600 \ --red-nodata 0 \ --green-nodata 0 \ --blue-nodata 0 python3 manage.py browsetype create "${COLLECTION}"_Product_GY01 "TRUE_COLOR" --traceback \ - --red "blue" \ + --red "red" \ --green "green" \ - --blue "red" \ - --red-range 50 450 \ - --green-range 50 800 \ + --blue "blue" \ + --red-range 50 600 \ + --green-range 50 600 \ --blue-range 50 600 \ --red-nodata 0 \ --green-nodata 0 \ --blue-nodata 0 python3 manage.py browsetype create "${COLLECTION}"_Product_GY01 "FALSE_COLOR" --traceback \ --red "nir" \ - --green "blue" \ + --green "red" \ --blue "green" \ - --red-range 50 1000 \ - --green-range 50 450 \ - --blue-range 50 800 \ + --red-range 50 800 \ + --green-range 50 600 \ + --blue-range 50 600 \ --red-nodata 0 \ --green-nodata 0 \ --blue-nodata 0 python3 manage.py browsetype create "${COLLECTION}"_Product_GY01 "NDVI" --traceback \ - --grey "(nir-blue)/(nir+blue)" --grey-range -1 1 + --grey "(nir-red)/(nir+red)" --grey-range -1 1 # IK02 python3 manage.py producttype create "${COLLECTION}"_Product_IK02 --traceback \ --coverage-type "RGBNir" @@ -562,51 +355,20 @@ if python3 manage.py id check "${COLLECTION}"; then --blue-nodata 0 python3 manage.py browsetype create "${COLLECTION}"_Product_RE00 "NDVI" --traceback \ --grey "(nir-red)/(nir+red)" --grey-range -1 1 - # RS02_2 - python3 manage.py producttype create "${COLLECTION}"_Product_RS02_2 --traceback \ - --coverage-type "sar_hh_gray" - python3 manage.py browsetype create "${COLLECTION}"_Product_RS02_2 --traceback \ - --red "gray" \ - --red-range 1 65535 \ - --red-nodata 0 - python3 manage.py browsetype create "${COLLECTION}"_Product_RS02_2 "TRUE_COLOR" --traceback \ - --red "gray" \ - --red-range 1 65535 \ - --red-nodata 0 - # RS02_3 - python3 manage.py producttype create "${COLLECTION}"_Product_RS02_3 --traceback \ - --coverage-type "sar_hh_vv_gray" - python3 manage.py browsetype create "${COLLECTION}"_Product_RS02_3 --traceback \ + # RS02 + python3 manage.py producttype create "${COLLECTION}"_Product_RS02 --traceback \ + --coverage-type "sar_hh_gray" \ + --coverage-type "sar_hh_vv_gray" \ + --coverage-type "sar_hh_hv_vh_vv_gray" + + python3 manage.py browsetype create "${COLLECTION}"_Product_RS02 --traceback \ --red "gray" \ --red-range 1 65535 \ --red-nodata 0 - python3 manage.py browsetype create "${COLLECTION}"_Product_RS02_3 "TRUE_COLOR" --traceback \ + python3 manage.py browsetype create "${COLLECTION}"_Product_RS02 "TRUE_COLOR" --traceback \ --red "gray" \ --red-range 1 65535 \ --red-nodata 0 - # RS02_7 - python3 manage.py producttype create "${COLLECTION}"_Product_RS02_7 --traceback \ - --coverage-type "sar_hh_hv_vh_vv_rgb" - python3 manage.py browsetype create "${COLLECTION}"_Product_RS02_7 --traceback \ - --red "red" \ - --red-range 1 65535 \ - --red-nodata 0 - python3 manage.py browsetype create "${COLLECTION}"_Product_RS02_7 "TRUE_COLOR" --traceback \ - --red "red" \ - --red-range 1 65535 \ - --red-nodata 0 - python3 manage.py browsetype create "${COLLECTION}"_Product_RS02_7 "FALSE_COLOR" --traceback \ - --red "red" \ - --green "green" \ - --blue "blue" \ - --red-range 1 65535 \ - --green-range 1 65535 \ - --blue-range 1 65535 \ - --red-nodata 0 \ - --green-nodata 0 \ - --blue-nodata 0 - python3 manage.py browsetype create "${COLLECTION}"_Product_RS02_7 "NDVI" --traceback \ - --grey "(vh-vv)/(vh+vv)" --grey-range -1 1 # SP04 python3 manage.py producttype create "${COLLECTION}"_Product_SP04 --traceback \ --coverage-type "RGBNirByte" @@ -665,51 +427,19 @@ if python3 manage.py id check "${COLLECTION}"; then --red-nodata 0 \ --green-nodata 0 \ --blue-nodata 0 - # TX01_2 - python3 manage.py producttype create "${COLLECTION}"_Product_TX01_2 --traceback \ - --coverage-type "sar_hh_gray" - python3 manage.py browsetype create "${COLLECTION}"_Product_TX01_2 --traceback \ - --red "gray" \ - --red-range 1 65535 \ - --red-nodata 0 - python3 manage.py browsetype create "${COLLECTION}"_Product_TX01_2 "TRUE_COLOR" --traceback \ - --red "gray" \ - --red-range 1 65535 \ - --red-nodata 0 - # TX01_3 - python3 manage.py producttype create "${COLLECTION}"_Product_TX01_3 --traceback \ - --coverage-type "sar_hh_vv_gray" - python3 manage.py browsetype create "${COLLECTION}"_Product_TX01_3 --traceback \ + # TX01 + python3 manage.py producttype create "${COLLECTION}"_Product_TX01 --traceback \ + --coverage-type "sar_hh_gray" \ + --coverage-type "sar_hh_vv_gray" \ + --coverage-type "sar_hh_hv_vh_vv_gray" + python3 manage.py browsetype create "${COLLECTION}"_Product_TX01 --traceback \ --red "gray" \ --red-range 1 65535 \ --red-nodata 0 - python3 manage.py browsetype create "${COLLECTION}"_Product_TX01_3 "TRUE_COLOR" --traceback \ + python3 manage.py browsetype create "${COLLECTION}"_Product_TX01 "TRUE_COLOR" --traceback \ --red "gray" \ --red-range 1 65535 \ --red-nodata 0 - # TX01_7 - python3 manage.py producttype create "${COLLECTION}"_Product_TX01_7 --traceback \ - --coverage-type "sar_hh_hv_vh_vv_rgb" - python3 manage.py browsetype create "${COLLECTION}"_Product_TX01_7 --traceback \ - --red "red" \ - --red-range 1 65535 \ - --red-nodata 0 - python3 manage.py browsetype create "${COLLECTION}"_Product_TX01_7 "TRUE_COLOR" --traceback \ - --red "red" \ - --red-range 1 65535 \ - --red-nodata 0 - python3 manage.py browsetype create "${COLLECTION}"_Product_TX01_7 "FALSE_COLOR" --traceback \ - --red "red" \ - --green "green" \ - --blue "blue" \ - --red-range 1 65535 \ - --green-range 1 65535 \ - --blue-range 1 65535 \ - --red-nodata 0 \ - --green-nodata 0 \ - --blue-nodata 0 - python3 manage.py browsetype create "${COLLECTION}"_Product_TX01_7 "NDVI" --traceback \ - --grey "(vh-vv)/(vh+vv)" --grey-range -1 1 # DM02 python3 manage.py producttype create "${COLLECTION}"_Product_DM02 --traceback \ --coverage-type "RGBNir" @@ -885,7 +615,6 @@ if python3 manage.py id check "${COLLECTION}"; then --blue-nodata 0 python3 manage.py browsetype create "${COLLECTION}"_Product_PH1A "NDVI" --traceback \ --grey "(nir-red)/(nir+red)" --grey-range -1 1 - # PH1B python3 manage.py producttype create "${COLLECTION}"_Product_PH1B --traceback \ --coverage-type "RGBNir" @@ -921,6 +650,7 @@ if python3 manage.py id check "${COLLECTION}"; then --blue-nodata 0 python3 manage.py browsetype create "${COLLECTION}"_Product_PH1B "NDVI" --traceback \ --grey "(nir-red)/(nir+red)" --grey-range -1 1 + # SK00 python3 manage.py producttype create "${COLLECTION}"_Product_SK00 --traceback \ --coverage-type "RGBNir" python3 manage.py browsetype create "${COLLECTION}"_Product_SK00 --traceback \ @@ -955,6 +685,7 @@ if python3 manage.py id check "${COLLECTION}"; then --blue-nodata 0 python3 manage.py browsetype create "${COLLECTION}"_Product_SK00 "NDVI" --traceback \ --grey "(nir-red)/(nir+red)" --grey-range -1 1 + # SW03 python3 manage.py producttype create "${COLLECTION}"_Product_SW03 --traceback \ --coverage-type "RGBNir" python3 manage.py browsetype create "${COLLECTION}"_Product_SW03 --traceback \ @@ -989,7 +720,7 @@ if python3 manage.py id check "${COLLECTION}"; then --blue-nodata 0 python3 manage.py browsetype create "${COLLECTION}"_Product_SW03 "NDVI" --traceback \ --grey "(nir-red)/(nir+red)" --grey-range -1 1 - # bands go in order b,g,r,nir and I did not change them in preprocessing + # PL00 bands go in order b,g,r,nir python3 manage.py producttype create "${COLLECTION}"_Product_PL00 --traceback \ --coverage-type "RGBNir" python3 manage.py browsetype create "${COLLECTION}"_Product_PL00 --traceback \ @@ -1034,7 +765,7 @@ if python3 manage.py id check "${COLLECTION}"; then --coverage-type "BGRReNir" \ --coverage-type "sar_hh_gray" \ --coverage-type "sar_hh_vv_gray" \ - --coverage-type "sar_hh_hv_vh_vv_rgb" \ + --coverage-type "sar_hh_hv_vh_vv_gray" \ --product-type "${COLLECTION}"_Product_PH1A \ --product-type "${COLLECTION}"_Product_PH1B \ --product-type "${COLLECTION}"_Product_DM02 \ @@ -1045,26 +776,15 @@ if python3 manage.py id check "${COLLECTION}"; then --product-type "${COLLECTION}"_Product_CS03 \ --product-type "${COLLECTION}"_Product_CS04 \ --product-type "${COLLECTION}"_Product_EW01 \ - --product-type "${COLLECTION}"_Product_EW02_3 \ - --product-type "${COLLECTION}"_Product_EW02_4 \ - --product-type "${COLLECTION}"_Product_EW02_8 \ - --product-type "${COLLECTION}"_Product_EW03_3 \ - --product-type "${COLLECTION}"_Product_EW03_4 \ - --product-type "${COLLECTION}"_Product_EW03_8 \ - --product-type "${COLLECTION}"_Product_EQ02_3 \ - --product-type "${COLLECTION}"_Product_EQ02_4 \ - --product-type "${COLLECTION}"_Product_GE01_4 \ - --product-type "${COLLECTION}"_Product_GE01_3 \ - --product-type "${COLLECTION}"_Product_GE01_1 \ + --product-type "${COLLECTION}"_Product_EW02 \ + --product-type "${COLLECTION}"_Product_EW03 \ + --product-type "${COLLECTION}"_Product_EQ02 \ + --product-type "${COLLECTION}"_Product_GE01 \ --product-type "${COLLECTION}"_Product_GY01 \ - --product-type "${COLLECTION}"_Product_TX01_2 \ - --product-type "${COLLECTION}"_Product_TX01_3 \ - --product-type "${COLLECTION}"_Product_TX01_7 \ + --product-type "${COLLECTION}"_Product_TX01 \ --product-type "${COLLECTION}"_Product_IK02 \ --product-type "${COLLECTION}"_Product_RE00 \ - --product-type "${COLLECTION}"_Product_RS02_2 \ - --product-type "${COLLECTION}"_Product_RS02_3 \ - --product-type "${COLLECTION}"_Product_RS02_7 \ + --product-type "${COLLECTION}"_Product_RS02 \ --product-type "${COLLECTION}"_Product_SP04 \ --product-type "${COLLECTION}"_Product_SP05 \ --product-type "${COLLECTION}"_Product_SP06 \ diff --git a/config/emg_preprocessor-config.yml b/config/emg_preprocessor-config.yml index 3ed463273e8f77b7f9ae1a3dde9b15b6a6e8ac49..9bb78a30b8e6e31b00127f424e7096a9df5b0294 100644 --- a/config/emg_preprocessor-config.yml +++ b/config/emg_preprocessor-config.yml @@ -63,10 +63,10 @@ preprocessing: group_by: ".*/(.*)_P..tif" sort_by: ".*_P(R|G|B|N).tif" order: - - R - - G - B + - G - N + - R RS02: data_file_globs: - "*imagery_*.tif" @@ -166,12 +166,14 @@ preprocessing: - "*IMG_*_PMS_*.tif" - "*IMG_*_MS_*.JP2" - "*IMG_*_MS_*.tif" + - "*.vrt" additional_file_globs: - "*RPC_*" - "*DIM_*" georeference: geotransforms: - type: rpc + - type: fix_geotrans - type: no_op PH1B: data_file_globs: @@ -179,12 +181,14 @@ preprocessing: - "*IMG_*_PMS_*.tif" - "*IMG_*_MS_*.JP2" - "*IMG_*_MS_*.tif" + - "*.vrt" additional_file_globs: - "*RPC_*" - "*DIM_*" georeference: geotransforms: - type: rpc + - type: fix_geotrans - type: no_op PL00: data_file_globs: diff --git a/config/emg_registrar-config.yml b/config/emg_registrar-config.yml index dd7acf4b05fc8f50f7a07b9c689a3b4a4fbfb52c..faa8c5120f03ac418fb73aff901e372dbca63e50 100644 --- a/config/emg_registrar-config.yml +++ b/config/emg_registrar-config.yml @@ -72,20 +72,15 @@ backends: - !env '${COLLECTION}' coverages: DM02: RGBNir - EQ02_3: + EQ02: ~: - product_type_name: !env '${COLLECTION}_Product_EQ02_3' + product_type_name: !env '${COLLECTION}_Product_EQ02' collections: - !env '${COLLECTION}' coverages: - EQ02_3: RGB - EQ02_4: - ~: - product_type_name: !env '${COLLECTION}_Product_EQ02_4' - collections: - - !env '${COLLECTION}' - coverages: - EQ02_4: RGBNir + EQ02: + - RGB + - BGRNir EW01: ~: product_type_name: !env '${COLLECTION}_Product_EW01' @@ -93,76 +88,44 @@ backends: - !env '${COLLECTION}' coverages: EW01: grayscale - EW02_3: - ~: - product_type_name: !env '${COLLECTION}_Product_EW02_3' - collections: - - !env '${COLLECTION}' - coverages: - EW02_3: RGB - EW02_4: - ~: - product_type_name: !env '${COLLECTION}_Product_EW02_4' - collections: - - !env '${COLLECTION}' - coverages: - EW02_4: RGBNir - EW02_8: - ~: - product_type_name: !env '${COLLECTION}_Product_EW02_8' - collections: - - !env '${COLLECTION}' - coverages: - EW02_8: CBGYRReNirNir2 - EW03_3: - ~: - product_type_name: !env '${COLLECTION}_Product_EW03_3' - collections: - - !env '${COLLECTION}' - coverages: - EW03_3: RGB - EW03_4: - ~: - product_type_name: !env '${COLLECTION}_Product_EW03_4' - collections: - - !env '${COLLECTION}' - coverages: - EW03_4: RGBNir - EW03_8: - ~: - product_type_name: !env '${COLLECTION}_Product_EW03_8' - collections: - - !env '${COLLECTION}' - coverages: - EW03_8: CBGYRReNirNir2 - GE01_4: + EW02: ~: - product_type_name: !env '${COLLECTION}_Product_GE01_4' + product_type_name: !env '${COLLECTION}_Product_EW02' collections: - !env '${COLLECTION}' coverages: - GE01_4: RGBNir - GE01_3: + EW02: + - RGB + - BGRNir + - CBGYRReNirNir2 + EW03: ~: - product_type_name: !env '${COLLECTION}_Product_GE01_3' + product_type_name: !env '${COLLECTION}_Product_EW03' collections: - !env '${COLLECTION}' coverages: - GE01_3: RGB - GE01_1: + EW03: + - RGB + - BGRNir + - CBGYRReNirNir2 + GE01: ~: - product_type_name: !env '${COLLECTION}_Product_GE01_1' + product_type_name: !env '${COLLECTION}_Product_GE01' collections: - !env '${COLLECTION}' coverages: - GE01_1: grayscale + GE01: + - RGB + - BGRNir GY01: ~: product_type_name: !env '${COLLECTION}_Product_GY01' collections: - !env '${COLLECTION}' coverages: - GY01: RGBNir + GY01: + - RGB + - BGRNir IK02: ~: product_type_name: !env '${COLLECTION}_Product_IK02' @@ -198,27 +161,16 @@ backends: - !env '${COLLECTION}' coverages: RE00: BGRReNir - RS02_2: - ~: - product_type_name: !env '${COLLECTION}_Product_RS02_2' - collections: - - !env '${COLLECTION}' - coverages: - RS02_2: sar_hh_gray - RS02_3: - ~: - product_type_name: !env '${COLLECTION}_Product_RS02_3' - collections: - - !env '${COLLECTION}' - coverages: - RS02_3: sar_hh_vv_gray - RS02_7: + RS02: ~: - product_type_name: !env '${COLLECTION}_Product_RS02_7' + product_type_name: !env '${COLLECTION}_Product_RS02' collections: - !env '${COLLECTION}' coverages: - RS02_7: sar_hh_hv_vh_vv_rgb + RS02: + - sar_hh_gray + - sar_hh_vv_gray + - sar_hh_hv_vh_vv_gray SP04: ~: product_type_name: !env '${COLLECTION}_Product_SP04' @@ -268,27 +220,16 @@ backends: - !env '${COLLECTION}' coverages: PL00: RGBNir - TX01_2: - ~: - product_type_name: !env '${COLLECTION}_Product_TX01_2' - collections: - - !env '${COLLECTION}' - coverages: - TX01_2: sar_hh_gray - TX01_3: - ~: - product_type_name: !env '${COLLECTION}_Product_TX01_3' - collections: - - !env '${COLLECTION}' - coverages: - TX01_3: sar_hh_vv_gray - TX01_7: + TX01: ~: - product_type_name: !env '${COLLECTION}_Product_TX01_7' + product_type_name: !env '${COLLECTION}_Product_TX01' collections: - !env '${COLLECTION}' coverages: - TX01_7: sar_hh_hv_vh_vv_rgb + TX01: + - sar_hh_gray + - sar_hh_vv_gray + - sar_hh_hv_vh_vv_gray post_handlers: - path: registrar.post_handlers.ReportingPostHandler diff --git a/core/registrar/backend.py b/core/registrar/backend.py index 056017e04464616e16b5520c1a98db0b9ce6b5af..b070498f519e08890f212b3e0270c4388ef2c93a 100644 --- a/core/registrar/backend.py +++ b/core/registrar/backend.py @@ -7,7 +7,9 @@ import json import django from django.db import transaction +from django.db.models import Count from django.contrib.gis.geos import GEOSGeometry, Polygon +from osgeo import gdal from .exceptions import RegistrationError from .context import Context @@ -105,7 +107,7 @@ class EOxServerBackend(Backend): container, _ = source.get_container_and_path(path) storage, created_storage = backends.Storage.objects.get_or_create( - name=source.name if source.container else f'{source.name}-{container}', + name=source.name if source.container else f'{source.name}{container}', url=container, storage_type='swift', storage_auth=storage_auth, @@ -129,14 +131,16 @@ class EOxServerBackend(Backend): from eoxserver.resources.coverages.registration.registrators.gdal import GDALRegistrator # get the mapping for this particular item - type_mapping = self.mapping[item.product_type] + try: + type_mapping = self.mapping[item.product_type] + except KeyError: + raise RegistrationError(f'Could not find mapping for {item.product_type} in configuration') mapping = type_mapping.get(item.product_level) or type_mapping.get(None) if not mapping: raise RegistrationError(f'Could not get mapping for {item.product_type} {item.product_level}') _, metadata_file = source.get_container_and_path(item.metadata_files[0]) - # metadata_file = '/'.join(item.metadata_files[0].split('/')[1:]) storage = self._get_storage_from_source(source, item.path) try: @@ -180,9 +184,21 @@ class EOxServerBackend(Backend): storage + [source.get_container_and_path(raster_item)[1]] for raster_item in (raster_items if isinstance(raster_items, list) else [raster_items]) ] - + if isinstance(coverage_type_name, list): + # determine which one to use based on number of bands + _, vsi_path = source.get_vsi_env_and_path(item.raster_files.get(raster_identifier)[0]) + ds = gdal.Open(vsi_path) + logger.debug("Raster has number of bands: '%s'" % ds.RasterCount) + # find covtype that has correct number of bands + try: + coverage_model = models.CoverageType.objects \ + .filter(name__in=coverage_type_name) \ + .annotate(field_count=Count('field_types')) \ + .get(field_count=ds.RasterCount) + except models.CoverageType.DoesNotExist: + raise RegistrationError(f'Coverage Band Count is different than in any of configured CoverageTypes') + coverage_type_name = coverage_model.name logger.info(f"Registering coverage{'s' if len(raster_items) > 1 else ''} {raster_items} as {coverage_type_name}") - report = GDALRegistrator().register( data_locations=raster_items, metadata_locations=[storage + [metadata_file]], diff --git a/core/registrar/source.py b/core/registrar/source.py index ce04366251c1c3711d79e0c010ceba9dcf118403..53b56ee510fd37e01ae7abf3279eeef7976366ab 100644 --- a/core/registrar/source.py +++ b/core/registrar/source.py @@ -67,7 +67,7 @@ class SwiftSource(Source): def get_container_and_path(self, path: str): container = self.container - if container is None: + if container is None or container == '': parts = (path[1:] if path.startswith('/') else path).split('/') container, path = parts[0], '/'.join(parts[1:]) diff --git a/core/rgbnir_definition.json b/core/rgbnir_definition.json index 6dc31cdc8a0b68a67a8de748314d1af3be30712a..ad8be6b63f544502a8bd114857511a4b03eb8e68 100644 --- a/core/rgbnir_definition.json +++ b/core/rgbnir_definition.json @@ -469,46 +469,10 @@ }, { "definition": "http://www.opengis.net/def/property/OGC/0/Radiance", - "description": "Red Channel", - "gdal_interpretation": "RedBand", - "identifier": "red", - "name": "red", - "nil_values": [ - { - "reason": "http://www.opengis.net/def/nil/OGC/0/unknown", - "value": 0 - } - ], - "uom": "W.m-2.Sr-1", - "significant_figures": 5, - "allowed_value_ranges": [ - [0, 65535] - ] - }, - { - "definition": "http://www.opengis.net/def/property/OGC/0/Radiance", - "description": "Green Channel", - "gdal_interpretation": "GreenBand", - "identifier": "green", - "name": "green", - "nil_values": [ - { - "reason": "http://www.opengis.net/def/nil/OGC/0/unknown", - "value": 0 - } - ], - "uom": "W.m-2.Sr-1", - "significant_figures": 5, - "allowed_value_ranges": [ - [0, 65535] - ] - }, - { - "definition": "http://www.opengis.net/def/property/OGC/0/Radiance", - "description": "Blue Channel", - "gdal_interpretation": "BlueBand", - "identifier": "blue", - "name": "blue", + "description": "Gray Channel", + "gdal_interpretation": "GrayBand", + "identifier": "gray", + "name": "gray", "nil_values": [ { "reason": "http://www.opengis.net/def/nil/OGC/0/unknown", @@ -523,7 +487,7 @@ } ], "data_type": "Uint16", - "name": "sar_hh_hv_vh_vv_rgb" + "name": "sar_hh_hv_vh_vv_gray" },{ "bands": [ { @@ -814,4 +778,81 @@ ], "data_type": "Float32", "name": "float32_grayscale" +},{ + "bands": [ + { + "definition": "http://www.opengis.net/def/property/OGC/0/Radiance", + "description": "Blue Channel", + "gdal_interpretation": "BlueBand", + "identifier": "blue", + "name": "blue", + "nil_values": [ + { + "reason": "http://www.opengis.net/def/nil/OGC/0/unknown", + "value": 0 + } + ], + "uom": "W.m-2.Sr-1", + "significant_figures": 5, + "allowed_value_ranges": [ + [0, 65535] + ] + }, + { + "definition": "http://www.opengis.net/def/property/OGC/0/Radiance", + "description": "Green Channel", + "gdal_interpretation": "GreenBand", + "identifier": "green", + "name": "green", + "nil_values": [ + { + "reason": "http://www.opengis.net/def/nil/OGC/0/unknown", + "value": 0 + } + ], + "uom": "W.m-2.Sr-1", + "significant_figures": 5, + "allowed_value_ranges": [ + [0, 65535] + ] + }, + { + "definition": "http://www.opengis.net/def/property/OGC/0/Radiance", + "description": "Red Channel", + "gdal_interpretation": "RedBand", + "identifier": "red", + "name": "red", + "nil_values": [ + { + "reason": "http://www.opengis.net/def/nil/OGC/0/unknown", + "value": 0 + } + ], + "uom": "W.m-2.Sr-1", + "significant_figures": 5, + "allowed_value_ranges": [ + [0, 65535] + ] + }, + { + "definition": "http://www.opengis.net/def/property/OGC/0/Radiance", + "description": "Nir Channel", + "gdal_interpretation": "NirBand", + "identifier": "nir", + "name": "nir", + "nil_values": [ + { + "reason": "http://www.opengis.net/def/nil/OGC/0/unknown", + "value": 0 + } + ], + "uom": "W.m-2.Sr-1", + "significant_figures": 5, + "allowed_value_ranges": [ + [0, 65535] + ] + } + ], + "data_type": "Uint16", + "name": "BGRNir" }] diff --git a/core/setup.py b/core/setup.py index 93112025b3dde3f53a4f33ca88c9f6013c33960a..223d0ed5738d4a44a88dc5e5d6233d1ea820725a 100644 --- a/core/setup.py +++ b/core/setup.py @@ -6,7 +6,7 @@ long_description = "" setup( name="registrar", # Replace with your own username - version="1.0.0-rc.2", + version="1.0.0", author="", author_email="", description="registrar for PVS", diff --git a/documentation/operator-guide/configuration.rst b/documentation/operator-guide/configuration.rst index de3e0084a312e782e18bdd162ca0243b978d3e79..7565e8f3af063f485805ae740ce4e76622bcef61 100644 --- a/documentation/operator-guide/configuration.rst +++ b/documentation/operator-guide/configuration.rst @@ -343,10 +343,6 @@ preprocessing What subdatasets to extract and how to name them. - data_file_glob - - A file glob pattern to select files to extract from. - subdataset_types Mapping of subdataset identifier to output filename postfix for diff --git a/preprocessor/preprocessor/config-schema.yaml b/preprocessor/preprocessor/config-schema.yaml index 10bb58f5f617c72ae85dce5d7c60b7f35dd69d29..f1c7138ec866b6de715367e669af2b4419c56aed 100644 --- a/preprocessor/preprocessor/config-schema.yaml +++ b/preprocessor/preprocessor/config-schema.yaml @@ -107,9 +107,6 @@ definitions: description: The definition of the subdataset extraction step. type: object properties: - data_file_glob: - description: The data file selector. - type: string subdataset_types: description: Mapping of subdataset identifier to output filename postfix for subdatasets to be extracted for each data file. type: object @@ -128,7 +125,7 @@ definitions: type: description: The type of georeferencing to apply. type: string - enum: [gcp, rpc, corner, world, no_op] # TODO: more + enum: [gcp, rpc, corner, world, no_op, fix_geotrans] # TODO: more options: description: Additional options for the georeferencing. Depends on the type of georeferencing. type: object diff --git a/preprocessor/preprocessor/steps/georeference.py b/preprocessor/preprocessor/steps/georeference.py index 80a8b43205130ddd2b5ca1dcb2af7700ca7040eb..afadc7bf1d863676eb39c5729737affd38aea85b 100644 --- a/preprocessor/preprocessor/steps/georeference.py +++ b/preprocessor/preprocessor/steps/georeference.py @@ -26,6 +26,8 @@ def georeference_step(source_dir: os.PathLike, target_dir: os.PathLike, preproce georef_func = corner_georef elif type_name == 'no_op': georef_func = no_op + elif type_name == 'fix_geotrans': + georef_func = fix_geotrans else: raise Exception('Invalid georeference type %s' % type_name) try: @@ -125,10 +127,6 @@ def corner_georef(input_filename: os.PathLike, target_filename: os.PathLike, cor orbit_direction = ds.GetMetadata()[orbit_direction_name].lower() metadata = ds.GetRasterBand(1).GetMetadata() - # from pprint import pprint - - # pprint (metadata) - # pprint(ds.GetMetadata()) bl, br, tl, tr = [ [float(num) for num in metadata[corner_name].split()] for corner_name in corner_names @@ -170,6 +168,57 @@ def no_op(input_filename: os.PathLike, target_filename: os.PathLike): shutil.move(input_filename, target_filename) +def fix_geotrans(input_filename: os.PathLike, target_filename: os.PathLike, warp_options: dict=None): + # assumes already existing geotransform and if xres=0,yres=0, fixes it + try: + ds = gdal.Open(input_filename, gdal.GA_Update) + except RuntimeError: + logger.warn('Can not open file by GDAL %s' % (input_filename)) + return + ds = correct_geo_transform(ds) + if ds.GetDriver().ShortName == 'JP2OpenJPEG': + # workaround for rewriting jp2 files and exception thrown by "USE_SRC_CODESTREAM=YES specified, but no codestream found" + # then needs .vrt configured in data_file_globs too + target_filename = replace_ext(target_filename, '.vrt'), + gdal.Warp( + target_filename, + ds, + **(warp_options or {}), + ) + + +def correct_geo_transform(src_ds): + # input - gdal dataset + # sets new geotransform if necessary by creating control points of a raster with switched height and width + # returns - gdal dataset + ulx, xres, xskew, uly, yskew, yres = src_ds.GetGeoTransform() + logger.debug("Testing for malformed geotransform") + # test geotransform if necessary to shift + if xres == 0.0 and yres == 0.0: + logger.info("Malformed geotransform xres,yres=0 detected, correcting.") + # malformed image, compute xres and yres switched in geotransform + lrx = ulx + (src_ds.RasterXSize * xskew) + lry = uly + (src_ds.RasterYSize * yskew) + # [ulx, lrx, lry, uly] - bounds = lon_min, lon_max, lat_min, lat_max + fp = [[0, src_ds.RasterXSize, src_ds.RasterXSize, 0], [0, 0, src_ds.RasterYSize, src_ds.RasterYSize]] + tp = [[ulx, lrx, lrx, ulx], [lry, lry, uly, uly]] + pix = list(zip(fp[0], fp[1])) + coor = list(zip(tp[0], tp[1])) + # compute the gdal.GCP parameters + gcps = [] + for index, txt in enumerate(pix): + gcps.append(gdal.GCP()) + gcps[index].GCPPixel = pix[index][0] + gcps[index].GCPLine = src_ds.RasterYSize - int(pix[index][1]) + gcps[index].GCPX = coor[index][0] + gcps[index].GCPY = coor[index][1] + # get correct geotransform from gcps + geotransform_new = gdal.GCPsToGeoTransform(gcps) + # overwrite geotransform with new + src_ds.SetGeoTransform(geotransform_new) + return src_ds + + def gcps_from_borders(size: Tuple[float, float], coords: List[Tuple[float, float]], orbit_direction: str, force_north_up: bool=False): x_size, y_size = size # expects coordinates in dict(.*border_left.*:[lat,lon],...) diff --git a/preprocessor/preprocessor/steps/subdataset.py b/preprocessor/preprocessor/steps/subdataset.py index 6438d27790b36e7924783d9950bb3d8518c08a98..6fe05bcb9c52a14fd1076ceede30fc829edfb8a8 100644 --- a/preprocessor/preprocessor/steps/subdataset.py +++ b/preprocessor/preprocessor/steps/subdataset.py @@ -1,14 +1,17 @@ import os -from os.path import join, splitext, basename, dirname +from os.path import join, splitext, basename, dirname, isdir from glob import glob from typing import Dict from ..util import replace_ext, gdal -def extract_subdataset_step(source_dir: os.PathLike, target_dir: os.PathLike, preprocessor_config: dict, data_file_glob: str, subdataset_types: Dict[str, str]=None): - datafiles = glob(join(source_dir, data_file_glob)) - if not datafiles: +def extract_subdataset_step(source_dir: os.PathLike, target_dir: os.PathLike, preprocessor_config: dict, subdataset_types: Dict[str, str]=None): + filenames = [] + for dataglob in preprocessor_config.get('data_file_globs', '*'): + for p in [path for path in glob(join(source_dir, '**', dataglob), recursive=True) if not isdir(path)]: + filenames.append(p) + if len(filenames) == 0: raise Exception('No datafiles were matched by the provided glob') for filename in datafiles: diff --git a/testing/testing_preprocessor_config.yml b/testing/testing_preprocessor_config.yml index 5edd267e9bf25a8765d16faa76f2f896519f97c8..827c7b064f3eb441ad8934a6ae14dab84900681d 100644 --- a/testing/testing_preprocessor_config.yml +++ b/testing/testing_preprocessor_config.yml @@ -61,11 +61,11 @@ preprocessing: stack_bands: group_by: ".*/(.*)_P..tif" sort_by: ".*_P(R|G|B|N).tif" - order: - - R - - G + order: # to match browse type in init_db - B + - G - N + - R MER_RR__2P: georeference: geotransforms: