regen schema

harvester/config-schema.json

+{
+  "title": "HarvesterAppConfig",
+  "type": "object",
+  "properties": {
+    "harvesters": {
+      "title": "Harvesters",
+      "type": "object",
+      "additionalProperties": {
+        "$ref": "#/definitions/HarvesterConfig"
+      }
+    },
+    "redis": {
+      "title": "Redis",
+      "default": {
+        "host": "vs-redis-master",
+        "port": 6379
+      },
+      "allOf": [
+        {
+          "$ref": "#/definitions/RedisConfig"
+        }
+      ]
+    },
+    "filesystems": {
+      "title": "Filesystems",
+      "type": "object",
+      "additionalProperties": {
+        "$ref": "#/definitions/FilesystemConfig"
+      }
+    }
+  },
+  "required": [
+    "harvesters"
+  ],
+  "definitions": {
+    "ResourceType": {
+      "title": "ResourceType",
+      "description": "An enumeration.",
+      "enum": [
+        "stacapi",
+        "staccatalog",
+        "filematcher",
+        "oads",
+        "opensearch"
+      ],
+      "type": "string"
+    },
+    "TimeConfig": {
+      "title": "TimeConfig",
+      "type": "object",
+      "properties": {
+        "begin": {
+          "title": "Begin",
+          "default": "2022-12-01T10:12:15.703595",
+          "type": "string"
+        },
+        "end": {
+          "title": "End",
+          "default": "2022-12-01T10:12:15.703605",
+          "type": "string"
+        }
+      }
+    },
+    "QueryConfig": {
+      "title": "QueryConfig",
+      "type": "object",
+      "properties": {
+        "time": {
+          "$ref": "#/definitions/TimeConfig"
+        },
+        "bbox": {
+          "title": "Bbox",
+          "type": "string"
+        },
+        "collection": {
+          "title": "Collection",
+          "type": "string"
+        },
+        "extra_params": {
+          "title": "Extra Params",
+          "type": "object",
+          "additionalProperties": {
+            "type": "string"
+          }
+        }
+      },
+      "required": [
+        "time",
+        "bbox"
+      ]
+    },
+    "STACAPIConfig": {
+      "title": "STACAPIConfig",
+      "type": "object",
+      "properties": {
+        "url": {
+          "title": "Url",
+          "type": "string"
+        },
+        "query": {
+          "$ref": "#/definitions/QueryConfig"
+        }
+      },
+      "required": [
+        "url",
+        "query"
+      ]
+    },
+    "STACCatalogConfig": {
+      "title": "STACCatalogConfig",
+      "type": "object",
+      "properties": {
+        "root_path": {
+          "title": "Root Path",
+          "type": "string"
+        },
+        "collection_id": {
+          "title": "Collection Id",
+          "type": "string"
+        },
+        "deduplicate": {
+          "title": "Deduplicate",
+          "default": false,
+          "type": "boolean"
+        },
+        "filesystem": {
+          "title": "Filesystem",
+          "default": "local",
+          "type": "string"
+        }
+      },
+      "required": [
+        "root_path"
+      ]
+    },
+    "FileMatcherConfig": {
+      "title": "FileMatcherConfig",
+      "type": "object",
+      "properties": {
+        "root_path": {
+          "title": "Root Path",
+          "type": "string"
+        },
+        "time_regex": {
+          "title": "Time Regex",
+          "type": "string"
+        },
+        "id_regex": {
+          "title": "Id Regex",
+          "type": "string"
+        },
+        "asset_id": {
+          "title": "Asset Id",
+          "type": "string"
+        },
+        "filesystem": {
+          "title": "Filesystem",
+          "default": "local",
+          "type": "string"
+        }
+      },
+      "required": [
+        "root_path",
+        "time_regex",
+        "id_regex",
+        "asset_id"
+      ]
+    },
+    "OADSConfig": {
+      "title": "OADSConfig",
+      "type": "object",
+      "properties": {
+        "url": {
+          "title": "Url",
+          "type": "string"
+        },
+        "use_oads_ext": {
+          "title": "Use Oads Ext",
+          "default": false,
+          "type": "boolean"
+        }
+      },
+      "required": [
+        "url"
+      ]
+    },
+    "JSONConfig": {
+      "title": "JSONConfig",
+      "type": "object",
+      "properties": {
+        "property_mapping": {
+          "title": "Property Mapping",
+          "type": "object",
+          "additionalProperties": {
+            "type": "string"
+          }
+        }
+      },
+      "required": [
+        "property_mapping"
+      ]
+    },
+    "ATOMXMLConfig": {
+      "title": "ATOMXMLConfig",
+      "type": "object",
+      "properties": {}
+    },
+    "FormatConfig": {
+      "title": "FormatConfig",
+      "type": "object",
+      "properties": {
+        "type": {
+          "title": "Type",
+          "type": "string"
+        },
+        "json": {
+          "$ref": "#/definitions/JSONConfig"
+        },
+        "atom_xml": {
+          "$ref": "#/definitions/ATOMXMLConfig"
+        }
+      },
+      "required": [
+        "type"
+      ]
+    },
+    "OpenSearchConfig": {
+      "title": "OpenSearchConfig",
+      "type": "object",
+      "properties": {
+        "url": {
+          "title": "Url",
+          "type": "string"
+        },
+        "query": {
+          "$ref": "#/definitions/QueryConfig"
+        },
+        "format": {
+          "$ref": "#/definitions/FormatConfig"
+        }
+      },
+      "required": [
+        "url",
+        "query",
+        "format"
+      ]
+    },
+    "ResourceConfig": {
+      "title": "ResourceConfig",
+      "type": "object",
+      "properties": {
+        "type": {
+          "$ref": "#/definitions/ResourceType"
+        },
+        "stacapi": {
+          "$ref": "#/definitions/STACAPIConfig"
+        },
+        "staccatalog": {
+          "$ref": "#/definitions/STACCatalogConfig"
+        },
+        "filematcher": {
+          "$ref": "#/definitions/FileMatcherConfig"
+        },
+        "oads": {
+          "$ref": "#/definitions/OADSConfig"
+        },
+        "opensearch": {
+          "$ref": "#/definitions/OpenSearchConfig"
+        }
+      },
+      "required": [
+        "type"
+      ]
+    },
+    "FilterConfig": {
+      "title": "FilterConfig",
+      "type": "object",
+      "properties": {
+        "context": {
+          "title": "Context",
+          "type": "object"
+        },
+        "expression": {
+          "title": "Expression",
+          "type": "object"
+        }
+      }
+    },
+    "OutputType": {
+      "title": "OutputType",
+      "description": "An enumeration.",
+      "enum": [
+        "console",
+        "queue"
+      ],
+      "type": "string"
+    },
+    "PostprocessorType": {
+      "title": "PostprocessorType",
+      "description": "An enumeration.",
+      "enum": [
+        "builtin",
+        "external"
+      ],
+      "type": "string"
+    },
+    "PostprocessorConfig": {
+      "title": "PostprocessorConfig",
+      "type": "object",
+      "properties": {
+        "type": {
+          "$ref": "#/definitions/PostprocessorType"
+        },
+        "process": {
+          "title": "Process",
+          "type": "string"
+        },
+        "kwargs": {
+          "title": "Kwargs",
+          "type": "object"
+        }
+      },
+      "required": [
+        "type",
+        "process"
+      ]
+    },
+    "HarvesterConfig": {
+      "title": "HarvesterConfig",
+      "type": "object",
+      "properties": {
+        "resource": {
+          "$ref": "#/definitions/ResourceConfig"
+        },
+        "filter": {
+          "$ref": "#/definitions/FilterConfig"
+        },
+        "output": {
+          "default": "queue",
+          "allOf": [
+            {
+              "$ref": "#/definitions/OutputType"
+            }
+          ]
+        },
+        "queue": {
+          "title": "Queue",
+          "type": "string"
+        },
+        "postprocessors": {
+          "title": "Postprocessors",
+          "type": "array",
+          "items": {
+            "$ref": "#/definitions/PostprocessorConfig"
+          }
+        }
+      },
+      "required": [
+        "resource"
+      ]
+    },
+    "RedisConfig": {
+      "title": "RedisConfig",
+      "type": "object",
+      "properties": {
+        "host": {
+          "title": "Host",
+          "default": "vs-redis-master",
+          "type": "string"
+        },
+        "port": {
+          "title": "Port",
+          "default": 6379,
+          "type": "integer"
+        }
+      }
+    },
+    "FilesystemType": {
+      "title": "FilesystemType",
+      "description": "An enumeration.",
+      "enum": [
+        "swift",
+        "s3",
+        "file"
+      ],
+      "type": "string"
+    },
+    "S3FilesystemConfig": {
+      "title": "S3FilesystemConfig",
+      "type": "object",
+      "properties": {
+        "access_key_id": {
+          "title": "Access Key Id",
+          "default": "",
+          "type": "string"
+        },
+        "secret_access_key": {
+          "title": "Secret Access Key",
+          "default": "",
+          "type": "string"
+        },
+        "endpoint_url": {
+          "title": "Endpoint Url",
+          "default": "",
+          "type": "string"
+        },
+        "region": {
+          "title": "Region",
+          "default": "",
+          "type": "string"
+        },
+        "public": {
+          "title": "Public",
+          "default": false,
+          "type": "boolean"
+        }
+      }
+    },
+    "SwiftFileystemConfig": {
+      "title": "SwiftFileystemConfig",
+      "type": "object",
+      "properties": {
+        "user": {
+          "title": "User",
+          "type": "string"
+        },
+        "key": {
+          "title": "Key",
+          "type": "string"
+        },
+        "region_name": {
+          "title": "Region Name",
+          "type": "string"
+        },
+        "authurl": {
+          "title": "Authurl",
+          "type": "string"
+        },
+        "auth_version": {
+          "title": "Auth Version",
+          "type": "string"
+        },
+        "tenant_name": {
+          "title": "Tenant Name",
+          "type": "string"
+        }
+      },
+      "required": [
+        "user",
+        "key",
+        "region_name",
+        "authurl",
+        "auth_version",
+        "tenant_name"
+      ]
+    },
+    "LocalFilesystemConfig": {
+      "title": "LocalFilesystemConfig",
+      "type": "object",
+      "properties": {}
+    },
+    "FilesystemConfig": {
+      "title": "FilesystemConfig",
+      "type": "object",
+      "properties": {
+        "type": {
+          "$ref": "#/definitions/FilesystemType"
+        },
+        "s3": {
+          "$ref": "#/definitions/S3FilesystemConfig"
+        },
+        "swift": {
+          "$ref": "#/definitions/SwiftFileystemConfig"
+        },
+        "local": {
+          "$ref": "#/definitions/LocalFilesystemConfig"
+        }
+      },
+      "required": [
+        "type"
+      ]
+    }
+  }
+}

harvester/config-schema.yaml

-$id: https://example.com/address.schema.json
-$schema: http://json-schema.org/draft-07/schema#
-type: object
-properties:
-  harvesters:
-    description: List of harvesters
-    type: array
-    items:
-      description: A harvester configuration
-      type: object
-      properties:
-        name:
-          description: Name of the harvester. Should be unique
-          type: string
-        queue:
-          description: Name of queue to send queried data to
-          type: string
-        endpoint:
-          description: Endpoint to be queried
-          type: object
-        collection:
-          description: Name of collection.
-          type: string
-          properties:
-            url:
-              description: url of the collection
-              type: string
-            type:
-              description: type of the endpoint
-              type: string
-              enum:
-                - STACAPI
-                - STACCatalog
-                - OpenSearch
-                - FTP
-                - S3
-                - Swift
-                - OGCAPI
-                - OADS
-        bbox:
-          description: Bounding box to be queried
-          type: string
-        time:
-          description: Sensing times to be used when querying
-          type: object
-          properties:
-            begin:
-              description: Begining of sensing time
-            end:
-              description: End of sensing time
-          required:
-            - begin
-
-      required:
-        - queue
-        - name
-        - endpoint
-        - mode