From d086f73b61f5329ba750331f8ed953e12a53c877 Mon Sep 17 00:00:00 2001 From: eyal rozen Date: Sun, 16 Aug 2026 13:28:22 +0300 Subject: [PATCH 1/5] feat: add new enrichment features config --- .../vectorStandard/namespace/v1.schema.json | 173 +++++++++++++++++ .../synchronizer/v4.schema.json | 179 ++++++++++++++++++ 2 files changed, 352 insertions(+) create mode 100644 schemas/vector/vectorStandard/namespace/v1.schema.json create mode 100644 schemas/vector/vectorStandard/synchronizer/v4.schema.json diff --git a/schemas/vector/vectorStandard/namespace/v1.schema.json b/schemas/vector/vectorStandard/namespace/v1.schema.json new file mode 100644 index 00000000..f789aa3e --- /dev/null +++ b/schemas/vector/vectorStandard/namespace/v1.schema.json @@ -0,0 +1,173 @@ +{ + "$id": "https://mapcolonies.com/vector/vectorStandard/namespace/v1", + "type": "object", + "title": "vectorStandardNamespaceSchemaV1", + "description": "A single vector standard namespace - one source database and the layer source describing it. Namespaces are consumed as an array, and the config package stops descending at arrays, so no x-env-value declared or inherited below this point is ever bound. Per namespace values, credentials in particular, are expected to come from referenced config instances.", + "required": ["name", "db", "layersFile", "bucket", "layerSource"], + "unevaluatedProperties": false, + "properties": { + "name": { + "type": "string", + "description": "The namespace identifier. Appears in the destination database primary keys, in the API path and as a key in the aliases file, so it is restricted to URL safe characters", + "pattern": "^[A-Za-z0-9][A-Za-z0-9_-]*$", + "maxLength": 63, + "examples": ["data-2024", "data-2025"] + }, + "db": { + "description": "The source database for this namespace. A config instance of https://mapcolonies.com/common/db/full/v3 can be referenced here directly", + "allOf": [ + { + "$ref": "https://mapcolonies.com/common/db/full/v3" + }, + { + "type": "object", + "required": ["type"], + "properties": { + "type": { + "type": "string", + "description": "the type of the database", + "default": "postgres" + } + } + } + ] + }, + "layersFile": { + "type": "string", + "description": "The path to this namespace's layers file. Each namespace owns its own file - sharing one across namespaces is always a mistake", + "examples": ["./config/layers/data-2025.json"] + }, + "bucket": { + "type": "string", + "description": "The S3 bucket holding this namespace's layer source. The connection itself is shared and configured once under dbs.s3" + }, + "layerSource": { + "$ref": "#/definitions/layerSource" + } + }, + "definitions": { + "layerSource": { + "type": "object", + "description": "Where layer identity and property aliases come from. The two modes are mutually exclusive: sharedLua requires the lua fields and carries the enrichment API, perLayerJson forbids enrichment outright, because without a layer id its url template cannot be resolved. The exclusivity is enforced by shape rather than by a runtime check", + "required": ["type"], + "unevaluatedProperties": false, + "properties": { + "type": { + "description": "Which layer source this namespace uses", + "enum": ["sharedLua", "perLayerJson"] + }, + "fileName": { + "type": "string", + "description": "sharedLua only. The key of the lua file within the bucket" + }, + "layersVariable": { + "type": "string", + "description": "sharedLua only. Lua layers variable name" + }, + "aliasFieldName": { + "type": "string", + "description": "sharedLua only. Lua alias field name" + }, + "idFieldName": { + "type": "string", + "description": "sharedLua only. Lua id field name" + }, + "nameFieldName": { + "type": "string", + "description": "sharedLua only. Lua name field name" + }, + "enrichment": { + "$ref": "#/definitions/enrichment" + }, + "prefix": { + "type": "string", + "description": "perLayerJson only. Optional key prefix within the bucket, prepended to the file name configured for each layer", + "default": "" + }, + "aliasLayerNameField": { + "type": "string", + "description": "perLayerJson only. The field holding the layer alias", + "default": "alias_layer_name" + }, + "fieldsField": { + "type": "string", + "description": "perLayerJson only. The field holding the array of field definitions", + "default": "fields" + }, + "fieldNameField": { + "type": "string", + "description": "perLayerJson only. The field within a field definition holding the source column name. Matched against the database columns case insensitively", + "default": "field_name" + }, + "aliasFieldNameField": { + "type": "string", + "description": "perLayerJson only. The field within a field definition holding the property alias", + "default": "alias_field_name" + } + }, + "if": { + "required": ["type"], + "properties": { + "type": { + "const": "sharedLua" + } + } + }, + "then": { + "required": ["fileName", "layersVariable", "aliasFieldName", "idFieldName", "nameFieldName", "enrichment"] + }, + "else": { + "not": { + "required": ["enrichment"] + } + } + }, + "enrichment": { + "type": "object", + "description": "sharedLua only. Property alias enrichment over http", + "required": ["enabled"], + "properties": { + "enabled": { + "type": "boolean", + "description": "enable enrichment", + "default": false + }, + "api": { + "type": "string", + "description": "Property alias name API. Supports the {layerName} and {layerId} placeholders" + }, + "propertiesPath": { + "type": "string", + "default": "{layerName}.properties" + }, + "aliasField": { + "type": "string", + "default": "alias" + }, + "requestTimeoutMilliseconds": { + "type": "integer", + "description": "Timeout for enrichment API requests in milliseconds", + "default": 5000 + } + }, + "unevaluatedProperties": false, + "if": { + "properties": { + "enabled": { + "const": true + } + } + }, + "then": { + "required": ["api", "propertiesPath", "aliasField"] + }, + "else": { + "properties": { + "enabled": { + "const": false + } + } + } + } + } +} diff --git a/schemas/vector/vectorStandard/synchronizer/v4.schema.json b/schemas/vector/vectorStandard/synchronizer/v4.schema.json new file mode 100644 index 00000000..b2b4615c --- /dev/null +++ b/schemas/vector/vectorStandard/synchronizer/v4.schema.json @@ -0,0 +1,179 @@ +{ + "$id": "https://mapcolonies.com/vector/vectorStandard/synchronizer/v4", + "type": "object", + "title": "vectorStandardSynchronizerSchemaV4", + "description": "Vector's standard synchronizer schema. One deployment serves several namespaces, each with its own source database and layer source, writing into a single destination catalog", + "allOf": [ + { + "$ref": "https://mapcolonies.com/common/boilerplate/v2" + }, + { + "type": "object", + "required": ["dbs", "namespaces", "schedule"], + "properties": { + "dbs": { + "type": "object", + "required": ["destination", "s3"], + "properties": { + "destination": { + "$ref": "#/definitions/destinationDb" + }, + "s3": { + "$ref": "#/definitions/s3" + } + } + }, + "namespaces": { + "$ref": "#/definitions/namespaces" + }, + "schedule": { + "$ref": "#/definitions/schedule" + }, + "indexNameFormat": { + "$ref": "#/definitions/indexNameFormat" + }, + "aliasesFile": { + "$ref": "#/definitions/aliasesFile" + }, + "typeMapFile": { + "$ref": "#/definitions/typeMapFile" + } + } + } + ], + "definitions": { + "namespaces": { + "type": "array", + "description": "The namespaces this deployment synchronizes. Each is an independent failure domain: an unreachable source database degrades its own namespace only. Names must be unique, which JSON Schema cannot express over object properties - it is validated at startup instead", + "minItems": 1, + "items": { + "$ref": "https://mapcolonies.com/vector/vectorStandard/namespace/v1" + } + }, + "destinationDb": { + "type": "object", + "description": "The single destination catalog every namespace writes into", + "required": ["database", "ssl", "type"], + "properties": { + "host": { + "type": "string", + "description": "the host of the database", + "default": "localhost", + "x-env-value": "DEST_DB_HOST" + }, + "port": { + "type": "integer", + "description": "the port of the database", + "default": 5432, + "x-env-value": "DEST_DB_PORT" + }, + "username": { + "type": "string", + "description": "the username of the database", + "default": "postgres", + "maxLength": 63, + "x-env-value": "DEST_DB_USERNAME" + }, + "password": { + "type": "string", + "description": "the password of the database", + "default": "postgres", + "x-env-value": "DEST_DB_PASSWORD" + }, + "schema": { + "type": "string", + "description": "the schema name of the database", + "default": "public", + "x-env-value": "DEST_DB_SCHEMA" + }, + "database": { + "type": "string", + "description": "the database name", + "maxLength": 63, + "x-env-value": "DEST_DB_NAME" + }, + "type": { + "type": "string", + "description": "the type of the database", + "default": "postgres", + "x-env-value": "DEST_DB_TYPE" + }, + "ssl": { + "type": "object", + "description": "ssl configuration", + "properties": { + "enabled": { + "type": "boolean", + "description": "enable ssl", + "default": false, + "x-env-value": "DEST_DB_ENABLE_SSL_AUTH" + }, + "ca": { + "type": "string", + "description": "the path to the ca file", + "x-env-value": "DEST_DB_CA_PATH" + }, + "cert": { + "type": "string", + "description": "the path to the cert file", + "x-env-value": "DEST_DB_CERT_PATH" + }, + "key": { + "type": "string", + "description": "the path to the key file", + "x-env-value": "DEST_DB_KEY_PATH" + } + }, + "unevaluatedProperties": false, + "if": { + "properties": { + "enabled": { + "const": true + } + } + }, + "then": { + "required": ["cert", "key"] + }, + "else": { + "properties": { + "enabled": { + "const": false + } + } + } + } + } + }, + "s3": { + "$ref": "https://mapcolonies.com/common/s3/partial/v1", + "description": "The shared S3 connection. Buckets are configured per namespace, so no bucket is declared here" + }, + "schedule": { + "type": "string", + "description": "The cron timing spec. One schedule drives every namespace; they are synchronized sequentially within a tick", + "default": "0 0 * * *", + "examples": ["*/1 * * * *"], + "x-env-value": "SYNC_FIELDS_CRON" + }, + "indexNameFormat": { + "type": "string", + "description": "The indexes' names in the source DB", + "default": "{layerName}_{column}_idx", + "examples": ["{layerName}_{column}_idx"], + "x-env-value": "INDEX_NAME_FORMAT" + }, + "aliasesFile": { + "type": "string", + "description": "The path to the aliases file, shared by every namespace. Keyed namespace then layer then property, with * accepted at the namespace and layer tiers", + "default": "./config/aliases.json", + "x-env-value": "ALIASES_FILE_PATH" + }, + "typeMapFile": { + "type": "string", + "description": "The path to the type map file, shared by every namespace since it describes postgres rather than the data", + "default": "./config/typeMap.json", + "x-env-value": "TYPE_MAP_FILE_PATH" + } + } +} From e74d135291286bad65c169934d86bf77d0a72488 Mon Sep 17 00:00:00 2001 From: eyal rozen Date: Wed, 19 Aug 2026 12:10:47 +0300 Subject: [PATCH 2/5] feat: improve enrichment union type --- .../vectorStandard/namespace/v1.schema.json | 28 ++++++++++--------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/schemas/vector/vectorStandard/namespace/v1.schema.json b/schemas/vector/vectorStandard/namespace/v1.schema.json index f789aa3e..8efa097a 100644 --- a/schemas/vector/vectorStandard/namespace/v1.schema.json +++ b/schemas/vector/vectorStandard/namespace/v1.schema.json @@ -135,19 +135,6 @@ "api": { "type": "string", "description": "Property alias name API. Supports the {layerName} and {layerId} placeholders" - }, - "propertiesPath": { - "type": "string", - "default": "{layerName}.properties" - }, - "aliasField": { - "type": "string", - "default": "alias" - }, - "requestTimeoutMilliseconds": { - "type": "integer", - "description": "Timeout for enrichment API requests in milliseconds", - "default": 5000 } }, "unevaluatedProperties": false, @@ -159,6 +146,21 @@ } }, "then": { + "properties": { + "propertiesPath": { + "type": "string", + "default": "{layerName}.properties" + }, + "aliasField": { + "type": "string", + "default": "alias" + }, + "requestTimeoutMilliseconds": { + "type": "integer", + "description": "Timeout for enrichment API requests in milliseconds", + "default": 5000 + } + }, "required": ["api", "propertiesPath", "aliasField"] }, "else": { From f95f315231a7842fc0b4479410616abc0ad30372 Mon Sep 17 00:00:00 2001 From: eyal rozen Date: Wed, 19 Aug 2026 12:21:05 +0300 Subject: [PATCH 3/5] fix: fix the union type --- .../vectorStandard/namespace/v1.schema.json | 52 ++++++++++--------- 1 file changed, 27 insertions(+), 25 deletions(-) diff --git a/schemas/vector/vectorStandard/namespace/v1.schema.json b/schemas/vector/vectorStandard/namespace/v1.schema.json index 8efa097a..017271e1 100644 --- a/schemas/vector/vectorStandard/namespace/v1.schema.json +++ b/schemas/vector/vectorStandard/namespace/v1.schema.json @@ -78,31 +78,6 @@ }, "enrichment": { "$ref": "#/definitions/enrichment" - }, - "prefix": { - "type": "string", - "description": "perLayerJson only. Optional key prefix within the bucket, prepended to the file name configured for each layer", - "default": "" - }, - "aliasLayerNameField": { - "type": "string", - "description": "perLayerJson only. The field holding the layer alias", - "default": "alias_layer_name" - }, - "fieldsField": { - "type": "string", - "description": "perLayerJson only. The field holding the array of field definitions", - "default": "fields" - }, - "fieldNameField": { - "type": "string", - "description": "perLayerJson only. The field within a field definition holding the source column name. Matched against the database columns case insensitively", - "default": "field_name" - }, - "aliasFieldNameField": { - "type": "string", - "description": "perLayerJson only. The field within a field definition holding the property alias", - "default": "alias_field_name" } }, "if": { @@ -119,6 +94,33 @@ "else": { "not": { "required": ["enrichment"] + }, + "properties": { + "prefix": { + "type": "string", + "description": "perLayerJson only. Optional key prefix within the bucket, prepended to the file name configured for each layer", + "default": "" + }, + "aliasLayerNameField": { + "type": "string", + "description": "perLayerJson only. The field holding the layer alias", + "default": "alias_layer_name" + }, + "fieldsField": { + "type": "string", + "description": "perLayerJson only. The field holding the array of field definitions", + "default": "fields" + }, + "fieldNameField": { + "type": "string", + "description": "perLayerJson only. The field within a field definition holding the source column name. Matched against the database columns case insensitively", + "default": "field_name" + }, + "aliasFieldNameField": { + "type": "string", + "description": "perLayerJson only. The field within a field definition holding the property alias", + "default": "alias_field_name" + } } } }, From 29ebce5d823160531c6940c7feb7b16f16951cd2 Mon Sep 17 00:00:00 2001 From: eyal rozen Date: Sun, 23 Aug 2026 12:32:25 +0300 Subject: [PATCH 4/5] feat: update discriminated types doe enrichment --- .../vectorStandard/namespace/v1.schema.json | 41 +++++++++++-------- 1 file changed, 23 insertions(+), 18 deletions(-) diff --git a/schemas/vector/vectorStandard/namespace/v1.schema.json b/schemas/vector/vectorStandard/namespace/v1.schema.json index 017271e1..1f60847f 100644 --- a/schemas/vector/vectorStandard/namespace/v1.schema.json +++ b/schemas/vector/vectorStandard/namespace/v1.schema.json @@ -126,7 +126,7 @@ }, "enrichment": { "type": "object", - "description": "sharedLua only. Property alias enrichment over http", + "description": "sharedLua only. Property alias enrichment over http. The settings exist only while it is enabled, so disabling it is exactly { \"enabled\": false }", "required": ["enabled"], "properties": { "enabled": { @@ -137,10 +137,26 @@ "api": { "type": "string", "description": "Property alias name API. Supports the {layerName} and {layerId} placeholders" + }, + "propertiesPath": { + "type": "string", + "description": "Path to the properties map within the API response. Supports the {layerName} placeholder", + "examples": ["{layerName}.properties"] + }, + "aliasField": { + "type": "string", + "description": "The field within a property entry holding its alias", + "examples": ["alias"] + }, + "requestTimeoutMilliseconds": { + "type": "integer", + "description": "Timeout for enrichment API requests in milliseconds", + "examples": [5000] } }, "unevaluatedProperties": false, "if": { + "required": ["enabled"], "properties": { "enabled": { "const": true @@ -148,28 +164,17 @@ } }, "then": { - "properties": { - "propertiesPath": { - "type": "string", - "default": "{layerName}.properties" - }, - "aliasField": { - "type": "string", - "default": "alias" - }, - "requestTimeoutMilliseconds": { - "type": "integer", - "description": "Timeout for enrichment API requests in milliseconds", - "default": 5000 - } - }, - "required": ["api", "propertiesPath", "aliasField"] + "required": ["api", "propertiesPath", "aliasField", "requestTimeoutMilliseconds"] }, "else": { "properties": { "enabled": { "const": false - } + }, + "api": false, + "propertiesPath": false, + "aliasField": false, + "requestTimeoutMilliseconds": false } } } From aa03aeac9165994cbd2aaff4f65cba5edc61556e Mon Sep 17 00:00:00 2001 From: eyal rozen Date: Sun, 23 Aug 2026 14:08:15 +0300 Subject: [PATCH 5/5] feat: add namespaces to api --- .../vector/vectorStandard/api/v2.schema.json | 61 +++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 schemas/vector/vectorStandard/api/v2.schema.json diff --git a/schemas/vector/vectorStandard/api/v2.schema.json b/schemas/vector/vectorStandard/api/v2.schema.json new file mode 100644 index 00000000..6d00632e --- /dev/null +++ b/schemas/vector/vectorStandard/api/v2.schema.json @@ -0,0 +1,61 @@ +{ + "$id": "https://mapcolonies.com/vector/vectorStandard/api/v2", + "type": "object", + "title": "vectorStandardApiSchemaV2", + "description": "Vector's standard API schema", + "allOf": [ + { + "$ref": "https://mapcolonies.com/common/boilerplate/v2" + }, + { + "$ref": "#/definitions/databases" + }, + { + "$ref": "#/definitions/namespaces" + } + ], + "definitions": { + "namespaces": { + "type": "object", + "required": ["namespaces"], + "properties": { + "namespaces": { + "type": "array", + "description": "The namespaces this deployment serves, by name. The synchronizer owns everything else about them, so only the names are repeated here", + "minItems": 1, + "items": { + "type": "string", + "description": "A namespace name, matching one the synchronizer is configured with", + "pattern": "^[A-Za-z0-9][A-Za-z0-9_-]*$", + "maxLength": 63, + "examples": ["data-2024", "data-2025"] + } + } + } + }, + "databases": { + "type": "object", + "required": ["db"], + "properties": { + "db": { + "allOf": [ + { + "$ref": "https://mapcolonies.com/common/db/full/v3" + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "x-env-value": "DB_TYPE", + "default": "postgres" + } + }, + "required": ["type"] + } + ] + } + } + } + } +}