From 081c2e87f1f67253d070e07b32bf8a86a6ddbe05 Mon Sep 17 00:00:00 2001 From: almog8k Date: Mon, 7 Sep 2026 10:28:31 +0300 Subject: [PATCH] refactor: remove dead seeding configuration and update to cache deletion jobs --- README.md | 18 +++-- config/custom-environment-variables.json | 27 -------- config/default.json | 12 ---- helm/templates/configmap.yaml | 8 --- helm/values.yaml | 10 --- src/common/interfaces.ts | 9 --- src/utils/geoUtils.ts | 88 ------------------------ tests/unit/mocks/configMock.ts | 11 --- 8 files changed, 11 insertions(+), 172 deletions(-) delete mode 100644 src/utils/geoUtils.ts diff --git a/README.md b/README.md index 5c11c2a..84d52fe 100644 --- a/README.md +++ b/README.md @@ -143,7 +143,6 @@ The service can be configured using environment variables or a configuration fil | INGESTION_NEW_JOB_TYPE | string | New ingestion job type | `"Ingestion_New"` | | INGESTION_UPDATE_JOB_TYPE | string | Update ingestion job type | `"Ingestion_Update"` | | INGESTION_SWAP_UPDATE_JOB_TYPE | string | Swap update job type | `"Ingestion_Swap_Update"` | -| INGESTION_SEED_JOB_TYPE | string | Seed job type | `"Ingestion_Seed"` | | TILES_MERGING_TASK_TYPE | string | Tiles merging task type | `"tiles-merging"` | | TILES_MERGING_TILE_BATCH_SIZE | number | Batch size for tile merging | `10000` | | TILES_MERGING_TASK_BATCH_SIZE | number | Batch size for task merging | `5` | @@ -151,10 +150,15 @@ The service can be configured using environment variables or a configuration fil | TILES_MERGING_RADIUS_BUFFER_UNITS | string | Units for radius buffer | `"meters"` | | TILES_MERGING_TRUNCATE_PRECISION | number | Precision for truncating | `6` | | TILES_MERGING_TRUNCATE_COORDINATES | number | Coordinates for truncating | `6` | -| TILES_SEEDING_TASK_TYPE | string | Tiles seeding task type | `"tiles-seeding"` | -| TILES_SEEDING_GRID | string | Grid configuration for tiles seeding | `"WorldCRS84"` | -| TILES_SEEDING_MAX_ZOOM | number | Maximum zoom level for seeding | `21` | -| TILES_SEEDING_SKIP_UNCACHED | boolean | Skip uncached tiles during seeding | `true` | +| UPDATE_CACHE_DELETION_JOB_TYPE | string | Cache deletion job type created after an update ingestion | `"Update_Delete_Cache"` | +| SWAP_CACHE_DELETION_JOB_TYPE | string | Cache deletion job type created after a swap-update ingestion | `"Swap_Delete_Cache"` | +| CACHE_DELETION_TASK_TYPE | string | Cache deletion task type | `"tiles-deletion"` | +| CACHE_DELETION_MAX_ZOOM | number | Maximum zoom level for the deleted tile ranges | `21` | +| CACHE_DELETION_TILE_BATCH_SIZE | number | Maximum tiles per range-deletion task | `100000` | +| CACHE_DELETION_MAX_RANGES_PER_TASK | number | Maximum tile ranges per task, bounding the serialized task params | `5000` | +| CACHE_DELETION_TASK_BATCH_SIZE | number | Number of tasks enqueued per job-manager call | `5` | +| CACHE_DELETION_GRACEFUL_RELOAD_MAX_SECONDS | number | Serving pods' config reload window, waited out before wiping cache keys | `300` | +| CACHE_DELETION_RELOAD_WINDOW_MARGIN_SECONDS | number | Margin added to the reload window | `8` | ## Export Configuration @@ -203,12 +207,12 @@ For both processing types, the service handles two primary task phases: - **For Update Ingestion:** - Updates catalog layers - - Creates seed jobs and tasks + - Creates a cache deletion job whose tasks delete the ingested footprint's tile ranges from MapProxy's redis cache - **For Swap Update Ingestion:** - Updates catalog layers - Handles layer updates in MapProxy when required - - Creates seed jobs and tasks + - Creates a cache deletion job whose task wipes the layer's entire redis cache prefix ## Export Processing diff --git a/config/custom-environment-variables.json b/config/custom-environment-variables.json index 93093c0..4a908c8 100644 --- a/config/custom-environment-variables.json +++ b/config/custom-environment-variables.json @@ -146,9 +146,6 @@ } }, "jobs": { - "seed": { - "type": "INGESTION_SEED_JOB_TYPE" - }, "updateCacheDeletion": { "type": "UPDATE_CACHE_DELETION_JOB_TYPE" }, @@ -200,30 +197,6 @@ "__name": "TILES_MERGING_TRUNCATE_COORDINATES", "__format": "number" } - }, - "tilesSeeding": { - "type": "TILES_SEEDING_TASK_TYPE", - "grid": "TILES_SEEDING_GRID", - "maxZoom": { - "__name": "TILES_SEEDING_MAX_ZOOM", - "__format": "number" - }, - "skipUncached": { - "__name": "TILES_SEEDING_SKIP_UNCACHED", - "__format": "boolean" - }, - "zoomThreshold": { - "__name": "TILES_SEEDING_ZOOM_THRESHOLD", - "__format": "number" - }, - "maxTilesPerSeedTask": { - "__name": "TILES_SEEDING_MAX_TILES_PER_SEED_TASK", - "__format": "number" - }, - "maxTilesPerCleanTask": { - "__name": "TILES_SEEDING_MAX_TILES_PER_CLEAN_TASK", - "__format": "number" - } } } }, diff --git a/config/default.json b/config/default.json index cc0aa5a..ce6b8c8 100644 --- a/config/default.json +++ b/config/default.json @@ -113,9 +113,6 @@ } }, "jobs": { - "seed": { - "type": "Ingestion_Seed" - }, "updateCacheDeletion": { "type": "Update_Delete_Cache" }, @@ -133,15 +130,6 @@ "truncatePrecision": 11, "truncateCoordinates": 2 }, - "tilesSeeding": { - "type": "tiles-seeding", - "grid": "WorldCRS84", - "maxZoom": 21, - "skipUncached": true, - "zoomThreshold": 16, - "maxTilesPerSeedTask": 500000, - "maxTilesPerCleanTask": 4000000 - }, "validation": { "type": "validation" }, diff --git a/helm/templates/configmap.yaml b/helm/templates/configmap.yaml index c349fb2..bac2296 100644 --- a/helm/templates/configmap.yaml +++ b/helm/templates/configmap.yaml @@ -57,7 +57,6 @@ data: INGESTION_UPDATE_JOB_TYPE: {{ $jobDefinitions.jobs.update.type | quote }} INGESTION_SWAP_UPDATE_JOB_TYPE: {{ $jobDefinitions.jobs.swapUpdate.type | quote }} INGESTION_DELETE_LAYER_JOB_TYPE: {{ $jobDefinitions.jobs.deleteLayer.type | quote }} - INGESTION_SEED_JOB_TYPE : {{ $jobDefinitions.jobs.seed.type | quote }} UPDATE_CACHE_DELETION_JOB_TYPE: {{ $jobDefinitions.jobs.updateCacheDeletion.type | quote }} SWAP_CACHE_DELETION_JOB_TYPE: {{ $jobDefinitions.jobs.swapCacheDeletion.type | quote }} EXPORT_JOB_TYPE: {{ $jobDefinitions.jobs.export.type | quote }} @@ -77,13 +76,6 @@ data: TILES_MERGING_RADIUS_BUFFER_UNITS: {{ $jobDefinitions.tasks.merge.radiusBufferUnits | quote }} TILES_MERGING_TRUNCATE_PRECISION: {{ $jobDefinitions.tasks.merge.truncatePrecision | quote }} TILES_MERGING_TRUNCATE_COORDINATES: {{ $jobDefinitions.tasks.merge.truncateCoordinates | quote }} - TILES_SEEDING_TASK_TYPE: {{ $jobDefinitions.tasks.seed.type | quote }} - TILES_SEEDING_GRID: {{ $jobDefinitions.tasks.seed.grid | quote }} - TILES_SEEDING_MAX_ZOOM: {{ $jobDefinitions.tasks.seed.maxZoom | quote }} - TILES_SEEDING_SKIP_UNCACHED: {{ $jobDefinitions.tasks.seed.skipUncached | quote }} - TILES_SEEDING_ZOOM_THRESHOLD: {{ $jobDefinitions.tasks.seed.zoomThreshold | quote }} - TILES_SEEDING_MAX_TILES_PER_SEED_TASK: {{ $jobDefinitions.tasks.seed.maxTilesPerSeedTask | quote }} - TILES_SEEDING_MAX_TILES_PER_CLEAN_TASK: {{ $jobDefinitions.tasks.seed.maxTilesPerCleanTask | quote }} CACHE_DELETION_TASK_TYPE: {{ $jobDefinitions.tasks.cacheDeletion.type | quote }} CACHE_DELETION_MAX_ZOOM: {{ $jobDefinitions.tasks.cacheDeletion.maxZoom | quote }} CACHE_DELETION_TILE_BATCH_SIZE: {{ $jobDefinitions.tasks.cacheDeletion.tileBatchSize | quote }} diff --git a/helm/values.yaml b/helm/values.yaml index ab0c1cd..cf3fb66 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -141,8 +141,6 @@ jobDefinitions: type: "" deleteLayer: type: "" - seed: - type: "" updateCacheDeletion: type: "" swapCacheDeletion: @@ -168,14 +166,6 @@ jobDefinitions: radiusBufferUnits: "degrees" # supported values are: "degrees" | "centimeters" | "meters" | "millimeters" | "kilometers" | "miles" | "inches" | "yards" | "feet" | "radians" truncatePrecision: 11 # coordinate decimal precision truncateCoordinates: 2 # maximum number of coordinates (primarly used to remove z coordinates) - seed: - type: "" - grid: "WorldCRS84" - maxZoom: 21 - skipUncached: true - zoomThreshold: 16 #zoom level threshold for high res seeding first seeding task will be 0->zoomThreshold - maxTilesPerSeedTask: 250000 - maxTilesPerCleanTask: 500000 export: type: "" tilesDeletion: diff --git a/src/common/interfaces.ts b/src/common/interfaces.ts index 7278bb1..2b04c48 100644 --- a/src/common/interfaces.ts +++ b/src/common/interfaces.ts @@ -61,7 +61,6 @@ export interface JobConfig { } export interface IngestionJobsConfig { - seed: JobConfig | undefined; updateCacheDeletion: JobConfig | undefined; swapCacheDeletion: JobConfig | undefined; } @@ -81,7 +80,6 @@ export interface ExportPollingJobsConfig { export interface IngestionTasksConfig { tilesMerging: TilesMergingTaskConfig; - tilesSeeding: TilesSeedingTaskConfig; tilesDeletion: TilesDeletionTaskConfig; cacheDeletion: CacheDeletionTaskConfig; } @@ -120,13 +118,6 @@ export interface TilesDeletionTaskConfig { taskBatchSize: number; } -export interface TilesSeedingTaskConfig { - type: string; - grid: string; - maxZoom: number; - skipUncached: boolean; -} - export interface CacheDeletionTaskConfig { type: string; maxZoom: number; diff --git a/src/utils/geoUtils.ts b/src/utils/geoUtils.ts deleted file mode 100644 index f89c0c8..0000000 --- a/src/utils/geoUtils.ts +++ /dev/null @@ -1,88 +0,0 @@ -import { feature, featureCollection, bbox, bboxPolygon, intersect } from '@turf/turf'; -import { featureToTilesCount } from '@map-colonies/mc-utils'; -import type { Feature, MultiPolygon, Polygon, BBox } from 'geojson'; - -/** - * Splits a geometry into smaller sub-geometries based on tile count limitations. - * - * This function is used when a geometry at a specific zoom level would generate - * more tiles than the maximum allowed per task. It divides the geometry into a - * grid of smaller geometries, each containing approximately the maximum number - * of tiles or fewer. - * - * The splitting process: - * 1. Calculates the total number of tiles the geometry would generate at the given zoom level - * 2. Determines the split factor (grid dimensions) needed to stay under the tile limit - * 3. Creates a grid of bounding boxes that cover the original geometry - * 4. Intersects each grid cell with the original geometry to create sub-geometries - * 5. Returns only the intersecting geometries (non-empty intersections) - * - * @param geometry - The polygon or multipolygon to split - * @param zoomLevel - The zoom level at which to calculate tile counts - * @param maxTiles - The maximum number of tiles allowed per sub-geometry - * @returns Array of sub-geometries, each containing at most maxTiles tiles - * - * @example - * ```typescript - * const largePolygon = { type: 'Polygon', coordinates: [...] }; - * const subGeometries = splitGeometryByTileCount(largePolygon, 18, 100000); - * // Returns multiple smaller geometries, each generating ≤100,000 tiles at zoom 18 - * ``` - */ -export function splitGeometryByTileCount(geometry: Polygon | MultiPolygon, zoomLevel: number, maxTiles: number): Feature[] { - const geometryBbox = bbox(geometry); - const [minX, minY, maxX, maxY] = geometryBbox; - - // Step 1: Calculate total tiles in the bbox - // This gives us the worst-case scenario - how many tiles would be generated - // if we rendered the entire bounding box at the given zoom level - const totalTiles = featureToTilesCount(feature(geometry), zoomLevel); - - // Step 2: Calculate the split factor - // The split factor determines how many pieces we need to divide the geometry into. - // We use the square root because we're creating a 2D grid (splitFactor x splitFactor). - // - // Example: If totalTiles = 1,000,000 and maxTiles = 100,000: - // - We need at least 10 pieces (1,000,000 / 100,000 = 10) - // - For a square grid: √10 ≈ 3.16, so we need at least a 4x4 grid (16 pieces) - // - This ensures each piece has ≤ 62,500 tiles (1,000,000 / 16) - const splitFactor = Math.ceil(Math.sqrt(totalTiles / maxTiles)); - - // Step 3: Calculate step sizes for the grid - // Divide the bounding box into equal-sized rectangles - // xStep = width of each grid cell, yStep = height of each grid cell - const xStep = (maxX - minX) / splitFactor; - const yStep = (maxY - minY) / splitFactor; - - const splitGeometries: Feature[] = []; - - // Step 4: Create grid of sub-geometries - // Generate splitFactor x splitFactor grid cells and intersect each with the original geometry - for (let i = 0; i < splitFactor; i++) { - for (let j = 0; j < splitFactor; j++) { - // Step 4a: Create bounding box for this grid cell - // Each cell covers a rectangular area of the original bounding box - const subBbox: BBox = [ - minX + i * xStep, // left edge - minY + j * yStep, // bottom edge - minX + (i + 1) * xStep, // right edge - minY + (j + 1) * yStep, // top edge - ]; - - // Step 4b: Convert the grid cell bbox to a polygon - const subPolygon = bboxPolygon(subBbox); - - // Step 4c: Find the intersection between the grid cell and original geometry - // This gives us only the part of the original geometry that falls within this grid cell - const intersection = intersect(featureCollection([subPolygon, feature(geometry)])); - - // Step 4d: Only keep non-empty intersections - // Some grid cells might not overlap with the geometry at all (e.g., if geometry has holes or irregular shape) - if (intersection) { - splitGeometries.push(intersection); - } - } - } - - return splitGeometries; -} diff --git a/tests/unit/mocks/configMock.ts b/tests/unit/mocks/configMock.ts index f7f5645..d037d2e 100644 --- a/tests/unit/mocks/configMock.ts +++ b/tests/unit/mocks/configMock.ts @@ -156,9 +156,6 @@ const registerDefaultConfig = (): void => { }, }, jobs: { - seed: { - type: 'Ingestion_Seed', - }, updateCacheDeletion: { type: 'Update_Delete_Cache', }, @@ -172,14 +169,6 @@ const registerDefaultConfig = (): void => { tileBatchSize: 10000, taskBatchSize: 2, }, - tilesSeeding: { - type: 'tilesSeeding', - grid: 'WorldCRS84', - maxZoom: 21, - skipUncached: true, - zoomThreshold: 16, - maxTilesPerSeedTask: 500000, - }, tilesDeletion: { type: 'tiles-deletion', tileBatchSize: 10000,