Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 11 additions & 7 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -143,18 +143,22 @@ The service can be configured using environment variables or a configuration fil
| INGESTION_NEW_JOB_TYPE | string | New ingestion job type | `"Ingestion_New"` |
| INGESTION_UPDATE_JOB_TYPE | string | Update ingestion job type | `"Ingestion_Update"` |
| INGESTION_SWAP_UPDATE_JOB_TYPE | string | Swap update job type | `"Ingestion_Swap_Update"` |
| INGESTION_SEED_JOB_TYPE | string | Seed job type | `"Ingestion_Seed"` |
| TILES_MERGING_TASK_TYPE | string | Tiles merging task type | `"tiles-merging"` |
| TILES_MERGING_TILE_BATCH_SIZE | number | Batch size for tile merging | `10000` |
| TILES_MERGING_TASK_BATCH_SIZE | number | Batch size for task merging | `5` |
| TILES_MERGING_RADIUS_BUFFER | number | Radius buffer for merging | `0` |
| TILES_MERGING_RADIUS_BUFFER_UNITS | string | Units for radius buffer | `"meters"` |
| TILES_MERGING_TRUNCATE_PRECISION | number | Precision for truncating | `6` |
| TILES_MERGING_TRUNCATE_COORDINATES | number | Coordinates for truncating | `6` |
| TILES_SEEDING_TASK_TYPE | string | Tiles seeding task type | `"tiles-seeding"` |
| TILES_SEEDING_GRID | string | Grid configuration for tiles seeding | `"WorldCRS84"` |
| TILES_SEEDING_MAX_ZOOM | number | Maximum zoom level for seeding | `21` |
| TILES_SEEDING_SKIP_UNCACHED | boolean | Skip uncached tiles during seeding | `true` |
| UPDATE_CACHE_DELETION_JOB_TYPE | string | Cache deletion job type created after an update ingestion | `"Update_Delete_Cache"` |
| SWAP_CACHE_DELETION_JOB_TYPE | string | Cache deletion job type created after a swap-update ingestion | `"Swap_Delete_Cache"` |
| CACHE_DELETION_TASK_TYPE | string | Cache deletion task type | `"tiles-deletion"` |
| CACHE_DELETION_MAX_ZOOM | number | Maximum zoom level for the deleted tile ranges | `21` |
| CACHE_DELETION_TILE_BATCH_SIZE | number | Maximum tiles per range-deletion task | `100000` |
| CACHE_DELETION_MAX_RANGES_PER_TASK | number | Maximum tile ranges per task, bounding the serialized task params | `5000` |
| CACHE_DELETION_TASK_BATCH_SIZE | number | Number of tasks enqueued per job-manager call | `5` |
| CACHE_DELETION_GRACEFUL_RELOAD_MAX_SECONDS | number | Serving pods' config reload window, waited out before wiping cache keys | `300` |
| CACHE_DELETION_RELOAD_WINDOW_MARGIN_SECONDS | number | Margin added to the reload window | `8` |

## Export Configuration

Expand Down Expand Up @@ -203,12 +207,12 @@ For both processing types, the service handles two primary task phases:

- **For Update Ingestion:**
- Updates catalog layers
- Creates seed jobs and tasks
- Creates a cache deletion job whose tasks delete the ingested footprint's tile ranges from MapProxy's redis cache

- **For Swap Update Ingestion:**
- Updates catalog layers
- Handles layer updates in MapProxy when required
- Creates seed jobs and tasks
- Creates a cache deletion job whose task wipes the layer's entire redis cache prefix

## Export Processing

Expand Down
27 changes: 0 additions & 27 deletions config/custom-environment-variables.json
Original file line number Diff line number Diff line change
Expand Up @@ -146,9 +146,6 @@
}
},
"jobs": {
"seed": {
"type": "INGESTION_SEED_JOB_TYPE"
},
"updateCacheDeletion": {
"type": "UPDATE_CACHE_DELETION_JOB_TYPE"
},
Expand Down Expand Up @@ -200,30 +197,6 @@
"__name": "TILES_MERGING_TRUNCATE_COORDINATES",
"__format": "number"
}
},
"tilesSeeding": {
"type": "TILES_SEEDING_TASK_TYPE",
"grid": "TILES_SEEDING_GRID",
"maxZoom": {
"__name": "TILES_SEEDING_MAX_ZOOM",
"__format": "number"
},
"skipUncached": {
"__name": "TILES_SEEDING_SKIP_UNCACHED",
"__format": "boolean"
},
"zoomThreshold": {
"__name": "TILES_SEEDING_ZOOM_THRESHOLD",
"__format": "number"
},
"maxTilesPerSeedTask": {
"__name": "TILES_SEEDING_MAX_TILES_PER_SEED_TASK",
"__format": "number"
},
"maxTilesPerCleanTask": {
"__name": "TILES_SEEDING_MAX_TILES_PER_CLEAN_TASK",
"__format": "number"
}
}
}
},
Expand Down
12 changes: 0 additions & 12 deletions config/default.json
Original file line number Diff line number Diff line change
Expand Up @@ -113,9 +113,6 @@
}
},
"jobs": {
"seed": {
"type": "Ingestion_Seed"
},
"updateCacheDeletion": {
"type": "Update_Delete_Cache"
},
Expand All @@ -133,15 +130,6 @@
"truncatePrecision": 11,
"truncateCoordinates": 2
},
"tilesSeeding": {
"type": "tiles-seeding",
"grid": "WorldCRS84",
"maxZoom": 21,
"skipUncached": true,
"zoomThreshold": 16,
"maxTilesPerSeedTask": 500000,
"maxTilesPerCleanTask": 4000000
},
"validation": {
"type": "validation"
},
Expand Down
8 changes: 0 additions & 8 deletions helm/templates/configmap.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,6 @@ data:
INGESTION_UPDATE_JOB_TYPE: {{ $jobDefinitions.jobs.update.type | quote }}
INGESTION_SWAP_UPDATE_JOB_TYPE: {{ $jobDefinitions.jobs.swapUpdate.type | quote }}
INGESTION_DELETE_LAYER_JOB_TYPE: {{ $jobDefinitions.jobs.deleteLayer.type | quote }}
INGESTION_SEED_JOB_TYPE : {{ $jobDefinitions.jobs.seed.type | quote }}
UPDATE_CACHE_DELETION_JOB_TYPE: {{ $jobDefinitions.jobs.updateCacheDeletion.type | quote }}
SWAP_CACHE_DELETION_JOB_TYPE: {{ $jobDefinitions.jobs.swapCacheDeletion.type | quote }}
EXPORT_JOB_TYPE: {{ $jobDefinitions.jobs.export.type | quote }}
Expand All @@ -77,13 +76,6 @@ data:
TILES_MERGING_RADIUS_BUFFER_UNITS: {{ $jobDefinitions.tasks.merge.radiusBufferUnits | quote }}
TILES_MERGING_TRUNCATE_PRECISION: {{ $jobDefinitions.tasks.merge.truncatePrecision | quote }}
TILES_MERGING_TRUNCATE_COORDINATES: {{ $jobDefinitions.tasks.merge.truncateCoordinates | quote }}
TILES_SEEDING_TASK_TYPE: {{ $jobDefinitions.tasks.seed.type | quote }}
TILES_SEEDING_GRID: {{ $jobDefinitions.tasks.seed.grid | quote }}
TILES_SEEDING_MAX_ZOOM: {{ $jobDefinitions.tasks.seed.maxZoom | quote }}
TILES_SEEDING_SKIP_UNCACHED: {{ $jobDefinitions.tasks.seed.skipUncached | quote }}
TILES_SEEDING_ZOOM_THRESHOLD: {{ $jobDefinitions.tasks.seed.zoomThreshold | quote }}
TILES_SEEDING_MAX_TILES_PER_SEED_TASK: {{ $jobDefinitions.tasks.seed.maxTilesPerSeedTask | quote }}
TILES_SEEDING_MAX_TILES_PER_CLEAN_TASK: {{ $jobDefinitions.tasks.seed.maxTilesPerCleanTask | quote }}
CACHE_DELETION_TASK_TYPE: {{ $jobDefinitions.tasks.cacheDeletion.type | quote }}
CACHE_DELETION_MAX_ZOOM: {{ $jobDefinitions.tasks.cacheDeletion.maxZoom | quote }}
CACHE_DELETION_TILE_BATCH_SIZE: {{ $jobDefinitions.tasks.cacheDeletion.tileBatchSize | quote }}
Expand Down
10 changes: 0 additions & 10 deletions helm/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -141,8 +141,6 @@ jobDefinitions:
type: ""
deleteLayer:
type: ""
seed:
type: ""
updateCacheDeletion:
type: ""
swapCacheDeletion:
Expand All @@ -168,14 +166,6 @@ jobDefinitions:
radiusBufferUnits: "degrees" # supported values are: "degrees" | "centimeters" | "meters" | "millimeters" | "kilometers" | "miles" | "inches" | "yards" | "feet" | "radians"
truncatePrecision: 11 # coordinate decimal precision
truncateCoordinates: 2 # maximum number of coordinates (primarly used to remove z coordinates)
seed:
type: ""
grid: "WorldCRS84"
maxZoom: 21
skipUncached: true
zoomThreshold: 16 #zoom level threshold for high res seeding first seeding task will be 0->zoomThreshold
maxTilesPerSeedTask: 250000
maxTilesPerCleanTask: 500000
export:
type: ""
tilesDeletion:
Expand Down
9 changes: 0 additions & 9 deletions src/common/interfaces.ts
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,6 @@ export interface JobConfig {
}

export interface IngestionJobsConfig {
seed: JobConfig | undefined;
updateCacheDeletion: JobConfig | undefined;
swapCacheDeletion: JobConfig | undefined;
}
Expand All @@ -81,7 +80,6 @@ export interface ExportPollingJobsConfig {

export interface IngestionTasksConfig {
tilesMerging: TilesMergingTaskConfig;
tilesSeeding: TilesSeedingTaskConfig;
tilesDeletion: TilesDeletionTaskConfig;
cacheDeletion: CacheDeletionTaskConfig;
}
Expand Down Expand Up @@ -120,13 +118,6 @@ export interface TilesDeletionTaskConfig {
taskBatchSize: number;
}

export interface TilesSeedingTaskConfig {
type: string;
grid: string;
maxZoom: number;
skipUncached: boolean;
}

export interface CacheDeletionTaskConfig {
type: string;
maxZoom: number;
Expand Down
88 changes: 0 additions & 88 deletions src/utils/geoUtils.ts

This file was deleted.

11 changes: 0 additions & 11 deletions tests/unit/mocks/configMock.ts
Original file line number Diff line number Diff line change
Expand Up @@ -156,9 +156,6 @@ const registerDefaultConfig = (): void => {
},
},
jobs: {
seed: {
type: 'Ingestion_Seed',
},
updateCacheDeletion: {
type: 'Update_Delete_Cache',
},
Expand All @@ -172,14 +169,6 @@ const registerDefaultConfig = (): void => {
tileBatchSize: 10000,
taskBatchSize: 2,
},
tilesSeeding: {
type: 'tilesSeeding',
grid: 'WorldCRS84',
maxZoom: 21,
skipUncached: true,
zoomThreshold: 16,
maxTilesPerSeedTask: 500000,
},
tilesDeletion: {
type: 'tiles-deletion',
tileBatchSize: 10000,
Expand Down
Loading