feat!: create the cache deletion job instead of the seeding job (MAPCO-11265)

config/custom-environment-variables.json

@@ -148,6 +148,12 @@
       "jobs": {
         "seed": {
           "type": "INGESTION_SEED_JOB_TYPE"
+        },
+        "updateCacheDeletion": {
+          "type": "UPDATE_CACHE_DELETION_JOB_TYPE"
+        },
+        "swapCacheDeletion": {
+          "type": "SWAP_CACHE_DELETION_JOB_TYPE"
         }
       },
       "tasks": {
@@ -162,6 +168,15 @@
             "__format": "number"
           }
         },
+        "cacheDeletion": {
+          "type": "CACHE_DELETION_TASK_TYPE",
+          "maxZoom": { "__name": "CACHE_DELETION_MAX_ZOOM", "__format": "number" },
+          "tileBatchSize": { "__name": "CACHE_DELETION_TILE_BATCH_SIZE", "__format": "number" },
+          "maxRangesPerTask": { "__name": "CACHE_DELETION_MAX_RANGES_PER_TASK", "__format": "number" },
+          "taskBatchSize": { "__name": "CACHE_DELETION_TASK_BATCH_SIZE", "__format": "number" },
+          "gracefulReloadMaxSeconds": { "__name": "CACHE_DELETION_GRACEFUL_RELOAD_MAX_SECONDS", "__format": "number" },
+          "reloadWindowMarginSeconds": { "__name": "CACHE_DELETION_RELOAD_WINDOW_MARGIN_SECONDS", "__format": "number" }
+        },
         "tilesMerging": {
           "type": "TILES_MERGING_TASK_TYPE",
           "tileBatchSize": {

config/default.json

@@ -115,6 +115,12 @@
       "jobs": {
         "seed": {
           "type": "Ingestion_Seed"
+        },
+        "updateCacheDeletion": {
+          "type": "Update_Delete_Cache"
+        },
+        "swapCacheDeletion": {
+          "type": "Swap_Delete_Cache"
         }
       },
       "tasks": {
@@ -143,6 +149,15 @@
           "type": "tiles-deletion",
           "tileBatchSize": 10000,
           "taskBatchSize": 5
+        },
+        "cacheDeletion": {
+          "type": "tiles-deletion",
+          "maxZoom": 21,
+          "tileBatchSize": 100000,
+          "maxRangesPerTask": 5000,
+          "taskBatchSize": 5,
+          "gracefulReloadMaxSeconds": 300,
+          "reloadWindowMarginSeconds": 8
         }
       }
     },

helm/templates/configmap.yaml

@@ -58,6 +58,8 @@ data:
   INGESTION_SWAP_UPDATE_JOB_TYPE: {{ $jobDefinitions.jobs.swapUpdate.type | quote }}
   INGESTION_DELETE_LAYER_JOB_TYPE: {{ $jobDefinitions.jobs.deleteLayer.type | quote }}
   INGESTION_SEED_JOB_TYPE : {{ $jobDefinitions.jobs.seed.type | quote }}
+  UPDATE_CACHE_DELETION_JOB_TYPE: {{ $jobDefinitions.jobs.updateCacheDeletion.type | quote }}
+  SWAP_CACHE_DELETION_JOB_TYPE: {{ $jobDefinitions.jobs.swapCacheDeletion.type | quote }}
   EXPORT_JOB_TYPE: {{ $jobDefinitions.jobs.export.type | quote }}
   EXPORT_CLEANUP_EXPIRATION_DAYS: {{ $jobDefinitions.jobs.export.cleanupExpirationDays | quote }}
   TILES_DELETION_TASK_TYPE: {{ $jobDefinitions.tasks.tilesDeletion.type | quote }}
@@ -82,6 +84,13 @@ data:
   TILES_SEEDING_ZOOM_THRESHOLD: {{ $jobDefinitions.tasks.seed.zoomThreshold | quote }}
   TILES_SEEDING_MAX_TILES_PER_SEED_TASK: {{ $jobDefinitions.tasks.seed.maxTilesPerSeedTask | quote }}
   TILES_SEEDING_MAX_TILES_PER_CLEAN_TASK: {{ $jobDefinitions.tasks.seed.maxTilesPerCleanTask | quote }}
+  CACHE_DELETION_TASK_TYPE: {{ $jobDefinitions.tasks.cacheDeletion.type | quote }}
+  CACHE_DELETION_MAX_ZOOM: {{ $jobDefinitions.tasks.cacheDeletion.maxZoom | quote }}
+  CACHE_DELETION_TILE_BATCH_SIZE: {{ $jobDefinitions.tasks.cacheDeletion.tileBatchSize | quote }}
+  CACHE_DELETION_MAX_RANGES_PER_TASK: {{ $jobDefinitions.tasks.cacheDeletion.maxRangesPerTask | quote }}
+  CACHE_DELETION_TASK_BATCH_SIZE: {{ $jobDefinitions.tasks.cacheDeletion.taskBatchSize | quote }}
+  CACHE_DELETION_GRACEFUL_RELOAD_MAX_SECONDS: {{ .Values.global.gracefulReloadMaxSeconds | quote }}
+  CACHE_DELETION_RELOAD_WINDOW_MARGIN_SECONDS: {{ $jobDefinitions.tasks.cacheDeletion.reloadWindowMarginSeconds | quote }}
   TILES_EXPORTING_TASK_TYPE: {{ $jobDefinitions.tasks.export.type | quote }}
   MAPPROXY_API_URL: {{ $serviceUrls.mapproxyApi | quote }}
   GEOSERVER_API_URL: {{ $serviceUrls.geoserverApiUrl | quote }}

helm/values.yaml

@@ -143,6 +143,10 @@ jobDefinitions:
       type: ""
     seed:
       type: ""
+    updateCacheDeletion:
+      type: ""
+    swapCacheDeletion:
+      type: ""
     export:
       type: ""
       cleanupExpirationDays: 14
@@ -178,6 +182,13 @@ jobDefinitions:
       type: ""
       tileBatchSize: 10000
       taskBatchSize: 5
+    cacheDeletion:
+      type: ""
+      maxZoom: 21
+      tileBatchSize: 100000 # max tiles per range-deletion task
+      maxRangesPerTask: 5000 # bounds the serialized task params for jagged footprints
+      taskBatchSize: 5
+      reloadWindowMarginSeconds: 8 # added to global.gracefulReloadMaxSeconds: mapproxinator's poll plus drain
 
   config:
     dequeueIntervalMs: 3000

package.json

@@ -46,9 +46,9 @@
     "@map-colonies/js-logger": "^5.0.0",
     "@map-colonies/mc-model-types": "^17.15.1",
     "@map-colonies/mc-priority-queue": "^9.1.2",
-    "@map-colonies/mc-utils": "^6.0.1",
+    "@map-colonies/mc-utils": "^6.1.0",
     "@map-colonies/prometheus": "^1.0.0",
-    "@map-colonies/raster-shared": "^9.0.0-alpha.0",
+    "@map-colonies/raster-shared": "9.0.0-alpha.1",
     "@map-colonies/read-pkg": "^1.0.0",
     "@map-colonies/schemas": "^1.18.0",
     "@map-colonies/shapefile-reader": "^1.0.1",

src/common/constants.ts

@@ -44,10 +44,3 @@ export const storageProviderToCacheTypeMap = new Map([
   [StorageProvider.S3, LayerCacheType.S3],
   [StorageProvider.REDIS, LayerCacheType.REDIS],
 ]);
-
-export const SeedMode = {
-  SEED: 'seed',
-  CLEAN: 'clean',
-} as const;
-
-export type SeedMode = (typeof SeedMode)[keyof typeof SeedMode];

src/common/errors.ts

@@ -80,11 +80,18 @@ export class LayerCacheNotFoundError extends Error {
   }
 }
 
-export class SeedJobCreationError extends Error {
-  public constructor(msg: string, err: Error) {
-    super(msg);
-    this.name = SeedJobCreationError.name;
-    this.stack = err.stack;
+export class UnsupportedGridError extends Error {
+  public constructor(grid: string, supported: readonly string[]) {
+    super(`Unsupported grid (${grid}) for cache deletion, supported grids: ${supported.join(', ')}`);
+    this.name = UnsupportedGridError.name;
+  }
+}
+
+export class UnexpectedCacheGridsError extends Error {
+  public constructor(cacheName: string, grids: readonly string[] | undefined) {
+    const reported = grids === undefined ? 'none' : grids.join(', ');
+    super(`Expected exactly one grid for cache ${cacheName}, mapproxy reported: ${reported}`);
+    this.name = UnexpectedCacheGridsError.name;
   }
 }
 

src/common/interfaces.ts

@@ -10,6 +10,8 @@ import type {
   InputFiles,
   LayerName,
   RasterLayerMetadata,
+  RedisDeleteStoredResourcesParams,
+  RedisTilesDeletionParams,
   TileFormatStrategy,
   TileOutputFormat,
 } from '@map-colonies/raster-shared';
@@ -24,7 +26,7 @@ import type {
   ingestionSwapUpdateFinalizeJobParamsSchema,
   ingestionUpdateFinalizeJobParamsSchema,
 } from '../utils/zod/schemas/jobParameters.schema';
-import type { LayerCacheType, SeedMode } from './constants';
+import type { LayerCacheType } from './constants';
 
 export type StepKey<T> = keyof T & { [K in keyof T]: T[K] extends boolean ? K : never }[keyof T]; // this is a utility type that extracts the keys of T that are of type boolean
 
@@ -60,6 +62,8 @@ export interface JobConfig {
 
 export interface IngestionJobsConfig {
   seed: JobConfig | undefined;
+  updateCacheDeletion: JobConfig | undefined;
+  swapCacheDeletion: JobConfig | undefined;
 }
 
 export interface IngestionPollingJobsConfig {
@@ -79,6 +83,7 @@ export interface IngestionTasksConfig {
   tilesMerging: TilesMergingTaskConfig;
   tilesSeeding: TilesSeedingTaskConfig;
   tilesDeletion: TilesDeletionTaskConfig;
+  cacheDeletion: CacheDeletionTaskConfig;
 }
 
 export interface ExportTasksConfig {
@@ -122,6 +127,16 @@ export interface TilesSeedingTaskConfig {
   skipUncached: boolean;
 }
 
+export interface CacheDeletionTaskConfig {
+  type: string;
+  maxZoom: number;
+  tileBatchSize: number;
+  maxRangesPerTask: number;
+  taskBatchSize: number;
+  gracefulReloadMaxSeconds: number;
+  reloadWindowMarginSeconds: number;
+}
+
 export interface TilesExportingTaskConfig {
   type: string;
 }
@@ -339,6 +354,8 @@ export interface GetMapproxyCacheResponse {
   cacheName: string;
   // eslint-disable-next-line @typescript-eslint/naming-convention
   cache: { type: LayerCacheType; directory?: string; directory_layout?: string; bucket_name?: string };
+  /** the grids the cache is served on; mapproxy derives one redis key prefix per grid */
+  grids?: string[];
 }
 //#endregion mapproxyApi
 
@@ -373,31 +390,21 @@ export type CatalogUpdateMetadata = Partial<RasterLayerMetadata>;
 export type PolygonPartsProcessPayload = Pick<PolygonPartsPayload, 'productId' | 'productType'> & { shouldClearEntities?: boolean };
 //#endregion PolygonPartsManagerClient
 
-//#region seedingJobCreator
+//#region cacheDeletionJobCreator
 
-export interface SeedJobParams {
+export interface CacheDeletionJobParams {
   layerName: LayerName;
   ingestionJob: IngestionUpdateFinalizeJob | IngestionSwapUpdateFinalizeJob;
 }
-export interface SeedTaskOptions {
-  mode: SeedMode;
-  grid: string;
-  fromZoomLevel: number;
-  toZoomLevel: number;
-  geometry: Footprint;
-  skipUncached: boolean;
-  layerId: string; // cache name as configured in mapproxy
-  refreshBefore: string;
-}
 
-export interface SeedTaskParams {
-  seedTasks: SeedTaskOptions[];
-  catalogId: string;
-  traceParentContext?: TraceParentContext;
-  cacheType: LayerCacheType;
-}
+/**
+ * Params of a `tiles-deletion` cache-deletion task. Both shapes share that task type; which one a
+ * task carries follows from the job type - Update_Delete_Cache means `ranges`, Swap_Delete_Cache
+ * means prefix-only - and that is also how cleaner picks the strategy to run.
+ */
+export type CacheDeletionTaskParams = RedisTilesDeletionParams | RedisDeleteStoredResourcesParams;
 
-//#endregion seedingJobCreator
+//#endregion cacheDeletionJobCreator
 
 //#region telemetry
 export interface TraceParentContext {

src/httpClients/mapproxyClient.ts

@@ -125,7 +125,7 @@ export class MapproxyApiClient extends HttpClient {
     return this.fetchLayerCache(layerName, this.layerCacheType);
   }
 
-  public async getRedisCacheName(getCacheReq: GetMapproxyCacheRequest): Promise<string> {
+  public async getRedisCache(getCacheReq: GetMapproxyCacheRequest): Promise<GetMapproxyCacheResponse> {
     const { layerName, cacheType } = getCacheReq;
     const res = await this.fetchLayerCache(layerName, cacheType);
     if (res === undefined) {
@@ -134,7 +134,7 @@ export class MapproxyApiClient extends HttpClient {
     if (res.cache.type !== LayerCacheType.REDIS) {
       throw new UnsupportedLayerCacheError(layerName, cacheType);
     }
-    return res.cacheName;
+    return res;
   }
 
   private async fetchLayerCache(layerName: LayerName, cacheType: LayerCacheType): Promise<GetMapproxyCacheResponse | undefined> {