From 78cd529cc14905adc37496a5040d34960df052b5 Mon Sep 17 00:00:00 2001 From: almog8k Date: Thu, 6 Aug 2026 16:29:23 +0300 Subject: [PATCH 1/7] feat: carry tiles storage on the tiles-deletion task params (MAPCO-11294) Adopt the provider-aware tilesDeletionParamsSchema from raster-shared 9.0.0-alpha.0 so the Cleaner locates tiles from the task params instead of its own configuration. The range-based tiles-deletion params are now a discriminated union on storageProvider, so the flat shape no longer typechecks: - FS -> { storageProvider, subPath } from storage.internalPvc.tilesSubPath - S3 -> { storageProvider, bucket } from S3.tilesBucket - tilesPath -> tilesRelativePath, sourceProvider -> storageProvider Storage is resolved once in the constructor and spread into every emitted task, mirroring DeleteLayerHandler.createCleanerTasks. Note: the emitted task params contract changed. The Cleaner keeps reading the FS sub-path from its own config until MAPCO-11295 lands. --- package-lock.json | 8 +- package.json | 2 +- src/task/models/deletionTaskManager.ts | 25 ++++- tests/unit/mocks/configMock.ts | 4 + .../tileDeletionTaskManager.spec.ts | 101 +++++++++++++++++- 5 files changed, 128 insertions(+), 12 deletions(-) diff --git a/package-lock.json b/package-lock.json index 3cbe2db..c048a32 100644 --- a/package-lock.json +++ b/package-lock.json @@ -22,7 +22,7 @@ "@map-colonies/mc-priority-queue": "^9.1.2", "@map-colonies/mc-utils": "^6.0.1", "@map-colonies/prometheus": "^1.0.0", - "@map-colonies/raster-shared": "^8.3.0-alpha.2", + "@map-colonies/raster-shared": "^9.0.0-alpha.0", "@map-colonies/read-pkg": "^1.0.0", "@map-colonies/schemas": "^1.18.0", "@map-colonies/shapefile-reader": "^1.0.1", @@ -6593,9 +6593,9 @@ } }, "node_modules/@map-colonies/raster-shared": { - "version": "8.3.0-alpha.2", - "resolved": "https://registry.npmjs.org/@map-colonies/raster-shared/-/raster-shared-8.3.0-alpha.2.tgz", - "integrity": "sha512-P2p0MddLonzxsG/MSkcKopfI05JsaWxe9BpfYPrVllZ7wCbrzyBUnsYsfP3Rl6jHeNlqWWPCJdHkpzvgTi0FOQ==", + "version": "9.0.0-alpha.0", + "resolved": "https://registry.npmjs.org/@map-colonies/raster-shared/-/raster-shared-9.0.0-alpha.0.tgz", + "integrity": "sha512-7NUVOMnlGaeSDYZu84wEXI25SyL/DatD/LHVBiGVP+CYWZ/8KjEe9WMH84EIPZS4VAlNpRbNZ770582AJNRjxQ==", "license": "ISC", "dependencies": { "@map-colonies/mc-priority-queue": "^9.1.0", diff --git a/package.json b/package.json index 655f7e4..70a5424 100644 --- a/package.json +++ b/package.json @@ -48,7 +48,7 @@ "@map-colonies/mc-priority-queue": "^9.1.2", "@map-colonies/mc-utils": "^6.0.1", "@map-colonies/prometheus": "^1.0.0", - "@map-colonies/raster-shared": "^8.3.0-alpha.2", + "@map-colonies/raster-shared": "^9.0.0-alpha.0", "@map-colonies/read-pkg": "^1.0.0", "@map-colonies/schemas": "^1.18.0", "@map-colonies/shapefile-reader": "^1.0.1", diff --git a/src/task/models/deletionTaskManager.ts b/src/task/models/deletionTaskManager.ts index 90a8f80..887a6c7 100644 --- a/src/task/models/deletionTaskManager.ts +++ b/src/task/models/deletionTaskManager.ts @@ -5,7 +5,14 @@ import { feature as turfFeature, featureCollection as turfFeatureCollection, uni import { inject, injectable } from 'tsyringe'; import type { Logger } from '@map-colonies/js-logger'; import { ShapefileChunkReader } from '@map-colonies/shapefile-reader'; -import type { IntersectionFeatureCollection, IngestionValidationTaskParams, TilesDeletionParams } from '@map-colonies/raster-shared'; +import type { + IntersectionFeatureCollection, + IngestionValidationTaskParams, + TilesDeletionParams, + FsStorage, + S3Storage, +} from '@map-colonies/raster-shared'; +import { SourceType } from '@map-colonies/raster-shared'; import type { ICreateTaskBody, ITaskResponse } from '@map-colonies/mc-priority-queue'; import { TaskHandler as QueueClient } from '@map-colonies/mc-priority-queue'; import type { MultiPolygon, Polygon } from 'geojson'; @@ -23,7 +30,7 @@ export class TileDeletionTaskManager { private readonly tileBatchSize: number; private readonly taskBatchSize: number; private readonly taskType: string; - private readonly sourceProvider: StorageProvider; + private readonly tilesStorage: S3Storage | FsStorage; private readonly shapefileReader: ShapefileChunkReader; public constructor( @@ -38,7 +45,7 @@ export class TileDeletionTaskManager { this.tileBatchSize = this.config.get('jobManagement.ingestion.tasks.tilesDeletion.tileBatchSize'); this.taskBatchSize = this.config.get('jobManagement.ingestion.tasks.tilesDeletion.taskBatchSize'); this.taskType = this.config.get('jobManagement.ingestion.tasks.tilesDeletion.type'); - this.sourceProvider = this.config.get('tilesStorageProvider'); + this.tilesStorage = this.resolveTilesStorage(); this.shapefileReader = new ShapefileChunkReader({ maxVerticesPerChunk: this.config.get('shapefileReader.maxVerticesPerChunk'), }); @@ -226,10 +233,10 @@ export class TileDeletionTaskManager { for await (const batch of batches) { const taskParameters: TilesDeletionParams = { - tilesPath: layerRelativePath, + ...this.tilesStorage, + tilesRelativePath: layerRelativePath, ranges: batch, fileExtension: tileOutputFormat.toLowerCase(), - sourceProvider: this.sourceProvider, }; yield { @@ -243,6 +250,14 @@ export class TileDeletionTaskManager { } } + private resolveTilesStorage(): S3Storage | FsStorage { + const provider = this.config.get('tilesStorageProvider'); + + return provider === SourceType.S3 + ? { storageProvider: provider, bucket: this.config.get('S3.tilesBucket') } + : { storageProvider: provider, subPath: this.config.get('storage.internalPvc.tilesSubPath') }; + } + private async fetchValidationTask(jobId: string): Promise> { const validationTaskType = this.config.get('jobManagement.ingestion.tasks.validation.type'); const validationTasks = await this.queueClient.jobManagerClient.findTasks({ diff --git a/tests/unit/mocks/configMock.ts b/tests/unit/mocks/configMock.ts index 469982a..b3b6b3b 100644 --- a/tests/unit/mocks/configMock.ts +++ b/tests/unit/mocks/configMock.ts @@ -92,6 +92,10 @@ const registerDefaultConfig = (): void => { maxVerticesPerChunk: 2500, }, ingestionSourcesDirPath: '/layerSources', + // eslint-disable-next-line @typescript-eslint/naming-convention + S3: { + tilesBucket: 'tiles-bucket', + }, tilesStorageProvider: 'FS', gpkgStorageProvider: 'FS', storage: { diff --git a/tests/unit/task/tileDeletionTaskManager/tileDeletionTaskManager.spec.ts b/tests/unit/task/tileDeletionTaskManager/tileDeletionTaskManager.spec.ts index 73a4c5c..9bac058 100644 --- a/tests/unit/task/tileDeletionTaskManager/tileDeletionTaskManager.spec.ts +++ b/tests/unit/task/tileDeletionTaskManager/tileDeletionTaskManager.spec.ts @@ -1,8 +1,10 @@ /* eslint-disable @typescript-eslint/unbound-method */ import { NotFoundError } from '@map-colonies/error-types'; -import type { TaskBlockDuplicationParam } from '@map-colonies/raster-shared'; +import type { TaskBlockDuplicationParam, TilesDeletionParams } from '@map-colonies/raster-shared'; +import { tilesDeletionParamsSchema } from '@map-colonies/raster-shared'; +import type { ICreateTaskBody } from '@map-colonies/mc-priority-queue'; import type { Polygon } from 'geojson'; -import { configMock, registerDefaultConfig } from '../../mocks/configMock'; +import { configMock, registerDefaultConfig, setValue } from '../../mocks/configMock'; import { ingestionUpdateJob } from '../../mocks/jobsMockData'; import { createFakeTask, @@ -193,6 +195,101 @@ describe('TileDeletionTaskManager', () => { }); }); + describe('tiles storage on the built task parameters', () => { + // a single conflict feature that yields at least one tile batch + const conflictGeometry: Polygon = { + type: 'Polygon', + coordinates: [ + [ + [0, 0], + [1, 0], + [1, 1], + [0, 1], + [0, 0], + ], + ], + }; + + const arrangeSingleIntersection = (): void => { + jobManagerClientMock.findTasks.mockResolvedValue([validationTaskWithResolutionErrors]); + vi.spyOn(reportUtils, 'readConflictFeatures').mockResolvedValue([ + // eslint-disable-next-line @typescript-eslint/naming-convention + { type: 'Feature', geometry: conflictGeometry, properties: { e_res: 'Resolution Conflict' } }, + ]); + // one zoom level with an intersection, then empty so the upward iteration stops + polygonPartsMangerClientMock.getIntersection + .mockResolvedValueOnce({ type: 'FeatureCollection', features: [{ type: 'Feature', geometry: conflictGeometry, properties: {} }] }) + .mockResolvedValue({ type: 'FeatureCollection', features: [] }); + }; + + const getCreatedTasks = (): TilesDeletionParams[] => { + const batches = jobManagerClientMock.createTaskForJob.mock.calls.map(([, batch]) => batch) as ICreateTaskBody[][]; + return batches.flat().map((task) => task.parameters); + }; + + it('should carry the configured subPath on every task when tiles are stored on FS', async () => { + setValue('tilesStorageProvider', 'FS'); + setValue('storage.internalPvc.tilesSubPath', 'raster/artifacts/tiles'); + const { tileDeletionTaskManager } = await setupTileDeletionTaskManagerTest(); + arrangeSingleIntersection(); + + await tileDeletionTaskManager.buildAndPushTasks(ingestionUpdateJob, task, polygonPartsEntityName, layerRelativePath); + + const createdTasks = getCreatedTasks(); + + expect(createdTasks.length).toBeGreaterThan(0); + + createdTasks.forEach((parameters) => { + expect(parameters).toStrictEqual({ + storageProvider: 'FS', + subPath: 'raster/artifacts/tiles', + tilesRelativePath: layerRelativePath, + fileExtension: ingestionUpdateJob.parameters.additionalParams.tileOutputFormat.toLowerCase(), + ranges: expect.any(Array) as unknown, + }); + }); + }); + + it('should carry the configured bucket on every task when tiles are stored on S3', async () => { + setValue('tilesStorageProvider', 'S3'); + setValue('S3.tilesBucket', 'tiles-bucket'); + const { tileDeletionTaskManager } = await setupTileDeletionTaskManagerTest(); + arrangeSingleIntersection(); + + await tileDeletionTaskManager.buildAndPushTasks(ingestionUpdateJob, task, polygonPartsEntityName, layerRelativePath); + + const createdTasks = getCreatedTasks(); + + expect(createdTasks.length).toBeGreaterThan(0); + + createdTasks.forEach((parameters) => { + expect(parameters).toStrictEqual({ + storageProvider: 'S3', + bucket: 'tiles-bucket', + tilesRelativePath: layerRelativePath, + fileExtension: ingestionUpdateJob.parameters.additionalParams.tileOutputFormat.toLowerCase(), + ranges: expect.any(Array) as unknown, + }); + }); + }); + + it('should produce task parameters that satisfy the raster-shared tiles-deletion schema', async () => { + setValue('tilesStorageProvider', 'FS'); + const { tileDeletionTaskManager } = await setupTileDeletionTaskManagerTest(); + arrangeSingleIntersection(); + + await tileDeletionTaskManager.buildAndPushTasks(ingestionUpdateJob, task, polygonPartsEntityName, layerRelativePath); + + const createdTasks = getCreatedTasks(); + + expect(createdTasks.length).toBeGreaterThan(0); + + createdTasks.forEach((parameters) => { + expect(() => tilesDeletionParamsSchema.parse(parameters)).not.toThrow(); + }); + }); + }); + describe('fetchValidationTask (via buildAndPushTasks)', () => { it('should use the configured validation task type when querying findTasks', async () => { const { tileDeletionTaskManager } = testContext; From 0280ac362c5c9b066ee4ab5c0edb72987bf33012 Mon Sep 17 00:00:00 2001 From: almog8k Date: Sun, 9 Aug 2026 13:29:02 +0300 Subject: [PATCH 2/7] refactor: remove unused StorageProvider type and update tile storage resolution logic --- src/common/constants.ts | 2 -- src/task/models/deletionTaskManager.ts | 10 +++++++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/common/constants.ts b/src/common/constants.ts index ee93d12..11f9aeb 100644 --- a/src/common/constants.ts +++ b/src/common/constants.ts @@ -1,8 +1,6 @@ import { SourceType } from '@map-colonies/raster-shared'; import { readPackageJsonSync } from '@map-colonies/read-pkg'; -export type StorageProvider = Exclude; - export const SERVICE_NAME = readPackageJsonSync().name ?? 'unknown_service'; export const SERVICE_VERSION = readPackageJsonSync().version ?? 'unknown_version'; export const DEFAULT_SERVER_PORT = 80; diff --git a/src/task/models/deletionTaskManager.ts b/src/task/models/deletionTaskManager.ts index 887a6c7..9048a5f 100644 --- a/src/task/models/deletionTaskManager.ts +++ b/src/task/models/deletionTaskManager.ts @@ -12,13 +12,13 @@ import type { FsStorage, S3Storage, } from '@map-colonies/raster-shared'; -import { SourceType } from '@map-colonies/raster-shared'; +import { StorageProvider } from '@map-colonies/raster-shared'; import type { ICreateTaskBody, ITaskResponse } from '@map-colonies/mc-priority-queue'; import { TaskHandler as QueueClient } from '@map-colonies/mc-priority-queue'; import type { MultiPolygon, Polygon } from 'geojson'; import { NotFoundError, UnprocessableEntityError } from '@map-colonies/error-types'; import type { IConfig, BuildDeletionTaskParams } from '../../common/interfaces'; -import { SERVICES, StorageProvider } from '../../common/constants'; +import { SERVICES } from '../../common/constants'; import { TaskMetrics } from '../../utils/metrics/taskMetrics'; import { createChildSpan } from '../../common/tracing'; import { IngestionCreateTasksTask, IngestionUpdateCreateTasksJob } from '../../utils/zod/schemas/job.schema'; @@ -253,7 +253,11 @@ export class TileDeletionTaskManager { private resolveTilesStorage(): S3Storage | FsStorage { const provider = this.config.get('tilesStorageProvider'); - return provider === SourceType.S3 + if (provider === StorageProvider.REDIS) { + throw new Error('Redis storage provider is not supported for tile deletion tasks'); + } + + return provider === StorageProvider.S3 ? { storageProvider: provider, bucket: this.config.get('S3.tilesBucket') } : { storageProvider: provider, subPath: this.config.get('storage.internalPvc.tilesSubPath') }; } From d337f3ce746e91318c4a60c57ddf914877a699c5 Mon Sep 17 00:00:00 2001 From: almog8k Date: Sun, 9 Aug 2026 13:51:25 +0300 Subject: [PATCH 3/7] refactor: rename getCreatedTasks to getTasksParametersParams for clarity --- .../tileDeletionTaskManager.spec.ts | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/tests/unit/task/tileDeletionTaskManager/tileDeletionTaskManager.spec.ts b/tests/unit/task/tileDeletionTaskManager/tileDeletionTaskManager.spec.ts index 9bac058..b93b3d8 100644 --- a/tests/unit/task/tileDeletionTaskManager/tileDeletionTaskManager.spec.ts +++ b/tests/unit/task/tileDeletionTaskManager/tileDeletionTaskManager.spec.ts @@ -222,7 +222,7 @@ describe('TileDeletionTaskManager', () => { .mockResolvedValue({ type: 'FeatureCollection', features: [] }); }; - const getCreatedTasks = (): TilesDeletionParams[] => { + const getTasksParametersParams = (): TilesDeletionParams[] => { const batches = jobManagerClientMock.createTaskForJob.mock.calls.map(([, batch]) => batch) as ICreateTaskBody[][]; return batches.flat().map((task) => task.parameters); }; @@ -235,11 +235,11 @@ describe('TileDeletionTaskManager', () => { await tileDeletionTaskManager.buildAndPushTasks(ingestionUpdateJob, task, polygonPartsEntityName, layerRelativePath); - const createdTasks = getCreatedTasks(); + const tasksParameters = getTasksParametersParams(); - expect(createdTasks.length).toBeGreaterThan(0); + expect(tasksParameters.length).toBeGreaterThan(0); - createdTasks.forEach((parameters) => { + tasksParameters.forEach((parameters) => { expect(parameters).toStrictEqual({ storageProvider: 'FS', subPath: 'raster/artifacts/tiles', @@ -258,11 +258,11 @@ describe('TileDeletionTaskManager', () => { await tileDeletionTaskManager.buildAndPushTasks(ingestionUpdateJob, task, polygonPartsEntityName, layerRelativePath); - const createdTasks = getCreatedTasks(); + const tasksParameters = getTasksParametersParams(); - expect(createdTasks.length).toBeGreaterThan(0); + expect(tasksParameters.length).toBeGreaterThan(0); - createdTasks.forEach((parameters) => { + tasksParameters.forEach((parameters) => { expect(parameters).toStrictEqual({ storageProvider: 'S3', bucket: 'tiles-bucket', @@ -280,11 +280,11 @@ describe('TileDeletionTaskManager', () => { await tileDeletionTaskManager.buildAndPushTasks(ingestionUpdateJob, task, polygonPartsEntityName, layerRelativePath); - const createdTasks = getCreatedTasks(); + const tasksParameters = getTasksParametersParams(); - expect(createdTasks.length).toBeGreaterThan(0); + expect(tasksParameters.length).toBeGreaterThan(0); - createdTasks.forEach((parameters) => { + tasksParameters.forEach((parameters) => { expect(() => tilesDeletionParamsSchema.parse(parameters)).not.toThrow(); }); }); From a6015dcd62d3f3eb09f17cc92c1e94fce157d9cc Mon Sep 17 00:00:00 2001 From: almog8k Date: Sun, 9 Aug 2026 14:02:09 +0300 Subject: [PATCH 4/7] refactor: update tile deletion task parameters to use specific types for ranges --- .../tileDeletionTaskManager.spec.ts | 33 +++++++------------ 1 file changed, 11 insertions(+), 22 deletions(-) diff --git a/tests/unit/task/tileDeletionTaskManager/tileDeletionTaskManager.spec.ts b/tests/unit/task/tileDeletionTaskManager/tileDeletionTaskManager.spec.ts index b93b3d8..40727b4 100644 --- a/tests/unit/task/tileDeletionTaskManager/tileDeletionTaskManager.spec.ts +++ b/tests/unit/task/tileDeletionTaskManager/tileDeletionTaskManager.spec.ts @@ -1,7 +1,12 @@ /* eslint-disable @typescript-eslint/unbound-method */ import { NotFoundError } from '@map-colonies/error-types'; -import type { TaskBlockDuplicationParam, TilesDeletionParams } from '@map-colonies/raster-shared'; -import { tilesDeletionParamsSchema } from '@map-colonies/raster-shared'; +import type { + FsTilesDeletionParams, + S3TilesDeletionParams, + TaskBlockDuplicationParam, + TileRange, + TilesDeletionParams, +} from '@map-colonies/raster-shared'; import type { ICreateTaskBody } from '@map-colonies/mc-priority-queue'; import type { Polygon } from 'geojson'; import { configMock, registerDefaultConfig, setValue } from '../../mocks/configMock'; @@ -245,8 +250,8 @@ describe('TileDeletionTaskManager', () => { subPath: 'raster/artifacts/tiles', tilesRelativePath: layerRelativePath, fileExtension: ingestionUpdateJob.parameters.additionalParams.tileOutputFormat.toLowerCase(), - ranges: expect.any(Array) as unknown, - }); + ranges: expect.any(Array) as TileRange[], + } satisfies FsTilesDeletionParams); }); }); @@ -268,24 +273,8 @@ describe('TileDeletionTaskManager', () => { bucket: 'tiles-bucket', tilesRelativePath: layerRelativePath, fileExtension: ingestionUpdateJob.parameters.additionalParams.tileOutputFormat.toLowerCase(), - ranges: expect.any(Array) as unknown, - }); - }); - }); - - it('should produce task parameters that satisfy the raster-shared tiles-deletion schema', async () => { - setValue('tilesStorageProvider', 'FS'); - const { tileDeletionTaskManager } = await setupTileDeletionTaskManagerTest(); - arrangeSingleIntersection(); - - await tileDeletionTaskManager.buildAndPushTasks(ingestionUpdateJob, task, polygonPartsEntityName, layerRelativePath); - - const tasksParameters = getTasksParametersParams(); - - expect(tasksParameters.length).toBeGreaterThan(0); - - tasksParameters.forEach((parameters) => { - expect(() => tilesDeletionParamsSchema.parse(parameters)).not.toThrow(); + ranges: expect.any(Array) as TileRange[], + } satisfies S3TilesDeletionParams); }); }); }); From f6272a69c485ccda3098b8f15e86bf19ea9b2641 Mon Sep 17 00:00:00 2001 From: almog8k Date: Sun, 9 Aug 2026 15:00:14 +0300 Subject: [PATCH 5/7] refactor: update storage provider references to use StorageProvider type --- src/common/constants.ts | 7 ++++--- src/httpClients/mapproxyClient.ts | 4 ++-- src/job/models/deletion/deleteLayerHandler.ts | 15 ++++++++------- src/job/models/export/exportJobHandler.ts | 10 ++-------- src/task/models/exportTaskManager.ts | 5 +++-- src/task/models/tileMergeTaskManager.ts | 6 +++--- 6 files changed, 22 insertions(+), 25 deletions(-) diff --git a/src/common/constants.ts b/src/common/constants.ts index 11f9aeb..bda0753 100644 --- a/src/common/constants.ts +++ b/src/common/constants.ts @@ -1,4 +1,4 @@ -import { SourceType } from '@map-colonies/raster-shared'; +import { StorageProvider } from '@map-colonies/raster-shared'; import { readPackageJsonSync } from '@map-colonies/read-pkg'; export const SERVICE_NAME = readPackageJsonSync().name ?? 'unknown_service'; @@ -40,8 +40,9 @@ export const LayerCacheType = { export type LayerCacheType = (typeof LayerCacheType)[keyof typeof LayerCacheType]; export const storageProviderToCacheTypeMap = new Map([ - [SourceType.FS, LayerCacheType.FS], - [SourceType.S3, LayerCacheType.S3], + [StorageProvider.FS, LayerCacheType.FS], + [StorageProvider.S3, LayerCacheType.S3], + [StorageProvider.REDIS, LayerCacheType.REDIS], ]); export const SeedMode = { diff --git a/src/httpClients/mapproxyClient.ts b/src/httpClients/mapproxyClient.ts index 120b13f..0c4136f 100644 --- a/src/httpClients/mapproxyClient.ts +++ b/src/httpClients/mapproxyClient.ts @@ -1,11 +1,11 @@ import type { Logger } from '@map-colonies/js-logger'; -import type { LayerName, TileOutputFormat } from '@map-colonies/raster-shared'; +import type { LayerName, StorageProvider, TileOutputFormat } from '@map-colonies/raster-shared'; import { context, SpanStatusCode, trace, type Tracer } from '@opentelemetry/api'; import { HttpClient, type IHttpRetryConfig } from '@map-colonies/mc-utils'; import { inject, injectable } from 'tsyringe'; import { NotFoundError } from '@map-colonies/error-types'; import type { IConfig, GetMapproxyCacheRequest, GetMapproxyCacheResponse, PublishMapLayerRequest } from '../common/interfaces'; -import { LayerCacheType, SERVICES, storageProviderToCacheTypeMap, StorageProvider } from '../common/constants'; +import { LayerCacheType, SERVICES, storageProviderToCacheTypeMap } from '../common/constants'; import { DeleteLayerError, LayerCacheNotFoundError, diff --git a/src/job/models/deletion/deleteLayerHandler.ts b/src/job/models/deletion/deleteLayerHandler.ts index 2c86664..af0bbf7 100644 --- a/src/job/models/deletion/deleteLayerHandler.ts +++ b/src/job/models/deletion/deleteLayerHandler.ts @@ -1,10 +1,10 @@ import type { Logger } from '@map-colonies/js-logger'; import { context, trace, type Tracer } from '@opentelemetry/api'; import { TaskHandler as QueueClient, type ICreateTaskBody } from '@map-colonies/mc-priority-queue'; -import { SourceType, type DeleteTaskParams, type DeleteStoredResourcesParams, type LayerName, Storage } from '@map-colonies/raster-shared'; +import { type DeleteTaskParams, type DeleteStoredResourcesParams, type LayerName, StorageProvider, Storage } from '@map-colonies/raster-shared'; import { inject, injectable } from 'tsyringe'; import type { IConfig, IJobHandler, JobAndTaskTelemetry, StepKey } from '../../../common/interfaces'; -import { SERVICES, type StorageProvider } from '../../../common/constants'; +import { SERVICES } from '../../../common/constants'; import { LayerCacheNotFoundError } from '../../../common/errors'; import { CatalogClient } from '../../../httpClients/catalogClient'; import { GeoserverClient } from '../../../httpClients/geoserverClient'; @@ -30,7 +30,8 @@ interface DeletionStep { @injectable() export class DeleteLayerHandler extends JobHandler implements IJobHandler { private readonly tilesDeletionType: string; - private readonly tilesStorageProvider: StorageProvider; + //TODO: when we support redis tiles deletion, change the type to StorageProvider and remove the Exclude<> wrapper + private readonly tilesStorageProvider: Exclude; private readonly tilesBucketConfig: string; private readonly tilesSubPathConfig: string; @@ -49,7 +50,7 @@ export class DeleteLayerHandler extends JobHandler implements IJobHandler('jobManagement.ingestion.tasks.tilesDeletion.type'); - this.tilesStorageProvider = this.config.get('tilesStorageProvider'); + this.tilesStorageProvider = this.config.get>('tilesStorageProvider'); this.tilesBucketConfig = this.config.get('S3.tilesBucket'); this.tilesSubPathConfig = this.config.get('storage.internalPvc.tilesSubPath'); } @@ -136,7 +137,7 @@ export class DeleteLayerHandler extends JobHandler implements IJobHandler Date: Sun, 9 Aug 2026 15:12:49 +0300 Subject: [PATCH 6/7] refactor: reorganize test structure and improve clarity in tile deletion tests --- .../tileDeletionTaskManager.spec.ts | 128 +++++++++--------- 1 file changed, 64 insertions(+), 64 deletions(-) diff --git a/tests/unit/task/tileDeletionTaskManager/tileDeletionTaskManager.spec.ts b/tests/unit/task/tileDeletionTaskManager/tileDeletionTaskManager.spec.ts index 40727b4..b6689d4 100644 --- a/tests/unit/task/tileDeletionTaskManager/tileDeletionTaskManager.spec.ts +++ b/tests/unit/task/tileDeletionTaskManager/tileDeletionTaskManager.spec.ts @@ -198,83 +198,83 @@ describe('TileDeletionTaskManager', () => { tileDeletionTaskManager.buildAndPushTasks(ingestionUpdateJob, task, polygonPartsEntityName, layerRelativePath) ).resolves.not.toThrow(); }); - }); - describe('tiles storage on the built task parameters', () => { - // a single conflict feature that yields at least one tile batch - const conflictGeometry: Polygon = { - type: 'Polygon', - coordinates: [ - [ - [0, 0], - [1, 0], - [1, 1], - [0, 1], - [0, 0], + describe('tiles storage on the built task parameters', () => { + // a single conflict feature that yields at least one tile batch + const conflictGeometry: Polygon = { + type: 'Polygon', + coordinates: [ + [ + [0, 0], + [1, 0], + [1, 1], + [0, 1], + [0, 0], + ], ], - ], - }; + }; - const arrangeSingleIntersection = (): void => { - jobManagerClientMock.findTasks.mockResolvedValue([validationTaskWithResolutionErrors]); - vi.spyOn(reportUtils, 'readConflictFeatures').mockResolvedValue([ - // eslint-disable-next-line @typescript-eslint/naming-convention - { type: 'Feature', geometry: conflictGeometry, properties: { e_res: 'Resolution Conflict' } }, - ]); - // one zoom level with an intersection, then empty so the upward iteration stops - polygonPartsMangerClientMock.getIntersection - .mockResolvedValueOnce({ type: 'FeatureCollection', features: [{ type: 'Feature', geometry: conflictGeometry, properties: {} }] }) - .mockResolvedValue({ type: 'FeatureCollection', features: [] }); - }; - - const getTasksParametersParams = (): TilesDeletionParams[] => { - const batches = jobManagerClientMock.createTaskForJob.mock.calls.map(([, batch]) => batch) as ICreateTaskBody[][]; - return batches.flat().map((task) => task.parameters); - }; - - it('should carry the configured subPath on every task when tiles are stored on FS', async () => { - setValue('tilesStorageProvider', 'FS'); - setValue('storage.internalPvc.tilesSubPath', 'raster/artifacts/tiles'); - const { tileDeletionTaskManager } = await setupTileDeletionTaskManagerTest(); - arrangeSingleIntersection(); + const arrangeSingleIntersection = (): void => { + jobManagerClientMock.findTasks.mockResolvedValue([validationTaskWithResolutionErrors]); + vi.spyOn(reportUtils, 'readConflictFeatures').mockResolvedValue([ + // eslint-disable-next-line @typescript-eslint/naming-convention + { type: 'Feature', geometry: conflictGeometry, properties: { e_res: 'Resolution Conflict' } }, + ]); + // one zoom level with an intersection, then empty so the upward iteration stops + polygonPartsMangerClientMock.getIntersection + .mockResolvedValueOnce({ type: 'FeatureCollection', features: [{ type: 'Feature', geometry: conflictGeometry, properties: {} }] }) + .mockResolvedValue({ type: 'FeatureCollection', features: [] }); + }; - await tileDeletionTaskManager.buildAndPushTasks(ingestionUpdateJob, task, polygonPartsEntityName, layerRelativePath); + const getTasksParametersParams = (): TilesDeletionParams[] => { + const batches = jobManagerClientMock.createTaskForJob.mock.calls.map(([, batch]) => batch) as ICreateTaskBody[][]; + return batches.flat().map((task) => task.parameters); + }; + + it('should carry the configured subPath on every task when tiles are stored on FS', async () => { + setValue('tilesStorageProvider', 'FS'); + setValue('storage.internalPvc.tilesSubPath', 'raster/artifacts/tiles'); + const { tileDeletionTaskManager } = await setupTileDeletionTaskManagerTest(); + arrangeSingleIntersection(); + + await tileDeletionTaskManager.buildAndPushTasks(ingestionUpdateJob, task, polygonPartsEntityName, layerRelativePath); - const tasksParameters = getTasksParametersParams(); + const tasksParameters = getTasksParametersParams(); - expect(tasksParameters.length).toBeGreaterThan(0); + expect(tasksParameters.length).toBeGreaterThan(0); - tasksParameters.forEach((parameters) => { - expect(parameters).toStrictEqual({ - storageProvider: 'FS', - subPath: 'raster/artifacts/tiles', - tilesRelativePath: layerRelativePath, - fileExtension: ingestionUpdateJob.parameters.additionalParams.tileOutputFormat.toLowerCase(), - ranges: expect.any(Array) as TileRange[], - } satisfies FsTilesDeletionParams); + tasksParameters.forEach((parameters) => { + expect(parameters).toStrictEqual({ + storageProvider: 'FS', + subPath: 'raster/artifacts/tiles', + tilesRelativePath: layerRelativePath, + fileExtension: ingestionUpdateJob.parameters.additionalParams.tileOutputFormat.toLowerCase(), + ranges: expect.any(Array) as TileRange[], + } satisfies FsTilesDeletionParams); + }); }); - }); - it('should carry the configured bucket on every task when tiles are stored on S3', async () => { - setValue('tilesStorageProvider', 'S3'); - setValue('S3.tilesBucket', 'tiles-bucket'); - const { tileDeletionTaskManager } = await setupTileDeletionTaskManagerTest(); - arrangeSingleIntersection(); + it('should carry the configured bucket on every task when tiles are stored on S3', async () => { + setValue('tilesStorageProvider', 'S3'); + setValue('S3.tilesBucket', 'tiles-bucket'); + const { tileDeletionTaskManager } = await setupTileDeletionTaskManagerTest(); + arrangeSingleIntersection(); - await tileDeletionTaskManager.buildAndPushTasks(ingestionUpdateJob, task, polygonPartsEntityName, layerRelativePath); + await tileDeletionTaskManager.buildAndPushTasks(ingestionUpdateJob, task, polygonPartsEntityName, layerRelativePath); - const tasksParameters = getTasksParametersParams(); + const tasksParameters = getTasksParametersParams(); - expect(tasksParameters.length).toBeGreaterThan(0); + expect(tasksParameters.length).toBeGreaterThan(0); - tasksParameters.forEach((parameters) => { - expect(parameters).toStrictEqual({ - storageProvider: 'S3', - bucket: 'tiles-bucket', - tilesRelativePath: layerRelativePath, - fileExtension: ingestionUpdateJob.parameters.additionalParams.tileOutputFormat.toLowerCase(), - ranges: expect.any(Array) as TileRange[], - } satisfies S3TilesDeletionParams); + tasksParameters.forEach((parameters) => { + expect(parameters).toStrictEqual({ + storageProvider: 'S3', + bucket: 'tiles-bucket', + tilesRelativePath: layerRelativePath, + fileExtension: ingestionUpdateJob.parameters.additionalParams.tileOutputFormat.toLowerCase(), + ranges: expect.any(Array) as TileRange[], + } satisfies S3TilesDeletionParams); + }); }); }); }); From f9cddcc0b5bd5e8585f81c7fa2100c95e343aea4 Mon Sep 17 00:00:00 2001 From: almog8k Date: Mon, 10 Aug 2026 13:50:57 +0300 Subject: [PATCH 7/7] refactor: rename getTasksParametersParams to getTasksParameters for clarity --- .../tileDeletionTaskManager/tileDeletionTaskManager.spec.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/unit/task/tileDeletionTaskManager/tileDeletionTaskManager.spec.ts b/tests/unit/task/tileDeletionTaskManager/tileDeletionTaskManager.spec.ts index b6689d4..696f158 100644 --- a/tests/unit/task/tileDeletionTaskManager/tileDeletionTaskManager.spec.ts +++ b/tests/unit/task/tileDeletionTaskManager/tileDeletionTaskManager.spec.ts @@ -226,7 +226,7 @@ describe('TileDeletionTaskManager', () => { .mockResolvedValue({ type: 'FeatureCollection', features: [] }); }; - const getTasksParametersParams = (): TilesDeletionParams[] => { + const getTasksParameters = (): TilesDeletionParams[] => { const batches = jobManagerClientMock.createTaskForJob.mock.calls.map(([, batch]) => batch) as ICreateTaskBody[][]; return batches.flat().map((task) => task.parameters); }; @@ -239,7 +239,7 @@ describe('TileDeletionTaskManager', () => { await tileDeletionTaskManager.buildAndPushTasks(ingestionUpdateJob, task, polygonPartsEntityName, layerRelativePath); - const tasksParameters = getTasksParametersParams(); + const tasksParameters = getTasksParameters(); expect(tasksParameters.length).toBeGreaterThan(0); @@ -262,7 +262,7 @@ describe('TileDeletionTaskManager', () => { await tileDeletionTaskManager.buildAndPushTasks(ingestionUpdateJob, task, polygonPartsEntityName, layerRelativePath); - const tasksParameters = getTasksParametersParams(); + const tasksParameters = getTasksParameters(); expect(tasksParameters.length).toBeGreaterThan(0);