diff --git a/package-lock.json b/package-lock.json index 4fd8b62..66210eb 100644 --- a/package-lock.json +++ b/package-lock.json @@ -22,7 +22,7 @@ "@map-colonies/mc-priority-queue": "^9.1.2", "@map-colonies/mc-utils": "^6.0.1", "@map-colonies/prometheus": "^1.0.0", - "@map-colonies/raster-shared": "^8.3.0", + "@map-colonies/raster-shared": "^9.0.0-alpha.0", "@map-colonies/read-pkg": "^1.0.0", "@map-colonies/schemas": "^1.18.0", "@map-colonies/shapefile-reader": "^1.0.1", @@ -6593,9 +6593,9 @@ } }, "node_modules/@map-colonies/raster-shared": { - "version": "8.3.0", - "resolved": "https://registry.npmjs.org/@map-colonies/raster-shared/-/raster-shared-8.3.0.tgz", - "integrity": "sha512-5faR0iBC9Dpxm7Z8fY+3d+5oolHqVk/urXeOq3nHuIEMOvwRT2TrTr9fmt6TK8DaCjyQXBeEU1nijvuPLHmrEw==", + "version": "9.0.0-alpha.0", + "resolved": "https://registry.npmjs.org/@map-colonies/raster-shared/-/raster-shared-9.0.0-alpha.0.tgz", + "integrity": "sha512-7NUVOMnlGaeSDYZu84wEXI25SyL/DatD/LHVBiGVP+CYWZ/8KjEe9WMH84EIPZS4VAlNpRbNZ770582AJNRjxQ==", "license": "ISC", "dependencies": { "@map-colonies/mc-priority-queue": "^9.1.0", diff --git a/package.json b/package.json index 0bdc127..4f6fc7e 100644 --- a/package.json +++ b/package.json @@ -48,7 +48,7 @@ "@map-colonies/mc-priority-queue": "^9.1.2", "@map-colonies/mc-utils": "^6.0.1", "@map-colonies/prometheus": "^1.0.0", - "@map-colonies/raster-shared": "^8.3.0", + "@map-colonies/raster-shared": "^9.0.0-alpha.0", "@map-colonies/read-pkg": "^1.0.0", "@map-colonies/schemas": "^1.18.0", "@map-colonies/shapefile-reader": "^1.0.1", diff --git a/src/common/constants.ts b/src/common/constants.ts index ee93d12..bda0753 100644 --- a/src/common/constants.ts +++ b/src/common/constants.ts @@ -1,8 +1,6 @@ -import { SourceType } from '@map-colonies/raster-shared'; +import { StorageProvider } from '@map-colonies/raster-shared'; import { readPackageJsonSync } from '@map-colonies/read-pkg'; -export type StorageProvider = Exclude; - export const SERVICE_NAME = readPackageJsonSync().name ?? 'unknown_service'; export const SERVICE_VERSION = readPackageJsonSync().version ?? 'unknown_version'; export const DEFAULT_SERVER_PORT = 80; @@ -42,8 +40,9 @@ export const LayerCacheType = { export type LayerCacheType = (typeof LayerCacheType)[keyof typeof LayerCacheType]; export const storageProviderToCacheTypeMap = new Map([ - [SourceType.FS, LayerCacheType.FS], - [SourceType.S3, LayerCacheType.S3], + [StorageProvider.FS, LayerCacheType.FS], + [StorageProvider.S3, LayerCacheType.S3], + [StorageProvider.REDIS, LayerCacheType.REDIS], ]); export const SeedMode = { diff --git a/src/httpClients/mapproxyClient.ts b/src/httpClients/mapproxyClient.ts index 120b13f..0c4136f 100644 --- a/src/httpClients/mapproxyClient.ts +++ b/src/httpClients/mapproxyClient.ts @@ -1,11 +1,11 @@ import type { Logger } from '@map-colonies/js-logger'; -import type { LayerName, TileOutputFormat } from '@map-colonies/raster-shared'; +import type { LayerName, StorageProvider, TileOutputFormat } from '@map-colonies/raster-shared'; import { context, SpanStatusCode, trace, type Tracer } from '@opentelemetry/api'; import { HttpClient, type IHttpRetryConfig } from '@map-colonies/mc-utils'; import { inject, injectable } from 'tsyringe'; import { NotFoundError } from '@map-colonies/error-types'; import type { IConfig, GetMapproxyCacheRequest, GetMapproxyCacheResponse, PublishMapLayerRequest } from '../common/interfaces'; -import { LayerCacheType, SERVICES, storageProviderToCacheTypeMap, StorageProvider } from '../common/constants'; +import { LayerCacheType, SERVICES, storageProviderToCacheTypeMap } from '../common/constants'; import { DeleteLayerError, LayerCacheNotFoundError, diff --git a/src/job/models/deletion/deleteLayerHandler.ts b/src/job/models/deletion/deleteLayerHandler.ts index 2c86664..af0bbf7 100644 --- a/src/job/models/deletion/deleteLayerHandler.ts +++ b/src/job/models/deletion/deleteLayerHandler.ts @@ -1,10 +1,10 @@ import type { Logger } from '@map-colonies/js-logger'; import { context, trace, type Tracer } from '@opentelemetry/api'; import { TaskHandler as QueueClient, type ICreateTaskBody } from '@map-colonies/mc-priority-queue'; -import { SourceType, type DeleteTaskParams, type DeleteStoredResourcesParams, type LayerName, Storage } from '@map-colonies/raster-shared'; +import { type DeleteTaskParams, type DeleteStoredResourcesParams, type LayerName, StorageProvider, Storage } from '@map-colonies/raster-shared'; import { inject, injectable } from 'tsyringe'; import type { IConfig, IJobHandler, JobAndTaskTelemetry, StepKey } from '../../../common/interfaces'; -import { SERVICES, type StorageProvider } from '../../../common/constants'; +import { SERVICES } from '../../../common/constants'; import { LayerCacheNotFoundError } from '../../../common/errors'; import { CatalogClient } from '../../../httpClients/catalogClient'; import { GeoserverClient } from '../../../httpClients/geoserverClient'; @@ -30,7 +30,8 @@ interface DeletionStep { @injectable() export class DeleteLayerHandler extends JobHandler implements IJobHandler { private readonly tilesDeletionType: string; - private readonly tilesStorageProvider: StorageProvider; + //TODO: when we support redis tiles deletion, change the type to StorageProvider and remove the Exclude<> wrapper + private readonly tilesStorageProvider: Exclude; private readonly tilesBucketConfig: string; private readonly tilesSubPathConfig: string; @@ -49,7 +50,7 @@ export class DeleteLayerHandler extends JobHandler implements IJobHandler('jobManagement.ingestion.tasks.tilesDeletion.type'); - this.tilesStorageProvider = this.config.get('tilesStorageProvider'); + this.tilesStorageProvider = this.config.get>('tilesStorageProvider'); this.tilesBucketConfig = this.config.get('S3.tilesBucket'); this.tilesSubPathConfig = this.config.get('storage.internalPvc.tilesSubPath'); } @@ -136,7 +137,7 @@ export class DeleteLayerHandler extends JobHandler implements IJobHandler('jobManagement.ingestion.tasks.tilesDeletion.tileBatchSize'); this.taskBatchSize = this.config.get('jobManagement.ingestion.tasks.tilesDeletion.taskBatchSize'); this.taskType = this.config.get('jobManagement.ingestion.tasks.tilesDeletion.type'); - this.sourceProvider = this.config.get('tilesStorageProvider'); + this.tilesStorage = this.resolveTilesStorage(); this.shapefileReader = new ShapefileChunkReader({ maxVerticesPerChunk: this.config.get('shapefileReader.maxVerticesPerChunk'), }); @@ -226,10 +233,10 @@ export class TileDeletionTaskManager { for await (const batch of batches) { const taskParameters: TilesDeletionParams = { - tilesPath: layerRelativePath, + ...this.tilesStorage, + tilesRelativePath: layerRelativePath, ranges: batch, fileExtension: tileOutputFormat.toLowerCase(), - sourceProvider: this.sourceProvider, }; yield { @@ -243,6 +250,18 @@ export class TileDeletionTaskManager { } } + private resolveTilesStorage(): S3Storage | FsStorage { + const provider = this.config.get('tilesStorageProvider'); + + if (provider === StorageProvider.REDIS) { + throw new Error('Redis storage provider is not supported for tile deletion tasks'); + } + + return provider === StorageProvider.S3 + ? { storageProvider: provider, bucket: this.config.get('S3.tilesBucket') } + : { storageProvider: provider, subPath: this.config.get('storage.internalPvc.tilesSubPath') }; + } + private async fetchValidationTask(jobId: string): Promise> { const validationTaskType = this.config.get('jobManagement.ingestion.tasks.validation.type'); const validationTasks = await this.queueClient.jobManagerClient.findTasks({ diff --git a/src/task/models/exportTaskManager.ts b/src/task/models/exportTaskManager.ts index 7cf1506..e97f641 100644 --- a/src/task/models/exportTaskManager.ts +++ b/src/task/models/exportTaskManager.ts @@ -11,12 +11,13 @@ import { polygonSchema, RasterLayerMetadata, SourceType, + StorageProvider, type RoiFeature, type RoiFeatureCollection, } from '@map-colonies/raster-shared'; import { BBox2d, bboxToTileRange, degreesPerPixelToZoomLevel, type ITileRange } from '@map-colonies/mc-utils'; import type { BBox, Feature, MultiPolygon, Polygon } from 'geojson'; -import { SERVICES, StorageProvider } from '../../common/constants'; +import { SERVICES } from '../../common/constants'; import type { IConfig, TaskSources, ZoomBoundsParameters } from '../../common/interfaces'; import type { ExportJob } from '../../utils/zod/schemas/job.schema'; import { createChildSpan } from '../../common/tracing'; @@ -24,7 +25,7 @@ import { createChildSpan } from '../../common/tracing'; @injectable() export class ExportTaskManager { private readonly allWorldBounds: BBox; - private readonly tilesProvider: SourceType; + private readonly tilesProvider: StorageProvider; public constructor( @inject(SERVICES.LOGGER) private readonly logger: Logger, @inject(SERVICES.CONFIG) private readonly config: IConfig, diff --git a/src/task/models/tileMergeTaskManager.ts b/src/task/models/tileMergeTaskManager.ts index 6e181b5..8a9d707 100644 --- a/src/task/models/tileMergeTaskManager.ts +++ b/src/task/models/tileMergeTaskManager.ts @@ -5,7 +5,7 @@ import { degreesPerPixelToZoomLevel, tileBatchGenerator, TileRanger } from '@map import { bbox, feature } from '@turf/turf'; import { inject, injectable } from 'tsyringe'; import type { Logger } from '@map-colonies/js-logger'; -import { type InputFiles } from '@map-colonies/raster-shared'; +import { StorageProvider, type InputFiles } from '@map-colonies/raster-shared'; import type { ICreateTaskBody } from '@map-colonies/mc-priority-queue'; import { TaskHandler as QueueClient } from '@map-colonies/mc-priority-queue'; import type { @@ -20,7 +20,7 @@ import type { ZoomDefinitions, FeatureTask, } from '../../common/interfaces'; -import { SERVICES, type StorageProvider } from '../../common/constants'; +import { SERVICES } from '../../common/constants'; import { fileExtensionExtractor } from '../../utils/fileUtil'; import { TaskMetrics } from '../../utils/metrics/taskMetrics'; import { createChildSpan } from '../../common/tracing'; @@ -29,7 +29,7 @@ import { Grid } from '../../common/interfaces'; @injectable() export class TileMergeTaskManager { - private readonly tilesStorageProvider: string; + private readonly tilesStorageProvider: StorageProvider; private readonly tileBatchSize: number; private readonly taskBatchSize: number; private readonly taskType: string; diff --git a/tests/unit/mocks/configMock.ts b/tests/unit/mocks/configMock.ts index 469982a..b3b6b3b 100644 --- a/tests/unit/mocks/configMock.ts +++ b/tests/unit/mocks/configMock.ts @@ -92,6 +92,10 @@ const registerDefaultConfig = (): void => { maxVerticesPerChunk: 2500, }, ingestionSourcesDirPath: '/layerSources', + // eslint-disable-next-line @typescript-eslint/naming-convention + S3: { + tilesBucket: 'tiles-bucket', + }, tilesStorageProvider: 'FS', gpkgStorageProvider: 'FS', storage: { diff --git a/tests/unit/task/tileDeletionTaskManager/tileDeletionTaskManager.spec.ts b/tests/unit/task/tileDeletionTaskManager/tileDeletionTaskManager.spec.ts index 73a4c5c..696f158 100644 --- a/tests/unit/task/tileDeletionTaskManager/tileDeletionTaskManager.spec.ts +++ b/tests/unit/task/tileDeletionTaskManager/tileDeletionTaskManager.spec.ts @@ -1,8 +1,15 @@ /* eslint-disable @typescript-eslint/unbound-method */ import { NotFoundError } from '@map-colonies/error-types'; -import type { TaskBlockDuplicationParam } from '@map-colonies/raster-shared'; +import type { + FsTilesDeletionParams, + S3TilesDeletionParams, + TaskBlockDuplicationParam, + TileRange, + TilesDeletionParams, +} from '@map-colonies/raster-shared'; +import type { ICreateTaskBody } from '@map-colonies/mc-priority-queue'; import type { Polygon } from 'geojson'; -import { configMock, registerDefaultConfig } from '../../mocks/configMock'; +import { configMock, registerDefaultConfig, setValue } from '../../mocks/configMock'; import { ingestionUpdateJob } from '../../mocks/jobsMockData'; import { createFakeTask, @@ -191,6 +198,85 @@ describe('TileDeletionTaskManager', () => { tileDeletionTaskManager.buildAndPushTasks(ingestionUpdateJob, task, polygonPartsEntityName, layerRelativePath) ).resolves.not.toThrow(); }); + + describe('tiles storage on the built task parameters', () => { + // a single conflict feature that yields at least one tile batch + const conflictGeometry: Polygon = { + type: 'Polygon', + coordinates: [ + [ + [0, 0], + [1, 0], + [1, 1], + [0, 1], + [0, 0], + ], + ], + }; + + const arrangeSingleIntersection = (): void => { + jobManagerClientMock.findTasks.mockResolvedValue([validationTaskWithResolutionErrors]); + vi.spyOn(reportUtils, 'readConflictFeatures').mockResolvedValue([ + // eslint-disable-next-line @typescript-eslint/naming-convention + { type: 'Feature', geometry: conflictGeometry, properties: { e_res: 'Resolution Conflict' } }, + ]); + // one zoom level with an intersection, then empty so the upward iteration stops + polygonPartsMangerClientMock.getIntersection + .mockResolvedValueOnce({ type: 'FeatureCollection', features: [{ type: 'Feature', geometry: conflictGeometry, properties: {} }] }) + .mockResolvedValue({ type: 'FeatureCollection', features: [] }); + }; + + const getTasksParameters = (): TilesDeletionParams[] => { + const batches = jobManagerClientMock.createTaskForJob.mock.calls.map(([, batch]) => batch) as ICreateTaskBody[][]; + return batches.flat().map((task) => task.parameters); + }; + + it('should carry the configured subPath on every task when tiles are stored on FS', async () => { + setValue('tilesStorageProvider', 'FS'); + setValue('storage.internalPvc.tilesSubPath', 'raster/artifacts/tiles'); + const { tileDeletionTaskManager } = await setupTileDeletionTaskManagerTest(); + arrangeSingleIntersection(); + + await tileDeletionTaskManager.buildAndPushTasks(ingestionUpdateJob, task, polygonPartsEntityName, layerRelativePath); + + const tasksParameters = getTasksParameters(); + + expect(tasksParameters.length).toBeGreaterThan(0); + + tasksParameters.forEach((parameters) => { + expect(parameters).toStrictEqual({ + storageProvider: 'FS', + subPath: 'raster/artifacts/tiles', + tilesRelativePath: layerRelativePath, + fileExtension: ingestionUpdateJob.parameters.additionalParams.tileOutputFormat.toLowerCase(), + ranges: expect.any(Array) as TileRange[], + } satisfies FsTilesDeletionParams); + }); + }); + + it('should carry the configured bucket on every task when tiles are stored on S3', async () => { + setValue('tilesStorageProvider', 'S3'); + setValue('S3.tilesBucket', 'tiles-bucket'); + const { tileDeletionTaskManager } = await setupTileDeletionTaskManagerTest(); + arrangeSingleIntersection(); + + await tileDeletionTaskManager.buildAndPushTasks(ingestionUpdateJob, task, polygonPartsEntityName, layerRelativePath); + + const tasksParameters = getTasksParameters(); + + expect(tasksParameters.length).toBeGreaterThan(0); + + tasksParameters.forEach((parameters) => { + expect(parameters).toStrictEqual({ + storageProvider: 'S3', + bucket: 'tiles-bucket', + tilesRelativePath: layerRelativePath, + fileExtension: ingestionUpdateJob.parameters.additionalParams.tileOutputFormat.toLowerCase(), + ranges: expect.any(Array) as TileRange[], + } satisfies S3TilesDeletionParams); + }); + }); + }); }); describe('fetchValidationTask (via buildAndPushTasks)', () => {