feat: add forbidden layers configuration for deletion (MAPCO-11281)

config/custom-environment-variables.json

@@ -107,6 +107,12 @@
       "__format": "json"
     }
   },
+  "deleteLayer": {
+    "forbiddenLayers": {
+      "__name": "FORBIDDEN_LAYERS_FOR_DELETION",
+      "__format": "json"
+    }
+  },
   "httpRetry": {
     "attempts": {
       "__name": "HTTP_RETRY_ATTEMPTS",

config/default.json

@@ -72,6 +72,9 @@
     ],
     "forbiddenJobTypesForParallelIngestion": ["Ingestion_New", "Ingestion_Update", "Ingestion_Swap_Update", "Delete_Layer"]
   },
+  "deleteLayer": {
+    "forbiddenLayers": []
+  },
   "httpRetry": {
     "attempts": 5,
     "delay": "exponential",

helm/templates/configmap.yaml

@@ -29,6 +29,7 @@ data:
   STORAGE_EXPLORER_VALID_FILE_EXTENSIONS: {{ .Values.env.storageExplorer.validFileExtensions | toJson | quote }}
   FORBIDDEN_TYPES_FOR_PARALLEL_INGESTION: {{ .Values.env.forbiddenJobTypesForParallelIngestion | toJson | quote }}
   SUPPORTED_INGESTION_SWAP_TYPES: {{ .Values.env.supportedIngestionSwapTypes | toJson | quote }}
+  FORBIDDEN_LAYERS_FOR_DELETION: {{ .Values.env.deleteLayer.forbiddenLayers | toJson | quote }}
   HTTP_RETRY_ATTEMPTS: {{ .Values.env.httpRetry.attempts | quote }}
   HTTP_RETRY_DELAY: {{ .Values.env.httpRetry.delay | quote }}
   HTTP_RETRY_RESET_TIMEOUT: {{ .Values.env.httpRetry.resetTimeout | quote }}

helm/values.yaml

@@ -174,6 +174,10 @@ env:
   - Ingestion_New
   - Ingestion_Update
   - Ingestion_Swap_Update
+  deleteLayer:
+    # layers that are protected from deletion, in the format of <productId>-<productType>
+    forbiddenLayers: []
+    # - 'VIVID_IHUD-Orthophoto'
 
 resources:
   enabled: false

openapi3.yaml

@@ -153,6 +153,13 @@ paths:
               schema:
                 $ref: >-
                   ./Schema/ingestionTrigger/responses/ingestionTriggerResponses.yaml#/components/schemas/errorMessage
+        '403':
+          description: Forbidden
+          content:
+            application/json:
+              schema:
+                $ref: >-
+                  ./Schema/ingestionTrigger/responses/ingestionTriggerResponses.yaml#/components/schemas/errorMessage
         '404':
           description: Not Found
           content:

src/ingestion/controllers/ingestionController.ts

@@ -1,4 +1,4 @@
-import { BadRequestError, ConflictError, NotFoundError } from '@map-colonies/error-types';
+import { BadRequestError, ConflictError, ForbiddenError, NotFoundError } from '@map-colonies/error-types';
 import type { RequestHandler } from 'express';
 import { HttpError } from 'express-openapi-validator/dist/framework/types';
 import { StatusCodes } from 'http-status-codes';
@@ -124,7 +124,9 @@ export class IngestionController {
 
       res.status(StatusCodes.OK).send(response);
     } catch (error) {
-      if (error instanceof NotFoundError) {
+      if (error instanceof ForbiddenError) {
+        (error as HttpError).status = StatusCodes.FORBIDDEN; //403
+      } else if (error instanceof NotFoundError) {
         (error as HttpError).status = StatusCodes.NOT_FOUND; //404
       } else if (error instanceof ConflictError) {
         (error as HttpError).status = StatusCodes.CONFLICT; //409

src/ingestion/models/ingestionManager.ts

@@ -1,6 +1,6 @@
 import { randomUUID } from 'node:crypto';
 import { relative } from 'node:path';
-import { ConflictError, NotFoundError } from '@map-colonies/error-types';
+import { ConflictError, ForbiddenError, NotFoundError } from '@map-colonies/error-types';
 import type { Logger } from '@map-colonies/js-logger';
 import {
   type IFindJobsByCriteriaBody,
@@ -24,6 +24,7 @@ import {
   type IngestionUpdateJobParams,
   type IngestionValidationTaskParams,
   type InputFiles,
+  type LayerName,
   type RasterProductTypes,
 } from '@map-colonies/raster-shared';
 import { withSpanAsyncV4, withSpanV4 } from '@map-colonies/telemetry';
@@ -80,6 +81,7 @@ export class IngestionManager {
   private readonly validationTaskType: string;
   private readonly finalizeTaskType: string;
   private readonly deleteTaskType: string;
+  private readonly forbiddenLayersForDeletion: LayerName[];
   private readonly sourceMount: string;
   private readonly jobTrackerServiceUrl: string;
 
@@ -113,6 +115,7 @@ export class IngestionManager {
     this.validationTaskType = config.get('jobManager.validationTaskType') as unknown as string;
     this.finalizeTaskType = config.get('jobManager.finalizeTaskType') as unknown as string;
     this.deleteTaskType = config.get('jobManager.deleteTaskType') as unknown as string;
+    this.forbiddenLayersForDeletion = config.get('deleteLayer.forbiddenLayers') as unknown as LayerName[];
     this.sourceMount = config.get('storageExplorer.layerSourceDir') as unknown as string;
     this.jobTrackerServiceUrl = config.get('services.jobTrackerServiceURL') as unknown as string;
   }
@@ -218,6 +221,7 @@ export class IngestionManager {
     activeSpan?.updateName('ingestionManager.deleteLayer');
 
     const rasterLayerMetadata = await this.getLayerMetadata(catalogId);
+    this.validateLayerIsNotForbiddenForDeletion(rasterLayerMetadata);
     this.validateLayerIsUnpublished(rasterLayerMetadata);
     await this.validateNoParallelJobs(rasterLayerMetadata.productId, rasterLayerMetadata.productType);
 
@@ -707,6 +711,21 @@ export class IngestionManager {
     return createJobRequest;
   }
 
+  @withSpanV4
+  private validateLayerIsNotForbiddenForDeletion(rasterLayerMetadata: RasterLayerMetadata): void {
+    const logCtx: LogContext = { ...this.logContext, function: this.validateLayerIsNotForbiddenForDeletion.name };
+    const { productId, productType } = rasterLayerMetadata;
+    const layerName = getMapServingLayerName(productId, productType);
+
+    if (this.forbiddenLayersForDeletion.includes(layerName)) {
+      const message = `Layer: ${layerName}, is configured as a forbidden layer for deletion and therefore cannot be deleted`;
+      this.logger.error({ msg: message, logContext: logCtx, layerName });
+      const error = new ForbiddenError(message);
+      trace.getActiveSpan()?.setAttribute('exception.type', error.status);
+      throw error;
+    }
+  }
+
   @withSpanV4
   private validateLayerIsUnpublished(rasterLayerMetadata: RasterLayerMetadata): void {
     if (rasterLayerMetadata.productStatus !== RecordStatus.UNPUBLISHED) {

tests/integration/ingestion/ingestion.spec.ts

@@ -27,6 +27,7 @@ import {
   createUpdateLayerRequest,
   generateCallbackUrl,
   generateMockJob,
+  getForbiddenLayerForDeletion,
   rasterLayerInputFilesGenerators,
   rasterLayerMetadataGenerators,
 } from '../../mocks/mockFactory';
@@ -2877,6 +2878,20 @@ describe('Ingestion', () => {
         }
       );
 
+      it('should return 403 status code when the layer is configured as forbidden for deletion', async () => {
+        const approver = faker.person.fullName();
+        const { productId, productType } = getForbiddenLayerForDeletion();
+        const catalogLayerResponse = createCatalogLayerResponse({ metadata: { productId, productType, productStatus: RecordStatus.UNPUBLISHED } });
+        const scope = nock(jobManagerURL).post('/jobs').reply(httpStatusCodes.OK, jobResponse);
+        nock(catalogServiceURL).post('/records/find', { id: catalogLayerResponse.metadata.id }).reply(httpStatusCodes.OK, [catalogLayerResponse]);
+
+        const response = await requestSender.deleteLayer(catalogLayerResponse.metadata.id, { approver });
+
+        expect(response).toSatisfyApiSpec();
+        expect(response.status).toBe(httpStatusCodes.FORBIDDEN);
+        expect(scope.isDone()).toBe(false);
+      });
+
       it('should return 409 status code when there are conflicting jobs', async () => {
         const approver = faker.person.fullName();
         const catalogLayerResponse = createCatalogLayerResponse({ metadata: { productStatus: RecordStatus.UNPUBLISHED } });

tests/mocks/configMock.ts

@@ -105,6 +105,9 @@ const registerDefaultConfig = (): void => {
       ],
       forbiddenJobTypesForParallelIngestion: ['Ingestion_New', 'Ingestion_Update', 'Delete_Layer'],
     },
+    deleteLayer: {
+      forbiddenLayers: ['VIVID_IHUD-Orthophoto'],
+    },
   };
 
   setConfigValues(config);

tests/mocks/mockFactory.ts

@@ -17,6 +17,7 @@ import {
   type IngestionSwapUpdateJobParams,
   type IngestionUpdateJobParams,
   type InputFiles,
+  type LayerName,
   type NewRasterLayerMetadata,
   type UpdateRasterLayerMetadata,
   JobTypes,
@@ -482,6 +483,15 @@ export const createCatalogLayerResponse = (rasterLayerCatalog?: DeepPartial<Rast
   return mergedRasterLayerCatalog;
 };
 
+/**
+ * Returns the first configured forbidden layer for deletion, split into its productId and productType parts
+ */
+export const getForbiddenLayerForDeletion = (): { forbiddenLayerName: LayerName; productId: string; productType: RasterProductTypes } => {
+  const forbiddenLayerName = configMock.get<LayerName[]>('deleteLayer.forbiddenLayers')[0]!;
+  const [productId, productType] = forbiddenLayerName.split('-') as [string, RasterProductTypes];
+  return { forbiddenLayerName, productId, productType };
+};
+
 export const createFindJobsParams = (findJobsParams: IFindJobsByCriteriaBody): IFindJobsByCriteriaBody => {
   const defaultFindJobsParams = {
     isCleaned: false,

tests/unit/ingestion/models/ingestionManager.spec.ts

@@ -1,8 +1,8 @@
 import { faker } from '@faker-js/faker';
-import { BadRequestError, ConflictError, NotFoundError } from '@map-colonies/error-types';
+import { BadRequestError, ConflictError, ForbiddenError, NotFoundError } from '@map-colonies/error-types';
 import { jsLogger } from '@map-colonies/js-logger';
 import { ICreateJobResponse, OperationStatus } from '@map-colonies/mc-priority-queue';
-import { getMapServingLayerName } from '@map-colonies/raster-shared';
+import { getMapServingLayerName, RasterProductTypes } from '@map-colonies/raster-shared';
 import { RecordStatus } from '@map-colonies/types';
 import { trace } from '@opentelemetry/api';
 import { container } from 'tsyringe';
@@ -27,6 +27,7 @@ import {
   generateMockJob,
   generateNewLayerRequest,
   generateUpdateLayerRequest,
+  getForbiddenLayerForDeletion,
 } from '../../../mocks/mockFactory';
 import { ChecksumProcessor } from '../../../../src/utils/hash/interfaces';
 import { CHECKSUM_PROCESSOR } from '../../../../src/utils/hash/constants';
@@ -680,6 +681,51 @@ describe('IngestionManager', () => {
       expect(createIngestionJobSpy).not.toHaveBeenCalled();
     });
 
+    it('should throw forbidden error when the layer is configured as forbidden for deletion', async () => {
+      const approver = faker.person.fullName();
+      const { forbiddenLayerName, productId, productType } = getForbiddenLayerForDeletion();
+      const catalogLayerResponse = createCatalogLayerResponse({ metadata: { productId, productType, productStatus: RecordStatus.UNPUBLISHED } });
+      const expectedErrorMessage = `Layer: ${forbiddenLayerName}, is configured as a forbidden layer for deletion and therefore cannot be deleted`;
+      findByIdSpy.mockResolvedValue([catalogLayerResponse]);
+
+      const promise = ingestionManager.deleteLayer(catalogLayerResponse.metadata.id, { approver });
+
+      await expect(promise).rejects.toThrow(new ForbiddenError(expectedErrorMessage));
+      expect(createIngestionJobSpy).not.toHaveBeenCalled();
+    });
+
+    it('should create delete layer job when only the productId matches a forbidden layer', async () => {
+      const approver = faker.person.fullName();
+      const { productId } = getForbiddenLayerForDeletion();
+      const catalogLayerResponse = createCatalogLayerResponse({
+        metadata: { productId, productType: RasterProductTypes.RASTER_MAP, productStatus: RecordStatus.UNPUBLISHED },
+      });
+      const createJobResponse: ICreateJobResponse = { id: faker.string.uuid(), taskIds: [faker.string.uuid()] };
+      findByIdSpy.mockResolvedValue([catalogLayerResponse]);
+      findJobsSpy.mockResolvedValue([]);
+      createIngestionJobSpy.mockResolvedValue(createJobResponse);
+
+      const response = await ingestionManager.deleteLayer(catalogLayerResponse.metadata.id, { approver });
+
+      expect(response).toStrictEqual({ jobId: createJobResponse.id, taskId: createJobResponse.taskIds[0] });
+    });
+
+    it('should create delete layer job when only the productType matches a forbidden layer', async () => {
+      const approver = faker.person.fullName();
+      const { productType } = getForbiddenLayerForDeletion();
+      const catalogLayerResponse = createCatalogLayerResponse({
+        metadata: { productType, productStatus: RecordStatus.UNPUBLISHED },
+      });
+      const createJobResponse: ICreateJobResponse = { id: faker.string.uuid(), taskIds: [faker.string.uuid()] };
+      findByIdSpy.mockResolvedValue([catalogLayerResponse]);
+      findJobsSpy.mockResolvedValue([]);
+      createIngestionJobSpy.mockResolvedValue(createJobResponse);
+
+      const response = await ingestionManager.deleteLayer(catalogLayerResponse.metadata.id, { approver });
+
+      expect(response).toStrictEqual({ jobId: createJobResponse.id, taskId: createJobResponse.taskIds[0] });
+    });
+
     it('should throw an error when job manager create delete layer job call throws an error', async () => {
       const approver = faker.person.fullName();
       const catalogLayerResponse = createCatalogLayerResponse({ metadata: { productStatus: RecordStatus.UNPUBLISHED } });