From 2dfaf1d7d74916d21b4a1b720d1cd14170858084 Mon Sep 17 00:00:00 2001 From: TULCHINSKI LIRAN Date: Wed, 30 Sep 2026 13:29:06 +0300 Subject: [PATCH 1/7] feat: wire update, publish/unpublish and job status to real backends (MAPCO-11833) Replace the stub PATCH handlers and add a job-status endpoint, making the service a real successor to 3d-gateway + store-trigger: - PATCH /record/{id}: validate the update (record exists, not being-deleted, v2 partial schema, footprint, classification, unique name), reject records that exist in extractable-management (409), convert the footprint to 2D and patch the catalog. - PATCH /record/status/{id}: validate the record, extractable guard, then change status in the catalog. - GET /jobs/{jobId}: return the Jobnik job status and completion percentage. Add CatalogCall getRecord/patchMetadata/changeStatus, an ExtractableCall client (externalServices.extractable), JobnikClient.getJobStatus, and a job module (model/controller/route mounted at /jobs). Unit and integration tests included. --- config/custom-environment-variables.json | 1 + config/default.json | 1 + helm/templates/configmap.yaml | 1 + helm/values.yaml | 1 + openapi3.yaml | 37 ++++++++++ src/common/config.ts | 4 +- src/common/util.ts | 9 +++ src/containerConfig.ts | 2 + src/externalServices/catalog/catalogCall.ts | 40 +++++++++- src/externalServices/catalog/interfaces.ts | 6 ++ .../extractableManagement/extractableCall.ts | 39 ++++++++++ src/externalServices/jobnik/jobnikClient.ts | 20 +++++ src/job/controllers/jobController.ts | 34 +++++++++ src/job/models/jobManager.ts | 29 ++++++++ src/job/routes/jobRouter.ts | 16 ++++ src/openapi.d.ts | 55 ++++++++++++++ src/record/controllers/recordController.ts | 8 +- src/record/models/recordManager.ts | 28 +++++-- src/serverBuilder.ts | 5 +- src/validator/validationManager.ts | 72 ++++++++++++++++++ tests/integration/record/record.spec.ts | 21 ++++++ .../unit/record/models/recordManager.spec.ts | 22 ++++-- .../unit/validator/validationManager.spec.ts | 74 ++++++++++++++++++- 23 files changed, 507 insertions(+), 18 deletions(-) create mode 100644 src/externalServices/extractableManagement/extractableCall.ts create mode 100644 src/job/controllers/jobController.ts create mode 100644 src/job/models/jobManager.ts create mode 100644 src/job/routes/jobRouter.ts diff --git a/config/custom-environment-variables.json b/config/custom-environment-variables.json index 30bd2cd..777ce0d 100644 --- a/config/custom-environment-variables.json +++ b/config/custom-environment-variables.json @@ -1,6 +1,7 @@ { "externalServices": { "catalog": "CATALOG_URL", + "extractable": "EXTRACTABLE_URL", "lookupTables": { "url": "LOOKUP_TABLES_URL", "subUrl": "LOOKUP_TABLES_SUB_URL" diff --git a/config/default.json b/config/default.json index 8de50f9..d383775 100644 --- a/config/default.json +++ b/config/default.json @@ -35,6 +35,7 @@ }, "externalServices": { "catalog": "http://127.0.0.1:8080", + "extractable": "http://127.0.0.1:8080", "lookupTables": { "url": "http://127.0.0.1:8080", "subUrl": "lookup-tables/lookupData" diff --git a/helm/templates/configmap.yaml b/helm/templates/configmap.yaml index 9b572c3..5fde69a 100644 --- a/helm/templates/configmap.yaml +++ b/helm/templates/configmap.yaml @@ -32,6 +32,7 @@ data: npm_config_cache: /tmp/ {{- with .Values.externalServices }} CATALOG_URL: {{ .catalog | quote }} + EXTRACTABLE_URL: {{ .extractable | quote }} {{- with .lookupTables }} LOOKUP_TABLES_URL: {{ .url | quote }} LOOKUP_TABLES_SUB_URL: {{ .subUrl | quote }} diff --git a/helm/values.yaml b/helm/values.yaml index e08d90b..6d85078 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -80,6 +80,7 @@ server: externalServices: catalog: '' + extractable: '' lookupTables: url: '' subUrl: '' diff --git a/openapi3.yaml b/openapi3.yaml index 74a26a0..01a0f62 100644 --- a/openapi3.yaml +++ b/openapi3.yaml @@ -126,6 +126,27 @@ paths: application/json: schema: $ref: '#/components/schemas/error' + /jobs/{jobId}: + get: + operationId: getJobStatus + tags: + - job + summary: Get the status and progress of a job + parameters: + - $ref: '#/components/parameters/jobId' + responses: + 200: + description: Job status + content: + application/json: + schema: + $ref: '#/components/schemas/jobStatusResponse' + 404: + description: Job not found + content: + application/json: + schema: + $ref: '#/components/schemas/error' security: - {} components: @@ -137,6 +158,13 @@ components: description: The record identifier schema: type: string + jobId: + name: jobId + in: path + required: true + description: The job identifier + schema: + type: string schemas: error: type: object @@ -193,3 +221,12 @@ components: properties: message: type: string + jobStatusResponse: + type: object + required: + - status + properties: + status: + type: string + percentage: + type: number diff --git a/src/common/config.ts b/src/common/config.ts index 8d3292d..4f46eb4 100644 --- a/src/common/config.ts +++ b/src/common/config.ts @@ -9,6 +9,7 @@ interface LookupTablesConfig { interface ExternalServicesConfig { lookupTables: LookupTablesConfig; catalog: string; + extractable: string; } interface JobManagerConfig { @@ -98,7 +99,7 @@ const opsTriggerConfigSchema = { }, externalServices: { type: 'object', - required: ['lookupTables', 'catalog'], + required: ['lookupTables', 'catalog', 'extractable'], properties: { lookupTables: { type: 'object', @@ -109,6 +110,7 @@ const opsTriggerConfigSchema = { }, }, catalog: { type: 'string' }, + extractable: { type: 'string' }, }, }, jobManager: { diff --git a/src/common/util.ts b/src/common/util.ts index 542af20..f4b4f2f 100644 --- a/src/common/util.ts +++ b/src/common/util.ts @@ -1,4 +1,13 @@ +import type { Polygon, Position } from 'geojson'; + +const TWO_DIMENSIONS = 2; + export const is3tz = (modelPath: string): boolean => modelPath.toLowerCase().endsWith('.3tz'); export const buildModelFilePath = (modelPath: string, tilesetFilename: string): string => is3tz(modelPath) ? modelPath : `${modelPath}/${tilesetFilename}`; + +export const convertPolygonTo2DPolygon = (polygon: Polygon): Polygon => ({ + ...polygon, + coordinates: polygon.coordinates.map((ring: Position[]) => ring.map((coordinate: Position) => coordinate.slice(0, TWO_DIMENSIONS))), +}); diff --git a/src/containerConfig.ts b/src/containerConfig.ts index 3dbe41f..8a4bf69 100644 --- a/src/containerConfig.ts +++ b/src/containerConfig.ts @@ -7,6 +7,7 @@ import { type InjectionObject, registerDependencies } from '@common/dependencyRe import { SERVICES, SERVICE_NAME } from '@common/constants'; import { getTracing } from '@common/tracing'; import { recordRouterFactory, RECORD_ROUTER_SYMBOL } from './record/routes/recordRouter'; +import { jobRouterFactory, JOB_ROUTER_SYMBOL } from './job/routes/jobRouter'; import { providerFactory } from './providers/getProvider'; import { getConfig } from './common/config'; @@ -33,6 +34,7 @@ export const registerExternalValues = async (options?: RegisterOptions): Promise { token: SERVICES.METRICS, provider: { useValue: metricsRegistry } }, { token: SERVICES.PROVIDER, provider: { useFactory: providerFactory } }, { token: RECORD_ROUTER_SYMBOL, provider: { useFactory: recordRouterFactory } }, + { token: JOB_ROUTER_SYMBOL, provider: { useFactory: jobRouterFactory } }, { token: 'onSignal', provider: { diff --git a/src/externalServices/catalog/catalogCall.ts b/src/externalServices/catalog/catalogCall.ts index 3f0fd9e..8e8c585 100644 --- a/src/externalServices/catalog/catalogCall.ts +++ b/src/externalServices/catalog/catalogCall.ts @@ -6,7 +6,7 @@ import { SERVICES } from '@common/constants'; import { AppError } from '@common/appError'; import type { ConfigType } from '@common/config'; import type { LogContext } from '@common/interfaces'; -import type { IFindRecordsPayload, Record3D } from './interfaces'; +import type { CatalogStatusPayload, CatalogUpdatePayload, IFindRecordsPayload, Record3D } from './interfaces'; @injectable() export class CatalogCall { @@ -50,4 +50,42 @@ export class CatalogCall { throw new AppError('catalog', StatusCodes.INTERNAL_SERVER_ERROR, 'Problem with catalog find', true); } } + + public async getRecord(identifier: string): Promise { + const logContext = { ...this.logContext, function: this.getRecord.name }; + this.logger.debug({ msg: `Getting record ${identifier} from catalog`, logContext }); + try { + const response = await axios.get(`${this.catalog}/metadata/${identifier}`, { + validateStatus: (status) => status === StatusCodes.OK.valueOf() || status === StatusCodes.NOT_FOUND.valueOf(), + }); + return response.status === StatusCodes.NOT_FOUND.valueOf() ? undefined : response.data; + } catch (err) { + this.logger.error({ msg: 'Something went wrong in catalog when getting a record', logContext, identifier, err }); + throw new AppError('catalog', StatusCodes.INTERNAL_SERVER_ERROR, 'Problem with catalog during record lookup', true); + } + } + + public async patchMetadata(identifier: string, payload: CatalogUpdatePayload): Promise { + const logContext = { ...this.logContext, function: this.patchMetadata.name }; + this.logger.debug({ msg: `Updating metadata for record ${identifier} in catalog`, logContext }); + try { + const response = await axios.patch(`${this.catalog}/metadata/${identifier}`, payload); + return response.data; + } catch (err) { + this.logger.error({ msg: 'Something went wrong in catalog when updating metadata', logContext, identifier, err }); + throw new AppError('catalog', StatusCodes.INTERNAL_SERVER_ERROR, 'Problem with catalog during metadata update', true); + } + } + + public async changeStatus(identifier: string, payload: CatalogStatusPayload): Promise { + const logContext = { ...this.logContext, function: this.changeStatus.name }; + this.logger.debug({ msg: `Changing status for record ${identifier} in catalog`, logContext }); + try { + const response = await axios.patch(`${this.catalog}/metadata/status/${identifier}`, payload); + return response.data; + } catch (err) { + this.logger.error({ msg: 'Something went wrong in catalog when changing status', logContext, identifier, err }); + throw new AppError('catalog', StatusCodes.INTERNAL_SERVER_ERROR, 'Problem with catalog during status change', true); + } + } } diff --git a/src/externalServices/catalog/interfaces.ts b/src/externalServices/catalog/interfaces.ts index 0b0609b..e3987df 100644 --- a/src/externalServices/catalog/interfaces.ts +++ b/src/externalServices/catalog/interfaces.ts @@ -17,3 +17,9 @@ export interface IFindRecordsPayload { classification?: string; productStatus?: string; } + +export type CatalogUpdatePayload = Record; + +export interface CatalogStatusPayload { + productStatus: string; +} diff --git a/src/externalServices/extractableManagement/extractableCall.ts b/src/externalServices/extractableManagement/extractableCall.ts new file mode 100644 index 0000000..02dbd22 --- /dev/null +++ b/src/externalServices/extractableManagement/extractableCall.ts @@ -0,0 +1,39 @@ +import axios from 'axios'; +import { inject, injectable } from 'tsyringe'; +import type { Logger } from '@map-colonies/js-logger'; +import { StatusCodes } from 'http-status-codes'; +import { SERVICES } from '@common/constants'; +import { AppError } from '@common/appError'; +import type { ConfigType } from '@common/config'; +import type { LogContext } from '@common/interfaces'; + +@injectable() +export class ExtractableCall { + private readonly logContext: LogContext; + private readonly extractable: string; + + public constructor( + @inject(SERVICES.CONFIG) private readonly config: ConfigType, + @inject(SERVICES.LOGGER) private readonly logger: Logger + ) { + this.extractable = this.config.get('externalServices.extractable'); + this.logContext = { + fileName: __filename, + class: ExtractableCall.name, + }; + } + + public async isExtractableRecordExists(recordName: string): Promise { + const logContext = { ...this.logContext, function: this.isExtractableRecordExists.name }; + this.logger.debug({ msg: `Checking record '${recordName}' in extractable service`, logContext }); + try { + const response = await axios.get(`${this.extractable}/records/${encodeURIComponent(recordName)}`, { + validateStatus: (status) => status === StatusCodes.OK.valueOf() || status === StatusCodes.NOT_FOUND.valueOf(), + }); + return response.status === StatusCodes.OK.valueOf(); + } catch (err) { + this.logger.error({ msg: 'Something went wrong in extractable when checking a record', logContext, recordName, err }); + throw new AppError('extractable', StatusCodes.INTERNAL_SERVER_ERROR, 'Problem with extractable during record lookup', true); + } + } +} diff --git a/src/externalServices/jobnik/jobnikClient.ts b/src/externalServices/jobnik/jobnikClient.ts index 593ce4e..0a9104e 100644 --- a/src/externalServices/jobnik/jobnikClient.ts +++ b/src/externalServices/jobnik/jobnikClient.ts @@ -3,6 +3,7 @@ import { StatusCodes } from 'http-status-codes'; import type { Logger } from '@map-colonies/js-logger'; import type { Registry } from 'prom-client'; import { JobnikSDK } from '@map-colonies/jobnik-sdk'; +import type { JobId } from '@map-colonies/jobnik-sdk'; import { IN_FLIGHT_JOB_STATUSES, SERVICES, STAGE_TYPES } from '@common/constants'; import { is3tz } from '@common/util'; import { AppError } from '@common/appError'; @@ -18,6 +19,11 @@ interface StageDescriptor { only3tz?: boolean; } +export interface JobStatusResponse { + status: string; + percentage?: number; +} + @singleton() export class JobnikClient { private readonly logContext: LogContext; @@ -69,6 +75,20 @@ export class JobnikClient { } } + public async getJobStatus(jobId: string): Promise { + const logContext = { ...this.logContext, function: this.getJobStatus.name }; + const { data, error, response } = await this.apiClient.GET('/v1/jobs/{jobId}', { params: { path: { jobId: jobId as JobId } } }); + if (error !== undefined) { + if (response.status === StatusCodes.NOT_FOUND.valueOf()) { + throw new AppError('badRequest', StatusCodes.NOT_FOUND, `job ${jobId} was not found`, true); + } + this.logger.error({ msg: 'failed querying Jobnik for job status', logContext, jobId, err: error }); + throw new AppError('jobnik', StatusCodes.INTERNAL_SERVER_ERROR, 'failed querying Jobnik for job status', false); + } + + return { status: data.status, percentage: data.percentage }; + } + public async createIngestionJob(payload: IngestionPayload): Promise { const logContext = { ...this.logContext, function: this.createIngestionJob.name }; const isArchive = is3tz(payload.modelPath); diff --git a/src/job/controllers/jobController.ts b/src/job/controllers/jobController.ts new file mode 100644 index 0000000..64148ce --- /dev/null +++ b/src/job/controllers/jobController.ts @@ -0,0 +1,34 @@ +import type { Logger } from '@map-colonies/js-logger'; +import { StatusCodes } from 'http-status-codes'; +import { injectable, inject } from 'tsyringe'; +import type { TypedRequestHandlers } from '@openapi'; +import { SERVICES } from '@common/constants'; +import type { LogContext } from '@common/interfaces'; +import { JobManager } from '../models/jobManager'; + +@injectable() +export class JobController { + private readonly logContext: LogContext; + + public constructor( + @inject(SERVICES.LOGGER) private readonly logger: Logger, + @inject(JobManager) private readonly manager: JobManager + ) { + this.logContext = { + fileName: __filename, + class: JobController.name, + }; + } + + public getJobStatus: TypedRequestHandlers['getJobStatus'] = async (req, res, next) => { + const logContext = { ...this.logContext, function: this.getJobStatus.name }; + const { jobId } = req.params; + try { + const status = await this.manager.getStatus(jobId); + return res.status(StatusCodes.OK).json(status); + } catch (err) { + this.logger.error({ msg: 'failed to get job status', logContext, err, jobId }); + return next(err); + } + }; +} diff --git a/src/job/models/jobManager.ts b/src/job/models/jobManager.ts new file mode 100644 index 0000000..f2767ce --- /dev/null +++ b/src/job/models/jobManager.ts @@ -0,0 +1,29 @@ +import type { Logger } from '@map-colonies/js-logger'; +import { inject, injectable } from 'tsyringe'; +import type { components } from '@openapi'; +import { SERVICES } from '@common/constants'; +import type { LogContext } from '@common/interfaces'; +import { JobnikClient } from '../../externalServices/jobnik/jobnikClient'; + +export type JobStatusResponse = components['schemas']['jobStatusResponse']; + +@injectable() +export class JobManager { + private readonly logContext: LogContext; + + public constructor( + @inject(SERVICES.LOGGER) private readonly logger: Logger, + @inject(JobnikClient) private readonly jobnik: JobnikClient + ) { + this.logContext = { + fileName: __filename, + class: JobManager.name, + }; + } + + public async getStatus(jobId: string): Promise { + const logContext = { ...this.logContext, function: this.getStatus.name }; + this.logger.info({ msg: 'fetching job status', logContext, jobId }); + return this.jobnik.getJobStatus(jobId); + } +} diff --git a/src/job/routes/jobRouter.ts b/src/job/routes/jobRouter.ts new file mode 100644 index 0000000..0c646d7 --- /dev/null +++ b/src/job/routes/jobRouter.ts @@ -0,0 +1,16 @@ +import { Router } from 'express'; +import type { FactoryFunction } from 'tsyringe'; +import { JobController } from '../controllers/jobController'; + +const jobRouterFactory: FactoryFunction = (dependencyContainer) => { + const router = Router(); + const controller = dependencyContainer.resolve(JobController); + + router.get('/:jobId', controller.getJobStatus); + + return router; +}; + +export const JOB_ROUTER_SYMBOL = Symbol('jobRouterFactory'); + +export { jobRouterFactory }; diff --git a/src/openapi.d.ts b/src/openapi.d.ts index ca8c610..b0d39d3 100644 --- a/src/openapi.d.ts +++ b/src/openapi.d.ts @@ -62,6 +62,23 @@ export type paths = { patch: operations['updateRecordStatus']; trace?: never; }; + '/jobs/{jobId}': { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** Get the status and progress of a job */ + get: operations['getJobStatus']; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; }; export type webhooks = Record; export type components = { @@ -94,11 +111,17 @@ export type components = { ackResponse: { message: string; }; + jobStatusResponse: { + status: string; + percentage?: number; + }; }; responses: never; parameters: { /** @description The record identifier */ recordId: string; + /** @description The job identifier */ + jobId: string; }; requestBodies: never; headers: never; @@ -270,5 +293,37 @@ export interface operations { }; }; }; + getJobStatus: { + parameters: { + query?: never; + header?: never; + path: { + /** @description The job identifier */ + jobId: components['parameters']['jobId']; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Job status */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + 'application/json': components['schemas']['jobStatusResponse']; + }; + }; + /** @description Job not found */ + 404: { + headers: { + [name: string]: unknown; + }; + content: { + 'application/json': components['schemas']['error']; + }; + }; + }; + }; } export type TypedRequestHandlers = ImportedTypedRequestHandlers; diff --git a/src/record/controllers/recordController.ts b/src/record/controllers/recordController.ts index af0b6ee..db8ea86 100644 --- a/src/record/controllers/recordController.ts +++ b/src/record/controllers/recordController.ts @@ -52,11 +52,11 @@ export class RecordController { } }; - public updateRecord: TypedRequestHandlers['updateRecord'] = (req, res, next) => { + public updateRecord: TypedRequestHandlers['updateRecord'] = async (req, res, next) => { const logContext = { ...this.logContext, function: this.updateRecord.name }; const { id } = req.params; try { - const ack = this.manager.updateMetadata(id, req.body); + const ack = await this.manager.updateMetadata(id, req.body); return res.status(StatusCodes.OK).json(ack); } catch (err) { this.logger.error({ msg: 'failed to update record metadata', logContext, err, recordId: id }); @@ -64,11 +64,11 @@ export class RecordController { } }; - public updateRecordStatus: TypedRequestHandlers['updateRecordStatus'] = (req, res, next) => { + public updateRecordStatus: TypedRequestHandlers['updateRecordStatus'] = async (req, res, next) => { const logContext = { ...this.logContext, function: this.updateRecordStatus.name }; const { id } = req.params; try { - const ack = this.manager.updateStatus(id, req.body); + const ack = await this.manager.updateStatus(id, req.body); return res.status(StatusCodes.OK).json(ack); } catch (err) { this.logger.error({ msg: 'failed to update record status', logContext, err, recordId: id }); diff --git a/src/record/models/recordManager.ts b/src/record/models/recordManager.ts index 37867d8..d1369fc 100644 --- a/src/record/models/recordManager.ts +++ b/src/record/models/recordManager.ts @@ -1,10 +1,13 @@ import type { Logger } from '@map-colonies/js-logger'; import { inject, injectable } from 'tsyringe'; +import type { Polygon } from 'geojson'; import type { components } from '@openapi'; import { SERVICES } from '@common/constants'; import type { LogContext } from '@common/interfaces'; +import { convertPolygonTo2DPolygon } from '@common/util'; import { ValidationManager } from '../../validator/validationManager'; import { JobnikClient } from '../../externalServices/jobnik/jobnikClient'; +import { CatalogCall } from '../../externalServices/catalog/catalogCall'; export type IngestionPayload = components['schemas']['ingestionPayload']; export type UpdatePayload = components['schemas']['updatePayload']; @@ -19,7 +22,8 @@ export class RecordManager { public constructor( @inject(SERVICES.LOGGER) private readonly logger: Logger, @inject(ValidationManager) private readonly validator: ValidationManager, - @inject(JobnikClient) private readonly jobnik: JobnikClient + @inject(JobnikClient) private readonly jobnik: JobnikClient, + @inject(CatalogCall) private readonly catalog: CatalogCall ) { this.logContext = { fileName: __filename, @@ -41,15 +45,29 @@ export class RecordManager { return this.jobnik.createDeleteJob(record); } - public updateMetadata(id: string, update: UpdatePayload): AckResponse { + public async updateMetadata(id: string, update: UpdatePayload): Promise { const logContext = { ...this.logContext, function: this.updateMetadata.name }; - this.logger.info({ msg: 'updating record metadata', logContext, recordId: id, fields: Object.keys(update) }); - return { message: `metadata update accepted for record ${id}` }; + this.logger.info({ msg: 'updating record metadata', logContext, recordId: id }); + const record = await this.validator.validateUpdate(id, update); + await this.validator.ensureRecordAbsentFromExtractable(record); + + const payload: Record = { ...update }; + if (payload.footprint !== undefined) { + payload.footprint = convertPolygonTo2DPolygon(payload.footprint as Polygon); + } + await this.catalog.patchMetadata(id, payload); + + return { message: `metadata updated for record ${id}` }; } - public updateStatus(id: string, payload: StatusPayload): AckResponse { + public async updateStatus(id: string, payload: StatusPayload): Promise { const logContext = { ...this.logContext, function: this.updateStatus.name }; this.logger.info({ msg: 'updating record status', logContext, recordId: id, status: payload.status }); + const record = await this.validator.validateStatusChange(id); + await this.validator.ensureRecordAbsentFromExtractable(record); + + await this.catalog.changeStatus(id, { productStatus: payload.status }); + return { message: `status ${payload.status} accepted for record ${id}` }; } } diff --git a/src/serverBuilder.ts b/src/serverBuilder.ts index 3e2fc98..eff4ee4 100644 --- a/src/serverBuilder.ts +++ b/src/serverBuilder.ts @@ -11,6 +11,7 @@ import { Registry } from 'prom-client'; import type { ConfigType } from '@common/config'; import { SERVICES } from '@common/constants'; import { RECORD_ROUTER_SYMBOL } from './record/routes/recordRouter'; +import { JOB_ROUTER_SYMBOL } from './job/routes/jobRouter'; @injectable() export class ServerBuilder { @@ -20,7 +21,8 @@ export class ServerBuilder { @inject(SERVICES.CONFIG) private readonly config: ConfigType, @inject(SERVICES.LOGGER) private readonly logger: Logger, @inject(SERVICES.METRICS) private readonly metricsRegistry: Registry, - @inject(RECORD_ROUTER_SYMBOL) private readonly recordRouter: Router + @inject(RECORD_ROUTER_SYMBOL) private readonly recordRouter: Router, + @inject(JOB_ROUTER_SYMBOL) private readonly jobRouter: Router ) { this.serverInstance = express(); } @@ -44,6 +46,7 @@ export class ServerBuilder { private buildRoutes(): void { this.serverInstance.use('/record', this.recordRouter); + this.serverInstance.use('/jobs', this.jobRouter); this.buildDocsRoutes(); } diff --git a/src/validator/validationManager.ts b/src/validator/validationManager.ts index 3dfdbf6..72d40a7 100644 --- a/src/validator/validationManager.ts +++ b/src/validator/validationManager.ts @@ -4,6 +4,7 @@ import { inject, injectable } from 'tsyringe'; import { new3DLayerMetadataSchema, geometrySchema, calculatePolygonFromTileset, type TileSetJson } from '@map-colonies/3d-shared'; import { area, feature, featureCollection, intersect, union } from '@turf/turf'; import type { Feature, MultiPolygon, Polygon } from 'geojson'; +import type { components } from '@openapi'; import { SERVICES } from '@common/constants'; import type { LogContext, ValidationResponse } from '@common/interfaces'; import { AppError } from '@common/appError'; @@ -12,15 +13,19 @@ import type { ConfigType } from '@common/config'; import { LookupTablesCall } from '../externalServices/lookupTables/lookupTablesCall'; import { CatalogCall } from '../externalServices/catalog/catalogCall'; import { JobnikClient } from '../externalServices/jobnik/jobnikClient'; +import { ExtractableCall } from '../externalServices/extractableManagement/extractableCall'; import { TilesetReader } from '../tileset/tilesetReader'; import type { Record3D } from '../externalServices/catalog/interfaces'; import type { Provider } from '../providers/interfaces'; import type { IngestionPayload } from '../record/models/recordManager'; +type UpdatePayload = components['schemas']['updatePayload']; + const FULL_COVERAGE_PERCENT = 100; const BLOCKED_DELETE_PRODUCT_TYPE = 'QuantizedMeshDTMBest'; const RECORD_STATUS_UNPUBLISHED = 'UNPUBLISHED'; +const RECORD_STATUS_BEING_DELETED = 'BEING_DELETED'; export const ERROR_METADATA_DATE = 'imagingTimeBeginUTC must not be later than imagingTimeEndUTC'; export const ERROR_METADATA_MISSING_DATE = 'imagingTimeBeginUTC and imagingTimeEndUTC are required'; @@ -35,6 +40,9 @@ export const ERROR_FILE_NOT_FOUND = 'The model files do not exist in the agreed export const ERROR_TILESET_INVALID = 'The tileset file is not a valid 3DTiles tileset'; export const ERROR_FOOTPRINT_FAR_FROM_MODEL = "Wrong footprint! footprint's coordinates is not even close to the model!"; export const ERROR_INTERSECTION_FAILED = 'An error caused during the validation of the intersection'; +export const ERROR_RECORD_NOT_FOUND = "record with the given identifier doesn't exist"; +export const ERROR_RECORD_BEING_DELETED = 'cannot change a record that is being deleted'; +export const ERROR_EXTRACTABLE_CONFLICT = 'the record exists in the extractable-management service and cannot be changed here'; @injectable() export class ValidationManager { @@ -48,6 +56,7 @@ export class ValidationManager { @inject(CatalogCall) private readonly catalog: CatalogCall, @inject(JobnikClient) private readonly jobnik: JobnikClient, @inject(TilesetReader) private readonly tilesetReader: TilesetReader, + @inject(ExtractableCall) private readonly extractable: ExtractableCall, @inject(SERVICES.PROVIDER) private readonly provider: Provider ) { this.percentageLimit = this.config.get('validation.percentageLimit'); @@ -103,6 +112,69 @@ export class ValidationManager { return record; } + public async validateUpdate(identifier: string, payload: UpdatePayload): Promise { + const record = await this.catalog.getRecord(identifier); + if (record === undefined) { + throw new AppError('notFound', StatusCodes.NOT_FOUND, ERROR_RECORD_NOT_FOUND, true); + } + if (record.productStatus === RECORD_STATUS_BEING_DELETED) { + throw new AppError('badRequest', StatusCodes.BAD_REQUEST, ERROR_RECORD_BEING_DELETED, true); + } + + const parsed = new3DLayerMetadataSchema.partial().safeParse(payload); + if (!parsed.success) { + const message = parsed.error.issues.map((issue) => `${issue.path.join('.')}: ${issue.message}`).join('; '); + throw new AppError('badRequest', StatusCodes.BAD_REQUEST, message, true); + } + + const footprint = (payload as { footprint?: unknown }).footprint; + if (footprint !== undefined && !geometrySchema.safeParse(footprint).success) { + throw new AppError('badRequest', StatusCodes.BAD_REQUEST, ERROR_METADATA_FOOTPRINT, true); + } + + const data = parsed.data; + if (data.classification !== undefined) { + await this.validateClassification(data.classification); + } + if (data.productName !== undefined) { + await this.validateProductNameUniqueExcept(data.productName, identifier); + } + + return record; + } + + public async validateStatusChange(identifier: string): Promise { + const record = await this.catalog.getRecord(identifier); + if (record === undefined) { + throw new AppError('notFound', StatusCodes.NOT_FOUND, ERROR_RECORD_NOT_FOUND, true); + } + if (record.productStatus === RECORD_STATUS_BEING_DELETED) { + throw new AppError('badRequest', StatusCodes.BAD_REQUEST, ERROR_RECORD_BEING_DELETED, true); + } + + return record; + } + + public async ensureRecordAbsentFromExtractable(record: Record3D): Promise { + const logContext = { ...this.logContext, function: this.ensureRecordAbsentFromExtractable.name }; + if (record.productName === undefined) { + return; + } + + const exists = await this.extractable.isExtractableRecordExists(record.productName); + this.logger.debug({ msg: 'extractable existence validation', logContext, productName: record.productName, exists }); + if (exists) { + throw new AppError('conflict', StatusCodes.CONFLICT, ERROR_EXTRACTABLE_CONFLICT, true); + } + } + + private async validateProductNameUniqueExcept(productName: string, identifier: string): Promise { + const records = await this.catalog.findRecords({ productName }); + if (records.some((existing) => existing.id !== identifier)) { + throw new AppError('badRequest', StatusCodes.BAD_REQUEST, ERROR_METADATA_PRODUCT_NAME_UNIQUE, true); + } + } + private async validateFileExists(modelPath: string, tilesetFilename: string): Promise { const logContext = { ...this.logContext, function: this.validateFileExists.name }; const path = buildModelFilePath(modelPath, tilesetFilename); diff --git a/tests/integration/record/record.spec.ts b/tests/integration/record/record.spec.ts index dcdbf34..a3ee84f 100644 --- a/tests/integration/record/record.spec.ts +++ b/tests/integration/record/record.spec.ts @@ -13,6 +13,7 @@ import { LookupTablesCall } from '@src/externalServices/lookupTables/lookupTable import { CatalogCall } from '@src/externalServices/catalog/catalogCall'; import { JobnikClient } from '@src/externalServices/jobnik/jobnikClient'; import { TilesetReader } from '@src/tileset/tilesetReader'; +import { ExtractableCall } from '@src/externalServices/extractableManagement/extractableCall'; import { buildValidMetadata } from '@tests/helpers/metadata'; const regionTilesetJson = readFileSync(join(__dirname, '../../helpers/tilesets/folder/tileset.json'), 'utf-8'); @@ -31,13 +32,18 @@ const lookupStub = { getClassifications: vi.fn().mockResolvedValue(['abc123']) } // findRecords by id (delete) → an existing deletable record; by productName (uniqueness) → none const catalogStub = { findRecords: vi.fn().mockImplementation((payload: { id?: string }) => (payload.id !== undefined ? [deletableRecord] : [])), + getRecord: vi.fn().mockResolvedValue(deletableRecord), + patchMetadata: vi.fn().mockResolvedValue(deletableRecord), + changeStatus: vi.fn().mockResolvedValue(deletableRecord), } as unknown as CatalogCall; const jobnikStub = { createIngestionJob: vi.fn().mockResolvedValue({ jobId: 'job-1', status: 'PENDING' }), createDeleteJob: vi.fn().mockResolvedValue({ jobId: 'del-1', status: 'PENDING' }), hasInFlightIngestionJob: vi.fn().mockResolvedValue(false), + getJobStatus: vi.fn().mockResolvedValue({ status: 'IN_PROGRESS', percentage: 42 }), } as unknown as JobnikClient; const tilesetReaderStub = { readTilesetJson: vi.fn().mockResolvedValue(regionTilesetJson) } as unknown as TilesetReader; +const extractableStub = { isExtractableRecordExists: vi.fn().mockResolvedValue(false) } as unknown as ExtractableCall; const providerStub = { fileExists: vi.fn().mockResolvedValue(true) }; const validMetadata = buildValidMetadata(); @@ -64,6 +70,7 @@ describe('record', function () { { token: CatalogCall, provider: { useValue: catalogStub } }, { token: JobnikClient, provider: { useValue: jobnikStub } }, { token: TilesetReader, provider: { useValue: tilesetReaderStub } }, + { token: ExtractableCall, provider: { useValue: extractableStub } }, { token: SERVICES.PROVIDER, provider: { useValue: providerStub } }, ], useChild: true, @@ -146,4 +153,18 @@ describe('record', function () { expect(response.status).toBe(httpStatusCodes.BAD_REQUEST); }); }); + + describe('GET /jobs/{jobId}', function () { + it('should return 200 with the job status and percentage', async function () { + const response = await requestSender.getJobStatus({ pathParams: { jobId: 'job-1' } }); + + expect(response).toSatisfyApiSpec(); + expect(response.status).toBe(httpStatusCodes.OK); + + const body = response.body as paths['/jobs/{jobId}']['get']['responses']['200']['content']['application/json']; + + expect(body.status).toBeTypeOf('string'); + expect(body.percentage).toBe(42); + }); + }); }); diff --git a/tests/unit/record/models/recordManager.spec.ts b/tests/unit/record/models/recordManager.spec.ts index 403dc42..bf5d1bd 100644 --- a/tests/unit/record/models/recordManager.spec.ts +++ b/tests/unit/record/models/recordManager.spec.ts @@ -3,21 +3,31 @@ import { describe, it, expect, beforeEach, vi } from 'vitest'; import { RecordManager, type IngestionPayload } from '@src/record/models/recordManager'; import type { ValidationManager } from '@src/validator/validationManager'; import type { JobnikClient } from '@src/externalServices/jobnik/jobnikClient'; +import type { CatalogCall } from '@src/externalServices/catalog/catalogCall'; const noopValidator = { validateIngestion: vi.fn().mockResolvedValue(undefined), validateDelete: vi.fn().mockResolvedValue({ id: 'rec-1', productName: 'afula' }), + validateUpdate: vi.fn().mockResolvedValue({ id: 'rec-1', productName: 'afula' }), + validateStatusChange: vi.fn().mockResolvedValue({ id: 'rec-1', productName: 'afula' }), + ensureRecordAbsentFromExtractable: vi.fn().mockResolvedValue(undefined), } as unknown as ValidationManager; const jobnikStub = { createIngestionJob: vi.fn().mockResolvedValue({ jobId: 'job-1', status: 'PENDING' }), createDeleteJob: vi.fn().mockResolvedValue({ jobId: 'del-1', status: 'PENDING' }), } as unknown as JobnikClient; +const patchMetadataMock = vi.fn().mockResolvedValue({ id: 'rec-1' }); +const changeStatusMock = vi.fn().mockResolvedValue({ id: 'rec-1' }); +const catalogStub = { + patchMetadata: patchMetadataMock, + changeStatus: changeStatusMock, +} as unknown as CatalogCall; describe('RecordManager', function () { let manager: RecordManager; beforeEach(async function () { - manager = new RecordManager(await jsLogger({ enabled: false }), noopValidator, jobnikStub); + manager = new RecordManager(await jsLogger({ enabled: false }), noopValidator, jobnikStub, catalogStub); }); describe('createIngestion', function () { @@ -45,17 +55,19 @@ describe('RecordManager', function () { }); describe('updateMetadata', function () { - it('should return an ack referencing the record id', function () { - const result = manager.updateMetadata('rec-1', { description: 'x' }); + it('should validate, guard extractable, patch the catalog and return an ack', async function () { + const result = await manager.updateMetadata('rec-1', { description: 'x' }); + expect(patchMetadataMock).toHaveBeenCalledWith('rec-1', expect.objectContaining({ description: 'x' })); expect(result.message).toContain('rec-1'); }); }); describe('updateStatus', function () { - it('should return an ack referencing the requested status', function () { - const result = manager.updateStatus('rec-1', { status: 'PUBLISHED' }); + it('should validate, guard extractable, change the catalog status and return an ack', async function () { + const result = await manager.updateStatus('rec-1', { status: 'PUBLISHED' }); + expect(changeStatusMock).toHaveBeenCalledWith('rec-1', { productStatus: 'PUBLISHED' }); expect(result.message).toContain('PUBLISHED'); }); }); diff --git a/tests/unit/validator/validationManager.spec.ts b/tests/unit/validator/validationManager.spec.ts index 18ef9aa..9e089ac 100644 --- a/tests/unit/validator/validationManager.spec.ts +++ b/tests/unit/validator/validationManager.spec.ts @@ -15,6 +15,9 @@ import { ERROR_FILE_NOT_FOUND, ERROR_PRODUCT_NAME_IN_FLIGHT, ERROR_FOOTPRINT_FAR_FROM_MODEL, + ERROR_RECORD_NOT_FOUND, + ERROR_RECORD_BEING_DELETED, + ERROR_EXTRACTABLE_CONFLICT, } from '@src/validator/validationManager'; import { AppError } from '@src/common/appError'; import type { ConfigType } from '@src/common/config'; @@ -22,6 +25,7 @@ import type { LookupTablesCall } from '@src/externalServices/lookupTables/lookup import type { CatalogCall } from '@src/externalServices/catalog/catalogCall'; import type { JobnikClient } from '@src/externalServices/jobnik/jobnikClient'; import type { TilesetReader } from '@src/tileset/tilesetReader'; +import type { ExtractableCall } from '@src/externalServices/extractableManagement/extractableCall'; import type { Provider } from '@src/providers/interfaces'; import type { IngestionPayload } from '@src/record/models/recordManager'; import { buildValidMetadata } from '@tests/helpers/metadata'; @@ -43,12 +47,17 @@ describe('ValidationManager', function () { let catalogStub: CatalogCall; let jobnikStub: JobnikClient; let tilesetReaderStub: TilesetReader; + let extractableStub: ExtractableCall; let providerStub: Provider; beforeEach(async function () { - catalogStub = { findRecords: vi.fn().mockResolvedValue([]) } as unknown as CatalogCall; + catalogStub = { + findRecords: vi.fn().mockResolvedValue([]), + getRecord: vi.fn().mockResolvedValue({ id: 'rec-1', productName: 'afula' }), + } as unknown as CatalogCall; jobnikStub = { hasInFlightIngestionJob: vi.fn().mockResolvedValue(false) } as unknown as JobnikClient; tilesetReaderStub = { readTilesetJson: vi.fn().mockResolvedValue(regionTilesetJson) } as unknown as TilesetReader; + extractableStub = { isExtractableRecordExists: vi.fn().mockResolvedValue(false) } as unknown as ExtractableCall; providerStub = { fileExists: vi.fn().mockResolvedValue(true) }; validator = new ValidationManager( configStub, @@ -57,6 +66,7 @@ describe('ValidationManager', function () { catalogStub, jobnikStub, tilesetReaderStub, + extractableStub, providerStub ); }); @@ -203,6 +213,68 @@ describe('ValidationManager', function () { expect((thrown as AppError).status).toBe(StatusCodes.BAD_REQUEST); }); + describe('validateUpdate', function () { + it('should return the record for a valid update payload', async function () { + await expect(validator.validateUpdate('rec-1', { description: 'x' })).resolves.toEqual({ id: 'rec-1', productName: 'afula' }); + }); + + it('should throw 404 when the record does not exist', async function () { + (catalogStub.getRecord as ReturnType).mockResolvedValueOnce(undefined); + + let thrown: unknown; + try { + await validator.validateUpdate('missing', { description: 'x' }); + } catch (err) { + thrown = err; + } + + expect(thrown).toBeInstanceOf(AppError); + expect((thrown as AppError).status).toBe(StatusCodes.NOT_FOUND); + expect((thrown as AppError).message).toBe(ERROR_RECORD_NOT_FOUND); + }); + + it('should throw 400 when the record is being deleted', async function () { + (catalogStub.getRecord as ReturnType).mockResolvedValueOnce({ id: 'rec-1', productStatus: 'BEING_DELETED' }); + + await expect(validator.validateUpdate('rec-1', { description: 'x' })).rejects.toThrow(ERROR_RECORD_BEING_DELETED); + }); + + it('should throw 400 when a renamed product name collides with another record', async function () { + (catalogStub.findRecords as ReturnType).mockResolvedValueOnce([{ id: 'other', productName: 'haifa' }]); + + await expect(validator.validateUpdate('rec-1', { productName: 'haifa' })).rejects.toThrow(AppError); + }); + }); + + describe('validateStatusChange', function () { + it('should throw 404 when the record does not exist', async function () { + (catalogStub.getRecord as ReturnType).mockResolvedValueOnce(undefined); + + await expect(validator.validateStatusChange('missing')).rejects.toThrow(ERROR_RECORD_NOT_FOUND); + }); + }); + + describe('ensureRecordAbsentFromExtractable', function () { + it('should throw 409 when the record exists in extractable-management', async function () { + (extractableStub.isExtractableRecordExists as ReturnType).mockResolvedValueOnce(true); + + let thrown: unknown; + try { + await validator.ensureRecordAbsentFromExtractable({ id: 'rec-1', productName: 'afula' }); + } catch (err) { + thrown = err; + } + + expect(thrown).toBeInstanceOf(AppError); + expect((thrown as AppError).status).toBe(StatusCodes.CONFLICT); + expect((thrown as AppError).message).toBe(ERROR_EXTRACTABLE_CONFLICT); + }); + + it('should pass when the record is absent from extractable-management', async function () { + await expect(validator.ensureRecordAbsentFromExtractable({ id: 'rec-1', productName: 'afula' })).resolves.toBeUndefined(); + }); + }); + describe('validateDelete', function () { it('should return the record when it is deletable', async function () { const record = { id: 'rec-1', productType: '3DPhotoRealistic', productStatus: 'UNPUBLISHED' }; From 9d411dc4ef50be2bc429ff796f22283a399bd6d1 Mon Sep 17 00:00:00 2001 From: TULCHINSKI LIRAN Date: Wed, 30 Sep 2026 16:01:19 +0300 Subject: [PATCH 2/7] feat: add validate endpoint, model-path/productId checks, delete status and bundle (MAPCO-11833) Bring the service closer to the 3d-gateway behaviour: - POST /record/validate: run the full ingestion validation and return {isValid, message} without creating a job. - Ingestion validation also enforces modelPath under the configured base path (validation.basePath) and productId uniqueness in the catalog. - Ingestion converts the footprint to 2D before creating the job. - Delete now sets the catalog record status to BEING_DELETED after the delete job is created. - Add an openapi:bundle script (redocly) producing bundledApi.yaml. Skipped by design (Jobnik model, not store-trigger): PV-path rewriting, pre-minted uuid modelId, StoreTriggerCall. --- .gitignore | 1 + config/custom-environment-variables.json | 3 +- config/default.json | 3 +- helm/templates/configmap.yaml | 1 + helm/values.yaml | 1 + openapi3.yaml | 29 +++++++++++ package.json | 1 + src/common/config.ts | 4 +- src/openapi.d.ts | 48 +++++++++++++++++++ src/record/controllers/recordController.ts | 11 +++++ src/record/models/recordManager.ts | 28 ++++++++++- src/record/routes/recordRouter.ts | 1 + src/validator/validationManager.ts | 22 +++++++++ tests/integration/record/record.spec.ts | 29 ++++++++++- .../unit/record/models/recordManager.spec.ts | 3 +- .../unit/validator/validationManager.spec.ts | 27 ++++++++++- 16 files changed, 205 insertions(+), 7 deletions(-) diff --git a/.gitignore b/.gitignore index 8ba7348..ce21358 100644 --- a/.gitignore +++ b/.gitignore @@ -113,3 +113,4 @@ jest_html_reporters.html reports config/local*.json +bundledApi.yaml diff --git a/config/custom-environment-variables.json b/config/custom-environment-variables.json index 777ce0d..e7c5e28 100644 --- a/config/custom-environment-variables.json +++ b/config/custom-environment-variables.json @@ -26,7 +26,8 @@ "percentageLimit": { "__name": "VALIDATION_PERCENTAGE_LIMIT", "__format": "number" - } + }, + "basePath": "VALIDATION_BASE_PATH" }, "provider": "PROVIDER_FROM", "NFS": { diff --git a/config/default.json b/config/default.json index d383775..3aa5098 100644 --- a/config/default.json +++ b/config/default.json @@ -54,7 +54,8 @@ } }, "validation": { - "percentageLimit": 10 + "percentageLimit": 10, + "basePath": "/app/models" }, "provider": "NFS", "NFS": { diff --git a/helm/templates/configmap.yaml b/helm/templates/configmap.yaml index 5fde69a..ed7a095 100644 --- a/helm/templates/configmap.yaml +++ b/helm/templates/configmap.yaml @@ -47,6 +47,7 @@ data: TASK_DELETE_TYPE: {{ .delete.taskType | quote }} {{- end }} VALIDATION_PERCENTAGE_LIMIT: {{ .Values.validation.percentageLimit | quote }} + VALIDATION_BASE_PATH: {{ .Values.validation.basePath | quote }} PROVIDER_FROM: {{ .Values.provider | quote }} {{- if eq .Values.provider "NFS" }} PV_SOURCE_PATH: {{ .Values.NFS.pvPath | quote }} diff --git a/helm/values.yaml b/helm/values.yaml index 6d85078..4a9b5c2 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -97,6 +97,7 @@ jobManager: validation: percentageLimit: 10 + basePath: '' provider: 'NFS' NFS: diff --git a/openapi3.yaml b/openapi3.yaml index 01a0f62..9250ab3 100644 --- a/openapi3.yaml +++ b/openapi3.yaml @@ -33,6 +33,26 @@ paths: application/json: schema: $ref: '#/components/schemas/error' + /record/validate: + post: + operationId: validateRecord + tags: + - record + summary: Validate an ingestion request without creating a job + description: Runs the full ingestion validation and returns the result without triggering a Jobnik job. + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/ingestionPayload' + responses: + 200: + description: Validation result + content: + application/json: + schema: + $ref: '#/components/schemas/validationResultResponse' /record/{id}: delete: operationId: deleteRecord @@ -230,3 +250,12 @@ components: type: string percentage: type: number + validationResultResponse: + type: object + required: + - isValid + properties: + isValid: + type: boolean + message: + type: string diff --git a/package.json b/package.json index 76b94a0..a7b9d79 100644 --- a/package.json +++ b/package.json @@ -14,6 +14,7 @@ "prelint:fix": "npm run format:fix", "prelint": "npm run format", "lint:openapi": "redocly lint openapi3.yaml", + "openapi:bundle": "redocly bundle openapi3.yaml -o bundledApi.yaml", "lint": "eslint .", "lint:fix": "eslint --fix .", "prebuild": "npm run clean && npm run generate:openapi-types", diff --git a/src/common/config.ts b/src/common/config.ts index 4f46eb4..0f28da5 100644 --- a/src/common/config.ts +++ b/src/common/config.ts @@ -27,6 +27,7 @@ interface JobManagerConfig { interface ValidationConfig { percentageLimit: number; + basePath: string; } interface NFSConfig { @@ -71,9 +72,10 @@ const opsTriggerConfigSchema = { provider: { type: 'string', enum: ['NFS', 'S3'] }, validation: { type: 'object', - required: ['percentageLimit'], + required: ['percentageLimit', 'basePath'], properties: { percentageLimit: { type: 'number' }, + basePath: { type: 'string' }, }, }, // eslint-disable-next-line @typescript-eslint/naming-convention diff --git a/src/openapi.d.ts b/src/openapi.d.ts index b0d39d3..bb6fa22 100644 --- a/src/openapi.d.ts +++ b/src/openapi.d.ts @@ -24,6 +24,26 @@ export type paths = { patch?: never; trace?: never; }; + '/record/validate': { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** + * Validate an ingestion request without creating a job + * @description Runs the full ingestion validation and returns the result without triggering a Jobnik job. + */ + post: operations['validateRecord']; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; '/record/{id}': { parameters: { query?: never; @@ -115,6 +135,10 @@ export type components = { status: string; percentage?: number; }; + validationResultResponse: { + isValid: boolean; + message?: string; + }; }; responses: never; parameters: { @@ -162,6 +186,30 @@ export interface operations { }; }; }; + validateRecord: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody: { + content: { + 'application/json': components['schemas']['ingestionPayload']; + }; + }; + responses: { + /** @description Validation result */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + 'application/json': components['schemas']['validationResultResponse']; + }; + }; + }; + }; deleteRecord: { parameters: { query?: never; diff --git a/src/record/controllers/recordController.ts b/src/record/controllers/recordController.ts index db8ea86..d40b219 100644 --- a/src/record/controllers/recordController.ts +++ b/src/record/controllers/recordController.ts @@ -40,6 +40,17 @@ export class RecordController { } }; + public validateRecord: TypedRequestHandlers['validateRecord'] = async (req, res, next) => { + const logContext = { ...this.logContext, function: this.validateRecord.name }; + try { + const result = await this.manager.validateIngestion(req.body); + return res.status(StatusCodes.OK).json(result); + } catch (err) { + this.logger.error({ msg: 'failed to validate ingestion request', logContext, err }); + return next(err); + } + }; + public deleteRecord: TypedRequestHandlers['deleteRecord'] = async (req, res, next) => { const logContext = { ...this.logContext, function: this.deleteRecord.name }; const { id } = req.params; diff --git a/src/record/models/recordManager.ts b/src/record/models/recordManager.ts index d1369fc..cf12ccf 100644 --- a/src/record/models/recordManager.ts +++ b/src/record/models/recordManager.ts @@ -4,16 +4,20 @@ import type { Polygon } from 'geojson'; import type { components } from '@openapi'; import { SERVICES } from '@common/constants'; import type { LogContext } from '@common/interfaces'; +import { AppError } from '@common/appError'; import { convertPolygonTo2DPolygon } from '@common/util'; import { ValidationManager } from '../../validator/validationManager'; import { JobnikClient } from '../../externalServices/jobnik/jobnikClient'; import { CatalogCall } from '../../externalServices/catalog/catalogCall'; +const RECORD_STATUS_BEING_DELETED = 'BEING_DELETED'; + export type IngestionPayload = components['schemas']['ingestionPayload']; export type UpdatePayload = components['schemas']['updatePayload']; export type StatusPayload = components['schemas']['statusPayload']; export type JobResponse = components['schemas']['jobResponse']; export type AckResponse = components['schemas']['ackResponse']; +export type ValidationResultResponse = components['schemas']['validationResultResponse']; @injectable() export class RecordManager { @@ -35,14 +39,36 @@ export class RecordManager { const logContext = { ...this.logContext, function: this.createIngestion.name }; this.logger.info({ msg: 'creating ingestion job', logContext, modelPath: payload.modelPath, tilesetFilename: payload.tilesetFilename }); await this.validator.validateIngestion(payload); + + const metadata = payload.metadata as Record; + if (metadata.footprint !== undefined) { + metadata.footprint = convertPolygonTo2DPolygon(metadata.footprint as Polygon); + } + return this.jobnik.createIngestionJob(payload); } + public async validateIngestion(payload: IngestionPayload): Promise { + const logContext = { ...this.logContext, function: this.validateIngestion.name }; + this.logger.info({ msg: 'validating ingestion request', logContext, modelPath: payload.modelPath }); + try { + await this.validator.validateIngestion(payload); + return { isValid: true }; + } catch (err) { + if (err instanceof AppError) { + return { isValid: false, message: err.message }; + } + throw err; + } + } + public async deleteRecord(id: string): Promise { const logContext = { ...this.logContext, function: this.deleteRecord.name }; this.logger.info({ msg: 'creating delete job', logContext, recordId: id }); const record = await this.validator.validateDelete(id); - return this.jobnik.createDeleteJob(record); + const job = await this.jobnik.createDeleteJob(record); + await this.catalog.changeStatus(id, { productStatus: RECORD_STATUS_BEING_DELETED }); + return job; } public async updateMetadata(id: string, update: UpdatePayload): Promise { diff --git a/src/record/routes/recordRouter.ts b/src/record/routes/recordRouter.ts index 4a5b495..077ac6f 100644 --- a/src/record/routes/recordRouter.ts +++ b/src/record/routes/recordRouter.ts @@ -7,6 +7,7 @@ const recordRouterFactory: FactoryFunction = (dependencyContainer) => { const controller = dependencyContainer.resolve(RecordController); router.post('/', controller.createRecord); + router.post('/validate', controller.validateRecord); router.patch('/status/:id', controller.updateRecordStatus); router.delete('/:id', controller.deleteRecord); router.patch('/:id', controller.updateRecord); diff --git a/src/validator/validationManager.ts b/src/validator/validationManager.ts index 72d40a7..3805e3e 100644 --- a/src/validator/validationManager.ts +++ b/src/validator/validationManager.ts @@ -43,11 +43,14 @@ export const ERROR_INTERSECTION_FAILED = 'An error caused during the validation export const ERROR_RECORD_NOT_FOUND = "record with the given identifier doesn't exist"; export const ERROR_RECORD_BEING_DELETED = 'cannot change a record that is being deleted'; export const ERROR_EXTRACTABLE_CONFLICT = 'the record exists in the extractable-management service and cannot be changed here'; +export const ERROR_MODEL_PATH_INVALID = 'Unknown model path! The model is not under the agreed base path'; +export const ERROR_PRODUCT_ID_EXISTS = 'a record with this productId already exists in the catalog'; @injectable() export class ValidationManager { private readonly logContext: LogContext; private readonly percentageLimit: number; + private readonly basePath: string; public constructor( @inject(SERVICES.CONFIG) private readonly config: ConfigType, @@ -60,6 +63,7 @@ export class ValidationManager { @inject(SERVICES.PROVIDER) private readonly provider: Provider ) { this.percentageLimit = this.config.get('validation.percentageLimit'); + this.basePath = this.config.get('validation.basePath'); this.logContext = { fileName: __filename, class: ValidationManager.name, @@ -83,10 +87,12 @@ export class ValidationManager { } this.validateDates(metadata.imagingTimeBeginUTC, metadata.imagingTimeEndUTC); + this.validateModelPath(payload.modelPath); await this.validateClassification(parsed.data.classification); await this.validateProductNameUnique(parsed.data.productName); await this.validateProductNameNotInFlight(parsed.data.productName); + await this.validateProductIdUnique(parsed.data.productId); await this.validateFileExists(payload.modelPath, payload.tilesetFilename); await this.validateTileset(payload, footprintResult.data); } @@ -175,6 +181,22 @@ export class ValidationManager { } } + private validateModelPath(modelPath: string): void { + if (!modelPath.startsWith(this.basePath)) { + throw new AppError('badRequest', StatusCodes.BAD_REQUEST, `${ERROR_MODEL_PATH_INVALID} (basePath: ${this.basePath})`, true); + } + } + + private async validateProductIdUnique(productId: string): Promise { + const logContext = { ...this.logContext, function: this.validateProductIdUnique.name }; + const records = await this.catalog.findRecords({ productId }); + this.logger.debug({ msg: 'product id uniqueness validation', logContext, productId, matches: records.length }); + + if (records.length > 0) { + throw new AppError('conflict', StatusCodes.CONFLICT, ERROR_PRODUCT_ID_EXISTS, true); + } + } + private async validateFileExists(modelPath: string, tilesetFilename: string): Promise { const logContext = { ...this.logContext, function: this.validateFileExists.name }; const path = buildModelFilePath(modelPath, tilesetFilename); diff --git a/tests/integration/record/record.spec.ts b/tests/integration/record/record.spec.ts index a3ee84f..659ac5c 100644 --- a/tests/integration/record/record.spec.ts +++ b/tests/integration/record/record.spec.ts @@ -49,7 +49,7 @@ const providerStub = { fileExists: vi.fn().mockResolvedValue(true) }; const validMetadata = buildValidMetadata(); const validIngestionPayload = { - modelPath: '/shared/models/afula', + modelPath: '/app/models/afula', tilesetFilename: 'tileset.json', metadata: validMetadata, }; @@ -154,6 +154,33 @@ describe('record', function () { }); }); + describe('POST /record/validate', function () { + it('should return 200 with isValid true for a valid request without creating a job', async function () { + const response = await requestSender.validateRecord({ requestBody: validIngestionPayload }); + + expect(response).toSatisfyApiSpec(); + expect(response.status).toBe(httpStatusCodes.OK); + + const body = response.body; + + expect(body.isValid).toBe(true); + }); + + it('should return 200 with isValid false and a message for an invalid request', async function () { + const response = await requestSender.validateRecord({ + requestBody: { ...validIngestionPayload, metadata: { ...validMetadata, productType: 'NOT_A_3D_TYPE' } }, + }); + + expect(response).toSatisfyApiSpec(); + expect(response.status).toBe(httpStatusCodes.OK); + + const body = response.body; + + expect(body.isValid).toBe(false); + expect(body.message).toBeTypeOf('string'); + }); + }); + describe('GET /jobs/{jobId}', function () { it('should return 200 with the job status and percentage', async function () { const response = await requestSender.getJobStatus({ pathParams: { jobId: 'job-1' } }); diff --git a/tests/unit/record/models/recordManager.spec.ts b/tests/unit/record/models/recordManager.spec.ts index bf5d1bd..59cc0d7 100644 --- a/tests/unit/record/models/recordManager.spec.ts +++ b/tests/unit/record/models/recordManager.spec.ts @@ -46,11 +46,12 @@ describe('RecordManager', function () { }); describe('deleteRecord', function () { - it('should return a job response', async function () { + it('should create the delete job and set the catalog status to BEING_DELETED', async function () { const result = await manager.deleteRecord('rec-1'); expect(result.jobId).toBeTypeOf('string'); expect(result.status).toBeTypeOf('string'); + expect(changeStatusMock).toHaveBeenCalledWith('rec-1', { productStatus: 'BEING_DELETED' }); }); }); diff --git a/tests/unit/validator/validationManager.spec.ts b/tests/unit/validator/validationManager.spec.ts index 9e089ac..91f0509 100644 --- a/tests/unit/validator/validationManager.spec.ts +++ b/tests/unit/validator/validationManager.spec.ts @@ -18,6 +18,7 @@ import { ERROR_RECORD_NOT_FOUND, ERROR_RECORD_BEING_DELETED, ERROR_EXTRACTABLE_CONFLICT, + ERROR_MODEL_PATH_INVALID, } from '@src/validator/validationManager'; import { AppError } from '@src/common/appError'; import type { ConfigType } from '@src/common/config'; @@ -34,7 +35,9 @@ const regionTilesetJson = readFileSync(join(__dirname, '../../helpers/tilesets/f const boxTilesetJson = JSON.stringify({ root: { boundingVolume: { box: [0, 0, 0, 100, 0, 0, 0, 100, 0, 0, 0, 100] } } }); const lookupStub = { getClassifications: vi.fn().mockResolvedValue(['abc123']) } as unknown as LookupTablesCall; -const configStub = { get: vi.fn().mockReturnValue(10) } as unknown as ConfigType; +const configStub = { + get: vi.fn((key: string) => (key === 'validation.basePath' ? '/shared/models' : 10)), +} as unknown as ConfigType; const ingest = (metadata: Record): IngestionPayload => ({ modelPath: '/shared/models/afula', @@ -154,6 +157,28 @@ describe('ValidationManager', function () { await expect(validator.validateIngestion(ingest(buildValidMetadata()))).rejects.toThrow(ERROR_FILE_NOT_FOUND); }); + it('should throw 400 when the model path is not under the agreed base path', async function () { + const payload = { ...ingest(buildValidMetadata()), modelPath: '/etc/passwd' }; + + await expect(validator.validateIngestion(payload)).rejects.toThrow(ERROR_MODEL_PATH_INVALID); + }); + + it('should throw 409 when the productId already exists in the catalog', async function () { + (catalogStub.findRecords as ReturnType).mockImplementation((query: { productId?: string; productName?: string }) => + query.productId !== undefined ? [{ id: 'existing', productId: query.productId }] : [] + ); + + let thrown: unknown; + try { + await validator.validateIngestion(ingest(buildValidMetadata())); + } catch (err) { + thrown = err; + } + + expect(thrown).toBeInstanceOf(AppError); + expect((thrown as AppError).status).toBe(StatusCodes.CONFLICT); + }); + it('should throw 400 when the footprint does not intersect the tileset model', async function () { const metadata = buildValidMetadata(); metadata.footprint = { From c03f02105c5a6ff5f5752e6db59caba279d252b5 Mon Sep 17 00:00:00 2001 From: TULCHINSKI LIRAN Date: Wed, 30 Sep 2026 16:18:54 +0300 Subject: [PATCH 3/7] feat: add canDelete endpoint and commit the bundled OpenApi (MAPCO-11833) - GET /record/canDelete/{id}: dry-run delete validation returning {isValid, message}, matching 3d-gateway's GET /models/canDelete/{recordId}. - Rename the bundle script to bundleOpenApi and commit bundledApi.yaml (tracked, like 3d-gateway and store-trigger; removed from .gitignore). --- .gitignore | 1 - bundledApi.yaml | 277 +++++++++++++++++++++ openapi3.yaml | 16 ++ package.json | 2 +- src/openapi.d.ts | 43 ++++ src/record/controllers/recordController.ts | 12 + src/record/models/recordManager.ts | 14 ++ src/record/routes/recordRouter.ts | 1 + tests/integration/record/record.spec.ts | 13 + 9 files changed, 377 insertions(+), 2 deletions(-) create mode 100644 bundledApi.yaml diff --git a/.gitignore b/.gitignore index ce21358..8ba7348 100644 --- a/.gitignore +++ b/.gitignore @@ -113,4 +113,3 @@ jest_html_reporters.html reports config/local*.json -bundledApi.yaml diff --git a/bundledApi.yaml b/bundledApi.yaml new file mode 100644 index 0000000..4cfd255 --- /dev/null +++ b/bundledApi.yaml @@ -0,0 +1,277 @@ +openapi: 3.0.1 +info: + title: 3d-ops-trigger + description: Entry point for 3D operations — validates requests and triggers Jobnik jobs + version: 1.0.0 + license: + name: MIT + url: https://opensource.org/licenses/MIT +security: + - {} +paths: + /record: + post: + operationId: createRecord + tags: + - record + summary: Start an ingestion process flow + description: Validates the request (light & fast) and creates a Jobnik ingestion job. + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/ingestionPayload' + responses: + '201': + description: Ingestion job created + content: + application/json: + schema: + $ref: '#/components/schemas/jobResponse' + '400': + description: Bad Request + content: + application/json: + schema: + $ref: '#/components/schemas/error' + /record/validate: + post: + operationId: validateRecord + tags: + - record + summary: Validate an ingestion request without creating a job + description: Runs the full ingestion validation and returns the result without triggering a Jobnik job. + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/ingestionPayload' + responses: + '200': + description: Validation result + content: + application/json: + schema: + $ref: '#/components/schemas/validationResultResponse' + /record/canDelete/{id}: + get: + operationId: canDeleteRecord + tags: + - record + summary: Validate whether a record can be deleted + description: Runs the delete validation and returns the result without creating a job. + parameters: + - $ref: '#/components/parameters/recordId' + responses: + '200': + description: Validation result + content: + application/json: + schema: + $ref: '#/components/schemas/validationResultResponse' + /record/{id}: + delete: + operationId: deleteRecord + tags: + - record + summary: Start a delete process flow + description: Validates the record can be deleted and creates a Jobnik delete job. + parameters: + - $ref: '#/components/parameters/recordId' + responses: + '200': + description: Delete job created + content: + application/json: + schema: + $ref: '#/components/schemas/jobResponse' + '400': + description: Bad Request + content: + application/json: + schema: + $ref: '#/components/schemas/error' + '404': + description: Record not found + content: + application/json: + schema: + $ref: '#/components/schemas/error' + patch: + operationId: updateRecord + tags: + - record + summary: Update metadata for a record + parameters: + - $ref: '#/components/parameters/recordId' + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/updatePayload' + responses: + '200': + description: Metadata updated + content: + application/json: + schema: + $ref: '#/components/schemas/ackResponse' + '400': + description: Bad Request + content: + application/json: + schema: + $ref: '#/components/schemas/error' + '404': + description: Record not found + content: + application/json: + schema: + $ref: '#/components/schemas/error' + /record/status/{id}: + patch: + operationId: updateRecordStatus + tags: + - record + summary: Update the publish/unpublish status of a record + parameters: + - $ref: '#/components/parameters/recordId' + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/statusPayload' + responses: + '200': + description: Status updated + content: + application/json: + schema: + $ref: '#/components/schemas/ackResponse' + '400': + description: Bad Request + content: + application/json: + schema: + $ref: '#/components/schemas/error' + '404': + description: Record not found + content: + application/json: + schema: + $ref: '#/components/schemas/error' + /jobs/{jobId}: + get: + operationId: getJobStatus + tags: + - job + summary: Get the status and progress of a job + parameters: + - $ref: '#/components/parameters/jobId' + responses: + '200': + description: Job status + content: + application/json: + schema: + $ref: '#/components/schemas/jobStatusResponse' + '404': + description: Job not found + content: + application/json: + schema: + $ref: '#/components/schemas/error' +components: + parameters: + recordId: + name: id + in: path + required: true + description: The record identifier + schema: + type: string + jobId: + name: jobId + in: path + required: true + description: The job identifier + schema: + type: string + schemas: + error: + type: object + required: + - message + properties: + message: + type: string + ingestionPayload: + type: object + required: + - modelPath + - tilesetFilename + - metadata + properties: + modelPath: + type: string + description: Path on the shared storage to the 3DTiles folder or 3TZ archive + tilesetFilename: + type: string + description: The tileset entry name (e.g. tileset.json) + metadata: + type: object + description: 3D record metadata (business validation applied downstream) + additionalProperties: true + updatePayload: + type: object + description: Partial metadata fields to update + additionalProperties: true + statusPayload: + type: object + required: + - status + properties: + status: + type: string + enum: + - PUBLISHED + - UNPUBLISHED + jobResponse: + type: object + required: + - jobId + - status + properties: + jobId: + type: string + status: + type: string + ackResponse: + type: object + required: + - message + properties: + message: + type: string + jobStatusResponse: + type: object + required: + - status + properties: + status: + type: string + percentage: + type: number + validationResultResponse: + type: object + required: + - isValid + properties: + isValid: + type: boolean + message: + type: string diff --git a/openapi3.yaml b/openapi3.yaml index 9250ab3..ae7a446 100644 --- a/openapi3.yaml +++ b/openapi3.yaml @@ -53,6 +53,22 @@ paths: application/json: schema: $ref: '#/components/schemas/validationResultResponse' + /record/canDelete/{id}: + get: + operationId: canDeleteRecord + tags: + - record + summary: Validate whether a record can be deleted + description: Runs the delete validation and returns the result without creating a job. + parameters: + - $ref: '#/components/parameters/recordId' + responses: + 200: + description: Validation result + content: + application/json: + schema: + $ref: '#/components/schemas/validationResultResponse' /record/{id}: delete: operationId: deleteRecord diff --git a/package.json b/package.json index a7b9d79..de2aeb0 100644 --- a/package.json +++ b/package.json @@ -14,7 +14,7 @@ "prelint:fix": "npm run format:fix", "prelint": "npm run format", "lint:openapi": "redocly lint openapi3.yaml", - "openapi:bundle": "redocly bundle openapi3.yaml -o bundledApi.yaml", + "bundleOpenApi": "redocly bundle openapi3.yaml -o bundledApi.yaml", "lint": "eslint .", "lint:fix": "eslint --fix .", "prebuild": "npm run clean && npm run generate:openapi-types", diff --git a/src/openapi.d.ts b/src/openapi.d.ts index bb6fa22..25eb5b6 100644 --- a/src/openapi.d.ts +++ b/src/openapi.d.ts @@ -44,6 +44,26 @@ export type paths = { patch?: never; trace?: never; }; + '/record/canDelete/{id}': { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * Validate whether a record can be deleted + * @description Runs the delete validation and returns the result without creating a job. + */ + get: operations['canDeleteRecord']; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; '/record/{id}': { parameters: { query?: never; @@ -210,6 +230,29 @@ export interface operations { }; }; }; + canDeleteRecord: { + parameters: { + query?: never; + header?: never; + path: { + /** @description The record identifier */ + id: components['parameters']['recordId']; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Validation result */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + 'application/json': components['schemas']['validationResultResponse']; + }; + }; + }; + }; deleteRecord: { parameters: { query?: never; diff --git a/src/record/controllers/recordController.ts b/src/record/controllers/recordController.ts index d40b219..ecba02c 100644 --- a/src/record/controllers/recordController.ts +++ b/src/record/controllers/recordController.ts @@ -51,6 +51,18 @@ export class RecordController { } }; + public canDeleteRecord: TypedRequestHandlers['canDeleteRecord'] = async (req, res, next) => { + const logContext = { ...this.logContext, function: this.canDeleteRecord.name }; + const { id } = req.params; + try { + const result = await this.manager.canDelete(id); + return res.status(StatusCodes.OK).json(result); + } catch (err) { + this.logger.error({ msg: 'failed to validate record deletability', logContext, err, recordId: id }); + return next(err); + } + }; + public deleteRecord: TypedRequestHandlers['deleteRecord'] = async (req, res, next) => { const logContext = { ...this.logContext, function: this.deleteRecord.name }; const { id } = req.params; diff --git a/src/record/models/recordManager.ts b/src/record/models/recordManager.ts index cf12ccf..d7cb86d 100644 --- a/src/record/models/recordManager.ts +++ b/src/record/models/recordManager.ts @@ -62,6 +62,20 @@ export class RecordManager { } } + public async canDelete(id: string): Promise { + const logContext = { ...this.logContext, function: this.canDelete.name }; + this.logger.info({ msg: 'validating record deletability', logContext, recordId: id }); + try { + await this.validator.validateDelete(id); + return { isValid: true }; + } catch (err) { + if (err instanceof AppError) { + return { isValid: false, message: err.message }; + } + throw err; + } + } + public async deleteRecord(id: string): Promise { const logContext = { ...this.logContext, function: this.deleteRecord.name }; this.logger.info({ msg: 'creating delete job', logContext, recordId: id }); diff --git a/src/record/routes/recordRouter.ts b/src/record/routes/recordRouter.ts index 077ac6f..aa98f7c 100644 --- a/src/record/routes/recordRouter.ts +++ b/src/record/routes/recordRouter.ts @@ -8,6 +8,7 @@ const recordRouterFactory: FactoryFunction = (dependencyContainer) => { router.post('/', controller.createRecord); router.post('/validate', controller.validateRecord); + router.get('/canDelete/:id', controller.canDeleteRecord); router.patch('/status/:id', controller.updateRecordStatus); router.delete('/:id', controller.deleteRecord); router.patch('/:id', controller.updateRecord); diff --git a/tests/integration/record/record.spec.ts b/tests/integration/record/record.spec.ts index 659ac5c..31e5427 100644 --- a/tests/integration/record/record.spec.ts +++ b/tests/integration/record/record.spec.ts @@ -124,6 +124,19 @@ describe('record', function () { }); }); + describe('GET /record/canDelete/{id}', function () { + it('should return 200 with isValid true for a deletable record', async function () { + const response = await requestSender.canDeleteRecord({ pathParams: { id: 'rec-1' } }); + + expect(response).toSatisfyApiSpec(); + expect(response.status).toBe(httpStatusCodes.OK); + + const body = response.body; + + expect(body.isValid).toBe(true); + }); + }); + describe('PATCH /record/{id}', function () { it('should return 200 and an ack for a metadata update', async function () { const response = await requestSender.updateRecord({ pathParams: { id: 'rec-1' }, requestBody: { description: 'updated' } }); From 487f7d2ccdd113071407b285c775f7253901e2f4 Mon Sep 17 00:00:00 2001 From: TULCHINSKI LIRAN Date: Wed, 30 Sep 2026 16:29:11 +0300 Subject: [PATCH 4/7] refactor: split API into jobOperations, jobStatus, models and metadata groups (MAPCO-11833) Restructure the routes to mirror 3d-gateway + store-trigger groupings instead of a single /record tag: - jobOperations: POST /jobOperations/ingestion, POST /jobOperations/delete - jobStatus: GET /jobStatus/{jobId} - models: POST /models/validate, GET /models/canDelete/{recordId} - metadata: PATCH /metadata/{identifier}, PATCH /metadata/status/{identifier} Delete is now POST with a body ({id}) like store-trigger. Each group has its own controller and router; RecordManager and JobManager remain the shared business models. OpenAPI, generated types, bundledApi and integration tests updated. --- bundledApi.yaml | 158 +++++++------ openapi3.yaml | 158 +++++++------ src/containerConfig.ts | 12 +- src/job/routes/jobRouter.ts | 16 -- .../controllers/jobOperationsController.ts | 54 +++++ .../routes/jobOperationsRouter.ts | 17 ++ .../controllers/jobStatusController.ts} | 6 +- src/jobStatus/routes/jobStatusRouter.ts | 16 ++ .../controllers/metadataController.ts | 46 ++++ src/metadata/routes/metadataRouter.ts | 17 ++ src/models/controllers/modelsController.ts | 45 ++++ src/models/routes/modelsRouter.ts | 17 ++ src/openapi.d.ts | 207 ++++++++++-------- src/record/controllers/recordController.ts | 101 --------- src/record/routes/recordRouter.ts | 21 -- src/serverBuilder.ts | 18 +- tests/integration/record/record.spec.ts | 101 +++++---- 17 files changed, 578 insertions(+), 432 deletions(-) delete mode 100644 src/job/routes/jobRouter.ts create mode 100644 src/jobOperations/controllers/jobOperationsController.ts create mode 100644 src/jobOperations/routes/jobOperationsRouter.ts rename src/{job/controllers/jobController.ts => jobStatus/controllers/jobStatusController.ts} (88%) create mode 100644 src/jobStatus/routes/jobStatusRouter.ts create mode 100644 src/metadata/controllers/metadataController.ts create mode 100644 src/metadata/routes/metadataRouter.ts create mode 100644 src/models/controllers/modelsController.ts create mode 100644 src/models/routes/modelsRouter.ts delete mode 100644 src/record/controllers/recordController.ts delete mode 100644 src/record/routes/recordRouter.ts diff --git a/bundledApi.yaml b/bundledApi.yaml index 4cfd255..edcb350 100644 --- a/bundledApi.yaml +++ b/bundledApi.yaml @@ -9,12 +9,12 @@ info: security: - {} paths: - /record: + /jobOperations/ingestion: post: - operationId: createRecord + operationId: createIngestion tags: - - record - summary: Start an ingestion process flow + - jobOperations + summary: Create a new ingestion job to invoke a new model ingestion flow description: Validates the request (light & fast) and creates a Jobnik ingestion job. requestBody: required: true @@ -35,35 +35,72 @@ paths: application/json: schema: $ref: '#/components/schemas/error' - /record/validate: + /jobOperations/delete: post: - operationId: validateRecord + operationId: createDelete tags: - - record - summary: Validate an ingestion request without creating a job - description: Runs the full ingestion validation and returns the result without triggering a Jobnik job. + - jobOperations + summary: Create a new delete job to invoke a delete flow + description: Validates the record can be deleted and creates a Jobnik delete job. requestBody: required: true content: application/json: schema: - $ref: '#/components/schemas/ingestionPayload' + $ref: '#/components/schemas/deletePayload' responses: '200': - description: Validation result + description: Delete job created content: application/json: schema: - $ref: '#/components/schemas/validationResultResponse' - /record/canDelete/{id}: + $ref: '#/components/schemas/jobResponse' + '400': + description: Bad Request + content: + application/json: + schema: + $ref: '#/components/schemas/error' + '404': + description: Record not found + content: + application/json: + schema: + $ref: '#/components/schemas/error' + /jobStatus/{jobId}: get: - operationId: canDeleteRecord + operationId: getJobStatus tags: - - record - summary: Validate whether a record can be deleted - description: Runs the delete validation and returns the result without creating a job. + - jobStatus + summary: Get the status and progress of a job parameters: - - $ref: '#/components/parameters/recordId' + - $ref: '#/components/parameters/jobId' + responses: + '200': + description: Job status + content: + application/json: + schema: + $ref: '#/components/schemas/jobStatusResponse' + '404': + description: Job not found + content: + application/json: + schema: + $ref: '#/components/schemas/error' + /models/validate: + post: + operationId: validateModel + tags: + - models + summary: Validate an ingestion request without creating a job + description: Runs the full ingestion validation and returns the result without triggering a Jobnik job. + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/ingestionPayload' responses: '200': description: Validation result @@ -71,41 +108,30 @@ paths: application/json: schema: $ref: '#/components/schemas/validationResultResponse' - /record/{id}: - delete: - operationId: deleteRecord + /models/canDelete/{recordId}: + get: + operationId: canDeleteModel tags: - - record - summary: Start a delete process flow - description: Validates the record can be deleted and creates a Jobnik delete job. + - models + summary: Validate whether a record can be deleted + description: Runs the delete validation and returns the result without creating a job. parameters: - $ref: '#/components/parameters/recordId' responses: '200': - description: Delete job created - content: - application/json: - schema: - $ref: '#/components/schemas/jobResponse' - '400': - description: Bad Request - content: - application/json: - schema: - $ref: '#/components/schemas/error' - '404': - description: Record not found + description: Validation result content: application/json: schema: - $ref: '#/components/schemas/error' + $ref: '#/components/schemas/validationResultResponse' + /metadata/{identifier}: patch: - operationId: updateRecord + operationId: updateMetadata tags: - - record - summary: Update metadata for a record + - metadata + summary: Update metadata of a model parameters: - - $ref: '#/components/parameters/recordId' + - $ref: '#/components/parameters/identifier' requestBody: required: true content: @@ -131,14 +157,14 @@ paths: application/json: schema: $ref: '#/components/schemas/error' - /record/status/{id}: + /metadata/status/{identifier}: patch: - operationId: updateRecordStatus + operationId: updateMetadataStatus tags: - - record - summary: Update the publish/unpublish status of a record + - metadata + summary: Update the publish/unpublish status of a model parameters: - - $ref: '#/components/parameters/recordId' + - $ref: '#/components/parameters/identifier' requestBody: required: true content: @@ -164,31 +190,17 @@ paths: application/json: schema: $ref: '#/components/schemas/error' - /jobs/{jobId}: - get: - operationId: getJobStatus - tags: - - job - summary: Get the status and progress of a job - parameters: - - $ref: '#/components/parameters/jobId' - responses: - '200': - description: Job status - content: - application/json: - schema: - $ref: '#/components/schemas/jobStatusResponse' - '404': - description: Job not found - content: - application/json: - schema: - $ref: '#/components/schemas/error' components: parameters: + identifier: + name: identifier + in: path + required: true + description: The record identifier + schema: + type: string recordId: - name: id + name: recordId in: path required: true description: The record identifier @@ -226,6 +238,14 @@ components: type: object description: 3D record metadata (business validation applied downstream) additionalProperties: true + deletePayload: + type: object + required: + - id + properties: + id: + type: string + description: The record identifier to delete updatePayload: type: object description: Partial metadata fields to update diff --git a/openapi3.yaml b/openapi3.yaml index ae7a446..7a9aea1 100644 --- a/openapi3.yaml +++ b/openapi3.yaml @@ -7,12 +7,12 @@ info: name: MIT url: https://opensource.org/licenses/MIT paths: - /record: + /jobOperations/ingestion: post: - operationId: createRecord + operationId: createIngestion tags: - - record - summary: Start an ingestion process flow + - jobOperations + summary: Create a new ingestion job to invoke a new model ingestion flow description: Validates the request (light & fast) and creates a Jobnik ingestion job. requestBody: required: true @@ -33,35 +33,72 @@ paths: application/json: schema: $ref: '#/components/schemas/error' - /record/validate: + /jobOperations/delete: post: - operationId: validateRecord + operationId: createDelete tags: - - record - summary: Validate an ingestion request without creating a job - description: Runs the full ingestion validation and returns the result without triggering a Jobnik job. + - jobOperations + summary: Create a new delete job to invoke a delete flow + description: Validates the record can be deleted and creates a Jobnik delete job. requestBody: required: true content: application/json: schema: - $ref: '#/components/schemas/ingestionPayload' + $ref: '#/components/schemas/deletePayload' responses: 200: - description: Validation result + description: Delete job created content: application/json: schema: - $ref: '#/components/schemas/validationResultResponse' - /record/canDelete/{id}: + $ref: '#/components/schemas/jobResponse' + 400: + description: Bad Request + content: + application/json: + schema: + $ref: '#/components/schemas/error' + 404: + description: Record not found + content: + application/json: + schema: + $ref: '#/components/schemas/error' + /jobStatus/{jobId}: get: - operationId: canDeleteRecord + operationId: getJobStatus tags: - - record - summary: Validate whether a record can be deleted - description: Runs the delete validation and returns the result without creating a job. + - jobStatus + summary: Get the status and progress of a job parameters: - - $ref: '#/components/parameters/recordId' + - $ref: '#/components/parameters/jobId' + responses: + 200: + description: Job status + content: + application/json: + schema: + $ref: '#/components/schemas/jobStatusResponse' + 404: + description: Job not found + content: + application/json: + schema: + $ref: '#/components/schemas/error' + /models/validate: + post: + operationId: validateModel + tags: + - models + summary: Validate an ingestion request without creating a job + description: Runs the full ingestion validation and returns the result without triggering a Jobnik job. + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/ingestionPayload' responses: 200: description: Validation result @@ -69,41 +106,30 @@ paths: application/json: schema: $ref: '#/components/schemas/validationResultResponse' - /record/{id}: - delete: - operationId: deleteRecord + /models/canDelete/{recordId}: + get: + operationId: canDeleteModel tags: - - record - summary: Start a delete process flow - description: Validates the record can be deleted and creates a Jobnik delete job. + - models + summary: Validate whether a record can be deleted + description: Runs the delete validation and returns the result without creating a job. parameters: - $ref: '#/components/parameters/recordId' responses: 200: - description: Delete job created - content: - application/json: - schema: - $ref: '#/components/schemas/jobResponse' - 400: - description: Bad Request - content: - application/json: - schema: - $ref: '#/components/schemas/error' - 404: - description: Record not found + description: Validation result content: application/json: schema: - $ref: '#/components/schemas/error' + $ref: '#/components/schemas/validationResultResponse' + /metadata/{identifier}: patch: - operationId: updateRecord + operationId: updateMetadata tags: - - record - summary: Update metadata for a record + - metadata + summary: Update metadata of a model parameters: - - $ref: '#/components/parameters/recordId' + - $ref: '#/components/parameters/identifier' requestBody: required: true content: @@ -129,14 +155,14 @@ paths: application/json: schema: $ref: '#/components/schemas/error' - /record/status/{id}: + /metadata/status/{identifier}: patch: - operationId: updateRecordStatus + operationId: updateMetadataStatus tags: - - record - summary: Update the publish/unpublish status of a record + - metadata + summary: Update the publish/unpublish status of a model parameters: - - $ref: '#/components/parameters/recordId' + - $ref: '#/components/parameters/identifier' requestBody: required: true content: @@ -162,33 +188,19 @@ paths: application/json: schema: $ref: '#/components/schemas/error' - /jobs/{jobId}: - get: - operationId: getJobStatus - tags: - - job - summary: Get the status and progress of a job - parameters: - - $ref: '#/components/parameters/jobId' - responses: - 200: - description: Job status - content: - application/json: - schema: - $ref: '#/components/schemas/jobStatusResponse' - 404: - description: Job not found - content: - application/json: - schema: - $ref: '#/components/schemas/error' security: - {} components: parameters: + identifier: + name: identifier + in: path + required: true + description: The record identifier + schema: + type: string recordId: - name: id + name: recordId in: path required: true description: The record identifier @@ -226,6 +238,14 @@ components: type: object description: 3D record metadata (business validation applied downstream) additionalProperties: true + deletePayload: + type: object + required: + - id + properties: + id: + type: string + description: The record identifier to delete updatePayload: type: object description: Partial metadata fields to update diff --git a/src/containerConfig.ts b/src/containerConfig.ts index 8a4bf69..690ebe1 100644 --- a/src/containerConfig.ts +++ b/src/containerConfig.ts @@ -6,8 +6,10 @@ import { jsLogger } from '@map-colonies/js-logger'; import { type InjectionObject, registerDependencies } from '@common/dependencyRegistration'; import { SERVICES, SERVICE_NAME } from '@common/constants'; import { getTracing } from '@common/tracing'; -import { recordRouterFactory, RECORD_ROUTER_SYMBOL } from './record/routes/recordRouter'; -import { jobRouterFactory, JOB_ROUTER_SYMBOL } from './job/routes/jobRouter'; +import { jobOperationsRouterFactory, JOB_OPERATIONS_ROUTER_SYMBOL } from './jobOperations/routes/jobOperationsRouter'; +import { jobStatusRouterFactory, JOB_STATUS_ROUTER_SYMBOL } from './jobStatus/routes/jobStatusRouter'; +import { modelsRouterFactory, MODELS_ROUTER_SYMBOL } from './models/routes/modelsRouter'; +import { metadataRouterFactory, METADATA_ROUTER_SYMBOL } from './metadata/routes/metadataRouter'; import { providerFactory } from './providers/getProvider'; import { getConfig } from './common/config'; @@ -33,8 +35,10 @@ export const registerExternalValues = async (options?: RegisterOptions): Promise { token: SERVICES.TRACER, provider: { useValue: tracer } }, { token: SERVICES.METRICS, provider: { useValue: metricsRegistry } }, { token: SERVICES.PROVIDER, provider: { useFactory: providerFactory } }, - { token: RECORD_ROUTER_SYMBOL, provider: { useFactory: recordRouterFactory } }, - { token: JOB_ROUTER_SYMBOL, provider: { useFactory: jobRouterFactory } }, + { token: JOB_OPERATIONS_ROUTER_SYMBOL, provider: { useFactory: jobOperationsRouterFactory } }, + { token: JOB_STATUS_ROUTER_SYMBOL, provider: { useFactory: jobStatusRouterFactory } }, + { token: MODELS_ROUTER_SYMBOL, provider: { useFactory: modelsRouterFactory } }, + { token: METADATA_ROUTER_SYMBOL, provider: { useFactory: metadataRouterFactory } }, { token: 'onSignal', provider: { diff --git a/src/job/routes/jobRouter.ts b/src/job/routes/jobRouter.ts deleted file mode 100644 index 0c646d7..0000000 --- a/src/job/routes/jobRouter.ts +++ /dev/null @@ -1,16 +0,0 @@ -import { Router } from 'express'; -import type { FactoryFunction } from 'tsyringe'; -import { JobController } from '../controllers/jobController'; - -const jobRouterFactory: FactoryFunction = (dependencyContainer) => { - const router = Router(); - const controller = dependencyContainer.resolve(JobController); - - router.get('/:jobId', controller.getJobStatus); - - return router; -}; - -export const JOB_ROUTER_SYMBOL = Symbol('jobRouterFactory'); - -export { jobRouterFactory }; diff --git a/src/jobOperations/controllers/jobOperationsController.ts b/src/jobOperations/controllers/jobOperationsController.ts new file mode 100644 index 0000000..32683fb --- /dev/null +++ b/src/jobOperations/controllers/jobOperationsController.ts @@ -0,0 +1,54 @@ +import type { Logger } from '@map-colonies/js-logger'; +import { StatusCodes } from 'http-status-codes'; +import { injectable, inject } from 'tsyringe'; +import { type Registry, Counter } from 'prom-client'; +import type { TypedRequestHandlers } from '@openapi'; +import { SERVICES } from '@common/constants'; +import type { LogContext } from '@common/interfaces'; +import { RecordManager } from '../../record/models/recordManager'; + +@injectable() +export class JobOperationsController { + private readonly logContext: LogContext; + private readonly ingestionJobCounter: Counter; + + public constructor( + @inject(SERVICES.LOGGER) private readonly logger: Logger, + @inject(RecordManager) private readonly manager: RecordManager, + @inject(SERVICES.METRICS) private readonly metricsRegistry: Registry + ) { + this.logContext = { + fileName: __filename, + class: JobOperationsController.name, + }; + this.ingestionJobCounter = new Counter({ + name: 'ingestion_jobs_created', + help: 'number of ingestion jobs created', + registers: [this.metricsRegistry], + }); + } + + public createIngestion: TypedRequestHandlers['createIngestion'] = async (req, res, next) => { + const logContext = { ...this.logContext, function: this.createIngestion.name }; + try { + const job = await this.manager.createIngestion(req.body); + this.ingestionJobCounter.inc(1); + return res.status(StatusCodes.CREATED).json(job); + } catch (err) { + this.logger.error({ msg: 'failed to create ingestion job', logContext, err }); + return next(err); + } + }; + + public createDelete: TypedRequestHandlers['createDelete'] = async (req, res, next) => { + const logContext = { ...this.logContext, function: this.createDelete.name }; + const { id } = req.body; + try { + const job = await this.manager.deleteRecord(id); + return res.status(StatusCodes.OK).json(job); + } catch (err) { + this.logger.error({ msg: 'failed to create delete job', logContext, err, recordId: id }); + return next(err); + } + }; +} diff --git a/src/jobOperations/routes/jobOperationsRouter.ts b/src/jobOperations/routes/jobOperationsRouter.ts new file mode 100644 index 0000000..c6d60a6 --- /dev/null +++ b/src/jobOperations/routes/jobOperationsRouter.ts @@ -0,0 +1,17 @@ +import { Router } from 'express'; +import type { FactoryFunction } from 'tsyringe'; +import { JobOperationsController } from '../controllers/jobOperationsController'; + +const jobOperationsRouterFactory: FactoryFunction = (dependencyContainer) => { + const router = Router(); + const controller = dependencyContainer.resolve(JobOperationsController); + + router.post('/ingestion', controller.createIngestion); + router.post('/delete', controller.createDelete); + + return router; +}; + +export const JOB_OPERATIONS_ROUTER_SYMBOL = Symbol('jobOperationsRouterFactory'); + +export { jobOperationsRouterFactory }; diff --git a/src/job/controllers/jobController.ts b/src/jobStatus/controllers/jobStatusController.ts similarity index 88% rename from src/job/controllers/jobController.ts rename to src/jobStatus/controllers/jobStatusController.ts index 64148ce..968b570 100644 --- a/src/job/controllers/jobController.ts +++ b/src/jobStatus/controllers/jobStatusController.ts @@ -4,10 +4,10 @@ import { injectable, inject } from 'tsyringe'; import type { TypedRequestHandlers } from '@openapi'; import { SERVICES } from '@common/constants'; import type { LogContext } from '@common/interfaces'; -import { JobManager } from '../models/jobManager'; +import { JobManager } from '../../job/models/jobManager'; @injectable() -export class JobController { +export class JobStatusController { private readonly logContext: LogContext; public constructor( @@ -16,7 +16,7 @@ export class JobController { ) { this.logContext = { fileName: __filename, - class: JobController.name, + class: JobStatusController.name, }; } diff --git a/src/jobStatus/routes/jobStatusRouter.ts b/src/jobStatus/routes/jobStatusRouter.ts new file mode 100644 index 0000000..9d3230a --- /dev/null +++ b/src/jobStatus/routes/jobStatusRouter.ts @@ -0,0 +1,16 @@ +import { Router } from 'express'; +import type { FactoryFunction } from 'tsyringe'; +import { JobStatusController } from '../controllers/jobStatusController'; + +const jobStatusRouterFactory: FactoryFunction = (dependencyContainer) => { + const router = Router(); + const controller = dependencyContainer.resolve(JobStatusController); + + router.get('/:jobId', controller.getJobStatus); + + return router; +}; + +export const JOB_STATUS_ROUTER_SYMBOL = Symbol('jobStatusRouterFactory'); + +export { jobStatusRouterFactory }; diff --git a/src/metadata/controllers/metadataController.ts b/src/metadata/controllers/metadataController.ts new file mode 100644 index 0000000..cdf0b58 --- /dev/null +++ b/src/metadata/controllers/metadataController.ts @@ -0,0 +1,46 @@ +import type { Logger } from '@map-colonies/js-logger'; +import { StatusCodes } from 'http-status-codes'; +import { injectable, inject } from 'tsyringe'; +import type { TypedRequestHandlers } from '@openapi'; +import { SERVICES } from '@common/constants'; +import type { LogContext } from '@common/interfaces'; +import { RecordManager } from '../../record/models/recordManager'; + +@injectable() +export class MetadataController { + private readonly logContext: LogContext; + + public constructor( + @inject(SERVICES.LOGGER) private readonly logger: Logger, + @inject(RecordManager) private readonly manager: RecordManager + ) { + this.logContext = { + fileName: __filename, + class: MetadataController.name, + }; + } + + public updateMetadata: TypedRequestHandlers['updateMetadata'] = async (req, res, next) => { + const logContext = { ...this.logContext, function: this.updateMetadata.name }; + const { identifier } = req.params; + try { + const ack = await this.manager.updateMetadata(identifier, req.body); + return res.status(StatusCodes.OK).json(ack); + } catch (err) { + this.logger.error({ msg: 'failed to update record metadata', logContext, err, recordId: identifier }); + return next(err); + } + }; + + public updateMetadataStatus: TypedRequestHandlers['updateMetadataStatus'] = async (req, res, next) => { + const logContext = { ...this.logContext, function: this.updateMetadataStatus.name }; + const { identifier } = req.params; + try { + const ack = await this.manager.updateStatus(identifier, req.body); + return res.status(StatusCodes.OK).json(ack); + } catch (err) { + this.logger.error({ msg: 'failed to update record status', logContext, err, recordId: identifier }); + return next(err); + } + }; +} diff --git a/src/metadata/routes/metadataRouter.ts b/src/metadata/routes/metadataRouter.ts new file mode 100644 index 0000000..c8ca7ed --- /dev/null +++ b/src/metadata/routes/metadataRouter.ts @@ -0,0 +1,17 @@ +import { Router } from 'express'; +import type { FactoryFunction } from 'tsyringe'; +import { MetadataController } from '../controllers/metadataController'; + +const metadataRouterFactory: FactoryFunction = (dependencyContainer) => { + const router = Router(); + const controller = dependencyContainer.resolve(MetadataController); + + router.patch('/status/:identifier', controller.updateMetadataStatus); + router.patch('/:identifier', controller.updateMetadata); + + return router; +}; + +export const METADATA_ROUTER_SYMBOL = Symbol('metadataRouterFactory'); + +export { metadataRouterFactory }; diff --git a/src/models/controllers/modelsController.ts b/src/models/controllers/modelsController.ts new file mode 100644 index 0000000..f001419 --- /dev/null +++ b/src/models/controllers/modelsController.ts @@ -0,0 +1,45 @@ +import type { Logger } from '@map-colonies/js-logger'; +import { StatusCodes } from 'http-status-codes'; +import { injectable, inject } from 'tsyringe'; +import type { TypedRequestHandlers } from '@openapi'; +import { SERVICES } from '@common/constants'; +import type { LogContext } from '@common/interfaces'; +import { RecordManager } from '../../record/models/recordManager'; + +@injectable() +export class ModelsController { + private readonly logContext: LogContext; + + public constructor( + @inject(SERVICES.LOGGER) private readonly logger: Logger, + @inject(RecordManager) private readonly manager: RecordManager + ) { + this.logContext = { + fileName: __filename, + class: ModelsController.name, + }; + } + + public validateModel: TypedRequestHandlers['validateModel'] = async (req, res, next) => { + const logContext = { ...this.logContext, function: this.validateModel.name }; + try { + const result = await this.manager.validateIngestion(req.body); + return res.status(StatusCodes.OK).json(result); + } catch (err) { + this.logger.error({ msg: 'failed to validate ingestion request', logContext, err }); + return next(err); + } + }; + + public canDeleteModel: TypedRequestHandlers['canDeleteModel'] = async (req, res, next) => { + const logContext = { ...this.logContext, function: this.canDeleteModel.name }; + const { recordId } = req.params; + try { + const result = await this.manager.canDelete(recordId); + return res.status(StatusCodes.OK).json(result); + } catch (err) { + this.logger.error({ msg: 'failed to validate record deletability', logContext, err, recordId }); + return next(err); + } + }; +} diff --git a/src/models/routes/modelsRouter.ts b/src/models/routes/modelsRouter.ts new file mode 100644 index 0000000..1716abb --- /dev/null +++ b/src/models/routes/modelsRouter.ts @@ -0,0 +1,17 @@ +import { Router } from 'express'; +import type { FactoryFunction } from 'tsyringe'; +import { ModelsController } from '../controllers/modelsController'; + +const modelsRouterFactory: FactoryFunction = (dependencyContainer) => { + const router = Router(); + const controller = dependencyContainer.resolve(ModelsController); + + router.post('/validate', controller.validateModel); + router.get('/canDelete/:recordId', controller.canDeleteModel); + + return router; +}; + +export const MODELS_ROUTER_SYMBOL = Symbol('modelsRouterFactory'); + +export { modelsRouterFactory }; diff --git a/src/openapi.d.ts b/src/openapi.d.ts index 25eb5b6..e4c5832 100644 --- a/src/openapi.d.ts +++ b/src/openapi.d.ts @@ -4,7 +4,7 @@ import type { TypedRequestHandlers as ImportedTypedRequestHandlers } from '@map-colonies/openapi-express-types'; export type paths = { - '/record': { + '/jobOperations/ingestion': { parameters: { query?: never; header?: never; @@ -14,17 +14,17 @@ export type paths = { get?: never; put?: never; /** - * Start an ingestion process flow + * Create a new ingestion job to invoke a new model ingestion flow * @description Validates the request (light & fast) and creates a Jobnik ingestion job. */ - post: operations['createRecord']; + post: operations['createIngestion']; delete?: never; options?: never; head?: never; patch?: never; trace?: never; }; - '/record/validate': { + '/jobOperations/delete': { parameters: { query?: never; header?: never; @@ -34,28 +34,25 @@ export type paths = { get?: never; put?: never; /** - * Validate an ingestion request without creating a job - * @description Runs the full ingestion validation and returns the result without triggering a Jobnik job. + * Create a new delete job to invoke a delete flow + * @description Validates the record can be deleted and creates a Jobnik delete job. */ - post: operations['validateRecord']; + post: operations['createDelete']; delete?: never; options?: never; head?: never; patch?: never; trace?: never; }; - '/record/canDelete/{id}': { + '/jobStatus/{jobId}': { parameters: { query?: never; header?: never; path?: never; cookie?: never; }; - /** - * Validate whether a record can be deleted - * @description Runs the delete validation and returns the result without creating a job. - */ - get: operations['canDeleteRecord']; + /** Get the status and progress of a job */ + get: operations['getJobStatus']; put?: never; post?: never; delete?: never; @@ -64,7 +61,7 @@ export type paths = { patch?: never; trace?: never; }; - '/record/{id}': { + '/models/validate': { parameters: { query?: never; header?: never; @@ -73,19 +70,38 @@ export type paths = { }; get?: never; put?: never; - post?: never; /** - * Start a delete process flow - * @description Validates the record can be deleted and creates a Jobnik delete job. + * Validate an ingestion request without creating a job + * @description Runs the full ingestion validation and returns the result without triggering a Jobnik job. + */ + post: operations['validateModel']; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + '/models/canDelete/{recordId}': { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * Validate whether a record can be deleted + * @description Runs the delete validation and returns the result without creating a job. */ - delete: operations['deleteRecord']; + get: operations['canDeleteModel']; + put?: never; + post?: never; + delete?: never; options?: never; head?: never; - /** Update metadata for a record */ - patch: operations['updateRecord']; + patch?: never; trace?: never; }; - '/record/status/{id}': { + '/metadata/{identifier}': { parameters: { query?: never; header?: never; @@ -98,25 +114,25 @@ export type paths = { delete?: never; options?: never; head?: never; - /** Update the publish/unpublish status of a record */ - patch: operations['updateRecordStatus']; + /** Update metadata of a model */ + patch: operations['updateMetadata']; trace?: never; }; - '/jobs/{jobId}': { + '/metadata/status/{identifier}': { parameters: { query?: never; header?: never; path?: never; cookie?: never; }; - /** Get the status and progress of a job */ - get: operations['getJobStatus']; + get?: never; put?: never; post?: never; delete?: never; options?: never; head?: never; - patch?: never; + /** Update the publish/unpublish status of a model */ + patch: operations['updateMetadataStatus']; trace?: never; }; }; @@ -136,6 +152,10 @@ export type components = { [key: string]: unknown; }; }; + deletePayload: { + /** @description The record identifier to delete */ + id: string; + }; /** @description Partial metadata fields to update */ updatePayload: { [key: string]: unknown; @@ -162,6 +182,8 @@ export type components = { }; responses: never; parameters: { + /** @description The record identifier */ + identifier: string; /** @description The record identifier */ recordId: string; /** @description The job identifier */ @@ -173,7 +195,7 @@ export type components = { }; export type $defs = Record; export interface operations { - createRecord: { + createIngestion: { parameters: { query?: never; header?: never; @@ -206,7 +228,7 @@ export interface operations { }; }; }; - validateRecord: { + createDelete: { parameters: { query?: never; header?: never; @@ -215,92 +237,125 @@ export interface operations { }; requestBody: { content: { - 'application/json': components['schemas']['ingestionPayload']; + 'application/json': components['schemas']['deletePayload']; }; }; responses: { - /** @description Validation result */ + /** @description Delete job created */ 200: { headers: { [name: string]: unknown; }; content: { - 'application/json': components['schemas']['validationResultResponse']; + 'application/json': components['schemas']['jobResponse']; + }; + }; + /** @description Bad Request */ + 400: { + headers: { + [name: string]: unknown; + }; + content: { + 'application/json': components['schemas']['error']; + }; + }; + /** @description Record not found */ + 404: { + headers: { + [name: string]: unknown; + }; + content: { + 'application/json': components['schemas']['error']; }; }; }; }; - canDeleteRecord: { + getJobStatus: { parameters: { query?: never; header?: never; path: { - /** @description The record identifier */ - id: components['parameters']['recordId']; + /** @description The job identifier */ + jobId: components['parameters']['jobId']; }; cookie?: never; }; requestBody?: never; responses: { - /** @description Validation result */ + /** @description Job status */ 200: { headers: { [name: string]: unknown; }; content: { - 'application/json': components['schemas']['validationResultResponse']; + 'application/json': components['schemas']['jobStatusResponse']; + }; + }; + /** @description Job not found */ + 404: { + headers: { + [name: string]: unknown; + }; + content: { + 'application/json': components['schemas']['error']; }; }; }; }; - deleteRecord: { + validateModel: { parameters: { query?: never; header?: never; - path: { - /** @description The record identifier */ - id: components['parameters']['recordId']; - }; + path?: never; cookie?: never; }; - requestBody?: never; + requestBody: { + content: { + 'application/json': components['schemas']['ingestionPayload']; + }; + }; responses: { - /** @description Delete job created */ + /** @description Validation result */ 200: { headers: { [name: string]: unknown; }; content: { - 'application/json': components['schemas']['jobResponse']; + 'application/json': components['schemas']['validationResultResponse']; }; }; - /** @description Bad Request */ - 400: { - headers: { - [name: string]: unknown; - }; - content: { - 'application/json': components['schemas']['error']; - }; + }; + }; + canDeleteModel: { + parameters: { + query?: never; + header?: never; + path: { + /** @description The record identifier */ + recordId: components['parameters']['recordId']; }; - /** @description Record not found */ - 404: { + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Validation result */ + 200: { headers: { [name: string]: unknown; }; content: { - 'application/json': components['schemas']['error']; + 'application/json': components['schemas']['validationResultResponse']; }; }; }; }; - updateRecord: { + updateMetadata: { parameters: { query?: never; header?: never; path: { /** @description The record identifier */ - id: components['parameters']['recordId']; + identifier: components['parameters']['identifier']; }; cookie?: never; }; @@ -339,13 +394,13 @@ export interface operations { }; }; }; - updateRecordStatus: { + updateMetadataStatus: { parameters: { query?: never; header?: never; path: { /** @description The record identifier */ - id: components['parameters']['recordId']; + identifier: components['parameters']['identifier']; }; cookie?: never; }; @@ -384,37 +439,5 @@ export interface operations { }; }; }; - getJobStatus: { - parameters: { - query?: never; - header?: never; - path: { - /** @description The job identifier */ - jobId: components['parameters']['jobId']; - }; - cookie?: never; - }; - requestBody?: never; - responses: { - /** @description Job status */ - 200: { - headers: { - [name: string]: unknown; - }; - content: { - 'application/json': components['schemas']['jobStatusResponse']; - }; - }; - /** @description Job not found */ - 404: { - headers: { - [name: string]: unknown; - }; - content: { - 'application/json': components['schemas']['error']; - }; - }; - }; - }; } export type TypedRequestHandlers = ImportedTypedRequestHandlers; diff --git a/src/record/controllers/recordController.ts b/src/record/controllers/recordController.ts deleted file mode 100644 index ecba02c..0000000 --- a/src/record/controllers/recordController.ts +++ /dev/null @@ -1,101 +0,0 @@ -import type { Logger } from '@map-colonies/js-logger'; -import { StatusCodes } from 'http-status-codes'; -import { injectable, inject } from 'tsyringe'; -import { type Registry, Counter } from 'prom-client'; -import type { TypedRequestHandlers } from '@openapi'; -import { SERVICES } from '@common/constants'; -import type { LogContext } from '@common/interfaces'; -import { RecordManager } from '../models/recordManager'; - -@injectable() -export class RecordController { - private readonly logContext: LogContext; - private readonly ingestionJobCounter: Counter; - - public constructor( - @inject(SERVICES.LOGGER) private readonly logger: Logger, - @inject(RecordManager) private readonly manager: RecordManager, - @inject(SERVICES.METRICS) private readonly metricsRegistry: Registry - ) { - this.logContext = { - fileName: __filename, - class: RecordController.name, - }; - this.ingestionJobCounter = new Counter({ - name: 'ingestion_jobs_created', - help: 'number of ingestion jobs created', - registers: [this.metricsRegistry], - }); - } - - public createRecord: TypedRequestHandlers['createRecord'] = async (req, res, next) => { - const logContext = { ...this.logContext, function: this.createRecord.name }; - try { - const job = await this.manager.createIngestion(req.body); - this.ingestionJobCounter.inc(1); - return res.status(StatusCodes.CREATED).json(job); - } catch (err) { - this.logger.error({ msg: 'failed to create ingestion job', logContext, err }); - return next(err); - } - }; - - public validateRecord: TypedRequestHandlers['validateRecord'] = async (req, res, next) => { - const logContext = { ...this.logContext, function: this.validateRecord.name }; - try { - const result = await this.manager.validateIngestion(req.body); - return res.status(StatusCodes.OK).json(result); - } catch (err) { - this.logger.error({ msg: 'failed to validate ingestion request', logContext, err }); - return next(err); - } - }; - - public canDeleteRecord: TypedRequestHandlers['canDeleteRecord'] = async (req, res, next) => { - const logContext = { ...this.logContext, function: this.canDeleteRecord.name }; - const { id } = req.params; - try { - const result = await this.manager.canDelete(id); - return res.status(StatusCodes.OK).json(result); - } catch (err) { - this.logger.error({ msg: 'failed to validate record deletability', logContext, err, recordId: id }); - return next(err); - } - }; - - public deleteRecord: TypedRequestHandlers['deleteRecord'] = async (req, res, next) => { - const logContext = { ...this.logContext, function: this.deleteRecord.name }; - const { id } = req.params; - try { - const job = await this.manager.deleteRecord(id); - return res.status(StatusCodes.OK).json(job); - } catch (err) { - this.logger.error({ msg: 'failed to create delete job', logContext, err, recordId: id }); - return next(err); - } - }; - - public updateRecord: TypedRequestHandlers['updateRecord'] = async (req, res, next) => { - const logContext = { ...this.logContext, function: this.updateRecord.name }; - const { id } = req.params; - try { - const ack = await this.manager.updateMetadata(id, req.body); - return res.status(StatusCodes.OK).json(ack); - } catch (err) { - this.logger.error({ msg: 'failed to update record metadata', logContext, err, recordId: id }); - return next(err); - } - }; - - public updateRecordStatus: TypedRequestHandlers['updateRecordStatus'] = async (req, res, next) => { - const logContext = { ...this.logContext, function: this.updateRecordStatus.name }; - const { id } = req.params; - try { - const ack = await this.manager.updateStatus(id, req.body); - return res.status(StatusCodes.OK).json(ack); - } catch (err) { - this.logger.error({ msg: 'failed to update record status', logContext, err, recordId: id }); - return next(err); - } - }; -} diff --git a/src/record/routes/recordRouter.ts b/src/record/routes/recordRouter.ts deleted file mode 100644 index aa98f7c..0000000 --- a/src/record/routes/recordRouter.ts +++ /dev/null @@ -1,21 +0,0 @@ -import { Router } from 'express'; -import type { FactoryFunction } from 'tsyringe'; -import { RecordController } from '../controllers/recordController'; - -const recordRouterFactory: FactoryFunction = (dependencyContainer) => { - const router = Router(); - const controller = dependencyContainer.resolve(RecordController); - - router.post('/', controller.createRecord); - router.post('/validate', controller.validateRecord); - router.get('/canDelete/:id', controller.canDeleteRecord); - router.patch('/status/:id', controller.updateRecordStatus); - router.delete('/:id', controller.deleteRecord); - router.patch('/:id', controller.updateRecord); - - return router; -}; - -export const RECORD_ROUTER_SYMBOL = Symbol('recordRouterFactory'); - -export { recordRouterFactory }; diff --git a/src/serverBuilder.ts b/src/serverBuilder.ts index eff4ee4..4c8e7fd 100644 --- a/src/serverBuilder.ts +++ b/src/serverBuilder.ts @@ -10,8 +10,10 @@ import { collectMetricsExpressMiddleware } from '@map-colonies/prometheus'; import { Registry } from 'prom-client'; import type { ConfigType } from '@common/config'; import { SERVICES } from '@common/constants'; -import { RECORD_ROUTER_SYMBOL } from './record/routes/recordRouter'; -import { JOB_ROUTER_SYMBOL } from './job/routes/jobRouter'; +import { JOB_OPERATIONS_ROUTER_SYMBOL } from './jobOperations/routes/jobOperationsRouter'; +import { JOB_STATUS_ROUTER_SYMBOL } from './jobStatus/routes/jobStatusRouter'; +import { MODELS_ROUTER_SYMBOL } from './models/routes/modelsRouter'; +import { METADATA_ROUTER_SYMBOL } from './metadata/routes/metadataRouter'; @injectable() export class ServerBuilder { @@ -21,8 +23,10 @@ export class ServerBuilder { @inject(SERVICES.CONFIG) private readonly config: ConfigType, @inject(SERVICES.LOGGER) private readonly logger: Logger, @inject(SERVICES.METRICS) private readonly metricsRegistry: Registry, - @inject(RECORD_ROUTER_SYMBOL) private readonly recordRouter: Router, - @inject(JOB_ROUTER_SYMBOL) private readonly jobRouter: Router + @inject(JOB_OPERATIONS_ROUTER_SYMBOL) private readonly jobOperationsRouter: Router, + @inject(JOB_STATUS_ROUTER_SYMBOL) private readonly jobStatusRouter: Router, + @inject(MODELS_ROUTER_SYMBOL) private readonly modelsRouter: Router, + @inject(METADATA_ROUTER_SYMBOL) private readonly metadataRouter: Router ) { this.serverInstance = express(); } @@ -45,8 +49,10 @@ export class ServerBuilder { } private buildRoutes(): void { - this.serverInstance.use('/record', this.recordRouter); - this.serverInstance.use('/jobs', this.jobRouter); + this.serverInstance.use('/jobOperations', this.jobOperationsRouter); + this.serverInstance.use('/jobStatus', this.jobStatusRouter); + this.serverInstance.use('/models', this.modelsRouter); + this.serverInstance.use('/metadata', this.metadataRouter); this.buildDocsRoutes(); } diff --git a/tests/integration/record/record.spec.ts b/tests/integration/record/record.spec.ts index 31e5427..acdd9de 100644 --- a/tests/integration/record/record.spec.ts +++ b/tests/integration/record/record.spec.ts @@ -29,7 +29,6 @@ const deletableRecord = { }; const lookupStub = { getClassifications: vi.fn().mockResolvedValue(['abc123']) } as unknown as LookupTablesCall; -// findRecords by id (delete) → an existing deletable record; by productName (uniqueness) → none const catalogStub = { findRecords: vi.fn().mockImplementation((payload: { id?: string }) => (payload.id !== undefined ? [deletableRecord] : [])), getRecord: vi.fn().mockResolvedValue(deletableRecord), @@ -54,7 +53,7 @@ const validIngestionPayload = { metadata: validMetadata, }; -describe('record', function () { +describe('3d-ops-trigger', function () { let requestSender: RequestSender; beforeAll(async function () { @@ -78,23 +77,23 @@ describe('record', function () { requestSender = await createRequestSender('openapi3.yaml', app); }); - describe('POST /record', function () { + describe('POST /jobOperations/ingestion', function () { it('should return 201 and a job response for a valid ingestion request', async function () { - const response = await requestSender.createRecord({ requestBody: validIngestionPayload }); + const response = await requestSender.createIngestion({ requestBody: validIngestionPayload }); expect(response).toSatisfyApiSpec(); expect(response.status).toBe(httpStatusCodes.CREATED); - const body = response.body as paths['/record']['post']['responses']['201']['content']['application/json']; + const body = response.body as paths['/jobOperations/ingestion']['post']['responses']['201']['content']['application/json']; expect(body.jobId).toBeTypeOf('string'); expect(body.status).toBeTypeOf('string'); }); it('should return 400 when a required field is missing', async function () { - const response = await requestSender.createRecord({ + const response = await requestSender.createIngestion({ // @ts-expect-error intentionally invalid: missing tilesetFilename and metadata - requestBody: { modelPath: '/shared/models/afula' }, + requestBody: { modelPath: '/app/models/afula' }, }); expect(response).toSatisfyApiSpec(); @@ -102,7 +101,7 @@ describe('record', function () { }); it('should return 400 when metadata fails business validation', async function () { - const response = await requestSender.createRecord({ + const response = await requestSender.createIngestion({ requestBody: { ...validIngestionPayload, metadata: { ...validMetadata, productType: 'NOT_A_3D_TYPE' } }, }); @@ -111,65 +110,63 @@ describe('record', function () { }); }); - describe('DELETE /record/{id}', function () { + describe('POST /jobOperations/delete', function () { it('should return 200 and a job response', async function () { - const response = await requestSender.deleteRecord({ pathParams: { id: 'rec-1' } }); + const response = await requestSender.createDelete({ requestBody: { id: 'rec-1' } }); expect(response).toSatisfyApiSpec(); expect(response.status).toBe(httpStatusCodes.OK); - const body = response.body as paths['/record/{id}']['delete']['responses']['200']['content']['application/json']; + const body = response.body as paths['/jobOperations/delete']['post']['responses']['200']['content']['application/json']; expect(body.jobId).toBeTypeOf('string'); }); }); - describe('GET /record/canDelete/{id}', function () { - it('should return 200 with isValid true for a deletable record', async function () { - const response = await requestSender.canDeleteRecord({ pathParams: { id: 'rec-1' } }); + describe('GET /jobStatus/{jobId}', function () { + it('should return 200 with the job status and percentage', async function () { + const response = await requestSender.getJobStatus({ pathParams: { jobId: 'job-1' } }); expect(response).toSatisfyApiSpec(); expect(response.status).toBe(httpStatusCodes.OK); - const body = response.body; + const body = response.body as paths['/jobStatus/{jobId}']['get']['responses']['200']['content']['application/json']; - expect(body.isValid).toBe(true); + expect(body.status).toBeTypeOf('string'); + expect(body.percentage).toBe(42); }); }); - describe('PATCH /record/{id}', function () { - it('should return 200 and an ack for a metadata update', async function () { - const response = await requestSender.updateRecord({ pathParams: { id: 'rec-1' }, requestBody: { description: 'updated' } }); + describe('POST /models/validate', function () { + it('should return 200 with isValid true for a valid request without creating a job', async function () { + const response = await requestSender.validateModel({ requestBody: validIngestionPayload }); expect(response).toSatisfyApiSpec(); expect(response.status).toBe(httpStatusCodes.OK); - expect(response.body.message).toBeTypeOf('string'); - }); - }); - describe('PATCH /record/status/{id}', function () { - it('should return 200 and an ack for a valid status change', async function () { - const response = await requestSender.updateRecordStatus({ pathParams: { id: 'rec-1' }, requestBody: { status: 'PUBLISHED' } }); + const body = response.body; - expect(response).toSatisfyApiSpec(); - expect(response.status).toBe(httpStatusCodes.OK); + expect(body.isValid).toBe(true); }); - it('should return 400 for an invalid status value', async function () { - const response = await requestSender.updateRecordStatus({ - pathParams: { id: 'rec-1' }, - // @ts-expect-error intentionally invalid status enum value - requestBody: { status: 'NOT_A_STATUS' }, + it('should return 200 with isValid false and a message for an invalid request', async function () { + const response = await requestSender.validateModel({ + requestBody: { ...validIngestionPayload, metadata: { ...validMetadata, productType: 'NOT_A_3D_TYPE' } }, }); expect(response).toSatisfyApiSpec(); - expect(response.status).toBe(httpStatusCodes.BAD_REQUEST); + expect(response.status).toBe(httpStatusCodes.OK); + + const body = response.body; + + expect(body.isValid).toBe(false); + expect(body.message).toBeTypeOf('string'); }); }); - describe('POST /record/validate', function () { - it('should return 200 with isValid true for a valid request without creating a job', async function () { - const response = await requestSender.validateRecord({ requestBody: validIngestionPayload }); + describe('GET /models/canDelete/{recordId}', function () { + it('should return 200 with isValid true for a deletable record', async function () { + const response = await requestSender.canDeleteModel({ pathParams: { recordId: 'rec-1' } }); expect(response).toSatisfyApiSpec(); expect(response.status).toBe(httpStatusCodes.OK); @@ -178,33 +175,35 @@ describe('record', function () { expect(body.isValid).toBe(true); }); + }); - it('should return 200 with isValid false and a message for an invalid request', async function () { - const response = await requestSender.validateRecord({ - requestBody: { ...validIngestionPayload, metadata: { ...validMetadata, productType: 'NOT_A_3D_TYPE' } }, - }); + describe('PATCH /metadata/{identifier}', function () { + it('should return 200 and an ack for a metadata update', async function () { + const response = await requestSender.updateMetadata({ pathParams: { identifier: 'rec-1' }, requestBody: { description: 'updated' } }); expect(response).toSatisfyApiSpec(); expect(response.status).toBe(httpStatusCodes.OK); - - const body = response.body; - - expect(body.isValid).toBe(false); - expect(body.message).toBeTypeOf('string'); + expect(response.body.message).toBeTypeOf('string'); }); }); - describe('GET /jobs/{jobId}', function () { - it('should return 200 with the job status and percentage', async function () { - const response = await requestSender.getJobStatus({ pathParams: { jobId: 'job-1' } }); + describe('PATCH /metadata/status/{identifier}', function () { + it('should return 200 and an ack for a valid status change', async function () { + const response = await requestSender.updateMetadataStatus({ pathParams: { identifier: 'rec-1' }, requestBody: { status: 'PUBLISHED' } }); expect(response).toSatisfyApiSpec(); expect(response.status).toBe(httpStatusCodes.OK); + }); - const body = response.body as paths['/jobs/{jobId}']['get']['responses']['200']['content']['application/json']; + it('should return 400 for an invalid status value', async function () { + const response = await requestSender.updateMetadataStatus({ + pathParams: { identifier: 'rec-1' }, + // @ts-expect-error intentionally invalid status enum value + requestBody: { status: 'NOT_A_STATUS' }, + }); - expect(body.status).toBeTypeOf('string'); - expect(body.percentage).toBe(42); + expect(response).toSatisfyApiSpec(); + expect(response.status).toBe(httpStatusCodes.BAD_REQUEST); }); }); }); From e7f6062ac211662fff72583045a19620fe753beb Mon Sep 17 00:00:00 2001 From: TULCHINSKI LIRAN Date: Wed, 30 Sep 2026 16:39:14 +0300 Subject: [PATCH 5/7] refactor: consolidate model operations under the records group (MAPCO-11833) Mirror 3d-gateway's models group (renamed to our domain term "record"): all model operations live under one records tag, metadata and jobStatus stay separate. - records: POST /records, POST /records/validate, GET /records/canDelete/{recordId}, DELETE /records/{recordId} - metadata: PATCH /metadata/{identifier}, PATCH /metadata/status/{identifier} - jobStatus: GET /jobStatus/{jobId} Delete reverts to a REST DELETE. Merge the jobOperations and models modules into a single records module (recordManager moved under it); update routers, serverBuilder, containerConfig, OpenAPI, bundledApi and tests. --- bundledApi.yaml | 130 ++++++------ openapi3.yaml | 130 ++++++------ src/containerConfig.ts | 6 +- src/externalServices/jobnik/jobnikClient.ts | 2 +- .../routes/jobOperationsRouter.ts | 17 -- .../controllers/metadataController.ts | 2 +- src/models/controllers/modelsController.ts | 45 ----- src/models/routes/modelsRouter.ts | 17 -- src/openapi.d.ts | 185 +++++++++--------- .../controllers/recordsController.ts} | 43 +++- .../models/recordManager.ts | 0 src/records/routes/recordsRouter.ts | 19 ++ src/serverBuilder.ts | 9 +- src/validator/validationManager.ts | 2 +- tests/integration/record/record.spec.ts | 74 +++---- .../jobnik/jobnikClient.spec.ts | 2 +- .../models/recordManager.spec.ts | 2 +- .../unit/validator/validationManager.spec.ts | 2 +- 18 files changed, 308 insertions(+), 379 deletions(-) delete mode 100644 src/jobOperations/routes/jobOperationsRouter.ts delete mode 100644 src/models/controllers/modelsController.ts delete mode 100644 src/models/routes/modelsRouter.ts rename src/{jobOperations/controllers/jobOperationsController.ts => records/controllers/recordsController.ts} (52%) rename src/{record => records}/models/recordManager.ts (100%) create mode 100644 src/records/routes/recordsRouter.ts rename tests/unit/{record => records}/models/recordManager.spec.ts (99%) diff --git a/bundledApi.yaml b/bundledApi.yaml index edcb350..9d62ccf 100644 --- a/bundledApi.yaml +++ b/bundledApi.yaml @@ -9,12 +9,12 @@ info: security: - {} paths: - /jobOperations/ingestion: + /records: post: - operationId: createIngestion + operationId: createRecord tags: - - jobOperations - summary: Create a new ingestion job to invoke a new model ingestion flow + - records + summary: Invoke a new model ingestion flow description: Validates the request (light & fast) and creates a Jobnik ingestion job. requestBody: required: true @@ -35,72 +35,35 @@ paths: application/json: schema: $ref: '#/components/schemas/error' - /jobOperations/delete: + /records/validate: post: - operationId: createDelete + operationId: validateRecord tags: - - jobOperations - summary: Create a new delete job to invoke a delete flow - description: Validates the record can be deleted and creates a Jobnik delete job. + - records + summary: Invoke a new model ingestion validation + description: Runs the full ingestion validation and returns the result without triggering a Jobnik job. requestBody: required: true content: application/json: schema: - $ref: '#/components/schemas/deletePayload' + $ref: '#/components/schemas/ingestionPayload' responses: '200': - description: Delete job created - content: - application/json: - schema: - $ref: '#/components/schemas/jobResponse' - '400': - description: Bad Request - content: - application/json: - schema: - $ref: '#/components/schemas/error' - '404': - description: Record not found + description: Validation result content: application/json: schema: - $ref: '#/components/schemas/error' - /jobStatus/{jobId}: + $ref: '#/components/schemas/validationResultResponse' + /records/canDelete/{recordId}: get: - operationId: getJobStatus + operationId: canDeleteRecord tags: - - jobStatus - summary: Get the status and progress of a job + - records + summary: Validate if a record can be removed + description: Runs the delete validation and returns the result without creating a job. parameters: - - $ref: '#/components/parameters/jobId' - responses: - '200': - description: Job status - content: - application/json: - schema: - $ref: '#/components/schemas/jobStatusResponse' - '404': - description: Job not found - content: - application/json: - schema: - $ref: '#/components/schemas/error' - /models/validate: - post: - operationId: validateModel - tags: - - models - summary: Validate an ingestion request without creating a job - description: Runs the full ingestion validation and returns the result without triggering a Jobnik job. - requestBody: - required: true - content: - application/json: - schema: - $ref: '#/components/schemas/ingestionPayload' + - $ref: '#/components/parameters/recordId' responses: '200': description: Validation result @@ -108,22 +71,34 @@ paths: application/json: schema: $ref: '#/components/schemas/validationResultResponse' - /models/canDelete/{recordId}: - get: - operationId: canDeleteModel + /records/{recordId}: + delete: + operationId: deleteRecord tags: - - models - summary: Validate whether a record can be deleted - description: Runs the delete validation and returns the result without creating a job. + - records + summary: Delete a model by record id + description: Validates the record can be deleted and creates a Jobnik delete job. parameters: - $ref: '#/components/parameters/recordId' responses: '200': - description: Validation result + description: Delete job created content: application/json: schema: - $ref: '#/components/schemas/validationResultResponse' + $ref: '#/components/schemas/jobResponse' + '400': + description: Bad Request + content: + application/json: + schema: + $ref: '#/components/schemas/error' + '404': + description: Record not found + content: + application/json: + schema: + $ref: '#/components/schemas/error' /metadata/{identifier}: patch: operationId: updateMetadata @@ -190,6 +165,27 @@ paths: application/json: schema: $ref: '#/components/schemas/error' + /jobStatus/{jobId}: + get: + operationId: getJobStatus + tags: + - jobStatus + summary: Get the status and progress of a job + parameters: + - $ref: '#/components/parameters/jobId' + responses: + '200': + description: Job status + content: + application/json: + schema: + $ref: '#/components/schemas/jobStatusResponse' + '404': + description: Job not found + content: + application/json: + schema: + $ref: '#/components/schemas/error' components: parameters: identifier: @@ -238,14 +234,6 @@ components: type: object description: 3D record metadata (business validation applied downstream) additionalProperties: true - deletePayload: - type: object - required: - - id - properties: - id: - type: string - description: The record identifier to delete updatePayload: type: object description: Partial metadata fields to update diff --git a/openapi3.yaml b/openapi3.yaml index 7a9aea1..491b9d1 100644 --- a/openapi3.yaml +++ b/openapi3.yaml @@ -7,12 +7,12 @@ info: name: MIT url: https://opensource.org/licenses/MIT paths: - /jobOperations/ingestion: + /records: post: - operationId: createIngestion + operationId: createRecord tags: - - jobOperations - summary: Create a new ingestion job to invoke a new model ingestion flow + - records + summary: Invoke a new model ingestion flow description: Validates the request (light & fast) and creates a Jobnik ingestion job. requestBody: required: true @@ -33,72 +33,35 @@ paths: application/json: schema: $ref: '#/components/schemas/error' - /jobOperations/delete: + /records/validate: post: - operationId: createDelete + operationId: validateRecord tags: - - jobOperations - summary: Create a new delete job to invoke a delete flow - description: Validates the record can be deleted and creates a Jobnik delete job. + - records + summary: Invoke a new model ingestion validation + description: Runs the full ingestion validation and returns the result without triggering a Jobnik job. requestBody: required: true content: application/json: schema: - $ref: '#/components/schemas/deletePayload' + $ref: '#/components/schemas/ingestionPayload' responses: 200: - description: Delete job created - content: - application/json: - schema: - $ref: '#/components/schemas/jobResponse' - 400: - description: Bad Request - content: - application/json: - schema: - $ref: '#/components/schemas/error' - 404: - description: Record not found + description: Validation result content: application/json: schema: - $ref: '#/components/schemas/error' - /jobStatus/{jobId}: + $ref: '#/components/schemas/validationResultResponse' + /records/canDelete/{recordId}: get: - operationId: getJobStatus + operationId: canDeleteRecord tags: - - jobStatus - summary: Get the status and progress of a job + - records + summary: Validate if a record can be removed + description: Runs the delete validation and returns the result without creating a job. parameters: - - $ref: '#/components/parameters/jobId' - responses: - 200: - description: Job status - content: - application/json: - schema: - $ref: '#/components/schemas/jobStatusResponse' - 404: - description: Job not found - content: - application/json: - schema: - $ref: '#/components/schemas/error' - /models/validate: - post: - operationId: validateModel - tags: - - models - summary: Validate an ingestion request without creating a job - description: Runs the full ingestion validation and returns the result without triggering a Jobnik job. - requestBody: - required: true - content: - application/json: - schema: - $ref: '#/components/schemas/ingestionPayload' + - $ref: '#/components/parameters/recordId' responses: 200: description: Validation result @@ -106,22 +69,34 @@ paths: application/json: schema: $ref: '#/components/schemas/validationResultResponse' - /models/canDelete/{recordId}: - get: - operationId: canDeleteModel + /records/{recordId}: + delete: + operationId: deleteRecord tags: - - models - summary: Validate whether a record can be deleted - description: Runs the delete validation and returns the result without creating a job. + - records + summary: Delete a model by record id + description: Validates the record can be deleted and creates a Jobnik delete job. parameters: - $ref: '#/components/parameters/recordId' responses: 200: - description: Validation result + description: Delete job created content: application/json: schema: - $ref: '#/components/schemas/validationResultResponse' + $ref: '#/components/schemas/jobResponse' + 400: + description: Bad Request + content: + application/json: + schema: + $ref: '#/components/schemas/error' + 404: + description: Record not found + content: + application/json: + schema: + $ref: '#/components/schemas/error' /metadata/{identifier}: patch: operationId: updateMetadata @@ -188,6 +163,27 @@ paths: application/json: schema: $ref: '#/components/schemas/error' + /jobStatus/{jobId}: + get: + operationId: getJobStatus + tags: + - jobStatus + summary: Get the status and progress of a job + parameters: + - $ref: '#/components/parameters/jobId' + responses: + 200: + description: Job status + content: + application/json: + schema: + $ref: '#/components/schemas/jobStatusResponse' + 404: + description: Job not found + content: + application/json: + schema: + $ref: '#/components/schemas/error' security: - {} components: @@ -238,14 +234,6 @@ components: type: object description: 3D record metadata (business validation applied downstream) additionalProperties: true - deletePayload: - type: object - required: - - id - properties: - id: - type: string - description: The record identifier to delete updatePayload: type: object description: Partial metadata fields to update diff --git a/src/containerConfig.ts b/src/containerConfig.ts index 690ebe1..75be200 100644 --- a/src/containerConfig.ts +++ b/src/containerConfig.ts @@ -6,9 +6,8 @@ import { jsLogger } from '@map-colonies/js-logger'; import { type InjectionObject, registerDependencies } from '@common/dependencyRegistration'; import { SERVICES, SERVICE_NAME } from '@common/constants'; import { getTracing } from '@common/tracing'; -import { jobOperationsRouterFactory, JOB_OPERATIONS_ROUTER_SYMBOL } from './jobOperations/routes/jobOperationsRouter'; +import { recordsRouterFactory, RECORDS_ROUTER_SYMBOL } from './records/routes/recordsRouter'; import { jobStatusRouterFactory, JOB_STATUS_ROUTER_SYMBOL } from './jobStatus/routes/jobStatusRouter'; -import { modelsRouterFactory, MODELS_ROUTER_SYMBOL } from './models/routes/modelsRouter'; import { metadataRouterFactory, METADATA_ROUTER_SYMBOL } from './metadata/routes/metadataRouter'; import { providerFactory } from './providers/getProvider'; import { getConfig } from './common/config'; @@ -35,9 +34,8 @@ export const registerExternalValues = async (options?: RegisterOptions): Promise { token: SERVICES.TRACER, provider: { useValue: tracer } }, { token: SERVICES.METRICS, provider: { useValue: metricsRegistry } }, { token: SERVICES.PROVIDER, provider: { useFactory: providerFactory } }, - { token: JOB_OPERATIONS_ROUTER_SYMBOL, provider: { useFactory: jobOperationsRouterFactory } }, + { token: RECORDS_ROUTER_SYMBOL, provider: { useFactory: recordsRouterFactory } }, { token: JOB_STATUS_ROUTER_SYMBOL, provider: { useFactory: jobStatusRouterFactory } }, - { token: MODELS_ROUTER_SYMBOL, provider: { useFactory: modelsRouterFactory } }, { token: METADATA_ROUTER_SYMBOL, provider: { useFactory: metadataRouterFactory } }, { token: 'onSignal', diff --git a/src/externalServices/jobnik/jobnikClient.ts b/src/externalServices/jobnik/jobnikClient.ts index 0a9104e..63caf44 100644 --- a/src/externalServices/jobnik/jobnikClient.ts +++ b/src/externalServices/jobnik/jobnikClient.ts @@ -9,7 +9,7 @@ import { is3tz } from '@common/util'; import { AppError } from '@common/appError'; import type { ConfigType, JobManagerConfig } from '@common/config'; import type { LogContext } from '@common/interfaces'; -import type { IngestionPayload, JobResponse } from '../../record/models/recordManager'; +import type { IngestionPayload, JobResponse } from '../../records/models/recordManager'; import type { Record3D } from '../catalog/interfaces'; interface StageDescriptor { diff --git a/src/jobOperations/routes/jobOperationsRouter.ts b/src/jobOperations/routes/jobOperationsRouter.ts deleted file mode 100644 index c6d60a6..0000000 --- a/src/jobOperations/routes/jobOperationsRouter.ts +++ /dev/null @@ -1,17 +0,0 @@ -import { Router } from 'express'; -import type { FactoryFunction } from 'tsyringe'; -import { JobOperationsController } from '../controllers/jobOperationsController'; - -const jobOperationsRouterFactory: FactoryFunction = (dependencyContainer) => { - const router = Router(); - const controller = dependencyContainer.resolve(JobOperationsController); - - router.post('/ingestion', controller.createIngestion); - router.post('/delete', controller.createDelete); - - return router; -}; - -export const JOB_OPERATIONS_ROUTER_SYMBOL = Symbol('jobOperationsRouterFactory'); - -export { jobOperationsRouterFactory }; diff --git a/src/metadata/controllers/metadataController.ts b/src/metadata/controllers/metadataController.ts index cdf0b58..c7da02e 100644 --- a/src/metadata/controllers/metadataController.ts +++ b/src/metadata/controllers/metadataController.ts @@ -4,7 +4,7 @@ import { injectable, inject } from 'tsyringe'; import type { TypedRequestHandlers } from '@openapi'; import { SERVICES } from '@common/constants'; import type { LogContext } from '@common/interfaces'; -import { RecordManager } from '../../record/models/recordManager'; +import { RecordManager } from '../../records/models/recordManager'; @injectable() export class MetadataController { diff --git a/src/models/controllers/modelsController.ts b/src/models/controllers/modelsController.ts deleted file mode 100644 index f001419..0000000 --- a/src/models/controllers/modelsController.ts +++ /dev/null @@ -1,45 +0,0 @@ -import type { Logger } from '@map-colonies/js-logger'; -import { StatusCodes } from 'http-status-codes'; -import { injectable, inject } from 'tsyringe'; -import type { TypedRequestHandlers } from '@openapi'; -import { SERVICES } from '@common/constants'; -import type { LogContext } from '@common/interfaces'; -import { RecordManager } from '../../record/models/recordManager'; - -@injectable() -export class ModelsController { - private readonly logContext: LogContext; - - public constructor( - @inject(SERVICES.LOGGER) private readonly logger: Logger, - @inject(RecordManager) private readonly manager: RecordManager - ) { - this.logContext = { - fileName: __filename, - class: ModelsController.name, - }; - } - - public validateModel: TypedRequestHandlers['validateModel'] = async (req, res, next) => { - const logContext = { ...this.logContext, function: this.validateModel.name }; - try { - const result = await this.manager.validateIngestion(req.body); - return res.status(StatusCodes.OK).json(result); - } catch (err) { - this.logger.error({ msg: 'failed to validate ingestion request', logContext, err }); - return next(err); - } - }; - - public canDeleteModel: TypedRequestHandlers['canDeleteModel'] = async (req, res, next) => { - const logContext = { ...this.logContext, function: this.canDeleteModel.name }; - const { recordId } = req.params; - try { - const result = await this.manager.canDelete(recordId); - return res.status(StatusCodes.OK).json(result); - } catch (err) { - this.logger.error({ msg: 'failed to validate record deletability', logContext, err, recordId }); - return next(err); - } - }; -} diff --git a/src/models/routes/modelsRouter.ts b/src/models/routes/modelsRouter.ts deleted file mode 100644 index 1716abb..0000000 --- a/src/models/routes/modelsRouter.ts +++ /dev/null @@ -1,17 +0,0 @@ -import { Router } from 'express'; -import type { FactoryFunction } from 'tsyringe'; -import { ModelsController } from '../controllers/modelsController'; - -const modelsRouterFactory: FactoryFunction = (dependencyContainer) => { - const router = Router(); - const controller = dependencyContainer.resolve(ModelsController); - - router.post('/validate', controller.validateModel); - router.get('/canDelete/:recordId', controller.canDeleteModel); - - return router; -}; - -export const MODELS_ROUTER_SYMBOL = Symbol('modelsRouterFactory'); - -export { modelsRouterFactory }; diff --git a/src/openapi.d.ts b/src/openapi.d.ts index e4c5832..07455f1 100644 --- a/src/openapi.d.ts +++ b/src/openapi.d.ts @@ -4,7 +4,7 @@ import type { TypedRequestHandlers as ImportedTypedRequestHandlers } from '@map-colonies/openapi-express-types'; export type paths = { - '/jobOperations/ingestion': { + '/records': { parameters: { query?: never; header?: never; @@ -14,17 +14,17 @@ export type paths = { get?: never; put?: never; /** - * Create a new ingestion job to invoke a new model ingestion flow + * Invoke a new model ingestion flow * @description Validates the request (light & fast) and creates a Jobnik ingestion job. */ - post: operations['createIngestion']; + post: operations['createRecord']; delete?: never; options?: never; head?: never; patch?: never; trace?: never; }; - '/jobOperations/delete': { + '/records/validate': { parameters: { query?: never; header?: never; @@ -34,25 +34,28 @@ export type paths = { get?: never; put?: never; /** - * Create a new delete job to invoke a delete flow - * @description Validates the record can be deleted and creates a Jobnik delete job. + * Invoke a new model ingestion validation + * @description Runs the full ingestion validation and returns the result without triggering a Jobnik job. */ - post: operations['createDelete']; + post: operations['validateRecord']; delete?: never; options?: never; head?: never; patch?: never; trace?: never; }; - '/jobStatus/{jobId}': { + '/records/canDelete/{recordId}': { parameters: { query?: never; header?: never; path?: never; cookie?: never; }; - /** Get the status and progress of a job */ - get: operations['getJobStatus']; + /** + * Validate if a record can be removed + * @description Runs the delete validation and returns the result without creating a job. + */ + get: operations['canDeleteRecord']; put?: never; post?: never; delete?: never; @@ -61,7 +64,7 @@ export type paths = { patch?: never; trace?: never; }; - '/models/validate': { + '/records/{recordId}': { parameters: { query?: never; header?: never; @@ -70,38 +73,35 @@ export type paths = { }; get?: never; put?: never; + post?: never; /** - * Validate an ingestion request without creating a job - * @description Runs the full ingestion validation and returns the result without triggering a Jobnik job. + * Delete a model by record id + * @description Validates the record can be deleted and creates a Jobnik delete job. */ - post: operations['validateModel']; - delete?: never; + delete: operations['deleteRecord']; options?: never; head?: never; patch?: never; trace?: never; }; - '/models/canDelete/{recordId}': { + '/metadata/{identifier}': { parameters: { query?: never; header?: never; path?: never; cookie?: never; }; - /** - * Validate whether a record can be deleted - * @description Runs the delete validation and returns the result without creating a job. - */ - get: operations['canDeleteModel']; + get?: never; put?: never; post?: never; delete?: never; options?: never; head?: never; - patch?: never; + /** Update metadata of a model */ + patch: operations['updateMetadata']; trace?: never; }; - '/metadata/{identifier}': { + '/metadata/status/{identifier}': { parameters: { query?: never; header?: never; @@ -114,25 +114,25 @@ export type paths = { delete?: never; options?: never; head?: never; - /** Update metadata of a model */ - patch: operations['updateMetadata']; + /** Update the publish/unpublish status of a model */ + patch: operations['updateMetadataStatus']; trace?: never; }; - '/metadata/status/{identifier}': { + '/jobStatus/{jobId}': { parameters: { query?: never; header?: never; path?: never; cookie?: never; }; - get?: never; + /** Get the status and progress of a job */ + get: operations['getJobStatus']; put?: never; post?: never; delete?: never; options?: never; head?: never; - /** Update the publish/unpublish status of a model */ - patch: operations['updateMetadataStatus']; + patch?: never; trace?: never; }; }; @@ -152,10 +152,6 @@ export type components = { [key: string]: unknown; }; }; - deletePayload: { - /** @description The record identifier to delete */ - id: string; - }; /** @description Partial metadata fields to update */ updatePayload: { [key: string]: unknown; @@ -195,7 +191,7 @@ export type components = { }; export type $defs = Record; export interface operations { - createIngestion: { + createRecord: { parameters: { query?: never; header?: never; @@ -228,7 +224,7 @@ export interface operations { }; }; }; - createDelete: { + validateRecord: { parameters: { query?: never; header?: never; @@ -237,83 +233,32 @@ export interface operations { }; requestBody: { content: { - 'application/json': components['schemas']['deletePayload']; + 'application/json': components['schemas']['ingestionPayload']; }; }; responses: { - /** @description Delete job created */ + /** @description Validation result */ 200: { headers: { [name: string]: unknown; }; content: { - 'application/json': components['schemas']['jobResponse']; - }; - }; - /** @description Bad Request */ - 400: { - headers: { - [name: string]: unknown; - }; - content: { - 'application/json': components['schemas']['error']; - }; - }; - /** @description Record not found */ - 404: { - headers: { - [name: string]: unknown; - }; - content: { - 'application/json': components['schemas']['error']; + 'application/json': components['schemas']['validationResultResponse']; }; }; }; }; - getJobStatus: { + canDeleteRecord: { parameters: { query?: never; header?: never; path: { - /** @description The job identifier */ - jobId: components['parameters']['jobId']; + /** @description The record identifier */ + recordId: components['parameters']['recordId']; }; cookie?: never; }; requestBody?: never; - responses: { - /** @description Job status */ - 200: { - headers: { - [name: string]: unknown; - }; - content: { - 'application/json': components['schemas']['jobStatusResponse']; - }; - }; - /** @description Job not found */ - 404: { - headers: { - [name: string]: unknown; - }; - content: { - 'application/json': components['schemas']['error']; - }; - }; - }; - }; - validateModel: { - parameters: { - query?: never; - header?: never; - path?: never; - cookie?: never; - }; - requestBody: { - content: { - 'application/json': components['schemas']['ingestionPayload']; - }; - }; responses: { /** @description Validation result */ 200: { @@ -326,7 +271,7 @@ export interface operations { }; }; }; - canDeleteModel: { + deleteRecord: { parameters: { query?: never; header?: never; @@ -338,13 +283,31 @@ export interface operations { }; requestBody?: never; responses: { - /** @description Validation result */ + /** @description Delete job created */ 200: { headers: { [name: string]: unknown; }; content: { - 'application/json': components['schemas']['validationResultResponse']; + 'application/json': components['schemas']['jobResponse']; + }; + }; + /** @description Bad Request */ + 400: { + headers: { + [name: string]: unknown; + }; + content: { + 'application/json': components['schemas']['error']; + }; + }; + /** @description Record not found */ + 404: { + headers: { + [name: string]: unknown; + }; + content: { + 'application/json': components['schemas']['error']; }; }; }; @@ -439,5 +402,37 @@ export interface operations { }; }; }; + getJobStatus: { + parameters: { + query?: never; + header?: never; + path: { + /** @description The job identifier */ + jobId: components['parameters']['jobId']; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Job status */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + 'application/json': components['schemas']['jobStatusResponse']; + }; + }; + /** @description Job not found */ + 404: { + headers: { + [name: string]: unknown; + }; + content: { + 'application/json': components['schemas']['error']; + }; + }; + }; + }; } export type TypedRequestHandlers = ImportedTypedRequestHandlers; diff --git a/src/jobOperations/controllers/jobOperationsController.ts b/src/records/controllers/recordsController.ts similarity index 52% rename from src/jobOperations/controllers/jobOperationsController.ts rename to src/records/controllers/recordsController.ts index 32683fb..7bca530 100644 --- a/src/jobOperations/controllers/jobOperationsController.ts +++ b/src/records/controllers/recordsController.ts @@ -5,10 +5,10 @@ import { type Registry, Counter } from 'prom-client'; import type { TypedRequestHandlers } from '@openapi'; import { SERVICES } from '@common/constants'; import type { LogContext } from '@common/interfaces'; -import { RecordManager } from '../../record/models/recordManager'; +import { RecordManager } from '../models/recordManager'; @injectable() -export class JobOperationsController { +export class RecordsController { private readonly logContext: LogContext; private readonly ingestionJobCounter: Counter; @@ -19,7 +19,7 @@ export class JobOperationsController { ) { this.logContext = { fileName: __filename, - class: JobOperationsController.name, + class: RecordsController.name, }; this.ingestionJobCounter = new Counter({ name: 'ingestion_jobs_created', @@ -28,8 +28,8 @@ export class JobOperationsController { }); } - public createIngestion: TypedRequestHandlers['createIngestion'] = async (req, res, next) => { - const logContext = { ...this.logContext, function: this.createIngestion.name }; + public createRecord: TypedRequestHandlers['createRecord'] = async (req, res, next) => { + const logContext = { ...this.logContext, function: this.createRecord.name }; try { const job = await this.manager.createIngestion(req.body); this.ingestionJobCounter.inc(1); @@ -40,14 +40,37 @@ export class JobOperationsController { } }; - public createDelete: TypedRequestHandlers['createDelete'] = async (req, res, next) => { - const logContext = { ...this.logContext, function: this.createDelete.name }; - const { id } = req.body; + public validateRecord: TypedRequestHandlers['validateRecord'] = async (req, res, next) => { + const logContext = { ...this.logContext, function: this.validateRecord.name }; try { - const job = await this.manager.deleteRecord(id); + const result = await this.manager.validateIngestion(req.body); + return res.status(StatusCodes.OK).json(result); + } catch (err) { + this.logger.error({ msg: 'failed to validate ingestion request', logContext, err }); + return next(err); + } + }; + + public canDeleteRecord: TypedRequestHandlers['canDeleteRecord'] = async (req, res, next) => { + const logContext = { ...this.logContext, function: this.canDeleteRecord.name }; + const { recordId } = req.params; + try { + const result = await this.manager.canDelete(recordId); + return res.status(StatusCodes.OK).json(result); + } catch (err) { + this.logger.error({ msg: 'failed to validate record deletability', logContext, err, recordId }); + return next(err); + } + }; + + public deleteRecord: TypedRequestHandlers['deleteRecord'] = async (req, res, next) => { + const logContext = { ...this.logContext, function: this.deleteRecord.name }; + const { recordId } = req.params; + try { + const job = await this.manager.deleteRecord(recordId); return res.status(StatusCodes.OK).json(job); } catch (err) { - this.logger.error({ msg: 'failed to create delete job', logContext, err, recordId: id }); + this.logger.error({ msg: 'failed to create delete job', logContext, err, recordId }); return next(err); } }; diff --git a/src/record/models/recordManager.ts b/src/records/models/recordManager.ts similarity index 100% rename from src/record/models/recordManager.ts rename to src/records/models/recordManager.ts diff --git a/src/records/routes/recordsRouter.ts b/src/records/routes/recordsRouter.ts new file mode 100644 index 0000000..c22f629 --- /dev/null +++ b/src/records/routes/recordsRouter.ts @@ -0,0 +1,19 @@ +import { Router } from 'express'; +import type { FactoryFunction } from 'tsyringe'; +import { RecordsController } from '../controllers/recordsController'; + +const recordsRouterFactory: FactoryFunction = (dependencyContainer) => { + const router = Router(); + const controller = dependencyContainer.resolve(RecordsController); + + router.post('/', controller.createRecord); + router.post('/validate', controller.validateRecord); + router.get('/canDelete/:recordId', controller.canDeleteRecord); + router.delete('/:recordId', controller.deleteRecord); + + return router; +}; + +export const RECORDS_ROUTER_SYMBOL = Symbol('recordsRouterFactory'); + +export { recordsRouterFactory }; diff --git a/src/serverBuilder.ts b/src/serverBuilder.ts index 4c8e7fd..900566d 100644 --- a/src/serverBuilder.ts +++ b/src/serverBuilder.ts @@ -10,9 +10,8 @@ import { collectMetricsExpressMiddleware } from '@map-colonies/prometheus'; import { Registry } from 'prom-client'; import type { ConfigType } from '@common/config'; import { SERVICES } from '@common/constants'; -import { JOB_OPERATIONS_ROUTER_SYMBOL } from './jobOperations/routes/jobOperationsRouter'; +import { RECORDS_ROUTER_SYMBOL } from './records/routes/recordsRouter'; import { JOB_STATUS_ROUTER_SYMBOL } from './jobStatus/routes/jobStatusRouter'; -import { MODELS_ROUTER_SYMBOL } from './models/routes/modelsRouter'; import { METADATA_ROUTER_SYMBOL } from './metadata/routes/metadataRouter'; @injectable() @@ -23,9 +22,8 @@ export class ServerBuilder { @inject(SERVICES.CONFIG) private readonly config: ConfigType, @inject(SERVICES.LOGGER) private readonly logger: Logger, @inject(SERVICES.METRICS) private readonly metricsRegistry: Registry, - @inject(JOB_OPERATIONS_ROUTER_SYMBOL) private readonly jobOperationsRouter: Router, + @inject(RECORDS_ROUTER_SYMBOL) private readonly recordsRouter: Router, @inject(JOB_STATUS_ROUTER_SYMBOL) private readonly jobStatusRouter: Router, - @inject(MODELS_ROUTER_SYMBOL) private readonly modelsRouter: Router, @inject(METADATA_ROUTER_SYMBOL) private readonly metadataRouter: Router ) { this.serverInstance = express(); @@ -49,9 +47,8 @@ export class ServerBuilder { } private buildRoutes(): void { - this.serverInstance.use('/jobOperations', this.jobOperationsRouter); + this.serverInstance.use('/records', this.recordsRouter); this.serverInstance.use('/jobStatus', this.jobStatusRouter); - this.serverInstance.use('/models', this.modelsRouter); this.serverInstance.use('/metadata', this.metadataRouter); this.buildDocsRoutes(); } diff --git a/src/validator/validationManager.ts b/src/validator/validationManager.ts index 3805e3e..94e4445 100644 --- a/src/validator/validationManager.ts +++ b/src/validator/validationManager.ts @@ -17,7 +17,7 @@ import { ExtractableCall } from '../externalServices/extractableManagement/extra import { TilesetReader } from '../tileset/tilesetReader'; import type { Record3D } from '../externalServices/catalog/interfaces'; import type { Provider } from '../providers/interfaces'; -import type { IngestionPayload } from '../record/models/recordManager'; +import type { IngestionPayload } from '../records/models/recordManager'; type UpdatePayload = components['schemas']['updatePayload']; diff --git a/tests/integration/record/record.spec.ts b/tests/integration/record/record.spec.ts index acdd9de..64e7ad5 100644 --- a/tests/integration/record/record.spec.ts +++ b/tests/integration/record/record.spec.ts @@ -77,21 +77,21 @@ describe('3d-ops-trigger', function () { requestSender = await createRequestSender('openapi3.yaml', app); }); - describe('POST /jobOperations/ingestion', function () { + describe('POST /records', function () { it('should return 201 and a job response for a valid ingestion request', async function () { - const response = await requestSender.createIngestion({ requestBody: validIngestionPayload }); + const response = await requestSender.createRecord({ requestBody: validIngestionPayload }); expect(response).toSatisfyApiSpec(); expect(response.status).toBe(httpStatusCodes.CREATED); - const body = response.body as paths['/jobOperations/ingestion']['post']['responses']['201']['content']['application/json']; + const body = response.body as paths['/records']['post']['responses']['201']['content']['application/json']; expect(body.jobId).toBeTypeOf('string'); expect(body.status).toBeTypeOf('string'); }); it('should return 400 when a required field is missing', async function () { - const response = await requestSender.createIngestion({ + const response = await requestSender.createRecord({ // @ts-expect-error intentionally invalid: missing tilesetFilename and metadata requestBody: { modelPath: '/app/models/afula' }, }); @@ -101,7 +101,7 @@ describe('3d-ops-trigger', function () { }); it('should return 400 when metadata fails business validation', async function () { - const response = await requestSender.createIngestion({ + const response = await requestSender.createRecord({ requestBody: { ...validIngestionPayload, metadata: { ...validMetadata, productType: 'NOT_A_3D_TYPE' } }, }); @@ -110,36 +110,9 @@ describe('3d-ops-trigger', function () { }); }); - describe('POST /jobOperations/delete', function () { - it('should return 200 and a job response', async function () { - const response = await requestSender.createDelete({ requestBody: { id: 'rec-1' } }); - - expect(response).toSatisfyApiSpec(); - expect(response.status).toBe(httpStatusCodes.OK); - - const body = response.body as paths['/jobOperations/delete']['post']['responses']['200']['content']['application/json']; - - expect(body.jobId).toBeTypeOf('string'); - }); - }); - - describe('GET /jobStatus/{jobId}', function () { - it('should return 200 with the job status and percentage', async function () { - const response = await requestSender.getJobStatus({ pathParams: { jobId: 'job-1' } }); - - expect(response).toSatisfyApiSpec(); - expect(response.status).toBe(httpStatusCodes.OK); - - const body = response.body as paths['/jobStatus/{jobId}']['get']['responses']['200']['content']['application/json']; - - expect(body.status).toBeTypeOf('string'); - expect(body.percentage).toBe(42); - }); - }); - - describe('POST /models/validate', function () { + describe('POST /records/validate', function () { it('should return 200 with isValid true for a valid request without creating a job', async function () { - const response = await requestSender.validateModel({ requestBody: validIngestionPayload }); + const response = await requestSender.validateRecord({ requestBody: validIngestionPayload }); expect(response).toSatisfyApiSpec(); expect(response.status).toBe(httpStatusCodes.OK); @@ -150,7 +123,7 @@ describe('3d-ops-trigger', function () { }); it('should return 200 with isValid false and a message for an invalid request', async function () { - const response = await requestSender.validateModel({ + const response = await requestSender.validateRecord({ requestBody: { ...validIngestionPayload, metadata: { ...validMetadata, productType: 'NOT_A_3D_TYPE' } }, }); @@ -164,9 +137,9 @@ describe('3d-ops-trigger', function () { }); }); - describe('GET /models/canDelete/{recordId}', function () { + describe('GET /records/canDelete/{recordId}', function () { it('should return 200 with isValid true for a deletable record', async function () { - const response = await requestSender.canDeleteModel({ pathParams: { recordId: 'rec-1' } }); + const response = await requestSender.canDeleteRecord({ pathParams: { recordId: 'rec-1' } }); expect(response).toSatisfyApiSpec(); expect(response.status).toBe(httpStatusCodes.OK); @@ -177,6 +150,19 @@ describe('3d-ops-trigger', function () { }); }); + describe('DELETE /records/{recordId}', function () { + it('should return 200 and a job response', async function () { + const response = await requestSender.deleteRecord({ pathParams: { recordId: 'rec-1' } }); + + expect(response).toSatisfyApiSpec(); + expect(response.status).toBe(httpStatusCodes.OK); + + const body = response.body as paths['/records/{recordId}']['delete']['responses']['200']['content']['application/json']; + + expect(body.jobId).toBeTypeOf('string'); + }); + }); + describe('PATCH /metadata/{identifier}', function () { it('should return 200 and an ack for a metadata update', async function () { const response = await requestSender.updateMetadata({ pathParams: { identifier: 'rec-1' }, requestBody: { description: 'updated' } }); @@ -206,4 +192,18 @@ describe('3d-ops-trigger', function () { expect(response.status).toBe(httpStatusCodes.BAD_REQUEST); }); }); + + describe('GET /jobStatus/{jobId}', function () { + it('should return 200 with the job status and percentage', async function () { + const response = await requestSender.getJobStatus({ pathParams: { jobId: 'job-1' } }); + + expect(response).toSatisfyApiSpec(); + expect(response.status).toBe(httpStatusCodes.OK); + + const body = response.body as paths['/jobStatus/{jobId}']['get']['responses']['200']['content']['application/json']; + + expect(body.status).toBeTypeOf('string'); + expect(body.percentage).toBe(42); + }); + }); }); diff --git a/tests/unit/externalServices/jobnik/jobnikClient.spec.ts b/tests/unit/externalServices/jobnik/jobnikClient.spec.ts index 395cc7e..3c7fb5a 100644 --- a/tests/unit/externalServices/jobnik/jobnikClient.spec.ts +++ b/tests/unit/externalServices/jobnik/jobnikClient.spec.ts @@ -4,7 +4,7 @@ import { Registry } from 'prom-client'; import { JobnikClient } from '@src/externalServices/jobnik/jobnikClient'; import { STAGE_TYPES } from '@src/common/constants'; import type { ConfigType } from '@src/common/config'; -import type { IngestionPayload } from '@src/record/models/recordManager'; +import type { IngestionPayload } from '@src/records/models/recordManager'; const jobManagerConfig = { url: 'http://job-manager', diff --git a/tests/unit/record/models/recordManager.spec.ts b/tests/unit/records/models/recordManager.spec.ts similarity index 99% rename from tests/unit/record/models/recordManager.spec.ts rename to tests/unit/records/models/recordManager.spec.ts index 59cc0d7..ef7f88c 100644 --- a/tests/unit/record/models/recordManager.spec.ts +++ b/tests/unit/records/models/recordManager.spec.ts @@ -1,6 +1,6 @@ import { jsLogger } from '@map-colonies/js-logger'; import { describe, it, expect, beforeEach, vi } from 'vitest'; -import { RecordManager, type IngestionPayload } from '@src/record/models/recordManager'; +import { RecordManager, type IngestionPayload } from '@src/records/models/recordManager'; import type { ValidationManager } from '@src/validator/validationManager'; import type { JobnikClient } from '@src/externalServices/jobnik/jobnikClient'; import type { CatalogCall } from '@src/externalServices/catalog/catalogCall'; diff --git a/tests/unit/validator/validationManager.spec.ts b/tests/unit/validator/validationManager.spec.ts index 91f0509..f088ca6 100644 --- a/tests/unit/validator/validationManager.spec.ts +++ b/tests/unit/validator/validationManager.spec.ts @@ -28,7 +28,7 @@ import type { JobnikClient } from '@src/externalServices/jobnik/jobnikClient'; import type { TilesetReader } from '@src/tileset/tilesetReader'; import type { ExtractableCall } from '@src/externalServices/extractableManagement/extractableCall'; import type { Provider } from '@src/providers/interfaces'; -import type { IngestionPayload } from '@src/record/models/recordManager'; +import type { IngestionPayload } from '@src/records/models/recordManager'; import { buildValidMetadata } from '@tests/helpers/metadata'; const regionTilesetJson = readFileSync(join(__dirname, '../../helpers/tilesets/folder/tileset.json'), 'utf-8'); From ddb35c84cac40e1db29610ea3f9a90b1a93b16f8 Mon Sep 17 00:00:00 2001 From: TULCHINSKI LIRAN Date: Wed, 30 Sep 2026 16:55:25 +0300 Subject: [PATCH 6/7] feat: mint modelId for ingestion jobs and harden the create flow (MAPCO-11833) Mint a modelId (uuid) at ingestion time, pass it into the Jobnik job data so the finalizer has a stable catalog id, and tag the trace span with it. Wrap the create flow in a try/catch that passes AppError through and maps unexpected errors to a 500, mirroring 3d-gateway's createModel. productSource and PV-path rewriting are intentionally omitted: productSource does not exist in the v2 metadata schema, and path rewriting is store-trigger mount-specific (revisit only if the Jobnik worker contract requires it). --- src/externalServices/jobnik/jobnikClient.ts | 4 +-- src/records/models/recordManager.ts | 28 ++++++++++++++----- .../jobnik/jobnikClient.spec.ts | 9 ++++-- 3 files changed, 29 insertions(+), 12 deletions(-) diff --git a/src/externalServices/jobnik/jobnikClient.ts b/src/externalServices/jobnik/jobnikClient.ts index 63caf44..52d11bb 100644 --- a/src/externalServices/jobnik/jobnikClient.ts +++ b/src/externalServices/jobnik/jobnikClient.ts @@ -89,13 +89,13 @@ export class JobnikClient { return { status: data.status, percentage: data.percentage }; } - public async createIngestionJob(payload: IngestionPayload): Promise { + public async createIngestionJob(payload: IngestionPayload, modelId: string): Promise { const logContext = { ...this.logContext, function: this.createIngestionJob.name }; const isArchive = is3tz(payload.modelPath); const job = await this.producer.createJob({ name: this.jobManager.ingestion.jobType, - data: { modelPath: payload.modelPath, tilesetFilename: payload.tilesetFilename, metadata: payload.metadata }, + data: { modelId, modelPath: payload.modelPath, tilesetFilename: payload.tilesetFilename, metadata: payload.metadata }, }); const stages: StageDescriptor[] = [ diff --git a/src/records/models/recordManager.ts b/src/records/models/recordManager.ts index d7cb86d..8ecb245 100644 --- a/src/records/models/recordManager.ts +++ b/src/records/models/recordManager.ts @@ -1,5 +1,8 @@ +import { randomUUID } from 'node:crypto'; import type { Logger } from '@map-colonies/js-logger'; import { inject, injectable } from 'tsyringe'; +import { trace } from '@opentelemetry/api'; +import { StatusCodes } from 'http-status-codes'; import type { Polygon } from 'geojson'; import type { components } from '@openapi'; import { SERVICES } from '@common/constants'; @@ -37,15 +40,26 @@ export class RecordManager { public async createIngestion(payload: IngestionPayload): Promise { const logContext = { ...this.logContext, function: this.createIngestion.name }; - this.logger.info({ msg: 'creating ingestion job', logContext, modelPath: payload.modelPath, tilesetFilename: payload.tilesetFilename }); - await this.validator.validateIngestion(payload); + const modelId = randomUUID(); + this.logger.info({ msg: 'creating ingestion job', logContext, modelId, modelPath: payload.modelPath, tilesetFilename: payload.tilesetFilename }); + trace.getActiveSpan()?.setAttribute('catalogId', modelId); - const metadata = payload.metadata as Record; - if (metadata.footprint !== undefined) { - metadata.footprint = convertPolygonTo2DPolygon(metadata.footprint as Polygon); - } + try { + await this.validator.validateIngestion(payload); + + const metadata = payload.metadata as Record; + if (metadata.footprint !== undefined) { + metadata.footprint = convertPolygonTo2DPolygon(metadata.footprint as Polygon); + } - return this.jobnik.createIngestionJob(payload); + return await this.jobnik.createIngestionJob(payload, modelId); + } catch (err) { + if (err instanceof AppError) { + throw err; + } + this.logger.error({ msg: 'unexpected error while creating ingestion job', logContext, modelId, err }); + throw new AppError('error', StatusCodes.INTERNAL_SERVER_ERROR, String(err), true); + } } public async validateIngestion(payload: IngestionPayload): Promise { diff --git a/tests/unit/externalServices/jobnik/jobnikClient.spec.ts b/tests/unit/externalServices/jobnik/jobnikClient.spec.ts index 3c7fb5a..95378a3 100644 --- a/tests/unit/externalServices/jobnik/jobnikClient.spec.ts +++ b/tests/unit/externalServices/jobnik/jobnikClient.spec.ts @@ -56,9 +56,12 @@ describe('JobnikClient', function () { }); it('should create the ingestion job and a validation stage for a folder input', async function () { - const result = await client.createIngestionJob(buildPayload('/shared/models/afula')); + const result = await client.createIngestionJob(buildPayload('/shared/models/afula'), 'model-1'); - expect(producerMock.createJob).toHaveBeenCalledWith(expect.objectContaining({ name: jobManagerConfig.ingestion.jobType })); + const createJobArg = (producerMock.createJob.mock.calls as [{ name: string; data: { modelId: string } }][])[0][0]; + + expect(createJobArg.name).toBe(jobManagerConfig.ingestion.jobType); + expect(createJobArg.data.modelId).toBe('model-1'); const folderStageTypes = (producerMock.createStage.mock.calls as [string, { type: string }][]).map((call) => call[1].type); @@ -79,7 +82,7 @@ describe('JobnikClient', function () { }); it('should also create the data-extraction and clear-data stages for a 3tz input', async function () { - await client.createIngestionJob(buildPayload('/shared/models/afula.3tz')); + await client.createIngestionJob(buildPayload('/shared/models/afula.3tz'), 'model-1'); const stageTypes = (producerMock.createStage.mock.calls as [string, { type: string }][]).map((call) => call[1].type); From 31ccdaeeee92a242ed5ca601b2482cbf0e23cb4e Mon Sep 17 00:00:00 2001 From: TULCHINSKI LIRAN Date: Thu, 1 Oct 2026 11:44:52 +0300 Subject: [PATCH 7/7] refactor: centralize DTOs/enums and split the validator (MAPCO-11833) - move record/job DTO aliases into @common/interfaces, dropping the duplicate UpdatePayload and the local JobStatusResponse - consume RecordStatus/ProductType from @map-colonies/types instead of hardcoded status/productType literals (adds @map-colonies/types + mime-types) - split ValidationManager into IngestionValidator, MetadataValidator and TilesetValidator, with shared errors.ts and a classification helper - dedupe CatalogCall error handling behind a single request helper --- package-lock.json | 157 ++++++--- package.json | 2 + src/common/interfaces.ts | 10 + src/externalServices/catalog/catalogCall.ts | 78 +++-- src/externalServices/jobnik/jobnikClient.ts | 8 +- src/job/models/jobManager.ts | 5 +- src/records/models/recordManager.ts | 45 +-- src/validator/classification.ts | 24 ++ src/validator/errors.ts | 18 + src/validator/ingestionValidator.ts | 138 ++++++++ src/validator/metadataValidator.ts | 127 +++++++ src/validator/tilesetValidator.ts | 98 ++++++ src/validator/validationManager.ts | 327 ------------------ .../jobnik/jobnikClient.spec.ts | 2 +- .../unit/records/models/recordManager.spec.ts | 14 +- ...ger.spec.ts => ingestionValidator.spec.ts} | 127 +------ .../unit/validator/metadataValidator.spec.ts | 126 +++++++ 17 files changed, 745 insertions(+), 561 deletions(-) create mode 100644 src/validator/classification.ts create mode 100644 src/validator/errors.ts create mode 100644 src/validator/ingestionValidator.ts create mode 100644 src/validator/metadataValidator.ts create mode 100644 src/validator/tilesetValidator.ts delete mode 100644 src/validator/validationManager.ts rename tests/unit/validator/{validationManager.spec.ts => ingestionValidator.spec.ts} (62%) create mode 100644 tests/unit/validator/metadataValidator.spec.ts diff --git a/package-lock.json b/package-lock.json index 08ceef5..d48d080 100644 --- a/package-lock.json +++ b/package-lock.json @@ -23,6 +23,7 @@ "@map-colonies/schemas": "^1.26.0", "@map-colonies/tracing": "^1.0.0", "@map-colonies/tracing-utils": "^2.0.0", + "@map-colonies/types": "^1.4.0", "@opentelemetry/api": "^1.9.1", "@turf/turf": "^7.4.0", "axios": "^1.20.0", @@ -30,6 +31,7 @@ "express": "^4.21.2", "express-openapi-validator": "^5.6.2", "http-status-codes": "^2.3.0", + "mime-types": "^2.1.35", "prom-client": "^15.1.3", "reflect-metadata": "^0.2.2", "tsyringe": "^4.8.0", @@ -1869,6 +1871,19 @@ "node": ">=24" } }, + "node_modules/@map-colonies/types": { + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/@map-colonies/types/-/types-1.10.0.tgz", + "integrity": "sha512-FwVrBAYzjnpmd9Jnq7Mex7axl8KR6o8JjKVGu3kDCGHz26Ru9qSCpzkA+zUVV8fKNxWVZSmDi7IiHEe9WsOalw==", + "license": "ISC", + "dependencies": { + "@types/geojson": "^7946.0.16", + "@types/mime-types": "^2.1.1", + "copyfiles": "^2.4.1", + "epsg-index": "^1.3.0", + "ts-morph": "^18.0.0" + } + }, "node_modules/@napi-rs/wasm-runtime": { "version": "1.1.6", "resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-1.1.6.tgz", @@ -1905,7 +1920,6 @@ "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", "integrity": "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==", - "dev": true, "license": "MIT", "dependencies": { "@nodelib/fs.stat": "2.0.5", @@ -1919,7 +1933,6 @@ "version": "2.0.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz", "integrity": "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==", - "dev": true, "license": "MIT", "engines": { "node": ">= 8" @@ -1929,7 +1942,6 @@ "version": "1.2.8", "resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz", "integrity": "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==", - "dev": true, "license": "MIT", "dependencies": { "@nodelib/fs.scandir": "2.1.5", @@ -4763,6 +4775,63 @@ "dev": true, "license": "MIT" }, + "node_modules/@ts-morph/common": { + "version": "0.19.0", + "resolved": "https://registry.npmjs.org/@ts-morph/common/-/common-0.19.0.tgz", + "integrity": "sha512-Unz/WHmd4pGax91rdIKWi51wnVUW11QttMEPpBiBgIewnc9UQIX7UDLxr5vRlqeByXCwhkF6VabSsI0raWcyAQ==", + "license": "MIT", + "dependencies": { + "fast-glob": "^3.2.12", + "minimatch": "^7.4.3", + "mkdirp": "^2.1.6", + "path-browserify": "^1.0.1" + } + }, + "node_modules/@ts-morph/common/node_modules/balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", + "license": "MIT" + }, + "node_modules/@ts-morph/common/node_modules/brace-expansion": { + "version": "2.1.7", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.1.7.tgz", + "integrity": "sha512-uZbew1NqdmPDTMJ8ah1y+b+9QEJrfkXFk3RcTQw3X0jW/xRUvFKsg1CfQdSYGdTbXZWExtU3J3ccxtnfw1Fi0g==", + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0" + } + }, + "node_modules/@ts-morph/common/node_modules/minimatch": { + "version": "7.4.9", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-7.4.9.tgz", + "integrity": "sha512-Brg/fp/iAVDOQoHxkuN5bEYhyQlZhxddI78yWsCbeEwTHXQjlNLtiJDUsp1GIptVqMI7/gkJMz4vVAc01mpoBw==", + "license": "ISC", + "dependencies": { + "brace-expansion": "^2.0.2" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/@ts-morph/common/node_modules/mkdirp": { + "version": "2.1.6", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-2.1.6.tgz", + "integrity": "sha512-+hEnITedc8LAtIP9u3HJDFIdcLV2vXP33sqLLIzkv1Db1zO/1OxbvYf0Y1OC/S/Qo5dxHXepofhmxL02PsKe+A==", + "license": "MIT", + "bin": { + "mkdirp": "dist/cjs/src/bin.js" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/@turf/along": { "version": "7.4.0", "resolved": "https://registry.npmjs.org/@turf/along/-/along-7.4.0.tgz", @@ -7736,6 +7805,12 @@ "integrity": "sha512-/pyBZWSLD2n0dcHE3hq8s8ZvcETHtEuF+3E7XVt0Ig2nvsVQXdghHVcEkIWjy9A0wKfTn97a/PSDYohKIlnP/w==", "license": "MIT" }, + "node_modules/@types/mime-types": { + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/@types/mime-types/-/mime-types-2.1.4.tgz", + "integrity": "sha512-lfU4b34HOri+kAY5UheuFMWPDOI+OPceBSHZKp69gEyTL/mmJ4cnU6Y/rlme3UL3GyOn6Y42hyIEw0/q8sWx5w==", + "license": "MIT" + }, "node_modules/@types/multer": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/@types/multer/-/multer-2.2.0.tgz", @@ -9083,7 +9158,6 @@ "version": "3.0.3", "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz", "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==", - "dev": true, "license": "MIT", "dependencies": { "fill-range": "^7.1.1" @@ -9418,6 +9492,12 @@ "node": ">=8" } }, + "node_modules/code-block-writer": { + "version": "12.0.0", + "resolved": "https://registry.npmjs.org/code-block-writer/-/code-block-writer-12.0.0.tgz", + "integrity": "sha512-q4dMFMlXtKR3XNBHyMHt/3pwYNA69EDk00lloMOaaUMKPUXBw6lpXtbu3MMVG6/uOihGnRDOlkyqsONEUj60+w==", + "license": "MIT" + }, "node_modules/color-convert": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", @@ -9542,7 +9622,6 @@ "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", - "dev": true, "license": "MIT" }, "node_modules/concat-stream": { @@ -9681,7 +9760,6 @@ "version": "2.4.1", "resolved": "https://registry.npmjs.org/copyfiles/-/copyfiles-2.4.1.tgz", "integrity": "sha512-fereAvAvxDrQDOXybk3Qu3dPbOoKoysFMWtkY3mv5BsL8//OSZVL5DCLYqgRfY5cWirgRzlC+WSrxp6Bo3eNZg==", - "dev": true, "license": "MIT", "dependencies": { "glob": "^7.0.5", @@ -9701,14 +9779,12 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", - "dev": true, "license": "MIT" }, "node_modules/copyfiles/node_modules/brace-expansion": { "version": "1.1.15", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.15.tgz", "integrity": "sha512-EwOCDEex4quD37XhqM3omwtMoJjr//isUZz1JopUNWms+4Z2ViyM/k1YIRePpoVNnQhENnxtFjLaxNHrT7xIUg==", - "dev": true, "license": "MIT", "dependencies": { "balanced-match": "^1.0.0", @@ -9719,7 +9795,6 @@ "version": "7.0.4", "resolved": "https://registry.npmjs.org/cliui/-/cliui-7.0.4.tgz", "integrity": "sha512-OcRE68cOsVMXp1Yvonl/fzkQOyjLSu/8bhPDfQt0e0/Eb283TKP20Fs2MqoPsr9SwA595rRCA+QMzYc9nBP+JQ==", - "dev": true, "license": "ISC", "dependencies": { "string-width": "^4.2.0", @@ -9731,7 +9806,6 @@ "version": "3.1.5", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.5.tgz", "integrity": "sha512-VgjWUsnnT6n+NUk6eZq77zeFdpW2LWDzP6zFGrCbHXiYNul5Dzqk2HHQ5uFH2DNW5Xbp8+jVzaeNt94ssEEl4w==", - "dev": true, "license": "ISC", "dependencies": { "brace-expansion": "^1.1.7" @@ -9744,7 +9818,6 @@ "version": "4.2.3", "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", - "dev": true, "license": "MIT", "dependencies": { "emoji-regex": "^8.0.0", @@ -9759,7 +9832,6 @@ "version": "6.0.1", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "dev": true, "license": "MIT", "dependencies": { "ansi-regex": "^5.0.1" @@ -9772,7 +9844,6 @@ "version": "16.2.2", "resolved": "https://registry.npmjs.org/yargs/-/yargs-16.2.2.tgz", "integrity": "sha512-Nt9ZJjXTv5R8MHbqby/wXQ6Gi0Bb3TcYZkR1bzuL4yB2OxWPkXknz513gEF0GoA6tn00UpbPvERW8rzCuWCA6w==", - "dev": true, "license": "MIT", "dependencies": { "cliui": "^7.0.2", @@ -9791,7 +9862,6 @@ "version": "20.2.9", "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-20.2.9.tgz", "integrity": "sha512-y11nGElTIV+CT3Zv9t7VKl+Q3hTQoT9a1Qzezhhl6Rp21gJ/IVTW7Z3y9EWXhuUBC2Shnf+DX0antecpAwSP8w==", - "dev": true, "license": "ISC", "engines": { "node": ">=10" @@ -9815,7 +9885,6 @@ "version": "1.0.3", "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz", "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==", - "dev": true, "license": "MIT" }, "node_modules/cosmiconfig": { @@ -10111,6 +10180,15 @@ "node": ">=6" } }, + "node_modules/epsg-index": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/epsg-index/-/epsg-index-1.3.0.tgz", + "integrity": "sha512-pWJ7lkMbH0Au+MJyBV4GxiX+KmOknPc6c5FqDAYD61Ju3LblR60VGmOlteBhK9Dm3oy++Jxa2kUimONeSvsKRA==", + "license": "ISC", + "engines": { + "node": ">=8" + } + }, "node_modules/error-ex": { "version": "1.3.4", "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.4.tgz", @@ -10768,7 +10846,6 @@ "version": "3.3.3", "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.3.tgz", "integrity": "sha512-7MptL8U0cqcFdzIzwOTHoilX9x5BrNqye7Z/LuC7kCMRio1EMSyqRK3BEAUD7sXRq4iT4AzTVuZdhgQ2TCvYLg==", - "dev": true, "license": "MIT", "dependencies": { "@nodelib/fs.stat": "^2.0.2", @@ -10785,7 +10862,6 @@ "version": "5.1.2", "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", - "dev": true, "license": "ISC", "dependencies": { "is-glob": "^4.0.1" @@ -10834,7 +10910,6 @@ "version": "1.20.1", "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.20.1.tgz", "integrity": "sha512-GGToxJ/w1x32s/D2EKND7kTil4n8OVk/9mycTc4VDza13lOvpUZTGX3mFSCtV9ksdGBVzvsyAVLM6mHFThxXxw==", - "dev": true, "license": "ISC", "dependencies": { "reusify": "^1.0.4" @@ -10882,7 +10957,6 @@ "version": "7.1.1", "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==", - "dev": true, "license": "MIT", "dependencies": { "to-regex-range": "^5.0.1" @@ -11073,7 +11147,6 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==", - "dev": true, "license": "ISC" }, "node_modules/fsevents": { @@ -11240,7 +11313,6 @@ "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", "deprecated": "Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me", - "dev": true, "license": "ISC", "dependencies": { "fs.realpath": "^1.0.0", @@ -11274,14 +11346,12 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", - "dev": true, "license": "MIT" }, "node_modules/glob/node_modules/brace-expansion": { "version": "1.1.15", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.15.tgz", "integrity": "sha512-EwOCDEex4quD37XhqM3omwtMoJjr//isUZz1JopUNWms+4Z2ViyM/k1YIRePpoVNnQhENnxtFjLaxNHrT7xIUg==", - "dev": true, "license": "MIT", "dependencies": { "balanced-match": "^1.0.0", @@ -11292,7 +11362,6 @@ "version": "3.1.5", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.5.tgz", "integrity": "sha512-VgjWUsnnT6n+NUk6eZq77zeFdpW2LWDzP6zFGrCbHXiYNul5Dzqk2HHQ5uFH2DNW5Xbp8+jVzaeNt94ssEEl4w==", - "dev": true, "license": "ISC", "dependencies": { "brace-expansion": "^1.1.7" @@ -11593,7 +11662,6 @@ "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", "deprecated": "This module is not supported, and leaks memory. Do not use it. Check out lru-cache if you want a good and tested way to coalesce async requests by a key value, which is much more comprehensive and powerful.", - "dev": true, "license": "ISC", "dependencies": { "once": "^1.3.0", @@ -11681,7 +11749,6 @@ "version": "2.1.1", "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==", - "dev": true, "license": "MIT", "engines": { "node": ">=0.10.0" @@ -11700,7 +11767,6 @@ "version": "4.0.3", "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==", - "dev": true, "license": "MIT", "dependencies": { "is-extglob": "^2.1.1" @@ -11726,7 +11792,6 @@ "version": "7.0.0", "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", - "dev": true, "license": "MIT", "engines": { "node": ">=0.12.0" @@ -11774,7 +11839,6 @@ "version": "0.0.1", "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz", "integrity": "sha512-D2S+3GLxWH+uhrNEcoh/fnmYeP8E8/zHl644d/jdA0g2uyXvy3sb0qxotE+ne0LtccHknQzWwZEzhak7oJ0COQ==", - "dev": true, "license": "MIT" }, "node_modules/isexe": { @@ -12498,7 +12562,6 @@ "version": "1.4.1", "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz", "integrity": "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==", - "dev": true, "license": "MIT", "engines": { "node": ">= 8" @@ -12517,7 +12580,6 @@ "version": "4.0.8", "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.8.tgz", "integrity": "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==", - "dev": true, "license": "MIT", "dependencies": { "braces": "^3.0.3", @@ -12531,7 +12593,6 @@ "version": "2.3.2", "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.2.tgz", "integrity": "sha512-V7+vQEJ06Z+c5tSye8S+nHUfI51xoXIXjHQ99cQtKUkQqqO1kO/KCJUfZXuB47h/YBlDhah2H3hdUGXn8ie0oA==", - "dev": true, "license": "MIT", "engines": { "node": ">=8.6" @@ -12634,7 +12695,6 @@ "version": "1.0.4", "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-1.0.4.tgz", "integrity": "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==", - "dev": true, "license": "MIT", "bin": { "mkdirp": "bin/cmd.js" @@ -12826,7 +12886,6 @@ "version": "0.0.0", "resolved": "https://registry.npmjs.org/noms/-/noms-0.0.0.tgz", "integrity": "sha512-lNDU9VJaOPxUmXcLb+HQFeUgQQPtMI24Gt6hgfuMHRJgMRHMF/qZ4HJD3GDru4sSw9IQl2jPjAYnQrdIeLbwow==", - "dev": true, "license": "ISC", "dependencies": { "inherits": "^2.0.1", @@ -12837,7 +12896,6 @@ "version": "1.0.34", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.0.34.tgz", "integrity": "sha512-ok1qVCJuRkNmvebYikljxJA/UEsKwLl2nI1OmaqAu4/UE+h0wKCHok4XkL/gvi39OacXvw59RJUOFUkDib2rHg==", - "dev": true, "license": "MIT", "dependencies": { "core-util-is": "~1.0.0", @@ -12850,7 +12908,6 @@ "version": "0.10.31", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz", "integrity": "sha512-ev2QzSzWPYmy9GuqfIVildA4OdcGLeFZQrq5ys6RtiuF+RQQiZWr8TZNyAcuVXyQRYfEO+MsoB/1BuQVhOJuoQ==", - "dev": true, "license": "MIT" }, "node_modules/normalize-path": { @@ -13596,6 +13653,12 @@ "node": ">= 0.8" } }, + "node_modules/path-browserify": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/path-browserify/-/path-browserify-1.0.1.tgz", + "integrity": "sha512-b7uo2UCUOYZcnF/3ID0lulOJi/bafxa1xPe7ZPsammBSpjSWQkjNxlt635YGS2MiR9GjvuXCtz2emr3jbsz98g==", + "license": "MIT" + }, "node_modules/path-exists": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", @@ -13610,7 +13673,6 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==", - "dev": true, "license": "MIT", "engines": { "node": ">=0.10.0" @@ -14051,7 +14113,6 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==", - "dev": true, "license": "MIT" }, "node_modules/process-warning": { @@ -14178,7 +14239,6 @@ "version": "1.2.3", "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==", - "dev": true, "funding": [ { "type": "github", @@ -14407,7 +14467,6 @@ "version": "1.1.0", "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.1.0.tgz", "integrity": "sha512-g6QUff04oZpHs0eG5p83rFLhHeV00ug/Yf9nZM6fLeUrPguBTkTQOdpAWWspMh55TZfVQDPaN3NQJfbVRAxdIw==", - "dev": true, "license": "MIT", "engines": { "iojs": ">=1.0.0", @@ -14496,7 +14555,6 @@ "version": "1.2.0", "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz", "integrity": "sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==", - "dev": true, "funding": [ { "type": "github", @@ -15216,7 +15274,6 @@ "version": "2.0.5", "resolved": "https://registry.npmjs.org/through2/-/through2-2.0.5.tgz", "integrity": "sha512-/mrRod8xqpA+IHSLyGCQ2s8SPHiCDEeQJSep1jqLYeEUClOFG2Qsh+4FU6G9VeqpZnGW/Su8LQGc4YKni5rYSQ==", - "dev": true, "license": "MIT", "dependencies": { "readable-stream": "~2.3.6", @@ -15227,14 +15284,12 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==", - "dev": true, "license": "MIT" }, "node_modules/through2/node_modules/readable-stream": { "version": "2.3.8", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", - "dev": true, "license": "MIT", "dependencies": { "core-util-is": "~1.0.0", @@ -15250,14 +15305,12 @@ "version": "5.1.2", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", - "dev": true, "license": "MIT" }, "node_modules/through2/node_modules/string_decoder": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", - "dev": true, "license": "MIT", "dependencies": { "safe-buffer": "~5.1.0" @@ -15317,7 +15370,6 @@ "version": "5.0.1", "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", - "dev": true, "license": "MIT", "dependencies": { "is-number": "^7.0.0" @@ -15417,6 +15469,16 @@ } } }, + "node_modules/ts-morph": { + "version": "18.0.0", + "resolved": "https://registry.npmjs.org/ts-morph/-/ts-morph-18.0.0.tgz", + "integrity": "sha512-Kg5u0mk19PIIe4islUI/HWRvm9bC1lHejK4S0oh1zaZ77TMZAEmQC0sHQYiu2RgCQFZKXz1fMVi/7nOOeirznA==", + "license": "MIT", + "dependencies": { + "@ts-morph/common": "~0.19.0", + "code-block-writer": "^12.0.0" + } + }, "node_modules/tsc-alias": { "version": "1.9.1", "resolved": "https://registry.npmjs.org/tsc-alias/-/tsc-alias-1.9.1.tgz", @@ -15644,7 +15706,6 @@ "version": "4.0.0", "resolved": "https://registry.npmjs.org/untildify/-/untildify-4.0.0.tgz", "integrity": "sha512-KK8xQ1mkzZeg9inewmFVDNkg3l5LUhoq9kN6iWYB/CC9YMG8HA+c1Q8HwDe6dEX7kErrEVNVBO3fWsVq5iDgtw==", - "dev": true, "license": "MIT", "engines": { "node": ">=8" diff --git a/package.json b/package.json index de2aeb0..33030e1 100644 --- a/package.json +++ b/package.json @@ -49,6 +49,7 @@ "@map-colonies/schemas": "^1.26.0", "@map-colonies/tracing": "^1.0.0", "@map-colonies/tracing-utils": "^2.0.0", + "@map-colonies/types": "^1.4.0", "@opentelemetry/api": "^1.9.1", "@turf/turf": "^7.4.0", "axios": "^1.20.0", @@ -56,6 +57,7 @@ "express": "^4.21.2", "express-openapi-validator": "^5.6.2", "http-status-codes": "^2.3.0", + "mime-types": "^2.1.35", "prom-client": "^15.1.3", "reflect-metadata": "^0.2.2", "tsyringe": "^4.8.0", diff --git a/src/common/interfaces.ts b/src/common/interfaces.ts index 6220fde..879c230 100644 --- a/src/common/interfaces.ts +++ b/src/common/interfaces.ts @@ -1,3 +1,13 @@ +import type { components } from '@openapi'; + +export type IngestionPayload = components['schemas']['ingestionPayload']; +export type UpdatePayload = components['schemas']['updatePayload']; +export type StatusPayload = components['schemas']['statusPayload']; +export type JobResponse = components['schemas']['jobResponse']; +export type AckResponse = components['schemas']['ackResponse']; +export type ValidationResultResponse = components['schemas']['validationResultResponse']; +export type JobStatusResponse = components['schemas']['jobStatusResponse']; + export interface IConfig { get: (setting: string) => T; has: (setting: string) => boolean; diff --git a/src/externalServices/catalog/catalogCall.ts b/src/externalServices/catalog/catalogCall.ts index 8e8c585..90696d0 100644 --- a/src/externalServices/catalog/catalogCall.ts +++ b/src/externalServices/catalog/catalogCall.ts @@ -27,7 +27,7 @@ export class CatalogCall { public async findRecords(payload: IFindRecordsPayload): Promise { const logContext = { ...this.logContext, function: this.findRecords.name }; this.logger.debug({ msg: `Searching in catalog ${this.catalog}/metadata/find`, logContext, payload }); - try { + return this.run(logContext, 'Something went wrong in catalog when trying to find records', 'Problem with catalog find', {}, async () => { const response = await axios.post(`${this.catalog}/metadata/find`, payload); if (response.status !== StatusCodes.OK.valueOf()) { this.logger.error({ msg: `Catalog returned unexpected status: ${response.status}`, logContext }); @@ -41,51 +41,71 @@ export class CatalogCall { this.logger.debug({ msg: `Found ${records.length} record(s) in catalog`, logContext }); return records; - } catch (err) { - if (err instanceof AppError) { - throw err; - } - - this.logger.error({ msg: 'Something went wrong in catalog when trying to find records', logContext, err }); - throw new AppError('catalog', StatusCodes.INTERNAL_SERVER_ERROR, 'Problem with catalog find', true); - } + }); } public async getRecord(identifier: string): Promise { const logContext = { ...this.logContext, function: this.getRecord.name }; this.logger.debug({ msg: `Getting record ${identifier} from catalog`, logContext }); - try { - const response = await axios.get(`${this.catalog}/metadata/${identifier}`, { - validateStatus: (status) => status === StatusCodes.OK.valueOf() || status === StatusCodes.NOT_FOUND.valueOf(), - }); - return response.status === StatusCodes.NOT_FOUND.valueOf() ? undefined : response.data; - } catch (err) { - this.logger.error({ msg: 'Something went wrong in catalog when getting a record', logContext, identifier, err }); - throw new AppError('catalog', StatusCodes.INTERNAL_SERVER_ERROR, 'Problem with catalog during record lookup', true); - } + return this.run( + logContext, + 'Something went wrong in catalog when getting a record', + 'Problem with catalog during record lookup', + { identifier }, + async () => { + const response = await axios.get(`${this.catalog}/metadata/${identifier}`, { + validateStatus: (status) => status === StatusCodes.OK.valueOf() || status === StatusCodes.NOT_FOUND.valueOf(), + }); + return response.status === StatusCodes.NOT_FOUND.valueOf() ? undefined : response.data; + } + ); } public async patchMetadata(identifier: string, payload: CatalogUpdatePayload): Promise { const logContext = { ...this.logContext, function: this.patchMetadata.name }; this.logger.debug({ msg: `Updating metadata for record ${identifier} in catalog`, logContext }); - try { - const response = await axios.patch(`${this.catalog}/metadata/${identifier}`, payload); - return response.data; - } catch (err) { - this.logger.error({ msg: 'Something went wrong in catalog when updating metadata', logContext, identifier, err }); - throw new AppError('catalog', StatusCodes.INTERNAL_SERVER_ERROR, 'Problem with catalog during metadata update', true); - } + return this.run( + logContext, + 'Something went wrong in catalog when updating metadata', + 'Problem with catalog during metadata update', + { identifier }, + async () => { + const response = await axios.patch(`${this.catalog}/metadata/${identifier}`, payload); + return response.data; + } + ); } public async changeStatus(identifier: string, payload: CatalogStatusPayload): Promise { const logContext = { ...this.logContext, function: this.changeStatus.name }; this.logger.debug({ msg: `Changing status for record ${identifier} in catalog`, logContext }); + return this.run( + logContext, + 'Something went wrong in catalog when changing status', + 'Problem with catalog during status change', + { identifier }, + async () => { + const response = await axios.patch(`${this.catalog}/metadata/status/${identifier}`, payload); + return response.data; + } + ); + } + + private async run( + logContext: LogContext, + errorLog: string, + failMessage: string, + details: Record, + fn: () => Promise + ): Promise { try { - const response = await axios.patch(`${this.catalog}/metadata/status/${identifier}`, payload); - return response.data; + return await fn(); } catch (err) { - this.logger.error({ msg: 'Something went wrong in catalog when changing status', logContext, identifier, err }); - throw new AppError('catalog', StatusCodes.INTERNAL_SERVER_ERROR, 'Problem with catalog during status change', true); + if (err instanceof AppError) { + throw err; + } + this.logger.error({ msg: errorLog, logContext, ...details, err }); + throw new AppError('catalog', StatusCodes.INTERNAL_SERVER_ERROR, failMessage, true); } } } diff --git a/src/externalServices/jobnik/jobnikClient.ts b/src/externalServices/jobnik/jobnikClient.ts index 52d11bb..52c1d69 100644 --- a/src/externalServices/jobnik/jobnikClient.ts +++ b/src/externalServices/jobnik/jobnikClient.ts @@ -8,8 +8,7 @@ import { IN_FLIGHT_JOB_STATUSES, SERVICES, STAGE_TYPES } from '@common/constants import { is3tz } from '@common/util'; import { AppError } from '@common/appError'; import type { ConfigType, JobManagerConfig } from '@common/config'; -import type { LogContext } from '@common/interfaces'; -import type { IngestionPayload, JobResponse } from '../../records/models/recordManager'; +import type { IngestionPayload, JobResponse, JobStatusResponse, LogContext } from '@common/interfaces'; import type { Record3D } from '../catalog/interfaces'; interface StageDescriptor { @@ -19,11 +18,6 @@ interface StageDescriptor { only3tz?: boolean; } -export interface JobStatusResponse { - status: string; - percentage?: number; -} - @singleton() export class JobnikClient { private readonly logContext: LogContext; diff --git a/src/job/models/jobManager.ts b/src/job/models/jobManager.ts index f2767ce..21949b3 100644 --- a/src/job/models/jobManager.ts +++ b/src/job/models/jobManager.ts @@ -1,12 +1,9 @@ import type { Logger } from '@map-colonies/js-logger'; import { inject, injectable } from 'tsyringe'; -import type { components } from '@openapi'; import { SERVICES } from '@common/constants'; -import type { LogContext } from '@common/interfaces'; +import type { JobStatusResponse, LogContext } from '@common/interfaces'; import { JobnikClient } from '../../externalServices/jobnik/jobnikClient'; -export type JobStatusResponse = components['schemas']['jobStatusResponse']; - @injectable() export class JobManager { private readonly logContext: LogContext; diff --git a/src/records/models/recordManager.ts b/src/records/models/recordManager.ts index 8ecb245..8fb11c8 100644 --- a/src/records/models/recordManager.ts +++ b/src/records/models/recordManager.ts @@ -4,31 +4,32 @@ import { inject, injectable } from 'tsyringe'; import { trace } from '@opentelemetry/api'; import { StatusCodes } from 'http-status-codes'; import type { Polygon } from 'geojson'; -import type { components } from '@openapi'; +import { RecordStatus } from '@map-colonies/types'; import { SERVICES } from '@common/constants'; -import type { LogContext } from '@common/interfaces'; +import type { + AckResponse, + IngestionPayload, + JobResponse, + LogContext, + StatusPayload, + UpdatePayload, + ValidationResultResponse, +} from '@common/interfaces'; import { AppError } from '@common/appError'; import { convertPolygonTo2DPolygon } from '@common/util'; -import { ValidationManager } from '../../validator/validationManager'; +import { IngestionValidator } from '../../validator/ingestionValidator'; +import { MetadataValidator } from '../../validator/metadataValidator'; import { JobnikClient } from '../../externalServices/jobnik/jobnikClient'; import { CatalogCall } from '../../externalServices/catalog/catalogCall'; -const RECORD_STATUS_BEING_DELETED = 'BEING_DELETED'; - -export type IngestionPayload = components['schemas']['ingestionPayload']; -export type UpdatePayload = components['schemas']['updatePayload']; -export type StatusPayload = components['schemas']['statusPayload']; -export type JobResponse = components['schemas']['jobResponse']; -export type AckResponse = components['schemas']['ackResponse']; -export type ValidationResultResponse = components['schemas']['validationResultResponse']; - @injectable() export class RecordManager { private readonly logContext: LogContext; public constructor( @inject(SERVICES.LOGGER) private readonly logger: Logger, - @inject(ValidationManager) private readonly validator: ValidationManager, + @inject(IngestionValidator) private readonly ingestionValidator: IngestionValidator, + @inject(MetadataValidator) private readonly metadataValidator: MetadataValidator, @inject(JobnikClient) private readonly jobnik: JobnikClient, @inject(CatalogCall) private readonly catalog: CatalogCall ) { @@ -45,7 +46,7 @@ export class RecordManager { trace.getActiveSpan()?.setAttribute('catalogId', modelId); try { - await this.validator.validateIngestion(payload); + await this.ingestionValidator.validateIngestion(payload); const metadata = payload.metadata as Record; if (metadata.footprint !== undefined) { @@ -66,7 +67,7 @@ export class RecordManager { const logContext = { ...this.logContext, function: this.validateIngestion.name }; this.logger.info({ msg: 'validating ingestion request', logContext, modelPath: payload.modelPath }); try { - await this.validator.validateIngestion(payload); + await this.ingestionValidator.validateIngestion(payload); return { isValid: true }; } catch (err) { if (err instanceof AppError) { @@ -80,7 +81,7 @@ export class RecordManager { const logContext = { ...this.logContext, function: this.canDelete.name }; this.logger.info({ msg: 'validating record deletability', logContext, recordId: id }); try { - await this.validator.validateDelete(id); + await this.metadataValidator.validateDelete(id); return { isValid: true }; } catch (err) { if (err instanceof AppError) { @@ -93,17 +94,17 @@ export class RecordManager { public async deleteRecord(id: string): Promise { const logContext = { ...this.logContext, function: this.deleteRecord.name }; this.logger.info({ msg: 'creating delete job', logContext, recordId: id }); - const record = await this.validator.validateDelete(id); + const record = await this.metadataValidator.validateDelete(id); const job = await this.jobnik.createDeleteJob(record); - await this.catalog.changeStatus(id, { productStatus: RECORD_STATUS_BEING_DELETED }); + await this.catalog.changeStatus(id, { productStatus: RecordStatus.BEING_DELETED }); return job; } public async updateMetadata(id: string, update: UpdatePayload): Promise { const logContext = { ...this.logContext, function: this.updateMetadata.name }; this.logger.info({ msg: 'updating record metadata', logContext, recordId: id }); - const record = await this.validator.validateUpdate(id, update); - await this.validator.ensureRecordAbsentFromExtractable(record); + const record = await this.metadataValidator.validateUpdate(id, update); + await this.metadataValidator.ensureRecordAbsentFromExtractable(record); const payload: Record = { ...update }; if (payload.footprint !== undefined) { @@ -117,8 +118,8 @@ export class RecordManager { public async updateStatus(id: string, payload: StatusPayload): Promise { const logContext = { ...this.logContext, function: this.updateStatus.name }; this.logger.info({ msg: 'updating record status', logContext, recordId: id, status: payload.status }); - const record = await this.validator.validateStatusChange(id); - await this.validator.ensureRecordAbsentFromExtractable(record); + const record = await this.metadataValidator.validateStatusChange(id); + await this.metadataValidator.ensureRecordAbsentFromExtractable(record); await this.catalog.changeStatus(id, { productStatus: payload.status }); diff --git a/src/validator/classification.ts b/src/validator/classification.ts new file mode 100644 index 0000000..91db638 --- /dev/null +++ b/src/validator/classification.ts @@ -0,0 +1,24 @@ +import type { Logger } from '@map-colonies/js-logger'; +import { StatusCodes } from 'http-status-codes'; +import type { LogContext } from '@common/interfaces'; +import { AppError } from '@common/appError'; +import type { LookupTablesCall } from '../externalServices/lookupTables/lookupTablesCall'; + +export const validateClassification = async ( + lookupTables: LookupTablesCall, + logger: Logger, + logContext: LogContext, + classification: string +): Promise => { + const classifications = await lookupTables.getClassifications(); + logger.debug({ msg: 'classification validation', logContext, classifications }); + + if (!classifications.includes(classification)) { + throw new AppError( + 'badRequest', + StatusCodes.BAD_REQUEST, + `classification is not a valid value. Optional values: ${classifications.join()}`, + true + ); + } +}; diff --git a/src/validator/errors.ts b/src/validator/errors.ts new file mode 100644 index 0000000..5817937 --- /dev/null +++ b/src/validator/errors.ts @@ -0,0 +1,18 @@ +export const ERROR_METADATA_DATE = 'imagingTimeBeginUTC must not be later than imagingTimeEndUTC'; +export const ERROR_METADATA_MISSING_DATE = 'imagingTimeBeginUTC and imagingTimeEndUTC are required'; +export const ERROR_METADATA_INVALID_DATE = 'imagingTimeBeginUTC and imagingTimeEndUTC must be valid dates'; +export const ERROR_METADATA_FOOTPRINT = 'Invalid footprint! Must be a GeoJSON Polygon or MultiPolygon with all-2D or all-3D coordinates'; +export const ERROR_METADATA_PRODUCT_NAME_UNIQUE = 'product name is not unique!'; +export const ERROR_PRODUCT_NAME_IN_FLIGHT = 'product name already has an in-flight ingestion job'; +export const ERROR_DELETE_RECORD_NOT_FOUND = "recordId doesn't match exactly one existing record"; +export const ERROR_DELETE_PRODUCT_TYPE = 'Cannot delete a record whose productType is "QuantizedMeshDTMBest"'; +export const ERROR_DELETE_STATUS = 'Cannot delete a record whose productStatus is not "UNPUBLISHED"'; +export const ERROR_FILE_NOT_FOUND = 'The model files do not exist in the agreed storage'; +export const ERROR_TILESET_INVALID = 'The tileset file is not a valid 3DTiles tileset'; +export const ERROR_FOOTPRINT_FAR_FROM_MODEL = "Wrong footprint! footprint's coordinates is not even close to the model!"; +export const ERROR_INTERSECTION_FAILED = 'An error caused during the validation of the intersection'; +export const ERROR_RECORD_NOT_FOUND = "record with the given identifier doesn't exist"; +export const ERROR_RECORD_BEING_DELETED = 'cannot change a record that is being deleted'; +export const ERROR_EXTRACTABLE_CONFLICT = 'the record exists in the extractable-management service and cannot be changed here'; +export const ERROR_MODEL_PATH_INVALID = 'Unknown model path! The model is not under the agreed base path'; +export const ERROR_PRODUCT_ID_EXISTS = 'a record with this productId already exists in the catalog'; diff --git a/src/validator/ingestionValidator.ts b/src/validator/ingestionValidator.ts new file mode 100644 index 0000000..7c4b0c7 --- /dev/null +++ b/src/validator/ingestionValidator.ts @@ -0,0 +1,138 @@ +import type { Logger } from '@map-colonies/js-logger'; +import { StatusCodes } from 'http-status-codes'; +import { inject, injectable } from 'tsyringe'; +import { new3DLayerMetadataSchema, geometrySchema } from '@map-colonies/3d-shared'; +import { SERVICES } from '@common/constants'; +import type { IngestionPayload, LogContext } from '@common/interfaces'; +import { AppError } from '@common/appError'; +import { buildModelFilePath } from '@common/util'; +import type { ConfigType } from '@common/config'; +import { LookupTablesCall } from '../externalServices/lookupTables/lookupTablesCall'; +import { CatalogCall } from '../externalServices/catalog/catalogCall'; +import { JobnikClient } from '../externalServices/jobnik/jobnikClient'; +import type { Provider } from '../providers/interfaces'; +import { TilesetValidator } from './tilesetValidator'; +import { validateClassification } from './classification'; +import { + ERROR_FILE_NOT_FOUND, + ERROR_METADATA_DATE, + ERROR_METADATA_FOOTPRINT, + ERROR_METADATA_INVALID_DATE, + ERROR_METADATA_MISSING_DATE, + ERROR_METADATA_PRODUCT_NAME_UNIQUE, + ERROR_MODEL_PATH_INVALID, + ERROR_PRODUCT_ID_EXISTS, + ERROR_PRODUCT_NAME_IN_FLIGHT, +} from './errors'; + +@injectable() +export class IngestionValidator { + private readonly logContext: LogContext; + private readonly basePath: string; + + public constructor( + @inject(SERVICES.CONFIG) private readonly config: ConfigType, + @inject(SERVICES.LOGGER) private readonly logger: Logger, + @inject(LookupTablesCall) private readonly lookupTables: LookupTablesCall, + @inject(CatalogCall) private readonly catalog: CatalogCall, + @inject(JobnikClient) private readonly jobnik: JobnikClient, + @inject(SERVICES.PROVIDER) private readonly provider: Provider, + @inject(TilesetValidator) private readonly tileset: TilesetValidator + ) { + this.basePath = this.config.get('validation.basePath'); + this.logContext = { + fileName: __filename, + class: IngestionValidator.name, + }; + } + + public async validateIngestion(payload: IngestionPayload): Promise { + const logContext = { ...this.logContext, function: this.validateIngestion.name }; + this.logger.info({ msg: 'ingestion validation start', logContext }); + + const metadata = payload.metadata; + const parsed = new3DLayerMetadataSchema.safeParse(metadata); + if (!parsed.success) { + const message = parsed.error.issues.map((issue) => `${issue.path.join('.')}: ${issue.message}`).join('; '); + throw new AppError('badRequest', StatusCodes.BAD_REQUEST, message, true); + } + + const footprintResult = geometrySchema.safeParse(metadata.footprint); + if (!footprintResult.success) { + throw new AppError('badRequest', StatusCodes.BAD_REQUEST, ERROR_METADATA_FOOTPRINT, true); + } + + this.validateDates(metadata.imagingTimeBeginUTC, metadata.imagingTimeEndUTC); + this.validateModelPath(payload.modelPath); + + await validateClassification(this.lookupTables, this.logger, logContext, parsed.data.classification); + await this.validateProductNameUnique(parsed.data.productName); + await this.validateProductNameNotInFlight(parsed.data.productName); + await this.validateProductIdUnique(parsed.data.productId); + await this.validateFileExists(payload.modelPath, payload.tilesetFilename); + await this.tileset.validateTileset(payload, footprintResult.data); + } + + private validateModelPath(modelPath: string): void { + if (!modelPath.startsWith(this.basePath)) { + throw new AppError('badRequest', StatusCodes.BAD_REQUEST, `${ERROR_MODEL_PATH_INVALID} (basePath: ${this.basePath})`, true); + } + } + + private async validateProductIdUnique(productId: string): Promise { + const logContext = { ...this.logContext, function: this.validateProductIdUnique.name }; + const records = await this.catalog.findRecords({ productId }); + this.logger.debug({ msg: 'product id uniqueness validation', logContext, productId, matches: records.length }); + + if (records.length > 0) { + throw new AppError('conflict', StatusCodes.CONFLICT, ERROR_PRODUCT_ID_EXISTS, true); + } + } + + private async validateFileExists(modelPath: string, tilesetFilename: string): Promise { + const logContext = { ...this.logContext, function: this.validateFileExists.name }; + const path = buildModelFilePath(modelPath, tilesetFilename); + const exists = await this.provider.fileExists(path); + this.logger.debug({ msg: 'file existence validation', logContext, path, exists }); + + if (!exists) { + throw new AppError('badRequest', StatusCodes.BAD_REQUEST, ERROR_FILE_NOT_FOUND, true); + } + } + + private async validateProductNameUnique(productName: string): Promise { + const logContext = { ...this.logContext, function: this.validateProductNameUnique.name }; + const records = await this.catalog.findRecords({ productName }); + this.logger.debug({ msg: 'product name uniqueness validation', logContext, productName, matches: records.length }); + + if (records.length > 0) { + throw new AppError('badRequest', StatusCodes.BAD_REQUEST, ERROR_METADATA_PRODUCT_NAME_UNIQUE, true); + } + } + + private async validateProductNameNotInFlight(productName: string): Promise { + const logContext = { ...this.logContext, function: this.validateProductNameNotInFlight.name }; + const inFlight = await this.jobnik.hasInFlightIngestionJob(productName); + this.logger.debug({ msg: 'in-flight duplicate validation', logContext, productName, inFlight }); + + if (inFlight) { + throw new AppError('conflict', StatusCodes.CONFLICT, ERROR_PRODUCT_NAME_IN_FLIGHT, true); + } + } + + private validateDates(start: unknown, end: unknown): void { + if (start === undefined || end === undefined) { + throw new AppError('badRequest', StatusCodes.BAD_REQUEST, ERROR_METADATA_MISSING_DATE, true); + } + + const startTime = new Date(start as string).getTime(); + const endTime = new Date(end as string).getTime(); + if (Number.isNaN(startTime) || Number.isNaN(endTime)) { + throw new AppError('badRequest', StatusCodes.BAD_REQUEST, ERROR_METADATA_INVALID_DATE, true); + } + + if (startTime > endTime) { + throw new AppError('badRequest', StatusCodes.BAD_REQUEST, ERROR_METADATA_DATE, true); + } + } +} diff --git a/src/validator/metadataValidator.ts b/src/validator/metadataValidator.ts new file mode 100644 index 0000000..35cb207 --- /dev/null +++ b/src/validator/metadataValidator.ts @@ -0,0 +1,127 @@ +import type { Logger } from '@map-colonies/js-logger'; +import { StatusCodes } from 'http-status-codes'; +import { inject, injectable } from 'tsyringe'; +import { new3DLayerMetadataSchema, geometrySchema } from '@map-colonies/3d-shared'; +import { ProductType, RecordStatus } from '@map-colonies/types'; +import { SERVICES } from '@common/constants'; +import type { LogContext, UpdatePayload } from '@common/interfaces'; +import { AppError } from '@common/appError'; +import { LookupTablesCall } from '../externalServices/lookupTables/lookupTablesCall'; +import { CatalogCall } from '../externalServices/catalog/catalogCall'; +import { ExtractableCall } from '../externalServices/extractableManagement/extractableCall'; +import type { Record3D } from '../externalServices/catalog/interfaces'; +import { validateClassification } from './classification'; +import { + ERROR_DELETE_PRODUCT_TYPE, + ERROR_DELETE_RECORD_NOT_FOUND, + ERROR_DELETE_STATUS, + ERROR_EXTRACTABLE_CONFLICT, + ERROR_METADATA_FOOTPRINT, + ERROR_METADATA_PRODUCT_NAME_UNIQUE, + ERROR_RECORD_BEING_DELETED, + ERROR_RECORD_NOT_FOUND, +} from './errors'; + +const BLOCKED_DELETE_PRODUCT_TYPE = ProductType.QUANTIZED_MESH_DTM_BEST; + +@injectable() +export class MetadataValidator { + private readonly logContext: LogContext; + + public constructor( + @inject(SERVICES.LOGGER) private readonly logger: Logger, + @inject(LookupTablesCall) private readonly lookupTables: LookupTablesCall, + @inject(CatalogCall) private readonly catalog: CatalogCall, + @inject(ExtractableCall) private readonly extractable: ExtractableCall + ) { + this.logContext = { + fileName: __filename, + class: MetadataValidator.name, + }; + } + + public async validateDelete(recordId: string): Promise { + const logContext = { ...this.logContext, function: this.validateDelete.name }; + const records = await this.catalog.findRecords({ id: recordId }); + this.logger.debug({ msg: 'delete validation', logContext, recordId, matches: records.length }); + + const [record] = records; + if (records.length !== 1 || record === undefined) { + throw new AppError('badRequest', StatusCodes.BAD_REQUEST, ERROR_DELETE_RECORD_NOT_FOUND, true); + } + + if (record.productType === BLOCKED_DELETE_PRODUCT_TYPE) { + throw new AppError('badRequest', StatusCodes.BAD_REQUEST, ERROR_DELETE_PRODUCT_TYPE, true); + } + + if (record.productStatus !== RecordStatus.UNPUBLISHED) { + throw new AppError('badRequest', StatusCodes.BAD_REQUEST, ERROR_DELETE_STATUS, true); + } + + return record; + } + + public async validateUpdate(identifier: string, payload: UpdatePayload): Promise { + const logContext = { ...this.logContext, function: this.validateUpdate.name }; + const record = await this.catalog.getRecord(identifier); + if (record === undefined) { + throw new AppError('notFound', StatusCodes.NOT_FOUND, ERROR_RECORD_NOT_FOUND, true); + } + if (record.productStatus === RecordStatus.BEING_DELETED) { + throw new AppError('badRequest', StatusCodes.BAD_REQUEST, ERROR_RECORD_BEING_DELETED, true); + } + + const parsed = new3DLayerMetadataSchema.partial().safeParse(payload); + if (!parsed.success) { + const message = parsed.error.issues.map((issue) => `${issue.path.join('.')}: ${issue.message}`).join('; '); + throw new AppError('badRequest', StatusCodes.BAD_REQUEST, message, true); + } + + const footprint = (payload as { footprint?: unknown }).footprint; + if (footprint !== undefined && !geometrySchema.safeParse(footprint).success) { + throw new AppError('badRequest', StatusCodes.BAD_REQUEST, ERROR_METADATA_FOOTPRINT, true); + } + + const data = parsed.data; + if (data.classification !== undefined) { + await validateClassification(this.lookupTables, this.logger, logContext, data.classification); + } + if (data.productName !== undefined) { + await this.validateProductNameUniqueExcept(data.productName, identifier); + } + + return record; + } + + public async validateStatusChange(identifier: string): Promise { + const record = await this.catalog.getRecord(identifier); + if (record === undefined) { + throw new AppError('notFound', StatusCodes.NOT_FOUND, ERROR_RECORD_NOT_FOUND, true); + } + if (record.productStatus === RecordStatus.BEING_DELETED) { + throw new AppError('badRequest', StatusCodes.BAD_REQUEST, ERROR_RECORD_BEING_DELETED, true); + } + + return record; + } + + public async ensureRecordAbsentFromExtractable(record: Record3D): Promise { + const logContext = { ...this.logContext, function: this.ensureRecordAbsentFromExtractable.name }; + if (record.productName === undefined) { + return; + } + + const exists = await this.extractable.isExtractableRecordExists(record.productName); + this.logger.debug({ msg: 'extractable existence validation', logContext, productName: record.productName, exists }); + if (exists) { + throw new AppError('conflict', StatusCodes.CONFLICT, ERROR_EXTRACTABLE_CONFLICT, true); + } + } + + private async validateProductNameUniqueExcept(productName: string, identifier: string): Promise { + const records = await this.catalog.findRecords({ productName }); + if (records.some((existing) => existing.id !== identifier)) { + throw new AppError('badRequest', StatusCodes.BAD_REQUEST, ERROR_METADATA_PRODUCT_NAME_UNIQUE, true); + } + } +} diff --git a/src/validator/tilesetValidator.ts b/src/validator/tilesetValidator.ts new file mode 100644 index 0000000..fed1fad --- /dev/null +++ b/src/validator/tilesetValidator.ts @@ -0,0 +1,98 @@ +import type { Logger } from '@map-colonies/js-logger'; +import { StatusCodes } from 'http-status-codes'; +import { inject, injectable } from 'tsyringe'; +import { calculatePolygonFromTileset, type TileSetJson } from '@map-colonies/3d-shared'; +import { area, feature, featureCollection, intersect, union } from '@turf/turf'; +import type { Feature, MultiPolygon, Polygon } from 'geojson'; +import { SERVICES } from '@common/constants'; +import type { IngestionPayload, LogContext, ValidationResponse } from '@common/interfaces'; +import { AppError } from '@common/appError'; +import type { ConfigType } from '@common/config'; +import { TilesetReader } from '../tileset/tilesetReader'; +import { ERROR_FOOTPRINT_FAR_FROM_MODEL, ERROR_INTERSECTION_FAILED, ERROR_TILESET_INVALID } from './errors'; + +const FULL_COVERAGE_PERCENT = 100; + +@injectable() +export class TilesetValidator { + private readonly logContext: LogContext; + private readonly percentageLimit: number; + + public constructor( + @inject(SERVICES.CONFIG) private readonly config: ConfigType, + @inject(SERVICES.LOGGER) private readonly logger: Logger, + @inject(TilesetReader) private readonly tilesetReader: TilesetReader + ) { + this.percentageLimit = this.config.get('validation.percentageLimit'); + this.logContext = { + fileName: __filename, + class: TilesetValidator.name, + }; + } + + public async validateTileset(payload: IngestionPayload, footprint: Polygon | MultiPolygon): Promise { + const content = await this.tilesetReader.readTilesetJson(payload.modelPath, payload.tilesetFilename); + + const { modelPolygon, response: polygonResponse } = this.getTilesetModelPolygon(content); + if (!polygonResponse.isValid || modelPolygon === undefined) { + this.rejectValidation(polygonResponse); + } + + const intersectionResponse = this.isFootprintAndModelIntersects(footprint, modelPolygon); + if (!intersectionResponse.isValid) { + this.rejectValidation(intersectionResponse); + } + } + + private getTilesetModelPolygon(content: string): { modelPolygon?: Polygon; response: ValidationResponse } { + const logContext = { ...this.logContext, function: this.getTilesetModelPolygon.name }; + try { + const tilesetJson = JSON.parse(content) as TileSetJson; + const modelPolygon = calculatePolygonFromTileset(tilesetJson); + this.logger.debug({ msg: 'extracted tileset model polygon', logContext, modelPolygon }); + + return { modelPolygon, response: { isValid: true } }; + } catch (err) { + const message = err instanceof Error ? err.message : ERROR_TILESET_INVALID; + this.logger.error({ msg: 'tileset polygon extraction failed', logContext, err }); + return { response: { isValid: false, message } }; + } + } + + private isFootprintAndModelIntersects(footprint: Polygon | MultiPolygon, modelPolygon: Polygon): ValidationResponse { + const logContext = { ...this.logContext, function: this.isFootprintAndModelIntersects.name }; + try { + const footprintFeature = feature(footprint); + const modelFeature: Feature = feature(modelPolygon); + + const intersection = intersect(featureCollection([footprintFeature, modelFeature])); + if (intersection === null) { + return { isValid: false, message: ERROR_FOOTPRINT_FAR_FROM_MODEL }; + } + + const combined = union(featureCollection([footprintFeature, modelFeature])); + const combinedArea = combined === null ? 0 : area(combined); + const coverage = combinedArea === 0 ? 0 : (FULL_COVERAGE_PERCENT * area(footprintFeature)) / combinedArea; + this.logger.debug({ msg: 'calculated footprint coverage of the model', logContext, coverage, percentageLimit: this.percentageLimit }); + + if (coverage < this.percentageLimit) { + return { + isValid: false, + message: `The footprint intersectection with the model doesn't reach minimum required threshhold, the coverage is: ${coverage}% when the minimum coverage is ${this.percentageLimit}%`, + }; + } + return { isValid: true }; + } catch (err) { + this.logger.error({ msg: ERROR_INTERSECTION_FAILED, logContext, err }); + return { isValid: false, message: ERROR_INTERSECTION_FAILED }; + } + } + + private rejectValidation(response: ValidationResponse): never { + const logContext = { ...this.logContext, function: this.rejectValidation.name }; + const message = response.message ?? ERROR_TILESET_INVALID; + this.logger.warn({ msg: 'ingestion validation failed', logContext, isValid: response.isValid, message }); + + throw new AppError('badRequest', StatusCodes.BAD_REQUEST, message, true); + } +} diff --git a/src/validator/validationManager.ts b/src/validator/validationManager.ts deleted file mode 100644 index 94e4445..0000000 --- a/src/validator/validationManager.ts +++ /dev/null @@ -1,327 +0,0 @@ -import type { Logger } from '@map-colonies/js-logger'; -import { StatusCodes } from 'http-status-codes'; -import { inject, injectable } from 'tsyringe'; -import { new3DLayerMetadataSchema, geometrySchema, calculatePolygonFromTileset, type TileSetJson } from '@map-colonies/3d-shared'; -import { area, feature, featureCollection, intersect, union } from '@turf/turf'; -import type { Feature, MultiPolygon, Polygon } from 'geojson'; -import type { components } from '@openapi'; -import { SERVICES } from '@common/constants'; -import type { LogContext, ValidationResponse } from '@common/interfaces'; -import { AppError } from '@common/appError'; -import { buildModelFilePath } from '@common/util'; -import type { ConfigType } from '@common/config'; -import { LookupTablesCall } from '../externalServices/lookupTables/lookupTablesCall'; -import { CatalogCall } from '../externalServices/catalog/catalogCall'; -import { JobnikClient } from '../externalServices/jobnik/jobnikClient'; -import { ExtractableCall } from '../externalServices/extractableManagement/extractableCall'; -import { TilesetReader } from '../tileset/tilesetReader'; -import type { Record3D } from '../externalServices/catalog/interfaces'; -import type { Provider } from '../providers/interfaces'; -import type { IngestionPayload } from '../records/models/recordManager'; - -type UpdatePayload = components['schemas']['updatePayload']; - -const FULL_COVERAGE_PERCENT = 100; - -const BLOCKED_DELETE_PRODUCT_TYPE = 'QuantizedMeshDTMBest'; -const RECORD_STATUS_UNPUBLISHED = 'UNPUBLISHED'; -const RECORD_STATUS_BEING_DELETED = 'BEING_DELETED'; - -export const ERROR_METADATA_DATE = 'imagingTimeBeginUTC must not be later than imagingTimeEndUTC'; -export const ERROR_METADATA_MISSING_DATE = 'imagingTimeBeginUTC and imagingTimeEndUTC are required'; -export const ERROR_METADATA_INVALID_DATE = 'imagingTimeBeginUTC and imagingTimeEndUTC must be valid dates'; -export const ERROR_METADATA_FOOTPRINT = 'Invalid footprint! Must be a GeoJSON Polygon or MultiPolygon with all-2D or all-3D coordinates'; -export const ERROR_METADATA_PRODUCT_NAME_UNIQUE = 'product name is not unique!'; -export const ERROR_PRODUCT_NAME_IN_FLIGHT = 'product name already has an in-flight ingestion job'; -export const ERROR_DELETE_RECORD_NOT_FOUND = "recordId doesn't match exactly one existing record"; -export const ERROR_DELETE_PRODUCT_TYPE = 'Cannot delete a record whose productType is "QuantizedMeshDTMBest"'; -export const ERROR_DELETE_STATUS = 'Cannot delete a record whose productStatus is not "UNPUBLISHED"'; -export const ERROR_FILE_NOT_FOUND = 'The model files do not exist in the agreed storage'; -export const ERROR_TILESET_INVALID = 'The tileset file is not a valid 3DTiles tileset'; -export const ERROR_FOOTPRINT_FAR_FROM_MODEL = "Wrong footprint! footprint's coordinates is not even close to the model!"; -export const ERROR_INTERSECTION_FAILED = 'An error caused during the validation of the intersection'; -export const ERROR_RECORD_NOT_FOUND = "record with the given identifier doesn't exist"; -export const ERROR_RECORD_BEING_DELETED = 'cannot change a record that is being deleted'; -export const ERROR_EXTRACTABLE_CONFLICT = 'the record exists in the extractable-management service and cannot be changed here'; -export const ERROR_MODEL_PATH_INVALID = 'Unknown model path! The model is not under the agreed base path'; -export const ERROR_PRODUCT_ID_EXISTS = 'a record with this productId already exists in the catalog'; - -@injectable() -export class ValidationManager { - private readonly logContext: LogContext; - private readonly percentageLimit: number; - private readonly basePath: string; - - public constructor( - @inject(SERVICES.CONFIG) private readonly config: ConfigType, - @inject(SERVICES.LOGGER) private readonly logger: Logger, - @inject(LookupTablesCall) private readonly lookupTables: LookupTablesCall, - @inject(CatalogCall) private readonly catalog: CatalogCall, - @inject(JobnikClient) private readonly jobnik: JobnikClient, - @inject(TilesetReader) private readonly tilesetReader: TilesetReader, - @inject(ExtractableCall) private readonly extractable: ExtractableCall, - @inject(SERVICES.PROVIDER) private readonly provider: Provider - ) { - this.percentageLimit = this.config.get('validation.percentageLimit'); - this.basePath = this.config.get('validation.basePath'); - this.logContext = { - fileName: __filename, - class: ValidationManager.name, - }; - } - - public async validateIngestion(payload: IngestionPayload): Promise { - const logContext = { ...this.logContext, function: this.validateIngestion.name }; - this.logger.info({ msg: 'ingestion validation start', logContext }); - - const metadata = payload.metadata; - const parsed = new3DLayerMetadataSchema.safeParse(metadata); - if (!parsed.success) { - const message = parsed.error.issues.map((issue) => `${issue.path.join('.')}: ${issue.message}`).join('; '); - throw new AppError('badRequest', StatusCodes.BAD_REQUEST, message, true); - } - - const footprintResult = geometrySchema.safeParse(metadata.footprint); - if (!footprintResult.success) { - throw new AppError('badRequest', StatusCodes.BAD_REQUEST, ERROR_METADATA_FOOTPRINT, true); - } - - this.validateDates(metadata.imagingTimeBeginUTC, metadata.imagingTimeEndUTC); - this.validateModelPath(payload.modelPath); - - await this.validateClassification(parsed.data.classification); - await this.validateProductNameUnique(parsed.data.productName); - await this.validateProductNameNotInFlight(parsed.data.productName); - await this.validateProductIdUnique(parsed.data.productId); - await this.validateFileExists(payload.modelPath, payload.tilesetFilename); - await this.validateTileset(payload, footprintResult.data); - } - - public async validateDelete(recordId: string): Promise { - const logContext = { ...this.logContext, function: this.validateDelete.name }; - const records = await this.catalog.findRecords({ id: recordId }); - this.logger.debug({ msg: 'delete validation', logContext, recordId, matches: records.length }); - - const [record] = records; - if (records.length !== 1 || record === undefined) { - throw new AppError('badRequest', StatusCodes.BAD_REQUEST, ERROR_DELETE_RECORD_NOT_FOUND, true); - } - - if (record.productType === BLOCKED_DELETE_PRODUCT_TYPE) { - throw new AppError('badRequest', StatusCodes.BAD_REQUEST, ERROR_DELETE_PRODUCT_TYPE, true); - } - - if (record.productStatus !== RECORD_STATUS_UNPUBLISHED) { - throw new AppError('badRequest', StatusCodes.BAD_REQUEST, ERROR_DELETE_STATUS, true); - } - - return record; - } - - public async validateUpdate(identifier: string, payload: UpdatePayload): Promise { - const record = await this.catalog.getRecord(identifier); - if (record === undefined) { - throw new AppError('notFound', StatusCodes.NOT_FOUND, ERROR_RECORD_NOT_FOUND, true); - } - if (record.productStatus === RECORD_STATUS_BEING_DELETED) { - throw new AppError('badRequest', StatusCodes.BAD_REQUEST, ERROR_RECORD_BEING_DELETED, true); - } - - const parsed = new3DLayerMetadataSchema.partial().safeParse(payload); - if (!parsed.success) { - const message = parsed.error.issues.map((issue) => `${issue.path.join('.')}: ${issue.message}`).join('; '); - throw new AppError('badRequest', StatusCodes.BAD_REQUEST, message, true); - } - - const footprint = (payload as { footprint?: unknown }).footprint; - if (footprint !== undefined && !geometrySchema.safeParse(footprint).success) { - throw new AppError('badRequest', StatusCodes.BAD_REQUEST, ERROR_METADATA_FOOTPRINT, true); - } - - const data = parsed.data; - if (data.classification !== undefined) { - await this.validateClassification(data.classification); - } - if (data.productName !== undefined) { - await this.validateProductNameUniqueExcept(data.productName, identifier); - } - - return record; - } - - public async validateStatusChange(identifier: string): Promise { - const record = await this.catalog.getRecord(identifier); - if (record === undefined) { - throw new AppError('notFound', StatusCodes.NOT_FOUND, ERROR_RECORD_NOT_FOUND, true); - } - if (record.productStatus === RECORD_STATUS_BEING_DELETED) { - throw new AppError('badRequest', StatusCodes.BAD_REQUEST, ERROR_RECORD_BEING_DELETED, true); - } - - return record; - } - - public async ensureRecordAbsentFromExtractable(record: Record3D): Promise { - const logContext = { ...this.logContext, function: this.ensureRecordAbsentFromExtractable.name }; - if (record.productName === undefined) { - return; - } - - const exists = await this.extractable.isExtractableRecordExists(record.productName); - this.logger.debug({ msg: 'extractable existence validation', logContext, productName: record.productName, exists }); - if (exists) { - throw new AppError('conflict', StatusCodes.CONFLICT, ERROR_EXTRACTABLE_CONFLICT, true); - } - } - - private async validateProductNameUniqueExcept(productName: string, identifier: string): Promise { - const records = await this.catalog.findRecords({ productName }); - if (records.some((existing) => existing.id !== identifier)) { - throw new AppError('badRequest', StatusCodes.BAD_REQUEST, ERROR_METADATA_PRODUCT_NAME_UNIQUE, true); - } - } - - private validateModelPath(modelPath: string): void { - if (!modelPath.startsWith(this.basePath)) { - throw new AppError('badRequest', StatusCodes.BAD_REQUEST, `${ERROR_MODEL_PATH_INVALID} (basePath: ${this.basePath})`, true); - } - } - - private async validateProductIdUnique(productId: string): Promise { - const logContext = { ...this.logContext, function: this.validateProductIdUnique.name }; - const records = await this.catalog.findRecords({ productId }); - this.logger.debug({ msg: 'product id uniqueness validation', logContext, productId, matches: records.length }); - - if (records.length > 0) { - throw new AppError('conflict', StatusCodes.CONFLICT, ERROR_PRODUCT_ID_EXISTS, true); - } - } - - private async validateFileExists(modelPath: string, tilesetFilename: string): Promise { - const logContext = { ...this.logContext, function: this.validateFileExists.name }; - const path = buildModelFilePath(modelPath, tilesetFilename); - const exists = await this.provider.fileExists(path); - this.logger.debug({ msg: 'file existence validation', logContext, path, exists }); - - if (!exists) { - throw new AppError('badRequest', StatusCodes.BAD_REQUEST, ERROR_FILE_NOT_FOUND, true); - } - } - - private async validateProductNameUnique(productName: string): Promise { - const logContext = { ...this.logContext, function: this.validateProductNameUnique.name }; - const records = await this.catalog.findRecords({ productName }); - this.logger.debug({ msg: 'product name uniqueness validation', logContext, productName, matches: records.length }); - - if (records.length > 0) { - throw new AppError('badRequest', StatusCodes.BAD_REQUEST, ERROR_METADATA_PRODUCT_NAME_UNIQUE, true); - } - } - - private async validateProductNameNotInFlight(productName: string): Promise { - const logContext = { ...this.logContext, function: this.validateProductNameNotInFlight.name }; - const inFlight = await this.jobnik.hasInFlightIngestionJob(productName); - this.logger.debug({ msg: 'in-flight duplicate validation', logContext, productName, inFlight }); - - if (inFlight) { - throw new AppError('conflict', StatusCodes.CONFLICT, ERROR_PRODUCT_NAME_IN_FLIGHT, true); - } - } - - private async validateTileset(payload: IngestionPayload, footprint: Polygon | MultiPolygon): Promise { - const content = await this.tilesetReader.readTilesetJson(payload.modelPath, payload.tilesetFilename); - - const { modelPolygon, response: polygonResponse } = this.getTilesetModelPolygon(content); - if (!polygonResponse.isValid || modelPolygon === undefined) { - this.rejectValidation(polygonResponse); - } - - const intersectionResponse = this.isFootprintAndModelIntersects(footprint, modelPolygon); - if (!intersectionResponse.isValid) { - this.rejectValidation(intersectionResponse); - } - } - - private getTilesetModelPolygon(content: string): { modelPolygon?: Polygon; response: ValidationResponse } { - const logContext = { ...this.logContext, function: this.getTilesetModelPolygon.name }; - try { - const tilesetJson = JSON.parse(content) as TileSetJson; - const modelPolygon = calculatePolygonFromTileset(tilesetJson); - this.logger.debug({ msg: 'extracted tileset model polygon', logContext, modelPolygon }); - - return { modelPolygon, response: { isValid: true } }; - } catch (err) { - const message = err instanceof Error ? err.message : ERROR_TILESET_INVALID; - this.logger.error({ msg: 'tileset polygon extraction failed', logContext, err }); - return { response: { isValid: false, message } }; - } - } - - private isFootprintAndModelIntersects(footprint: Polygon | MultiPolygon, modelPolygon: Polygon): ValidationResponse { - const logContext = { ...this.logContext, function: this.isFootprintAndModelIntersects.name }; - try { - const footprintFeature = feature(footprint); - const modelFeature: Feature = feature(modelPolygon); - - const intersection = intersect(featureCollection([footprintFeature, modelFeature])); - if (intersection === null) { - return { isValid: false, message: ERROR_FOOTPRINT_FAR_FROM_MODEL }; - } - - const combined = union(featureCollection([footprintFeature, modelFeature])); - const combinedArea = combined === null ? 0 : area(combined); - const coverage = combinedArea === 0 ? 0 : (FULL_COVERAGE_PERCENT * area(footprintFeature)) / combinedArea; - this.logger.debug({ msg: 'calculated footprint coverage of the model', logContext, coverage, percentageLimit: this.percentageLimit }); - - if (coverage < this.percentageLimit) { - return { - isValid: false, - message: `The footprint intersectection with the model doesn't reach minimum required threshhold, the coverage is: ${coverage}% when the minimum coverage is ${this.percentageLimit}%`, - }; - } - return { isValid: true }; - } catch (err) { - this.logger.error({ msg: ERROR_INTERSECTION_FAILED, logContext, err }); - return { isValid: false, message: ERROR_INTERSECTION_FAILED }; - } - } - - private rejectValidation(response: ValidationResponse): never { - const logContext = { ...this.logContext, function: this.rejectValidation.name }; - const message = response.message ?? ERROR_TILESET_INVALID; - this.logger.warn({ msg: 'ingestion validation failed', logContext, isValid: response.isValid, message }); - - throw new AppError('badRequest', StatusCodes.BAD_REQUEST, message, true); - } - - private validateDates(start: unknown, end: unknown): void { - if (start === undefined || end === undefined) { - throw new AppError('badRequest', StatusCodes.BAD_REQUEST, ERROR_METADATA_MISSING_DATE, true); - } - - const startTime = new Date(start as string).getTime(); - const endTime = new Date(end as string).getTime(); - if (Number.isNaN(startTime) || Number.isNaN(endTime)) { - throw new AppError('badRequest', StatusCodes.BAD_REQUEST, ERROR_METADATA_INVALID_DATE, true); - } - - if (startTime > endTime) { - throw new AppError('badRequest', StatusCodes.BAD_REQUEST, ERROR_METADATA_DATE, true); - } - } - - private async validateClassification(classification: string): Promise { - const logContext = { ...this.logContext, function: this.validateClassification.name }; - const classifications = await this.lookupTables.getClassifications(); - this.logger.debug({ msg: 'classification validation', logContext, classifications }); - - if (!classifications.includes(classification)) { - throw new AppError( - 'badRequest', - StatusCodes.BAD_REQUEST, - `classification is not a valid value. Optional values: ${classifications.join()}`, - true - ); - } - } -} diff --git a/tests/unit/externalServices/jobnik/jobnikClient.spec.ts b/tests/unit/externalServices/jobnik/jobnikClient.spec.ts index 95378a3..ccb0fed 100644 --- a/tests/unit/externalServices/jobnik/jobnikClient.spec.ts +++ b/tests/unit/externalServices/jobnik/jobnikClient.spec.ts @@ -4,7 +4,7 @@ import { Registry } from 'prom-client'; import { JobnikClient } from '@src/externalServices/jobnik/jobnikClient'; import { STAGE_TYPES } from '@src/common/constants'; import type { ConfigType } from '@src/common/config'; -import type { IngestionPayload } from '@src/records/models/recordManager'; +import type { IngestionPayload } from '@src/common/interfaces'; const jobManagerConfig = { url: 'http://job-manager', diff --git a/tests/unit/records/models/recordManager.spec.ts b/tests/unit/records/models/recordManager.spec.ts index ef7f88c..e2dc26b 100644 --- a/tests/unit/records/models/recordManager.spec.ts +++ b/tests/unit/records/models/recordManager.spec.ts @@ -1,17 +1,21 @@ import { jsLogger } from '@map-colonies/js-logger'; import { describe, it, expect, beforeEach, vi } from 'vitest'; -import { RecordManager, type IngestionPayload } from '@src/records/models/recordManager'; -import type { ValidationManager } from '@src/validator/validationManager'; +import { RecordManager } from '@src/records/models/recordManager'; +import type { IngestionPayload } from '@src/common/interfaces'; +import type { IngestionValidator } from '@src/validator/ingestionValidator'; +import type { MetadataValidator } from '@src/validator/metadataValidator'; import type { JobnikClient } from '@src/externalServices/jobnik/jobnikClient'; import type { CatalogCall } from '@src/externalServices/catalog/catalogCall'; -const noopValidator = { +const ingestionValidatorStub = { validateIngestion: vi.fn().mockResolvedValue(undefined), +} as unknown as IngestionValidator; +const metadataValidatorStub = { validateDelete: vi.fn().mockResolvedValue({ id: 'rec-1', productName: 'afula' }), validateUpdate: vi.fn().mockResolvedValue({ id: 'rec-1', productName: 'afula' }), validateStatusChange: vi.fn().mockResolvedValue({ id: 'rec-1', productName: 'afula' }), ensureRecordAbsentFromExtractable: vi.fn().mockResolvedValue(undefined), -} as unknown as ValidationManager; +} as unknown as MetadataValidator; const jobnikStub = { createIngestionJob: vi.fn().mockResolvedValue({ jobId: 'job-1', status: 'PENDING' }), createDeleteJob: vi.fn().mockResolvedValue({ jobId: 'del-1', status: 'PENDING' }), @@ -27,7 +31,7 @@ describe('RecordManager', function () { let manager: RecordManager; beforeEach(async function () { - manager = new RecordManager(await jsLogger({ enabled: false }), noopValidator, jobnikStub, catalogStub); + manager = new RecordManager(await jsLogger({ enabled: false }), ingestionValidatorStub, metadataValidatorStub, jobnikStub, catalogStub); }); describe('createIngestion', function () { diff --git a/tests/unit/validator/validationManager.spec.ts b/tests/unit/validator/ingestionValidator.spec.ts similarity index 62% rename from tests/unit/validator/validationManager.spec.ts rename to tests/unit/validator/ingestionValidator.spec.ts index f088ca6..2c4da59 100644 --- a/tests/unit/validator/validationManager.spec.ts +++ b/tests/unit/validator/ingestionValidator.spec.ts @@ -3,32 +3,26 @@ import { join } from 'node:path'; import { jsLogger } from '@map-colonies/js-logger'; import { describe, it, expect, beforeEach, vi } from 'vitest'; import { StatusCodes } from 'http-status-codes'; +import { IngestionValidator } from '@src/validator/ingestionValidator'; +import { TilesetValidator } from '@src/validator/tilesetValidator'; import { - ValidationManager, ERROR_METADATA_DATE, ERROR_METADATA_FOOTPRINT, ERROR_METADATA_MISSING_DATE, ERROR_METADATA_INVALID_DATE, - ERROR_DELETE_RECORD_NOT_FOUND, - ERROR_DELETE_PRODUCT_TYPE, - ERROR_DELETE_STATUS, ERROR_FILE_NOT_FOUND, ERROR_PRODUCT_NAME_IN_FLIGHT, ERROR_FOOTPRINT_FAR_FROM_MODEL, - ERROR_RECORD_NOT_FOUND, - ERROR_RECORD_BEING_DELETED, - ERROR_EXTRACTABLE_CONFLICT, ERROR_MODEL_PATH_INVALID, -} from '@src/validator/validationManager'; +} from '@src/validator/errors'; import { AppError } from '@src/common/appError'; import type { ConfigType } from '@src/common/config'; import type { LookupTablesCall } from '@src/externalServices/lookupTables/lookupTablesCall'; import type { CatalogCall } from '@src/externalServices/catalog/catalogCall'; import type { JobnikClient } from '@src/externalServices/jobnik/jobnikClient'; import type { TilesetReader } from '@src/tileset/tilesetReader'; -import type { ExtractableCall } from '@src/externalServices/extractableManagement/extractableCall'; import type { Provider } from '@src/providers/interfaces'; -import type { IngestionPayload } from '@src/records/models/recordManager'; +import type { IngestionPayload } from '@src/common/interfaces'; import { buildValidMetadata } from '@tests/helpers/metadata'; const regionTilesetJson = readFileSync(join(__dirname, '../../helpers/tilesets/folder/tileset.json'), 'utf-8'); @@ -45,33 +39,23 @@ const ingest = (metadata: Record): IngestionPayload => ({ metadata, }); -describe('ValidationManager', function () { - let validator: ValidationManager; +describe('IngestionValidator', function () { + let validator: IngestionValidator; let catalogStub: CatalogCall; let jobnikStub: JobnikClient; let tilesetReaderStub: TilesetReader; - let extractableStub: ExtractableCall; let providerStub: Provider; beforeEach(async function () { catalogStub = { findRecords: vi.fn().mockResolvedValue([]), - getRecord: vi.fn().mockResolvedValue({ id: 'rec-1', productName: 'afula' }), } as unknown as CatalogCall; jobnikStub = { hasInFlightIngestionJob: vi.fn().mockResolvedValue(false) } as unknown as JobnikClient; tilesetReaderStub = { readTilesetJson: vi.fn().mockResolvedValue(regionTilesetJson) } as unknown as TilesetReader; - extractableStub = { isExtractableRecordExists: vi.fn().mockResolvedValue(false) } as unknown as ExtractableCall; providerStub = { fileExists: vi.fn().mockResolvedValue(true) }; - validator = new ValidationManager( - configStub, - await jsLogger({ enabled: false }), - lookupStub, - catalogStub, - jobnikStub, - tilesetReaderStub, - extractableStub, - providerStub - ); + const logger = await jsLogger({ enabled: false }); + const tilesetValidator = new TilesetValidator(configStub, logger, tilesetReaderStub); + validator = new IngestionValidator(configStub, logger, lookupStub, catalogStub, jobnikStub, providerStub, tilesetValidator); }); it('should pass a fully valid ingestion payload', async function () { @@ -237,97 +221,4 @@ describe('ValidationManager', function () { expect(thrown).toBeInstanceOf(AppError); expect((thrown as AppError).status).toBe(StatusCodes.BAD_REQUEST); }); - - describe('validateUpdate', function () { - it('should return the record for a valid update payload', async function () { - await expect(validator.validateUpdate('rec-1', { description: 'x' })).resolves.toEqual({ id: 'rec-1', productName: 'afula' }); - }); - - it('should throw 404 when the record does not exist', async function () { - (catalogStub.getRecord as ReturnType).mockResolvedValueOnce(undefined); - - let thrown: unknown; - try { - await validator.validateUpdate('missing', { description: 'x' }); - } catch (err) { - thrown = err; - } - - expect(thrown).toBeInstanceOf(AppError); - expect((thrown as AppError).status).toBe(StatusCodes.NOT_FOUND); - expect((thrown as AppError).message).toBe(ERROR_RECORD_NOT_FOUND); - }); - - it('should throw 400 when the record is being deleted', async function () { - (catalogStub.getRecord as ReturnType).mockResolvedValueOnce({ id: 'rec-1', productStatus: 'BEING_DELETED' }); - - await expect(validator.validateUpdate('rec-1', { description: 'x' })).rejects.toThrow(ERROR_RECORD_BEING_DELETED); - }); - - it('should throw 400 when a renamed product name collides with another record', async function () { - (catalogStub.findRecords as ReturnType).mockResolvedValueOnce([{ id: 'other', productName: 'haifa' }]); - - await expect(validator.validateUpdate('rec-1', { productName: 'haifa' })).rejects.toThrow(AppError); - }); - }); - - describe('validateStatusChange', function () { - it('should throw 404 when the record does not exist', async function () { - (catalogStub.getRecord as ReturnType).mockResolvedValueOnce(undefined); - - await expect(validator.validateStatusChange('missing')).rejects.toThrow(ERROR_RECORD_NOT_FOUND); - }); - }); - - describe('ensureRecordAbsentFromExtractable', function () { - it('should throw 409 when the record exists in extractable-management', async function () { - (extractableStub.isExtractableRecordExists as ReturnType).mockResolvedValueOnce(true); - - let thrown: unknown; - try { - await validator.ensureRecordAbsentFromExtractable({ id: 'rec-1', productName: 'afula' }); - } catch (err) { - thrown = err; - } - - expect(thrown).toBeInstanceOf(AppError); - expect((thrown as AppError).status).toBe(StatusCodes.CONFLICT); - expect((thrown as AppError).message).toBe(ERROR_EXTRACTABLE_CONFLICT); - }); - - it('should pass when the record is absent from extractable-management', async function () { - await expect(validator.ensureRecordAbsentFromExtractable({ id: 'rec-1', productName: 'afula' })).resolves.toBeUndefined(); - }); - }); - - describe('validateDelete', function () { - it('should return the record when it is deletable', async function () { - const record = { id: 'rec-1', productType: '3DPhotoRealistic', productStatus: 'UNPUBLISHED' }; - (catalogStub.findRecords as ReturnType).mockResolvedValueOnce([record]); - - await expect(validator.validateDelete('rec-1')).resolves.toEqual(record); - }); - - it('should throw when the record is not found', async function () { - (catalogStub.findRecords as ReturnType).mockResolvedValueOnce([]); - - await expect(validator.validateDelete('missing')).rejects.toThrow(ERROR_DELETE_RECORD_NOT_FOUND); - }); - - it('should throw when the productType is blocked', async function () { - (catalogStub.findRecords as ReturnType).mockResolvedValueOnce([ - { id: 'rec-1', productType: 'QuantizedMeshDTMBest', productStatus: 'UNPUBLISHED' }, - ]); - - await expect(validator.validateDelete('rec-1')).rejects.toThrow(ERROR_DELETE_PRODUCT_TYPE); - }); - - it('should throw when the record is not unpublished', async function () { - (catalogStub.findRecords as ReturnType).mockResolvedValueOnce([ - { id: 'rec-1', productType: '3DPhotoRealistic', productStatus: 'PUBLISHED' }, - ]); - - await expect(validator.validateDelete('rec-1')).rejects.toThrow(ERROR_DELETE_STATUS); - }); - }); }); diff --git a/tests/unit/validator/metadataValidator.spec.ts b/tests/unit/validator/metadataValidator.spec.ts new file mode 100644 index 0000000..27d441f --- /dev/null +++ b/tests/unit/validator/metadataValidator.spec.ts @@ -0,0 +1,126 @@ +import { jsLogger } from '@map-colonies/js-logger'; +import { describe, it, expect, beforeEach, vi } from 'vitest'; +import { StatusCodes } from 'http-status-codes'; +import { MetadataValidator } from '@src/validator/metadataValidator'; +import { + ERROR_DELETE_RECORD_NOT_FOUND, + ERROR_DELETE_PRODUCT_TYPE, + ERROR_DELETE_STATUS, + ERROR_RECORD_NOT_FOUND, + ERROR_RECORD_BEING_DELETED, + ERROR_EXTRACTABLE_CONFLICT, +} from '@src/validator/errors'; +import { AppError } from '@src/common/appError'; +import type { LookupTablesCall } from '@src/externalServices/lookupTables/lookupTablesCall'; +import type { CatalogCall } from '@src/externalServices/catalog/catalogCall'; +import type { ExtractableCall } from '@src/externalServices/extractableManagement/extractableCall'; + +const lookupStub = { getClassifications: vi.fn().mockResolvedValue(['abc123']) } as unknown as LookupTablesCall; + +describe('MetadataValidator', function () { + let validator: MetadataValidator; + let catalogStub: CatalogCall; + let extractableStub: ExtractableCall; + + beforeEach(async function () { + catalogStub = { + findRecords: vi.fn().mockResolvedValue([]), + getRecord: vi.fn().mockResolvedValue({ id: 'rec-1', productName: 'afula' }), + } as unknown as CatalogCall; + extractableStub = { isExtractableRecordExists: vi.fn().mockResolvedValue(false) } as unknown as ExtractableCall; + validator = new MetadataValidator(await jsLogger({ enabled: false }), lookupStub, catalogStub, extractableStub); + }); + + describe('validateUpdate', function () { + it('should return the record for a valid update payload', async function () { + await expect(validator.validateUpdate('rec-1', { description: 'x' })).resolves.toEqual({ id: 'rec-1', productName: 'afula' }); + }); + + it('should throw 404 when the record does not exist', async function () { + (catalogStub.getRecord as ReturnType).mockResolvedValueOnce(undefined); + + let thrown: unknown; + try { + await validator.validateUpdate('missing', { description: 'x' }); + } catch (err) { + thrown = err; + } + + expect(thrown).toBeInstanceOf(AppError); + expect((thrown as AppError).status).toBe(StatusCodes.NOT_FOUND); + expect((thrown as AppError).message).toBe(ERROR_RECORD_NOT_FOUND); + }); + + it('should throw 400 when the record is being deleted', async function () { + (catalogStub.getRecord as ReturnType).mockResolvedValueOnce({ id: 'rec-1', productStatus: 'BEING_DELETED' }); + + await expect(validator.validateUpdate('rec-1', { description: 'x' })).rejects.toThrow(ERROR_RECORD_BEING_DELETED); + }); + + it('should throw 400 when a renamed product name collides with another record', async function () { + (catalogStub.findRecords as ReturnType).mockResolvedValueOnce([{ id: 'other', productName: 'haifa' }]); + + await expect(validator.validateUpdate('rec-1', { productName: 'haifa' })).rejects.toThrow(AppError); + }); + }); + + describe('validateStatusChange', function () { + it('should throw 404 when the record does not exist', async function () { + (catalogStub.getRecord as ReturnType).mockResolvedValueOnce(undefined); + + await expect(validator.validateStatusChange('missing')).rejects.toThrow(ERROR_RECORD_NOT_FOUND); + }); + }); + + describe('ensureRecordAbsentFromExtractable', function () { + it('should throw 409 when the record exists in extractable-management', async function () { + (extractableStub.isExtractableRecordExists as ReturnType).mockResolvedValueOnce(true); + + let thrown: unknown; + try { + await validator.ensureRecordAbsentFromExtractable({ id: 'rec-1', productName: 'afula' }); + } catch (err) { + thrown = err; + } + + expect(thrown).toBeInstanceOf(AppError); + expect((thrown as AppError).status).toBe(StatusCodes.CONFLICT); + expect((thrown as AppError).message).toBe(ERROR_EXTRACTABLE_CONFLICT); + }); + + it('should pass when the record is absent from extractable-management', async function () { + await expect(validator.ensureRecordAbsentFromExtractable({ id: 'rec-1', productName: 'afula' })).resolves.toBeUndefined(); + }); + }); + + describe('validateDelete', function () { + it('should return the record when it is deletable', async function () { + const record = { id: 'rec-1', productType: '3DPhotoRealistic', productStatus: 'UNPUBLISHED' }; + (catalogStub.findRecords as ReturnType).mockResolvedValueOnce([record]); + + await expect(validator.validateDelete('rec-1')).resolves.toEqual(record); + }); + + it('should throw when the record is not found', async function () { + (catalogStub.findRecords as ReturnType).mockResolvedValueOnce([]); + + await expect(validator.validateDelete('missing')).rejects.toThrow(ERROR_DELETE_RECORD_NOT_FOUND); + }); + + it('should throw when the productType is blocked', async function () { + (catalogStub.findRecords as ReturnType).mockResolvedValueOnce([ + { id: 'rec-1', productType: 'QuantizedMeshDTMBest', productStatus: 'UNPUBLISHED' }, + ]); + + await expect(validator.validateDelete('rec-1')).rejects.toThrow(ERROR_DELETE_PRODUCT_TYPE); + }); + + it('should throw when the record is not unpublished', async function () { + (catalogStub.findRecords as ReturnType).mockResolvedValueOnce([ + { id: 'rec-1', productType: '3DPhotoRealistic', productStatus: 'PUBLISHED' }, + ]); + + await expect(validator.validateDelete('rec-1')).rejects.toThrow(ERROR_DELETE_STATUS); + }); + }); +});