diff --git a/bundledApi.yaml b/bundledApi.yaml new file mode 100644 index 0000000..9d62ccf --- /dev/null +++ b/bundledApi.yaml @@ -0,0 +1,285 @@ +openapi: 3.0.1 +info: + title: 3d-ops-trigger + description: Entry point for 3D operations — validates requests and triggers Jobnik jobs + version: 1.0.0 + license: + name: MIT + url: https://opensource.org/licenses/MIT +security: + - {} +paths: + /records: + post: + operationId: createRecord + tags: + - records + summary: Invoke a new model ingestion flow + description: Validates the request (light & fast) and creates a Jobnik ingestion job. + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/ingestionPayload' + responses: + '201': + description: Ingestion job created + content: + application/json: + schema: + $ref: '#/components/schemas/jobResponse' + '400': + description: Bad Request + content: + application/json: + schema: + $ref: '#/components/schemas/error' + /records/validate: + post: + operationId: validateRecord + tags: + - records + summary: Invoke a new model ingestion validation + description: Runs the full ingestion validation and returns the result without triggering a Jobnik job. + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/ingestionPayload' + responses: + '200': + description: Validation result + content: + application/json: + schema: + $ref: '#/components/schemas/validationResultResponse' + /records/canDelete/{recordId}: + get: + operationId: canDeleteRecord + tags: + - records + summary: Validate if a record can be removed + description: Runs the delete validation and returns the result without creating a job. + parameters: + - $ref: '#/components/parameters/recordId' + responses: + '200': + description: Validation result + content: + application/json: + schema: + $ref: '#/components/schemas/validationResultResponse' + /records/{recordId}: + delete: + operationId: deleteRecord + tags: + - records + summary: Delete a model by record id + description: Validates the record can be deleted and creates a Jobnik delete job. + parameters: + - $ref: '#/components/parameters/recordId' + responses: + '200': + description: Delete job created + content: + application/json: + schema: + $ref: '#/components/schemas/jobResponse' + '400': + description: Bad Request + content: + application/json: + schema: + $ref: '#/components/schemas/error' + '404': + description: Record not found + content: + application/json: + schema: + $ref: '#/components/schemas/error' + /metadata/{identifier}: + patch: + operationId: updateMetadata + tags: + - metadata + summary: Update metadata of a model + parameters: + - $ref: '#/components/parameters/identifier' + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/updatePayload' + responses: + '200': + description: Metadata updated + content: + application/json: + schema: + $ref: '#/components/schemas/ackResponse' + '400': + description: Bad Request + content: + application/json: + schema: + $ref: '#/components/schemas/error' + '404': + description: Record not found + content: + application/json: + schema: + $ref: '#/components/schemas/error' + /metadata/status/{identifier}: + patch: + operationId: updateMetadataStatus + tags: + - metadata + summary: Update the publish/unpublish status of a model + parameters: + - $ref: '#/components/parameters/identifier' + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/statusPayload' + responses: + '200': + description: Status updated + content: + application/json: + schema: + $ref: '#/components/schemas/ackResponse' + '400': + description: Bad Request + content: + application/json: + schema: + $ref: '#/components/schemas/error' + '404': + description: Record not found + content: + application/json: + schema: + $ref: '#/components/schemas/error' + /jobStatus/{jobId}: + get: + operationId: getJobStatus + tags: + - jobStatus + summary: Get the status and progress of a job + parameters: + - $ref: '#/components/parameters/jobId' + responses: + '200': + description: Job status + content: + application/json: + schema: + $ref: '#/components/schemas/jobStatusResponse' + '404': + description: Job not found + content: + application/json: + schema: + $ref: '#/components/schemas/error' +components: + parameters: + identifier: + name: identifier + in: path + required: true + description: The record identifier + schema: + type: string + recordId: + name: recordId + in: path + required: true + description: The record identifier + schema: + type: string + jobId: + name: jobId + in: path + required: true + description: The job identifier + schema: + type: string + schemas: + error: + type: object + required: + - message + properties: + message: + type: string + ingestionPayload: + type: object + required: + - modelPath + - tilesetFilename + - metadata + properties: + modelPath: + type: string + description: Path on the shared storage to the 3DTiles folder or 3TZ archive + tilesetFilename: + type: string + description: The tileset entry name (e.g. tileset.json) + metadata: + type: object + description: 3D record metadata (business validation applied downstream) + additionalProperties: true + updatePayload: + type: object + description: Partial metadata fields to update + additionalProperties: true + statusPayload: + type: object + required: + - status + properties: + status: + type: string + enum: + - PUBLISHED + - UNPUBLISHED + jobResponse: + type: object + required: + - jobId + - status + properties: + jobId: + type: string + status: + type: string + ackResponse: + type: object + required: + - message + properties: + message: + type: string + jobStatusResponse: + type: object + required: + - status + properties: + status: + type: string + percentage: + type: number + validationResultResponse: + type: object + required: + - isValid + properties: + isValid: + type: boolean + message: + type: string diff --git a/config/custom-environment-variables.json b/config/custom-environment-variables.json index 30bd2cd..e7c5e28 100644 --- a/config/custom-environment-variables.json +++ b/config/custom-environment-variables.json @@ -1,6 +1,7 @@ { "externalServices": { "catalog": "CATALOG_URL", + "extractable": "EXTRACTABLE_URL", "lookupTables": { "url": "LOOKUP_TABLES_URL", "subUrl": "LOOKUP_TABLES_SUB_URL" @@ -25,7 +26,8 @@ "percentageLimit": { "__name": "VALIDATION_PERCENTAGE_LIMIT", "__format": "number" - } + }, + "basePath": "VALIDATION_BASE_PATH" }, "provider": "PROVIDER_FROM", "NFS": { diff --git a/config/default.json b/config/default.json index 8de50f9..3aa5098 100644 --- a/config/default.json +++ b/config/default.json @@ -35,6 +35,7 @@ }, "externalServices": { "catalog": "http://127.0.0.1:8080", + "extractable": "http://127.0.0.1:8080", "lookupTables": { "url": "http://127.0.0.1:8080", "subUrl": "lookup-tables/lookupData" @@ -53,7 +54,8 @@ } }, "validation": { - "percentageLimit": 10 + "percentageLimit": 10, + "basePath": "/app/models" }, "provider": "NFS", "NFS": { diff --git a/helm/templates/configmap.yaml b/helm/templates/configmap.yaml index 9b572c3..ed7a095 100644 --- a/helm/templates/configmap.yaml +++ b/helm/templates/configmap.yaml @@ -32,6 +32,7 @@ data: npm_config_cache: /tmp/ {{- with .Values.externalServices }} CATALOG_URL: {{ .catalog | quote }} + EXTRACTABLE_URL: {{ .extractable | quote }} {{- with .lookupTables }} LOOKUP_TABLES_URL: {{ .url | quote }} LOOKUP_TABLES_SUB_URL: {{ .subUrl | quote }} @@ -46,6 +47,7 @@ data: TASK_DELETE_TYPE: {{ .delete.taskType | quote }} {{- end }} VALIDATION_PERCENTAGE_LIMIT: {{ .Values.validation.percentageLimit | quote }} + VALIDATION_BASE_PATH: {{ .Values.validation.basePath | quote }} PROVIDER_FROM: {{ .Values.provider | quote }} {{- if eq .Values.provider "NFS" }} PV_SOURCE_PATH: {{ .Values.NFS.pvPath | quote }} diff --git a/helm/values.yaml b/helm/values.yaml index e08d90b..4a9b5c2 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -80,6 +80,7 @@ server: externalServices: catalog: '' + extractable: '' lookupTables: url: '' subUrl: '' @@ -96,6 +97,7 @@ jobManager: validation: percentageLimit: 10 + basePath: '' provider: 'NFS' NFS: diff --git a/openapi3.yaml b/openapi3.yaml index 74a26a0..491b9d1 100644 --- a/openapi3.yaml +++ b/openapi3.yaml @@ -7,12 +7,12 @@ info: name: MIT url: https://opensource.org/licenses/MIT paths: - /record: + /records: post: operationId: createRecord tags: - - record - summary: Start an ingestion process flow + - records + summary: Invoke a new model ingestion flow description: Validates the request (light & fast) and creates a Jobnik ingestion job. requestBody: required: true @@ -33,12 +33,48 @@ paths: application/json: schema: $ref: '#/components/schemas/error' - /record/{id}: + /records/validate: + post: + operationId: validateRecord + tags: + - records + summary: Invoke a new model ingestion validation + description: Runs the full ingestion validation and returns the result without triggering a Jobnik job. + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/ingestionPayload' + responses: + 200: + description: Validation result + content: + application/json: + schema: + $ref: '#/components/schemas/validationResultResponse' + /records/canDelete/{recordId}: + get: + operationId: canDeleteRecord + tags: + - records + summary: Validate if a record can be removed + description: Runs the delete validation and returns the result without creating a job. + parameters: + - $ref: '#/components/parameters/recordId' + responses: + 200: + description: Validation result + content: + application/json: + schema: + $ref: '#/components/schemas/validationResultResponse' + /records/{recordId}: delete: operationId: deleteRecord tags: - - record - summary: Start a delete process flow + - records + summary: Delete a model by record id description: Validates the record can be deleted and creates a Jobnik delete job. parameters: - $ref: '#/components/parameters/recordId' @@ -61,13 +97,14 @@ paths: application/json: schema: $ref: '#/components/schemas/error' + /metadata/{identifier}: patch: - operationId: updateRecord + operationId: updateMetadata tags: - - record - summary: Update metadata for a record + - metadata + summary: Update metadata of a model parameters: - - $ref: '#/components/parameters/recordId' + - $ref: '#/components/parameters/identifier' requestBody: required: true content: @@ -93,14 +130,14 @@ paths: application/json: schema: $ref: '#/components/schemas/error' - /record/status/{id}: + /metadata/status/{identifier}: patch: - operationId: updateRecordStatus + operationId: updateMetadataStatus tags: - - record - summary: Update the publish/unpublish status of a record + - metadata + summary: Update the publish/unpublish status of a model parameters: - - $ref: '#/components/parameters/recordId' + - $ref: '#/components/parameters/identifier' requestBody: required: true content: @@ -126,17 +163,52 @@ paths: application/json: schema: $ref: '#/components/schemas/error' + /jobStatus/{jobId}: + get: + operationId: getJobStatus + tags: + - jobStatus + summary: Get the status and progress of a job + parameters: + - $ref: '#/components/parameters/jobId' + responses: + 200: + description: Job status + content: + application/json: + schema: + $ref: '#/components/schemas/jobStatusResponse' + 404: + description: Job not found + content: + application/json: + schema: + $ref: '#/components/schemas/error' security: - {} components: parameters: + identifier: + name: identifier + in: path + required: true + description: The record identifier + schema: + type: string recordId: - name: id + name: recordId in: path required: true description: The record identifier schema: type: string + jobId: + name: jobId + in: path + required: true + description: The job identifier + schema: + type: string schemas: error: type: object @@ -193,3 +265,21 @@ components: properties: message: type: string + jobStatusResponse: + type: object + required: + - status + properties: + status: + type: string + percentage: + type: number + validationResultResponse: + type: object + required: + - isValid + properties: + isValid: + type: boolean + message: + type: string diff --git a/package-lock.json b/package-lock.json index 08ceef5..d48d080 100644 --- a/package-lock.json +++ b/package-lock.json @@ -23,6 +23,7 @@ "@map-colonies/schemas": "^1.26.0", "@map-colonies/tracing": "^1.0.0", "@map-colonies/tracing-utils": "^2.0.0", + "@map-colonies/types": "^1.4.0", "@opentelemetry/api": "^1.9.1", "@turf/turf": "^7.4.0", "axios": "^1.20.0", @@ -30,6 +31,7 @@ "express": "^4.21.2", "express-openapi-validator": "^5.6.2", "http-status-codes": "^2.3.0", + "mime-types": "^2.1.35", "prom-client": "^15.1.3", "reflect-metadata": "^0.2.2", "tsyringe": "^4.8.0", @@ -1869,6 +1871,19 @@ "node": ">=24" } }, + "node_modules/@map-colonies/types": { + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/@map-colonies/types/-/types-1.10.0.tgz", + "integrity": "sha512-FwVrBAYzjnpmd9Jnq7Mex7axl8KR6o8JjKVGu3kDCGHz26Ru9qSCpzkA+zUVV8fKNxWVZSmDi7IiHEe9WsOalw==", + "license": "ISC", + "dependencies": { + "@types/geojson": "^7946.0.16", + "@types/mime-types": "^2.1.1", + "copyfiles": "^2.4.1", + "epsg-index": "^1.3.0", + "ts-morph": "^18.0.0" + } + }, "node_modules/@napi-rs/wasm-runtime": { "version": "1.1.6", "resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-1.1.6.tgz", @@ -1905,7 +1920,6 @@ "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", "integrity": "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==", - "dev": true, "license": "MIT", "dependencies": { "@nodelib/fs.stat": "2.0.5", @@ -1919,7 +1933,6 @@ "version": "2.0.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz", "integrity": "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==", - "dev": true, "license": "MIT", "engines": { "node": ">= 8" @@ -1929,7 +1942,6 @@ "version": "1.2.8", "resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz", "integrity": "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==", - "dev": true, "license": "MIT", "dependencies": { "@nodelib/fs.scandir": "2.1.5", @@ -4763,6 +4775,63 @@ "dev": true, "license": "MIT" }, + "node_modules/@ts-morph/common": { + "version": "0.19.0", + "resolved": "https://registry.npmjs.org/@ts-morph/common/-/common-0.19.0.tgz", + "integrity": "sha512-Unz/WHmd4pGax91rdIKWi51wnVUW11QttMEPpBiBgIewnc9UQIX7UDLxr5vRlqeByXCwhkF6VabSsI0raWcyAQ==", + "license": "MIT", + "dependencies": { + "fast-glob": "^3.2.12", + "minimatch": "^7.4.3", + "mkdirp": "^2.1.6", + "path-browserify": "^1.0.1" + } + }, + "node_modules/@ts-morph/common/node_modules/balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", + "license": "MIT" + }, + "node_modules/@ts-morph/common/node_modules/brace-expansion": { + "version": "2.1.7", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.1.7.tgz", + "integrity": "sha512-uZbew1NqdmPDTMJ8ah1y+b+9QEJrfkXFk3RcTQw3X0jW/xRUvFKsg1CfQdSYGdTbXZWExtU3J3ccxtnfw1Fi0g==", + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0" + } + }, + "node_modules/@ts-morph/common/node_modules/minimatch": { + "version": "7.4.9", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-7.4.9.tgz", + "integrity": "sha512-Brg/fp/iAVDOQoHxkuN5bEYhyQlZhxddI78yWsCbeEwTHXQjlNLtiJDUsp1GIptVqMI7/gkJMz4vVAc01mpoBw==", + "license": "ISC", + "dependencies": { + "brace-expansion": "^2.0.2" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/@ts-morph/common/node_modules/mkdirp": { + "version": "2.1.6", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-2.1.6.tgz", + "integrity": "sha512-+hEnITedc8LAtIP9u3HJDFIdcLV2vXP33sqLLIzkv1Db1zO/1OxbvYf0Y1OC/S/Qo5dxHXepofhmxL02PsKe+A==", + "license": "MIT", + "bin": { + "mkdirp": "dist/cjs/src/bin.js" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/@turf/along": { "version": "7.4.0", "resolved": "https://registry.npmjs.org/@turf/along/-/along-7.4.0.tgz", @@ -7736,6 +7805,12 @@ "integrity": "sha512-/pyBZWSLD2n0dcHE3hq8s8ZvcETHtEuF+3E7XVt0Ig2nvsVQXdghHVcEkIWjy9A0wKfTn97a/PSDYohKIlnP/w==", "license": "MIT" }, + "node_modules/@types/mime-types": { + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/@types/mime-types/-/mime-types-2.1.4.tgz", + "integrity": "sha512-lfU4b34HOri+kAY5UheuFMWPDOI+OPceBSHZKp69gEyTL/mmJ4cnU6Y/rlme3UL3GyOn6Y42hyIEw0/q8sWx5w==", + "license": "MIT" + }, "node_modules/@types/multer": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/@types/multer/-/multer-2.2.0.tgz", @@ -9083,7 +9158,6 @@ "version": "3.0.3", "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz", "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==", - "dev": true, "license": "MIT", "dependencies": { "fill-range": "^7.1.1" @@ -9418,6 +9492,12 @@ "node": ">=8" } }, + "node_modules/code-block-writer": { + "version": "12.0.0", + "resolved": "https://registry.npmjs.org/code-block-writer/-/code-block-writer-12.0.0.tgz", + "integrity": "sha512-q4dMFMlXtKR3XNBHyMHt/3pwYNA69EDk00lloMOaaUMKPUXBw6lpXtbu3MMVG6/uOihGnRDOlkyqsONEUj60+w==", + "license": "MIT" + }, "node_modules/color-convert": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", @@ -9542,7 +9622,6 @@ "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", - "dev": true, "license": "MIT" }, "node_modules/concat-stream": { @@ -9681,7 +9760,6 @@ "version": "2.4.1", "resolved": "https://registry.npmjs.org/copyfiles/-/copyfiles-2.4.1.tgz", "integrity": "sha512-fereAvAvxDrQDOXybk3Qu3dPbOoKoysFMWtkY3mv5BsL8//OSZVL5DCLYqgRfY5cWirgRzlC+WSrxp6Bo3eNZg==", - "dev": true, "license": "MIT", "dependencies": { "glob": "^7.0.5", @@ -9701,14 +9779,12 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", - "dev": true, "license": "MIT" }, "node_modules/copyfiles/node_modules/brace-expansion": { "version": "1.1.15", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.15.tgz", "integrity": "sha512-EwOCDEex4quD37XhqM3omwtMoJjr//isUZz1JopUNWms+4Z2ViyM/k1YIRePpoVNnQhENnxtFjLaxNHrT7xIUg==", - "dev": true, "license": "MIT", "dependencies": { "balanced-match": "^1.0.0", @@ -9719,7 +9795,6 @@ "version": "7.0.4", "resolved": "https://registry.npmjs.org/cliui/-/cliui-7.0.4.tgz", "integrity": "sha512-OcRE68cOsVMXp1Yvonl/fzkQOyjLSu/8bhPDfQt0e0/Eb283TKP20Fs2MqoPsr9SwA595rRCA+QMzYc9nBP+JQ==", - "dev": true, "license": "ISC", "dependencies": { "string-width": "^4.2.0", @@ -9731,7 +9806,6 @@ "version": "3.1.5", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.5.tgz", "integrity": "sha512-VgjWUsnnT6n+NUk6eZq77zeFdpW2LWDzP6zFGrCbHXiYNul5Dzqk2HHQ5uFH2DNW5Xbp8+jVzaeNt94ssEEl4w==", - "dev": true, "license": "ISC", "dependencies": { "brace-expansion": "^1.1.7" @@ -9744,7 +9818,6 @@ "version": "4.2.3", "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", - "dev": true, "license": "MIT", "dependencies": { "emoji-regex": "^8.0.0", @@ -9759,7 +9832,6 @@ "version": "6.0.1", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "dev": true, "license": "MIT", "dependencies": { "ansi-regex": "^5.0.1" @@ -9772,7 +9844,6 @@ "version": "16.2.2", "resolved": "https://registry.npmjs.org/yargs/-/yargs-16.2.2.tgz", "integrity": "sha512-Nt9ZJjXTv5R8MHbqby/wXQ6Gi0Bb3TcYZkR1bzuL4yB2OxWPkXknz513gEF0GoA6tn00UpbPvERW8rzCuWCA6w==", - "dev": true, "license": "MIT", "dependencies": { "cliui": "^7.0.2", @@ -9791,7 +9862,6 @@ "version": "20.2.9", "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-20.2.9.tgz", "integrity": "sha512-y11nGElTIV+CT3Zv9t7VKl+Q3hTQoT9a1Qzezhhl6Rp21gJ/IVTW7Z3y9EWXhuUBC2Shnf+DX0antecpAwSP8w==", - "dev": true, "license": "ISC", "engines": { "node": ">=10" @@ -9815,7 +9885,6 @@ "version": "1.0.3", "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz", "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==", - "dev": true, "license": "MIT" }, "node_modules/cosmiconfig": { @@ -10111,6 +10180,15 @@ "node": ">=6" } }, + "node_modules/epsg-index": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/epsg-index/-/epsg-index-1.3.0.tgz", + "integrity": "sha512-pWJ7lkMbH0Au+MJyBV4GxiX+KmOknPc6c5FqDAYD61Ju3LblR60VGmOlteBhK9Dm3oy++Jxa2kUimONeSvsKRA==", + "license": "ISC", + "engines": { + "node": ">=8" + } + }, "node_modules/error-ex": { "version": "1.3.4", "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.4.tgz", @@ -10768,7 +10846,6 @@ "version": "3.3.3", "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.3.tgz", "integrity": "sha512-7MptL8U0cqcFdzIzwOTHoilX9x5BrNqye7Z/LuC7kCMRio1EMSyqRK3BEAUD7sXRq4iT4AzTVuZdhgQ2TCvYLg==", - "dev": true, "license": "MIT", "dependencies": { "@nodelib/fs.stat": "^2.0.2", @@ -10785,7 +10862,6 @@ "version": "5.1.2", "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", - "dev": true, "license": "ISC", "dependencies": { "is-glob": "^4.0.1" @@ -10834,7 +10910,6 @@ "version": "1.20.1", "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.20.1.tgz", "integrity": "sha512-GGToxJ/w1x32s/D2EKND7kTil4n8OVk/9mycTc4VDza13lOvpUZTGX3mFSCtV9ksdGBVzvsyAVLM6mHFThxXxw==", - "dev": true, "license": "ISC", "dependencies": { "reusify": "^1.0.4" @@ -10882,7 +10957,6 @@ "version": "7.1.1", "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==", - "dev": true, "license": "MIT", "dependencies": { "to-regex-range": "^5.0.1" @@ -11073,7 +11147,6 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==", - "dev": true, "license": "ISC" }, "node_modules/fsevents": { @@ -11240,7 +11313,6 @@ "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", "deprecated": "Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me", - "dev": true, "license": "ISC", "dependencies": { "fs.realpath": "^1.0.0", @@ -11274,14 +11346,12 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", - "dev": true, "license": "MIT" }, "node_modules/glob/node_modules/brace-expansion": { "version": "1.1.15", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.15.tgz", "integrity": "sha512-EwOCDEex4quD37XhqM3omwtMoJjr//isUZz1JopUNWms+4Z2ViyM/k1YIRePpoVNnQhENnxtFjLaxNHrT7xIUg==", - "dev": true, "license": "MIT", "dependencies": { "balanced-match": "^1.0.0", @@ -11292,7 +11362,6 @@ "version": "3.1.5", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.5.tgz", "integrity": "sha512-VgjWUsnnT6n+NUk6eZq77zeFdpW2LWDzP6zFGrCbHXiYNul5Dzqk2HHQ5uFH2DNW5Xbp8+jVzaeNt94ssEEl4w==", - "dev": true, "license": "ISC", "dependencies": { "brace-expansion": "^1.1.7" @@ -11593,7 +11662,6 @@ "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", "deprecated": "This module is not supported, and leaks memory. Do not use it. Check out lru-cache if you want a good and tested way to coalesce async requests by a key value, which is much more comprehensive and powerful.", - "dev": true, "license": "ISC", "dependencies": { "once": "^1.3.0", @@ -11681,7 +11749,6 @@ "version": "2.1.1", "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==", - "dev": true, "license": "MIT", "engines": { "node": ">=0.10.0" @@ -11700,7 +11767,6 @@ "version": "4.0.3", "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==", - "dev": true, "license": "MIT", "dependencies": { "is-extglob": "^2.1.1" @@ -11726,7 +11792,6 @@ "version": "7.0.0", "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", - "dev": true, "license": "MIT", "engines": { "node": ">=0.12.0" @@ -11774,7 +11839,6 @@ "version": "0.0.1", "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz", "integrity": "sha512-D2S+3GLxWH+uhrNEcoh/fnmYeP8E8/zHl644d/jdA0g2uyXvy3sb0qxotE+ne0LtccHknQzWwZEzhak7oJ0COQ==", - "dev": true, "license": "MIT" }, "node_modules/isexe": { @@ -12498,7 +12562,6 @@ "version": "1.4.1", "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz", "integrity": "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==", - "dev": true, "license": "MIT", "engines": { "node": ">= 8" @@ -12517,7 +12580,6 @@ "version": "4.0.8", "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.8.tgz", "integrity": "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==", - "dev": true, "license": "MIT", "dependencies": { "braces": "^3.0.3", @@ -12531,7 +12593,6 @@ "version": "2.3.2", "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.2.tgz", "integrity": "sha512-V7+vQEJ06Z+c5tSye8S+nHUfI51xoXIXjHQ99cQtKUkQqqO1kO/KCJUfZXuB47h/YBlDhah2H3hdUGXn8ie0oA==", - "dev": true, "license": "MIT", "engines": { "node": ">=8.6" @@ -12634,7 +12695,6 @@ "version": "1.0.4", "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-1.0.4.tgz", "integrity": "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==", - "dev": true, "license": "MIT", "bin": { "mkdirp": "bin/cmd.js" @@ -12826,7 +12886,6 @@ "version": "0.0.0", "resolved": "https://registry.npmjs.org/noms/-/noms-0.0.0.tgz", "integrity": "sha512-lNDU9VJaOPxUmXcLb+HQFeUgQQPtMI24Gt6hgfuMHRJgMRHMF/qZ4HJD3GDru4sSw9IQl2jPjAYnQrdIeLbwow==", - "dev": true, "license": "ISC", "dependencies": { "inherits": "^2.0.1", @@ -12837,7 +12896,6 @@ "version": "1.0.34", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.0.34.tgz", "integrity": "sha512-ok1qVCJuRkNmvebYikljxJA/UEsKwLl2nI1OmaqAu4/UE+h0wKCHok4XkL/gvi39OacXvw59RJUOFUkDib2rHg==", - "dev": true, "license": "MIT", "dependencies": { "core-util-is": "~1.0.0", @@ -12850,7 +12908,6 @@ "version": "0.10.31", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz", "integrity": "sha512-ev2QzSzWPYmy9GuqfIVildA4OdcGLeFZQrq5ys6RtiuF+RQQiZWr8TZNyAcuVXyQRYfEO+MsoB/1BuQVhOJuoQ==", - "dev": true, "license": "MIT" }, "node_modules/normalize-path": { @@ -13596,6 +13653,12 @@ "node": ">= 0.8" } }, + "node_modules/path-browserify": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/path-browserify/-/path-browserify-1.0.1.tgz", + "integrity": "sha512-b7uo2UCUOYZcnF/3ID0lulOJi/bafxa1xPe7ZPsammBSpjSWQkjNxlt635YGS2MiR9GjvuXCtz2emr3jbsz98g==", + "license": "MIT" + }, "node_modules/path-exists": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", @@ -13610,7 +13673,6 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==", - "dev": true, "license": "MIT", "engines": { "node": ">=0.10.0" @@ -14051,7 +14113,6 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==", - "dev": true, "license": "MIT" }, "node_modules/process-warning": { @@ -14178,7 +14239,6 @@ "version": "1.2.3", "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==", - "dev": true, "funding": [ { "type": "github", @@ -14407,7 +14467,6 @@ "version": "1.1.0", "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.1.0.tgz", "integrity": "sha512-g6QUff04oZpHs0eG5p83rFLhHeV00ug/Yf9nZM6fLeUrPguBTkTQOdpAWWspMh55TZfVQDPaN3NQJfbVRAxdIw==", - "dev": true, "license": "MIT", "engines": { "iojs": ">=1.0.0", @@ -14496,7 +14555,6 @@ "version": "1.2.0", "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz", "integrity": "sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==", - "dev": true, "funding": [ { "type": "github", @@ -15216,7 +15274,6 @@ "version": "2.0.5", "resolved": "https://registry.npmjs.org/through2/-/through2-2.0.5.tgz", "integrity": "sha512-/mrRod8xqpA+IHSLyGCQ2s8SPHiCDEeQJSep1jqLYeEUClOFG2Qsh+4FU6G9VeqpZnGW/Su8LQGc4YKni5rYSQ==", - "dev": true, "license": "MIT", "dependencies": { "readable-stream": "~2.3.6", @@ -15227,14 +15284,12 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==", - "dev": true, "license": "MIT" }, "node_modules/through2/node_modules/readable-stream": { "version": "2.3.8", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", - "dev": true, "license": "MIT", "dependencies": { "core-util-is": "~1.0.0", @@ -15250,14 +15305,12 @@ "version": "5.1.2", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", - "dev": true, "license": "MIT" }, "node_modules/through2/node_modules/string_decoder": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", - "dev": true, "license": "MIT", "dependencies": { "safe-buffer": "~5.1.0" @@ -15317,7 +15370,6 @@ "version": "5.0.1", "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", - "dev": true, "license": "MIT", "dependencies": { "is-number": "^7.0.0" @@ -15417,6 +15469,16 @@ } } }, + "node_modules/ts-morph": { + "version": "18.0.0", + "resolved": "https://registry.npmjs.org/ts-morph/-/ts-morph-18.0.0.tgz", + "integrity": "sha512-Kg5u0mk19PIIe4islUI/HWRvm9bC1lHejK4S0oh1zaZ77TMZAEmQC0sHQYiu2RgCQFZKXz1fMVi/7nOOeirznA==", + "license": "MIT", + "dependencies": { + "@ts-morph/common": "~0.19.0", + "code-block-writer": "^12.0.0" + } + }, "node_modules/tsc-alias": { "version": "1.9.1", "resolved": "https://registry.npmjs.org/tsc-alias/-/tsc-alias-1.9.1.tgz", @@ -15644,7 +15706,6 @@ "version": "4.0.0", "resolved": "https://registry.npmjs.org/untildify/-/untildify-4.0.0.tgz", "integrity": "sha512-KK8xQ1mkzZeg9inewmFVDNkg3l5LUhoq9kN6iWYB/CC9YMG8HA+c1Q8HwDe6dEX7kErrEVNVBO3fWsVq5iDgtw==", - "dev": true, "license": "MIT", "engines": { "node": ">=8" diff --git a/package.json b/package.json index 76b94a0..33030e1 100644 --- a/package.json +++ b/package.json @@ -14,6 +14,7 @@ "prelint:fix": "npm run format:fix", "prelint": "npm run format", "lint:openapi": "redocly lint openapi3.yaml", + "bundleOpenApi": "redocly bundle openapi3.yaml -o bundledApi.yaml", "lint": "eslint .", "lint:fix": "eslint --fix .", "prebuild": "npm run clean && npm run generate:openapi-types", @@ -48,6 +49,7 @@ "@map-colonies/schemas": "^1.26.0", "@map-colonies/tracing": "^1.0.0", "@map-colonies/tracing-utils": "^2.0.0", + "@map-colonies/types": "^1.4.0", "@opentelemetry/api": "^1.9.1", "@turf/turf": "^7.4.0", "axios": "^1.20.0", @@ -55,6 +57,7 @@ "express": "^4.21.2", "express-openapi-validator": "^5.6.2", "http-status-codes": "^2.3.0", + "mime-types": "^2.1.35", "prom-client": "^15.1.3", "reflect-metadata": "^0.2.2", "tsyringe": "^4.8.0", diff --git a/src/common/config.ts b/src/common/config.ts index 8d3292d..0f28da5 100644 --- a/src/common/config.ts +++ b/src/common/config.ts @@ -9,6 +9,7 @@ interface LookupTablesConfig { interface ExternalServicesConfig { lookupTables: LookupTablesConfig; catalog: string; + extractable: string; } interface JobManagerConfig { @@ -26,6 +27,7 @@ interface JobManagerConfig { interface ValidationConfig { percentageLimit: number; + basePath: string; } interface NFSConfig { @@ -70,9 +72,10 @@ const opsTriggerConfigSchema = { provider: { type: 'string', enum: ['NFS', 'S3'] }, validation: { type: 'object', - required: ['percentageLimit'], + required: ['percentageLimit', 'basePath'], properties: { percentageLimit: { type: 'number' }, + basePath: { type: 'string' }, }, }, // eslint-disable-next-line @typescript-eslint/naming-convention @@ -98,7 +101,7 @@ const opsTriggerConfigSchema = { }, externalServices: { type: 'object', - required: ['lookupTables', 'catalog'], + required: ['lookupTables', 'catalog', 'extractable'], properties: { lookupTables: { type: 'object', @@ -109,6 +112,7 @@ const opsTriggerConfigSchema = { }, }, catalog: { type: 'string' }, + extractable: { type: 'string' }, }, }, jobManager: { diff --git a/src/common/interfaces.ts b/src/common/interfaces.ts index 6220fde..879c230 100644 --- a/src/common/interfaces.ts +++ b/src/common/interfaces.ts @@ -1,3 +1,13 @@ +import type { components } from '@openapi'; + +export type IngestionPayload = components['schemas']['ingestionPayload']; +export type UpdatePayload = components['schemas']['updatePayload']; +export type StatusPayload = components['schemas']['statusPayload']; +export type JobResponse = components['schemas']['jobResponse']; +export type AckResponse = components['schemas']['ackResponse']; +export type ValidationResultResponse = components['schemas']['validationResultResponse']; +export type JobStatusResponse = components['schemas']['jobStatusResponse']; + export interface IConfig { get: (setting: string) => T; has: (setting: string) => boolean; diff --git a/src/common/util.ts b/src/common/util.ts index 542af20..f4b4f2f 100644 --- a/src/common/util.ts +++ b/src/common/util.ts @@ -1,4 +1,13 @@ +import type { Polygon, Position } from 'geojson'; + +const TWO_DIMENSIONS = 2; + export const is3tz = (modelPath: string): boolean => modelPath.toLowerCase().endsWith('.3tz'); export const buildModelFilePath = (modelPath: string, tilesetFilename: string): string => is3tz(modelPath) ? modelPath : `${modelPath}/${tilesetFilename}`; + +export const convertPolygonTo2DPolygon = (polygon: Polygon): Polygon => ({ + ...polygon, + coordinates: polygon.coordinates.map((ring: Position[]) => ring.map((coordinate: Position) => coordinate.slice(0, TWO_DIMENSIONS))), +}); diff --git a/src/containerConfig.ts b/src/containerConfig.ts index 3dbe41f..75be200 100644 --- a/src/containerConfig.ts +++ b/src/containerConfig.ts @@ -6,7 +6,9 @@ import { jsLogger } from '@map-colonies/js-logger'; import { type InjectionObject, registerDependencies } from '@common/dependencyRegistration'; import { SERVICES, SERVICE_NAME } from '@common/constants'; import { getTracing } from '@common/tracing'; -import { recordRouterFactory, RECORD_ROUTER_SYMBOL } from './record/routes/recordRouter'; +import { recordsRouterFactory, RECORDS_ROUTER_SYMBOL } from './records/routes/recordsRouter'; +import { jobStatusRouterFactory, JOB_STATUS_ROUTER_SYMBOL } from './jobStatus/routes/jobStatusRouter'; +import { metadataRouterFactory, METADATA_ROUTER_SYMBOL } from './metadata/routes/metadataRouter'; import { providerFactory } from './providers/getProvider'; import { getConfig } from './common/config'; @@ -32,7 +34,9 @@ export const registerExternalValues = async (options?: RegisterOptions): Promise { token: SERVICES.TRACER, provider: { useValue: tracer } }, { token: SERVICES.METRICS, provider: { useValue: metricsRegistry } }, { token: SERVICES.PROVIDER, provider: { useFactory: providerFactory } }, - { token: RECORD_ROUTER_SYMBOL, provider: { useFactory: recordRouterFactory } }, + { token: RECORDS_ROUTER_SYMBOL, provider: { useFactory: recordsRouterFactory } }, + { token: JOB_STATUS_ROUTER_SYMBOL, provider: { useFactory: jobStatusRouterFactory } }, + { token: METADATA_ROUTER_SYMBOL, provider: { useFactory: metadataRouterFactory } }, { token: 'onSignal', provider: { diff --git a/src/externalServices/catalog/catalogCall.ts b/src/externalServices/catalog/catalogCall.ts index 3f0fd9e..90696d0 100644 --- a/src/externalServices/catalog/catalogCall.ts +++ b/src/externalServices/catalog/catalogCall.ts @@ -6,7 +6,7 @@ import { SERVICES } from '@common/constants'; import { AppError } from '@common/appError'; import type { ConfigType } from '@common/config'; import type { LogContext } from '@common/interfaces'; -import type { IFindRecordsPayload, Record3D } from './interfaces'; +import type { CatalogStatusPayload, CatalogUpdatePayload, IFindRecordsPayload, Record3D } from './interfaces'; @injectable() export class CatalogCall { @@ -27,7 +27,7 @@ export class CatalogCall { public async findRecords(payload: IFindRecordsPayload): Promise { const logContext = { ...this.logContext, function: this.findRecords.name }; this.logger.debug({ msg: `Searching in catalog ${this.catalog}/metadata/find`, logContext, payload }); - try { + return this.run(logContext, 'Something went wrong in catalog when trying to find records', 'Problem with catalog find', {}, async () => { const response = await axios.post(`${this.catalog}/metadata/find`, payload); if (response.status !== StatusCodes.OK.valueOf()) { this.logger.error({ msg: `Catalog returned unexpected status: ${response.status}`, logContext }); @@ -41,13 +41,71 @@ export class CatalogCall { this.logger.debug({ msg: `Found ${records.length} record(s) in catalog`, logContext }); return records; + }); + } + + public async getRecord(identifier: string): Promise { + const logContext = { ...this.logContext, function: this.getRecord.name }; + this.logger.debug({ msg: `Getting record ${identifier} from catalog`, logContext }); + return this.run( + logContext, + 'Something went wrong in catalog when getting a record', + 'Problem with catalog during record lookup', + { identifier }, + async () => { + const response = await axios.get(`${this.catalog}/metadata/${identifier}`, { + validateStatus: (status) => status === StatusCodes.OK.valueOf() || status === StatusCodes.NOT_FOUND.valueOf(), + }); + return response.status === StatusCodes.NOT_FOUND.valueOf() ? undefined : response.data; + } + ); + } + + public async patchMetadata(identifier: string, payload: CatalogUpdatePayload): Promise { + const logContext = { ...this.logContext, function: this.patchMetadata.name }; + this.logger.debug({ msg: `Updating metadata for record ${identifier} in catalog`, logContext }); + return this.run( + logContext, + 'Something went wrong in catalog when updating metadata', + 'Problem with catalog during metadata update', + { identifier }, + async () => { + const response = await axios.patch(`${this.catalog}/metadata/${identifier}`, payload); + return response.data; + } + ); + } + + public async changeStatus(identifier: string, payload: CatalogStatusPayload): Promise { + const logContext = { ...this.logContext, function: this.changeStatus.name }; + this.logger.debug({ msg: `Changing status for record ${identifier} in catalog`, logContext }); + return this.run( + logContext, + 'Something went wrong in catalog when changing status', + 'Problem with catalog during status change', + { identifier }, + async () => { + const response = await axios.patch(`${this.catalog}/metadata/status/${identifier}`, payload); + return response.data; + } + ); + } + + private async run( + logContext: LogContext, + errorLog: string, + failMessage: string, + details: Record, + fn: () => Promise + ): Promise { + try { + return await fn(); } catch (err) { if (err instanceof AppError) { throw err; } - - this.logger.error({ msg: 'Something went wrong in catalog when trying to find records', logContext, err }); - throw new AppError('catalog', StatusCodes.INTERNAL_SERVER_ERROR, 'Problem with catalog find', true); + this.logger.error({ msg: errorLog, logContext, ...details, err }); + throw new AppError('catalog', StatusCodes.INTERNAL_SERVER_ERROR, failMessage, true); } } } diff --git a/src/externalServices/catalog/interfaces.ts b/src/externalServices/catalog/interfaces.ts index 0b0609b..e3987df 100644 --- a/src/externalServices/catalog/interfaces.ts +++ b/src/externalServices/catalog/interfaces.ts @@ -17,3 +17,9 @@ export interface IFindRecordsPayload { classification?: string; productStatus?: string; } + +export type CatalogUpdatePayload = Record; + +export interface CatalogStatusPayload { + productStatus: string; +} diff --git a/src/externalServices/extractableManagement/extractableCall.ts b/src/externalServices/extractableManagement/extractableCall.ts new file mode 100644 index 0000000..02dbd22 --- /dev/null +++ b/src/externalServices/extractableManagement/extractableCall.ts @@ -0,0 +1,39 @@ +import axios from 'axios'; +import { inject, injectable } from 'tsyringe'; +import type { Logger } from '@map-colonies/js-logger'; +import { StatusCodes } from 'http-status-codes'; +import { SERVICES } from '@common/constants'; +import { AppError } from '@common/appError'; +import type { ConfigType } from '@common/config'; +import type { LogContext } from '@common/interfaces'; + +@injectable() +export class ExtractableCall { + private readonly logContext: LogContext; + private readonly extractable: string; + + public constructor( + @inject(SERVICES.CONFIG) private readonly config: ConfigType, + @inject(SERVICES.LOGGER) private readonly logger: Logger + ) { + this.extractable = this.config.get('externalServices.extractable'); + this.logContext = { + fileName: __filename, + class: ExtractableCall.name, + }; + } + + public async isExtractableRecordExists(recordName: string): Promise { + const logContext = { ...this.logContext, function: this.isExtractableRecordExists.name }; + this.logger.debug({ msg: `Checking record '${recordName}' in extractable service`, logContext }); + try { + const response = await axios.get(`${this.extractable}/records/${encodeURIComponent(recordName)}`, { + validateStatus: (status) => status === StatusCodes.OK.valueOf() || status === StatusCodes.NOT_FOUND.valueOf(), + }); + return response.status === StatusCodes.OK.valueOf(); + } catch (err) { + this.logger.error({ msg: 'Something went wrong in extractable when checking a record', logContext, recordName, err }); + throw new AppError('extractable', StatusCodes.INTERNAL_SERVER_ERROR, 'Problem with extractable during record lookup', true); + } + } +} diff --git a/src/externalServices/jobnik/jobnikClient.ts b/src/externalServices/jobnik/jobnikClient.ts index 593ce4e..52c1d69 100644 --- a/src/externalServices/jobnik/jobnikClient.ts +++ b/src/externalServices/jobnik/jobnikClient.ts @@ -3,12 +3,12 @@ import { StatusCodes } from 'http-status-codes'; import type { Logger } from '@map-colonies/js-logger'; import type { Registry } from 'prom-client'; import { JobnikSDK } from '@map-colonies/jobnik-sdk'; +import type { JobId } from '@map-colonies/jobnik-sdk'; import { IN_FLIGHT_JOB_STATUSES, SERVICES, STAGE_TYPES } from '@common/constants'; import { is3tz } from '@common/util'; import { AppError } from '@common/appError'; import type { ConfigType, JobManagerConfig } from '@common/config'; -import type { LogContext } from '@common/interfaces'; -import type { IngestionPayload, JobResponse } from '../../record/models/recordManager'; +import type { IngestionPayload, JobResponse, JobStatusResponse, LogContext } from '@common/interfaces'; import type { Record3D } from '../catalog/interfaces'; interface StageDescriptor { @@ -69,13 +69,27 @@ export class JobnikClient { } } - public async createIngestionJob(payload: IngestionPayload): Promise { + public async getJobStatus(jobId: string): Promise { + const logContext = { ...this.logContext, function: this.getJobStatus.name }; + const { data, error, response } = await this.apiClient.GET('/v1/jobs/{jobId}', { params: { path: { jobId: jobId as JobId } } }); + if (error !== undefined) { + if (response.status === StatusCodes.NOT_FOUND.valueOf()) { + throw new AppError('badRequest', StatusCodes.NOT_FOUND, `job ${jobId} was not found`, true); + } + this.logger.error({ msg: 'failed querying Jobnik for job status', logContext, jobId, err: error }); + throw new AppError('jobnik', StatusCodes.INTERNAL_SERVER_ERROR, 'failed querying Jobnik for job status', false); + } + + return { status: data.status, percentage: data.percentage }; + } + + public async createIngestionJob(payload: IngestionPayload, modelId: string): Promise { const logContext = { ...this.logContext, function: this.createIngestionJob.name }; const isArchive = is3tz(payload.modelPath); const job = await this.producer.createJob({ name: this.jobManager.ingestion.jobType, - data: { modelPath: payload.modelPath, tilesetFilename: payload.tilesetFilename, metadata: payload.metadata }, + data: { modelId, modelPath: payload.modelPath, tilesetFilename: payload.tilesetFilename, metadata: payload.metadata }, }); const stages: StageDescriptor[] = [ diff --git a/src/job/models/jobManager.ts b/src/job/models/jobManager.ts new file mode 100644 index 0000000..21949b3 --- /dev/null +++ b/src/job/models/jobManager.ts @@ -0,0 +1,26 @@ +import type { Logger } from '@map-colonies/js-logger'; +import { inject, injectable } from 'tsyringe'; +import { SERVICES } from '@common/constants'; +import type { JobStatusResponse, LogContext } from '@common/interfaces'; +import { JobnikClient } from '../../externalServices/jobnik/jobnikClient'; + +@injectable() +export class JobManager { + private readonly logContext: LogContext; + + public constructor( + @inject(SERVICES.LOGGER) private readonly logger: Logger, + @inject(JobnikClient) private readonly jobnik: JobnikClient + ) { + this.logContext = { + fileName: __filename, + class: JobManager.name, + }; + } + + public async getStatus(jobId: string): Promise { + const logContext = { ...this.logContext, function: this.getStatus.name }; + this.logger.info({ msg: 'fetching job status', logContext, jobId }); + return this.jobnik.getJobStatus(jobId); + } +} diff --git a/src/jobStatus/controllers/jobStatusController.ts b/src/jobStatus/controllers/jobStatusController.ts new file mode 100644 index 0000000..968b570 --- /dev/null +++ b/src/jobStatus/controllers/jobStatusController.ts @@ -0,0 +1,34 @@ +import type { Logger } from '@map-colonies/js-logger'; +import { StatusCodes } from 'http-status-codes'; +import { injectable, inject } from 'tsyringe'; +import type { TypedRequestHandlers } from '@openapi'; +import { SERVICES } from '@common/constants'; +import type { LogContext } from '@common/interfaces'; +import { JobManager } from '../../job/models/jobManager'; + +@injectable() +export class JobStatusController { + private readonly logContext: LogContext; + + public constructor( + @inject(SERVICES.LOGGER) private readonly logger: Logger, + @inject(JobManager) private readonly manager: JobManager + ) { + this.logContext = { + fileName: __filename, + class: JobStatusController.name, + }; + } + + public getJobStatus: TypedRequestHandlers['getJobStatus'] = async (req, res, next) => { + const logContext = { ...this.logContext, function: this.getJobStatus.name }; + const { jobId } = req.params; + try { + const status = await this.manager.getStatus(jobId); + return res.status(StatusCodes.OK).json(status); + } catch (err) { + this.logger.error({ msg: 'failed to get job status', logContext, err, jobId }); + return next(err); + } + }; +} diff --git a/src/jobStatus/routes/jobStatusRouter.ts b/src/jobStatus/routes/jobStatusRouter.ts new file mode 100644 index 0000000..9d3230a --- /dev/null +++ b/src/jobStatus/routes/jobStatusRouter.ts @@ -0,0 +1,16 @@ +import { Router } from 'express'; +import type { FactoryFunction } from 'tsyringe'; +import { JobStatusController } from '../controllers/jobStatusController'; + +const jobStatusRouterFactory: FactoryFunction = (dependencyContainer) => { + const router = Router(); + const controller = dependencyContainer.resolve(JobStatusController); + + router.get('/:jobId', controller.getJobStatus); + + return router; +}; + +export const JOB_STATUS_ROUTER_SYMBOL = Symbol('jobStatusRouterFactory'); + +export { jobStatusRouterFactory }; diff --git a/src/metadata/controllers/metadataController.ts b/src/metadata/controllers/metadataController.ts new file mode 100644 index 0000000..c7da02e --- /dev/null +++ b/src/metadata/controllers/metadataController.ts @@ -0,0 +1,46 @@ +import type { Logger } from '@map-colonies/js-logger'; +import { StatusCodes } from 'http-status-codes'; +import { injectable, inject } from 'tsyringe'; +import type { TypedRequestHandlers } from '@openapi'; +import { SERVICES } from '@common/constants'; +import type { LogContext } from '@common/interfaces'; +import { RecordManager } from '../../records/models/recordManager'; + +@injectable() +export class MetadataController { + private readonly logContext: LogContext; + + public constructor( + @inject(SERVICES.LOGGER) private readonly logger: Logger, + @inject(RecordManager) private readonly manager: RecordManager + ) { + this.logContext = { + fileName: __filename, + class: MetadataController.name, + }; + } + + public updateMetadata: TypedRequestHandlers['updateMetadata'] = async (req, res, next) => { + const logContext = { ...this.logContext, function: this.updateMetadata.name }; + const { identifier } = req.params; + try { + const ack = await this.manager.updateMetadata(identifier, req.body); + return res.status(StatusCodes.OK).json(ack); + } catch (err) { + this.logger.error({ msg: 'failed to update record metadata', logContext, err, recordId: identifier }); + return next(err); + } + }; + + public updateMetadataStatus: TypedRequestHandlers['updateMetadataStatus'] = async (req, res, next) => { + const logContext = { ...this.logContext, function: this.updateMetadataStatus.name }; + const { identifier } = req.params; + try { + const ack = await this.manager.updateStatus(identifier, req.body); + return res.status(StatusCodes.OK).json(ack); + } catch (err) { + this.logger.error({ msg: 'failed to update record status', logContext, err, recordId: identifier }); + return next(err); + } + }; +} diff --git a/src/metadata/routes/metadataRouter.ts b/src/metadata/routes/metadataRouter.ts new file mode 100644 index 0000000..c8ca7ed --- /dev/null +++ b/src/metadata/routes/metadataRouter.ts @@ -0,0 +1,17 @@ +import { Router } from 'express'; +import type { FactoryFunction } from 'tsyringe'; +import { MetadataController } from '../controllers/metadataController'; + +const metadataRouterFactory: FactoryFunction = (dependencyContainer) => { + const router = Router(); + const controller = dependencyContainer.resolve(MetadataController); + + router.patch('/status/:identifier', controller.updateMetadataStatus); + router.patch('/:identifier', controller.updateMetadata); + + return router; +}; + +export const METADATA_ROUTER_SYMBOL = Symbol('metadataRouterFactory'); + +export { metadataRouterFactory }; diff --git a/src/openapi.d.ts b/src/openapi.d.ts index ca8c610..07455f1 100644 --- a/src/openapi.d.ts +++ b/src/openapi.d.ts @@ -4,7 +4,7 @@ import type { TypedRequestHandlers as ImportedTypedRequestHandlers } from '@map-colonies/openapi-express-types'; export type paths = { - '/record': { + '/records': { parameters: { query?: never; header?: never; @@ -14,7 +14,7 @@ export type paths = { get?: never; put?: never; /** - * Start an ingestion process flow + * Invoke a new model ingestion flow * @description Validates the request (light & fast) and creates a Jobnik ingestion job. */ post: operations['createRecord']; @@ -24,7 +24,47 @@ export type paths = { patch?: never; trace?: never; }; - '/record/{id}': { + '/records/validate': { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** + * Invoke a new model ingestion validation + * @description Runs the full ingestion validation and returns the result without triggering a Jobnik job. + */ + post: operations['validateRecord']; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + '/records/canDelete/{recordId}': { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * Validate if a record can be removed + * @description Runs the delete validation and returns the result without creating a job. + */ + get: operations['canDeleteRecord']; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + '/records/{recordId}': { parameters: { query?: never; header?: never; @@ -35,17 +75,33 @@ export type paths = { put?: never; post?: never; /** - * Start a delete process flow + * Delete a model by record id * @description Validates the record can be deleted and creates a Jobnik delete job. */ delete: operations['deleteRecord']; options?: never; head?: never; - /** Update metadata for a record */ - patch: operations['updateRecord']; + patch?: never; + trace?: never; + }; + '/metadata/{identifier}': { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + /** Update metadata of a model */ + patch: operations['updateMetadata']; trace?: never; }; - '/record/status/{id}': { + '/metadata/status/{identifier}': { parameters: { query?: never; header?: never; @@ -58,8 +114,25 @@ export type paths = { delete?: never; options?: never; head?: never; - /** Update the publish/unpublish status of a record */ - patch: operations['updateRecordStatus']; + /** Update the publish/unpublish status of a model */ + patch: operations['updateMetadataStatus']; + trace?: never; + }; + '/jobStatus/{jobId}': { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** Get the status and progress of a job */ + get: operations['getJobStatus']; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; trace?: never; }; }; @@ -94,11 +167,23 @@ export type components = { ackResponse: { message: string; }; + jobStatusResponse: { + status: string; + percentage?: number; + }; + validationResultResponse: { + isValid: boolean; + message?: string; + }; }; responses: never; parameters: { + /** @description The record identifier */ + identifier: string; /** @description The record identifier */ recordId: string; + /** @description The job identifier */ + jobId: string; }; requestBodies: never; headers: never; @@ -139,13 +224,60 @@ export interface operations { }; }; }; + validateRecord: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody: { + content: { + 'application/json': components['schemas']['ingestionPayload']; + }; + }; + responses: { + /** @description Validation result */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + 'application/json': components['schemas']['validationResultResponse']; + }; + }; + }; + }; + canDeleteRecord: { + parameters: { + query?: never; + header?: never; + path: { + /** @description The record identifier */ + recordId: components['parameters']['recordId']; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Validation result */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + 'application/json': components['schemas']['validationResultResponse']; + }; + }; + }; + }; deleteRecord: { parameters: { query?: never; header?: never; path: { /** @description The record identifier */ - id: components['parameters']['recordId']; + recordId: components['parameters']['recordId']; }; cookie?: never; }; @@ -180,13 +312,13 @@ export interface operations { }; }; }; - updateRecord: { + updateMetadata: { parameters: { query?: never; header?: never; path: { /** @description The record identifier */ - id: components['parameters']['recordId']; + identifier: components['parameters']['identifier']; }; cookie?: never; }; @@ -225,13 +357,13 @@ export interface operations { }; }; }; - updateRecordStatus: { + updateMetadataStatus: { parameters: { query?: never; header?: never; path: { /** @description The record identifier */ - id: components['parameters']['recordId']; + identifier: components['parameters']['identifier']; }; cookie?: never; }; @@ -270,5 +402,37 @@ export interface operations { }; }; }; + getJobStatus: { + parameters: { + query?: never; + header?: never; + path: { + /** @description The job identifier */ + jobId: components['parameters']['jobId']; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Job status */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + 'application/json': components['schemas']['jobStatusResponse']; + }; + }; + /** @description Job not found */ + 404: { + headers: { + [name: string]: unknown; + }; + content: { + 'application/json': components['schemas']['error']; + }; + }; + }; + }; } export type TypedRequestHandlers = ImportedTypedRequestHandlers; diff --git a/src/record/models/recordManager.ts b/src/record/models/recordManager.ts deleted file mode 100644 index 37867d8..0000000 --- a/src/record/models/recordManager.ts +++ /dev/null @@ -1,55 +0,0 @@ -import type { Logger } from '@map-colonies/js-logger'; -import { inject, injectable } from 'tsyringe'; -import type { components } from '@openapi'; -import { SERVICES } from '@common/constants'; -import type { LogContext } from '@common/interfaces'; -import { ValidationManager } from '../../validator/validationManager'; -import { JobnikClient } from '../../externalServices/jobnik/jobnikClient'; - -export type IngestionPayload = components['schemas']['ingestionPayload']; -export type UpdatePayload = components['schemas']['updatePayload']; -export type StatusPayload = components['schemas']['statusPayload']; -export type JobResponse = components['schemas']['jobResponse']; -export type AckResponse = components['schemas']['ackResponse']; - -@injectable() -export class RecordManager { - private readonly logContext: LogContext; - - public constructor( - @inject(SERVICES.LOGGER) private readonly logger: Logger, - @inject(ValidationManager) private readonly validator: ValidationManager, - @inject(JobnikClient) private readonly jobnik: JobnikClient - ) { - this.logContext = { - fileName: __filename, - class: RecordManager.name, - }; - } - - public async createIngestion(payload: IngestionPayload): Promise { - const logContext = { ...this.logContext, function: this.createIngestion.name }; - this.logger.info({ msg: 'creating ingestion job', logContext, modelPath: payload.modelPath, tilesetFilename: payload.tilesetFilename }); - await this.validator.validateIngestion(payload); - return this.jobnik.createIngestionJob(payload); - } - - public async deleteRecord(id: string): Promise { - const logContext = { ...this.logContext, function: this.deleteRecord.name }; - this.logger.info({ msg: 'creating delete job', logContext, recordId: id }); - const record = await this.validator.validateDelete(id); - return this.jobnik.createDeleteJob(record); - } - - public updateMetadata(id: string, update: UpdatePayload): AckResponse { - const logContext = { ...this.logContext, function: this.updateMetadata.name }; - this.logger.info({ msg: 'updating record metadata', logContext, recordId: id, fields: Object.keys(update) }); - return { message: `metadata update accepted for record ${id}` }; - } - - public updateStatus(id: string, payload: StatusPayload): AckResponse { - const logContext = { ...this.logContext, function: this.updateStatus.name }; - this.logger.info({ msg: 'updating record status', logContext, recordId: id, status: payload.status }); - return { message: `status ${payload.status} accepted for record ${id}` }; - } -} diff --git a/src/record/routes/recordRouter.ts b/src/record/routes/recordRouter.ts deleted file mode 100644 index 4a5b495..0000000 --- a/src/record/routes/recordRouter.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { Router } from 'express'; -import type { FactoryFunction } from 'tsyringe'; -import { RecordController } from '../controllers/recordController'; - -const recordRouterFactory: FactoryFunction = (dependencyContainer) => { - const router = Router(); - const controller = dependencyContainer.resolve(RecordController); - - router.post('/', controller.createRecord); - router.patch('/status/:id', controller.updateRecordStatus); - router.delete('/:id', controller.deleteRecord); - router.patch('/:id', controller.updateRecord); - - return router; -}; - -export const RECORD_ROUTER_SYMBOL = Symbol('recordRouterFactory'); - -export { recordRouterFactory }; diff --git a/src/record/controllers/recordController.ts b/src/records/controllers/recordsController.ts similarity index 65% rename from src/record/controllers/recordController.ts rename to src/records/controllers/recordsController.ts index af0b6ee..7bca530 100644 --- a/src/record/controllers/recordController.ts +++ b/src/records/controllers/recordsController.ts @@ -8,7 +8,7 @@ import type { LogContext } from '@common/interfaces'; import { RecordManager } from '../models/recordManager'; @injectable() -export class RecordController { +export class RecordsController { private readonly logContext: LogContext; private readonly ingestionJobCounter: Counter; @@ -19,7 +19,7 @@ export class RecordController { ) { this.logContext = { fileName: __filename, - class: RecordController.name, + class: RecordsController.name, }; this.ingestionJobCounter = new Counter({ name: 'ingestion_jobs_created', @@ -40,38 +40,37 @@ export class RecordController { } }; - public deleteRecord: TypedRequestHandlers['deleteRecord'] = async (req, res, next) => { - const logContext = { ...this.logContext, function: this.deleteRecord.name }; - const { id } = req.params; + public validateRecord: TypedRequestHandlers['validateRecord'] = async (req, res, next) => { + const logContext = { ...this.logContext, function: this.validateRecord.name }; try { - const job = await this.manager.deleteRecord(id); - return res.status(StatusCodes.OK).json(job); + const result = await this.manager.validateIngestion(req.body); + return res.status(StatusCodes.OK).json(result); } catch (err) { - this.logger.error({ msg: 'failed to create delete job', logContext, err, recordId: id }); + this.logger.error({ msg: 'failed to validate ingestion request', logContext, err }); return next(err); } }; - public updateRecord: TypedRequestHandlers['updateRecord'] = (req, res, next) => { - const logContext = { ...this.logContext, function: this.updateRecord.name }; - const { id } = req.params; + public canDeleteRecord: TypedRequestHandlers['canDeleteRecord'] = async (req, res, next) => { + const logContext = { ...this.logContext, function: this.canDeleteRecord.name }; + const { recordId } = req.params; try { - const ack = this.manager.updateMetadata(id, req.body); - return res.status(StatusCodes.OK).json(ack); + const result = await this.manager.canDelete(recordId); + return res.status(StatusCodes.OK).json(result); } catch (err) { - this.logger.error({ msg: 'failed to update record metadata', logContext, err, recordId: id }); + this.logger.error({ msg: 'failed to validate record deletability', logContext, err, recordId }); return next(err); } }; - public updateRecordStatus: TypedRequestHandlers['updateRecordStatus'] = (req, res, next) => { - const logContext = { ...this.logContext, function: this.updateRecordStatus.name }; - const { id } = req.params; + public deleteRecord: TypedRequestHandlers['deleteRecord'] = async (req, res, next) => { + const logContext = { ...this.logContext, function: this.deleteRecord.name }; + const { recordId } = req.params; try { - const ack = this.manager.updateStatus(id, req.body); - return res.status(StatusCodes.OK).json(ack); + const job = await this.manager.deleteRecord(recordId); + return res.status(StatusCodes.OK).json(job); } catch (err) { - this.logger.error({ msg: 'failed to update record status', logContext, err, recordId: id }); + this.logger.error({ msg: 'failed to create delete job', logContext, err, recordId }); return next(err); } }; diff --git a/src/records/models/recordManager.ts b/src/records/models/recordManager.ts new file mode 100644 index 0000000..8fb11c8 --- /dev/null +++ b/src/records/models/recordManager.ts @@ -0,0 +1,128 @@ +import { randomUUID } from 'node:crypto'; +import type { Logger } from '@map-colonies/js-logger'; +import { inject, injectable } from 'tsyringe'; +import { trace } from '@opentelemetry/api'; +import { StatusCodes } from 'http-status-codes'; +import type { Polygon } from 'geojson'; +import { RecordStatus } from '@map-colonies/types'; +import { SERVICES } from '@common/constants'; +import type { + AckResponse, + IngestionPayload, + JobResponse, + LogContext, + StatusPayload, + UpdatePayload, + ValidationResultResponse, +} from '@common/interfaces'; +import { AppError } from '@common/appError'; +import { convertPolygonTo2DPolygon } from '@common/util'; +import { IngestionValidator } from '../../validator/ingestionValidator'; +import { MetadataValidator } from '../../validator/metadataValidator'; +import { JobnikClient } from '../../externalServices/jobnik/jobnikClient'; +import { CatalogCall } from '../../externalServices/catalog/catalogCall'; + +@injectable() +export class RecordManager { + private readonly logContext: LogContext; + + public constructor( + @inject(SERVICES.LOGGER) private readonly logger: Logger, + @inject(IngestionValidator) private readonly ingestionValidator: IngestionValidator, + @inject(MetadataValidator) private readonly metadataValidator: MetadataValidator, + @inject(JobnikClient) private readonly jobnik: JobnikClient, + @inject(CatalogCall) private readonly catalog: CatalogCall + ) { + this.logContext = { + fileName: __filename, + class: RecordManager.name, + }; + } + + public async createIngestion(payload: IngestionPayload): Promise { + const logContext = { ...this.logContext, function: this.createIngestion.name }; + const modelId = randomUUID(); + this.logger.info({ msg: 'creating ingestion job', logContext, modelId, modelPath: payload.modelPath, tilesetFilename: payload.tilesetFilename }); + trace.getActiveSpan()?.setAttribute('catalogId', modelId); + + try { + await this.ingestionValidator.validateIngestion(payload); + + const metadata = payload.metadata as Record; + if (metadata.footprint !== undefined) { + metadata.footprint = convertPolygonTo2DPolygon(metadata.footprint as Polygon); + } + + return await this.jobnik.createIngestionJob(payload, modelId); + } catch (err) { + if (err instanceof AppError) { + throw err; + } + this.logger.error({ msg: 'unexpected error while creating ingestion job', logContext, modelId, err }); + throw new AppError('error', StatusCodes.INTERNAL_SERVER_ERROR, String(err), true); + } + } + + public async validateIngestion(payload: IngestionPayload): Promise { + const logContext = { ...this.logContext, function: this.validateIngestion.name }; + this.logger.info({ msg: 'validating ingestion request', logContext, modelPath: payload.modelPath }); + try { + await this.ingestionValidator.validateIngestion(payload); + return { isValid: true }; + } catch (err) { + if (err instanceof AppError) { + return { isValid: false, message: err.message }; + } + throw err; + } + } + + public async canDelete(id: string): Promise { + const logContext = { ...this.logContext, function: this.canDelete.name }; + this.logger.info({ msg: 'validating record deletability', logContext, recordId: id }); + try { + await this.metadataValidator.validateDelete(id); + return { isValid: true }; + } catch (err) { + if (err instanceof AppError) { + return { isValid: false, message: err.message }; + } + throw err; + } + } + + public async deleteRecord(id: string): Promise { + const logContext = { ...this.logContext, function: this.deleteRecord.name }; + this.logger.info({ msg: 'creating delete job', logContext, recordId: id }); + const record = await this.metadataValidator.validateDelete(id); + const job = await this.jobnik.createDeleteJob(record); + await this.catalog.changeStatus(id, { productStatus: RecordStatus.BEING_DELETED }); + return job; + } + + public async updateMetadata(id: string, update: UpdatePayload): Promise { + const logContext = { ...this.logContext, function: this.updateMetadata.name }; + this.logger.info({ msg: 'updating record metadata', logContext, recordId: id }); + const record = await this.metadataValidator.validateUpdate(id, update); + await this.metadataValidator.ensureRecordAbsentFromExtractable(record); + + const payload: Record = { ...update }; + if (payload.footprint !== undefined) { + payload.footprint = convertPolygonTo2DPolygon(payload.footprint as Polygon); + } + await this.catalog.patchMetadata(id, payload); + + return { message: `metadata updated for record ${id}` }; + } + + public async updateStatus(id: string, payload: StatusPayload): Promise { + const logContext = { ...this.logContext, function: this.updateStatus.name }; + this.logger.info({ msg: 'updating record status', logContext, recordId: id, status: payload.status }); + const record = await this.metadataValidator.validateStatusChange(id); + await this.metadataValidator.ensureRecordAbsentFromExtractable(record); + + await this.catalog.changeStatus(id, { productStatus: payload.status }); + + return { message: `status ${payload.status} accepted for record ${id}` }; + } +} diff --git a/src/records/routes/recordsRouter.ts b/src/records/routes/recordsRouter.ts new file mode 100644 index 0000000..c22f629 --- /dev/null +++ b/src/records/routes/recordsRouter.ts @@ -0,0 +1,19 @@ +import { Router } from 'express'; +import type { FactoryFunction } from 'tsyringe'; +import { RecordsController } from '../controllers/recordsController'; + +const recordsRouterFactory: FactoryFunction = (dependencyContainer) => { + const router = Router(); + const controller = dependencyContainer.resolve(RecordsController); + + router.post('/', controller.createRecord); + router.post('/validate', controller.validateRecord); + router.get('/canDelete/:recordId', controller.canDeleteRecord); + router.delete('/:recordId', controller.deleteRecord); + + return router; +}; + +export const RECORDS_ROUTER_SYMBOL = Symbol('recordsRouterFactory'); + +export { recordsRouterFactory }; diff --git a/src/serverBuilder.ts b/src/serverBuilder.ts index 3e2fc98..900566d 100644 --- a/src/serverBuilder.ts +++ b/src/serverBuilder.ts @@ -10,7 +10,9 @@ import { collectMetricsExpressMiddleware } from '@map-colonies/prometheus'; import { Registry } from 'prom-client'; import type { ConfigType } from '@common/config'; import { SERVICES } from '@common/constants'; -import { RECORD_ROUTER_SYMBOL } from './record/routes/recordRouter'; +import { RECORDS_ROUTER_SYMBOL } from './records/routes/recordsRouter'; +import { JOB_STATUS_ROUTER_SYMBOL } from './jobStatus/routes/jobStatusRouter'; +import { METADATA_ROUTER_SYMBOL } from './metadata/routes/metadataRouter'; @injectable() export class ServerBuilder { @@ -20,7 +22,9 @@ export class ServerBuilder { @inject(SERVICES.CONFIG) private readonly config: ConfigType, @inject(SERVICES.LOGGER) private readonly logger: Logger, @inject(SERVICES.METRICS) private readonly metricsRegistry: Registry, - @inject(RECORD_ROUTER_SYMBOL) private readonly recordRouter: Router + @inject(RECORDS_ROUTER_SYMBOL) private readonly recordsRouter: Router, + @inject(JOB_STATUS_ROUTER_SYMBOL) private readonly jobStatusRouter: Router, + @inject(METADATA_ROUTER_SYMBOL) private readonly metadataRouter: Router ) { this.serverInstance = express(); } @@ -43,7 +47,9 @@ export class ServerBuilder { } private buildRoutes(): void { - this.serverInstance.use('/record', this.recordRouter); + this.serverInstance.use('/records', this.recordsRouter); + this.serverInstance.use('/jobStatus', this.jobStatusRouter); + this.serverInstance.use('/metadata', this.metadataRouter); this.buildDocsRoutes(); } diff --git a/src/validator/classification.ts b/src/validator/classification.ts new file mode 100644 index 0000000..91db638 --- /dev/null +++ b/src/validator/classification.ts @@ -0,0 +1,24 @@ +import type { Logger } from '@map-colonies/js-logger'; +import { StatusCodes } from 'http-status-codes'; +import type { LogContext } from '@common/interfaces'; +import { AppError } from '@common/appError'; +import type { LookupTablesCall } from '../externalServices/lookupTables/lookupTablesCall'; + +export const validateClassification = async ( + lookupTables: LookupTablesCall, + logger: Logger, + logContext: LogContext, + classification: string +): Promise => { + const classifications = await lookupTables.getClassifications(); + logger.debug({ msg: 'classification validation', logContext, classifications }); + + if (!classifications.includes(classification)) { + throw new AppError( + 'badRequest', + StatusCodes.BAD_REQUEST, + `classification is not a valid value. Optional values: ${classifications.join()}`, + true + ); + } +}; diff --git a/src/validator/errors.ts b/src/validator/errors.ts new file mode 100644 index 0000000..5817937 --- /dev/null +++ b/src/validator/errors.ts @@ -0,0 +1,18 @@ +export const ERROR_METADATA_DATE = 'imagingTimeBeginUTC must not be later than imagingTimeEndUTC'; +export const ERROR_METADATA_MISSING_DATE = 'imagingTimeBeginUTC and imagingTimeEndUTC are required'; +export const ERROR_METADATA_INVALID_DATE = 'imagingTimeBeginUTC and imagingTimeEndUTC must be valid dates'; +export const ERROR_METADATA_FOOTPRINT = 'Invalid footprint! Must be a GeoJSON Polygon or MultiPolygon with all-2D or all-3D coordinates'; +export const ERROR_METADATA_PRODUCT_NAME_UNIQUE = 'product name is not unique!'; +export const ERROR_PRODUCT_NAME_IN_FLIGHT = 'product name already has an in-flight ingestion job'; +export const ERROR_DELETE_RECORD_NOT_FOUND = "recordId doesn't match exactly one existing record"; +export const ERROR_DELETE_PRODUCT_TYPE = 'Cannot delete a record whose productType is "QuantizedMeshDTMBest"'; +export const ERROR_DELETE_STATUS = 'Cannot delete a record whose productStatus is not "UNPUBLISHED"'; +export const ERROR_FILE_NOT_FOUND = 'The model files do not exist in the agreed storage'; +export const ERROR_TILESET_INVALID = 'The tileset file is not a valid 3DTiles tileset'; +export const ERROR_FOOTPRINT_FAR_FROM_MODEL = "Wrong footprint! footprint's coordinates is not even close to the model!"; +export const ERROR_INTERSECTION_FAILED = 'An error caused during the validation of the intersection'; +export const ERROR_RECORD_NOT_FOUND = "record with the given identifier doesn't exist"; +export const ERROR_RECORD_BEING_DELETED = 'cannot change a record that is being deleted'; +export const ERROR_EXTRACTABLE_CONFLICT = 'the record exists in the extractable-management service and cannot be changed here'; +export const ERROR_MODEL_PATH_INVALID = 'Unknown model path! The model is not under the agreed base path'; +export const ERROR_PRODUCT_ID_EXISTS = 'a record with this productId already exists in the catalog'; diff --git a/src/validator/ingestionValidator.ts b/src/validator/ingestionValidator.ts new file mode 100644 index 0000000..7c4b0c7 --- /dev/null +++ b/src/validator/ingestionValidator.ts @@ -0,0 +1,138 @@ +import type { Logger } from '@map-colonies/js-logger'; +import { StatusCodes } from 'http-status-codes'; +import { inject, injectable } from 'tsyringe'; +import { new3DLayerMetadataSchema, geometrySchema } from '@map-colonies/3d-shared'; +import { SERVICES } from '@common/constants'; +import type { IngestionPayload, LogContext } from '@common/interfaces'; +import { AppError } from '@common/appError'; +import { buildModelFilePath } from '@common/util'; +import type { ConfigType } from '@common/config'; +import { LookupTablesCall } from '../externalServices/lookupTables/lookupTablesCall'; +import { CatalogCall } from '../externalServices/catalog/catalogCall'; +import { JobnikClient } from '../externalServices/jobnik/jobnikClient'; +import type { Provider } from '../providers/interfaces'; +import { TilesetValidator } from './tilesetValidator'; +import { validateClassification } from './classification'; +import { + ERROR_FILE_NOT_FOUND, + ERROR_METADATA_DATE, + ERROR_METADATA_FOOTPRINT, + ERROR_METADATA_INVALID_DATE, + ERROR_METADATA_MISSING_DATE, + ERROR_METADATA_PRODUCT_NAME_UNIQUE, + ERROR_MODEL_PATH_INVALID, + ERROR_PRODUCT_ID_EXISTS, + ERROR_PRODUCT_NAME_IN_FLIGHT, +} from './errors'; + +@injectable() +export class IngestionValidator { + private readonly logContext: LogContext; + private readonly basePath: string; + + public constructor( + @inject(SERVICES.CONFIG) private readonly config: ConfigType, + @inject(SERVICES.LOGGER) private readonly logger: Logger, + @inject(LookupTablesCall) private readonly lookupTables: LookupTablesCall, + @inject(CatalogCall) private readonly catalog: CatalogCall, + @inject(JobnikClient) private readonly jobnik: JobnikClient, + @inject(SERVICES.PROVIDER) private readonly provider: Provider, + @inject(TilesetValidator) private readonly tileset: TilesetValidator + ) { + this.basePath = this.config.get('validation.basePath'); + this.logContext = { + fileName: __filename, + class: IngestionValidator.name, + }; + } + + public async validateIngestion(payload: IngestionPayload): Promise { + const logContext = { ...this.logContext, function: this.validateIngestion.name }; + this.logger.info({ msg: 'ingestion validation start', logContext }); + + const metadata = payload.metadata; + const parsed = new3DLayerMetadataSchema.safeParse(metadata); + if (!parsed.success) { + const message = parsed.error.issues.map((issue) => `${issue.path.join('.')}: ${issue.message}`).join('; '); + throw new AppError('badRequest', StatusCodes.BAD_REQUEST, message, true); + } + + const footprintResult = geometrySchema.safeParse(metadata.footprint); + if (!footprintResult.success) { + throw new AppError('badRequest', StatusCodes.BAD_REQUEST, ERROR_METADATA_FOOTPRINT, true); + } + + this.validateDates(metadata.imagingTimeBeginUTC, metadata.imagingTimeEndUTC); + this.validateModelPath(payload.modelPath); + + await validateClassification(this.lookupTables, this.logger, logContext, parsed.data.classification); + await this.validateProductNameUnique(parsed.data.productName); + await this.validateProductNameNotInFlight(parsed.data.productName); + await this.validateProductIdUnique(parsed.data.productId); + await this.validateFileExists(payload.modelPath, payload.tilesetFilename); + await this.tileset.validateTileset(payload, footprintResult.data); + } + + private validateModelPath(modelPath: string): void { + if (!modelPath.startsWith(this.basePath)) { + throw new AppError('badRequest', StatusCodes.BAD_REQUEST, `${ERROR_MODEL_PATH_INVALID} (basePath: ${this.basePath})`, true); + } + } + + private async validateProductIdUnique(productId: string): Promise { + const logContext = { ...this.logContext, function: this.validateProductIdUnique.name }; + const records = await this.catalog.findRecords({ productId }); + this.logger.debug({ msg: 'product id uniqueness validation', logContext, productId, matches: records.length }); + + if (records.length > 0) { + throw new AppError('conflict', StatusCodes.CONFLICT, ERROR_PRODUCT_ID_EXISTS, true); + } + } + + private async validateFileExists(modelPath: string, tilesetFilename: string): Promise { + const logContext = { ...this.logContext, function: this.validateFileExists.name }; + const path = buildModelFilePath(modelPath, tilesetFilename); + const exists = await this.provider.fileExists(path); + this.logger.debug({ msg: 'file existence validation', logContext, path, exists }); + + if (!exists) { + throw new AppError('badRequest', StatusCodes.BAD_REQUEST, ERROR_FILE_NOT_FOUND, true); + } + } + + private async validateProductNameUnique(productName: string): Promise { + const logContext = { ...this.logContext, function: this.validateProductNameUnique.name }; + const records = await this.catalog.findRecords({ productName }); + this.logger.debug({ msg: 'product name uniqueness validation', logContext, productName, matches: records.length }); + + if (records.length > 0) { + throw new AppError('badRequest', StatusCodes.BAD_REQUEST, ERROR_METADATA_PRODUCT_NAME_UNIQUE, true); + } + } + + private async validateProductNameNotInFlight(productName: string): Promise { + const logContext = { ...this.logContext, function: this.validateProductNameNotInFlight.name }; + const inFlight = await this.jobnik.hasInFlightIngestionJob(productName); + this.logger.debug({ msg: 'in-flight duplicate validation', logContext, productName, inFlight }); + + if (inFlight) { + throw new AppError('conflict', StatusCodes.CONFLICT, ERROR_PRODUCT_NAME_IN_FLIGHT, true); + } + } + + private validateDates(start: unknown, end: unknown): void { + if (start === undefined || end === undefined) { + throw new AppError('badRequest', StatusCodes.BAD_REQUEST, ERROR_METADATA_MISSING_DATE, true); + } + + const startTime = new Date(start as string).getTime(); + const endTime = new Date(end as string).getTime(); + if (Number.isNaN(startTime) || Number.isNaN(endTime)) { + throw new AppError('badRequest', StatusCodes.BAD_REQUEST, ERROR_METADATA_INVALID_DATE, true); + } + + if (startTime > endTime) { + throw new AppError('badRequest', StatusCodes.BAD_REQUEST, ERROR_METADATA_DATE, true); + } + } +} diff --git a/src/validator/metadataValidator.ts b/src/validator/metadataValidator.ts new file mode 100644 index 0000000..35cb207 --- /dev/null +++ b/src/validator/metadataValidator.ts @@ -0,0 +1,127 @@ +import type { Logger } from '@map-colonies/js-logger'; +import { StatusCodes } from 'http-status-codes'; +import { inject, injectable } from 'tsyringe'; +import { new3DLayerMetadataSchema, geometrySchema } from '@map-colonies/3d-shared'; +import { ProductType, RecordStatus } from '@map-colonies/types'; +import { SERVICES } from '@common/constants'; +import type { LogContext, UpdatePayload } from '@common/interfaces'; +import { AppError } from '@common/appError'; +import { LookupTablesCall } from '../externalServices/lookupTables/lookupTablesCall'; +import { CatalogCall } from '../externalServices/catalog/catalogCall'; +import { ExtractableCall } from '../externalServices/extractableManagement/extractableCall'; +import type { Record3D } from '../externalServices/catalog/interfaces'; +import { validateClassification } from './classification'; +import { + ERROR_DELETE_PRODUCT_TYPE, + ERROR_DELETE_RECORD_NOT_FOUND, + ERROR_DELETE_STATUS, + ERROR_EXTRACTABLE_CONFLICT, + ERROR_METADATA_FOOTPRINT, + ERROR_METADATA_PRODUCT_NAME_UNIQUE, + ERROR_RECORD_BEING_DELETED, + ERROR_RECORD_NOT_FOUND, +} from './errors'; + +const BLOCKED_DELETE_PRODUCT_TYPE = ProductType.QUANTIZED_MESH_DTM_BEST; + +@injectable() +export class MetadataValidator { + private readonly logContext: LogContext; + + public constructor( + @inject(SERVICES.LOGGER) private readonly logger: Logger, + @inject(LookupTablesCall) private readonly lookupTables: LookupTablesCall, + @inject(CatalogCall) private readonly catalog: CatalogCall, + @inject(ExtractableCall) private readonly extractable: ExtractableCall + ) { + this.logContext = { + fileName: __filename, + class: MetadataValidator.name, + }; + } + + public async validateDelete(recordId: string): Promise { + const logContext = { ...this.logContext, function: this.validateDelete.name }; + const records = await this.catalog.findRecords({ id: recordId }); + this.logger.debug({ msg: 'delete validation', logContext, recordId, matches: records.length }); + + const [record] = records; + if (records.length !== 1 || record === undefined) { + throw new AppError('badRequest', StatusCodes.BAD_REQUEST, ERROR_DELETE_RECORD_NOT_FOUND, true); + } + + if (record.productType === BLOCKED_DELETE_PRODUCT_TYPE) { + throw new AppError('badRequest', StatusCodes.BAD_REQUEST, ERROR_DELETE_PRODUCT_TYPE, true); + } + + if (record.productStatus !== RecordStatus.UNPUBLISHED) { + throw new AppError('badRequest', StatusCodes.BAD_REQUEST, ERROR_DELETE_STATUS, true); + } + + return record; + } + + public async validateUpdate(identifier: string, payload: UpdatePayload): Promise { + const logContext = { ...this.logContext, function: this.validateUpdate.name }; + const record = await this.catalog.getRecord(identifier); + if (record === undefined) { + throw new AppError('notFound', StatusCodes.NOT_FOUND, ERROR_RECORD_NOT_FOUND, true); + } + if (record.productStatus === RecordStatus.BEING_DELETED) { + throw new AppError('badRequest', StatusCodes.BAD_REQUEST, ERROR_RECORD_BEING_DELETED, true); + } + + const parsed = new3DLayerMetadataSchema.partial().safeParse(payload); + if (!parsed.success) { + const message = parsed.error.issues.map((issue) => `${issue.path.join('.')}: ${issue.message}`).join('; '); + throw new AppError('badRequest', StatusCodes.BAD_REQUEST, message, true); + } + + const footprint = (payload as { footprint?: unknown }).footprint; + if (footprint !== undefined && !geometrySchema.safeParse(footprint).success) { + throw new AppError('badRequest', StatusCodes.BAD_REQUEST, ERROR_METADATA_FOOTPRINT, true); + } + + const data = parsed.data; + if (data.classification !== undefined) { + await validateClassification(this.lookupTables, this.logger, logContext, data.classification); + } + if (data.productName !== undefined) { + await this.validateProductNameUniqueExcept(data.productName, identifier); + } + + return record; + } + + public async validateStatusChange(identifier: string): Promise { + const record = await this.catalog.getRecord(identifier); + if (record === undefined) { + throw new AppError('notFound', StatusCodes.NOT_FOUND, ERROR_RECORD_NOT_FOUND, true); + } + if (record.productStatus === RecordStatus.BEING_DELETED) { + throw new AppError('badRequest', StatusCodes.BAD_REQUEST, ERROR_RECORD_BEING_DELETED, true); + } + + return record; + } + + public async ensureRecordAbsentFromExtractable(record: Record3D): Promise { + const logContext = { ...this.logContext, function: this.ensureRecordAbsentFromExtractable.name }; + if (record.productName === undefined) { + return; + } + + const exists = await this.extractable.isExtractableRecordExists(record.productName); + this.logger.debug({ msg: 'extractable existence validation', logContext, productName: record.productName, exists }); + if (exists) { + throw new AppError('conflict', StatusCodes.CONFLICT, ERROR_EXTRACTABLE_CONFLICT, true); + } + } + + private async validateProductNameUniqueExcept(productName: string, identifier: string): Promise { + const records = await this.catalog.findRecords({ productName }); + if (records.some((existing) => existing.id !== identifier)) { + throw new AppError('badRequest', StatusCodes.BAD_REQUEST, ERROR_METADATA_PRODUCT_NAME_UNIQUE, true); + } + } +} diff --git a/src/validator/tilesetValidator.ts b/src/validator/tilesetValidator.ts new file mode 100644 index 0000000..fed1fad --- /dev/null +++ b/src/validator/tilesetValidator.ts @@ -0,0 +1,98 @@ +import type { Logger } from '@map-colonies/js-logger'; +import { StatusCodes } from 'http-status-codes'; +import { inject, injectable } from 'tsyringe'; +import { calculatePolygonFromTileset, type TileSetJson } from '@map-colonies/3d-shared'; +import { area, feature, featureCollection, intersect, union } from '@turf/turf'; +import type { Feature, MultiPolygon, Polygon } from 'geojson'; +import { SERVICES } from '@common/constants'; +import type { IngestionPayload, LogContext, ValidationResponse } from '@common/interfaces'; +import { AppError } from '@common/appError'; +import type { ConfigType } from '@common/config'; +import { TilesetReader } from '../tileset/tilesetReader'; +import { ERROR_FOOTPRINT_FAR_FROM_MODEL, ERROR_INTERSECTION_FAILED, ERROR_TILESET_INVALID } from './errors'; + +const FULL_COVERAGE_PERCENT = 100; + +@injectable() +export class TilesetValidator { + private readonly logContext: LogContext; + private readonly percentageLimit: number; + + public constructor( + @inject(SERVICES.CONFIG) private readonly config: ConfigType, + @inject(SERVICES.LOGGER) private readonly logger: Logger, + @inject(TilesetReader) private readonly tilesetReader: TilesetReader + ) { + this.percentageLimit = this.config.get('validation.percentageLimit'); + this.logContext = { + fileName: __filename, + class: TilesetValidator.name, + }; + } + + public async validateTileset(payload: IngestionPayload, footprint: Polygon | MultiPolygon): Promise { + const content = await this.tilesetReader.readTilesetJson(payload.modelPath, payload.tilesetFilename); + + const { modelPolygon, response: polygonResponse } = this.getTilesetModelPolygon(content); + if (!polygonResponse.isValid || modelPolygon === undefined) { + this.rejectValidation(polygonResponse); + } + + const intersectionResponse = this.isFootprintAndModelIntersects(footprint, modelPolygon); + if (!intersectionResponse.isValid) { + this.rejectValidation(intersectionResponse); + } + } + + private getTilesetModelPolygon(content: string): { modelPolygon?: Polygon; response: ValidationResponse } { + const logContext = { ...this.logContext, function: this.getTilesetModelPolygon.name }; + try { + const tilesetJson = JSON.parse(content) as TileSetJson; + const modelPolygon = calculatePolygonFromTileset(tilesetJson); + this.logger.debug({ msg: 'extracted tileset model polygon', logContext, modelPolygon }); + + return { modelPolygon, response: { isValid: true } }; + } catch (err) { + const message = err instanceof Error ? err.message : ERROR_TILESET_INVALID; + this.logger.error({ msg: 'tileset polygon extraction failed', logContext, err }); + return { response: { isValid: false, message } }; + } + } + + private isFootprintAndModelIntersects(footprint: Polygon | MultiPolygon, modelPolygon: Polygon): ValidationResponse { + const logContext = { ...this.logContext, function: this.isFootprintAndModelIntersects.name }; + try { + const footprintFeature = feature(footprint); + const modelFeature: Feature = feature(modelPolygon); + + const intersection = intersect(featureCollection([footprintFeature, modelFeature])); + if (intersection === null) { + return { isValid: false, message: ERROR_FOOTPRINT_FAR_FROM_MODEL }; + } + + const combined = union(featureCollection([footprintFeature, modelFeature])); + const combinedArea = combined === null ? 0 : area(combined); + const coverage = combinedArea === 0 ? 0 : (FULL_COVERAGE_PERCENT * area(footprintFeature)) / combinedArea; + this.logger.debug({ msg: 'calculated footprint coverage of the model', logContext, coverage, percentageLimit: this.percentageLimit }); + + if (coverage < this.percentageLimit) { + return { + isValid: false, + message: `The footprint intersectection with the model doesn't reach minimum required threshhold, the coverage is: ${coverage}% when the minimum coverage is ${this.percentageLimit}%`, + }; + } + return { isValid: true }; + } catch (err) { + this.logger.error({ msg: ERROR_INTERSECTION_FAILED, logContext, err }); + return { isValid: false, message: ERROR_INTERSECTION_FAILED }; + } + } + + private rejectValidation(response: ValidationResponse): never { + const logContext = { ...this.logContext, function: this.rejectValidation.name }; + const message = response.message ?? ERROR_TILESET_INVALID; + this.logger.warn({ msg: 'ingestion validation failed', logContext, isValid: response.isValid, message }); + + throw new AppError('badRequest', StatusCodes.BAD_REQUEST, message, true); + } +} diff --git a/src/validator/validationManager.ts b/src/validator/validationManager.ts deleted file mode 100644 index 3dfdbf6..0000000 --- a/src/validator/validationManager.ts +++ /dev/null @@ -1,233 +0,0 @@ -import type { Logger } from '@map-colonies/js-logger'; -import { StatusCodes } from 'http-status-codes'; -import { inject, injectable } from 'tsyringe'; -import { new3DLayerMetadataSchema, geometrySchema, calculatePolygonFromTileset, type TileSetJson } from '@map-colonies/3d-shared'; -import { area, feature, featureCollection, intersect, union } from '@turf/turf'; -import type { Feature, MultiPolygon, Polygon } from 'geojson'; -import { SERVICES } from '@common/constants'; -import type { LogContext, ValidationResponse } from '@common/interfaces'; -import { AppError } from '@common/appError'; -import { buildModelFilePath } from '@common/util'; -import type { ConfigType } from '@common/config'; -import { LookupTablesCall } from '../externalServices/lookupTables/lookupTablesCall'; -import { CatalogCall } from '../externalServices/catalog/catalogCall'; -import { JobnikClient } from '../externalServices/jobnik/jobnikClient'; -import { TilesetReader } from '../tileset/tilesetReader'; -import type { Record3D } from '../externalServices/catalog/interfaces'; -import type { Provider } from '../providers/interfaces'; -import type { IngestionPayload } from '../record/models/recordManager'; - -const FULL_COVERAGE_PERCENT = 100; - -const BLOCKED_DELETE_PRODUCT_TYPE = 'QuantizedMeshDTMBest'; -const RECORD_STATUS_UNPUBLISHED = 'UNPUBLISHED'; - -export const ERROR_METADATA_DATE = 'imagingTimeBeginUTC must not be later than imagingTimeEndUTC'; -export const ERROR_METADATA_MISSING_DATE = 'imagingTimeBeginUTC and imagingTimeEndUTC are required'; -export const ERROR_METADATA_INVALID_DATE = 'imagingTimeBeginUTC and imagingTimeEndUTC must be valid dates'; -export const ERROR_METADATA_FOOTPRINT = 'Invalid footprint! Must be a GeoJSON Polygon or MultiPolygon with all-2D or all-3D coordinates'; -export const ERROR_METADATA_PRODUCT_NAME_UNIQUE = 'product name is not unique!'; -export const ERROR_PRODUCT_NAME_IN_FLIGHT = 'product name already has an in-flight ingestion job'; -export const ERROR_DELETE_RECORD_NOT_FOUND = "recordId doesn't match exactly one existing record"; -export const ERROR_DELETE_PRODUCT_TYPE = 'Cannot delete a record whose productType is "QuantizedMeshDTMBest"'; -export const ERROR_DELETE_STATUS = 'Cannot delete a record whose productStatus is not "UNPUBLISHED"'; -export const ERROR_FILE_NOT_FOUND = 'The model files do not exist in the agreed storage'; -export const ERROR_TILESET_INVALID = 'The tileset file is not a valid 3DTiles tileset'; -export const ERROR_FOOTPRINT_FAR_FROM_MODEL = "Wrong footprint! footprint's coordinates is not even close to the model!"; -export const ERROR_INTERSECTION_FAILED = 'An error caused during the validation of the intersection'; - -@injectable() -export class ValidationManager { - private readonly logContext: LogContext; - private readonly percentageLimit: number; - - public constructor( - @inject(SERVICES.CONFIG) private readonly config: ConfigType, - @inject(SERVICES.LOGGER) private readonly logger: Logger, - @inject(LookupTablesCall) private readonly lookupTables: LookupTablesCall, - @inject(CatalogCall) private readonly catalog: CatalogCall, - @inject(JobnikClient) private readonly jobnik: JobnikClient, - @inject(TilesetReader) private readonly tilesetReader: TilesetReader, - @inject(SERVICES.PROVIDER) private readonly provider: Provider - ) { - this.percentageLimit = this.config.get('validation.percentageLimit'); - this.logContext = { - fileName: __filename, - class: ValidationManager.name, - }; - } - - public async validateIngestion(payload: IngestionPayload): Promise { - const logContext = { ...this.logContext, function: this.validateIngestion.name }; - this.logger.info({ msg: 'ingestion validation start', logContext }); - - const metadata = payload.metadata; - const parsed = new3DLayerMetadataSchema.safeParse(metadata); - if (!parsed.success) { - const message = parsed.error.issues.map((issue) => `${issue.path.join('.')}: ${issue.message}`).join('; '); - throw new AppError('badRequest', StatusCodes.BAD_REQUEST, message, true); - } - - const footprintResult = geometrySchema.safeParse(metadata.footprint); - if (!footprintResult.success) { - throw new AppError('badRequest', StatusCodes.BAD_REQUEST, ERROR_METADATA_FOOTPRINT, true); - } - - this.validateDates(metadata.imagingTimeBeginUTC, metadata.imagingTimeEndUTC); - - await this.validateClassification(parsed.data.classification); - await this.validateProductNameUnique(parsed.data.productName); - await this.validateProductNameNotInFlight(parsed.data.productName); - await this.validateFileExists(payload.modelPath, payload.tilesetFilename); - await this.validateTileset(payload, footprintResult.data); - } - - public async validateDelete(recordId: string): Promise { - const logContext = { ...this.logContext, function: this.validateDelete.name }; - const records = await this.catalog.findRecords({ id: recordId }); - this.logger.debug({ msg: 'delete validation', logContext, recordId, matches: records.length }); - - const [record] = records; - if (records.length !== 1 || record === undefined) { - throw new AppError('badRequest', StatusCodes.BAD_REQUEST, ERROR_DELETE_RECORD_NOT_FOUND, true); - } - - if (record.productType === BLOCKED_DELETE_PRODUCT_TYPE) { - throw new AppError('badRequest', StatusCodes.BAD_REQUEST, ERROR_DELETE_PRODUCT_TYPE, true); - } - - if (record.productStatus !== RECORD_STATUS_UNPUBLISHED) { - throw new AppError('badRequest', StatusCodes.BAD_REQUEST, ERROR_DELETE_STATUS, true); - } - - return record; - } - - private async validateFileExists(modelPath: string, tilesetFilename: string): Promise { - const logContext = { ...this.logContext, function: this.validateFileExists.name }; - const path = buildModelFilePath(modelPath, tilesetFilename); - const exists = await this.provider.fileExists(path); - this.logger.debug({ msg: 'file existence validation', logContext, path, exists }); - - if (!exists) { - throw new AppError('badRequest', StatusCodes.BAD_REQUEST, ERROR_FILE_NOT_FOUND, true); - } - } - - private async validateProductNameUnique(productName: string): Promise { - const logContext = { ...this.logContext, function: this.validateProductNameUnique.name }; - const records = await this.catalog.findRecords({ productName }); - this.logger.debug({ msg: 'product name uniqueness validation', logContext, productName, matches: records.length }); - - if (records.length > 0) { - throw new AppError('badRequest', StatusCodes.BAD_REQUEST, ERROR_METADATA_PRODUCT_NAME_UNIQUE, true); - } - } - - private async validateProductNameNotInFlight(productName: string): Promise { - const logContext = { ...this.logContext, function: this.validateProductNameNotInFlight.name }; - const inFlight = await this.jobnik.hasInFlightIngestionJob(productName); - this.logger.debug({ msg: 'in-flight duplicate validation', logContext, productName, inFlight }); - - if (inFlight) { - throw new AppError('conflict', StatusCodes.CONFLICT, ERROR_PRODUCT_NAME_IN_FLIGHT, true); - } - } - - private async validateTileset(payload: IngestionPayload, footprint: Polygon | MultiPolygon): Promise { - const content = await this.tilesetReader.readTilesetJson(payload.modelPath, payload.tilesetFilename); - - const { modelPolygon, response: polygonResponse } = this.getTilesetModelPolygon(content); - if (!polygonResponse.isValid || modelPolygon === undefined) { - this.rejectValidation(polygonResponse); - } - - const intersectionResponse = this.isFootprintAndModelIntersects(footprint, modelPolygon); - if (!intersectionResponse.isValid) { - this.rejectValidation(intersectionResponse); - } - } - - private getTilesetModelPolygon(content: string): { modelPolygon?: Polygon; response: ValidationResponse } { - const logContext = { ...this.logContext, function: this.getTilesetModelPolygon.name }; - try { - const tilesetJson = JSON.parse(content) as TileSetJson; - const modelPolygon = calculatePolygonFromTileset(tilesetJson); - this.logger.debug({ msg: 'extracted tileset model polygon', logContext, modelPolygon }); - - return { modelPolygon, response: { isValid: true } }; - } catch (err) { - const message = err instanceof Error ? err.message : ERROR_TILESET_INVALID; - this.logger.error({ msg: 'tileset polygon extraction failed', logContext, err }); - return { response: { isValid: false, message } }; - } - } - - private isFootprintAndModelIntersects(footprint: Polygon | MultiPolygon, modelPolygon: Polygon): ValidationResponse { - const logContext = { ...this.logContext, function: this.isFootprintAndModelIntersects.name }; - try { - const footprintFeature = feature(footprint); - const modelFeature: Feature = feature(modelPolygon); - - const intersection = intersect(featureCollection([footprintFeature, modelFeature])); - if (intersection === null) { - return { isValid: false, message: ERROR_FOOTPRINT_FAR_FROM_MODEL }; - } - - const combined = union(featureCollection([footprintFeature, modelFeature])); - const combinedArea = combined === null ? 0 : area(combined); - const coverage = combinedArea === 0 ? 0 : (FULL_COVERAGE_PERCENT * area(footprintFeature)) / combinedArea; - this.logger.debug({ msg: 'calculated footprint coverage of the model', logContext, coverage, percentageLimit: this.percentageLimit }); - - if (coverage < this.percentageLimit) { - return { - isValid: false, - message: `The footprint intersectection with the model doesn't reach minimum required threshhold, the coverage is: ${coverage}% when the minimum coverage is ${this.percentageLimit}%`, - }; - } - return { isValid: true }; - } catch (err) { - this.logger.error({ msg: ERROR_INTERSECTION_FAILED, logContext, err }); - return { isValid: false, message: ERROR_INTERSECTION_FAILED }; - } - } - - private rejectValidation(response: ValidationResponse): never { - const logContext = { ...this.logContext, function: this.rejectValidation.name }; - const message = response.message ?? ERROR_TILESET_INVALID; - this.logger.warn({ msg: 'ingestion validation failed', logContext, isValid: response.isValid, message }); - - throw new AppError('badRequest', StatusCodes.BAD_REQUEST, message, true); - } - - private validateDates(start: unknown, end: unknown): void { - if (start === undefined || end === undefined) { - throw new AppError('badRequest', StatusCodes.BAD_REQUEST, ERROR_METADATA_MISSING_DATE, true); - } - - const startTime = new Date(start as string).getTime(); - const endTime = new Date(end as string).getTime(); - if (Number.isNaN(startTime) || Number.isNaN(endTime)) { - throw new AppError('badRequest', StatusCodes.BAD_REQUEST, ERROR_METADATA_INVALID_DATE, true); - } - - if (startTime > endTime) { - throw new AppError('badRequest', StatusCodes.BAD_REQUEST, ERROR_METADATA_DATE, true); - } - } - - private async validateClassification(classification: string): Promise { - const logContext = { ...this.logContext, function: this.validateClassification.name }; - const classifications = await this.lookupTables.getClassifications(); - this.logger.debug({ msg: 'classification validation', logContext, classifications }); - - if (!classifications.includes(classification)) { - throw new AppError( - 'badRequest', - StatusCodes.BAD_REQUEST, - `classification is not a valid value. Optional values: ${classifications.join()}`, - true - ); - } - } -} diff --git a/tests/integration/record/record.spec.ts b/tests/integration/record/record.spec.ts index dcdbf34..64e7ad5 100644 --- a/tests/integration/record/record.spec.ts +++ b/tests/integration/record/record.spec.ts @@ -13,6 +13,7 @@ import { LookupTablesCall } from '@src/externalServices/lookupTables/lookupTable import { CatalogCall } from '@src/externalServices/catalog/catalogCall'; import { JobnikClient } from '@src/externalServices/jobnik/jobnikClient'; import { TilesetReader } from '@src/tileset/tilesetReader'; +import { ExtractableCall } from '@src/externalServices/extractableManagement/extractableCall'; import { buildValidMetadata } from '@tests/helpers/metadata'; const regionTilesetJson = readFileSync(join(__dirname, '../../helpers/tilesets/folder/tileset.json'), 'utf-8'); @@ -28,27 +29,31 @@ const deletableRecord = { }; const lookupStub = { getClassifications: vi.fn().mockResolvedValue(['abc123']) } as unknown as LookupTablesCall; -// findRecords by id (delete) → an existing deletable record; by productName (uniqueness) → none const catalogStub = { findRecords: vi.fn().mockImplementation((payload: { id?: string }) => (payload.id !== undefined ? [deletableRecord] : [])), + getRecord: vi.fn().mockResolvedValue(deletableRecord), + patchMetadata: vi.fn().mockResolvedValue(deletableRecord), + changeStatus: vi.fn().mockResolvedValue(deletableRecord), } as unknown as CatalogCall; const jobnikStub = { createIngestionJob: vi.fn().mockResolvedValue({ jobId: 'job-1', status: 'PENDING' }), createDeleteJob: vi.fn().mockResolvedValue({ jobId: 'del-1', status: 'PENDING' }), hasInFlightIngestionJob: vi.fn().mockResolvedValue(false), + getJobStatus: vi.fn().mockResolvedValue({ status: 'IN_PROGRESS', percentage: 42 }), } as unknown as JobnikClient; const tilesetReaderStub = { readTilesetJson: vi.fn().mockResolvedValue(regionTilesetJson) } as unknown as TilesetReader; +const extractableStub = { isExtractableRecordExists: vi.fn().mockResolvedValue(false) } as unknown as ExtractableCall; const providerStub = { fileExists: vi.fn().mockResolvedValue(true) }; const validMetadata = buildValidMetadata(); const validIngestionPayload = { - modelPath: '/shared/models/afula', + modelPath: '/app/models/afula', tilesetFilename: 'tileset.json', metadata: validMetadata, }; -describe('record', function () { +describe('3d-ops-trigger', function () { let requestSender: RequestSender; beforeAll(async function () { @@ -64,6 +69,7 @@ describe('record', function () { { token: CatalogCall, provider: { useValue: catalogStub } }, { token: JobnikClient, provider: { useValue: jobnikStub } }, { token: TilesetReader, provider: { useValue: tilesetReaderStub } }, + { token: ExtractableCall, provider: { useValue: extractableStub } }, { token: SERVICES.PROVIDER, provider: { useValue: providerStub } }, ], useChild: true, @@ -71,14 +77,14 @@ describe('record', function () { requestSender = await createRequestSender('openapi3.yaml', app); }); - describe('POST /record', function () { + describe('POST /records', function () { it('should return 201 and a job response for a valid ingestion request', async function () { const response = await requestSender.createRecord({ requestBody: validIngestionPayload }); expect(response).toSatisfyApiSpec(); expect(response.status).toBe(httpStatusCodes.CREATED); - const body = response.body as paths['/record']['post']['responses']['201']['content']['application/json']; + const body = response.body as paths['/records']['post']['responses']['201']['content']['application/json']; expect(body.jobId).toBeTypeOf('string'); expect(body.status).toBeTypeOf('string'); @@ -87,7 +93,7 @@ describe('record', function () { it('should return 400 when a required field is missing', async function () { const response = await requestSender.createRecord({ // @ts-expect-error intentionally invalid: missing tilesetFilename and metadata - requestBody: { modelPath: '/shared/models/afula' }, + requestBody: { modelPath: '/app/models/afula' }, }); expect(response).toSatisfyApiSpec(); @@ -104,22 +110,62 @@ describe('record', function () { }); }); - describe('DELETE /record/{id}', function () { + describe('POST /records/validate', function () { + it('should return 200 with isValid true for a valid request without creating a job', async function () { + const response = await requestSender.validateRecord({ requestBody: validIngestionPayload }); + + expect(response).toSatisfyApiSpec(); + expect(response.status).toBe(httpStatusCodes.OK); + + const body = response.body; + + expect(body.isValid).toBe(true); + }); + + it('should return 200 with isValid false and a message for an invalid request', async function () { + const response = await requestSender.validateRecord({ + requestBody: { ...validIngestionPayload, metadata: { ...validMetadata, productType: 'NOT_A_3D_TYPE' } }, + }); + + expect(response).toSatisfyApiSpec(); + expect(response.status).toBe(httpStatusCodes.OK); + + const body = response.body; + + expect(body.isValid).toBe(false); + expect(body.message).toBeTypeOf('string'); + }); + }); + + describe('GET /records/canDelete/{recordId}', function () { + it('should return 200 with isValid true for a deletable record', async function () { + const response = await requestSender.canDeleteRecord({ pathParams: { recordId: 'rec-1' } }); + + expect(response).toSatisfyApiSpec(); + expect(response.status).toBe(httpStatusCodes.OK); + + const body = response.body; + + expect(body.isValid).toBe(true); + }); + }); + + describe('DELETE /records/{recordId}', function () { it('should return 200 and a job response', async function () { - const response = await requestSender.deleteRecord({ pathParams: { id: 'rec-1' } }); + const response = await requestSender.deleteRecord({ pathParams: { recordId: 'rec-1' } }); expect(response).toSatisfyApiSpec(); expect(response.status).toBe(httpStatusCodes.OK); - const body = response.body as paths['/record/{id}']['delete']['responses']['200']['content']['application/json']; + const body = response.body as paths['/records/{recordId}']['delete']['responses']['200']['content']['application/json']; expect(body.jobId).toBeTypeOf('string'); }); }); - describe('PATCH /record/{id}', function () { + describe('PATCH /metadata/{identifier}', function () { it('should return 200 and an ack for a metadata update', async function () { - const response = await requestSender.updateRecord({ pathParams: { id: 'rec-1' }, requestBody: { description: 'updated' } }); + const response = await requestSender.updateMetadata({ pathParams: { identifier: 'rec-1' }, requestBody: { description: 'updated' } }); expect(response).toSatisfyApiSpec(); expect(response.status).toBe(httpStatusCodes.OK); @@ -127,17 +173,17 @@ describe('record', function () { }); }); - describe('PATCH /record/status/{id}', function () { + describe('PATCH /metadata/status/{identifier}', function () { it('should return 200 and an ack for a valid status change', async function () { - const response = await requestSender.updateRecordStatus({ pathParams: { id: 'rec-1' }, requestBody: { status: 'PUBLISHED' } }); + const response = await requestSender.updateMetadataStatus({ pathParams: { identifier: 'rec-1' }, requestBody: { status: 'PUBLISHED' } }); expect(response).toSatisfyApiSpec(); expect(response.status).toBe(httpStatusCodes.OK); }); it('should return 400 for an invalid status value', async function () { - const response = await requestSender.updateRecordStatus({ - pathParams: { id: 'rec-1' }, + const response = await requestSender.updateMetadataStatus({ + pathParams: { identifier: 'rec-1' }, // @ts-expect-error intentionally invalid status enum value requestBody: { status: 'NOT_A_STATUS' }, }); @@ -146,4 +192,18 @@ describe('record', function () { expect(response.status).toBe(httpStatusCodes.BAD_REQUEST); }); }); + + describe('GET /jobStatus/{jobId}', function () { + it('should return 200 with the job status and percentage', async function () { + const response = await requestSender.getJobStatus({ pathParams: { jobId: 'job-1' } }); + + expect(response).toSatisfyApiSpec(); + expect(response.status).toBe(httpStatusCodes.OK); + + const body = response.body as paths['/jobStatus/{jobId}']['get']['responses']['200']['content']['application/json']; + + expect(body.status).toBeTypeOf('string'); + expect(body.percentage).toBe(42); + }); + }); }); diff --git a/tests/unit/externalServices/jobnik/jobnikClient.spec.ts b/tests/unit/externalServices/jobnik/jobnikClient.spec.ts index 395cc7e..ccb0fed 100644 --- a/tests/unit/externalServices/jobnik/jobnikClient.spec.ts +++ b/tests/unit/externalServices/jobnik/jobnikClient.spec.ts @@ -4,7 +4,7 @@ import { Registry } from 'prom-client'; import { JobnikClient } from '@src/externalServices/jobnik/jobnikClient'; import { STAGE_TYPES } from '@src/common/constants'; import type { ConfigType } from '@src/common/config'; -import type { IngestionPayload } from '@src/record/models/recordManager'; +import type { IngestionPayload } from '@src/common/interfaces'; const jobManagerConfig = { url: 'http://job-manager', @@ -56,9 +56,12 @@ describe('JobnikClient', function () { }); it('should create the ingestion job and a validation stage for a folder input', async function () { - const result = await client.createIngestionJob(buildPayload('/shared/models/afula')); + const result = await client.createIngestionJob(buildPayload('/shared/models/afula'), 'model-1'); - expect(producerMock.createJob).toHaveBeenCalledWith(expect.objectContaining({ name: jobManagerConfig.ingestion.jobType })); + const createJobArg = (producerMock.createJob.mock.calls as [{ name: string; data: { modelId: string } }][])[0][0]; + + expect(createJobArg.name).toBe(jobManagerConfig.ingestion.jobType); + expect(createJobArg.data.modelId).toBe('model-1'); const folderStageTypes = (producerMock.createStage.mock.calls as [string, { type: string }][]).map((call) => call[1].type); @@ -79,7 +82,7 @@ describe('JobnikClient', function () { }); it('should also create the data-extraction and clear-data stages for a 3tz input', async function () { - await client.createIngestionJob(buildPayload('/shared/models/afula.3tz')); + await client.createIngestionJob(buildPayload('/shared/models/afula.3tz'), 'model-1'); const stageTypes = (producerMock.createStage.mock.calls as [string, { type: string }][]).map((call) => call[1].type); diff --git a/tests/unit/record/models/recordManager.spec.ts b/tests/unit/record/models/recordManager.spec.ts deleted file mode 100644 index 403dc42..0000000 --- a/tests/unit/record/models/recordManager.spec.ts +++ /dev/null @@ -1,62 +0,0 @@ -import { jsLogger } from '@map-colonies/js-logger'; -import { describe, it, expect, beforeEach, vi } from 'vitest'; -import { RecordManager, type IngestionPayload } from '@src/record/models/recordManager'; -import type { ValidationManager } from '@src/validator/validationManager'; -import type { JobnikClient } from '@src/externalServices/jobnik/jobnikClient'; - -const noopValidator = { - validateIngestion: vi.fn().mockResolvedValue(undefined), - validateDelete: vi.fn().mockResolvedValue({ id: 'rec-1', productName: 'afula' }), -} as unknown as ValidationManager; -const jobnikStub = { - createIngestionJob: vi.fn().mockResolvedValue({ jobId: 'job-1', status: 'PENDING' }), - createDeleteJob: vi.fn().mockResolvedValue({ jobId: 'del-1', status: 'PENDING' }), -} as unknown as JobnikClient; - -describe('RecordManager', function () { - let manager: RecordManager; - - beforeEach(async function () { - manager = new RecordManager(await jsLogger({ enabled: false }), noopValidator, jobnikStub); - }); - - describe('createIngestion', function () { - it('should return a job response with a jobId and status', async function () { - const payload: IngestionPayload = { - modelPath: '/shared/models/afula', - tilesetFilename: 'tileset.json', - metadata: { productName: 'afula' }, - }; - - const result = await manager.createIngestion(payload); - - expect(result.jobId).toBeTypeOf('string'); - expect(result.status).toBeTypeOf('string'); - }); - }); - - describe('deleteRecord', function () { - it('should return a job response', async function () { - const result = await manager.deleteRecord('rec-1'); - - expect(result.jobId).toBeTypeOf('string'); - expect(result.status).toBeTypeOf('string'); - }); - }); - - describe('updateMetadata', function () { - it('should return an ack referencing the record id', function () { - const result = manager.updateMetadata('rec-1', { description: 'x' }); - - expect(result.message).toContain('rec-1'); - }); - }); - - describe('updateStatus', function () { - it('should return an ack referencing the requested status', function () { - const result = manager.updateStatus('rec-1', { status: 'PUBLISHED' }); - - expect(result.message).toContain('PUBLISHED'); - }); - }); -}); diff --git a/tests/unit/records/models/recordManager.spec.ts b/tests/unit/records/models/recordManager.spec.ts new file mode 100644 index 0000000..e2dc26b --- /dev/null +++ b/tests/unit/records/models/recordManager.spec.ts @@ -0,0 +1,79 @@ +import { jsLogger } from '@map-colonies/js-logger'; +import { describe, it, expect, beforeEach, vi } from 'vitest'; +import { RecordManager } from '@src/records/models/recordManager'; +import type { IngestionPayload } from '@src/common/interfaces'; +import type { IngestionValidator } from '@src/validator/ingestionValidator'; +import type { MetadataValidator } from '@src/validator/metadataValidator'; +import type { JobnikClient } from '@src/externalServices/jobnik/jobnikClient'; +import type { CatalogCall } from '@src/externalServices/catalog/catalogCall'; + +const ingestionValidatorStub = { + validateIngestion: vi.fn().mockResolvedValue(undefined), +} as unknown as IngestionValidator; +const metadataValidatorStub = { + validateDelete: vi.fn().mockResolvedValue({ id: 'rec-1', productName: 'afula' }), + validateUpdate: vi.fn().mockResolvedValue({ id: 'rec-1', productName: 'afula' }), + validateStatusChange: vi.fn().mockResolvedValue({ id: 'rec-1', productName: 'afula' }), + ensureRecordAbsentFromExtractable: vi.fn().mockResolvedValue(undefined), +} as unknown as MetadataValidator; +const jobnikStub = { + createIngestionJob: vi.fn().mockResolvedValue({ jobId: 'job-1', status: 'PENDING' }), + createDeleteJob: vi.fn().mockResolvedValue({ jobId: 'del-1', status: 'PENDING' }), +} as unknown as JobnikClient; +const patchMetadataMock = vi.fn().mockResolvedValue({ id: 'rec-1' }); +const changeStatusMock = vi.fn().mockResolvedValue({ id: 'rec-1' }); +const catalogStub = { + patchMetadata: patchMetadataMock, + changeStatus: changeStatusMock, +} as unknown as CatalogCall; + +describe('RecordManager', function () { + let manager: RecordManager; + + beforeEach(async function () { + manager = new RecordManager(await jsLogger({ enabled: false }), ingestionValidatorStub, metadataValidatorStub, jobnikStub, catalogStub); + }); + + describe('createIngestion', function () { + it('should return a job response with a jobId and status', async function () { + const payload: IngestionPayload = { + modelPath: '/shared/models/afula', + tilesetFilename: 'tileset.json', + metadata: { productName: 'afula' }, + }; + + const result = await manager.createIngestion(payload); + + expect(result.jobId).toBeTypeOf('string'); + expect(result.status).toBeTypeOf('string'); + }); + }); + + describe('deleteRecord', function () { + it('should create the delete job and set the catalog status to BEING_DELETED', async function () { + const result = await manager.deleteRecord('rec-1'); + + expect(result.jobId).toBeTypeOf('string'); + expect(result.status).toBeTypeOf('string'); + expect(changeStatusMock).toHaveBeenCalledWith('rec-1', { productStatus: 'BEING_DELETED' }); + }); + }); + + describe('updateMetadata', function () { + it('should validate, guard extractable, patch the catalog and return an ack', async function () { + const result = await manager.updateMetadata('rec-1', { description: 'x' }); + + expect(patchMetadataMock).toHaveBeenCalledWith('rec-1', expect.objectContaining({ description: 'x' })); + expect(result.message).toContain('rec-1'); + }); + }); + + describe('updateStatus', function () { + it('should validate, guard extractable, change the catalog status and return an ack', async function () { + const result = await manager.updateStatus('rec-1', { status: 'PUBLISHED' }); + + expect(changeStatusMock).toHaveBeenCalledWith('rec-1', { productStatus: 'PUBLISHED' }); + expect(result.message).toContain('PUBLISHED'); + }); + }); +}); diff --git a/tests/unit/validator/validationManager.spec.ts b/tests/unit/validator/ingestionValidator.spec.ts similarity index 77% rename from tests/unit/validator/validationManager.spec.ts rename to tests/unit/validator/ingestionValidator.spec.ts index 18ef9aa..2c4da59 100644 --- a/tests/unit/validator/validationManager.spec.ts +++ b/tests/unit/validator/ingestionValidator.spec.ts @@ -3,19 +3,18 @@ import { join } from 'node:path'; import { jsLogger } from '@map-colonies/js-logger'; import { describe, it, expect, beforeEach, vi } from 'vitest'; import { StatusCodes } from 'http-status-codes'; +import { IngestionValidator } from '@src/validator/ingestionValidator'; +import { TilesetValidator } from '@src/validator/tilesetValidator'; import { - ValidationManager, ERROR_METADATA_DATE, ERROR_METADATA_FOOTPRINT, ERROR_METADATA_MISSING_DATE, ERROR_METADATA_INVALID_DATE, - ERROR_DELETE_RECORD_NOT_FOUND, - ERROR_DELETE_PRODUCT_TYPE, - ERROR_DELETE_STATUS, ERROR_FILE_NOT_FOUND, ERROR_PRODUCT_NAME_IN_FLIGHT, ERROR_FOOTPRINT_FAR_FROM_MODEL, -} from '@src/validator/validationManager'; + ERROR_MODEL_PATH_INVALID, +} from '@src/validator/errors'; import { AppError } from '@src/common/appError'; import type { ConfigType } from '@src/common/config'; import type { LookupTablesCall } from '@src/externalServices/lookupTables/lookupTablesCall'; @@ -23,14 +22,16 @@ import type { CatalogCall } from '@src/externalServices/catalog/catalogCall'; import type { JobnikClient } from '@src/externalServices/jobnik/jobnikClient'; import type { TilesetReader } from '@src/tileset/tilesetReader'; import type { Provider } from '@src/providers/interfaces'; -import type { IngestionPayload } from '@src/record/models/recordManager'; +import type { IngestionPayload } from '@src/common/interfaces'; import { buildValidMetadata } from '@tests/helpers/metadata'; const regionTilesetJson = readFileSync(join(__dirname, '../../helpers/tilesets/folder/tileset.json'), 'utf-8'); const boxTilesetJson = JSON.stringify({ root: { boundingVolume: { box: [0, 0, 0, 100, 0, 0, 0, 100, 0, 0, 0, 100] } } }); const lookupStub = { getClassifications: vi.fn().mockResolvedValue(['abc123']) } as unknown as LookupTablesCall; -const configStub = { get: vi.fn().mockReturnValue(10) } as unknown as ConfigType; +const configStub = { + get: vi.fn((key: string) => (key === 'validation.basePath' ? '/shared/models' : 10)), +} as unknown as ConfigType; const ingest = (metadata: Record): IngestionPayload => ({ modelPath: '/shared/models/afula', @@ -38,27 +39,23 @@ const ingest = (metadata: Record): IngestionPayload => ({ metadata, }); -describe('ValidationManager', function () { - let validator: ValidationManager; +describe('IngestionValidator', function () { + let validator: IngestionValidator; let catalogStub: CatalogCall; let jobnikStub: JobnikClient; let tilesetReaderStub: TilesetReader; let providerStub: Provider; beforeEach(async function () { - catalogStub = { findRecords: vi.fn().mockResolvedValue([]) } as unknown as CatalogCall; + catalogStub = { + findRecords: vi.fn().mockResolvedValue([]), + } as unknown as CatalogCall; jobnikStub = { hasInFlightIngestionJob: vi.fn().mockResolvedValue(false) } as unknown as JobnikClient; tilesetReaderStub = { readTilesetJson: vi.fn().mockResolvedValue(regionTilesetJson) } as unknown as TilesetReader; providerStub = { fileExists: vi.fn().mockResolvedValue(true) }; - validator = new ValidationManager( - configStub, - await jsLogger({ enabled: false }), - lookupStub, - catalogStub, - jobnikStub, - tilesetReaderStub, - providerStub - ); + const logger = await jsLogger({ enabled: false }); + const tilesetValidator = new TilesetValidator(configStub, logger, tilesetReaderStub); + validator = new IngestionValidator(configStub, logger, lookupStub, catalogStub, jobnikStub, providerStub, tilesetValidator); }); it('should pass a fully valid ingestion payload', async function () { @@ -144,6 +141,28 @@ describe('ValidationManager', function () { await expect(validator.validateIngestion(ingest(buildValidMetadata()))).rejects.toThrow(ERROR_FILE_NOT_FOUND); }); + it('should throw 400 when the model path is not under the agreed base path', async function () { + const payload = { ...ingest(buildValidMetadata()), modelPath: '/etc/passwd' }; + + await expect(validator.validateIngestion(payload)).rejects.toThrow(ERROR_MODEL_PATH_INVALID); + }); + + it('should throw 409 when the productId already exists in the catalog', async function () { + (catalogStub.findRecords as ReturnType).mockImplementation((query: { productId?: string; productName?: string }) => + query.productId !== undefined ? [{ id: 'existing', productId: query.productId }] : [] + ); + + let thrown: unknown; + try { + await validator.validateIngestion(ingest(buildValidMetadata())); + } catch (err) { + thrown = err; + } + + expect(thrown).toBeInstanceOf(AppError); + expect((thrown as AppError).status).toBe(StatusCodes.CONFLICT); + }); + it('should throw 400 when the footprint does not intersect the tileset model', async function () { const metadata = buildValidMetadata(); metadata.footprint = { @@ -202,35 +221,4 @@ describe('ValidationManager', function () { expect(thrown).toBeInstanceOf(AppError); expect((thrown as AppError).status).toBe(StatusCodes.BAD_REQUEST); }); - - describe('validateDelete', function () { - it('should return the record when it is deletable', async function () { - const record = { id: 'rec-1', productType: '3DPhotoRealistic', productStatus: 'UNPUBLISHED' }; - (catalogStub.findRecords as ReturnType).mockResolvedValueOnce([record]); - - await expect(validator.validateDelete('rec-1')).resolves.toEqual(record); - }); - - it('should throw when the record is not found', async function () { - (catalogStub.findRecords as ReturnType).mockResolvedValueOnce([]); - - await expect(validator.validateDelete('missing')).rejects.toThrow(ERROR_DELETE_RECORD_NOT_FOUND); - }); - - it('should throw when the productType is blocked', async function () { - (catalogStub.findRecords as ReturnType).mockResolvedValueOnce([ - { id: 'rec-1', productType: 'QuantizedMeshDTMBest', productStatus: 'UNPUBLISHED' }, - ]); - - await expect(validator.validateDelete('rec-1')).rejects.toThrow(ERROR_DELETE_PRODUCT_TYPE); - }); - - it('should throw when the record is not unpublished', async function () { - (catalogStub.findRecords as ReturnType).mockResolvedValueOnce([ - { id: 'rec-1', productType: '3DPhotoRealistic', productStatus: 'PUBLISHED' }, - ]); - - await expect(validator.validateDelete('rec-1')).rejects.toThrow(ERROR_DELETE_STATUS); - }); - }); }); diff --git a/tests/unit/validator/metadataValidator.spec.ts b/tests/unit/validator/metadataValidator.spec.ts new file mode 100644 index 0000000..27d441f --- /dev/null +++ b/tests/unit/validator/metadataValidator.spec.ts @@ -0,0 +1,126 @@ +import { jsLogger } from '@map-colonies/js-logger'; +import { describe, it, expect, beforeEach, vi } from 'vitest'; +import { StatusCodes } from 'http-status-codes'; +import { MetadataValidator } from '@src/validator/metadataValidator'; +import { + ERROR_DELETE_RECORD_NOT_FOUND, + ERROR_DELETE_PRODUCT_TYPE, + ERROR_DELETE_STATUS, + ERROR_RECORD_NOT_FOUND, + ERROR_RECORD_BEING_DELETED, + ERROR_EXTRACTABLE_CONFLICT, +} from '@src/validator/errors'; +import { AppError } from '@src/common/appError'; +import type { LookupTablesCall } from '@src/externalServices/lookupTables/lookupTablesCall'; +import type { CatalogCall } from '@src/externalServices/catalog/catalogCall'; +import type { ExtractableCall } from '@src/externalServices/extractableManagement/extractableCall'; + +const lookupStub = { getClassifications: vi.fn().mockResolvedValue(['abc123']) } as unknown as LookupTablesCall; + +describe('MetadataValidator', function () { + let validator: MetadataValidator; + let catalogStub: CatalogCall; + let extractableStub: ExtractableCall; + + beforeEach(async function () { + catalogStub = { + findRecords: vi.fn().mockResolvedValue([]), + getRecord: vi.fn().mockResolvedValue({ id: 'rec-1', productName: 'afula' }), + } as unknown as CatalogCall; + extractableStub = { isExtractableRecordExists: vi.fn().mockResolvedValue(false) } as unknown as ExtractableCall; + validator = new MetadataValidator(await jsLogger({ enabled: false }), lookupStub, catalogStub, extractableStub); + }); + + describe('validateUpdate', function () { + it('should return the record for a valid update payload', async function () { + await expect(validator.validateUpdate('rec-1', { description: 'x' })).resolves.toEqual({ id: 'rec-1', productName: 'afula' }); + }); + + it('should throw 404 when the record does not exist', async function () { + (catalogStub.getRecord as ReturnType).mockResolvedValueOnce(undefined); + + let thrown: unknown; + try { + await validator.validateUpdate('missing', { description: 'x' }); + } catch (err) { + thrown = err; + } + + expect(thrown).toBeInstanceOf(AppError); + expect((thrown as AppError).status).toBe(StatusCodes.NOT_FOUND); + expect((thrown as AppError).message).toBe(ERROR_RECORD_NOT_FOUND); + }); + + it('should throw 400 when the record is being deleted', async function () { + (catalogStub.getRecord as ReturnType).mockResolvedValueOnce({ id: 'rec-1', productStatus: 'BEING_DELETED' }); + + await expect(validator.validateUpdate('rec-1', { description: 'x' })).rejects.toThrow(ERROR_RECORD_BEING_DELETED); + }); + + it('should throw 400 when a renamed product name collides with another record', async function () { + (catalogStub.findRecords as ReturnType).mockResolvedValueOnce([{ id: 'other', productName: 'haifa' }]); + + await expect(validator.validateUpdate('rec-1', { productName: 'haifa' })).rejects.toThrow(AppError); + }); + }); + + describe('validateStatusChange', function () { + it('should throw 404 when the record does not exist', async function () { + (catalogStub.getRecord as ReturnType).mockResolvedValueOnce(undefined); + + await expect(validator.validateStatusChange('missing')).rejects.toThrow(ERROR_RECORD_NOT_FOUND); + }); + }); + + describe('ensureRecordAbsentFromExtractable', function () { + it('should throw 409 when the record exists in extractable-management', async function () { + (extractableStub.isExtractableRecordExists as ReturnType).mockResolvedValueOnce(true); + + let thrown: unknown; + try { + await validator.ensureRecordAbsentFromExtractable({ id: 'rec-1', productName: 'afula' }); + } catch (err) { + thrown = err; + } + + expect(thrown).toBeInstanceOf(AppError); + expect((thrown as AppError).status).toBe(StatusCodes.CONFLICT); + expect((thrown as AppError).message).toBe(ERROR_EXTRACTABLE_CONFLICT); + }); + + it('should pass when the record is absent from extractable-management', async function () { + await expect(validator.ensureRecordAbsentFromExtractable({ id: 'rec-1', productName: 'afula' })).resolves.toBeUndefined(); + }); + }); + + describe('validateDelete', function () { + it('should return the record when it is deletable', async function () { + const record = { id: 'rec-1', productType: '3DPhotoRealistic', productStatus: 'UNPUBLISHED' }; + (catalogStub.findRecords as ReturnType).mockResolvedValueOnce([record]); + + await expect(validator.validateDelete('rec-1')).resolves.toEqual(record); + }); + + it('should throw when the record is not found', async function () { + (catalogStub.findRecords as ReturnType).mockResolvedValueOnce([]); + + await expect(validator.validateDelete('missing')).rejects.toThrow(ERROR_DELETE_RECORD_NOT_FOUND); + }); + + it('should throw when the productType is blocked', async function () { + (catalogStub.findRecords as ReturnType).mockResolvedValueOnce([ + { id: 'rec-1', productType: 'QuantizedMeshDTMBest', productStatus: 'UNPUBLISHED' }, + ]); + + await expect(validator.validateDelete('rec-1')).rejects.toThrow(ERROR_DELETE_PRODUCT_TYPE); + }); + + it('should throw when the record is not unpublished', async function () { + (catalogStub.findRecords as ReturnType).mockResolvedValueOnce([ + { id: 'rec-1', productType: '3DPhotoRealistic', productStatus: 'PUBLISHED' }, + ]); + + await expect(validator.validateDelete('rec-1')).rejects.toThrow(ERROR_DELETE_STATUS); + }); + }); +});