diff --git a/README.md b/README.md index abe1eae..64663d1 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,7 @@ Services notifies Job Tracker on completed tasks, and Job Tracker handles the re ## API Check out the OpenAPI spec [here](/openapi3.yaml) + #### Get all items ```http @@ -61,7 +62,7 @@ Start the server |`JOB_DEFINITIONS_JOB_SWAP_UPDATE` |Job name. |Ingestion_Swap_Update | |`JOB_DEFINITIONS_TASK_INIT` |Task name. |init | |`JOB_DEFINITIONS_TASK_MERGE` |Task name. |merge | -|`JOB_DEFINITIONS_TASK_DELETION` |Task name. |merge | +|`JOB_DEFINITIONS_TASK_TILES_DELETION`|Task name. |tiles-deletion | |`JOB_DEFINITIONS_TASK_POLYGON_PARTS`|Task name. |polygon-parts | |`JOB_DEFINITIONS_TASK_FINALIZE` |Task name. |finalize | |`HTTP_RETRY_ATTEMPTS` |How many retries should the service make if a request fails. |5 | diff --git a/config/custom-environment-variables.json b/config/custom-environment-variables.json index 03fbc73..e527ae7 100644 --- a/config/custom-environment-variables.json +++ b/config/custom-environment-variables.json @@ -68,18 +68,20 @@ "update": "JOB_DEFINITIONS_JOB_UPDATE", "swapUpdate": "JOB_DEFINITIONS_JOB_SWAP_UPDATE", "export": "JOB_DEFINITIONS_JOB_EXPORT", - "seed": "JOB_DEFINITIONS_JOB_SEED" + "seed": "JOB_DEFINITIONS_JOB_SEED", + "deleteLayer": "JOB_DEFINITIONS_JOB_DELETE_LAYER" }, "tasks": { "validation": "JOB_DEFINITIONS_TASK_VALIDATION", "init": "JOB_DEFINITIONS_TASK_INIT", "createTasks": "JOB_DEFINITIONS_TASK_CREATE_TASKS", "merge": "JOB_DEFINITIONS_TASK_MERGE", - "tilesDeletion": "JOB_DEFINITIONS_TASK_DELETION", + "tilesDeletion": "JOB_DEFINITIONS_TASK_TILES_DELETION", "polygonParts": "JOB_DEFINITIONS_TASK_POLYGON_PARTS", "export": "JOB_DEFINITIONS_TASK_EXPORT", "finalize": "JOB_DEFINITIONS_TASK_FINALIZE", - "seed": "JOB_DEFINITIONS_TASK_SEED" + "seed": "JOB_DEFINITIONS_TASK_SEED", + "delete": "JOB_DEFINITIONS_TASK_DELETE" }, "suspendingTaskTypes": { "__name": "JOB_DEFINITIONS_SUSPENDING_TASKS", @@ -98,6 +100,10 @@ "seedTasksFlow": { "__name": "SEED_TASKS_FLOW", "__format": "json" + }, + "deleteLayerTasksFlow": { + "__name": "DELETE_LAYER_TASKS_FLOW", + "__format": "json" } } } diff --git a/config/default.json b/config/default.json index 7d9eae3..8e57929 100644 --- a/config/default.json +++ b/config/default.json @@ -53,7 +53,8 @@ "update": "Ingestion_Update", "swapUpdate": "Ingestion_Swap_Update", "export": "Export", - "seed": "TilesSeeding" + "seed": "TilesSeeding", + "deleteLayer": "Delete_Layer" }, "tasks": { "validation": "validation", @@ -63,13 +64,16 @@ "createTasks": "create-tasks", "export": "tilesExporting", "finalize": "finalize", - "seed": "TilesSeeding" + "seed": "TilesSeeding", + "tilesDeletion": "tiles-deletion", + "delete": "delete" }, "suspendingTaskTypes": ["polygon-parts"] }, "taskFlowManager": { "ingestionTasksFlow": ["validation", "create-merge-tasks", "tilesMerging", "finalize"], "exportTasksFlow": ["init", "tilesExporting", "polygon-parts", "finalize"], - "seedTasksFlow": ["TilesSeeding"] + "seedTasksFlow": ["TilesSeeding"], + "deleteLayerTasksFlow": ["delete", "tiles-deletion"] } } diff --git a/helm/templates/configmap.yaml b/helm/templates/configmap.yaml index 4c7c231..cb9dfea 100644 --- a/helm/templates/configmap.yaml +++ b/helm/templates/configmap.yaml @@ -33,19 +33,22 @@ data: JOB_DEFINITIONS_JOB_SWAP_UPDATE: {{ $jobDefinitions.jobs.swapUpdate.type | quote }} JOB_DEFINITIONS_JOB_EXPORT: {{ $jobDefinitions.jobs.export.type | quote }} JOB_DEFINITIONS_JOB_SEED: {{ $jobDefinitions.jobs.seed.type | quote }} + JOB_DEFINITIONS_JOB_DELETE_LAYER: {{ $jobDefinitions.jobs.deleteLayer.type | quote }} JOB_DEFINITIONS_TASK_INIT: {{ $jobDefinitions.tasks.init.type | quote }} JOB_DEFINITIONS_TASK_VALIDATION: {{ $jobDefinitions.tasks.validation.type | quote }} JOB_DEFINITIONS_TASK_MERGE: {{ $jobDefinitions.tasks.merge.type | quote }} - JOB_DEFINITIONS_TASK_DELETION: {{ $jobDefinitions.tasks.tilesDeletion.type | quote }} + JOB_DEFINITIONS_TASK_TILES_DELETION: {{ $jobDefinitions.tasks.tilesDeletion.type | quote }} JOB_DEFINITIONS_TASK_POLYGON_PARTS: {{ $jobDefinitions.tasks.polygonParts.type | quote }} JOB_DEFINITIONS_TASK_CREATE_TASKS: {{ $jobDefinitions.tasks.createTasks.type | quote }} JOB_DEFINITIONS_TASK_EXPORT: {{ $jobDefinitions.tasks.export.type | quote }} JOB_DEFINITIONS_TASK_FINALIZE: {{ $jobDefinitions.tasks.finalize.type | quote }} JOB_DEFINITIONS_TASK_SEED: {{ $jobDefinitions.tasks.seed.type | quote }} + JOB_DEFINITIONS_TASK_DELETE: {{ $jobDefinitions.tasks.delete.type | quote }} JOB_DEFINITIONS_SUSPENDING_TASKS: {{ $suspendingTaskTypes | quote }} INGESTION_TASKS_FLOW: {{ .Values.taskFlowManager.ingestionTasksFlow | toJson | quote }} EXPORT_TASKS_FLOW: {{ .Values.taskFlowManager.exportTasksFlow | toJson | quote }} SEED_TASKS_FLOW: {{ .Values.taskFlowManager.seedTasksFlow | toJson | quote }} + DELETE_LAYER_TASKS_FLOW: {{ .Values.taskFlowManager.deleteLayerTasksFlow | toJson | quote }} {{- with .Values.configManagement }} CONFIG_NAME: {{ .name | quote }} CONFIG_VERSION: {{ .version | quote }} diff --git a/helm/values.yaml b/helm/values.yaml index 20b566c..36a3138 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -95,6 +95,8 @@ jobDefinitions: type: '' seed: type: '' + deleteLayer: + type: '' tasks: validation: type: '' @@ -115,6 +117,8 @@ jobDefinitions: type: '' seed: type: '' + delete: + type: '' taskFlowManager: ingestionTasksFlow: @@ -130,6 +134,9 @@ taskFlowManager: - finalize seedTasksFlow: - TilesSeeding + deleteLayerTasksFlow: + - delete + - tiles-deletion env: port: 80 targetPort: 8080 diff --git a/package-lock.json b/package-lock.json index 6a25788..974f6c1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -21,7 +21,7 @@ "@map-colonies/mc-utils": "^5.1.0", "@map-colonies/openapi-express-viewer": "^3.0.0", "@map-colonies/prometheus": "^1.0.0", - "@map-colonies/raster-shared": "^7.10.2", + "@map-colonies/raster-shared": "8.3.0-alpha.0", "@map-colonies/read-pkg": "0.0.1", "@map-colonies/schemas": "^1.18.0", "@map-colonies/telemetry": "^6.0.0", @@ -161,7 +161,6 @@ "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.18.2.tgz", "integrity": "sha512-A8pri1YJiC5UnkdrWcmfZTJTV85b4UXTAfImGmCfYmax4TR9Cw8sDS0MOk++Gp2mE/BefVJ5nwy5yzqNJbP/DQ==", "dev": true, - "peer": true, "dependencies": { "@ampproject/remapping": "^2.1.0", "@babel/code-frame": "^7.16.7", @@ -3287,7 +3286,6 @@ "resolved": "https://registry.npmjs.org/@map-colonies/mc-utils/-/mc-utils-5.1.0.tgz", "integrity": "sha512-4eL4ykgJFrFSPP18AbeJUJklwmzJieormWIvEc8c8PFRXU5eN1lRUal60xAhghpmzEQnudRG1kV3NWIibUjw2w==", "license": "ISC", - "peer": true, "dependencies": { "@map-colonies/types": "^1.9.0", "@turf/turf": "^6.5.0", @@ -3414,9 +3412,9 @@ } }, "node_modules/@map-colonies/raster-shared": { - "version": "7.10.2", - "resolved": "https://registry.npmjs.org/@map-colonies/raster-shared/-/raster-shared-7.10.2.tgz", - "integrity": "sha512-EKPtETrKPWOZTpVQ0CEs2CFaC2cU9FlXa+Ylss+Jg7wmFEaN4BKR2hUw5kjtsgD69lqL9fyFCWQysBFc7h4iTQ==", + "version": "8.3.0-alpha.0", + "resolved": "https://registry.npmjs.org/@map-colonies/raster-shared/-/raster-shared-8.3.0-alpha.0.tgz", + "integrity": "sha512-xcleqPf+KSOObCqHuKm1nng7WQRAZ0JrX12qe1U8pX8iijZ7pS7QMrHnPM7DFU5cxMoxkgnRC8hVzd1SMLiErQ==", "license": "ISC", "dependencies": { "@map-colonies/mc-priority-queue": "^9.1.0", @@ -3440,8 +3438,7 @@ "version": "1.18.0", "resolved": "https://registry.npmjs.org/@map-colonies/schemas/-/schemas-1.18.0.tgz", "integrity": "sha512-G6OB/xogqzcLrDU+sXok8ECFbdzQDjI9AE+eMZP6Bcl8BZArkkFI/wNz14HCD2NE81p4SNaeAKUuyGZBqzRJyg==", - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/@map-colonies/telemetry": { "version": "6.0.0", @@ -3467,7 +3464,6 @@ "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.7.0.tgz", "integrity": "sha512-AdY5wvN0P2vXBi3b29hxZgSFvdhdxPB9+f0B6s//P9Q8nibRWeA3cHm8UmLpio9ABigkVHJ5NMPk+Mz8VCCyrw==", "license": "Apache-2.0", - "peer": true, "engines": { "node": ">=8.0.0" } @@ -3772,7 +3768,6 @@ "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-2.6.0.tgz", "integrity": "sha512-HLM1v2cbZ4TgYN6KEOj+Bbj8rAKriOdkF9Ed3tG25FoprSiQl7kYc+RRT6fUZGOvx0oMi5U67GoFdT+XUn8zEg==", "license": "Apache-2.0", - "peer": true, "dependencies": { "@opentelemetry/semantic-conventions": "^1.29.0" }, @@ -4945,7 +4940,6 @@ "resolved": "https://registry.npmjs.org/@map-colonies/types/-/types-1.9.0.tgz", "integrity": "sha512-TUM82wWLCV49vFTs16h4OYo15FjlnhMeMGLE922CeA0QNJyEkyaJG7IHvrM0LG793Ep4qeBMeB1izJQaN2fVYQ==", "license": "ISC", - "peer": true, "dependencies": { "@types/geojson": "^7946.0.16", "@types/mime-types": "^2.1.1", @@ -5022,7 +5016,6 @@ "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.9.0.tgz", "integrity": "sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg==", "license": "Apache-2.0", - "peer": true, "engines": { "node": ">=8.0.0" } @@ -5031,7 +5024,6 @@ "version": "0.46.0", "resolved": "https://registry.npmjs.org/@opentelemetry/api-logs/-/api-logs-0.46.0.tgz", "integrity": "sha512-+9BcqfiEDGPXEIo+o3tso/aqGM5dGbGwAkGVp3FPpZ8GlkK1YlaKRd9gMVyPaeRATwvO5wYGGnCsAc/sMMM9Qw==", - "peer": true, "dependencies": { "@opentelemetry/api": "^1.0.0" }, @@ -9626,6 +9618,7 @@ "os": [ "darwin" ], + "peer": true, "engines": { "node": ">=10" } @@ -9642,6 +9635,7 @@ "os": [ "darwin" ], + "peer": true, "engines": { "node": ">=10" } @@ -9658,6 +9652,7 @@ "os": [ "linux" ], + "peer": true, "engines": { "node": ">=10" } @@ -9674,6 +9669,7 @@ "os": [ "linux" ], + "peer": true, "engines": { "node": ">=10" } @@ -9690,6 +9686,7 @@ "os": [ "linux" ], + "peer": true, "engines": { "node": ">=10" } @@ -9706,6 +9703,7 @@ "os": [ "linux" ], + "peer": true, "engines": { "node": ">=10" } @@ -9722,6 +9720,7 @@ "os": [ "linux" ], + "peer": true, "engines": { "node": ">=10" } @@ -9738,6 +9737,7 @@ "os": [ "win32" ], + "peer": true, "engines": { "node": ">=10" } @@ -9754,6 +9754,7 @@ "os": [ "win32" ], + "peer": true, "engines": { "node": ">=10" } @@ -9770,6 +9771,7 @@ "os": [ "win32" ], + "peer": true, "engines": { "node": ">=10" } @@ -9802,6 +9804,7 @@ "resolved": "https://registry.npmjs.org/@swc/types/-/types-0.1.25.tgz", "integrity": "sha512-iAoY/qRhNH8a/hBvm3zKj9qQ4oc2+3w1unPJa2XvTK3XjeLXtzcCingVPw/9e5mn1+0yPqxcBGp9Jf0pkfMb1g==", "license": "Apache-2.0", + "peer": true, "dependencies": { "@swc/counter": "^0.1.3" } @@ -11824,8 +11827,7 @@ "node_modules/@types/json-schema": { "version": "7.0.15", "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz", - "integrity": "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==", - "peer": true + "integrity": "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==" }, "node_modules/@types/keygrip": { "version": "1.0.6", @@ -11940,8 +11942,7 @@ "version": "20.5.1", "resolved": "https://registry.npmjs.org/@types/node/-/node-20.5.1.tgz", "integrity": "sha512-4tT2UrL5LBqDwoed9wZ6N3umC4Yhz3W3FloMmiiG4JwmUJWpie0c7lcnUNd4gtMKuDEO4wRVS8B6Xa0uMRsMKg==", - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/@types/normalize-package-data": { "version": "2.4.1", @@ -12145,7 +12146,6 @@ "integrity": "sha512-klQbnPAAiGYFyI02+znpBRLyjL4/BrBd0nyWkdC0s/6xFLkXYQ8OoRrSkqacS1ddVxf/LDyODIKbQ5TgKAf/Fg==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@typescript-eslint/scope-manager": "8.56.1", "@typescript-eslint/types": "8.56.1", @@ -12681,7 +12681,6 @@ "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.16.0.tgz", "integrity": "sha512-UVJyE9MttOsBQIDKw1skb9nAwQuR5wuGD3+82K6JgJlm/Y+KI92oNsMNGZCYdDsVtRHSak0pcV5Dno5+4jh9sw==", "license": "MIT", - "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -12745,7 +12744,6 @@ "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.18.0.tgz", "integrity": "sha512-PlXPeEWMXMZ7sPYOHqmDyCJzcfNrUr3fGNKtezX14ykXOEIvyK81d+qydx89KY5O71FKMPaQ2vBfBFI5NHR63A==", "license": "MIT", - "peer": true, "dependencies": { "fast-deep-equal": "^3.1.3", "fast-uri": "^3.0.1", @@ -12914,7 +12912,6 @@ "resolved": "https://registry.npmjs.org/axios/-/axios-0.21.4.tgz", "integrity": "sha512-ut5vewkiu8jjGBdqpM44XxjuCjq9LAKeHVmoVfHVzy8eHgxxq8SbAVQNovDA8mVi05kP0Ea/n/UzcSHcTJQfNg==", "license": "MIT", - "peer": true, "dependencies": { "follow-redirects": "^1.14.0" } @@ -13341,7 +13338,6 @@ } ], "license": "MIT", - "peer": true, "dependencies": { "baseline-browser-mapping": "^2.9.0", "caniuse-lite": "^1.0.30001759", @@ -14213,7 +14209,6 @@ "integrity": "sha512-kcZ6+W5QzcJ3P1Mt+83OUv/oHFqZHIx8DuxG6eZ5RGMERoLqp4BuGjhHLYGK+Kf5XVkQvqBSmAy/nGWN3qDgEA==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "import-fresh": "^3.3.0", "js-yaml": "^4.1.0", @@ -15096,7 +15091,6 @@ "integrity": "sha512-hr4ihw+DBqcvrsEDioRO31Z17x71pUYoNe/4h6Z0wB72p7MU7/9gH8Q3s12NFhHPfYBBOV3qyfUxmr/Yn3shnQ==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "env-paths": "^2.2.1", "import-fresh": "^3.3.0", @@ -15929,7 +15923,6 @@ "integrity": "sha512-vPZZsiOKaBAIATpFE2uMI4w5IRwdv/FpQ+qZZMR4E+PeOcM4OeoEbqxRMnywdxP19TyB/3h6QBB0EWon7letSQ==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@typescript-eslint/types": "^8.35.0", "comment-parser": "^1.4.1", @@ -16006,7 +15999,6 @@ "integrity": "sha512-ZCGr7vTH2WSo2hrK5oM2RULFmMruQ7W3cX7YfwoTiPfzTGTFBMmrVIz45jZHd++cGKj/kWf02li/RhTGcANJSA==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@typescript-eslint/utils": "^8.0.0" }, @@ -16519,7 +16511,6 @@ "resolved": "https://registry.npmjs.org/express/-/express-4.22.1.tgz", "integrity": "sha512-F2X8g9P1X7uCPZMA3MVf9wcTqlyNp7IhH5qPCI0izhaOIYXaW9L535tGA3qmjRzpH+bZczqq7hVKxTR4NWnu+g==", "license": "MIT", - "peer": true, "dependencies": { "accepts": "~1.3.8", "array-flatten": "1.1.1", @@ -18383,7 +18374,6 @@ "resolved": "https://registry.npmjs.org/jest/-/jest-29.5.0.tgz", "integrity": "sha512-juMg3he2uru1QoXX078zTa7pO85QyB9xajZc6bU+d9yEGwrKX6+vGmJQ3UdVZsvTEUARIdObzH68QItim6OSSQ==", "dev": true, - "peer": true, "dependencies": { "@jest/core": "^29.5.0", "@jest/types": "^29.5.0", @@ -21050,7 +21040,6 @@ "integrity": "sha512-UczzB+0nnwGotYSgllfARAqWCJ5e/skuV2K/l+Zyck/H6pJIhLXuBnz+6vn2i211o7DtbE78HQtsYEKICHGI+g==", "dev": true, "license": "MIT", - "peer": true, "funding": { "type": "opencollective", "url": "https://opencollective.com/mobx" @@ -22342,7 +22331,6 @@ "resolved": "https://registry.npmjs.org/pino/-/pino-10.3.1.tgz", "integrity": "sha512-r34yH/GlQpKZbU1BvFFqOjhISRo1MNx1tWYsYvmj6KIRHSPMT2+yHOEb1SG6NMvRoHRF0a07kCOox/9yakl1vg==", "license": "MIT", - "peer": true, "dependencies": { "@pinojs/redact": "^0.4.0", "atomic-sleep": "^1.0.0", @@ -22613,7 +22601,6 @@ "integrity": "sha512-UOnG6LftzbdaHZcKoPFtOcCKztrQ57WkHDeRD9t/PTQtmT0NHSeWWepj6pS0z/N7+08BHFDQVUrfmfMRcZwbMg==", "dev": true, "license": "MIT", - "peer": true, "bin": { "prettier": "bin/prettier.cjs" }, @@ -22766,7 +22753,6 @@ "resolved": "https://registry.npmjs.org/prom-client/-/prom-client-15.1.3.tgz", "integrity": "sha512-6ZiOBfCywsD4k1BN9IX0uZhF+tJkV8q8llP64G5Hajs4JOeVLPCwpPVcpXy3BwYiUGgyJzsJJQeOIv7+hDSq8g==", "license": "Apache-2.0", - "peer": true, "dependencies": { "@opentelemetry/api": "^1.4.0", "tdigest": "^0.1.1" @@ -23008,7 +22994,6 @@ "integrity": "sha512-Ku/hhYbVjOQnXDZFv2+RibmLFGwFdeeKHFcOTlrt7xplBnya5OGn/hIRDsqDiSUcfORsDC7MPxwork8jBwsIWA==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=0.10.0" } @@ -23019,7 +23004,6 @@ "integrity": "sha512-yELu4WmLPw5Mr/lmeEpox5rw3RETacE++JgHqQzd2dg+YbJuat3jH4ingc+WPZhxaoFzdv9y33G+F7Nl5O0GBg==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "scheduler": "^0.27.0" }, @@ -24377,7 +24361,6 @@ "integrity": "sha512-J72R4ltw0UBVUlEjTzI0gg2STOqlI9JBhQOL4Dxt7aJOnnSesy0qJDn4PYfMCafk9cWOaVg129Pesl5o+DIh0Q==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@emotion/is-prop-valid": "1.4.0", "@emotion/unitless": "0.10.0", @@ -24687,7 +24670,6 @@ "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=12" }, @@ -24890,7 +24872,6 @@ "resolved": "https://registry.npmjs.org/ts-node/-/ts-node-10.9.1.tgz", "integrity": "sha512-NtVysVPkxxrwFGUUxGYhfux8k78pQB3JqYBXlLRZgdGUqTO5wU/UyHop5p70iEbGhB7q5KmiZiU0Y3KlJrScEw==", "dev": true, - "peer": true, "dependencies": { "@cspotcode/source-map-support": "^0.8.0", "@tsconfig/node10": "^1.0.7", diff --git a/package.json b/package.json index c1f43af..a12bc0e 100644 --- a/package.json +++ b/package.json @@ -51,7 +51,7 @@ "@map-colonies/mc-utils": "^5.1.0", "@map-colonies/openapi-express-viewer": "^3.0.0", "@map-colonies/prometheus": "^1.0.0", - "@map-colonies/raster-shared": "^7.10.2", + "@map-colonies/raster-shared": "8.3.0-alpha.0", "@map-colonies/read-pkg": "0.0.1", "@map-colonies/schemas": "^1.18.0", "@map-colonies/telemetry": "^6.0.0", diff --git a/src/common/interfaces.ts b/src/common/interfaces.ts index 7d4e60e..73d8e26 100644 --- a/src/common/interfaces.ts +++ b/src/common/interfaces.ts @@ -22,6 +22,7 @@ export interface IJobDefinitionsConfig { swapUpdate: string; export: string; seed: string; + deleteLayer: string; }; tasks: { polygonParts: string; @@ -33,6 +34,7 @@ export interface IJobDefinitionsConfig { export: string; seed: string; tilesDeletion: string; + delete: string; }; suspendingTaskTypes: string[]; } diff --git a/src/tasks/handlers/deleteLayer/deleteLayerHandler.ts b/src/tasks/handlers/deleteLayer/deleteLayerHandler.ts new file mode 100644 index 0000000..74d919b --- /dev/null +++ b/src/tasks/handlers/deleteLayer/deleteLayerHandler.ts @@ -0,0 +1,33 @@ +import type { Logger } from '@map-colonies/js-logger'; +import type { IJobResponse, ITaskResponse, JobManagerClient } from '@map-colonies/mc-priority-queue'; +import { injectable, inject } from 'tsyringe'; +import type { ConfigType } from '@src/common/config'; +import type { TaskType, TaskTypes } from '../../../common/interfaces'; +import { SERVICES } from '../../../common/constants'; +import { JobHandler } from '../jobHandler'; + +@injectable() +export class DeleteLayerJobHandler extends JobHandler { + protected readonly tasksFlow: TaskTypes; + protected readonly excludedTypes: TaskTypes; + protected readonly blockedDuplicationTypes: TaskTypes; + + public constructor( + @inject(SERVICES.LOGGER) logger: Logger, + @inject(SERVICES.CONFIG) config: ConfigType, + jobManagerClient: JobManagerClient, + job: IJobResponse, + task: ITaskResponse + ) { + super(logger, config, jobManagerClient, job, task); + this.tasksFlow = this.config.get('taskFlowManager.deleteLayerTasksFlow') as unknown as TaskTypes; + this.excludedTypes = [this.jobDefinitions.tasks.delete, this.jobDefinitions.tasks.tilesDeletion]; + this.blockedDuplicationTypes = []; + + this.initializeTaskOperations(); + } + + public override isJobCompleted = (taskType: TaskType): boolean => { + return this.job.completedTasks === this.job.taskCount && taskType === this.jobDefinitions.tasks.tilesDeletion; + }; +} diff --git a/src/tasks/handlers/jobHandlerFactory.ts b/src/tasks/handlers/jobHandlerFactory.ts index 143e4c8..47c9fa8 100644 --- a/src/tasks/handlers/jobHandlerFactory.ts +++ b/src/tasks/handlers/jobHandlerFactory.ts @@ -7,6 +7,7 @@ import type { JobHandler } from './jobHandler'; import { IngestionJobHandler } from './ingestion/ingestionHandler'; import { ExportJobHandler } from './export/exportHandler'; import { SeedJobHandler } from './seed/seedHandler'; +import { DeleteLayerJobHandler } from './deleteLayer/deleteLayerHandler'; export function getJobHandler( jobHandlerType: string, @@ -31,6 +32,9 @@ export function getJobHandler( case jobDefinitions.jobs.seed: { return new SeedJobHandler(logger, config, jobManagerClient, job, task); } + case jobDefinitions.jobs.deleteLayer: { + return new DeleteLayerJobHandler(logger, config, jobManagerClient, job, task); + } default: throw new BadRequestError(`${jobHandlerType} job type is invalid`); } diff --git a/tests/integration/tasks/deleteLayer/taskManager.spec.ts b/tests/integration/tasks/deleteLayer/taskManager.spec.ts new file mode 100644 index 0000000..09f6058 --- /dev/null +++ b/tests/integration/tasks/deleteLayer/taskManager.spec.ts @@ -0,0 +1,158 @@ +import nock, { cleanAll, isDone, pendingMocks } from 'nock'; +import { OperationStatus } from '@map-colonies/mc-priority-queue'; +import { StatusCodes as httpStatusCodes } from 'http-status-codes'; +import { trace } from '@opentelemetry/api'; +import { jsLogger } from '@map-colonies/js-logger'; +import { initConfig } from '../../../../src/common/config'; +import { configMock } from '../../../mocks/configMock'; +import { getApp } from '../../../../src/app'; +import type { IJobManagerConfig, IJobDefinitionsConfig } from '../../../../src/common/interfaces'; +import { getDeleteLayerJobMock, getTaskMock } from '../../../mocks/jobMocks'; +import { calculateJobPercentage } from '../../../../src/utils/jobUtils'; +import { SERVICES } from '../../../../src/common/constants'; +import { registerExternalValues } from '../../../../src/containerConfig'; +import { TasksRequestSender } from '../helpers/requestSender'; +import { getTestContainerConfig, resetContainer } from '../helpers/containerConfig'; + +describe('tasks', function () { + let requestSender: TasksRequestSender; + let jobManagerConfigMock: IJobManagerConfig; + let jobDefinitionsConfig: IJobDefinitionsConfig; + + beforeAll(async function () { + await initConfig(true); + }); + + beforeEach(async function () { + const [app] = await getApp({ + override: [...(await getTestContainerConfig())], + useChild: true, + }); + + await registerExternalValues({ + override: [ + { token: SERVICES.LOGGER, provider: { useValue: await jsLogger({ enabled: false }) } }, + { token: SERVICES.CONFIG, provider: { useValue: configMock } }, + { token: SERVICES.TRACER, provider: { useValue: trace.getTracer('testTracer') } }, + ], + }); + + requestSender = new TasksRequestSender(app); + jobManagerConfigMock = configMock.get('jobManagement.config') as unknown as IJobManagerConfig; + jobDefinitionsConfig = configMock.get('jobDefinitions') as IJobDefinitionsConfig; + cleanAll(); + }); + + afterEach(function () { + resetContainer(); + jest.restoreAllMocks(); + if (!isDone()) { + throw new Error(`Not all nock interceptors were used: ${JSON.stringify(pendingMocks())}`); + } + }); + + describe('Happy Path', function () { + it('should return 200 and only update progress when getting completed delete task and no other task is pending', async () => { + // mocks + const mockDeleteLayerJob = getDeleteLayerJobMock({ taskCount: 1, completedTasks: 1 }); + const mockDeleteTask = getTaskMock(mockDeleteLayerJob.id, { + type: jobDefinitionsConfig.tasks.delete, + status: OperationStatus.COMPLETED, + }); + + nock(jobManagerConfigMock.jobManagerBaseUrl).post('/tasks/find', { id: mockDeleteTask.id }).reply(httpStatusCodes.OK, [mockDeleteTask]); + nock(jobManagerConfigMock.jobManagerBaseUrl) + .get(`/jobs/${mockDeleteLayerJob.id}`) + .query({ shouldReturnTasks: false }) + .reply(httpStatusCodes.OK, mockDeleteLayerJob); + const taskPercentage = calculateJobPercentage(mockDeleteLayerJob.completedTasks, mockDeleteLayerJob.taskCount); + nock(jobManagerConfigMock.jobManagerBaseUrl).put(`/jobs/${mockDeleteLayerJob.id}`, { percentage: taskPercentage }).reply(httpStatusCodes.OK); + + // action + const response = await requestSender.handleTaskNotification(mockDeleteTask.id); + + // expectation + expect(response.status).toBe(httpStatusCodes.OK); + expect(response).toSatisfyApiSpec(); + }); + + it('should return 200 and complete job when getting completed excluded "tilesDeletion" task and no other task is pending', async () => { + // mocks + const mockDeleteLayerJob = getDeleteLayerJobMock({ taskCount: 3, completedTasks: 3 }); + const mockExcludedTask = getTaskMock(mockDeleteLayerJob.id, { + type: jobDefinitionsConfig.tasks.tilesDeletion, + status: OperationStatus.COMPLETED, + }); + + nock(jobManagerConfigMock.jobManagerBaseUrl).post('/tasks/find', { id: mockExcludedTask.id }).reply(httpStatusCodes.OK, [mockExcludedTask]); + nock(jobManagerConfigMock.jobManagerBaseUrl) + .get(`/jobs/${mockDeleteLayerJob.id}`) + .query({ shouldReturnTasks: false }) + .reply(httpStatusCodes.OK, mockDeleteLayerJob); + nock(jobManagerConfigMock.jobManagerBaseUrl) + .put(`/jobs/${mockDeleteLayerJob.id}`, { status: OperationStatus.COMPLETED, percentage: 100 }) + .reply(httpStatusCodes.OK); + + // action + const response = await requestSender.handleTaskNotification(mockExcludedTask.id); + + // expectation + expect(response.status).toBe(httpStatusCodes.OK); + expect(response).toSatisfyApiSpec(); + }); + + it('should return 200 and fail job when getting failed delete task', async () => { + // mocks + const mockDeleteLayerJob = getDeleteLayerJobMock(); + const mockDeleteTask = getTaskMock(mockDeleteLayerJob.id, { + type: jobDefinitionsConfig.tasks.delete, + status: OperationStatus.FAILED, + reason: 'delete task failed', + }); + + nock(jobManagerConfigMock.jobManagerBaseUrl).post('/tasks/find', { id: mockDeleteTask.id }).reply(httpStatusCodes.OK, [mockDeleteTask]); + nock(jobManagerConfigMock.jobManagerBaseUrl) + .get(`/jobs/${mockDeleteLayerJob.id}`) + .query({ shouldReturnTasks: false }) + .reply(httpStatusCodes.OK, mockDeleteLayerJob); + nock(jobManagerConfigMock.jobManagerBaseUrl) + .put(`/jobs/${mockDeleteLayerJob.id}`, { status: OperationStatus.FAILED, reason: mockDeleteTask.reason }) + .reply(httpStatusCodes.OK); + + // action + const response = await requestSender.handleTaskNotification(mockDeleteTask.id); + + // expectation + expect(response.status).toBe(httpStatusCodes.OK); + expect(response).toSatisfyApiSpec(); + }); + + it('should return 200 and fail job when getting failed tiles-deletion task', async () => { + // mocks + const mockDeleteLayerJob = getDeleteLayerJobMock(); + const mockTilesDeletionTask = getTaskMock(mockDeleteLayerJob.id, { + type: jobDefinitionsConfig.tasks.tilesDeletion, + status: OperationStatus.FAILED, + reason: 'tiles-deletion task failed', + }); + + nock(jobManagerConfigMock.jobManagerBaseUrl) + .post('/tasks/find', { id: mockTilesDeletionTask.id }) + .reply(httpStatusCodes.OK, [mockTilesDeletionTask]); + nock(jobManagerConfigMock.jobManagerBaseUrl) + .get(`/jobs/${mockDeleteLayerJob.id}`) + .query({ shouldReturnTasks: false }) + .reply(httpStatusCodes.OK, mockDeleteLayerJob); + nock(jobManagerConfigMock.jobManagerBaseUrl) + .put(`/jobs/${mockDeleteLayerJob.id}`, { status: OperationStatus.FAILED, reason: mockTilesDeletionTask.reason }) + .reply(httpStatusCodes.OK); + + // action + const response = await requestSender.handleTaskNotification(mockTilesDeletionTask.id); + + // expectation + expect(response.status).toBe(httpStatusCodes.OK); + expect(response).toSatisfyApiSpec(); + }); + }); +}); diff --git a/tests/mocks/configMock.ts b/tests/mocks/configMock.ts index 05685e2..40845a1 100644 --- a/tests/mocks/configMock.ts +++ b/tests/mocks/configMock.ts @@ -83,6 +83,7 @@ const registerDefaultConfig = (): void => { swapUpdate: 'Ingestion_Swap_Update', export: 'Export', seed: 'TilesSeeding', + deleteLayer: 'Delete_Layer', }, tasks: { validation: 'validation', @@ -93,6 +94,8 @@ const registerDefaultConfig = (): void => { finalize: 'finalize', export: 'tilesExporting', seed: 'TilesSeeding', + tilesDeletion: 'tiles-deletion', + delete: 'delete', }, suspendingTaskTypes: ['validation'], }, @@ -100,6 +103,7 @@ const registerDefaultConfig = (): void => { ingestionTasksFlow: ['validation', 'create-tasks', 'tilesMerging', 'finalize'], exportTasksFlow: ['init', 'tilesExporting', 'polygon-parts', 'finalize'], seedTasksFlow: ['TilesSeeding'], + deleteLayerTasksFlow: ['delete', 'tiles-deletion'], }, }; diff --git a/tests/mocks/jobMocks.ts b/tests/mocks/jobMocks.ts index f1c0737..6b7e4d0 100644 --- a/tests/mocks/jobMocks.ts +++ b/tests/mocks/jobMocks.ts @@ -109,6 +109,39 @@ export const getSeedingJobMock = (override?: Partial>): IJobResponse => { + const defaultJobMock = { + id: faker.string.uuid(), + resourceId: 'test', + version: '1.0', + type: 'Delete_Layer', + description: '', + status: OperationStatus.IN_PROGRESS, + percentage: 0, + reason: '', + domain: 'RASTER', + isCleaned: false, + priority: 0, + parameters: { approver: 'test-approver' }, + expirationDate: undefined, + internalId: faker.string.uuid(), + producerName: undefined, + productName: 'test', + productType: 'Orthophoto', + additionalIdentifiers: '', + taskCount: 1, + completedTasks: 0, + failedTasks: 0, + expiredTasks: 0, + pendingTasks: 0, + inProgressTasks: 0, + abortedTasks: 0, + created: faker.date.anytime().toString(), + updated: faker.date.anytime().toString(), + }; + return { ...defaultJobMock, ...override }; +}; + export const getTaskMock = (jobId: string, override?: Partial, 'jobId'>>): ITaskResponse => { const defaultTaskMock = { id: faker.string.uuid(), diff --git a/tests/unit/tasks/handlers/jobHandler.spec.ts b/tests/unit/tasks/handlers/jobHandler.spec.ts index ddddb3f..9f28438 100644 --- a/tests/unit/tasks/handlers/jobHandler.spec.ts +++ b/tests/unit/tasks/handlers/jobHandler.spec.ts @@ -23,6 +23,7 @@ describe('JobHandler', () => { { mockJob: createTestJob(jobDefinitionsConfig.jobs.swapUpdate), taskType: jobDefinitionsConfig.tasks.merge }, { mockJob: createTestJob(jobDefinitionsConfig.jobs.export), taskType: jobDefinitionsConfig.tasks.export }, { mockJob: createTestJob(jobDefinitionsConfig.jobs.seed), taskType: jobDefinitionsConfig.tasks.seed }, + { mockJob: createTestJob(jobDefinitionsConfig.jobs.deleteLayer), taskType: jobDefinitionsConfig.tasks.delete }, ]; const testCaseHandlerLog = '$mockJob.type handler'; @@ -58,9 +59,11 @@ describe('JobHandler', () => { }); describe('handleCompletedNotification', () => { - const nonSeedCases = testCases.filter(({ mockJob }) => mockJob.type !== jobDefinitionsConfig.jobs.seed); // removing seed job test case as finalize task type is not handled there + const finalizeCases = testCases.filter( + ({ mockJob }) => mockJob.type !== jobDefinitionsConfig.jobs.seed && mockJob.type !== jobDefinitionsConfig.jobs.deleteLayer + ); // removing seed and deleteLayer job test cases as finalize task type is not handled there - it.each(nonSeedCases)( + it.each(finalizeCases)( `should complete job when all of the task are completed and task type is "finalize" - ${testCaseHandlerLog}`, async ({ mockJob }) => { mockJob = { ...mockJob, completedTasks: 10, taskCount: 10 }; diff --git a/tests/unit/tasks/handlers/jobHandlerFactory.spec.ts b/tests/unit/tasks/handlers/jobHandlerFactory.spec.ts index 84e53f6..580ebbd 100644 --- a/tests/unit/tasks/handlers/jobHandlerFactory.spec.ts +++ b/tests/unit/tasks/handlers/jobHandlerFactory.spec.ts @@ -7,6 +7,7 @@ import { getJobHandler } from '../../../../src/tasks/handlers/jobHandlerFactory' import type { IJobDefinitionsConfig } from '../../../../src/common/interfaces'; import { IngestionJobHandler } from '../../../../src/tasks/handlers/ingestion/ingestionHandler'; import { ExportJobHandler } from '../../../../src/tasks/handlers/export/exportHandler'; +import { DeleteLayerJobHandler } from '../../../../src/tasks/handlers/deleteLayer/deleteLayerHandler'; // Test helper functions const createTestLogger = async (): Promise => jsLogger({ enabled: false }); @@ -87,6 +88,20 @@ describe('jobHandlerFactory', () => { // Then: should create ExportJobHandler expect(handler).toBeInstanceOf(ExportJobHandler); }); + + it('should create DeleteLayerJobHandler for deleteLayer job type', () => { + // Given: deleteLayer job and task + const jobType = jobDefinitionsConfig.jobs.deleteLayer; + const mockJob = createTestJob(jobType); + const mockTask = getTaskMock(mockJob.id); + const mockQueueClient = createMockQueueClient(); + + // When: creating handler for deleteLayer job type + const handler = getJobHandler(jobType, jobDefinitionsConfig, logger, mockQueueClient, configMock, mockJob, mockTask); + + // Then: should create DeleteLayerJobHandler + expect(handler).toBeInstanceOf(DeleteLayerJobHandler); + }); }); describe('Error handling', () => {