feat: redis storage provider and cache deletion jobs (MAPCO-11263)

README.md

@@ -63,22 +63,26 @@ npm run test:watch
 
 #### Integration tests
 
-`npm run test:integration` needs an S3-compatible server. By default it starts a Minio
-[testcontainer](https://testcontainers.com/) automatically.
- The image is pinned to the release deployed in our Azure environment; bump it in
-`tests/integration/helpers/minioContainer.ts` when that environment moves.
+`npm run test:integration` needs an S3-compatible server and a Redis server. By default it starts
+a Minio and a Redis [testcontainer](https://testcontainers.com/) automatically.
+The images are pinned to the releases deployed in our environments; bump them in
+`tests/integration/helpers/minioContainer.ts` and `tests/integration/helpers/redisContainer.ts`
+when those environments move.
 
-To run against an already-running Minio instead, set `TEST_MINIO_ENDPOINT`:
+To run against already-running servers instead, set the `TEST_*` variables:
 
-| Variable                 | Default      | Purpose                                                            |
-| ------------------------ | ------------ | ------------------------------------------------------------------ |
+| Variable                 | Default      | Purpose                                                              |
+| ------------------------ | ------------ | -------------------------------------------------------------------- |
 | `TEST_MINIO_ENDPOINT`    | _(unset)_    | Point the suite at an existing Minio. Unset means start a container. |
-| `TEST_MINIO_ACCESS_KEY`  | `minioadmin` | Access key for that server.                                        |
-| `TEST_MINIO_SECRET_KEY`  | `minioadmin` | Secret key for that server.                                        |
-
-> **The suite creates and deletes buckets on whichever endpoint it is given.** Never point
-> `TEST_MINIO_ENDPOINT` at a shared or deployed environment, and beware of leaving it exported in a
-> shell profile. Each run prints which mode it selected and against which endpoint.
+| `TEST_MINIO_ACCESS_KEY`  | `minioadmin` | Access key for that server.                                          |
+| `TEST_MINIO_SECRET_KEY`  | `minioadmin` | Secret key for that server.                                          |
+| `TEST_REDIS_HOST`        | _(unset)_    | Point the suite at an existing Redis. Unset means start a container. |
+| `TEST_REDIS_PORT`        | `6379`       | Port for that server.                                                |
+
+> **The suite creates and deletes buckets on whichever Minio it is given, and calls `FLUSHDB` on
+> whichever Redis it is given.** Never point `TEST_MINIO_ENDPOINT` or `TEST_REDIS_HOST` at a shared
+> or deployed environment, and beware of leaving them exported in a shell profile. Each run prints
+> which mode it selected and against which server.
 
 ## Customizing the Boilerplate
 

config/custom-environment-variables.json

@@ -110,6 +110,33 @@
         "__name": "FS_SUB_PATHS",
         "__format": "json"
       }
+    },
+    "redis": {
+      "delete": {
+        "batchSize": {
+          "__name": "REDIS_DELETE_BATCH_SIZE",
+          "__format": "number"
+        },
+        "scanCount": {
+          "__name": "REDIS_SCAN_COUNT",
+          "__format": "number"
+        }
+      },
+      "host": "REDIS_HOST",
+      "port": {
+        "__name": "REDIS_PORT",
+        "__format": "number"
+      },
+      "db": {
+        "__name": "REDIS_DB",
+        "__format": "number"
+      },
+      "username": "REDIS_USERNAME",
+      "password": "REDIS_PASSWORD",
+      "tlsEnabled": {
+        "__name": "REDIS_TLS_ENABLED",
+        "__format": "boolean"
+      }
     }
   },
   "strategies": {

config/default.json

@@ -34,6 +34,14 @@
         {
           "job": "Delete_Layer",
           "task": "artifacts-deletion"
+        },
+        {
+          "job": "Update_Delete_Cache",
+          "task": "tiles-deletion"
+        },
+        {
+          "job": "Swap_Delete_Cache",
+          "task": "tiles-deletion"
         }
       ]
     }
@@ -58,6 +66,12 @@
       },
       "deleteLayer": {
         "type": "Delete_Layer"
+      },
+      "updateCacheDeletion": {
+        "type": "Update_Delete_Cache"
+      },
+      "swapCacheDeletion": {
+        "type": "Swap_Delete_Cache"
       }
     },
     "tasks": {
@@ -77,7 +91,7 @@
     "shouldResetTimeout": true
   },
   "storage": {
-    "cleanupStorageProviders": ["FS", "S3"],
+    "cleanupStorageProviders": ["FS", "S3", "REDIS"],
     "s3": {
       "delete": {
         "batchSize": 1000
@@ -97,6 +111,15 @@
       "subPaths": {
         "tilesSubPath": "tiles"
       }
+    },
+    "redis": {
+      "delete": {
+        "batchSize": 1000,
+        "scanCount": 1000
+      },
+      "host": "localhost",
+      "port": 6379,
+      "db": 0
     }
   },
   "strategies": {

helm/templates/_tplValues.tpl

@@ -57,6 +57,11 @@ Custom definitions
 {{- include "common.tplvalues.merge" ( dict "values" ( list .Values.storage .Values.global.storage ) "context" . ) }}
 {{- end -}}
 
+{{/* storage.redis, then global.storage.redis, then the shared global.redis block */}}
+{{- define "common.redis.merged" -}}
+{{- include "common.tplvalues.merge" ( dict "values" ( list ((.Values.storage).redis | default dict) ((.Values.global.storage).redis | default dict) (.Values.global.redis | default dict) ) "context" . ) }}
+{{- end -}}
+
 {{- define "common.ca.merged" -}}
 {{- include "common.tplvalues.merge" ( dict "values" ( list .Values.ca .Values.global.ca ) "context" . ) }}
 {{- end -}}

helm/templates/configmap.yaml

@@ -3,6 +3,7 @@
 {{- $storage := fromYaml (include "common.storage.merged" .) -}}
 {{- $s3 := ($storage.s3) | default dict -}}
 {{- $fs := ($storage.fs) | default dict -}}
+{{- $redis := fromYaml (include "common.redis.merged" .) -}}
 {{- $internalPvc := (($fs).internalPvc) | default dict -}}
 {{- $fsBasePath := clean (printf "/%s" $internalPvc.mountPath) -}}
 {{- if .Values.enabled -}}
@@ -74,6 +75,22 @@ data:
   {{- end }}
   FS_SUB_PATHS: {{ $subPaths | toJson | quote }}
   {{- end }}
+  {{- if has "REDIS" $storage.cleanupStorageProviders }}
+  REDIS_DELETE_BATCH_SIZE: {{ $redis.delete.batchSize | default 1000 | quote }}
+  REDIS_HOST: {{ $redis.host | quote }}
+  REDIS_PORT: {{ $redis.port | default 6379 | quote }}
+  REDIS_DB: {{ $redis.db | default 0 | quote }}
+  REDIS_SCAN_COUNT: {{ $redis.delete.scanCount | default 1000 | quote }}
+  REDIS_TLS_ENABLED: {{ $redis.tlsEnabled | default false | quote }}
+  {{- with $redis.auth }}
+  {{- if .enabled }}
+  {{- if .username }}
+  REDIS_USERNAME: {{ .username | quote }}
+  {{- end }}
+  REDIS_PASSWORD: {{ .password | quote }}
+  {{- end }}
+  {{- end }}
+  {{- end }}
   {{- with .Values.env.strategies.tilesDeletion }}
   TILES_DELETION_BATCH_SIZE: {{ .batchSize | default 1000 | quote }}
   TILES_DELETION_CONCURRENCY: {{ .concurrency | default 10 | quote }}

helm/values.yaml

@@ -4,6 +4,7 @@ global:
   metrics: {}
   jobDefinitions: {}
   storage: {}
+  redis: {}
   serviceUrls: {}
   ca: {}
 
@@ -13,7 +14,7 @@ serviceUrls:
   jobTracker: ""
 
 storage:
-  cleanupStorageProviders: {}
+  cleanupStorageProviders: [] # any of "FS", "S3", "REDIS"
   s3:
     delete:
       batchSize: 1000
@@ -30,6 +31,19 @@ storage:
       name: ""
       mountPath: ""
       tilesSubPath: "" # e.g. folder/tiles
+  redis:
+    host: ""
+    port: 6379
+    db: 0
+    # Same nesting as global.redis.auth; username is sent only when non-empty
+    auth:
+      enabled: false
+      username: ""
+      password: ""
+    tlsEnabled: false
+    delete:
+      batchSize: 1000
+      scanCount: 1000
 
 mclabels:
   component: backend
@@ -115,13 +129,20 @@ env:
   queue:
     heartbeatIntervalMs: 1000
     dequeueIntervalMs: 3000
+  jobnik:
+    worker:
+      concurrency: 1
   worker:
     capabilities:
       pairs:
         - job: "Ingestion_Update"
           task: "tiles-deletion"
         - job: "Ingestion_Swap_Update"
           task: "tiles-deletion"
+        - job: "Update_Delete_Cache"
+          task: "tiles-deletion"
+        - job: "Swap_Delete_Cache"
+          task: "tiles-deletion"
   httpRetry:
     attempts: 3
     delay: "exponential"

package.json

@@ -39,13 +39,14 @@
     "@map-colonies/js-logger": "^5.0.0",
     "@map-colonies/mc-priority-queue": "^9.1.2",
     "@map-colonies/mc-utils": "^6.0.1",
-    "@map-colonies/raster-shared": "^9.0.0-alpha.0",
+    "@map-colonies/raster-shared": "^9.0.0",
     "@map-colonies/read-pkg": "^1.0.0",
     "@map-colonies/schemas": "^1.20.0",
     "@map-colonies/telemetry": "^10.0.1",
     "@opentelemetry/api": "^1.9.0",
     "compression": "^1.8.0",
     "express": "^4.21.2",
+    "ioredis": "^6.0.0",
     "prom-client": "^15.1.3",
     "reflect-metadata": "^0.2.2",
     "tsyringe": "^4.8.0",

src/cleaner/clients/index.ts

@@ -0,0 +1,2 @@
+export { createRedisConnection } from './redisClient';
+export { createS3Client } from './s3Client';