feat: add ingestion metadata validation (MAPCO-11589)

config/custom-environment-variables.json

@@ -0,0 +1,9 @@
+{
+  "externalServices": {
+    "catalog": "CATALOG_URL",
+    "lookupTables": {
+      "url": "LOOKUP_TABLES_URL",
+      "subUrl": "LOOKUP_TABLES_SUB_URL"
+    }
+  }
+}

config/default.json

@@ -32,5 +32,12 @@
         "options": null
       }
     }
+  },
+  "externalServices": {
+    "catalog": "http://127.0.0.1:8080",
+    "lookupTables": {
+      "url": "http://127.0.0.1:8080",
+      "subUrl": "lookup-tables/lookupData"
+    }
   }
 }

helm/templates/configmap.yaml

@@ -30,6 +30,13 @@ data:
   {{- end }}
   {{- end }}
   npm_config_cache: /tmp/
+  {{- with .Values.externalServices }}
+  CATALOG_URL: {{ .catalog | quote }}
+  {{- with .lookupTables }}
+  LOOKUP_TABLES_URL: {{ .url | quote }}
+  LOOKUP_TABLES_SUB_URL: {{ .subUrl | quote }}
+  {{- end }}
+  {{- end }}
   {{- with .Values.configManagement }}
   CONFIG_NAME: {{ .name | quote }}
   CONFIG_VERSION: {{ .version | quote }}

helm/values.yaml

@@ -78,6 +78,12 @@ server:
   responseCompressionEnabled: true
   requestPayloadLimit: 1mb
 
+externalServices:
+  catalog: ''
+  lookupTables:
+    url: ''
+    subUrl: ''
+
 telemetry:
   logger:
     level: info

package.json

@@ -37,9 +37,9 @@
     "@godaddy/terminus": "^4.12.1",
     "@map-colonies/3d-shared": "file:../../3d-general/3d-shared",
     "@map-colonies/config": "^4.0.1",
-    "@map-colonies/jobnik-sdk": "^1.0.1",
     "@map-colonies/error-express-handler": "^4.0.0",
     "@map-colonies/express-access-log-middleware": "^5.0.0",
+    "@map-colonies/jobnik-sdk": "^1.0.1",
     "@map-colonies/js-logger": "^5.0.0",
     "@map-colonies/openapi-express-viewer": "^5.0.0",
     "@map-colonies/prometheus": "^1.0.0",
@@ -48,6 +48,7 @@
     "@map-colonies/tracing": "^1.0.0",
     "@map-colonies/tracing-utils": "^2.0.0",
     "@opentelemetry/api": "^1.9.1",
+    "axios": "^1.20.0",
     "compression": "^1.8.0",
     "express": "^4.21.2",
     "express-openapi-validator": "^5.6.2",

src/common/appError.ts

@@ -0,0 +1,20 @@
+import type { StatusCodes } from 'http-status-codes';
+
+export class AppError extends Error {
+  public override readonly name: string;
+  public readonly status: StatusCodes;
+  public readonly isOperational: boolean;
+
+  public constructor(name: string, status: StatusCodes, description: string, isOperational: boolean) {
+    super(description);
+
+    Object.setPrototypeOf(this, new.target.prototype);
+
+    this.name = name;
+    this.status = status;
+    this.isOperational = isOperational;
+    this.message = description;
+
+    Error.captureStackTrace(this);
+  }
+}

src/common/config.ts

@@ -1,21 +1,56 @@
 import { type ConfigInstance, config } from '@map-colonies/config';
 import { commonBoilerplateV3, type commonBoilerplateV3Type } from '@map-colonies/schemas';
 
-// Choose here the type of the config instance and import this type from the entire application
-type ConfigType = ConfigInstance<commonBoilerplateV3Type>;
+interface LookupTablesConfig {
+  url: string;
+  subUrl: string;
+}
+
+interface ExternalServicesConfig {
+  lookupTables: LookupTablesConfig;
+  catalog: string;
+}
+
+type OpsTriggerConfigType = commonBoilerplateV3Type & { externalServices: ExternalServicesConfig };
+
+type ConfigType = ConfigInstance<OpsTriggerConfigType>;
+
+const opsTriggerConfigSchema = {
+  $id: 'https://mapcolonies.com/3d/opsTrigger/v1',
+  type: 'object',
+  allOf: [
+    { $ref: commonBoilerplateV3.$id },
+    {
+      type: 'object',
+      required: ['externalServices'],
+      properties: {
+        externalServices: {
+          type: 'object',
+          required: ['lookupTables', 'catalog'],
+          properties: {
+            lookupTables: {
+              type: 'object',
+              required: ['url', 'subUrl'],
+              properties: {
+                url: { type: 'string' },
+                subUrl: { type: 'string' },
+              },
+            },
+            catalog: { type: 'string' },
+          },
+        },
+      },
+    },
+  ],
+};
 
 let configInstance: ConfigType | undefined;
 
-/**
- * Initializes the configuration by fetching it from the server.
- * This should only be called from the instrumentation file.
- * @returns A Promise that resolves when the configuration is successfully initialized.
- */
 async function initConfig(offlineMode?: boolean): Promise<void> {
-  configInstance = await config({
-    schema: commonBoilerplateV3,
+  configInstance = (await config({
+    schema: opsTriggerConfigSchema as unknown as typeof commonBoilerplateV3,
     offlineMode,
-  });
+  })) as unknown as ConfigType;
 }
 
 function getConfig(): ConfigType {
@@ -26,4 +61,4 @@ function getConfig(): ConfigType {
 }
 
 export { getConfig, initConfig };
-export type { ConfigType };
+export type { ConfigType, LookupTablesConfig, ExternalServicesConfig };

src/externalServices/catalog/catalogCall.ts

@@ -0,0 +1,53 @@
+import axios from 'axios';
+import { inject, injectable } from 'tsyringe';
+import type { Logger } from '@map-colonies/js-logger';
+import { StatusCodes } from 'http-status-codes';
+import { SERVICES } from '@common/constants';
+import { AppError } from '@common/appError';
+import type { ConfigType } from '@common/config';
+import type { LogContext } from '@common/interfaces';
+import type { IFindRecordsPayload, Record3D } from './interfaces';
+
+@injectable()
+export class CatalogCall {
+  private readonly logContext: LogContext;
+  private readonly catalog: string;
+
+  public constructor(
+    @inject(SERVICES.CONFIG) private readonly config: ConfigType,
+    @inject(SERVICES.LOGGER) private readonly logger: Logger
+  ) {
+    this.catalog = this.config.get('externalServices.catalog');
+    this.logContext = {
+      fileName: __filename,
+      class: CatalogCall.name,
+    };
+  }
+
+  public async findRecords(payload: IFindRecordsPayload): Promise<Record3D[]> {
+    const logContext = { ...this.logContext, function: this.findRecords.name };
+    this.logger.debug({ msg: `Searching in catalog ${this.catalog}/metadata/find`, logContext, payload });
+    try {
+      const response = await axios.post<Record3D[]>(`${this.catalog}/metadata/find`, payload);
+      if (response.status !== StatusCodes.OK.valueOf()) {
+        this.logger.error({ msg: `Catalog returned unexpected status: ${response.status}`, logContext });
+        throw new AppError('catalog', StatusCodes.INTERNAL_SERVER_ERROR, 'Problem with catalog during record lookup', true);
+      }
+
+      const records = response.data;
+      if (!Array.isArray(records)) {
+        return [];
+      }
+
+      this.logger.debug({ msg: `Found ${records.length} record(s) in catalog`, logContext });
+      return records;
+    } catch (err) {
+      if (err instanceof AppError) {
+        throw err;
+      }
+
+      this.logger.error({ msg: 'Something went wrong in catalog when trying to find records', logContext, err });
+      throw new AppError('catalog', StatusCodes.INTERNAL_SERVER_ERROR, 'Problem with catalog find', true);
+    }
+  }
+}

src/externalServices/catalog/interfaces.ts

@@ -0,0 +1,17 @@
+export interface Record3D {
+  id: string;
+  productId?: string;
+  productName?: string;
+  productVersion?: number;
+  productStatus?: string;
+  links?: string;
+}
+
+export interface IFindRecordsPayload {
+  id?: string;
+  productId?: string;
+  productName?: string;
+  productType?: string;
+  classification?: string;
+  productStatus?: string;
+}

src/externalServices/lookupTables/interfaces.ts

@@ -0,0 +1,5 @@
+export interface ILookupOption {
+  value: string;
+  translationCode: string;
+  properties?: Record<string, unknown>;
+}