From 4dd247074dcccf5a3b9466752e14554d90765207 Mon Sep 17 00:00:00 2001 From: shimoncohen Date: Thu, 3 Sep 2026 11:03:47 +0300 Subject: [PATCH 01/25] docs: geotiff heights migration implementation plan Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01ATQerkHAV3Vur82HVVFack --- .../2026-09-03-geotiff-heights-migration.md | 1055 +++++++++++++++++ 1 file changed, 1055 insertions(+) create mode 100644 docs/superpowers/plans/2026-09-03-geotiff-heights-migration.md diff --git a/docs/superpowers/plans/2026-09-03-geotiff-heights-migration.md b/docs/superpowers/plans/2026-09-03-geotiff-heights-migration.md new file mode 100644 index 0000000..ede51df --- /dev/null +++ b/docs/superpowers/plans/2026-09-03-geotiff-heights-migration.md @@ -0,0 +1,1055 @@ +# GeoTIFF Heights Migration Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Replace the Cesium quantized-mesh terrain engine in `dem-heights` with direct COG (Cloud-Optimized GeoTIFF) sampling read over HTTP range requests from the S3 gateway, keeping the `/points` API contract identical. + +**Architecture:** Positions flow as WGS84 **degrees** end-to-end (no more radians round-trip). Discovery stays on CSW but filters for a `GEOTIFF` link instead of `TERRAIN_QMESH`. `DEMTerrainCacheManager` opens one `GeotiffHeightProvider` per catalog record via `geotiff.js` `fromUrl` (range reads through the gateway, token auth). `HeightsManager` selects a provider per point (product-type + footprint + resolution), groups points by provider, and samples heights via bilinear interpolation, mapping the tiff nodata value (`-32768`) and out-of-footprint points to `null`. Cesium is removed entirely. + +**Tech Stack:** TypeScript (CommonJS build, `target es2021`), Node 20, Express, tsyringe DI, `geotiff@^3.0.5` (ships a CJS entry — plain `import` works), `@turf/boolean-point-in-polygon`, Jest 28, `config`. + +--- + +## Background facts (verified before this plan) + +- Cesium coupling lives in: `DEMTerrainCacheManager.ts`, `heightsManager.ts`, `utilities.ts`, `interfaces.ts`, plus the two radian middlewares (`dataToRadians.ts`, `dataToDegrees.ts`) and their consumers (`controller`, `validateRequest`, `router`, tests). +- The radian middlewares exist ONLY because Cesium works in radians. GeoTIFF geotransform math is in degrees, so both middlewares and `radiansToOriginalPositionsMap` are deleted. +- Live env (verified on OCP `dem-dev`): `nginx-s3-gateway` proxies S3 **Range** (`206`/`Content-Range`), OPA auth passes with token as **queryParam** `?token=`. COGs already uploaded at MinIO bucket `dem-dev` under prefix `cogs/`. geotiff.js `fromUrl` through the internal gateway opened + sampled a real COG (`206.99 m`). +- Use the `tiled_ovr` COG (native 30 m, tiled 256, 13 overviews) — NOT the coarse `_COG` variant. +- Tiff nodata = `-32768` on all GEO tiffs. +- Catalog `productType` values look like `QUANTIZED_MESH_DTM_BEST`; selection uses `.includes('DTM'|'DSM')`, so it keeps working regardless of exact string. + +## File Structure + +**Create** +- `src/heights/models/geotiffHeightProvider.ts` — opens a COG and samples heights (bilinear + nodata). One responsibility: raster → height. +- `tests/unit/heights/models/geotiffHeightProvider.spec.ts` — unit tests for sampling math via a mocked `geotiff` module. + +**Modify** +- `src/heights/interfaces.ts` — drop Cesium types; add `GeoPoint`, `PosWithProvider`, `HeightProviders`; keep `TerrainTypes`, `PosWithHeight`. +- `src/heights/models/DEMTerrainCacheManager.ts` — build `GeotiffHeightProvider`s from `GEOTIFF` links (filename + DI symbol unchanged to limit churn). +- `src/heights/models/heightsManager.ts` — provider selection in degrees, group-by-provider, bilinear sampling; remove Cesium + tile clustering + density throw. +- `src/heights/utilities.ts` — delete tile-clustering; keep `generateChecksum` + `isSame`. +- `src/workerCatalogRecords.ts` — CSW filter `TERRAIN_QMESH` → `GEOTIFF`. +- `src/containerConfig.ts` — call `initProviders` (renamed) instead of `initTerrainProviders`. +- `src/heights/controllers/heightsController.ts` — request positions typed `GeoPoint[]`; drop `radiansToOriginalPositionsMap`. +- `src/heights/middlewares/validateRequest.ts` — type `GeoPoint[]`; drop Cesium import. +- `src/heights/routes/heightsRouter.ts` — remove the two radian middlewares from the chain. +- `package.json` — remove `cesium`, add `geotiff`. +- `tests/configurations/testContainerConfig.ts` — records use `GEOTIFF` links; mock `GeotiffHeightProvider.fromUrl`. +- `tests/unit/heights/models/heightModel.spec.ts` — degrees input; mock geotiff provider. +- `tests/unit/heights/middlewares/heightsMiddlewares.spec.ts` — remove radian/degree middleware tests. +- `tests/integration/heights/heights.spec.ts` — remove Cesium spy. + +**Delete** +- `src/heights/middlewares/dataToRadians.ts` +- `src/heights/middlewares/dataToDegrees.ts` + +## Conventions for every task + +- Run a single unit spec: `npx jest --config=./tests/configurations/unit/jest.config.js ` +- Run all unit: `npm run test:unit` +- Run integration: `npm run test:integration` +- Type-check/build: `npm run build` +- Lint fix: `npm run lint:fix` +- Node must be 20: `nvm use 20` first (config@3.3.7 crashes on Node 24). +- Commit after each task. Conventional Commits: this migration is `feat:`/`refactor:`; dependency swap is `build:` (per repo convention, NOT `chore:`). End commit messages with the two trailer lines required by the repo (Co-Authored-By + Claude-Session). + +--- + +### Task 1: Add geotiff dependency, remove cesium + +**Files:** +- Modify: `package.json` + +- [ ] **Step 1: Swap the dependency** + +Run: +```bash +nvm use 20 +npm uninstall cesium +npm install geotiff@^3.0.5 +``` + +- [ ] **Step 2: Verify geotiff resolves as CJS** + +Run: `node -e "console.log(typeof require('geotiff').fromUrl)"` +Expected: `function` + +- [ ] **Step 3: Verify cesium is gone from the tree** + +Run: `node -e "try{require('cesium');console.log('STILL PRESENT')}catch(e){console.log('removed')}"` +Expected: `removed` + +- [ ] **Step 4: Commit** + +```bash +git add package.json package-lock.json +git commit -m "build: replace cesium terrain engine with geotiff" +``` + +--- + +### Task 2: New coordinate + provider types + +**Files:** +- Modify: `src/heights/interfaces.ts` + +- [ ] **Step 1: Replace the file contents** + +```typescript +import type GeotiffHeightProvider from './models/geotiffHeightProvider'; + +export enum TerrainTypes { + DSM = 'DSM', + DTM = 'DTM', + MIXED = 'MIXED', +} + +export interface GeoPoint { + longitude: number; // WGS84 degrees + latitude: number; // WGS84 degrees + height?: number | null; +} + +export interface PosWithHeight extends GeoPoint { + height: number | null; + productId?: string; +} + +export interface PosWithProvider extends GeoPoint { + providerKey?: string; +} + +export type HeightProviders = Record; +``` + +- [ ] **Step 2: Expect type errors elsewhere (fine for now)** + +Run: `npx tsc --noEmit -p tsconfig.json || true` +Expected: errors only in files that still import removed symbols (`PosWithTerrainProvider`, `TerrainProviders`, `cesium`). These are fixed in later tasks. The type import of `GeotiffHeightProvider` will error until Task 3 creates the file — that is expected. + +- [ ] **Step 3: Commit** + +```bash +git add src/heights/interfaces.ts +git commit -m "refactor: degrees-based GeoPoint and height provider types" +``` + +--- + +### Task 3: GeotiffHeightProvider — sampling engine (TDD) + +**Files:** +- Create: `src/heights/models/geotiffHeightProvider.ts` +- Test: `tests/unit/heights/models/geotiffHeightProvider.spec.ts` + +- [ ] **Step 1: Write the failing test** + +```typescript +import GeotiffHeightProvider from '../../../../src/heights/models/geotiffHeightProvider'; + +jest.mock('geotiff', () => ({ + fromUrl: jest.fn(), +})); + +// eslint-disable-next-line @typescript-eslint/no-var-requires +const { fromUrl } = require('geotiff') as { fromUrl: jest.Mock }; + +// origin [34,33], resolution [0.001,-0.001], 100x100, nodata -32768 +const makeImage = (band: number[], nodata: number | null = -32768) => ({ + getOrigin: () => [34.0, 33.0, 0], + getResolution: () => [0.001, -0.001], + getWidth: () => 100, + getHeight: () => 100, + getGDALNoData: () => nodata, + readRasters: jest.fn().mockResolvedValue([Int16Array.from(band)]), +}); + +const mockTiff = (image: unknown): void => { + fromUrl.mockResolvedValue({ getImage: jest.fn().mockResolvedValue(image) }); +}; + +describe('GeotiffHeightProvider', () => { + afterEach(() => jest.clearAllMocks()); + + it('bilinearly interpolates the height at a point', async () => { + // point at pixel (0,0)+0.5 in both axes → center of the 2x2 window + mockTiff(makeImage([200, 202, 204, 206])); + const provider = await GeotiffHeightProvider.fromUrl('http://gw/cogs/x.tif'); + + const [height] = await provider.sample([{ longitude: 34.0005, latitude: 32.9995 }]); + + expect(height).toBeCloseTo(203, 5); + }); + + it('returns null when any neighbor is nodata', async () => { + mockTiff(makeImage([200, -32768, 204, 206])); + const provider = await GeotiffHeightProvider.fromUrl('http://gw/cogs/x.tif'); + + const [height] = await provider.sample([{ longitude: 34.0005, latitude: 32.9995 }]); + + expect(height).toBeNull(); + }); + + it('returns null for a point outside the raster', async () => { + mockTiff(makeImage([200, 202, 204, 206])); + const provider = await GeotiffHeightProvider.fromUrl('http://gw/cogs/x.tif'); + + const [height] = await provider.sample([{ longitude: 10.0, latitude: 10.0 }]); + + expect(height).toBeNull(); + }); + + it('passes token headers through to fromUrl', async () => { + mockTiff(makeImage([1, 1, 1, 1])); + await GeotiffHeightProvider.fromUrl('http://gw/cogs/x.tif', { 'x-api-key': 'T' }); + + expect(fromUrl).toHaveBeenCalledWith('http://gw/cogs/x.tif', { headers: { 'x-api-key': 'T' } }); + }); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `npx jest --config=./tests/configurations/unit/jest.config.js geotiffHeightProvider` +Expected: FAIL — `Cannot find module '.../geotiffHeightProvider'`. + +- [ ] **Step 3: Create the implementation** + +```typescript +import { fromUrl } from 'geotiff'; +import type { GeoTIFFImage } from 'geotiff'; +import { GeoPoint } from '../interfaces'; + +const DEFAULT_NODATA = -32768; + +export default class GeotiffHeightProvider { + private constructor( + private readonly image: GeoTIFFImage, + private readonly originX: number, + private readonly originY: number, + private readonly pixelWidth: number, // degrees/pixel, positive + private readonly pixelHeight: number, // degrees/pixel, negative + private readonly rasterWidth: number, + private readonly rasterHeight: number, + private readonly noData: number + ) {} + + public static async fromUrl(url: string, headers?: Record): Promise { + const tiff = await fromUrl(url, headers ? { headers } : {}); + const image = await tiff.getImage(0); + const [originX, originY] = image.getOrigin(); + const [pixelWidth, pixelHeight] = image.getResolution(); + const noData = image.getGDALNoData() ?? DEFAULT_NODATA; + + return new GeotiffHeightProvider(image, originX, originY, pixelWidth, pixelHeight, image.getWidth(), image.getHeight(), noData); + } + + public async sample(points: GeoPoint[]): Promise<(number | null)[]> { + return Promise.all(points.map(async (point) => this.sampleOne(point))); + } + + private async sampleOne(point: GeoPoint): Promise { + const fx = (point.longitude - this.originX) / this.pixelWidth; + const fy = (point.latitude - this.originY) / this.pixelHeight; + const px = Math.floor(fx); + const py = Math.floor(fy); + + // Need a full 2x2 neighborhood for bilinear interpolation. + if (px < 0 || py < 0 || px >= this.rasterWidth - 1 || py >= this.rasterHeight - 1) { + return null; + } + + const dx = fx - px; + const dy = fy - py; + + const raster = (await this.image.readRasters({ window: [px, py, px + 2, py + 2] })) as unknown as number[][]; + const band = raster[0]; + const [tl, tr, bl, br] = [band[0], band[1], band[2], band[3]]; + + if ([tl, tr, bl, br].some((value) => value === this.noData)) { + return null; + } + + const top = tl + (tr - tl) * dx; + const bottom = bl + (br - bl) * dx; + + return top + (bottom - top) * dy; + } +} +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `npx jest --config=./tests/configurations/unit/jest.config.js geotiffHeightProvider` +Expected: PASS (4 tests). + +- [ ] **Step 5: Commit** + +```bash +git add src/heights/models/geotiffHeightProvider.ts tests/unit/heights/models/geotiffHeightProvider.spec.ts +git commit -m "feat: geotiff COG height provider with bilinear sampling" +``` + +--- + +### Task 4: DEMTerrainCacheManager builds geotiff providers + +**Files:** +- Modify: `src/heights/models/DEMTerrainCacheManager.ts` + +Rationale for keeping the filename and DI symbol: the symbol is only an injection token; keeping both avoids churn across `containerConfig.ts` and every test import. + +- [ ] **Step 1: Replace the file contents** + +```typescript +import { inject, injectable } from 'tsyringe'; +import { IConfig } from 'config'; +import { PycswDemCatalogRecord } from '@map-colonies/mc-model-types'; +import { HeightProviders } from '../interfaces'; +import { SERVICES } from '../../common/constants'; +import GeotiffHeightProvider from './geotiffHeightProvider'; + +const GEOTIFF_PROTOCOL = 'GEOTIFF'; +const COGS_FOLDER = 'cogs/'; + +@injectable() +export default class DEMTerrainCacheManager { + public heightProviders: HeightProviders = {}; + + public constructor(@inject(SERVICES.CONFIG) private readonly config: IConfig) {} + + public async initProviders(demCatalogRecords: PycswDemCatalogRecord[]): Promise { + const heightProviders: HeightProviders = {}; + + const geotiffRecords = demCatalogRecords.filter((record) => record.links?.some((link) => link.protocol === GEOTIFF_PROTOCOL)); + + for (const record of geotiffRecords) { + const link = record.links?.find((currentLink) => currentLink.protocol === GEOTIFF_PROTOCOL); + + if (link) { + const objectUrl = this.transformRouteToObjectUrl(link.url as string); + const { url, headers } = this.buildAuthenticatedUrl(objectUrl); + heightProviders[record.id as string] = await GeotiffHeightProvider.fromUrl(url, headers); + } + } + + this.heightProviders = heightProviders; + } + + private transformRouteToObjectUrl(linkUrl: string): string { + const serviceURL = this.config.get('s3Gateway.url'); + + return `${serviceURL}/${COGS_FOLDER}${linkUrl.split(COGS_FOLDER)[1]}`; + } + + private buildAuthenticatedUrl(objectUrl: string): { url: string; headers?: Record } { + const injectionType = this.config.get('accessToken.injectionType'); + const attributeName = this.config.get('accessToken.attributeName'); + const tokenValue = this.config.get('accessToken.tokenValue'); + + if (injectionType.toLowerCase() === 'header') { + return { url: objectUrl, headers: { [attributeName]: tokenValue } }; + } + + if (injectionType.toLowerCase() === 'queryparam') { + const separator = objectUrl.includes('?') ? '&' : '?'; + return { url: `${objectUrl}${separator}${attributeName}=${encodeURIComponent(tokenValue)}` }; + } + + return { url: objectUrl }; + } +} +``` + +- [ ] **Step 2: Update the DI wiring caller** + +In `src/containerConfig.ts`, change the one call inside the worker `updateValue` handler from: +```typescript + await demTerrainCacheManager.initTerrainProviders(dataValue); +``` +to: +```typescript + await demTerrainCacheManager.initProviders(dataValue); +``` + +- [ ] **Step 3: Build to type-check these two files** + +Run: `npm run build` +Expected: `heightsManager.ts` still errors (fixed next task); `DEMTerrainCacheManager.ts` and `containerConfig.ts` compile clean. + +- [ ] **Step 4: Commit** + +```bash +git add src/heights/models/DEMTerrainCacheManager.ts src/containerConfig.ts +git commit -m "feat: open geotiff providers from GEOTIFF catalog links" +``` + +--- + +### Task 5: HeightsManager samples via geotiff (TDD) + +**Files:** +- Modify: `src/heights/models/heightsManager.ts` +- Modify: `tests/configurations/testContainerConfig.ts` +- Modify: `tests/unit/heights/models/heightModel.spec.ts` + +- [ ] **Step 1: Update the test container to GEOTIFF links and mock the provider** + +In `tests/configurations/testContainerConfig.ts`: + +1. Add imports at the top (after existing imports): +```typescript +import GeotiffHeightProvider from '../../src/heights/models/geotiffHeightProvider'; +``` + +2. In BOTH catalog records, change the `links` entry `protocol` and `url`: +```typescript + links: [ + { + __typename: 'Link', + name: '', + description: '', + protocol: 'GEOTIFF', + url: 'https://tiles-dev.mapcolonies.net/api/dem/v1/cogs/combined_srtm_30_100_il_ever.tif', + }, + ], +``` +(For the second record use `.../cogs/srtm100.tif`.) + +3. Immediately before the `await (async ...)` IIFE, install the provider mock so `initProviders` does not hit the network: +```typescript + jest.spyOn(GeotiffHeightProvider, 'fromUrl').mockResolvedValue({ + sample: async (points: { longitude: number; latitude: number }[]) => points.map(() => 100), + } as unknown as GeotiffHeightProvider); +``` + +4. Change the init call: +```typescript + if (shouldInitTerrainProviders) { + await demTestTerrainCacheManager.initProviders(demTestCatalogRecords as unknown as PycswDemCatalogRecord[]); + } +``` + +- [ ] **Step 2: Rewrite the model unit test for degrees input** + +Replace the entire contents of `tests/unit/heights/models/heightModel.spec.ts`: + +```typescript +import { container } from 'tsyringe'; +import { HeightsManager } from '../../../../src/heights/models/heightsManager'; +import mockJsonPoints, { positionsOutsideOfProviders, emptyPositionsRequest } from '../../../../src/heights/MOCKS/mockData'; +import { GetHeightsPointsRequest } from '../../../../src/heights/controllers/heightsController'; +import { PosWithHeight, TerrainTypes } from '../../../../src/heights/interfaces'; +import { registerTestValues } from '../../../configurations/testContainerConfig'; + +describe('Get Heights model', function () { + const mockJsonData = mockJsonPoints as GetHeightsPointsRequest; + const mockJsonDataOutOfBounds = positionsOutsideOfProviders as GetHeightsPointsRequest; + + let heightsManager: HeightsManager; + + beforeEach(async function () { + await registerTestValues(); + heightsManager = container.resolve(HeightsManager); + }); + + afterEach(() => { + container.reset(); + container.clearInstances(); + jest.clearAllMocks(); + }); + + describe('Given valid parameters', function () { + it('Should return positions with height and productId', async function () { + const result = await heightsManager.getPoints(mockJsonData.positions, TerrainTypes.MIXED); + + expect(result).toHaveLength(mockJsonData.positions.length); + for (const position of result) { + expect(position.height).toBe(100); + expect(position.productId).toBeDefined(); + } + }); + + it('Should return null heights and no productId when no provider matches the product type', async function () { + const result = await heightsManager.getPoints(mockJsonData.positions, TerrainTypes.DSM); + + expect(result).toHaveLength(mockJsonData.positions.length); + for (const position of result) { + expect(position.height).toBeNull(); + expect(position.productId).toBeUndefined(); + } + }); + + it('Should return height only for the positions inside a provider footprint', async function () { + const result = await heightsManager.getPoints(mockJsonDataOutOfBounds.positions, TerrainTypes.MIXED); + + expect(result).toHaveLength(mockJsonDataOutOfBounds.positions.length); + for (const position of result) { + expect(position.longitude).toBeDefined(); + expect(position.latitude).toBeDefined(); + + const isNullHeight = (position.height as number | null) === null; + expect(typeof position.productId === 'undefined').toEqual(isNullHeight); + } + }); + }); + + describe('Given invalid params', function () { + it('Should return empty array for empty positions', async function () { + await expect(heightsManager.getPoints((emptyPositionsRequest as unknown as GetHeightsPointsRequest).positions, TerrainTypes.MIXED)).resolves.toEqual([]); + }); + }); +}); +``` + +Note: `mockData.ts` already exports plain `{ longitude, latitude }` objects (degrees), so no `Cartographic.fromDegrees` conversion is needed anymore. + +- [ ] **Step 3: Run the test to verify it fails** + +Run: `npx jest --config=./tests/configurations/unit/jest.config.js heightModel` +Expected: FAIL — `heightsManager.ts` still imports `cesium`/`sampleTerrainMostDetailed`; compile/runtime error. + +- [ ] **Step 4: Rewrite `heightsManager.ts`** + +Replace the entire contents of `src/heights/models/heightsManager.ts`: + +```typescript +import { Polygon } from 'geojson'; +import client from 'prom-client'; +import { container, inject, injectable } from 'tsyringe'; +import { Logger } from '@map-colonies/js-logger'; +import PromisePool from '@supercharge/promise-pool/dist'; +import booleanPointInPolygon from '@turf/boolean-point-in-polygon'; +import { Feature } from '@turf/turf'; +import { CommonErrors } from '../../common/commonErrors'; +import { SERVICES } from '../../common/constants'; +import { IConfig } from '../../common/interfaces'; +import { CATALOG_RECORDS_MAP, DEM_TERRAIN_CACHE_MANAGER } from '../../containerConfig'; +import { GeoPoint, PosWithHeight, PosWithProvider, TerrainTypes } from '../interfaces'; +import DEMTerrainCacheManager from './DEMTerrainCacheManager'; +import { CatalogRecords } from './catalogRecords'; + +export interface ICoordinates { + longitude: string; + latitude: string; +} + +export interface IHeightModel { + dem: number; +} + +@injectable() +export class HeightsManager { + private runningRequests = 0; + + private get catalogRecordsMap() { + return container.resolve(CATALOG_RECORDS_MAP).getValue(); + } + + private get heightProviders() { + return container.resolve(DEM_TERRAIN_CACHE_MANAGER).heightProviders; + } + + private readonly elevationsRequestsCounter?: client.Counter<'points_number'>; + + public constructor( + @inject(SERVICES.LOGGER) private readonly logger: Logger, + @inject(CommonErrors) private readonly commonErrors: CommonErrors, + @inject(SERVICES.CONFIG) private readonly config: IConfig, + @inject(SERVICES.METRICS_REGISTRY) registry?: client.Registry + ) { + if (registry !== undefined) { + // eslint-disable-next-line @typescript-eslint/no-this-alias + const self = this; + new client.Gauge({ + name: 'elevations_current_requests_count', + help: 'Currently running elevations requests', + collect(): void { + this.set(self.runningRequests); + }, + registers: [registry], + }); + + this.elevationsRequestsCounter = new client.Counter({ + name: 'elevations_requests_total', + help: 'Total elevations requests', + labelNames: ['points_number'] as const, + registers: [registry], + }); + } + } + + public async getPoints(points: GeoPoint[], requestedProductType: TerrainTypes, reqCtx?: Record): Promise { + this.logger.info({ pointsNumber: points.length, location: '[HeightsManager] [getPoints]', ...reqCtx }); + + this.runningRequests++; + // eslint-disable-next-line @typescript-eslint/naming-convention + this.elevationsRequestsCounter?.inc({ points_number: points.length }); + + if (points.length === 0) { + this.runningRequests--; + return []; + } + + const result = await this.samplePositionsHeights(points, requestedProductType, reqCtx); + + this.logger.info({ + totalRequests: result.totalRequests, + pointsNumber: points.length, + location: '[HeightsManager] [getPoints]', + ...reqCtx, + }); + + this.runningRequests--; + return result.positions; + } + + private async samplePositionsHeights( + positionsArr: GeoPoint[], + requestedProductType: TerrainTypes, + reqCtx?: Record + ): Promise<{ positions: PosWithHeight[]; totalRequests: number }> { + const attachProviderStart = performance.now(); + const positionsWithProviders = this.attachProviderToPositions(positionsArr, requestedProductType); + this.logger.info({ + attachProviderTime: performance.now() - attachProviderStart, + pointsNumber: positionsArr.length, + location: '[HeightsManager] [samplePositionsHeights]', + ...reqCtx, + }); + + // Group points by the provider chosen for them (null = no provider). + const groups = new Map(); + for (const position of positionsWithProviders) { + const key = position.providerKey ?? null; + const bucket = groups.get(key) ?? []; + bucket.push({ longitude: position.longitude, latitude: position.latitude }); + groups.set(key, bucket); + } + + const groupEntries = [...groups.entries()]; + const finalPositionsWithHeights: PosWithHeight[] = []; + + const { results } = await PromisePool.for(groupEntries) + .withConcurrency(Math.max(1, groupEntries.length)) + .process(async ([providerKey, points]) => { + if (providerKey === null) { + return points.map((point) => ({ ...point, height: null } as PosWithHeight)); + } + + const samplingStart = performance.now(); + const provider = this.heightProviders[providerKey]; + const record = this.catalogRecordsMap[providerKey]; + const heights = await provider.sample(points); + + this.logger.info({ + terrainSamplingTime: performance.now() - samplingStart, + providerId: providerKey, + pointsNumber: positionsArr.length, + location: '[HeightsManager] [samplePositionsHeights]', + ...reqCtx, + }); + + return points.map((point, index) => { + const height = heights[index]; + return { + ...point, + height, + ...(height !== null ? { productId: record.productId as string } : {}), + } as PosWithHeight; + }); + }); + + finalPositionsWithHeights.push(...(results as PosWithHeight[][]).flat()); + + return { positions: finalPositionsWithHeights, totalRequests: groupEntries.length }; + } + + private attachProviderToPositions(positions: GeoPoint[], requestedProductType: TerrainTypes): PosWithProvider[] { + /* + * Filter providers by requested product type (unless MIXED) + * Filter providers by footprint point intersection + * Sort by highest resolution (lower resolutionMeter is better), tie-break on newest updateDate + * Attach the best provider key to the point + */ + return positions.map((position) => { + const providerEntries = Object.entries(this.heightProviders); + + const productTypeFiltered = + requestedProductType !== TerrainTypes.MIXED + ? providerEntries.filter(([key]) => this.catalogRecordsMap[key].productType?.includes(requestedProductType)) + : providerEntries; + + const footprintFiltered = productTypeFiltered.filter(([key]) => + booleanPointInPolygon([position.longitude, position.latitude], this.catalogRecordsMap[key].footprint as Feature) + ); + + const sorted = footprintFiltered.sort(([aKey], [bKey]) => { + const A_BEFORE_B = -1; + const B_BEFORE_A = 1; + const recordA = this.catalogRecordsMap[aKey]; + const recordB = this.catalogRecordsMap[bKey]; + + switch (true) { + case (recordA.resolutionMeter as number) < (recordB.resolutionMeter as number): + return A_BEFORE_B; + case (recordA.resolutionMeter as number) > (recordB.resolutionMeter as number): + return B_BEFORE_A; + default: { + const dateB = new Date(recordB.updateDate as string | number | Date); + const dateA = new Date(recordA.updateDate as string | number | Date); + return dateB.getTime() - dateA.getTime(); + } + } + }); + + if (sorted.length === 0) { + return { ...position } as PosWithProvider; + } + + const [providerKey] = sorted[0]; + return { ...position, providerKey } as PosWithProvider; + }); + } +} +``` + +Notes: +- The `maximumTilesPerRequest` density throw and `RequestScheduler.update()` are intentionally removed — both were quantized-mesh tile concerns. Point-count limiting stays in `validateRequestMiddleware`. +- `commonErrors` stays injected (used elsewhere in the class surface / future use); leaving the constructor param avoids DI signature churn. + +- [ ] **Step 5: Run the model test to verify it passes** + +Run: `npx jest --config=./tests/configurations/unit/jest.config.js heightModel` +Expected: PASS (4 tests). + +- [ ] **Step 6: Commit** + +```bash +git add src/heights/models/heightsManager.ts tests/configurations/testContainerConfig.ts tests/unit/heights/models/heightModel.spec.ts +git commit -m "feat: sample point heights from geotiff providers in degrees" +``` + +--- + +### Task 6: Trim utilities to checksum helpers only + +**Files:** +- Modify: `src/heights/utilities.ts` + +- [ ] **Step 1: Replace the file contents** + +```typescript +import crypto from 'crypto'; + +export const generateChecksum = (str: string, algorithm?: string, encoding?: crypto.BinaryToTextEncoding): string => { + return crypto + .createHash(algorithm ?? 'md5') + .update(str, 'utf8') + .digest(encoding ?? 'hex'); +}; + +// eslint-disable-next-line @typescript-eslint/no-explicit-any +export const isSame = (src1: any, src2: any): boolean => { + return generateChecksum(JSON.stringify(src1)) === generateChecksum(JSON.stringify(src2)); +}; +``` + +- [ ] **Step 2: Confirm no remaining imports of the removed clustering function** + +Run: `grep -rn "cartographicArrayClusteringForHeightRequests\|PositionsWithProviderKey" src tests` +Expected: no matches. + +- [ ] **Step 3: Commit** + +```bash +git add src/heights/utilities.ts +git commit -m "refactor: drop quantized-mesh tile clustering utilities" +``` + +--- + +### Task 7: CSW discovery filters GEOTIFF links + +**Files:** +- Modify: `src/workerCatalogRecords.ts` + +- [ ] **Step 1: Change the CSW filter** + +In `src/workerCatalogRecords.ts`, replace the `mc:links` filter value: +```typescript + { + field: 'mc:links', + like: 'GEOTIFF', + }, +``` +and update the two comment lines above it to reference `GEOTIFF` instead of `TERRAIN_PROVIDER`/`TERRAIN_QMESH`: +```typescript + // ******* DEM profile links carry the object protocol. We match records exposing a GEOTIFF link + // ******* (COG served from the S3 gateway) via a LIKE filter on the LINKS field. +``` + +- [ ] **Step 2: Build to type-check** + +Run: `npm run build` +Expected: compiles (any remaining errors are in controller/middleware/router — fixed next tasks). + +- [ ] **Step 3: Commit** + +```bash +git add src/workerCatalogRecords.ts +git commit -m "feat: discover GEOTIFF catalog records instead of qmesh" +``` + +--- + +### Task 8: Controller + validate types to degrees; drop radians map + +**Files:** +- Modify: `src/heights/controllers/heightsController.ts` +- Modify: `src/heights/middlewares/validateRequest.ts` + +- [ ] **Step 1: Update the controller request type** + +In `src/heights/controllers/heightsController.ts`: + +1. Replace the cesium import line: +```typescript +import { Cartographic } from 'cesium'; +``` +with: +```typescript +import { GeoPoint } from '../interfaces'; +``` + +2. Change the request interface (remove `radiansToOriginalPositionsMap`, retype `positions`): +```typescript +export interface GetHeightsPointsRequest { + positions: GeoPoint[]; + productType?: TerrainTypes; +} +``` + +`PosWithHeight` and `TerrainTypes` are already imported from `../interfaces`. The `getPoints` handler body is unchanged (it already passes `userInput.positions`). + +- [ ] **Step 2: Update validateRequest** + +In `src/heights/middlewares/validateRequest.ts`: + +1. Replace the cesium import: +```typescript +import { Cartographic } from 'cesium'; +``` +with: +```typescript +import { GeoPoint } from '../interfaces'; +``` + +2. Change the points type: +```typescript + const points: GeoPoint[] = req.body.positions; +``` + +- [ ] **Step 3: Build to type-check these files** + +Run: `npm run build` +Expected: only router + deleted-middleware errors remain (next task). + +- [ ] **Step 4: Commit** + +```bash +git add src/heights/controllers/heightsController.ts src/heights/middlewares/validateRequest.ts +git commit -m "refactor: type points as degrees, drop radians request map" +``` + +--- + +### Task 9: Remove radian middlewares from the route + +**Files:** +- Delete: `src/heights/middlewares/dataToRadians.ts` +- Delete: `src/heights/middlewares/dataToDegrees.ts` +- Modify: `src/heights/routes/heightsRouter.ts` + +- [ ] **Step 1: Delete the two middlewares** + +Run: +```bash +git rm src/heights/middlewares/dataToRadians.ts src/heights/middlewares/dataToDegrees.ts +``` + +- [ ] **Step 2: Update the router** + +In `src/heights/routes/heightsRouter.ts`: + +1. Remove these two imports: +```typescript +import { positionResAsDegreesMiddleware } from '../middlewares/dataToDegrees'; +import { convertReqPositionToRadiansMiddleware } from '../middlewares/dataToRadians'; +``` + +2. Change the `router.post('/points', ...)` chain to drop both middlewares — final chain: +```typescript + router.post( + '/points', + createReqCtxMiddleware(logger), + validateRequestMiddleware(config, logger, commonErrors), + controller.getPoints, + addProductsDictionaryMiddleware(logger, productMetadataFields), + sendResponseMiddleware(logger) + ); +``` + +- [ ] **Step 3: Full build** + +Run: `npm run build` +Expected: PASS — no compile errors anywhere. + +- [ ] **Step 4: Confirm cesium is fully gone from source** + +Run: `grep -rn "cesium\|Cartographic\|radians" src` +Expected: no matches. + +- [ ] **Step 5: Commit** + +```bash +git add -A +git commit -m "refactor: remove radian conversion middlewares from points route" +``` + +--- + +### Task 10: Fix the middleware unit test + +**Files:** +- Modify: `tests/unit/heights/middlewares/heightsMiddlewares.spec.ts` + +- [ ] **Step 1: Remove the deleted-middleware tests and imports** + +In `tests/unit/heights/middlewares/heightsMiddlewares.spec.ts`: + +1. Remove these imports: +```typescript +import { Cartographic } from 'cesium'; +import { positionResAsDegreesMiddleware } from '../../../../src/heights/middlewares/dataToDegrees'; +import { convertReqPositionToRadiansMiddleware } from '../../../../src/heights/middlewares/dataToRadians'; +``` + +2. Remove the two now-unused declarations: +```typescript + let dataToRadiansMiddleware: GetHeightsHandler; + let dataToDegreesMiddleware: GetHeightsHandler; +``` + +3. Remove their assignments in `beforeAll`: +```typescript + dataToRadiansMiddleware = convertReqPositionToRadiansMiddleware(logger); + dataToDegreesMiddleware = positionResAsDegreesMiddleware(logger); +``` + +4. Delete the entire `describe('Data to radians middleware', ...)` and `describe('Data to degrees middleware', ...)` blocks. + +The remaining blocks — `Create request id middleware`, `Add products dictionary`, `Validate request middleware` — stay unchanged. (The `PosWithHeight` import becomes unused after deleting the degrees block; remove it too if the linter flags it.) + +- [ ] **Step 2: Run the middleware unit test** + +Run: `npx jest --config=./tests/configurations/unit/jest.config.js heightsMiddlewares` +Expected: PASS (reqCtx, products dictionary, validate empty, validate too-many). + +- [ ] **Step 3: Commit** + +```bash +git add tests/unit/heights/middlewares/heightsMiddlewares.spec.ts +git commit -m "test: drop radian middleware unit tests" +``` + +--- + +### Task 11: Fix the integration test + +**Files:** +- Modify: `tests/integration/heights/heights.spec.ts` + +The integration suite boots the real app (the CSW worker's network fetch fails harmlessly in test, leaving providers empty), so its assertions only require `200` + response shape + the null-height invariants — all of which hold with no providers. It only needs the Cesium spy removed. + +- [ ] **Step 1: Remove Cesium from the integration test** + +In `tests/integration/heights/heights.spec.ts`: + +1. Remove the import: +```typescript +import { Cartesian2, Cartographic, CesiumTerrainProvider } from 'cesium'; +``` + +2. Remove the spy declaration: +```typescript + let cesiumTerrainProviderFromUrlSpy: jest.SpyInstance; +``` + +3. Remove the entire `cesiumTerrainProviderFromUrlSpy = jest.spyOn(...)` + `.mockReturnValue({...})` block inside `beforeAll` (the `availability`/`tilingScheme` mock). + +`basicPositionResponse` is only referenced in commented-out code; leave it or delete it — either compiles. + +- [ ] **Step 2: Run the integration suite** + +Run: `npm run test:integration` +Expected: PASS. All valid-params cases return `200` with correct `data` length; invalid-params return `400` with `TOO_MANY_POINTS_ERROR` / `EMPTY_POSITIONS_ARRAY`. + +- [ ] **Step 3: Commit** + +```bash +git add tests/integration/heights/heights.spec.ts +git commit -m "test: remove cesium terrain mock from heights integration" +``` + +--- + +### Task 12: Full verification pass + +**Files:** none (verification only) + +- [ ] **Step 1: Lint** + +Run: `npm run lint` +Expected: no errors. If any, `npm run lint:fix` then re-run and commit with `style:`. + +- [ ] **Step 2: Full test suite** + +Run: `npm test` +Expected: unit + integration all green. + +- [ ] **Step 3: Build** + +Run: `npm run build` +Expected: clean compile, `dist/` produced. + +- [ ] **Step 4: Confirm no leftover qmesh/cesium references** + +Run: `grep -rn "cesium\|Cartographic\|TERRAIN_QMESH\|sampleTerrainMostDetailed\|radians" src tests` +Expected: no matches. + +- [ ] **Step 5: Commit any lint fixes** + +```bash +git add -A +git commit -m "style: lint fixes for geotiff migration" || echo "nothing to commit" +``` + +--- + +## Out-of-repo deployment tasks (NOT code — do after merge, coordinate with ops) + +These are required for the service to actually serve heights in `dem-dev`, but they are data/infra, not part of this repo's TDD cycle: + +1. **Catalog record** — the DEM CSW record(s) must expose a `GEOTIFF` link whose URL resolves (after the `cogs/` split) to the object key in bucket `dem-dev`, e.g. `https://tiles-dev.mapcolonies.net/api/dem/v1/cogs/dtm_srtm30wgs84geo_tiled256_ovr_lzw.tif`. Today the live record has a `TERRAIN_QMESH` link to `terrains/srtm100`. +2. **COG upload** — ensure the `tiled_ovr` COG (native 30 m, tiled, overviews) is the object served — NOT the coarse `_COG`. SRTM100 COG is not yet under `cogs/`; upload if 100 m coverage is required. +3. **helm-charts** — no chart change is strictly required (the service reads `s3Gateway.url` + `accessToken`, both already set to the internal gateway with token as queryParam). Confirm the gateway route/prefix `cogs/` is reachable via `dem-nginx-s3-gateway-internal`. + +## Self-Review notes + +- **Spec coverage:** engine swap (Tasks 3–6), discovery (Task 7), degrees end-to-end + middleware removal (Tasks 2, 8, 9), dependency swap (Task 1), tests (Tasks 5, 10, 11), verification (Task 12). API contract (`/points`, `openapi3.yaml`, response shape) is unchanged by design — no task needed. +- **Type consistency:** `GeoPoint` / `PosWithHeight` / `PosWithProvider` / `HeightProviders` defined in Task 2 are used consistently in Tasks 3–5, 8. Cache manager members `heightProviders` + `initProviders` defined in Task 4 are consumed in Task 5 and `containerConfig` (Task 4 step 2). `GeotiffHeightProvider.fromUrl(url, headers?)` signature defined in Task 3 matches its callers in Task 4 and the mock in Task 5. +- **Null-height invariant:** integration test #115-equivalent requires `productId === undefined ⇔ height === null`. Enforced in `samplePositionsHeights` (productId attached only when `height !== null`) and for the no-provider bucket. +- **nodata:** `-32768` default plus `getGDALNoData()`; any nodata neighbor → `null` (Task 3). From 76e5467d9cc560d1d2abfaa605fcad93e921c09c Mon Sep 17 00:00:00 2001 From: shimoncohen Date: Thu, 3 Sep 2026 11:07:41 +0300 Subject: [PATCH 02/25] build: replace cesium terrain engine with geotiff Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01ATQerkHAV3Vur82HVVFack --- package-lock.json | 612 +++++++++++----------------------------------- package.json | 3 +- 2 files changed, 146 insertions(+), 469 deletions(-) diff --git a/package-lock.json b/package-lock.json index fcbf511..4ddcaa1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -31,12 +31,12 @@ "@turf/turf": "^6.5.0", "@types/geojson": "^7946.0.10", "axios": "^0.21.1", - "cesium": "^1.114.0", "compression": "^1.7.4", "config": "^3.3.12", "cross-fetch": "^3.1.5", "express": "^4.18.1", "express-openapi-validator": "^4.13.8", + "geotiff": "^3.0.5", "http-status-codes": "^2.2.0", "lodash": "^4.17.21", "node-fetch": "^3.3.1", @@ -53,7 +53,6 @@ "@map-colonies/eslint-config": "^3.1.1", "@map-colonies/prettier-config": "0.0.1", "@redocly/openapi-cli": "^1.0.0-beta.94", - "@types/cesium": "^1.70.0", "@types/compression": "^1.7.2", "@types/config": "^3.3.0", "@types/express": "^4.17.13", @@ -2037,90 +2036,6 @@ "integrity": "sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==", "dev": true }, - "node_modules/@cesium/engine": { - "version": "26.2.0", - "resolved": "https://registry.npmjs.org/@cesium/engine/-/engine-26.2.0.tgz", - "integrity": "sha512-BsfHZpg6LvlUSaTfvzfEzo9JQd9+PtMbzoylwls+aRj6bnqGNTF5F2sMC1eja5L3wg725NyiWQ8s84sIM9Dr6g==", - "license": "Apache-2.0", - "dependencies": { - "@cesium/wasm-splats": "^0.1.0-alpha.2", - "@spz-loader/core": "0.3.1", - "@tweenjs/tween.js": "^25.0.0", - "@zip.js/zip.js": "^2.8.1", - "autolinker": "^4.0.0", - "bitmap-sdf": "^1.0.3", - "dompurify": "^3.3.0", - "draco3d": "^1.5.1", - "earcut": "3.0.2", - "grapheme-splitter": "^1.0.4", - "jsep": "^1.3.8", - "kdbush": "^4.0.1", - "ktx-parse": "^1.0.0", - "lerc": "^2.0.0", - "mersenne-twister": "^1.1.0", - "meshoptimizer": "^1.0.1", - "pako": "^3.0.0", - "protobufjs": "^8.6.5", - "rbush": "^4.0.1", - "topojson-client": "^3.1.0", - "urijs": "^1.19.7" - }, - "engines": { - "node": ">=22.0.0" - } - }, - "node_modules/@cesium/engine/node_modules/earcut": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/earcut/-/earcut-3.0.2.tgz", - "integrity": "sha512-X7hshQbLyMJ/3RPhyObLARM2sNxxmRALLKx1+NVFFnQ9gKzmCrxm9+uLIAdBcvc8FNLpctqlQ2V6AE92Ol9UDQ==", - "license": "ISC" - }, - "node_modules/@cesium/engine/node_modules/protobufjs": { - "version": "8.7.2", - "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-8.7.2.tgz", - "integrity": "sha512-oTVHV+oelUBtiu5iTuTNNZ0eLYsXSMxry4cgr30mayNkgIZL6qZ0IOQVPuSWGcyAaXKl/XgqwWHIC3a0khYVBA==", - "license": "BSD-3-Clause", - "dependencies": { - "long": "^5.3.2" - }, - "engines": { - "node": ">=12.0.0" - } - }, - "node_modules/@cesium/engine/node_modules/quickselect": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/quickselect/-/quickselect-3.0.0.tgz", - "integrity": "sha512-XdjUArbK4Bm5fLLvlm5KpTFOiOThgfWWI4axAZDWg4E/0mKdZyI9tNEfds27qCi1ze/vwTR16kvmmGhRra3c2g==", - "license": "ISC" - }, - "node_modules/@cesium/engine/node_modules/rbush": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/rbush/-/rbush-4.0.1.tgz", - "integrity": "sha512-IP0UpfeWQujYC8Jg162rMNc01Rf0gWMMAb2Uxus/Q0qOFw4lCcq6ZnQEZwUoJqWyUGJ9th7JjwI4yIWo+uvoAQ==", - "license": "MIT", - "dependencies": { - "quickselect": "^3.0.0" - } - }, - "node_modules/@cesium/wasm-splats": { - "version": "0.1.0-alpha.2", - "resolved": "https://registry.npmjs.org/@cesium/wasm-splats/-/wasm-splats-0.1.0-alpha.2.tgz", - "integrity": "sha512-t9pMkknv31hhIbLpMa8yPvmqfpvs5UkUjgqlQv9SeO8VerCXOYnyP8/486BDaFrztM0A7FMbRjsXtNeKvqQghA==", - "license": "Apache-2.0" - }, - "node_modules/@cesium/widgets": { - "version": "16.1.1", - "resolved": "https://registry.npmjs.org/@cesium/widgets/-/widgets-16.1.1.tgz", - "integrity": "sha512-um2JznRouJ2pfC0FNnEgJJU7WrCDrYFeyqCdiSsdbZgdW4q/R9ysDpj0VO/H2U4r4t6HDn9pA3HgAYpuIsdUWg==", - "license": "Apache-2.0", - "dependencies": { - "@cesium/engine": "^26.2.0", - "nosleep.js": "^0.12.0" - }, - "engines": { - "node": ">=22.0.0" - } - }, "node_modules/@commitlint/cli": { "version": "17.0.1", "resolved": "https://registry.npmjs.org/@commitlint/cli/-/cli-17.0.1.tgz", @@ -5899,6 +5814,12 @@ "node": ">=14" } }, + "node_modules/@petamoriken/float16": { + "version": "3.9.3", + "resolved": "https://registry.npmjs.org/@petamoriken/float16/-/float16-3.9.3.tgz", + "integrity": "sha512-8awtpHXCx/bNpFt4mt2xdkgtgVvKqty8VbjHI/WWWQuEw+KLzFot3f4+LkQY9YmOtq7A5GdOnqoIC8Pdygjk2g==", + "license": "MIT" + }, "node_modules/@protobufjs/aspromise": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", @@ -6132,16 +6053,6 @@ "@sinonjs/commons": "^1.7.0" } }, - "node_modules/@spz-loader/core": { - "version": "0.3.1", - "resolved": "https://registry.npmjs.org/@spz-loader/core/-/core-0.3.1.tgz", - "integrity": "sha512-8qJ1WIBXaJu8HjnJAjYniE0kYcr0kCe5Hp7kDzYiGVvvd7zyrOBwbF5imoW5mvwx1Qba0hxGEK5R9jEoaHKJFA==", - "license": "Apache-2.0", - "engines": { - "node": ">=16", - "pnpm": ">=8" - } - }, "node_modules/@supercharge/promise-pool": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/@supercharge/promise-pool/-/promise-pool-3.0.0.tgz", @@ -7764,12 +7675,6 @@ "url": "https://opencollective.com/turf" } }, - "node_modules/@tweenjs/tween.js": { - "version": "25.0.0", - "resolved": "https://registry.npmjs.org/@tweenjs/tween.js/-/tween.js-25.0.0.tgz", - "integrity": "sha512-XKLA6syeBUaPzx4j3qwMqzzq+V4uo72BnlbOjmuljLrRqdsd3qnzvZZoxvMHZ23ndsRS4aufU6JOZYpCbU6T1A==", - "license": "MIT" - }, "node_modules/@types/accepts": { "version": "1.3.5", "resolved": "https://registry.npmjs.org/@types/accepts/-/accepts-1.3.5.tgz", @@ -7847,16 +7752,6 @@ "integrity": "sha512-6ckxMjBBD8URvjB6J3NcnuAn5Pkl7t3TizAg+xdlzzQGSPSmBcXf8KoIH0ua/i+tio+ZRUHEXp0HEmvaR4kt0w==", "dev": true }, - "node_modules/@types/cesium": { - "version": "1.70.0", - "resolved": "https://registry.npmjs.org/@types/cesium/-/cesium-1.70.0.tgz", - "integrity": "sha512-awCNt8Exv6oT+GALMgorfLYw43H5/99ZdKpG4Ovtft7dKpaAPkLuqpD1AobSjjtJ7xaOKKTj5fzYj+JQnPuzAQ==", - "deprecated": "This is a stub types definition. cesium provides its own type definitions, so you do not need this installed.", - "dev": true, - "dependencies": { - "cesium": "*" - } - }, "node_modules/@types/compression": { "version": "1.7.2", "resolved": "https://registry.npmjs.org/@types/compression/-/compression-1.7.2.tgz", @@ -8281,13 +8176,6 @@ "integrity": "sha512-RBz2uRZVCXuMg93WD//aTS5B120QlT4lR/gL+935QtGsKHLS6sCtZBaKfWjIfk7ZXv/r8mtGbwjVIee6/3XTow==", "dev": true }, - "node_modules/@types/trusted-types": { - "version": "2.0.7", - "resolved": "https://registry.npmjs.org/@types/trusted-types/-/trusted-types-2.0.7.tgz", - "integrity": "sha512-ScaPdn1dQczgbl0QFTeTOmVHFULt394XJgOQNoyVhZ6r2vLnMLJfBPd53SB52T/3G36VI1/g2MZaX0cwDuXsfw==", - "license": "MIT", - "optional": true - }, "node_modules/@types/uuid": { "version": "8.3.4", "resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-8.3.4.tgz", @@ -8615,17 +8503,6 @@ "resolved": "https://registry.npmjs.org/@yarnpkg/lockfile/-/lockfile-1.1.0.tgz", "integrity": "sha512-GpSwvyXOcOOlV70vbnzjj4fW5xW/FdUF6nQEt1ENy7m4ZCczi1+/buVUPAqmGfqznsORNFzUMjctTIp8a9tuCQ==" }, - "node_modules/@zip.js/zip.js": { - "version": "2.8.36", - "resolved": "https://registry.npmjs.org/@zip.js/zip.js/-/zip.js-2.8.36.tgz", - "integrity": "sha512-3CT63BXqYh781nZhNVOktg71oM1496BZhBHDjP5aOZTDmsL2nLkldbLDJFatFJt00j94TFnsZu2pWNijL4Eorw==", - "license": "BSD-3-Clause", - "engines": { - "bun": ">=0.7.0", - "deno": ">=1.0.0", - "node": ">=18.0.0" - } - }, "node_modules/abbrev": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz", @@ -8986,24 +8863,6 @@ "node": ">=8.0.0" } }, - "node_modules/autolinker": { - "version": "4.1.5", - "resolved": "https://registry.npmjs.org/autolinker/-/autolinker-4.1.5.tgz", - "integrity": "sha512-vEfYZPmvVOIuE567XBVCsx8SBgOYtjB2+S1iAaJ+HgH+DNjAcrHem2hmAeC9yaNGWayicv4yR+9UaJlkF3pvtw==", - "license": "MIT", - "dependencies": { - "tslib": "^2.8.1" - }, - "engines": { - "pnpm": ">=10.10.0" - } - }, - "node_modules/autolinker/node_modules/tslib": { - "version": "2.8.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", - "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", - "license": "0BSD" - }, "node_modules/axe-core": { "version": "4.4.2", "resolved": "https://registry.npmjs.org/axe-core/-/axe-core-4.4.2.tgz", @@ -9345,12 +9204,6 @@ "resolved": "https://registry.npmjs.org/bintrees/-/bintrees-1.0.2.tgz", "integrity": "sha512-VOMgTMwjAaUG580SXn3LacVgjurrbMme7ZZNYGSSV7mmtY6QQRh0Eg3pwIcntQ77DErK1L0NxkbetjcoXzVwKw==" }, - "node_modules/bitmap-sdf": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/bitmap-sdf/-/bitmap-sdf-1.0.4.tgz", - "integrity": "sha512-1G3U4n5JE6RAiALMxu0p1XmeZkTeCwGKykzsLTCqVzfSDaN6S7fKnkIkfejogz+iwqBWc0UYAIKnKHNN7pSfDg==", - "license": "MIT" - }, "node_modules/body-parser": { "version": "1.20.0", "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.0.tgz", @@ -9593,37 +9446,6 @@ } ] }, - "node_modules/cesium": { - "version": "1.144.0", - "resolved": "https://registry.npmjs.org/cesium/-/cesium-1.144.0.tgz", - "integrity": "sha512-g1TX13KvTUhX9zjlJNn3De5BwuHyzuQzp1+1ITpGJJmIORCGnxVKAiu9/OkQByTWNYKgNtoEuPLrnWFhl0ao5g==", - "license": "Apache-2.0", - "workspaces": [ - "packages/engine", - "packages/widgets", - "packages/sandcastle" - ], - "dependencies": { - "@cesium/engine": "^26.2.0", - "@cesium/widgets": "^16.1.1", - "protobufjs": "^8.6.5" - }, - "engines": { - "node": ">=22.0.0" - } - }, - "node_modules/cesium/node_modules/protobufjs": { - "version": "8.7.2", - "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-8.7.2.tgz", - "integrity": "sha512-oTVHV+oelUBtiu5iTuTNNZ0eLYsXSMxry4cgr30mayNkgIZL6qZ0IOQVPuSWGcyAaXKl/XgqwWHIC3a0khYVBA==", - "license": "BSD-3-Clause", - "dependencies": { - "long": "^5.3.2" - }, - "engines": { - "node": ">=12.0.0" - } - }, "node_modules/chalk": { "version": "2.4.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", @@ -10474,15 +10296,6 @@ "node": ">=0.10.0" } }, - "node_modules/dompurify": { - "version": "3.4.13", - "resolved": "https://registry.npmjs.org/dompurify/-/dompurify-3.4.13.tgz", - "integrity": "sha512-2vmYIoqjze2d+kakP8S/nS5shfsl587kzwEjcGlTdiksUVgFHnFCsLYDVj/JNqJVOQZGSYBTmuycv0PodwmnMQ==", - "license": "(MPL-2.0 OR Apache-2.0)", - "optionalDependencies": { - "@types/trusted-types": "^2.0.7" - } - }, "node_modules/dot-prop": { "version": "5.3.0", "resolved": "https://registry.npmjs.org/dot-prop/-/dot-prop-5.3.0.tgz", @@ -10495,12 +10308,6 @@ "node": ">=8" } }, - "node_modules/draco3d": { - "version": "1.5.7", - "resolved": "https://registry.npmjs.org/draco3d/-/draco3d-1.5.7.tgz", - "integrity": "sha512-m6WCKt/erDXcw+70IJXnG7M3awwQPAsZvJGX5zY7beBqpELw6RDGkYVU0W43AFxye4pDZ5i2Lbyc/NNGqwjUVQ==", - "license": "Apache-2.0" - }, "node_modules/earcut": { "version": "2.2.4", "resolved": "https://registry.npmjs.org/earcut/-/earcut-2.2.4.tgz", @@ -12180,6 +11987,59 @@ "resolved": "https://registry.npmjs.org/@types/geojson/-/geojson-7946.0.8.tgz", "integrity": "sha512-1rkryxURpr6aWP7R786/UQOkJ3PcpQiWkAXBmdWc7ryFWqN6a4xfK7BtjXvFBKO9LjQ+MWQSWxYeZX1OApnArA==" }, + "node_modules/geotiff": { + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/geotiff/-/geotiff-3.0.5.tgz", + "integrity": "sha512-OWcL9S9+yDZ6iAlXMt32T1iwUApJM8UiD47xbm6ZP1h33d10fqkPs14EG/ttT5EnefpZSx3G15iDFC5FxUNUwA==", + "license": "MIT", + "dependencies": { + "@petamoriken/float16": "^3.9.3", + "lerc": "^3.0.0", + "pako": "^2.0.4", + "parse-headers": "^2.0.2", + "quick-lru": "^6.1.1", + "web-worker": "^1.5.0", + "xml-utils": "^1.10.2", + "zstddec": "^0.2.0" + }, + "engines": { + "node": ">=10.19" + } + }, + "node_modules/geotiff/node_modules/lerc": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/lerc/-/lerc-3.0.0.tgz", + "integrity": "sha512-Rm4J/WaHhRa93nCN2mwWDZFoRVF18G1f47C+kvQWyHGEZxFpTUi73p7lMVSAndyxGt6lJ2/CFbOcf9ra5p8aww==", + "license": "Apache-2.0" + }, + "node_modules/geotiff/node_modules/pako": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/pako/-/pako-2.2.0.tgz", + "integrity": "sha512-zJq6RP/5q+TO2OpFV3FHzlPnFjmkb7Nc99a5SNjJE+uu/PkpChs+NIZSSzbBoD+6kjiISXjfYdwj1ZRQ81dz/w==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/puzrin" + }, + { + "type": "github", + "url": "https://github.com/sponsors/nodeca" + } + ], + "license": "(MIT AND Zlib)" + }, + "node_modules/geotiff/node_modules/quick-lru": { + "version": "6.1.2", + "resolved": "https://registry.npmjs.org/quick-lru/-/quick-lru-6.1.2.tgz", + "integrity": "sha512-AAFUA5O1d83pIHEhJwWCq/RQcRukCkn/NSm2QsTEMle5f2hP0ChI2+3Xb051PZCkLryI/Ir1MVKviT2FIloaTQ==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/get-caller-file": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", @@ -12362,7 +12222,8 @@ "node_modules/grapheme-splitter": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/grapheme-splitter/-/grapheme-splitter-1.0.4.tgz", - "integrity": "sha512-bzh50DW9kTPM00T8y4o8vQg89Di9oLJVLW/KaOGIXJWP/iqCN6WKYkbNOF04vFLJhwcpYUh9ydh/+5vpOqV4YQ==" + "integrity": "sha512-bzh50DW9kTPM00T8y4o8vQg89Di9oLJVLW/KaOGIXJWP/iqCN6WKYkbNOF04vFLJhwcpYUh9ydh/+5vpOqV4YQ==", + "dev": true }, "node_modules/handlebars": { "version": "4.7.7", @@ -15016,15 +14877,6 @@ "js-yaml": "bin/js-yaml.js" } }, - "node_modules/jsep": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/jsep/-/jsep-1.4.0.tgz", - "integrity": "sha512-B7qPcEVE3NVkmSJbaYxvv4cHkVW7DQsZz13pUMrfS8z8Q/BuShN+gcTXrUlPiGqM2/t/EEaI030bpxMqY8gMlw==", - "license": "MIT", - "engines": { - "node": ">= 10.16.0" - } - }, "node_modules/jsesc": { "version": "2.5.2", "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-2.5.2.tgz", @@ -15148,12 +15000,6 @@ "node": ">=4.0" } }, - "node_modules/kdbush": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/kdbush/-/kdbush-4.1.0.tgz", - "integrity": "sha512-e9vurzrXJQrFX6ckpHP3bvj5l+9CnYzkxDNnNQ1h2QTqdWsUAJgXiKdGNcOa1EY85dU8KbQ+z/FdQdB7P+9yfQ==", - "license": "ISC" - }, "node_modules/kind-of": { "version": "6.0.3", "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", @@ -15180,12 +15026,6 @@ "node": ">=6" } }, - "node_modules/ktx-parse": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/ktx-parse/-/ktx-parse-1.1.0.tgz", - "integrity": "sha512-mKp3y+FaYgR7mXWAbyyzpa/r1zDWeaunH+INJO4fou3hb45XuNSwar+7llrRyvpMWafxSIi99RNFJ05MHedaJQ==", - "license": "MIT" - }, "node_modules/language-subtag-registry": { "version": "0.3.21", "resolved": "https://registry.npmjs.org/language-subtag-registry/-/language-subtag-registry-0.3.21.tgz", @@ -15201,12 +15041,6 @@ "language-subtag-registry": "~0.3.2" } }, - "node_modules/lerc": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/lerc/-/lerc-2.0.0.tgz", - "integrity": "sha512-7qo1Mq8ZNmaR4USHHm615nEW2lPeeWJ3bTyoqFbd35DLx0LUH7C6ptt5FDCTAlbIzs3+WKrk5SkJvw8AFDE2hg==", - "license": "Apache-2.0" - }, "node_modules/levn": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz", @@ -15435,18 +15269,6 @@ "node": ">= 8" } }, - "node_modules/mersenne-twister": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/mersenne-twister/-/mersenne-twister-1.1.0.tgz", - "integrity": "sha512-mUYWsMKNrm4lfygPkL3OfGzOPTR2DBlTkBNHM//F6hGp8cLThY897crAlk3/Jo17LEOOjQUrNAx6DvgO77QJkA==", - "license": "MIT" - }, - "node_modules/meshoptimizer": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/meshoptimizer/-/meshoptimizer-1.2.0.tgz", - "integrity": "sha512-davRZeIJbxJrE24cwQle7ZDsxjdk/OphNOV83oX+efQinyoHY9Jcyz3MHbaoG0qySZajldGztNZ1RN/T19PZsg==", - "license": "MIT" - }, "node_modules/methods": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz", @@ -15802,12 +15624,6 @@ "node": ">=0.10.0" } }, - "node_modules/nosleep.js": { - "version": "0.12.0", - "resolved": "https://registry.npmjs.org/nosleep.js/-/nosleep.js-0.12.0.tgz", - "integrity": "sha512-9d1HbpKLh3sdWlhXMhU6MMH+wQzKkrgfRkYV0EBdvt99YJfj0ilCJrWRDYG2130Tm4GXbEoTCx5b34JSaP+HhA==", - "license": "MIT" - }, "node_modules/npm-run-path": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-4.0.1.tgz", @@ -16198,22 +16014,6 @@ "node": ">=6" } }, - "node_modules/pako": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/pako/-/pako-3.0.1.tgz", - "integrity": "sha512-GupotUUI0mlhugKjUs4bjOwLt3nrehy9Ys2dxC0GtgVef5cnKggkDMmf2bq2poCCuVXopWPmqsc9VDT2iJUy+w==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/puzrin" - }, - { - "type": "github", - "url": "https://github.com/sponsors/nodeca" - } - ], - "license": "(MIT AND Zlib)" - }, "node_modules/parent-module": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", @@ -16226,6 +16026,12 @@ "node": ">=6" } }, + "node_modules/parse-headers": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/parse-headers/-/parse-headers-2.0.6.tgz", + "integrity": "sha512-Tz11t3uKztEW5FEVZnj1ox8GKblWn+PvHY9TmJV5Mll2uHEwRdR/5Li1OlXoECjLYkApdhWy44ocONwXLiKO5A==", + "license": "MIT" + }, "node_modules/parse-json": { "version": "5.2.0", "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-5.2.0.tgz", @@ -18811,12 +18617,6 @@ "node": ">=6" } }, - "node_modules/urijs": { - "version": "1.19.11", - "resolved": "https://registry.npmjs.org/urijs/-/urijs-1.19.11.tgz", - "integrity": "sha512-HXgFDgDommxn5/bIv0cnQZsPhHDA90NPHD6+c/v21U5+Sx5hoP8+dP9IZXBU1gIfvdRfhG8cel9QNPeionfcCQ==", - "license": "MIT" - }, "node_modules/url-value-parser": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/url-value-parser/-/url-value-parser-2.2.0.tgz", @@ -18937,6 +18737,12 @@ "node": ">= 8" } }, + "node_modules/web-worker": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/web-worker/-/web-worker-1.5.0.tgz", + "integrity": "sha512-RiMReJrTAiA+mBjGONMnjVDP2u3p9R1vkcGz6gDIrOMT3oGuYwX2WRMYI9ipkphSuE5XKEhydbhNEJh4NY9mlw==", + "license": "Apache-2.0" + }, "node_modules/webidl-conversions": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", @@ -19087,6 +18893,12 @@ } } }, + "node_modules/xml-utils": { + "version": "1.10.2", + "resolved": "https://registry.npmjs.org/xml-utils/-/xml-utils-1.10.2.tgz", + "integrity": "sha512-RqM+2o1RYs6T8+3DzDSoTRAUfrvaejbVHcp3+thnAtDKo8LskR+HomLajEy5UjTz24rpka7AxVBRR3g2wTUkJA==", + "license": "CC0-1.0" + }, "node_modules/xml2js": { "version": "0.4.23", "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.23.tgz", @@ -19223,6 +19035,12 @@ "funding": { "url": "https://github.com/sponsors/sindresorhus" } + }, + "node_modules/zstddec": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/zstddec/-/zstddec-0.2.0.tgz", + "integrity": "sha512-oyPnDa1X5c13+Y7mA/FDMNJrn4S8UNBe0KCqtDmor40Re7ALrPN6npFwyYVRRh+PqozZQdeg23QtbcamZnG5rA==", + "license": "MIT AND BSD-3-Clause" } }, "dependencies": { @@ -20592,76 +20410,6 @@ "integrity": "sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==", "dev": true }, - "@cesium/engine": { - "version": "26.2.0", - "resolved": "https://registry.npmjs.org/@cesium/engine/-/engine-26.2.0.tgz", - "integrity": "sha512-BsfHZpg6LvlUSaTfvzfEzo9JQd9+PtMbzoylwls+aRj6bnqGNTF5F2sMC1eja5L3wg725NyiWQ8s84sIM9Dr6g==", - "requires": { - "@cesium/wasm-splats": "^0.1.0-alpha.2", - "@spz-loader/core": "0.3.1", - "@tweenjs/tween.js": "^25.0.0", - "@zip.js/zip.js": "^2.8.1", - "autolinker": "^4.0.0", - "bitmap-sdf": "^1.0.3", - "dompurify": "^3.3.0", - "draco3d": "^1.5.1", - "earcut": "3.0.2", - "grapheme-splitter": "^1.0.4", - "jsep": "^1.3.8", - "kdbush": "^4.0.1", - "ktx-parse": "^1.0.0", - "lerc": "^2.0.0", - "mersenne-twister": "^1.1.0", - "meshoptimizer": "^1.0.1", - "pako": "^3.0.0", - "protobufjs": "^8.6.5", - "rbush": "^4.0.1", - "topojson-client": "^3.1.0", - "urijs": "^1.19.7" - }, - "dependencies": { - "earcut": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/earcut/-/earcut-3.0.2.tgz", - "integrity": "sha512-X7hshQbLyMJ/3RPhyObLARM2sNxxmRALLKx1+NVFFnQ9gKzmCrxm9+uLIAdBcvc8FNLpctqlQ2V6AE92Ol9UDQ==" - }, - "protobufjs": { - "version": "8.7.2", - "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-8.7.2.tgz", - "integrity": "sha512-oTVHV+oelUBtiu5iTuTNNZ0eLYsXSMxry4cgr30mayNkgIZL6qZ0IOQVPuSWGcyAaXKl/XgqwWHIC3a0khYVBA==", - "requires": { - "long": "^5.3.2" - } - }, - "quickselect": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/quickselect/-/quickselect-3.0.0.tgz", - "integrity": "sha512-XdjUArbK4Bm5fLLvlm5KpTFOiOThgfWWI4axAZDWg4E/0mKdZyI9tNEfds27qCi1ze/vwTR16kvmmGhRra3c2g==" - }, - "rbush": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/rbush/-/rbush-4.0.1.tgz", - "integrity": "sha512-IP0UpfeWQujYC8Jg162rMNc01Rf0gWMMAb2Uxus/Q0qOFw4lCcq6ZnQEZwUoJqWyUGJ9th7JjwI4yIWo+uvoAQ==", - "requires": { - "quickselect": "^3.0.0" - } - } - } - }, - "@cesium/wasm-splats": { - "version": "0.1.0-alpha.2", - "resolved": "https://registry.npmjs.org/@cesium/wasm-splats/-/wasm-splats-0.1.0-alpha.2.tgz", - "integrity": "sha512-t9pMkknv31hhIbLpMa8yPvmqfpvs5UkUjgqlQv9SeO8VerCXOYnyP8/486BDaFrztM0A7FMbRjsXtNeKvqQghA==" - }, - "@cesium/widgets": { - "version": "16.1.1", - "resolved": "https://registry.npmjs.org/@cesium/widgets/-/widgets-16.1.1.tgz", - "integrity": "sha512-um2JznRouJ2pfC0FNnEgJJU7WrCDrYFeyqCdiSsdbZgdW4q/R9ysDpj0VO/H2U4r4t6HDn9pA3HgAYpuIsdUWg==", - "requires": { - "@cesium/engine": "^26.2.0", - "nosleep.js": "^0.12.0" - } - }, "@commitlint/cli": { "version": "17.0.1", "resolved": "https://registry.npmjs.org/@commitlint/cli/-/cli-17.0.1.tgz", @@ -23358,6 +23106,11 @@ "resolved": "https://registry.npmjs.org/@opentelemetry/semantic-conventions/-/semantic-conventions-1.16.0.tgz", "integrity": "sha512-jpbpOY0ahD/uTF16h4/NkFatwlulCpkvecK8zVJgHRfjK/ymJAw1Cg45KXiVSJML8uiMmRBciea/+gebrD398g==" }, + "@petamoriken/float16": { + "version": "3.9.3", + "resolved": "https://registry.npmjs.org/@petamoriken/float16/-/float16-3.9.3.tgz", + "integrity": "sha512-8awtpHXCx/bNpFt4mt2xdkgtgVvKqty8VbjHI/WWWQuEw+KLzFot3f4+LkQY9YmOtq7A5GdOnqoIC8Pdygjk2g==" + }, "@protobufjs/aspromise": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", @@ -23569,11 +23322,6 @@ "@sinonjs/commons": "^1.7.0" } }, - "@spz-loader/core": { - "version": "0.3.1", - "resolved": "https://registry.npmjs.org/@spz-loader/core/-/core-0.3.1.tgz", - "integrity": "sha512-8qJ1WIBXaJu8HjnJAjYniE0kYcr0kCe5Hp7kDzYiGVvvd7zyrOBwbF5imoW5mvwx1Qba0hxGEK5R9jEoaHKJFA==" - }, "@supercharge/promise-pool": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/@supercharge/promise-pool/-/promise-pool-3.0.0.tgz", @@ -24876,11 +24624,6 @@ "d3-voronoi": "1.1.2" } }, - "@tweenjs/tween.js": { - "version": "25.0.0", - "resolved": "https://registry.npmjs.org/@tweenjs/tween.js/-/tween.js-25.0.0.tgz", - "integrity": "sha512-XKLA6syeBUaPzx4j3qwMqzzq+V4uo72BnlbOjmuljLrRqdsd3qnzvZZoxvMHZ23ndsRS4aufU6JOZYpCbU6T1A==" - }, "@types/accepts": { "version": "1.3.5", "resolved": "https://registry.npmjs.org/@types/accepts/-/accepts-1.3.5.tgz", @@ -24958,15 +24701,6 @@ "integrity": "sha512-6ckxMjBBD8URvjB6J3NcnuAn5Pkl7t3TizAg+xdlzzQGSPSmBcXf8KoIH0ua/i+tio+ZRUHEXp0HEmvaR4kt0w==", "dev": true }, - "@types/cesium": { - "version": "1.70.0", - "resolved": "https://registry.npmjs.org/@types/cesium/-/cesium-1.70.0.tgz", - "integrity": "sha512-awCNt8Exv6oT+GALMgorfLYw43H5/99ZdKpG4Ovtft7dKpaAPkLuqpD1AobSjjtJ7xaOKKTj5fzYj+JQnPuzAQ==", - "dev": true, - "requires": { - "cesium": "*" - } - }, "@types/compression": { "version": "1.7.2", "resolved": "https://registry.npmjs.org/@types/compression/-/compression-1.7.2.tgz", @@ -25389,12 +25123,6 @@ "integrity": "sha512-RBz2uRZVCXuMg93WD//aTS5B120QlT4lR/gL+935QtGsKHLS6sCtZBaKfWjIfk7ZXv/r8mtGbwjVIee6/3XTow==", "dev": true }, - "@types/trusted-types": { - "version": "2.0.7", - "resolved": "https://registry.npmjs.org/@types/trusted-types/-/trusted-types-2.0.7.tgz", - "integrity": "sha512-ScaPdn1dQczgbl0QFTeTOmVHFULt394XJgOQNoyVhZ6r2vLnMLJfBPd53SB52T/3G36VI1/g2MZaX0cwDuXsfw==", - "optional": true - }, "@types/uuid": { "version": "8.3.4", "resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-8.3.4.tgz", @@ -25582,11 +25310,6 @@ "resolved": "https://registry.npmjs.org/@yarnpkg/lockfile/-/lockfile-1.1.0.tgz", "integrity": "sha512-GpSwvyXOcOOlV70vbnzjj4fW5xW/FdUF6nQEt1ENy7m4ZCczi1+/buVUPAqmGfqznsORNFzUMjctTIp8a9tuCQ==" }, - "@zip.js/zip.js": { - "version": "2.8.36", - "resolved": "https://registry.npmjs.org/@zip.js/zip.js/-/zip.js-2.8.36.tgz", - "integrity": "sha512-3CT63BXqYh781nZhNVOktg71oM1496BZhBHDjP5aOZTDmsL2nLkldbLDJFatFJt00j94TFnsZu2pWNijL4Eorw==" - }, "abbrev": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz", @@ -25858,21 +25581,6 @@ "resolved": "https://registry.npmjs.org/atomic-sleep/-/atomic-sleep-1.0.0.tgz", "integrity": "sha512-kNOjDqAh7px0XWNI+4QbzoiR/nTkHAWNud2uvnJquD1/x5a7EQZMJT0AczqK0Qn67oY/TTQ1LbUKajZpp3I9tQ==" }, - "autolinker": { - "version": "4.1.5", - "resolved": "https://registry.npmjs.org/autolinker/-/autolinker-4.1.5.tgz", - "integrity": "sha512-vEfYZPmvVOIuE567XBVCsx8SBgOYtjB2+S1iAaJ+HgH+DNjAcrHem2hmAeC9yaNGWayicv4yR+9UaJlkF3pvtw==", - "requires": { - "tslib": "^2.8.1" - }, - "dependencies": { - "tslib": { - "version": "2.8.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", - "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==" - } - } - }, "axe-core": { "version": "4.4.2", "resolved": "https://registry.npmjs.org/axe-core/-/axe-core-4.4.2.tgz", @@ -26137,11 +25845,6 @@ "resolved": "https://registry.npmjs.org/bintrees/-/bintrees-1.0.2.tgz", "integrity": "sha512-VOMgTMwjAaUG580SXn3LacVgjurrbMme7ZZNYGSSV7mmtY6QQRh0Eg3pwIcntQ77DErK1L0NxkbetjcoXzVwKw==" }, - "bitmap-sdf": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/bitmap-sdf/-/bitmap-sdf-1.0.4.tgz", - "integrity": "sha512-1G3U4n5JE6RAiALMxu0p1XmeZkTeCwGKykzsLTCqVzfSDaN6S7fKnkIkfejogz+iwqBWc0UYAIKnKHNN7pSfDg==" - }, "body-parser": { "version": "1.20.0", "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.0.tgz", @@ -26309,26 +26012,6 @@ "integrity": "sha512-0ZFjnlCaXNOAYcV7i+TtdKBp0L/3XEU2MF/x6Du1lrh+SRX4IfzIVL4HNJg5pB2PmFb8rszIGyOvsZnqqRoc2g==", "dev": true }, - "cesium": { - "version": "1.144.0", - "resolved": "https://registry.npmjs.org/cesium/-/cesium-1.144.0.tgz", - "integrity": "sha512-g1TX13KvTUhX9zjlJNn3De5BwuHyzuQzp1+1ITpGJJmIORCGnxVKAiu9/OkQByTWNYKgNtoEuPLrnWFhl0ao5g==", - "requires": { - "@cesium/engine": "^26.2.0", - "@cesium/widgets": "^16.1.1", - "protobufjs": "^8.6.5" - }, - "dependencies": { - "protobufjs": { - "version": "8.7.2", - "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-8.7.2.tgz", - "integrity": "sha512-oTVHV+oelUBtiu5iTuTNNZ0eLYsXSMxry4cgr30mayNkgIZL6qZ0IOQVPuSWGcyAaXKl/XgqwWHIC3a0khYVBA==", - "requires": { - "long": "^5.3.2" - } - } - } - }, "chalk": { "version": "2.4.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", @@ -27013,14 +26696,6 @@ "esutils": "^2.0.2" } }, - "dompurify": { - "version": "3.4.13", - "resolved": "https://registry.npmjs.org/dompurify/-/dompurify-3.4.13.tgz", - "integrity": "sha512-2vmYIoqjze2d+kakP8S/nS5shfsl587kzwEjcGlTdiksUVgFHnFCsLYDVj/JNqJVOQZGSYBTmuycv0PodwmnMQ==", - "requires": { - "@types/trusted-types": "^2.0.7" - } - }, "dot-prop": { "version": "5.3.0", "resolved": "https://registry.npmjs.org/dot-prop/-/dot-prop-5.3.0.tgz", @@ -27030,11 +26705,6 @@ "is-obj": "^2.0.0" } }, - "draco3d": { - "version": "1.5.7", - "resolved": "https://registry.npmjs.org/draco3d/-/draco3d-1.5.7.tgz", - "integrity": "sha512-m6WCKt/erDXcw+70IJXnG7M3awwQPAsZvJGX5zY7beBqpELw6RDGkYVU0W43AFxye4pDZ5i2Lbyc/NNGqwjUVQ==" - }, "earcut": { "version": "2.2.4", "resolved": "https://registry.npmjs.org/earcut/-/earcut-2.2.4.tgz", @@ -28296,6 +27966,38 @@ } } }, + "geotiff": { + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/geotiff/-/geotiff-3.0.5.tgz", + "integrity": "sha512-OWcL9S9+yDZ6iAlXMt32T1iwUApJM8UiD47xbm6ZP1h33d10fqkPs14EG/ttT5EnefpZSx3G15iDFC5FxUNUwA==", + "requires": { + "@petamoriken/float16": "^3.9.3", + "lerc": "^3.0.0", + "pako": "^2.0.4", + "parse-headers": "^2.0.2", + "quick-lru": "^6.1.1", + "web-worker": "^1.5.0", + "xml-utils": "^1.10.2", + "zstddec": "^0.2.0" + }, + "dependencies": { + "lerc": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/lerc/-/lerc-3.0.0.tgz", + "integrity": "sha512-Rm4J/WaHhRa93nCN2mwWDZFoRVF18G1f47C+kvQWyHGEZxFpTUi73p7lMVSAndyxGt6lJ2/CFbOcf9ra5p8aww==" + }, + "pako": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/pako/-/pako-2.2.0.tgz", + "integrity": "sha512-zJq6RP/5q+TO2OpFV3FHzlPnFjmkb7Nc99a5SNjJE+uu/PkpChs+NIZSSzbBoD+6kjiISXjfYdwj1ZRQ81dz/w==" + }, + "quick-lru": { + "version": "6.1.2", + "resolved": "https://registry.npmjs.org/quick-lru/-/quick-lru-6.1.2.tgz", + "integrity": "sha512-AAFUA5O1d83pIHEhJwWCq/RQcRukCkn/NSm2QsTEMle5f2hP0ChI2+3Xb051PZCkLryI/Ir1MVKviT2FIloaTQ==" + } + } + }, "get-caller-file": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", @@ -28426,7 +28128,8 @@ "grapheme-splitter": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/grapheme-splitter/-/grapheme-splitter-1.0.4.tgz", - "integrity": "sha512-bzh50DW9kTPM00T8y4o8vQg89Di9oLJVLW/KaOGIXJWP/iqCN6WKYkbNOF04vFLJhwcpYUh9ydh/+5vpOqV4YQ==" + "integrity": "sha512-bzh50DW9kTPM00T8y4o8vQg89Di9oLJVLW/KaOGIXJWP/iqCN6WKYkbNOF04vFLJhwcpYUh9ydh/+5vpOqV4YQ==", + "dev": true }, "handlebars": { "version": "4.7.7", @@ -30366,11 +30069,6 @@ "argparse": "^2.0.1" } }, - "jsep": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/jsep/-/jsep-1.4.0.tgz", - "integrity": "sha512-B7qPcEVE3NVkmSJbaYxvv4cHkVW7DQsZz13pUMrfS8z8Q/BuShN+gcTXrUlPiGqM2/t/EEaI030bpxMqY8gMlw==" - }, "jsesc": { "version": "2.5.2", "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-2.5.2.tgz", @@ -30460,11 +30158,6 @@ "object.assign": "^4.1.2" } }, - "kdbush": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/kdbush/-/kdbush-4.1.0.tgz", - "integrity": "sha512-e9vurzrXJQrFX6ckpHP3bvj5l+9CnYzkxDNnNQ1h2QTqdWsUAJgXiKdGNcOa1EY85dU8KbQ+z/FdQdB7P+9yfQ==" - }, "kind-of": { "version": "6.0.3", "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", @@ -30485,11 +30178,6 @@ "integrity": "sha512-eTIzlVOSUR+JxdDFepEYcBMtZ9Qqdef+rnzWdRZuMbOywu5tO2w2N7rqjoANZ5k9vywhL6Br1VRjUIgTQx4E8w==", "dev": true }, - "ktx-parse": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/ktx-parse/-/ktx-parse-1.1.0.tgz", - "integrity": "sha512-mKp3y+FaYgR7mXWAbyyzpa/r1zDWeaunH+INJO4fou3hb45XuNSwar+7llrRyvpMWafxSIi99RNFJ05MHedaJQ==" - }, "language-subtag-registry": { "version": "0.3.21", "resolved": "https://registry.npmjs.org/language-subtag-registry/-/language-subtag-registry-0.3.21.tgz", @@ -30505,11 +30193,6 @@ "language-subtag-registry": "~0.3.2" } }, - "lerc": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/lerc/-/lerc-2.0.0.tgz", - "integrity": "sha512-7qo1Mq8ZNmaR4USHHm615nEW2lPeeWJ3bTyoqFbd35DLx0LUH7C6ptt5FDCTAlbIzs3+WKrk5SkJvw8AFDE2hg==" - }, "levn": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz", @@ -30696,16 +30379,6 @@ "integrity": "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==", "dev": true }, - "mersenne-twister": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/mersenne-twister/-/mersenne-twister-1.1.0.tgz", - "integrity": "sha512-mUYWsMKNrm4lfygPkL3OfGzOPTR2DBlTkBNHM//F6hGp8cLThY897crAlk3/Jo17LEOOjQUrNAx6DvgO77QJkA==" - }, - "meshoptimizer": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/meshoptimizer/-/meshoptimizer-1.2.0.tgz", - "integrity": "sha512-davRZeIJbxJrE24cwQle7ZDsxjdk/OphNOV83oX+efQinyoHY9Jcyz3MHbaoG0qySZajldGztNZ1RN/T19PZsg==" - }, "methods": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz", @@ -30974,11 +30647,6 @@ "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==", "dev": true }, - "nosleep.js": { - "version": "0.12.0", - "resolved": "https://registry.npmjs.org/nosleep.js/-/nosleep.js-0.12.0.tgz", - "integrity": "sha512-9d1HbpKLh3sdWlhXMhU6MMH+wQzKkrgfRkYV0EBdvt99YJfj0ilCJrWRDYG2130Tm4GXbEoTCx5b34JSaP+HhA==" - }, "npm-run-path": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-4.0.1.tgz", @@ -31282,11 +30950,6 @@ "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==" }, - "pako": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/pako/-/pako-3.0.1.tgz", - "integrity": "sha512-GupotUUI0mlhugKjUs4bjOwLt3nrehy9Ys2dxC0GtgVef5cnKggkDMmf2bq2poCCuVXopWPmqsc9VDT2iJUy+w==" - }, "parent-module": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", @@ -31296,6 +30959,11 @@ "callsites": "^3.0.0" } }, + "parse-headers": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/parse-headers/-/parse-headers-2.0.6.tgz", + "integrity": "sha512-Tz11t3uKztEW5FEVZnj1ox8GKblWn+PvHY9TmJV5Mll2uHEwRdR/5Li1OlXoECjLYkApdhWy44ocONwXLiKO5A==" + }, "parse-json": { "version": "5.2.0", "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-5.2.0.tgz", @@ -33234,11 +32902,6 @@ } } }, - "urijs": { - "version": "1.19.11", - "resolved": "https://registry.npmjs.org/urijs/-/urijs-1.19.11.tgz", - "integrity": "sha512-HXgFDgDommxn5/bIv0cnQZsPhHDA90NPHD6+c/v21U5+Sx5hoP8+dP9IZXBU1gIfvdRfhG8cel9QNPeionfcCQ==" - }, "url-value-parser": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/url-value-parser/-/url-value-parser-2.2.0.tgz", @@ -33340,6 +33003,11 @@ "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.2.1.tgz", "integrity": "sha512-e0MO3wdXWKrLbL0DgGnUV7WHVuw9OUvL4hjgnPkIeEvESk74gAITi5G606JtZPp39cd8HA9VQzCIvA49LpPN5Q==" }, + "web-worker": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/web-worker/-/web-worker-1.5.0.tgz", + "integrity": "sha512-RiMReJrTAiA+mBjGONMnjVDP2u3p9R1vkcGz6gDIrOMT3oGuYwX2WRMYI9ipkphSuE5XKEhydbhNEJh4NY9mlw==" + }, "webidl-conversions": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", @@ -33448,6 +33116,11 @@ "dev": true, "requires": {} }, + "xml-utils": { + "version": "1.10.2", + "resolved": "https://registry.npmjs.org/xml-utils/-/xml-utils-1.10.2.tgz", + "integrity": "sha512-RqM+2o1RYs6T8+3DzDSoTRAUfrvaejbVHcp3+thnAtDKo8LskR+HomLajEy5UjTz24rpka7AxVBRR3g2wTUkJA==" + }, "xml2js": { "version": "0.4.23", "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.23.tgz", @@ -33547,6 +33220,11 @@ "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", "dev": true + }, + "zstddec": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/zstddec/-/zstddec-0.2.0.tgz", + "integrity": "sha512-oyPnDa1X5c13+Y7mA/FDMNJrn4S8UNBe0KCqtDmor40Re7ALrPN6npFwyYVRRh+PqozZQdeg23QtbcamZnG5rA==" } } } diff --git a/package.json b/package.json index ab5908c..dc08141 100644 --- a/package.json +++ b/package.json @@ -57,12 +57,12 @@ "@turf/turf": "^6.5.0", "@types/geojson": "^7946.0.10", "axios": "^0.21.1", - "cesium": "^1.114.0", "compression": "^1.7.4", "config": "^3.3.12", "cross-fetch": "^3.1.5", "express": "^4.18.1", "express-openapi-validator": "^4.13.8", + "geotiff": "^3.0.5", "http-status-codes": "^2.2.0", "lodash": "^4.17.21", "node-fetch": "^3.3.1", @@ -79,7 +79,6 @@ "@map-colonies/eslint-config": "^3.1.1", "@map-colonies/prettier-config": "0.0.1", "@redocly/openapi-cli": "^1.0.0-beta.94", - "@types/cesium": "^1.70.0", "@types/compression": "^1.7.2", "@types/config": "^3.3.0", "@types/express": "^4.17.13", From e47b5618ed91a5b4151952fa2f03c84e2be20c6b Mon Sep 17 00:00:00 2001 From: shimoncohen Date: Thu, 3 Sep 2026 11:08:45 +0300 Subject: [PATCH 03/25] refactor: degrees-based GeoPoint and height provider types Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01ATQerkHAV3Vur82HVVFack --- src/heights/interfaces.ts | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/src/heights/interfaces.ts b/src/heights/interfaces.ts index df163f6..e8aa20a 100644 --- a/src/heights/interfaces.ts +++ b/src/heights/interfaces.ts @@ -1,4 +1,4 @@ -import { Cartographic, CesiumTerrainProvider } from 'cesium'; +import type GeotiffHeightProvider from './models/geotiffHeightProvider'; export enum TerrainTypes { DSM = 'DSM', @@ -6,13 +6,19 @@ export enum TerrainTypes { MIXED = 'MIXED', } -export interface PosWithHeight extends Cartographic { - productId: string; +export interface GeoPoint { + longitude: number; // WGS84 degrees + latitude: number; // WGS84 degrees + height?: number | null; } -export interface PosWithTerrainProvider extends Cartographic { - terrainProvider?: CesiumTerrainProvider; +export interface PosWithHeight extends GeoPoint { + height: number | null; + productId?: string; +} + +export interface PosWithProvider extends GeoPoint { providerKey?: string; } -export type TerrainProviders = Record; +export type HeightProviders = Record; From fb8ef1a19f1667b61d69e86a7926bd477fd37a5e Mon Sep 17 00:00:00 2001 From: shimoncohen Date: Thu, 3 Sep 2026 11:13:46 +0300 Subject: [PATCH 04/25] test: drop obsolete cesium mock from jest global setup Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01ATQerkHAV3Vur82HVVFack --- tests/configurations/jest.setup.ts | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/tests/configurations/jest.setup.ts b/tests/configurations/jest.setup.ts index 38f8828..d2c9bc6 100644 --- a/tests/configurations/jest.setup.ts +++ b/tests/configurations/jest.setup.ts @@ -1,20 +1 @@ -/* eslint-disable @typescript-eslint/no-unsafe-return */ -/* eslint-disable @typescript-eslint/no-unsafe-assignment */ - -import Cesium, { Cartographic, CesiumTerrainProvider } from 'cesium'; import 'reflect-metadata'; - -jest.mock('cesium', () => { - const originalCesium = jest.requireActual('cesium'); - - return { - // eslint-disable-next-line @typescript-eslint/naming-convention - __esModule: true, - ...originalCesium, - sampleTerrainMostDetailed: jest - .fn() - .mockImplementation(async (provider: CesiumTerrainProvider, positions: Cesium.Cartographic[]): Promise => { - return Promise.resolve(positions.map((pos) => ({ ...pos, height: 66 } as Cartographic))); - }), - }; -}); From 2a94786b4ec98caf413d8a5cd36b44d12174ef40 Mon Sep 17 00:00:00 2001 From: shimoncohen Date: Thu, 3 Sep 2026 11:13:51 +0300 Subject: [PATCH 05/25] feat: geotiff COG height provider with bilinear sampling Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01ATQerkHAV3Vur82HVVFack --- src/heights/models/geotiffHeightProvider.ts | 60 ++++++++++++++++++ .../models/geotiffHeightProvider.spec.ts | 61 +++++++++++++++++++ 2 files changed, 121 insertions(+) create mode 100644 src/heights/models/geotiffHeightProvider.ts create mode 100644 tests/unit/heights/models/geotiffHeightProvider.spec.ts diff --git a/src/heights/models/geotiffHeightProvider.ts b/src/heights/models/geotiffHeightProvider.ts new file mode 100644 index 0000000..6b72f05 --- /dev/null +++ b/src/heights/models/geotiffHeightProvider.ts @@ -0,0 +1,60 @@ +import { fromUrl } from 'geotiff'; +import type { GeoTIFFImage } from 'geotiff'; +import { GeoPoint } from '../interfaces'; + +const DEFAULT_NODATA = -32768; + +export default class GeotiffHeightProvider { + private constructor( + private readonly image: GeoTIFFImage, + private readonly originX: number, + private readonly originY: number, + private readonly pixelWidth: number, // degrees/pixel, positive + private readonly pixelHeight: number, // degrees/pixel, negative + private readonly rasterWidth: number, + private readonly rasterHeight: number, + private readonly noData: number + ) {} + + public static async fromUrl(url: string, headers?: Record): Promise { + const tiff = await fromUrl(url, headers ? { headers } : {}); + const image = await tiff.getImage(0); + const [originX, originY] = image.getOrigin(); + const [pixelWidth, pixelHeight] = image.getResolution(); + const noData = image.getGDALNoData() ?? DEFAULT_NODATA; + + return new GeotiffHeightProvider(image, originX, originY, pixelWidth, pixelHeight, image.getWidth(), image.getHeight(), noData); + } + + public async sample(points: GeoPoint[]): Promise<(number | null)[]> { + return Promise.all(points.map(async (point) => this.sampleOne(point))); + } + + private async sampleOne(point: GeoPoint): Promise { + const fx = (point.longitude - this.originX) / this.pixelWidth; + const fy = (point.latitude - this.originY) / this.pixelHeight; + const px = Math.floor(fx); + const py = Math.floor(fy); + + // Need a full 2x2 neighborhood for bilinear interpolation. + if (px < 0 || py < 0 || px >= this.rasterWidth - 1 || py >= this.rasterHeight - 1) { + return null; + } + + const dx = fx - px; + const dy = fy - py; + + const raster = (await this.image.readRasters({ window: [px, py, px + 2, py + 2] })) as unknown as number[][]; + const band = raster[0]; + const [tl, tr, bl, br] = [band[0], band[1], band[2], band[3]]; + + if ([tl, tr, bl, br].some((value) => value === this.noData)) { + return null; + } + + const top = tl + (tr - tl) * dx; + const bottom = bl + (br - bl) * dx; + + return top + (bottom - top) * dy; + } +} diff --git a/tests/unit/heights/models/geotiffHeightProvider.spec.ts b/tests/unit/heights/models/geotiffHeightProvider.spec.ts new file mode 100644 index 0000000..3bc9c9c --- /dev/null +++ b/tests/unit/heights/models/geotiffHeightProvider.spec.ts @@ -0,0 +1,61 @@ +import GeotiffHeightProvider from '../../../../src/heights/models/geotiffHeightProvider'; + +jest.mock('geotiff', () => ({ + fromUrl: jest.fn(), +})); + +// eslint-disable-next-line @typescript-eslint/no-var-requires +const { fromUrl } = require('geotiff') as { fromUrl: jest.Mock }; + +// origin [34,33], resolution [0.001,-0.001], 100x100, nodata -32768 +const makeImage = (band: number[], nodata: number | null = -32768) => ({ + getOrigin: () => [34.0, 33.0, 0], + getResolution: () => [0.001, -0.001], + getWidth: () => 100, + getHeight: () => 100, + getGDALNoData: () => nodata, + readRasters: jest.fn().mockResolvedValue([Int16Array.from(band)]), +}); + +const mockTiff = (image: unknown): void => { + fromUrl.mockResolvedValue({ getImage: jest.fn().mockResolvedValue(image) }); +}; + +describe('GeotiffHeightProvider', () => { + afterEach(() => jest.clearAllMocks()); + + it('bilinearly interpolates the height at a point', async () => { + // point at pixel (0,0)+0.5 in both axes → center of the 2x2 window + mockTiff(makeImage([200, 202, 204, 206])); + const provider = await GeotiffHeightProvider.fromUrl('http://gw/cogs/x.tif'); + + const [height] = await provider.sample([{ longitude: 34.0005, latitude: 32.9995 }]); + + expect(height).toBeCloseTo(203, 5); + }); + + it('returns null when any neighbor is nodata', async () => { + mockTiff(makeImage([200, -32768, 204, 206])); + const provider = await GeotiffHeightProvider.fromUrl('http://gw/cogs/x.tif'); + + const [height] = await provider.sample([{ longitude: 34.0005, latitude: 32.9995 }]); + + expect(height).toBeNull(); + }); + + it('returns null for a point outside the raster', async () => { + mockTiff(makeImage([200, 202, 204, 206])); + const provider = await GeotiffHeightProvider.fromUrl('http://gw/cogs/x.tif'); + + const [height] = await provider.sample([{ longitude: 10.0, latitude: 10.0 }]); + + expect(height).toBeNull(); + }); + + it('passes token headers through to fromUrl', async () => { + mockTiff(makeImage([1, 1, 1, 1])); + await GeotiffHeightProvider.fromUrl('http://gw/cogs/x.tif', { 'x-api-key': 'T' }); + + expect(fromUrl).toHaveBeenCalledWith('http://gw/cogs/x.tif', { headers: { 'x-api-key': 'T' } }); + }); +}); From 1914d02ff8fd7b4780ecc872768abeb78647a19c Mon Sep 17 00:00:00 2001 From: shimoncohen Date: Thu, 3 Sep 2026 11:14:19 +0300 Subject: [PATCH 06/25] docs: record jest.setup.ts cesium-mock removal in plan Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01ATQerkHAV3Vur82HVVFack --- docs/superpowers/plans/2026-09-03-geotiff-heights-migration.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/superpowers/plans/2026-09-03-geotiff-heights-migration.md b/docs/superpowers/plans/2026-09-03-geotiff-heights-migration.md index ede51df..6596367 100644 --- a/docs/superpowers/plans/2026-09-03-geotiff-heights-migration.md +++ b/docs/superpowers/plans/2026-09-03-geotiff-heights-migration.md @@ -36,6 +36,7 @@ - `src/heights/middlewares/validateRequest.ts` — type `GeoPoint[]`; drop Cesium import. - `src/heights/routes/heightsRouter.ts` — remove the two radian middlewares from the chain. - `package.json` — remove `cesium`, add `geotiff`. +- `tests/configurations/jest.setup.ts` — remove the global `jest.mock('cesium', ...)` (mocked `sampleTerrainMostDetailed`, now obsolete); keep only `import 'reflect-metadata';`. Runs before EVERY unit + integration spec, so it must be de-cesium'd or all specs fail to load. (Handled during Task 3, since it blocks Task 3's own spec.) - `tests/configurations/testContainerConfig.ts` — records use `GEOTIFF` links; mock `GeotiffHeightProvider.fromUrl`. - `tests/unit/heights/models/heightModel.spec.ts` — degrees input; mock geotiff provider. - `tests/unit/heights/middlewares/heightsMiddlewares.spec.ts` — remove radian/degree middleware tests. From bb5b0aebf08d12d44c2d8a229ed9de5c62279ff1 Mon Sep 17 00:00:00 2001 From: shimoncohen Date: Thu, 3 Sep 2026 11:18:03 +0300 Subject: [PATCH 07/25] feat: open geotiff providers from GEOTIFF catalog links Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01ATQerkHAV3Vur82HVVFack --- src/containerConfig.ts | 2 +- src/heights/models/DEMTerrainCacheManager.ts | 68 +++++++++----------- 2 files changed, 33 insertions(+), 37 deletions(-) diff --git a/src/containerConfig.ts b/src/containerConfig.ts index 563b436..785fb45 100644 --- a/src/containerConfig.ts +++ b/src/containerConfig.ts @@ -51,7 +51,7 @@ export const registerExternalValues = async (options?: RegisterOptions): Promise if (!isSame(dataValue, Object.values(catalogRecordsServiceInstance.getValue()))) { catalogRecordsServiceInstance.setValue(Object.fromEntries(dataValue.map((record) => [record.id as string, record]))); - await demTerrainCacheManager.initTerrainProviders(dataValue); + await demTerrainCacheManager.initProviders(dataValue); logger.info({ msg: `CatalogRecords UPDATED - ${dataValue.length} records fetched`, diff --git a/src/heights/models/DEMTerrainCacheManager.ts b/src/heights/models/DEMTerrainCacheManager.ts index 3927a91..eb69f7e 100644 --- a/src/heights/models/DEMTerrainCacheManager.ts +++ b/src/heights/models/DEMTerrainCacheManager.ts @@ -1,61 +1,57 @@ import { inject, injectable } from 'tsyringe'; import { IConfig } from 'config'; -import { CesiumTerrainProvider, Resource } from 'cesium'; import { PycswDemCatalogRecord } from '@map-colonies/mc-model-types'; -import { TerrainProviders } from '../interfaces'; +import { HeightProviders } from '../interfaces'; import { SERVICES } from '../../common/constants'; +import GeotiffHeightProvider from './geotiffHeightProvider'; -const QMESH_PROTOCOL = 'TERRAIN_QMESH'; -const TERRAINS_FOLDER = 'terrains/'; +const GEOTIFF_PROTOCOL = 'GEOTIFF'; +const COGS_FOLDER = 'cogs/'; @injectable() export default class DEMTerrainCacheManager { - public terrainProviders: TerrainProviders = {}; - public constructor(@inject(SERVICES.CONFIG) private readonly config: IConfig) {} + public heightProviders: HeightProviders = {}; - private transformRouteToService(cswUrl: string) { - const serviceURL = this.config.get('s3Gateway.url'); + public constructor(@inject(SERVICES.CONFIG) private readonly config: IConfig) {} - return `${serviceURL}/${TERRAINS_FOLDER}${cswUrl.split(TERRAINS_FOLDER)[1]}`; - } + public async initProviders(demCatalogRecords: PycswDemCatalogRecord[]): Promise { + const heightProviders: HeightProviders = {}; - private getTokenResource(url: string): Resource { - const tokenProps: Record = { url }; + const geotiffRecords = demCatalogRecords.filter((record) => record.links?.some((link) => link.protocol === GEOTIFF_PROTOCOL)); - const injectionType = this.config.get('accessToken.injectionType'); - const attributeName = this.config.get('accessToken.attributeName'); - const tokenValue = this.config.get('accessToken.tokenValue'); + for (const record of geotiffRecords) { + const link = record.links?.find((currentLink) => currentLink.protocol === GEOTIFF_PROTOCOL); - if (injectionType && injectionType.toLowerCase() === 'header') { - tokenProps.headers = { - [attributeName]: tokenValue, - } as Record; - } else if (injectionType && injectionType.toLowerCase() === 'queryparam') { - tokenProps.queryParameters = { - [attributeName]: tokenValue, - } as Record; + if (link) { + const objectUrl = this.transformRouteToObjectUrl(link.url as string); + const { url, headers } = this.buildAuthenticatedUrl(objectUrl); + heightProviders[record.id as string] = await GeotiffHeightProvider.fromUrl(url, headers); + } } - return new Resource({ ...(tokenProps as unknown as Resource) }); + this.heightProviders = heightProviders; } - public async initTerrainProviders(demCatalogRecords: PycswDemCatalogRecord[]): Promise { - const terrainProviders: TerrainProviders = {}; + private transformRouteToObjectUrl(linkUrl: string): string { + const serviceURL = this.config.get('s3Gateway.url'); - const qmeshRecords = demCatalogRecords.filter((record) => { - return record.links?.some((link) => link.protocol === QMESH_PROTOCOL); - }); + return `${serviceURL}/${COGS_FOLDER}${linkUrl.split(COGS_FOLDER)[1]}`; + } - for (const record of qmeshRecords) { - const recordProviderLink = record.links?.find((link) => link.protocol === QMESH_PROTOCOL); + private buildAuthenticatedUrl(objectUrl: string): { url: string; headers?: Record } { + const injectionType = this.config.get('accessToken.injectionType'); + const attributeName = this.config.get('accessToken.attributeName'); + const tokenValue = this.config.get('accessToken.tokenValue'); - if (recordProviderLink) { - const provider = await CesiumTerrainProvider.fromUrl(this.getTokenResource(this.transformRouteToService(recordProviderLink.url as string))); + if (injectionType.toLowerCase() === 'header') { + return { url: objectUrl, headers: { [attributeName]: tokenValue } }; + } - terrainProviders[record.id as string] = provider; - } + if (injectionType.toLowerCase() === 'queryparam') { + const separator = objectUrl.includes('?') ? '&' : '?'; + return { url: `${objectUrl}${separator}${attributeName}=${encodeURIComponent(tokenValue)}` }; } - this.terrainProviders = terrainProviders; + return { url: objectUrl }; } } From b68c6a4bc8a5a6ea541798a98a81f9537825fac6 Mon Sep 17 00:00:00 2001 From: shimoncohen Date: Thu, 3 Sep 2026 11:31:40 +0300 Subject: [PATCH 08/25] test: mock geotiff module globally to avoid nested-ESM load under jest Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01ATQerkHAV3Vur82HVVFack --- tests/configurations/jest.setup.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/configurations/jest.setup.ts b/tests/configurations/jest.setup.ts index d2c9bc6..b6b10a2 100644 --- a/tests/configurations/jest.setup.ts +++ b/tests/configurations/jest.setup.ts @@ -1 +1,5 @@ import 'reflect-metadata'; + +// geotiff ships a nested ESM-only quick-lru that Jest's CJS runtime cannot parse. +// Mock the module so the real graph never loads; tests stub provider behavior directly. +jest.mock('geotiff', () => ({ fromUrl: jest.fn() })); From 5011651e2aed4773f024465658e4746eeee1af88 Mon Sep 17 00:00:00 2001 From: shimoncohen Date: Thu, 3 Sep 2026 11:31:45 +0300 Subject: [PATCH 09/25] feat: sample point heights from geotiff providers in degrees Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01ATQerkHAV3Vur82HVVFack --- src/heights/models/heightsManager.ts | 343 ++++-------------- tests/configurations/testContainerConfig.ts | 15 +- tests/unit/heights/models/heightModel.spec.ts | 120 +----- 3 files changed, 84 insertions(+), 394 deletions(-) diff --git a/src/heights/models/heightsManager.ts b/src/heights/models/heightsManager.ts index c262871..80c14de 100644 --- a/src/heights/models/heightsManager.ts +++ b/src/heights/models/heightsManager.ts @@ -1,4 +1,3 @@ -import { Cartographic, Math as CesiumMath, RequestScheduler, sampleTerrainMostDetailed } from 'cesium'; import { Polygon } from 'geojson'; import client from 'prom-client'; import { container, inject, injectable } from 'tsyringe'; @@ -10,8 +9,7 @@ import { CommonErrors } from '../../common/commonErrors'; import { SERVICES } from '../../common/constants'; import { IConfig } from '../../common/interfaces'; import { CATALOG_RECORDS_MAP, DEM_TERRAIN_CACHE_MANAGER } from '../../containerConfig'; -import { PosWithHeight, PosWithTerrainProvider, TerrainTypes } from '../interfaces'; -import { cartographicArrayClusteringForHeightRequests } from '../utilities'; +import { GeoPoint, PosWithHeight, PosWithProvider, TerrainTypes } from '../interfaces'; import DEMTerrainCacheManager from './DEMTerrainCacheManager'; import { CatalogRecords } from './catalogRecords'; @@ -32,8 +30,8 @@ export class HeightsManager { return container.resolve(CATALOG_RECORDS_MAP).getValue(); } - private get terrainProviders() { - return container.resolve(DEM_TERRAIN_CACHE_MANAGER).terrainProviders; + private get heightProviders() { + return container.resolve(DEM_TERRAIN_CACHE_MANAGER).heightProviders; } private readonly elevationsRequestsCounter?: client.Counter<'points_number'>; @@ -45,7 +43,7 @@ export class HeightsManager { @inject(SERVICES.METRICS_REGISTRY) registry?: client.Registry ) { if (registry !== undefined) { - // eslint-disable-next-line @typescript-eslint/no-this-alias + // eslint-disable-next-line @typescript-eslint/no-this-alias const self = this; new client.Gauge({ name: 'elevations_current_requests_count', @@ -65,12 +63,8 @@ export class HeightsManager { } } - public async getPoints(points: Cartographic[], requestedProductType: TerrainTypes, reqCtx?: Record): Promise { - this.logger.info({ - pointsNumber: points.length, - location: '[HeightsManager] [getPoints]', - ...reqCtx, - }); + public async getPoints(points: GeoPoint[], requestedProductType: TerrainTypes, reqCtx?: Record): Promise { + this.logger.info({ pointsNumber: points.length, location: '[HeightsManager] [getPoints]', ...reqCtx }); this.runningRequests++; // eslint-disable-next-line @typescript-eslint/naming-convention @@ -95,323 +89,110 @@ export class HeightsManager { } private async samplePositionsHeights( - positionsArr: Cartographic[], + positionsArr: GeoPoint[], requestedProductType: TerrainTypes, reqCtx?: Record ): Promise<{ positions: PosWithHeight[]; totalRequests: number }> { - const MAX_REQ_PER_BATCH = 150; - const MAXIMUM_TILES_PER_REQUEST = this.config.has('maximumTilesPerRequest') ? this.config.get('maximumTilesPerRequest') : undefined; - - // Attach providers const attachProviderStart = performance.now(); - - const positionsWithProviders = this.attachTerrainProviderToPositions(positionsArr, requestedProductType); - - const attachProviderEnd = performance.now(); - + const positionsWithProviders = this.attachProviderToPositions(positionsArr, requestedProductType); this.logger.info({ - attachProviderTime: attachProviderEnd - attachProviderStart, + attachProviderTime: performance.now() - attachProviderStart, pointsNumber: positionsArr.length, location: '[HeightsManager] [samplePositionsHeights]', ...reqCtx, }); - // Positions Clustering - const clusteringStart = performance.now(); - - const { optimizedCluster: sampleTerrainClusteredPositions, totalRequests } = cartographicArrayClusteringForHeightRequests( - positionsWithProviders, - MAX_REQ_PER_BATCH - ); - - const clusteringEnd = performance.now(); - - this.logger.info({ - clusteringTime: clusteringEnd - clusteringStart, - pointsNumber: positionsArr.length, - location: '[HeightsManager] [samplePositionsHeights]', - ...reqCtx, - }); - - if (typeof MAXIMUM_TILES_PER_REQUEST !== 'undefined' && totalRequests > MAXIMUM_TILES_PER_REQUEST) { - this.logger.error({ - msg: 'Points density is too low to compute.', - totalRequests: totalRequests, - location: '[HeightsManager] [samplePositionsHeights]', - ...reqCtx, - }); - this.runningRequests--; - throw this.commonErrors.POINTS_DENSITY_TOO_LOW_ERROR; + // Group points by the provider chosen for them (null = no provider). + const groups = new Map(); + for (const position of positionsWithProviders) { + const key = position.providerKey ?? null; + const bucket = groups.get(key) ?? []; + bucket.push({ longitude: position.longitude, latitude: position.latitude }); + groups.set(key, bucket); } + const groupEntries = [...groups.entries()]; const finalPositionsWithHeights: PosWithHeight[] = []; - // Terrain sampling batches - const { results } = await PromisePool.for(sampleTerrainClusteredPositions) - .withConcurrency(sampleTerrainClusteredPositions.length) - .useCorrespondingResults() - .process(async (batch) => { - const samplingStart = performance.now(); - - if (batch.providerKey === null) { - this.logger.info({ - msg: `No terrain to sample these positions`, - positionsOutsideOfProviders: JSON.stringify(batch.positions), - positionsOutsideOfProvidersCount: batch.positions.length, - pointsNumber: positionsArr.length, - location: '[HeightsManager] [samplePositionsHeights]', - ...reqCtx, - }); - - return batch.positions; + const { results } = await PromisePool.for(groupEntries) + .withConcurrency(Math.max(1, groupEntries.length)) + .process(async ([providerKey, points]) => { + if (providerKey === null) { + return points.map((point) => ({ ...point, height: null } as PosWithHeight)); } - const provider = this.terrainProviders[batch.providerKey]; - const qmeshRecord = this.catalogRecordsMap[batch.providerKey]; - - const positionsWithHeights = await sampleTerrainMostDetailed(provider, batch.positions); - - // Attach additional info on top of each position returned via the catalog record (productId added as ref to products dictionary metadata). - positionsWithHeights.forEach((pos) => { - (pos as unknown as Record)['productId'] = qmeshRecord['productId']; - }); - - const samplingEnd = performance.now(); + const samplingStart = performance.now(); + const provider = this.heightProviders[providerKey]; + const record = this.catalogRecordsMap[providerKey]; + const heights = await provider.sample(points); this.logger.info({ - terrainSamplingTime: samplingEnd - samplingStart, - providerId: batch.providerKey, + terrainSamplingTime: performance.now() - samplingStart, + providerId: providerKey, pointsNumber: positionsArr.length, location: '[HeightsManager] [samplePositionsHeights]', ...reqCtx, }); - return positionsWithHeights as PosWithHeight[]; + return points.map((point, index) => { + const height = heights[index]; + return { + ...point, + height, + ...(height !== null ? { productId: record.productId as string } : {}), + } as PosWithHeight; + }); }); finalPositionsWithHeights.push(...(results as PosWithHeight[][]).flat()); - // CESIUM without involving a VIEWER (visualization) behaves differently and doesn't manage a REQUESTS cleanup - // Full EXPLANATOIN is here: https://github.com/CesiumGS/cesium/issues/7670 - // @ts-ignore - // eslint-disable-next-line @typescript-eslint/no-unsafe-call - RequestScheduler.update(); - - return { positions: finalPositionsWithHeights, totalRequests }; + return { positions: finalPositionsWithHeights, totalRequests: groupEntries.length }; } - private attachTerrainProviderToPositions(positions: Cartographic[], requestedProductType: TerrainTypes): PosWithTerrainProvider[] { + private attachProviderToPositions(positions: GeoPoint[], requestedProductType: TerrainTypes): PosWithProvider[] { /* - * Filter terrain providers by requested product type - * Filter terrain providers by footprint point intersection - * Sort by highest resolution (Lower is better) - * Attach to the point the (first) provider with highest resolution + * Filter providers by requested product type (unless MIXED) + * Filter providers by footprint point intersection + * Sort by highest resolution (lower resolutionMeter is better), tie-break on newest updateDate + * Attach the best provider key to the point */ - return positions.map((position) => { - const terrainProvidersEntries = Object.entries(this.terrainProviders); + const providerEntries = Object.entries(this.heightProviders); - // Filter terrain providers by requested product type if not MIXED - const productTypeFilteredTerrains = + const productTypeFiltered = requestedProductType !== TerrainTypes.MIXED - ? terrainProvidersEntries.filter(([terrainKey]) => { - const qmeshRecord = this.catalogRecordsMap[terrainKey]; - return qmeshRecord.productType?.includes(requestedProductType); - }) - : terrainProvidersEntries; + ? providerEntries.filter(([key]) => this.catalogRecordsMap[key].productType?.includes(requestedProductType)) + : providerEntries; - // Filter terrain providers by footprint point intersection - const terrainsFilterByFootprint = productTypeFilteredTerrains.filter(([terrainKey]) => { - const qmeshRecord = this.catalogRecordsMap[terrainKey]; - const isPointInFootprint = booleanPointInPolygon( - [CesiumMath.toDegrees(position.longitude), CesiumMath.toDegrees(position.latitude)], - qmeshRecord.footprint as Feature - ); - return isPointInFootprint; - }); + const footprintFiltered = productTypeFiltered.filter(([key]) => + booleanPointInPolygon([position.longitude, position.latitude], this.catalogRecordsMap[key].footprint as Feature) + ); - // Sort by highest resolution (Lower is better) - const sortedTerrainsByResolution = terrainsFilterByFootprint.sort(([terrainAKey], [terrainBKey]) => { + const sorted = footprintFiltered.sort(([aKey], [bKey]) => { const A_BEFORE_B = -1; const B_BEFORE_A = 1; - - const qmeshRecordA = this.catalogRecordsMap[terrainAKey]; - const qmeshRecordB = this.catalogRecordsMap[terrainBKey]; - - let dateB = null, - dateA = null; + const recordA = this.catalogRecordsMap[aKey]; + const recordB = this.catalogRecordsMap[bKey]; switch (true) { - case (qmeshRecordA.resolutionMeter as number) < (qmeshRecordB.resolutionMeter as number): + case (recordA.resolutionMeter as number) < (recordB.resolutionMeter as number): return A_BEFORE_B; - case (qmeshRecordA.resolutionMeter as number) > (qmeshRecordB.resolutionMeter as number): + case (recordA.resolutionMeter as number) > (recordB.resolutionMeter as number): return B_BEFORE_A; - default: - // Equal resolutions, compare update date - dateB = new Date(qmeshRecordB.updateDate as string | number | Date); - dateA = new Date(qmeshRecordA.updateDate as string | number | Date); + default: { + const dateB = new Date(recordB.updateDate as string | number | Date); + const dateA = new Date(recordA.updateDate as string | number | Date); return dateB.getTime() - dateA.getTime(); + } } }); - if (sortedTerrainsByResolution.length === 0) { - return { - ...position, - } as PosWithTerrainProvider; + if (sorted.length === 0) { + return { ...position } as PosWithProvider; } - // Attach to the point the (first) provider with highest resolution - const [terrainKey, provider] = sortedTerrainsByResolution[0]; - - return { - ...position, - providerKey: terrainKey, - terrainProvider: provider, - } as PosWithTerrainProvider; + const [providerKey] = sorted[0]; + return { ...position, providerKey } as PosWithProvider; }); } - - // public async getPath(path: GeoJSON): Promise { - // this.logger.info({ msg: 'Getting path heights' }); - // const start = new Date(); - // const result = await this.sample(path, { level: 11 }); - // const end = new Date(); - // console.log(result); - // console.log(`${end.getTime() - start.getTime()} ms`); - // return result; - // } - - // public async getPolygon(polygon: GeoJSON): Promise { - // this.logger.info({ msg: 'Getting polygon heights' }); - - // const polygonBbox = bbox(polygon); - // const cellSide = 600.0; // distance between points (in units) - // const options = { - // units: 'meters' as Units, // used in calculating cellSide, can be: degrees, radians, miles, or kilometers (default) - // mask: (polygon as FeatureCollection).features[0] as Feature // if passed a Polygon or MultiPolygon, the grid Points will be created only inside it - // }; - - // // Creates a Point grid from a bounding box, FeatureCollection or Feature. - // const polygonPointGrid = pointGrid(polygonBbox, cellSide, options); // grid of points inside the given polygon - - // // let bbox = turf.bbox(polygon); - // // let cellWidth = 0.05; - // // let cellHeight = 0.05; - - // // let bufferedBbox = turf.bbox(turf.buffer(polygon, cellWidth, {units: 'kilometers'})); - // // let options = { units: "kilometers", mask: polygon}; - // // let squareGrid = turf.squareGrid( - // // bufferedBbox, - // // // bbox, - // // cellWidth, - // // options - // // ); - - // // turf.featureEach(squareGrid, function (currentFeature, featureIndex) { - // // let intersected = turf.intersect(polygon.features[0], currentFeature); - // // }); - - // const start = new Date(); - // const result = await this.sample(polygonPointGrid, { level: 11 }); - // const end = new Date(); - // console.log(result); - // console.log(`${end.getTime() - start.getTime()} ms`); - // return result; - // } - - // public async getHeights(geojson: GeoJSON): Promise { - // this.logger.info({ msg: 'Getting heights' }); - // const start = new Date(); - // const result = await this.sample(geojson, { level: 11 }); - // const end = new Date(); - // console.log(result); - // console.log(`${end.getTime() - start.getTime()} ms`); - // return result; - // } - - // public async getHeight(coordinates: ICoordinates): Promise { - // this.logger.info({ msg: 'Getting height' }); - // const point: GeoJSON = { - // "type": "Feature", - // "properties": {}, - // "geometry": { - // "type": "Point", - // "coordinates": [ - // ...Object.values(coordinates).map(value => +value) - // ] - // } - // }; - // const start = new Date(); - // const result = await this.sample(point, { level: 11 }); - // const end = new Date(); - // console.log(result); - // console.log(`${end.getTime() - start.getTime()} ms`); - // return { dem: ((result as Feature).geometry as Point).coordinates[2] }; - // } - - // private async sample(data: GeoJSON, options?: { level: number }): Promise { - // return new Promise((resolve, reject) => { - // const positions: Cartographic[] = []; - // const dPositions: {longitude: number, latitude: number}[] = []; - // const paths: string[][] = []; - - // traverse(data).forEach(function() { - // // @ts-ignore - // if (this.isLeaf && Number.isFinite(this.node) && Array.isArray(this.parent?.node) && this.parent?.node.length > 1 && this.key == '0') { - // positions.push(Cartographic.fromDegrees(this.parent?.node[0], this.parent?.node[1])); - // dPositions.push({ longitude: th // Here we should attach additional info on top of each position returned via the catalog record.is.parent?.node[0], latitude: this.parent?.node[1] }); - // paths.push(this.parent?.path as string[]); - // } - // }); - - // if (positions.length && dPositions.length) { - - // const CesiumProvider = new CesiumTerrainProvider({ - // url: config.get('terrainProviderUrl') - // }); - - // const MIN_ZOOM_LEVEL = 0; - // const MAX_ZOOM_LEVEL = 11; - - // let sampleTerrainPromise; - // if (options?.level) { - // if ((options.level < MIN_ZOOM_LEVEL) || (options.level > MAX_ZOOM_LEVEL)) { - // return reject(new Error(`Level must between ${MIN_ZOOM_LEVEL} and ${MAX_ZOOM_LEVEL}`)); - // } - // sampleTerrainPromise = sampleTerrain(CesiumProvider, options.level, positions); - // } else { - // sampleTerrainPromise = sampleTerrainMostDetailed(CesiumProvider, positions); - // // sampleTerrainPromise = fetch('https://dem-int-proxy-production-nginx-s3-gateway-route-integration.apps.j1lk3njp.eastus.aroapp.io/terrains/srtm100/11/2446/1394.terrain?token=eyJhbGciOiJSUzI1NiIsImtpZCI6Im1hcC1jb2xvbmllcy1pbnQifQ.eyJkIjpbInJhc3RlciIsInJhc3RlcldtcyIsInJhc3RlckV4cG9ydCIsImRlbSIsInZlY3RvciIsIjNkIl0sImlhdCI6MTY3NDYzMjM0Niwic3ViIjoibWFwY29sb25pZXMtYXBwIiwiaXNzIjoibWFwY29sb25pZXMtdG9rZW4tY2xpIn0.D1u28gFlxf_Z1bzIiRHZonUgrdWwhZy8DtmQj15cIzaABRUrGV2n_OJlgWTuNfrao0SbUZb_s0_qUUW6Gz_zO3ET2bVx5xQjBu0CaIWdmUPDjEYr6tw-eZx8EjFFIyq3rs-Fo0daVY9cX1B2aGW_GeJir1oMnJUURhABYRoh60azzl_utee9UdhDpnr_QElNtzJZIKogngsxCWp7tI7wkTuNCBaQM7aLEcymk0ktxlWEAt1E0nGt1R-bx-HnPeeQyZlxx4UQ1nuYTijpz7N8poaCCExOFeafj9T7megv2BzTrKWgfM1eai8srSgNa3I5wKuW0EyYnGZxdbJe8aseZg&kuku='+dPositions[i].longitude.toFixed(8)); - // } - // sampleTerrainPromise.then(() => { - // paths.forEach((path, i) => { - // if (path.length) { - // traverse(data).set(path, [dPositions[i].longitude, dPositions[i].latitude, positions[i].height ? Number(positions[i].height.toFixed(2)) : undefined]); - // } else { - // return reject (new Error('Invalid GeoJSON input file')); - // } - // }); - // return resolve(data); - // }).catch(function (e) { - // return reject(e); - // }); - // // sampleTerrainPromise.then(() => { - // // return resolve({ - // // "type": "Feature", - // // "properties": { - // // "al": "kuku" - // // }, - // // "geometry": { - // // "coordinates": [ - - // // ], - // // "type": "Polygon" - // // }}) - // // }); - - // } else { - // return reject(new Error('No coordinates found in input file')); - // } - // }); - // } } diff --git a/tests/configurations/testContainerConfig.ts b/tests/configurations/testContainerConfig.ts index ec1f0d9..7af5551 100644 --- a/tests/configurations/testContainerConfig.ts +++ b/tests/configurations/testContainerConfig.ts @@ -6,6 +6,7 @@ import { PycswDemCatalogRecord } from '@map-colonies/mc-model-types'; import { SERVICES } from '../../src/common/constants'; import { CATALOG_RECORDS_MAP, DEM_TERRAIN_CACHE_MANAGER, PRODUCT_METADATA_FIELDS } from '../../src/containerConfig'; import DEMTerrainCacheManager from '../../src/heights/models/DEMTerrainCacheManager'; +import GeotiffHeightProvider from '../../src/heights/models/geotiffHeightProvider'; import { CatalogRecords } from '../../src/heights/models/catalogRecords'; async function registerTestValues(shouldInitTerrainProviders = true): Promise { @@ -74,8 +75,8 @@ async function registerTestValues(shouldInitTerrainProviders = true): Promise points.map(() => 100), + } as unknown as GeotiffHeightProvider); + await (async (): Promise => { const catalogTestRecordsServiceInstance = container.resolve(CATALOG_RECORDS_MAP); const demTestTerrainCacheManager = container.resolve(DEM_TERRAIN_CACHE_MANAGER); @@ -157,7 +162,7 @@ async function registerTestValues(shouldInitTerrainProviders = true): Promise { - return Cartographic.fromDegrees(pos.longitude, pos.latitude); - }); - - mockJsonDataLowDensity.positions = mockJsonDataLowDensity.positions.map((pos) => { - return Cartographic.fromDegrees(pos.longitude, pos.latitude); - }); - - mockJsonDataOutOfBounds.positions = mockJsonDataOutOfBounds.positions.map((pos) => { - return Cartographic.fromDegrees(pos.longitude, pos.latitude); - }); - - cesiumTerrainProviderFromUrlSpy = jest.spyOn(CesiumTerrainProvider, 'fromUrl'); - - cesiumTerrainProviderFromUrlSpy.mockReturnValue({ - availability: { - available: true, - computeMaximumLevelAtPosition: () => { - return 13; - }, - }, - tilingScheme: { - positionToTileXY: () => { - // Always a maximum of 3 "Tile requests" - - const xys = [new Cartesian2(1, 2), new Cartesian2(3, 4), new Cartesian2(5, 6)]; - const randomXy = Math.floor(Math.random() * xys.length); - - return xys[randomXy]; - }, - }, - }); - }); beforeEach(async function () { await registerTestValues(); - heightsManager = container.resolve(HeightsManager); }); @@ -76,86 +23,43 @@ describe('Get Heights model', function () { }); describe('Given valid parameters', function () { - it('Should return positions with height and extra metadata fields', async function () { - const result = await heightsManager.getPoints(mockJsonData.positions, TerrainTypes.MIXED); - - expect(result).toHaveLength(mockJsonData.positions.length); - - const getHeightsResProperties = Object.keys(basicPositionResponse); - - for (const position of result) { - for (const key of getHeightsResProperties) { - expect(position[key as keyof PosWithHeight]).toBeTruthy(); - } - } - }); - it('Should return positions with height and productId', async function () { const result = await heightsManager.getPoints(mockJsonData.positions, TerrainTypes.MIXED); expect(result).toHaveLength(mockJsonData.positions.length); - for (const position of result) { - expect(position['productId']).toBeDefined(); + expect(position.height).toBe(100); + expect(position.productId).toBeDefined(); } }); - it('Should return the positions with null heights and no fields if no provider match for the request', async function () { - const nonExistingTerrainType = TerrainTypes.DSM; - const result = await heightsManager.getPoints(mockJsonData.positions, nonExistingTerrainType); + it('Should return null heights and no productId when no provider matches the product type', async function () { + const result = await heightsManager.getPoints(mockJsonData.positions, TerrainTypes.DSM); expect(result).toHaveLength(mockJsonData.positions.length); - for (const position of result) { expect(position.height).toBeNull(); - expect(position['productId']).toBeUndefined(); + expect(position.productId).toBeUndefined(); } }); - it('Should be able to return height and data for only a part of the positions', async function () { + it('Should return height only for the positions inside a provider footprint', async function () { const result = await heightsManager.getPoints(mockJsonDataOutOfBounds.positions, TerrainTypes.MIXED); expect(result).toHaveLength(mockJsonDataOutOfBounds.positions.length); - for (const position of result) { - expect(position['latitude'] && position['longitude']).toBeDefined(); + expect(position.longitude).toBeDefined(); + expect(position.latitude).toBeDefined(); const isNullHeight = (position.height as number | null) === null; - - expect(typeof position['productId'] === 'undefined').toEqual(isNullHeight); + expect(typeof position.productId === 'undefined').toEqual(isNullHeight); } }); }); describe('Given invalid params', function () { - beforeAll(function () { - cesiumTerrainProviderFromUrlSpy = jest.spyOn(CesiumTerrainProvider, 'fromUrl'); - - cesiumTerrainProviderFromUrlSpy.mockReturnValue({ - availability: { - available: true, - computeMaximumLevelAtPosition: () => { - return 13; - }, - }, - tilingScheme: { - positionToTileXY: (position: Cartographic) => { - // Making sure there is no tiles overlapping for any position. so that each position is a "request". (Assuming unique positions) - return new Cartesian2(position.latitude, position.longitude); - }, - }, - }); - }); - - it.skip('Should throw dansity too low error for 150+ points (As configured)', async function () { - await expect(heightsManager.getPoints(mockJsonDataLowDensity.positions, TerrainTypes.MIXED)).rejects.toHaveProperty( - 'errorCode', - CommonErrorCodes.POINTS_DENSITY_TOO_LOW_ERROR - ); - }); - - it('Should return empty array', async function () { - await expect(heightsManager.getPoints(emptyPositionsRequest.positions, TerrainTypes.MIXED)).resolves.toEqual([]); + it('Should return empty array for empty positions', async function () { + await expect(heightsManager.getPoints((emptyPositionsRequest as unknown as GetHeightsPointsRequest).positions, TerrainTypes.MIXED)).resolves.toEqual([]); }); }); }); From 51f76728d2235a83deabcac1503dfc0557dc5d62 Mon Sep 17 00:00:00 2001 From: shimoncohen Date: Thu, 3 Sep 2026 11:33:30 +0300 Subject: [PATCH 10/25] refactor: drop quantized-mesh tile clustering utilities Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01ATQerkHAV3Vur82HVVFack --- src/heights/utilities.ts | 91 +++------------------------------------- 1 file changed, 5 insertions(+), 86 deletions(-) diff --git a/src/heights/utilities.ts b/src/heights/utilities.ts index 565379c..cfeae87 100644 --- a/src/heights/utilities.ts +++ b/src/heights/utilities.ts @@ -1,94 +1,13 @@ -/* eslint-disable import/exports-last */ - import crypto from 'crypto'; -import { Cartographic } from 'cesium'; -import { PosWithTerrainProvider } from './interfaces'; - -export interface PositionsWithProviderKey { - positions: Cartographic[]; - providerKey: string | null; -} - -const NO_PROVIDER_KEY = 'NO-PROVIDER'; - -// eslint-disable-next-line @typescript-eslint/naming-convention -const createClustersByTerrainProvider = (data: PositionsWithProviderKey[], maxRequestsPerBatch: number): PositionsWithProviderKey[] => { - const clusters: (PositionsWithProviderKey & { count: number })[] = []; - - // Group positions by providerKey and split into subArrays - for (let i = 0; i < data.length; i++) { - const item = data[i]; - const positions = item.positions; - const batchedPos = clusters.find((value) => value.providerKey === item.providerKey && value.count < maxRequestsPerBatch); - if (batchedPos) { - batchedPos.positions.push(...positions); - batchedPos.count++; - } else { - clusters.push({ providerKey: item.providerKey === NO_PROVIDER_KEY ? null : item.providerKey, positions, count: 1 }); - } - } - - // @ts-ignore - return clusters; -}; - -export const cartographicArrayClusteringForHeightRequests = ( - positions: PosWithTerrainProvider[], - maxRequestsPerBatch = 1 -): { optimizedCluster: PositionsWithProviderKey[]; totalRequests: number } => { - const positionsClustersByTile = new Map(); - - positions.forEach((pos) => { - // Check if position doesn't have attached terrain provider - if (typeof pos.terrainProvider === 'undefined' || typeof pos.providerKey === 'undefined') { - const positionTilePath = NO_PROVIDER_KEY; - - const currentPosInTile = positionsClustersByTile.get(positionTilePath) ?? []; - - positionsClustersByTile.set(positionTilePath, [ - ...currentPosInTile, - { latitude: pos.latitude, longitude: pos.longitude, height: null } as unknown as Cartographic, - ]); - - return; - } - - // Get max level for position. - const maxLevelAtPos = pos.terrainProvider.availability!.computeMaximumLevelAtPosition(pos); - - // Get correspond tile. - const posTile = pos.terrainProvider.tilingScheme.positionToTileXY(pos, maxLevelAtPos); - - // Create unique key per tile matched. - const positionTilePath = `${pos.providerKey}_${maxLevelAtPos}_${posTile.x}_${posTile.y}`; - - // Add position to pos array in dictionary - const currentPosInTile = positionsClustersByTile.get(positionTilePath) ?? []; - - positionsClustersByTile.set(positionTilePath, [...currentPosInTile, { latitude: pos.latitude, longitude: pos.longitude } as Cartographic]); - }); - - // Create batches of length up to max requests, by provider. - - const clusteredPositionsWithProviderKey: PositionsWithProviderKey[] = Array.from(positionsClustersByTile).map(([key, val]) => { - const clusterProviderKey = key.split('_')[0]; - return { providerKey: clusterProviderKey, positions: val }; - }); - - const newOptimizedCluster = createClustersByTerrainProvider(clusteredPositionsWithProviderKey, maxRequestsPerBatch); - - const totalRequests = positionsClustersByTile.size - +positionsClustersByTile.has(NO_PROVIDER_KEY); - - return { optimizedCluster: newOptimizedCluster, totalRequests: totalRequests }; -}; -export const generateChecksum = (str: string, algorithm?: string, encoding?: crypto.BinaryToTextEncoding) => { +export const generateChecksum = (str: string, algorithm?: string, encoding?: crypto.BinaryToTextEncoding): string => { return crypto - .createHash(algorithm || 'md5') + .createHash(algorithm ?? 'md5') .update(str, 'utf8') - .digest(encoding || 'hex'); + .digest(encoding ?? 'hex'); }; -export const isSame = (src1: any, src2: any) => { +// eslint-disable-next-line @typescript-eslint/no-explicit-any +export const isSame = (src1: any, src2: any): boolean => { return generateChecksum(JSON.stringify(src1)) === generateChecksum(JSON.stringify(src2)); }; From 5a3b9fab13c64379ee85d1b93fc93497043ee242 Mon Sep 17 00:00:00 2001 From: shimoncohen Date: Thu, 3 Sep 2026 11:33:40 +0300 Subject: [PATCH 11/25] feat: discover GEOTIFF catalog records instead of qmesh Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01ATQerkHAV3Vur82HVVFack --- src/workerCatalogRecords.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/workerCatalogRecords.ts b/src/workerCatalogRecords.ts index d91ff69..a33c2e2 100644 --- a/src/workerCatalogRecords.ts +++ b/src/workerCatalogRecords.ts @@ -19,15 +19,15 @@ const END_RECORD = 1000; const getCatalogRecords = async (): Promise => { const res = await cswClient.getRecords(START_RECORD, END_RECORD, { filter: [ - // ******* DEM profile has special BOOLEAN field 'mc:hasTerrain' which holds an indication of TERRAIN_PROVIDER - // ******* Probably there is a bug when filtering by BOOLEAN field. Instead used LIKE filter that looks in LINKS field + // ******* DEM profile links carry the object protocol. We match records exposing a GEOTIFF link + // ******* (COG served from the S3 gateway) via a LIKE filter on the LINKS field. // { // field: 'mc:hasTerrain', // eq: 'True', // }, { field: 'mc:links', - like: 'TERRAIN_QMESH', + like: 'GEOTIFF', }, { field: 'mc:productStatus', From b641fe0c9e5f888ced601539e17df05e4ef28418 Mon Sep 17 00:00:00 2001 From: shimoncohen Date: Thu, 3 Sep 2026 11:34:13 +0300 Subject: [PATCH 12/25] refactor: type points as degrees, drop radians request map Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01ATQerkHAV3Vur82HVVFack --- src/heights/controllers/heightsController.ts | 6 ++---- src/heights/middlewares/validateRequest.ts | 4 ++-- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/src/heights/controllers/heightsController.ts b/src/heights/controllers/heightsController.ts index 73621c8..b8e283a 100644 --- a/src/heights/controllers/heightsController.ts +++ b/src/heights/controllers/heightsController.ts @@ -1,13 +1,11 @@ import { RequestHandler } from 'express'; -import { Cartographic } from 'cesium'; import { injectable, inject } from 'tsyringe'; import { HeightsManager } from '../models/heightsManager'; -import { PosWithHeight, TerrainTypes } from '../interfaces'; +import { GeoPoint, PosWithHeight, TerrainTypes } from '../interfaces'; import { CommonErrors } from '../../common/commonErrors'; export interface GetHeightsPointsRequest { - positions: Cartographic[]; - radiansToOriginalPositionsMap: Map; + positions: GeoPoint[]; productType?: TerrainTypes; } diff --git a/src/heights/middlewares/validateRequest.ts b/src/heights/middlewares/validateRequest.ts index 2330ddc..5281f2a 100644 --- a/src/heights/middlewares/validateRequest.ts +++ b/src/heights/middlewares/validateRequest.ts @@ -1,8 +1,8 @@ import { Logger } from '@map-colonies/js-logger'; -import { Cartographic } from 'cesium'; import { CommonErrors } from '../../common/commonErrors'; import { IConfig } from '../../common/interfaces'; import { GetHeightsHandler } from '../controllers/heightsController'; +import { GeoPoint } from '../interfaces'; export const validateRequestMiddleware: (config: IConfig, logger: Logger, commonErrors: CommonErrors) => GetHeightsHandler = ( config, @@ -11,7 +11,7 @@ export const validateRequestMiddleware: (config: IConfig, logger: Logger, common ) => { return (req, res, next) => { const MAXIMUM_TILES_PER_REQUEST = config.has('maximumTilesPerRequest') ? config.get('maximumTilesPerRequest') : undefined; - const points: Cartographic[] = req.body.positions; + const points: GeoPoint[] = req.body.positions; if (points.length === 0) { logger.error({ msg: 'Points array is empty.', pointsNumber: points.length, location: '[validateRequestMiddleware]', ...res.locals.reqCtx }); From d410d58c1fc5532654f191947dc64769a59ec906 Mon Sep 17 00:00:00 2001 From: shimoncohen Date: Thu, 3 Sep 2026 11:35:06 +0300 Subject: [PATCH 13/25] refactor: remove radian conversion middlewares from points route Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01ATQerkHAV3Vur82HVVFack --- src/heights/MOCKS/mockData.ts | 1 - src/heights/middlewares/dataToDegrees.ts | 35 ------------------------ src/heights/middlewares/dataToRadians.ts | 34 ----------------------- src/heights/routes/heightsRouter.ts | 4 --- 4 files changed, 74 deletions(-) delete mode 100644 src/heights/middlewares/dataToDegrees.ts delete mode 100644 src/heights/middlewares/dataToRadians.ts diff --git a/src/heights/MOCKS/mockData.ts b/src/heights/MOCKS/mockData.ts index 0137ed3..7337aaa 100644 --- a/src/heights/MOCKS/mockData.ts +++ b/src/heights/MOCKS/mockData.ts @@ -305,7 +305,6 @@ export const positionsOutsideOfProviders = { export const emptyPositionsRequest = { positions: [], - radiansToOriginalPositionsMap: new Map(), }; export default { diff --git a/src/heights/middlewares/dataToDegrees.ts b/src/heights/middlewares/dataToDegrees.ts deleted file mode 100644 index 5f8f93d..0000000 --- a/src/heights/middlewares/dataToDegrees.ts +++ /dev/null @@ -1,35 +0,0 @@ -import { Logger } from '@map-colonies/js-logger'; -import type { GetHeightsHandler } from '../controllers/heightsController'; -import { PosWithHeight } from '../interfaces'; - -export const positionResAsDegreesMiddleware: (logger: Logger) => GetHeightsHandler = (logger) => { - return (req, res, next) => { - const startTime = performance.now(); - const posArray = res.locals.positions as PosWithHeight[]; - const radiansToOriginalPositionsMap = req.body.radiansToOriginalPositionsMap; - - const posInDegrees = posArray.map(({ latitude, longitude, ...other }) => { - // Using the original position requested. - const originalPositionKey = `${longitude};${latitude}`; - const [originalLongitude, originalLatitude] = radiansToOriginalPositionsMap.get(originalPositionKey)?.split(';') ?? []; - - return { - latitude: Number(originalLatitude), - longitude: Number(originalLongitude), - ...other, - }; - }); - - const endTime = performance.now(); - - logger.info({ - convertToDegreesTime: endTime - startTime, - pointsNumber: posArray.length, - location: '[positionResAsDegreesMiddleware]', - ...res.locals.reqCtx, - }); - res.locals.positions = posInDegrees; - - next(); - }; -}; diff --git a/src/heights/middlewares/dataToRadians.ts b/src/heights/middlewares/dataToRadians.ts deleted file mode 100644 index 7761d3c..0000000 --- a/src/heights/middlewares/dataToRadians.ts +++ /dev/null @@ -1,34 +0,0 @@ -import { Cartographic } from 'cesium'; -import { Logger } from '@map-colonies/js-logger'; -import type { GetHeightsHandler } from '../controllers/heightsController'; - -export const convertReqPositionToRadiansMiddleware: (logger: Logger) => GetHeightsHandler = (logger) => { - return (req, res, next) => { - const startTime = performance.now(); - const posArray = req.body.positions; - const radiansToOriginalPositionsMap = new Map(); - - const positions = req.body.positions.map((pos) => { - const radiansPosition = Cartographic.fromDegrees(pos.longitude, pos.latitude); - - // Populating the dictionary between the original requested positions and the radians we work with internally. - // Key is "longitude;latitude" - radiansToOriginalPositionsMap.set(`${radiansPosition.longitude};${radiansPosition.latitude}`, `${pos.longitude};${pos.latitude}`); - - return radiansPosition; - }); - - const endTime = performance.now(); - - logger.info({ - convertToRadiansTime: endTime - startTime, - pointsNumber: posArray.length, - location: '[convertReqPositionToRadiansMiddleware]', - ...res.locals.reqCtx, - }); - - req.body = { ...req.body, radiansToOriginalPositionsMap, positions }; - - next(); - }; -}; diff --git a/src/heights/routes/heightsRouter.ts b/src/heights/routes/heightsRouter.ts index e1cfd9f..622a172 100644 --- a/src/heights/routes/heightsRouter.ts +++ b/src/heights/routes/heightsRouter.ts @@ -7,8 +7,6 @@ import { IConfig } from '../../common/interfaces'; import { PRODUCT_METADATA_FIELDS } from '../../containerConfig'; import { HeightsController } from '../controllers/heightsController'; import { createReqCtxMiddleware } from '../middlewares/createReqCtx'; -import { positionResAsDegreesMiddleware } from '../middlewares/dataToDegrees'; -import { convertReqPositionToRadiansMiddleware } from '../middlewares/dataToRadians'; import { sendResponseMiddleware } from '../middlewares/sendResponse'; import { validateRequestMiddleware } from '../middlewares/validateRequest'; import { addProductsDictionaryMiddleware } from '../middlewares/addProductsDictionary'; @@ -25,9 +23,7 @@ const heightsRouterFactory: FactoryFunction = (dependencyContainer) => { '/points', createReqCtxMiddleware(logger), validateRequestMiddleware(config, logger, commonErrors), - convertReqPositionToRadiansMiddleware(logger), controller.getPoints, - positionResAsDegreesMiddleware(logger), addProductsDictionaryMiddleware(logger, productMetadataFields), sendResponseMiddleware(logger) ); From 7310a7aaadbb9d15e9dca1890e8150e13d86eea6 Mon Sep 17 00:00:00 2001 From: shimoncohen Date: Thu, 3 Sep 2026 11:36:21 +0300 Subject: [PATCH 14/25] test: drop radian middleware unit tests Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01ATQerkHAV3Vur82HVVFack --- .../middlewares/heightsMiddlewares.spec.ts | 82 +------------------ 1 file changed, 2 insertions(+), 80 deletions(-) diff --git a/tests/unit/heights/middlewares/heightsMiddlewares.spec.ts b/tests/unit/heights/middlewares/heightsMiddlewares.spec.ts index a93eea5..8fc1b26 100644 --- a/tests/unit/heights/middlewares/heightsMiddlewares.spec.ts +++ b/tests/unit/heights/middlewares/heightsMiddlewares.spec.ts @@ -1,6 +1,5 @@ import { Logger } from '@map-colonies/js-logger'; import { PycswDemCatalogRecord } from '@map-colonies/mc-model-types'; -import { Cartographic } from 'cesium'; import { NextFunction, Request, Response } from 'express'; import { container } from 'tsyringe'; import { isUuid } from 'uuidv4'; @@ -8,12 +7,9 @@ import { CommonErrors } from '../../../../src/common/commonErrors'; import { SERVICES } from '../../../../src/common/constants'; import { IConfig } from '../../../../src/common/interfaces'; import { CATALOG_RECORDS_MAP, PRODUCT_METADATA_FIELDS } from '../../../../src/containerConfig'; -import { GetHeightsHandler, GetHeightsPointsRequest } from '../../../../src/heights/controllers/heightsController'; -import { PosWithHeight } from '../../../../src/heights/interfaces'; +import { GetHeightsHandler } from '../../../../src/heights/controllers/heightsController'; import { addProductsDictionaryMiddleware } from '../../../../src/heights/middlewares/addProductsDictionary'; import { createReqCtxMiddleware } from '../../../../src/heights/middlewares/createReqCtx'; -import { positionResAsDegreesMiddleware } from '../../../../src/heights/middlewares/dataToDegrees'; -import { convertReqPositionToRadiansMiddleware } from '../../../../src/heights/middlewares/dataToRadians'; import { validateRequestMiddleware } from '../../../../src/heights/middlewares/validateRequest'; import { CatalogRecords } from '../../../../src/heights/models/catalogRecords'; import { registerTestValues } from '../../../configurations/testContainerConfig'; @@ -26,8 +22,6 @@ describe('Get heights middlewares', function () { let config: IConfig; let commonErrors: CommonErrors; let reqCtxMiddleware: GetHeightsHandler; - let dataToRadiansMiddleware: GetHeightsHandler; - let dataToDegreesMiddleware: GetHeightsHandler; let reqValidateMiddleware: GetHeightsHandler; let addProdDictionaryMiddleware: GetHeightsHandler; let productMetadataFields: string[]; @@ -41,8 +35,6 @@ describe('Get heights middlewares', function () { productMetadataFields = container.resolve(PRODUCT_METADATA_FIELDS); reqCtxMiddleware = createReqCtxMiddleware(logger); - dataToRadiansMiddleware = convertReqPositionToRadiansMiddleware(logger); - dataToDegreesMiddleware = positionResAsDegreesMiddleware(logger); reqValidateMiddleware = validateRequestMiddleware(config, logger, commonErrors); addProdDictionaryMiddleware = addProductsDictionaryMiddleware(logger, productMetadataFields); }); @@ -54,9 +46,7 @@ describe('Get heights middlewares', function () { } as Response; mockRequest = { - body: { - radiansToOriginalPositionsMap: new Map(), - }, + body: {}, } as Request; mockNext = jest.fn(); @@ -71,74 +61,6 @@ describe('Get heights middlewares', function () { }); }); - describe('Data to radians middleware', function () { - const position = { longitude: 86.82918540404042, latitude: 27.888257 }; - const expectedResponseInRadians = { - longitude: 1.5154551721251082, - latitude: 0.4867419072923562, - height: 0, - } as Cartographic; - - beforeEach(function () { - mockResponse = { - locals: {}, - } as Response; - - mockRequest = { - body: { - positions: [position], - radiansToOriginalPositionsMap: new Map([ - [`${expectedResponseInRadians.longitude};${expectedResponseInRadians.latitude}`, `${position.longitude};${position.latitude}`], - ]), - }, - } as Request; - - mockNext = jest.fn(); - }); - - it('Should receive positions array via req.body.positions and attach back positions in radians', function () { - // @ts-ignore - dataToRadiansMiddleware(mockRequest, mockResponse, mockNext); - expect((mockRequest.body as GetHeightsPointsRequest).positions[0]).toEqual(expectedResponseInRadians); - }); - }); - - describe('Data to degrees middleware', function () { - const positionInRadians = { longitude: 1.5154551721251082, latitude: 0.4867419072923562 }; - const expectedResponseInDegrees = { - longitude: 86.82918540404042, - latitude: 27.888257, - } as Cartographic; - - beforeEach(function () { - mockResponse = { - locals: { - positions: [positionInRadians], - }, - } as unknown as Response; - - mockRequest = { - body: { - radiansToOriginalPositionsMap: new Map([ - [ - `${positionInRadians.longitude};${positionInRadians.latitude}`, - `${expectedResponseInDegrees.longitude};${expectedResponseInDegrees.latitude}`, - ], - ]), - }, - } as Request; - - mockNext = jest.fn(); - }); - - it('Should receive positions array via res.locals.positions and attach back positions in degrees', function () { - // @ts-ignore - dataToDegreesMiddleware(mockRequest, mockResponse, mockNext); - - expect((mockResponse.locals.positions as PosWithHeight[])[0]).toEqual(expectedResponseInDegrees); - }); - }); - describe('Add products dictionary', function () { beforeEach(function () { mockResponse = { From 2a1538b51f6e3aa9f33fd65087457c1db3bd6e33 Mon Sep 17 00:00:00 2001 From: shimoncohen Date: Thu, 3 Sep 2026 11:39:38 +0300 Subject: [PATCH 15/25] test: remove cesium terrain mock from heights integration Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01ATQerkHAV3Vur82HVVFack --- tests/integration/heights/heights.spec.ts | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/tests/integration/heights/heights.spec.ts b/tests/integration/heights/heights.spec.ts index aa7d415..5fa426a 100644 --- a/tests/integration/heights/heights.spec.ts +++ b/tests/integration/heights/heights.spec.ts @@ -1,6 +1,5 @@ /* eslint-disable @typescript-eslint/no-unsafe-return */ import config from 'config'; -import { Cartesian2, Cartographic, CesiumTerrainProvider } from 'cesium'; import { Application } from 'express'; import httpStatusCodes from 'http-status-codes'; import jsLogger from '@map-colonies/js-logger'; @@ -22,7 +21,6 @@ describe('heights', function () { const mockJsonDataOutOfBounds = positionsOutsideOfProviders as GetHeightsPointsRequest; let requestSender: HeightsRequestSender; - let cesiumTerrainProviderFromUrlSpy: jest.SpyInstance; let productMetadataFields: string[]; const basicPositionResponse: PosWithHeight = { @@ -35,25 +33,6 @@ describe('heights', function () { beforeAll(async function () { productMetadataFields = config.get('productMetadataFields').split(','); - cesiumTerrainProviderFromUrlSpy = jest.spyOn(CesiumTerrainProvider, 'fromUrl'); - - cesiumTerrainProviderFromUrlSpy.mockReturnValue({ - availability: { - available: true, - computeMaximumLevelAtPosition: () => { - return 13; - }, - }, - tilingScheme: { - positionToTileXY: (position: Cartographic) => { - // Making sure there are no overlapping tiles for any of the given positions, - // so that each position is a "request" - // (Assuming unique positions) - return new Cartesian2(position.latitude, position.longitude); - }, - }, - }); - const app = await getApp({ override: [{ token: SERVICES.LOGGER, provider: { useValue: jsLogger({ enabled: false }) } }], }); From deb2863cd8fde1696d1ab4d9a0d9602fb9cc5dc3 Mon Sep 17 00:00:00 2001 From: shimoncohen Date: Thu, 3 Sep 2026 11:42:32 +0300 Subject: [PATCH 16/25] test: cover checksum utilities Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01ATQerkHAV3Vur82HVVFack --- tests/unit/heights/utilities.spec.ts | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 tests/unit/heights/utilities.spec.ts diff --git a/tests/unit/heights/utilities.spec.ts b/tests/unit/heights/utilities.spec.ts new file mode 100644 index 0000000..020dcd1 --- /dev/null +++ b/tests/unit/heights/utilities.spec.ts @@ -0,0 +1,21 @@ +import { generateChecksum, isSame } from '../../../src/heights/utilities'; + +describe('utilities', () => { + describe('generateChecksum', () => { + it('is deterministic for the same input', () => { + expect(generateChecksum('abc')).toEqual(generateChecksum('abc')); + }); + it('differs for different input', () => { + expect(generateChecksum('abc')).not.toEqual(generateChecksum('abd')); + }); + }); + + describe('isSame', () => { + it('returns true for deeply equal objects', () => { + expect(isSame({ a: 1, b: [2, 3] }, { a: 1, b: [2, 3] })).toBe(true); + }); + it('returns false for different objects', () => { + expect(isSame({ a: 1 }, { a: 2 })).toBe(false); + }); + }); +}); From 5eec55e3d0696018dcc65474fb706841135035b1 Mon Sep 17 00:00:00 2001 From: shimoncohen Date: Thu, 3 Sep 2026 11:42:32 +0300 Subject: [PATCH 17/25] test: exclude worker-only utilities helper from integration coverage Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01ATQerkHAV3Vur82HVVFack --- tests/configurations/integration/jest.config.js | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/configurations/integration/jest.config.js b/tests/configurations/integration/jest.config.js index efa2634..7556c70 100644 --- a/tests/configurations/integration/jest.config.js +++ b/tests/configurations/integration/jest.config.js @@ -9,7 +9,15 @@ module.exports = { }, coverageReporters: ['text', 'html'], collectCoverage: true, - collectCoverageFrom: ['/src/**/*.ts', '!*/node_modules/', '!/vendor/**', '!*/common/**', '!**/models/**', '!/src/*'], + collectCoverageFrom: [ + '/src/**/*.ts', + '!*/node_modules/', + '!/vendor/**', + '!*/common/**', + '!**/models/**', + '!/src/*', + '!/src/heights/utilities.ts', // helper reached only via CSW worker (inert in integration); unit-covered instead + ], coverageDirectory: '/coverage', rootDir: '../../../.', testMatch: ['/tests/integration/**/*.spec.ts'], From bf5e2eb2672e27fcac80e4339623373a56973dbe Mon Sep 17 00:00:00 2001 From: shimoncohen Date: Thu, 3 Sep 2026 11:53:21 +0300 Subject: [PATCH 18/25] fix: isolate per-record geotiff provider failures and validate link url Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01ATQerkHAV3Vur82HVVFack --- src/heights/models/DEMTerrainCacheManager.ts | 25 ++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/src/heights/models/DEMTerrainCacheManager.ts b/src/heights/models/DEMTerrainCacheManager.ts index eb69f7e..881025e 100644 --- a/src/heights/models/DEMTerrainCacheManager.ts +++ b/src/heights/models/DEMTerrainCacheManager.ts @@ -1,5 +1,6 @@ import { inject, injectable } from 'tsyringe'; import { IConfig } from 'config'; +import { Logger } from '@map-colonies/js-logger'; import { PycswDemCatalogRecord } from '@map-colonies/mc-model-types'; import { HeightProviders } from '../interfaces'; import { SERVICES } from '../../common/constants'; @@ -12,7 +13,10 @@ const COGS_FOLDER = 'cogs/'; export default class DEMTerrainCacheManager { public heightProviders: HeightProviders = {}; - public constructor(@inject(SERVICES.CONFIG) private readonly config: IConfig) {} + public constructor( + @inject(SERVICES.CONFIG) private readonly config: IConfig, + @inject(SERVICES.LOGGER) private readonly logger: Logger + ) {} public async initProviders(demCatalogRecords: PycswDemCatalogRecord[]): Promise { const heightProviders: HeightProviders = {}; @@ -21,11 +25,21 @@ export default class DEMTerrainCacheManager { for (const record of geotiffRecords) { const link = record.links?.find((currentLink) => currentLink.protocol === GEOTIFF_PROTOCOL); + if (!link) { + continue; + } - if (link) { + try { const objectUrl = this.transformRouteToObjectUrl(link.url as string); const { url, headers } = this.buildAuthenticatedUrl(objectUrl); heightProviders[record.id as string] = await GeotiffHeightProvider.fromUrl(url, headers); + } catch (err) { + this.logger.error({ + msg: 'Failed to open geotiff provider; skipping record', + recordId: record.id, + err, + location: '[DEMTerrainCacheManager] [initProviders]', + }); } } @@ -34,8 +48,11 @@ export default class DEMTerrainCacheManager { private transformRouteToObjectUrl(linkUrl: string): string { const serviceURL = this.config.get('s3Gateway.url'); - - return `${serviceURL}/${COGS_FOLDER}${linkUrl.split(COGS_FOLDER)[1]}`; + const objectKey = linkUrl.split(COGS_FOLDER)[1]; + if (objectKey === undefined) { + throw new Error(`GEOTIFF link URL missing '${COGS_FOLDER}' segment: ${linkUrl}`); + } + return `${serviceURL}/${COGS_FOLDER}${objectKey}`; } private buildAuthenticatedUrl(objectUrl: string): { url: string; headers?: Record } { From 01330651fcddca07afe29e32732056503c2152d7 Mon Sep 17 00:00:00 2001 From: shimoncohen Date: Thu, 3 Sep 2026 11:53:21 +0300 Subject: [PATCH 19/25] fix: prevent running-requests leak on error and treat NaN pixels as nodata Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01ATQerkHAV3Vur82HVVFack --- src/heights/models/geotiffHeightProvider.ts | 2 +- src/heights/models/heightsManager.ts | 26 ++++++++++----------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/heights/models/geotiffHeightProvider.ts b/src/heights/models/geotiffHeightProvider.ts index 6b72f05..a1ffad5 100644 --- a/src/heights/models/geotiffHeightProvider.ts +++ b/src/heights/models/geotiffHeightProvider.ts @@ -48,7 +48,7 @@ export default class GeotiffHeightProvider { const band = raster[0]; const [tl, tr, bl, br] = [band[0], band[1], band[2], band[3]]; - if ([tl, tr, bl, br].some((value) => value === this.noData)) { + if ([tl, tr, bl, br].some((value) => value === this.noData || Number.isNaN(value))) { return null; } diff --git a/src/heights/models/heightsManager.ts b/src/heights/models/heightsManager.ts index 80c14de..cbf72ca 100644 --- a/src/heights/models/heightsManager.ts +++ b/src/heights/models/heightsManager.ts @@ -66,26 +66,26 @@ export class HeightsManager { public async getPoints(points: GeoPoint[], requestedProductType: TerrainTypes, reqCtx?: Record): Promise { this.logger.info({ pointsNumber: points.length, location: '[HeightsManager] [getPoints]', ...reqCtx }); - this.runningRequests++; // eslint-disable-next-line @typescript-eslint/naming-convention this.elevationsRequestsCounter?.inc({ points_number: points.length }); if (points.length === 0) { - this.runningRequests--; return []; } - const result = await this.samplePositionsHeights(points, requestedProductType, reqCtx); - - this.logger.info({ - totalRequests: result.totalRequests, - pointsNumber: points.length, - location: '[HeightsManager] [getPoints]', - ...reqCtx, - }); - - this.runningRequests--; - return result.positions; + this.runningRequests++; + try { + const result = await this.samplePositionsHeights(points, requestedProductType, reqCtx); + this.logger.info({ + totalRequests: result.totalRequests, + pointsNumber: points.length, + location: '[HeightsManager] [getPoints]', + ...reqCtx, + }); + return result.positions; + } finally { + this.runningRequests--; + } } private async samplePositionsHeights( From f1e959acfa667bcc62e447555164e4e2d4f5f40b Mon Sep 17 00:00:00 2001 From: shimoncohen Date: Thu, 3 Sep 2026 11:53:21 +0300 Subject: [PATCH 20/25] test: cover geotiff sampling happy path end-to-end Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01ATQerkHAV3Vur82HVVFack --- tests/integration/heights/heights.spec.ts | 63 +++++++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/tests/integration/heights/heights.spec.ts b/tests/integration/heights/heights.spec.ts index 5fa426a..aab21bf 100644 --- a/tests/integration/heights/heights.spec.ts +++ b/tests/integration/heights/heights.spec.ts @@ -3,10 +3,16 @@ import config from 'config'; import { Application } from 'express'; import httpStatusCodes from 'http-status-codes'; import jsLogger from '@map-colonies/js-logger'; +import { container } from 'tsyringe'; +import { PycswDemCatalogRecord } from '@map-colonies/mc-model-types'; import { getApp } from '../../../src/app'; import { SERVICES } from '../../../src/common/constants'; +import { CATALOG_RECORDS_MAP, DEM_TERRAIN_CACHE_MANAGER } from '../../../src/containerConfig'; import { GetHeightsPointsRequest, GetHeightsPointsResponse } from '../../../src/heights/controllers/heightsController'; import { PosWithHeight, TerrainTypes } from '../../../src/heights/interfaces'; +import { CatalogRecords } from '../../../src/heights/models/catalogRecords'; +import DEMTerrainCacheManager from '../../../src/heights/models/DEMTerrainCacheManager'; +import GeotiffHeightProvider from '../../../src/heights/models/geotiffHeightProvider'; import mockJsonPoints, { emptyPositionsRequest, moreThen150RequestsPositions, @@ -142,4 +148,61 @@ describe('heights', function () { }); }); }); + + describe('Given seeded geotiff providers', function () { + beforeAll(async function () { + jest.spyOn(GeotiffHeightProvider, 'fromUrl').mockResolvedValue({ + sample: async (points: { longitude: number; latitude: number }[]) => points.map(() => 123), + } as unknown as GeotiffHeightProvider); + + const records = [ + { + id: 'rec1', + productId: 'test_prod', + productType: 'DTM', + resolutionMeter: 30, + updateDate: '2023-05-08T17:44:01.000Z', + absoluteAccuracyLEP90: 9e-7, + productStatus: 'PUBLISHED', + footprint: { + type: 'Polygon', + coordinates: [ + [ + [34, 32], + [34, 33], + [36, 33], + [36, 32], + [34, 32], + ], + ], + }, + links: [{ protocol: 'GEOTIFF', url: 'https://tiles-dev.mapcolonies.net/api/dem/v1/cogs/a.tif' }], + }, + ]; + + container + .resolve(CATALOG_RECORDS_MAP) + .setValue(Object.fromEntries(records.map((r) => [r.id, r])) as unknown as Record); + await container.resolve(DEM_TERRAIN_CACHE_MANAGER).initProviders(records as unknown as PycswDemCatalogRecord[]); + }); + + afterAll(function () { + container.resolve(CATALOG_RECORDS_MAP).setValue({}); + jest.restoreAllMocks(); + }); + + it('returns real heights, productId, and products metadata for a point inside a footprint', async function () { + const response = await requestSender.getPoints({ positions: [{ longitude: 35.0, latitude: 32.5 }] } as GetHeightsPointsRequest); + + expect(response.status).toBe(httpStatusCodes.OK); + const body = response.body as GetHeightsPointsResponse; + expect(body.data).toHaveLength(1); + expect(body.data[0].height).toBe(123); + expect(body.data[0].productId).toBe('test_prod'); + expect(body.products['test_prod']).toBeDefined(); + ['productType', 'updateDate', 'resolutionMeter', 'absoluteAccuracyLEP90'].forEach((field) => { + expect((body.products['test_prod'] as Record)[field]).toBeDefined(); + }); + }); + }); }); From a9b4a9e2044aec8edc64e78ed4981294d1c19711 Mon Sep 17 00:00:00 2001 From: shimoncohen Date: Thu, 10 Sep 2026 11:56:12 +0300 Subject: [PATCH 21/25] fix: preserve input point order and address PR review Response to PR #52 review (syncush): - Preserve input order in samplePositionsHeights: points were bucketed by provider then flattened, so responses reordered points that spanned multiple providers or fell outside coverage. Now scatter results back to each point's original index. - Bound GeotiffHeightProvider.sample concurrency (config samplingConcurrency, default 16) so large batches don't open N simultaneous gateway connections. - Build query-param authenticated URL with WHATWG URL API. - Drop unused CommonErrors injection from HeightsManager. - Add tests: input-order regression across providers/gaps, and DEMTerrainCacheManager provider-init failure isolation. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01ATQerkHAV3Vur82HVVFack --- config/custom-environment-variables.json | 3 +- config/default.json | 3 +- src/heights/models/DEMTerrainCacheManager.ts | 13 ++++--- src/heights/models/geotiffHeightProvider.ts | 35 ++++++++++++++++--- src/heights/models/heightsManager.ts | 34 +++++++++--------- .../models/DEMTerrainCacheManager.spec.ts | 31 ++++++++++++++++ tests/unit/heights/models/heightModel.spec.ts | 19 +++++++++- 7 files changed, 107 insertions(+), 31 deletions(-) create mode 100644 tests/unit/heights/models/DEMTerrainCacheManager.spec.ts diff --git a/config/custom-environment-variables.json b/config/custom-environment-variables.json index e04eb9b..b2abd14 100644 --- a/config/custom-environment-variables.json +++ b/config/custom-environment-variables.json @@ -62,5 +62,6 @@ }, "synchRecordsInterval": "SYNCH_RECORDS_INTERVAL", "productMetadataFields": "productType,updateDate,resolutionMeter,absoluteAccuracyLEP90", - "maximumTilesPerRequest": "MAXIMUM_TILES_PER_REQUEST" + "maximumTilesPerRequest": "MAXIMUM_TILES_PER_REQUEST", + "samplingConcurrency": "SAMPLING_CONCURRENCY" } diff --git a/config/default.json b/config/default.json index 508fecb..29203df 100644 --- a/config/default.json +++ b/config/default.json @@ -50,5 +50,6 @@ }, "synchRecordsInterval": 600000, "productMetadataFields": "productType,updateDate,resolutionMeter,absoluteAccuracyLEP90", - "maximumTilesPerRequest": 150 + "maximumTilesPerRequest": 150, + "samplingConcurrency": 16 } diff --git a/src/heights/models/DEMTerrainCacheManager.ts b/src/heights/models/DEMTerrainCacheManager.ts index 881025e..c39f3aa 100644 --- a/src/heights/models/DEMTerrainCacheManager.ts +++ b/src/heights/models/DEMTerrainCacheManager.ts @@ -13,15 +13,13 @@ const COGS_FOLDER = 'cogs/'; export default class DEMTerrainCacheManager { public heightProviders: HeightProviders = {}; - public constructor( - @inject(SERVICES.CONFIG) private readonly config: IConfig, - @inject(SERVICES.LOGGER) private readonly logger: Logger - ) {} + public constructor(@inject(SERVICES.CONFIG) private readonly config: IConfig, @inject(SERVICES.LOGGER) private readonly logger: Logger) {} public async initProviders(demCatalogRecords: PycswDemCatalogRecord[]): Promise { const heightProviders: HeightProviders = {}; const geotiffRecords = demCatalogRecords.filter((record) => record.links?.some((link) => link.protocol === GEOTIFF_PROTOCOL)); + const samplingConcurrency = Number(this.config.get('samplingConcurrency')); for (const record of geotiffRecords) { const link = record.links?.find((currentLink) => currentLink.protocol === GEOTIFF_PROTOCOL); @@ -32,7 +30,7 @@ export default class DEMTerrainCacheManager { try { const objectUrl = this.transformRouteToObjectUrl(link.url as string); const { url, headers } = this.buildAuthenticatedUrl(objectUrl); - heightProviders[record.id as string] = await GeotiffHeightProvider.fromUrl(url, headers); + heightProviders[record.id as string] = await GeotiffHeightProvider.fromUrl(url, headers, samplingConcurrency); } catch (err) { this.logger.error({ msg: 'Failed to open geotiff provider; skipping record', @@ -65,8 +63,9 @@ export default class DEMTerrainCacheManager { } if (injectionType.toLowerCase() === 'queryparam') { - const separator = objectUrl.includes('?') ? '&' : '?'; - return { url: `${objectUrl}${separator}${attributeName}=${encodeURIComponent(tokenValue)}` }; + const url = new URL(objectUrl); + url.searchParams.set(attributeName, tokenValue); + return { url: url.toString() }; } return { url: objectUrl }; diff --git a/src/heights/models/geotiffHeightProvider.ts b/src/heights/models/geotiffHeightProvider.ts index a1ffad5..6713ee0 100644 --- a/src/heights/models/geotiffHeightProvider.ts +++ b/src/heights/models/geotiffHeightProvider.ts @@ -1,8 +1,10 @@ import { fromUrl } from 'geotiff'; import type { GeoTIFFImage } from 'geotiff'; +import PromisePool from '@supercharge/promise-pool/dist'; import { GeoPoint } from '../interfaces'; const DEFAULT_NODATA = -32768; +const DEFAULT_SAMPLING_CONCURRENCY = 16; export default class GeotiffHeightProvider { private constructor( @@ -13,21 +15,46 @@ export default class GeotiffHeightProvider { private readonly pixelHeight: number, // degrees/pixel, negative private readonly rasterWidth: number, private readonly rasterHeight: number, - private readonly noData: number + private readonly noData: number, + private readonly samplingConcurrency: number ) {} - public static async fromUrl(url: string, headers?: Record): Promise { + public static async fromUrl( + url: string, + headers?: Record, + samplingConcurrency: number = DEFAULT_SAMPLING_CONCURRENCY + ): Promise { const tiff = await fromUrl(url, headers ? { headers } : {}); const image = await tiff.getImage(0); const [originX, originY] = image.getOrigin(); const [pixelWidth, pixelHeight] = image.getResolution(); const noData = image.getGDALNoData() ?? DEFAULT_NODATA; - return new GeotiffHeightProvider(image, originX, originY, pixelWidth, pixelHeight, image.getWidth(), image.getHeight(), noData); + return new GeotiffHeightProvider( + image, + originX, + originY, + pixelWidth, + pixelHeight, + image.getWidth(), + image.getHeight(), + noData, + samplingConcurrency + ); } public async sample(points: GeoPoint[]): Promise<(number | null)[]> { - return Promise.all(points.map(async (point) => this.sampleOne(point))); + // Bound concurrent range reads so a large batch doesn't open N simultaneous connections to the gateway. + const heights = new Array(points.length); + await PromisePool.for(points.map((point, index) => ({ point, index }))) + .withConcurrency(this.samplingConcurrency) + .handleError((error) => { + throw error; + }) + .process(async ({ point, index }) => { + heights[index] = await this.sampleOne(point); + }); + return heights; } private async sampleOne(point: GeoPoint): Promise { diff --git a/src/heights/models/heightsManager.ts b/src/heights/models/heightsManager.ts index cbf72ca..0c00359 100644 --- a/src/heights/models/heightsManager.ts +++ b/src/heights/models/heightsManager.ts @@ -5,7 +5,6 @@ import { Logger } from '@map-colonies/js-logger'; import PromisePool from '@supercharge/promise-pool/dist'; import booleanPointInPolygon from '@turf/boolean-point-in-polygon'; import { Feature } from '@turf/turf'; -import { CommonErrors } from '../../common/commonErrors'; import { SERVICES } from '../../common/constants'; import { IConfig } from '../../common/interfaces'; import { CATALOG_RECORDS_MAP, DEM_TERRAIN_CACHE_MANAGER } from '../../containerConfig'; @@ -38,7 +37,6 @@ export class HeightsManager { public constructor( @inject(SERVICES.LOGGER) private readonly logger: Logger, - @inject(CommonErrors) private readonly commonErrors: CommonErrors, @inject(SERVICES.CONFIG) private readonly config: IConfig, @inject(SERVICES.METRICS_REGISTRY) registry?: client.Registry ) { @@ -102,29 +100,33 @@ export class HeightsManager { ...reqCtx, }); - // Group points by the provider chosen for them (null = no provider). - const groups = new Map(); - for (const position of positionsWithProviders) { + // Group points by the provider chosen for them (null = no provider), keeping each point's + // original index so results can be scattered back in input order. + const groups = new Map(); + positionsWithProviders.forEach((position, index) => { const key = position.providerKey ?? null; const bucket = groups.get(key) ?? []; - bucket.push({ longitude: position.longitude, latitude: position.latitude }); + bucket.push({ point: { longitude: position.longitude, latitude: position.latitude }, index }); groups.set(key, bucket); - } + }); const groupEntries = [...groups.entries()]; - const finalPositionsWithHeights: PosWithHeight[] = []; + const finalPositionsWithHeights = new Array(positionsWithProviders.length); - const { results } = await PromisePool.for(groupEntries) + await PromisePool.for(groupEntries) .withConcurrency(Math.max(1, groupEntries.length)) - .process(async ([providerKey, points]) => { + .process(async ([providerKey, entries]) => { if (providerKey === null) { - return points.map((point) => ({ ...point, height: null } as PosWithHeight)); + entries.forEach(({ point, index }) => { + finalPositionsWithHeights[index] = { ...point, height: null } as PosWithHeight; + }); + return; } const samplingStart = performance.now(); const provider = this.heightProviders[providerKey]; const record = this.catalogRecordsMap[providerKey]; - const heights = await provider.sample(points); + const heights = await provider.sample(entries.map(({ point }) => point)); this.logger.info({ terrainSamplingTime: performance.now() - samplingStart, @@ -134,9 +136,9 @@ export class HeightsManager { ...reqCtx, }); - return points.map((point, index) => { - const height = heights[index]; - return { + entries.forEach(({ point, index }, i) => { + const height = heights[i]; + finalPositionsWithHeights[index] = { ...point, height, ...(height !== null ? { productId: record.productId as string } : {}), @@ -144,8 +146,6 @@ export class HeightsManager { }); }); - finalPositionsWithHeights.push(...(results as PosWithHeight[][]).flat()); - return { positions: finalPositionsWithHeights, totalRequests: groupEntries.length }; } diff --git a/tests/unit/heights/models/DEMTerrainCacheManager.spec.ts b/tests/unit/heights/models/DEMTerrainCacheManager.spec.ts new file mode 100644 index 0000000..4a2aa6a --- /dev/null +++ b/tests/unit/heights/models/DEMTerrainCacheManager.spec.ts @@ -0,0 +1,31 @@ +import config from 'config'; +import jsLogger from '@map-colonies/js-logger'; +import { PycswDemCatalogRecord } from '@map-colonies/mc-model-types'; +import DEMTerrainCacheManager from '../../../../src/heights/models/DEMTerrainCacheManager'; +import GeotiffHeightProvider from '../../../../src/heights/models/geotiffHeightProvider'; + +describe('DEMTerrainCacheManager', () => { + afterEach(() => jest.restoreAllMocks()); + + it('skips a record whose provider fails to open and still registers the rest', async () => { + const records = [ + { id: 'bad', links: [{ protocol: 'GEOTIFF', url: 'https://gw/cogs/bad.tif' }] }, + { id: 'good', links: [{ protocol: 'GEOTIFF', url: 'https://gw/cogs/good.tif' }] }, + ] as unknown as PycswDemCatalogRecord[]; + + const fromUrlSpy = jest + .spyOn(GeotiffHeightProvider, 'fromUrl') + .mockRejectedValueOnce(new Error('open failed')) + .mockResolvedValueOnce({} as GeotiffHeightProvider); + + const logger = jsLogger({ enabled: false }); + const errorSpy = jest.spyOn(logger, 'error'); + + const manager = new DEMTerrainCacheManager(config, logger); + await manager.initProviders(records); + + expect(fromUrlSpy).toHaveBeenCalledTimes(2); + expect(Object.keys(manager.heightProviders)).toEqual(['good']); + expect(errorSpy).toHaveBeenCalled(); + }); +}); diff --git a/tests/unit/heights/models/heightModel.spec.ts b/tests/unit/heights/models/heightModel.spec.ts index 397b246..b9b28f3 100644 --- a/tests/unit/heights/models/heightModel.spec.ts +++ b/tests/unit/heights/models/heightModel.spec.ts @@ -43,6 +43,21 @@ describe('Get Heights model', function () { } }); + it('Should preserve input order when points span providers and out-of-coverage gaps', async function () { + const positions = [ + { longitude: 35.2, latitude: 32.0 }, // inside footprint + { longitude: 0, latitude: 0 }, // outside every footprint + { longitude: 35.5, latitude: 32.5 }, // inside footprint + ]; + + const result = await heightsManager.getPoints(positions, TerrainTypes.MIXED); + + expect(result).toHaveLength(3); + expect(result[0]).toMatchObject({ longitude: 35.2, latitude: 32.0, height: 100 }); + expect(result[1]).toMatchObject({ longitude: 0, latitude: 0, height: null }); + expect(result[2]).toMatchObject({ longitude: 35.5, latitude: 32.5, height: 100 }); + }); + it('Should return height only for the positions inside a provider footprint', async function () { const result = await heightsManager.getPoints(mockJsonDataOutOfBounds.positions, TerrainTypes.MIXED); @@ -59,7 +74,9 @@ describe('Get Heights model', function () { describe('Given invalid params', function () { it('Should return empty array for empty positions', async function () { - await expect(heightsManager.getPoints((emptyPositionsRequest as unknown as GetHeightsPointsRequest).positions, TerrainTypes.MIXED)).resolves.toEqual([]); + await expect( + heightsManager.getPoints((emptyPositionsRequest as unknown as GetHeightsPointsRequest).positions, TerrainTypes.MIXED) + ).resolves.toEqual([]); }); }); }); From 8f3c57568f3ed9f7033b6ab501d7992faa13cb55 Mon Sep 17 00:00:00 2001 From: shimoncohen Date: Thu, 24 Sep 2026 11:56:22 +0300 Subject: [PATCH 22/25] fix: address review on heights sampling and cog url parsing - samplePositionsHeights: add PromisePool .handleError that rethrows, so a provider.sample() failure surfaces instead of leaving undefined height slots. - Snapshot heightProviders + catalogRecordsMap once per request and thread them through attachProviderToPositions, so a background rebuild mid-request can't make a chosen providerKey resolve to an undefined provider/record. - attachProviderToPositions: hoist providerEntries + productTypeFiltered out of the per-point map (they don't vary by point). - Only attach productId when defined (drops the undefined-as-string cast). - transformRouteToObjectUrl: slice from the first 'cogs/' to the end instead of split('cogs/')[1], which dropped trailing segments on repeated 'cogs/'. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01ATQerkHAV3Vur82HVVFack --- src/heights/models/DEMTerrainCacheManager.ts | 6 ++- src/heights/models/heightsManager.ts | 50 +++++++++++++------- 2 files changed, 38 insertions(+), 18 deletions(-) diff --git a/src/heights/models/DEMTerrainCacheManager.ts b/src/heights/models/DEMTerrainCacheManager.ts index c39f3aa..dfb204f 100644 --- a/src/heights/models/DEMTerrainCacheManager.ts +++ b/src/heights/models/DEMTerrainCacheManager.ts @@ -46,10 +46,12 @@ export default class DEMTerrainCacheManager { private transformRouteToObjectUrl(linkUrl: string): string { const serviceURL = this.config.get('s3Gateway.url'); - const objectKey = linkUrl.split(COGS_FOLDER)[1]; - if (objectKey === undefined) { + // Take everything from the first 'cogs/' to the end — split()[1] would drop trailing + // segments if 'cogs/' appears more than once in the URL. + if (!linkUrl.includes(COGS_FOLDER)) { throw new Error(`GEOTIFF link URL missing '${COGS_FOLDER}' segment: ${linkUrl}`); } + const objectKey = linkUrl.slice(linkUrl.indexOf(COGS_FOLDER) + COGS_FOLDER.length); return `${serviceURL}/${COGS_FOLDER}${objectKey}`; } diff --git a/src/heights/models/heightsManager.ts b/src/heights/models/heightsManager.ts index 0c00359..abe2075 100644 --- a/src/heights/models/heightsManager.ts +++ b/src/heights/models/heightsManager.ts @@ -2,13 +2,14 @@ import { Polygon } from 'geojson'; import client from 'prom-client'; import { container, inject, injectable } from 'tsyringe'; import { Logger } from '@map-colonies/js-logger'; +import { PycswDemCatalogRecord } from '@map-colonies/mc-model-types'; import PromisePool from '@supercharge/promise-pool/dist'; import booleanPointInPolygon from '@turf/boolean-point-in-polygon'; import { Feature } from '@turf/turf'; import { SERVICES } from '../../common/constants'; import { IConfig } from '../../common/interfaces'; import { CATALOG_RECORDS_MAP, DEM_TERRAIN_CACHE_MANAGER } from '../../containerConfig'; -import { GeoPoint, PosWithHeight, PosWithProvider, TerrainTypes } from '../interfaces'; +import { GeoPoint, HeightProviders, PosWithHeight, PosWithProvider, TerrainTypes } from '../interfaces'; import DEMTerrainCacheManager from './DEMTerrainCacheManager'; import { CatalogRecords } from './catalogRecords'; @@ -91,8 +92,15 @@ export class HeightsManager { requestedProductType: TerrainTypes, reqCtx?: Record ): Promise<{ positions: PosWithHeight[]; totalRequests: number }> { + // Snapshot the provider map and catalog once per request. Both are rebuilt wholesale on a + // background timer; resolving them fresh per access could pair a providerKey chosen against + // one snapshot with a provider/catalog map from a later rebuild (provider becomes undefined, + // sample() throws). + const heightProviders = this.heightProviders; + const catalogRecordsMap = this.catalogRecordsMap; + const attachProviderStart = performance.now(); - const positionsWithProviders = this.attachProviderToPositions(positionsArr, requestedProductType); + const positionsWithProviders = this.attachProviderToPositions(positionsArr, requestedProductType, heightProviders, catalogRecordsMap); this.logger.info({ attachProviderTime: performance.now() - attachProviderStart, pointsNumber: positionsArr.length, @@ -115,6 +123,11 @@ export class HeightsManager { await PromisePool.for(groupEntries) .withConcurrency(Math.max(1, groupEntries.length)) + .handleError((error) => { + // Without this, promise-pool silently collects sample() failures and resolves normally, + // leaving height slots undefined. Rethrow so the request fails loudly instead. + throw error; + }) .process(async ([providerKey, entries]) => { if (providerKey === null) { entries.forEach(({ point, index }) => { @@ -124,8 +137,8 @@ export class HeightsManager { } const samplingStart = performance.now(); - const provider = this.heightProviders[providerKey]; - const record = this.catalogRecordsMap[providerKey]; + const provider = heightProviders[providerKey]; + const record = catalogRecordsMap[providerKey]; const heights = await provider.sample(entries.map(({ point }) => point)); this.logger.info({ @@ -141,7 +154,7 @@ export class HeightsManager { finalPositionsWithHeights[index] = { ...point, height, - ...(height !== null ? { productId: record.productId as string } : {}), + ...(height !== null && record.productId !== undefined ? { productId: record.productId } : {}), } as PosWithHeight; }); }); @@ -149,30 +162,35 @@ export class HeightsManager { return { positions: finalPositionsWithHeights, totalRequests: groupEntries.length }; } - private attachProviderToPositions(positions: GeoPoint[], requestedProductType: TerrainTypes): PosWithProvider[] { + private attachProviderToPositions( + positions: GeoPoint[], + requestedProductType: TerrainTypes, + heightProviders: HeightProviders, + catalogRecordsMap: Record + ): PosWithProvider[] { /* * Filter providers by requested product type (unless MIXED) * Filter providers by footprint point intersection * Sort by highest resolution (lower resolutionMeter is better), tie-break on newest updateDate * Attach the best provider key to the point */ - return positions.map((position) => { - const providerEntries = Object.entries(this.heightProviders); - - const productTypeFiltered = - requestedProductType !== TerrainTypes.MIXED - ? providerEntries.filter(([key]) => this.catalogRecordsMap[key].productType?.includes(requestedProductType)) - : providerEntries; + // Provider list and product-type filter are the same for every point — compute once. + const providerEntries = Object.entries(heightProviders); + const productTypeFiltered = + requestedProductType !== TerrainTypes.MIXED + ? providerEntries.filter(([key]) => catalogRecordsMap[key].productType?.includes(requestedProductType)) + : providerEntries; + return positions.map((position) => { const footprintFiltered = productTypeFiltered.filter(([key]) => - booleanPointInPolygon([position.longitude, position.latitude], this.catalogRecordsMap[key].footprint as Feature) + booleanPointInPolygon([position.longitude, position.latitude], catalogRecordsMap[key].footprint as Feature) ); const sorted = footprintFiltered.sort(([aKey], [bKey]) => { const A_BEFORE_B = -1; const B_BEFORE_A = 1; - const recordA = this.catalogRecordsMap[aKey]; - const recordB = this.catalogRecordsMap[bKey]; + const recordA = catalogRecordsMap[aKey]; + const recordB = catalogRecordsMap[bKey]; switch (true) { case (recordA.resolutionMeter as number) < (recordB.resolutionMeter as number): From fd4b5efe107b359f6bc3f5db8e0f8510feaaffe8 Mon Sep 17 00:00:00 2001 From: shimoncohen Date: Mon, 28 Sep 2026 12:03:28 +0300 Subject: [PATCH 23/25] test: assert heights reject on provider sample failure Co-Authored-By: Claude Opus 5.5 (1M context) --- tests/unit/heights/models/heightModel.spec.ts | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/tests/unit/heights/models/heightModel.spec.ts b/tests/unit/heights/models/heightModel.spec.ts index b9b28f3..56d4bc7 100644 --- a/tests/unit/heights/models/heightModel.spec.ts +++ b/tests/unit/heights/models/heightModel.spec.ts @@ -2,7 +2,9 @@ import { container } from 'tsyringe'; import { HeightsManager } from '../../../../src/heights/models/heightsManager'; import mockJsonPoints, { positionsOutsideOfProviders, emptyPositionsRequest } from '../../../../src/heights/MOCKS/mockData'; import { GetHeightsPointsRequest } from '../../../../src/heights/controllers/heightsController'; -import { PosWithHeight, TerrainTypes } from '../../../../src/heights/interfaces'; +import { TerrainTypes } from '../../../../src/heights/interfaces'; +import { DEM_TERRAIN_CACHE_MANAGER } from '../../../../src/containerConfig'; +import DEMTerrainCacheManager from '../../../../src/heights/models/DEMTerrainCacheManager'; import { registerTestValues } from '../../../configurations/testContainerConfig'; describe('Get Heights model', function () { @@ -66,12 +68,23 @@ describe('Get Heights model', function () { expect(position.longitude).toBeDefined(); expect(position.latitude).toBeDefined(); - const isNullHeight = (position.height as number | null) === null; + const isNullHeight = position.height === null; expect(typeof position.productId === 'undefined').toEqual(isNullHeight); } }); }); + describe('Given a provider failure', function () { + it('Should reject instead of silently returning positions with undefined heights when a provider sample fails', async function () { + const cacheManager = container.resolve(DEM_TERRAIN_CACHE_MANAGER); + for (const provider of Object.values(cacheManager.heightProviders)) { + jest.spyOn(provider, 'sample').mockRejectedValue(new Error('range read failed')); + } + + await expect(heightsManager.getPoints(mockJsonData.positions, TerrainTypes.MIXED)).rejects.toThrow('range read failed'); + }); + }); + describe('Given invalid params', function () { it('Should return empty array for empty positions', async function () { await expect( From af49979bf40bb080dbfd259b94ed8837d2bfbc89 Mon Sep 17 00:00:00 2001 From: shimoncohen Date: Mon, 28 Sep 2026 12:09:14 +0300 Subject: [PATCH 24/25] docs: untrack superpowers implementation plan Co-Authored-By: Claude Opus 5.5 (1M context) --- .../2026-09-03-geotiff-heights-migration.md | 1056 ----------------- 1 file changed, 1056 deletions(-) delete mode 100644 docs/superpowers/plans/2026-09-03-geotiff-heights-migration.md diff --git a/docs/superpowers/plans/2026-09-03-geotiff-heights-migration.md b/docs/superpowers/plans/2026-09-03-geotiff-heights-migration.md deleted file mode 100644 index 6596367..0000000 --- a/docs/superpowers/plans/2026-09-03-geotiff-heights-migration.md +++ /dev/null @@ -1,1056 +0,0 @@ -# GeoTIFF Heights Migration Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Replace the Cesium quantized-mesh terrain engine in `dem-heights` with direct COG (Cloud-Optimized GeoTIFF) sampling read over HTTP range requests from the S3 gateway, keeping the `/points` API contract identical. - -**Architecture:** Positions flow as WGS84 **degrees** end-to-end (no more radians round-trip). Discovery stays on CSW but filters for a `GEOTIFF` link instead of `TERRAIN_QMESH`. `DEMTerrainCacheManager` opens one `GeotiffHeightProvider` per catalog record via `geotiff.js` `fromUrl` (range reads through the gateway, token auth). `HeightsManager` selects a provider per point (product-type + footprint + resolution), groups points by provider, and samples heights via bilinear interpolation, mapping the tiff nodata value (`-32768`) and out-of-footprint points to `null`. Cesium is removed entirely. - -**Tech Stack:** TypeScript (CommonJS build, `target es2021`), Node 20, Express, tsyringe DI, `geotiff@^3.0.5` (ships a CJS entry — plain `import` works), `@turf/boolean-point-in-polygon`, Jest 28, `config`. - ---- - -## Background facts (verified before this plan) - -- Cesium coupling lives in: `DEMTerrainCacheManager.ts`, `heightsManager.ts`, `utilities.ts`, `interfaces.ts`, plus the two radian middlewares (`dataToRadians.ts`, `dataToDegrees.ts`) and their consumers (`controller`, `validateRequest`, `router`, tests). -- The radian middlewares exist ONLY because Cesium works in radians. GeoTIFF geotransform math is in degrees, so both middlewares and `radiansToOriginalPositionsMap` are deleted. -- Live env (verified on OCP `dem-dev`): `nginx-s3-gateway` proxies S3 **Range** (`206`/`Content-Range`), OPA auth passes with token as **queryParam** `?token=`. COGs already uploaded at MinIO bucket `dem-dev` under prefix `cogs/`. geotiff.js `fromUrl` through the internal gateway opened + sampled a real COG (`206.99 m`). -- Use the `tiled_ovr` COG (native 30 m, tiled 256, 13 overviews) — NOT the coarse `_COG` variant. -- Tiff nodata = `-32768` on all GEO tiffs. -- Catalog `productType` values look like `QUANTIZED_MESH_DTM_BEST`; selection uses `.includes('DTM'|'DSM')`, so it keeps working regardless of exact string. - -## File Structure - -**Create** -- `src/heights/models/geotiffHeightProvider.ts` — opens a COG and samples heights (bilinear + nodata). One responsibility: raster → height. -- `tests/unit/heights/models/geotiffHeightProvider.spec.ts` — unit tests for sampling math via a mocked `geotiff` module. - -**Modify** -- `src/heights/interfaces.ts` — drop Cesium types; add `GeoPoint`, `PosWithProvider`, `HeightProviders`; keep `TerrainTypes`, `PosWithHeight`. -- `src/heights/models/DEMTerrainCacheManager.ts` — build `GeotiffHeightProvider`s from `GEOTIFF` links (filename + DI symbol unchanged to limit churn). -- `src/heights/models/heightsManager.ts` — provider selection in degrees, group-by-provider, bilinear sampling; remove Cesium + tile clustering + density throw. -- `src/heights/utilities.ts` — delete tile-clustering; keep `generateChecksum` + `isSame`. -- `src/workerCatalogRecords.ts` — CSW filter `TERRAIN_QMESH` → `GEOTIFF`. -- `src/containerConfig.ts` — call `initProviders` (renamed) instead of `initTerrainProviders`. -- `src/heights/controllers/heightsController.ts` — request positions typed `GeoPoint[]`; drop `radiansToOriginalPositionsMap`. -- `src/heights/middlewares/validateRequest.ts` — type `GeoPoint[]`; drop Cesium import. -- `src/heights/routes/heightsRouter.ts` — remove the two radian middlewares from the chain. -- `package.json` — remove `cesium`, add `geotiff`. -- `tests/configurations/jest.setup.ts` — remove the global `jest.mock('cesium', ...)` (mocked `sampleTerrainMostDetailed`, now obsolete); keep only `import 'reflect-metadata';`. Runs before EVERY unit + integration spec, so it must be de-cesium'd or all specs fail to load. (Handled during Task 3, since it blocks Task 3's own spec.) -- `tests/configurations/testContainerConfig.ts` — records use `GEOTIFF` links; mock `GeotiffHeightProvider.fromUrl`. -- `tests/unit/heights/models/heightModel.spec.ts` — degrees input; mock geotiff provider. -- `tests/unit/heights/middlewares/heightsMiddlewares.spec.ts` — remove radian/degree middleware tests. -- `tests/integration/heights/heights.spec.ts` — remove Cesium spy. - -**Delete** -- `src/heights/middlewares/dataToRadians.ts` -- `src/heights/middlewares/dataToDegrees.ts` - -## Conventions for every task - -- Run a single unit spec: `npx jest --config=./tests/configurations/unit/jest.config.js ` -- Run all unit: `npm run test:unit` -- Run integration: `npm run test:integration` -- Type-check/build: `npm run build` -- Lint fix: `npm run lint:fix` -- Node must be 20: `nvm use 20` first (config@3.3.7 crashes on Node 24). -- Commit after each task. Conventional Commits: this migration is `feat:`/`refactor:`; dependency swap is `build:` (per repo convention, NOT `chore:`). End commit messages with the two trailer lines required by the repo (Co-Authored-By + Claude-Session). - ---- - -### Task 1: Add geotiff dependency, remove cesium - -**Files:** -- Modify: `package.json` - -- [ ] **Step 1: Swap the dependency** - -Run: -```bash -nvm use 20 -npm uninstall cesium -npm install geotiff@^3.0.5 -``` - -- [ ] **Step 2: Verify geotiff resolves as CJS** - -Run: `node -e "console.log(typeof require('geotiff').fromUrl)"` -Expected: `function` - -- [ ] **Step 3: Verify cesium is gone from the tree** - -Run: `node -e "try{require('cesium');console.log('STILL PRESENT')}catch(e){console.log('removed')}"` -Expected: `removed` - -- [ ] **Step 4: Commit** - -```bash -git add package.json package-lock.json -git commit -m "build: replace cesium terrain engine with geotiff" -``` - ---- - -### Task 2: New coordinate + provider types - -**Files:** -- Modify: `src/heights/interfaces.ts` - -- [ ] **Step 1: Replace the file contents** - -```typescript -import type GeotiffHeightProvider from './models/geotiffHeightProvider'; - -export enum TerrainTypes { - DSM = 'DSM', - DTM = 'DTM', - MIXED = 'MIXED', -} - -export interface GeoPoint { - longitude: number; // WGS84 degrees - latitude: number; // WGS84 degrees - height?: number | null; -} - -export interface PosWithHeight extends GeoPoint { - height: number | null; - productId?: string; -} - -export interface PosWithProvider extends GeoPoint { - providerKey?: string; -} - -export type HeightProviders = Record; -``` - -- [ ] **Step 2: Expect type errors elsewhere (fine for now)** - -Run: `npx tsc --noEmit -p tsconfig.json || true` -Expected: errors only in files that still import removed symbols (`PosWithTerrainProvider`, `TerrainProviders`, `cesium`). These are fixed in later tasks. The type import of `GeotiffHeightProvider` will error until Task 3 creates the file — that is expected. - -- [ ] **Step 3: Commit** - -```bash -git add src/heights/interfaces.ts -git commit -m "refactor: degrees-based GeoPoint and height provider types" -``` - ---- - -### Task 3: GeotiffHeightProvider — sampling engine (TDD) - -**Files:** -- Create: `src/heights/models/geotiffHeightProvider.ts` -- Test: `tests/unit/heights/models/geotiffHeightProvider.spec.ts` - -- [ ] **Step 1: Write the failing test** - -```typescript -import GeotiffHeightProvider from '../../../../src/heights/models/geotiffHeightProvider'; - -jest.mock('geotiff', () => ({ - fromUrl: jest.fn(), -})); - -// eslint-disable-next-line @typescript-eslint/no-var-requires -const { fromUrl } = require('geotiff') as { fromUrl: jest.Mock }; - -// origin [34,33], resolution [0.001,-0.001], 100x100, nodata -32768 -const makeImage = (band: number[], nodata: number | null = -32768) => ({ - getOrigin: () => [34.0, 33.0, 0], - getResolution: () => [0.001, -0.001], - getWidth: () => 100, - getHeight: () => 100, - getGDALNoData: () => nodata, - readRasters: jest.fn().mockResolvedValue([Int16Array.from(band)]), -}); - -const mockTiff = (image: unknown): void => { - fromUrl.mockResolvedValue({ getImage: jest.fn().mockResolvedValue(image) }); -}; - -describe('GeotiffHeightProvider', () => { - afterEach(() => jest.clearAllMocks()); - - it('bilinearly interpolates the height at a point', async () => { - // point at pixel (0,0)+0.5 in both axes → center of the 2x2 window - mockTiff(makeImage([200, 202, 204, 206])); - const provider = await GeotiffHeightProvider.fromUrl('http://gw/cogs/x.tif'); - - const [height] = await provider.sample([{ longitude: 34.0005, latitude: 32.9995 }]); - - expect(height).toBeCloseTo(203, 5); - }); - - it('returns null when any neighbor is nodata', async () => { - mockTiff(makeImage([200, -32768, 204, 206])); - const provider = await GeotiffHeightProvider.fromUrl('http://gw/cogs/x.tif'); - - const [height] = await provider.sample([{ longitude: 34.0005, latitude: 32.9995 }]); - - expect(height).toBeNull(); - }); - - it('returns null for a point outside the raster', async () => { - mockTiff(makeImage([200, 202, 204, 206])); - const provider = await GeotiffHeightProvider.fromUrl('http://gw/cogs/x.tif'); - - const [height] = await provider.sample([{ longitude: 10.0, latitude: 10.0 }]); - - expect(height).toBeNull(); - }); - - it('passes token headers through to fromUrl', async () => { - mockTiff(makeImage([1, 1, 1, 1])); - await GeotiffHeightProvider.fromUrl('http://gw/cogs/x.tif', { 'x-api-key': 'T' }); - - expect(fromUrl).toHaveBeenCalledWith('http://gw/cogs/x.tif', { headers: { 'x-api-key': 'T' } }); - }); -}); -``` - -- [ ] **Step 2: Run test to verify it fails** - -Run: `npx jest --config=./tests/configurations/unit/jest.config.js geotiffHeightProvider` -Expected: FAIL — `Cannot find module '.../geotiffHeightProvider'`. - -- [ ] **Step 3: Create the implementation** - -```typescript -import { fromUrl } from 'geotiff'; -import type { GeoTIFFImage } from 'geotiff'; -import { GeoPoint } from '../interfaces'; - -const DEFAULT_NODATA = -32768; - -export default class GeotiffHeightProvider { - private constructor( - private readonly image: GeoTIFFImage, - private readonly originX: number, - private readonly originY: number, - private readonly pixelWidth: number, // degrees/pixel, positive - private readonly pixelHeight: number, // degrees/pixel, negative - private readonly rasterWidth: number, - private readonly rasterHeight: number, - private readonly noData: number - ) {} - - public static async fromUrl(url: string, headers?: Record): Promise { - const tiff = await fromUrl(url, headers ? { headers } : {}); - const image = await tiff.getImage(0); - const [originX, originY] = image.getOrigin(); - const [pixelWidth, pixelHeight] = image.getResolution(); - const noData = image.getGDALNoData() ?? DEFAULT_NODATA; - - return new GeotiffHeightProvider(image, originX, originY, pixelWidth, pixelHeight, image.getWidth(), image.getHeight(), noData); - } - - public async sample(points: GeoPoint[]): Promise<(number | null)[]> { - return Promise.all(points.map(async (point) => this.sampleOne(point))); - } - - private async sampleOne(point: GeoPoint): Promise { - const fx = (point.longitude - this.originX) / this.pixelWidth; - const fy = (point.latitude - this.originY) / this.pixelHeight; - const px = Math.floor(fx); - const py = Math.floor(fy); - - // Need a full 2x2 neighborhood for bilinear interpolation. - if (px < 0 || py < 0 || px >= this.rasterWidth - 1 || py >= this.rasterHeight - 1) { - return null; - } - - const dx = fx - px; - const dy = fy - py; - - const raster = (await this.image.readRasters({ window: [px, py, px + 2, py + 2] })) as unknown as number[][]; - const band = raster[0]; - const [tl, tr, bl, br] = [band[0], band[1], band[2], band[3]]; - - if ([tl, tr, bl, br].some((value) => value === this.noData)) { - return null; - } - - const top = tl + (tr - tl) * dx; - const bottom = bl + (br - bl) * dx; - - return top + (bottom - top) * dy; - } -} -``` - -- [ ] **Step 4: Run test to verify it passes** - -Run: `npx jest --config=./tests/configurations/unit/jest.config.js geotiffHeightProvider` -Expected: PASS (4 tests). - -- [ ] **Step 5: Commit** - -```bash -git add src/heights/models/geotiffHeightProvider.ts tests/unit/heights/models/geotiffHeightProvider.spec.ts -git commit -m "feat: geotiff COG height provider with bilinear sampling" -``` - ---- - -### Task 4: DEMTerrainCacheManager builds geotiff providers - -**Files:** -- Modify: `src/heights/models/DEMTerrainCacheManager.ts` - -Rationale for keeping the filename and DI symbol: the symbol is only an injection token; keeping both avoids churn across `containerConfig.ts` and every test import. - -- [ ] **Step 1: Replace the file contents** - -```typescript -import { inject, injectable } from 'tsyringe'; -import { IConfig } from 'config'; -import { PycswDemCatalogRecord } from '@map-colonies/mc-model-types'; -import { HeightProviders } from '../interfaces'; -import { SERVICES } from '../../common/constants'; -import GeotiffHeightProvider from './geotiffHeightProvider'; - -const GEOTIFF_PROTOCOL = 'GEOTIFF'; -const COGS_FOLDER = 'cogs/'; - -@injectable() -export default class DEMTerrainCacheManager { - public heightProviders: HeightProviders = {}; - - public constructor(@inject(SERVICES.CONFIG) private readonly config: IConfig) {} - - public async initProviders(demCatalogRecords: PycswDemCatalogRecord[]): Promise { - const heightProviders: HeightProviders = {}; - - const geotiffRecords = demCatalogRecords.filter((record) => record.links?.some((link) => link.protocol === GEOTIFF_PROTOCOL)); - - for (const record of geotiffRecords) { - const link = record.links?.find((currentLink) => currentLink.protocol === GEOTIFF_PROTOCOL); - - if (link) { - const objectUrl = this.transformRouteToObjectUrl(link.url as string); - const { url, headers } = this.buildAuthenticatedUrl(objectUrl); - heightProviders[record.id as string] = await GeotiffHeightProvider.fromUrl(url, headers); - } - } - - this.heightProviders = heightProviders; - } - - private transformRouteToObjectUrl(linkUrl: string): string { - const serviceURL = this.config.get('s3Gateway.url'); - - return `${serviceURL}/${COGS_FOLDER}${linkUrl.split(COGS_FOLDER)[1]}`; - } - - private buildAuthenticatedUrl(objectUrl: string): { url: string; headers?: Record } { - const injectionType = this.config.get('accessToken.injectionType'); - const attributeName = this.config.get('accessToken.attributeName'); - const tokenValue = this.config.get('accessToken.tokenValue'); - - if (injectionType.toLowerCase() === 'header') { - return { url: objectUrl, headers: { [attributeName]: tokenValue } }; - } - - if (injectionType.toLowerCase() === 'queryparam') { - const separator = objectUrl.includes('?') ? '&' : '?'; - return { url: `${objectUrl}${separator}${attributeName}=${encodeURIComponent(tokenValue)}` }; - } - - return { url: objectUrl }; - } -} -``` - -- [ ] **Step 2: Update the DI wiring caller** - -In `src/containerConfig.ts`, change the one call inside the worker `updateValue` handler from: -```typescript - await demTerrainCacheManager.initTerrainProviders(dataValue); -``` -to: -```typescript - await demTerrainCacheManager.initProviders(dataValue); -``` - -- [ ] **Step 3: Build to type-check these two files** - -Run: `npm run build` -Expected: `heightsManager.ts` still errors (fixed next task); `DEMTerrainCacheManager.ts` and `containerConfig.ts` compile clean. - -- [ ] **Step 4: Commit** - -```bash -git add src/heights/models/DEMTerrainCacheManager.ts src/containerConfig.ts -git commit -m "feat: open geotiff providers from GEOTIFF catalog links" -``` - ---- - -### Task 5: HeightsManager samples via geotiff (TDD) - -**Files:** -- Modify: `src/heights/models/heightsManager.ts` -- Modify: `tests/configurations/testContainerConfig.ts` -- Modify: `tests/unit/heights/models/heightModel.spec.ts` - -- [ ] **Step 1: Update the test container to GEOTIFF links and mock the provider** - -In `tests/configurations/testContainerConfig.ts`: - -1. Add imports at the top (after existing imports): -```typescript -import GeotiffHeightProvider from '../../src/heights/models/geotiffHeightProvider'; -``` - -2. In BOTH catalog records, change the `links` entry `protocol` and `url`: -```typescript - links: [ - { - __typename: 'Link', - name: '', - description: '', - protocol: 'GEOTIFF', - url: 'https://tiles-dev.mapcolonies.net/api/dem/v1/cogs/combined_srtm_30_100_il_ever.tif', - }, - ], -``` -(For the second record use `.../cogs/srtm100.tif`.) - -3. Immediately before the `await (async ...)` IIFE, install the provider mock so `initProviders` does not hit the network: -```typescript - jest.spyOn(GeotiffHeightProvider, 'fromUrl').mockResolvedValue({ - sample: async (points: { longitude: number; latitude: number }[]) => points.map(() => 100), - } as unknown as GeotiffHeightProvider); -``` - -4. Change the init call: -```typescript - if (shouldInitTerrainProviders) { - await demTestTerrainCacheManager.initProviders(demTestCatalogRecords as unknown as PycswDemCatalogRecord[]); - } -``` - -- [ ] **Step 2: Rewrite the model unit test for degrees input** - -Replace the entire contents of `tests/unit/heights/models/heightModel.spec.ts`: - -```typescript -import { container } from 'tsyringe'; -import { HeightsManager } from '../../../../src/heights/models/heightsManager'; -import mockJsonPoints, { positionsOutsideOfProviders, emptyPositionsRequest } from '../../../../src/heights/MOCKS/mockData'; -import { GetHeightsPointsRequest } from '../../../../src/heights/controllers/heightsController'; -import { PosWithHeight, TerrainTypes } from '../../../../src/heights/interfaces'; -import { registerTestValues } from '../../../configurations/testContainerConfig'; - -describe('Get Heights model', function () { - const mockJsonData = mockJsonPoints as GetHeightsPointsRequest; - const mockJsonDataOutOfBounds = positionsOutsideOfProviders as GetHeightsPointsRequest; - - let heightsManager: HeightsManager; - - beforeEach(async function () { - await registerTestValues(); - heightsManager = container.resolve(HeightsManager); - }); - - afterEach(() => { - container.reset(); - container.clearInstances(); - jest.clearAllMocks(); - }); - - describe('Given valid parameters', function () { - it('Should return positions with height and productId', async function () { - const result = await heightsManager.getPoints(mockJsonData.positions, TerrainTypes.MIXED); - - expect(result).toHaveLength(mockJsonData.positions.length); - for (const position of result) { - expect(position.height).toBe(100); - expect(position.productId).toBeDefined(); - } - }); - - it('Should return null heights and no productId when no provider matches the product type', async function () { - const result = await heightsManager.getPoints(mockJsonData.positions, TerrainTypes.DSM); - - expect(result).toHaveLength(mockJsonData.positions.length); - for (const position of result) { - expect(position.height).toBeNull(); - expect(position.productId).toBeUndefined(); - } - }); - - it('Should return height only for the positions inside a provider footprint', async function () { - const result = await heightsManager.getPoints(mockJsonDataOutOfBounds.positions, TerrainTypes.MIXED); - - expect(result).toHaveLength(mockJsonDataOutOfBounds.positions.length); - for (const position of result) { - expect(position.longitude).toBeDefined(); - expect(position.latitude).toBeDefined(); - - const isNullHeight = (position.height as number | null) === null; - expect(typeof position.productId === 'undefined').toEqual(isNullHeight); - } - }); - }); - - describe('Given invalid params', function () { - it('Should return empty array for empty positions', async function () { - await expect(heightsManager.getPoints((emptyPositionsRequest as unknown as GetHeightsPointsRequest).positions, TerrainTypes.MIXED)).resolves.toEqual([]); - }); - }); -}); -``` - -Note: `mockData.ts` already exports plain `{ longitude, latitude }` objects (degrees), so no `Cartographic.fromDegrees` conversion is needed anymore. - -- [ ] **Step 3: Run the test to verify it fails** - -Run: `npx jest --config=./tests/configurations/unit/jest.config.js heightModel` -Expected: FAIL — `heightsManager.ts` still imports `cesium`/`sampleTerrainMostDetailed`; compile/runtime error. - -- [ ] **Step 4: Rewrite `heightsManager.ts`** - -Replace the entire contents of `src/heights/models/heightsManager.ts`: - -```typescript -import { Polygon } from 'geojson'; -import client from 'prom-client'; -import { container, inject, injectable } from 'tsyringe'; -import { Logger } from '@map-colonies/js-logger'; -import PromisePool from '@supercharge/promise-pool/dist'; -import booleanPointInPolygon from '@turf/boolean-point-in-polygon'; -import { Feature } from '@turf/turf'; -import { CommonErrors } from '../../common/commonErrors'; -import { SERVICES } from '../../common/constants'; -import { IConfig } from '../../common/interfaces'; -import { CATALOG_RECORDS_MAP, DEM_TERRAIN_CACHE_MANAGER } from '../../containerConfig'; -import { GeoPoint, PosWithHeight, PosWithProvider, TerrainTypes } from '../interfaces'; -import DEMTerrainCacheManager from './DEMTerrainCacheManager'; -import { CatalogRecords } from './catalogRecords'; - -export interface ICoordinates { - longitude: string; - latitude: string; -} - -export interface IHeightModel { - dem: number; -} - -@injectable() -export class HeightsManager { - private runningRequests = 0; - - private get catalogRecordsMap() { - return container.resolve(CATALOG_RECORDS_MAP).getValue(); - } - - private get heightProviders() { - return container.resolve(DEM_TERRAIN_CACHE_MANAGER).heightProviders; - } - - private readonly elevationsRequestsCounter?: client.Counter<'points_number'>; - - public constructor( - @inject(SERVICES.LOGGER) private readonly logger: Logger, - @inject(CommonErrors) private readonly commonErrors: CommonErrors, - @inject(SERVICES.CONFIG) private readonly config: IConfig, - @inject(SERVICES.METRICS_REGISTRY) registry?: client.Registry - ) { - if (registry !== undefined) { - // eslint-disable-next-line @typescript-eslint/no-this-alias - const self = this; - new client.Gauge({ - name: 'elevations_current_requests_count', - help: 'Currently running elevations requests', - collect(): void { - this.set(self.runningRequests); - }, - registers: [registry], - }); - - this.elevationsRequestsCounter = new client.Counter({ - name: 'elevations_requests_total', - help: 'Total elevations requests', - labelNames: ['points_number'] as const, - registers: [registry], - }); - } - } - - public async getPoints(points: GeoPoint[], requestedProductType: TerrainTypes, reqCtx?: Record): Promise { - this.logger.info({ pointsNumber: points.length, location: '[HeightsManager] [getPoints]', ...reqCtx }); - - this.runningRequests++; - // eslint-disable-next-line @typescript-eslint/naming-convention - this.elevationsRequestsCounter?.inc({ points_number: points.length }); - - if (points.length === 0) { - this.runningRequests--; - return []; - } - - const result = await this.samplePositionsHeights(points, requestedProductType, reqCtx); - - this.logger.info({ - totalRequests: result.totalRequests, - pointsNumber: points.length, - location: '[HeightsManager] [getPoints]', - ...reqCtx, - }); - - this.runningRequests--; - return result.positions; - } - - private async samplePositionsHeights( - positionsArr: GeoPoint[], - requestedProductType: TerrainTypes, - reqCtx?: Record - ): Promise<{ positions: PosWithHeight[]; totalRequests: number }> { - const attachProviderStart = performance.now(); - const positionsWithProviders = this.attachProviderToPositions(positionsArr, requestedProductType); - this.logger.info({ - attachProviderTime: performance.now() - attachProviderStart, - pointsNumber: positionsArr.length, - location: '[HeightsManager] [samplePositionsHeights]', - ...reqCtx, - }); - - // Group points by the provider chosen for them (null = no provider). - const groups = new Map(); - for (const position of positionsWithProviders) { - const key = position.providerKey ?? null; - const bucket = groups.get(key) ?? []; - bucket.push({ longitude: position.longitude, latitude: position.latitude }); - groups.set(key, bucket); - } - - const groupEntries = [...groups.entries()]; - const finalPositionsWithHeights: PosWithHeight[] = []; - - const { results } = await PromisePool.for(groupEntries) - .withConcurrency(Math.max(1, groupEntries.length)) - .process(async ([providerKey, points]) => { - if (providerKey === null) { - return points.map((point) => ({ ...point, height: null } as PosWithHeight)); - } - - const samplingStart = performance.now(); - const provider = this.heightProviders[providerKey]; - const record = this.catalogRecordsMap[providerKey]; - const heights = await provider.sample(points); - - this.logger.info({ - terrainSamplingTime: performance.now() - samplingStart, - providerId: providerKey, - pointsNumber: positionsArr.length, - location: '[HeightsManager] [samplePositionsHeights]', - ...reqCtx, - }); - - return points.map((point, index) => { - const height = heights[index]; - return { - ...point, - height, - ...(height !== null ? { productId: record.productId as string } : {}), - } as PosWithHeight; - }); - }); - - finalPositionsWithHeights.push(...(results as PosWithHeight[][]).flat()); - - return { positions: finalPositionsWithHeights, totalRequests: groupEntries.length }; - } - - private attachProviderToPositions(positions: GeoPoint[], requestedProductType: TerrainTypes): PosWithProvider[] { - /* - * Filter providers by requested product type (unless MIXED) - * Filter providers by footprint point intersection - * Sort by highest resolution (lower resolutionMeter is better), tie-break on newest updateDate - * Attach the best provider key to the point - */ - return positions.map((position) => { - const providerEntries = Object.entries(this.heightProviders); - - const productTypeFiltered = - requestedProductType !== TerrainTypes.MIXED - ? providerEntries.filter(([key]) => this.catalogRecordsMap[key].productType?.includes(requestedProductType)) - : providerEntries; - - const footprintFiltered = productTypeFiltered.filter(([key]) => - booleanPointInPolygon([position.longitude, position.latitude], this.catalogRecordsMap[key].footprint as Feature) - ); - - const sorted = footprintFiltered.sort(([aKey], [bKey]) => { - const A_BEFORE_B = -1; - const B_BEFORE_A = 1; - const recordA = this.catalogRecordsMap[aKey]; - const recordB = this.catalogRecordsMap[bKey]; - - switch (true) { - case (recordA.resolutionMeter as number) < (recordB.resolutionMeter as number): - return A_BEFORE_B; - case (recordA.resolutionMeter as number) > (recordB.resolutionMeter as number): - return B_BEFORE_A; - default: { - const dateB = new Date(recordB.updateDate as string | number | Date); - const dateA = new Date(recordA.updateDate as string | number | Date); - return dateB.getTime() - dateA.getTime(); - } - } - }); - - if (sorted.length === 0) { - return { ...position } as PosWithProvider; - } - - const [providerKey] = sorted[0]; - return { ...position, providerKey } as PosWithProvider; - }); - } -} -``` - -Notes: -- The `maximumTilesPerRequest` density throw and `RequestScheduler.update()` are intentionally removed — both were quantized-mesh tile concerns. Point-count limiting stays in `validateRequestMiddleware`. -- `commonErrors` stays injected (used elsewhere in the class surface / future use); leaving the constructor param avoids DI signature churn. - -- [ ] **Step 5: Run the model test to verify it passes** - -Run: `npx jest --config=./tests/configurations/unit/jest.config.js heightModel` -Expected: PASS (4 tests). - -- [ ] **Step 6: Commit** - -```bash -git add src/heights/models/heightsManager.ts tests/configurations/testContainerConfig.ts tests/unit/heights/models/heightModel.spec.ts -git commit -m "feat: sample point heights from geotiff providers in degrees" -``` - ---- - -### Task 6: Trim utilities to checksum helpers only - -**Files:** -- Modify: `src/heights/utilities.ts` - -- [ ] **Step 1: Replace the file contents** - -```typescript -import crypto from 'crypto'; - -export const generateChecksum = (str: string, algorithm?: string, encoding?: crypto.BinaryToTextEncoding): string => { - return crypto - .createHash(algorithm ?? 'md5') - .update(str, 'utf8') - .digest(encoding ?? 'hex'); -}; - -// eslint-disable-next-line @typescript-eslint/no-explicit-any -export const isSame = (src1: any, src2: any): boolean => { - return generateChecksum(JSON.stringify(src1)) === generateChecksum(JSON.stringify(src2)); -}; -``` - -- [ ] **Step 2: Confirm no remaining imports of the removed clustering function** - -Run: `grep -rn "cartographicArrayClusteringForHeightRequests\|PositionsWithProviderKey" src tests` -Expected: no matches. - -- [ ] **Step 3: Commit** - -```bash -git add src/heights/utilities.ts -git commit -m "refactor: drop quantized-mesh tile clustering utilities" -``` - ---- - -### Task 7: CSW discovery filters GEOTIFF links - -**Files:** -- Modify: `src/workerCatalogRecords.ts` - -- [ ] **Step 1: Change the CSW filter** - -In `src/workerCatalogRecords.ts`, replace the `mc:links` filter value: -```typescript - { - field: 'mc:links', - like: 'GEOTIFF', - }, -``` -and update the two comment lines above it to reference `GEOTIFF` instead of `TERRAIN_PROVIDER`/`TERRAIN_QMESH`: -```typescript - // ******* DEM profile links carry the object protocol. We match records exposing a GEOTIFF link - // ******* (COG served from the S3 gateway) via a LIKE filter on the LINKS field. -``` - -- [ ] **Step 2: Build to type-check** - -Run: `npm run build` -Expected: compiles (any remaining errors are in controller/middleware/router — fixed next tasks). - -- [ ] **Step 3: Commit** - -```bash -git add src/workerCatalogRecords.ts -git commit -m "feat: discover GEOTIFF catalog records instead of qmesh" -``` - ---- - -### Task 8: Controller + validate types to degrees; drop radians map - -**Files:** -- Modify: `src/heights/controllers/heightsController.ts` -- Modify: `src/heights/middlewares/validateRequest.ts` - -- [ ] **Step 1: Update the controller request type** - -In `src/heights/controllers/heightsController.ts`: - -1. Replace the cesium import line: -```typescript -import { Cartographic } from 'cesium'; -``` -with: -```typescript -import { GeoPoint } from '../interfaces'; -``` - -2. Change the request interface (remove `radiansToOriginalPositionsMap`, retype `positions`): -```typescript -export interface GetHeightsPointsRequest { - positions: GeoPoint[]; - productType?: TerrainTypes; -} -``` - -`PosWithHeight` and `TerrainTypes` are already imported from `../interfaces`. The `getPoints` handler body is unchanged (it already passes `userInput.positions`). - -- [ ] **Step 2: Update validateRequest** - -In `src/heights/middlewares/validateRequest.ts`: - -1. Replace the cesium import: -```typescript -import { Cartographic } from 'cesium'; -``` -with: -```typescript -import { GeoPoint } from '../interfaces'; -``` - -2. Change the points type: -```typescript - const points: GeoPoint[] = req.body.positions; -``` - -- [ ] **Step 3: Build to type-check these files** - -Run: `npm run build` -Expected: only router + deleted-middleware errors remain (next task). - -- [ ] **Step 4: Commit** - -```bash -git add src/heights/controllers/heightsController.ts src/heights/middlewares/validateRequest.ts -git commit -m "refactor: type points as degrees, drop radians request map" -``` - ---- - -### Task 9: Remove radian middlewares from the route - -**Files:** -- Delete: `src/heights/middlewares/dataToRadians.ts` -- Delete: `src/heights/middlewares/dataToDegrees.ts` -- Modify: `src/heights/routes/heightsRouter.ts` - -- [ ] **Step 1: Delete the two middlewares** - -Run: -```bash -git rm src/heights/middlewares/dataToRadians.ts src/heights/middlewares/dataToDegrees.ts -``` - -- [ ] **Step 2: Update the router** - -In `src/heights/routes/heightsRouter.ts`: - -1. Remove these two imports: -```typescript -import { positionResAsDegreesMiddleware } from '../middlewares/dataToDegrees'; -import { convertReqPositionToRadiansMiddleware } from '../middlewares/dataToRadians'; -``` - -2. Change the `router.post('/points', ...)` chain to drop both middlewares — final chain: -```typescript - router.post( - '/points', - createReqCtxMiddleware(logger), - validateRequestMiddleware(config, logger, commonErrors), - controller.getPoints, - addProductsDictionaryMiddleware(logger, productMetadataFields), - sendResponseMiddleware(logger) - ); -``` - -- [ ] **Step 3: Full build** - -Run: `npm run build` -Expected: PASS — no compile errors anywhere. - -- [ ] **Step 4: Confirm cesium is fully gone from source** - -Run: `grep -rn "cesium\|Cartographic\|radians" src` -Expected: no matches. - -- [ ] **Step 5: Commit** - -```bash -git add -A -git commit -m "refactor: remove radian conversion middlewares from points route" -``` - ---- - -### Task 10: Fix the middleware unit test - -**Files:** -- Modify: `tests/unit/heights/middlewares/heightsMiddlewares.spec.ts` - -- [ ] **Step 1: Remove the deleted-middleware tests and imports** - -In `tests/unit/heights/middlewares/heightsMiddlewares.spec.ts`: - -1. Remove these imports: -```typescript -import { Cartographic } from 'cesium'; -import { positionResAsDegreesMiddleware } from '../../../../src/heights/middlewares/dataToDegrees'; -import { convertReqPositionToRadiansMiddleware } from '../../../../src/heights/middlewares/dataToRadians'; -``` - -2. Remove the two now-unused declarations: -```typescript - let dataToRadiansMiddleware: GetHeightsHandler; - let dataToDegreesMiddleware: GetHeightsHandler; -``` - -3. Remove their assignments in `beforeAll`: -```typescript - dataToRadiansMiddleware = convertReqPositionToRadiansMiddleware(logger); - dataToDegreesMiddleware = positionResAsDegreesMiddleware(logger); -``` - -4. Delete the entire `describe('Data to radians middleware', ...)` and `describe('Data to degrees middleware', ...)` blocks. - -The remaining blocks — `Create request id middleware`, `Add products dictionary`, `Validate request middleware` — stay unchanged. (The `PosWithHeight` import becomes unused after deleting the degrees block; remove it too if the linter flags it.) - -- [ ] **Step 2: Run the middleware unit test** - -Run: `npx jest --config=./tests/configurations/unit/jest.config.js heightsMiddlewares` -Expected: PASS (reqCtx, products dictionary, validate empty, validate too-many). - -- [ ] **Step 3: Commit** - -```bash -git add tests/unit/heights/middlewares/heightsMiddlewares.spec.ts -git commit -m "test: drop radian middleware unit tests" -``` - ---- - -### Task 11: Fix the integration test - -**Files:** -- Modify: `tests/integration/heights/heights.spec.ts` - -The integration suite boots the real app (the CSW worker's network fetch fails harmlessly in test, leaving providers empty), so its assertions only require `200` + response shape + the null-height invariants — all of which hold with no providers. It only needs the Cesium spy removed. - -- [ ] **Step 1: Remove Cesium from the integration test** - -In `tests/integration/heights/heights.spec.ts`: - -1. Remove the import: -```typescript -import { Cartesian2, Cartographic, CesiumTerrainProvider } from 'cesium'; -``` - -2. Remove the spy declaration: -```typescript - let cesiumTerrainProviderFromUrlSpy: jest.SpyInstance; -``` - -3. Remove the entire `cesiumTerrainProviderFromUrlSpy = jest.spyOn(...)` + `.mockReturnValue({...})` block inside `beforeAll` (the `availability`/`tilingScheme` mock). - -`basicPositionResponse` is only referenced in commented-out code; leave it or delete it — either compiles. - -- [ ] **Step 2: Run the integration suite** - -Run: `npm run test:integration` -Expected: PASS. All valid-params cases return `200` with correct `data` length; invalid-params return `400` with `TOO_MANY_POINTS_ERROR` / `EMPTY_POSITIONS_ARRAY`. - -- [ ] **Step 3: Commit** - -```bash -git add tests/integration/heights/heights.spec.ts -git commit -m "test: remove cesium terrain mock from heights integration" -``` - ---- - -### Task 12: Full verification pass - -**Files:** none (verification only) - -- [ ] **Step 1: Lint** - -Run: `npm run lint` -Expected: no errors. If any, `npm run lint:fix` then re-run and commit with `style:`. - -- [ ] **Step 2: Full test suite** - -Run: `npm test` -Expected: unit + integration all green. - -- [ ] **Step 3: Build** - -Run: `npm run build` -Expected: clean compile, `dist/` produced. - -- [ ] **Step 4: Confirm no leftover qmesh/cesium references** - -Run: `grep -rn "cesium\|Cartographic\|TERRAIN_QMESH\|sampleTerrainMostDetailed\|radians" src tests` -Expected: no matches. - -- [ ] **Step 5: Commit any lint fixes** - -```bash -git add -A -git commit -m "style: lint fixes for geotiff migration" || echo "nothing to commit" -``` - ---- - -## Out-of-repo deployment tasks (NOT code — do after merge, coordinate with ops) - -These are required for the service to actually serve heights in `dem-dev`, but they are data/infra, not part of this repo's TDD cycle: - -1. **Catalog record** — the DEM CSW record(s) must expose a `GEOTIFF` link whose URL resolves (after the `cogs/` split) to the object key in bucket `dem-dev`, e.g. `https://tiles-dev.mapcolonies.net/api/dem/v1/cogs/dtm_srtm30wgs84geo_tiled256_ovr_lzw.tif`. Today the live record has a `TERRAIN_QMESH` link to `terrains/srtm100`. -2. **COG upload** — ensure the `tiled_ovr` COG (native 30 m, tiled, overviews) is the object served — NOT the coarse `_COG`. SRTM100 COG is not yet under `cogs/`; upload if 100 m coverage is required. -3. **helm-charts** — no chart change is strictly required (the service reads `s3Gateway.url` + `accessToken`, both already set to the internal gateway with token as queryParam). Confirm the gateway route/prefix `cogs/` is reachable via `dem-nginx-s3-gateway-internal`. - -## Self-Review notes - -- **Spec coverage:** engine swap (Tasks 3–6), discovery (Task 7), degrees end-to-end + middleware removal (Tasks 2, 8, 9), dependency swap (Task 1), tests (Tasks 5, 10, 11), verification (Task 12). API contract (`/points`, `openapi3.yaml`, response shape) is unchanged by design — no task needed. -- **Type consistency:** `GeoPoint` / `PosWithHeight` / `PosWithProvider` / `HeightProviders` defined in Task 2 are used consistently in Tasks 3–5, 8. Cache manager members `heightProviders` + `initProviders` defined in Task 4 are consumed in Task 5 and `containerConfig` (Task 4 step 2). `GeotiffHeightProvider.fromUrl(url, headers?)` signature defined in Task 3 matches its callers in Task 4 and the mock in Task 5. -- **Null-height invariant:** integration test #115-equivalent requires `productId === undefined ⇔ height === null`. Enforced in `samplePositionsHeights` (productId attached only when `height !== null`) and for the no-provider bucket. -- **nodata:** `-32768` default plus `getGDALNoData()`; any nodata neighbor → `null` (Task 3). From 620d66ccc71ff46c94c7ceb30d909cbb6365295e Mon Sep 17 00:00:00 2001 From: shimoncohen Date: Mon, 28 Sep 2026 14:14:13 +0300 Subject: [PATCH 25/25] style: fix eslint errors Co-Authored-By: Claude Opus 5.5 (1M context) --- src/heights/models/geotiffHeightProvider.ts | 5 ++++- tests/configurations/testContainerConfig.ts | 4 +++- tests/integration/heights/heights.spec.ts | 4 ++-- tests/unit/heights/models/geotiffHeightProvider.spec.ts | 2 ++ 4 files changed, 11 insertions(+), 4 deletions(-) diff --git a/src/heights/models/geotiffHeightProvider.ts b/src/heights/models/geotiffHeightProvider.ts index 6713ee0..697195b 100644 --- a/src/heights/models/geotiffHeightProvider.ts +++ b/src/heights/models/geotiffHeightProvider.ts @@ -5,6 +5,7 @@ import { GeoPoint } from '../interfaces'; const DEFAULT_NODATA = -32768; const DEFAULT_SAMPLING_CONCURRENCY = 16; +const BILINEAR_WINDOW_SIZE = 2; export default class GeotiffHeightProvider { private constructor( @@ -71,7 +72,9 @@ export default class GeotiffHeightProvider { const dx = fx - px; const dy = fy - py; - const raster = (await this.image.readRasters({ window: [px, py, px + 2, py + 2] })) as unknown as number[][]; + const raster = (await this.image.readRasters({ + window: [px, py, px + BILINEAR_WINDOW_SIZE, py + BILINEAR_WINDOW_SIZE], + })) as unknown as number[][]; const band = raster[0]; const [tl, tr, bl, br] = [band[0], band[1], band[2], band[3]]; diff --git a/tests/configurations/testContainerConfig.ts b/tests/configurations/testContainerConfig.ts index 7af5551..d1af628 100644 --- a/tests/configurations/testContainerConfig.ts +++ b/tests/configurations/testContainerConfig.ts @@ -9,6 +9,8 @@ import DEMTerrainCacheManager from '../../src/heights/models/DEMTerrainCacheMana import GeotiffHeightProvider from '../../src/heights/models/geotiffHeightProvider'; import { CatalogRecords } from '../../src/heights/models/catalogRecords'; +const MOCK_SAMPLE_HEIGHT = 100; + async function registerTestValues(shouldInitTerrainProviders = true): Promise { /* eslint-disable */ const demTestCatalogRecords = [ @@ -150,7 +152,7 @@ async function registerTestValues(shouldInitTerrainProviders = true): Promise points.map(() => 100), + sample: async (points: { longitude: number; latitude: number }[]) => Promise.resolve(points.map(() => MOCK_SAMPLE_HEIGHT)), } as unknown as GeotiffHeightProvider); await (async (): Promise => { diff --git a/tests/integration/heights/heights.spec.ts b/tests/integration/heights/heights.spec.ts index aab21bf..af335a8 100644 --- a/tests/integration/heights/heights.spec.ts +++ b/tests/integration/heights/heights.spec.ts @@ -121,7 +121,7 @@ describe('heights', function () { for (const position of (response.body as GetHeightsPointsResponse).data) { expect(position['latitude'] && position['longitude']).toBeDefined(); - const isNullHeight = (position.height as number | null) === null; + const isNullHeight = position.height === null; expect(typeof position['productId'] === 'undefined').toEqual(isNullHeight); } @@ -152,7 +152,7 @@ describe('heights', function () { describe('Given seeded geotiff providers', function () { beforeAll(async function () { jest.spyOn(GeotiffHeightProvider, 'fromUrl').mockResolvedValue({ - sample: async (points: { longitude: number; latitude: number }[]) => points.map(() => 123), + sample: async (points: { longitude: number; latitude: number }[]) => Promise.resolve(points.map(() => 123)), } as unknown as GeotiffHeightProvider); const records = [ diff --git a/tests/unit/heights/models/geotiffHeightProvider.spec.ts b/tests/unit/heights/models/geotiffHeightProvider.spec.ts index 3bc9c9c..9b5976e 100644 --- a/tests/unit/heights/models/geotiffHeightProvider.spec.ts +++ b/tests/unit/heights/models/geotiffHeightProvider.spec.ts @@ -54,8 +54,10 @@ describe('GeotiffHeightProvider', () => { it('passes token headers through to fromUrl', async () => { mockTiff(makeImage([1, 1, 1, 1])); + // eslint-disable-next-line @typescript-eslint/naming-convention await GeotiffHeightProvider.fromUrl('http://gw/cogs/x.tif', { 'x-api-key': 'T' }); + // eslint-disable-next-line @typescript-eslint/naming-convention expect(fromUrl).toHaveBeenCalledWith('http://gw/cogs/x.tif', { headers: { 'x-api-key': 'T' } }); }); });