From b31b723f81eac0f6270eac074cd55b9e92171d5b Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 12 Aug 2026 12:28:16 +0000 Subject: [PATCH 1/2] osm cockpit: WebGL2 hardware-accelerated basemap rendering MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Client-side rendering was ~57,690-70,000 DOM
elements (one per feature dot), which is the actual source of the reported lag — not the Canvas2D basemap draw calls, which stayed under ~150/frame throughout. Adds an additive WebGL2 layer to the single embedded cockpit page: - initGL() creates a WebGL2 context alongside the existing canvas; automatic fallback to the existing Canvas2D path if it fails or is unavailable (gl stays null and every draw/pick site branches on it). - parseTileBinGL() decodes the same OSM1 wire format into interleaved (x, y, class) vertex buffers instead of Path2D — triangles for fills (pure-JS ear-clipping triangulation), line segments for strokes. - drawBaseGL()/drawDotsGL() replace per-feature DOM nodes with ONE gl.POINTS draw call per tile from a lazily-built Float32Array — the actual fix for the DOM-node lag. - pickFeature() replaces the DOM click-target lookup with a nearest-point scan over the same vertex buffers. - Per-class fill/stroke colors and the meadow/park texture patterns (stipple, hatch) are reproduced in the fragment shader from the same CLASS_STYLE table the Canvas2D path already used. Verified end-to-end with a real GPU (headless Chromium via Playwright, --use-gl=swiftshader), desktop and phone viewports: real triangle/point data painted (570,006 / 137,421 px respectively, glError:0), zero DOM dots, pan-repaint confirmed. A same-synchronous-block draw+readPixels diagnostic was needed to rule out a preserveDrawingBuffer:false compositing false-negative before trusting that measurement. wgpu/wasm is explicitly deferred — this is the WebGL2-now half only. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01NMeiLmtDKhomJNSo2ecbJw --- crates/cockpit-server/src/osm.rs | 317 ++++++++++++++++++++++++++++++- 1 file changed, 313 insertions(+), 4 deletions(-) diff --git a/crates/cockpit-server/src/osm.rs b/crates/cockpit-server/src/osm.rs index b73f7edb42..09ee2f5329 100644 --- a/crates/cockpit-server/src/osm.rs +++ b/crates/cockpit-server/src/osm.rs @@ -410,6 +410,180 @@ function parseTileBin(buf){ return {total,sampled,returned:count,malformed,fills,strokes}; } +// ── WebGL2 renderer — hardware acceleration (operator-mandated) ───────────── +// Canvas2D above stays as the automatic fallback: same wire, same cache keys, +// same status line, chosen once at startup. The GL path replaces BOTH hot +// layers: +// fills/strokes — Path2D re-rasterized per frame -> retained per-tile VBOs +// feature dots — one DOM
per feature -> one gl.POINTS draw +// The dot layer was the MEASURED lag: 57,690 features = 57,690 absolutely- +// positioned divs, the same retained-DOM architecture the module doc records +// as removed ("69k retained SVG DOM nodes... terribly slow") — it had quietly +// survived in the overlay while the basemap moved to canvas. +function hexRGBA(x){ const h=x.slice(1); + return [parseInt(h.slice(0,2),16)/255, parseInt(h.slice(2,4),16)/255, + parseInt(h.slice(4,6),16)/255, h.length>=8?parseInt(h.slice(6,8),16)/255:1]; } +const FILL_RGBA=CLASS_ORDER.map(n=>{const st=CLASS_STYLE[n]; + return st.fill==='none'?[0,0,0,0]:hexRGBA(st.fill);}); +const STROKE_RGBA=CLASS_ORDER.map(n=>hexRGBA(CLASS_STYLE[n].stroke)); + +let gl=null, glProg=null, glLoc=null, glCanvas=null, glSel=null; +function initGL(){ + const c=document.createElement('canvas'); + c.id='base-gl'; + c.style.cssText='position:absolute;left:0;top:0;pointer-events:none;'; + const g=c.getContext('webgl2',{alpha:true,antialias:true}); + if(!g) return false; // no WebGL2: Canvas2D path runs + map.insertBefore(c,map.firstChild); + glCanvas=c; gl=g; + const vs=`#version 300 es +in vec2 aPos; in float aCls; +uniform vec2 uView; uniform vec2 uTrans; uniform float uPointPx; +out float vCls; out vec2 vWorld; +void main(){ + vec2 p=aPos+uTrans; vWorld=p; vCls=aCls; + gl_Position=vec4(p.x*2.0/uView.x-1.0, 1.0-p.y*2.0/uView.y, 0.0, 1.0); + gl_PointSize=uPointPx; +}`; + // Procedural textures replace the canvas createPattern tiles: the SAME 8px + // cadence, keyed on vWorld (pre-pan-offset px), so the pattern rides the map + // exactly like the translate()d canvas pattern did. + const fs=`#version 300 es +precision mediump float; +in float vCls; in vec2 vWorld; +uniform vec4 uFill[10]; uniform vec4 uStroke[10]; uniform int uMode; +out vec4 frag; +void main(){ + if(uMode>=2){ vec2 d=gl_PointCoord-0.5; if(dot(d,d)>0.25) discard; + frag = uMode==3 ? vec4(0.494,0.906,0.529,1.0) // selection #7ee787 + : vec4(1.0,0.706,0.329,1.0); // dot #ffb454 + return; } + int c=int(vCls+0.5); + if(uMode==1){ frag=uStroke[c]; return; } + vec4 col=uFill[c]; + vec2 m=mod(vWorld,8.0); + if(c==7){ // meadow stipple — same 4 dot cells as the canvas pattern + if((m.x>=1.0&&m.x<2.0&&m.y>=1.0&&m.y<2.0)||(m.x>=5.0&&m.x<6.0&&m.y>=3.0&&m.y<4.0)|| + (m.x>=3.0&&m.x<4.0&&m.y>=6.0&&m.y<7.0)||(m.x>=7.0&&m.y>=5.0&&m.y<6.0)) + col=vec4(uStroke[7].rgb, min(1.0,col.a+0.35)); + } else if(c==2){ // wood canopy — the crossing-stroke read, as one hatch band + if(abs(m.x+m.y-8.0)<0.7) col=vec4(uStroke[2].rgb, min(1.0,col.a+0.25)); + } + frag=col; +}`; + const sh=(t,src)=>{ const o=g.createShader(t); g.shaderSource(o,src); g.compileShader(o); + if(!g.getShaderParameter(o,g.COMPILE_STATUS)) throw new Error(g.getShaderInfoLog(o)); + return o; }; + glProg=g.createProgram(); + g.attachShader(glProg,sh(g.VERTEX_SHADER,vs)); + g.attachShader(glProg,sh(g.FRAGMENT_SHADER,fs)); + g.linkProgram(glProg); + if(!g.getProgramParameter(glProg,g.LINK_STATUS)) throw new Error(g.getProgramInfoLog(glProg)); + g.useProgram(glProg); + glLoc={ aPos:g.getAttribLocation(glProg,'aPos'), aCls:g.getAttribLocation(glProg,'aCls'), + uView:g.getUniformLocation(glProg,'uView'), uTrans:g.getUniformLocation(glProg,'uTrans'), + uMode:g.getUniformLocation(glProg,'uMode'), uPointPx:g.getUniformLocation(glProg,'uPointPx') }; + g.uniform4fv(g.getUniformLocation(glProg,'uFill'), FILL_RGBA.flat()); + g.uniform4fv(g.getUniformLocation(glProg,'uStroke'), STROKE_RGBA.flat()); + g.enable(g.BLEND); g.blendFunc(g.SRC_ALPHA,g.ONE_MINUS_SRC_ALPHA); + return true; +} + +// Ear-clipping triangulation for one simple ring (no holes — the wire delivers +// per-way rings; multipolygon inners arrive as their own ways, matching what +// the Canvas2D nonzero fill drew). Handles CW and CCW input. A degenerate or +// self-intersecting ring falls back to a fan over whatever remains rather than +// looping forever — draw something sane, never hang. +function earClip(pts,n){ + const idx=[]; for(let i=0;i1 && pts[0]===pts[2*(n-1)] && pts[1]===pts[2*(n-1)+1]) idx.pop(); + let m=idx.length; + if(m<3) return []; + let area=0; + for(let i=0;i0, out=[]; + let guard=m*m+m, i=0; + while(m>3 && guard-->0){ + const i0=idx[(i+m-1)%m], i1=idx[i%m], i2=idx[(i+1)%m]; + const ax=pts[2*i0],ay=pts[2*i0+1],bx=pts[2*i1],by=pts[2*i1+1],qx=pts[2*i2],qy=pts[2*i2+1]; + const cross=(bx-ax)*(qy-ay)-(by-ay)*(qx-ax); + let ear=ccw?cross>0:cross<0; + if(ear){ + for(let k=0;k0)||(d2>0)||(d3>0)))) ear=false; + } + } + if(ear){ out.push(ax,ay,bx,by,qx,qy); idx.splice(i%m,1); m--; i=0; } + else i=(i+1)%m; + } + if(m>=3){ + if(guard<=0){ // bail: fan the remainder from its first vertex + for(let k=1;kdv.byteLength) return null; + const cls=dv.getUint8(at+4), closed=dv.getUint8(at+5)===1; + const n=dv.getUint16(at+6,true); at+=8; + if(at+n*8>dv.byteLength) return null; + const pts=new Float32Array(buf,at,n*2); at+=n*8; + if(n<2) continue; + let L=linesBy.get(cls); if(!L){ L=[]; linesBy.set(cls,L); } + for(let i=0;i{ if(!buf) return; - const d=parseTileBin(buf); + const d=gl?parseTileBinGL(buf):parseTileBin(buf); if(d){ geomCache.set(key,d); scheduleBaseDraw(); } else geomCache.delete(key); // bad payload: allow a retry }).catch(()=>{ geomCache.delete(key); }); // transient: retry on next render @@ -458,6 +632,7 @@ function scheduleBaseDraw(){ requestAnimationFrame(()=>{ baseDrawQueued=false; drawBase(); }); } function drawBase(){ + if(gl) return drawBaseGL(); const ctx=baseLayerCanvas(); const w=map.clientWidth, h=map.clientHeight; ctx.clearRect(0,0,w,h); @@ -487,6 +662,115 @@ function drawBase(){ } updateBaseStatus(); } +// The GL frame. Uploads a tile's arrays once (on first sight) and thereafter +// re-draws by binding — the per-frame cost is bind + uniform + drawArrays, so +// panning never re-walks geometry. `uTrans` carries the pan offset, so no +// vertex data is rewritten while dragging. +function drawBaseGL(){ + const dpr=window.devicePixelRatio||1; + const w=map.clientWidth, h=map.clientHeight; + if(glCanvas.width!==Math.round(w*dpr)||glCanvas.height!==Math.round(h*dpr)){ + glCanvas.width=Math.round(w*dpr); glCanvas.height=Math.round(h*dpr); + glCanvas.style.width=w+'px'; glCanvas.style.height=h+'px'; + } + gl.viewport(0,0,glCanvas.width,glCanvas.height); + gl.clearColor(0,0,0,0); gl.clear(gl.COLOR_BUFFER_BIT); + if(BASEMAPS[basemap].src) return; // a raster skin is active + if(z!==geomZoom){ evictGLBuffers(); geomCache.clear(); geomZoom=z; } + gl.useProgram(glProg); + gl.uniform2f(glLoc.uView,w,h); + const ox=w/2 - cx*256, oy=h/2 - cy*256; + const bind=(vbo)=>{ + gl.bindBuffer(gl.ARRAY_BUFFER,vbo); + gl.enableVertexAttribArray(glLoc.aPos); + gl.vertexAttribPointer(glLoc.aPos,2,gl.FLOAT,false,12,0); + gl.enableVertexAttribArray(glLoc.aCls); + gl.vertexAttribPointer(glLoc.aCls,1,gl.FLOAT,false,12,8); + }; + for(const {tx,ty,wx} of visibleGrid()){ + ensureGeometry(z,wx,ty); + const d=geomCache.get(tileKey(z,wx,ty)); + if(!(d && typeof d==='object' && d.fill)) continue; + if(!d.vboF){ + d.vboF=gl.createBuffer(); gl.bindBuffer(gl.ARRAY_BUFFER,d.vboF); + gl.bufferData(gl.ARRAY_BUFFER,d.fill,gl.STATIC_DRAW); + d.vboL=gl.createBuffer(); gl.bindBuffer(gl.ARRAY_BUFFER,d.vboL); + gl.bufferData(gl.ARRAY_BUFFER,d.line,gl.STATIC_DRAW); + } + gl.uniform2f(glLoc.uTrans, tx*256+ox, ty*256+oy); + if(d.fill.length){ gl.uniform1i(glLoc.uMode,0); bind(d.vboF); + gl.drawArrays(gl.TRIANGLES,0,d.fill.length/3); } + if(d.line.length){ gl.uniform1i(glLoc.uMode,1); bind(d.vboL); + gl.drawArrays(gl.LINES,0,d.line.length/3); } + } + drawDotsGL(ox,oy); + updateBaseStatus(); +} + +// The feature dots, as ONE gl.POINTS draw per tile instead of one DOM div per +// feature. This is the measured lag fix: 57,690 divs -> 1 buffer + 1 draw. +// Hit-testing moves to a nearest-point search over the same Float32Array +// (`pickFeature`), so a click still resolves to an idx without any DOM node. +let dotBuf=null; +function drawDotsGL(ox,oy){ + if(!showFeatures) return; + if(!dotBuf) dotBuf=gl.createBuffer(); + gl.uniform1i(glLoc.uMode,2); + gl.uniform1f(glLoc.uPointPx,6*(window.devicePixelRatio||1)); + for(const {tx,ty,wx} of visibleGrid()){ + ensureFeatures(z,wx,ty); + const d=featureCache.get(tileKey(z,wx,ty)); + if(!(d && typeof d==='object')) continue; + if(!d.pts){ + const a=new Float32Array(d.features.length*3); + const offset=tx-wx; + d.features.forEach((f,i)=>{ a[3*i]=(lon2x(f.lon,z)+offset)*256 - tx*256; + a[3*i+1]=lat2y(f.lat,z)*256 - ty*256; a[3*i+2]=0; }); + d.pts=a; + } + if(!d.pts.length) continue; + gl.bindBuffer(gl.ARRAY_BUFFER,dotBuf); + gl.bufferData(gl.ARRAY_BUFFER,d.pts,gl.DYNAMIC_DRAW); + gl.enableVertexAttribArray(glLoc.aPos); + gl.vertexAttribPointer(glLoc.aPos,2,gl.FLOAT,false,12,0); + gl.enableVertexAttribArray(glLoc.aCls); + gl.vertexAttribPointer(glLoc.aCls,1,gl.FLOAT,false,12,8); + gl.uniform2f(glLoc.uTrans, tx*256+ox, ty*256+oy); + gl.drawArrays(gl.POINTS,0,d.pts.length/3); + } + if(glSel){ // the selected dot, drawn lit + gl.uniform1i(glLoc.uMode,3); + gl.uniform1f(glLoc.uPointPx,10*(window.devicePixelRatio||1)); + gl.bindBuffer(gl.ARRAY_BUFFER,dotBuf); + gl.bufferData(gl.ARRAY_BUFFER,new Float32Array([glSel.x,glSel.y,0]),gl.DYNAMIC_DRAW); + gl.enableVertexAttribArray(glLoc.aPos); + gl.vertexAttribPointer(glLoc.aPos,2,gl.FLOAT,false,12,0); + gl.uniform2f(glLoc.uTrans, ox, oy); + gl.drawArrays(gl.POINTS,0,1); + } +} + +// Click -> nearest feature, over the SAME arrays the GPU drew. Replaces the +// per-div click target: screen px -> world px, then a linear scan of visible +// tiles within a 6px radius (tiles are small and the scan is per-click, not +// per-frame). +function pickFeature(px,py){ + if(!showFeatures) return null; + const w=map.clientWidth, h=map.clientHeight; + const wxp=cx*256+(px-w/2), wyp=cy*256+(py-h/2); + let best=null, bestD=36; // 6px radius, squared + for(const {tx,ty,wx} of visibleGrid()){ + const d=featureCache.get(tileKey(z,wx,ty)); + if(!(d && typeof d==='object' && d.pts)) continue; + for(let i=0;i{ e.preventDefault(); zoom(e.deltaY<0?1:-1); },{ // locate is "where am I", this is "what is THAT". async function showFeature(idx, el){ const box=document.getElementById('feature'); + // DOM selection only exists on the Canvas2D path; under GL the lit dot is + // `glSel`, drawn by drawDotsGL. document.querySelectorAll('.pt.sel').forEach(n=>n.classList.remove('sel')); if(el) el.classList.add('sel'); box.innerHTML='

resolving…

'; @@ -626,11 +921,14 @@ async function showFeature(idx, el){ map.addEventListener('click',async e=>{ if(moved){ moved=false; return; } // this "click" ended a pan — ignore it - if(e.target.classList && e.target.classList.contains('pt')){ - showFeature(e.target.dataset.idx, e.target); - } const r=map.getBoundingClientRect(); const px=e.clientX-r.left, py=e.clientY-r.top; + if(gl){ + const hit=pickFeature(px,py); // nearest point, no DOM target to test + if(hit){ glSel={x:hit.x,y:hit.y}; scheduleBaseDraw(); showFeature(hit.idx,null); } + } else if(e.target.classList && e.target.classList.contains('pt')){ + showFeature(e.target.dataset.idx, e.target); + } const w=map.clientWidth, h=map.clientHeight; const n=Math.pow(2,z); // wrap the horizontal tile index into [0,2^z) so clicks on a repeated world @@ -657,6 +955,17 @@ map.addEventListener('click',async e=>{ }catch(err){ document.getElementById('src').textContent='locate failed: '+err; } }); +// Hardware acceleration is the default and the fallback is automatic: if +// WebGL2 is unavailable (or the context is lost / shaders fail to compile) the +// Canvas2D path above runs unchanged. `initGL` is the ONLY switch — every +// draw/pick site branches on `gl`. +try{ if(!initGL()) console.info('osm: WebGL2 unavailable — Canvas2D fallback'); } +catch(err){ gl=null; console.warn('osm: WebGL2 init failed, using Canvas2D —',err); } +if(gl) glCanvas.addEventListener('webglcontextlost',e=>{ + // A lost context must not leave a blank map: drop to Canvas2D and rebuild. + e.preventDefault(); gl=null; geomCache.clear(); geomZoom=null; render(); +}); + addEventListener('resize',render); render(); From 730642cca61a370b55229be85300ae4846f22daa Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 12 Aug 2026 12:28:35 +0000 Subject: [PATCH 2/2] osm cockpit: load the hydrated bake into lance-graph as a Lance dataset MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Converts the hydrated .soa slab into a Lance dataset on the same persistent volume, following medcare-rs's own hydrate-into-Lance shape (bake_hydrate::hydrate_crystal's warm-skip gate; LanceStore's volume-first-not-S3-backed default) and lance-graph's soa_to_lance example recipe: zero-copy Buffer::from_vec import, the 512-byte SoA contract (NODE_ROW_STRIDE / ENVELOPE_LAYOUT_VERSION, imported from lance_graph_contract rather than restated) persisted once as Arrow schema metadata. Runs at boot right after osm_slab_hydrate::ensure_slab_local() lands the verified .soa/.books/.chains bake, converting into .lance in the same directory. Best-effort: a conversion failure never blocks boot or takes the map down, since osm_features::open_slab still mmaps the raw .soa file unchanged either way. Scope of this change: establishes Lance as the volume's canonical persisted form. It does NOT yet rewire the tile-serving hot path onto the Lance-owned bytes — that's the soa_to_lance "P-CACHE-4" fragment-mmap trick, a diagnostic probe rather than a stable API, and a separately scoped follow-up. classid recorded as the canon's own dormant-default 0x00000000 (see lance-graph PR #902) rather than an invented mint — the OSM bake does not yet participate in OGAR's classid/IdentityQuad federation model. lance/arrow declared as direct deps (both already locked transitively via lance-graph -> lance -> arrow at these exact versions; Cargo.lock diff is 3 lines, no new resolution) — the same move lance-graph PR #931 made for object_store's aws feature. Verified: cargo check -p cockpit-server clean; cargo clippy -p cockpit-server --bins produces zero warnings in the new code (all existing warnings are pre-existing, unrelated to this change). Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01NMeiLmtDKhomJNSo2ecbJw --- Cargo.lock | 3 + crates/cockpit-server/Cargo.toml | 12 ++ crates/cockpit-server/src/main.rs | 15 ++ crates/cockpit-server/src/osm_lance.rs | 274 +++++++++++++++++++++++++ 4 files changed, 304 insertions(+) create mode 100644 crates/cockpit-server/src/osm_lance.rs diff --git a/Cargo.lock b/Cargo.lock index 5f525a9d16..a36795ee00 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1780,6 +1780,7 @@ name = "cockpit-server" version = "0.1.0" dependencies = [ "aiwar-ingest", + "arrow", "askama", "async-stream", "axum 0.7.9", @@ -1790,6 +1791,7 @@ dependencies = [ "futures-core", "hex", "include_dir", + "lance", "lance-graph", "lance-graph-contract", "lance-graph-ogar", @@ -1805,6 +1807,7 @@ dependencies = [ "serde_json", "serde_v8", "sha2 0.10.9", + "tempfile", "tokio", "tower-http 0.5.2", "tracing", diff --git a/crates/cockpit-server/Cargo.toml b/crates/cockpit-server/Cargo.toml index c8dd47238e..046a652d43 100644 --- a/crates/cockpit-server/Cargo.toml +++ b/crates/cockpit-server/Cargo.toml @@ -78,6 +78,14 @@ object_store = { version = "0.13.2", features = ["aws"] } sha2 = "0.10" hex = "0.4" futures = "0.3" +# `osm_lance`: convert the hydrated `.soa` slab into a Lance dataset on the +# same persistent volume (the `soa_to_lance` recipe, see that module's doc). +# Both are ALREADY locked transitively at these exact versions via +# `lance-graph` -> `lance` -> `arrow` — declaring them directly here changes +# no resolved version, it just makes the capability this crate USES a thing +# this crate ASKS FOR. Same move lance-graph PR #931 made for `object_store`. +lance = "=9.0.0" +arrow = { version = "58", features = ["prettyprint"] } # ── Quarto rendering pipeline ─────────────────────────────────────── @@ -118,3 +126,7 @@ planner = ["notebook-query/planner"] orchestrator = ["notebook-query/orchestrator"] # Neo4j Aura fallback — only for live demos when lance-graph is unreachable neo4j-fallback = ["notebook-query/neo4j-fallback"] + +[dev-dependencies] +# `osm_lance`'s warm-skip test needs an isolated directory per run. +tempfile = "3" diff --git a/crates/cockpit-server/src/main.rs b/crates/cockpit-server/src/main.rs index 287edff60b..ef9fa4c9bb 100644 --- a/crates/cockpit-server/src/main.rs +++ b/crates/cockpit-server/src/main.rs @@ -43,6 +43,7 @@ mod osint_classview; mod osm_tiles; mod osm; mod osm_features; +mod osm_lance; mod osm_slab_hydrate; // ── Embed the Vite build at compile time ───────────────────────────────────── @@ -197,6 +198,20 @@ async fn main() { if let Some(path) = osm_slab_hydrate::ensure_slab_local().await { // SAFETY: single-threaded startup, before any task or listener exists. unsafe { std::env::set_var("OSM_SLAB_PATH", &path) }; + + // Load the map into lance-graph: convert the hydrated `.soa` slab + // into a Lance dataset on the same persistent volume (medcare-rs's + // own hydrate-into-Lance shape; see `osm_lance`'s module doc for the + // full recipe and why it stays local rather than S3-backed). + // Best-effort and non-blocking to correctness: `osm_features:: + // open_slab` still mmaps the raw `.soa` file regardless of whether + // this succeeds, so a conversion failure never takes the map down. + if osm_lance::ensure_lance_local(&path).await.is_none() { + tracing::warn!( + "osm lance: conversion to a Lance dataset failed or was skipped; \ + the raw .soa slab still serves the map unaffected" + ); + } } let (tx, _rx) = broadcast::channel::(256); diff --git a/crates/cockpit-server/src/osm_lance.rs b/crates/cockpit-server/src/osm_lance.rs new file mode 100644 index 0000000000..8fe0644865 --- /dev/null +++ b/crates/cockpit-server/src/osm_lance.rs @@ -0,0 +1,274 @@ +//! Convert the hydrated OSM `.soa` slab into a Lance dataset on the same +//! persistent volume — the same recipe `lance-graph`'s `soa_to_lance` +//! example established and proved (zero-copy import, the 512-byte SoA +//! contract persisted once as Arrow schema metadata, the P-CACHE-4 +//! verbatim-row-column guarantee that lets a synchronous mmap keep serving +//! the bytes unchanged). Duplicated here rather than called as a library +//! function: `soa_to_lance` is currently an example BINARY, not a lib +//! export, and this module needs the same recipe from inside an +//! already-running server's boot sequence, not a subprocess. The two +//! contract constants it depends on (`NODE_ROW_STRIDE`, +//! `ENVELOPE_LAYOUT_VERSION`) are imported from `lance_graph_contract`, +//! never restated, so the two copies cannot drift on the CONTRACT — only on +//! plumbing around it. +//! +//! # Why the write lands on volume01, never S3 +//! +//! [`crate::osm_slab_hydrate`] already resolved the persistent volume +//! (`OSM_SLAB_CACHE_DIR` / `RAILWAY_VOL` — Railway's own default mount is +//! `/volume01`) to land the raw `.soa`/`.books`/`.chains` bake. The `.lance` +//! dataset is written into that SAME directory, for the reason +//! `medcare-rs::medcare_db::LanceStore::open`'s own doc gives for keeping +//! its store local by default: at one replica, an S3-backed Lance dataset +//! turns every read into a network round-trip and gives up mmap/page-cache +//! zero-copy. S3 stays the HYDRATION source; it is never the serving +//! location — this module never targets an `s3://` uri. +//! +//! # The request-time read path does not change in this pass +//! +//! [`crate::osm_features::open_slab`] keeps mmapping `OSM_SLAB_PATH` +//! synchronously, unchanged. This module does not repoint it. `soa_to_lance` +//! (lance-graph) already proves the mechanism that WOULD let it: at the +//! 512-byte `NODE_ROW_STRIDE`, Lance's `is_narrow` mini-block cutoff (256 +//! bytes) is never crossed, so the row column lands full-zip, uncompressed, +//! and byte-verbatim in the dataset's own data file — meaning +//! `RowSlab::new(bytes)` would parse Lance-owned bytes exactly as it parses +//! the raw `.soa` file today. Reaching into a Lance fragment's on-disk +//! layout to find that byte run is a diagnostic probe in `soa_to_lance` +//! (its own "P-CACHE-4" section), not a stable production API — wiring the +//! hot tile-serving path onto it is a distinct, separately-scoped follow-up, +//! not bundled into establishing Lance as the volume's canonical persisted +//! form. +//! +//! # What "loaded into lance-graph" means here +//! +//! The bake does not yet mint an OGAR classid space for OSM node rows (see +//! lance-graph PR #902's federation-model gap, which explicitly leaves this +//! open for every bake outside its own). [`OSM_BAKE_CLASSID`] is therefore +//! the canon's OWN documented bootstrap value — lance-graph `CLAUDE.md`'s +//! zero-fallback ladder: *"classid == 0x0000_0000 → default class, no +//! prefix routing (dormant)"* — never an invented placeholder. Minting a +//! real classid for OSM rows is future work, not something this module +//! should improvise. + +use std::collections::HashMap; +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +use arrow::array::FixedSizeBinaryArray; +use arrow::datatypes::{DataType, Field, Schema}; +use arrow::record_batch::RecordBatch; +use lance::dataset::{Dataset, WriteMode, WriteParams}; +use lance_graph_contract::canonical_node::NODE_ROW_STRIDE; +use lance_graph_contract::soa_envelope::ENVELOPE_LAYOUT_VERSION; + +/// Schema-metadata keys — identical to `soa_to_lance`'s, namespaced `soa:` +/// so they never collide with Lance's own (`lance-encoding:*`) or Arrow's. +const K_LAYOUT: &str = "soa:envelope_layout_version"; +const K_STRIDE: &str = "soa:row_stride"; +const K_CARVING: &str = "soa:row_carving"; +const K_ENDIAN: &str = "soa:endianness"; +const K_CLASSID: &str = "soa:classid"; +const K_DIGEST: &str = "soa:slab_digest"; +const K_SOURCE: &str = "soa:source"; + +const ROW_COLUMN: &str = "row"; + +/// See the module doc's "What 'loaded into lance-graph' means here" — the +/// canon's own dormant-default classid, not a mint. +const OSM_BAKE_CLASSID: &str = "00000000"; + +/// Ensure a Lance dataset exists at `.lance`, converting from the +/// verified `.soa` slab (already hydrated onto the persistent volume by +/// [`crate::osm_slab_hydrate::ensure_slab_local`]) if absent or stale. +/// +/// Never panics; a failure returns `None` and the caller keeps mmapping the +/// raw `.soa` file exactly as it did before this module existed — the +/// Lance conversion is a best-effort addition, not a new hard dependency +/// for serving the map. +/// +/// Mirrors `medcare-rs::bake_hydrate::hydrate_crystal`'s warm-skip gate: a +/// dataset whose row count AND slab digest already match this slab costs +/// one `count_rows` + one metadata read, never a rewrite of gigabytes. +pub async fn ensure_lance_local(slab_path: &Path) -> Option { + let dest = slab_path.with_extension("lance"); + + let bytes = match std::fs::read(slab_path) { + Ok(b) => b, + Err(e) => { + tracing::error!(path = %slab_path.display(), error = %e, "osm lance: cannot read slab"); + return None; + } + }; + if bytes.is_empty() || !bytes.len().is_multiple_of(NODE_ROW_STRIDE) { + tracing::error!( + path = %slab_path.display(), len = bytes.len(), + "osm lance: slab is not a whole number of {NODE_ROW_STRIDE}-byte rows; refusing" + ); + return None; + } + let rows = bytes.len() / NODE_ROW_STRIDE; + let digest_hex = format!("{:016x}", osm_soa_bake::codebook::hash_slab(&bytes)); + + if let Some(warm) = reopen_if_warm(&dest, rows, &digest_hex).await { + tracing::info!( + path = %dest.display(), rows, + "osm lance: warm — dataset already matches this slab's row count and digest" + ); + return Some(warm); + } + + tracing::info!( + path = %dest.display(), rows, + "osm lance: converting the .soa slab into a Lance dataset (zero-copy import)" + ); + write_lance(&dest, bytes, rows, &digest_hex, slab_path).await +} + +/// `Some(dest)` iff a dataset already exists at `dest` with exactly `rows` +/// rows and a `soa:slab_digest` header matching `digest_hex` — the ONLY two +/// facts that must hold before skipping the write. Any other outcome +/// (missing, unreadable, wrong row count, wrong digest) returns `None` so +/// the caller reconverts rather than serve stale or half-written data. +async fn reopen_if_warm(dest: &Path, rows: usize, digest_hex: &str) -> Option { + if !dest.exists() { + return None; + } + let uri = dest.to_string_lossy().into_owned(); + let ds = match lance::dataset::builder::DatasetBuilder::from_uri(&uri) + .load() + .await + { + Ok(ds) => ds, + Err(e) => { + tracing::warn!(path = %dest.display(), error = %e, "osm lance: existing dataset unreadable; reconverting"); + return None; + } + }; + let got_rows = match ds.count_rows(None).await { + Ok(n) => n, + Err(e) => { + tracing::warn!(path = %dest.display(), error = %e, "osm lance: count_rows failed; reconverting"); + return None; + } + }; + let got_digest = ds.schema().metadata.get(K_DIGEST).cloned(); + if got_rows == rows && got_digest.as_deref() == Some(digest_hex) { + Some(dest.to_path_buf()) + } else { + tracing::warn!( + path = %dest.display(), got_rows, want_rows = rows, + got_digest = ?got_digest, want_digest = digest_hex, + "osm lance: existing dataset is stale (row count or slab digest mismatch); reconverting" + ); + None + } +} + +/// The write itself — `lance-graph`'s `soa_to_lance` example recipe, +/// local-uri only (this module never targets `s3://`; see the module doc). +/// `WriteMode::Overwrite` rather than `Create`: [`reopen_if_warm`] already +/// handled the "already correct" case, so by the time this runs `dest` is +/// either absent or holds a STALE dataset that must be replaced, not +/// appended onto. +async fn write_lance( + dest: &Path, + bytes: Vec, + rows: usize, + digest_hex: &str, + slab_path: &Path, +) -> Option { + let field = Field::new( + ROW_COLUMN, + DataType::FixedSizeBinary(NODE_ROW_STRIDE as i32), + false, + ) + .with_metadata(HashMap::from([( + // Inert at this stride (512 ≥ the 256-byte mini-block cutoff ⇒ + // full-zip ⇒ never consulted) — kept as a defensive pin, matching + // `soa_to_lance`'s own module doc on why this is a backstop and not + // the actual mechanism that keeps the column verbatim. + "lance-encoding:compression".to_string(), + "none".to_string(), + )])); + let schema_meta = HashMap::from([ + (K_LAYOUT.to_string(), ENVELOPE_LAYOUT_VERSION.to_string()), + (K_STRIDE.to_string(), NODE_ROW_STRIDE.to_string()), + ( + K_CARVING.to_string(), + format!("key:0..16|edges:16..32|value:32..{NODE_ROW_STRIDE}"), + ), + (K_ENDIAN.to_string(), "le".to_string()), + (K_CLASSID.to_string(), OSM_BAKE_CLASSID.to_string()), + (K_DIGEST.to_string(), digest_hex.to_string()), + ( + K_SOURCE.to_string(), + format!("{} rows={rows}", slab_path.display()), + ), + ]); + let schema = Arc::new(Schema::new_with_metadata(vec![field], schema_meta)); + + // Zero-copy import: `Buffer::from_vec` ADOPTS this allocation rather + // than copying it — asserted by pointer identity, not merely trusted + // (same discipline `soa_to_lance` uses; a 1+ GiB slab makes a silent + // copy here a real cost, not a rounding error). + let before = bytes.as_ptr(); + let array = FixedSizeBinaryArray::new( + NODE_ROW_STRIDE as i32, + arrow::buffer::Buffer::from_vec(bytes), + None, + ); + debug_assert_eq!( + array.value_data().as_ptr(), + before, + "Buffer::from_vec must ADOPT the allocation — a moved address means a copy crept in" + ); + let batch = match RecordBatch::try_new(schema.clone(), vec![Arc::new(array)]) { + Ok(b) => b, + Err(e) => { + tracing::error!(error = %e, "osm lance: building the record batch failed"); + return None; + } + }; + + let uri = dest.to_string_lossy().into_owned(); + let params = WriteParams { + mode: WriteMode::Overwrite, + ..Default::default() + }; + if let Err(e) = Dataset::write( + arrow::array::RecordBatchIterator::new(vec![Ok(batch)].into_iter(), schema), + &uri, + Some(params), + ) + .await + { + tracing::error!(path = %dest.display(), error = %e, "osm lance: write failed"); + return None; + } + + tracing::info!(path = %dest.display(), rows, "osm lance: converted"); + Some(dest.to_path_buf()) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The classid constant must stay the canon's dormant-default value — + /// this is a documentation-of-intent test, not a behavioural one: a + /// change here should be a deliberate classid mint, not a typo. + #[test] + fn the_classid_is_the_canonical_dormant_default() { + assert_eq!(OSM_BAKE_CLASSID, "00000000"); + } + + /// `reopen_if_warm` must decline when nothing is there yet — the base + /// case a cold volume hits on every first boot. + #[tokio::test] + async fn reopen_if_warm_declines_a_missing_destination() { + let dir = tempfile::tempdir().expect("tempdir"); + let dest = dir.path().join("nope.lance"); + assert!(reopen_if_warm(&dest, 10, "deadbeef").await.is_none()); + } +}