From 43d3552f5791f52351b1fa3a61029998c220e23e Mon Sep 17 00:00:00 2001 From: G Date: Mon, 3 Aug 2026 10:18:21 +0530 Subject: [PATCH] deleting older java based custom code for codec Signed-off-by: G --- .../DocValuesRandomAccessBenchmark.java | 272 --- .../benchmark/DocValuesSkipperBenchmark.java | 229 --- .../parquet/ParquetDataFormatPlugin.java | 26 +- .../opensearch/parquet/ParquetSettings.java | 48 - .../parquet/bridge/ParquetColumnReader.java | 543 ------ .../opensearch/parquet/bridge/RustBridge.java | 346 ---- .../ParquetDocValuesDirectoryReader.java | 54 +- .../codec/ParquetDocValuesLeafReader.java | 15 +- .../codec/ParquetDocValuesProducer.java | 101 +- .../codec/RowIdRemappingDocValues.java | 32 +- .../parquet/codec/SharedProducerRegistry.java | 7 +- .../parquet/codec/UninvertedOrdinals.java | 13 +- .../codec/UninvertedOrdinalsCache.java | 3 +- .../parquet/codec/cache/CacheStats.java | 124 -- .../codec/cache/QueryParquetStats.java | 157 -- .../parquet/codec/cache/RowIdStats.java | 60 - .../ParquetLiquidCacheClearRestAction.java | 65 - .../src/main/rust/src/ffm.rs | 1669 +---------------- .../src/main/rust/src/lib.rs | 1 - .../src/main/rust/src/liquid_page_cache.rs | 405 ---- .../parquet/ParquetSettingsTests.java | 11 - .../bridge/ParquetColumnReaderTests.java | 270 --- .../codec/DataFusionDocValuesTests.java | 8 - 23 files changed, 38 insertions(+), 4421 deletions(-) delete mode 100644 sandbox/plugins/parquet-data-format/benchmarks/src/main/java/org/opensearch/parquet/benchmark/DocValuesRandomAccessBenchmark.java delete mode 100644 sandbox/plugins/parquet-data-format/benchmarks/src/main/java/org/opensearch/parquet/benchmark/DocValuesSkipperBenchmark.java delete mode 100644 sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/bridge/ParquetColumnReader.java delete mode 100644 sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/codec/cache/CacheStats.java delete mode 100644 sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/codec/cache/QueryParquetStats.java delete mode 100644 sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/codec/cache/RowIdStats.java delete mode 100644 sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/rest/ParquetLiquidCacheClearRestAction.java delete mode 100644 sandbox/plugins/parquet-data-format/src/main/rust/src/liquid_page_cache.rs delete mode 100644 sandbox/plugins/parquet-data-format/src/test/java/org/opensearch/parquet/bridge/ParquetColumnReaderTests.java diff --git a/sandbox/plugins/parquet-data-format/benchmarks/src/main/java/org/opensearch/parquet/benchmark/DocValuesRandomAccessBenchmark.java b/sandbox/plugins/parquet-data-format/benchmarks/src/main/java/org/opensearch/parquet/benchmark/DocValuesRandomAccessBenchmark.java deleted file mode 100644 index f09d64c92a890..0000000000000 --- a/sandbox/plugins/parquet-data-format/benchmarks/src/main/java/org/opensearch/parquet/benchmark/DocValuesRandomAccessBenchmark.java +++ /dev/null @@ -1,272 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.parquet.benchmark; - -import org.apache.arrow.c.ArrowArray; -import org.apache.arrow.c.ArrowSchema; -import org.apache.arrow.c.Data; -import org.apache.arrow.memory.BufferAllocator; -import org.apache.arrow.memory.RootAllocator; -import org.apache.arrow.vector.BigIntVector; -import org.apache.arrow.vector.IntVector; -import org.apache.arrow.vector.VectorSchemaRoot; -import org.apache.arrow.vector.types.pojo.ArrowType; -import org.apache.arrow.vector.types.pojo.Field; -import org.apache.arrow.vector.types.pojo.FieldType; -import org.apache.arrow.vector.types.pojo.Schema; -import org.opensearch.nativebridge.spi.ArrowExport; -import org.opensearch.parquet.bridge.NativeParquetWriter; -import org.opensearch.parquet.bridge.ParquetColumnReader; -import org.opensearch.parquet.bridge.ParquetSortConfig; -import org.opensearch.parquet.bridge.RustBridge; -import org.opensearch.parquet.codec.ParquetPhysicalType; -import org.opensearch.parquet.codec.cache.BufferPool; -import org.opensearch.parquet.codec.iter.ParquetNumericDocValues; -import org.openjdk.jmh.annotations.Benchmark; -import org.openjdk.jmh.annotations.BenchmarkMode; -import org.openjdk.jmh.annotations.Fork; -import org.openjdk.jmh.annotations.Level; -import org.openjdk.jmh.annotations.Measurement; -import org.openjdk.jmh.annotations.Mode; -import org.openjdk.jmh.annotations.OperationsPerInvocation; -import org.openjdk.jmh.annotations.OutputTimeUnit; -import org.openjdk.jmh.annotations.Param; -import org.openjdk.jmh.annotations.Scope; -import org.openjdk.jmh.annotations.Setup; -import org.openjdk.jmh.annotations.State; -import org.openjdk.jmh.annotations.TearDown; -import org.openjdk.jmh.annotations.Warmup; -import org.openjdk.jmh.infra.Blackhole; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.List; -import java.util.Random; -import java.util.concurrent.TimeUnit; - -/** - * JMH benchmark for {@link ParquetNumericDocValues#advanceExact} under different access - * patterns, isolating the native page-decode path (`parquet_decode_page_at_row`). - * - *

Access patterns: - *

- * - *

Parameters cover both presence-pack paths (required-column memset vs branchless - * def-level compare) and both value-expand paths (all-present tight widening loop vs - * bitset pop-and-scatter): {@code nullFraction=0.0} exercises the former of each pair, - * {@code nullFraction=0.3} the latter. {@code columnType} covers the i64 passthrough - * (INT64) and the SIMD-widening (INT32) conversions. - * - *

Run with: - * 

- * ./gradlew -Dsandbox.enabled=true :sandbox:plugins:parquet-data-format:benchmarks:run \
- *   --args 'DocValuesRandomAccessBenchmark'
- *
- */ -@Fork(1) -@Warmup(iterations = 3, time = 5, timeUnit = TimeUnit.SECONDS) -@Measurement(iterations = 5, time = 5, timeUnit = TimeUnit.SECONDS) -@BenchmarkMode(Mode.AverageTime) -@OutputTimeUnit(TimeUnit.MICROSECONDS) -@State(Scope.Benchmark) -public class DocValuesRandomAccessBenchmark { - - /** Total rows in the benchmark file. 1M rows / default 20k page_row_limit ≈ 50 pages. */ - @Param({ "1000000" }) - private int rows; - - /** Fraction of null rows. 0.0 → required-column fast paths; 0.3 → nullable scatter paths. */ - @Param({ "0.0", "0.3" }) - private double nullFraction; - - /** INT64 = raw i64 passthrough; INT32 = widening conversion (vectorizable). */ - @Param({ "INT64", "INT32" }) - private String columnType; - - /** Number of advanceExact calls per invocation (fixed target array, fixed seed). */ - private static final int TARGETS = 100_000; - /** Rows per Arrow batch handed to the native writer during setup. */ - private static final int BATCH_ROWS = 100_000; - - private BufferAllocator allocator; - private Path file; - private BufferPool bufferPool; - private ParquetColumnReader reader; - private ParquetNumericDocValues docValues; - private int[] randomTargets; - private int[] pingPongTargets; - - @Setup(Level.Trial) - public void setupTrial() throws Exception { - RustBridge.initLogger(); - allocator = new RootAllocator(); - file = Files.createTempDirectory("dv-bench").resolve("bench.parquet"); - writeFile(); - - bufferPool = new BufferPool(); - String column = columnType.equals("INT64") ? "val_i64" : "val_i32"; - ParquetPhysicalType physical = columnType.equals("INT64") ? ParquetPhysicalType.INT64 : ParquetPhysicalType.INT32; - reader = ParquetColumnReader.open(file, column, physical, false, bufferPool); - docValues = new ParquetNumericDocValues(reader, rows); - - // Fixed seed so every fork/param combination replays the identical target sequence. - Random random = new Random(42); - randomTargets = new int[TARGETS]; - for (int i = 0; i < TARGETS; i++) { - randomTargets[i] = random.nextInt(rows); - } - // Alternate between the first and last page so every call is a page miss. - pingPongTargets = new int[TARGETS]; - for (int i = 0; i < TARGETS; i++) { - pingPongTargets[i] = (i % 2 == 0) ? (i / 2) % 1000 : rows - 1 - ((i / 2) % 1000); - } - } - - @TearDown(Level.Trial) - public void tearDownTrial() throws Exception { - if (reader != null) { - reader.close(); - } - if (bufferPool != null) { - bufferPool.close(); - } - if (allocator != null) { - allocator.close(); - } - } - - /** - * Baseline: ascending scan of the whole column. L1/L2 hit rate ≈ (1 - pages/rows); - * decode cost is amortized over ~20k rows per FFM call. - */ - @Benchmark - @OperationsPerInvocation(1_000_000) - public void sequentialScan(Blackhole bh) throws IOException { - for (int d = 0; d < rows; d++) { - if (docValues.advanceExact(d)) { - bh.consume(docValues.longValue()); - } - } - } - - /** - * The optimized case: uniform random targets, ~98% resident-page misses, so throughput - * is dominated by the native page decode (scratch reuse + branchless presence pack + - * direct-to-outbuf expand). - */ - @Benchmark - @OperationsPerInvocation(TARGETS) - public void randomAccess(Blackhole bh) throws IOException { - for (int t : randomTargets) { - if (docValues.advanceExact(t)) { - bh.consume(docValues.longValue()); - } - } - } - - /** Worst case: every advanceExact evicts the resident page — a pure cold-decode measurement. */ - @Benchmark - @OperationsPerInvocation(TARGETS) - public void pageMissPingPong(Blackhole bh) throws IOException { - for (int t : pingPongTargets) { - if (docValues.advanceExact(t)) { - bh.consume(docValues.longValue()); - } - } - } - - /** - * Per-query lifecycle cost: open a fresh column reader (the once-per-field-per-query step a - * real search pays for every producer), read a handful of scattered docs, close. Dominated - * by {@link ParquetColumnReader#open}'s metadata work — schema resolution + page-layout - * (OffsetIndex/ColumnIndex) computation + ColumnPageIndex marshal — which the node-level - * file-metadata cache converts from a per-open parse into an Arc-clone lookup. This is the - * benchmark that shows the "dvm-equivalent" win; the decode benchmarks above open once per - * trial and cannot see it. - */ - @Benchmark - public long openReadClose(Blackhole bh) throws IOException { - String column = columnType.equals("INT64") ? "val_i64" : "val_i32"; - ParquetPhysicalType physical = columnType.equals("INT64") ? ParquetPhysicalType.INT64 : ParquetPhysicalType.INT32; - try (BufferPool pool = new BufferPool(); ParquetColumnReader r = ParquetColumnReader.open(file, column, physical, false, pool)) { - ParquetNumericDocValues dv = new ParquetNumericDocValues(r, rows); - long sum = 0; - // Touch a few scattered docs so the open isn't dead-code-eliminated and the reader - // exercises a realistic first-access pattern (a couple of page decodes). - for (int i = 0; i < 8; i++) { - int t = randomTargets[i * (TARGETS / 8)]; - if (dv.advanceExact(t)) { - sum += dv.longValue(); - } - } - bh.consume(sum); - return sum; - } - } - - // ── data generation ── - - private void writeFile() throws Exception { - Schema schema = new Schema( - List.of( - new Field("val_i64", FieldType.nullable(new ArrowType.Int(64, true)), null), - new Field("val_i32", FieldType.nullable(new ArrowType.Int(32, true)), null) - ) - ); - - NativeParquetWriter writer = new NativeParquetWriter(file.toString()); - try (ArrowExport schemaExport = exportSchema(schema)) { - writer.initialize("dv-bench-index", schemaExport.getSchemaAddress(), ParquetSortConfig.empty(), 0L); - } - - // Deterministic values and null placement so all runs measure identical data. - Random random = new Random(7); - for (int start = 0; start < rows; start += BATCH_ROWS) { - int batch = Math.min(BATCH_ROWS, rows - start); - try (VectorSchemaRoot root = VectorSchemaRoot.create(schema, allocator)) { - BigIntVector i64Vec = (BigIntVector) root.getVector("val_i64"); - IntVector i32Vec = (IntVector) root.getVector("val_i32"); - for (int i = 0; i < batch; i++) { - if (nullFraction > 0 && random.nextDouble() < nullFraction) { - i64Vec.setNull(i); - i32Vec.setNull(i); - } else { - i64Vec.setSafe(i, random.nextLong()); - i32Vec.setSafe(i, random.nextInt()); - } - } - root.setRowCount(batch); - - ArrowArray array = ArrowArray.allocateNew(allocator); - ArrowSchema arrowSchema = ArrowSchema.allocateNew(allocator); - Data.exportVectorSchemaRoot(allocator, root, null, array, arrowSchema); - try (ArrowExport export = new ArrowExport(array, arrowSchema)) { - writer.write(export.getArrayAddress(), export.getSchemaAddress()); - } - } - } - writer.flush(); - } - - private ArrowExport exportSchema(Schema schema) { - ArrowSchema arrowSchema = ArrowSchema.allocateNew(allocator); - Data.exportSchema(allocator, schema, null, arrowSchema); - return new ArrowExport(null, arrowSchema); - } -} diff --git a/sandbox/plugins/parquet-data-format/benchmarks/src/main/java/org/opensearch/parquet/benchmark/DocValuesSkipperBenchmark.java b/sandbox/plugins/parquet-data-format/benchmarks/src/main/java/org/opensearch/parquet/benchmark/DocValuesSkipperBenchmark.java deleted file mode 100644 index eddafb6a3686a..0000000000000 --- a/sandbox/plugins/parquet-data-format/benchmarks/src/main/java/org/opensearch/parquet/benchmark/DocValuesSkipperBenchmark.java +++ /dev/null @@ -1,229 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.parquet.benchmark; - -import org.apache.arrow.c.ArrowArray; -import org.apache.arrow.c.ArrowSchema; -import org.apache.arrow.c.Data; -import org.apache.arrow.memory.BufferAllocator; -import org.apache.arrow.memory.RootAllocator; -import org.apache.arrow.vector.BigIntVector; -import org.apache.arrow.vector.VectorSchemaRoot; -import org.apache.arrow.vector.types.pojo.ArrowType; -import org.apache.arrow.vector.types.pojo.Field; -import org.apache.arrow.vector.types.pojo.FieldType; -import org.apache.arrow.vector.types.pojo.Schema; -import org.apache.lucene.search.DocIdSetIterator; -import org.opensearch.nativebridge.spi.ArrowExport; -import org.opensearch.parquet.bridge.NativeParquetWriter; -import org.opensearch.parquet.bridge.ParquetColumnReader; -import org.opensearch.parquet.bridge.ParquetSortConfig; -import org.opensearch.parquet.bridge.RustBridge; -import org.opensearch.parquet.codec.ParquetDocValuesSkipper; -import org.opensearch.parquet.codec.ParquetPhysicalType; -import org.opensearch.parquet.codec.cache.BufferPool; -import org.opensearch.parquet.codec.iter.ParquetNumericDocValues; -import org.openjdk.jmh.annotations.Benchmark; -import org.openjdk.jmh.annotations.BenchmarkMode; -import org.openjdk.jmh.annotations.Fork; -import org.openjdk.jmh.annotations.Level; -import org.openjdk.jmh.annotations.Measurement; -import org.openjdk.jmh.annotations.Mode; -import org.openjdk.jmh.annotations.OutputTimeUnit; -import org.openjdk.jmh.annotations.Param; -import org.openjdk.jmh.annotations.Scope; -import org.openjdk.jmh.annotations.Setup; -import org.openjdk.jmh.annotations.State; -import org.openjdk.jmh.annotations.TearDown; -import org.openjdk.jmh.annotations.Warmup; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.List; -import java.util.concurrent.TimeUnit; - -/** - * JMH benchmark demonstrating the efficacy of {@link ParquetDocValuesSkipper}: a counting - * range-scan over a value-sorted column (the shape of a range filter or date-histogram over a - * timestamp-sorted log index, where Parquet page min/max are tight and disjoint), implemented - * two ways: - * - * - * - *

Both methods compute the identical count (asserted at setup), so the score difference is - * pure page-exclusion. {@code selectivity} controls the fraction of rows (and, with sorted - * data, pages) the range covers: at 0.01 the skipper visits ~1% of pages; at 1.0 it visits all - * of them and measures pure skipper overhead over the full scan. - * - *

Run with: - * 

- * ./gradlew -Dsandbox.enabled=true :sandbox:plugins:parquet-data-format:benchmarks:run \
- *   --args 'DocValuesSkipperBenchmark'
- *
- */ -@Fork(1) -@Warmup(iterations = 2, time = 3, timeUnit = TimeUnit.SECONDS) -@Measurement(iterations = 3, time = 3, timeUnit = TimeUnit.SECONDS) -@BenchmarkMode(Mode.AverageTime) -@OutputTimeUnit(TimeUnit.MILLISECONDS) -@State(Scope.Benchmark) -public class DocValuesSkipperBenchmark { - - /** Total rows; 1M rows / default 20k page_row_limit ≈ 50 pages. */ - @Param({ "1000000" }) - private int rows; - - /** - * Fraction of rows the range covers, centered in the value space. With sorted data this is - * also the fraction of pages that intersect the range. - */ - @Param({ "0.01", "0.1", "1.0" }) - private double selectivity; - - private static final int BATCH_ROWS = 100_000; - - private BufferAllocator allocator; - private Path file; - private BufferPool bufferPool; - private ParquetColumnReader reader; - private ParquetNumericDocValues docValues; - private long rangeLo; - private long rangeHi; - private long expectedCount; - - @Setup(Level.Trial) - public void setupTrial() throws Exception { - RustBridge.initLogger(); - allocator = new RootAllocator(); - file = Files.createTempDirectory("skipper-bench").resolve("bench.parquet"); - writeSortedFile(); - - bufferPool = new BufferPool(); - reader = ParquetColumnReader.open(file, "ts", ParquetPhysicalType.INT64, false, bufferPool); - docValues = new ParquetNumericDocValues(reader, rows); - - // Value of row i is i (sorted), so a centered range of `selectivity * rows` values - // matches exactly that many docs. - long span = Math.max(1, (long) (rows * selectivity)); - rangeLo = (rows - span) / 2; - rangeHi = rangeLo + span - 1; - expectedCount = rangeHi - rangeLo + 1; - - // Both strategies must agree before we measure either. - long full = rangeScanFullIteration(); - long skip = rangeScanWithSkipper(); - if (full != expectedCount || skip != expectedCount) { - throw new IllegalStateException("strategy mismatch: full=" + full + " skipper=" + skip + " expected=" + expectedCount); - } - } - - @TearDown(Level.Trial) - public void tearDownTrial() throws Exception { - if (reader != null) { - reader.close(); - } - if (bufferPool != null) { - bufferPool.close(); - } - if (allocator != null) { - allocator.close(); - } - } - - /** Pre-skipper: advanceExact every doc and test its value. Decodes every page. */ - @Benchmark - public long rangeScanFullIteration() throws IOException { - long count = 0; - for (int doc = 0; doc < rows; doc++) { - if (docValues.advanceExact(doc)) { - long v = docValues.longValue(); - if (v >= rangeLo && v <= rangeHi) { - count++; - } - } - } - return count; - } - - /** - * Skipper-driven: {@code advance(rangeLo, rangeHi)} lands on the next page whose - * [min, max] intersects the range; excluded pages cost one metadata comparison each. - * Only intersecting pages are iterated (and hence decoded). - */ - @Benchmark - public long rangeScanWithSkipper() throws IOException { - ParquetDocValuesSkipper skipper = new ParquetDocValuesSkipper(reader.pageIndex(), rows); - long count = 0; - skipper.advance(rangeLo, rangeHi); - while (skipper.minDocID(0) != DocIdSetIterator.NO_MORE_DOCS) { - int end = skipper.maxDocID(0); - for (int doc = Math.max(skipper.minDocID(0), 0); doc <= end; doc++) { - if (docValues.advanceExact(doc)) { - long v = docValues.longValue(); - if (v >= rangeLo && v <= rangeHi) { - count++; - } - } - } - if (end >= rows - 1) { - break; - } - skipper.advance(end + 1); - // Re-apply the range so the skipper jumps over non-intersecting pages. - skipper.advance(rangeLo, rangeHi); - } - return count; - } - - // ── data generation ── - - /** Writes a single INT64 column "ts" with value(row i) == i — sorted, no nulls. */ - private void writeSortedFile() throws Exception { - Schema schema = new Schema(List.of(new Field("ts", FieldType.nullable(new ArrowType.Int(64, true)), null))); - - NativeParquetWriter writer = new NativeParquetWriter(file.toString()); - try (ArrowExport schemaExport = exportSchema(schema)) { - writer.initialize("skipper-bench-index", schemaExport.getSchemaAddress(), ParquetSortConfig.empty(), 0L); - } - - for (int start = 0; start < rows; start += BATCH_ROWS) { - int batch = Math.min(BATCH_ROWS, rows - start); - try (VectorSchemaRoot root = VectorSchemaRoot.create(schema, allocator)) { - BigIntVector tsVec = (BigIntVector) root.getVector("ts"); - for (int i = 0; i < batch; i++) { - tsVec.setSafe(i, start + i); - } - root.setRowCount(batch); - - ArrowArray array = ArrowArray.allocateNew(allocator); - ArrowSchema arrowSchema = ArrowSchema.allocateNew(allocator); - Data.exportVectorSchemaRoot(allocator, root, null, array, arrowSchema); - try (ArrowExport export = new ArrowExport(array, arrowSchema)) { - writer.write(export.getArrayAddress(), export.getSchemaAddress()); - } - } - } - writer.flush(); - } - - private ArrowExport exportSchema(Schema schema) { - ArrowSchema arrowSchema = ArrowSchema.allocateNew(allocator); - Data.exportSchema(allocator, schema, null, arrowSchema); - return new ArrowExport(null, arrowSchema); - } -} diff --git a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/ParquetDataFormatPlugin.java b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/ParquetDataFormatPlugin.java index fe91cf30c4f96..d9381019dd013 100644 --- a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/ParquetDataFormatPlugin.java +++ b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/ParquetDataFormatPlugin.java @@ -44,7 +44,6 @@ import org.opensearch.parquet.engine.ParquetDataFormat; import org.opensearch.parquet.engine.ParquetIndexingEngine; import org.opensearch.parquet.fields.ArrowSchemaBuilder; -import org.opensearch.parquet.rest.ParquetLiquidCacheClearRestAction; import org.opensearch.parquet.stats.ParquetStatsProvider; import org.opensearch.parquet.stats.transport.ParquetNodeStatsActionType; import org.opensearch.parquet.stats.transport.ParquetNodeStatsRestAction; @@ -140,30 +139,12 @@ public Collection createComponents( long mergeMax = ParquetSettings.MERGE_POOL_MAX.get(this.settings); RustBridge.initMemoryPools(writeMax, mergeMax); - // Enable the codec-owned cross-query decoded-page cache (liquid-cache core) when configured. - // Off by default → the DocValues decode path is unchanged. - boolean liquidCacheEnabled = ParquetSettings.LIQUID_CACHE_ENABLED.get(this.settings); - if (liquidCacheEnabled) { - long liquidCacheMaxBytes = ParquetSettings.LIQUID_CACHE_MAX_BYTES.get(this.settings).getBytes(); - // Mount the liquid t4 store under the node's data directory (real disk, always writable — - // never tmpfs), so the cache does not depend on TMPDIR/-Djava.io.tmpdir pointing at a - // suitable filesystem. Falls back to java.io.tmpdir only if no data path is configured. - Path[] dataPaths = nodeEnvironment.nodeDataPaths(); - Path liquidCacheDir = (dataPaths != null && dataPaths.length > 0) - ? dataPaths[0].resolve("parquet_liquid_cache") - : environment.tmpDir(); - RustBridge.liquidCacheSetEnabled(true, liquidCacheMaxBytes, liquidCacheDir.toString()); - } - - ParquetDocValuesProducer.setDecodePath(ParquetSettings.DOCVALUES_DECODE_PATH.get(this.settings)); ParquetDocValuesProducer.setInitialBatchSize(ParquetSettings.DOCVALUES_INITIAL_BATCH_SIZE.get(this.settings)); ParquetDocValuesProducer.setDiagnostics(ParquetSettings.DOCVALUES_DIAGNOSTICS.get(this.settings)); ParquetDocValuesProducer.setDictionaryMaxTerms(ParquetSettings.DOCVALUES_DICTIONARY_MAX_TERMS.get(this.settings)); ParquetDocValuesProducer.setDictionaryCacheBytes(ParquetSettings.DOCVALUES_DICTIONARY_CACHE_BYTES.get(this.settings)); ParquetDocValuesProducer.setUninvertMaxDiskBytes(ParquetSettings.DOCVALUES_UNINVERT_MAX_DISK_BYTES.get(this.settings)); org.opensearch.parquet.codec.UninvertedOrdinalsCache.setOrdsDir(environment.dataFiles()[0].resolve("parquet-ords")); - clusterService.getClusterSettings() - .addSettingsUpdateConsumer(ParquetSettings.DOCVALUES_DECODE_PATH, ParquetDocValuesProducer::setDecodePath); clusterService.getClusterSettings() .addSettingsUpdateConsumer(ParquetSettings.DOCVALUES_INITIAL_BATCH_SIZE, ParquetDocValuesProducer::setInitialBatchSize); clusterService.getClusterSettings() @@ -173,7 +154,10 @@ public Collection createComponents( clusterService.getClusterSettings() .addSettingsUpdateConsumer(ParquetSettings.DOCVALUES_DICTIONARY_CACHE_BYTES, ParquetDocValuesProducer::setDictionaryCacheBytes); clusterService.getClusterSettings() - .addSettingsUpdateConsumer(ParquetSettings.DOCVALUES_UNINVERT_MAX_DISK_BYTES, ParquetDocValuesProducer::setUninvertMaxDiskBytes); + .addSettingsUpdateConsumer( + ParquetSettings.DOCVALUES_UNINVERT_MAX_DISK_BYTES, + ParquetDocValuesProducer::setUninvertMaxDiskBytes + ); // Register virtual pools if allocator is available (arrow-base loaded) if (nativeAllocator != null) { @@ -325,7 +309,7 @@ public List getRestHandlers( IndexNameExpressionResolver indexNameExpressionResolver, Supplier nodesInCluster ) { - return List.of(new ParquetStatsRestAction(), new ParquetNodeStatsRestAction(), new ParquetLiquidCacheClearRestAction()); + return List.of(new ParquetStatsRestAction(), new ParquetNodeStatsRestAction()); } /** diff --git a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/ParquetSettings.java b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/ParquetSettings.java index ceae3808b96af..8866da50e15b2 100644 --- a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/ParquetSettings.java +++ b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/ParquetSettings.java @@ -217,43 +217,6 @@ private ParquetSettings() {} Setting.Property.NodeScope ); - /** - * Enables the codec-owned cross-query decoded-page cache (backed by liquid-cache core). When off - * (the default), the DocValues codec decode path is unchanged and the native cache is never - * consulted. Node-scoped: set in {@code opensearch.yml} or via {@code -E} at launch. - */ - public static final Setting LIQUID_CACHE_ENABLED = Setting.boolSetting( - "parquet.liquid_cache.enabled", - false, - Setting.Property.NodeScope - ); - - /** - * Memory budget for the codec-owned decoded-page cache. A value of 0 leaves the native liquid - * default budget. Only consulted when {@link #LIQUID_CACHE_ENABLED} is true. - */ - public static final Setting LIQUID_CACHE_MAX_BYTES = Setting.byteSizeSetting( - "parquet.liquid_cache.max_bytes", - new ByteSizeValue(0, ByteSizeUnit.BYTES), - Setting.Property.NodeScope - ); - - public static final String DECODE_PATH_CODEC_NATIVE = "codec_native"; - public static final String DECODE_PATH_DATAFUSION = "datafusion"; - public static final Set VALID_DECODE_PATHS = Set.of(DECODE_PATH_CODEC_NATIVE, DECODE_PATH_DATAFUSION); - - /** - * Selects the DocValues decoder. The DataFusion path retains one Arrow reader per accessed - * column and advances it by row using page-level OffsetIndex skips. - */ - public static final Setting DOCVALUES_DECODE_PATH = new Setting<>( - "parquet.docvalues.decode_path", - DECODE_PATH_CODEC_NATIVE, - ParquetSettings::validateDecodePath, - Setting.Property.NodeScope, - Setting.Property.Dynamic - ); - /** Starting Arrow decode window for the adaptive DataFusion DocValues cursor. */ public static final Setting DOCVALUES_INITIAL_BATCH_SIZE = Setting.intSetting( "parquet.docvalues.initial_batch_size", @@ -307,14 +270,6 @@ private ParquetSettings() {} Setting.Property.Dynamic ); - private static String validateDecodePath(String value) { - String normalized = value.toLowerCase(Locale.ROOT); - if (VALID_DECODE_PATHS.contains(normalized) == false) { - throw new IllegalArgumentException("Invalid parquet.docvalues.decode_path: " + value + ". Valid values: " + VALID_DECODE_PATHS); - } - return normalized; - } - /** * Minimum number of variable-width (string/binary) non-sort columns required to activate * deferred data loading during merge. Below this threshold, all columns are decoded eagerly @@ -947,9 +902,6 @@ public static List> getSettings() { MERGE_BATCH_SIZE, MERGE_RAYON_THREADS, MERGE_IO_THREADS, - LIQUID_CACHE_ENABLED, - LIQUID_CACHE_MAX_BYTES, - DOCVALUES_DECODE_PATH, DOCVALUES_INITIAL_BATCH_SIZE, DOCVALUES_DIAGNOSTICS, DOCVALUES_DICTIONARY_MAX_TERMS, diff --git a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/bridge/ParquetColumnReader.java b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/bridge/ParquetColumnReader.java deleted file mode 100644 index 55f3728406298..0000000000000 --- a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/bridge/ParquetColumnReader.java +++ /dev/null @@ -1,543 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.parquet.bridge; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.apache.lucene.util.ArrayUtil; -import org.apache.lucene.util.LongsRef; -import org.opensearch.parquet.codec.ParquetPhysicalType; -import org.opensearch.parquet.codec.cache.BufferPool; -import org.opensearch.parquet.codec.cache.CacheStats; -import org.opensearch.parquet.codec.cache.ColumnPageIndex; -import org.opensearch.parquet.codec.cache.PageCache; -import org.opensearch.parquet.codec.cache.QueryParquetStats; - -import java.io.Closeable; -import java.io.IOException; -import java.lang.foreign.MemorySegment; -import java.lang.foreign.ValueLayout; -import java.nio.charset.StandardCharsets; -import java.nio.file.Path; - -/** - * FFM wrapper over a single native Parquet column-reader handle, and the owner of that - * column's performance state (the Layer 3/4 {@link ColumnPageIndex} built at open, and the - * current Layer 1/2 {@link PageCache}). - * - *

This is the only class besides {@link RustBridge} that deals with the native - * column-reader surface; iterator implementations (task 5+) see only the plain-Java - * {@link PageCache} / {@link ColumnPageIndex} data structures. - * - *

Threading: a reader is single-threaded (one segment per query thread). It is not safe - * for concurrent use. {@link #close()} is idempotent. - * - *

Buffer ownership: scratch out-buffers handed to the native functions are drawn from a - * shared {@link BufferPool} (Layer 5). The slow-path reads and the page decode both follow - * the grow-and-retry overflow protocol: on {@link RustBridge#RC_OVERFLOW} the required sizes - * are read from the out-parameters, larger buffers are obtained from the pool, and the call - * is retried exactly once. - */ -public final class ParquetColumnReader implements Closeable, NumericPageReader, BinaryPageReader { - - // Dedicated timing channel — separate from the query-stats logger so timing (which takes - // nanoTime) can be toggled independently. When not at TRACE, no nanoTime is taken. - private static final Logger timingLog = LogManager.getLogger("org.opensearch.parquet.timing"); - - /** Sentinel handle for a reader whose native handle has been released. */ - private static final long CLOSED_HANDLE = -1L; - - private final ParquetPhysicalType type; - private final boolean repeated; - private final BufferPool bufferPool; - private final Path file; - private final String column; - private final CacheStats stats = new CacheStats(); - - private long handle; - private ColumnPageIndex pageIndex; - private PageCache cache; - - /** - * Rotation index for the page decode out-buffer slots. The resident {@link PageCache} holds - * off-heap views of the slots the last decode wrote; alternating between two slot families - * ("pageValue0"/"pageValue1", ...) guarantees the next decode never overwrites the segments - * the current cache still serves. Flipped only after a successful decode. - */ - private int decodeSlot; - - /** Optional per-query accumulator; this reader's {@link #stats} are folded in on {@link #close()}. */ - private QueryParquetStats queryStats; - - private ParquetColumnReader(long handle, Path file, String column, ParquetPhysicalType type, boolean repeated, BufferPool bufferPool) { - this.handle = handle; - this.file = file; - this.column = column; - this.type = type; - this.repeated = repeated; - this.bufferPool = bufferPool; - } - - /** - * Opens a column reader for {@code column} in the Parquet {@code file}, validating that - * the column exists and its physical type matches {@code expected}. Eagerly loads the - * column's {@link ColumnPageIndex} (Layer 3/4). - * - * @param repeated whether the column is multi-valued (max repetition level > 0) - * @throws IOException if the file/column cannot be opened or the type mismatches - */ - public static ParquetColumnReader open(Path file, String column, ParquetPhysicalType expected, boolean repeated, BufferPool pool) - throws IOException { - long h = RustBridge.openColumnReader(file.toString(), column, expected.code()); - ParquetColumnReader reader = new ParquetColumnReader(h, file, column, expected, repeated, pool); - try { - reader.pageIndex = reader.loadPageIndex(); - } catch (IOException | RuntimeException e) { - // Never leak the native handle if page-index load fails after open. - reader.close(); - throw e; - } - return reader; - } - - /** The column's Parquet physical type. */ - public ParquetPhysicalType type() { - return type; - } - - /** Whether the column is multi-valued. */ - public boolean isRepeated() { - return repeated; - } - - /** The Layer 3/4 page index, loaded at {@link #open}. */ - public ColumnPageIndex pageIndex() { - return pageIndex; - } - - /** Per-column cache hit/miss counters across all caching layers (for diagnostics). */ - public CacheStats stats() { - return stats; - } - - /** - * Registers this reader's {@link #stats} with the per-query accumulator immediately, so the - * counters are summed live at end-of-query regardless of reader-close ordering. Optional; when - * unset (e.g. low-level tests) no roll-up happens. - */ - public void setQueryStats(QueryParquetStats queryStats) { - this.queryStats = queryStats; - if (queryStats != null) { - queryStats.register(stats); - } - } - - /** - * Returns the currently cached page, or {@code null} when no page is loaded or the last - * {@link #loadPageContaining(long)} landed on an all-nulls page (Layer 4 skip). - */ - public PageCache cache() { - return cache; - } - - // ── Slow-path single/repeated reads (used by ordinal-table construction, task 6) ── - - /** Result of a single-valued read: whether the row had a value, and its raw {@code long} bits. */ - public record Value(boolean present, long bits) { - public static final Value ABSENT = new Value(false, 0L); - } - - /** - * Slow-path single-value read at global {@code row}. For primitive columns the returned - * {@code bits} are the raw value bits (INT32 sign-extended, FLOAT/DOUBLE via - * {@code *toRawBits}, BOOL as 0/1). For {@code BYTE_ARRAY} columns use - * {@link #readBytesAtRow(long)} instead. - */ - public Value readValueAtRow(long row) throws IOException { - ensureOpen(); - stats.slowValueRead(); - MemorySegment present = bufferPool.longOut("present"); - MemorySegment longOut = bufferPool.longOut("long"); - MemorySegment lenOut = bufferPool.longOut("len"); - long rc = RustBridge.readValueAtRow(handle, row, present, longOut, MemorySegment.NULL, 0L, lenOut); - if (rc == RustBridge.RC_OVERFLOW) { - // Primitive reads never overflow (no byte payload); a BYTE_ARRAY column was - // queried through the primitive path. - throw new IOException("readValueAtRow: unexpected overflow for primitive read at row " + row); - } - boolean isPresent = present.get(ValueLayout.JAVA_LONG, 0) != 0L; - return isPresent ? new Value(true, longOut.get(ValueLayout.JAVA_LONG, 0)) : Value.ABSENT; - } - - /** - * Slow-path single-value read of a {@code BYTE_ARRAY} column at global {@code row}. - * Returns the value bytes, or {@code null} when the row is null. Follows the - * grow-and-retry overflow protocol. - */ - public byte[] readBytesAtRow(long row) throws IOException { - ensureOpen(); - stats.slowValueRead(); - MemorySegment present = bufferPool.longOut("present"); - MemorySegment longOut = bufferPool.longOut("long"); - MemorySegment lenOut = bufferPool.longOut("len"); - - long cap = 64; - MemorySegment buf = bufferPool.bytes("value", cap); - long rc = RustBridge.readValueAtRow(handle, row, present, longOut, buf, cap, lenOut); - if (rc == RustBridge.RC_OVERFLOW) { - long required = lenOut.get(ValueLayout.JAVA_LONG, 0); - buf = bufferPool.bytes("value", required); - cap = required; - rc = RustBridge.readValueAtRow(handle, row, present, longOut, buf, cap, lenOut); - if (rc == RustBridge.RC_OVERFLOW) { - throw new IOException("readBytesAtRow: overflow persisted after retry at row " + row); - } - } - if (present.get(ValueLayout.JAVA_LONG, 0) == 0L) { - return null; - } - long len = lenOut.get(ValueLayout.JAVA_LONG, 0); - if (len < 0) { - return null; - } - return buf.asSlice(0, len).toArray(ValueLayout.JAVA_BYTE); - } - - /** Result of a repeated primitive read: the per-value raw {@code long} bits, in row order. */ - public record RepeatedValues(long[] bits) { - public static final RepeatedValues EMPTY = new RepeatedValues(new long[0]); - - public int count() { - return bits.length; - } - } - - /** - * Slow-path repeated read of a primitive column at global {@code row}. Returns the raw - * {@code long} bits of each repeated value, in row order. Follows the grow-and-retry - * overflow protocol. - */ - public RepeatedValues readRepeatedAtRow(long row) throws IOException { - ensureOpen(); - stats.slowRepeatedRead(); - MemorySegment countOut = bufferPool.longOut("count"); - - long cap = 8; - MemorySegment longs = bufferPool.longs("repeated", cap); - long rc = RustBridge.readRepeatedAtRow(handle, row, countOut, longs, cap, MemorySegment.NULL, MemorySegment.NULL, 0L); - if (rc == RustBridge.RC_OVERFLOW) { - long required = countOut.get(ValueLayout.JAVA_LONG, 0); - longs = bufferPool.longs("repeated", required); - cap = required; - rc = RustBridge.readRepeatedAtRow(handle, row, countOut, longs, cap, MemorySegment.NULL, MemorySegment.NULL, 0L); - if (rc == RustBridge.RC_OVERFLOW) { - throw new IOException("readRepeatedAtRow: overflow persisted after retry at row " + row); - } - } - int count = (int) countOut.get(ValueLayout.JAVA_LONG, 0); - if (count == 0) { - return RepeatedValues.EMPTY; - } - long[] out = longs.asSlice(0, (long) count * ValueLayout.JAVA_LONG.byteSize()).toArray(ValueLayout.JAVA_LONG); - return new RepeatedValues(out); - } - - @Override - public void readRepeatedLongsAtRow(long row, LongsRef dst) throws IOException { - RepeatedValues values = readRepeatedAtRow(row); - int count = values.count(); - dst.longs = ArrayUtil.grow(dst.longs, count); - dst.offset = 0; - dst.length = count; - System.arraycopy(values.bits(), 0, dst.longs, 0, count); - } - - /** - * Slow-path repeated read of a {@code BYTE_ARRAY} column at global {@code row}. Returns one - * {@code byte[]} per repeated value in row order, or {@code null} when the row's list is - * empty. Follows the grow-and-retry overflow protocol for both the element-count buffer - * and the concatenated-bytes buffer. - */ - @Override - public byte[][] readRepeatedBytesAtRow(long row) throws IOException { - ensureOpen(); - stats.slowRepeatedRead(); - MemorySegment countOut = bufferPool.longOut("count"); - - long countCap = 8; - long byteCap = 256; - MemorySegment offsets = bufferPool.longs("repeatedOffsets", countCap + 1); - MemorySegment byteBuf = bufferPool.bytes("repeatedBytes", byteCap); - - long rc = RustBridge.readRepeatedAtRow(handle, row, countOut, MemorySegment.NULL, countCap, byteBuf, offsets, byteCap); - if (rc == RustBridge.RC_OVERFLOW) { - long requiredCount = countOut.get(ValueLayout.JAVA_LONG, 0); - countCap = Math.max(requiredCount, countCap); - offsets = bufferPool.longs("repeatedOffsets", countCap + 1); - // First retry establishes the offsets so we can learn the required byte size. - rc = RustBridge.readRepeatedAtRow(handle, row, countOut, MemorySegment.NULL, countCap, byteBuf, offsets, byteCap); - if (rc == RustBridge.RC_OVERFLOW) { - // Byte buffer too small: offsets[count] reports the required total byte size. - int cnt = (int) countOut.get(ValueLayout.JAVA_LONG, 0); - long requiredBytes = offsets.getAtIndex(ValueLayout.JAVA_LONG, cnt); - byteCap = Math.max(requiredBytes, byteCap); - byteBuf = bufferPool.bytes("repeatedBytes", byteCap); - rc = RustBridge.readRepeatedAtRow(handle, row, countOut, MemorySegment.NULL, countCap, byteBuf, offsets, byteCap); - if (rc == RustBridge.RC_OVERFLOW) { - throw new IOException("readRepeatedBytesAtRow: overflow persisted after retry at row " + row); - } - } - } - - int count = (int) countOut.get(ValueLayout.JAVA_LONG, 0); - if (count == 0) { - return null; - } - byte[][] out = new byte[count][]; - for (int i = 0; i < count; i++) { - int start = (int) offsets.getAtIndex(ValueLayout.JAVA_LONG, i); - int end = (int) offsets.getAtIndex(ValueLayout.JAVA_LONG, i + 1); - out[i] = byteBuf.asSlice(start, (long) (end - start)).toArray(ValueLayout.JAVA_BYTE); - } - return out; - } - - // ── Page index + page decode (Layer 1-4 hot path) ── - - /** - * Loads the page that contains global {@code row} into the cache (Layer 1/2), applying - * the Layer 4 all-nulls skip first. On an all-nulls page the cache is set to {@code null} - * and no decode happens. Single-valued columns only; repeated columns use the slow path. - */ - public void loadPageContaining(long row) throws IOException { - boolean t = timingLog.isTraceEnabled(); - long start = t ? System.nanoTime() : 0L; - try { - ensureOpen(); - // Layer 3 — OffsetIndex jump-table lookup (consulted on every page miss). - stats.pageIndexLookup(); - int pageIdx = pageIndex.pageForRow(row); - if (pageIdx < 0) { - throw new IOException("loadPageContaining: row " + row + " out of range (rows " + pageIndex.totalRows() + ")"); - } - if (pageIndex.isAllNulls(pageIdx)) { - // Layer 4 — whole page is null, resolved with no decode. - stats.allNullPageSkip(); - cache = null; // Layer 4 — whole page is null, no decode. - return; - } - // FFM — page decode crossing. - stats.pageDecode(); - cache = decodePage(row); - } finally { - if (t) stats.addLoadPageNanos(System.nanoTime() - start); - } - } - - private PageCache decodePage(long row) throws IOException { - boolean t = timingLog.isTraceEnabled(); - long decodeStart = t ? System.nanoTime() : 0L; - try { - return decodePage0(row, t); - } finally { - if (t) stats.addDecodePageNanos(System.nanoTime() - decodeStart); - } - } - - private PageCache decodePage0(long row, boolean t) throws IOException { - MemorySegment firstRowOut = bufferPool.longOut("firstRow"); - MemorySegment lastRowOut = bufferPool.longOut("lastRow"); - MemorySegment valueLenOut = bufferPool.longOut("valueLen"); - - // Size the page from the index so the first attempt usually fits. - int pageIdx = pageIndex.pageForRow(row); - int rows = (int) pageIndex.numRowsOf(pageIdx); - int presenceWords = (rows + 63) >>> 6; - - // Slot naming carries TWO independent guarantees, both required for correctness: - // 1. Per-column ("...:" + column): the PageCache serves values/presence as in-place - // off-heap views of these slots, so two columns aggregated in the SAME query (e.g. - // sum(age) + avg(score)) must NOT share a slot — otherwise the second column's decode - // overwrites the first column's still-resident cached page. A shared slot corrupts - // cross-column reads (age would read score's bits). - // 2. Rotating (+ decodeSlot, flipped after each successful decode): within ONE column, - // the next page decode must not overwrite the segments the current resident page still - // serves; alternating two families keeps the resident view untouched until page-after-next. - // Both are needed: (1) alone leaves a single column clobbering its own resident page; (2) - // alone (the state before this fix) leaves two columns colliding on the same family. - String valueSlot = "pageValue:" + column + decodeSlot; - String offsetsSlot = "pageOffsets:" + column + decodeSlot; - String presenceSlot = "pagePresence:" + column + decodeSlot; - - long valueCap = (long) rows * ValueLayout.JAVA_LONG.byteSize(); - long offsetsCap = type.isPrimitive() ? 0 : (rows + 1); - MemorySegment valueBuf = bufferPool.bytes(valueSlot, Math.max(valueCap, 1)); - MemorySegment offsets = type.isPrimitive() ? MemorySegment.NULL : bufferPool.ints(offsetsSlot, offsetsCap); - MemorySegment presence = bufferPool.longs(presenceSlot, presenceWords); - - long ffmStart = t ? System.nanoTime() : 0L; - long rc = RustBridge.decodePageAtRow( - handle, - row, - firstRowOut, - lastRowOut, - valueBuf, - valueCap, - valueLenOut, - offsets, - offsetsCap, - presence, - presenceWords - ); - if (t) stats.addFfmDecodeNanos(System.nanoTime() - ffmStart); - - if (rc == RustBridge.RC_OVERFLOW) { - // Re-size from the reported page range + value length and retry once. - long fr = firstRowOut.get(ValueLayout.JAVA_LONG, 0); - long lr = lastRowOut.get(ValueLayout.JAVA_LONG, 0); - int actualRows = (int) (lr - fr + 1); - int actualPresenceWords = (actualRows + 63) >>> 6; - long requiredValueBytes = valueLenOut.get(ValueLayout.JAVA_LONG, 0); - - valueCap = Math.max(requiredValueBytes, 1); - offsetsCap = type.isPrimitive() ? 0 : (actualRows + 1); - valueBuf = bufferPool.bytes(valueSlot, valueCap); - offsets = type.isPrimitive() ? MemorySegment.NULL : bufferPool.ints(offsetsSlot, offsetsCap); - presence = bufferPool.longs(presenceSlot, actualPresenceWords); - - long ffmRetryStart = t ? System.nanoTime() : 0L; - rc = RustBridge.decodePageAtRow( - handle, - row, - firstRowOut, - lastRowOut, - valueBuf, - valueCap, - valueLenOut, - offsets, - offsetsCap, - presence, - actualPresenceWords - ); - if (t) stats.addFfmDecodeNanos(System.nanoTime() - ffmRetryStart); - if (rc == RustBridge.RC_OVERFLOW) { - throw new IOException("decodePageAtRow: overflow persisted after retry at row " + row); - } - presenceWords = actualPresenceWords; - } - - long firstRow = firstRowOut.get(ValueLayout.JAVA_LONG, 0); - long lastRow = lastRowOut.get(ValueLayout.JAVA_LONG, 0); - long valueLen = valueLenOut.get(ValueLayout.JAVA_LONG, 0); - int pageRows = (int) (lastRow - firstRow + 1); - - PageCache pc = new PageCache(); - pc.firstRow = firstRow; - pc.lastRow = lastRow; - // Serve the decoded page in place: off-heap views of the slots this decode wrote — - // zero on-heap copies. The slot rotation above keeps these segments untouched until - // the page after next; the pool's arena keeps them valid until the producer closes. - pc.presenceBits = presence.asSlice(0, (long) ((pageRows + 63) >>> 6) * ValueLayout.JAVA_LONG.byteSize()); - - if (type.isPrimitive()) { - pc.values = valueBuf.asSlice(0, (long) pageRows * ValueLayout.JAVA_LONG.byteSize()); - } else { - // BYTE_ARRAY keeps heap copies: BinaryDocValues hands out BytesRef, whose contract - // requires a heap byte[]. Presence is still served off-heap. - pc.byteBuf = valueLen > 0 ? valueBuf.asSlice(0, valueLen).toArray(ValueLayout.JAVA_BYTE) : new byte[0]; - pc.byteOffsets = offsets.asSlice(0, (long) (pageRows + 1) * ValueLayout.JAVA_INT.byteSize()).toArray(ValueLayout.JAVA_INT); - } - // Flip AFTER a successful decode so a failed decode retries into the same family and - // the resident cache's views (the other family) were never at risk. - decodeSlot ^= 1; - return pc; - } - - private ColumnPageIndex loadPageIndex() throws IOException { - long numPages = RustBridge.getColumnNumPages(handle); - int n = Math.toIntExact(numPages); - - MemorySegment firstRow = bufferPool.longs("idxFirstRow", Math.max(n, 1)); - MemorySegment fileOffset = bufferPool.longs("idxFileOffset", Math.max(n, 1)); - MemorySegment compressed = bufferPool.ints("idxCompressed", Math.max(n, 1)); - MemorySegment nullCount = bufferPool.longs("idxNullCount", Math.max(n, 1)); - MemorySegment minLong = bufferPool.longs("idxMin", Math.max(n, 1)); - MemorySegment maxLong = bufferPool.longs("idxMax", Math.max(n, 1)); - MemorySegment actualPages = bufferPool.longOut("idxActualPages"); - - long rc = RustBridge.getColumnPageIndex(handle, firstRow, fileOffset, compressed, nullCount, minLong, maxLong, n, actualPages); - if (rc == RustBridge.RC_OVERFLOW) { - // Page count grew between the two calls (shouldn't happen for an immutable - // file, but handle defensively): re-size to the reported count and retry once. - n = Math.toIntExact(actualPages.get(ValueLayout.JAVA_LONG, 0)); - firstRow = bufferPool.longs("idxFirstRow", Math.max(n, 1)); - fileOffset = bufferPool.longs("idxFileOffset", Math.max(n, 1)); - compressed = bufferPool.ints("idxCompressed", Math.max(n, 1)); - nullCount = bufferPool.longs("idxNullCount", Math.max(n, 1)); - minLong = bufferPool.longs("idxMin", Math.max(n, 1)); - maxLong = bufferPool.longs("idxMax", Math.max(n, 1)); - rc = RustBridge.getColumnPageIndex(handle, firstRow, fileOffset, compressed, nullCount, minLong, maxLong, n, actualPages); - if (rc == RustBridge.RC_OVERFLOW) { - throw new IOException("getColumnPageIndex: overflow persisted after retry"); - } - } - - long[] firstRowArr = toLongArray(firstRow, n); - long[] fileOffsetArr = toLongArray(fileOffset, n); - int[] compressedArr = toIntArray(compressed, n); - long[] nullCountArr = toLongArray(nullCount, n); - long[] minArr = toLongArray(minLong, n); - long[] maxArr = toLongArray(maxLong, n); - - // The per-page first-row offsets don't encode the last page's length, so take the - // file's authoritative row count from metadata. open() is invoked once per column, - // so this extra metadata read is not on the hot path. - long totalRows = RustBridge.getFileMetadata(file.toString()).numRows(); - return new ColumnPageIndex(firstRowArr, fileOffsetArr, compressedArr, nullCountArr, minArr, maxArr, totalRows); - } - - private static long[] toLongArray(MemorySegment seg, int n) { - if (n == 0) { - return new long[0]; - } - return seg.asSlice(0, (long) n * ValueLayout.JAVA_LONG.byteSize()).toArray(ValueLayout.JAVA_LONG); - } - - private static int[] toIntArray(MemorySegment seg, int n) { - if (n == 0) { - return new int[0]; - } - return seg.asSlice(0, (long) n * ValueLayout.JAVA_INT.byteSize()).toArray(ValueLayout.JAVA_INT); - } - - /** Decodes a UTF-8 byte slice from the binary page cache. */ - public static String utf8(byte[] bytes) { - return new String(bytes, StandardCharsets.UTF_8); - } - - private void ensureOpen() { - if (handle == CLOSED_HANDLE) { - throw new IllegalStateException("ParquetColumnReader is closed"); - } - } - - /** Idempotent: releases the native handle exactly once; never leaks on repeated calls. */ - @Override - public void close() throws IOException { - if (handle == CLOSED_HANDLE) { - return; - } - // Per-query roll-up is handled by registration at open (see setQueryStats) and emitted once - // per query on the dedicated stats channel; no per-column detail line here. - long h = handle; - handle = CLOSED_HANDLE; - cache = null; - RustBridge.closeColumnReader(h); - } -} diff --git a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/bridge/RustBridge.java b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/bridge/RustBridge.java index f11e167ad8be0..60253bf27c1a1 100644 --- a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/bridge/RustBridge.java +++ b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/bridge/RustBridge.java @@ -52,19 +52,6 @@ public class RustBridge { private static final MethodHandle GET_POOL_STATS; // DocValues codec — column-reader functions (tasks 1.1 + 1.2) - private static final MethodHandle OPEN_COLUMN_READER; - private static final MethodHandle CLOSE_COLUMN_READER; - private static final MethodHandle OPEN_COLUMN_READER_COUNT; - private static final MethodHandle LIQUID_CACHE_SET_ENABLED; - private static final MethodHandle LIQUID_CACHE_CLEAR; - private static final MethodHandle LIQUID_CACHE_STATS; - private static final MethodHandle TIMING_SET_ENABLED; - private static final MethodHandle TIMING_SNAPSHOT; - private static final MethodHandle READ_VALUE_AT_ROW; - private static final MethodHandle READ_REPEATED_AT_ROW; - private static final MethodHandle GET_COLUMN_NUM_PAGES; - private static final MethodHandle GET_COLUMN_PAGE_INDEX; - private static final MethodHandle DECODE_PAGE_AT_ROW; private static final MethodHandle DF_OPEN_ITER; private static final MethodHandle DF_CLOSE_ITER; private static final MethodHandle DF_RESET_ITER; @@ -320,124 +307,6 @@ public class RustBridge { ); // ── DocValues codec column-reader functions ── - OPEN_COLUMN_READER = linker.downcallHandle( - lib.find("parquet_open_column_reader").orElseThrow(), - FunctionDescriptor.of( - ValueLayout.JAVA_LONG, - ValueLayout.ADDRESS, // file_ptr - ValueLayout.JAVA_LONG, // file_len - ValueLayout.ADDRESS, // col_ptr - ValueLayout.JAVA_LONG, // col_len - ValueLayout.JAVA_INT // expected_type - ) - ); - CLOSE_COLUMN_READER = linker.downcallHandle( - lib.find("parquet_close_column_reader").orElseThrow(), - FunctionDescriptor.of(ValueLayout.JAVA_LONG, ValueLayout.JAVA_LONG) - ); - OPEN_COLUMN_READER_COUNT = linker.downcallHandle( - lib.find("parquet_open_column_reader_count").orElseThrow(), - FunctionDescriptor.of(ValueLayout.JAVA_LONG) - ); - LIQUID_CACHE_SET_ENABLED = linker.downcallHandle( - lib.find("parquet_liquid_cache_set_enabled").orElseThrow(), - FunctionDescriptor.of( - ValueLayout.JAVA_LONG, // status (< 0 error pointer) - ValueLayout.JAVA_INT, // enabled - ValueLayout.JAVA_LONG, // max_memory_bytes - ValueLayout.ADDRESS, // cache_dir ptr - ValueLayout.JAVA_LONG // cache_dir len - ) - ); - LIQUID_CACHE_CLEAR = linker.downcallHandle( - lib.find("parquet_liquid_cache_clear").orElseThrow(), - FunctionDescriptor.of(ValueLayout.JAVA_LONG) // status (< 0 error pointer) - ); - LIQUID_CACHE_STATS = linker.downcallHandle( - lib.find("parquet_liquid_cache_stats").orElseThrow(), - FunctionDescriptor.of( - ValueLayout.JAVA_LONG, // status (< 0 error pointer) - ValueLayout.ADDRESS, // hits_out - ValueLayout.ADDRESS, // misses_out - ValueLayout.ADDRESS // puts_out - ) - ); - TIMING_SET_ENABLED = linker.downcallHandle( - lib.find("parquet_timing_set_enabled").orElseThrow(), - FunctionDescriptor.of(ValueLayout.JAVA_LONG, ValueLayout.JAVA_INT) // status, enabled - ); - TIMING_SNAPSHOT = linker.downcallHandle( - lib.find("parquet_timing_snapshot").orElseThrow(), - FunctionDescriptor.of( - ValueLayout.JAVA_LONG, // status - ValueLayout.ADDRESS, // get_out - ValueLayout.ADDRESS, // decode_out - ValueLayout.ADDRESS // put_out - ) - ); - READ_VALUE_AT_ROW = linker.downcallHandle( - lib.find("parquet_read_value_at_row").orElseThrow(), - FunctionDescriptor.of( - ValueLayout.JAVA_LONG, - ValueLayout.JAVA_LONG, // handle - ValueLayout.JAVA_LONG, // row - ValueLayout.ADDRESS, // out_present - ValueLayout.ADDRESS, // out_long - ValueLayout.ADDRESS, // out_buf - ValueLayout.JAVA_LONG, // out_buf_cap - ValueLayout.ADDRESS // out_len - ) - ); - READ_REPEATED_AT_ROW = linker.downcallHandle( - lib.find("parquet_read_repeated_at_row").orElseThrow(), - FunctionDescriptor.of( - ValueLayout.JAVA_LONG, - ValueLayout.JAVA_LONG, // handle - ValueLayout.JAVA_LONG, // row - ValueLayout.ADDRESS, // out_count - ValueLayout.ADDRESS, // out_longs - ValueLayout.JAVA_LONG, // out_long_cap - ValueLayout.ADDRESS, // out_byte_buf - ValueLayout.ADDRESS, // out_byte_offsets - ValueLayout.JAVA_LONG // out_byte_buf_cap - ) - ); - GET_COLUMN_NUM_PAGES = linker.downcallHandle( - lib.find("parquet_get_column_num_pages").orElseThrow(), - FunctionDescriptor.of(ValueLayout.JAVA_LONG, ValueLayout.JAVA_LONG) - ); - GET_COLUMN_PAGE_INDEX = linker.downcallHandle( - lib.find("parquet_get_column_page_index").orElseThrow(), - FunctionDescriptor.of( - ValueLayout.JAVA_LONG, - ValueLayout.JAVA_LONG, // handle - ValueLayout.ADDRESS, // out_first_row - ValueLayout.ADDRESS, // out_file_offset - ValueLayout.ADDRESS, // out_compressed_size - ValueLayout.ADDRESS, // out_null_count - ValueLayout.ADDRESS, // out_min_long - ValueLayout.ADDRESS, // out_max_long - ValueLayout.JAVA_LONG, // out_buf_capacity - ValueLayout.ADDRESS // out_actual_pages - ) - ); - DECODE_PAGE_AT_ROW = linker.downcallHandle( - lib.find("parquet_decode_page_at_row").orElseThrow(), - FunctionDescriptor.of( - ValueLayout.JAVA_LONG, - ValueLayout.JAVA_LONG, // handle - ValueLayout.JAVA_LONG, // row - ValueLayout.ADDRESS, // out_first_row - ValueLayout.ADDRESS, // out_last_row - ValueLayout.ADDRESS, // out_value_buf - ValueLayout.JAVA_LONG, // out_value_buf_cap - ValueLayout.ADDRESS, // out_value_actual_len - ValueLayout.ADDRESS, // out_byte_offsets - ValueLayout.JAVA_LONG, // out_byte_offsets_cap - ValueLayout.ADDRESS, // out_presence_bitset - ValueLayout.JAVA_LONG // out_presence_bits_cap - ) - ); DF_OPEN_ITER = linker.downcallHandle( lib.find("parquet_df_open_iter").orElseThrow(), FunctionDescriptor.of( @@ -996,221 +865,6 @@ private static long invokeChecked(MethodHandle handle, Object... args) throws IO return NativeCall.invokeIOStatic(handle, args); } - /** - * Opens a per-column reader over {@code file} for {@code column}, validating that the - * column exists and its physical type matches {@code expectedTypeCode} - * (0=INT32, 1=INT64, 2=FLOAT, 3=DOUBLE, 4=BOOL, 5=BYTE_ARRAY). - * - * @return a {@code >= 0} opaque reader handle (a Rust-side i64; lives until {@link #closeColumnReader}) - * @throws IOException if the column is missing, the type mismatches, or the file cannot be read - */ - public static long openColumnReader(String file, String column, int expectedTypeCode) throws IOException { - try (var call = new NativeCall()) { - var f = call.str(file); - var c = call.str(column); - return call.invokeIO(OPEN_COLUMN_READER, f.segment(), f.len(), c.segment(), c.len(), expectedTypeCode); - } - } - - /** - * Closes a column reader handle, releasing its native file handle and buffers. - * Safe to call with an already-closed/unknown handle only if the caller guards it; - * the native side returns an error for unknown handles. - * - * @throws IOException if the handle is unknown - */ - public static void closeColumnReader(long handle) throws IOException { - invokeChecked(CLOSE_COLUMN_READER, handle); - } - - /** - * Debug-only: returns the number of currently open native column-reader handles. - * Used by Property 7 (native handle non-leakage). - */ - public static long openColumnReaderCount() { - return NativeCall.invokeStatic(OPEN_COLUMN_READER_COUNT); - } - - /** - * Enables or disables the cross-query decoded-page cache (codec-owned liquid instance) and sets - * its memory budget in bytes. Called once at plugin init when the {@code parquet_liquid_cache} - * feature flag is on. When disabled (the default), {@code parquet_decode_page_at_row} never - * consults the cache and the decode path is unchanged. A {@code maxMemoryBytes} of 0 leaves the - * native default budget. - */ - public static void liquidCacheSetEnabled(boolean enabled, long maxMemoryBytes, String cacheDir) { - try (var call = new NativeCall()) { - var dir = call.str(cacheDir); - call.invoke(LIQUID_CACHE_SET_ENABLED, enabled ? 1 : 0, maxMemoryBytes, dir.segment(), dir.len()); - } - } - - /** - * Clears the codec-owned liquid decoded-page cache (in-memory index + spilled {@code t4} entries) - * on this node, without disabling it. A no-op when the cache is disabled or not yet built. Used by - * the {@code POST /_plugins/parquet/liquid_cache/_clear} REST action for cold-start benchmarking. - */ - public static void liquidCacheClear() { - NativeCall.invokeStatic(LIQUID_CACHE_CLEAR); - } - - /** - * Process-wide liquid decoded-page cache event counters, monotonic since process start. - * - * @param hits pages served from liquid without a Parquet decode - * @param misses liquid {@code get} found nothing, so the caller decoded from Parquet - * @param puts decoded pages inserted into liquid - */ - public record LiquidCacheStats(long hits, long misses, long puts) { - } - - /** - * Snapshots the codec liquid cache event counters. Cheap (three relaxed atomic loads on the - * native side); read before and after a query to compute per-query deltas. Returns zeros when - * the cache is disabled. - */ - public static LiquidCacheStats liquidCacheStats() { - try (var call = new NativeCall()) { - var hits = call.longOut(); - var misses = call.longOut(); - var puts = call.longOut(); - call.invoke(LIQUID_CACHE_STATS, hits, misses, puts); - return new LiquidCacheStats( - hits.get(ValueLayout.JAVA_LONG, 0), - misses.get(ValueLayout.JAVA_LONG, 0), - puts.get(ValueLayout.JAVA_LONG, 0) - ); - } - } - - /** Cumulative page-decode phase timers (nanos since process start): get/decode/put. */ - public record TimingStats(long getNanos, long decodeNanos, long putNanos) { - } - - /** Enables/disables native page-decode phase timing. Called when the timing logger toggles. */ - public static void timingSetEnabled(boolean enabled) { - NativeCall.invokeStatic(TIMING_SET_ENABLED, enabled ? 1 : 0); - } - - /** Snapshots the native get/decode/put phase timers (cumulative nanos). */ - public static TimingStats timingSnapshot() { - try (var call = new NativeCall()) { - var g = call.longOut(); - var d = call.longOut(); - var p = call.longOut(); - call.invoke(TIMING_SNAPSHOT, g, d, p); - return new TimingStats(g.get(ValueLayout.JAVA_LONG, 0), d.get(ValueLayout.JAVA_LONG, 0), p.get(ValueLayout.JAVA_LONG, 0)); - } - } - - /** - * Slow-path single-value read at {@code row} into caller-provided out-segments. - * The caller (ParquetColumnReader) owns the segments. Returns the native status: - * {@code 0} on success, {@link #RC_OVERFLOW} when {@code outBuf} was too small - * (required length written to {@code outLen}); throws on a native error. - */ - static long readValueAtRow( - long handle, - long row, - MemorySegment outPresent, - MemorySegment outLong, - MemorySegment outBuf, - long outBufCap, - MemorySegment outLen - ) throws IOException { - return invokeChecked(READ_VALUE_AT_ROW, handle, row, outPresent, outLong, outBuf, outBufCap, outLen); - } - - /** - * Slow-path repeated read at {@code row} into caller-provided out-segments. - * Returns the native status: {@code 0} on success, {@link #RC_OVERFLOW} when a - * buffer was too small (required element count in {@code outCount}; required byte - * size in {@code outByteOffsets[count]} when only the byte buffer overflowed). - */ - static long readRepeatedAtRow( - long handle, - long row, - MemorySegment outCount, - MemorySegment outLongs, - long outLongCap, - MemorySegment outByteBuf, - MemorySegment outByteOffsets, - long outByteBufCap - ) throws IOException { - return invokeChecked(READ_REPEATED_AT_ROW, handle, row, outCount, outLongs, outLongCap, outByteBuf, outByteOffsets, outByteBufCap); - } - - /** Returns the number of pages in the column (used to pre-size the page-index arrays). */ - static long getColumnNumPages(long handle) throws IOException { - return invokeChecked(GET_COLUMN_NUM_PAGES, handle); - } - - /** - * Loads the column's per-page jump table + stats (Layer 3/4) into caller-provided - * parallel out-segments, each of capacity {@code outBufCapacity} pages. Returns the - * native status: {@code 0} on success, {@link #RC_OVERFLOW} when capacity is too - * small (true page count written to {@code outActualPages}). - */ - static long getColumnPageIndex( - long handle, - MemorySegment outFirstRow, - MemorySegment outFileOffset, - MemorySegment outCompressedSize, - MemorySegment outNullCount, - MemorySegment outMinLong, - MemorySegment outMaxLong, - long outBufCapacity, - MemorySegment outActualPages - ) throws IOException { - return invokeChecked( - GET_COLUMN_PAGE_INDEX, - handle, - outFirstRow, - outFileOffset, - outCompressedSize, - outNullCount, - outMinLong, - outMaxLong, - outBufCapacity, - outActualPages - ); - } - - /** - * Decodes the page containing {@code row} (Layer 1 values + Layer 2 presence bitset) - * into caller-provided out-segments. Returns the native status: {@code 0} on success, - * {@link #RC_OVERFLOW} when a buffer was too small (page row range and required value - * byte length are still written so the caller can size every buffer and retry once). - */ - static long decodePageAtRow( - long handle, - long row, - MemorySegment outFirstRow, - MemorySegment outLastRow, - MemorySegment outValueBuf, - long outValueBufCap, - MemorySegment outValueActualLen, - MemorySegment outByteOffsets, - long outByteOffsetsCap, - MemorySegment outPresenceBitset, - long outPresenceBitsCap - ) throws IOException { - return invokeChecked( - DECODE_PAGE_AT_ROW, - handle, - row, - outFirstRow, - outLastRow, - outValueBuf, - outValueBufCap, - outValueActualLen, - outByteOffsets, - outByteOffsetsCap, - outPresenceBitset, - outPresenceBitsCap - ); - } - /** Opens a forward-only DataFusion/Arrow cursor over one Parquet column. */ public static long dfOpenIter(String file, String column, int initialBatchSize) throws IOException { try (var call = new NativeCall()) { diff --git a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/codec/ParquetDocValuesDirectoryReader.java b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/codec/ParquetDocValuesDirectoryReader.java index 66c63faf723fd..bd8ead7b26151 100644 --- a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/codec/ParquetDocValuesDirectoryReader.java +++ b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/codec/ParquetDocValuesDirectoryReader.java @@ -8,14 +8,10 @@ package org.opensearch.parquet.codec; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.FilterDirectoryReader; import org.apache.lucene.index.LeafReader; import org.opensearch.index.mapper.MapperService; -import org.opensearch.parquet.bridge.RustBridge; -import org.opensearch.parquet.codec.cache.QueryParquetStats; import java.io.IOException; import java.io.UncheckedIOException; @@ -37,35 +33,11 @@ public final class ParquetDocValuesDirectoryReader extends FilterDirectoryReader // Dedicated stats channel, NOT the class-named logger, so the per-query summary can be toggled // in isolation (logger.org.opensearch.parquet.stats.query=TRACE) without turning on any other - // codec class's logging. Enabling it affects only this one diagnostic line. - private static final Logger statsLogger = LogManager.getLogger("org.opensearch.parquet.stats.query"); - - // Parallel timing channel — toggled independently of statsLogger. When at TRACE, the native - // phase timers are enabled and baselined at query start, and per-query nanoTime is accumulated. - private static final Logger timingLog = LogManager.getLogger("org.opensearch.parquet.timing"); - private final MapperService mapperService; - private final QueryParquetStats queryStats; - private ParquetDocValuesDirectoryReader(DirectoryReader in, MapperService mapperService, QueryParquetStats queryStats) - throws IOException { - super(in, new ParquetSubReaderWrapper(mapperService, queryStats)); + private ParquetDocValuesDirectoryReader(DirectoryReader in, MapperService mapperService) throws IOException { + super(in, new ParquetSubReaderWrapper(mapperService)); this.mapperService = mapperService; - this.queryStats = queryStats; - // Baseline the process-wide liquid counters so doClose() can report per-query deltas. Only - // when the summary will actually be logged — the snapshot is an FFM crossing, so this keeps - // it fully off (no native call) on the raw-performance path where TRACE is disabled. - if (statsLogger.isTraceEnabled()) { - RustBridge.LiquidCacheStats base = RustBridge.liquidCacheStats(); - queryStats.captureLiquidBaseline(base.hits(), base.misses(), base.puts()); - } - // Parallel timing path: enable native phase timers and baseline them so doClose() can report - // per-query deltas. Fully off (no native call, no nanoTime) when the timing channel is not at TRACE. - if (timingLog.isTraceEnabled()) { - RustBridge.timingSetEnabled(true); - RustBridge.TimingStats tbase = RustBridge.timingSnapshot(); - queryStats.captureTimingBaseline(tbase.getNanos(), tbase.decodeNanos(), tbase.putNanos()); - } } /** @@ -73,13 +45,12 @@ private ParquetDocValuesDirectoryReader(DirectoryReader in, MapperService mapper * code paths. */ public static DirectoryReader wrap(DirectoryReader in, MapperService mapperService) throws IOException { - return new ParquetDocValuesDirectoryReader(in, mapperService, new QueryParquetStats()); + return new ParquetDocValuesDirectoryReader(in, mapperService); } @Override protected DirectoryReader doWrapDirectoryReader(DirectoryReader in) throws IOException { - // A reopened reader is a fresh search view; give it its own accumulator. - return new ParquetDocValuesDirectoryReader(in, mapperService, new QueryParquetStats()); + return new ParquetDocValuesDirectoryReader(in, mapperService); } @Override @@ -112,17 +83,6 @@ protected void doClose() throws IOException { first = e; } } - // The per-query [PARQUET_DV_QUERY_STATS] summary is TRACE-only so it is NOT emitted during - // raw-performance runs. Counters are always accumulated (cheap); to see the summary enable - // the dedicated stats channel: logger.org.opensearch.parquet.stats.query=TRACE. - if (queryStats != null && queryStats.isEmpty() == false && statsLogger.isTraceEnabled()) { - RustBridge.LiquidCacheStats now = RustBridge.liquidCacheStats(); - statsLogger.trace("[PARQUET_DV_QUERY_STATS] {}", queryStats.summary(now.hits(), now.misses(), now.puts())); - } - if (queryStats != null && queryStats.isEmpty() == false && timingLog.isTraceEnabled()) { - RustBridge.TimingStats tnow = RustBridge.timingSnapshot(); - timingLog.trace("[PARQUET_DV_TIMING] {}", queryStats.timingSummary(tnow.getNanos(), tnow.decodeNanos(), tnow.putNanos())); - } } if (first != null) { throw first; @@ -132,17 +92,15 @@ protected void doClose() throws IOException { /** Per-leaf wrapper that swaps in {@link ParquetDocValuesLeafReader} when applicable. */ private static final class ParquetSubReaderWrapper extends SubReaderWrapper { private final MapperService mapperService; - private final QueryParquetStats queryStats; - private ParquetSubReaderWrapper(MapperService mapperService, QueryParquetStats queryStats) { + private ParquetSubReaderWrapper(MapperService mapperService) { this.mapperService = mapperService; - this.queryStats = queryStats; } @Override public LeafReader wrap(LeafReader reader) { try { - return ParquetDocValuesLeafReader.wrapIfApplicable(reader, mapperService, queryStats); + return ParquetDocValuesLeafReader.wrapIfApplicable(reader, mapperService); } catch (IOException e) { // SubReaderWrapper.wrap cannot throw checked exceptions; surface as unchecked so // the search fails loudly rather than silently dropping Parquet doc values. diff --git a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/codec/ParquetDocValuesLeafReader.java b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/codec/ParquetDocValuesLeafReader.java index 8a9d506b1f6c5..c400e5ce26766 100644 --- a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/codec/ParquetDocValuesLeafReader.java +++ b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/codec/ParquetDocValuesLeafReader.java @@ -34,7 +34,6 @@ import org.opensearch.index.engine.dataformat.DocumentInput; import org.opensearch.index.mapper.MappedFieldType; import org.opensearch.index.mapper.MapperService; -import org.opensearch.parquet.codec.cache.QueryParquetStats; import org.opensearch.parquet.codec.iter.ParquetDictionarySortedDocValues; import org.opensearch.parquet.codec.iter.ParquetSortedDocValues; import org.opensearch.parquet.codec.iter.ParquetUninvertedSortedDocValues; @@ -101,22 +100,19 @@ public final class ParquetDocValuesLeafReader extends SequentialStoredFieldsLeaf private final SegmentReadState segmentReadState; /** Per-query stats accumulator shared across all leaves of one search; may be null in tests. */ - private final QueryParquetStats queryStats; private ParquetDocValuesLeafReader( LeafReader in, MapperService mapperService, SegmentReadState segmentReadState, Map parquetFields, - FieldInfos mergedFieldInfos, - QueryParquetStats queryStats + FieldInfos mergedFieldInfos ) { super(in); this.mapperService = mapperService; this.segmentReadState = segmentReadState; this.parquetFields = parquetFields; this.mergedFieldInfos = mergedFieldInfos; - this.queryStats = queryStats; } /** @@ -124,7 +120,7 @@ private ParquetDocValuesLeafReader( * segment and the mapping declares at least one Parquet-codec-supported field that is missing * doc values in the Lucene segment. Otherwise returns {@code in} unwrapped. */ - public static LeafReader wrapIfApplicable(LeafReader in, MapperService mapperService, QueryParquetStats queryStats) throws IOException { + public static LeafReader wrapIfApplicable(LeafReader in, MapperService mapperService) throws IOException { SegmentReader segmentReader; try { segmentReader = Lucene.segmentReader(in); @@ -187,7 +183,7 @@ public static LeafReader wrapIfApplicable(LeafReader in, MapperService mapperSer } FieldInfos mergedInfos = new FieldInfos(merged.toArray(new FieldInfo[0])); - return new ParquetDocValuesLeafReader(in, mapperService, state, parquetFields, mergedInfos, queryStats); + return new ParquetDocValuesLeafReader(in, mapperService, state, parquetFields, mergedInfos); } /** @@ -230,7 +226,6 @@ private static FieldInfo newDocValuesFieldInfo(String name, int number, DocValue private synchronized ParquetDocValuesProducer producer() throws IOException { if (producerInitialized == false) { producer = new ParquetDocValuesProducer(segmentReadState, mapperService); - producer.setQueryStats(queryStats); producerInitialized = true; } if (producer != null && producer.isClosed()) { @@ -240,9 +235,7 @@ private synchronized ParquetDocValuesProducer producer() throws IOException { // itself closes — the contract every reader-keyed cache in Lucene/OpenSearch assumes. ParquetDocValuesProducer shared = SharedProducerRegistry.get(in.getCoreCacheHelper(), segmentReadState, mapperService); if (shared == null) { - throw new IllegalStateException( - "doc values requested after the search closed and the segment has no core cache identity" - ); + throw new IllegalStateException("doc values requested after the search closed and the segment has no core cache identity"); } return shared; } diff --git a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/codec/ParquetDocValuesProducer.java b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/codec/ParquetDocValuesProducer.java index 3b42bb48a681a..16103bc35d4f8 100644 --- a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/codec/ParquetDocValuesProducer.java +++ b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/codec/ParquetDocValuesProducer.java @@ -22,14 +22,11 @@ import org.apache.lucene.index.SortedSetDocValues; import org.opensearch.index.mapper.MappedFieldType; import org.opensearch.index.mapper.MapperService; -import org.opensearch.parquet.ParquetSettings; import org.opensearch.parquet.bridge.BinaryPageReader; import org.opensearch.parquet.bridge.DataFusionColumnReader; -import org.opensearch.parquet.bridge.ParquetColumnReader; import org.opensearch.parquet.bridge.ParquetFileMetadata; import org.opensearch.parquet.bridge.RustBridge; import org.opensearch.parquet.codec.cache.BufferPool; -import org.opensearch.parquet.codec.cache.QueryParquetStats; import org.opensearch.parquet.codec.iter.ParquetBinaryDocValues; import org.opensearch.parquet.codec.iter.ParquetNumericDocValues; import org.opensearch.parquet.codec.iter.ParquetSortedDocValues; @@ -38,7 +35,6 @@ import java.io.IOException; import java.nio.file.Path; -import java.util.HashMap; import java.util.Locale; import java.util.Map; @@ -66,18 +62,12 @@ public final class ParquetDocValuesProducer extends DocValuesProducer { private static final Logger logger = LogManager.getLogger(ParquetDocValuesProducer.class); - private static volatile boolean useDataFusionDecodePath; private static volatile int dataFusionInitialBatchSize = 32; private static volatile boolean dataFusionDiagnostics; private static volatile int dictionaryMaxTerms = 65536; private static volatile long dictionaryCacheBytes = 64 * 1024 * 1024; private static volatile long uninvertMaxDiskBytes = 2L * 1024 * 1024 * 1024; - /** Updates the node-wide DocValues decode path. */ - public static void setDecodePath(String decodePath) { - useDataFusionDecodePath = ParquetSettings.DECODE_PATH_DATAFUSION.equals(decodePath); - } - /** Updates the cardinality budget for dictionary-rank keyword ordinals. */ public static void setDictionaryMaxTerms(int maxTerms) { dictionaryMaxTerms = maxTerms; @@ -150,12 +140,10 @@ public static synchronized void setDiagnostics(boolean diagnostics) { private final long parquetRowCount; private final BufferPool bufferPool = new BufferPool(); - private final Map columnReaders = new java.util.concurrent.ConcurrentHashMap<>(); private final Map dataFusionColumnReaders = new java.util.concurrent.ConcurrentHashMap<>(); private final java.util.List dedicatedReaders = java.util.Collections.synchronizedList(new java.util.ArrayList<>()); /** Optional per-query accumulator; propagated to each column reader so its stats roll up at close. */ - private QueryParquetStats queryStats; private boolean closed; @@ -202,52 +190,27 @@ public ParquetDocValuesProducer(SegmentReadState state, MapperService mapperServ } } - /** - * Attaches the per-query accumulator. The accumulator is propagated to every column reader - * (existing and future) so each reader's stats roll up into the query total when it closes. - */ - public void setQueryStats(QueryParquetStats queryStats) { - this.queryStats = queryStats; - if (queryStats != null) { - for (ParquetColumnReader reader : columnReaders.values()) { - reader.setQueryStats(queryStats); - } - } - } - // ── DocValuesProducer API ── @Override public NumericDocValues getNumeric(FieldInfo field) throws IOException { ensureOpen(); validate(field, DocValuesType.NUMERIC); - if (useDataFusionDecodePath) { - return new ParquetNumericDocValues(dataFusionReaderFor(field, false), maxDoc); - } - ParquetColumnReader reader = readerFor(field, false); - return new ParquetNumericDocValues(reader, maxDoc); + return new ParquetNumericDocValues(dataFusionReaderFor(field, false), maxDoc); } @Override public SortedNumericDocValues getSortedNumeric(FieldInfo field) throws IOException { ensureOpen(); validate(field, DocValuesType.SORTED_NUMERIC); - if (useDataFusionDecodePath) { - return new ParquetSortedNumericDocValues(dataFusionReaderFor(field, true), maxDoc); - } - ParquetColumnReader reader = readerFor(field, true); - return new ParquetSortedNumericDocValues(reader, maxDoc); + return new ParquetSortedNumericDocValues(dataFusionReaderFor(field, true), maxDoc); } @Override public BinaryDocValues getBinary(FieldInfo field) throws IOException { ensureOpen(); validate(field, DocValuesType.BINARY); - if (useDataFusionDecodePath) { - return new ParquetBinaryDocValues(dataFusionReaderFor(field, false), maxDoc); - } - ParquetColumnReader reader = readerFor(field, false); - return new ParquetBinaryDocValues(reader, maxDoc); + return new ParquetBinaryDocValues(dataFusionReaderFor(field, false), maxDoc); } @Override @@ -289,14 +252,7 @@ public DocValuesSkipper getSkipper(FieldInfo field) throws IOException { // define independent Lucene document ranges. Do not expose unsafe stats. return null; } - if (useDataFusionDecodePath) { - return new ParquetDocValuesSkipper(dataFusionReaderFor(field, false).pageIndex(), maxDoc); - } - // Match the repeated flag the field's DV accessor will use — readerFor caches by field - // name, so opening here with a mismatched flag would poison the cache for the accessor. - boolean repeated = field.getDocValuesType() == DocValuesType.SORTED_NUMERIC; - ParquetColumnReader reader = readerFor(field, repeated); - return new ParquetDocValuesSkipper(reader.pageIndex(), maxDoc); + return new ParquetDocValuesSkipper(dataFusionReaderFor(field, false).pageIndex(), maxDoc); } /** @@ -329,16 +285,6 @@ public void close() throws IOException { // ([PARQUET_DV_QUERY_STATS]); no per-segment detail line here. closed = true; IOException first = null; - for (ParquetColumnReader reader : columnReaders.values()) { - try { - reader.close(); - } catch (IOException | RuntimeException e) { - if (first == null && e instanceof IOException io) { - first = io; - } - // Suppress per-reader errors so every reader gets a chance to close. - } - } for (DataFusionColumnReader reader : dataFusionColumnReaders.values()) { try { reader.close(); @@ -361,7 +307,6 @@ public void close() throws IOException { } } dedicatedReaders.clear(); - columnReaders.clear(); dataFusionColumnReaders.clear(); bufferPool.close(); if (first != null) { @@ -401,17 +346,6 @@ private ParquetPhysicalType physicalType(FieldInfo field) { }; } - private synchronized ParquetColumnReader readerFor(FieldInfo field, boolean repeated) throws IOException { - ParquetColumnReader reader = columnReaders.get(field.getName()); - if (reader == null) { - reader = ParquetColumnReader.open(parquetFile, field.getName(), physicalType(field), repeated, bufferPool); - reader.setQueryStats(queryStats); - columnReaders.put(field.getName(), reader); - } - return reader; - } - - /** * A dedicated (non-shared) binary reader for one streaming sorted iterator. Concurrent * segment-search slices each obtain their own DocValues instance; sharing one forward @@ -438,21 +372,18 @@ long nonNullRowCount(FieldInfo field) throws IOException { } private synchronized BinaryPageReader binaryReaderFor(FieldInfo field, boolean repeated) throws IOException { - if (useDataFusionDecodePath) { - DataFusionColumnReader reader = DataFusionColumnReader.open( - parquetFile, - field.getName(), - physicalType(field), - repeated, - bufferPool, - dataFusionInitialBatchSize - ); - dedicatedReaders.add(reader); - return reader; - } - // codec_native path: keep the shared per-field reader (its iterators tolerate sharing, - // and its pool slots are not instance-scoped). - return readerFor(field, repeated); + // Sorted iterators need instance-scoped cursors (shared producers are accessed + // concurrently), so each gets a dedicated reader with instance-unique pool slots. + DataFusionColumnReader reader = DataFusionColumnReader.open( + parquetFile, + field.getName(), + physicalType(field), + repeated, + bufferPool, + dataFusionInitialBatchSize + ); + dedicatedReaders.add(reader); + return reader; } private synchronized DataFusionColumnReader dataFusionReaderFor(FieldInfo field, boolean repeated) throws IOException { diff --git a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/codec/RowIdRemappingDocValues.java b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/codec/RowIdRemappingDocValues.java index 4815f01aa247e..eed1f03ca1540 100644 --- a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/codec/RowIdRemappingDocValues.java +++ b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/codec/RowIdRemappingDocValues.java @@ -15,7 +15,6 @@ import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.util.BytesRef; -import org.opensearch.parquet.codec.cache.RowIdStats; import java.io.IOException; @@ -310,36 +309,9 @@ public TermsEnum termsEnum() throws IOException { * resolver (backed by its own {@code __row_id__} iterator) per codec iterator. */ static RowIdResolver resolverFrom(SortedNumericDocValues rowIdDocValues) { - return resolverFrom(rowIdDocValues, null); - } - - /** - * As {@link #resolverFrom(SortedNumericDocValues)}, but records the lookup count into - * {@code stats} (and marks IDENTITY when there is no row-id field). When {@code stats} is null, - * no instrumentation is added (the resolver is the plain hot path). - * - *

Only the lookup count is recorded — never per-call timing. The per-document work is on the - * order of tens of nanoseconds, comparable to {@code System.nanoTime()} itself, so timing each - * call cannot produce a trustworthy figure; the actual wall-clock spent here is measured with a - * CPU flamegraph instead (see {@link RowIdStats}). - */ - static RowIdResolver resolverFrom(SortedNumericDocValues rowIdDocValues, RowIdStats stats) { if (rowIdDocValues == null) { - if (stats != null) { - stats.markIdentity(); - } return RowIdResolver.IDENTITY; } - if (stats == null) { - return docId -> { - if (rowIdDocValues.advanceExact(docId) == false) { - throw new IllegalStateException( - "missing __row_id__ doc value for docId=" + docId + "; cannot translate to Parquet row position" - ); - } - return rowIdDocValues.nextValue(); - }; - } return docId -> { if (rowIdDocValues.advanceExact(docId) == false) { throw new IllegalStateException( @@ -347,9 +319,7 @@ static RowIdResolver resolverFrom(SortedNumericDocValues rowIdDocValues, RowIdSt ); } // __row_id__ is single-valued; take the first (and only) value. - long rowId = rowIdDocValues.nextValue(); - stats.recordLookup(); - return rowId; + return rowIdDocValues.nextValue(); }; } } diff --git a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/codec/SharedProducerRegistry.java b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/codec/SharedProducerRegistry.java index 1f448cdb2254b..2ab54a71830e4 100644 --- a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/codec/SharedProducerRegistry.java +++ b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/codec/SharedProducerRegistry.java @@ -43,11 +43,8 @@ private SharedProducerRegistry() {} * on first use. Returns {@code null} when the segment exposes no core cache helper (no safe * lifecycle to attach to — callers must fail rather than leak). */ - static ParquetDocValuesProducer get( - IndexReader.CacheHelper coreHelper, - SegmentReadState segmentReadState, - MapperService mapperService - ) throws IOException { + static ParquetDocValuesProducer get(IndexReader.CacheHelper coreHelper, SegmentReadState segmentReadState, MapperService mapperService) + throws IOException { if (coreHelper == null) { return null; } diff --git a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/codec/UninvertedOrdinals.java b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/codec/UninvertedOrdinals.java index c105b844e497f..eb2aaa41784ba 100644 --- a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/codec/UninvertedOrdinals.java +++ b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/codec/UninvertedOrdinals.java @@ -19,8 +19,8 @@ import org.apache.lucene.store.MMapDirectory; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.LongValues; -import org.apache.lucene.util.packed.DirectWriter; import org.apache.lucene.util.packed.DirectReader; +import org.apache.lucene.util.packed.DirectWriter; import org.apache.lucene.util.packed.PackedInts; import java.io.Closeable; @@ -192,16 +192,7 @@ static UninvertedOrdinals build( + "refusing uninverted ordinals to avoid silent undercounts" ); } - return new UninvertedOrdinals( - directory, - input, - ords, - checkpoints.toArray(new BytesRef[0]), - terms, - (int) termCount, - size, - fileName - ); + return new UninvertedOrdinals(directory, input, ords, checkpoints.toArray(new BytesRef[0]), terms, (int) termCount, size, fileName); } /** The segment ordinal for {@code doc}, or -1 when the document has no value. */ diff --git a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/codec/UninvertedOrdinalsCache.java b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/codec/UninvertedOrdinalsCache.java index ac429710e3daf..a2d8b749a1c21 100644 --- a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/codec/UninvertedOrdinalsCache.java +++ b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/codec/UninvertedOrdinalsCache.java @@ -65,8 +65,7 @@ public static void shutdown() { * rebuild fixes (segment data moved on after an unclean stop). Only a failure on a FRESH * build is genuine (unindexed stored values) and latches the field ineligible. */ - private static UninvertedOrdinals buildWithRetry(String fileKey, Terms terms, int maxDoc, long expectedNonNullDocs) - throws IOException { + private static UninvertedOrdinals buildWithRetry(String fileKey, Terms terms, int maxDoc, long expectedNonNullDocs) throws IOException { String fileName = "parquet-ords-" + fileKey + ".ord"; boolean preExisting = java.nio.file.Files.exists(ORDS_DIR.resolve(fileName)); try { diff --git a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/codec/cache/CacheStats.java b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/codec/cache/CacheStats.java deleted file mode 100644 index c2cd9a71f36ef..0000000000000 --- a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/codec/cache/CacheStats.java +++ /dev/null @@ -1,124 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.parquet.codec.cache; - -/** - * Per-column hit/miss counters for the Parquet DocValues cache layers, used to measure how much - * each caching layer contributes during a query. One instance is owned by each - * {@code ParquetColumnReader}; it is single-threaded (one segment per query thread), so counters - * are plain {@code long}s with no synchronization. - * - *

Layer mapping (per the codec design): - *

    - *
  • Layer 3 — OffsetIndex jump table ({@code pageForRow} binary search), consulted on every miss.
    • - *
  • Layer 4 — page-stat all-nulls skip; a miss resolved without decoding the page.
    • - *
  • FFM — calls that cross the native boundary (page decodes and slow-path single/repeated - * reads). This is the cost the upper layers exist to avoid.
    • - *
- */ -public final class CacheStats { - - // Layer 3 — OffsetIndex jump-table lookups (one per miss). - private long pageIndexLookups; - - // Layer 4 — all-nulls page skips (a miss resolved with no decode). - private long allNullPageSkips; - - // FFM boundary crossings. - private long pageDecodes; // parquet_decode_page_at_row - private long slowValueReads; // parquet_read_value_at_row (single) - private long slowRepeatedReads; // parquet_read_repeated_at_row - - // Phase timers (nanos), accumulated only when the timing logger is at TRACE. - private long loadPageNanos; // total time in loadPageContaining - private long decodePageNanos; // total time in decodePage - private long ffmDecodeNanos; // time spent in the parquet_decode_page_at_row FFM crossing(s) - - /** Records a Layer 3 jump-table lookup ({@code pageForRow}). */ - public void pageIndexLookup() { - pageIndexLookups++; - } - - /** Records a Layer 4 all-nulls page skip (miss resolved without a decode). */ - public void allNullPageSkip() { - allNullPageSkips++; - } - - /** Records an FFM page decode crossing. */ - public void pageDecode() { - pageDecodes++; - } - - /** Records an FFM slow-path single-value read crossing. */ - public void slowValueRead() { - slowValueReads++; - } - - /** Records an FFM slow-path repeated-value read crossing. */ - public void slowRepeatedRead() { - slowRepeatedReads++; - } - - public long pageDecodes() { - return pageDecodes; - } - - public long allNullPageSkips() { - return allNullPageSkips; - } - - public long pageIndexLookups() { - return pageIndexLookups; - } - - public long slowValueReads() { - return slowValueReads; - } - - public long slowRepeatedReads() { - return slowRepeatedReads; - } - - /** Total FFM boundary crossings across all access paths. */ - public long ffmCrossings() { - return pageDecodes + slowValueReads + slowRepeatedReads; - } - - /** Adds elapsed nanos spent in {@code loadPageContaining}. */ - public void addLoadPageNanos(long n) { - loadPageNanos += n; - } - - /** Adds elapsed nanos spent in {@code decodePage}. */ - public void addDecodePageNanos(long n) { - decodePageNanos += n; - } - - /** Adds elapsed nanos spent in the {@code parquet_decode_page_at_row} FFM crossing. */ - public void addFfmDecodeNanos(long n) { - ffmDecodeNanos += n; - } - - public long loadPageNanos() { - return loadPageNanos; - } - - public long decodePageNanos() { - return decodePageNanos; - } - - public long ffmDecodeNanos() { - return ffmDecodeNanos; - } - - /** True when no access has been recorded (used to suppress empty summaries). */ - public boolean isEmpty() { - return pageDecodes == 0 && pageIndexLookups == 0 && allNullPageSkips == 0 && slowValueReads == 0 && slowRepeatedReads == 0; - } -} diff --git a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/codec/cache/QueryParquetStats.java b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/codec/cache/QueryParquetStats.java deleted file mode 100644 index 69fa3ba1109f4..0000000000000 --- a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/codec/cache/QueryParquetStats.java +++ /dev/null @@ -1,157 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.parquet.codec.cache; - -import java.util.Locale; -import java.util.concurrent.ConcurrentLinkedQueue; - -/** - * Query-scoped accumulator that sums per-column {@link CacheStats} across every segment touched by - * a single search, so the Parquet read-path cost can be summarized in one log line per query. - * - *

One instance is created per search (by {@code ParquetDocValuesDirectoryReader}) and shared by - * all the per-segment {@code ParquetColumnReader}s that search opens. Each column reader - * {@link #register(CacheStats) registers} its {@link CacheStats} when it is opened; the values are - * summed live at {@link #summary(long, long, long)} time. Registering at open (rather than merging at close) - * means the roll-up does not depend on reader-close ordering — by the time the per-query summary is - * produced (end of search), every registered reader has finished collecting, so the live sums are - * final. The registry is a {@link ConcurrentLinkedQueue} so concurrent segment slices can register - * safely, and each reader mutates only its own {@link CacheStats} during collection. - */ -public final class QueryParquetStats { - - private final ConcurrentLinkedQueue registered = new ConcurrentLinkedQueue<>(); - - // Liquid-cache event counters are process-wide and monotonic. We snapshot them at query start - // (only when the stats summary is enabled — the snapshot itself is an FFM crossing) and report - // the delta in the summary. Meaningful for the single-query-at-a-time benchmark case; under - // concurrent queries the delta is an over-count (other queries' liquid events land in the window), - // which is acceptable for a diagnostic. -1 baseline means "not captured" → liquid line suppressed. - private long liquidHitsBase = -1; - private long liquidMissesBase = -1; - private long liquidPutsBase = -1; - - // Native page-decode phase timers are process-wide and cumulative (nanos). Same baseline/delta - // treatment as the liquid counters above; -1 baseline means "not captured" so the rust portion - // of the timing line is suppressed. - private long getNanosBase = -1; - private long decodeNanosBase = -1; - private long putNanosBase = -1; - - /** - * Records the liquid counter baseline at query start so {@link #summary(long, long, long)} can report per-query - * deltas. Call only when the summary will actually be emitted (this reads native counters over - * FFM); skipping it leaves the liquid line out of the summary at zero cost. - */ - public void captureLiquidBaseline(long hits, long misses, long puts) { - this.liquidHitsBase = hits; - this.liquidMissesBase = misses; - this.liquidPutsBase = puts; - } - - /** - * Records the native phase-timer baseline (cumulative nanos) at query start so - * {@link #timingSummary(long, long, long)} can report the per-query rust delta. Call only when - * the timing summary will be emitted (this reads native timers over FFM). - */ - public void captureTimingBaseline(long getNanos, long decodeNanos, long putNanos) { - this.getNanosBase = getNanos; - this.decodeNanosBase = decodeNanos; - this.putNanosBase = putNanos; - } - - /** Registers a column reader's stats; its counters are summed live when {@link #summary(long, long, long)} runs. */ - public void register(CacheStats s) { - if (s != null) { - registered.add(s); - } - } - - /** True when nothing was recorded (used to suppress an empty summary). */ - public boolean isEmpty() { - return registered.isEmpty(); - } - - /** - * A single-line, human-readable per-query summary. The {@code liquid*Now} arguments are the - * current process-wide liquid counters (read by the caller over FFM); the reported liquid line - * is their delta from the baseline captured at query start. Pass any value when no baseline was - * captured — the liquid line is suppressed unless a baseline exists. - */ - public String summary(long liquidHitsNow, long liquidMissesNow, long liquidPutsNow) { - long columns = 0; - long jumpTableLookups = 0, allNullSkips = 0; - long pageDecodes = 0, slowValueReads = 0, slowRepeatedReads = 0; - for (CacheStats s : registered) { - columns++; - jumpTableLookups += s.pageIndexLookups(); - allNullSkips += s.allNullPageSkips(); - pageDecodes += s.pageDecodes(); - slowValueReads += s.slowValueReads(); - slowRepeatedReads += s.slowRepeatedReads(); - } - - // Liquid line: per-query deltas from the baseline captured at query start. Suppressed when - // no baseline was taken (stats disabled path). hits = pages served from liquid, decoded = - // liquid misses that fell through to a Parquet decode, puts = pages inserted into liquid. - String liquidLine = ""; - if (liquidHitsBase >= 0) { - long lh = liquidHitsNow - liquidHitsBase; - long lm = liquidMissesNow - liquidMissesBase; - long lp = liquidPutsNow - liquidPutsBase; - long lget = lh + lm; - double lHitRate = lget == 0 ? 0.0 : (double) lh / lget * 100.0; - liquidLine = String.format(Locale.ROOT, " | liquid: hits=%d decoded=%d puts=%d (liquidHitRate=%.2f%%)", lh, lm, lp, lHitRate); - } - return String.format( - Locale.ROOT, - "segments/columns=%d | L3 jumpTableLookups=%d | L4 allNullSkips=%d | " - + "FFM: pageDecodes=%d slowValueReads=%d slowRepeatedReads=%d%s", - columns, - jumpTableLookups, - allNullSkips, - pageDecodes, - slowValueReads, - slowRepeatedReads, - liquidLine - ); - } - - /** - * A single-line, human-readable per-query timing summary (all values in milliseconds). The Java - * timers (loadPage/decodePage/ffmCrossing) are summed live from the registered {@link CacheStats}; - * the {@code *Now} arguments are the current process-wide native phase timers (read by the caller - * over FFM) and are reported as a delta from the baseline captured at query start. The rust portion - * is suppressed when no baseline was captured. - */ - public String timingSummary(long getNow, long decodeNow, long putNow) { - long loadPageNanos = 0, decodePageNanos = 0, ffmDecodeNanos = 0; - for (CacheStats s : registered) { - loadPageNanos += s.loadPageNanos(); - decodePageNanos += s.decodePageNanos(); - ffmDecodeNanos += s.ffmDecodeNanos(); - } - - String rustLine = ""; - if (getNanosBase >= 0) { - long getD = getNow - getNanosBase; - long decodeD = decodeNow - decodeNanosBase; - long putD = putNow - putNanosBase; - rustLine = String.format(Locale.ROOT, " | rust: get=%.1fms decode=%.1fms put=%.1fms", getD / 1e6, decodeD / 1e6, putD / 1e6); - } - return String.format( - Locale.ROOT, - "loadPage=%.1fms decodePage=%.1fms ffmCrossing=%.1fms%s", - loadPageNanos / 1e6, - decodePageNanos / 1e6, - ffmDecodeNanos / 1e6, - rustLine - ); - } -} diff --git a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/codec/cache/RowIdStats.java b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/codec/cache/RowIdStats.java deleted file mode 100644 index 7ea9e50ff31ac..0000000000000 --- a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/codec/cache/RowIdStats.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.parquet.codec.cache; - -/** - * Per-resolver counters for the docId→Parquet-row translation layer (the - * {@code RowIdRemappingDocValues} resolver backed by {@code __row_id__}). - * - *

One instance is created per codec DocValues iterator (each builds its own resolver), so this is - * single-threaded for the lifetime of one iterator — counters are plain {@code long}s with no - * synchronization, mirroring {@link CacheStats}. It is registered once with the query-scoped - * {@code QueryParquetStats} and summed live at end of query. - * - *

Counts only — no per-call timing

- * Only counts are tracked here ({@link #lookups()} and the {@link #isIdentity() identity} - * flag), never per-call time. {@code toRowId(docId)} runs once per document (up to 100M+ times per - * query); the work per call is on the order of tens of nanoseconds, comparable to - * {@code System.nanoTime()} itself, so code-timing each call cannot produce a trustworthy figure - * (it inflates and, when extrapolated, exceeds the whole query time). The honest signal is the - * exact lookup count here; the actual wall-clock spent in this layer is obtained from a CPU - * flamegraph (async-profiler), which attributes real per-method time without instrumentation - * overhead. - */ -public final class RowIdStats { - - /** True when this resolver is the no-op IDENTITY mapping (segment has docId == rowId). */ - private boolean identity; - - /** Number of {@code toRowId} calls that performed a {@code __row_id__} lookup. */ - private long lookups; - - /** Marks this resolver as the IDENTITY (no-op) mapping. */ - public void markIdentity() { - identity = true; - } - - /** Records one {@code __row_id__} lookup (called per document on a backed resolver). */ - public void recordLookup() { - lookups++; - } - - public boolean isIdentity() { - return identity; - } - - public long lookups() { - return lookups; - } - - /** True when this resolver did no work (not used / no lookups and not marked identity). */ - public boolean isEmpty() { - return identity == false && lookups == 0; - } -} diff --git a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/rest/ParquetLiquidCacheClearRestAction.java b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/rest/ParquetLiquidCacheClearRestAction.java deleted file mode 100644 index 7874329419eef..0000000000000 --- a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/rest/ParquetLiquidCacheClearRestAction.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.parquet.rest; - -import org.opensearch.common.annotation.ExperimentalApi; -import org.opensearch.core.rest.RestStatus; -import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.parquet.bridge.RustBridge; -import org.opensearch.rest.BaseRestHandler; -import org.opensearch.rest.BytesRestResponse; -import org.opensearch.rest.RestRequest; -import org.opensearch.transport.client.node.NodeClient; - -import java.io.IOException; -import java.util.List; - -import static org.opensearch.rest.RestRequest.Method.POST; - -/** - * REST handler for {@code POST /_plugins/parquet/liquid_cache/_clear}. - * - *

Clears the codec-owned liquid decoded-page cache (in-memory index + spilled {@code t4} - * entries) on the coordinating node, without disabling it — the next reads re-decode and - * re-populate. Intended for cold-start benchmarking so a "cold" measurement no longer requires a - * full node restart. A no-op when the cache is disabled or not yet built. - * - *

Node-local: the clear runs in-process against this node's cache static. On a multi-node - * cluster, hit each node (or route via {@code _nodes}) to clear all — for the single-node bench - * box this handler is sufficient. - * - * @opensearch.experimental - */ -@ExperimentalApi -public final class ParquetLiquidCacheClearRestAction extends BaseRestHandler { - - @Override - public String getName() { - return "parquet_liquid_cache_clear_action"; - } - - @Override - public List routes() { - return List.of(new Route(POST, "/_plugins/parquet/liquid_cache/_clear")); - } - - @Override - protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { - return channel -> { - RustBridge.liquidCacheClear(); - try (XContentBuilder builder = channel.newBuilder()) { - builder.startObject(); - builder.field("acknowledged", true); - builder.field("cleared", "parquet_liquid_cache"); - builder.endObject(); - channel.sendResponse(new BytesRestResponse(RestStatus.OK, builder)); - } - }; - } -} diff --git a/sandbox/plugins/parquet-data-format/src/main/rust/src/ffm.rs b/sandbox/plugins/parquet-data-format/src/main/rust/src/ffm.rs index 289cf253ff677..1ef89adeb3c87 100644 --- a/sandbox/plugins/parquet-data-format/src/main/rust/src/ffm.rs +++ b/sandbox/plugins/parquet-data-format/src/main/rust/src/ffm.rs @@ -11,45 +11,16 @@ //! Return convention: `>= 0` success, `< 0` error pointer (negate to get ptr, //! call `native_error_message`/`native_error_free`). -use std::collections::HashMap; use std::slice; use std::str; -use std::sync::atomic::{AtomicBool, AtomicI64, AtomicU64, Ordering}; -use std::sync::Mutex; -use std::time::Instant; -use lazy_static::lazy_static; -use native_bridge_common::{ffm_safe, log_debug}; +use native_bridge_common::ffm_safe; -/// Optional page-decode phase timing (T5/T6/T7), gated so it costs nothing when off. /// /// `TIMING_ENABLED` is flipped by Java (`parquet_timing_set_enabled`) only while the /// `org.opensearch.parquet.timing` logger is at TRACE. When false, the decode path does a single /// relaxed atomic load per page and takes no `Instant::now()` — identical latency to no timing. /// When true, per-page elapsed nanos are accumulated into three counters, read back via -/// `parquet_timing_snapshot`: -/// - GET : time in `get_page_into_outbuf` (liquid fetch + memcpy on a hit; cheap miss check) -/// - DECODE : time in `decode_primitive_page` (the actual Parquet decode into the out-buffer) -/// - PUT : time in `put_page_from_outbuf` (build Arrow array + insert into liquid) -pub(crate) mod timing { - use super::*; - pub(crate) static TIMING_ENABLED: AtomicBool = AtomicBool::new(false); - pub(crate) static GET_NANOS: AtomicU64 = AtomicU64::new(0); - pub(crate) static DECODE_NANOS: AtomicU64 = AtomicU64::new(0); - pub(crate) static PUT_NANOS: AtomicU64 = AtomicU64::new(0); - - #[inline] - pub(crate) fn on() -> bool { - TIMING_ENABLED.load(Ordering::Relaxed) - } - - /// Adds `start.elapsed()` nanos to `counter`. Only call when `on()` was already true. - #[inline] - pub(crate) fn record(counter: &AtomicU64, start: Instant) { - counter.fetch_add(start.elapsed().as_nanos() as u64, Ordering::Relaxed); - } -} - use crate::native_settings::NativeSettings; use crate::field_config::FieldConfig; use crate::merge; @@ -817,1641 +788,3 @@ pub unsafe extern "C" fn parquet_get_pool_stats(out_buf: *mut i64) { *out_buf.add(i) = *val as i64; } } -// Parquet column reader (DocValues codec — Strategy 1 FFM bridge) -// --------------------------------------------------------------------------- -// -// Read-only per-column random-access reader used by the Lucene -// `ParquetDocValuesProducer`. The reader exposes the column's physical values -// by row position so the Java side can materialise per-document doc values. -// -// Return convention (consistent with the rest of this file): -// - `>= 0` success. For reads, `0` (`RC_OK`) means "value(s) written"; the -// positive sentinel `RC_OVERFLOW` (1) means "caller buffer too small — -// required sizes were written to the out-parameters, retry once". -// - `< 0` error pointer (negate, then `native_error_message`/`native_error_free`). -// -// NOTE on the overflow sentinel: the design sketch refers to this as -// `-E_OVERFLOW`, but this file's established FFM contract reserves *all* -// negative returns for heap error pointers (negated `Box`/`CString` address). -// Returning a small negative constant would be indistinguishable from — and -// dereferenced as — an error pointer by `native_error_message`, corrupting -// memory. We therefore signal overflow with a positive status code, mirroring -// the existing `parquet_finalize_writer` precedent (`Ok(1)` == "no writer"). -// The Java wrapper checks `ret == RC_OVERFLOW` before the `ret < 0` error path. - -use std::fs::File; -use std::sync::MutexGuard; - -use parquet::basic::Type as PhysicalType; -use parquet::column::reader::{ColumnReader, ColumnReaderImpl}; -use parquet::data_type::DataType as ParquetDataType; -use parquet::file::page_index::column_index::ColumnIndexMetaData; -use parquet::file::reader::{FileReader, SerializedFileReader}; -use parquet::file::serialized_reader::ReadOptionsBuilder; - -/// Read succeeded; value(s) written to the caller buffers. -pub(crate) const RC_OK: i64 = 0; -/// A caller buffer was too small. Required sizes were written to the -/// out-parameters; the caller should grow its buffers and retry once. -pub(crate) const RC_OVERFLOW: i64 = 1; - -/// `expected_type` discriminants exchanged with Java -/// (matches `ParquetPhysicalType` on the Java side). -const TYPE_INT32: i32 = 0; -const TYPE_INT64: i32 = 1; -const TYPE_FLOAT: i32 = 2; -const TYPE_DOUBLE: i32 = 3; -const TYPE_BOOL: i32 = 4; -const TYPE_BYTE_ARRAY: i32 = 5; - -/// One open per-column reader. Owns the file handle (via `SerializedFileReader`) -/// and the row-group layout needed to translate a global row position into a -/// `(row_group, local_offset)` pair. Random access reopens the relevant row -/// group per call — this is the documented *slow path*; the hot path goes -/// through `parquet_decode_page_at_row` (page-resident caching on the Java side). -struct ColumnReaderState { - reader: SerializedFileReader, - /// Leaf column index within the Parquet schema descriptor. - leaf_idx: usize, - /// Physical type of the column (validated against the caller's expectation). - physical_type: PhysicalType, - /// True when the column has a repetition level > 0 (multi-valued). - repeated: bool, - /// Max definition level of the column (0 = required; >0 = optional/nested). - max_def_level: i16, - /// Total number of rows (records) in the file. - row_count: i64, - /// Global row index of the first row in each row group. - rg_first_row: Vec, - /// Number of rows in each row group. - rg_num_rows: Vec, - /// Per-page layout (Layer 3 jump table + Layer 4 page stats), ascending by - /// `global_first_row`. Built once at `open()` from the Parquet OffsetIndex + - /// ColumnIndex when present, else one entry per row group as a fallback. - pages: Vec, - /// Codec-local file id (path→id) for the cross-query decoded-page cache key. See - /// `crate::liquid_page_cache`. - liquid_file_id: u32, - /// Sequential-decode cursor retained by `parquet_decode_page_at_row` across - /// calls. `None` until the first page decode and whenever it has been - /// invalidated (row-group change, backwards seek, or a decode error). The - /// random-access slow paths (`parquet_read_value_at_row`, - /// `parquet_read_repeated_at_row`) never touch it. - cursor: Option, - /// Pre-allocated scratch buffers for page decoding, reused across calls to - /// `parquet_decode_page_at_row` to eliminate per-call heap allocations. - /// Vecs keep their capacity across `.clear()` so after the first cold miss - /// all subsequent decodes are allocation-free on the steady-state path. - scratch: DecodeScratch, -} - -/// Retained cursor for `parquet_decode_page_at_row`. Keeps the typed column -/// reader open across consecutive page decodes within the same row group so -/// the hot ascending-doc-ID path skips forward from its current position -/// instead of reopening the row group (re-reading row-group metadata and the -/// dictionary page) for every page. -struct CursorState { - /// The row group index this cursor was opened for. - rg_idx: usize, - /// The typed column reader, retained across pages within the same row group. - col_reader: ColumnReader, - /// Global row position: the next record read from this cursor would start at this row. - position: i64, -} - -/// Pre-allocated scratch buffers for page decoding. Kept on `ColumnReaderState` and cleared (not -/// reallocated) between calls, so steady-state decoding is allocation-free after the first miss. -struct DecodeScratch { - /// Definition levels returned by `read_records`. - def_levels: Vec, - /// Dense non-null values read from the page (re-interpretable as any primitive via transmute). - /// Stored as raw bytes; capacity is in units of the largest primitive (i64 = 8 bytes). - values_i64: Vec, - values_i32: Vec, - values_f32: Vec, - values_f64: Vec, - values_bool: Vec, -} - -impl DecodeScratch { - fn new() -> Self { - DecodeScratch { - def_levels: Vec::new(), - values_i64: Vec::new(), - values_i32: Vec::new(), - values_f32: Vec::new(), - values_f64: Vec::new(), - values_bool: Vec::new(), - } - } -} - -/// One row-aligned page in the column (or one row group, in the no-page-index -/// fallback). All row indices are global (file-relative). -#[derive(Clone)] -struct PageEntry { - /// Global index of the first row in the page. - global_first_row: i64, - /// Number of rows in the page. - num_rows: i64, - /// Byte offset of the page in the file (Layer 3). 0 when unknown. - file_offset: i64, - /// Compressed page size in bytes (Layer 3). 0 when unknown. - compressed_size: i32, - /// Number of nulls in the page (Layer 4). -1 when unknown. - null_count: i64, - /// Min value raw bits (Layer 4); meaningful only for numeric columns with a - /// page index, else 0. - min_long: i64, - /// Max value raw bits (Layer 4); meaningful only for numeric columns with a - /// page index, else 0. - max_long: i64, - /// Row group containing the page. - rg_idx: usize, - /// Index of the page's first row within its row group. - local_first_row: i64, -} - -/// Gets or builds the per-file metadata cache entry. First call for a given path parses the -/// footer + page index; subsequent calls (other columns, other queries) get an Arc clone in -/// O(1). The page layout for a specific column is computed lazily within the cached entry. -fn get_or_build_file_metadata(filename: &str) -> Result, String> { - let mut cache = FILE_METADATA_CACHE - .lock() - .map_err(|_| "file metadata cache mutex poisoned".to_string())?; - if let Some(entry) = cache.get(filename) { - return Ok(std::sync::Arc::clone(entry)); - } - - // First access to this file: parse footer + page index. - let file = File::open(filename).map_err(|e| format!("Failed to open '{}': {}", filename, e))?; - let options = ReadOptionsBuilder::new().with_page_index().build(); - let reader = SerializedFileReader::new_with_options(file, options) - .map_err(|e| format!("Failed to read parquet metadata '{}': {}", filename, e))?; - - let metadata = reader.metadata(); - let schema = metadata.file_metadata().schema_descr_ptr(); - let row_count = metadata.file_metadata().num_rows(); - - let n_rg = metadata.num_row_groups(); - let mut rg_first_row = Vec::with_capacity(n_rg); - let mut rg_num_rows = Vec::with_capacity(n_rg); - let mut acc = 0i64; - for i in 0..n_rg { - let rn = metadata.row_group(i).num_rows(); - rg_first_row.push(acc); - rg_num_rows.push(rn); - acc += rn; - } - - // Pre-compute per-column descriptors. - let columns: Vec<_> = (0..schema.num_columns()) - .map(|i| { - let d = schema.column(i); - (i, d.physical_type(), d.max_rep_level(), d.max_def_level()) - }) - .collect(); - - // Pre-compute page layouts for ALL columns (small per column; avoids per-column re-parse). - let mut column_pages = HashMap::new(); - for &(leaf_idx, phys, _, _) in &columns { - let pages = build_page_layout(metadata, leaf_idx, phys, &rg_first_row, &rg_num_rows); - column_pages.insert(leaf_idx, pages); - } - - let entry = std::sync::Arc::new(FileMetadataCache { - schema, - row_count, - rg_first_row, - rg_num_rows, - column_pages, - columns, - }); - cache.insert(filename.to_string(), std::sync::Arc::clone(&entry)); - Ok(entry) -} - -impl ColumnReaderState { - fn open(filename: &str, column: &str, expected_type: i32) -> Result { - // Use the node-level metadata cache — first call parses; subsequent calls are O(1). - let fmc = get_or_build_file_metadata(filename)?; - - // Resolve column from cached schema descriptor (no file I/O, no re-parse). - let mut found: Option<(usize, PhysicalType, i16, i16)> = None; - for i in 0..fmc.schema.num_columns() { - let descr = fmc.schema.column(i); - if descr.name() == column || descr.path().string() == column { - found = Some((i, descr.physical_type(), descr.max_rep_level(), descr.max_def_level())); - break; - } - } - let (leaf_idx, phys, max_rep, max_def) = found.ok_or_else(|| { - format!("Column '{}' not found in parquet file '{}'", column, filename) - })?; - - let actual = physical_type_code(phys); - if actual != expected_type { - return Err(format!( - "Column '{}' physical type mismatch in '{}': expected type code {}, found {:?} (code {})", - column, filename, expected_type, phys, actual - )); - } - - // Get the pre-computed page layout from the cache. - let pages = fmc.column_pages.get(&leaf_idx) - .cloned() - .unwrap_or_default(); - - // Open a file handle for this column reader WITH the page index loaded: the retained - // cursor's skip_records uses the page index internally for efficient page-hopping. The - // per-file metadata cache above saves the JAVA-FACING costs (schema resolution, page - // layout computation, ColumnPageIndex FFM marshal) — this reader's page-index load is - // cheap (already in OS page cache from the first parse) and necessary for decode perf. - let file = File::open(filename).map_err(|e| format!("Failed to open '{}': {}", filename, e))?; - let options = ReadOptionsBuilder::new().with_page_index().build(); - let reader = SerializedFileReader::new_with_options(file, options) - .map_err(|e| format!("Failed to read parquet '{}': {}", filename, e))?; - - let liquid_file_id = crate::liquid_page_cache::file_id(filename); - - Ok(ColumnReaderState { - reader, - leaf_idx, - physical_type: phys, - repeated: max_rep > 0, - max_def_level: max_def, - row_count: fmc.row_count, - rg_first_row: fmc.rg_first_row.clone(), - rg_num_rows: fmc.rg_num_rows.clone(), - pages, - liquid_file_id, - cursor: None, - scratch: DecodeScratch::new(), - }) - } - - /// Translate a global row position into `(row_group_index, local_offset)`. - fn locate(&self, row: i64) -> Result<(usize, i64), String> { - for i in 0..self.rg_first_row.len() { - let start = self.rg_first_row[i]; - let end = start + self.rg_num_rows[i]; - if row >= start && row < end { - return Ok((i, row - start)); - } - } - Err(format!("Row {} not found in any row group (row count {})", row, self.row_count)) - } - - /// Find the index of the page containing global row `row` (binary search over - /// the ascending page layout). - fn page_for_row(&self, row: i64) -> Result { - // partition_point finds the first page whose global_first_row > row; the - // page we want is the one immediately before it. - let p = self.pages.partition_point(|e| e.global_first_row <= row); - if p == 0 { - return Err(format!("Row {} precedes the first page (row count {})", row, self.row_count)); - } - let idx = p - 1; - let entry = &self.pages[idx]; - if row >= entry.global_first_row && row < entry.global_first_row + entry.num_rows { - Ok(idx) - } else { - Err(format!("Row {} not found in any page (row count {})", row, self.row_count)) - } - } -} - -/// Builds the per-page layout for a column. Prefers the Parquet OffsetIndex + -/// ColumnIndex (true page granularity); falls back to one entry per row group -/// when the file has no page index. -fn build_page_layout( - metadata: &parquet::file::metadata::ParquetMetaData, - leaf_idx: usize, - phys: PhysicalType, - rg_first_row: &[i64], - rg_num_rows: &[i64], -) -> Vec { - let n_rg = metadata.num_row_groups(); - let offset_index = metadata.offset_index(); - let column_index = metadata.column_index(); - - let mut pages: Vec = Vec::new(); - - for rg in 0..n_rg { - let oi_pages = offset_index - .and_then(|oi| oi.get(rg)) - .and_then(|cols| cols.get(leaf_idx)); - let ci = column_index - .and_then(|ci| ci.get(rg)) - .and_then(|cols| cols.get(leaf_idx)); - - match oi_pages { - Some(oi) => { - let locations = oi.page_locations(); - let rg_rows = rg_num_rows[rg]; - for (p, loc) in locations.iter().enumerate() { - let local_first = loc.first_row_index; - let next_local = if p + 1 < locations.len() { - locations[p + 1].first_row_index - } else { - rg_rows - }; - let num_rows = next_local - local_first; - let null_count = ci.and_then(|c| c.null_count(p)).unwrap_or(-1); - let (min_long, max_long) = ci - .map(|c| page_min_max(c, p, phys)) - .unwrap_or(MINMAX_UNKNOWN); - pages.push(PageEntry { - global_first_row: rg_first_row[rg] + local_first, - num_rows, - file_offset: loc.offset, - compressed_size: loc.compressed_page_size, - null_count, - min_long, - max_long, - rg_idx: rg, - local_first_row: local_first, - }); - } - } - None => { - // Fallback: treat the whole row group as a single "page". - let cc = metadata.row_group(rg).column(leaf_idx); - let null_count = cc - .statistics() - .and_then(|s| s.null_count_opt()) - .map(|n| n as i64) - .unwrap_or(-1); - let compressed = cc.compressed_size().min(i32::MAX as i64) as i32; - pages.push(PageEntry { - global_first_row: rg_first_row[rg], - num_rows: rg_num_rows[rg], - file_offset: cc.data_page_offset(), - compressed_size: compressed, - null_count, - min_long: MINMAX_UNKNOWN.0, - max_long: MINMAX_UNKNOWN.1, - rg_idx: rg, - local_first_row: 0, - }); - } - } - } - - pages -} - -/// Sentinel pair meaning "min/max unknown": the widest possible range, so a consumer making -/// skip decisions (the DocValuesSkipper) can never wrongly exclude the page. Distinguishable -/// from real data only in that real data spanning the full i64 range behaves identically — -/// which is exactly the safe behavior. -const MINMAX_UNKNOWN: (i64, i64) = (i64::MIN, i64::MAX); - -/// Extracts the per-page min/max as raw i64 bits from a typed ColumnIndex. -/// Returns [`MINMAX_UNKNOWN`] for byte-array/unsupported columns (binary min/max is not -/// exchanged as i64) and for pages whose stats are absent. -fn page_min_max(ci: &ColumnIndexMetaData, idx: usize, _phys: PhysicalType) -> (i64, i64) { - // A stat may be absent per page (stats disabled, or an all-null page). Report the unknown - // sentinel rather than 0 — 0 is indistinguishable from a real value and would let a - // skipper wrongly exclude pages. - match ci { - ColumnIndexMetaData::INT32(p) => match (p.min_value(idx), p.max_value(idx)) { - (Some(min), Some(max)) => (*min as i64, *max as i64), - _ => MINMAX_UNKNOWN, - }, - ColumnIndexMetaData::INT64(p) => match (p.min_value(idx), p.max_value(idx)) { - (Some(min), Some(max)) => (*min, *max), - _ => MINMAX_UNKNOWN, - }, - ColumnIndexMetaData::FLOAT(p) => match (p.min_value(idx), p.max_value(idx)) { - (Some(min), Some(max)) => (min.to_bits() as i64, max.to_bits() as i64), - _ => MINMAX_UNKNOWN, - }, - ColumnIndexMetaData::DOUBLE(p) => match (p.min_value(idx), p.max_value(idx)) { - (Some(min), Some(max)) => (min.to_bits() as i64, max.to_bits() as i64), - _ => MINMAX_UNKNOWN, - }, - ColumnIndexMetaData::BOOLEAN(p) => match (p.min_value(idx), p.max_value(idx)) { - (Some(min), Some(max)) => (if *min { 1 } else { 0 }, if *max { 1 } else { 0 }), - _ => MINMAX_UNKNOWN, - }, - _ => MINMAX_UNKNOWN, - } -} - -/// Maps a Parquet physical type to the Java-facing `expected_type` discriminant. -/// Returns `-1` for unsupported physical types (e.g. INT96), which can never -/// match a valid expectation and therefore surfaces as a clear mismatch error. -fn physical_type_code(t: PhysicalType) -> i32 { - match t { - PhysicalType::BOOLEAN => TYPE_BOOL, - PhysicalType::INT32 => TYPE_INT32, - PhysicalType::INT64 => TYPE_INT64, - PhysicalType::FLOAT => TYPE_FLOAT, - PhysicalType::DOUBLE => TYPE_DOUBLE, - PhysicalType::BYTE_ARRAY => TYPE_BYTE_ARRAY, - PhysicalType::FIXED_LEN_BYTE_ARRAY => TYPE_BYTE_ARRAY, - PhysicalType::INT96 => -1, - } -} - -/// Reads exactly one record (after skipping `skip` records) from a typed column -/// reader, returning that record's non-null values. For a single-valued column -/// the result holds 0 (null) or 1 value; for a repeated column it holds all the -/// values of the record. -fn read_record_values( - r: &mut ColumnReaderImpl, - skip: usize, -) -> Result, String> { - if skip > 0 { - let skipped = r.skip_records(skip).map_err(|e| e.to_string())?; - if skipped < skip { - return Err(format!("requested skip of {} records but only {} available", skip, skipped)); - } - } - let mut def_levels: Vec = Vec::new(); - let mut rep_levels: Vec = Vec::new(); - let mut values: Vec = Vec::new(); - r.read_records(1, Some(&mut def_levels), Some(&mut rep_levels), &mut values) - .map_err(|e| e.to_string())?; - Ok(values) -} - -/// Cached per-file metadata: footer + page-index parse results shared across all column readers -/// for the same file. Parquet files are immutable (changed data = new file = new path), so entries -/// never need invalidation — they can only be evicted when a file is deleted (shard close). This -/// is the `.dvm` equivalent: parsed once at first column open, then every subsequent query reuses -/// it without FFM/IO, at node lifetime scope. -struct FileMetadataCache { - /// Schema descriptor pointer (for column lookup). - schema: std::sync::Arc, - /// Number of rows in the file. - row_count: i64, - /// Per-row-group: global first row. - rg_first_row: Vec, - /// Per-row-group: number of rows. - rg_num_rows: Vec, - /// Per-column page layouts, keyed by leaf column index. Computed lazily per column. - column_pages: HashMap>, - /// Per-column descriptor cache: (leaf_idx, physical_type, max_rep_level, max_def_level). - /// Retained for future dictionary pre-warm; column lookup currently goes through `schema`. - #[allow(dead_code)] - columns: Vec<(usize, PhysicalType, i16, i16)>, -} - -lazy_static! { - /// Node-level file metadata cache. Keyed by absolute file path. Entries are never - /// invalidated (immutable files) — evicted only on explicit `parquet_evict_file_metadata`. - static ref FILE_METADATA_CACHE: Mutex>> = Mutex::new(HashMap::new()); - - /// Per-handle registry of open column readers, keyed by an opaque i64 handle. - /// Mirrors the writer-side handle pattern; serialised behind a single mutex - /// since column readers are not shared across threads. - static ref COLUMN_READERS: Mutex> = Mutex::new(HashMap::new()); -} - -/// Monotonic handle allocator. Always `>= 0`, so a returned handle is never -/// confused with the `< 0` error-pointer convention. -static NEXT_COLUMN_READER_HANDLE: AtomicI64 = AtomicI64::new(0); - -/// Locks the column-reader registry, converting a poisoned mutex into a normal -/// FFM error instead of propagating the panic. -fn lock_readers<'a>() -> Result>, String> { - COLUMN_READERS - .lock() - .map_err(|_| "column reader registry mutex poisoned".to_string()) -} - -/// Evicts a file's cached metadata (footer + page index) from the node-level cache, e.g. on -/// shard close or file deletion. No-op if the file isn't cached. Returns 0 on success. -#[ffm_safe] -#[no_mangle] -pub unsafe extern "C" fn parquet_evict_file_metadata( - file_ptr: *const u8, - file_len: i64, -) -> i64 { - let filename = str_from_raw(file_ptr, file_len) - .map_err(|e| format!("parquet_evict_file_metadata: {}", e))? - .to_string(); - FILE_METADATA_CACHE - .lock() - .map_err(|_| "file metadata cache mutex poisoned".to_string())? - .remove(&filename); - Ok(RC_OK) -} - -/// Opens a per-column reader over `file` for `col`, validating that the column -/// exists and its physical type matches `expected_type` -/// (0=INT32,1=INT64,2=FLOAT,3=DOUBLE,4=BOOL,5=BYTE_ARRAY). -/// -/// Returns `>= 0` handle id on success, `< 0` negated error pointer on failure. -#[ffm_safe] -#[no_mangle] -pub unsafe extern "C" fn parquet_open_column_reader( - file_ptr: *const u8, - file_len: i64, - col_ptr: *const u8, - col_len: i64, - expected_type: i32, -) -> i64 { - let filename = str_from_raw(file_ptr, file_len) - .map_err(|e| format!("parquet_open_column_reader file: {}", e))? - .to_string(); - let column = str_from_raw(col_ptr, col_len) - .map_err(|e| format!("parquet_open_column_reader column: {}", e))? - .to_string(); - - let state = ColumnReaderState::open(&filename, &column, expected_type)?; - - let handle = NEXT_COLUMN_READER_HANDLE.fetch_add(1, Ordering::SeqCst); - lock_readers()?.insert(handle, state); - log_debug!( - "parquet_open_column_reader: file={}, column={}, handle={}", - filename, column, handle - ); - Ok(handle) -} - -/// Closes a column reader handle and releases its file handle and buffers. -/// Returns `0` on success, a `< 0` error pointer if the handle is unknown. -#[ffm_safe] -#[no_mangle] -pub unsafe extern "C" fn parquet_close_column_reader(handle: i64) -> i64 { - match lock_readers()?.remove(&handle) { - Some(_) => { - log_debug!("parquet_close_column_reader: handle={}", handle); - Ok(RC_OK) - } - None => Err(format!("parquet_close_column_reader: unknown handle {}", handle)), - } -} - -/// Debug-only symbol: returns the number of currently open column-reader -/// handles. Used by Property 7 (native handle non-leakage). Never errors; -/// recovers from a poisoned mutex rather than panicking. -#[no_mangle] -pub unsafe extern "C" fn parquet_open_column_reader_count() -> i64 { - match COLUMN_READERS.lock() { - Ok(guard) => guard.len() as i64, - Err(poisoned) => poisoned.into_inner().len() as i64, - } -} - -/// Enable/disable the cross-query decoded-page cache and set its memory budget (bytes) + store -/// directory. Called by Java at plugin init when the `parquet_liquid_cache` feature flag is on. When -/// disabled (the default), `parquet_decode_page_at_row` never consults the cache and the decode -/// path is unchanged. A `max_memory_bytes` of 0 leaves the liquid-cache default budget. `cache_dir` -/// must point to a writable directory on real disk (Java passes a path under the node data dir); the -/// `t4` store is mounted inside it. If the store can't be built the cache silently disables itself. -#[ffm_safe] -#[no_mangle] -pub unsafe extern "C" fn parquet_liquid_cache_set_enabled( - enabled: i32, - max_memory_bytes: i64, - cache_dir_ptr: *const u8, - cache_dir_len: i64, -) -> i64 { - let bytes = if max_memory_bytes > 0 { max_memory_bytes as usize } else { 0 }; - let cache_dir = str_from_raw(cache_dir_ptr, cache_dir_len) - .map_err(|e| format!("parquet_liquid_cache_set_enabled cache_dir: {}", e))? - .to_string(); - crate::liquid_page_cache::set_enabled(enabled != 0, bytes, &cache_dir); - Ok(0) -} - -/// Clears the codec liquid cache (all decoded pages, in-memory + spilled). Returns 0 on success; -/// a no-op when the cache is disabled or not yet built. Never consults or blocks the decode path. -#[ffm_safe] -#[no_mangle] -pub unsafe extern "C" fn parquet_liquid_cache_clear() -> i64 { - crate::liquid_page_cache::clear(); - Ok(0) -} - -/// Snapshots the process-wide liquid event counters into the three out-pointers: -/// `hits` (pages served from liquid), `misses` (get found nothing → caller decodes), and `puts` -/// (decoded pages inserted). Monotonic since process start; a caller computes per-query deltas by -/// reading before and after. Returns 0 on success. Null out-pointers are skipped. -#[ffm_safe] -#[no_mangle] -pub unsafe extern "C" fn parquet_liquid_cache_stats( - hits_out: *mut i64, - misses_out: *mut i64, - puts_out: *mut i64, -) -> i64 { - let (hits, misses, puts) = crate::liquid_page_cache::stats_snapshot(); - if !hits_out.is_null() { - *hits_out = hits as i64; - } - if !misses_out.is_null() { - *misses_out = misses as i64; - } - if !puts_out.is_null() { - *puts_out = puts as i64; - } - Ok(0) -} - -/// Enables/disables page-decode phase timing (get/decode/put). Java flips this on only while the -/// `org.opensearch.parquet.timing` logger is at TRACE. When off, the decode path takes no -/// `Instant::now()` (one relaxed atomic load per page). Returns 0. -#[ffm_safe] -#[no_mangle] -pub unsafe extern "C" fn parquet_timing_set_enabled(enabled: i32) -> i64 { - timing::TIMING_ENABLED.store(enabled != 0, Ordering::Relaxed); - Ok(0) -} - -/// Snapshots the page-decode phase timers (cumulative nanos since process start) into the three -/// out-pointers: `get` (get_page_into_outbuf), `decode` (decode_primitive_page), `put` -/// (put_page_from_outbuf). Caller computes per-query deltas by reading before and after. Returns 0. -#[ffm_safe] -#[no_mangle] -pub unsafe extern "C" fn parquet_timing_snapshot( - get_out: *mut i64, - decode_out: *mut i64, - put_out: *mut i64, -) -> i64 { - if !get_out.is_null() { - *get_out = timing::GET_NANOS.load(Ordering::Relaxed) as i64; - } - if !decode_out.is_null() { - *decode_out = timing::DECODE_NANOS.load(Ordering::Relaxed) as i64; - } - if !put_out.is_null() { - *put_out = timing::PUT_NANOS.load(Ordering::Relaxed) as i64; - } - Ok(0) -} - -/// Slow-path single-value read at `row`. -/// -/// On success writes: -/// - `out_present` = 1 if the row has a value, 0 if null/absent -/// - `out_long` = the value's raw bits for primitive columns: -/// INT32 sign-extended to i64; INT64 verbatim; -/// FLOAT = `f32::to_bits` (zero-extended); -/// DOUBLE = `f64::to_bits`; -/// BOOL = 0 or 1 -/// - for BYTE_ARRAY columns: the value bytes are copied into `out_buf` and -/// `out_len` is set to the byte length (or -1 when the value is null). -/// -/// Returns a `< 0` error pointer naming the row when `row >= row_count`, when -/// the handle is unknown, or when `out_buf` is too small for a BYTE_ARRAY value -/// (in which case `out_len` is set to the required length first). -#[ffm_safe] -#[no_mangle] -pub unsafe extern "C" fn parquet_read_value_at_row( - handle: i64, - row: i64, - out_present: *mut i64, - out_long: *mut i64, - out_buf: *mut u8, - out_buf_cap: i64, - out_len: *mut i64, -) -> i64 { - let mut guard = lock_readers()?; - let state = guard - .get_mut(&handle) - .ok_or_else(|| format!("parquet_read_value_at_row: unknown handle {}", handle))?; - - if row < 0 { - return Err(format!("parquet_read_value_at_row: negative row {}", row)); - } - if row >= state.row_count { - return Err(format!( - "parquet_read_value_at_row: row {} out of range (row count {})", - row, state.row_count - )); - } - - // Default outputs: absent value. - if !out_present.is_null() { - *out_present = 0; - } - if !out_long.is_null() { - *out_long = 0; - } - if !out_len.is_null() { - *out_len = -1; - } - - let (rg_idx, local) = state.locate(row)?; - let rg = state.reader.get_row_group(rg_idx).map_err(|e| e.to_string())?; - let col = rg.get_column_reader(state.leaf_idx).map_err(|e| e.to_string())?; - let local = local as usize; - - match col { - ColumnReader::Int32ColumnReader(mut r) => { - if let Some(v) = read_record_values(&mut r, local)?.first() { - set_present(out_present, out_long, *v as i64); - } - } - ColumnReader::Int64ColumnReader(mut r) => { - if let Some(v) = read_record_values(&mut r, local)?.first() { - set_present(out_present, out_long, *v); - } - } - ColumnReader::FloatColumnReader(mut r) => { - if let Some(v) = read_record_values(&mut r, local)?.first() { - set_present(out_present, out_long, v.to_bits() as i64); - } - } - ColumnReader::DoubleColumnReader(mut r) => { - if let Some(v) = read_record_values(&mut r, local)?.first() { - set_present(out_present, out_long, v.to_bits() as i64); - } - } - ColumnReader::BoolColumnReader(mut r) => { - if let Some(v) = read_record_values(&mut r, local)?.first() { - set_present(out_present, out_long, if *v { 1 } else { 0 }); - } - } - ColumnReader::ByteArrayColumnReader(mut r) => { - if let Some(v) = read_record_values(&mut r, local)?.first() { - return write_bytes_value(v.data(), out_present, out_buf, out_buf_cap, out_len); - } - } - ColumnReader::FixedLenByteArrayColumnReader(mut r) => { - if let Some(v) = read_record_values(&mut r, local)?.first() { - return write_bytes_value(v.data(), out_present, out_buf, out_buf_cap, out_len); - } - } - ColumnReader::Int96ColumnReader(_) => { - return Err("parquet_read_value_at_row: INT96 columns are not supported".to_string()); - } - } - - Ok(RC_OK) -} - -/// Marks a primitive value present and stores its raw bits. -unsafe fn set_present(out_present: *mut i64, out_long: *mut i64, bits: i64) { - if !out_present.is_null() { - *out_present = 1; - } - if !out_long.is_null() { - *out_long = bits; - } -} - -/// Copies a single BYTE_ARRAY value into the caller buffer. Sets `out_present=1` -/// and `out_len` to the byte length. Returns `RC_OVERFLOW` (after recording the -/// required length in `out_len`) when the value does not fit in `out_buf_cap`, -/// so the caller can grow its buffer and retry once. -unsafe fn write_bytes_value( - bytes: &[u8], - out_present: *mut i64, - out_buf: *mut u8, - out_buf_cap: i64, - out_len: *mut i64, -) -> Result { - if !out_present.is_null() { - *out_present = 1; - } - let n = bytes.len(); - if !out_len.is_null() { - *out_len = n as i64; - } - if (n as i64) > out_buf_cap || (n > 0 && out_buf.is_null()) { - return Ok(RC_OVERFLOW); - } - if n > 0 { - std::ptr::copy_nonoverlapping(bytes.as_ptr(), out_buf, n); - } - Ok(RC_OK) -} - -/// Slow-path repeated read at `row` for a repeated (multi-valued) column. -/// -/// Capacity contract: `out_long_cap` is the maximum element count for *both* -/// primitive and BYTE_ARRAY columns; `out_byte_offsets` (BYTE_ARRAY only) must -/// have capacity `out_long_cap + 1`. -/// -/// On success (`RC_OK`): -/// - `out_count` = number of values at the row -/// - primitive columns: raw bits (see `parquet_read_value_at_row`) written to -/// `out_longs` -/// - BYTE_ARRAY columns: concatenated bytes in `out_byte_buf`, CSR offsets -/// (length `count + 1`) in `out_byte_offsets` -/// -/// On `RC_OVERFLOW`: `out_count` holds the required element count. When the -/// element count fits but only the byte buffer is too small, the full CSR -/// offsets are still written so `out_byte_offsets[count]` reports the required -/// total byte size, enabling a single retry. -#[ffm_safe] -#[no_mangle] -pub unsafe extern "C" fn parquet_read_repeated_at_row( - handle: i64, - row: i64, - out_count: *mut i64, - out_longs: *mut i64, - out_long_cap: i64, - out_byte_buf: *mut u8, - out_byte_offsets: *mut i64, - out_byte_buf_cap: i64, -) -> i64 { - let mut guard = lock_readers()?; - let state = guard - .get_mut(&handle) - .ok_or_else(|| format!("parquet_read_repeated_at_row: unknown handle {}", handle))?; - - if row < 0 { - return Err(format!("parquet_read_repeated_at_row: negative row {}", row)); - } - if row >= state.row_count { - return Err(format!( - "parquet_read_repeated_at_row: row {} out of range (row count {})", - row, state.row_count - )); - } - - if !out_count.is_null() { - *out_count = 0; - } - - let (rg_idx, local) = state.locate(row)?; - let rg = state.reader.get_row_group(rg_idx).map_err(|e| e.to_string())?; - let col = rg.get_column_reader(state.leaf_idx).map_err(|e| e.to_string())?; - let local = local as usize; - - match col { - ColumnReader::Int32ColumnReader(mut r) => { - let vals = read_record_values(&mut r, local)?; - write_primitive_repeated(vals.iter().map(|v| *v as i64), vals.len(), out_count, out_longs, out_long_cap) - } - ColumnReader::Int64ColumnReader(mut r) => { - let vals = read_record_values(&mut r, local)?; - write_primitive_repeated(vals.iter().copied(), vals.len(), out_count, out_longs, out_long_cap) - } - ColumnReader::FloatColumnReader(mut r) => { - let vals = read_record_values(&mut r, local)?; - write_primitive_repeated(vals.iter().map(|v| v.to_bits() as i64), vals.len(), out_count, out_longs, out_long_cap) - } - ColumnReader::DoubleColumnReader(mut r) => { - let vals = read_record_values(&mut r, local)?; - write_primitive_repeated(vals.iter().map(|v| v.to_bits() as i64), vals.len(), out_count, out_longs, out_long_cap) - } - ColumnReader::BoolColumnReader(mut r) => { - let vals = read_record_values(&mut r, local)?; - write_primitive_repeated(vals.iter().map(|v| if *v { 1i64 } else { 0i64 }), vals.len(), out_count, out_longs, out_long_cap) - } - ColumnReader::ByteArrayColumnReader(mut r) => { - let vals = read_record_values(&mut r, local)?; - let slices: Vec<&[u8]> = vals.iter().map(|v| v.data()).collect(); - write_bytes_repeated(&slices, out_count, out_long_cap, out_byte_buf, out_byte_offsets, out_byte_buf_cap) - } - ColumnReader::FixedLenByteArrayColumnReader(mut r) => { - let vals = read_record_values(&mut r, local)?; - let slices: Vec<&[u8]> = vals.iter().map(|v| v.data()).collect(); - write_bytes_repeated(&slices, out_count, out_long_cap, out_byte_buf, out_byte_offsets, out_byte_buf_cap) - } - ColumnReader::Int96ColumnReader(_) => { - Err("parquet_read_repeated_at_row: INT96 columns are not supported".to_string()) - } - } -} - -/// Writes repeated primitive values to `out_longs`, or reports overflow. -unsafe fn write_primitive_repeated( - values: impl Iterator, - count: usize, - out_count: *mut i64, - out_longs: *mut i64, - out_long_cap: i64, -) -> Result { - if !out_count.is_null() { - *out_count = count as i64; - } - if (count as i64) > out_long_cap || out_longs.is_null() { - return Ok(RC_OVERFLOW); - } - for (i, v) in values.enumerate() { - *out_longs.add(i) = v; - } - Ok(RC_OK) -} - -/// Writes repeated BYTE_ARRAY values (CSR layout) to the caller buffers, or -/// reports overflow with required sizes. -unsafe fn write_bytes_repeated( - slices: &[&[u8]], - out_count: *mut i64, - out_long_cap: i64, - out_byte_buf: *mut u8, - out_byte_offsets: *mut i64, - out_byte_buf_cap: i64, -) -> Result { - let count = slices.len(); - let total_bytes: usize = slices.iter().map(|s| s.len()).sum(); - if !out_count.is_null() { - *out_count = count as i64; - } - - // Element-count overflow: cannot safely write offsets (capacity is count+1). - if (count as i64) > out_long_cap { - return Ok(RC_OVERFLOW); - } - - // Element count fits: write the full CSR offsets so that, even on a byte - // overflow, out_byte_offsets[count] == total_bytes reports the required size. - if !out_byte_offsets.is_null() { - let mut acc = 0i64; - for (i, s) in slices.iter().enumerate() { - *out_byte_offsets.add(i) = acc; - acc += s.len() as i64; - } - *out_byte_offsets.add(count) = acc; - } - - if (total_bytes as i64) > out_byte_buf_cap || (total_bytes > 0 && out_byte_buf.is_null()) { - return Ok(RC_OVERFLOW); - } - - let mut acc = 0usize; - for s in slices { - if !s.is_empty() { - std::ptr::copy_nonoverlapping(s.as_ptr(), out_byte_buf.add(acc), s.len()); - } - acc += s.len(); - } - Ok(RC_OK) -} - -// --------------------------------------------------------------------------- -// Page-index loader + page decoder (DocValues codec — cache Layers 1-4) -// --------------------------------------------------------------------------- -// -// These are the hot-path functions used by the Java `ParquetColumnReader`: -// - `parquet_get_column_num_pages` — page count, so Java can pre-size buffers -// - `parquet_get_column_page_index` — Layer 3 jump table + Layer 4 page stats -// - `parquet_decode_page_at_row` — Layer 1 values + Layer 2 presence bitset -// -// All row indices exchanged here are global (file-relative), consistent with -// the Row ID = Doc ID invariant. - -/// Returns the number of pages in the column (`>= 0`), or a `< 0` error pointer -/// for an unknown handle. Java reads this first to size the parallel arrays -/// passed to `parquet_get_column_page_index`. -#[ffm_safe] -#[no_mangle] -pub unsafe extern "C" fn parquet_get_column_num_pages(handle: i64) -> i64 { - let guard = lock_readers()?; - let state = guard - .get(&handle) - .ok_or_else(|| format!("parquet_get_column_num_pages: unknown handle {}", handle))?; - Ok(state.pages.len() as i64) -} - -/// Layer 3 + 4: writes the column's per-page jump table and page statistics into -/// caller-provided parallel arrays, each of capacity `out_buf_capacity` -/// (= the page count from `parquet_get_column_num_pages`). -/// -/// Arrays (length = page count): -/// - `out_first_row` global index of the page's first row -/// - `out_file_offset` byte offset of the page in the file (0 if unknown) -/// - `out_compressed_size` compressed page size in bytes (0 if unknown) -/// - `out_null_count` nulls in the page, or -1 when unknown -/// - `out_min_long` per-page min raw bits (numeric only; 0 otherwise) -/// - `out_max_long` per-page max raw bits (numeric only; 0 otherwise) -/// -/// `out_actual_pages` always receives the true page count. Returns `RC_OVERFLOW` -/// (a positive sentinel) without writing the arrays when `out_buf_capacity` is -/// smaller than the page count, so the caller can grow and retry. Returns a -/// `< 0` error pointer for an unknown handle. -#[ffm_safe] -#[no_mangle] -pub unsafe extern "C" fn parquet_get_column_page_index( - handle: i64, - out_first_row: *mut i64, - out_file_offset: *mut i64, - out_compressed_size: *mut i32, - out_null_count: *mut i64, - out_min_long: *mut i64, - out_max_long: *mut i64, - out_buf_capacity: i64, - out_actual_pages: *mut i64, -) -> i64 { - let guard = lock_readers()?; - let state = guard - .get(&handle) - .ok_or_else(|| format!("parquet_get_column_page_index: unknown handle {}", handle))?; - - let n = state.pages.len(); - if !out_actual_pages.is_null() { - *out_actual_pages = n as i64; - } - if (n as i64) > out_buf_capacity { - return Ok(RC_OVERFLOW); - } - - for (i, e) in state.pages.iter().enumerate() { - if !out_first_row.is_null() { - *out_first_row.add(i) = e.global_first_row; - } - if !out_file_offset.is_null() { - *out_file_offset.add(i) = e.file_offset; - } - if !out_compressed_size.is_null() { - *out_compressed_size.add(i) = e.compressed_size; - } - if !out_null_count.is_null() { - *out_null_count.add(i) = e.null_count; - } - if !out_min_long.is_null() { - *out_min_long.add(i) = e.min_long; - } - if !out_max_long.is_null() { - *out_max_long.add(i) = e.max_long; - } - } - Ok(RC_OK) -} - -/// Decodes one primitive page through a (possibly retained) typed column reader and writes -/// values + packed presence straight into the caller's out-buffers. `skip` is relative to the -/// reader's current position, NOT the row group start — the retained-cursor caller computes it -/// from the cursor position so a reused reader only skips forward the remaining distance. -#[allow(clippy::too_many_arguments)] -unsafe fn decode_primitive_page( - r: &mut ColumnReaderImpl, - skip: usize, - num_rows: usize, - max_def_level: i16, - effective_null_count: i64, - def_scratch: &mut Vec, - val_scratch: &mut Vec, - to_bits: impl Fn(T::T) -> i64, - out_value_buf: *mut u8, - out_presence_bitset: *mut i64, -) -> Result<(), String> -where - T::T: Copy, -{ - // DECODE_NANOS: the actual Parquet decode (records + presence + expand into the out-buffer). - let decode_timer = if timing::on() { Some(Instant::now()) } else { None }; - decode_page_records(r, skip, num_rows, def_scratch, val_scratch)?; - pack_presence_from_def_levels(def_scratch, max_def_level, num_rows, out_presence_bitset); - expand_to_outbuf(val_scratch, to_bits, effective_null_count, num_rows, out_value_buf, out_presence_bitset as *const i64); - if let Some(t) = decode_timer { - timing::record(&timing::DECODE_NANOS, t); - } - Ok(()) -} - -/// Decodes one page's worth of single-valued records, returning a per-row -/// presence flag (`true` = value present) and the dense list of non-null values -/// in row order. `skip` records are skipped first, then `num_rows` records are -/// read. Works for required columns (`max_def_level == 0`, all present) and -/// optional non-repeated columns. -fn decode_page_records( - r: &mut ColumnReaderImpl, - skip: usize, - num_rows: usize, - scratch_def: &mut Vec, - scratch_vals: &mut Vec, -) -> Result<(), String> { - if skip > 0 { - let skipped = r.skip_records(skip).map_err(|e| e.to_string())?; - if skipped < skip { - return Err(format!( - "page decode: requested skip of {} records but only {} available", - skip, skipped - )); - } - } - - scratch_def.clear(); - scratch_vals.clear(); - scratch_def.reserve(num_rows.saturating_sub(scratch_def.capacity())); - scratch_vals.reserve(num_rows.saturating_sub(scratch_vals.capacity())); - - let (records_read, _values_read, _levels_read) = r - .read_records(num_rows, Some(scratch_def), None, scratch_vals) - .map_err(|e| e.to_string())?; - if records_read < num_rows { - return Err(format!( - "page decode: expected {} records but read {}", - num_rows, records_read - )); - } - Ok(()) -} - -/// Packs definition levels directly into a little-endian `long[]` bitset in the -/// caller's out-buffer, with bit `i` set when `def_levels[i] == max_def_level`. -/// When `max_def_level == 0` (required column), all bits are set. -/// -/// Uses a branchless comparison that auto-vectorizes (LLVM emits pcmpeqw + pack -/// on x86 AVX2, cmeq on aarch64 NEON). Eliminates both the intermediate `Vec` -/// and the per-bit branch of the old `write_presence_bitset`. -/// -/// Returns the number of words written. Caller must ensure capacity is sufficient. -#[inline] -unsafe fn pack_presence_from_def_levels( - def_levels: &[i16], - max_def_level: i16, - num_rows: usize, - out: *mut i64, -) { - let words = (num_rows + 63) / 64; - if max_def_level == 0 { - // Required column: every row is present → all-ones, mask tail. - for w in 0..words { - let remaining = num_rows - w * 64; - if remaining >= 64 { - *out.add(w) = -1i64; // all bits set - } else { - *out.add(w) = ((1u64 << remaining) - 1) as i64; - } - } - } else { - // Optional column: branchless pack. The inner loop auto-vectorizes because - // `(d == max_def_level) as u64` is a conditional-move / compare instruction. - for w in 0..words { - let mut bits: u64 = 0; - let base = w * 64; - let end = (base + 64).min(num_rows); - for b in base..end { - bits |= ((*def_levels.get_unchecked(b) == max_def_level) as u64) << (b - base); - } - *out.add(w) = bits as i64; - } - } -} - -/// Expands dense non-null values directly into the caller's out-buffer as per-row -/// raw i64 bits, using the packed presence bitset to scatter. Null slots are -/// written as 0. Eliminates the intermediate `Vec` allocation. -/// -/// The inner scatter loop is split by nullability: when `null_count == 0` the -/// entire dense buffer can be converted with a tight widening loop that -/// auto-vectorizes (LLVM emits vpmovsxdq / sshll). The nullable path reads the -/// packed bits we just wrote and scatters accordingly. -/// -/// # Safety -/// `out` must be valid for `num_rows * 8` bytes. `presence_bits` must contain -/// the packed bitset already written by `pack_presence_from_def_levels`. -#[inline] -unsafe fn expand_to_outbuf( - dense: &[T], - to_bits: impl Fn(T) -> i64, - null_count: i64, - num_rows: usize, - out: *mut u8, - presence_bits: *const i64, -) { - let out_i64 = out as *mut i64; - if null_count == 0 { - // All rows present — tight conversion loop, no branching, SIMD-friendly. - for i in 0..num_rows { - *out_i64.add(i) = to_bits(*dense.get_unchecked(i)); - } - } else { - // Scatter using the packed presence bits. Read one word at a time and use - // trailing_zeros to jump to set bits (pop-and-scatter pattern). - // First zero-fill so null slots hold 0 without an explicit branch. - std::ptr::write_bytes(out, 0, num_rows * 8); - let mut di = 0usize; - let words = (num_rows + 63) / 64; - for w in 0..words { - let mut bits = *presence_bits.add(w) as u64; - while bits != 0 { - let b = bits.trailing_zeros() as usize; - let row = w * 64 + b; - *out_i64.add(row) = to_bits(*dense.get_unchecked(di)); - di += 1; - bits &= bits - 1; // clear lowest set bit - } - } - } -} - -/// Packs a per-row presence slice into a little-endian `long[]` bitset (bit i set -/// when row i is present), writing into `out` (capacity `out_words`). Returns the -/// number of words required; on capacity shortfall writes nothing and the caller -/// treats the positive required count as an overflow signal. -/// -/// Retained for the BYTE_ARRAY path which still uses `Vec` presence. -unsafe fn write_presence_bitset(presence: &[bool], out: *mut i64, out_words: i64) -> i64 { - let words_needed = ((presence.len() + 63) / 64) as i64; - if words_needed > out_words || out.is_null() { - return words_needed; - } - for w in 0..words_needed as usize { - let mut bits: u64 = 0; - let base = w * 64; - for b in 0..64 { - let idx = base + b; - if idx >= presence.len() { - break; - } - if presence[idx] { - bits |= 1u64 << b; - } - } - *out.add(w) = bits as i64; - } - words_needed -} - -/// Layer 1 + 2: decode the page containing global row `row` into caller buffers. -/// -/// On success (`RC_OK`): -/// - `out_first_row` / `out_last_row` = inclusive global row range of the page -/// - primitive columns: per-row raw bits written to `out_value_buf` -/// (interpreted as `long[]`, one slot per row; null rows hold 0); -/// `out_value_actual_len` = `rows * 8` -/// - BYTE_ARRAY columns: concatenated value bytes in `out_value_buf`, per-row -/// CSR offsets (length `rows + 1`) in `out_byte_offsets`; -/// `out_value_actual_len` = total bytes used -/// - `out_presence_bitset` = packed `long[]`, one bit per row -/// -/// On `RC_OVERFLOW` (a positive sentinel): `out_first_row`, `out_last_row` and -/// `out_value_actual_len` are populated so the caller can size every buffer -/// (values = `out_value_actual_len` bytes; offsets = `rows + 1`; presence = -/// `ceil(rows / 64)` words) and retry once. Returns a `< 0` error pointer for an -/// unknown handle, an out-of-range row, or a repeated (multi-valued) column. -#[ffm_safe] -#[no_mangle] -pub unsafe extern "C" fn parquet_decode_page_at_row( - handle: i64, - row: i64, - out_first_row: *mut i64, - out_last_row: *mut i64, - out_value_buf: *mut u8, - out_value_buf_cap: i64, - out_value_actual_len: *mut i64, - out_byte_offsets: *mut i32, - out_byte_offsets_cap: i64, - out_presence_bitset: *mut i64, - out_presence_bits_cap: i64, -) -> i64 { - let mut guard = lock_readers()?; - let state = guard - .get_mut(&handle) - .ok_or_else(|| format!("parquet_decode_page_at_row: unknown handle {}", handle))?; - - if row < 0 || row >= state.row_count { - return Err(format!( - "parquet_decode_page_at_row: row {} out of range (row count {})", - row, state.row_count - )); - } - if state.repeated { - return Err(format!( - "parquet_decode_page_at_row: column is repeated (multi-valued); use parquet_read_repeated_at_row (handle {})", - handle - )); - } - - let page_idx = state.page_for_row(row)?; - // Copy out the page's coordinates before borrowing the reader mutably. - let (rg_idx, local_first, num_rows, first_global) = { - let e = &state.pages[page_idx]; - (e.rg_idx, e.local_first_row, e.num_rows, e.global_first_row) - }; - let num_rows_usize = num_rows as usize; - let max_def_level = state.max_def_level; - let physical_type = state.physical_type; - - // Always report the page row range so the caller can bound its cache and - // size buffers even on the overflow path. - if !out_first_row.is_null() { - *out_first_row = first_global; - } - if !out_last_row.is_null() { - *out_last_row = first_global + num_rows - 1; - } - - // Cross-query decoded-page cache (codec-owned liquid instance). When enabled, a hit serves the - // decoded page from the node-level cache and skips the Parquet decode below entirely; a miss - // decodes as usual and backfills the cache (see the primitive arms). Keyed by - // (file, column, page); primitives only. No-op when the feature flag is off. - let lc_eid = if crate::liquid_page_cache::enabled() { - Some(crate::liquid_page_cache::entry_id( - state.liquid_file_id, - state.leaf_idx as u32, - page_idx as u32, - )) - } else { - None - }; - if let Some(eid) = lc_eid { - // Serve a cache hit straight into the caller's out-buffers (two memcpys), bypassing the - // Vec+Vec rebuild that get_page does and its recopy through write_primitive_page. - let get_timer = if timing::on() { Some(Instant::now()) } else { None }; - let got = crate::liquid_page_cache::get_page_into_outbuf( - eid, out_value_buf, out_value_buf_cap, out_value_actual_len, - out_presence_bitset, out_presence_bits_cap, - ); - if let Some(t) = get_timer { - timing::record(&timing::GET_NANOS, t); - } - if let Some(rc) = got { - return Ok(rc); - } - } - - // Check capacity up front for primitive types so we can write directly into the out-buffers - // without needing an intermediate allocation. BYTE_ARRAY still uses the old path since its - // total byte length isn't known until after decode. - let is_primitive = physical_type != PhysicalType::BYTE_ARRAY - && physical_type != PhysicalType::FIXED_LEN_BYTE_ARRAY; - - if is_primitive { - let value_bytes = (num_rows_usize * 8) as i64; - if !out_value_actual_len.is_null() { - *out_value_actual_len = value_bytes; - } - let presence_words = ((num_rows_usize + 63) / 64) as i64; - if value_bytes > out_value_buf_cap - || out_value_buf.is_null() - || presence_words > out_presence_bits_cap - || out_presence_bitset.is_null() - { - return Ok(RC_OVERFLOW); - } - } - - // Get the null_count from the page entry for the expand path (0 means all-present). - let page_null_count = state.pages[page_idx].null_count; - // If null_count is unknown (-1), treat as potentially nullable. - let effective_null_count = if page_null_count < 0 { 1 } else { page_null_count }; - - // Retained-cursor reader acquisition. A typed column reader can only move forward, so it - // is reusable exactly when the target page is in the same row group at-or-ahead of the - // cursor's position; then the skip is the remaining forward distance and — crucially — - // the dictionary page and row-group metadata are NOT re-read. Any backward jump or - // row-group change falls back to a fresh reader (dictionary re-decoded once). - // The cursor is take()n up front so a decode error leaves it invalidated; each arm - // re-installs it only after a successful decode. - let (col, skip) = match state.cursor.take() { - Some(c) if c.rg_idx == rg_idx && c.position <= first_global => { - (c.col_reader, (first_global - c.position) as usize) - } - _ => { - let rg = state.reader.get_row_group(rg_idx).map_err(|e| e.to_string())?; - let col = rg.get_column_reader(state.leaf_idx).map_err(|e| e.to_string())?; - (col, local_first as usize) - } - }; - // Position of the reader after this page is consumed; stored on the re-installed cursor. - let next_position = first_global + num_rows; - - // Decode and write directly to out-buffers. Primitive types use the optimized - // zero-alloc path (scratch buffers reused across calls, presence bits packed - // branchlessly from def_levels, values scattered directly to the out-buffer). - // BYTE_ARRAY/FIXED_LEN_BYTE_ARRAY still use the prior path since their total - // value byte length is data-dependent and must be computed before overflow check. - match col { - ColumnReader::Int32ColumnReader(mut r) => { - let scratch = &mut state.scratch; - decode_primitive_page( - &mut r, skip, num_rows_usize, max_def_level, effective_null_count, - &mut scratch.def_levels, &mut scratch.values_i32, |v| v as i64, - out_value_buf, out_presence_bitset, - )?; - state.cursor = Some(CursorState { rg_idx, col_reader: ColumnReader::Int32ColumnReader(r), position: next_position }); - if let Some(eid) = lc_eid { - crate::liquid_page_cache::put_page_from_outbuf(eid, out_value_buf, out_presence_bitset, num_rows_usize); - } - return Ok(RC_OK); - } - ColumnReader::Int64ColumnReader(mut r) => { - let scratch = &mut state.scratch; - decode_primitive_page( - &mut r, skip, num_rows_usize, max_def_level, effective_null_count, - &mut scratch.def_levels, &mut scratch.values_i64, |v| v, - out_value_buf, out_presence_bitset, - )?; - state.cursor = Some(CursorState { rg_idx, col_reader: ColumnReader::Int64ColumnReader(r), position: next_position }); - if let Some(eid) = lc_eid { - crate::liquid_page_cache::put_page_from_outbuf(eid, out_value_buf, out_presence_bitset, num_rows_usize); - } - return Ok(RC_OK); - } - ColumnReader::FloatColumnReader(mut r) => { - let scratch = &mut state.scratch; - decode_primitive_page( - &mut r, skip, num_rows_usize, max_def_level, effective_null_count, - &mut scratch.def_levels, &mut scratch.values_f32, |v| v.to_bits() as i64, - out_value_buf, out_presence_bitset, - )?; - state.cursor = Some(CursorState { rg_idx, col_reader: ColumnReader::FloatColumnReader(r), position: next_position }); - if let Some(eid) = lc_eid { - crate::liquid_page_cache::put_page_from_outbuf(eid, out_value_buf, out_presence_bitset, num_rows_usize); - } - return Ok(RC_OK); - } - ColumnReader::DoubleColumnReader(mut r) => { - let scratch = &mut state.scratch; - decode_primitive_page( - &mut r, skip, num_rows_usize, max_def_level, effective_null_count, - &mut scratch.def_levels, &mut scratch.values_f64, |v| v.to_bits() as i64, - out_value_buf, out_presence_bitset, - )?; - state.cursor = Some(CursorState { rg_idx, col_reader: ColumnReader::DoubleColumnReader(r), position: next_position }); - if let Some(eid) = lc_eid { - crate::liquid_page_cache::put_page_from_outbuf(eid, out_value_buf, out_presence_bitset, num_rows_usize); - } - return Ok(RC_OK); - } - ColumnReader::BoolColumnReader(mut r) => { - let scratch = &mut state.scratch; - decode_primitive_page( - &mut r, skip, num_rows_usize, max_def_level, effective_null_count, - &mut scratch.def_levels, &mut scratch.values_bool, |v| if v { 1i64 } else { 0i64 }, - out_value_buf, out_presence_bitset, - )?; - state.cursor = Some(CursorState { rg_idx, col_reader: ColumnReader::BoolColumnReader(r), position: next_position }); - if let Some(eid) = lc_eid { - crate::liquid_page_cache::put_page_from_outbuf(eid, out_value_buf, out_presence_bitset, num_rows_usize); - } - return Ok(RC_OK); - } - ColumnReader::ByteArrayColumnReader(mut r) => { - let scratch = &mut state.scratch; - let mut byte_values: Vec = Vec::new(); - decode_page_records(&mut r, skip, num_rows_usize, &mut scratch.def_levels, &mut byte_values)?; - let presence: Vec = if max_def_level == 0 { - vec![true; num_rows_usize] - } else { - scratch.def_levels.iter().take(num_rows_usize).map(|d| *d == max_def_level).collect() - }; - let slices = expand_bytes(&presence, &byte_values); - let rc = write_bytes_page( - &slices, &presence, out_value_buf, out_value_buf_cap, out_value_actual_len, - out_byte_offsets, out_byte_offsets_cap, out_presence_bitset, out_presence_bits_cap, - )?; - // The page was fully consumed even on RC_OVERFLOW, so the cursor position is - // next_position either way; the overflow retry of the same row then simply - // misses the cursor (position > first_global) and opens a fresh reader. - state.cursor = Some(CursorState { rg_idx, col_reader: ColumnReader::ByteArrayColumnReader(r), position: next_position }); - return Ok(rc); - } - ColumnReader::FixedLenByteArrayColumnReader(mut r) => { - let scratch = &mut state.scratch; - let mut flba_values: Vec = Vec::new(); - decode_page_records(&mut r, skip, num_rows_usize, &mut scratch.def_levels, &mut flba_values)?; - let presence: Vec = if max_def_level == 0 { - vec![true; num_rows_usize] - } else { - scratch.def_levels.iter().take(num_rows_usize).map(|d| *d == max_def_level).collect() - }; - let slices = expand_flba(&presence, &flba_values); - let rc = write_bytes_page( - &slices, &presence, out_value_buf, out_value_buf_cap, out_value_actual_len, - out_byte_offsets, out_byte_offsets_cap, out_presence_bitset, out_presence_bits_cap, - )?; - // See the ByteArray arm: page fully consumed even on RC_OVERFLOW. - state.cursor = Some(CursorState { rg_idx, col_reader: ColumnReader::FixedLenByteArrayColumnReader(r), position: next_position }); - return Ok(rc); - } - ColumnReader::Int96ColumnReader(_) => { - let _ = physical_type; // silence unused in this arm - Err("parquet_decode_page_at_row: INT96 columns are not supported".to_string()) - } - } -} - -/// Expands dense non-null primitive values into a per-row `i64` slot vector -/// (null rows hold 0), applying `to_bits` to each present value. -/// Retained for the `get_page` A/B path in liquid_page_cache; no longer on the hot decode path. -#[allow(dead_code)] -fn expand_primitive(presence: &[bool], dense: &[T], to_bits: impl Fn(T) -> i64) -> Vec { - let mut out = Vec::with_capacity(presence.len()); - let mut di = 0usize; - for &present in presence { - if present { - out.push(to_bits(dense[di])); - di += 1; - } else { - out.push(0); - } - } - out -} - -/// Expands dense non-null BYTE_ARRAY values into a per-row slice vector (null -/// rows map to an empty slice). -fn expand_bytes<'a>(presence: &[bool], dense: &'a [parquet::data_type::ByteArray]) -> Vec<&'a [u8]> { - let mut out: Vec<&[u8]> = Vec::with_capacity(presence.len()); - let mut di = 0usize; - for &present in presence { - if present { - out.push(dense[di].data()); - di += 1; - } else { - out.push(&[]); - } - } - out -} - -/// Expands dense non-null FIXED_LEN_BYTE_ARRAY values into a per-row slice vector. -fn expand_flba<'a>( - presence: &[bool], - dense: &'a [parquet::data_type::FixedLenByteArray], -) -> Vec<&'a [u8]> { - let mut out: Vec<&[u8]> = Vec::with_capacity(presence.len()); - let mut di = 0usize; - for &present in presence { - if present { - out.push(dense[di].data()); - di += 1; - } else { - out.push(&[]); - } - } - out -} - -/// Writes a decoded primitive page (per-row raw bits + presence bitset) to the -/// caller buffers, or returns `RC_OVERFLOW` after recording the required value -/// byte length. -/// Retained for reference / potential fallback use; the hot path now uses -/// `expand_to_outbuf` + `pack_presence_from_def_levels` directly. -#[allow(dead_code)] -unsafe fn write_primitive_page( - longs: &[i64], - presence: &[bool], - out_value_buf: *mut u8, - out_value_buf_cap: i64, - out_value_actual_len: *mut i64, - out_presence_bitset: *mut i64, - out_presence_bits_cap: i64, -) -> Result { - let value_bytes = (longs.len() * 8) as i64; - if !out_value_actual_len.is_null() { - *out_value_actual_len = value_bytes; - } - - let presence_words = ((presence.len() + 63) / 64) as i64; - if value_bytes > out_value_buf_cap - || out_value_buf.is_null() - || presence_words > out_presence_bits_cap - || out_presence_bitset.is_null() - { - return Ok(RC_OVERFLOW); - } - - // Write values as native-endian i64 words (Java reads them as a long[] via - // a MemorySegment using native byte order). Copy raw bytes rather than - // storing through a *mut i64 — out_value_buf is a u8 buffer with no - // guaranteed 8-byte alignment, so an aligned i64 store would be UB. - if !longs.is_empty() { - std::ptr::copy_nonoverlapping( - longs.as_ptr() as *const u8, - out_value_buf, - longs.len() * 8, - ); - } - write_presence_bitset(presence, out_presence_bitset, out_presence_bits_cap); - Ok(RC_OK) -} - -/// Writes a decoded BYTE_ARRAY page (concatenated bytes + CSR offsets + presence -/// bitset) to the caller buffers, or returns `RC_OVERFLOW` after recording the -/// required total byte length. -unsafe fn write_bytes_page( - slices: &[&[u8]], - presence: &[bool], - out_value_buf: *mut u8, - out_value_buf_cap: i64, - out_value_actual_len: *mut i64, - out_byte_offsets: *mut i32, - out_byte_offsets_cap: i64, - out_presence_bitset: *mut i64, - out_presence_bits_cap: i64, -) -> Result { - let total_bytes: usize = slices.iter().map(|s| s.len()).sum(); - if !out_value_actual_len.is_null() { - *out_value_actual_len = total_bytes as i64; - } - - let offsets_needed = (slices.len() + 1) as i64; - let presence_words = ((presence.len() + 63) / 64) as i64; - if (total_bytes as i64) > out_value_buf_cap - || (total_bytes > 0 && out_value_buf.is_null()) - || offsets_needed > out_byte_offsets_cap - || out_byte_offsets.is_null() - || presence_words > out_presence_bits_cap - || out_presence_bitset.is_null() - { - return Ok(RC_OVERFLOW); - } - - let mut acc: i32 = 0; - for (i, s) in slices.iter().enumerate() { - *out_byte_offsets.add(i) = acc; - if !s.is_empty() { - std::ptr::copy_nonoverlapping(s.as_ptr(), out_value_buf.add(acc as usize), s.len()); - } - acc += s.len() as i32; - } - *out_byte_offsets.add(slices.len()) = acc; - - write_presence_bitset(presence, out_presence_bitset, out_presence_bits_cap); - Ok(RC_OK) -} diff --git a/sandbox/plugins/parquet-data-format/src/main/rust/src/lib.rs b/sandbox/plugins/parquet-data-format/src/main/rust/src/lib.rs index 699f12311123e..9a2fac354e97c 100644 --- a/sandbox/plugins/parquet-data-format/src/main/rust/src/lib.rs +++ b/sandbox/plugins/parquet-data-format/src/main/rust/src/lib.rs @@ -14,7 +14,6 @@ mod tests; pub mod writer; pub mod ffm; -pub mod liquid_page_cache; pub mod memory; pub mod native_settings; pub mod field_config; diff --git a/sandbox/plugins/parquet-data-format/src/main/rust/src/liquid_page_cache.rs b/sandbox/plugins/parquet-data-format/src/main/rust/src/liquid_page_cache.rs deleted file mode 100644 index 98d4ac25bb9c6..0000000000000 --- a/sandbox/plugins/parquet-data-format/src/main/rust/src/liquid_page_cache.rs +++ /dev/null @@ -1,405 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -//! Codec-owned, cross-query decoded-page cache backed by liquid-cache's core API. -//! -//! The Parquet DocValues codec's `PageCache` is per-query scratch: every query re-decodes the -//! same Parquet pages. This module gives the codec a **node-level** (process-lifetime) cache of -//! decoded primitive pages so a later query reuses a page an earlier query already decoded — -//! the cross-query tier the codec otherwise lacks. -//! -//! Design (v1): -//! - A single process-global `Arc` (liquid core), built lazily. This is a -//! **codec-owned** instance, independent of the DataFusion/PPL liquid cache — same technology, -//! separate instance and keyspace. Sharing the DataFusion instance is a later optimization. -//! - Entries are keyed by `(file_id, column_id, page_idx)` packed into liquid's `EntryID` (a -//! `usize`). `file_id` comes from a codec-local path→id registry, so the key carries file -//! identity (and Parquet's immutable-file/generation model means changed data = new path = -//! new key = automatic miss — no invalidation logic needed). -//! - Values are cached as an Arrow `Int64Array` (with a null buffer derived from the page's -//! presence bits). On a hit we convert back to the raw `Vec` + `Vec` the codec's -//! `write_primitive_page` already consumes, so the Java/PageCache/per-doc path is byte-identical -//! whether the page was decoded or served from cache. -//! - liquid's `insert`/`get` are async; we drive them on a dedicated single-threaded tokio runtime -//! via `block_on`, mirroring `merge::io_task`'s `OnceLock` pattern (the codec's FFM -//! entry points are synchronous `extern "C"`). -//! -//! Primitives only (INT32/INT64/date → i64 words). BYTE_ARRAY/keyword pages are not cached here. -//! Gated by `set_enabled(true)` from Java; when disabled every entry point is a cheap no-op and the -//! codec's decode path is unchanged. - -use std::collections::HashMap; -use std::future::IntoFuture; -use std::path::PathBuf; -use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; -use std::sync::{Arc, Mutex, OnceLock}; - -use arrow::array::{Array, ArrayRef, Int64Array}; -use liquid_cache::cache::{EntryID, LiquidCache, LiquidCacheBuilder}; -use tokio::runtime::Runtime; - -/// The process-global codec-owned decoded-page cache, built on first use. `None` when the cache is -/// disabled or its one-time build failed (e.g. the store directory could not be mounted) — in that -/// case the codec silently falls back to decoding every page, rather than failing the query. -static CACHE: OnceLock>> = OnceLock::new(); - -/// Dedicated runtime for driving liquid's async `insert`/`get` from the synchronous FFM path. -static RT: OnceLock = OnceLock::new(); - -/// Master on/off switch, set by Java at init. Off by default → the codec decode path is untouched. -/// May be flipped back off internally if the cache fails to build (see `cache`). -static ENABLED: AtomicBool = AtomicBool::new(false); - -/// Process-wide liquid-cache event counters (monotonic, Relaxed). A single `long[]`-worth of -/// atomics incremented on the cache get/put paths; readers snapshot them via -/// `parquet_liquid_cache_stats`. Cost is one relaxed atomic add per page-boundary event (never -/// per-doc), so they stay on unconditionally without measurable latency. -/// - HITS : a `get_page*` served the page from liquid (no Parquet decode) -/// - MISSES : a `get_page*` found nothing → the caller decodes from Parquet -/// - PUTS : a decoded page was inserted into liquid (`put_page`) -static LIQUID_HITS: AtomicU64 = AtomicU64::new(0); -static LIQUID_MISSES: AtomicU64 = AtomicU64::new(0); -static LIQUID_PUTS: AtomicU64 = AtomicU64::new(0); - -/// Snapshot of the liquid event counters: `(hits, misses, puts)`. -pub fn stats_snapshot() -> (u64, u64, u64) { - ( - LIQUID_HITS.load(Ordering::Relaxed), - LIQUID_MISSES.load(Ordering::Relaxed), - LIQUID_PUTS.load(Ordering::Relaxed), - ) -} - -/// Configured max memory budget for the cache (bytes). Applied when the cache is first built. -static MAX_MEMORY_BYTES: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0); - -/// Directory under which the liquid `t4` store is mounted, supplied by Java at init (a writable -/// path derived from the node's data directory — never tmpfs). Empty until `set_enabled` runs. -static CACHE_DIR: OnceLock> = OnceLock::new(); - -/// Codec-local file path → small integer id registry, so entries carry file identity without -/// depending on DataFusion's file numbering. -static FILE_IDS: OnceLock>> = OnceLock::new(); - -/// Enable/disable the cache and set the memory budget + store directory. Called by Java at plugin -/// init when the `parquet_liquid_cache` feature flag is on. `cache_dir` must be a writable directory -/// on real disk (the caller passes a path under the node's data dir); the `t4` store is mounted -/// inside it. A `max_memory_bytes` of 0 leaves the liquid default. -pub fn set_enabled(enabled: bool, max_memory_bytes: usize, cache_dir: &str) { - MAX_MEMORY_BYTES.store(max_memory_bytes, Ordering::Relaxed); - let slot = CACHE_DIR.get_or_init(|| Mutex::new(String::new())); - if let Ok(mut guard) = slot.lock() { - *guard = cache_dir.to_string(); - } - ENABLED.store(enabled, Ordering::Relaxed); -} - -/// True when the cache should be consulted. Cheap relaxed load on the hot path. -#[inline] -pub fn enabled() -> bool { - ENABLED.load(Ordering::Relaxed) -} - -/// Clears all cached decoded pages (the in-memory index and any spilled `t4` entries) without -/// disabling the cache or tearing down the runtime: resets the liquid index and budget usage, so -/// subsequent reads re-decode and re-populate. A no-op when the cache is disabled or not yet built. -/// Never panics — a clear failure must not poison the column-reader mutex or fail doc-values reads. -pub fn clear() { - if let Some(c) = cache() { - c.reset(); - crate::log_info!("liquid_page_cache: cache cleared"); - } -} - -fn runtime() -> &'static Runtime { - RT.get_or_init(|| { - tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .expect("liquid_page_cache: failed to build tokio runtime") - }) -} - -/// Build the `Arc` once, mounting a `t4` store at `/parquet_liquid_cache.t4`. -/// Returns `None` on any failure (missing/unwritable dir, mount error) after logging — the caller -/// then disables the cache so the decode path continues unaffected. Never panics: a cache problem -/// must not poison the column-reader mutex or fail doc-values reads. -fn build_cache() -> Option> { - let dir = CACHE_DIR - .get() - .and_then(|m| m.lock().ok().map(|g| g.clone())) - .unwrap_or_default(); - if dir.is_empty() { - crate::log_error!("liquid_page_cache: no cache_dir configured; disabling codec liquid cache"); - return None; - } - let base = PathBuf::from(&dir).join(format!("parquet_liquid_cache_{}", std::process::id())); - if let Err(e) = std::fs::create_dir_all(&base) { - crate::log_error!( - "liquid_page_cache: failed to create cache dir {:?}: {}; disabling codec liquid cache", - base, e - ); - return None; - } - let store_path = base.join("store.t4"); - let store = match runtime().block_on(t4::mount(&store_path)) { - Ok(s) => s, - Err(e) => { - crate::log_error!( - "liquid_page_cache: failed to mount t4 store at {:?}: {}; disabling codec liquid cache", - store_path, e - ); - return None; - } - }; - let mut builder = LiquidCacheBuilder::new().with_store(store); - let budget = MAX_MEMORY_BYTES.load(Ordering::Relaxed); - if budget > 0 { - builder = builder.with_max_memory_bytes(budget); - } - crate::log_info!("liquid_page_cache: codec liquid cache initialized at {:?}", base); - Some(runtime().block_on(builder.build())) -} - -/// Access the process-global cache, building it once. On build failure this returns `None` and -/// flips `ENABLED` off so subsequent `get_page`/`put_page` calls short-circuit without retrying. -fn cache() -> Option<&'static Arc> { - let slot = CACHE.get_or_init(build_cache); - if slot.is_none() { - ENABLED.store(false, Ordering::Relaxed); - } - slot.as_ref() -} - -/// Resolve (or assign) a stable small id for a Parquet file path. Codec-local; independent of any -/// DataFusion file numbering. -pub fn file_id(path: &str) -> u32 { - let map = FILE_IDS.get_or_init(|| Mutex::new(HashMap::new())); - let mut guard = map.lock().expect("liquid_page_cache: file id registry poisoned"); - let next = guard.len() as u32; - *guard.entry(path.to_string()).or_insert(next) -} - -/// Pack `(file_id, column_id, page_idx)` into a liquid `EntryID`. u16 column id + u32 page fit -/// alongside the file id in a usize on 64-bit targets. -#[inline] -pub fn entry_id(file_id: u32, column_id: u32, page_idx: u32) -> EntryID { - let v = ((file_id as usize) << 48) | ((column_id as usize) << 32) | (page_idx as usize); - EntryID::from(v) -} - -/// Look up a cached decoded page. Returns `(longs, presence)` in the exact form the decode arms -/// produce (`longs[i]` valid iff `presence[i]`), or `None` on a miss. -/// -/// Retained alongside the faster [`get_page_into_outbuf`] so the two hit paths can be A/B profiled -/// (this one materializes a `Vec` + `Vec`; the other writes straight to the FFM buffers). -/// Not on the live hot path — `parquet_decode_page_at_row` calls `get_page_into_outbuf`. -#[allow(dead_code)] -pub fn get_page(eid: EntryID) -> Option<(Vec, Vec)> { - let cache = cache()?; - let array: ArrayRef = runtime().block_on(cache.get(&eid).read())?; - let int_array = array.as_any().downcast_ref::()?; - let len = int_array.len(); - let mut longs = Vec::with_capacity(len); - let mut presence = Vec::with_capacity(len); - for i in 0..len { - if int_array.is_null(i) { - longs.push(0); - presence.push(false); - } else { - longs.push(int_array.value(i)); - presence.push(true); - } - } - Some((longs, presence)) -} - -/// Look up a cached decoded page and write it **straight into the caller's FFM out-buffers**, -/// skipping the intermediate `Vec` + `Vec` rebuild that [`get_page`] does. This is the -/// hot-path variant: on a warm aggregation the cached `Int64Array` already stores the data in the -/// exact layout Java reads, so a hit is two `memcpy`s (values + validity) instead of an -/// element-by-element loop that is then recopied by `write_primitive_page`. -/// -/// Buffer contract mirrors `write_primitive_page` exactly: -/// - writes `out_value_actual_len = len * 8` up front, -/// - returns `Some(RC_OVERFLOW)` if either out-buffer is too small (caller sizes buffers and -/// retries — the actual length is already populated), -/// - returns `Some(RC_OK)` after copying, or -/// - returns `None` on a cache miss (caller then decodes normally). -/// -/// Layout equivalences that make this a raw copy (little-endian target, which the codec value -/// buffer already assumes): -/// - `Int64Array::values()` derefs to `&[i64]` in native-endian order — identical to the FFM value -/// buffer Java reads as a `long[]`. -/// - Arrow's validity bitmap is LSB-first packed bytes with `1 == valid == present`; the codec -/// presence bitset is a little-endian `long[]` with bit `i` == row `i` present. On little-endian -/// the byte layouts coincide, so a byte copy of the validity buffer reproduces -/// `write_presence_bitset`. Null value slots hold 0 and are never read by Java (every read is -/// gated on `isPresent`), so copying them verbatim is behavior-preserving. -/// -/// # Safety -/// The out pointers must be valid for writes of their stated capacities (`out_value_buf_cap` bytes; -/// `out_presence_bits_cap` `i64` words), matching the `parquet_decode_page_at_row` FFM contract. -pub unsafe fn get_page_into_outbuf( - eid: EntryID, - out_value_buf: *mut u8, - out_value_buf_cap: i64, - out_value_actual_len: *mut i64, - out_presence_bitset: *mut i64, - out_presence_bits_cap: i64, -) -> Option { - let cache = cache()?; - // Count the liquid outcome at the page-boundary get: Some => hit (served without a Parquet - // decode), None => miss (the caller decodes and typically puts). One relaxed atomic per page. - let array: ArrayRef = match runtime().block_on(cache.get(&eid).read()) { - Some(a) => { - LIQUID_HITS.fetch_add(1, Ordering::Relaxed); - a - } - None => { - LIQUID_MISSES.fetch_add(1, Ordering::Relaxed); - return None; - } - }; - let int_array = array.as_any().downcast_ref::()?; - let len = int_array.len(); - - let value_bytes = (len * 8) as i64; - if !out_value_actual_len.is_null() { - *out_value_actual_len = value_bytes; - } - let presence_words = ((len + 63) / 64) as i64; - if value_bytes > out_value_buf_cap - || out_value_buf.is_null() - || presence_words > out_presence_bits_cap - || out_presence_bitset.is_null() - { - return Some(crate::ffm::RC_OVERFLOW); - } - if len == 0 { - return Some(crate::ffm::RC_OK); - } - - // Zero the whole presence word region first so any trailing bits past `len` are clean; only the - // low `len` bits are ever read by Java, but this keeps the buffer well-defined. - let presence_bytes = (presence_words as usize) * 8; - std::ptr::write_bytes(out_presence_bitset as *mut u8, 0, presence_bytes); - let presence_dst = out_presence_bitset as *mut u8; - - // Cached arrays are always freshly built (offset 0) in `put_page`, so the fast raw-copy path is - // the norm. Guard on the array/validity offsets anyway and fall back to element-wise if a sliced - // array ever reaches here, so correctness never depends on the layout assumption. - let arr_offset = int_array.offset(); - let nulls = int_array.nulls(); - let nulls_unaligned = nulls.map(|n| n.inner().offset() != 0).unwrap_or(false); - - if arr_offset == 0 && !nulls_unaligned { - // Values: one memcpy of the native-endian i64 words. - std::ptr::copy_nonoverlapping(int_array.values().as_ptr() as *const u8, out_value_buf, len * 8); - // Presence: byte copy of the validity bitmap, or all-ones when the column has no nulls. - match nulls { - None => { - let full = len / 8; - std::ptr::write_bytes(presence_dst, 0xFF, full); - let rem = len % 8; - if rem > 0 { - *presence_dst.add(full) = ((1u16 << rem) - 1) as u8; - } - } - Some(nb) => { - let validity: &[u8] = nb.inner().values(); - let n = validity.len().min(presence_bytes); - std::ptr::copy_nonoverlapping(validity.as_ptr(), presence_dst, n); - } - } - } else { - // Rare fallback: sliced array. Copy element by element into the out-buffers. Write values - // as raw bytes (out_value_buf has no guaranteed 8-byte alignment, so an aligned i64 store - // would be UB — mirrors write_primitive_page). - for i in 0..len { - let word: i64 = if int_array.is_null(i) { - 0 - } else { - *presence_dst.add(i / 8) |= 1u8 << (i % 8); - int_array.value(i) - }; - std::ptr::copy_nonoverlapping(&word as *const i64 as *const u8, out_value_buf.add(i * 8), 8); - } - } - Some(crate::ffm::RC_OK) -} - -/// Cache a decoded primitive page. `longs[i]` is meaningful only where `presence[i]` is true; -/// null rows are stored as Arrow nulls so a later `get_page` reconstructs presence exactly. -pub fn put_page(eid: EntryID, longs: &[i64], presence: &[bool]) { - debug_assert_eq!(longs.len(), presence.len()); - let cache = match cache() { - Some(c) => c, - None => return, - }; - let array: Int64Array = longs - .iter() - .zip(presence.iter()) - .map(|(&v, &present)| if present { Some(v) } else { None }) - .collect(); - let array_ref: ArrayRef = Arc::new(array); - // Best-effort: a CacheFull error just means this page is not cached this time. - // `insert`/`get` return builder types that implement `IntoFuture`, so convert before block_on. - let _ = runtime().block_on(cache.insert(eid, array_ref).into_future()); - LIQUID_PUTS.fetch_add(1, Ordering::Relaxed); -} - -/// Cache a decoded primitive page directly from the FFM out-buffers. Reads the already-written -/// value buffer (native-endian `i64` words, one per row) and the packed presence bitset to build -/// the Arrow `Int64Array` for liquid cache insertion. Avoids re-scanning the source data. -/// -/// # Safety -/// `value_buf` must point to `num_rows * 8` valid bytes. `presence_bits` must point to -/// `ceil(num_rows/64)` valid `i64` words — both written by the preceding decode step. -pub unsafe fn put_page_from_outbuf( - eid: EntryID, - value_buf: *const u8, - presence_bits: *const i64, - num_rows: usize, -) { - // PUT_NANOS: build the Arrow array + insert into liquid. Timer gated by the ffm timing flag. - let put_timer = if crate::ffm::timing::on() { Some(std::time::Instant::now()) } else { None }; - let cache = match cache() { - Some(c) => c, - None => return, - }; - - // Build the Arrow Int64Array from the raw out-buffers. The value buffer is already in - // native-endian i64 layout (null slots hold 0); the presence bitset is already LSB-packed - // matching Arrow's validity bitmap byte layout on little-endian. - let values_slice = std::slice::from_raw_parts(value_buf as *const i64, num_rows); - let presence_words = (num_rows + 63) / 64; - let presence_bytes = presence_words * 8; - - // Arrow NullBuffer is LSB-packed bytes. On little-endian our i64 packed bitset has the same - // byte representation, so we can copy the raw bytes. - let validity_bytes = std::slice::from_raw_parts(presence_bits as *const u8, presence_bytes); - - use arrow::buffer::{Buffer, NullBuffer}; - use arrow::datatypes::Int64Type; - use arrow::array::PrimitiveArray; - - let values_buf = Buffer::from_slice_ref(values_slice); - let null_buf = NullBuffer::new(arrow::buffer::BooleanBuffer::new( - Buffer::from_slice_ref(validity_bytes), - 0, - num_rows, - )); - let array = PrimitiveArray::::new(values_buf.into(), Some(null_buf)); - let array_ref: ArrayRef = Arc::new(array); - let _ = runtime().block_on(cache.insert(eid, array_ref).into_future()); - LIQUID_PUTS.fetch_add(1, Ordering::Relaxed); - if let Some(t) = put_timer { - crate::ffm::timing::record(&crate::ffm::timing::PUT_NANOS, t); - } -} diff --git a/sandbox/plugins/parquet-data-format/src/test/java/org/opensearch/parquet/ParquetSettingsTests.java b/sandbox/plugins/parquet-data-format/src/test/java/org/opensearch/parquet/ParquetSettingsTests.java index 2f56b375a4929..73c85b87294d2 100644 --- a/sandbox/plugins/parquet-data-format/src/test/java/org/opensearch/parquet/ParquetSettingsTests.java +++ b/sandbox/plugins/parquet-data-format/src/test/java/org/opensearch/parquet/ParquetSettingsTests.java @@ -21,17 +21,6 @@ public class ParquetSettingsTests extends OpenSearchTestCase { - public void testDocValuesDecodePath() { - assertEquals(ParquetSettings.DECODE_PATH_CODEC_NATIVE, ParquetSettings.DOCVALUES_DECODE_PATH.get(Settings.EMPTY)); - Settings settings = Settings.builder().put("parquet.docvalues.decode_path", "DATAFUSION").build(); - assertEquals(ParquetSettings.DECODE_PATH_DATAFUSION, ParquetSettings.DOCVALUES_DECODE_PATH.get(settings)); - } - - public void testInvalidDocValuesDecodePathThrows() { - Settings settings = Settings.builder().put("parquet.docvalues.decode_path", "other").build(); - expectThrows(IllegalArgumentException.class, () -> ParquetSettings.DOCVALUES_DECODE_PATH.get(settings)); - } - public void testDocValuesInitialBatchSize() { assertEquals(32, ParquetSettings.DOCVALUES_INITIAL_BATCH_SIZE.get(Settings.EMPTY).intValue()); Settings settings = Settings.builder().put("parquet.docvalues.initial_batch_size", 4196).build(); diff --git a/sandbox/plugins/parquet-data-format/src/test/java/org/opensearch/parquet/bridge/ParquetColumnReaderTests.java b/sandbox/plugins/parquet-data-format/src/test/java/org/opensearch/parquet/bridge/ParquetColumnReaderTests.java deleted file mode 100644 index f8054ee275e6a..0000000000000 --- a/sandbox/plugins/parquet-data-format/src/test/java/org/opensearch/parquet/bridge/ParquetColumnReaderTests.java +++ /dev/null @@ -1,270 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.parquet.bridge; - -import org.apache.arrow.c.ArrowArray; -import org.apache.arrow.c.ArrowSchema; -import org.apache.arrow.c.Data; -import org.apache.arrow.memory.BufferAllocator; -import org.apache.arrow.memory.RootAllocator; -import org.apache.arrow.vector.BigIntVector; -import org.apache.arrow.vector.IntVector; -import org.apache.arrow.vector.VarCharVector; -import org.apache.arrow.vector.VectorSchemaRoot; -import org.apache.arrow.vector.types.pojo.ArrowType; -import org.apache.arrow.vector.types.pojo.Field; -import org.apache.arrow.vector.types.pojo.FieldType; -import org.apache.arrow.vector.types.pojo.Schema; -import org.opensearch.nativebridge.spi.ArrowExport; -import org.opensearch.parquet.codec.ParquetPhysicalType; -import org.opensearch.parquet.codec.cache.BufferPool; -import org.opensearch.parquet.codec.cache.ColumnPageIndex; -import org.opensearch.parquet.codec.cache.PageCache; -import org.opensearch.parquet.codec.iter.ParquetNumericDocValues; -import org.opensearch.test.OpenSearchTestCase; - -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.nio.file.Path; -import java.util.List; - -/** - * Unit tests for {@link ParquetColumnReader} (task 2.4). Writes a small Parquet file with - * the native writer, then exercises single/repeated reads, null handling, error paths, the - * page index, and page-decode cache population. - */ -public class ParquetColumnReaderTests extends OpenSearchTestCase { - - private BufferAllocator allocator; - private Schema schema; - - @Override - public void setUp() throws Exception { - super.setUp(); - RustBridge.initLogger(); - allocator = new RootAllocator(); - schema = new Schema( - List.of( - new Field("id", FieldType.nullable(new ArrowType.Int(32, true)), null), - new Field("name", FieldType.nullable(new ArrowType.Utf8()), null), - new Field("score", FieldType.nullable(new ArrowType.Int(64, true)), null) - ) - ); - } - - @Override - public void tearDown() throws Exception { - allocator.close(); - super.tearDown(); - } - - public void testReadIntColumnSingleValues() throws Exception { - Path file = writeFile(new int[] { 10, 20, 30 }, new String[] { "alice", "bob", "carol" }, new long[] { 100L, 200L, 300L }); - try (BufferPool pool = new BufferPool(); ParquetColumnReader r = open(file, "id", ParquetPhysicalType.INT32, false, pool)) { - assertEquals(new ParquetColumnReader.Value(true, 10L), r.readValueAtRow(0)); - assertEquals(new ParquetColumnReader.Value(true, 20L), r.readValueAtRow(1)); - assertEquals(new ParquetColumnReader.Value(true, 30L), r.readValueAtRow(2)); - } - } - - public void testReadBinaryColumnWithNull() throws Exception { - Path file = writeFileWithNullName(new int[] { 1, 2, 3 }, new String[] { "alice", null, "carol" }, new long[] { 1, 2, 3 }); - try (BufferPool pool = new BufferPool(); ParquetColumnReader r = open(file, "name", ParquetPhysicalType.BYTE_ARRAY, false, pool)) { - assertEquals("alice", new String(r.readBytesAtRow(0), StandardCharsets.UTF_8)); - assertNull(r.readBytesAtRow(1)); - assertEquals("carol", new String(r.readBytesAtRow(2), StandardCharsets.UTF_8)); - } - } - - public void testRowOutOfRangeThrows() throws Exception { - Path file = writeFile(new int[] { 1, 2 }, new String[] { "a", "b" }, new long[] { 1, 2 }); - try (BufferPool pool = new BufferPool(); ParquetColumnReader r = open(file, "id", ParquetPhysicalType.INT32, false, pool)) { - IOException e = expectThrows(IOException.class, () -> r.readValueAtRow(99)); - assertTrue("error should name the row: " + e.getMessage(), e.getMessage().contains("99")); - } - } - - public void testMissingColumnThrows() throws Exception { - Path file = writeFile(new int[] { 1 }, new String[] { "a" }, new long[] { 1 }); - try (BufferPool pool = new BufferPool()) { - IOException e = expectThrows(IOException.class, () -> open(file, "nope", ParquetPhysicalType.INT32, false, pool)); - assertTrue("error should name the missing column: " + e.getMessage(), e.getMessage().contains("nope")); - } - } - - public void testTypeMismatchThrows() throws Exception { - Path file = writeFile(new int[] { 1 }, new String[] { "a" }, new long[] { 1 }); - try (BufferPool pool = new BufferPool()) { - // id is INT32 — asking for BYTE_ARRAY must fail. - IOException e = expectThrows(IOException.class, () -> open(file, "id", ParquetPhysicalType.BYTE_ARRAY, false, pool)); - assertTrue( - "error should mention a mismatch: " + e.getMessage(), - e.getMessage().toLowerCase(java.util.Locale.ROOT).contains("mismatch") - ); - } - } - - public void testPageIndexAndDecode() throws Exception { - Path file = writeFile(new int[] { 10, 20, 30 }, new String[] { "a", "b", "c" }, new long[] { 1, 2, 3 }); - try (BufferPool pool = new BufferPool(); ParquetColumnReader r = open(file, "id", ParquetPhysicalType.INT32, false, pool)) { - ColumnPageIndex idx = r.pageIndex(); - assertTrue("expected at least one page", idx.pageCount() >= 1); - assertEquals(0L, idx.firstRowOf(0)); - assertEquals(3L, idx.totalRows()); - assertEquals(0, idx.pageForRow(0)); - assertEquals(0, idx.pageForRow(2)); - - r.loadPageContaining(1); - PageCache cache = r.cache(); - assertNotNull("page should be cached", cache); - assertEquals(0L, cache.firstRow); - assertEquals(2L, cache.lastRow); - assertTrue(cache.isPresent(1)); - assertEquals(20L, cache.valueAt(1)); - assertEquals(10L, cache.valueAt(0)); - assertEquals(30L, cache.valueAt(2)); - } - } - - public void testCloseIsIdempotentAndHandlesDoNotLeak() throws Exception { - Path file = writeFile(new int[] { 1, 2 }, new String[] { "a", "b" }, new long[] { 1, 2 }); - long before = RustBridge.openColumnReaderCount(); - try (BufferPool pool = new BufferPool()) { - ParquetColumnReader r = open(file, "id", ParquetPhysicalType.INT32, false, pool); - assertEquals(before + 1, RustBridge.openColumnReaderCount()); - r.close(); - r.close(); // idempotent - assertEquals(before, RustBridge.openColumnReaderCount()); - } - } - - public void testDataFusionReaderUsesAdaptiveBatchesAndSharedNumericIterator() throws Exception { - int rowCount = 100; - int[] ids = new int[rowCount]; - String[] names = new String[rowCount]; - long[] scores = new long[rowCount]; - for (int i = 0; i < rowCount; i++) { - ids[i] = i; - names[i] = "value-" + i; - scores[i] = i * 10L; - } - Path file = writeFile(ids, names, scores); - long before = RustBridge.dfOpenIterCount(); - - try (BufferPool pool = new BufferPool(); DataFusionColumnReader reader = DataFusionColumnReader.open(file, "id", pool)) { - assertEquals(before + 1, RustBridge.dfOpenIterCount()); - ParquetNumericDocValues values = new ParquetNumericDocValues(reader, rowCount); - - assertTrue(values.advanceExact(0)); - assertEquals(0L, values.longValue()); - assertEquals(0L, reader.cache().firstRow); - assertEquals(31L, reader.cache().lastRow); - - assertTrue(values.advanceExact(32)); - assertEquals(32L, values.longValue()); - assertEquals(32L, reader.cache().firstRow); - assertEquals(95L, reader.cache().lastRow); - - assertTrue(values.advanceExact(99)); - assertEquals(99L, values.longValue()); - assertEquals(99L, reader.cache().firstRow); - assertEquals(99L, reader.cache().lastRow); - } - assertEquals(before, RustBridge.dfOpenIterCount()); - } - - public void testDataFusionReaderReopensForBackwardSeeks() throws Exception { - int rowCount = 100; - int[] ids = new int[rowCount]; - String[] names = new String[rowCount]; - long[] scores = new long[rowCount]; - for (int i = 0; i < rowCount; i++) { - ids[i] = i; - names[i] = "value-" + i; - scores[i] = i * 10L; - } - Path file = writeFile(ids, names, scores); - long before = RustBridge.dfOpenIterCount(); - - try (BufferPool pool = new BufferPool(); DataFusionColumnReader reader = DataFusionColumnReader.open(file, "id", pool)) { - // Two iterators over the same column share one native cursor — e.g. a parent - // aggregation and a filtered sub-aggregation advancing at different rates. - ParquetNumericDocValues leading = new ParquetNumericDocValues(reader, rowCount); - ParquetNumericDocValues trailing = new ParquetNumericDocValues(reader, rowCount); - - assertTrue(leading.advanceExact(90)); - assertEquals(90L, leading.longValue()); - - // The forward-only cursor is past row 5; the reader must reopen, not fail. - assertTrue(trailing.advanceExact(5)); - assertEquals(5L, trailing.longValue()); - - // Forward progress still works after the reopen. - assertTrue(leading.advanceExact(95)); - assertEquals(95L, leading.longValue()); - - // Reopen swaps cursors one-for-one: exactly one remains live. - assertEquals(before + 1, RustBridge.dfOpenIterCount()); - } - assertEquals(before, RustBridge.dfOpenIterCount()); - } - - // ── helpers ── - - private static ParquetColumnReader open(Path file, String col, ParquetPhysicalType type, boolean repeated, BufferPool pool) - throws IOException { - return ParquetColumnReader.open(file, col, type, repeated, pool); - } - - private Path writeFile(int[] ids, String[] names, long[] scores) throws Exception { - return writeFileWithNullName(ids, names, scores); - } - - private Path writeFileWithNullName(int[] ids, String[] names, long[] scores) throws Exception { - Path file = createTempDir().resolve("colreader.parquet"); - NativeParquetWriter writer = new NativeParquetWriter(file.toString()); - try (ArrowExport schemaExport = exportSchema()) { - writer.initialize("test-index", schemaExport.getSchemaAddress(), ParquetSortConfig.empty(), 0L); - } - try (ArrowExport export = exportData(ids, names, scores)) { - writer.write(export.getArrayAddress(), export.getSchemaAddress()); - } - writer.flush(); - return file; - } - - private ArrowExport exportSchema() { - ArrowSchema arrowSchema = ArrowSchema.allocateNew(allocator); - Data.exportSchema(allocator, schema, null, arrowSchema); - return new ArrowExport(null, arrowSchema); - } - - private ArrowExport exportData(int[] ids, String[] names, long[] scores) { - try (VectorSchemaRoot root = VectorSchemaRoot.create(schema, allocator)) { - IntVector idVec = (IntVector) root.getVector("id"); - VarCharVector nameVec = (VarCharVector) root.getVector("name"); - BigIntVector scoreVec = (BigIntVector) root.getVector("score"); - for (int i = 0; i < ids.length; i++) { - idVec.setSafe(i, ids[i]); - if (names[i] == null) { - nameVec.setNull(i); - } else { - nameVec.setSafe(i, names[i].getBytes(StandardCharsets.UTF_8)); - } - scoreVec.setSafe(i, scores[i]); - } - root.setRowCount(ids.length); - - ArrowArray array = ArrowArray.allocateNew(allocator); - ArrowSchema arrowSchema = ArrowSchema.allocateNew(allocator); - Data.exportVectorSchemaRoot(allocator, root, null, array, arrowSchema); - return new ArrowExport(array, arrowSchema); - } - } -} diff --git a/sandbox/plugins/parquet-data-format/src/test/java/org/opensearch/parquet/codec/DataFusionDocValuesTests.java b/sandbox/plugins/parquet-data-format/src/test/java/org/opensearch/parquet/codec/DataFusionDocValuesTests.java index bc3a96e415f8f..7557a2f3ffaa2 100644 --- a/sandbox/plugins/parquet-data-format/src/test/java/org/opensearch/parquet/codec/DataFusionDocValuesTests.java +++ b/sandbox/plugins/parquet-data-format/src/test/java/org/opensearch/parquet/codec/DataFusionDocValuesTests.java @@ -38,7 +38,6 @@ import org.apache.lucene.util.StringHelper; import org.apache.lucene.util.Version; import org.opensearch.nativebridge.spi.ArrowExport; -import org.opensearch.parquet.ParquetSettings; import org.opensearch.parquet.bridge.NativeParquetWriter; import org.opensearch.parquet.bridge.ParquetSortConfig; import org.opensearch.parquet.bridge.RustBridge; @@ -59,12 +58,10 @@ public void setUp() throws Exception { super.setUp(); RustBridge.initLogger(); allocator = new RootAllocator(); - ParquetDocValuesProducer.setDecodePath(ParquetSettings.DECODE_PATH_DATAFUSION); } @Override public void tearDown() throws Exception { - ParquetDocValuesProducer.setDecodePath(ParquetSettings.DECODE_PATH_CODEC_NATIVE); allocator.close(); super.tearDown(); } @@ -76,7 +73,6 @@ public void testRepeatedTypesUseDataFusionWithoutCodecNativeFallback() throws Ex FieldInfo numbers = fieldInfo("numbers", 0, DocValuesType.SORTED_NUMERIC); FieldInfo tags = fieldInfo("tags", 1, DocValuesType.SORTED_SET); - long nativeReadersBefore = RustBridge.openColumnReaderCount(); long dataFusionReadersBefore = RustBridge.dfOpenIterCount(); try ( @@ -84,7 +80,6 @@ public void testRepeatedTypesUseDataFusionWithoutCodecNativeFallback() throws Ex ParquetDocValuesProducer producer = new ParquetDocValuesProducer(segmentReadState(directory, parquetFile, numbers, tags), null) ) { SortedNumericDocValues numeric = producer.getSortedNumeric(numbers); - assertEquals(nativeReadersBefore, RustBridge.openColumnReaderCount()); assertEquals(dataFusionReadersBefore + 1, RustBridge.dfOpenIterCount()); assertTrue(numeric.advanceExact(0)); @@ -100,7 +95,6 @@ public void testRepeatedTypesUseDataFusionWithoutCodecNativeFallback() throws Ex assertEquals(8L, numeric.nextValue()); SortedSetDocValues sortedSet = producer.getSortedSet(tags); - assertEquals(nativeReadersBefore, RustBridge.openColumnReaderCount()); assertEquals(dataFusionReadersBefore + 2, RustBridge.dfOpenIterCount()); assertTrue(sortedSet.advanceExact(0)); @@ -114,10 +108,8 @@ public void testRepeatedTypesUseDataFusionWithoutCodecNativeFallback() throws Ex assertEquals(new BytesRef("alpha"), sortedSet.lookupOrd(sortedSet.nextOrd())); assertEquals(new BytesRef("omega"), sortedSet.lookupOrd(sortedSet.nextOrd())); - assertEquals(nativeReadersBefore, RustBridge.openColumnReaderCount()); } - assertEquals(nativeReadersBefore, RustBridge.openColumnReaderCount()); assertEquals(dataFusionReadersBefore, RustBridge.dfOpenIterCount()); }