From 73e83ded751796f8ca651ff6585868d6f5f1b6be Mon Sep 17 00:00:00 2001 From: Darek Date: Tue, 20 Oct 2026 14:27:34 +0200 Subject: [PATCH] perf(processor): reuse Intersect's per-probe scratch buffers Intersect allocated on every probed row: probeHTs built a vector of vectors it never read. Each intersected tuple allocated five more buffers: the tuple pointers, the lists, the list order, the left selection vector (up to 16 KB) and the list of selection vectors to slice. These are now members sized once in initLocalStateInternal, so probing does not allocate. initLocalStateInternal also resets the carry state (probedFlatTuples, tupleIdxPerBuildSide, carryBuildSideIdx), which only the constructor set, so a reused operator starts clean as its comment says. Co-Authored-By: Claude Opus 5.5 --- .../processor/operator/intersect/intersect.h | 12 +++- .../operator/intersect/intersect.cpp | 58 ++++++++++--------- test/test_files/cyclic/large_lists.test | 38 ++++++++++++ 3 files changed, 79 insertions(+), 29 deletions(-) create mode 100644 test/test_files/cyclic/large_lists.test diff --git a/src/include/processor/operator/intersect/intersect.h b/src/include/processor/operator/intersect/intersect.h index f932ca1581..650919c5ad 100644 --- a/src/include/processor/operator/intersect/intersect.h +++ b/src/include/processor/operator/intersect/intersect.h @@ -61,9 +61,8 @@ class Intersect : public PhysicalOperator { // Left is always the one with less num of values. static void twoWayIntersect(common::nodeID_t* leftNodeIDs, common::SelectionVector& lSelVector, common::nodeID_t* rightNodeIDs, common::SelectionVector& rSelVector); - void intersectLists(const std::vector& listsToIntersect); - void populatePayloads(const std::vector& tuples, - const std::vector& listIdxes); + void intersectLists(); + void populatePayloads(); bool hasNextTuplesToIntersect(); uint32_t getNumBuilds() { return sharedHTs.size(); } @@ -86,6 +85,13 @@ class Intersect : public PhysicalOperator { uint32_t carryBuildSideIdx; // Aligned copy of the right-hand list being intersected (see intersectLists). std::vector alignedRightList; + // Per-tuple scratch, so probing does not allocate. The vectors hold one entry per build side + // and are sized in initLocalStateInternal; leftSelVector has DEFAULT_VECTOR_CAPACITY. + std::vector flatTuplesToIntersect; + std::vector listsToIntersect; + std::vector listIdxes; + std::vector selVectorsForIntersectedLists; + common::SelectionVector leftSelVector; }; } // namespace processor diff --git a/src/processor/operator/intersect/intersect.cpp b/src/processor/operator/intersect/intersect.cpp index 98b322ef15..bd1962671f 100644 --- a/src/processor/operator/intersect/intersect.cpp +++ b/src/processor/operator/intersect/intersect.cpp @@ -47,10 +47,19 @@ void Intersect::initLocalStateInternal(ResultSet* resultSet, ExecutionContext* / sharedHT->getHashTable()->getTableSchema()->getColumn(1)->isFlat()); } alignedRightList.resize(DEFAULT_VECTOR_CAPACITY); + flatTuplesToIntersect.assign(sharedHTs.size(), nullptr); + listsToIntersect.assign(sharedHTs.size(), overflow_value_t{}); + listIdxes.assign(sharedHTs.size(), 0); + selVectorsForIntersectedLists.clear(); + selVectorsForIntersectedLists.reserve(sharedHTs.size()); + for (auto& tuples : probedFlatTuples) { + tuples.clear(); + } + std::fill(tupleIdxPerBuildSide.begin(), tupleIdxPerBuildSide.end(), 0); + carryBuildSideIdx = -1u; } void Intersect::probeHTs() { - std::vector> flatTuples(probeKeyVectors.size()); hash_t hashVal = 0; for (auto i = 0u; i < probeKeyVectors.size(); i++) { DASSERT(probeKeyVectors[i]->state->isFlat()); @@ -85,9 +94,10 @@ void Intersect::twoWayIntersect(nodeID_t* leftNodeIDs, SelectionVector& lSelVect rSelVector.setToFiltered(outputValuePosition); } -static std::vector fetchListsToIntersectFromTuples( - const std::vector& tuples, const std::vector& isFlatValue) { - std::vector listsToIntersect(tuples.size()); +// Overwrites every entry of listsToIntersect, which has one entry per tuple. +static void fetchListsToIntersectFromTuples(const std::vector& tuples, + const std::vector& isFlatValue, std::vector& listsToIntersect) { + DASSERT(listsToIntersect.size() == tuples.size()); for (auto i = 0u; i < tuples.size(); i++) { if (isFlatValue[i]) { listsToIntersect[i] = @@ -97,12 +107,12 @@ static std::vector fetchListsToIntersectFromTuples( memcpy(&listsToIntersect[i], tuples[i] + sizeof(nodeID_t), sizeof(overflow_value_t)); } } - return listsToIntersect; } -static std::vector swapSmallestListToFront(std::vector& lists) { - DASSERT(lists.size() >= 2); - std::vector listIdxes(lists.size()); +// Overwrites every entry of listIdxes, which has one entry per list. +static void swapSmallestListToFront(std::vector& lists, + std::vector& listIdxes) { + DASSERT(lists.size() >= 2 && listIdxes.size() == lists.size()); iota(listIdxes.begin(), listIdxes.end(), 0); uint32_t smallestListIdx = 0; for (auto i = 1u; i < lists.size(); i++) { @@ -114,7 +124,6 @@ static std::vector swapSmallestListToFront(std::vector& selVectorsToSlice, @@ -129,7 +138,7 @@ static void sliceSelVectors(const std::vector& selVectorsToSli } } -void Intersect::intersectLists(const std::vector& listsToIntersect) { +void Intersect::intersectLists() { if (listsToIntersect[0].numElements == 0) { outKeyVector->state->getSelVectorUnsafe().setSelSize(0); return; @@ -137,9 +146,8 @@ void Intersect::intersectLists(const std::vector& listsToInter DASSERT(listsToIntersect[0].numElements <= DEFAULT_VECTOR_CAPACITY); memcpy(outKeyVector->getData(), listsToIntersect[0].value, listsToIntersect[0].numElements * sizeof(nodeID_t)); - SelectionVector lSelVector(listsToIntersect[0].numElements); - lSelVector.setSelSize(listsToIntersect[0].numElements); - std::vector selVectorsForIntersectedLists; + leftSelVector.setToUnfiltered(listsToIntersect[0].numElements); + selVectorsForIntersectedLists.clear(); intersectSelVectors[0]->setToUnfiltered(listsToIntersect[0].numElements); selVectorsForIntersectedLists.push_back(intersectSelVectors[0].get()); // Lists live in packed factorized-table/unflat storage without alignment padding, so the raw @@ -157,24 +165,23 @@ void Intersect::intersectLists(const std::vector& listsToInter memcpy(alignedRightList.data(), rightList.value, rightList.numElements * sizeof(nodeID_t)); } - twoWayIntersect((nodeID_t*)outKeyVector->getData(), lSelVector, alignedRightList.data(), + twoWayIntersect((nodeID_t*)outKeyVector->getData(), leftSelVector, alignedRightList.data(), *intersectSelVectors[i + 1]); // Here we need to slice all selVectors that have been previously intersected, as all these // lists need to be selected synchronously to read payloads correctly. - sliceSelVectors(selVectorsForIntersectedLists, lSelVector); - lSelVector.setToUnfiltered(); + sliceSelVectors(selVectorsForIntersectedLists, leftSelVector); + leftSelVector.setToUnfiltered(); selVectorsForIntersectedLists.push_back(intersectSelVectors[i + 1].get()); } - outKeyVector->state->getSelVectorUnsafe().setSelSize(lSelVector.getSelSize()); + outKeyVector->state->getSelVectorUnsafe().setSelSize(leftSelVector.getSelSize()); } -void Intersect::populatePayloads(const std::vector& tuples, - const std::vector& listIdxes) { +void Intersect::populatePayloads() { for (auto i = 0u; i < listIdxes.size(); i++) { auto listIdx = listIdxes[i]; sharedHTs[listIdx]->getHashTable()->getFactorizedTable()->lookup( payloadVectorsToScanInto[listIdx], intersectSelVectors[i].get(), - payloadColumnIdxesToScanFrom[listIdx], tuples[listIdx]); + payloadColumnIdxesToScanFrom[listIdx], flatTuplesToIntersect[listIdx]); } } @@ -218,16 +225,15 @@ bool Intersect::getNextTuplesInternal(ExecutionContext* context) { // too large to fit in a single ValueVector, we end up chunking the list as multiple tuples // in FTable. Thus, when performing the intersection, we need to perform cartesian product // between all flat tuples probed from all build sides. - std::vector flatTuplesToIntersect(getNumBuilds()); for (auto i = 0u; i < getNumBuilds(); i++) { flatTuplesToIntersect[i] = probedFlatTuples[i][tupleIdxPerBuildSide[i]]; } - auto listsToIntersect = - fetchListsToIntersectFromTuples(flatTuplesToIntersect, isIntersectListAFlatValue); - auto listIdxes = swapSmallestListToFront(listsToIntersect); - intersectLists(listsToIntersect); + fetchListsToIntersectFromTuples(flatTuplesToIntersect, isIntersectListAFlatValue, + listsToIntersect); + swapSmallestListToFront(listsToIntersect, listIdxes); + intersectLists(); if (outKeyVector->state->getSelVector().getSelSize() != 0) { - populatePayloads(flatTuplesToIntersect, listIdxes); + populatePayloads(); } if (!hasNextTuplesToIntersect()) { carryBuildSideIdx = -1u; diff --git a/test/test_files/cyclic/large_lists.test b/test/test_files/cyclic/large_lists.test new file mode 100644 index 0000000000..6d235a9715 --- /dev/null +++ b/test/test_files/cyclic/large_lists.test @@ -0,0 +1,38 @@ +-DATASET CSV empty + +-- + +-CASE IntersectListsLongerThanOneVector +-STATEMENT CREATE NODE TABLE N(id INT64 PRIMARY KEY); +---- ok +-STATEMENT CREATE REL TABLE E(FROM N TO N); +---- ok +-STATEMENT UNWIND range(0, 3000) AS i CREATE (:N {id: i}); +---- ok +# Node 0 points at the other 3000 nodes, so its list is split over two 2048-entry build tuples +# and a single probe intersects several list pairs. +-STATEMENT MATCH (h:N {id: 0}), (n:N) WHERE n.id > 0 CREATE (h)-[:E]->(n); +---- ok +-STATEMENT MATCH (a:N), (b:N) WHERE a.id > 0 AND b.id = a.id + 1 CREATE (a)-[:E]->(b); +---- ok +-STATEMENT MATCH (a:N), (b:N) WHERE a.id > 0 AND b.id = a.id + 2 CREATE (a)-[:E]->(b); +---- ok +-LOG Triangle +-STATEMENT MATCH (a:N)-[e1:E]->(b:N)-[e2:E]->(c:N), (a)-[e3:E]->(c) + HINT (((a JOIN e1) JOIN b) MULTI_JOIN e2 MULTI_JOIN e3) JOIN c + RETURN COUNT(*); +---- 1 +8995 +-STATEMENT MATCH (a:N)-[e1:E]->(b:N)-[e2:E]->(c:N), (a)-[e3:E]->(c) + HINT (((a JOIN e1) JOIN b) MULTI_JOIN e2 MULTI_JOIN e3) JOIN c + RETURN a.id, b.id, c.id ORDER BY c.id DESC, b.id DESC LIMIT 3; +---- 3 +0|2999|3000 +2998|2999|3000 +0|2998|3000 +-LOG FourClique +-STATEMENT MATCH (a:N)-[e1:E]->(b:N)-[e2:E]->(c:N), (a)-[e3:E]->(c), (a)-[e4:E]->(d:N), (b)-[e5:E]->(d), (c)-[e6:E]->(d) + HINT (((((a JOIN e1) JOIN b) MULTI_JOIN e2 MULTI_JOIN e3) JOIN c) MULTI_JOIN e4 MULTI_JOIN e5 MULTI_JOIN e6) JOIN d + RETURN COUNT(*), SUM(a.id + b.id * 3 + c.id * 5 + d.id * 7); +---- 1 +2998|67489477