diff --git a/AUTHORS b/AUTHORS index 648c3aa52212d..f2984e57ba7f5 100644 --- a/AUTHORS +++ b/AUTHORS @@ -230,6 +230,7 @@ - Jonny007-MKD <1-23-4-5@web.de> - Jos Poortvliet - Jose Quinteiro + - Josh Duffy - Josh Richards - Joshua Trees - Juan Pablo Villafáñez diff --git a/apps/files/lib/Search/FilesSearchProvider.php b/apps/files/lib/Search/FilesSearchProvider.php index 93af94db391c9..a91c7fe819b97 100644 --- a/apps/files/lib/Search/FilesSearchProvider.php +++ b/apps/files/lib/Search/FilesSearchProvider.php @@ -10,12 +10,14 @@ namespace OCA\Files\Search; use InvalidArgumentException; +use OC\Files\Search\FilesSearchQuery; use OC\Files\Search\SearchBinaryOperator; use OC\Files\Search\SearchComparison; use OC\Files\Search\SearchOrder; use OC\Files\Search\SearchQuery; use OC\Search\Filter\GroupFilter; use OC\Search\Filter\UserFilter; +use OC\SystemConfig; use OCP\Files\FileInfo; use OCP\Files\IMimeTypeDetector; use OCP\Files\IRootFolder; @@ -42,6 +44,7 @@ public function __construct( private IMimeTypeDetector $mimeTypeDetector, private IRootFolder $rootFolder, private IPreview $previewManager, + private ?SystemConfig $systemConfig = null, ) { } @@ -161,12 +164,30 @@ private function buildSearchQuery(ISearchQuery $query, IUser $user): SearchQuery }; } - return new SearchQuery( + $cursor = (int)$query->getCursor(); + $soleNameLike = count($comparisons) === 1 + && $comparisons[0] instanceof SearchComparison + && $comparisons[0]->getField() === 'name' + && $comparisons[0]->getType() === ISearchComparison::COMPARE_LIKE + && $comparisons[0]->getExtra() === '' + && is_string($comparisons[0]->getValue()); + $bounded = $this->systemConfig?->getValue('files_search_bounded_name', false) === true + && $query->getSortOrder() === ISearchQuery::SORT_DATE_DESC + && $soleNameLike + && $query->getLimit() >= 1 + && $query->getLimit() <= 100 + && $cursor >= 0 + && $cursor <= 1024 - $query->getLimit(); + $queryClass = $bounded ? FilesSearchQuery::class : SearchQuery::class; + return new $queryClass( new SearchBinaryOperator(SearchBinaryOperator::OPERATOR_AND, $comparisons), $query->getLimit(), - (int)$query->getCursor(), + $cursor, $query->getSortOrder() === ISearchQuery::SORT_DATE_DESC - ? [new SearchOrder(ISearchOrder::DIRECTION_DESCENDING, 'mtime')] + ? array_merge([new SearchOrder(ISearchOrder::DIRECTION_DESCENDING, 'mtime')], + // The bounded sole-name opt-in deliberately makes equal-mtime pages + // deterministic. All non-admitted requests retain their native order. + $bounded ? [new SearchOrder(ISearchOrder::DIRECTION_DESCENDING, 'fileid')] : []) : [], $user ); diff --git a/apps/files/tests/Search/FilesSearchProviderTest.php b/apps/files/tests/Search/FilesSearchProviderTest.php new file mode 100644 index 0000000000000..8db96ca432941 --- /dev/null +++ b/apps/files/tests/Search/FilesSearchProviderTest.php @@ -0,0 +1,165 @@ + + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +namespace OCA\Files\Tests\Search; + +use ArrayIterator; +use OC\Files\Search\FilesSearchQuery; +use OC\Files\Search\SearchBinaryOperator; +use OC\Files\Search\SearchComparison; +use OC\Files\Search\SearchQuery as FilesQuery; +use OC\Search\Filter\StringFilter; +use OC\SystemConfig; +use OCA\Files\Search\FilesSearchProvider; +use OCP\Files\IMimeTypeDetector; +use OCP\Files\IRootFolder; +use OCP\Files\IUserFolder; +use OCP\Files\Search\ISearchBinaryOperator; +use OCP\Files\Search\ISearchComparison; +use OCP\Files\Search\ISearchOrder; +use OCP\Files\Search\ISearchQuery as IFilesSearchQuery; +use OCP\IL10N; +use OCP\IPreview; +use OCP\IURLGenerator; +use OCP\IUser; +use OCP\Search\IFilter; +use OCP\Search\IFilterCollection; +use OCP\Search\ISearchQuery; +use PHPUnit\Framework\Attributes\DataProvider; +use Test\TestCase; + +class FilesSearchProviderTest extends TestCase { + /** + * @param array $filters + */ + private function searchAndCaptureQuery(bool $enabled, array $filters, int $sortOrder = ISearchQuery::SORT_DATE_DESC, int $limit = 50, int|string|null $cursor = 0): FilesQuery { + $user = $this->createMock(IUser::class); + $user->method('getUID')->willReturn('user'); + $folder = $this->createMock(IUserFolder::class); + $captured = null; + $folder->expects($this->once()) + ->method('search') + ->willReturnCallback(function (IFilesSearchQuery $query) use (&$captured): array { + $captured = $query; + return []; + }); + + $rootFolder = $this->createMock(IRootFolder::class); + $rootFolder->expects($this->once()) + ->method('getUserFolder') + ->with('user') + ->willReturn($folder); + $l10n = $this->createMock(IL10N::class); + $l10n->method('t')->willReturnArgument(0); + $config = $this->createMock(SystemConfig::class); + $config->method('getValue') + ->with('files_search_bounded_name', false) + ->willReturn($enabled); + + $provider = new FilesSearchProvider( + $l10n, + $this->createMock(IURLGenerator::class), + $this->createMock(IMimeTypeDetector::class), + $rootFolder, + $this->createMock(IPreview::class), + $config, + ); + $query = $this->createMock(ISearchQuery::class); + $query->method('getFilters')->willReturn($this->filters($filters)); + $query->method('getSortOrder')->willReturn($sortOrder); + $query->method('getLimit')->willReturn($limit); + $query->method('getCursor')->willReturn($cursor); + $provider->search($user, $query); + + self::assertInstanceOf(FilesQuery::class, $captured); + return $captured; + } + + /** @param array $filters */ + private function filters(array $filters): IFilterCollection { + return new class($filters) implements IFilterCollection { + /** @param array $filters */ + public function __construct( + private array $filters, + ) { + } + + #[\Override] + public function has(string $name): bool { + return isset($this->filters[$name]); + } + + #[\Override] + public function get(string $name): ?IFilter { + return $this->filters[$name] ?? null; + } + + #[\Override] + public function getIterator(): \Traversable { + return new ArrayIterator($this->filters); + } + + #[\Override] + public function count(): int { + return count($this->filters); + } + }; + } + + public function testBoundedSearchIsOffByDefault(): void { + $query = $this->searchAndCaptureQuery(false, ['term' => new StringFilter('invoice')]); + + self::assertNotInstanceOf(FilesSearchQuery::class, $query); + self::assertSame(1, count($query->getOrder())); + self::assertSame('mtime', $query->getOrder()[0]->getField()); + } + + public function testEnabledSoleTermUsesBoundedQueryAndStableOrder(): void { + $query = $this->searchAndCaptureQuery(true, ['term' => new StringFilter('invoice')], ISearchQuery::SORT_DATE_DESC, 25, 12); + + self::assertInstanceOf(FilesSearchQuery::class, $query); + self::assertSame(25, $query->getLimit()); + self::assertSame(12, $query->getOffset()); + self::assertCount(2, $query->getOrder()); + self::assertSame('mtime', $query->getOrder()[0]->getField()); + self::assertSame(ISearchOrder::DIRECTION_DESCENDING, $query->getOrder()[0]->getDirection()); + self::assertSame('fileid', $query->getOrder()[1]->getField()); + self::assertSame(ISearchOrder::DIRECTION_DESCENDING, $query->getOrder()[1]->getDirection()); + $operation = $query->getSearchOperation(); + self::assertInstanceOf(SearchBinaryOperator::class, $operation); + self::assertSame(ISearchBinaryOperator::OPERATOR_AND, $operation->getType()); + self::assertCount(1, $operation->getArguments()); + self::assertInstanceOf(SearchComparison::class, $operation->getArguments()[0]); + self::assertSame(ISearchComparison::COMPARE_LIKE, $operation->getArguments()[0]->getType()); + self::assertSame('name', $operation->getArguments()[0]->getField()); + self::assertSame('%invoice%', $operation->getArguments()[0]->getValue()); + } + + /** + * @return array, int, int, int|string|null}> + */ + public static function boundedFallbackProvider(): array { + return [ + 'non-date sort' => [['term' => new StringFilter('invoice')], 0, 25, 0], + 'extra request filter' => [['term' => new StringFilter('invoice'), 'type' => new StringFilter('text')], ISearchQuery::SORT_DATE_DESC, 25, 0], + 'zero limit' => [['term' => new StringFilter('invoice')], ISearchQuery::SORT_DATE_DESC, 0, 0], + 'limit over maximum' => [['term' => new StringFilter('invoice')], ISearchQuery::SORT_DATE_DESC, 101, 0], + 'negative cursor' => [['term' => new StringFilter('invoice')], ISearchQuery::SORT_DATE_DESC, 25, -1], + 'cursor beyond bounded window' => [['term' => new StringFilter('invoice')], ISearchQuery::SORT_DATE_DESC, 100, 925], + ]; + } + + #[DataProvider('boundedFallbackProvider')] + public function testEnabledSearchFallsBackWhenRequestIsOutsideBoundedContract(array $filters, int $sortOrder, int $limit, int|string|null $cursor): void { + $query = $this->searchAndCaptureQuery(true, $filters, $sortOrder, $limit, $cursor); + + self::assertNotInstanceOf(FilesSearchQuery::class, $query); + self::assertCount($sortOrder === ISearchQuery::SORT_DATE_DESC ? 1 : 0, $query->getOrder()); + } +} diff --git a/config/config.sample.php b/config/config.sample.php index da3de23d25202..79679bd48039c 100644 --- a/config/config.sample.php +++ b/config/config.sample.php @@ -3129,6 +3129,31 @@ */ 'unified_search.enabled' => false, + /** + * Opt in to bounded, date-sorted Files name searches on MariaDB 10.11. + * Enable this setting, then run ``occ db:add-missing-indices`` to install + * the optional covering indexes. Index creation can take time on a large + * file cache and adds storage and write overhead. Disabling this setting + * does not remove the indexes. Other database versions and unsupported + * query shapes retain native search. + * + * Eligible date-sorted requests use descending file IDs to break equal + * modification-time ties, including when the database capability declines. + * Defaults to ``false``. + */ + 'files_search_bounded_name' => false, + + /** + * Required budgets for bounded Files name searches. Without a valid record, + * the database capability declines. Values must be integers: statement and + * lock seconds in 1..10, total seconds in 2..30, with lock <= statement < total. + * Initial connection negotiation is not bounded by these session settings. + * + * For example: ``['statement_seconds' => 2, 'lock_seconds' => 1, 'total_seconds' => 5]``. + * Defaults to ``null``; no numeric budget is chosen automatically. + */ + 'files_search_bounded_name_limits' => null, + /** * Enable features that do not yet comply with accessibility standards. * diff --git a/core/Listener/AddMissingIndicesListener.php b/core/Listener/AddMissingIndicesListener.php index 22cf7d76735c6..e7c0ddd08400c 100644 --- a/core/Listener/AddMissingIndicesListener.php +++ b/core/Listener/AddMissingIndicesListener.php @@ -9,6 +9,7 @@ namespace OC\Core\Listener; +use OC\SystemConfig; use OCP\DB\Events\AddMissingIndicesEvent; use OCP\EventDispatcher\Event; use OCP\EventDispatcher\IEventListener; @@ -20,6 +21,7 @@ class AddMissingIndicesListener implements IEventListener { public function __construct( private readonly IDBConnection $connection, + private readonly SystemConfig $systemConfig, ) { } @@ -78,6 +80,18 @@ public function handle(Event $event): void { 'fs_name_hash', ['name'] ); + if ($this->supportsBoundedFilesNameSearch()) { + $event->addMissingIndex( + 'filecache', + 'fs_storage_name_mtime_fileid', + ['storage', 'name', 'mtime', 'fileid'], + ); + $event->addMissingIndex( + 'filecache', + 'fs_storage_mtime_fileid_name', + ['storage', 'mtime', 'fileid', 'name'], + ); + } $event->addMissingIndex( 'twofactor_providers', @@ -236,4 +250,24 @@ public function handle(Event $event): void { ['scheduled_at'] ); } + + private function supportsBoundedFilesNameSearch(): bool { + if ($this->systemConfig->getValue('files_search_bounded_name', false) !== true + || $this->connection->getDatabaseProvider(true) !== IDBConnection::PLATFORM_MARIADB) { + return false; + } + + try { + $result = $this->connection->executeQuery('SELECT VERSION()'); + try { + $version = $result->fetchOne(); + } finally { + $result->closeCursor(); + } + } catch (\Throwable) { + return false; + } + + return is_string($version) && preg_match('/^10\\.11\\.[0-9]+(?:-|$)/', $version) === 1; + } } diff --git a/lib/composer/composer/autoload_classmap.php b/lib/composer/composer/autoload_classmap.php index b3b5b63fb3a1e..b87e70125cffa 100644 --- a/lib/composer/composer/autoload_classmap.php +++ b/lib/composer/composer/autoload_classmap.php @@ -1787,6 +1787,7 @@ 'OC\\DB\\ConnectionFactory' => $baseDir . '/lib/private/DB/ConnectionFactory.php', 'OC\\DB\\DbDataCollector' => $baseDir . '/lib/private/DB/DbDataCollector.php', 'OC\\DB\\Exceptions\\DbalException' => $baseDir . '/lib/private/DB/Exceptions/DbalException.php', + 'OC\\DB\\FilesSearchSnapshot' => $baseDir . '/lib/private/DB/FilesSearchSnapshot.php', 'OC\\DB\\Middleware\\ConnectionActivityConnection' => $baseDir . '/lib/private/DB/Middleware/ConnectionActivityConnection.php', 'OC\\DB\\Middleware\\ConnectionActivityDriver' => $baseDir . '/lib/private/DB/Middleware/ConnectionActivityDriver.php', 'OC\\DB\\Middleware\\ConnectionActivityMiddleware' => $baseDir . '/lib/private/DB/Middleware/ConnectionActivityMiddleware.php', @@ -1975,6 +1976,7 @@ 'OC\\Files\\ObjectStore\\Swift' => $baseDir . '/lib/private/Files/ObjectStore/Swift.php', 'OC\\Files\\ObjectStore\\SwiftFactory' => $baseDir . '/lib/private/Files/ObjectStore/SwiftFactory.php', 'OC\\Files\\ObjectStore\\SwiftV2CachingAuthService' => $baseDir . '/lib/private/Files/ObjectStore/SwiftV2CachingAuthService.php', + 'OC\\Files\\Search\\FilesSearchQuery' => $baseDir . '/lib/private/Files/Search/FilesSearchQuery.php', 'OC\\Files\\Search\\QueryOptimizer\\FlattenNestedBool' => $baseDir . '/lib/private/Files/Search/QueryOptimizer/FlattenNestedBool.php', 'OC\\Files\\Search\\QueryOptimizer\\FlattenSingleArgumentBinaryOperation' => $baseDir . '/lib/private/Files/Search/QueryOptimizer/FlattenSingleArgumentBinaryOperation.php', 'OC\\Files\\Search\\QueryOptimizer\\MergeDistributiveOperations' => $baseDir . '/lib/private/Files/Search/QueryOptimizer/MergeDistributiveOperations.php', diff --git a/lib/composer/composer/autoload_static.php b/lib/composer/composer/autoload_static.php index 35c872fd7527d..c8c3bffe589d7 100644 --- a/lib/composer/composer/autoload_static.php +++ b/lib/composer/composer/autoload_static.php @@ -1828,6 +1828,7 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2 'OC\\DB\\ConnectionFactory' => __DIR__ . '/../../..' . '/lib/private/DB/ConnectionFactory.php', 'OC\\DB\\DbDataCollector' => __DIR__ . '/../../..' . '/lib/private/DB/DbDataCollector.php', 'OC\\DB\\Exceptions\\DbalException' => __DIR__ . '/../../..' . '/lib/private/DB/Exceptions/DbalException.php', + 'OC\\DB\\FilesSearchSnapshot' => __DIR__ . '/../../..' . '/lib/private/DB/FilesSearchSnapshot.php', 'OC\\DB\\Middleware\\ConnectionActivityConnection' => __DIR__ . '/../../..' . '/lib/private/DB/Middleware/ConnectionActivityConnection.php', 'OC\\DB\\Middleware\\ConnectionActivityDriver' => __DIR__ . '/../../..' . '/lib/private/DB/Middleware/ConnectionActivityDriver.php', 'OC\\DB\\Middleware\\ConnectionActivityMiddleware' => __DIR__ . '/../../..' . '/lib/private/DB/Middleware/ConnectionActivityMiddleware.php', @@ -2016,6 +2017,7 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2 'OC\\Files\\ObjectStore\\Swift' => __DIR__ . '/../../..' . '/lib/private/Files/ObjectStore/Swift.php', 'OC\\Files\\ObjectStore\\SwiftFactory' => __DIR__ . '/../../..' . '/lib/private/Files/ObjectStore/SwiftFactory.php', 'OC\\Files\\ObjectStore\\SwiftV2CachingAuthService' => __DIR__ . '/../../..' . '/lib/private/Files/ObjectStore/SwiftV2CachingAuthService.php', + 'OC\\Files\\Search\\FilesSearchQuery' => __DIR__ . '/../../..' . '/lib/private/Files/Search/FilesSearchQuery.php', 'OC\\Files\\Search\\QueryOptimizer\\FlattenNestedBool' => __DIR__ . '/../../..' . '/lib/private/Files/Search/QueryOptimizer/FlattenNestedBool.php', 'OC\\Files\\Search\\QueryOptimizer\\FlattenSingleArgumentBinaryOperation' => __DIR__ . '/../../..' . '/lib/private/Files/Search/QueryOptimizer/FlattenSingleArgumentBinaryOperation.php', 'OC\\Files\\Search\\QueryOptimizer\\MergeDistributiveOperations' => __DIR__ . '/../../..' . '/lib/private/Files/Search/QueryOptimizer/MergeDistributiveOperations.php', diff --git a/lib/private/DB/FilesSearchSnapshot.php b/lib/private/DB/FilesSearchSnapshot.php new file mode 100644 index 0000000000000..5918a46a75d0f --- /dev/null +++ b/lib/private/DB/FilesSearchSnapshot.php @@ -0,0 +1,830 @@ + + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +namespace OC\DB; + +use Doctrine\DBAL\TransactionIsolationLevel; +use OC\DB\Middleware\ConnectionActivityNotifier; +use OC\SystemConfig; +use OCP\DB\QueryBuilder\IQueryBuilder; +use OCP\IDBConnection; + +/** + * Opt-in capability for a bounded Files name-search prepass. + * + * This deliberately lives in the internal DB layer. Files callers receive an + * IDBConnection only and must not inspect a ConnectionAdapter or connection + * parameters. Until the capability checks below succeed, callers use their + * existing query path. + * + * @internal + */ +final class FilesSearchSnapshot { + private const LIMIT_KEYS = ['statement_seconds', 'lock_seconds', 'total_seconds']; + private const MIN_STATEMENT_SECONDS = 1; + private const MAX_STATEMENT_SECONDS = 10; + private const MIN_LOCK_SECONDS = 1; + private const MAX_LOCK_SECONDS = 10; + private const MIN_TOTAL_SECONDS = 2; + private const MAX_TOTAL_SECONDS = 30; + + private bool $busy = false; + private ?int $activeDeadline = null; + private ?int $activeStatementSeconds = null; + + public function __construct( + private SystemConfig $config, + private ConnectionFactory $connectionFactory, + ) { + } + + /** + * Run a bounded prepass on an owned, read-only snapshot. + * + * A null return means that the capability is unavailable before the + * optimized body was admitted. An empty array is a successful body result. + * Once admitted, errors intentionally propagate: retrying or falling back + * after an optimized body begins could conceal an uncertain effect. + * + * @param callable(IDBConnection,callable():void,callable():bool,callable(int,string,int,bool):?array):array $body + * @return array|null + */ + public function run(IDBConnection $shared, callable $body): ?array { + if ($this->config->getValue('files_search_bounded_name', false) !== true + || PHP_INT_SIZE !== 8 + || $this->busy + || $shared->inTransaction() + || !($shared instanceof ConnectionAdapter) + || $this->config->getValue('dbtype', 'sqlite') !== 'mysql' + || $this->config->getValue('dbpersistent', false) !== false + || $this->config->getValue('dbreplica', []) !== [] + || $this->config->getValue('dbsharding', []) !== []) { + return null; + } + $limits = $this->readBoundedNameLimits(); + if ($limits === null) { + // There is deliberately no numeric production default. The opt-in + // route is unavailable until an administrator supplies this exact, + // typed server-side admission record. + return null; + } + $deadline = hrtime(true) + ($limits['total_seconds'] * 1_000_000_000); + + $this->busy = true; + $owned = null; + $originalIsolation = null; + $originalReadOnly = null; + $originalLimits = null; + $result = null; + $primaryFailure = null; + $unavailable = false; + + try { + $actual = $shared->getInner(); + $expectedParams = $this->connectionFactory->createConnectionParams('', [], 'mysql'); + if ($actual->getShardDefinition('filecache') !== null + || !$this->hasFactoryOwnedActivityNotifier($actual->getParams()) + || !$this->connectionParamsMatch($actual->getParams(), $expectedParams) + || !$this->isSingleNonPersistentTopology($expectedParams)) { + $unavailable = true; + } else { + $owned = $this->connectionFactory->getConnection('mysql', []); + // Complete the configured primary connection/listener setup before + // changing the transaction characteristics for this owned session. + // A session-level timeout cannot bound this first connection + // negotiation. The monotonic budget is checked as soon as it returns. + $owned->ensureConnectedToPrimary(); + if (!$this->hasRemainingBudget($deadline)) { + $unavailable = true; + } else { + try { + // Capturing the prior per-session values is required for exact + // restoration. Like the initial connection negotiation, that first + // local session read cannot itself be capped by a value not yet set. + $originalLimits = $this->readSessionLimits($owned); + if ($originalLimits === null || !$this->applySessionLimits($owned, $limits)) { + $unavailable = true; + } else { + $this->activeDeadline = $deadline; + $this->activeStatementSeconds = $limits['statement_seconds']; + } + } catch (\Throwable) { + // This is still before the optimized body. A server which cannot + // prove and install the requested session limits declines the route. + $unavailable = true; + } + if ($unavailable || !$this->hasRemainingBudget($deadline)) { + $unavailable = true; + } else { + // Version and catalog checks are now covered by max_statement_time. + if (!$this->hasRemainingBudget($deadline) || !$this->isMariaDb1011($owned) || !$this->hasRemainingBudget($deadline) || $owned->inTransaction()) { + $unavailable = true; + } else { + $table = $expectedParams['tablePrefix'] . 'filecache'; + if (!$this->hasRemainingBudget($deadline) || !$this->hasStorageNameMtimeFileidIndex($owned, $table) || !$this->hasRemainingBudget($deadline)) { + $unavailable = true; + } else { + $this->assertWithinBudget($deadline); + $originalIsolation = $this->readSessionIsolation($owned); + $originalReadOnly = $this->readSessionReadOnly($owned); + $this->assertWithinBudget($deadline); + $owned->setTransactionIsolation(TransactionIsolationLevel::REPEATABLE_READ); + $owned->executeStatement('SET SESSION TRANSACTION READ ONLY'); + $owned->beginTransaction(); + + // A closed DBAL connection can reconnect lazily. Never allow + // the Files continuation to silently run in a new snapshot. + $identity = $this->readSnapshotIdentity($owned); + $assertActive = function () use ($owned, $identity, $deadline): void { + $this->assertWithinBudget($deadline); + if (!$owned->isConnected() || !$owned->inTransaction()) { + throw new \RuntimeException('Files search snapshot is no longer active'); + } + $this->armRemainingStatementTimeout($owned); + if ($this->readSnapshotIdentity($owned) !== $identity) { + throw new \RuntimeException('Files search snapshot session changed'); + } + $this->assertWithinBudget($deadline); + }; + $metadataTable = $expectedParams['tablePrefix'] . 'files_metadata'; + $verifyOneToOne = function () use ($owned, $table, $metadataTable, $assertActive): bool { + // This query runs only after the caller's coarse read has + // acquired its snapshot. It is DB-only: Files cannot choose + // table identifiers; the rank caller declines on false. + $assertActive(); + $valid = $this->hasOneToOneFileMetadataRelation($owned, $table, $metadataTable); + $assertActive(); + return $valid; + }; + $assertActive(); + $recentNameIds = function (int $storage, string $pattern, int $limit, bool $widen) use ($owned, $table, $metadataTable, $expectedParams, $assertActive): ?array { + $assertActive(); + if ($storage < 1 || $limit < 1 || $limit > 1024) { + return null; + } + $collation = $this->filesNameCollation($expectedParams); + if ($collation === null + || !$this->acquireRecentMetadataRead($owned, $table, $metadataTable, $assertActive) + || !$this->hasNonNullBigIntMtime($owned, $table, $assertActive) + || !$this->hasOneToOneFileMetadataRelation($owned, $table, $metadataTable)) { + return null; + } + // The tiny joined PK read above has acquired the filecache/metadata + // metadata locks in this owned RR transaction. Certify catalog facts + // only after that acquisition, before any storage-range scan. + $assertActive(); + if (!$this->hasStorageMtimeFileidNameIndex($owned, $table)) { + return null; + } + $assertActive(); + if (!$this->hasSafeStorageMtimeRange($owned, $table, $metadataTable, $storage, $assertActive)) { + return null; + } + + $filecache = $owned->quoteIdentifier($table); + $metadata = $owned->quoteIdentifier($metadataTable); + $outerLimit = $widen ? 2048 : $limit; + $statement = $this->executeBoundedQuery($owned, + 'SELECT `recent`.`fileid` FROM (' + . 'SELECT `file`.`fileid`, `file`.`mtime`, `file`.`name`, `meta`.`file_id` AS `metadata_file_id` FROM ' . $filecache . ' `file` ' + . 'LEFT JOIN ' . $metadata . ' `meta` ON `file`.`fileid` = `meta`.`file_id` ' + . 'WHERE `file`.`storage` = ? ORDER BY `file`.`mtime` DESC, `file`.`fileid` DESC LIMIT 2048' + . ') `recent` WHERE `recent`.`name` COLLATE ' . $collation . ' LIKE ? ' + . 'ORDER BY `recent`.`mtime` DESC, `recent`.`fileid` DESC LIMIT ' . $outerLimit, + [$storage, $pattern], + [IQueryBuilder::PARAM_INT, IQueryBuilder::PARAM_STR], + ); + try { + $rows = $statement->fetchAll(); + } finally { + $statement->free(); + } + $assertActive(); + if (count($rows) > $outerLimit) { + throw new \UnexpectedValueException('Files search recent candidate bound violated'); + } + $ids = []; + foreach ($rows as $row) { + if (!is_array($row)) { + throw new \UnexpectedValueException('Files search recent candidate row is malformed'); + } + $value = $row['fileid'] ?? null; + if ((!is_int($value) && !is_string($value)) || !preg_match('/^[1-9][0-9]*$/D', (string)$value)) { + throw new \UnexpectedValueException('Files search recent candidate id is malformed'); + } + $id = filter_var($value, FILTER_VALIDATE_INT); + if ($id === false || isset($ids[$id])) { + throw new \UnexpectedValueException('Files search recent candidate ids are malformed'); + } + $ids[$id] = true; + } + return $rows; + }; + $value = $body(new ConnectionAdapter($owned), $assertActive, $verifyOneToOne, $recentNameIds); + $assertActive(); + if (!is_array($value)) { + throw new \UnexpectedValueException('Files search snapshot body must return an array'); + } + $result = $value; + } + } + } + } + } + } catch (\Throwable $e) { + $primaryFailure = $e; + } + + $cleanupFailure = $this->cleanup( + $owned, + $originalIsolation, + $originalReadOnly, + $originalLimits, + ); + $this->busy = false; + $this->activeDeadline = null; + $this->activeStatementSeconds = null; + + if ($cleanupFailure !== null) { + throw new \RuntimeException( + 'Unable to release the Files search snapshot', + 0, + $primaryFailure ?? $cleanupFailure, + ); + } + if ($primaryFailure !== null) { + throw $primaryFailure; + } + + return $unavailable ? null : $result; + } + + /** + * @param array $params + */ + private function isSingleNonPersistentTopology(array $params): bool { + return empty($params['persistent']) + && empty($params['sharding']) + && isset($params['primary'], $params['replica']) + && is_array($params['replica']) + && count($params['replica']) === 1 + && $params['primary'] === $params['replica'][0]; + } + + /** + * ConnectionFactory appends this process-local notifier after creating the + * public connection parameter record. It observes activity only; it is not + * part of routing, credentials, or the database topology. + * + * @param array $params + */ + private function hasFactoryOwnedActivityNotifier(array $params): bool { + return isset($params['activity_notifier']) + && $params['activity_notifier'] instanceof ConnectionActivityNotifier; + } + + /** + * @param array $actual + * @param array $expected + */ + private function connectionParamsMatch(array $actual, array $expected): bool { + unset($actual['activity_notifier']); + return $actual === $expected; + } + + /** @return array{statement_seconds:int,lock_seconds:int,total_seconds:int}|null */ + private function readBoundedNameLimits(): ?array { + $raw = $this->config->getValue('files_search_bounded_name_limits', null); + if (!is_array($raw) + || count($raw) !== count(self::LIMIT_KEYS) + || array_diff(array_keys($raw), self::LIMIT_KEYS) !== [] + || array_diff(self::LIMIT_KEYS, array_keys($raw)) !== []) { + return null; + } + foreach (self::LIMIT_KEYS as $key) { + if (!is_int($raw[$key])) { + return null; + } + } + $statement = $raw['statement_seconds']; + $lock = $raw['lock_seconds']; + $total = $raw['total_seconds']; + if ($statement < self::MIN_STATEMENT_SECONDS || $statement > self::MAX_STATEMENT_SECONDS + || $lock < self::MIN_LOCK_SECONDS || $lock > self::MAX_LOCK_SECONDS + || $total < self::MIN_TOTAL_SECONDS || $total > self::MAX_TOTAL_SECONDS + || $lock > $statement || $statement >= $total) { + return null; + } + return [ + 'statement_seconds' => $statement, + 'lock_seconds' => $lock, + 'total_seconds' => $total, + ]; + } + + private function hasRemainingBudget(int $deadline): bool { + return hrtime(true) < $deadline; + } + + private function assertWithinBudget(int $deadline): void { + if (!$this->hasRemainingBudget($deadline)) { + throw new \RuntimeException('Files search snapshot budget exhausted'); + } + } + + private function currentStatementLimit(): string { + if ($this->activeDeadline === null || $this->activeStatementSeconds === null) { + throw new \LogicException('Files search statement budget is not active'); + } + $remainingNanoseconds = $this->activeDeadline - hrtime(true); + // A SET STATEMENT decimal below one microsecond is neither useful nor a + // portable MariaDB contract. Decline by throwing before dispatch instead. + if ($remainingNanoseconds < 1_000) { + throw new \RuntimeException('Files search snapshot budget exhausted'); + } + $microseconds = min( + intdiv($remainingNanoseconds, 1_000), + $this->activeStatementSeconds * 1_000_000, + ); + if ($microseconds < 1) { + throw new \RuntimeException('Files search snapshot budget exhausted'); + } + return sprintf('%d.%06d', intdiv($microseconds, 1_000_000), $microseconds % 1_000_000); + } + + /** + * Dispatch one internal SELECT with a cap no larger than its current + * remaining monotonic budget. DBAL prepared-parameter support for MariaDB's + * SET STATEMENT ... FOR form remains a native-test gate. + * + * @param list $params + * @param list $types + */ + private function executeBoundedQuery(Connection $connection, string $sql, array $params = [], array $types = []): mixed { + if (strncasecmp(ltrim($sql), 'SELECT', 6) !== 0) { + throw new \LogicException('Files search bounded query must be a SELECT'); + } + return $connection->executeQuery( + 'SET STATEMENT max_statement_time = ' . $this->currentStatementLimit() . ' FOR ' . $sql, + $params, + $types, + ); + } + + private function armRemainingStatementTimeout(Connection $connection): void { + // This is intentionally done immediately before handing the owned + // connection to the external helper/native continuation. It limits each + // later statement to the then-remaining window, but cannot turn opaque + // multi-statement continuation work into a hard HTTP/request cancellation. + $connection->executeStatement('SET SESSION max_statement_time = ' . $this->currentStatementLimit()); + } + + /** @return array{max_statement_time:string,lock_wait_timeout:int,innodb_lock_wait_timeout:int}|null */ + private function readSessionLimits(Connection $connection): ?array { + $statement = $connection->executeQuery( + 'SELECT CAST(@@SESSION.max_statement_time AS CHAR) AS max_statement_time, ' + . '@@SESSION.lock_wait_timeout AS lock_wait_timeout, ' + . '@@SESSION.innodb_lock_wait_timeout AS innodb_lock_wait_timeout', + ); + try { + $row = $statement->fetchAssociative(); + } finally { + $statement->free(); + } + $row = is_array($row) ? $row : []; + $max = $row['max_statement_time'] ?? null; + $lock = $row['lock_wait_timeout'] ?? null; + $innodbLock = $row['innodb_lock_wait_timeout'] ?? null; + if (($max = $this->canonicalMaxStatementTime($max)) === null + || ($lock = $this->sessionNonNegativeInteger($lock)) === null + || ($innodbLock = $this->sessionNonNegativeInteger($innodbLock)) === null) { + return null; + } + return [ + 'max_statement_time' => $max, + 'lock_wait_timeout' => $lock, + 'innodb_lock_wait_timeout' => $innodbLock, + ]; + } + + private function canonicalMaxStatementTime(mixed $value): ?string { + if (is_int($value)) { + $value = (string)$value; + } elseif (is_float($value)) { + if (!is_finite($value) || $value < 0) { + return null; + } + $value = sprintf('%.6F', $value); + } + if (!is_string($value) || preg_match('/^(?:0|[1-9][0-9]*)(?:\.[0-9]+)?$/D', $value) !== 1) { + return null; + } + if (str_contains($value, '.')) { + $value = rtrim(rtrim($value, '0'), '.'); + } + return $value === '' ? '0' : $value; + } + + private function sessionNonNegativeInteger(mixed $value): ?int { + if ((!is_int($value) && !is_string($value)) || !preg_match('/^(?:0|[1-9][0-9]*)$/D', (string)$value)) { + return null; + } + $integer = filter_var($value, FILTER_VALIDATE_INT); + return $integer === false || $integer < 0 ? null : $integer; + } + + /** @param array{statement_seconds:int,lock_seconds:int,total_seconds:int} $limits */ + private function applySessionLimits(Connection $connection, array $limits): bool { + // The values are validated native integers before this method builds SQL. + $connection->executeStatement('SET SESSION max_statement_time = ' . $limits['statement_seconds']); + $connection->executeStatement('SET SESSION lock_wait_timeout = ' . $limits['lock_seconds']); + $connection->executeStatement('SET SESSION innodb_lock_wait_timeout = ' . $limits['lock_seconds']); + $actual = $this->readSessionLimits($connection); + return $actual !== null + && $actual['max_statement_time'] === (string)$limits['statement_seconds'] + && $actual['lock_wait_timeout'] === $limits['lock_seconds'] + && $actual['innodb_lock_wait_timeout'] === $limits['lock_seconds']; + } + + /** @param array{max_statement_time:string,lock_wait_timeout:int,innodb_lock_wait_timeout:int} $limits */ + private function restoreSessionLimits(Connection $connection, array $limits): void { + // These values came from the same live session and were syntax-checked by + // readSessionLimits(). Cleanup never opens a replacement connection. + $connection->executeStatement('SET SESSION max_statement_time = ' . $limits['max_statement_time']); + $connection->executeStatement('SET SESSION lock_wait_timeout = ' . $limits['lock_wait_timeout']); + $connection->executeStatement('SET SESSION innodb_lock_wait_timeout = ' . $limits['innodb_lock_wait_timeout']); + } + + private function isMariaDb1011(Connection $connection): bool { + if ($connection->getDatabaseProvider(true) !== IDBConnection::PLATFORM_MARIADB) { + return false; + } + $statement = $this->executeBoundedQuery($connection, 'SELECT VERSION()'); + try { + $version = (string)$statement->fetchOne(); + } finally { + $statement->free(); + } + return preg_match('/^10\\.11\\.\\d+(?:-|$)/', $version) === 1; + } + + private function hasStorageNameMtimeFileidIndex(Connection $connection, string $table): bool { + $statement = $this->executeBoundedQuery($connection, + 'SELECT INDEX_NAME, NON_UNIQUE, INDEX_TYPE, SEQ_IN_INDEX, COLUMN_NAME, SUB_PART, IGNORED ' + . 'FROM INFORMATION_SCHEMA.STATISTICS ' + . 'WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ? ' + . 'ORDER BY INDEX_NAME, SEQ_IN_INDEX', + [$table], + ); + try { + $indexes = []; + foreach ($statement->fetchAll() as $row) { + $indexes[(string)$row['INDEX_NAME']][] = $row; + } + } finally { + $statement->free(); + } + + return $this->hasExactIndex($indexes, ['storage', 'name', 'mtime', 'fileid'], true); + } + + private function hasStorageMtimeFileidNameIndex(Connection $connection, string $table): bool { + $statement = $this->executeBoundedQuery($connection, + 'SELECT INDEX_NAME, NON_UNIQUE, INDEX_TYPE, SEQ_IN_INDEX, COLUMN_NAME, SUB_PART, IGNORED, COLLATION ' + . 'FROM INFORMATION_SCHEMA.STATISTICS ' + . 'WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ? ' + . 'ORDER BY INDEX_NAME, SEQ_IN_INDEX', + [$table], + ); + try { + $indexes = []; + foreach ($statement->fetchAll() as $row) { + $indexes[(string)$row['INDEX_NAME']][] = $row; + } + } finally { + $statement->free(); + } + return $this->hasExactIndex( + $indexes, + ['storage', 'mtime', 'fileid', 'name'], + true, + null, + ['A', 'A', 'A', 'A'], + ); + } + + /** + * Return the exact MySQL ILIKE collation used by the configured connection, + * or decline the recent route when that equivalence is not certain. + * + * @param array $params + */ + private function filesNameCollation(array $params): ?string { + $collation = $params['collation'] ?? (($params['charset'] ?? 'utf8') . '_general_ci'); + // MySqlExpressionBuilder and AdapterMySQL use the same configured + // collation-or-charset-plus-_general_ci formula for ILIKE. + // This small route is established only for utf8mb4 general_ci. + return $collation === 'utf8mb4_general_ci' ? $collation : null; + } + + /** Acquire only the metadata locks needed for later catalog certification. */ + private function acquireRecentMetadataRead(Connection $connection, string $filecacheTable, string $metadataTable, callable $assertActive): bool { + $filecache = $connection->quoteIdentifier($filecacheTable); + $metadata = $connection->quoteIdentifier($metadataTable); + $assertActive(); + $statement = $this->executeBoundedQuery($connection, + 'SELECT `file`.`fileid`, `meta`.`file_id` AS `metadata_file_id` FROM ' . $filecache . ' `file` ' + . 'LEFT JOIN ' . $metadata . ' `meta` ON `file`.`fileid` = `meta`.`file_id` ' + . 'ORDER BY `file`.`fileid` ASC LIMIT 1', + ); + try { + $statement->fetchAssociative(); + } finally { + $statement->free(); + } + $assertActive(); + return true; + } + + private function hasNonNullBigIntMtime(Connection $connection, string $filecacheTable, callable $assertActive): bool { + $assertActive(); + $column = $this->executeBoundedQuery($connection, + 'SELECT DATA_TYPE, IS_NULLABLE, COLUMN_TYPE FROM INFORMATION_SCHEMA.COLUMNS ' + . 'WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ? AND COLUMN_NAME = \'mtime\'', + [$filecacheTable], + ); + try { + $mtime = $column->fetchAssociative(); + } finally { + $column->free(); + } + $assertActive(); + return is_array($mtime) + && strtolower((string)($mtime['DATA_TYPE'] ?? '')) === 'bigint' + && (string)($mtime['IS_NULLABLE'] ?? '') === 'NO' + && preg_match('/^bigint(?:\([0-9]+\))?(?: unsigned)?$/', strtolower(trim((string)($mtime['COLUMN_TYPE'] ?? '')))) === 1; + } + + private function hasSafeStorageMtimeRange(Connection $connection, string $filecacheTable, string $metadataTable, int $storage, callable $assertActive): bool { + $filecache = $connection->quoteIdentifier($filecacheTable); + $metadata = $connection->quoteIdentifier($metadataTable); + $extrema = []; + foreach (['ASC', 'DESC'] as $direction) { + $assertActive(); + $statement = $this->executeBoundedQuery($connection, + 'SELECT `file`.`mtime`, `meta`.`file_id` AS `metadata_file_id` FROM ' . $filecache . ' `file` ' + . 'LEFT JOIN ' . $metadata . ' `meta` ON `file`.`fileid` = `meta`.`file_id` ' + . 'WHERE `file`.`storage` = ? ORDER BY `file`.`mtime` ' . $direction . ' LIMIT 1', + [$storage], + [IQueryBuilder::PARAM_INT], + ); + try { + $value = $statement->fetchOne(); + } finally { + $statement->free(); + } + $assertActive(); + if ($value === false || (!is_int($value) && !is_string($value)) || !preg_match('/^-?[0-9]+$/D', (string)$value)) { + return false; + } + $integer = filter_var($value, FILTER_VALIDATE_INT); + if ($integer === false || $integer < -9007199254740991 || $integer > 9007199254740991) { + return false; + } + $extrema[] = $integer; + } + if (count($extrema) !== 2) { + return false; + } + return count($extrema) === 2; + } + + private function hasOneToOneFileMetadataRelation(Connection $connection, string $filecacheTable, string $metadataTable): bool { + $tables = $this->executeBoundedQuery($connection, + 'SELECT TABLE_NAME, ENGINE FROM INFORMATION_SCHEMA.TABLES ' + . 'WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME IN (?, ?)', + [$filecacheTable, $metadataTable], + ); + try { + $engines = []; + foreach ($tables->fetchAll() as $row) { + $engines[(string)$row['TABLE_NAME']] = strtoupper((string)$row['ENGINE']); + } + } finally { + $tables->free(); + } + if (($engines[$filecacheTable] ?? null) !== 'INNODB' || ($engines[$metadataTable] ?? null) !== 'INNODB') { + return false; + } + // The unique metadata key only establishes a one-to-one SQL join when + // both join columns have the same non-coercing integral representation. + // For example, a unique VARCHAR metadata key could contain both "1" and + // "01", which compare equal to a BIGINT fileid in MariaDB. Do this + // physical-schema check inside the owned snapshot, after the coarse read. + if (!$this->hasCompatibleFileIdColumns($connection, $filecacheTable, $metadataTable)) { + return false; + } + + $statement = $this->executeBoundedQuery($connection, + 'SELECT TABLE_NAME, INDEX_NAME, NON_UNIQUE, INDEX_TYPE, SEQ_IN_INDEX, COLUMN_NAME, SUB_PART, IGNORED ' + . 'FROM INFORMATION_SCHEMA.STATISTICS ' + . 'WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME IN (?, ?) ' + . 'ORDER BY TABLE_NAME, INDEX_NAME, SEQ_IN_INDEX', + [$filecacheTable, $metadataTable], + ); + try { + $indexes = []; + foreach ($statement->fetchAll() as $row) { + $indexes[(string)$row['TABLE_NAME']][(string)$row['INDEX_NAME']][] = $row; + } + } finally { + $statement->free(); + } + return $this->hasExactIndex($indexes[$filecacheTable] ?? [], ['fileid'], false, 'PRIMARY') + && $this->hasExactIndex($indexes[$metadataTable] ?? [], ['file_id'], false); + } + + private function hasCompatibleFileIdColumns(Connection $connection, string $filecacheTable, string $metadataTable): bool { + $statement = $this->executeBoundedQuery($connection, + 'SELECT TABLE_NAME, COLUMN_NAME, DATA_TYPE, IS_NULLABLE, COLUMN_TYPE ' + . 'FROM INFORMATION_SCHEMA.COLUMNS ' + . 'WHERE TABLE_SCHEMA = DATABASE() ' + . 'AND ((TABLE_NAME = ? AND COLUMN_NAME = \'fileid\') ' + . 'OR (TABLE_NAME = ? AND COLUMN_NAME = \'file_id\'))', + [$filecacheTable, $metadataTable], + ); + try { + $columns = []; + foreach ($statement->fetchAll() as $row) { + $columns[(string)$row['TABLE_NAME']] = $row; + } + } finally { + $statement->free(); + } + + $filecache = $columns[$filecacheTable] ?? null; + $metadata = $columns[$metadataTable] ?? null; + if (!is_array($filecache) || !is_array($metadata) + || (string)($filecache['DATA_TYPE'] ?? '') !== 'bigint' + || (string)($metadata['DATA_TYPE'] ?? '') !== 'bigint' + || (string)($filecache['IS_NULLABLE'] ?? '') !== 'NO' + || (string)($metadata['IS_NULLABLE'] ?? '') !== 'NO') { + return false; + } + + $filecacheType = strtolower(trim((string)($filecache['COLUMN_TYPE'] ?? ''))); + $metadataType = strtolower(trim((string)($metadata['COLUMN_TYPE'] ?? ''))); + return preg_match('/^bigint(?:\\([0-9]+\\))?(?: unsigned)?$/', $filecacheType) === 1 + && $filecacheType === $metadataType; + } + + /** + * @param array>> $indexes + * @param list $columns + */ + private function hasExactIndex(array $indexes, array $columns, bool $nonUnique, ?string $requiredName = null, ?array $directions = null): bool { + foreach ($indexes as $name => $indexColumns) { + if (($requiredName !== null && $name !== $requiredName) || count($indexColumns) !== count($columns)) { + continue; + } + $matches = true; + foreach ($indexColumns as $position => $column) { + if ((int)$column['NON_UNIQUE'] !== ($nonUnique ? 1 : 0) + || strtoupper((string)$column['INDEX_TYPE']) !== 'BTREE' + || (int)$column['SEQ_IN_INDEX'] !== $position + 1 + || (string)$column['COLUMN_NAME'] !== $columns[$position] + || ($directions !== null && strtoupper((string)($column['COLLATION'] ?? '')) !== $directions[$position]) + || $column['SUB_PART'] !== null + || strtoupper((string)$column['IGNORED']) === 'YES') { + $matches = false; + break; + } + } + if ($matches) { + return true; + } + } + return false; + } + + private function readSessionIsolation(Connection $connection): string { + // MariaDB 10.11 exposes the session value as tx_isolation. The + // transaction_isolation spelling is only its newer MariaDB 11.1 name. + $statement = $this->executeBoundedQuery($connection, 'SELECT @@SESSION.tx_isolation'); + try { + $isolation = strtoupper(str_replace(' ', '-', (string)$statement->fetchOne())); + } finally { + $statement->free(); + } + if (!in_array($isolation, ['READ-UNCOMMITTED', 'READ-COMMITTED', 'REPEATABLE-READ', 'SERIALIZABLE'], true)) { + throw new \RuntimeException('Unsupported Files search snapshot isolation'); + } + return $isolation; + } + + private function readSessionReadOnly(Connection $connection): bool { + $statement = $this->executeBoundedQuery($connection, 'SELECT @@SESSION.tx_read_only'); + try { + $value = $statement->fetchOne(); + } finally { + $statement->free(); + } + if ($value !== 0 && $value !== 1 && $value !== '0' && $value !== '1') { + throw new \RuntimeException('Unsupported Files search snapshot read-only state'); + } + return (bool)$value; + } + + /** @return array{connection: string, isolation: string, readOnly: string, active: string} */ + private function readSnapshotIdentity(Connection $connection): array { + $statement = $this->executeBoundedQuery($connection, + 'SELECT CONNECTION_ID() AS snapshot_id, @@SESSION.tx_isolation AS snapshot_isolation, ' + . '@@SESSION.tx_read_only AS snapshot_read_only, @@SESSION.in_transaction AS snapshot_active', + ); + try { + $row = $statement->fetchAssociative(); + } finally { + $statement->free(); + } + if (!is_array($row) + || !preg_match('/^[1-9][0-9]*$/D', (string)($row['snapshot_id'] ?? '')) + || (string)($row['snapshot_isolation'] ?? '') !== 'REPEATABLE-READ' + || (string)($row['snapshot_read_only'] ?? '') !== '1' + || (string)($row['snapshot_active'] ?? '') !== '1') { + throw new \RuntimeException('Files search snapshot session is not repeatable read-only'); + } + return [ + 'connection' => (string)$row['snapshot_id'], + 'isolation' => (string)$row['snapshot_isolation'], + 'readOnly' => (string)$row['snapshot_read_only'], + 'active' => (string)$row['snapshot_active'], + ]; + } + + private function cleanup( + ?Connection $owned, + ?string $originalIsolation, + ?bool $originalReadOnly, + ?array $originalLimits, + ): ?\Throwable { + if ($owned === null || !$owned->isConnected()) { + // Session settings no longer exist once this nonpersistent owned + // connection is closed. Restoring them must not open a new session. + return null; + } + + $failure = null; + try { + // beginTransaction() can throw after it has incremented the DBAL nesting + // state, so this cannot rely on a post-call bookkeeping flag. + if ($owned->inTransaction()) { + $owned->rollBack(); + } + } catch (\Throwable $e) { + $failure = $e; + } + try { + if ($owned->isConnected() && $originalIsolation !== null) { + $owned->setTransactionIsolation($this->transactionIsolationLevel($originalIsolation)); + } + } catch (\Throwable $e) { + $failure ??= $e; + } + try { + if ($owned->isConnected() && $originalReadOnly !== null) { + $owned->executeStatement('SET SESSION TRANSACTION ' . ($originalReadOnly ? 'READ ONLY' : 'READ WRITE')); + } + } catch (\Throwable $e) { + $failure ??= $e; + } + try { + if ($owned->isConnected() && $originalLimits !== null) { + $this->restoreSessionLimits($owned, $originalLimits); + } + } catch (\Throwable $e) { + $failure ??= $e; + } + try { + $owned->close(); + } catch (\Throwable $e) { + $failure ??= $e; + } + return $failure; + } + + private function transactionIsolationLevel(string $isolation): int { + return match ($isolation) { + 'READ-UNCOMMITTED' => TransactionIsolationLevel::READ_UNCOMMITTED, + 'READ-COMMITTED' => TransactionIsolationLevel::READ_COMMITTED, + 'REPEATABLE-READ' => TransactionIsolationLevel::REPEATABLE_READ, + 'SERIALIZABLE' => TransactionIsolationLevel::SERIALIZABLE, + default => throw new \LogicException('Unsupported Files search snapshot isolation'), + }; + } +} diff --git a/lib/private/Files/Cache/QuerySearchHelper.php b/lib/private/Files/Cache/QuerySearchHelper.php index 97af2b9fd8a34..b08e4cd9219f7 100644 --- a/lib/private/Files/Cache/QuerySearchHelper.php +++ b/lib/private/Files/Cache/QuerySearchHelper.php @@ -7,7 +7,9 @@ namespace OC\Files\Cache; +use OC\DB\FilesSearchSnapshot; use OC\Files\Cache\Wrapper\CacheJail; +use OC\Files\Search\FilesSearchQuery; use OC\Files\Search\QueryOptimizer\QueryOptimizer; use OC\Files\Search\SearchBinaryOperator; use OC\Files\Search\SearchComparison; @@ -21,6 +23,7 @@ use OCP\Files\Search\ISearchBinaryOperator; use OCP\Files\Search\ISearchComparison; use OCP\Files\Search\ISearchOperator; +use OCP\Files\Search\ISearchOrder; use OCP\Files\Search\ISearchQuery; use OCP\FilesMetadata\IFilesMetadataManager; use OCP\FilesMetadata\IMetadataQuery; @@ -36,9 +39,22 @@ public function __construct( private readonly QueryOptimizer $queryOptimizer, private readonly IGroupManager $groupManager, private readonly IFilesMetadataManager $filesMetadataManager, + private readonly ?FilesSearchSnapshot $filesSearchSnapshot = null, ) { } + private function withConnection(IDBConnection $connection): self { + return new self( + $this->mimetypeLoader, + $connection, + $this->searchBuilder, + $this->queryOptimizer, + $this->groupManager, + $this->filesMetadataManager, + null, + ); + } + protected function getQueryBuilder(): CacheQueryBuilder { return new CacheQueryBuilder( $this->connection->getQueryBuilder(), @@ -142,6 +158,120 @@ protected function equipQueryForShares(CacheQueryBuilder $query): void { * @return array */ public function searchInCaches(ISearchQuery $searchQuery, array $caches): array { + $context = $this->filesSearchContext($searchQuery, $caches); + $prefix = $context === null ? null : $this->rankPrefix($searchQuery); + if ($context !== null && $prefix !== null && $this->filesSearchSnapshot !== null) { + $results = $this->filesSearchSnapshot->run( + $this->connection, + function (IDBConnection $connection, callable $assertActive, callable $verifyOneToOne, callable $recentNameIds) use ($searchQuery, $caches, $context, $prefix): array { + $helper = $this->withConnection($connection); + $recentStorage = count($context[1]) === 1 ? $context[1][0] : null; + $namePattern = $context[0]->getValue(); + if (is_int($recentStorage) && $recentStorage > 0 && is_string($namePattern)) { + $recentRows = $recentNameIds($recentStorage, $namePattern, $prefix, false); + $assertActive(); + if (is_array($recentRows) && count($recentRows) === $prefix && $verifyOneToOne()) { + $recentIds = $helper->candidateIds($recentRows, $prefix); + $restrictedQuery = $helper->queryWithIds($searchQuery, $recentIds); + $probe = $helper->idProbeBuilder($restrictedQuery, $caches); + $assertActive(); + $probeResult = $probe->executeQuery(); + try { + $probeRows = $probeResult->fetchAllAssociative(); + } finally { + $probeResult->closeCursor(); + } + $assertActive(); + if (count($probeRows) > $searchQuery->getLimit()) { + throw new \UnexpectedValueException('Files search recent probe bound violated'); + } + if (count($probeRows) === $searchQuery->getLimit()) { + return $helper->searchInCaches($restrictedQuery, $caches); + } + $widenedRows = $recentNameIds($recentStorage, $namePattern, $prefix, true); + $assertActive(); + if (is_array($widenedRows)) { + $widenedIds = $helper->candidateIds($widenedRows, 2048); + if ($widenedIds !== []) { + $widenedQuery = $helper->queryWithIds($searchQuery, $widenedIds); + $widenedProbe = $helper->idProbeBuilder($widenedQuery, $caches); + $assertActive(); + $widenedResult = $widenedProbe->executeQuery(); + try { + $widenedProbeRows = $widenedResult->fetchAllAssociative(); + } finally { + $widenedResult->closeCursor(); + } + $assertActive(); + if (count($widenedProbeRows) > $searchQuery->getLimit()) { + throw new \UnexpectedValueException('Files search widened probe bound violated'); + } + if (count($widenedProbeRows) === $searchQuery->getLimit()) { + return $helper->searchInCaches($widenedQuery, $caches); + } + } + } + } + } + $builder = $helper->getQueryBuilder(); + $builder->selectFileCache('file', false); + $metadataQuery = $builder->selectMetadata(); + $coarseOperation = new SearchBinaryOperator(ISearchBinaryOperator::OPERATOR_AND, [ + $context[0], + new SearchComparison(ISearchComparison::COMPARE_IN, 'storage', $context[1]), + ]); + $searchExpr = $helper->searchBuilder->searchOperatorToDBExpr($builder, $coarseOperation, $metadataQuery); + if ($searchExpr) { + $builder->andWhere($searchExpr); + } + $helper->searchBuilder->addSearchOrdersToQuery($builder, $searchQuery->getOrder(), $metadataQuery); + $builder->setMaxResults($prefix); + $builder->select('file.fileid'); + $assertActive(); + $result = $builder->executeQuery(); + try { + $rows = $result->fetchAllAssociative(); + } finally { + $result->closeCursor(); + } + $assertActive(); + if (count($rows) > $prefix) { + throw new \UnexpectedValueException('Files search candidate bound violated'); + } + if (!$verifyOneToOne()) { + return $helper->searchInCaches($searchQuery, $caches); + } + $ids = $helper->candidateIds($rows, $prefix); + if ($ids === []) { + return array_fill_keys(array_keys($caches), []); + } + $restrictedQuery = $helper->queryWithIds($searchQuery, $ids); + if (count($ids) === $prefix) { + $probe = $helper->idProbeBuilder($restrictedQuery, $caches); + $assertActive(); + $probeResult = $probe->executeQuery(); + try { + $probeRows = $probeResult->fetchAllAssociative(); + } finally { + $probeResult->closeCursor(); + } + $assertActive(); + if (count($probeRows) > $searchQuery->getLimit()) { + throw new \UnexpectedValueException('Files search probe bound violated'); + } + if (count($probeRows) !== $searchQuery->getLimit()) { + return $helper->searchInCaches($searchQuery, $caches); + } + } + $assertActive(); + return $helper->searchInCaches($restrictedQuery, $caches); + }, + ); + if ($results !== null) { + return $results; + } + } + // search in multiple caches at once by creating one query in the following format // SELECT ... FROM oc_filecache WHERE // @@ -218,6 +348,133 @@ public function searchInCaches(ISearchQuery $searchQuery, array $caches): array return $results; } + /** @return array{0: SearchComparison, 1: list}|null */ + private function filesSearchContext(ISearchQuery $query, array $caches): ?array { + if ($this->connection->getShardDefinition('filecache') !== null + || !$query instanceof FilesSearchQuery || count($caches) !== 1 || $query->getSelectFields() !== [] + || $query->getLimit() < 1 || $query->getLimit() > 100 || $query->getOffset() < 0) { + return null; + } + $order = $query->getOrder(); + if (count($order) !== 2 || $order[0]->getField() !== 'mtime' || $order[0]->getDirection() !== ISearchOrder::DIRECTION_DESCENDING + || $order[1]->getField() !== 'fileid' || $order[1]->getDirection() !== ISearchOrder::DIRECTION_DESCENDING + || $order[0]->getExtra() !== '' || $order[1]->getExtra() !== '') { + return null; + } + $operation = $query->getSearchOperation(); + if ($operation instanceof SearchBinaryOperator && $operation->getType() === ISearchBinaryOperator::OPERATOR_AND + && count($operation->getArguments()) === 1) { + $operation = $operation->getArguments()[0]; + } + if (!$operation instanceof SearchComparison || $operation->getField() !== 'name' + || $operation->getType() !== ISearchComparison::COMPARE_LIKE || $operation->getExtra() !== '' + || !is_string($operation->getValue())) { + return null; + } + $cache = reset($caches); + if (!$cache instanceof CacheJail || $cache->getGetUnjailedRoot() !== 'files') { + return null; + } + $storages = $this->storageSuperset($cache->getQueryFilterForStorage()); + return $storages === null || $storages === [] ? null : [$operation, $storages]; + } + + private function rankPrefix(ISearchQuery $query): ?int { + $limit = $query->getLimit(); + $offset = $query->getOffset(); + if ($limit < 1 || $offset < 0 || $limit > 1024 || $offset > 1024 - $limit) { + return null; + } + return $offset + $limit; + } + + /** @param list> $rows + * @return list */ + private function candidateIds(array $rows, int $bound): array { + if (count($rows) > $bound) { + throw new \UnexpectedValueException('Files search candidate bound violated'); + } + $ids = []; + foreach ($rows as $row) { + $value = $row['fileid'] ?? null; + if ((!is_int($value) && !is_string($value)) || !preg_match('/^[1-9][0-9]*$/D', (string)$value)) { + throw new \UnexpectedValueException('Files search candidate fileid is invalid'); + } + $id = filter_var($value, FILTER_VALIDATE_INT); + if ($id === false || isset($ids[$id])) { + throw new \UnexpectedValueException('Files search candidate fileid is invalid'); + } + $ids[$id] = $id; + } + return array_values($ids); + } + + /** @param list $ids */ + private function queryWithIds(ISearchQuery $query, array $ids): SearchQuery { + return new SearchQuery( + new SearchBinaryOperator(ISearchBinaryOperator::OPERATOR_AND, [ + $query->getSearchOperation(), + new SearchComparison(ISearchComparison::COMPARE_IN, 'fileid', $ids), + ]), + $query->getLimit(), + $query->getOffset(), + $query->getOrder(), + $query->getUser(), + $query->limitToHome(), + $query->getSelectFields(), + ); + } + + private function idProbeBuilder(ISearchQuery $query, array $caches): CacheQueryBuilder { + $builder = $this->getQueryBuilder(); + $builder->selectFileCache('file', false); + $metadataQuery = $builder->selectMetadata(); + $this->applySearchConstraints($builder, $query, $caches, $metadataQuery); + $builder->select('file.fileid'); + return $builder; + } + + /** @return list|null */ + private function storageSuperset(ISearchOperator $operator): ?array { + if ($operator instanceof SearchComparison && $operator->getField() === 'storage' && $operator->getExtra() === '') { + $values = match ($operator->getType()) { + ISearchComparison::COMPARE_EQUAL => [$operator->getValue()], + ISearchComparison::COMPARE_IN => $operator->getValue(), + default => null, + }; + if (!is_array($values) || $values === [] || count($values) > 16) { + return null; + } + foreach ($values as $id) { + if (!is_int($id) || $id < 1) { + return null; + } + } + return array_values(array_unique($values)); + } + if (!$operator instanceof SearchBinaryOperator || $operator->getArguments() === []) { + return null; + } + $type = $operator->getType(); + if ($type !== ISearchBinaryOperator::OPERATOR_AND && $type !== ISearchBinaryOperator::OPERATOR_OR) { + return null; + } + $sets = array_map($this->storageSuperset(...), $operator->getArguments()); + if ($type === ISearchBinaryOperator::OPERATOR_OR && in_array(null, $sets, true)) { + return null; + } + $sets = array_values(array_filter($sets, static fn ($set) => $set !== null)); + if ($sets === []) { + return null; + } + $result = array_shift($sets); + foreach ($sets as $set) { + $result = $type === ISearchBinaryOperator::OPERATOR_AND ? array_intersect($result, $set) : array_merge($result, $set); + } + $result = array_values(array_unique($result)); + return count($result) > 16 ? null : $result; + } + protected function requireUser(ISearchQuery $searchQuery): IUser { $user = $searchQuery->getUser(); if ($user === null) { diff --git a/lib/private/Files/Search/FilesSearchQuery.php b/lib/private/Files/Search/FilesSearchQuery.php new file mode 100644 index 0000000000000..3cbe36e5b32d1 --- /dev/null +++ b/lib/private/Files/Search/FilesSearchQuery.php @@ -0,0 +1,14 @@ + + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +namespace OC\Files\Search; + +/** Internal Files-provider opt-in. No mutable hints or request-controlled flags. */ +final class FilesSearchQuery extends SearchQuery { +} diff --git a/tests/Core/Listener/AddMissingIndicesListenerTest.php b/tests/Core/Listener/AddMissingIndicesListenerTest.php new file mode 100644 index 0000000000000..83abd9278b08a --- /dev/null +++ b/tests/Core/Listener/AddMissingIndicesListenerTest.php @@ -0,0 +1,85 @@ + + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +namespace Tests\Core\Listener; + +use OC\Core\Listener\AddMissingIndicesListener; +use OC\SystemConfig; +use OCP\DB\Events\AddMissingIndicesEvent; +use OCP\DB\IResult; +use OCP\IDBConnection; +use Test\TestCase; + +class AddMissingIndicesListenerTest extends TestCase { + public function testRegistersBoundedFilesSearchIndicesForEnabledMariaDb1011(): void { + $connection = $this->createMock(IDBConnection::class); + $connection->method('getDatabaseProvider')->willReturn(IDBConnection::PLATFORM_MARIADB); + $result = $this->createMock(IResult::class); + $result->method('fetchOne')->willReturn('10.11.14-MariaDB'); + $result->expects($this->once())->method('closeCursor')->willReturn(true); + $connection->expects($this->once())->method('executeQuery')->with('SELECT VERSION()')->willReturn($result); + + $config = $this->createMock(SystemConfig::class); + $config->expects($this->once())->method('getValue')->with('files_search_bounded_name', false)->willReturn(true); + + $event = new AddMissingIndicesEvent(); + (new AddMissingIndicesListener($connection, $config))->handle($event); + + $indices = array_column($event->getMissingIndices(), null, 'indexName'); + $this->assertSame([ + 'tableName' => 'filecache', + 'indexName' => 'fs_storage_name_mtime_fileid', + 'columns' => ['storage', 'name', 'mtime', 'fileid'], + 'options' => [], + 'dropUnnamedIndex' => false, + 'uniqueIndex' => false, + ], $indices['fs_storage_name_mtime_fileid']); + $this->assertSame([ + 'tableName' => 'filecache', + 'indexName' => 'fs_storage_mtime_fileid_name', + 'columns' => ['storage', 'mtime', 'fileid', 'name'], + 'options' => [], + 'dropUnnamedIndex' => false, + 'uniqueIndex' => false, + ], $indices['fs_storage_mtime_fileid_name']); + } + + public function testDoesNotRegisterBoundedFilesSearchIndicesWhenFeatureIsDisabled(): void { + $connection = $this->createMock(IDBConnection::class); + $connection->method('getDatabaseProvider')->willReturn(IDBConnection::PLATFORM_MARIADB); + $connection->expects($this->never())->method('executeQuery'); + $config = $this->createMock(SystemConfig::class); + $config->expects($this->once())->method('getValue')->with('files_search_bounded_name', false)->willReturn(false); + + $event = new AddMissingIndicesEvent(); + (new AddMissingIndicesListener($connection, $config))->handle($event); + + $names = array_column($event->getMissingIndices(), 'indexName'); + $this->assertNotContains('fs_storage_name_mtime_fileid', $names); + $this->assertNotContains('fs_storage_mtime_fileid_name', $names); + } + + public function testDoesNotRegisterBoundedFilesSearchIndicesForOtherMariaDbVersions(): void { + $connection = $this->createMock(IDBConnection::class); + $connection->method('getDatabaseProvider')->willReturn(IDBConnection::PLATFORM_MARIADB); + $result = $this->createMock(IResult::class); + $result->method('fetchOne')->willReturn('10.6.23-MariaDB'); + $result->expects($this->once())->method('closeCursor')->willReturn(true); + $connection->expects($this->once())->method('executeQuery')->with('SELECT VERSION()')->willReturn($result); + $config = $this->createMock(SystemConfig::class); + $config->method('getValue')->with('files_search_bounded_name', false)->willReturn(true); + + $event = new AddMissingIndicesEvent(); + (new AddMissingIndicesListener($connection, $config))->handle($event); + + $names = array_column($event->getMissingIndices(), 'indexName'); + $this->assertNotContains('fs_storage_name_mtime_fileid', $names); + $this->assertNotContains('fs_storage_mtime_fileid_name', $names); + } +} diff --git a/tests/lib/DB/FilesSearchSnapshotIntegrationTest.php b/tests/lib/DB/FilesSearchSnapshotIntegrationTest.php new file mode 100644 index 0000000000000..ddc7f7c764ef7 --- /dev/null +++ b/tests/lib/DB/FilesSearchSnapshotIntegrationTest.php @@ -0,0 +1,180 @@ + + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +namespace Test\DB; + +use OC\DB\Connection; +use OC\DB\ConnectionAdapter; +use OC\DB\ConnectionFactory; +use OC\DB\FilesSearchSnapshot; +use OC\DB\SchemaWrapper; +use OC\Files\Storage\Temporary; +use OC\SystemConfig; +use OCP\IDBConnection; +use OCP\Server; +use PHPUnit\Framework\Attributes\Group; +use Test\TestCase; + +#[Group('DB')] +class FilesSearchSnapshotIntegrationTest extends TestCase { + private IDBConnection $shared; + private FilesSearchSnapshot $snapshot; + private array $createdIndexes = []; + + protected function setUp(): void { + parent::setUp(); + $this->shared = Server::get(IDBConnection::class); + if ($this->shared->getDatabaseProvider(true) !== IDBConnection::PLATFORM_MARIADB) { + $this->markTestSkipped('The bounded search capability targets MariaDB 10.11.'); + } + $result = $this->shared->executeQuery('SELECT VERSION()'); + try { + $version = (string)$result->fetchOne(); + } finally { + $result->closeCursor(); + } + if (preg_match('/^10\.11\.\d+(?:-|$)/', $version) !== 1) { + $this->markTestSkipped('The bounded search capability targets MariaDB 10.11.'); + } + $this->overwriteSystemConfig('files_search_bounded_name', true); + $this->overwriteSystemConfig('files_search_bounded_name_limits', [ + 'statement_seconds' => 2, + 'lock_seconds' => 1, + 'total_seconds' => 5, + ]); + $schema = new SchemaWrapper(Server::get(Connection::class)); + $table = $schema->getTable('filecache'); + foreach ([ + 'fs_storage_name_mtime_fileid' => ['storage', 'name', 'mtime', 'fileid'], + 'fs_storage_mtime_fileid_name' => ['storage', 'mtime', 'fileid', 'name'], + ] as $name => $columns) { + if (!$table->hasIndex($name)) { + $table->addIndex($columns, $name); + $this->createdIndexes[] = $name; + } + } + if ($this->createdIndexes !== []) { + $this->shared->migrateToSchema($schema->getWrappedSchema()); + } + $this->snapshot = new FilesSearchSnapshot(Server::get(SystemConfig::class), Server::get(ConnectionFactory::class)); + } + + protected function tearDown(): void { + try { + if ($this->createdIndexes !== []) { + $schema = new SchemaWrapper(Server::get(Connection::class)); + $table = $schema->getTable('filecache'); + foreach ($this->createdIndexes as $name) { + $table->dropIndex($name); + } + $this->shared->migrateToSchema($schema->getWrappedSchema()); + } + } finally { + parent::tearDown(); + } + } + + public function testOwnedReadOnlySnapshotDoesNotChangeSharedSession(): void { + $before = $this->sessionState($this->shared); + $called = false; + $actual = $this->snapshot->run($this->shared, function (IDBConnection $owned, callable $assertActive, callable $verifyOneToOne) use ($before, &$called): array { + $called = true; + self::assertInstanceOf(ConnectionAdapter::class, $owned); + $state = $this->sessionState($owned); + self::assertNotSame($before['session_id'], $state['session_id']); + self::assertSame('REPEATABLE-READ', $state['isolation_level']); + self::assertSame('1', (string)$state['read_only']); + self::assertSame('1', (string)$state['active']); + $assertActive(); + self::assertTrue($verifyOneToOne()); + return ['native-snapshot']; + }); + self::assertTrue($called, 'The capability must be admitted, not silently fall back.'); + self::assertSame(['native-snapshot'], $actual); + self::assertSame($before, $this->sessionState($this->shared)); + self::assertFalse($this->shared->inTransaction()); + } + + public function testAdmittedExceptionKeepsIdentityAndCapabilityCanBeReused(): void { + $failure = new \DomainException('snapshot fixture failure'); + try { + $this->snapshot->run($this->shared, static function () use ($failure): array { + throw $failure; + }); + self::fail('An admitted error must not become native fallback or an empty result.'); + } catch (\DomainException $actual) { + self::assertSame($failure, $actual); + } + self::assertFalse($this->shared->inTransaction()); + self::assertSame([], $this->snapshot->run($this->shared, static fn (): array => [])); + } + + public function testClosedOwnedConnectionCannotSilentlyReconnect(): void { + $before = $this->sessionState($this->shared); + try { + $this->snapshot->run($this->shared, static function (IDBConnection $owned, callable $assertActive): array { + self::assertInstanceOf(ConnectionAdapter::class, $owned); + $owned->getInner()->close(); + $assertActive(); + return []; + }); + self::fail('A lost owned snapshot must remain an error.'); + } catch (\RuntimeException $actual) { + self::assertSame('Files search snapshot is no longer active', $actual->getMessage()); + } + self::assertSame($before, $this->sessionState($this->shared)); + } + + public function testReentrantCallDeclinesWithoutInvokingItsBody(): void { + $actual = $this->snapshot->run($this->shared, function (): array { + self::assertNull($this->snapshot->run($this->shared, static function (): array { + self::fail('A busy capability must decline before dispatching a second body.'); + })); + return ['outer']; + }); + self::assertSame(['outer'], $actual); + } + + public function testRecentCandidatesUseDeterministicDescendingTies(): void { + $storage = new Temporary([]); + $cache = $storage->getCache(); + $storage->getScanner()->scan(''); + $ids = []; + try { + for ($i = 0; $i < 3; $i++) { + $ids[] = $cache->put('snapshot-fixture-' . $i, [ + 'size' => 1, + 'mtime' => 100, + 'mimetype' => 'text/plain', + ]); + } + $expected = array_reverse($ids); + $actual = $this->snapshot->run($this->shared, static function (IDBConnection $owned, callable $assertActive, callable $verifyOneToOne, callable $recentNameIds) use ($cache): array { + self::assertTrue($verifyOneToOne()); + $recent = $recentNameIds($cache->getNumericStorageId(), '%snapshot-fixture%', 2, false); + self::assertIsArray($recent, 'The native recent route must be admitted.'); + $assertActive(); + return array_map(static fn (array $row): int => (int)$row['fileid'], $recent); + }); + self::assertSame(array_slice($expected, 0, 2), $actual); + } finally { + $cache->clear(); + $storage->cleanUp(); + } + } + + private function sessionState(IDBConnection $connection): array { + $result = $connection->executeQuery('SELECT CONNECTION_ID() AS session_id, @@SESSION.tx_isolation AS isolation_level, @@SESSION.tx_read_only AS read_only, @@SESSION.in_transaction AS active, @@SESSION.max_statement_time AS statement_limit, @@SESSION.lock_wait_timeout AS lock_limit, @@SESSION.innodb_lock_wait_timeout AS innodb_limit'); + try { + return $result->fetchAssociative(); + } finally { + $result->closeCursor(); + } + } +} diff --git a/tests/lib/DB/FilesSearchSnapshotTest.php b/tests/lib/DB/FilesSearchSnapshotTest.php new file mode 100644 index 0000000000000..09c0995d39da3 --- /dev/null +++ b/tests/lib/DB/FilesSearchSnapshotTest.php @@ -0,0 +1,101 @@ + + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +namespace Test\DB; + +use OC\DB\ConnectionFactory; +use OC\DB\FilesSearchSnapshot; +use OC\DB\Middleware\ConnectionActivityNotifier; +use OC\SystemConfig; +use Test\TestCase; + +class FilesSearchSnapshotTest extends TestCase { + public function testAcceptsTheAscendingIndexUsedForReverseRecentScan(): void { + $snapshot = new FilesSearchSnapshot( + $this->createMock(SystemConfig::class), + $this->createMock(ConnectionFactory::class), + ); + $index = [ + 'fs_storage_mtime_fileid_name' => [ + $this->indexColumn(1, 'storage', 'A'), + $this->indexColumn(2, 'mtime', 'A'), + $this->indexColumn(3, 'fileid', 'A'), + $this->indexColumn(4, 'name', 'A'), + ], + ]; + + $this->assertTrue(self::invokePrivate( + $snapshot, + 'hasExactIndex', + [$index, ['storage', 'mtime', 'fileid', 'name'], true, null, ['A', 'A', 'A', 'A']], + )); + } + + public function testRejectsMixedDirectionIndexForReverseRecentScan(): void { + $snapshot = new FilesSearchSnapshot( + $this->createMock(SystemConfig::class), + $this->createMock(ConnectionFactory::class), + ); + $index = [ + 'fs_storage_mtime_fileid_name' => [ + $this->indexColumn(1, 'storage', 'A'), + $this->indexColumn(2, 'mtime', 'D'), + $this->indexColumn(3, 'fileid', 'A'), + $this->indexColumn(4, 'name', 'A'), + ], + ]; + + $this->assertFalse(self::invokePrivate( + $snapshot, + 'hasExactIndex', + [$index, ['storage', 'mtime', 'fileid', 'name'], true, null, ['A', 'A', 'A', 'A']], + )); + } + + public function testAcceptsOnlyTheFactoryOwnedActivityNotifierDifference(): void { + $snapshot = $this->newSnapshot(); + $expected = ['host' => 'database', 'dbname' => 'nextcloud']; + $actual = $expected; + $actual['activity_notifier'] = new ConnectionActivityNotifier(); + + self::assertTrue(self::invokePrivate($snapshot, 'hasFactoryOwnedActivityNotifier', [$actual])); + self::assertTrue(self::invokePrivate($snapshot, 'connectionParamsMatch', [$actual, $expected])); + } + + public function testRejectsUnknownConnectionParameterDifference(): void { + $snapshot = $this->newSnapshot(); + $expected = ['host' => 'database', 'dbname' => 'nextcloud']; + $actual = $expected; + $actual['activity_notifier'] = new ConnectionActivityNotifier(); + $actual['unknown_routing_parameter'] = 'different'; + + self::assertTrue(self::invokePrivate($snapshot, 'hasFactoryOwnedActivityNotifier', [$actual])); + self::assertFalse(self::invokePrivate($snapshot, 'connectionParamsMatch', [$actual, $expected])); + } + + private function newSnapshot(): FilesSearchSnapshot { + return new FilesSearchSnapshot( + $this->createMock(SystemConfig::class), + $this->createMock(ConnectionFactory::class), + ); + } + + /** @return array */ + private function indexColumn(int $position, string $name, string $direction): array { + return [ + 'NON_UNIQUE' => 1, + 'INDEX_TYPE' => 'BTREE', + 'SEQ_IN_INDEX' => $position, + 'COLUMN_NAME' => $name, + 'SUB_PART' => null, + 'IGNORED' => 'NO', + 'COLLATION' => $direction, + ]; + } +} diff --git a/tests/lib/Files/Cache/QuerySearchHelperTest.php b/tests/lib/Files/Cache/QuerySearchHelperTest.php new file mode 100644 index 0000000000000..3d8077c0b1a85 --- /dev/null +++ b/tests/lib/Files/Cache/QuerySearchHelperTest.php @@ -0,0 +1,302 @@ + + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +namespace Test\Files\Cache; + +use OC\DB\Connection; +use OC\DB\ConnectionAdapter; +use OC\DB\ConnectionFactory; +use OC\DB\FilesSearchSnapshot; +use OC\DB\QueryBuilder\Sharded\ShardDefinition; +use OC\Files\Cache\Cache; +use OC\Files\Cache\CacheDependencies; +use OC\Files\Cache\QuerySearchHelper; +use OC\Files\Cache\SearchBuilder; +use OC\Files\Cache\Wrapper\CacheJail; +use OC\Files\Search\FilesSearchQuery; +use OC\Files\Search\QueryOptimizer\QueryOptimizer; +use OC\Files\Search\SearchComparison; +use OC\Files\Search\SearchOrder; +use OC\Files\Search\SearchQuery; +use OC\Files\Storage\Temporary; +use OC\SystemConfig; +use OC\User\DisplayNameCache; +use OCP\DB\QueryBuilder\Sharded\IShardMapper; +use OCP\EventDispatcher\IEventDispatcher; +use OCP\Files\Cache\ICacheEntry; +use OCP\Files\IMimeTypeLoader; +use OCP\Files\Search\ISearchComparison; +use OCP\Files\Search\ISearchOrder; +use OCP\FilesMetadata\IFilesMetadataManager; +use OCP\IDBConnection; +use OCP\IGroupManager; +use OCP\Server; +use PHPUnit\Framework\Attributes\Group; +use Psr\Log\LoggerInterface; +use Test\TestCase; + +#[Group('DB')] +class QuerySearchHelperTest extends TestCase { + private SystemConfig $baseConfig; + + #[\Override] + protected function setUp(): void { + parent::setUp(); + $this->baseConfig = Server::get(SystemConfig::class); + } + + /** @param list|null $readKeys */ + private function helper(bool $enabled = true, ?array &$readKeys = null, ?IDBConnection $connection = null): QuerySearchHelper { + $config = $this->createMock(SystemConfig::class); + $config->method('getValue') + ->willReturnCallback(function (string $key, mixed $default = '') use ($enabled, &$readKeys): mixed { + if ($readKeys !== null) { + $readKeys[] = $key; + } + return match ($key) { + 'files_search_bounded_name' => $enabled, + 'files_search_bounded_name_limits' => [ + 'statement_seconds' => 2, + 'lock_seconds' => 1, + 'total_seconds' => 3, + ], + default => $this->baseConfig->getValue($key, $default), + }; + }); + + return new QuerySearchHelper( + Server::get(IMimeTypeLoader::class), + $connection ?? Server::get(IDBConnection::class), + Server::get(SearchBuilder::class), + Server::get(QueryOptimizer::class), + Server::get(IGroupManager::class), + Server::get(IFilesMetadataManager::class), + new FilesSearchSnapshot($config, Server::get(ConnectionFactory::class)), + ); + } + + private function cache(QuerySearchHelper $helper): Cache { + $storage = new Temporary([]); + return new Cache($storage, new CacheDependencies( + Server::get(IMimeTypeLoader::class), + Server::get(IDBConnection::class), + Server::get(IEventDispatcher::class), + $helper, + $this->baseConfig, + Server::get(LoggerInterface::class), + Server::get(IFilesMetadataManager::class), + Server::get(DisplayNameCache::class), + )); + } + + private function boundedQuery(int $limit = 2, int $offset = 0): FilesSearchQuery { + return new FilesSearchQuery( + new SearchComparison(ISearchComparison::COMPARE_LIKE, 'name', '%needle%'), + $limit, + $offset, + [ + new SearchOrder(ISearchOrder::DIRECTION_DESCENDING, 'mtime'), + new SearchOrder(ISearchOrder::DIRECTION_DESCENDING, 'fileid'), + ], + ); + } + + private function nativeQuery(int $limit = 2, int $offset = 0): SearchQuery { + return new SearchQuery( + new SearchComparison(ISearchComparison::COMPARE_LIKE, 'name', '%needle%'), + $limit, + $offset, + [ + new SearchOrder(ISearchOrder::DIRECTION_DESCENDING, 'mtime'), + new SearchOrder(ISearchOrder::DIRECTION_DESCENDING, 'fileid'), + ], + ); + } + + /** @param list $entries + * @return list + */ + private function ids(array $entries): array { + return array_map(static fn (ICacheEntry $entry): int => $entry->getId(), $entries); + } + + /** @param list $entries + * @return list> + */ + private function payloads(array $entries): array { + return array_map(static function (ICacheEntry $entry): array { + self::assertTrue(method_exists($entry, 'getData')); + /** @var array $data */ + $data = $entry->getData(); + return $data; + }, $entries); + } + + public function testBoundedRootJailRetainsNativeResultOrderAndPaging(): void { + $readKeys = []; + $helper = $this->helper(true, $readKeys); + $cache = $this->cache($helper); + $cache->put('', ['size' => 0, 'mtime' => 200, 'mimetype' => ICacheEntry::DIRECTORY_MIMETYPE]); + $cache->put('files', ['size' => 0, 'mtime' => 200, 'mimetype' => ICacheEntry::DIRECTORY_MIMETYPE]); + $ids = [ + $cache->put('files/needle-a.txt', ['size' => 1, 'mtime' => 100, 'mimetype' => 'text/plain']), + $cache->put('files/needle-b.txt', ['size' => 1, 'mtime' => 100, 'mimetype' => 'text/plain']), + $cache->put('files/needle-c.txt', ['size' => 1, 'mtime' => 100, 'mimetype' => 'text/plain']), + ]; + $jail = new CacheJail($cache, 'files'); + + $bounded = $helper->searchInCaches($this->boundedQuery(), ['files' => $jail]); + $native = $helper->searchInCaches($this->nativeQuery(), ['files' => $jail]); + rsort($ids, SORT_NUMERIC); + + self::assertSame(array_slice($ids, 0, 2), $this->ids($bounded['files'])); + self::assertSame($this->ids($native['files']), $this->ids($bounded['files'])); + self::assertSame($this->payloads($native['files']), $this->payloads($bounded['files'])); + // These values are read only by FilesSearchSnapshot::run(). Their presence + // proves the public helper admitted the bounded snapshot preflight instead + // of merely comparing two native fallback calls. + self::assertContains('files_search_bounded_name_limits', $readKeys); + self::assertContains('dbtype', $readKeys); + + $boundedPage = $helper->searchInCaches($this->boundedQuery(2, 1), ['files' => $jail]); + $nativePage = $helper->searchInCaches($this->nativeQuery(2, 1), ['files' => $jail]); + self::assertSame(array_slice($ids, 1, 2), $this->ids($boundedPage['files'])); + self::assertSame($this->ids($nativePage['files']), $this->ids($boundedPage['files'])); + self::assertSame($this->payloads($nativePage['files']), $this->payloads($boundedPage['files'])); + } + + public function testWidenedCandidatesPreserveRootJailPayloads(): void { + $readKeys = []; + $helper = $this->helper(true, $readKeys); + $cache = $this->cache($helper); + $cache->put('', ['size' => 0, 'mtime' => 400, 'mimetype' => ICacheEntry::DIRECTORY_MIMETYPE]); + $cache->put('files', ['size' => 0, 'mtime' => 400, 'mimetype' => ICacheEntry::DIRECTORY_MIMETYPE]); + $cache->put('outside-1-needle.txt', ['size' => 1, 'mtime' => 300, 'mimetype' => 'text/plain']); + $cache->put('outside-2-needle.txt', ['size' => 1, 'mtime' => 200, 'mimetype' => 'text/plain']); + $inside = $cache->put('files/older-needle.txt', ['size' => 7, 'mtime' => 100, 'mimetype' => 'text/plain']); + $jail = new CacheJail($cache, 'files'); + + $bounded = $helper->searchInCaches($this->boundedQuery(1), ['files' => $jail]); + $native = $helper->searchInCaches($this->nativeQuery(1), ['files' => $jail]); + + self::assertSame([$inside], $this->ids($bounded['files'])); + self::assertSame($this->payloads($native['files']), $this->payloads($bounded['files'])); + self::assertContains('files_search_bounded_name_limits', $readKeys); + } + + public function testNoMatchRetainsNativeEmptyResult(): void { + $helper = $this->helper(); + $cache = $this->cache($helper); + $cache->put('', ['size' => 0, 'mtime' => 200, 'mimetype' => ICacheEntry::DIRECTORY_MIMETYPE]); + $cache->put('files', ['size' => 0, 'mtime' => 200, 'mimetype' => ICacheEntry::DIRECTORY_MIMETYPE]); + $cache->put('files/other.txt', ['size' => 1, 'mtime' => 100, 'mimetype' => 'text/plain']); + $jail = new CacheJail($cache, 'files'); + + $bounded = $helper->searchInCaches($this->boundedQuery(), ['files' => $jail]); + $native = $helper->searchInCaches($this->nativeQuery(), ['files' => $jail]); + + self::assertSame([], $bounded['files']); + self::assertSame($this->payloads($native['files']), $this->payloads($bounded['files'])); + } + + public function testNestedJailKeepsNativeScopeAndFallsBackToNativeSemantics(): void { + $helper = $this->helper(); + $cache = $this->cache($helper); + $cache->put('', ['size' => 0, 'mtime' => 200, 'mimetype' => ICacheEntry::DIRECTORY_MIMETYPE]); + $cache->put('files', ['size' => 0, 'mtime' => 200, 'mimetype' => ICacheEntry::DIRECTORY_MIMETYPE]); + $cache->put('files/private', ['size' => 0, 'mtime' => 200, 'mimetype' => ICacheEntry::DIRECTORY_MIMETYPE]); + $inside = $cache->put('files/private/needle-inside.txt', ['size' => 1, 'mtime' => 100, 'mimetype' => 'text/plain']); + $cache->put('files/needle-outside.txt', ['size' => 1, 'mtime' => 100, 'mimetype' => 'text/plain']); + $jail = new CacheJail($cache, 'files/private'); + + $bounded = $helper->searchInCaches($this->boundedQuery(), ['private' => $jail]); + $native = $helper->searchInCaches($this->nativeQuery(), ['private' => $jail]); + + self::assertSame([$inside], $this->ids($bounded['private'])); + self::assertSame($this->ids($native['private']), $this->ids($bounded['private'])); + self::assertSame($this->payloads($native['private']), $this->payloads($bounded['private'])); + } + + public function testSelectedFieldsSkipSnapshotAndKeepNativePayload(): void { + $readKeys = []; + $helper = $this->helper(true, $readKeys); + $cache = $this->cache($helper); + $cache->put('', ['size' => 0, 'mtime' => 200, 'mimetype' => ICacheEntry::DIRECTORY_MIMETYPE]); + $cache->put('files', ['size' => 0, 'mtime' => 200, 'mimetype' => ICacheEntry::DIRECTORY_MIMETYPE]); + $id = $cache->put('files/needle.txt', ['size' => 1, 'mtime' => 100, 'mimetype' => 'text/plain']); + $jail = new CacheJail($cache, 'files'); + $bounded = new FilesSearchQuery( + new SearchComparison(ISearchComparison::COMPARE_LIKE, 'name', '%needle%'), + 2, + 0, + [ + new SearchOrder(ISearchOrder::DIRECTION_DESCENDING, 'mtime'), + new SearchOrder(ISearchOrder::DIRECTION_DESCENDING, 'fileid'), + ], + null, + false, + ['metadata_etag'], + ); + $native = new SearchQuery( + $bounded->getSearchOperation(), + $bounded->getLimit(), + $bounded->getOffset(), + $bounded->getOrder(), + null, + false, + ['metadata_etag'], + ); + + $boundedResults = $helper->searchInCaches($bounded, ['files' => $jail]); + $nativeResults = $helper->searchInCaches($native, ['files' => $jail]); + + self::assertSame([$id], $this->ids($boundedResults['files'])); + self::assertSame($this->payloads($nativeResults['files']), $this->payloads($boundedResults['files'])); + self::assertSame([], $readKeys); + } + + public function testDisabledSnapshotUsesNativeSearchSemantics(): void { + $helper = $this->helper(false); + $cache = $this->cache($helper); + $cache->put('', ['size' => 0, 'mtime' => 200, 'mimetype' => ICacheEntry::DIRECTORY_MIMETYPE]); + $cache->put('files', ['size' => 0, 'mtime' => 200, 'mimetype' => ICacheEntry::DIRECTORY_MIMETYPE]); + $id = $cache->put('files/needle.txt', ['size' => 1, 'mtime' => 100, 'mimetype' => 'text/plain']); + $jail = new CacheJail($cache, 'files'); + + $bounded = $helper->searchInCaches($this->boundedQuery(), ['files' => $jail]); + $native = $helper->searchInCaches($this->nativeQuery(), ['files' => $jail]); + + self::assertSame([$id], $this->ids($bounded['files'])); + self::assertSame($this->ids($native['files']), $this->ids($bounded['files'])); + self::assertSame($this->payloads($native['files']), $this->payloads($bounded['files'])); + } + + public function testShardedConnectionSkipsSnapshotBeforeConfigAdmission(): void { + $actual = Server::get(IDBConnection::class); + self::assertInstanceOf(ConnectionAdapter::class, $actual); + $mapper = $this->createMock(IShardMapper::class); + $shard = new ShardDefinition('filecache', 'fileid', [], 'storage', $mapper, [], [], 0, 0); + $inner = $this->createMock(Connection::class); + $inner->method('getShardDefinition')->with('filecache')->willReturn($shard); + $inner->method('getQueryBuilder')->willReturn($actual->getQueryBuilder()); + $connection = new ConnectionAdapter($inner); + $readKeys = []; + $helper = $this->helper(true, $readKeys, $connection); + $cache = $this->cache($helper); + $cache->put('', ['size' => 0, 'mtime' => 200, 'mimetype' => ICacheEntry::DIRECTORY_MIMETYPE]); + $cache->put('files', ['size' => 0, 'mtime' => 200, 'mimetype' => ICacheEntry::DIRECTORY_MIMETYPE]); + $id = $cache->put('files/needle.txt', ['size' => 1, 'mtime' => 100, 'mimetype' => 'text/plain']); + $jail = new CacheJail($cache, 'files'); + + $results = $helper->searchInCaches($this->boundedQuery(), ['files' => $jail]); + + self::assertSame([$id], $this->ids($results['files'])); + self::assertSame([], $readKeys); + } +}