Improve file-cache search performance for large caches

AUTHORS

@@ -230,6 +230,7 @@
  - Jonny007-MKD <1-23-4-5@web.de>
  - Jos Poortvliet <jos@opensuse.org>
  - Jose Quinteiro <github@quinteiro.org>
+ - Josh Duffy <joshduff.jd@gmail.com>
  - Josh Richards <josh.t.richards@gmail.com>
  - Joshua Trees <me@jtrees.io>
  - Juan Pablo Villafáñez <jvillafanez@solidgear.es>

apps/files/lib/Search/FilesSearchProvider.php

@@ -10,12 +10,14 @@
 namespace OCA\Files\Search;
 
 use InvalidArgumentException;
+use OC\Files\Search\FilesSearchQuery;
 use OC\Files\Search\SearchBinaryOperator;
 use OC\Files\Search\SearchComparison;
 use OC\Files\Search\SearchOrder;
 use OC\Files\Search\SearchQuery;
 use OC\Search\Filter\GroupFilter;
 use OC\Search\Filter\UserFilter;
+use OC\SystemConfig;
 use OCP\Files\FileInfo;
 use OCP\Files\IMimeTypeDetector;
 use OCP\Files\IRootFolder;
@@ -42,6 +44,7 @@ public function __construct(
 		private IMimeTypeDetector $mimeTypeDetector,
 		private IRootFolder $rootFolder,
 		private IPreview $previewManager,
+		private ?SystemConfig $systemConfig = null,
 	) {
 	}
 
@@ -161,12 +164,30 @@ private function buildSearchQuery(ISearchQuery $query, IUser $user): SearchQuery
 			};
 		}
 
-		return new SearchQuery(
+		$cursor = (int)$query->getCursor();
+		$soleNameLike = count($comparisons) === 1
+			&& $comparisons[0] instanceof SearchComparison
+			&& $comparisons[0]->getField() === 'name'
+			&& $comparisons[0]->getType() === ISearchComparison::COMPARE_LIKE
+			&& $comparisons[0]->getExtra() === ''
+			&& is_string($comparisons[0]->getValue());
+		$bounded = $this->systemConfig?->getValue('files_search_bounded_name', false) === true
+			&& $query->getSortOrder() === ISearchQuery::SORT_DATE_DESC
+			&& $soleNameLike
+			&& $query->getLimit() >= 1
+			&& $query->getLimit() <= 100
+			&& $cursor >= 0
+			&& $cursor <= 1024 - $query->getLimit();
+		$queryClass = $bounded ? FilesSearchQuery::class : SearchQuery::class;
+		return new $queryClass(
 			new SearchBinaryOperator(SearchBinaryOperator::OPERATOR_AND, $comparisons),
 			$query->getLimit(),
-			(int)$query->getCursor(),
+			$cursor,
 			$query->getSortOrder() === ISearchQuery::SORT_DATE_DESC
-				? [new SearchOrder(ISearchOrder::DIRECTION_DESCENDING, 'mtime')]
+				? array_merge([new SearchOrder(ISearchOrder::DIRECTION_DESCENDING, 'mtime')],
+					// The bounded sole-name opt-in deliberately makes equal-mtime pages
+					// deterministic. All non-admitted requests retain their native order.
+					$bounded ? [new SearchOrder(ISearchOrder::DIRECTION_DESCENDING, 'fileid')] : [])
 				: [],
 			$user
 		);

apps/files/tests/Search/FilesSearchProviderTest.php

@@ -0,0 +1,165 @@
+<?php
+
+declare(strict_types=1);
+
+/**
+ * SPDX-FileCopyrightText: 2026 Josh Duffy <joshduff.jd@gmail.com>
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+namespace OCA\Files\Tests\Search;
+
+use ArrayIterator;
+use OC\Files\Search\FilesSearchQuery;
+use OC\Files\Search\SearchBinaryOperator;
+use OC\Files\Search\SearchComparison;
+use OC\Files\Search\SearchQuery as FilesQuery;
+use OC\Search\Filter\StringFilter;
+use OC\SystemConfig;
+use OCA\Files\Search\FilesSearchProvider;
+use OCP\Files\IMimeTypeDetector;
+use OCP\Files\IRootFolder;
+use OCP\Files\IUserFolder;
+use OCP\Files\Search\ISearchBinaryOperator;
+use OCP\Files\Search\ISearchComparison;
+use OCP\Files\Search\ISearchOrder;
+use OCP\Files\Search\ISearchQuery as IFilesSearchQuery;
+use OCP\IL10N;
+use OCP\IPreview;
+use OCP\IURLGenerator;
+use OCP\IUser;
+use OCP\Search\IFilter;
+use OCP\Search\IFilterCollection;
+use OCP\Search\ISearchQuery;
+use PHPUnit\Framework\Attributes\DataProvider;
+use Test\TestCase;
+
+class FilesSearchProviderTest extends TestCase {
+	/**
+	 * @param array<string, IFilter> $filters
+	 */
+	private function searchAndCaptureQuery(bool $enabled, array $filters, int $sortOrder = ISearchQuery::SORT_DATE_DESC, int $limit = 50, int|string|null $cursor = 0): FilesQuery {
+		$user = $this->createMock(IUser::class);
+		$user->method('getUID')->willReturn('user');
+		$folder = $this->createMock(IUserFolder::class);
+		$captured = null;
+		$folder->expects($this->once())
+			->method('search')
+			->willReturnCallback(function (IFilesSearchQuery $query) use (&$captured): array {
+				$captured = $query;
+				return [];
+			});
+
+		$rootFolder = $this->createMock(IRootFolder::class);
+		$rootFolder->expects($this->once())
+			->method('getUserFolder')
+			->with('user')
+			->willReturn($folder);
+		$l10n = $this->createMock(IL10N::class);
+		$l10n->method('t')->willReturnArgument(0);
+		$config = $this->createMock(SystemConfig::class);
+		$config->method('getValue')
+			->with('files_search_bounded_name', false)
+			->willReturn($enabled);
+
+		$provider = new FilesSearchProvider(
+			$l10n,
+			$this->createMock(IURLGenerator::class),
+			$this->createMock(IMimeTypeDetector::class),
+			$rootFolder,
+			$this->createMock(IPreview::class),
+			$config,
+		);
+		$query = $this->createMock(ISearchQuery::class);
+		$query->method('getFilters')->willReturn($this->filters($filters));
+		$query->method('getSortOrder')->willReturn($sortOrder);
+		$query->method('getLimit')->willReturn($limit);
+		$query->method('getCursor')->willReturn($cursor);
+		$provider->search($user, $query);
+
+		self::assertInstanceOf(FilesQuery::class, $captured);
+		return $captured;
+	}
+
+	/** @param array<string, IFilter> $filters */
+	private function filters(array $filters): IFilterCollection {
+		return new class($filters) implements IFilterCollection {
+			/** @param array<string, IFilter> $filters */
+			public function __construct(
+				private array $filters,
+			) {
+			}
+
+			#[\Override]
+			public function has(string $name): bool {
+				return isset($this->filters[$name]);
+			}
+
+			#[\Override]
+			public function get(string $name): ?IFilter {
+				return $this->filters[$name] ?? null;
+			}
+
+			#[\Override]
+			public function getIterator(): \Traversable {
+				return new ArrayIterator($this->filters);
+			}
+
+			#[\Override]
+			public function count(): int {
+				return count($this->filters);
+			}
+		};
+	}
+
+	public function testBoundedSearchIsOffByDefault(): void {
+		$query = $this->searchAndCaptureQuery(false, ['term' => new StringFilter('invoice')]);
+
+		self::assertNotInstanceOf(FilesSearchQuery::class, $query);
+		self::assertSame(1, count($query->getOrder()));
+		self::assertSame('mtime', $query->getOrder()[0]->getField());
+	}
+
+	public function testEnabledSoleTermUsesBoundedQueryAndStableOrder(): void {
+		$query = $this->searchAndCaptureQuery(true, ['term' => new StringFilter('invoice')], ISearchQuery::SORT_DATE_DESC, 25, 12);
+
+		self::assertInstanceOf(FilesSearchQuery::class, $query);
+		self::assertSame(25, $query->getLimit());
+		self::assertSame(12, $query->getOffset());
+		self::assertCount(2, $query->getOrder());
+		self::assertSame('mtime', $query->getOrder()[0]->getField());
+		self::assertSame(ISearchOrder::DIRECTION_DESCENDING, $query->getOrder()[0]->getDirection());
+		self::assertSame('fileid', $query->getOrder()[1]->getField());
+		self::assertSame(ISearchOrder::DIRECTION_DESCENDING, $query->getOrder()[1]->getDirection());
+		$operation = $query->getSearchOperation();
+		self::assertInstanceOf(SearchBinaryOperator::class, $operation);
+		self::assertSame(ISearchBinaryOperator::OPERATOR_AND, $operation->getType());
+		self::assertCount(1, $operation->getArguments());
+		self::assertInstanceOf(SearchComparison::class, $operation->getArguments()[0]);
+		self::assertSame(ISearchComparison::COMPARE_LIKE, $operation->getArguments()[0]->getType());
+		self::assertSame('name', $operation->getArguments()[0]->getField());
+		self::assertSame('%invoice%', $operation->getArguments()[0]->getValue());
+	}
+
+	/**
+	 * @return array<string, array{array<string, IFilter>, int, int, int|string|null}>
+	 */
+	public static function boundedFallbackProvider(): array {
+		return [
+			'non-date sort' => [['term' => new StringFilter('invoice')], 0, 25, 0],
+			'extra request filter' => [['term' => new StringFilter('invoice'), 'type' => new StringFilter('text')], ISearchQuery::SORT_DATE_DESC, 25, 0],
+			'zero limit' => [['term' => new StringFilter('invoice')], ISearchQuery::SORT_DATE_DESC, 0, 0],
+			'limit over maximum' => [['term' => new StringFilter('invoice')], ISearchQuery::SORT_DATE_DESC, 101, 0],
+			'negative cursor' => [['term' => new StringFilter('invoice')], ISearchQuery::SORT_DATE_DESC, 25, -1],
+			'cursor beyond bounded window' => [['term' => new StringFilter('invoice')], ISearchQuery::SORT_DATE_DESC, 100, 925],
+		];
+	}
+
+	#[DataProvider('boundedFallbackProvider')]
+	public function testEnabledSearchFallsBackWhenRequestIsOutsideBoundedContract(array $filters, int $sortOrder, int $limit, int|string|null $cursor): void {
+		$query = $this->searchAndCaptureQuery(true, $filters, $sortOrder, $limit, $cursor);
+
+		self::assertNotInstanceOf(FilesSearchQuery::class, $query);
+		self::assertCount($sortOrder === ISearchQuery::SORT_DATE_DESC ? 1 : 0, $query->getOrder());
+	}
+}

config/config.sample.php

@@ -3129,6 +3129,31 @@
 	 */
 	'unified_search.enabled' => false,
 
+	/**
+	 * Opt in to bounded, date-sorted Files name searches on MariaDB 10.11.
+	 * Enable this setting, then run ``occ db:add-missing-indices`` to install
+	 * the optional covering indexes. Index creation can take time on a large
+	 * file cache and adds storage and write overhead. Disabling this setting
+	 * does not remove the indexes. Other database versions and unsupported
+	 * query shapes retain native search.
+	 *
+	 * Eligible date-sorted requests use descending file IDs to break equal
+	 * modification-time ties, including when the database capability declines.
+	 * Defaults to ``false``.
+	 */
+	'files_search_bounded_name' => false,
+
+	/**
+	 * Required budgets for bounded Files name searches. Without a valid record,
+	 * the database capability declines. Values must be integers: statement and
+	 * lock seconds in 1..10, total seconds in 2..30, with lock <= statement < total.
+	 * Initial connection negotiation is not bounded by these session settings.
+	 *
+	 * For example: ``['statement_seconds' => 2, 'lock_seconds' => 1, 'total_seconds' => 5]``.
+	 * Defaults to ``null``; no numeric budget is chosen automatically.
+	 */
+	'files_search_bounded_name_limits' => null,
+
 	/**
 	 * Enable features that do not yet comply with accessibility standards.
 	 *

core/Listener/AddMissingIndicesListener.php

@@ -9,6 +9,7 @@
 
 namespace OC\Core\Listener;
 
+use OC\SystemConfig;
 use OCP\DB\Events\AddMissingIndicesEvent;
 use OCP\EventDispatcher\Event;
 use OCP\EventDispatcher\IEventListener;
@@ -20,6 +21,7 @@
 class AddMissingIndicesListener implements IEventListener {
 	public function __construct(
 		private readonly IDBConnection $connection,
+		private readonly SystemConfig $systemConfig,
 	) {
 	}
 
@@ -78,6 +80,18 @@ public function handle(Event $event): void {
 			'fs_name_hash',
 			['name']
 		);
+		if ($this->supportsBoundedFilesNameSearch()) {
+			$event->addMissingIndex(
+				'filecache',
+				'fs_storage_name_mtime_fileid',
+				['storage', 'name', 'mtime', 'fileid'],
+			);
+			$event->addMissingIndex(
+				'filecache',
+				'fs_storage_mtime_fileid_name',
+				['storage', 'mtime', 'fileid', 'name'],
+			);
+		}
 
 		$event->addMissingIndex(
 			'twofactor_providers',
@@ -236,4 +250,24 @@ public function handle(Event $event): void {
 			['scheduled_at']
 		);
 	}
+
+	private function supportsBoundedFilesNameSearch(): bool {
+		if ($this->systemConfig->getValue('files_search_bounded_name', false) !== true
+			|| $this->connection->getDatabaseProvider(true) !== IDBConnection::PLATFORM_MARIADB) {
+			return false;
+		}
+
+		try {
+			$result = $this->connection->executeQuery('SELECT VERSION()');
+			try {
+				$version = $result->fetchOne();
+			} finally {
+				$result->closeCursor();
+			}
+		} catch (\Throwable) {
+			return false;
+		}
+
+		return is_string($version) && preg_match('/^10\\.11\\.[0-9]+(?:-|$)/', $version) === 1;
+	}
 }

lib/composer/composer/autoload_classmap.php

@@ -1787,6 +1787,7 @@
     'OC\\DB\\ConnectionFactory' => $baseDir . '/lib/private/DB/ConnectionFactory.php',
     'OC\\DB\\DbDataCollector' => $baseDir . '/lib/private/DB/DbDataCollector.php',
     'OC\\DB\\Exceptions\\DbalException' => $baseDir . '/lib/private/DB/Exceptions/DbalException.php',
+    'OC\\DB\\FilesSearchSnapshot' => $baseDir . '/lib/private/DB/FilesSearchSnapshot.php',
     'OC\\DB\\Middleware\\ConnectionActivityConnection' => $baseDir . '/lib/private/DB/Middleware/ConnectionActivityConnection.php',
     'OC\\DB\\Middleware\\ConnectionActivityDriver' => $baseDir . '/lib/private/DB/Middleware/ConnectionActivityDriver.php',
     'OC\\DB\\Middleware\\ConnectionActivityMiddleware' => $baseDir . '/lib/private/DB/Middleware/ConnectionActivityMiddleware.php',
@@ -1975,6 +1976,7 @@
     'OC\\Files\\ObjectStore\\Swift' => $baseDir . '/lib/private/Files/ObjectStore/Swift.php',
     'OC\\Files\\ObjectStore\\SwiftFactory' => $baseDir . '/lib/private/Files/ObjectStore/SwiftFactory.php',
     'OC\\Files\\ObjectStore\\SwiftV2CachingAuthService' => $baseDir . '/lib/private/Files/ObjectStore/SwiftV2CachingAuthService.php',
+    'OC\\Files\\Search\\FilesSearchQuery' => $baseDir . '/lib/private/Files/Search/FilesSearchQuery.php',
     'OC\\Files\\Search\\QueryOptimizer\\FlattenNestedBool' => $baseDir . '/lib/private/Files/Search/QueryOptimizer/FlattenNestedBool.php',
     'OC\\Files\\Search\\QueryOptimizer\\FlattenSingleArgumentBinaryOperation' => $baseDir . '/lib/private/Files/Search/QueryOptimizer/FlattenSingleArgumentBinaryOperation.php',
     'OC\\Files\\Search\\QueryOptimizer\\MergeDistributiveOperations' => $baseDir . '/lib/private/Files/Search/QueryOptimizer/MergeDistributiveOperations.php',

lib/composer/composer/autoload_static.php

@@ -1828,6 +1828,7 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2
         'OC\\DB\\ConnectionFactory' => __DIR__ . '/../../..' . '/lib/private/DB/ConnectionFactory.php',
         'OC\\DB\\DbDataCollector' => __DIR__ . '/../../..' . '/lib/private/DB/DbDataCollector.php',
         'OC\\DB\\Exceptions\\DbalException' => __DIR__ . '/../../..' . '/lib/private/DB/Exceptions/DbalException.php',
+        'OC\\DB\\FilesSearchSnapshot' => __DIR__ . '/../../..' . '/lib/private/DB/FilesSearchSnapshot.php',
         'OC\\DB\\Middleware\\ConnectionActivityConnection' => __DIR__ . '/../../..' . '/lib/private/DB/Middleware/ConnectionActivityConnection.php',
         'OC\\DB\\Middleware\\ConnectionActivityDriver' => __DIR__ . '/../../..' . '/lib/private/DB/Middleware/ConnectionActivityDriver.php',
         'OC\\DB\\Middleware\\ConnectionActivityMiddleware' => __DIR__ . '/../../..' . '/lib/private/DB/Middleware/ConnectionActivityMiddleware.php',
@@ -2016,6 +2017,7 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2
         'OC\\Files\\ObjectStore\\Swift' => __DIR__ . '/../../..' . '/lib/private/Files/ObjectStore/Swift.php',
         'OC\\Files\\ObjectStore\\SwiftFactory' => __DIR__ . '/../../..' . '/lib/private/Files/ObjectStore/SwiftFactory.php',
         'OC\\Files\\ObjectStore\\SwiftV2CachingAuthService' => __DIR__ . '/../../..' . '/lib/private/Files/ObjectStore/SwiftV2CachingAuthService.php',
+        'OC\\Files\\Search\\FilesSearchQuery' => __DIR__ . '/../../..' . '/lib/private/Files/Search/FilesSearchQuery.php',
         'OC\\Files\\Search\\QueryOptimizer\\FlattenNestedBool' => __DIR__ . '/../../..' . '/lib/private/Files/Search/QueryOptimizer/FlattenNestedBool.php',
         'OC\\Files\\Search\\QueryOptimizer\\FlattenSingleArgumentBinaryOperation' => __DIR__ . '/../../..' . '/lib/private/Files/Search/QueryOptimizer/FlattenSingleArgumentBinaryOperation.php',
         'OC\\Files\\Search\\QueryOptimizer\\MergeDistributiveOperations' => __DIR__ . '/../../..' . '/lib/private/Files/Search/QueryOptimizer/MergeDistributiveOperations.php',