Skip to content

Commit d8ca3fd

Browse files
CarlSchwanCarlSchwan
committed
refactor(preview-migration): Simplify the preview migration
Move the filecache queries from the job into the migration service, so the job only walks the preview folders of its partition, and share the source file lookup and the preview creation between the batch and the single file migration. Assisted-by: ClaudeCode:claude-opus-5-5 Signed-off-by: Carl Schwan <carl@carlschwan.eu>
1 parent 41afe3d commit d8ca3fd

4 files changed

Lines changed: 186 additions & 196 deletions

File tree

‎core/BackgroundJobs/PreviewMigrationJob.php‎

Lines changed: 25 additions & 77 deletions
Original file line numberDiff line numberDiff line change
@@ -9,18 +9,13 @@
99

1010
namespace OC\Core\BackgroundJobs;
1111

12-
use OC\Files\Cache\CacheEntry;
1312
use OC\Preview\PreviewMigrationService;
1413
use OCP\AppFramework\Utility\ITimeFactory;
1514
use OCP\BackgroundJob\IJobList;
1615
use OCP\BackgroundJob\TimedJob;
17-
use OCP\DB\QueryBuilder\IQueryBuilder;
18-
use OCP\Files\FileInfo;
19-
use OCP\Files\IMimeTypeLoader;
2016
use OCP\Files\IRootFolder;
2117
use OCP\IAppConfig;
2218
use OCP\IConfig;
23-
use OCP\IDBConnection;
2419
use Override;
2520
use Psr\Log\LoggerInterface;
2621

@@ -36,8 +31,6 @@ public function __construct(
3631
private readonly IRootFolder $rootFolder,
3732
private readonly PreviewMigrationService $migrationService,
3833
private readonly IJobList $jobList,
39-
private readonly IDBConnection $connection,
40-
private readonly IMimeTypeLoader $mimeTypeLoader,
4134
private readonly LoggerInterface $logger,
4235
) {
4336
parent::__construct($time);
@@ -81,8 +74,7 @@ private function runPartition(int $partition): bool {
8174
return true;
8275
}
8376

84-
$cache = $storage->getCache();
85-
$previewRootId = $cache->getId(rtrim($this->previewRootPath, '/'));
77+
$previewRootId = $storage->getCache()->getId(rtrim($this->previewRootPath, '/'));
8678
if ($previewRootId === -1) {
8779
$this->logger->warning('Preview migration skipped: no preview root found at "{path}" on storage "{storageId}".', [
8880
'path' => $this->previewRootPath,
@@ -92,97 +84,53 @@ private function runPartition(int $partition): bool {
9284
}
9385

9486
$startTime = time();
95-
$storageId = $cache->getNumericStorageId();
96-
$folderMimeTypeId = $this->mimeTypeLoader->getId(FileInfo::MIMETYPE_FOLDER);
97-
$foldersToVisit = [[$previewRootId, '', 0]];
87+
$foldersToVisit = [['id' => $previewRootId, 'name' => '', 'depth' => 0]];
9888
$foldersToMigrate = [];
9989
// Folders without preview files, by depth; removed once their children are gone.
10090
$emptyFolders = [];
10191

10292
while ($foldersToVisit !== []) {
103-
$folders = [];
104-
foreach (array_splice($foldersToVisit, -self::BATCH_SIZE) as [$folderId, $folderName, $depth]) {
105-
if ($depth === 1 && !$this->belongsToPartition($folderName, $partition)) {
106-
continue;
93+
$folders = array_filter(
94+
array_splice($foldersToVisit, -self::BATCH_SIZE),
95+
fn (array $folder): bool => $folder['depth'] !== 1 || $this->belongsToPartition($folder['name'], $partition),
96+
);
97+
$children = $this->migrationService->getFolderChildren(array_column($folders, 'id'));
98+
99+
foreach ($folders as $folder) {
100+
foreach ($children[$folder['id']]['folders'] as $subFolder) {
101+
$foldersToVisit[] = [...$subFolder, 'depth' => $folder['depth'] + 1];
107102
}
108-
$folders[$folderId] = [$folderName, $depth];
109-
}
110-
if ($folders === []) {
111-
continue;
112-
}
113103

114-
$entries = array_fill_keys(array_keys($folders), []);
115-
$qb = $this->connection->getQueryBuilder();
116-
$qb->select('fileid', 'parent', 'name', 'mimetype', 'size', 'mtime')
117-
->from('filecache')
118-
->where($qb->expr()->in('parent', $qb->createNamedParameter(array_keys($folders), IQueryBuilder::PARAM_INT_ARRAY)))
119-
->hintShardKey('storage', $storageId);
120-
$cursor = $qb->executeQuery();
121-
while ($row = $cursor->fetchAssociative()) {
122-
$parent = (int)$row['parent'];
123-
if ((int)$row['mimetype'] === $folderMimeTypeId) {
124-
$foldersToVisit[] = [(int)$row['fileid'], $row['name'], $folders[$parent][1] + 1];
125-
} else {
126-
$entries[$parent][] = new CacheEntry($row);
127-
}
128-
}
129-
$cursor->closeCursor();
130-
131-
foreach ($folders as $folderId => [$folderName, $depth]) {
132-
if ($entries[$folderId] === [] || !ctype_digit($folderName)) {
133-
if ($depth > 0) {
134-
$emptyFolders[$depth][] = $folderId;
135-
}
136-
continue;
104+
$files = $children[$folder['id']]['files'];
105+
if ($files !== [] && ctype_digit($folder['name'])) {
106+
$foldersToMigrate[] = [
107+
'fileId' => (int)$folder['name'],
108+
'folderId' => $folder['id'],
109+
'flat' => $folder['depth'] === 1,
110+
'entries' => $files,
111+
];
112+
} elseif ($folder['depth'] > 0) {
113+
$emptyFolders[$folder['depth']][] = $folder['id'];
137114
}
138-
$foldersToMigrate[] = ['fileId' => (int)$folderName, 'folderId' => $folderId, 'flat' => $depth === 1, 'entries' => $entries[$folderId]];
139115
}
140116

141117
if (count($foldersToMigrate) >= self::BATCH_SIZE) {
142-
$this->migrateFolders($foldersToMigrate);
118+
$this->migrationService->migrateFolders($foldersToMigrate);
143119
$foldersToMigrate = [];
144120

145121
if (time() - $startTime > 3600) {
146122
return false;
147123
}
148124
}
149125
}
150-
$this->migrateFolders($foldersToMigrate);
151-
152-
krsort($emptyFolders);
153-
foreach ($emptyFolders as $folderIds) {
154-
foreach (array_chunk($folderIds, 1000) as $chunk) {
155-
$qb = $this->connection->getQueryBuilder();
156-
$qb->selectDistinct('parent')
157-
->from('filecache')
158-
->where($qb->expr()->in('parent', $qb->createNamedParameter($chunk, IQueryBuilder::PARAM_INT_ARRAY)))
159-
->hintShardKey('storage', $storageId);
160-
$nonEmpty = array_map('intval', $qb->executeQuery()->fetchFirstColumn());
161-
$this->migrationService->deleteOldFileCacheEntries(array_values(array_diff($chunk, $nonEmpty)));
162-
}
163-
}
164126

165-
return true;
166-
}
127+
$this->migrationService->migrateFolders($foldersToMigrate);
128+
$this->migrationService->deleteEmptyFolders($emptyFolders);
167129

168-
/**
169-
* @param list<array{fileId: int, folderId: int, flat: bool, entries: list<CacheEntry>}> $folders
170-
*/
171-
private function migrateFolders(array $folders): void {
172-
try {
173-
$this->migrationService->migrateFolders($folders);
174-
} catch (\Exception $e) {
175-
$this->logger->error('Failed to migrate previews of file ids: ' . implode(', ', array_column($folders, 'fileId')), [
176-
'exception' => $e,
177-
]);
178-
}
130+
return true;
179131
}
180132

181133
private function belongsToPartition(string $folderName, int $partition): bool {
182-
if ($partition < 0 || $partition >= self::PARTITIONS) {
183-
return false;
184-
}
185-
186134
if (ctype_digit($folderName)) {
187135
return ((int)$folderName % self::PARTITIONS) === $partition;
188136
}

‎lib/private/Preview/Db/PreviewMapper.php‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,8 @@ public function insert(Entity $entity): Entity {
6060
/** @var Preview $preview */
6161
$preview = $entity;
6262

63+
// The version row reuses the preview id, so it has to exist before it is written.
64+
$preview->generateId();
6365
$preview->setMimetypeId($this->mimeTypeLoader->getId($preview->getMimeType()));
6466
$preview->setSourceMimetypeId($this->mimeTypeLoader->getId($preview->getSourceMimeType()));
6567

0 commit comments

Comments
 (0)