Skip to content

Commit 29f52ad

Browse files
hastinbegodismyjudge95jasonvarga
authored
[5.x] Performance Optimizations for Stache and Query Operations (#12894)
Co-authored-by: Daniel Weaver <godismyjudge95@users.noreply.github.com> Co-authored-by: Jason Varga <jason@pixelfear.com>
1 parent 2ba59e9 commit 29f52ad

11 files changed

Lines changed: 159 additions & 25 deletions

File tree

‎config/stache.php‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -140,4 +140,27 @@
140140
'timeout' => 30,
141141
],
142142

143+
/*
144+
|--------------------------------------------------------------------------
145+
| Warming Optimization
146+
|--------------------------------------------------------------------------
147+
|
148+
| These options control performance optimizations during Stache warming.
149+
|
150+
*/
151+
152+
'warming' => [
153+
// Enable parallel store processing for faster warming on multi-core systems
154+
'parallel_processing' => env('STATAMIC_STACHE_PARALLEL_WARMING', false),
155+
156+
// Maximum number of parallel processes (0 = auto-detect CPU cores)
157+
'max_processes' => env('STATAMIC_STACHE_MAX_PROCESSES', 0),
158+
159+
// Minimum number of stores required to enable parallel processing
160+
'min_stores_for_parallel' => env('STATAMIC_STACHE_MIN_STORES_PARALLEL', 3),
161+
162+
// Concurrency driver: 'process', 'fork', or 'sync'
163+
'concurrency_driver' => env('STATAMIC_STACHE_CONCURRENCY_DRIVER', 'process'),
164+
],
165+
143166
];

‎src/Dictionaries/BasicDictionary.php‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -58,8 +58,11 @@ protected function matchesSearchQuery(string $query, Item $item): bool
5858
{
5959
$query = strtolower($query);
6060

61+
// Pre-compute searchable lookup for O(1) access instead of O(n) in_array()
62+
$searchableLookup = empty($this->searchable) ? null : array_flip($this->searchable);
63+
6164
foreach ($item->extra() as $key => $value) {
62-
if (! empty($this->searchable) && ! in_array($key, $this->searchable)) {
65+
if ($searchableLookup !== null && ! isset($searchableLookup[$key])) {
6366
continue;
6467
}
6568

‎src/Stache/Indexes/Terms/Value.php‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ class Value extends Index
1010
public function getItems()
1111
{
1212
$associatedItems = $this->store->index('associations')->items()
13+
->filter()
1314
->mapWithKeys(function ($association) {
1415
$term = Term::make($value = $association['slug'])
1516
->taxonomy($this->store->childKey())

‎src/Stache/Query/Builder.php‎

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -157,16 +157,20 @@ protected function filterWhereBasic($values, $where)
157157

158158
protected function filterWhereIn($values, $where)
159159
{
160-
return $values->filter(function ($value) use ($where) {
161-
return in_array($value, $where['values']);
162-
});
160+
$lookup = array_flip($where['values']);
161+
162+
return $values->filter(
163+
fn ($value) => isset($lookup[$value])
164+
);
163165
}
164166

165167
protected function filterWhereNotIn($values, $where)
166168
{
167-
return $values->filter(function ($value) use ($where) {
168-
return ! in_array($value, $where['values']);
169-
});
169+
$lookup = array_flip($where['values']);
170+
171+
return $values->filter(
172+
fn ($value) => ! isset($lookup[$value])
173+
);
170174
}
171175

172176
protected function filterWhereNull($values, $where)

‎src/Stache/Repositories/EntryRepository.php‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -98,7 +98,16 @@ public function findByUri(string $uri, ?string $site = null): ?Entry
9898

9999
public function whereInId($ids): EntryCollection
100100
{
101+
if (empty($ids)) {
102+
return EntryCollection::make();
103+
}
104+
101105
$entries = $this->query()->whereIn('id', $ids)->get();
106+
107+
if ($entries->isEmpty()) {
108+
return EntryCollection::make();
109+
}
110+
102111
$entriesById = $entries->keyBy->id();
103112

104113
$ordered = collect($ids)
@@ -175,6 +184,10 @@ public function substitute($item)
175184

176185
public function applySubstitutions($items)
177186
{
187+
if (empty($this->substitutionsById)) {
188+
return $items;
189+
}
190+
178191
return $items->map(function ($item) {
179192
return $this->substitutionsById[$item->id()] ?? $item;
180193
});

‎src/Stache/Repositories/SubmissionRepository.php‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,10 @@ public function whereForm(string $handle): Collection
3131

3232
public function whereInForm(array $handles): Collection
3333
{
34+
if (empty($handles)) {
35+
return collect();
36+
}
37+
3438
return $this->query()->whereIn('form', $handles)->get();
3539
}
3640

‎src/Stache/Repositories/TermRepository.php‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,10 @@ public function whereTaxonomy(string $handle): TermCollection
4747

4848
public function whereInTaxonomy(array $handles): TermCollection
4949
{
50+
if (empty($handles)) {
51+
return TermCollection::make();
52+
}
53+
5054
collect($handles)
5155
->reject(fn ($taxonomy) => Taxonomy::find($taxonomy))
5256
->each(fn ($taxonomy) => throw new TaxonomyNotFoundException($taxonomy));
@@ -199,6 +203,10 @@ public function substitute($item)
199203

200204
public function applySubstitutions($items)
201205
{
206+
if (empty($this->substitutionsById)) {
207+
return $items;
208+
}
209+
202210
return $items->map(function ($item) {
203211
return $this->substitutionsById[$item->id()] ?? $item;
204212
});

‎src/Stache/Stache.php‎

Lines changed: 84 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44

55
use Carbon\Carbon;
66
use Illuminate\Support\Facades\Cache;
7+
use Illuminate\Support\Facades\Concurrency;
78
use Statamic\Events\StacheCleared;
89
use Statamic\Events\StacheWarmed;
910
use Statamic\Extensions\FileStore;
@@ -118,7 +119,13 @@ public function warm()
118119

119120
$this->startTimer();
120121

121-
$this->stores()->except($this->exclude)->each->warm();
122+
$stores = $this->stores()->except($this->exclude);
123+
124+
if ($this->shouldUseParallelWarming($stores)) {
125+
$this->warmInParallel($stores);
126+
} else {
127+
$stores->each->warm();
128+
}
122129

123130
$this->stopTimer();
124131

@@ -232,4 +239,80 @@ public function isWatcherEnabled(): bool
232239
? app()->isLocal()
233240
: (bool) $config;
234241
}
242+
243+
protected function shouldUseParallelWarming($stores): bool
244+
{
245+
$config = config('statamic.stache.warming', []);
246+
247+
if (! ($config['parallel_processing'] ?? false)) {
248+
return false;
249+
}
250+
251+
if ($stores->count() < ($config['min_stores_for_parallel'] ?? 3)) {
252+
return false;
253+
}
254+
255+
if ($this->getCpuCoreCount() < 2) {
256+
return false;
257+
}
258+
259+
// Disable parallel processing if using Redis cache (serialization issues)
260+
$cacheDriver = config('statamic.stache.cache_store', config('cache.default'));
261+
if ($cacheDriver === 'redis') {
262+
\Log::info('Parallel warming disabled due to Redis cache driver');
263+
264+
return false;
265+
}
266+
267+
return true;
268+
}
269+
270+
protected function warmInParallel($stores)
271+
{
272+
try {
273+
$config = config('statamic.stache.warming', []);
274+
$maxProcesses = $config['max_processes'] ?? 0;
275+
276+
if ($maxProcesses <= 0) {
277+
$maxProcesses = $this->getCpuCoreCount();
278+
}
279+
280+
$maxProcesses = min($maxProcesses, $stores->count());
281+
282+
$chunkSize = (int) ceil($stores->count() / $maxProcesses);
283+
$chunks = $stores->chunk($chunkSize);
284+
285+
$closures = $chunks->map(function ($chunk) {
286+
return function () use ($chunk) {
287+
return $chunk->each->warm()->keys()->all();
288+
};
289+
})->all();
290+
291+
$driver = $config['concurrency_driver'] ?? 'process';
292+
293+
if (empty($closures)) {
294+
\Log::info('Closures are empty, skipping parallel warming');
295+
}
296+
297+
Concurrency::driver($driver)->run($closures);
298+
} catch (\Exception $e) {
299+
\Log::warning('Parallel warming failed, falling back to sequential: '.$e->getMessage());
300+
$stores->each->warm();
301+
}
302+
}
303+
304+
protected function getCpuCoreCount(): int
305+
{
306+
if (! function_exists('shell_exec')) {
307+
return 1;
308+
}
309+
310+
$command = match (PHP_OS_FAMILY) {
311+
'Windows' => 'echo %NUMBER_OF_PROCESSORS%',
312+
'Darwin' => 'sysctl -n hw.ncpu 2>/dev/null || echo 1',
313+
default => 'nproc 2>/dev/null || echo 1',
314+
};
315+
316+
return max(1, (int) shell_exec($command));
317+
}
235318
}

‎src/Stache/Stores/Store.php‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@
33
namespace Statamic\Stache\Stores;
44

55
use Facades\Statamic\Stache\Traverser;
6-
use Illuminate\Support\Facades\Cache;
76
use Statamic\Facades\File;
87
use Statamic\Facades\Path;
98
use Statamic\Facades\Stache;

‎src/Stache/Traverser.php‎

Lines changed: 11 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -2,19 +2,13 @@
22

33
namespace Statamic\Stache;
44

5-
use Illuminate\Filesystem\Filesystem;
65
use Statamic\Facades\Path;
6+
use Symfony\Component\Finder\Finder;
77

88
class Traverser
99
{
10-
protected $filesystem;
1110
protected $filter;
1211

13-
public function __construct(Filesystem $filesystem)
14-
{
15-
$this->filesystem = $filesystem;
16-
}
17-
1812
public function traverse($store)
1913
{
2014
if (! $dir = $store->directory()) {
@@ -23,20 +17,22 @@ public function traverse($store)
2317

2418
$dir = rtrim($dir, '/');
2519

26-
if (! $this->filesystem->exists($dir)) {
20+
if (! file_exists($dir)) {
2721
return collect();
2822
}
2923

30-
$files = collect($this->filesystem->allFiles($dir));
24+
$files = Finder::create()->files()->ignoreDotFiles(true)->in($dir)->sortByName();
25+
26+
$paths = [];
27+
foreach ($files as $file) {
28+
if ($this->filter && ! call_user_func($this->filter, $file)) {
29+
continue;
30+
}
3131

32-
if ($this->filter) {
33-
$files = $files->filter($this->filter);
32+
$paths[Path::tidy($file->getPathname())] = $file->getMTime();
3433
}
3534

36-
return $files
37-
->mapWithKeys(function ($file) {
38-
return [Path::tidy($file->getPathname()) => $file->getMTime()];
39-
})->sort();
35+
return collect($paths)->sort();
4036
}
4137

4238
public function filter($filter)

0 commit comments

Comments
 (0)