Repository navigation
ci: retain useful clang-tidy cache entries #8105
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,58 @@ | ||
| #!/usr/bin/env python3 | ||
| # Copyright (c) 2026 The Dash Core developers | ||
| # Distributed under the MIT software license, see the accompanying | ||
| # file COPYING or http://www.opensource.org/licenses/mit-license.php. | ||
|
|
||
| """Trim the local clang-tidy cache to an allocated-size limit, oldest files first.""" | ||
|
|
||
| import argparse | ||
| from pathlib import Path | ||
| import subprocess | ||
|
|
||
|
|
||
| def cache_size(cache_dir: Path) -> int: | ||
| return int(subprocess.check_output(["du", "-sk", str(cache_dir)], text=True, encoding="utf8").split()[0]) * 1024 | ||
|
|
||
|
|
||
| def cleanup(cache_dir: Path, max_size_mb: int) -> None: | ||
| limit = max_size_mb * 1024 * 1024 | ||
| size = cache_size(cache_dir) | ||
| if size <= limit: | ||
| return | ||
|
|
||
| print(f"Cache size {size / 1024 / 1024:.1f} MiB exceeds limit {max_size_mb} MiB, cleaning old entries...") | ||
| files = sorted((path.stat().st_mtime_ns, path) for path in cache_dir.rglob("*") if path.is_file()) | ||
| removed = 0 | ||
| for _, path in files: | ||
| path.unlink() | ||
| removed += 1 | ||
| parent = path.parent | ||
| while parent != cache_dir: | ||
| try: | ||
| parent.rmdir() | ||
| except OSError: | ||
| break | ||
| parent = parent.parent | ||
| # Outputs vary greatly in size, and stats files count too. Check bytes, | ||
| # not a fixed percentage of files, including the remaining directories. | ||
| size = cache_size(cache_dir) | ||
| if size <= limit: | ||
| break | ||
|
Comment on lines
+36
to
+40
Collaborator
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 💬 Nitpick: Avoid a full cache traversal after every deleted file Each iteration invokes cache_size(), which starts a new du process and traverses the remaining cache tree. Deleting k files from an n-file cache therefore performs O(k*n) metadata work, becoming quadratic when most entries must be removed. The early return avoids this cost below budget, and remeasurement correctly includes directory allocation, so this is not a correctness problem. Track allocated blocks for files and removed directories, then verify with du near the limit, rather than rescanning after every unlink; retain the final measured-budget check. source: |
||
|
|
||
| print(f"Removed {removed} oldest cache files; cache size after cleanup: {size / 1024 / 1024:.1f} MiB") | ||
| if size > limit: | ||
| raise RuntimeError("Could not trim ctcache to its size limit") | ||
|
|
||
|
|
||
| def main() -> None: | ||
| parser = argparse.ArgumentParser(description=__doc__) | ||
| parser.add_argument("cache_dir", type=Path) | ||
| parser.add_argument("max_size_mb", type=int) | ||
| args = parser.parse_args() | ||
| if args.max_size_mb <= 0: | ||
| parser.error("max_size_mb must be positive") | ||
| cleanup(args.cache_dir.resolve(), args.max_size_mb) | ||
|
|
||
|
|
||
| if __name__ == "__main__": | ||
| main() | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,77 @@ | ||
| #!/usr/bin/env python3 | ||
| # Copyright (c) 2026 The Dash Core developers | ||
| # Distributed under the MIT software license, see the accompanying | ||
| # file COPYING or http://www.opensource.org/licenses/mit-license.php. | ||
|
|
||
| """Regression tests for the lint cache's disk budget (no clang-tidy required).""" | ||
|
|
||
| import os | ||
| from pathlib import Path | ||
| import subprocess | ||
| import sys | ||
| import tempfile | ||
| import time | ||
| import unittest | ||
|
|
||
|
|
||
| SCRIPT = Path(__file__).with_name("cleanup-ctcache.py") | ||
| MIB = 1024 * 1024 | ||
|
|
||
|
|
||
| class CleanupCTCacheTest(unittest.TestCase): | ||
| def setUp(self): | ||
| self.temp = tempfile.TemporaryDirectory() | ||
| self.addCleanup(self.temp.cleanup) | ||
| self.cache = Path(self.temp.name) | ||
|
|
||
| def write_entry(self, name, size, timestamp): | ||
| path = self.cache / name | ||
| path.parent.mkdir(parents=True, exist_ok=True) | ||
| path.write_bytes(b"x" * size) | ||
| os.utime(path, (timestamp, timestamp)) | ||
| return path | ||
|
|
||
| def size_kib(self): | ||
| return int(subprocess.check_output(["du", "-sk", str(self.cache)], text=True, encoding="utf8").split()[0]) | ||
|
|
||
| def trim(self, limit): | ||
| return subprocess.run( | ||
| [sys.executable, str(SCRIPT), str(self.cache), str(limit)], | ||
| check=True, capture_output=True, text=True, encoding="utf8", | ||
| ).stdout | ||
|
|
||
| def test_under_budget_keeps_even_old_entries(self): | ||
| old = self.write_entry("ab/old", MIB // 2, 1) | ||
| before = self.size_kib() | ||
| self.trim(1) | ||
| self.assertTrue(old.exists()) | ||
| self.assertEqual(before, self.size_kib()) | ||
|
|
||
| def test_unequal_outputs_and_stats_trim_to_budget_oldest_first(self): | ||
| # Removing 20% of these files only removes tiny stats, not enough bytes. | ||
| now = int(time.time()) | ||
| stats = [self.write_entry(f"stats/{i:02x}", 4, now - 200 + i) for i in range(10)] | ||
| large = self.write_entry("ab/old large output", 2 * MIB, now - 100) | ||
| hot = self.write_entry("cd/recent output", MIB // 2, now) | ||
| self.assertGreater(self.size_kib(), 1024) | ||
| output = self.trim(1) | ||
| self.assertLessEqual(self.size_kib(), 1024) | ||
| self.assertTrue(hot.exists()) | ||
| self.assertFalse(large.exists()) | ||
| self.assertTrue(all(not path.exists() for path in stats)) | ||
| self.assertFalse((self.cache / "ab").exists()) | ||
| self.assertIn("Removed 11 oldest cache files", output) | ||
| self.assertEqual(self.trim(1), "") | ||
|
|
||
| def test_empty_cache_and_invalid_limit(self): | ||
| self.trim(1) | ||
| result = subprocess.run( | ||
| [sys.executable, str(SCRIPT), str(self.cache), "0"], | ||
| capture_output=True, text=True, encoding="utf8", | ||
| ) | ||
| self.assertNotEqual(result.returncode, 0) | ||
| self.assertIn("must be positive", result.stderr) | ||
|
|
||
|
|
||
| if __name__ == "__main__": | ||
| unittest.main() |
Uh oh!
There was an error while loading. Please reload this page.