From 0732cba8eb20c99d9e3d093bbb03e9eec3e176c8 Mon Sep 17 00:00:00 2001 From: JakimPL Date: Sat, 12 Sep 2026 18:26:41 +0200 Subject: [PATCH 01/36] Added: compression search budget --- docs/concepts/compression.md | 13 ++-- src/sampletones_player/compression/budget.py | 61 +++++++++++++++ src/sampletones_player/compression/encode.py | 4 + src/sampletones_player/compression/search.py | 77 +++++++++++++------ src/sampletones_player/compression/song.py | 4 + tests/integration/nsf/corpus.py | 7 +- .../nsf/test_compression_report.py | 2 +- .../compression/test_budget.py | 61 +++++++++++++++ .../compression/test_search.py | 71 +++++++++++++++++ 9 files changed, 269 insertions(+), 31 deletions(-) create mode 100644 src/sampletones_player/compression/budget.py create mode 100644 tests/unit/sampletones_player/compression/test_budget.py create mode 100644 tests/unit/sampletones_player/compression/test_search.py diff --git a/docs/concepts/compression.md b/docs/concepts/compression.md index 9fab475a1..05ff10a6a 100644 --- a/docs/concepts/compression.md +++ b/docs/concepts/compression.md @@ -264,11 +264,13 @@ transposition it makes possible. ## 7. Limitations -- **The search struggles on dense reconstructions.** A reconstruction whose planes turn - over at nearly every tick offers enormous numbers of candidates, and past a cap the - search stops gathering and earns nothing for that song. It bites at lengths where the - song already exceeds the program area, so it costs a diagnosis rather than a song, but - a dense export of two minutes is stored materially larger than the same song at one. +- **The search reads a dense plane only so far.** A reconstruction whose planes turn over + at nearly every tick offers enormous numbers of candidates, and a round gathers a fixed + number of them, shared among the planes by what each has to offer. A plane that spells out + little is read whole; a dense one is read as far as its share reaches, and the figures + beyond that point go unsearched. It bites at lengths where the song already exceeds the + program area, so it costs a diagnosis rather than a song, but a dense export of two + minutes is stored materially larger than the same song at one. - **The dictionary holds 255 phrases.** A project of roughly 30 to 60 samples fills it from its instruments alone, at which point the search has no room left to work in and further samples compete for slots on measured value. @@ -303,6 +305,7 @@ Where things live: | the dictionary, its entries and its pruning | `sampletones_player.compression.dictionary` | | phrases the instruments offer | `sampletones_player.compression.seeds` | | phrases the search earns | `sampletones_player.compression.search` | +| how much work the search spends | `sampletones_player.compression.budget` | | what a phrase plays against a plane | `sampletones_player.compression.matches` | | encoding, and the decoder the driver is held to | `sampletones_player.compression.encode`, `.decode` | | the opcode layout and its bounds | `sampletones_player.specification.compression` | diff --git a/src/sampletones_player/compression/budget.py b/src/sampletones_player/compression/budget.py new file mode 100644 index 000000000..bc2fbb87f --- /dev/null +++ b/src/sampletones_player/compression/budget.py @@ -0,0 +1,61 @@ +from typing import Final, List, Sequence, Tuple + +from pydantic import BaseModel, ConfigDict, Field + + +class SearchBudget(BaseModel): + """How much work the search spends on the phrases a song's own planes repeat. + + A round gathers candidate windows from what the parse still spells out, ranks them, and + confirms the best few by parsing the song again with each one added. Each figure here + bounds one of those steps, so the budget is what an export trades against the bytes the + search earns. + + Attributes: + candidate_entries: The candidate windows one round gathers over every plane together, + shared among the planes by what each has to offer. + rounds: The rounds the search runs, each adding at most one phrase. + confirmed_candidates: The best-ranked candidates a round parses the song again with. + """ + + model_config = ConfigDict(extra="forbid", frozen=True) + + candidate_entries: int = Field(ge=1) + rounds: int = Field(ge=0) + confirmed_candidates: int = Field(ge=1) + + +DEFAULT_SEARCH_BUDGET: Final[SearchBudget] = SearchBudget( + candidate_entries=200_000, + rounds=64, + confirmed_candidates=3, +) + + +def shares( + demands: Sequence[int], + total: int, +) -> Tuple[int, ...]: + """Divides ``total`` among claimants, meeting small demands in full and splitting what + remains evenly among the larger ones. + + Claimants are served from the smallest demand upward, each taking the lesser of its demand + and an even share of what is left, so a claimant asking for little leaves its surplus to the + rest and a claimant asking for much is held to the same share as its peers. + + Args: + demands: What each claimant would take, given the room. + total: What there is to share. + + Returns: + Tuple[int, ...]: What each claimant receives, in the order the demands were given. + """ + granted: List[int] = [0] * len(demands) + remaining = total + pending = len(demands) + for claimant in sorted(range(len(demands)), key=lambda index: demands[index]): + granted[claimant] = min(demands[claimant], remaining // pending) + remaining -= granted[claimant] + pending -= 1 + + return tuple(granted) diff --git a/src/sampletones_player/compression/encode.py b/src/sampletones_player/compression/encode.py index 1cf52ccfe..708ebdc2f 100644 --- a/src/sampletones_player/compression/encode.py +++ b/src/sampletones_player/compression/encode.py @@ -2,6 +2,7 @@ from typing import Dict, Final, FrozenSet, Iterable, Sequence, Tuple from sampletones_player.compression.admit import admit_seeds +from sampletones_player.compression.budget import DEFAULT_SEARCH_BUDGET, SearchBudget from sampletones_player.compression.compressed import CompressedPlanes from sampletones_player.compression.dictionary.phrase import Phrase from sampletones_player.compression.dictionary.prune import prune @@ -119,6 +120,7 @@ def encode_planes( *, options: CodecOptions, boundaries: FrozenSet[int], + budget: SearchBudget = DEFAULT_SEARCH_BUDGET, report: CodecReporter = silent_reporter, ) -> CompressedPlanes: """Compresses a song's planes into the dictionary and streams the driver reads. @@ -135,6 +137,7 @@ def encode_planes( seeds: The phrases the song's instruments offer. options: Which of the codec's layers the encoding is built from. boundaries: The ticks a token starts on, beyond the first tick of the song. + budget: How much work the search spends beyond the phrases the instruments seed. report: Hears what the run holds each time it looks up, and answers whether it goes on. Returns: @@ -170,6 +173,7 @@ def encode_planes( options, entries, monitor, + budget, ) table, parses = _settle( diff --git a/src/sampletones_player/compression/search.py b/src/sampletones_player/compression/search.py index 830e350cf..5ea60b2a9 100644 --- a/src/sampletones_player/compression/search.py +++ b/src/sampletones_player/compression/search.py @@ -1,5 +1,6 @@ -from typing import Dict, Final, FrozenSet, List, NamedTuple, Sequence +from typing import Callable, Dict, Final, FrozenSet, Iterator, List, NamedTuple, Sequence, Tuple +from sampletones_player.compression.budget import SearchBudget, shares from sampletones_player.compression.dictionary.phrase import Phrase, phrase_entry_size from sampletones_player.compression.dictionary.table import PhraseTable, phrase_table from sampletones_player.compression.matches.cache import MatchCache @@ -14,9 +15,6 @@ MIN_CANDIDATE_LENGTH: Final[int] = 3 MAX_CANDIDATE_LENGTH: Final[int] = 48 -MAX_CANDIDATE_ENTRIES: Final[int] = 200_000 -MAX_SEARCH_ROUNDS: Final[int] = 64 -CONFIRMED_CANDIDATES: Final[int] = 3 MIN_OCCURRENCES: Final[int] = 2 UNSHIFTED_OCCURRENCE_TRANSPOSE: Final[int] = 0 SHIFTED_OCCURRENCE_TRANSPOSE: Final[int] = 1 @@ -49,28 +47,58 @@ def _residue_spans(parse: Parse) -> List[_Span]: return spans +def _windows(parse: Parse) -> Iterator[Tuple[int, int]]: + """Each position the parse still spells out, beside the longest candidate starting there.""" + for span in _residue_spans(parse): + for position in range(span.start, span.end): + yield position, min(MAX_CANDIDATE_LENGTH, span.end - position) + + +def _entries(longest: int) -> int: + return max(0, longest - MIN_CANDIDATE_LENGTH + 1) + + +def _demand(parse: Parse) -> int: + return sum(_entries(longest) for _, longest in _windows(parse)) + + +def _gather_plane( + plane: int, + index: PlaneIndex, + parse: Parse, + allowance: int, + gather: Callable[[bytes, List[_Occurrence]], List[_Occurrence]], +) -> None: + differences = index.differences + entries = 0 + for position, longest in _windows(parse): + if entries >= allowance: + return + + occurrence = _Occurrence(plane=plane, position=position) + for length in range(MIN_CANDIDATE_LENGTH, longest + 1): + gather(differences[position : position + length - 1], []).append(occurrence) + + entries += _entries(longest) + + def _candidates( indices: Sequence[PlaneIndex], parses: Sequence[Parse], monitor: CodecMonitor, + entries: int, ) -> Dict[bytes, List[_Occurrence]]: + """Every candidate shape the planes still spell out, beside where each occurs. + + The entries a round gathers are shared among the planes by what each has to offer, so a + plane that spells out little is read whole and a dense one is read as far as its share + reaches, and every plane is heard from. + """ found: Dict[bytes, List[_Occurrence]] = {} - gather = found.setdefault - entries = 0 - for plane, (index, parse) in enumerate(zip(indices, parses)): + allowances = shares([_demand(parse) for parse in parses], entries) + for plane, (index, parse, allowance) in enumerate(zip(indices, parses, allowances)): monitor.poll() - differences = index.differences - for span in _residue_spans(parse): - if entries > MAX_CANDIDATE_ENTRIES: - return found - - for position in range(span.start, span.end): - longest = min(MAX_CANDIDATE_LENGTH, span.end - position) - occurrence = _Occurrence(plane=plane, position=position) - for length in range(MIN_CANDIDATE_LENGTH, longest + 1): - gather(differences[position : position + length - 1], []).append(occurrence) - - entries += max(0, longest - MIN_CANDIDATE_LENGTH + 1) + _gather_plane(plane, index, parse, allowance, found.setdefault) return found @@ -112,9 +140,10 @@ def _ranked( parses: Sequence[Parse], phrase_id: int, monitor: CodecMonitor, + budget: SearchBudget, ) -> List[_Candidate]: ranked: List[_Candidate] = [] - for key, occurrences in _candidates(indices, parses, monitor).items(): + for key, occurrences in _candidates(indices, parses, monitor, budget.candidate_entries).items(): if len(occurrences) < MIN_OCCURRENCES: continue @@ -130,7 +159,7 @@ def _ranked( ranked.append(_Candidate(gain=gain, body=body)) ranked.sort(key=lambda candidate: (-candidate.gain, candidate.body)) - return ranked[:CONFIRMED_CANDIDATES] + return ranked[: budget.confirmed_candidates] def _total(table: PhraseTable, parses: Sequence[Parse]) -> int: @@ -143,6 +172,7 @@ def search_phrases( options: CodecOptions, boundaries: FrozenSet[int], monitor: CodecMonitor, + budget: SearchBudget, ) -> PhraseTable: """Fills the dictionary with the phrases the song's own planes repeat. @@ -161,6 +191,7 @@ def search_phrases( options: Which of the codec's layers the encoding is built from. boundaries: The ticks a token starts on. monitor: Carries the run's reckoning of itself onward. + budget: How many candidates a round gathers and confirms, and how many rounds run. Returns: PhraseTable: The seeded phrases alongside the ones the search earned. @@ -172,12 +203,12 @@ def search_phrases( parses = parse_planes(cache, table, options, boundaries, monitor) total = _total(table, parses) monitor.reached(len(table), total) - for _ in range(MAX_SEARCH_ROUNDS): + for _ in range(budget.rounds): if len(table) == MAX_PHRASE_IDS: return table settled = False - for candidate in _ranked(indices, parses, len(table), monitor): + for candidate in _ranked(indices, parses, len(table), monitor, budget): offered = Phrase(body=candidate.body) enlarged = phrase_table(table.phrases + (offered,)) trial = parse_planes_offered( diff --git a/src/sampletones_player/compression/song.py b/src/sampletones_player/compression/song.py index 7a38582a7..df0f34741 100644 --- a/src/sampletones_player/compression/song.py +++ b/src/sampletones_player/compression/song.py @@ -1,5 +1,6 @@ from typing import FrozenSet, Optional, Sequence +from sampletones_player.compression.budget import DEFAULT_SEARCH_BUDGET, SearchBudget from sampletones_player.compression.compressed import CompressedPlanes from sampletones_player.compression.decode import decode_planes from sampletones_player.compression.dictionary.phrase import Phrase @@ -26,6 +27,7 @@ def compress_song( *, seeds: Sequence[Phrase], loop_tick: Optional[int] = None, + budget: SearchBudget = DEFAULT_SEARCH_BUDGET, report: CodecReporter = silent_reporter, ) -> CompressedPlanes: """Compresses a song's four register streams into the dictionary and streams a file carries. @@ -39,6 +41,7 @@ def compress_song( pitches: The timer each pitch sounds at, which is what turns a timer into an index. seeds: The phrases the song's instruments offer the dictionary. loop_tick: The tick the song returns to once it ends, or ``None`` where it stops there. + budget: How much work the search spends beyond the phrases the instruments seed. report: Hears what the codec holds each time it looks up, and answers whether it goes on. Returns: @@ -53,6 +56,7 @@ def compress_song( seeds, options=EVERY_LAYER, boundaries=_entries(loop_tick), + budget=budget, report=report, ) diff --git a/tests/integration/nsf/corpus.py b/tests/integration/nsf/corpus.py index db7d107d7..20188586b 100644 --- a/tests/integration/nsf/corpus.py +++ b/tests/integration/nsf/corpus.py @@ -229,17 +229,20 @@ def build_corpus( instrument_catalog: Dict[str, Sample], integration_project: Project, ) -> Tuple[CorpusEntry, ...]: - """The songs the codec is measured on: each sample alone, and the arrangement at two lengths. + """The songs the codec is measured on: each sample alone, the arrangement at two lengths, and + a minute of dense reconstruction. Args: instrument_catalog: The samples the integration suite reads. integration_project: The arrangement those samples are played in. Returns: - Tuple[CorpusEntry, ...]: The samples first, then the arrangement, then the long one. + Tuple[CorpusEntry, ...]: The samples first, then the arrangement, the long one, and the + reconstruction. """ return ( *sample_entries(instrument_catalog, integration_project.settings), arrangement_entry(ARRANGEMENT, integration_project), arrangement_entry(LONG_ARRANGEMENT, lengthened_arrangement(integration_project, TARGET_SECONDS)), + reconstruction_entry(RECONSTRUCTION, RECONSTRUCTION_SECONDS), ) diff --git a/tests/integration/nsf/test_compression_report.py b/tests/integration/nsf/test_compression_report.py index a7863419d..db0899580 100644 --- a/tests/integration/nsf/test_compression_report.py +++ b/tests/integration/nsf/test_compression_report.py @@ -116,7 +116,7 @@ def corpus( instrument_catalog: Dict[str, Sample], integration_project: Project, ) -> Tuple[CorpusEntry, ...]: - """The songs the report measures: each sample alone, and the arrangement at two lengths.""" + """The songs the report measures: each sample alone, the arrangement at two lengths, and a dense minute.""" return build_corpus(instrument_catalog, integration_project) diff --git a/tests/unit/sampletones_player/compression/test_budget.py b/tests/unit/sampletones_player/compression/test_budget.py new file mode 100644 index 000000000..e39a02db0 --- /dev/null +++ b/tests/unit/sampletones_player/compression/test_budget.py @@ -0,0 +1,61 @@ +from dataclasses import dataclass +from typing import Tuple + +import pytest +from pydantic import ValidationError + +from sampletones_player.compression.budget import DEFAULT_SEARCH_BUDGET, SearchBudget, shares +from tests.suite.base import BaseTestSuite +from tests.suite.case import BaseRegularTestCase + + +class TestSharesMeetSmallDemandsAndSplitTheRest(BaseTestSuite): + """A claimant asking for little is served whole; the larger ones split what remains evenly.""" + + @dataclass(frozen=True, kw_only=True) + class TestCase(BaseRegularTestCase): + demands: Tuple[int, ...] + total: int + expected: Tuple[int, ...] + + test_cases = ( + TestCase(label="every_demand_fits", demands=(5, 5, 5), total=30, expected=(5, 5, 5)), + TestCase(label="a_small_demand_leaves_its_surplus", demands=(100, 1, 100), total=41, expected=(20, 1, 20)), + TestCase(label="a_zero_demand_takes_nothing", demands=(0, 7), total=3, expected=(0, 3)), + TestCase(label="equal_demands_split_evenly", demands=(9, 9), total=8, expected=(4, 4)), + TestCase(label="a_total_short_of_one_each", demands=(9, 9, 9), total=2, expected=(0, 1, 1)), + TestCase(label="no_claimants", demands=(), total=10, expected=()), + ) + + @pytest.mark.parametrize( + "test_case", + test_cases, + ids=lambda test_case: test_case.label, + ) + def test_shares(self, test_case: TestCase) -> None: + assert shares(test_case.demands, test_case.total) == test_case.expected + + def test_the_total_is_never_exceeded(self) -> None: + demands = (30, 70, 10, 90) + for total in range(0, 220, 7): + assert sum(shares(demands, total)) <= total + + +class TestTheBudgetBoundsEveryStep: + """Each figure is at least what one round can act on, and the default states them all.""" + + def test_the_default_gathers_ranks_and_confirms(self) -> None: + assert DEFAULT_SEARCH_BUDGET.candidate_entries >= 1 + assert DEFAULT_SEARCH_BUDGET.rounds >= 1 + assert DEFAULT_SEARCH_BUDGET.confirmed_candidates >= 1 + + def test_a_round_gathers_at_least_one_entry(self) -> None: + with pytest.raises(ValidationError): + SearchBudget(candidate_entries=0, rounds=1, confirmed_candidates=1) + + def test_a_round_confirms_at_least_one_candidate(self) -> None: + with pytest.raises(ValidationError): + SearchBudget(candidate_entries=1, rounds=1, confirmed_candidates=0) + + def test_no_rounds_is_a_budget(self) -> None: + assert SearchBudget(candidate_entries=1, rounds=0, confirmed_candidates=1).rounds == 0 diff --git a/tests/unit/sampletones_player/compression/test_search.py b/tests/unit/sampletones_player/compression/test_search.py new file mode 100644 index 000000000..7a8d7ccab --- /dev/null +++ b/tests/unit/sampletones_player/compression/test_search.py @@ -0,0 +1,71 @@ +from random import Random +from typing import Final, FrozenSet + +import pytest + +from sampletones_player.compression.budget import SearchBudget +from sampletones_player.compression.dictionary.table import PhraseTable, phrase_table +from sampletones_player.compression.matches.cache import MatchCache +from sampletones_player.compression.matches.index import PlaneIndex +from sampletones_player.compression.options import EVERY_LAYER +from sampletones_player.compression.progress.monitor import CodecMonitor +from sampletones_player.compression.search import search_phrases +from sampletones_player.specification.binary import BYTE_VALUES +from sampletones_shared.utils.progress import silent_reporter + +STREAM_START: Final[FrozenSet[int]] = frozenset({0}) +FIGURE: Final[bytes] = b"\x10\x18\x14\x22\x1c\x30\x11\x19\x15\x23\x1d\x31\x12\x1a\x16\x24" +FIGURE_REPEATS: Final[int] = 6 +CROWDED_TICKS: Final[int] = 600 +CROWDED_SEED: Final[int] = 20260912 +REPEATING: Final[bytes] = FIGURE * FIGURE_REPEATS +NARROW_ENTRIES: Final[int] = 8_000 + + +def _crowded() -> bytes: + """A plane of unrelated values, which offers many candidates and repeats none of them.""" + random = Random(CROWDED_SEED) + return bytes(random.randrange(BYTE_VALUES) for _ in range(CROWDED_TICKS)) + + +def _searched(budget: SearchBudget) -> PhraseTable: + cache = MatchCache([PlaneIndex.from_plane(_crowded()), PlaneIndex.from_plane(REPEATING)]) + return search_phrases( + cache, + phrase_table(()), + EVERY_LAYER, + STREAM_START, + CodecMonitor(silent_reporter), + budget, + ) + + +@pytest.fixture(name="narrow") +def narrow_fixture() -> SearchBudget: + """A round too small for the crowded plane alone, and ample for the repeating one.""" + return SearchBudget(candidate_entries=NARROW_ENTRIES, rounds=64, confirmed_candidates=3) + + +class TestEveryPlaneIsHeardFrom: + """A crowded plane takes its share of a round and leaves the rest to the planes after it.""" + + def test_a_plane_after_a_crowded_one_earns_its_phrase(self, narrow: SearchBudget) -> None: + table = _searched(narrow) + assert any(phrase.body in REPEATING for phrase in table.phrases) + + def test_the_crowded_plane_earns_nothing(self, narrow: SearchBudget) -> None: + crowded = _crowded() + table = _searched(narrow) + assert all(phrase.body not in crowded for phrase in table.phrases) + + +class TestTheRoundsBoundWhatTheSearchEarns: + """Each round adds at most one phrase, so the rounds cap the dictionary the search fills.""" + + def test_no_rounds_leaves_the_table_as_seeded(self) -> None: + table = _searched(SearchBudget(candidate_entries=NARROW_ENTRIES, rounds=0, confirmed_candidates=3)) + assert len(table) == 0 + + def test_one_round_adds_one_phrase(self) -> None: + table = _searched(SearchBudget(candidate_entries=NARROW_ENTRIES, rounds=1, confirmed_candidates=3)) + assert len(table) == 1 From f1960f6e4e9592b21d7b1911244ee7d69a3bc63b Mon Sep 17 00:00:00 2001 From: JakimPL Date: Sat, 12 Sep 2026 19:01:59 +0200 Subject: [PATCH 02/36] Added: the compression study harness and its accounting report --- Makefile | 6 +- pyproject.toml | 3 +- scripts/codec_study/__init__.py | 0 scripts/codec_study/accounting/__init__.py | 0 scripts/codec_study/accounting/coincident.py | 37 +++ scripts/codec_study/accounting/dictionary.py | 66 +++++ scripts/codec_study/accounting/finding.py | 24 ++ scripts/codec_study/accounting/fixed.py | 63 +++++ scripts/codec_study/accounting/pairs.py | 57 ++++ scripts/codec_study/accounting/rows.py | 138 ++++++++++ scripts/codec_study/accounting/runs.py | 32 +++ scripts/codec_study/accounting/shares.py | 116 ++++++++ scripts/codec_study/accounting/tokens.py | 105 ++++++++ scripts/codec_study/corpus/__init__.py | 0 scripts/codec_study/corpus/build.py | 43 +++ scripts/codec_study/corpus/projects.py | 94 +++++++ scripts/codec_study/corpus/reconstructions.py | 61 +++++ scripts/codec_study/corpus/song.py | 49 ++++ scripts/codec_study/manifest.py | 125 +++++++++ scripts/codec_study/measure.py | 76 ++++++ scripts/codec_study/report/__init__.py | 0 scripts/codec_study/report/aggregate.py | 112 ++++++++ scripts/codec_study/report/rows.py | 87 ++++++ scripts/codec_study/report/run.py | 114 ++++++++ scripts/codec_study/report/writers.py | 42 +++ scripts/codec_study/variants/__init__.py | 0 scripts/codec_study/variants/production.py | 36 +++ scripts/codec_study/variants/registry.py | 26 ++ scripts/codec_study/variants/variant.py | 37 +++ scripts/compression_study.py | 129 +++++++++ src/sampletones_config/boundaries/tokens.yaml | 21 ++ src/sampletones_player/py.typed | 0 .../scripts/codec_study/test_accounting.py | 252 ++++++++++++++++++ 33 files changed, 1949 insertions(+), 2 deletions(-) create mode 100644 scripts/codec_study/__init__.py create mode 100644 scripts/codec_study/accounting/__init__.py create mode 100644 scripts/codec_study/accounting/coincident.py create mode 100644 scripts/codec_study/accounting/dictionary.py create mode 100644 scripts/codec_study/accounting/finding.py create mode 100644 scripts/codec_study/accounting/fixed.py create mode 100644 scripts/codec_study/accounting/pairs.py create mode 100644 scripts/codec_study/accounting/rows.py create mode 100644 scripts/codec_study/accounting/runs.py create mode 100644 scripts/codec_study/accounting/shares.py create mode 100644 scripts/codec_study/accounting/tokens.py create mode 100644 scripts/codec_study/corpus/__init__.py create mode 100644 scripts/codec_study/corpus/build.py create mode 100644 scripts/codec_study/corpus/projects.py create mode 100644 scripts/codec_study/corpus/reconstructions.py create mode 100644 scripts/codec_study/corpus/song.py create mode 100644 scripts/codec_study/manifest.py create mode 100644 scripts/codec_study/measure.py create mode 100644 scripts/codec_study/report/__init__.py create mode 100644 scripts/codec_study/report/aggregate.py create mode 100644 scripts/codec_study/report/rows.py create mode 100644 scripts/codec_study/report/run.py create mode 100644 scripts/codec_study/report/writers.py create mode 100644 scripts/codec_study/variants/__init__.py create mode 100644 scripts/codec_study/variants/production.py create mode 100644 scripts/codec_study/variants/registry.py create mode 100644 scripts/codec_study/variants/variant.py create mode 100644 scripts/compression_study.py create mode 100644 src/sampletones_player/py.typed create mode 100644 tests/unit/scripts/codec_study/test_accounting.py diff --git a/Makefile b/Makefile index 6b75d682a..1555ed7b1 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ .PHONY: help setup install build release system-deps run clean pre-commit test benchmarks \ - ftm-samples nsf-samples nsf-render compression-report icons player check-import-boundary check-tag-names check-unused-tags check-rendered-literals \ + ftm-samples nsf-samples nsf-render compression-report compression-study icons player check-import-boundary check-tag-names check-unused-tags check-rendered-literals \ check-language-keys check-palette-colors check-shortcut-actions calibration lint pylint mypy format ifeq ($(OS),Windows_NT) @@ -74,6 +74,7 @@ help: @echo $(Q) make nsf-samples - Emit example .nsf files to build/nsf via the integration suite$(Q) @echo $(Q) make nsf-render - Render the .nsf files in build/nsf to waves with ffmpeg$(Q) @echo $(Q) make compression-report - Measure the song codec into build/compression$(Q) + @echo $(Q) make compression-study - Measure the song codec over the projects and stems on this machine; the report lands in Documents/SampleToNES/compression (ARGS=--quick for a short run)$(Q) @echo $(Q) make icons - Generate the icon suite into src/sampletones_assets/icons$(Q) @echo $(Q) make player - Assemble the NES player driver with cc65$(Q) @echo $(Q) make calibration - Score the reconstruction corpus; the report lands in Documents/SampleToNES/calibration$(Q) @@ -130,6 +131,9 @@ compression-report: export SAMPLETONES_COMPRESSION_OUTPUT_DIR := build/compressi compression-report: uv run python -m pytest tests/integration/nsf/test_compression_report.py +compression-study: + uv run scripts/compression_study.py $(ARGS) + icons: uv run --group assets python scripts/assets/icons.py diff --git a/pyproject.toml b/pyproject.toml index 20448235e..1b2959935 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -122,6 +122,7 @@ target-version = ["py312"] profile = "black" line_length = 120 known_first_party = [ + "codec_study", "sampletones", "sampletones_application", "sampletones_assets", @@ -134,7 +135,7 @@ known_first_party = [ [tool.pytest.ini_options] addopts = "--import-mode=importlib" -pythonpath = ["."] +pythonpath = [".", "scripts"] [tool.coverage.run] source = [ diff --git a/scripts/codec_study/__init__.py b/scripts/codec_study/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/scripts/codec_study/accounting/__init__.py b/scripts/codec_study/accounting/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/scripts/codec_study/accounting/coincident.py b/scripts/codec_study/accounting/coincident.py new file mode 100644 index 000000000..bf4414bc7 --- /dev/null +++ b/scripts/codec_study/accounting/coincident.py @@ -0,0 +1,37 @@ +from typing import Final, Mapping, Sequence + +from codec_study.accounting.finding import NOTHING, Finding +from codec_study.accounting.tokens import ReadToken +from sampletones_player.compression.planes.order import PlaneOrder +from sampletones_player.specification.compression import OPCODE_SIZE + +CONTROL_SUFFIX: Final[str] = "_control" +VALUE_SUFFIX: Final[str] = "_value" + + +def coincident_starts(tokens: Mapping[str, Sequence[ReadToken]]) -> Finding: + """What one opcode for a channel's control and value would spare where both start a token + on the same tick (H5). + + A note usually changes its timbre and its pitch on the same tick, so the two planes pay two + opcodes for one moment. One opcode announcing both is the bound stated here: a byte per + coincidence, before the cost of telling the two apart. + + Args: + tokens: Every plane's tokens, under the plane's name in the song block. + + Returns: + Finding: The opcode bytes at coincident starts, and the bytes one opcode would spare. + """ + finding = NOTHING + for name in PlaneOrder.names(): + if not name.endswith(CONTROL_SUFFIX): + continue + + channel = name.removesuffix(CONTROL_SUFFIX) + control = {token.tick for token in tokens[name]} + value = {token.tick for token in tokens[f"{channel}{VALUE_SUFFIX}"]} + shared = len(control & value) + finding += Finding(2 * OPCODE_SIZE * shared, OPCODE_SIZE * shared) + + return finding diff --git a/scripts/codec_study/accounting/dictionary.py b/scripts/codec_study/accounting/dictionary.py new file mode 100644 index 000000000..af2bb2767 --- /dev/null +++ b/scripts/codec_study/accounting/dictionary.py @@ -0,0 +1,66 @@ +from collections import Counter +from typing import Dict, Final, Iterable + +from codec_study.accounting.finding import NOTHING, Finding +from codec_study.accounting.runs import runs +from codec_study.accounting.tokens import ReadToken +from sampletones_player.compression.dictionary.table import PhraseTable +from sampletones_player.specification.compression import PHRASE_COUNT_SIZE + +REPEATED: Final[int] = 2 +MIN_RLE_RUN: Final[int] = 3 +RLE_RUN_SIZE: Final[int] = 2 +DEFAULT_COUNT_SIZE: Final[int] = 1 + + +def plateaus(table: PhraseTable) -> Finding: + """What run-length coding inside phrase bodies would spare on the plateaus they hold (H2). + + A phrase is stored verbatim, a value resting for several ticks spelled out once per tick. + A run coded as its value and its count takes two bytes however long it rests. + + Args: + table: The dictionary. + + Returns: + Finding: The body bytes sitting in a repeat, and the bytes run-length coding would spare. + """ + finding = NOTHING + for phrase in table.phrases: + for run in runs(phrase.body): + if run >= REPEATED: + finding += Finding(run - 1, max(0, run - RLE_RUN_SIZE) if run >= MIN_RLE_RUN else 0) + + return finding + + +def default_counts( + table: PhraseTable, + tokens: Iterable[ReadToken], +) -> Finding: + """What a default count stored with each phrase would spare on the count bytes (H4). + + Every phrase token carries the ticks it plays for. A phrase played mostly for one length + could state that length once in its body, and a token playing it that long could leave the + count byte out. + + Args: + table: The dictionary. + tokens: Every token of every plane. + + Returns: + Finding: The count bytes phrase tokens carry, and the bytes a default would spare. + """ + counts: Dict[int, Counter[int]] = {phrase_id: Counter() for phrase_id in range(len(table))} + for token in tokens: + if token.phrase_id is not None: + counts[token.phrase_id][token.ticks] += 1 + + targeted = PHRASE_COUNT_SIZE * sum(sum(counter.values()) for counter in counts.values()) + saving = 0 + for counter in counts.values(): + if counter: + (_, modal), *_ = counter.most_common(1) + saving += max(0, PHRASE_COUNT_SIZE * modal - DEFAULT_COUNT_SIZE) + + return Finding(targeted, saving) diff --git a/scripts/codec_study/accounting/finding.py b/scripts/codec_study/accounting/finding.py new file mode 100644 index 000000000..4b04d4351 --- /dev/null +++ b/scripts/codec_study/accounting/finding.py @@ -0,0 +1,24 @@ +from __future__ import annotations + +from typing import Final, NamedTuple + + +class Finding(NamedTuple): + """What one hypothesis reaches in one encoding. + + Attributes: + targeted: The bytes the hypothesis acts upon, as the encoding writes them today. + saving: The bytes it would spare under the pricing the hypothesis states. + """ + + targeted: int + saving: int + + def __add__(self, other: object) -> Finding: + if not isinstance(other, Finding): + return NotImplemented + + return Finding(self.targeted + other.targeted, self.saving + other.saving) + + +NOTHING: Final[Finding] = Finding(0, 0) diff --git a/scripts/codec_study/accounting/fixed.py b/scripts/codec_study/accounting/fixed.py new file mode 100644 index 000000000..57404cc60 --- /dev/null +++ b/scripts/codec_study/accounting/fixed.py @@ -0,0 +1,63 @@ +from dataclasses import dataclass +from typing import Final + +from codec_study.accounting.finding import Finding +from codec_study.corpus.song import StudySong +from sampletones_player.compression.compressed import CompressedPlanes +from sampletones_player.specification.binary import WORD_SIZE +from sampletones_player.specification.compression import ( + PHRASE_TABLE_ENTRY_SIZE, + PLANE_COUNT, +) +from sampletones_player.specification.song import SONG_HEADER_SIZE + +TIMER_SIZE: Final[int] = WORD_SIZE + + +@dataclass(frozen=True) +class FixedOverheads: + """The bytes a song block pays before any tick is described (H7). + + Attributes: + header: The song header. + pitch_table: The pitch table as written, every pitch the format can name. + pitch_table_used: The pitch table cut at the highest pitch the song sounds. + loop_entries: The re-entry offsets written whether or not the song loops. + phrase_offsets: The table of offsets a sequential walk over the phrases could replace. + """ + + header: int + pitch_table: int + pitch_table_used: int + loop_entries: int + phrase_offsets: int + + @property + def finding(self) -> Finding: + """The fixed bytes, and what trimming the table, the entries and the offsets would spare.""" + targeted = self.header + self.pitch_table + self.phrase_offsets + saving = (self.pitch_table - self.pitch_table_used) + self.loop_entries + self.phrase_offsets + return Finding(targeted, saving) + + +def fixed_overheads( + song: StudySong, + compressed: CompressedPlanes, +) -> FixedOverheads: + """Accounts for the bytes a song block pays regardless of its length. + + Args: + song: The song, for the pitches it sounds. + compressed: The encoding, for the phrases it holds. + + Returns: + FixedOverheads: The fixed bytes, part by part. + """ + highest = max(max(channel.value) for channel in (song.planes.pulse1, song.planes.pulse2, song.planes.triangle)) + return FixedOverheads( + header=SONG_HEADER_SIZE, + pitch_table=len(song.pitches.data), + pitch_table_used=TIMER_SIZE * (highest + 1), + loop_entries=WORD_SIZE * PLANE_COUNT, + phrase_offsets=PHRASE_TABLE_ENTRY_SIZE * len(compressed.phrases), + ) diff --git a/scripts/codec_study/accounting/pairs.py b/scripts/codec_study/accounting/pairs.py new file mode 100644 index 000000000..fd9becc65 --- /dev/null +++ b/scripts/codec_study/accounting/pairs.py @@ -0,0 +1,57 @@ +from typing import Final, Sequence + +from codec_study.accounting.finding import NOTHING, Finding +from codec_study.accounting.runs import ramps +from codec_study.accounting.tokens import ReadToken +from sampletones_player.specification.compression import TokenTag + +SET_HOLD_SIZE: Final[int] = 2 +SINGLE_VALUE: Final[int] = 1 +RAMP_TOKEN_SIZE: Final[int] = 3 +MIN_RAMP_LENGTH: Final[int] = 3 + + +def set_holds(tokens: Sequence[ReadToken]) -> Finding: + """What a set-then-hold token would spare on a single value followed by a hold (H3). + + A plane that steps to a value and rests there pays a literal of one value and a hold, three + bytes. A token carrying the value and the count together takes two bytes on an opcode of its + own, which is the bound stated here; on the free escape it takes three and spares nothing. + + Args: + tokens: The tokens of one plane's stream. + + Returns: + Finding: The bytes such pairs take, and the bytes a two-byte token would spare. + """ + finding = NOTHING + for token, following in zip(tokens, tokens[1:]): + if token.tag is TokenTag.LITERAL and len(token.payload) == SINGLE_VALUE and following.tag is TokenTag.HOLD: + now = token.size + following.size + finding += Finding(now, now - SET_HOLD_SIZE) + + return finding + + +def ramps_in_literals(tokens: Sequence[ReadToken]) -> Finding: + """What a ramp token would spare on constant-step runs inside literals (H6). + + A fade or a slide spells every step out. A token naming the first value, the step and the + count takes three bytes however long the ramp runs. + + Args: + tokens: The tokens of one plane's stream. + + Returns: + Finding: The payload bytes lying in ramps, and the bytes ramp tokens would spare. + """ + finding = NOTHING + for token in tokens: + if token.tag is not TokenTag.LITERAL: + continue + + for length in ramps(token.payload): + if length >= MIN_RAMP_LENGTH: + finding += Finding(length, max(0, length - RAMP_TOKEN_SIZE)) + + return finding diff --git a/scripts/codec_study/accounting/rows.py b/scripts/codec_study/accounting/rows.py new file mode 100644 index 000000000..12ce84884 --- /dev/null +++ b/scripts/codec_study/accounting/rows.py @@ -0,0 +1,138 @@ +from dataclasses import dataclass +from typing import Dict, Final, List, Tuple + +from codec_study.accounting.coincident import coincident_starts +from codec_study.accounting.dictionary import default_counts, plateaus +from codec_study.accounting.finding import NOTHING, Finding +from codec_study.accounting.fixed import fixed_overheads +from codec_study.accounting.pairs import ramps_in_literals, set_holds +from codec_study.accounting.shares import PlaneShares, plane_shares +from codec_study.accounting.tokens import ReadToken, read_tokens +from codec_study.measure import Measurement +from sampletones_player.compression.planes.order import PlaneOrder + +HYPOTHESES: Final[Tuple[Tuple[str, str], ...]] = ( + ("H1", "hold chains"), + ("H2", "plateaus in bodies"), + ("H3", "set-then-hold"), + ("H4", "default counts"), + ("H5", "coincident starts"), + ("H6", "ramps in literals"), + ("H7", "fixed tables"), +) +LEADING_COLUMNS: Final[Tuple[str, ...]] = ( + "group", + "song", + "variant", + "block", + "dictionary", + "streams", + "hold opcodes", + "literal opcodes", + "literal payload", + "phrase tokens", + "idle planes", + "idle bytes", + "bend bytes", +) +COLUMNS: Final[Tuple[str, ...]] = ( + *LEADING_COLUMNS, + *(f"{label} {column}" for label, _ in HYPOTHESES for column in ("targeted", "saving", "share")), +) + + +@dataclass(frozen=True) +class AccountingRow: + """Where one encoding's bytes go, and what each hypothesis would reach in it. + + Attributes: + group: Which kind of song it is. + song: The song's name. + variant: The variant the encoding was built by. + block: The bytes the whole song block takes. + dictionary: The bytes the dictionary takes. + streams: The bytes the streams take together. + planes: Every plane's shares, in the order the song block writes them. + findings: What each hypothesis reaches, in the order of ``HYPOTHESES``. + """ + + group: str + song: str + variant: str + block: int + dictionary: int + streams: int + planes: Tuple[PlaneShares, ...] + findings: Tuple[Finding, ...] + + @property + def idle_planes(self) -> Tuple[PlaneShares, ...]: + """The planes keeping one value throughout the song.""" + return tuple(plane for plane in self.planes if plane.idle) + + @property + def cells(self) -> Tuple[str, ...]: + """The row as the table prints it, column by column.""" + leading = ( + self.group, + self.song, + self.variant, + f"{self.block}", + f"{self.dictionary}", + f"{self.streams}", + f"{sum(plane.hold_opcodes for plane in self.planes)}", + f"{sum(plane.literal_opcodes for plane in self.planes)}", + f"{sum(plane.literal_payload for plane in self.planes)}", + f"{sum(plane.phrase_bytes for plane in self.planes)}", + f"{len(self.idle_planes)}", + f"{sum(plane.stream for plane in self.idle_planes)}", + f"{sum(plane.stream for plane in self.planes if plane.bend)}", + ) + findings: List[str] = [] + for finding in self.findings: + findings.extend((f"{finding.targeted}", f"{finding.saving}", self.share(finding.saving))) + + return (*leading, *findings) + + def share(self, saving: int) -> str: + """``saving`` as a percentage of the whole block.""" + return f"{100.0 * saving / self.block:.1f}%" + + +def account(measurement: Measurement) -> AccountingRow: + """Reads one encoding back and states what every hypothesis would reach in it. + + Args: + measurement: The encoding. + + Returns: + AccountingRow: The shares and the findings. + """ + compressed = measurement.compressed + tokens: Dict[str, Tuple[ReadToken, ...]] = { + name: read_tokens(stream) for name, stream in zip(PlaneOrder.names(), compressed.streams) + } + planes = tuple( + plane_shares(name, plane, tokens[name]) + for name, plane in zip(PlaneOrder.names(), measurement.song.planes.planes) + ) + every_token = [token for name in PlaneOrder.names() for token in tokens[name]] + findings = ( + sum((plane.hold_chains for plane in planes), NOTHING), + plateaus(compressed.phrases), + sum((set_holds(tokens[name]) for name in PlaneOrder.names()), NOTHING), + default_counts(compressed.phrases, every_token), + coincident_starts(tokens), + sum((ramps_in_literals(tokens[name]) for name in PlaneOrder.names()), NOTHING), + fixed_overheads(measurement.song, compressed).finding, + ) + return AccountingRow( + group=measurement.song.group.value, + song=measurement.song.name, + variant=measurement.variant, + block=measurement.block, + dictionary=measurement.dictionary, + streams=measurement.streams, + planes=planes, + findings=findings, + ) diff --git a/scripts/codec_study/accounting/runs.py b/scripts/codec_study/accounting/runs.py new file mode 100644 index 000000000..d4b9200b5 --- /dev/null +++ b/scripts/codec_study/accounting/runs.py @@ -0,0 +1,32 @@ +from itertools import groupby +from typing import Tuple + +from sampletones_player.specification.binary import BYTE_VALUES + + +def runs(data: bytes) -> Tuple[int, ...]: + """The lengths of the runs of one value ``data`` is made of, in order. + + Args: + data: The bytes to read. + + Returns: + Tuple[int, ...]: One length per run; they sum to the length of ``data``. + """ + return tuple(len(list(group)) for _, group in groupby(data)) + + +def ramps(data: bytes) -> Tuple[int, ...]: + """The lengths of the constant-step runs ``data`` is made of, steps of zero left aside. + + A ramp is a series stepping by the same amount from one value to the next, a fade or a + slide, and its length counts the values it covers. + + Args: + data: The bytes to read. + + Returns: + Tuple[int, ...]: The length of every ramp of at least two values. + """ + steps = tuple((following - value) % BYTE_VALUES for value, following in zip(data, data[1:])) + return tuple(len(list(group)) + 1 for step, group in groupby(steps) if step != 0) diff --git a/scripts/codec_study/accounting/shares.py b/scripts/codec_study/accounting/shares.py new file mode 100644 index 000000000..dc514f774 --- /dev/null +++ b/scripts/codec_study/accounting/shares.py @@ -0,0 +1,116 @@ +from dataclasses import dataclass +from math import ceil +from typing import Final, List, Sequence + +from codec_study.accounting.finding import NOTHING, Finding +from codec_study.accounting.tokens import ReadToken +from sampletones_player.specification.compression import ( + MAX_HOLD_TICKS, + OPCODE_SIZE, + TokenTag, +) + +WIDE_HOLD_SIZE: Final[int] = 2 +WIDE_HOLD_UNITS: Final[int] = MAX_HOLD_TICKS +BEND_SUFFIX: Final[str] = "_bend" + + +@dataclass(frozen=True) +class PlaneShares: + """Where one plane's stream bytes go, token kind by token kind. + + Attributes: + name: The plane's name in the song block. + stream: The bytes the stream takes. + hold_opcodes: The bytes spent on hold opcodes. + literal_opcodes: The bytes spent on literal opcodes. + literal_payload: The bytes spent on the values literals carry. + phrase_bytes: The bytes spent on phrase tokens. + idle: Whether the plane keeps one value throughout the song. + bend: Whether the plane is a bend plane. + hold_chains: What a wide hold would reach (H1): runs of consecutive holds. + """ + + name: str + stream: int + hold_opcodes: int + literal_opcodes: int + literal_payload: int + phrase_bytes: int + idle: bool + bend: bool + hold_chains: Finding + + +def plane_shares( + name: str, + plane: bytes, + tokens: Sequence[ReadToken], +) -> PlaneShares: + """Accounts for one plane's stream. + + Args: + name: The plane's name in the song block. + plane: The values the plane plays. + tokens: The tokens the plane's stream was read back as. + + Returns: + PlaneShares: The bytes by token kind, and what a wide hold would reach. + """ + return PlaneShares( + name=name, + stream=sum(token.size for token in tokens), + hold_opcodes=sum(token.size for token in tokens if token.tag is TokenTag.HOLD), + literal_opcodes=sum(OPCODE_SIZE for token in tokens if token.tag is TokenTag.LITERAL), + literal_payload=sum(len(token.payload) for token in tokens), + phrase_bytes=sum(token.size for token in tokens if token.names_a_phrase), + idle=len(set(plane)) == 1, + bend=name.endswith(BEND_SUFFIX), + hold_chains=hold_chains(tokens), + ) + + +def hold_chains(tokens: Sequence[ReadToken]) -> Finding: + """What a wide hold would spare on runs of consecutive holds (H1). + + A hold covers at most ``MAX_HOLD_TICKS``, so a plane resting longer pays an opcode per that + many ticks. A wide hold on the free escape takes two bytes and counts full holds instead of + ticks, so a chain is priced as the wide holds its full holds need and one plain hold for + the ticks left over. + + Args: + tokens: The tokens of one plane's stream. + + Returns: + Finding: The hold opcodes in chains, and the bytes wide holds would spare on them. + """ + finding = NOTHING + for chain in _chains(tokens): + full, rest = divmod(sum(token.ticks for token in chain), MAX_HOLD_TICKS) + if full == 0: + continue + + now = OPCODE_SIZE * len(chain) + wide = WIDE_HOLD_SIZE * ceil(full / WIDE_HOLD_UNITS) + (OPCODE_SIZE if rest else 0) + finding += Finding(now, max(0, now - wide)) + + return finding + + +def _chains(tokens: Sequence[ReadToken]) -> List[List[ReadToken]]: + chains: List[List[ReadToken]] = [] + current: List[ReadToken] = [] + for token in tokens: + if token.tag is TokenTag.HOLD: + current.append(token) + continue + + if len(current) > 1: + chains.append(current) + + current = [] + + if len(current) > 1: + chains.append(current) + + return chains diff --git a/scripts/codec_study/accounting/tokens.py b/scripts/codec_study/accounting/tokens.py new file mode 100644 index 000000000..73deed04f --- /dev/null +++ b/scripts/codec_study/accounting/tokens.py @@ -0,0 +1,105 @@ +from typing import List, NamedTuple, Optional, Tuple + +from sampletones_player.compression.tokens.span import token_span +from sampletones_player.specification.compression import ( + OPCODE_SIZE, + PHRASE_COUNT_SIZE, + PHRASE_ESCAPE_SIZE, + PHRASE_ID_ESCAPE, + TOKEN_OPERAND_MASK, + TOKEN_TAG_MASK, + TokenTag, +) + + +class ReadToken(NamedTuple): + """One token read back from a written stream, with everything the accounting asks of it. + + Attributes: + tag: What kind of token it is. + tick: The tick the token starts on. + size: The bytes the token takes. + ticks: The ticks the token covers. + payload: The values a literal carries, empty for every other kind. + phrase_id: The phrase a phrase token names, ``None`` for every other kind. + transpose: The shift a transposed phrase is played at, zero otherwise. + """ + + tag: TokenTag + tick: int + size: int + ticks: int + payload: bytes + phrase_id: Optional[int] + transpose: int + + @property + def names_a_phrase(self) -> bool: + """Whether the token plays a phrase from the dictionary.""" + return self.phrase_id is not None + + +def _phrase_operands( + stream: bytes, + position: int, + operand: int, + *, + transposed: bool, +) -> Tuple[int, int]: + after = position + OPCODE_SIZE + phrase_id = operand + if operand == PHRASE_ID_ESCAPE: + phrase_id = stream[after] + after += PHRASE_ESCAPE_SIZE + + transpose = stream[after + PHRASE_COUNT_SIZE] if transposed else 0 + return phrase_id, transpose + + +def read_tokens(stream: bytes) -> Tuple[ReadToken, ...]: + """Reads a plane's stream back as the tokens it was written from. + + Args: + stream: The plane's token stream. + + Returns: + Tuple[ReadToken, ...]: The tokens in the order they are read. + """ + tokens: List[ReadToken] = [] + position = 0 + tick = 0 + while position < len(stream): + span = token_span(stream, position) + tag = TokenTag(stream[position] & TOKEN_TAG_MASK) + operand = stream[position] & TOKEN_OPERAND_MASK + payload = b"" + phrase_id: Optional[int] = None + transpose = 0 + match tag: + case TokenTag.LITERAL: + payload = stream[position + OPCODE_SIZE : position + span.size] + case TokenTag.PHRASE | TokenTag.TRANSPOSED_PHRASE: + phrase_id, transpose = _phrase_operands( + stream, + position, + operand, + transposed=tag is TokenTag.TRANSPOSED_PHRASE, + ) + case TokenTag.HOLD: + pass + + tokens.append( + ReadToken( + tag=tag, + tick=tick, + size=span.size, + ticks=span.ticks, + payload=payload, + phrase_id=phrase_id, + transpose=transpose, + ) + ) + position += span.size + tick += span.ticks + + return tuple(tokens) diff --git a/scripts/codec_study/corpus/__init__.py b/scripts/codec_study/corpus/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/scripts/codec_study/corpus/build.py b/scripts/codec_study/corpus/build.py new file mode 100644 index 000000000..cf7220479 --- /dev/null +++ b/scripts/codec_study/corpus/build.py @@ -0,0 +1,43 @@ +from typing import List, Tuple + +from codec_study.corpus.projects import lengthened_song, project_song +from codec_study.corpus.reconstructions import reconstruction_paths, reconstruction_song +from codec_study.corpus.song import StudySong +from codec_study.manifest import StudyManifest, StudySource +from sampletones_shared.logger import logger + + +def build_corpus(manifest: StudyManifest) -> Tuple[StudySong, ...]: + """Reads every song the manifest names. + + Args: + manifest: What the run reads. + + Returns: + Tuple[StudySong, ...]: The projects as they stand, then each one lengthened, then every + stem in manifest order. + """ + songs: List[StudySong] = [] + for source in manifest.projects: + logger.info(f"Reading project {source.path}") + songs.append(project_song(source.path)) + + for source in manifest.projects: + logger.info(f"Lengthening project {source.path} to {manifest.lengthen_seconds} s") + songs.append(lengthened_song(source.path, manifest.lengthen_seconds)) + + for source in manifest.reconstructions: + songs.extend(_reconstructions(source)) + + return tuple(songs) + + +def _reconstructions(source: StudySource) -> List[StudySong]: + paths = reconstruction_paths(source.path) + songs: List[StudySong] = [] + for path in paths: + logger.info(f"Reading stem {path}") + name = source.label if len(paths) == 1 else f"{source.label}/{path.stem}" + songs.append(reconstruction_song(name, path)) + + return songs diff --git a/scripts/codec_study/corpus/projects.py b/scripts/codec_study/corpus/projects.py new file mode 100644 index 000000000..0419ff070 --- /dev/null +++ b/scripts/codec_study/corpus/projects.py @@ -0,0 +1,94 @@ +from math import ceil +from pathlib import Path + +from codec_study.corpus.song import SongGroup, StudySong +from sampletones_core.performance import song_instructions +from sampletones_core.project.container import ProjectContainer +from sampletones_core.project.project import Project +from sampletones_core.project.tuning import tuning_from_project +from sampletones_core.timers.utils import get_timer_table +from sampletones_core.timing import SongTiming +from sampletones_player.builder import streams_from_instructions +from sampletones_player.compression.pitch import PitchTable +from sampletones_player.compression.planes.separate import planes_from_streams +from sampletones_player.compression.seeds import phrases_from_project +from sampletones_shared.utils.progress import silent_reporter + + +def project_song(path: Path) -> StudySong: + """Reads a project file as the song its arrangement plays. + + Args: + path: The project file. + + Returns: + StudySong: The song, seeded with the phrases the project's instruments offer. + """ + return _song(path.stem, SongGroup.PROJECT, path, ProjectContainer.load(path)) + + +def lengthened_song( + path: Path, + seconds: int, +) -> StudySong: + """Reads a project file as its arrangement repeated until the song lasts ``seconds``. + + A project on disk is a few seconds of patterns, and an export is measured against a song of + minutes, so the whole order is played through as many times as that takes. + + Args: + path: The project file. + seconds: How long the song is to last. + + Returns: + StudySong: The song, seeded with the phrases the project's instruments offer. + """ + project = ProjectContainer.load(path) + return _song( + f"{path.stem} ({seconds} s)", + SongGroup.LONG_PROJECT, + path, + _repeated(project, seconds), + ) + + +def _song( + name: str, + group: SongGroup, + source: Path, + project: Project, +) -> StudySong: + tuning = tuning_from_project(project) + pitches = PitchTable.from_tuning(tuning) + streams = streams_from_instructions( + song_instructions(project, silent_reporter), + get_timer_table(tuning), + ) + return StudySong( + name=name, + group=group, + source=source, + planes=planes_from_streams(streams, pitches), + seeds=phrases_from_project(project, tuning), + pitches=pitches, + ) + + +def _repeated( + project: Project, + seconds: int, +) -> Project: + groove = SongTiming.from_project(project).groove() + repetitions = max(1, ceil(seconds * project.settings.nes_frequency / groove.total_ticks)) + longer = Project.create( + rows_per_pattern=project.song.rows_per_pattern, + settings=project.settings, + ) + for voice in project.voices: + longer.voices.append(voice) + + longer.song = project.song.model_copy(deep=True) + for _ in range(repetitions - 1): + longer.song.order.extend(dict(frame) for frame in project.song.order) + + return longer diff --git a/scripts/codec_study/corpus/reconstructions.py b/scripts/codec_study/corpus/reconstructions.py new file mode 100644 index 000000000..79d171c37 --- /dev/null +++ b/scripts/codec_study/corpus/reconstructions.py @@ -0,0 +1,61 @@ +from pathlib import Path +from typing import Final, Tuple + +from codec_study.corpus.song import SongGroup, StudySong +from sampletones_core.reconstructions import Reconstruction +from sampletones_core.timers.utils import get_timer_table +from sampletones_player.builder import streams_from_instructions +from sampletones_player.compression.dictionary.phrase import Phrase +from sampletones_player.compression.pitch import PitchTable +from sampletones_player.compression.planes.separate import planes_from_streams + +STEM_SUFFIX: Final[str] = ".stn" +NO_SEEDS: Final[Tuple[Phrase, ...]] = () + + +def reconstruction_paths(path: Path) -> Tuple[Path, ...]: + """The stem files ``path`` names: the file itself, or every stem under a directory. + + Args: + path: A stem file or a directory holding stems. + + Returns: + Tuple[Path, ...]: The stem files, a directory's in sorted order. + """ + if path.is_dir(): + return tuple(sorted(path.rglob(f"*{STEM_SUFFIX}"))) + + return (path,) + + +def reconstruction_song( + name: str, + path: Path, +) -> StudySong: + """Reads a stem file as the song the console plays it as. + + A reconstruction sounds each of its slices once, so the song offers the dictionary nothing + and the search fills it from what the streams themselves repeat. + + Args: + name: What the song is called in a report. + path: The stem file. + + Returns: + StudySong: The song, at the tuning and rate the stem was reconstructed at. + """ + reconstruction = Reconstruction.load(path) + tuning = reconstruction.config.tuning + pitches = PitchTable.from_tuning(tuning) + streams = streams_from_instructions( + reconstruction.instructions, + get_timer_table(tuning), + ) + return StudySong( + name=name, + group=SongGroup.RECONSTRUCTION, + source=path, + planes=planes_from_streams(streams, pitches), + seeds=NO_SEEDS, + pitches=pitches, + ) diff --git a/scripts/codec_study/corpus/song.py b/scripts/codec_study/corpus/song.py new file mode 100644 index 000000000..473655eef --- /dev/null +++ b/scripts/codec_study/corpus/song.py @@ -0,0 +1,49 @@ +from dataclasses import dataclass +from enum import StrEnum +from pathlib import Path +from typing import Tuple + +from sampletones_player.compression.dictionary.phrase import Phrase +from sampletones_player.compression.pitch import PitchTable +from sampletones_player.compression.planes.song import SongPlanes + + +class SongGroup(StrEnum): + """Which kind of song a corpus entry is, which is what the report aggregates over. + + Attributes: + PROJECT: A project as it stands on disk, a few seconds of patterns. + LONG_PROJECT: A project with its order repeated to the length an export is measured at. + RECONSTRUCTION: A stem reconstructed from audio, its planes turning over at nearly + every tick. + """ + + PROJECT = "project" + LONG_PROJECT = "project-long" + RECONSTRUCTION = "reconstruction" + + +@dataclass(frozen=True) +class StudySong: + """One song the study measures the codec on. + + Attributes: + name: What the song is called in a report. + group: Which kind of song it is. + source: The file the song was read from. + planes: The planes the codec compresses. + seeds: The phrases the song's instruments offer the dictionary. + pitches: The timer each pitch of the song sounds at. + """ + + name: str + group: SongGroup + source: Path + planes: SongPlanes + seeds: Tuple[Phrase, ...] + pitches: PitchTable + + @property + def ticks(self) -> int: + """The ticks the song lasts.""" + return self.planes.ticks diff --git a/scripts/codec_study/manifest.py b/scripts/codec_study/manifest.py new file mode 100644 index 000000000..03bb76017 --- /dev/null +++ b/scripts/codec_study/manifest.py @@ -0,0 +1,125 @@ +from __future__ import annotations + +from pathlib import Path +from typing import Final, Tuple + +from pydantic import BaseModel, ConfigDict, Field + +from sampletones_shared.paths.user import PROJECTS_DIRECTORY, RECONSTRUCTIONS_DIRECTORY + +PROJECT_SUFFIX: Final[str] = ".stp" +DEFAULT_PROJECT_NAMES: Final[Tuple[str, ...]] = ("Amen", "Demo", "Tempo", "Test") +QUICK_PROJECT_NAMES: Final[Tuple[str, ...]] = ("Test",) +LEAD_VOCALS: Final[str] = "lead-vocals" +DEFAULT_RECONSTRUCTIONS: Final[Tuple[Tuple[str, str], ...]] = ( + ( + LEAD_VOCALS, + "sr_44100_nf_60_sm_cqt_tg_100_gn_p_ch_7376fe40aec68c41696a37cbf89baa55/0 Lead Vocals.stn", + ), + ( + "payoff-cqt", + "sr_44100_nf_60_sm_cqt_tg_100_gn_PTN_ch_124286f3189248af752bf09c37323e5e/Payoff [Revenge A] Stems (123 BPM)", + ), + ( + "payoff-fft", + "sr_44100_nf_60_sm_fft_tg_0_gn_PTN_ch_6d3be4658f9804463a01f2b34e3f0d16/Payoff [Revenge A] Stems (123 BPM)", + ), +) + + +class StudySource(BaseModel): + """One file or directory the corpus is read from. + + Attributes: + label: What songs read from the source are called in a report. + path: The file, or the directory whose stems are read. + """ + + model_config = ConfigDict(extra="forbid", frozen=True) + + label: str + path: Path + + @classmethod + def at(cls, path: Path) -> StudySource: + """A source labeled by its own name: a file's stem, or a directory's name. + + Args: + path: The file or directory. + + Returns: + StudySource: The source under that label. + """ + return cls(label=path.name if path.is_dir() else path.stem, path=path) + + +class StudyManifest(BaseModel): + """What one run of the study reads and measures. + + Attributes: + projects: The project files, each measured as it stands and at the lengthened duration. + reconstructions: The stem files and directories of stems. + lengthen_seconds: How long each project's lengthened copy lasts. + variants: The names of the variants every song is encoded under. + """ + + model_config = ConfigDict(extra="forbid", frozen=True) + + projects: Tuple[StudySource, ...] + reconstructions: Tuple[StudySource, ...] + lengthen_seconds: int = Field(ge=1) + variants: Tuple[str, ...] = Field(min_length=1) + + @classmethod + def load(cls, path: Path) -> StudyManifest: + """Reads a manifest a run wrote, or one written by hand. + + Args: + path: The manifest file, as JSON. + + Returns: + StudyManifest: The manifest. + """ + return cls.model_validate_json(path.read_text(encoding="utf-8")) + + def save(self, path: Path) -> None: + """Writes the manifest beside a run's report, so the run can be repeated. + + Args: + path: Where the manifest is written, as JSON. + """ + path.write_text(self.model_dump_json(indent=2), encoding="utf-8") + + @classmethod + def default( + cls, + *, + lengthen_seconds: int, + variants: Tuple[str, ...], + quick: bool, + ) -> StudyManifest: + """The corpus on this machine: the projects and stems the study is settled on. + + Args: + lengthen_seconds: How long each project's lengthened copy lasts. + variants: The names of the variants every song is encoded under. + quick: Whether to read one small project and one stem, for a run that checks the + harness rather than the codec. + + Returns: + StudyManifest: The manifest. + """ + project_names = QUICK_PROJECT_NAMES if quick else DEFAULT_PROJECT_NAMES + reconstructions = tuple( + StudySource(label=label, path=RECONSTRUCTIONS_DIRECTORY / relative) + for label, relative in DEFAULT_RECONSTRUCTIONS + if label == LEAD_VOCALS or not quick + ) + return cls( + projects=tuple( + StudySource(label=name, path=PROJECTS_DIRECTORY / f"{name}{PROJECT_SUFFIX}") for name in project_names + ), + reconstructions=reconstructions, + lengthen_seconds=lengthen_seconds, + variants=variants, + ) diff --git a/scripts/codec_study/measure.py b/scripts/codec_study/measure.py new file mode 100644 index 000000000..7f67775e0 --- /dev/null +++ b/scripts/codec_study/measure.py @@ -0,0 +1,76 @@ +from dataclasses import dataclass +from time import process_time +from typing import Callable + +from codec_study.corpus.song import StudySong +from sampletones_player.compression.compressed import CompressedPlanes +from sampletones_player.compression.decode import decode_planes +from sampletones_player.specification.song import SONG_HEADER_SIZE + +Encoder = Callable[[StudySong], CompressedPlanes] + + +@dataclass(frozen=True) +class Measurement: + """One song encoded under one variant, with what the encoding cost and whether it plays back. + + Attributes: + song: The song encoded. + variant: The name of the variant the encoding was built by. + compressed: The dictionary and the token streams the variant wrote. + seconds: The processor time the encoding took. + lossless: Whether the streams play back to the planes they were written from. + """ + + song: StudySong + variant: str + compressed: CompressedPlanes + seconds: float + lossless: bool + + @property + def block(self) -> int: + """The bytes the whole song block takes: header, pitch table, dictionary and streams.""" + return SONG_HEADER_SIZE + len(self.song.pitches.data) + self.compressed.size + + @property + def dictionary(self) -> int: + """The bytes the dictionary takes.""" + return self.compressed.phrases.size + + @property + def streams(self) -> int: + """The bytes the token streams take together.""" + return sum(len(stream) for stream in self.compressed.streams) + + @property + def bytes_per_tick(self) -> float: + """The bytes each tick of the song costs, the whole block counted.""" + return self.block / self.song.ticks + + +def measure( + song: StudySong, + variant: str, + encode: Encoder, +) -> Measurement: + """Encodes a song under a variant, timing the encoding and playing it back. + + Args: + song: The song to encode. + variant: The name of the variant. + encode: What the variant writes the song as. + + Returns: + Measurement: The encoding, its cost and whether it plays back. + """ + started = process_time() + compressed = encode(song) + seconds = process_time() - started + return Measurement( + song=song, + variant=variant, + compressed=compressed, + seconds=seconds, + lossless=decode_planes(compressed) == song.planes, + ) diff --git a/scripts/codec_study/report/__init__.py b/scripts/codec_study/report/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/scripts/codec_study/report/aggregate.py b/scripts/codec_study/report/aggregate.py new file mode 100644 index 000000000..4026d9734 --- /dev/null +++ b/scripts/codec_study/report/aggregate.py @@ -0,0 +1,112 @@ +from dataclasses import dataclass +from typing import Dict, Final, List, Sequence, Tuple + +from codec_study.corpus.song import SongGroup +from codec_study.measure import Measurement + +COLUMNS: Final[Tuple[str, ...]] = ( + "group", + "variant", + "songs", + "ticks", + "block", + "bytes per tick", + "change", + "worst song", + "seconds", +) + + +@dataclass(frozen=True) +class GroupRow: + """One kind of song under one variant, summed over the songs of that kind. + + Attributes: + group: Which kind of song it is. + variant: The variant the encodings were built by. + songs: How many songs the group holds. + ticks: The ticks the songs last together. + block: The bytes the song blocks take together. + baseline: The bytes the same songs take under the baseline. + worst: The largest growth any one song shows against its baseline, as a ratio. + seconds: The processor time the encodings took together. + """ + + group: str + variant: str + songs: int + ticks: int + block: int + baseline: int + worst: float + seconds: float + + @property + def change(self) -> float: + """How the group's bytes stand against the baseline, as a ratio; negative is smaller.""" + return self.block / self.baseline - 1.0 + + @property + def cells(self) -> Tuple[str, ...]: + """The row as the table prints it, column by column.""" + return ( + self.group, + self.variant, + f"{self.songs}", + f"{self.ticks}", + f"{self.block}", + f"{self.block / self.ticks:.3f}", + f"{100.0 * self.change:+.1f}%", + f"{100.0 * self.worst:+.1f}%", + f"{self.seconds:.2f}", + ) + + +def group_rows( + measurements: Sequence[Measurement], + baseline: str, +) -> Tuple[GroupRow, ...]: + """Sums the measurements over each kind of song, variant by variant. + + Args: + measurements: Every song under every variant. + baseline: The name of the variant the others are held against. + + Returns: + Tuple[GroupRow, ...]: One row per kind of song and variant, kinds in their declared + order and variants in the order they were measured. + """ + reference: Dict[str, int] = { + measurement.song.name: measurement.block for measurement in measurements if measurement.variant == baseline + } + variants = list(dict.fromkeys(measurement.variant for measurement in measurements)) + rows: List[GroupRow] = [] + for group in SongGroup: + for variant in variants: + chosen = [ + measurement + for measurement in measurements + if measurement.song.group is group and measurement.variant == variant + ] + if chosen: + rows.append(_group_row(group, variant, chosen, reference)) + + return tuple(rows) + + +def _group_row( + group: SongGroup, + variant: str, + chosen: Sequence[Measurement], + reference: Dict[str, int], +) -> GroupRow: + return GroupRow( + group=group.value, + variant=variant, + songs=len(chosen), + ticks=sum(measurement.song.ticks for measurement in chosen), + block=sum(measurement.block for measurement in chosen), + baseline=sum(reference[measurement.song.name] for measurement in chosen), + worst=max(measurement.block / reference[measurement.song.name] - 1.0 for measurement in chosen), + seconds=sum(measurement.seconds for measurement in chosen), + ) diff --git a/scripts/codec_study/report/rows.py b/scripts/codec_study/report/rows.py new file mode 100644 index 000000000..ac3a6c365 --- /dev/null +++ b/scripts/codec_study/report/rows.py @@ -0,0 +1,87 @@ +from dataclasses import dataclass +from typing import Final, Tuple + +from codec_study.measure import Measurement + +COLUMNS: Final[Tuple[str, ...]] = ( + "group", + "song", + "variant", + "ticks", + "block", + "bytes per tick", + "dictionary", + "streams", + "phrases", + "seconds", + "lossless", +) + + +@dataclass(frozen=True) +class StudyRow: + """One song measured under one variant, as the report prints it. + + Attributes: + group: Which kind of song it is. + song: The song's name. + variant: The variant the encoding was built by. + ticks: The ticks the song lasts. + block: The bytes the whole song block takes. + dictionary: The bytes the dictionary takes. + streams: The bytes the streams take together. + phrases: The phrases the dictionary holds. + seconds: The processor time the encoding took. + lossless: Whether the encoding plays back to its planes. + """ + + group: str + song: str + variant: str + ticks: int + block: int + dictionary: int + streams: int + phrases: int + seconds: float + lossless: bool + + @property + def cells(self) -> Tuple[str, ...]: + """The row as the table prints it, column by column.""" + return ( + self.group, + self.song, + self.variant, + f"{self.ticks}", + f"{self.block}", + f"{self.block / self.ticks:.3f}", + f"{self.dictionary}", + f"{self.streams}", + f"{self.phrases}", + f"{self.seconds:.2f}", + "yes" if self.lossless else "no", + ) + + +def study_row(measurement: Measurement) -> StudyRow: + """The report row of one measurement. + + Args: + measurement: The encoding and its cost. + + Returns: + StudyRow: The row. + """ + return StudyRow( + group=measurement.song.group.value, + song=measurement.song.name, + variant=measurement.variant, + ticks=measurement.song.ticks, + block=measurement.block, + dictionary=measurement.dictionary, + streams=measurement.streams, + phrases=len(measurement.compressed.phrases), + seconds=measurement.seconds, + lossless=measurement.lossless, + ) diff --git a/scripts/codec_study/report/run.py b/scripts/codec_study/report/run.py new file mode 100644 index 000000000..6a78edbee --- /dev/null +++ b/scripts/codec_study/report/run.py @@ -0,0 +1,114 @@ +import subprocess +from datetime import UTC, datetime +from pathlib import Path +from typing import Final, List, Optional, Sequence + +from codec_study.accounting import rows as accounting +from codec_study.manifest import StudyManifest +from codec_study.measure import Measurement +from codec_study.report import aggregate +from codec_study.report import rows as songs +from codec_study.report.writers import markdown_table, write_csv +from codec_study.variants.production import BASELINE +from codec_study.variants.variant import Variant +from sampletones_shared.paths.source import REPOSITORY_ROOT +from sampletones_shared.paths.user import USER_PATH_DOCUMENTS + +DEFAULT_OUTPUT_ROOT: Final[Path] = USER_PATH_DOCUMENTS / "compression" +RUN_STAMP: Final[str] = "run-%Y%m%d-%H%M%S" +REPORT_CSV: Final[str] = "report.csv" +REPORT_MARKDOWN: Final[str] = "report.md" +ACCOUNTING_CSV: Final[str] = "accounting.csv" +MANIFEST_JSON: Final[str] = "manifest.json" +UNKNOWN_COMMIT: Final[str] = "unknown" + + +def run_directory(output: Optional[Path]) -> Path: + """The directory a run writes into, created where it is missing. + + Args: + output: The directory asked for, or ``None`` for a stamped one under the documents. + + Returns: + Path: The directory. + """ + directory = output or DEFAULT_OUTPUT_ROOT / datetime.now(UTC).strftime(RUN_STAMP) + directory.mkdir(parents=True, exist_ok=True) + return directory + + +def commit_hash() -> str: + """The short hash of the commit the repository stands at, or a marker where git answers nothing.""" + completed = subprocess.run( + ["git", "rev-parse", "--short", "HEAD"], + capture_output=True, + text=True, + check=False, + cwd=REPOSITORY_ROOT, + ) + return completed.stdout.strip() or UNKNOWN_COMMIT + + +def write_run( + directory: Path, + manifest: StudyManifest, + variants: Sequence[Variant], + measurements: Sequence[Measurement], +) -> None: + """Writes a run's report, its accounting and the manifest that reproduces it. + + Args: + directory: The run's directory. + manifest: What the run read. + variants: The variants every song was encoded under. + measurements: Every song under every variant, in the order measured. + """ + song_rows = [songs.study_row(measurement) for measurement in measurements] + group_rows = aggregate.group_rows(measurements, BASELINE.name) + accounting_rows = [accounting.account(measurement) for measurement in measurements] + manifest.save(directory / MANIFEST_JSON) + write_csv(directory / REPORT_CSV, songs.COLUMNS, [row.cells for row in song_rows]) + write_csv(directory / ACCOUNTING_CSV, accounting.COLUMNS, [row.cells for row in accounting_rows]) + lines = _header(manifest, variants) + lines.extend(("## Groups", "", *markdown_table(aggregate.COLUMNS, [row.cells for row in group_rows]), "")) + lines.extend(("## Songs", "", *markdown_table(songs.COLUMNS, [row.cells for row in song_rows]), "")) + lines.extend(("## Accounting", "", *_accounting_table(accounting_rows), "")) + (directory / REPORT_MARKDOWN).write_text("\n".join(lines), encoding="utf-8") + + +def _header( + manifest: StudyManifest, + variants: Sequence[Variant], +) -> List[str]: + named = ", ".join(f"{variant.name} ({variant.kind.value})" for variant in variants) + return [ + "# Compression study", + "", + f"Commit: {commit_hash()}", + f"Date: {datetime.now(UTC).isoformat(timespec='seconds')}", + f"Manifest: {MANIFEST_JSON}, {len(manifest.projects)} projects lengthened to " + f"{manifest.lengthen_seconds} s, {len(manifest.reconstructions)} reconstruction sources", + f"Variants: {named}", + "", + "Each accounting share is the saving a hypothesis would reach, as a share of the whole song block.", + "", + ] + + +def _accounting_table(rows: Sequence[accounting.AccountingRow]) -> List[str]: + columns = ("group", "song", "variant", "block", "dictionary", "idle bytes", "bend bytes") + labels = tuple(f"{label} {name}" for label, name in accounting.HYPOTHESES) + cells = [ + ( + row.group, + row.song, + row.variant, + f"{row.block}", + row.share(row.dictionary), + row.share(sum(plane.stream for plane in row.idle_planes)), + row.share(sum(plane.stream for plane in row.planes if plane.bend)), + *(row.share(finding.saving) for finding in row.findings), + ) + for row in rows + ] + return markdown_table((*columns, *labels), cells) diff --git a/scripts/codec_study/report/writers.py b/scripts/codec_study/report/writers.py new file mode 100644 index 000000000..cc4243512 --- /dev/null +++ b/scripts/codec_study/report/writers.py @@ -0,0 +1,42 @@ +import csv +from pathlib import Path +from typing import List, Sequence + + +def write_csv( + path: Path, + columns: Sequence[str], + rows: Sequence[Sequence[str]], +) -> None: + """Writes a table another tool reads. + + Args: + path: Where the table is written. + columns: The header row. + rows: The rows, each as its cells. + """ + with path.open("w", encoding="utf-8", newline="") as handle: + writer = csv.writer(handle) + writer.writerow(columns) + writer.writerows(rows) + + +def markdown_table( + columns: Sequence[str], + rows: Sequence[Sequence[str]], +) -> List[str]: + """A table a reader reads, as the lines of a markdown document. + + Args: + columns: The header row. + rows: The rows, each as its cells. + + Returns: + List[str]: The header, the rule and one line per row. + """ + lines = [ + "| " + " | ".join(columns) + " |", + "|" + "|".join("---" for _ in columns) + "|", + ] + lines.extend("| " + " | ".join(row) + " |" for row in rows) + return lines diff --git a/scripts/codec_study/variants/__init__.py b/scripts/codec_study/variants/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/scripts/codec_study/variants/production.py b/scripts/codec_study/variants/production.py new file mode 100644 index 000000000..d1413d9fa --- /dev/null +++ b/scripts/codec_study/variants/production.py @@ -0,0 +1,36 @@ +from typing import Final + +from codec_study.corpus.song import StudySong +from codec_study.variants.variant import Variant, VariantKind +from sampletones_player.compression.compressed import CompressedPlanes +from sampletones_player.compression.encode import encode_planes +from sampletones_player.compression.options import EVERY_LAYER + +BASELINE_NAME: Final[str] = "baseline" +NO_LOOP_BOUNDARIES: Final[frozenset[int]] = frozenset() + + +def encode_baseline(song: StudySong) -> CompressedPlanes: + """Encodes a song as the export does, every layer on and the search at its default budget. + + Args: + song: The song to encode. + + Returns: + CompressedPlanes: The dictionary and the token streams. + """ + return encode_planes( + song.planes, + song.seeds, + options=EVERY_LAYER, + boundaries=NO_LOOP_BOUNDARIES, + ) + + +BASELINE: Final[Variant] = Variant( + name=BASELINE_NAME, + hypothesis="", + kind=VariantKind.BASELINE, + note="", + encode=encode_baseline, +) diff --git a/scripts/codec_study/variants/registry.py b/scripts/codec_study/variants/registry.py new file mode 100644 index 000000000..ade9040fc --- /dev/null +++ b/scripts/codec_study/variants/registry.py @@ -0,0 +1,26 @@ +from typing import Dict, Final, Sequence, Tuple + +from codec_study.variants.production import BASELINE +from codec_study.variants.variant import Variant + +VARIANTS: Final[Dict[str, Variant]] = {BASELINE.name: BASELINE} + + +def selected_variants(names: Sequence[str]) -> Tuple[Variant, ...]: + """The variants a run encodes every song under, the baseline always first. + + Args: + names: The names the run asks for. + + Returns: + Tuple[Variant, ...]: The baseline, then the named variants in the order given. + + Raises: + KeyError: If a name is registered to no variant. + """ + unknown = [name for name in names if name not in VARIANTS] + if unknown: + raise KeyError(f"no variant is called {', '.join(unknown)}; the registry holds {', '.join(VARIANTS)}") + + chosen = [VARIANTS[name] for name in names if name != BASELINE.name] + return (BASELINE, *chosen) diff --git a/scripts/codec_study/variants/variant.py b/scripts/codec_study/variants/variant.py new file mode 100644 index 000000000..81e743b3f --- /dev/null +++ b/scripts/codec_study/variants/variant.py @@ -0,0 +1,37 @@ +from dataclasses import dataclass +from enum import StrEnum + +from codec_study.measure import Encoder + + +class VariantKind(StrEnum): + """What a variant changes to earn its bytes. + + Attributes: + BASELINE: The production codec as it stands. + ENCODER: A change to how the encoder chooses, readable by the driver as it stands. + FORMAT: A change to the token grammar, which the driver has to learn. + """ + + BASELINE = "baseline" + ENCODER = "encoder" + FORMAT = "format" + + +@dataclass(frozen=True) +class Variant: + """One way of encoding a song the study prices against the baseline. + + Attributes: + name: What the variant is called in a report. + hypothesis: The hypothesis the variant measures, by its label in the plan. + kind: What the variant changes. + note: What the driver would have to do, where the variant changes the format. + encode: What the variant writes a song as. + """ + + name: str + hypothesis: str + kind: VariantKind + note: str + encode: Encoder diff --git a/scripts/compression_study.py b/scripts/compression_study.py new file mode 100644 index 000000000..b474cd544 --- /dev/null +++ b/scripts/compression_study.py @@ -0,0 +1,129 @@ +import argparse +from pathlib import Path +from typing import Final, List, Optional, Tuple + +from codec_study.corpus.build import build_corpus +from codec_study.manifest import StudyManifest, StudySource +from codec_study.measure import Measurement, measure +from codec_study.report.run import run_directory, write_run +from codec_study.variants.registry import selected_variants +from sampletones_shared.logger import logger + +DEFAULT_LENGTHEN_SECONDS: Final[int] = 180 +DEFAULT_VARIANTS: Final[str] = "baseline" + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Measure the song codec over the projects and stems on this machine.", + ) + parser.add_argument( + "--manifest", + type=Path, + default=None, + help="A manifest a run wrote; its lengthening and variants stand in for the options below.", + ) + parser.add_argument( + "--project", + type=Path, + action="append", + default=[], + help="A project file to measure, in place of the default corpus; repeatable.", + ) + parser.add_argument( + "--reconstruction", + type=Path, + action="append", + default=[], + help="A stem file, or a directory of stems, in place of the default corpus; repeatable.", + ) + parser.add_argument( + "--output", + type=Path, + default=None, + help="Output directory of the run.", + ) + parser.add_argument( + "--lengthen", + type=int, + default=DEFAULT_LENGTHEN_SECONDS, + help="Seconds each project's lengthened copy lasts.", + ) + parser.add_argument( + "--variants", + type=str, + default=DEFAULT_VARIANTS, + help="Comma-separated variants every song is encoded under; the baseline always runs.", + ) + parser.add_argument( + "--quick", + action="store_true", + help="Read one small project and one stem, to check the harness.", + ) + arguments = parser.parse_args() + + manifest = _manifest( + arguments.manifest, + projects=arguments.project, + reconstructions=arguments.reconstruction, + lengthen_seconds=arguments.lengthen, + variants=_names(arguments.variants), + quick=arguments.quick, + ) + variants = selected_variants(manifest.variants) + directory = run_directory(arguments.output) + corpus = build_corpus(manifest) + + measurements: List[Measurement] = [] + for song in corpus: + for variant in variants: + logger.info(f"Encoding {song.name} ({song.ticks} ticks) under {variant.name}") + measurement = measure(song, variant.name, variant.encode) + if not measurement.lossless: + raise ValueError(f"{variant.name} wrote {song.name} as streams that play back differently") + + logger.info( + f" {measurement.block} bytes, {measurement.bytes_per_tick:.3f} bytes per tick, " + f"{len(measurement.compressed.phrases)} phrases, {measurement.seconds:.1f} s" + ) + measurements.append(measurement) + + write_run(directory, manifest, variants, measurements) + logger.info(f"Report written to {directory}") + + +def _names(variants: str) -> Tuple[str, ...]: + return tuple(name.strip() for name in variants.split(",") if name.strip()) + + +def _manifest( + path: Optional[Path], + *, + projects: List[Path], + reconstructions: List[Path], + lengthen_seconds: int, + variants: Tuple[str, ...], + quick: bool, +) -> StudyManifest: + manifest = ( + StudyManifest.load(path) + if path is not None + else StudyManifest.default( + lengthen_seconds=lengthen_seconds, + variants=variants, + quick=quick, + ) + ) + if not projects and not reconstructions: + return manifest + + return StudyManifest( + projects=tuple(StudySource.at(project) for project in projects), + reconstructions=tuple(StudySource.at(reconstruction) for reconstruction in reconstructions), + lengthen_seconds=manifest.lengthen_seconds, + variants=manifest.variants, + ) + + +if __name__ == "__main__": + main() diff --git a/src/sampletones_config/boundaries/tokens.yaml b/src/sampletones_config/boundaries/tokens.yaml index f144f828a..8d089e732 100644 --- a/src/sampletones_config/boundaries/tokens.yaml +++ b/src/sampletones_config/boundaries/tokens.yaml @@ -19,3 +19,24 @@ ui/panels must not bind a structural depth theme (TAG_GLOBAL_THEME_PANEL_SURFACE/GROUND); only the layout primitives own depth (TabColumns binds the column, card() binds the card), and a panel binds only semantic themes + +- root: sampletones_core + pattern: "**/*.py" + forbidden: '\bcodec_study\b' + message: >- + sampletones_core is shipped code; the compression study harness (scripts/codec_study) stays + outside it, and a finding it earns lands as production code of its own + +- root: sampletones_player + pattern: "**/*.py" + forbidden: '\bcodec_study\b' + message: >- + sampletones_player is shipped code; the compression study harness (scripts/codec_study) stays + outside it, and a finding it earns lands as production code of its own + +- root: sampletones_shared + pattern: "**/*.py" + forbidden: '\bcodec_study\b' + message: >- + sampletones_shared is shipped code; the compression study harness (scripts/codec_study) stays + outside it, and a finding it earns lands as production code of its own diff --git a/src/sampletones_player/py.typed b/src/sampletones_player/py.typed new file mode 100644 index 000000000..e69de29bb diff --git a/tests/unit/scripts/codec_study/test_accounting.py b/tests/unit/scripts/codec_study/test_accounting.py new file mode 100644 index 000000000..253235cc4 --- /dev/null +++ b/tests/unit/scripts/codec_study/test_accounting.py @@ -0,0 +1,252 @@ +from dataclasses import dataclass +from typing import Dict, Final, Sequence, Tuple + +import pytest + +from codec_study.accounting.coincident import coincident_starts +from codec_study.accounting.dictionary import default_counts, plateaus +from codec_study.accounting.finding import Finding +from codec_study.accounting.pairs import ramps_in_literals, set_holds +from codec_study.accounting.runs import ramps, runs +from codec_study.accounting.shares import hold_chains +from codec_study.accounting.tokens import ReadToken, read_tokens +from sampletones_player.compression.dictionary.phrase import Phrase +from sampletones_player.compression.dictionary.table import phrase_table +from sampletones_player.compression.encode import emit +from sampletones_player.compression.planes.order import PlaneOrder +from sampletones_player.compression.tokens.hold import HoldToken +from sampletones_player.compression.tokens.literal import LiteralToken +from sampletones_player.compression.tokens.phrase import PhraseToken +from sampletones_player.compression.tokens.types import TokenUnion +from sampletones_player.specification.compression import MAX_HOLD_TICKS, TokenTag +from tests.suite.base import BaseTestSuite +from tests.suite.case import BaseRegularTestCase + +ESCAPED_PHRASE_ID: Final[int] = 70 +REST: Final[int] = 32 + + +class TestReadTokensReadsBackWhatEmitWrote: + """Every token comes back with its tag, where it starts, what it takes and what it carries.""" + + tokens: Final[Tuple[TokenUnion, ...]] = ( + HoldToken(ticks=3), + LiteralToken(values=b"\x01\x02"), + PhraseToken(phrase_id=2, ticks=4, transpose=0), + PhraseToken(phrase_id=ESCAPED_PHRASE_ID, ticks=2, transpose=3), + ) + + def test_the_tokens_come_back_in_order(self) -> None: + read = read_tokens(emit(self.tokens)) + + assert [token.tag for token in read] == [ + TokenTag.HOLD, + TokenTag.LITERAL, + TokenTag.PHRASE, + TokenTag.TRANSPOSED_PHRASE, + ] + assert [token.tick for token in read] == [0, 3, 5, 9] + assert [token.ticks for token in read] == [3, 2, 4, 2] + assert [token.size for token in read] == [token.size for token in self.tokens] + + def test_the_operands_come_back(self) -> None: + read = read_tokens(emit(self.tokens)) + + assert [token.payload for token in read] == [b"", b"\x01\x02", b"", b""] + assert [token.phrase_id for token in read] == [None, None, 2, ESCAPED_PHRASE_ID] + assert [token.transpose for token in read] == [0, 0, 0, 3] + + +class TestRuns(BaseTestSuite): + @dataclass(frozen=True, kw_only=True) + class TestCase(BaseRegularTestCase): + data: bytes + expected: Tuple[int, ...] + + test_cases = ( + TestCase(label="empty", data=b"", expected=()), + TestCase(label="one value", data=b"\x07", expected=(1,)), + TestCase(label="two plateaus", data=b"\x01\x01\x01\x02\x02", expected=(3, 2)), + TestCase(label="no repeat", data=b"\x01\x02\x03", expected=(1, 1, 1)), + ) + + @pytest.mark.parametrize("test_case", test_cases, ids=lambda test_case: test_case.label) + def test_the_run_lengths_cover_the_data(self, test_case: TestCase) -> None: + assert runs(test_case.data) == test_case.expected + assert sum(test_case.expected) == len(test_case.data) + + +class TestRamps(BaseTestSuite): + @dataclass(frozen=True, kw_only=True) + class TestCase(BaseRegularTestCase): + data: bytes + expected: Tuple[int, ...] + + test_cases = ( + TestCase(label="a rise", data=b"\x01\x02\x03\x04\x05", expected=(5,)), + TestCase(label="a plateau steps nowhere", data=b"\x05\x05\x05", expected=()), + TestCase(label="a rise across the byte wraps", data=b"\xfe\xff\x00\x01", expected=(4,)), + TestCase(label="two steps of different size", data=b"\x01\x02\x04", expected=(2, 2)), + TestCase(label="a fall after a rest", data=b"\x09\x09\x08\x07", expected=(3,)), + ) + + @pytest.mark.parametrize("test_case", test_cases, ids=lambda test_case: test_case.label) + def test_the_constant_step_runs_are_found(self, test_case: TestCase) -> None: + assert ramps(test_case.data) == test_case.expected + + +class TestHoldChains(BaseTestSuite): + """A wide hold pays two bytes for up to sixty-four full holds, and one for the ticks left.""" + + @dataclass(frozen=True, kw_only=True) + class TestCase(BaseRegularTestCase): + holds: Tuple[int, ...] + expected: Finding + + test_cases = ( + TestCase(label="one hold is no chain", holds=(MAX_HOLD_TICKS,), expected=Finding(0, 0)), + TestCase(label="a full hold and a rest break even", holds=(MAX_HOLD_TICKS, REST), expected=Finding(2, 0)), + TestCase( + label="three full holds and a rest spare one", + holds=(MAX_HOLD_TICKS, MAX_HOLD_TICKS, MAX_HOLD_TICKS, REST), + expected=Finding(4, 1), + ), + TestCase( + label="sixty-four full holds become one wide hold", + holds=(MAX_HOLD_TICKS,) * MAX_HOLD_TICKS, + expected=Finding(MAX_HOLD_TICKS, MAX_HOLD_TICKS - 2), + ), + TestCase( + label="sixty-five full holds need two wide holds", + holds=(MAX_HOLD_TICKS,) * (MAX_HOLD_TICKS + 1), + expected=Finding(MAX_HOLD_TICKS + 1, MAX_HOLD_TICKS + 1 - 4), + ), + ) + + @pytest.mark.parametrize("test_case", test_cases, ids=lambda test_case: test_case.label) + def test_the_chain_is_priced_against_wide_holds(self, test_case: TestCase) -> None: + stream = emit([HoldToken(ticks=ticks) for ticks in test_case.holds]) + + assert hold_chains(read_tokens(stream)) == test_case.expected + + def test_a_literal_breaks_the_chain(self) -> None: + stream = emit( + [ + HoldToken(ticks=MAX_HOLD_TICKS), + LiteralToken(values=b"\x01"), + HoldToken(ticks=MAX_HOLD_TICKS), + HoldToken(ticks=MAX_HOLD_TICKS), + ] + ) + + assert hold_chains(read_tokens(stream)) == Finding(2, 0) + + +class TestSetHolds(BaseTestSuite): + @dataclass(frozen=True, kw_only=True) + class TestCase(BaseRegularTestCase): + tokens: Tuple[TokenUnion, ...] + expected: Finding + + test_cases = ( + TestCase( + label="a value then a rest is a pair", + tokens=(LiteralToken(values=b"\x05"), HoldToken(ticks=10)), + expected=Finding(3, 1), + ), + TestCase( + label="two values then a rest is no pair", + tokens=(LiteralToken(values=b"\x05\x06"), HoldToken(ticks=10)), + expected=Finding(0, 0), + ), + TestCase( + label="a rest then a value is no pair", + tokens=(HoldToken(ticks=10), LiteralToken(values=b"\x05")), + expected=Finding(0, 0), + ), + TestCase( + label="every pair counts", + tokens=( + LiteralToken(values=b"\x05"), + HoldToken(ticks=10), + LiteralToken(values=b"\x06"), + HoldToken(ticks=10), + ), + expected=Finding(6, 2), + ), + ) + + @pytest.mark.parametrize("test_case", test_cases, ids=lambda test_case: test_case.label) + def test_the_pairs_are_priced_at_two_bytes(self, test_case: TestCase) -> None: + assert set_holds(read_tokens(emit(list(test_case.tokens)))) == test_case.expected + + +class TestRampsInLiterals(BaseTestSuite): + @dataclass(frozen=True, kw_only=True) + class TestCase(BaseRegularTestCase): + values: bytes + expected: Finding + + test_cases = ( + TestCase(label="a rise of five", values=b"\x01\x02\x03\x04\x05", expected=Finding(5, 2)), + TestCase(label="a rise of three breaks even", values=b"\x01\x02\x03", expected=Finding(3, 0)), + TestCase(label="a rise of two is no ramp", values=b"\x01\x02", expected=Finding(0, 0)), + TestCase(label="a plateau is no ramp", values=b"\x05\x05\x05\x05", expected=Finding(0, 0)), + ) + + @pytest.mark.parametrize("test_case", test_cases, ids=lambda test_case: test_case.label) + def test_the_ramps_are_priced_at_three_bytes(self, test_case: TestCase) -> None: + assert ramps_in_literals(read_tokens(emit([LiteralToken(values=test_case.values)]))) == test_case.expected + + def test_a_ramp_inside_a_hold_is_none(self) -> None: + assert ramps_in_literals(read_tokens(emit([HoldToken(ticks=5)]))) == Finding(0, 0) + + +def _starting_at(ticks: Sequence[int]) -> Tuple[ReadToken, ...]: + return tuple( + ReadToken( + tag=TokenTag.HOLD, + tick=tick, + size=1, + ticks=1, + payload=b"", + phrase_id=None, + transpose=0, + ) + for tick in ticks + ) + + +class TestCoincidentStarts: + def test_a_channel_counts_the_ticks_both_its_planes_start_a_token_on(self) -> None: + tokens: Dict[str, Tuple[ReadToken, ...]] = {name: _starting_at((0,)) for name in PlaneOrder.names()} + tokens["pulse1_control"] = _starting_at((0, 5)) + tokens["pulse1_value"] = _starting_at((0, 5, 9)) + tokens["noise_value"] = _starting_at((0, 7)) + + assert coincident_starts(tokens) == Finding(10, 5) + + +class TestPlateausInBodies: + def test_the_repeats_beyond_a_runs_first_byte_are_targeted(self) -> None: + table = phrase_table((Phrase(body=b"\x01\x01\x01\x02\x02"), Phrase(body=b"\x03\x04"))) + + assert plateaus(table) == Finding(3, 1) + + +class TestDefaultCounts: + def test_the_modal_count_of_each_phrase_spares_its_bytes_but_one(self) -> None: + table = phrase_table((Phrase(body=b"\x01\x02"), Phrase(body=b"\x03\x04"))) + tokens = read_tokens( + emit( + [ + PhraseToken(phrase_id=0, ticks=4, transpose=0), + PhraseToken(phrase_id=0, ticks=4, transpose=0), + PhraseToken(phrase_id=0, ticks=4, transpose=2), + PhraseToken(phrase_id=0, ticks=6, transpose=0), + PhraseToken(phrase_id=1, ticks=2, transpose=0), + ] + ) + ) + + assert default_counts(table, tokens) == Finding(5, 2) From aa5e8c61c414a272ea02449c14ec2d509513491e Mon Sep 17 00:00:00 2001 From: JakimPL Date: Sat, 12 Sep 2026 19:30:21 +0200 Subject: [PATCH 03/36] Measured: seed transforms, search budgets and strategy depth --- scripts/codec_study/report/run.py | 15 +- scripts/codec_study/variants/production.py | 153 +++++++++++++++++- scripts/codec_study/variants/registry.py | 12 +- scripts/codec_study/variants/seeds.py | 92 +++++++++++ scripts/codec_study/variants/strategy.py | 72 +++++++++ scripts/codec_study/variants/variant.py | 14 ++ scripts/compression_study.py | 18 ++- .../unit/scripts/codec_study/test_variants.py | 153 ++++++++++++++++++ 8 files changed, 516 insertions(+), 13 deletions(-) create mode 100644 scripts/codec_study/variants/seeds.py create mode 100644 scripts/codec_study/variants/strategy.py create mode 100644 tests/unit/scripts/codec_study/test_variants.py diff --git a/scripts/codec_study/report/run.py b/scripts/codec_study/report/run.py index 6a78edbee..6c0faa91f 100644 --- a/scripts/codec_study/report/run.py +++ b/scripts/codec_study/report/run.py @@ -54,6 +54,7 @@ def write_run( manifest: StudyManifest, variants: Sequence[Variant], measurements: Sequence[Measurement], + derived: Sequence[Measurement], ) -> None: """Writes a run's report, its accounting and the manifest that reproduces it. @@ -62,14 +63,18 @@ def write_run( manifest: What the run read. variants: The variants every song was encoded under. measurements: Every song under every variant, in the order measured. + derived: The strategy-depth measurements drawn from those, reported beside them and + left out of the accounting, which reads each encoding once. """ - song_rows = [songs.study_row(measurement) for measurement in measurements] - group_rows = aggregate.group_rows(measurements, BASELINE.name) + reported = (*measurements, *derived) + song_rows = [songs.study_row(measurement) for measurement in reported] + group_rows = aggregate.group_rows(reported, BASELINE.name) accounting_rows = [accounting.account(measurement) for measurement in measurements] manifest.save(directory / MANIFEST_JSON) write_csv(directory / REPORT_CSV, songs.COLUMNS, [row.cells for row in song_rows]) write_csv(directory / ACCOUNTING_CSV, accounting.COLUMNS, [row.cells for row in accounting_rows]) lines = _header(manifest, variants) + lines.extend(("## Variants", "", *_variants_table(variants), "")) lines.extend(("## Groups", "", *markdown_table(aggregate.COLUMNS, [row.cells for row in group_rows]), "")) lines.extend(("## Songs", "", *markdown_table(songs.COLUMNS, [row.cells for row in song_rows]), "")) lines.extend(("## Accounting", "", *_accounting_table(accounting_rows), "")) @@ -95,6 +100,12 @@ def _header( ] +def _variants_table(variants: Sequence[Variant]) -> List[str]: + columns = ("variant", "hypothesis", "kind", "driver") + cells = [(variant.name, variant.hypothesis, variant.kind.value, variant.note) for variant in variants] + return markdown_table(columns, cells) + + def _accounting_table(rows: Sequence[accounting.AccountingRow]) -> List[str]: columns = ("group", "song", "variant", "block", "dictionary", "idle bytes", "bend bytes") labels = tuple(f"{label} {name}" for label, name in accounting.HYPOTHESES) diff --git a/scripts/codec_study/variants/production.py b/scripts/codec_study/variants/production.py index d1413d9fa..1d7d3f7e1 100644 --- a/scripts/codec_study/variants/production.py +++ b/scripts/codec_study/variants/production.py @@ -1,36 +1,181 @@ -from typing import Final +from typing import Callable, Final, Sequence, Tuple from codec_study.corpus.song import StudySong +from codec_study.measure import Encoder +from codec_study.variants.seeds import split, trimmed, whole_and_split from codec_study.variants.variant import Variant, VariantKind +from sampletones_player.compression.budget import DEFAULT_SEARCH_BUDGET, SearchBudget from sampletones_player.compression.compressed import CompressedPlanes +from sampletones_player.compression.dictionary.phrase import Phrase from sampletones_player.compression.encode import encode_planes from sampletones_player.compression.options import EVERY_LAYER +SeedTransform = Callable[[Sequence[Phrase]], Tuple[Phrase, ...]] + BASELINE_NAME: Final[str] = "baseline" NO_LOOP_BOUNDARIES: Final[frozenset[int]] = frozenset() +SEED_SPLITTING: Final[str] = "H2'" +SEARCH_BUDGET: Final[str] = "H8" +DRIVER_UNCHANGED: Final[str] = "driver unchanged" +SPLIT_THRESHOLDS: Final[Tuple[int, ...]] = (2, 3, 4, 6, 8) +LIGHT_ENTRIES: Final[int] = 50_000 +WIDE_ENTRIES: Final[int] = 1_000_000 +LONG_ROUNDS: Final[int] = 255 +LIGHT_CONFIRMED: Final[int] = 1 +DEEP_CONFIRMED: Final[int] = 8 -def encode_baseline(song: StudySong) -> CompressedPlanes: - """Encodes a song as the export does, every layer on and the search at its default budget. +def encode_production( + song: StudySong, + *, + seeds: Sequence[Phrase], + budget: SearchBudget, +) -> CompressedPlanes: + """Encodes a song as the export does, every layer on, over the seeds and budget given. Args: song: The song to encode. + seeds: The phrases offered to the dictionary. + budget: How much work the search spends. Returns: CompressedPlanes: The dictionary and the token streams. """ return encode_planes( song.planes, - song.seeds, + seeds, options=EVERY_LAYER, boundaries=NO_LOOP_BOUNDARIES, + budget=budget, + ) + + +def encode_baseline(song: StudySong) -> CompressedPlanes: + """Encodes a song as the export does today. + + Args: + song: The song to encode. + + Returns: + CompressedPlanes: The dictionary and the token streams. + """ + return encode_production(song, seeds=song.seeds, budget=DEFAULT_SEARCH_BUDGET) + + +def seed_encoder(transform: SeedTransform) -> Encoder: + """An encoder offering the dictionary the song's seeds after ``transform``. + + Args: + transform: What the seeds become before they are offered. + + Returns: + Encoder: The encoder, searching at the default budget. + """ + + def encode(song: StudySong) -> CompressedPlanes: + return encode_production(song, seeds=transform(song.seeds), budget=DEFAULT_SEARCH_BUDGET) + + return encode + + +def budget_encoder(budget: SearchBudget) -> Encoder: + """An encoder searching under ``budget`` over the song's own seeds. + + Args: + budget: How much work the search spends. + + Returns: + Encoder: The encoder. + """ + + def encode(song: StudySong) -> CompressedPlanes: + return encode_production(song, seeds=song.seeds, budget=budget) + + return encode + + +def _seed_variant( + name: str, + transform: SeedTransform, +) -> Variant: + return Variant( + name=name, + hypothesis=SEED_SPLITTING, + kind=VariantKind.ENCODER, + note=DRIVER_UNCHANGED, + encode=seed_encoder(transform), + needs_seeds=True, ) +def _budget_variant( + name: str, + budget: SearchBudget, +) -> Variant: + return Variant( + name=name, + hypothesis=SEARCH_BUDGET, + kind=VariantKind.ENCODER, + note=DRIVER_UNCHANGED, + encode=budget_encoder(budget), + needs_seeds=False, + ) + + +def _splitter(threshold: int) -> SeedTransform: + return lambda seeds: split(seeds, threshold) + + +def _both(threshold: int) -> SeedTransform: + return lambda seeds: whole_and_split(seeds, threshold) + + BASELINE: Final[Variant] = Variant( name=BASELINE_NAME, hypothesis="", kind=VariantKind.BASELINE, note="", encode=encode_baseline, + needs_seeds=False, +) + +SEED_VARIANTS: Final[Tuple[Variant, ...]] = ( + _seed_variant("seeds-trimmed", trimmed), + *(_seed_variant(f"seeds-split-{threshold}", _splitter(threshold)) for threshold in SPLIT_THRESHOLDS), + *(_seed_variant(f"seeds-both-{threshold}", _both(threshold)) for threshold in SPLIT_THRESHOLDS), +) + +BUDGET_VARIANTS: Final[Tuple[Variant, ...]] = ( + _budget_variant( + "search-light", + SearchBudget( + candidate_entries=LIGHT_ENTRIES, + rounds=DEFAULT_SEARCH_BUDGET.rounds, + confirmed_candidates=LIGHT_CONFIRMED, + ), + ), + _budget_variant( + "search-long", + SearchBudget( + candidate_entries=DEFAULT_SEARCH_BUDGET.candidate_entries, + rounds=LONG_ROUNDS, + confirmed_candidates=DEFAULT_SEARCH_BUDGET.confirmed_candidates, + ), + ), + _budget_variant( + "search-wide", + SearchBudget( + candidate_entries=WIDE_ENTRIES, + rounds=DEFAULT_SEARCH_BUDGET.rounds, + confirmed_candidates=DEFAULT_SEARCH_BUDGET.confirmed_candidates, + ), + ), + _budget_variant( + "search-deep", + SearchBudget( + candidate_entries=WIDE_ENTRIES, + rounds=LONG_ROUNDS, + confirmed_candidates=DEEP_CONFIRMED, + ), + ), ) diff --git a/scripts/codec_study/variants/registry.py b/scripts/codec_study/variants/registry.py index ade9040fc..f10f5d27e 100644 --- a/scripts/codec_study/variants/registry.py +++ b/scripts/codec_study/variants/registry.py @@ -1,16 +1,19 @@ from typing import Dict, Final, Sequence, Tuple -from codec_study.variants.production import BASELINE +from codec_study.variants.production import BASELINE, BUDGET_VARIANTS, SEED_VARIANTS from codec_study.variants.variant import Variant -VARIANTS: Final[Dict[str, Variant]] = {BASELINE.name: BASELINE} +EVERY_VARIANT: Final[str] = "all" +VARIANTS: Final[Dict[str, Variant]] = { + variant.name: variant for variant in (BASELINE, *SEED_VARIANTS, *BUDGET_VARIANTS) +} def selected_variants(names: Sequence[str]) -> Tuple[Variant, ...]: """The variants a run encodes every song under, the baseline always first. Args: - names: The names the run asks for. + names: The names the run asks for, or ``all`` for every registered variant. Returns: Tuple[Variant, ...]: The baseline, then the named variants in the order given. @@ -18,6 +21,9 @@ def selected_variants(names: Sequence[str]) -> Tuple[Variant, ...]: Raises: KeyError: If a name is registered to no variant. """ + if EVERY_VARIANT in names: + return tuple(VARIANTS.values()) + unknown = [name for name in names if name not in VARIANTS] if unknown: raise KeyError(f"no variant is called {', '.join(unknown)}; the registry holds {', '.join(VARIANTS)}") diff --git a/scripts/codec_study/variants/seeds.py b/scripts/codec_study/variants/seeds.py new file mode 100644 index 000000000..d0eed4053 --- /dev/null +++ b/scripts/codec_study/variants/seeds.py @@ -0,0 +1,92 @@ +from typing import Final, List, Sequence, Tuple + +from sampletones_player.compression.dictionary.phrase import Phrase + +MIN_PIECE_LENGTH: Final[int] = 2 + + +def trimmed(seeds: Sequence[Phrase]) -> Tuple[Phrase, ...]: + """Every seed with its trailing plateau cut to one value. + + A token playing a phrase past its end holds the final value onward, so the ticks a body + spends resting on its last value are ticks the token covers for free. + + Args: + seeds: The phrases the song's instruments offer. + + Returns: + Tuple[Phrase, ...]: The same phrases, each ending where its last value first appears. + """ + return tuple(Phrase(body=_trimmed_body(seed.body)) for seed in seeds) + + +def split( + seeds: Sequence[Phrase], + threshold: int, +) -> Tuple[Phrase, ...]: + """Every seed cut at the plateaus of at least ``threshold`` ticks it holds. + + A plateau inside a body is stored one byte per tick, where a token's count would cover the + rest for free. Cutting the body where a plateau begins keeps the plateau's first value at + the end of one piece, so the token playing that piece holds it, and the next piece starts + where the body moves again. + + Args: + seeds: The phrases the song's instruments offer. + threshold: The ticks a plateau rests for before the body is cut there. + + Returns: + Tuple[Phrase, ...]: The pieces, each of at least two values, in the order they are cut. + """ + return tuple(Phrase(body=piece) for seed in seeds for piece in _pieces(seed.body, threshold)) + + +def whole_and_split( + seeds: Sequence[Phrase], + threshold: int, +) -> Tuple[Phrase, ...]: + """Every seed as it stands, followed by its pieces, for the encoder to weigh against each other. + + Args: + seeds: The phrases the song's instruments offer. + threshold: The ticks a plateau rests for before the body is cut there. + + Returns: + Tuple[Phrase, ...]: The whole phrases, then the pieces. + """ + return (*seeds, *split(seeds, threshold)) + + +def _trimmed_body(body: bytes) -> bytes: + end = len(body) + while end > 1 and body[end - 1] == body[end - 2]: + end -= 1 + + return body[:end] + + +def _pieces( + body: bytes, + threshold: int, +) -> List[bytes]: + pieces: List[bytes] = [] + current = bytearray() + start = 0 + while start < len(body): + end = start + while end < len(body) and body[end] == body[start]: + end += 1 + + if end - start >= threshold: + current.append(body[start]) + pieces.append(bytes(current)) + current = bytearray() + else: + current.extend(body[start:end]) + + start = end + + if current: + pieces.append(bytes(current)) + + return [piece for piece in pieces if len(piece) >= MIN_PIECE_LENGTH] diff --git a/scripts/codec_study/variants/strategy.py b/scripts/codec_study/variants/strategy.py new file mode 100644 index 000000000..bfcd3d93f --- /dev/null +++ b/scripts/codec_study/variants/strategy.py @@ -0,0 +1,72 @@ +from typing import Dict, Final, List, Optional, Sequence, Tuple + +from codec_study.measure import Measurement + +STRATEGY_ORDER: Final[Tuple[str, ...]] = ( + "baseline", + "seeds-trimmed", + "seeds-both-4", + "seeds-split-4", + "search-long", + "search-wide", + "search-deep", +) +DEPTH_PREFIX: Final[str] = "depth-" + + +def depth_measurements( + measurements: Sequence[Measurement], + order: Sequence[str], +) -> Tuple[Measurement, ...]: + """What an export trying the first ``d`` strategies and keeping the smallest would write. + + An export option choosing how deep to search is modeled as a list of encoder-side + strategies tried in order: at each depth a song is written by whichever of the strategies so + far made it smallest, and the time is what all of them took together. A strategy a song + offers nothing to keeps the song where the previous depth left it. + + Args: + measurements: Every song under every variant the run measured. + order: The strategies, in the order an export would try them. + + Returns: + Tuple[Measurement, ...]: One measurement per song and depth, named ``depth-d``, in song + order then depth order. + """ + present = [name for name in order if any(measurement.variant == name for measurement in measurements)] + derived: List[Measurement] = [] + for name in dict.fromkeys(measurement.song.name for measurement in measurements): + by_variant: Dict[str, Measurement] = { + measurement.variant: measurement for measurement in measurements if measurement.song.name == name + } + derived.extend(_song_depths(by_variant, present)) + + return tuple(derived) + + +def _song_depths( + by_variant: Dict[str, Measurement], + present: Sequence[str], +) -> List[Measurement]: + depths: List[Measurement] = [] + best: Optional[Measurement] = None + seconds = 0.0 + for depth, name in enumerate(present, start=1): + measured = by_variant.get(name) + if measured is not None: + seconds += measured.seconds + if best is None or measured.block < best.block: + best = measured + + if best is not None: + depths.append( + Measurement( + song=best.song, + variant=f"{DEPTH_PREFIX}{depth}", + compressed=best.compressed, + seconds=seconds, + lossless=best.lossless, + ) + ) + + return depths diff --git a/scripts/codec_study/variants/variant.py b/scripts/codec_study/variants/variant.py index 81e743b3f..4fb9ee88d 100644 --- a/scripts/codec_study/variants/variant.py +++ b/scripts/codec_study/variants/variant.py @@ -1,6 +1,7 @@ from dataclasses import dataclass from enum import StrEnum +from codec_study.corpus.song import StudySong from codec_study.measure import Encoder @@ -28,6 +29,7 @@ class Variant: kind: What the variant changes. note: What the driver would have to do, where the variant changes the format. encode: What the variant writes a song as. + needs_seeds: Whether the variant changes anything only where a song offers seeds. """ name: str @@ -35,3 +37,15 @@ class Variant: kind: VariantKind note: str encode: Encoder + needs_seeds: bool + + def applies(self, song: StudySong) -> bool: + """Whether encoding ``song`` under this variant can differ from the baseline. + + Args: + song: The song. + + Returns: + bool: Whether the variant is worth measuring on it. + """ + return bool(song.seeds) or not self.needs_seeds diff --git a/scripts/compression_study.py b/scripts/compression_study.py index b474cd544..5c33856cd 100644 --- a/scripts/compression_study.py +++ b/scripts/compression_study.py @@ -6,11 +6,12 @@ from codec_study.manifest import StudyManifest, StudySource from codec_study.measure import Measurement, measure from codec_study.report.run import run_directory, write_run -from codec_study.variants.registry import selected_variants +from codec_study.variants.registry import EVERY_VARIANT, selected_variants +from codec_study.variants.strategy import STRATEGY_ORDER, depth_measurements from sampletones_shared.logger import logger DEFAULT_LENGTHEN_SECONDS: Final[int] = 180 -DEFAULT_VARIANTS: Final[str] = "baseline" +DEFAULT_VARIANTS: Final[str] = EVERY_VARIANT def main() -> None: @@ -53,7 +54,7 @@ def main() -> None: "--variants", type=str, default=DEFAULT_VARIANTS, - help="Comma-separated variants every song is encoded under; the baseline always runs.", + help="Comma-separated variants every song is encoded under, or all; the baseline always runs.", ) parser.add_argument( "--quick", @@ -77,6 +78,9 @@ def main() -> None: measurements: List[Measurement] = [] for song in corpus: for variant in variants: + if not variant.applies(song): + continue + logger.info(f"Encoding {song.name} ({song.ticks} ticks) under {variant.name}") measurement = measure(song, variant.name, variant.encode) if not measurement.lossless: @@ -88,7 +92,13 @@ def main() -> None: ) measurements.append(measurement) - write_run(directory, manifest, variants, measurements) + write_run( + directory, + manifest, + variants, + measurements, + depth_measurements(measurements, STRATEGY_ORDER), + ) logger.info(f"Report written to {directory}") diff --git a/tests/unit/scripts/codec_study/test_variants.py b/tests/unit/scripts/codec_study/test_variants.py new file mode 100644 index 000000000..992022259 --- /dev/null +++ b/tests/unit/scripts/codec_study/test_variants.py @@ -0,0 +1,153 @@ +from dataclasses import dataclass +from pathlib import Path +from typing import Final, Tuple + +import pytest + +from codec_study.corpus.song import SongGroup, StudySong +from codec_study.measure import Measurement +from codec_study.variants.seeds import split, trimmed, whole_and_split +from codec_study.variants.strategy import DEPTH_PREFIX, depth_measurements +from sampletones_player.compression.compressed import CompressedPlanes +from sampletones_player.compression.dictionary.phrase import Phrase +from sampletones_player.compression.dictionary.table import phrase_table +from sampletones_player.compression.encode import emit +from sampletones_player.compression.pitch import PitchTable +from sampletones_player.compression.planes.order import PlaneOrder +from sampletones_player.compression.planes.song import SongPlanes +from sampletones_player.compression.tokens.hold import HoldToken +from sampletones_player.compression.tokens.literal import LiteralToken +from sampletones_player.specification.compression import PLANE_COUNT +from sampletones_shared.music import Tuning +from tests.suite.base import BaseTestSuite +from tests.suite.case import BaseRegularTestCase + +TICKS: Final[int] = 4 +BASELINE: Final[str] = "baseline" +TRIMMED: Final[str] = "seeds-trimmed" +WIDE: Final[str] = "search-wide" +ORDER: Final[Tuple[str, ...]] = (BASELINE, TRIMMED, "seeds-split-4", WIDE) + + +class TestTrimmed(BaseTestSuite): + @dataclass(frozen=True, kw_only=True) + class TestCase(BaseRegularTestCase): + body: bytes + expected: bytes + + test_cases = ( + TestCase( + label="a trailing plateau ends at its first value", body=b"\x01\x02\x03\x03\x03", expected=b"\x01\x02\x03" + ), + TestCase(label="a body of one value keeps one", body=b"\x05\x05", expected=b"\x05"), + TestCase(label="a body moving to its end stays", body=b"\x01\x02", expected=b"\x01\x02"), + TestCase(label="a plateau inside the body stays", body=b"\x01\x01\x02", expected=b"\x01\x01\x02"), + ) + + @pytest.mark.parametrize("test_case", test_cases, ids=lambda test_case: test_case.label) + def test_the_tail_is_cut(self, test_case: TestCase) -> None: + assert trimmed((Phrase(body=test_case.body),)) == (Phrase(body=test_case.expected),) + + +class TestSplit(BaseTestSuite): + @dataclass(frozen=True, kw_only=True) + class TestCase(BaseRegularTestCase): + body: bytes + threshold: int + expected: Tuple[bytes, ...] + + test_cases = ( + TestCase( + label="plateaus of three cut the body", + body=b"\x01\x02\x02\x02\x03\x04\x04\x04\x04\x05", + threshold=3, + expected=(b"\x01\x02", b"\x03\x04"), + ), + TestCase( + label="a threshold above every plateau keeps the body whole", + body=b"\x01\x02\x02\x02\x03", + threshold=4, + expected=(b"\x01\x02\x02\x02\x03",), + ), + TestCase( + label="plateaus of two cut too", + body=b"\x01\x01\x02\x03\x03\x04", + threshold=2, + expected=(b"\x02\x03",), + ), + TestCase( + label="a piece of one value is dropped", + body=b"\x01\x02\x02\x02\x03", + threshold=3, + expected=(b"\x01\x02",), + ), + ) + + @pytest.mark.parametrize("test_case", test_cases, ids=lambda test_case: test_case.label) + def test_the_pieces_end_where_a_plateau_begins(self, test_case: TestCase) -> None: + assert split((Phrase(body=test_case.body),), test_case.threshold) == tuple( + Phrase(body=piece) for piece in test_case.expected + ) + + def test_whole_and_split_offers_both(self) -> None: + seeds = (Phrase(body=b"\x01\x02\x02\x02\x03\x04"),) + + assert whole_and_split(seeds, 3) == (*seeds, Phrase(body=b"\x01\x02"), Phrase(body=b"\x03\x04")) + + +def _song() -> StudySong: + return StudySong( + name="song", + group=SongGroup.PROJECT, + source=Path("song.stp"), + planes=SongPlanes.from_order(PlaneOrder.across([bytes(TICKS)] * PLANE_COUNT)), + seeds=(), + pitches=PitchTable.from_tuning(Tuning()), + ) + + +def _measurement( + song: StudySong, + variant: str, + *, + spelled_out: bool, + seconds: float, +) -> Measurement: + stream = emit([LiteralToken(values=bytes(TICKS))] if spelled_out else [HoldToken(ticks=TICKS)]) + return Measurement( + song=song, + variant=variant, + compressed=CompressedPlanes( + phrases=phrase_table(()), + streams=PlaneOrder.across([stream] * PLANE_COUNT), + ticks=TICKS, + ), + seconds=seconds, + lossless=True, + ) + + +class TestDepthMeasurements: + def test_each_depth_keeps_the_smallest_so_far_and_sums_the_time(self) -> None: + song = _song() + measurements = ( + _measurement(song, BASELINE, spelled_out=True, seconds=1.0), + _measurement(song, TRIMMED, spelled_out=False, seconds=2.0), + _measurement(song, WIDE, spelled_out=True, seconds=4.0), + ) + + depths = depth_measurements(measurements, ORDER) + + assert [depth.variant for depth in depths] == [f"{DEPTH_PREFIX}{depth}" for depth in (1, 2, 3)] + assert [depth.block for depth in depths] == [ + measurements[0].block, + measurements[1].block, + measurements[1].block, + ] + assert [depth.seconds for depth in depths] == [1.0, 3.0, 7.0] + + def test_a_strategy_no_song_was_measured_under_takes_no_depth(self) -> None: + song = _song() + measurements = (_measurement(song, BASELINE, spelled_out=True, seconds=1.0),) + + assert [depth.variant for depth in depth_measurements(measurements, ORDER)] == [f"{DEPTH_PREFIX}1"] From f7b9207eca483f32b60a79f1665ed943f7535416 Mon Sep 17 00:00:00 2001 From: JakimPL Date: Sat, 12 Sep 2026 20:17:20 +0200 Subject: [PATCH 04/36] Priced: candidate token grammars in a study-side parser --- scripts/codec_study/accounting/rows.py | 10 +- scripts/codec_study/measure.py | 98 ++++- scripts/codec_study/report/rows.py | 2 +- scripts/codec_study/report/run.py | 21 +- scripts/codec_study/sandbox/__init__.py | 0 scripts/codec_study/sandbox/context.py | 43 ++ scripts/codec_study/sandbox/costs.py | 93 ++++ scripts/codec_study/sandbox/decode.py | 53 +++ scripts/codec_study/sandbox/defaults.py | 44 ++ scripts/codec_study/sandbox/edges/__init__.py | 0 .../codec_study/sandbox/edges/generator.py | 53 +++ scripts/codec_study/sandbox/edges/holds.py | 79 ++++ scripts/codec_study/sandbox/edges/literals.py | 29 ++ scripts/codec_study/sandbox/edges/phrases.py | 57 +++ scripts/codec_study/sandbox/edges/set_hold.py | 45 ++ scripts/codec_study/sandbox/encode.py | 80 ++++ scripts/codec_study/sandbox/grammar.py | 40 ++ scripts/codec_study/sandbox/parse.py | 117 ++++++ scripts/codec_study/sandbox/reference.py | 108 +++++ scripts/codec_study/sandbox/shortest.py | 86 ++++ scripts/codec_study/sandbox/tokens.py | 59 +++ scripts/codec_study/sandbox/verify.py | 51 +++ scripts/codec_study/variants/baselines.py | 91 ++++ scripts/codec_study/variants/production.py | 51 ++- scripts/codec_study/variants/registry.py | 42 +- scripts/codec_study/variants/sandbox.py | 178 ++++++++ scripts/codec_study/variants/strategy.py | 5 +- scripts/compression_study.py | 5 +- .../unit/scripts/codec_study/test_sandbox.py | 397 ++++++++++++++++++ .../unit/scripts/codec_study/test_variants.py | 15 +- 30 files changed, 1880 insertions(+), 72 deletions(-) create mode 100644 scripts/codec_study/sandbox/__init__.py create mode 100644 scripts/codec_study/sandbox/context.py create mode 100644 scripts/codec_study/sandbox/costs.py create mode 100644 scripts/codec_study/sandbox/decode.py create mode 100644 scripts/codec_study/sandbox/defaults.py create mode 100644 scripts/codec_study/sandbox/edges/__init__.py create mode 100644 scripts/codec_study/sandbox/edges/generator.py create mode 100644 scripts/codec_study/sandbox/edges/holds.py create mode 100644 scripts/codec_study/sandbox/edges/literals.py create mode 100644 scripts/codec_study/sandbox/edges/phrases.py create mode 100644 scripts/codec_study/sandbox/edges/set_hold.py create mode 100644 scripts/codec_study/sandbox/encode.py create mode 100644 scripts/codec_study/sandbox/grammar.py create mode 100644 scripts/codec_study/sandbox/parse.py create mode 100644 scripts/codec_study/sandbox/reference.py create mode 100644 scripts/codec_study/sandbox/shortest.py create mode 100644 scripts/codec_study/sandbox/tokens.py create mode 100644 scripts/codec_study/sandbox/verify.py create mode 100644 scripts/codec_study/variants/baselines.py create mode 100644 scripts/codec_study/variants/sandbox.py create mode 100644 tests/unit/scripts/codec_study/test_sandbox.py diff --git a/scripts/codec_study/accounting/rows.py b/scripts/codec_study/accounting/rows.py index 12ce84884..aed9bbcaa 100644 --- a/scripts/codec_study/accounting/rows.py +++ b/scripts/codec_study/accounting/rows.py @@ -9,6 +9,7 @@ from codec_study.accounting.shares import PlaneShares, plane_shares from codec_study.accounting.tokens import ReadToken, read_tokens from codec_study.measure import Measurement +from sampletones_player.compression.compressed import CompressedPlanes from sampletones_player.compression.planes.order import PlaneOrder HYPOTHESES: Final[Tuple[Tuple[str, str], ...]] = ( @@ -99,16 +100,19 @@ def share(self, saving: int) -> str: return f"{100.0 * saving / self.block:.1f}%" -def account(measurement: Measurement) -> AccountingRow: +def account( + measurement: Measurement, + compressed: CompressedPlanes, +) -> AccountingRow: """Reads one encoding back and states what every hypothesis would reach in it. Args: - measurement: The encoding. + measurement: The encoding under the song and variant it belongs to. + compressed: Its streams as the driver reads them. Returns: AccountingRow: The shares and the findings. """ - compressed = measurement.compressed tokens: Dict[str, Tuple[ReadToken, ...]] = { name: read_tokens(stream) for name, stream in zip(PlaneOrder.names(), compressed.streams) } diff --git a/scripts/codec_study/measure.py b/scripts/codec_study/measure.py index 7f67775e0..e277fe028 100644 --- a/scripts/codec_study/measure.py +++ b/scripts/codec_study/measure.py @@ -1,47 +1,83 @@ from dataclasses import dataclass -from time import process_time -from typing import Callable +from typing import Callable, Optional, Tuple from codec_study.corpus.song import StudySong from sampletones_player.compression.compressed import CompressedPlanes from sampletones_player.compression.decode import decode_planes from sampletones_player.specification.song import SONG_HEADER_SIZE -Encoder = Callable[[StudySong], CompressedPlanes] + +@dataclass(frozen=True) +class Encoding: + """What a variant wrote a song as, priced in the bytes the song block counts. + + A variant of the encoder writes the grammar the driver reads today, and its streams are + kept as the bytes they are. A variant of the grammar is priced at token level, its streams + stated as the bytes each would take, so the two kinds sit in one report on the same terms. + + Attributes: + phrases: The phrases the dictionary holds. + dictionary: The bytes the dictionary takes. + streams: The bytes each plane's stream takes, in the order the song block writes them. + seconds: The processor time the encoding took. + lossless: Whether the streams play back to the planes they were written from. + written: The streams as the driver reads them, where the variant writes its grammar. + """ + + phrases: int + dictionary: int + streams: Tuple[int, ...] + seconds: float + lossless: bool + written: Optional[CompressedPlanes] + + +Encoder = Callable[[StudySong], Encoding] @dataclass(frozen=True) class Measurement: - """One song encoded under one variant, with what the encoding cost and whether it plays back. + """One song encoded under one variant. Attributes: song: The song encoded. variant: The name of the variant the encoding was built by. - compressed: The dictionary and the token streams the variant wrote. - seconds: The processor time the encoding took. - lossless: Whether the streams play back to the planes they were written from. + encoding: What the variant wrote, and what writing it cost. """ song: StudySong variant: str - compressed: CompressedPlanes - seconds: float - lossless: bool + encoding: Encoding @property def block(self) -> int: """The bytes the whole song block takes: header, pitch table, dictionary and streams.""" - return SONG_HEADER_SIZE + len(self.song.pitches.data) + self.compressed.size + return SONG_HEADER_SIZE + len(self.song.pitches.data) + self.dictionary + self.streams @property def dictionary(self) -> int: """The bytes the dictionary takes.""" - return self.compressed.phrases.size + return self.encoding.dictionary @property def streams(self) -> int: """The bytes the token streams take together.""" - return sum(len(stream) for stream in self.compressed.streams) + return sum(self.encoding.streams) + + @property + def phrases(self) -> int: + """The phrases the dictionary holds.""" + return self.encoding.phrases + + @property + def seconds(self) -> float: + """The processor time the encoding took.""" + return self.encoding.seconds + + @property + def lossless(self) -> bool: + """Whether the streams play back to the planes they were written from.""" + return self.encoding.lossless @property def bytes_per_tick(self) -> float: @@ -49,12 +85,37 @@ def bytes_per_tick(self) -> float: return self.block / self.song.ticks +def production_encoding( + song: StudySong, + compressed: CompressedPlanes, + seconds: float, +) -> Encoding: + """What the production codec wrote a song as, played back through the production decoder. + + Args: + song: The song encoded. + compressed: The dictionary and the streams the codec wrote. + seconds: The processor time the encoding took. + + Returns: + Encoding: The encoding, its streams kept as written. + """ + return Encoding( + phrases=len(compressed.phrases), + dictionary=compressed.phrases.size, + streams=tuple(len(stream) for stream in compressed.streams), + seconds=seconds, + lossless=decode_planes(compressed) == song.planes, + written=compressed, + ) + + def measure( song: StudySong, variant: str, encode: Encoder, ) -> Measurement: - """Encodes a song under a variant, timing the encoding and playing it back. + """Encodes a song under a variant. Args: song: The song to encode. @@ -62,15 +123,10 @@ def measure( encode: What the variant writes the song as. Returns: - Measurement: The encoding, its cost and whether it plays back. + Measurement: The encoding under the song and variant it belongs to. """ - started = process_time() - compressed = encode(song) - seconds = process_time() - started return Measurement( song=song, variant=variant, - compressed=compressed, - seconds=seconds, - lossless=decode_planes(compressed) == song.planes, + encoding=encode(song), ) diff --git a/scripts/codec_study/report/rows.py b/scripts/codec_study/report/rows.py index ac3a6c365..07592b44b 100644 --- a/scripts/codec_study/report/rows.py +++ b/scripts/codec_study/report/rows.py @@ -81,7 +81,7 @@ def study_row(measurement: Measurement) -> StudyRow: block=measurement.block, dictionary=measurement.dictionary, streams=measurement.streams, - phrases=len(measurement.compressed.phrases), + phrases=measurement.phrases, seconds=measurement.seconds, lossless=measurement.lossless, ) diff --git a/scripts/codec_study/report/run.py b/scripts/codec_study/report/run.py index 6c0faa91f..0c5b7815a 100644 --- a/scripts/codec_study/report/run.py +++ b/scripts/codec_study/report/run.py @@ -1,7 +1,7 @@ import subprocess from datetime import UTC, datetime from pathlib import Path -from typing import Final, List, Optional, Sequence +from typing import Final, Iterator, List, Optional, Sequence, Tuple from codec_study.accounting import rows as accounting from codec_study.manifest import StudyManifest @@ -9,8 +9,9 @@ from codec_study.report import aggregate from codec_study.report import rows as songs from codec_study.report.writers import markdown_table, write_csv -from codec_study.variants.production import BASELINE +from codec_study.variants.production import BASELINE_NAME from codec_study.variants.variant import Variant +from sampletones_player.compression.compressed import CompressedPlanes from sampletones_shared.paths.source import REPOSITORY_ROOT from sampletones_shared.paths.user import USER_PATH_DOCUMENTS @@ -64,12 +65,14 @@ def write_run( variants: The variants every song was encoded under. measurements: Every song under every variant, in the order measured. derived: The strategy-depth measurements drawn from those, reported beside them and - left out of the accounting, which reads each encoding once. + left out of the accounting, which reads each written encoding once. """ reported = (*measurements, *derived) song_rows = [songs.study_row(measurement) for measurement in reported] - group_rows = aggregate.group_rows(reported, BASELINE.name) - accounting_rows = [accounting.account(measurement) for measurement in measurements] + group_rows = aggregate.group_rows(reported, BASELINE_NAME) + accounting_rows = [ + accounting.account(measurement, compressed) for measurement, compressed in _written(measurements) + ] manifest.save(directory / MANIFEST_JSON) write_csv(directory / REPORT_CSV, songs.COLUMNS, [row.cells for row in song_rows]) write_csv(directory / ACCOUNTING_CSV, accounting.COLUMNS, [row.cells for row in accounting_rows]) @@ -81,6 +84,14 @@ def write_run( (directory / REPORT_MARKDOWN).write_text("\n".join(lines), encoding="utf-8") +def _written(measurements: Sequence[Measurement]) -> Iterator[Tuple[Measurement, CompressedPlanes]]: + """The measurements whose streams the driver reads as they stand, beside those streams.""" + for measurement in measurements: + written = measurement.encoding.written + if written is not None: + yield measurement, written + + def _header( manifest: StudyManifest, variants: Sequence[Variant], diff --git a/scripts/codec_study/sandbox/__init__.py b/scripts/codec_study/sandbox/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/scripts/codec_study/sandbox/context.py b/scripts/codec_study/sandbox/context.py new file mode 100644 index 000000000..1e23f6058 --- /dev/null +++ b/scripts/codec_study/sandbox/context.py @@ -0,0 +1,43 @@ +from dataclasses import dataclass +from typing import Tuple + +from codec_study.sandbox.costs import Costs +from sampletones_player.compression.matches.index import PlaneIndex +from sampletones_player.compression.matches.matcher import PhraseMatcher +from sampletones_player.compression.parse.boundaries import Boundaries +from sampletones_player.specification.compression import INITIAL_PLANE_VALUE + + +@dataclass(frozen=True) +class PlaneContext: + """One plane as a grammar reads it: its values, the phrases it may play, and the terms. + + Attributes: + index: The plane, its steps and its runs. + matcher: Which phrases the plane plays at a tick, and for how long. + boundaries: The ticks a token starts on, read either way from every tick. + transposition: Whether a phrase may play at a shift. + defaults: The default count of each phrase, by id, zero where the phrase has none. + costs: The bytes each token takes. + """ + + index: PlaneIndex + matcher: PhraseMatcher + boundaries: Boundaries + transposition: bool + defaults: Tuple[int, ...] + costs: Costs + + def value_before(self, position: int) -> int: + """The value the plane holds as ``position`` is reached, which a hold there keeps. + + The driver seeds every plane before the first token, so the stream's first tick is + reached holding that value like any other. + + Args: + position: The tick. + + Returns: + int: The value. + """ + return self.index.plane[position - 1] if position > 0 else INITIAL_PLANE_VALUE diff --git a/scripts/codec_study/sandbox/costs.py b/scripts/codec_study/sandbox/costs.py new file mode 100644 index 000000000..23d3f1703 --- /dev/null +++ b/scripts/codec_study/sandbox/costs.py @@ -0,0 +1,93 @@ +from dataclasses import dataclass +from typing import Final + +from sampletones_player.compression.dictionary.table import PhraseTable +from sampletones_player.compression.tokens.sizes import hold_size +from sampletones_player.specification.compression import ( + CHEAP_PHRASE_IDS, + OPCODE_SIZE, + PHRASE_COUNT_SIZE, + PHRASE_ESCAPE_SIZE, + TRANSPOSE_SIZE, +) + +VALUE_SIZE: Final[int] = 1 +DEFAULT_COUNT_SIZE: Final[int] = 1 +DEFAULT_FLAG_BITS: Final[int] = 1 + + +@dataclass(frozen=True) +class Costs: + """The bytes each token takes under one grammar. + + A token on the free escape is a phrase opcode followed by a count of zero, which the codec + as it stands never writes, so the opcode's operand is what such a token carries: the sizes + it counts are as many as the cheap ids a phrase opcode names. + + Attributes: + opcode: The bytes an opcode takes. + hold: The bytes a hold takes. + wide_hold: The bytes a wide hold takes. + set_hold: The bytes a set-hold takes. + phrase_count: The bytes a phrase token's count takes. + phrase_escape: The bytes naming a phrase beyond the cheap ids takes. + transpose: The bytes a shift takes. + operands: The operand values below the escape: the cheap ids a phrase opcode names, and + the sizes a token on the escape counts. + default_entry: The bytes a table entry grows by to carry the phrase's default count. + """ + + opcode: int + hold: int + wide_hold: int + set_hold: int + phrase_count: int + phrase_escape: int + transpose: int + operands: int + default_entry: int + + def literal(self, length: int) -> int: + """The bytes a literal of ``length`` values takes.""" + return self.opcode + length + + def phrase( + self, + phrase_id: int, + transpose: int, + *, + default: bool, + ) -> int: + """The bytes a token playing a phrase takes. + + Args: + phrase_id: Position the phrase takes in the table. + transpose: The shift every byte of the phrase is played at. + default: Whether the token plays the phrase's default count and carries none. + + Returns: + int: The bytes the token takes. + """ + escape = self.phrase_escape if phrase_id >= self.operands else 0 + count = 0 if default else self.phrase_count + shift = self.transpose if transpose else 0 + return self.opcode + escape + count + shift + + def dictionary(self, table: PhraseTable) -> int: + """The bytes ``table`` takes in the song block under this grammar.""" + return table.size + self.default_entry * len(table) + + +PRODUCTION_COSTS: Final[Costs] = Costs( + opcode=OPCODE_SIZE, + hold=hold_size(), + wide_hold=OPCODE_SIZE + PHRASE_COUNT_SIZE, + set_hold=OPCODE_SIZE + PHRASE_COUNT_SIZE + VALUE_SIZE, + phrase_count=PHRASE_COUNT_SIZE, + phrase_escape=PHRASE_ESCAPE_SIZE, + transpose=TRANSPOSE_SIZE, + operands=CHEAP_PHRASE_IDS, + default_entry=0, +) +SET_HOLD_BOUND: Final[int] = OPCODE_SIZE + VALUE_SIZE +DEFAULT_COUNT_OPERANDS: Final[int] = (CHEAP_PHRASE_IDS + 1) // (1 << DEFAULT_FLAG_BITS) - 1 diff --git a/scripts/codec_study/sandbox/decode.py b/scripts/codec_study/sandbox/decode.py new file mode 100644 index 000000000..7929484ee --- /dev/null +++ b/scripts/codec_study/sandbox/decode.py @@ -0,0 +1,53 @@ +from typing import Sequence + +from codec_study.sandbox.tokens import Hold, Literal, Play, SetHold, StudyToken, WideHold +from sampletones_player.compression.dictionary.table import PhraseTable +from sampletones_player.specification.binary import BYTE_VALUES +from sampletones_player.specification.compression import INITIAL_PLANE_VALUE + + +def _played( + token: Play, + table: PhraseTable, +) -> bytes: + body = table[token.phrase_id].body + last = len(body) - 1 + return bytes((body[min(offset, last)] + token.transpose) % BYTE_VALUES for offset in range(token.ticks)) + + +def play_tokens( + tokens: Sequence[StudyToken], + table: PhraseTable, + ticks: int, +) -> bytes: + """Plays a plane's tokens back into the values they write, tick by tick. + + This is the reading a driver of the grammar would perform, stated over tokens: a hold + keeps the value reached, a phrase played past its end holds its final value onward, and + every plane opens on the value the driver seeds it to. + + Args: + tokens: The tokens the plane is written as, in the order they are read. + table: The dictionary the tokens name. + ticks: The ticks the song lasts. + + Returns: + bytes: The values the plane writes, one per tick. + """ + values = bytearray() + current = INITIAL_PLANE_VALUE + for token in tokens: + match token: + case Hold() | WideHold(): + played = bytes([current]) * token.ticks + case SetHold(): + played = bytes([token.value]) * token.ticks + case Literal(): + played = token.values + case Play(): + played = _played(token, table) + + current = played[-1] + values.extend(played) + + return bytes(values[:ticks]) diff --git a/scripts/codec_study/sandbox/defaults.py b/scripts/codec_study/sandbox/defaults.py new file mode 100644 index 000000000..e73396316 --- /dev/null +++ b/scripts/codec_study/sandbox/defaults.py @@ -0,0 +1,44 @@ +from collections import Counter +from typing import Final, List, Sequence, Tuple + +from codec_study.sandbox.parse import StudyParse +from codec_study.sandbox.tokens import Play + +NO_DEFAULT: Final[int] = 0 + + +def no_defaults(phrases: int) -> Tuple[int, ...]: + """A default count for none of ``phrases`` phrases.""" + return (NO_DEFAULT,) * phrases + + +def modal_counts( + parses: Sequence[StudyParse], + phrases: int, +) -> Tuple[int, ...]: + """The count each phrase is played at most often across the parses (H4). + + A phrase's default count is the one its tokens would leave unstated most often, so it is + read off the parse as it stands. A phrase the parses never play keeps no default. + + Args: + parses: Every plane's parse. + phrases: The phrases the table holds. + + Returns: + Tuple[int, ...]: The default count of each phrase, by id, zero where it has none. + """ + counters: List[Counter[int]] = [Counter() for _ in range(phrases)] + for parse in parses: + for token in parse.tokens: + if isinstance(token, Play): + counters[token.phrase_id][token.ticks] += 1 + + return tuple(_modal(counter) for counter in counters) + + +def _modal(counter: Counter[int]) -> int: + if not counter: + return NO_DEFAULT + + return min(counter, key=lambda ticks: (-counter[ticks], ticks)) diff --git a/scripts/codec_study/sandbox/edges/__init__.py b/scripts/codec_study/sandbox/edges/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/scripts/codec_study/sandbox/edges/generator.py b/scripts/codec_study/sandbox/edges/generator.py new file mode 100644 index 000000000..f215ea93f --- /dev/null +++ b/scripts/codec_study/sandbox/edges/generator.py @@ -0,0 +1,53 @@ +from typing import Iterable, Protocol + +from codec_study.sandbox.context import PlaneContext +from codec_study.sandbox.shortest import Shortest +from codec_study.sandbox.tokens import StudyToken + + +class EdgeGenerator(Protocol): + """Offers every token of one kind that may start at a tick of a plane.""" + + def __call__( + self, + context: PlaneContext, + shortest: Shortest[StudyToken], + position: int, + reach: int, + *, + holdable: bool, + ) -> None: + """Relaxes the edges of the tokens starting at ``position``. + + Args: + context: The plane and the terms the grammar reads it on. + shortest: The search the edges join. + position: The tick the tokens start on. + reach: The most ticks a token may cover from there. + holdable: Whether a token may keep the value the plane holds as the tick is reached. + """ + + +def offered_lengths( + longest: int, + *, + least: int, + every: bool, +) -> Iterable[int]: + """The lengths a token kind is offered at from one tick. + + The codec as it stands offers the longest token alone, and a grammar asking for every + length offers each one from ``least`` up to it. + + Args: + longest: The most ticks the token may cover. + least: The fewest ticks a token of the kind covers. + every: Whether every length is offered. + + Returns: + Iterable[int]: The lengths, the longest one last. + """ + if every: + return range(least, longest + 1) + + return (longest,) diff --git a/scripts/codec_study/sandbox/edges/holds.py b/scripts/codec_study/sandbox/edges/holds.py new file mode 100644 index 000000000..5139d5d78 --- /dev/null +++ b/scripts/codec_study/sandbox/edges/holds.py @@ -0,0 +1,79 @@ +from typing import Final + +from codec_study.sandbox.context import PlaneContext +from codec_study.sandbox.edges.generator import EdgeGenerator, offered_lengths +from codec_study.sandbox.shortest import Shortest +from codec_study.sandbox.tokens import Hold, StudyToken, WideHold +from sampletones_player.specification.compression import MAX_HOLD_TICKS + +LEAST_HOLD_TICKS: Final[int] = 1 +LEAST_WIDE_BLOCKS: Final[int] = 1 + + +def hold_edges(*, every_length: bool) -> EdgeGenerator: + """Holds, as the codec offers them: over the value the plane holds as the tick is reached. + + Args: + every_length: Whether a hold of every length is offered beside the longest one. + + Returns: + EdgeGenerator: The generator. + """ + + def relax( + context: PlaneContext, + shortest: Shortest[StudyToken], + position: int, + reach: int, + *, + holdable: bool, + ) -> None: + index = context.index + if not holdable or index.plane[position] != context.value_before(position): + return + + longest = min(MAX_HOLD_TICKS, index.runs[position], reach) + cost = shortest.costs[position] + context.costs.hold + for ticks in offered_lengths(longest, least=LEAST_HOLD_TICKS, every=every_length): + if shortest.improves(position + ticks, cost): + shortest.relax(position, position + ticks, cost, Hold(ticks=ticks)) + + return relax + + +def wide_hold_edges(*, every_length: bool) -> EdgeGenerator: + """Wide holds (H1): whole blocks of a plain hold's longest reach, counted by one token. + + A wide hold rides the free escape, so it counts as many blocks as the escape has operand + values, and a run shorter than a block is left to a plain hold. + + Args: + every_length: Whether a wide hold of every block count is offered beside the longest. + + Returns: + EdgeGenerator: The generator. + """ + + def relax( + context: PlaneContext, + shortest: Shortest[StudyToken], + position: int, + reach: int, + *, + holdable: bool, + ) -> None: + index = context.index + if not holdable or index.plane[position] != context.value_before(position): + return + + longest = min(context.costs.operands, index.runs[position] // MAX_HOLD_TICKS, reach // MAX_HOLD_TICKS) + if longest < LEAST_WIDE_BLOCKS: + return + + cost = shortest.costs[position] + context.costs.wide_hold + for blocks in offered_lengths(longest, least=LEAST_WIDE_BLOCKS, every=every_length): + end = position + blocks * MAX_HOLD_TICKS + if shortest.improves(end, cost): + shortest.relax(position, end, cost, WideHold(blocks=blocks)) + + return relax diff --git a/scripts/codec_study/sandbox/edges/literals.py b/scripts/codec_study/sandbox/edges/literals.py new file mode 100644 index 000000000..dd38994ff --- /dev/null +++ b/scripts/codec_study/sandbox/edges/literals.py @@ -0,0 +1,29 @@ +from codec_study.sandbox.context import PlaneContext +from codec_study.sandbox.shortest import Shortest +from codec_study.sandbox.tokens import Literal, StudyToken +from sampletones_player.compression.parse.literals import LiteralWindow + + +def relax_literal( + context: PlaneContext, + shortest: Shortest[StudyToken], + window: LiteralWindow, + position: int, + earliest: int, +) -> None: + """Spelling the values out reaches ``position`` from wherever that costs least. + + Literals are the one token every grammar keeps, priced as the codec prices them: an opcode + and the values, from the start the window finds cheapest. + + Args: + context: The plane and the terms the grammar reads it on. + shortest: The search the edge joins. + window: The starts a literal ending here may take, the cheapest kept at the front. + position: The tick the literal ends at. + earliest: The earliest tick the literal may start on. + """ + start = window.cheapest(position, earliest) + cost = shortest.costs[start] + context.costs.literal(position - start) + if shortest.improves(position, cost): + shortest.relax(start, position, cost, Literal(values=context.index.plane[start:position])) diff --git a/scripts/codec_study/sandbox/edges/phrases.py b/scripts/codec_study/sandbox/edges/phrases.py new file mode 100644 index 000000000..650f9a76d --- /dev/null +++ b/scripts/codec_study/sandbox/edges/phrases.py @@ -0,0 +1,57 @@ +from codec_study.sandbox.context import PlaneContext +from codec_study.sandbox.edges.generator import EdgeGenerator +from codec_study.sandbox.shortest import Shortest +from codec_study.sandbox.tokens import Play, StudyToken +from sampletones_player.specification.compression import MAX_PHRASE_TICKS + + +def phrase_edges(*, defaults: bool) -> EdgeGenerator: + """Phrases, as the codec offers them: each one the plane plays from the tick, played whole. + + With defaults on, a phrase played at least its default count from the tick is offered at + that count too, as the token that carries no count of its own (H4). + + Args: + defaults: Whether the default-count token is offered. + + Returns: + EdgeGenerator: The generator. + """ + + def relax( + context: PlaneContext, + shortest: Shortest[StudyToken], + position: int, + reach: int, + *, + holdable: bool, + ) -> None: + del holdable + cost = shortest.costs[position] + costs = context.costs + for phrase_id, ticks, transpose in context.matcher.matches( + position, + min(MAX_PHRASE_TICKS, reach), + transposition=context.transposition, + ): + stated = cost + costs.phrase(phrase_id, transpose, default=False) + if shortest.improves(position + ticks, stated): + shortest.relax( + position, + position + ticks, + stated, + Play(phrase_id=phrase_id, ticks=ticks, transpose=transpose, default=False), + ) + + count = context.defaults[phrase_id] if defaults else 0 + if 0 < count <= ticks: + unstated = cost + costs.phrase(phrase_id, transpose, default=True) + if shortest.improves(position + count, unstated): + shortest.relax( + position, + position + count, + unstated, + Play(phrase_id=phrase_id, ticks=count, transpose=transpose, default=True), + ) + + return relax diff --git a/scripts/codec_study/sandbox/edges/set_hold.py b/scripts/codec_study/sandbox/edges/set_hold.py new file mode 100644 index 000000000..cfe04d19c --- /dev/null +++ b/scripts/codec_study/sandbox/edges/set_hold.py @@ -0,0 +1,45 @@ +from typing import Final + +from codec_study.sandbox.context import PlaneContext +from codec_study.sandbox.edges.generator import EdgeGenerator, offered_lengths +from codec_study.sandbox.shortest import Shortest +from codec_study.sandbox.tokens import SetHold, StudyToken + +LEAST_SET_HOLD_TICKS: Final[int] = 2 + + +def set_hold_edges(*, every_length: bool) -> EdgeGenerator: + """Set-holds (H3): one token taking a value and keeping it for a run of ticks. + + The token names its value, so it starts anywhere a run does, the stream's first tick and a + loop entry included. A run of one tick is a literal's, which costs the same and carries + more. + + Args: + every_length: Whether a set-hold of every length is offered beside the longest one. + + Returns: + EdgeGenerator: The generator. + """ + + def relax( + context: PlaneContext, + shortest: Shortest[StudyToken], + position: int, + reach: int, + *, + holdable: bool, + ) -> None: + del holdable + index = context.index + longest = min(context.costs.operands, index.runs[position], reach) + if longest < LEAST_SET_HOLD_TICKS: + return + + value = index.plane[position] + cost = shortest.costs[position] + context.costs.set_hold + for ticks in offered_lengths(longest, least=LEAST_SET_HOLD_TICKS, every=every_length): + if shortest.improves(position + ticks, cost): + shortest.relax(position, position + ticks, cost, SetHold(value=value, ticks=ticks)) + + return relax diff --git a/scripts/codec_study/sandbox/encode.py b/scripts/codec_study/sandbox/encode.py new file mode 100644 index 000000000..bfd1c1f34 --- /dev/null +++ b/scripts/codec_study/sandbox/encode.py @@ -0,0 +1,80 @@ +from time import process_time +from typing import Final, Sequence, Tuple + +from codec_study.measure import Encoding +from codec_study.sandbox.defaults import modal_counts, no_defaults +from codec_study.sandbox.grammar import Grammar +from codec_study.sandbox.parse import StudyParse, parse_plane +from codec_study.sandbox.reference import Reference +from codec_study.sandbox.verify import plays_back + +DEFAULT_ROUNDS: Final[int] = 3 + + +def encode_grammar( + reference: Reference, + grammar: Grammar, +) -> Encoding: + """Prices a song under a grammar, over the dictionary the production codec settled on. + + Args: + reference: The song, its dictionary and its matches. + grammar: The grammar. + + Returns: + Encoding: The bytes each stream would take, timed, and whether the tokens play back. + """ + started = process_time() + parses = _parses(reference, grammar) + seconds = process_time() - started + return Encoding( + phrases=len(reference.table), + dictionary=grammar.costs.dictionary(reference.table), + streams=tuple(parse.size for parse in parses), + seconds=seconds, + lossless=plays_back(parses, reference.table, reference.song.planes), + written=None, + ) + + +def _parse_all( + reference: Reference, + grammar: Grammar, + defaults: Sequence[int], +) -> Tuple[StudyParse, ...]: + return tuple(parse_plane(context, grammar) for context in reference.contexts(grammar.costs, defaults)) + + +def _streams(parses: Sequence[StudyParse]) -> int: + return sum(parse.size for parse in parses) + + +def _parses( + reference: Reference, + grammar: Grammar, +) -> Tuple[StudyParse, ...]: + """Every plane under the grammar, its default counts settled where it carries them. + + A default count is read off the parse it serves, so the first reading takes the counts the + baseline parse plays each phrase at, and each further round reads them off the parse that + used them, stopping where they stand still or stop paying. + """ + phrases = len(reference.table) + if not grammar.default_counts: + return _parse_all(reference, grammar, no_defaults(phrases)) + + defaults = modal_counts(reference.baseline, phrases) + parses = _parse_all(reference, grammar, defaults) + for _ in range(DEFAULT_ROUNDS - 1): + refined = modal_counts(parses, phrases) + if refined == defaults: + break + + trial = _parse_all(reference, grammar, refined) + if _streams(trial) >= _streams(parses): + break + + defaults = refined + parses = trial + + return parses diff --git a/scripts/codec_study/sandbox/grammar.py b/scripts/codec_study/sandbox/grammar.py new file mode 100644 index 000000000..a0c12b51a --- /dev/null +++ b/scripts/codec_study/sandbox/grammar.py @@ -0,0 +1,40 @@ +from dataclasses import dataclass +from typing import Final + +from codec_study.sandbox.costs import PRODUCTION_COSTS, Costs + + +@dataclass(frozen=True) +class Grammar: + """One token grammar the sandbox prices a plane under. + + The baseline grammar is the codec as it stands, and every other one switches a change on + over it, so what a change earns is read as the difference between the two on the same plane + and the same dictionary. + + Attributes: + every_length: Whether a hold of every length is offered beside the longest one (H9a). + start_hold: Whether a hold may open a stream, over the value the driver seeds every + plane to (H9b). + wide_hold: Whether a wide hold covers blocks of a plain hold's longest reach at once (H1). + set_hold: Whether one token sets a value and holds it (H3). + default_counts: Whether a phrase carries a default count a token may leave unstated (H4). + costs: The bytes each token takes. + """ + + every_length: bool + start_hold: bool + wide_hold: bool + set_hold: bool + default_counts: bool + costs: Costs + + +BASELINE_GRAMMAR: Final[Grammar] = Grammar( + every_length=False, + start_hold=False, + wide_hold=False, + set_hold=False, + default_counts=False, + costs=PRODUCTION_COSTS, +) diff --git a/scripts/codec_study/sandbox/parse.py b/scripts/codec_study/sandbox/parse.py new file mode 100644 index 000000000..ad3526e0a --- /dev/null +++ b/scripts/codec_study/sandbox/parse.py @@ -0,0 +1,117 @@ +from dataclasses import dataclass +from typing import List, Sequence, Tuple + +from codec_study.sandbox.context import PlaneContext +from codec_study.sandbox.edges.generator import EdgeGenerator +from codec_study.sandbox.edges.holds import hold_edges, wide_hold_edges +from codec_study.sandbox.edges.literals import relax_literal +from codec_study.sandbox.edges.phrases import phrase_edges +from codec_study.sandbox.edges.set_hold import set_hold_edges +from codec_study.sandbox.grammar import Grammar +from codec_study.sandbox.shortest import Shortest +from codec_study.sandbox.tokens import StudyToken +from sampletones_player.compression.parse.literals import LiteralWindow +from sampletones_player.specification.compression import MAX_LITERAL_BYTES + + +@dataclass(frozen=True) +class StudyParse: + """A plane read as the tokens of one grammar, alongside what each of its prefixes costs. + + Attributes: + tokens: The tokens the plane is written as, in the order they are read. + costs: The bytes each prefix of the plane takes, the whole plane's cost last. + """ + + tokens: Tuple[StudyToken, ...] + costs: Tuple[int, ...] + + @property + def size(self) -> int: + """The bytes the plane's token stream takes.""" + return self.costs[-1] + + +def generators(grammar: Grammar) -> Tuple[EdgeGenerator, ...]: + """The token kinds a grammar offers from a tick, each as the edges it adds to the search. + + Args: + grammar: The grammar. + + Returns: + Tuple[EdgeGenerator, ...]: The generators, holds first and phrases last. + """ + chosen: List[EdgeGenerator] = [hold_edges(every_length=grammar.every_length)] + if grammar.wide_hold: + chosen.append(wide_hold_edges(every_length=grammar.every_length)) + + if grammar.set_hold: + chosen.append(set_hold_edges(every_length=grammar.every_length)) + + chosen.append(phrase_edges(defaults=grammar.default_counts)) + return tuple(chosen) + + +def _relax_forward( + offered: Sequence[EdgeGenerator], + context: PlaneContext, + shortest: Shortest[StudyToken], + position: int, + reach: int, + *, + holdable: bool, +) -> None: + for generator in offered: + generator( + context, + shortest, + position, + reach, + holdable=holdable, + ) + + +def parse_plane( + context: PlaneContext, + grammar: Grammar, +) -> StudyParse: + """Reads a plane as the cheapest token stream ``grammar`` allows. + + This is the production parse with the token kinds plugged in: literals reach every tick + from their cheapest start, and the grammar's other kinds are offered forward from each tick + a token may start on. + + Args: + context: The plane and the terms the grammar reads it on. + grammar: The grammar. + + Returns: + StudyParse: The tokens the plane is written as, and what each of its prefixes costs. + """ + ticks = context.index.ticks + entries = context.boundaries + previous = entries.previous + following = entries.following + offered = generators(grammar) + shortest: Shortest[StudyToken] = Shortest.across(ticks) + window = LiteralWindow(shortest.costs) + _relax_forward(offered, context, shortest, 0, following[0], holdable=grammar.start_hold) + for position in range(1, ticks + 1): + relax_literal( + context, + shortest, + window, + position, + max(position - MAX_LITERAL_BYTES, previous[position]), + ) + if position < ticks: + _relax_forward( + offered, + context, + shortest, + position, + following[position] - position, + holdable=position not in entries.entries, + ) + + return StudyParse(tokens=shortest.walk(ticks), costs=tuple(shortest.costs)) diff --git a/scripts/codec_study/sandbox/reference.py b/scripts/codec_study/sandbox/reference.py new file mode 100644 index 000000000..122544064 --- /dev/null +++ b/scripts/codec_study/sandbox/reference.py @@ -0,0 +1,108 @@ +from dataclasses import dataclass +from typing import Final, FrozenSet, Sequence, Tuple + +from codec_study.corpus.song import StudySong +from codec_study.sandbox.context import PlaneContext +from codec_study.sandbox.costs import PRODUCTION_COSTS, Costs +from codec_study.sandbox.defaults import no_defaults +from codec_study.sandbox.grammar import BASELINE_GRAMMAR +from codec_study.sandbox.parse import StudyParse, parse_plane +from codec_study.sandbox.verify import verify_baseline +from sampletones_player.compression.compressed import CompressedPlanes +from sampletones_player.compression.dictionary.table import PhraseTable +from sampletones_player.compression.encode import STREAM_START +from sampletones_player.compression.matches.cache import MatchCache +from sampletones_player.compression.matches.index import PlaneIndex +from sampletones_player.compression.matches.matcher import PhraseMatcher +from sampletones_player.compression.options import EVERY_LAYER +from sampletones_player.compression.parse.boundaries import Boundaries + +STREAM_ENTRIES: Final[FrozenSet[int]] = frozenset({STREAM_START}) + + +@dataclass(frozen=True) +class Reference: + """One song as every grammar is priced against it. + + The dictionary is the one the production codec settled on, held fixed across the grammars: + what a grammar earns is read on the streams alone, and how the search and the settling + would answer a new grammar is left to its production layer to measure. + + Attributes: + song: The song. + table: The dictionary the production codec settled on. + cache: What each phrase plays against each plane, shared by every grammar's parse. + baseline: Every plane under the baseline grammar, held to the production streams. + """ + + song: StudySong + table: PhraseTable + cache: MatchCache + baseline: Tuple[StudyParse, ...] + + def contexts( + self, + costs: Costs, + defaults: Sequence[int], + ) -> Tuple[PlaneContext, ...]: + """Every plane of the song under one grammar's terms. + + Args: + costs: The bytes each token takes. + defaults: The default count of each phrase, by id. + + Returns: + Tuple[PlaneContext, ...]: One context per plane, in song-block order. + """ + return _contexts(self.cache, self.table, costs, defaults) + + +def _contexts( + cache: MatchCache, + table: PhraseTable, + costs: Costs, + defaults: Sequence[int], +) -> Tuple[PlaneContext, ...]: + contexts = [] + for plane, index in enumerate(cache.indices): + contexts.append( + PlaneContext( + index=index, + matcher=PhraseMatcher(table, plane, cache), + boundaries=Boundaries.across(index.ticks, STREAM_ENTRIES), + transposition=EVERY_LAYER.transposition, + defaults=tuple(defaults), + costs=costs, + ) + ) + + return tuple(contexts) + + +def reference( + song: StudySong, + compressed: CompressedPlanes, +) -> Reference: + """Reads a song into what every grammar is priced against, proving the parser on the way. + + Args: + song: The song. + compressed: The production encoding of the song. + + Returns: + Reference: The song, its dictionary, its matches and its baseline parse. + + Raises: + ValueError: If the baseline grammar prices a plane differently from the codec's stream. + """ + cache = MatchCache(PlaneIndex.from_plane(plane) for plane in song.planes.planes) + table = compressed.phrases + contexts = _contexts(cache, table, PRODUCTION_COSTS, no_defaults(len(table))) + baseline = tuple(parse_plane(context, BASELINE_GRAMMAR) for context in contexts) + verify_baseline(baseline, compressed) + return Reference( + song=song, + table=table, + cache=cache, + baseline=baseline, + ) diff --git a/scripts/codec_study/sandbox/shortest.py b/scripts/codec_study/sandbox/shortest.py new file mode 100644 index 000000000..8bfa57f44 --- /dev/null +++ b/scripts/codec_study/sandbox/shortest.py @@ -0,0 +1,86 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import Final, Generic, List, Optional, Tuple, TypeVar + +UNREACHED: Final[int] = -1 + +Token = TypeVar("Token") + + +@dataclass +class Shortest(Generic[Token]): + """The cheapest way found so far of reaching each tick of a plane, over any token kind. + + Every token is an edge from the tick it starts on to the tick after the ones it covers, and + its cost is the bytes it takes, so the cheapest path across the plane is its encoding. This + is the production search stated over whichever tokens a grammar offers. + + Attributes: + costs: The bytes reaching each tick takes, ticks not yet reached holding ``UNREACHED``. + origins: The tick each one was reached from. + tokens: The token each tick was reached by. + """ + + costs: List[int] + origins: List[int] + tokens: List[Optional[Token]] + + @classmethod + def across(cls, ticks: int) -> Shortest[Token]: + """Opens a search over a plane of ``ticks`` ticks, its first tick free to reach.""" + return cls( + costs=[0] + [UNREACHED] * ticks, + origins=[0] * (ticks + 1), + tokens=[None] * (ticks + 1), + ) + + def improves(self, end: int, cost: int) -> bool: + """Whether reaching ``end`` for ``cost`` beats what it has been reached for so far.""" + return self.costs[end] == UNREACHED or cost < self.costs[end] + + def relax( + self, + start: int, + end: int, + cost: int, + token: Token, + ) -> None: + """Takes a token as the way to reach ``end`` where it is the cheapest one found. + + Args: + start: The tick the token starts on. + end: The tick following the ones the token covers. + cost: The bytes reaching ``end`` through this token takes. + token: The token covering the ticks between the two. + """ + if self.costs[end] != UNREACHED and self.costs[end] <= cost: + return + + self.costs[end] = cost + self.origins[end] = start + self.tokens[end] = token + + def walk(self, ticks: int) -> Tuple[Token, ...]: + """Reads the tokens of the cheapest path back from ``ticks`` to the plane's first tick. + + Args: + ticks: The tick the path ends at, which is the ticks the plane covers. + + Returns: + Tuple[Token, ...]: The tokens, in the order they are read. + + Raises: + ValueError: If a tick along the path was never reached. + """ + read: List[Token] = [] + position = ticks + while position > 0: + token = self.tokens[position] + if token is None: + raise ValueError(f"tick {position} of the plane was left unreachable") + + read.append(token) + position = self.origins[position] + + return tuple(reversed(read)) diff --git a/scripts/codec_study/sandbox/tokens.py b/scripts/codec_study/sandbox/tokens.py new file mode 100644 index 000000000..4accffcb2 --- /dev/null +++ b/scripts/codec_study/sandbox/tokens.py @@ -0,0 +1,59 @@ +from dataclasses import dataclass +from typing import Union + +from sampletones_player.specification.compression import MAX_HOLD_TICKS + + +@dataclass(frozen=True) +class Hold: + """The plane keeps the value it reached, for ``ticks`` ticks.""" + + ticks: int + + +@dataclass(frozen=True) +class WideHold: + """The plane keeps the value it reached for ``blocks`` blocks of a plain hold's longest reach (H1).""" + + blocks: int + + @property + def ticks(self) -> int: + """The ticks the token covers.""" + return self.blocks * MAX_HOLD_TICKS + + +@dataclass(frozen=True) +class SetHold: + """The plane takes ``value`` and keeps it for ``ticks`` ticks (H3).""" + + value: int + ticks: int + + +@dataclass(frozen=True) +class Literal: + """The plane takes the values verbatim, one per tick.""" + + values: bytes + + @property + def ticks(self) -> int: + """The ticks the token covers.""" + return len(self.values) + + +@dataclass(frozen=True) +class Play: + """The plane plays a phrase from the table, shifted by ``transpose``, for ``ticks`` ticks. + + A default play covers the phrase's own default count and carries no count of its own (H4). + """ + + phrase_id: int + ticks: int + transpose: int + default: bool + + +StudyToken = Union[Hold, WideHold, SetHold, Literal, Play] diff --git a/scripts/codec_study/sandbox/verify.py b/scripts/codec_study/sandbox/verify.py new file mode 100644 index 000000000..11db3bf8b --- /dev/null +++ b/scripts/codec_study/sandbox/verify.py @@ -0,0 +1,51 @@ +from typing import Sequence + +from codec_study.sandbox.decode import play_tokens +from codec_study.sandbox.parse import StudyParse +from sampletones_player.compression.compressed import CompressedPlanes +from sampletones_player.compression.dictionary.table import PhraseTable +from sampletones_player.compression.planes.order import PlaneOrder +from sampletones_player.compression.planes.song import SongPlanes + + +def verify_baseline( + parses: Sequence[StudyParse], + compressed: CompressedPlanes, +) -> None: + """Holds the sandbox's baseline grammar to the streams the production codec wrote. + + The sandbox prices every grammar on its own parser, so the parser is proven on the one + grammar whose answer is known: each plane under the baseline grammar costs exactly the + bytes the production stream takes, or the study is reading the planes wrong. + + Args: + parses: Every plane under the baseline grammar. + compressed: The production encoding of the same planes over the same dictionary. + + Raises: + ValueError: If a plane is priced differently from the stream the codec wrote. + """ + for name, parse, stream in zip(PlaneOrder.names(), parses, compressed.streams): + if parse.size != len(stream): + raise ValueError( + f"the sandbox reads {name} as {parse.size} bytes where the codec wrote {len(stream)}; " + "the baseline grammar has parted from the production parser" + ) + + +def plays_back( + parses: Sequence[StudyParse], + table: PhraseTable, + planes: SongPlanes, +) -> bool: + """Whether every plane's tokens play back to the plane they were written from. + + Args: + parses: Every plane's parse, in the order the song block writes them. + table: The dictionary the tokens name. + planes: The planes the parses were read from. + + Returns: + bool: Whether the encoding is lossless. + """ + return all(play_tokens(parse.tokens, table, planes.ticks) == plane for parse, plane in zip(parses, planes.planes)) diff --git a/scripts/codec_study/variants/baselines.py b/scripts/codec_study/variants/baselines.py new file mode 100644 index 000000000..1e899f4ce --- /dev/null +++ b/scripts/codec_study/variants/baselines.py @@ -0,0 +1,91 @@ +from time import process_time +from typing import Dict + +from codec_study.corpus.song import StudySong +from codec_study.measure import Encoding, production_encoding +from codec_study.sandbox.reference import Reference, reference +from codec_study.variants.production import BASELINE_NAME, compress_baseline +from codec_study.variants.variant import Variant, VariantKind +from sampletones_player.compression.compressed import CompressedPlanes + + +class Baselines: + """The production encoding of each song, taken once and read by every variant built on it. + + A run measures a song under every variant before moving to the next, so one song's + encoding is kept at a time: the baseline variant writes it, and the sandbox reads the + dictionary and the planes' matches from there in place of encoding the song again. A + song asked for before its baseline ran is encoded on the spot. + """ + + def __init__(self) -> None: + self._compressed: Dict[str, CompressedPlanes] = {} + self._references: Dict[str, Reference] = {} + + def encode(self, song: StudySong) -> Encoding: + """Encodes a song as the export does today, keeping the result for the sandbox. + + Args: + song: The song to encode. + + Returns: + Encoding: The encoding, its streams kept as written. + """ + started = process_time() + compressed = compress_baseline(song) + seconds = process_time() - started + self._compressed = {song.name: compressed} + self._references = {} + return production_encoding(song, compressed, seconds) + + def compressed(self, song: StudySong) -> CompressedPlanes: + """The production encoding of ``song``. + + Args: + song: The song. + + Returns: + CompressedPlanes: The dictionary and the streams the codec wrote. + """ + remembered = self._compressed.get(song.name) + if remembered is None: + remembered = compress_baseline(song) + self._compressed = {song.name: remembered} + self._references = {} + + return remembered + + def reference(self, song: StudySong) -> Reference: + """What every grammar prices ``song`` against. + + Args: + song: The song. + + Returns: + Reference: The song, its dictionary, its matches and its baseline parse. + """ + remembered = self._references.get(song.name) + if remembered is None: + remembered = reference(song, self.compressed(song)) + self._references = {song.name: remembered} + + return remembered + + +def baseline_variant(baselines: Baselines) -> Variant: + """The codec as it stands, which every other variant is held against. + + Args: + baselines: Where the encodings are kept for the variants built on them. + + Returns: + Variant: The baseline. + """ + return Variant( + name=BASELINE_NAME, + hypothesis="", + kind=VariantKind.BASELINE, + note="", + encode=baselines.encode, + needs_seeds=False, + ) diff --git a/scripts/codec_study/variants/production.py b/scripts/codec_study/variants/production.py index 1d7d3f7e1..19dc85118 100644 --- a/scripts/codec_study/variants/production.py +++ b/scripts/codec_study/variants/production.py @@ -1,7 +1,8 @@ +from time import process_time from typing import Callable, Final, Sequence, Tuple from codec_study.corpus.song import StudySong -from codec_study.measure import Encoder +from codec_study.measure import Encoder, Encoding, production_encoding from codec_study.variants.seeds import split, trimmed, whole_and_split from codec_study.variants.variant import Variant, VariantKind from sampletones_player.compression.budget import DEFAULT_SEARCH_BUDGET, SearchBudget @@ -25,16 +26,16 @@ DEEP_CONFIRMED: Final[int] = 8 -def encode_production( +def compress( song: StudySong, *, seeds: Sequence[Phrase], budget: SearchBudget, ) -> CompressedPlanes: - """Encodes a song as the export does, every layer on, over the seeds and budget given. + """Compresses a song as the export does, every layer on, over the seeds and budget given. Args: - song: The song to encode. + song: The song to compress. seeds: The phrases offered to the dictionary. budget: How much work the search spends. @@ -50,16 +51,37 @@ def encode_production( ) -def encode_baseline(song: StudySong) -> CompressedPlanes: - """Encodes a song as the export does today. +def compress_baseline(song: StudySong) -> CompressedPlanes: + """Compresses a song as the export does today. Args: - song: The song to encode. + song: The song to compress. Returns: CompressedPlanes: The dictionary and the token streams. """ - return encode_production(song, seeds=song.seeds, budget=DEFAULT_SEARCH_BUDGET) + return compress(song, seeds=song.seeds, budget=DEFAULT_SEARCH_BUDGET) + + +def encode_production( + song: StudySong, + *, + seeds: Sequence[Phrase], + budget: SearchBudget, +) -> Encoding: + """Encodes a song as the export does, timing the run and playing the result back. + + Args: + song: The song to encode. + seeds: The phrases offered to the dictionary. + budget: How much work the search spends. + + Returns: + Encoding: The encoding, its streams kept as written. + """ + started = process_time() + compressed = compress(song, seeds=seeds, budget=budget) + return production_encoding(song, compressed, process_time() - started) def seed_encoder(transform: SeedTransform) -> Encoder: @@ -72,7 +94,7 @@ def seed_encoder(transform: SeedTransform) -> Encoder: Encoder: The encoder, searching at the default budget. """ - def encode(song: StudySong) -> CompressedPlanes: + def encode(song: StudySong) -> Encoding: return encode_production(song, seeds=transform(song.seeds), budget=DEFAULT_SEARCH_BUDGET) return encode @@ -88,7 +110,7 @@ def budget_encoder(budget: SearchBudget) -> Encoder: Encoder: The encoder. """ - def encode(song: StudySong) -> CompressedPlanes: + def encode(song: StudySong) -> Encoding: return encode_production(song, seeds=song.seeds, budget=budget) return encode @@ -130,15 +152,6 @@ def _both(threshold: int) -> SeedTransform: return lambda seeds: whole_and_split(seeds, threshold) -BASELINE: Final[Variant] = Variant( - name=BASELINE_NAME, - hypothesis="", - kind=VariantKind.BASELINE, - note="", - encode=encode_baseline, - needs_seeds=False, -) - SEED_VARIANTS: Final[Tuple[Variant, ...]] = ( _seed_variant("seeds-trimmed", trimmed), *(_seed_variant(f"seeds-split-{threshold}", _splitter(threshold)) for threshold in SPLIT_THRESHOLDS), diff --git a/scripts/codec_study/variants/registry.py b/scripts/codec_study/variants/registry.py index f10f5d27e..405daf45c 100644 --- a/scripts/codec_study/variants/registry.py +++ b/scripts/codec_study/variants/registry.py @@ -1,19 +1,40 @@ from typing import Dict, Final, Sequence, Tuple -from codec_study.variants.production import BASELINE, BUDGET_VARIANTS, SEED_VARIANTS +from codec_study.variants.baselines import Baselines, baseline_variant +from codec_study.variants.production import BASELINE_NAME, BUDGET_VARIANTS, SEED_VARIANTS +from codec_study.variants.sandbox import grammar_variants from codec_study.variants.variant import Variant EVERY_VARIANT: Final[str] = "all" -VARIANTS: Final[Dict[str, Variant]] = { - variant.name: variant for variant in (BASELINE, *SEED_VARIANTS, *BUDGET_VARIANTS) -} -def selected_variants(names: Sequence[str]) -> Tuple[Variant, ...]: +def variants(baselines: Baselines) -> Dict[str, Variant]: + """Every variant a run may encode under, by name, the baseline first. + + Args: + baselines: Where the production encodings are kept for the variants built on them. + + Returns: + Dict[str, Variant]: The variants, in the order a run encodes them. + """ + every = ( + baseline_variant(baselines), + *SEED_VARIANTS, + *BUDGET_VARIANTS, + *grammar_variants(baselines), + ) + return {variant.name: variant for variant in every} + + +def selected_variants( + names: Sequence[str], + baselines: Baselines, +) -> Tuple[Variant, ...]: """The variants a run encodes every song under, the baseline always first. Args: names: The names the run asks for, or ``all`` for every registered variant. + baselines: Where the production encodings are kept for the variants built on them. Returns: Tuple[Variant, ...]: The baseline, then the named variants in the order given. @@ -21,12 +42,13 @@ def selected_variants(names: Sequence[str]) -> Tuple[Variant, ...]: Raises: KeyError: If a name is registered to no variant. """ + registered = variants(baselines) if EVERY_VARIANT in names: - return tuple(VARIANTS.values()) + return tuple(registered.values()) - unknown = [name for name in names if name not in VARIANTS] + unknown = [name for name in names if name not in registered] if unknown: - raise KeyError(f"no variant is called {', '.join(unknown)}; the registry holds {', '.join(VARIANTS)}") + raise KeyError(f"no variant is called {', '.join(unknown)}; the registry holds {', '.join(registered)}") - chosen = [VARIANTS[name] for name in names if name != BASELINE.name] - return (BASELINE, *chosen) + chosen = [registered[name] for name in names if name != BASELINE_NAME] + return (registered[BASELINE_NAME], *chosen) diff --git a/scripts/codec_study/variants/sandbox.py b/scripts/codec_study/variants/sandbox.py new file mode 100644 index 000000000..7a7aa8b32 --- /dev/null +++ b/scripts/codec_study/variants/sandbox.py @@ -0,0 +1,178 @@ +from dataclasses import replace +from typing import Final, NamedTuple, Tuple + +from codec_study.corpus.song import StudySong +from codec_study.measure import Encoder, Encoding +from codec_study.sandbox.costs import ( + DEFAULT_COUNT_OPERANDS, + DEFAULT_COUNT_SIZE, + PRODUCTION_COSTS, + SET_HOLD_BOUND, + Costs, +) +from codec_study.sandbox.encode import encode_grammar +from codec_study.sandbox.grammar import BASELINE_GRAMMAR, Grammar +from codec_study.variants.baselines import Baselines +from codec_study.variants.variant import Variant, VariantKind + +WIDE_HOLD: Final[str] = "H1" +SET_HOLD: Final[str] = "H3" +DEFAULT_COUNT: Final[str] = "H4" +HOLD_EDGES: Final[str] = "H9" +DRIVER_UNCHANGED: Final[str] = "driver unchanged" +WIDE_HOLD_NOTE: Final[str] = ( + "a phrase opcode with a zero count; the block count can sit in the idle shift field and " + "reload in the token fetch, the tick path unchanged" +) +SET_HOLD_NOTE: Final[str] = ( + "a transposed-phrase opcode with a zero count and the value behind it, three bytes; one compare in the token fetch" +) +SET_HOLD_BOUND_NOTE: Final[str] = ( + "a two-byte set-hold needs an opcode of its own, so this bounds what any reallocation of the opcode space reaches" +) +DEFAULT_COUNT_NOTE: Final[str] = ( + "one operand bit says the count is the phrase's own, leaving 31 cheap ids; each table entry " + "grows by a byte, read once when a phrase is set" +) +COMBINED_NOTE: Final[str] = "every change the free escape carries at once, the escape counting 31 blocks and 31 ticks" + +DEFAULT_COUNT_COSTS: Final[Costs] = replace( + PRODUCTION_COSTS, + operands=DEFAULT_COUNT_OPERANDS, + default_entry=DEFAULT_COUNT_SIZE, +) +SET_HOLD_BOUND_COSTS: Final[Costs] = replace(PRODUCTION_COSTS, set_hold=SET_HOLD_BOUND) + + +class GrammarVariant(NamedTuple): + """One grammar the sandbox prices, and how the report names it. + + Attributes: + name: What the variant is called in a report. + hypothesis: The hypothesis the variant measures, by its label in the plan. + kind: What the variant changes. + note: What the driver would have to do. + grammar: The grammar. + """ + + name: str + hypothesis: str + kind: VariantKind + note: str + grammar: Grammar + + +GRAMMAR_VARIANTS: Final[Tuple[GrammarVariant, ...]] = ( + GrammarVariant( + name="start-hold", + hypothesis=HOLD_EDGES, + kind=VariantKind.ENCODER, + note=f"{DRIVER_UNCHANGED}: every plane is seeded before its first token, so a hold may open a stream", + grammar=replace(BASELINE_GRAMMAR, start_hold=True), + ), + GrammarVariant( + name="every-hold", + hypothesis=HOLD_EDGES, + kind=VariantKind.ENCODER, + note=DRIVER_UNCHANGED, + grammar=replace(BASELINE_GRAMMAR, every_length=True), + ), + GrammarVariant( + name="wide-hold", + hypothesis=WIDE_HOLD, + kind=VariantKind.FORMAT, + note=WIDE_HOLD_NOTE, + grammar=replace(BASELINE_GRAMMAR, wide_hold=True), + ), + GrammarVariant( + name="wide-hold+start-hold", + hypothesis=f"{WIDE_HOLD}+{HOLD_EDGES}", + kind=VariantKind.FORMAT, + note=WIDE_HOLD_NOTE, + grammar=replace(BASELINE_GRAMMAR, wide_hold=True, start_hold=True), + ), + GrammarVariant( + name="set-hold-3", + hypothesis=SET_HOLD, + kind=VariantKind.FORMAT, + note=SET_HOLD_NOTE, + grammar=replace(BASELINE_GRAMMAR, set_hold=True), + ), + GrammarVariant( + name="set-hold-2", + hypothesis=SET_HOLD, + kind=VariantKind.FORMAT, + note=SET_HOLD_BOUND_NOTE, + grammar=replace(BASELINE_GRAMMAR, set_hold=True, costs=SET_HOLD_BOUND_COSTS), + ), + GrammarVariant( + name="default-count", + hypothesis=DEFAULT_COUNT, + kind=VariantKind.FORMAT, + note=DEFAULT_COUNT_NOTE, + grammar=replace(BASELINE_GRAMMAR, default_counts=True, costs=DEFAULT_COUNT_COSTS), + ), + GrammarVariant( + name="wide-hold+default-count", + hypothesis=f"{WIDE_HOLD}+{DEFAULT_COUNT}", + kind=VariantKind.FORMAT, + note=f"{WIDE_HOLD_NOTE}; {DEFAULT_COUNT_NOTE}", + grammar=replace(BASELINE_GRAMMAR, wide_hold=True, default_counts=True, costs=DEFAULT_COUNT_COSTS), + ), + GrammarVariant( + name="escape-grammar", + hypothesis=f"{WIDE_HOLD}+{SET_HOLD}+{DEFAULT_COUNT}+{HOLD_EDGES}", + kind=VariantKind.FORMAT, + note=COMBINED_NOTE, + grammar=replace( + BASELINE_GRAMMAR, + start_hold=True, + wide_hold=True, + set_hold=True, + default_counts=True, + costs=DEFAULT_COUNT_COSTS, + ), + ), +) + + +def grammar_encoder( + baselines: Baselines, + grammar: Grammar, +) -> Encoder: + """An encoder pricing a song under ``grammar`` over its production dictionary. + + Args: + baselines: Where the production encodings are kept. + grammar: The grammar. + + Returns: + Encoder: The encoder. + """ + + def encode(song: StudySong) -> Encoding: + return encode_grammar(baselines.reference(song), grammar) + + return encode + + +def grammar_variants(baselines: Baselines) -> Tuple[Variant, ...]: + """Every grammar the sandbox prices, as the variants a run encodes under. + + Args: + baselines: Where the production encodings are kept. + + Returns: + Tuple[Variant, ...]: The variants, in the order of ``GRAMMAR_VARIANTS``. + """ + return tuple( + Variant( + name=entry.name, + hypothesis=entry.hypothesis, + kind=entry.kind, + note=entry.note, + encode=grammar_encoder(baselines, entry.grammar), + needs_seeds=False, + ) + for entry in GRAMMAR_VARIANTS + ) diff --git a/scripts/codec_study/variants/strategy.py b/scripts/codec_study/variants/strategy.py index bfcd3d93f..00762c0dd 100644 --- a/scripts/codec_study/variants/strategy.py +++ b/scripts/codec_study/variants/strategy.py @@ -1,3 +1,4 @@ +from dataclasses import replace from typing import Dict, Final, List, Optional, Sequence, Tuple from codec_study.measure import Measurement @@ -63,9 +64,7 @@ def _song_depths( Measurement( song=best.song, variant=f"{DEPTH_PREFIX}{depth}", - compressed=best.compressed, - seconds=seconds, - lossless=best.lossless, + encoding=replace(best.encoding, seconds=seconds), ) ) diff --git a/scripts/compression_study.py b/scripts/compression_study.py index 5c33856cd..350e35f59 100644 --- a/scripts/compression_study.py +++ b/scripts/compression_study.py @@ -6,6 +6,7 @@ from codec_study.manifest import StudyManifest, StudySource from codec_study.measure import Measurement, measure from codec_study.report.run import run_directory, write_run +from codec_study.variants.baselines import Baselines from codec_study.variants.registry import EVERY_VARIANT, selected_variants from codec_study.variants.strategy import STRATEGY_ORDER, depth_measurements from sampletones_shared.logger import logger @@ -71,7 +72,7 @@ def main() -> None: variants=_names(arguments.variants), quick=arguments.quick, ) - variants = selected_variants(manifest.variants) + variants = selected_variants(manifest.variants, Baselines()) directory = run_directory(arguments.output) corpus = build_corpus(manifest) @@ -88,7 +89,7 @@ def main() -> None: logger.info( f" {measurement.block} bytes, {measurement.bytes_per_tick:.3f} bytes per tick, " - f"{len(measurement.compressed.phrases)} phrases, {measurement.seconds:.1f} s" + f"{measurement.phrases} phrases, {measurement.seconds:.1f} s" ) measurements.append(measurement) diff --git a/tests/unit/scripts/codec_study/test_sandbox.py b/tests/unit/scripts/codec_study/test_sandbox.py new file mode 100644 index 000000000..b30476112 --- /dev/null +++ b/tests/unit/scripts/codec_study/test_sandbox.py @@ -0,0 +1,397 @@ +from dataclasses import dataclass, replace +from pathlib import Path +from random import Random +from typing import Final, FrozenSet, List, Sequence, Tuple + +import pytest + +from codec_study.corpus.song import SongGroup, StudySong +from codec_study.sandbox.context import PlaneContext +from codec_study.sandbox.costs import PRODUCTION_COSTS, SET_HOLD_BOUND, Costs +from codec_study.sandbox.decode import play_tokens +from codec_study.sandbox.defaults import modal_counts, no_defaults +from codec_study.sandbox.edges.generator import offered_lengths +from codec_study.sandbox.encode import encode_grammar +from codec_study.sandbox.grammar import BASELINE_GRAMMAR, Grammar +from codec_study.sandbox.parse import StudyParse, parse_plane +from codec_study.sandbox.reference import reference +from codec_study.sandbox.tokens import Hold, Literal, Play, SetHold, StudyToken, WideHold +from codec_study.sandbox.verify import verify_baseline +from codec_study.variants.production import compress_baseline +from codec_study.variants.sandbox import DEFAULT_COUNT_COSTS, GRAMMAR_VARIANTS, GrammarVariant +from sampletones_player.compression.dictionary.phrase import Phrase +from sampletones_player.compression.dictionary.table import PhraseTable, phrase_table +from sampletones_player.compression.encode import STREAM_START +from sampletones_player.compression.matches.cache import MatchCache +from sampletones_player.compression.matches.index import PlaneIndex +from sampletones_player.compression.matches.matcher import PhraseMatcher +from sampletones_player.compression.options import EVERY_LAYER +from sampletones_player.compression.parse.boundaries import Boundaries +from sampletones_player.compression.parse.plane import parse_plane as parse_production +from sampletones_player.compression.pitch import PitchTable +from sampletones_player.compression.planes.order import PlaneOrder +from sampletones_player.compression.planes.song import SongPlanes +from sampletones_player.specification.compression import ( + CHEAP_PHRASE_IDS, + MAX_HOLD_TICKS, + PLANE_COUNT, +) +from sampletones_shared.music import Tuning +from tests.suite.base import BaseTestSuite +from tests.suite.case import BaseRegularTestCase + +ENTRIES: Final[FrozenSet[int]] = frozenset({STREAM_START}) +FIGURES: Final[Tuple[bytes, ...]] = (b"\x0a\x0c\x0f\x0f\x0f\x0f\x0f\x0f", b"\x03\x04\x05\x06") +TABLE: Final[PhraseTable] = phrase_table(Phrase(body=body) for body in FIGURES) +NO_TABLE: Final[PhraseTable] = phrase_table(()) +SONG_TICKS: Final[int] = 200 +SEED: Final[int] = 20260912 +WIDE_HOLD: Final[Grammar] = replace(BASELINE_GRAMMAR, wide_hold=True) +START_HOLD: Final[Grammar] = replace(BASELINE_GRAMMAR, start_hold=True) +EVERY_HOLD: Final[Grammar] = replace(BASELINE_GRAMMAR, every_length=True) +SET_HOLD_ESCAPE: Final[Grammar] = replace(BASELINE_GRAMMAR, set_hold=True) +SET_HOLD_REALLOCATED: Final[Grammar] = replace( + BASELINE_GRAMMAR, set_hold=True, costs=replace(PRODUCTION_COSTS, set_hold=SET_HOLD_BOUND) +) +DEFAULT_COUNTS: Final[Grammar] = replace(BASELINE_GRAMMAR, default_counts=True, costs=DEFAULT_COUNT_COSTS) + + +def _figures_plane(random: Random, ticks: int) -> bytes: + values = bytearray() + while len(values) < ticks: + figure = random.choice(FIGURES) + shift = random.randrange(20) + values.extend((value + shift) % 256 for value in figure) + values.extend([values[-1]] * random.randrange(7)) + + return bytes(values[:ticks]) + + +def _runs_plane(random: Random, ticks: int) -> bytes: + values = bytearray() + while len(values) < ticks: + values.extend([random.randrange(16)] * random.randint(1, 40)) + + return bytes(values[:ticks]) + + +def _dense_plane(random: Random, ticks: int) -> bytes: + return bytes(random.randrange(50) for _ in range(ticks)) + + +def _song(ticks: int) -> StudySong: + random = Random(SEED) + planes: List[bytes] = [] + for plane in range(PLANE_COUNT): + if plane % 3 == 2: + planes.append(bytes(ticks)) + elif plane % 3 == 0: + planes.append(_runs_plane(random, ticks)) + else: + planes.append(_figures_plane(random, ticks)) + + return StudySong( + name="song", + group=SongGroup.PROJECT, + source=Path("song.stp"), + planes=SongPlanes.from_order(PlaneOrder.across(planes)), + seeds=tuple(Phrase(body=body) for body in FIGURES), + pitches=PitchTable.from_tuning(Tuning()), + ) + + +def _context( + plane: bytes, + table: PhraseTable, + costs: Costs, + defaults: Sequence[int], +) -> PlaneContext: + cache = MatchCache([PlaneIndex.from_plane(plane)]) + return PlaneContext( + index=cache.index(0), + matcher=PhraseMatcher(table, 0, cache), + boundaries=Boundaries.across(len(plane), ENTRIES), + transposition=EVERY_LAYER.transposition, + defaults=tuple(defaults), + costs=costs, + ) + + +def _parse( + plane: bytes, + table: PhraseTable, + grammar: Grammar, + defaults: Sequence[int], +) -> StudyParse: + return parse_plane(_context(plane, table, grammar.costs, defaults), grammar) + + +def _production_size(plane: bytes, table: PhraseTable) -> int: + cache = MatchCache([PlaneIndex.from_plane(plane)]) + return parse_production(PhraseMatcher(table, 0, cache), EVERY_LAYER, ENTRIES).size + + +class TestTheBaselineGrammarReadsAsTheCodec(BaseTestSuite): + @dataclass(frozen=True, kw_only=True) + class TestCase(BaseRegularTestCase): + plane: bytes + + test_cases = ( + TestCase(label="an idle plane", plane=bytes(SONG_TICKS)), + TestCase(label="a ramp", plane=bytes(range(SONG_TICKS))), + TestCase(label="two runs", plane=bytes([5] * 10 + [7] * 10)), + TestCase(label="figures at several pitches", plane=_figures_plane(Random(SEED), SONG_TICKS)), + TestCase(label="runs of many lengths", plane=_runs_plane(Random(SEED), SONG_TICKS)), + TestCase(label="dense values", plane=_dense_plane(Random(SEED), SONG_TICKS)), + ) + + @pytest.mark.parametrize("test_case", test_cases, ids=lambda test_case: test_case.label) + def test_the_plane_costs_what_the_production_parser_charges(self, test_case: TestCase) -> None: + parse = _parse(test_case.plane, TABLE, BASELINE_GRAMMAR, no_defaults(len(TABLE))) + + assert parse.size == _production_size(test_case.plane, TABLE) + + @pytest.mark.parametrize("test_case", test_cases, ids=lambda test_case: test_case.label) + def test_the_tokens_play_back(self, test_case: TestCase) -> None: + parse = _parse(test_case.plane, TABLE, BASELINE_GRAMMAR, no_defaults(len(TABLE))) + + assert play_tokens(parse.tokens, TABLE, len(test_case.plane)) == test_case.plane + + +class TestPlayTokens(BaseTestSuite): + @dataclass(frozen=True, kw_only=True) + class TestCase(BaseRegularTestCase): + tokens: Tuple[StudyToken, ...] + expected: bytes + + test_cases = ( + TestCase( + label="a hold keeps the value a literal reached", + tokens=(Literal(values=b"\x07"), Hold(ticks=3)), + expected=b"\x07\x07\x07\x07", + ), + TestCase( + label="a hold opening the stream keeps the seeded value", + tokens=(Hold(ticks=2),), + expected=b"\x00\x00", + ), + TestCase( + label="a wide hold counts blocks of the longest plain hold", + tokens=(Literal(values=b"\x09"), WideHold(blocks=2)), + expected=b"\x09" * (1 + 2 * MAX_HOLD_TICKS), + ), + TestCase( + label="a set-hold takes its value and keeps it", + tokens=(SetHold(value=5, ticks=3), Hold(ticks=1)), + expected=b"\x05\x05\x05\x05", + ), + TestCase( + label="a phrase played past its end holds its final value", + tokens=(Play(phrase_id=1, ticks=6, transpose=0, default=False),), + expected=b"\x03\x04\x05\x06\x06\x06", + ), + TestCase( + label="a shifted phrase plays every value shifted", + tokens=(Play(phrase_id=1, ticks=2, transpose=10, default=True),), + expected=b"\x0d\x0e", + ), + ) + + @pytest.mark.parametrize("test_case", test_cases, ids=lambda test_case: test_case.label) + def test_the_values_are_played(self, test_case: TestCase) -> None: + assert play_tokens(test_case.tokens, TABLE, len(test_case.expected)) == test_case.expected + + def test_the_song_end_cuts_the_final_token(self) -> None: + assert play_tokens((Literal(values=b"\x01"), Hold(ticks=5)), NO_TABLE, 3) == b"\x01\x01\x01" + + +class TestGrammarsPriceAPlane(BaseTestSuite): + @dataclass(frozen=True, kw_only=True) + class TestCase(BaseRegularTestCase): + plane: bytes + table: PhraseTable + grammar: Grammar + defaults: Tuple[int, ...] + expected: int + + idle = bytes(SONG_TICKS) + two_runs = bytes([5] * 10 + [7] * 10) + run_then_figure = bytes([5] * 6 + [9] * 5) + figure_table = phrase_table((Phrase(body=b"\x05\x05\x09"),)) + repeated = bytes(b"\x01\x02\x03\x03" * 3) + repeated_table = phrase_table((Phrase(body=b"\x01\x02\x03"),)) + test_cases = ( + TestCase( + label="an idle plane pays a literal and a hold per block", + plane=idle, + table=NO_TABLE, + grammar=BASELINE_GRAMMAR, + defaults=(), + expected=6, + ), + TestCase( + label="a hold may open the stream over the seeded value", + plane=idle, + table=NO_TABLE, + grammar=START_HOLD, + defaults=(), + expected=4, + ), + TestCase( + label="a wide hold covers the full blocks at once", + plane=idle, + table=NO_TABLE, + grammar=WIDE_HOLD, + defaults=(), + expected=5, + ), + TestCase( + label="a wide hold opening the stream costs two bytes and a remainder", + plane=idle, + table=NO_TABLE, + grammar=replace(WIDE_HOLD, start_hold=True), + defaults=(), + expected=3, + ), + TestCase( + label="two runs pay a literal and a hold each", + plane=two_runs, + table=NO_TABLE, + grammar=BASELINE_GRAMMAR, + defaults=(), + expected=6, + ), + TestCase( + label="a three-byte set-hold breaks even with a literal and a hold", + plane=two_runs, + table=NO_TABLE, + grammar=SET_HOLD_ESCAPE, + defaults=(), + expected=6, + ), + TestCase( + label="a two-byte set-hold spares the hold", + plane=two_runs, + table=NO_TABLE, + grammar=SET_HOLD_REALLOCATED, + defaults=(), + expected=4, + ), + TestCase( + label="the longest hold alone overshoots where a figure begins", + plane=run_then_figure, + table=figure_table, + grammar=BASELINE_GRAMMAR, + defaults=(0,), + expected=6, + ), + TestCase( + label="a shorter hold lets the figure start where it matches", + plane=run_then_figure, + table=figure_table, + grammar=EVERY_HOLD, + defaults=(0,), + expected=5, + ), + TestCase( + label="a phrase played at its default count carries no count", + plane=repeated, + table=repeated_table, + grammar=DEFAULT_COUNTS, + defaults=(4,), + expected=3, + ), + TestCase( + label="a phrase played at another count carries it", + plane=repeated, + table=repeated_table, + grammar=DEFAULT_COUNTS, + defaults=(3,), + expected=6, + ), + ) + + @pytest.mark.parametrize("test_case", test_cases, ids=lambda test_case: test_case.label) + def test_the_plane_costs_what_the_grammar_charges(self, test_case: TestCase) -> None: + parse = _parse(test_case.plane, test_case.table, test_case.grammar, test_case.defaults) + + assert parse.size == test_case.expected + assert play_tokens(parse.tokens, test_case.table, len(test_case.plane)) == test_case.plane + + +class TestCosts: + def test_a_phrase_beyond_the_cheap_ids_pays_the_escape(self) -> None: + assert PRODUCTION_COSTS.phrase(CHEAP_PHRASE_IDS - 1, 0, default=False) == 2 + assert PRODUCTION_COSTS.phrase(CHEAP_PHRASE_IDS, 0, default=False) == 3 + assert PRODUCTION_COSTS.phrase(0, 3, default=False) == 3 + + def test_default_counts_halve_the_cheap_ids_and_widen_the_entries(self) -> None: + assert DEFAULT_COUNT_COSTS.phrase(30, 0, default=True) == 1 + assert DEFAULT_COUNT_COSTS.phrase(31, 0, default=True) == 2 + assert DEFAULT_COUNT_COSTS.dictionary(TABLE) == TABLE.size + len(TABLE) + + def test_offered_lengths_are_the_longest_alone_or_every_one(self) -> None: + assert tuple(offered_lengths(5, least=2, every=False)) == (5,) + assert tuple(offered_lengths(5, least=2, every=True)) == (2, 3, 4, 5) + + +class TestModalCounts: + def test_the_count_played_most_often_is_the_default(self) -> None: + parse = StudyParse( + tokens=( + Play(phrase_id=0, ticks=4, transpose=0, default=False), + Play(phrase_id=0, ticks=8, transpose=2, default=False), + Play(phrase_id=0, ticks=4, transpose=0, default=True), + Hold(ticks=1), + ), + costs=(0, 2, 5, 6, 7), + ) + + assert modal_counts((parse,), 2) == (4, 0) + + def test_a_tie_takes_the_shorter_count(self) -> None: + parse = StudyParse( + tokens=( + Play(phrase_id=0, ticks=8, transpose=0, default=False), + Play(phrase_id=0, ticks=4, transpose=0, default=False), + ), + costs=(0, 2, 4), + ) + + assert modal_counts((parse,), 1) == (4,) + + +class TestTheReferenceHoldsTheSandboxToTheCodec: + song = _song(SONG_TICKS) + compressed = compress_baseline(song) + + def test_the_baseline_parse_costs_what_the_codec_wrote(self) -> None: + read = reference(self.song, self.compressed) + + assert [parse.size for parse in read.baseline] == [len(stream) for stream in self.compressed.streams] + + def test_a_stream_priced_differently_stops_the_study(self) -> None: + read = reference(self.song, self.compressed) + tampered = self.compressed.model_copy( + update={"streams": PlaneOrder.across([stream + b"\x00" for stream in self.compressed.streams])} + ) + + with pytest.raises(ValueError, match="parted from the production parser"): + verify_baseline(read.baseline, tampered) + + @pytest.mark.parametrize("entry", GRAMMAR_VARIANTS, ids=lambda entry: entry.name) + def test_every_grammar_plays_back_and_adds_no_bytes_over_a_small_table(self, entry: GrammarVariant) -> None: + read = reference(self.song, self.compressed) + + encoding = encode_grammar(read, entry.grammar) + + assert encoding.lossless + assert encoding.written is None + assert sum(encoding.streams) <= sum(len(stream) for stream in self.compressed.streams) + + def test_wide_holds_opening_the_stream_reduce_an_idle_plane_to_three_bytes(self) -> None: + read = reference(self.song, self.compressed) + + encoding = encode_grammar(read, replace(WIDE_HOLD, start_hold=True)) + + assert [encoding.streams[plane] for plane in range(2, PLANE_COUNT, 3)] == [3, 3, 3] diff --git a/tests/unit/scripts/codec_study/test_variants.py b/tests/unit/scripts/codec_study/test_variants.py index 992022259..1c8109fcc 100644 --- a/tests/unit/scripts/codec_study/test_variants.py +++ b/tests/unit/scripts/codec_study/test_variants.py @@ -5,7 +5,7 @@ import pytest from codec_study.corpus.song import SongGroup, StudySong -from codec_study.measure import Measurement +from codec_study.measure import Measurement, production_encoding from codec_study.variants.seeds import split, trimmed, whole_and_split from codec_study.variants.strategy import DEPTH_PREFIX, depth_measurements from sampletones_player.compression.compressed import CompressedPlanes @@ -114,16 +114,15 @@ def _measurement( seconds: float, ) -> Measurement: stream = emit([LiteralToken(values=bytes(TICKS))] if spelled_out else [HoldToken(ticks=TICKS)]) + compressed = CompressedPlanes( + phrases=phrase_table(()), + streams=PlaneOrder.across([stream] * PLANE_COUNT), + ticks=TICKS, + ) return Measurement( song=song, variant=variant, - compressed=CompressedPlanes( - phrases=phrase_table(()), - streams=PlaneOrder.across([stream] * PLANE_COUNT), - ticks=TICKS, - ), - seconds=seconds, - lossless=True, + encoding=production_encoding(song, compressed, seconds), ) From 7aafe67806856224b3d364d4b959e5a53e72f658 Mon Sep 17 00:00:00 2001 From: JakimPL Date: Sat, 12 Sep 2026 20:58:02 +0200 Subject: [PATCH 05/36] Recorded: the compression study verdicts --- docs/development/player.md | 12 ++ scripts/codec_study/report/run.py | 16 +- scripts/codec_study/report/verdicts.py | 174 ++++++++++++++++ .../unit/scripts/codec_study/test_verdicts.py | 195 ++++++++++++++++++ 4 files changed, 396 insertions(+), 1 deletion(-) create mode 100644 scripts/codec_study/report/verdicts.py create mode 100644 tests/unit/scripts/codec_study/test_verdicts.py diff --git a/docs/development/player.md b/docs/development/player.md index b3fcf59db..a58b6dbf6 100644 --- a/docs/development/player.md +++ b/docs/development/player.md @@ -41,6 +41,17 @@ what the samples leave uncovered. off on its own, and `make compression-report` writes what each one saves across a corpus of songs. The format's constants are settled from that report rather than from argument. +**A change to the codec is measured before it is built.** `make compression-study` reads the +projects and stems on this machine, encodes every song under every candidate change, and +writes the sizes, the times and a verdict per candidate under +`Documents/SampleToNES/compression`. A candidate is one of two things. A new way of choosing +tokens is encoded and played back by the production codec itself. A new token grammar is +priced in bytes by a study parser, which first has to reproduce the production parser's +bytes on today's grammar. The rule is printed in the report: a candidate earns a production +layer when it saves 3% over the projects or 5% over the reconstructions and grows no song by +more than 1%. The study lives under `scripts/codec_study` and stays out of the shipped +packages. + ## The song a file carries `Song` is the compressed song: the dictionary, one token stream per plane, the timer table, @@ -141,6 +152,7 @@ The chain runs from the register values upward, and each link is held on its own | The codec is lossless | every encoding decodes to the planes it was written from, over a corpus | | The codec is safe | a plane the codec finds nothing in stays within its literal bound | | The ratio | `make compression-report` — bytes per tick and ticks that fit, per layer | +| What a change would save | `make compression-study` — the songs on this machine under every candidate change, with a verdict each | | The byte layout | a hand-built song serializes to expected bytes | | The assembly agrees with the specification | the include's equates are read and compared field by field | | The driver behaves | the assembled image on a 6502 emulator against `RegisterTrace.from_song`, over several rates and over songs that repeat | diff --git a/scripts/codec_study/report/run.py b/scripts/codec_study/report/run.py index 0c5b7815a..a78126905 100644 --- a/scripts/codec_study/report/run.py +++ b/scripts/codec_study/report/run.py @@ -8,6 +8,7 @@ from codec_study.measure import Measurement from codec_study.report import aggregate from codec_study.report import rows as songs +from codec_study.report import verdicts from codec_study.report.writers import markdown_table, write_csv from codec_study.variants.production import BASELINE_NAME from codec_study.variants.variant import Variant @@ -20,6 +21,7 @@ REPORT_CSV: Final[str] = "report.csv" REPORT_MARKDOWN: Final[str] = "report.md" ACCOUNTING_CSV: Final[str] = "accounting.csv" +VERDICTS_CSV: Final[str] = "verdicts.csv" MANIFEST_JSON: Final[str] = "manifest.json" UNKNOWN_COMMIT: Final[str] = "unknown" @@ -57,7 +59,7 @@ def write_run( measurements: Sequence[Measurement], derived: Sequence[Measurement], ) -> None: - """Writes a run's report, its accounting and the manifest that reproduces it. + """Writes a run's report, its verdicts, its accounting and the manifest that reproduces it. Args: directory: The run's directory. @@ -70,14 +72,26 @@ def write_run( reported = (*measurements, *derived) song_rows = [songs.study_row(measurement) for measurement in reported] group_rows = aggregate.group_rows(reported, BASELINE_NAME) + verdict_rows = verdicts.verdict_rows(group_rows, measurements, variants) accounting_rows = [ accounting.account(measurement, compressed) for measurement, compressed in _written(measurements) ] manifest.save(directory / MANIFEST_JSON) write_csv(directory / REPORT_CSV, songs.COLUMNS, [row.cells for row in song_rows]) + write_csv(directory / VERDICTS_CSV, verdicts.COLUMNS, [row.cells for row in verdict_rows]) write_csv(directory / ACCOUNTING_CSV, accounting.COLUMNS, [row.cells for row in accounting_rows]) lines = _header(manifest, variants) lines.extend(("## Variants", "", *_variants_table(variants), "")) + lines.extend( + ( + "## Verdicts", + "", + verdicts.RULE, + "", + *markdown_table(verdicts.COLUMNS, [row.cells for row in verdict_rows]), + "", + ) + ) lines.extend(("## Groups", "", *markdown_table(aggregate.COLUMNS, [row.cells for row in group_rows]), "")) lines.extend(("## Songs", "", *markdown_table(songs.COLUMNS, [row.cells for row in song_rows]), "")) lines.extend(("## Accounting", "", *_accounting_table(accounting_rows), "")) diff --git a/scripts/codec_study/report/verdicts.py b/scripts/codec_study/report/verdicts.py new file mode 100644 index 000000000..b98946ab9 --- /dev/null +++ b/scripts/codec_study/report/verdicts.py @@ -0,0 +1,174 @@ +from dataclasses import dataclass +from enum import StrEnum +from typing import Dict, Final, List, Optional, Sequence, Tuple + +from codec_study.corpus.song import SongGroup +from codec_study.measure import Measurement +from codec_study.report.aggregate import GroupRow +from codec_study.variants.variant import Variant, VariantKind + +PROJECT_BAR: Final[float] = 0.03 +RECONSTRUCTION_BAR: Final[float] = 0.05 +REGRESSION_BAR: Final[float] = 0.01 +PROJECT_GROUPS: Final[Tuple[SongGroup, ...]] = (SongGroup.PROJECT, SongGroup.LONG_PROJECT) +UNMEASURED: Final[str] = "" +COLUMNS: Final[Tuple[str, ...]] = ( + "variant", + "hypothesis", + "kind", + "projects", + "reconstructions", + "worst song", + "verdict", +) +RULE: Final[str] = ( + f"A variant graduates when it saves {100 * PROJECT_BAR:.0f}% over the projects, short and " + f"lengthened together, or {100 * RECONSTRUCTION_BAR:.0f}% over the reconstructions, with no song " + f"growing by more than {100 * REGRESSION_BAR:.0f}%. A variant priced at token level is provisional " + "until its production layer confirms the figure." +) + + +class Verdict(StrEnum): + """What the decision rule says of a variant. + + Attributes: + GRADUATES: The variant saves enough, harms no song, and its bytes are the driver's. + PROVISIONAL: The variant saves enough and harms no song, priced at token level. + REJECTED: The variant saves too little, or grows a song past the bar. + """ + + GRADUATES = "graduates" + PROVISIONAL = "provisional" + REJECTED = "rejected" + + +@dataclass(frozen=True) +class VerdictRow: + """One variant judged by the decision rule. + + Attributes: + variant: The variant's name. + hypothesis: The hypothesis the variant measures. + kind: What the variant changes. + projects: How the projects' bytes stand against the baseline, short and lengthened + together, as a ratio; ``None`` where the variant measured none. + reconstructions: The same over the reconstructions. + worst: The largest growth any one song shows against its baseline, as a ratio. + verdict: What the rule says. + """ + + variant: str + hypothesis: str + kind: VariantKind + projects: Optional[float] + reconstructions: Optional[float] + worst: float + verdict: Verdict + + @property + def cells(self) -> Tuple[str, ...]: + """The row as the table prints it, column by column.""" + return ( + self.variant, + self.hypothesis, + self.kind.value, + _percent(self.projects), + _percent(self.reconstructions), + f"{100.0 * self.worst:+.1f}%", + self.verdict.value, + ) + + +def _percent(change: Optional[float]) -> str: + return UNMEASURED if change is None else f"{100.0 * change:+.1f}%" + + +def _change(rows: Sequence[GroupRow]) -> Optional[float]: + if not rows: + return None + + return sum(row.block for row in rows) / sum(row.baseline for row in rows) - 1.0 + + +def _earned( + projects: Optional[float], + reconstructions: Optional[float], +) -> bool: + over_projects = projects is not None and projects <= -PROJECT_BAR + over_reconstructions = reconstructions is not None and reconstructions <= -RECONSTRUCTION_BAR + return over_projects or over_reconstructions + + +def judge( + projects: Optional[float], + reconstructions: Optional[float], + worst: float, + *, + priced: bool, +) -> Verdict: + """Applies the decision rule to one variant's aggregates. + + Args: + projects: The change over the projects, short and lengthened together. + reconstructions: The change over the reconstructions. + worst: The largest growth any one song shows. + priced: Whether the variant's bytes were priced at token level. + + Returns: + Verdict: What the rule says. + """ + if worst > REGRESSION_BAR or not _earned(projects, reconstructions): + return Verdict.REJECTED + + return Verdict.PROVISIONAL if priced else Verdict.GRADUATES + + +def verdict_rows( + group_rows: Sequence[GroupRow], + measurements: Sequence[Measurement], + variants: Sequence[Variant], +) -> Tuple[VerdictRow, ...]: + """Judges every variant the run measured, the baseline excepted. + + Args: + group_rows: The measurements summed over each kind of song, variant by variant. + measurements: Every song under every variant. + variants: The variants the run encoded under. + + Returns: + Tuple[VerdictRow, ...]: One row per measured variant, in the order given. + """ + by_variant: Dict[str, List[GroupRow]] = {} + for row in group_rows: + by_variant.setdefault(row.variant, []).append(row) + + priced = {measurement.variant for measurement in measurements if measurement.encoding.written is None} + rows: List[VerdictRow] = [] + for variant in variants: + if variant.kind is VariantKind.BASELINE or variant.name not in by_variant: + continue + + rows.append(_verdict_row(variant, by_variant[variant.name], priced=variant.name in priced)) + + return tuple(rows) + + +def _verdict_row( + variant: Variant, + rows: Sequence[GroupRow], + *, + priced: bool, +) -> VerdictRow: + projects = _change([row for row in rows if SongGroup(row.group) in PROJECT_GROUPS]) + reconstructions = _change([row for row in rows if SongGroup(row.group) is SongGroup.RECONSTRUCTION]) + worst = max(row.worst for row in rows) + return VerdictRow( + variant=variant.name, + hypothesis=variant.hypothesis, + kind=variant.kind, + projects=projects, + reconstructions=reconstructions, + worst=worst, + verdict=judge(projects, reconstructions, worst, priced=priced), + ) diff --git a/tests/unit/scripts/codec_study/test_verdicts.py b/tests/unit/scripts/codec_study/test_verdicts.py new file mode 100644 index 000000000..b465c8040 --- /dev/null +++ b/tests/unit/scripts/codec_study/test_verdicts.py @@ -0,0 +1,195 @@ +from dataclasses import dataclass, replace +from pathlib import Path +from typing import Final, Optional, Tuple + +import pytest + +from codec_study.corpus.song import SongGroup, StudySong +from codec_study.measure import Encoding, Measurement +from codec_study.report.aggregate import group_rows +from codec_study.report.verdicts import Verdict, judge, verdict_rows +from codec_study.variants.variant import Variant, VariantKind +from sampletones_player.compression.pitch import PitchTable +from sampletones_player.compression.planes.order import PlaneOrder +from sampletones_player.compression.planes.song import SongPlanes +from sampletones_player.specification.compression import PLANE_COUNT +from sampletones_shared.music import Tuning +from tests.suite.base import BaseTestSuite +from tests.suite.case import BaseRegularTestCase + +TICKS: Final[int] = 4 +BASELINE: Final[str] = "baseline" +STREAM: Final[int] = 100 + + +class TestJudge(BaseTestSuite): + @dataclass(frozen=True, kw_only=True) + class TestCase(BaseRegularTestCase): + projects: Optional[float] + reconstructions: Optional[float] + worst: float + priced: bool + expected: Verdict + + test_cases = ( + TestCase( + label="saving enough over the projects graduates", + projects=-0.03, + reconstructions=0.0, + worst=0.0, + priced=False, + expected=Verdict.GRADUATES, + ), + TestCase( + label="saving enough over the reconstructions graduates", + projects=0.0, + reconstructions=-0.05, + worst=-0.02, + priced=False, + expected=Verdict.GRADUATES, + ), + TestCase( + label="a token-level price is provisional", + projects=-0.1, + reconstructions=-0.1, + worst=0.0, + priced=True, + expected=Verdict.PROVISIONAL, + ), + TestCase( + label="a song growing past the bar rejects the variant", + projects=-0.1, + reconstructions=-0.1, + worst=0.011, + priced=False, + expected=Verdict.REJECTED, + ), + TestCase( + label="saving too little rejects the variant", + projects=-0.029, + reconstructions=-0.049, + worst=0.0, + priced=False, + expected=Verdict.REJECTED, + ), + TestCase( + label="a group the variant left unmeasured counts for nothing", + projects=None, + reconstructions=-0.05, + worst=0.0, + priced=False, + expected=Verdict.GRADUATES, + ), + ) + + @pytest.mark.parametrize("test_case", test_cases, ids=lambda test_case: test_case.label) + def test_the_rule_answers(self, test_case: TestCase) -> None: + verdict = judge( + test_case.projects, + test_case.reconstructions, + test_case.worst, + priced=test_case.priced, + ) + + assert verdict is test_case.expected + + +def _song(name: str, group: SongGroup) -> StudySong: + return StudySong( + name=name, + group=group, + source=Path(f"{name}.stp"), + planes=SongPlanes.from_order(PlaneOrder.across([bytes(TICKS)] * PLANE_COUNT)), + seeds=(), + pitches=PitchTable.from_tuning(Tuning()), + ) + + +def _measurement( + song: StudySong, + variant: str, + streams: int, +) -> Measurement: + return Measurement( + song=song, + variant=variant, + encoding=Encoding( + phrases=0, + dictionary=0, + streams=(streams,) + (0,) * (PLANE_COUNT - 1), + seconds=0.0, + lossless=True, + written=None, + ), + ) + + +def _variant(name: str, kind: VariantKind) -> Variant: + return Variant( + name=name, + hypothesis="H", + kind=kind, + note="", + encode=lambda song: Encoding( + phrases=0, + dictionary=0, + streams=(), + seconds=0.0, + lossless=True, + written=None, + ), + needs_seeds=False, + ) + + +class TestVerdictRows: + short = _song("short", SongGroup.PROJECT) + long = _song("long", SongGroup.LONG_PROJECT) + stem = _song("stem", SongGroup.RECONSTRUCTION) + variants: Tuple[Variant, ...] = ( + _variant(BASELINE, VariantKind.BASELINE), + _variant("shrinks", VariantKind.FORMAT), + _variant("projects-only", VariantKind.ENCODER), + ) + + def test_the_projects_are_judged_together_and_the_baseline_is_left_out(self) -> None: + measurements = ( + _measurement(self.short, BASELINE, STREAM), + _measurement(self.long, BASELINE, STREAM), + _measurement(self.stem, BASELINE, STREAM), + _measurement(self.short, "shrinks", STREAM), + _measurement(self.long, "shrinks", STREAM - 60), + _measurement(self.stem, "shrinks", STREAM - 10), + _measurement(self.short, "projects-only", STREAM - 40), + _measurement(self.long, "projects-only", STREAM - 40), + ) + baseline = 2 * measurements[0].block + + rows = verdict_rows(group_rows(measurements, BASELINE), measurements, self.variants) + + assert [row.variant for row in rows] == ["shrinks", "projects-only"] + assert rows[0].projects == pytest.approx(-60 / baseline) + assert rows[0].reconstructions == pytest.approx(-10 / measurements[2].block) + assert rows[0].worst == pytest.approx(0.0) + assert rows[0].verdict is Verdict.PROVISIONAL + assert rows[1].reconstructions is None + assert rows[1].cells[4] == "" + + def test_a_variant_the_run_left_out_takes_no_row(self) -> None: + measurements = (_measurement(self.short, BASELINE, STREAM), _measurement(self.short, "shrinks", STREAM)) + + rows = verdict_rows(group_rows(measurements, BASELINE), measurements, self.variants) + + assert [row.variant for row in rows] == ["shrinks"] + assert rows[0].verdict is Verdict.REJECTED + + def test_a_written_encoding_graduates_outright(self) -> None: + written = replace(self.variants[2], kind=VariantKind.ENCODER) + measurements = ( + _measurement(self.short, BASELINE, STREAM), + _measurement(self.short, written.name, STREAM - 50), + ) + + rows = verdict_rows(group_rows(measurements, BASELINE), measurements, (self.variants[0], written)) + + assert rows[0].verdict is Verdict.PROVISIONAL From f2a599a564441afffb1e9f92b3e08c8d35f9bf30 Mon Sep 17 00:00:00 2001 From: JakimPL Date: Sun, 13 Sep 2026 14:32:19 +0200 Subject: [PATCH 06/36] Rewrote: the build and release scripts in Python --- .github/workflows/ci.yml | 8 +- .github/workflows/workflow.yml | 27 +- CHANGELOG.md | 6 +- Makefile | 37 +-- README.md | 2 +- docs/development/dependencies.md | 12 +- docs/development/tooling.md | 73 ++++++ docs/development/undo.md | 2 +- docs/guide/installation.md | 2 +- docs/index.md | 1 + install.bat | 11 +- install.sh | 15 +- pyproject.toml | 2 + run.bat | 2 - run.sh | 3 - scripts/bootstrap/__init__.py | 0 scripts/bootstrap/interpreter.py | 26 ++ scripts/bootstrap/platforms/__init__.py | 0 scripts/bootstrap/platforms/factory.py | 37 +++ scripts/bootstrap/platforms/linux.py | 91 +++++++ scripts/bootstrap/platforms/macos.py | 70 ++++++ scripts/bootstrap/platforms/protocol.py | 69 +++++ scripts/bootstrap/platforms/windows.py | 55 ++++ scripts/bootstrap/preflight.py | 89 +++++++ scripts/bootstrap/processes.py | 70 ++++++ scripts/bootstrap/repository.py | 17 ++ scripts/bootstrap/venv_build.py | 94 +++++++ scripts/build_environment.py | 47 ++++ scripts/bundle.py | 235 ++++++++++++++++++ scripts/ci/__init__.py | 0 scripts/ci/checks/__init__.py | 0 scripts/clean.py | 63 +++++ scripts/linux/build/build.sh | 71 ------ scripts/linux/build/clean.sh | 10 - scripts/linux/build/dependencies.sh | 31 --- scripts/linux/build/icons.sh | 12 - scripts/linux/build/preflight.sh | 53 ---- scripts/linux/build/python.sh | 14 -- scripts/linux/build/sampletones.sh | 28 --- scripts/linux/build/venv.sh | 15 -- scripts/linux/lib/root.sh | 10 - scripts/macos/build/build_env.sh | 15 -- scripts/macos/build/dependencies.sh | 17 -- scripts/macos/build/no_bundle.sh | 15 -- .../release_environment.py} | 0 scripts/setup_environment.py | 102 ++++++++ scripts/system_dependencies.py | 37 +++ scripts/windows/build/build.bat | 81 ------ scripts/windows/build/clean.bat | 18 -- scripts/windows/build/icons.bat | 14 -- scripts/windows/build/preflight.bat | 51 ---- scripts/windows/build/python.bat | 22 -- scripts/windows/build/sampletones.bat | 28 --- scripts/windows/build/venv.bat | 14 -- scripts/windows/lib/root.bat | 7 - .../config/deployment/deployment.py | 2 +- tests/suite/bootstrap.py | 72 ++++++ .../scripts/bootstrap/test_interpreter.py | 16 ++ .../unit/scripts/bootstrap/test_platforms.py | 115 +++++++++ .../unit/scripts/bootstrap/test_preflight.py | 98 ++++++++ .../unit/scripts/bootstrap/test_processes.py | 36 +++ .../unit/scripts/bootstrap/test_repository.py | 9 + .../unit/scripts/bootstrap/test_venv_build.py | 56 +++++ tests/unit/scripts/test_build_environment.py | 35 +++ tests/unit/scripts/test_bundle.py | 148 +++++++++++ tests/unit/scripts/test_clean.py | 46 ++++ tests/unit/scripts/test_setup_environment.py | 45 ++++ .../unit/scripts/test_system_dependencies.py | 40 +++ 68 files changed, 1924 insertions(+), 625 deletions(-) create mode 100644 docs/development/tooling.md delete mode 100644 run.bat delete mode 100755 run.sh create mode 100644 scripts/bootstrap/__init__.py create mode 100644 scripts/bootstrap/interpreter.py create mode 100644 scripts/bootstrap/platforms/__init__.py create mode 100644 scripts/bootstrap/platforms/factory.py create mode 100644 scripts/bootstrap/platforms/linux.py create mode 100644 scripts/bootstrap/platforms/macos.py create mode 100644 scripts/bootstrap/platforms/protocol.py create mode 100644 scripts/bootstrap/platforms/windows.py create mode 100644 scripts/bootstrap/preflight.py create mode 100644 scripts/bootstrap/processes.py create mode 100644 scripts/bootstrap/repository.py create mode 100644 scripts/bootstrap/venv_build.py create mode 100644 scripts/build_environment.py create mode 100644 scripts/bundle.py create mode 100644 scripts/ci/__init__.py create mode 100644 scripts/ci/checks/__init__.py create mode 100644 scripts/clean.py delete mode 100755 scripts/linux/build/build.sh delete mode 100755 scripts/linux/build/clean.sh delete mode 100755 scripts/linux/build/dependencies.sh delete mode 100644 scripts/linux/build/icons.sh delete mode 100755 scripts/linux/build/preflight.sh delete mode 100755 scripts/linux/build/python.sh delete mode 100755 scripts/linux/build/sampletones.sh delete mode 100755 scripts/linux/build/venv.sh delete mode 100755 scripts/linux/lib/root.sh delete mode 100755 scripts/macos/build/build_env.sh delete mode 100755 scripts/macos/build/dependencies.sh delete mode 100755 scripts/macos/build/no_bundle.sh rename scripts/{release_env_hook.py => runtime_hooks/release_environment.py} (100%) create mode 100644 scripts/setup_environment.py create mode 100644 scripts/system_dependencies.py delete mode 100644 scripts/windows/build/build.bat delete mode 100644 scripts/windows/build/clean.bat delete mode 100644 scripts/windows/build/icons.bat delete mode 100644 scripts/windows/build/preflight.bat delete mode 100644 scripts/windows/build/python.bat delete mode 100644 scripts/windows/build/sampletones.bat delete mode 100644 scripts/windows/build/venv.bat delete mode 100644 scripts/windows/lib/root.bat create mode 100644 tests/suite/bootstrap.py create mode 100644 tests/unit/scripts/bootstrap/test_interpreter.py create mode 100644 tests/unit/scripts/bootstrap/test_platforms.py create mode 100644 tests/unit/scripts/bootstrap/test_preflight.py create mode 100644 tests/unit/scripts/bootstrap/test_processes.py create mode 100644 tests/unit/scripts/bootstrap/test_repository.py create mode 100644 tests/unit/scripts/bootstrap/test_venv_build.py create mode 100644 tests/unit/scripts/test_build_environment.py create mode 100644 tests/unit/scripts/test_bundle.py create mode 100644 tests/unit/scripts/test_clean.py create mode 100644 tests/unit/scripts/test_setup_environment.py create mode 100644 tests/unit/scripts/test_system_dependencies.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 687508232..2d44cc60b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,7 +27,7 @@ jobs: enable-cache: true - name: Install system libraries - run: bash scripts/linux/build/dependencies.sh + run: python3 scripts/system_dependencies.py - name: Install the development environment run: uv sync --group dev @@ -63,13 +63,13 @@ jobs: - name: Install system libraries (Linux) if: runner.os == 'Linux' - run: bash scripts/linux/build/dependencies.sh + run: python3 scripts/system_dependencies.py - name: Install system libraries (macOS) if: runner.os == 'macOS' run: | - bash scripts/macos/build/dependencies.sh - bash scripts/macos/build/build_env.sh >> "$GITHUB_ENV" + python3 scripts/system_dependencies.py + python3 scripts/build_environment.py >> "$GITHUB_ENV" - name: Install the development environment run: uv sync --group dev diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml index f94d55538..958c6e8ab 100644 --- a/.github/workflows/workflow.yml +++ b/.github/workflows/workflow.yml @@ -78,8 +78,8 @@ jobs: - name: Install PortAudio (macOS) if: runner.os == 'macOS' run: | - bash repository/scripts/macos/build/dependencies.sh - bash repository/scripts/macos/build/build_env.sh >> "$GITHUB_ENV" + python3 repository/scripts/system_dependencies.py + python3 repository/scripts/build_environment.py >> "$GITHUB_ENV" - name: Install the wheel and check the entry point shell: bash @@ -114,28 +114,11 @@ jobs: - name: Install system libraries (Linux) if: runner.os == 'Linux' - run: bash scripts/linux/build/dependencies.sh + run: python scripts/system_dependencies.py - - name: Create the build environment the bundle scripts expect + - name: Build the bundle shell: bash - run: | - python -m venv .venv-build - if [ "$RUNNER_OS" = "Windows" ]; then - venv_python=.venv-build/Scripts/python.exe - else - venv_python=.venv-build/bin/python - fi - "$venv_python" -m pip install --upgrade pip - "$venv_python" -m pip install ".[build]" --group assets - - - name: Build the bundle (Linux) - if: runner.os == 'Linux' - run: bash scripts/linux/build/build.sh --release - - - name: Build the bundle (Windows) - if: runner.os == 'Windows' - shell: cmd - run: scripts\windows\build\build.bat --release + run: python scripts/bundle.py --release - name: Check the bundle runs and carries its notices shell: bash diff --git a/CHANGELOG.md b/CHANGELOG.md index b30d05826..89b4fcdac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,9 +3,9 @@ ## v0.3.2 * Added NSF player and export. -* Added stems conversion: mix several recordings into one reconstruction -* Matched each stem against its own recording, so a stem plays what was recorded on it -* Added a per-source channel cap +* Added stems conversion: mix several recordings into one reconstruction. +* Matched each stem against its own recording, so a stem plays what was recorded on it. +* Added a per-source channel cap. * Bumped the reconstruction data-version to `2.2` with backward compatibility for `2.1`. ## v0.3.1 [2026-08-18] diff --git a/Makefile b/Makefile index 1555ed7b1..f7de00ac2 100644 --- a/Makefile +++ b/Makefile @@ -16,15 +16,11 @@ ifeq ($(UNAME_S),Windows) SCRIPTS_DIR := scripts/windows SCRIPT_EXT := .bat RUN_SCRIPT := - BUILD_SCRIPT := install.bat - EXECUTABLE := sampletones.exe PYTHON := python else SCRIPTS_DIR := scripts/linux SCRIPT_EXT := .sh RUN_SCRIPT := bash - BUILD_SCRIPT := ./install.sh - EXECUTABLE := sampletones PYTHON := python3 endif @@ -40,32 +36,13 @@ else Q := " endif -BUILD_COMMAND := $(RUN_SCRIPT) $(BUILD_SCRIPT) -RELEASE_COMMAND := $(RUN_SCRIPT) $(BUILD_SCRIPT) --release -SYSTEM_DEPS_COMMAND := bash scripts/linux/build/dependencies.sh -SETUP_ENV := - -ifeq ($(UNAME_S),Darwin) - MACOS_NO_BUNDLE := bash scripts/macos/build/no_bundle.sh - BUILD_COMMAND := $(MACOS_NO_BUNDLE) 'make build' - RELEASE_COMMAND := $(MACOS_NO_BUNDLE) 'make release' - SYSTEM_DEPS_COMMAND := bash scripts/macos/build/dependencies.sh - SETUP_ENV := ARCHFLAGS="-arch $(shell uname -m)" -endif - GPU ?= auto -GPU_EXTRA := -ifeq ($(filter 0,$(GPU)),) -ifneq ($(filter setup,$(MAKECMDGOALS)),) - GPU_EXTRA := $(shell $(PYTHON) scripts/detect_cuda.py --extra) -endif -endif help: @echo $(Q)Available targets:$(Q) @echo $(Q) make setup - Set up development environment (uv); GPU auto-detected, GPU=0 forces CPU$(Q) @echo $(Q) make pre-commit - Install pre-commit hooks$(Q) - @echo $(Q) make system-deps - Install system packages required to build and run (Debian-based, or Homebrew on macOS)$(Q) + @echo $(Q) make system-deps - Install system packages required to build and run (apt on Debian-based Linux, Homebrew on macOS)$(Q) @echo $(Q) make build - Compile standalone executable (development deployment config: DEBUG, strict history)$(Q) @echo $(Q) make release - Compile standalone executable with the release deployment config (INFO, self-healing history)$(Q) @echo $(Q) make test - Run unit tests with coverage$(Q) @@ -84,28 +61,26 @@ help: @echo $(Q) make run - Run SampleToNES application$(Q) setup: - $(SETUP_ENV) uv sync --group dev $(if $(GPU_EXTRA),--extra $(GPU_EXTRA),) - $(MAKE) icons - $(SETUP_ENV) uv tool install --force $(if $(GPU_EXTRA),".[$(GPU_EXTRA)]",.) + $(PYTHON) scripts/setup_environment.py --gpu $(GPU) install: $(MAKE) setup $(MAKE) build build: - $(BUILD_COMMAND) + $(PYTHON) scripts/bundle.py release: - $(RELEASE_COMMAND) + $(PYTHON) scripts/bundle.py --release system-deps: - $(SYSTEM_DEPS_COMMAND) + $(PYTHON) scripts/system_dependencies.py run: uv run sampletones clean: - $(call script,build/clean) + $(PYTHON) scripts/clean.py pre-commit: $(call script,dev/pre_commit) diff --git a/README.md b/README.md index 170ba6561..7ce8659b9 100644 --- a/README.md +++ b/README.md @@ -85,7 +85,7 @@ You only need Python 3.12. #### Linux -1. Install the audio and file-dialog system packages: `make system-deps` (or run `./scripts/linux/build/dependencies.sh`). +1. Install the audio and file-dialog system packages: `make system-deps` (or run `python3 scripts/system_dependencies.py`). 2. Install Python 3.12, then run `./install.sh` in a terminal. It builds a `bin/sampletones` executable. 3. Run `./bin/sampletones` to start. diff --git a/docs/development/dependencies.md b/docs/development/dependencies.md index 103d68085..ac08699a4 100644 --- a/docs/development/dependencies.md +++ b/docs/development/dependencies.md @@ -20,9 +20,9 @@ Instruction libraries and reconstructions are serialized with [MessagePack](http ## Audio playback -Playback goes through PortAudio, reached with the `pyaudio` package. PyPI carries `pyaudio` wheels for Windows, so Linux and macOS compile it on install and need the PortAudio headers and library on the machine. Linux takes them from the distribution packages listed in `scripts/linux/build/dependencies.sh`; macOS takes them from Homebrew through `scripts/macos/build/dependencies.sh`. +Playback goes through PortAudio, reached with the `pyaudio` package. PyPI carries `pyaudio` wheels for Windows, so Linux and macOS compile it on install and need the PortAudio headers and library on the machine. `scripts/system_dependencies.py` installs them: the distribution packages through apt on Linux, PortAudio through Homebrew on macOS. -Compiling on macOS also depends on the interpreter's architecture. The python.org installer ships a universal2 build, which compiles extensions for both Apple Silicon and Intel, while Homebrew's `libportaudio` carries the machine's own architecture. Pinning `ARCHFLAGS` to `uname -m` settles it on the native one: `make setup` sets it directly, and the CI workflows take it from `scripts/macos/build/build_env.sh`, which reports it as a `KEY=VALUE` line alongside the PortAudio prefix for a Homebrew installed outside its usual place. +Compiling on macOS also depends on the interpreter's architecture. The python.org installer ships a universal2 build, which compiles extensions for both Apple Silicon and Intel, while Homebrew's `libportaudio` carries the machine's own architecture. Pinning `ARCHFLAGS` to `uname -m` settles it on the native one: `make setup` sets it directly, and the CI workflows take it from `scripts/build_environment.py`, which reports it as a `KEY=VALUE` line alongside the PortAudio prefix for a Homebrew installed outside its usual place. ## Audio rendering @@ -66,7 +66,7 @@ with, and every wheel, bundle and test run finds them where they lie. `make icon from the mark, and the `icons` pre-push hook writes them for a push that touches either directory, holding the committed files to what the mark describes. CI runs that same hook. -Pillow is a build-time tool, and the bundle scripts pass `--exclude-module PIL` to hold it to that: +Pillow is a build-time tool, and the bundle script passes `--exclude-module PIL` to hold it to that: `pygments`, which arrives with `rich`, offers an image formatter that imports Pillow where it is installed, and PyInstaller follows that import into the bundle. The application reads its icons as files, so the exclusion spares every bundle Pillow's extension modules and the imaging libraries @@ -124,8 +124,8 @@ Three tools serve the player, each reached by one command: | py65 | `make test` | `uv sync --group dev` | the `dev` dependency group | | ffmpeg with `libgme` | `make nsf-render` | the system's package manager | the machine listening to an export | -`scripts/linux/build/dependencies.sh` and its macOS counterpart carry what building and running the -application needs, and the workflows install the `dev` group, so py65 is the one of the three CI +`scripts/system_dependencies.py` carries what building and running the application needs, and the +workflows install the `dev` group, so py65 is the one of the three CI reaches — the suite verifies the driver through it alone. cc65 and ffmpeg stay on the machine of whoever runs `make player` or `make nsf-render`, and a workflow that assembles the driver or renders a wave is what would put them in those scripts. The application itself calls neither: an export is @@ -133,7 +133,7 @@ written by the package's own code, from the committed `driver.bin`. ## Linux (standalone executable) -Building a standalone executable on Linux needs the PortAudio, Tk and OpenGL/X11 system packages. Install them with `make system-deps` (or run `scripts/linux/build/dependencies.sh`), which holds the full list. +Building a standalone executable on Linux needs the PortAudio, Tk and OpenGL/X11 system packages. Install them with `make system-deps` (or run `python3 scripts/system_dependencies.py`), which holds the full list. PortAudio is required. Tk backs the file dialogs where neither a portal nor a desktop tool answers, and `make release` requires it so the shipped executable stays self-contained. diff --git a/docs/development/tooling.md b/docs/development/tooling.md new file mode 100644 index 000000000..88d018261 --- /dev/null +++ b/docs/development/tooling.md @@ -0,0 +1,73 @@ +# Tooling + +This document governs the scripts under `scripts/` and the `Makefile`: what runs on the system +interpreter, what runs in the project environment, and the rules each kind holds to. Read it +before adding a script or a make target. Which packages may import which is +[package layers](packages.md); the libraries and tools the scripts reach for are +[dependencies](dependencies.md). + +## Principles + +**1. Two interpreters, two kinds of script.** A *bootstrap script* runs on the system interpreter, +before or beside the project environment: it creates the environment, installs system packages, +builds the standalone bundle, cleans the tree. It imports the standard library and the other +bootstrap modules, nothing else, so it runs on a machine that has Python and nothing more. A +*tool script* runs inside the project environment, through `uv run`, and imports the project's +packages freely. + +**2. A bootstrap script installs nothing into the interpreter it runs on.** Every package a build +installs lands in `.venv-build`, a virtual environment of its own, and pip is told to refuse any +interpreter outside a virtual environment. System packages are a step of their own, +`make system-deps`, and the only one that asks for administrator rights. A release build reaches +neither uv nor the developer's environment, so it runs the same on a clean machine. + +**3. One script per operation, the same on every system.** What differs between systems (the +launcher's name and extension, the icon, the package manager, the compiler flags audio playback +needs) sits behind one `Platform` protocol with an implementation per system, chosen by a +factory from `platform.system()`. A script never branches on the operating system itself, and a +system the project does not build on is refused by name. + +**4. A script is a library with a thin face.** Pure functions assemble the commands and take the +decisions; `main` parses the arguments and wires in the real runner and the real environment. +Tests call the functions with a runner that records what it was asked to run, so the build is +verified without building. + +**5. The Makefile is the developer's index, one line per target.** A target names the script +that does the work and passes its flag. The two shell files at the root, `install.sh` and +`install.bat`, exist for the double-click path and call the same bundle script. + +## The bootstrap scripts + +| Script | Target | What it does | +|---|---|---| +| `bundle.py` | `make build`, `make release` | Creates `.venv-build`, installs the package with the `build` extra, checks the interpreter carries PortAudio (and Tk, for a release), writes the bundle with PyInstaller, runs its self-check, and copies the notices beside a release | +| `setup_environment.py` | `make setup` | Reads the NVIDIA driver, synchronizes the development environment with the matching GPU extra, writes the icons, installs the global `sampletones` command | +| `system_dependencies.py` | `make system-deps` | Installs the system packages: apt on Debian-based Linux, Homebrew on macOS, nothing on Windows | +| `build_environment.py` | CI | Prints the compiler flags a macOS build exports, one `KEY=VALUE` per line | +| `clean.py` | `make clean` | Removes the build outputs, the coverage reports and the bytecode caches | +| `detect_cuda.py` | via `setup_environment.py` | Maps the driver's CUDA version to the CuPy extra | +| `runtime_hooks/release_environment.py` | build input | The PyInstaller runtime hook that gives a release bundle its deployment defaults | +| `ci/` | the release workflow | The gates a release passes: the tag matches the version, the bundle ships its notices and starts | + +`scripts/bootstrap/` holds what they share: the repository root (`repository.py`), the +interpreter version check (`interpreter.py`), running a command and holding it to success +(`processes.py`), the build environment and the installs into it (`venv_build.py`), the +preflight of the build interpreter (`preflight.py`), and the platforms (`platforms/`). + +## The tool scripts + +`calibration.py`, `compression_study.py`, `nsf_render.py`, `player.py`, `assets/icons.py` and +the checks under `checks/` import the project's packages and run inside its environment, from +the make target that names each. The checks are also pre-commit hooks; +[architecture](architecture.md#enforcement) lists them. + +## Who governs what + +| Concern | Owner | +|---|---| +| What differs between systems | `scripts/bootstrap/platforms/` | +| Where a build installs | `scripts/bootstrap/venv_build.py` | +| What a bundle has to carry before it is built | `scripts/bootstrap/preflight.py` | +| The PyInstaller invocation | `scripts/bundle.py` | +| The GPU extra a machine gets | `scripts/detect_cuda.py` | +| What a release bundle is held to | `scripts/ci/checks/bundle.py` | diff --git a/docs/development/undo.md b/docs/development/undo.md index 21fc240fc..1321705ed 100644 --- a/docs/development/undo.md +++ b/docs/development/undo.md @@ -87,7 +87,7 @@ cursor and repaints rows in place via an index-keyed diff. values, so an edit path reaching the project outside a transaction raises `UntrackedMutationError` at once — in a development run and in the test suite alike, since several tests build the whole application and read that file. A user build takes the opposite -values from `scripts/release_env_hook.py`, so a gap that reaches a release is healed into an +values from `scripts/runtime_hooks/release_environment.py`, so a gap that reaches a release is healed into an `UNTRACKED` entry rather than shown to the user. The gap therefore surfaces where it can be fixed and stays quiet where it cannot. diff --git a/docs/guide/installation.md b/docs/guide/installation.md index 74efb5882..5b888a5d2 100644 --- a/docs/guide/installation.md +++ b/docs/guide/installation.md @@ -37,7 +37,7 @@ A ready-to-run executable built on your machine. You only need Python 3.12. ### Linux 1. Install the audio and file-dialog system packages: `make system-deps` (or run - `./scripts/linux/build/dependencies.sh`). + `python3 scripts/system_dependencies.py`). 2. Install Python 3.12, then run `./install.sh` in a terminal. It builds a `bin/sampletones` executable. 3. Run `./bin/sampletones` to start. diff --git a/docs/index.md b/docs/index.md index 58b11407e..ee57da776 100644 --- a/docs/index.md +++ b/docs/index.md @@ -74,6 +74,7 @@ The [**development**](development/) section is for contributors. - [Configuration](development/config-organization.md) — how the YAML configuration package is laid out. - [Coding guidelines](development/guidelines.md) — conventions for the codebase. - [Dependencies](development/dependencies.md) — the libraries _SampleToNES_ builds on. +- [Tooling](development/tooling.md) — the scripts and the Makefile: what runs on the system interpreter, what runs in the project environment. - [Bugs and to-dos](development/bugs-and-todos.md) — the working ledger of known gaps. ## Glossary diff --git a/install.bat b/install.bat index 1c375bb96..599f7f56c 100644 --- a/install.bat +++ b/install.bat @@ -1,13 +1,4 @@ @echo off -setlocal - -set SCRIPT_DIR=%~dp0 - -call "%SCRIPT_DIR%scripts\windows\lib\root.bat" || exit /b - -call "%SCRIPT_DIR%scripts\windows\build\python.bat" || exit /b -call "%SCRIPT_DIR%scripts\windows\build\venv.bat" || exit /b -call "%SCRIPT_DIR%scripts\windows\build\sampletones.bat" %* || exit /b -call "%SCRIPT_DIR%scripts\windows\build\build.bat" %* || exit /b +python "%~dp0scripts\bundle.py" %* pause diff --git a/install.sh b/install.sh index 73eea2e26..2bdbee4e3 100755 --- a/install.sh +++ b/install.sh @@ -1,16 +1,3 @@ #!/usr/bin/env bash -set -e - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" - -if [[ "$(uname -s)" == "Darwin" ]]; then - exec bash "${SCRIPT_DIR}/scripts/macos/build/no_bundle.sh" "./install.sh" -fi - -source "${SCRIPT_DIR}/scripts/linux/lib/root.sh" - -source "${SCRIPT_DIR}/scripts/linux/build/python.sh" -source "${SCRIPT_DIR}/scripts/linux/build/venv.sh" -bash "${SCRIPT_DIR}/scripts/linux/build/sampletones.sh" "$@" -bash "${SCRIPT_DIR}/scripts/linux/build/build.sh" "$@" +exec python3 "$(dirname "$0")/scripts/bundle.py" "$@" diff --git a/pyproject.toml b/pyproject.toml index 1b2959935..efe629583 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -122,6 +122,8 @@ target-version = ["py312"] profile = "black" line_length = 120 known_first_party = [ + "bootstrap", + "ci", "codec_study", "sampletones", "sampletones_application", diff --git a/run.bat b/run.bat deleted file mode 100644 index 170ec8572..000000000 --- a/run.bat +++ /dev/null @@ -1,2 +0,0 @@ -@echo off -uv run sampletones %* diff --git a/run.sh b/run.sh deleted file mode 100755 index 1716dccd6..000000000 --- a/run.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env bash -set -e -uv run sampletones "$@" diff --git a/scripts/bootstrap/__init__.py b/scripts/bootstrap/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/scripts/bootstrap/interpreter.py b/scripts/bootstrap/interpreter.py new file mode 100644 index 000000000..902800d2f --- /dev/null +++ b/scripts/bootstrap/interpreter.py @@ -0,0 +1,26 @@ +import sys +from typing import Final, Tuple + +REQUIRED_VERSION: Final[Tuple[int, int]] = (3, 12) +DOWNLOADS: Final[str] = "https://www.python.org/downloads/" + + +def require_python(version: Tuple[int, int]) -> None: + """Holds the interpreter running the script to ``version`` or newer. + + Args: + version: The oldest major and minor version the operation runs on. + + Raises: + SystemExit: If the interpreter is older, naming where a newer one is downloaded. + """ + running = sys.version_info + if (running.major, running.minor) >= version: + print(f"Detected Python version: {running.major}.{running.minor}.{running.micro}") + return + + major, minor = version + raise SystemExit( + f"ERROR: Python {major}.{minor} or newer is required.\n" + f"Please install Python {major}.{minor}+ from {DOWNLOADS}" + ) diff --git a/scripts/bootstrap/platforms/__init__.py b/scripts/bootstrap/platforms/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/scripts/bootstrap/platforms/factory.py b/scripts/bootstrap/platforms/factory.py new file mode 100644 index 000000000..8b5d4df71 --- /dev/null +++ b/scripts/bootstrap/platforms/factory.py @@ -0,0 +1,37 @@ +import platform +from typing import Dict, Final + +from bootstrap.platforms.linux import LINUX, Linux +from bootstrap.platforms.macos import DARWIN, MacOS +from bootstrap.platforms.protocol import Platform +from bootstrap.platforms.windows import WINDOWS, Windows + +PLATFORMS: Final[Dict[str, Platform]] = { + LINUX: Linux(), + WINDOWS: Windows(), + DARWIN: MacOS(), +} + + +def platform_named(system: str) -> Platform: + """The platform a system name selects. + + Args: + system: The name ``platform.system()`` reports. + + Returns: + Platform: The platform. + + Raises: + SystemExit: If SampleToNES supports no system of that name. + """ + chosen = PLATFORMS.get(system) + if chosen is None: + raise SystemExit(f"ERROR: {system} is not a system SampleToNES builds on; supported: {', '.join(PLATFORMS)}.") + + return chosen + + +def current_platform() -> Platform: + """The platform the script runs on.""" + return platform_named(platform.system()) diff --git a/scripts/bootstrap/platforms/linux.py b/scripts/bootstrap/platforms/linux.py new file mode 100644 index 000000000..79beb6c57 --- /dev/null +++ b/scripts/bootstrap/platforms/linux.py @@ -0,0 +1,91 @@ +from pathlib import Path +from typing import Final, Optional, Sequence, Tuple + +LINUX: Final[str] = "Linux" +POSIX_LAUNCHER: Final[str] = "sampletones" +POSIX_INTERPRETER: Final[Tuple[str, str]] = ("bin", "python") +PNG_ICON: Final[str] = "src/sampletones_assets/icons/sampletones.png" +SYSTEM_PACKAGES: Final[Tuple[str, ...]] = ( + "libportaudio2", + "libasound-dev", + "libpulse-dev", + "portaudio19-dev", + "python3-tk", + "tk-dev", + "tcl-dev", + "libgl1", + "libegl1", + "libx11-6", + "libx11-xcb1", + "libxcursor1", + "libxi6", + "libxinerama1", + "libxrandr2", + "libxrender1", + "libxxf86vm1", +) + + +def posix_interpreter(environment: Path) -> Path: + """The interpreter a POSIX virtual environment at ``environment`` runs.""" + return environment.joinpath(*POSIX_INTERPRETER) + + +def posix_launcher(distribution: Path, *, release: bool) -> Path: + """The executable PyInstaller writes under ``distribution`` on a POSIX system.""" + if release: + return distribution / POSIX_LAUNCHER / POSIX_LAUNCHER + + return distribution / POSIX_LAUNCHER + + +class Linux: + """A Debian-based Linux: packages through apt, a launcher without an extension.""" + + @property + def name(self) -> str: + return LINUX + + @property + def bundles(self) -> bool: + return True + + @property + def icon(self) -> str: + return PNG_ICON + + @property + def pyaudio_advice(self) -> str: + return ( + "Run 'make system-deps' to install the PortAudio packages, then 'make build' to reinstall the dependencies." + ) + + @property + def tkinter_advice(self) -> str: + return "Run 'make system-deps' to install python3-tk, then build again." + + @property + def tkinter_warning(self) -> str: + return ( + "This bundle opens file dialogs through zenity or kdialog, which the machine running it has to " + "provide. Run 'make system-deps' to install python3-tk and carry Tk as a self-contained fallback." + ) + + def interpreter(self, environment: Path) -> Path: + return posix_interpreter(environment) + + def launcher(self, distribution: Path, *, release: bool) -> Path: + return posix_launcher(distribution, release=release) + + def missing_package_manager(self) -> Optional[str]: + return None + + def system_packages(self) -> Sequence[Sequence[str]]: + return ( + ("sudo", "apt-get", "update"), + ("sudo", "apt-get", "install", "-y", *SYSTEM_PACKAGES), + ) + + def build_environment(self, *, machine: str, portaudio_prefix: str) -> Sequence[str]: + del machine, portaudio_prefix + return () diff --git a/scripts/bootstrap/platforms/macos.py b/scripts/bootstrap/platforms/macos.py new file mode 100644 index 000000000..582a1155c --- /dev/null +++ b/scripts/bootstrap/platforms/macos.py @@ -0,0 +1,70 @@ +from pathlib import Path +from typing import Final, Optional, Sequence + +from bootstrap.platforms.linux import PNG_ICON, posix_interpreter, posix_launcher + +DARWIN: Final[str] = "Darwin" +HOMEBREW: Final[str] = "brew" +HOMEBREW_SITE: Final[str] = "https://brew.sh" +PORTAUDIO: Final[str] = "portaudio" + + +class MacOS: + """macOS: PortAudio through Homebrew, and the application run from source.""" + + @property + def name(self) -> str: + return DARWIN + + @property + def bundles(self) -> bool: + return False + + @property + def icon(self) -> str: + return PNG_ICON + + @property + def pyaudio_advice(self) -> str: + return "Run 'make system-deps' to install PortAudio through Homebrew." + + @property + def tkinter_advice(self) -> str: + return "Install Python from python.org, which includes Tk." + + @property + def tkinter_warning(self) -> str: + return "This bundle opens no file dialogs. Install Python from python.org to include Tk." + + def interpreter(self, environment: Path) -> Path: + return posix_interpreter(environment) + + def launcher(self, distribution: Path, *, release: bool) -> Path: + return posix_launcher(distribution, release=release) + + def missing_package_manager(self) -> Optional[str]: + return ( + "ERROR: Homebrew is required to install the macOS system dependencies.\n" + f"Install it from {HOMEBREW_SITE}, then run this script again." + ) + + def system_packages(self) -> Sequence[Sequence[str]]: + return ((HOMEBREW, "install", PORTAUDIO),) + + def build_environment(self, *, machine: str, portaudio_prefix: str) -> Sequence[str]: + """The flags compiling audio playback against Homebrew's PortAudio on the native architecture. + + Raises: + SystemExit: If Homebrew reported no PortAudio prefix. + """ + if not portaudio_prefix: + raise SystemExit( + "ERROR: Homebrew is required to locate the PortAudio headers and library.\n" + "Run 'make system-deps' first." + ) + + return ( + f"CFLAGS=-I{portaudio_prefix}/include", + f"LDFLAGS=-L{portaudio_prefix}/lib", + f"ARCHFLAGS=-arch {machine}", + ) diff --git a/scripts/bootstrap/platforms/protocol.py b/scripts/bootstrap/platforms/protocol.py new file mode 100644 index 000000000..addb7db55 --- /dev/null +++ b/scripts/bootstrap/platforms/protocol.py @@ -0,0 +1,69 @@ +from pathlib import Path +from typing import Optional, Protocol, Sequence + + +class Platform(Protocol): + """What building and running SampleToNES has to know about the system it happens on.""" + + @property + def name(self) -> str: + """The name ``platform.system()`` reports for the system.""" + + @property + def bundles(self) -> bool: + """Whether a standalone bundle is built on the system.""" + + @property + def icon(self) -> str: + """The icon file a bundle is stamped with, relative to the repository.""" + + @property + def pyaudio_advice(self) -> str: + """How to supply PortAudio to the build interpreter.""" + + @property + def tkinter_advice(self) -> str: + """How to supply Tk to the build interpreter for a release bundle.""" + + @property + def tkinter_warning(self) -> str: + """What a development bundle built without Tk does about file dialogs.""" + + def interpreter(self, environment: Path) -> Path: + """The interpreter a virtual environment at ``environment`` runs. + + Args: + environment: The virtual environment's directory. + + Returns: + Path: The interpreter. + """ + + def launcher(self, distribution: Path, *, release: bool) -> Path: + """The executable PyInstaller writes under ``distribution``. + + Args: + distribution: The directory the bundle is written into. + release: Whether the bundle is a release, which is a directory beside its launcher. + + Returns: + Path: The launcher. + """ + + def missing_package_manager(self) -> Optional[str]: + """What stands in the way of installing system packages, or ``None`` where nothing does.""" + + def system_packages(self) -> Sequence[Sequence[str]]: + """The commands that install the system packages the application needs, in order.""" + + def build_environment(self, *, machine: str, portaudio_prefix: str) -> Sequence[str]: + """The ``KEY=VALUE`` lines a build exports so audio playback compiles against PortAudio. + + Args: + machine: The processor architecture ``platform.machine()`` reports. + portaudio_prefix: Where the package manager installed PortAudio, or empty where the + system carries it without one. + + Returns: + Sequence[str]: The lines, empty where the build needs none. + """ diff --git a/scripts/bootstrap/platforms/windows.py b/scripts/bootstrap/platforms/windows.py new file mode 100644 index 000000000..7f02ef126 --- /dev/null +++ b/scripts/bootstrap/platforms/windows.py @@ -0,0 +1,55 @@ +from pathlib import Path +from typing import Final, Optional, Sequence, Tuple + +WINDOWS: Final[str] = "Windows" +WINDOWS_LAUNCHER: Final[str] = "sampletones.exe" +BUNDLE_DIRECTORY: Final[str] = "sampletones" +WINDOWS_INTERPRETER: Final[Tuple[str, str]] = ("Scripts", "python.exe") +ICO_ICON: Final[str] = "src/sampletones_assets/icons/sampletones.ico" + + +class Windows: + """Windows: the official Python installer carries what the application needs.""" + + @property + def name(self) -> str: + return WINDOWS + + @property + def bundles(self) -> bool: + return True + + @property + def icon(self) -> str: + return ICO_ICON + + @property + def pyaudio_advice(self) -> str: + return "Run install.bat from the project root to reinstall the dependencies." + + @property + def tkinter_advice(self) -> str: + return "Install Python from python.org or the Microsoft Store, which both include Tk, then build again." + + @property + def tkinter_warning(self) -> str: + return "This bundle opens no file dialogs. Install Python from python.org or the Microsoft Store to include Tk." + + def interpreter(self, environment: Path) -> Path: + return environment.joinpath(*WINDOWS_INTERPRETER) + + def launcher(self, distribution: Path, *, release: bool) -> Path: + if release: + return distribution / BUNDLE_DIRECTORY / WINDOWS_LAUNCHER + + return distribution / WINDOWS_LAUNCHER + + def missing_package_manager(self) -> Optional[str]: + return None + + def system_packages(self) -> Sequence[Sequence[str]]: + return () + + def build_environment(self, *, machine: str, portaudio_prefix: str) -> Sequence[str]: + del machine, portaudio_prefix + return () diff --git a/scripts/bootstrap/preflight.py b/scripts/bootstrap/preflight.py new file mode 100644 index 000000000..c317bbf34 --- /dev/null +++ b/scripts/bootstrap/preflight.py @@ -0,0 +1,89 @@ +from pathlib import Path +from typing import Final, Mapping + +from bootstrap.platforms.protocol import Platform +from bootstrap.processes import Runner + +PYAUDIO: Final[str] = "pyaudio" +TKINTER: Final[str] = "tkinter" + + +def can_import( + python: Path, + module: str, + *, + runner: Runner, + cwd: Path, + environment: Mapping[str, str], +) -> bool: + """Whether the interpreter at ``python`` imports ``module``. + + Args: + python: The interpreter. + module: The module's name. + runner: What runs the probe. + cwd: The directory the probe runs in. + environment: The variables the probe sees. + + Returns: + bool: Whether the import succeeds. + """ + status = runner( + (str(python), "-c", f"import {module}"), + cwd=cwd, + environment=environment, + quiet=True, + ) + return status == 0 + + +def check_build_interpreter( + python: Path, + platform: Platform, + *, + release: bool, + runner: Runner, + cwd: Path, + environment: Mapping[str, str], +) -> None: + """Holds the build interpreter to what a bundle has to carry. + + Audio playback is required of every bundle. Tk is required of a release bundle, so the + shipped executable opens file dialogs on its own; a development bundle built without it is + warned about what it leans on instead. + + Args: + python: The build environment's interpreter. + platform: The system the build runs on. + release: Whether the bundle is a release. + runner: What runs the probes. + cwd: The directory the probes run in. + environment: The variables the probes see. + + Raises: + SystemExit: If the interpreter is missing, cannot play audio, or a release lacks Tk. + """ + print("Checking the build environment...") + if not python.is_file(): + raise SystemExit( + f"ERROR: build interpreter not found at {python}.\nRun 'make build' to create the build environment." + ) + + if not can_import(python, PYAUDIO, runner=runner, cwd=cwd, environment=environment): + raise SystemExit( + "ERROR: the build interpreter cannot import pyaudio, so the bundle would carry no audio playback.\n" + f"{platform.pyaudio_advice}" + ) + + print(f"{PYAUDIO}: available") + if can_import(python, TKINTER, runner=runner, cwd=cwd, environment=environment): + print(f"{TKINTER}: available") + return + + if release: + raise SystemExit( + "ERROR: the build interpreter cannot import tkinter, so a release bundle would depend on the " + f"machine running it for file dialogs.\n{platform.tkinter_advice}" + ) + + print(f"WARNING: the build interpreter cannot import tkinter. {platform.tkinter_warning}") diff --git a/scripts/bootstrap/processes.py b/scripts/bootstrap/processes.py new file mode 100644 index 000000000..2364d3c97 --- /dev/null +++ b/scripts/bootstrap/processes.py @@ -0,0 +1,70 @@ +import shlex +import subprocess +from pathlib import Path +from typing import Mapping, Protocol, Sequence + + +class Runner(Protocol): + """Runs a command to completion and answers with its exit status.""" + + def __call__( + self, + command: Sequence[str], + *, + cwd: Path, + environment: Mapping[str, str], + quiet: bool, + ) -> int: ... + + +def run( + command: Sequence[str], + *, + cwd: Path, + environment: Mapping[str, str], + quiet: bool, +) -> int: + """Runs a command in ``cwd`` under ``environment`` and answers with its exit status. + + A quiet run keeps the command's output to itself, which is what a probe asks for. + + Args: + command: The program and its arguments. + cwd: The directory the command runs in. + environment: The variables the command sees. + quiet: Whether the command's output is captured instead of shown. + + Returns: + int: The command's exit status. + """ + completed = subprocess.run( + list(command), + cwd=str(cwd), + env=dict(environment), + check=False, + capture_output=quiet, + ) + return completed.returncode + + +def expect_success( + runner: Runner, + command: Sequence[str], + *, + cwd: Path, + environment: Mapping[str, str], +) -> None: + """Runs a command the operation depends on. + + Args: + runner: What runs the command. + command: The program and its arguments. + cwd: The directory the command runs in. + environment: The variables the command sees. + + Raises: + SystemExit: If the command exits with a status other than zero. + """ + status = runner(command, cwd=cwd, environment=environment, quiet=False) + if status != 0: + raise SystemExit(f"ERROR: {shlex.join(command)} exited with status {status}") diff --git a/scripts/bootstrap/repository.py b/scripts/bootstrap/repository.py new file mode 100644 index 000000000..60c84a159 --- /dev/null +++ b/scripts/bootstrap/repository.py @@ -0,0 +1,17 @@ +from pathlib import Path +from typing import Final + +REPOSITORY_ROOT: Final[Path] = Path(__file__).resolve().parents[2] +ENTRY_PACKAGE: Final[Path] = REPOSITORY_ROOT / "src" / "sampletones" + + +def repository_root() -> Path: + """The repository the scripts belong to, which every build and clean-up runs against. + + Raises: + FileNotFoundError: If the scripts lie outside a SampleToNES checkout. + """ + if not ENTRY_PACKAGE.is_dir(): + raise FileNotFoundError(f"SampleToNES project root not found (expected {ENTRY_PACKAGE})") + + return REPOSITORY_ROOT diff --git a/scripts/bootstrap/venv_build.py b/scripts/bootstrap/venv_build.py new file mode 100644 index 000000000..f0dd7ac09 --- /dev/null +++ b/scripts/bootstrap/venv_build.py @@ -0,0 +1,94 @@ +import sys +from pathlib import Path +from typing import Dict, Final, Mapping, Sequence + +from bootstrap.platforms.protocol import Platform +from bootstrap.processes import Runner, expect_success + +BUILD_ENVIRONMENT: Final[str] = ".venv-build" +PIP_REQUIRE_VIRTUALENV: Final[str] = "PIP_REQUIRE_VIRTUALENV" +GROUP_FLAG: Final[str] = "--group" + + +def build_environment( + root: Path, + *, + runner: Runner, + environment: Mapping[str, str], +) -> Path: + """The virtual environment a bundle is built in, created under ``root`` where it is missing. + + Every package a build installs lands here, so the interpreter running the script stays as + it was found. + + Args: + root: The repository. + runner: What runs the command creating the environment. + environment: The variables the command sees. + + Returns: + Path: The environment's directory. + """ + directory = root / BUILD_ENVIRONMENT + if directory.is_dir(): + print("Virtual environment already exists.") + return directory + + print("Creating virtual environment...") + expect_success( + runner, + (sys.executable, "-m", "venv", str(directory)), + cwd=root, + environment=environment, + ) + print("Virtual environment created.") + return directory + + +def install( + root: Path, + python: Path, + *, + extras: Sequence[str], + groups: Sequence[str], + runner: Runner, + environment: Mapping[str, str], +) -> None: + """Installs the package with ``extras`` and ``groups`` into the environment ``python`` runs. + + Pip is told to refuse any interpreter outside a virtual environment, so an install reaches + the build environment alone. + + Args: + root: The repository, which is the package installed. + python: The build environment's interpreter. + extras: The optional-dependency extras installed with the package. + groups: The dependency groups installed beside it. + runner: What runs the commands. + environment: The variables the commands see. + """ + guarded: Dict[str, str] = {**environment, PIP_REQUIRE_VIRTUALENV: "1"} + print("Installing dependencies...") + expect_success( + runner, + (str(python), "-m", "pip", "install", "--upgrade", "pip"), + cwd=root, + environment=guarded, + ) + print(f"Installing with extras: {','.join(extras)}") + group_flags = [flag for group in groups for flag in (GROUP_FLAG, group)] + expect_success( + runner, + (str(python), "-m", "pip", "install", f".[{','.join(extras)}]", *group_flags), + cwd=root, + environment=guarded, + ) + print("sampletones Python package installed successfully.") + + +def interpreter( + root: Path, + platform: Platform, +) -> Path: + """The interpreter of the build environment under ``root``.""" + return platform.interpreter(root / BUILD_ENVIRONMENT) diff --git a/scripts/build_environment.py b/scripts/build_environment.py new file mode 100644 index 000000000..cf952a14c --- /dev/null +++ b/scripts/build_environment.py @@ -0,0 +1,47 @@ +import argparse +import platform as running +import shutil +import subprocess +import sys +from typing import Final, Sequence + +from bootstrap.platforms.factory import current_platform + +HOMEBREW: Final[str] = "brew" +PORTAUDIO: Final[str] = "portaudio" + + +def portaudio_prefix() -> str: + """Where Homebrew installed PortAudio, or empty where Homebrew is absent.""" + if shutil.which(HOMEBREW) is None: + return "" + + completed = subprocess.run( + [HOMEBREW, "--prefix", PORTAUDIO], + capture_output=True, + text=True, + check=False, + ) + if completed.returncode != 0: + return "" + + return completed.stdout.strip() + + +def main(argv: Sequence[str]) -> int: + """Prints the variables a build exports so audio playback compiles, one ``KEY=VALUE`` per line.""" + parser = argparse.ArgumentParser(description="Print the build environment audio playback compiles under.") + parser.parse_args(list(argv)) + + lines = current_platform().build_environment( + machine=running.machine(), + portaudio_prefix=portaudio_prefix(), + ) + for line in lines: + print(line) + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) diff --git a/scripts/bundle.py b/scripts/bundle.py new file mode 100644 index 000000000..6941f2404 --- /dev/null +++ b/scripts/bundle.py @@ -0,0 +1,235 @@ +import argparse +import os +import shutil +import sys +from dataclasses import dataclass +from pathlib import Path +from typing import Final, List, Mapping, Sequence, Tuple + +from bootstrap.interpreter import REQUIRED_VERSION, require_python +from bootstrap.platforms.factory import current_platform +from bootstrap.platforms.protocol import Platform +from bootstrap.preflight import check_build_interpreter +from bootstrap.processes import Runner, expect_success, run +from bootstrap.repository import repository_root +from bootstrap.venv_build import build_environment, install, interpreter + +BUNDLE_NAME: Final[str] = "sampletones" +DISTRIBUTION: Final[str] = "bin" +ENTRY: Final[str] = "src/sampletones/__main__.py" +RELEASE_HOOK: Final[str] = "scripts/runtime_hooks/release_environment.py" +ICONS_SCRIPT: Final[str] = "scripts/assets/icons.py" +SELF_CHECK: Final[str] = "--self-check" +BUILD_EXTRA: Final[str] = "build" +GPU_EXTRA: Final[str] = "gpu" +GROUPS: Final[Tuple[str, ...]] = ("assets",) +DATA: Final[Tuple[Tuple[str, str], ...]] = ( + ("src/sampletones_assets/icons", "assets/icons"), + ("src/sampletones_assets/fonts", "assets/fonts"), + ("src/sampletones_config", "config"), + ("src/sampletones_player/driver/binary", "sampletones_player/driver/binary"), +) +DATA_SEPARATOR: Final[str] = ":" +EXCLUDED_MODULES: Final[Tuple[str, ...]] = ("PIL",) +NOTICES: Final[Tuple[str, ...]] = ("LICENSE", "THIRD-PARTY-NOTICES.md", "THIRD-PARTY-LICENSES.txt") +NO_BUNDLE: Final[str] = ( + "ERROR: a standalone bundle is built on Linux and Windows.\n" + "On macOS, SampleToNES runs from source:\n" + "\n" + " make system-deps\n" + " make setup\n" + " make run\n" + "\n" + "See docs/guide/installation.md for the full steps." +) + + +@dataclass(frozen=True) +class BundleOptions: + """How a bundle is built. + + Attributes: + release: Whether the bundle is a release: a directory beside its launcher, carrying the + release deployment configuration and the notices. + gpu: Whether the bundle carries GPU support. + """ + + release: bool + gpu: bool + + +def extras(options: BundleOptions) -> Tuple[str, ...]: + """The optional-dependency extras a bundle is built with.""" + if options.gpu: + return (BUILD_EXTRA, GPU_EXTRA) + + return (BUILD_EXTRA,) + + +def pyinstaller_command( + python: Path, + platform: Platform, + options: BundleOptions, +) -> List[str]: + """The PyInstaller invocation that writes the bundle. + + Args: + python: The build environment's interpreter. + platform: The system the bundle is built for. + options: How the bundle is built. + + Returns: + List[str]: The command, run from the repository root. + """ + command = [ + str(python), + "-m", + "PyInstaller", + "--name", + BUNDLE_NAME, + "--onedir" if options.release else "--onefile", + "--noconfirm", + "--distpath", + DISTRIBUTION, + "--icon", + platform.icon, + ] + for source, destination in DATA: + command.extend(("--add-data", f"{source}{DATA_SEPARATOR}{destination}")) + + command.extend(("--copy-metadata", BUNDLE_NAME)) + for module in EXCLUDED_MODULES: + command.extend(("--exclude-module", module)) + + if options.release: + command.extend(("--runtime-hook", RELEASE_HOOK)) + + command.append(ENTRY) + return command + + +def remove_previous(distribution: Path) -> None: + """Removes what an earlier build left under ``distribution``, whichever shape it took.""" + for previous in (distribution / BUNDLE_NAME, distribution / f"{BUNDLE_NAME}.exe"): + if previous.is_dir(): + print(f"Removing the previous artifact: {previous}") + shutil.rmtree(previous) + elif previous.exists(): + print(f"Removing the previous artifact: {previous}") + previous.unlink() + + +def copy_notices(root: Path, bundle: Path) -> None: + """Places the license and notice files beside a release bundle's launcher.""" + for notice in NOTICES: + shutil.copyfile(root / notice, bundle / notice) + + print(f"Bundled notices: {', '.join(NOTICES)}") + + +def build_bundle( + root: Path, + platform: Platform, + options: BundleOptions, + *, + runner: Runner, + environment: Mapping[str, str], +) -> Path: + """Builds the standalone bundle in a build environment of its own and verifies it starts. + + Args: + root: The repository. + platform: The system the bundle is built on and for. + options: How the bundle is built. + runner: What runs the commands. + environment: The variables the commands see. + + Returns: + Path: The launcher the bundle offers. + + Raises: + SystemExit: If a step fails, or PyInstaller produced no launcher. + """ + if options.release: + print("Release build: onedir bundle, injecting release deployment configuration") + + build_environment(root, runner=runner, environment=environment) + python = interpreter(root, platform) + install( + root, + python, + extras=extras(options), + groups=GROUPS, + runner=runner, + environment=environment, + ) + check_build_interpreter( + python, + platform, + release=options.release, + runner=runner, + cwd=root, + environment=environment, + ) + print("Generating the icon suite...") + expect_success(runner, (str(python), ICONS_SCRIPT), cwd=root, environment=environment) + + distribution = root / DISTRIBUTION + remove_previous(distribution) + print("Building executable...") + expect_success( + runner, + pyinstaller_command(python, platform, options), + cwd=root, + environment=environment, + ) + launcher = platform.launcher(distribution, release=options.release) + if not launcher.is_file(): + raise SystemExit(f"Build failed: PyInstaller produced no executable at {launcher}.") + + print("Verifying the bundle...") + status = runner((str(launcher), SELF_CHECK), cwd=root, environment=environment, quiet=False) + if status != 0: + raise SystemExit(f"Build failed: {launcher} did not pass its self-check.") + + if options.release: + copy_notices(root, launcher.parent) + + print(f"Build complete: {launcher}") + return launcher + + +def main(argv: Sequence[str]) -> int: + """Builds the standalone bundle, as a development build or a release.""" + parser = argparse.ArgumentParser(description="Build the standalone SampleToNES bundle.") + parser.add_argument( + "--release", + action="store_true", + help="build a release: a directory bundle with the release deployment configuration and the notices", + ) + parser.add_argument( + "--gpu", + action="store_true", + help="build with GPU support", + ) + arguments = parser.parse_args(list(argv)) + options = BundleOptions(release=arguments.release, gpu=arguments.gpu) + + require_python(REQUIRED_VERSION) + platform = current_platform() + if not platform.bundles: + print(NO_BUNDLE, file=sys.stderr) + return 1 + + build_bundle( + repository_root(), + platform, + options, + runner=run, + environment=os.environ, + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) diff --git a/scripts/ci/__init__.py b/scripts/ci/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/scripts/ci/checks/__init__.py b/scripts/ci/checks/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/scripts/clean.py b/scripts/clean.py new file mode 100644 index 000000000..519f36241 --- /dev/null +++ b/scripts/clean.py @@ -0,0 +1,63 @@ +import argparse +import os +import shutil +import sys +from pathlib import Path +from typing import Final, Sequence, Tuple + +from bootstrap.repository import repository_root + +ARTIFACTS: Final[Tuple[str, ...]] = ("bin", "build", "dist", "htmlcov", ".coverage") +ARTIFACT_PATTERNS: Final[Tuple[str, ...]] = ("*.spec",) +CACHE_DIRECTORIES: Final[Tuple[str, ...]] = ("__pycache__",) +CACHE_DIRECTORY_SUFFIXES: Final[Tuple[str, ...]] = (".egg-info",) +CACHE_FILE_SUFFIXES: Final[Tuple[str, ...]] = (".pyc",) +LEFT_ALONE: Final[Tuple[str, ...]] = (".git", ".venv", ".venv-build") + + +def _remove(path: Path) -> None: + if path.is_dir(): + shutil.rmtree(path) + elif path.exists(): + path.unlink() + + +def remove_artifacts(root: Path) -> None: + """Removes the build outputs and the coverage reports under ``root``.""" + for name in ARTIFACTS: + _remove(root / name) + + for pattern in ARTIFACT_PATTERNS: + for path in root.glob(pattern): + _remove(path) + + +def remove_caches(root: Path) -> None: + """Removes the bytecode caches and packaging leftovers under ``root``, the environments left alone.""" + for directory, subdirectories, files in os.walk(root): + subdirectories[:] = [name for name in subdirectories if name not in LEFT_ALONE] + for name in list(subdirectories): + if name in CACHE_DIRECTORIES or name.endswith(CACHE_DIRECTORY_SUFFIXES): + shutil.rmtree(Path(directory) / name) + subdirectories.remove(name) + + for name in files: + if name.endswith(CACHE_FILE_SUFFIXES): + (Path(directory) / name).unlink() + + +def main(argv: Sequence[str]) -> int: + """Removes the build artifacts and cache files from the repository.""" + parser = argparse.ArgumentParser(description="Remove build artifacts and cache files.") + parser.parse_args(list(argv)) + + root = repository_root() + print("Removing build artifacts and temporary files...") + remove_artifacts(root) + remove_caches(root) + print("Cleaned build artifacts and temporary files.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) diff --git a/scripts/linux/build/build.sh b/scripts/linux/build/build.sh deleted file mode 100755 index 198b0acf7..000000000 --- a/scripts/linux/build/build.sh +++ /dev/null @@ -1,71 +0,0 @@ -#!/usr/bin/env bash - -set -e - -SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) -. "$SCRIPT_DIR/../lib/root.sh" - -PROJECT_DIR=$(CDPATH= cd -- "$SCRIPT_DIR/../../.." && pwd) -VENV_DIR="$PROJECT_DIR/.venv-build" -VENV_PY="$VENV_DIR/bin/python" - -RELEASE=0 -BUNDLE_ARGS=(--onefile) -RELEASE_HOOK_ARGS=() -for arg in "$@"; do - if [[ "$arg" == "--release" ]]; then - echo "Release build: onedir bundle, injecting release deployment configuration" - RELEASE=1 - BUNDLE_ARGS=(--onedir) - RELEASE_HOOK_ARGS=(--runtime-hook scripts/release_env_hook.py) - fi -done - -if [[ "${RELEASE}" == "1" ]]; then - EXECUTABLE="bin/sampletones/sampletones" -else - EXECUTABLE="bin/sampletones" -fi - -bash "$SCRIPT_DIR/preflight.sh" "$@" -bash "$SCRIPT_DIR/icons.sh" - -if [[ -e "${PROJECT_DIR}/bin/sampletones" ]]; then - echo "Removing the previous artifact: ./bin/sampletones" - rm -rf "${PROJECT_DIR}/bin/sampletones" -fi - -echo "Building executable..." -"$VENV_PY" -m PyInstaller --name sampletones \ - "${BUNDLE_ARGS[@]}" \ - --noconfirm \ - --distpath ./bin \ - --icon "src/sampletones_assets/icons/sampletones.png" \ - --add-data "src/sampletones_assets/icons:assets/icons" \ - --add-data "src/sampletones_assets/fonts:assets/fonts" \ - --add-data "src/sampletones_config:config" \ - --add-data "src/sampletones_player/driver/binary:sampletones_player/driver/binary" \ - --copy-metadata sampletones \ - --exclude-module PIL \ - "${RELEASE_HOOK_ARGS[@]}" \ - "src/sampletones/__main__.py" - -if [[ ! -x "${EXECUTABLE}" ]]; then - echo "Build failed: PyInstaller produced no executable at ./${EXECUTABLE}." >&2 - exit 1 -fi - -echo "Verifying the bundle..." -if ! "${PROJECT_DIR}/${EXECUTABLE}" --self-check; then - echo "Build failed: ./${EXECUTABLE} did not pass its self-check." >&2 - exit 1 -fi - -if [[ "${RELEASE}" == "1" ]]; then - for notice in LICENSE THIRD-PARTY-NOTICES.md THIRD-PARTY-LICENSES.txt; do - cp "${notice}" "bin/sampletones/${notice}" - done - echo "Bundled notices: LICENSE, THIRD-PARTY-NOTICES.md, THIRD-PARTY-LICENSES.txt" -fi - -echo "Build complete: ./${EXECUTABLE}" diff --git a/scripts/linux/build/clean.sh b/scripts/linux/build/clean.sh deleted file mode 100755 index 932a5fe58..000000000 --- a/scripts/linux/build/clean.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "${BASH_SOURCE[0]}")/../lib/root.sh" - -echo "Removing build artifacts and temporary files..." -rm -rf bin/ build/ dist/ *.spec htmlcov/ .coverage -find . -type d -name "__pycache__" -exec rm -rf {} + 2>/dev/null || true -find . -type d -name "*.egg-info" -exec rm -rf {} + 2>/dev/null || true -find . -type f -name "*.pyc" -delete -echo "Cleaned build artifacts and temporary files." diff --git a/scripts/linux/build/dependencies.sh b/scripts/linux/build/dependencies.sh deleted file mode 100755 index 7b1758276..000000000 --- a/scripts/linux/build/dependencies.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env bash - -set -e - -PACKAGES=( - # PortAudio: audio playback through pyaudio - libportaudio2 - libasound-dev - libpulse-dev - portaudio19-dev - # Tk: file dialogs where kdialog and zenity are absent - python3-tk - tk-dev - tcl-dev - # OpenGL and X11: the window DearPyGui opens through GLFW - libgl1 - libegl1 - libx11-6 - libx11-xcb1 - libxcursor1 - libxi6 - libxinerama1 - libxrandr2 - libxrender1 - libxxf86vm1 -) - -echo "Installing system dependencies (requires sudo)" -sudo apt-get update -sudo apt-get install -y "${PACKAGES[@]}" -echo "System dependencies installed." diff --git a/scripts/linux/build/icons.sh b/scripts/linux/build/icons.sh deleted file mode 100644 index df7bf35b8..000000000 --- a/scripts/linux/build/icons.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env bash - -set -e - -SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) -. "$SCRIPT_DIR/../lib/root.sh" - -PROJECT_DIR=$(CDPATH= cd -- "$SCRIPT_DIR/../../.." && pwd) -VENV_PY="$PROJECT_DIR/.venv-build/bin/python" - -echo "Generating the icon suite..." -"$VENV_PY" scripts/assets/icons.py diff --git a/scripts/linux/build/preflight.sh b/scripts/linux/build/preflight.sh deleted file mode 100755 index 5008bde05..000000000 --- a/scripts/linux/build/preflight.sh +++ /dev/null @@ -1,53 +0,0 @@ -#!/usr/bin/env bash - -set -e - -SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) -. "$SCRIPT_DIR/../lib/root.sh" - -PROJECT_DIR=$(CDPATH= cd -- "$SCRIPT_DIR/../../.." && pwd) -VENV_DIR="$PROJECT_DIR/.venv-build" -VENV_PY="$VENV_DIR/bin/python" - -RELEASE=0 -for arg in "$@"; do - if [[ "$arg" == "--release" ]]; then - RELEASE=1 - fi -done - -can_import() { - "$VENV_PY" -c "import $1" >/dev/null 2>&1 -} - -echo "Checking the build environment..." - -if [[ ! -x "$VENV_PY" ]]; then - echo "ERROR: build interpreter not found at ${VENV_PY}." >&2 - echo "Run './install.sh' from the project root to create the build environment." >&2 - exit 1 -fi - -if ! can_import pyaudio; then - echo "ERROR: the build interpreter cannot import pyaudio, so the bundle would carry no audio playback." >&2 - echo "Run 'make system-deps' to install the PortAudio packages, then './install.sh' to reinstall dependencies." >&2 - exit 1 -fi - -echo "pyaudio: available" - -if can_import tkinter; then - echo "tkinter: available" - exit 0 -fi - -if [[ "${RELEASE}" == "1" ]]; then - echo "ERROR: the build interpreter cannot import tkinter, so a release bundle would depend on the" >&2 - echo "target machine providing zenity or kdialog for file dialogs." >&2 - echo "Run 'make system-deps' to install python3-tk, then build again." >&2 - exit 1 -fi - -echo "WARNING: the build interpreter cannot import tkinter." -echo "This bundle opens file dialogs through zenity or kdialog, which the machine running it has to provide." -echo "Run 'make system-deps' to install python3-tk and carry Tk as a self-contained fallback." diff --git a/scripts/linux/build/python.sh b/scripts/linux/build/python.sh deleted file mode 100755 index fec272e0a..000000000 --- a/scripts/linux/build/python.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env bash - -set -e - -PYVER=$(python3 --version 2>&1 | awk '{print $2}') -echo "Detected Python version: $PYVER" -MAJOR=$(echo $PYVER | cut -d. -f1) -MINOR=$(echo $PYVER | cut -d. -f2) -if [ "$MAJOR" -lt 3 ] || { [ "$MAJOR" -eq 3 ] && [ "$MINOR" -lt 12 ]; }; then - echo - echo "ERROR: Python 3.12 or newer is required." - echo "Please install Python 3.12+ from https://www.python.org/downloads/" - exit 1 -fi diff --git a/scripts/linux/build/sampletones.sh b/scripts/linux/build/sampletones.sh deleted file mode 100755 index d5accc984..000000000 --- a/scripts/linux/build/sampletones.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/env bash - -set -e - -SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) -. "$SCRIPT_DIR/../lib/root.sh" - -PROJECT_DIR=$(CDPATH= cd -- "$SCRIPT_DIR/../../.." && pwd) -VENV_DIR="$PROJECT_DIR/.venv-build" -VENV_PY="$VENV_DIR/bin/python" - -EXTRAS=("build") -for arg in "$@"; do - case $arg in - --gpu) - EXTRAS+=("gpu") - ;; - esac -done - -echo "Installing dependencies..." -"$VENV_PY" -m pip install --upgrade pip - -EXTRAS_STR=$(IFS=,; echo "${EXTRAS[*]}") -echo "Installing with extras: $EXTRAS_STR" -"$VENV_PY" -m pip install ".[$EXTRAS_STR]" --group assets - -echo "sampletones Python package installed successfully." diff --git a/scripts/linux/build/venv.sh b/scripts/linux/build/venv.sh deleted file mode 100755 index 29b7ad326..000000000 --- a/scripts/linux/build/venv.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env bash - -set -e - -source "$(dirname "${BASH_SOURCE[0]}")/../lib/root.sh" - -if [[ -d ".venv-build" ]]; then - echo "Virtual environment already exists." -else - echo "Creating virtual environment..." - python3 -m venv .venv-build - echo "Virtual environment created." -fi - -return 0 diff --git a/scripts/linux/lib/root.sh b/scripts/linux/lib/root.sh deleted file mode 100755 index a5859c2eb..000000000 --- a/scripts/linux/lib/root.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env bash - -_PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)" - -if [[ ! -d "${_PROJECT_ROOT}/src/sampletones" ]]; then - echo "ERROR: SampleToNES project root not found (expected ${_PROJECT_ROOT}/src/sampletones)." >&2 - exit 1 -fi - -cd "${_PROJECT_ROOT}" diff --git a/scripts/macos/build/build_env.sh b/scripts/macos/build/build_env.sh deleted file mode 100755 index f6d195827..000000000 --- a/scripts/macos/build/build_env.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env bash - -set -e - -if ! command -v brew >/dev/null 2>&1; then - echo "ERROR: Homebrew is required to locate the PortAudio headers and library." >&2 - echo "Run scripts/macos/build/dependencies.sh first." >&2 - exit 1 -fi - -PORTAUDIO_PREFIX=$(brew --prefix portaudio) - -echo "CFLAGS=-I${PORTAUDIO_PREFIX}/include" -echo "LDFLAGS=-L${PORTAUDIO_PREFIX}/lib" -echo "ARCHFLAGS=-arch $(uname -m)" diff --git a/scripts/macos/build/dependencies.sh b/scripts/macos/build/dependencies.sh deleted file mode 100755 index ca5049082..000000000 --- a/scripts/macos/build/dependencies.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env bash - -set -e - -PACKAGES=( - portaudio -) - -if ! command -v brew >/dev/null 2>&1; then - echo "ERROR: Homebrew is required to install the macOS system dependencies." >&2 - echo "Install it from https://brew.sh, then run this script again." >&2 - exit 1 -fi - -echo "Installing system dependencies through Homebrew" -brew install "${PACKAGES[@]}" -echo "System dependencies installed." diff --git a/scripts/macos/build/no_bundle.sh b/scripts/macos/build/no_bundle.sh deleted file mode 100755 index b6cfb4ec7..000000000 --- a/scripts/macos/build/no_bundle.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env bash - -set -e - -OPERATION="${1:-this command}" - -echo "ERROR: ${OPERATION} supports Linux and Windows." >&2 -echo "On macOS, SampleToNES runs from source:" >&2 -echo >&2 -echo " make system-deps" >&2 -echo " make setup" >&2 -echo " make run" >&2 -echo >&2 -echo "See docs/guide/installation.md for the full steps." >&2 -exit 1 diff --git a/scripts/release_env_hook.py b/scripts/runtime_hooks/release_environment.py similarity index 100% rename from scripts/release_env_hook.py rename to scripts/runtime_hooks/release_environment.py diff --git a/scripts/setup_environment.py b/scripts/setup_environment.py new file mode 100644 index 000000000..66d3c065b --- /dev/null +++ b/scripts/setup_environment.py @@ -0,0 +1,102 @@ +import argparse +import os +import platform as running +import sys +from typing import Dict, Final, List, Mapping, Optional, Sequence + +import detect_cuda + +from bootstrap.processes import expect_success, run +from bootstrap.repository import repository_root + +GPU_AUTO: Final[str] = "auto" +GPU_OFF: Final[str] = "0" +DEFAULT_GPU: Final[str] = GPU_AUTO +DEVELOPMENT_GROUP: Final[str] = "dev" +ICONS_COMMAND: Final[Sequence[str]] = ("uv", "run", "--group", "assets", "python", "scripts/assets/icons.py") +DARWIN: Final[str] = "Darwin" +ARCHFLAGS: Final[str] = "ARCHFLAGS" + + +def gpu_extra(choice: str, *, system: str) -> Optional[str]: + """The optional-dependency extra a GPU choice selects. + + Args: + choice: ``auto`` to read the NVIDIA driver, ``0`` for the CPU backend, or an extra's name. + system: The name ``platform.system()`` reports. + + Returns: + Optional[str]: The extra, or ``None`` for the CPU backend. + """ + if choice == GPU_OFF: + return None + + if choice == GPU_AUTO: + detection = detect_cuda.detect(system=system) + print(detection.reason, file=sys.stderr) + return detection.extra + + return choice + + +def setup_commands(extra: Optional[str]) -> List[List[str]]: + """The commands that create the development environment and install the global command. + + Args: + extra: The GPU extra installed with the package, or ``None`` for the CPU backend. + + Returns: + List[List[str]]: The commands, in order. + """ + synchronize = ["uv", "sync", "--group", DEVELOPMENT_GROUP] + package = "." + if extra is not None: + synchronize.extend(("--extra", extra)) + package = f".[{extra}]" + + return [ + synchronize, + list(ICONS_COMMAND), + ["uv", "tool", "install", "--force", package], + ] + + +def setup_environment_variables( + base: Mapping[str, str], + *, + system: str, + machine: str, +) -> Dict[str, str]: + """The variables the setup commands see: the caller's, pinned to the native architecture on macOS. + + Homebrew's PortAudio carries the machine's own architecture while a python.org interpreter + compiles for both, so the flag settles audio playback on the native one. + """ + variables = dict(base) + if system == DARWIN: + variables[ARCHFLAGS] = f"-arch {machine}" + + return variables + + +def main(argv: Sequence[str]) -> int: + """Creates the development environment and installs the global ``sampletones`` command.""" + parser = argparse.ArgumentParser(description="Set up the SampleToNES development environment.") + parser.add_argument( + "--gpu", + default=DEFAULT_GPU, + help="auto to match the NVIDIA driver, 0 for the CPU backend, or the name of a GPU extra", + ) + arguments = parser.parse_args(list(argv)) + + root = repository_root() + system = running.system() + environment = setup_environment_variables(os.environ, system=system, machine=running.machine()) + for command in setup_commands(gpu_extra(arguments.gpu, system=system)): + expect_success(run, command, cwd=root, environment=environment) + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) diff --git a/scripts/system_dependencies.py b/scripts/system_dependencies.py new file mode 100644 index 000000000..8957664fa --- /dev/null +++ b/scripts/system_dependencies.py @@ -0,0 +1,37 @@ +import argparse +import os +import sys +from typing import Sequence + +from bootstrap.platforms.factory import current_platform +from bootstrap.processes import expect_success, run +from bootstrap.repository import repository_root + + +def main(argv: Sequence[str]) -> int: + """Installs the system packages building and running the application needs on this machine.""" + parser = argparse.ArgumentParser(description="Install the system packages SampleToNES needs.") + parser.parse_args(list(argv)) + + platform = current_platform() + missing = platform.missing_package_manager() + if missing is not None: + print(missing, file=sys.stderr) + return 1 + + commands = platform.system_packages() + if not commands: + print(f"Nothing to install on {platform.name}: the Python installer carries what the application needs.") + return 0 + + root = repository_root() + print("Installing system dependencies...") + for command in commands: + expect_success(run, command, cwd=root, environment=os.environ) + + print("System dependencies installed.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) diff --git a/scripts/windows/build/build.bat b/scripts/windows/build/build.bat deleted file mode 100644 index f2567962c..000000000 --- a/scripts/windows/build/build.bat +++ /dev/null @@ -1,81 +0,0 @@ -@echo off -setlocal EnableExtensions - -set "SCRIPT_DIR=%~dp0" -call "%SCRIPT_DIR%\..\lib\root.bat" || exit /b 1 - -set "PROJECT_DIR=%SCRIPT_DIR%..\..\.." -set "VENV_DIR=%PROJECT_DIR%\.venv-build" -set "VENV_PY=%VENV_DIR%\Scripts\python.exe" - -set RELEASE=0 -set BUNDLE_MODE=--onefile -set RELEASE_HOOK= -:parse_args -if "%~1"=="" goto build -if "%~1"=="--release" ( - echo Release build: onedir bundle, injecting release deployment configuration - set RELEASE=1 - set BUNDLE_MODE=--onedir - set RELEASE_HOOK=--runtime-hook scripts\release_env_hook.py -) -shift -goto parse_args - -:build -if "%RELEASE%"=="1" ( - set EXECUTABLE=bin\sampletones\sampletones.exe -) else ( - set EXECUTABLE=bin\sampletones.exe -) - -call "%SCRIPT_DIR%preflight.bat" %* || exit /b 1 -call "%SCRIPT_DIR%icons.bat" || exit /b 1 - -if exist "bin\sampletones.exe" ( - echo Removing the previous artifact: bin\sampletones.exe - del /Q "bin\sampletones.exe" || exit /b 1 -) - -if exist "bin\sampletones\" ( - echo Removing the previous artifact: bin\sampletones - rmdir /S /Q "bin\sampletones" || exit /b 1 -) - -echo Building executable... -"%VENV_PY%" -m PyInstaller --name sampletones ^ - %BUNDLE_MODE% ^ - --noconfirm ^ - --distpath bin ^ - --icon "src\sampletones_assets\icons\sampletones.ico" ^ - --add-data "src\sampletones_assets\icons;assets\icons" ^ - --add-data "src\sampletones_assets\fonts;assets\fonts" ^ - --add-data "src\sampletones_config;config" ^ - --add-data "src\sampletones_player\driver\binary;sampletones_player\driver\binary" ^ - --copy-metadata sampletones ^ - --exclude-module PIL ^ - %RELEASE_HOOK% ^ - "src\sampletones\__main__.py" || exit /b - -if not exist "%EXECUTABLE%" ( - echo Build failed: PyInstaller produced no executable at %EXECUTABLE%.>&2 - exit /b 1 -) - -echo Verifying the bundle... -"%EXECUTABLE%" --self-check -if errorlevel 1 ( - echo Build failed: %EXECUTABLE% did not pass its self-check.>&2 - exit /b 1 -) - -if "%RELEASE%"=="1" ( - copy /Y LICENSE bin\sampletones\LICENSE >nul || exit /b 1 - copy /Y THIRD-PARTY-NOTICES.md bin\sampletones\THIRD-PARTY-NOTICES.md >nul || exit /b 1 - copy /Y THIRD-PARTY-LICENSES.txt bin\sampletones\THIRD-PARTY-LICENSES.txt >nul || exit /b 1 - echo Bundled notices: LICENSE, THIRD-PARTY-NOTICES.md, THIRD-PARTY-LICENSES.txt -) - -echo Build complete: .\%EXECUTABLE% - -exit /b 0 diff --git a/scripts/windows/build/clean.bat b/scripts/windows/build/clean.bat deleted file mode 100644 index a41b8bf8d..000000000 --- a/scripts/windows/build/clean.bat +++ /dev/null @@ -1,18 +0,0 @@ -@echo off -setlocal - -call "%~dp0..\lib\root.bat" || exit /b 1 - -echo Cleaning build artifacts... -if exist bin rmdir /s /q bin 2>nul -if exist build rmdir /s /q build 2>nul -if exist dist rmdir /s /q dist 2>nul -if exist htmlcov rmdir /s /q htmlcov 2>nul -if exist .coverage del /q .coverage 2>nul -if exist *.spec del /q *.spec 2>nul -for /d /r . %%d in (__pycache__) do @if exist "%%d" rmdir /s /q "%%d" 2>nul -for /d /r . %%d in (*.egg-info) do @if exist "%%d" rmdir /s /q "%%d" 2>nul -del /s /q *.pyc 2>nul - -echo Clean complete. -exit /b 0 diff --git a/scripts/windows/build/icons.bat b/scripts/windows/build/icons.bat deleted file mode 100644 index d4ee7f14a..000000000 --- a/scripts/windows/build/icons.bat +++ /dev/null @@ -1,14 +0,0 @@ -@echo off -setlocal EnableExtensions - -set "SCRIPT_DIR=%~dp0" -call "%SCRIPT_DIR%\..\lib\root.bat" || exit /b 1 - -set "PROJECT_DIR=%SCRIPT_DIR%..\..\.." -set "VENV_DIR=%PROJECT_DIR%\.venv-build" -set "VENV_PY=%VENV_DIR%\Scripts\python.exe" - -echo Generating the icon suite... -"%VENV_PY%" scripts\assets\icons.py || exit /b 1 - -exit /b 0 diff --git a/scripts/windows/build/preflight.bat b/scripts/windows/build/preflight.bat deleted file mode 100644 index 6cb6a7467..000000000 --- a/scripts/windows/build/preflight.bat +++ /dev/null @@ -1,51 +0,0 @@ -@echo off -setlocal EnableExtensions - -set "SCRIPT_DIR=%~dp0" -call "%SCRIPT_DIR%\..\lib\root.bat" || exit /b 1 - -set "PROJECT_DIR=%SCRIPT_DIR%..\..\.." -set "VENV_DIR=%PROJECT_DIR%\.venv-build" -set "VENV_PY=%VENV_DIR%\Scripts\python.exe" - -set RELEASE=0 -:parse_args -if "%~1"=="" goto check -if "%~1"=="--release" set RELEASE=1 -shift -goto parse_args - -:check -echo Checking the build environment... - -if not exist "%VENV_PY%" ( - echo ERROR: build interpreter not found at %VENV_PY%.>&2 - echo Run install.bat from the project root to create the build environment.>&2 - exit /b 1 -) - -"%VENV_PY%" -c "import pyaudio" >nul 2>&1 -if errorlevel 1 ( - echo ERROR: the build interpreter cannot import pyaudio, so the bundle would carry no audio playback.>&2 - echo Run install.bat from the project root to reinstall the dependencies.>&2 - exit /b 1 -) - -echo pyaudio: available - -"%VENV_PY%" -c "import tkinter" >nul 2>&1 -if errorlevel 1 goto missing_tkinter - -echo tkinter: available -exit /b 0 - -:missing_tkinter -if "%RELEASE%"=="1" ( - echo ERROR: the build interpreter cannot import tkinter, so a release bundle would open no file dialogs.>&2 - echo Install Python from python.org or the Microsoft Store, which both include Tk, then build again.>&2 - exit /b 1 -) - -echo WARNING: the build interpreter cannot import tkinter. -echo This bundle opens no file dialogs. Install Python from python.org or the Microsoft Store to include Tk. -exit /b 0 diff --git a/scripts/windows/build/python.bat b/scripts/windows/build/python.bat deleted file mode 100644 index 381804ed4..000000000 --- a/scripts/windows/build/python.bat +++ /dev/null @@ -1,22 +0,0 @@ -@echo off -setlocal - -for /f "tokens=2 delims= " %%v in ('python --version 2^>^&1') do set PYTHON_VERSION=%%v -echo Detected Python version: %PYTHON_VERSION% - -for /f "tokens=1,2 delims=." %%a in ("%PYTHON_VERSION%") do set PY_MAJOR=%%a& set PY_MINOR=%%b - -set /a VERSION_OK=0 -if %PY_MAJOR% GEQ 3 ( - if %PY_MINOR% GEQ 12 set VERSION_OK=1 -) - -if not %VERSION_OK%==1 ( - echo. - echo ERROR: Python 3.12 or newer is required. - echo Please install Python 3.12+ from https://www.python.org/downloads/ - pause - exit /b 1 -) - -exit /b 0 diff --git a/scripts/windows/build/sampletones.bat b/scripts/windows/build/sampletones.bat deleted file mode 100644 index 902ea3726..000000000 --- a/scripts/windows/build/sampletones.bat +++ /dev/null @@ -1,28 +0,0 @@ -@echo off -setlocal enabledelayedexpansion -setlocal EnableExtensions - -set "SCRIPT_DIR=%~dp0" -call "%SCRIPT_DIR%\..\lib\root.bat" || exit /b 1 - -set "PROJECT_DIR=%SCRIPT_DIR%..\..\.." -set "VENV_DIR=%PROJECT_DIR%\.venv-build" -set "VENV_PY=%VENV_DIR%\Scripts\python.exe" - -set "EXTRAS=build" - -:parse_args -if "%~1"=="" goto install -if "%~1"=="--gpu" set "EXTRAS=!EXTRAS!,gpu" -shift -goto parse_args - -:install -echo Installing dependencies... -"%VENV_PY%" -m pip install --upgrade pip - -echo Installing with extras: !EXTRAS! -"%VENV_PY%" -m pip install ".[!EXTRAS!]" --group assets || exit /b 1 - -echo sampletones Python package installed successfully. -exit /b 0 diff --git a/scripts/windows/build/venv.bat b/scripts/windows/build/venv.bat deleted file mode 100644 index 85f4d6307..000000000 --- a/scripts/windows/build/venv.bat +++ /dev/null @@ -1,14 +0,0 @@ -@echo off -setlocal - -call "%~dp0..\lib\root.bat" || exit /b 1 - -if exist ".venv-build" ( - echo Virtual environment already exists. -) else ( - echo Creating virtual environment... - python -m venv .venv-build || exit /b - echo Virtual environment created. -) - -exit /b 0 diff --git a/scripts/windows/lib/root.bat b/scripts/windows/lib/root.bat deleted file mode 100644 index f9952477a..000000000 --- a/scripts/windows/lib/root.bat +++ /dev/null @@ -1,7 +0,0 @@ -if not exist "%~dp0..\..\..\src\sampletones\" ( - echo ERROR: SampleToNES project root not found.>&2 - exit /b 1 -) - -cd /d "%~dp0..\..\.." || exit /b 1 -exit /b 0 diff --git a/src/sampletones_application/config/deployment/deployment.py b/src/sampletones_application/config/deployment/deployment.py index a89c411f8..d537bc7b5 100644 --- a/src/sampletones_application/config/deployment/deployment.py +++ b/src/sampletones_application/config/deployment/deployment.py @@ -23,7 +23,7 @@ class DeploymentConfig(BaseModel, frozen=True): development values — verbose logging, and strict history so a missing transaction is reported the moment it happens rather than healed in silence — since the source tree is where an edit is written and where that report is worth having. A release build injects the - user-facing values through ``scripts/release_env_hook.py``, so a shipped artifact is quiet + user-facing values through ``scripts/runtime_hooks/release_environment.py``, so a shipped artifact is quiet and self-healing whatever the tree it was built from said. The ``SAMPLETONES_LOG_LEVEL`` and ``SAMPLETONES_STRICT_HISTORY`` environment variables set diff --git a/tests/suite/bootstrap.py b/tests/suite/bootstrap.py new file mode 100644 index 000000000..71b98c81a --- /dev/null +++ b/tests/suite/bootstrap.py @@ -0,0 +1,72 @@ +from dataclasses import dataclass +from pathlib import Path +from typing import Callable, Dict, List, Mapping, Optional, Sequence, Tuple + + +@dataclass(frozen=True) +class RecordedCommand: + """One command a recording runner was asked to run. + + Attributes: + command: The program and its arguments. + cwd: The directory it was to run in. + environment: The variables it was to see. + quiet: Whether its output was to be captured. + """ + + command: Tuple[str, ...] + cwd: Path + environment: Dict[str, str] + quiet: bool + + @property + def line(self) -> str: + """The command as one line.""" + return " ".join(self.command) + + +class RecordingRunner: + """A runner that records every command and answers with the status a test assigned it. + + A status is assigned by a fragment of the command line, so a probe such as ``import pyaudio`` + can be made to fail while everything else succeeds. A callback runs on every command, which + lets a test leave behind the files a real command would have written. + """ + + def __init__( + self, + statuses: Mapping[str, int], + on_run: Optional[Callable[[Sequence[str]], None]], + ) -> None: + self.commands: List[RecordedCommand] = [] + self._statuses = dict(statuses) + self._on_run = on_run + + def __call__( + self, + command: Sequence[str], + *, + cwd: Path, + environment: Mapping[str, str], + quiet: bool, + ) -> int: + recorded = RecordedCommand( + command=tuple(command), + cwd=cwd, + environment=dict(environment), + quiet=quiet, + ) + self.commands.append(recorded) + if self._on_run is not None: + self._on_run(command) + + for fragment, status in self._statuses.items(): + if fragment in recorded.line: + return status + + return 0 + + @property + def lines(self) -> List[str]: + """Every recorded command as one line, in the order run.""" + return [recorded.line for recorded in self.commands] diff --git a/tests/unit/scripts/bootstrap/test_interpreter.py b/tests/unit/scripts/bootstrap/test_interpreter.py new file mode 100644 index 000000000..d9b698abc --- /dev/null +++ b/tests/unit/scripts/bootstrap/test_interpreter.py @@ -0,0 +1,16 @@ +import pytest + +from bootstrap.interpreter import DOWNLOADS, require_python + + +class TestRequirePython: + def test_an_interpreter_at_least_the_version_passes(self, capsys: pytest.CaptureFixture[str]) -> None: + require_python((3, 8)) + + assert "Detected Python version" in capsys.readouterr().out + + def test_an_older_interpreter_is_refused_with_the_download_site(self) -> None: + with pytest.raises(SystemExit) as refused: + require_python((99, 0)) + + assert DOWNLOADS in str(refused.value) diff --git a/tests/unit/scripts/bootstrap/test_platforms.py b/tests/unit/scripts/bootstrap/test_platforms.py new file mode 100644 index 000000000..b7cb20846 --- /dev/null +++ b/tests/unit/scripts/bootstrap/test_platforms.py @@ -0,0 +1,115 @@ +from dataclasses import dataclass +from pathlib import Path + +import pytest + +from bootstrap.platforms.factory import platform_named +from bootstrap.platforms.linux import Linux +from bootstrap.platforms.macos import MacOS +from bootstrap.platforms.windows import Windows +from tests.suite.base import BaseTestSuite +from tests.suite.case import BaseRegularTestCase + + +class TestPlatformNamed(BaseTestSuite): + @dataclass(frozen=True, kw_only=True) + class TestCase(BaseRegularTestCase): + system: str + bundles: bool + + test_cases = ( + TestCase(label="Linux builds a bundle", system="Linux", bundles=True), + TestCase(label="Windows builds a bundle", system="Windows", bundles=True), + TestCase(label="macOS runs from source", system="Darwin", bundles=False), + ) + + @pytest.mark.parametrize("test_case", test_cases, ids=lambda test_case: test_case.label) + def test_the_system_name_selects_its_platform(self, test_case: TestCase) -> None: + platform = platform_named(test_case.system) + + assert platform.name == test_case.system + assert platform.bundles is test_case.bundles + + def test_an_unknown_system_is_refused_by_name(self) -> None: + with pytest.raises(SystemExit, match="Plan 9"): + platform_named("Plan 9") + + +class TestLaunchers(BaseTestSuite): + @dataclass(frozen=True, kw_only=True) + class TestCase(BaseRegularTestCase): + system: str + release: bool + expected: str + + test_cases = ( + TestCase(label="a Linux development bundle is one file", system="Linux", release=False, expected="sampletones"), + TestCase( + label="a Linux release is a directory beside its launcher", + system="Linux", + release=True, + expected="sampletones/sampletones", + ), + TestCase( + label="a Windows development bundle carries an extension", + system="Windows", + release=False, + expected="sampletones.exe", + ), + TestCase( + label="a Windows release is a directory beside its launcher", + system="Windows", + release=True, + expected="sampletones/sampletones.exe", + ), + ) + + @pytest.mark.parametrize("test_case", test_cases, ids=lambda test_case: test_case.label) + def test_the_launcher_lies_where_pyinstaller_writes_it(self, test_case: TestCase) -> None: + launcher = platform_named(test_case.system).launcher(Path("bin"), release=test_case.release) + + assert launcher == Path("bin") / test_case.expected + + +class TestInterpreters: + def test_a_posix_environment_runs_bin_python(self) -> None: + assert Linux().interpreter(Path(".venv-build")) == Path(".venv-build/bin/python") + + def test_a_windows_environment_runs_scripts_python(self) -> None: + assert Windows().interpreter(Path(".venv-build")) == Path(".venv-build/Scripts/python.exe") + + +class TestSystemPackages: + def test_linux_updates_apt_before_installing(self) -> None: + commands = Linux().system_packages() + + assert [command[:2] for command in commands] == [("sudo", "apt-get"), ("sudo", "apt-get")] + assert "portaudio19-dev" in commands[1] + assert "python3-tk" in commands[1] + + def test_macos_installs_portaudio_through_homebrew(self) -> None: + assert MacOS().system_packages() == (("brew", "install", "portaudio"),) + assert MacOS().missing_package_manager() is not None + + def test_windows_installs_nothing(self) -> None: + assert Windows().system_packages() == () + assert Windows().missing_package_manager() is None + + +class TestBuildEnvironment: + def test_macos_exports_the_portaudio_flags_on_the_native_architecture(self) -> None: + lines = MacOS().build_environment(machine="arm64", portaudio_prefix="/opt/homebrew/opt/portaudio") + + assert lines == ( + "CFLAGS=-I/opt/homebrew/opt/portaudio/include", + "LDFLAGS=-L/opt/homebrew/opt/portaudio/lib", + "ARCHFLAGS=-arch arm64", + ) + + def test_macos_without_homebrew_is_refused(self) -> None: + with pytest.raises(SystemExit, match="Homebrew"): + MacOS().build_environment(machine="arm64", portaudio_prefix="") + + def test_other_systems_export_nothing(self) -> None: + assert Linux().build_environment(machine="x86_64", portaudio_prefix="") == () + assert Windows().build_environment(machine="AMD64", portaudio_prefix="") == () diff --git a/tests/unit/scripts/bootstrap/test_preflight.py b/tests/unit/scripts/bootstrap/test_preflight.py new file mode 100644 index 000000000..43a4149f8 --- /dev/null +++ b/tests/unit/scripts/bootstrap/test_preflight.py @@ -0,0 +1,98 @@ +from pathlib import Path + +import pytest + +from bootstrap.platforms.linux import Linux +from bootstrap.preflight import can_import, check_build_interpreter +from tests.suite.bootstrap import RecordingRunner + + +@pytest.fixture +def python(tmp_path: Path) -> Path: + interpreter = tmp_path / "python" + interpreter.write_text("") + return interpreter + + +class TestCanImport: + def test_the_probe_is_quiet_and_answers_the_status(self, python: Path, tmp_path: Path) -> None: + runner = RecordingRunner({"import tkinter": 1}, None) + + assert can_import(python, "pyaudio", runner=runner, cwd=tmp_path, environment={}) + assert not can_import(python, "tkinter", runner=runner, cwd=tmp_path, environment={}) + assert all(recorded.quiet for recorded in runner.commands) + + +class TestCheckBuildInterpreter: + def test_a_missing_interpreter_is_refused(self, tmp_path: Path) -> None: + with pytest.raises(SystemExit, match="build interpreter not found"): + check_build_interpreter( + tmp_path / "absent", + Linux(), + release=False, + runner=RecordingRunner({}, None), + cwd=tmp_path, + environment={}, + ) + + def test_an_interpreter_without_audio_playback_is_refused_with_the_platform_s_advice( + self, + python: Path, + tmp_path: Path, + ) -> None: + with pytest.raises(SystemExit, match="make system-deps"): + check_build_interpreter( + python, + Linux(), + release=False, + runner=RecordingRunner({"import pyaudio": 1}, None), + cwd=tmp_path, + environment={}, + ) + + def test_a_release_without_tk_is_refused(self, python: Path, tmp_path: Path) -> None: + with pytest.raises(SystemExit, match="tkinter"): + check_build_interpreter( + python, + Linux(), + release=True, + runner=RecordingRunner({"import tkinter": 1}, None), + cwd=tmp_path, + environment={}, + ) + + def test_a_development_bundle_without_tk_is_warned( + self, + python: Path, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + ) -> None: + check_build_interpreter( + python, + Linux(), + release=False, + runner=RecordingRunner({"import tkinter": 1}, None), + cwd=tmp_path, + environment={}, + ) + + assert "WARNING" in capsys.readouterr().out + + def test_an_interpreter_carrying_both_passes( + self, + python: Path, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + ) -> None: + check_build_interpreter( + python, + Linux(), + release=True, + runner=RecordingRunner({}, None), + cwd=tmp_path, + environment={}, + ) + + output = capsys.readouterr().out + assert "pyaudio: available" in output + assert "tkinter: available" in output diff --git a/tests/unit/scripts/bootstrap/test_processes.py b/tests/unit/scripts/bootstrap/test_processes.py new file mode 100644 index 000000000..f55a02ae5 --- /dev/null +++ b/tests/unit/scripts/bootstrap/test_processes.py @@ -0,0 +1,36 @@ +import sys +from pathlib import Path + +import pytest + +from bootstrap.processes import expect_success, run +from tests.suite.bootstrap import RecordingRunner + + +class TestRun: + def test_the_status_is_the_command_s_own(self, tmp_path: Path) -> None: + status = run( + (sys.executable, "-c", "raise SystemExit(3)"), + cwd=tmp_path, + environment={}, + quiet=True, + ) + + assert status == 3 + + +class TestExpectSuccess: + def test_a_succeeding_command_passes(self, tmp_path: Path) -> None: + runner = RecordingRunner({}, None) + + expect_success(runner, ("echo", "hello"), cwd=tmp_path, environment={"KEY": "value"}) + + assert runner.lines == ["echo hello"] + assert runner.commands[0].environment == {"KEY": "value"} + assert not runner.commands[0].quiet + + def test_a_failing_command_stops_the_script_naming_it(self, tmp_path: Path) -> None: + runner = RecordingRunner({"echo": 2}, None) + + with pytest.raises(SystemExit, match="echo hello exited with status 2"): + expect_success(runner, ("echo", "hello"), cwd=tmp_path, environment={}) diff --git a/tests/unit/scripts/bootstrap/test_repository.py b/tests/unit/scripts/bootstrap/test_repository.py new file mode 100644 index 000000000..24706cc59 --- /dev/null +++ b/tests/unit/scripts/bootstrap/test_repository.py @@ -0,0 +1,9 @@ +from bootstrap.repository import repository_root + + +class TestRepositoryRoot: + def test_the_root_holds_the_project_and_the_entry_package(self) -> None: + root = repository_root() + + assert (root / "pyproject.toml").is_file() + assert (root / "src" / "sampletones" / "__main__.py").is_file() diff --git a/tests/unit/scripts/bootstrap/test_venv_build.py b/tests/unit/scripts/bootstrap/test_venv_build.py new file mode 100644 index 000000000..f9b0bfe86 --- /dev/null +++ b/tests/unit/scripts/bootstrap/test_venv_build.py @@ -0,0 +1,56 @@ +import sys +from pathlib import Path + +from bootstrap.platforms.linux import Linux +from bootstrap.venv_build import BUILD_ENVIRONMENT, build_environment, install, interpreter +from tests.suite.bootstrap import RecordingRunner + + +class TestBuildEnvironment: + def test_a_missing_environment_is_created_by_the_running_interpreter(self, tmp_path: Path) -> None: + runner = RecordingRunner({}, None) + + directory = build_environment(tmp_path, runner=runner, environment={}) + + assert directory == tmp_path / BUILD_ENVIRONMENT + assert runner.lines == [f"{sys.executable} -m venv {directory}"] + + def test_an_existing_environment_is_kept(self, tmp_path: Path) -> None: + (tmp_path / BUILD_ENVIRONMENT).mkdir() + runner = RecordingRunner({}, None) + + build_environment(tmp_path, runner=runner, environment={}) + + assert runner.lines == [] + + +class TestInstall: + def test_pip_is_upgraded_then_the_package_installed_with_its_extras_and_groups(self, tmp_path: Path) -> None: + runner = RecordingRunner({}, None) + python = tmp_path / "python" + + install( + tmp_path, + python, + extras=("build", "gpu"), + groups=("assets",), + runner=runner, + environment={"PATH": "/usr/bin"}, + ) + + assert runner.lines == [ + f"{python} -m pip install --upgrade pip", + f"{python} -m pip install .[build,gpu] --group assets", + ] + + def test_every_install_refuses_an_interpreter_outside_a_virtual_environment(self, tmp_path: Path) -> None: + runner = RecordingRunner({}, None) + + install(tmp_path, tmp_path / "python", extras=("build",), groups=(), runner=runner, environment={}) + + assert all(recorded.environment["PIP_REQUIRE_VIRTUALENV"] == "1" for recorded in runner.commands) + + +class TestInterpreter: + def test_the_interpreter_lies_in_the_build_environment(self, tmp_path: Path) -> None: + assert interpreter(tmp_path, Linux()) == tmp_path / BUILD_ENVIRONMENT / "bin" / "python" diff --git a/tests/unit/scripts/test_build_environment.py b/tests/unit/scripts/test_build_environment.py new file mode 100644 index 000000000..dd047fd2f --- /dev/null +++ b/tests/unit/scripts/test_build_environment.py @@ -0,0 +1,35 @@ +import pytest + +from bootstrap.platforms.linux import Linux +from bootstrap.platforms.macos import MacOS +from tests.suite.scripts import load_script + +build_environment = load_script("build_environment.py") + + +class TestMain: + def test_macos_prints_the_flags_one_per_line( + self, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], + ) -> None: + monkeypatch.setattr(build_environment, "current_platform", MacOS) + monkeypatch.setattr(build_environment, "portaudio_prefix", lambda: "/opt/homebrew/opt/portaudio") + monkeypatch.setattr(build_environment.running, "machine", lambda: "arm64") + + assert build_environment.main([]) == 0 + assert capsys.readouterr().out.splitlines() == [ + "CFLAGS=-I/opt/homebrew/opt/portaudio/include", + "LDFLAGS=-L/opt/homebrew/opt/portaudio/lib", + "ARCHFLAGS=-arch arm64", + ] + + def test_linux_prints_nothing( + self, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], + ) -> None: + monkeypatch.setattr(build_environment, "current_platform", Linux) + + assert build_environment.main([]) == 0 + assert capsys.readouterr().out == "" diff --git a/tests/unit/scripts/test_bundle.py b/tests/unit/scripts/test_bundle.py new file mode 100644 index 000000000..2811d5fa3 --- /dev/null +++ b/tests/unit/scripts/test_bundle.py @@ -0,0 +1,148 @@ +from pathlib import Path +from typing import Sequence + +import pytest + +from bootstrap.platforms.linux import Linux +from bootstrap.platforms.macos import MacOS +from tests.suite.bootstrap import RecordingRunner +from tests.suite.scripts import load_script + +bundle = load_script("bundle.py") + + +class TestPyInstallerCommand: + def test_a_release_is_a_directory_with_the_runtime_hook(self) -> None: + options = bundle.BundleOptions(release=True, gpu=False) + + command = bundle.pyinstaller_command(Path("python"), Linux(), options) + + assert command[:3] == ["python", "-m", "PyInstaller"] + assert "--onedir" in command + assert "--runtime-hook" in command + assert command[command.index("--runtime-hook") + 1] == bundle.RELEASE_HOOK + assert command[-1] == bundle.ENTRY + + def test_a_development_bundle_is_one_file_without_the_hook(self) -> None: + options = bundle.BundleOptions(release=False, gpu=False) + + command = bundle.pyinstaller_command(Path("python"), Linux(), options) + + assert "--onefile" in command + assert "--runtime-hook" not in command + + def test_the_data_and_the_exclusions_ride_along(self) -> None: + command = bundle.pyinstaller_command(Path("python"), Linux(), bundle.BundleOptions(release=False, gpu=False)) + + data = [command[index + 1] for index, flag in enumerate(command) if flag == "--add-data"] + assert data == [f"{source}:{destination}" for source, destination in bundle.DATA] + assert command[command.index("--exclude-module") + 1] == "PIL" + assert command[command.index("--icon") + 1] == Linux().icon + + +class TestExtras: + def test_the_build_extra_is_always_installed_and_gpu_on_request(self) -> None: + assert bundle.extras(bundle.BundleOptions(release=False, gpu=False)) == ("build",) + assert bundle.extras(bundle.BundleOptions(release=False, gpu=True)) == ("build", "gpu") + + +class TestRemovePrevious: + def test_a_previous_file_and_directory_are_removed(self, tmp_path: Path) -> None: + (tmp_path / "sampletones").mkdir() + (tmp_path / "sampletones.exe").write_text("") + + bundle.remove_previous(tmp_path) + + assert list(tmp_path.iterdir()) == [] + + +def _repository(tmp_path: Path) -> Path: + for notice in bundle.NOTICES: + (tmp_path / notice).write_text(notice) + + return tmp_path + + +class TestBuildBundle: + def test_a_release_runs_every_step_in_order_and_places_the_notices(self, tmp_path: Path) -> None: + root = _repository(tmp_path) + platform = Linux() + options = bundle.BundleOptions(release=True, gpu=False) + launcher = platform.launcher(root / bundle.DISTRIBUTION, release=True) + + def leave_behind(command: Sequence[str]) -> None: + if "venv" in command: + platform.interpreter(root / ".venv-build").parent.mkdir(parents=True) + platform.interpreter(root / ".venv-build").write_text("") + + if "PyInstaller" in command: + launcher.parent.mkdir(parents=True) + launcher.write_text("") + + runner = RecordingRunner({}, leave_behind) + + built = bundle.build_bundle(root, platform, options, runner=runner, environment={}) + + assert built == launcher + assert runner.lines[0].endswith(".venv-build") + assert "pip install --upgrade pip" in runner.lines[1] + assert ".[build]" in runner.lines[2] + assert "import pyaudio" in runner.lines[3] + assert "import tkinter" in runner.lines[4] + assert runner.lines[5].endswith(bundle.ICONS_SCRIPT) + assert "PyInstaller" in runner.lines[6] + assert runner.lines[7] == f"{launcher} --self-check" + assert all((launcher.parent / notice).read_text() == notice for notice in bundle.NOTICES) + + def test_a_bundle_pyinstaller_never_wrote_is_reported(self, tmp_path: Path) -> None: + root = _repository(tmp_path) + platform = Linux() + + def leave_behind(command: Sequence[str]) -> None: + if "venv" in command: + platform.interpreter(root / ".venv-build").parent.mkdir(parents=True) + platform.interpreter(root / ".venv-build").write_text("") + + with pytest.raises(SystemExit, match="produced no executable"): + bundle.build_bundle( + root, + platform, + bundle.BundleOptions(release=False, gpu=False), + runner=RecordingRunner({}, leave_behind), + environment={}, + ) + + def test_a_launcher_failing_its_self_check_fails_the_build(self, tmp_path: Path) -> None: + root = _repository(tmp_path) + platform = Linux() + launcher = platform.launcher(root / bundle.DISTRIBUTION, release=False) + + def leave_behind(command: Sequence[str]) -> None: + if "venv" in command: + platform.interpreter(root / ".venv-build").parent.mkdir(parents=True) + platform.interpreter(root / ".venv-build").write_text("") + + if "PyInstaller" in command: + launcher.parent.mkdir(parents=True, exist_ok=True) + launcher.write_text("") + + with pytest.raises(SystemExit, match="self-check"): + bundle.build_bundle( + root, + platform, + bundle.BundleOptions(release=False, gpu=False), + runner=RecordingRunner({"--self-check": 1}, leave_behind), + environment={}, + ) + + +class TestMain: + def test_a_system_without_bundles_is_told_to_run_from_source( + self, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], + ) -> None: + monkeypatch.setattr(bundle, "current_platform", MacOS) + + assert bundle.main([]) == 1 + assert "make setup" in capsys.readouterr().err diff --git a/tests/unit/scripts/test_clean.py b/tests/unit/scripts/test_clean.py new file mode 100644 index 000000000..5d65aa15e --- /dev/null +++ b/tests/unit/scripts/test_clean.py @@ -0,0 +1,46 @@ +from pathlib import Path + +from tests.suite.scripts import load_script + +clean = load_script("clean.py") + + +def _tree(root: Path) -> None: + for directory in ( + "bin", + "build", + "dist", + "htmlcov", + "src/__pycache__", + "src/sampletones.egg-info", + ".venv/__pycache__", + ): + (root / directory).mkdir(parents=True) + + for file in (".coverage", "sampletones.spec", "src/module.pyc", "src/module.py", ".venv/cached.pyc"): + (root / file).write_text("") + + +class TestRemoveArtifacts: + def test_the_build_outputs_and_reports_go(self, tmp_path: Path) -> None: + _tree(tmp_path) + + clean.remove_artifacts(tmp_path) + + assert not any((tmp_path / name).exists() for name in ("bin", "build", "dist", "htmlcov", ".coverage")) + assert not (tmp_path / "sampletones.spec").exists() + assert (tmp_path / "src" / "module.py").exists() + + +class TestRemoveCaches: + def test_the_caches_go_and_the_environments_stay(self, tmp_path: Path) -> None: + _tree(tmp_path) + + clean.remove_caches(tmp_path) + + assert not (tmp_path / "src" / "__pycache__").exists() + assert not (tmp_path / "src" / "sampletones.egg-info").exists() + assert not (tmp_path / "src" / "module.pyc").exists() + assert (tmp_path / "src" / "module.py").exists() + assert (tmp_path / ".venv" / "__pycache__").exists() + assert (tmp_path / ".venv" / "cached.pyc").exists() diff --git a/tests/unit/scripts/test_setup_environment.py b/tests/unit/scripts/test_setup_environment.py new file mode 100644 index 000000000..ae6e7570a --- /dev/null +++ b/tests/unit/scripts/test_setup_environment.py @@ -0,0 +1,45 @@ +from tests.suite.scripts import load_script + +setup_environment = load_script("setup_environment.py") + + +class TestGpuExtra: + def test_zero_keeps_the_cpu_backend(self) -> None: + assert setup_environment.gpu_extra("0", system="Linux") is None + + def test_a_named_extra_is_taken_as_given(self) -> None: + assert setup_environment.gpu_extra("gpu-cuda11", system="Linux") == "gpu-cuda11" + + def test_auto_on_macos_keeps_the_cpu_backend(self) -> None: + assert setup_environment.gpu_extra("auto", system="Darwin") is None + + +class TestSetupCommands: + def test_the_cpu_backend_synchronizes_and_installs_the_bare_package(self) -> None: + commands = setup_environment.setup_commands(None) + + assert commands[0] == ["uv", "sync", "--group", "dev"] + assert commands[1][-1].endswith("icons.py") + assert commands[2] == ["uv", "tool", "install", "--force", "."] + + def test_a_gpu_extra_reaches_both_installs(self) -> None: + commands = setup_environment.setup_commands("gpu") + + assert commands[0] == ["uv", "sync", "--group", "dev", "--extra", "gpu"] + assert commands[2] == ["uv", "tool", "install", "--force", ".[gpu]"] + + +class TestSetupEnvironmentVariables: + def test_macos_pins_the_native_architecture(self) -> None: + variables = setup_environment.setup_environment_variables( + {"PATH": "/usr/bin"}, system="Darwin", machine="arm64" + ) + + assert variables == {"PATH": "/usr/bin", "ARCHFLAGS": "-arch arm64"} + + def test_other_systems_pass_the_variables_through(self) -> None: + variables = setup_environment.setup_environment_variables( + {"PATH": "/usr/bin"}, system="Linux", machine="x86_64" + ) + + assert variables == {"PATH": "/usr/bin"} diff --git a/tests/unit/scripts/test_system_dependencies.py b/tests/unit/scripts/test_system_dependencies.py new file mode 100644 index 000000000..bb5f15c34 --- /dev/null +++ b/tests/unit/scripts/test_system_dependencies.py @@ -0,0 +1,40 @@ +import pytest + +from bootstrap.platforms.linux import Linux +from bootstrap.platforms.macos import MacOS +from bootstrap.platforms.windows import Windows +from tests.suite.bootstrap import RecordingRunner +from tests.suite.scripts import load_script + +system_dependencies = load_script("system_dependencies.py") + + +class TestMain: + def test_windows_has_nothing_to_install( + self, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], + ) -> None: + monkeypatch.setattr(system_dependencies, "current_platform", Windows) + + assert system_dependencies.main([]) == 0 + assert "Nothing to install" in capsys.readouterr().out + + def test_linux_runs_the_apt_commands_in_order(self, monkeypatch: pytest.MonkeyPatch) -> None: + runner = RecordingRunner({}, None) + monkeypatch.setattr(system_dependencies, "current_platform", Linux) + monkeypatch.setattr(system_dependencies, "run", runner) + + assert system_dependencies.main([]) == 0 + assert runner.lines[0] == "sudo apt-get update" + assert runner.lines[1].startswith("sudo apt-get install -y") + + def test_macos_without_homebrew_is_refused( + self, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], + ) -> None: + monkeypatch.setattr(system_dependencies, "current_platform", MacOS) + + assert system_dependencies.main([]) == 1 + assert "Homebrew" in capsys.readouterr().err From 5b906946a0ad3b2907ea4eab89bf093ed87dbe43 Mon Sep 17 00:00:00 2001 From: JakimPL Date: Sun, 13 Sep 2026 14:49:14 +0200 Subject: [PATCH 07/36] Rewrote: the development scripts in Python --- .github/workflows/ci.yml | 11 +-- Makefile | 55 ++++--------- docs/development/tooling.md | 19 +++-- pyproject.toml | 1 + scripts/bootstrap/passes.py | 51 ++++++++++++ scripts/bootstrap/processes.py | 6 +- scripts/formatting.py | 56 ++++++++++++++ scripts/hooks.py | 33 ++++++++ scripts/lint.py | 55 +++++++++++++ scripts/linux/dev/format.sh | 11 --- scripts/linux/dev/lint.sh | 19 ----- scripts/linux/dev/mypy.sh | 11 --- scripts/linux/dev/pre_commit.sh | 8 -- scripts/linux/dev/pylint.sh | 11 --- scripts/linux/dev/tests.sh | 23 ------ scripts/run_tests.py | 77 +++++++++++++++++++ scripts/windows/dev/format.bat | 11 --- scripts/windows/dev/lint.bat | 23 ------ scripts/windows/dev/mypy.bat | 10 --- scripts/windows/dev/pre_commit.bat | 9 --- scripts/windows/dev/pylint.bat | 10 --- scripts/windows/dev/tests.bat | 32 -------- tests/unit/scripts/bootstrap/test_passes.py | 30 ++++++++ .../unit/scripts/bootstrap/test_processes.py | 16 ++++ tests/unit/scripts/test_formatting.py | 39 ++++++++++ tests/unit/scripts/test_hooks.py | 20 +++++ tests/unit/scripts/test_lint.py | 55 +++++++++++++ tests/unit/scripts/test_run_tests.py | 53 +++++++++++++ 28 files changed, 524 insertions(+), 231 deletions(-) create mode 100644 scripts/bootstrap/passes.py create mode 100644 scripts/formatting.py create mode 100644 scripts/hooks.py create mode 100644 scripts/lint.py delete mode 100755 scripts/linux/dev/format.sh delete mode 100755 scripts/linux/dev/lint.sh delete mode 100755 scripts/linux/dev/mypy.sh delete mode 100755 scripts/linux/dev/pre_commit.sh delete mode 100755 scripts/linux/dev/pylint.sh delete mode 100755 scripts/linux/dev/tests.sh create mode 100644 scripts/run_tests.py delete mode 100644 scripts/windows/dev/format.bat delete mode 100644 scripts/windows/dev/lint.bat delete mode 100644 scripts/windows/dev/mypy.bat delete mode 100644 scripts/windows/dev/pre_commit.bat delete mode 100644 scripts/windows/dev/pylint.bat delete mode 100644 scripts/windows/dev/tests.bat create mode 100644 tests/unit/scripts/bootstrap/test_passes.py create mode 100644 tests/unit/scripts/test_formatting.py create mode 100644 tests/unit/scripts/test_hooks.py create mode 100644 tests/unit/scripts/test_lint.py create mode 100644 tests/unit/scripts/test_run_tests.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2d44cc60b..d8fd5eaf5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -74,11 +74,6 @@ jobs: - name: Install the development environment run: uv sync --group dev - - name: Run doctests - run: uv run python -m pytest src/ --doctest-modules --no-cov - - - name: Run the unit and integration suites with coverage - run: uv run python -m pytest -n auto --cov --ignore=tests/benchmarks - - - name: Run the benchmarks - run: uv run python -m pytest tests/benchmarks --no-cov + - name: Run the doctests, the covered suite and the benchmarks + shell: bash + run: ${{ runner.os == 'Windows' && 'python' || 'python3' }} scripts/run_tests.py --workers auto diff --git a/Makefile b/Makefile index f7de00ac2..170f25dec 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,7 @@ -.PHONY: help setup install build release system-deps run clean pre-commit test benchmarks \ - ftm-samples nsf-samples nsf-render compression-report compression-study icons player check-import-boundary check-tag-names check-unused-tags check-rendered-literals \ - check-language-keys check-palette-colors check-shortcut-actions calibration lint pylint mypy format +.PHONY: help setup install system-deps build release run clean pre-commit test benchmarks lint format \ + ftm-samples nsf-samples nsf-render compression-report compression-study icons player calibration \ + check-import-boundary check-tag-names check-unused-tags check-rendered-literals check-language-keys \ + check-palette-colors check-shortcut-actions ifeq ($(OS),Windows_NT) ifeq ($(MSYSTEM),) @@ -13,27 +14,11 @@ UNAME_S := $(shell uname -s) endif ifeq ($(UNAME_S),Windows) - SCRIPTS_DIR := scripts/windows - SCRIPT_EXT := .bat - RUN_SCRIPT := PYTHON := python + Q := else - SCRIPTS_DIR := scripts/linux - SCRIPT_EXT := .sh - RUN_SCRIPT := bash PYTHON := python3 -endif - -ifeq ($(UNAME_S),Windows) -script = $(subst /,\,$(SCRIPTS_DIR)/$(1)$(SCRIPT_EXT)) -else -script = $(RUN_SCRIPT) $(SCRIPTS_DIR)/$(1)$(SCRIPT_EXT) -endif - -ifeq ($(UNAME_S),Windows) -Q := -else -Q := " + Q := " endif GPU ?= auto @@ -45,7 +30,7 @@ help: @echo $(Q) make system-deps - Install system packages required to build and run (apt on Debian-based Linux, Homebrew on macOS)$(Q) @echo $(Q) make build - Compile standalone executable (development deployment config: DEBUG, strict history)$(Q) @echo $(Q) make release - Compile standalone executable with the release deployment config (INFO, self-healing history)$(Q) - @echo $(Q) make test - Run unit tests with coverage$(Q) + @echo $(Q) make test - Run the doctests, the covered suite and the benchmarks$(Q) @echo $(Q) make benchmarks - Run the measured-duration suite on its own$(Q) @echo $(Q) make ftm-samples - Emit example .ftm files to build/ftm via the integration suite$(Q) @echo $(Q) make nsf-samples - Emit example .nsf files to build/nsf via the integration suite$(Q) @@ -56,7 +41,7 @@ help: @echo $(Q) make player - Assemble the NES player driver with cc65$(Q) @echo $(Q) make calibration - Score the reconstruction corpus; the report lands in Documents/SampleToNES/calibration$(Q) @echo $(Q) make clean - Remove build artifacts and cache files$(Q) - @echo $(Q) make lint - Run linting (pylint, mypy)$(Q) + @echo $(Q) make lint - Run mypy and pylint (ARGS=--mypy or ARGS=--pylint for one of them)$(Q) @echo $(Q) make format - Auto-format code (isort, black)$(Q) @echo $(Q) make run - Run SampleToNES application$(Q) @@ -83,13 +68,19 @@ clean: $(PYTHON) scripts/clean.py pre-commit: - $(call script,dev/pre_commit) + $(PYTHON) scripts/hooks.py test: - $(call script,dev/tests) + $(PYTHON) scripts/run_tests.py benchmarks: - uv run python -m pytest tests/benchmarks --no-cov -s + $(PYTHON) scripts/run_tests.py --only benchmarks + +lint: + $(PYTHON) scripts/lint.py $(ARGS) + +format: + $(PYTHON) scripts/formatting.py ftm-samples: export SAMPLETONES_FTM_OUTPUT_DIR := build/ftm ftm-samples: @@ -138,15 +129,3 @@ check-shortcut-actions: calibration: uv run scripts/calibration.py - -lint: - $(call script,dev/lint) - -pylint: - $(call script,dev/pylint) - -mypy: - $(call script,dev/mypy) - -format: - $(call script,dev/format) diff --git a/docs/development/tooling.md b/docs/development/tooling.md index 88d018261..1050a66bd 100644 --- a/docs/development/tooling.md +++ b/docs/development/tooling.md @@ -10,10 +10,10 @@ before adding a script or a make target. Which packages may import which is **1. Two interpreters, two kinds of script.** A *bootstrap script* runs on the system interpreter, before or beside the project environment: it creates the environment, installs system packages, -builds the standalone bundle, cleans the tree. It imports the standard library and the other -bootstrap modules, nothing else, so it runs on a machine that has Python and nothing more. A -*tool script* runs inside the project environment, through `uv run`, and imports the project's -packages freely. +builds the standalone bundle, cleans the tree, and runs the tests, the linters and the formatters +the environment provides. It imports the standard library and the other bootstrap modules, +nothing else, so it runs on a machine that has Python and nothing more. A *tool script* runs +inside the project environment, through `uv run`, and imports the project's packages freely. **2. A bootstrap script installs nothing into the interpreter it runs on.** Every package a build installs lands in `.venv-build`, a virtual environment of its own, and pip is told to refuse any @@ -45,14 +45,19 @@ that does the work and passes its flag. The two shell files at the root, `instal | `system_dependencies.py` | `make system-deps` | Installs the system packages: apt on Debian-based Linux, Homebrew on macOS, nothing on Windows | | `build_environment.py` | CI | Prints the compiler flags a macOS build exports, one `KEY=VALUE` per line | | `clean.py` | `make clean` | Removes the build outputs, the coverage reports and the bytecode caches | +| `run_tests.py` | `make test`, `make benchmarks` | Runs the doctests, the covered suite across six workers, and the benchmarks, every pass whatever the earlier ones reported; `--only` picks one pass and `--workers` sets the count | +| `lint.py` | `make lint` | Runs mypy over the files `pyproject.toml` configures and pylint over `src/` and `scripts/`; `--mypy` or `--pylint` picks one, and named paths narrow both | +| `formatting.py` | `make format` | Runs isort, then black, over `src/`, `tests/` and `scripts/`, or over the paths named | +| `hooks.py` | `make pre-commit` | Installs the git hooks pre-commit runs at commit and at push | | `detect_cuda.py` | via `setup_environment.py` | Maps the driver's CUDA version to the CuPy extra | | `runtime_hooks/release_environment.py` | build input | The PyInstaller runtime hook that gives a release bundle its deployment defaults | | `ci/` | the release workflow | The gates a release passes: the tag matches the version, the bundle ships its notices and starts | `scripts/bootstrap/` holds what they share: the repository root (`repository.py`), the interpreter version check (`interpreter.py`), running a command and holding it to success -(`processes.py`), the build environment and the installs into it (`venv_build.py`), the -preflight of the build interpreter (`preflight.py`), and the platforms (`platforms/`). +(`processes.py`), a run of named passes that reports every failure at once (`passes.py`), the +build environment and the installs into it (`venv_build.py`), the preflight of the build +interpreter (`preflight.py`), and the platforms (`platforms/`). ## The tool scripts @@ -69,5 +74,7 @@ the make target that names each. The checks are also pre-commit hooks; | Where a build installs | `scripts/bootstrap/venv_build.py` | | What a bundle has to carry before it is built | `scripts/bootstrap/preflight.py` | | The PyInstaller invocation | `scripts/bundle.py` | +| The passes `make test` runs, and their order | `scripts/run_tests.py` | +| What `make lint` and `make format` sweep | `scripts/lint.py`, `scripts/formatting.py` | | The GPU extra a machine gets | `scripts/detect_cuda.py` | | What a release bundle is held to | `scripts/ci/checks/bundle.py` | diff --git a/pyproject.toml b/pyproject.toml index efe629583..52ebb90ba 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -164,6 +164,7 @@ files = [ "src/sampletones_player", "src/sampletones_shared", "src/sampletones_synthesis", + "scripts", ] exclude = ["tests"] disallow_subclassing_any = true diff --git a/scripts/bootstrap/passes.py b/scripts/bootstrap/passes.py new file mode 100644 index 000000000..4fe91b1e5 --- /dev/null +++ b/scripts/bootstrap/passes.py @@ -0,0 +1,51 @@ +from dataclasses import dataclass +from pathlib import Path +from typing import List, Mapping, Sequence, Tuple + +from bootstrap.processes import Runner + + +@dataclass(frozen=True) +class Pass: + """One step of a run that reports every failure at once: named, announced, run as one command. + + Attributes: + name: What the step is called in the report and on the command line. + announcement: The line printed as the step starts. + command: The program and its arguments, run from the repository root. + """ + + name: str + announcement: str + command: Tuple[str, ...] + + +def run_passes( + passes: Sequence[Pass], + *, + root: Path, + runner: Runner, + environment: Mapping[str, str], +) -> List[str]: + """Runs every pass in order and names the ones that failed. + + Every pass runs whatever the earlier ones reported, so one run shows everything that is + wrong. + + Args: + passes: The steps, in order. + root: The repository, which every command runs in. + runner: What runs the commands. + environment: The variables the commands see. + + Returns: + List[str]: The names of the passes that exited with a failure, in order. + """ + failed: List[str] = [] + for current in passes: + print(current.announcement) + status = runner(current.command, cwd=root, environment=environment, quiet=False) + if status != 0: + failed.append(current.name) + + return failed diff --git a/scripts/bootstrap/processes.py b/scripts/bootstrap/processes.py index 2364d3c97..6d40f468a 100644 --- a/scripts/bootstrap/processes.py +++ b/scripts/bootstrap/processes.py @@ -1,5 +1,6 @@ import shlex import subprocess +import sys from pathlib import Path from typing import Mapping, Protocol, Sequence @@ -26,7 +27,9 @@ def run( ) -> int: """Runs a command in ``cwd`` under ``environment`` and answers with its exit status. - A quiet run keeps the command's output to itself, which is what a probe asks for. + A quiet run keeps the command's output to itself, which is what a probe asks for. Whatever + the script printed reaches the terminal before the command's own output, so a log read + through a pipe keeps the announcements ahead of what they announce. Args: command: The program and its arguments. @@ -37,6 +40,7 @@ def run( Returns: int: The command's exit status. """ + sys.stdout.flush() completed = subprocess.run( list(command), cwd=str(cwd), diff --git a/scripts/formatting.py b/scripts/formatting.py new file mode 100644 index 000000000..5129c5cb7 --- /dev/null +++ b/scripts/formatting.py @@ -0,0 +1,56 @@ +import argparse +import os +import sys +from pathlib import Path +from typing import Final, Mapping, Sequence, Tuple + +from bootstrap.processes import Runner, expect_success, run +from bootstrap.repository import repository_root + +FORMATTED_TREES: Final[Tuple[str, ...]] = ("src", "tests", "scripts") +ISORT: Final[Tuple[str, ...]] = ("uv", "run", "python", "-m", "isort") +BLACK: Final[Tuple[str, ...]] = ("uv", "run", "python", "-m", "black") + + +def format_code( + root: Path, + paths: Sequence[str], + *, + runner: Runner, + environment: Mapping[str, str], +) -> None: + """Sorts the imports and formats the code under ``paths``, isort first so black settles the result. + + Args: + root: The repository, which the formatters run in. + paths: The files or directories formatted. + runner: What runs the formatters. + environment: The variables the formatters see. + + Raises: + SystemExit: If a formatter fails, before the next one runs. + """ + print("Formatting imports with isort...") + expect_success(runner, (*ISORT, *paths), cwd=root, environment=environment) + print("Formatting code with black...") + expect_success(runner, (*BLACK, *paths), cwd=root, environment=environment) + + +def main(argv: Sequence[str]) -> int: + """Formats the source, test and script trees, or the paths named.""" + parser = argparse.ArgumentParser(description="Format the SampleToNES code with isort and black.") + parser.add_argument("paths", nargs="*", help="files or directories to format in place of the whole trees") + arguments = parser.parse_args(list(argv)) + + format_code( + repository_root(), + tuple(arguments.paths) or FORMATTED_TREES, + runner=run, + environment=os.environ, + ) + print("Code formatting complete.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) diff --git a/scripts/hooks.py b/scripts/hooks.py new file mode 100644 index 000000000..614e11d38 --- /dev/null +++ b/scripts/hooks.py @@ -0,0 +1,33 @@ +import argparse +import os +import sys +from typing import Final, Sequence, Tuple + +from bootstrap.processes import expect_success, run +from bootstrap.repository import repository_root + +INSTALL_HOOKS: Final[Tuple[str, ...]] = ( + "uv", + "run", + "pre-commit", + "install", + "--hook-type", + "pre-commit", + "--hook-type", + "pre-push", +) + + +def main(argv: Sequence[str]) -> int: + """Installs the git hooks pre-commit runs at commit and at push.""" + parser = argparse.ArgumentParser(description="Install the pre-commit hooks.") + parser.parse_args(list(argv)) + + print("Installing pre-commit hooks...") + expect_success(run, INSTALL_HOOKS, cwd=repository_root(), environment=os.environ) + print("Pre-commit hooks installed.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) diff --git a/scripts/lint.py b/scripts/lint.py new file mode 100644 index 000000000..050c6ac81 --- /dev/null +++ b/scripts/lint.py @@ -0,0 +1,55 @@ +import argparse +import os +import sys +from typing import Final, Sequence, Set, Tuple + +from bootstrap.passes import Pass, run_passes +from bootstrap.processes import run +from bootstrap.repository import repository_root + +MYPY: Final[str] = "mypy" +PYLINT: Final[str] = "pylint" +LINTED_TREES: Final[Tuple[str, ...]] = ("src", "scripts") + + +def linters(paths: Sequence[str]) -> Tuple[Pass, ...]: + """Mypy and pylint, in that order, over ``paths``. + + Without paths, mypy reads the files ``pyproject.toml`` configures and pylint sweeps the + source and script trees. + """ + return ( + Pass( + MYPY, + "Running type checking with mypy...", + ("uv", "run", "python", "-m", MYPY, *paths), + ), + Pass( + PYLINT, + "Running linting with pylint...", + ("uv", "run", "python", "-m", PYLINT, *(paths or LINTED_TREES)), + ), + ) + + +def main(argv: Sequence[str]) -> int: + """Type checks and lints the code, and reports which linter failed.""" + parser = argparse.ArgumentParser(description="Type check and lint the SampleToNES code.") + parser.add_argument("--mypy", action="store_true", help="run mypy alone") + parser.add_argument("--pylint", action="store_true", help="run pylint alone") + parser.add_argument("paths", nargs="*", help="files or directories to lint in place of the whole trees") + arguments = parser.parse_args(list(argv)) + + chosen: Set[str] = {name for name, wanted in ((MYPY, arguments.mypy), (PYLINT, arguments.pylint)) if wanted} + passes = tuple(linter for linter in linters(tuple(arguments.paths)) if not chosen or linter.name in chosen) + failed = run_passes(passes, root=repository_root(), runner=run, environment=os.environ) + if failed: + print(f"Linting failed: {', '.join(failed)}.") + return 1 + + print("All linting checks passed.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) diff --git a/scripts/linux/dev/format.sh b/scripts/linux/dev/format.sh deleted file mode 100755 index 16bcc7d03..000000000 --- a/scripts/linux/dev/format.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env bash - -set -e - -echo "Formatting imports with isort..." -uv run python -m isort src/ tests/ - -echo "Formatting code with black..." -uv run python -m black src/ tests/ - -echo "Code formatting complete." diff --git a/scripts/linux/dev/lint.sh b/scripts/linux/dev/lint.sh deleted file mode 100755 index 6ac482b7a..000000000 --- a/scripts/linux/dev/lint.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env bash - -set +e - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" - -bash "${SCRIPT_DIR}/mypy.sh" -MYPY_EXIT=$? - -bash "${SCRIPT_DIR}/pylint.sh" -PYLINT_EXIT=$? - -if [[ $MYPY_EXIT -ne 0 ]] || [[ $PYLINT_EXIT -ne 0 ]]; then - echo "Linting failed." - exit 1 -fi - -echo "All linting checks passed." -exit 0 diff --git a/scripts/linux/dev/mypy.sh b/scripts/linux/dev/mypy.sh deleted file mode 100755 index 409357be7..000000000 --- a/scripts/linux/dev/mypy.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env bash - -set +e - -echo "Running type checking with mypy..." -if [ $# -eq 0 ]; then - uv run python -m mypy -else - uv run python -m mypy "$@" -fi -exit $? diff --git a/scripts/linux/dev/pre_commit.sh b/scripts/linux/dev/pre_commit.sh deleted file mode 100755 index bb30b8619..000000000 --- a/scripts/linux/dev/pre_commit.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env bash - -set -e - -echo "Installing pre-commit hooks..." -uv run pre-commit install -uv run pre-commit install --hook-type pre-commit --hook-type pre-push -echo "Pre-commit hooks installed successfully." diff --git a/scripts/linux/dev/pylint.sh b/scripts/linux/dev/pylint.sh deleted file mode 100755 index 6a4ccad62..000000000 --- a/scripts/linux/dev/pylint.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env bash - -set +e - -echo "Running linting with pylint..." -if [ $# -eq 0 ]; then - uv run python -m pylint src/sampletones -else - uv run python -m pylint "$@" -fi -exit $? diff --git a/scripts/linux/dev/tests.sh b/scripts/linux/dev/tests.sh deleted file mode 100755 index 46d4f865a..000000000 --- a/scripts/linux/dev/tests.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env bash - -set +e - -echo "Running doctests..." -uv run python -m pytest src/ --doctest-modules --no-cov -DOCTEST_EXIT=$? - -echo "Running benchmarks..." -uv run python -m pytest tests/benchmarks --no-cov -BENCHMARK_EXIT=$? - -echo "Running pytest with coverage..." -uv run python -m pytest -n 6 --cov --ignore=tests/benchmarks -PYTEST_EXIT=$? - -if [[ $DOCTEST_EXIT -ne 0 ]] || [[ $PYTEST_EXIT -ne 0 ]] || [[ $BENCHMARK_EXIT -ne 0 ]]; then - echo "Tests failed." - exit 1 -fi - -echo "All tests passed." -exit 0 diff --git a/scripts/run_tests.py b/scripts/run_tests.py new file mode 100644 index 000000000..7b905b543 --- /dev/null +++ b/scripts/run_tests.py @@ -0,0 +1,77 @@ +import argparse +import os +import sys +from typing import Final, Optional, Sequence, Tuple + +from bootstrap.passes import Pass, run_passes +from bootstrap.processes import run +from bootstrap.repository import repository_root + +DOCTESTS: Final[str] = "doctests" +SUITE: Final[str] = "suite" +BENCHMARKS: Final[str] = "benchmarks" +DEFAULT_WORKERS: Final[str] = "6" +PYTEST: Final[Tuple[str, ...]] = ("uv", "run", "python", "-m", "pytest") + + +def planned_passes(workers: str) -> Tuple[Pass, ...]: + """The passes of a test run, in order: the doctests, the covered suite, the benchmarks. + + The covered suite runs across ``workers`` pytest workers. The benchmarks run last, serial, + uncovered and with their output shown, so a measured duration is the code's own cost and + its reading reaches the terminal. + + Args: + workers: The worker count for the covered suite, or ``auto`` for one per processor. + """ + return ( + Pass( + DOCTESTS, + "Running doctests...", + (*PYTEST, "src/", "--doctest-modules", "--no-cov"), + ), + Pass( + SUITE, + "Running pytest with coverage...", + (*PYTEST, "-n", workers, "--cov", "--ignore=tests/benchmarks"), + ), + Pass( + BENCHMARKS, + "Running benchmarks...", + (*PYTEST, "tests/benchmarks", "--no-cov", "-s"), + ), + ) + + +def selected(passes: Sequence[Pass], only: Optional[str]) -> Tuple[Pass, ...]: + """The passes a run performs: all of them, or the one ``only`` names.""" + return tuple(current for current in passes if only is None or current.name == only) + + +def main(argv: Sequence[str]) -> int: + """Runs the doctests, the covered suite and the benchmarks, and reports which failed.""" + parser = argparse.ArgumentParser(description="Run the SampleToNES tests.") + parser.add_argument("--only", choices=(DOCTESTS, SUITE, BENCHMARKS), help="run one pass alone") + parser.add_argument( + "--workers", + default=DEFAULT_WORKERS, + help="pytest workers for the covered suite: a count, or auto for one per processor", + ) + arguments = parser.parse_args(list(argv)) + + failed = run_passes( + selected(planned_passes(arguments.workers), arguments.only), + root=repository_root(), + runner=run, + environment=os.environ, + ) + if failed: + print(f"Tests failed: {', '.join(failed)}.") + return 1 + + print("All tests passed.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) diff --git a/scripts/windows/dev/format.bat b/scripts/windows/dev/format.bat deleted file mode 100644 index cd1717a4c..000000000 --- a/scripts/windows/dev/format.bat +++ /dev/null @@ -1,11 +0,0 @@ -@echo off -setlocal - -echo Formatting imports with isort... -uv run python -m isort src/ tests/ || exit /b - -echo Formatting code with black... -uv run python -m black src/ tests/ || exit /b - -echo Code formatting complete. -exit /b 0 diff --git a/scripts/windows/dev/lint.bat b/scripts/windows/dev/lint.bat deleted file mode 100644 index fba418f24..000000000 --- a/scripts/windows/dev/lint.bat +++ /dev/null @@ -1,23 +0,0 @@ -@echo off -setlocal - -set SCRIPT_DIR=%~dp0 - -call "%SCRIPT_DIR%mypy.bat" -set MYPY_EXIT=%ERRORLEVEL% - -call "%SCRIPT_DIR%pylint.bat" -set PYLINT_EXIT=%ERRORLEVEL% - -if not %MYPY_EXIT%==0 ( - echo Linting failed. - exit /b 1 -) - -if not %PYLINT_EXIT%==0 ( - echo Linting failed. - exit /b 1 -) - -echo All linting checks passed. -exit /b 0 diff --git a/scripts/windows/dev/mypy.bat b/scripts/windows/dev/mypy.bat deleted file mode 100644 index 686c7716f..000000000 --- a/scripts/windows/dev/mypy.bat +++ /dev/null @@ -1,10 +0,0 @@ -@echo off -setlocal - -echo Running type checking with mypy... -if "%~1"=="" ( - uv run python -m mypy -) else ( - uv run python -m mypy %* -) -exit /b %ERRORLEVEL% diff --git a/scripts/windows/dev/pre_commit.bat b/scripts/windows/dev/pre_commit.bat deleted file mode 100644 index deebc171d..000000000 --- a/scripts/windows/dev/pre_commit.bat +++ /dev/null @@ -1,9 +0,0 @@ -@echo off -setlocal - -echo Installing pre-commit hooks... -uv run pre-commit install || exit /b -uv run pre-commit install --hook-type pre-commit --hook-type pre-push || exit /b -echo Pre-commit hooks installed successfully. - -exit /b 0 diff --git a/scripts/windows/dev/pylint.bat b/scripts/windows/dev/pylint.bat deleted file mode 100644 index 5e6aa0fa0..000000000 --- a/scripts/windows/dev/pylint.bat +++ /dev/null @@ -1,10 +0,0 @@ -@echo off -setlocal - -echo Running linting with pylint... -if "%~1"=="" ( - uv run python -m pylint src/sampletones -) else ( - uv run python -m pylint %* -) -exit /b %ERRORLEVEL% diff --git a/scripts/windows/dev/tests.bat b/scripts/windows/dev/tests.bat deleted file mode 100644 index fd0d1659b..000000000 --- a/scripts/windows/dev/tests.bat +++ /dev/null @@ -1,32 +0,0 @@ -@echo off -setlocal - -echo Running doctests... -uv run python -m pytest src/ --doctest-modules --no-cov -set DOCTEST_EXIT=%ERRORLEVEL% - -echo Running benchmarks... -uv run python -m pytest tests/benchmarks --no-cov -set BENCHMARK_EXIT=%ERRORLEVEL% - -echo Running pytest with coverage... -uv run python -m pytest -n 6 --cov --ignore=tests/benchmarks -set PYTEST_EXIT=%ERRORLEVEL% - -if not %DOCTEST_EXIT%==0 ( - echo Tests failed. - exit /b 1 -) - -if not %PYTEST_EXIT%==0 ( - echo Tests failed. - exit /b 1 -) - -if not %BENCHMARK_EXIT%==0 ( - echo Tests failed. - exit /b 1 -) - -echo All tests passed. -exit /b 0 diff --git a/tests/unit/scripts/bootstrap/test_passes.py b/tests/unit/scripts/bootstrap/test_passes.py new file mode 100644 index 000000000..43f5fe507 --- /dev/null +++ b/tests/unit/scripts/bootstrap/test_passes.py @@ -0,0 +1,30 @@ +from pathlib import Path + +import pytest + +from bootstrap.passes import Pass, run_passes +from tests.suite.bootstrap import RecordingRunner + +PASSES = ( + Pass("first", "First...", ("first", "command")), + Pass("second", "Second...", ("second", "command")), + Pass("third", "Third...", ("third", "command")), +) + + +class TestRunPasses: + def test_every_pass_runs_and_the_failed_ones_are_named(self, capsys: pytest.CaptureFixture[str]) -> None: + runner = RecordingRunner({"second": 1}, None) + + failed = run_passes(PASSES, root=Path("/repository"), runner=runner, environment={"PATH": "/usr/bin"}) + + assert failed == ["second"] + assert runner.lines == ["first command", "second command", "third command"] + assert all(recorded.cwd == Path("/repository") for recorded in runner.commands) + assert all(recorded.environment == {"PATH": "/usr/bin"} for recorded in runner.commands) + assert capsys.readouterr().out == "First...\nSecond...\nThird...\n" + + def test_a_clean_run_names_nothing(self) -> None: + runner = RecordingRunner({}, None) + + assert run_passes(PASSES, root=Path("/repository"), runner=runner, environment={}) == [] diff --git a/tests/unit/scripts/bootstrap/test_processes.py b/tests/unit/scripts/bootstrap/test_processes.py index f55a02ae5..4a5b310d2 100644 --- a/tests/unit/scripts/bootstrap/test_processes.py +++ b/tests/unit/scripts/bootstrap/test_processes.py @@ -1,3 +1,4 @@ +import os import sys from pathlib import Path @@ -18,6 +19,21 @@ def test_the_status_is_the_command_s_own(self, tmp_path: Path) -> None: assert status == 3 + def test_what_the_script_printed_leads_the_command_s_output( + self, + tmp_path: Path, + capfd: pytest.CaptureFixture[str], + ) -> None: + print("before") + run( + (sys.executable, "-c", "print('inside')"), + cwd=tmp_path, + environment=os.environ, + quiet=False, + ) + + assert capfd.readouterr().out == "before\ninside\n" + class TestExpectSuccess: def test_a_succeeding_command_passes(self, tmp_path: Path) -> None: diff --git a/tests/unit/scripts/test_formatting.py b/tests/unit/scripts/test_formatting.py new file mode 100644 index 000000000..01a62ba3c --- /dev/null +++ b/tests/unit/scripts/test_formatting.py @@ -0,0 +1,39 @@ +import pytest + +from tests.suite.bootstrap import RecordingRunner +from tests.suite.scripts import load_script + +formatting = load_script("formatting.py") + + +class TestMain: + def test_isort_runs_before_black_over_the_three_trees( + self, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], + ) -> None: + runner = RecordingRunner({}, None) + monkeypatch.setattr(formatting, "run", runner) + + assert formatting.main([]) == 0 + assert runner.lines == [ + "uv run python -m isort src tests scripts", + "uv run python -m black src tests scripts", + ] + assert "Code formatting complete." in capsys.readouterr().out + + def test_named_paths_replace_the_trees(self, monkeypatch: pytest.MonkeyPatch) -> None: + runner = RecordingRunner({}, None) + monkeypatch.setattr(formatting, "run", runner) + + assert formatting.main(["scripts/lint.py"]) == 0 + assert all(line.endswith(" scripts/lint.py") for line in runner.lines) + + def test_a_failing_formatter_stops_the_run(self, monkeypatch: pytest.MonkeyPatch) -> None: + runner = RecordingRunner({"isort": 1}, None) + monkeypatch.setattr(formatting, "run", runner) + + with pytest.raises(SystemExit, match="isort"): + formatting.main([]) + + assert len(runner.lines) == 1 diff --git a/tests/unit/scripts/test_hooks.py b/tests/unit/scripts/test_hooks.py new file mode 100644 index 000000000..bbd9a0529 --- /dev/null +++ b/tests/unit/scripts/test_hooks.py @@ -0,0 +1,20 @@ +import pytest + +from tests.suite.bootstrap import RecordingRunner +from tests.suite.scripts import load_script + +hooks = load_script("hooks.py") + + +class TestMain: + def test_both_hook_stages_are_installed( + self, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], + ) -> None: + runner = RecordingRunner({}, None) + monkeypatch.setattr(hooks, "run", runner) + + assert hooks.main([]) == 0 + assert runner.lines == ["uv run pre-commit install --hook-type pre-commit --hook-type pre-push"] + assert "Pre-commit hooks installed." in capsys.readouterr().out diff --git a/tests/unit/scripts/test_lint.py b/tests/unit/scripts/test_lint.py new file mode 100644 index 000000000..6f3f1587d --- /dev/null +++ b/tests/unit/scripts/test_lint.py @@ -0,0 +1,55 @@ +import pytest + +from tests.suite.bootstrap import RecordingRunner +from tests.suite.scripts import load_script + +lint = load_script("lint.py") + + +class TestLinters: + def test_mypy_reads_its_configuration_and_pylint_sweeps_the_trees(self) -> None: + mypy, pylint = lint.linters(()) + + assert mypy.command == ("uv", "run", "python", "-m", "mypy") + assert pylint.command == ("uv", "run", "python", "-m", "pylint", "src", "scripts") + + def test_named_paths_reach_both(self) -> None: + mypy, pylint = lint.linters(("scripts/lint.py",)) + + assert mypy.command[-1] == "scripts/lint.py" + assert pylint.command[-1] == "scripts/lint.py" + + +class TestMain: + def test_both_linters_run_by_default( + self, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], + ) -> None: + runner = RecordingRunner({}, None) + monkeypatch.setattr(lint, "run", runner) + + assert lint.main([]) == 0 + assert [line.split()[-1] for line in runner.lines[:1]] == ["mypy"] + assert "pylint" in runner.lines[1] + assert "All linting checks passed." in capsys.readouterr().out + + def test_a_flag_picks_one_linter(self, monkeypatch: pytest.MonkeyPatch) -> None: + runner = RecordingRunner({}, None) + monkeypatch.setattr(lint, "run", runner) + + assert lint.main(["--pylint"]) == 0 + assert len(runner.lines) == 1 + assert "pylint src scripts" in runner.lines[0] + + def test_a_failing_linter_stops_nothing_and_is_named( + self, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], + ) -> None: + runner = RecordingRunner({"mypy": 1}, None) + monkeypatch.setattr(lint, "run", runner) + + assert lint.main([]) == 1 + assert len(runner.lines) == 2 + assert "Linting failed: mypy." in capsys.readouterr().out diff --git a/tests/unit/scripts/test_run_tests.py b/tests/unit/scripts/test_run_tests.py new file mode 100644 index 000000000..2eb92987b --- /dev/null +++ b/tests/unit/scripts/test_run_tests.py @@ -0,0 +1,53 @@ +import pytest + +from tests.suite.bootstrap import RecordingRunner +from tests.suite.scripts import load_script + +run_tests = load_script("run_tests.py") + + +class TestPlannedPasses: + def test_the_doctests_run_first_and_the_benchmarks_last(self) -> None: + passes = run_tests.planned_passes("6") + + assert [current.name for current in passes] == ["doctests", "suite", "benchmarks"] + assert "--doctest-modules" in passes[0].command + assert passes[1].command[-4:] == ("-n", "6", "--cov", "--ignore=tests/benchmarks") + assert "--no-cov" in passes[2].command + assert "-s" in passes[2].command + + +class TestSelected: + def test_a_name_keeps_one_pass(self) -> None: + passes = run_tests.planned_passes("6") + + assert run_tests.selected(passes, "suite") == (passes[1],) + assert run_tests.selected(passes, None) == passes + + +class TestMain: + def test_only_runs_the_named_pass( + self, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], + ) -> None: + runner = RecordingRunner({}, None) + monkeypatch.setattr(run_tests, "run", runner) + + assert run_tests.main(["--only", "benchmarks"]) == 0 + assert len(runner.lines) == 1 + assert "tests/benchmarks" in runner.lines[0] + assert "All tests passed." in capsys.readouterr().out + + def test_a_failing_pass_stops_nothing_and_is_named( + self, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], + ) -> None: + runner = RecordingRunner({"--doctest-modules": 1}, None) + monkeypatch.setattr(run_tests, "run", runner) + + assert run_tests.main(["--workers", "auto"]) == 1 + assert len(runner.lines) == 3 + assert "-n auto" in runner.lines[1] + assert "Tests failed: doctests." in capsys.readouterr().out From cff95e4fb775d8cee9f9803d136efb01724a1628 Mon Sep 17 00:00:00 2001 From: JakimPL Date: Sun, 13 Sep 2026 14:59:31 +0200 Subject: [PATCH 08/36] Held: the bootstrap scripts to the standard library --- docs/development/config-organization.md | 6 +- docs/development/packages.md | 10 +- docs/development/tooling.md | 5 +- scripts/checks/import_boundary.py | 34 +++-- scripts/checks/unused_tags.py | 4 +- src/sampletones_config/README.md | 2 +- .../boundaries/standalone.yaml | 13 ++ .../meta/import_boundary/configs/rules.py | 10 +- .../meta/import_boundary/standalone.py | 142 ++++++++++++++++++ src/sampletones_shared/paths/source.py | 1 + tests/suite/scripts.py | 4 +- .../import_boundary/configs/test_rules.py | 30 +++- .../meta/import_boundary/test_standalone.py | 126 ++++++++++++++++ .../sampletones_shared/paths/test_source.py | 9 +- .../scripts/checks/test_import_boundary.py | 18 +++ 15 files changed, 388 insertions(+), 26 deletions(-) create mode 100644 src/sampletones_config/boundaries/standalone.yaml create mode 100644 src/sampletones_shared/meta/import_boundary/standalone.py create mode 100644 tests/unit/sampletones_shared/meta/import_boundary/test_standalone.py diff --git a/docs/development/config-organization.md b/docs/development/config-organization.md index d2994b588..73bedb7a2 100644 --- a/docs/development/config-organization.md +++ b/docs/development/config-organization.md @@ -169,9 +169,9 @@ sites across `application.py` and the tab coordinators that read it, and the ~10 that import `SchedulingBehavior` as a type. Boundaries is the domain a developer tool reads. It states the layer graphs the packages -divide into, the imports each part of the application stays clear of, and the spellings a -tree keeps out, and `scripts/checks/import_boundary.py` runs it over the source tree on every -commit. A declaration draws on the named prefix groups `general.yaml` holds, so a set several +divide into, the imports each part of the application stays clear of, the spellings a tree +keeps out, and the standard-library rule the bootstrap scripts under `scripts/` hold to, and +`scripts/checks/import_boundary.py` runs it over the source and scripts trees on every commit. A declaration draws on the named prefix groups `general.yaml` holds, so a set several rules reach for is written once and each rule names it, and a name reaching no group is refused as the domain is read. The bundle carries the domain because `--add-data` copies `sampletones_config` whole — the terms `calibration/` already ships on. diff --git a/docs/development/packages.md b/docs/development/packages.md index f246dd5a8..997df40f1 100644 --- a/docs/development/packages.md +++ b/docs/development/packages.md @@ -123,4 +123,12 @@ Three parts share the work. `sampletones_config/boundaries/` states what the bou `sampletones_shared/meta/import_boundary/` validates that statement and holds the mechanism — reading a module line by line, resolving a unit to the modules it owns, deriving a rule from a graph and reporting what crosses it — beside the source layer the other checks read the tree through. -`scripts/checks/import_boundary.py` runs them over a source tree and prints what they find. +`scripts/checks/import_boundary.py` runs them over the source and scripts trees and prints what +they find. + +The scripts tree is held to a rule of its own, `boundaries/standalone.yaml`. A bootstrap script +runs on the system interpreter, so it imports the standard library and the scripts tree itself, +and a name in that tree that stands in for a standard-library module is reported too, since the +tree sits on the import path. The tool scripts still under `scripts/` are excluded by name until +each moves into the project, and an exclusion naming no file fails the tests, so a move takes its +exclusion with it. [Tooling](tooling.md) states the principle. diff --git a/docs/development/tooling.md b/docs/development/tooling.md index 1050a66bd..88cd4b7c2 100644 --- a/docs/development/tooling.md +++ b/docs/development/tooling.md @@ -12,7 +12,9 @@ before adding a script or a make target. Which packages may import which is before or beside the project environment: it creates the environment, installs system packages, builds the standalone bundle, cleans the tree, and runs the tests, the linters and the formatters the environment provides. It imports the standard library and the other bootstrap modules, -nothing else, so it runs on a machine that has Python and nothing more. A *tool script* runs +nothing else, so it runs on a machine that has Python and nothing more. The import boundary +check holds the tree to that: `sampletones_config/boundaries/standalone.yaml` names the scripts, +and an import beyond the standard library and the tree fails the hook. A *tool script* runs inside the project environment, through `uv run`, and imports the project's packages freely. **2. A bootstrap script installs nothing into the interpreter it runs on.** Every package a build @@ -70,6 +72,7 @@ the make target that names each. The checks are also pre-commit hooks; | Concern | Owner | |---|---| +| What a bootstrap script may import | `sampletones_config/boundaries/standalone.yaml` | | What differs between systems | `scripts/bootstrap/platforms/` | | Where a build installs | `scripts/bootstrap/venv_build.py` | | What a bundle has to carry before it is built | `scripts/bootstrap/preflight.py` | diff --git a/scripts/checks/import_boundary.py b/scripts/checks/import_boundary.py index c906623b0..7c903ba61 100755 --- a/scripts/checks/import_boundary.py +++ b/scripts/checks/import_boundary.py @@ -19,13 +19,17 @@ express — e.g. that panels never compose a column suffix (`SUF_PANEL_*`) or parent into another panel's container. +A standalone rule holds the bootstrap scripts under `scripts/` to the standard library and the +tree they sit in, since they run on the system interpreter before the project environment exists, +and reports a name in that tree that stands in for a standard-library module. + `sampletones_config/boundaries/` declares what the boundaries are and `sampletones_shared/meta/import_boundary/` holds how they are read and reported; this script runs -them over a source tree and prints what they find. +them over the source and scripts trees and prints what they find. Usage: python scripts/checks/import_boundary.py [files...] # check specific files - python scripts/checks/import_boundary.py --all # run all rules against the source tree + python scripts/checks/import_boundary.py --all # run all rules against both trees """ import argparse @@ -35,7 +39,8 @@ from sampletones_shared.meta.import_boundary.check import check_boundaries from sampletones_shared.meta.import_boundary.configs.rules import ImportBoundaryRules -from sampletones_shared.paths.source import SOURCE_ROOT +from sampletones_shared.meta.import_boundary.standalone import check_standalone +from sampletones_shared.paths.source import SCRIPTS_ROOT, SOURCE_ROOT def main(argv: Sequence[str]) -> int: @@ -52,7 +57,7 @@ def main(argv: Sequence[str]) -> int: parser.add_argument( "--all", action="store_true", - help=f"check every module under {SOURCE_ROOT.name}/ instead of named files", + help=f"check every module under {SOURCE_ROOT.name}/ and {SCRIPTS_ROOT.name}/ instead of named files", ) parser.add_argument( "--source", @@ -60,17 +65,26 @@ def main(argv: Sequence[str]) -> int: default=SOURCE_ROOT, help="source root the rule roots are named within", ) + parser.add_argument( + "--scripts", + type=Path, + default=SCRIPTS_ROOT, + help="scripts tree the standalone rules are written against", + ) arguments = parser.parse_args(list(argv)) files: List[Path] = arguments.files selection = None if arguments.all else {path.resolve() for path in files} boundaries = ImportBoundaryRules.load() - violations = check_boundaries( - arguments.source, - boundaries.boundary_rules(), - boundaries.tokens, - selection, - ) + violations = [ + *check_boundaries( + arguments.source, + boundaries.boundary_rules(), + boundaries.tokens, + selection, + ), + *check_standalone(arguments.scripts, boundaries.standalone, selection), + ] if not violations: return 0 diff --git a/scripts/checks/unused_tags.py b/scripts/checks/unused_tags.py index c2a87b5ce..2dc8c9fc5 100755 --- a/scripts/checks/unused_tags.py +++ b/scripts/checks/unused_tags.py @@ -22,13 +22,13 @@ from sampletones_shared.meta.source.modules import SourceModule, discover_modules from sampletones_shared.meta.source.packages import package_directory from sampletones_shared.meta.source.references import count_identifier_loads -from sampletones_shared.paths.source import REPOSITORY_ROOT, SOURCE_ROOT +from sampletones_shared.paths.source import REPOSITORY_ROOT, SCRIPTS_ROOT, SOURCE_ROOT TAGS_PACKAGE: Final[Path] = package_directory("sampletones_application", "tags") REFERENCE_ROOTS: Final[Tuple[Path, ...]] = ( SOURCE_ROOT, REPOSITORY_ROOT / "tests", - REPOSITORY_ROOT / "scripts", + SCRIPTS_ROOT, ) FRAGMENT_PREFIXES: Final[Tuple[str, ...]] = ("TAG_", "SUF_", "PRE_") diff --git a/src/sampletones_config/README.md b/src/sampletones_config/README.md index 767fb77f6..430c78194 100644 --- a/src/sampletones_config/README.md +++ b/src/sampletones_config/README.md @@ -19,7 +19,7 @@ The data package must not import a schema, and a schema package must not inline |-----------|---------|--------------| | `application/` | Deployment-time environment knobs | `DeploymentConfig` | | `behavior/` | Non-visual runtime behavior | `BehaviorConfig` | -| `boundaries/` | The imports the source tree is held to | `ImportBoundaryRules` | +| `boundaries/` | The imports the source and scripts trees are held to | `ImportBoundaryRules` | | `calibration/` | DSP calibration tuning | `CorpusConfig`, `RefereeConfig` | | `keybindings/` | The key combinations each named action answers | `ShortcutScheme` | | `lang/` | Interface strings (i18n) | `LanguageManager` | diff --git a/src/sampletones_config/boundaries/standalone.yaml b/src/sampletones_config/boundaries/standalone.yaml new file mode 100644 index 000000000..ed2ac61bf --- /dev/null +++ b/src/sampletones_config/boundaries/standalone.yaml @@ -0,0 +1,13 @@ +- pattern: "**/*.py" + excluding: + - "calibration.py" + - "compression_study.py" + - "nsf_render.py" + - "player.py" + - "assets/**/*.py" + - "checks/**/*.py" + - "codec_study/**/*.py" + reserved: [build, test, tests] + message: >- + a bootstrap script runs on the system interpreter, so it imports the standard library and + the scripts tree alone; code that needs the project environment is a tool, run through uv diff --git a/src/sampletones_shared/meta/import_boundary/configs/rules.py b/src/sampletones_shared/meta/import_boundary/configs/rules.py index b4fc43a46..a4069986e 100644 --- a/src/sampletones_shared/meta/import_boundary/configs/rules.py +++ b/src/sampletones_shared/meta/import_boundary/configs/rules.py @@ -7,6 +7,7 @@ from sampletones_shared.meta.import_boundary.configs.paths import BOUNDARIES_DIRECTORY from sampletones_shared.meta.import_boundary.graph import LayerGraph from sampletones_shared.meta.import_boundary.rule import BoundaryRule +from sampletones_shared.meta.import_boundary.standalone import StandaloneRule from sampletones_shared.meta.import_boundary.token import TokenRule from sampletones_shared.utils.serialization import load_yaml_model_dir @@ -14,17 +15,19 @@ class ImportBoundaryRules(BaseModel): """Every boundary the source tree is held to, as the shipped configuration states it. - The declaration comes in three forms, each a fragment of its own. A layer graph names a tree + The declaration comes in four forms, each a fragment of its own. A layer graph names a tree of modules and what each unit may import, and amounts to one rule per unit. A boundary declaration names one directory and the imports it stays clear of. A token rule names a - spelling a tree keeps out. The general vocabulary holds the prefix sets the declarations draw - on, so a set several of them reach for is written once. + spelling a tree keeps out. A standalone rule names the scripts that run on the system + interpreter and holds them to the standard library. The general vocabulary holds the prefix + sets the declarations draw on, so a set several of them reach for is written once. Attributes: general: The names the declarations are written in. graphs: Each layer graph the source tree divides into, under the name the documents give it. rules: The boundaries written directly. tokens: The spellings kept out of the trees they name. + standalone: The scripts held to the standard library and the tree they sit in. """ model_config = ConfigDict(extra="forbid", frozen=True) @@ -33,6 +36,7 @@ class ImportBoundaryRules(BaseModel): graphs: Dict[str, LayerGraph] rules: Tuple[BoundaryDeclaration, ...] tokens: Tuple[TokenRule, ...] + standalone: Tuple[StandaloneRule, ...] @model_validator(mode="after") def _validate_every_named_group_is_declared(self) -> Self: diff --git a/src/sampletones_shared/meta/import_boundary/standalone.py b/src/sampletones_shared/meta/import_boundary/standalone.py new file mode 100644 index 000000000..b21162dd4 --- /dev/null +++ b/src/sampletones_shared/meta/import_boundary/standalone.py @@ -0,0 +1,142 @@ +import sys +from pathlib import Path +from typing import Final, List, Optional, Sequence, Set, Tuple + +from pydantic import BaseModel, ConfigDict + +from sampletones_shared.meta.import_boundary.imports import imported_module +from sampletones_shared.meta.import_boundary.lines import numbered_lines +from sampletones_shared.meta.import_boundary.scope import rule_modules +from sampletones_shared.meta.import_boundary.units import MODULE_SUFFIX +from sampletones_shared.meta.import_boundary.violation import Violation +from sampletones_shared.meta.source.modules import ( + MODULE_SEPARATOR, + PACKAGE_INITIALIZER, + SOURCE_PATTERN, + source_paths, +) + +SHADOWING: Final[str] = "stands in for a module the tree sits beside" + + +def local_names(root: Path) -> Set[str]: + """The names a script under ``root`` reaches in the tree itself: its modules and its packages. + + A script runs with its tree on the import path, so a module beside it and a package holding + an initializer are reached by their bare names. + + Args: + root: The tree the scripts sit in. + + Returns: + Set[str]: The importable names the tree offers. + """ + modules = {path.stem for path in root.glob(SOURCE_PATTERN)} + packages = {path.name for path in root.iterdir() if (path / f"{PACKAGE_INITIALIZER}{MODULE_SUFFIX}").is_file()} + return modules | packages + + +class StandaloneRule(BaseModel): + """One tree of scripts that run on the system interpreter, and what they may import. + + A script the rule reaches imports the standard library and the tree it sits in, so it runs + on a machine that has Python and nothing more. The tree sits on the import path beside the + standard library and the repository's own packages, so a name in it that stands in for one + of theirs is reported as well. + + Attributes: + pattern: Glob naming the scripts the rule reaches, written against the tree's root. + excluding: Globs naming the scripts the rule leaves to the project environment. + reserved: Names the tree keeps clear of beyond the standard library's. + message: What the rule holds, printed where a script imports past it. + """ + + model_config = ConfigDict(extra="forbid", frozen=True) + + pattern: str + excluding: Tuple[str, ...] = () + reserved: Tuple[str, ...] = () + message: str + + def violations(self, path: Path, local: Set[str]) -> List[Violation]: + """Every import one script takes beyond the standard library and the tree. + + A relative import stays inside the tree by construction, and a name the tree offers is + reached the way a script reaches it when run from the tree. + + Args: + path: Script to read. + local: The names the tree offers, as `local_names` reads them. + + Returns: + List[Violation]: The imports the rule reports, in line order. + + Raises: + OSError: If the script cannot be read. + """ + violations: List[Violation] = [] + for line_number, line in numbered_lines(path): + module = imported_module(line) + if module is None: + continue + + top = module.split(MODULE_SEPARATOR, 1)[0] + if top and top not in sys.stdlib_module_names and top not in local: + violations.append(Violation.at(self.message, path, line_number, line)) + + return violations + + def shadowing(self, root: Path, paths: Sequence[Path]) -> List[Violation]: + """Every name in the tree that stands in for a standard-library or reserved module. + + A name is reported once, at the first script that carries it, since a directory is + named by every script under it. + + Args: + root: The tree the scripts sit in, resolved. + paths: The scripts the rule reaches, resolved and in path order. + + Returns: + List[Violation]: One violation per offending name, in the order the tree is read. + """ + taken = set(sys.stdlib_module_names) | set(self.reserved) + seen: Set[str] = set() + violations: List[Violation] = [] + for path in paths: + for part in path.relative_to(root).with_suffix("").parts: + if part in taken and part not in seen: + seen.add(part) + violations.append(Violation(kind=f"{part} {SHADOWING}", location=str(path))) + + return violations + + +def check_standalone( + root: Path, + rules: Sequence[StandaloneRule], + selection: Optional[Set[Path]], +) -> List[Violation]: + """Every import and name the rules forbid under a tree of standalone scripts. + + Args: + root: The tree the rules are written against. + rules: What the scripts may import. + selection: Resolved paths to narrow the check to, or `None` to check the whole tree. + + Returns: + List[Violation]: What the rules report, the shadowing names before the imports. + + Raises: + NotADirectoryError: If the root names no directory. + FileNotFoundError: If the root holds no module to read. + """ + tree = root.resolve() + swept = {path.resolve() for path in source_paths([tree])} + local = local_names(tree) + violations: List[Violation] = [] + for rule in rules: + paths = rule_modules(tree, rule.pattern, rule.excluding, swept, selection) + violations.extend(rule.shadowing(tree, paths)) + violations.extend(violation for path in paths for violation in rule.violations(path, local)) + + return violations diff --git a/src/sampletones_shared/paths/source.py b/src/sampletones_shared/paths/source.py index 51dd98fa8..517910b4b 100644 --- a/src/sampletones_shared/paths/source.py +++ b/src/sampletones_shared/paths/source.py @@ -3,3 +3,4 @@ SOURCE_ROOT: Final[Path] = Path(__file__).resolve().parents[2] REPOSITORY_ROOT: Final[Path] = SOURCE_ROOT.parent +SCRIPTS_ROOT: Final[Path] = REPOSITORY_ROOT / "scripts" diff --git a/tests/suite/scripts.py b/tests/suite/scripts.py index 413fa9d7a..66cf1ce7b 100644 --- a/tests/suite/scripts.py +++ b/tests/suite/scripts.py @@ -1,7 +1,7 @@ import importlib.util from types import ModuleType -from sampletones_shared.paths.source import REPOSITORY_ROOT +from sampletones_shared.paths.source import SCRIPTS_ROOT def load_script(relative_path: str) -> ModuleType: @@ -10,7 +10,7 @@ def load_script(relative_path: str) -> ModuleType: The scripts under ``scripts/`` are entry points invoked by path from workflows, hooks and the Makefile, so importing them the same way keeps a test exercising the module the tooling runs. """ - path = REPOSITORY_ROOT / "scripts" / relative_path + path = SCRIPTS_ROOT / relative_path spec = importlib.util.spec_from_file_location(path.stem, path) assert spec is not None and spec.loader is not None diff --git a/tests/unit/sampletones_shared/meta/import_boundary/configs/test_rules.py b/tests/unit/sampletones_shared/meta/import_boundary/configs/test_rules.py index 58e9acf81..0aab27b23 100644 --- a/tests/unit/sampletones_shared/meta/import_boundary/configs/test_rules.py +++ b/tests/unit/sampletones_shared/meta/import_boundary/configs/test_rules.py @@ -12,7 +12,8 @@ from sampletones_shared.meta.import_boundary.graph import reached_units from sampletones_shared.meta.import_boundary.rule import BoundaryRule from sampletones_shared.meta.import_boundary.scope import rule_modules -from sampletones_shared.paths.source import SOURCE_ROOT +from sampletones_shared.meta.import_boundary.standalone import check_standalone +from sampletones_shared.paths.source import SCRIPTS_ROOT, SOURCE_ROOT from tests.suite.source import swept_paths, write_module BOUNDARIES: Final[ImportBoundaryRules] = ImportBoundaryRules.load() @@ -29,6 +30,7 @@ ASSEMBLER_IMPORT: Final[str] = "from sampletones_player.driver.assembler.builder import build_driver\n" DRIVER_IMPORT: Final[str] = "from sampletones_player.driver.image import DriverImage\n" PANEL_SUFFIX: Final[str] = "def build() -> None:\n dpg.add_group(parent=SUF_PANEL_LEFT)\n" +THIRD_PARTY_IMPORT: Final[str] = "import numpy\n" def reached_modules(rule: BoundaryRule) -> List[Path]: @@ -108,6 +110,7 @@ def test_a_declaration_naming_no_declared_group_is_refused(self) -> None: ), ), tokens=(), + standalone=(), ) @@ -151,6 +154,24 @@ def test_a_panel_composing_a_column_suffix_is_reported(self, tmp_path: Path) -> assert len(reported(tmp_path)) == 1 +class TestStandaloneRules: + """The scripts that run on the system interpreter, held to the standard library.""" + + def test_the_scripts_tree_holds_to_the_rule(self) -> None: + assert check_standalone(SCRIPTS_ROOT, BOUNDARIES.standalone, None) == [] + + def test_a_bootstrap_script_reaching_a_third_party_package_is_reported(self, tmp_path: Path) -> None: + write_module(tmp_path, "bundle.py", THIRD_PARTY_IMPORT) + + reported = check_standalone(tmp_path, BOUNDARIES.standalone, None) + + assert [violation.kind for violation in reported] == [rule.message for rule in BOUNDARIES.standalone] + + def test_every_excluded_glob_names_a_script_still_in_the_tree(self) -> None: + """A tool that moved into the project takes its exclusion with it.""" + assert all(list(SCRIPTS_ROOT.glob(glob)) for rule in BOUNDARIES.standalone for glob in rule.excluding) + + class TestRuleCoverage: """A rule naming no module of the tree reads as a clean tree, so each one reaches something.""" @@ -162,3 +183,10 @@ def test_every_boundary_rule_reaches_a_module(self) -> None: def test_every_token_rule_reaches_a_module(self) -> None: assert all(list((SOURCE_ROOT / rule.root).glob(rule.pattern)) for rule in BOUNDARIES.tokens) + + def test_every_standalone_rule_reaches_a_script(self) -> None: + swept = swept_paths(SCRIPTS_ROOT) + + assert all( + rule_modules(SCRIPTS_ROOT, rule.pattern, rule.excluding, swept, None) for rule in BOUNDARIES.standalone + ) diff --git a/tests/unit/sampletones_shared/meta/import_boundary/test_standalone.py b/tests/unit/sampletones_shared/meta/import_boundary/test_standalone.py new file mode 100644 index 000000000..eb184a54c --- /dev/null +++ b/tests/unit/sampletones_shared/meta/import_boundary/test_standalone.py @@ -0,0 +1,126 @@ +from pathlib import Path +from typing import Final, List + +import pytest + +from sampletones_shared.meta.import_boundary.standalone import ( + StandaloneRule, + check_standalone, + local_names, +) +from tests.suite.source import write_module + +MESSAGE: Final[str] = "a bootstrap script imports the standard library and the tree alone" + +RULE: Final[StandaloneRule] = StandaloneRule( + pattern="**/*.py", + excluding=("tools/**/*.py",), + reserved=("tests",), + message=MESSAGE, +) + +THIRD_PARTY: Final[str] = "import numpy\n" +REACHABLE: Final[str] = ( + "import argparse\n" + "from pathlib import Path\n" + "from bootstrap.processes import run\n" + "import detect_cuda\n" + "from . import sibling\n" +) + + +def _tree(root: Path) -> None: + write_module(root / "bootstrap", "__init__.py", "") + write_module(root / "bootstrap", "processes.py", "import subprocess\n") + write_module(root, "detect_cuda.py", "import re\n") + + +def kinds(root: Path) -> List[str]: + """What the rule reports over a tree a test builds.""" + return [violation.kind for violation in check_standalone(root, (RULE,), None)] + + +class TestLocalNames: + def test_the_modules_and_the_packages_of_the_tree_are_its_names(self, tmp_path: Path) -> None: + _tree(tmp_path) + (tmp_path / "runtime_hooks").mkdir() + + assert local_names(tmp_path) == {"bootstrap", "detect_cuda"} + + +class TestStandaloneViolations: + """The imports of one script that reach past the standard library and the tree.""" + + def test_a_third_party_import_is_reported_with_the_rules_message(self, tmp_path: Path) -> None: + path = write_module(tmp_path, "bundle.py", THIRD_PARTY) + + assert [violation.kind for violation in RULE.violations(path, set())] == [MESSAGE] + + def test_the_standard_library_and_the_tree_stay_reachable(self, tmp_path: Path) -> None: + path = write_module(tmp_path, "bundle.py", REACHABLE) + + assert RULE.violations(path, {"bootstrap", "detect_cuda"}) == [] + + def test_the_report_names_the_line_the_import_sits_on(self, tmp_path: Path) -> None: + path = write_module(tmp_path, "bundle.py", f"import sys\n{THIRD_PARTY}") + + assert RULE.violations(path, set())[0].location == f"{path}:2: import numpy" + + +class TestShadowing: + """A name in the tree that stands in for a module the tree sits beside.""" + + def test_a_standard_library_name_is_reported_once_at_its_first_script(self, tmp_path: Path) -> None: + first = write_module(tmp_path / "venv", "first.py", "") + write_module(tmp_path / "venv", "second.py", "") + + reported = RULE.shadowing(tmp_path, [first, tmp_path / "venv" / "second.py"]) + + assert [violation.kind for violation in reported] == ["venv stands in for a module the tree sits beside"] + assert reported[0].location == str(first) + + def test_a_reserved_name_is_reported(self, tmp_path: Path) -> None: + path = write_module(tmp_path, "tests.py", "") + + assert [violation.kind for violation in RULE.shadowing(tmp_path, [path])] == [ + "tests stands in for a module the tree sits beside", + ] + + def test_a_name_of_the_trees_own_is_left_alone(self, tmp_path: Path) -> None: + path = write_module(tmp_path / "bootstrap", "venv_build.py", "") + + assert RULE.shadowing(tmp_path, [path]) == [] + + +class TestCheckStandalone: + """Every rule read over one tree, from the sweep to the report.""" + + def test_a_clean_tree_reports_nothing(self, tmp_path: Path) -> None: + _tree(tmp_path) + write_module(tmp_path, "bundle.py", REACHABLE) + + assert kinds(tmp_path) == [] + + def test_an_excluded_script_is_left_to_the_project_environment(self, tmp_path: Path) -> None: + _tree(tmp_path) + write_module(tmp_path / "tools", "calibration.py", THIRD_PARTY) + + assert kinds(tmp_path) == [] + + def test_the_shadowing_names_lead_the_imports(self, tmp_path: Path) -> None: + _tree(tmp_path) + write_module(tmp_path, "bundle.py", THIRD_PARTY) + write_module(tmp_path, "platform.py", "") + + assert kinds(tmp_path) == ["platform stands in for a module the tree sits beside", MESSAGE] + + def test_a_selection_narrows_the_check(self, tmp_path: Path) -> None: + _tree(tmp_path) + checked = write_module(tmp_path, "checked.py", THIRD_PARTY) + write_module(tmp_path, "other.py", THIRD_PARTY) + + assert len(check_standalone(tmp_path, (RULE,), {checked.resolve()})) == 1 + + def test_a_tree_holding_no_script_stops_the_check(self, tmp_path: Path) -> None: + with pytest.raises(FileNotFoundError): + check_standalone(tmp_path, (RULE,), None) diff --git a/tests/unit/sampletones_shared/paths/test_source.py b/tests/unit/sampletones_shared/paths/test_source.py index 6d5f1ec83..36a210b2c 100644 --- a/tests/unit/sampletones_shared/paths/test_source.py +++ b/tests/unit/sampletones_shared/paths/test_source.py @@ -1,4 +1,4 @@ -from sampletones_shared.paths.source import REPOSITORY_ROOT, SOURCE_ROOT +from sampletones_shared.paths.source import REPOSITORY_ROOT, SCRIPTS_ROOT, SOURCE_ROOT PROJECT_FILE = "pyproject.toml" SHARED_PACKAGE = "sampletones_shared" @@ -18,4 +18,9 @@ def test_the_repository_root_holds_the_project_file(self) -> None: assert (REPOSITORY_ROOT / PROJECT_FILE).is_file() def test_the_repository_root_holds_the_scripts_the_checks_run_from(self) -> None: - assert (REPOSITORY_ROOT / "scripts" / "checks").is_dir() + assert (SCRIPTS_ROOT / "checks").is_dir() + + +class TestScriptsRoot: + def test_the_scripts_root_holds_the_bootstrap_tree(self) -> None: + assert (SCRIPTS_ROOT / "bootstrap" / "__init__.py").is_file() diff --git a/tests/unit/scripts/checks/test_import_boundary.py b/tests/unit/scripts/checks/test_import_boundary.py index 2e0798f3b..c0b1daef0 100644 --- a/tests/unit/scripts/checks/test_import_boundary.py +++ b/tests/unit/scripts/checks/test_import_boundary.py @@ -12,6 +12,7 @@ VISUAL_IMPORT: Final[str] = "import dearpygui.dearpygui as dpg\n" PLAIN_IMPORT: Final[str] = "from sampletones_core.project.project import Project\n" +THIRD_PARTY_IMPORT: Final[str] = "import numpy\n" class TestMain: @@ -32,6 +33,23 @@ def test_a_forbidden_import_is_reported_where_it_sits( assert f"{path}:1" in error assert "dearpygui" in error + def test_a_bootstrap_script_reaching_past_the_standard_library_is_reported( + self, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + ) -> None: + write_module(tmp_path / "src" / APPLICATION / "logic", "clean.py", PLAIN_IMPORT) + path = write_module(tmp_path / "scripts", "bundle.py", THIRD_PARTY_IMPORT) + + exit_code = check_import_boundary.main( + ["--all", "--source", str(tmp_path / "src"), "--scripts", str(tmp_path / "scripts")], + ) + + assert exit_code == 1 + error = capsys.readouterr().err + assert f"{path}:1" in error + assert "system interpreter" in error + def test_named_files_narrow_the_run_to_themselves( self, tmp_path: Path, From 77059f9c0477252745d7b7385a21f61a8cb5c573 Mon Sep 17 00:00:00 2001 From: JakimPL Date: Sun, 13 Sep 2026 15:15:39 +0200 Subject: [PATCH 09/36] Named: every command the sampletones entry runs --- .github/workflows/workflow.yml | 2 +- .gitignore | 2 +- README.md | 11 +- docs/api/index.md | 4 +- docs/concepts/instruction-library.md | 2 +- docs/development/packages.md | 2 +- docs/development/tooling.md | 103 ++++-- docs/formats/instruction-libraries.md | 2 +- docs/formats/reconstructions.md | 15 + docs/guide/command-line.md | 60 ++-- scripts/bundle.py | 2 +- src/sampletones/__main__.py | 223 +------------ .../commands}/__init__.py | 0 src/sampletones/commands/convert.py | 81 +++++ src/sampletones/commands/library.py | 37 +++ src/sampletones/commands/open.py | 68 ++++ src/sampletones/commands/options.py | 10 + src/sampletones/commands/registry.py | 11 + src/sampletones/commands/run.py | 36 +++ src/sampletones/commands/self_check.py | 23 ++ src/sampletones/dispatcher.py | 58 ++++ src/sampletones_core/calibration/runner.py | 2 +- src/sampletones_core/headless/__init__.py | 0 src/sampletones_core/headless/config.py | 9 + src/sampletones_core/headless/conversion.py | 293 ++++++++++++++++++ .../{scripts => headless}/library.py | 0 .../scripts/reconstruction.py | 151 --------- src/sampletones_shared/command.py | 23 ++ tests/suite/commands.py | 26 ++ .../unit/sampletones/commands/test_convert.py | 126 ++++++++ .../unit/sampletones/commands/test_library.py | 29 ++ tests/unit/sampletones/commands/test_open.py | 60 ++++ .../sampletones/commands/test_registry.py | 13 + tests/unit/sampletones/commands/test_run.py | 28 ++ .../sampletones/commands/test_self_check.py | 13 + tests/unit/sampletones/test_dispatcher.py | 64 ++++ tests/unit/sampletones/test_self_check.py | 40 +++ .../sampletones_core/headless/test_config.py | 15 + .../headless/test_conversion.py | 178 +++++++++++ tests/unit/sampletones_shared/test_command.py | 31 ++ tests/unit/scripts/test_bundle.py | 4 +- 41 files changed, 1413 insertions(+), 444 deletions(-) rename src/{sampletones_core/scripts => sampletones/commands}/__init__.py (100%) create mode 100644 src/sampletones/commands/convert.py create mode 100644 src/sampletones/commands/library.py create mode 100644 src/sampletones/commands/open.py create mode 100644 src/sampletones/commands/options.py create mode 100644 src/sampletones/commands/registry.py create mode 100644 src/sampletones/commands/run.py create mode 100644 src/sampletones/commands/self_check.py create mode 100644 src/sampletones/dispatcher.py create mode 100644 src/sampletones_core/headless/__init__.py create mode 100644 src/sampletones_core/headless/config.py create mode 100644 src/sampletones_core/headless/conversion.py rename src/sampletones_core/{scripts => headless}/library.py (100%) delete mode 100644 src/sampletones_core/scripts/reconstruction.py create mode 100644 src/sampletones_shared/command.py create mode 100644 tests/suite/commands.py create mode 100644 tests/unit/sampletones/commands/test_convert.py create mode 100644 tests/unit/sampletones/commands/test_library.py create mode 100644 tests/unit/sampletones/commands/test_open.py create mode 100644 tests/unit/sampletones/commands/test_registry.py create mode 100644 tests/unit/sampletones/commands/test_run.py create mode 100644 tests/unit/sampletones/commands/test_self_check.py create mode 100644 tests/unit/sampletones/test_dispatcher.py create mode 100644 tests/unit/sampletones/test_self_check.py create mode 100644 tests/unit/sampletones_core/headless/test_config.py create mode 100644 tests/unit/sampletones_core/headless/test_conversion.py create mode 100644 tests/unit/sampletones_shared/test_command.py diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml index 958c6e8ab..53800c847 100644 --- a/.github/workflows/workflow.yml +++ b/.github/workflows/workflow.yml @@ -90,7 +90,7 @@ jobs: - name: Check the wheel carries every resource it needs at startup shell: bash - run: sampletones --self-check + run: sampletones self-check bundle: name: Standalone bundle (${{ matrix.platform }}) diff --git a/.gitignore b/.gitignore index 585ea73dc..10e71d195 100644 --- a/.gitignore +++ b/.gitignore @@ -16,7 +16,7 @@ wheels/ sampletones !src/sampletones -!tests/sampletones +!tests/unit/sampletones *.pyc *.pyo diff --git a/README.md b/README.md index 7ce8659b9..4098bb697 100644 --- a/README.md +++ b/README.md @@ -123,15 +123,16 @@ Your configuration, instruction libraries (`.ins`), and reconstructions (`.stn`) ### Command line -You can run without the GUI to use a custom config, generate an instruction library, or reconstruct a file: +Every operation is a named command, and `sampletones` alone starts the interface: ```sh -sampletones --config # run with a custom config -sampletones --generate --config # generate an instruction library -sampletones --config --output # reconstruct an audio file +sampletones run --config # start with a custom config +sampletones open # start with a project, reconstruction or library loaded +sampletones convert --config -o # reconstruct a recording without the GUI +sampletones library --config # generate an instruction library ``` -Run `sampletones --help` for all options. +Run `sampletones --help` for the commands and `sampletones --help` for a command's options. ## Documentation diff --git a/docs/api/index.md b/docs/api/index.md index a2752db75..81819275e 100644 --- a/docs/api/index.md +++ b/docs/api/index.md @@ -77,13 +77,13 @@ A reconstruction searches an [instruction library](../formats/instruction-librar ```python from sampletones import Config -from sampletones_core.scripts.library import generate_library +from sampletones_core.headless.library import generate_library config = Config.load("config.json") generate_library(config) # renders every instruction and writes the .ins library ``` -The same step is reached from the application's _Instructions_ tab, or on the command line with `sampletones --generate --config config.json`. +The same step is reached from the application's _Instructions_ tab, or on the command line with `sampletones library --config config.json`. ### Reconstruct a sample diff --git a/docs/concepts/instruction-library.md b/docs/concepts/instruction-library.md index c0890acd3..374bd6dc8 100644 --- a/docs/concepts/instruction-library.md +++ b/docs/concepts/instruction-library.md @@ -52,7 +52,7 @@ representation so their spectra are directly comparable. ## Generating and exploring Generate a library from the _Instructions_ tab, or on the command line with -`sampletones --generate`. Generation renders every instruction and stores its +`sampletones library`. Generation renders every instruction and stores its waveform and spectrum; a configuration that has no library yet is also built automatically the first time a reconstruction needs it. Regenerating a library that already exists replaces it. diff --git a/docs/development/packages.md b/docs/development/packages.md index 997df40f1..4049fea1d 100644 --- a/docs/development/packages.md +++ b/docs/development/packages.md @@ -48,7 +48,7 @@ graph TD | `sampletones_core` | The reconstruction engine, the project model, playing a song out into instructions, and the tracker export formats | `sampletones_shared`, `sampletones_synthesis` | | `sampletones_player` | The NES player: the register model, the re-clocking schedule, the 6502 driver and the NSF file | `sampletones_shared`, `sampletones_core` | | `sampletones_application` | The DearPyGui front end | `sampletones_shared`, `sampletones_core`, `sampletones_player` | -| `sampletones` | The command-line entry point and the startup self-check | `sampletones_shared`, `sampletones_core`, `sampletones_application` | +| `sampletones` | The command-line entry: the dispatcher, the commands and the startup self-check | `sampletones_shared`, `sampletones_core`, `sampletones_application` | Third-party imports are the package author's own choice and stand outside this table. diff --git a/docs/development/tooling.md b/docs/development/tooling.md index 88cd4b7c2..f9e14e925 100644 --- a/docs/development/tooling.md +++ b/docs/development/tooling.md @@ -1,42 +1,81 @@ # Tooling -This document governs the scripts under `scripts/` and the `Makefile`: what runs on the system -interpreter, what runs in the project environment, and the rules each kind holds to. Read it -before adding a script or a make target. Which packages may import which is -[package layers](packages.md); the libraries and tools the scripts reach for are -[dependencies](dependencies.md). +This document governs how the repository is run: the `sampletones` command and what it offers, +the scripts under `scripts/`, and the `Makefile`. Read it before adding a command, a script or a +make target. Which packages may import which is [package layers](packages.md); the libraries and +tools the scripts reach for are [dependencies](dependencies.md). ## Principles -**1. Two interpreters, two kinds of script.** A *bootstrap script* runs on the system interpreter, -before or beside the project environment: it creates the environment, installs system packages, -builds the standalone bundle, cleans the tree, and runs the tests, the linters and the formatters -the environment provides. It imports the standard library and the other bootstrap modules, -nothing else, so it runs on a machine that has Python and nothing more. The import boundary -check holds the tree to that: `sampletones_config/boundaries/standalone.yaml` names the scripts, -and an import beyond the standard library and the tree fails the hook. A *tool script* runs -inside the project environment, through `uv run`, and imports the project's packages freely. - -**2. A bootstrap script installs nothing into the interpreter it runs on.** Every package a build -installs lands in `.venv-build`, a virtual environment of its own, and pip is told to refuse any -interpreter outside a virtual environment. System packages are a step of their own, -`make system-deps`, and the only one that asks for administrator rights. A release build reaches -neither uv nor the developer's environment, so it runs the same on a clean machine. - -**3. One script per operation, the same on every system.** What differs between systems (the +**1. One door.** Everything a person runs by hand is a `sampletones` command with its own parser +and help, always named: `sampletones` alone is `sampletones run`, a file is opened with +`sampletones open `, a recording is converted with `sampletones convert `. A +command's name says what it does, in plain words. + +**2. Three kinds of runnable code, told apart by who runs them and what they may import.** The +*application* is what a user installs. A *tool* runs inside the project environment, through +`uv run`: it may import any package, and only the command line reaches it. A *bootstrap script* +runs on the system interpreter, before or beside the environment: it creates the environment, +installs system packages, builds the standalone bundle, cleans the tree, and runs the tests, the +linters and the formatters the environment provides. It imports the standard library and the other +bootstrap modules, nothing else, so it runs on a machine that has Python and nothing more, and it +installs nothing into the interpreter it runs on: every package a build installs lands in +`.venv-build`, a virtual environment of its own, and pip is told to refuse any interpreter outside +one. System packages are a step of their own, `make system-deps`, the only one that asks for +administrator rights. The import boundary check holds the tree to the rule: +`sampletones_config/boundaries/standalone.yaml` names the scripts, and an import beyond the +standard library and the tree fails the hook. + +**3. What earns a place on the command.** An operation is a *user command* when its input and +output are the user's own files and it needs nothing beyond the installed package. It is a +*developer command* when it reads or writes the repository or measures the code on this machine, +so it needs a checkout and the project environment. It is a *bootstrap script* when it must run +without the environment. A test is run by pytest and is never wrapped in a command; a tool the +tests exercise is a function they call. + +**4. A tool is a library with a thin face.** The work is a function that takes values and returns +values; the command module parses the arguments, calls it and prints. A command module imports the +standard library, the command type and its constants at module level, and its implementation +inside `run`, so listing the commands loads no tool. A bootstrap script is shaped the same way: +pure functions assemble the commands and take the decisions, `main` wires in the real runner and +the real environment, and the tests call the functions with a runner that records what it was +asked to run, so a build is verified without building. + +**5. The import graph is declared once and checked.** `sampletones` is the top of the graph, and +the bootstrap tree is a root of its own under the standard-library rule. [Package layers](packages.md) +holds the graph and its enforcement. + +**6. Tests mirror the tree.** `tests/unit//` mirrors `src//` and +`tests/unit/scripts/` mirrors `scripts/`. Code and its tests move in one change. + +**7. One script per operation, the same on every system.** What differs between systems (the launcher's name and extension, the icon, the package manager, the compiler flags audio playback -needs) sits behind one `Platform` protocol with an implementation per system, chosen by a -factory from `platform.system()`. A script never branches on the operating system itself, and a -system the project does not build on is refused by name. +needs) sits behind one `Platform` protocol with an implementation per system, chosen by a factory +from `platform.system()`. A script never branches on the operating system itself, and a system the +project does not build on is refused by name. The Makefile is the developer's index, one line per +target: a target names the script that does the work and passes its flag. The two shell files at +the root, `install.sh` and `install.bat`, exist for the double-click path and call the same bundle +script. + +## The commands -**4. A script is a library with a thin face.** Pure functions assemble the commands and take the -decisions; `main` parses the arguments and wires in the real runner and the real environment. -Tests call the functions with a runner that records what it was asked to run, so the build is -verified without building. +`src/sampletones/` is the entry package. `dispatcher.py` builds one parser over the commands and +runs the one named; `commands/` holds one module per command and `commands/registry.py` lists +them. A command is a frozen `Command` (`sampletones_shared/command.py`): its name, one line of +help, the function adding its options to a parser, and the function running it over the parsed +arguments. Each command turns its arguments into a frozen record, field by field, before it works. + +| Command | What it does | +|---|---| +| `run [--config FILE]` | Starts the application | +| `open PATH [--config FILE]` | Starts the application with a `.stp` project, a `.stn` reconstruction or an `.ins` library loaded; a recording is refused with the `convert` line to run instead | +| `convert SOURCE... [-o FILE] [--config FILE] [--channels LIST \| --stems FILE]` | Reconstructs recordings into one `.stn` file, or every recording under one directory file by file. `--stems` names a JSON file holding the setup the `.stn` record stores, its entries paired with the sources in order; the pairing is printed before the run | +| `library [--config FILE]` | Generates the instruction library for a configuration | +| `self-check` | Verifies that the build's imports, bundled resources and configuration files are usable | -**5. The Makefile is the developer's index, one line per target.** A target names the script -that does the work and passes its flag. The two shell files at the root, `install.sh` and -`install.bat`, exist for the double-click path and call the same bundle script. +`--version` and `--help` are flags of the entry itself. The headless runs behind `convert` and +`library` live in `sampletones_core/headless/`, where the calibration reuses them. Developer +commands join the registry as the tools they run move into their own package. ## The bootstrap scripts @@ -72,6 +111,8 @@ the make target that names each. The checks are also pre-commit hooks; | Concern | Owner | |---|---| +| Which commands the entry offers | `src/sampletones/commands/registry.py` | +| What a command is | `src/sampletones_shared/command.py` | | What a bootstrap script may import | `sampletones_config/boundaries/standalone.yaml` | | What differs between systems | `scripts/bootstrap/platforms/` | | Where a build installs | `scripts/bootstrap/venv_build.py` | diff --git a/docs/formats/instruction-libraries.md b/docs/formats/instruction-libraries.md index bfa6fdd01..3d2629163 100644 --- a/docs/formats/instruction-libraries.md +++ b/docs/formats/instruction-libraries.md @@ -8,7 +8,7 @@ it is built, see [Instruction library](../concepts/instruction-library.md); this page documents the file. Libraries are generated from the _Instructions_ tab (or with -`sampletones --generate`) and stored in the documents folder. +`sampletones library`) and stored in the documents folder. ## Contents diff --git a/docs/formats/reconstructions.md b/docs/formats/reconstructions.md index 489991cee..ca1f9bcb3 100644 --- a/docs/formats/reconstructions.md +++ b/docs/formats/reconstructions.md @@ -59,6 +59,21 @@ to the player, which is the record a channel edited down to empty envelopes reac as well. A file naming a stream for the channels it plays alone reads as the whole four, with the rest coming back standing by. +The stems setup is also what `sampletones convert --stems` reads, written as JSON with the +same fields: one entry per recording, in the order the recordings are given, each naming the +channels it may occupy and the ones it bends; a hierarchy listing the stem ids by precedence +level; and a channel cap. Two recordings, the first on the pulses and the second on the rest: + +```json +{ + "entries": [ + {"id": 0, "settings": {"channels": ["pulse1", "pulse2"], "bends": ["pulse1", "pulse2"]}}, + {"id": 1, "settings": {"channels": ["triangle", "noise"], "bends": ["triangle"]}} + ], + "hierarchy": {"levels": [[0], [1]]} +} +``` + ## Detached reconstructions A reconstruction normally remembers the path to its source audio. Embedding one diff --git a/docs/guide/command-line.md b/docs/guide/command-line.md index 76a2d5912..22072399f 100644 --- a/docs/guide/command-line.md +++ b/docs/guide/command-line.md @@ -1,41 +1,41 @@ # Command line -You can run _SampleToNES_ from a terminal — to reconstruct without opening the -interface, to generate a library, or to open a file directly in the app. Run it -with no arguments to launch the GUI as usual. +You can run _SampleToNES_ from a terminal: to reconstruct without opening the interface, to +generate a library, or to open a file directly in the app. Every operation is a named command, +and `sampletones` alone starts the interface. -The command is `sampletones` when installed from source; a standalone build is the -executable you made (`./bin/sampletones` on Linux, `bin\sampletones.exe` on Windows). +The command is `sampletones` when installed from source; a standalone build is the executable you +made (`./bin/sampletones` on Linux, `bin\sampletones.exe` on Windows). + +## Commands + +| Command | Purpose | +| --- | --- | +| `sampletones`, `sampletones run` | start the interface | +| `sampletones open ` | start the interface with a `.stp` project, a `.stn` reconstruction or an `.ins` library loaded | +| `sampletones convert ...` | reconstruct recordings into a `.stn` file, or every recording under one folder | +| `sampletones library` | build the instruction library for a configuration, then exit | +| `sampletones self-check` | verify that this build's imports, bundled resources and configuration files are all usable, then exit | +| `sampletones --version` | print the version | + +Every command lists its options with `--help`. ## Common tasks -* **Launch the interface** — `sampletones` -* **Reconstruct a file** — `sampletones input.wav -o output.stn` -* **Reconstruct a folder** — `sampletones path/to/folder` reconstructs every audio - file inside it. +* **Reconstruct a file** — `sampletones convert input.wav -o output.stn` +* **Reconstruct a folder** — `sampletones convert path/to/folder` reconstructs every audio + file inside it, into the reconstructions folder your configuration names. * **Choose the channels** — add `--channels pulse1,pulse2` to reconstruct onto those two alone; without it a run uses pulse 1, triangle, and noise. -* **Open a file in the app** — `sampletones song.stp` opens the interface preloaded - with it; a `.stn` reconstruction or `.ins` library works the same way. -* **Use a specific configuration** — add `--config my-config.json`; otherwise your - saved configuration is used (`config.json`, or built-in defaults if you have not - saved one yet). -* **Generate a library and exit** — `sampletones --generate --config my-config.json` -* **Check the version** — `sampletones --version` -* **Check that a build works** — `sampletones --self-check` - -## Options - -| Option | Purpose | -| --- | --- | -| `path` | (positional) an audio file or folder to reconstruct, or a `.stn` / `.ins` / `.stp` file to open in the app. Omit it to launch the interface. | -| `--output`, `-o` | output path for a reconstruction | -| `--channels` | channels the reconstruction may use, comma separated (default: `pulse1,triangle,noise`) | -| `--config`, `-c` | path to a configuration `.json` (default: your saved `config.json`) | -| `--generate`, `-g` | build the instruction library for the configuration, then exit | -| `--version`, `-v` | print the version and exit | -| `--self-check` | verify that this build's imports, bundled resources, and configuration files are all usable, then exit | -| `--help`, `-h` | show the full option list | +* **Mix several recordings into one reconstruction** — `sampletones convert bass.wav lead.wav + --stems stems.json`. The stems file describes the same setup the interface's stems list + builds: one entry per recording, in order, each naming the channels it may occupy and the + ones it bends. The command prints which recording plays under which stem before it starts. + [Reconstructions](../formats/reconstructions.md) shows the file. +* **Use a specific configuration** — add `--config my-config.json` to `run`, `open`, `convert` + or `library`; otherwise your saved configuration is used (`config.json`, or built-in defaults + if you have not saved one yet). +* **Generate a library and exit** — `sampletones library --config my-config.json` GPU acceleration is selected at setup, not per run: `make setup` detects a supported NVIDIA driver and installs the matching build (`make setup GPU=0` forces the CPU diff --git a/scripts/bundle.py b/scripts/bundle.py index 6941f2404..fa976c243 100644 --- a/scripts/bundle.py +++ b/scripts/bundle.py @@ -19,7 +19,7 @@ ENTRY: Final[str] = "src/sampletones/__main__.py" RELEASE_HOOK: Final[str] = "scripts/runtime_hooks/release_environment.py" ICONS_SCRIPT: Final[str] = "scripts/assets/icons.py" -SELF_CHECK: Final[str] = "--self-check" +SELF_CHECK: Final[str] = "self-check" BUILD_EXTRA: Final[str] = "build" GPU_EXTRA: Final[str] = "gpu" GROUPS: Final[Tuple[str, ...]] = ("assets",) diff --git a/src/sampletones/__main__.py b/src/sampletones/__main__.py index 234e5e91d..c17e0f4ba 100644 --- a/src/sampletones/__main__.py +++ b/src/sampletones/__main__.py @@ -1,224 +1,15 @@ -import argparse import multiprocessing -from argparse import RawTextHelpFormatter -from dataclasses import dataclass -from pathlib import Path -from typing import TYPE_CHECKING, Optional +import sys -from sampletones_shared.paths.extensions import EXT_FILES_AUDIO +from sampletones.commands.registry import COMMANDS +from sampletones.dispatcher import dispatch -if TYPE_CHECKING: - from typing import List - from sampletones_core.configs import Config - from sampletones_core.constants.enums import ChannelName - -HELP_PATH = """Path to either: - * audio file path/directory to reconstruct - * reconstruction .stn file to load a reconstruction - * instructions library .ins file to load a library""" - -HELP_OUTPUT = """Output path for reconstruction.""" - -HELP_CHANNELS = """Channels the reconstruction may use, comma separated - (pulse1, pulse2, triangle, noise; default: pulse1,triangle,noise)""" - -HELP_CONFIG = """Path to a configuration .json file - (if not provided, default configuration will be used)""" - -HELP_GENERATE = """Generate library data for given configuration - (using default one if not provided)""" - -HELP_HELP = """Show this help message and exit""" - -HELP_VERSION = "Show application version information" - -HELP_SELF_CHECK = """Verify that the imports, bundled resources - and configuration files this build ships are all usable""" - - -@dataclass(frozen=True) -class ProgramArguments: - path: Optional[Path] = None - output: Optional[Path] = None - config: Optional[Path] = None - channels: Optional[str] = None - - help: bool = False - version: bool = False - generate: bool = False - self_check: bool = False - - -def _load_config(config_path: Optional[Path]) -> "Config": - from sampletones_core.configs import Config - - return Config.load(config_path) if config_path else Config.default() - - -def _channels(stated: Optional[str]) -> "List[ChannelName]": - """The channels a run hands out: the ones named on the command line, or the usual three. - - Raises: - SystemExit: If a name is not one of the channels the hardware has. - """ - from sampletones_core.constants.enums import DEFAULT_CHANNELS, ChannelName - - if stated is None: - return list(DEFAULT_CHANNELS) - - names = [name.strip() for name in stated.split(",") if name.strip()] - try: - return [ChannelName(name) for name in names] - except ValueError as exception: - raise SystemExit(f"Unknown channel in --channels: {exception}") from exception - - -def main() -> None: - parser = argparse.ArgumentParser( - prog="SampleToNES", - add_help=False, - formatter_class=RawTextHelpFormatter, - ) - parser.add_argument( - "path", - nargs="?", - default=None, - help=HELP_PATH, - ) - parser.add_argument( - "--output", - "-o", - type=Path, - default=None, - help=HELP_OUTPUT, - ) - parser.add_argument( - "--channels", - type=str, - default=None, - help=HELP_CHANNELS, - ) - parser.add_argument( - "--config", - "-c", - type=Path, - default=None, - help=HELP_CONFIG, - ) - parser.add_argument( - "--generate", - "-g", - action="store_true", - help=HELP_GENERATE, - ) - parser.add_argument( - "--help", - "-h", - action="store_true", - help=HELP_HELP, - ) - parser.add_argument( - "--version", - "-v", - action="store_true", - help=HELP_VERSION, - ) - parser.add_argument( - "--self-check", - action="store_true", - help=HELP_SELF_CHECK, - ) - args: ProgramArguments = ProgramArguments(**vars(parser.parse_args())) - - if args.help: - parser.print_help() - return None - - if args.version: - from sampletones_shared.application import ( - SAMPLETONES_NAME_VERSION, - ) - - return print(SAMPLETONES_NAME_VERSION) - - if args.self_check: - from sampletones.self_check import run_self_check - - raise SystemExit(run_self_check()) - - from sampletones_shared.array import report_array_backend - - report_array_backend() - - config_path = Path(args.config) if args.config else None - output_path = Path(args.output) if args.output else None - - from sampletones_shared.paths.extensions import ( - EXT_FILE_LIBRARY, - EXT_FILE_PROJECT, - EXT_FILE_RECONSTRUCTION, - ) - - if args.generate: - from sampletones_core.scripts.library import generate_library - - config = _load_config(config_path) - return generate_library(config) - - project_path: Optional[Path] = None - library_path: Optional[Path] = None - reconstruction_path: Optional[Path] = None - - if args.path: - path = Path(args.path) - if path.is_file(): - suffix = path.suffix.lower() - if suffix == EXT_FILE_PROJECT: - project_path = path - - elif suffix == EXT_FILE_RECONSTRUCTION: - reconstruction_path = path - - elif suffix == EXT_FILE_LIBRARY: - library_path = path - - elif suffix in EXT_FILES_AUDIO: - from sampletones_core.scripts.reconstruction import ( - reconstruct_file, - ) - - config = _load_config(config_path) - return reconstruct_file(path, config, _channels(args.channels), output_path) - - else: - raise RuntimeError( - f"Unsupported file extension, only audio ({', '.join(EXT_FILES_AUDIO)})," - f"{EXT_FILE_RECONSTRUCTION} reconstruction, " - f"and {EXT_FILE_LIBRARY} library files are supported." - ) - - elif path.is_dir(): - from sampletones_core.scripts.reconstruction import ( - reconstruct_directory, - ) - - config = _load_config(config_path) - return reconstruct_directory(path, config, _channels(args.channels)) - - else: - raise RuntimeError("Unsupported path type or file extension.") - - from sampletones.run import run_application - - return run_application( - config_path, - library_path=library_path, - reconstruction_path=reconstruction_path, - project_path=project_path, - ) +def main() -> int: + """Runs the command named on the command line, which is what the ``sampletones`` entry does.""" + return dispatch(COMMANDS, sys.argv[1:]) if __name__ == "__main__": multiprocessing.freeze_support() - main() + raise SystemExit(main()) diff --git a/src/sampletones_core/scripts/__init__.py b/src/sampletones/commands/__init__.py similarity index 100% rename from src/sampletones_core/scripts/__init__.py rename to src/sampletones/commands/__init__.py diff --git a/src/sampletones/commands/convert.py b/src/sampletones/commands/convert.py new file mode 100644 index 000000000..0db922895 --- /dev/null +++ b/src/sampletones/commands/convert.py @@ -0,0 +1,81 @@ +from argparse import ArgumentParser, Namespace +from dataclasses import dataclass +from pathlib import Path +from typing import Final, Optional, Tuple + +from sampletones.commands.options import add_config_option +from sampletones_shared.command import Command + +NAME: Final[str] = "convert" +HELP: Final[str] = "reconstruct recordings into a .stn file" +SOURCES_HELP: Final[str] = "recordings mixed into one reconstruction, or one directory converted file by file" +OUTPUT_HELP: Final[str] = ( + "where the reconstruction is written; without it, the configuration's reconstructions directory" +) +CHANNELS_HELP: Final[str] = ( + "channels the reconstruction may use, comma separated: pulse1, pulse2, triangle, noise; " + "without it pulse1, triangle and noise" +) +STEMS_HELP: Final[str] = "a JSON file holding a stems setup; its entries pair with the sources in order" + + +@dataclass(frozen=True) +class ConvertArguments: + """What a conversion is given: the sources, where the result goes, and how the channels are handed out.""" + + sources: Tuple[Path, ...] + output: Optional[Path] + config: Optional[Path] + channels: Optional[str] + stems: Optional[Path] + + +def configure(parser: ArgumentParser) -> None: + parser.add_argument("sources", nargs="+", type=Path, help=SOURCES_HELP) + parser.add_argument("--output", "-o", type=Path, default=None, help=OUTPUT_HELP) + add_config_option(parser) + setup = parser.add_mutually_exclusive_group() + setup.add_argument("--channels", type=str, default=None, help=CHANNELS_HELP) + setup.add_argument("--stems", type=Path, default=None, help=STEMS_HELP) + + +def run(arguments: Namespace) -> int: + """Reconstructs the sources under the stems setup the options describe. + + Raises: + SystemExit: If a channel is unknown, the stems file is no setup, or the sources and the + setup pair up wrong. + """ + given = ConvertArguments( + sources=tuple(arguments.sources), + output=arguments.output, + config=arguments.config, + channels=arguments.channels, + stems=arguments.stems, + ) + + from sampletones_core.headless.config import load_config + from sampletones_core.headless.conversion import ( + ConversionRequest, + channels_named, + classic_setup, + load_stems, + reconstruct, + ) + from sampletones_shared.array import report_array_backend + + try: + stems = load_stems(given.stems) if given.stems is not None else classic_setup(channels_named(given.channels)) + request = ConversionRequest(sources=given.sources, stems=stems, output_path=given.output) + except (TypeError, ValueError) as error: + raise SystemExit(str(error)) from error + + for line in request.pairing(): + print(line) + + report_array_backend() + reconstruct(request, load_config(given.config)) + return 0 + + +CONVERT: Final[Command] = Command(name=NAME, help=HELP, configure=configure, run=run) diff --git a/src/sampletones/commands/library.py b/src/sampletones/commands/library.py new file mode 100644 index 000000000..bed80c66c --- /dev/null +++ b/src/sampletones/commands/library.py @@ -0,0 +1,37 @@ +from argparse import ArgumentParser, Namespace +from dataclasses import dataclass +from pathlib import Path +from typing import Final, Optional + +from sampletones.commands.options import add_config_option +from sampletones_shared.command import Command + +NAME: Final[str] = "library" +HELP: Final[str] = "generate the instruction library for a configuration" + + +@dataclass(frozen=True) +class LibraryArguments: + """What a library generation is given: the configuration the library is built for, if any.""" + + config: Optional[Path] + + +def configure(parser: ArgumentParser) -> None: + add_config_option(parser) + + +def run(arguments: Namespace) -> int: + """Generates the instruction library for the configuration.""" + given = LibraryArguments(config=arguments.config) + + from sampletones_core.headless.config import load_config + from sampletones_core.headless.library import generate_library + from sampletones_shared.array import report_array_backend + + report_array_backend() + generate_library(load_config(given.config)) + return 0 + + +LIBRARY: Final[Command] = Command(name=NAME, help=HELP, configure=configure, run=run) diff --git a/src/sampletones/commands/open.py b/src/sampletones/commands/open.py new file mode 100644 index 000000000..9bb7528ad --- /dev/null +++ b/src/sampletones/commands/open.py @@ -0,0 +1,68 @@ +from argparse import ArgumentParser, Namespace +from dataclasses import dataclass +from pathlib import Path +from typing import Final, Optional + +from sampletones.commands.options import add_config_option +from sampletones_shared.command import Command + +NAME: Final[str] = "open" +HELP: Final[str] = "start the application with a project, reconstruction or library loaded" +PATH_HELP: Final[str] = "a .stp project, a .stn reconstruction or an .ins library" + + +@dataclass(frozen=True) +class OpenArguments: + """What an opening run is given: the file to load and the configuration to start with.""" + + path: Path + config: Optional[Path] + + +def configure(parser: ArgumentParser) -> None: + parser.add_argument("path", type=Path, help=PATH_HELP) + add_config_option(parser) + + +def run(arguments: Namespace) -> int: + """Starts the application with the file loaded. + + Raises: + SystemExit: If the path names no file, a recording, or a file of another kind. + """ + given = OpenArguments(path=arguments.path, config=arguments.config) + + from sampletones_shared.paths.extensions import ( + EXT_FILE_LIBRARY, + EXT_FILE_PROJECT, + EXT_FILE_RECONSTRUCTION, + EXT_FILES_AUDIO, + ) + + if not given.path.is_file(): + raise SystemExit(f"No file at {given.path}.") + + suffix = given.path.suffix.lower() + if suffix in EXT_FILES_AUDIO: + raise SystemExit(f"{given.path} is a recording; run: sampletones convert {given.path}") + + if suffix not in (EXT_FILE_PROJECT, EXT_FILE_RECONSTRUCTION, EXT_FILE_LIBRARY): + raise SystemExit( + f"{given.path} is neither a {EXT_FILE_PROJECT} project, a {EXT_FILE_RECONSTRUCTION} " + f"reconstruction nor an {EXT_FILE_LIBRARY} library." + ) + + from sampletones.run import run_application + from sampletones_shared.array import report_array_backend + + report_array_backend() + run_application( + given.config, + project_path=given.path if suffix == EXT_FILE_PROJECT else None, + reconstruction_path=given.path if suffix == EXT_FILE_RECONSTRUCTION else None, + library_path=given.path if suffix == EXT_FILE_LIBRARY else None, + ) + return 0 + + +OPEN: Final[Command] = Command(name=NAME, help=HELP, configure=configure, run=run) diff --git a/src/sampletones/commands/options.py b/src/sampletones/commands/options.py new file mode 100644 index 000000000..9de454668 --- /dev/null +++ b/src/sampletones/commands/options.py @@ -0,0 +1,10 @@ +from argparse import ArgumentParser +from pathlib import Path +from typing import Final + +CONFIG_HELP: Final[str] = "a configuration .json file; without it, the saved configuration or the built-in defaults" + + +def add_config_option(parser: ArgumentParser) -> None: + """Adds the ``--config`` option the commands reading a configuration share.""" + parser.add_argument("--config", "-c", type=Path, default=None, help=CONFIG_HELP) diff --git a/src/sampletones/commands/registry.py b/src/sampletones/commands/registry.py new file mode 100644 index 000000000..135d954ae --- /dev/null +++ b/src/sampletones/commands/registry.py @@ -0,0 +1,11 @@ +from typing import Final, Tuple + +from sampletones.commands.convert import CONVERT +from sampletones.commands.library import LIBRARY +from sampletones.commands.open import OPEN +from sampletones.commands.run import RUN +from sampletones.commands.self_check import SELF_CHECK +from sampletones_shared.command import Command + +USER_COMMANDS: Final[Tuple[Command, ...]] = (RUN, OPEN, CONVERT, LIBRARY, SELF_CHECK) +COMMANDS: Final[Tuple[Command, ...]] = USER_COMMANDS diff --git a/src/sampletones/commands/run.py b/src/sampletones/commands/run.py new file mode 100644 index 000000000..68439e1e8 --- /dev/null +++ b/src/sampletones/commands/run.py @@ -0,0 +1,36 @@ +from argparse import ArgumentParser, Namespace +from dataclasses import dataclass +from pathlib import Path +from typing import Final, Optional + +from sampletones.commands.options import add_config_option +from sampletones_shared.command import Command + +NAME: Final[str] = "run" +HELP: Final[str] = "start the application" + + +@dataclass(frozen=True) +class RunArguments: + """What a run of the application is given: the configuration it starts with, if any.""" + + config: Optional[Path] + + +def configure(parser: ArgumentParser) -> None: + add_config_option(parser) + + +def run(arguments: Namespace) -> int: + """Starts the application.""" + given = RunArguments(config=arguments.config) + + from sampletones.run import run_application + from sampletones_shared.array import report_array_backend + + report_array_backend() + run_application(given.config) + return 0 + + +RUN: Final[Command] = Command(name=NAME, help=HELP, configure=configure, run=run) diff --git a/src/sampletones/commands/self_check.py b/src/sampletones/commands/self_check.py new file mode 100644 index 000000000..df4e3bc95 --- /dev/null +++ b/src/sampletones/commands/self_check.py @@ -0,0 +1,23 @@ +from argparse import ArgumentParser, Namespace +from typing import Final + +from sampletones_shared.command import Command + +NAME: Final[str] = "self-check" +HELP: Final[str] = "verify that this build's imports, bundled resources and configuration files are usable" + + +def configure(parser: ArgumentParser) -> None: + del parser + + +def run(arguments: Namespace) -> int: + """Runs every startup check and answers with the status a packaged build is held to.""" + del arguments + + from sampletones.self_check import run_self_check + + return run_self_check() + + +SELF_CHECK: Final[Command] = Command(name=NAME, help=HELP, configure=configure, run=run) diff --git a/src/sampletones/dispatcher.py b/src/sampletones/dispatcher.py new file mode 100644 index 000000000..9e1d315ee --- /dev/null +++ b/src/sampletones/dispatcher.py @@ -0,0 +1,58 @@ +from argparse import ArgumentParser +from typing import Final, Sequence + +from sampletones_shared.application import SAMPLETONES_NAME_VERSION +from sampletones_shared.command import Command + +PROGRAM: Final[str] = "sampletones" +DESCRIPTION: Final[str] = "SampleToNES turns recordings into NES instruments and plays them back." +DEFAULT_COMMAND: Final[str] = "run" +COMMAND_FIELD: Final[str] = "command" +COMMAND_METAVAR: Final[str] = "" + + +def build_parser(commands: Sequence[Command]) -> ArgumentParser: + """The parser over ``commands``: one subcommand each, with the version and the help as flags. + + Args: + commands: The commands on offer, listed in the help in this order. + + Returns: + ArgumentParser: The parser the entry runs. + + Raises: + ValueError: If two commands share a name. + """ + names = [command.name for command in commands] + repeated = sorted({name for name in names if names.count(name) > 1}) + if repeated: + raise ValueError(f"the commands share a name: {', '.join(repeated)}") + + parser = ArgumentParser( + prog=PROGRAM, + description=DESCRIPTION, + epilog=f"Run '{PROGRAM} {COMMAND_METAVAR} --help' for a command's options.", + ) + parser.add_argument("--version", "-v", action="version", version=SAMPLETONES_NAME_VERSION) + subparsers = parser.add_subparsers(dest=COMMAND_FIELD, metavar=COMMAND_METAVAR, required=True) + for command in commands: + subparser = subparsers.add_parser(command.name, help=command.help, description=command.help) + command.configure(subparser) + + return parser + + +def dispatch(commands: Sequence[Command], argv: Sequence[str]) -> int: + """Runs the command ``argv`` names, and the default command when it names none. + + Args: + commands: The commands on offer. + argv: The arguments after the program name. + + Returns: + int: The exit status the command answers with. + """ + parser = build_parser(commands) + arguments = parser.parse_args(list(argv) or [DEFAULT_COMMAND]) + command = next(command for command in commands if command.name == arguments.command) + return command.run(arguments) diff --git a/src/sampletones_core/calibration/runner.py b/src/sampletones_core/calibration/runner.py index c1388eca6..46bccb2c1 100644 --- a/src/sampletones_core/calibration/runner.py +++ b/src/sampletones_core/calibration/runner.py @@ -11,9 +11,9 @@ SpectrumMethod, ) from sampletones_core.fft import Window +from sampletones_core.headless.library import generate_library from sampletones_core.library import InstructionLibrary from sampletones_core.reconstructions import Reconstructor -from sampletones_core.scripts.library import generate_library from sampletones_shared.logger import logger from .corpus.item import CorpusItem diff --git a/src/sampletones_core/headless/__init__.py b/src/sampletones_core/headless/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/sampletones_core/headless/config.py b/src/sampletones_core/headless/config.py new file mode 100644 index 000000000..e1d807512 --- /dev/null +++ b/src/sampletones_core/headless/config.py @@ -0,0 +1,9 @@ +from pathlib import Path +from typing import Optional + +from sampletones_core.configs import Config + + +def load_config(path: Optional[Path]) -> Config: + """The configuration a headless run uses: the file named, or the saved one with the defaults behind it.""" + return Config.load(path) if path is not None else Config.default() diff --git a/src/sampletones_core/headless/conversion.py b/src/sampletones_core/headless/conversion.py new file mode 100644 index 000000000..e0c2bd07e --- /dev/null +++ b/src/sampletones_core/headless/conversion.py @@ -0,0 +1,293 @@ +from pathlib import Path +from typing import Final, List, Optional, Self, Sequence, Tuple + +from pydantic import BaseModel, ConfigDict, Field, model_validator +from tqdm import tqdm + +from sampletones_core.configs import Config +from sampletones_core.constants.enums import ( + DEFAULT_CHANNELS, + ChannelName, + bending_channels, + ordered_channels, +) +from sampletones_core.headless.library import generate_library +from sampletones_core.library import InstructionLibrary +from sampletones_core.parallelization import TaskProgress, TaskStatus +from sampletones_core.reconstructions import Reconstructor +from sampletones_core.reconstructions.converter import ( + ConversionJob, + DirectoryConversion, + ReconstructionConverter, + reconstruct_job, +) +from sampletones_core.reconstructions.converter.paths import group_output_path +from sampletones_core.reconstructions.progress import ReconstructionProgress +from sampletones_core.reconstructions.reconstructor.stems.configs.config import StemsConfig +from sampletones_core.reconstructions.reconstructor.stems.configs.entry import StemEntry +from sampletones_shared.logger import logger, null_logger +from sampletones_shared.utils.serialization import load_json + +BAR_STEPS: Final[int] = 1000 +CHANNEL_SEPARATOR: Final[str] = "," + + +def channels_named(stated: Optional[str]) -> List[ChannelName]: + """The channels a run hands out: the ones named, comma separated, or the usual three. + + Raises: + ValueError: If a name is none of the channels the hardware has. + """ + if stated is None: + return list(DEFAULT_CHANNELS) + + names = [name.strip() for name in stated.split(CHANNEL_SEPARATOR) if name.strip()] + channels: List[ChannelName] = [] + for name in names: + try: + channels.append(ChannelName(name)) + except ValueError as error: + known = ", ".join(channel.value for channel in ChannelName) + raise ValueError(f"Unknown channel {name!r}; the channels are {known}.") from error + + return channels + + +def classic_setup(channels: Sequence[ChannelName]) -> StemsConfig: + """The setup a single-source conversion runs under: one stem over the channels it was given.""" + ordered = ordered_channels(frozenset(channels)) + return StemsConfig.single_entry(ordered, bending_channels(ordered)) + + +def load_stems(path: Path) -> StemsConfig: + """The stems setup a JSON file holds, validated the way the ``.stn`` record is. + + Raises: + TypeError: If the file holds anything other than a mapping. + ValueError: If the mapping is no stems setup. + """ + loaded = load_json(path) + if not isinstance(loaded, dict): + raise TypeError(f"Stems file {path} must hold a mapping, got {type(loaded).__name__}") + + return StemsConfig.model_validate(loaded) + + +def describe_stem(entry: StemEntry) -> str: + """One stem as the pairing names it: its id, the channels it may occupy and the ones it bends.""" + channels = ", ".join(channel.value for channel in entry.settings.channels) + bends = ", ".join(channel.value for channel in entry.settings.bends) + bending = f", bending {bends}" if bends else "" + return f"stem {entry.id} on {channels}{bending}" + + +class ConversionRequest(BaseModel): + """What a headless conversion is asked for: the sources, the stems setup and where the result goes. + + The i-th source plays under the i-th entry of the setup. One directory stands for every + recording under it, each converted alone under the setup's one stem, into the + configuration's reconstructions directory. + + Attributes: + sources: The recordings, or one directory of them. + stems: The setup handing the channels out. + output_path: The file the reconstruction of the recordings is written to, or ``None`` + for the configuration's own directory. + """ + + model_config = ConfigDict(frozen=True) + + sources: Tuple[Path, ...] = Field(min_length=1) + stems: StemsConfig + output_path: Optional[Path] + + @property + def directory(self) -> Optional[Path]: + """The one directory the request converts file by file, or ``None`` for recordings.""" + if len(self.sources) == 1 and self.sources[0].is_dir(): + return self.sources[0] + + return None + + @model_validator(mode="after") + def _sources_are_recordings_or_one_directory(self) -> Self: + """Raises: + ValueError: If a directory stands among several sources. + """ + if self.directory is None and any(source.is_dir() for source in self.sources): + raise ValueError("Sources are recordings, or one directory alone.") + + return self + + @model_validator(mode="after") + def _entries_pair_with_sources(self) -> Self: + """Raises: + ValueError: If the setup holds a different number of stems than there are sources. + """ + entries = len(self.stems.entries) + if self.directory is not None and entries != 1: + raise ValueError(f"A directory is converted file by file under one stem; the setup holds {entries}.") + + if self.directory is None and entries != len(self.sources): + raise ValueError( + f"{len(self.sources)} sources for {entries} stems; a setup pairs one stem with each source, in order." + ) + + return self + + @model_validator(mode="after") + def _output_names_the_one_file(self) -> Self: + """Raises: + ValueError: If an output path is given for a directory, whose reconstructions land in + the configuration's directory. + """ + if self.directory is not None and self.output_path is not None: + raise ValueError( + "A directory's reconstructions land in the configuration's reconstructions directory; " + "an output path names the one file recordings are mixed into." + ) + + return self + + def pairing(self) -> List[str]: + """One line per source naming the stem it plays under, in the order they pair.""" + directory = self.directory + if directory is not None: + return [f"{directory.name}/: every recording under {describe_stem(self.stems.entries[0])}"] + + return [f"{source.name}: {describe_stem(entry)}" for source, entry in zip(self.sources, self.stems.entries)] + + +def reconstruct(request: ConversionRequest, config: Config) -> None: + """Builds what the request asks for: one reconstruction of the recordings, or one per file of the directory.""" + directory = request.directory + if directory is not None: + reconstruct_directory(directory, config, request.stems) + return + + reconstruct_sources(request.sources, config, request.stems, request.output_path) + + +def reconstruct_sources( + sources: Tuple[Path, ...], + config: Config, + stems: StemsConfig, + output_path: Optional[Path], +) -> None: + """Mixes the recordings into one reconstruction and writes it, showing the progress as a bar. + + A file already standing at the output path is kept, and the run says so. + + Args: + sources: The recordings, one per stem of the setup. + config: The configuration selecting the library and the matching settings. + stems: The setup handing the channels out. + output_path: The file written, or ``None`` for the configuration's own directory. + """ + if output_path is None: + output_path = group_output_path(config, sources, stems.covered_channels) + + if output_path.exists(): + logger.info(f"Reconstruction {output_path} exists, skipping") + return + + names = ", ".join(source.name for source in sources) + logger.info(f"Starting reconstruction of {names}") + job = ConversionJob(sources=sources, stems=stems, output_path=output_path) + progress_bar = tqdm(total=BAR_STEPS, desc=f"Reconstructing {output_path.stem}", unit="step") + + def on_progress(progress: ReconstructionProgress) -> bool: + progress_bar.set_postfix_str(progress.stage) + progress_bar.update(round(progress.fraction * BAR_STEPS) - progress_bar.n) + return True + + try: + reconstruct_job((Reconstructor(config, stems.covered_channels), job, on_progress)) + finally: + progress_bar.close() + + logger.info(f"Reconstruction file saved to {output_path}") + + +def reconstruct_directory( + directory: Path, + config: Config, + stems: StemsConfig, +) -> None: + """Reconstructs every recording under the directory, each alone under the setup's stem. + + The library the configuration names is generated first where it is missing. The results + mirror the directory's tree inside the configuration's reconstructions directory. + + Args: + directory: The directory of recordings. + config: The configuration selecting the library and the matching settings. + stems: The one-stem setup every recording is converted under. + """ + library = InstructionLibrary.from_config(config) + if not library.exists(config): + logger.warning("Library does not exist for the given configuration, generating a new library") + generate_library(config) + + progress_bar = tqdm(total=0, desc=f"Reconstructing {directory.name}", unit="file") + + def on_start() -> None: + progress_bar.disable = False + logger.info(f"Starting reconstruction for directory {directory}") + + def on_completed(written: Tuple[Path, ...]) -> None: + logger.info(f"Reconstructed {len(written)} files from {directory}") + progress_bar.close() + + def on_progress( + task_status: TaskStatus, + task_progress: TaskProgress, + ) -> None: + progress_bar.disable = False + total = task_progress.total + if total and total != progress_bar.total: + progress_bar.total = total + progress_bar.refresh() + + delta = int(task_progress.completed) - int(progress_bar.n) + if delta > 0: + progress_bar.update(delta) + + if task_progress.current_item: + progress_bar.set_description(f"{directory.name}: {task_progress.current_item}") + + if task_status in ( + TaskStatus.COMPLETED, + TaskStatus.CANCELED, + TaskStatus.FAILED, + ): + progress_bar.close() + + def on_canceled() -> None: + logger.info("Reconstruction canceled by user") + progress_bar.close() + + def on_error(_exception: Exception) -> None: + progress_bar.close() + + converter = ReconstructionConverter( + config, + DirectoryConversion(directory=directory, stems=stems), + logger=null_logger, + ) + + converter.set_callbacks( + on_start=on_start, + on_completed=on_completed, + on_progress=on_progress, + on_canceled=on_canceled, + on_error=on_error, + ) + + try: + converter.start() + converter.wait() + except KeyboardInterrupt: + logger.info("Reconstruction interrupted by user") + finally: + progress_bar.close() diff --git a/src/sampletones_core/scripts/library.py b/src/sampletones_core/headless/library.py similarity index 100% rename from src/sampletones_core/scripts/library.py rename to src/sampletones_core/headless/library.py diff --git a/src/sampletones_core/scripts/reconstruction.py b/src/sampletones_core/scripts/reconstruction.py deleted file mode 100644 index c61d73725..000000000 --- a/src/sampletones_core/scripts/reconstruction.py +++ /dev/null @@ -1,151 +0,0 @@ -from pathlib import Path -from typing import Final, Optional, Sequence, Tuple - -from tqdm import tqdm - -from sampletones_core.configs import Config -from sampletones_core.constants.enums import ( - ChannelName, - bending_channels, - ordered_channels, -) -from sampletones_core.library import InstructionLibrary -from sampletones_core.parallelization import TaskProgress, TaskStatus -from sampletones_core.reconstructions import Reconstructor -from sampletones_core.reconstructions.converter import ( - ConversionJob, - DirectoryConversion, - ReconstructionConverter, - reconstruct_job, -) -from sampletones_core.reconstructions.converter.paths import get_output_path -from sampletones_core.reconstructions.progress import ReconstructionProgress -from sampletones_core.reconstructions.reconstructor.stems.configs.config import StemsConfig -from sampletones_core.scripts.library import generate_library -from sampletones_shared.logger import logger, null_logger - -BAR_STEPS: Final[int] = 1000 - - -def reconstruct_file( - input_path: Path, - config: Config, - channels: Sequence[ChannelName], - output_path: Optional[Path] = None, -) -> None: - if output_path is None: - output_path = get_output_path(config, input_path, frozenset(channels)) - - if output_path.exists(): - logger.info(f"Reconstructing file {input_path} exists, skipping") - return - - if input_path.is_dir(): - raise IsADirectoryError(f"Expected a file path, got directory path: {input_path}") - - logger.info(f"Starting reconstruction for file {input_path}") - job = ConversionJob( - sources=(input_path,), - stems=_classic_setup(channels), - output_path=output_path, - ) - progress_bar = tqdm(total=BAR_STEPS, desc=f"Reconstructing {input_path.name}", unit="step") - - def on_progress(progress: ReconstructionProgress) -> bool: - progress_bar.set_postfix_str(progress.stage) - progress_bar.update(round(progress.fraction * BAR_STEPS) - progress_bar.n) - return True - - try: - reconstruct_job((Reconstructor(config, frozenset(channels)), job, on_progress)) - finally: - progress_bar.close() - - logger.info(f"Reconstruction file saved to {output_path}") - - -def reconstruct_directory( - input_path: Path, - config: Config, - channels: Sequence[ChannelName], - output_path: Optional[Path] = None, -) -> None: - if output_path is None: - output_path = get_output_path(config, input_path, frozenset(channels)) - - if not input_path.is_dir(): - raise NotADirectoryError(f"Expected a directory path, got file path: {input_path}") - - library = InstructionLibrary.from_config(config) - if not library.exists(config): - logger.warning("Library does not exist for the given configuration, generating a new library") - generate_library(config) - - progress_bar = tqdm(total=0, desc=f"Reconstructing {input_path.name}", unit="file") - - def on_start() -> None: - progress_bar.disable = False - logger.info(f"Starting reconstruction for directory {input_path}") - - def on_completed(_written: Tuple[Path, ...]) -> None: - logger.info(f"Reconstruction directory saved to {output_path}") - progress_bar.close() - - def on_progress( - task_status: TaskStatus, - task_progress: TaskProgress, - ) -> None: - progress_bar.disable = False - total = task_progress.total - if total and total != progress_bar.total: - progress_bar.total = total - progress_bar.refresh() - - delta = int(task_progress.completed) - int(progress_bar.n) - if delta > 0: - progress_bar.update(delta) - - if task_progress.current_item: - progress_bar.set_description(f"{input_path.name}: {task_progress.current_item}") - - if task_status in ( - TaskStatus.COMPLETED, - TaskStatus.CANCELED, - TaskStatus.FAILED, - ): - progress_bar.close() - - def on_canceled() -> None: - logger.info("Reconstruction canceled by user") - progress_bar.close() - - def on_error(_exception: Exception) -> None: - progress_bar.close() - - converter = ReconstructionConverter( - config, - DirectoryConversion(directory=input_path, stems=_classic_setup(channels)), - logger=null_logger, - ) - - converter.set_callbacks( - on_start=on_start, - on_completed=on_completed, - on_progress=on_progress, - on_canceled=on_canceled, - on_error=on_error, - ) - - try: - converter.start() - converter.wait() - except KeyboardInterrupt: - logger.info("Reconstruction interrupted by user") - finally: - progress_bar.close() - - -def _classic_setup(channels: Sequence[ChannelName]) -> StemsConfig: - """The setup a single-source conversion runs under: one stem over the channels it was given.""" - ordered = ordered_channels(frozenset(channels)) - return StemsConfig.single_entry(ordered, bending_channels(ordered)) diff --git a/src/sampletones_shared/command.py b/src/sampletones_shared/command.py new file mode 100644 index 000000000..a9490fc22 --- /dev/null +++ b/src/sampletones_shared/command.py @@ -0,0 +1,23 @@ +from argparse import ArgumentParser, Namespace +from dataclasses import dataclass +from typing import Callable + + +@dataclass(frozen=True) +class Command: + """One named operation the ``sampletones`` entry runs: its options and what it does. + + A command module declares one of these at import time and keeps its implementation behind + ``run``, so listing the commands loads nothing a command needs to do its work. + + Attributes: + name: The word that selects the command on the command line. + help: One line saying what the command does, shown in the command list. + configure: Adds the command's arguments to the parser it is given. + run: Performs the command over the parsed arguments and answers with the exit status. + """ + + name: str + help: str + configure: Callable[[ArgumentParser], None] + run: Callable[[Namespace], int] diff --git a/tests/suite/commands.py b/tests/suite/commands.py new file mode 100644 index 000000000..c10ba206a --- /dev/null +++ b/tests/suite/commands.py @@ -0,0 +1,26 @@ +from pathlib import Path +from typing import Dict, List, Optional + + +class RecordedApplication: + """A stand-in for the application launcher that records what each start was given.""" + + def __init__(self) -> None: + self.starts: List[Dict[str, Optional[Path]]] = [] + + def __call__( + self, + config_path: Optional[Path] = None, + *, + library_path: Optional[Path] = None, + reconstruction_path: Optional[Path] = None, + project_path: Optional[Path] = None, + ) -> None: + self.starts.append( + { + "config": config_path, + "library": library_path, + "reconstruction": reconstruction_path, + "project": project_path, + } + ) diff --git a/tests/unit/sampletones/commands/test_convert.py b/tests/unit/sampletones/commands/test_convert.py new file mode 100644 index 000000000..92078335c --- /dev/null +++ b/tests/unit/sampletones/commands/test_convert.py @@ -0,0 +1,126 @@ +import json +from pathlib import Path +from typing import List + +import pytest + +from sampletones.commands.registry import COMMANDS +from sampletones.dispatcher import dispatch +from sampletones_core.configs import Config +from sampletones_core.constants.enums import DEFAULT_CHANNELS, ChannelName +from sampletones_core.headless.conversion import ConversionRequest, classic_setup +from sampletones_core.reconstructions.reconstructor.stems.configs.config import StemsConfig +from sampletones_core.reconstructions.reconstructor.stems.configs.entry import StemEntry +from sampletones_core.reconstructions.reconstructor.stems.configs.hierarchy import StemsHierarchy +from sampletones_core.reconstructions.reconstructor.stems.configs.settings import StemSettings + +RECONSTRUCTION = "sampletones_core.headless.conversion.reconstruct" +LOADER = "sampletones_core.headless.config.load_config" + + +class RecordedReconstruction: + def __init__(self) -> None: + self.requests: List[ConversionRequest] = [] + + def __call__(self, request: ConversionRequest, config: Config) -> None: + del config + self.requests.append(request) + + +@pytest.fixture(name="reconstruction") +def reconstruction_fixture(monkeypatch: pytest.MonkeyPatch) -> RecordedReconstruction: + recorded = RecordedReconstruction() + monkeypatch.setattr(RECONSTRUCTION, recorded) + monkeypatch.setattr(LOADER, lambda path: Config()) + return recorded + + +def _recording(tmp_path: Path, name: str) -> Path: + path = tmp_path / name + path.write_bytes(b"") + return path + + +def _two_stems() -> StemsConfig: + return StemsConfig( + entries=[ + StemEntry( + id=0, + settings=StemSettings(channels=[ChannelName.PULSE1, ChannelName.PULSE2], bends=[ChannelName.PULSE1]), + ), + StemEntry( + id=1, + settings=StemSettings(channels=[ChannelName.TRIANGLE], bends=[ChannelName.TRIANGLE]), + ), + ], + hierarchy=StemsHierarchy(levels=[[0], [1]]), + ) + + +class TestConvert: + def test_the_channels_named_become_one_stem(self, reconstruction: RecordedReconstruction, tmp_path: Path) -> None: + source = _recording(tmp_path, "song.wav") + output = tmp_path / "song.stn" + + status = dispatch(COMMANDS, ["convert", str(source), "--channels", "pulse1,pulse2", "-o", str(output)]) + + assert status == 0 + request = reconstruction.requests[0] + assert request.sources == (source,) + assert request.stems == classic_setup([ChannelName.PULSE1, ChannelName.PULSE2]) + assert request.output_path == output + + def test_without_channels_the_usual_three_are_used( + self, + reconstruction: RecordedReconstruction, + tmp_path: Path, + ) -> None: + source = _recording(tmp_path, "song.wav") + + assert dispatch(COMMANDS, ["convert", str(source)]) == 0 + assert reconstruction.requests[0].stems == classic_setup(DEFAULT_CHANNELS) + assert reconstruction.requests[0].output_path is None + + def test_a_stems_file_pairs_its_entries_with_the_sources_in_order( + self, + reconstruction: RecordedReconstruction, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + ) -> None: + bass = _recording(tmp_path, "bass.wav") + lead = _recording(tmp_path, "lead.wav") + stems = _two_stems() + setup = tmp_path / "stems.json" + setup.write_text(json.dumps(stems.model_dump(mode="json")), encoding="utf-8") + + assert dispatch(COMMANDS, ["convert", str(bass), str(lead), "--stems", str(setup)]) == 0 + assert reconstruction.requests[0].stems == stems + printed = capsys.readouterr().out + assert "bass.wav: stem 0 on pulse1, pulse2, bending pulse1" in printed + assert "lead.wav: stem 1 on triangle, bending triangle" in printed + + def test_a_setup_pairing_wrong_is_refused(self, reconstruction: RecordedReconstruction, tmp_path: Path) -> None: + source = _recording(tmp_path, "song.wav") + setup = tmp_path / "stems.json" + setup.write_text(json.dumps(_two_stems().model_dump(mode="json")), encoding="utf-8") + + with pytest.raises(SystemExit, match="1 sources for 2 stems"): + dispatch(COMMANDS, ["convert", str(source), "--stems", str(setup)]) + + assert reconstruction.requests == [] + + def test_an_unknown_channel_is_refused(self, reconstruction: RecordedReconstruction, tmp_path: Path) -> None: + source = _recording(tmp_path, "song.wav") + + with pytest.raises(SystemExit, match="Unknown channel 'pulse3'"): + dispatch(COMMANDS, ["convert", str(source), "--channels", "pulse3"]) + + assert reconstruction.requests == [] + + def test_channels_and_stems_exclude_each_other(self, tmp_path: Path) -> None: + source = _recording(tmp_path, "song.wav") + + with pytest.raises(SystemExit) as leaving: + dispatch(COMMANDS, ["convert", str(source), "--channels", "pulse1", "--stems", "stems.json"]) + + assert leaving.value.code == 2 diff --git a/tests/unit/sampletones/commands/test_library.py b/tests/unit/sampletones/commands/test_library.py new file mode 100644 index 000000000..02fbd2088 --- /dev/null +++ b/tests/unit/sampletones/commands/test_library.py @@ -0,0 +1,29 @@ +from pathlib import Path +from typing import List + +import pytest + +from sampletones.commands.registry import COMMANDS +from sampletones.dispatcher import dispatch +from sampletones_core.configs import Config + +GENERATOR = "sampletones_core.headless.library.generate_library" +LOADER = "sampletones_core.headless.config.load_config" + + +class TestLibrary: + def test_the_library_is_generated_for_the_configuration_named(self, monkeypatch: pytest.MonkeyPatch) -> None: + generated: List[Config] = [] + loaded: List[Path] = [] + configuration = Config() + + def load_config(path: Path) -> Config: + loaded.append(path) + return configuration + + monkeypatch.setattr(GENERATOR, generated.append) + monkeypatch.setattr(LOADER, load_config) + + assert dispatch(COMMANDS, ["library", "--config", "custom.json"]) == 0 + assert loaded == [Path("custom.json")] + assert generated == [configuration] diff --git a/tests/unit/sampletones/commands/test_open.py b/tests/unit/sampletones/commands/test_open.py new file mode 100644 index 000000000..ddd373808 --- /dev/null +++ b/tests/unit/sampletones/commands/test_open.py @@ -0,0 +1,60 @@ +from pathlib import Path + +import pytest + +from sampletones.commands.registry import COMMANDS +from sampletones.dispatcher import dispatch +from tests.suite.commands import RecordedApplication + +LAUNCHER = "sampletones.run.run_application" + + +def _file(tmp_path: Path, name: str) -> Path: + path = tmp_path / name + path.write_bytes(b"") + return path + + +class TestOpen: + @pytest.mark.parametrize( + ("name", "field"), + [("song.stp", "project"), ("song.stn", "reconstruction"), ("library.ins", "library")], + ) + def test_a_file_is_loaded_by_its_kind( + self, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + name: str, + field: str, + ) -> None: + application = RecordedApplication() + monkeypatch.setattr(LAUNCHER, application) + path = _file(tmp_path, name) + + assert dispatch(COMMANDS, ["open", str(path), "--config", "custom.json"]) == 0 + start = application.starts[0] + assert start[field] == path + assert start["config"] == Path("custom.json") + assert [key for key, value in start.items() if value is None] == [ + key for key in ("library", "reconstruction", "project") if key != field + ] + + def test_a_recording_is_pointed_at_convert(self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + application = RecordedApplication() + monkeypatch.setattr(LAUNCHER, application) + path = _file(tmp_path, "song.wav") + + with pytest.raises(SystemExit, match=f"sampletones convert {path}"): + dispatch(COMMANDS, ["open", str(path)]) + + assert application.starts == [] + + def test_a_file_of_another_kind_is_refused(self, tmp_path: Path) -> None: + path = _file(tmp_path, "notes.txt") + + with pytest.raises(SystemExit, match="neither"): + dispatch(COMMANDS, ["open", str(path)]) + + def test_a_missing_file_is_refused(self, tmp_path: Path) -> None: + with pytest.raises(SystemExit, match="No file at"): + dispatch(COMMANDS, ["open", str(tmp_path / "absent.stp")]) diff --git a/tests/unit/sampletones/commands/test_registry.py b/tests/unit/sampletones/commands/test_registry.py new file mode 100644 index 000000000..c0d62dd95 --- /dev/null +++ b/tests/unit/sampletones/commands/test_registry.py @@ -0,0 +1,13 @@ +from sampletones.commands.registry import COMMANDS, USER_COMMANDS +from sampletones.dispatcher import DEFAULT_COMMAND, build_parser + + +class TestRegistry: + def test_the_user_commands_are_the_ones_the_guide_names(self) -> None: + assert [command.name for command in USER_COMMANDS] == ["run", "open", "convert", "library", "self-check"] + + def test_the_default_command_is_registered(self) -> None: + assert DEFAULT_COMMAND in {command.name for command in COMMANDS} + + def test_the_registry_builds_one_parser(self) -> None: + assert build_parser(COMMANDS).format_help() diff --git a/tests/unit/sampletones/commands/test_run.py b/tests/unit/sampletones/commands/test_run.py new file mode 100644 index 000000000..72a9f347d --- /dev/null +++ b/tests/unit/sampletones/commands/test_run.py @@ -0,0 +1,28 @@ +from pathlib import Path + +import pytest + +from sampletones.commands.registry import COMMANDS +from sampletones.dispatcher import dispatch +from tests.suite.commands import RecordedApplication + +LAUNCHER = "sampletones.run.run_application" + + +class TestRun: + def test_no_arguments_start_the_application_with_the_saved_configuration( + self, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + application = RecordedApplication() + monkeypatch.setattr(LAUNCHER, application) + + assert dispatch(COMMANDS, []) == 0 + assert application.starts == [{"config": None, "library": None, "reconstruction": None, "project": None}] + + def test_a_configuration_reaches_the_application(self, monkeypatch: pytest.MonkeyPatch) -> None: + application = RecordedApplication() + monkeypatch.setattr(LAUNCHER, application) + + assert dispatch(COMMANDS, ["run", "--config", "custom.json"]) == 0 + assert application.starts[0]["config"] == Path("custom.json") diff --git a/tests/unit/sampletones/commands/test_self_check.py b/tests/unit/sampletones/commands/test_self_check.py new file mode 100644 index 000000000..2748135e4 --- /dev/null +++ b/tests/unit/sampletones/commands/test_self_check.py @@ -0,0 +1,13 @@ +import pytest + +from sampletones.commands.registry import COMMANDS +from sampletones.dispatcher import dispatch + +CHECK = "sampletones.self_check.run_self_check" + + +class TestSelfCheck: + def test_the_status_is_the_checks_own(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(CHECK, lambda: 7) + + assert dispatch(COMMANDS, ["self-check"]) == 7 diff --git a/tests/unit/sampletones/test_dispatcher.py b/tests/unit/sampletones/test_dispatcher.py new file mode 100644 index 000000000..8b4e9d143 --- /dev/null +++ b/tests/unit/sampletones/test_dispatcher.py @@ -0,0 +1,64 @@ +from argparse import ArgumentParser, Namespace +from typing import List, Tuple + +import pytest + +from sampletones.dispatcher import DEFAULT_COMMAND, build_parser, dispatch +from sampletones_shared.application import SAMPLETONES_NAME_VERSION +from sampletones_shared.command import Command + + +def _command(name: str, calls: List[Tuple[str, bool]]) -> Command: + def configure(parser: ArgumentParser) -> None: + parser.add_argument("--flag", action="store_true") + + def run(arguments: Namespace) -> int: + calls.append((name, arguments.flag)) + return 3 + + return Command(name=name, help=f"{name} help", configure=configure, run=run) + + +class TestBuildParser: + def test_two_commands_sharing_a_name_are_refused(self) -> None: + calls: List[Tuple[str, bool]] = [] + + with pytest.raises(ValueError, match="share a name: twin"): + build_parser((_command("twin", calls), _command("twin", calls))) + + def test_the_version_is_a_flag_of_the_entry(self, capsys: pytest.CaptureFixture[str]) -> None: + parser = build_parser((_command(DEFAULT_COMMAND, []),)) + + with pytest.raises(SystemExit) as leaving: + parser.parse_args(["--version"]) + + assert leaving.value.code == 0 + assert SAMPLETONES_NAME_VERSION in capsys.readouterr().out + + def test_every_command_is_listed_with_its_help(self) -> None: + parser = build_parser((_command("first", []), _command("second", []))) + + listing = parser.format_help() + + assert "first help" in listing + assert "second help" in listing + + +class TestDispatch: + def test_no_arguments_run_the_default_command(self) -> None: + calls: List[Tuple[str, bool]] = [] + + assert dispatch((_command(DEFAULT_COMMAND, calls), _command("other", calls)), []) == 3 + assert calls == [(DEFAULT_COMMAND, False)] + + def test_a_named_command_runs_with_its_arguments(self) -> None: + calls: List[Tuple[str, bool]] = [] + + assert dispatch((_command(DEFAULT_COMMAND, calls), _command("other", calls)), ["other", "--flag"]) == 3 + assert calls == [("other", True)] + + def test_an_unknown_command_is_refused(self) -> None: + with pytest.raises(SystemExit) as leaving: + dispatch((_command(DEFAULT_COMMAND, []),), ["absent"]) + + assert leaving.value.code == 2 diff --git a/tests/unit/sampletones/test_self_check.py b/tests/unit/sampletones/test_self_check.py new file mode 100644 index 000000000..ac2476f7d --- /dev/null +++ b/tests/unit/sampletones/test_self_check.py @@ -0,0 +1,40 @@ +from unittest.mock import patch + +import pytest + +from sampletones.self_check import CHECKS, FAILURE_STATUS, SUCCESS_STATUS, run_self_check +from sampletones_shared.exceptions import FileDialogUnavailableError + +RESOURCES_MODULE = "sampletones_application.ui.resources.resources" +SELECTION_MODULE = "sampletones_application.utils.file_dialogs.selection" + + +class TestRunSelfCheck: + def test_source_checkout_passes(self, capsys: pytest.CaptureFixture[str]) -> None: + status = run_self_check() + output = capsys.readouterr().out + + assert status == SUCCESS_STATUS + for check in CHECKS: + assert check.name in output + + def test_missing_resource_fails(self, capsys: pytest.CaptureFixture[str]) -> None: + with patch(f"{RESOURCES_MODULE}.get_font_path", side_effect=FileNotFoundError("Resource not found")): + status = run_self_check() + + captured = capsys.readouterr() + + assert status == FAILURE_STATUS + assert "resources" in captured.err + assert "FileNotFoundError" in captured.err + assert "file dialog backend" not in captured.out + + def test_unavailable_dialog_backend_fails(self, capsys: pytest.CaptureFixture[str]) -> None: + with patch( + f"{SELECTION_MODULE}.select_file_dialog_backend", + side_effect=FileDialogUnavailableError("No file dialog backend is available."), + ): + status = run_self_check() + + assert status == FAILURE_STATUS + assert "file dialog backend" in capsys.readouterr().err diff --git a/tests/unit/sampletones_core/headless/test_config.py b/tests/unit/sampletones_core/headless/test_config.py new file mode 100644 index 000000000..f37588181 --- /dev/null +++ b/tests/unit/sampletones_core/headless/test_config.py @@ -0,0 +1,15 @@ +from pathlib import Path + +from sampletones_core.configs import Config +from sampletones_core.headless.config import load_config + + +class TestLoadConfig: + def test_a_file_named_is_read(self, tmp_path: Path) -> None: + path = tmp_path / "config.json" + path.write_text("{}", encoding="utf-8") + + assert load_config(path) == Config() + + def test_nothing_named_is_the_saved_configuration(self) -> None: + assert load_config(None) == Config.default() diff --git a/tests/unit/sampletones_core/headless/test_conversion.py b/tests/unit/sampletones_core/headless/test_conversion.py new file mode 100644 index 000000000..76e663b52 --- /dev/null +++ b/tests/unit/sampletones_core/headless/test_conversion.py @@ -0,0 +1,178 @@ +import json +from pathlib import Path +from typing import List, Tuple + +import pytest + +from sampletones_core.configs import Config +from sampletones_core.constants.enums import DEFAULT_CHANNELS, ChannelName +from sampletones_core.headless import conversion +from sampletones_core.headless.conversion import ( + ConversionRequest, + channels_named, + classic_setup, + describe_stem, + load_stems, + reconstruct, +) +from sampletones_core.reconstructions.reconstructor.stems.configs.config import StemsConfig +from sampletones_core.reconstructions.reconstructor.stems.configs.entry import StemEntry +from sampletones_core.reconstructions.reconstructor.stems.configs.hierarchy import StemsHierarchy +from sampletones_core.reconstructions.reconstructor.stems.configs.settings import StemSettings + + +def _recording(tmp_path: Path, name: str) -> Path: + path = tmp_path / name + path.write_bytes(b"") + return path + + +def _two_stems() -> StemsConfig: + return StemsConfig( + entries=[ + StemEntry( + id=0, + settings=StemSettings(channels=[ChannelName.PULSE1, ChannelName.PULSE2], bends=[ChannelName.PULSE1]), + ), + StemEntry( + id=1, + settings=StemSettings(channels=[ChannelName.NOISE], bends=[]), + ), + ], + hierarchy=StemsHierarchy(levels=[[0, 1]]), + ) + + +class TestChannelsNamed: + def test_nothing_named_is_the_usual_three(self) -> None: + assert channels_named(None) == list(DEFAULT_CHANNELS) + + def test_names_are_read_in_order_with_their_spaces_stripped(self) -> None: + assert channels_named("noise, pulse1") == [ChannelName.NOISE, ChannelName.PULSE1] + + def test_an_unknown_name_is_refused_with_the_known_ones(self) -> None: + with pytest.raises(ValueError, match="Unknown channel 'pulse3'; the channels are pulse1, pulse2"): + channels_named("pulse1,pulse3") + + +class TestClassicSetup: + def test_one_stem_holds_the_channels_in_order_and_bends_the_toned_ones(self) -> None: + setup = classic_setup([ChannelName.NOISE, ChannelName.PULSE1]) + + assert len(setup.entries) == 1 + assert setup.entries[0].settings.channels == [ChannelName.PULSE1, ChannelName.NOISE] + assert setup.entries[0].settings.bends == [ChannelName.PULSE1] + + +class TestLoadStems: + def test_a_setup_written_as_json_reads_back(self, tmp_path: Path) -> None: + stems = _two_stems() + path = tmp_path / "stems.json" + path.write_text(json.dumps(stems.model_dump(mode="json")), encoding="utf-8") + + assert load_stems(path) == stems + + def test_a_file_holding_no_mapping_is_refused(self, tmp_path: Path) -> None: + path = tmp_path / "stems.json" + path.write_text("[]", encoding="utf-8") + + with pytest.raises(TypeError, match="must hold a mapping"): + load_stems(path) + + def test_a_mapping_that_is_no_setup_is_refused(self, tmp_path: Path) -> None: + path = tmp_path / "stems.json" + path.write_text(json.dumps({"entries": [{"id": 0, "settings": {"channels": ["pulse9"], "bends": []}}]})) + + with pytest.raises(ValueError): + load_stems(path) + + +class TestDescribeStem: + def test_a_stem_names_its_channels_and_its_bends(self) -> None: + first, second = _two_stems().entries + + assert describe_stem(first) == "stem 0 on pulse1, pulse2, bending pulse1" + assert describe_stem(second) == "stem 1 on noise" + + +class TestConversionRequest: + def test_recordings_pair_with_the_entries_in_order(self, tmp_path: Path) -> None: + bass = _recording(tmp_path, "bass.wav") + drums = _recording(tmp_path, "drums.wav") + + request = ConversionRequest(sources=(bass, drums), stems=_two_stems(), output_path=None) + + assert request.directory is None + assert request.pairing() == [ + "bass.wav: stem 0 on pulse1, pulse2, bending pulse1", + "drums.wav: stem 1 on noise", + ] + + def test_a_directory_is_converted_file_by_file_under_one_stem(self, tmp_path: Path) -> None: + request = ConversionRequest(sources=(tmp_path,), stems=classic_setup(DEFAULT_CHANNELS), output_path=None) + + assert request.directory == tmp_path + assert request.pairing() == [ + f"{tmp_path.name}/: every recording under stem 0 on pulse1, triangle, noise, bending pulse1, triangle" + ] + + def test_a_count_mismatch_is_refused(self, tmp_path: Path) -> None: + with pytest.raises(ValueError, match="1 sources for 2 stems"): + ConversionRequest(sources=(_recording(tmp_path, "bass.wav"),), stems=_two_stems(), output_path=None) + + def test_a_directory_under_several_stems_is_refused(self, tmp_path: Path) -> None: + with pytest.raises(ValueError, match="under one stem; the setup holds 2"): + ConversionRequest(sources=(tmp_path,), stems=_two_stems(), output_path=None) + + def test_a_directory_among_recordings_is_refused(self, tmp_path: Path) -> None: + with pytest.raises(ValueError, match="one directory alone"): + ConversionRequest( + sources=(_recording(tmp_path, "bass.wav"), tmp_path), stems=_two_stems(), output_path=None + ) + + def test_an_output_path_for_a_directory_is_refused(self, tmp_path: Path) -> None: + with pytest.raises(ValueError, match="an output path names the one file"): + ConversionRequest( + sources=(tmp_path,), + stems=classic_setup(DEFAULT_CHANNELS), + output_path=tmp_path / "out.stn", + ) + + def test_no_source_is_refused(self) -> None: + with pytest.raises(ValueError): + ConversionRequest(sources=(), stems=classic_setup(DEFAULT_CHANNELS), output_path=None) + + +class TestReconstruct: + def test_recordings_are_mixed_into_one_file(self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + calls: List[Tuple[Tuple[Path, ...], Path]] = [] + + def reconstruct_sources( + sources: Tuple[Path, ...], config: Config, stems: StemsConfig, output_path: Path + ) -> None: + del config, stems + calls.append((sources, output_path)) + + monkeypatch.setattr(conversion, "reconstruct_sources", reconstruct_sources) + source = _recording(tmp_path, "song.wav") + request = ConversionRequest( + sources=(source,), stems=classic_setup(DEFAULT_CHANNELS), output_path=tmp_path / "x.stn" + ) + + reconstruct(request, Config()) + + assert calls == [((source,), tmp_path / "x.stn")] + + def test_a_directory_is_walked(self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + walked: List[Path] = [] + + def reconstruct_directory(directory: Path, config: Config, stems: StemsConfig) -> None: + del config, stems + walked.append(directory) + + monkeypatch.setattr(conversion, "reconstruct_directory", reconstruct_directory) + request = ConversionRequest(sources=(tmp_path,), stems=classic_setup(DEFAULT_CHANNELS), output_path=None) + + reconstruct(request, Config()) + + assert walked == [tmp_path] diff --git a/tests/unit/sampletones_shared/test_command.py b/tests/unit/sampletones_shared/test_command.py new file mode 100644 index 000000000..c929c6c9b --- /dev/null +++ b/tests/unit/sampletones_shared/test_command.py @@ -0,0 +1,31 @@ +from argparse import ArgumentParser, Namespace +from dataclasses import FrozenInstanceError + +import pytest + +from sampletones_shared.command import Command + + +def _configure(parser: ArgumentParser) -> None: + parser.add_argument("--flag", action="store_true") + + +def _run(arguments: Namespace) -> int: + return 3 if arguments.flag else 0 + + +class TestCommand: + def test_a_command_carries_its_parser_and_its_work(self) -> None: + command = Command(name="probe", help="probe the thing", configure=_configure, run=_run) + parser = ArgumentParser() + + command.configure(parser) + + assert command.run(parser.parse_args(["--flag"])) == 3 + assert command.run(parser.parse_args([])) == 0 + + def test_a_command_is_settled_once_declared(self) -> None: + command = Command(name="probe", help="probe the thing", configure=_configure, run=_run) + + with pytest.raises(FrozenInstanceError): + command.name = "other" # type: ignore[misc] diff --git a/tests/unit/scripts/test_bundle.py b/tests/unit/scripts/test_bundle.py index 2811d5fa3..cefbb5da8 100644 --- a/tests/unit/scripts/test_bundle.py +++ b/tests/unit/scripts/test_bundle.py @@ -91,7 +91,7 @@ def leave_behind(command: Sequence[str]) -> None: assert "import tkinter" in runner.lines[4] assert runner.lines[5].endswith(bundle.ICONS_SCRIPT) assert "PyInstaller" in runner.lines[6] - assert runner.lines[7] == f"{launcher} --self-check" + assert runner.lines[7] == f"{launcher} self-check" assert all((launcher.parent / notice).read_text() == notice for notice in bundle.NOTICES) def test_a_bundle_pyinstaller_never_wrote_is_reported(self, tmp_path: Path) -> None: @@ -131,7 +131,7 @@ def leave_behind(command: Sequence[str]) -> None: root, platform, bundle.BundleOptions(release=False, gpu=False), - runner=RecordingRunner({"--self-check": 1}, leave_behind), + runner=RecordingRunner({"self-check": 1}, leave_behind), environment={}, ) From 69102658205477753280c23b974b106474467117 Mon Sep 17 00:00:00 2001 From: JakimPL Date: Sun, 13 Sep 2026 15:25:21 +0200 Subject: [PATCH 10/36] Opened: the tools package to the command line --- .github/workflows/workflow.yml | 1 + docs/development/packages.md | 21 ++++++++++- docs/development/tooling.md | 34 ++++++++++++++++-- pyproject.toml | 4 +++ src/sampletones/commands/registry.py | 3 +- src/sampletones_config/boundaries/graphs.yaml | 3 +- src/sampletones_config/boundaries/tokens.yaml | 35 ++++++++++++++++++ src/sampletones_tools/__init__.py | 0 src/sampletones_tools/checkout.py | 32 +++++++++++++++++ src/sampletones_tools/py.typed | 0 src/sampletones_tools/registry.py | 5 +++ .../sampletones/commands/test_registry.py | 22 ++++++++++++ .../import_boundary/configs/test_rules.py | 28 +++++++++++++++ tests/unit/sampletones_tools/test_checkout.py | 36 +++++++++++++++++++ tests/unit/sampletones_tools/test_registry.py | 10 ++++++ 15 files changed, 229 insertions(+), 5 deletions(-) create mode 100644 src/sampletones_tools/__init__.py create mode 100644 src/sampletones_tools/checkout.py create mode 100644 src/sampletones_tools/py.typed create mode 100644 src/sampletones_tools/registry.py create mode 100644 tests/unit/sampletones_tools/test_checkout.py create mode 100644 tests/unit/sampletones_tools/test_registry.py diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml index 53800c847..e666c1d66 100644 --- a/.github/workflows/workflow.yml +++ b/.github/workflows/workflow.yml @@ -87,6 +87,7 @@ jobs: python -m pip install --upgrade pip python -m pip install dist/*.whl sampletones --version + sampletones --help - name: Check the wheel carries every resource it needs at startup shell: bash diff --git a/docs/development/packages.md b/docs/development/packages.md index 4049fea1d..eb6afd064 100644 --- a/docs/development/packages.md +++ b/docs/development/packages.md @@ -17,6 +17,7 @@ has come — inside one process and across the pool's workers — is [`progress. ```mermaid graph TD ENTRY["sampletones\n(entry point)"] + TOOLS["sampletones_tools\n(developer tools)"] APP["sampletones_application\n(GUI)"] PLAYER["sampletones_player\n(NES player)"] CORE["sampletones_core\n(reconstruction engine)"] @@ -27,6 +28,12 @@ graph TD ENTRY --> APP ENTRY --> CORE + ENTRY --> TOOLS + TOOLS --> APP + TOOLS --> PLAYER + TOOLS --> CORE + TOOLS --> ASSETS + TOOLS --> SHARED APP --> PLAYER APP --> CORE PLAYER --> CORE @@ -48,10 +55,17 @@ graph TD | `sampletones_core` | The reconstruction engine, the project model, playing a song out into instructions, and the tracker export formats | `sampletones_shared`, `sampletones_synthesis` | | `sampletones_player` | The NES player: the register model, the re-clocking schedule, the 6502 driver and the NSF file | `sampletones_shared`, `sampletones_core` | | `sampletones_application` | The DearPyGui front end | `sampletones_shared`, `sampletones_core`, `sampletones_player` | -| `sampletones` | The command-line entry: the dispatcher, the commands and the startup self-check | `sampletones_shared`, `sampletones_core`, `sampletones_application` | +| `sampletones_tools` | Everything a developer runs and the application does not: the developer commands and the libraries behind them | `sampletones_shared`, `sampletones_assets`, `sampletones_core`, `sampletones_player`, `sampletones_application` | +| `sampletones` | The command-line entry: the dispatcher, the commands and the startup self-check | `sampletones_shared`, `sampletones_core`, `sampletones_application`, `sampletones_tools` | Third-party imports are the package author's own choice and stand outside this table. +**The tools package is reached from the command line alone.** `sampletones_tools` holds what a +developer runs and the application never imports: the developer commands and the libraries behind +them. `sampletones` is its one importer, appending the developer commands to the user commands, so +the wheel and the bundle carry the tools and no shipped package depends on them. +[Tooling](tooling.md) states what a tool is and what a developer command does in an installed copy. + **The reconstruction engine stands below the console player.** A reconstruction is produced, saved and exported to a tracker with `sampletones_player` absent from the process, which is what lets the player's format move while the engine holds still. The consequence is that an export backend @@ -115,6 +129,11 @@ out of reach, so an edge is declared before it is taken. The hook audits the who every commit (`make check-import-boundary`), which means adding an edge to a table is how a new dependency is opened, and removing one enumerates the work of closing it. +Five token rules hold the shipped packages to the tools edge a second way: a module of +`sampletones_application`, `sampletones_core`, `sampletones_player`, `sampletones_shared` or +`sampletones_assets` that spells `sampletones_tools` at all is reported, so the edge is closed in +words as well as in imports. + A graph answers for its own well-formedness as it is read: a unit reaching a unit the graph leaves undeclared is refused, and so is a graph whose units reach themselves, since a unit's layers state a level only where the units stand in an order. diff --git a/docs/development/tooling.md b/docs/development/tooling.md index f9e14e925..0ead9852b 100644 --- a/docs/development/tooling.md +++ b/docs/development/tooling.md @@ -74,8 +74,36 @@ arguments. Each command turns its arguments into a frozen record, field by field | `self-check` | Verifies that the build's imports, bundled resources and configuration files are usable | `--version` and `--help` are flags of the entry itself. The headless runs behind `convert` and -`library` live in `sampletones_core/headless/`, where the calibration reuses them. Developer -commands join the registry as the tools they run move into their own package. +`library` live in `sampletones_core/headless/`, where the calibration reuses them. The developer +commands are listed by `sampletones_tools/registry.py` and join as the tools they run move into +that package. + +## The tools package + +`src/sampletones_tools/` holds every tool the running application does not use, in subpackages by +subject, and `sampletones_tools/registry.py` lists the developer commands they offer. +`sampletones/commands/registry.py` appends them to the user commands, which is the one import of +the tools package; [package layers](packages.md) holds the edge. The wheel and the bundle carry the +package, so every command exists in every copy of the program, and three rules decide what a +developer command does there: + +- **The checkout guard.** `sampletones_tools/checkout.py` holds `require_checkout(command)`: the + repository root must hold `pyproject.toml` beside `src/`, or the command exits naming + `uv run sampletones ` in a checkout. Every developer command that reads or writes the + repository calls it first. A command that measures the code on this machine runs anywhere. +- **No default derived from the repository.** An emitter takes a required `--output`; a measurement + defaults to the user's Documents. Nothing a developer command writes lands beside an installed + package. +- **Package data is read from the package**, through `importlib.resources`, never through a path + under the repository, so it ships in the wheel and the bundle. + +Developer commands are run as `uv run sampletones ` from a checkout; the `sampletones` +command `make setup` installs is a wheel and refuses the guarded ones the same way. A command module +imports pillow, py65 and the like inside `run`, and a test imports the registry in a subprocess and +asserts they stay out of `sys.modules`, since a startup failure in any tool module would break every +invocation, the GUI included. The editable install puts `src/` on the path whole, so a checkout +sees the tools package whatever the wheel lists; hatchling's `dev-mode-exact` stays off for that +reason. ## The bootstrap scripts @@ -112,6 +140,8 @@ the make target that names each. The checks are also pre-commit hooks; | Concern | Owner | |---|---| | Which commands the entry offers | `src/sampletones/commands/registry.py` | +| Which developer commands exist | `src/sampletones_tools/registry.py` | +| Whether a command runs outside a checkout | `src/sampletones_tools/checkout.py` | | What a command is | `src/sampletones_shared/command.py` | | What a bootstrap script may import | `sampletones_config/boundaries/standalone.yaml` | | What differs between systems | `scripts/bootstrap/platforms/` | diff --git a/pyproject.toml b/pyproject.toml index 52ebb90ba..5b1b12d09 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -112,6 +112,7 @@ packages = [ "src/sampletones_player", "src/sampletones_shared", "src/sampletones_synthesis", + "src/sampletones_tools", ] [tool.black] @@ -133,6 +134,7 @@ known_first_party = [ "sampletones_player", "sampletones_shared", "sampletones_synthesis", + "sampletones_tools", ] [tool.pytest.ini_options] @@ -147,6 +149,7 @@ source = [ "sampletones_player", "sampletones_shared", "sampletones_synthesis", + "sampletones_tools", ] [tool.coverage.report] @@ -164,6 +167,7 @@ files = [ "src/sampletones_player", "src/sampletones_shared", "src/sampletones_synthesis", + "src/sampletones_tools", "scripts", ] exclude = ["tests"] diff --git a/src/sampletones/commands/registry.py b/src/sampletones/commands/registry.py index 135d954ae..540865cce 100644 --- a/src/sampletones/commands/registry.py +++ b/src/sampletones/commands/registry.py @@ -6,6 +6,7 @@ from sampletones.commands.run import RUN from sampletones.commands.self_check import SELF_CHECK from sampletones_shared.command import Command +from sampletones_tools.registry import DEVELOPER_COMMANDS USER_COMMANDS: Final[Tuple[Command, ...]] = (RUN, OPEN, CONVERT, LIBRARY, SELF_CHECK) -COMMANDS: Final[Tuple[Command, ...]] = USER_COMMANDS +COMMANDS: Final[Tuple[Command, ...]] = (*USER_COMMANDS, *DEVELOPER_COMMANDS) diff --git a/src/sampletones_config/boundaries/graphs.yaml b/src/sampletones_config/boundaries/graphs.yaml index eda8d30e2..2392db121 100644 --- a/src/sampletones_config/boundaries/graphs.yaml +++ b/src/sampletones_config/boundaries/graphs.yaml @@ -7,7 +7,8 @@ packages: sampletones_core: [sampletones_shared, sampletones_synthesis] sampletones_player: [sampletones_shared, sampletones_core] sampletones_application: [sampletones_shared, sampletones_core, sampletones_player] - sampletones: [sampletones_shared, sampletones_core, sampletones_application] + sampletones_tools: [sampletones_shared, sampletones_assets, sampletones_core, sampletones_player, sampletones_application] + sampletones: [sampletones_shared, sampletones_core, sampletones_application, sampletones_tools] player: root: sampletones_player diff --git a/src/sampletones_config/boundaries/tokens.yaml b/src/sampletones_config/boundaries/tokens.yaml index 8d089e732..0f1b0f31f 100644 --- a/src/sampletones_config/boundaries/tokens.yaml +++ b/src/sampletones_config/boundaries/tokens.yaml @@ -40,3 +40,38 @@ message: >- sampletones_shared is shipped code; the compression study harness (scripts/codec_study) stays outside it, and a finding it earns lands as production code of its own + +- root: sampletones_application + pattern: "**/*.py" + forbidden: '\bsampletones_tools\b' + message: >- + sampletones_application is shipped code and names no tool; the command line is the one importer of + sampletones_tools + +- root: sampletones_core + pattern: "**/*.py" + forbidden: '\bsampletones_tools\b' + message: >- + sampletones_core is shipped code and names no tool; the command line is the one importer of + sampletones_tools + +- root: sampletones_player + pattern: "**/*.py" + forbidden: '\bsampletones_tools\b' + message: >- + sampletones_player is shipped code and names no tool; the command line is the one importer of + sampletones_tools + +- root: sampletones_shared + pattern: "**/*.py" + forbidden: '\bsampletones_tools\b' + message: >- + sampletones_shared is shipped code and names no tool; the command line is the one importer of + sampletones_tools + +- root: sampletones_assets + pattern: "**/*.py" + forbidden: '\bsampletones_tools\b' + message: >- + sampletones_assets is shipped code and names no tool; the command line is the one importer of + sampletones_tools diff --git a/src/sampletones_tools/__init__.py b/src/sampletones_tools/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/sampletones_tools/checkout.py b/src/sampletones_tools/checkout.py new file mode 100644 index 000000000..abaf6db53 --- /dev/null +++ b/src/sampletones_tools/checkout.py @@ -0,0 +1,32 @@ +from pathlib import Path +from typing import Final + +from sampletones_shared.paths.source import REPOSITORY_ROOT + +PROJECT_FILE: Final[str] = "pyproject.toml" +SOURCE_DIRECTORY: Final[str] = "src" +CHECKOUT_ADVICE: Final[str] = ( + "This command reads the repository, so it runs from a checkout: clone SampleToNES, run " + "'make setup', then 'uv run sampletones {command}'." +) + + +def is_checkout(root: Path) -> bool: + """Whether ``root`` is a SampleToNES checkout: the project file beside the source tree.""" + return (root / PROJECT_FILE).is_file() and (root / SOURCE_DIRECTORY).is_dir() + + +def require_checkout(command: str) -> None: + """Holds a developer command to a checkout, where the repository it reads or writes is. + + An installed copy, from the wheel or the bundle, carries the package without the repository + around it, so the refusal names the way to run the command there. + + Args: + command: The command line the advice names, such as ``check import-boundary --all``. + + Raises: + SystemExit: If the package runs outside a checkout. + """ + if not is_checkout(REPOSITORY_ROOT): + raise SystemExit(CHECKOUT_ADVICE.format(command=command)) diff --git a/src/sampletones_tools/py.typed b/src/sampletones_tools/py.typed new file mode 100644 index 000000000..e69de29bb diff --git a/src/sampletones_tools/registry.py b/src/sampletones_tools/registry.py new file mode 100644 index 000000000..e802789f7 --- /dev/null +++ b/src/sampletones_tools/registry.py @@ -0,0 +1,5 @@ +from typing import Final, Tuple + +from sampletones_shared.command import Command + +DEVELOPER_COMMANDS: Final[Tuple[Command, ...]] = () diff --git a/tests/unit/sampletones/commands/test_registry.py b/tests/unit/sampletones/commands/test_registry.py index c0d62dd95..a7374dc69 100644 --- a/tests/unit/sampletones/commands/test_registry.py +++ b/tests/unit/sampletones/commands/test_registry.py @@ -1,13 +1,35 @@ +import subprocess +import sys +from typing import Final, Tuple + from sampletones.commands.registry import COMMANDS, USER_COMMANDS from sampletones.dispatcher import DEFAULT_COMMAND, build_parser +from sampletones_tools.registry import DEVELOPER_COMMANDS + +HEAVY_PACKAGES: Final[Tuple[str, ...]] = ("PIL", "py65", "pytest", "dearpygui") +PROBE: Final[str] = "import sys, sampletones.commands.registry; print(sorted(set(sys.modules) & set(sys.argv[1:])))" class TestRegistry: def test_the_user_commands_are_the_ones_the_guide_names(self) -> None: assert [command.name for command in USER_COMMANDS] == ["run", "open", "convert", "library", "self-check"] + def test_the_commands_are_the_user_ones_followed_by_the_developer_ones(self) -> None: + assert COMMANDS == (*USER_COMMANDS, *DEVELOPER_COMMANDS) + def test_the_default_command_is_registered(self) -> None: assert DEFAULT_COMMAND in {command.name for command in COMMANDS} def test_the_registry_builds_one_parser(self) -> None: assert build_parser(COMMANDS).format_help() + + def test_listing_the_commands_loads_no_tool(self) -> None: + """A tool's dependency imported at module level would break every invocation, the GUI included.""" + completed = subprocess.run( + [sys.executable, "-c", PROBE, *HEAVY_PACKAGES], + check=True, + capture_output=True, + text=True, + ) + + assert completed.stdout.strip() == "[]" diff --git a/tests/unit/sampletones_shared/meta/import_boundary/configs/test_rules.py b/tests/unit/sampletones_shared/meta/import_boundary/configs/test_rules.py index 0aab27b23..f01e747b0 100644 --- a/tests/unit/sampletones_shared/meta/import_boundary/configs/test_rules.py +++ b/tests/unit/sampletones_shared/meta/import_boundary/configs/test_rules.py @@ -21,6 +21,8 @@ APPLICATION: Final[str] = "sampletones_application" CORE: Final[str] = "sampletones_core" PLAYER: Final[str] = "sampletones_player" +TOOLS: Final[str] = "sampletones_tools" +ENTRY: Final[str] = "sampletones" ASSEMBLER: Final[str] = "sampletones_player.driver.assembler" VISUAL_IMPORT: Final[str] = "import dearpygui.dearpygui as dpg\n" @@ -29,6 +31,7 @@ PLAYER_IMPORT: Final[str] = "from sampletones_player.song import Song\n" ASSEMBLER_IMPORT: Final[str] = "from sampletones_player.driver.assembler.builder import build_driver\n" DRIVER_IMPORT: Final[str] = "from sampletones_player.driver.image import DriverImage\n" +TOOLS_IMPORT: Final[str] = "from sampletones_tools.registry import DEVELOPER_COMMANDS\n" PANEL_SUFFIX: Final[str] = "def build() -> None:\n dpg.add_group(parent=SUF_PANEL_LEFT)\n" THIRD_PARTY_IMPORT: Final[str] = "import numpy\n" @@ -70,6 +73,18 @@ def test_the_synthesis_package_stands_below_the_reconstruction_engine(self) -> N """Equal temperament sits in `sampletones_shared`, so synthesis reaches no engine module.""" assert CORE not in reached_units(self.LAYERS, "sampletones_synthesis") + def test_the_entry_is_the_one_importer_of_the_tools(self) -> None: + """The wheel carries the tools, and the command line is where a developer reaches them.""" + assert {unit for unit, layers in self.LAYERS.items() if TOOLS in layers} == {ENTRY} + + def test_no_shipped_package_reaches_the_tools(self) -> None: + shipped = set(self.LAYERS) - {ENTRY, TOOLS} + + assert all(TOOLS not in reached_units(self.LAYERS, package) for package in shipped) + + def test_the_tools_reach_the_application(self) -> None: + assert APPLICATION in self.LAYERS[TOOLS] + class TestPlayerGraph: """The player's own subpackages, and the order they may reach each other in.""" @@ -153,6 +168,19 @@ def test_a_panel_composing_a_column_suffix_is_reported(self, tmp_path: Path) -> assert len(reported(tmp_path)) == 1 + def test_a_shipped_module_naming_the_tools_is_reported_by_the_graph_and_by_name(self, tmp_path: Path) -> None: + write_module(tmp_path / CORE / "formats", "tooling.py", TOOLS_IMPORT) + + kinds = reported(tmp_path) + + assert TOOLS in kinds + assert any("names no tool" in kind for kind in kinds) + + def test_the_entry_naming_the_tools_is_left_alone(self, tmp_path: Path) -> None: + write_module(tmp_path / ENTRY / "commands", "registry.py", TOOLS_IMPORT) + + assert reported(tmp_path) == [] + class TestStandaloneRules: """The scripts that run on the system interpreter, held to the standard library.""" diff --git a/tests/unit/sampletones_tools/test_checkout.py b/tests/unit/sampletones_tools/test_checkout.py new file mode 100644 index 000000000..9cdaf98e1 --- /dev/null +++ b/tests/unit/sampletones_tools/test_checkout.py @@ -0,0 +1,36 @@ +from pathlib import Path + +import pytest + +from sampletones_tools import checkout +from sampletones_tools.checkout import is_checkout, require_checkout + +COMMAND = "check import-boundary --all" + + +class TestIsCheckout: + def test_the_project_file_beside_the_source_tree_makes_a_checkout(self, tmp_path: Path) -> None: + (tmp_path / "pyproject.toml").write_text("", encoding="utf-8") + (tmp_path / "src").mkdir() + + assert is_checkout(tmp_path) + + def test_an_installed_copy_is_no_checkout(self, tmp_path: Path) -> None: + (tmp_path / "site-packages").mkdir() + + assert not is_checkout(tmp_path) + + +class TestRequireCheckout: + def test_the_repository_passes(self) -> None: + require_checkout(COMMAND) + + def test_outside_a_checkout_the_command_is_refused_with_the_way_to_run_it( + self, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + ) -> None: + monkeypatch.setattr(checkout, "REPOSITORY_ROOT", tmp_path) + + with pytest.raises(SystemExit, match=f"uv run sampletones {COMMAND}"): + require_checkout(COMMAND) diff --git a/tests/unit/sampletones_tools/test_registry.py b/tests/unit/sampletones_tools/test_registry.py new file mode 100644 index 000000000..b01cdc8d5 --- /dev/null +++ b/tests/unit/sampletones_tools/test_registry.py @@ -0,0 +1,10 @@ +from sampletones.commands.registry import USER_COMMANDS +from sampletones_tools.registry import DEVELOPER_COMMANDS + + +class TestDeveloperCommands: + def test_every_developer_command_has_a_name_of_its_own(self) -> None: + names = [command.name for command in DEVELOPER_COMMANDS] + + assert len(set(names)) == len(names) + assert not set(names) & {command.name for command in USER_COMMANDS} From c54a7ff5d11d8bfcdc5aab380816825102f2dbd8 Mon Sep 17 00:00:00 2001 From: JakimPL Date: Sun, 13 Sep 2026 15:36:25 +0200 Subject: [PATCH 11/36] Moved: synthesis and calibration into the tools package --- Makefile | 6 +- docs/concepts/calibration.md | 10 +- docs/concepts/reconstruction.md | 4 +- docs/development/architecture.md | 2 +- docs/development/config-organization.md | 4 +- docs/development/packages.md | 8 +- docs/development/tooling.md | 19 ++- pyproject.toml | 4 - scripts/calibration.py | 109 --------------- src/sampletones/commands/convert.py | 2 +- src/sampletones/commands/library.py | 2 +- src/sampletones/commands/open.py | 2 +- src/sampletones/commands/run.py | 2 +- src/sampletones_config/README.md | 2 +- src/sampletones_config/boundaries/graphs.yaml | 3 +- .../boundaries/standalone.yaml | 1 - .../options.py | 0 .../calibration/__init__.py | 0 src/sampletones_tools/calibration/command.py | 88 ++++++++++++ .../calibration/config/__init__.py | 0 .../calibration/config/corpus.py | 2 +- .../calibration/config/mix.py | 0 .../calibration/config/noise.py | 0 .../calibration/config/referee.py | 2 +- .../calibration/config/timbre.py | 2 +- .../calibration/config/tone.py | 0 .../calibration/config/transient.py | 0 .../calibration/corpus/__init__.py | 0 .../calibration/corpus/item.py | 0 .../calibration/corpus/synthesis.py | 26 ++-- .../calibration/corpus/writer.py | 0 .../calibration/paths.py | 0 .../calibration/referee/__init__.py | 0 .../calibration/referee/auditory.py | 2 +- .../calibration/referee/factory.py | 2 +- .../calibration/referee/protocol.py | 0 .../calibration/referee/zimtohrli.py | 0 .../calibration/report.py | 0 .../calibration/runner.py | 0 src/sampletones_tools/calibration/session.py | 126 ++++++++++++++++++ src/sampletones_tools/registry.py | 3 +- .../synthesis}/__init__.py | 0 .../synthesis}/envelopes/__init__.py | 0 .../synthesis}/envelopes/exponential_decay.py | 0 .../synthesis}/envelopes/linear_attack.py | 0 .../synthesis}/envelopes/linear_ramp.py | 0 .../synthesis}/envelopes/types.py | 0 .../synthesis}/filters/__init__.py | 0 .../filters/butterworth_highpass.py | 0 .../synthesis}/filters/types.py | 0 .../synthesis}/frequency.py | 0 .../synthesis}/oscillators/__init__.py | 0 .../oscillators/exponential_glide.py | 2 +- .../synthesis}/oscillators/geometric_sweep.py | 2 +- .../synthesis}/oscillators/pulse.py | 2 +- .../synthesis}/oscillators/sine.py | 2 +- .../synthesis}/oscillators/types.py | 0 .../synthesis}/oscillators/walk_noise.py | 0 .../synthesis}/oscillators/white_noise.py | 0 .../synthesis}/protocols.py | 0 .../synthesis}/py.typed | 0 .../synthesis}/voice/__init__.py | 0 .../synthesis}/voice/layer.py | 4 +- .../synthesis}/voice/voice.py | 2 +- tests/integration/assets/synth_config.py | 2 +- .../import_boundary/configs/test_rules.py | 4 - .../calibration/__init__.py | 0 .../calibration/config/__init__.py | 0 .../calibration/config/test_corpus.py | 2 +- .../calibration/config/test_referee.py | 2 +- .../calibration/corpus/__init__.py | 0 .../calibration/corpus/conftest.py | 6 +- .../calibration/corpus/test_synthesis.py | 6 +- .../calibration/corpus/test_writer.py | 4 +- .../calibration/referee/__init__.py | 0 .../calibration/referee/test_auditory.py | 4 +- .../calibration/test_command.py | 83 ++++++++++++ .../calibration/test_runner.py | 2 +- .../calibration/test_session.py | 71 ++++++++++ .../synthesis}/__init__.py | 0 .../synthesis}/conftest.py | 0 .../synthesis}/envelopes/__init__.py | 0 .../synthesis}/envelopes/test_envelopes.py | 6 +- .../synthesis}/filters/__init__.py | 0 .../filters/test_butterworth_highpass.py | 2 +- .../synthesis}/oscillators/__init__.py | 0 .../synthesis}/oscillators/test_noise.py | 4 +- .../synthesis}/oscillators/test_pulse.py | 2 +- .../synthesis}/oscillators/test_sine.py | 2 +- .../synthesis}/oscillators/test_sweeps.py | 8 +- .../synthesis}/test_frequency.py | 2 +- .../synthesis}/test_unions.py | 22 +-- .../synthesis}/voice/__init__.py | 0 .../synthesis}/voice/test_layer.py | 8 +- .../synthesis}/voice/test_voice.py | 6 +- 95 files changed, 472 insertions(+), 223 deletions(-) delete mode 100644 scripts/calibration.py rename src/{sampletones/commands => sampletones_shared}/options.py (100%) rename src/{sampletones_core => sampletones_tools}/calibration/__init__.py (100%) create mode 100644 src/sampletones_tools/calibration/command.py rename src/{sampletones_core => sampletones_tools}/calibration/config/__init__.py (100%) rename src/{sampletones_core => sampletones_tools}/calibration/config/corpus.py (96%) rename src/{sampletones_core => sampletones_tools}/calibration/config/mix.py (100%) rename src/{sampletones_core => sampletones_tools}/calibration/config/noise.py (100%) rename src/{sampletones_core => sampletones_tools}/calibration/config/referee.py (95%) rename src/{sampletones_core => sampletones_tools}/calibration/config/timbre.py (86%) rename src/{sampletones_core => sampletones_tools}/calibration/config/tone.py (100%) rename src/{sampletones_core => sampletones_tools}/calibration/config/transient.py (100%) rename src/{sampletones_core => sampletones_tools}/calibration/corpus/__init__.py (100%) rename src/{sampletones_core => sampletones_tools}/calibration/corpus/item.py (100%) rename src/{sampletones_core => sampletones_tools}/calibration/corpus/synthesis.py (86%) rename src/{sampletones_core => sampletones_tools}/calibration/corpus/writer.py (100%) rename src/{sampletones_core => sampletones_tools}/calibration/paths.py (100%) rename src/{sampletones_core => sampletones_tools}/calibration/referee/__init__.py (100%) rename src/{sampletones_core => sampletones_tools}/calibration/referee/auditory.py (98%) rename src/{sampletones_core => sampletones_tools}/calibration/referee/factory.py (92%) rename src/{sampletones_core => sampletones_tools}/calibration/referee/protocol.py (100%) rename src/{sampletones_core => sampletones_tools}/calibration/referee/zimtohrli.py (100%) rename src/{sampletones_core => sampletones_tools}/calibration/report.py (100%) rename src/{sampletones_core => sampletones_tools}/calibration/runner.py (100%) create mode 100644 src/sampletones_tools/calibration/session.py rename src/{sampletones_synthesis => sampletones_tools/synthesis}/__init__.py (100%) rename src/{sampletones_synthesis => sampletones_tools/synthesis}/envelopes/__init__.py (100%) rename src/{sampletones_synthesis => sampletones_tools/synthesis}/envelopes/exponential_decay.py (100%) rename src/{sampletones_synthesis => sampletones_tools/synthesis}/envelopes/linear_attack.py (100%) rename src/{sampletones_synthesis => sampletones_tools/synthesis}/envelopes/linear_ramp.py (100%) rename src/{sampletones_synthesis => sampletones_tools/synthesis}/envelopes/types.py (100%) rename src/{sampletones_synthesis => sampletones_tools/synthesis}/filters/__init__.py (100%) rename src/{sampletones_synthesis => sampletones_tools/synthesis}/filters/butterworth_highpass.py (100%) rename src/{sampletones_synthesis => sampletones_tools/synthesis}/filters/types.py (100%) rename src/{sampletones_synthesis => sampletones_tools/synthesis}/frequency.py (100%) rename src/{sampletones_synthesis => sampletones_tools/synthesis}/oscillators/__init__.py (100%) rename src/{sampletones_synthesis => sampletones_tools/synthesis}/oscillators/exponential_glide.py (95%) rename src/{sampletones_synthesis => sampletones_tools/synthesis}/oscillators/geometric_sweep.py (95%) rename src/{sampletones_synthesis => sampletones_tools/synthesis}/oscillators/pulse.py (93%) rename src/{sampletones_synthesis => sampletones_tools/synthesis}/oscillators/sine.py (91%) rename src/{sampletones_synthesis => sampletones_tools/synthesis}/oscillators/types.py (100%) rename src/{sampletones_synthesis => sampletones_tools/synthesis}/oscillators/walk_noise.py (100%) rename src/{sampletones_synthesis => sampletones_tools/synthesis}/oscillators/white_noise.py (100%) rename src/{sampletones_synthesis => sampletones_tools/synthesis}/protocols.py (100%) rename src/{sampletones_synthesis => sampletones_tools/synthesis}/py.typed (100%) rename src/{sampletones_synthesis => sampletones_tools/synthesis}/voice/__init__.py (100%) rename src/{sampletones_synthesis => sampletones_tools/synthesis}/voice/layer.py (90%) rename src/{sampletones_synthesis => sampletones_tools/synthesis}/voice/voice.py (97%) rename tests/unit/{sampletones_core => sampletones_tools}/calibration/__init__.py (100%) rename tests/unit/{sampletones_core => sampletones_tools}/calibration/config/__init__.py (100%) rename tests/unit/{sampletones_core => sampletones_tools}/calibration/config/test_corpus.py (98%) rename tests/unit/{sampletones_core => sampletones_tools}/calibration/config/test_referee.py (96%) rename tests/unit/{sampletones_core => sampletones_tools}/calibration/corpus/__init__.py (100%) rename tests/unit/{sampletones_core => sampletones_tools}/calibration/corpus/conftest.py (68%) rename tests/unit/{sampletones_core => sampletones_tools}/calibration/corpus/test_synthesis.py (87%) rename tests/unit/{sampletones_core => sampletones_tools}/calibration/corpus/test_writer.py (78%) rename tests/unit/{sampletones_core => sampletones_tools}/calibration/referee/__init__.py (100%) rename tests/unit/{sampletones_core => sampletones_tools}/calibration/referee/test_auditory.py (94%) create mode 100644 tests/unit/sampletones_tools/calibration/test_command.py rename tests/unit/{sampletones_core => sampletones_tools}/calibration/test_runner.py (95%) create mode 100644 tests/unit/sampletones_tools/calibration/test_session.py rename tests/unit/{sampletones_synthesis => sampletones_tools/synthesis}/__init__.py (100%) rename tests/unit/{sampletones_synthesis => sampletones_tools/synthesis}/conftest.py (100%) rename tests/unit/{sampletones_synthesis => sampletones_tools/synthesis}/envelopes/__init__.py (100%) rename tests/unit/{sampletones_synthesis => sampletones_tools/synthesis}/envelopes/test_envelopes.py (87%) rename tests/unit/{sampletones_synthesis => sampletones_tools/synthesis}/filters/__init__.py (100%) rename tests/unit/{sampletones_synthesis => sampletones_tools/synthesis}/filters/test_butterworth_highpass.py (90%) rename tests/unit/{sampletones_synthesis => sampletones_tools/synthesis}/oscillators/__init__.py (100%) rename tests/unit/{sampletones_synthesis => sampletones_tools/synthesis}/oscillators/test_noise.py (89%) rename tests/unit/{sampletones_synthesis => sampletones_tools/synthesis}/oscillators/test_pulse.py (93%) rename tests/unit/{sampletones_synthesis => sampletones_tools/synthesis}/oscillators/test_sine.py (95%) rename tests/unit/{sampletones_synthesis => sampletones_tools/synthesis}/oscillators/test_sweeps.py (90%) rename tests/unit/{sampletones_synthesis => sampletones_tools/synthesis}/test_frequency.py (95%) rename tests/unit/{sampletones_synthesis => sampletones_tools/synthesis}/test_unions.py (79%) rename tests/unit/{sampletones_synthesis => sampletones_tools/synthesis}/voice/__init__.py (100%) rename tests/unit/{sampletones_synthesis => sampletones_tools/synthesis}/voice/test_layer.py (86%) rename tests/unit/{sampletones_synthesis => sampletones_tools/synthesis}/voice/test_voice.py (94%) diff --git a/Makefile b/Makefile index 170f25dec..8e10d6959 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ .PHONY: help setup install system-deps build release run clean pre-commit test benchmarks lint format \ - ftm-samples nsf-samples nsf-render compression-report compression-study icons player calibration \ + ftm-samples nsf-samples nsf-render compression-report compression-study icons player \ check-import-boundary check-tag-names check-unused-tags check-rendered-literals check-language-keys \ check-palette-colors check-shortcut-actions @@ -39,7 +39,6 @@ help: @echo $(Q) make compression-study - Measure the song codec over the projects and stems on this machine; the report lands in Documents/SampleToNES/compression (ARGS=--quick for a short run)$(Q) @echo $(Q) make icons - Generate the icon suite into src/sampletones_assets/icons$(Q) @echo $(Q) make player - Assemble the NES player driver with cc65$(Q) - @echo $(Q) make calibration - Score the reconstruction corpus; the report lands in Documents/SampleToNES/calibration$(Q) @echo $(Q) make clean - Remove build artifacts and cache files$(Q) @echo $(Q) make lint - Run mypy and pylint (ARGS=--mypy or ARGS=--pylint for one of them)$(Q) @echo $(Q) make format - Auto-format code (isort, black)$(Q) @@ -126,6 +125,3 @@ check-palette-colors: check-shortcut-actions: uv run scripts/checks/shortcut_actions.py - -calibration: - uv run scripts/calibration.py diff --git a/docs/concepts/calibration.md b/docs/concepts/calibration.md index 37d63ef4a..2e08554da 100644 --- a/docs/concepts/calibration.md +++ b/docs/concepts/calibration.md @@ -6,13 +6,13 @@ spectral/temporal blend, the Viterbi transition weights — and their best value an empirical question. **Calibration** answers it with a repeatable experiment: reconstruct a fixed probe corpus under several candidate configurations and let independent judges score the results, so configuration decisions rest on measured -quality and targeted listening. The harness lives in `sampletones_core.calibration` -and runs as a script: +quality and targeted listening. The harness lives in `sampletones_tools.calibration` +and runs as a developer command from a checkout: ``` -python scripts/calibration.py [--config ] [--methods fft,cqt] +uv run sampletones calibration [--config ] [--methods fft,cqt] [--perceptual-exponents 0.5,1.0] [--temporal-weights 0.1,0.3] - [--channels pulse1,triangle,noise] + [--channels pulse1,triangle,noise] [--output ] ``` The base configuration comes from `--config` when given; otherwise the saved @@ -28,7 +28,7 @@ It has three moving parts: signals in six categories: steady tones across the pitch range, pulse timbres of several duty cycles, white and dark noise, tone-plus-noise mixes, percussive transients (snare, kick, pluck) and a crescendo probing the dynamic - range. Each probe is a `sampletones_synthesis` voice — oscillators, envelopes + range. Each probe is a `sampletones_tools.synthesis` voice — oscillators, envelopes and filters composed from one shared, exactly-rendered configuration vocabulary — built from the probe families in `sampletones_config/calibration/corpus.yaml`. Each category isolates one kind of decision the criterion must get diff --git a/docs/concepts/reconstruction.md b/docs/concepts/reconstruction.md index 35c522f40..3d56895a8 100644 --- a/docs/concepts/reconstruction.md +++ b/docs/concepts/reconstruction.md @@ -406,5 +406,5 @@ Package map: | audio I/O and level | `sampletones_core.audio` | | tracker export | `sampletones_core.exporters` | | pitch refinement | `sampletones_core.reconstructions.reconstructor.refinement` | -| criterion calibration | `sampletones_core.calibration` | -| analytic waveform synthesis | `sampletones_synthesis` | +| criterion calibration | `sampletones_tools.calibration` | +| analytic waveform synthesis | `sampletones_tools.synthesis` | diff --git a/docs/development/architecture.md b/docs/development/architecture.md index 79d215183..825fa5706 100644 --- a/docs/development/architecture.md +++ b/docs/development/architecture.md @@ -98,7 +98,7 @@ A new exclusive operation joins by contributing its `is_active` to the authority Where behavior depends on the operating system, the desktop environment, or an external command-line tool, that variation is expressed as a `Protocol` with one implementation per target, chosen by a runtime factory — never as platform branches scattered through the callers. The factory probes availability (`locate_program`) and environment (`System.current()`, `XDG_CURRENT_DESKTOP`) and returns the implementation that fits; callers depend only on the Protocol and read identically on every platform. -Each tool's quirks stay sealed inside its own implementation and are named in that class's docstring, where a reader meets them beside the code they explain; the guarantee callers depend on — that a saved file carries one of the offered extensions — is enforced once in the API layer above every backend. `utils/file_dialogs/` applies this to native file dialogs: a `FileDialogBackend` Protocol in `protocol.py`, with desktop-portal, `kdialog`, `zenity`, and `tkinter` implementations under `backends/`, selected by `select_file_dialog_backend()`. `sampletones_core/calibration/referee/` follows the same shape with its `build_referees()` factory. +Each tool's quirks stay sealed inside its own implementation and are named in that class's docstring, where a reader meets them beside the code they explain; the guarantee callers depend on — that a saved file carries one of the offered extensions — is enforced once in the API layer above every backend. `utils/file_dialogs/` applies this to native file dialogs: a `FileDialogBackend` Protocol in `protocol.py`, with desktop-portal, `kdialog`, `zenity`, and `tkinter` implementations under `backends/`, selected by `select_file_dialog_backend()`. `sampletones_tools/calibration/referee/` follows the same shape with its `build_referees()` factory. Ordering the implementations is part of the factory's job: where several are available, the one that expresses the most wins. A save offering several file types is answered by the portal because it alone reports which type was chosen, so an export names its format in the type selector; a backend answering with a name alone leaves the extension to be read from the name, and the API layer settles it either way. diff --git a/docs/development/config-organization.md b/docs/development/config-organization.md index 73bedb7a2..95d8e4897 100644 --- a/docs/development/config-organization.md +++ b/docs/development/config-organization.md @@ -32,7 +32,7 @@ empty `__init__.py`. Each schema lives with its reader: - `sampletones_application` owns the layout, theme, palettes, keybindings, language, behavior, and deployment schemas. -- `sampletones_core` owns the calibration schemas. +- `sampletones_tools` owns the calibration schemas. - `sampletones_shared` owns the import-boundary schemas and the loader primitives (`load_yaml_model`, `load_yaml_model_dir`). @@ -133,7 +133,7 @@ each value sits in the tree stays in the factory. | Application | `application/` | `DeploymentConfig` (`sampletones_application/config/deployment/`) | `DeploymentConfig.load()`, with `SAMPLETONES_*` env overrides | | Behavior | `behavior/` | `BehaviorConfig` (`sampletones_application/layout/behavior.py`) | folded into `LayoutConfig.behavior` by `load_layout_config` | | Boundaries | `boundaries/` | `ImportBoundaryRules` (`sampletones_shared/meta/import_boundary/configs/`) | `ImportBoundaryRules.load()` | -| Calibration | `calibration/` | `CorpusConfig`, `RefereeConfig` (`sampletones_core/calibration/config/`) | each model's own `.load()` | +| Calibration | `calibration/` | `CorpusConfig`, `RefereeConfig` (`sampletones_tools/calibration/config/`) | each model's own `.load()` | | Keybindings | `keybindings/` | `ShortcutScheme` (`sampletones_application/utils/gui/shortcuts/`) | `ShortcutCatalog.load()`, indexed by scheme name | | Language | `lang/` | `LanguageManager` (`sampletones_application/categories/`) | flat string map keyed `page.panel.text_type.element`, each key validated at load | | Layout | `layout/` | `LayoutConfig` (`sampletones_application/layout/config.py`) | `load_layout_config` (`layout/loader.py`) | diff --git a/docs/development/packages.md b/docs/development/packages.md index eb6afd064..70526c981 100644 --- a/docs/development/packages.md +++ b/docs/development/packages.md @@ -21,7 +21,6 @@ graph TD APP["sampletones_application\n(GUI)"] PLAYER["sampletones_player\n(NES player)"] CORE["sampletones_core\n(reconstruction engine)"] - SYNTH["sampletones_synthesis\n(waveform synthesis)"] ASSETS["sampletones_assets\n(mark and fonts)"] SHARED["sampletones_shared\n(facts and helpers)"] CONFIG["sampletones_config\n(shipped YAML)"] @@ -37,9 +36,7 @@ graph TD APP --> PLAYER APP --> CORE PLAYER --> CORE - CORE --> SYNTH ASSETS --> SHARED - SYNTH --> SHARED CORE --> SHARED PLAYER --> SHARED APP --> SHARED @@ -51,11 +48,10 @@ graph TD | `sampletones_shared` | Facts and helpers any package holds: constants, exception families, paths, the logger, the array backend, the source layer the checks read the tree through, and the schema these boundaries are declared in | — | | `sampletones_config` | The shipped YAML — layout, palettes, themes, keybindings, language, calibration, and these boundaries themselves — reached as package data rather than by import | — | | `sampletones_assets` | The application mark and the bundled fonts, with the code that draws the mark | `sampletones_shared` | -| `sampletones_synthesis` | Analytic waveform synthesis: oscillators, envelopes, layers and voices | `sampletones_shared` | -| `sampletones_core` | The reconstruction engine, the project model, playing a song out into instructions, and the tracker export formats | `sampletones_shared`, `sampletones_synthesis` | +| `sampletones_core` | The reconstruction engine, the project model, playing a song out into instructions, and the tracker export formats | `sampletones_shared` | | `sampletones_player` | The NES player: the register model, the re-clocking schedule, the 6502 driver and the NSF file | `sampletones_shared`, `sampletones_core` | | `sampletones_application` | The DearPyGui front end | `sampletones_shared`, `sampletones_core`, `sampletones_player` | -| `sampletones_tools` | Everything a developer runs and the application does not: the developer commands and the libraries behind them | `sampletones_shared`, `sampletones_assets`, `sampletones_core`, `sampletones_player`, `sampletones_application` | +| `sampletones_tools` | Everything a developer runs and the application does not: analytic waveform synthesis, the calibration harness, and the developer commands that run them | `sampletones_shared`, `sampletones_assets`, `sampletones_core`, `sampletones_player`, `sampletones_application` | | `sampletones` | The command-line entry: the dispatcher, the commands and the startup self-check | `sampletones_shared`, `sampletones_core`, `sampletones_application`, `sampletones_tools` | Third-party imports are the package author's own choice and stand outside this table. diff --git a/docs/development/tooling.md b/docs/development/tooling.md index 0ead9852b..1c5f43627 100644 --- a/docs/development/tooling.md +++ b/docs/development/tooling.md @@ -74,9 +74,16 @@ arguments. Each command turns its arguments into a frozen record, field by field | `self-check` | Verifies that the build's imports, bundled resources and configuration files are usable | `--version` and `--help` are flags of the entry itself. The headless runs behind `convert` and -`library` live in `sampletones_core/headless/`, where the calibration reuses them. The developer -commands are listed by `sampletones_tools/registry.py` and join as the tools they run move into -that package. +`library` live in `sampletones_core/headless/`, where the calibration reuses them. + +The developer commands, listed by `sampletones_tools/registry.py` and run as +`uv run sampletones ` from a checkout: + +| Command | What it does | +|---|---| +| `calibration [--config FILE] [-o DIR] [--methods LIST] [--perceptual-exponents LIST] [--temporal-weights LIST] [--channels LIST]` | Reconstructs the calibration corpus under every variant of the sweep, scores it with every referee, and writes the reports; without `-o` the run lands in a timestamped directory under Documents/SampleToNES/calibration | + +More join as the tools they run move into the package. ## The tools package @@ -130,9 +137,9 @@ interpreter (`preflight.py`), and the platforms (`platforms/`). ## The tool scripts -`calibration.py`, `compression_study.py`, `nsf_render.py`, `player.py`, `assets/icons.py` and -the checks under `checks/` import the project's packages and run inside its environment, from -the make target that names each. The checks are also pre-commit hooks; +`compression_study.py`, `nsf_render.py`, `player.py`, `assets/icons.py` and the checks under +`checks/` import the project's packages and run inside its environment, from the make target +that names each. The checks are also pre-commit hooks; [architecture](architecture.md#enforcement) lists them. ## Who governs what diff --git a/pyproject.toml b/pyproject.toml index 5b1b12d09..b6b1a722e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -111,7 +111,6 @@ packages = [ "src/sampletones_core", "src/sampletones_player", "src/sampletones_shared", - "src/sampletones_synthesis", "src/sampletones_tools", ] @@ -133,7 +132,6 @@ known_first_party = [ "sampletones_core", "sampletones_player", "sampletones_shared", - "sampletones_synthesis", "sampletones_tools", ] @@ -148,7 +146,6 @@ source = [ "sampletones_core", "sampletones_player", "sampletones_shared", - "sampletones_synthesis", "sampletones_tools", ] @@ -166,7 +163,6 @@ files = [ "src/sampletones_core", "src/sampletones_player", "src/sampletones_shared", - "src/sampletones_synthesis", "src/sampletones_tools", "scripts", ] diff --git a/scripts/calibration.py b/scripts/calibration.py deleted file mode 100644 index 4bf8d57d0..000000000 --- a/scripts/calibration.py +++ /dev/null @@ -1,109 +0,0 @@ -import argparse -from datetime import UTC, datetime -from pathlib import Path -from typing import Final - -from sampletones_core.calibration.config.corpus import CorpusConfig -from sampletones_core.calibration.corpus.synthesis import build_corpus -from sampletones_core.calibration.corpus.writer import write_corpus -from sampletones_core.calibration.referee.factory import build_referees -from sampletones_core.calibration.report import write_csv, write_markdown -from sampletones_core.calibration.runner import build_variants, evaluate_variants -from sampletones_core.configs import Config -from sampletones_core.constants.enums import ( - DEFAULT_CHANNELS, - ChannelName, - SpectrumMethod, -) -from sampletones_shared.logger import logger -from sampletones_shared.paths.user import USER_PATH_DOCUMENTS - -DEFAULT_OUTPUT_ROOT: Final[Path] = USER_PATH_DOCUMENTS / "calibration" -DEFAULT_METHODS: Final[str] = f"{SpectrumMethod.FFT.value},{SpectrumMethod.CQT.value}" -DEFAULT_PERCEPTUAL_EXPONENTS: Final[str] = "1.0" -DEFAULT_CHANNEL_NAMES: Final[str] = ",".join(generator.value for generator in DEFAULT_CHANNELS) - - -def main() -> None: - parser = argparse.ArgumentParser( - description="Reconstruct the calibration corpus and score it with referees.", - ) - parser.add_argument( - "--config", - type=Path, - default=None, - help="Base configuration path.", - ) - parser.add_argument( - "--output", - type=Path, - default=None, - help="Output directory of the run.", - ) - parser.add_argument( - "--methods", - type=str, - default=DEFAULT_METHODS, - help="Comma-separated spectrum methods.", - ) - parser.add_argument( - "--perceptual-exponents", - type=str, - default=DEFAULT_PERCEPTUAL_EXPONENTS, - help="Comma-separated values of metric.perceptual_exponent.", - ) - parser.add_argument( - "--temporal-weights", - type=str, - default="", - help="Comma-separated values of weights.temporal_loss_weight; empty keeps the base blend.", - ) - parser.add_argument( - "--channels", - type=str, - default=DEFAULT_CHANNEL_NAMES, - help="Comma-separated channels every variant reconstructs with.", - ) - arguments = parser.parse_args() - - channels = [ChannelName(name.strip()) for name in arguments.channels.split(",") if name.strip()] - if not channels: - parser.error("--channels requires at least one channel name") - - base = Config.load(arguments.config) if arguments.config else Config.default() - base = base.model_copy( - update={ - "generation": base.generation.model_copy( - update={"channels": channels}, - ) - }, - ) - output = arguments.output or DEFAULT_OUTPUT_ROOT / datetime.now(UTC).strftime("run-%Y%m%d-%H%M%S") - output.mkdir(parents=True, exist_ok=True) - - methods = [SpectrumMethod(name.strip()) for name in arguments.methods.split(",") if name.strip()] - if not methods: - parser.error("--methods requires at least one spectrum method") - - exponents = [float(value) for value in arguments.perceptual_exponents.split(",") if value.strip()] - temporal_weights = [float(value) for value in arguments.temporal_weights.split(",") if value.strip()] - - sample_rate = base.library.sample_rate - items = build_corpus(sample_rate, config=CorpusConfig.load()) - item_paths = write_corpus(items, output / "corpus", sample_rate) - referees = build_referees(sample_rate) - variants = build_variants(base, methods, exponents, temporal_weights) - - channel_names = ", ".join(channel.value for channel in channels) - logger.info( - f"Evaluating {len(variants)} variants x {len(items)} items x {len(referees)} referees on {channel_names}" - ) - rows = evaluate_variants(variants, items, item_paths, referees) - - write_csv(rows, output / "report.csv") - write_markdown(rows, output / "report.md") - logger.info(f"Report written to {output}") - - -if __name__ == "__main__": - main() diff --git a/src/sampletones/commands/convert.py b/src/sampletones/commands/convert.py index 0db922895..8074b8b24 100644 --- a/src/sampletones/commands/convert.py +++ b/src/sampletones/commands/convert.py @@ -3,8 +3,8 @@ from pathlib import Path from typing import Final, Optional, Tuple -from sampletones.commands.options import add_config_option from sampletones_shared.command import Command +from sampletones_shared.options import add_config_option NAME: Final[str] = "convert" HELP: Final[str] = "reconstruct recordings into a .stn file" diff --git a/src/sampletones/commands/library.py b/src/sampletones/commands/library.py index bed80c66c..2e83d8c15 100644 --- a/src/sampletones/commands/library.py +++ b/src/sampletones/commands/library.py @@ -3,8 +3,8 @@ from pathlib import Path from typing import Final, Optional -from sampletones.commands.options import add_config_option from sampletones_shared.command import Command +from sampletones_shared.options import add_config_option NAME: Final[str] = "library" HELP: Final[str] = "generate the instruction library for a configuration" diff --git a/src/sampletones/commands/open.py b/src/sampletones/commands/open.py index 9bb7528ad..c6c576cb2 100644 --- a/src/sampletones/commands/open.py +++ b/src/sampletones/commands/open.py @@ -3,8 +3,8 @@ from pathlib import Path from typing import Final, Optional -from sampletones.commands.options import add_config_option from sampletones_shared.command import Command +from sampletones_shared.options import add_config_option NAME: Final[str] = "open" HELP: Final[str] = "start the application with a project, reconstruction or library loaded" diff --git a/src/sampletones/commands/run.py b/src/sampletones/commands/run.py index 68439e1e8..241c088ac 100644 --- a/src/sampletones/commands/run.py +++ b/src/sampletones/commands/run.py @@ -3,8 +3,8 @@ from pathlib import Path from typing import Final, Optional -from sampletones.commands.options import add_config_option from sampletones_shared.command import Command +from sampletones_shared.options import add_config_option NAME: Final[str] = "run" HELP: Final[str] = "start the application" diff --git a/src/sampletones_config/README.md b/src/sampletones_config/README.md index 430c78194..268bf4b59 100644 --- a/src/sampletones_config/README.md +++ b/src/sampletones_config/README.md @@ -8,7 +8,7 @@ programmatic role is to be importable so consumers can resolve its directory The schema that validates each file lives in the **consuming** package: - `sampletones_application` — layout, theme, palettes, language, behavior, deployment. -- `sampletones_core` — calibration. +- `sampletones_tools` — calibration. - `sampletones_shared` — the import boundaries and the loader primitives. The data package must not import a schema, and a schema package must not inline data. diff --git a/src/sampletones_config/boundaries/graphs.yaml b/src/sampletones_config/boundaries/graphs.yaml index 2392db121..bf8caa62f 100644 --- a/src/sampletones_config/boundaries/graphs.yaml +++ b/src/sampletones_config/boundaries/graphs.yaml @@ -3,8 +3,7 @@ packages: sampletones_shared: [] sampletones_config: [] sampletones_assets: [sampletones_shared] - sampletones_synthesis: [sampletones_shared] - sampletones_core: [sampletones_shared, sampletones_synthesis] + sampletones_core: [sampletones_shared] sampletones_player: [sampletones_shared, sampletones_core] sampletones_application: [sampletones_shared, sampletones_core, sampletones_player] sampletones_tools: [sampletones_shared, sampletones_assets, sampletones_core, sampletones_player, sampletones_application] diff --git a/src/sampletones_config/boundaries/standalone.yaml b/src/sampletones_config/boundaries/standalone.yaml index ed2ac61bf..efd401216 100644 --- a/src/sampletones_config/boundaries/standalone.yaml +++ b/src/sampletones_config/boundaries/standalone.yaml @@ -1,6 +1,5 @@ - pattern: "**/*.py" excluding: - - "calibration.py" - "compression_study.py" - "nsf_render.py" - "player.py" diff --git a/src/sampletones/commands/options.py b/src/sampletones_shared/options.py similarity index 100% rename from src/sampletones/commands/options.py rename to src/sampletones_shared/options.py diff --git a/src/sampletones_core/calibration/__init__.py b/src/sampletones_tools/calibration/__init__.py similarity index 100% rename from src/sampletones_core/calibration/__init__.py rename to src/sampletones_tools/calibration/__init__.py diff --git a/src/sampletones_tools/calibration/command.py b/src/sampletones_tools/calibration/command.py new file mode 100644 index 000000000..846bc791b --- /dev/null +++ b/src/sampletones_tools/calibration/command.py @@ -0,0 +1,88 @@ +from argparse import ArgumentParser, Namespace +from dataclasses import dataclass +from pathlib import Path +from typing import Final, Optional + +from sampletones_shared.command import Command +from sampletones_shared.options import add_config_option + +NAME: Final[str] = "calibration" +HELP: Final[str] = "score the reconstruction corpus under candidate configurations" +OUTPUT_HELP: Final[str] = ( + "the directory the run writes into; without it, a timestamped directory under Documents/SampleToNES/calibration" +) +METHODS_HELP: Final[str] = "spectrum methods to evaluate, comma separated; without it fft and cqt" +EXPONENTS_HELP: Final[str] = "values of metric.perceptual_exponent to evaluate, comma separated; without it 1.0" +WEIGHTS_HELP: Final[str] = ( + "values of weights.temporal_loss_weight to evaluate, comma separated; without it the base blend" +) +CHANNELS_HELP: Final[str] = ( + "channels every variant reconstructs with, comma separated; without it pulse1, triangle and noise" +) + + +@dataclass(frozen=True) +class CalibrationArguments: + """What a calibration run is given, as written on the command line.""" + + config: Optional[Path] + output: Optional[Path] + methods: Optional[str] + perceptual_exponents: Optional[str] + temporal_weights: Optional[str] + channels: Optional[str] + + +def configure(parser: ArgumentParser) -> None: + add_config_option(parser) + parser.add_argument("--output", "-o", type=Path, default=None, help=OUTPUT_HELP) + parser.add_argument("--methods", type=str, default=None, help=METHODS_HELP) + parser.add_argument("--perceptual-exponents", type=str, default=None, help=EXPONENTS_HELP) + parser.add_argument("--temporal-weights", type=str, default=None, help=WEIGHTS_HELP) + parser.add_argument("--channels", type=str, default=None, help=CHANNELS_HELP) + + +def run(arguments: Namespace) -> int: + """Runs the calibration the options describe. + + Raises: + SystemExit: If a method, a value or a channel is unknown. + """ + given = CalibrationArguments( + config=arguments.config, + output=arguments.output, + methods=arguments.methods, + perceptual_exponents=arguments.perceptual_exponents, + temporal_weights=arguments.temporal_weights, + channels=arguments.channels, + ) + + from sampletones_core.headless.config import load_config + from sampletones_core.headless.conversion import channels_named + from sampletones_tools.calibration.session import ( + BASE_BLEND, + DEFAULT_PERCEPTUAL_EXPONENTS, + CalibrationRequest, + calibrate, + default_output, + floats_named, + methods_named, + ) + + try: + request = CalibrationRequest( + base=load_config(given.config), + output=given.output if given.output is not None else default_output(), + methods=methods_named(given.methods), + perceptual_exponents=floats_named(given.perceptual_exponents, DEFAULT_PERCEPTUAL_EXPONENTS), + temporal_weights=floats_named(given.temporal_weights, BASE_BLEND), + channels=channels_named(given.channels), + ) + except ValueError as error: + raise SystemExit(str(error)) from error + + calibrate(request) + return 0 + + +CALIBRATION: Final[Command] = Command(name=NAME, help=HELP, configure=configure, run=run) diff --git a/src/sampletones_core/calibration/config/__init__.py b/src/sampletones_tools/calibration/config/__init__.py similarity index 100% rename from src/sampletones_core/calibration/config/__init__.py rename to src/sampletones_tools/calibration/config/__init__.py diff --git a/src/sampletones_core/calibration/config/corpus.py b/src/sampletones_tools/calibration/config/corpus.py similarity index 96% rename from src/sampletones_core/calibration/config/corpus.py rename to src/sampletones_tools/calibration/config/corpus.py index b9fab70c5..ad3c45bfe 100644 --- a/src/sampletones_core/calibration/config/corpus.py +++ b/src/sampletones_tools/calibration/config/corpus.py @@ -2,8 +2,8 @@ from pydantic import BaseModel, Field -from sampletones_core.calibration.paths import CORPUS_CONFIG_PATH from sampletones_shared.utils.serialization import load_yaml_model +from sampletones_tools.calibration.paths import CORPUS_CONFIG_PATH from .mix import MixConfig from .noise import NoiseConfig diff --git a/src/sampletones_core/calibration/config/mix.py b/src/sampletones_tools/calibration/config/mix.py similarity index 100% rename from src/sampletones_core/calibration/config/mix.py rename to src/sampletones_tools/calibration/config/mix.py diff --git a/src/sampletones_core/calibration/config/noise.py b/src/sampletones_tools/calibration/config/noise.py similarity index 100% rename from src/sampletones_core/calibration/config/noise.py rename to src/sampletones_tools/calibration/config/noise.py diff --git a/src/sampletones_core/calibration/config/referee.py b/src/sampletones_tools/calibration/config/referee.py similarity index 95% rename from src/sampletones_core/calibration/config/referee.py rename to src/sampletones_tools/calibration/config/referee.py index 9e4e2e65a..bb4cafc6d 100644 --- a/src/sampletones_core/calibration/config/referee.py +++ b/src/sampletones_tools/calibration/config/referee.py @@ -2,8 +2,8 @@ from pydantic import BaseModel, Field, PositiveInt -from sampletones_core.calibration.paths import REFEREE_CONFIG_PATH from sampletones_shared.utils.serialization import load_yaml_model +from sampletones_tools.calibration.paths import REFEREE_CONFIG_PATH class RefereeConfig(BaseModel, frozen=True): diff --git a/src/sampletones_core/calibration/config/timbre.py b/src/sampletones_tools/calibration/config/timbre.py similarity index 86% rename from src/sampletones_core/calibration/config/timbre.py rename to src/sampletones_tools/calibration/config/timbre.py index 0febd84ae..86c31f775 100644 --- a/src/sampletones_core/calibration/config/timbre.py +++ b/src/sampletones_tools/calibration/config/timbre.py @@ -2,7 +2,7 @@ from pydantic import BaseModel, Field -from sampletones_synthesis.oscillators.pulse import DutyCycle +from sampletones_tools.synthesis.oscillators.pulse import DutyCycle class TimbreConfig(BaseModel, frozen=True): diff --git a/src/sampletones_core/calibration/config/tone.py b/src/sampletones_tools/calibration/config/tone.py similarity index 100% rename from src/sampletones_core/calibration/config/tone.py rename to src/sampletones_tools/calibration/config/tone.py diff --git a/src/sampletones_core/calibration/config/transient.py b/src/sampletones_tools/calibration/config/transient.py similarity index 100% rename from src/sampletones_core/calibration/config/transient.py rename to src/sampletones_tools/calibration/config/transient.py diff --git a/src/sampletones_core/calibration/corpus/__init__.py b/src/sampletones_tools/calibration/corpus/__init__.py similarity index 100% rename from src/sampletones_core/calibration/corpus/__init__.py rename to src/sampletones_tools/calibration/corpus/__init__.py diff --git a/src/sampletones_core/calibration/corpus/item.py b/src/sampletones_tools/calibration/corpus/item.py similarity index 100% rename from src/sampletones_core/calibration/corpus/item.py rename to src/sampletones_tools/calibration/corpus/item.py diff --git a/src/sampletones_core/calibration/corpus/synthesis.py b/src/sampletones_tools/calibration/corpus/synthesis.py similarity index 86% rename from src/sampletones_core/calibration/corpus/synthesis.py rename to src/sampletones_tools/calibration/corpus/synthesis.py index b8815670a..52bbfe694 100644 --- a/src/sampletones_core/calibration/corpus/synthesis.py +++ b/src/sampletones_tools/calibration/corpus/synthesis.py @@ -3,21 +3,21 @@ import numpy as np from sampletones_core.audio.processing import clip_audio -from sampletones_core.calibration.config.corpus import CorpusConfig -from sampletones_synthesis.envelopes.exponential_decay import ExponentialDecayEnvelope -from sampletones_synthesis.envelopes.linear_attack import LinearAttackEnvelope -from sampletones_synthesis.envelopes.linear_ramp import LinearRampEnvelope -from sampletones_synthesis.envelopes.types import EnvelopeUnion -from sampletones_synthesis.oscillators.exponential_glide import ( +from sampletones_tools.calibration.config.corpus import CorpusConfig +from sampletones_tools.synthesis.envelopes.exponential_decay import ExponentialDecayEnvelope +from sampletones_tools.synthesis.envelopes.linear_attack import LinearAttackEnvelope +from sampletones_tools.synthesis.envelopes.linear_ramp import LinearRampEnvelope +from sampletones_tools.synthesis.envelopes.types import EnvelopeUnion +from sampletones_tools.synthesis.oscillators.exponential_glide import ( ExponentialGlideOscillator, ) -from sampletones_synthesis.oscillators.pulse import PulseOscillator -from sampletones_synthesis.oscillators.sine import SineOscillator -from sampletones_synthesis.oscillators.types import OscillatorUnion -from sampletones_synthesis.oscillators.walk_noise import WalkNoiseOscillator -from sampletones_synthesis.oscillators.white_noise import WhiteNoiseOscillator -from sampletones_synthesis.voice.layer import Layer -from sampletones_synthesis.voice.voice import Voice +from sampletones_tools.synthesis.oscillators.pulse import PulseOscillator +from sampletones_tools.synthesis.oscillators.sine import SineOscillator +from sampletones_tools.synthesis.oscillators.types import OscillatorUnion +from sampletones_tools.synthesis.oscillators.walk_noise import WalkNoiseOscillator +from sampletones_tools.synthesis.oscillators.white_noise import WhiteNoiseOscillator +from sampletones_tools.synthesis.voice.layer import Layer +from sampletones_tools.synthesis.voice.voice import Voice from .item import CorpusItem diff --git a/src/sampletones_core/calibration/corpus/writer.py b/src/sampletones_tools/calibration/corpus/writer.py similarity index 100% rename from src/sampletones_core/calibration/corpus/writer.py rename to src/sampletones_tools/calibration/corpus/writer.py diff --git a/src/sampletones_core/calibration/paths.py b/src/sampletones_tools/calibration/paths.py similarity index 100% rename from src/sampletones_core/calibration/paths.py rename to src/sampletones_tools/calibration/paths.py diff --git a/src/sampletones_core/calibration/referee/__init__.py b/src/sampletones_tools/calibration/referee/__init__.py similarity index 100% rename from src/sampletones_core/calibration/referee/__init__.py rename to src/sampletones_tools/calibration/referee/__init__.py diff --git a/src/sampletones_core/calibration/referee/auditory.py b/src/sampletones_tools/calibration/referee/auditory.py similarity index 98% rename from src/sampletones_core/calibration/referee/auditory.py rename to src/sampletones_tools/calibration/referee/auditory.py index f5f0e3046..db124d054 100644 --- a/src/sampletones_core/calibration/referee/auditory.py +++ b/src/sampletones_tools/calibration/referee/auditory.py @@ -3,7 +3,7 @@ import numpy as np from scipy.signal import stft -from sampletones_core.calibration.config.referee import RefereeConfig +from sampletones_tools.calibration.config.referee import RefereeConfig ERB_RATE_SCALE: Final[float] = 21.4 ERB_RATE_FACTOR: Final[float] = 4.37e-3 diff --git a/src/sampletones_core/calibration/referee/factory.py b/src/sampletones_tools/calibration/referee/factory.py similarity index 92% rename from src/sampletones_core/calibration/referee/factory.py rename to src/sampletones_tools/calibration/referee/factory.py index e6310e878..7c9fb8885 100644 --- a/src/sampletones_core/calibration/referee/factory.py +++ b/src/sampletones_tools/calibration/referee/factory.py @@ -1,6 +1,6 @@ from typing import List -from sampletones_core.calibration.config.referee import RefereeConfig +from sampletones_tools.calibration.config.referee import RefereeConfig from .auditory import MultiResolutionAuditoryReferee from .protocol import Referee diff --git a/src/sampletones_core/calibration/referee/protocol.py b/src/sampletones_tools/calibration/referee/protocol.py similarity index 100% rename from src/sampletones_core/calibration/referee/protocol.py rename to src/sampletones_tools/calibration/referee/protocol.py diff --git a/src/sampletones_core/calibration/referee/zimtohrli.py b/src/sampletones_tools/calibration/referee/zimtohrli.py similarity index 100% rename from src/sampletones_core/calibration/referee/zimtohrli.py rename to src/sampletones_tools/calibration/referee/zimtohrli.py diff --git a/src/sampletones_core/calibration/report.py b/src/sampletones_tools/calibration/report.py similarity index 100% rename from src/sampletones_core/calibration/report.py rename to src/sampletones_tools/calibration/report.py diff --git a/src/sampletones_core/calibration/runner.py b/src/sampletones_tools/calibration/runner.py similarity index 100% rename from src/sampletones_core/calibration/runner.py rename to src/sampletones_tools/calibration/runner.py diff --git a/src/sampletones_tools/calibration/session.py b/src/sampletones_tools/calibration/session.py new file mode 100644 index 000000000..0266a256d --- /dev/null +++ b/src/sampletones_tools/calibration/session.py @@ -0,0 +1,126 @@ +from datetime import UTC, datetime +from pathlib import Path +from typing import Final, List, Optional, Sequence, Tuple + +from pydantic import BaseModel, ConfigDict, Field + +from sampletones_core.configs import Config +from sampletones_core.constants.enums import ChannelName, SpectrumMethod +from sampletones_shared.logger import logger +from sampletones_shared.paths.user import USER_PATH_DOCUMENTS +from sampletones_tools.calibration.config.corpus import CorpusConfig +from sampletones_tools.calibration.corpus.synthesis import build_corpus +from sampletones_tools.calibration.corpus.writer import write_corpus +from sampletones_tools.calibration.referee.factory import build_referees +from sampletones_tools.calibration.report import write_csv, write_markdown +from sampletones_tools.calibration.runner import build_variants, evaluate_variants + +DEFAULT_METHODS: Final[Tuple[SpectrumMethod, ...]] = (SpectrumMethod.FFT, SpectrumMethod.CQT) +DEFAULT_PERCEPTUAL_EXPONENTS: Final[Tuple[float, ...]] = (1.0,) +BASE_BLEND: Final[Tuple[float, ...]] = () +OUTPUT_DIRECTORY: Final[str] = "calibration" +RUN_STAMP: Final[str] = "run-%Y%m%d-%H%M%S" +CORPUS_DIRECTORY: Final[str] = "corpus" +CSV_REPORT: Final[str] = "report.csv" +MARKDOWN_REPORT: Final[str] = "report.md" +LIST_SEPARATOR: Final[str] = "," + + +def methods_named(stated: Optional[str]) -> List[SpectrumMethod]: + """The spectrum methods a run evaluates: the ones named, comma separated, or FFT and CQT. + + Raises: + ValueError: If a name is none of the spectrum methods. + """ + if stated is None: + return list(DEFAULT_METHODS) + + names = [name.strip() for name in stated.split(LIST_SEPARATOR) if name.strip()] + methods: List[SpectrumMethod] = [] + for name in names: + try: + methods.append(SpectrumMethod(name)) + except ValueError as error: + known = ", ".join(method.value for method in SpectrumMethod) + raise ValueError(f"Unknown spectrum method {name!r}; the methods are {known}.") from error + + return methods + + +def floats_named(stated: Optional[str], default: Sequence[float]) -> List[float]: + """The values a sweep takes: the ones named, comma separated, or ``default``. + + Raises: + ValueError: If a value is no number. + """ + if stated is None: + return list(default) + + values: List[float] = [] + for value in (piece.strip() for piece in stated.split(LIST_SEPARATOR) if piece.strip()): + try: + values.append(float(value)) + except ValueError as error: + raise ValueError(f"Not a number: {value!r}.") from error + + return values + + +def default_output() -> Path: + """A timestamped run directory under the user's calibration documents.""" + return USER_PATH_DOCUMENTS / OUTPUT_DIRECTORY / datetime.now(UTC).strftime(RUN_STAMP) + + +class CalibrationRequest(BaseModel): + """What a calibration run is asked for: the base configuration, the sweep and where it writes. + + Attributes: + base: The configuration every value the sweep leaves untouched comes from. + output: The directory the corpus and the reports are written into. + methods: The spectrum methods evaluated. + perceptual_exponents: The values of ``metric.perceptual_exponent`` evaluated. + temporal_weights: The values of ``weights.temporal_loss_weight`` evaluated; empty keeps + the base blend. + channels: The channels every variant reconstructs with. + """ + + model_config = ConfigDict(frozen=True) + + base: Config + output: Path + methods: List[SpectrumMethod] = Field(min_length=1) + perceptual_exponents: List[float] = Field(min_length=1) + temporal_weights: List[float] + channels: List[ChannelName] = Field(min_length=1) + + def pinned_base(self) -> Config: + """The base configuration reconstructing with the channels the run pins.""" + generation = self.base.generation.model_copy(update={"channels": self.channels}) + return self.base.model_copy(update={"generation": generation}) + + +def calibrate(request: CalibrationRequest) -> Path: + """Reconstructs the corpus under every variant, scores it with every referee and writes the reports. + + Returns: + Path: The directory holding the corpus, ``report.csv`` and ``report.md``. + """ + base = request.pinned_base() + request.output.mkdir(parents=True, exist_ok=True) + + sample_rate = base.library.sample_rate + items = build_corpus(sample_rate, config=CorpusConfig.load()) + item_paths = write_corpus(items, request.output / CORPUS_DIRECTORY, sample_rate) + referees = build_referees(sample_rate) + variants = build_variants(base, request.methods, request.perceptual_exponents, request.temporal_weights) + + channel_names = ", ".join(channel.value for channel in request.channels) + logger.info( + f"Evaluating {len(variants)} variants x {len(items)} items x {len(referees)} referees on {channel_names}" + ) + rows = evaluate_variants(variants, items, item_paths, referees) + + write_csv(rows, request.output / CSV_REPORT) + write_markdown(rows, request.output / MARKDOWN_REPORT) + logger.info(f"Report written to {request.output}") + return request.output diff --git a/src/sampletones_tools/registry.py b/src/sampletones_tools/registry.py index e802789f7..f75d8c374 100644 --- a/src/sampletones_tools/registry.py +++ b/src/sampletones_tools/registry.py @@ -1,5 +1,6 @@ from typing import Final, Tuple from sampletones_shared.command import Command +from sampletones_tools.calibration.command import CALIBRATION -DEVELOPER_COMMANDS: Final[Tuple[Command, ...]] = () +DEVELOPER_COMMANDS: Final[Tuple[Command, ...]] = (CALIBRATION,) diff --git a/src/sampletones_synthesis/__init__.py b/src/sampletones_tools/synthesis/__init__.py similarity index 100% rename from src/sampletones_synthesis/__init__.py rename to src/sampletones_tools/synthesis/__init__.py diff --git a/src/sampletones_synthesis/envelopes/__init__.py b/src/sampletones_tools/synthesis/envelopes/__init__.py similarity index 100% rename from src/sampletones_synthesis/envelopes/__init__.py rename to src/sampletones_tools/synthesis/envelopes/__init__.py diff --git a/src/sampletones_synthesis/envelopes/exponential_decay.py b/src/sampletones_tools/synthesis/envelopes/exponential_decay.py similarity index 100% rename from src/sampletones_synthesis/envelopes/exponential_decay.py rename to src/sampletones_tools/synthesis/envelopes/exponential_decay.py diff --git a/src/sampletones_synthesis/envelopes/linear_attack.py b/src/sampletones_tools/synthesis/envelopes/linear_attack.py similarity index 100% rename from src/sampletones_synthesis/envelopes/linear_attack.py rename to src/sampletones_tools/synthesis/envelopes/linear_attack.py diff --git a/src/sampletones_synthesis/envelopes/linear_ramp.py b/src/sampletones_tools/synthesis/envelopes/linear_ramp.py similarity index 100% rename from src/sampletones_synthesis/envelopes/linear_ramp.py rename to src/sampletones_tools/synthesis/envelopes/linear_ramp.py diff --git a/src/sampletones_synthesis/envelopes/types.py b/src/sampletones_tools/synthesis/envelopes/types.py similarity index 100% rename from src/sampletones_synthesis/envelopes/types.py rename to src/sampletones_tools/synthesis/envelopes/types.py diff --git a/src/sampletones_synthesis/filters/__init__.py b/src/sampletones_tools/synthesis/filters/__init__.py similarity index 100% rename from src/sampletones_synthesis/filters/__init__.py rename to src/sampletones_tools/synthesis/filters/__init__.py diff --git a/src/sampletones_synthesis/filters/butterworth_highpass.py b/src/sampletones_tools/synthesis/filters/butterworth_highpass.py similarity index 100% rename from src/sampletones_synthesis/filters/butterworth_highpass.py rename to src/sampletones_tools/synthesis/filters/butterworth_highpass.py diff --git a/src/sampletones_synthesis/filters/types.py b/src/sampletones_tools/synthesis/filters/types.py similarity index 100% rename from src/sampletones_synthesis/filters/types.py rename to src/sampletones_tools/synthesis/filters/types.py diff --git a/src/sampletones_synthesis/frequency.py b/src/sampletones_tools/synthesis/frequency.py similarity index 100% rename from src/sampletones_synthesis/frequency.py rename to src/sampletones_tools/synthesis/frequency.py diff --git a/src/sampletones_synthesis/oscillators/__init__.py b/src/sampletones_tools/synthesis/oscillators/__init__.py similarity index 100% rename from src/sampletones_synthesis/oscillators/__init__.py rename to src/sampletones_tools/synthesis/oscillators/__init__.py diff --git a/src/sampletones_synthesis/oscillators/exponential_glide.py b/src/sampletones_tools/synthesis/oscillators/exponential_glide.py similarity index 95% rename from src/sampletones_synthesis/oscillators/exponential_glide.py rename to src/sampletones_tools/synthesis/oscillators/exponential_glide.py index 31254045a..fef55d880 100644 --- a/src/sampletones_synthesis/oscillators/exponential_glide.py +++ b/src/sampletones_tools/synthesis/oscillators/exponential_glide.py @@ -3,7 +3,7 @@ import numpy as np from pydantic import BaseModel, ConfigDict, Field -from sampletones_synthesis.frequency import FrequencySpec, resolve_frequency +from sampletones_tools.synthesis.frequency import FrequencySpec, resolve_frequency class ExponentialGlideOscillator(BaseModel): diff --git a/src/sampletones_synthesis/oscillators/geometric_sweep.py b/src/sampletones_tools/synthesis/oscillators/geometric_sweep.py similarity index 95% rename from src/sampletones_synthesis/oscillators/geometric_sweep.py rename to src/sampletones_tools/synthesis/oscillators/geometric_sweep.py index db207f3c8..8fdd899c7 100644 --- a/src/sampletones_synthesis/oscillators/geometric_sweep.py +++ b/src/sampletones_tools/synthesis/oscillators/geometric_sweep.py @@ -3,7 +3,7 @@ import numpy as np from pydantic import BaseModel, ConfigDict -from sampletones_synthesis.frequency import FrequencySpec, resolve_frequency +from sampletones_tools.synthesis.frequency import FrequencySpec, resolve_frequency class GeometricSweepOscillator(BaseModel): diff --git a/src/sampletones_synthesis/oscillators/pulse.py b/src/sampletones_tools/synthesis/oscillators/pulse.py similarity index 93% rename from src/sampletones_synthesis/oscillators/pulse.py rename to src/sampletones_tools/synthesis/oscillators/pulse.py index 8bb5e90d4..bfaca5790 100644 --- a/src/sampletones_synthesis/oscillators/pulse.py +++ b/src/sampletones_tools/synthesis/oscillators/pulse.py @@ -3,7 +3,7 @@ import numpy as np from pydantic import BaseModel, ConfigDict, Field -from sampletones_synthesis.frequency import FrequencySpec, resolve_frequency +from sampletones_tools.synthesis.frequency import FrequencySpec, resolve_frequency DutyCycle = Annotated[float, Field(gt=0.0, lt=1.0)] diff --git a/src/sampletones_synthesis/oscillators/sine.py b/src/sampletones_tools/synthesis/oscillators/sine.py similarity index 91% rename from src/sampletones_synthesis/oscillators/sine.py rename to src/sampletones_tools/synthesis/oscillators/sine.py index 3e2e96ed3..9abeb685e 100644 --- a/src/sampletones_synthesis/oscillators/sine.py +++ b/src/sampletones_tools/synthesis/oscillators/sine.py @@ -3,7 +3,7 @@ import numpy as np from pydantic import BaseModel, ConfigDict -from sampletones_synthesis.frequency import FrequencySpec, resolve_frequency +from sampletones_tools.synthesis.frequency import FrequencySpec, resolve_frequency class SineOscillator(BaseModel): diff --git a/src/sampletones_synthesis/oscillators/types.py b/src/sampletones_tools/synthesis/oscillators/types.py similarity index 100% rename from src/sampletones_synthesis/oscillators/types.py rename to src/sampletones_tools/synthesis/oscillators/types.py diff --git a/src/sampletones_synthesis/oscillators/walk_noise.py b/src/sampletones_tools/synthesis/oscillators/walk_noise.py similarity index 100% rename from src/sampletones_synthesis/oscillators/walk_noise.py rename to src/sampletones_tools/synthesis/oscillators/walk_noise.py diff --git a/src/sampletones_synthesis/oscillators/white_noise.py b/src/sampletones_tools/synthesis/oscillators/white_noise.py similarity index 100% rename from src/sampletones_synthesis/oscillators/white_noise.py rename to src/sampletones_tools/synthesis/oscillators/white_noise.py diff --git a/src/sampletones_synthesis/protocols.py b/src/sampletones_tools/synthesis/protocols.py similarity index 100% rename from src/sampletones_synthesis/protocols.py rename to src/sampletones_tools/synthesis/protocols.py diff --git a/src/sampletones_synthesis/py.typed b/src/sampletones_tools/synthesis/py.typed similarity index 100% rename from src/sampletones_synthesis/py.typed rename to src/sampletones_tools/synthesis/py.typed diff --git a/src/sampletones_synthesis/voice/__init__.py b/src/sampletones_tools/synthesis/voice/__init__.py similarity index 100% rename from src/sampletones_synthesis/voice/__init__.py rename to src/sampletones_tools/synthesis/voice/__init__.py diff --git a/src/sampletones_synthesis/voice/layer.py b/src/sampletones_tools/synthesis/voice/layer.py similarity index 90% rename from src/sampletones_synthesis/voice/layer.py rename to src/sampletones_tools/synthesis/voice/layer.py index 31b8429d7..5ea85b401 100644 --- a/src/sampletones_synthesis/voice/layer.py +++ b/src/sampletones_tools/synthesis/voice/layer.py @@ -3,8 +3,8 @@ import numpy as np from pydantic import BaseModel, ConfigDict, Field -from sampletones_synthesis.envelopes.types import EnvelopeUnion -from sampletones_synthesis.oscillators.types import OscillatorUnion +from sampletones_tools.synthesis.envelopes.types import EnvelopeUnion +from sampletones_tools.synthesis.oscillators.types import OscillatorUnion class Layer(BaseModel): diff --git a/src/sampletones_synthesis/voice/voice.py b/src/sampletones_tools/synthesis/voice/voice.py similarity index 97% rename from src/sampletones_synthesis/voice/voice.py rename to src/sampletones_tools/synthesis/voice/voice.py index 968502c5a..89187ebcd 100644 --- a/src/sampletones_synthesis/voice/voice.py +++ b/src/sampletones_tools/synthesis/voice/voice.py @@ -3,7 +3,7 @@ import numpy as np from pydantic import BaseModel, ConfigDict, Field -from sampletones_synthesis.filters.types import FilterUnion +from sampletones_tools.synthesis.filters.types import FilterUnion from .layer import Layer diff --git a/tests/integration/assets/synth_config.py b/tests/integration/assets/synth_config.py index 1cff14660..971287fd2 100644 --- a/tests/integration/assets/synth_config.py +++ b/tests/integration/assets/synth_config.py @@ -4,7 +4,7 @@ from sampletones_shared.types.path import Pathlike from sampletones_shared.utils.serialization import load_yaml_model -from sampletones_synthesis.voice.voice import Voice +from sampletones_tools.synthesis.voice.voice import Voice class SynthConfig(BaseModel): diff --git a/tests/unit/sampletones_shared/meta/import_boundary/configs/test_rules.py b/tests/unit/sampletones_shared/meta/import_boundary/configs/test_rules.py index f01e747b0..31f35c450 100644 --- a/tests/unit/sampletones_shared/meta/import_boundary/configs/test_rules.py +++ b/tests/unit/sampletones_shared/meta/import_boundary/configs/test_rules.py @@ -69,10 +69,6 @@ def test_the_reconstruction_engine_stays_clear_of_the_console_player(self) -> No def test_the_console_player_reads_the_reconstruction_engine(self) -> None: assert CORE in self.LAYERS[PLAYER] - def test_the_synthesis_package_stands_below_the_reconstruction_engine(self) -> None: - """Equal temperament sits in `sampletones_shared`, so synthesis reaches no engine module.""" - assert CORE not in reached_units(self.LAYERS, "sampletones_synthesis") - def test_the_entry_is_the_one_importer_of_the_tools(self) -> None: """The wheel carries the tools, and the command line is where a developer reaches them.""" assert {unit for unit, layers in self.LAYERS.items() if TOOLS in layers} == {ENTRY} diff --git a/tests/unit/sampletones_core/calibration/__init__.py b/tests/unit/sampletones_tools/calibration/__init__.py similarity index 100% rename from tests/unit/sampletones_core/calibration/__init__.py rename to tests/unit/sampletones_tools/calibration/__init__.py diff --git a/tests/unit/sampletones_core/calibration/config/__init__.py b/tests/unit/sampletones_tools/calibration/config/__init__.py similarity index 100% rename from tests/unit/sampletones_core/calibration/config/__init__.py rename to tests/unit/sampletones_tools/calibration/config/__init__.py diff --git a/tests/unit/sampletones_core/calibration/config/test_corpus.py b/tests/unit/sampletones_tools/calibration/config/test_corpus.py similarity index 98% rename from tests/unit/sampletones_core/calibration/config/test_corpus.py rename to tests/unit/sampletones_tools/calibration/config/test_corpus.py index b09e60af6..4d2dae139 100644 --- a/tests/unit/sampletones_core/calibration/config/test_corpus.py +++ b/tests/unit/sampletones_tools/calibration/config/test_corpus.py @@ -4,7 +4,7 @@ import pytest from pydantic import ValidationError -from sampletones_core.calibration.config.corpus import CorpusConfig +from sampletones_tools.calibration.config.corpus import CorpusConfig from tests.suite.case import BaseRegularTestCase VALID_TRANSIENT: Final[Dict[str, Any]] = { diff --git a/tests/unit/sampletones_core/calibration/config/test_referee.py b/tests/unit/sampletones_tools/calibration/config/test_referee.py similarity index 96% rename from tests/unit/sampletones_core/calibration/config/test_referee.py rename to tests/unit/sampletones_tools/calibration/config/test_referee.py index 21120e5ad..eb5659cf3 100644 --- a/tests/unit/sampletones_core/calibration/config/test_referee.py +++ b/tests/unit/sampletones_tools/calibration/config/test_referee.py @@ -4,7 +4,7 @@ import pytest from pydantic import ValidationError -from sampletones_core.calibration.config.referee import RefereeConfig +from sampletones_tools.calibration.config.referee import RefereeConfig from tests.suite.case import BaseRegularTestCase VALID_FIELDS: Final[Dict[str, Any]] = { diff --git a/tests/unit/sampletones_core/calibration/corpus/__init__.py b/tests/unit/sampletones_tools/calibration/corpus/__init__.py similarity index 100% rename from tests/unit/sampletones_core/calibration/corpus/__init__.py rename to tests/unit/sampletones_tools/calibration/corpus/__init__.py diff --git a/tests/unit/sampletones_core/calibration/corpus/conftest.py b/tests/unit/sampletones_tools/calibration/corpus/conftest.py similarity index 68% rename from tests/unit/sampletones_core/calibration/corpus/conftest.py rename to tests/unit/sampletones_tools/calibration/corpus/conftest.py index db4080c0d..6a89d2862 100644 --- a/tests/unit/sampletones_core/calibration/corpus/conftest.py +++ b/tests/unit/sampletones_tools/calibration/corpus/conftest.py @@ -2,9 +2,9 @@ import pytest -from sampletones_core.calibration.config.corpus import CorpusConfig -from sampletones_core.calibration.corpus.item import CorpusItem -from sampletones_core.calibration.corpus.synthesis import build_corpus +from sampletones_tools.calibration.config.corpus import CorpusConfig +from sampletones_tools.calibration.corpus.item import CorpusItem +from sampletones_tools.calibration.corpus.synthesis import build_corpus SAMPLE_RATE: Final[int] = 22050 diff --git a/tests/unit/sampletones_core/calibration/corpus/test_synthesis.py b/tests/unit/sampletones_tools/calibration/corpus/test_synthesis.py similarity index 87% rename from tests/unit/sampletones_core/calibration/corpus/test_synthesis.py rename to tests/unit/sampletones_tools/calibration/corpus/test_synthesis.py index 38350b1cc..45cd11514 100644 --- a/tests/unit/sampletones_core/calibration/corpus/test_synthesis.py +++ b/tests/unit/sampletones_tools/calibration/corpus/test_synthesis.py @@ -2,9 +2,9 @@ import numpy as np -from sampletones_core.calibration.config.corpus import CorpusConfig -from sampletones_core.calibration.corpus.item import CorpusItem -from sampletones_core.calibration.corpus.synthesis import build_corpus +from sampletones_tools.calibration.config.corpus import CorpusConfig +from sampletones_tools.calibration.corpus.item import CorpusItem +from sampletones_tools.calibration.corpus.synthesis import build_corpus EXPECTED_CATEGORIES: Final[FrozenSet[str]] = frozenset({"tone", "timbre", "noise", "mix", "transient", "dynamics"}) diff --git a/tests/unit/sampletones_core/calibration/corpus/test_writer.py b/tests/unit/sampletones_tools/calibration/corpus/test_writer.py similarity index 78% rename from tests/unit/sampletones_core/calibration/corpus/test_writer.py rename to tests/unit/sampletones_tools/calibration/corpus/test_writer.py index 0520366e3..9d1873d65 100644 --- a/tests/unit/sampletones_core/calibration/corpus/test_writer.py +++ b/tests/unit/sampletones_tools/calibration/corpus/test_writer.py @@ -1,8 +1,8 @@ from pathlib import Path from typing import List -from sampletones_core.calibration.corpus.item import CorpusItem -from sampletones_core.calibration.corpus.writer import write_corpus +from sampletones_tools.calibration.corpus.item import CorpusItem +from sampletones_tools.calibration.corpus.writer import write_corpus class TestWriteCorpus: diff --git a/tests/unit/sampletones_core/calibration/referee/__init__.py b/tests/unit/sampletones_tools/calibration/referee/__init__.py similarity index 100% rename from tests/unit/sampletones_core/calibration/referee/__init__.py rename to tests/unit/sampletones_tools/calibration/referee/__init__.py diff --git a/tests/unit/sampletones_core/calibration/referee/test_auditory.py b/tests/unit/sampletones_tools/calibration/referee/test_auditory.py similarity index 94% rename from tests/unit/sampletones_core/calibration/referee/test_auditory.py rename to tests/unit/sampletones_tools/calibration/referee/test_auditory.py index 7581f7804..63a20e856 100644 --- a/tests/unit/sampletones_core/calibration/referee/test_auditory.py +++ b/tests/unit/sampletones_tools/calibration/referee/test_auditory.py @@ -3,8 +3,8 @@ import numpy as np import pytest -from sampletones_core.calibration.config.referee import RefereeConfig -from sampletones_core.calibration.referee.auditory import MultiResolutionAuditoryReferee +from sampletones_tools.calibration.config.referee import RefereeConfig +from sampletones_tools.calibration.referee.auditory import MultiResolutionAuditoryReferee SAMPLE_RATE: Final[int] = 22050 SIGNAL_SECONDS: Final[float] = 1.0 diff --git a/tests/unit/sampletones_tools/calibration/test_command.py b/tests/unit/sampletones_tools/calibration/test_command.py new file mode 100644 index 000000000..58c013354 --- /dev/null +++ b/tests/unit/sampletones_tools/calibration/test_command.py @@ -0,0 +1,83 @@ +from pathlib import Path +from typing import List + +import pytest + +from sampletones.commands.registry import COMMANDS +from sampletones.dispatcher import dispatch +from sampletones_core.configs import Config +from sampletones_core.constants.enums import DEFAULT_CHANNELS, ChannelName, SpectrumMethod +from sampletones_shared.paths.user import USER_PATH_DOCUMENTS +from sampletones_tools.calibration.session import CalibrationRequest + +CALIBRATE = "sampletones_tools.calibration.session.calibrate" +LOADER = "sampletones_core.headless.config.load_config" + + +class RecordedCalibration: + def __init__(self) -> None: + self.requests: List[CalibrationRequest] = [] + + def __call__(self, request: CalibrationRequest) -> Path: + self.requests.append(request) + return request.output + + +@pytest.fixture(name="calibration") +def calibration_fixture(monkeypatch: pytest.MonkeyPatch) -> RecordedCalibration: + recorded = RecordedCalibration() + monkeypatch.setattr(CALIBRATE, recorded) + monkeypatch.setattr(LOADER, lambda path: Config()) + return recorded + + +class TestCalibration: + def test_the_sweep_is_read_from_the_options(self, calibration: RecordedCalibration, tmp_path: Path) -> None: + status = dispatch( + COMMANDS, + [ + "calibration", + "--methods", + "fft", + "--perceptual-exponents", + "0.5,1", + "--temporal-weights", + "0.25", + "--channels", + "pulse1", + "-o", + str(tmp_path / "run"), + ], + ) + + assert status == 0 + request = calibration.requests[0] + assert request.methods == [SpectrumMethod.FFT] + assert request.perceptual_exponents == [0.5, 1.0] + assert request.temporal_weights == [0.25] + assert request.channels == [ChannelName.PULSE1] + assert request.output == tmp_path / "run" + + def test_without_options_the_run_sweeps_both_methods_into_the_documents( + self, + calibration: RecordedCalibration, + ) -> None: + assert dispatch(COMMANDS, ["calibration"]) == 0 + request = calibration.requests[0] + assert request.methods == [SpectrumMethod.FFT, SpectrumMethod.CQT] + assert request.perceptual_exponents == [1.0] + assert request.temporal_weights == [] + assert request.channels == list(DEFAULT_CHANNELS) + assert request.output.parent == USER_PATH_DOCUMENTS / "calibration" + + def test_an_unknown_method_is_refused(self, calibration: RecordedCalibration) -> None: + with pytest.raises(SystemExit, match="Unknown spectrum method"): + dispatch(COMMANDS, ["calibration", "--methods", "dct"]) + + assert calibration.requests == [] + + def test_a_value_that_is_no_number_is_refused(self, calibration: RecordedCalibration) -> None: + with pytest.raises(SystemExit, match="Not a number"): + dispatch(COMMANDS, ["calibration", "--perceptual-exponents", "high"]) + + assert calibration.requests == [] diff --git a/tests/unit/sampletones_core/calibration/test_runner.py b/tests/unit/sampletones_tools/calibration/test_runner.py similarity index 95% rename from tests/unit/sampletones_core/calibration/test_runner.py rename to tests/unit/sampletones_tools/calibration/test_runner.py index 2eb447e31..329c1dbcb 100644 --- a/tests/unit/sampletones_core/calibration/test_runner.py +++ b/tests/unit/sampletones_tools/calibration/test_runner.py @@ -3,9 +3,9 @@ import pytest -from sampletones_core.calibration.runner import build_variants from sampletones_core.configs import Config from sampletones_core.constants.enums import SpectrumMethod +from sampletones_tools.calibration.runner import build_variants METHODS: Final[List[SpectrumMethod]] = [SpectrumMethod.FFT, SpectrumMethod.CQT] EXPONENTS: Final[List[float]] = [1.0] diff --git a/tests/unit/sampletones_tools/calibration/test_session.py b/tests/unit/sampletones_tools/calibration/test_session.py new file mode 100644 index 000000000..0870b333e --- /dev/null +++ b/tests/unit/sampletones_tools/calibration/test_session.py @@ -0,0 +1,71 @@ +from pathlib import Path + +import pytest + +from sampletones_core.configs import Config +from sampletones_core.constants.enums import ChannelName, SpectrumMethod +from sampletones_shared.paths.user import USER_PATH_DOCUMENTS +from sampletones_tools.calibration.session import ( + CalibrationRequest, + default_output, + floats_named, + methods_named, +) + + +class TestMethodsNamed: + def test_nothing_named_is_fft_and_cqt(self) -> None: + assert methods_named(None) == [SpectrumMethod.FFT, SpectrumMethod.CQT] + + def test_names_are_read_in_order(self) -> None: + assert methods_named("cqt, fft") == [SpectrumMethod.CQT, SpectrumMethod.FFT] + + def test_an_unknown_method_is_refused_with_the_known_ones(self) -> None: + with pytest.raises(ValueError, match="Unknown spectrum method 'dct'; the methods are"): + methods_named("fft,dct") + + +class TestFloatsNamed: + def test_nothing_named_is_the_default(self) -> None: + assert floats_named(None, (1.0,)) == [1.0] + assert floats_named(None, ()) == [] + + def test_values_are_read_in_order(self) -> None: + assert floats_named("0.5, 1", ()) == [0.5, 1.0] + + def test_a_value_that_is_no_number_is_refused(self) -> None: + with pytest.raises(ValueError, match="Not a number: 'high'"): + floats_named("0.5,high", ()) + + +class TestDefaultOutput: + def test_a_run_lands_in_a_timestamped_directory_under_the_documents(self) -> None: + output = default_output() + + assert output.parent == USER_PATH_DOCUMENTS / "calibration" + assert output.name.startswith("run-") + + +class TestCalibrationRequest: + def test_the_base_is_pinned_to_the_channels(self, tmp_path: Path) -> None: + request = CalibrationRequest( + base=Config(), + output=tmp_path, + methods=[SpectrumMethod.FFT], + perceptual_exponents=[1.0], + temporal_weights=[], + channels=[ChannelName.PULSE1], + ) + + assert request.pinned_base().generation.channels == [ChannelName.PULSE1] + + def test_an_empty_sweep_is_refused(self, tmp_path: Path) -> None: + with pytest.raises(ValueError): + CalibrationRequest( + base=Config(), + output=tmp_path, + methods=[], + perceptual_exponents=[1.0], + temporal_weights=[], + channels=[ChannelName.PULSE1], + ) diff --git a/tests/unit/sampletones_synthesis/__init__.py b/tests/unit/sampletones_tools/synthesis/__init__.py similarity index 100% rename from tests/unit/sampletones_synthesis/__init__.py rename to tests/unit/sampletones_tools/synthesis/__init__.py diff --git a/tests/unit/sampletones_synthesis/conftest.py b/tests/unit/sampletones_tools/synthesis/conftest.py similarity index 100% rename from tests/unit/sampletones_synthesis/conftest.py rename to tests/unit/sampletones_tools/synthesis/conftest.py diff --git a/tests/unit/sampletones_synthesis/envelopes/__init__.py b/tests/unit/sampletones_tools/synthesis/envelopes/__init__.py similarity index 100% rename from tests/unit/sampletones_synthesis/envelopes/__init__.py rename to tests/unit/sampletones_tools/synthesis/envelopes/__init__.py diff --git a/tests/unit/sampletones_synthesis/envelopes/test_envelopes.py b/tests/unit/sampletones_tools/synthesis/envelopes/test_envelopes.py similarity index 87% rename from tests/unit/sampletones_synthesis/envelopes/test_envelopes.py rename to tests/unit/sampletones_tools/synthesis/envelopes/test_envelopes.py index 2984acaf7..957c37ba3 100644 --- a/tests/unit/sampletones_synthesis/envelopes/test_envelopes.py +++ b/tests/unit/sampletones_tools/synthesis/envelopes/test_envelopes.py @@ -3,9 +3,9 @@ import numpy as np import pytest -from sampletones_synthesis.envelopes.exponential_decay import ExponentialDecayEnvelope -from sampletones_synthesis.envelopes.linear_attack import LinearAttackEnvelope -from sampletones_synthesis.envelopes.linear_ramp import LinearRampEnvelope +from sampletones_tools.synthesis.envelopes.exponential_decay import ExponentialDecayEnvelope +from sampletones_tools.synthesis.envelopes.linear_attack import LinearAttackEnvelope +from sampletones_tools.synthesis.envelopes.linear_ramp import LinearRampEnvelope TIME_CONSTANT_SECONDS: Final[float] = 0.25 ATTACK_SECONDS: Final[float] = 0.005 diff --git a/tests/unit/sampletones_synthesis/filters/__init__.py b/tests/unit/sampletones_tools/synthesis/filters/__init__.py similarity index 100% rename from tests/unit/sampletones_synthesis/filters/__init__.py rename to tests/unit/sampletones_tools/synthesis/filters/__init__.py diff --git a/tests/unit/sampletones_synthesis/filters/test_butterworth_highpass.py b/tests/unit/sampletones_tools/synthesis/filters/test_butterworth_highpass.py similarity index 90% rename from tests/unit/sampletones_synthesis/filters/test_butterworth_highpass.py rename to tests/unit/sampletones_tools/synthesis/filters/test_butterworth_highpass.py index 90e2623fa..3c4ddc7ae 100644 --- a/tests/unit/sampletones_synthesis/filters/test_butterworth_highpass.py +++ b/tests/unit/sampletones_tools/synthesis/filters/test_butterworth_highpass.py @@ -2,7 +2,7 @@ import numpy as np -from sampletones_synthesis.filters.butterworth_highpass import ButterworthHighpassFilter +from sampletones_tools.synthesis.filters.butterworth_highpass import ButterworthHighpassFilter CUTOFF_HZ: Final[float] = 2000.0 ORDER: Final[int] = 4 diff --git a/tests/unit/sampletones_synthesis/oscillators/__init__.py b/tests/unit/sampletones_tools/synthesis/oscillators/__init__.py similarity index 100% rename from tests/unit/sampletones_synthesis/oscillators/__init__.py rename to tests/unit/sampletones_tools/synthesis/oscillators/__init__.py diff --git a/tests/unit/sampletones_synthesis/oscillators/test_noise.py b/tests/unit/sampletones_tools/synthesis/oscillators/test_noise.py similarity index 89% rename from tests/unit/sampletones_synthesis/oscillators/test_noise.py rename to tests/unit/sampletones_tools/synthesis/oscillators/test_noise.py index 727df82a5..23fa37c32 100644 --- a/tests/unit/sampletones_synthesis/oscillators/test_noise.py +++ b/tests/unit/sampletones_tools/synthesis/oscillators/test_noise.py @@ -3,8 +3,8 @@ import numpy as np import pytest -from sampletones_synthesis.oscillators.walk_noise import WalkNoiseOscillator -from sampletones_synthesis.oscillators.white_noise import WhiteNoiseOscillator +from sampletones_tools.synthesis.oscillators.walk_noise import WalkNoiseOscillator +from sampletones_tools.synthesis.oscillators.white_noise import WhiteNoiseOscillator SEED: Final[int] = 99 diff --git a/tests/unit/sampletones_synthesis/oscillators/test_pulse.py b/tests/unit/sampletones_tools/synthesis/oscillators/test_pulse.py similarity index 93% rename from tests/unit/sampletones_synthesis/oscillators/test_pulse.py rename to tests/unit/sampletones_tools/synthesis/oscillators/test_pulse.py index bb4c3c707..c8d6f7c11 100644 --- a/tests/unit/sampletones_synthesis/oscillators/test_pulse.py +++ b/tests/unit/sampletones_tools/synthesis/oscillators/test_pulse.py @@ -3,7 +3,7 @@ import numpy as np import pytest -from sampletones_synthesis.oscillators.pulse import PulseOscillator +from sampletones_tools.synthesis.oscillators.pulse import PulseOscillator FREQUENCY: Final[float] = 220.0 DUTY_CYCLES: Final[Tuple[float, ...]] = (0.125, 0.25, 0.5) diff --git a/tests/unit/sampletones_synthesis/oscillators/test_sine.py b/tests/unit/sampletones_tools/synthesis/oscillators/test_sine.py similarity index 95% rename from tests/unit/sampletones_synthesis/oscillators/test_sine.py rename to tests/unit/sampletones_tools/synthesis/oscillators/test_sine.py index d015852f2..cd5393e30 100644 --- a/tests/unit/sampletones_synthesis/oscillators/test_sine.py +++ b/tests/unit/sampletones_tools/synthesis/oscillators/test_sine.py @@ -3,7 +3,7 @@ import numpy as np import pytest -from sampletones_synthesis.oscillators.sine import SineOscillator +from sampletones_tools.synthesis.oscillators.sine import SineOscillator FREQUENCY: Final[float] = 440.0 diff --git a/tests/unit/sampletones_synthesis/oscillators/test_sweeps.py b/tests/unit/sampletones_tools/synthesis/oscillators/test_sweeps.py similarity index 90% rename from tests/unit/sampletones_synthesis/oscillators/test_sweeps.py rename to tests/unit/sampletones_tools/synthesis/oscillators/test_sweeps.py index 49c1a351a..25667dfa0 100644 --- a/tests/unit/sampletones_synthesis/oscillators/test_sweeps.py +++ b/tests/unit/sampletones_tools/synthesis/oscillators/test_sweeps.py @@ -5,10 +5,10 @@ import pytest from scipy.integrate import cumulative_trapezoid -from sampletones_synthesis.oscillators.exponential_glide import ExponentialGlideOscillator -from sampletones_synthesis.oscillators.geometric_sweep import GeometricSweepOscillator -from sampletones_synthesis.oscillators.sine import SineOscillator -from sampletones_synthesis.protocols import Oscillator +from sampletones_tools.synthesis.oscillators.exponential_glide import ExponentialGlideOscillator +from sampletones_tools.synthesis.oscillators.geometric_sweep import GeometricSweepOscillator +from sampletones_tools.synthesis.oscillators.sine import SineOscillator +from sampletones_tools.synthesis.protocols import Oscillator FREQUENCY_START: Final[float] = 392.0 FREQUENCY_END: Final[float] = 44.0 diff --git a/tests/unit/sampletones_synthesis/test_frequency.py b/tests/unit/sampletones_tools/synthesis/test_frequency.py similarity index 95% rename from tests/unit/sampletones_synthesis/test_frequency.py rename to tests/unit/sampletones_tools/synthesis/test_frequency.py index 314c0b5b2..b408ae398 100644 --- a/tests/unit/sampletones_synthesis/test_frequency.py +++ b/tests/unit/sampletones_tools/synthesis/test_frequency.py @@ -4,7 +4,7 @@ import pytest from pydantic import TypeAdapter, ValidationError -from sampletones_synthesis.frequency import FrequencySpec, resolve_frequency +from sampletones_tools.synthesis.frequency import FrequencySpec, resolve_frequency FREQUENCY_SPEC_ADAPTER: Final[TypeAdapter[Any]] = TypeAdapter(FrequencySpec) diff --git a/tests/unit/sampletones_synthesis/test_unions.py b/tests/unit/sampletones_tools/synthesis/test_unions.py similarity index 79% rename from tests/unit/sampletones_synthesis/test_unions.py rename to tests/unit/sampletones_tools/synthesis/test_unions.py index 1b70adaa3..21a6bb4e7 100644 --- a/tests/unit/sampletones_synthesis/test_unions.py +++ b/tests/unit/sampletones_tools/synthesis/test_unions.py @@ -4,17 +4,17 @@ import pytest from pydantic import BaseModel, TypeAdapter, ValidationError -from sampletones_synthesis.envelopes.exponential_decay import ExponentialDecayEnvelope -from sampletones_synthesis.envelopes.linear_attack import LinearAttackEnvelope -from sampletones_synthesis.envelopes.linear_ramp import LinearRampEnvelope -from sampletones_synthesis.envelopes.types import EnvelopeUnion -from sampletones_synthesis.oscillators.exponential_glide import ExponentialGlideOscillator -from sampletones_synthesis.oscillators.geometric_sweep import GeometricSweepOscillator -from sampletones_synthesis.oscillators.pulse import PulseOscillator -from sampletones_synthesis.oscillators.sine import SineOscillator -from sampletones_synthesis.oscillators.types import OscillatorUnion -from sampletones_synthesis.oscillators.walk_noise import WalkNoiseOscillator -from sampletones_synthesis.oscillators.white_noise import WhiteNoiseOscillator +from sampletones_tools.synthesis.envelopes.exponential_decay import ExponentialDecayEnvelope +from sampletones_tools.synthesis.envelopes.linear_attack import LinearAttackEnvelope +from sampletones_tools.synthesis.envelopes.linear_ramp import LinearRampEnvelope +from sampletones_tools.synthesis.envelopes.types import EnvelopeUnion +from sampletones_tools.synthesis.oscillators.exponential_glide import ExponentialGlideOscillator +from sampletones_tools.synthesis.oscillators.geometric_sweep import GeometricSweepOscillator +from sampletones_tools.synthesis.oscillators.pulse import PulseOscillator +from sampletones_tools.synthesis.oscillators.sine import SineOscillator +from sampletones_tools.synthesis.oscillators.types import OscillatorUnion +from sampletones_tools.synthesis.oscillators.walk_noise import WalkNoiseOscillator +from sampletones_tools.synthesis.oscillators.white_noise import WhiteNoiseOscillator OSCILLATOR_ADAPTER: Final[TypeAdapter[Any]] = TypeAdapter(OscillatorUnion) ENVELOPE_ADAPTER: Final[TypeAdapter[Any]] = TypeAdapter(EnvelopeUnion) diff --git a/tests/unit/sampletones_synthesis/voice/__init__.py b/tests/unit/sampletones_tools/synthesis/voice/__init__.py similarity index 100% rename from tests/unit/sampletones_synthesis/voice/__init__.py rename to tests/unit/sampletones_tools/synthesis/voice/__init__.py diff --git a/tests/unit/sampletones_synthesis/voice/test_layer.py b/tests/unit/sampletones_tools/synthesis/voice/test_layer.py similarity index 86% rename from tests/unit/sampletones_synthesis/voice/test_layer.py rename to tests/unit/sampletones_tools/synthesis/voice/test_layer.py index 92ee232e1..70237aa10 100644 --- a/tests/unit/sampletones_synthesis/voice/test_layer.py +++ b/tests/unit/sampletones_tools/synthesis/voice/test_layer.py @@ -3,10 +3,10 @@ import numpy as np import pytest -from sampletones_synthesis.envelopes.exponential_decay import ExponentialDecayEnvelope -from sampletones_synthesis.envelopes.linear_attack import LinearAttackEnvelope -from sampletones_synthesis.oscillators.sine import SineOscillator -from sampletones_synthesis.voice.layer import Layer +from sampletones_tools.synthesis.envelopes.exponential_decay import ExponentialDecayEnvelope +from sampletones_tools.synthesis.envelopes.linear_attack import LinearAttackEnvelope +from sampletones_tools.synthesis.oscillators.sine import SineOscillator +from sampletones_tools.synthesis.voice.layer import Layer FREQUENCY: Final[float] = 220.0 TIME_CONSTANT_SECONDS: Final[float] = 0.3 diff --git a/tests/unit/sampletones_synthesis/voice/test_voice.py b/tests/unit/sampletones_tools/synthesis/voice/test_voice.py similarity index 94% rename from tests/unit/sampletones_synthesis/voice/test_voice.py rename to tests/unit/sampletones_tools/synthesis/voice/test_voice.py index 6323253a1..e21cd3213 100644 --- a/tests/unit/sampletones_synthesis/voice/test_voice.py +++ b/tests/unit/sampletones_tools/synthesis/voice/test_voice.py @@ -4,9 +4,9 @@ import pytest from pydantic import ValidationError -from sampletones_synthesis.oscillators.sine import SineOscillator -from sampletones_synthesis.voice.layer import Layer -from sampletones_synthesis.voice.voice import Voice +from sampletones_tools.synthesis.oscillators.sine import SineOscillator +from sampletones_tools.synthesis.voice.layer import Layer +from sampletones_tools.synthesis.voice.voice import Voice DURATION_SECONDS: Final[float] = 0.5 LOW_FREQUENCY: Final[float] = 220.0 From a61586e1682ddf82a97fc7ff8f3e4b2ca62c0748 Mon Sep 17 00:00:00 2001 From: JakimPL Date: Sun, 13 Sep 2026 15:50:13 +0200 Subject: [PATCH 12/36] Moved: the driver toolchain and the trace into the tools package --- Makefile | 10 +- docs/development/dependencies.md | 21 ++-- docs/development/packages.md | 17 +-- docs/development/player.md | 10 +- docs/development/tooling.md | 7 +- docs/formats/nsf.md | 2 +- pyproject.toml | 4 - scripts/player.py | 43 ------- src/sampletones_config/boundaries/graphs.yaml | 2 - .../boundaries/standalone.yaml | 2 - .../driver/assembler/layout.py | 14 --- src/sampletones_player/driver/image.py | 2 +- .../player}/__init__.py | 0 .../player/assembler}/__init__.py | 0 .../player}/assembler/builder.py | 26 +++-- .../player}/assembler/labels.py | 0 .../player/assembler/layout.py | 19 +++ .../player/assembler/report.py | 15 +++ .../player}/assembler/toolchain.py | 0 .../player/assembly}/__init__.py | 0 .../player}/assembly/include/nes.inc | 0 .../player}/assembly/include/song.inc | 0 .../player}/assembly/nsf.cfg | 0 .../player}/assembly/source/channels.s | 0 .../player}/assembly/source/clock.s | 0 .../player}/assembly/source/driver.s | 0 src/sampletones_tools/player/command.py | 54 +++++++++ .../player}/trace/__init__.py | 0 .../player}/trace/trace.py | 2 +- .../player}/trace/write.py | 0 src/sampletones_tools/registry.py | 4 +- src/sampletones_tools/samples/__init__.py | 0 src/sampletones_tools/samples/command.py | 55 +++++++++ .../sampletones_tools/samples/render.py | 110 +++++++++--------- tests/integration/nsf/console/instructions.py | 2 +- tests/integration/nsf/console/machine.py | 4 +- tests/integration/nsf/console/session.py | 2 +- tests/integration/nsf/test_driver_bend.py | 2 +- tests/integration/nsf/test_driver_trace.py | 2 +- .../import_boundary/configs/test_rules.py | 17 --- .../player/assembler/__init__.py | 0 .../player}/assembler/test_builder.py | 6 +- .../player}/assembler/test_labels.py | 6 +- .../player/assembler/test_report.py | 19 +++ .../player}/assembler/test_toolchain.py | 6 +- .../sampletones_tools/player/test_command.py | 65 +++++++++++ .../player}/test_song_include.py | 7 +- .../player/trace/__init__.py | 0 .../player}/trace/test_trace.py | 4 +- .../sampletones_tools/samples/test_command.py | 51 ++++++++ .../sampletones_tools/samples/test_render.py | 44 +++++++ 51 files changed, 451 insertions(+), 205 deletions(-) delete mode 100755 scripts/player.py delete mode 100644 src/sampletones_player/driver/assembler/layout.py rename src/{sampletones_player/driver/assembler => sampletones_tools/player}/__init__.py (100%) rename src/{sampletones_player/trace => sampletones_tools/player/assembler}/__init__.py (100%) rename src/{sampletones_player/driver => sampletones_tools/player}/assembler/builder.py (81%) rename src/{sampletones_player/driver => sampletones_tools/player}/assembler/labels.py (100%) create mode 100644 src/sampletones_tools/player/assembler/layout.py create mode 100644 src/sampletones_tools/player/assembler/report.py rename src/{sampletones_player/driver => sampletones_tools/player}/assembler/toolchain.py (100%) rename {tests/unit/sampletones_player/driver/assembler => src/sampletones_tools/player/assembly}/__init__.py (100%) rename src/{sampletones_player/driver => sampletones_tools/player}/assembly/include/nes.inc (100%) rename src/{sampletones_player/driver => sampletones_tools/player}/assembly/include/song.inc (100%) rename src/{sampletones_player/driver => sampletones_tools/player}/assembly/nsf.cfg (100%) rename src/{sampletones_player/driver => sampletones_tools/player}/assembly/source/channels.s (100%) rename src/{sampletones_player/driver => sampletones_tools/player}/assembly/source/clock.s (100%) rename src/{sampletones_player/driver => sampletones_tools/player}/assembly/source/driver.s (100%) create mode 100644 src/sampletones_tools/player/command.py rename {tests/unit/sampletones_player => src/sampletones_tools/player}/trace/__init__.py (100%) rename src/{sampletones_player => sampletones_tools/player}/trace/trace.py (98%) rename src/{sampletones_player => sampletones_tools/player}/trace/write.py (100%) create mode 100644 src/sampletones_tools/samples/__init__.py create mode 100644 src/sampletones_tools/samples/command.py rename scripts/nsf_render.py => src/sampletones_tools/samples/render.py (66%) create mode 100644 tests/unit/sampletones_tools/player/assembler/__init__.py rename tests/unit/{sampletones_player/driver => sampletones_tools/player}/assembler/test_builder.py (93%) rename tests/unit/{sampletones_player/driver => sampletones_tools/player}/assembler/test_labels.py (97%) create mode 100644 tests/unit/sampletones_tools/player/assembler/test_report.py rename tests/unit/{sampletones_player/driver => sampletones_tools/player}/assembler/test_toolchain.py (96%) create mode 100644 tests/unit/sampletones_tools/player/test_command.py rename tests/unit/{sampletones_player/driver => sampletones_tools/player}/test_song_include.py (95%) create mode 100644 tests/unit/sampletones_tools/player/trace/__init__.py rename tests/unit/{sampletones_player => sampletones_tools/player}/trace/test_trace.py (98%) create mode 100644 tests/unit/sampletones_tools/samples/test_command.py create mode 100644 tests/unit/sampletones_tools/samples/test_render.py diff --git a/Makefile b/Makefile index 8e10d6959..216c9ef22 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ .PHONY: help setup install system-deps build release run clean pre-commit test benchmarks lint format \ - ftm-samples nsf-samples nsf-render compression-report compression-study icons player \ + ftm-samples nsf-samples compression-report compression-study icons \ check-import-boundary check-tag-names check-unused-tags check-rendered-literals check-language-keys \ check-palette-colors check-shortcut-actions @@ -34,11 +34,9 @@ help: @echo $(Q) make benchmarks - Run the measured-duration suite on its own$(Q) @echo $(Q) make ftm-samples - Emit example .ftm files to build/ftm via the integration suite$(Q) @echo $(Q) make nsf-samples - Emit example .nsf files to build/nsf via the integration suite$(Q) - @echo $(Q) make nsf-render - Render the .nsf files in build/nsf to waves with ffmpeg$(Q) @echo $(Q) make compression-report - Measure the song codec into build/compression$(Q) @echo $(Q) make compression-study - Measure the song codec over the projects and stems on this machine; the report lands in Documents/SampleToNES/compression (ARGS=--quick for a short run)$(Q) @echo $(Q) make icons - Generate the icon suite into src/sampletones_assets/icons$(Q) - @echo $(Q) make player - Assemble the NES player driver with cc65$(Q) @echo $(Q) make clean - Remove build artifacts and cache files$(Q) @echo $(Q) make lint - Run mypy and pylint (ARGS=--mypy or ARGS=--pylint for one of them)$(Q) @echo $(Q) make format - Auto-format code (isort, black)$(Q) @@ -89,9 +87,6 @@ nsf-samples: export SAMPLETONES_NSF_OUTPUT_DIR := build/nsf nsf-samples: uv run python -m pytest tests/integration/nsf -nsf-render: nsf-samples - uv run scripts/nsf_render.py - compression-report: export SAMPLETONES_COMPRESSION_OUTPUT_DIR := build/compression compression-report: uv run python -m pytest tests/integration/nsf/test_compression_report.py @@ -102,9 +97,6 @@ compression-study: icons: uv run --group assets python scripts/assets/icons.py -player: - uv run scripts/player.py - check-import-boundary: uv run scripts/checks/import_boundary.py --all diff --git a/docs/development/dependencies.md b/docs/development/dependencies.md index ac08699a4..a669495df 100644 --- a/docs/development/dependencies.md +++ b/docs/development/dependencies.md @@ -74,11 +74,12 @@ that come with them. `scripts/ci/checks/bundle.py` holds the release bundles to ## NES player driver -The player that runs on the console is 6502 assembly, and `src/sampletones_player/driver` holds it -in three parts: `assembly/` carries the sources, their includes and the linker configuration, -`binary/` carries the assembled `driver.bin`, and `assembler/` carries the Python that turns one -into the other. `make player` runs `scripts/player.py` over that package, so the build behaves the -same on every system the project supports. +The player that runs on the console is 6502 assembly, held in three parts: +`src/sampletones_tools/player/assembly/` carries the sources, their includes and the linker +configuration, `src/sampletones_tools/player/assembler/` carries the Python that assembles them, +and `src/sampletones_player/driver/binary/` carries the assembled `driver.bin` the application +ships. `uv run sampletones driver` runs the build, so it behaves the same on every system the +project supports. Assembling needs `ca65` and `ld65` from [cc65](https://cc65.github.io/) — on Debian and Ubuntu, `sudo apt install cc65`, and a build names the equivalent for whichever system it runs on when the @@ -88,7 +89,7 @@ The assembled `driver.bin` is committed, so a checkout carries the player and ex needs no assembler. A jump table leads the image, which fixes the addresses an NSF header names whatever the driver's length, so the exporter states them from `specification/driver.py` and a build holds the linker's own labels to them before it writes anything. Editing the assembly means -running `make player` again and committing what it writes; the driver's test suite rebuilds the +running `uv run sampletones driver` again and committing what it writes; the driver's test suite rebuilds the sources and holds the committed image to them wherever cc65 is installed. The wheel carries the assembled image alone, which is all an installed copy reads. @@ -108,7 +109,7 @@ is a developer dependency, outside both the wheel and the bundles, and its BSD l project's own terms untouched. Listening to a real APU needs [ffmpeg](https://ffmpeg.org/) carrying the `libgme` demuxer, which -is a build option rather than a given: `make nsf-render` asks the installed ffmpeg which demuxers +is a build option rather than a given: `uv run sampletones nsf render` asks the installed ffmpeg which demuxers it holds and names this system's install command before it decodes anything. It exports the example files and renders each one to a wave beside it, its length read out of the song block the file carries. That is an ear rather than a gate: the register trace is what the driver answers to, and @@ -120,14 +121,14 @@ Three tools serve the player, each reached by one command: | Tool | Run by | Installed with | Reaches | | --- | --- | --- | --- | -| cc65 (`ca65`, `ld65`) | `make player` | the system's package manager | the machine assembling the driver | +| cc65 (`ca65`, `ld65`) | `sampletones driver` | the system's package manager | the machine assembling the driver | | py65 | `make test` | `uv sync --group dev` | the `dev` dependency group | -| ffmpeg with `libgme` | `make nsf-render` | the system's package manager | the machine listening to an export | +| ffmpeg with `libgme` | `sampletones nsf render` | the system's package manager | the machine listening to an export | `scripts/system_dependencies.py` carries what building and running the application needs, and the workflows install the `dev` group, so py65 is the one of the three CI reaches — the suite verifies the driver through it alone. cc65 and ffmpeg stay on the machine of -whoever runs `make player` or `make nsf-render`, and a workflow that assembles the driver or renders +whoever runs `sampletones driver` or `sampletones nsf render`, and a workflow that assembles the driver or renders a wave is what would put them in those scripts. The application itself calls neither: an export is written by the package's own code, from the committed `driver.bin`. diff --git a/docs/development/packages.md b/docs/development/packages.md index 70526c981..ac79da956 100644 --- a/docs/development/packages.md +++ b/docs/development/packages.md @@ -51,7 +51,7 @@ graph TD | `sampletones_core` | The reconstruction engine, the project model, playing a song out into instructions, and the tracker export formats | `sampletones_shared` | | `sampletones_player` | The NES player: the register model, the re-clocking schedule, the 6502 driver and the NSF file | `sampletones_shared`, `sampletones_core` | | `sampletones_application` | The DearPyGui front end | `sampletones_shared`, `sampletones_core`, `sampletones_player` | -| `sampletones_tools` | Everything a developer runs and the application does not: analytic waveform synthesis, the calibration harness, and the developer commands that run them | `sampletones_shared`, `sampletones_assets`, `sampletones_core`, `sampletones_player`, `sampletones_application` | +| `sampletones_tools` | Everything a developer runs and the application does not: analytic waveform synthesis, the calibration harness, the driver toolchain and the register trace, and the developer commands that run them | `sampletones_shared`, `sampletones_assets`, `sampletones_core`, `sampletones_player`, `sampletones_application` | | `sampletones` | The command-line entry: the dispatcher, the commands and the startup self-check | `sampletones_shared`, `sampletones_core`, `sampletones_application`, `sampletones_tools` | Third-party imports are the package author's own choice and stand outside this table. @@ -101,18 +101,19 @@ them. | `compression/` | The planes a song separates into, the dictionary its tokens name, and the codec that reads them both ways | `specification/`, `registers/` | | `song.py` | `Song` — the compressed planes, the timer table, the schedule and the loop point as one value | `clock/`, `registers/`, `compression/` | | `builder.py` | The song a reconstruction or an export request plays as, its instructions encoded, its planes compressed and its rate scheduled | `song.py`, `registers/`, `clock/`, `compression/` | -| `trace/` | `RegisterTrace` — what the driver is expected to write, call by call | `song.py`, `specification/` | | `nsf/` | The song block, the header and the `.nsf` file the console loads | `song.py`, `specification/`, `compression/`, `driver/` | | `driver/` | The assembled 6502 driver and the addresses its build reports | `specification/` | -| `driver/assembler/` | The cc65 build: the layout, the toolchain, the linker map reader and the builder | `driver/`, `specification/` | | `export.py` | `NSFBackend` — the export seam answered in `.nsf` files, holding the driver every one of them carries and saying which stage a run is in | `builder.py`, `nsf/`, `driver/`, `compression/` | -### The build toolchain is a developer tool +### The toolchain and the oracle live with the tools -`driver/assembler/` runs `ca65` and `ld65` over `driver/assembly/` to produce the committed -`driver/binary/driver.bin`. It is reached from `scripts/player.py` and from the tests, and the wheel -carries the binary alone — so a module of the shipped tree that imported it would break an installed -copy, and no unit above declares it. The developer toolchain it needs is described in +`sampletones_tools/player/assembler/` runs `ca65` and `ld65` over the assembly sources, their +includes and the linker configuration in `sampletones_tools/player/assembly/`, read as package +data, to produce the committed `driver/binary/driver.bin`; `uv run sampletones driver` runs it, +and the tests rebuild the sources and hold the committed image to them wherever cc65 is +installed. `sampletones_tools/player/trace/` holds `RegisterTrace`, what the driver is expected +to write call by call, which the emulator tests hold the assembled driver to. The application +ships the binary alone. The toolchain the build needs is described in [`dependencies.md`](dependencies.md). --- diff --git a/docs/development/player.md b/docs/development/player.md index a58b6dbf6..f8dd8bff3 100644 --- a/docs/development/player.md +++ b/docs/development/player.md @@ -3,7 +3,7 @@ This document governs `sampletones_player`: the 6502 driver an exported `.nsf` carries, the codec that fits a song into the console's program area, and the chain that holds both to what the application plays. Read it before changing the assembly under -`driver/assembly/`, anything under `compression/`, or the way a song is built in +`sampletones_tools/player/assembly/`, anything under `compression/`, or the way a song is built in `builder.py`. The byte layout the two sides meet on is [the NSF format](../formats/nsf.md); where the package sits among the others is [package layers](packages.md). @@ -159,7 +159,7 @@ The chain runs from the register values upward, and each link is held on its own | The driver's arithmetic | a song stating a bend outright, held to the divider each tick is meant to sound at | | The audio | a captured trace re-rendered against the reconstruction's own approximation | | The whole export | a project exported, played on the emulator, and read back as the instructions the sequencer sounds | -| Listening | `make nsf-samples` then `make nsf-render`, or any NSF player | +| Listening | `make nsf-samples` then `uv run sampletones nsf render --directory build/nsf`, or any NSF player | | Speed | `make benchmarks` — the encoder's own cost on the shapes that scale worst | The audio comparison is the one that catches a mistake the trace would let through: the @@ -168,9 +168,9 @@ the reconstruction was built as. ## Building the driver -`make player` assembles the sources with cc65 and writes `driver/binary/driver.bin`, which -is committed beside them — exporting an `.nsf` needs no assembler, and the wheel carries the -binary alone. +`uv run sampletones driver` assembles the sources under `sampletones_tools/player/assembly/` +with cc65 and writes `sampletones_player/driver/binary/driver.bin`, which is committed — +exporting an `.nsf` needs no assembler, and the application ships the binary alone. The link line names our own configuration and our own object files, with the CPU stated outright. That is the guardrail that keeps the shipped image entirely ours: reaching for a diff --git a/docs/development/tooling.md b/docs/development/tooling.md index 1c5f43627..7518eadb3 100644 --- a/docs/development/tooling.md +++ b/docs/development/tooling.md @@ -82,6 +82,8 @@ The developer commands, listed by `sampletones_tools/registry.py` and run as | Command | What it does | |---|---| | `calibration [--config FILE] [-o DIR] [--methods LIST] [--perceptual-exponents LIST] [--temporal-weights LIST] [--channels LIST]` | Reconstructs the calibration corpus under every variant of the sweep, scores it with every referee, and writes the reports; without `-o` the run lands in a timestamped directory under Documents/SampleToNES/calibration | +| `driver [--directory DIR]` | Assembles the NES player driver with cc65 and prints the layout the build produced; without `--directory` it writes the driver the package ships, which needs a checkout | +| `nsf render --directory DIR [--tail SECONDS]` | Renders every exported `.nsf` file in the directory to a wave beside it, through ffmpeg's libgme demuxer | More join as the tools they run move into the package. @@ -137,9 +139,8 @@ interpreter (`preflight.py`), and the platforms (`platforms/`). ## The tool scripts -`compression_study.py`, `nsf_render.py`, `player.py`, `assets/icons.py` and the checks under -`checks/` import the project's packages and run inside its environment, from the make target -that names each. The checks are also pre-commit hooks; +`compression_study.py`, `assets/icons.py` and the checks under `checks/` import the project's +packages and run inside its environment, from the make target that names each. The checks are also pre-commit hooks; [architecture](architecture.md#enforcement) lists them. ## Who governs what diff --git a/docs/formats/nsf.md b/docs/formats/nsf.md index bc9ae9733..de3838c94 100644 --- a/docs/formats/nsf.md +++ b/docs/formats/nsf.md @@ -16,7 +16,7 @@ and the song that driver plays. Every constant named here has a counterpart under `sampletones_player/specification/`, and the assembly reads the same figures from -`driver/assembly/include/song.inc`. The two are held against each other by a test, so a +`sampletones_tools/player/assembly/include/song.inc`. The two are held against each other by a test, so a change made in one file and forgotten in the other is reported by name. ## A. The file diff --git a/pyproject.toml b/pyproject.toml index b6b1a722e..8abe2100c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -99,10 +99,6 @@ build-backend = "hatchling.build" conflicts = [[{ extra = "gpu" }, { extra = "gpu-cuda11" }]] [tool.hatch.build.targets.wheel] -exclude = [ - "src/sampletones_player/driver/assembler", - "src/sampletones_player/driver/assembly", -] packages = [ "src/sampletones", "src/sampletones_application", diff --git a/scripts/player.py b/scripts/player.py deleted file mode 100755 index d96e2847c..000000000 --- a/scripts/player.py +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env python3 - -import argparse -import sys -from pathlib import Path -from typing import Sequence - -from sampletones_player.driver.assembler.builder import build_driver -from sampletones_player.driver.assembler.layout import BINARY_DIRECTORY -from sampletones_player.specification.driver import DRIVER_CODE_NAME -from sampletones_shared.exceptions import DriverBuildError - - -def main(argv: Sequence[str]) -> int: - """Assembles the NES player driver and reports the layout the build produced.""" - - parser = argparse.ArgumentParser( - description="Assemble the NES player driver with cc65.", - ) - parser.add_argument( - "--directory", - type=Path, - default=BINARY_DIRECTORY, - help="directory receiving the assembled driver", - ) - arguments = parser.parse_args(list(argv)) - - try: - image = build_driver(arguments.directory) - except DriverBuildError as error: - print(error, file=sys.stderr) - return 1 - - addresses = image.addresses - print(f"{DRIVER_CODE_NAME} {len(image.code)} bytes, ${addresses.load:04X}-${addresses.song - 1:04X}") - print(f"init ${addresses.init:04X}") - print(f"play ${addresses.play:04X}") - print(f"song ${addresses.song:04X}") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main(sys.argv[1:])) diff --git a/src/sampletones_config/boundaries/graphs.yaml b/src/sampletones_config/boundaries/graphs.yaml index bf8caa62f..eb6d16180 100644 --- a/src/sampletones_config/boundaries/graphs.yaml +++ b/src/sampletones_config/boundaries/graphs.yaml @@ -20,8 +20,6 @@ player: compression: [specification, registers] song.py: [clock, registers, compression] builder.py: [song.py, registers, clock, compression] - trace: [song.py, specification] nsf: [song.py, specification, compression, driver] export.py: [builder.py, nsf, driver, compression] driver: [specification] - driver/assembler: [driver, specification] diff --git a/src/sampletones_config/boundaries/standalone.yaml b/src/sampletones_config/boundaries/standalone.yaml index efd401216..b2738a020 100644 --- a/src/sampletones_config/boundaries/standalone.yaml +++ b/src/sampletones_config/boundaries/standalone.yaml @@ -1,8 +1,6 @@ - pattern: "**/*.py" excluding: - "compression_study.py" - - "nsf_render.py" - - "player.py" - "assets/**/*.py" - "checks/**/*.py" - "codec_study/**/*.py" diff --git a/src/sampletones_player/driver/assembler/layout.py b/src/sampletones_player/driver/assembler/layout.py deleted file mode 100644 index 6e312bbb7..000000000 --- a/src/sampletones_player/driver/assembler/layout.py +++ /dev/null @@ -1,14 +0,0 @@ -from pathlib import Path -from typing import Final, Tuple - -from sampletones_player.specification.driver import DRIVER_BINARY_DIRECTORY -from sampletones_shared.paths.source import SOURCE_ROOT - -DRIVER_DIRECTORY: Final[Path] = SOURCE_ROOT / "sampletones_player" / "driver" -ASSEMBLY_DIRECTORY: Final[Path] = DRIVER_DIRECTORY / "assembly" -INCLUDE_DIRECTORY: Final[Path] = ASSEMBLY_DIRECTORY / "include" -SOURCE_DIRECTORY: Final[Path] = ASSEMBLY_DIRECTORY / "source" -LINKER_CONFIGURATION: Final[Path] = ASSEMBLY_DIRECTORY / "nsf.cfg" -BINARY_DIRECTORY: Final[Path] = DRIVER_DIRECTORY / DRIVER_BINARY_DIRECTORY - -SOURCE_NAMES: Final[Tuple[str, ...]] = ("driver.s", "clock.s", "channels.s") diff --git a/src/sampletones_player/driver/image.py b/src/sampletones_player/driver/image.py index f16292b57..79cbe1d22 100644 --- a/src/sampletones_player/driver/image.py +++ b/src/sampletones_player/driver/image.py @@ -16,7 +16,7 @@ class DriverImage(BaseModel): """The assembled player, paired with the addresses it lays out. - The 6502 program that plays a song is written in assembly, built once by ``make player`` and + The 6502 program that plays a song is written in assembly, built once by ``sampletones driver`` and committed beside its sources, so exporting an NSF needs no assembler. Pairing the bytes with their addresses is what lets the exporter name the routines in an NSF header and place the song where the driver looks for it. diff --git a/src/sampletones_player/driver/assembler/__init__.py b/src/sampletones_tools/player/__init__.py similarity index 100% rename from src/sampletones_player/driver/assembler/__init__.py rename to src/sampletones_tools/player/__init__.py diff --git a/src/sampletones_player/trace/__init__.py b/src/sampletones_tools/player/assembler/__init__.py similarity index 100% rename from src/sampletones_player/trace/__init__.py rename to src/sampletones_tools/player/assembler/__init__.py diff --git a/src/sampletones_player/driver/assembler/builder.py b/src/sampletones_tools/player/assembler/builder.py similarity index 81% rename from src/sampletones_player/driver/assembler/builder.py rename to src/sampletones_tools/player/assembler/builder.py index aae212b2c..eef811e62 100644 --- a/src/sampletones_player/driver/assembler/builder.py +++ b/src/sampletones_tools/player/assembler/builder.py @@ -1,19 +1,21 @@ +from importlib.resources import as_file from pathlib import Path from tempfile import TemporaryDirectory from typing import Final, List from sampletones_player.driver.addresses import DriverAddresses -from sampletones_player.driver.assembler.labels import read_addresses -from sampletones_player.driver.assembler.layout import ( +from sampletones_player.driver.image import DriverImage +from sampletones_player.specification.driver import DRIVER_CODE_NAME +from sampletones_shared.exceptions import DriverBuildError +from sampletones_tools.player.assembler.labels import read_addresses +from sampletones_tools.player.assembler.layout import ( INCLUDE_DIRECTORY, LINKER_CONFIGURATION, SOURCE_DIRECTORY, SOURCE_NAMES, + assembly, ) -from sampletones_player.driver.assembler.toolchain import Toolchain -from sampletones_player.driver.image import DriverImage -from sampletones_player.specification.driver import DRIVER_CODE_NAME -from sampletones_shared.exceptions import DriverBuildError +from sampletones_tools.player.assembler.toolchain import Toolchain LABELS_NAME: Final[str] = "driver.labels" OBJECT_SUFFIX: Final[str] = ".o" @@ -39,12 +41,12 @@ def build_driver(destination: Path) -> DriverImage: built to answer at. """ toolchain = Toolchain.locate() - with TemporaryDirectory() as directory: + with as_file(assembly()) as sources, TemporaryDirectory() as directory: work_directory = Path(directory) - objects = assemble_sources(toolchain, work_directory) + objects = assemble_sources(toolchain, sources, work_directory) assembled = work_directory / DRIVER_CODE_NAME labels = work_directory / LABELS_NAME - toolchain.link(LINKER_CONFIGURATION, objects, assembled, labels) + toolchain.link(sources / LINKER_CONFIGURATION, objects, assembled, labels) image = DriverImage( code=assembled.read_bytes(), addresses=read_addresses(labels), @@ -56,11 +58,13 @@ def build_driver(destination: Path) -> DriverImage: return image -def assemble_sources(toolchain: Toolchain, work_directory: Path) -> List[Path]: +def assemble_sources(toolchain: Toolchain, sources: Path, work_directory: Path) -> List[Path]: """Assembles every source the driver is built from. Args: toolchain: The cc65 programs the build runs. + sources: The assembly package as a directory: the sources, their includes and the + linker configuration. work_directory: The directory receiving the object files. Returns: @@ -72,7 +76,7 @@ def assemble_sources(toolchain: Toolchain, work_directory: Path) -> List[Path]: objects: List[Path] = [] for name in SOURCE_NAMES: object_file = (work_directory / name).with_suffix(OBJECT_SUFFIX) - toolchain.assemble(SOURCE_DIRECTORY / name, INCLUDE_DIRECTORY, object_file) + toolchain.assemble(sources / SOURCE_DIRECTORY / name, sources / INCLUDE_DIRECTORY, object_file) objects.append(object_file) return objects diff --git a/src/sampletones_player/driver/assembler/labels.py b/src/sampletones_tools/player/assembler/labels.py similarity index 100% rename from src/sampletones_player/driver/assembler/labels.py rename to src/sampletones_tools/player/assembler/labels.py diff --git a/src/sampletones_tools/player/assembler/layout.py b/src/sampletones_tools/player/assembler/layout.py new file mode 100644 index 000000000..04d461cc8 --- /dev/null +++ b/src/sampletones_tools/player/assembler/layout.py @@ -0,0 +1,19 @@ +from importlib.resources import files +from importlib.resources.abc import Traversable +from pathlib import Path +from typing import Final, Tuple + +from sampletones_player.specification.driver import DRIVER_BINARY_DIRECTORY +from sampletones_shared.paths.source import SOURCE_ROOT + +ASSEMBLY_PACKAGE: Final[str] = "sampletones_tools.player.assembly" +INCLUDE_DIRECTORY: Final[str] = "include" +SOURCE_DIRECTORY: Final[str] = "source" +LINKER_CONFIGURATION: Final[str] = "nsf.cfg" +SOURCE_NAMES: Final[Tuple[str, ...]] = ("driver.s", "clock.s", "channels.s") +BINARY_DIRECTORY: Final[Path] = SOURCE_ROOT / "sampletones_player" / "driver" / DRIVER_BINARY_DIRECTORY + + +def assembly() -> Traversable: + """The assembly sources, their includes and the linker configuration, read from the package.""" + return files(ASSEMBLY_PACKAGE) diff --git a/src/sampletones_tools/player/assembler/report.py b/src/sampletones_tools/player/assembler/report.py new file mode 100644 index 000000000..f7afd1155 --- /dev/null +++ b/src/sampletones_tools/player/assembler/report.py @@ -0,0 +1,15 @@ +from typing import List + +from sampletones_player.driver.image import DriverImage +from sampletones_player.specification.driver import DRIVER_CODE_NAME + + +def layout_lines(image: DriverImage) -> List[str]: + """The layout a build produced, one line per figure a reader checks against the header.""" + addresses = image.addresses + return [ + f"{DRIVER_CODE_NAME} {len(image.code)} bytes, ${addresses.load:04X}-${addresses.song - 1:04X}", + f"init ${addresses.init:04X}", + f"play ${addresses.play:04X}", + f"song ${addresses.song:04X}", + ] diff --git a/src/sampletones_player/driver/assembler/toolchain.py b/src/sampletones_tools/player/assembler/toolchain.py similarity index 100% rename from src/sampletones_player/driver/assembler/toolchain.py rename to src/sampletones_tools/player/assembler/toolchain.py diff --git a/tests/unit/sampletones_player/driver/assembler/__init__.py b/src/sampletones_tools/player/assembly/__init__.py similarity index 100% rename from tests/unit/sampletones_player/driver/assembler/__init__.py rename to src/sampletones_tools/player/assembly/__init__.py diff --git a/src/sampletones_player/driver/assembly/include/nes.inc b/src/sampletones_tools/player/assembly/include/nes.inc similarity index 100% rename from src/sampletones_player/driver/assembly/include/nes.inc rename to src/sampletones_tools/player/assembly/include/nes.inc diff --git a/src/sampletones_player/driver/assembly/include/song.inc b/src/sampletones_tools/player/assembly/include/song.inc similarity index 100% rename from src/sampletones_player/driver/assembly/include/song.inc rename to src/sampletones_tools/player/assembly/include/song.inc diff --git a/src/sampletones_player/driver/assembly/nsf.cfg b/src/sampletones_tools/player/assembly/nsf.cfg similarity index 100% rename from src/sampletones_player/driver/assembly/nsf.cfg rename to src/sampletones_tools/player/assembly/nsf.cfg diff --git a/src/sampletones_player/driver/assembly/source/channels.s b/src/sampletones_tools/player/assembly/source/channels.s similarity index 100% rename from src/sampletones_player/driver/assembly/source/channels.s rename to src/sampletones_tools/player/assembly/source/channels.s diff --git a/src/sampletones_player/driver/assembly/source/clock.s b/src/sampletones_tools/player/assembly/source/clock.s similarity index 100% rename from src/sampletones_player/driver/assembly/source/clock.s rename to src/sampletones_tools/player/assembly/source/clock.s diff --git a/src/sampletones_player/driver/assembly/source/driver.s b/src/sampletones_tools/player/assembly/source/driver.s similarity index 100% rename from src/sampletones_player/driver/assembly/source/driver.s rename to src/sampletones_tools/player/assembly/source/driver.s diff --git a/src/sampletones_tools/player/command.py b/src/sampletones_tools/player/command.py new file mode 100644 index 000000000..2b11dfde1 --- /dev/null +++ b/src/sampletones_tools/player/command.py @@ -0,0 +1,54 @@ +from argparse import ArgumentParser, Namespace +from dataclasses import dataclass +from pathlib import Path +from typing import Final, Optional + +from sampletones_shared.command import Command + +NAME: Final[str] = "driver" +HELP: Final[str] = "assemble the NES player driver with cc65" +DIRECTORY_HELP: Final[str] = "the directory receiving the assembled driver; without it, the driver the package ships" + + +@dataclass(frozen=True) +class DriverArguments: + """What a driver build is given: where the image goes, if anywhere but the package.""" + + directory: Optional[Path] + + +def configure(parser: ArgumentParser) -> None: + parser.add_argument("--directory", type=Path, default=None, help=DIRECTORY_HELP) + + +def run(arguments: Namespace) -> int: + """Assembles the driver and prints the layout the build produced. + + Writing the driver the package ships needs a checkout, since that is where the package is. + + Raises: + SystemExit: If the build runs outside a checkout without a directory of its own, or fails. + """ + given = DriverArguments(directory=arguments.directory) + + from sampletones_shared.exceptions.player import DriverBuildError + from sampletones_tools.checkout import require_checkout + from sampletones_tools.player.assembler.builder import build_driver + from sampletones_tools.player.assembler.layout import BINARY_DIRECTORY + from sampletones_tools.player.assembler.report import layout_lines + + if given.directory is None: + require_checkout(NAME) + + try: + image = build_driver(given.directory if given.directory is not None else BINARY_DIRECTORY) + except DriverBuildError as error: + raise SystemExit(str(error)) from error + + for line in layout_lines(image): + print(line) + + return 0 + + +DRIVER: Final[Command] = Command(name=NAME, help=HELP, configure=configure, run=run) diff --git a/tests/unit/sampletones_player/trace/__init__.py b/src/sampletones_tools/player/trace/__init__.py similarity index 100% rename from tests/unit/sampletones_player/trace/__init__.py rename to src/sampletones_tools/player/trace/__init__.py diff --git a/src/sampletones_player/trace/trace.py b/src/sampletones_tools/player/trace/trace.py similarity index 98% rename from src/sampletones_player/trace/trace.py rename to src/sampletones_tools/player/trace/trace.py index cd9bd185b..e9cccfdc3 100644 --- a/src/sampletones_player/trace/trace.py +++ b/src/sampletones_tools/player/trace/trace.py @@ -21,7 +21,7 @@ SILENCED_REGISTER, SWEEP_DISABLED, ) -from sampletones_player.trace.write import RegisterWrite +from sampletones_tools.player.trace.write import RegisterWrite FIRST_TICK: Final[int] = 0 diff --git a/src/sampletones_player/trace/write.py b/src/sampletones_tools/player/trace/write.py similarity index 100% rename from src/sampletones_player/trace/write.py rename to src/sampletones_tools/player/trace/write.py diff --git a/src/sampletones_tools/registry.py b/src/sampletones_tools/registry.py index f75d8c374..ccc075d17 100644 --- a/src/sampletones_tools/registry.py +++ b/src/sampletones_tools/registry.py @@ -2,5 +2,7 @@ from sampletones_shared.command import Command from sampletones_tools.calibration.command import CALIBRATION +from sampletones_tools.player.command import DRIVER +from sampletones_tools.samples.command import NSF -DEVELOPER_COMMANDS: Final[Tuple[Command, ...]] = (CALIBRATION,) +DEVELOPER_COMMANDS: Final[Tuple[Command, ...]] = (CALIBRATION, DRIVER, NSF) diff --git a/src/sampletones_tools/samples/__init__.py b/src/sampletones_tools/samples/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/sampletones_tools/samples/command.py b/src/sampletones_tools/samples/command.py new file mode 100644 index 000000000..7b9aec954 --- /dev/null +++ b/src/sampletones_tools/samples/command.py @@ -0,0 +1,55 @@ +from argparse import ArgumentParser, Namespace +from dataclasses import dataclass +from pathlib import Path +from typing import Final + +from sampletones_shared.command import Command + +NAME: Final[str] = "nsf" +HELP: Final[str] = "render exported .nsf files to waves" +ACTION_FIELD: Final[str] = "action" +ACTION_METAVAR: Final[str] = "" +RENDER: Final[str] = "render" +RENDER_HELP: Final[str] = "render the .nsf files in a directory to waves through ffmpeg's libgme demuxer" +DIRECTORY_HELP: Final[str] = "the directory holding the exported .nsf files" +TAIL_HELP: Final[str] = "seconds kept past the end of each song" +DEFAULT_TAIL_SECONDS: Final[float] = 0.5 + + +@dataclass(frozen=True) +class RenderArguments: + """What a render is given: the directory of exported files and the tail each wave keeps.""" + + directory: Path + tail: float + + +def configure(parser: ArgumentParser) -> None: + actions = parser.add_subparsers(dest=ACTION_FIELD, metavar=ACTION_METAVAR, required=True) + render = actions.add_parser(RENDER, help=RENDER_HELP, description=RENDER_HELP) + render.add_argument("--directory", type=Path, required=True, help=DIRECTORY_HELP) + render.add_argument("--tail", type=float, default=DEFAULT_TAIL_SECONDS, help=TAIL_HELP) + + +def run(arguments: Namespace) -> int: + """Renders the exported files and prints each wave written. + + Raises: + SystemExit: If ffmpeg is unusable or rejects a file. + """ + given = RenderArguments(directory=arguments.directory, tail=arguments.tail) + + from sampletones_tools.samples.render import RenderingError, render_directory + + try: + rendered = render_directory(given.directory, given.tail) + except RenderingError as error: + raise SystemExit(str(error)) from error + + for wave in rendered: + print(f"{wave.destination} {wave.seconds:.3f} s") + + return 0 + + +NSF: Final[Command] = Command(name=NAME, help=HELP, configure=configure, run=run) diff --git a/scripts/nsf_render.py b/src/sampletones_tools/samples/render.py similarity index 66% rename from scripts/nsf_render.py rename to src/sampletones_tools/samples/render.py index 922c58407..ae3477856 100755 --- a/scripts/nsf_render.py +++ b/src/sampletones_tools/samples/render.py @@ -1,11 +1,8 @@ -#!/usr/bin/env python3 - -import argparse import struct import subprocess -import sys +from dataclasses import dataclass from pathlib import Path -from typing import Dict, Final, List, Sequence +from typing import Dict, Final, List from sampletones_player.driver.image import DriverImage from sampletones_player.specification.clock import ( @@ -18,6 +15,7 @@ STEP_WHOLE_OFFSET, TOTAL_TICKS_OFFSET, ) +from sampletones_shared.exceptions import SampleToNESError from sampletones_shared.paths.extensions import EXT_FILE_NSF, EXT_FILE_WAVE from sampletones_shared.utils.system.programs import ( locate_program, @@ -25,8 +23,6 @@ ) from sampletones_shared.utils.system.system import System -SAMPLES_DIRECTORY: Final[Path] = Path("build") / "nsf" -TAIL_SECONDS: Final[float] = 0.5 FFMPEG: Final[str] = "ffmpeg" GME_FORMAT: Final[str] = "libgme" WORD: Final[str] = " bool: """Whether the installed ffmpeg carries the demuxer an exported file is read through. @@ -60,6 +75,22 @@ def decodes_exports() -> bool: return GME_FORMAT in reported.stdout +def require_renderer() -> None: + """Holds a render to an ffmpeg that decodes exported files. + + Raises: + RenderingError: If ffmpeg is absent, naming how this system installs it, or carries no + libgme demuxer. + """ + if locate_program(FFMPEG) is None: + raise RenderingError(missing_program_message(FFMPEG, RENDER_PURPOSE, INSTALL_HINTS)) + + if not decodes_exports(): + raise RenderingError( + f"{FFMPEG} reports no {GME_FORMAT} demuxer; rendering needs a build made with --enable-libgme" + ) + + def song_seconds(data: bytes, code_length: int) -> float: """How long the song in an exported file lasts, read out of the block behind the driver. @@ -110,62 +141,35 @@ def render(source: Path, destination: Path, seconds: float) -> None: ) -def main(argv: Sequence[str]) -> int: - """Renders every exported file in a directory to a wave beside it.""" - - parser = argparse.ArgumentParser( - description="Render exported .nsf files to waves with ffmpeg's libgme demuxer.", - ) - parser.add_argument( - "--directory", - type=Path, - default=SAMPLES_DIRECTORY, - help="directory holding the exported .nsf files", - ) - parser.add_argument( - "--tail", - type=float, - default=TAIL_SECONDS, - help="seconds to keep past the end of each song", - ) - arguments = parser.parse_args(list(argv)) +def render_directory(directory: Path, tail_seconds: float) -> List[RenderedWave]: + """Renders every exported file in a directory to a wave beside it. - if locate_program(FFMPEG) is None: - print(missing_program_message(FFMPEG, RENDER_PURPOSE, INSTALL_HINTS), file=sys.stderr) - return 1 + Args: + directory: The directory holding the `.nsf` files. + tail_seconds: How much to keep past the end of each song. - if not decodes_exports(): - print( - f"{FFMPEG} reports no {GME_FORMAT} demuxer; rendering needs a build made with --enable-libgme", - file=sys.stderr, - ) - return 1 + Returns: + List[RenderedWave]: The waves written, in path order. - sources: List[Path] = sorted(arguments.directory.glob(f"*{EXT_FILE_NSF}")) + Raises: + RenderingError: If ffmpeg is unusable, the directory holds no exported file, or ffmpeg + rejects one. + """ + require_renderer() + sources = sorted(directory.glob(f"*{EXT_FILE_NSF}")) if not sources: - print( - f"no {EXT_FILE_NSF} files in {arguments.directory}; run make nsf-samples first", - file=sys.stderr, - ) - return 1 + raise RenderingError(f"no {EXT_FILE_NSF} files in {directory}") code_length = len(DriverImage.load().code) + rendered: List[RenderedWave] = [] for source in sources: destination = source.with_suffix(EXT_FILE_WAVE) - seconds = song_seconds(source.read_bytes(), code_length) + arguments.tail + seconds = song_seconds(source.read_bytes(), code_length) + tail_seconds try: render(source, destination, seconds) except subprocess.CalledProcessError as error: - print( - f"{FFMPEG} rejected {source}: exit status {error.returncode}", - file=sys.stderr, - ) - return 1 - - print(f"{destination} {seconds:.3f} s") - - return 0 + raise RenderingError(f"{FFMPEG} rejected {source}: exit status {error.returncode}") from error + rendered.append(RenderedWave(source=source, destination=destination, seconds=seconds)) -if __name__ == "__main__": - raise SystemExit(main(sys.argv[1:])) + return rendered diff --git a/tests/integration/nsf/console/instructions.py b/tests/integration/nsf/console/instructions.py index c82f8bbf1..0fdb700c3 100644 --- a/tests/integration/nsf/console/instructions.py +++ b/tests/integration/nsf/console/instructions.py @@ -22,7 +22,7 @@ TRIANGLE_COUNTER_CONTROL, TRIANGLE_SOUNDING_RELOAD, ) -from sampletones_player.trace.trace import RegisterTrace +from sampletones_tools.player.trace.trace import RegisterTrace from tests.integration.nsf.console.machine import register_file TRIANGLE_SOUNDING: Final[int] = TRIANGLE_COUNTER_CONTROL | TRIANGLE_SOUNDING_RELOAD diff --git a/tests/integration/nsf/console/machine.py b/tests/integration/nsf/console/machine.py index afa2a1c56..87edc8778 100644 --- a/tests/integration/nsf/console/machine.py +++ b/tests/integration/nsf/console/machine.py @@ -9,8 +9,8 @@ APU_FRAME_COUNTER, FIRST_CHANNEL_REGISTER, ) -from sampletones_player.trace.trace import RegisterTrace -from sampletones_player.trace.write import RegisterWrite +from sampletones_tools.player.trace.trace import RegisterTrace +from sampletones_tools.player.trace.write import RegisterWrite RETURN_SENTINEL: Final[int] = 0xFFF0 STACK_PAGE: Final[int] = 0x0100 diff --git a/tests/integration/nsf/console/session.py b/tests/integration/nsf/console/session.py index 356116ee6..7f64b337c 100644 --- a/tests/integration/nsf/console/session.py +++ b/tests/integration/nsf/console/session.py @@ -4,7 +4,7 @@ from sampletones_player.nsf.file import nsf_to_bytes from sampletones_player.nsf.information import NSFInformation from sampletones_player.song import Song -from sampletones_player.trace.trace import RegisterTrace +from sampletones_tools.player.trace.trace import RegisterTrace from tests.integration.nsf.console.machine import Console TRAILING_CALLS: Final[int] = 2 diff --git a/tests/integration/nsf/test_driver_bend.py b/tests/integration/nsf/test_driver_bend.py index 1db9a4803..fc9356c54 100644 --- a/tests/integration/nsf/test_driver_bend.py +++ b/tests/integration/nsf/test_driver_bend.py @@ -7,7 +7,7 @@ from sampletones_player.song import Song from sampletones_player.specification.binary import BYTE_VALUES from sampletones_player.specification.registers import PULSE1_TIMER_HIGH, TIMER_HIGH_SHIFT -from sampletones_player.trace.trace import RegisterTrace +from sampletones_tools.player.trace.trace import RegisterTrace from tests.integration.nsf.console.instructions import channel_values, timer_value from tests.integration.nsf.console.machine import register_file from tests.integration.nsf.console.session import captured_trace, play_calls_covering diff --git a/tests/integration/nsf/test_driver_trace.py b/tests/integration/nsf/test_driver_trace.py index 8640103d5..90f662f8c 100644 --- a/tests/integration/nsf/test_driver_trace.py +++ b/tests/integration/nsf/test_driver_trace.py @@ -11,7 +11,7 @@ from sampletones_player.specification.binary import WORD_SIZE from sampletones_player.specification.nsf import PROGRAM_SIZE from sampletones_player.specification.song import STEP_FRACTION_OFFSET, STEP_WHOLE_OFFSET -from sampletones_player.trace.trace import RegisterTrace +from sampletones_tools.player.trace.trace import RegisterTrace from tests.integration.nsf.console.session import ( TRAILING_CALLS, captured_trace, diff --git a/tests/unit/sampletones_shared/meta/import_boundary/configs/test_rules.py b/tests/unit/sampletones_shared/meta/import_boundary/configs/test_rules.py index 31f35c450..510919c8e 100644 --- a/tests/unit/sampletones_shared/meta/import_boundary/configs/test_rules.py +++ b/tests/unit/sampletones_shared/meta/import_boundary/configs/test_rules.py @@ -23,14 +23,11 @@ PLAYER: Final[str] = "sampletones_player" TOOLS: Final[str] = "sampletones_tools" ENTRY: Final[str] = "sampletones" -ASSEMBLER: Final[str] = "sampletones_player.driver.assembler" VISUAL_IMPORT: Final[str] = "import dearpygui.dearpygui as dpg\n" CONTRACT_IMPORT: Final[str] = "from sampletones_application.services.result import ServiceResult\n" PLAIN_IMPORT: Final[str] = "from sampletones_core.project.project import Project\n" PLAYER_IMPORT: Final[str] = "from sampletones_player.song import Song\n" -ASSEMBLER_IMPORT: Final[str] = "from sampletones_player.driver.assembler.builder import build_driver\n" -DRIVER_IMPORT: Final[str] = "from sampletones_player.driver.image import DriverImage\n" TOOLS_IMPORT: Final[str] = "from sampletones_tools.registry import DEVELOPER_COMMANDS\n" PANEL_SUFFIX: Final[str] = "def build() -> None:\n dpg.add_group(parent=SUF_PANEL_LEFT)\n" THIRD_PARTY_IMPORT: Final[str] = "import numpy\n" @@ -90,10 +87,6 @@ class TestPlayerGraph: def test_the_specification_is_the_layer_everything_stands_on(self) -> None: assert self.LAYERS["specification"] == () - def test_the_build_toolchain_is_reached_from_no_shipped_module(self) -> None: - """`driver/assembler/` stays outside the wheel, so an import of it breaks an installed copy.""" - assert all("driver/assembler" not in layers for layers in self.LAYERS.values()) - def test_the_driver_is_reached_through_the_file_that_writes_the_nsf(self) -> None: assert "driver" in self.LAYERS["nsf"] @@ -149,16 +142,6 @@ def test_the_console_player_reading_the_engine_is_left_alone(self, tmp_path: Pat assert reported(tmp_path) == [] - def test_a_shipped_module_reaching_the_build_toolchain_is_reported(self, tmp_path: Path) -> None: - write_module(tmp_path / PLAYER / "nsf", "file.py", ASSEMBLER_IMPORT) - - assert reported(tmp_path) == [ASSEMBLER] - - def test_the_build_toolchain_reads_the_driver_it_assembles(self, tmp_path: Path) -> None: - write_module(tmp_path / PLAYER / "driver" / "assembler", "builder.py", DRIVER_IMPORT) - - assert reported(tmp_path) == [] - def test_a_panel_composing_a_column_suffix_is_reported(self, tmp_path: Path) -> None: write_module(tmp_path / APPLICATION / "ui" / "panels", "left.py", PANEL_SUFFIX) diff --git a/tests/unit/sampletones_tools/player/assembler/__init__.py b/tests/unit/sampletones_tools/player/assembler/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/unit/sampletones_player/driver/assembler/test_builder.py b/tests/unit/sampletones_tools/player/assembler/test_builder.py similarity index 93% rename from tests/unit/sampletones_player/driver/assembler/test_builder.py rename to tests/unit/sampletones_tools/player/assembler/test_builder.py index 0c8479146..fea7fbb4a 100644 --- a/tests/unit/sampletones_player/driver/assembler/test_builder.py +++ b/tests/unit/sampletones_tools/player/assembler/test_builder.py @@ -5,8 +5,6 @@ import pytest from sampletones_player.driver.addresses import DriverAddresses -from sampletones_player.driver.assembler.builder import build_driver, verify_addresses -from sampletones_player.driver.assembler.toolchain import ASSEMBLER from sampletones_player.driver.image import DriverImage from sampletones_player.specification.driver import ( DRIVER_CODE_NAME, @@ -15,6 +13,8 @@ LOAD_ADDRESS, ) from sampletones_shared.exceptions import DriverBuildError +from sampletones_tools.player.assembler.builder import build_driver, verify_addresses +from sampletones_tools.player.assembler.toolchain import ASSEMBLER from tests.suite.base import BaseTestSuite DISPLACEMENT: Final[int] = 0x0100 @@ -57,7 +57,7 @@ class TestTheDriverBuild(BaseTestSuite): """The committed driver against the sources it is built from.""" def test_the_committed_driver_matches_its_sources(self, built_driver: DriverImage) -> None: - message = f"{DRIVER_CODE_NAME} is behind its sources: run `make player`" + message = f"{DRIVER_CODE_NAME} is behind its sources: run `uv run sampletones driver`" assert built_driver.code == DriverImage.load().code, message def test_the_linker_lays_the_driver_out_where_it_is_declared(self, built_driver: DriverImage) -> None: diff --git a/tests/unit/sampletones_player/driver/assembler/test_labels.py b/tests/unit/sampletones_tools/player/assembler/test_labels.py similarity index 97% rename from tests/unit/sampletones_player/driver/assembler/test_labels.py rename to tests/unit/sampletones_tools/player/assembler/test_labels.py index 411d50765..0fa438e84 100644 --- a/tests/unit/sampletones_player/driver/assembler/test_labels.py +++ b/tests/unit/sampletones_tools/player/assembler/test_labels.py @@ -3,7 +3,9 @@ import pytest -from sampletones_player.driver.assembler.labels import ( +from sampletones_player.specification.driver import INIT_ADDRESS, LOAD_ADDRESS, PLAY_ADDRESS +from sampletones_shared.exceptions import DriverBuildError +from sampletones_tools.player.assembler.labels import ( INIT_SYMBOL, LOAD_SYMBOL, PLAY_SYMBOL, @@ -11,8 +13,6 @@ read_addresses, read_labels, ) -from sampletones_player.specification.driver import INIT_ADDRESS, LOAD_ADDRESS, PLAY_ADDRESS -from sampletones_shared.exceptions import DriverBuildError from tests.suite.base import BaseTestSuite LABELS_NAME: Final[str] = "driver.labels" diff --git a/tests/unit/sampletones_tools/player/assembler/test_report.py b/tests/unit/sampletones_tools/player/assembler/test_report.py new file mode 100644 index 000000000..95932fb7a --- /dev/null +++ b/tests/unit/sampletones_tools/player/assembler/test_report.py @@ -0,0 +1,19 @@ +from typing import Final + +from sampletones_player.driver.addresses import DriverAddresses +from sampletones_player.driver.image import DriverImage +from sampletones_player.specification.driver import JUMP_ABSOLUTE_OPCODE +from sampletones_tools.player.assembler.report import layout_lines + +RETURN_OPCODE: Final[int] = 0x60 +CODE: Final[bytes] = bytes((JUMP_ABSOLUTE_OPCODE, 0x00, 0x80, JUMP_ABSOLUTE_OPCODE, 0x00, 0x80, RETURN_OPCODE)) + + +class TestLayoutLines: + def test_the_figures_a_reader_checks_against_the_header(self) -> None: + image = DriverImage(code=CODE, addresses=DriverAddresses.for_code(len(CODE))) + + lines = layout_lines(image) + + assert lines[0] == "driver.bin 7 bytes, $8000-$8006" + assert lines[1:] == ["init $8000", "play $8003", "song $8007"] diff --git a/tests/unit/sampletones_player/driver/assembler/test_toolchain.py b/tests/unit/sampletones_tools/player/assembler/test_toolchain.py similarity index 96% rename from tests/unit/sampletones_player/driver/assembler/test_toolchain.py rename to tests/unit/sampletones_tools/player/assembler/test_toolchain.py index 89602b787..4ee145f24 100644 --- a/tests/unit/sampletones_player/driver/assembler/test_toolchain.py +++ b/tests/unit/sampletones_tools/player/assembler/test_toolchain.py @@ -5,14 +5,14 @@ import pytest -from sampletones_player.driver.assembler.toolchain import ( +from sampletones_shared.exceptions import DriverBuildError, ToolchainMissingError +from sampletones_shared.utils.system.system import System +from sampletones_tools.player.assembler.toolchain import ( ASSEMBLER, INSTALL_HINTS, LINKER, Toolchain, ) -from sampletones_shared.exceptions import DriverBuildError, ToolchainMissingError -from sampletones_shared.utils.system.system import System from tests.suite.base import BaseTestSuite FAILING_PROGRAM: Final[str] = "import sys; sys.stderr.write('boom'); sys.exit(1)" diff --git a/tests/unit/sampletones_tools/player/test_command.py b/tests/unit/sampletones_tools/player/test_command.py new file mode 100644 index 000000000..32540ffd0 --- /dev/null +++ b/tests/unit/sampletones_tools/player/test_command.py @@ -0,0 +1,65 @@ +from pathlib import Path +from typing import Final, List + +import pytest + +from sampletones.commands.registry import COMMANDS +from sampletones.dispatcher import dispatch +from sampletones_player.driver.addresses import DriverAddresses +from sampletones_player.driver.image import DriverImage +from sampletones_player.specification.driver import JUMP_ABSOLUTE_OPCODE +from sampletones_shared.exceptions.player import DriverBuildError +from sampletones_tools.player.assembler.layout import BINARY_DIRECTORY + +BUILDER: Final[str] = "sampletones_tools.player.assembler.builder.build_driver" +GUARD: Final[str] = "sampletones_tools.checkout.require_checkout" +RETURN_OPCODE: Final[int] = 0x60 +CODE: Final[bytes] = bytes((JUMP_ABSOLUTE_OPCODE, 0x00, 0x80, JUMP_ABSOLUTE_OPCODE, 0x00, 0x80, RETURN_OPCODE)) +IMAGE: Final[DriverImage] = DriverImage(code=CODE, addresses=DriverAddresses.for_code(len(CODE))) + + +class RecordedBuild: + def __init__(self) -> None: + self.destinations: List[Path] = [] + + def __call__(self, destination: Path) -> DriverImage: + self.destinations.append(destination) + return IMAGE + + +def _refuse(command: str) -> None: + raise AssertionError(f"the checkout guard ran for {command}") + + +class TestDriver: + def test_without_a_directory_the_shipped_driver_is_written_from_a_checkout( + self, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], + ) -> None: + build = RecordedBuild() + guarded: List[str] = [] + monkeypatch.setattr(BUILDER, build) + monkeypatch.setattr(GUARD, guarded.append) + + assert dispatch(COMMANDS, ["driver"]) == 0 + assert build.destinations == [BINARY_DIRECTORY] + assert guarded == ["driver"] + assert "driver.bin 7 bytes" in capsys.readouterr().out + + def test_a_directory_of_its_own_needs_no_checkout(self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + build = RecordedBuild() + monkeypatch.setattr(BUILDER, build) + monkeypatch.setattr(GUARD, _refuse) + + assert dispatch(COMMANDS, ["driver", "--directory", str(tmp_path)]) == 0 + assert build.destinations == [tmp_path] + + def test_a_failing_build_is_reported(self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + def fail(destination: Path) -> DriverImage: + raise DriverBuildError(f"ca65 failed: boom in {destination}") + + monkeypatch.setattr(BUILDER, fail) + + with pytest.raises(SystemExit, match="ca65 failed"): + dispatch(COMMANDS, ["driver", "--directory", str(tmp_path)]) diff --git a/tests/unit/sampletones_player/driver/test_song_include.py b/tests/unit/sampletones_tools/player/test_song_include.py similarity index 95% rename from tests/unit/sampletones_player/driver/test_song_include.py rename to tests/unit/sampletones_tools/player/test_song_include.py index 042e4b300..6daddf079 100644 --- a/tests/unit/sampletones_player/driver/test_song_include.py +++ b/tests/unit/sampletones_tools/player/test_song_include.py @@ -1,4 +1,5 @@ import ast +from importlib.resources.abc import Traversable from pathlib import Path from typing import Dict, Final @@ -6,7 +7,6 @@ from sampletones_player.compression.pitch import PITCH_COUNT from sampletones_player.compression.planes.order import PlaneOrder -from sampletones_player.driver.assembler.layout import INCLUDE_DIRECTORY from sampletones_player.specification.binary import WORD_SIZE from sampletones_player.specification.compression import ( OPCODE_SIZE, @@ -31,6 +31,7 @@ TIMER_TABLE_OFFSET, TOTAL_TICKS_OFFSET, ) +from sampletones_tools.player.assembler.layout import INCLUDE_DIRECTORY, assembly SONG_INCLUDE: Final[str] = "song.inc" HEXADECIMAL_MARKER: Final[str] = "$" @@ -81,7 +82,7 @@ def _value(node: ast.expr, defined: Dict[str, int]) -> int: raise ValueError(f"an equate reads {ast.dump(node)}, which the include holds no form for") -def read_equates(path: Path) -> Dict[str, int]: +def read_equates(path: Traversable) -> Dict[str, int]: """Reads the constants an assembly include states, each over the ones stated before it. The driver and the exporter read one song block, so what the assembly believes about the @@ -108,7 +109,7 @@ def read_equates(path: Path) -> Dict[str, int]: @pytest.fixture(name="equates", scope="module") def equates_fixture() -> Dict[str, int]: - return read_equates(INCLUDE_DIRECTORY / SONG_INCLUDE) + return read_equates(assembly() / INCLUDE_DIRECTORY / SONG_INCLUDE) class TestTheDriverReadsTheBlockTheExporterWrites: diff --git a/tests/unit/sampletones_tools/player/trace/__init__.py b/tests/unit/sampletones_tools/player/trace/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/unit/sampletones_player/trace/test_trace.py b/tests/unit/sampletones_tools/player/trace/test_trace.py similarity index 98% rename from tests/unit/sampletones_player/trace/test_trace.py rename to tests/unit/sampletones_tools/player/trace/test_trace.py index f800fd3bc..f585cc9a4 100644 --- a/tests/unit/sampletones_player/trace/test_trace.py +++ b/tests/unit/sampletones_tools/player/trace/test_trace.py @@ -22,8 +22,8 @@ SWEEP_DISABLED, TRIANGLE_TIMER_HIGH, ) -from sampletones_player.trace.trace import RegisterTrace -from sampletones_player.trace.write import RegisterWrite +from sampletones_tools.player.trace.trace import RegisterTrace +from sampletones_tools.player.trace.write import RegisterWrite from tests.suite.base import BaseTestSuite from tests.suite.case import BaseAutolabelTestCase from tests.suite.player import ( diff --git a/tests/unit/sampletones_tools/samples/test_command.py b/tests/unit/sampletones_tools/samples/test_command.py new file mode 100644 index 000000000..584e46ee7 --- /dev/null +++ b/tests/unit/sampletones_tools/samples/test_command.py @@ -0,0 +1,51 @@ +from pathlib import Path +from typing import Final, List, Tuple + +import pytest + +from sampletones.commands.registry import COMMANDS +from sampletones.dispatcher import dispatch +from sampletones_tools.samples.render import RenderedWave, RenderingError + +RENDERER: Final[str] = "sampletones_tools.samples.render.render_directory" + + +class TestNsfRender: + def test_every_wave_written_is_printed( + self, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + ) -> None: + calls: List[Tuple[Path, float]] = [] + + def render_directory(directory: Path, tail_seconds: float) -> List[RenderedWave]: + calls.append((directory, tail_seconds)) + return [RenderedWave(source=directory / "a.nsf", destination=directory / "a.wav", seconds=1.5)] + + monkeypatch.setattr(RENDERER, render_directory) + + assert dispatch(COMMANDS, ["nsf", "render", "--directory", str(tmp_path)]) == 0 + assert calls == [(tmp_path, 0.5)] + assert f"{tmp_path / 'a.wav'} 1.500 s" in capsys.readouterr().out + + def test_the_directory_is_required(self) -> None: + with pytest.raises(SystemExit) as leaving: + dispatch(COMMANDS, ["nsf", "render"]) + + assert leaving.value.code == 2 + + def test_an_action_is_required(self) -> None: + with pytest.raises(SystemExit) as leaving: + dispatch(COMMANDS, ["nsf"]) + + assert leaving.value.code == 2 + + def test_a_rendering_failure_is_reported(self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + def fail(directory: Path, tail_seconds: float) -> List[RenderedWave]: + raise RenderingError(f"ffmpeg is missing for {directory} at {tail_seconds}") + + monkeypatch.setattr(RENDERER, fail) + + with pytest.raises(SystemExit, match="ffmpeg is missing"): + dispatch(COMMANDS, ["nsf", "render", "--directory", str(tmp_path)]) diff --git a/tests/unit/sampletones_tools/samples/test_render.py b/tests/unit/sampletones_tools/samples/test_render.py new file mode 100644 index 000000000..132ece2c3 --- /dev/null +++ b/tests/unit/sampletones_tools/samples/test_render.py @@ -0,0 +1,44 @@ +import struct +from pathlib import Path + +import pytest + +from sampletones_player.specification.clock import FIXED_POINT_SCALE, NTSC_FRAME_RATE +from sampletones_player.specification.nsf import HEADER_SIZE +from sampletones_player.specification.song import ( + STEP_FRACTION_OFFSET, + STEP_WHOLE_OFFSET, + TOTAL_TICKS_OFFSET, +) +from sampletones_tools.samples import render +from sampletones_tools.samples.render import RenderingError, render_directory, song_seconds + +CODE_LENGTH = 10 + + +def _exported(ticks: int, whole: int, fraction: int) -> bytes: + block = bytearray(max(TOTAL_TICKS_OFFSET, STEP_FRACTION_OFFSET, STEP_WHOLE_OFFSET) + 2) + struct.pack_into(" None: + data = _exported(ticks=600, whole=1, fraction=0) + + assert song_seconds(data, CODE_LENGTH) == pytest.approx(600 / float(NTSC_FRAME_RATE)) + + def test_a_fractional_step_takes_fewer_calls(self) -> None: + data = _exported(ticks=600, whole=1, fraction=FIXED_POINT_SCALE // 2) + + assert song_seconds(data, CODE_LENGTH) == pytest.approx(400 / float(NTSC_FRAME_RATE)) + + +class TestRenderDirectory: + def test_a_directory_without_exports_is_refused(self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + monkeypatch.setattr(render, "require_renderer", lambda: None) + + with pytest.raises(RenderingError, match="no .nsf files"): + render_directory(tmp_path, 0.5) From 2a50aae9934818a60fb56dc6305a101260134b96 Mon Sep 17 00:00:00 2001 From: JakimPL Date: Sun, 13 Sep 2026 16:00:33 +0200 Subject: [PATCH 13/36] Moved: the mark and the icon suite into the tools package --- .pre-commit-config.yaml | 4 +- LICENSE | 2 +- Makefile | 6 +-- THIRD-PARTY-NOTICES.md | 2 +- docs/development/dependencies.md | 19 +++---- docs/development/packages.md | 7 ++- docs/development/tooling.md | 7 +-- scripts/assets/icons.py | 46 ----------------- scripts/bootstrap/venv_build.py | 8 +-- scripts/bundle.py | 6 --- scripts/setup_environment.py | 2 - src/sampletones_config/boundaries/graphs.yaml | 2 +- .../boundaries/standalone.yaml | 1 - .../assets}/__init__.py | 0 src/sampletones_tools/assets/command.py | 48 +++++++++++++++++ .../assets/mark}/__init__.py | 0 .../assets}/mark/geometry.py | 4 +- .../assets}/mark/mark.yaml | 0 .../assets}/mark/paths.py | 2 +- .../assets}/mark/raster.py | 4 +- .../assets}/mark/specification/__init__.py | 12 ++--- .../assets}/mark/specification/colors.py | 0 .../assets}/mark/specification/frame.py | 0 .../assets}/mark/specification/point.py | 0 .../assets}/mark/specification/render.py | 0 .../assets}/mark/specification/waves.py | 2 +- .../assets}/mark/suite.py | 6 +-- .../assets}/mark/template.svg | 0 .../assets}/mark/vector.py | 8 +-- src/sampletones_tools/assets/paths.py | 7 +++ src/sampletones_tools/registry.py | 3 +- .../assets}/mark/__init__.py | 0 .../assets}/mark/test_geometry.py | 4 +- .../assets}/mark/test_raster.py | 4 +- .../assets}/mark/test_specification.py | 2 +- .../assets}/mark/test_suite.py | 4 +- .../assets}/mark/test_vector.py | 4 +- .../sampletones_tools/assets/test_command.py | 51 +++++++++++++++++++ .../unit/scripts/bootstrap/test_venv_build.py | 7 ++- tests/unit/scripts/test_bundle.py | 5 +- tests/unit/scripts/test_setup_environment.py | 6 +-- 41 files changed, 169 insertions(+), 126 deletions(-) delete mode 100755 scripts/assets/icons.py rename src/{sampletones_assets/mark => sampletones_tools/assets}/__init__.py (100%) create mode 100644 src/sampletones_tools/assets/command.py rename {tests/unit/sampletones_assets => src/sampletones_tools/assets/mark}/__init__.py (100%) rename src/{sampletones_assets => sampletones_tools/assets}/mark/geometry.py (95%) rename src/{sampletones_assets => sampletones_tools/assets}/mark/mark.yaml (100%) rename src/{sampletones_assets => sampletones_tools/assets}/mark/paths.py (71%) rename src/{sampletones_assets => sampletones_tools/assets}/mark/raster.py (96%) rename src/{sampletones_assets => sampletones_tools/assets}/mark/specification/__init__.py (73%) rename src/{sampletones_assets => sampletones_tools/assets}/mark/specification/colors.py (100%) rename src/{sampletones_assets => sampletones_tools/assets}/mark/specification/frame.py (100%) rename src/{sampletones_assets => sampletones_tools/assets}/mark/specification/point.py (100%) rename src/{sampletones_assets => sampletones_tools/assets}/mark/specification/render.py (100%) rename src/{sampletones_assets => sampletones_tools/assets}/mark/specification/waves.py (96%) rename src/{sampletones_assets => sampletones_tools/assets}/mark/suite.py (91%) rename src/{sampletones_assets => sampletones_tools/assets}/mark/template.svg (100%) rename src/{sampletones_assets => sampletones_tools/assets}/mark/vector.py (89%) create mode 100644 src/sampletones_tools/assets/paths.py rename tests/unit/{sampletones_assets => sampletones_tools/assets}/mark/__init__.py (100%) rename tests/unit/{sampletones_assets => sampletones_tools/assets}/mark/test_geometry.py (94%) rename tests/unit/{sampletones_assets => sampletones_tools/assets}/mark/test_raster.py (93%) rename tests/unit/{sampletones_assets => sampletones_tools/assets}/mark/test_specification.py (98%) rename tests/unit/{sampletones_assets => sampletones_tools/assets}/mark/test_suite.py (94%) rename tests/unit/{sampletones_assets => sampletones_tools/assets}/mark/test_vector.py (93%) create mode 100644 tests/unit/sampletones_tools/assets/test_command.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a8eee20e8..29fcd0dbb 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -109,9 +109,9 @@ repos: - id: icons name: icons - entry: uv run python scripts/assets/icons.py + entry: uv run sampletones icons language: system - files: ^src/sampletones_assets/(icons|mark)/ + files: ^src/(sampletones_assets/icons|sampletones_tools/assets/mark)/ pass_filenames: false verbose: true stages: diff --git a/LICENSE b/LICENSE index 5918803cf..119ae7fc2 100644 --- a/LICENSE +++ b/LICENSE @@ -24,7 +24,7 @@ SOFTWARE. The MIT license above covers the SampleToNES source code and the application icons under `src/sampletones_assets/icons/`, which are drawn from the mark -declared in `src/sampletones_assets/mark/`. +declared in `src/sampletones_tools/assets/mark/`. Font files bundled under `src/sampletones_assets/fonts/` are the work of third parties and remain under their own licenses (SIL Open Font License 1.1 and the diff --git a/Makefile b/Makefile index 216c9ef22..b68d5fb74 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ .PHONY: help setup install system-deps build release run clean pre-commit test benchmarks lint format \ - ftm-samples nsf-samples compression-report compression-study icons \ + ftm-samples nsf-samples compression-report compression-study \ check-import-boundary check-tag-names check-unused-tags check-rendered-literals check-language-keys \ check-palette-colors check-shortcut-actions @@ -36,7 +36,6 @@ help: @echo $(Q) make nsf-samples - Emit example .nsf files to build/nsf via the integration suite$(Q) @echo $(Q) make compression-report - Measure the song codec into build/compression$(Q) @echo $(Q) make compression-study - Measure the song codec over the projects and stems on this machine; the report lands in Documents/SampleToNES/compression (ARGS=--quick for a short run)$(Q) - @echo $(Q) make icons - Generate the icon suite into src/sampletones_assets/icons$(Q) @echo $(Q) make clean - Remove build artifacts and cache files$(Q) @echo $(Q) make lint - Run mypy and pylint (ARGS=--mypy or ARGS=--pylint for one of them)$(Q) @echo $(Q) make format - Auto-format code (isort, black)$(Q) @@ -94,9 +93,6 @@ compression-report: compression-study: uv run scripts/compression_study.py $(ARGS) -icons: - uv run --group assets python scripts/assets/icons.py - check-import-boundary: uv run scripts/checks/import_boundary.py --all diff --git a/THIRD-PARTY-NOTICES.md b/THIRD-PARTY-NOTICES.md index 99cd24b32..f702953e4 100644 --- a/THIRD-PARTY-NOTICES.md +++ b/THIRD-PARTY-NOTICES.md @@ -112,7 +112,7 @@ CUDA components from their publishers straight to your machine. ## Build-time tooling -The application icons are drawn by `sampletones_assets.mark` and rasterized with +The application icons are drawn by `sampletones_tools.assets.mark` and rasterized with [Pillow](https://pypi.org/project/Pillow/), which is under the [MIT-CMU license](https://github.com/python-pillow/Pillow/blob/main/LICENSE). Pillow belongs to the `assets` dependency group alone, so `pip`/`uv` installs it on the machine that diff --git a/docs/development/dependencies.md b/docs/development/dependencies.md index a669495df..94e62ff66 100644 --- a/docs/development/dependencies.md +++ b/docs/development/dependencies.md @@ -53,20 +53,21 @@ Dialogs open through the XDG desktop portal (`org.freedesktop.portal.FileChooser ## Application icon -The icon suite in `src/sampletones_assets/icons` is generated from the mark declared beside it in -`src/sampletones_assets/mark`: `mark.yaml` carries the geometry, colors and rasterization +The icon suite in `src/sampletones_assets/icons` is generated from the mark declared in +`src/sampletones_tools/assets/mark`: `mark.yaml` carries the geometry, colors and rasterization settings, validated as a `Mark`, and `template.svg` is the vector the rendered geometry fills. The -package writes the whole suite — the vector `sampletones.svg` and the rasters the application -ships, `sampletones.png` and the multi-resolution `sampletones.ico` — and `scripts/assets/icons.py` +tools package writes the whole suite — the vector `sampletones.svg` and the rasters the application +ships, `sampletones.png` and the multi-resolution `sampletones.ico` — and `uv run sampletones icons` points it at the directory the icons are shipped from. Rasterization uses Pillow, declared in the -`assets` dependency group. +`assets` dependency group, which the `dev` group includes. The whole suite is committed, so a plain checkout carries the icons the application opens its window -with, and every wheel, bundle and test run finds them where they lie. `make icons` writes them again -from the mark, and the `icons` pre-push hook writes them for a push that touches either directory, -holding the committed files to what the mark describes. CI runs that same hook. +with, and every wheel, bundle and test run finds them where they lie. `uv run sampletones icons` +writes them again from the mark, and the `icons` pre-push hook writes them for a push that touches +either directory, holding the committed files to what the mark describes. CI runs that same hook. -Pillow is a build-time tool, and the bundle script passes `--exclude-module PIL` to hold it to that: +Pillow is a developer tool the build environment never installs, and the bundle script passes +`--exclude-module PIL` besides, to hold it to that: `pygments`, which arrives with `rich`, offers an image formatter that imports Pillow where it is installed, and PyInstaller follows that import into the bundle. The application reads its icons as files, so the exclusion spares every bundle Pillow's extension modules and the imaging libraries diff --git a/docs/development/packages.md b/docs/development/packages.md index ac79da956..832346f16 100644 --- a/docs/development/packages.md +++ b/docs/development/packages.md @@ -21,7 +21,7 @@ graph TD APP["sampletones_application\n(GUI)"] PLAYER["sampletones_player\n(NES player)"] CORE["sampletones_core\n(reconstruction engine)"] - ASSETS["sampletones_assets\n(mark and fonts)"] + ASSETS["sampletones_assets\n(icons and fonts)"] SHARED["sampletones_shared\n(facts and helpers)"] CONFIG["sampletones_config\n(shipped YAML)"] @@ -36,7 +36,6 @@ graph TD APP --> PLAYER APP --> CORE PLAYER --> CORE - ASSETS --> SHARED CORE --> SHARED PLAYER --> SHARED APP --> SHARED @@ -47,11 +46,11 @@ graph TD |---------|---------|------------| | `sampletones_shared` | Facts and helpers any package holds: constants, exception families, paths, the logger, the array backend, the source layer the checks read the tree through, and the schema these boundaries are declared in | — | | `sampletones_config` | The shipped YAML — layout, palettes, themes, keybindings, language, calibration, and these boundaries themselves — reached as package data rather than by import | — | -| `sampletones_assets` | The application mark and the bundled fonts, with the code that draws the mark | `sampletones_shared` | +| `sampletones_assets` | The application icons and the bundled fonts, reached as package data | — | | `sampletones_core` | The reconstruction engine, the project model, playing a song out into instructions, and the tracker export formats | `sampletones_shared` | | `sampletones_player` | The NES player: the register model, the re-clocking schedule, the 6502 driver and the NSF file | `sampletones_shared`, `sampletones_core` | | `sampletones_application` | The DearPyGui front end | `sampletones_shared`, `sampletones_core`, `sampletones_player` | -| `sampletones_tools` | Everything a developer runs and the application does not: analytic waveform synthesis, the calibration harness, the driver toolchain and the register trace, and the developer commands that run them | `sampletones_shared`, `sampletones_assets`, `sampletones_core`, `sampletones_player`, `sampletones_application` | +| `sampletones_tools` | Everything a developer runs and the application does not: analytic waveform synthesis, the calibration harness, the driver toolchain and the register trace, the mark the icons are drawn from, and the developer commands that run them | `sampletones_shared`, `sampletones_assets`, `sampletones_core`, `sampletones_player`, `sampletones_application` | | `sampletones` | The command-line entry: the dispatcher, the commands and the startup self-check | `sampletones_shared`, `sampletones_core`, `sampletones_application`, `sampletones_tools` | Third-party imports are the package author's own choice and stand outside this table. diff --git a/docs/development/tooling.md b/docs/development/tooling.md index 7518eadb3..5e96c276e 100644 --- a/docs/development/tooling.md +++ b/docs/development/tooling.md @@ -83,6 +83,7 @@ The developer commands, listed by `sampletones_tools/registry.py` and run as |---|---| | `calibration [--config FILE] [-o DIR] [--methods LIST] [--perceptual-exponents LIST] [--temporal-weights LIST] [--channels LIST]` | Reconstructs the calibration corpus under every variant of the sweep, scores it with every referee, and writes the reports; without `-o` the run lands in a timestamped directory under Documents/SampleToNES/calibration | | `driver [--directory DIR]` | Assembles the NES player driver with cc65 and prints the layout the build produced; without `--directory` it writes the driver the package ships, which needs a checkout | +| `icons [--directory DIR]` | Writes the icon suite from the mark; without `--directory` it writes the icons the package ships, which needs a checkout | | `nsf render --directory DIR [--tail SECONDS]` | Renders every exported `.nsf` file in the directory to a wave beside it, through ffmpeg's libgme demuxer | More join as the tools they run move into the package. @@ -119,7 +120,7 @@ reason. | Script | Target | What it does | |---|---|---| | `bundle.py` | `make build`, `make release` | Creates `.venv-build`, installs the package with the `build` extra, checks the interpreter carries PortAudio (and Tk, for a release), writes the bundle with PyInstaller, runs its self-check, and copies the notices beside a release | -| `setup_environment.py` | `make setup` | Reads the NVIDIA driver, synchronizes the development environment with the matching GPU extra, writes the icons, installs the global `sampletones` command | +| `setup_environment.py` | `make setup` | Reads the NVIDIA driver, synchronizes the development environment with the matching GPU extra, installs the global `sampletones` command | | `system_dependencies.py` | `make system-deps` | Installs the system packages: apt on Debian-based Linux, Homebrew on macOS, nothing on Windows | | `build_environment.py` | CI | Prints the compiler flags a macOS build exports, one `KEY=VALUE` per line | | `clean.py` | `make clean` | Removes the build outputs, the coverage reports and the bytecode caches | @@ -139,8 +140,8 @@ interpreter (`preflight.py`), and the platforms (`platforms/`). ## The tool scripts -`compression_study.py`, `assets/icons.py` and the checks under `checks/` import the project's -packages and run inside its environment, from the make target that names each. The checks are also pre-commit hooks; +`compression_study.py` and the checks under `checks/` import the project's packages and run +inside its environment, from the make target that names each. The checks are also pre-commit hooks; [architecture](architecture.md#enforcement) lists them. ## Who governs what diff --git a/scripts/assets/icons.py b/scripts/assets/icons.py deleted file mode 100755 index c6484eb94..000000000 --- a/scripts/assets/icons.py +++ /dev/null @@ -1,46 +0,0 @@ -#!/usr/bin/env python3 - -""" -Writes the application icon suite from the packaged mark definition. - -The mark, its template and the code drawing them live in `sampletones_assets/mark`; this -script points them at the directory the icons are shipped from. - -Usage: - python scripts/assets/icons.py # write the suite into src/sampletones_assets/icons -""" - -import argparse -import sys -from pathlib import Path -from typing import Final, Sequence - -from sampletones_assets.mark.specification import Mark -from sampletones_assets.mark.suite import write_icon_suite - -REPOSITORY_ROOT: Final[Path] = Path(__file__).resolve().parents[2] -ICONS_DIRECTORY: Final[Path] = REPOSITORY_ROOT / "src" / "sampletones_assets" / "icons" - - -def main(argv: Sequence[str]) -> int: - """Writes the icon suite and reports each file it produced.""" - - parser = argparse.ArgumentParser( - description="Write the application icon suite from the mark definition.", - ) - parser.add_argument( - "--directory", - type=Path, - default=ICONS_DIRECTORY, - help="directory receiving the icon files", - ) - arguments = parser.parse_args(list(argv)) - - for path in write_icon_suite(arguments.directory, Mark.load()): - print(f"Wrote {path}") - - return 0 - - -if __name__ == "__main__": - raise SystemExit(main(sys.argv[1:])) diff --git a/scripts/bootstrap/venv_build.py b/scripts/bootstrap/venv_build.py index f0dd7ac09..c8928cc7d 100644 --- a/scripts/bootstrap/venv_build.py +++ b/scripts/bootstrap/venv_build.py @@ -7,7 +7,6 @@ BUILD_ENVIRONMENT: Final[str] = ".venv-build" PIP_REQUIRE_VIRTUALENV: Final[str] = "PIP_REQUIRE_VIRTUALENV" -GROUP_FLAG: Final[str] = "--group" def build_environment( @@ -50,11 +49,10 @@ def install( python: Path, *, extras: Sequence[str], - groups: Sequence[str], runner: Runner, environment: Mapping[str, str], ) -> None: - """Installs the package with ``extras`` and ``groups`` into the environment ``python`` runs. + """Installs the package with ``extras`` into the environment ``python`` runs. Pip is told to refuse any interpreter outside a virtual environment, so an install reaches the build environment alone. @@ -63,7 +61,6 @@ def install( root: The repository, which is the package installed. python: The build environment's interpreter. extras: The optional-dependency extras installed with the package. - groups: The dependency groups installed beside it. runner: What runs the commands. environment: The variables the commands see. """ @@ -76,10 +73,9 @@ def install( environment=guarded, ) print(f"Installing with extras: {','.join(extras)}") - group_flags = [flag for group in groups for flag in (GROUP_FLAG, group)] expect_success( runner, - (str(python), "-m", "pip", "install", f".[{','.join(extras)}]", *group_flags), + (str(python), "-m", "pip", "install", f".[{','.join(extras)}]"), cwd=root, environment=guarded, ) diff --git a/scripts/bundle.py b/scripts/bundle.py index fa976c243..054ac2a2a 100644 --- a/scripts/bundle.py +++ b/scripts/bundle.py @@ -18,11 +18,9 @@ DISTRIBUTION: Final[str] = "bin" ENTRY: Final[str] = "src/sampletones/__main__.py" RELEASE_HOOK: Final[str] = "scripts/runtime_hooks/release_environment.py" -ICONS_SCRIPT: Final[str] = "scripts/assets/icons.py" SELF_CHECK: Final[str] = "self-check" BUILD_EXTRA: Final[str] = "build" GPU_EXTRA: Final[str] = "gpu" -GROUPS: Final[Tuple[str, ...]] = ("assets",) DATA: Final[Tuple[Tuple[str, str], ...]] = ( ("src/sampletones_assets/icons", "assets/icons"), ("src/sampletones_assets/fonts", "assets/fonts"), @@ -159,7 +157,6 @@ def build_bundle( root, python, extras=extras(options), - groups=GROUPS, runner=runner, environment=environment, ) @@ -171,9 +168,6 @@ def build_bundle( cwd=root, environment=environment, ) - print("Generating the icon suite...") - expect_success(runner, (str(python), ICONS_SCRIPT), cwd=root, environment=environment) - distribution = root / DISTRIBUTION remove_previous(distribution) print("Building executable...") diff --git a/scripts/setup_environment.py b/scripts/setup_environment.py index 66d3c065b..990eff8cc 100644 --- a/scripts/setup_environment.py +++ b/scripts/setup_environment.py @@ -13,7 +13,6 @@ GPU_OFF: Final[str] = "0" DEFAULT_GPU: Final[str] = GPU_AUTO DEVELOPMENT_GROUP: Final[str] = "dev" -ICONS_COMMAND: Final[Sequence[str]] = ("uv", "run", "--group", "assets", "python", "scripts/assets/icons.py") DARWIN: Final[str] = "Darwin" ARCHFLAGS: Final[str] = "ARCHFLAGS" @@ -56,7 +55,6 @@ def setup_commands(extra: Optional[str]) -> List[List[str]]: return [ synchronize, - list(ICONS_COMMAND), ["uv", "tool", "install", "--force", package], ] diff --git a/src/sampletones_config/boundaries/graphs.yaml b/src/sampletones_config/boundaries/graphs.yaml index eb6d16180..746220db7 100644 --- a/src/sampletones_config/boundaries/graphs.yaml +++ b/src/sampletones_config/boundaries/graphs.yaml @@ -2,7 +2,7 @@ packages: layers: sampletones_shared: [] sampletones_config: [] - sampletones_assets: [sampletones_shared] + sampletones_assets: [] sampletones_core: [sampletones_shared] sampletones_player: [sampletones_shared, sampletones_core] sampletones_application: [sampletones_shared, sampletones_core, sampletones_player] diff --git a/src/sampletones_config/boundaries/standalone.yaml b/src/sampletones_config/boundaries/standalone.yaml index b2738a020..8d7103283 100644 --- a/src/sampletones_config/boundaries/standalone.yaml +++ b/src/sampletones_config/boundaries/standalone.yaml @@ -1,7 +1,6 @@ - pattern: "**/*.py" excluding: - "compression_study.py" - - "assets/**/*.py" - "checks/**/*.py" - "codec_study/**/*.py" reserved: [build, test, tests] diff --git a/src/sampletones_assets/mark/__init__.py b/src/sampletones_tools/assets/__init__.py similarity index 100% rename from src/sampletones_assets/mark/__init__.py rename to src/sampletones_tools/assets/__init__.py diff --git a/src/sampletones_tools/assets/command.py b/src/sampletones_tools/assets/command.py new file mode 100644 index 000000000..d003d00e9 --- /dev/null +++ b/src/sampletones_tools/assets/command.py @@ -0,0 +1,48 @@ +from argparse import ArgumentParser, Namespace +from dataclasses import dataclass +from pathlib import Path +from typing import Final, Optional + +from sampletones_shared.command import Command + +NAME: Final[str] = "icons" +HELP: Final[str] = "write the application icon suite from the mark" +DIRECTORY_HELP: Final[str] = "the directory receiving the icon files; without it, the icons the package ships" + + +@dataclass(frozen=True) +class IconsArguments: + """What an icon suite run is given: where the files go, if anywhere but the package.""" + + directory: Optional[Path] + + +def configure(parser: ArgumentParser) -> None: + parser.add_argument("--directory", type=Path, default=None, help=DIRECTORY_HELP) + + +def run(arguments: Namespace) -> int: + """Writes the icon suite and reports each file produced. + + Writing the icons the package ships needs a checkout, since that is where the package is. + + Raises: + SystemExit: If the run writes the shipped icons outside a checkout. + """ + given = IconsArguments(directory=arguments.directory) + + from sampletones_tools.assets.mark.specification import Mark + from sampletones_tools.assets.mark.suite import write_icon_suite + from sampletones_tools.assets.paths import ICONS_DIRECTORY + from sampletones_tools.checkout import require_checkout + + if given.directory is None: + require_checkout(NAME) + + for path in write_icon_suite(given.directory if given.directory is not None else ICONS_DIRECTORY, Mark.load()): + print(f"Wrote {path}") + + return 0 + + +ICONS: Final[Command] = Command(name=NAME, help=HELP, configure=configure, run=run) diff --git a/tests/unit/sampletones_assets/__init__.py b/src/sampletones_tools/assets/mark/__init__.py similarity index 100% rename from tests/unit/sampletones_assets/__init__.py rename to src/sampletones_tools/assets/mark/__init__.py diff --git a/src/sampletones_assets/mark/geometry.py b/src/sampletones_tools/assets/mark/geometry.py similarity index 95% rename from src/sampletones_assets/mark/geometry.py rename to src/sampletones_tools/assets/mark/geometry.py index 5fd855bdd..fcbac6320 100644 --- a/src/sampletones_assets/mark/geometry.py +++ b/src/sampletones_tools/assets/mark/geometry.py @@ -2,8 +2,8 @@ from dataclasses import dataclass from typing import List -from sampletones_assets.mark.specification.point import CubicCurve, Point -from sampletones_assets.mark.specification.waves import MarkSine, MarkSquare +from sampletones_tools.assets.mark.specification.point import CubicCurve, Point +from sampletones_tools.assets.mark.specification.waves import MarkSine, MarkSquare @dataclass(frozen=True) diff --git a/src/sampletones_assets/mark/mark.yaml b/src/sampletones_tools/assets/mark/mark.yaml similarity index 100% rename from src/sampletones_assets/mark/mark.yaml rename to src/sampletones_tools/assets/mark/mark.yaml diff --git a/src/sampletones_assets/mark/paths.py b/src/sampletones_tools/assets/mark/paths.py similarity index 71% rename from src/sampletones_assets/mark/paths.py rename to src/sampletones_tools/assets/mark/paths.py index 4b9decd97..60ea38088 100644 --- a/src/sampletones_assets/mark/paths.py +++ b/src/sampletones_tools/assets/mark/paths.py @@ -2,6 +2,6 @@ from pathlib import Path from typing import Final -MARK_DIRECTORY: Final[Path] = Path(str(files("sampletones_assets.mark"))) +MARK_DIRECTORY: Final[Path] = Path(str(files("sampletones_tools.assets.mark"))) MARK_PATH: Final[Path] = MARK_DIRECTORY / "mark.yaml" TEMPLATE_PATH: Final[Path] = MARK_DIRECTORY / "template.svg" diff --git a/src/sampletones_assets/mark/raster.py b/src/sampletones_tools/assets/mark/raster.py similarity index 96% rename from src/sampletones_assets/mark/raster.py rename to src/sampletones_tools/assets/mark/raster.py index e7d388bd9..537130d0f 100644 --- a/src/sampletones_assets/mark/raster.py +++ b/src/sampletones_tools/assets/mark/raster.py @@ -2,10 +2,10 @@ from PIL import Image, ImageDraw -from sampletones_assets.mark.geometry import sine_points, square_rectangles -from sampletones_assets.mark.specification import Mark from sampletones_shared.types.application import ColorRGBA from sampletones_shared.utils.color import parse_hex_color, with_alpha_fraction +from sampletones_tools.assets.mark.geometry import sine_points, square_rectangles +from sampletones_tools.assets.mark.specification import Mark TRANSPARENT: Final[ColorRGBA] = (0, 0, 0, 0) OPAQUE: Final[int] = 255 diff --git a/src/sampletones_assets/mark/specification/__init__.py b/src/sampletones_tools/assets/mark/specification/__init__.py similarity index 73% rename from src/sampletones_assets/mark/specification/__init__.py rename to src/sampletones_tools/assets/mark/specification/__init__.py index 6e78e8710..a2f386226 100644 --- a/src/sampletones_assets/mark/specification/__init__.py +++ b/src/sampletones_tools/assets/mark/specification/__init__.py @@ -2,12 +2,12 @@ from pydantic import BaseModel, Field -from sampletones_assets.mark.paths import MARK_PATH -from sampletones_assets.mark.specification.colors import MarkColors -from sampletones_assets.mark.specification.frame import MarkFrame -from sampletones_assets.mark.specification.render import MarkRender -from sampletones_assets.mark.specification.waves import MarkWaves from sampletones_shared.utils.serialization import load_yaml_model +from sampletones_tools.assets.mark.paths import MARK_PATH +from sampletones_tools.assets.mark.specification.colors import MarkColors +from sampletones_tools.assets.mark.specification.frame import MarkFrame +from sampletones_tools.assets.mark.specification.render import MarkRender +from sampletones_tools.assets.mark.specification.waves import MarkWaves class Mark(BaseModel, extra="forbid", frozen=True): @@ -30,7 +30,7 @@ def load(cls) -> Self: """Load the packaged mark definition. Returns: - The mark validated from `sampletones_assets/mark/mark.yaml`. + The mark validated from `sampletones_tools/assets/mark/mark.yaml`. Raises: TypeError: If the definition file holds anything other than a mapping. diff --git a/src/sampletones_assets/mark/specification/colors.py b/src/sampletones_tools/assets/mark/specification/colors.py similarity index 100% rename from src/sampletones_assets/mark/specification/colors.py rename to src/sampletones_tools/assets/mark/specification/colors.py diff --git a/src/sampletones_assets/mark/specification/frame.py b/src/sampletones_tools/assets/mark/specification/frame.py similarity index 100% rename from src/sampletones_assets/mark/specification/frame.py rename to src/sampletones_tools/assets/mark/specification/frame.py diff --git a/src/sampletones_assets/mark/specification/point.py b/src/sampletones_tools/assets/mark/specification/point.py similarity index 100% rename from src/sampletones_assets/mark/specification/point.py rename to src/sampletones_tools/assets/mark/specification/point.py diff --git a/src/sampletones_assets/mark/specification/render.py b/src/sampletones_tools/assets/mark/specification/render.py similarity index 100% rename from src/sampletones_assets/mark/specification/render.py rename to src/sampletones_tools/assets/mark/specification/render.py diff --git a/src/sampletones_assets/mark/specification/waves.py b/src/sampletones_tools/assets/mark/specification/waves.py similarity index 96% rename from src/sampletones_assets/mark/specification/waves.py rename to src/sampletones_tools/assets/mark/specification/waves.py index b97a1c35f..cb5b6f9b3 100644 --- a/src/sampletones_assets/mark/specification/waves.py +++ b/src/sampletones_tools/assets/mark/specification/waves.py @@ -3,7 +3,7 @@ from pydantic import BaseModel, Field, PositiveFloat, model_validator -from sampletones_assets.mark.specification.point import CubicCurve, Point +from sampletones_tools.assets.mark.specification.point import CubicCurve, Point class MarkSine(BaseModel, extra="forbid", frozen=True): diff --git a/src/sampletones_assets/mark/suite.py b/src/sampletones_tools/assets/mark/suite.py similarity index 91% rename from src/sampletones_assets/mark/suite.py rename to src/sampletones_tools/assets/mark/suite.py index a464b50a0..76adaba53 100644 --- a/src/sampletones_assets/mark/suite.py +++ b/src/sampletones_tools/assets/mark/suite.py @@ -3,14 +3,14 @@ from PIL import Image -from sampletones_assets.mark.raster import MarkRaster -from sampletones_assets.mark.specification import Mark -from sampletones_assets.mark.vector import render_vector from sampletones_shared.paths.resources import ( ICON_UNIX_FILENAME, ICON_VECTOR_FILENAME, ICON_WIN_FILENAME, ) +from sampletones_tools.assets.mark.raster import MarkRaster +from sampletones_tools.assets.mark.specification import Mark +from sampletones_tools.assets.mark.vector import render_vector def _resized(master: Image.Image, size: int) -> Image.Image: diff --git a/src/sampletones_assets/mark/template.svg b/src/sampletones_tools/assets/mark/template.svg similarity index 100% rename from src/sampletones_assets/mark/template.svg rename to src/sampletones_tools/assets/mark/template.svg diff --git a/src/sampletones_assets/mark/vector.py b/src/sampletones_tools/assets/mark/vector.py similarity index 89% rename from src/sampletones_assets/mark/vector.py rename to src/sampletones_tools/assets/mark/vector.py index f9ba412c9..0f8cded6e 100644 --- a/src/sampletones_assets/mark/vector.py +++ b/src/sampletones_tools/assets/mark/vector.py @@ -2,10 +2,10 @@ from string import Template from typing import Dict -from sampletones_assets.mark.paths import TEMPLATE_PATH -from sampletones_assets.mark.specification import Mark -from sampletones_assets.mark.specification.point import Point -from sampletones_assets.mark.specification.waves import MarkSine, MarkSquare +from sampletones_tools.assets.mark.paths import TEMPLATE_PATH +from sampletones_tools.assets.mark.specification import Mark +from sampletones_tools.assets.mark.specification.point import Point +from sampletones_tools.assets.mark.specification.waves import MarkSine, MarkSquare def _number(value: float) -> str: diff --git a/src/sampletones_tools/assets/paths.py b/src/sampletones_tools/assets/paths.py new file mode 100644 index 000000000..4d4c72e04 --- /dev/null +++ b/src/sampletones_tools/assets/paths.py @@ -0,0 +1,7 @@ +from pathlib import Path +from typing import Final + +from sampletones_shared.paths.resources import ICON_DIRECTORY +from sampletones_shared.paths.source import SOURCE_ROOT + +ICONS_DIRECTORY: Final[Path] = SOURCE_ROOT / "sampletones_assets" / ICON_DIRECTORY diff --git a/src/sampletones_tools/registry.py b/src/sampletones_tools/registry.py index ccc075d17..e0cd7b9c3 100644 --- a/src/sampletones_tools/registry.py +++ b/src/sampletones_tools/registry.py @@ -1,8 +1,9 @@ from typing import Final, Tuple from sampletones_shared.command import Command +from sampletones_tools.assets.command import ICONS from sampletones_tools.calibration.command import CALIBRATION from sampletones_tools.player.command import DRIVER from sampletones_tools.samples.command import NSF -DEVELOPER_COMMANDS: Final[Tuple[Command, ...]] = (CALIBRATION, DRIVER, NSF) +DEVELOPER_COMMANDS: Final[Tuple[Command, ...]] = (CALIBRATION, DRIVER, ICONS, NSF) diff --git a/tests/unit/sampletones_assets/mark/__init__.py b/tests/unit/sampletones_tools/assets/mark/__init__.py similarity index 100% rename from tests/unit/sampletones_assets/mark/__init__.py rename to tests/unit/sampletones_tools/assets/mark/__init__.py diff --git a/tests/unit/sampletones_assets/mark/test_geometry.py b/tests/unit/sampletones_tools/assets/mark/test_geometry.py similarity index 94% rename from tests/unit/sampletones_assets/mark/test_geometry.py rename to tests/unit/sampletones_tools/assets/mark/test_geometry.py index 78279edb4..fb92405ee 100644 --- a/tests/unit/sampletones_assets/mark/test_geometry.py +++ b/tests/unit/sampletones_tools/assets/mark/test_geometry.py @@ -3,8 +3,8 @@ import pytest -from sampletones_assets.mark.geometry import Rectangle, sine_points, square_rectangles -from sampletones_assets.mark.specification import Mark +from sampletones_tools.assets.mark.geometry import Rectangle, sine_points, square_rectangles +from sampletones_tools.assets.mark.specification import Mark SAMPLES: Final[int] = 5 diff --git a/tests/unit/sampletones_assets/mark/test_raster.py b/tests/unit/sampletones_tools/assets/mark/test_raster.py similarity index 93% rename from tests/unit/sampletones_assets/mark/test_raster.py rename to tests/unit/sampletones_tools/assets/mark/test_raster.py index 91b5ae74c..acef28217 100644 --- a/tests/unit/sampletones_assets/mark/test_raster.py +++ b/tests/unit/sampletones_tools/assets/mark/test_raster.py @@ -2,9 +2,9 @@ import pytest -from sampletones_assets.mark.raster import MarkRaster -from sampletones_assets.mark.specification import Mark from sampletones_shared.utils.color import parse_hex_color +from sampletones_tools.assets.mark.raster import MarkRaster +from sampletones_tools.assets.mark.specification import Mark CORNER: Final[Tuple[int, int]] = (0, 0) ALPHA: Final[int] = 3 diff --git a/tests/unit/sampletones_assets/mark/test_specification.py b/tests/unit/sampletones_tools/assets/mark/test_specification.py similarity index 98% rename from tests/unit/sampletones_assets/mark/test_specification.py rename to tests/unit/sampletones_tools/assets/mark/test_specification.py index a3f51d363..9048f54d0 100644 --- a/tests/unit/sampletones_assets/mark/test_specification.py +++ b/tests/unit/sampletones_tools/assets/mark/test_specification.py @@ -4,7 +4,7 @@ import pytest from pydantic import ValidationError -from sampletones_assets.mark.specification import Mark +from sampletones_tools.assets.mark.specification import Mark from tests.suite.case import BaseRegularTestCase VALID_FRAME: Final[Dict[str, Any]] = { diff --git a/tests/unit/sampletones_assets/mark/test_suite.py b/tests/unit/sampletones_tools/assets/mark/test_suite.py similarity index 94% rename from tests/unit/sampletones_assets/mark/test_suite.py rename to tests/unit/sampletones_tools/assets/mark/test_suite.py index b0a8f3e3c..97821fcf3 100644 --- a/tests/unit/sampletones_assets/mark/test_suite.py +++ b/tests/unit/sampletones_tools/assets/mark/test_suite.py @@ -3,13 +3,13 @@ import pytest from PIL import Image -from sampletones_assets.mark.specification import Mark -from sampletones_assets.mark.suite import write_icon_suite from sampletones_shared.paths.resources import ( ICON_UNIX_FILENAME, ICON_VECTOR_FILENAME, ICON_WIN_FILENAME, ) +from sampletones_tools.assets.mark.specification import Mark +from sampletones_tools.assets.mark.suite import write_icon_suite RGBA_MODE = "RGBA" ICO_SIZES_KEY = "sizes" diff --git a/tests/unit/sampletones_assets/mark/test_vector.py b/tests/unit/sampletones_tools/assets/mark/test_vector.py similarity index 93% rename from tests/unit/sampletones_assets/mark/test_vector.py rename to tests/unit/sampletones_tools/assets/mark/test_vector.py index 108abb064..e73925b68 100644 --- a/tests/unit/sampletones_assets/mark/test_vector.py +++ b/tests/unit/sampletones_tools/assets/mark/test_vector.py @@ -1,9 +1,9 @@ from importlib.resources import files from pathlib import Path -from sampletones_assets.mark.specification import Mark -from sampletones_assets.mark.vector import render_vector from sampletones_shared.paths.resources import ICON_VECTOR_FILENAME +from sampletones_tools.assets.mark.specification import Mark +from sampletones_tools.assets.mark.vector import render_vector PLACEHOLDER_PREFIX = "$" REPLACEMENT_COLOR = "#010203" diff --git a/tests/unit/sampletones_tools/assets/test_command.py b/tests/unit/sampletones_tools/assets/test_command.py new file mode 100644 index 000000000..60c4dca77 --- /dev/null +++ b/tests/unit/sampletones_tools/assets/test_command.py @@ -0,0 +1,51 @@ +from pathlib import Path +from typing import Final, List, Tuple + +import pytest + +from sampletones.commands.registry import COMMANDS +from sampletones.dispatcher import dispatch +from sampletones_tools.assets.mark.specification import Mark +from sampletones_tools.assets.paths import ICONS_DIRECTORY + +WRITER: Final[str] = "sampletones_tools.assets.mark.suite.write_icon_suite" +GUARD: Final[str] = "sampletones_tools.checkout.require_checkout" + + +class RecordedSuite: + def __init__(self) -> None: + self.writes: List[Tuple[Path, Mark]] = [] + + def __call__(self, directory: Path, mark: Mark) -> List[Path]: + self.writes.append((directory, mark)) + return [directory / "sampletones.svg"] + + +def _refuse(command: str) -> None: + raise AssertionError(f"the checkout guard ran for {command}") + + +class TestIcons: + def test_without_a_directory_the_shipped_icons_are_written_from_a_checkout( + self, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], + ) -> None: + suite = RecordedSuite() + guarded: List[str] = [] + monkeypatch.setattr(WRITER, suite) + monkeypatch.setattr(GUARD, guarded.append) + + assert dispatch(COMMANDS, ["icons"]) == 0 + assert [directory for directory, _ in suite.writes] == [ICONS_DIRECTORY] + assert suite.writes[0][1] == Mark.load() + assert guarded == ["icons"] + assert f"Wrote {ICONS_DIRECTORY / 'sampletones.svg'}" in capsys.readouterr().out + + def test_a_directory_of_its_own_needs_no_checkout(self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + suite = RecordedSuite() + monkeypatch.setattr(WRITER, suite) + monkeypatch.setattr(GUARD, _refuse) + + assert dispatch(COMMANDS, ["icons", "--directory", str(tmp_path)]) == 0 + assert [directory for directory, _ in suite.writes] == [tmp_path] diff --git a/tests/unit/scripts/bootstrap/test_venv_build.py b/tests/unit/scripts/bootstrap/test_venv_build.py index f9b0bfe86..d6406103f 100644 --- a/tests/unit/scripts/bootstrap/test_venv_build.py +++ b/tests/unit/scripts/bootstrap/test_venv_build.py @@ -25,7 +25,7 @@ def test_an_existing_environment_is_kept(self, tmp_path: Path) -> None: class TestInstall: - def test_pip_is_upgraded_then_the_package_installed_with_its_extras_and_groups(self, tmp_path: Path) -> None: + def test_pip_is_upgraded_then_the_package_installed_with_its_extras(self, tmp_path: Path) -> None: runner = RecordingRunner({}, None) python = tmp_path / "python" @@ -33,20 +33,19 @@ def test_pip_is_upgraded_then_the_package_installed_with_its_extras_and_groups(s tmp_path, python, extras=("build", "gpu"), - groups=("assets",), runner=runner, environment={"PATH": "/usr/bin"}, ) assert runner.lines == [ f"{python} -m pip install --upgrade pip", - f"{python} -m pip install .[build,gpu] --group assets", + f"{python} -m pip install .[build,gpu]", ] def test_every_install_refuses_an_interpreter_outside_a_virtual_environment(self, tmp_path: Path) -> None: runner = RecordingRunner({}, None) - install(tmp_path, tmp_path / "python", extras=("build",), groups=(), runner=runner, environment={}) + install(tmp_path, tmp_path / "python", extras=("build",), runner=runner, environment={}) assert all(recorded.environment["PIP_REQUIRE_VIRTUALENV"] == "1" for recorded in runner.commands) diff --git a/tests/unit/scripts/test_bundle.py b/tests/unit/scripts/test_bundle.py index cefbb5da8..51faf5ae3 100644 --- a/tests/unit/scripts/test_bundle.py +++ b/tests/unit/scripts/test_bundle.py @@ -89,9 +89,8 @@ def leave_behind(command: Sequence[str]) -> None: assert ".[build]" in runner.lines[2] assert "import pyaudio" in runner.lines[3] assert "import tkinter" in runner.lines[4] - assert runner.lines[5].endswith(bundle.ICONS_SCRIPT) - assert "PyInstaller" in runner.lines[6] - assert runner.lines[7] == f"{launcher} self-check" + assert "PyInstaller" in runner.lines[5] + assert runner.lines[6] == f"{launcher} self-check" assert all((launcher.parent / notice).read_text() == notice for notice in bundle.NOTICES) def test_a_bundle_pyinstaller_never_wrote_is_reported(self, tmp_path: Path) -> None: diff --git a/tests/unit/scripts/test_setup_environment.py b/tests/unit/scripts/test_setup_environment.py index ae6e7570a..3b5e7cfca 100644 --- a/tests/unit/scripts/test_setup_environment.py +++ b/tests/unit/scripts/test_setup_environment.py @@ -18,15 +18,15 @@ class TestSetupCommands: def test_the_cpu_backend_synchronizes_and_installs_the_bare_package(self) -> None: commands = setup_environment.setup_commands(None) + assert len(commands) == 2 assert commands[0] == ["uv", "sync", "--group", "dev"] - assert commands[1][-1].endswith("icons.py") - assert commands[2] == ["uv", "tool", "install", "--force", "."] + assert commands[1] == ["uv", "tool", "install", "--force", "."] def test_a_gpu_extra_reaches_both_installs(self) -> None: commands = setup_environment.setup_commands("gpu") assert commands[0] == ["uv", "sync", "--group", "dev", "--extra", "gpu"] - assert commands[2] == ["uv", "tool", "install", "--force", ".[gpu]"] + assert commands[1] == ["uv", "tool", "install", "--force", ".[gpu]"] class TestSetupEnvironmentVariables: From 2a10bdcfcee72fab39dbe6fc19fc733e63a8aa6b Mon Sep 17 00:00:00 2001 From: JakimPL Date: Sun, 13 Sep 2026 16:16:00 +0200 Subject: [PATCH 14/36] Moved: the source checks into the tools package --- .github/workflows/workflow.yml | 4 + .pre-commit-config.yaml | 14 +-- Makefile | 25 +---- docs/development/architecture.md | 24 ++--- docs/development/config-organization.md | 9 +- docs/development/packages.md | 14 +-- docs/development/tooling.md | 7 +- scripts/checks/import_boundary.py | 100 ------------------ src/sampletones_config/README.md | 4 +- .../boundaries/standalone.yaml | 1 - src/sampletones_tools/assets/command.py | 7 +- .../checks}/__init__.py | 0 .../checks/boundary}/__init__.py | 0 .../checks/boundary}/check.py | 10 +- .../checks/boundary/configs}/__init__.py | 0 .../checks/boundary}/configs/declaration.py | 4 +- .../checks/boundary}/configs/general.py | 0 .../checks/boundary}/configs/paths.py | 0 .../checks/boundary}/configs/rules.py | 14 +-- .../checks/boundary}/graph.py | 4 +- .../checks/boundary}/imports.py | 2 +- .../checks/boundary}/lines.py | 2 +- .../checks/boundary}/rule.py | 6 +- .../checks/boundary}/scope.py | 0 .../checks/boundary}/standalone.py | 12 +-- .../checks/boundary}/token.py | 4 +- .../checks/boundary}/units.py | 2 +- .../checks/boundary}/violation.py | 0 src/sampletones_tools/checks/command.py | 33 ++++++ .../checks/commands}/__init__.py | 0 .../checks/commands/import_boundary.py | 54 ++++++++++ .../checks/commands/language_keys.py | 42 ++++++++ .../checks/commands/palette_colors.py | 48 +++++++++ .../checks/commands/rendered_literals.py | 33 ++++++ .../checks/commands/shortcut_actions.py | 23 ++++ .../checks/commands/tag_names.py | 36 +++++++ .../checks/commands/unused_tags.py | 47 ++++++++ .../checks/import_boundary.py | 39 +++++++ .../checks/language_keys.py | 60 ++--------- .../checks/palette_colors.py | 88 ++++----------- src/sampletones_tools/checks/registry.py | 20 ++++ .../checks/rendered_literals.py | 48 ++------- .../checks/shortcut_actions.py | 49 +-------- .../checks/source}/__init__.py | 0 .../checks}/source/annotations.py | 2 +- .../checks/source/bindings}/__init__.py | 0 .../checks}/source/bindings/containers.py | 4 +- .../checks}/source/bindings/environment.py | 0 .../checks}/source/bindings/scopes.py | 8 +- .../checks}/source/bindings/statements.py | 4 +- .../checks}/source/classes.py | 2 +- .../checks}/source/constants.py | 0 .../checks}/source/index.py | 6 +- .../checks}/source/lookups.py | 10 +- .../checks}/source/modules.py | 2 +- .../checks}/source/nodes.py | 0 .../checks}/source/packages.py | 0 .../checks}/source/references.py | 0 .../checks}/source/subscripts.py | 2 +- .../checks}/source/values.py | 4 +- .../sampletones_tools}/checks/tag_names.py | 57 +++------- .../sampletones_tools}/checks/unused_tags.py | 57 ++-------- src/sampletones_tools/player/command.py | 9 +- src/sampletones_tools/registry.py | 3 +- .../tooling/test_check_commands.py | 60 ++++------- tests/suite/source.py | 4 +- .../sampletones_shared/paths/test_source.py | 3 - .../checks/boundary}/__init__.py | 0 .../checks/boundary/configs}/__init__.py | 0 .../boundary}/configs/test_declaration.py | 4 +- .../checks/boundary}/configs/test_general.py | 2 +- .../checks/boundary}/configs/test_rules.py | 16 +-- .../checks/boundary}/test_check.py | 6 +- .../checks/boundary}/test_graph.py | 2 +- .../checks/boundary}/test_imports.py | 2 +- .../checks/boundary}/test_rule.py | 2 +- .../checks/boundary}/test_scope.py | 2 +- .../checks/boundary}/test_standalone.py | 2 +- .../checks/boundary}/test_token.py | 2 +- .../checks/boundary}/test_units.py | 2 +- .../checks/boundary}/test_violation.py | 2 +- .../checks/source/__init__.py | 0 .../checks/source/bindings/__init__.py | 0 .../source/bindings/test_containers.py | 2 +- .../source/bindings/test_environment.py | 2 +- .../checks}/source/bindings/test_scopes.py | 4 +- .../source/bindings/test_statements.py | 2 +- .../checks}/source/test_annotations.py | 2 +- .../checks}/source/test_classes.py | 2 +- .../checks}/source/test_constants.py | 2 +- .../checks}/source/test_index.py | 4 +- .../checks}/source/test_lookups.py | 8 +- .../checks}/source/test_modules.py | 4 +- .../checks}/source/test_nodes.py | 2 +- .../checks}/source/test_packages.py | 4 +- .../checks}/source/test_references.py | 2 +- .../checks}/source/test_subscripts.py | 4 +- .../checks}/source/test_values.py | 4 +- .../sampletones_tools/checks/test_command.py | 61 +++++++++++ .../checks/test_import_boundary.py | 25 +++-- .../checks/test_language_keys.py | 18 ++-- .../checks/test_palette_colors.py | 8 +- .../checks/test_rendered_literals.py | 12 +-- .../checks/test_shortcut_actions.py | 6 +- .../checks/test_tag_names.py | 14 +-- .../checks/test_unused_tags.py | 12 +-- 106 files changed, 726 insertions(+), 648 deletions(-) delete mode 100755 scripts/checks/import_boundary.py rename src/{sampletones_shared/meta/import_boundary => sampletones_tools/checks}/__init__.py (100%) rename src/{sampletones_shared/meta/import_boundary/configs => sampletones_tools/checks/boundary}/__init__.py (100%) rename src/{sampletones_shared/meta/import_boundary => sampletones_tools/checks/boundary}/check.py (83%) rename src/{sampletones_shared/meta/source => sampletones_tools/checks/boundary/configs}/__init__.py (100%) rename src/{sampletones_shared/meta/import_boundary => sampletones_tools/checks/boundary}/configs/declaration.py (93%) rename src/{sampletones_shared/meta/import_boundary => sampletones_tools/checks/boundary}/configs/general.py (100%) rename src/{sampletones_shared/meta/import_boundary => sampletones_tools/checks/boundary}/configs/paths.py (100%) rename src/{sampletones_shared/meta/import_boundary => sampletones_tools/checks/boundary}/configs/rules.py (84%) rename src/{sampletones_shared/meta/import_boundary => sampletones_tools/checks/boundary}/graph.py (96%) rename src/{sampletones_shared/meta/import_boundary => sampletones_tools/checks/boundary}/imports.py (91%) rename src/{sampletones_shared/meta/import_boundary => sampletones_tools/checks/boundary}/lines.py (91%) rename src/{sampletones_shared/meta/import_boundary => sampletones_tools/checks/boundary}/rule.py (91%) rename src/{sampletones_shared/meta/import_boundary => sampletones_tools/checks/boundary}/scope.py (100%) rename src/{sampletones_shared/meta/import_boundary => sampletones_tools/checks/boundary}/standalone.py (92%) rename src/{sampletones_shared/meta/import_boundary => sampletones_tools/checks/boundary}/token.py (92%) rename src/{sampletones_shared/meta/import_boundary => sampletones_tools/checks/boundary}/units.py (94%) rename src/{sampletones_shared/meta/import_boundary => sampletones_tools/checks/boundary}/violation.py (100%) create mode 100644 src/sampletones_tools/checks/command.py rename src/{sampletones_shared/meta/source/bindings => sampletones_tools/checks/commands}/__init__.py (100%) create mode 100644 src/sampletones_tools/checks/commands/import_boundary.py create mode 100644 src/sampletones_tools/checks/commands/language_keys.py create mode 100644 src/sampletones_tools/checks/commands/palette_colors.py create mode 100644 src/sampletones_tools/checks/commands/rendered_literals.py create mode 100644 src/sampletones_tools/checks/commands/shortcut_actions.py create mode 100644 src/sampletones_tools/checks/commands/tag_names.py create mode 100644 src/sampletones_tools/checks/commands/unused_tags.py create mode 100755 src/sampletones_tools/checks/import_boundary.py rename {scripts => src/sampletones_tools}/checks/language_keys.py (79%) rename {scripts => src/sampletones_tools}/checks/palette_colors.py (72%) create mode 100644 src/sampletones_tools/checks/registry.py rename {scripts => src/sampletones_tools}/checks/rendered_literals.py (67%) rename {scripts => src/sampletones_tools}/checks/shortcut_actions.py (78%) rename {tests/unit/sampletones_shared/meta/import_boundary => src/sampletones_tools/checks/source}/__init__.py (100%) rename src/{sampletones_shared/meta => sampletones_tools/checks}/source/annotations.py (97%) rename {tests/unit/sampletones_shared/meta/import_boundary/configs => src/sampletones_tools/checks/source/bindings}/__init__.py (100%) rename src/{sampletones_shared/meta => sampletones_tools/checks}/source/bindings/containers.py (96%) rename src/{sampletones_shared/meta => sampletones_tools/checks}/source/bindings/environment.py (100%) rename src/{sampletones_shared/meta => sampletones_tools/checks}/source/bindings/scopes.py (94%) rename src/{sampletones_shared/meta => sampletones_tools/checks}/source/bindings/statements.py (95%) rename src/{sampletones_shared/meta => sampletones_tools/checks}/source/classes.py (93%) rename src/{sampletones_shared/meta => sampletones_tools/checks}/source/constants.py (100%) rename src/{sampletones_shared/meta => sampletones_tools/checks}/source/index.py (85%) rename src/{sampletones_shared/meta => sampletones_tools/checks}/source/lookups.py (92%) rename src/{sampletones_shared/meta => sampletones_tools/checks}/source/modules.py (98%) rename src/{sampletones_shared/meta => sampletones_tools/checks}/source/nodes.py (100%) rename src/{sampletones_shared/meta => sampletones_tools/checks}/source/packages.py (100%) rename src/{sampletones_shared/meta => sampletones_tools/checks}/source/references.py (100%) rename src/{sampletones_shared/meta => sampletones_tools/checks}/source/subscripts.py (94%) rename src/{sampletones_shared/meta => sampletones_tools/checks}/source/values.py (96%) rename {scripts => src/sampletones_tools}/checks/tag_names.py (78%) rename {scripts => src/sampletones_tools}/checks/unused_tags.py (61%) rename tests/unit/{sampletones_shared/meta/source => sampletones_tools/checks/boundary}/__init__.py (100%) rename tests/unit/{sampletones_shared/meta/source/bindings => sampletones_tools/checks/boundary/configs}/__init__.py (100%) rename tests/unit/{sampletones_shared/meta/import_boundary => sampletones_tools/checks/boundary}/configs/test_declaration.py (90%) rename tests/unit/{sampletones_shared/meta/import_boundary => sampletones_tools/checks/boundary}/configs/test_general.py (91%) rename tests/unit/{sampletones_shared/meta/import_boundary => sampletones_tools/checks/boundary}/configs/test_rules.py (92%) rename tests/unit/{sampletones_shared/meta/import_boundary => sampletones_tools/checks/boundary}/test_check.py (94%) rename tests/unit/{sampletones_shared/meta/import_boundary => sampletones_tools/checks/boundary}/test_graph.py (97%) rename tests/unit/{sampletones_shared/meta/import_boundary => sampletones_tools/checks/boundary}/test_imports.py (95%) rename tests/unit/{sampletones_shared/meta/import_boundary => sampletones_tools/checks/boundary}/test_rule.py (97%) rename tests/unit/{sampletones_shared/meta/import_boundary => sampletones_tools/checks/boundary}/test_scope.py (97%) rename tests/unit/{sampletones_shared/meta/import_boundary => sampletones_tools/checks/boundary}/test_standalone.py (98%) rename tests/unit/{sampletones_shared/meta/import_boundary => sampletones_tools/checks/boundary}/test_token.py (96%) rename tests/unit/{sampletones_shared/meta/import_boundary => sampletones_tools/checks/boundary}/test_units.py (92%) rename tests/unit/{sampletones_shared/meta/import_boundary => sampletones_tools/checks/boundary}/test_violation.py (92%) create mode 100644 tests/unit/sampletones_tools/checks/source/__init__.py create mode 100644 tests/unit/sampletones_tools/checks/source/bindings/__init__.py rename tests/unit/{sampletones_shared/meta => sampletones_tools/checks}/source/bindings/test_containers.py (98%) rename tests/unit/{sampletones_shared/meta => sampletones_tools/checks}/source/bindings/test_environment.py (94%) rename tests/unit/{sampletones_shared/meta => sampletones_tools/checks}/source/bindings/test_scopes.py (97%) rename tests/unit/{sampletones_shared/meta => sampletones_tools/checks}/source/bindings/test_statements.py (98%) rename tests/unit/{sampletones_shared/meta => sampletones_tools/checks}/source/test_annotations.py (99%) rename tests/unit/{sampletones_shared/meta => sampletones_tools/checks}/source/test_classes.py (96%) rename tests/unit/{sampletones_shared/meta => sampletones_tools/checks}/source/test_constants.py (96%) rename tests/unit/{sampletones_shared/meta => sampletones_tools/checks}/source/test_index.py (92%) rename tests/unit/{sampletones_shared/meta => sampletones_tools/checks}/source/test_lookups.py (96%) rename tests/unit/{sampletones_shared/meta => sampletones_tools/checks}/source/test_modules.py (97%) rename tests/unit/{sampletones_shared/meta => sampletones_tools/checks}/source/test_nodes.py (98%) rename tests/unit/{sampletones_shared/meta => sampletones_tools/checks}/source/test_packages.py (86%) rename tests/unit/{sampletones_shared/meta => sampletones_tools/checks}/source/test_references.py (95%) rename tests/unit/{sampletones_shared/meta => sampletones_tools/checks}/source/test_subscripts.py (93%) rename tests/unit/{sampletones_shared/meta => sampletones_tools/checks}/source/test_values.py (97%) create mode 100644 tests/unit/sampletones_tools/checks/test_command.py rename tests/unit/{scripts => sampletones_tools}/checks/test_import_boundary.py (69%) rename tests/unit/{scripts => sampletones_tools}/checks/test_language_keys.py (94%) rename tests/unit/{scripts => sampletones_tools}/checks/test_palette_colors.py (95%) rename tests/unit/{scripts => sampletones_tools}/checks/test_rendered_literals.py (91%) rename tests/unit/{scripts => sampletones_tools}/checks/test_shortcut_actions.py (96%) rename tests/unit/{scripts => sampletones_tools}/checks/test_tag_names.py (93%) rename tests/unit/{scripts => sampletones_tools}/checks/test_unused_tags.py (91%) diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml index e666c1d66..4271cc6a1 100644 --- a/.github/workflows/workflow.yml +++ b/.github/workflows/workflow.yml @@ -93,6 +93,10 @@ jobs: shell: bash run: sampletones self-check + - name: Check a developer command refuses to run outside a checkout + shell: bash + run: sampletones check import-boundary --all 2>&1 | grep "runs from a checkout" + bundle: name: Standalone bundle (${{ matrix.platform }}) if: startsWith(github.ref, 'refs/tags/v') diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 29fcd0dbb..966d43b3f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -35,7 +35,7 @@ repos: hooks: - id: import-boundary name: import boundary - entry: uv run scripts/checks/import_boundary.py --all + entry: uv run sampletones check import-boundary --all language: system types: [python] pass_filenames: false @@ -43,7 +43,7 @@ repos: - id: unused-tags name: unused tags - entry: uv run scripts/checks/unused_tags.py + entry: uv run sampletones check unused-tags language: system types: [python] pass_filenames: false @@ -51,7 +51,7 @@ repos: - id: rendered-literals name: rendered literals - entry: uv run scripts/checks/rendered_literals.py + entry: uv run sampletones check rendered-literals language: system files: ^tests/.*\.py$ pass_filenames: false @@ -59,7 +59,7 @@ repos: - id: tag-names name: tag names - entry: uv run scripts/checks/tag_names.py --all + entry: uv run sampletones check tag-names --all language: system types: [python] pass_filenames: false @@ -67,7 +67,7 @@ repos: - id: palette-colors name: palette colors - entry: uv run scripts/checks/palette_colors.py + entry: uv run sampletones check palette-colors language: system files: (^src/sampletones_application/.*\.py|^src/sampletones_config/.*\.yaml)$ pass_filenames: false @@ -75,7 +75,7 @@ repos: - id: language-keys name: language keys - entry: uv run scripts/checks/language_keys.py + entry: uv run sampletones check language-keys language: system files: (\.py|^src/sampletones_config/lang/.*\.yaml)$ pass_filenames: false @@ -83,7 +83,7 @@ repos: - id: shortcut-actions name: shortcut actions - entry: uv run scripts/checks/shortcut_actions.py + entry: uv run sampletones check shortcut-actions language: system files: (\.py|^src/sampletones_config/keybindings/.*\.yaml)$ pass_filenames: false diff --git a/Makefile b/Makefile index b68d5fb74..fdda1aa24 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,5 @@ .PHONY: help setup install system-deps build release run clean pre-commit test benchmarks lint format \ - ftm-samples nsf-samples compression-report compression-study \ - check-import-boundary check-tag-names check-unused-tags check-rendered-literals check-language-keys \ - check-palette-colors check-shortcut-actions + ftm-samples nsf-samples compression-report compression-study ifeq ($(OS),Windows_NT) ifeq ($(MSYSTEM),) @@ -92,24 +90,3 @@ compression-report: compression-study: uv run scripts/compression_study.py $(ARGS) - -check-import-boundary: - uv run scripts/checks/import_boundary.py --all - -check-tag-names: - uv run scripts/checks/tag_names.py --all - -check-unused-tags: - uv run scripts/checks/unused_tags.py - -check-rendered-literals: - uv run scripts/checks/rendered_literals.py - -check-language-keys: - uv run scripts/checks/language_keys.py - -check-palette-colors: - uv run scripts/checks/palette_colors.py - -check-shortcut-actions: - uv run scripts/checks/shortcut_actions.py diff --git a/docs/development/architecture.md b/docs/development/architecture.md index 825fa5706..da283775c 100644 --- a/docs/development/architecture.md +++ b/docs/development/architecture.md @@ -41,7 +41,7 @@ These principles govern every structural decision in the codebase. Each layer imports only from the layers below it. Coordinators, at the top, reach every layer they orchestrate. The UI layer knows only view models and shared utilities. Logic owns domain state and produces view models. Services, at the bottom of the application stack, know only the core libraries and thread-safe utilities — a service is driven through a logic-side `Protocol` and reports through its result-contract types, so even logic reaches a service only through inversion. -The load-bearing prohibitions: nothing in `logic/` or `services/` imports `ui/` or `coordinators/`, and `services/` imports neither `logic/` nor `view_model/`. The authoritative import matrix is the **May import / Must not import** pair in each Layer Reference section below; the boundary script enforces it (see Enforcement). +The load-bearing prohibitions: nothing in `logic/` or `services/` imports `ui/` or `coordinators/`, and `services/` imports neither `logic/` nor `view_model/`. The authoritative import matrix is the **May import / Must not import** pair in each Layer Reference section below; the boundary check enforces it (see Enforcement). ### 2. DPG stays in the visual layers @@ -126,20 +126,20 @@ A menu item is a view of an action: `ShortcutManager.add_menu_item(shortcut_id, Two mechanisms keep the codebase aligned with this document. -**Import-expressible contracts are enforced by a check.** `sampletones_config/boundaries/rules.yaml` states one rule per layer, mirroring the **Must not import** lists in the Layer Reference; the Layer Reference is the source of truth, and a divergence between it and the configuration is itself a defect. The same domain holds the order the repository's packages import each other in, and the layering inside `sampletones_player`, both declared as layer tables in `docs/development/packages.md`. `sampletones_config/boundaries/` declares what the boundaries are, `sampletones_shared/meta/import_boundary/` holds how they are read and reported, and `scripts/checks/import_boundary.py` (a pre-commit hook, also run via `make check-import-boundary`) runs them over the source tree. A rule names the prefixes it reaches through the groups `boundaries/general.yaml` declares, so the interface several layers stay clear of is written once and each rule names it. Where a layer may consume another layer's data contract while its implementation stays out of reach (logic and the service result types), the rule names the contracts group that stays in reach. The hook audits the entire source tree on every commit (`--all`), so strengthening a rule surfaces violations in files a commit never touched. That property sets the working idiom for structural refactors: turn the stricter rule on first, and let the failing hook enumerate the remaining work. +**Import-expressible contracts are enforced by a check.** `sampletones_config/boundaries/rules.yaml` states one rule per layer, mirroring the **Must not import** lists in the Layer Reference; the Layer Reference is the source of truth, and a divergence between it and the configuration is itself a defect. The same domain holds the order the repository's packages import each other in, and the layering inside `sampletones_player`, both declared as layer tables in `docs/development/packages.md`. `sampletones_config/boundaries/` declares what the boundaries are, `sampletones_tools/checks/boundary/` holds how they are read and reported, and `sampletones check import-boundary` (a pre-commit hook, run as `uv run sampletones check import-boundary --all`) runs them over the source and scripts trees. A rule names the prefixes it reaches through the groups `boundaries/general.yaml` declares, so the interface several layers stay clear of is written once and each rule names it. Where a layer may consume another layer's data contract while its implementation stays out of reach (logic and the service result types), the rule names the contracts group that stays in reach. The hook audits the entire source tree on every commit (`--all`), so strengthening a rule surfaces violations in files a commit never touched. That property sets the working idiom for structural refactors: turn the stricter rule on first, and let the failing hook enumerate the remaining work. -**The identifier vocabularies, the declarations that complete them, and the shapes a case may not take are enforced the same way.** Further scripts under `scripts/checks/` run whole-tree as pre-commit hooks, each also available as a `make check-*` target: +**The identifier vocabularies, the declarations that complete them, and the shapes a case may not take are enforced the same way.** Further checks, each a `sampletones check ` command under `sampletones_tools/checks/`, run whole-tree as pre-commit hooks: -| Hook | Script | What it holds | -|------|--------|---------------| -| `language-keys` | `language_keys.py` | Code and `en.yaml` against each other, in both directions: a literal key names an entry, every entry is reached by some lookup, and a lookup states values the check can read (principle 8) | -| `tag-names` | `tag_names.py` | A tag constant's name against the tag it composes (principle 9) | -| `unused-tags` | `unused_tags.py` | Every `TAG_*`/`SUF_*`/`PRE_*` the `tags/` package declares against the reads of it across `src/`, `tests/`, and `scripts/`, where an import alone stands at no reads | -| `palette-colors` | `palette_colors.py` | A color as a token up to the moment it is drawn with: an attribute assigned a resolved `rgba`, a theme color filled outside the palette bindings, and a hex literal in the shipped configuration outside `palettes/` (principle 13) | -| `shortcut-actions` | `shortcut_actions.py` | Every action against the links it needs: a combination in every shipped scheme, a name the keybindings editor lists it by, and — for an application-scope action — the call it makes, whether its own binding or a family (principle 14) | -| `rendered-literals` | `rendered_literals.py` | A case against the text it renders: an equality holding `str(...)` or an f-string against a written-out string pins whatever the platform or the build decided (`guidelines.md` § Tests) | +| Hook | Command | What it holds | +|------|---------|---------------| +| `language-keys` | `check language-keys` | Code and `en.yaml` against each other, in both directions: a literal key names an entry, every entry is reached by some lookup, and a lookup states values the check can read (principle 8) | +| `tag-names` | `check tag-names --all` | A tag constant's name against the tag it composes (principle 9) | +| `unused-tags` | `check unused-tags` | Every `TAG_*`/`SUF_*`/`PRE_*` the `tags/` package declares against the reads of it across `src/`, `tests/`, and `scripts/`, where an import alone stands at no reads | +| `palette-colors` | `check palette-colors` | A color as a token up to the moment it is drawn with: an attribute assigned a resolved `rgba`, a theme color filled outside the palette bindings, and a hex literal in the shipped configuration outside `palettes/` (principle 13) | +| `shortcut-actions` | `check shortcut-actions` | Every action against the links it needs: a combination in every shipped scheme, a name the keybindings editor lists it by, and — for an application-scope action — the call it makes, whether its own binding or a family (principle 14) | +| `rendered-literals` | `check rendered-literals` | A case against the text it renders: an equality holding `str(...)` or an f-string against a written-out string pins whatever the platform or the build decided (`guidelines.md` § Tests) | -They read the source as an AST through the shared layer in `sampletones_shared/meta/source/`, which discovers modules, resolves the receiver a subscript sits on, and expands an enum-annotated key part to its members; the palette and shortcut checks read the shipped YAML beside it. That layer derives each package directory from its own location and reports a root it finds nothing at, so a check that sweeps nothing fails loudly where it would otherwise pass clean. Because the checks are global by nature — a dead entry and an unread fragment are both absences — the hooks pass whole-tree rather than filenames. +They read the source as an AST through the source layer in `sampletones_tools/checks/source/`, which discovers modules, resolves the receiver a subscript sits on, and expands an enum-annotated key part to its members; the palette and shortcut checks read the shipped YAML beside it. That layer derives each package directory from its own location and reports a root it finds nothing at, so a check that sweeps nothing fails loudly where it would otherwise pass clean. Because the checks are global by nature — a dead entry and an unread fragment are both absences — the hooks pass whole-tree rather than filenames. **Behavioral contracts are enforced by review.** Contracts a grep cannot see — where state lives, which methods touch DPG, how errors travel — are upheld in code review against this document. A change that alters one of them lands with the edit stating the new contract, and one that knowingly leaves a distance behind lands with an entry in `docs/development/bugs-and-todos.md § Architecture` — `guidelines.md` § Documents holds that rule. The ledger, not the codebase, is the memory of what is currently out of line. diff --git a/docs/development/config-organization.md b/docs/development/config-organization.md index 95d8e4897..0cfd0c7d7 100644 --- a/docs/development/config-organization.md +++ b/docs/development/config-organization.md @@ -32,9 +32,8 @@ empty `__init__.py`. Each schema lives with its reader: - `sampletones_application` owns the layout, theme, palettes, keybindings, language, behavior, and deployment schemas. -- `sampletones_tools` owns the calibration schemas. -- `sampletones_shared` owns the import-boundary schemas and the loader primitives - (`load_yaml_model`, `load_yaml_model_dir`). +- `sampletones_tools` owns the calibration and the import-boundary schemas. +- `sampletones_shared` owns the loader primitives (`load_yaml_model`, `load_yaml_model_dir`). So the data carries the values and the consumer carries the meaning, and the two evolve on their own terms. @@ -132,7 +131,7 @@ each value sits in the tree stays in the factory. |--------|-----------|--------------|------------------| | Application | `application/` | `DeploymentConfig` (`sampletones_application/config/deployment/`) | `DeploymentConfig.load()`, with `SAMPLETONES_*` env overrides | | Behavior | `behavior/` | `BehaviorConfig` (`sampletones_application/layout/behavior.py`) | folded into `LayoutConfig.behavior` by `load_layout_config` | -| Boundaries | `boundaries/` | `ImportBoundaryRules` (`sampletones_shared/meta/import_boundary/configs/`) | `ImportBoundaryRules.load()` | +| Boundaries | `boundaries/` | `ImportBoundaryRules` (`sampletones_tools/checks/boundary/configs/`) | `ImportBoundaryRules.load()` | | Calibration | `calibration/` | `CorpusConfig`, `RefereeConfig` (`sampletones_tools/calibration/config/`) | each model's own `.load()` | | Keybindings | `keybindings/` | `ShortcutScheme` (`sampletones_application/utils/gui/shortcuts/`) | `ShortcutCatalog.load()`, indexed by scheme name | | Language | `lang/` | `LanguageManager` (`sampletones_application/categories/`) | flat string map keyed `page.panel.text_type.element`, each key validated at load | @@ -171,7 +170,7 @@ that import `SchedulingBehavior` as a type. Boundaries is the domain a developer tool reads. It states the layer graphs the packages divide into, the imports each part of the application stays clear of, the spellings a tree keeps out, and the standard-library rule the bootstrap scripts under `scripts/` hold to, and -`scripts/checks/import_boundary.py` runs it over the source and scripts trees on every commit. A declaration draws on the named prefix groups `general.yaml` holds, so a set several +`sampletones check import-boundary` runs it over the source and scripts trees on every commit. A declaration draws on the named prefix groups `general.yaml` holds, so a set several rules reach for is written once and each rule names it, and a name reaching no group is refused as the domain is read. The bundle carries the domain because `--add-data` copies `sampletones_config` whole — the terms `calibration/` already ships on. diff --git a/docs/development/packages.md b/docs/development/packages.md index 832346f16..f662d83de 100644 --- a/docs/development/packages.md +++ b/docs/development/packages.md @@ -44,13 +44,13 @@ graph TD | Package | Purpose | May import | |---------|---------|------------| -| `sampletones_shared` | Facts and helpers any package holds: constants, exception families, paths, the logger, the array backend, the source layer the checks read the tree through, and the schema these boundaries are declared in | — | +| `sampletones_shared` | Facts and helpers any package holds: constants, exception families, paths, the logger, the array backend, and the command type the entry and the tools share | — | | `sampletones_config` | The shipped YAML — layout, palettes, themes, keybindings, language, calibration, and these boundaries themselves — reached as package data rather than by import | — | | `sampletones_assets` | The application icons and the bundled fonts, reached as package data | — | | `sampletones_core` | The reconstruction engine, the project model, playing a song out into instructions, and the tracker export formats | `sampletones_shared` | | `sampletones_player` | The NES player: the register model, the re-clocking schedule, the 6502 driver and the NSF file | `sampletones_shared`, `sampletones_core` | | `sampletones_application` | The DearPyGui front end | `sampletones_shared`, `sampletones_core`, `sampletones_player` | -| `sampletones_tools` | Everything a developer runs and the application does not: analytic waveform synthesis, the calibration harness, the driver toolchain and the register trace, the mark the icons are drawn from, and the developer commands that run them | `sampletones_shared`, `sampletones_assets`, `sampletones_core`, `sampletones_player`, `sampletones_application` | +| `sampletones_tools` | Everything a developer runs and the application does not: analytic waveform synthesis, the calibration harness, the driver toolchain and the register trace, the mark the icons are drawn from, the source checks, and the developer commands that run them | `sampletones_shared`, `sampletones_assets`, `sampletones_core`, `sampletones_player`, `sampletones_application` | | `sampletones` | The command-line entry: the dispatcher, the commands and the startup self-check | `sampletones_shared`, `sampletones_core`, `sampletones_application`, `sampletones_tools` | Third-party imports are the package author's own choice and stand outside this table. @@ -122,7 +122,7 @@ ships the binary alone. The toolchain the build needs is described in `sampletones_config/boundaries/graphs.yaml` declares both graphs as layer tables — each unit and the units it may import — and the rule the check runs derives from them: every unit a table leaves out is out of reach, so an edge is declared before it is taken. The hook audits the whole source tree on -every commit (`make check-import-boundary`), which means adding an edge to a table is how a new +every commit (`uv run sampletones check import-boundary --all`), which means adding an edge to a table is how a new dependency is opened, and removing one enumerates the work of closing it. Five token rules hold the shipped packages to the tools edge a second way: a module of @@ -135,11 +135,11 @@ undeclared is refused, and so is a graph whose units reach themselves, since a u level only where the units stand in an order. Three parts share the work. `sampletones_config/boundaries/` states what the boundaries are. -`sampletones_shared/meta/import_boundary/` validates that statement and holds the mechanism — +`sampletones_tools/checks/boundary/` validates that statement and holds the mechanism — reading a module line by line, resolving a unit to the modules it owns, deriving a rule from a graph -and reporting what crosses it — beside the source layer the other checks read the tree through. -`scripts/checks/import_boundary.py` runs them over the source and scripts trees and prints what -they find. +and reporting what crosses it — beside the source layer the other checks read the tree through, +`sampletones_tools/checks/source/`. `sampletones check import-boundary` runs them over the source +and scripts trees and prints what they find. The scripts tree is held to a rule of its own, `boundaries/standalone.yaml`. A bootstrap script runs on the system interpreter, so it imports the standard library and the scripts tree itself, diff --git a/docs/development/tooling.md b/docs/development/tooling.md index 5e96c276e..2e10c2f17 100644 --- a/docs/development/tooling.md +++ b/docs/development/tooling.md @@ -82,6 +82,7 @@ The developer commands, listed by `sampletones_tools/registry.py` and run as | Command | What it does | |---|---| | `calibration [--config FILE] [-o DIR] [--methods LIST] [--perceptual-exponents LIST] [--temporal-weights LIST] [--channels LIST]` | Reconstructs the calibration corpus under every variant of the sweep, scores it with every referee, and writes the reports; without `-o` the run lands in a timestamped directory under Documents/SampleToNES/calibration | +| `check [options]` | Holds the tree to one of its checks: `import-boundary`, `language-keys`, `palette-colors`, `rendered-literals`, `shortcut-actions`, `tag-names`, `unused-tags`; each is a pre-commit hook, and [architecture](architecture.md#enforcement) says what each holds. Needs a checkout | | `driver [--directory DIR]` | Assembles the NES player driver with cc65 and prints the layout the build produced; without `--directory` it writes the driver the package ships, which needs a checkout | | `icons [--directory DIR]` | Writes the icon suite from the mark; without `--directory` it writes the icons the package ships, which needs a checkout | | `nsf render --directory DIR [--tail SECONDS]` | Renders every exported `.nsf` file in the directory to a wave beside it, through ffmpeg's libgme demuxer | @@ -140,9 +141,8 @@ interpreter (`preflight.py`), and the platforms (`platforms/`). ## The tool scripts -`compression_study.py` and the checks under `checks/` import the project's packages and run -inside its environment, from the make target that names each. The checks are also pre-commit hooks; -[architecture](architecture.md#enforcement) lists them. +`compression_study.py` imports the project's packages and runs inside its environment, from the +make target that names it. ## Who governs what @@ -150,6 +150,7 @@ inside its environment, from the make target that names each. The checks are als |---|---| | Which commands the entry offers | `src/sampletones/commands/registry.py` | | Which developer commands exist | `src/sampletones_tools/registry.py` | +| Which checks the tree is held to | `src/sampletones_tools/checks/registry.py` | | Whether a command runs outside a checkout | `src/sampletones_tools/checkout.py` | | What a command is | `src/sampletones_shared/command.py` | | What a bootstrap script may import | `sampletones_config/boundaries/standalone.yaml` | diff --git a/scripts/checks/import_boundary.py b/scripts/checks/import_boundary.py deleted file mode 100755 index 7c903ba61..000000000 --- a/scripts/checks/import_boundary.py +++ /dev/null @@ -1,100 +0,0 @@ -#!/usr/bin/env python3 - -""" -Enforces the import boundaries the source tree is layered by. - -A layer graph names a tree of modules, the units it divides into, and the units each one may -import; every other unit is out of reach, so an edge across the graph is declared before it is -taken. The packages under `src/` are one such graph — `sampletones_core` sits below -`sampletones_player`, which is what keeps the reconstruction engine clear of the console player — -and the player's own subpackages are another, where `driver/assembler/` reaches the cc65 toolchain -and stays outside the wheel, so no shipped module imports it. - -Boundary rules state a contract the other way round, by the import prefixes a directory stays clear -of, and carry the contract modules exempt from those prefixes: a layer may consume another layer's -data contract (e.g. a service's result types) while its implementation modules stay out of reach. -`sampletones_application` is layered that way. - -Token rules additionally forbid a regex within a file glob, enforcing contracts a prefix cannot -express — e.g. that panels never compose a column suffix (`SUF_PANEL_*`) or parent into another -panel's container. - -A standalone rule holds the bootstrap scripts under `scripts/` to the standard library and the -tree they sit in, since they run on the system interpreter before the project environment exists, -and reports a name in that tree that stands in for a standard-library module. - -`sampletones_config/boundaries/` declares what the boundaries are and -`sampletones_shared/meta/import_boundary/` holds how they are read and reported; this script runs -them over the source and scripts trees and prints what they find. - -Usage: - python scripts/checks/import_boundary.py [files...] # check specific files - python scripts/checks/import_boundary.py --all # run all rules against both trees -""" - -import argparse -import sys -from pathlib import Path -from typing import List, Sequence - -from sampletones_shared.meta.import_boundary.check import check_boundaries -from sampletones_shared.meta.import_boundary.configs.rules import ImportBoundaryRules -from sampletones_shared.meta.import_boundary.standalone import check_standalone -from sampletones_shared.paths.source import SCRIPTS_ROOT, SOURCE_ROOT - - -def main(argv: Sequence[str]) -> int: - """Report every import and token the layer boundaries forbid.""" - parser = argparse.ArgumentParser( - description="Check the import boundaries the source tree is layered by.", - ) - parser.add_argument( - "files", - nargs="*", - type=Path, - help="modules to check", - ) - parser.add_argument( - "--all", - action="store_true", - help=f"check every module under {SOURCE_ROOT.name}/ and {SCRIPTS_ROOT.name}/ instead of named files", - ) - parser.add_argument( - "--source", - type=Path, - default=SOURCE_ROOT, - help="source root the rule roots are named within", - ) - parser.add_argument( - "--scripts", - type=Path, - default=SCRIPTS_ROOT, - help="scripts tree the standalone rules are written against", - ) - arguments = parser.parse_args(list(argv)) - - files: List[Path] = arguments.files - selection = None if arguments.all else {path.resolve() for path in files} - boundaries = ImportBoundaryRules.load() - violations = [ - *check_boundaries( - arguments.source, - boundaries.boundary_rules(), - boundaries.tokens, - selection, - ), - *check_standalone(arguments.scripts, boundaries.standalone, selection), - ] - if not violations: - return 0 - - print("Layer boundary violation(s) found:", file=sys.stderr) - for kind, location in violations: - print(f" [forbidden: {kind}] {location}", file=sys.stderr) - - print(f"\nFound {len(violations)} violation(s) in total.", file=sys.stderr) - return 1 - - -if __name__ == "__main__": - raise SystemExit(main(sys.argv[1:])) diff --git a/src/sampletones_config/README.md b/src/sampletones_config/README.md index 268bf4b59..3d4d030c9 100644 --- a/src/sampletones_config/README.md +++ b/src/sampletones_config/README.md @@ -8,8 +8,8 @@ programmatic role is to be importable so consumers can resolve its directory The schema that validates each file lives in the **consuming** package: - `sampletones_application` — layout, theme, palettes, language, behavior, deployment. -- `sampletones_tools` — calibration. -- `sampletones_shared` — the import boundaries and the loader primitives. +- `sampletones_tools` — calibration and the import boundaries. +- `sampletones_shared` — the loader primitives. The data package must not import a schema, and a schema package must not inline data. diff --git a/src/sampletones_config/boundaries/standalone.yaml b/src/sampletones_config/boundaries/standalone.yaml index 8d7103283..2f485b236 100644 --- a/src/sampletones_config/boundaries/standalone.yaml +++ b/src/sampletones_config/boundaries/standalone.yaml @@ -1,7 +1,6 @@ - pattern: "**/*.py" excluding: - "compression_study.py" - - "checks/**/*.py" - "codec_study/**/*.py" reserved: [build, test, tests] message: >- diff --git a/src/sampletones_tools/assets/command.py b/src/sampletones_tools/assets/command.py index d003d00e9..38505e62f 100644 --- a/src/sampletones_tools/assets/command.py +++ b/src/sampletones_tools/assets/command.py @@ -31,14 +31,15 @@ def run(arguments: Namespace) -> int: """ given = IconsArguments(directory=arguments.directory) - from sampletones_tools.assets.mark.specification import Mark - from sampletones_tools.assets.mark.suite import write_icon_suite - from sampletones_tools.assets.paths import ICONS_DIRECTORY from sampletones_tools.checkout import require_checkout if given.directory is None: require_checkout(NAME) + from sampletones_tools.assets.mark.specification import Mark + from sampletones_tools.assets.mark.suite import write_icon_suite + from sampletones_tools.assets.paths import ICONS_DIRECTORY + for path in write_icon_suite(given.directory if given.directory is not None else ICONS_DIRECTORY, Mark.load()): print(f"Wrote {path}") diff --git a/src/sampletones_shared/meta/import_boundary/__init__.py b/src/sampletones_tools/checks/__init__.py similarity index 100% rename from src/sampletones_shared/meta/import_boundary/__init__.py rename to src/sampletones_tools/checks/__init__.py diff --git a/src/sampletones_shared/meta/import_boundary/configs/__init__.py b/src/sampletones_tools/checks/boundary/__init__.py similarity index 100% rename from src/sampletones_shared/meta/import_boundary/configs/__init__.py rename to src/sampletones_tools/checks/boundary/__init__.py diff --git a/src/sampletones_shared/meta/import_boundary/check.py b/src/sampletones_tools/checks/boundary/check.py similarity index 83% rename from src/sampletones_shared/meta/import_boundary/check.py rename to src/sampletones_tools/checks/boundary/check.py index c5604faec..8781c7c70 100644 --- a/src/sampletones_shared/meta/import_boundary/check.py +++ b/src/sampletones_tools/checks/boundary/check.py @@ -1,11 +1,11 @@ from pathlib import Path from typing import List, Optional, Sequence, Set -from sampletones_shared.meta.import_boundary.rule import BoundaryRule -from sampletones_shared.meta.import_boundary.scope import rule_modules -from sampletones_shared.meta.import_boundary.token import TokenRule -from sampletones_shared.meta.import_boundary.violation import Violation -from sampletones_shared.meta.source.modules import source_paths +from sampletones_tools.checks.boundary.rule import BoundaryRule +from sampletones_tools.checks.boundary.scope import rule_modules +from sampletones_tools.checks.boundary.token import TokenRule +from sampletones_tools.checks.boundary.violation import Violation +from sampletones_tools.checks.source.modules import source_paths def check_boundaries( diff --git a/src/sampletones_shared/meta/source/__init__.py b/src/sampletones_tools/checks/boundary/configs/__init__.py similarity index 100% rename from src/sampletones_shared/meta/source/__init__.py rename to src/sampletones_tools/checks/boundary/configs/__init__.py diff --git a/src/sampletones_shared/meta/import_boundary/configs/declaration.py b/src/sampletones_tools/checks/boundary/configs/declaration.py similarity index 93% rename from src/sampletones_shared/meta/import_boundary/configs/declaration.py rename to src/sampletones_tools/checks/boundary/configs/declaration.py index 5957647a3..addfc9c5b 100644 --- a/src/sampletones_shared/meta/import_boundary/configs/declaration.py +++ b/src/sampletones_tools/checks/boundary/configs/declaration.py @@ -2,8 +2,8 @@ from pydantic import BaseModel, ConfigDict -from sampletones_shared.meta.import_boundary.configs.general import GeneralBoundaries -from sampletones_shared.meta.import_boundary.rule import BoundaryRule +from sampletones_tools.checks.boundary.configs.general import GeneralBoundaries +from sampletones_tools.checks.boundary.rule import BoundaryRule class BoundaryDeclaration(BaseModel): diff --git a/src/sampletones_shared/meta/import_boundary/configs/general.py b/src/sampletones_tools/checks/boundary/configs/general.py similarity index 100% rename from src/sampletones_shared/meta/import_boundary/configs/general.py rename to src/sampletones_tools/checks/boundary/configs/general.py diff --git a/src/sampletones_shared/meta/import_boundary/configs/paths.py b/src/sampletones_tools/checks/boundary/configs/paths.py similarity index 100% rename from src/sampletones_shared/meta/import_boundary/configs/paths.py rename to src/sampletones_tools/checks/boundary/configs/paths.py diff --git a/src/sampletones_shared/meta/import_boundary/configs/rules.py b/src/sampletones_tools/checks/boundary/configs/rules.py similarity index 84% rename from src/sampletones_shared/meta/import_boundary/configs/rules.py rename to src/sampletones_tools/checks/boundary/configs/rules.py index a4069986e..89784d669 100644 --- a/src/sampletones_shared/meta/import_boundary/configs/rules.py +++ b/src/sampletones_tools/checks/boundary/configs/rules.py @@ -2,14 +2,14 @@ from pydantic import BaseModel, ConfigDict, model_validator -from sampletones_shared.meta.import_boundary.configs.declaration import BoundaryDeclaration -from sampletones_shared.meta.import_boundary.configs.general import GeneralBoundaries -from sampletones_shared.meta.import_boundary.configs.paths import BOUNDARIES_DIRECTORY -from sampletones_shared.meta.import_boundary.graph import LayerGraph -from sampletones_shared.meta.import_boundary.rule import BoundaryRule -from sampletones_shared.meta.import_boundary.standalone import StandaloneRule -from sampletones_shared.meta.import_boundary.token import TokenRule from sampletones_shared.utils.serialization import load_yaml_model_dir +from sampletones_tools.checks.boundary.configs.declaration import BoundaryDeclaration +from sampletones_tools.checks.boundary.configs.general import GeneralBoundaries +from sampletones_tools.checks.boundary.configs.paths import BOUNDARIES_DIRECTORY +from sampletones_tools.checks.boundary.graph import LayerGraph +from sampletones_tools.checks.boundary.rule import BoundaryRule +from sampletones_tools.checks.boundary.standalone import StandaloneRule +from sampletones_tools.checks.boundary.token import TokenRule class ImportBoundaryRules(BaseModel): diff --git a/src/sampletones_shared/meta/import_boundary/graph.py b/src/sampletones_tools/checks/boundary/graph.py similarity index 96% rename from src/sampletones_shared/meta/import_boundary/graph.py rename to src/sampletones_tools/checks/boundary/graph.py index dede55c51..9842a9a54 100644 --- a/src/sampletones_shared/meta/import_boundary/graph.py +++ b/src/sampletones_tools/checks/boundary/graph.py @@ -2,8 +2,8 @@ from pydantic import BaseModel, ConfigDict, model_validator -from sampletones_shared.meta.import_boundary.rule import BoundaryRule -from sampletones_shared.meta.import_boundary.units import ( +from sampletones_tools.checks.boundary.rule import BoundaryRule +from sampletones_tools.checks.boundary.units import ( nested_globs, unit_glob, unit_prefix, diff --git a/src/sampletones_shared/meta/import_boundary/imports.py b/src/sampletones_tools/checks/boundary/imports.py similarity index 91% rename from src/sampletones_shared/meta/import_boundary/imports.py rename to src/sampletones_tools/checks/boundary/imports.py index 5a4f50508..83a41277e 100644 --- a/src/sampletones_shared/meta/import_boundary/imports.py +++ b/src/sampletones_tools/checks/boundary/imports.py @@ -1,7 +1,7 @@ import re from typing import Final, Optional -from sampletones_shared.meta.source.modules import MODULE_SEPARATOR +from sampletones_tools.checks.source.modules import MODULE_SEPARATOR IMPORT_PATTERN: Final[re.Pattern[str]] = re.compile(r"^\s*(import|from)\s+([\w.]+)") diff --git a/src/sampletones_shared/meta/import_boundary/lines.py b/src/sampletones_tools/checks/boundary/lines.py similarity index 91% rename from src/sampletones_shared/meta/import_boundary/lines.py rename to src/sampletones_tools/checks/boundary/lines.py index 3b6f2d28a..3bdd39d96 100644 --- a/src/sampletones_shared/meta/import_boundary/lines.py +++ b/src/sampletones_tools/checks/boundary/lines.py @@ -1,7 +1,7 @@ from pathlib import Path from typing import Iterator, Tuple -from sampletones_shared.meta.source.modules import SOURCE_ENCODING +from sampletones_tools.checks.source.modules import SOURCE_ENCODING def numbered_lines(path: Path) -> Iterator[Tuple[int, str]]: diff --git a/src/sampletones_shared/meta/import_boundary/rule.py b/src/sampletones_tools/checks/boundary/rule.py similarity index 91% rename from src/sampletones_shared/meta/import_boundary/rule.py rename to src/sampletones_tools/checks/boundary/rule.py index 8ab77c3d5..5a4e1339b 100644 --- a/src/sampletones_shared/meta/import_boundary/rule.py +++ b/src/sampletones_tools/checks/boundary/rule.py @@ -3,12 +3,12 @@ from pydantic import BaseModel, ConfigDict -from sampletones_shared.meta.import_boundary.imports import ( +from sampletones_tools.checks.boundary.imports import ( imported_module, matches_prefix, ) -from sampletones_shared.meta.import_boundary.lines import numbered_lines -from sampletones_shared.meta.import_boundary.violation import Violation +from sampletones_tools.checks.boundary.lines import numbered_lines +from sampletones_tools.checks.boundary.violation import Violation class BoundaryRule(BaseModel): diff --git a/src/sampletones_shared/meta/import_boundary/scope.py b/src/sampletones_tools/checks/boundary/scope.py similarity index 100% rename from src/sampletones_shared/meta/import_boundary/scope.py rename to src/sampletones_tools/checks/boundary/scope.py diff --git a/src/sampletones_shared/meta/import_boundary/standalone.py b/src/sampletones_tools/checks/boundary/standalone.py similarity index 92% rename from src/sampletones_shared/meta/import_boundary/standalone.py rename to src/sampletones_tools/checks/boundary/standalone.py index b21162dd4..9ca4e23e0 100644 --- a/src/sampletones_shared/meta/import_boundary/standalone.py +++ b/src/sampletones_tools/checks/boundary/standalone.py @@ -4,12 +4,12 @@ from pydantic import BaseModel, ConfigDict -from sampletones_shared.meta.import_boundary.imports import imported_module -from sampletones_shared.meta.import_boundary.lines import numbered_lines -from sampletones_shared.meta.import_boundary.scope import rule_modules -from sampletones_shared.meta.import_boundary.units import MODULE_SUFFIX -from sampletones_shared.meta.import_boundary.violation import Violation -from sampletones_shared.meta.source.modules import ( +from sampletones_tools.checks.boundary.imports import imported_module +from sampletones_tools.checks.boundary.lines import numbered_lines +from sampletones_tools.checks.boundary.scope import rule_modules +from sampletones_tools.checks.boundary.units import MODULE_SUFFIX +from sampletones_tools.checks.boundary.violation import Violation +from sampletones_tools.checks.source.modules import ( MODULE_SEPARATOR, PACKAGE_INITIALIZER, SOURCE_PATTERN, diff --git a/src/sampletones_shared/meta/import_boundary/token.py b/src/sampletones_tools/checks/boundary/token.py similarity index 92% rename from src/sampletones_shared/meta/import_boundary/token.py rename to src/sampletones_tools/checks/boundary/token.py index 51891e26d..e99660945 100644 --- a/src/sampletones_shared/meta/import_boundary/token.py +++ b/src/sampletones_tools/checks/boundary/token.py @@ -4,8 +4,8 @@ from pydantic import BaseModel, ConfigDict, field_validator -from sampletones_shared.meta.import_boundary.lines import numbered_lines -from sampletones_shared.meta.import_boundary.violation import Violation +from sampletones_tools.checks.boundary.lines import numbered_lines +from sampletones_tools.checks.boundary.violation import Violation class TokenRule(BaseModel): diff --git a/src/sampletones_shared/meta/import_boundary/units.py b/src/sampletones_tools/checks/boundary/units.py similarity index 94% rename from src/sampletones_shared/meta/import_boundary/units.py rename to src/sampletones_tools/checks/boundary/units.py index c87b06d0a..a594604b5 100644 --- a/src/sampletones_shared/meta/import_boundary/units.py +++ b/src/sampletones_tools/checks/boundary/units.py @@ -1,6 +1,6 @@ from typing import Final, Iterable, Tuple -from sampletones_shared.meta.source.modules import MODULE_SEPARATOR +from sampletones_tools.checks.source.modules import MODULE_SEPARATOR MODULE_SUFFIX: Final[str] = ".py" PATH_SEPARATOR: Final[str] = "/" diff --git a/src/sampletones_shared/meta/import_boundary/violation.py b/src/sampletones_tools/checks/boundary/violation.py similarity index 100% rename from src/sampletones_shared/meta/import_boundary/violation.py rename to src/sampletones_tools/checks/boundary/violation.py diff --git a/src/sampletones_tools/checks/command.py b/src/sampletones_tools/checks/command.py new file mode 100644 index 000000000..fe46657d2 --- /dev/null +++ b/src/sampletones_tools/checks/command.py @@ -0,0 +1,33 @@ +from argparse import ArgumentParser, Namespace +from typing import Final + +from sampletones_shared.command import Command +from sampletones_tools.checks.registry import GATES + +NAME: Final[str] = "check" +HELP: Final[str] = "hold the repository to one of its source checks" +GATE_FIELD: Final[str] = "gate" +GATE_METAVAR: Final[str] = "" + + +def configure(parser: ArgumentParser) -> None: + gates = parser.add_subparsers(dest=GATE_FIELD, metavar=GATE_METAVAR, required=True) + for gate in GATES: + gate.configure(gates.add_parser(gate.name, help=gate.help, description=gate.help)) + + +def run(arguments: Namespace) -> int: + """Runs the check named, from a checkout, and answers with its status. + + Raises: + SystemExit: If the package runs outside a checkout. + """ + gate = next(gate for gate in GATES if gate.name == arguments.gate) + + from sampletones_tools.checkout import require_checkout + + require_checkout(f"{NAME} {gate.name}") + return gate.run(arguments) + + +CHECK: Final[Command] = Command(name=NAME, help=HELP, configure=configure, run=run) diff --git a/src/sampletones_shared/meta/source/bindings/__init__.py b/src/sampletones_tools/checks/commands/__init__.py similarity index 100% rename from src/sampletones_shared/meta/source/bindings/__init__.py rename to src/sampletones_tools/checks/commands/__init__.py diff --git a/src/sampletones_tools/checks/commands/import_boundary.py b/src/sampletones_tools/checks/commands/import_boundary.py new file mode 100644 index 000000000..c1275a6c4 --- /dev/null +++ b/src/sampletones_tools/checks/commands/import_boundary.py @@ -0,0 +1,54 @@ +from argparse import ArgumentParser, Namespace +from dataclasses import dataclass +from pathlib import Path +from typing import Final, Optional, Tuple + +from sampletones_shared.command import Command + +NAME: Final[str] = "import-boundary" +HELP: Final[str] = "hold the source and scripts trees to the declared import boundaries" +FILES_HELP: Final[str] = "modules to check" +ALL_HELP: Final[str] = "check every module under the source and scripts trees instead of named files" +SOURCE_HELP: Final[str] = "source root the rule roots are named within; without it, the repository's src" +SCRIPTS_HELP: Final[str] = "scripts tree the standalone rules are written against; without it, the repository's scripts" + + +@dataclass(frozen=True) +class ImportBoundaryArguments: + """What a boundary check is given: the modules or the whole trees, and where the trees are.""" + + files: Tuple[Path, ...] + everything: bool + source: Optional[Path] + scripts: Optional[Path] + + +def configure(parser: ArgumentParser) -> None: + parser.add_argument("files", nargs="*", type=Path, help=FILES_HELP) + parser.add_argument("--all", action="store_true", help=ALL_HELP) + parser.add_argument("--source", type=Path, default=None, help=SOURCE_HELP) + parser.add_argument("--scripts", type=Path, default=None, help=SCRIPTS_HELP) + + +def run(arguments: Namespace) -> int: + """Reports every import and token the boundaries forbid.""" + given = ImportBoundaryArguments( + files=tuple(arguments.files), + everything=arguments.all, + source=arguments.source, + scripts=arguments.scripts, + ) + + from sampletones_shared.paths.source import SCRIPTS_ROOT, SOURCE_ROOT + from sampletones_tools.checks.import_boundary import check_imports, report + + selection = None if given.everything else {path.resolve() for path in given.files} + violations = check_imports( + given.source if given.source is not None else SOURCE_ROOT, + given.scripts if given.scripts is not None else SCRIPTS_ROOT, + selection, + ) + return report(violations) + + +IMPORT_BOUNDARY: Final[Command] = Command(name=NAME, help=HELP, configure=configure, run=run) diff --git a/src/sampletones_tools/checks/commands/language_keys.py b/src/sampletones_tools/checks/commands/language_keys.py new file mode 100644 index 000000000..4caff6ac1 --- /dev/null +++ b/src/sampletones_tools/checks/commands/language_keys.py @@ -0,0 +1,42 @@ +from argparse import ArgumentParser, Namespace +from dataclasses import dataclass +from pathlib import Path +from typing import Final, Optional + +from sampletones_shared.command import Command + +NAME: Final[str] = "language-keys" +HELP: Final[str] = "hold the code and the language file to each other, in both directions" +SOURCE_HELP: Final[str] = "root of the sources to read; without it, the repository's src" +LANGUAGE_FILE_HELP: Final[str] = "language file to check against; without it, the shipped en.yaml" + + +@dataclass(frozen=True) +class LanguageKeysArguments: + """What a language key check is given: the sources and the language file.""" + + source: Optional[Path] + language_file: Optional[Path] + + +def configure(parser: ArgumentParser) -> None: + parser.add_argument("--source", type=Path, default=None, help=SOURCE_HELP) + parser.add_argument("--language-file", type=Path, default=None, help=LANGUAGE_FILE_HELP) + + +def run(arguments: Namespace) -> int: + """Reports every disagreement between the language file and the lookups reading it.""" + given = LanguageKeysArguments(source=arguments.source, language_file=arguments.language_file) + + from sampletones_application.paths import LANG_EN + from sampletones_shared.paths.source import SOURCE_ROOT + from sampletones_tools.checks.language_keys import check_language_keys, report + + findings = check_language_keys( + given.source if given.source is not None else SOURCE_ROOT, + given.language_file if given.language_file is not None else LANG_EN, + ) + return report(findings) + + +LANGUAGE_KEYS: Final[Command] = Command(name=NAME, help=HELP, configure=configure, run=run) diff --git a/src/sampletones_tools/checks/commands/palette_colors.py b/src/sampletones_tools/checks/commands/palette_colors.py new file mode 100644 index 000000000..fb12c1b35 --- /dev/null +++ b/src/sampletones_tools/checks/commands/palette_colors.py @@ -0,0 +1,48 @@ +from argparse import ArgumentParser, Namespace +from dataclasses import dataclass +from pathlib import Path +from typing import Final, Optional + +from sampletones_shared.command import Command + +NAME: Final[str] = "palette-colors" +HELP: Final[str] = "hold every color to a palette token until it is drawn with" +PACKAGE_HELP: Final[str] = "package whose color reads to check; without it, the application package" +CONFIG_HELP: Final[str] = ( + "shipped configuration package whose colors must name palette tokens; without it, the shipped one" +) +PALETTES_HELP: Final[str] = "directory holding the palettes, where color values belong; without it, the shipped one" + + +@dataclass(frozen=True) +class PaletteColorsArguments: + """What a palette check is given: the package, the configuration and the palettes.""" + + package: Optional[Path] + config: Optional[Path] + palettes: Optional[Path] + + +def configure(parser: ArgumentParser) -> None: + parser.add_argument("--package", type=Path, default=None, help=PACKAGE_HELP) + parser.add_argument("--config", type=Path, default=None, help=CONFIG_HELP) + parser.add_argument("--palettes", type=Path, default=None, help=PALETTES_HELP) + + +def run(arguments: Namespace) -> int: + """Reports every color the application stores resolved or the configuration writes out.""" + given = PaletteColorsArguments(package=arguments.package, config=arguments.config, palettes=arguments.palettes) + + from sampletones_application.paths import PALETTES_DIRECTORY + from sampletones_shared.paths.resources import CONFIG_DIRECTORY + from sampletones_tools.checks.palette_colors import APPLICATION_PACKAGE, check_colors, report + + findings = check_colors( + given.package if given.package is not None else APPLICATION_PACKAGE, + given.config if given.config is not None else CONFIG_DIRECTORY, + given.palettes if given.palettes is not None else PALETTES_DIRECTORY, + ) + return report(findings) + + +PALETTE_COLORS: Final[Command] = Command(name=NAME, help=HELP, configure=configure, run=run) diff --git a/src/sampletones_tools/checks/commands/rendered_literals.py b/src/sampletones_tools/checks/commands/rendered_literals.py new file mode 100644 index 000000000..8f403d994 --- /dev/null +++ b/src/sampletones_tools/checks/commands/rendered_literals.py @@ -0,0 +1,33 @@ +from argparse import ArgumentParser, Namespace +from dataclasses import dataclass +from pathlib import Path +from typing import Final, Tuple + +from sampletones_shared.command import Command + +NAME: Final[str] = "rendered-literals" +HELP: Final[str] = "hold every case to comparing values, never the text it rendered" +TESTS_HELP: Final[str] = "directory of cases to read, repeatable; without it, the repository's tests" + + +@dataclass(frozen=True) +class RenderedLiteralsArguments: + """What a rendered literal check is given: the directories of cases.""" + + roots: Tuple[Path, ...] + + +def configure(parser: ArgumentParser) -> None: + parser.add_argument("--tests", type=Path, action="append", dest="roots", default=[], help=TESTS_HELP) + + +def run(arguments: Namespace) -> int: + """Reports every case comparing text it rendered with a literal it spelled out.""" + given = RenderedLiteralsArguments(roots=tuple(arguments.roots)) + + from sampletones_tools.checks.rendered_literals import TEST_ROOTS, check_cases, report + + return report(check_cases(given.roots or TEST_ROOTS)) + + +RENDERED_LITERALS: Final[Command] = Command(name=NAME, help=HELP, configure=configure, run=run) diff --git a/src/sampletones_tools/checks/commands/shortcut_actions.py b/src/sampletones_tools/checks/commands/shortcut_actions.py new file mode 100644 index 000000000..3c0b6bb7d --- /dev/null +++ b/src/sampletones_tools/checks/commands/shortcut_actions.py @@ -0,0 +1,23 @@ +from argparse import ArgumentParser, Namespace +from typing import Final + +from sampletones_shared.command import Command + +NAME: Final[str] = "shortcut-actions" +HELP: Final[str] = "hold every action to its keys, its name and the call it makes" + + +def configure(parser: ArgumentParser) -> None: + del parser + + +def run(arguments: Namespace) -> int: + """Reports every action left short of a combination, a name, or the call it makes.""" + del arguments + + from sampletones_tools.checks.shortcut_actions import check, report + + return report(check()) + + +SHORTCUT_ACTIONS: Final[Command] = Command(name=NAME, help=HELP, configure=configure, run=run) diff --git a/src/sampletones_tools/checks/commands/tag_names.py b/src/sampletones_tools/checks/commands/tag_names.py new file mode 100644 index 000000000..f333cff4d --- /dev/null +++ b/src/sampletones_tools/checks/commands/tag_names.py @@ -0,0 +1,36 @@ +from argparse import ArgumentParser, Namespace +from dataclasses import dataclass +from pathlib import Path +from typing import Final, Tuple + +from sampletones_shared.command import Command + +NAME: Final[str] = "tag-names" +HELP: Final[str] = "hold every tag constant's name to the tag it composes" +FILES_HELP: Final[str] = "modules to check" +ALL_HELP: Final[str] = "check every module of the tags package instead of named files" + + +@dataclass(frozen=True) +class TagNamesArguments: + """What a tag name check is given: the modules, or the whole tags package.""" + + files: Tuple[Path, ...] + everything: bool + + +def configure(parser: ArgumentParser) -> None: + parser.add_argument("files", nargs="*", type=Path, help=FILES_HELP) + parser.add_argument("--all", action="store_true", help=ALL_HELP) + + +def run(arguments: Namespace) -> int: + """Reports any tag constant whose name departs from the tag it composes.""" + given = TagNamesArguments(files=tuple(arguments.files), everything=arguments.all) + + from sampletones_tools.checks.tag_names import check_tags, report + + return report(check_tags(given.files, given.everything)) + + +TAG_NAMES: Final[Command] = Command(name=NAME, help=HELP, configure=configure, run=run) diff --git a/src/sampletones_tools/checks/commands/unused_tags.py b/src/sampletones_tools/checks/commands/unused_tags.py new file mode 100644 index 000000000..f45eccc93 --- /dev/null +++ b/src/sampletones_tools/checks/commands/unused_tags.py @@ -0,0 +1,47 @@ +from argparse import ArgumentParser, Namespace +from dataclasses import dataclass +from pathlib import Path +from typing import Final, Optional, Tuple + +from sampletones_shared.command import Command + +NAME: Final[str] = "unused-tags" +HELP: Final[str] = "hold every declared tag fragment to a read somewhere in the repository" +TAGS_HELP: Final[str] = "package declaring the tag fragments; without it, the application's tags package" +REFERENCE_ROOT_HELP: Final[str] = "directory to count reads in, repeatable; without it, src, tests and scripts" + + +@dataclass(frozen=True) +class UnusedTagsArguments: + """What an unused tag check is given: the tags package and the directories read for references.""" + + tags: Optional[Path] + reference_roots: Tuple[Path, ...] + + +def configure(parser: ArgumentParser) -> None: + parser.add_argument("--tags", type=Path, default=None, help=TAGS_HELP) + parser.add_argument( + "--reference-root", + type=Path, + action="append", + dest="reference_roots", + default=[], + help=REFERENCE_ROOT_HELP, + ) + + +def run(arguments: Namespace) -> int: + """Reports every tag fragment the repository declares and never reads.""" + given = UnusedTagsArguments(tags=arguments.tags, reference_roots=tuple(arguments.reference_roots)) + + from sampletones_tools.checks.unused_tags import REFERENCE_ROOTS, TAGS_PACKAGE, check_reads, report + + unread = check_reads( + given.tags if given.tags is not None else TAGS_PACKAGE, + given.reference_roots or REFERENCE_ROOTS, + ) + return report(unread) + + +UNUSED_TAGS: Final[Command] = Command(name=NAME, help=HELP, configure=configure, run=run) diff --git a/src/sampletones_tools/checks/import_boundary.py b/src/sampletones_tools/checks/import_boundary.py new file mode 100755 index 000000000..8501fe349 --- /dev/null +++ b/src/sampletones_tools/checks/import_boundary.py @@ -0,0 +1,39 @@ +import sys +from pathlib import Path +from typing import List, Optional, Sequence, Set + +from sampletones_tools.checks.boundary.check import check_boundaries +from sampletones_tools.checks.boundary.configs.rules import ImportBoundaryRules +from sampletones_tools.checks.boundary.standalone import check_standalone +from sampletones_tools.checks.boundary.violation import Violation + + +def check_imports(source: Path, scripts: Path, selection: Optional[Set[Path]]) -> List[Violation]: + """Every import and token the shipped boundaries forbid under the source and scripts trees. + + Args: + source: Source root the rule roots are named within. + scripts: Scripts tree the standalone rules are written against. + selection: Resolved paths to narrow the check to, or `None` to check both trees whole. + + Returns: + List[Violation]: What the rules report, the source tree's before the scripts tree's. + """ + boundaries = ImportBoundaryRules.load() + return [ + *check_boundaries(source, boundaries.boundary_rules(), boundaries.tokens, selection), + *check_standalone(scripts, boundaries.standalone, selection), + ] + + +def report(violations: Sequence[Violation]) -> int: + """Prints every violation on the standard error stream and answers with the exit status.""" + if not violations: + return 0 + + print("Layer boundary violation(s) found:", file=sys.stderr) + for kind, location in violations: + print(f" [forbidden: {kind}] {location}", file=sys.stderr) + + print(f"\nFound {len(violations)} violation(s) in total.", file=sys.stderr) + return 1 diff --git a/scripts/checks/language_keys.py b/src/sampletones_tools/checks/language_keys.py similarity index 79% rename from scripts/checks/language_keys.py rename to src/sampletones_tools/checks/language_keys.py index d6a55601c..c939e3abe 100755 --- a/scripts/checks/language_keys.py +++ b/src/sampletones_tools/checks/language_keys.py @@ -1,24 +1,3 @@ -#!/usr/bin/env python3 - -""" -Checks the language file and the lookups that read it against each other, in both directions. - -Every lookup on a `LanguageManager` states a key, so the check reads each one and holds it against -`en.yaml`. A key spelled entirely from literals must name an entry, and every entry must be reachable -from some lookup — a key part arriving in a variable stands for each member of the enum it is -annotated with, which is why a dynamic part names a concrete element enum. An element enum is found -by what it derives from, so one lives wherever its domain lives and the check reads it there. - -Three things are reported: - broken lookup — a literal key the language file holds no entry for - unresolved part — a key part the check reads no values from - unreached entry — an entry no lookup asks for - -Usage: - python scripts/checks/language_keys.py -""" - -import argparse import importlib import inspect import sys @@ -31,15 +10,14 @@ from sampletones_application.categories import hierarchy from sampletones_application.categories.abstract import AbstractElement -from sampletones_application.paths import LANG_EN from sampletones_application.tags.compose import TAG_SEPARATOR -from sampletones_shared.meta.source.classes import declared_subclasses -from sampletones_shared.meta.source.index import source_index -from sampletones_shared.meta.source.lookups import LookupSite, tree_lookups -from sampletones_shared.meta.source.modules import discover_modules, module_name -from sampletones_shared.meta.source.packages import package_directory -from sampletones_shared.meta.source.values import EnumMembers, EnumTable from sampletones_shared.paths.source import SOURCE_ROOT +from sampletones_tools.checks.source.classes import declared_subclasses +from sampletones_tools.checks.source.index import source_index +from sampletones_tools.checks.source.lookups import LookupSite, tree_lookups +from sampletones_tools.checks.source.modules import discover_modules, module_name +from sampletones_tools.checks.source.packages import package_directory +from sampletones_tools.checks.source.values import EnumMembers, EnumTable EnumPredicate = Callable[[object], bool] @@ -275,26 +253,8 @@ def check_language_keys(source: Path, language_file: Path) -> List[Finding]: ] -def main(argv: Sequence[str]) -> int: - """Report every disagreement between the language file and the lookups reading it.""" - parser = argparse.ArgumentParser( - description="Check language keys against the language file, in both directions.", - ) - parser.add_argument( - "--source", - type=Path, - default=SOURCE_ROOT, - help="root of the sources to read", - ) - parser.add_argument( - "--language-file", - type=Path, - default=LANG_EN, - help="language file to check against", - ) - arguments = parser.parse_args(list(argv)) - - findings = check_language_keys(arguments.source, arguments.language_file) +def report(findings: Sequence[Finding]) -> int: + """Prints every finding on the standard error stream and answers with the exit status.""" if not findings: return 0 @@ -304,7 +264,3 @@ def main(argv: Sequence[str]) -> int: print(f"\nFound {len(findings)} language key finding(s).", file=sys.stderr) return 1 - - -if __name__ == "__main__": - raise SystemExit(main(sys.argv[1:])) diff --git a/scripts/checks/palette_colors.py b/src/sampletones_tools/checks/palette_colors.py similarity index 72% rename from scripts/checks/palette_colors.py rename to src/sampletones_tools/checks/palette_colors.py index c3fa9e4cb..276b4d0b3 100755 --- a/scripts/checks/palette_colors.py +++ b/src/sampletones_tools/checks/palette_colors.py @@ -1,19 +1,3 @@ -#!/usr/bin/env python3 - -""" -Checks that a color stays a palette token until the moment it is drawn with. - -`BaseColor.rgba` answers with the palette active right now, so a consumer that holds the -token follows a palette swap and one that stores the answer keeps the shade it read at -construction. The check reports the three ways that contract is lost: an attribute assigned the -resolved value, a theme color filled outside the palette bindings that record it, and a color -written into the shipped configuration as a literal instead of a palette token. - -Usage: - python scripts/checks/palette_colors.py # check the source tree and the config package -""" - -import argparse import ast import logging import re @@ -23,12 +7,10 @@ from pathlib import Path from typing import Final, List, NamedTuple, Tuple, Union -from sampletones_application.paths import PALETTES_DIRECTORY from sampletones_shared.logger import logger -from sampletones_shared.meta.source.modules import SourceModule, discover_modules -from sampletones_shared.meta.source.nodes import terminal_name -from sampletones_shared.meta.source.packages import package_directory -from sampletones_shared.paths.resources import CONFIG_DIRECTORY +from sampletones_tools.checks.source.modules import SourceModule, discover_modules +from sampletones_tools.checks.source.nodes import terminal_name +from sampletones_tools.checks.source.packages import package_directory APPLICATION_PACKAGE: Final[Path] = package_directory("sampletones_application") @@ -195,63 +177,37 @@ def find_literal_colors(package: Path, palettes: Path) -> List[ColorFinding]: return [finding for path in paths if palettes not in path.parents for finding in literal_colors(path)] -def main(argv: Sequence[str]) -> int: - """Report every color the application stores resolved or the configuration writes out.""" +def check_colors(package: Path, config: Path, palettes: Path) -> List[ColorFinding]: + """Every color the application stores resolved or the configuration writes out. + + Args: + package: Package whose color reads are checked. + config: Shipped configuration package whose colors must name palette tokens. + palettes: Directory holding the palettes, where color values belong. + Returns: + List[ColorFinding]: The detached colors of the package, then the literal colors of the + configuration. + """ logger.set_level(level=logging.ERROR) bindings_module, theme_color_helper = dpg_module_helper() - - parser = argparse.ArgumentParser( - description="Check that a color stays a palette token until it is drawn with.", - ) - parser.add_argument( - "--package", - type=Path, - default=APPLICATION_PACKAGE, - help="package whose color reads to check", - ) - parser.add_argument( - "--config", - type=Path, - default=CONFIG_DIRECTORY, - help="shipped configuration package whose colors must name palette tokens", - ) - parser.add_argument( - "--palettes", - type=Path, - default=PALETTES_DIRECTORY, - help="directory holding the palettes, where color values belong", - ) - arguments = parser.parse_args(list(argv)) - findings = find_detached_colors( - arguments.package, + package, bindings_module=bindings_module, theme_color_helper=theme_color_helper, ) - findings.extend( - find_literal_colors( - arguments.config, - arguments.palettes, - ) - ) + findings.extend(find_literal_colors(config, palettes)) + return findings + +def report(findings: Sequence[ColorFinding]) -> int: + """Prints every finding on the standard error stream and answers with the exit status.""" if not findings: return 0 - print( - "Color(s) that stop following the active palette:", - file=sys.stderr, - ) + print("Color(s) that stop following the active palette:", file=sys.stderr) for location, message in findings: print(f" {location}: {message}", file=sys.stderr) - print( - f"\nFound {len(findings)} color(s) detached from the palette.", - file=sys.stderr, - ) + print(f"\nFound {len(findings)} color(s) detached from the palette.", file=sys.stderr) return 1 - - -if __name__ == "__main__": - raise SystemExit(main(sys.argv[1:])) diff --git a/src/sampletones_tools/checks/registry.py b/src/sampletones_tools/checks/registry.py new file mode 100644 index 000000000..b1cc2af33 --- /dev/null +++ b/src/sampletones_tools/checks/registry.py @@ -0,0 +1,20 @@ +from typing import Final, Tuple + +from sampletones_shared.command import Command +from sampletones_tools.checks.commands.import_boundary import IMPORT_BOUNDARY +from sampletones_tools.checks.commands.language_keys import LANGUAGE_KEYS +from sampletones_tools.checks.commands.palette_colors import PALETTE_COLORS +from sampletones_tools.checks.commands.rendered_literals import RENDERED_LITERALS +from sampletones_tools.checks.commands.shortcut_actions import SHORTCUT_ACTIONS +from sampletones_tools.checks.commands.tag_names import TAG_NAMES +from sampletones_tools.checks.commands.unused_tags import UNUSED_TAGS + +GATES: Final[Tuple[Command, ...]] = ( + IMPORT_BOUNDARY, + LANGUAGE_KEYS, + PALETTE_COLORS, + RENDERED_LITERALS, + SHORTCUT_ACTIONS, + TAG_NAMES, + UNUSED_TAGS, +) diff --git a/scripts/checks/rendered_literals.py b/src/sampletones_tools/checks/rendered_literals.py similarity index 67% rename from scripts/checks/rendered_literals.py rename to src/sampletones_tools/checks/rendered_literals.py index 954acc18b..f7150ea6c 100755 --- a/scripts/checks/rendered_literals.py +++ b/src/sampletones_tools/checks/rendered_literals.py @@ -1,31 +1,10 @@ -#!/usr/bin/env python3 - -""" -Checks that no case holds text it rendered itself against a literal it spelled out. - -Rendering a value and comparing the result with a written-out string pins whatever decided that -rendering. `str(path)` reads one way on Windows and another elsewhere, and a formatted setting reads -whatever the build ships, so a case written that way passes where it was authored and fails where it -is run next. Comparing the values instead — a `Path` against a `Path`, a setting against the -configuration it comes from — holds on every platform and survives every tuning. - -Equality is what pins a rendering entire, so that is what the check reads; a case holding a -fragment picks one clear of anything a platform decides. Every hit therefore reads under one of two -rules in `guidelines.md`: a case assumes no one platform, or a shipped value is a choice rather than -a contract. - -Usage: - python scripts/checks/rendered_literals.py -""" - -import argparse import ast import sys from pathlib import Path from typing import Final, Iterator, List, NamedTuple, Sequence, Tuple, Type -from sampletones_shared.meta.source.modules import SourceModule, discover_modules from sampletones_shared.paths.source import REPOSITORY_ROOT +from sampletones_tools.checks.source.modules import SourceModule, discover_modules TEST_ROOTS: Final[Tuple[Path, ...]] = (REPOSITORY_ROOT / "tests",) @@ -107,22 +86,13 @@ def findings(modules: Sequence[SourceModule]) -> List[Finding]: return [finding for module in modules for finding in module_findings(module)] -def main(argv: Sequence[str]) -> int: - """Report every case comparing text it rendered with a literal it spelled out.""" - parser = argparse.ArgumentParser( - description="Check that no case holds rendered text against a spelled-out literal.", - ) - parser.add_argument( - "--tests", - type=Path, - action="append", - dest="roots", - help="directory of cases to read, repeatable", - ) - arguments = parser.parse_args(list(argv)) +def check_cases(roots: Sequence[Path]) -> List[Finding]: + """Every case under the roots comparing text it rendered with a literal it spelled out.""" + return findings(discover_modules(roots)) + - roots: Tuple[Path, ...] = tuple(arguments.roots or TEST_ROOTS) - found = findings(discover_modules(roots)) +def report(found: Sequence[Finding]) -> int: + """Prints every finding on the standard error stream and answers with the exit status.""" if not found: return 0 @@ -136,7 +106,3 @@ def main(argv: Sequence[str]) -> int: file=sys.stderr, ) return 1 - - -if __name__ == "__main__": - raise SystemExit(main(sys.argv[1:])) diff --git a/scripts/checks/shortcut_actions.py b/src/sampletones_tools/checks/shortcut_actions.py similarity index 78% rename from scripts/checks/shortcut_actions.py rename to src/sampletones_tools/checks/shortcut_actions.py index b8ff915ed..ba380ffcd 100755 --- a/scripts/checks/shortcut_actions.py +++ b/src/sampletones_tools/checks/shortcut_actions.py @@ -1,29 +1,3 @@ -#!/usr/bin/env python3 - -""" -Checks that every action a key or a menu item reaches is declared the whole way through. - -An action is one `ShortcutId`, and naming it is the first of the links it needs: every shipped -keybinding scheme states the combination that fires it, `KeybindingActionElements` names it so the -keybindings editor can list it, and an application-scope action states the call it makes — either -its own entry in the shell's binding map, or membership of a family that maps a whole enum onto -one call. - -The scheme a build loads validates itself as it loads, so this check covers what that validation -cannot reach: the schemes a platform other than this one ships, the editor's vocabulary, and the -call behind a menu item, which otherwise goes missing until the application is constructed. - -Four things are reported: - unanswered action — an action a shipped scheme states no combination for - unnamed action — a rebindable action `KeybindingActionElements` has no member for - uncalled action — an application-scope action no binding and no family answers - stale entry — a name a scheme or the editor states that no action carries - -Usage: - python scripts/checks/shortcut_actions.py -""" - -import argparse import ast import sys from pathlib import Path @@ -53,8 +27,8 @@ ShortcutCategory, ShortcutId, ) -from sampletones_shared.meta.source.modules import SourceModule, parse_module -from sampletones_shared.meta.source.packages import package_directory +from sampletones_tools.checks.source.modules import SourceModule, parse_module +from sampletones_tools.checks.source.packages import package_directory SHORTCUTS_MODULE: Final[Path] = Path(ids_module.__file__) ELEMENTS_MODULE: Final[Path] = Path(settings_module.__file__) @@ -234,14 +208,8 @@ def check() -> List[Finding]: ] -def main(argv: Sequence[str]) -> int: - """Report every action left short of a combination, a name, or the call it makes.""" - parser = argparse.ArgumentParser( - description="Check that every action is declared the whole way through.", - ) - parser.parse_args(list(argv)) - - findings = check() +def report(findings: Sequence[Finding]) -> int: + """Prints every finding on the standard error stream and answers with the exit status.""" if not findings: return 0 @@ -249,12 +217,5 @@ def main(argv: Sequence[str]) -> int: for kind, location, message in findings: print(f" {kind} | {location}: {message}", file=sys.stderr) - print( - f"\nFound {len(findings)} incomplete action declaration(s).", - file=sys.stderr, - ) + print(f"\nFound {len(findings)} incomplete action declaration(s).", file=sys.stderr) return 1 - - -if __name__ == "__main__": - raise SystemExit(main(sys.argv[1:])) diff --git a/tests/unit/sampletones_shared/meta/import_boundary/__init__.py b/src/sampletones_tools/checks/source/__init__.py similarity index 100% rename from tests/unit/sampletones_shared/meta/import_boundary/__init__.py rename to src/sampletones_tools/checks/source/__init__.py diff --git a/src/sampletones_shared/meta/source/annotations.py b/src/sampletones_tools/checks/source/annotations.py similarity index 97% rename from src/sampletones_shared/meta/source/annotations.py rename to src/sampletones_tools/checks/source/annotations.py index 5994feb7e..e71278bfe 100644 --- a/src/sampletones_shared/meta/source/annotations.py +++ b/src/sampletones_tools/checks/source/annotations.py @@ -1,7 +1,7 @@ import ast from typing import Final, FrozenSet, Optional, Tuple -from sampletones_shared.meta.source.nodes import slice_elements, terminal_name +from sampletones_tools.checks.source.nodes import slice_elements, terminal_name ANNOTATION_WRAPPERS: Final[FrozenSet[str]] = frozenset({"Annotated", "ClassVar", "Final", "Optional"}) diff --git a/tests/unit/sampletones_shared/meta/import_boundary/configs/__init__.py b/src/sampletones_tools/checks/source/bindings/__init__.py similarity index 100% rename from tests/unit/sampletones_shared/meta/import_boundary/configs/__init__.py rename to src/sampletones_tools/checks/source/bindings/__init__.py diff --git a/src/sampletones_shared/meta/source/bindings/containers.py b/src/sampletones_tools/checks/source/bindings/containers.py similarity index 96% rename from src/sampletones_shared/meta/source/bindings/containers.py rename to src/sampletones_tools/checks/source/bindings/containers.py index 0e9d81e5e..75c3d1bf8 100644 --- a/src/sampletones_shared/meta/source/bindings/containers.py +++ b/src/sampletones_tools/checks/source/bindings/containers.py @@ -1,8 +1,8 @@ import ast from typing import Dict, Final, FrozenSet, Mapping, NamedTuple, Optional, Tuple -from sampletones_shared.meta.source.annotations import annotation_item_types -from sampletones_shared.meta.source.nodes import expression_spelling +from sampletones_tools.checks.source.annotations import annotation_item_types +from sampletones_tools.checks.source.nodes import expression_spelling KEY_ACCESSOR: Final[str] = "keys" VALUE_ACCESSOR: Final[str] = "values" diff --git a/src/sampletones_shared/meta/source/bindings/environment.py b/src/sampletones_tools/checks/source/bindings/environment.py similarity index 100% rename from src/sampletones_shared/meta/source/bindings/environment.py rename to src/sampletones_tools/checks/source/bindings/environment.py diff --git a/src/sampletones_shared/meta/source/bindings/scopes.py b/src/sampletones_tools/checks/source/bindings/scopes.py similarity index 94% rename from src/sampletones_shared/meta/source/bindings/scopes.py rename to src/sampletones_tools/checks/source/bindings/scopes.py index 38362f18c..95d3af019 100644 --- a/src/sampletones_shared/meta/source/bindings/scopes.py +++ b/src/sampletones_tools/checks/source/bindings/scopes.py @@ -2,21 +2,21 @@ from dataclasses import dataclass from typing import Dict, List, NamedTuple, Optional -from sampletones_shared.meta.source.bindings.containers import ( +from sampletones_tools.checks.source.bindings.containers import ( ItemTypes, container_item_types, iterated_container, iterated_types, ) -from sampletones_shared.meta.source.bindings.environment import TypeEnvironment -from sampletones_shared.meta.source.bindings.statements import ( +from sampletones_tools.checks.source.bindings.environment import TypeEnvironment +from sampletones_tools.checks.source.bindings.statements import ( AliasStatement, LoopStatement, Statement, TypeStatement, read_statement, ) -from sampletones_shared.meta.source.nodes import expression_spelling, is_attribute_spelling, nested_scopes, own_nodes +from sampletones_tools.checks.source.nodes import expression_spelling, is_attribute_spelling, nested_scopes, own_nodes @dataclass(frozen=True) diff --git a/src/sampletones_shared/meta/source/bindings/statements.py b/src/sampletones_tools/checks/source/bindings/statements.py similarity index 95% rename from src/sampletones_shared/meta/source/bindings/statements.py rename to src/sampletones_tools/checks/source/bindings/statements.py index cadd9a9a2..7d4d55182 100644 --- a/src/sampletones_shared/meta/source/bindings/statements.py +++ b/src/sampletones_tools/checks/source/bindings/statements.py @@ -1,8 +1,8 @@ import ast from typing import NamedTuple, Optional, Union -from sampletones_shared.meta.source.annotations import annotation_type_name -from sampletones_shared.meta.source.nodes import expression_spelling, terminal_name +from sampletones_tools.checks.source.annotations import annotation_type_name +from sampletones_tools.checks.source.nodes import expression_spelling, terminal_name class TypeStatement(NamedTuple): diff --git a/src/sampletones_shared/meta/source/classes.py b/src/sampletones_tools/checks/source/classes.py similarity index 93% rename from src/sampletones_shared/meta/source/classes.py rename to src/sampletones_tools/checks/source/classes.py index 02bac0a34..f4f1483b8 100644 --- a/src/sampletones_shared/meta/source/classes.py +++ b/src/sampletones_tools/checks/source/classes.py @@ -1,7 +1,7 @@ import ast from typing import List -from sampletones_shared.meta.source.nodes import terminal_name +from sampletones_tools.checks.source.nodes import terminal_name def declared_subclasses(tree: ast.Module, base: str) -> List[str]: diff --git a/src/sampletones_shared/meta/source/constants.py b/src/sampletones_tools/checks/source/constants.py similarity index 100% rename from src/sampletones_shared/meta/source/constants.py rename to src/sampletones_tools/checks/source/constants.py diff --git a/src/sampletones_shared/meta/source/index.py b/src/sampletones_tools/checks/source/index.py similarity index 85% rename from src/sampletones_shared/meta/source/index.py rename to src/sampletones_tools/checks/source/index.py index b61822453..8c670ac28 100644 --- a/src/sampletones_shared/meta/source/index.py +++ b/src/sampletones_tools/checks/source/index.py @@ -2,9 +2,9 @@ from dataclasses import dataclass from typing import Dict, Iterable, Mapping, Tuple -from sampletones_shared.meta.source.bindings.containers import container_item_types -from sampletones_shared.meta.source.constants import module_constants -from sampletones_shared.meta.source.modules import SourceModule +from sampletones_tools.checks.source.bindings.containers import container_item_types +from sampletones_tools.checks.source.constants import module_constants +from sampletones_tools.checks.source.modules import SourceModule @dataclass(frozen=True) diff --git a/src/sampletones_shared/meta/source/lookups.py b/src/sampletones_tools/checks/source/lookups.py similarity index 92% rename from src/sampletones_shared/meta/source/lookups.py rename to src/sampletones_tools/checks/source/lookups.py index 6266cd6d9..6bdfb5de0 100644 --- a/src/sampletones_shared/meta/source/lookups.py +++ b/src/sampletones_tools/checks/source/lookups.py @@ -3,11 +3,11 @@ from itertools import product from typing import Iterable, List, Sequence, Tuple -from sampletones_shared.meta.source.bindings.scopes import module_scopes -from sampletones_shared.meta.source.index import SourceIndex -from sampletones_shared.meta.source.modules import SourceModule -from sampletones_shared.meta.source.subscripts import SubscriptSite, find_subscripts -from sampletones_shared.meta.source.values import EnumTable, ResolvedValues, ValueResolver +from sampletones_tools.checks.source.bindings.scopes import module_scopes +from sampletones_tools.checks.source.index import SourceIndex +from sampletones_tools.checks.source.modules import SourceModule +from sampletones_tools.checks.source.subscripts import SubscriptSite, find_subscripts +from sampletones_tools.checks.source.values import EnumTable, ResolvedValues, ValueResolver @dataclass(frozen=True) diff --git a/src/sampletones_shared/meta/source/modules.py b/src/sampletones_tools/checks/source/modules.py similarity index 98% rename from src/sampletones_shared/meta/source/modules.py rename to src/sampletones_tools/checks/source/modules.py index d2b5efd7e..bdb2a23ac 100644 --- a/src/sampletones_shared/meta/source/modules.py +++ b/src/sampletones_tools/checks/source/modules.py @@ -3,7 +3,7 @@ from pathlib import Path from typing import Final, Iterable, List -from sampletones_shared.meta.source.nodes import PositionedNode +from sampletones_tools.checks.source.nodes import PositionedNode SOURCE_PATTERN: Final[str] = "*.py" SOURCE_ENCODING: Final[str] = "utf-8-sig" diff --git a/src/sampletones_shared/meta/source/nodes.py b/src/sampletones_tools/checks/source/nodes.py similarity index 100% rename from src/sampletones_shared/meta/source/nodes.py rename to src/sampletones_tools/checks/source/nodes.py diff --git a/src/sampletones_shared/meta/source/packages.py b/src/sampletones_tools/checks/source/packages.py similarity index 100% rename from src/sampletones_shared/meta/source/packages.py rename to src/sampletones_tools/checks/source/packages.py diff --git a/src/sampletones_shared/meta/source/references.py b/src/sampletones_tools/checks/source/references.py similarity index 100% rename from src/sampletones_shared/meta/source/references.py rename to src/sampletones_tools/checks/source/references.py diff --git a/src/sampletones_shared/meta/source/subscripts.py b/src/sampletones_tools/checks/source/subscripts.py similarity index 94% rename from src/sampletones_shared/meta/source/subscripts.py rename to src/sampletones_tools/checks/source/subscripts.py index d942ea362..4121b15e5 100644 --- a/src/sampletones_shared/meta/source/subscripts.py +++ b/src/sampletones_tools/checks/source/subscripts.py @@ -2,7 +2,7 @@ from dataclasses import dataclass from typing import Collection, List, Tuple -from sampletones_shared.meta.source.nodes import expression_spelling, own_nodes, slice_elements +from sampletones_tools.checks.source.nodes import expression_spelling, own_nodes, slice_elements @dataclass(frozen=True) diff --git a/src/sampletones_shared/meta/source/values.py b/src/sampletones_tools/checks/source/values.py similarity index 96% rename from src/sampletones_shared/meta/source/values.py rename to src/sampletones_tools/checks/source/values.py index 4abaf4cb7..6a1c1afe2 100644 --- a/src/sampletones_shared/meta/source/values.py +++ b/src/sampletones_tools/checks/source/values.py @@ -2,8 +2,8 @@ from dataclasses import dataclass from typing import Final, FrozenSet, Mapping, Tuple -from sampletones_shared.meta.source.bindings.environment import TypeEnvironment -from sampletones_shared.meta.source.nodes import expression_spelling, terminal_name +from sampletones_tools.checks.source.bindings.environment import TypeEnvironment +from sampletones_tools.checks.source.nodes import expression_spelling, terminal_name EnumMembers = Mapping[str, str] EnumTable = Mapping[str, EnumMembers] diff --git a/scripts/checks/tag_names.py b/src/sampletones_tools/checks/tag_names.py similarity index 78% rename from scripts/checks/tag_names.py rename to src/sampletones_tools/checks/tag_names.py index e2757858d..02754081f 100755 --- a/scripts/checks/tag_names.py +++ b/src/sampletones_tools/checks/tag_names.py @@ -1,19 +1,3 @@ -#!/usr/bin/env python3 - -""" -Checks that every tag constant is named after the tag it composes. - -`TAG_GLOBAL_WINDOW_MAIN = TagName(Page.GLOBAL, Panel.IMPLICIT, Widget.WINDOW, "main")` composes the -tag `global.window.main`, so its name is that tag upper-cased with each separator turned into an -underscore, behind the `TAG_` prefix. Reading the name therefore states the tag, and reading the tag -states where its constant lives. - -Usage: - python scripts/checks/tag_names.py [files...] # check the given modules - python scripts/checks/tag_names.py --all # check every module of the tags package -""" - -import argparse import ast import sys from enum import StrEnum @@ -23,10 +7,10 @@ from sampletones_application.categories.hierarchy import Page, Panel, Widget from sampletones_application.categories.key.tag import TagName from sampletones_application.tags.compose import TAG_SEPARATOR -from sampletones_shared.meta.source.constants import ModuleConstant, module_constants -from sampletones_shared.meta.source.modules import SourceModule, discover_modules, parse_module -from sampletones_shared.meta.source.nodes import terminal_name -from sampletones_shared.meta.source.packages import package_directory +from sampletones_tools.checks.source.constants import ModuleConstant, module_constants +from sampletones_tools.checks.source.modules import SourceModule, discover_modules, parse_module +from sampletones_tools.checks.source.nodes import terminal_name +from sampletones_tools.checks.source.packages import package_directory TAGS_PACKAGE: Final[Path] = package_directory("sampletones_application", "tags") @@ -202,27 +186,14 @@ def check_modules(modules: Sequence[SourceModule]) -> List[TagFinding]: return [finding for module in modules for finding in check_module(module)] -def main(argv: Sequence[str]) -> int: - """Report any tag constant whose name departs from the tag it composes.""" - parser = argparse.ArgumentParser( - description="Check that tag constant names state the tags they compose.", - ) - parser.add_argument( - "files", - nargs="*", - type=Path, - help="modules to check", - ) - parser.add_argument( - "--all", - action="store_true", - help=f"check every module under {TAGS_PACKAGE.name}/ instead of named files", - ) - arguments = parser.parse_args(list(argv)) - - files: List[Path] = arguments.files - modules = discover_modules([TAGS_PACKAGE]) if arguments.all else [parse_module(path) for path in files] - findings = check_modules(modules) +def check_tags(files: Sequence[Path], everything: bool) -> List[TagFinding]: + """The findings over the named modules, or over the whole tags package when ``everything``.""" + modules = discover_modules([TAGS_PACKAGE]) if everything else [parse_module(path) for path in files] + return check_modules(modules) + + +def report(findings: Sequence[TagFinding]) -> int: + """Prints every finding on the standard error stream and answers with the exit status.""" if not findings: return 0 @@ -232,7 +203,3 @@ def main(argv: Sequence[str]) -> int: print(f"\nFound {len(findings)} tag name(s) to fix.", file=sys.stderr) return 1 - - -if __name__ == "__main__": - raise SystemExit(main(sys.argv[1:])) diff --git a/scripts/checks/unused_tags.py b/src/sampletones_tools/checks/unused_tags.py similarity index 61% rename from scripts/checks/unused_tags.py rename to src/sampletones_tools/checks/unused_tags.py index 2dc8c9fc5..dd8acecb6 100755 --- a/scripts/checks/unused_tags.py +++ b/src/sampletones_tools/checks/unused_tags.py @@ -1,28 +1,13 @@ -#!/usr/bin/env python3 - -""" -Checks that every tag fragment the tags package declares is read somewhere. - -A tag or a suffix nobody reads still reads as part of the interface vocabulary, so it invites a -second constant for the same widget. The check counts reads across the sources, the tests, and the -scripts: a fragment feeding another fragment's value counts as read, while an import alone does not, -which is what makes a re-exported yet unread fragment visible. - -Usage: - python scripts/checks/unused_tags.py -""" - -import argparse import sys from collections import Counter from pathlib import Path from typing import Dict, Final, List, NamedTuple, Sequence, Tuple -from sampletones_shared.meta.source.constants import module_constants -from sampletones_shared.meta.source.modules import SourceModule, discover_modules -from sampletones_shared.meta.source.packages import package_directory -from sampletones_shared.meta.source.references import count_identifier_loads from sampletones_shared.paths.source import REPOSITORY_ROOT, SCRIPTS_ROOT, SOURCE_ROOT +from sampletones_tools.checks.source.constants import module_constants +from sampletones_tools.checks.source.modules import SourceModule, discover_modules +from sampletones_tools.checks.source.packages import package_directory +from sampletones_tools.checks.source.references import count_identifier_loads TAGS_PACKAGE: Final[Path] = package_directory("sampletones_application", "tags") REFERENCE_ROOTS: Final[Tuple[Path, ...]] = ( @@ -98,31 +83,15 @@ def unread_fragments( return [fragment for fragment in fragments if counts.get(fragment.name, 0) == 0] -def main(argv: Sequence[str]) -> int: - """Report every tag fragment the repository declares and never reads.""" - parser = argparse.ArgumentParser( - description="Check that every declared tag fragment is read somewhere.", - ) - parser.add_argument( - "--tags", - type=Path, - default=TAGS_PACKAGE, - help="package declaring the tag fragments", - ) - parser.add_argument( - "--reference-root", - type=Path, - action="append", - dest="reference_roots", - help="directory to count reads in, repeatable", - ) - arguments = parser.parse_args(list(argv)) - - tags: Path = arguments.tags - reference_roots: Tuple[Path, ...] = tuple(arguments.reference_roots or REFERENCE_ROOTS) +def check_reads(tags: Path, reference_roots: Sequence[Path]) -> List[Fragment]: + """Every tag fragment the package declares and the reference roots never read.""" fragments = declared_fragments(discover_modules([tags])) counts = reference_counts(discover_modules(reference_roots)) - unread = unread_fragments(fragments, counts) + return unread_fragments(fragments, counts) + + +def report(unread: Sequence[Fragment]) -> int: + """Prints every unread fragment on the standard error stream and answers with the exit status.""" if not unread: return 0 @@ -132,7 +101,3 @@ def main(argv: Sequence[str]) -> int: print(f"\nFound {len(unread)} unread tag fragment(s).", file=sys.stderr) return 1 - - -if __name__ == "__main__": - raise SystemExit(main(sys.argv[1:])) diff --git a/src/sampletones_tools/player/command.py b/src/sampletones_tools/player/command.py index 2b11dfde1..daac75206 100644 --- a/src/sampletones_tools/player/command.py +++ b/src/sampletones_tools/player/command.py @@ -31,15 +31,16 @@ def run(arguments: Namespace) -> int: """ given = DriverArguments(directory=arguments.directory) - from sampletones_shared.exceptions.player import DriverBuildError from sampletones_tools.checkout import require_checkout - from sampletones_tools.player.assembler.builder import build_driver - from sampletones_tools.player.assembler.layout import BINARY_DIRECTORY - from sampletones_tools.player.assembler.report import layout_lines if given.directory is None: require_checkout(NAME) + from sampletones_shared.exceptions.player import DriverBuildError + from sampletones_tools.player.assembler.builder import build_driver + from sampletones_tools.player.assembler.layout import BINARY_DIRECTORY + from sampletones_tools.player.assembler.report import layout_lines + try: image = build_driver(given.directory if given.directory is not None else BINARY_DIRECTORY) except DriverBuildError as error: diff --git a/src/sampletones_tools/registry.py b/src/sampletones_tools/registry.py index e0cd7b9c3..c70aa2f13 100644 --- a/src/sampletones_tools/registry.py +++ b/src/sampletones_tools/registry.py @@ -3,7 +3,8 @@ from sampletones_shared.command import Command from sampletones_tools.assets.command import ICONS from sampletones_tools.calibration.command import CALIBRATION +from sampletones_tools.checks.command import CHECK from sampletones_tools.player.command import DRIVER from sampletones_tools.samples.command import NSF -DEVELOPER_COMMANDS: Final[Tuple[Command, ...]] = (CALIBRATION, DRIVER, ICONS, NSF) +DEVELOPER_COMMANDS: Final[Tuple[Command, ...]] = (CALIBRATION, CHECK, DRIVER, ICONS, NSF) diff --git a/tests/integration/tooling/test_check_commands.py b/tests/integration/tooling/test_check_commands.py index a265cea4b..69f756c6f 100644 --- a/tests/integration/tooling/test_check_commands.py +++ b/tests/integration/tooling/test_check_commands.py @@ -1,69 +1,53 @@ from pathlib import Path -from typing import Dict, Final, List, Optional +from typing import Dict, Final, List import yaml +from sampletones.commands.registry import COMMANDS +from sampletones.dispatcher import build_parser from sampletones_shared.paths.source import REPOSITORY_ROOT +from sampletones_tools.checks.registry import GATES PRE_COMMIT_CONFIG: Final[Path] = REPOSITORY_ROOT / ".pre-commit-config.yaml" -MAKEFILE: Final[Path] = REPOSITORY_ROOT / "Makefile" FILE_ENCODING: Final[str] = "utf-8" LOCAL_REPOSITORY: Final[str] = "local" -CHECK_SCRIPTS: Final[str] = "scripts/checks/" -TARGET_PREFIX: Final[str] = "check-" -SCRIPT_SUFFIX: Final[str] = ".py" -RECIPE_PREFIX: Final[str] = "\t" -TARGET_SUFFIX: Final[str] = ":" +ENTRY_PREFIX: Final[str] = "uv run sampletones " +CHECK_PREFIX: Final[str] = f"{ENTRY_PREFIX}check " def check_hooks() -> List[Dict[str, object]]: - """Every local hook running one of the check scripts, as the configuration declares it.""" + """Every local hook running one of the checks, as the configuration declares it.""" config = yaml.safe_load(PRE_COMMIT_CONFIG.read_text(encoding=FILE_ENCODING)) return [ hook for repository in config["repos"] if repository["repo"] == LOCAL_REPOSITORY for hook in repository["hooks"] - if CHECK_SCRIPTS in str(hook["entry"]) + if str(hook["entry"]).startswith(CHECK_PREFIX) ] -def check_targets() -> Dict[str, str]: - """The command each `check-*` target of the Makefile runs, keyed by target name.""" - targets: Dict[str, str] = {} - target: Optional[str] = None - for line in MAKEFILE.read_text(encoding=FILE_ENCODING).splitlines(): - if line.startswith(TARGET_PREFIX) and line.endswith(TARGET_SUFFIX): - target = line.removesuffix(TARGET_SUFFIX) - elif target is not None and line.startswith(RECIPE_PREFIX): - targets[target] = line.strip() - target = None - - return targets - - -def script_path(entry: str) -> Path: - """The check script an entry runs, taken from the words the entry is written with.""" - return REPOSITORY_ROOT / next(word for word in entry.split() if word.endswith(SCRIPT_SUFFIX)) +def hook_gate(hook: Dict[str, object]) -> str: + """The check a hook runs, taken from the words the entry is written with.""" + return str(hook["entry"]).removeprefix(CHECK_PREFIX).split()[0] class TestCheckHooks: - def test_the_configuration_declares_a_hook_for_every_check_script(self) -> None: - scripts = {path.name for path in (REPOSITORY_ROOT / CHECK_SCRIPTS).glob(f"*{SCRIPT_SUFFIX}")} + def test_the_configuration_declares_a_hook_for_every_check(self) -> None: + hooks = check_hooks() + + assert GATES + assert {hook_gate(hook) for hook in hooks} == {gate.name for gate in GATES} + assert len(hooks) == len(GATES) - assert {script_path(str(hook["entry"])).name for hook in check_hooks()} == scripts + def test_every_check_hook_parses_as_the_entry_runs_it(self) -> None: + """A hook's entry is a command line the entry accepts, so a renamed option fails here.""" + parser = build_parser(COMMANDS) - def test_every_check_hook_names_a_script_that_is_there(self) -> None: - assert all(script_path(str(hook["entry"])).is_file() for hook in check_hooks()) + for hook in check_hooks(): + parser.parse_args(str(hook["entry"]).removeprefix(ENTRY_PREFIX).split()) def test_every_check_hook_sweeps_the_whole_tree(self) -> None: """A hook handed the staged files checks the staged subset, which passes what it never reads.""" assert all(hook["pass_filenames"] is False for hook in check_hooks()) - - -class TestCheckTargets: - def test_each_hook_and_its_make_target_run_the_same_command(self) -> None: - commands = {f"{TARGET_PREFIX}{hook['id']}": str(hook["entry"]) for hook in check_hooks()} - - assert check_targets() == commands diff --git a/tests/suite/source.py b/tests/suite/source.py index 86a038f75..60f9078d3 100644 --- a/tests/suite/source.py +++ b/tests/suite/source.py @@ -3,8 +3,8 @@ from textwrap import dedent from typing import Iterable, Set -from sampletones_shared.meta.source.bindings.scopes import Scope -from sampletones_shared.meta.source.modules import source_paths +from sampletones_tools.checks.source.bindings.scopes import Scope +from sampletones_tools.checks.source.modules import source_paths def parse_source(source: str) -> ast.Module: diff --git a/tests/unit/sampletones_shared/paths/test_source.py b/tests/unit/sampletones_shared/paths/test_source.py index 36a210b2c..426673b00 100644 --- a/tests/unit/sampletones_shared/paths/test_source.py +++ b/tests/unit/sampletones_shared/paths/test_source.py @@ -17,9 +17,6 @@ class TestRepositoryRoot: def test_the_repository_root_holds_the_project_file(self) -> None: assert (REPOSITORY_ROOT / PROJECT_FILE).is_file() - def test_the_repository_root_holds_the_scripts_the_checks_run_from(self) -> None: - assert (SCRIPTS_ROOT / "checks").is_dir() - class TestScriptsRoot: def test_the_scripts_root_holds_the_bootstrap_tree(self) -> None: diff --git a/tests/unit/sampletones_shared/meta/source/__init__.py b/tests/unit/sampletones_tools/checks/boundary/__init__.py similarity index 100% rename from tests/unit/sampletones_shared/meta/source/__init__.py rename to tests/unit/sampletones_tools/checks/boundary/__init__.py diff --git a/tests/unit/sampletones_shared/meta/source/bindings/__init__.py b/tests/unit/sampletones_tools/checks/boundary/configs/__init__.py similarity index 100% rename from tests/unit/sampletones_shared/meta/source/bindings/__init__.py rename to tests/unit/sampletones_tools/checks/boundary/configs/__init__.py diff --git a/tests/unit/sampletones_shared/meta/import_boundary/configs/test_declaration.py b/tests/unit/sampletones_tools/checks/boundary/configs/test_declaration.py similarity index 90% rename from tests/unit/sampletones_shared/meta/import_boundary/configs/test_declaration.py rename to tests/unit/sampletones_tools/checks/boundary/configs/test_declaration.py index de73889d2..333b71165 100644 --- a/tests/unit/sampletones_shared/meta/import_boundary/configs/test_declaration.py +++ b/tests/unit/sampletones_tools/checks/boundary/configs/test_declaration.py @@ -1,7 +1,7 @@ from typing import Final -from sampletones_shared.meta.import_boundary.configs.declaration import BoundaryDeclaration -from sampletones_shared.meta.import_boundary.configs.general import GeneralBoundaries +from sampletones_tools.checks.boundary.configs.declaration import BoundaryDeclaration +from sampletones_tools.checks.boundary.configs.general import GeneralBoundaries ROOT: Final[str] = "package" PATTERN: Final[str] = "logic/**/*.py" diff --git a/tests/unit/sampletones_shared/meta/import_boundary/configs/test_general.py b/tests/unit/sampletones_tools/checks/boundary/configs/test_general.py similarity index 91% rename from tests/unit/sampletones_shared/meta/import_boundary/configs/test_general.py rename to tests/unit/sampletones_tools/checks/boundary/configs/test_general.py index 51b85a3c6..c1b86c2ea 100644 --- a/tests/unit/sampletones_shared/meta/import_boundary/configs/test_general.py +++ b/tests/unit/sampletones_tools/checks/boundary/configs/test_general.py @@ -2,7 +2,7 @@ import pytest -from sampletones_shared.meta.import_boundary.configs.general import GeneralBoundaries +from sampletones_tools.checks.boundary.configs.general import GeneralBoundaries GENERAL: Final[GeneralBoundaries] = GeneralBoundaries( groups={ diff --git a/tests/unit/sampletones_shared/meta/import_boundary/configs/test_rules.py b/tests/unit/sampletones_tools/checks/boundary/configs/test_rules.py similarity index 92% rename from tests/unit/sampletones_shared/meta/import_boundary/configs/test_rules.py rename to tests/unit/sampletones_tools/checks/boundary/configs/test_rules.py index 510919c8e..6efe16a1c 100644 --- a/tests/unit/sampletones_shared/meta/import_boundary/configs/test_rules.py +++ b/tests/unit/sampletones_tools/checks/boundary/configs/test_rules.py @@ -5,15 +5,15 @@ import pytest from pydantic import ValidationError -from sampletones_shared.meta.import_boundary.check import check_boundaries -from sampletones_shared.meta.import_boundary.configs.declaration import BoundaryDeclaration -from sampletones_shared.meta.import_boundary.configs.general import GeneralBoundaries -from sampletones_shared.meta.import_boundary.configs.rules import ImportBoundaryRules -from sampletones_shared.meta.import_boundary.graph import reached_units -from sampletones_shared.meta.import_boundary.rule import BoundaryRule -from sampletones_shared.meta.import_boundary.scope import rule_modules -from sampletones_shared.meta.import_boundary.standalone import check_standalone from sampletones_shared.paths.source import SCRIPTS_ROOT, SOURCE_ROOT +from sampletones_tools.checks.boundary.check import check_boundaries +from sampletones_tools.checks.boundary.configs.declaration import BoundaryDeclaration +from sampletones_tools.checks.boundary.configs.general import GeneralBoundaries +from sampletones_tools.checks.boundary.configs.rules import ImportBoundaryRules +from sampletones_tools.checks.boundary.graph import reached_units +from sampletones_tools.checks.boundary.rule import BoundaryRule +from sampletones_tools.checks.boundary.scope import rule_modules +from sampletones_tools.checks.boundary.standalone import check_standalone from tests.suite.source import swept_paths, write_module BOUNDARIES: Final[ImportBoundaryRules] = ImportBoundaryRules.load() diff --git a/tests/unit/sampletones_shared/meta/import_boundary/test_check.py b/tests/unit/sampletones_tools/checks/boundary/test_check.py similarity index 94% rename from tests/unit/sampletones_shared/meta/import_boundary/test_check.py rename to tests/unit/sampletones_tools/checks/boundary/test_check.py index 7cd39473c..59554dc72 100644 --- a/tests/unit/sampletones_shared/meta/import_boundary/test_check.py +++ b/tests/unit/sampletones_tools/checks/boundary/test_check.py @@ -3,9 +3,9 @@ import pytest -from sampletones_shared.meta.import_boundary.check import check_boundaries -from sampletones_shared.meta.import_boundary.rule import BoundaryRule -from sampletones_shared.meta.import_boundary.token import TokenRule +from sampletones_tools.checks.boundary.check import check_boundaries +from sampletones_tools.checks.boundary.rule import BoundaryRule +from sampletones_tools.checks.boundary.token import TokenRule from tests.suite.source import write_module FORBIDDEN: Final[str] = "from other_package.module import Thing\n" diff --git a/tests/unit/sampletones_shared/meta/import_boundary/test_graph.py b/tests/unit/sampletones_tools/checks/boundary/test_graph.py similarity index 97% rename from tests/unit/sampletones_shared/meta/import_boundary/test_graph.py rename to tests/unit/sampletones_tools/checks/boundary/test_graph.py index d6857ef97..f7006ffef 100644 --- a/tests/unit/sampletones_shared/meta/import_boundary/test_graph.py +++ b/tests/unit/sampletones_tools/checks/boundary/test_graph.py @@ -3,7 +3,7 @@ import pytest from pydantic import ValidationError -from sampletones_shared.meta.import_boundary.graph import LayerGraph, reached_units +from sampletones_tools.checks.boundary.graph import LayerGraph, reached_units GRAPH: Final[LayerGraph] = LayerGraph( root="package", diff --git a/tests/unit/sampletones_shared/meta/import_boundary/test_imports.py b/tests/unit/sampletones_tools/checks/boundary/test_imports.py similarity index 95% rename from tests/unit/sampletones_shared/meta/import_boundary/test_imports.py rename to tests/unit/sampletones_tools/checks/boundary/test_imports.py index 681e2c086..bb9bd8bdc 100644 --- a/tests/unit/sampletones_shared/meta/import_boundary/test_imports.py +++ b/tests/unit/sampletones_tools/checks/boundary/test_imports.py @@ -3,7 +3,7 @@ import pytest -from sampletones_shared.meta.import_boundary.imports import imported_module, matches_prefix +from sampletones_tools.checks.boundary.imports import imported_module, matches_prefix from tests.suite.base import BaseTestSuite from tests.suite.case import BaseAutolabelTestCase diff --git a/tests/unit/sampletones_shared/meta/import_boundary/test_rule.py b/tests/unit/sampletones_tools/checks/boundary/test_rule.py similarity index 97% rename from tests/unit/sampletones_shared/meta/import_boundary/test_rule.py rename to tests/unit/sampletones_tools/checks/boundary/test_rule.py index a6058a404..bf398cb4a 100644 --- a/tests/unit/sampletones_shared/meta/import_boundary/test_rule.py +++ b/tests/unit/sampletones_tools/checks/boundary/test_rule.py @@ -1,7 +1,7 @@ from pathlib import Path from typing import Final -from sampletones_shared.meta.import_boundary.rule import BoundaryRule +from sampletones_tools.checks.boundary.rule import BoundaryRule from tests.suite.source import write_module FORBIDDEN: Final[str] = "from other_package.module import Thing\n" diff --git a/tests/unit/sampletones_shared/meta/import_boundary/test_scope.py b/tests/unit/sampletones_tools/checks/boundary/test_scope.py similarity index 97% rename from tests/unit/sampletones_shared/meta/import_boundary/test_scope.py rename to tests/unit/sampletones_tools/checks/boundary/test_scope.py index 92050e731..94b58a597 100644 --- a/tests/unit/sampletones_shared/meta/import_boundary/test_scope.py +++ b/tests/unit/sampletones_tools/checks/boundary/test_scope.py @@ -1,7 +1,7 @@ from pathlib import Path from typing import Final -from sampletones_shared.meta.import_boundary.scope import rule_modules +from sampletones_tools.checks.boundary.scope import rule_modules from tests.suite.source import swept_paths, write_module LOGIC_PATTERN: Final[str] = "logic/**/*.py" diff --git a/tests/unit/sampletones_shared/meta/import_boundary/test_standalone.py b/tests/unit/sampletones_tools/checks/boundary/test_standalone.py similarity index 98% rename from tests/unit/sampletones_shared/meta/import_boundary/test_standalone.py rename to tests/unit/sampletones_tools/checks/boundary/test_standalone.py index eb184a54c..f34745270 100644 --- a/tests/unit/sampletones_shared/meta/import_boundary/test_standalone.py +++ b/tests/unit/sampletones_tools/checks/boundary/test_standalone.py @@ -3,7 +3,7 @@ import pytest -from sampletones_shared.meta.import_boundary.standalone import ( +from sampletones_tools.checks.boundary.standalone import ( StandaloneRule, check_standalone, local_names, diff --git a/tests/unit/sampletones_shared/meta/import_boundary/test_token.py b/tests/unit/sampletones_tools/checks/boundary/test_token.py similarity index 96% rename from tests/unit/sampletones_shared/meta/import_boundary/test_token.py rename to tests/unit/sampletones_tools/checks/boundary/test_token.py index d1b3018a9..ec2881921 100644 --- a/tests/unit/sampletones_shared/meta/import_boundary/test_token.py +++ b/tests/unit/sampletones_tools/checks/boundary/test_token.py @@ -4,7 +4,7 @@ import pytest from pydantic import ValidationError -from sampletones_shared.meta.import_boundary.token import TokenRule +from sampletones_tools.checks.boundary.token import TokenRule from tests.suite.source import write_module MESSAGE: Final[str] = "a panel receives its parent through create_panel(parent)" diff --git a/tests/unit/sampletones_shared/meta/import_boundary/test_units.py b/tests/unit/sampletones_tools/checks/boundary/test_units.py similarity index 92% rename from tests/unit/sampletones_shared/meta/import_boundary/test_units.py rename to tests/unit/sampletones_tools/checks/boundary/test_units.py index d824a4adf..a3df988a2 100644 --- a/tests/unit/sampletones_shared/meta/import_boundary/test_units.py +++ b/tests/unit/sampletones_tools/checks/boundary/test_units.py @@ -1,4 +1,4 @@ -from sampletones_shared.meta.import_boundary.units import nested_globs, unit_glob, unit_prefix +from sampletones_tools.checks.boundary.units import nested_globs, unit_glob, unit_prefix PLAYER = "sampletones_player" diff --git a/tests/unit/sampletones_shared/meta/import_boundary/test_violation.py b/tests/unit/sampletones_tools/checks/boundary/test_violation.py similarity index 92% rename from tests/unit/sampletones_shared/meta/import_boundary/test_violation.py rename to tests/unit/sampletones_tools/checks/boundary/test_violation.py index af8e8a0f5..2b4939a8f 100644 --- a/tests/unit/sampletones_shared/meta/import_boundary/test_violation.py +++ b/tests/unit/sampletones_tools/checks/boundary/test_violation.py @@ -1,6 +1,6 @@ from pathlib import Path -from sampletones_shared.meta.import_boundary.violation import Violation +from sampletones_tools.checks.boundary.violation import Violation class TestViolationLocation: diff --git a/tests/unit/sampletones_tools/checks/source/__init__.py b/tests/unit/sampletones_tools/checks/source/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/unit/sampletones_tools/checks/source/bindings/__init__.py b/tests/unit/sampletones_tools/checks/source/bindings/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/unit/sampletones_shared/meta/source/bindings/test_containers.py b/tests/unit/sampletones_tools/checks/source/bindings/test_containers.py similarity index 98% rename from tests/unit/sampletones_shared/meta/source/bindings/test_containers.py rename to tests/unit/sampletones_tools/checks/source/bindings/test_containers.py index b66c9d09e..c99070413 100644 --- a/tests/unit/sampletones_shared/meta/source/bindings/test_containers.py +++ b/tests/unit/sampletones_tools/checks/source/bindings/test_containers.py @@ -1,7 +1,7 @@ import ast from typing import Dict, Final, List, Optional, Tuple -from sampletones_shared.meta.source.bindings.containers import ( +from sampletones_tools.checks.source.bindings.containers import ( container_item_types, iterated_container, iterated_types, diff --git a/tests/unit/sampletones_shared/meta/source/bindings/test_environment.py b/tests/unit/sampletones_tools/checks/source/bindings/test_environment.py similarity index 94% rename from tests/unit/sampletones_shared/meta/source/bindings/test_environment.py rename to tests/unit/sampletones_tools/checks/source/bindings/test_environment.py index 79bc0bb9c..a088ae164 100644 --- a/tests/unit/sampletones_shared/meta/source/bindings/test_environment.py +++ b/tests/unit/sampletones_tools/checks/source/bindings/test_environment.py @@ -1,6 +1,6 @@ from typing import Final -from sampletones_shared.meta.source.bindings.environment import TypeEnvironment +from sampletones_tools.checks.source.bindings.environment import TypeEnvironment ENVIRONMENT: Final[TypeEnvironment] = TypeEnvironment( types={ diff --git a/tests/unit/sampletones_shared/meta/source/bindings/test_scopes.py b/tests/unit/sampletones_tools/checks/source/bindings/test_scopes.py similarity index 97% rename from tests/unit/sampletones_shared/meta/source/bindings/test_scopes.py rename to tests/unit/sampletones_tools/checks/source/bindings/test_scopes.py index aa4b6d2aa..f9b7c6a82 100644 --- a/tests/unit/sampletones_shared/meta/source/bindings/test_scopes.py +++ b/tests/unit/sampletones_tools/checks/source/bindings/test_scopes.py @@ -1,7 +1,7 @@ from typing import Final, List, Mapping, Tuple -from sampletones_shared.meta.source.bindings.environment import TypeEnvironment -from sampletones_shared.meta.source.bindings.scopes import Scope, module_scopes +from sampletones_tools.checks.source.bindings.environment import TypeEnvironment +from sampletones_tools.checks.source.bindings.scopes import Scope, module_scopes from tests.suite.source import parse_source, scope_named PANEL_SOURCE: Final[str] = """ diff --git a/tests/unit/sampletones_shared/meta/source/bindings/test_statements.py b/tests/unit/sampletones_tools/checks/source/bindings/test_statements.py similarity index 98% rename from tests/unit/sampletones_shared/meta/source/bindings/test_statements.py rename to tests/unit/sampletones_tools/checks/source/bindings/test_statements.py index 8aa1043fe..6b1467ea5 100644 --- a/tests/unit/sampletones_shared/meta/source/bindings/test_statements.py +++ b/tests/unit/sampletones_tools/checks/source/bindings/test_statements.py @@ -1,7 +1,7 @@ import ast from typing import Optional, Type -from sampletones_shared.meta.source.bindings.statements import ( +from sampletones_tools.checks.source.bindings.statements import ( AliasStatement, LoopStatement, Statement, diff --git a/tests/unit/sampletones_shared/meta/source/test_annotations.py b/tests/unit/sampletones_tools/checks/source/test_annotations.py similarity index 99% rename from tests/unit/sampletones_shared/meta/source/test_annotations.py rename to tests/unit/sampletones_tools/checks/source/test_annotations.py index 36b3de5a7..c00b4c58e 100644 --- a/tests/unit/sampletones_shared/meta/source/test_annotations.py +++ b/tests/unit/sampletones_tools/checks/source/test_annotations.py @@ -4,7 +4,7 @@ import pytest -from sampletones_shared.meta.source.annotations import ( +from sampletones_tools.checks.source.annotations import ( annotation_item_types, annotation_type_name, unwrap_annotation, diff --git a/tests/unit/sampletones_shared/meta/source/test_classes.py b/tests/unit/sampletones_tools/checks/source/test_classes.py similarity index 96% rename from tests/unit/sampletones_shared/meta/source/test_classes.py rename to tests/unit/sampletones_tools/checks/source/test_classes.py index 00603297f..4174ab6ed 100644 --- a/tests/unit/sampletones_shared/meta/source/test_classes.py +++ b/tests/unit/sampletones_tools/checks/source/test_classes.py @@ -1,6 +1,6 @@ from typing import Final, List -from sampletones_shared.meta.source.classes import declared_subclasses +from sampletones_tools.checks.source.classes import declared_subclasses from tests.suite.source import parse_source CLASSES_SOURCE: Final[str] = """ diff --git a/tests/unit/sampletones_shared/meta/source/test_constants.py b/tests/unit/sampletones_tools/checks/source/test_constants.py similarity index 96% rename from tests/unit/sampletones_shared/meta/source/test_constants.py rename to tests/unit/sampletones_tools/checks/source/test_constants.py index 1c76fadd1..6ea61c404 100644 --- a/tests/unit/sampletones_shared/meta/source/test_constants.py +++ b/tests/unit/sampletones_tools/checks/source/test_constants.py @@ -1,7 +1,7 @@ import ast from typing import Dict, Final, List -from sampletones_shared.meta.source.constants import ModuleConstant, module_constants +from sampletones_tools.checks.source.constants import ModuleConstant, module_constants from tests.suite.source import parse_source CONSTANTS_SOURCE: Final[str] = """ diff --git a/tests/unit/sampletones_shared/meta/source/test_index.py b/tests/unit/sampletones_tools/checks/source/test_index.py similarity index 92% rename from tests/unit/sampletones_shared/meta/source/test_index.py rename to tests/unit/sampletones_tools/checks/source/test_index.py index 7e50e97c0..4828305b6 100644 --- a/tests/unit/sampletones_shared/meta/source/test_index.py +++ b/tests/unit/sampletones_tools/checks/source/test_index.py @@ -2,8 +2,8 @@ from pathlib import Path from typing import Final -from sampletones_shared.meta.source.index import SourceIndex, source_index -from sampletones_shared.meta.source.modules import SourceModule +from sampletones_tools.checks.source.index import SourceIndex, source_index +from sampletones_tools.checks.source.modules import SourceModule from tests.suite.source import parse_source TAGS_SOURCE: Final[str] = """ diff --git a/tests/unit/sampletones_shared/meta/source/test_lookups.py b/tests/unit/sampletones_tools/checks/source/test_lookups.py similarity index 96% rename from tests/unit/sampletones_shared/meta/source/test_lookups.py rename to tests/unit/sampletones_tools/checks/source/test_lookups.py index 1db807dbc..613a8da8c 100644 --- a/tests/unit/sampletones_shared/meta/source/test_lookups.py +++ b/tests/unit/sampletones_tools/checks/source/test_lookups.py @@ -4,15 +4,15 @@ import pytest -from sampletones_shared.meta.source.index import source_index -from sampletones_shared.meta.source.lookups import ( +from sampletones_tools.checks.source.index import source_index +from sampletones_tools.checks.source.lookups import ( LookupSite, composed_values, module_lookups, tree_lookups, ) -from sampletones_shared.meta.source.modules import SourceModule -from sampletones_shared.meta.source.values import UNRESOLVED, EnumTable, ResolvedValues +from sampletones_tools.checks.source.modules import SourceModule +from sampletones_tools.checks.source.values import UNRESOLVED, EnumTable, ResolvedValues from tests.suite.base import BaseTestSuite from tests.suite.case import BaseRegularTestCase from tests.suite.source import parse_source diff --git a/tests/unit/sampletones_shared/meta/source/test_modules.py b/tests/unit/sampletones_tools/checks/source/test_modules.py similarity index 97% rename from tests/unit/sampletones_shared/meta/source/test_modules.py rename to tests/unit/sampletones_tools/checks/source/test_modules.py index 777586ccb..bd5cfa7f8 100644 --- a/tests/unit/sampletones_shared/meta/source/test_modules.py +++ b/tests/unit/sampletones_tools/checks/source/test_modules.py @@ -5,7 +5,7 @@ import pytest -from sampletones_shared.meta.source.modules import ( +from sampletones_tools.checks.source.modules import ( discover_modules, is_visible, module_name, @@ -52,7 +52,7 @@ def test_a_location_names_the_path_and_the_line(self, tmp_path: Path) -> None: class TestIsVisible: def test_a_plain_path_is_visible(self) -> None: - assert is_visible(Path("src/sampletones_shared/meta/source/modules.py")) + assert is_visible(Path("src/sampletones_tools/checks/source/modules.py")) def test_a_hidden_directory_hides_the_path(self) -> None: assert not is_visible(Path(".venv/lib/python3.12/ast.py")) diff --git a/tests/unit/sampletones_shared/meta/source/test_nodes.py b/tests/unit/sampletones_tools/checks/source/test_nodes.py similarity index 98% rename from tests/unit/sampletones_shared/meta/source/test_nodes.py rename to tests/unit/sampletones_tools/checks/source/test_nodes.py index 1d68f1b39..fc939274b 100644 --- a/tests/unit/sampletones_shared/meta/source/test_nodes.py +++ b/tests/unit/sampletones_tools/checks/source/test_nodes.py @@ -1,7 +1,7 @@ import ast from typing import Final, Iterable, List -from sampletones_shared.meta.source.nodes import ( +from sampletones_tools.checks.source.nodes import ( expression_spelling, is_attribute_spelling, nested_scopes, diff --git a/tests/unit/sampletones_shared/meta/source/test_packages.py b/tests/unit/sampletones_tools/checks/source/test_packages.py similarity index 86% rename from tests/unit/sampletones_shared/meta/source/test_packages.py rename to tests/unit/sampletones_tools/checks/source/test_packages.py index e18e8b853..7859a88e5 100644 --- a/tests/unit/sampletones_shared/meta/source/test_packages.py +++ b/tests/unit/sampletones_tools/checks/source/test_packages.py @@ -1,7 +1,7 @@ import pytest -from sampletones_shared.meta.source.packages import package_directory from sampletones_shared.paths.source import SOURCE_ROOT +from sampletones_tools.checks.source.packages import package_directory SHARED_PACKAGE = "sampletones_shared" APPLICATION_PACKAGE = "sampletones_application" @@ -12,7 +12,7 @@ def test_a_top_level_package_sits_under_the_source_root(self) -> None: assert package_directory(SHARED_PACKAGE) == SOURCE_ROOT / SHARED_PACKAGE def test_a_subpackage_is_named_part_by_part(self) -> None: - assert package_directory(SHARED_PACKAGE, "meta", "source") == SOURCE_ROOT / SHARED_PACKAGE / "meta" / "source" + assert package_directory(SHARED_PACKAGE, "utils", "system") == SOURCE_ROOT / SHARED_PACKAGE / "utils" / "system" def test_the_answer_is_a_directory_a_sweep_reads_under(self) -> None: """A package resource resolves to `__init__.py`, which a sweep reads nothing under.""" diff --git a/tests/unit/sampletones_shared/meta/source/test_references.py b/tests/unit/sampletones_tools/checks/source/test_references.py similarity index 95% rename from tests/unit/sampletones_shared/meta/source/test_references.py rename to tests/unit/sampletones_tools/checks/source/test_references.py index 50456eef8..195d98aa6 100644 --- a/tests/unit/sampletones_shared/meta/source/test_references.py +++ b/tests/unit/sampletones_tools/checks/source/test_references.py @@ -1,6 +1,6 @@ from typing import Dict, Final -from sampletones_shared.meta.source.references import count_identifier_loads +from sampletones_tools.checks.source.references import count_identifier_loads from tests.suite.source import parse_source REFERENCES_SOURCE: Final[str] = """ diff --git a/tests/unit/sampletones_shared/meta/source/test_subscripts.py b/tests/unit/sampletones_tools/checks/source/test_subscripts.py similarity index 93% rename from tests/unit/sampletones_shared/meta/source/test_subscripts.py rename to tests/unit/sampletones_tools/checks/source/test_subscripts.py index 5285a216a..88437fc37 100644 --- a/tests/unit/sampletones_shared/meta/source/test_subscripts.py +++ b/tests/unit/sampletones_tools/checks/source/test_subscripts.py @@ -1,7 +1,7 @@ from typing import Final, List -from sampletones_shared.meta.source.bindings.scopes import module_scopes -from sampletones_shared.meta.source.subscripts import SubscriptSite, find_subscripts +from sampletones_tools.checks.source.bindings.scopes import module_scopes +from sampletones_tools.checks.source.subscripts import SubscriptSite, find_subscripts from tests.suite.source import parse_source, scope_named RECEIVER_TYPE: Final[str] = "LanguageManager" diff --git a/tests/unit/sampletones_shared/meta/source/test_values.py b/tests/unit/sampletones_tools/checks/source/test_values.py similarity index 97% rename from tests/unit/sampletones_shared/meta/source/test_values.py rename to tests/unit/sampletones_tools/checks/source/test_values.py index 733ad0e94..025ab8349 100644 --- a/tests/unit/sampletones_shared/meta/source/test_values.py +++ b/tests/unit/sampletones_tools/checks/source/test_values.py @@ -4,8 +4,8 @@ import pytest -from sampletones_shared.meta.source.bindings.environment import TypeEnvironment -from sampletones_shared.meta.source.values import ( +from sampletones_tools.checks.source.bindings.environment import TypeEnvironment +from sampletones_tools.checks.source.values import ( UNRESOLVED, EnumTable, ResolvedValues, diff --git a/tests/unit/sampletones_tools/checks/test_command.py b/tests/unit/sampletones_tools/checks/test_command.py new file mode 100644 index 000000000..6b3d52989 --- /dev/null +++ b/tests/unit/sampletones_tools/checks/test_command.py @@ -0,0 +1,61 @@ +from argparse import ArgumentParser, Namespace +from typing import Final, List + +import pytest + +from sampletones.commands.registry import COMMANDS +from sampletones.dispatcher import build_parser, dispatch +from sampletones_shared.command import Command +from sampletones_tools.checks import command +from sampletones_tools.checks.registry import GATES + +GUARD: Final[str] = "sampletones_tools.checkout.require_checkout" + + +def _gate(name: str, calls: List[str]) -> Command: + def configure(parser: ArgumentParser) -> None: + parser.add_argument("--flag", action="store_true") + + def run(arguments: Namespace) -> int: + calls.append(f"{name} {arguments.flag}") + return 3 + + return Command(name=name, help=f"{name} help", configure=configure, run=run) + + +class TestCheck: + def test_every_gate_is_listed(self, capsys: pytest.CaptureFixture[str]) -> None: + with pytest.raises(SystemExit) as leaving: + build_parser(COMMANDS).parse_args(["check", "--help"]) + + assert leaving.value.code == 0 + listing = capsys.readouterr().out + assert GATES + assert all(gate.name in listing for gate in GATES) + + def test_the_gate_named_runs_from_a_checkout(self, monkeypatch: pytest.MonkeyPatch) -> None: + calls: List[str] = [] + guarded: List[str] = [] + monkeypatch.setattr(command, "GATES", (_gate("probe", calls),)) + monkeypatch.setattr(GUARD, guarded.append) + + assert dispatch(COMMANDS, ["check", "probe", "--flag"]) == 3 + assert calls == ["probe True"] + assert guarded == ["check probe"] + + def test_an_unknown_check_is_refused(self) -> None: + with pytest.raises(SystemExit) as leaving: + dispatch(COMMANDS, ["check", "absent"]) + + assert leaving.value.code == 2 + + def test_a_check_is_required(self) -> None: + with pytest.raises(SystemExit) as leaving: + dispatch(COMMANDS, ["check"]) + + assert leaving.value.code == 2 + + def test_every_gate_has_a_name_of_its_own(self) -> None: + names = [gate.name for gate in GATES] + + assert len(set(names)) == len(names) diff --git a/tests/unit/scripts/checks/test_import_boundary.py b/tests/unit/sampletones_tools/checks/test_import_boundary.py similarity index 69% rename from tests/unit/scripts/checks/test_import_boundary.py rename to tests/unit/sampletones_tools/checks/test_import_boundary.py index c0b1daef0..c5f372b05 100644 --- a/tests/unit/scripts/checks/test_import_boundary.py +++ b/tests/unit/sampletones_tools/checks/test_import_boundary.py @@ -3,11 +3,11 @@ import pytest -from tests.suite.scripts import load_script +from sampletones.commands.registry import COMMANDS +from sampletones.dispatcher import dispatch +from sampletones_tools.checks import import_boundary as check_import_boundary from tests.suite.source import write_module -check_import_boundary = load_script("checks/import_boundary.py") - APPLICATION: Final[str] = "sampletones_application" VISUAL_IMPORT: Final[str] = "import dearpygui.dearpygui as dpg\n" @@ -17,7 +17,7 @@ class TestMain: def test_the_repository_holds_its_import_boundaries(self) -> None: - assert check_import_boundary.main(["--all"]) == 0 + assert dispatch(COMMANDS, ["check", "import-boundary", "--all"]) == 0 def test_a_forbidden_import_is_reported_where_it_sits( self, @@ -26,7 +26,7 @@ def test_a_forbidden_import_is_reported_where_it_sits( ) -> None: path = write_module(tmp_path / APPLICATION / "logic", "direct.py", VISUAL_IMPORT) - exit_code = check_import_boundary.main(["--all", "--source", str(tmp_path)]) + exit_code = dispatch(COMMANDS, ["check", "import-boundary", "--all", "--source", str(tmp_path)]) assert exit_code == 1 error = capsys.readouterr().err @@ -41,8 +41,17 @@ def test_a_bootstrap_script_reaching_past_the_standard_library_is_reported( write_module(tmp_path / "src" / APPLICATION / "logic", "clean.py", PLAIN_IMPORT) path = write_module(tmp_path / "scripts", "bundle.py", THIRD_PARTY_IMPORT) - exit_code = check_import_boundary.main( - ["--all", "--source", str(tmp_path / "src"), "--scripts", str(tmp_path / "scripts")], + exit_code = dispatch( + COMMANDS, + [ + "check", + "import-boundary", + "--all", + "--source", + str(tmp_path / "src"), + "--scripts", + str(tmp_path / "scripts"), + ], ) assert exit_code == 1 @@ -58,5 +67,5 @@ def test_named_files_narrow_the_run_to_themselves( write_module(tmp_path / APPLICATION / "logic", "reported.py", VISUAL_IMPORT) clean = write_module(tmp_path / APPLICATION / "logic", "clean.py", PLAIN_IMPORT) - assert check_import_boundary.main([str(clean), "--source", str(tmp_path)]) == 0 + assert dispatch(COMMANDS, ["check", "import-boundary", str(clean), "--source", str(tmp_path)]) == 0 assert capsys.readouterr().err == "" diff --git a/tests/unit/scripts/checks/test_language_keys.py b/tests/unit/sampletones_tools/checks/test_language_keys.py similarity index 94% rename from tests/unit/scripts/checks/test_language_keys.py rename to tests/unit/sampletones_tools/checks/test_language_keys.py index 3e74ffab2..6f47d2b5b 100644 --- a/tests/unit/scripts/checks/test_language_keys.py +++ b/tests/unit/sampletones_tools/checks/test_language_keys.py @@ -3,15 +3,15 @@ import pytest +from sampletones.commands.registry import COMMANDS +from sampletones.dispatcher import dispatch from sampletones_application.categories.elements.global_ import DialogElements from sampletones_application.logic.history.action import HistoryAction from sampletones_application.paths import LANG_EN -from sampletones_shared.meta.source.lookups import LookupSite -from sampletones_shared.meta.source.modules import source_paths -from sampletones_shared.meta.source.values import EnumTable -from tests.suite.scripts import load_script - -check_language_keys = load_script("checks/language_keys.py") +from sampletones_tools.checks import language_keys as check_language_keys +from sampletones_tools.checks.source.lookups import LookupSite +from sampletones_tools.checks.source.modules import source_paths +from sampletones_tools.checks.source.values import EnumTable ENUMS: Final[EnumTable] = check_language_keys.enum_table() @@ -251,7 +251,7 @@ def test_the_language_file_is_there_to_read(self) -> None: class TestMain: def test_the_repository_and_its_language_file_agree(self) -> None: - assert check_language_keys.main([]) == 0 + assert dispatch(COMMANDS, ["check", "language-keys"]) == 0 def test_a_disagreement_is_reported_by_kind( self, @@ -261,7 +261,9 @@ def test_a_disagreement_is_reported_by_kind( source = source_tree(tmp_path, LOOKUP_SOURCE.format(key=ABSENT_KEY)) entries = language_file(tmp_path, f'{OK_KEY}: "OK"\n') - exit_code = check_language_keys.main(["--source", str(source), "--language-file", str(entries)]) + exit_code = dispatch( + COMMANDS, ["check", "language-keys", "--source", str(source), "--language-file", str(entries)] + ) assert exit_code == 1 error = capsys.readouterr().err diff --git a/tests/unit/scripts/checks/test_palette_colors.py b/tests/unit/sampletones_tools/checks/test_palette_colors.py similarity index 95% rename from tests/unit/scripts/checks/test_palette_colors.py rename to tests/unit/sampletones_tools/checks/test_palette_colors.py index 9e13f2873..c0bc27174 100644 --- a/tests/unit/scripts/checks/test_palette_colors.py +++ b/tests/unit/sampletones_tools/checks/test_palette_colors.py @@ -4,14 +4,12 @@ from pytest import fixture from sampletones_application.paths import PALETTES_DIRECTORY -from sampletones_shared.meta.source.modules import SourceModule, source_paths from sampletones_shared.paths.resources import CONFIG_DIRECTORY -from scripts.checks.palette_colors import dpg_module_helper -from tests.suite.scripts import load_script +from sampletones_tools.checks import palette_colors as check_palette_colors +from sampletones_tools.checks.palette_colors import dpg_module_helper +from sampletones_tools.checks.source.modules import SourceModule, source_paths from tests.suite.source import parse_source -check_palette_colors = load_script("checks/palette_colors.py") - PANEL_MODULE: Final[Path] = Path("ui/panel.py") PANEL_SOURCE: Final[str] = """ diff --git a/tests/unit/scripts/checks/test_rendered_literals.py b/tests/unit/sampletones_tools/checks/test_rendered_literals.py similarity index 91% rename from tests/unit/scripts/checks/test_rendered_literals.py rename to tests/unit/sampletones_tools/checks/test_rendered_literals.py index e38c0c0f0..833df5d15 100644 --- a/tests/unit/scripts/checks/test_rendered_literals.py +++ b/tests/unit/sampletones_tools/checks/test_rendered_literals.py @@ -4,14 +4,14 @@ import pytest -from sampletones_shared.meta.source.modules import SourceModule +from sampletones.commands.registry import COMMANDS +from sampletones.dispatcher import dispatch +from sampletones_tools.checks import rendered_literals as check_rendered_literals +from sampletones_tools.checks.source.modules import SourceModule from tests.suite.base import BaseTestSuite from tests.suite.case import BaseRegularTestCase -from tests.suite.scripts import load_script from tests.suite.source import parse_source -check_rendered_literals = load_script("checks/rendered_literals.py") - CASE_MODULE: Final[Path] = Path("tests/unit/test_card.py") @@ -99,7 +99,7 @@ def test_it_is_passed_over(self, test_case: TestCase) -> None: class TestMain: def test_no_case_in_the_repository_holds_rendered_text_against_a_literal(self) -> None: - assert check_rendered_literals.main([]) == 0 + assert dispatch(COMMANDS, ["check", "rendered-literals"]) == 0 def test_a_case_that_does_is_reported_where_it_sits( self, @@ -114,7 +114,7 @@ def test_a_case_that_does_is_reported_where_it_sits( encoding="utf-8", ) - exit_code = check_rendered_literals.main(["--tests", str(cases)]) + exit_code = dispatch(COMMANDS, ["check", "rendered-literals", "--tests", str(cases)]) assert exit_code == 1 assert f"{module}:2" in capsys.readouterr().err diff --git a/tests/unit/scripts/checks/test_shortcut_actions.py b/tests/unit/sampletones_tools/checks/test_shortcut_actions.py similarity index 96% rename from tests/unit/scripts/checks/test_shortcut_actions.py rename to tests/unit/sampletones_tools/checks/test_shortcut_actions.py index 569d95c5a..1c821a66a 100644 --- a/tests/unit/scripts/checks/test_shortcut_actions.py +++ b/tests/unit/sampletones_tools/checks/test_shortcut_actions.py @@ -7,12 +7,10 @@ ShortcutCategory, ShortcutId, ) -from sampletones_shared.meta.source.modules import SourceModule -from tests.suite.scripts import load_script +from sampletones_tools.checks import shortcut_actions as check_shortcut_actions +from sampletones_tools.checks.source.modules import SourceModule from tests.suite.source import parse_source -check_shortcut_actions = load_script("checks/shortcut_actions.py") - SCHEME: Final[Path] = Path("keybindings/default.yaml") BOUND_ACTION: Final[ShortcutId] = ShortcutId.NEW_PROJECT FAMILY_ACTION: Final[ShortcutId] = ShortcutId.EXPORT_PROJECT_FAMITRACKER diff --git a/tests/unit/scripts/checks/test_tag_names.py b/tests/unit/sampletones_tools/checks/test_tag_names.py similarity index 93% rename from tests/unit/scripts/checks/test_tag_names.py rename to tests/unit/sampletones_tools/checks/test_tag_names.py index 3b03f4fcb..a9b7d51b4 100644 --- a/tests/unit/scripts/checks/test_tag_names.py +++ b/tests/unit/sampletones_tools/checks/test_tag_names.py @@ -4,17 +4,17 @@ import pytest +from sampletones.commands.registry import COMMANDS +from sampletones.dispatcher import dispatch from sampletones_application.categories.hierarchy import Page, Panel, Widget from sampletones_application.categories.key.tag import TagName -from sampletones_shared.meta.source.modules import SourceModule, source_paths from sampletones_shared.paths.source import SOURCE_ROOT +from sampletones_tools.checks import tag_names as check_tag_names +from sampletones_tools.checks.source.modules import SourceModule, source_paths from tests.suite.base import BaseTestSuite from tests.suite.case import BaseRegularTestCase -from tests.suite.scripts import load_script from tests.suite.source import parse_source -check_tag_names = load_script("checks/tag_names.py") - MODULE_PATH: Final[Path] = Path("src/sampletones_application/tags/general.py") WINDOW_TAG: Final[str] = 'TagName(Page.GLOBAL, Panel.IMPLICIT, Widget.WINDOW, "main")' @@ -157,7 +157,7 @@ def test_the_tags_package_sits_under_the_source_root(self) -> None: class TestMain: def test_the_tags_package_names_its_tags_after_them(self) -> None: - assert check_tag_names.main(["--all"]) == 0 + assert dispatch(COMMANDS, ["check", "tag-names", "--all"]) == 0 def test_a_mistyped_tag_is_reported_where_it_sits( self, @@ -167,7 +167,7 @@ def test_a_mistyped_tag_is_reported_where_it_sits( module = tmp_path / "tags.py" module.write_text(f"TAG_GLOBAL_WINDOW_MIAN = {WINDOW_TAG}\n", encoding="utf-8") - assert check_tag_names.main([str(module)]) == 1 + assert dispatch(COMMANDS, ["check", "tag-names", str(module)]) == 1 error = capsys.readouterr().err assert f"{module}:1" in error @@ -181,5 +181,5 @@ def test_a_well_named_module_reports_nothing( module = tmp_path / "tags.py" module.write_text(f"TAG_GLOBAL_WINDOW_MAIN = {WINDOW_TAG}\n", encoding="utf-8") - assert check_tag_names.main([str(module)]) == 0 + assert dispatch(COMMANDS, ["check", "tag-names", str(module)]) == 0 assert capsys.readouterr().err == "" diff --git a/tests/unit/scripts/checks/test_unused_tags.py b/tests/unit/sampletones_tools/checks/test_unused_tags.py similarity index 91% rename from tests/unit/scripts/checks/test_unused_tags.py rename to tests/unit/sampletones_tools/checks/test_unused_tags.py index 1a6b033f5..633fa86a8 100644 --- a/tests/unit/scripts/checks/test_unused_tags.py +++ b/tests/unit/sampletones_tools/checks/test_unused_tags.py @@ -3,13 +3,13 @@ import pytest -from sampletones_shared.meta.source.modules import SourceModule, source_paths +from sampletones.commands.registry import COMMANDS +from sampletones.dispatcher import dispatch from sampletones_shared.paths.source import SOURCE_ROOT -from tests.suite.scripts import load_script +from sampletones_tools.checks import unused_tags as check_unused_tags +from sampletones_tools.checks.source.modules import SourceModule, source_paths from tests.suite.source import parse_source -check_unused_tags = load_script("checks/unused_tags.py") - TAGS_MODULE: Final[Path] = Path("tags/general.py") PANEL_MODULE: Final[Path] = Path("ui/panel.py") @@ -102,7 +102,7 @@ def test_the_tags_package_sits_under_the_source_root(self) -> None: class TestMain: def test_the_repository_reads_every_fragment_it_declares(self) -> None: - assert check_unused_tags.main([]) == 0 + assert dispatch(COMMANDS, ["check", "unused-tags"]) == 0 def test_an_unread_fragment_is_reported_where_it_sits( self, @@ -116,7 +116,7 @@ def test_an_unread_fragment_is_reported_where_it_sits( sources.mkdir() (sources / "panel.py").write_text("show(TAG_READ)\n", encoding="utf-8") - exit_code = check_unused_tags.main(["--tags", str(tags), "--reference-root", str(sources)]) + exit_code = dispatch(COMMANDS, ["check", "unused-tags", "--tags", str(tags), "--reference-root", str(sources)]) assert exit_code == 1 error = capsys.readouterr().err From bbfc3cccd5d61d371a1331ff34595aaf22097aba Mon Sep 17 00:00:00 2001 From: JakimPL Date: Sun, 13 Sep 2026 16:26:09 +0200 Subject: [PATCH 15/36] Moved: the codec study into the tools package --- Makefile | 6 +- docs/development/player.md | 6 +- docs/development/tooling.md | 6 +- pyproject.toml | 1 - scripts/compression_study.py | 140 ------------------ .../boundaries/standalone.yaml | 3 - src/sampletones_config/boundaries/tokens.yaml | 21 --- .../sampletones_tools/codec}/__init__.py | 0 src/sampletones_tools/codec/command.py | 88 +++++++++++ .../codec/study}/__init__.py | 0 .../codec/study/accounting}/__init__.py | 0 .../codec/study}/accounting/coincident.py | 4 +- .../codec/study}/accounting/dictionary.py | 6 +- .../codec/study}/accounting/finding.py | 0 .../codec/study}/accounting/fixed.py | 4 +- .../codec/study}/accounting/pairs.py | 6 +- .../codec/study}/accounting/rows.py | 16 +- .../codec/study}/accounting/runs.py | 0 .../codec/study}/accounting/shares.py | 4 +- .../codec/study}/accounting/tokens.py | 0 .../codec/study/corpus}/__init__.py | 0 .../codec/study}/corpus/build.py | 8 +- .../codec/study}/corpus/projects.py | 2 +- .../codec/study}/corpus/reconstructions.py | 2 +- .../codec/study}/corpus/song.py | 0 .../codec/study}/manifest.py | 0 .../sampletones_tools/codec/study}/measure.py | 2 +- .../codec/study/report}/__init__.py | 0 .../codec/study}/report/aggregate.py | 4 +- .../codec/study}/report/rows.py | 2 +- .../codec/study}/report/run.py | 18 +-- .../codec/study}/report/verdicts.py | 8 +- .../codec/study}/report/writers.py | 0 .../codec/study/sandbox}/__init__.py | 0 .../codec/study}/sandbox/context.py | 2 +- .../codec/study}/sandbox/costs.py | 0 .../codec/study}/sandbox/decode.py | 2 +- .../codec/study}/sandbox/defaults.py | 4 +- .../codec/study/sandbox/edges}/__init__.py | 0 .../codec/study}/sandbox/edges/generator.py | 6 +- .../codec/study}/sandbox/edges/holds.py | 8 +- .../codec/study}/sandbox/edges/literals.py | 6 +- .../codec/study}/sandbox/edges/phrases.py | 8 +- .../codec/study}/sandbox/edges/set_hold.py | 8 +- .../codec/study}/sandbox/encode.py | 12 +- .../codec/study}/sandbox/grammar.py | 2 +- .../codec/study}/sandbox/parse.py | 18 +-- .../codec/study}/sandbox/reference.py | 14 +- .../codec/study}/sandbox/shortest.py | 0 .../codec/study}/sandbox/tokens.py | 0 .../codec/study}/sandbox/verify.py | 4 +- src/sampletones_tools/codec/study/session.py | 109 ++++++++++++++ .../codec/study/variants/__init__.py | 0 .../codec/study}/variants/baselines.py | 10 +- .../codec/study}/variants/production.py | 8 +- .../codec/study}/variants/registry.py | 8 +- .../codec/study}/variants/sandbox.py | 14 +- .../codec/study}/variants/seeds.py | 0 .../codec/study}/variants/strategy.py | 2 +- .../codec/study}/variants/variant.py | 4 +- src/sampletones_tools/registry.py | 3 +- .../codec/study}/test_accounting.py | 14 +- .../codec/study}/test_sandbox.py | 28 ++-- .../codec/study/test_session.py | 43 ++++++ .../codec/study}/test_variants.py | 8 +- .../codec/study}/test_verdicts.py | 10 +- .../sampletones_tools/codec/test_command.py | 70 +++++++++ 67 files changed, 460 insertions(+), 322 deletions(-) delete mode 100644 scripts/compression_study.py rename {scripts/codec_study => src/sampletones_tools/codec}/__init__.py (100%) create mode 100644 src/sampletones_tools/codec/command.py rename {scripts/codec_study/accounting => src/sampletones_tools/codec/study}/__init__.py (100%) rename {scripts/codec_study/corpus => src/sampletones_tools/codec/study/accounting}/__init__.py (100%) rename {scripts/codec_study => src/sampletones_tools/codec/study}/accounting/coincident.py (90%) rename {scripts/codec_study => src/sampletones_tools/codec/study}/accounting/dictionary.py (91%) rename {scripts/codec_study => src/sampletones_tools/codec/study}/accounting/finding.py (100%) rename {scripts/codec_study => src/sampletones_tools/codec/study}/accounting/fixed.py (94%) rename {scripts/codec_study => src/sampletones_tools/codec/study}/accounting/pairs.py (90%) rename {scripts/codec_study => src/sampletones_tools/codec/study}/accounting/rows.py (87%) rename {scripts/codec_study => src/sampletones_tools/codec/study}/accounting/runs.py (100%) rename {scripts/codec_study => src/sampletones_tools/codec/study}/accounting/shares.py (96%) rename {scripts/codec_study => src/sampletones_tools/codec/study}/accounting/tokens.py (100%) rename {scripts/codec_study/report => src/sampletones_tools/codec/study/corpus}/__init__.py (100%) rename {scripts/codec_study => src/sampletones_tools/codec/study}/corpus/build.py (78%) rename {scripts/codec_study => src/sampletones_tools/codec/study}/corpus/projects.py (97%) rename {scripts/codec_study => src/sampletones_tools/codec/study}/corpus/reconstructions.py (96%) rename {scripts/codec_study => src/sampletones_tools/codec/study}/corpus/song.py (100%) rename {scripts/codec_study => src/sampletones_tools/codec/study}/manifest.py (100%) rename {scripts/codec_study => src/sampletones_tools/codec/study}/measure.py (98%) rename {scripts/codec_study/sandbox => src/sampletones_tools/codec/study/report}/__init__.py (100%) rename {scripts/codec_study => src/sampletones_tools/codec/study}/report/aggregate.py (96%) rename {scripts/codec_study => src/sampletones_tools/codec/study}/report/rows.py (97%) rename {scripts/codec_study => src/sampletones_tools/codec/study}/report/run.py (90%) rename {scripts/codec_study => src/sampletones_tools/codec/study}/report/verdicts.py (95%) rename {scripts/codec_study => src/sampletones_tools/codec/study}/report/writers.py (100%) rename {scripts/codec_study/sandbox/edges => src/sampletones_tools/codec/study/sandbox}/__init__.py (100%) rename {scripts/codec_study => src/sampletones_tools/codec/study}/sandbox/context.py (96%) rename {scripts/codec_study => src/sampletones_tools/codec/study}/sandbox/costs.py (100%) rename {scripts/codec_study => src/sampletones_tools/codec/study}/sandbox/decode.py (93%) rename {scripts/codec_study => src/sampletones_tools/codec/study}/sandbox/defaults.py (90%) rename {scripts/codec_study/variants => src/sampletones_tools/codec/study/sandbox/edges}/__init__.py (100%) rename {scripts/codec_study => src/sampletones_tools/codec/study}/sandbox/edges/generator.py (87%) rename {scripts/codec_study => src/sampletones_tools/codec/study}/sandbox/edges/holds.py (88%) rename {scripts/codec_study => src/sampletones_tools/codec/study}/sandbox/edges/literals.py (83%) rename {scripts/codec_study => src/sampletones_tools/codec/study}/sandbox/edges/phrases.py (86%) rename {scripts/codec_study => src/sampletones_tools/codec/study}/sandbox/edges/set_hold.py (80%) rename {scripts/codec_study => src/sampletones_tools/codec/study}/sandbox/encode.py (84%) rename {scripts/codec_study => src/sampletones_tools/codec/study}/sandbox/grammar.py (93%) rename {scripts/codec_study => src/sampletones_tools/codec/study}/sandbox/parse.py (83%) rename {scripts/codec_study => src/sampletones_tools/codec/study}/sandbox/reference.py (87%) rename {scripts/codec_study => src/sampletones_tools/codec/study}/sandbox/shortest.py (100%) rename {scripts/codec_study => src/sampletones_tools/codec/study}/sandbox/tokens.py (100%) rename {scripts/codec_study => src/sampletones_tools/codec/study}/sandbox/verify.py (93%) create mode 100644 src/sampletones_tools/codec/study/session.py create mode 100644 src/sampletones_tools/codec/study/variants/__init__.py rename {scripts/codec_study => src/sampletones_tools/codec/study}/variants/baselines.py (87%) rename {scripts/codec_study => src/sampletones_tools/codec/study}/variants/production.py (94%) rename {scripts/codec_study => src/sampletones_tools/codec/study}/variants/registry.py (82%) rename {scripts/codec_study => src/sampletones_tools/codec/study}/variants/sandbox.py (91%) rename {scripts/codec_study => src/sampletones_tools/codec/study}/variants/seeds.py (100%) rename {scripts/codec_study => src/sampletones_tools/codec/study}/variants/strategy.py (97%) rename {scripts/codec_study => src/sampletones_tools/codec/study}/variants/variant.py (92%) rename tests/unit/{scripts/codec_study => sampletones_tools/codec/study}/test_accounting.py (94%) rename tests/unit/{scripts/codec_study => sampletones_tools/codec/study}/test_sandbox.py (92%) create mode 100644 tests/unit/sampletones_tools/codec/study/test_session.py rename tests/unit/{scripts/codec_study => sampletones_tools/codec/study}/test_variants.py (94%) rename tests/unit/{scripts/codec_study => sampletones_tools/codec/study}/test_verdicts.py (94%) create mode 100644 tests/unit/sampletones_tools/codec/test_command.py diff --git a/Makefile b/Makefile index fdda1aa24..08c93824d 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ .PHONY: help setup install system-deps build release run clean pre-commit test benchmarks lint format \ - ftm-samples nsf-samples compression-report compression-study + ftm-samples nsf-samples compression-report ifeq ($(OS),Windows_NT) ifeq ($(MSYSTEM),) @@ -33,7 +33,6 @@ help: @echo $(Q) make ftm-samples - Emit example .ftm files to build/ftm via the integration suite$(Q) @echo $(Q) make nsf-samples - Emit example .nsf files to build/nsf via the integration suite$(Q) @echo $(Q) make compression-report - Measure the song codec into build/compression$(Q) - @echo $(Q) make compression-study - Measure the song codec over the projects and stems on this machine; the report lands in Documents/SampleToNES/compression (ARGS=--quick for a short run)$(Q) @echo $(Q) make clean - Remove build artifacts and cache files$(Q) @echo $(Q) make lint - Run mypy and pylint (ARGS=--mypy or ARGS=--pylint for one of them)$(Q) @echo $(Q) make format - Auto-format code (isort, black)$(Q) @@ -87,6 +86,3 @@ nsf-samples: compression-report: export SAMPLETONES_COMPRESSION_OUTPUT_DIR := build/compression compression-report: uv run python -m pytest tests/integration/nsf/test_compression_report.py - -compression-study: - uv run scripts/compression_study.py $(ARGS) diff --git a/docs/development/player.md b/docs/development/player.md index f8dd8bff3..10f0071ea 100644 --- a/docs/development/player.md +++ b/docs/development/player.md @@ -41,7 +41,7 @@ what the samples leave uncovered. off on its own, and `make compression-report` writes what each one saves across a corpus of songs. The format's constants are settled from that report rather than from argument. -**A change to the codec is measured before it is built.** `make compression-study` reads the +**A change to the codec is measured before it is built.** `uv run sampletones codec study` reads the projects and stems on this machine, encodes every song under every candidate change, and writes the sizes, the times and a verdict per candidate under `Documents/SampleToNES/compression`. A candidate is one of two things. A new way of choosing @@ -49,7 +49,7 @@ tokens is encoded and played back by the production codec itself. A new token gr priced in bytes by a study parser, which first has to reproduce the production parser's bytes on today's grammar. The rule is printed in the report: a candidate earns a production layer when it saves 3% over the projects or 5% over the reconstructions and grows no song by -more than 1%. The study lives under `scripts/codec_study` and stays out of the shipped +more than 1%. The study lives under `sampletones_tools/codec/study`, outside the shipped packages. ## The song a file carries @@ -152,7 +152,7 @@ The chain runs from the register values upward, and each link is held on its own | The codec is lossless | every encoding decodes to the planes it was written from, over a corpus | | The codec is safe | a plane the codec finds nothing in stays within its literal bound | | The ratio | `make compression-report` — bytes per tick and ticks that fit, per layer | -| What a change would save | `make compression-study` — the songs on this machine under every candidate change, with a verdict each | +| What a change would save | `uv run sampletones codec study` — the songs on this machine under every candidate change, with a verdict each | | The byte layout | a hand-built song serializes to expected bytes | | The assembly agrees with the specification | the include's equates are read and compared field by field | | The driver behaves | the assembled image on a 6502 emulator against `RegisterTrace.from_song`, over several rates and over songs that repeat | diff --git a/docs/development/tooling.md b/docs/development/tooling.md index 2e10c2f17..3e8a14e5c 100644 --- a/docs/development/tooling.md +++ b/docs/development/tooling.md @@ -83,6 +83,7 @@ The developer commands, listed by `sampletones_tools/registry.py` and run as |---|---| | `calibration [--config FILE] [-o DIR] [--methods LIST] [--perceptual-exponents LIST] [--temporal-weights LIST] [--channels LIST]` | Reconstructs the calibration corpus under every variant of the sweep, scores it with every referee, and writes the reports; without `-o` the run lands in a timestamped directory under Documents/SampleToNES/calibration | | `check [options]` | Holds the tree to one of its checks: `import-boundary`, `language-keys`, `palette-colors`, `rendered-literals`, `shortcut-actions`, `tag-names`, `unused-tags`; each is a pre-commit hook, and [architecture](architecture.md#enforcement) says what each holds. Needs a checkout | +| `codec study [--manifest FILE] [--project FILE]... [--reconstruction PATH]... [-o DIR] [--lengthen SECONDS] [--variants LIST] [--quick]` | Encodes the projects and stems on this machine under every candidate change to the codec and writes the sizes, the times and a verdict per candidate; without `-o` the run lands under Documents/SampleToNES/compression | | `driver [--directory DIR]` | Assembles the NES player driver with cc65 and prints the layout the build produced; without `--directory` it writes the driver the package ships, which needs a checkout | | `icons [--directory DIR]` | Writes the icon suite from the mark; without `--directory` it writes the icons the package ships, which needs a checkout | | `nsf render --directory DIR [--tail SECONDS]` | Renders every exported `.nsf` file in the directory to a wave beside it, through ffmpeg's libgme demuxer | @@ -139,11 +140,6 @@ interpreter version check (`interpreter.py`), running a command and holding it t build environment and the installs into it (`venv_build.py`), the preflight of the build interpreter (`preflight.py`), and the platforms (`platforms/`). -## The tool scripts - -`compression_study.py` imports the project's packages and runs inside its environment, from the -make target that names it. - ## Who governs what | Concern | Owner | diff --git a/pyproject.toml b/pyproject.toml index 8abe2100c..5912fdbc7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -120,7 +120,6 @@ line_length = 120 known_first_party = [ "bootstrap", "ci", - "codec_study", "sampletones", "sampletones_application", "sampletones_assets", diff --git a/scripts/compression_study.py b/scripts/compression_study.py deleted file mode 100644 index 350e35f59..000000000 --- a/scripts/compression_study.py +++ /dev/null @@ -1,140 +0,0 @@ -import argparse -from pathlib import Path -from typing import Final, List, Optional, Tuple - -from codec_study.corpus.build import build_corpus -from codec_study.manifest import StudyManifest, StudySource -from codec_study.measure import Measurement, measure -from codec_study.report.run import run_directory, write_run -from codec_study.variants.baselines import Baselines -from codec_study.variants.registry import EVERY_VARIANT, selected_variants -from codec_study.variants.strategy import STRATEGY_ORDER, depth_measurements -from sampletones_shared.logger import logger - -DEFAULT_LENGTHEN_SECONDS: Final[int] = 180 -DEFAULT_VARIANTS: Final[str] = EVERY_VARIANT - - -def main() -> None: - parser = argparse.ArgumentParser( - description="Measure the song codec over the projects and stems on this machine.", - ) - parser.add_argument( - "--manifest", - type=Path, - default=None, - help="A manifest a run wrote; its lengthening and variants stand in for the options below.", - ) - parser.add_argument( - "--project", - type=Path, - action="append", - default=[], - help="A project file to measure, in place of the default corpus; repeatable.", - ) - parser.add_argument( - "--reconstruction", - type=Path, - action="append", - default=[], - help="A stem file, or a directory of stems, in place of the default corpus; repeatable.", - ) - parser.add_argument( - "--output", - type=Path, - default=None, - help="Output directory of the run.", - ) - parser.add_argument( - "--lengthen", - type=int, - default=DEFAULT_LENGTHEN_SECONDS, - help="Seconds each project's lengthened copy lasts.", - ) - parser.add_argument( - "--variants", - type=str, - default=DEFAULT_VARIANTS, - help="Comma-separated variants every song is encoded under, or all; the baseline always runs.", - ) - parser.add_argument( - "--quick", - action="store_true", - help="Read one small project and one stem, to check the harness.", - ) - arguments = parser.parse_args() - - manifest = _manifest( - arguments.manifest, - projects=arguments.project, - reconstructions=arguments.reconstruction, - lengthen_seconds=arguments.lengthen, - variants=_names(arguments.variants), - quick=arguments.quick, - ) - variants = selected_variants(manifest.variants, Baselines()) - directory = run_directory(arguments.output) - corpus = build_corpus(manifest) - - measurements: List[Measurement] = [] - for song in corpus: - for variant in variants: - if not variant.applies(song): - continue - - logger.info(f"Encoding {song.name} ({song.ticks} ticks) under {variant.name}") - measurement = measure(song, variant.name, variant.encode) - if not measurement.lossless: - raise ValueError(f"{variant.name} wrote {song.name} as streams that play back differently") - - logger.info( - f" {measurement.block} bytes, {measurement.bytes_per_tick:.3f} bytes per tick, " - f"{measurement.phrases} phrases, {measurement.seconds:.1f} s" - ) - measurements.append(measurement) - - write_run( - directory, - manifest, - variants, - measurements, - depth_measurements(measurements, STRATEGY_ORDER), - ) - logger.info(f"Report written to {directory}") - - -def _names(variants: str) -> Tuple[str, ...]: - return tuple(name.strip() for name in variants.split(",") if name.strip()) - - -def _manifest( - path: Optional[Path], - *, - projects: List[Path], - reconstructions: List[Path], - lengthen_seconds: int, - variants: Tuple[str, ...], - quick: bool, -) -> StudyManifest: - manifest = ( - StudyManifest.load(path) - if path is not None - else StudyManifest.default( - lengthen_seconds=lengthen_seconds, - variants=variants, - quick=quick, - ) - ) - if not projects and not reconstructions: - return manifest - - return StudyManifest( - projects=tuple(StudySource.at(project) for project in projects), - reconstructions=tuple(StudySource.at(reconstruction) for reconstruction in reconstructions), - lengthen_seconds=manifest.lengthen_seconds, - variants=manifest.variants, - ) - - -if __name__ == "__main__": - main() diff --git a/src/sampletones_config/boundaries/standalone.yaml b/src/sampletones_config/boundaries/standalone.yaml index 2f485b236..599690650 100644 --- a/src/sampletones_config/boundaries/standalone.yaml +++ b/src/sampletones_config/boundaries/standalone.yaml @@ -1,7 +1,4 @@ - pattern: "**/*.py" - excluding: - - "compression_study.py" - - "codec_study/**/*.py" reserved: [build, test, tests] message: >- a bootstrap script runs on the system interpreter, so it imports the standard library and diff --git a/src/sampletones_config/boundaries/tokens.yaml b/src/sampletones_config/boundaries/tokens.yaml index 0f1b0f31f..0eeed6346 100644 --- a/src/sampletones_config/boundaries/tokens.yaml +++ b/src/sampletones_config/boundaries/tokens.yaml @@ -20,27 +20,6 @@ only the layout primitives own depth (TabColumns binds the column, card() binds the card), and a panel binds only semantic themes -- root: sampletones_core - pattern: "**/*.py" - forbidden: '\bcodec_study\b' - message: >- - sampletones_core is shipped code; the compression study harness (scripts/codec_study) stays - outside it, and a finding it earns lands as production code of its own - -- root: sampletones_player - pattern: "**/*.py" - forbidden: '\bcodec_study\b' - message: >- - sampletones_player is shipped code; the compression study harness (scripts/codec_study) stays - outside it, and a finding it earns lands as production code of its own - -- root: sampletones_shared - pattern: "**/*.py" - forbidden: '\bcodec_study\b' - message: >- - sampletones_shared is shipped code; the compression study harness (scripts/codec_study) stays - outside it, and a finding it earns lands as production code of its own - - root: sampletones_application pattern: "**/*.py" forbidden: '\bsampletones_tools\b' diff --git a/scripts/codec_study/__init__.py b/src/sampletones_tools/codec/__init__.py similarity index 100% rename from scripts/codec_study/__init__.py rename to src/sampletones_tools/codec/__init__.py diff --git a/src/sampletones_tools/codec/command.py b/src/sampletones_tools/codec/command.py new file mode 100644 index 000000000..abfa6fb3d --- /dev/null +++ b/src/sampletones_tools/codec/command.py @@ -0,0 +1,88 @@ +from argparse import ArgumentParser, Namespace +from dataclasses import dataclass +from pathlib import Path +from typing import Final, Optional, Tuple + +from sampletones_shared.command import Command + +NAME: Final[str] = "codec" +HELP: Final[str] = "measure the song codec" +ACTION_FIELD: Final[str] = "action" +ACTION_METAVAR: Final[str] = "" +STUDY: Final[str] = "study" +STUDY_HELP: Final[str] = ( + "encode the projects and stems on this machine under every candidate change, with a verdict each" +) +MANIFEST_HELP: Final[str] = "a manifest a run wrote; its lengthening and variants stand in for the options below" +PROJECT_HELP: Final[str] = "a project file to measure in place of the corpus, repeatable" +RECONSTRUCTION_HELP: Final[str] = "a stem file, or a directory of stems, to measure in place of the corpus, repeatable" +OUTPUT_HELP: Final[str] = ( + "the directory the run writes into; without it, a timestamped directory under Documents/SampleToNES/compression" +) +LENGTHEN_HELP: Final[str] = "seconds each project's lengthened copy lasts" +VARIANTS_HELP: Final[str] = ( + "variants every song is encoded under, comma separated; without it every one, and the baseline always runs" +) +QUICK_HELP: Final[str] = "read one small project and one stem, to check the harness" +DEFAULT_LENGTHEN_SECONDS: Final[int] = 180 + + +@dataclass(frozen=True) +class StudyArguments: + """What a study run is given, as written on the command line.""" + + manifest: Optional[Path] + projects: Tuple[Path, ...] + reconstructions: Tuple[Path, ...] + output: Optional[Path] + lengthen: int + variants: Optional[str] + quick: bool + + +def configure(parser: ArgumentParser) -> None: + actions = parser.add_subparsers(dest=ACTION_FIELD, metavar=ACTION_METAVAR, required=True) + study = actions.add_parser(STUDY, help=STUDY_HELP, description=STUDY_HELP) + study.add_argument("--manifest", type=Path, default=None, help=MANIFEST_HELP) + study.add_argument("--project", type=Path, action="append", dest="projects", default=[], help=PROJECT_HELP) + study.add_argument( + "--reconstruction", + type=Path, + action="append", + dest="reconstructions", + default=[], + help=RECONSTRUCTION_HELP, + ) + study.add_argument("--output", "-o", type=Path, default=None, help=OUTPUT_HELP) + study.add_argument("--lengthen", type=int, default=DEFAULT_LENGTHEN_SECONDS, help=LENGTHEN_HELP) + study.add_argument("--variants", type=str, default=None, help=VARIANTS_HELP) + study.add_argument("--quick", action="store_true", help=QUICK_HELP) + + +def run(arguments: Namespace) -> int: + """Measures the codec the way the action describes.""" + given = StudyArguments( + manifest=arguments.manifest, + projects=tuple(arguments.projects), + reconstructions=tuple(arguments.reconstructions), + output=arguments.output, + lengthen=arguments.lengthen, + variants=arguments.variants, + quick=arguments.quick, + ) + + from sampletones_tools.codec.study.session import resolve_manifest, run_study, variant_names + + manifest = resolve_manifest( + given.manifest, + projects=given.projects, + reconstructions=given.reconstructions, + lengthen_seconds=given.lengthen, + variants=variant_names(given.variants), + quick=given.quick, + ) + run_study(manifest, given.output) + return 0 + + +CODEC: Final[Command] = Command(name=NAME, help=HELP, configure=configure, run=run) diff --git a/scripts/codec_study/accounting/__init__.py b/src/sampletones_tools/codec/study/__init__.py similarity index 100% rename from scripts/codec_study/accounting/__init__.py rename to src/sampletones_tools/codec/study/__init__.py diff --git a/scripts/codec_study/corpus/__init__.py b/src/sampletones_tools/codec/study/accounting/__init__.py similarity index 100% rename from scripts/codec_study/corpus/__init__.py rename to src/sampletones_tools/codec/study/accounting/__init__.py diff --git a/scripts/codec_study/accounting/coincident.py b/src/sampletones_tools/codec/study/accounting/coincident.py similarity index 90% rename from scripts/codec_study/accounting/coincident.py rename to src/sampletones_tools/codec/study/accounting/coincident.py index bf4414bc7..408058e11 100644 --- a/scripts/codec_study/accounting/coincident.py +++ b/src/sampletones_tools/codec/study/accounting/coincident.py @@ -1,9 +1,9 @@ from typing import Final, Mapping, Sequence -from codec_study.accounting.finding import NOTHING, Finding -from codec_study.accounting.tokens import ReadToken from sampletones_player.compression.planes.order import PlaneOrder from sampletones_player.specification.compression import OPCODE_SIZE +from sampletones_tools.codec.study.accounting.finding import NOTHING, Finding +from sampletones_tools.codec.study.accounting.tokens import ReadToken CONTROL_SUFFIX: Final[str] = "_control" VALUE_SUFFIX: Final[str] = "_value" diff --git a/scripts/codec_study/accounting/dictionary.py b/src/sampletones_tools/codec/study/accounting/dictionary.py similarity index 91% rename from scripts/codec_study/accounting/dictionary.py rename to src/sampletones_tools/codec/study/accounting/dictionary.py index af2bb2767..1307bd4b2 100644 --- a/scripts/codec_study/accounting/dictionary.py +++ b/src/sampletones_tools/codec/study/accounting/dictionary.py @@ -1,11 +1,11 @@ from collections import Counter from typing import Dict, Final, Iterable -from codec_study.accounting.finding import NOTHING, Finding -from codec_study.accounting.runs import runs -from codec_study.accounting.tokens import ReadToken from sampletones_player.compression.dictionary.table import PhraseTable from sampletones_player.specification.compression import PHRASE_COUNT_SIZE +from sampletones_tools.codec.study.accounting.finding import NOTHING, Finding +from sampletones_tools.codec.study.accounting.runs import runs +from sampletones_tools.codec.study.accounting.tokens import ReadToken REPEATED: Final[int] = 2 MIN_RLE_RUN: Final[int] = 3 diff --git a/scripts/codec_study/accounting/finding.py b/src/sampletones_tools/codec/study/accounting/finding.py similarity index 100% rename from scripts/codec_study/accounting/finding.py rename to src/sampletones_tools/codec/study/accounting/finding.py diff --git a/scripts/codec_study/accounting/fixed.py b/src/sampletones_tools/codec/study/accounting/fixed.py similarity index 94% rename from scripts/codec_study/accounting/fixed.py rename to src/sampletones_tools/codec/study/accounting/fixed.py index 57404cc60..ea098b36f 100644 --- a/scripts/codec_study/accounting/fixed.py +++ b/src/sampletones_tools/codec/study/accounting/fixed.py @@ -1,8 +1,6 @@ from dataclasses import dataclass from typing import Final -from codec_study.accounting.finding import Finding -from codec_study.corpus.song import StudySong from sampletones_player.compression.compressed import CompressedPlanes from sampletones_player.specification.binary import WORD_SIZE from sampletones_player.specification.compression import ( @@ -10,6 +8,8 @@ PLANE_COUNT, ) from sampletones_player.specification.song import SONG_HEADER_SIZE +from sampletones_tools.codec.study.accounting.finding import Finding +from sampletones_tools.codec.study.corpus.song import StudySong TIMER_SIZE: Final[int] = WORD_SIZE diff --git a/scripts/codec_study/accounting/pairs.py b/src/sampletones_tools/codec/study/accounting/pairs.py similarity index 90% rename from scripts/codec_study/accounting/pairs.py rename to src/sampletones_tools/codec/study/accounting/pairs.py index fd9becc65..03918d148 100644 --- a/scripts/codec_study/accounting/pairs.py +++ b/src/sampletones_tools/codec/study/accounting/pairs.py @@ -1,9 +1,9 @@ from typing import Final, Sequence -from codec_study.accounting.finding import NOTHING, Finding -from codec_study.accounting.runs import ramps -from codec_study.accounting.tokens import ReadToken from sampletones_player.specification.compression import TokenTag +from sampletones_tools.codec.study.accounting.finding import NOTHING, Finding +from sampletones_tools.codec.study.accounting.runs import ramps +from sampletones_tools.codec.study.accounting.tokens import ReadToken SET_HOLD_SIZE: Final[int] = 2 SINGLE_VALUE: Final[int] = 1 diff --git a/scripts/codec_study/accounting/rows.py b/src/sampletones_tools/codec/study/accounting/rows.py similarity index 87% rename from scripts/codec_study/accounting/rows.py rename to src/sampletones_tools/codec/study/accounting/rows.py index aed9bbcaa..ff0e9c832 100644 --- a/scripts/codec_study/accounting/rows.py +++ b/src/sampletones_tools/codec/study/accounting/rows.py @@ -1,16 +1,16 @@ from dataclasses import dataclass from typing import Dict, Final, List, Tuple -from codec_study.accounting.coincident import coincident_starts -from codec_study.accounting.dictionary import default_counts, plateaus -from codec_study.accounting.finding import NOTHING, Finding -from codec_study.accounting.fixed import fixed_overheads -from codec_study.accounting.pairs import ramps_in_literals, set_holds -from codec_study.accounting.shares import PlaneShares, plane_shares -from codec_study.accounting.tokens import ReadToken, read_tokens -from codec_study.measure import Measurement from sampletones_player.compression.compressed import CompressedPlanes from sampletones_player.compression.planes.order import PlaneOrder +from sampletones_tools.codec.study.accounting.coincident import coincident_starts +from sampletones_tools.codec.study.accounting.dictionary import default_counts, plateaus +from sampletones_tools.codec.study.accounting.finding import NOTHING, Finding +from sampletones_tools.codec.study.accounting.fixed import fixed_overheads +from sampletones_tools.codec.study.accounting.pairs import ramps_in_literals, set_holds +from sampletones_tools.codec.study.accounting.shares import PlaneShares, plane_shares +from sampletones_tools.codec.study.accounting.tokens import ReadToken, read_tokens +from sampletones_tools.codec.study.measure import Measurement HYPOTHESES: Final[Tuple[Tuple[str, str], ...]] = ( ("H1", "hold chains"), diff --git a/scripts/codec_study/accounting/runs.py b/src/sampletones_tools/codec/study/accounting/runs.py similarity index 100% rename from scripts/codec_study/accounting/runs.py rename to src/sampletones_tools/codec/study/accounting/runs.py diff --git a/scripts/codec_study/accounting/shares.py b/src/sampletones_tools/codec/study/accounting/shares.py similarity index 96% rename from scripts/codec_study/accounting/shares.py rename to src/sampletones_tools/codec/study/accounting/shares.py index dc514f774..9f722363a 100644 --- a/scripts/codec_study/accounting/shares.py +++ b/src/sampletones_tools/codec/study/accounting/shares.py @@ -2,13 +2,13 @@ from math import ceil from typing import Final, List, Sequence -from codec_study.accounting.finding import NOTHING, Finding -from codec_study.accounting.tokens import ReadToken from sampletones_player.specification.compression import ( MAX_HOLD_TICKS, OPCODE_SIZE, TokenTag, ) +from sampletones_tools.codec.study.accounting.finding import NOTHING, Finding +from sampletones_tools.codec.study.accounting.tokens import ReadToken WIDE_HOLD_SIZE: Final[int] = 2 WIDE_HOLD_UNITS: Final[int] = MAX_HOLD_TICKS diff --git a/scripts/codec_study/accounting/tokens.py b/src/sampletones_tools/codec/study/accounting/tokens.py similarity index 100% rename from scripts/codec_study/accounting/tokens.py rename to src/sampletones_tools/codec/study/accounting/tokens.py diff --git a/scripts/codec_study/report/__init__.py b/src/sampletones_tools/codec/study/corpus/__init__.py similarity index 100% rename from scripts/codec_study/report/__init__.py rename to src/sampletones_tools/codec/study/corpus/__init__.py diff --git a/scripts/codec_study/corpus/build.py b/src/sampletones_tools/codec/study/corpus/build.py similarity index 78% rename from scripts/codec_study/corpus/build.py rename to src/sampletones_tools/codec/study/corpus/build.py index cf7220479..aec03a639 100644 --- a/scripts/codec_study/corpus/build.py +++ b/src/sampletones_tools/codec/study/corpus/build.py @@ -1,10 +1,10 @@ from typing import List, Tuple -from codec_study.corpus.projects import lengthened_song, project_song -from codec_study.corpus.reconstructions import reconstruction_paths, reconstruction_song -from codec_study.corpus.song import StudySong -from codec_study.manifest import StudyManifest, StudySource from sampletones_shared.logger import logger +from sampletones_tools.codec.study.corpus.projects import lengthened_song, project_song +from sampletones_tools.codec.study.corpus.reconstructions import reconstruction_paths, reconstruction_song +from sampletones_tools.codec.study.corpus.song import StudySong +from sampletones_tools.codec.study.manifest import StudyManifest, StudySource def build_corpus(manifest: StudyManifest) -> Tuple[StudySong, ...]: diff --git a/scripts/codec_study/corpus/projects.py b/src/sampletones_tools/codec/study/corpus/projects.py similarity index 97% rename from scripts/codec_study/corpus/projects.py rename to src/sampletones_tools/codec/study/corpus/projects.py index 0419ff070..9897231eb 100644 --- a/scripts/codec_study/corpus/projects.py +++ b/src/sampletones_tools/codec/study/corpus/projects.py @@ -1,7 +1,6 @@ from math import ceil from pathlib import Path -from codec_study.corpus.song import SongGroup, StudySong from sampletones_core.performance import song_instructions from sampletones_core.project.container import ProjectContainer from sampletones_core.project.project import Project @@ -13,6 +12,7 @@ from sampletones_player.compression.planes.separate import planes_from_streams from sampletones_player.compression.seeds import phrases_from_project from sampletones_shared.utils.progress import silent_reporter +from sampletones_tools.codec.study.corpus.song import SongGroup, StudySong def project_song(path: Path) -> StudySong: diff --git a/scripts/codec_study/corpus/reconstructions.py b/src/sampletones_tools/codec/study/corpus/reconstructions.py similarity index 96% rename from scripts/codec_study/corpus/reconstructions.py rename to src/sampletones_tools/codec/study/corpus/reconstructions.py index 79d171c37..e7fd84b49 100644 --- a/scripts/codec_study/corpus/reconstructions.py +++ b/src/sampletones_tools/codec/study/corpus/reconstructions.py @@ -1,13 +1,13 @@ from pathlib import Path from typing import Final, Tuple -from codec_study.corpus.song import SongGroup, StudySong from sampletones_core.reconstructions import Reconstruction from sampletones_core.timers.utils import get_timer_table from sampletones_player.builder import streams_from_instructions from sampletones_player.compression.dictionary.phrase import Phrase from sampletones_player.compression.pitch import PitchTable from sampletones_player.compression.planes.separate import planes_from_streams +from sampletones_tools.codec.study.corpus.song import SongGroup, StudySong STEM_SUFFIX: Final[str] = ".stn" NO_SEEDS: Final[Tuple[Phrase, ...]] = () diff --git a/scripts/codec_study/corpus/song.py b/src/sampletones_tools/codec/study/corpus/song.py similarity index 100% rename from scripts/codec_study/corpus/song.py rename to src/sampletones_tools/codec/study/corpus/song.py diff --git a/scripts/codec_study/manifest.py b/src/sampletones_tools/codec/study/manifest.py similarity index 100% rename from scripts/codec_study/manifest.py rename to src/sampletones_tools/codec/study/manifest.py diff --git a/scripts/codec_study/measure.py b/src/sampletones_tools/codec/study/measure.py similarity index 98% rename from scripts/codec_study/measure.py rename to src/sampletones_tools/codec/study/measure.py index e277fe028..e116fcec4 100644 --- a/scripts/codec_study/measure.py +++ b/src/sampletones_tools/codec/study/measure.py @@ -1,10 +1,10 @@ from dataclasses import dataclass from typing import Callable, Optional, Tuple -from codec_study.corpus.song import StudySong from sampletones_player.compression.compressed import CompressedPlanes from sampletones_player.compression.decode import decode_planes from sampletones_player.specification.song import SONG_HEADER_SIZE +from sampletones_tools.codec.study.corpus.song import StudySong @dataclass(frozen=True) diff --git a/scripts/codec_study/sandbox/__init__.py b/src/sampletones_tools/codec/study/report/__init__.py similarity index 100% rename from scripts/codec_study/sandbox/__init__.py rename to src/sampletones_tools/codec/study/report/__init__.py diff --git a/scripts/codec_study/report/aggregate.py b/src/sampletones_tools/codec/study/report/aggregate.py similarity index 96% rename from scripts/codec_study/report/aggregate.py rename to src/sampletones_tools/codec/study/report/aggregate.py index 4026d9734..512042d87 100644 --- a/scripts/codec_study/report/aggregate.py +++ b/src/sampletones_tools/codec/study/report/aggregate.py @@ -1,8 +1,8 @@ from dataclasses import dataclass from typing import Dict, Final, List, Sequence, Tuple -from codec_study.corpus.song import SongGroup -from codec_study.measure import Measurement +from sampletones_tools.codec.study.corpus.song import SongGroup +from sampletones_tools.codec.study.measure import Measurement COLUMNS: Final[Tuple[str, ...]] = ( "group", diff --git a/scripts/codec_study/report/rows.py b/src/sampletones_tools/codec/study/report/rows.py similarity index 97% rename from scripts/codec_study/report/rows.py rename to src/sampletones_tools/codec/study/report/rows.py index 07592b44b..38188afaa 100644 --- a/scripts/codec_study/report/rows.py +++ b/src/sampletones_tools/codec/study/report/rows.py @@ -1,7 +1,7 @@ from dataclasses import dataclass from typing import Final, Tuple -from codec_study.measure import Measurement +from sampletones_tools.codec.study.measure import Measurement COLUMNS: Final[Tuple[str, ...]] = ( "group", diff --git a/scripts/codec_study/report/run.py b/src/sampletones_tools/codec/study/report/run.py similarity index 90% rename from scripts/codec_study/report/run.py rename to src/sampletones_tools/codec/study/report/run.py index a78126905..006b38d98 100644 --- a/scripts/codec_study/report/run.py +++ b/src/sampletones_tools/codec/study/report/run.py @@ -3,18 +3,18 @@ from pathlib import Path from typing import Final, Iterator, List, Optional, Sequence, Tuple -from codec_study.accounting import rows as accounting -from codec_study.manifest import StudyManifest -from codec_study.measure import Measurement -from codec_study.report import aggregate -from codec_study.report import rows as songs -from codec_study.report import verdicts -from codec_study.report.writers import markdown_table, write_csv -from codec_study.variants.production import BASELINE_NAME -from codec_study.variants.variant import Variant from sampletones_player.compression.compressed import CompressedPlanes from sampletones_shared.paths.source import REPOSITORY_ROOT from sampletones_shared.paths.user import USER_PATH_DOCUMENTS +from sampletones_tools.codec.study.accounting import rows as accounting +from sampletones_tools.codec.study.manifest import StudyManifest +from sampletones_tools.codec.study.measure import Measurement +from sampletones_tools.codec.study.report import aggregate +from sampletones_tools.codec.study.report import rows as songs +from sampletones_tools.codec.study.report import verdicts +from sampletones_tools.codec.study.report.writers import markdown_table, write_csv +from sampletones_tools.codec.study.variants.production import BASELINE_NAME +from sampletones_tools.codec.study.variants.variant import Variant DEFAULT_OUTPUT_ROOT: Final[Path] = USER_PATH_DOCUMENTS / "compression" RUN_STAMP: Final[str] = "run-%Y%m%d-%H%M%S" diff --git a/scripts/codec_study/report/verdicts.py b/src/sampletones_tools/codec/study/report/verdicts.py similarity index 95% rename from scripts/codec_study/report/verdicts.py rename to src/sampletones_tools/codec/study/report/verdicts.py index b98946ab9..761a7643c 100644 --- a/scripts/codec_study/report/verdicts.py +++ b/src/sampletones_tools/codec/study/report/verdicts.py @@ -2,10 +2,10 @@ from enum import StrEnum from typing import Dict, Final, List, Optional, Sequence, Tuple -from codec_study.corpus.song import SongGroup -from codec_study.measure import Measurement -from codec_study.report.aggregate import GroupRow -from codec_study.variants.variant import Variant, VariantKind +from sampletones_tools.codec.study.corpus.song import SongGroup +from sampletones_tools.codec.study.measure import Measurement +from sampletones_tools.codec.study.report.aggregate import GroupRow +from sampletones_tools.codec.study.variants.variant import Variant, VariantKind PROJECT_BAR: Final[float] = 0.03 RECONSTRUCTION_BAR: Final[float] = 0.05 diff --git a/scripts/codec_study/report/writers.py b/src/sampletones_tools/codec/study/report/writers.py similarity index 100% rename from scripts/codec_study/report/writers.py rename to src/sampletones_tools/codec/study/report/writers.py diff --git a/scripts/codec_study/sandbox/edges/__init__.py b/src/sampletones_tools/codec/study/sandbox/__init__.py similarity index 100% rename from scripts/codec_study/sandbox/edges/__init__.py rename to src/sampletones_tools/codec/study/sandbox/__init__.py diff --git a/scripts/codec_study/sandbox/context.py b/src/sampletones_tools/codec/study/sandbox/context.py similarity index 96% rename from scripts/codec_study/sandbox/context.py rename to src/sampletones_tools/codec/study/sandbox/context.py index 1e23f6058..60051a41f 100644 --- a/scripts/codec_study/sandbox/context.py +++ b/src/sampletones_tools/codec/study/sandbox/context.py @@ -1,11 +1,11 @@ from dataclasses import dataclass from typing import Tuple -from codec_study.sandbox.costs import Costs from sampletones_player.compression.matches.index import PlaneIndex from sampletones_player.compression.matches.matcher import PhraseMatcher from sampletones_player.compression.parse.boundaries import Boundaries from sampletones_player.specification.compression import INITIAL_PLANE_VALUE +from sampletones_tools.codec.study.sandbox.costs import Costs @dataclass(frozen=True) diff --git a/scripts/codec_study/sandbox/costs.py b/src/sampletones_tools/codec/study/sandbox/costs.py similarity index 100% rename from scripts/codec_study/sandbox/costs.py rename to src/sampletones_tools/codec/study/sandbox/costs.py diff --git a/scripts/codec_study/sandbox/decode.py b/src/sampletones_tools/codec/study/sandbox/decode.py similarity index 93% rename from scripts/codec_study/sandbox/decode.py rename to src/sampletones_tools/codec/study/sandbox/decode.py index 7929484ee..720064704 100644 --- a/scripts/codec_study/sandbox/decode.py +++ b/src/sampletones_tools/codec/study/sandbox/decode.py @@ -1,9 +1,9 @@ from typing import Sequence -from codec_study.sandbox.tokens import Hold, Literal, Play, SetHold, StudyToken, WideHold from sampletones_player.compression.dictionary.table import PhraseTable from sampletones_player.specification.binary import BYTE_VALUES from sampletones_player.specification.compression import INITIAL_PLANE_VALUE +from sampletones_tools.codec.study.sandbox.tokens import Hold, Literal, Play, SetHold, StudyToken, WideHold def _played( diff --git a/scripts/codec_study/sandbox/defaults.py b/src/sampletones_tools/codec/study/sandbox/defaults.py similarity index 90% rename from scripts/codec_study/sandbox/defaults.py rename to src/sampletones_tools/codec/study/sandbox/defaults.py index e73396316..34c697823 100644 --- a/scripts/codec_study/sandbox/defaults.py +++ b/src/sampletones_tools/codec/study/sandbox/defaults.py @@ -1,8 +1,8 @@ from collections import Counter from typing import Final, List, Sequence, Tuple -from codec_study.sandbox.parse import StudyParse -from codec_study.sandbox.tokens import Play +from sampletones_tools.codec.study.sandbox.parse import StudyParse +from sampletones_tools.codec.study.sandbox.tokens import Play NO_DEFAULT: Final[int] = 0 diff --git a/scripts/codec_study/variants/__init__.py b/src/sampletones_tools/codec/study/sandbox/edges/__init__.py similarity index 100% rename from scripts/codec_study/variants/__init__.py rename to src/sampletones_tools/codec/study/sandbox/edges/__init__.py diff --git a/scripts/codec_study/sandbox/edges/generator.py b/src/sampletones_tools/codec/study/sandbox/edges/generator.py similarity index 87% rename from scripts/codec_study/sandbox/edges/generator.py rename to src/sampletones_tools/codec/study/sandbox/edges/generator.py index f215ea93f..33a080306 100644 --- a/scripts/codec_study/sandbox/edges/generator.py +++ b/src/sampletones_tools/codec/study/sandbox/edges/generator.py @@ -1,8 +1,8 @@ from typing import Iterable, Protocol -from codec_study.sandbox.context import PlaneContext -from codec_study.sandbox.shortest import Shortest -from codec_study.sandbox.tokens import StudyToken +from sampletones_tools.codec.study.sandbox.context import PlaneContext +from sampletones_tools.codec.study.sandbox.shortest import Shortest +from sampletones_tools.codec.study.sandbox.tokens import StudyToken class EdgeGenerator(Protocol): diff --git a/scripts/codec_study/sandbox/edges/holds.py b/src/sampletones_tools/codec/study/sandbox/edges/holds.py similarity index 88% rename from scripts/codec_study/sandbox/edges/holds.py rename to src/sampletones_tools/codec/study/sandbox/edges/holds.py index 5139d5d78..d8f703298 100644 --- a/scripts/codec_study/sandbox/edges/holds.py +++ b/src/sampletones_tools/codec/study/sandbox/edges/holds.py @@ -1,10 +1,10 @@ from typing import Final -from codec_study.sandbox.context import PlaneContext -from codec_study.sandbox.edges.generator import EdgeGenerator, offered_lengths -from codec_study.sandbox.shortest import Shortest -from codec_study.sandbox.tokens import Hold, StudyToken, WideHold from sampletones_player.specification.compression import MAX_HOLD_TICKS +from sampletones_tools.codec.study.sandbox.context import PlaneContext +from sampletones_tools.codec.study.sandbox.edges.generator import EdgeGenerator, offered_lengths +from sampletones_tools.codec.study.sandbox.shortest import Shortest +from sampletones_tools.codec.study.sandbox.tokens import Hold, StudyToken, WideHold LEAST_HOLD_TICKS: Final[int] = 1 LEAST_WIDE_BLOCKS: Final[int] = 1 diff --git a/scripts/codec_study/sandbox/edges/literals.py b/src/sampletones_tools/codec/study/sandbox/edges/literals.py similarity index 83% rename from scripts/codec_study/sandbox/edges/literals.py rename to src/sampletones_tools/codec/study/sandbox/edges/literals.py index dd38994ff..8c19c1950 100644 --- a/scripts/codec_study/sandbox/edges/literals.py +++ b/src/sampletones_tools/codec/study/sandbox/edges/literals.py @@ -1,7 +1,7 @@ -from codec_study.sandbox.context import PlaneContext -from codec_study.sandbox.shortest import Shortest -from codec_study.sandbox.tokens import Literal, StudyToken from sampletones_player.compression.parse.literals import LiteralWindow +from sampletones_tools.codec.study.sandbox.context import PlaneContext +from sampletones_tools.codec.study.sandbox.shortest import Shortest +from sampletones_tools.codec.study.sandbox.tokens import Literal, StudyToken def relax_literal( diff --git a/scripts/codec_study/sandbox/edges/phrases.py b/src/sampletones_tools/codec/study/sandbox/edges/phrases.py similarity index 86% rename from scripts/codec_study/sandbox/edges/phrases.py rename to src/sampletones_tools/codec/study/sandbox/edges/phrases.py index 650f9a76d..8b5791611 100644 --- a/scripts/codec_study/sandbox/edges/phrases.py +++ b/src/sampletones_tools/codec/study/sandbox/edges/phrases.py @@ -1,8 +1,8 @@ -from codec_study.sandbox.context import PlaneContext -from codec_study.sandbox.edges.generator import EdgeGenerator -from codec_study.sandbox.shortest import Shortest -from codec_study.sandbox.tokens import Play, StudyToken from sampletones_player.specification.compression import MAX_PHRASE_TICKS +from sampletones_tools.codec.study.sandbox.context import PlaneContext +from sampletones_tools.codec.study.sandbox.edges.generator import EdgeGenerator +from sampletones_tools.codec.study.sandbox.shortest import Shortest +from sampletones_tools.codec.study.sandbox.tokens import Play, StudyToken def phrase_edges(*, defaults: bool) -> EdgeGenerator: diff --git a/scripts/codec_study/sandbox/edges/set_hold.py b/src/sampletones_tools/codec/study/sandbox/edges/set_hold.py similarity index 80% rename from scripts/codec_study/sandbox/edges/set_hold.py rename to src/sampletones_tools/codec/study/sandbox/edges/set_hold.py index cfe04d19c..17a62f2e7 100644 --- a/scripts/codec_study/sandbox/edges/set_hold.py +++ b/src/sampletones_tools/codec/study/sandbox/edges/set_hold.py @@ -1,9 +1,9 @@ from typing import Final -from codec_study.sandbox.context import PlaneContext -from codec_study.sandbox.edges.generator import EdgeGenerator, offered_lengths -from codec_study.sandbox.shortest import Shortest -from codec_study.sandbox.tokens import SetHold, StudyToken +from sampletones_tools.codec.study.sandbox.context import PlaneContext +from sampletones_tools.codec.study.sandbox.edges.generator import EdgeGenerator, offered_lengths +from sampletones_tools.codec.study.sandbox.shortest import Shortest +from sampletones_tools.codec.study.sandbox.tokens import SetHold, StudyToken LEAST_SET_HOLD_TICKS: Final[int] = 2 diff --git a/scripts/codec_study/sandbox/encode.py b/src/sampletones_tools/codec/study/sandbox/encode.py similarity index 84% rename from scripts/codec_study/sandbox/encode.py rename to src/sampletones_tools/codec/study/sandbox/encode.py index bfd1c1f34..ff6279c8d 100644 --- a/scripts/codec_study/sandbox/encode.py +++ b/src/sampletones_tools/codec/study/sandbox/encode.py @@ -1,12 +1,12 @@ from time import process_time from typing import Final, Sequence, Tuple -from codec_study.measure import Encoding -from codec_study.sandbox.defaults import modal_counts, no_defaults -from codec_study.sandbox.grammar import Grammar -from codec_study.sandbox.parse import StudyParse, parse_plane -from codec_study.sandbox.reference import Reference -from codec_study.sandbox.verify import plays_back +from sampletones_tools.codec.study.measure import Encoding +from sampletones_tools.codec.study.sandbox.defaults import modal_counts, no_defaults +from sampletones_tools.codec.study.sandbox.grammar import Grammar +from sampletones_tools.codec.study.sandbox.parse import StudyParse, parse_plane +from sampletones_tools.codec.study.sandbox.reference import Reference +from sampletones_tools.codec.study.sandbox.verify import plays_back DEFAULT_ROUNDS: Final[int] = 3 diff --git a/scripts/codec_study/sandbox/grammar.py b/src/sampletones_tools/codec/study/sandbox/grammar.py similarity index 93% rename from scripts/codec_study/sandbox/grammar.py rename to src/sampletones_tools/codec/study/sandbox/grammar.py index a0c12b51a..1827e2924 100644 --- a/scripts/codec_study/sandbox/grammar.py +++ b/src/sampletones_tools/codec/study/sandbox/grammar.py @@ -1,7 +1,7 @@ from dataclasses import dataclass from typing import Final -from codec_study.sandbox.costs import PRODUCTION_COSTS, Costs +from sampletones_tools.codec.study.sandbox.costs import PRODUCTION_COSTS, Costs @dataclass(frozen=True) diff --git a/scripts/codec_study/sandbox/parse.py b/src/sampletones_tools/codec/study/sandbox/parse.py similarity index 83% rename from scripts/codec_study/sandbox/parse.py rename to src/sampletones_tools/codec/study/sandbox/parse.py index ad3526e0a..82d3b6511 100644 --- a/scripts/codec_study/sandbox/parse.py +++ b/src/sampletones_tools/codec/study/sandbox/parse.py @@ -1,17 +1,17 @@ from dataclasses import dataclass from typing import List, Sequence, Tuple -from codec_study.sandbox.context import PlaneContext -from codec_study.sandbox.edges.generator import EdgeGenerator -from codec_study.sandbox.edges.holds import hold_edges, wide_hold_edges -from codec_study.sandbox.edges.literals import relax_literal -from codec_study.sandbox.edges.phrases import phrase_edges -from codec_study.sandbox.edges.set_hold import set_hold_edges -from codec_study.sandbox.grammar import Grammar -from codec_study.sandbox.shortest import Shortest -from codec_study.sandbox.tokens import StudyToken from sampletones_player.compression.parse.literals import LiteralWindow from sampletones_player.specification.compression import MAX_LITERAL_BYTES +from sampletones_tools.codec.study.sandbox.context import PlaneContext +from sampletones_tools.codec.study.sandbox.edges.generator import EdgeGenerator +from sampletones_tools.codec.study.sandbox.edges.holds import hold_edges, wide_hold_edges +from sampletones_tools.codec.study.sandbox.edges.literals import relax_literal +from sampletones_tools.codec.study.sandbox.edges.phrases import phrase_edges +from sampletones_tools.codec.study.sandbox.edges.set_hold import set_hold_edges +from sampletones_tools.codec.study.sandbox.grammar import Grammar +from sampletones_tools.codec.study.sandbox.shortest import Shortest +from sampletones_tools.codec.study.sandbox.tokens import StudyToken @dataclass(frozen=True) diff --git a/scripts/codec_study/sandbox/reference.py b/src/sampletones_tools/codec/study/sandbox/reference.py similarity index 87% rename from scripts/codec_study/sandbox/reference.py rename to src/sampletones_tools/codec/study/sandbox/reference.py index 122544064..a02973524 100644 --- a/scripts/codec_study/sandbox/reference.py +++ b/src/sampletones_tools/codec/study/sandbox/reference.py @@ -1,13 +1,6 @@ from dataclasses import dataclass from typing import Final, FrozenSet, Sequence, Tuple -from codec_study.corpus.song import StudySong -from codec_study.sandbox.context import PlaneContext -from codec_study.sandbox.costs import PRODUCTION_COSTS, Costs -from codec_study.sandbox.defaults import no_defaults -from codec_study.sandbox.grammar import BASELINE_GRAMMAR -from codec_study.sandbox.parse import StudyParse, parse_plane -from codec_study.sandbox.verify import verify_baseline from sampletones_player.compression.compressed import CompressedPlanes from sampletones_player.compression.dictionary.table import PhraseTable from sampletones_player.compression.encode import STREAM_START @@ -16,6 +9,13 @@ from sampletones_player.compression.matches.matcher import PhraseMatcher from sampletones_player.compression.options import EVERY_LAYER from sampletones_player.compression.parse.boundaries import Boundaries +from sampletones_tools.codec.study.corpus.song import StudySong +from sampletones_tools.codec.study.sandbox.context import PlaneContext +from sampletones_tools.codec.study.sandbox.costs import PRODUCTION_COSTS, Costs +from sampletones_tools.codec.study.sandbox.defaults import no_defaults +from sampletones_tools.codec.study.sandbox.grammar import BASELINE_GRAMMAR +from sampletones_tools.codec.study.sandbox.parse import StudyParse, parse_plane +from sampletones_tools.codec.study.sandbox.verify import verify_baseline STREAM_ENTRIES: Final[FrozenSet[int]] = frozenset({STREAM_START}) diff --git a/scripts/codec_study/sandbox/shortest.py b/src/sampletones_tools/codec/study/sandbox/shortest.py similarity index 100% rename from scripts/codec_study/sandbox/shortest.py rename to src/sampletones_tools/codec/study/sandbox/shortest.py diff --git a/scripts/codec_study/sandbox/tokens.py b/src/sampletones_tools/codec/study/sandbox/tokens.py similarity index 100% rename from scripts/codec_study/sandbox/tokens.py rename to src/sampletones_tools/codec/study/sandbox/tokens.py diff --git a/scripts/codec_study/sandbox/verify.py b/src/sampletones_tools/codec/study/sandbox/verify.py similarity index 93% rename from scripts/codec_study/sandbox/verify.py rename to src/sampletones_tools/codec/study/sandbox/verify.py index 11db3bf8b..a4728a5e3 100644 --- a/scripts/codec_study/sandbox/verify.py +++ b/src/sampletones_tools/codec/study/sandbox/verify.py @@ -1,11 +1,11 @@ from typing import Sequence -from codec_study.sandbox.decode import play_tokens -from codec_study.sandbox.parse import StudyParse from sampletones_player.compression.compressed import CompressedPlanes from sampletones_player.compression.dictionary.table import PhraseTable from sampletones_player.compression.planes.order import PlaneOrder from sampletones_player.compression.planes.song import SongPlanes +from sampletones_tools.codec.study.sandbox.decode import play_tokens +from sampletones_tools.codec.study.sandbox.parse import StudyParse def verify_baseline( diff --git a/src/sampletones_tools/codec/study/session.py b/src/sampletones_tools/codec/study/session.py new file mode 100644 index 000000000..aae4a851e --- /dev/null +++ b/src/sampletones_tools/codec/study/session.py @@ -0,0 +1,109 @@ +from pathlib import Path +from typing import Final, List, Optional, Tuple + +from sampletones_shared.logger import logger +from sampletones_tools.codec.study.corpus.build import build_corpus +from sampletones_tools.codec.study.manifest import StudyManifest, StudySource +from sampletones_tools.codec.study.measure import Measurement, measure +from sampletones_tools.codec.study.report.run import run_directory, write_run +from sampletones_tools.codec.study.variants.baselines import Baselines +from sampletones_tools.codec.study.variants.registry import EVERY_VARIANT, selected_variants +from sampletones_tools.codec.study.variants.strategy import STRATEGY_ORDER, depth_measurements + +LIST_SEPARATOR: Final[str] = "," + + +def variant_names(stated: Optional[str]) -> Tuple[str, ...]: + """The variants a run encodes under: the ones named, comma separated, or every one.""" + if stated is None: + return (EVERY_VARIANT,) + + return tuple(name.strip() for name in stated.split(LIST_SEPARATOR) if name.strip()) + + +def resolve_manifest( + path: Optional[Path], + *, + projects: Tuple[Path, ...], + reconstructions: Tuple[Path, ...], + lengthen_seconds: int, + variants: Tuple[str, ...], + quick: bool, +) -> StudyManifest: + """The manifest a run measures: the one a file states, or the corpus on this machine. + + Projects or reconstructions named outright stand in for the corpus while the manifest's + lengthening and variants stay. + + Args: + path: A manifest a run wrote, or ``None`` for the corpus on this machine. + projects: Project files measured in place of the corpus. + reconstructions: Stem files, or directories of stems, measured in place of the corpus. + lengthen_seconds: How long each project's lengthened copy lasts. + variants: The names of the variants every song is encoded under. + quick: Whether to read one small project and one stem, to check the harness. + """ + manifest = ( + StudyManifest.load(path) + if path is not None + else StudyManifest.default( + lengthen_seconds=lengthen_seconds, + variants=variants, + quick=quick, + ) + ) + if not projects and not reconstructions: + return manifest + + return StudyManifest( + projects=tuple(StudySource.at(project) for project in projects), + reconstructions=tuple(StudySource.at(reconstruction) for reconstruction in reconstructions), + lengthen_seconds=manifest.lengthen_seconds, + variants=manifest.variants, + ) + + +def run_study(manifest: StudyManifest, output: Optional[Path]) -> Path: + """Encodes every song of the manifest under every variant and writes the run. + + Args: + manifest: What is measured and under which variants. + output: The directory the run writes into, or ``None`` for a stamped one under the + documents. + + Returns: + Path: The directory holding the report, the accounting and the verdicts. + + Raises: + ValueError: If a variant writes a song as streams that play back differently. + """ + variants = selected_variants(manifest.variants, Baselines()) + directory = run_directory(output) + corpus = build_corpus(manifest) + + measurements: List[Measurement] = [] + for song in corpus: + for variant in variants: + if not variant.applies(song): + continue + + logger.info(f"Encoding {song.name} ({song.ticks} ticks) under {variant.name}") + measurement = measure(song, variant.name, variant.encode) + if not measurement.lossless: + raise ValueError(f"{variant.name} wrote {song.name} as streams that play back differently") + + logger.info( + f" {measurement.block} bytes, {measurement.bytes_per_tick:.3f} bytes per tick, " + f"{measurement.phrases} phrases, {measurement.seconds:.1f} s" + ) + measurements.append(measurement) + + write_run( + directory, + manifest, + variants, + measurements, + depth_measurements(measurements, STRATEGY_ORDER), + ) + logger.info(f"Report written to {directory}") + return directory diff --git a/src/sampletones_tools/codec/study/variants/__init__.py b/src/sampletones_tools/codec/study/variants/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/scripts/codec_study/variants/baselines.py b/src/sampletones_tools/codec/study/variants/baselines.py similarity index 87% rename from scripts/codec_study/variants/baselines.py rename to src/sampletones_tools/codec/study/variants/baselines.py index 1e899f4ce..c47e54475 100644 --- a/scripts/codec_study/variants/baselines.py +++ b/src/sampletones_tools/codec/study/variants/baselines.py @@ -1,12 +1,12 @@ from time import process_time from typing import Dict -from codec_study.corpus.song import StudySong -from codec_study.measure import Encoding, production_encoding -from codec_study.sandbox.reference import Reference, reference -from codec_study.variants.production import BASELINE_NAME, compress_baseline -from codec_study.variants.variant import Variant, VariantKind from sampletones_player.compression.compressed import CompressedPlanes +from sampletones_tools.codec.study.corpus.song import StudySong +from sampletones_tools.codec.study.measure import Encoding, production_encoding +from sampletones_tools.codec.study.sandbox.reference import Reference, reference +from sampletones_tools.codec.study.variants.production import BASELINE_NAME, compress_baseline +from sampletones_tools.codec.study.variants.variant import Variant, VariantKind class Baselines: diff --git a/scripts/codec_study/variants/production.py b/src/sampletones_tools/codec/study/variants/production.py similarity index 94% rename from scripts/codec_study/variants/production.py rename to src/sampletones_tools/codec/study/variants/production.py index 19dc85118..3be6af54a 100644 --- a/scripts/codec_study/variants/production.py +++ b/src/sampletones_tools/codec/study/variants/production.py @@ -1,15 +1,15 @@ from time import process_time from typing import Callable, Final, Sequence, Tuple -from codec_study.corpus.song import StudySong -from codec_study.measure import Encoder, Encoding, production_encoding -from codec_study.variants.seeds import split, trimmed, whole_and_split -from codec_study.variants.variant import Variant, VariantKind from sampletones_player.compression.budget import DEFAULT_SEARCH_BUDGET, SearchBudget from sampletones_player.compression.compressed import CompressedPlanes from sampletones_player.compression.dictionary.phrase import Phrase from sampletones_player.compression.encode import encode_planes from sampletones_player.compression.options import EVERY_LAYER +from sampletones_tools.codec.study.corpus.song import StudySong +from sampletones_tools.codec.study.measure import Encoder, Encoding, production_encoding +from sampletones_tools.codec.study.variants.seeds import split, trimmed, whole_and_split +from sampletones_tools.codec.study.variants.variant import Variant, VariantKind SeedTransform = Callable[[Sequence[Phrase]], Tuple[Phrase, ...]] diff --git a/scripts/codec_study/variants/registry.py b/src/sampletones_tools/codec/study/variants/registry.py similarity index 82% rename from scripts/codec_study/variants/registry.py rename to src/sampletones_tools/codec/study/variants/registry.py index 405daf45c..5a5a09e81 100644 --- a/scripts/codec_study/variants/registry.py +++ b/src/sampletones_tools/codec/study/variants/registry.py @@ -1,9 +1,9 @@ from typing import Dict, Final, Sequence, Tuple -from codec_study.variants.baselines import Baselines, baseline_variant -from codec_study.variants.production import BASELINE_NAME, BUDGET_VARIANTS, SEED_VARIANTS -from codec_study.variants.sandbox import grammar_variants -from codec_study.variants.variant import Variant +from sampletones_tools.codec.study.variants.baselines import Baselines, baseline_variant +from sampletones_tools.codec.study.variants.production import BASELINE_NAME, BUDGET_VARIANTS, SEED_VARIANTS +from sampletones_tools.codec.study.variants.sandbox import grammar_variants +from sampletones_tools.codec.study.variants.variant import Variant EVERY_VARIANT: Final[str] = "all" diff --git a/scripts/codec_study/variants/sandbox.py b/src/sampletones_tools/codec/study/variants/sandbox.py similarity index 91% rename from scripts/codec_study/variants/sandbox.py rename to src/sampletones_tools/codec/study/variants/sandbox.py index 7a7aa8b32..41dd122c3 100644 --- a/scripts/codec_study/variants/sandbox.py +++ b/src/sampletones_tools/codec/study/variants/sandbox.py @@ -1,19 +1,19 @@ from dataclasses import replace from typing import Final, NamedTuple, Tuple -from codec_study.corpus.song import StudySong -from codec_study.measure import Encoder, Encoding -from codec_study.sandbox.costs import ( +from sampletones_tools.codec.study.corpus.song import StudySong +from sampletones_tools.codec.study.measure import Encoder, Encoding +from sampletones_tools.codec.study.sandbox.costs import ( DEFAULT_COUNT_OPERANDS, DEFAULT_COUNT_SIZE, PRODUCTION_COSTS, SET_HOLD_BOUND, Costs, ) -from codec_study.sandbox.encode import encode_grammar -from codec_study.sandbox.grammar import BASELINE_GRAMMAR, Grammar -from codec_study.variants.baselines import Baselines -from codec_study.variants.variant import Variant, VariantKind +from sampletones_tools.codec.study.sandbox.encode import encode_grammar +from sampletones_tools.codec.study.sandbox.grammar import BASELINE_GRAMMAR, Grammar +from sampletones_tools.codec.study.variants.baselines import Baselines +from sampletones_tools.codec.study.variants.variant import Variant, VariantKind WIDE_HOLD: Final[str] = "H1" SET_HOLD: Final[str] = "H3" diff --git a/scripts/codec_study/variants/seeds.py b/src/sampletones_tools/codec/study/variants/seeds.py similarity index 100% rename from scripts/codec_study/variants/seeds.py rename to src/sampletones_tools/codec/study/variants/seeds.py diff --git a/scripts/codec_study/variants/strategy.py b/src/sampletones_tools/codec/study/variants/strategy.py similarity index 97% rename from scripts/codec_study/variants/strategy.py rename to src/sampletones_tools/codec/study/variants/strategy.py index 00762c0dd..211783f66 100644 --- a/scripts/codec_study/variants/strategy.py +++ b/src/sampletones_tools/codec/study/variants/strategy.py @@ -1,7 +1,7 @@ from dataclasses import replace from typing import Dict, Final, List, Optional, Sequence, Tuple -from codec_study.measure import Measurement +from sampletones_tools.codec.study.measure import Measurement STRATEGY_ORDER: Final[Tuple[str, ...]] = ( "baseline", diff --git a/scripts/codec_study/variants/variant.py b/src/sampletones_tools/codec/study/variants/variant.py similarity index 92% rename from scripts/codec_study/variants/variant.py rename to src/sampletones_tools/codec/study/variants/variant.py index 4fb9ee88d..47ea3754e 100644 --- a/scripts/codec_study/variants/variant.py +++ b/src/sampletones_tools/codec/study/variants/variant.py @@ -1,8 +1,8 @@ from dataclasses import dataclass from enum import StrEnum -from codec_study.corpus.song import StudySong -from codec_study.measure import Encoder +from sampletones_tools.codec.study.corpus.song import StudySong +from sampletones_tools.codec.study.measure import Encoder class VariantKind(StrEnum): diff --git a/src/sampletones_tools/registry.py b/src/sampletones_tools/registry.py index c70aa2f13..5ba1fcb58 100644 --- a/src/sampletones_tools/registry.py +++ b/src/sampletones_tools/registry.py @@ -4,7 +4,8 @@ from sampletones_tools.assets.command import ICONS from sampletones_tools.calibration.command import CALIBRATION from sampletones_tools.checks.command import CHECK +from sampletones_tools.codec.command import CODEC from sampletones_tools.player.command import DRIVER from sampletones_tools.samples.command import NSF -DEVELOPER_COMMANDS: Final[Tuple[Command, ...]] = (CALIBRATION, CHECK, DRIVER, ICONS, NSF) +DEVELOPER_COMMANDS: Final[Tuple[Command, ...]] = (CALIBRATION, CHECK, CODEC, DRIVER, ICONS, NSF) diff --git a/tests/unit/scripts/codec_study/test_accounting.py b/tests/unit/sampletones_tools/codec/study/test_accounting.py similarity index 94% rename from tests/unit/scripts/codec_study/test_accounting.py rename to tests/unit/sampletones_tools/codec/study/test_accounting.py index 253235cc4..f22839b39 100644 --- a/tests/unit/scripts/codec_study/test_accounting.py +++ b/tests/unit/sampletones_tools/codec/study/test_accounting.py @@ -3,13 +3,6 @@ import pytest -from codec_study.accounting.coincident import coincident_starts -from codec_study.accounting.dictionary import default_counts, plateaus -from codec_study.accounting.finding import Finding -from codec_study.accounting.pairs import ramps_in_literals, set_holds -from codec_study.accounting.runs import ramps, runs -from codec_study.accounting.shares import hold_chains -from codec_study.accounting.tokens import ReadToken, read_tokens from sampletones_player.compression.dictionary.phrase import Phrase from sampletones_player.compression.dictionary.table import phrase_table from sampletones_player.compression.encode import emit @@ -19,6 +12,13 @@ from sampletones_player.compression.tokens.phrase import PhraseToken from sampletones_player.compression.tokens.types import TokenUnion from sampletones_player.specification.compression import MAX_HOLD_TICKS, TokenTag +from sampletones_tools.codec.study.accounting.coincident import coincident_starts +from sampletones_tools.codec.study.accounting.dictionary import default_counts, plateaus +from sampletones_tools.codec.study.accounting.finding import Finding +from sampletones_tools.codec.study.accounting.pairs import ramps_in_literals, set_holds +from sampletones_tools.codec.study.accounting.runs import ramps, runs +from sampletones_tools.codec.study.accounting.shares import hold_chains +from sampletones_tools.codec.study.accounting.tokens import ReadToken, read_tokens from tests.suite.base import BaseTestSuite from tests.suite.case import BaseRegularTestCase diff --git a/tests/unit/scripts/codec_study/test_sandbox.py b/tests/unit/sampletones_tools/codec/study/test_sandbox.py similarity index 92% rename from tests/unit/scripts/codec_study/test_sandbox.py rename to tests/unit/sampletones_tools/codec/study/test_sandbox.py index b30476112..5fc9699e3 100644 --- a/tests/unit/scripts/codec_study/test_sandbox.py +++ b/tests/unit/sampletones_tools/codec/study/test_sandbox.py @@ -5,20 +5,6 @@ import pytest -from codec_study.corpus.song import SongGroup, StudySong -from codec_study.sandbox.context import PlaneContext -from codec_study.sandbox.costs import PRODUCTION_COSTS, SET_HOLD_BOUND, Costs -from codec_study.sandbox.decode import play_tokens -from codec_study.sandbox.defaults import modal_counts, no_defaults -from codec_study.sandbox.edges.generator import offered_lengths -from codec_study.sandbox.encode import encode_grammar -from codec_study.sandbox.grammar import BASELINE_GRAMMAR, Grammar -from codec_study.sandbox.parse import StudyParse, parse_plane -from codec_study.sandbox.reference import reference -from codec_study.sandbox.tokens import Hold, Literal, Play, SetHold, StudyToken, WideHold -from codec_study.sandbox.verify import verify_baseline -from codec_study.variants.production import compress_baseline -from codec_study.variants.sandbox import DEFAULT_COUNT_COSTS, GRAMMAR_VARIANTS, GrammarVariant from sampletones_player.compression.dictionary.phrase import Phrase from sampletones_player.compression.dictionary.table import PhraseTable, phrase_table from sampletones_player.compression.encode import STREAM_START @@ -37,6 +23,20 @@ PLANE_COUNT, ) from sampletones_shared.music import Tuning +from sampletones_tools.codec.study.corpus.song import SongGroup, StudySong +from sampletones_tools.codec.study.sandbox.context import PlaneContext +from sampletones_tools.codec.study.sandbox.costs import PRODUCTION_COSTS, SET_HOLD_BOUND, Costs +from sampletones_tools.codec.study.sandbox.decode import play_tokens +from sampletones_tools.codec.study.sandbox.defaults import modal_counts, no_defaults +from sampletones_tools.codec.study.sandbox.edges.generator import offered_lengths +from sampletones_tools.codec.study.sandbox.encode import encode_grammar +from sampletones_tools.codec.study.sandbox.grammar import BASELINE_GRAMMAR, Grammar +from sampletones_tools.codec.study.sandbox.parse import StudyParse, parse_plane +from sampletones_tools.codec.study.sandbox.reference import reference +from sampletones_tools.codec.study.sandbox.tokens import Hold, Literal, Play, SetHold, StudyToken, WideHold +from sampletones_tools.codec.study.sandbox.verify import verify_baseline +from sampletones_tools.codec.study.variants.production import compress_baseline +from sampletones_tools.codec.study.variants.sandbox import DEFAULT_COUNT_COSTS, GRAMMAR_VARIANTS, GrammarVariant from tests.suite.base import BaseTestSuite from tests.suite.case import BaseRegularTestCase diff --git a/tests/unit/sampletones_tools/codec/study/test_session.py b/tests/unit/sampletones_tools/codec/study/test_session.py new file mode 100644 index 000000000..2ec115a62 --- /dev/null +++ b/tests/unit/sampletones_tools/codec/study/test_session.py @@ -0,0 +1,43 @@ +from pathlib import Path + +from sampletones_tools.codec.study.manifest import StudyManifest, StudySource +from sampletones_tools.codec.study.session import resolve_manifest, variant_names +from sampletones_tools.codec.study.variants.registry import EVERY_VARIANT + + +class TestVariantNames: + def test_nothing_named_is_every_variant(self) -> None: + assert variant_names(None) == (EVERY_VARIANT,) + + def test_names_are_read_in_order_with_their_spaces_stripped(self) -> None: + assert variant_names("wide-hold, default-count") == ("wide-hold", "default-count") + + +class TestResolveManifest: + def test_sources_named_outright_stand_in_for_the_corpus(self) -> None: + manifest = resolve_manifest( + None, + projects=(Path("songs/one.stp"),), + reconstructions=(Path("stems/two"),), + lengthen_seconds=30, + variants=("wide-hold",), + quick=False, + ) + + assert manifest.projects == (StudySource(label="one", path=Path("songs/one.stp")),) + assert manifest.reconstructions == (StudySource(label="two", path=Path("stems/two")),) + assert manifest.lengthen_seconds == 30 + assert manifest.variants == ("wide-hold",) + + def test_without_sources_the_corpus_on_this_machine_is_read(self) -> None: + manifest = resolve_manifest( + None, + projects=(), + reconstructions=(), + lengthen_seconds=30, + variants=(EVERY_VARIANT,), + quick=True, + ) + + assert manifest == StudyManifest.default(lengthen_seconds=30, variants=(EVERY_VARIANT,), quick=True) + assert manifest.projects diff --git a/tests/unit/scripts/codec_study/test_variants.py b/tests/unit/sampletones_tools/codec/study/test_variants.py similarity index 94% rename from tests/unit/scripts/codec_study/test_variants.py rename to tests/unit/sampletones_tools/codec/study/test_variants.py index 1c8109fcc..32e842639 100644 --- a/tests/unit/scripts/codec_study/test_variants.py +++ b/tests/unit/sampletones_tools/codec/study/test_variants.py @@ -4,10 +4,6 @@ import pytest -from codec_study.corpus.song import SongGroup, StudySong -from codec_study.measure import Measurement, production_encoding -from codec_study.variants.seeds import split, trimmed, whole_and_split -from codec_study.variants.strategy import DEPTH_PREFIX, depth_measurements from sampletones_player.compression.compressed import CompressedPlanes from sampletones_player.compression.dictionary.phrase import Phrase from sampletones_player.compression.dictionary.table import phrase_table @@ -19,6 +15,10 @@ from sampletones_player.compression.tokens.literal import LiteralToken from sampletones_player.specification.compression import PLANE_COUNT from sampletones_shared.music import Tuning +from sampletones_tools.codec.study.corpus.song import SongGroup, StudySong +from sampletones_tools.codec.study.measure import Measurement, production_encoding +from sampletones_tools.codec.study.variants.seeds import split, trimmed, whole_and_split +from sampletones_tools.codec.study.variants.strategy import DEPTH_PREFIX, depth_measurements from tests.suite.base import BaseTestSuite from tests.suite.case import BaseRegularTestCase diff --git a/tests/unit/scripts/codec_study/test_verdicts.py b/tests/unit/sampletones_tools/codec/study/test_verdicts.py similarity index 94% rename from tests/unit/scripts/codec_study/test_verdicts.py rename to tests/unit/sampletones_tools/codec/study/test_verdicts.py index b465c8040..e4530544f 100644 --- a/tests/unit/scripts/codec_study/test_verdicts.py +++ b/tests/unit/sampletones_tools/codec/study/test_verdicts.py @@ -4,16 +4,16 @@ import pytest -from codec_study.corpus.song import SongGroup, StudySong -from codec_study.measure import Encoding, Measurement -from codec_study.report.aggregate import group_rows -from codec_study.report.verdicts import Verdict, judge, verdict_rows -from codec_study.variants.variant import Variant, VariantKind from sampletones_player.compression.pitch import PitchTable from sampletones_player.compression.planes.order import PlaneOrder from sampletones_player.compression.planes.song import SongPlanes from sampletones_player.specification.compression import PLANE_COUNT from sampletones_shared.music import Tuning +from sampletones_tools.codec.study.corpus.song import SongGroup, StudySong +from sampletones_tools.codec.study.measure import Encoding, Measurement +from sampletones_tools.codec.study.report.aggregate import group_rows +from sampletones_tools.codec.study.report.verdicts import Verdict, judge, verdict_rows +from sampletones_tools.codec.study.variants.variant import Variant, VariantKind from tests.suite.base import BaseTestSuite from tests.suite.case import BaseRegularTestCase diff --git a/tests/unit/sampletones_tools/codec/test_command.py b/tests/unit/sampletones_tools/codec/test_command.py new file mode 100644 index 000000000..53fca9d0c --- /dev/null +++ b/tests/unit/sampletones_tools/codec/test_command.py @@ -0,0 +1,70 @@ +from pathlib import Path +from typing import Final, List, Optional, Tuple + +import pytest + +from sampletones.commands.registry import COMMANDS +from sampletones.dispatcher import dispatch +from sampletones_tools.codec.study.manifest import StudyManifest + +RUNNER: Final[str] = "sampletones_tools.codec.study.session.run_study" + + +class RecordedStudy: + def __init__(self) -> None: + self.runs: List[Tuple[StudyManifest, Optional[Path]]] = [] + + def __call__(self, manifest: StudyManifest, output: Optional[Path]) -> Path: + self.runs.append((manifest, output)) + return output if output is not None else Path("run") + + +class TestCodecStudy: + def test_the_sources_and_the_sweep_are_read_from_the_options( + self, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + ) -> None: + study = RecordedStudy() + monkeypatch.setattr(RUNNER, study) + + status = dispatch( + COMMANDS, + [ + "codec", + "study", + "--project", + "songs/one.stp", + "--reconstruction", + "stems/two", + "--variants", + "wide-hold", + "--lengthen", + "30", + "-o", + str(tmp_path), + ], + ) + + assert status == 0 + manifest, output = study.runs[0] + assert [source.path for source in manifest.projects] == [Path("songs/one.stp")] + assert [source.path for source in manifest.reconstructions] == [Path("stems/two")] + assert manifest.variants == ("wide-hold",) + assert manifest.lengthen_seconds == 30 + assert output == tmp_path + + def test_a_quick_run_reads_the_small_corpus_into_the_documents(self, monkeypatch: pytest.MonkeyPatch) -> None: + study = RecordedStudy() + monkeypatch.setattr(RUNNER, study) + + assert dispatch(COMMANDS, ["codec", "study", "--quick"]) == 0 + manifest, output = study.runs[0] + assert manifest == StudyManifest.default(lengthen_seconds=180, variants=("all",), quick=True) + assert output is None + + def test_an_action_is_required(self) -> None: + with pytest.raises(SystemExit) as leaving: + dispatch(COMMANDS, ["codec"]) + + assert leaving.value.code == 2 From 913508797c168d8b05f6dd934d5b40fb40fd35be Mon Sep 17 00:00:00 2001 From: JakimPL Date: Sun, 13 Sep 2026 17:32:27 +0200 Subject: [PATCH 16/36] Moved: the sample corpus and its emitters into the tools package --- Makefile | 18 +- docs/concepts/compression.md | 2 +- docs/development/packages.md | 2 +- docs/development/player.md | 6 +- docs/development/tooling.md | 5 +- src/sampletones_tools/codec/command.py | 46 +++- .../codec/report}/__init__.py | 0 .../sampletones_tools/codec/report}/corpus.py | 24 +- .../codec/report/encoding.py | 221 ++++++++++++++++ .../sampletones_tools/codec/report/rows.py | 0 src/sampletones_tools/codec/report/session.py | 56 ++++ .../sampletones_tools/codec/report}/songs.py | 0 src/sampletones_tools/corpus/__init__.py | 0 src/sampletones_tools/corpus/build.py | 56 ++++ src/sampletones_tools/corpus/catalog.py | 218 ++++++++++++++++ .../corpus/config/__init__.py | 0 .../corpus}/config/module.yaml | 0 .../corpus}/config/reconstruction.yaml | 0 .../corpus}/config/song.yaml | 0 .../corpus}/config/synth.yaml | 0 src/sampletones_tools/corpus/module.py | 23 ++ src/sampletones_tools/corpus/paths.py | 9 + src/sampletones_tools/corpus/song.py | 121 +++++++++ src/sampletones_tools/corpus/synth.py | 21 ++ src/sampletones_tools/registry.py | 6 +- src/sampletones_tools/samples/bitphase.py | 39 +++ .../samples/commands/__init__.py | 0 src/sampletones_tools/samples/commands/btp.py | 34 +++ src/sampletones_tools/samples/commands/ftm.py | 34 +++ .../samples/{command.py => commands/nsf.py} | 32 ++- .../samples/commands/options.py | 29 +++ src/sampletones_tools/samples/emit.py | 24 ++ src/sampletones_tools/samples/famitracker.py | 22 ++ src/sampletones_tools/samples/nsf.py | 47 ++++ tests/benchmarks/test_compression.py | 2 +- tests/integration/assets/module_config.py | 20 -- tests/integration/assets/reconstruction.py | 240 ------------------ tests/integration/assets/song_loader.py | 94 ------- tests/integration/assets/synth_config.py | 20 -- tests/integration/bitphase/conftest.py | 22 +- .../integration/bitphase/test_btp_pipeline.py | 13 +- tests/integration/conftest.py | 37 +-- tests/integration/famitracker/conftest.py | 14 +- tests/integration/nsf/conftest.py | 18 +- tests/integration/nsf/exports.py | 10 - tests/integration/nsf/test_backend.py | 10 +- .../nsf/test_compression_report.py | 229 ++--------------- tests/integration/nsf/test_driver_audio.py | 2 +- tests/integration/nsf/test_driver_bend.py | 2 +- tests/integration/nsf/test_driver_trace.py | 2 +- tests/integration/nsf/test_nsf_pipeline.py | 2 +- tests/integration/nsf/test_song_export.py | 2 +- tests/integration/output.py | 49 ---- tests/integration/paths.py | 30 --- .../reconstruction/test_conversion_jobs.py | 6 +- .../reconstruction/test_decoding.py | 2 +- .../test_stems_reconstruction.py | 4 +- tests/integration/samples/__init__.py | 0 tests/integration/samples/test_emitters.py | 49 ++++ tests/suite/stems.py | 78 +++++- .../sampletones_tools/codec/test_command.py | 30 +++ .../sampletones_tools/corpus/test_build.py | 35 +++ .../sampletones_tools/corpus/test_catalog.py | 13 + .../sampletones_tools/corpus/test_module.py | 9 + .../sampletones_tools/corpus/test_song.py | 74 ++++++ .../samples/commands/test_btp.py | 43 ++++ .../samples/commands/test_ftm.py | 43 ++++ .../{test_command.py => commands/test_nsf.py} | 32 +++ 68 files changed, 1498 insertions(+), 833 deletions(-) rename {tests/integration/assets => src/sampletones_tools/codec/report}/__init__.py (100%) rename {tests/integration/nsf => src/sampletones_tools/codec/report}/corpus.py (92%) create mode 100644 src/sampletones_tools/codec/report/encoding.py rename tests/integration/nsf/report.py => src/sampletones_tools/codec/report/rows.py (100%) create mode 100644 src/sampletones_tools/codec/report/session.py rename {tests/integration/nsf => src/sampletones_tools/codec/report}/songs.py (100%) create mode 100644 src/sampletones_tools/corpus/__init__.py create mode 100644 src/sampletones_tools/corpus/build.py create mode 100644 src/sampletones_tools/corpus/catalog.py create mode 100644 src/sampletones_tools/corpus/config/__init__.py rename {tests/integration => src/sampletones_tools/corpus}/config/module.yaml (100%) rename {tests/integration => src/sampletones_tools/corpus}/config/reconstruction.yaml (100%) rename {tests/integration => src/sampletones_tools/corpus}/config/song.yaml (100%) rename {tests/integration => src/sampletones_tools/corpus}/config/synth.yaml (100%) create mode 100644 src/sampletones_tools/corpus/module.py create mode 100644 src/sampletones_tools/corpus/paths.py create mode 100644 src/sampletones_tools/corpus/song.py create mode 100644 src/sampletones_tools/corpus/synth.py create mode 100644 src/sampletones_tools/samples/bitphase.py create mode 100644 src/sampletones_tools/samples/commands/__init__.py create mode 100644 src/sampletones_tools/samples/commands/btp.py create mode 100644 src/sampletones_tools/samples/commands/ftm.py rename src/sampletones_tools/samples/{command.py => commands/nsf.py} (61%) create mode 100644 src/sampletones_tools/samples/commands/options.py create mode 100644 src/sampletones_tools/samples/emit.py create mode 100644 src/sampletones_tools/samples/famitracker.py create mode 100644 src/sampletones_tools/samples/nsf.py delete mode 100644 tests/integration/assets/module_config.py delete mode 100644 tests/integration/assets/reconstruction.py delete mode 100644 tests/integration/assets/song_loader.py delete mode 100644 tests/integration/assets/synth_config.py delete mode 100644 tests/integration/nsf/exports.py delete mode 100644 tests/integration/output.py delete mode 100644 tests/integration/paths.py create mode 100644 tests/integration/samples/__init__.py create mode 100644 tests/integration/samples/test_emitters.py create mode 100644 tests/unit/sampletones_tools/corpus/test_build.py create mode 100644 tests/unit/sampletones_tools/corpus/test_catalog.py create mode 100644 tests/unit/sampletones_tools/corpus/test_module.py create mode 100644 tests/unit/sampletones_tools/corpus/test_song.py create mode 100644 tests/unit/sampletones_tools/samples/commands/test_btp.py create mode 100644 tests/unit/sampletones_tools/samples/commands/test_ftm.py rename tests/unit/sampletones_tools/samples/{test_command.py => commands/test_nsf.py} (62%) diff --git a/Makefile b/Makefile index 08c93824d..454e2dd24 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,4 @@ -.PHONY: help setup install system-deps build release run clean pre-commit test benchmarks lint format \ - ftm-samples nsf-samples compression-report +.PHONY: help setup install system-deps build release run clean pre-commit test benchmarks lint format ifeq ($(OS),Windows_NT) ifeq ($(MSYSTEM),) @@ -30,9 +29,6 @@ help: @echo $(Q) make release - Compile standalone executable with the release deployment config (INFO, self-healing history)$(Q) @echo $(Q) make test - Run the doctests, the covered suite and the benchmarks$(Q) @echo $(Q) make benchmarks - Run the measured-duration suite on its own$(Q) - @echo $(Q) make ftm-samples - Emit example .ftm files to build/ftm via the integration suite$(Q) - @echo $(Q) make nsf-samples - Emit example .nsf files to build/nsf via the integration suite$(Q) - @echo $(Q) make compression-report - Measure the song codec into build/compression$(Q) @echo $(Q) make clean - Remove build artifacts and cache files$(Q) @echo $(Q) make lint - Run mypy and pylint (ARGS=--mypy or ARGS=--pylint for one of them)$(Q) @echo $(Q) make format - Auto-format code (isort, black)$(Q) @@ -74,15 +70,3 @@ lint: format: $(PYTHON) scripts/formatting.py - -ftm-samples: export SAMPLETONES_FTM_OUTPUT_DIR := build/ftm -ftm-samples: - uv run python -m pytest tests/integration/famitracker - -nsf-samples: export SAMPLETONES_NSF_OUTPUT_DIR := build/nsf -nsf-samples: - uv run python -m pytest tests/integration/nsf - -compression-report: export SAMPLETONES_COMPRESSION_OUTPUT_DIR := build/compression -compression-report: - uv run python -m pytest tests/integration/nsf/test_compression_report.py diff --git a/docs/concepts/compression.md b/docs/concepts/compression.md index 05ff10a6a..95889a817 100644 --- a/docs/concepts/compression.md +++ b/docs/concepts/compression.md @@ -255,7 +255,7 @@ minutes at 60 Hz**, against the 49 seconds a record per tick reaches. Encoding i about two seconds; decoding it costs the console around twenty instructions per plane per tick, comfortably inside a video frame. -`make compression-report` writes this table over a corpus of songs, and the format's +`uv run sampletones codec report` writes this table over a corpus of songs, and the format's constants are settled from it. Two of them were settled against expectation: splitting the duty cycle out of the control byte into a plane of its own **costs** 14 %, because volume and duty turn over together and a split pays two opcodes for what one covers; diff --git a/docs/development/packages.md b/docs/development/packages.md index f662d83de..622d64c83 100644 --- a/docs/development/packages.md +++ b/docs/development/packages.md @@ -50,7 +50,7 @@ graph TD | `sampletones_core` | The reconstruction engine, the project model, playing a song out into instructions, and the tracker export formats | `sampletones_shared` | | `sampletones_player` | The NES player: the register model, the re-clocking schedule, the 6502 driver and the NSF file | `sampletones_shared`, `sampletones_core` | | `sampletones_application` | The DearPyGui front end | `sampletones_shared`, `sampletones_core`, `sampletones_player` | -| `sampletones_tools` | Everything a developer runs and the application does not: analytic waveform synthesis, the calibration harness, the driver toolchain and the register trace, the mark the icons are drawn from, the source checks, and the developer commands that run them | `sampletones_shared`, `sampletones_assets`, `sampletones_core`, `sampletones_player`, `sampletones_application` | +| `sampletones_tools` | Everything a developer runs and the application does not: analytic waveform synthesis, the calibration harness, the driver toolchain and the register trace, the mark the icons are drawn from, the source checks, the synthetic corpus with its sample emitters, and the developer commands that run them | `sampletones_shared`, `sampletones_assets`, `sampletones_core`, `sampletones_player`, `sampletones_application` | | `sampletones` | The command-line entry: the dispatcher, the commands and the startup self-check | `sampletones_shared`, `sampletones_core`, `sampletones_application`, `sampletones_tools` | Third-party imports are the package author's own choice and stand outside this table. diff --git a/docs/development/player.md b/docs/development/player.md index 10f0071ea..e402657da 100644 --- a/docs/development/player.md +++ b/docs/development/player.md @@ -38,7 +38,7 @@ planes it writes, and every row playing it becomes a token naming that entry. Se what the samples leave uncovered. **Every layer earns its place on measured ground.** Each stage of the codec can be switched -off on its own, and `make compression-report` writes what each one saves across a corpus of +off on its own, and `uv run sampletones codec report` writes what each one saves across a corpus of songs. The format's constants are settled from that report rather than from argument. **A change to the codec is measured before it is built.** `uv run sampletones codec study` reads the @@ -151,7 +151,7 @@ The chain runs from the register values upward, and each link is held on its own |---|---| | The codec is lossless | every encoding decodes to the planes it was written from, over a corpus | | The codec is safe | a plane the codec finds nothing in stays within its literal bound | -| The ratio | `make compression-report` — bytes per tick and ticks that fit, per layer | +| The ratio | `uv run sampletones codec report` — bytes per tick and ticks that fit, per layer | | What a change would save | `uv run sampletones codec study` — the songs on this machine under every candidate change, with a verdict each | | The byte layout | a hand-built song serializes to expected bytes | | The assembly agrees with the specification | the include's equates are read and compared field by field | @@ -159,7 +159,7 @@ The chain runs from the register values upward, and each link is held on its own | The driver's arithmetic | a song stating a bend outright, held to the divider each tick is meant to sound at | | The audio | a captured trace re-rendered against the reconstruction's own approximation | | The whole export | a project exported, played on the emulator, and read back as the instructions the sequencer sounds | -| Listening | `make nsf-samples` then `uv run sampletones nsf render --directory build/nsf`, or any NSF player | +| Listening | `uv run sampletones nsf samples -o build/nsf` then `uv run sampletones nsf render --directory build/nsf`, or any NSF player | | Speed | `make benchmarks` — the encoder's own cost on the shapes that scale worst | The audio comparison is the one that catches a mistake the trace would let through: the diff --git a/docs/development/tooling.md b/docs/development/tooling.md index 3e8a14e5c..c0e3763a3 100644 --- a/docs/development/tooling.md +++ b/docs/development/tooling.md @@ -83,13 +83,13 @@ The developer commands, listed by `sampletones_tools/registry.py` and run as |---|---| | `calibration [--config FILE] [-o DIR] [--methods LIST] [--perceptual-exponents LIST] [--temporal-weights LIST] [--channels LIST]` | Reconstructs the calibration corpus under every variant of the sweep, scores it with every referee, and writes the reports; without `-o` the run lands in a timestamped directory under Documents/SampleToNES/calibration | | `check [options]` | Holds the tree to one of its checks: `import-boundary`, `language-keys`, `palette-colors`, `rendered-literals`, `shortcut-actions`, `tag-names`, `unused-tags`; each is a pre-commit hook, and [architecture](architecture.md#enforcement) says what each holds. Needs a checkout | +| `btp samples -o DIR`, `ftm samples -o DIR`, `nsf samples -o DIR` | Builds the synthetic corpus and writes it as example files: the arrangement as two Bitphase documents (at its tempo and as a groove), the arrangement as a FamiTracker module, or each sample and the arrangement as `.nsf` programs | +| `codec report -o DIR` | Compresses the synthetic corpus under every layer of the codec and writes the report the format's constants are settled from, as CSV and Markdown | | `codec study [--manifest FILE] [--project FILE]... [--reconstruction PATH]... [-o DIR] [--lengthen SECONDS] [--variants LIST] [--quick]` | Encodes the projects and stems on this machine under every candidate change to the codec and writes the sizes, the times and a verdict per candidate; without `-o` the run lands under Documents/SampleToNES/compression | | `driver [--directory DIR]` | Assembles the NES player driver with cc65 and prints the layout the build produced; without `--directory` it writes the driver the package ships, which needs a checkout | | `icons [--directory DIR]` | Writes the icon suite from the mark; without `--directory` it writes the icons the package ships, which needs a checkout | | `nsf render --directory DIR [--tail SECONDS]` | Renders every exported `.nsf` file in the directory to a wave beside it, through ffmpeg's libgme demuxer | -More join as the tools they run move into the package. - ## The tools package `src/sampletones_tools/` holds every tool the running application does not use, in subpackages by @@ -148,6 +148,7 @@ interpreter (`preflight.py`), and the platforms (`platforms/`). | Which developer commands exist | `src/sampletones_tools/registry.py` | | Which checks the tree is held to | `src/sampletones_tools/checks/registry.py` | | Whether a command runs outside a checkout | `src/sampletones_tools/checkout.py` | +| The synthetic corpus the emitters and the integration tests share | `src/sampletones_tools/corpus/` | | What a command is | `src/sampletones_shared/command.py` | | What a bootstrap script may import | `sampletones_config/boundaries/standalone.yaml` | | What differs between systems | `scripts/bootstrap/platforms/` | diff --git a/src/sampletones_tools/codec/command.py b/src/sampletones_tools/codec/command.py index abfa6fb3d..3677fa4a2 100644 --- a/src/sampletones_tools/codec/command.py +++ b/src/sampletones_tools/codec/command.py @@ -6,9 +6,12 @@ from sampletones_shared.command import Command NAME: Final[str] = "codec" -HELP: Final[str] = "measure the song codec" +HELP: Final[str] = "measure the song codec on the synthetic corpus or on songs of this machine" ACTION_FIELD: Final[str] = "action" ACTION_METAVAR: Final[str] = "" +REPORT: Final[str] = "report" +REPORT_HELP: Final[str] = "compress the synthetic corpus under every layer of the codec and write the report tables" +REPORT_OUTPUT_HELP: Final[str] = "the directory the report is written into, created when missing" STUDY: Final[str] = "study" STUDY_HELP: Final[str] = ( "encode the projects and stems on this machine under every candidate change, with a verdict each" @@ -27,6 +30,13 @@ DEFAULT_LENGTHEN_SECONDS: Final[int] = 180 +@dataclass(frozen=True) +class ReportArguments: + """What a report run is given: the directory the tables are written into.""" + + output: Path + + @dataclass(frozen=True) class StudyArguments: """What a study run is given, as written on the command line.""" @@ -42,6 +52,8 @@ class StudyArguments: def configure(parser: ArgumentParser) -> None: actions = parser.add_subparsers(dest=ACTION_FIELD, metavar=ACTION_METAVAR, required=True) + report = actions.add_parser(REPORT, help=REPORT_HELP, description=REPORT_HELP) + report.add_argument("--output", "-o", type=Path, required=True, help=REPORT_OUTPUT_HELP) study = actions.add_parser(STUDY, help=STUDY_HELP, description=STUDY_HELP) study.add_argument("--manifest", type=Path, default=None, help=MANIFEST_HELP) study.add_argument("--project", type=Path, action="append", dest="projects", default=[], help=PROJECT_HELP) @@ -61,16 +73,32 @@ def configure(parser: ArgumentParser) -> None: def run(arguments: Namespace) -> int: """Measures the codec the way the action describes.""" - given = StudyArguments( - manifest=arguments.manifest, - projects=tuple(arguments.projects), - reconstructions=tuple(arguments.reconstructions), - output=arguments.output, - lengthen=arguments.lengthen, - variants=arguments.variants, - quick=arguments.quick, + if arguments.action == REPORT: + return _report(ReportArguments(output=arguments.output)) + + return _study( + StudyArguments( + manifest=arguments.manifest, + projects=tuple(arguments.projects), + reconstructions=tuple(arguments.reconstructions), + output=arguments.output, + lengthen=arguments.lengthen, + variants=arguments.variants, + quick=arguments.quick, + ) ) + +def _report(given: ReportArguments) -> int: + from sampletones_tools.codec.report.session import run_report + + for path in run_report(given.output): + print(f"Wrote {path}") + + return 0 + + +def _study(given: StudyArguments) -> int: from sampletones_tools.codec.study.session import resolve_manifest, run_study, variant_names manifest = resolve_manifest( diff --git a/tests/integration/assets/__init__.py b/src/sampletones_tools/codec/report/__init__.py similarity index 100% rename from tests/integration/assets/__init__.py rename to src/sampletones_tools/codec/report/__init__.py diff --git a/tests/integration/nsf/corpus.py b/src/sampletones_tools/codec/report/corpus.py similarity index 92% rename from tests/integration/nsf/corpus.py rename to src/sampletones_tools/codec/report/corpus.py index 20188586b..f220720a4 100644 --- a/tests/integration/nsf/corpus.py +++ b/src/sampletones_tools/codec/report/corpus.py @@ -30,7 +30,7 @@ from sampletones_player.compression.seeds import phrases_from_project from sampletones_player.song import Song from sampletones_shared.music import Tuning -from tests.integration.nsf.songs import RECORD_BYTES_PER_TICK, lengthened +from sampletones_tools.codec.report.songs import RECORD_BYTES_PER_TICK, lengthened ARRANGEMENT: Final[str] = "arrangement" LONG_ARRANGEMENT: Final[str] = "arrangement, three minutes" @@ -81,20 +81,20 @@ def _sample_project( def sample_entries( - instrument_catalog: Dict[str, Sample], + catalog: Dict[str, Sample], settings: ProjectSettings, ) -> Tuple[CorpusEntry, ...]: """Each catalog sample as a song of its own, played at the tuning it was reconstructed at. Args: - instrument_catalog: The samples the integration suite reads. + catalog: The samples of the synthetic corpus, by name. settings: The project settings a sample is seeded under. Returns: Tuple[CorpusEntry, ...]: One entry per sample, in catalog order. """ entries: List[CorpusEntry] = [] - for name, sample in instrument_catalog.items(): + for name, sample in catalog.items(): tuning = sample.reconstruction.config.library.tuning entries.append( CorpusEntry( @@ -225,24 +225,24 @@ def reconstruction_entry(name: str, seconds: int) -> CorpusEntry: ) -def build_corpus( - instrument_catalog: Dict[str, Sample], - integration_project: Project, +def corpus_entries( + catalog: Dict[str, Sample], + project: Project, ) -> Tuple[CorpusEntry, ...]: """The songs the codec is measured on: each sample alone, the arrangement at two lengths, and a minute of dense reconstruction. Args: - instrument_catalog: The samples the integration suite reads. - integration_project: The arrangement those samples are played in. + catalog: The samples of the synthetic corpus, by name. + project: The arrangement those samples are played in. Returns: Tuple[CorpusEntry, ...]: The samples first, then the arrangement, the long one, and the reconstruction. """ return ( - *sample_entries(instrument_catalog, integration_project.settings), - arrangement_entry(ARRANGEMENT, integration_project), - arrangement_entry(LONG_ARRANGEMENT, lengthened_arrangement(integration_project, TARGET_SECONDS)), + *sample_entries(catalog, project.settings), + arrangement_entry(ARRANGEMENT, project), + arrangement_entry(LONG_ARRANGEMENT, lengthened_arrangement(project, TARGET_SECONDS)), reconstruction_entry(RECONSTRUCTION, RECONSTRUCTION_SECONDS), ) diff --git a/src/sampletones_tools/codec/report/encoding.py b/src/sampletones_tools/codec/report/encoding.py new file mode 100644 index 000000000..16abe11c8 --- /dev/null +++ b/src/sampletones_tools/codec/report/encoding.py @@ -0,0 +1,221 @@ +from dataclasses import dataclass +from time import process_time +from typing import Final, List, Sequence, Tuple + +from sampletones_player.compression.compressed import CompressedPlanes +from sampletones_player.compression.dictionary.table import phrase_table +from sampletones_player.compression.encode import STREAM_START, encode_planes +from sampletones_player.compression.matches.cache import MatchCache +from sampletones_player.compression.matches.index import PlaneIndex +from sampletones_player.compression.matches.matcher import PhraseMatcher +from sampletones_player.compression.options import CodecOptions +from sampletones_player.compression.parse.plane import parse_plane +from sampletones_player.compression.planes.song import SongPlanes +from sampletones_player.registers.streams import ChannelStreams +from sampletones_player.specification.registers import DUTY_CYCLE_SHIFT +from sampletones_tools.codec.report.corpus import CorpusEntry +from sampletones_tools.codec.report.rows import ReportRow + +LITERALS: Final[str] = "literals" +HOLDS: Final[str] = "holds" +INSTRUMENTS: Final[str] = "instruments" +TRANSPOSITION: Final[str] = "transposition" +SEARCH: Final[str] = "search" +RECORDS: Final[str] = "records" +REGISTER_PLANES: Final[str] = "register planes" +SPLIT_CONTROL: Final[str] = "split control" +CONTROL_LEVEL_MASK: Final[int] = 0x3F +HOLDS_OPTIONS: Final[CodecOptions] = CodecOptions(holds=True, phrases=False, transposition=False, search=False) + +PLANE_VARIANTS: Final[Tuple[Tuple[str, CodecOptions], ...]] = ( + (LITERALS, CodecOptions(holds=False, phrases=False, transposition=False, search=False)), + (HOLDS, HOLDS_OPTIONS), + (INSTRUMENTS, CodecOptions(holds=True, phrases=True, transposition=False, search=False)), + (TRANSPOSITION, CodecOptions(holds=True, phrases=True, transposition=True, search=False)), + (SEARCH, CodecOptions(holds=True, phrases=True, transposition=True, search=True)), +) + + +@dataclass(frozen=True) +class Encoding: + """One corpus song compressed under one variant of the codec. + + Attributes: + entry: The song compressed. + variant: The layers the codec was switched on with. + planes: The planes the song separates into. + compressed: What the encoder wrote. + seconds: The processor time the encoding took. + """ + + entry: CorpusEntry + variant: str + planes: SongPlanes + compressed: CompressedPlanes + seconds: float + + @property + def size(self) -> int: + """The bytes the dictionary, the streams and the pitch table take together.""" + return self.compressed.size + len(self.entry.pitches.data) + + @property + def streams(self) -> int: + """The bytes the token streams take, which is the part that grows with the song.""" + return sum(len(stream) for stream in self.compressed.streams) + + +def encode_corpus(entries: Sequence[CorpusEntry]) -> Tuple[Encoding, ...]: + """Compresses every corpus song under every variant of the codec, timing each encoding. + + Args: + entries: The songs to compress. + + Returns: + Tuple[Encoding, ...]: The encodings, song by song, each in the order of the variants. + """ + encodings: List[Encoding] = [] + for entry in entries: + planes = entry.planes + for variant, options in PLANE_VARIANTS: + started = process_time() + compressed = encode_planes( + planes, + entry.seeds, + options=options, + boundaries=frozenset(), + ) + encodings.append( + Encoding( + entry=entry, + variant=variant, + planes=planes, + compressed=compressed, + seconds=process_time() - started, + ) + ) + + return tuple(encodings) + + +def report_rows( + entries: Sequence[CorpusEntry], + encodings: Sequence[Encoding], + space: int, +) -> Tuple[ReportRow, ...]: + """The report's measurements: per song, the baselines and then each of its encodings. + + Args: + entries: The songs measured. + encodings: Their encodings, as `encode_corpus` returns them. + space: The program area a song is written into. + + Returns: + Tuple[ReportRow, ...]: The rows, in the order the report prints them. + """ + rows: List[ReportRow] = [] + for entry in entries: + rows.extend(_baseline_rows(entry, space)) + rows.extend(_encoded_row(encoding, space) for encoding in encodings if encoding.entry is entry) + + return tuple(rows) + + +def _encoded_row(encoding: Encoding, space: int) -> ReportRow: + entry = encoding.entry + return ReportRow( + corpus=entry.name, + variant=encoding.variant, + ticks=entry.song.ticks, + size=encoding.size, + variable=encoding.streams, + phrases=len(encoding.compressed.phrases), + dictionary=encoding.compressed.phrases.size, + seconds=encoding.seconds, + records=entry.records, + space=space, + ) + + +def _baseline_rows(entry: CorpusEntry, space: int) -> Tuple[ReportRow, ...]: + return ( + ReportRow( + corpus=entry.name, + variant=RECORDS, + ticks=entry.song.ticks, + size=entry.records, + variable=entry.records, + phrases=0, + dictionary=0, + seconds=0.0, + records=entry.records, + space=space, + ), + _measured_row(entry, REGISTER_PLANES, _register_planes(entry.song.streams), 0, space), + _measured_row( + entry, + SPLIT_CONTROL, + _split_control_planes(entry.planes), + len(entry.pitches.data), + space, + ), + ) + + +def _measured_row( + entry: CorpusEntry, + variant: str, + planes: Sequence[bytes], + fixed: int, + space: int, +) -> ReportRow: + started = process_time() + coded = _coded_size(planes, HOLDS_OPTIONS) + return ReportRow( + corpus=entry.name, + variant=variant, + ticks=entry.song.ticks, + size=fixed + coded, + variable=coded, + phrases=0, + dictionary=0, + seconds=process_time() - started, + records=entry.records, + space=space, + ) + + +def _register_planes(streams: ChannelStreams) -> Tuple[bytes, ...]: + return tuple( + bytes(tick.values[register] for tick in stream) + for stream in streams.padded + for register in range(len(stream[0].values)) + ) + + +def _split_control_planes(planes: SongPlanes) -> Tuple[bytes, ...]: + split: List[bytes] = [] + for channels in planes.ordered: + if channels in (planes.pulse1, planes.pulse2): + split.append(bytes(control >> DUTY_CYCLE_SHIFT for control in channels.control)) + split.append(bytes(control & CONTROL_LEVEL_MASK for control in channels.control)) + else: + split.append(channels.control) + + split.append(channels.value) + + return tuple(split) + + +def _coded_size(planes: Sequence[bytes], options: CodecOptions) -> int: + cache = MatchCache(PlaneIndex.from_plane(plane) for plane in planes) + table = phrase_table(()) + entries = frozenset({STREAM_START}) + return sum( + parse_plane( + PhraseMatcher(table, plane, cache), + options, + entries, + ).size + for plane in range(len(planes)) + ) diff --git a/tests/integration/nsf/report.py b/src/sampletones_tools/codec/report/rows.py similarity index 100% rename from tests/integration/nsf/report.py rename to src/sampletones_tools/codec/report/rows.py diff --git a/src/sampletones_tools/codec/report/session.py b/src/sampletones_tools/codec/report/session.py new file mode 100644 index 000000000..e0e4d250d --- /dev/null +++ b/src/sampletones_tools/codec/report/session.py @@ -0,0 +1,56 @@ +from pathlib import Path +from tempfile import TemporaryDirectory +from typing import Final, Sequence, Tuple + +from sampletones_player.driver.image import DriverImage +from sampletones_player.specification.compression import PLANE_COUNT, PLANE_STATE_SIZE +from sampletones_tools.codec.report.corpus import CorpusEntry, corpus_entries +from sampletones_tools.codec.report.encoding import Encoding, encode_corpus, report_rows +from sampletones_tools.codec.report.rows import write_csv, write_markdown +from sampletones_tools.codec.report.songs import available_bytes +from sampletones_tools.corpus.build import build_corpus + +CSV_FILENAME: Final[str] = "report.csv" +MARKDOWN_FILENAME: Final[str] = "report.md" + + +def write_report( + entries: Sequence[CorpusEntry], + encodings: Sequence[Encoding], + space: int, + output: Path, +) -> Tuple[Path, Path]: + """Writes the measurements as a table another tool reads and a table a reader reads. + + Args: + entries: The songs measured. + encodings: Their encodings. + space: The program area a song is written into. + output: The directory the two tables are written into, created when missing. + + Returns: + Tuple[Path, Path]: The CSV table and the Markdown table. + """ + rows = report_rows(entries, encodings, space) + output.mkdir(parents=True, exist_ok=True) + csv_path = output / CSV_FILENAME + markdown_path = output / MARKDOWN_FILENAME + write_csv(rows, csv_path) + write_markdown(rows, markdown_path, PLANE_COUNT * PLANE_STATE_SIZE) + return csv_path, markdown_path + + +def run_report(output: Path) -> Tuple[Path, Path]: + """Builds the synthetic corpus, compresses it under every variant and writes the report. + + Args: + output: The directory the report is written into. + + Returns: + Tuple[Path, Path]: The CSV table and the Markdown table. + """ + with TemporaryDirectory() as recordings: + corpus = build_corpus(Path(recordings)) + + entries = corpus_entries(corpus.catalog, corpus.project) + return write_report(entries, encode_corpus(entries), available_bytes(DriverImage.load()), output) diff --git a/tests/integration/nsf/songs.py b/src/sampletones_tools/codec/report/songs.py similarity index 100% rename from tests/integration/nsf/songs.py rename to src/sampletones_tools/codec/report/songs.py diff --git a/src/sampletones_tools/corpus/__init__.py b/src/sampletones_tools/corpus/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/sampletones_tools/corpus/build.py b/src/sampletones_tools/corpus/build.py new file mode 100644 index 000000000..29485ea9f --- /dev/null +++ b/src/sampletones_tools/corpus/build.py @@ -0,0 +1,56 @@ +from dataclasses import dataclass +from typing import Dict + +from sampletones_core.project.project import Project +from sampletones_core.project.settings import ProjectSettings +from sampletones_core.project.voices.sample import Sample +from sampletones_shared.types.path import Pathlike +from sampletones_tools.corpus.catalog import CatalogSpec, build_catalog +from sampletones_tools.corpus.module import ModuleConfig +from sampletones_tools.corpus.song import SongSpec, build_song +from sampletones_tools.corpus.synth import SynthConfig + + +@dataclass(frozen=True) +class Corpus: + """The synthetic corpus: the reconstructed samples and the arrangement that plays them. + + Attributes: + catalog: The samples, by name. + project: The arrangement, carrying the samples as its voices. + """ + + catalog: Dict[str, Sample] + project: Project + + +def build_project( + catalog: Dict[str, Sample], + module_config: ModuleConfig, + song_spec: SongSpec, +) -> Project: + """The arrangement playing the catalog under the module's identity and playback settings.""" + settings = ProjectSettings( + tempo=module_config.tempo, + speed=module_config.speed, + nes_frequency=module_config.nes_frequency, + ) + project = Project.create(title=module_config.title, author=module_config.author, settings=settings) + for sample in catalog.values(): + project.voices.append(sample) + + project.song = build_song(song_spec, catalog) + return project + + +def build_corpus(tmp_dir: Pathlike) -> Corpus: + """Renders, reconstructs and arranges the corpus the package describes. + + Args: + tmp_dir: Where the rendered recordings are written before they are reconstructed. + + Returns: + Corpus: The samples and the arrangement. + """ + catalog = build_catalog(CatalogSpec.load(), SynthConfig.load(), tmp_dir=tmp_dir) + return Corpus(catalog=catalog, project=build_project(catalog, ModuleConfig.load(), SongSpec.load())) diff --git a/src/sampletones_tools/corpus/catalog.py b/src/sampletones_tools/corpus/catalog.py new file mode 100644 index 000000000..a8bf761a9 --- /dev/null +++ b/src/sampletones_tools/corpus/catalog.py @@ -0,0 +1,218 @@ +from pathlib import Path +from typing import Dict, Final, FrozenSet, List, Self, Sequence, Tuple + +import numpy as np +from pydantic import BaseModel, ConfigDict + +from sampletones_core.audio import write_wave +from sampletones_core.audio.processing import normalize +from sampletones_core.configs import Config, InstructionsLibraryConfig +from sampletones_core.constants.enums import ChannelName, GeneratorClassName, SpectrumMethod +from sampletones_core.fft import Window +from sampletones_core.fft.features import get_feature_extractor +from sampletones_core.generators import get_generators_by_channels +from sampletones_core.instructions import InstructionUnion +from sampletones_core.library import InstructionLibrary, InstructionLibraryData +from sampletones_core.library.creator.creation import generate_instruction +from sampletones_core.project.voices.sample import Sample +from sampletones_core.reconstructions import Reconstruction, Reconstructor +from sampletones_shared.types.path import Pathlike +from sampletones_shared.utils.serialization import load_yaml_model +from sampletones_tools.corpus.paths import CATALOG_CONFIG_PATH +from sampletones_tools.corpus.synth import SynthConfig + +INSTRUCTIONS_PER_GENERATOR: Final[int] = 48 +CHANNELS: Final[List[ChannelName]] = [ + ChannelName.PULSE1, + ChannelName.TRIANGLE, + ChannelName.NOISE, +] + + +class ReconstructionSettings(BaseModel): + """The settings the one in-memory library every instrument is matched against is built with.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + spectrum_method: SpectrumMethod + transformation_gamma: int + instructions_per_generator: int + + +class InstrumentSpec(BaseModel): + """One instrument of the catalog: the voice it is rendered from and the channels it covers.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + name: str + synth: str + channels: List[ChannelName] + + +class CatalogSpec(BaseModel): + """The reconstructed sample catalog: the library settings and the instruments built under them.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + reconstruction: ReconstructionSettings + instruments: List[InstrumentSpec] + + @classmethod + def load(cls) -> Self: + """The catalog the package ships.""" + return load_yaml_model(CATALOG_CONFIG_PATH, cls) + + +def build_mini_library( + config: Config, + *, + per_generator: int = INSTRUCTIONS_PER_GENERATOR, +) -> InstructionLibrary: + """Builds a small in-memory instruction library covering pulse/triangle/noise. + + Candidates are sampled with an even stride across each channel's instruction + space so pitch, volume and period are represented, rather than a biased prefix. + """ + window = Window.from_config(config) + extractor = get_feature_extractor(config, window) + generators = { + generator.class_name(): generator for generator in get_generators_by_channels(config, CHANNELS).values() + } + sampled: List[Tuple[GeneratorClassName, InstructionUnion]] = [ + (class_name, instruction) + for class_name, generator in generators.items() + for instruction in _evenly_sampled(generator.get_possible_instructions(), per_generator) + ] + data = dict( + generate_instruction(generators, class_name, instruction, extractor) for class_name, instruction in sampled + ) + + library = InstructionLibrary() + library.data[library.create_key(config, window)] = InstructionLibraryData.create(config, data) + return library + + +def _evenly_sampled(candidates: Sequence[InstructionUnion], count: int) -> List[InstructionUnion]: + stride = max(1, len(candidates) // count) + return list(candidates[::stride][:count]) + + +def reconstruct_sample( + audio: np.ndarray, + config: Config, + library: InstructionLibrary, + channels: FrozenSet[ChannelName], + *, + tmp_dir: Pathlike, + name: str, +) -> Reconstruction: + """Runs the real reconstruction pipeline on ``audio`` via a temp WAV. + + Raises: + ValueError: If the reconstructor returns no reconstruction for the recording. + """ + path = Path(tmp_dir) / f"{name}.wav" + write_wave(path, config.library.sample_rate, audio) + reconstruction = Reconstructor(config, channels, library=library)(path) + if reconstruction is None: + raise ValueError(f"Reconstruction of '{name}' produced no result") + + return reconstruction + + +def make_sample( + name: str, + audio: np.ndarray, + config: Config, + library: InstructionLibrary, + *, + tmp_dir: Pathlike, + expected_slices: FrozenSet[ChannelName], +) -> Sample: + """Reconstructs ``audio`` into a `Sample` playing exactly the expected channels. + + Raises: + ValueError: If the reconstruction plays other channels than ``expected_slices``. + """ + reconstruction = reconstruct_sample( + audio, + config, + library, + expected_slices, + tmp_dir=tmp_dir, + name=name, + ) + played = frozenset(reconstruction.playing_channels) + if played != expected_slices: + raise ValueError(f"Sample '{name}' covers {set(played)}, expected {set(expected_slices)}") + + return Sample(name=name, reconstruction=reconstruction) + + +def build_catalog( + spec: CatalogSpec, + synth_config: SynthConfig, + *, + tmp_dir: Pathlike, +) -> Dict[str, Sample]: + """Builds the reconstructed sample catalog the spec describes. + + The spec fixes the reconstruction settings (spectrum method, gamma, library size) shared by + one in-memory library, and per instrument the voice to render and the channel slices to + cover. Voice parameters come from ``synth_config``. + + Args: + spec: The catalog: the library settings and the instruments. + synth_config: The voices the instruments are rendered from. + tmp_dir: Where the rendered recordings are written before they are reconstructed. + + Returns: + Dict[str, Sample]: The samples, by name, in the order the spec lists them. + """ + settings = spec.reconstruction + library_config = InstructionsLibraryConfig( + spectrum_method=settings.spectrum_method, + transformation_gamma=settings.transformation_gamma, + ) + sample_rate = library_config.sample_rate + library = build_mini_library( + Config(library=library_config), + per_generator=settings.instructions_per_generator, + ) + + catalog: Dict[str, Sample] = {} + for instrument in spec.instruments: + config = Config(library=library_config) + audio = _render_instrument(synth_config, instrument.synth, sample_rate=sample_rate) + catalog[instrument.name] = make_sample( + instrument.name, + audio, + config, + library, + tmp_dir=tmp_dir, + expected_slices=frozenset(instrument.channels), + ) + + return catalog + + +def _render_instrument( + synth_config: SynthConfig, + name: str, + *, + sample_rate: int, +) -> np.ndarray: + """Renders a named voice at peak level 1.0. + + A fresh channel seeded from the synth configuration keeps every instrument reproducible + independently of catalog order. + """ + voice = synth_config.voices[name] + generator = np.random.default_rng(synth_config.seed) + instrument: np.ndarray = normalize( + voice.render( + sample_rate=sample_rate, + generator=generator, + ) + ) + return instrument diff --git a/src/sampletones_tools/corpus/config/__init__.py b/src/sampletones_tools/corpus/config/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/integration/config/module.yaml b/src/sampletones_tools/corpus/config/module.yaml similarity index 100% rename from tests/integration/config/module.yaml rename to src/sampletones_tools/corpus/config/module.yaml diff --git a/tests/integration/config/reconstruction.yaml b/src/sampletones_tools/corpus/config/reconstruction.yaml similarity index 100% rename from tests/integration/config/reconstruction.yaml rename to src/sampletones_tools/corpus/config/reconstruction.yaml diff --git a/tests/integration/config/song.yaml b/src/sampletones_tools/corpus/config/song.yaml similarity index 100% rename from tests/integration/config/song.yaml rename to src/sampletones_tools/corpus/config/song.yaml diff --git a/tests/integration/config/synth.yaml b/src/sampletones_tools/corpus/config/synth.yaml similarity index 100% rename from tests/integration/config/synth.yaml rename to src/sampletones_tools/corpus/config/synth.yaml diff --git a/src/sampletones_tools/corpus/module.py b/src/sampletones_tools/corpus/module.py new file mode 100644 index 000000000..c8796d2fc --- /dev/null +++ b/src/sampletones_tools/corpus/module.py @@ -0,0 +1,23 @@ +from typing import Self + +from pydantic import BaseModel, ConfigDict + +from sampletones_shared.utils.serialization import load_yaml_model +from sampletones_tools.corpus.paths import MODULE_CONFIG_PATH + + +class ModuleConfig(BaseModel): + """Module identity and playback settings that shape the exported files.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + title: str + author: str + tempo: int + speed: int + nes_frequency: int + + @classmethod + def load(cls) -> Self: + """The module settings the package ships.""" + return load_yaml_model(MODULE_CONFIG_PATH, cls) diff --git a/src/sampletones_tools/corpus/paths.py b/src/sampletones_tools/corpus/paths.py new file mode 100644 index 000000000..3ee0a3057 --- /dev/null +++ b/src/sampletones_tools/corpus/paths.py @@ -0,0 +1,9 @@ +from importlib.resources import files +from pathlib import Path +from typing import Final + +CONFIG_DIRECTORY: Final[Path] = Path(str(files("sampletones_tools.corpus.config"))) +SYNTH_CONFIG_PATH: Final[Path] = CONFIG_DIRECTORY / "synth.yaml" +CATALOG_CONFIG_PATH: Final[Path] = CONFIG_DIRECTORY / "reconstruction.yaml" +MODULE_CONFIG_PATH: Final[Path] = CONFIG_DIRECTORY / "module.yaml" +SONG_PATH: Final[Path] = CONFIG_DIRECTORY / "song.yaml" diff --git a/src/sampletones_tools/corpus/song.py b/src/sampletones_tools/corpus/song.py new file mode 100644 index 000000000..fe899938e --- /dev/null +++ b/src/sampletones_tools/corpus/song.py @@ -0,0 +1,121 @@ +from typing import Dict, List, Mapping, Optional, Self, Sequence + +from pydantic import BaseModel, ConfigDict + +from sampletones_core.constants.enums import ChannelName +from sampletones_core.project.patterns.channel import Channel +from sampletones_core.project.patterns.pattern import Pattern +from sampletones_core.project.patterns.row import Row +from sampletones_core.project.song import Song +from sampletones_core.project.voices.note_off import NoteOff +from sampletones_core.project.voices.note_on import NoteOn +from sampletones_core.project.voices.sample import Sample +from sampletones_shared.utils.serialization import load_yaml_model +from sampletones_tools.corpus.paths import SONG_PATH + + +class RowSpec(BaseModel): + """One written row of a pattern: what it plays, or that it releases the note. + + Attributes: + row: The row's index in its pattern. + sample: The sample the row plays, or ``None`` for a row that plays none. + transpose: The semitones the row transposes the sample by, or ``None`` to leave it. + volume: The volume the row sets, or ``None`` to leave it. + off: Whether the row releases the note. + """ + + model_config = ConfigDict(frozen=True, extra="forbid") + + row: int + sample: Optional[str] = None + transpose: Optional[int] = None + volume: Optional[int] = None + off: bool = False + + +class ChannelSpec(BaseModel): + """The patterns one channel holds, each as the rows written into it.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + patterns: Dict[int, List[RowSpec]] + + +class SongSpec(BaseModel): + """The arrangement: the pattern length, the order and the channels' patterns.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + rows_per_pattern: int + order: List[Dict[ChannelName, int]] + channels: Dict[ChannelName, ChannelSpec] + + @classmethod + def load(cls) -> Self: + """The arrangement the package ships.""" + return load_yaml_model(SONG_PATH, cls) + + +def _order(frames: Sequence[Mapping[ChannelName, int]]) -> List[Dict[ChannelName, Optional[int]]]: + return [{channel: frame.get(channel) for channel in ChannelName.items()} for frame in frames] + + +def _row(spec: RowSpec, channel: ChannelName, samples_by_name: Mapping[str, Sample]) -> Row: + if spec.off: + return Row(command=NoteOff(), volume=spec.volume) + + if spec.sample is None: + return Row(transpose=spec.transpose, volume=spec.volume) + + sample = samples_by_name[spec.sample] + if channel not in sample.reconstruction.playing_channels: + raise ValueError(f"Sample '{spec.sample}' has no '{channel.value}' slice for the {channel.value} channel") + + return Row(command=NoteOn(voice_id=sample.id), transpose=spec.transpose, volume=spec.volume) + + +def _pattern( + row_specs: Sequence[RowSpec], + rows_per_pattern: int, + channel: ChannelName, + samples_by_name: Mapping[str, Sample], +) -> Pattern: + rows = [Row() for _ in range(rows_per_pattern)] + for spec in row_specs: + rows[spec.row] = _row(spec, channel, samples_by_name) + + return Pattern(rows=rows) + + +def _channels( + channel_specs: Mapping[ChannelName, ChannelSpec], + rows_per_pattern: int, + samples_by_name: Mapping[str, Sample], +) -> Dict[ChannelName, Channel]: + channels: Dict[ChannelName, Channel] = {} + for channel, spec in channel_specs.items(): + patterns = { + index: _pattern(row_specs, rows_per_pattern, channel, samples_by_name) + for index, row_specs in spec.patterns.items() + } + channels[channel] = Channel(name=channel, patterns=patterns) + + for channel in ChannelName.items(): + channels.setdefault(channel, Channel(name=channel, patterns={})) + + return channels + + +def build_song(spec: SongSpec, samples_by_name: Mapping[str, Sample]) -> Song: + """The song the spec describes, playing the named samples. + + Raises: + KeyError: If a row names a sample the catalog lacks. + ValueError: If a row plays a sample on a channel the sample has no slice for. + """ + return Song( + rows_per_pattern=spec.rows_per_pattern, + order=_order(spec.order), + channels=_channels(spec.channels, spec.rows_per_pattern, samples_by_name), + ) diff --git a/src/sampletones_tools/corpus/synth.py b/src/sampletones_tools/corpus/synth.py new file mode 100644 index 000000000..f7fab5a43 --- /dev/null +++ b/src/sampletones_tools/corpus/synth.py @@ -0,0 +1,21 @@ +from typing import Dict, Self + +from pydantic import BaseModel, ConfigDict + +from sampletones_shared.utils.serialization import load_yaml_model +from sampletones_tools.corpus.paths import SYNTH_CONFIG_PATH +from sampletones_tools.synthesis.voice.voice import Voice + + +class SynthConfig(BaseModel): + """The named synthesizer voices the corpus is rendered from and their shared noise seed.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + seed: int + voices: Dict[str, Voice] + + @classmethod + def load(cls) -> Self: + """The voices the package ships.""" + return load_yaml_model(SYNTH_CONFIG_PATH, cls) diff --git a/src/sampletones_tools/registry.py b/src/sampletones_tools/registry.py index 5ba1fcb58..938fa119f 100644 --- a/src/sampletones_tools/registry.py +++ b/src/sampletones_tools/registry.py @@ -6,6 +6,8 @@ from sampletones_tools.checks.command import CHECK from sampletones_tools.codec.command import CODEC from sampletones_tools.player.command import DRIVER -from sampletones_tools.samples.command import NSF +from sampletones_tools.samples.commands.btp import BTP +from sampletones_tools.samples.commands.ftm import FTM +from sampletones_tools.samples.commands.nsf import NSF -DEVELOPER_COMMANDS: Final[Tuple[Command, ...]] = (CALIBRATION, CHECK, CODEC, DRIVER, ICONS, NSF) +DEVELOPER_COMMANDS: Final[Tuple[Command, ...]] = (BTP, CALIBRATION, CHECK, CODEC, DRIVER, FTM, ICONS, NSF) diff --git a/src/sampletones_tools/samples/bitphase.py b/src/sampletones_tools/samples/bitphase.py new file mode 100644 index 000000000..d8debf72c --- /dev/null +++ b/src/sampletones_tools/samples/bitphase.py @@ -0,0 +1,39 @@ +from pathlib import Path +from typing import Final, List + +from sampletones_core.formats.bitphase.btp import write_btp +from sampletones_core.formats.bitphase.builder import project_to_bitphase +from sampletones_core.project.project import Project +from sampletones_tools.corpus.build import Corpus + +DOCUMENT_FILENAME: Final[str] = "drums.btp" +GROOVE_DOCUMENT_FILENAME: Final[str] = "drums-groove.btp" +GROOVE_TEMPO: Final[int] = 210 + + +def at_tempo(project: Project, tempo: int) -> Project: + """The same project played at another tempo, leaving the given project as it is.""" + return Project( + metadata=project.metadata, + info=project.info, + settings=project.settings.model_copy(update={"tempo": tempo}), + voices=project.voices, + song=project.song, + ) + + +def write_samples(corpus: Corpus, output: Path) -> List[Path]: + """Writes the corpus arrangement as two Bitphase documents: at its own tempo and as a groove. + + Args: + corpus: The samples and the arrangement. + output: The directory the documents are written into. + + Returns: + List[Path]: The document at the song's tempo, then the one carrying a groove. + """ + document = output / DOCUMENT_FILENAME + write_btp(document, project_to_bitphase(corpus.project)) + groove_document = output / GROOVE_DOCUMENT_FILENAME + write_btp(groove_document, project_to_bitphase(at_tempo(corpus.project, GROOVE_TEMPO))) + return [document, groove_document] diff --git a/src/sampletones_tools/samples/commands/__init__.py b/src/sampletones_tools/samples/commands/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/sampletones_tools/samples/commands/btp.py b/src/sampletones_tools/samples/commands/btp.py new file mode 100644 index 000000000..46605000a --- /dev/null +++ b/src/sampletones_tools/samples/commands/btp.py @@ -0,0 +1,34 @@ +from argparse import ArgumentParser, Namespace +from typing import Final + +from sampletones_shared.command import Command +from sampletones_tools.samples.commands.options import ( + ACTION_FIELD, + ACTION_METAVAR, + SAMPLES, + SamplesArguments, + add_output_option, + print_written, +) + +NAME: Final[str] = "btp" +HELP: Final[str] = "write example Bitphase documents (.btp) from the synthetic corpus" +SAMPLES_HELP: Final[str] = "write the corpus arrangement as Bitphase documents (.btp), at its own tempo and as a groove" + + +def configure(parser: ArgumentParser) -> None: + actions = parser.add_subparsers(dest=ACTION_FIELD, metavar=ACTION_METAVAR, required=True) + add_output_option(actions.add_parser(SAMPLES, help=SAMPLES_HELP, description=SAMPLES_HELP)) + + +def run(arguments: Namespace) -> int: + """Builds the corpus, writes its arrangement and prints each file written.""" + given = SamplesArguments(output=arguments.output) + + from sampletones_tools.samples.bitphase import write_samples + from sampletones_tools.samples.emit import emit_samples + + return print_written(emit_samples(given.output, write_samples)) + + +BTP: Final[Command] = Command(name=NAME, help=HELP, configure=configure, run=run) diff --git a/src/sampletones_tools/samples/commands/ftm.py b/src/sampletones_tools/samples/commands/ftm.py new file mode 100644 index 000000000..2355efa7a --- /dev/null +++ b/src/sampletones_tools/samples/commands/ftm.py @@ -0,0 +1,34 @@ +from argparse import ArgumentParser, Namespace +from typing import Final + +from sampletones_shared.command import Command +from sampletones_tools.samples.commands.options import ( + ACTION_FIELD, + ACTION_METAVAR, + SAMPLES, + SamplesArguments, + add_output_option, + print_written, +) + +NAME: Final[str] = "ftm" +HELP: Final[str] = "write an example FamiTracker module (.ftm) from the synthetic corpus" +SAMPLES_HELP: Final[str] = "write the corpus arrangement as a FamiTracker module (.ftm)" + + +def configure(parser: ArgumentParser) -> None: + actions = parser.add_subparsers(dest=ACTION_FIELD, metavar=ACTION_METAVAR, required=True) + add_output_option(actions.add_parser(SAMPLES, help=SAMPLES_HELP, description=SAMPLES_HELP)) + + +def run(arguments: Namespace) -> int: + """Builds the corpus, writes its arrangement and prints each file written.""" + given = SamplesArguments(output=arguments.output) + + from sampletones_tools.samples.emit import emit_samples + from sampletones_tools.samples.famitracker import write_samples + + return print_written(emit_samples(given.output, write_samples)) + + +FTM: Final[Command] = Command(name=NAME, help=HELP, configure=configure, run=run) diff --git a/src/sampletones_tools/samples/command.py b/src/sampletones_tools/samples/commands/nsf.py similarity index 61% rename from src/sampletones_tools/samples/command.py rename to src/sampletones_tools/samples/commands/nsf.py index 7b9aec954..014b96d85 100644 --- a/src/sampletones_tools/samples/command.py +++ b/src/sampletones_tools/samples/commands/nsf.py @@ -4,11 +4,18 @@ from typing import Final from sampletones_shared.command import Command +from sampletones_tools.samples.commands.options import ( + ACTION_FIELD, + ACTION_METAVAR, + SAMPLES, + SamplesArguments, + add_output_option, + print_written, +) NAME: Final[str] = "nsf" -HELP: Final[str] = "render exported .nsf files to waves" -ACTION_FIELD: Final[str] = "action" -ACTION_METAVAR: Final[str] = "" +HELP: Final[str] = "write example .nsf files from the synthetic corpus and render .nsf files to waves" +SAMPLES_HELP: Final[str] = "write each corpus sample and the corpus arrangement as .nsf files" RENDER: Final[str] = "render" RENDER_HELP: Final[str] = "render the .nsf files in a directory to waves through ffmpeg's libgme demuxer" DIRECTORY_HELP: Final[str] = "the directory holding the exported .nsf files" @@ -26,19 +33,34 @@ class RenderArguments: def configure(parser: ArgumentParser) -> None: actions = parser.add_subparsers(dest=ACTION_FIELD, metavar=ACTION_METAVAR, required=True) + samples = actions.add_parser(SAMPLES, help=SAMPLES_HELP, description=SAMPLES_HELP) + add_output_option(samples) render = actions.add_parser(RENDER, help=RENDER_HELP, description=RENDER_HELP) render.add_argument("--directory", type=Path, required=True, help=DIRECTORY_HELP) render.add_argument("--tail", type=float, default=DEFAULT_TAIL_SECONDS, help=TAIL_HELP) def run(arguments: Namespace) -> int: + """Writes the example files or renders exported ones, as the action names.""" + if arguments.action == SAMPLES: + return _write_samples(SamplesArguments(output=arguments.output)) + + return _render(RenderArguments(directory=arguments.directory, tail=arguments.tail)) + + +def _write_samples(given: SamplesArguments) -> int: + from sampletones_tools.samples.emit import emit_samples + from sampletones_tools.samples.nsf import write_samples + + return print_written(emit_samples(given.output, write_samples)) + + +def _render(given: RenderArguments) -> int: """Renders the exported files and prints each wave written. Raises: SystemExit: If ffmpeg is unusable or rejects a file. """ - given = RenderArguments(directory=arguments.directory, tail=arguments.tail) - from sampletones_tools.samples.render import RenderingError, render_directory try: diff --git a/src/sampletones_tools/samples/commands/options.py b/src/sampletones_tools/samples/commands/options.py new file mode 100644 index 000000000..1b4343472 --- /dev/null +++ b/src/sampletones_tools/samples/commands/options.py @@ -0,0 +1,29 @@ +from argparse import ArgumentParser +from dataclasses import dataclass +from pathlib import Path +from typing import Final, Sequence + +ACTION_FIELD: Final[str] = "action" +ACTION_METAVAR: Final[str] = "" +SAMPLES: Final[str] = "samples" +OUTPUT_HELP: Final[str] = "the directory the files are written into, created when missing" + + +@dataclass(frozen=True) +class SamplesArguments: + """What an emitter run is given: the directory its files are written into.""" + + output: Path + + +def add_output_option(parser: ArgumentParser) -> None: + """Adds the required output directory an emitter writes into.""" + parser.add_argument("--output", "-o", type=Path, required=True, help=OUTPUT_HELP) + + +def print_written(paths: Sequence[Path]) -> int: + """Prints each file an emitter wrote and returns the command's success status.""" + for path in paths: + print(f"Wrote {path}") + + return 0 diff --git a/src/sampletones_tools/samples/emit.py b/src/sampletones_tools/samples/emit.py new file mode 100644 index 000000000..18e05ad3e --- /dev/null +++ b/src/sampletones_tools/samples/emit.py @@ -0,0 +1,24 @@ +from pathlib import Path +from tempfile import TemporaryDirectory +from typing import Callable, List + +from sampletones_tools.corpus.build import Corpus, build_corpus + +Emitter = Callable[[Corpus, Path], List[Path]] + + +def emit_samples(output: Path, emitter: Emitter) -> List[Path]: + """Builds the synthetic corpus and hands it to an emitter writing into ``output``. + + Args: + output: The directory the files are written into, created when missing. + emitter: The writer of one format. + + Returns: + List[Path]: The files the emitter wrote. + """ + with TemporaryDirectory() as recordings: + corpus = build_corpus(Path(recordings)) + + output.mkdir(parents=True, exist_ok=True) + return emitter(corpus, output) diff --git a/src/sampletones_tools/samples/famitracker.py b/src/sampletones_tools/samples/famitracker.py new file mode 100644 index 000000000..a3dba6459 --- /dev/null +++ b/src/sampletones_tools/samples/famitracker.py @@ -0,0 +1,22 @@ +from pathlib import Path +from typing import Final, List + +from sampletones_core.formats.famitracker.export import write_ftm +from sampletones_tools.corpus.build import Corpus + +MODULE_FILENAME: Final[str] = "drums.ftm" + + +def write_samples(corpus: Corpus, output: Path) -> List[Path]: + """Writes the corpus arrangement as one FamiTracker module. + + Args: + corpus: The samples and the arrangement. + output: The directory the module is written into. + + Returns: + List[Path]: The module written. + """ + destination = output / MODULE_FILENAME + write_ftm(destination, corpus.project) + return [destination] diff --git a/src/sampletones_tools/samples/nsf.py b/src/sampletones_tools/samples/nsf.py new file mode 100644 index 000000000..1831fad73 --- /dev/null +++ b/src/sampletones_tools/samples/nsf.py @@ -0,0 +1,47 @@ +from pathlib import Path +from typing import Final, List + +from sampletones_core.exports.request import ProjectExport +from sampletones_player.builder import song_from_reconstruction +from sampletones_player.driver.image import DriverImage +from sampletones_player.export import NSFBackend +from sampletones_player.nsf.file import write_nsf +from sampletones_player.nsf.information import NSFInformation +from sampletones_shared.paths.extensions import EXT_FILE_NSF +from sampletones_tools.corpus.build import Corpus + +ARTIST: Final[str] = "Integration" +SONG_NAME: Final[str] = "song" + + +def exported_information(name: str) -> NSFInformation: + """The header text an exported sample carries.""" + return NSFInformation(title=name, artist=ARTIST) + + +def write_samples(corpus: Corpus, output: Path) -> List[Path]: + """Writes each corpus sample as a program of its own, then the arrangement through the backend. + + Args: + corpus: The samples and the arrangement. + output: The directory the files are written into. + + Returns: + List[Path]: The files written, the samples first and the arrangement last. + """ + image = DriverImage.load() + written: List[Path] = [] + for name, sample in corpus.catalog.items(): + destination = output / f"{name}{EXT_FILE_NSF}" + write_nsf( + destination, + song_from_reconstruction(sample.reconstruction, loop_tick=None), + exported_information(name), + image, + ) + written.append(destination) + + arrangement = output / f"{SONG_NAME}{EXT_FILE_NSF}" + NSFBackend().write_project(arrangement, ProjectExport(project=corpus.project)) + written.append(arrangement) + return written diff --git a/tests/benchmarks/test_compression.py b/tests/benchmarks/test_compression.py index 083074ef8..0753900f3 100644 --- a/tests/benchmarks/test_compression.py +++ b/tests/benchmarks/test_compression.py @@ -6,7 +6,7 @@ from sampletones_core.project.project import Project from sampletones_player.compression.encode import encode_planes from sampletones_player.compression.options import EVERY_LAYER -from tests.integration.nsf.corpus import ( +from sampletones_tools.codec.report.corpus import ( LONG_ARRANGEMENT, RECONSTRUCTION, RECONSTRUCTION_SECONDS, diff --git a/tests/integration/assets/module_config.py b/tests/integration/assets/module_config.py deleted file mode 100644 index 3b52384d5..000000000 --- a/tests/integration/assets/module_config.py +++ /dev/null @@ -1,20 +0,0 @@ -from pydantic import BaseModel, ConfigDict - -from sampletones_shared.types.path import Pathlike -from sampletones_shared.utils.serialization import load_yaml - - -class ModuleConfig(BaseModel): - """Module identity and playback settings that shape the exported ``.ftm``.""" - - model_config = ConfigDict(frozen=True, extra="forbid") - - title: str - author: str - tempo: int - speed: int - nes_frequency: int - - -def load_module_config(path: Pathlike) -> ModuleConfig: - return ModuleConfig.model_validate(load_yaml(path)) diff --git a/tests/integration/assets/reconstruction.py b/tests/integration/assets/reconstruction.py deleted file mode 100644 index a03457c13..000000000 --- a/tests/integration/assets/reconstruction.py +++ /dev/null @@ -1,240 +0,0 @@ -from pathlib import Path -from typing import Any, Dict, Final, FrozenSet, List, Tuple - -import numpy as np - -from sampletones_core.audio import write_wave -from sampletones_core.audio.processing import normalize -from sampletones_core.configs import Config, InstructionsLibraryConfig -from sampletones_core.configs.generation import GenerationConfig -from sampletones_core.constants.enums import ( - DEFAULT_CHANNELS, - ChannelName, - HierarchyMode, - SpectrumMethod, - bending_channels, -) -from sampletones_core.fft import Window -from sampletones_core.fft.features import get_feature_extractor -from sampletones_core.generators import get_generators_by_channels -from sampletones_core.instructions import InstructionUnion -from sampletones_core.library import ( - InstructionLibrary, - InstructionLibraryData, - InstructionLibraryFragment, -) -from sampletones_core.project.voices.sample import Sample -from sampletones_core.reconstructions import Reconstruction, Reconstructor -from sampletones_core.reconstructions.reconstructor.stems.configs.config import StemsConfig -from sampletones_core.reconstructions.reconstructor.stems.configs.entry import StemEntry -from sampletones_core.reconstructions.reconstructor.stems.configs.hierarchy import StemsHierarchy -from sampletones_core.reconstructions.reconstructor.stems.configs.settings import StemSettings -from sampletones_shared.types.path import Pathlike -from sampletones_shared.utils.serialization import load_yaml -from tests.integration.assets.synth_config import SynthConfig - -INSTRUCTIONS_PER_GENERATOR: Final[int] = 48 -CHANNELS: Final[List[ChannelName]] = [ - ChannelName.PULSE1, - ChannelName.TRIANGLE, - ChannelName.NOISE, -] - -STEM_A_ID: Final[int] = 0 -STEM_B_ID: Final[int] = 1 -STEM_C_ID: Final[int] = 2 -THREE_STEM_CHANNELS: Final[List[ChannelName]] = [ - ChannelName.PULSE1, - ChannelName.PULSE2, - ChannelName.TRIANGLE, - ChannelName.NOISE, -] -THREE_STEM_ENTRY_CHANNELS: Final[Dict[int, List[ChannelName]]] = { - STEM_A_ID: [ChannelName.PULSE1, ChannelName.TRIANGLE, ChannelName.NOISE], - STEM_B_ID: [ChannelName.PULSE2, ChannelName.TRIANGLE], - STEM_C_ID: [ChannelName.PULSE1, ChannelName.NOISE], -} -STEM_RECORDING_DURATION_SECONDS: Final[float] = 0.5 - - -def three_stem_config() -> StemsConfig: - """Builds the three-stem setup the stems tests share. - - Stems a (pulse 1, triangle, noise) and b (pulse 2, triangle) pick on the first - hierarchy level, stem c (pulse 1, noise) on the second. - """ - return StemsConfig( - entries=[ - StemEntry(id=stem_id, settings=StemSettings(channels=channels, bends=bending_channels(channels))) - for stem_id, channels in THREE_STEM_ENTRY_CHANNELS.items() - ], - hierarchy=StemsHierarchy( - levels=[[STEM_A_ID, STEM_B_ID], [STEM_C_ID]], - mode=HierarchyMode.STRICT, - ), - channel_cap=1, - ) - - -def three_stem_reconstruction_config() -> Config: - """Builds a reconstruction config for the three-stem example.""" - return Config() - - -def write_three_stem_recordings( - config: Config, - tmp_dir: Pathlike, -) -> Tuple[Path, Path, Path]: - """Writes three distinct stem recordings a, b, c and returns their paths in order.""" - sample_rate = config.library.sample_rate - count = int(sample_rate * STEM_RECORDING_DURATION_SECONDS) - time = np.arange(count) / sample_rate - recordings = { - "a": 0.5 * np.sin(2 * np.pi * 440.0 * time), - "b": 0.4 * np.sin(2 * np.pi * 220.0 * time), - "c": np.random.default_rng(93).uniform(-0.3, 0.3, count), - } - - paths: List[Path] = [] - for name, audio in recordings.items(): - path = Path(tmp_dir) / f"stem_{name}.wav" - write_wave(path, sample_rate, audio) - paths.append(path) - - return paths[0], paths[1], paths[2] - - -def build_mini_library( - config: Config, - *, - per_generator: int = INSTRUCTIONS_PER_GENERATOR, -) -> InstructionLibrary: - """Builds a small in-memory instruction library covering pulse/triangle/noise. - - Candidates are sampled with an even stride across each channel's instruction - space so pitch, volume and period are represented, rather than a biased prefix. - """ - window = Window.from_config(config) - extractor = get_feature_extractor(config, window) - channels = get_generators_by_channels(config, CHANNELS) - - data: Dict[InstructionUnion, InstructionLibraryFragment[Any]] = {} - for channel in channels.values(): - candidates = list(channel.get_possible_instructions()) - stride = max(1, len(candidates) // per_generator) - for instruction in candidates[::stride][:per_generator]: - data[instruction] = InstructionLibraryFragment.create(channel, instruction, extractor) - - library = InstructionLibrary() - library.data[library.create_key(config, window)] = InstructionLibraryData.create(config, data) - return library - - -def reconstruct_sample( - audio: np.ndarray, - config: Config, - library: InstructionLibrary, - channels: FrozenSet[ChannelName], - *, - tmp_dir: Pathlike, - name: str, -) -> Reconstruction: - """Runs the real reconstruction pipeline on ``audio`` via a temp WAV.""" - path = Path(tmp_dir) / f"{name}.wav" - write_wave(path, config.library.sample_rate, audio) - reconstruction = Reconstructor(config, channels, library=library)(path) - if reconstruction is None: - raise AssertionError(f"Reconstruction of '{name}' produced no result") - - return reconstruction - - -def make_sample( - name: str, - audio: np.ndarray, - config: Config, - library: InstructionLibrary, - *, - tmp_dir: Pathlike, - expected_slices: FrozenSet[ChannelName], - loop: bool = False, -) -> Sample: - """Reconstructs ``audio`` into a `Sample`, asserting the channels it plays.""" - reconstruction = reconstruct_sample( - audio, - config, - library, - expected_slices, - tmp_dir=tmp_dir, - name=name, - ) - played = frozenset(reconstruction.playing_channels) - if played != expected_slices: - raise AssertionError(f"Sample '{name}' covers {set(played)}, expected {set(expected_slices)}") - - return Sample(name=name, reconstruction=reconstruction) - - -def load_instrument_catalog( - spec_path: Pathlike, - synth_config: SynthConfig, - *, - tmp_dir: Pathlike, -) -> Dict[str, Sample]: - """Builds the reconstructed sample catalog described by a JSON spec. - - The spec fixes the reconstruction settings (spectrum method, gamma, library - size) shared by one in-memory library, and per instrument the synth to run and - the channel slices to cover. Synth parameters come from ``synth_config``. - Returns a name -> `Sample` mapping. - """ - spec = load_yaml(spec_path) - settings = spec["reconstruction"] - library_config = InstructionsLibraryConfig( - spectrum_method=SpectrumMethod(settings["spectrum_method"]), - transformation_gamma=settings["transformation_gamma"], - ) - sample_rate = library_config.sample_rate - library = build_mini_library( - Config(library=library_config), - per_generator=settings["instructions_per_generator"], - ) - - catalog: Dict[str, Sample] = {} - for entry in spec["instruments"]: - channels = [ChannelName(name) for name in entry["channels"]] - config = Config(library=library_config) - audio = _render_instrument(synth_config, entry["synth"], sample_rate=sample_rate) - catalog[entry["name"]] = make_sample( - entry["name"], - audio, - config, - library, - tmp_dir=tmp_dir, - expected_slices=frozenset(channels), - ) - - return catalog - - -def _render_instrument( - synth_config: SynthConfig, - name: str, - *, - sample_rate: int, -) -> np.ndarray: - """ - Render a named voice at peak level 1.0. - - A fresh channel seeded from the synth configuration keeps every instrument - reproducible independently of catalog order. - """ - voice = synth_config.voices[name] - generator = np.random.default_rng(synth_config.seed) - instrument: np.ndarray = normalize( - voice.render( - sample_rate=sample_rate, - generator=generator, - ) - ) - return instrument diff --git a/tests/integration/assets/song_loader.py b/tests/integration/assets/song_loader.py deleted file mode 100644 index e8f48294c..000000000 --- a/tests/integration/assets/song_loader.py +++ /dev/null @@ -1,94 +0,0 @@ -from typing import Any, Dict, List, Mapping, Optional - -from sampletones_core.constants.enums import ChannelName -from sampletones_core.project.patterns.channel import Channel -from sampletones_core.project.patterns.pattern import Pattern -from sampletones_core.project.patterns.row import Row -from sampletones_core.project.song import Song -from sampletones_core.project.voices.note_off import NoteOff -from sampletones_core.project.voices.note_on import NoteOn -from sampletones_core.project.voices.sample import Sample -from sampletones_shared.types.path import Pathlike -from sampletones_shared.utils.serialization import load_yaml - -RowSpec = Dict[str, Any] - - -def _order( - order_specs: List[Dict[str, int]], -) -> List[Dict[ChannelName, Optional[int]]]: - frames: List[Dict[ChannelName, Optional[int]]] = [] - for spec in order_specs: - frames.append({channel: spec.get(channel.value) for channel in ChannelName.items()}) - - return frames - - -def _row(spec: RowSpec, channel: ChannelName, samples_by_name: Mapping[str, Sample]) -> Row: - transpose = spec.get("transpose") - volume = spec.get("volume") - - if spec.get("off"): - return Row(command=NoteOff(), volume=volume) - - sample_name = spec.get("sample") - if sample_name is None: - return Row(transpose=transpose, volume=volume) - - sample = samples_by_name[sample_name] - if channel not in sample.reconstruction.instructions: - raise ValueError(f"Sample '{sample_name}' has no '{channel.value}' slice for the {channel.value} channel") - - command = NoteOn(voice_id=sample.id) - return Row(command=command, transpose=transpose, volume=volume) - - -def _pattern( - row_specs: List[RowSpec], - rows_per_pattern: int, - channel: ChannelName, - samples_by_name: Mapping[str, Sample], -) -> Pattern: - rows = [Row() for _ in range(rows_per_pattern)] - for spec in row_specs: - rows[spec["row"]] = _row(spec, channel, samples_by_name) - - return Pattern(rows=rows) - - -def _channels( - channels_spec: Dict[str, Dict[str, Any]], - rows_per_pattern: int, - samples_by_name: Mapping[str, Sample], -) -> Dict[ChannelName, Channel]: - channels: Dict[ChannelName, Channel] = {} - for name, spec in channels_spec.items(): - channel = ChannelName(name) - patterns = { - int(index): _pattern( - row_specs, - rows_per_pattern, - channel, - samples_by_name, - ) - for index, row_specs in spec["patterns"].items() - } - channels[channel] = Channel( - name=channel, - patterns=patterns, - ) - - for channel in ChannelName.items(): - channels.setdefault(channel, Channel(name=channel, patterns={})) - - return channels - - -def load_song(path: Pathlike, samples_by_name: Mapping[str, Sample]) -> Song: - document = load_yaml(path) - rows_per_pattern = document["rows_per_pattern"] - return Song( - rows_per_pattern=rows_per_pattern, - order=_order(document["order"]), - channels=_channels(document["channels"], rows_per_pattern, samples_by_name), - ) diff --git a/tests/integration/assets/synth_config.py b/tests/integration/assets/synth_config.py deleted file mode 100644 index 971287fd2..000000000 --- a/tests/integration/assets/synth_config.py +++ /dev/null @@ -1,20 +0,0 @@ -from typing import Dict - -from pydantic import BaseModel, ConfigDict - -from sampletones_shared.types.path import Pathlike -from sampletones_shared.utils.serialization import load_yaml_model -from sampletones_tools.synthesis.voice.voice import Voice - - -class SynthConfig(BaseModel): - """The named synthesizer voices of the integration suite and their shared noise seed.""" - - model_config = ConfigDict(frozen=True, extra="forbid") - - seed: int - voices: Dict[str, Voice] - - -def load_synth_config(path: Pathlike) -> SynthConfig: - return load_yaml_model(path, SynthConfig) diff --git a/tests/integration/bitphase/conftest.py b/tests/integration/bitphase/conftest.py index d1dd846e8..0007eda69 100644 --- a/tests/integration/bitphase/conftest.py +++ b/tests/integration/bitphase/conftest.py @@ -1,29 +1,17 @@ from pathlib import Path -from typing import Optional import pytest -from tests.integration.output import resolve_output_directory, resolve_output_path -from tests.integration.paths import ( - BTP_OUTPUT_ENV, - DOCUMENT_FILENAME, - GROOVE_DOCUMENT_FILENAME, -) - - -@pytest.fixture(scope="session") -def btp_output_dir() -> Optional[Path]: - """The persistent output directory ``SAMPLETONES_BTP_OUTPUT_DIR`` names.""" - return resolve_output_directory(BTP_OUTPUT_ENV) +from sampletones_tools.samples.bitphase import DOCUMENT_FILENAME, GROOVE_DOCUMENT_FILENAME @pytest.fixture -def document_path(btp_output_dir: Optional[Path], tmp_path: Path) -> Path: +def document_path(tmp_path: Path) -> Path: """Where a produced ``.btp`` is written.""" - return resolve_output_path(btp_output_dir, tmp_path, DOCUMENT_FILENAME) + return tmp_path / DOCUMENT_FILENAME @pytest.fixture -def groove_document_path(btp_output_dir: Optional[Path], tmp_path: Path) -> Path: +def groove_document_path(tmp_path: Path) -> Path: """Where the document carrying a groove is written, beside the one at the song's own tempo.""" - return resolve_output_path(btp_output_dir, tmp_path, GROOVE_DOCUMENT_FILENAME) + return tmp_path / GROOVE_DOCUMENT_FILENAME diff --git a/tests/integration/bitphase/test_btp_pipeline.py b/tests/integration/bitphase/test_btp_pipeline.py index 85ad771ae..fc96efa83 100644 --- a/tests/integration/bitphase/test_btp_pipeline.py +++ b/tests/integration/bitphase/test_btp_pipeline.py @@ -45,6 +45,7 @@ ) from sampletones_core.project.project import Project from sampletones_core.timing import Meter, RowRate, calculate_groove +from sampletones_tools.samples.bitphase import GROOVE_TEMPO, at_tempo from tests.suite.bitphase import ( BITPHASE_NO_EFFECTS, LoadedEffect, @@ -56,7 +57,6 @@ ) EXPECTED_INSTRUMENT_COUNT: Final[int] = 5 -GROOVE_TEMPO: Final[int] = 210 PLAYED_CHANNELS: Final[List[int]] = [ int(ChannelIndex.SQUARE1), int(ChannelIndex.SQUARE2), @@ -75,17 +75,6 @@ def note_index(note: LoadedNote) -> int: return note.name - int(NoteName.C) + (note.octave - FIRST_OCTAVE) * NOTE_RANGE -def at_tempo(project: Project, tempo: int) -> Project: - """The same project played at another tempo, leaving the session-wide fixture as it is.""" - return Project( - metadata=project.metadata, - info=project.info, - settings=project.settings.model_copy(update={"tempo": tempo}), - voices=project.voices, - song=project.song, - ) - - @pytest.fixture def document(integration_project: Project, document_path: Path) -> LoadedProject: write_btp(document_path, project_to_bitphase(integration_project)) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 8d6b920c8..8776d3dc6 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -4,19 +4,12 @@ import pytest from sampletones_core.project.project import Project -from sampletones_core.project.settings import ProjectSettings from sampletones_core.project.voices.sample import Sample -from sampletones_core.structures import IdentifiedCollection -from tests.integration.assets.module_config import ModuleConfig, load_module_config -from tests.integration.assets.reconstruction import load_instrument_catalog -from tests.integration.assets.song_loader import load_song -from tests.integration.assets.synth_config import SynthConfig, load_synth_config -from tests.integration.paths import ( - MODULE_CONFIG_PATH, - RECONSTRUCTION_CONFIG_PATH, - SONG_PATH, - SYNTH_CONFIG_PATH, -) +from sampletones_tools.corpus.build import build_project +from sampletones_tools.corpus.catalog import CatalogSpec, build_catalog +from sampletones_tools.corpus.module import ModuleConfig +from sampletones_tools.corpus.song import SongSpec +from sampletones_tools.corpus.synth import SynthConfig @pytest.fixture(scope="session") @@ -26,31 +19,19 @@ def audio_directory(tmp_path_factory: pytest.TempPathFactory) -> Path: @pytest.fixture(scope="session") def synth_config() -> SynthConfig: - return load_synth_config(SYNTH_CONFIG_PATH) + return SynthConfig.load() @pytest.fixture(scope="session") def module_config() -> ModuleConfig: - return load_module_config(MODULE_CONFIG_PATH) + return ModuleConfig.load() @pytest.fixture(scope="session") def instrument_catalog(audio_directory: Path, synth_config: SynthConfig) -> Dict[str, Sample]: - return load_instrument_catalog(RECONSTRUCTION_CONFIG_PATH, synth_config, tmp_dir=audio_directory) + return build_catalog(CatalogSpec.load(), synth_config, tmp_dir=audio_directory) @pytest.fixture(scope="session") def integration_project(instrument_catalog: Dict[str, Sample], module_config: ModuleConfig) -> Project: - voices: IdentifiedCollection[Sample] = IdentifiedCollection() - for sample in instrument_catalog.values(): - voices.append(sample) - - settings = ProjectSettings( - tempo=module_config.tempo, - speed=module_config.speed, - nes_frequency=module_config.nes_frequency, - ) - project = Project.create(title=module_config.title, author=module_config.author, settings=settings) - project.voices = voices - project.song = load_song(SONG_PATH, instrument_catalog) - return project + return build_project(instrument_catalog, module_config, SongSpec.load()) diff --git a/tests/integration/famitracker/conftest.py b/tests/integration/famitracker/conftest.py index b10b93fe7..1ffa52897 100644 --- a/tests/integration/famitracker/conftest.py +++ b/tests/integration/famitracker/conftest.py @@ -1,19 +1,11 @@ from pathlib import Path -from typing import Optional import pytest -from tests.integration.output import resolve_output_directory, resolve_output_path -from tests.integration.paths import FTM_OUTPUT_ENV, MODULE_FILENAME - - -@pytest.fixture(scope="session") -def ftm_output_dir() -> Optional[Path]: - """The persistent output directory ``SAMPLETONES_FTM_OUTPUT_DIR`` names.""" - return resolve_output_directory(FTM_OUTPUT_ENV) +from sampletones_tools.samples.famitracker import MODULE_FILENAME @pytest.fixture -def module_path(ftm_output_dir: Optional[Path], tmp_path: Path) -> Path: +def module_path(tmp_path: Path) -> Path: """Where a produced ``.ftm`` is written.""" - return resolve_output_path(ftm_output_dir, tmp_path, MODULE_FILENAME) + return tmp_path / MODULE_FILENAME diff --git a/tests/integration/nsf/conftest.py b/tests/integration/nsf/conftest.py index 1a2567f14..9c166fd90 100644 --- a/tests/integration/nsf/conftest.py +++ b/tests/integration/nsf/conftest.py @@ -1,5 +1,5 @@ from pathlib import Path -from typing import Dict, Final, Optional +from typing import Dict, Final import pytest @@ -8,26 +8,14 @@ from sampletones_player.driver.image import DriverImage from sampletones_player.song import Song from sampletones_shared.paths.extensions import EXT_FILE_NSF -from tests.integration.output import resolve_output_directory, resolve_output_path -from tests.integration.paths import NSF_OUTPUT_ENV EXPORTED_SAMPLE: Final[str] = "lead" -@pytest.fixture(scope="session") -def nsf_output_dir() -> Optional[Path]: - """The persistent output directory ``SAMPLETONES_NSF_OUTPUT_DIR`` names.""" - return resolve_output_directory(NSF_OUTPUT_ENV) - - @pytest.fixture -def nsf_paths( - nsf_output_dir: Optional[Path], - tmp_path: Path, - instrument_catalog: Dict[str, Sample], -) -> Dict[str, Path]: +def nsf_paths(tmp_path: Path, instrument_catalog: Dict[str, Sample]) -> Dict[str, Path]: """Where each sample's produced ``.nsf`` is written.""" - return {name: resolve_output_path(nsf_output_dir, tmp_path, f"{name}{EXT_FILE_NSF}") for name in instrument_catalog} + return {name: tmp_path / f"{name}{EXT_FILE_NSF}" for name in instrument_catalog} @pytest.fixture(scope="session") diff --git a/tests/integration/nsf/exports.py b/tests/integration/nsf/exports.py deleted file mode 100644 index c7041fafb..000000000 --- a/tests/integration/nsf/exports.py +++ /dev/null @@ -1,10 +0,0 @@ -from typing import Final - -from sampletones_player.nsf.information import NSFInformation - -ARTIST: Final[str] = "Integration" - - -def exported_information(name: str) -> NSFInformation: - """The header text an exported sample carries.""" - return NSFInformation(title=name, artist=ARTIST) diff --git a/tests/integration/nsf/test_backend.py b/tests/integration/nsf/test_backend.py index a080aa36a..6f516b180 100644 --- a/tests/integration/nsf/test_backend.py +++ b/tests/integration/nsf/test_backend.py @@ -1,5 +1,5 @@ from pathlib import Path -from typing import Dict, Final, List, Optional +from typing import Dict, Final, List import numpy as np import pytest @@ -36,7 +36,6 @@ captured_run, play_calls_reaching, ) -from tests.integration.output import resolve_output_path ChannelInstructions = Dict[ChannelName, List[InstructionUnion]] @@ -204,7 +203,6 @@ def project_song(integration_project: Project) -> Song: def project_file( backend: NSFBackend, integration_project: Project, - nsf_output_dir: Optional[Path], tmp_path_factory: pytest.TempPathFactory, ) -> Path: """The whole arrangement written through the backend the application registers. @@ -212,11 +210,7 @@ def project_file( It joins the samples in the emitted artifacts, so the arrangement can be listened to beside the instruments it is built from. """ - destination = resolve_output_path( - nsf_output_dir, - tmp_path_factory.mktemp("nsf-project"), - f"{PROJECT_ARTIFACT}{EXT_FILE_NSF}", - ) + destination = tmp_path_factory.mktemp("nsf-project") / f"{PROJECT_ARTIFACT}{EXT_FILE_NSF}" backend.write_project(destination, ProjectExport(project=integration_project)) return destination diff --git a/tests/integration/nsf/test_compression_report.py b/tests/integration/nsf/test_compression_report.py index db0899580..1f446d38c 100644 --- a/tests/integration/nsf/test_compression_report.py +++ b/tests/integration/nsf/test_compression_report.py @@ -1,114 +1,26 @@ -from dataclasses import dataclass from math import ceil from pathlib import Path -from time import process_time -from typing import Dict, Final, List, Optional, Sequence, Tuple +from typing import Dict, Tuple import pytest from sampletones_core.project.project import Project from sampletones_core.project.voices.sample import Sample -from sampletones_player.compression.compressed import CompressedPlanes from sampletones_player.compression.decode import decode_planes -from sampletones_player.compression.dictionary.table import phrase_table -from sampletones_player.compression.encode import STREAM_START, encode_planes -from sampletones_player.compression.matches.cache import MatchCache -from sampletones_player.compression.matches.index import PlaneIndex -from sampletones_player.compression.matches.matcher import PhraseMatcher -from sampletones_player.compression.options import CodecOptions -from sampletones_player.compression.parse.plane import parse_plane from sampletones_player.compression.planes.rebuild import streams_from_planes -from sampletones_player.compression.planes.song import SongPlanes from sampletones_player.driver.image import DriverImage -from sampletones_player.registers.streams import ChannelStreams -from sampletones_player.specification.compression import ( - MAX_LITERAL_BYTES, - PLANE_COUNT, - PLANE_STATE_SIZE, -) -from sampletones_player.specification.registers import DUTY_CYCLE_SHIFT +from sampletones_player.specification.compression import MAX_LITERAL_BYTES from sampletones_player.specification.song import SONG_HEADER_SIZE -from tests.integration.nsf.corpus import LONG_ARRANGEMENT, CorpusEntry, build_corpus -from tests.integration.nsf.report import ReportRow, write_csv, write_markdown -from tests.integration.nsf.songs import available_bytes -from tests.integration.output import resolve_output_directory, resolve_output_path -from tests.integration.paths import COMPRESSION_OUTPUT_ENV - -LITERALS: Final[str] = "literals" -HOLDS: Final[str] = "holds" -SEARCH: Final[str] = "search" -RECORDS: Final[str] = "records" -REGISTER_PLANES: Final[str] = "register planes" -SPLIT_CONTROL: Final[str] = "split control" -CONTROL_LEVEL_MASK: Final[int] = 0x3F - -PLANE_VARIANTS: Final[Tuple[Tuple[str, CodecOptions], ...]] = ( - (LITERALS, CodecOptions(holds=False, phrases=False, transposition=False, search=False)), - (HOLDS, CodecOptions(holds=True, phrases=False, transposition=False, search=False)), - ("instruments", CodecOptions(holds=True, phrases=True, transposition=False, search=False)), - ("transposition", CodecOptions(holds=True, phrases=True, transposition=True, search=False)), - (SEARCH, CodecOptions(holds=True, phrases=True, transposition=True, search=True)), +from sampletones_tools.codec.report.corpus import LONG_ARRANGEMENT, CorpusEntry, corpus_entries +from sampletones_tools.codec.report.encoding import ( + LITERALS, + SEARCH, + Encoding, + encode_corpus, + report_rows, ) - -CSV_FILENAME: Final[str] = "report.csv" -MARKDOWN_FILENAME: Final[str] = "report.md" - - -@dataclass(frozen=True) -class Encoding: - """One corpus song compressed under one variant of the codec.""" - - entry: CorpusEntry - variant: str - planes: SongPlanes - compressed: CompressedPlanes - seconds: float - - @property - def size(self) -> int: - """The bytes the dictionary, the streams and the pitch table take together.""" - return self.compressed.size + len(self.entry.pitches.data) - - @property - def streams(self) -> int: - """The bytes the token streams take, which is the part that grows with the song.""" - return sum(len(stream) for stream in self.compressed.streams) - - -def _register_planes(streams: ChannelStreams) -> Tuple[bytes, ...]: - return tuple( - bytes(tick.values[register] for tick in stream) - for stream in streams.padded - for register in range(len(stream[0].values)) - ) - - -def _split_control_planes(planes: SongPlanes) -> Tuple[bytes, ...]: - split: List[bytes] = [] - for channels in planes.ordered: - if channels in (planes.pulse1, planes.pulse2): - split.append(bytes(control >> DUTY_CYCLE_SHIFT for control in channels.control)) - split.append(bytes(control & CONTROL_LEVEL_MASK for control in channels.control)) - else: - split.append(channels.control) - - split.append(channels.value) - - return tuple(split) - - -def _coded_size(planes: Sequence[bytes], options: CodecOptions) -> int: - cache = MatchCache(PlaneIndex.from_plane(plane) for plane in planes) - table = phrase_table(()) - entries = frozenset({STREAM_START}) - return sum( - parse_plane( - PhraseMatcher(table, plane, cache), - options, - entries, - ).size - for plane in range(len(planes)) - ) +from sampletones_tools.codec.report.session import write_report +from sampletones_tools.codec.report.songs import available_bytes @pytest.fixture(scope="module") @@ -117,113 +29,13 @@ def corpus( integration_project: Project, ) -> Tuple[CorpusEntry, ...]: """The songs the report measures: each sample alone, the arrangement at two lengths, and a dense minute.""" - return build_corpus(instrument_catalog, integration_project) + return corpus_entries(instrument_catalog, integration_project) @pytest.fixture(scope="module") def encodings(corpus: Tuple[CorpusEntry, ...]) -> Tuple[Encoding, ...]: """Every corpus song compressed under every variant of the codec.""" - encoded: List[Encoding] = [] - for entry in corpus: - planes = entry.planes - for variant, options in PLANE_VARIANTS: - started = process_time() - compressed = encode_planes( - planes, - entry.seeds, - options=options, - boundaries=frozenset(), - ) - encoded.append( - Encoding( - entry=entry, - variant=variant, - planes=planes, - compressed=compressed, - seconds=process_time() - started, - ) - ) - - return tuple(encoded) - - -def _measured_row( - entry: CorpusEntry, - variant: str, - planes: Sequence[bytes], - fixed: int, - space: int, -) -> ReportRow: - _, holds = PLANE_VARIANTS[1] - started = process_time() - coded = _coded_size(planes, holds) - return ReportRow( - corpus=entry.name, - variant=variant, - ticks=entry.song.ticks, - size=fixed + coded, - variable=coded, - phrases=0, - dictionary=0, - seconds=process_time() - started, - records=entry.records, - space=space, - ) - - -def _baseline_rows(entry: CorpusEntry, space: int) -> Tuple[ReportRow, ...]: - return ( - ReportRow( - corpus=entry.name, - variant=RECORDS, - ticks=entry.song.ticks, - size=entry.records, - variable=entry.records, - phrases=0, - dictionary=0, - seconds=0.0, - records=entry.records, - space=space, - ), - _measured_row(entry, REGISTER_PLANES, _register_planes(entry.song.streams), 0, space), - _measured_row( - entry, - SPLIT_CONTROL, - _split_control_planes(entry.planes), - len(entry.pitches.data), - space, - ), - ) - - -def _rows( - corpus: Sequence[CorpusEntry], - encodings: Sequence[Encoding], - space: int, -) -> Tuple[ReportRow, ...]: - rows: List[ReportRow] = [] - for entry in corpus: - rows.extend(_baseline_rows(entry, space)) - for encoding in encodings: - if encoding.entry is not entry: - continue - - rows.append( - ReportRow( - corpus=entry.name, - variant=encoding.variant, - ticks=entry.song.ticks, - size=encoding.size, - variable=encoding.streams, - phrases=len(encoding.compressed.phrases), - dictionary=encoding.compressed.phrases.size, - seconds=encoding.seconds, - records=entry.records, - space=space, - ) - ) - - return tuple(rows) + return encode_corpus(corpus) class TestTheCodecAnswersWithTheSongItWasGiven: @@ -302,19 +114,10 @@ def test_the_report_is_written( corpus: Tuple[CorpusEntry, ...], encodings: Tuple[Encoding, ...], driver_image: DriverImage, - compression_output_dir: Optional[Path], tmp_path: Path, ) -> None: - rows = _rows(corpus, encodings, available_bytes(driver_image)) - csv_path = resolve_output_path(compression_output_dir, tmp_path, CSV_FILENAME) - markdown_path = resolve_output_path(compression_output_dir, tmp_path, MARKDOWN_FILENAME) - write_csv(rows, csv_path) - write_markdown(rows, markdown_path, PLANE_COUNT * PLANE_STATE_SIZE) + space = available_bytes(driver_image) + csv_path, markdown_path = write_report(corpus, encodings, space, tmp_path) + rows = report_rows(corpus, encodings, space) assert csv_path.read_text(encoding="utf-8").count("\n") == len(rows) + 1 assert markdown_path.exists() - - -@pytest.fixture(scope="session") -def compression_output_dir() -> Optional[Path]: - """The persistent output directory ``SAMPLETONES_COMPRESSION_OUTPUT_DIR`` names.""" - return resolve_output_directory(COMPRESSION_OUTPUT_ENV) diff --git a/tests/integration/nsf/test_driver_audio.py b/tests/integration/nsf/test_driver_audio.py index 02018b35c..fabd72ac7 100644 --- a/tests/integration/nsf/test_driver_audio.py +++ b/tests/integration/nsf/test_driver_audio.py @@ -11,9 +11,9 @@ from sampletones_core.timers.utils import get_timer_table from sampletones_player.builder import song_from_reconstruction from sampletones_player.registers.playable import playable +from sampletones_tools.samples.nsf import exported_information from tests.integration.nsf.console.instructions import instructions_from_trace, sounded_approximation from tests.integration.nsf.console.session import captured_trace -from tests.integration.nsf.exports import exported_information ChannelInstructions = Dict[ChannelName, List[InstructionUnion]] diff --git a/tests/integration/nsf/test_driver_bend.py b/tests/integration/nsf/test_driver_bend.py index fc9356c54..5a5f3fa0b 100644 --- a/tests/integration/nsf/test_driver_bend.py +++ b/tests/integration/nsf/test_driver_bend.py @@ -8,10 +8,10 @@ from sampletones_player.specification.binary import BYTE_VALUES from sampletones_player.specification.registers import PULSE1_TIMER_HIGH, TIMER_HIGH_SHIFT from sampletones_tools.player.trace.trace import RegisterTrace +from sampletones_tools.samples.nsf import exported_information from tests.integration.nsf.console.instructions import channel_values, timer_value from tests.integration.nsf.console.machine import register_file from tests.integration.nsf.console.session import captured_trace, play_calls_covering -from tests.integration.nsf.exports import exported_information from tests.suite.base import BaseTestSuite from tests.suite.case import BaseAutolabelTestCase from tests.suite.player import PLAYER_PITCHES, bent_song diff --git a/tests/integration/nsf/test_driver_trace.py b/tests/integration/nsf/test_driver_trace.py index 90f662f8c..8ec5ea9de 100644 --- a/tests/integration/nsf/test_driver_trace.py +++ b/tests/integration/nsf/test_driver_trace.py @@ -12,6 +12,7 @@ from sampletones_player.specification.nsf import PROGRAM_SIZE from sampletones_player.specification.song import STEP_FRACTION_OFFSET, STEP_WHOLE_OFFSET from sampletones_tools.player.trace.trace import RegisterTrace +from sampletones_tools.samples.nsf import exported_information from tests.integration.nsf.console.session import ( TRAILING_CALLS, captured_trace, @@ -19,7 +20,6 @@ play_calls_covering, play_calls_reaching, ) -from tests.integration.nsf.exports import exported_information from tests.suite.base import BaseTestSuite from tests.suite.case import BaseAutolabelTestCase diff --git a/tests/integration/nsf/test_nsf_pipeline.py b/tests/integration/nsf/test_nsf_pipeline.py index fda31e4cf..3f61d3eeb 100644 --- a/tests/integration/nsf/test_nsf_pipeline.py +++ b/tests/integration/nsf/test_nsf_pipeline.py @@ -29,7 +29,7 @@ TOTAL_TICKS_OFFSET, ) from sampletones_shared.paths.extensions import EXT_FILE_RECONSTRUCTION -from tests.integration.nsf.exports import exported_information +from sampletones_tools.samples.nsf import exported_information def song_block(data: bytes, image: DriverImage) -> bytes: diff --git a/tests/integration/nsf/test_song_export.py b/tests/integration/nsf/test_song_export.py index 7f566958c..09d4e78d1 100644 --- a/tests/integration/nsf/test_song_export.py +++ b/tests/integration/nsf/test_song_export.py @@ -13,7 +13,7 @@ TOTAL_TICKS_OFFSET, ) from sampletones_shared.exceptions import SongTooLargeError -from tests.integration.nsf.songs import RECORD_BYTES_PER_TICK, available_bytes +from sampletones_tools.codec.report.songs import RECORD_BYTES_PER_TICK, available_bytes def read_word(data: bytes, offset: int) -> int: diff --git a/tests/integration/output.py b/tests/integration/output.py deleted file mode 100644 index 423b89d30..000000000 --- a/tests/integration/output.py +++ /dev/null @@ -1,49 +0,0 @@ -import os -import shutil -from pathlib import Path -from typing import Optional - -from tests.integration.paths import REPO_ROOT - - -def resolve_output_directory(variable: str) -> Optional[Path]: - """Reads the persistent output directory an environment variable names. - - Emission is opt-in so an ordinary (and parallel) run writes only to ``tmp_path``. - A named directory is cleaned once per session, so each run leaves a fresh set of - files there. - - Args: - variable: Environment variable naming the directory. - - Returns: - Optional[Path]: The prepared directory, or ``None`` while emission is unasked for. - """ - configured = os.environ.get(variable) - if not configured: - return None - - directory = Path(configured) - if not directory.is_absolute(): - directory = REPO_ROOT / directory - - if directory.exists(): - shutil.rmtree(directory) - - directory.mkdir(parents=True, exist_ok=True) - return directory - - -def resolve_output_path(output_directory: Optional[Path], tmp_path: Path, filename: str) -> Path: - """Locates a produced file: the persistent directory where one is named, else ``tmp_path``. - - Args: - output_directory: The persistent directory, or ``None`` while emission is unasked for. - tmp_path: The test's own temporary directory. - filename: Name the produced file carries. - - Returns: - Path: Where the file is written. - """ - base = output_directory if output_directory is not None else tmp_path - return base / filename diff --git a/tests/integration/paths.py b/tests/integration/paths.py deleted file mode 100644 index c27997891..000000000 --- a/tests/integration/paths.py +++ /dev/null @@ -1,30 +0,0 @@ -from pathlib import Path -from typing import Final - - -def _repo_root() -> Path: - for parent in Path(__file__).resolve().parents: - if (parent / "pyproject.toml").exists(): - return parent - - raise RuntimeError("Could not locate the repository root") - - -REPO_ROOT: Final[Path] = _repo_root() -INTEGRATION_ROOT: Final[Path] = Path(__file__).parent - -CONFIG_DIRECTORY: Final[Path] = INTEGRATION_ROOT / "config" -SYNTH_CONFIG_PATH: Final[Path] = CONFIG_DIRECTORY / "synth.yaml" -RECONSTRUCTION_CONFIG_PATH: Final[Path] = CONFIG_DIRECTORY / "reconstruction.yaml" -MODULE_CONFIG_PATH: Final[Path] = CONFIG_DIRECTORY / "module.yaml" -SONG_PATH: Final[Path] = CONFIG_DIRECTORY / "song.yaml" - -MODULE_FILENAME: Final[str] = "drums.ftm" -FTM_OUTPUT_ENV: Final[str] = "SAMPLETONES_FTM_OUTPUT_DIR" - -DOCUMENT_FILENAME: Final[str] = "drums.btp" -GROOVE_DOCUMENT_FILENAME: Final[str] = "drums-groove.btp" -BTP_OUTPUT_ENV: Final[str] = "SAMPLETONES_BTP_OUTPUT_DIR" - -NSF_OUTPUT_ENV: Final[str] = "SAMPLETONES_NSF_OUTPUT_DIR" -COMPRESSION_OUTPUT_ENV: Final[str] = "SAMPLETONES_COMPRESSION_OUTPUT_DIR" diff --git a/tests/integration/reconstruction/test_conversion_jobs.py b/tests/integration/reconstruction/test_conversion_jobs.py index 488621ee4..672505513 100644 --- a/tests/integration/reconstruction/test_conversion_jobs.py +++ b/tests/integration/reconstruction/test_conversion_jobs.py @@ -21,14 +21,14 @@ from sampletones_core.reconstructions.stage import ReconstructionStage from sampletones_shared.exceptions import OperationCanceled from sampletones_shared.utils.progress import silent_reporter -from tests.integration.assets.reconstruction import ( +from sampletones_tools.corpus.catalog import build_mini_library +from tests.suite.progress import FIRST_REPORT, RecordingReporter, reported_stages +from tests.suite.stems import ( THREE_STEM_CHANNELS, - build_mini_library, three_stem_config, three_stem_reconstruction_config, write_three_stem_recordings, ) -from tests.suite.progress import FIRST_REPORT, RecordingReporter, reported_stages def _writing_to(config: Config, directory: Path) -> Config: diff --git a/tests/integration/reconstruction/test_decoding.py b/tests/integration/reconstruction/test_decoding.py index cda43bd2a..66b110c6b 100644 --- a/tests/integration/reconstruction/test_decoding.py +++ b/tests/integration/reconstruction/test_decoding.py @@ -13,7 +13,7 @@ ) from sampletones_core.instructions import InstructionUnion from sampletones_core.reconstructions import Reconstruction, Reconstructor -from tests.integration.assets.reconstruction import build_mini_library +from sampletones_tools.corpus.catalog import build_mini_library _DURATION_SECONDS: Final[float] = 1.0 _LOWER_TONE: Final[float] = 440.0 diff --git a/tests/integration/reconstruction/test_stems_reconstruction.py b/tests/integration/reconstruction/test_stems_reconstruction.py index ef8df05ba..ac927745f 100644 --- a/tests/integration/reconstruction/test_stems_reconstruction.py +++ b/tests/integration/reconstruction/test_stems_reconstruction.py @@ -21,13 +21,13 @@ from sampletones_core.reconstructions.reconstructor.stems.configs.entry import StemEntry from sampletones_core.reconstructions.reconstructor.stems.configs.hierarchy import StemsHierarchy from sampletones_core.reconstructions.reconstructor.stems.configs.settings import StemSettings -from tests.integration.assets.reconstruction import ( +from sampletones_tools.corpus.catalog import build_mini_library +from tests.suite.stems import ( STEM_A_ID, STEM_B_ID, STEM_C_ID, STEM_RECORDING_DURATION_SECONDS, THREE_STEM_CHANNELS, - build_mini_library, three_stem_config, three_stem_reconstruction_config, write_three_stem_recordings, diff --git a/tests/integration/samples/__init__.py b/tests/integration/samples/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/integration/samples/test_emitters.py b/tests/integration/samples/test_emitters.py new file mode 100644 index 000000000..268c3a03e --- /dev/null +++ b/tests/integration/samples/test_emitters.py @@ -0,0 +1,49 @@ +from pathlib import Path +from typing import Dict + +import pytest + +from sampletones_core.formats.bitphase.specification.channels import CHANNEL_LABELS +from sampletones_core.project.project import Project +from sampletones_core.project.voices.sample import Sample +from sampletones_player.specification.nsf import NSF_MAGIC +from sampletones_shared.paths.extensions import EXT_FILE_NSF +from sampletones_tools.corpus.build import Corpus +from sampletones_tools.samples import bitphase, famitracker, nsf +from tests.suite.bitphase import parse_btp +from tests.suite.famitracker import parse_ftm + + +@pytest.fixture(scope="module") +def corpus(instrument_catalog: Dict[str, Sample], integration_project: Project) -> Corpus: + """The session's catalog and arrangement, as the emitters are handed them.""" + return Corpus(catalog=instrument_catalog, project=integration_project) + + +class TestTheEmittersWriteTheCorpus: + """Each format's emitter writes the files a player of that format opens.""" + + def test_the_nsf_emitter_writes_every_sample_then_the_arrangement(self, corpus: Corpus, tmp_path: Path) -> None: + written = nsf.write_samples(corpus, tmp_path) + + assert [path.name for path in written] == [ + *(f"{name}{EXT_FILE_NSF}" for name in corpus.catalog), + f"{nsf.SONG_NAME}{EXT_FILE_NSF}", + ] + assert all(path.read_bytes()[: len(NSF_MAGIC)] == NSF_MAGIC for path in written) + + def test_the_famitracker_emitter_writes_one_module(self, corpus: Corpus, tmp_path: Path) -> None: + written = famitracker.write_samples(corpus, tmp_path) + + assert written == [tmp_path / famitracker.MODULE_FILENAME] + assert parse_ftm(written[0].read_bytes()).instruments + + def test_the_bitphase_emitter_writes_the_song_at_its_tempo_and_as_a_groove( + self, + corpus: Corpus, + tmp_path: Path, + ) -> None: + written = bitphase.write_samples(corpus, tmp_path) + + assert written == [tmp_path / bitphase.DOCUMENT_FILENAME, tmp_path / bitphase.GROOVE_DOCUMENT_FILENAME] + assert all(parse_btp(path.read_bytes(), list(CHANNEL_LABELS)).songs for path in written) diff --git a/tests/suite/stems.py b/tests/suite/stems.py index 3dd8ba7dc..8bbcff6e1 100644 --- a/tests/suite/stems.py +++ b/tests/suite/stems.py @@ -1,10 +1,37 @@ -from typing import List, Mapping, Sequence +from pathlib import Path +from typing import Dict, Final, List, Mapping, Sequence, Tuple +import numpy as np + +from sampletones_core.audio import write_wave +from sampletones_core.configs import Config from sampletones_core.constants.algorithm import DEFAULT_STEMS_CHANNEL_CAP -from sampletones_core.constants.enums import ChannelName, bending_channels +from sampletones_core.constants.enums import ChannelName, HierarchyMode, bending_channels from sampletones_core.instructions import InstructionUnion from sampletones_core.reconstructions.reconstruction.stems.channel_assignment import ChannelAssignment from sampletones_core.reconstructions.reconstruction.stems.data import StemsData +from sampletones_core.reconstructions.reconstructor.stems.configs.config import StemsConfig +from sampletones_core.reconstructions.reconstructor.stems.configs.entry import StemEntry +from sampletones_core.reconstructions.reconstructor.stems.configs.hierarchy import StemsHierarchy +from sampletones_core.reconstructions.reconstructor.stems.configs.settings import StemSettings +from sampletones_shared.types.path import Pathlike + +STEM_A_ID: Final[int] = 0 +STEM_B_ID: Final[int] = 1 +STEM_C_ID: Final[int] = 2 +THREE_STEM_CHANNELS: Final[List[ChannelName]] = [ + ChannelName.PULSE1, + ChannelName.PULSE2, + ChannelName.TRIANGLE, + ChannelName.NOISE, +] +THREE_STEM_ENTRY_CHANNELS: Final[Dict[int, List[ChannelName]]] = { + STEM_A_ID: [ChannelName.PULSE1, ChannelName.TRIANGLE, ChannelName.NOISE], + STEM_B_ID: [ChannelName.PULSE2, ChannelName.TRIANGLE], + STEM_C_ID: [ChannelName.PULSE1, ChannelName.NOISE], +} +STEM_RECORDING_DURATION_SECONDS: Final[float] = 0.5 +RECORDING_SEED: Final[int] = 93 def single_entry_stems_data( @@ -22,3 +49,50 @@ def single_entry_stems_data( bending_channels(channels), assignments, ) + + +def three_stem_config() -> StemsConfig: + """Builds the three-stem setup the stems tests share. + + Stems a (pulse 1, triangle, noise) and b (pulse 2, triangle) pick on the first + hierarchy level, stem c (pulse 1, noise) on the second. + """ + return StemsConfig( + entries=[ + StemEntry(id=stem_id, settings=StemSettings(channels=channels, bends=bending_channels(channels))) + for stem_id, channels in THREE_STEM_ENTRY_CHANNELS.items() + ], + hierarchy=StemsHierarchy( + levels=[[STEM_A_ID, STEM_B_ID], [STEM_C_ID]], + mode=HierarchyMode.STRICT, + ), + channel_cap=1, + ) + + +def three_stem_reconstruction_config() -> Config: + """Builds a reconstruction config for the three-stem example.""" + return Config() + + +def write_three_stem_recordings( + config: Config, + tmp_dir: Pathlike, +) -> Tuple[Path, Path, Path]: + """Writes three distinct stem recordings a, b, c and returns their paths in order.""" + sample_rate = config.library.sample_rate + count = int(sample_rate * STEM_RECORDING_DURATION_SECONDS) + time = np.arange(count) / sample_rate + recordings = { + "a": 0.5 * np.sin(2 * np.pi * 440.0 * time), + "b": 0.4 * np.sin(2 * np.pi * 220.0 * time), + "c": np.random.default_rng(RECORDING_SEED).uniform(-0.3, 0.3, count), + } + + paths: List[Path] = [] + for name, audio in recordings.items(): + path = Path(tmp_dir) / f"stem_{name}.wav" + write_wave(path, sample_rate, audio) + paths.append(path) + + return paths[0], paths[1], paths[2] diff --git a/tests/unit/sampletones_tools/codec/test_command.py b/tests/unit/sampletones_tools/codec/test_command.py index 53fca9d0c..3ae0a13b1 100644 --- a/tests/unit/sampletones_tools/codec/test_command.py +++ b/tests/unit/sampletones_tools/codec/test_command.py @@ -8,6 +8,7 @@ from sampletones_tools.codec.study.manifest import StudyManifest RUNNER: Final[str] = "sampletones_tools.codec.study.session.run_study" +REPORTER: Final[str] = "sampletones_tools.codec.report.session.run_report" class RecordedStudy: @@ -19,6 +20,35 @@ def __call__(self, manifest: StudyManifest, output: Optional[Path]) -> Path: return output if output is not None else Path("run") +class TestCodecReport: + def test_the_report_is_written_into_the_output_and_its_tables_are_printed( + self, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + ) -> None: + outputs: List[Path] = [] + + def run_report(output: Path) -> Tuple[Path, Path]: + outputs.append(output) + return output / "report.csv", output / "report.md" + + monkeypatch.setattr(REPORTER, run_report) + + assert dispatch(COMMANDS, ["codec", "report", "-o", str(tmp_path)]) == 0 + assert outputs == [tmp_path] + assert capsys.readouterr().out.splitlines() == [ + f"Wrote {tmp_path / 'report.csv'}", + f"Wrote {tmp_path / 'report.md'}", + ] + + def test_the_output_is_required(self) -> None: + with pytest.raises(SystemExit) as leaving: + dispatch(COMMANDS, ["codec", "report"]) + + assert leaving.value.code == 2 + + class TestCodecStudy: def test_the_sources_and_the_sweep_are_read_from_the_options( self, diff --git a/tests/unit/sampletones_tools/corpus/test_build.py b/tests/unit/sampletones_tools/corpus/test_build.py new file mode 100644 index 000000000..8fcbd8d65 --- /dev/null +++ b/tests/unit/sampletones_tools/corpus/test_build.py @@ -0,0 +1,35 @@ +from typing import Dict + +from sampletones_core.constants.enums import ChannelName +from sampletones_core.project.voices.sample import Sample +from sampletones_tools.corpus.build import build_project +from sampletones_tools.corpus.module import ModuleConfig +from sampletones_tools.corpus.song import ChannelSpec, RowSpec, SongSpec +from tests.suite.performance import make_pulse_reconstruction, make_triangle_reconstruction + +MODULE: ModuleConfig = ModuleConfig(title="Demo", author="Someone", tempo=125, speed=3, nes_frequency=50) + + +def _catalog() -> Dict[str, Sample]: + return { + "lead": Sample(name="lead", reconstruction=make_pulse_reconstruction()), + "bass": Sample(name="bass", reconstruction=make_triangle_reconstruction()), + } + + +class TestBuildProject: + def test_the_project_carries_the_module_the_voices_and_the_song(self) -> None: + catalog = _catalog() + spec = SongSpec( + rows_per_pattern=2, + order=[{ChannelName.TRIANGLE: 0}], + channels={ChannelName.TRIANGLE: ChannelSpec(patterns={0: [RowSpec(row=0, sample="bass")]})}, + ) + + project = build_project(catalog, MODULE, spec) + + assert (project.info.title, project.info.author) == ("Demo", "Someone") + assert (project.settings.tempo, project.settings.speed, project.settings.nes_frequency) == (125, 3, 50) + assert [voice.name for voice in project.voices] == ["lead", "bass"] + assert project.song.rows_per_pattern == 2 + assert project.song.channels[ChannelName.TRIANGLE].patterns[0].rows[0].volume is None diff --git a/tests/unit/sampletones_tools/corpus/test_catalog.py b/tests/unit/sampletones_tools/corpus/test_catalog.py new file mode 100644 index 000000000..9b0e86efa --- /dev/null +++ b/tests/unit/sampletones_tools/corpus/test_catalog.py @@ -0,0 +1,13 @@ +from sampletones_tools.corpus.catalog import CatalogSpec +from sampletones_tools.corpus.synth import SynthConfig + + +class TestCatalogSpec: + def test_every_shipped_instrument_is_rendered_from_a_voice_the_synthesizer_holds(self) -> None: + voices = SynthConfig.load().voices + + instruments = CatalogSpec.load().instruments + + assert instruments + assert all(instrument.synth in voices for instrument in instruments) + assert all(instrument.channels for instrument in instruments) diff --git a/tests/unit/sampletones_tools/corpus/test_module.py b/tests/unit/sampletones_tools/corpus/test_module.py new file mode 100644 index 000000000..a5146c346 --- /dev/null +++ b/tests/unit/sampletones_tools/corpus/test_module.py @@ -0,0 +1,9 @@ +from sampletones_tools.corpus.module import ModuleConfig + + +class TestModuleConfig: + def test_the_shipped_module_loads_from_the_package(self) -> None: + module = ModuleConfig.load() + + assert module.title + assert module.tempo > 0 diff --git a/tests/unit/sampletones_tools/corpus/test_song.py b/tests/unit/sampletones_tools/corpus/test_song.py new file mode 100644 index 000000000..6644d0b91 --- /dev/null +++ b/tests/unit/sampletones_tools/corpus/test_song.py @@ -0,0 +1,74 @@ +from typing import Dict + +import pytest + +from sampletones_core.constants.enums import ChannelName +from sampletones_core.project.voices.note_off import NoteOff +from sampletones_core.project.voices.note_on import NoteOn +from sampletones_core.project.voices.sample import Sample +from sampletones_tools.corpus.song import ChannelSpec, RowSpec, SongSpec, build_song +from tests.suite.performance import make_noise_reconstruction, make_pulse_reconstruction + +ROWS_PER_PATTERN: int = 4 + + +def _catalog() -> Dict[str, Sample]: + return { + "lead": Sample(name="lead", reconstruction=make_pulse_reconstruction()), + "hihat": Sample(name="hihat", reconstruction=make_noise_reconstruction()), + } + + +def _spec(channel: ChannelName, rows: Dict[int, RowSpec]) -> SongSpec: + return SongSpec( + rows_per_pattern=ROWS_PER_PATTERN, + order=[{channel: 0}], + channels={channel: ChannelSpec(patterns={0: list(rows.values())})}, + ) + + +class TestSongSpec: + def test_the_shipped_arrangement_loads_from_the_package(self) -> None: + spec = SongSpec.load() + + assert spec.rows_per_pattern > 0 + assert spec.order + assert set(spec.channels) <= set(ChannelName.items()) + + +class TestBuildSong: + def test_every_written_row_reaches_its_pattern(self) -> None: + catalog = _catalog() + spec = _spec( + ChannelName.PULSE1, + { + 0: RowSpec(row=0, sample="lead", transpose=12, volume=10), + 2: RowSpec(row=2, off=True), + 3: RowSpec(row=3, volume=4), + }, + ) + + song = build_song(spec, catalog) + + rows = song.channels[ChannelName.PULSE1].patterns[0].rows + assert len(rows) == ROWS_PER_PATTERN + assert rows[0].command == NoteOn(voice_id=catalog["lead"].id) + assert (rows[0].transpose, rows[0].volume) == (12, 10) + assert rows[1].command is None + assert isinstance(rows[2].command, NoteOff) + assert (rows[3].command, rows[3].volume) == (None, 4) + + def test_the_order_names_every_channel_and_every_channel_is_present(self) -> None: + song = build_song(_spec(ChannelName.NOISE, {0: RowSpec(row=0, sample="hihat")}), _catalog()) + + assert song.order == [{channel: 0 if channel == ChannelName.NOISE else None for channel in ChannelName.items()}] + assert set(song.channels) == set(ChannelName.items()) + assert song.channels[ChannelName.PULSE1].patterns == {} + + def test_a_sample_the_catalog_lacks_is_refused(self) -> None: + with pytest.raises(KeyError, match="snare"): + build_song(_spec(ChannelName.NOISE, {0: RowSpec(row=0, sample="snare")}), _catalog()) + + def test_a_sample_on_a_channel_it_has_no_slice_for_is_refused(self) -> None: + with pytest.raises(ValueError, match="Sample 'lead' has no 'noise' slice"): + build_song(_spec(ChannelName.NOISE, {0: RowSpec(row=0, sample="lead")}), _catalog()) diff --git a/tests/unit/sampletones_tools/samples/commands/test_btp.py b/tests/unit/sampletones_tools/samples/commands/test_btp.py new file mode 100644 index 000000000..02c5801b2 --- /dev/null +++ b/tests/unit/sampletones_tools/samples/commands/test_btp.py @@ -0,0 +1,43 @@ +from pathlib import Path +from typing import Final, List, Tuple + +import pytest + +from sampletones.commands.registry import COMMANDS +from sampletones.dispatcher import dispatch +from sampletones_tools.samples.bitphase import write_samples +from sampletones_tools.samples.emit import Emitter + +EMITTER: Final[str] = "sampletones_tools.samples.emit.emit_samples" + + +class TestBtpSamples: + def test_the_bitphase_emitter_writes_into_the_output_and_every_file_is_printed( + self, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + ) -> None: + calls: List[Tuple[Path, Emitter]] = [] + + def emit_samples(output: Path, emitter: Emitter) -> List[Path]: + calls.append((output, emitter)) + return [output / "drums.btp"] + + monkeypatch.setattr(EMITTER, emit_samples) + + assert dispatch(COMMANDS, ["btp", "samples", "-o", str(tmp_path)]) == 0 + assert calls == [(tmp_path, write_samples)] + assert capsys.readouterr().out.splitlines() == [f"Wrote {tmp_path / 'drums.btp'}"] + + def test_the_output_is_required(self) -> None: + with pytest.raises(SystemExit) as leaving: + dispatch(COMMANDS, ["btp", "samples"]) + + assert leaving.value.code == 2 + + def test_an_action_is_required(self) -> None: + with pytest.raises(SystemExit) as leaving: + dispatch(COMMANDS, ["btp"]) + + assert leaving.value.code == 2 diff --git a/tests/unit/sampletones_tools/samples/commands/test_ftm.py b/tests/unit/sampletones_tools/samples/commands/test_ftm.py new file mode 100644 index 000000000..21d0b8c81 --- /dev/null +++ b/tests/unit/sampletones_tools/samples/commands/test_ftm.py @@ -0,0 +1,43 @@ +from pathlib import Path +from typing import Final, List, Tuple + +import pytest + +from sampletones.commands.registry import COMMANDS +from sampletones.dispatcher import dispatch +from sampletones_tools.samples.emit import Emitter +from sampletones_tools.samples.famitracker import write_samples + +EMITTER: Final[str] = "sampletones_tools.samples.emit.emit_samples" + + +class TestFtmSamples: + def test_the_famitracker_emitter_writes_into_the_output_and_every_file_is_printed( + self, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + ) -> None: + calls: List[Tuple[Path, Emitter]] = [] + + def emit_samples(output: Path, emitter: Emitter) -> List[Path]: + calls.append((output, emitter)) + return [output / "drums.ftm"] + + monkeypatch.setattr(EMITTER, emit_samples) + + assert dispatch(COMMANDS, ["ftm", "samples", "-o", str(tmp_path)]) == 0 + assert calls == [(tmp_path, write_samples)] + assert capsys.readouterr().out.splitlines() == [f"Wrote {tmp_path / 'drums.ftm'}"] + + def test_the_output_is_required(self) -> None: + with pytest.raises(SystemExit) as leaving: + dispatch(COMMANDS, ["ftm", "samples"]) + + assert leaving.value.code == 2 + + def test_an_action_is_required(self) -> None: + with pytest.raises(SystemExit) as leaving: + dispatch(COMMANDS, ["ftm"]) + + assert leaving.value.code == 2 diff --git a/tests/unit/sampletones_tools/samples/test_command.py b/tests/unit/sampletones_tools/samples/commands/test_nsf.py similarity index 62% rename from tests/unit/sampletones_tools/samples/test_command.py rename to tests/unit/sampletones_tools/samples/commands/test_nsf.py index 584e46ee7..1ef0970c2 100644 --- a/tests/unit/sampletones_tools/samples/test_command.py +++ b/tests/unit/sampletones_tools/samples/commands/test_nsf.py @@ -5,9 +5,41 @@ from sampletones.commands.registry import COMMANDS from sampletones.dispatcher import dispatch +from sampletones_tools.samples.emit import Emitter +from sampletones_tools.samples.nsf import write_samples from sampletones_tools.samples.render import RenderedWave, RenderingError RENDERER: Final[str] = "sampletones_tools.samples.render.render_directory" +EMITTER: Final[str] = "sampletones_tools.samples.emit.emit_samples" + + +class TestNsfSamples: + def test_the_nsf_emitter_writes_into_the_output_and_every_file_is_printed( + self, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + ) -> None: + calls: List[Tuple[Path, Emitter]] = [] + + def emit_samples(output: Path, emitter: Emitter) -> List[Path]: + calls.append((output, emitter)) + return [output / "kick.nsf", output / "song.nsf"] + + monkeypatch.setattr(EMITTER, emit_samples) + + assert dispatch(COMMANDS, ["nsf", "samples", "--output", str(tmp_path)]) == 0 + assert calls == [(tmp_path, write_samples)] + assert capsys.readouterr().out.splitlines() == [ + f"Wrote {tmp_path / 'kick.nsf'}", + f"Wrote {tmp_path / 'song.nsf'}", + ] + + def test_the_output_is_required(self) -> None: + with pytest.raises(SystemExit) as leaving: + dispatch(COMMANDS, ["nsf", "samples"]) + + assert leaving.value.code == 2 class TestNsfRender: From c2089f2108fefef40e846de1096b5d95a9f4d763 Mon Sep 17 00:00:00 2001 From: JakimPL Date: Sun, 13 Sep 2026 17:41:38 +0200 Subject: [PATCH 17/36] Organized: the development documents into subdirectories --- docs/concepts/stems.md | 2 +- docs/development/{ => application}/browser.md | 0 .../{ => application}/config-organization.md | 0 .../development/{ => application}/keyboard.md | 4 +- docs/development/{ => application}/palette.md | 4 +- .../development/{ => application}/playback.md | 2 +- .../{ => application}/render-thread.md | 2 +- .../{ => application}/sequencer-blocks.md | 8 ++-- docs/development/{ => application}/undo.md | 0 .../{ => application}/vocabularies.md | 2 +- docs/development/architecture.md | 20 +++++----- docs/development/guidelines.md | 5 ++- docs/development/packages.md | 2 +- docs/development/player.md | 2 +- .../{ => release}/compatibility.md | 6 +-- .../development/{ => release}/dependencies.md | 2 +- docs/development/tooling.md | 9 +++-- docs/formats/projects.md | 2 +- docs/formats/reconstructions.md | 2 +- docs/index.md | 37 ++++++++++++------- .../coordinators/playback/router.py | 2 +- src/sampletones_config/README.md | 2 +- 22 files changed, 63 insertions(+), 52 deletions(-) rename docs/development/{ => application}/browser.md (100%) rename docs/development/{ => application}/config-organization.md (100%) rename docs/development/{ => application}/keyboard.md (99%) rename docs/development/{ => application}/palette.md (94%) rename docs/development/{ => application}/playback.md (99%) rename docs/development/{ => application}/render-thread.md (99%) rename docs/development/{ => application}/sequencer-blocks.md (98%) rename docs/development/{ => application}/undo.md (100%) rename docs/development/{ => application}/vocabularies.md (99%) rename docs/development/{ => release}/compatibility.md (97%) rename docs/development/{ => release}/dependencies.md (99%) diff --git a/docs/concepts/stems.md b/docs/concepts/stems.md index f6d377f93..633cd7a2d 100644 --- a/docs/concepts/stems.md +++ b/docs/concepts/stems.md @@ -255,7 +255,7 @@ silences them. channels a stem newly reaches, and closing the document empties the card. 6. **Listening choices stay out of the document.** The ticked set is session state, like every choice that shapes what is heard — see - [Playback](../development/playback.md). Saving the reconstruction records + [Playback](../development/application/playback.md). Saving the reconstruction records the assignment, never the selection. So is the banding: collapsing the levels changes how the card draws, never what it describes. 7. **Removing a recording edits the document.** Where a box steers listening, diff --git a/docs/development/browser.md b/docs/development/application/browser.md similarity index 100% rename from docs/development/browser.md rename to docs/development/application/browser.md diff --git a/docs/development/config-organization.md b/docs/development/application/config-organization.md similarity index 100% rename from docs/development/config-organization.md rename to docs/development/application/config-organization.md diff --git a/docs/development/keyboard.md b/docs/development/application/keyboard.md similarity index 99% rename from docs/development/keyboard.md rename to docs/development/application/keyboard.md index 5572905cc..9c3b6aeff 100644 --- a/docs/development/keyboard.md +++ b/docs/development/application/keyboard.md @@ -6,7 +6,7 @@ declared and shown. It governs `utils/gui/keyboard/`, `utils/gui/shortcuts/`, th `sampletones_config/keybindings/`, and the menu surfaces that print an action. Consult it when giving a panel keys of its own, adding a shortcut, or putting an action on a menu. -The design truths it realizes are principles 12 and 14 of [`architecture.md`](architecture.md): +The design truths it realizes are principles 12 and 14 of [`architecture.md`](../architecture.md): one dispatcher owns the keyboard, and an action is declared once. This document holds the mechanism behind both. @@ -136,7 +136,7 @@ holds every shipped scheme to the same action set. ## Actions Declaring an action is a chain of four links, and the `shortcut-actions` check holds every one of -them (see [`architecture.md`](architecture.md) § Enforcement): +them (see [`architecture.md`](../architecture.md) § Enforcement): | Link | Where | What it states | |------|-------|----------------| diff --git a/docs/development/palette.md b/docs/development/application/palette.md similarity index 94% rename from docs/development/palette.md rename to docs/development/application/palette.md index 854cf7785..8ec3ed2be 100644 --- a/docs/development/palette.md +++ b/docs/development/application/palette.md @@ -4,7 +4,7 @@ This document describes how a color is written, composed, and handed to DearPyGu palette change costs. It governs `utils/palette/` and `utils/gui/palette/`. Consult it when adding a color the interface draws with, or a shade the interface derives from one. -The design truth it realizes is principle 13 of [`architecture.md`](architecture.md): a color is a +The design truth it realizes is principle 13 of [`architecture.md`](../architecture.md): a color is a token, resolved where it is drawn. This document holds the mechanism. --- @@ -37,4 +37,4 @@ DearPyGui holds as table state. The `palette-colors` hook holds all three rules — an attribute assigned a resolved `rgba`, a theme color filled outside the palette bindings, and a hex literal in the shipped configuration outside -`palettes/` (see [`architecture.md`](architecture.md) § Enforcement). +`palettes/` (see [`architecture.md`](../architecture.md) § Enforcement). diff --git a/docs/development/playback.md b/docs/development/application/playback.md similarity index 99% rename from docs/development/playback.md rename to docs/development/application/playback.md index c7c5b9f4e..0061ae77b 100644 --- a/docs/development/playback.md +++ b/docs/development/application/playback.md @@ -3,7 +3,7 @@ This document governs sound across the application: what may be heard, who decides, and what each transport command means. Consult it when adding audio a user can start, a surface that starts it, or a control over what is heard. The contracts here bind every tab and every player. It complements -`docs/development/architecture.md`, `docs/development/keyboard.md` (which owns the keyboard-routing +`docs/development/architecture.md`, `docs/development/application/keyboard.md` (which owns the keyboard-routing layer) and `docs/development/guidelines.md`. --- diff --git a/docs/development/render-thread.md b/docs/development/application/render-thread.md similarity index 99% rename from docs/development/render-thread.md rename to docs/development/application/render-thread.md index 9049d3eb2..2f04b15b0 100644 --- a/docs/development/render-thread.md +++ b/docs/development/application/render-thread.md @@ -6,7 +6,7 @@ its context, and what each crossing costs. It governs `utils/gui/render_thread.p a worker thread has something to show, when a gesture rebuilds widgets, or when work needs a frame to have been drawn first. -The design truth it realizes is principle 6 of [`architecture.md`](architecture.md): DearPyGui's +The design truth it realizes is principle 6 of [`architecture.md`](../architecture.md): DearPyGui's context belongs to the render thread. This document holds the mechanism. --- diff --git a/docs/development/sequencer-blocks.md b/docs/development/application/sequencer-blocks.md similarity index 98% rename from docs/development/sequencer-blocks.md rename to docs/development/application/sequencer-blocks.md index f8b779cfe..314167f89 100644 --- a/docs/development/sequencer-blocks.md +++ b/docs/development/application/sequencer-blocks.md @@ -7,8 +7,8 @@ and both grids — the tracker's pattern rows and the order's frames — carry t This document states the rules those gestures follow, how a block leaves the app as text, how a selection is drawn, and how a grid's actions reach the menus and the keyboard that fire them. The layering they sit in is -[Architecture](architecture.md); the conventions the code is held to are the -[coding guidelines](guidelines.md). +[Architecture](../architecture.md); the conventions the code is held to are the +[coding guidelines](../guidelines.md). ## Three vocabularies, kept apart @@ -123,7 +123,7 @@ SampleToNES/1 order rows=1 positions=0..1 The form and its reading live in `logic/sequencer/clipboard/`, which deals in blocks and strings alone; the desktop's clipboard is reached through `utils/gui/clipboard.py::TextClipboard`, one more piece of external behavior standing behind -a protocol ([Architecture](architecture.md), principle 11). The sequencer coordinator wires +a protocol ([Architecture](../architecture.md), principle 11). The sequencer coordinator wires the two. **A field prints what the grid prints in its cell**, which is what carries the three states @@ -172,7 +172,7 @@ needs no selection made first, and a menu raised inside a selection acts on the One builder means an action added to a grid appears at every door, and the accelerator **Edit** prints is the one that grid answers to, since a binding is declared once and every -reader of it reads that entry ([Architecture](architecture.md), principle 12). +reader of it reads that entry ([Architecture](../architecture.md), principle 12). `EditRouter` (`coordinators/edit/`) is the menu-side counterpart of the `KeyRouter` the keyboard runs through. Each surface states whether it owns the editing gestures at this diff --git a/docs/development/undo.md b/docs/development/application/undo.md similarity index 100% rename from docs/development/undo.md rename to docs/development/application/undo.md diff --git a/docs/development/vocabularies.md b/docs/development/application/vocabularies.md similarity index 99% rename from docs/development/vocabularies.md rename to docs/development/application/vocabularies.md index 8b8f4690c..1181ff4ac 100644 --- a/docs/development/vocabularies.md +++ b/docs/development/application/vocabularies.md @@ -6,7 +6,7 @@ are held to the source whole-tree by a pre-commit hook, and both keep in one pla otherwise be restated at every use. Consult this document when adding a string the reader sees, or a widget another module reaches. -The design truths it realizes are principles 8 and 9 of [`architecture.md`](architecture.md): all +The design truths it realizes are principles 8 and 9 of [`architecture.md`](../architecture.md): all display text comes from `LanguageManager`, and `tags/` holds only DPG identifiers. This document holds the grammar behind both. diff --git a/docs/development/architecture.md b/docs/development/architecture.md index da283775c..6fdee6c11 100644 --- a/docs/development/architecture.md +++ b/docs/development/architecture.md @@ -2,7 +2,7 @@ This document describes the design of `sampletones_application` — the GUI front-end of _SampleToNES_. It is prescriptive: it states the contracts each layer must honor, in the form they are enforced, and the rationale behind them. Use it as the reference when deciding where new code belongs. -Concrete classes and modules appear throughout as **examples** that anchor a rule; the rules bind every instance, named or not. Known deviations from these contracts are tracked in `docs/development/bugs-and-todos.md`. Coding-level rules live in `docs/development/guidelines.md`; the undo subsystem has its own design document, `docs/development/undo.md`, the audio transport has `docs/development/playback.md`, the reconstruction browser has `docs/development/browser.md`, the YAML configuration package has `docs/development/config-organization.md`, how a long operation says how far it has come has `docs/development/progress.md`, the keyboard and the actions it reaches have `docs/development/keyboard.md`, the identifier vocabularies have `docs/development/vocabularies.md`, colors and palettes have `docs/development/palette.md`, and the packages the repository divides into have `docs/development/packages.md`. +Concrete classes and modules appear throughout as **examples** that anchor a rule; the rules bind every instance, named or not. Known deviations from these contracts are tracked in `docs/development/bugs-and-todos.md`. Coding-level rules live in `docs/development/guidelines.md`; the undo subsystem has its own design document, `docs/development/application/undo.md`, the audio transport has `docs/development/application/playback.md`, the reconstruction browser has `docs/development/application/browser.md`, the YAML configuration package has `docs/development/application/config-organization.md`, how a long operation says how far it has come has `docs/development/progress.md`, the keyboard and the actions it reaches have `docs/development/application/keyboard.md`, the identifier vocabularies have `docs/development/application/vocabularies.md`, colors and palettes have `docs/development/application/palette.md`, and the packages the repository divides into have `docs/development/packages.md`. --- @@ -67,7 +67,7 @@ This decouples widget construction (which happens during `create_panel()`) from ### 6. DearPyGui's context belongs to the render thread -The thread that created the DearPyGui context is the only one that may build, configure, or destroy an item, and an item freed from another thread is freed with no Python thread state — a crash rather than a glitch. So work reaching the interface from anywhere else arrives on that thread first, through a crossing named for what it carries: a background result is queued for the render loop to drain, a worker's touch of a widget goes through `on_render_thread`, and a gesture DearPyGui gathered is run at the top of a frame. A crossing that would hold the frames up puts its waiting on a thread of its own, and work that needs a drawn frame names the frame it is picked up on. The four crossings, the helpers that make them, and the hazard each one answers are in [`render-thread.md`](render-thread.md). +The thread that created the DearPyGui context is the only one that may build, configure, or destroy an item, and an item freed from another thread is freed with no Python thread state — a crash rather than a glitch. So work reaching the interface from anywhere else arrives on that thread first, through a crossing named for what it carries: a background result is queued for the render loop to drain, a worker's touch of a widget goes through `on_render_thread`, and a gesture DearPyGui gathered is run at the top of a frame. A crossing that would hold the frames up puts its waiting on a thread of its own, and work that needs a drawn frame names the frame it is picked up on. The four crossings, the helpers that make them, and the hazard each one answers are in [`render-thread.md`](application/render-thread.md). ### 7. Construction flows from the composition root @@ -77,11 +77,11 @@ The thread that created the DearPyGui context is the only one that may build, co ### 8. All display text comes from `LanguageManager` -Every user-visible string is looked up on `LanguageManager` by the key the language file spells — `page.panel.text_type.element` — and resolves at the point of use, so a language change takes effect on the next read. `en.yaml` is a flat map keyed exactly this way, which makes the text system the single source of truth, lets a reader hold a key against the language file by eye, and enables future localization. A lookup states its key in a form the `language-keys` hook can read, so every key the code spells names an entry and every entry the file holds is reached. The grammar, the forms a lookup takes, and where each element enum lives are in [`vocabularies.md`](vocabularies.md). Log messages are developer-facing and exempt. +Every user-visible string is looked up on `LanguageManager` by the key the language file spells — `page.panel.text_type.element` — and resolves at the point of use, so a language change takes effect on the next read. `en.yaml` is a flat map keyed exactly this way, which makes the text system the single source of truth, lets a reader hold a key against the language file by eye, and enables future localization. A lookup states its key in a form the `language-keys` hook can read, so every key the code spells names an entry and every entry the file holds is reached. The grammar, the forms a lookup takes, and where each element enum lives are in [`vocabularies.md`](application/vocabularies.md). Log messages are developer-facing and exempt. ### 9. `tags/` holds only DPG identifiers -The `tags/` package contains only DPG widget string identifiers: `TAG_*` whole tags, and `SUF_*`/`PRE_*` fragments that compose into them. Dimensions, colors, timings, and display strings live in YAML configuration loaded at startup (`layout/`). Every tag reaches its final spelling through one composer, and a constant's name states the tag it composes, which the `tag-names` hook holds it to. The composer, the `TagName` spelling, and the rules a fragment follows are in [`vocabularies.md`](vocabularies.md). +The `tags/` package contains only DPG widget string identifiers: `TAG_*` whole tags, and `SUF_*`/`PRE_*` fragments that compose into them. Dimensions, colors, timings, and display strings live in YAML configuration loaded at startup (`layout/`). Every tag reaches its final spelling through one composer, and a constant's name states the tag it composes, which the `tag-names` hook holds it to. The composer, the `TagName` spelling, and the rules a fragment follows are in [`vocabularies.md`](application/vocabularies.md). ### 10. Exclusive operations expose a lifecycle-accurate active state @@ -108,17 +108,17 @@ DearPyGui gives every key handler the same global reach, so priority and consume A binding is declared once and read by everyone who prints or fires it: `ShortcutId` names the action together with the category that answers it, and the scheme under `sampletones_config/keybindings/` decides the combination, so a printed key and the handler behind it stay in step by construction. -The router is constructed at the composition root and injected into every consumer (principle 7). The scopes, the focus query, the modal stack, the key vocabulary, and how a scheme is chosen, layered, and edited are in [`keyboard.md`](keyboard.md). +The router is constructed at the composition root and injected into every consumer (principle 7). The scopes, the focus query, the modal stack, the key vocabulary, and how a scheme is chosen, layered, and edited are in [`keyboard.md`](application/keyboard.md). ### 13. A color is a token, resolved where it is drawn -A color is written as a palette token and stays one until it reaches DearPyGui. `BaseColor` (`utils/palette/colors/`) carries what was written, and its `rgba` property answers with the palette active at the moment of the read, so whoever holds the color follows a palette swap. Every annotation names `BaseColor`; the read happens where the value is handed to a widget, and what a consumer keeps is the token. What DearPyGui has already taken a copy of is registered with `PaletteBindings` rather than remembered by whoever set it, so a palette change is one switch. The `palette-colors` hook holds all three rules (see Enforcement); the color forms and the switch itself are in [`palette.md`](palette.md). +A color is written as a palette token and stays one until it reaches DearPyGui. `BaseColor` (`utils/palette/colors/`) carries what was written, and its `rgba` property answers with the palette active at the moment of the read, so whoever holds the color follows a palette swap. Every annotation names `BaseColor`; the read happens where the value is handed to a widget, and what a consumer keeps is the token. What DearPyGui has already taken a copy of is registered with `PaletteBindings` rather than remembered by whoever set it, so a palette change is one switch. The `palette-colors` hook holds all three rules (see Enforcement); the color forms and the switch itself are in [`palette.md`](application/palette.md). ### 14. An action is declared once; whoever shows it prints it An **action** is one `ShortcutId` — the name a key press, a menu item, and a context item all reach one behavior by. Declaring one is a chain of four links: the action and the category that answers it, its keys in every shipped scheme, the one call it makes, and the label the keybindings editor lists it by. The `shortcut-actions` check holds every link (see Enforcement). -A menu item is a view of an action: `ShortcutManager.add_menu_item(shortcut_id, ...)` takes both the accelerator and the call from the action and keeps the item under it, so a rebind re-prints the key already on screen. A set of actions several menus show is stated by one builder belonging to whoever owns them, and each door decides where to print it. A menu whose contents follow a selection states them when it is opened. The four links, the kinds of action that state their call differently, and the mechanism behind a restated menu are in [`keyboard.md`](keyboard.md). +A menu item is a view of an action: `ShortcutManager.add_menu_item(shortcut_id, ...)` takes both the accelerator and the call from the action and keeps the item under it, so a rebind re-prints the key already on screen. A set of actions several menus show is stated by one builder belonging to whoever owns them, and each door decides where to print it. A menu whose contents follow a selection states them when it is opened. The four links, the kinds of action that state their call differently, and the mechanism behind a restated menu are in [`keyboard.md`](application/keyboard.md). --- @@ -207,9 +207,9 @@ They read the source as an AST through the source layer in `sampletones_tools/ch *Logic objects* (e.g. `ConverterLogic`) orchestrate multi-step workflows within a feature area. They subscribe to services and translate service results into view model updates. -`logic/history/` implements the session-scoped undo engine (`HistoryManager`); its invariants and mechanics are documented in `docs/development/undo.md`. +`logic/history/` implements the session-scoped undo engine (`HistoryManager`); its invariants and mechanics are documented in `docs/development/application/undo.md`. -`logic/reconstruction/browser/` builds the tree of reconstructions both browser tabs render (`BrowserManager`); its pipeline, node vocabulary and shaping rules are documented in `docs/development/browser.md`. +`logic/reconstruction/browser/` builds the tree of reconstructions both browser tabs render (`BrowserManager`); its pipeline, node vocabulary and shaping rules are documented in `docs/development/application/browser.md`. **Contracts:** - Logic classes produce view models and may therefore import `view_model/`; they import neither `ui/` nor `coordinators/`. @@ -245,7 +245,7 @@ They read the source as an AST through the source layer in `sampletones_tools/ch There are two coordinator kinds: -*Domain coordinators* manage a cross-cutting concern that spans the whole application lifecycle — e.g. `ProjectCoordinator` (project file I/O, save confirmations), `PlaybackRouter` (the single transport over the shared output device, acting on the active tab's source or the engaged one — see `docs/development/playback.md`), or `EditRouter` (the single edit surface behind the menu bar's Edit menu, which shows the actions of the grid holding the cursor — see `docs/development/sequencer-blocks.md`). +*Domain coordinators* manage a cross-cutting concern that spans the whole application lifecycle — e.g. `ProjectCoordinator` (project file I/O, save confirmations), `PlaybackRouter` (the single transport over the shared output device, acting on the active tab's source or the engaged one — see `docs/development/application/playback.md`), or `EditRouter` (the single edit surface behind the menu bar's Edit menu, which shows the actions of the grid holding the cursor — see `docs/development/application/sequencer-blocks.md`). *Tab coordinators* own everything for one tab: they instantiate its panels, logic objects, and tab-scoped services, wire their callbacks together, and provide `create_tab()` — the single method that builds the DPG widget tree for that tab. Tab coordinators present a narrow public API of intent-level methods (`set_input_path`, `display_reconstruction`, …) and keep their panels and logic objects private. diff --git a/docs/development/guidelines.md b/docs/development/guidelines.md index 3ca1c3ccc..30c604180 100644 --- a/docs/development/guidelines.md +++ b/docs/development/guidelines.md @@ -2,7 +2,7 @@ These rules govern the Python in this repository. They complement `docs/development/architecture.md` (ownership and layering) and -`docs/development/config-organization.md` (configuration). +`docs/development/application/config-organization.md` (configuration). ## General @@ -19,7 +19,7 @@ These rules govern the Python in this repository. They complement 1. Prefer `pathlib.Path` over `os.path`. 1. Separate function options with `*`, and choose positional arguments intentionally. 1. Change internal APIs, configs, and data shapes freely; preserve backward compatibility only when the user explicitly asks. -1. Move a stored data version once per release. The version a build writes between releases is still being written, so a further change to that format extends the upgrade step already pending — one step carries the whole distance from the version the last release shipped. See `compatibility.md`. +1. Move a stored data version once per release. The version a build writes between releases is still being written, so a further change to that format extends the upgrade step already pending — one step carries the whole distance from the version the last release shipped. See [data compatibility](release/compatibility.md). 1. Run `pre-commit` on new files after each change. ## Ownership @@ -75,6 +75,7 @@ These rules govern the Python in this repository. They complement ## Documents 1. A document under `docs/` explains a subsystem to someone about to change it. Open by stating what it governs and when to consult it, so a reader learns in one paragraph whether they are in the right place. +1. A development document sits beside what it governs. The top of `docs/development/` holds what spans the repository's packages; `application/` holds what governs the graphical application alone, and `release/` what a release ships and keeps compatible. `docs/index.md` lists every document under the heading of its directory. 1. Lead with principles, then mechanics. A principle is a design truth you reason from; state the principles first, and let concrete conventions and reference tables follow as the way each principle is realized. 1. Keep principles, conventions, and descriptions distinct. A principle is a reason; a convention is a handy mechanic that serves it; a description is a fact about how something works. A convention promoted to a principle, or a principle buried in a description, misleads the reader about what is load-bearing. 1. Prefer a few strong principles to many narrow rules. When several rules are facets of one idea, state the idea once and derive them. A growing list of ad-hoc rules signals a principle that has gone unstated. diff --git a/docs/development/packages.md b/docs/development/packages.md index 622d64c83..b7c55a302 100644 --- a/docs/development/packages.md +++ b/docs/development/packages.md @@ -113,7 +113,7 @@ and the tests rebuild the sources and hold the committed image to them wherever installed. `sampletones_tools/player/trace/` holds `RegisterTrace`, what the driver is expected to write call by call, which the emulator tests hold the assembled driver to. The application ships the binary alone. The toolchain the build needs is described in -[`dependencies.md`](dependencies.md). +[`dependencies.md`](release/dependencies.md). --- diff --git a/docs/development/player.md b/docs/development/player.md index e402657da..910cd5e1d 100644 --- a/docs/development/player.md +++ b/docs/development/player.md @@ -179,4 +179,4 @@ package distributes. A build also holds the linker's own labels against the addr exporter states without one, so the committed image and the header describing it cannot drift apart. -Installing cc65 is covered in [dependencies](dependencies.md). +Installing cc65 is covered in [dependencies](release/dependencies.md). diff --git a/docs/development/compatibility.md b/docs/development/release/compatibility.md similarity index 97% rename from docs/development/compatibility.md rename to docs/development/release/compatibility.md index fca02ffdc..c2ffba193 100644 --- a/docs/development/compatibility.md +++ b/docs/development/release/compatibility.md @@ -9,9 +9,9 @@ The upgrades live in `sampletones_core/compatibility` and run at the load boundary of each format, before deserialization. The formats' own documents describe their stored shape and versioning: -- [`formats/reconstructions.md`](../formats/reconstructions.md) -- [`formats/instruction-libraries.md`](../formats/instruction-libraries.md) -- [`formats/projects.md`](../formats/projects.md) +- [`formats/reconstructions.md`](../../formats/reconstructions.md) +- [`formats/instruction-libraries.md`](../../formats/instruction-libraries.md) +- [`formats/projects.md`](../../formats/projects.md) ## Principles diff --git a/docs/development/dependencies.md b/docs/development/release/dependencies.md similarity index 99% rename from docs/development/dependencies.md rename to docs/development/release/dependencies.md index 94e62ff66..8570264bb 100644 --- a/docs/development/dependencies.md +++ b/docs/development/release/dependencies.md @@ -12,7 +12,7 @@ The core depends on common Python packages: * `librosa` * `cupy` (optional; enables the GPU backend, with the build selected for your NVIDIA driver) -See [GPU acceleration](../guide/installation.md#gpu-acceleration) for enabling it. +See [GPU acceleration](../../guide/installation.md#gpu-acceleration) for enabling it. ## Serialization diff --git a/docs/development/tooling.md b/docs/development/tooling.md index c0e3763a3..e301218c0 100644 --- a/docs/development/tooling.md +++ b/docs/development/tooling.md @@ -1,9 +1,10 @@ # Tooling -This document governs how the repository is run: the `sampletones` command and what it offers, -the scripts under `scripts/`, and the `Makefile`. Read it before adding a command, a script or a -make target. Which packages may import which is [package layers](packages.md); the libraries and -tools the scripts reach for are [dependencies](dependencies.md). +This document governs how the repository is run: the `sampletones` command and what it offers, the +tools package behind its developer commands, the scripts under `scripts/`, and the `Makefile`. Read +it before adding a command, a tool, a script or a make target. Which packages may import which is +[package layers](packages.md); the libraries and tools the scripts reach for are +[dependencies](release/dependencies.md). ## Principles diff --git a/docs/formats/projects.md b/docs/formats/projects.md index d11b4d613..7da88b12d 100644 --- a/docs/formats/projects.md +++ b/docs/formats/projects.md @@ -70,7 +70,7 @@ _SampleToNES_ requires that version to match the one it supports and declines an incompatible file rather than misreading it. A file written at a version the upgrade chain reaches is migrated in memory to the current shape before deserialization (see -[Data compatibility](../development/compatibility.md)). Unknown or extra fields +[Data compatibility](../development/release/compatibility.md)). Unknown or extra fields within a matching version are ignored, which leaves room for the format to grow. The current format version is 1.1. It gathers the pool under `voices`, each record diff --git a/docs/formats/reconstructions.md b/docs/formats/reconstructions.md index ca1f9bcb3..718a99f20 100644 --- a/docs/formats/reconstructions.md +++ b/docs/formats/reconstructions.md @@ -89,7 +89,7 @@ _SampleToNES_ requires that version to match the one it supports and declines a file written by an incompatible version rather than misreading it. A file written at a version the upgrade chain reaches is migrated in memory to the current shape before deserialization (see -[Data compatibility](../development/compatibility.md)); the application version +[Data compatibility](../development/release/compatibility.md)); the application version is stored alongside the data version, for reference. The current data version is 2.2. Version 2.2 renamed the per-channel stream and diff --git a/docs/index.md b/docs/index.md index ee57da776..ee95a56f9 100644 --- a/docs/index.md +++ b/docs/index.md @@ -57,26 +57,35 @@ worked examples. ## Development -The [**development**](development/) section is for contributors. +The [**development**](development/) section is for contributors. The documents at its top +span the whole repository; the documents about the graphical application and about a release +each have a directory of their own. - [Architecture](development/architecture.md) — the application's layers and the contracts between them. - [Package layers](development/packages.md) — the packages the repository divides into, and the order they import each other in. -- [Undo engine](development/undo.md) — the design of the undo/redo subsystem. -- [Sequencer blocks](development/sequencer-blocks.md) — the rules copy, cut, paste and delete follow on both grids. -- [Keyboard and actions](development/keyboard.md) — how a press reaches behavior, and how an action is declared and shown. -- [Identifier vocabularies](development/vocabularies.md) — the keys display text is looked up by, and the tags DearPyGui knows a widget by. -- [Colors and palettes](development/palette.md) — how a color is written, composed, and handed to DearPyGui. -- [The render thread](development/render-thread.md) — how work reaches DearPyGui from another thread, and what each crossing costs. -- [Playback](development/playback.md) — the audio transport shared by every view, and rendering the song to a file. -- [Progress](development/progress.md) — how a long operation says how far it has come, in one process and across the pool's workers. -- [Console player](development/player.md) — the 6502 driver an `.nsf` carries, the codec that fits a song beside it, and how both are verified. -- [Reconstruction browser](development/browser.md) — how a reconstructions directory becomes the tree both browser tabs render, and what narrows it. -- [Configuration](development/config-organization.md) — how the YAML configuration package is laid out. +- [Tooling](development/tooling.md) — the `sampletones` command, the tools package and the bootstrap scripts: what each runs on and what it may import. - [Coding guidelines](development/guidelines.md) — conventions for the codebase. -- [Dependencies](development/dependencies.md) — the libraries _SampleToNES_ builds on. -- [Tooling](development/tooling.md) — the scripts and the Makefile: what runs on the system interpreter, what runs in the project environment. +- [Console player](development/player.md) — the 6502 driver an `.nsf` carries, the codec that fits a song beside it, and how both are verified. +- [Progress](development/progress.md) — how a long operation says how far it has come, in one process and across the pool's workers. - [Bugs and to-dos](development/bugs-and-todos.md) — the working ledger of known gaps. +### The application + +- [Undo engine](development/application/undo.md) — the design of the undo/redo subsystem. +- [Sequencer blocks](development/application/sequencer-blocks.md) — the rules copy, cut, paste and delete follow on both grids. +- [Keyboard and actions](development/application/keyboard.md) — how a press reaches behavior, and how an action is declared and shown. +- [Identifier vocabularies](development/application/vocabularies.md) — the keys display text is looked up by, and the tags DearPyGui knows a widget by. +- [Colors and palettes](development/application/palette.md) — how a color is written, composed, and handed to DearPyGui. +- [The render thread](development/application/render-thread.md) — how work reaches DearPyGui from another thread, and what each crossing costs. +- [Playback](development/application/playback.md) — the audio transport shared by every view, and rendering the song to a file. +- [Reconstruction browser](development/application/browser.md) — how a reconstructions directory becomes the tree both browser tabs render, and what narrows it. +- [Configuration](development/application/config-organization.md) — how the YAML configuration package is laid out. + +### Releases + +- [Data compatibility](development/release/compatibility.md) — the upgrades that bring a file an older version saved up to the current format as it loads. +- [Dependencies](development/release/dependencies.md) — the libraries _SampleToNES_ builds on. + ## Glossary The [glossary](glossary.md) defines the recurring terms — NES hardware, the diff --git a/src/sampletones_application/coordinators/playback/router.py b/src/sampletones_application/coordinators/playback/router.py index 194460f3d..ab1206d27 100644 --- a/src/sampletones_application/coordinators/playback/router.py +++ b/src/sampletones_application/coordinators/playback/router.py @@ -13,7 +13,7 @@ class PlaybackRouter: Every command acts on one *target*: the active tab's own source when that tab has something to play, and otherwise the intentional source currently engaged (owning the device). Preview sounds belong to no source and answer only to Stop. The full contract is documented in - ``docs/development/playback.md``. + ``docs/development/application/playback.md``. It is stateless: the target is recomputed from the live sources on each call, so the transport stays in step with tab changes and playback transitions automatically. diff --git a/src/sampletones_config/README.md b/src/sampletones_config/README.md index 3d4d030c9..2d9d3728c 100644 --- a/src/sampletones_config/README.md +++ b/src/sampletones_config/README.md @@ -29,4 +29,4 @@ The data package must not import a schema, and a schema package must not inline The rules for where a value belongs, how the directories nest, and how each domain is loaded are prescriptive and documented in -[`docs/development/config-organization.md`](../../docs/development/config-organization.md). +[`docs/development/application/config-organization.md`](../../docs/development/application/config-organization.md). From 6b40b703d27b45168f205dcc9ce4ec74313f4a55 Mon Sep 17 00:00:00 2001 From: JakimPL Date: Sun, 13 Sep 2026 18:37:12 +0200 Subject: [PATCH 18/36] Reformat --- src/sampletones/__init__.py | 2 +- src/sampletones/commands/convert.py | 7 ++- src/sampletones/commands/library.py | 7 ++- src/sampletones/commands/open.py | 7 ++- src/sampletones/commands/run.py | 7 ++- src/sampletones/commands/self_check.py | 7 ++- src/sampletones/dispatcher.py | 14 ++++- src/sampletones/self_check.py | 14 ++++- src/sampletones_tools/assets/command.py | 7 ++- src/sampletones_tools/assets/mark/geometry.py | 16 +++++- src/sampletones_tools/assets/mark/raster.py | 21 +++++-- src/sampletones_tools/assets/mark/suite.py | 12 +++- src/sampletones_tools/calibration/command.py | 7 ++- .../calibration/config/corpus.py | 36 +++++++++--- .../calibration/config/referee.py | 18 ++++-- .../calibration/corpus/synthesis.py | 23 ++++++-- .../calibration/referee/auditory.py | 20 ++++++- src/sampletones_tools/calibration/report.py | 10 +++- src/sampletones_tools/calibration/session.py | 7 ++- .../checks/boundary/standalone.py | 7 ++- src/sampletones_tools/checks/command.py | 21 ++++++- .../checks/commands/import_boundary.py | 7 ++- .../checks/commands/language_keys.py | 26 +++++++-- .../checks/commands/palette_colors.py | 19 ++++++- .../checks/commands/rendered_literals.py | 22 +++++++- .../checks/commands/shortcut_actions.py | 7 ++- .../checks/commands/tag_names.py | 12 +++- .../checks/commands/unused_tags.py | 19 ++++++- src/sampletones_tools/checks/language_keys.py | 22 +++++++- .../checks/palette_colors.py | 5 +- .../checks/rendered_literals.py | 10 +++- .../checks/shortcut_actions.py | 5 +- .../checks/source/lookups.py | 24 +++++++- src/sampletones_tools/checks/tag_names.py | 28 ++++++++-- .../codec/report/encoding.py | 55 +++++++++++++++++-- src/sampletones_tools/codec/study/manifest.py | 1 + .../codec/study/report/verdicts.py | 8 ++- .../codec/study/sandbox/parse.py | 19 ++++++- .../codec/study/variants/production.py | 6 +- .../codec/study/variants/sandbox.py | 5 +- src/sampletones_tools/corpus/build.py | 11 +++- src/sampletones_tools/corpus/song.py | 17 +++++- src/sampletones_tools/player/command.py | 7 ++- src/sampletones_tools/registry.py | 11 +++- src/sampletones_tools/samples/commands/btp.py | 17 +++++- src/sampletones_tools/samples/commands/ftm.py | 17 +++++- src/sampletones_tools/samples/commands/nsf.py | 43 ++++++++++++--- src/sampletones_tools/samples/nsf.py | 5 +- src/sampletones_tools/samples/render.py | 12 +++- 49 files changed, 598 insertions(+), 112 deletions(-) diff --git a/src/sampletones/__init__.py b/src/sampletones/__init__.py index 60d9ed3cc..a059dd93b 100644 --- a/src/sampletones/__init__.py +++ b/src/sampletones/__init__.py @@ -74,9 +74,9 @@ def __getattr__(name: str) -> Any: __all__ = [ + "ChannelName", "Config", "Generator", - "ChannelName", "Instruction", "InstructionLibrary", "NoiseGenerator", diff --git a/src/sampletones/commands/convert.py b/src/sampletones/commands/convert.py index 8074b8b24..66183ac2e 100644 --- a/src/sampletones/commands/convert.py +++ b/src/sampletones/commands/convert.py @@ -78,4 +78,9 @@ def run(arguments: Namespace) -> int: return 0 -CONVERT: Final[Command] = Command(name=NAME, help=HELP, configure=configure, run=run) +CONVERT: Final[Command] = Command( + name=NAME, + help=HELP, + configure=configure, + run=run, +) diff --git a/src/sampletones/commands/library.py b/src/sampletones/commands/library.py index 2e83d8c15..09308690f 100644 --- a/src/sampletones/commands/library.py +++ b/src/sampletones/commands/library.py @@ -34,4 +34,9 @@ def run(arguments: Namespace) -> int: return 0 -LIBRARY: Final[Command] = Command(name=NAME, help=HELP, configure=configure, run=run) +LIBRARY: Final[Command] = Command( + name=NAME, + help=HELP, + configure=configure, + run=run, +) diff --git a/src/sampletones/commands/open.py b/src/sampletones/commands/open.py index c6c576cb2..8db31885c 100644 --- a/src/sampletones/commands/open.py +++ b/src/sampletones/commands/open.py @@ -65,4 +65,9 @@ def run(arguments: Namespace) -> int: return 0 -OPEN: Final[Command] = Command(name=NAME, help=HELP, configure=configure, run=run) +OPEN: Final[Command] = Command( + name=NAME, + help=HELP, + configure=configure, + run=run, +) diff --git a/src/sampletones/commands/run.py b/src/sampletones/commands/run.py index 241c088ac..9f5a3e1d1 100644 --- a/src/sampletones/commands/run.py +++ b/src/sampletones/commands/run.py @@ -33,4 +33,9 @@ def run(arguments: Namespace) -> int: return 0 -RUN: Final[Command] = Command(name=NAME, help=HELP, configure=configure, run=run) +RUN: Final[Command] = Command( + name=NAME, + help=HELP, + configure=configure, + run=run, +) diff --git a/src/sampletones/commands/self_check.py b/src/sampletones/commands/self_check.py index df4e3bc95..a9957caa0 100644 --- a/src/sampletones/commands/self_check.py +++ b/src/sampletones/commands/self_check.py @@ -20,4 +20,9 @@ def run(arguments: Namespace) -> int: return run_self_check() -SELF_CHECK: Final[Command] = Command(name=NAME, help=HELP, configure=configure, run=run) +SELF_CHECK: Final[Command] = Command( + name=NAME, + help=HELP, + configure=configure, + run=run, +) diff --git a/src/sampletones/dispatcher.py b/src/sampletones/dispatcher.py index 9e1d315ee..1c01c4f37 100644 --- a/src/sampletones/dispatcher.py +++ b/src/sampletones/dispatcher.py @@ -33,8 +33,18 @@ def build_parser(commands: Sequence[Command]) -> ArgumentParser: description=DESCRIPTION, epilog=f"Run '{PROGRAM} {COMMAND_METAVAR} --help' for a command's options.", ) - parser.add_argument("--version", "-v", action="version", version=SAMPLETONES_NAME_VERSION) - subparsers = parser.add_subparsers(dest=COMMAND_FIELD, metavar=COMMAND_METAVAR, required=True) + parser.add_argument( + "--version", + "-v", + action="version", + version=SAMPLETONES_NAME_VERSION, + ) + subparsers = parser.add_subparsers( + dest=COMMAND_FIELD, + metavar=COMMAND_METAVAR, + required=True, + ) + for command in commands: subparser = subparsers.add_parser(command.name, help=command.help, description=command.help) command.configure(subparser) diff --git a/src/sampletones/self_check.py b/src/sampletones/self_check.py index 30fe3b407..9841e8f94 100644 --- a/src/sampletones/self_check.py +++ b/src/sampletones/self_check.py @@ -115,7 +115,10 @@ def _check_language() -> str: def _check_resources() -> str: from sampletones_application.ui.resources.items import FontResource, IconResource - from sampletones_application.ui.resources.resources import get_font_path, get_icon_path + from sampletones_application.ui.resources.resources import ( + get_font_path, + get_icon_path, + ) for font in FontResource: get_font_path(font) @@ -138,7 +141,9 @@ def _check_export_backends() -> str: def _check_file_dialog_backend() -> str: - from sampletones_application.utils.file_dialogs.selection import select_file_dialog_backend + from sampletones_application.utils.file_dialogs.selection import ( + select_file_dialog_backend, + ) return type(select_file_dialog_backend()).__name__ @@ -168,7 +173,10 @@ def run_self_check() -> int: try: detail = check.run() except CHECK_FAILURES as exception: - print(f"{FAILURE_PREFIX} {check.name}: {type(exception).__name__}: {exception}", file=sys.stderr) + print( + f"{FAILURE_PREFIX} {check.name}: {type(exception).__name__}: {exception}", + file=sys.stderr, + ) return FAILURE_STATUS print(f"{SUCCESS_PREFIX} {check.name}: {detail}") diff --git a/src/sampletones_tools/assets/command.py b/src/sampletones_tools/assets/command.py index 38505e62f..340729397 100644 --- a/src/sampletones_tools/assets/command.py +++ b/src/sampletones_tools/assets/command.py @@ -46,4 +46,9 @@ def run(arguments: Namespace) -> int: return 0 -ICONS: Final[Command] = Command(name=NAME, help=HELP, configure=configure, run=run) +ICONS: Final[Command] = Command( + name=NAME, + help=HELP, + configure=configure, + run=run, +) diff --git a/src/sampletones_tools/assets/mark/geometry.py b/src/sampletones_tools/assets/mark/geometry.py index fcbac6320..e268dd3b9 100644 --- a/src/sampletones_tools/assets/mark/geometry.py +++ b/src/sampletones_tools/assets/mark/geometry.py @@ -38,8 +38,20 @@ def _cubic_point( progress: float, ) -> Point: return Point( - x=_cubic_coordinate(start.x, curve.control_start.x, curve.control_end.x, curve.end.x, progress), - y=_cubic_coordinate(start.y, curve.control_start.y, curve.control_end.y, curve.end.y, progress), + x=_cubic_coordinate( + start.x, + curve.control_start.x, + curve.control_end.x, + curve.end.x, + progress, + ), + y=_cubic_coordinate( + start.y, + curve.control_start.y, + curve.control_end.y, + curve.end.y, + progress, + ), ) diff --git a/src/sampletones_tools/assets/mark/raster.py b/src/sampletones_tools/assets/mark/raster.py index 537130d0f..ccd244c20 100644 --- a/src/sampletones_tools/assets/mark/raster.py +++ b/src/sampletones_tools/assets/mark/raster.py @@ -55,7 +55,11 @@ def _background(self) -> Image.Image: size = (self.canvas, self.canvas) top = Image.new("RGB", size, self.mark.colors.background.top) bottom = Image.new("RGB", size, self.mark.colors.background.bottom) - shaded = Image.composite(bottom, top, Image.linear_gradient("L").resize(size)) + shaded = Image.composite( + bottom, + top, + Image.linear_gradient("L").resize(size), + ) background = Image.new("RGBA", size, TRANSPARENT) background.paste(shaded, mask=self._frame_mask()) @@ -77,7 +81,10 @@ def _draw_sine(self, draw: ImageDraw.ImageDraw) -> None: what holds the outline smooth along its whole sweep. """ radius = self.mark.waves.width * self.scale / 2 - for point in sine_points(self.mark.waves.sine, self.mark.render.curve_samples): + for point in sine_points( + self.mark.waves.sine, + self.mark.render.curve_samples, + ): center_x, center_y = point.x * self.scale, point.y * self.scale draw.ellipse( ( @@ -90,7 +97,10 @@ def _draw_sine(self, draw: ImageDraw.ImageDraw) -> None: ) def _draw_square(self, draw: ImageDraw.ImageDraw) -> None: - for rectangle in square_rectangles(self.mark.waves.square, self.mark.waves.width): + for rectangle in square_rectangles( + self.mark.waves.square, + self.mark.waves.width, + ): draw.rectangle( ( round(rectangle.left * self.scale), @@ -113,4 +123,7 @@ def _rim(self) -> Image.Image: return overlay def _rim_color(self) -> ColorRGBA: - return with_alpha_fraction(parse_hex_color(self.mark.colors.rim), self.mark.frame.rim.opacity) + return with_alpha_fraction( + parse_hex_color(self.mark.colors.rim), + self.mark.frame.rim.opacity, + ) diff --git a/src/sampletones_tools/assets/mark/suite.py b/src/sampletones_tools/assets/mark/suite.py index 76adaba53..a6bd27c00 100644 --- a/src/sampletones_tools/assets/mark/suite.py +++ b/src/sampletones_tools/assets/mark/suite.py @@ -62,6 +62,14 @@ def write_icon_suite(directory: Path, mark: Mark) -> List[Path]: return [ _write_vector(directory / ICON_VECTOR_FILENAME, mark), - _write_raster(directory / ICON_UNIX_FILENAME, master, mark.render.raster_size), - _write_windows_icon(directory / ICON_WIN_FILENAME, master, mark.render.windows_sizes), + _write_raster( + directory / ICON_UNIX_FILENAME, + master, + mark.render.raster_size, + ), + _write_windows_icon( + directory / ICON_WIN_FILENAME, + master, + mark.render.windows_sizes, + ), ] diff --git a/src/sampletones_tools/calibration/command.py b/src/sampletones_tools/calibration/command.py index 846bc791b..035957bea 100644 --- a/src/sampletones_tools/calibration/command.py +++ b/src/sampletones_tools/calibration/command.py @@ -85,4 +85,9 @@ def run(arguments: Namespace) -> int: return 0 -CALIBRATION: Final[Command] = Command(name=NAME, help=HELP, configure=configure, run=run) +CALIBRATION: Final[Command] = Command( + name=NAME, + help=HELP, + configure=configure, + run=run, +) diff --git a/src/sampletones_tools/calibration/config/corpus.py b/src/sampletones_tools/calibration/config/corpus.py index ad3c45bfe..d6805445f 100644 --- a/src/sampletones_tools/calibration/config/corpus.py +++ b/src/sampletones_tools/calibration/config/corpus.py @@ -23,18 +23,38 @@ class CorpusConfig(BaseModel, frozen=True): remaining adjustable in one place. """ - seed: int = Field(ge=0, description="Seed of the random generator behind the stochastic probes.") - item_seconds: float = Field(gt=0.0, description="Duration of every corpus item in seconds.") - amplitude: float = Field(gt=0.0, le=1.0, description="Scale applied to every unit-level probe.") + seed: int = Field( + ge=0, + description="Seed of the random generator behind the stochastic probes.", + ) + item_seconds: float = Field( + gt=0.0, + description="Duration of every corpus item in seconds.", + ) + amplitude: float = Field( + gt=0.0, + le=1.0, + description="Scale applied to every unit-level probe.", + ) reference_frequency: float = Field( gt=0.0, description="Anchor tone in Hz shared by the mix, pluck, and crescendo probes.", ) - tone: ToneConfig = Field(description="Steady sine probes.") - timbre: TimbreConfig = Field(description="Pulse-wave probes.") - noise: NoiseConfig = Field(description="Broadband noise probes.") - mix: MixConfig = Field(description="Tone-plus-noise probes.") - transient: TransientConfig = Field(description="Percussive probes.") + tone: ToneConfig = Field( + description="Steady sine probes.", + ) + timbre: TimbreConfig = Field( + description="Pulse-wave probes.", + ) + noise: NoiseConfig = Field( + description="Broadband noise probes.", + ) + mix: MixConfig = Field( + description="Tone-plus-noise probes.", + ) + transient: TransientConfig = Field( + description="Percussive probes.", + ) @classmethod def load(cls) -> Self: diff --git a/src/sampletones_tools/calibration/config/referee.py b/src/sampletones_tools/calibration/config/referee.py index bb4cafc6d..2d8992116 100644 --- a/src/sampletones_tools/calibration/config/referee.py +++ b/src/sampletones_tools/calibration/config/referee.py @@ -19,10 +19,20 @@ class RefereeConfig(BaseModel, frozen=True): min_length=1, description="STFT window sizes covering the time-frequency trade-off.", ) - hop_divisor: PositiveInt = Field(description="Window size divided by this gives the STFT hop.") - band_count: PositiveInt = Field(description="Number of ERB-spaced aggregation bands.") - low_frequency: float = Field(gt=0.0, description="Lower bound of the band axis in Hz.") - energy_floor: float = Field(gt=0.0, description="Absolute floor keeping silent band energies finite.") + hop_divisor: PositiveInt = Field( + description="Window size divided by this gives the STFT hop.", + ) + band_count: PositiveInt = Field( + description="Number of ERB-spaced aggregation bands.", + ) + low_frequency: float = Field( + gt=0.0, + description="Lower bound of the band axis in Hz.", + ) + energy_floor: float = Field( + gt=0.0, + description="Absolute floor keeping silent band energies finite.", + ) audibility_range_decibels: float = Field( gt=0.0, description="Audible range below the reference's loudest band; quieter content saturates.", diff --git a/src/sampletones_tools/calibration/corpus/synthesis.py b/src/sampletones_tools/calibration/corpus/synthesis.py index 52bbfe694..e24450e6b 100644 --- a/src/sampletones_tools/calibration/corpus/synthesis.py +++ b/src/sampletones_tools/calibration/corpus/synthesis.py @@ -4,7 +4,9 @@ from sampletones_core.audio.processing import clip_audio from sampletones_tools.calibration.config.corpus import CorpusConfig -from sampletones_tools.synthesis.envelopes.exponential_decay import ExponentialDecayEnvelope +from sampletones_tools.synthesis.envelopes.exponential_decay import ( + ExponentialDecayEnvelope, +) from sampletones_tools.synthesis.envelopes.linear_attack import LinearAttackEnvelope from sampletones_tools.synthesis.envelopes.linear_ramp import LinearRampEnvelope from sampletones_tools.synthesis.envelopes.types import EnvelopeUnion @@ -193,8 +195,16 @@ def _transient_probes(config: CorpusConfig) -> List[Probe]: "transient", _voice(config, _layer(WhiteNoiseOscillator(kind="white_noise"), snare_decay)), ), - ("transient-kick", "transient", _voice(config, _layer(kick_glide, kick_decay))), - ("transient-pluck", "transient", _voice(config, _layer(reference_tone, pluck_attack, pluck_decay))), + ( + "transient-kick", + "transient", + _voice(config, _layer(kick_glide, kick_decay)), + ), + ( + "transient-pluck", + "transient", + _voice(config, _layer(reference_tone, pluck_attack, pluck_decay)), + ), ] @@ -228,7 +238,12 @@ def _voice(config: CorpusConfig, *layers: Layer) -> Voice: ) -def _item(name: str, category: str, audio: np.ndarray, config: CorpusConfig) -> CorpusItem: +def _item( + name: str, + category: str, + audio: np.ndarray, + config: CorpusConfig, +) -> CorpusItem: """ Assemble a corpus item from a unit-scale probe. diff --git a/src/sampletones_tools/calibration/referee/auditory.py b/src/sampletones_tools/calibration/referee/auditory.py index db124d054..aebb873cd 100644 --- a/src/sampletones_tools/calibration/referee/auditory.py +++ b/src/sampletones_tools/calibration/referee/auditory.py @@ -64,7 +64,12 @@ def score(self, reference: np.ndarray, estimate: np.ndarray) -> float: return float(np.mean(distances)) - def _band_energy(self, audio: np.ndarray, window_size: int, band_matrix: np.ndarray) -> np.ndarray: + def _band_energy( + self, + audio: np.ndarray, + window_size: int, + band_matrix: np.ndarray, + ) -> np.ndarray: hop = window_size // self.config.hop_divisor _, _, spectrum = stft( audio.astype(np.float64), @@ -77,7 +82,12 @@ def _band_energy(self, audio: np.ndarray, window_size: int, band_matrix: np.ndar return band_energy @classmethod - def _band_matrix(cls, sample_rate: int, window_size: int, config: RefereeConfig) -> np.ndarray: + def _band_matrix( + cls, + sample_rate: int, + window_size: int, + config: RefereeConfig, + ) -> np.ndarray: """ Rectangular aggregation matrix from STFT bins onto ERB-spaced bands. @@ -93,7 +103,11 @@ def _band_matrix(cls, sample_rate: int, window_size: int, config: RefereeConfig) config.band_count + 1, ) bin_rates = cls._erb_rate(frequencies) - band_indices = np.clip(np.searchsorted(edges_rate, bin_rates, side="right") - 1, 0, config.band_count - 1) + band_indices = np.clip( + np.searchsorted(edges_rate, bin_rates, side="right") - 1, + 0, + config.band_count - 1, + ) matrix = np.zeros((config.band_count, frequencies.shape[0])) matrix[band_indices, np.arange(frequencies.shape[0])] = 1.0 diff --git a/src/sampletones_tools/calibration/report.py b/src/sampletones_tools/calibration/report.py index f9a9ceae2..1f42d0f35 100644 --- a/src/sampletones_tools/calibration/report.py +++ b/src/sampletones_tools/calibration/report.py @@ -20,7 +20,15 @@ def write_csv(rows: List[CalibrationRow], path: Path) -> None: writer = csv.writer(handle) writer.writerow(["variant", "item", "category", "referee", "score"]) for row in rows: - writer.writerow([row.variant, row.item, row.category, row.referee, f"{row.score:.6f}"]) + writer.writerow( + [ + row.variant, + row.item, + row.category, + row.referee, + f"{row.score:.6f}", + ] + ) def write_markdown(rows: List[CalibrationRow], path: Path) -> None: diff --git a/src/sampletones_tools/calibration/session.py b/src/sampletones_tools/calibration/session.py index 0266a256d..87155b373 100644 --- a/src/sampletones_tools/calibration/session.py +++ b/src/sampletones_tools/calibration/session.py @@ -112,7 +112,12 @@ def calibrate(request: CalibrationRequest) -> Path: items = build_corpus(sample_rate, config=CorpusConfig.load()) item_paths = write_corpus(items, request.output / CORPUS_DIRECTORY, sample_rate) referees = build_referees(sample_rate) - variants = build_variants(base, request.methods, request.perceptual_exponents, request.temporal_weights) + variants = build_variants( + base, + request.methods, + request.perceptual_exponents, + request.temporal_weights, + ) channel_names = ", ".join(channel.value for channel in request.channels) logger.info( diff --git a/src/sampletones_tools/checks/boundary/standalone.py b/src/sampletones_tools/checks/boundary/standalone.py index 9ca4e23e0..eb558ecf9 100644 --- a/src/sampletones_tools/checks/boundary/standalone.py +++ b/src/sampletones_tools/checks/boundary/standalone.py @@ -106,7 +106,12 @@ def shadowing(self, root: Path, paths: Sequence[Path]) -> List[Violation]: for part in path.relative_to(root).with_suffix("").parts: if part in taken and part not in seen: seen.add(part) - violations.append(Violation(kind=f"{part} {SHADOWING}", location=str(path))) + violations.append( + Violation( + kind=f"{part} {SHADOWING}", + location=str(path), + ) + ) return violations diff --git a/src/sampletones_tools/checks/command.py b/src/sampletones_tools/checks/command.py index fe46657d2..156138c2e 100644 --- a/src/sampletones_tools/checks/command.py +++ b/src/sampletones_tools/checks/command.py @@ -11,9 +11,19 @@ def configure(parser: ArgumentParser) -> None: - gates = parser.add_subparsers(dest=GATE_FIELD, metavar=GATE_METAVAR, required=True) + gates = parser.add_subparsers( + dest=GATE_FIELD, + metavar=GATE_METAVAR, + required=True, + ) for gate in GATES: - gate.configure(gates.add_parser(gate.name, help=gate.help, description=gate.help)) + gate.configure( + gates.add_parser( + gate.name, + help=gate.help, + description=gate.help, + ) + ) def run(arguments: Namespace) -> int: @@ -30,4 +40,9 @@ def run(arguments: Namespace) -> int: return gate.run(arguments) -CHECK: Final[Command] = Command(name=NAME, help=HELP, configure=configure, run=run) +CHECK: Final[Command] = Command( + name=NAME, + help=HELP, + configure=configure, + run=run, +) diff --git a/src/sampletones_tools/checks/commands/import_boundary.py b/src/sampletones_tools/checks/commands/import_boundary.py index c1275a6c4..b4bf78188 100644 --- a/src/sampletones_tools/checks/commands/import_boundary.py +++ b/src/sampletones_tools/checks/commands/import_boundary.py @@ -51,4 +51,9 @@ def run(arguments: Namespace) -> int: return report(violations) -IMPORT_BOUNDARY: Final[Command] = Command(name=NAME, help=HELP, configure=configure, run=run) +IMPORT_BOUNDARY: Final[Command] = Command( + name=NAME, + help=HELP, + configure=configure, + run=run, +) diff --git a/src/sampletones_tools/checks/commands/language_keys.py b/src/sampletones_tools/checks/commands/language_keys.py index 4caff6ac1..0c46a3151 100644 --- a/src/sampletones_tools/checks/commands/language_keys.py +++ b/src/sampletones_tools/checks/commands/language_keys.py @@ -20,13 +20,26 @@ class LanguageKeysArguments: def configure(parser: ArgumentParser) -> None: - parser.add_argument("--source", type=Path, default=None, help=SOURCE_HELP) - parser.add_argument("--language-file", type=Path, default=None, help=LANGUAGE_FILE_HELP) + parser.add_argument( + "--source", + type=Path, + default=None, + help=SOURCE_HELP, + ) + parser.add_argument( + "--language-file", + type=Path, + default=None, + help=LANGUAGE_FILE_HELP, + ) def run(arguments: Namespace) -> int: """Reports every disagreement between the language file and the lookups reading it.""" - given = LanguageKeysArguments(source=arguments.source, language_file=arguments.language_file) + given = LanguageKeysArguments( + source=arguments.source, + language_file=arguments.language_file, + ) from sampletones_application.paths import LANG_EN from sampletones_shared.paths.source import SOURCE_ROOT @@ -39,4 +52,9 @@ def run(arguments: Namespace) -> int: return report(findings) -LANGUAGE_KEYS: Final[Command] = Command(name=NAME, help=HELP, configure=configure, run=run) +LANGUAGE_KEYS: Final[Command] = Command( + name=NAME, + help=HELP, + configure=configure, + run=run, +) diff --git a/src/sampletones_tools/checks/commands/palette_colors.py b/src/sampletones_tools/checks/commands/palette_colors.py index fb12c1b35..588ddde2c 100644 --- a/src/sampletones_tools/checks/commands/palette_colors.py +++ b/src/sampletones_tools/checks/commands/palette_colors.py @@ -31,11 +31,19 @@ def configure(parser: ArgumentParser) -> None: def run(arguments: Namespace) -> int: """Reports every color the application stores resolved or the configuration writes out.""" - given = PaletteColorsArguments(package=arguments.package, config=arguments.config, palettes=arguments.palettes) + given = PaletteColorsArguments( + package=arguments.package, + config=arguments.config, + palettes=arguments.palettes, + ) from sampletones_application.paths import PALETTES_DIRECTORY from sampletones_shared.paths.resources import CONFIG_DIRECTORY - from sampletones_tools.checks.palette_colors import APPLICATION_PACKAGE, check_colors, report + from sampletones_tools.checks.palette_colors import ( + APPLICATION_PACKAGE, + check_colors, + report, + ) findings = check_colors( given.package if given.package is not None else APPLICATION_PACKAGE, @@ -45,4 +53,9 @@ def run(arguments: Namespace) -> int: return report(findings) -PALETTE_COLORS: Final[Command] = Command(name=NAME, help=HELP, configure=configure, run=run) +PALETTE_COLORS: Final[Command] = Command( + name=NAME, + help=HELP, + configure=configure, + run=run, +) diff --git a/src/sampletones_tools/checks/commands/rendered_literals.py b/src/sampletones_tools/checks/commands/rendered_literals.py index 8f403d994..67dd0a045 100644 --- a/src/sampletones_tools/checks/commands/rendered_literals.py +++ b/src/sampletones_tools/checks/commands/rendered_literals.py @@ -18,16 +18,32 @@ class RenderedLiteralsArguments: def configure(parser: ArgumentParser) -> None: - parser.add_argument("--tests", type=Path, action="append", dest="roots", default=[], help=TESTS_HELP) + parser.add_argument( + "--tests", + type=Path, + action="append", + dest="roots", + default=[], + help=TESTS_HELP, + ) def run(arguments: Namespace) -> int: """Reports every case comparing text it rendered with a literal it spelled out.""" given = RenderedLiteralsArguments(roots=tuple(arguments.roots)) - from sampletones_tools.checks.rendered_literals import TEST_ROOTS, check_cases, report + from sampletones_tools.checks.rendered_literals import ( + TEST_ROOTS, + check_cases, + report, + ) return report(check_cases(given.roots or TEST_ROOTS)) -RENDERED_LITERALS: Final[Command] = Command(name=NAME, help=HELP, configure=configure, run=run) +RENDERED_LITERALS: Final[Command] = Command( + name=NAME, + help=HELP, + configure=configure, + run=run, +) diff --git a/src/sampletones_tools/checks/commands/shortcut_actions.py b/src/sampletones_tools/checks/commands/shortcut_actions.py index 3c0b6bb7d..4d20fc24b 100644 --- a/src/sampletones_tools/checks/commands/shortcut_actions.py +++ b/src/sampletones_tools/checks/commands/shortcut_actions.py @@ -20,4 +20,9 @@ def run(arguments: Namespace) -> int: return report(check()) -SHORTCUT_ACTIONS: Final[Command] = Command(name=NAME, help=HELP, configure=configure, run=run) +SHORTCUT_ACTIONS: Final[Command] = Command( + name=NAME, + help=HELP, + configure=configure, + run=run, +) diff --git a/src/sampletones_tools/checks/commands/tag_names.py b/src/sampletones_tools/checks/commands/tag_names.py index f333cff4d..bc4640f41 100644 --- a/src/sampletones_tools/checks/commands/tag_names.py +++ b/src/sampletones_tools/checks/commands/tag_names.py @@ -26,11 +26,19 @@ def configure(parser: ArgumentParser) -> None: def run(arguments: Namespace) -> int: """Reports any tag constant whose name departs from the tag it composes.""" - given = TagNamesArguments(files=tuple(arguments.files), everything=arguments.all) + given = TagNamesArguments( + files=tuple(arguments.files), + everything=arguments.all, + ) from sampletones_tools.checks.tag_names import check_tags, report return report(check_tags(given.files, given.everything)) -TAG_NAMES: Final[Command] = Command(name=NAME, help=HELP, configure=configure, run=run) +TAG_NAMES: Final[Command] = Command( + name=NAME, + help=HELP, + configure=configure, + run=run, +) diff --git a/src/sampletones_tools/checks/commands/unused_tags.py b/src/sampletones_tools/checks/commands/unused_tags.py index f45eccc93..eaa545740 100644 --- a/src/sampletones_tools/checks/commands/unused_tags.py +++ b/src/sampletones_tools/checks/commands/unused_tags.py @@ -33,9 +33,17 @@ def configure(parser: ArgumentParser) -> None: def run(arguments: Namespace) -> int: """Reports every tag fragment the repository declares and never reads.""" - given = UnusedTagsArguments(tags=arguments.tags, reference_roots=tuple(arguments.reference_roots)) + given = UnusedTagsArguments( + tags=arguments.tags, + reference_roots=tuple(arguments.reference_roots), + ) - from sampletones_tools.checks.unused_tags import REFERENCE_ROOTS, TAGS_PACKAGE, check_reads, report + from sampletones_tools.checks.unused_tags import ( + REFERENCE_ROOTS, + TAGS_PACKAGE, + check_reads, + report, + ) unread = check_reads( given.tags if given.tags is not None else TAGS_PACKAGE, @@ -44,4 +52,9 @@ def run(arguments: Namespace) -> int: return report(unread) -UNUSED_TAGS: Final[Command] = Command(name=NAME, help=HELP, configure=configure, run=run) +UNUSED_TAGS: Final[Command] = Command( + name=NAME, + help=HELP, + configure=configure, + run=run, +) diff --git a/src/sampletones_tools/checks/language_keys.py b/src/sampletones_tools/checks/language_keys.py index c939e3abe..0653965bb 100755 --- a/src/sampletones_tools/checks/language_keys.py +++ b/src/sampletones_tools/checks/language_keys.py @@ -4,7 +4,17 @@ from enum import Enum, EnumMeta from pathlib import Path from types import ModuleType -from typing import Callable, Dict, Final, List, Mapping, NamedTuple, Optional, Sequence, Type +from typing import ( + Callable, + Dict, + Final, + List, + Mapping, + NamedTuple, + Optional, + Sequence, + Type, +) import yaml @@ -129,7 +139,10 @@ def entry_key(line: str) -> Optional[str]: def entry_lines(path: Path) -> Dict[str, int]: """Where each key sits in the language file, read line by line.""" lines: Dict[str, int] = {} - for number, line in enumerate(path.read_text(encoding=LANGUAGE_ENCODING).splitlines(), start=FIRST_LINE): + for number, line in enumerate( + path.read_text(encoding=LANGUAGE_ENCODING).splitlines(), + start=FIRST_LINE, + ): key = entry_key(line) if key is not None: lines.setdefault(key, number) @@ -157,7 +170,10 @@ def language_entries(path: Path) -> Dict[str, int]: return {str(key): lines.get(str(key), FIRST_LINE) for key in raw} -def broken_lookups(sites: Sequence[LookupSite], entries: Mapping[str, int]) -> List[Finding]: +def broken_lookups( + sites: Sequence[LookupSite], + entries: Mapping[str, int], +) -> List[Finding]: """Every literal key a lookup asks for that the language file holds no entry for. Args: diff --git a/src/sampletones_tools/checks/palette_colors.py b/src/sampletones_tools/checks/palette_colors.py index 276b4d0b3..1f3604e43 100755 --- a/src/sampletones_tools/checks/palette_colors.py +++ b/src/sampletones_tools/checks/palette_colors.py @@ -209,5 +209,8 @@ def report(findings: Sequence[ColorFinding]) -> int: for location, message in findings: print(f" {location}: {message}", file=sys.stderr) - print(f"\nFound {len(findings)} color(s) detached from the palette.", file=sys.stderr) + print( + f"\nFound {len(findings)} color(s) detached from the palette.", + file=sys.stderr, + ) return 1 diff --git a/src/sampletones_tools/checks/rendered_literals.py b/src/sampletones_tools/checks/rendered_literals.py index f7150ea6c..c198fa957 100755 --- a/src/sampletones_tools/checks/rendered_literals.py +++ b/src/sampletones_tools/checks/rendered_literals.py @@ -96,9 +96,15 @@ def report(found: Sequence[Finding]) -> int: if not found: return 0 - print("Case(s) holding rendered text against a spelled-out literal:", file=sys.stderr) + print( + "Case(s) holding rendered text against a spelled-out literal:", + file=sys.stderr, + ) for location, rendering in found: - print(f" {location}: {rendering} compared with a written-out string", file=sys.stderr) + print( + f" {location}: {rendering} compared with a written-out string", + file=sys.stderr, + ) print( f"\nFound {len(found)} such comparison(s). Compare the values rather than their text: " diff --git a/src/sampletones_tools/checks/shortcut_actions.py b/src/sampletones_tools/checks/shortcut_actions.py index ba380ffcd..92373030f 100755 --- a/src/sampletones_tools/checks/shortcut_actions.py +++ b/src/sampletones_tools/checks/shortcut_actions.py @@ -217,5 +217,8 @@ def report(findings: Sequence[Finding]) -> int: for kind, location, message in findings: print(f" {kind} | {location}: {message}", file=sys.stderr) - print(f"\nFound {len(findings)} incomplete action declaration(s).", file=sys.stderr) + print( + f"\nFound {len(findings)} incomplete action declaration(s).", + file=sys.stderr, + ) return 1 diff --git a/src/sampletones_tools/checks/source/lookups.py b/src/sampletones_tools/checks/source/lookups.py index 6bdfb5de0..43d8abc7b 100644 --- a/src/sampletones_tools/checks/source/lookups.py +++ b/src/sampletones_tools/checks/source/lookups.py @@ -7,7 +7,11 @@ from sampletones_tools.checks.source.index import SourceIndex from sampletones_tools.checks.source.modules import SourceModule from sampletones_tools.checks.source.subscripts import SubscriptSite, find_subscripts -from sampletones_tools.checks.source.values import EnumTable, ResolvedValues, ValueResolver +from sampletones_tools.checks.source.values import ( + EnumTable, + ResolvedValues, + ValueResolver, +) @dataclass(frozen=True) @@ -29,7 +33,10 @@ def resolved(self) -> bool: return not self.unresolved_parts -def composed_values(resolutions: Sequence[ResolvedValues], separator: str) -> Tuple[str, ...]: +def composed_values( + resolutions: Sequence[ResolvedValues], + separator: str, +) -> Tuple[str, ...]: """Every string a lookup's resolved parts spell together. A lookup of one part spells that part's values. A lookup of several parts spells one string per @@ -107,7 +114,18 @@ def module_lookups( enums=enums, constants=index.constants, ) - sites.extend(lookup_site(module, site, resolver, separator) for site in find_subscripts(scope.node, receivers)) + sites.extend( + lookup_site( + module, + site, + resolver, + separator, + ) + for site in find_subscripts( + scope.node, + receivers, + ) + ) return sites diff --git a/src/sampletones_tools/checks/tag_names.py b/src/sampletones_tools/checks/tag_names.py index 02754081f..2c34de76c 100755 --- a/src/sampletones_tools/checks/tag_names.py +++ b/src/sampletones_tools/checks/tag_names.py @@ -2,13 +2,27 @@ import sys from enum import StrEnum from pathlib import Path -from typing import Dict, Final, List, NamedTuple, Optional, Sequence, Tuple, Type, TypeVar +from typing import ( + Dict, + Final, + List, + NamedTuple, + Optional, + Sequence, + Tuple, + Type, + TypeVar, +) from sampletones_application.categories.hierarchy import Page, Panel, Widget from sampletones_application.categories.key.tag import TagName from sampletones_application.tags.compose import TAG_SEPARATOR from sampletones_tools.checks.source.constants import ModuleConstant, module_constants -from sampletones_tools.checks.source.modules import SourceModule, discover_modules, parse_module +from sampletones_tools.checks.source.modules import ( + SourceModule, + discover_modules, + parse_module, +) from sampletones_tools.checks.source.nodes import terminal_name from sampletones_tools.checks.source.packages import package_directory @@ -82,7 +96,10 @@ def call_arguments(call: ast.Call) -> Optional[Dict[str, ast.expr]]: return arguments -def hierarchy_member(node: ast.expr, enum: Type[EnumMember]) -> Optional[EnumMember]: +def hierarchy_member( + node: ast.expr, + enum: Type[EnumMember], +) -> Optional[EnumMember]: """The hierarchy member an argument names, such as `Page.GLOBAL`. Args: @@ -128,7 +145,10 @@ def composed_tag(arguments: Dict[str, ast.expr]) -> Optional[TagName]: return TagName(page, panel, widget, element) -def check_constant(module: SourceModule, constant: ModuleConstant) -> Optional[TagFinding]: +def check_constant( + module: SourceModule, + constant: ModuleConstant, +) -> Optional[TagFinding]: """Checks one constant of a tags module, where it is bound to a tag. Args: diff --git a/src/sampletones_tools/codec/report/encoding.py b/src/sampletones_tools/codec/report/encoding.py index 16abe11c8..414c27ba7 100644 --- a/src/sampletones_tools/codec/report/encoding.py +++ b/src/sampletones_tools/codec/report/encoding.py @@ -25,14 +25,51 @@ REGISTER_PLANES: Final[str] = "register planes" SPLIT_CONTROL: Final[str] = "split control" CONTROL_LEVEL_MASK: Final[int] = 0x3F -HOLDS_OPTIONS: Final[CodecOptions] = CodecOptions(holds=True, phrases=False, transposition=False, search=False) +HOLDS_OPTIONS: Final[CodecOptions] = CodecOptions( + holds=True, + phrases=False, + transposition=False, + search=False, +) PLANE_VARIANTS: Final[Tuple[Tuple[str, CodecOptions], ...]] = ( - (LITERALS, CodecOptions(holds=False, phrases=False, transposition=False, search=False)), + ( + LITERALS, + CodecOptions( + holds=False, + phrases=False, + transposition=False, + search=False, + ), + ), (HOLDS, HOLDS_OPTIONS), - (INSTRUMENTS, CodecOptions(holds=True, phrases=True, transposition=False, search=False)), - (TRANSPOSITION, CodecOptions(holds=True, phrases=True, transposition=True, search=False)), - (SEARCH, CodecOptions(holds=True, phrases=True, transposition=True, search=True)), + ( + INSTRUMENTS, + CodecOptions( + holds=True, + phrases=True, + transposition=False, + search=False, + ), + ), + ( + TRANSPOSITION, + CodecOptions( + holds=True, + phrases=True, + transposition=True, + search=False, + ), + ), + ( + SEARCH, + CodecOptions( + holds=True, + phrases=True, + transposition=True, + search=True, + ), + ), ) @@ -151,7 +188,13 @@ def _baseline_rows(entry: CorpusEntry, space: int) -> Tuple[ReportRow, ...]: records=entry.records, space=space, ), - _measured_row(entry, REGISTER_PLANES, _register_planes(entry.song.streams), 0, space), + _measured_row( + entry, + REGISTER_PLANES, + _register_planes(entry.song.streams), + 0, + space, + ), _measured_row( entry, SPLIT_CONTROL, diff --git a/src/sampletones_tools/codec/study/manifest.py b/src/sampletones_tools/codec/study/manifest.py index 03bb76017..c0b4b9924 100644 --- a/src/sampletones_tools/codec/study/manifest.py +++ b/src/sampletones_tools/codec/study/manifest.py @@ -7,6 +7,7 @@ from sampletones_shared.paths.user import PROJECTS_DIRECTORY, RECONSTRUCTIONS_DIRECTORY +# TODO: to remove completely PROJECT_SUFFIX: Final[str] = ".stp" DEFAULT_PROJECT_NAMES: Final[Tuple[str, ...]] = ("Amen", "Demo", "Tempo", "Test") QUICK_PROJECT_NAMES: Final[Tuple[str, ...]] = ("Test",) diff --git a/src/sampletones_tools/codec/study/report/verdicts.py b/src/sampletones_tools/codec/study/report/verdicts.py index 761a7643c..8dbcbe677 100644 --- a/src/sampletones_tools/codec/study/report/verdicts.py +++ b/src/sampletones_tools/codec/study/report/verdicts.py @@ -149,7 +149,13 @@ def verdict_rows( if variant.kind is VariantKind.BASELINE or variant.name not in by_variant: continue - rows.append(_verdict_row(variant, by_variant[variant.name], priced=variant.name in priced)) + rows.append( + _verdict_row( + variant, + by_variant[variant.name], + priced=variant.name in priced, + ) + ) return tuple(rows) diff --git a/src/sampletones_tools/codec/study/sandbox/parse.py b/src/sampletones_tools/codec/study/sandbox/parse.py index 82d3b6511..b6b5c416d 100644 --- a/src/sampletones_tools/codec/study/sandbox/parse.py +++ b/src/sampletones_tools/codec/study/sandbox/parse.py @@ -5,7 +5,10 @@ from sampletones_player.specification.compression import MAX_LITERAL_BYTES from sampletones_tools.codec.study.sandbox.context import PlaneContext from sampletones_tools.codec.study.sandbox.edges.generator import EdgeGenerator -from sampletones_tools.codec.study.sandbox.edges.holds import hold_edges, wide_hold_edges +from sampletones_tools.codec.study.sandbox.edges.holds import ( + hold_edges, + wide_hold_edges, +) from sampletones_tools.codec.study.sandbox.edges.literals import relax_literal from sampletones_tools.codec.study.sandbox.edges.phrases import phrase_edges from sampletones_tools.codec.study.sandbox.edges.set_hold import set_hold_edges @@ -95,7 +98,14 @@ def parse_plane( offered = generators(grammar) shortest: Shortest[StudyToken] = Shortest.across(ticks) window = LiteralWindow(shortest.costs) - _relax_forward(offered, context, shortest, 0, following[0], holdable=grammar.start_hold) + _relax_forward( + offered, + context, + shortest, + 0, + following[0], + holdable=grammar.start_hold, + ) for position in range(1, ticks + 1): relax_literal( context, @@ -114,4 +124,7 @@ def parse_plane( holdable=position not in entries.entries, ) - return StudyParse(tokens=shortest.walk(ticks), costs=tuple(shortest.costs)) + return StudyParse( + tokens=shortest.walk(ticks), + costs=tuple(shortest.costs), + ) diff --git a/src/sampletones_tools/codec/study/variants/production.py b/src/sampletones_tools/codec/study/variants/production.py index 3be6af54a..e483c40e9 100644 --- a/src/sampletones_tools/codec/study/variants/production.py +++ b/src/sampletones_tools/codec/study/variants/production.py @@ -95,7 +95,11 @@ def seed_encoder(transform: SeedTransform) -> Encoder: """ def encode(song: StudySong) -> Encoding: - return encode_production(song, seeds=transform(song.seeds), budget=DEFAULT_SEARCH_BUDGET) + return encode_production( + song, + seeds=transform(song.seeds), + budget=DEFAULT_SEARCH_BUDGET, + ) return encode diff --git a/src/sampletones_tools/codec/study/variants/sandbox.py b/src/sampletones_tools/codec/study/variants/sandbox.py index 41dd122c3..bb63a6769 100644 --- a/src/sampletones_tools/codec/study/variants/sandbox.py +++ b/src/sampletones_tools/codec/study/variants/sandbox.py @@ -41,7 +41,10 @@ operands=DEFAULT_COUNT_OPERANDS, default_entry=DEFAULT_COUNT_SIZE, ) -SET_HOLD_BOUND_COSTS: Final[Costs] = replace(PRODUCTION_COSTS, set_hold=SET_HOLD_BOUND) +SET_HOLD_BOUND_COSTS: Final[Costs] = replace( + PRODUCTION_COSTS, + set_hold=SET_HOLD_BOUND, +) class GrammarVariant(NamedTuple): diff --git a/src/sampletones_tools/corpus/build.py b/src/sampletones_tools/corpus/build.py index 29485ea9f..fc1c8f59f 100644 --- a/src/sampletones_tools/corpus/build.py +++ b/src/sampletones_tools/corpus/build.py @@ -35,7 +35,11 @@ def build_project( speed=module_config.speed, nes_frequency=module_config.nes_frequency, ) - project = Project.create(title=module_config.title, author=module_config.author, settings=settings) + project = Project.create( + title=module_config.title, + author=module_config.author, + settings=settings, + ) for sample in catalog.values(): project.voices.append(sample) @@ -53,4 +57,7 @@ def build_corpus(tmp_dir: Pathlike) -> Corpus: Corpus: The samples and the arrangement. """ catalog = build_catalog(CatalogSpec.load(), SynthConfig.load(), tmp_dir=tmp_dir) - return Corpus(catalog=catalog, project=build_project(catalog, ModuleConfig.load(), SongSpec.load())) + return Corpus( + catalog=catalog, + project=build_project(catalog, ModuleConfig.load(), SongSpec.load()), + ) diff --git a/src/sampletones_tools/corpus/song.py b/src/sampletones_tools/corpus/song.py index fe899938e..5ae846682 100644 --- a/src/sampletones_tools/corpus/song.py +++ b/src/sampletones_tools/corpus/song.py @@ -57,7 +57,9 @@ def load(cls) -> Self: return load_yaml_model(SONG_PATH, cls) -def _order(frames: Sequence[Mapping[ChannelName, int]]) -> List[Dict[ChannelName, Optional[int]]]: +def _order( + frames: Sequence[Mapping[ChannelName, int]], +) -> List[Dict[ChannelName, Optional[int]]]: return [{channel: frame.get(channel) for channel in ChannelName.items()} for frame in frames] @@ -72,7 +74,11 @@ def _row(spec: RowSpec, channel: ChannelName, samples_by_name: Mapping[str, Samp if channel not in sample.reconstruction.playing_channels: raise ValueError(f"Sample '{spec.sample}' has no '{channel.value}' slice for the {channel.value} channel") - return Row(command=NoteOn(voice_id=sample.id), transpose=spec.transpose, volume=spec.volume) + return Row( + command=NoteOn(voice_id=sample.id), + transpose=spec.transpose, + volume=spec.volume, + ) def _pattern( @@ -96,7 +102,12 @@ def _channels( channels: Dict[ChannelName, Channel] = {} for channel, spec in channel_specs.items(): patterns = { - index: _pattern(row_specs, rows_per_pattern, channel, samples_by_name) + index: _pattern( + row_specs, + rows_per_pattern, + channel, + samples_by_name, + ) for index, row_specs in spec.patterns.items() } channels[channel] = Channel(name=channel, patterns=patterns) diff --git a/src/sampletones_tools/player/command.py b/src/sampletones_tools/player/command.py index daac75206..bd4d65807 100644 --- a/src/sampletones_tools/player/command.py +++ b/src/sampletones_tools/player/command.py @@ -52,4 +52,9 @@ def run(arguments: Namespace) -> int: return 0 -DRIVER: Final[Command] = Command(name=NAME, help=HELP, configure=configure, run=run) +DRIVER: Final[Command] = Command( + name=NAME, + help=HELP, + configure=configure, + run=run, +) diff --git a/src/sampletones_tools/registry.py b/src/sampletones_tools/registry.py index 938fa119f..27995f58a 100644 --- a/src/sampletones_tools/registry.py +++ b/src/sampletones_tools/registry.py @@ -10,4 +10,13 @@ from sampletones_tools.samples.commands.ftm import FTM from sampletones_tools.samples.commands.nsf import NSF -DEVELOPER_COMMANDS: Final[Tuple[Command, ...]] = (BTP, CALIBRATION, CHECK, CODEC, DRIVER, FTM, ICONS, NSF) +DEVELOPER_COMMANDS: Final[Tuple[Command, ...]] = ( + BTP, + CALIBRATION, + CHECK, + CODEC, + DRIVER, + FTM, + ICONS, + NSF, +) diff --git a/src/sampletones_tools/samples/commands/btp.py b/src/sampletones_tools/samples/commands/btp.py index 46605000a..fb7dfb124 100644 --- a/src/sampletones_tools/samples/commands/btp.py +++ b/src/sampletones_tools/samples/commands/btp.py @@ -17,8 +17,14 @@ def configure(parser: ArgumentParser) -> None: - actions = parser.add_subparsers(dest=ACTION_FIELD, metavar=ACTION_METAVAR, required=True) - add_output_option(actions.add_parser(SAMPLES, help=SAMPLES_HELP, description=SAMPLES_HELP)) + actions = parser.add_subparsers( + dest=ACTION_FIELD, + metavar=ACTION_METAVAR, + required=True, + ) + add_output_option( + actions.add_parser(SAMPLES, help=SAMPLES_HELP, description=SAMPLES_HELP), + ) def run(arguments: Namespace) -> int: @@ -31,4 +37,9 @@ def run(arguments: Namespace) -> int: return print_written(emit_samples(given.output, write_samples)) -BTP: Final[Command] = Command(name=NAME, help=HELP, configure=configure, run=run) +BTP: Final[Command] = Command( + name=NAME, + help=HELP, + configure=configure, + run=run, +) diff --git a/src/sampletones_tools/samples/commands/ftm.py b/src/sampletones_tools/samples/commands/ftm.py index 2355efa7a..aacbc783e 100644 --- a/src/sampletones_tools/samples/commands/ftm.py +++ b/src/sampletones_tools/samples/commands/ftm.py @@ -17,8 +17,14 @@ def configure(parser: ArgumentParser) -> None: - actions = parser.add_subparsers(dest=ACTION_FIELD, metavar=ACTION_METAVAR, required=True) - add_output_option(actions.add_parser(SAMPLES, help=SAMPLES_HELP, description=SAMPLES_HELP)) + actions = parser.add_subparsers( + dest=ACTION_FIELD, + metavar=ACTION_METAVAR, + required=True, + ) + add_output_option( + actions.add_parser(SAMPLES, help=SAMPLES_HELP, description=SAMPLES_HELP), + ) def run(arguments: Namespace) -> int: @@ -31,4 +37,9 @@ def run(arguments: Namespace) -> int: return print_written(emit_samples(given.output, write_samples)) -FTM: Final[Command] = Command(name=NAME, help=HELP, configure=configure, run=run) +FTM: Final[Command] = Command( + name=NAME, + help=HELP, + configure=configure, + run=run, +) diff --git a/src/sampletones_tools/samples/commands/nsf.py b/src/sampletones_tools/samples/commands/nsf.py index 014b96d85..7671686d9 100644 --- a/src/sampletones_tools/samples/commands/nsf.py +++ b/src/sampletones_tools/samples/commands/nsf.py @@ -32,12 +32,34 @@ class RenderArguments: def configure(parser: ArgumentParser) -> None: - actions = parser.add_subparsers(dest=ACTION_FIELD, metavar=ACTION_METAVAR, required=True) - samples = actions.add_parser(SAMPLES, help=SAMPLES_HELP, description=SAMPLES_HELP) + actions = parser.add_subparsers( + dest=ACTION_FIELD, + metavar=ACTION_METAVAR, + required=True, + ) + samples = actions.add_parser( + SAMPLES, + help=SAMPLES_HELP, + description=SAMPLES_HELP, + ) add_output_option(samples) - render = actions.add_parser(RENDER, help=RENDER_HELP, description=RENDER_HELP) - render.add_argument("--directory", type=Path, required=True, help=DIRECTORY_HELP) - render.add_argument("--tail", type=float, default=DEFAULT_TAIL_SECONDS, help=TAIL_HELP) + render = actions.add_parser( + RENDER, + help=RENDER_HELP, + description=RENDER_HELP, + ) + render.add_argument( + "--directory", + type=Path, + required=True, + help=DIRECTORY_HELP, + ) + render.add_argument( + "--tail", + type=float, + default=DEFAULT_TAIL_SECONDS, + help=TAIL_HELP, + ) def run(arguments: Namespace) -> int: @@ -45,7 +67,9 @@ def run(arguments: Namespace) -> int: if arguments.action == SAMPLES: return _write_samples(SamplesArguments(output=arguments.output)) - return _render(RenderArguments(directory=arguments.directory, tail=arguments.tail)) + return _render( + RenderArguments(directory=arguments.directory, tail=arguments.tail), + ) def _write_samples(given: SamplesArguments) -> int: @@ -74,4 +98,9 @@ def _render(given: RenderArguments) -> int: return 0 -NSF: Final[Command] = Command(name=NAME, help=HELP, configure=configure, run=run) +NSF: Final[Command] = Command( + name=NAME, + help=HELP, + configure=configure, + run=run, +) diff --git a/src/sampletones_tools/samples/nsf.py b/src/sampletones_tools/samples/nsf.py index 1831fad73..b7016e966 100644 --- a/src/sampletones_tools/samples/nsf.py +++ b/src/sampletones_tools/samples/nsf.py @@ -42,6 +42,9 @@ def write_samples(corpus: Corpus, output: Path) -> List[Path]: written.append(destination) arrangement = output / f"{SONG_NAME}{EXT_FILE_NSF}" - NSFBackend().write_project(arrangement, ProjectExport(project=corpus.project)) + NSFBackend().write_project( + arrangement, + ProjectExport(project=corpus.project), + ) written.append(arrangement) return written diff --git a/src/sampletones_tools/samples/render.py b/src/sampletones_tools/samples/render.py index ae3477856..8da4f30da 100755 --- a/src/sampletones_tools/samples/render.py +++ b/src/sampletones_tools/samples/render.py @@ -83,7 +83,9 @@ def require_renderer() -> None: libgme demuxer. """ if locate_program(FFMPEG) is None: - raise RenderingError(missing_program_message(FFMPEG, RENDER_PURPOSE, INSTALL_HINTS)) + raise RenderingError( + missing_program_message(FFMPEG, RENDER_PURPOSE, INSTALL_HINTS), + ) if not decodes_exports(): raise RenderingError( @@ -170,6 +172,12 @@ def render_directory(directory: Path, tail_seconds: float) -> List[RenderedWave] except subprocess.CalledProcessError as error: raise RenderingError(f"{FFMPEG} rejected {source}: exit status {error.returncode}") from error - rendered.append(RenderedWave(source=source, destination=destination, seconds=seconds)) + rendered.append( + RenderedWave( + source=source, + destination=destination, + seconds=seconds, + ) + ) return rendered From e0e4f4875f0fe0ca4cc7190b3c7556264cf0ee98 Mon Sep 17 00:00:00 2001 From: JakimPL Date: Sun, 13 Sep 2026 18:59:36 +0200 Subject: [PATCH 19/36] Removed: the codec study's corpus of local files --- docs/development/player.md | 8 +- docs/development/tooling.md | 7 +- src/sampletones_tools/codec/command.py | 55 +++++++----- src/sampletones_tools/codec/study/manifest.py | 85 +++++-------------- src/sampletones_tools/codec/study/session.py | 41 ++++----- .../codec/study/test_session.py | 59 +++++++++++-- .../sampletones_tools/codec/test_command.py | 18 +++- 7 files changed, 151 insertions(+), 122 deletions(-) diff --git a/docs/development/player.md b/docs/development/player.md index 910cd5e1d..65ff6b590 100644 --- a/docs/development/player.md +++ b/docs/development/player.md @@ -42,9 +42,9 @@ off on its own, and `uv run sampletones codec report` writes what each one saves songs. The format's constants are settled from that report rather than from argument. **A change to the codec is measured before it is built.** `uv run sampletones codec study` reads the -projects and stems on this machine, encodes every song under every candidate change, and -writes the sizes, the times and a verdict per candidate under -`Documents/SampleToNES/compression`. A candidate is one of two things. A new way of choosing +projects and stems named on its command line, encodes every song under every candidate change, +and writes the sizes, the times, a verdict per candidate and the manifest that repeats the run +under `Documents/SampleToNES/compression`. A candidate is one of two things. A new way of choosing tokens is encoded and played back by the production codec itself. A new token grammar is priced in bytes by a study parser, which first has to reproduce the production parser's bytes on today's grammar. The rule is printed in the report: a candidate earns a production @@ -152,7 +152,7 @@ The chain runs from the register values upward, and each link is held on its own | The codec is lossless | every encoding decodes to the planes it was written from, over a corpus | | The codec is safe | a plane the codec finds nothing in stays within its literal bound | | The ratio | `uv run sampletones codec report` — bytes per tick and ticks that fit, per layer | -| What a change would save | `uv run sampletones codec study` — the songs on this machine under every candidate change, with a verdict each | +| What a change would save | `uv run sampletones codec study` — the projects and stems it is given, under every candidate change, with a verdict each | | The byte layout | a hand-built song serializes to expected bytes | | The assembly agrees with the specification | the include's equates are read and compared field by field | | The driver behaves | the assembled image on a 6502 emulator against `RegisterTrace.from_song`, over several rates and over songs that repeat | diff --git a/docs/development/tooling.md b/docs/development/tooling.md index e301218c0..0f27ee96a 100644 --- a/docs/development/tooling.md +++ b/docs/development/tooling.md @@ -86,7 +86,7 @@ The developer commands, listed by `sampletones_tools/registry.py` and run as | `check [options]` | Holds the tree to one of its checks: `import-boundary`, `language-keys`, `palette-colors`, `rendered-literals`, `shortcut-actions`, `tag-names`, `unused-tags`; each is a pre-commit hook, and [architecture](architecture.md#enforcement) says what each holds. Needs a checkout | | `btp samples -o DIR`, `ftm samples -o DIR`, `nsf samples -o DIR` | Builds the synthetic corpus and writes it as example files: the arrangement as two Bitphase documents (at its tempo and as a groove), the arrangement as a FamiTracker module, or each sample and the arrangement as `.nsf` programs | | `codec report -o DIR` | Compresses the synthetic corpus under every layer of the codec and writes the report the format's constants are settled from, as CSV and Markdown | -| `codec study [--manifest FILE] [--project FILE]... [--reconstruction PATH]... [-o DIR] [--lengthen SECONDS] [--variants LIST] [--quick]` | Encodes the projects and stems on this machine under every candidate change to the codec and writes the sizes, the times and a verdict per candidate; without `-o` the run lands under Documents/SampleToNES/compression | +| `codec study [--manifest FILE] [--project FILE]... [--reconstruction PATH]... [-o DIR] [--lengthen SECONDS] [--variants LIST]` | Encodes the projects and stems it is given, or the ones a manifest names, under every candidate change to the codec and writes the sizes, the times, a verdict per candidate and the manifest that repeats the run; without `-o` the run lands under Documents/SampleToNES/compression | | `driver [--directory DIR]` | Assembles the NES player driver with cc65 and prints the layout the build produced; without `--directory` it writes the driver the package ships, which needs a checkout | | `icons [--directory DIR]` | Writes the icon suite from the mark; without `--directory` it writes the icons the package ships, which needs a checkout | | `nsf render --directory DIR [--tail SECONDS]` | Renders every exported `.nsf` file in the directory to a wave beside it, through ffmpeg's libgme demuxer | @@ -97,7 +97,7 @@ The developer commands, listed by `sampletones_tools/registry.py` and run as subject, and `sampletones_tools/registry.py` lists the developer commands they offer. `sampletones/commands/registry.py` appends them to the user commands, which is the one import of the tools package; [package layers](packages.md) holds the edge. The wheel and the bundle carry the -package, so every command exists in every copy of the program, and three rules decide what a +package, so every command exists in every copy of the program, and four rules decide what a developer command does there: - **The checkout guard.** `sampletones_tools/checkout.py` holds `require_checkout(command)`: the @@ -109,6 +109,9 @@ developer command does there: package. - **Package data is read from the package**, through `importlib.resources`, never through a path under the repository, so it ships in the wheel and the bundle. +- **A tool reads the files it is given.** What a tool measures or converts arrives on its command + line or in a file a run wrote; the code names no file on one machine, so every run starts from + what the person running it has. Developer commands are run as `uv run sampletones ` from a checkout; the `sampletones` command `make setup` installs is a wheel and refuses the guarded ones the same way. A command module diff --git a/src/sampletones_tools/codec/command.py b/src/sampletones_tools/codec/command.py index 3677fa4a2..fd296af06 100644 --- a/src/sampletones_tools/codec/command.py +++ b/src/sampletones_tools/codec/command.py @@ -13,12 +13,16 @@ REPORT_HELP: Final[str] = "compress the synthetic corpus under every layer of the codec and write the report tables" REPORT_OUTPUT_HELP: Final[str] = "the directory the report is written into, created when missing" STUDY: Final[str] = "study" -STUDY_HELP: Final[str] = ( - "encode the projects and stems on this machine under every candidate change, with a verdict each" +STUDY_HELP: Final[str] = "encode the projects and stems named under every candidate change, with a verdict each" +MANIFEST_HELP: Final[str] = ( + "a manifest a run wrote, measured again; its lengthening and variants stand in for the options below" +) +PROJECT_HELP: Final[str] = ( + "a project file to measure, repeatable; a run names at least one project, reconstruction or manifest" +) +RECONSTRUCTION_HELP: Final[str] = ( + "a stem file, or a directory of stems, to measure, repeatable; named sources replace a manifest's own" ) -MANIFEST_HELP: Final[str] = "a manifest a run wrote; its lengthening and variants stand in for the options below" -PROJECT_HELP: Final[str] = "a project file to measure in place of the corpus, repeatable" -RECONSTRUCTION_HELP: Final[str] = "a stem file, or a directory of stems, to measure in place of the corpus, repeatable" OUTPUT_HELP: Final[str] = ( "the directory the run writes into; without it, a timestamped directory under Documents/SampleToNES/compression" ) @@ -26,7 +30,6 @@ VARIANTS_HELP: Final[str] = ( "variants every song is encoded under, comma separated; without it every one, and the baseline always runs" ) -QUICK_HELP: Final[str] = "read one small project and one stem, to check the harness" DEFAULT_LENGTHEN_SECONDS: Final[int] = 180 @@ -47,7 +50,6 @@ class StudyArguments: output: Optional[Path] lengthen: int variants: Optional[str] - quick: bool def configure(parser: ArgumentParser) -> None: @@ -68,7 +70,6 @@ def configure(parser: ArgumentParser) -> None: study.add_argument("--output", "-o", type=Path, default=None, help=OUTPUT_HELP) study.add_argument("--lengthen", type=int, default=DEFAULT_LENGTHEN_SECONDS, help=LENGTHEN_HELP) study.add_argument("--variants", type=str, default=None, help=VARIANTS_HELP) - study.add_argument("--quick", action="store_true", help=QUICK_HELP) def run(arguments: Namespace) -> int: @@ -84,7 +85,6 @@ def run(arguments: Namespace) -> int: output=arguments.output, lengthen=arguments.lengthen, variants=arguments.variants, - quick=arguments.quick, ) ) @@ -99,18 +99,35 @@ def _report(given: ReportArguments) -> int: def _study(given: StudyArguments) -> int: - from sampletones_tools.codec.study.session import resolve_manifest, run_study, variant_names - - manifest = resolve_manifest( - given.manifest, - projects=given.projects, - reconstructions=given.reconstructions, - lengthen_seconds=given.lengthen, - variants=variant_names(given.variants), - quick=given.quick, + """Measures the sources a run names. + + Raises: + SystemExit: If the run names neither a source nor a manifest. + """ + from sampletones_tools.codec.study.session import ( + resolve_manifest, + run_study, + variant_names, ) + + try: + manifest = resolve_manifest( + given.manifest, + projects=given.projects, + reconstructions=given.reconstructions, + lengthen_seconds=given.lengthen, + variants=variant_names(given.variants), + ) + except ValueError as error: + raise SystemExit(str(error)) from error + run_study(manifest, given.output) return 0 -CODEC: Final[Command] = Command(name=NAME, help=HELP, configure=configure, run=run) +CODEC: Final[Command] = Command( + name=NAME, + help=HELP, + configure=configure, + run=run, +) diff --git a/src/sampletones_tools/codec/study/manifest.py b/src/sampletones_tools/codec/study/manifest.py index c0b4b9924..76db22ec8 100644 --- a/src/sampletones_tools/codec/study/manifest.py +++ b/src/sampletones_tools/codec/study/manifest.py @@ -1,30 +1,11 @@ -from __future__ import annotations - from pathlib import Path -from typing import Final, Tuple - -from pydantic import BaseModel, ConfigDict, Field - -from sampletones_shared.paths.user import PROJECTS_DIRECTORY, RECONSTRUCTIONS_DIRECTORY - -# TODO: to remove completely -PROJECT_SUFFIX: Final[str] = ".stp" -DEFAULT_PROJECT_NAMES: Final[Tuple[str, ...]] = ("Amen", "Demo", "Tempo", "Test") -QUICK_PROJECT_NAMES: Final[Tuple[str, ...]] = ("Test",) -LEAD_VOCALS: Final[str] = "lead-vocals" -DEFAULT_RECONSTRUCTIONS: Final[Tuple[Tuple[str, str], ...]] = ( - ( - LEAD_VOCALS, - "sr_44100_nf_60_sm_cqt_tg_100_gn_p_ch_7376fe40aec68c41696a37cbf89baa55/0 Lead Vocals.stn", - ), - ( - "payoff-cqt", - "sr_44100_nf_60_sm_cqt_tg_100_gn_PTN_ch_124286f3189248af752bf09c37323e5e/Payoff [Revenge A] Stems (123 BPM)", - ), - ( - "payoff-fft", - "sr_44100_nf_60_sm_fft_tg_0_gn_PTN_ch_6d3be4658f9804463a01f2b34e3f0d16/Payoff [Revenge A] Stems (123 BPM)", - ), +from typing import Final, Self, Tuple + +from pydantic import BaseModel, ConfigDict, Field, model_validator + +NO_SOURCE: Final[str] = ( + "The study reads the files it is given: name a project with --project, a stem file or a directory of " + "stems with --reconstruction, or a manifest a run wrote with --manifest." ) @@ -42,14 +23,14 @@ class StudySource(BaseModel): path: Path @classmethod - def at(cls, path: Path) -> StudySource: + def at(cls, path: Path) -> Self: """A source labeled by its own name: a file's stem, or a directory's name. Args: path: The file or directory. Returns: - StudySource: The source under that label. + Self: The source under that label. """ return cls(label=path.name if path.is_dir() else path.stem, path=path) @@ -71,15 +52,25 @@ class StudyManifest(BaseModel): lengthen_seconds: int = Field(ge=1) variants: Tuple[str, ...] = Field(min_length=1) + @model_validator(mode="after") + def _names_a_source(self) -> Self: + """Raises: + ValueError: If the manifest names neither a project nor a reconstruction. + """ + if not self.projects and not self.reconstructions: + raise ValueError(NO_SOURCE) + + return self + @classmethod - def load(cls, path: Path) -> StudyManifest: + def load(cls, path: Path) -> Self: """Reads a manifest a run wrote, or one written by hand. Args: path: The manifest file, as JSON. Returns: - StudyManifest: The manifest. + Self: The manifest. """ return cls.model_validate_json(path.read_text(encoding="utf-8")) @@ -90,37 +81,3 @@ def save(self, path: Path) -> None: path: Where the manifest is written, as JSON. """ path.write_text(self.model_dump_json(indent=2), encoding="utf-8") - - @classmethod - def default( - cls, - *, - lengthen_seconds: int, - variants: Tuple[str, ...], - quick: bool, - ) -> StudyManifest: - """The corpus on this machine: the projects and stems the study is settled on. - - Args: - lengthen_seconds: How long each project's lengthened copy lasts. - variants: The names of the variants every song is encoded under. - quick: Whether to read one small project and one stem, for a run that checks the - harness rather than the codec. - - Returns: - StudyManifest: The manifest. - """ - project_names = QUICK_PROJECT_NAMES if quick else DEFAULT_PROJECT_NAMES - reconstructions = tuple( - StudySource(label=label, path=RECONSTRUCTIONS_DIRECTORY / relative) - for label, relative in DEFAULT_RECONSTRUCTIONS - if label == LEAD_VOCALS or not quick - ) - return cls( - projects=tuple( - StudySource(label=name, path=PROJECTS_DIRECTORY / f"{name}{PROJECT_SUFFIX}") for name in project_names - ), - reconstructions=reconstructions, - lengthen_seconds=lengthen_seconds, - variants=variants, - ) diff --git a/src/sampletones_tools/codec/study/session.py b/src/sampletones_tools/codec/study/session.py index aae4a851e..cf7daa191 100644 --- a/src/sampletones_tools/codec/study/session.py +++ b/src/sampletones_tools/codec/study/session.py @@ -3,7 +3,7 @@ from sampletones_shared.logger import logger from sampletones_tools.codec.study.corpus.build import build_corpus -from sampletones_tools.codec.study.manifest import StudyManifest, StudySource +from sampletones_tools.codec.study.manifest import NO_SOURCE, StudyManifest, StudySource from sampletones_tools.codec.study.measure import Measurement, measure from sampletones_tools.codec.study.report.run import run_directory, write_run from sampletones_tools.codec.study.variants.baselines import Baselines @@ -28,38 +28,33 @@ def resolve_manifest( reconstructions: Tuple[Path, ...], lengthen_seconds: int, variants: Tuple[str, ...], - quick: bool, ) -> StudyManifest: - """The manifest a run measures: the one a file states, or the corpus on this machine. + """The manifest a run measures: the sources named outright, or the ones a manifest file states. - Projects or reconstructions named outright stand in for the corpus while the manifest's - lengthening and variants stay. + Sources named outright stand in for a manifest's own, while its lengthening and variants stay. Args: - path: A manifest a run wrote, or ``None`` for the corpus on this machine. - projects: Project files measured in place of the corpus. - reconstructions: Stem files, or directories of stems, measured in place of the corpus. - lengthen_seconds: How long each project's lengthened copy lasts. - variants: The names of the variants every song is encoded under. - quick: Whether to read one small project and one stem, to check the harness. + path: A manifest a run wrote, or ``None`` to measure the sources named outright. + projects: Project files to measure. + reconstructions: Stem files, or directories of stems, to measure. + lengthen_seconds: How long each project's lengthened copy lasts, unless a manifest states it. + variants: The names of the variants every song is encoded under, unless a manifest states them. + + Raises: + ValueError: If neither a manifest nor a source is named. """ - manifest = ( - StudyManifest.load(path) - if path is not None - else StudyManifest.default( - lengthen_seconds=lengthen_seconds, - variants=variants, - quick=quick, - ) - ) - if not projects and not reconstructions: + if path is None and not projects and not reconstructions: + raise ValueError(NO_SOURCE) + + manifest = StudyManifest.load(path) if path is not None else None + if manifest is not None and not projects and not reconstructions: return manifest return StudyManifest( projects=tuple(StudySource.at(project) for project in projects), reconstructions=tuple(StudySource.at(reconstruction) for reconstruction in reconstructions), - lengthen_seconds=manifest.lengthen_seconds, - variants=manifest.variants, + lengthen_seconds=manifest.lengthen_seconds if manifest is not None else lengthen_seconds, + variants=manifest.variants if manifest is not None else variants, ) diff --git a/tests/unit/sampletones_tools/codec/study/test_session.py b/tests/unit/sampletones_tools/codec/study/test_session.py index 2ec115a62..8a18f02b0 100644 --- a/tests/unit/sampletones_tools/codec/study/test_session.py +++ b/tests/unit/sampletones_tools/codec/study/test_session.py @@ -1,5 +1,7 @@ from pathlib import Path +import pytest + from sampletones_tools.codec.study.manifest import StudyManifest, StudySource from sampletones_tools.codec.study.session import resolve_manifest, variant_names from sampletones_tools.codec.study.variants.registry import EVERY_VARIANT @@ -14,14 +16,13 @@ def test_names_are_read_in_order_with_their_spaces_stripped(self) -> None: class TestResolveManifest: - def test_sources_named_outright_stand_in_for_the_corpus(self) -> None: + def test_sources_named_outright_are_measured(self) -> None: manifest = resolve_manifest( None, projects=(Path("songs/one.stp"),), reconstructions=(Path("stems/two"),), lengthen_seconds=30, variants=("wide-hold",), - quick=False, ) assert manifest.projects == (StudySource(label="one", path=Path("songs/one.stp")),) @@ -29,15 +30,59 @@ def test_sources_named_outright_stand_in_for_the_corpus(self) -> None: assert manifest.lengthen_seconds == 30 assert manifest.variants == ("wide-hold",) - def test_without_sources_the_corpus_on_this_machine_is_read(self) -> None: + def test_a_run_naming_no_source_and_no_manifest_is_refused(self) -> None: + with pytest.raises(ValueError, match="reads the files it is given"): + resolve_manifest( + None, + projects=(), + reconstructions=(), + lengthen_seconds=30, + variants=(EVERY_VARIANT,), + ) + + def test_a_manifest_file_is_measured_as_it_stands(self, tmp_path: Path) -> None: + written = StudyManifest( + projects=(StudySource(label="one", path=Path("songs/one.stp")),), + reconstructions=(), + lengthen_seconds=45, + variants=("wide-hold",), + ) + path = tmp_path / "manifest.json" + written.save(path) + manifest = resolve_manifest( - None, + path, projects=(), reconstructions=(), lengthen_seconds=30, variants=(EVERY_VARIANT,), - quick=True, ) - assert manifest == StudyManifest.default(lengthen_seconds=30, variants=(EVERY_VARIANT,), quick=True) - assert manifest.projects + assert manifest == written + + def test_sources_named_outright_replace_a_manifest_s_own_and_keep_its_sweep(self, tmp_path: Path) -> None: + path = tmp_path / "manifest.json" + StudyManifest( + projects=(StudySource(label="one", path=Path("songs/one.stp")),), + reconstructions=(), + lengthen_seconds=45, + variants=("wide-hold",), + ).save(path) + + manifest = resolve_manifest( + path, + projects=(), + reconstructions=(Path("stems/two"),), + lengthen_seconds=30, + variants=(EVERY_VARIANT,), + ) + + assert manifest.projects == () + assert manifest.reconstructions == (StudySource(label="two", path=Path("stems/two")),) + assert (manifest.lengthen_seconds, manifest.variants) == (45, ("wide-hold",)) + + +class TestStudyManifest: + def test_a_manifest_naming_no_source_is_refused(self) -> None: + with pytest.raises(ValueError, match="reads the files it is given"): + StudyManifest(projects=(), reconstructions=(), lengthen_seconds=45, variants=("wide-hold",)) diff --git a/tests/unit/sampletones_tools/codec/test_command.py b/tests/unit/sampletones_tools/codec/test_command.py index 3ae0a13b1..9e8cc7f87 100644 --- a/tests/unit/sampletones_tools/codec/test_command.py +++ b/tests/unit/sampletones_tools/codec/test_command.py @@ -5,7 +5,9 @@ from sampletones.commands.registry import COMMANDS from sampletones.dispatcher import dispatch +from sampletones_tools.codec.command import DEFAULT_LENGTHEN_SECONDS from sampletones_tools.codec.study.manifest import StudyManifest +from sampletones_tools.codec.study.variants.registry import EVERY_VARIANT RUNNER: Final[str] = "sampletones_tools.codec.study.session.run_study" REPORTER: Final[str] = "sampletones_tools.codec.report.session.run_report" @@ -84,13 +86,23 @@ def test_the_sources_and_the_sweep_are_read_from_the_options( assert manifest.lengthen_seconds == 30 assert output == tmp_path - def test_a_quick_run_reads_the_small_corpus_into_the_documents(self, monkeypatch: pytest.MonkeyPatch) -> None: + def test_a_run_naming_no_source_is_refused_with_the_way_to_name_one(self, monkeypatch: pytest.MonkeyPatch) -> None: study = RecordedStudy() monkeypatch.setattr(RUNNER, study) - assert dispatch(COMMANDS, ["codec", "study", "--quick"]) == 0 + with pytest.raises(SystemExit, match="reads the files it is given"): + dispatch(COMMANDS, ["codec", "study"]) + + assert study.runs == [] + + def test_the_lengthening_defaults_to_its_constant(self, monkeypatch: pytest.MonkeyPatch) -> None: + study = RecordedStudy() + monkeypatch.setattr(RUNNER, study) + + assert dispatch(COMMANDS, ["codec", "study", "--project", "songs/one.stp"]) == 0 manifest, output = study.runs[0] - assert manifest == StudyManifest.default(lengthen_seconds=180, variants=("all",), quick=True) + assert manifest.lengthen_seconds == DEFAULT_LENGTHEN_SECONDS + assert manifest.variants == (EVERY_VARIANT,) assert output is None def test_an_action_is_required(self) -> None: From 054e5d1b0eb7588f8288b155f133b632a27cebaf Mon Sep 17 00:00:00 2001 From: JakimPL Date: Sun, 13 Sep 2026 19:19:04 +0200 Subject: [PATCH 20/36] Fixed: the CI and build breakages --- .github/workflows/workflow.yml | 6 +++- Makefile | 16 ++++------- install.bat | 3 ++ scripts/bootstrap/platforms/macos.py | 4 +++ scripts/bootstrap/preflight.py | 7 +---- scripts/bootstrap/venv_build.py | 26 ++++++++--------- scripts/bundle.py | 10 +++++-- scripts/setup_environment.py | 11 ++++++-- .../unit/scripts/bootstrap/test_platforms.py | 19 +++++++++++-- .../unit/scripts/bootstrap/test_preflight.py | 15 +--------- .../unit/scripts/bootstrap/test_processes.py | 2 +- .../unit/scripts/bootstrap/test_venv_build.py | 28 +++++++++++-------- tests/unit/scripts/test_setup_environment.py | 21 ++++++++++++++ .../unit/scripts/test_system_dependencies.py | 11 ++++++++ 14 files changed, 111 insertions(+), 68 deletions(-) diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml index 4271cc6a1..15bd9b2f4 100644 --- a/.github/workflows/workflow.yml +++ b/.github/workflows/workflow.yml @@ -95,7 +95,11 @@ jobs: - name: Check a developer command refuses to run outside a checkout shell: bash - run: sampletones check import-boundary --all 2>&1 | grep "runs from a checkout" + run: | + if refusal=$(sampletones check import-boundary --all 2>&1); then + exit 1 + fi + grep "runs from a checkout" <<< "$refusal" bundle: name: Standalone bundle (${{ matrix.platform }}) diff --git a/Makefile b/Makefile index 454e2dd24..6062d7afe 100644 --- a/Makefile +++ b/Makefile @@ -1,21 +1,15 @@ .PHONY: help setup install system-deps build release run clean pre-commit test benchmarks lint format ifeq ($(OS),Windows_NT) -ifeq ($(MSYSTEM),) -UNAME_S := Windows +PYTHON := python else -UNAME_S := $(shell uname -s) -endif -else -UNAME_S := $(shell uname -s) +PYTHON := python3 endif -ifeq ($(UNAME_S),Windows) - PYTHON := python - Q := +ifeq ($(OS)$(MSYSTEM),Windows_NT) +Q := else - PYTHON := python3 - Q := " +Q := " endif GPU ?= auto diff --git a/install.bat b/install.bat index 599f7f56c..4d4861042 100644 --- a/install.bat +++ b/install.bat @@ -1,4 +1,7 @@ @echo off +setlocal python "%~dp0scripts\bundle.py" %* +set STATUS=%ERRORLEVEL% pause +exit /b %STATUS% diff --git a/scripts/bootstrap/platforms/macos.py b/scripts/bootstrap/platforms/macos.py index 582a1155c..4fdfa2cee 100644 --- a/scripts/bootstrap/platforms/macos.py +++ b/scripts/bootstrap/platforms/macos.py @@ -1,3 +1,4 @@ +import shutil from pathlib import Path from typing import Final, Optional, Sequence @@ -43,6 +44,9 @@ def launcher(self, distribution: Path, *, release: bool) -> Path: return posix_launcher(distribution, release=release) def missing_package_manager(self) -> Optional[str]: + if shutil.which(HOMEBREW) is not None: + return None + return ( "ERROR: Homebrew is required to install the macOS system dependencies.\n" f"Install it from {HOMEBREW_SITE}, then run this script again." diff --git a/scripts/bootstrap/preflight.py b/scripts/bootstrap/preflight.py index c317bbf34..8ba01e08b 100644 --- a/scripts/bootstrap/preflight.py +++ b/scripts/bootstrap/preflight.py @@ -61,14 +61,9 @@ def check_build_interpreter( environment: The variables the probes see. Raises: - SystemExit: If the interpreter is missing, cannot play audio, or a release lacks Tk. + SystemExit: If the interpreter cannot play audio, or a release lacks Tk. """ print("Checking the build environment...") - if not python.is_file(): - raise SystemExit( - f"ERROR: build interpreter not found at {python}.\nRun 'make build' to create the build environment." - ) - if not can_import(python, PYAUDIO, runner=runner, cwd=cwd, environment=environment): raise SystemExit( "ERROR: the build interpreter cannot import pyaudio, so the bundle would carry no audio playback.\n" diff --git a/scripts/bootstrap/venv_build.py b/scripts/bootstrap/venv_build.py index c8928cc7d..f7e83e883 100644 --- a/scripts/bootstrap/venv_build.py +++ b/scripts/bootstrap/venv_build.py @@ -11,37 +11,41 @@ def build_environment( root: Path, + platform: Platform, *, runner: Runner, environment: Mapping[str, str], ) -> Path: - """The virtual environment a bundle is built in, created under ``root`` where it is missing. + """The interpreter of the virtual environment a bundle is built in, created under ``root`` where it is missing. Every package a build installs lands here, so the interpreter running the script stays as - it was found. + it was found. An environment is present once its interpreter is; a directory an interrupted + creation left behind is cleared and created again. Args: root: The repository. + platform: The system, which places the interpreter inside the environment. runner: What runs the command creating the environment. environment: The variables the command sees. Returns: - Path: The environment's directory. + Path: The environment's interpreter. """ directory = root / BUILD_ENVIRONMENT - if directory.is_dir(): + python = platform.interpreter(directory) + if python.is_file(): print("Virtual environment already exists.") - return directory + return python print("Creating virtual environment...") expect_success( runner, - (sys.executable, "-m", "venv", str(directory)), + (sys.executable, "-m", "venv", "--clear", str(directory)), cwd=root, environment=environment, ) print("Virtual environment created.") - return directory + return python def install( @@ -80,11 +84,3 @@ def install( environment=guarded, ) print("sampletones Python package installed successfully.") - - -def interpreter( - root: Path, - platform: Platform, -) -> Path: - """The interpreter of the build environment under ``root``.""" - return platform.interpreter(root / BUILD_ENVIRONMENT) diff --git a/scripts/bundle.py b/scripts/bundle.py index 054ac2a2a..46d20a5ee 100644 --- a/scripts/bundle.py +++ b/scripts/bundle.py @@ -12,7 +12,7 @@ from bootstrap.preflight import check_build_interpreter from bootstrap.processes import Runner, expect_success, run from bootstrap.repository import repository_root -from bootstrap.venv_build import build_environment, install, interpreter +from bootstrap.venv_build import build_environment, install BUNDLE_NAME: Final[str] = "sampletones" DISTRIBUTION: Final[str] = "bin" @@ -151,8 +151,12 @@ def build_bundle( if options.release: print("Release build: onedir bundle, injecting release deployment configuration") - build_environment(root, runner=runner, environment=environment) - python = interpreter(root, platform) + python = build_environment( + root, + platform, + runner=runner, + environment=environment, + ) install( root, python, diff --git a/scripts/setup_environment.py b/scripts/setup_environment.py index 990eff8cc..3df283db8 100644 --- a/scripts/setup_environment.py +++ b/scripts/setup_environment.py @@ -2,7 +2,7 @@ import os import platform as running import sys -from typing import Dict, Final, List, Mapping, Optional, Sequence +from typing import Dict, Final, List, Mapping, Optional, Sequence, Tuple import detect_cuda @@ -11,6 +11,12 @@ GPU_AUTO: Final[str] = "auto" GPU_OFF: Final[str] = "0" +GPU_CHOICES: Final[Tuple[str, ...]] = ( + GPU_AUTO, + GPU_OFF, + detect_cuda.EXTRA_GPU, + detect_cuda.EXTRA_GPU_CUDA11, +) DEFAULT_GPU: Final[str] = GPU_AUTO DEVELOPMENT_GROUP: Final[str] = "dev" DARWIN: Final[str] = "Darwin" @@ -83,7 +89,8 @@ def main(argv: Sequence[str]) -> int: parser.add_argument( "--gpu", default=DEFAULT_GPU, - help="auto to match the NVIDIA driver, 0 for the CPU backend, or the name of a GPU extra", + choices=GPU_CHOICES, + help="auto to match the NVIDIA driver, 0 for the CPU backend, or the GPU extra to install", ) arguments = parser.parse_args(list(argv)) diff --git a/tests/unit/scripts/bootstrap/test_platforms.py b/tests/unit/scripts/bootstrap/test_platforms.py index b7cb20846..9bbf972f4 100644 --- a/tests/unit/scripts/bootstrap/test_platforms.py +++ b/tests/unit/scripts/bootstrap/test_platforms.py @@ -3,9 +3,10 @@ import pytest +from bootstrap.platforms import macos from bootstrap.platforms.factory import platform_named from bootstrap.platforms.linux import Linux -from bootstrap.platforms.macos import MacOS +from bootstrap.platforms.macos import HOMEBREW, HOMEBREW_SITE, MacOS from bootstrap.platforms.windows import Windows from tests.suite.base import BaseTestSuite from tests.suite.case import BaseRegularTestCase @@ -88,8 +89,20 @@ def test_linux_updates_apt_before_installing(self) -> None: assert "python3-tk" in commands[1] def test_macos_installs_portaudio_through_homebrew(self) -> None: - assert MacOS().system_packages() == (("brew", "install", "portaudio"),) - assert MacOS().missing_package_manager() is not None + assert MacOS().system_packages() == ((HOMEBREW, "install", "portaudio"),) + + def test_macos_with_homebrew_has_its_package_manager(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(macos.shutil, "which", lambda name: f"/opt/homebrew/bin/{name}") + + assert MacOS().missing_package_manager() is None + + def test_macos_without_homebrew_names_where_to_get_it(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(macos.shutil, "which", lambda name: None) + + refusal = MacOS().missing_package_manager() + + assert refusal is not None + assert HOMEBREW_SITE in refusal def test_windows_installs_nothing(self) -> None: assert Windows().system_packages() == () diff --git a/tests/unit/scripts/bootstrap/test_preflight.py b/tests/unit/scripts/bootstrap/test_preflight.py index 43a4149f8..001ae8aa7 100644 --- a/tests/unit/scripts/bootstrap/test_preflight.py +++ b/tests/unit/scripts/bootstrap/test_preflight.py @@ -9,9 +9,7 @@ @pytest.fixture def python(tmp_path: Path) -> Path: - interpreter = tmp_path / "python" - interpreter.write_text("") - return interpreter + return tmp_path / "python" class TestCanImport: @@ -24,17 +22,6 @@ def test_the_probe_is_quiet_and_answers_the_status(self, python: Path, tmp_path: class TestCheckBuildInterpreter: - def test_a_missing_interpreter_is_refused(self, tmp_path: Path) -> None: - with pytest.raises(SystemExit, match="build interpreter not found"): - check_build_interpreter( - tmp_path / "absent", - Linux(), - release=False, - runner=RecordingRunner({}, None), - cwd=tmp_path, - environment={}, - ) - def test_an_interpreter_without_audio_playback_is_refused_with_the_platform_s_advice( self, python: Path, diff --git a/tests/unit/scripts/bootstrap/test_processes.py b/tests/unit/scripts/bootstrap/test_processes.py index 4a5b310d2..1267cadbe 100644 --- a/tests/unit/scripts/bootstrap/test_processes.py +++ b/tests/unit/scripts/bootstrap/test_processes.py @@ -32,7 +32,7 @@ def test_what_the_script_printed_leads_the_command_s_output( quiet=False, ) - assert capfd.readouterr().out == "before\ninside\n" + assert capfd.readouterr().out.splitlines() == ["before", "inside"] class TestExpectSuccess: diff --git a/tests/unit/scripts/bootstrap/test_venv_build.py b/tests/unit/scripts/bootstrap/test_venv_build.py index d6406103f..295d94271 100644 --- a/tests/unit/scripts/bootstrap/test_venv_build.py +++ b/tests/unit/scripts/bootstrap/test_venv_build.py @@ -2,7 +2,7 @@ from pathlib import Path from bootstrap.platforms.linux import Linux -from bootstrap.venv_build import BUILD_ENVIRONMENT, build_environment, install, interpreter +from bootstrap.venv_build import BUILD_ENVIRONMENT, build_environment, install from tests.suite.bootstrap import RecordingRunner @@ -10,18 +10,27 @@ class TestBuildEnvironment: def test_a_missing_environment_is_created_by_the_running_interpreter(self, tmp_path: Path) -> None: runner = RecordingRunner({}, None) - directory = build_environment(tmp_path, runner=runner, environment={}) + python = build_environment(tmp_path, Linux(), runner=runner, environment={}) - assert directory == tmp_path / BUILD_ENVIRONMENT - assert runner.lines == [f"{sys.executable} -m venv {directory}"] + assert python == Linux().interpreter(tmp_path / BUILD_ENVIRONMENT) + assert runner.lines == [f"{sys.executable} -m venv --clear {tmp_path / BUILD_ENVIRONMENT}"] - def test_an_existing_environment_is_kept(self, tmp_path: Path) -> None: + def test_an_environment_with_its_interpreter_is_kept(self, tmp_path: Path) -> None: + python = Linux().interpreter(tmp_path / BUILD_ENVIRONMENT) + python.parent.mkdir(parents=True) + python.write_text("") + runner = RecordingRunner({}, None) + + assert build_environment(tmp_path, Linux(), runner=runner, environment={}) == python + assert runner.lines == [] + + def test_a_directory_an_interrupted_creation_left_is_created_again(self, tmp_path: Path) -> None: (tmp_path / BUILD_ENVIRONMENT).mkdir() runner = RecordingRunner({}, None) - build_environment(tmp_path, runner=runner, environment={}) + build_environment(tmp_path, Linux(), runner=runner, environment={}) - assert runner.lines == [] + assert runner.lines == [f"{sys.executable} -m venv --clear {tmp_path / BUILD_ENVIRONMENT}"] class TestInstall: @@ -48,8 +57,3 @@ def test_every_install_refuses_an_interpreter_outside_a_virtual_environment(self install(tmp_path, tmp_path / "python", extras=("build",), runner=runner, environment={}) assert all(recorded.environment["PIP_REQUIRE_VIRTUALENV"] == "1" for recorded in runner.commands) - - -class TestInterpreter: - def test_the_interpreter_lies_in_the_build_environment(self, tmp_path: Path) -> None: - assert interpreter(tmp_path, Linux()) == tmp_path / BUILD_ENVIRONMENT / "bin" / "python" diff --git a/tests/unit/scripts/test_setup_environment.py b/tests/unit/scripts/test_setup_environment.py index 3b5e7cfca..c665ecbfb 100644 --- a/tests/unit/scripts/test_setup_environment.py +++ b/tests/unit/scripts/test_setup_environment.py @@ -1,3 +1,6 @@ +import pytest + +from tests.suite.bootstrap import RecordingRunner from tests.suite.scripts import load_script setup_environment = load_script("setup_environment.py") @@ -14,6 +17,24 @@ def test_auto_on_macos_keeps_the_cpu_backend(self) -> None: assert setup_environment.gpu_extra("auto", system="Darwin") is None +class TestMain: + def test_a_gpu_choice_outside_the_extras_is_refused_before_anything_runs( + self, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], + ) -> None: + runner = RecordingRunner({}, None) + monkeypatch.setattr(setup_environment, "run", runner) + + with pytest.raises(SystemExit) as exit_info: + setup_environment.main(["--gpu", "1"]) + + refusal = capsys.readouterr().err + assert exit_info.value.code == 2 + assert runner.lines == [] + assert all(choice in refusal for choice in setup_environment.GPU_CHOICES) + + class TestSetupCommands: def test_the_cpu_backend_synchronizes_and_installs_the_bare_package(self) -> None: commands = setup_environment.setup_commands(None) diff --git a/tests/unit/scripts/test_system_dependencies.py b/tests/unit/scripts/test_system_dependencies.py index bb5f15c34..01ed5bbcb 100644 --- a/tests/unit/scripts/test_system_dependencies.py +++ b/tests/unit/scripts/test_system_dependencies.py @@ -1,5 +1,6 @@ import pytest +from bootstrap.platforms import macos from bootstrap.platforms.linux import Linux from bootstrap.platforms.macos import MacOS from bootstrap.platforms.windows import Windows @@ -29,12 +30,22 @@ def test_linux_runs_the_apt_commands_in_order(self, monkeypatch: pytest.MonkeyPa assert runner.lines[0] == "sudo apt-get update" assert runner.lines[1].startswith("sudo apt-get install -y") + def test_macos_with_homebrew_installs_portaudio(self, monkeypatch: pytest.MonkeyPatch) -> None: + runner = RecordingRunner({}, None) + monkeypatch.setattr(system_dependencies, "current_platform", MacOS) + monkeypatch.setattr(system_dependencies, "run", runner) + monkeypatch.setattr(macos.shutil, "which", lambda name: f"/opt/homebrew/bin/{name}") + + assert system_dependencies.main([]) == 0 + assert runner.lines == [" ".join(command) for command in MacOS().system_packages()] + def test_macos_without_homebrew_is_refused( self, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], ) -> None: monkeypatch.setattr(system_dependencies, "current_platform", MacOS) + monkeypatch.setattr(macos.shutil, "which", lambda name: None) assert system_dependencies.main([]) == 1 assert "Homebrew" in capsys.readouterr().err From a61c3e0c824aecd494d7f0ae65210bacb7a187b6 Mon Sep 17 00:00:00 2001 From: JakimPL Date: Sun, 13 Sep 2026 19:28:41 +0200 Subject: [PATCH 21/36] Split: the test run into suite, doctests and benchmarks targets --- .github/workflows/ci.yml | 14 ++++- .pre-commit-config.yaml | 23 +++++++- Makefile | 14 +++-- docs/development/guidelines.md | 2 +- docs/development/tooling.md | 4 +- scripts/run_tests.py | 60 ++++++++++---------- tests/unit/scripts/test_run_tests.py | 83 ++++++++++++++++++---------- 7 files changed, 128 insertions(+), 72 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d8fd5eaf5..9169e3305 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -74,6 +74,16 @@ jobs: - name: Install the development environment run: uv sync --group dev - - name: Run the doctests, the covered suite and the benchmarks + - name: Run the doctests shell: bash - run: ${{ runner.os == 'Windows' && 'python' || 'python3' }} scripts/run_tests.py --workers auto + run: ${{ runner.os == 'Windows' && 'python' || 'python3' }} scripts/run_tests.py doctests + + - name: Run the test suite with coverage + if: ${{ !cancelled() }} + shell: bash + run: ${{ runner.os == 'Windows' && 'python' || 'python3' }} scripts/run_tests.py suite --workers auto + + - name: Run the benchmarks + if: ${{ !cancelled() }} + shell: bash + run: ${{ runner.os == 'Windows' && 'python' || 'python3' }} scripts/run_tests.py benchmarks diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 966d43b3f..a3d32077d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -117,8 +117,8 @@ repos: stages: - pre-push - - id: pytest - name: pytest + - id: test + name: test entry: make test language: system types: @@ -126,3 +126,22 @@ repos: stages: - pre-push pass_filenames: false + + - id: test-docs + name: test docs + entry: make test-docs + language: system + files: ^src/.*\.py$ + stages: + - pre-push + pass_filenames: false + + - id: benchmarks + name: benchmarks + entry: make benchmarks + language: system + types: + - python + stages: + - pre-push + pass_filenames: false diff --git a/Makefile b/Makefile index 6062d7afe..a2e9dee94 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: help setup install system-deps build release run clean pre-commit test benchmarks lint format +.PHONY: help setup install system-deps build release run clean pre-commit test test-docs benchmarks lint format ifeq ($(OS),Windows_NT) PYTHON := python @@ -21,8 +21,9 @@ help: @echo $(Q) make system-deps - Install system packages required to build and run (apt on Debian-based Linux, Homebrew on macOS)$(Q) @echo $(Q) make build - Compile standalone executable (development deployment config: DEBUG, strict history)$(Q) @echo $(Q) make release - Compile standalone executable with the release deployment config (INFO, self-healing history)$(Q) - @echo $(Q) make test - Run the doctests, the covered suite and the benchmarks$(Q) - @echo $(Q) make benchmarks - Run the measured-duration suite on its own$(Q) + @echo $(Q) make test - Run the test suite with coverage$(Q) + @echo $(Q) make test-docs - Run the doctests$(Q) + @echo $(Q) make benchmarks - Run the measured-duration suite$(Q) @echo $(Q) make clean - Remove build artifacts and cache files$(Q) @echo $(Q) make lint - Run mypy and pylint (ARGS=--mypy or ARGS=--pylint for one of them)$(Q) @echo $(Q) make format - Auto-format code (isort, black)$(Q) @@ -54,10 +55,13 @@ pre-commit: $(PYTHON) scripts/hooks.py test: - $(PYTHON) scripts/run_tests.py + $(PYTHON) scripts/run_tests.py suite + +test-docs: + $(PYTHON) scripts/run_tests.py doctests benchmarks: - $(PYTHON) scripts/run_tests.py --only benchmarks + $(PYTHON) scripts/run_tests.py benchmarks lint: $(PYTHON) scripts/lint.py $(ARGS) diff --git a/docs/development/guidelines.md b/docs/development/guidelines.md index 30c604180..811a92124 100644 --- a/docs/development/guidelines.md +++ b/docs/development/guidelines.md @@ -97,7 +97,7 @@ These rules govern the Python in this repository. They complement 1. A test file mirrors the ownership of the code it exercises. 1. When functionality moves between packages, move its direct unit tests in the same change. -1. **A test whose assertion is a measured duration lives in `tests/benchmarks/`.** The gated suite runs across six workers and under coverage, which multiplies the cost of the code being measured, so those tests run in a pass of their own — serial and uncovered — where the reading is the code's own cost. `make test` runs that pass after the covered one, and `make benchmarks` runs it alone. +1. **A test whose assertion is a measured duration lives in `tests/benchmarks/`.** The gated suite runs across six workers and under coverage, which multiplies the cost of the code being measured, so those tests run in a pass of their own — serial and uncovered — where the reading is the code's own cost. `make test` runs the covered suite, and `make benchmarks` runs the measured pass. 1. Parametrize tests that share a body, using a test-case dataclass. 1. Test case classes and cases themselves should be defined inside the testing class, unless these objects are shared between test classes. A suite inherits from `BaseTestSuite` and names its case class `TestCase`, which inherits from `BaseRegularTestCase`, or from `BaseAutolabelTestCase` where the case derives its own label. The parametrized argument carries the case as `test_case`. 1. For a multi-step scenario, use a test-scenario suite class — a series of functions with assertions. diff --git a/docs/development/tooling.md b/docs/development/tooling.md index 0f27ee96a..924adf6c2 100644 --- a/docs/development/tooling.md +++ b/docs/development/tooling.md @@ -130,7 +130,7 @@ reason. | `system_dependencies.py` | `make system-deps` | Installs the system packages: apt on Debian-based Linux, Homebrew on macOS, nothing on Windows | | `build_environment.py` | CI | Prints the compiler flags a macOS build exports, one `KEY=VALUE` per line | | `clean.py` | `make clean` | Removes the build outputs, the coverage reports and the bytecode caches | -| `run_tests.py` | `make test`, `make benchmarks` | Runs the doctests, the covered suite across six workers, and the benchmarks, every pass whatever the earlier ones reported; `--only` picks one pass and `--workers` sets the count | +| `run_tests.py` | `make test`, `make test-docs`, `make benchmarks` | Runs one pass of the tests, named on its command line: `suite`, the covered suite across six workers (`--workers` sets the count); `doctests`; or `benchmarks`, serial and uncovered. Each pass is a target, a pre-push hook and a CI step of its own, so a failure names its pass | | `lint.py` | `make lint` | Runs mypy over the files `pyproject.toml` configures and pylint over `src/` and `scripts/`; `--mypy` or `--pylint` picks one, and named paths narrow both | | `formatting.py` | `make format` | Runs isort, then black, over `src/`, `tests/` and `scripts/`, or over the paths named | | `hooks.py` | `make pre-commit` | Installs the git hooks pre-commit runs at commit and at push | @@ -159,7 +159,7 @@ interpreter (`preflight.py`), and the platforms (`platforms/`). | Where a build installs | `scripts/bootstrap/venv_build.py` | | What a bundle has to carry before it is built | `scripts/bootstrap/preflight.py` | | The PyInstaller invocation | `scripts/bundle.py` | -| The passes `make test` runs, and their order | `scripts/run_tests.py` | +| The test passes and the command each runs | `scripts/run_tests.py` | | What `make lint` and `make format` sweep | `scripts/lint.py`, `scripts/formatting.py` | | The GPU extra a machine gets | `scripts/detect_cuda.py` | | What a release bundle is held to | `scripts/ci/checks/bundle.py` | diff --git a/scripts/run_tests.py b/scripts/run_tests.py index 7b905b543..73c34c855 100644 --- a/scripts/run_tests.py +++ b/scripts/run_tests.py @@ -1,57 +1,57 @@ import argparse import os import sys -from typing import Final, Optional, Sequence, Tuple +from typing import Dict, Final, Sequence, Tuple -from bootstrap.passes import Pass, run_passes +from bootstrap.passes import Pass from bootstrap.processes import run from bootstrap.repository import repository_root -DOCTESTS: Final[str] = "doctests" SUITE: Final[str] = "suite" +DOCTESTS: Final[str] = "doctests" BENCHMARKS: Final[str] = "benchmarks" DEFAULT_WORKERS: Final[str] = "6" PYTEST: Final[Tuple[str, ...]] = ("uv", "run", "python", "-m", "pytest") -def planned_passes(workers: str) -> Tuple[Pass, ...]: - """The passes of a test run, in order: the doctests, the covered suite, the benchmarks. +def planned_passes(workers: str) -> Dict[str, Pass]: + """The passes a test run is made of, by name: the covered suite, the doctests, the benchmarks. - The covered suite runs across ``workers`` pytest workers. The benchmarks run last, serial, - uncovered and with their output shown, so a measured duration is the code's own cost and - its reading reaches the terminal. + Each pass is a target and a hook of its own, so a failure names the pass it belongs to. The + covered suite runs across ``workers`` pytest workers. The benchmarks run serial, uncovered + and with their output shown, so a measured duration is the code's own cost and its reading + reaches the terminal. Args: workers: The worker count for the covered suite, or ``auto`` for one per processor. + + Returns: + Dict[str, Pass]: Every pass, under its name. """ - return ( - Pass( - DOCTESTS, - "Running doctests...", - (*PYTEST, "src/", "--doctest-modules", "--no-cov"), - ), + passes = ( Pass( SUITE, "Running pytest with coverage...", (*PYTEST, "-n", workers, "--cov", "--ignore=tests/benchmarks"), ), + Pass( + DOCTESTS, + "Running doctests...", + (*PYTEST, "src/", "--doctest-modules", "--no-cov"), + ), Pass( BENCHMARKS, "Running benchmarks...", (*PYTEST, "tests/benchmarks", "--no-cov", "-s"), ), ) - - -def selected(passes: Sequence[Pass], only: Optional[str]) -> Tuple[Pass, ...]: - """The passes a run performs: all of them, or the one ``only`` names.""" - return tuple(current for current in passes if only is None or current.name == only) + return {current.name: current for current in passes} def main(argv: Sequence[str]) -> int: - """Runs the doctests, the covered suite and the benchmarks, and reports which failed.""" - parser = argparse.ArgumentParser(description="Run the SampleToNES tests.") - parser.add_argument("--only", choices=(DOCTESTS, SUITE, BENCHMARKS), help="run one pass alone") + """Runs one pass of the tests and exits with the status pytest gave it.""" + parser = argparse.ArgumentParser(description="Run one pass of the SampleToNES tests.") + parser.add_argument("name", choices=(SUITE, DOCTESTS, BENCHMARKS), help="the pass to run") parser.add_argument( "--workers", default=DEFAULT_WORKERS, @@ -59,18 +59,14 @@ def main(argv: Sequence[str]) -> int: ) arguments = parser.parse_args(list(argv)) - failed = run_passes( - selected(planned_passes(arguments.workers), arguments.only), - root=repository_root(), - runner=run, + chosen = planned_passes(arguments.workers)[arguments.name] + print(chosen.announcement) + return run( + chosen.command, + cwd=repository_root(), environment=os.environ, + quiet=False, ) - if failed: - print(f"Tests failed: {', '.join(failed)}.") - return 1 - - print("All tests passed.") - return 0 if __name__ == "__main__": diff --git a/tests/unit/scripts/test_run_tests.py b/tests/unit/scripts/test_run_tests.py index 2eb92987b..b0be40d0f 100644 --- a/tests/unit/scripts/test_run_tests.py +++ b/tests/unit/scripts/test_run_tests.py @@ -1,53 +1,80 @@ +from dataclasses import dataclass +from typing import Tuple + import pytest +from tests.suite.base import BaseTestSuite from tests.suite.bootstrap import RecordingRunner +from tests.suite.case import BaseRegularTestCase from tests.suite.scripts import load_script run_tests = load_script("run_tests.py") class TestPlannedPasses: - def test_the_doctests_run_first_and_the_benchmarks_last(self) -> None: - passes = run_tests.planned_passes("6") + def test_every_pass_is_found_under_its_name(self) -> None: + passes = run_tests.planned_passes(run_tests.DEFAULT_WORKERS) + + assert all(name == current.name for name, current in passes.items()) + assert set(passes) == {run_tests.SUITE, run_tests.DOCTESTS, run_tests.BENCHMARKS} + + def test_the_suite_is_covered_across_the_workers_and_leaves_the_benchmarks_out(self) -> None: + command = run_tests.planned_passes("auto")[run_tests.SUITE].command + + assert command[-4:] == ("-n", "auto", "--cov", "--ignore=tests/benchmarks") - assert [current.name for current in passes] == ["doctests", "suite", "benchmarks"] - assert "--doctest-modules" in passes[0].command - assert passes[1].command[-4:] == ("-n", "6", "--cov", "--ignore=tests/benchmarks") - assert "--no-cov" in passes[2].command - assert "-s" in passes[2].command + def test_the_doctests_read_the_sources_uncovered(self) -> None: + command = run_tests.planned_passes(run_tests.DEFAULT_WORKERS)[run_tests.DOCTESTS].command + assert "--doctest-modules" in command + assert "--no-cov" in command -class TestSelected: - def test_a_name_keeps_one_pass(self) -> None: - passes = run_tests.planned_passes("6") + def test_the_benchmarks_run_serial_uncovered_and_show_their_readings(self) -> None: + command = run_tests.planned_passes(run_tests.DEFAULT_WORKERS)[run_tests.BENCHMARKS].command - assert run_tests.selected(passes, "suite") == (passes[1],) - assert run_tests.selected(passes, None) == passes + assert "tests/benchmarks" in command + assert "--no-cov" in command + assert "-s" in command + assert "-n" not in command class TestMain: - def test_only_runs_the_named_pass( - self, - monkeypatch: pytest.MonkeyPatch, - capsys: pytest.CaptureFixture[str], - ) -> None: + def test_the_named_pass_runs_alone(self, monkeypatch: pytest.MonkeyPatch) -> None: runner = RecordingRunner({}, None) monkeypatch.setattr(run_tests, "run", runner) - assert run_tests.main(["--only", "benchmarks"]) == 0 - assert len(runner.lines) == 1 - assert "tests/benchmarks" in runner.lines[0] - assert "All tests passed." in capsys.readouterr().out + assert run_tests.main([run_tests.DOCTESTS]) == 0 + assert runner.lines == [ + " ".join(run_tests.planned_passes(run_tests.DEFAULT_WORKERS)[run_tests.DOCTESTS].command) + ] - def test_a_failing_pass_stops_nothing_and_is_named( + def test_the_status_is_pytest_s_own(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(run_tests, "run", RecordingRunner({"pytest": 5}, None)) + + assert run_tests.main([run_tests.SUITE, "--workers", "auto"]) == 5 + + +class TestRefusedPasses(BaseTestSuite): + @dataclass(frozen=True, kw_only=True) + class TestCase(BaseRegularTestCase): + argv: Tuple[str, ...] + + test_cases = ( + TestCase(label="a missing pass", argv=()), + TestCase(label="an unknown pass", argv=("everything",)), + ) + + @pytest.mark.parametrize("test_case", test_cases, ids=lambda test_case: test_case.label) + def test_a_pass_outside_the_three_is_refused_before_anything_runs( self, monkeypatch: pytest.MonkeyPatch, - capsys: pytest.CaptureFixture[str], + test_case: TestCase, ) -> None: - runner = RecordingRunner({"--doctest-modules": 1}, None) + runner = RecordingRunner({}, None) monkeypatch.setattr(run_tests, "run", runner) - assert run_tests.main(["--workers", "auto"]) == 1 - assert len(runner.lines) == 3 - assert "-n auto" in runner.lines[1] - assert "Tests failed: doctests." in capsys.readouterr().out + with pytest.raises(SystemExit) as exit_info: + run_tests.main(test_case.argv) + + assert exit_info.value.code == 2 + assert runner.lines == [] From 6a2759f9628edd2a52ce695d0b2980da75236d88 Mon Sep 17 00:00:00 2001 From: JakimPL Date: Sun, 13 Sep 2026 19:42:08 +0200 Subject: [PATCH 22/36] Lightened: the command registry and the tools' failure messages --- docs/development/tooling.md | 15 ++-- src/sampletones/commands/convert.py | 9 +-- src/sampletones/commands/open.py | 6 +- src/sampletones/dispatcher.py | 7 +- src/sampletones_core/headless/conversion.py | 29 ++++++-- src/sampletones_shared/utils/validation.py | 33 ++++++++- src/sampletones_tools/assets/command.py | 8 +-- src/sampletones_tools/calibration/__init__.py | 37 ---------- src/sampletones_tools/calibration/command.py | 3 +- src/sampletones_tools/codec/command.py | 12 ++-- .../codec/study/report/run.py | 28 +++++--- src/sampletones_tools/codec/study/session.py | 32 +++++++-- .../codec/study/variants/registry.py | 6 +- .../unit/sampletones/commands/test_convert.py | 26 +++++++ .../sampletones/commands/test_registry.py | 68 +++++++++++++++---- tests/unit/sampletones/test_dispatcher.py | 8 ++- .../headless/test_conversion.py | 22 +++++- .../utils/test_validation.py | 37 +++++++++- .../sampletones_tools/assets/test_command.py | 14 ++-- .../codec/study/report/test_run.py | 38 +++++++++++ .../sampletones_tools/codec/test_command.py | 46 +++++++++++-- 21 files changed, 374 insertions(+), 110 deletions(-) create mode 100644 tests/unit/sampletones_tools/codec/study/report/test_run.py diff --git a/docs/development/tooling.md b/docs/development/tooling.md index 924adf6c2..5c0e9dcd6 100644 --- a/docs/development/tooling.md +++ b/docs/development/tooling.md @@ -70,7 +70,7 @@ arguments. Each command turns its arguments into a frozen record, field by field |---|---| | `run [--config FILE]` | Starts the application | | `open PATH [--config FILE]` | Starts the application with a `.stp` project, a `.stn` reconstruction or an `.ins` library loaded; a recording is refused with the `convert` line to run instead | -| `convert SOURCE... [-o FILE] [--config FILE] [--channels LIST \| --stems FILE]` | Reconstructs recordings into one `.stn` file, or every recording under one directory file by file. `--stems` names a JSON file holding the setup the `.stn` record stores, its entries paired with the sources in order; the pairing is printed before the run | +| `convert SOURCE... [-o FILE] [--config FILE] [--channels LIST \| --stems FILE]` | Reconstructs recordings into one `.stn` file, or every recording under one directory file by file. `--stems` names a JSON file holding the setup the `.stn` record stores, its entries paired with the sources in order; the pairing is printed before the run, and a missing source, a file other than a recording or a missing stems file is refused first | | `library [--config FILE]` | Generates the instruction library for a configuration | | `self-check` | Verifies that the build's imports, bundled resources and configuration files are usable | @@ -88,7 +88,7 @@ The developer commands, listed by `sampletones_tools/registry.py` and run as | `codec report -o DIR` | Compresses the synthetic corpus under every layer of the codec and writes the report the format's constants are settled from, as CSV and Markdown | | `codec study [--manifest FILE] [--project FILE]... [--reconstruction PATH]... [-o DIR] [--lengthen SECONDS] [--variants LIST]` | Encodes the projects and stems it is given, or the ones a manifest names, under every candidate change to the codec and writes the sizes, the times, a verdict per candidate and the manifest that repeats the run; without `-o` the run lands under Documents/SampleToNES/compression | | `driver [--directory DIR]` | Assembles the NES player driver with cc65 and prints the layout the build produced; without `--directory` it writes the driver the package ships, which needs a checkout | -| `icons [--directory DIR]` | Writes the icon suite from the mark; without `--directory` it writes the icons the package ships, which needs a checkout | +| `icons [--directory DIR]` | Writes the icon suite from the mark, into `--directory` or over the icons the package ships. Needs a checkout | | `nsf render --directory DIR [--tail SECONDS]` | Renders every exported `.nsf` file in the directory to a wave beside it, through ffmpeg's libgme demuxer | ## The tools package @@ -103,7 +103,8 @@ developer command does there: - **The checkout guard.** `sampletones_tools/checkout.py` holds `require_checkout(command)`: the repository root must hold `pyproject.toml` beside `src/`, or the command exits naming `uv run sampletones ` in a checkout. Every developer command that reads or writes the - repository calls it first. A command that measures the code on this machine runs anywhere. + repository calls it first, and so does one that needs a development dependency, as `icons` needs + Pillow. A command that measures the code on this machine runs anywhere. - **No default derived from the repository.** An emitter takes a required `--output`; a measurement defaults to the user's Documents. Nothing a developer command writes lands beside an installed package. @@ -115,9 +116,11 @@ developer command does there: Developer commands are run as `uv run sampletones ` from a checkout; the `sampletones` command `make setup` installs is a wheel and refuses the guarded ones the same way. A command module -imports pillow, py65 and the like inside `run`, and a test imports the registry in a subprocess and -asserts they stay out of `sys.modules`, since a startup failure in any tool module would break every -invocation, the GUI included. The editable install puts `src/` on the path whole, so a checkout +imports pillow, NumPy and the like inside `run`, and a test imports the registry in a subprocess and +asserts that only the command, registry and package modules of the tools load and no heavy library +does, since a startup failure in any tool module would break every invocation, the GUI included. A +command reports a refused value in one line: `describe_failure` in +`sampletones_shared/utils/validation.py` renders a validation error the way a person reads it. The editable install puts `src/` on the path whole, so a checkout sees the tools package whatever the wheel lists; hatchling's `dev-mode-exact` stays off for that reason. diff --git a/src/sampletones/commands/convert.py b/src/sampletones/commands/convert.py index 66183ac2e..e396d0e9e 100644 --- a/src/sampletones/commands/convert.py +++ b/src/sampletones/commands/convert.py @@ -43,8 +43,8 @@ def run(arguments: Namespace) -> int: """Reconstructs the sources under the stems setup the options describe. Raises: - SystemExit: If a channel is unknown, the stems file is no setup, or the sources and the - setup pair up wrong. + SystemExit: If a source is missing or no recording, a channel is unknown, the stems file + is missing or no setup, or the sources and the setup pair up wrong. """ given = ConvertArguments( sources=tuple(arguments.sources), @@ -63,12 +63,13 @@ def run(arguments: Namespace) -> int: reconstruct, ) from sampletones_shared.array import report_array_backend + from sampletones_shared.utils.validation import describe_failure try: stems = load_stems(given.stems) if given.stems is not None else classic_setup(channels_named(given.channels)) request = ConversionRequest(sources=given.sources, stems=stems, output_path=given.output) - except (TypeError, ValueError) as error: - raise SystemExit(str(error)) from error + except ValueError as error: + raise SystemExit(describe_failure(error)) from error for line in request.pairing(): print(line) diff --git a/src/sampletones/commands/open.py b/src/sampletones/commands/open.py index 8db31885c..b9d92871c 100644 --- a/src/sampletones/commands/open.py +++ b/src/sampletones/commands/open.py @@ -32,20 +32,20 @@ def run(arguments: Namespace) -> int: """ given = OpenArguments(path=arguments.path, config=arguments.config) + from sampletones_core.reconstructions.converter.paths import is_audio_file from sampletones_shared.paths.extensions import ( EXT_FILE_LIBRARY, EXT_FILE_PROJECT, EXT_FILE_RECONSTRUCTION, - EXT_FILES_AUDIO, ) if not given.path.is_file(): raise SystemExit(f"No file at {given.path}.") - suffix = given.path.suffix.lower() - if suffix in EXT_FILES_AUDIO: + if is_audio_file(given.path): raise SystemExit(f"{given.path} is a recording; run: sampletones convert {given.path}") + suffix = given.path.suffix.lower() if suffix not in (EXT_FILE_PROJECT, EXT_FILE_RECONSTRUCTION, EXT_FILE_LIBRARY): raise SystemExit( f"{given.path} is neither a {EXT_FILE_PROJECT} project, a {EXT_FILE_RECONSTRUCTION} " diff --git a/src/sampletones/dispatcher.py b/src/sampletones/dispatcher.py index 1c01c4f37..5a33dd278 100644 --- a/src/sampletones/dispatcher.py +++ b/src/sampletones/dispatcher.py @@ -9,6 +9,11 @@ DEFAULT_COMMAND: Final[str] = "run" COMMAND_FIELD: Final[str] = "command" COMMAND_METAVAR: Final[str] = "" +DEVELOPER_GUIDE: Final[str] = "docs/development/tooling.md" +EPILOG: Final[str] = ( + f"Run '{PROGRAM} {COMMAND_METAVAR} --help' for a command's options. The commands for developing " + f"SampleToNES run from a checkout as 'uv run {PROGRAM} {COMMAND_METAVAR}'; {DEVELOPER_GUIDE} lists them." +) def build_parser(commands: Sequence[Command]) -> ArgumentParser: @@ -31,7 +36,7 @@ def build_parser(commands: Sequence[Command]) -> ArgumentParser: parser = ArgumentParser( prog=PROGRAM, description=DESCRIPTION, - epilog=f"Run '{PROGRAM} {COMMAND_METAVAR} --help' for a command's options.", + epilog=EPILOG, ) parser.add_argument( "--version", diff --git a/src/sampletones_core/headless/conversion.py b/src/sampletones_core/headless/conversion.py index e0c2bd07e..fee328669 100644 --- a/src/sampletones_core/headless/conversion.py +++ b/src/sampletones_core/headless/conversion.py @@ -21,11 +21,12 @@ ReconstructionConverter, reconstruct_job, ) -from sampletones_core.reconstructions.converter.paths import group_output_path +from sampletones_core.reconstructions.converter.paths import group_output_path, is_audio_file from sampletones_core.reconstructions.progress import ReconstructionProgress from sampletones_core.reconstructions.reconstructor.stems.configs.config import StemsConfig from sampletones_core.reconstructions.reconstructor.stems.configs.entry import StemEntry from sampletones_shared.logger import logger, null_logger +from sampletones_shared.paths.extensions import EXT_FILES_AUDIO from sampletones_shared.utils.serialization import load_json BAR_STEPS: Final[int] = 1000 @@ -63,12 +64,15 @@ def load_stems(path: Path) -> StemsConfig: """The stems setup a JSON file holds, validated the way the ``.stn`` record is. Raises: - TypeError: If the file holds anything other than a mapping. - ValueError: If the mapping is no stems setup. + ValueError: If no file stands at the path, the file holds anything other than a JSON + mapping, or the mapping is no stems setup. """ + if not path.is_file(): + raise ValueError(f"No stems file at {path}.") + loaded = load_json(path) if not isinstance(loaded, dict): - raise TypeError(f"Stems file {path} must hold a mapping, got {type(loaded).__name__}") + raise ValueError(f"Stems file {path} must hold a mapping, got {type(loaded).__name__}.") return StemsConfig.model_validate(loaded) @@ -112,10 +116,21 @@ def directory(self) -> Optional[Path]: @model_validator(mode="after") def _sources_are_recordings_or_one_directory(self) -> Self: """Raises: - ValueError: If a directory stands among several sources. + ValueError: If a directory stands among several sources, or a source names nothing or + a file other than a recording. """ - if self.directory is None and any(source.is_dir() for source in self.sources): - raise ValueError("Sources are recordings, or one directory alone.") + if self.directory is not None: + return self + + for source in self.sources: + if source.is_dir(): + raise ValueError("Sources are recordings, or one directory alone.") + + if not source.exists(): + raise ValueError(f"No file at {source}.") + + if not is_audio_file(source): + raise ValueError(f"{source} is no recording; a source is a {', '.join(EXT_FILES_AUDIO)} file.") return self diff --git a/src/sampletones_shared/utils/validation.py b/src/sampletones_shared/utils/validation.py index f4d15912f..9e6cb67b9 100644 --- a/src/sampletones_shared/utils/validation.py +++ b/src/sampletones_shared/utils/validation.py @@ -3,12 +3,14 @@ import copy from collections.abc import Mapping from dataclasses import dataclass -from typing import Any, Generic, List, Optional, Tuple, Type, TypeVar, Union +from typing import Any, Final, Generic, List, Optional, Tuple, Type, TypeVar, Union from pydantic import BaseModel, ValidationError +from pydantic_core import ErrorDetails ModelTypeT = TypeVar("ModelTypeT", bound=BaseModel) Location = Tuple[Union[str, int], ...] +VALUE_ERROR: Final[str] = "value_error" @dataclass(frozen=True) @@ -48,6 +50,35 @@ def flatten_location(location: Location) -> str: return "".join(parts) +def describe_failure(error: ValueError) -> str: + """ + Renders a refused value the way a command line reports it, one line per problem. + + A reason a validator raised keeps its own words, and a broken constraint is named by the + field it binds, so a person reads what to change in place of Pydantic's diagnostic layout. + + Args: + error: The failure, a Pydantic validation error or a plain ``ValueError``. + + Returns: + str: The reasons, one per line. + """ + match error: + case ValidationError(): + return "\n".join(_describe_detail(detail) for detail in error.errors()) + case _: + return str(error) + + +def _describe_detail(detail: ErrorDetails) -> str: + context = detail.get("ctx") + if detail["type"] == VALUE_ERROR and context is not None: + return str(context["error"]) + + location = flatten_location(tuple(detail["loc"])) + return f"{location}: {detail['msg']}" if location else detail["msg"] + + def validate_with_recovery( model_class: Type[ModelTypeT], data: Mapping[str, Any], diff --git a/src/sampletones_tools/assets/command.py b/src/sampletones_tools/assets/command.py index 340729397..197a27e55 100644 --- a/src/sampletones_tools/assets/command.py +++ b/src/sampletones_tools/assets/command.py @@ -24,17 +24,17 @@ def configure(parser: ArgumentParser) -> None: def run(arguments: Namespace) -> int: """Writes the icon suite and reports each file produced. - Writing the icons the package ships needs a checkout, since that is where the package is. + The suite is rasterized with Pillow, which the project environment carries, so the command + runs from a checkout wherever it writes. Raises: - SystemExit: If the run writes the shipped icons outside a checkout. + SystemExit: If the run happens outside a checkout. """ given = IconsArguments(directory=arguments.directory) from sampletones_tools.checkout import require_checkout - if given.directory is None: - require_checkout(NAME) + require_checkout(NAME) from sampletones_tools.assets.mark.specification import Mark from sampletones_tools.assets.mark.suite import write_icon_suite diff --git a/src/sampletones_tools/calibration/__init__.py b/src/sampletones_tools/calibration/__init__.py index e63aed347..e69de29bb 100644 --- a/src/sampletones_tools/calibration/__init__.py +++ b/src/sampletones_tools/calibration/__init__.py @@ -1,37 +0,0 @@ -from .config.corpus import CorpusConfig -from .config.referee import RefereeConfig -from .corpus.item import CorpusItem -from .corpus.synthesis import build_corpus -from .corpus.writer import write_corpus -from .referee.auditory import MultiResolutionAuditoryReferee -from .referee.factory import build_referees -from .referee.protocol import Referee -from .referee.zimtohrli import ZimtohrliReferee, find_zimtohrli -from .report import write_csv, write_markdown -from .runner import ( - CalibrationRow, - CalibrationVariant, - build_variants, - ensure_library, - evaluate_variants, -) - -__all__ = [ - "CalibrationRow", - "CalibrationVariant", - "CorpusConfig", - "CorpusItem", - "MultiResolutionAuditoryReferee", - "Referee", - "RefereeConfig", - "ZimtohrliReferee", - "build_corpus", - "build_referees", - "build_variants", - "ensure_library", - "evaluate_variants", - "find_zimtohrli", - "write_corpus", - "write_csv", - "write_markdown", -] diff --git a/src/sampletones_tools/calibration/command.py b/src/sampletones_tools/calibration/command.py index 035957bea..fec1b19c5 100644 --- a/src/sampletones_tools/calibration/command.py +++ b/src/sampletones_tools/calibration/command.py @@ -59,6 +59,7 @@ def run(arguments: Namespace) -> int: from sampletones_core.headless.config import load_config from sampletones_core.headless.conversion import channels_named + from sampletones_shared.utils.validation import describe_failure from sampletones_tools.calibration.session import ( BASE_BLEND, DEFAULT_PERCEPTUAL_EXPONENTS, @@ -79,7 +80,7 @@ def run(arguments: Namespace) -> int: channels=channels_named(given.channels), ) except ValueError as error: - raise SystemExit(str(error)) from error + raise SystemExit(describe_failure(error)) from error calibrate(request) return 0 diff --git a/src/sampletones_tools/codec/command.py b/src/sampletones_tools/codec/command.py index fd296af06..830f230d2 100644 --- a/src/sampletones_tools/codec/command.py +++ b/src/sampletones_tools/codec/command.py @@ -6,7 +6,7 @@ from sampletones_shared.command import Command NAME: Final[str] = "codec" -HELP: Final[str] = "measure the song codec on the synthetic corpus or on songs of this machine" +HELP: Final[str] = "measure the song codec on the synthetic corpus or on the songs named" ACTION_FIELD: Final[str] = "action" ACTION_METAVAR: Final[str] = "" REPORT: Final[str] = "report" @@ -102,11 +102,14 @@ def _study(given: StudyArguments) -> int: """Measures the sources a run names. Raises: - SystemExit: If the run names neither a source nor a manifest. + SystemExit: If the run names neither a source nor a manifest, the manifest is missing or + broken, the lengthening is below one second, or a variant is unknown. """ + from sampletones_shared.utils.validation import describe_failure from sampletones_tools.codec.study.session import ( resolve_manifest, run_study, + study_variants, variant_names, ) @@ -118,10 +121,11 @@ def _study(given: StudyArguments) -> int: lengthen_seconds=given.lengthen, variants=variant_names(given.variants), ) + variants = study_variants(manifest.variants) except ValueError as error: - raise SystemExit(str(error)) from error + raise SystemExit(describe_failure(error)) from error - run_study(manifest, given.output) + run_study(manifest, variants, given.output) return 0 diff --git a/src/sampletones_tools/codec/study/report/run.py b/src/sampletones_tools/codec/study/report/run.py index 006b38d98..9281528ac 100644 --- a/src/sampletones_tools/codec/study/report/run.py +++ b/src/sampletones_tools/codec/study/report/run.py @@ -6,6 +6,7 @@ from sampletones_player.compression.compressed import CompressedPlanes from sampletones_shared.paths.source import REPOSITORY_ROOT from sampletones_shared.paths.user import USER_PATH_DOCUMENTS +from sampletones_tools.checkout import is_checkout from sampletones_tools.codec.study.accounting import rows as accounting from sampletones_tools.codec.study.manifest import StudyManifest from sampletones_tools.codec.study.measure import Measurement @@ -41,14 +42,25 @@ def run_directory(output: Optional[Path]) -> Path: def commit_hash() -> str: - """The short hash of the commit the repository stands at, or a marker where git answers nothing.""" - completed = subprocess.run( - ["git", "rev-parse", "--short", "HEAD"], - capture_output=True, - text=True, - check=False, - cwd=REPOSITORY_ROOT, - ) + """The short hash of the commit the checkout stands at, which dates the run's report. + + An installed copy has no checkout of its own around it, and a machine may run without git, so + either records ``unknown``. + """ + if not is_checkout(REPOSITORY_ROOT): + return UNKNOWN_COMMIT + + try: + completed = subprocess.run( + ["git", "rev-parse", "--short", "HEAD"], + capture_output=True, + text=True, + check=False, + cwd=REPOSITORY_ROOT, + ) + except FileNotFoundError: + return UNKNOWN_COMMIT + return completed.stdout.strip() or UNKNOWN_COMMIT diff --git a/src/sampletones_tools/codec/study/session.py b/src/sampletones_tools/codec/study/session.py index cf7daa191..8535768c8 100644 --- a/src/sampletones_tools/codec/study/session.py +++ b/src/sampletones_tools/codec/study/session.py @@ -1,5 +1,5 @@ from pathlib import Path -from typing import Final, List, Optional, Tuple +from typing import Final, List, Optional, Sequence, Tuple from sampletones_shared.logger import logger from sampletones_tools.codec.study.corpus.build import build_corpus @@ -9,6 +9,7 @@ from sampletones_tools.codec.study.variants.baselines import Baselines from sampletones_tools.codec.study.variants.registry import EVERY_VARIANT, selected_variants from sampletones_tools.codec.study.variants.strategy import STRATEGY_ORDER, depth_measurements +from sampletones_tools.codec.study.variants.variant import Variant LIST_SEPARATOR: Final[str] = "," @@ -41,11 +42,15 @@ def resolve_manifest( variants: The names of the variants every song is encoded under, unless a manifest states them. Raises: - ValueError: If neither a manifest nor a source is named. + ValueError: If neither a manifest nor a source is named, no manifest stands at the path, + or the manifest breaks one of its bounds. """ if path is None and not projects and not reconstructions: raise ValueError(NO_SOURCE) + if path is not None and not path.is_file(): + raise ValueError(f"No manifest at {path}.") + manifest = StudyManifest.load(path) if path is not None else None if manifest is not None and not projects and not reconstructions: return manifest @@ -58,11 +63,29 @@ def resolve_manifest( ) -def run_study(manifest: StudyManifest, output: Optional[Path]) -> Path: +def study_variants(names: Sequence[str]) -> Tuple[Variant, ...]: + """The variants a run encodes under, the baseline first, over production encodings of their own. + + Args: + names: The names a manifest states, or ``all``. + + Raises: + ValueError: If a name is registered to no variant. + """ + return selected_variants(names, Baselines()) + + +def run_study( + manifest: StudyManifest, + variants: Sequence[Variant], + output: Optional[Path], +) -> Path: """Encodes every song of the manifest under every variant and writes the run. Args: - manifest: What is measured and under which variants. + manifest: What is measured. + variants: The variants every song is encoded under, as ``study_variants`` selects them + from the manifest's names. output: The directory the run writes into, or ``None`` for a stamped one under the documents. @@ -72,7 +95,6 @@ def run_study(manifest: StudyManifest, output: Optional[Path]) -> Path: Raises: ValueError: If a variant writes a song as streams that play back differently. """ - variants = selected_variants(manifest.variants, Baselines()) directory = run_directory(output) corpus = build_corpus(manifest) diff --git a/src/sampletones_tools/codec/study/variants/registry.py b/src/sampletones_tools/codec/study/variants/registry.py index 5a5a09e81..5b300cf29 100644 --- a/src/sampletones_tools/codec/study/variants/registry.py +++ b/src/sampletones_tools/codec/study/variants/registry.py @@ -40,7 +40,7 @@ def selected_variants( Tuple[Variant, ...]: The baseline, then the named variants in the order given. Raises: - KeyError: If a name is registered to no variant. + ValueError: If a name is registered to no variant. """ registered = variants(baselines) if EVERY_VARIANT in names: @@ -48,7 +48,9 @@ def selected_variants( unknown = [name for name in names if name not in registered] if unknown: - raise KeyError(f"no variant is called {', '.join(unknown)}; the registry holds {', '.join(registered)}") + raise ValueError( + f"No variant is called {', '.join(unknown)}; the variants are {', '.join(registered)} or {EVERY_VARIANT}." + ) chosen = [registered[name] for name in names if name != BASELINE_NAME] return (registered[BASELINE_NAME], *chosen) diff --git a/tests/unit/sampletones/commands/test_convert.py b/tests/unit/sampletones/commands/test_convert.py index 92078335c..f1ce34721 100644 --- a/tests/unit/sampletones/commands/test_convert.py +++ b/tests/unit/sampletones/commands/test_convert.py @@ -117,6 +117,32 @@ def test_an_unknown_channel_is_refused(self, reconstruction: RecordedReconstruct assert reconstruction.requests == [] + def test_a_missing_source_is_refused_in_one_line( + self, reconstruction: RecordedReconstruction, tmp_path: Path + ) -> None: + with pytest.raises(SystemExit) as leaving: + dispatch(COMMANDS, ["convert", str(tmp_path / "absent.wav")]) + + assert str(leaving.value) == f"No file at {tmp_path / 'absent.wav'}." + assert reconstruction.requests == [] + + def test_a_project_is_refused_as_no_recording(self, reconstruction: RecordedReconstruction, tmp_path: Path) -> None: + project = _recording(tmp_path, "song.stp") + + with pytest.raises(SystemExit, match="is no recording") as leaving: + dispatch(COMMANDS, ["convert", str(project)]) + + assert len(str(leaving.value).splitlines()) == 1 + assert reconstruction.requests == [] + + def test_a_missing_stems_file_is_refused(self, reconstruction: RecordedReconstruction, tmp_path: Path) -> None: + source = _recording(tmp_path, "song.wav") + + with pytest.raises(SystemExit, match="No stems file at"): + dispatch(COMMANDS, ["convert", str(source), "--stems", str(tmp_path / "absent.json")]) + + assert reconstruction.requests == [] + def test_channels_and_stems_exclude_each_other(self, tmp_path: Path) -> None: source = _recording(tmp_path, "song.wav") diff --git a/tests/unit/sampletones/commands/test_registry.py b/tests/unit/sampletones/commands/test_registry.py index a7374dc69..c112f5557 100644 --- a/tests/unit/sampletones/commands/test_registry.py +++ b/tests/unit/sampletones/commands/test_registry.py @@ -1,13 +1,55 @@ +import json import subprocess import sys -from typing import Final, Tuple +from importlib.util import find_spec +from typing import Final, List, Tuple + +import pytest from sampletones.commands.registry import COMMANDS, USER_COMMANDS from sampletones.dispatcher import DEFAULT_COMMAND, build_parser from sampletones_tools.registry import DEVELOPER_COMMANDS -HEAVY_PACKAGES: Final[Tuple[str, ...]] = ("PIL", "py65", "pytest", "dearpygui") -PROBE: Final[str] = "import sys, sampletones.commands.registry; print(sorted(set(sys.modules) & set(sys.argv[1:])))" +TOOLS_PACKAGE: Final[str] = "sampletones_tools" +FACE_MODULES: Final[Tuple[str, ...]] = ("command", "registry") +FACE_PACKAGE: Final[str] = "commands" +HEAVY_MODULES: Final[Tuple[str, ...]] = ( + "numpy", + "scipy", + "librosa", + "cupy", + "PIL", + "py65", + "pytest", + "dearpygui", + "sampletones_shared.paths.user", +) +PROBE: Final[str] = "import json, sys, sampletones.commands.registry; print(json.dumps(sorted(sys.modules)))" + + +def _is_face(module: str) -> bool: + """Whether a tools module is one a command list reads: a package initializer, a command, a + registry, or a module under ``commands``.""" + spec = find_spec(module) + parts = module.split(".") + return ( + (spec is not None and spec.submodule_search_locations is not None) + or parts[-1] in FACE_MODULES + or FACE_PACKAGE in parts[:-1] + ) + + +@pytest.fixture(scope="module") +def loaded_modules() -> List[str]: + """Every module a fresh interpreter holds once it lists the commands.""" + completed = subprocess.run( + [sys.executable, "-c", PROBE], + check=True, + capture_output=True, + text=True, + ) + modules: List[str] = json.loads(completed.stdout) + return modules class TestRegistry: @@ -23,13 +65,15 @@ def test_the_default_command_is_registered(self) -> None: def test_the_registry_builds_one_parser(self) -> None: assert build_parser(COMMANDS).format_help() - def test_listing_the_commands_loads_no_tool(self) -> None: - """A tool's dependency imported at module level would break every invocation, the GUI included.""" - completed = subprocess.run( - [sys.executable, "-c", PROBE, *HEAVY_PACKAGES], - check=True, - capture_output=True, - text=True, - ) - assert completed.stdout.strip() == "[]" +class TestListingTheCommands: + """A module a tool needs, loaded with the command list, would slow and could break every invocation, the GUI included.""" + + def test_only_the_faces_of_the_tools_are_loaded(self, loaded_modules: List[str]) -> None: + tools = [module for module in loaded_modules if module.startswith(f"{TOOLS_PACKAGE}.")] + + assert tools + assert [module for module in tools if not _is_face(module)] == [] + + def test_no_heavy_module_is_loaded(self, loaded_modules: List[str]) -> None: + assert set(loaded_modules) & set(HEAVY_MODULES) == set() diff --git a/tests/unit/sampletones/test_dispatcher.py b/tests/unit/sampletones/test_dispatcher.py index 8b4e9d143..c24b7cf9e 100644 --- a/tests/unit/sampletones/test_dispatcher.py +++ b/tests/unit/sampletones/test_dispatcher.py @@ -3,7 +3,7 @@ import pytest -from sampletones.dispatcher import DEFAULT_COMMAND, build_parser, dispatch +from sampletones.dispatcher import DEFAULT_COMMAND, DEVELOPER_GUIDE, build_parser, dispatch from sampletones_shared.application import SAMPLETONES_NAME_VERSION from sampletones_shared.command import Command @@ -35,6 +35,12 @@ def test_the_version_is_a_flag_of_the_entry(self, capsys: pytest.CaptureFixture[ assert leaving.value.code == 0 assert SAMPLETONES_NAME_VERSION in capsys.readouterr().out + def test_the_listing_says_where_the_developer_commands_run_and_what_lists_them(self) -> None: + listing = " ".join(build_parser((_command("first", []),)).format_help().split()) + + assert "uv run sampletones" in listing + assert DEVELOPER_GUIDE in listing + def test_every_command_is_listed_with_its_help(self) -> None: parser = build_parser((_command("first", []), _command("second", []))) diff --git a/tests/unit/sampletones_core/headless/test_conversion.py b/tests/unit/sampletones_core/headless/test_conversion.py index 76e663b52..b12ebd4a9 100644 --- a/tests/unit/sampletones_core/headless/test_conversion.py +++ b/tests/unit/sampletones_core/headless/test_conversion.py @@ -76,9 +76,13 @@ def test_a_file_holding_no_mapping_is_refused(self, tmp_path: Path) -> None: path = tmp_path / "stems.json" path.write_text("[]", encoding="utf-8") - with pytest.raises(TypeError, match="must hold a mapping"): + with pytest.raises(ValueError, match="must hold a mapping"): load_stems(path) + def test_a_missing_file_is_refused_by_its_path(self, tmp_path: Path) -> None: + with pytest.raises(ValueError, match="No stems file at"): + load_stems(tmp_path / "absent.json") + def test_a_mapping_that_is_no_setup_is_refused(self, tmp_path: Path) -> None: path = tmp_path / "stems.json" path.write_text(json.dumps({"entries": [{"id": 0, "settings": {"channels": ["pulse9"], "bends": []}}]})) @@ -138,6 +142,22 @@ def test_an_output_path_for_a_directory_is_refused(self, tmp_path: Path) -> None output_path=tmp_path / "out.stn", ) + def test_a_missing_source_is_refused_by_its_path(self, tmp_path: Path) -> None: + with pytest.raises(ValueError, match="No file at"): + ConversionRequest( + sources=(tmp_path / "absent.wav",), + stems=classic_setup(DEFAULT_CHANNELS), + output_path=None, + ) + + def test_a_file_other_than_a_recording_is_refused(self, tmp_path: Path) -> None: + with pytest.raises(ValueError, match="is no recording"): + ConversionRequest( + sources=(_recording(tmp_path, "song.stp"),), + stems=classic_setup(DEFAULT_CHANNELS), + output_path=None, + ) + def test_no_source_is_refused(self) -> None: with pytest.raises(ValueError): ConversionRequest(sources=(), stems=classic_setup(DEFAULT_CHANNELS), output_path=None) diff --git a/tests/unit/sampletones_shared/utils/test_validation.py b/tests/unit/sampletones_shared/utils/test_validation.py index fa867de34..c0ac56f3c 100644 --- a/tests/unit/sampletones_shared/utils/test_validation.py +++ b/tests/unit/sampletones_shared/utils/test_validation.py @@ -5,13 +5,14 @@ from typing import Any, Dict, List, Tuple import pytest -from pydantic import BaseModel, ConfigDict, Field, ValidationError +from pydantic import BaseModel, ConfigDict, Field, ValidationError, field_validator from sampletones_application.config.session.application.config import ApplicationConfig from sampletones_application.config.session.state.state import ApplicationState from sampletones_core.configs import Config from sampletones_shared.utils.validation import ( Location, + describe_failure, flatten_location, validate_with_recovery, ) @@ -209,6 +210,40 @@ def test_flatten(self, test_case: TestCase) -> None: assert flatten_location(test_case.location) == test_case.expected +class Refusing(BaseModel): + model_config = ConfigDict(extra="forbid") + + leaf: Leaf + count: int = Field(ge=1) + + @field_validator("count") + @classmethod + def _count_is_odd(cls, count: int) -> int: + if count % 2 == 0: + raise ValueError(f"The count {count} is even.") + + return count + + +class TestDescribeFailure: + def test_a_plain_value_error_reads_as_its_message(self) -> None: + assert describe_failure(ValueError("No file at song.wav.")) == "No file at song.wav." + + def test_a_raised_reason_keeps_its_own_words(self) -> None: + with pytest.raises(ValidationError) as failure: + Refusing(leaf=Leaf(), count=2) + + assert describe_failure(failure.value) == "The count 2 is even." + + def test_each_broken_constraint_is_one_line_naming_its_field(self) -> None: + with pytest.raises(ValidationError) as failure: + Refusing.model_validate({"leaf": {"value": 11}, "count": 0}) + + lines = describe_failure(failure.value).splitlines() + + assert [line.split(":")[0] for line in lines] == ["leaf.value", "count"] + + class TestRecoveryOnRealModels: def test_config_keeps_valid_settings_and_drops_incompatible(self) -> None: raw = { diff --git a/tests/unit/sampletones_tools/assets/test_command.py b/tests/unit/sampletones_tools/assets/test_command.py index 60c4dca77..69abe20fd 100644 --- a/tests/unit/sampletones_tools/assets/test_command.py +++ b/tests/unit/sampletones_tools/assets/test_command.py @@ -21,10 +21,6 @@ def __call__(self, directory: Path, mark: Mark) -> List[Path]: return [directory / "sampletones.svg"] -def _refuse(command: str) -> None: - raise AssertionError(f"the checkout guard ran for {command}") - - class TestIcons: def test_without_a_directory_the_shipped_icons_are_written_from_a_checkout( self, @@ -42,10 +38,16 @@ def test_without_a_directory_the_shipped_icons_are_written_from_a_checkout( assert guarded == ["icons"] assert f"Wrote {ICONS_DIRECTORY / 'sampletones.svg'}" in capsys.readouterr().out - def test_a_directory_of_its_own_needs_no_checkout(self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + def test_a_directory_of_its_own_is_written_from_a_checkout_too( + self, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + ) -> None: suite = RecordedSuite() + guarded: List[str] = [] monkeypatch.setattr(WRITER, suite) - monkeypatch.setattr(GUARD, _refuse) + monkeypatch.setattr(GUARD, guarded.append) assert dispatch(COMMANDS, ["icons", "--directory", str(tmp_path)]) == 0 assert [directory for directory, _ in suite.writes] == [tmp_path] + assert guarded == ["icons"] diff --git a/tests/unit/sampletones_tools/codec/study/report/test_run.py b/tests/unit/sampletones_tools/codec/study/report/test_run.py new file mode 100644 index 000000000..dbdf563ec --- /dev/null +++ b/tests/unit/sampletones_tools/codec/study/report/test_run.py @@ -0,0 +1,38 @@ +import subprocess +from typing import List, Sequence + +import pytest + +from sampletones_tools.codec.study.report import run +from sampletones_tools.codec.study.report.run import UNKNOWN_COMMIT, commit_hash + + +class TestCommitHash: + def test_a_copy_outside_a_checkout_records_unknown_without_asking_git( + self, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + asked: List[Sequence[str]] = [] + monkeypatch.setattr(run, "is_checkout", lambda root: False) + monkeypatch.setattr(run.subprocess, "run", lambda command, **options: asked.append(command)) + + assert commit_hash() == UNKNOWN_COMMIT + assert asked == [] + + def test_a_machine_without_git_records_unknown(self, monkeypatch: pytest.MonkeyPatch) -> None: + def missing_git(command: Sequence[str], **options: object) -> subprocess.CompletedProcess[str]: + raise FileNotFoundError(command[0]) + + monkeypatch.setattr(run, "is_checkout", lambda root: True) + monkeypatch.setattr(run.subprocess, "run", missing_git) + + assert commit_hash() == UNKNOWN_COMMIT + + def test_a_checkout_records_what_git_reports(self, monkeypatch: pytest.MonkeyPatch) -> None: + def reporting_git(command: Sequence[str], **options: object) -> subprocess.CompletedProcess[str]: + return subprocess.CompletedProcess(command, 0, stdout="abc1234\n", stderr="") + + monkeypatch.setattr(run, "is_checkout", lambda root: True) + monkeypatch.setattr(run.subprocess, "run", reporting_git) + + assert commit_hash() == "abc1234" diff --git a/tests/unit/sampletones_tools/codec/test_command.py b/tests/unit/sampletones_tools/codec/test_command.py index 9e8cc7f87..2ab8a65d8 100644 --- a/tests/unit/sampletones_tools/codec/test_command.py +++ b/tests/unit/sampletones_tools/codec/test_command.py @@ -1,5 +1,6 @@ +from dataclasses import dataclass from pathlib import Path -from typing import Final, List, Optional, Tuple +from typing import Final, List, Optional, Sequence, Tuple import pytest @@ -7,7 +8,11 @@ from sampletones.dispatcher import dispatch from sampletones_tools.codec.command import DEFAULT_LENGTHEN_SECONDS from sampletones_tools.codec.study.manifest import StudyManifest +from sampletones_tools.codec.study.variants.production import BASELINE_NAME from sampletones_tools.codec.study.variants.registry import EVERY_VARIANT +from sampletones_tools.codec.study.variants.variant import Variant +from tests.suite.base import BaseTestSuite +from tests.suite.case import BaseRegularTestCase RUNNER: Final[str] = "sampletones_tools.codec.study.session.run_study" REPORTER: Final[str] = "sampletones_tools.codec.report.session.run_report" @@ -15,10 +20,10 @@ class RecordedStudy: def __init__(self) -> None: - self.runs: List[Tuple[StudyManifest, Optional[Path]]] = [] + self.runs: List[Tuple[StudyManifest, Tuple[str, ...], Optional[Path]]] = [] - def __call__(self, manifest: StudyManifest, output: Optional[Path]) -> Path: - self.runs.append((manifest, output)) + def __call__(self, manifest: StudyManifest, variants: Sequence[Variant], output: Optional[Path]) -> Path: + self.runs.append((manifest, tuple(variant.name for variant in variants), output)) return output if output is not None else Path("run") @@ -79,10 +84,11 @@ def test_the_sources_and_the_sweep_are_read_from_the_options( ) assert status == 0 - manifest, output = study.runs[0] + manifest, variants, output = study.runs[0] assert [source.path for source in manifest.projects] == [Path("songs/one.stp")] assert [source.path for source in manifest.reconstructions] == [Path("stems/two")] assert manifest.variants == ("wide-hold",) + assert variants == (BASELINE_NAME, "wide-hold") assert manifest.lengthen_seconds == 30 assert output == tmp_path @@ -100,7 +106,7 @@ def test_the_lengthening_defaults_to_its_constant(self, monkeypatch: pytest.Monk monkeypatch.setattr(RUNNER, study) assert dispatch(COMMANDS, ["codec", "study", "--project", "songs/one.stp"]) == 0 - manifest, output = study.runs[0] + manifest, _, output = study.runs[0] assert manifest.lengthen_seconds == DEFAULT_LENGTHEN_SECONDS assert manifest.variants == (EVERY_VARIANT,) assert output is None @@ -110,3 +116,31 @@ def test_an_action_is_required(self) -> None: dispatch(COMMANDS, ["codec"]) assert leaving.value.code == 2 + + +class TestRefusedStudies(BaseTestSuite): + @dataclass(frozen=True, kw_only=True) + class TestCase(BaseRegularTestCase): + argv: Tuple[str, ...] + refusal: str + + test_cases = ( + TestCase(label="an unknown variant", argv=("--variants", "bogus"), refusal="No variant is called bogus"), + TestCase(label="no lengthening", argv=("--lengthen", "0"), refusal="lengthen_seconds: Input should be"), + TestCase(label="a missing manifest", argv=("--manifest", "absent.json"), refusal="No manifest at"), + ) + + @pytest.mark.parametrize("test_case", test_cases, ids=lambda test_case: test_case.label) + def test_a_run_asking_the_impossible_is_refused_in_one_line_before_it_starts( + self, + monkeypatch: pytest.MonkeyPatch, + test_case: TestCase, + ) -> None: + study = RecordedStudy() + monkeypatch.setattr(RUNNER, study) + + with pytest.raises(SystemExit, match=test_case.refusal) as leaving: + dispatch(COMMANDS, ["codec", "study", "--project", "songs/one.stp", *test_case.argv]) + + assert len(str(leaving.value).splitlines()) == 1 + assert study.runs == [] From a249579a7d75f5e78a482ea7f4b370a1fc20e4ef Mon Sep 17 00:00:00 2001 From: JakimPL Date: Sun, 13 Sep 2026 19:52:12 +0200 Subject: [PATCH 23/36] Shared: one table writer for every report --- src/sampletones_shared/utils/tables.py | 59 +++++++++++++++++++ src/sampletones_tools/calibration/report.py | 59 ++++++++----------- src/sampletones_tools/codec/report/rows.py | 30 ++++------ src/sampletones_tools/codec/report/session.py | 8 +-- .../codec/study/report/run.py | 46 +++++++-------- .../codec/study/report/writers.py | 42 ------------- .../sampletones_shared/utils/test_tables.py | 39 ++++++++++++ 7 files changed, 159 insertions(+), 124 deletions(-) create mode 100644 src/sampletones_shared/utils/tables.py delete mode 100644 src/sampletones_tools/codec/study/report/writers.py create mode 100644 tests/unit/sampletones_shared/utils/test_tables.py diff --git a/src/sampletones_shared/utils/tables.py b/src/sampletones_shared/utils/tables.py new file mode 100644 index 000000000..76370119c --- /dev/null +++ b/src/sampletones_shared/utils/tables.py @@ -0,0 +1,59 @@ +import csv +from pathlib import Path +from typing import Final, List, Self, Sequence, Tuple + +from pydantic import BaseModel, ConfigDict, model_validator + +MARKDOWN_RULE: Final[str] = "---" +MARKDOWN_SEPARATOR: Final[str] = " | " + + +class Table(BaseModel): + """A report's table: a header and rows of cells already written as text. + + A report builds a table once and writes it both ways, as CSV another tool reads and as the + lines of a Markdown document a person reads, so the two copies carry the same cells. + + Attributes: + columns: The header row. + rows: The rows, each with one cell per column. + """ + + model_config = ConfigDict(frozen=True) + + columns: Tuple[str, ...] + rows: Tuple[Tuple[str, ...], ...] + + @model_validator(mode="after") + def _rows_fill_the_columns(self) -> Self: + """Raises: + ValueError: If a row holds a different number of cells than there are columns. + """ + for index, row in enumerate(self.rows): + if len(row) != len(self.columns): + raise ValueError(f"Row {index} holds {len(row)} cells for {len(self.columns)} columns.") + + return self + + def write_csv(self, path: Path) -> None: + """Writes the table as UTF-8 CSV, the header first. + + Args: + path: Where the table is written. + """ + with path.open("w", encoding="utf-8", newline="") as handle: + writer = csv.writer(handle) + writer.writerow(self.columns) + writer.writerows(self.rows) + + def markdown_lines(self) -> List[str]: + """The table as the lines of a Markdown document: the header, the rule and one line per row.""" + return [ + self._markdown_line(self.columns), + "|" + "|".join(MARKDOWN_RULE for _ in self.columns) + "|", + *(self._markdown_line(row) for row in self.rows), + ] + + @staticmethod + def _markdown_line(cells: Sequence[str]) -> str: + return f"| {MARKDOWN_SEPARATOR.join(cells)} |" diff --git a/src/sampletones_tools/calibration/report.py b/src/sampletones_tools/calibration/report.py index 1f42d0f35..d104199f8 100644 --- a/src/sampletones_tools/calibration/report.py +++ b/src/sampletones_tools/calibration/report.py @@ -1,11 +1,15 @@ -import csv from collections import defaultdict from pathlib import Path -from typing import Dict, Iterable, List, Tuple +from typing import Dict, Final, Iterable, List, Tuple import numpy as np -from .runner import CalibrationRow +from sampletones_shared.utils.tables import Table +from sampletones_tools.calibration.runner import CalibrationRow + +CSV_COLUMNS: Final[Tuple[str, ...]] = ("variant", "item", "category", "referee", "score") +VARIANT_COLUMN: Final[str] = "variant" +OVERALL_COLUMN: Final[str] = "overall" def write_csv(rows: List[CalibrationRow], path: Path) -> None: @@ -16,19 +20,10 @@ def write_csv(rows: List[CalibrationRow], path: Path) -> None: rows: Scored rows from the runner. path: Target CSV path. """ - with path.open("w", newline="") as handle: - writer = csv.writer(handle) - writer.writerow(["variant", "item", "category", "referee", "score"]) - for row in rows: - writer.writerow( - [ - row.variant, - row.item, - row.category, - row.referee, - f"{row.score:.6f}", - ] - ) + Table( + columns=CSV_COLUMNS, + rows=tuple((row.variant, row.item, row.category, row.referee, f"{row.score:.6f}") for row in rows), + ).write_csv(path) def write_markdown(rows: List[CalibrationRow], path: Path) -> None: @@ -41,25 +36,23 @@ def write_markdown(rows: List[CalibrationRow], path: Path) -> None: path: Target markdown path. """ lines: List[str] = ["# Calibration report", ""] - for referee in _ordered(row.referee for row in rows): referee_rows = [row for row in rows if row.referee == referee] - categories = _ordered(row.category for row in referee_rows) - variants = _ordered(row.variant for row in referee_rows) - means = _mean_scores(referee_rows) - - lines.append(f"## {referee}") - lines.append("") - lines.append("| variant | " + " | ".join(categories) + " | overall |") - lines.append("|---" * (len(categories) + 2) + "|") - for variant in variants: - cells = [f"{means.get((variant, category), float('nan')):.3f}" for category in categories] - overall = np.mean([row.score for row in referee_rows if row.variant == variant]) - lines.append(f"| {variant} | " + " | ".join(cells) + f" | {float(overall):.3f} |") - - lines.append("") - - path.write_text("\n".join(lines)) + lines.extend((f"## {referee}", "", *_referee_table(referee_rows).markdown_lines(), "")) + + path.write_text("\n".join(lines), encoding="utf-8") + + +def _referee_table(rows: List[CalibrationRow]) -> Table: + categories = _ordered(row.category for row in rows) + means = _mean_scores(rows) + cells: List[Tuple[str, ...]] = [] + for variant in _ordered(row.variant for row in rows): + scores = [f"{means.get((variant, category), float('nan')):.3f}" for category in categories] + overall = np.mean([row.score for row in rows if row.variant == variant]) + cells.append((variant, *scores, f"{float(overall):.3f}")) + + return Table(columns=(VARIANT_COLUMN, *categories, OVERALL_COLUMN), rows=tuple(cells)) def _mean_scores(rows: List[CalibrationRow]) -> Dict[Tuple[str, str], float]: diff --git a/src/sampletones_tools/codec/report/rows.py b/src/sampletones_tools/codec/report/rows.py index c689926fe..408272fec 100644 --- a/src/sampletones_tools/codec/report/rows.py +++ b/src/sampletones_tools/codec/report/rows.py @@ -1,8 +1,9 @@ -import csv from dataclasses import dataclass from pathlib import Path from typing import Final, Sequence, Tuple +from sampletones_shared.utils.tables import Table + COLUMNS: Final[Tuple[str, ...]] = ( "corpus", "variant", @@ -81,26 +82,17 @@ def cells(self) -> Tuple[str, ...]: ) -def write_csv(rows: Sequence[ReportRow], path: Path) -> None: - """Writes the measurements as a table another tool reads. - - Args: - rows: The measurements, in the order they are reported. - path: Where the table is written. - """ - with path.open("w", encoding="utf-8", newline="") as handle: - writer = csv.writer(handle) - writer.writerow(COLUMNS) - for row in rows: - writer.writerow(row.cells) +def report_table(rows: Sequence[ReportRow]) -> Table: + """The measurements as the report's table, in the order they are reported.""" + return Table(columns=COLUMNS, rows=tuple(row.cells for row in rows)) -def write_markdown(rows: Sequence[ReportRow], path: Path, state: int) -> None: - """Writes the measurements as a table a reader reads. +def write_markdown(table: Table, path: Path, state: int) -> None: + """Writes the measurements as a document a reader reads. Args: - rows: The measurements, in the order they are reported. - path: Where the table is written. + table: The measurements. + path: Where the document is written. state: The zero-page bytes the decoder's plane state takes. """ lines = [ @@ -108,8 +100,6 @@ def write_markdown(rows: Sequence[ReportRow], path: Path, state: int) -> None: "", f"Decoder state: {state} bytes of zero page.", "", - "| " + " | ".join(COLUMNS) + " |", - "|" + "|".join("---" for _ in COLUMNS) + "|", + *table.markdown_lines(), ] - lines.extend("| " + " | ".join(row.cells) + " |" for row in rows) path.write_text("\n".join(lines) + "\n", encoding="utf-8") diff --git a/src/sampletones_tools/codec/report/session.py b/src/sampletones_tools/codec/report/session.py index e0e4d250d..19a38b902 100644 --- a/src/sampletones_tools/codec/report/session.py +++ b/src/sampletones_tools/codec/report/session.py @@ -6,7 +6,7 @@ from sampletones_player.specification.compression import PLANE_COUNT, PLANE_STATE_SIZE from sampletones_tools.codec.report.corpus import CorpusEntry, corpus_entries from sampletones_tools.codec.report.encoding import Encoding, encode_corpus, report_rows -from sampletones_tools.codec.report.rows import write_csv, write_markdown +from sampletones_tools.codec.report.rows import report_table, write_markdown from sampletones_tools.codec.report.songs import available_bytes from sampletones_tools.corpus.build import build_corpus @@ -31,12 +31,12 @@ def write_report( Returns: Tuple[Path, Path]: The CSV table and the Markdown table. """ - rows = report_rows(entries, encodings, space) + table = report_table(report_rows(entries, encodings, space)) output.mkdir(parents=True, exist_ok=True) csv_path = output / CSV_FILENAME markdown_path = output / MARKDOWN_FILENAME - write_csv(rows, csv_path) - write_markdown(rows, markdown_path, PLANE_COUNT * PLANE_STATE_SIZE) + table.write_csv(csv_path) + write_markdown(table, markdown_path, PLANE_COUNT * PLANE_STATE_SIZE) return csv_path, markdown_path diff --git a/src/sampletones_tools/codec/study/report/run.py b/src/sampletones_tools/codec/study/report/run.py index 9281528ac..4c6c07757 100644 --- a/src/sampletones_tools/codec/study/report/run.py +++ b/src/sampletones_tools/codec/study/report/run.py @@ -6,6 +6,7 @@ from sampletones_player.compression.compressed import CompressedPlanes from sampletones_shared.paths.source import REPOSITORY_ROOT from sampletones_shared.paths.user import USER_PATH_DOCUMENTS +from sampletones_shared.utils.tables import Table from sampletones_tools.checkout import is_checkout from sampletones_tools.codec.study.accounting import rows as accounting from sampletones_tools.codec.study.manifest import StudyManifest @@ -13,7 +14,6 @@ from sampletones_tools.codec.study.report import aggregate from sampletones_tools.codec.study.report import rows as songs from sampletones_tools.codec.study.report import verdicts -from sampletones_tools.codec.study.report.writers import markdown_table, write_csv from sampletones_tools.codec.study.variants.production import BASELINE_NAME from sampletones_tools.codec.study.variants.variant import Variant @@ -88,25 +88,21 @@ def write_run( accounting_rows = [ accounting.account(measurement, compressed) for measurement, compressed in _written(measurements) ] + song_table = Table(columns=songs.COLUMNS, rows=tuple(row.cells for row in song_rows)) + verdict_table = Table(columns=verdicts.COLUMNS, rows=tuple(row.cells for row in verdict_rows)) manifest.save(directory / MANIFEST_JSON) - write_csv(directory / REPORT_CSV, songs.COLUMNS, [row.cells for row in song_rows]) - write_csv(directory / VERDICTS_CSV, verdicts.COLUMNS, [row.cells for row in verdict_rows]) - write_csv(directory / ACCOUNTING_CSV, accounting.COLUMNS, [row.cells for row in accounting_rows]) - lines = _header(manifest, variants) - lines.extend(("## Variants", "", *_variants_table(variants), "")) - lines.extend( - ( - "## Verdicts", - "", - verdicts.RULE, - "", - *markdown_table(verdicts.COLUMNS, [row.cells for row in verdict_rows]), - "", - ) + song_table.write_csv(directory / REPORT_CSV) + verdict_table.write_csv(directory / VERDICTS_CSV) + Table(columns=accounting.COLUMNS, rows=tuple(row.cells for row in accounting_rows)).write_csv( + directory / ACCOUNTING_CSV ) - lines.extend(("## Groups", "", *markdown_table(aggregate.COLUMNS, [row.cells for row in group_rows]), "")) - lines.extend(("## Songs", "", *markdown_table(songs.COLUMNS, [row.cells for row in song_rows]), "")) - lines.extend(("## Accounting", "", *_accounting_table(accounting_rows), "")) + lines = _header(manifest, variants) + lines.extend(("## Variants", "", *_variants_table(variants).markdown_lines(), "")) + lines.extend(("## Verdicts", "", verdicts.RULE, "", *verdict_table.markdown_lines(), "")) + group_table = Table(columns=aggregate.COLUMNS, rows=tuple(row.cells for row in group_rows)) + lines.extend(("## Groups", "", *group_table.markdown_lines(), "")) + lines.extend(("## Songs", "", *song_table.markdown_lines(), "")) + lines.extend(("## Accounting", "", *_accounting_table(accounting_rows).markdown_lines(), "")) (directory / REPORT_MARKDOWN).write_text("\n".join(lines), encoding="utf-8") @@ -137,16 +133,16 @@ def _header( ] -def _variants_table(variants: Sequence[Variant]) -> List[str]: +def _variants_table(variants: Sequence[Variant]) -> Table: columns = ("variant", "hypothesis", "kind", "driver") - cells = [(variant.name, variant.hypothesis, variant.kind.value, variant.note) for variant in variants] - return markdown_table(columns, cells) + cells = tuple((variant.name, variant.hypothesis, variant.kind.value, variant.note) for variant in variants) + return Table(columns=columns, rows=cells) -def _accounting_table(rows: Sequence[accounting.AccountingRow]) -> List[str]: +def _accounting_table(rows: Sequence[accounting.AccountingRow]) -> Table: columns = ("group", "song", "variant", "block", "dictionary", "idle bytes", "bend bytes") labels = tuple(f"{label} {name}" for label, name in accounting.HYPOTHESES) - cells = [ + cells = tuple( ( row.group, row.song, @@ -158,5 +154,5 @@ def _accounting_table(rows: Sequence[accounting.AccountingRow]) -> List[str]: *(row.share(finding.saving) for finding in row.findings), ) for row in rows - ] - return markdown_table((*columns, *labels), cells) + ) + return Table(columns=(*columns, *labels), rows=cells) diff --git a/src/sampletones_tools/codec/study/report/writers.py b/src/sampletones_tools/codec/study/report/writers.py deleted file mode 100644 index cc4243512..000000000 --- a/src/sampletones_tools/codec/study/report/writers.py +++ /dev/null @@ -1,42 +0,0 @@ -import csv -from pathlib import Path -from typing import List, Sequence - - -def write_csv( - path: Path, - columns: Sequence[str], - rows: Sequence[Sequence[str]], -) -> None: - """Writes a table another tool reads. - - Args: - path: Where the table is written. - columns: The header row. - rows: The rows, each as its cells. - """ - with path.open("w", encoding="utf-8", newline="") as handle: - writer = csv.writer(handle) - writer.writerow(columns) - writer.writerows(rows) - - -def markdown_table( - columns: Sequence[str], - rows: Sequence[Sequence[str]], -) -> List[str]: - """A table a reader reads, as the lines of a markdown document. - - Args: - columns: The header row. - rows: The rows, each as its cells. - - Returns: - List[str]: The header, the rule and one line per row. - """ - lines = [ - "| " + " | ".join(columns) + " |", - "|" + "|".join("---" for _ in columns) + "|", - ] - lines.extend("| " + " | ".join(row) + " |" for row in rows) - return lines diff --git a/tests/unit/sampletones_shared/utils/test_tables.py b/tests/unit/sampletones_shared/utils/test_tables.py new file mode 100644 index 000000000..3147ed78e --- /dev/null +++ b/tests/unit/sampletones_shared/utils/test_tables.py @@ -0,0 +1,39 @@ +import csv +from pathlib import Path + +import pytest + +from sampletones_shared.utils.tables import Table + + +@pytest.fixture(name="table") +def table_fixture() -> Table: + return Table( + columns=("song", "bytes"), + rows=(("Theme, reprise", "120"), ("Café", "7")), + ) + + +class TestTable: + def test_the_csv_reads_back_cell_for_cell(self, table: Table, tmp_path: Path) -> None: + path = tmp_path / "report.csv" + + table.write_csv(path) + + with path.open(encoding="utf-8", newline="") as handle: + assert [tuple(row) for row in csv.reader(handle)] == [table.columns, *table.rows] + + def test_the_markdown_is_the_header_the_rule_and_a_line_per_row(self, table: Table) -> None: + assert table.markdown_lines() == [ + "| song | bytes |", + "|---|---|", + "| Theme, reprise | 120 |", + "| Café | 7 |", + ] + + def test_a_table_without_rows_is_its_header_and_rule(self) -> None: + assert Table(columns=("song",), rows=()).markdown_lines() == ["| song |", "|---|"] + + def test_a_row_short_of_the_columns_is_refused(self) -> None: + with pytest.raises(ValueError, match="Row 1 holds 1 cells for 2 columns"): + Table(columns=("song", "bytes"), rows=(("one", "1"), ("two",))) From 67ead94bdcbee1b6c38fd556e49a29e167a7c7d3 Mon Sep 17 00:00:00 2001 From: JakimPL Date: Sun, 13 Sep 2026 20:09:10 +0200 Subject: [PATCH 24/36] Tidied: the tools package after the review --- docs/development/packages.md | 7 +- docs/development/release/dependencies.md | 9 +- docs/development/tooling.md | 9 +- src/sampletones/commands/convert.py | 7 +- src/sampletones_config/boundaries/graphs.yaml | 2 +- src/sampletones_core/headless/conversion.py | 308 ------------------ .../headless/conversion/__init__.py} | 0 .../headless/conversion/console.py | 21 ++ .../headless/conversion/request.py | 144 ++++++++ .../headless/conversion/runners.py | 158 +++++++++ src/sampletones_shared/paths/source.py | 1 - src/sampletones_shared/utils/text.py | 23 +- src/sampletones_tools/assets/command.py | 10 +- src/sampletones_tools/calibration/command.py | 2 +- src/sampletones_tools/calibration/session.py | 12 +- .../checks/boundary/standalone.py | 4 +- .../checks/commands/import_boundary.py | 3 +- .../checks/commands/palette_colors.py | 8 +- src/sampletones_tools/checks/paths.py | 6 + src/sampletones_tools/checks/unused_tags.py | 3 +- src/sampletones_tools/codec/command.py | 3 +- src/sampletones_tools/codec/report/session.py | 38 ++- src/sampletones_tools/codec/report/songs.py | 4 +- .../codec/study/report/run.py | 4 +- src/sampletones_tools/codec/study/session.py | 7 +- src/sampletones_tools/corpus/build.py | 13 +- src/sampletones_tools/player/command.py | 14 +- src/sampletones_tools/runs.py | 19 ++ src/sampletones_tools/samples/emit.py | 7 +- src/sampletones_tools/samples/nsf.py | 9 +- tests/benchmarks/conftest.py | 8 +- tests/integration/conftest.py | 30 +- tests/integration/nsf/header.py | 8 + .../nsf/test_compression_report.py | 62 ++-- tests/integration/nsf/test_driver_audio.py | 4 +- tests/integration/nsf/test_driver_bend.py | 8 +- tests/integration/nsf/test_driver_trace.py | 10 +- tests/integration/nsf/test_nsf_pipeline.py | 8 +- tests/integration/samples/test_emitters.py | 28 +- tests/suite/scripts.py | 2 +- .../unit/sampletones/commands/test_convert.py | 4 +- .../headless/conversion/stems.py | 29 ++ .../headless/conversion/test_console.py | 34 ++ .../test_request.py} | 112 ++----- .../headless/conversion/test_runners.py | 47 +++ .../sampletones_shared/paths/test_source.py | 7 +- .../sampletones_shared/utils/test_text.py | 16 +- .../sampletones_tools/assets/test_command.py | 2 +- .../calibration/test_command.py | 4 +- .../calibration/test_session.py | 7 +- .../checks/boundary/configs/test_rules.py | 11 +- .../checks/boundary/test_standalone.py | 7 +- .../sampletones_tools/checks/test_paths.py | 6 + .../codec/study/test_session.py | 10 + .../sampletones_tools/codec/test_command.py | 10 +- .../sampletones_tools/player/test_command.py | 4 +- tests/unit/sampletones_tools/test_runs.py | 15 + 57 files changed, 755 insertions(+), 593 deletions(-) delete mode 100644 src/sampletones_core/headless/conversion.py rename src/{sampletones_tools/synthesis/py.typed => sampletones_core/headless/conversion/__init__.py} (100%) create mode 100644 src/sampletones_core/headless/conversion/console.py create mode 100644 src/sampletones_core/headless/conversion/request.py create mode 100644 src/sampletones_core/headless/conversion/runners.py create mode 100644 src/sampletones_tools/checks/paths.py create mode 100644 src/sampletones_tools/runs.py create mode 100644 tests/integration/nsf/header.py create mode 100644 tests/unit/sampletones_core/headless/conversion/stems.py create mode 100644 tests/unit/sampletones_core/headless/conversion/test_console.py rename tests/unit/sampletones_core/headless/{test_conversion.py => conversion/test_request.py} (53%) create mode 100644 tests/unit/sampletones_core/headless/conversion/test_runners.py create mode 100644 tests/unit/sampletones_tools/checks/test_paths.py create mode 100644 tests/unit/sampletones_tools/test_runs.py diff --git a/docs/development/packages.md b/docs/development/packages.md index b7c55a302..c48f177f0 100644 --- a/docs/development/packages.md +++ b/docs/development/packages.md @@ -31,7 +31,6 @@ graph TD TOOLS --> APP TOOLS --> PLAYER TOOLS --> CORE - TOOLS --> ASSETS TOOLS --> SHARED APP --> PLAYER APP --> CORE @@ -50,7 +49,7 @@ graph TD | `sampletones_core` | The reconstruction engine, the project model, playing a song out into instructions, and the tracker export formats | `sampletones_shared` | | `sampletones_player` | The NES player: the register model, the re-clocking schedule, the 6502 driver and the NSF file | `sampletones_shared`, `sampletones_core` | | `sampletones_application` | The DearPyGui front end | `sampletones_shared`, `sampletones_core`, `sampletones_player` | -| `sampletones_tools` | Everything a developer runs and the application does not: analytic waveform synthesis, the calibration harness, the driver toolchain and the register trace, the mark the icons are drawn from, the source checks, the synthetic corpus with its sample emitters, and the developer commands that run them | `sampletones_shared`, `sampletones_assets`, `sampletones_core`, `sampletones_player`, `sampletones_application` | +| `sampletones_tools` | Everything a developer runs and the application does not: analytic waveform synthesis, the calibration harness, the driver toolchain and the register trace, the mark the icons are drawn from, the source checks, the synthetic corpus with its sample emitters, and the developer commands that run them | `sampletones_shared`, `sampletones_core`, `sampletones_player`, `sampletones_application` | | `sampletones` | The command-line entry: the dispatcher, the commands and the startup self-check | `sampletones_shared`, `sampletones_core`, `sampletones_application`, `sampletones_tools` | Third-party imports are the package author's own choice and stand outside this table. @@ -144,6 +143,4 @@ and scripts trees and prints what they find. The scripts tree is held to a rule of its own, `boundaries/standalone.yaml`. A bootstrap script runs on the system interpreter, so it imports the standard library and the scripts tree itself, and a name in that tree that stands in for a standard-library module is reported too, since the -tree sits on the import path. The tool scripts still under `scripts/` are excluded by name until -each moves into the project, and an exclusion naming no file fails the tests, so a move takes its -exclusion with it. [Tooling](tooling.md) states the principle. +tree sits on the import path. [Tooling](tooling.md) states the principle. diff --git a/docs/development/release/dependencies.md b/docs/development/release/dependencies.md index 8570264bb..3c34a7b4e 100644 --- a/docs/development/release/dependencies.md +++ b/docs/development/release/dependencies.md @@ -92,7 +92,8 @@ whatever the driver's length, so the exporter states them from `specification/dr build holds the linker's own labels to them before it writes anything. Editing the assembly means running `uv run sampletones driver` again and committing what it writes; the driver's test suite rebuilds the sources and holds the committed image to them wherever cc65 is installed. The wheel carries the -assembled image alone, which is all an installed copy reads. +assembled image, which is what exporting reads, and the assembly sources inside the tools package, +so `sampletones driver -o DIR` assembles the driver from an installed copy too. cc65 is distributed under the zlib license, and the driver stays clear of it: the link line names our own object files and our own `nsf.cfg`, so nothing of cc65's start-up code or libraries reaches @@ -111,9 +112,9 @@ project's own terms untouched. Listening to a real APU needs [ffmpeg](https://ffmpeg.org/) carrying the `libgme` demuxer, which is a build option rather than a given: `uv run sampletones nsf render` asks the installed ffmpeg which demuxers -it holds and names this system's install command before it decodes anything. It exports the example -files and renders each one to a wave beside it, its length read out of the song block the file -carries. That is an ear rather than a gate: the register trace is what the driver answers to, and +it holds and names this system's install command before it decodes anything. `nsf samples -o DIR` +writes the example files, and `nsf render --directory DIR` renders each one to a wave beside it, its +length read out of the song block the file carries. That is an ear rather than a gate: the register trace is what the driver answers to, and the wave is what a person listens to. ### The player's tools diff --git a/docs/development/tooling.md b/docs/development/tooling.md index 5c0e9dcd6..87921a2e4 100644 --- a/docs/development/tooling.md +++ b/docs/development/tooling.md @@ -64,7 +64,10 @@ script. runs the one named; `commands/` holds one module per command and `commands/registry.py` lists them. A command is a frozen `Command` (`sampletones_shared/command.py`): its name, one line of help, the function adding its options to a parser, and the function running it over the parsed -arguments. Each command turns its arguments into a frozen record, field by field, before it works. +arguments. Each command turns its arguments into a frozen record, field by field, before it works; a +command with actions, such as `codec`, reads the action first and builds the record that action takes. +A command writing files names their directory `--output` (`-o`), and an option naming an input says +what it reads, so `--config` is a configuration file wherever it appears. | Command | What it does | |---|---| @@ -87,8 +90,8 @@ The developer commands, listed by `sampletones_tools/registry.py` and run as | `btp samples -o DIR`, `ftm samples -o DIR`, `nsf samples -o DIR` | Builds the synthetic corpus and writes it as example files: the arrangement as two Bitphase documents (at its tempo and as a groove), the arrangement as a FamiTracker module, or each sample and the arrangement as `.nsf` programs | | `codec report -o DIR` | Compresses the synthetic corpus under every layer of the codec and writes the report the format's constants are settled from, as CSV and Markdown | | `codec study [--manifest FILE] [--project FILE]... [--reconstruction PATH]... [-o DIR] [--lengthen SECONDS] [--variants LIST]` | Encodes the projects and stems it is given, or the ones a manifest names, under every candidate change to the codec and writes the sizes, the times, a verdict per candidate and the manifest that repeats the run; without `-o` the run lands under Documents/SampleToNES/compression | -| `driver [--directory DIR]` | Assembles the NES player driver with cc65 and prints the layout the build produced; without `--directory` it writes the driver the package ships, which needs a checkout | -| `icons [--directory DIR]` | Writes the icon suite from the mark, into `--directory` or over the icons the package ships. Needs a checkout | +| `driver [-o DIR]` | Assembles the NES player driver with cc65 and prints the layout the build produced; without `-o` it writes the driver the package ships, which needs a checkout | +| `icons [-o DIR]` | Writes the icon suite from the mark, into `-o` or over the icons the package ships. Needs a checkout | | `nsf render --directory DIR [--tail SECONDS]` | Renders every exported `.nsf` file in the directory to a wave beside it, through ffmpeg's libgme demuxer | ## The tools package diff --git a/src/sampletones/commands/convert.py b/src/sampletones/commands/convert.py index e396d0e9e..fd765d242 100644 --- a/src/sampletones/commands/convert.py +++ b/src/sampletones/commands/convert.py @@ -55,13 +55,14 @@ def run(arguments: Namespace) -> int: ) from sampletones_core.headless.config import load_config - from sampletones_core.headless.conversion import ( + from sampletones_core.headless.conversion.console import pairing_lines + from sampletones_core.headless.conversion.request import ( ConversionRequest, channels_named, classic_setup, load_stems, - reconstruct, ) + from sampletones_core.headless.conversion.runners import reconstruct from sampletones_shared.array import report_array_backend from sampletones_shared.utils.validation import describe_failure @@ -71,7 +72,7 @@ def run(arguments: Namespace) -> int: except ValueError as error: raise SystemExit(describe_failure(error)) from error - for line in request.pairing(): + for line in pairing_lines(request): print(line) report_array_backend() diff --git a/src/sampletones_config/boundaries/graphs.yaml b/src/sampletones_config/boundaries/graphs.yaml index 746220db7..593e11c7e 100644 --- a/src/sampletones_config/boundaries/graphs.yaml +++ b/src/sampletones_config/boundaries/graphs.yaml @@ -6,7 +6,7 @@ packages: sampletones_core: [sampletones_shared] sampletones_player: [sampletones_shared, sampletones_core] sampletones_application: [sampletones_shared, sampletones_core, sampletones_player] - sampletones_tools: [sampletones_shared, sampletones_assets, sampletones_core, sampletones_player, sampletones_application] + sampletones_tools: [sampletones_shared, sampletones_core, sampletones_player, sampletones_application] sampletones: [sampletones_shared, sampletones_core, sampletones_application, sampletones_tools] player: diff --git a/src/sampletones_core/headless/conversion.py b/src/sampletones_core/headless/conversion.py deleted file mode 100644 index fee328669..000000000 --- a/src/sampletones_core/headless/conversion.py +++ /dev/null @@ -1,308 +0,0 @@ -from pathlib import Path -from typing import Final, List, Optional, Self, Sequence, Tuple - -from pydantic import BaseModel, ConfigDict, Field, model_validator -from tqdm import tqdm - -from sampletones_core.configs import Config -from sampletones_core.constants.enums import ( - DEFAULT_CHANNELS, - ChannelName, - bending_channels, - ordered_channels, -) -from sampletones_core.headless.library import generate_library -from sampletones_core.library import InstructionLibrary -from sampletones_core.parallelization import TaskProgress, TaskStatus -from sampletones_core.reconstructions import Reconstructor -from sampletones_core.reconstructions.converter import ( - ConversionJob, - DirectoryConversion, - ReconstructionConverter, - reconstruct_job, -) -from sampletones_core.reconstructions.converter.paths import group_output_path, is_audio_file -from sampletones_core.reconstructions.progress import ReconstructionProgress -from sampletones_core.reconstructions.reconstructor.stems.configs.config import StemsConfig -from sampletones_core.reconstructions.reconstructor.stems.configs.entry import StemEntry -from sampletones_shared.logger import logger, null_logger -from sampletones_shared.paths.extensions import EXT_FILES_AUDIO -from sampletones_shared.utils.serialization import load_json - -BAR_STEPS: Final[int] = 1000 -CHANNEL_SEPARATOR: Final[str] = "," - - -def channels_named(stated: Optional[str]) -> List[ChannelName]: - """The channels a run hands out: the ones named, comma separated, or the usual three. - - Raises: - ValueError: If a name is none of the channels the hardware has. - """ - if stated is None: - return list(DEFAULT_CHANNELS) - - names = [name.strip() for name in stated.split(CHANNEL_SEPARATOR) if name.strip()] - channels: List[ChannelName] = [] - for name in names: - try: - channels.append(ChannelName(name)) - except ValueError as error: - known = ", ".join(channel.value for channel in ChannelName) - raise ValueError(f"Unknown channel {name!r}; the channels are {known}.") from error - - return channels - - -def classic_setup(channels: Sequence[ChannelName]) -> StemsConfig: - """The setup a single-source conversion runs under: one stem over the channels it was given.""" - ordered = ordered_channels(frozenset(channels)) - return StemsConfig.single_entry(ordered, bending_channels(ordered)) - - -def load_stems(path: Path) -> StemsConfig: - """The stems setup a JSON file holds, validated the way the ``.stn`` record is. - - Raises: - ValueError: If no file stands at the path, the file holds anything other than a JSON - mapping, or the mapping is no stems setup. - """ - if not path.is_file(): - raise ValueError(f"No stems file at {path}.") - - loaded = load_json(path) - if not isinstance(loaded, dict): - raise ValueError(f"Stems file {path} must hold a mapping, got {type(loaded).__name__}.") - - return StemsConfig.model_validate(loaded) - - -def describe_stem(entry: StemEntry) -> str: - """One stem as the pairing names it: its id, the channels it may occupy and the ones it bends.""" - channels = ", ".join(channel.value for channel in entry.settings.channels) - bends = ", ".join(channel.value for channel in entry.settings.bends) - bending = f", bending {bends}" if bends else "" - return f"stem {entry.id} on {channels}{bending}" - - -class ConversionRequest(BaseModel): - """What a headless conversion is asked for: the sources, the stems setup and where the result goes. - - The i-th source plays under the i-th entry of the setup. One directory stands for every - recording under it, each converted alone under the setup's one stem, into the - configuration's reconstructions directory. - - Attributes: - sources: The recordings, or one directory of them. - stems: The setup handing the channels out. - output_path: The file the reconstruction of the recordings is written to, or ``None`` - for the configuration's own directory. - """ - - model_config = ConfigDict(frozen=True) - - sources: Tuple[Path, ...] = Field(min_length=1) - stems: StemsConfig - output_path: Optional[Path] - - @property - def directory(self) -> Optional[Path]: - """The one directory the request converts file by file, or ``None`` for recordings.""" - if len(self.sources) == 1 and self.sources[0].is_dir(): - return self.sources[0] - - return None - - @model_validator(mode="after") - def _sources_are_recordings_or_one_directory(self) -> Self: - """Raises: - ValueError: If a directory stands among several sources, or a source names nothing or - a file other than a recording. - """ - if self.directory is not None: - return self - - for source in self.sources: - if source.is_dir(): - raise ValueError("Sources are recordings, or one directory alone.") - - if not source.exists(): - raise ValueError(f"No file at {source}.") - - if not is_audio_file(source): - raise ValueError(f"{source} is no recording; a source is a {', '.join(EXT_FILES_AUDIO)} file.") - - return self - - @model_validator(mode="after") - def _entries_pair_with_sources(self) -> Self: - """Raises: - ValueError: If the setup holds a different number of stems than there are sources. - """ - entries = len(self.stems.entries) - if self.directory is not None and entries != 1: - raise ValueError(f"A directory is converted file by file under one stem; the setup holds {entries}.") - - if self.directory is None and entries != len(self.sources): - raise ValueError( - f"{len(self.sources)} sources for {entries} stems; a setup pairs one stem with each source, in order." - ) - - return self - - @model_validator(mode="after") - def _output_names_the_one_file(self) -> Self: - """Raises: - ValueError: If an output path is given for a directory, whose reconstructions land in - the configuration's directory. - """ - if self.directory is not None and self.output_path is not None: - raise ValueError( - "A directory's reconstructions land in the configuration's reconstructions directory; " - "an output path names the one file recordings are mixed into." - ) - - return self - - def pairing(self) -> List[str]: - """One line per source naming the stem it plays under, in the order they pair.""" - directory = self.directory - if directory is not None: - return [f"{directory.name}/: every recording under {describe_stem(self.stems.entries[0])}"] - - return [f"{source.name}: {describe_stem(entry)}" for source, entry in zip(self.sources, self.stems.entries)] - - -def reconstruct(request: ConversionRequest, config: Config) -> None: - """Builds what the request asks for: one reconstruction of the recordings, or one per file of the directory.""" - directory = request.directory - if directory is not None: - reconstruct_directory(directory, config, request.stems) - return - - reconstruct_sources(request.sources, config, request.stems, request.output_path) - - -def reconstruct_sources( - sources: Tuple[Path, ...], - config: Config, - stems: StemsConfig, - output_path: Optional[Path], -) -> None: - """Mixes the recordings into one reconstruction and writes it, showing the progress as a bar. - - A file already standing at the output path is kept, and the run says so. - - Args: - sources: The recordings, one per stem of the setup. - config: The configuration selecting the library and the matching settings. - stems: The setup handing the channels out. - output_path: The file written, or ``None`` for the configuration's own directory. - """ - if output_path is None: - output_path = group_output_path(config, sources, stems.covered_channels) - - if output_path.exists(): - logger.info(f"Reconstruction {output_path} exists, skipping") - return - - names = ", ".join(source.name for source in sources) - logger.info(f"Starting reconstruction of {names}") - job = ConversionJob(sources=sources, stems=stems, output_path=output_path) - progress_bar = tqdm(total=BAR_STEPS, desc=f"Reconstructing {output_path.stem}", unit="step") - - def on_progress(progress: ReconstructionProgress) -> bool: - progress_bar.set_postfix_str(progress.stage) - progress_bar.update(round(progress.fraction * BAR_STEPS) - progress_bar.n) - return True - - try: - reconstruct_job((Reconstructor(config, stems.covered_channels), job, on_progress)) - finally: - progress_bar.close() - - logger.info(f"Reconstruction file saved to {output_path}") - - -def reconstruct_directory( - directory: Path, - config: Config, - stems: StemsConfig, -) -> None: - """Reconstructs every recording under the directory, each alone under the setup's stem. - - The library the configuration names is generated first where it is missing. The results - mirror the directory's tree inside the configuration's reconstructions directory. - - Args: - directory: The directory of recordings. - config: The configuration selecting the library and the matching settings. - stems: The one-stem setup every recording is converted under. - """ - library = InstructionLibrary.from_config(config) - if not library.exists(config): - logger.warning("Library does not exist for the given configuration, generating a new library") - generate_library(config) - - progress_bar = tqdm(total=0, desc=f"Reconstructing {directory.name}", unit="file") - - def on_start() -> None: - progress_bar.disable = False - logger.info(f"Starting reconstruction for directory {directory}") - - def on_completed(written: Tuple[Path, ...]) -> None: - logger.info(f"Reconstructed {len(written)} files from {directory}") - progress_bar.close() - - def on_progress( - task_status: TaskStatus, - task_progress: TaskProgress, - ) -> None: - progress_bar.disable = False - total = task_progress.total - if total and total != progress_bar.total: - progress_bar.total = total - progress_bar.refresh() - - delta = int(task_progress.completed) - int(progress_bar.n) - if delta > 0: - progress_bar.update(delta) - - if task_progress.current_item: - progress_bar.set_description(f"{directory.name}: {task_progress.current_item}") - - if task_status in ( - TaskStatus.COMPLETED, - TaskStatus.CANCELED, - TaskStatus.FAILED, - ): - progress_bar.close() - - def on_canceled() -> None: - logger.info("Reconstruction canceled by user") - progress_bar.close() - - def on_error(_exception: Exception) -> None: - progress_bar.close() - - converter = ReconstructionConverter( - config, - DirectoryConversion(directory=directory, stems=stems), - logger=null_logger, - ) - - converter.set_callbacks( - on_start=on_start, - on_completed=on_completed, - on_progress=on_progress, - on_canceled=on_canceled, - on_error=on_error, - ) - - try: - converter.start() - converter.wait() - except KeyboardInterrupt: - logger.info("Reconstruction interrupted by user") - finally: - progress_bar.close() diff --git a/src/sampletones_tools/synthesis/py.typed b/src/sampletones_core/headless/conversion/__init__.py similarity index 100% rename from src/sampletones_tools/synthesis/py.typed rename to src/sampletones_core/headless/conversion/__init__.py diff --git a/src/sampletones_core/headless/conversion/console.py b/src/sampletones_core/headless/conversion/console.py new file mode 100644 index 000000000..41b6fbd5f --- /dev/null +++ b/src/sampletones_core/headless/conversion/console.py @@ -0,0 +1,21 @@ +from typing import List + +from sampletones_core.headless.conversion.request import ConversionRequest +from sampletones_core.reconstructions.reconstructor.stems.configs.entry import StemEntry + + +def describe_stem(entry: StemEntry) -> str: + """One stem as the pairing names it: its id, the channels it may occupy and the ones it bends.""" + channels = ", ".join(channel.value for channel in entry.settings.channels) + bends = ", ".join(channel.value for channel in entry.settings.bends) + bending = f", bending {bends}" if bends else "" + return f"stem {entry.id} on {channels}{bending}" + + +def pairing_lines(request: ConversionRequest) -> List[str]: + """One line per source naming the stem it plays under, in the order they pair.""" + directory = request.directory + if directory is not None: + return [f"{directory.name}/: every recording under {describe_stem(request.stems.entries[0])}"] + + return [f"{source.name}: {describe_stem(entry)}" for source, entry in zip(request.sources, request.stems.entries)] diff --git a/src/sampletones_core/headless/conversion/request.py b/src/sampletones_core/headless/conversion/request.py new file mode 100644 index 000000000..47f816ee4 --- /dev/null +++ b/src/sampletones_core/headless/conversion/request.py @@ -0,0 +1,144 @@ +from functools import cached_property +from pathlib import Path +from typing import List, Optional, Self, Sequence, Tuple + +from pydantic import BaseModel, ConfigDict, Field, model_validator + +from sampletones_core.constants.enums import ( + DEFAULT_CHANNELS, + ChannelName, + bending_channels, + ordered_channels, +) +from sampletones_core.reconstructions.converter.paths import is_audio_file +from sampletones_core.reconstructions.reconstructor.stems.configs.config import StemsConfig +from sampletones_shared.paths.extensions import EXT_FILES_AUDIO +from sampletones_shared.utils.serialization import load_json +from sampletones_shared.utils.text import listed_items + + +def channels_named(stated: Optional[str]) -> List[ChannelName]: + """The channels a run hands out: the ones named, comma separated, or the usual three. + + Raises: + ValueError: If a name is none of the channels the hardware has. + """ + if stated is None: + return list(DEFAULT_CHANNELS) + + channels: List[ChannelName] = [] + for name in listed_items(stated): + try: + channels.append(ChannelName(name)) + except ValueError as error: + known = ", ".join(channel.value for channel in ChannelName) + raise ValueError(f"Unknown channel {name!r}; the channels are {known}.") from error + + return channels + + +def classic_setup(channels: Sequence[ChannelName]) -> StemsConfig: + """The setup a single-source conversion runs under: one stem over the channels it was given.""" + ordered = ordered_channels(frozenset(channels)) + return StemsConfig.single_entry(ordered, bending_channels(ordered)) + + +def load_stems(path: Path) -> StemsConfig: + """The stems setup a JSON file holds, validated the way the ``.stn`` record is. + + Raises: + ValueError: If no file stands at the path, the file holds anything other than a JSON + mapping, or the mapping is no stems setup. + """ + if not path.is_file(): + raise ValueError(f"No stems file at {path}.") + + loaded = load_json(path) + if not isinstance(loaded, dict): + raise ValueError(f"Stems file {path} must hold a mapping, got {type(loaded).__name__}.") + + return StemsConfig.model_validate(loaded) + + +class ConversionRequest(BaseModel): + """What a headless conversion is asked for: the sources, the stems setup and where the result goes. + + The i-th source plays under the i-th entry of the setup. One directory stands for every + recording under it, each converted alone under the setup's one stem, into the + configuration's reconstructions directory. + + Attributes: + sources: The recordings, or one directory of them. + stems: The setup handing the channels out. + output_path: The file the reconstruction of the recordings is written to, or ``None`` + for the configuration's own directory. + """ + + model_config = ConfigDict(frozen=True) + + sources: Tuple[Path, ...] = Field(min_length=1) + stems: StemsConfig + output_path: Optional[Path] + + @cached_property + def directory(self) -> Optional[Path]: + """The one directory the request converts file by file, or ``None`` for recordings. + + The sources are classified once, when the request is validated, and every later reading + answers from that. + """ + if len(self.sources) == 1 and self.sources[0].is_dir(): + return self.sources[0] + + return None + + @model_validator(mode="after") + def _sources_are_recordings_or_one_directory(self) -> Self: + """Raises: + ValueError: If a directory stands among several sources, or a source names nothing or + a file other than a recording. + """ + if self.directory is not None: + return self + + for source in self.sources: + if source.is_dir(): + raise ValueError("Sources are recordings, or one directory alone.") + + if not source.exists(): + raise ValueError(f"No file at {source}.") + + if not is_audio_file(source): + raise ValueError(f"{source} is no recording; a source is a {', '.join(EXT_FILES_AUDIO)} file.") + + return self + + @model_validator(mode="after") + def _entries_pair_with_sources(self) -> Self: + """Raises: + ValueError: If the setup holds a different number of stems than there are sources. + """ + entries = len(self.stems.entries) + if self.directory is not None and entries != 1: + raise ValueError(f"A directory is converted file by file under one stem; the setup holds {entries}.") + + if self.directory is None and entries != len(self.sources): + raise ValueError( + f"{len(self.sources)} sources for {entries} stems; a setup pairs one stem with each source, in order." + ) + + return self + + @model_validator(mode="after") + def _output_names_the_one_file(self) -> Self: + """Raises: + ValueError: If an output path is given for a directory, whose reconstructions land in + the configuration's directory. + """ + if self.directory is not None and self.output_path is not None: + raise ValueError( + "A directory's reconstructions land in the configuration's reconstructions directory; " + "an output path names the one file recordings are mixed into." + ) + + return self diff --git a/src/sampletones_core/headless/conversion/runners.py b/src/sampletones_core/headless/conversion/runners.py new file mode 100644 index 000000000..5ec8ced89 --- /dev/null +++ b/src/sampletones_core/headless/conversion/runners.py @@ -0,0 +1,158 @@ +from pathlib import Path +from typing import Final, Optional, Tuple + +from tqdm import tqdm + +from sampletones_core.configs import Config +from sampletones_core.headless.conversion.request import ConversionRequest +from sampletones_core.headless.library import generate_library +from sampletones_core.library import InstructionLibrary +from sampletones_core.parallelization import TaskProgress, TaskStatus +from sampletones_core.reconstructions import Reconstructor +from sampletones_core.reconstructions.converter import ( + ConversionJob, + DirectoryConversion, + ReconstructionConverter, + reconstruct_job, +) +from sampletones_core.reconstructions.converter.paths import group_output_path +from sampletones_core.reconstructions.progress import ReconstructionProgress +from sampletones_core.reconstructions.reconstructor.stems.configs.config import StemsConfig +from sampletones_shared.logger import logger, null_logger + +BAR_STEPS: Final[int] = 1000 + + +def reconstruct(request: ConversionRequest, config: Config) -> None: + """Builds what the request asks for: one reconstruction of the recordings, or one per file of the directory.""" + directory = request.directory + if directory is not None: + reconstruct_directory(directory, config, request.stems) + return + + reconstruct_sources(request.sources, config, request.stems, request.output_path) + + +def reconstruct_sources( + sources: Tuple[Path, ...], + config: Config, + stems: StemsConfig, + output_path: Optional[Path], +) -> None: + """Mixes the recordings into one reconstruction and writes it, showing the progress as a bar. + + A file already standing at the output path is kept, and the run says so. + + Args: + sources: The recordings, one per stem of the setup. + config: The configuration selecting the library and the matching settings. + stems: The setup handing the channels out. + output_path: The file written, or ``None`` for the configuration's own directory. + """ + if output_path is None: + output_path = group_output_path(config, sources, stems.covered_channels) + + if output_path.exists(): + logger.info(f"Reconstruction {output_path} exists, skipping") + return + + names = ", ".join(source.name for source in sources) + logger.info(f"Starting reconstruction of {names}") + job = ConversionJob(sources=sources, stems=stems, output_path=output_path) + progress_bar = tqdm(total=BAR_STEPS, desc=f"Reconstructing {output_path.stem}", unit="step") + + def on_progress(progress: ReconstructionProgress) -> bool: + progress_bar.set_postfix_str(progress.stage) + progress_bar.update(round(progress.fraction * BAR_STEPS) - progress_bar.n) + return True + + try: + reconstruct_job((Reconstructor(config, stems.covered_channels), job, on_progress)) + finally: + progress_bar.close() + + logger.info(f"Reconstruction file saved to {output_path}") + + +def reconstruct_directory( + directory: Path, + config: Config, + stems: StemsConfig, +) -> None: + """Reconstructs every recording under the directory, each alone under the setup's stem. + + The library the configuration names is generated first where it is missing. The results + mirror the directory's tree inside the configuration's reconstructions directory. + + Args: + directory: The directory of recordings. + config: The configuration selecting the library and the matching settings. + stems: The one-stem setup every recording is converted under. + """ + library = InstructionLibrary.from_config(config) + if not library.exists(config): + logger.warning("Library does not exist for the given configuration, generating a new library") + generate_library(config) + + progress_bar = tqdm(total=0, desc=f"Reconstructing {directory.name}", unit="file") + + def on_start() -> None: + progress_bar.disable = False + logger.info(f"Starting reconstruction for directory {directory}") + + def on_completed(written: Tuple[Path, ...]) -> None: + logger.info(f"Reconstructed {len(written)} files from {directory}") + progress_bar.close() + + def on_progress( + task_status: TaskStatus, + task_progress: TaskProgress, + ) -> None: + progress_bar.disable = False + total = task_progress.total + if total and total != progress_bar.total: + progress_bar.total = total + progress_bar.refresh() + + delta = int(task_progress.completed) - int(progress_bar.n) + if delta > 0: + progress_bar.update(delta) + + if task_progress.current_item: + progress_bar.set_description(f"{directory.name}: {task_progress.current_item}") + + if task_status in ( + TaskStatus.COMPLETED, + TaskStatus.CANCELED, + TaskStatus.FAILED, + ): + progress_bar.close() + + def on_canceled() -> None: + logger.info("Reconstruction canceled by user") + progress_bar.close() + + def on_error(_exception: Exception) -> None: + progress_bar.close() + + converter = ReconstructionConverter( + config, + DirectoryConversion(directory=directory, stems=stems), + logger=null_logger, + ) + + converter.set_callbacks( + on_start=on_start, + on_completed=on_completed, + on_progress=on_progress, + on_canceled=on_canceled, + on_error=on_error, + ) + + try: + converter.start() + converter.wait() + except KeyboardInterrupt: + logger.info("Reconstruction interrupted by user") + finally: + progress_bar.close() diff --git a/src/sampletones_shared/paths/source.py b/src/sampletones_shared/paths/source.py index 517910b4b..51dd98fa8 100644 --- a/src/sampletones_shared/paths/source.py +++ b/src/sampletones_shared/paths/source.py @@ -3,4 +3,3 @@ SOURCE_ROOT: Final[Path] = Path(__file__).resolve().parents[2] REPOSITORY_ROOT: Final[Path] = SOURCE_ROOT.parent -SCRIPTS_ROOT: Final[Path] = REPOSITORY_ROOT / "scripts" diff --git a/src/sampletones_shared/utils/text.py b/src/sampletones_shared/utils/text.py index 1abc9f9e8..d1954b5bc 100644 --- a/src/sampletones_shared/utils/text.py +++ b/src/sampletones_shared/utils/text.py @@ -1,9 +1,10 @@ import re -from typing import Final, Tuple, TypeAlias +from typing import Final, List, Tuple, TypeAlias NaturalSortKey: TypeAlias = Tuple[Tuple[int, str], ...] _DIGIT_RUN_PATTERN: Final[re.Pattern[str]] = re.compile(r"(\d+)") +LIST_SEPARATOR: Final[str] = "," def natural_sort_key(text: str) -> NaturalSortKey: @@ -30,3 +31,23 @@ def natural_sort_key(text: str) -> NaturalSortKey: (int(part), "") if part.isdecimal() else (0, part.casefold()) for part in _DIGIT_RUN_PATTERN.split(text) ) return tokens + ((0, text),) + + +def listed_items(stated: str) -> List[str]: + """ + Reads a comma-separated list the way a command line states one. + + Each item keeps its own words with the spaces around it stripped, and an empty item, as a + trailing comma leaves, is dropped. + + Args: + stated: The list as written. + + Returns: + The items, in the order written. + + Examples: + >>> listed_items("pulse1, triangle,,noise,") + ['pulse1', 'triangle', 'noise'] + """ + return [item.strip() for item in stated.split(LIST_SEPARATOR) if item.strip()] diff --git a/src/sampletones_tools/assets/command.py b/src/sampletones_tools/assets/command.py index 197a27e55..85c9cffc0 100644 --- a/src/sampletones_tools/assets/command.py +++ b/src/sampletones_tools/assets/command.py @@ -7,18 +7,18 @@ NAME: Final[str] = "icons" HELP: Final[str] = "write the application icon suite from the mark" -DIRECTORY_HELP: Final[str] = "the directory receiving the icon files; without it, the icons the package ships" +OUTPUT_HELP: Final[str] = "the directory receiving the icon files; without it, the icons the package ships" @dataclass(frozen=True) class IconsArguments: """What an icon suite run is given: where the files go, if anywhere but the package.""" - directory: Optional[Path] + output: Optional[Path] def configure(parser: ArgumentParser) -> None: - parser.add_argument("--directory", type=Path, default=None, help=DIRECTORY_HELP) + parser.add_argument("--output", "-o", type=Path, default=None, help=OUTPUT_HELP) def run(arguments: Namespace) -> int: @@ -30,7 +30,7 @@ def run(arguments: Namespace) -> int: Raises: SystemExit: If the run happens outside a checkout. """ - given = IconsArguments(directory=arguments.directory) + given = IconsArguments(output=arguments.output) from sampletones_tools.checkout import require_checkout @@ -40,7 +40,7 @@ def run(arguments: Namespace) -> int: from sampletones_tools.assets.mark.suite import write_icon_suite from sampletones_tools.assets.paths import ICONS_DIRECTORY - for path in write_icon_suite(given.directory if given.directory is not None else ICONS_DIRECTORY, Mark.load()): + for path in write_icon_suite(given.output if given.output is not None else ICONS_DIRECTORY, Mark.load()): print(f"Wrote {path}") return 0 diff --git a/src/sampletones_tools/calibration/command.py b/src/sampletones_tools/calibration/command.py index fec1b19c5..237eb1208 100644 --- a/src/sampletones_tools/calibration/command.py +++ b/src/sampletones_tools/calibration/command.py @@ -58,7 +58,7 @@ def run(arguments: Namespace) -> int: ) from sampletones_core.headless.config import load_config - from sampletones_core.headless.conversion import channels_named + from sampletones_core.headless.conversion.request import channels_named from sampletones_shared.utils.validation import describe_failure from sampletones_tools.calibration.session import ( BASE_BLEND, diff --git a/src/sampletones_tools/calibration/session.py b/src/sampletones_tools/calibration/session.py index 87155b373..ec37c02d4 100644 --- a/src/sampletones_tools/calibration/session.py +++ b/src/sampletones_tools/calibration/session.py @@ -1,4 +1,3 @@ -from datetime import UTC, datetime from pathlib import Path from typing import Final, List, Optional, Sequence, Tuple @@ -8,22 +7,22 @@ from sampletones_core.constants.enums import ChannelName, SpectrumMethod from sampletones_shared.logger import logger from sampletones_shared.paths.user import USER_PATH_DOCUMENTS +from sampletones_shared.utils.text import listed_items from sampletones_tools.calibration.config.corpus import CorpusConfig from sampletones_tools.calibration.corpus.synthesis import build_corpus from sampletones_tools.calibration.corpus.writer import write_corpus from sampletones_tools.calibration.referee.factory import build_referees from sampletones_tools.calibration.report import write_csv, write_markdown from sampletones_tools.calibration.runner import build_variants, evaluate_variants +from sampletones_tools.runs import stamped_run_directory DEFAULT_METHODS: Final[Tuple[SpectrumMethod, ...]] = (SpectrumMethod.FFT, SpectrumMethod.CQT) DEFAULT_PERCEPTUAL_EXPONENTS: Final[Tuple[float, ...]] = (1.0,) BASE_BLEND: Final[Tuple[float, ...]] = () OUTPUT_DIRECTORY: Final[str] = "calibration" -RUN_STAMP: Final[str] = "run-%Y%m%d-%H%M%S" CORPUS_DIRECTORY: Final[str] = "corpus" CSV_REPORT: Final[str] = "report.csv" MARKDOWN_REPORT: Final[str] = "report.md" -LIST_SEPARATOR: Final[str] = "," def methods_named(stated: Optional[str]) -> List[SpectrumMethod]: @@ -35,9 +34,8 @@ def methods_named(stated: Optional[str]) -> List[SpectrumMethod]: if stated is None: return list(DEFAULT_METHODS) - names = [name.strip() for name in stated.split(LIST_SEPARATOR) if name.strip()] methods: List[SpectrumMethod] = [] - for name in names: + for name in listed_items(stated): try: methods.append(SpectrumMethod(name)) except ValueError as error: @@ -57,7 +55,7 @@ def floats_named(stated: Optional[str], default: Sequence[float]) -> List[float] return list(default) values: List[float] = [] - for value in (piece.strip() for piece in stated.split(LIST_SEPARATOR) if piece.strip()): + for value in listed_items(stated): try: values.append(float(value)) except ValueError as error: @@ -68,7 +66,7 @@ def floats_named(stated: Optional[str], default: Sequence[float]) -> List[float] def default_output() -> Path: """A timestamped run directory under the user's calibration documents.""" - return USER_PATH_DOCUMENTS / OUTPUT_DIRECTORY / datetime.now(UTC).strftime(RUN_STAMP) + return stamped_run_directory(USER_PATH_DOCUMENTS / OUTPUT_DIRECTORY) class CalibrationRequest(BaseModel): diff --git a/src/sampletones_tools/checks/boundary/standalone.py b/src/sampletones_tools/checks/boundary/standalone.py index eb558ecf9..bd7ab4b1c 100644 --- a/src/sampletones_tools/checks/boundary/standalone.py +++ b/src/sampletones_tools/checks/boundary/standalone.py @@ -46,7 +46,6 @@ class StandaloneRule(BaseModel): Attributes: pattern: Glob naming the scripts the rule reaches, written against the tree's root. - excluding: Globs naming the scripts the rule leaves to the project environment. reserved: Names the tree keeps clear of beyond the standard library's. message: What the rule holds, printed where a script imports past it. """ @@ -54,7 +53,6 @@ class StandaloneRule(BaseModel): model_config = ConfigDict(extra="forbid", frozen=True) pattern: str - excluding: Tuple[str, ...] = () reserved: Tuple[str, ...] = () message: str @@ -140,7 +138,7 @@ def check_standalone( local = local_names(tree) violations: List[Violation] = [] for rule in rules: - paths = rule_modules(tree, rule.pattern, rule.excluding, swept, selection) + paths = rule_modules(tree, rule.pattern, (), swept, selection) violations.extend(rule.shadowing(tree, paths)) violations.extend(violation for path in paths for violation in rule.violations(path, local)) diff --git a/src/sampletones_tools/checks/commands/import_boundary.py b/src/sampletones_tools/checks/commands/import_boundary.py index b4bf78188..285a71f95 100644 --- a/src/sampletones_tools/checks/commands/import_boundary.py +++ b/src/sampletones_tools/checks/commands/import_boundary.py @@ -39,8 +39,9 @@ def run(arguments: Namespace) -> int: scripts=arguments.scripts, ) - from sampletones_shared.paths.source import SCRIPTS_ROOT, SOURCE_ROOT + from sampletones_shared.paths.source import SOURCE_ROOT from sampletones_tools.checks.import_boundary import check_imports, report + from sampletones_tools.checks.paths import SCRIPTS_ROOT selection = None if given.everything else {path.resolve() for path in given.files} violations = check_imports( diff --git a/src/sampletones_tools/checks/commands/palette_colors.py b/src/sampletones_tools/checks/commands/palette_colors.py index 588ddde2c..22c495e63 100644 --- a/src/sampletones_tools/checks/commands/palette_colors.py +++ b/src/sampletones_tools/checks/commands/palette_colors.py @@ -19,13 +19,13 @@ class PaletteColorsArguments: """What a palette check is given: the package, the configuration and the palettes.""" package: Optional[Path] - config: Optional[Path] + config_directory: Optional[Path] palettes: Optional[Path] def configure(parser: ArgumentParser) -> None: parser.add_argument("--package", type=Path, default=None, help=PACKAGE_HELP) - parser.add_argument("--config", type=Path, default=None, help=CONFIG_HELP) + parser.add_argument("--config-directory", type=Path, default=None, help=CONFIG_HELP) parser.add_argument("--palettes", type=Path, default=None, help=PALETTES_HELP) @@ -33,7 +33,7 @@ def run(arguments: Namespace) -> int: """Reports every color the application stores resolved or the configuration writes out.""" given = PaletteColorsArguments( package=arguments.package, - config=arguments.config, + config_directory=arguments.config_directory, palettes=arguments.palettes, ) @@ -47,7 +47,7 @@ def run(arguments: Namespace) -> int: findings = check_colors( given.package if given.package is not None else APPLICATION_PACKAGE, - given.config if given.config is not None else CONFIG_DIRECTORY, + given.config_directory if given.config_directory is not None else CONFIG_DIRECTORY, given.palettes if given.palettes is not None else PALETTES_DIRECTORY, ) return report(findings) diff --git a/src/sampletones_tools/checks/paths.py b/src/sampletones_tools/checks/paths.py new file mode 100644 index 000000000..becbb4168 --- /dev/null +++ b/src/sampletones_tools/checks/paths.py @@ -0,0 +1,6 @@ +from pathlib import Path +from typing import Final + +from sampletones_shared.paths.source import REPOSITORY_ROOT + +SCRIPTS_ROOT: Final[Path] = REPOSITORY_ROOT / "scripts" diff --git a/src/sampletones_tools/checks/unused_tags.py b/src/sampletones_tools/checks/unused_tags.py index dd8acecb6..8c0287de2 100755 --- a/src/sampletones_tools/checks/unused_tags.py +++ b/src/sampletones_tools/checks/unused_tags.py @@ -3,7 +3,8 @@ from pathlib import Path from typing import Dict, Final, List, NamedTuple, Sequence, Tuple -from sampletones_shared.paths.source import REPOSITORY_ROOT, SCRIPTS_ROOT, SOURCE_ROOT +from sampletones_shared.paths.source import REPOSITORY_ROOT, SOURCE_ROOT +from sampletones_tools.checks.paths import SCRIPTS_ROOT from sampletones_tools.checks.source.constants import module_constants from sampletones_tools.checks.source.modules import SourceModule, discover_modules from sampletones_tools.checks.source.packages import package_directory diff --git a/src/sampletones_tools/codec/command.py b/src/sampletones_tools/codec/command.py index 830f230d2..6e029b369 100644 --- a/src/sampletones_tools/codec/command.py +++ b/src/sampletones_tools/codec/command.py @@ -92,7 +92,8 @@ def run(arguments: Namespace) -> int: def _report(given: ReportArguments) -> int: from sampletones_tools.codec.report.session import run_report - for path in run_report(given.output): + report = run_report(given.output) + for path in (report.csv_path, report.markdown_path): print(f"Wrote {path}") return 0 diff --git a/src/sampletones_tools/codec/report/session.py b/src/sampletones_tools/codec/report/session.py index 19a38b902..5a8d66019 100644 --- a/src/sampletones_tools/codec/report/session.py +++ b/src/sampletones_tools/codec/report/session.py @@ -1,5 +1,5 @@ +from dataclasses import dataclass from pathlib import Path -from tempfile import TemporaryDirectory from typing import Final, Sequence, Tuple from sampletones_player.driver.image import DriverImage @@ -8,12 +8,29 @@ from sampletones_tools.codec.report.encoding import Encoding, encode_corpus, report_rows from sampletones_tools.codec.report.rows import report_table, write_markdown from sampletones_tools.codec.report.songs import available_bytes -from sampletones_tools.corpus.build import build_corpus +from sampletones_tools.corpus.build import build_synthetic_corpus CSV_FILENAME: Final[str] = "report.csv" MARKDOWN_FILENAME: Final[str] = "report.md" +@dataclass(frozen=True) +class CompressionReport: + """What a report run measured, and the tables it wrote. + + Attributes: + entries: The songs measured. + encodings: Every song compressed under every variant of the codec. + csv_path: The table another tool reads. + markdown_path: The table a reader reads. + """ + + entries: Tuple[CorpusEntry, ...] + encodings: Tuple[Encoding, ...] + csv_path: Path + markdown_path: Path + + def write_report( entries: Sequence[CorpusEntry], encodings: Sequence[Encoding], @@ -40,17 +57,22 @@ def write_report( return csv_path, markdown_path -def run_report(output: Path) -> Tuple[Path, Path]: +def run_report(output: Path) -> CompressionReport: """Builds the synthetic corpus, compresses it under every variant and writes the report. Args: output: The directory the report is written into. Returns: - Tuple[Path, Path]: The CSV table and the Markdown table. + CompressionReport: The songs, their encodings and the tables written. """ - with TemporaryDirectory() as recordings: - corpus = build_corpus(Path(recordings)) - + corpus = build_synthetic_corpus() entries = corpus_entries(corpus.catalog, corpus.project) - return write_report(entries, encode_corpus(entries), available_bytes(DriverImage.load()), output) + encodings = encode_corpus(entries) + csv_path, markdown_path = write_report(entries, encodings, available_bytes(DriverImage.load()), output) + return CompressionReport( + entries=entries, + encodings=encodings, + csv_path=csv_path, + markdown_path=markdown_path, + ) diff --git a/src/sampletones_tools/codec/report/songs.py b/src/sampletones_tools/codec/report/songs.py index db464f778..01499c2cb 100644 --- a/src/sampletones_tools/codec/report/songs.py +++ b/src/sampletones_tools/codec/report/songs.py @@ -15,8 +15,8 @@ def available_bytes(driver_image: DriverImage) -> int: def lengthened(project: Project, frames: int) -> Project: """``project`` with its order repeated to ``frames`` positions, over the same samples. - The song is copied rather than edited so the session's own project keeps the arrangement - every other case reads. + The lengthened song is a copy of its own, so ``project`` keeps its arrangement for every other + reader. """ longer = Project.create( rows_per_pattern=project.song.rows_per_pattern, diff --git a/src/sampletones_tools/codec/study/report/run.py b/src/sampletones_tools/codec/study/report/run.py index 4c6c07757..d7e963b3f 100644 --- a/src/sampletones_tools/codec/study/report/run.py +++ b/src/sampletones_tools/codec/study/report/run.py @@ -16,9 +16,9 @@ from sampletones_tools.codec.study.report import verdicts from sampletones_tools.codec.study.variants.production import BASELINE_NAME from sampletones_tools.codec.study.variants.variant import Variant +from sampletones_tools.runs import stamped_run_directory DEFAULT_OUTPUT_ROOT: Final[Path] = USER_PATH_DOCUMENTS / "compression" -RUN_STAMP: Final[str] = "run-%Y%m%d-%H%M%S" REPORT_CSV: Final[str] = "report.csv" REPORT_MARKDOWN: Final[str] = "report.md" ACCOUNTING_CSV: Final[str] = "accounting.csv" @@ -36,7 +36,7 @@ def run_directory(output: Optional[Path]) -> Path: Returns: Path: The directory. """ - directory = output or DEFAULT_OUTPUT_ROOT / datetime.now(UTC).strftime(RUN_STAMP) + directory = output or stamped_run_directory(DEFAULT_OUTPUT_ROOT) directory.mkdir(parents=True, exist_ok=True) return directory diff --git a/src/sampletones_tools/codec/study/session.py b/src/sampletones_tools/codec/study/session.py index 8535768c8..61935f1de 100644 --- a/src/sampletones_tools/codec/study/session.py +++ b/src/sampletones_tools/codec/study/session.py @@ -1,7 +1,8 @@ from pathlib import Path -from typing import Final, List, Optional, Sequence, Tuple +from typing import List, Optional, Sequence, Tuple from sampletones_shared.logger import logger +from sampletones_shared.utils.text import listed_items from sampletones_tools.codec.study.corpus.build import build_corpus from sampletones_tools.codec.study.manifest import NO_SOURCE, StudyManifest, StudySource from sampletones_tools.codec.study.measure import Measurement, measure @@ -11,15 +12,13 @@ from sampletones_tools.codec.study.variants.strategy import STRATEGY_ORDER, depth_measurements from sampletones_tools.codec.study.variants.variant import Variant -LIST_SEPARATOR: Final[str] = "," - def variant_names(stated: Optional[str]) -> Tuple[str, ...]: """The variants a run encodes under: the ones named, comma separated, or every one.""" if stated is None: return (EVERY_VARIANT,) - return tuple(name.strip() for name in stated.split(LIST_SEPARATOR) if name.strip()) + return tuple(listed_items(stated)) def resolve_manifest( diff --git a/src/sampletones_tools/corpus/build.py b/src/sampletones_tools/corpus/build.py index fc1c8f59f..1eb460bf4 100644 --- a/src/sampletones_tools/corpus/build.py +++ b/src/sampletones_tools/corpus/build.py @@ -1,10 +1,11 @@ from dataclasses import dataclass +from pathlib import Path +from tempfile import TemporaryDirectory from typing import Dict from sampletones_core.project.project import Project from sampletones_core.project.settings import ProjectSettings from sampletones_core.project.voices.sample import Sample -from sampletones_shared.types.path import Pathlike from sampletones_tools.corpus.catalog import CatalogSpec, build_catalog from sampletones_tools.corpus.module import ModuleConfig from sampletones_tools.corpus.song import SongSpec, build_song @@ -47,16 +48,18 @@ def build_project( return project -def build_corpus(tmp_dir: Pathlike) -> Corpus: +def build_synthetic_corpus() -> Corpus: """Renders, reconstructs and arranges the corpus the package describes. - Args: - tmp_dir: Where the rendered recordings are written before they are reconstructed. + The recordings are rendered into a temporary directory of the build's own, which is gone once + their reconstructions are made. Returns: Corpus: The samples and the arrangement. """ - catalog = build_catalog(CatalogSpec.load(), SynthConfig.load(), tmp_dir=tmp_dir) + with TemporaryDirectory() as recordings: + catalog = build_catalog(CatalogSpec.load(), SynthConfig.load(), tmp_dir=Path(recordings)) + return Corpus( catalog=catalog, project=build_project(catalog, ModuleConfig.load(), SongSpec.load()), diff --git a/src/sampletones_tools/player/command.py b/src/sampletones_tools/player/command.py index bd4d65807..620d4ccb4 100644 --- a/src/sampletones_tools/player/command.py +++ b/src/sampletones_tools/player/command.py @@ -7,18 +7,18 @@ NAME: Final[str] = "driver" HELP: Final[str] = "assemble the NES player driver with cc65" -DIRECTORY_HELP: Final[str] = "the directory receiving the assembled driver; without it, the driver the package ships" +OUTPUT_HELP: Final[str] = "the directory receiving the assembled driver; without it, the driver the package ships" @dataclass(frozen=True) class DriverArguments: """What a driver build is given: where the image goes, if anywhere but the package.""" - directory: Optional[Path] + output: Optional[Path] def configure(parser: ArgumentParser) -> None: - parser.add_argument("--directory", type=Path, default=None, help=DIRECTORY_HELP) + parser.add_argument("--output", "-o", type=Path, default=None, help=OUTPUT_HELP) def run(arguments: Namespace) -> int: @@ -27,13 +27,13 @@ def run(arguments: Namespace) -> int: Writing the driver the package ships needs a checkout, since that is where the package is. Raises: - SystemExit: If the build runs outside a checkout without a directory of its own, or fails. + SystemExit: If the build runs outside a checkout without an output of its own, or fails. """ - given = DriverArguments(directory=arguments.directory) + given = DriverArguments(output=arguments.output) from sampletones_tools.checkout import require_checkout - if given.directory is None: + if given.output is None: require_checkout(NAME) from sampletones_shared.exceptions.player import DriverBuildError @@ -42,7 +42,7 @@ def run(arguments: Namespace) -> int: from sampletones_tools.player.assembler.report import layout_lines try: - image = build_driver(given.directory if given.directory is not None else BINARY_DIRECTORY) + image = build_driver(given.output if given.output is not None else BINARY_DIRECTORY) except DriverBuildError as error: raise SystemExit(str(error)) from error diff --git a/src/sampletones_tools/runs.py b/src/sampletones_tools/runs.py new file mode 100644 index 000000000..25e2a5b3e --- /dev/null +++ b/src/sampletones_tools/runs.py @@ -0,0 +1,19 @@ +from datetime import UTC, datetime +from pathlib import Path +from typing import Final + +RUN_STAMP: Final[str] = "run-%Y%m%d-%H%M%S" + + +def stamped_run_directory(root: Path) -> Path: + """The directory a measurement run lands in under ``root``, named by the moment it starts. + + The stamp is UTC and sorts by name, so the runs under one root read in the order they ran. + + Args: + root: The directory the runs of one measurement share. + + Returns: + Path: The run's directory, which the run creates. + """ + return root / datetime.now(UTC).strftime(RUN_STAMP) diff --git a/src/sampletones_tools/samples/emit.py b/src/sampletones_tools/samples/emit.py index 18e05ad3e..d334d1255 100644 --- a/src/sampletones_tools/samples/emit.py +++ b/src/sampletones_tools/samples/emit.py @@ -1,8 +1,7 @@ from pathlib import Path -from tempfile import TemporaryDirectory from typing import Callable, List -from sampletones_tools.corpus.build import Corpus, build_corpus +from sampletones_tools.corpus.build import Corpus, build_synthetic_corpus Emitter = Callable[[Corpus, Path], List[Path]] @@ -17,8 +16,6 @@ def emit_samples(output: Path, emitter: Emitter) -> List[Path]: Returns: List[Path]: The files the emitter wrote. """ - with TemporaryDirectory() as recordings: - corpus = build_corpus(Path(recordings)) - + corpus = build_synthetic_corpus() output.mkdir(parents=True, exist_ok=True) return emitter(corpus, output) diff --git a/src/sampletones_tools/samples/nsf.py b/src/sampletones_tools/samples/nsf.py index b7016e966..ca577b01c 100644 --- a/src/sampletones_tools/samples/nsf.py +++ b/src/sampletones_tools/samples/nsf.py @@ -10,13 +10,12 @@ from sampletones_shared.paths.extensions import EXT_FILE_NSF from sampletones_tools.corpus.build import Corpus -ARTIST: Final[str] = "Integration" SONG_NAME: Final[str] = "song" -def exported_information(name: str) -> NSFInformation: - """The header text an exported sample carries.""" - return NSFInformation(title=name, artist=ARTIST) +def exported_information(name: str, author: str) -> NSFInformation: + """The header text an exported sample carries: its own name, by the author of the arrangement.""" + return NSFInformation(title=name, artist=author) def write_samples(corpus: Corpus, output: Path) -> List[Path]: @@ -36,7 +35,7 @@ def write_samples(corpus: Corpus, output: Path) -> List[Path]: write_nsf( destination, song_from_reconstruction(sample.reconstruction, loop_tick=None), - exported_information(name), + exported_information(name, corpus.project.info.author), image, ) written.append(destination) diff --git a/tests/benchmarks/conftest.py b/tests/benchmarks/conftest.py index 3c2949f8d..aaa5d2942 100644 --- a/tests/benchmarks/conftest.py +++ b/tests/benchmarks/conftest.py @@ -1,15 +1,11 @@ from tests.integration.conftest import ( - audio_directory, instrument_catalog, integration_project, - module_config, - synth_config, + synthetic_corpus, ) __all__ = [ - "audio_directory", "instrument_catalog", "integration_project", - "module_config", - "synth_config", + "synthetic_corpus", ] diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 8776d3dc6..9b5566dac 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -1,37 +1,23 @@ -from pathlib import Path from typing import Dict import pytest from sampletones_core.project.project import Project from sampletones_core.project.voices.sample import Sample -from sampletones_tools.corpus.build import build_project -from sampletones_tools.corpus.catalog import CatalogSpec, build_catalog -from sampletones_tools.corpus.module import ModuleConfig -from sampletones_tools.corpus.song import SongSpec -from sampletones_tools.corpus.synth import SynthConfig +from sampletones_tools.corpus.build import Corpus, build_synthetic_corpus @pytest.fixture(scope="session") -def audio_directory(tmp_path_factory: pytest.TempPathFactory) -> Path: - return tmp_path_factory.mktemp("integration_audio") +def synthetic_corpus() -> Corpus: + """The corpus the emitters and the report are built from, built once for the session.""" + return build_synthetic_corpus() @pytest.fixture(scope="session") -def synth_config() -> SynthConfig: - return SynthConfig.load() +def instrument_catalog(synthetic_corpus: Corpus) -> Dict[str, Sample]: + return synthetic_corpus.catalog @pytest.fixture(scope="session") -def module_config() -> ModuleConfig: - return ModuleConfig.load() - - -@pytest.fixture(scope="session") -def instrument_catalog(audio_directory: Path, synth_config: SynthConfig) -> Dict[str, Sample]: - return build_catalog(CatalogSpec.load(), synth_config, tmp_dir=audio_directory) - - -@pytest.fixture(scope="session") -def integration_project(instrument_catalog: Dict[str, Sample], module_config: ModuleConfig) -> Project: - return build_project(instrument_catalog, module_config, SongSpec.load()) +def integration_project(synthetic_corpus: Corpus) -> Project: + return synthetic_corpus.project diff --git a/tests/integration/nsf/header.py b/tests/integration/nsf/header.py new file mode 100644 index 000000000..6f93dbf76 --- /dev/null +++ b/tests/integration/nsf/header.py @@ -0,0 +1,8 @@ +from sampletones_player.nsf.information import NSFInformation +from sampletones_tools.corpus.module import ModuleConfig +from sampletones_tools.samples.nsf import exported_information + + +def sample_information(name: str) -> NSFInformation: + """The header text ``nsf samples`` gives a corpus sample: its name, by the corpus's author.""" + return exported_information(name, ModuleConfig.load().author) diff --git a/tests/integration/nsf/test_compression_report.py b/tests/integration/nsf/test_compression_report.py index 1f446d38c..ad0d08041 100644 --- a/tests/integration/nsf/test_compression_report.py +++ b/tests/integration/nsf/test_compression_report.py @@ -1,41 +1,43 @@ +import csv from math import ceil -from pathlib import Path -from typing import Dict, Tuple +from typing import Dict, List, Tuple import pytest -from sampletones_core.project.project import Project -from sampletones_core.project.voices.sample import Sample from sampletones_player.compression.decode import decode_planes from sampletones_player.compression.planes.rebuild import streams_from_planes from sampletones_player.driver.image import DriverImage from sampletones_player.specification.compression import MAX_LITERAL_BYTES from sampletones_player.specification.song import SONG_HEADER_SIZE -from sampletones_tools.codec.report.corpus import LONG_ARRANGEMENT, CorpusEntry, corpus_entries +from sampletones_shared.utils.tables import Table +from sampletones_tools.codec.report import session +from sampletones_tools.codec.report.corpus import LONG_ARRANGEMENT from sampletones_tools.codec.report.encoding import ( LITERALS, + PLANE_VARIANTS, + RECORDS, + REGISTER_PLANES, SEARCH, + SPLIT_CONTROL, Encoding, - encode_corpus, - report_rows, ) -from sampletones_tools.codec.report.session import write_report +from sampletones_tools.codec.report.session import CompressionReport, run_report from sampletones_tools.codec.report.songs import available_bytes +from sampletones_tools.corpus.build import Corpus @pytest.fixture(scope="module") -def corpus( - instrument_catalog: Dict[str, Sample], - integration_project: Project, -) -> Tuple[CorpusEntry, ...]: - """The songs the report measures: each sample alone, the arrangement at two lengths, and a dense minute.""" - return corpus_entries(instrument_catalog, integration_project) +def report(synthetic_corpus: Corpus, tmp_path_factory: pytest.TempPathFactory) -> CompressionReport: + """The report ``codec report`` writes, over the session's corpus in place of a build of its own.""" + with pytest.MonkeyPatch.context() as monkeypatch: + monkeypatch.setattr(session, "build_synthetic_corpus", lambda: synthetic_corpus) + return run_report(tmp_path_factory.mktemp("compression_report")) @pytest.fixture(scope="module") -def encodings(corpus: Tuple[CorpusEntry, ...]) -> Tuple[Encoding, ...]: +def encodings(report: CompressionReport) -> Tuple[Encoding, ...]: """Every corpus song compressed under every variant of the codec.""" - return encode_corpus(corpus) + return report.encodings class TestTheCodecAnswersWithTheSongItWasGiven: @@ -109,15 +111,19 @@ def test_every_layer_undercuts_a_record_per_tick( class TestTheReportStatesWhatEachLayerSaves: """The measurements the format's constants are settled from.""" - def test_the_report_is_written( - self, - corpus: Tuple[CorpusEntry, ...], - encodings: Tuple[Encoding, ...], - driver_image: DriverImage, - tmp_path: Path, - ) -> None: - space = available_bytes(driver_image) - csv_path, markdown_path = write_report(corpus, encodings, space, tmp_path) - rows = report_rows(corpus, encodings, space) - assert csv_path.read_text(encoding="utf-8").count("\n") == len(rows) + 1 - assert markdown_path.exists() + def test_the_report_holds_the_baselines_then_every_variant_per_song(self, report: CompressionReport) -> None: + with report.csv_path.open(encoding="utf-8", newline="") as handle: + header, *rows = list(csv.reader(handle)) + + variants: Dict[str, List[str]] = {} + for row in rows: + variants.setdefault(row[header.index("corpus")], []).append(row[header.index("variant")]) + + assert list(variants) == [entry.name for entry in report.entries] + assert all( + listed == [RECORDS, REGISTER_PLANES, SPLIT_CONTROL, *(name for name, _ in PLANE_VARIANTS)] + for listed in variants.values() + ) + table = Table(columns=tuple(header), rows=tuple(tuple(row) for row in rows)).markdown_lines() + document = report.markdown_path.read_text(encoding="utf-8").splitlines() + assert document[document.index(table[0]) : document.index(table[0]) + len(table)] == table diff --git a/tests/integration/nsf/test_driver_audio.py b/tests/integration/nsf/test_driver_audio.py index fabd72ac7..14efacc1c 100644 --- a/tests/integration/nsf/test_driver_audio.py +++ b/tests/integration/nsf/test_driver_audio.py @@ -11,9 +11,9 @@ from sampletones_core.timers.utils import get_timer_table from sampletones_player.builder import song_from_reconstruction from sampletones_player.registers.playable import playable -from sampletones_tools.samples.nsf import exported_information from tests.integration.nsf.console.instructions import instructions_from_trace, sounded_approximation from tests.integration.nsf.console.session import captured_trace +from tests.integration.nsf.header import sample_information ChannelInstructions = Dict[ChannelName, List[InstructionUnion]] @@ -21,7 +21,7 @@ def played_by_console(sample: Sample) -> ChannelInstructions: """The per-tick instructions the console sounds, read back out of the registers it wrote.""" song = song_from_reconstruction(sample.reconstruction, loop_tick=None) - trace = captured_trace(song, exported_information(sample.name)) + trace = captured_trace(song, sample_information(sample.name)) return instructions_from_trace(trace, get_timer_table(sample.reconstruction.config.tuning)) diff --git a/tests/integration/nsf/test_driver_bend.py b/tests/integration/nsf/test_driver_bend.py index 5a5f3fa0b..3e600fc6d 100644 --- a/tests/integration/nsf/test_driver_bend.py +++ b/tests/integration/nsf/test_driver_bend.py @@ -8,10 +8,10 @@ from sampletones_player.specification.binary import BYTE_VALUES from sampletones_player.specification.registers import PULSE1_TIMER_HIGH, TIMER_HIGH_SHIFT from sampletones_tools.player.trace.trace import RegisterTrace -from sampletones_tools.samples.nsf import exported_information from tests.integration.nsf.console.instructions import channel_values, timer_value from tests.integration.nsf.console.machine import register_file from tests.integration.nsf.console.session import captured_trace, play_calls_covering +from tests.integration.nsf.header import sample_information from tests.suite.base import BaseTestSuite from tests.suite.case import BaseAutolabelTestCase from tests.suite.player import PLAYER_PITCHES, bent_song @@ -35,7 +35,7 @@ def sounded_dividers(song: Song) -> List[int]: Returns: List[int]: One divider per tick the song covers, in order. """ - trace = captured_trace(song, exported_information(SONG_NAME)) + trace = captured_trace(song, sample_information(SONG_NAME)) dividers = [] for registers in register_file(trace): values = channel_values(registers, ChannelName.PULSE1) @@ -96,7 +96,7 @@ def test_the_console_sounds_the_divider_the_bend_states(self, test_case: TestCas @pytest.mark.parametrize("test_case", test_cases, ids=lambda case: case.label) def test_the_model_states_the_writes_the_driver_makes(self, test_case: TestCase) -> None: song = bent_song(test_case.pitch_index, test_case.bends, NTSC_RATE) - trace = captured_trace(song, exported_information(SONG_NAME)) + trace = captured_trace(song, sample_information(SONG_NAME)) assert trace == RegisterTrace.from_song(song, play_calls_covering(song)) @@ -127,7 +127,7 @@ def test_the_high_half_is_written_wherever_the_bend_moves_it(self, crossing: Son halves = [(divider + bend) >> TIMER_HIGH_SHIFT for bend in self.BENDS] crossings = sum(1 for earlier, later in zip(halves, halves[1:]) if earlier != later) - trace = captured_trace(crossing, exported_information(SONG_NAME)) + trace = captured_trace(crossing, sample_information(SONG_NAME)) written = [write for writes in trace.play_calls for write in writes if write.address == PULSE1_TIMER_HIGH] assert crossings assert len(written) == crossings diff --git a/tests/integration/nsf/test_driver_trace.py b/tests/integration/nsf/test_driver_trace.py index 8ec5ea9de..abf98d12e 100644 --- a/tests/integration/nsf/test_driver_trace.py +++ b/tests/integration/nsf/test_driver_trace.py @@ -12,7 +12,6 @@ from sampletones_player.specification.nsf import PROGRAM_SIZE from sampletones_player.specification.song import STEP_FRACTION_OFFSET, STEP_WHOLE_OFFSET from sampletones_tools.player.trace.trace import RegisterTrace -from sampletones_tools.samples.nsf import exported_information from tests.integration.nsf.console.session import ( TRAILING_CALLS, captured_trace, @@ -20,6 +19,7 @@ play_calls_covering, play_calls_reaching, ) +from tests.integration.nsf.header import sample_information from tests.suite.base import BaseTestSuite from tests.suite.case import BaseAutolabelTestCase @@ -33,7 +33,7 @@ @pytest.fixture def trace(song: Song, sample: Sample) -> RegisterTrace: """Every APU write the assembled driver makes over a full run of the sample.""" - return captured_trace(song, exported_information(sample.name)) + return captured_trace(song, sample_information(sample.name)) @pytest.fixture @@ -101,7 +101,7 @@ def test_the_driver_writes_what_the_model_states( reclocked = sample.reconstruction.with_nes_frequency(test_case.expected) song = song_from_reconstruction(reclocked, loop_tick=None) - trace = captured_trace(song, exported_information(sample.name)) + trace = captured_trace(song, sample_information(sample.name)) assert trace == RegisterTrace.from_song(song, play_calls_covering(song)) @pytest.mark.parametrize("test_case", test_cases, ids=lambda case: case.label) @@ -166,7 +166,7 @@ def test_the_driver_writes_what_the_model_states( covered = song.ticks + test_case.expected * (song.ticks - song.loop_tick) calls = play_calls_reaching(song, covered) - trace = captured_trace_over(song, exported_information(sample.name), calls) + trace = captured_trace_over(song, sample_information(sample.name), calls) assert trace == RegisterTrace.from_song(song, calls) @pytest.mark.parametrize("test_case", test_cases, ids=lambda case: case.label) @@ -181,5 +181,5 @@ def test_the_run_keeps_sounding_past_the_songs_end( covered = song.ticks + test_case.expected * (song.ticks - song.loop_tick) calls = play_calls_reaching(song, covered) - trace = captured_trace_over(song, exported_information(sample.name), calls) + trace = captured_trace_over(song, sample_information(sample.name), calls) assert any(writes for writes in trace.play_calls[-TRAILING_CALLS:]) diff --git a/tests/integration/nsf/test_nsf_pipeline.py b/tests/integration/nsf/test_nsf_pipeline.py index 3f61d3eeb..98d739f48 100644 --- a/tests/integration/nsf/test_nsf_pipeline.py +++ b/tests/integration/nsf/test_nsf_pipeline.py @@ -29,7 +29,7 @@ TOTAL_TICKS_OFFSET, ) from sampletones_shared.paths.extensions import EXT_FILE_RECONSTRUCTION -from sampletones_tools.samples.nsf import exported_information +from tests.integration.nsf.header import sample_information def song_block(data: bytes, image: DriverImage) -> bytes: @@ -52,7 +52,7 @@ def exported( driver_image: DriverImage, ) -> bytes: destination = nsf_paths[sample.name] - write_nsf(destination, song, exported_information(sample.name), driver_image) + write_nsf(destination, song, sample_information(sample.name), driver_image) return destination.read_bytes() @@ -67,7 +67,7 @@ def test_every_sample_reaches_a_playable_file( ) -> None: for name, sample in instrument_catalog.items(): song = song_from_reconstruction(sample.reconstruction, loop_tick=None) - write_nsf(nsf_paths[name], song, exported_information(name), driver_image) + write_nsf(nsf_paths[name], song, sample_information(name), driver_image) assert nsf_paths[name].read_bytes()[: len(NSF_MAGIC)] == NSF_MAGIC def test_the_file_carries_the_shipped_driver(self, exported: bytes, driver_image: DriverImage) -> None: @@ -148,7 +148,7 @@ def test_the_round_trip_leaves_the_exported_bytes_alone( sample.reconstruction.save(stored) reloaded = song_from_reconstruction(Reconstruction.load(stored), loop_tick=None) - information = exported_information(sample.name) + information = sample_information(sample.name) before = tmp_path / "before.nsf" after = tmp_path / "after.nsf" write_nsf(before, song, information, driver_image) diff --git a/tests/integration/samples/test_emitters.py b/tests/integration/samples/test_emitters.py index 268c3a03e..99e8bc6f4 100644 --- a/tests/integration/samples/test_emitters.py +++ b/tests/integration/samples/test_emitters.py @@ -1,39 +1,43 @@ from pathlib import Path -from typing import Dict import pytest from sampletones_core.formats.bitphase.specification.channels import CHANNEL_LABELS -from sampletones_core.project.project import Project -from sampletones_core.project.voices.sample import Sample -from sampletones_player.specification.nsf import NSF_MAGIC +from sampletones_player.specification.nsf import ARTIST_OFFSET, NSF_MAGIC, STRING_FIELD_SIZE from sampletones_shared.paths.extensions import EXT_FILE_NSF from sampletones_tools.corpus.build import Corpus -from sampletones_tools.samples import bitphase, famitracker, nsf +from sampletones_tools.samples import bitphase, emit, famitracker, nsf +from sampletones_tools.samples.emit import emit_samples from tests.suite.bitphase import parse_btp from tests.suite.famitracker import parse_ftm -@pytest.fixture(scope="module") -def corpus(instrument_catalog: Dict[str, Sample], integration_project: Project) -> Corpus: - """The session's catalog and arrangement, as the emitters are handed them.""" - return Corpus(catalog=instrument_catalog, project=integration_project) +def _artist(data: bytes) -> str: + return data[ARTIST_OFFSET : ARTIST_OFFSET + STRING_FIELD_SIZE].split(b"\x00", 1)[0].decode("utf-8") + + +@pytest.fixture(name="corpus") +def corpus_fixture(monkeypatch: pytest.MonkeyPatch, synthetic_corpus: Corpus) -> Corpus: + """The session's corpus, which the samples commands hand their emitter in place of a build of their own.""" + monkeypatch.setattr(emit, "build_synthetic_corpus", lambda: synthetic_corpus) + return synthetic_corpus class TestTheEmittersWriteTheCorpus: """Each format's emitter writes the files a player of that format opens.""" def test_the_nsf_emitter_writes_every_sample_then_the_arrangement(self, corpus: Corpus, tmp_path: Path) -> None: - written = nsf.write_samples(corpus, tmp_path) + written = emit_samples(tmp_path, nsf.write_samples) assert [path.name for path in written] == [ *(f"{name}{EXT_FILE_NSF}" for name in corpus.catalog), f"{nsf.SONG_NAME}{EXT_FILE_NSF}", ] assert all(path.read_bytes()[: len(NSF_MAGIC)] == NSF_MAGIC for path in written) + assert all(_artist(path.read_bytes()) == corpus.project.info.author for path in written) def test_the_famitracker_emitter_writes_one_module(self, corpus: Corpus, tmp_path: Path) -> None: - written = famitracker.write_samples(corpus, tmp_path) + written = emit_samples(tmp_path, famitracker.write_samples) assert written == [tmp_path / famitracker.MODULE_FILENAME] assert parse_ftm(written[0].read_bytes()).instruments @@ -43,7 +47,7 @@ def test_the_bitphase_emitter_writes_the_song_at_its_tempo_and_as_a_groove( corpus: Corpus, tmp_path: Path, ) -> None: - written = bitphase.write_samples(corpus, tmp_path) + written = emit_samples(tmp_path, bitphase.write_samples) assert written == [tmp_path / bitphase.DOCUMENT_FILENAME, tmp_path / bitphase.GROOVE_DOCUMENT_FILENAME] assert all(parse_btp(path.read_bytes(), list(CHANNEL_LABELS)).songs for path in written) diff --git a/tests/suite/scripts.py b/tests/suite/scripts.py index 66cf1ce7b..26e6393ee 100644 --- a/tests/suite/scripts.py +++ b/tests/suite/scripts.py @@ -1,7 +1,7 @@ import importlib.util from types import ModuleType -from sampletones_shared.paths.source import SCRIPTS_ROOT +from sampletones_tools.checks.paths import SCRIPTS_ROOT def load_script(relative_path: str) -> ModuleType: diff --git a/tests/unit/sampletones/commands/test_convert.py b/tests/unit/sampletones/commands/test_convert.py index f1ce34721..78ce418c5 100644 --- a/tests/unit/sampletones/commands/test_convert.py +++ b/tests/unit/sampletones/commands/test_convert.py @@ -8,13 +8,13 @@ from sampletones.dispatcher import dispatch from sampletones_core.configs import Config from sampletones_core.constants.enums import DEFAULT_CHANNELS, ChannelName -from sampletones_core.headless.conversion import ConversionRequest, classic_setup +from sampletones_core.headless.conversion.request import ConversionRequest, classic_setup from sampletones_core.reconstructions.reconstructor.stems.configs.config import StemsConfig from sampletones_core.reconstructions.reconstructor.stems.configs.entry import StemEntry from sampletones_core.reconstructions.reconstructor.stems.configs.hierarchy import StemsHierarchy from sampletones_core.reconstructions.reconstructor.stems.configs.settings import StemSettings -RECONSTRUCTION = "sampletones_core.headless.conversion.reconstruct" +RECONSTRUCTION = "sampletones_core.headless.conversion.runners.reconstruct" LOADER = "sampletones_core.headless.config.load_config" diff --git a/tests/unit/sampletones_core/headless/conversion/stems.py b/tests/unit/sampletones_core/headless/conversion/stems.py new file mode 100644 index 000000000..89728ff60 --- /dev/null +++ b/tests/unit/sampletones_core/headless/conversion/stems.py @@ -0,0 +1,29 @@ +from pathlib import Path + +from sampletones_core.constants.enums import ChannelName +from sampletones_core.reconstructions.reconstructor.stems.configs.config import StemsConfig +from sampletones_core.reconstructions.reconstructor.stems.configs.entry import StemEntry +from sampletones_core.reconstructions.reconstructor.stems.configs.hierarchy import StemsHierarchy +from sampletones_core.reconstructions.reconstructor.stems.configs.settings import StemSettings + + +def recording(tmp_path: Path, name: str) -> Path: + path = tmp_path / name + path.write_bytes(b"") + return path + + +def two_stems() -> StemsConfig: + return StemsConfig( + entries=[ + StemEntry( + id=0, + settings=StemSettings(channels=[ChannelName.PULSE1, ChannelName.PULSE2], bends=[ChannelName.PULSE1]), + ), + StemEntry( + id=1, + settings=StemSettings(channels=[ChannelName.NOISE], bends=[]), + ), + ], + hierarchy=StemsHierarchy(levels=[[0, 1]]), + ) diff --git a/tests/unit/sampletones_core/headless/conversion/test_console.py b/tests/unit/sampletones_core/headless/conversion/test_console.py new file mode 100644 index 000000000..1fa89df21 --- /dev/null +++ b/tests/unit/sampletones_core/headless/conversion/test_console.py @@ -0,0 +1,34 @@ +from pathlib import Path + +from sampletones_core.constants.enums import DEFAULT_CHANNELS +from sampletones_core.headless.conversion.console import describe_stem, pairing_lines +from sampletones_core.headless.conversion.request import ConversionRequest, classic_setup +from tests.unit.sampletones_core.headless.conversion.stems import recording, two_stems + + +class TestDescribeStem: + def test_a_stem_names_its_channels_and_its_bends(self) -> None: + first, second = two_stems().entries + + assert describe_stem(first) == "stem 0 on pulse1, pulse2, bending pulse1" + assert describe_stem(second) == "stem 1 on noise" + + +class TestPairingLines: + def test_recordings_pair_with_the_entries_in_order(self, tmp_path: Path) -> None: + bass = recording(tmp_path, "bass.wav") + drums = recording(tmp_path, "drums.wav") + + request = ConversionRequest(sources=(bass, drums), stems=two_stems(), output_path=None) + + assert pairing_lines(request) == [ + "bass.wav: stem 0 on pulse1, pulse2, bending pulse1", + "drums.wav: stem 1 on noise", + ] + + def test_a_directory_names_the_one_stem_every_recording_plays_under(self, tmp_path: Path) -> None: + request = ConversionRequest(sources=(tmp_path,), stems=classic_setup(DEFAULT_CHANNELS), output_path=None) + + assert pairing_lines(request) == [ + f"{tmp_path.name}/: every recording under stem 0 on pulse1, triangle, noise, bending pulse1, triangle" + ] diff --git a/tests/unit/sampletones_core/headless/test_conversion.py b/tests/unit/sampletones_core/headless/conversion/test_request.py similarity index 53% rename from tests/unit/sampletones_core/headless/test_conversion.py rename to tests/unit/sampletones_core/headless/conversion/test_request.py index b12ebd4a9..9f73bdb60 100644 --- a/tests/unit/sampletones_core/headless/test_conversion.py +++ b/tests/unit/sampletones_core/headless/conversion/test_request.py @@ -1,46 +1,16 @@ import json from pathlib import Path -from typing import List, Tuple import pytest -from sampletones_core.configs import Config from sampletones_core.constants.enums import DEFAULT_CHANNELS, ChannelName -from sampletones_core.headless import conversion -from sampletones_core.headless.conversion import ( +from sampletones_core.headless.conversion.request import ( ConversionRequest, channels_named, classic_setup, - describe_stem, load_stems, - reconstruct, ) -from sampletones_core.reconstructions.reconstructor.stems.configs.config import StemsConfig -from sampletones_core.reconstructions.reconstructor.stems.configs.entry import StemEntry -from sampletones_core.reconstructions.reconstructor.stems.configs.hierarchy import StemsHierarchy -from sampletones_core.reconstructions.reconstructor.stems.configs.settings import StemSettings - - -def _recording(tmp_path: Path, name: str) -> Path: - path = tmp_path / name - path.write_bytes(b"") - return path - - -def _two_stems() -> StemsConfig: - return StemsConfig( - entries=[ - StemEntry( - id=0, - settings=StemSettings(channels=[ChannelName.PULSE1, ChannelName.PULSE2], bends=[ChannelName.PULSE1]), - ), - StemEntry( - id=1, - settings=StemSettings(channels=[ChannelName.NOISE], bends=[]), - ), - ], - hierarchy=StemsHierarchy(levels=[[0, 1]]), - ) +from tests.unit.sampletones_core.headless.conversion.stems import recording, two_stems class TestChannelsNamed: @@ -66,7 +36,7 @@ def test_one_stem_holds_the_channels_in_order_and_bends_the_toned_ones(self) -> class TestLoadStems: def test_a_setup_written_as_json_reads_back(self, tmp_path: Path) -> None: - stems = _two_stems() + stems = two_stems() path = tmp_path / "stems.json" path.write_text(json.dumps(stems.model_dump(mode="json")), encoding="utf-8") @@ -91,48 +61,41 @@ def test_a_mapping_that_is_no_setup_is_refused(self, tmp_path: Path) -> None: load_stems(path) -class TestDescribeStem: - def test_a_stem_names_its_channels_and_its_bends(self) -> None: - first, second = _two_stems().entries - - assert describe_stem(first) == "stem 0 on pulse1, pulse2, bending pulse1" - assert describe_stem(second) == "stem 1 on noise" - - class TestConversionRequest: def test_recordings_pair_with_the_entries_in_order(self, tmp_path: Path) -> None: - bass = _recording(tmp_path, "bass.wav") - drums = _recording(tmp_path, "drums.wav") + bass = recording(tmp_path, "bass.wav") + drums = recording(tmp_path, "drums.wav") - request = ConversionRequest(sources=(bass, drums), stems=_two_stems(), output_path=None) + request = ConversionRequest(sources=(bass, drums), stems=two_stems(), output_path=None) assert request.directory is None - assert request.pairing() == [ - "bass.wav: stem 0 on pulse1, pulse2, bending pulse1", - "drums.wav: stem 1 on noise", - ] + assert request.sources == (bass, drums) def test_a_directory_is_converted_file_by_file_under_one_stem(self, tmp_path: Path) -> None: request = ConversionRequest(sources=(tmp_path,), stems=classic_setup(DEFAULT_CHANNELS), output_path=None) assert request.directory == tmp_path - assert request.pairing() == [ - f"{tmp_path.name}/: every recording under stem 0 on pulse1, triangle, noise, bending pulse1, triangle" - ] + + def test_the_sources_are_classified_once_when_the_request_is_made(self, tmp_path: Path) -> None: + directory = tmp_path / "recordings" + directory.mkdir() + request = ConversionRequest(sources=(directory,), stems=classic_setup(DEFAULT_CHANNELS), output_path=None) + + directory.rmdir() + + assert request.directory == directory def test_a_count_mismatch_is_refused(self, tmp_path: Path) -> None: with pytest.raises(ValueError, match="1 sources for 2 stems"): - ConversionRequest(sources=(_recording(tmp_path, "bass.wav"),), stems=_two_stems(), output_path=None) + ConversionRequest(sources=(recording(tmp_path, "bass.wav"),), stems=two_stems(), output_path=None) def test_a_directory_under_several_stems_is_refused(self, tmp_path: Path) -> None: with pytest.raises(ValueError, match="under one stem; the setup holds 2"): - ConversionRequest(sources=(tmp_path,), stems=_two_stems(), output_path=None) + ConversionRequest(sources=(tmp_path,), stems=two_stems(), output_path=None) def test_a_directory_among_recordings_is_refused(self, tmp_path: Path) -> None: with pytest.raises(ValueError, match="one directory alone"): - ConversionRequest( - sources=(_recording(tmp_path, "bass.wav"), tmp_path), stems=_two_stems(), output_path=None - ) + ConversionRequest(sources=(recording(tmp_path, "bass.wav"), tmp_path), stems=two_stems(), output_path=None) def test_an_output_path_for_a_directory_is_refused(self, tmp_path: Path) -> None: with pytest.raises(ValueError, match="an output path names the one file"): @@ -153,7 +116,7 @@ def test_a_missing_source_is_refused_by_its_path(self, tmp_path: Path) -> None: def test_a_file_other_than_a_recording_is_refused(self, tmp_path: Path) -> None: with pytest.raises(ValueError, match="is no recording"): ConversionRequest( - sources=(_recording(tmp_path, "song.stp"),), + sources=(recording(tmp_path, "song.stp"),), stems=classic_setup(DEFAULT_CHANNELS), output_path=None, ) @@ -161,38 +124,3 @@ def test_a_file_other_than_a_recording_is_refused(self, tmp_path: Path) -> None: def test_no_source_is_refused(self) -> None: with pytest.raises(ValueError): ConversionRequest(sources=(), stems=classic_setup(DEFAULT_CHANNELS), output_path=None) - - -class TestReconstruct: - def test_recordings_are_mixed_into_one_file(self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: - calls: List[Tuple[Tuple[Path, ...], Path]] = [] - - def reconstruct_sources( - sources: Tuple[Path, ...], config: Config, stems: StemsConfig, output_path: Path - ) -> None: - del config, stems - calls.append((sources, output_path)) - - monkeypatch.setattr(conversion, "reconstruct_sources", reconstruct_sources) - source = _recording(tmp_path, "song.wav") - request = ConversionRequest( - sources=(source,), stems=classic_setup(DEFAULT_CHANNELS), output_path=tmp_path / "x.stn" - ) - - reconstruct(request, Config()) - - assert calls == [((source,), tmp_path / "x.stn")] - - def test_a_directory_is_walked(self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: - walked: List[Path] = [] - - def reconstruct_directory(directory: Path, config: Config, stems: StemsConfig) -> None: - del config, stems - walked.append(directory) - - monkeypatch.setattr(conversion, "reconstruct_directory", reconstruct_directory) - request = ConversionRequest(sources=(tmp_path,), stems=classic_setup(DEFAULT_CHANNELS), output_path=None) - - reconstruct(request, Config()) - - assert walked == [tmp_path] diff --git a/tests/unit/sampletones_core/headless/conversion/test_runners.py b/tests/unit/sampletones_core/headless/conversion/test_runners.py new file mode 100644 index 000000000..6361f68aa --- /dev/null +++ b/tests/unit/sampletones_core/headless/conversion/test_runners.py @@ -0,0 +1,47 @@ +from pathlib import Path +from typing import List, Tuple + +import pytest + +from sampletones_core.configs import Config +from sampletones_core.constants.enums import DEFAULT_CHANNELS +from sampletones_core.headless.conversion import runners +from sampletones_core.headless.conversion.request import ConversionRequest, classic_setup +from sampletones_core.headless.conversion.runners import reconstruct +from sampletones_core.reconstructions.reconstructor.stems.configs.config import StemsConfig +from tests.unit.sampletones_core.headless.conversion.stems import recording + + +class TestReconstruct: + def test_recordings_are_mixed_into_one_file(self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + calls: List[Tuple[Tuple[Path, ...], Path]] = [] + + def reconstruct_sources( + sources: Tuple[Path, ...], config: Config, stems: StemsConfig, output_path: Path + ) -> None: + del config, stems + calls.append((sources, output_path)) + + monkeypatch.setattr(runners, "reconstruct_sources", reconstruct_sources) + source = recording(tmp_path, "song.wav") + request = ConversionRequest( + sources=(source,), stems=classic_setup(DEFAULT_CHANNELS), output_path=tmp_path / "x.stn" + ) + + reconstruct(request, Config()) + + assert calls == [((source,), tmp_path / "x.stn")] + + def test_a_directory_is_walked(self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + walked: List[Path] = [] + + def reconstruct_directory(directory: Path, config: Config, stems: StemsConfig) -> None: + del config, stems + walked.append(directory) + + monkeypatch.setattr(runners, "reconstruct_directory", reconstruct_directory) + request = ConversionRequest(sources=(tmp_path,), stems=classic_setup(DEFAULT_CHANNELS), output_path=None) + + reconstruct(request, Config()) + + assert walked == [tmp_path] diff --git a/tests/unit/sampletones_shared/paths/test_source.py b/tests/unit/sampletones_shared/paths/test_source.py index 426673b00..3b55bd6a7 100644 --- a/tests/unit/sampletones_shared/paths/test_source.py +++ b/tests/unit/sampletones_shared/paths/test_source.py @@ -1,4 +1,4 @@ -from sampletones_shared.paths.source import REPOSITORY_ROOT, SCRIPTS_ROOT, SOURCE_ROOT +from sampletones_shared.paths.source import REPOSITORY_ROOT, SOURCE_ROOT PROJECT_FILE = "pyproject.toml" SHARED_PACKAGE = "sampletones_shared" @@ -16,8 +16,3 @@ def test_the_source_root_is_where_this_package_lives(self) -> None: class TestRepositoryRoot: def test_the_repository_root_holds_the_project_file(self) -> None: assert (REPOSITORY_ROOT / PROJECT_FILE).is_file() - - -class TestScriptsRoot: - def test_the_scripts_root_holds_the_bootstrap_tree(self) -> None: - assert (SCRIPTS_ROOT / "bootstrap" / "__init__.py").is_file() diff --git a/tests/unit/sampletones_shared/utils/test_text.py b/tests/unit/sampletones_shared/utils/test_text.py index c5b960799..a15c73d10 100644 --- a/tests/unit/sampletones_shared/utils/test_text.py +++ b/tests/unit/sampletones_shared/utils/test_text.py @@ -2,7 +2,7 @@ import pytest -from sampletones_shared.utils.text import natural_sort_key +from sampletones_shared.utils.text import LIST_SEPARATOR, listed_items, natural_sort_key class TestNumbers: @@ -51,3 +51,17 @@ def test_one_name_reaches_one_key(self) -> None: def test_a_name_the_reader_alone_can_spell(self) -> None: """A digit-like glyph outside the decimal digits is text, and the key states it as text.""" assert sorted(["m²", "m1"], key=natural_sort_key) == ["m1", "m²"] + + +class TestListedItems: + def test_items_keep_their_order_with_the_spaces_around_them_stripped(self) -> None: + assert listed_items(" fft ,cqt") == ["fft", "cqt"] + + def test_empty_items_are_dropped(self) -> None: + assert listed_items(f"pulse1{LIST_SEPARATOR}{LIST_SEPARATOR} {LIST_SEPARATOR}noise{LIST_SEPARATOR}") == [ + "pulse1", + "noise", + ] + + def test_an_empty_statement_lists_nothing(self) -> None: + assert listed_items("") == [] diff --git a/tests/unit/sampletones_tools/assets/test_command.py b/tests/unit/sampletones_tools/assets/test_command.py index 69abe20fd..723388def 100644 --- a/tests/unit/sampletones_tools/assets/test_command.py +++ b/tests/unit/sampletones_tools/assets/test_command.py @@ -48,6 +48,6 @@ def test_a_directory_of_its_own_is_written_from_a_checkout_too( monkeypatch.setattr(WRITER, suite) monkeypatch.setattr(GUARD, guarded.append) - assert dispatch(COMMANDS, ["icons", "--directory", str(tmp_path)]) == 0 + assert dispatch(COMMANDS, ["icons", "--output", str(tmp_path)]) == 0 assert [directory for directory, _ in suite.writes] == [tmp_path] assert guarded == ["icons"] diff --git a/tests/unit/sampletones_tools/calibration/test_command.py b/tests/unit/sampletones_tools/calibration/test_command.py index 58c013354..3e80f8d4b 100644 --- a/tests/unit/sampletones_tools/calibration/test_command.py +++ b/tests/unit/sampletones_tools/calibration/test_command.py @@ -8,7 +8,7 @@ from sampletones_core.configs import Config from sampletones_core.constants.enums import DEFAULT_CHANNELS, ChannelName, SpectrumMethod from sampletones_shared.paths.user import USER_PATH_DOCUMENTS -from sampletones_tools.calibration.session import CalibrationRequest +from sampletones_tools.calibration.session import OUTPUT_DIRECTORY, CalibrationRequest CALIBRATE = "sampletones_tools.calibration.session.calibrate" LOADER = "sampletones_core.headless.config.load_config" @@ -68,7 +68,7 @@ def test_without_options_the_run_sweeps_both_methods_into_the_documents( assert request.perceptual_exponents == [1.0] assert request.temporal_weights == [] assert request.channels == list(DEFAULT_CHANNELS) - assert request.output.parent == USER_PATH_DOCUMENTS / "calibration" + assert request.output.parent == USER_PATH_DOCUMENTS / OUTPUT_DIRECTORY def test_an_unknown_method_is_refused(self, calibration: RecordedCalibration) -> None: with pytest.raises(SystemExit, match="Unknown spectrum method"): diff --git a/tests/unit/sampletones_tools/calibration/test_session.py b/tests/unit/sampletones_tools/calibration/test_session.py index 0870b333e..85d4da3db 100644 --- a/tests/unit/sampletones_tools/calibration/test_session.py +++ b/tests/unit/sampletones_tools/calibration/test_session.py @@ -1,3 +1,4 @@ +from datetime import datetime from pathlib import Path import pytest @@ -6,11 +7,13 @@ from sampletones_core.constants.enums import ChannelName, SpectrumMethod from sampletones_shared.paths.user import USER_PATH_DOCUMENTS from sampletones_tools.calibration.session import ( + OUTPUT_DIRECTORY, CalibrationRequest, default_output, floats_named, methods_named, ) +from sampletones_tools.runs import RUN_STAMP class TestMethodsNamed: @@ -42,8 +45,8 @@ class TestDefaultOutput: def test_a_run_lands_in_a_timestamped_directory_under_the_documents(self) -> None: output = default_output() - assert output.parent == USER_PATH_DOCUMENTS / "calibration" - assert output.name.startswith("run-") + assert output.parent == USER_PATH_DOCUMENTS / OUTPUT_DIRECTORY + assert datetime.strptime(output.name, RUN_STAMP) class TestCalibrationRequest: diff --git a/tests/unit/sampletones_tools/checks/boundary/configs/test_rules.py b/tests/unit/sampletones_tools/checks/boundary/configs/test_rules.py index 6efe16a1c..2b9602d7c 100644 --- a/tests/unit/sampletones_tools/checks/boundary/configs/test_rules.py +++ b/tests/unit/sampletones_tools/checks/boundary/configs/test_rules.py @@ -5,7 +5,7 @@ import pytest from pydantic import ValidationError -from sampletones_shared.paths.source import SCRIPTS_ROOT, SOURCE_ROOT +from sampletones_shared.paths.source import SOURCE_ROOT from sampletones_tools.checks.boundary.check import check_boundaries from sampletones_tools.checks.boundary.configs.declaration import BoundaryDeclaration from sampletones_tools.checks.boundary.configs.general import GeneralBoundaries @@ -14,6 +14,7 @@ from sampletones_tools.checks.boundary.rule import BoundaryRule from sampletones_tools.checks.boundary.scope import rule_modules from sampletones_tools.checks.boundary.standalone import check_standalone +from sampletones_tools.checks.paths import SCRIPTS_ROOT from tests.suite.source import swept_paths, write_module BOUNDARIES: Final[ImportBoundaryRules] = ImportBoundaryRules.load() @@ -174,10 +175,6 @@ def test_a_bootstrap_script_reaching_a_third_party_package_is_reported(self, tmp assert [violation.kind for violation in reported] == [rule.message for rule in BOUNDARIES.standalone] - def test_every_excluded_glob_names_a_script_still_in_the_tree(self) -> None: - """A tool that moved into the project takes its exclusion with it.""" - assert all(list(SCRIPTS_ROOT.glob(glob)) for rule in BOUNDARIES.standalone for glob in rule.excluding) - class TestRuleCoverage: """A rule naming no module of the tree reads as a clean tree, so each one reaches something.""" @@ -194,6 +191,4 @@ def test_every_token_rule_reaches_a_module(self) -> None: def test_every_standalone_rule_reaches_a_script(self) -> None: swept = swept_paths(SCRIPTS_ROOT) - assert all( - rule_modules(SCRIPTS_ROOT, rule.pattern, rule.excluding, swept, None) for rule in BOUNDARIES.standalone - ) + assert all(rule_modules(SCRIPTS_ROOT, rule.pattern, (), swept, None) for rule in BOUNDARIES.standalone) diff --git a/tests/unit/sampletones_tools/checks/boundary/test_standalone.py b/tests/unit/sampletones_tools/checks/boundary/test_standalone.py index f34745270..6057c838a 100644 --- a/tests/unit/sampletones_tools/checks/boundary/test_standalone.py +++ b/tests/unit/sampletones_tools/checks/boundary/test_standalone.py @@ -14,7 +14,6 @@ RULE: Final[StandaloneRule] = StandaloneRule( pattern="**/*.py", - excluding=("tools/**/*.py",), reserved=("tests",), message=MESSAGE, ) @@ -101,11 +100,11 @@ def test_a_clean_tree_reports_nothing(self, tmp_path: Path) -> None: assert kinds(tmp_path) == [] - def test_an_excluded_script_is_left_to_the_project_environment(self, tmp_path: Path) -> None: + def test_a_script_in_a_subdirectory_is_held_to_the_rule(self, tmp_path: Path) -> None: _tree(tmp_path) - write_module(tmp_path / "tools", "calibration.py", THIRD_PARTY) + write_module(tmp_path / "ci", "archive.py", THIRD_PARTY) - assert kinds(tmp_path) == [] + assert kinds(tmp_path) == [MESSAGE] def test_the_shadowing_names_lead_the_imports(self, tmp_path: Path) -> None: _tree(tmp_path) diff --git a/tests/unit/sampletones_tools/checks/test_paths.py b/tests/unit/sampletones_tools/checks/test_paths.py new file mode 100644 index 000000000..d94e64cef --- /dev/null +++ b/tests/unit/sampletones_tools/checks/test_paths.py @@ -0,0 +1,6 @@ +from sampletones_tools.checks.paths import SCRIPTS_ROOT + + +class TestScriptsRoot: + def test_the_scripts_root_holds_the_bootstrap_tree(self) -> None: + assert (SCRIPTS_ROOT / "bootstrap" / "__init__.py").is_file() diff --git a/tests/unit/sampletones_tools/codec/study/test_session.py b/tests/unit/sampletones_tools/codec/study/test_session.py index 8a18f02b0..4e250ea6a 100644 --- a/tests/unit/sampletones_tools/codec/study/test_session.py +++ b/tests/unit/sampletones_tools/codec/study/test_session.py @@ -40,6 +40,16 @@ def test_a_run_naming_no_source_and_no_manifest_is_refused(self) -> None: variants=(EVERY_VARIANT,), ) + def test_a_manifest_missing_from_its_path_is_refused(self, tmp_path: Path) -> None: + with pytest.raises(ValueError, match="No manifest at"): + resolve_manifest( + tmp_path / "absent.json", + projects=(), + reconstructions=(), + lengthen_seconds=30, + variants=(EVERY_VARIANT,), + ) + def test_a_manifest_file_is_measured_as_it_stands(self, tmp_path: Path) -> None: written = StudyManifest( projects=(StudySource(label="one", path=Path("songs/one.stp")),), diff --git a/tests/unit/sampletones_tools/codec/test_command.py b/tests/unit/sampletones_tools/codec/test_command.py index 2ab8a65d8..49d8edc5b 100644 --- a/tests/unit/sampletones_tools/codec/test_command.py +++ b/tests/unit/sampletones_tools/codec/test_command.py @@ -7,6 +7,7 @@ from sampletones.commands.registry import COMMANDS from sampletones.dispatcher import dispatch from sampletones_tools.codec.command import DEFAULT_LENGTHEN_SECONDS +from sampletones_tools.codec.report.session import CompressionReport from sampletones_tools.codec.study.manifest import StudyManifest from sampletones_tools.codec.study.variants.production import BASELINE_NAME from sampletones_tools.codec.study.variants.registry import EVERY_VARIANT @@ -36,9 +37,14 @@ def test_the_report_is_written_into_the_output_and_its_tables_are_printed( ) -> None: outputs: List[Path] = [] - def run_report(output: Path) -> Tuple[Path, Path]: + def run_report(output: Path) -> CompressionReport: outputs.append(output) - return output / "report.csv", output / "report.md" + return CompressionReport( + entries=(), + encodings=(), + csv_path=output / "report.csv", + markdown_path=output / "report.md", + ) monkeypatch.setattr(REPORTER, run_report) diff --git a/tests/unit/sampletones_tools/player/test_command.py b/tests/unit/sampletones_tools/player/test_command.py index 32540ffd0..c9b7d56e6 100644 --- a/tests/unit/sampletones_tools/player/test_command.py +++ b/tests/unit/sampletones_tools/player/test_command.py @@ -52,7 +52,7 @@ def test_a_directory_of_its_own_needs_no_checkout(self, monkeypatch: pytest.Monk monkeypatch.setattr(BUILDER, build) monkeypatch.setattr(GUARD, _refuse) - assert dispatch(COMMANDS, ["driver", "--directory", str(tmp_path)]) == 0 + assert dispatch(COMMANDS, ["driver", "--output", str(tmp_path)]) == 0 assert build.destinations == [tmp_path] def test_a_failing_build_is_reported(self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: @@ -62,4 +62,4 @@ def fail(destination: Path) -> DriverImage: monkeypatch.setattr(BUILDER, fail) with pytest.raises(SystemExit, match="ca65 failed"): - dispatch(COMMANDS, ["driver", "--directory", str(tmp_path)]) + dispatch(COMMANDS, ["driver", "--output", str(tmp_path)]) diff --git a/tests/unit/sampletones_tools/test_runs.py b/tests/unit/sampletones_tools/test_runs.py new file mode 100644 index 000000000..d54d92994 --- /dev/null +++ b/tests/unit/sampletones_tools/test_runs.py @@ -0,0 +1,15 @@ +from datetime import UTC, datetime +from pathlib import Path + +from sampletones_tools.runs import RUN_STAMP, stamped_run_directory + + +class TestStampedRunDirectory: + def test_a_run_lands_under_the_root_named_by_its_start(self, tmp_path: Path) -> None: + before = datetime.now(UTC).replace(microsecond=0) + + directory = stamped_run_directory(tmp_path) + + started = datetime.strptime(directory.name, RUN_STAMP).replace(tzinfo=UTC) + assert directory.parent == tmp_path + assert before <= started <= datetime.now(UTC) From 6d08f892d63c589ba6d0ac21470b8b10c464134b Mon Sep 17 00:00:00 2001 From: JakimPL Date: Sun, 13 Sep 2026 20:18:01 +0200 Subject: [PATCH 25/36] Moved: the calibration configuration into the tools package --- docs/concepts/calibration.md | 4 ++-- .../application/config-organization.md | 21 ++++++++++++------- src/sampletones_config/README.md | 3 +-- .../calibration/config/corpus.py | 4 ++-- .../calibration/config}/corpus.yaml | 0 .../calibration/config/referee.py | 2 +- .../calibration/config}/referee.yaml | 0 src/sampletones_tools/calibration/paths.py | 5 ++--- 8 files changed, 21 insertions(+), 18 deletions(-) rename src/{sampletones_config/calibration => sampletones_tools/calibration/config}/corpus.yaml (100%) rename src/{sampletones_config/calibration => sampletones_tools/calibration/config}/referee.yaml (100%) diff --git a/docs/concepts/calibration.md b/docs/concepts/calibration.md index 2e08554da..dfff8c513 100644 --- a/docs/concepts/calibration.md +++ b/docs/concepts/calibration.md @@ -31,7 +31,7 @@ It has three moving parts: range. Each probe is a `sampletones_tools.synthesis` voice — oscillators, envelopes and filters composed from one shared, exactly-rendered configuration vocabulary — built from the probe families in - `sampletones_config/calibration/corpus.yaml`. Each category isolates one kind of decision the criterion must get + `sampletones_tools/calibration/config/corpus.yaml`. Each category isolates one kind of decision the criterion must get right — pitch, timbre, noise balance, attack sharpness, level tracking — and the fixed seed makes every run bit-identical, so scores are comparable across runs and code changes. @@ -51,7 +51,7 @@ It has three moving parts: resolves with a single frame length. Band energies are floored at a fixed audibility range below the reference's loudest band, so the score reflects audible content and holds steady under a common gain. Its tuning is a - `RefereeConfig` loaded from `sampletones_config/calibration/referee.yaml`. + `RefereeConfig` loaded from `sampletones_tools/calibration/config/referee.yaml`. When the [zimtohrli](https://github.com/google/zimtohrli) binary is installed it joins automatically as a second, psychoacoustic referee. diff --git a/docs/development/application/config-organization.md b/docs/development/application/config-organization.md index 0cfd0c7d7..6cb4a0096 100644 --- a/docs/development/application/config-organization.md +++ b/docs/development/application/config-organization.md @@ -10,8 +10,8 @@ is read; use it as the reference when adding or moving a value. It sits alongsid first: - **Shipped configuration** — the `sampletones_config` YAML package: layout, theme, - palettes, keybindings, language, behavior, deployment, calibration, and the import - boundaries. *(This document.)* + palettes, keybindings, language, behavior, deployment, and the import boundaries. + *(This document.)* - **Runtime user preferences** — mutable state persisted to the user profile (`sampletones_application/config`, e.g. `PlaybackConfig`, `ShortcutsConfig`, `ApplicationState`), governed by that package. @@ -32,16 +32,22 @@ empty `__init__.py`. Each schema lives with its reader: - `sampletones_application` owns the layout, theme, palettes, keybindings, language, behavior, and deployment schemas. -- `sampletones_tools` owns the calibration and the import-boundary schemas. +- `sampletones_tools` owns the import-boundary schemas. - `sampletones_shared` owns the loader primitives (`load_yaml_model`, `load_yaml_model_dir`). So the data carries the values and the consumer carries the meaning, and the two evolve on their own terms. +Data only a developer tool reads ships with that tool, beside the schema reading it: the +calibration tuning sits in `sampletones_tools/calibration/config/` and the synthetic corpus in +`sampletones_tools/corpus/config/`, each read through `importlib.resources`. `sampletones_config` +holds what the application reads and the import boundaries, which state the repository's +package layers for every tree and which [package layers](../packages.md) refers to. + ### 2. The top level is organized by domain `sampletones_config` has one top-level directory per schema family and its loader: -`application`, `behavior`, `boundaries`, `calibration`, `keybindings`, `lang`, `layout`, +`application`, `behavior`, `boundaries`, `keybindings`, `lang`, `layout`, `palettes`, `theme`. Each domain owns its schema and its load path (see [Domains](#domains)). A new domain is a new top-level directory with its own schema owner and loader. @@ -132,7 +138,6 @@ each value sits in the tree stays in the factory. | Application | `application/` | `DeploymentConfig` (`sampletones_application/config/deployment/`) | `DeploymentConfig.load()`, with `SAMPLETONES_*` env overrides | | Behavior | `behavior/` | `BehaviorConfig` (`sampletones_application/layout/behavior.py`) | folded into `LayoutConfig.behavior` by `load_layout_config` | | Boundaries | `boundaries/` | `ImportBoundaryRules` (`sampletones_tools/checks/boundary/configs/`) | `ImportBoundaryRules.load()` | -| Calibration | `calibration/` | `CorpusConfig`, `RefereeConfig` (`sampletones_tools/calibration/config/`) | each model's own `.load()` | | Keybindings | `keybindings/` | `ShortcutScheme` (`sampletones_application/utils/gui/shortcuts/`) | `ShortcutCatalog.load()`, indexed by scheme name | | Language | `lang/` | `LanguageManager` (`sampletones_application/categories/`) | flat string map keyed `page.panel.text_type.element`, each key validated at load | | Layout | `layout/` | `LayoutConfig` (`sampletones_application/layout/config.py`) | `load_layout_config` (`layout/loader.py`) | @@ -172,8 +177,8 @@ divide into, the imports each part of the application stays clear of, the spelli keeps out, and the standard-library rule the bootstrap scripts under `scripts/` hold to, and `sampletones check import-boundary` runs it over the source and scripts trees on every commit. A declaration draws on the named prefix groups `general.yaml` holds, so a set several rules reach for is written once and each rule names it, and a name reaching no group is -refused as the domain is read. The bundle carries the domain because `--add-data` copies -`sampletones_config` whole — the terms `calibration/` already ships on. +refused as the domain is read. The bundle carries the domain with the rest of +`sampletones_config`. --- @@ -199,6 +204,6 @@ Three load mechanisms serve the three grouping schemes: file on disk. `ShortcutCatalog.load()` reads `keybindings/` the same way, keyed by `ShortcutScheme.name`. -Deployment, calibration and the boundaries each load through a bespoke `.load()` +Deployment and the boundaries each load through a bespoke `.load()` classmethod over the same low-level primitives in `sampletones_shared/utils/serialization.py` — the one module that calls `yaml.safe_load`. diff --git a/src/sampletones_config/README.md b/src/sampletones_config/README.md index 2d9d3728c..204682e19 100644 --- a/src/sampletones_config/README.md +++ b/src/sampletones_config/README.md @@ -8,7 +8,7 @@ programmatic role is to be importable so consumers can resolve its directory The schema that validates each file lives in the **consuming** package: - `sampletones_application` — layout, theme, palettes, language, behavior, deployment. -- `sampletones_tools` — calibration and the import boundaries. +- `sampletones_tools` — the import boundaries. - `sampletones_shared` — the loader primitives. The data package must not import a schema, and a schema package must not inline data. @@ -20,7 +20,6 @@ The data package must not import a schema, and a schema package must not inline | `application/` | Deployment-time environment knobs | `DeploymentConfig` | | `behavior/` | Non-visual runtime behavior | `BehaviorConfig` | | `boundaries/` | The imports the source and scripts trees are held to | `ImportBoundaryRules` | -| `calibration/` | DSP calibration tuning | `CorpusConfig`, `RefereeConfig` | | `keybindings/` | The key combinations each named action answers | `ShortcutScheme` | | `lang/` | Interface strings (i18n) | `LanguageManager` | | `layout/` | UI geometry, dimensions, fonts | `LayoutConfig` | diff --git a/src/sampletones_tools/calibration/config/corpus.py b/src/sampletones_tools/calibration/config/corpus.py index d6805445f..1eb2e1307 100644 --- a/src/sampletones_tools/calibration/config/corpus.py +++ b/src/sampletones_tools/calibration/config/corpus.py @@ -18,7 +18,7 @@ class CorpusConfig(BaseModel, frozen=True): Every probe is synthesized at unit scale and multiplied by the corpus amplitude, so the per-class parameters compose under one loudness - convention. Values are loaded from the packaged `calibration/corpus.yaml`, + convention. Values are loaded from the `corpus.yaml` beside this model, so corpus content stays reproducible across calibration runs while remaining adjustable in one place. """ @@ -62,7 +62,7 @@ def load(cls) -> Self: Load the packaged corpus tuning. Returns: - The corpus configuration validated from `sampletones_config/calibration/corpus.yaml`. + The corpus configuration validated from `sampletones_tools/calibration/config/corpus.yaml`. Raises: TypeError: If the configuration file holds anything other than a mapping. diff --git a/src/sampletones_config/calibration/corpus.yaml b/src/sampletones_tools/calibration/config/corpus.yaml similarity index 100% rename from src/sampletones_config/calibration/corpus.yaml rename to src/sampletones_tools/calibration/config/corpus.yaml diff --git a/src/sampletones_tools/calibration/config/referee.py b/src/sampletones_tools/calibration/config/referee.py index 2d8992116..d70bc9d5d 100644 --- a/src/sampletones_tools/calibration/config/referee.py +++ b/src/sampletones_tools/calibration/config/referee.py @@ -44,7 +44,7 @@ def load(cls) -> Self: Load the packaged referee tuning. Returns: - The referee configuration validated from `sampletones_config/calibration/referee.yaml`. + The referee configuration validated from `sampletones_tools/calibration/config/referee.yaml`. Raises: TypeError: If the configuration file holds anything other than a mapping. diff --git a/src/sampletones_config/calibration/referee.yaml b/src/sampletones_tools/calibration/config/referee.yaml similarity index 100% rename from src/sampletones_config/calibration/referee.yaml rename to src/sampletones_tools/calibration/config/referee.yaml diff --git a/src/sampletones_tools/calibration/paths.py b/src/sampletones_tools/calibration/paths.py index 751bbb533..5deb181a1 100644 --- a/src/sampletones_tools/calibration/paths.py +++ b/src/sampletones_tools/calibration/paths.py @@ -1,8 +1,7 @@ +from importlib.resources import files from pathlib import Path from typing import Final -from sampletones_shared.paths.resources import CONFIG_DIRECTORY - -CALIBRATION_CONFIG_DIRECTORY: Final[Path] = CONFIG_DIRECTORY / "calibration" +CALIBRATION_CONFIG_DIRECTORY: Final[Path] = Path(str(files("sampletones_tools.calibration.config"))) REFEREE_CONFIG_PATH: Final[Path] = CALIBRATION_CONFIG_DIRECTORY / "referee.yaml" CORPUS_CONFIG_PATH: Final[Path] = CALIBRATION_CONFIG_DIRECTORY / "corpus.yaml" From 0f84b83a9416b5f8ae0636e2e91504c132f51bb0 Mon Sep 17 00:00:00 2001 From: JakimPL Date: Sun, 13 Sep 2026 20:41:55 +0200 Subject: [PATCH 26/36] Centralized: the bootstrap scripts' layout, platform facts and release gates --- .github/workflows/workflow.yml | 11 +- .../application/config-organization.md | 2 +- docs/development/release/dependencies.md | 2 +- docs/development/tooling.md | 48 ++-- .../{ci/zip_bundle.py => archive_bundle.py} | 36 ++- scripts/bootstrap/cuda.py | 163 ++++++++++++ scripts/bootstrap/files.py | 22 ++ scripts/bootstrap/layout.py | 31 +++ scripts/bootstrap/platforms/bundling.py | 38 +++ scripts/bootstrap/platforms/linux.py | 64 +++-- scripts/bootstrap/platforms/macos.py | 88 ++++--- scripts/bootstrap/platforms/protocol.py | 62 ++--- scripts/bootstrap/platforms/windows.py | 66 ++--- scripts/bootstrap/preflight.py | 12 +- scripts/bootstrap/project.py | 88 +++++++ scripts/bootstrap/repository.py | 17 -- scripts/bootstrap/venv_build.py | 4 +- scripts/build_environment.py | 30 +-- scripts/bundle.py | 93 +++---- scripts/ci/__init__.py | 0 scripts/ci/checks/__init__.py | 0 scripts/ci/checks/bundle.py | 83 ------- scripts/ci/checks/version_tag.py | 46 ---- scripts/clean.py | 45 ++-- scripts/detect_cuda.py | 235 ------------------ scripts/formatting.py | 9 +- scripts/hooks.py | 27 +- scripts/lint.py | 55 +++- scripts/run_tests.py | 39 ++- scripts/setup_environment.py | 79 +++--- scripts/system_dependencies.py | 49 +++- scripts/verify_bundle.py | 97 ++++++++ scripts/verify_version_tag.py | 33 +++ .../ui/resources/loader.py | 6 - src/sampletones_shared/paths/package.py | 27 ++ src/sampletones_shared/paths/resources.py | 12 +- src/sampletones_tools/assets/mark/paths.py | 5 +- src/sampletones_tools/calibration/paths.py | 5 +- src/sampletones_tools/checkout.py | 8 +- src/sampletones_tools/corpus/paths.py | 5 +- tests/suite/bootstrap.py | 31 ++- .../sampletones_shared/paths/test_package.py | 33 +++ .../bootstrap/platforms/test_bundling.py | 50 ++++ .../bootstrap/platforms/test_factory.py | 30 +++ .../scripts/bootstrap/platforms/test_linux.py | 27 ++ .../scripts/bootstrap/platforms/test_macos.py | 64 +++++ .../bootstrap/platforms/test_windows.py | 27 ++ .../test_cuda.py} | 97 ++++---- tests/unit/scripts/bootstrap/test_files.py | 23 ++ tests/unit/scripts/bootstrap/test_layout.py | 9 + .../unit/scripts/bootstrap/test_platforms.py | 128 ---------- .../unit/scripts/bootstrap/test_preflight.py | 8 +- tests/unit/scripts/bootstrap/test_project.py | 50 ++++ .../unit/scripts/bootstrap/test_repository.py | 9 - .../unit/scripts/bootstrap/test_venv_build.py | 11 +- tests/unit/scripts/ci/checks/test_bundle.py | 198 --------------- .../scripts/ci/checks/test_version_tag.py | 139 ----------- ...t_zip_bundle.py => test_archive_bundle.py} | 69 +++-- tests/unit/scripts/test_build_environment.py | 13 +- tests/unit/scripts/test_bundle.py | 160 ++++++------ tests/unit/scripts/test_formatting.py | 39 ++- tests/unit/scripts/test_hooks.py | 19 +- tests/unit/scripts/test_lint.py | 43 ++-- tests/unit/scripts/test_run_tests.py | 30 +-- tests/unit/scripts/test_setup_environment.py | 68 +++-- .../unit/scripts/test_system_dependencies.py | 45 ++-- tests/unit/scripts/test_verify_bundle.py | 105 ++++++++ tests/unit/scripts/test_verify_version_tag.py | 43 ++++ 68 files changed, 1808 insertions(+), 1502 deletions(-) rename scripts/{ci/zip_bundle.py => archive_bundle.py} (55%) create mode 100644 scripts/bootstrap/cuda.py create mode 100644 scripts/bootstrap/files.py create mode 100644 scripts/bootstrap/layout.py create mode 100644 scripts/bootstrap/platforms/bundling.py create mode 100644 scripts/bootstrap/project.py delete mode 100644 scripts/bootstrap/repository.py delete mode 100644 scripts/ci/__init__.py delete mode 100644 scripts/ci/checks/__init__.py delete mode 100644 scripts/ci/checks/bundle.py delete mode 100644 scripts/ci/checks/version_tag.py delete mode 100644 scripts/detect_cuda.py create mode 100644 scripts/verify_bundle.py create mode 100644 scripts/verify_version_tag.py create mode 100644 src/sampletones_shared/paths/package.py create mode 100644 tests/unit/sampletones_shared/paths/test_package.py create mode 100644 tests/unit/scripts/bootstrap/platforms/test_bundling.py create mode 100644 tests/unit/scripts/bootstrap/platforms/test_factory.py create mode 100644 tests/unit/scripts/bootstrap/platforms/test_linux.py create mode 100644 tests/unit/scripts/bootstrap/platforms/test_macos.py create mode 100644 tests/unit/scripts/bootstrap/platforms/test_windows.py rename tests/unit/scripts/{test_detect_cuda.py => bootstrap/test_cuda.py} (68%) create mode 100644 tests/unit/scripts/bootstrap/test_files.py create mode 100644 tests/unit/scripts/bootstrap/test_layout.py delete mode 100644 tests/unit/scripts/bootstrap/test_platforms.py create mode 100644 tests/unit/scripts/bootstrap/test_project.py delete mode 100644 tests/unit/scripts/bootstrap/test_repository.py delete mode 100644 tests/unit/scripts/ci/checks/test_bundle.py delete mode 100644 tests/unit/scripts/ci/checks/test_version_tag.py rename tests/unit/scripts/{ci/test_zip_bundle.py => test_archive_bundle.py} (57%) create mode 100644 tests/unit/scripts/test_verify_bundle.py create mode 100644 tests/unit/scripts/test_verify_version_tag.py diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml index 15bd9b2f4..bd22d7aa4 100644 --- a/.github/workflows/workflow.yml +++ b/.github/workflows/workflow.yml @@ -32,10 +32,7 @@ jobs: - name: Verify tag matches project version if: startsWith(github.ref, 'refs/tags/v') - run: | - python3 scripts/ci/checks/version_tag.py \ - --tag "$GITHUB_REF_NAME" \ - --project-version "$(uv version --short)" + run: python3 scripts/verify_version_tag.py --tag "$GITHUB_REF_NAME" - name: Build sdist and wheel run: uv build @@ -131,13 +128,11 @@ jobs: - name: Check the bundle runs and carries its notices shell: bash - run: python scripts/ci/checks/bundle.py bin/sampletones + run: python scripts/verify_bundle.py - name: Zip the bundle shell: bash - run: | - name="sampletones-${GITHUB_REF_NAME}-${{ matrix.platform }}" - python scripts/ci/zip_bundle.py bin/sampletones "bundles/${name}.zip" --root "${name}" + run: python scripts/archive_bundle.py --label "${{ matrix.platform }}" - uses: actions/upload-artifact@v7 with: diff --git a/docs/development/application/config-organization.md b/docs/development/application/config-organization.md index 6cb4a0096..06181ad48 100644 --- a/docs/development/application/config-organization.md +++ b/docs/development/application/config-organization.md @@ -40,7 +40,7 @@ on their own terms. Data only a developer tool reads ships with that tool, beside the schema reading it: the calibration tuning sits in `sampletones_tools/calibration/config/` and the synthetic corpus in -`sampletones_tools/corpus/config/`, each read through `importlib.resources`. `sampletones_config` +`sampletones_tools/corpus/config/`, each placed by `package_directory`. `sampletones_config` holds what the application reads and the import boundaries, which state the repository's package layers for every tree and which [package layers](../packages.md) refers to. diff --git a/docs/development/release/dependencies.md b/docs/development/release/dependencies.md index 3c34a7b4e..50f79d118 100644 --- a/docs/development/release/dependencies.md +++ b/docs/development/release/dependencies.md @@ -71,7 +71,7 @@ Pillow is a developer tool the build environment never installs, and the bundle `pygments`, which arrives with `rich`, offers an image formatter that imports Pillow where it is installed, and PyInstaller follows that import into the bundle. The application reads its icons as files, so the exclusion spares every bundle Pillow's extension modules and the imaging libraries -that come with them. `scripts/ci/checks/bundle.py` holds the release bundles to it. +that come with them. `scripts/verify_bundle.py` holds the release bundles to it. ## NES player driver diff --git a/docs/development/tooling.md b/docs/development/tooling.md index 87921a2e4..05cb4f251 100644 --- a/docs/development/tooling.md +++ b/docs/development/tooling.md @@ -111,8 +111,10 @@ developer command does there: - **No default derived from the repository.** An emitter takes a required `--output`; a measurement defaults to the user's Documents. Nothing a developer command writes lands beside an installed package. -- **Package data is read from the package**, through `importlib.resources`, never through a path - under the repository, so it ships in the wheel and the bundle. +- **Package data is read from the package.** `package_directory` in + `sampletones_shared/paths/package.py` places a package from the import system's own record, so + the same path holds in a checkout, in the wheel and in the bundle, where PyInstaller unpacks each + package's data beside its modules. - **A tool reads the files it is given.** What a tool measures or converts arrives on its command line or in a file a run wrote; the code names no file on one machine, so every run starts from what the person running it has. @@ -123,7 +125,8 @@ imports pillow, NumPy and the like inside `run`, and a test imports the registry asserts that only the command, registry and package modules of the tools load and no heavy library does, since a startup failure in any tool module would break every invocation, the GUI included. A command reports a refused value in one line: `describe_failure` in -`sampletones_shared/utils/validation.py` renders a validation error the way a person reads it. The editable install puts `src/` on the path whole, so a checkout +`sampletones_shared/utils/validation.py` renders a validation error the way a person reads it. The +editable install puts `src/` on the path whole, so a checkout sees the tools package whatever the wheel lists; hatchling's `dev-mode-exact` stays off for that reason. @@ -131,7 +134,7 @@ reason. | Script | Target | What it does | |---|---|---| -| `bundle.py` | `make build`, `make release` | Creates `.venv-build`, installs the package with the `build` extra, checks the interpreter carries PortAudio (and Tk, for a release), writes the bundle with PyInstaller, runs its self-check, and copies the notices beside a release | +| `bundle.py` | `make build`, `make release` | Creates `.venv-build`, installs the package with the `build` extra, checks the interpreter carries PortAudio (and Tk, for a release), writes the bundle with PyInstaller, runs its self-check, and copies the notices beside a release. Every package the wheel carries brings its data files at its own package path, so the frozen application finds them where an installed one does | | `setup_environment.py` | `make setup` | Reads the NVIDIA driver, synchronizes the development environment with the matching GPU extra, installs the global `sampletones` command | | `system_dependencies.py` | `make system-deps` | Installs the system packages: apt on Debian-based Linux, Homebrew on macOS, nothing on Windows | | `build_environment.py` | CI | Prints the compiler flags a macOS build exports, one `KEY=VALUE` per line | @@ -140,15 +143,30 @@ reason. | `lint.py` | `make lint` | Runs mypy over the files `pyproject.toml` configures and pylint over `src/` and `scripts/`; `--mypy` or `--pylint` picks one, and named paths narrow both | | `formatting.py` | `make format` | Runs isort, then black, over `src/`, `tests/` and `scripts/`, or over the paths named | | `hooks.py` | `make pre-commit` | Installs the git hooks pre-commit runs at commit and at push | -| `detect_cuda.py` | via `setup_environment.py` | Maps the driver's CUDA version to the CuPy extra | | `runtime_hooks/release_environment.py` | build input | The PyInstaller runtime hook that gives a release bundle its deployment defaults | -| `ci/` | the release workflow | The gates a release passes: the tag matches the version, the bundle ships its notices and starts | - -`scripts/bootstrap/` holds what they share: the repository root (`repository.py`), the -interpreter version check (`interpreter.py`), running a command and holding it to success -(`processes.py`), a run of named passes that reports every failure at once (`passes.py`), the -build environment and the installs into it (`venv_build.py`), the preflight of the build -interpreter (`preflight.py`), and the platforms (`platforms/`). +| `verify_version_tag.py` | the release workflow | Holds the release tag to the version `pyproject.toml` records | +| `verify_bundle.py` | the release workflow | Holds the release bundle to its notices, keeps the build tools out of it, and starts its launcher | +| `archive_bundle.py` | the release workflow | Zips the release bundle into `bundles/`, named by the version and the `--label` of the platform | + +`scripts/bootstrap/` holds what they share, one fact in one place: + +- `layout.py`: the repository root and every path and list a script names: `bin`, `bundles`, + `.venv-build`, the runtime hook, the notices, the build tools a bundle leaves out, and what + `make clean` removes. +- `project.py`: what `pyproject.toml` states, read with `tomllib`: the name, the version, the + entry module, the wheel's packages, and the extras and groups the scripts install, which it + holds the file to. +- `platforms/`: what differs between systems, behind `Platform`, with `Bundling` holding what a + bundle takes on a system that builds one. +- `cuda.py`: the NVIDIA driver's CUDA version and the CuPy extra it selects. +- `interpreter.py`, `processes.py`, `passes.py`, `files.py`: the interpreter version check, + running a command and holding it to success, a run of named passes that reports every failure + at once, and removing a file or a tree. +- `venv_build.py`, `preflight.py`: the build environment and the installs into it, and the + preflight of the build interpreter. + +Every script's work is a function taking the runner and the variables; `main` parses the arguments +and passes in the real ones, and the tests pass a `RecordingRunner`. ## Who governs what @@ -162,10 +180,12 @@ interpreter (`preflight.py`), and the platforms (`platforms/`). | What a command is | `src/sampletones_shared/command.py` | | What a bootstrap script may import | `sampletones_config/boundaries/standalone.yaml` | | What differs between systems | `scripts/bootstrap/platforms/` | +| Where a script finds a path, a notice or a clean target | `scripts/bootstrap/layout.py` | +| What the scripts read from `pyproject.toml` | `scripts/bootstrap/project.py` | | Where a build installs | `scripts/bootstrap/venv_build.py` | | What a bundle has to carry before it is built | `scripts/bootstrap/preflight.py` | | The PyInstaller invocation | `scripts/bundle.py` | | The test passes and the command each runs | `scripts/run_tests.py` | | What `make lint` and `make format` sweep | `scripts/lint.py`, `scripts/formatting.py` | -| The GPU extra a machine gets | `scripts/detect_cuda.py` | -| What a release bundle is held to | `scripts/ci/checks/bundle.py` | +| The GPU extra a machine gets | `scripts/bootstrap/cuda.py` | +| What a release is held to | `scripts/verify_version_tag.py`, `scripts/verify_bundle.py` | diff --git a/scripts/ci/zip_bundle.py b/scripts/archive_bundle.py similarity index 55% rename from scripts/ci/zip_bundle.py rename to scripts/archive_bundle.py index f8a523bef..171653c27 100644 --- a/scripts/ci/zip_bundle.py +++ b/scripts/archive_bundle.py @@ -4,7 +4,24 @@ from pathlib import Path from typing import Final, List, Sequence +from bootstrap.layout import BUNDLES, DISTRIBUTION, repository_root +from bootstrap.project import Project, read_project + ARCHIVE_COMPRESSION: Final[int] = zipfile.ZIP_DEFLATED +ARCHIVE_SUFFIX: Final[str] = ".zip" + + +def archive_root(project: Project, label: str) -> str: + """The directory every archived entry sits under, which names the archive too. + + Args: + project: The project, whose name and version lead the name. + label: The platform the bundle was built for, such as ``windows-x86_64``. + + Returns: + str: The name, such as ``sampletones-v0.3.0-windows-x86_64``. + """ + return f"{project.name}-v{project.version}-{label}" def bundle_entries(source: Path) -> List[Path]: @@ -18,7 +35,7 @@ def archive_name(path: Path, *, source: Path, root: str) -> str: def write_archive(source: Path, archive: Path, *, root: str) -> List[Path]: - """Archive a built bundle directory, placing every entry under ``root``. + """Archives a built bundle directory, placing every entry under ``root``. Each entry is read where it lies, which keeps the archive available while a virus scanner or a process that ran the executable holds a handle inside the directory, and carries over the @@ -34,20 +51,21 @@ def write_archive(source: Path, archive: Path, *, root: str) -> List[Path]: def main(argv: Sequence[str]) -> int: - """Archive a built bundle directory under a versioned root directory.""" - parser = argparse.ArgumentParser(description="Archive a built bundle directory under a versioned root.") - parser.add_argument("source", type=Path, help="the built bundle directory, such as bin/sampletones") - parser.add_argument("archive", type=Path, help="the path of the zip file to write") - parser.add_argument("--root", required=True, help="the directory name every archived entry sits under") + """Archives the release bundle into the bundles directory, named by the version and the platform.""" + parser = argparse.ArgumentParser(description="Archive the release bundle under a versioned root.") + parser.add_argument("--label", required=True, help="the platform the bundle was built for, such as windows-x86_64") arguments = parser.parse_args(list(argv)) - source: Path = arguments.source - archive: Path = arguments.archive + root = repository_root() + project = read_project(root) + source = root / DISTRIBUTION / project.name if not source.is_dir(): print(f"::error::Bundle directory {source} is missing") return 1 - entries = write_archive(source, archive, root=arguments.root) + name = archive_root(project, arguments.label) + archive = root / BUNDLES / f"{name}{ARCHIVE_SUFFIX}" + entries = write_archive(source, archive, root=name) print(f"Archived {len(entries)} entries from {source} into {archive}") return 0 diff --git a/scripts/bootstrap/cuda.py b/scripts/bootstrap/cuda.py new file mode 100644 index 000000000..309469fd3 --- /dev/null +++ b/scripts/bootstrap/cuda.py @@ -0,0 +1,163 @@ +import re +import shutil +import subprocess +from dataclasses import dataclass +from pathlib import Path +from typing import Final, Mapping, Optional, Sequence, Tuple + +from bootstrap.platforms.protocol import Platform +from bootstrap.project import GPU_CUDA11_EXTRA, GPU_EXTRA + +NVIDIA_SMI: Final[str] = "nvidia-smi" +CUDA12_MAJOR: Final[int] = 12 +CUDA11_MAJOR: Final[int] = 11 +CUDA_VERSION_PATTERN: Final[re.Pattern[str]] = re.compile(r"CUDA Version\s*:?\s*(\d+)\.(\d+)") +QUERY_ARGUMENTS: Final[Tuple[Tuple[str, ...], ...]] = ((), ("-q",)) + + +@dataclass(frozen=True, kw_only=True) +class CudaDetection: + """The NVIDIA driver capability observed on the host and the CuPy extra it maps to. + + Attributes: + system: The system the detection ran on. + nvidia_smi: The driver's ``nvidia-smi``, where one was found. + cuda_version: The newest CUDA version the driver supports, as ``(major, minor)``. + extra: The optional-dependency extra matching the driver, or ``None`` for the CPU backend. + reason: One line saying what was found and what it selects. + """ + + system: str + nvidia_smi: Optional[Path] + cuda_version: Optional[Tuple[int, int]] + extra: Optional[str] + reason: str + + +def find_nvidia_smi(platform: Platform, environment: Mapping[str, str]) -> Optional[Path]: + """The driver's ``nvidia-smi``: on ``PATH``, or at a location the system's driver installs it. + + Args: + platform: The system, which names the locations off ``PATH``. + environment: The variables those locations are read from. + + Returns: + Optional[Path]: The executable, or ``None`` where no NVIDIA driver is installed. + """ + located = shutil.which(NVIDIA_SMI) + if located is not None: + return Path(located) + + return next((candidate for candidate in platform.nvidia_smi_locations(environment) if candidate.exists()), None) + + +def _run_nvidia_smi(nvidia_smi: Path, arguments: Sequence[str]) -> Optional[str]: + try: + completed = subprocess.run( + [str(nvidia_smi), *arguments], + capture_output=True, + text=True, + check=False, + ) + except (OSError, subprocess.SubprocessError): + return None + + if completed.returncode != 0: + return None + + return completed.stdout + completed.stderr + + +def query_driver_cuda_version(nvidia_smi: Path) -> Optional[Tuple[int, int]]: + """The newest CUDA version the driver supports, as ``(major, minor)``. + + ``nvidia-smi`` reports the driver's newest supported CUDA version, which governs CuPy wheel + selection: CuPy wheels bind to the driver, whatever CUDA Toolkit the system carries. The + default table carries the value, and ``-q`` provides a structured fallback for formats that + omit it. + """ + for arguments in QUERY_ARGUMENTS: + output = _run_nvidia_smi(nvidia_smi, arguments) + if output is None: + continue + + match = CUDA_VERSION_PATTERN.search(output) + if match is not None: + return int(match.group(1)), int(match.group(2)) + + return None + + +def select_extra(cuda_version: Optional[Tuple[int, int]]) -> Optional[str]: + """The optional-dependency extra matching a driver's CUDA version. + + CUDA 12 and newer drivers map to the ``gpu`` extra (``cupy-cuda12x``): a CUDA 12 wheel runs on + every 12.x driver through Enhanced Compatibility and on newer drivers through backward + compatibility. CUDA 11 drivers map to the legacy ``gpu-cuda11`` extra. Older drivers map to + ``None``, keeping the CPU backend. + """ + if cuda_version is None: + return None + + major = cuda_version[0] + if major >= CUDA12_MAJOR: + return GPU_EXTRA + + if major == CUDA11_MAJOR: + return GPU_CUDA11_EXTRA + + return None + + +def _describe(*, cuda_version: Optional[Tuple[int, int]], extra: Optional[str]) -> str: + if cuda_version is None: + return "NVIDIA driver detected, though its CUDA version was unreadable; keeping the CPU (NumPy) backend." + + major, minor = cuda_version + if extra is None: + return ( + f"Detected CUDA {major}.{minor}, which predates the supported CuPy builds; " + "keeping the CPU (NumPy) backend." + ) + + return f"Detected an NVIDIA driver supporting CUDA {major}.{minor}; selecting the '{extra}' extra." + + +def detect(platform: Platform, environment: Mapping[str, str]) -> CudaDetection: + """The CUDA capability of the host and the CuPy extra that matches it. + + Args: + platform: The system the detection runs on. + environment: The variables the driver's locations are read from. + + Returns: + CudaDetection: What was found and the extra it selects. + """ + if not platform.cuda: + return CudaDetection( + system=platform.name, + nvidia_smi=None, + cuda_version=None, + extra=None, + reason=f"NVIDIA CUDA runs on Linux and Windows; on {platform.name}, keeping the CPU (NumPy) backend.", + ) + + nvidia_smi = find_nvidia_smi(platform, environment) + if nvidia_smi is None: + return CudaDetection( + system=platform.name, + nvidia_smi=None, + cuda_version=None, + extra=None, + reason="No NVIDIA driver detected (nvidia-smi is absent); keeping the CPU (NumPy) backend.", + ) + + cuda_version = query_driver_cuda_version(nvidia_smi) + extra = select_extra(cuda_version) + return CudaDetection( + system=platform.name, + nvidia_smi=nvidia_smi, + cuda_version=cuda_version, + extra=extra, + reason=_describe(cuda_version=cuda_version, extra=extra), + ) diff --git a/scripts/bootstrap/files.py b/scripts/bootstrap/files.py new file mode 100644 index 000000000..d38000af7 --- /dev/null +++ b/scripts/bootstrap/files.py @@ -0,0 +1,22 @@ +import shutil +from pathlib import Path + + +def remove_path(path: Path) -> bool: + """Removes what stands at ``path``, a directory with everything under it or a single file. + + Args: + path: The file or directory. + + Returns: + bool: Whether anything stood there. + """ + if path.is_dir(): + shutil.rmtree(path) + return True + + if path.exists(): + path.unlink() + return True + + return False diff --git a/scripts/bootstrap/layout.py b/scripts/bootstrap/layout.py new file mode 100644 index 000000000..30d91ac04 --- /dev/null +++ b/scripts/bootstrap/layout.py @@ -0,0 +1,31 @@ +from pathlib import Path +from typing import Final, Tuple + +REPOSITORY_ROOT: Final[Path] = Path(__file__).resolve().parents[2] +PROJECT_FILE: Final[str] = "pyproject.toml" +SOURCE_DIRECTORY: Final[str] = "src" +DISTRIBUTION: Final[str] = "bin" +BUNDLES: Final[str] = "bundles" +BUILD_ENVIRONMENT: Final[str] = ".venv-build" +RELEASE_HOOK: Final[str] = "scripts/runtime_hooks/release_environment.py" +NOTICES: Final[Tuple[str, ...]] = ("LICENSE", "THIRD-PARTY-NOTICES.md", "THIRD-PARTY-LICENSES.txt") +BUILD_TOOLS: Final[Tuple[str, ...]] = ("PIL",) +CLEAN_ARTIFACTS: Final[Tuple[str, ...]] = (DISTRIBUTION, "build", "dist", "htmlcov", ".coverage") +CLEAN_PATTERNS: Final[Tuple[str, ...]] = ("*.spec",) +CACHE_DIRECTORIES: Final[Tuple[str, ...]] = ("__pycache__",) +CACHE_DIRECTORY_SUFFIXES: Final[Tuple[str, ...]] = (".egg-info",) +CACHE_FILE_SUFFIXES: Final[Tuple[str, ...]] = (".pyc",) +ENVIRONMENTS: Final[Tuple[str, ...]] = (".git", ".venv", BUILD_ENVIRONMENT) + + +def repository_root() -> Path: + """The repository the scripts belong to, which every build and clean-up runs against. + + Raises: + FileNotFoundError: If the scripts lie outside a SampleToNES checkout. + """ + project = REPOSITORY_ROOT / PROJECT_FILE + if not project.is_file() or not (REPOSITORY_ROOT / SOURCE_DIRECTORY).is_dir(): + raise FileNotFoundError(f"SampleToNES project root not found (expected {project} beside {SOURCE_DIRECTORY})") + + return REPOSITORY_ROOT diff --git a/scripts/bootstrap/platforms/bundling.py b/scripts/bootstrap/platforms/bundling.py new file mode 100644 index 000000000..02eaad2ac --- /dev/null +++ b/scripts/bootstrap/platforms/bundling.py @@ -0,0 +1,38 @@ +from dataclasses import dataclass +from pathlib import Path + + +@dataclass(frozen=True) +class Bundling: + """What building a standalone bundle takes on one system. + + Attributes: + icon: The icon file the bundle is stamped with, relative to the repository. + executable_suffix: What the launcher's file name ends with. + pyaudio_advice: How to supply PortAudio to the build interpreter. + tkinter_advice: How to supply Tk to the build interpreter for a release bundle. + tkinter_warning: What a development bundle built without Tk does about file dialogs. + """ + + icon: str + executable_suffix: str + pyaudio_advice: str + tkinter_advice: str + tkinter_warning: str + + def launcher(self, distribution: Path, *, name: str, release: bool) -> Path: + """The executable PyInstaller writes under ``distribution``. + + Args: + distribution: The directory the bundle is written into. + name: The bundle's name. + release: Whether the bundle is a release, which is a directory beside its launcher. + + Returns: + Path: The launcher. + """ + executable = f"{name}{self.executable_suffix}" + if release: + return distribution / name / executable + + return distribution / executable diff --git a/scripts/bootstrap/platforms/linux.py b/scripts/bootstrap/platforms/linux.py index 79beb6c57..324b07f2f 100644 --- a/scripts/bootstrap/platforms/linux.py +++ b/scripts/bootstrap/platforms/linux.py @@ -1,8 +1,9 @@ from pathlib import Path -from typing import Final, Optional, Sequence, Tuple +from typing import Dict, Final, Mapping, Optional, Sequence, Tuple + +from bootstrap.platforms.bundling import Bundling LINUX: Final[str] = "Linux" -POSIX_LAUNCHER: Final[str] = "sampletones" POSIX_INTERPRETER: Final[Tuple[str, str]] = ("bin", "python") PNG_ICON: Final[str] = "src/sampletones_assets/icons/sampletones.png" SYSTEM_PACKAGES: Final[Tuple[str, ...]] = ( @@ -24,6 +25,18 @@ "libxrender1", "libxxf86vm1", ) +BUNDLING: Final[Bundling] = Bundling( + icon=PNG_ICON, + executable_suffix="", + pyaudio_advice=( + "Run 'make system-deps' to install the PortAudio packages, then 'make build' to reinstall the dependencies." + ), + tkinter_advice="Run 'make system-deps' to install python3-tk, then build again.", + tkinter_warning=( + "This bundle opens file dialogs through zenity or kdialog, which the machine running it has to " + "provide. Run 'make system-deps' to install python3-tk and carry Tk as a self-contained fallback." + ), +) def posix_interpreter(environment: Path) -> Path: @@ -31,14 +44,6 @@ def posix_interpreter(environment: Path) -> Path: return environment.joinpath(*POSIX_INTERPRETER) -def posix_launcher(distribution: Path, *, release: bool) -> Path: - """The executable PyInstaller writes under ``distribution`` on a POSIX system.""" - if release: - return distribution / POSIX_LAUNCHER / POSIX_LAUNCHER - - return distribution / POSIX_LAUNCHER - - class Linux: """A Debian-based Linux: packages through apt, a launcher without an extension.""" @@ -47,35 +52,14 @@ def name(self) -> str: return LINUX @property - def bundles(self) -> bool: + def cuda(self) -> bool: return True - @property - def icon(self) -> str: - return PNG_ICON - - @property - def pyaudio_advice(self) -> str: - return ( - "Run 'make system-deps' to install the PortAudio packages, then 'make build' to reinstall the dependencies." - ) - - @property - def tkinter_advice(self) -> str: - return "Run 'make system-deps' to install python3-tk, then build again." - - @property - def tkinter_warning(self) -> str: - return ( - "This bundle opens file dialogs through zenity or kdialog, which the machine running it has to " - "provide. Run 'make system-deps' to install python3-tk and carry Tk as a self-contained fallback." - ) - def interpreter(self, environment: Path) -> Path: return posix_interpreter(environment) - def launcher(self, distribution: Path, *, release: bool) -> Path: - return posix_launcher(distribution, release=release) + def bundling(self) -> Bundling: + return BUNDLING def missing_package_manager(self) -> Optional[str]: return None @@ -86,6 +70,14 @@ def system_packages(self) -> Sequence[Sequence[str]]: ("sudo", "apt-get", "install", "-y", *SYSTEM_PACKAGES), ) - def build_environment(self, *, machine: str, portaudio_prefix: str) -> Sequence[str]: - del machine, portaudio_prefix + def setup_variables(self, base: Mapping[str, str], *, machine: str) -> Dict[str, str]: + del machine + return dict(base) + + def build_flags(self, *, machine: str) -> Sequence[str]: + del machine + return () + + def nvidia_smi_locations(self, environment: Mapping[str, str]) -> Sequence[Path]: + del environment return () diff --git a/scripts/bootstrap/platforms/macos.py b/scripts/bootstrap/platforms/macos.py index 4fdfa2cee..2c3b02474 100644 --- a/scripts/bootstrap/platforms/macos.py +++ b/scripts/bootstrap/platforms/macos.py @@ -1,47 +1,71 @@ import shutil +import subprocess from pathlib import Path -from typing import Final, Optional, Sequence +from typing import Dict, Final, Mapping, Optional, Sequence -from bootstrap.platforms.linux import PNG_ICON, posix_interpreter, posix_launcher +from bootstrap.platforms.bundling import Bundling +from bootstrap.platforms.linux import posix_interpreter DARWIN: Final[str] = "Darwin" HOMEBREW: Final[str] = "brew" HOMEBREW_SITE: Final[str] = "https://brew.sh" PORTAUDIO: Final[str] = "portaudio" +ARCHFLAGS: Final[str] = "ARCHFLAGS" +NO_BUNDLE: Final[str] = ( + "ERROR: a standalone bundle is built on Linux and Windows.\n" + "On macOS, SampleToNES runs from source:\n" + "\n" + " make system-deps\n" + " make setup\n" + " make run\n" + "\n" + "See docs/guide/installation.md for the full steps." +) -class MacOS: - """macOS: PortAudio through Homebrew, and the application run from source.""" +def homebrew_prefix(package: str) -> str: + """Where Homebrew installed ``package``, or empty where Homebrew or the package is absent.""" + if shutil.which(HOMEBREW) is None: + return "" - @property - def name(self) -> str: - return DARWIN + completed = subprocess.run( + [HOMEBREW, "--prefix", package], + capture_output=True, + text=True, + check=False, + ) + if completed.returncode != 0: + return "" - @property - def bundles(self) -> bool: - return False + return completed.stdout.strip() - @property - def icon(self) -> str: - return PNG_ICON - @property - def pyaudio_advice(self) -> str: - return "Run 'make system-deps' to install PortAudio through Homebrew." +def architecture_flag(machine: str) -> str: + """The flag compiling an extension for the machine's own architecture alone.""" + return f"-arch {machine}" + + +class MacOS: + """macOS: PortAudio through Homebrew, and the application run from source. + + Homebrew's PortAudio carries the machine's own architecture while a python.org interpreter + compiles for both, so the setup and a build pin ``ARCHFLAGS`` to the native one. NVIDIA CUDA + has no driver on the system, so the CPU backend is the one it runs. + """ @property - def tkinter_advice(self) -> str: - return "Install Python from python.org, which includes Tk." + def name(self) -> str: + return DARWIN @property - def tkinter_warning(self) -> str: - return "This bundle opens no file dialogs. Install Python from python.org to include Tk." + def cuda(self) -> bool: + return False def interpreter(self, environment: Path) -> Path: return posix_interpreter(environment) - def launcher(self, distribution: Path, *, release: bool) -> Path: - return posix_launcher(distribution, release=release) + def bundling(self) -> Bundling: + raise SystemExit(NO_BUNDLE) def missing_package_manager(self) -> Optional[str]: if shutil.which(HOMEBREW) is not None: @@ -55,20 +79,28 @@ def missing_package_manager(self) -> Optional[str]: def system_packages(self) -> Sequence[Sequence[str]]: return ((HOMEBREW, "install", PORTAUDIO),) - def build_environment(self, *, machine: str, portaudio_prefix: str) -> Sequence[str]: + def setup_variables(self, base: Mapping[str, str], *, machine: str) -> Dict[str, str]: + return {**base, ARCHFLAGS: architecture_flag(machine)} + + def build_flags(self, *, machine: str) -> Sequence[str]: """The flags compiling audio playback against Homebrew's PortAudio on the native architecture. Raises: - SystemExit: If Homebrew reported no PortAudio prefix. + SystemExit: If Homebrew reports no PortAudio prefix. """ - if not portaudio_prefix: + prefix = homebrew_prefix(PORTAUDIO) + if not prefix: raise SystemExit( "ERROR: Homebrew is required to locate the PortAudio headers and library.\n" "Run 'make system-deps' first." ) return ( - f"CFLAGS=-I{portaudio_prefix}/include", - f"LDFLAGS=-L{portaudio_prefix}/lib", - f"ARCHFLAGS=-arch {machine}", + f"CFLAGS=-I{prefix}/include", + f"LDFLAGS=-L{prefix}/lib", + f"{ARCHFLAGS}={architecture_flag(machine)}", ) + + def nvidia_smi_locations(self, environment: Mapping[str, str]) -> Sequence[Path]: + del environment + return () diff --git a/scripts/bootstrap/platforms/protocol.py b/scripts/bootstrap/platforms/protocol.py index addb7db55..469e5d880 100644 --- a/scripts/bootstrap/platforms/protocol.py +++ b/scripts/bootstrap/platforms/protocol.py @@ -1,5 +1,7 @@ from pathlib import Path -from typing import Optional, Protocol, Sequence +from typing import Dict, Mapping, Optional, Protocol, Sequence + +from bootstrap.platforms.bundling import Bundling class Platform(Protocol): @@ -10,24 +12,8 @@ def name(self) -> str: """The name ``platform.system()`` reports for the system.""" @property - def bundles(self) -> bool: - """Whether a standalone bundle is built on the system.""" - - @property - def icon(self) -> str: - """The icon file a bundle is stamped with, relative to the repository.""" - - @property - def pyaudio_advice(self) -> str: - """How to supply PortAudio to the build interpreter.""" - - @property - def tkinter_advice(self) -> str: - """How to supply Tk to the build interpreter for a release bundle.""" - - @property - def tkinter_warning(self) -> str: - """What a development bundle built without Tk does about file dialogs.""" + def cuda(self) -> bool: + """Whether an NVIDIA driver with CUDA can run on the system.""" def interpreter(self, environment: Path) -> Path: """The interpreter a virtual environment at ``environment`` runs. @@ -39,15 +25,14 @@ def interpreter(self, environment: Path) -> Path: Path: The interpreter. """ - def launcher(self, distribution: Path, *, release: bool) -> Path: - """The executable PyInstaller writes under ``distribution``. - - Args: - distribution: The directory the bundle is written into. - release: Whether the bundle is a release, which is a directory beside its launcher. + def bundling(self) -> Bundling: + """What building a standalone bundle takes on the system. Returns: - Path: The launcher. + Bundling: The icon, the launcher's name and the advice a build gives. + + Raises: + SystemExit: If the system builds no bundle, naming how SampleToNES runs there. """ def missing_package_manager(self) -> Optional[str]: @@ -56,14 +41,33 @@ def missing_package_manager(self) -> Optional[str]: def system_packages(self) -> Sequence[Sequence[str]]: """The commands that install the system packages the application needs, in order.""" - def build_environment(self, *, machine: str, portaudio_prefix: str) -> Sequence[str]: + def setup_variables(self, base: Mapping[str, str], *, machine: str) -> Dict[str, str]: + """The variables the development setup runs under: ``base`` and what the system adds to it. + + Args: + base: The caller's variables. + machine: The processor architecture ``platform.machine()`` reports. + + Returns: + Dict[str, str]: The variables. + """ + + def build_flags(self, *, machine: str) -> Sequence[str]: """The ``KEY=VALUE`` lines a build exports so audio playback compiles against PortAudio. Args: machine: The processor architecture ``platform.machine()`` reports. - portaudio_prefix: Where the package manager installed PortAudio, or empty where the - system carries it without one. Returns: Sequence[str]: The lines, empty where the build needs none. """ + + def nvidia_smi_locations(self, environment: Mapping[str, str]) -> Sequence[Path]: + """Where the NVIDIA driver installs ``nvidia-smi`` outside ``PATH``. + + Args: + environment: The variables the locations are read from. + + Returns: + Sequence[Path]: The candidates, in the order they are tried. + """ diff --git a/scripts/bootstrap/platforms/windows.py b/scripts/bootstrap/platforms/windows.py index 7f02ef126..41e8d7382 100644 --- a/scripts/bootstrap/platforms/windows.py +++ b/scripts/bootstrap/platforms/windows.py @@ -1,48 +1,47 @@ from pathlib import Path -from typing import Final, Optional, Sequence, Tuple +from typing import Dict, Final, Mapping, Optional, Sequence, Tuple + +from bootstrap.platforms.bundling import Bundling WINDOWS: Final[str] = "Windows" -WINDOWS_LAUNCHER: Final[str] = "sampletones.exe" -BUNDLE_DIRECTORY: Final[str] = "sampletones" WINDOWS_INTERPRETER: Final[Tuple[str, str]] = ("Scripts", "python.exe") ICO_ICON: Final[str] = "src/sampletones_assets/icons/sampletones.ico" +EXECUTABLE_SUFFIX: Final[str] = ".exe" +NVIDIA_SMI_LOCATIONS: Final[Tuple[Tuple[str, str], ...]] = ( + ("SystemRoot", "System32/nvidia-smi.exe"), + ("ProgramFiles", "NVIDIA Corporation/NVSMI/nvidia-smi.exe"), +) +BUNDLING: Final[Bundling] = Bundling( + icon=ICO_ICON, + executable_suffix=EXECUTABLE_SUFFIX, + pyaudio_advice="Run install.bat from the project root to reinstall the dependencies.", + tkinter_advice="Install Python from python.org or the Microsoft Store, which both include Tk, then build again.", + tkinter_warning=( + "This bundle opens no file dialogs. Install Python from python.org or the Microsoft Store to include Tk." + ), +) class Windows: - """Windows: the official Python installer carries what the application needs.""" + """Windows: the official Python installer carries what the application needs. + + The NVIDIA driver installs ``nvidia-smi.exe`` into fixed system locations that ``PATH`` may + leave out, so CUDA detection tries those too. + """ @property def name(self) -> str: return WINDOWS @property - def bundles(self) -> bool: + def cuda(self) -> bool: return True - @property - def icon(self) -> str: - return ICO_ICON - - @property - def pyaudio_advice(self) -> str: - return "Run install.bat from the project root to reinstall the dependencies." - - @property - def tkinter_advice(self) -> str: - return "Install Python from python.org or the Microsoft Store, which both include Tk, then build again." - - @property - def tkinter_warning(self) -> str: - return "This bundle opens no file dialogs. Install Python from python.org or the Microsoft Store to include Tk." - def interpreter(self, environment: Path) -> Path: return environment.joinpath(*WINDOWS_INTERPRETER) - def launcher(self, distribution: Path, *, release: bool) -> Path: - if release: - return distribution / BUNDLE_DIRECTORY / WINDOWS_LAUNCHER - - return distribution / WINDOWS_LAUNCHER + def bundling(self) -> Bundling: + return BUNDLING def missing_package_manager(self) -> Optional[str]: return None @@ -50,6 +49,17 @@ def missing_package_manager(self) -> Optional[str]: def system_packages(self) -> Sequence[Sequence[str]]: return () - def build_environment(self, *, machine: str, portaudio_prefix: str) -> Sequence[str]: - del machine, portaudio_prefix + def setup_variables(self, base: Mapping[str, str], *, machine: str) -> Dict[str, str]: + del machine + return dict(base) + + def build_flags(self, *, machine: str) -> Sequence[str]: + del machine return () + + def nvidia_smi_locations(self, environment: Mapping[str, str]) -> Sequence[Path]: + return tuple( + Path(environment[variable]) / relative + for variable, relative in NVIDIA_SMI_LOCATIONS + if environment.get(variable) + ) diff --git a/scripts/bootstrap/preflight.py b/scripts/bootstrap/preflight.py index 8ba01e08b..1757f71f7 100644 --- a/scripts/bootstrap/preflight.py +++ b/scripts/bootstrap/preflight.py @@ -1,7 +1,7 @@ from pathlib import Path from typing import Final, Mapping -from bootstrap.platforms.protocol import Platform +from bootstrap.platforms.bundling import Bundling from bootstrap.processes import Runner PYAUDIO: Final[str] = "pyaudio" @@ -39,7 +39,7 @@ def can_import( def check_build_interpreter( python: Path, - platform: Platform, + bundling: Bundling, *, release: bool, runner: Runner, @@ -54,7 +54,7 @@ def check_build_interpreter( Args: python: The build environment's interpreter. - platform: The system the build runs on. + bundling: What building a bundle takes on the system, whose advice a refusal gives. release: Whether the bundle is a release. runner: What runs the probes. cwd: The directory the probes run in. @@ -67,7 +67,7 @@ def check_build_interpreter( if not can_import(python, PYAUDIO, runner=runner, cwd=cwd, environment=environment): raise SystemExit( "ERROR: the build interpreter cannot import pyaudio, so the bundle would carry no audio playback.\n" - f"{platform.pyaudio_advice}" + f"{bundling.pyaudio_advice}" ) print(f"{PYAUDIO}: available") @@ -78,7 +78,7 @@ def check_build_interpreter( if release: raise SystemExit( "ERROR: the build interpreter cannot import tkinter, so a release bundle would depend on the " - f"machine running it for file dialogs.\n{platform.tkinter_advice}" + f"machine running it for file dialogs.\n{bundling.tkinter_advice}" ) - print(f"WARNING: the build interpreter cannot import tkinter. {platform.tkinter_warning}") + print(f"WARNING: the build interpreter cannot import tkinter. {bundling.tkinter_warning}") diff --git a/scripts/bootstrap/project.py b/scripts/bootstrap/project.py new file mode 100644 index 000000000..bc8a6ef02 --- /dev/null +++ b/scripts/bootstrap/project.py @@ -0,0 +1,88 @@ +import tomllib +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Dict, Final, Tuple + +from bootstrap.layout import PROJECT_FILE, SOURCE_DIRECTORY + +BUILD_EXTRA: Final[str] = "build" +GPU_EXTRA: Final[str] = "gpu" +GPU_CUDA11_EXTRA: Final[str] = "gpu-cuda11" +DEVELOPMENT_GROUP: Final[str] = "dev" +NAMED_EXTRAS: Final[Tuple[str, ...]] = (BUILD_EXTRA, GPU_EXTRA, GPU_CUDA11_EXTRA) +NAMED_GROUPS: Final[Tuple[str, ...]] = (DEVELOPMENT_GROUP,) +ENTRY_SEPARATOR: Final[str] = ":" +MODULE_SEPARATOR: Final[str] = "." +MODULE_SUFFIX: Final[str] = ".py" + + +@dataclass(frozen=True) +class Project: + """What ``pyproject.toml`` states that a build, a setup and a release gate read. + + Attributes: + name: The distribution's name, which the command and the bundle carry too. + version: The version a release is tagged with. + entry_module: The module the command runs, read from ``[project.scripts]``. + packages: The import packages the wheel carries. + extras: The optional-dependency extras. + groups: The dependency groups. + """ + + name: str + version: str + entry_module: str + packages: Tuple[str, ...] + extras: Tuple[str, ...] + groups: Tuple[str, ...] + + @property + def entry_script(self) -> str: + """The entry module as a path from the repository root, which PyInstaller starts from.""" + return f"{SOURCE_DIRECTORY}/{self.entry_module.replace(MODULE_SEPARATOR, '/')}{MODULE_SUFFIX}" + + +def parse_project(document: Dict[str, Any]) -> Project: + """The project a parsed ``pyproject.toml`` states, held to the names the scripts rely on. + + Args: + document: The file as ``tomllib`` reads it. + + Returns: + Project: The project. + + Raises: + SystemExit: If the file lacks the command, an extra or a group the scripts name. + """ + table = document["project"] + name: str = table["name"] + scripts: Dict[str, str] = table.get("scripts", {}) + if name not in scripts: + raise SystemExit(f"ERROR: {PROJECT_FILE} names no '{name}' command under [project.scripts].") + + project = Project( + name=name, + version=table["version"], + entry_module=scripts[name].split(ENTRY_SEPARATOR, 1)[0], + packages=tuple( + Path(package).name for package in document["tool"]["hatch"]["build"]["targets"]["wheel"]["packages"] + ), + extras=tuple(table.get("optional-dependencies", {})), + groups=tuple(document.get("dependency-groups", {})), + ) + missing = [extra for extra in NAMED_EXTRAS if extra not in project.extras] + missing.extend(group for group in NAMED_GROUPS if group not in project.groups) + if missing: + raise SystemExit(f"ERROR: {PROJECT_FILE} lacks what the scripts install: {', '.join(missing)}.") + + return project + + +def read_project(root: Path) -> Project: + """The project the repository at ``root`` states. + + Raises: + SystemExit: If the file lacks the command, an extra or a group the scripts name. + """ + with (root / PROJECT_FILE).open("rb") as handle: + return parse_project(tomllib.load(handle)) diff --git a/scripts/bootstrap/repository.py b/scripts/bootstrap/repository.py deleted file mode 100644 index 60c84a159..000000000 --- a/scripts/bootstrap/repository.py +++ /dev/null @@ -1,17 +0,0 @@ -from pathlib import Path -from typing import Final - -REPOSITORY_ROOT: Final[Path] = Path(__file__).resolve().parents[2] -ENTRY_PACKAGE: Final[Path] = REPOSITORY_ROOT / "src" / "sampletones" - - -def repository_root() -> Path: - """The repository the scripts belong to, which every build and clean-up runs against. - - Raises: - FileNotFoundError: If the scripts lie outside a SampleToNES checkout. - """ - if not ENTRY_PACKAGE.is_dir(): - raise FileNotFoundError(f"SampleToNES project root not found (expected {ENTRY_PACKAGE})") - - return REPOSITORY_ROOT diff --git a/scripts/bootstrap/venv_build.py b/scripts/bootstrap/venv_build.py index f7e83e883..7aab0b423 100644 --- a/scripts/bootstrap/venv_build.py +++ b/scripts/bootstrap/venv_build.py @@ -2,14 +2,14 @@ from pathlib import Path from typing import Dict, Final, Mapping, Sequence +from bootstrap.layout import BUILD_ENVIRONMENT from bootstrap.platforms.protocol import Platform from bootstrap.processes import Runner, expect_success -BUILD_ENVIRONMENT: Final[str] = ".venv-build" PIP_REQUIRE_VIRTUALENV: Final[str] = "PIP_REQUIRE_VIRTUALENV" -def build_environment( +def ensure_build_venv( root: Path, platform: Platform, *, diff --git a/scripts/build_environment.py b/scripts/build_environment.py index cf952a14c..e749fd293 100644 --- a/scripts/build_environment.py +++ b/scripts/build_environment.py @@ -1,43 +1,17 @@ import argparse import platform as running -import shutil -import subprocess import sys -from typing import Final, Sequence +from typing import Sequence from bootstrap.platforms.factory import current_platform -HOMEBREW: Final[str] = "brew" -PORTAUDIO: Final[str] = "portaudio" - - -def portaudio_prefix() -> str: - """Where Homebrew installed PortAudio, or empty where Homebrew is absent.""" - if shutil.which(HOMEBREW) is None: - return "" - - completed = subprocess.run( - [HOMEBREW, "--prefix", PORTAUDIO], - capture_output=True, - text=True, - check=False, - ) - if completed.returncode != 0: - return "" - - return completed.stdout.strip() - def main(argv: Sequence[str]) -> int: """Prints the variables a build exports so audio playback compiles, one ``KEY=VALUE`` per line.""" parser = argparse.ArgumentParser(description="Print the build environment audio playback compiles under.") parser.parse_args(list(argv)) - lines = current_platform().build_environment( - machine=running.machine(), - portaudio_prefix=portaudio_prefix(), - ) - for line in lines: + for line in current_platform().build_flags(machine=running.machine()): print(line) return 0 diff --git a/scripts/bundle.py b/scripts/bundle.py index 46d20a5ee..0cb5f4471 100644 --- a/scripts/bundle.py +++ b/scripts/bundle.py @@ -6,40 +6,18 @@ from pathlib import Path from typing import Final, List, Mapping, Sequence, Tuple +from bootstrap.files import remove_path from bootstrap.interpreter import REQUIRED_VERSION, require_python +from bootstrap.layout import BUILD_TOOLS, DISTRIBUTION, NOTICES, RELEASE_HOOK, repository_root +from bootstrap.platforms.bundling import Bundling from bootstrap.platforms.factory import current_platform from bootstrap.platforms.protocol import Platform from bootstrap.preflight import check_build_interpreter from bootstrap.processes import Runner, expect_success, run -from bootstrap.repository import repository_root -from bootstrap.venv_build import build_environment, install +from bootstrap.project import BUILD_EXTRA, GPU_EXTRA, Project, read_project +from bootstrap.venv_build import ensure_build_venv, install -BUNDLE_NAME: Final[str] = "sampletones" -DISTRIBUTION: Final[str] = "bin" -ENTRY: Final[str] = "src/sampletones/__main__.py" -RELEASE_HOOK: Final[str] = "scripts/runtime_hooks/release_environment.py" SELF_CHECK: Final[str] = "self-check" -BUILD_EXTRA: Final[str] = "build" -GPU_EXTRA: Final[str] = "gpu" -DATA: Final[Tuple[Tuple[str, str], ...]] = ( - ("src/sampletones_assets/icons", "assets/icons"), - ("src/sampletones_assets/fonts", "assets/fonts"), - ("src/sampletones_config", "config"), - ("src/sampletones_player/driver/binary", "sampletones_player/driver/binary"), -) -DATA_SEPARATOR: Final[str] = ":" -EXCLUDED_MODULES: Final[Tuple[str, ...]] = ("PIL",) -NOTICES: Final[Tuple[str, ...]] = ("LICENSE", "THIRD-PARTY-NOTICES.md", "THIRD-PARTY-LICENSES.txt") -NO_BUNDLE: Final[str] = ( - "ERROR: a standalone bundle is built on Linux and Windows.\n" - "On macOS, SampleToNES runs from source:\n" - "\n" - " make system-deps\n" - " make setup\n" - " make run\n" - "\n" - "See docs/guide/installation.md for the full steps." -) @dataclass(frozen=True) @@ -66,14 +44,19 @@ def extras(options: BundleOptions) -> Tuple[str, ...]: def pyinstaller_command( python: Path, - platform: Platform, + bundling: Bundling, + project: Project, options: BundleOptions, ) -> List[str]: """The PyInstaller invocation that writes the bundle. + Every package the wheel carries brings its data files along at its own package path, so the + frozen application reads them through ``importlib.resources`` the way an installed one does. + Args: python: The build environment's interpreter. - platform: The system the bundle is built for. + bundling: What building a bundle takes on the system. + project: The project the bundle is built from. options: How the bundle is built. Returns: @@ -84,37 +67,36 @@ def pyinstaller_command( "-m", "PyInstaller", "--name", - BUNDLE_NAME, + project.name, "--onedir" if options.release else "--onefile", "--noconfirm", "--distpath", DISTRIBUTION, "--icon", - platform.icon, + bundling.icon, ] - for source, destination in DATA: - command.extend(("--add-data", f"{source}{DATA_SEPARATOR}{destination}")) + for package in project.packages: + command.extend(("--collect-data", package)) - command.extend(("--copy-metadata", BUNDLE_NAME)) - for module in EXCLUDED_MODULES: + command.extend(("--copy-metadata", project.name)) + for module in BUILD_TOOLS: command.extend(("--exclude-module", module)) if options.release: command.extend(("--runtime-hook", RELEASE_HOOK)) - command.append(ENTRY) + command.append(project.entry_script) return command -def remove_previous(distribution: Path) -> None: - """Removes what an earlier build left under ``distribution``, whichever shape it took.""" - for previous in (distribution / BUNDLE_NAME, distribution / f"{BUNDLE_NAME}.exe"): - if previous.is_dir(): - print(f"Removing the previous artifact: {previous}") - shutil.rmtree(previous) - elif previous.exists(): - print(f"Removing the previous artifact: {previous}") - previous.unlink() +def remove_previous(distribution: Path, bundling: Bundling, name: str) -> None: + """Removes what an earlier build left under ``distribution``, a release's directory or a single file.""" + for previous in ( + bundling.launcher(distribution, name=name, release=True).parent, + bundling.launcher(distribution, name=name, release=False), + ): + if remove_path(previous): + print(f"Removed the previous artifact: {previous}") def copy_notices(root: Path, bundle: Path) -> None: @@ -146,12 +128,14 @@ def build_bundle( Path: The launcher the bundle offers. Raises: - SystemExit: If a step fails, or PyInstaller produced no launcher. + SystemExit: If the system builds no bundle, a step fails, or PyInstaller produced no launcher. """ + bundling = platform.bundling() + project = read_project(root) if options.release: print("Release build: onedir bundle, injecting release deployment configuration") - python = build_environment( + python = ensure_build_venv( root, platform, runner=runner, @@ -166,22 +150,22 @@ def build_bundle( ) check_build_interpreter( python, - platform, + bundling, release=options.release, runner=runner, cwd=root, environment=environment, ) distribution = root / DISTRIBUTION - remove_previous(distribution) + remove_previous(distribution, bundling, project.name) print("Building executable...") expect_success( runner, - pyinstaller_command(python, platform, options), + pyinstaller_command(python, bundling, project, options), cwd=root, environment=environment, ) - launcher = platform.launcher(distribution, release=options.release) + launcher = bundling.launcher(distribution, name=project.name, release=options.release) if not launcher.is_file(): raise SystemExit(f"Build failed: PyInstaller produced no executable at {launcher}.") @@ -214,14 +198,9 @@ def main(argv: Sequence[str]) -> int: options = BundleOptions(release=arguments.release, gpu=arguments.gpu) require_python(REQUIRED_VERSION) - platform = current_platform() - if not platform.bundles: - print(NO_BUNDLE, file=sys.stderr) - return 1 - build_bundle( repository_root(), - platform, + current_platform(), options, runner=run, environment=os.environ, diff --git a/scripts/ci/__init__.py b/scripts/ci/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/scripts/ci/checks/__init__.py b/scripts/ci/checks/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/scripts/ci/checks/bundle.py b/scripts/ci/checks/bundle.py deleted file mode 100644 index acef01234..000000000 --- a/scripts/ci/checks/bundle.py +++ /dev/null @@ -1,83 +0,0 @@ -import argparse -import platform -import subprocess -import sys -from pathlib import Path -from typing import Final, List, Sequence - -WINDOWS: Final[str] = "Windows" -WINDOWS_LAUNCHER: Final[str] = "sampletones.exe" -POSIX_LAUNCHER: Final[str] = "sampletones" -VERSION_FLAG: Final[str] = "--version" - -REQUIRED_NOTICES: Final[Sequence[str]] = ( - "LICENSE", - "THIRD-PARTY-NOTICES.md", - "THIRD-PARTY-LICENSES.txt", -) - -INTERNAL_DIRECTORY: Final[str] = "_internal" -BUILD_TOOLS: Final[Sequence[str]] = ("PIL",) - - -def launcher_path(bundle: Path, *, system: str) -> Path: - """The executable a built bundle offers on the platform it was built for.""" - name = WINDOWS_LAUNCHER if system == WINDOWS else POSIX_LAUNCHER - return bundle / name - - -def missing_notices(bundle: Path) -> List[str]: - """The license and notice files a release bundle must ship that are absent from it.""" - return [name for name in REQUIRED_NOTICES if not (bundle / name).is_file()] - - -def carried_build_tools(bundle: Path) -> List[str]: - """The build-time packages found in a bundle, which the notices place on the build machine. - - A bundle carries the application and its runtime dependencies. Tooling that only draws the - assets belongs to the machine that builds it, so finding it here means the notices describe - a different set of components than the bundle ships. - """ - directories = (bundle, bundle / INTERNAL_DIRECTORY) - return [name for name in BUILD_TOOLS if any((directory / name).is_dir() for directory in directories)] - - -def main(argv: Sequence[str]) -> int: - """Confirm a built bundle ships its notices, holds to them, and that its launcher starts.""" - parser = argparse.ArgumentParser( - description="Verify a built bundle before it is archived.", - ) - parser.add_argument( - "bundle", - type=Path, - help="the built bundle directory, such as bin/sampletones", - ) - arguments = parser.parse_args(list(argv)) - - bundle: Path = arguments.bundle - absent = missing_notices(bundle) - if absent: - print(f"::error::Bundle {bundle} is missing {', '.join(absent)}") - return 1 - - carried = carried_build_tools(bundle) - if carried: - print(f"::error::Bundle {bundle} carries build-time tooling its notices leave out: {', '.join(carried)}") - return 1 - - launcher = launcher_path(bundle, system=platform.system()) - if not launcher.is_file(): - print(f"::error::Bundle {bundle} offers no launcher at {launcher}") - return 1 - - completed = subprocess.run([str(launcher), VERSION_FLAG], check=False) - if completed.returncode != 0: - print(f"::error::Launcher {launcher} exited with status {completed.returncode}") - return completed.returncode - - print(f"Bundle {bundle} ships its notices and its launcher starts") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main(sys.argv[1:])) diff --git a/scripts/ci/checks/version_tag.py b/scripts/ci/checks/version_tag.py deleted file mode 100644 index 47d4438c7..000000000 --- a/scripts/ci/checks/version_tag.py +++ /dev/null @@ -1,46 +0,0 @@ -import argparse -import sys -from typing import Final, Sequence - -TAG_PREFIX: Final[str] = "v" - - -def version_from_tag(tag: str) -> str: - """The project version a release tag names, read from the tag with its ``v`` prefix dropped.""" - return tag.removeprefix(TAG_PREFIX) - - -def tag_names_version(*, tag: str, project_version: str) -> bool: - """Whether a release tag names the version recorded in the project metadata.""" - return version_from_tag(tag) == project_version - - -def main(argv: Sequence[str]) -> int: - """Confirm a release tag and the project metadata agree on the version being released.""" - parser = argparse.ArgumentParser( - description="Compare a release tag against the project version.", - ) - parser.add_argument( - "--tag", - required=True, - help="the release tag being built, such as v0.3.0", - ) - parser.add_argument( - "--project-version", - required=True, - help="the version recorded in pyproject.toml", - ) - arguments = parser.parse_args(list(argv)) - - tag: str = arguments.tag - project_version: str = arguments.project_version - if not tag_names_version(tag=tag, project_version=project_version): - print(f"::error::Tag {tag} names a version other than the pyproject version {project_version}") - return 1 - - print(f"Version {project_version} matches tag {tag}") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main(sys.argv[1:])) diff --git a/scripts/clean.py b/scripts/clean.py index 519f36241..08a690ee6 100644 --- a/scripts/clean.py +++ b/scripts/clean.py @@ -1,49 +1,42 @@ import argparse -import os -import shutil import sys from pathlib import Path -from typing import Final, Sequence, Tuple +from typing import Sequence -from bootstrap.repository import repository_root - -ARTIFACTS: Final[Tuple[str, ...]] = ("bin", "build", "dist", "htmlcov", ".coverage") -ARTIFACT_PATTERNS: Final[Tuple[str, ...]] = ("*.spec",) -CACHE_DIRECTORIES: Final[Tuple[str, ...]] = ("__pycache__",) -CACHE_DIRECTORY_SUFFIXES: Final[Tuple[str, ...]] = (".egg-info",) -CACHE_FILE_SUFFIXES: Final[Tuple[str, ...]] = (".pyc",) -LEFT_ALONE: Final[Tuple[str, ...]] = (".git", ".venv", ".venv-build") - - -def _remove(path: Path) -> None: - if path.is_dir(): - shutil.rmtree(path) - elif path.exists(): - path.unlink() +from bootstrap.files import remove_path +from bootstrap.layout import ( + CACHE_DIRECTORIES, + CACHE_DIRECTORY_SUFFIXES, + CACHE_FILE_SUFFIXES, + CLEAN_ARTIFACTS, + CLEAN_PATTERNS, + ENVIRONMENTS, + repository_root, +) def remove_artifacts(root: Path) -> None: """Removes the build outputs and the coverage reports under ``root``.""" - for name in ARTIFACTS: - _remove(root / name) + for name in CLEAN_ARTIFACTS: + remove_path(root / name) - for pattern in ARTIFACT_PATTERNS: + for pattern in CLEAN_PATTERNS: for path in root.glob(pattern): - _remove(path) + remove_path(path) def remove_caches(root: Path) -> None: """Removes the bytecode caches and packaging leftovers under ``root``, the environments left alone.""" - for directory, subdirectories, files in os.walk(root): - subdirectories[:] = [name for name in subdirectories if name not in LEFT_ALONE] + for directory, subdirectories, files in root.walk(): + subdirectories[:] = [name for name in subdirectories if name not in ENVIRONMENTS] for name in list(subdirectories): if name in CACHE_DIRECTORIES or name.endswith(CACHE_DIRECTORY_SUFFIXES): - shutil.rmtree(Path(directory) / name) + remove_path(directory / name) subdirectories.remove(name) for name in files: if name.endswith(CACHE_FILE_SUFFIXES): - (Path(directory) / name).unlink() + remove_path(directory / name) def main(argv: Sequence[str]) -> int: diff --git a/scripts/detect_cuda.py b/scripts/detect_cuda.py deleted file mode 100644 index 9aa905228..000000000 --- a/scripts/detect_cuda.py +++ /dev/null @@ -1,235 +0,0 @@ -import argparse -import os -import platform -import re -import shutil -import subprocess -import sys -from dataclasses import dataclass -from pathlib import Path -from typing import Final, List, Optional, Sequence, Tuple - -NVIDIA_SMI: Final[str] = "nvidia-smi" - -DARWIN: Final[str] = "Darwin" -WINDOWS: Final[str] = "Windows" - -EXTRA_GPU: Final[str] = "gpu" -EXTRA_GPU_CUDA11: Final[str] = "gpu-cuda11" - -CUDA12_MAJOR: Final[int] = 12 -CUDA11_MAJOR: Final[int] = 11 - -CUDA_VERSION_PATTERN: Final[re.Pattern[str]] = re.compile(r"CUDA Version\s*:?\s*(\d+)\.(\d+)") - -WINDOWS_NVIDIA_SMI_SUBPATHS: Final[Sequence[Tuple[str, str]]] = ( - ("SystemRoot", "System32/nvidia-smi.exe"), - ("ProgramFiles", "NVIDIA Corporation/NVSMI/nvidia-smi.exe"), -) - - -@dataclass(frozen=True, kw_only=True) -class CudaDetection: - """The NVIDIA driver capability observed on the host and the CuPy extra it maps to.""" - - system: str - nvidia_smi: Optional[Path] - cuda_version: Optional[Tuple[int, int]] - extra: Optional[str] - reason: str - - -def _windows_nvidia_smi_candidates() -> List[Path]: - """Return the fixed Windows locations where the driver installs ``nvidia-smi.exe``.""" - candidates: List[Path] = [] - for variable, relative in WINDOWS_NVIDIA_SMI_SUBPATHS: - root = os.environ.get(variable) - if root: - candidates.append(Path(root) / relative) - - return candidates - - -def find_nvidia_smi(*, system: str) -> Optional[Path]: - """Return the path to ``nvidia-smi`` when an NVIDIA driver is installed. - - ``nvidia-smi`` ships with the driver and lands on ``PATH`` on Linux and Windows. On Windows - it may also sit in a fixed system location off ``PATH``, so probe those as a fallback. - """ - located = shutil.which(NVIDIA_SMI) - if located is not None: - return Path(located) - - if system == WINDOWS: - for candidate in _windows_nvidia_smi_candidates(): - if candidate.exists(): - return candidate - return None - - -def _run_nvidia_smi( - nvidia_smi: Path, - arguments: Sequence[str], -) -> Optional[str]: - """Return the combined output of an ``nvidia-smi`` invocation that succeeds.""" - try: - completed = subprocess.run( - [str(nvidia_smi), *arguments], - capture_output=True, - text=True, - check=False, - ) - except (OSError, subprocess.SubprocessError): - return None - - if completed.returncode != 0: - return None - - return completed.stdout + completed.stderr - - -def query_driver_cuda_version(nvidia_smi: Path) -> Optional[Tuple[int, int]]: - """Return the maximum CUDA version the driver supports, as ``(major, minor)``. - - ``nvidia-smi`` reports the driver's maximum supported CUDA version, which governs CuPy wheel - selection: CuPy wheels bind to the driver rather than to a system CUDA Toolkit. The default - table carries the value, and ``-q`` provides a structured fallback for formats that omit it. - """ - for arguments in ((), ("-q",)): - output = _run_nvidia_smi(nvidia_smi, arguments) - if output is None: - continue - match = CUDA_VERSION_PATTERN.search(output) - if match is not None: - return int(match.group(1)), int(match.group(2)) - - return None - - -def select_extra(cuda_version: Optional[Tuple[int, int]]) -> Optional[str]: - """Return the optional-dependency extra matching a driver's CUDA version. - - CUDA 12 and newer drivers map to the ``gpu`` extra (``cupy-cuda12x``): a CUDA 12 wheel runs on - every 12.x driver through Enhanced Compatibility and on newer drivers through backward - compatibility. CUDA 11 drivers map to the legacy ``gpu-cuda11`` extra. Older drivers map to - ``None``, keeping the CPU backend. - """ - if cuda_version is None: - return None - - major = cuda_version[0] - if major >= CUDA12_MAJOR: - return EXTRA_GPU - - if major == CUDA11_MAJOR: - return EXTRA_GPU_CUDA11 - - return None - - -def _describe( - *, - cuda_version: Optional[Tuple[int, int]], - extra: Optional[str], -) -> str: - """Return a one-line summary of a driver-present detection outcome.""" - if cuda_version is None: - return "NVIDIA driver detected, though its CUDA version was unreadable; keeping the CPU (NumPy) backend." - major, minor = cuda_version - if extra is None: - return ( - f"Detected CUDA {major}.{minor}, which predates the supported CuPy builds; " - "keeping the CPU (NumPy) backend." - ) - return f"Detected an NVIDIA driver supporting CUDA {major}.{minor}; selecting the '{extra}' extra." - - -def detect(*, system: str) -> CudaDetection: - """Return the CUDA capability of the host and the CuPy extra that matches it. - - macOS keeps the CPU backend, since NVIDIA CUDA is available only on Linux and Windows. On - those systems the driver's ``nvidia-smi`` supplies the CUDA version that drives selection. - """ - if system == DARWIN: - return CudaDetection( - system=system, - nvidia_smi=None, - cuda_version=None, - extra=None, - reason="NVIDIA CUDA is available on Linux and Windows; on macOS, keeping the CPU (NumPy) backend.", - ) - - nvidia_smi = find_nvidia_smi(system=system) - if nvidia_smi is None: - return CudaDetection( - system=system, - nvidia_smi=None, - cuda_version=None, - extra=None, - reason="No NVIDIA driver detected (nvidia-smi is absent); keeping the CPU (NumPy) backend.", - ) - - cuda_version = query_driver_cuda_version(nvidia_smi) - extra = select_extra(cuda_version) - return CudaDetection( - system=system, - nvidia_smi=nvidia_smi, - cuda_version=cuda_version, - extra=extra, - reason=_describe(cuda_version=cuda_version, extra=extra), - ) - - -def _format_version(cuda_version: Optional[Tuple[int, int]]) -> str: - """Return a human-readable CUDA version, or ``unknown`` when it was unreadable.""" - if cuda_version is None: - return "unknown" - - major, minor = cuda_version - return f"{major}.{minor}" - - -def _print_report(detection: CudaDetection) -> None: - """Print the full detection outcome to stdout for a human reader.""" - nvidia_smi = str(detection.nvidia_smi) if detection.nvidia_smi is not None else "not found" - extra = detection.extra if detection.extra is not None else "none (CPU)" - lines = [ - f"System: {detection.system}", - f"nvidia-smi: {nvidia_smi}", - f"Driver CUDA: {_format_version(detection.cuda_version)}", - f"Selected extra: {extra}", - f"Summary: {detection.reason}", - ] - print("\n".join(lines)) - - -def main(argv: Sequence[str]) -> int: - """Report the detected CUDA capability, or print the matching extra for install scripts. - - Under ``--extra`` the chosen extra name is written to stdout (empty when the CPU backend is - kept) so a Makefile can capture it, and the summary is written to stderr. Without it, a full - report is written to stdout. - """ - parser = argparse.ArgumentParser( - description="Select the CuPy extra matching the local NVIDIA driver.", - ) - parser.add_argument( - "--extra", - action="store_true", - help="print the chosen optional-dependency extra to stdout (empty when the CPU backend is kept)", - ) - arguments = parser.parse_args(list(argv)) - detection = detect(system=platform.system()) - - if arguments.extra: - print(detection.reason, file=sys.stderr) - if detection.extra is not None: - print(detection.extra) - else: - _print_report(detection) - - return 0 - - -if __name__ == "__main__": - raise SystemExit(main(sys.argv[1:])) diff --git a/scripts/formatting.py b/scripts/formatting.py index 5129c5cb7..410d7f8a1 100644 --- a/scripts/formatting.py +++ b/scripts/formatting.py @@ -4,8 +4,8 @@ from pathlib import Path from typing import Final, Mapping, Sequence, Tuple +from bootstrap.layout import repository_root from bootstrap.processes import Runner, expect_success, run -from bootstrap.repository import repository_root FORMATTED_TREES: Final[Tuple[str, ...]] = ("src", "tests", "scripts") ISORT: Final[Tuple[str, ...]] = ("uv", "run", "python", "-m", "isort") @@ -36,6 +36,11 @@ def format_code( expect_success(runner, (*BLACK, *paths), cwd=root, environment=environment) +def formatted_paths(named: Sequence[str]) -> Tuple[str, ...]: + """The paths a run formats: the ones named, or the source, test and script trees.""" + return tuple(named) or FORMATTED_TREES + + def main(argv: Sequence[str]) -> int: """Formats the source, test and script trees, or the paths named.""" parser = argparse.ArgumentParser(description="Format the SampleToNES code with isort and black.") @@ -44,7 +49,7 @@ def main(argv: Sequence[str]) -> int: format_code( repository_root(), - tuple(arguments.paths) or FORMATTED_TREES, + formatted_paths(tuple(arguments.paths)), runner=run, environment=os.environ, ) diff --git a/scripts/hooks.py b/scripts/hooks.py index 614e11d38..96fb13c01 100644 --- a/scripts/hooks.py +++ b/scripts/hooks.py @@ -1,10 +1,11 @@ import argparse import os import sys -from typing import Final, Sequence, Tuple +from pathlib import Path +from typing import Final, Mapping, Sequence, Tuple -from bootstrap.processes import expect_success, run -from bootstrap.repository import repository_root +from bootstrap.layout import repository_root +from bootstrap.processes import Runner, expect_success, run INSTALL_HOOKS: Final[Tuple[str, ...]] = ( "uv", @@ -18,14 +19,28 @@ ) +def install_hooks( + root: Path, + *, + runner: Runner, + environment: Mapping[str, str], +) -> None: + """Installs the git hooks pre-commit runs at commit and at push. + + Raises: + SystemExit: If pre-commit fails to install them. + """ + print("Installing pre-commit hooks...") + expect_success(runner, INSTALL_HOOKS, cwd=root, environment=environment) + print("Pre-commit hooks installed.") + + def main(argv: Sequence[str]) -> int: """Installs the git hooks pre-commit runs at commit and at push.""" parser = argparse.ArgumentParser(description="Install the pre-commit hooks.") parser.parse_args(list(argv)) - print("Installing pre-commit hooks...") - expect_success(run, INSTALL_HOOKS, cwd=repository_root(), environment=os.environ) - print("Pre-commit hooks installed.") + install_hooks(repository_root(), runner=run, environment=os.environ) return 0 diff --git a/scripts/lint.py b/scripts/lint.py index 050c6ac81..ab5a6be06 100644 --- a/scripts/lint.py +++ b/scripts/lint.py @@ -1,11 +1,12 @@ import argparse import os import sys -from typing import Final, Sequence, Set, Tuple +from pathlib import Path +from typing import Final, Mapping, Sequence, Set, Tuple +from bootstrap.layout import repository_root from bootstrap.passes import Pass, run_passes -from bootstrap.processes import run -from bootstrap.repository import repository_root +from bootstrap.processes import Runner, run MYPY: Final[str] = "mypy" PYLINT: Final[str] = "pylint" @@ -32,6 +33,39 @@ def linters(paths: Sequence[str]) -> Tuple[Pass, ...]: ) +def lint_code( + root: Path, + passes: Sequence[Pass], + *, + runner: Runner, + environment: Mapping[str, str], +) -> int: + """Runs every linter asked for and names the ones that failed. + + Args: + root: The repository, which the linters run in. + passes: The linters, in order. + runner: What runs them. + environment: The variables they see. + + Returns: + int: 0 where every linter passed, 1 otherwise. + """ + failed = run_passes(passes, root=root, runner=runner, environment=environment) + if failed: + print(f"Linting failed: {', '.join(failed)}.") + return 1 + + print("All linting checks passed.") + return 0 + + +def chosen_linters(paths: Sequence[str], *, mypy: bool, pylint: bool) -> Tuple[Pass, ...]: + """The linters a run asks for: the ones flagged, or both where none is.""" + chosen: Set[str] = {name for name, wanted in ((MYPY, mypy), (PYLINT, pylint)) if wanted} + return tuple(linter for linter in linters(paths) if not chosen or linter.name in chosen) + + def main(argv: Sequence[str]) -> int: """Type checks and lints the code, and reports which linter failed.""" parser = argparse.ArgumentParser(description="Type check and lint the SampleToNES code.") @@ -40,15 +74,12 @@ def main(argv: Sequence[str]) -> int: parser.add_argument("paths", nargs="*", help="files or directories to lint in place of the whole trees") arguments = parser.parse_args(list(argv)) - chosen: Set[str] = {name for name, wanted in ((MYPY, arguments.mypy), (PYLINT, arguments.pylint)) if wanted} - passes = tuple(linter for linter in linters(tuple(arguments.paths)) if not chosen or linter.name in chosen) - failed = run_passes(passes, root=repository_root(), runner=run, environment=os.environ) - if failed: - print(f"Linting failed: {', '.join(failed)}.") - return 1 - - print("All linting checks passed.") - return 0 + return lint_code( + repository_root(), + chosen_linters(tuple(arguments.paths), mypy=arguments.mypy, pylint=arguments.pylint), + runner=run, + environment=os.environ, + ) if __name__ == "__main__": diff --git a/scripts/run_tests.py b/scripts/run_tests.py index 73c34c855..6da4bc94e 100644 --- a/scripts/run_tests.py +++ b/scripts/run_tests.py @@ -1,11 +1,12 @@ import argparse import os import sys -from typing import Dict, Final, Sequence, Tuple +from pathlib import Path +from typing import Dict, Final, Mapping, Sequence, Tuple +from bootstrap.layout import repository_root from bootstrap.passes import Pass -from bootstrap.processes import run -from bootstrap.repository import repository_root +from bootstrap.processes import Runner, run SUITE: Final[str] = "suite" DOCTESTS: Final[str] = "doctests" @@ -48,6 +49,28 @@ def planned_passes(workers: str) -> Dict[str, Pass]: return {current.name: current for current in passes} +def run_pass( + chosen: Pass, + root: Path, + *, + runner: Runner, + environment: Mapping[str, str], +) -> int: + """Announces one pass and runs it from the repository root. + + Args: + chosen: The pass. + root: The repository. + runner: What runs the pass. + environment: The variables pytest sees. + + Returns: + int: The status pytest exited with. + """ + print(chosen.announcement) + return runner(chosen.command, cwd=root, environment=environment, quiet=False) + + def main(argv: Sequence[str]) -> int: """Runs one pass of the tests and exits with the status pytest gave it.""" parser = argparse.ArgumentParser(description="Run one pass of the SampleToNES tests.") @@ -59,13 +82,11 @@ def main(argv: Sequence[str]) -> int: ) arguments = parser.parse_args(list(argv)) - chosen = planned_passes(arguments.workers)[arguments.name] - print(chosen.announcement) - return run( - chosen.command, - cwd=repository_root(), + return run_pass( + planned_passes(arguments.workers)[arguments.name], + repository_root(), + runner=run, environment=os.environ, - quiet=False, ) diff --git a/scripts/setup_environment.py b/scripts/setup_environment.py index 3df283db8..e1358e9ba 100644 --- a/scripts/setup_environment.py +++ b/scripts/setup_environment.py @@ -2,33 +2,38 @@ import os import platform as running import sys -from typing import Dict, Final, List, Mapping, Optional, Sequence, Tuple +from pathlib import Path +from typing import Final, List, Mapping, Optional, Sequence, Tuple -import detect_cuda - -from bootstrap.processes import expect_success, run -from bootstrap.repository import repository_root +from bootstrap.cuda import detect +from bootstrap.layout import repository_root +from bootstrap.platforms.factory import current_platform +from bootstrap.platforms.protocol import Platform +from bootstrap.processes import Runner, expect_success, run +from bootstrap.project import DEVELOPMENT_GROUP, GPU_CUDA11_EXTRA, GPU_EXTRA, read_project GPU_AUTO: Final[str] = "auto" GPU_OFF: Final[str] = "0" GPU_CHOICES: Final[Tuple[str, ...]] = ( GPU_AUTO, GPU_OFF, - detect_cuda.EXTRA_GPU, - detect_cuda.EXTRA_GPU_CUDA11, + GPU_EXTRA, + GPU_CUDA11_EXTRA, ) DEFAULT_GPU: Final[str] = GPU_AUTO -DEVELOPMENT_GROUP: Final[str] = "dev" -DARWIN: Final[str] = "Darwin" -ARCHFLAGS: Final[str] = "ARCHFLAGS" -def gpu_extra(choice: str, *, system: str) -> Optional[str]: +def gpu_extra( + choice: str, + platform: Platform, + environment: Mapping[str, str], +) -> Optional[str]: """The optional-dependency extra a GPU choice selects. Args: choice: ``auto`` to read the NVIDIA driver, ``0`` for the CPU backend, or an extra's name. - system: The name ``platform.system()`` reports. + platform: The system the setup runs on. + environment: The variables the driver's locations are read from. Returns: Optional[str]: The extra, or ``None`` for the CPU backend. @@ -37,7 +42,7 @@ def gpu_extra(choice: str, *, system: str) -> Optional[str]: return None if choice == GPU_AUTO: - detection = detect_cuda.detect(system=system) + detection = detect(platform, environment) print(detection.reason, file=sys.stderr) return detection.extra @@ -65,22 +70,31 @@ def setup_commands(extra: Optional[str]) -> List[List[str]]: ] -def setup_environment_variables( - base: Mapping[str, str], +def set_up_environment( + root: Path, + platform: Platform, + extra: Optional[str], *, - system: str, machine: str, -) -> Dict[str, str]: - """The variables the setup commands see: the caller's, pinned to the native architecture on macOS. + runner: Runner, + environment: Mapping[str, str], +) -> None: + """Synchronizes the development environment and installs the global ``sampletones`` command. - Homebrew's PortAudio carries the machine's own architecture while a python.org interpreter - compiles for both, so the flag settles audio playback on the native one. - """ - variables = dict(base) - if system == DARWIN: - variables[ARCHFLAGS] = f"-arch {machine}" + Args: + root: The repository. + platform: The system the setup runs on, which adds the variables the build needs. + extra: The GPU extra installed with the package, or ``None`` for the CPU backend. + machine: The processor architecture ``platform.machine()`` reports. + runner: What runs the commands. + environment: The caller's variables. - return variables + Raises: + SystemExit: If a command fails. + """ + variables = platform.setup_variables(environment, machine=machine) + for command in setup_commands(extra): + expect_success(runner, command, cwd=root, environment=variables) def main(argv: Sequence[str]) -> int: @@ -95,11 +109,16 @@ def main(argv: Sequence[str]) -> int: arguments = parser.parse_args(list(argv)) root = repository_root() - system = running.system() - environment = setup_environment_variables(os.environ, system=system, machine=running.machine()) - for command in setup_commands(gpu_extra(arguments.gpu, system=system)): - expect_success(run, command, cwd=root, environment=environment) - + read_project(root) + platform = current_platform() + set_up_environment( + root, + platform, + gpu_extra(arguments.gpu, platform, os.environ), + machine=running.machine(), + runner=run, + environment=os.environ, + ) return 0 diff --git a/scripts/system_dependencies.py b/scripts/system_dependencies.py index 8957664fa..a8071a5a8 100644 --- a/scripts/system_dependencies.py +++ b/scripts/system_dependencies.py @@ -1,19 +1,36 @@ import argparse import os import sys -from typing import Sequence +from pathlib import Path +from typing import Mapping, Sequence +from bootstrap.layout import repository_root from bootstrap.platforms.factory import current_platform -from bootstrap.processes import expect_success, run -from bootstrap.repository import repository_root +from bootstrap.platforms.protocol import Platform +from bootstrap.processes import Runner, expect_success, run -def main(argv: Sequence[str]) -> int: - """Installs the system packages building and running the application needs on this machine.""" - parser = argparse.ArgumentParser(description="Install the system packages SampleToNES needs.") - parser.parse_args(list(argv)) +def install_system_packages( + root: Path, + platform: Platform, + *, + runner: Runner, + environment: Mapping[str, str], +) -> int: + """Installs the system packages building and running the application needs. - platform = current_platform() + Args: + root: The repository, which the commands run in. + platform: The system, which names its package manager and packages. + runner: What runs the commands. + environment: The variables the commands see. + + Returns: + int: The exit status: 1 where the package manager is missing, 0 otherwise. + + Raises: + SystemExit: If an install command fails. + """ missing = platform.missing_package_manager() if missing is not None: print(missing, file=sys.stderr) @@ -24,14 +41,26 @@ def main(argv: Sequence[str]) -> int: print(f"Nothing to install on {platform.name}: the Python installer carries what the application needs.") return 0 - root = repository_root() print("Installing system dependencies...") for command in commands: - expect_success(run, command, cwd=root, environment=os.environ) + expect_success(runner, command, cwd=root, environment=environment) print("System dependencies installed.") return 0 +def main(argv: Sequence[str]) -> int: + """Installs the system packages building and running the application needs on this machine.""" + parser = argparse.ArgumentParser(description="Install the system packages SampleToNES needs.") + parser.parse_args(list(argv)) + + return install_system_packages( + repository_root(), + current_platform(), + runner=run, + environment=os.environ, + ) + + if __name__ == "__main__": raise SystemExit(main(sys.argv[1:])) diff --git a/scripts/verify_bundle.py b/scripts/verify_bundle.py new file mode 100644 index 000000000..d60e25cbe --- /dev/null +++ b/scripts/verify_bundle.py @@ -0,0 +1,97 @@ +import argparse +import os +import sys +from pathlib import Path +from typing import Final, List, Mapping, Sequence + +from bootstrap.layout import BUILD_TOOLS, DISTRIBUTION, NOTICES, repository_root +from bootstrap.platforms.factory import current_platform +from bootstrap.platforms.protocol import Platform +from bootstrap.processes import Runner, run +from bootstrap.project import read_project + +VERSION_FLAG: Final[str] = "--version" +INTERNAL_DIRECTORY: Final[str] = "_internal" + + +def missing_notices(bundle: Path) -> List[str]: + """The license and notice files a release bundle ships that are absent from it.""" + return [name for name in NOTICES if not (bundle / name).is_file()] + + +def carried_build_tools(bundle: Path) -> List[str]: + """The build-time packages found in a bundle, which the notices place on the build machine. + + A bundle carries the application and its runtime dependencies. Tooling that draws the assets + belongs to the machine that builds it, so finding it here means the notices describe a + different set of components than the bundle ships. + """ + directories = (bundle, bundle / INTERNAL_DIRECTORY) + return [name for name in BUILD_TOOLS if any((directory / name).is_dir() for directory in directories)] + + +def bundle_failures( + root: Path, + platform: Platform, + *, + runner: Runner, + environment: Mapping[str, str], +) -> List[str]: + """What keeps the release bundle under ``root`` from shipping, each as a line of its own. + + The bundle ships its notices, carries none of the build-time tooling they leave out, and + offers a launcher that starts. + + Args: + root: The repository the bundle was built in. + platform: The system the bundle was built for. + runner: What runs the launcher. + environment: The variables the launcher sees. + + Returns: + List[str]: The failures, empty for a bundle ready to archive. + + Raises: + SystemExit: If the system builds no bundle. + """ + name = read_project(root).name + launcher = platform.bundling().launcher(root / DISTRIBUTION, name=name, release=True) + bundle = launcher.parent + failures: List[str] = [] + absent = missing_notices(bundle) + if absent: + failures.append(f"Bundle {bundle} is missing {', '.join(absent)}") + + carried = carried_build_tools(bundle) + if carried: + failures.append(f"Bundle {bundle} carries build-time tooling its notices leave out: {', '.join(carried)}") + + if not launcher.is_file(): + failures.append(f"Bundle {bundle} offers no launcher at {launcher}") + return failures + + status = runner((str(launcher), VERSION_FLAG), cwd=root, environment=environment, quiet=False) + if status != 0: + failures.append(f"Launcher {launcher} exited with status {status}") + + return failures + + +def main(argv: Sequence[str]) -> int: + """Confirms the release bundle ships its notices, holds to them, and that its launcher starts.""" + parser = argparse.ArgumentParser(description="Verify the release bundle before it is archived.") + parser.parse_args(list(argv)) + + failures = bundle_failures(repository_root(), current_platform(), runner=run, environment=os.environ) + for failure in failures: + print(f"::error::{failure}") + + if failures: + return 1 + + print("The bundle ships its notices and its launcher starts") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) diff --git a/scripts/verify_version_tag.py b/scripts/verify_version_tag.py new file mode 100644 index 000000000..513848b66 --- /dev/null +++ b/scripts/verify_version_tag.py @@ -0,0 +1,33 @@ +import argparse +import sys +from typing import Final, Sequence + +from bootstrap.layout import repository_root +from bootstrap.project import read_project + +TAG_PREFIX: Final[str] = "v" + + +def version_from_tag(tag: str) -> str: + """The project version a release tag names, read from the tag with its ``v`` prefix dropped.""" + return tag.removeprefix(TAG_PREFIX) + + +def main(argv: Sequence[str]) -> int: + """Confirms a release tag names the version ``pyproject.toml`` records.""" + parser = argparse.ArgumentParser(description="Compare a release tag against the project version.") + parser.add_argument("--tag", required=True, help="the release tag being built, such as v0.3.0") + arguments = parser.parse_args(list(argv)) + + tag: str = arguments.tag + version = read_project(repository_root()).version + if version_from_tag(tag) != version: + print(f"::error::Tag {tag} names a version other than the project version {version}") + return 1 + + print(f"Version {version} matches tag {tag}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) diff --git a/src/sampletones_application/ui/resources/loader.py b/src/sampletones_application/ui/resources/loader.py index 5c61dc3d2..35dad7f1c 100644 --- a/src/sampletones_application/ui/resources/loader.py +++ b/src/sampletones_application/ui/resources/loader.py @@ -1,4 +1,3 @@ -import sys from importlib.resources import files from importlib.resources.abc import Traversable from pathlib import Path @@ -12,11 +11,6 @@ def __init__(self, resource_directory: Pathlike) -> None: self.resource_directory = Path(resource_directory) def _get_package_path(self, resource_name: str) -> Union[Path, Traversable]: - if getattr(sys, "frozen", False): - base_path = Path(sys._MEIPASS) # type: ignore[attr-defined] # pylint: disable=protected-access - resource_type = self.resource_directory.name - return base_path / "assets" / resource_type / resource_name - package_name = f"sampletones_assets.{self.resource_directory.name}" return files(package_name).joinpath(resource_name) diff --git a/src/sampletones_shared/paths/package.py b/src/sampletones_shared/paths/package.py new file mode 100644 index 000000000..a6f6daa41 --- /dev/null +++ b/src/sampletones_shared/paths/package.py @@ -0,0 +1,27 @@ +from importlib.util import find_spec +from pathlib import Path + + +def package_directory(package: str) -> Path: + """The directory an import package's files lie in, its data files among them. + + The import system's own record of where the package lives places it, which holds in a + checkout, in an installed copy and in a bundle. PyInstaller unpacks a package's collected data + into a directory of the package's name, which a package the application never imports as code + reaches as a namespace package, so the location is read from the spec whatever kind of + package answers. + + Args: + package: The package's dotted name. + + Returns: + Path: The directory. + + Raises: + FileNotFoundError: If no package of that name is importable. + """ + spec = find_spec(package) + if spec is None or spec.submodule_search_locations is None: + raise FileNotFoundError(f"No package {package} is importable to place its directory by") + + return Path(next(iter(spec.submodule_search_locations))) diff --git a/src/sampletones_shared/paths/resources.py b/src/sampletones_shared/paths/resources.py index 9da9292dd..2e415f2ff 100644 --- a/src/sampletones_shared/paths/resources.py +++ b/src/sampletones_shared/paths/resources.py @@ -1,15 +1,9 @@ -import sys -from importlib.resources import files from pathlib import Path -from typing import Final, Optional +from typing import Final -_BUNDLE_ROOT: Final[Optional[str]] = getattr(sys, "_MEIPASS", None) +from sampletones_shared.paths.package import package_directory -CONFIG_DIRECTORY: Final[Path] = ( - Path(_BUNDLE_ROOT) / "config" if _BUNDLE_ROOT is not None else Path(str(files("sampletones_config"))) -) - -ASSETS_DIRECTORY: Final[str] = "assets" +CONFIG_DIRECTORY: Final[Path] = package_directory("sampletones_config") ICON_DIRECTORY: Final[str] = "icons" ICON_WIN_FILENAME: Final[str] = "sampletones.ico" diff --git a/src/sampletones_tools/assets/mark/paths.py b/src/sampletones_tools/assets/mark/paths.py index 60ea38088..b1c3afd27 100644 --- a/src/sampletones_tools/assets/mark/paths.py +++ b/src/sampletones_tools/assets/mark/paths.py @@ -1,7 +1,8 @@ -from importlib.resources import files from pathlib import Path from typing import Final -MARK_DIRECTORY: Final[Path] = Path(str(files("sampletones_tools.assets.mark"))) +from sampletones_shared.paths.package import package_directory + +MARK_DIRECTORY: Final[Path] = package_directory("sampletones_tools.assets.mark") MARK_PATH: Final[Path] = MARK_DIRECTORY / "mark.yaml" TEMPLATE_PATH: Final[Path] = MARK_DIRECTORY / "template.svg" diff --git a/src/sampletones_tools/calibration/paths.py b/src/sampletones_tools/calibration/paths.py index 5deb181a1..2e407c1ea 100644 --- a/src/sampletones_tools/calibration/paths.py +++ b/src/sampletones_tools/calibration/paths.py @@ -1,7 +1,8 @@ -from importlib.resources import files from pathlib import Path from typing import Final -CALIBRATION_CONFIG_DIRECTORY: Final[Path] = Path(str(files("sampletones_tools.calibration.config"))) +from sampletones_shared.paths.package import package_directory + +CALIBRATION_CONFIG_DIRECTORY: Final[Path] = package_directory("sampletones_tools.calibration.config") REFEREE_CONFIG_PATH: Final[Path] = CALIBRATION_CONFIG_DIRECTORY / "referee.yaml" CORPUS_CONFIG_PATH: Final[Path] = CALIBRATION_CONFIG_DIRECTORY / "corpus.yaml" diff --git a/src/sampletones_tools/checkout.py b/src/sampletones_tools/checkout.py index abaf6db53..ea09b3193 100644 --- a/src/sampletones_tools/checkout.py +++ b/src/sampletones_tools/checkout.py @@ -6,8 +6,8 @@ PROJECT_FILE: Final[str] = "pyproject.toml" SOURCE_DIRECTORY: Final[str] = "src" CHECKOUT_ADVICE: Final[str] = ( - "This command reads the repository, so it runs from a checkout: clone SampleToNES, run " - "'make setup', then 'uv run sampletones {command}'." + "This command needs the repository and its project environment, so it runs from a checkout: " + "clone SampleToNES, run 'make setup', then 'uv run sampletones {command}'." ) @@ -17,10 +17,10 @@ def is_checkout(root: Path) -> bool: def require_checkout(command: str) -> None: - """Holds a developer command to a checkout, where the repository it reads or writes is. + """Holds a developer command to a checkout, where the repository and the project environment are. An installed copy, from the wheel or the bundle, carries the package without the repository - around it, so the refusal names the way to run the command there. + around it or the development dependencies, so the refusal names the way to run the command there. Args: command: The command line the advice names, such as ``check import-boundary --all``. diff --git a/src/sampletones_tools/corpus/paths.py b/src/sampletones_tools/corpus/paths.py index 3ee0a3057..92c1cf78f 100644 --- a/src/sampletones_tools/corpus/paths.py +++ b/src/sampletones_tools/corpus/paths.py @@ -1,8 +1,9 @@ -from importlib.resources import files from pathlib import Path from typing import Final -CONFIG_DIRECTORY: Final[Path] = Path(str(files("sampletones_tools.corpus.config"))) +from sampletones_shared.paths.package import package_directory + +CONFIG_DIRECTORY: Final[Path] = package_directory("sampletones_tools.corpus.config") SYNTH_CONFIG_PATH: Final[Path] = CONFIG_DIRECTORY / "synth.yaml" CATALOG_CONFIG_PATH: Final[Path] = CONFIG_DIRECTORY / "reconstruction.yaml" MODULE_CONFIG_PATH: Final[Path] = CONFIG_DIRECTORY / "module.yaml" diff --git a/tests/suite/bootstrap.py b/tests/suite/bootstrap.py index 71b98c81a..0c17d4b38 100644 --- a/tests/suite/bootstrap.py +++ b/tests/suite/bootstrap.py @@ -1,6 +1,28 @@ from dataclasses import dataclass from pathlib import Path -from typing import Callable, Dict, List, Mapping, Optional, Sequence, Tuple +from typing import Callable, Dict, Final, List, Mapping, Optional, Sequence, Tuple + +PROJECT_NAME: Final[str] = "sampletones" +PROJECT_VERSION: Final[str] = "0.3.0" +PROJECT_DOCUMENT: Final[str] = f""" +[project] +name = "{PROJECT_NAME}" +version = "{PROJECT_VERSION}" + +[project.scripts] +{PROJECT_NAME} = "{PROJECT_NAME}.__main__:main" + +[project.optional-dependencies] +build = ["pyinstaller"] +gpu = ["cupy-cuda12x"] +gpu-cuda11 = ["cupy-cuda11x"] + +[dependency-groups] +dev = ["pytest"] + +[tool.hatch.build.targets.wheel] +packages = ["src/{PROJECT_NAME}", "src/{PROJECT_NAME}_core"] +""" @dataclass(frozen=True) @@ -70,3 +92,10 @@ def __call__( def lines(self) -> List[str]: """Every recorded command as one line, in the order run.""" return [recorded.line for recorded in self.commands] + + +def write_project(root: Path) -> Path: + """Writes a ``pyproject.toml`` under ``root`` stating what the scripts read, and answers with ``root``.""" + root.mkdir(parents=True, exist_ok=True) + (root / "pyproject.toml").write_text(PROJECT_DOCUMENT, encoding="utf-8") + return root diff --git a/tests/unit/sampletones_shared/paths/test_package.py b/tests/unit/sampletones_shared/paths/test_package.py new file mode 100644 index 000000000..07337a8bd --- /dev/null +++ b/tests/unit/sampletones_shared/paths/test_package.py @@ -0,0 +1,33 @@ +from pathlib import Path + +import pytest + +import sampletones_config +from sampletones_shared.paths.package import package_directory + + +class TestPackageDirectory: + def test_the_directory_holds_the_package_s_modules_and_data(self) -> None: + directory = package_directory("sampletones_config") + + assert directory == Path(sampletones_config.__file__).parent + assert (directory / "application").is_dir() + + def test_a_nested_package_is_placed_by_its_own_location(self) -> None: + assert package_directory("sampletones_tools.corpus.config").name == "config" + + def test_a_directory_of_data_alone_is_placed_as_a_namespace_package( + self, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + ) -> None: + """A bundle unpacks a package the application never imports as code this way.""" + (tmp_path / "unpacked_data").mkdir() + (tmp_path / "unpacked_data" / "values.yaml").write_text("") + monkeypatch.syspath_prepend(str(tmp_path)) + + assert package_directory("unpacked_data") == tmp_path / "unpacked_data" + + def test_an_absent_package_is_refused(self) -> None: + with pytest.raises(FileNotFoundError, match="sampletones_absent"): + package_directory("sampletones_absent") diff --git a/tests/unit/scripts/bootstrap/platforms/test_bundling.py b/tests/unit/scripts/bootstrap/platforms/test_bundling.py new file mode 100644 index 000000000..172169618 --- /dev/null +++ b/tests/unit/scripts/bootstrap/platforms/test_bundling.py @@ -0,0 +1,50 @@ +from dataclasses import dataclass +from pathlib import Path + +import pytest + +from bootstrap.platforms.bundling import Bundling +from tests.suite.base import BaseTestSuite +from tests.suite.case import BaseRegularTestCase + +NAME = "sampletones" + + +def _bundling(executable_suffix: str) -> Bundling: + return Bundling( + icon="icon.png", + executable_suffix=executable_suffix, + pyaudio_advice="", + tkinter_advice="", + tkinter_warning="", + ) + + +class TestLauncher(BaseTestSuite): + @dataclass(frozen=True, kw_only=True) + class TestCase(BaseRegularTestCase): + suffix: str + release: bool + expected: Path + + test_cases = ( + TestCase(label="a development bundle is one file", suffix="", release=False, expected=Path("bin", NAME)), + TestCase( + label="a release is a directory beside its launcher", + suffix="", + release=True, + expected=Path("bin", NAME, NAME), + ), + TestCase( + label="the suffix ends the launcher's name", + suffix=".exe", + release=True, + expected=Path("bin", NAME, f"{NAME}.exe"), + ), + ) + + @pytest.mark.parametrize("test_case", test_cases, ids=lambda test_case: test_case.label) + def test_the_launcher_lies_where_pyinstaller_writes_it(self, test_case: TestCase) -> None: + launcher = _bundling(test_case.suffix).launcher(Path("bin"), name=NAME, release=test_case.release) + + assert launcher == test_case.expected diff --git a/tests/unit/scripts/bootstrap/platforms/test_factory.py b/tests/unit/scripts/bootstrap/platforms/test_factory.py new file mode 100644 index 000000000..0f25c0cab --- /dev/null +++ b/tests/unit/scripts/bootstrap/platforms/test_factory.py @@ -0,0 +1,30 @@ +from dataclasses import dataclass + +import pytest + +from bootstrap.platforms.factory import platform_named +from bootstrap.platforms.linux import LINUX +from bootstrap.platforms.macos import DARWIN +from bootstrap.platforms.windows import WINDOWS +from tests.suite.base import BaseTestSuite +from tests.suite.case import BaseRegularTestCase + + +class TestPlatformNamed(BaseTestSuite): + @dataclass(frozen=True, kw_only=True) + class TestCase(BaseRegularTestCase): + system: str + + test_cases = ( + TestCase(label="Linux", system=LINUX), + TestCase(label="Windows", system=WINDOWS), + TestCase(label="macOS", system=DARWIN), + ) + + @pytest.mark.parametrize("test_case", test_cases, ids=lambda test_case: test_case.label) + def test_the_system_name_selects_its_platform(self, test_case: TestCase) -> None: + assert platform_named(test_case.system).name == test_case.system + + def test_an_unknown_system_is_refused_by_name(self) -> None: + with pytest.raises(SystemExit, match="Plan 9"): + platform_named("Plan 9") diff --git a/tests/unit/scripts/bootstrap/platforms/test_linux.py b/tests/unit/scripts/bootstrap/platforms/test_linux.py new file mode 100644 index 000000000..cc237c4d9 --- /dev/null +++ b/tests/unit/scripts/bootstrap/platforms/test_linux.py @@ -0,0 +1,27 @@ +from pathlib import Path + +from bootstrap.platforms.linux import Linux + + +class TestLinux: + def test_a_virtual_environment_runs_bin_python(self) -> None: + assert Linux().interpreter(Path(".venv-build")) == Path(".venv-build", "bin", "python") + + def test_a_bundle_launcher_carries_no_extension(self) -> None: + assert Linux().bundling().launcher(Path("bin"), name="sampletones", release=False) == Path("bin", "sampletones") + + def test_apt_is_updated_before_the_packages_are_installed(self) -> None: + commands = Linux().system_packages() + + assert [command[:2] for command in commands] == [("sudo", "apt-get"), ("sudo", "apt-get")] + assert "portaudio19-dev" in commands[1] + assert "python3-tk" in commands[1] + assert Linux().missing_package_manager() is None + + def test_the_setup_and_the_build_take_the_variables_as_given(self) -> None: + assert Linux().setup_variables({"PATH": "/usr/bin"}, machine="x86_64") == {"PATH": "/usr/bin"} + assert Linux().build_flags(machine="x86_64") == () + + def test_the_driver_is_looked_up_on_the_path_alone(self) -> None: + assert Linux().cuda + assert Linux().nvidia_smi_locations({"SystemRoot": "C:/Windows"}) == () diff --git a/tests/unit/scripts/bootstrap/platforms/test_macos.py b/tests/unit/scripts/bootstrap/platforms/test_macos.py new file mode 100644 index 000000000..fddbb26ad --- /dev/null +++ b/tests/unit/scripts/bootstrap/platforms/test_macos.py @@ -0,0 +1,64 @@ +from pathlib import Path + +import pytest + +from bootstrap.platforms import macos +from bootstrap.platforms.macos import ARCHFLAGS, HOMEBREW, HOMEBREW_SITE, PORTAUDIO, MacOS + +PORTAUDIO_PREFIX = "/opt/homebrew/opt/portaudio" + + +class TestMacOS: + def test_a_virtual_environment_runs_bin_python(self) -> None: + assert MacOS().interpreter(Path(".venv-build")) == Path(".venv-build", "bin", "python") + + def test_a_bundle_is_refused_with_the_way_to_run_from_source(self) -> None: + with pytest.raises(SystemExit, match="make setup"): + MacOS().bundling() + + def test_portaudio_is_installed_through_homebrew(self) -> None: + assert MacOS().system_packages() == ((HOMEBREW, "install", PORTAUDIO),) + + def test_with_homebrew_the_package_manager_is_present(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(macos.shutil, "which", lambda name: f"/opt/homebrew/bin/{name}") + + assert MacOS().missing_package_manager() is None + + def test_without_homebrew_the_refusal_names_where_to_get_it(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(macos.shutil, "which", lambda name: None) + + refusal = MacOS().missing_package_manager() + + assert refusal is not None + assert HOMEBREW_SITE in refusal + + def test_the_setup_pins_the_native_architecture(self) -> None: + variables = MacOS().setup_variables({"PATH": "/usr/bin"}, machine="arm64") + + assert variables == {"PATH": "/usr/bin", ARCHFLAGS: "-arch arm64"} + + def test_a_build_compiles_against_homebrew_s_portaudio(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(macos, "homebrew_prefix", lambda package: PORTAUDIO_PREFIX) + + assert MacOS().build_flags(machine="arm64") == ( + f"CFLAGS=-I{PORTAUDIO_PREFIX}/include", + f"LDFLAGS=-L{PORTAUDIO_PREFIX}/lib", + f"{ARCHFLAGS}=-arch arm64", + ) + + def test_a_build_without_homebrew_s_portaudio_is_refused(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(macos, "homebrew_prefix", lambda package: "") + + with pytest.raises(SystemExit, match="Homebrew"): + MacOS().build_flags(machine="arm64") + + def test_the_cpu_backend_is_the_one_it_runs(self) -> None: + assert not MacOS().cuda + assert MacOS().nvidia_smi_locations({}) == () + + +class TestHomebrewPrefix: + def test_without_homebrew_the_prefix_is_empty(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(macos.shutil, "which", lambda name: None) + + assert macos.homebrew_prefix(PORTAUDIO) == "" diff --git a/tests/unit/scripts/bootstrap/platforms/test_windows.py b/tests/unit/scripts/bootstrap/platforms/test_windows.py new file mode 100644 index 000000000..6ddc30017 --- /dev/null +++ b/tests/unit/scripts/bootstrap/platforms/test_windows.py @@ -0,0 +1,27 @@ +from pathlib import Path + +from bootstrap.platforms.windows import Windows + + +class TestWindows: + def test_a_virtual_environment_runs_scripts_python(self) -> None: + assert Windows().interpreter(Path(".venv-build")) == Path(".venv-build", "Scripts", "python.exe") + + def test_a_bundle_launcher_carries_the_executable_extension(self) -> None: + launcher = Windows().bundling().launcher(Path("bin"), name="sampletones", release=True) + + assert launcher == Path("bin", "sampletones", "sampletones.exe") + + def test_the_installer_carries_every_system_package(self) -> None: + assert Windows().system_packages() == () + assert Windows().missing_package_manager() is None + + def test_the_setup_and_the_build_take_the_variables_as_given(self) -> None: + assert Windows().setup_variables({"PATH": "C:/Python"}, machine="AMD64") == {"PATH": "C:/Python"} + assert Windows().build_flags(machine="AMD64") == () + + def test_the_driver_s_fixed_locations_are_read_from_the_variables_that_are_set(self) -> None: + locations = Windows().nvidia_smi_locations({"SystemRoot": "C:/Windows"}) + + assert Windows().cuda + assert locations == (Path("C:/Windows", "System32", "nvidia-smi.exe"),) diff --git a/tests/unit/scripts/test_detect_cuda.py b/tests/unit/scripts/bootstrap/test_cuda.py similarity index 68% rename from tests/unit/scripts/test_detect_cuda.py rename to tests/unit/scripts/bootstrap/test_cuda.py index 60012eb6d..998f09dbb 100644 --- a/tests/unit/scripts/test_detect_cuda.py +++ b/tests/unit/scripts/bootstrap/test_cuda.py @@ -7,11 +7,13 @@ import pytest +from bootstrap import cuda +from bootstrap.platforms.linux import Linux +from bootstrap.platforms.macos import MacOS +from bootstrap.platforms.windows import Windows +from bootstrap.project import GPU_CUDA11_EXTRA, GPU_EXTRA from tests.suite.base import BaseTestSuite from tests.suite.case import BaseRegularTestCase -from tests.suite.scripts import load_script - -detect_cuda = load_script("detect_cuda.py") def _completed( @@ -42,32 +44,32 @@ class TestCase(BaseRegularTestCase): TestCase( label="cuda_12_0_selects_gpu", cuda_version=(12, 0), - expected="gpu", + expected=GPU_EXTRA, ), TestCase( label="cuda_12_9_selects_gpu", cuda_version=(12, 9), - expected="gpu", + expected=GPU_EXTRA, ), TestCase( label="cuda_13_0_selects_gpu", cuda_version=(13, 0), - expected="gpu", + expected=GPU_EXTRA, ), TestCase( label="cuda_14_2_selects_gpu", cuda_version=(14, 2), - expected="gpu", + expected=GPU_EXTRA, ), TestCase( label="cuda_11_8_selects_legacy", cuda_version=(11, 8), - expected="gpu-cuda11", + expected=GPU_CUDA11_EXTRA, ), TestCase( label="cuda_11_0_selects_legacy", cuda_version=(11, 0), - expected="gpu-cuda11", + expected=GPU_CUDA11_EXTRA, ), TestCase( label="cuda_10_2_keeps_cpu", @@ -87,7 +89,7 @@ class TestCase(BaseRegularTestCase): ids=lambda test_case: test_case.label, ) def test_select_extra(self, test_case: TestCase) -> None: - assert detect_cuda.select_extra(test_case.cuda_version) == test_case.expected + assert cuda.select_extra(test_case.cuda_version) == test_case.expected class TestQueryDriverCudaVersion(BaseTestSuite): @@ -135,8 +137,8 @@ def fake_run( ) -> subprocess.CompletedProcess[str]: return _completed(test_case.output) - monkeypatch.setattr(detect_cuda.subprocess, "run", fake_run) - assert detect_cuda.query_driver_cuda_version(Path("nvidia-smi")) == test_case.expected + monkeypatch.setattr(cuda.subprocess, "run", fake_run) + assert cuda.query_driver_cuda_version(Path("nvidia-smi")) == test_case.expected def test_falls_back_to_query_flag( self, @@ -149,8 +151,8 @@ def fake_run( output = QUERY_OUTPUT_CUDA11 if "-q" in command else NO_VERSION_OUTPUT return _completed(output) - monkeypatch.setattr(detect_cuda.subprocess, "run", fake_run) - assert detect_cuda.query_driver_cuda_version(Path("nvidia-smi")) == (11, 8) + monkeypatch.setattr(cuda.subprocess, "run", fake_run) + assert cuda.query_driver_cuda_version(Path("nvidia-smi")) == (11, 8) def test_missing_executable_keeps_cpu( self, @@ -162,8 +164,8 @@ def fake_run( ) -> subprocess.CompletedProcess[str]: raise OSError("nvidia-smi is not executable") - monkeypatch.setattr(detect_cuda.subprocess, "run", fake_run) - assert detect_cuda.query_driver_cuda_version(Path("nvidia-smi")) is None + monkeypatch.setattr(cuda.subprocess, "run", fake_run) + assert cuda.query_driver_cuda_version(Path("nvidia-smi")) is None def test_nonzero_return_code_keeps_cpu( self, @@ -175,50 +177,55 @@ def fake_run( ) -> subprocess.CompletedProcess[str]: return _completed(TABLE_OUTPUT_CUDA12, returncode=9) - monkeypatch.setattr(detect_cuda.subprocess, "run", fake_run) - assert detect_cuda.query_driver_cuda_version(Path("nvidia-smi")) is None + monkeypatch.setattr(cuda.subprocess, "run", fake_run) + assert cuda.query_driver_cuda_version(Path("nvidia-smi")) is None class TestFindNvidiaSmi: def test_uses_path_when_present(self, monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr( - detect_cuda.shutil, - "which", - lambda name: "/usr/bin/nvidia-smi", - ) - assert detect_cuda.find_nvidia_smi(system="Linux") == Path("/usr/bin/nvidia-smi") + monkeypatch.setattr(cuda.shutil, "which", lambda name: "/usr/bin/nvidia-smi") + + assert cuda.find_nvidia_smi(Linux(), {}) == Path("/usr/bin/nvidia-smi") def test_absent_on_linux(self, monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr(detect_cuda.shutil, "which", lambda name: None) - assert detect_cuda.find_nvidia_smi(system="Linux") is None + monkeypatch.setattr(cuda.shutil, "which", lambda name: None) + + assert cuda.find_nvidia_smi(Linux(), {}) is None + + def test_a_fixed_windows_location_stands_in_for_the_path( + self, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + ) -> None: + monkeypatch.setattr(cuda.shutil, "which", lambda name: None) + (tmp_path / "System32").mkdir() + (tmp_path / "System32" / "nvidia-smi.exe").write_bytes(b"") + + assert ( + cuda.find_nvidia_smi(Windows(), {"SystemRoot": str(tmp_path)}) == tmp_path / "System32" / "nvidia-smi.exe" + ) class TestDetect: def test_macos_keeps_cpu(self) -> None: - detection = detect_cuda.detect(system="Darwin") + detection = cuda.detect(MacOS(), {}) + assert detection.extra is None assert detection.cuda_version is None def test_no_driver_keeps_cpu(self, monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr(detect_cuda.shutil, "which", lambda name: None) - detection = detect_cuda.detect(system="Linux") + monkeypatch.setattr(cuda.shutil, "which", lambda name: None) + + detection = cuda.detect(Linux(), {}) + assert detection.extra is None assert detection.nvidia_smi is None - def test_selects_gpu_for_cuda12_driver( - self, - monkeypatch: pytest.MonkeyPatch, - ) -> None: - monkeypatch.setattr( - detect_cuda.shutil, - "which", - lambda name: "/usr/bin/nvidia-smi", - ) - monkeypatch.setattr( - detect_cuda.subprocess, - "run", - lambda command, **_: _completed(TABLE_OUTPUT_CUDA12), - ) - detection = detect_cuda.detect(system="Linux") + def test_selects_gpu_for_cuda12_driver(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(cuda.shutil, "which", lambda name: "/usr/bin/nvidia-smi") + monkeypatch.setattr(cuda.subprocess, "run", lambda command, **_: _completed(TABLE_OUTPUT_CUDA12)) + + detection = cuda.detect(Linux(), {}) + assert detection.cuda_version == (12, 4) - assert detection.extra == "gpu" + assert detection.extra == GPU_EXTRA diff --git a/tests/unit/scripts/bootstrap/test_files.py b/tests/unit/scripts/bootstrap/test_files.py new file mode 100644 index 000000000..d20fafc60 --- /dev/null +++ b/tests/unit/scripts/bootstrap/test_files.py @@ -0,0 +1,23 @@ +from pathlib import Path + +from bootstrap.files import remove_path + + +class TestRemovePath: + def test_a_directory_goes_with_everything_under_it(self, tmp_path: Path) -> None: + directory = tmp_path / "bin" + (directory / "nested").mkdir(parents=True) + (directory / "nested" / "file").write_text("") + + assert remove_path(directory) + assert not directory.exists() + + def test_a_file_goes(self, tmp_path: Path) -> None: + file = tmp_path / "sampletones.spec" + file.write_text("") + + assert remove_path(file) + assert not file.exists() + + def test_an_absent_path_is_reported_as_nothing_removed(self, tmp_path: Path) -> None: + assert not remove_path(tmp_path / "absent") diff --git a/tests/unit/scripts/bootstrap/test_layout.py b/tests/unit/scripts/bootstrap/test_layout.py new file mode 100644 index 000000000..a8308884e --- /dev/null +++ b/tests/unit/scripts/bootstrap/test_layout.py @@ -0,0 +1,9 @@ +from bootstrap.layout import PROJECT_FILE, SOURCE_DIRECTORY, repository_root + + +class TestRepositoryRoot: + def test_the_root_holds_the_project_beside_the_sources(self) -> None: + root = repository_root() + + assert (root / PROJECT_FILE).is_file() + assert (root / SOURCE_DIRECTORY / "sampletones" / "__main__.py").is_file() diff --git a/tests/unit/scripts/bootstrap/test_platforms.py b/tests/unit/scripts/bootstrap/test_platforms.py deleted file mode 100644 index 9bbf972f4..000000000 --- a/tests/unit/scripts/bootstrap/test_platforms.py +++ /dev/null @@ -1,128 +0,0 @@ -from dataclasses import dataclass -from pathlib import Path - -import pytest - -from bootstrap.platforms import macos -from bootstrap.platforms.factory import platform_named -from bootstrap.platforms.linux import Linux -from bootstrap.platforms.macos import HOMEBREW, HOMEBREW_SITE, MacOS -from bootstrap.platforms.windows import Windows -from tests.suite.base import BaseTestSuite -from tests.suite.case import BaseRegularTestCase - - -class TestPlatformNamed(BaseTestSuite): - @dataclass(frozen=True, kw_only=True) - class TestCase(BaseRegularTestCase): - system: str - bundles: bool - - test_cases = ( - TestCase(label="Linux builds a bundle", system="Linux", bundles=True), - TestCase(label="Windows builds a bundle", system="Windows", bundles=True), - TestCase(label="macOS runs from source", system="Darwin", bundles=False), - ) - - @pytest.mark.parametrize("test_case", test_cases, ids=lambda test_case: test_case.label) - def test_the_system_name_selects_its_platform(self, test_case: TestCase) -> None: - platform = platform_named(test_case.system) - - assert platform.name == test_case.system - assert platform.bundles is test_case.bundles - - def test_an_unknown_system_is_refused_by_name(self) -> None: - with pytest.raises(SystemExit, match="Plan 9"): - platform_named("Plan 9") - - -class TestLaunchers(BaseTestSuite): - @dataclass(frozen=True, kw_only=True) - class TestCase(BaseRegularTestCase): - system: str - release: bool - expected: str - - test_cases = ( - TestCase(label="a Linux development bundle is one file", system="Linux", release=False, expected="sampletones"), - TestCase( - label="a Linux release is a directory beside its launcher", - system="Linux", - release=True, - expected="sampletones/sampletones", - ), - TestCase( - label="a Windows development bundle carries an extension", - system="Windows", - release=False, - expected="sampletones.exe", - ), - TestCase( - label="a Windows release is a directory beside its launcher", - system="Windows", - release=True, - expected="sampletones/sampletones.exe", - ), - ) - - @pytest.mark.parametrize("test_case", test_cases, ids=lambda test_case: test_case.label) - def test_the_launcher_lies_where_pyinstaller_writes_it(self, test_case: TestCase) -> None: - launcher = platform_named(test_case.system).launcher(Path("bin"), release=test_case.release) - - assert launcher == Path("bin") / test_case.expected - - -class TestInterpreters: - def test_a_posix_environment_runs_bin_python(self) -> None: - assert Linux().interpreter(Path(".venv-build")) == Path(".venv-build/bin/python") - - def test_a_windows_environment_runs_scripts_python(self) -> None: - assert Windows().interpreter(Path(".venv-build")) == Path(".venv-build/Scripts/python.exe") - - -class TestSystemPackages: - def test_linux_updates_apt_before_installing(self) -> None: - commands = Linux().system_packages() - - assert [command[:2] for command in commands] == [("sudo", "apt-get"), ("sudo", "apt-get")] - assert "portaudio19-dev" in commands[1] - assert "python3-tk" in commands[1] - - def test_macos_installs_portaudio_through_homebrew(self) -> None: - assert MacOS().system_packages() == ((HOMEBREW, "install", "portaudio"),) - - def test_macos_with_homebrew_has_its_package_manager(self, monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr(macos.shutil, "which", lambda name: f"/opt/homebrew/bin/{name}") - - assert MacOS().missing_package_manager() is None - - def test_macos_without_homebrew_names_where_to_get_it(self, monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr(macos.shutil, "which", lambda name: None) - - refusal = MacOS().missing_package_manager() - - assert refusal is not None - assert HOMEBREW_SITE in refusal - - def test_windows_installs_nothing(self) -> None: - assert Windows().system_packages() == () - assert Windows().missing_package_manager() is None - - -class TestBuildEnvironment: - def test_macos_exports_the_portaudio_flags_on_the_native_architecture(self) -> None: - lines = MacOS().build_environment(machine="arm64", portaudio_prefix="/opt/homebrew/opt/portaudio") - - assert lines == ( - "CFLAGS=-I/opt/homebrew/opt/portaudio/include", - "LDFLAGS=-L/opt/homebrew/opt/portaudio/lib", - "ARCHFLAGS=-arch arm64", - ) - - def test_macos_without_homebrew_is_refused(self) -> None: - with pytest.raises(SystemExit, match="Homebrew"): - MacOS().build_environment(machine="arm64", portaudio_prefix="") - - def test_other_systems_export_nothing(self) -> None: - assert Linux().build_environment(machine="x86_64", portaudio_prefix="") == () - assert Windows().build_environment(machine="AMD64", portaudio_prefix="") == () diff --git a/tests/unit/scripts/bootstrap/test_preflight.py b/tests/unit/scripts/bootstrap/test_preflight.py index 001ae8aa7..08f061205 100644 --- a/tests/unit/scripts/bootstrap/test_preflight.py +++ b/tests/unit/scripts/bootstrap/test_preflight.py @@ -30,7 +30,7 @@ def test_an_interpreter_without_audio_playback_is_refused_with_the_platform_s_ad with pytest.raises(SystemExit, match="make system-deps"): check_build_interpreter( python, - Linux(), + Linux().bundling(), release=False, runner=RecordingRunner({"import pyaudio": 1}, None), cwd=tmp_path, @@ -41,7 +41,7 @@ def test_a_release_without_tk_is_refused(self, python: Path, tmp_path: Path) -> with pytest.raises(SystemExit, match="tkinter"): check_build_interpreter( python, - Linux(), + Linux().bundling(), release=True, runner=RecordingRunner({"import tkinter": 1}, None), cwd=tmp_path, @@ -56,7 +56,7 @@ def test_a_development_bundle_without_tk_is_warned( ) -> None: check_build_interpreter( python, - Linux(), + Linux().bundling(), release=False, runner=RecordingRunner({"import tkinter": 1}, None), cwd=tmp_path, @@ -73,7 +73,7 @@ def test_an_interpreter_carrying_both_passes( ) -> None: check_build_interpreter( python, - Linux(), + Linux().bundling(), release=True, runner=RecordingRunner({}, None), cwd=tmp_path, diff --git a/tests/unit/scripts/bootstrap/test_project.py b/tests/unit/scripts/bootstrap/test_project.py new file mode 100644 index 000000000..6ef73c8ad --- /dev/null +++ b/tests/unit/scripts/bootstrap/test_project.py @@ -0,0 +1,50 @@ +import tomllib +from pathlib import Path + +import pytest + +from bootstrap.layout import repository_root +from bootstrap.project import NAMED_EXTRAS, NAMED_GROUPS, parse_project, read_project +from tests.suite.bootstrap import PROJECT_DOCUMENT, PROJECT_NAME, PROJECT_VERSION, write_project + + +class TestReadProject: + def test_the_repository_states_every_name_the_scripts_install(self) -> None: + project = read_project(repository_root()) + + assert set(NAMED_EXTRAS) <= set(project.extras) + assert set(NAMED_GROUPS) <= set(project.groups) + assert (repository_root() / project.entry_script).is_file() + assert all((repository_root() / "src" / package).is_dir() for package in project.packages) + + def test_the_facts_are_read_from_the_file(self, tmp_path: Path) -> None: + project = read_project(write_project(tmp_path)) + + assert project.name == PROJECT_NAME + assert project.version == PROJECT_VERSION + assert project.entry_module == f"{PROJECT_NAME}.__main__" + assert project.entry_script == f"src/{PROJECT_NAME}/__main__.py" + assert project.packages == (PROJECT_NAME, f"{PROJECT_NAME}_core") + + +class TestParseProject: + def test_a_missing_extra_is_refused_by_name(self) -> None: + document = tomllib.loads(PROJECT_DOCUMENT) + del document["project"]["optional-dependencies"]["gpu-cuda11"] + + with pytest.raises(SystemExit, match="gpu-cuda11"): + parse_project(document) + + def test_a_missing_group_is_refused_by_name(self) -> None: + document = tomllib.loads(PROJECT_DOCUMENT) + del document["dependency-groups"]["dev"] + + with pytest.raises(SystemExit, match="dev"): + parse_project(document) + + def test_a_project_without_its_command_is_refused(self) -> None: + document = tomllib.loads(PROJECT_DOCUMENT) + del document["project"]["scripts"] + + with pytest.raises(SystemExit, match=r"\[project.scripts\]"): + parse_project(document) diff --git a/tests/unit/scripts/bootstrap/test_repository.py b/tests/unit/scripts/bootstrap/test_repository.py deleted file mode 100644 index 24706cc59..000000000 --- a/tests/unit/scripts/bootstrap/test_repository.py +++ /dev/null @@ -1,9 +0,0 @@ -from bootstrap.repository import repository_root - - -class TestRepositoryRoot: - def test_the_root_holds_the_project_and_the_entry_package(self) -> None: - root = repository_root() - - assert (root / "pyproject.toml").is_file() - assert (root / "src" / "sampletones" / "__main__.py").is_file() diff --git a/tests/unit/scripts/bootstrap/test_venv_build.py b/tests/unit/scripts/bootstrap/test_venv_build.py index 295d94271..ce8bdc9de 100644 --- a/tests/unit/scripts/bootstrap/test_venv_build.py +++ b/tests/unit/scripts/bootstrap/test_venv_build.py @@ -1,16 +1,17 @@ import sys from pathlib import Path +from bootstrap.layout import BUILD_ENVIRONMENT from bootstrap.platforms.linux import Linux -from bootstrap.venv_build import BUILD_ENVIRONMENT, build_environment, install +from bootstrap.venv_build import ensure_build_venv, install from tests.suite.bootstrap import RecordingRunner -class TestBuildEnvironment: +class TestEnsureBuildVenv: def test_a_missing_environment_is_created_by_the_running_interpreter(self, tmp_path: Path) -> None: runner = RecordingRunner({}, None) - python = build_environment(tmp_path, Linux(), runner=runner, environment={}) + python = ensure_build_venv(tmp_path, Linux(), runner=runner, environment={}) assert python == Linux().interpreter(tmp_path / BUILD_ENVIRONMENT) assert runner.lines == [f"{sys.executable} -m venv --clear {tmp_path / BUILD_ENVIRONMENT}"] @@ -21,14 +22,14 @@ def test_an_environment_with_its_interpreter_is_kept(self, tmp_path: Path) -> No python.write_text("") runner = RecordingRunner({}, None) - assert build_environment(tmp_path, Linux(), runner=runner, environment={}) == python + assert ensure_build_venv(tmp_path, Linux(), runner=runner, environment={}) == python assert runner.lines == [] def test_a_directory_an_interrupted_creation_left_is_created_again(self, tmp_path: Path) -> None: (tmp_path / BUILD_ENVIRONMENT).mkdir() runner = RecordingRunner({}, None) - build_environment(tmp_path, Linux(), runner=runner, environment={}) + ensure_build_venv(tmp_path, Linux(), runner=runner, environment={}) assert runner.lines == [f"{sys.executable} -m venv --clear {tmp_path / BUILD_ENVIRONMENT}"] diff --git a/tests/unit/scripts/ci/checks/test_bundle.py b/tests/unit/scripts/ci/checks/test_bundle.py deleted file mode 100644 index 06e339dcb..000000000 --- a/tests/unit/scripts/ci/checks/test_bundle.py +++ /dev/null @@ -1,198 +0,0 @@ -import subprocess -from dataclasses import dataclass -from pathlib import Path -from typing import Any, List, Sequence - -import pytest - -from tests.suite.base import BaseTestSuite -from tests.suite.case import BaseRegularTestCase -from tests.suite.scripts import load_script - -check_bundle = load_script("ci/checks/bundle.py") - -NOTICES = ("LICENSE", "THIRD-PARTY-NOTICES.md", "THIRD-PARTY-LICENSES.txt") - - -@pytest.fixture -def bundle(tmp_path: Path) -> Path: - source = tmp_path / "bin" / "sampletones" - source.mkdir(parents=True) - for name in NOTICES: - (source / name).write_text(name) - - return source - - -def _install_launcher(bundle: Path) -> Path: - launcher: Path = check_bundle.launcher_path( - bundle, - system=check_bundle.platform.system(), - ) - launcher.write_bytes(b"launcher") - return launcher - - -def _stub_run( - monkeypatch: pytest.MonkeyPatch, - *, - returncode: int, -) -> List[Sequence[str]]: - commands: List[Sequence[str]] = [] - - def fake_run( - command: Sequence[str], - **_: Any, - ) -> subprocess.CompletedProcess[str]: - commands.append(command) - return subprocess.CompletedProcess( - args=list(command), - returncode=returncode, - ) - - monkeypatch.setattr(check_bundle.subprocess, "run", fake_run) - return commands - - -class TestLauncherPath(BaseTestSuite): - @dataclass(frozen=True, kw_only=True) - class TestCase(BaseRegularTestCase): - system: str - expected: str - - test_cases = ( - TestCase( - label="windows_launcher_carries_an_extension", - system="Windows", - expected="sampletones.exe", - ), - TestCase( - label="linux_launcher", - system="Linux", - expected="sampletones", - ), - TestCase( - label="macos_launcher", - system="Darwin", - expected="sampletones", - ), - ) - - @pytest.mark.parametrize( - "test_case", - test_cases, - ids=lambda test_case: test_case.label, - ) - def test_launcher_path( - self, - test_case: "TestLauncherPath.TestCase", - tmp_path: Path, - ) -> None: - assert check_bundle.launcher_path(tmp_path, system=test_case.system).name == test_case.expected - - -class TestMissingNotices: - def test_a_complete_bundle_lacks_nothing(self, bundle: Path) -> None: - assert check_bundle.missing_notices(bundle) == [] - - def test_every_absent_notice_is_reported(self, bundle: Path) -> None: - (bundle / "LICENSE").unlink() - (bundle / "THIRD-PARTY-LICENSES.txt").unlink() - - assert check_bundle.missing_notices(bundle) == [ - "LICENSE", - "THIRD-PARTY-LICENSES.txt", - ] - - def test_a_notice_directory_counts_as_absent(self, bundle: Path) -> None: - (bundle / "LICENSE").unlink() - (bundle / "LICENSE").mkdir() - - assert check_bundle.missing_notices(bundle) == ["LICENSE"] - - -class TestCarriedBuildTools: - def test_an_application_bundle_holds_to_its_notices(self, bundle: Path) -> None: - assert check_bundle.carried_build_tools(bundle) == [] - - def test_a_build_tool_beside_the_application_is_reported(self, bundle: Path) -> None: - (bundle / check_bundle.INTERNAL_DIRECTORY / "PIL").mkdir(parents=True) - - assert check_bundle.carried_build_tools(bundle) == ["PIL"] - - def test_a_build_tool_beside_the_launcher_is_reported(self, bundle: Path) -> None: - (bundle / "PIL").mkdir() - - assert check_bundle.carried_build_tools(bundle) == ["PIL"] - - -class TestMain: - def test_a_complete_bundle_passes( - self, - bundle: Path, - monkeypatch: pytest.MonkeyPatch, - ) -> None: - launcher = _install_launcher(bundle) - commands = _stub_run(monkeypatch, returncode=0) - - assert check_bundle.main([str(bundle)]) == 0 - assert commands == [[str(launcher), "--version"]] - - def test_a_missing_notice_is_annotated_as_an_error( - self, - bundle: Path, - capsys: pytest.CaptureFixture[str], - ) -> None: - _install_launcher(bundle) - (bundle / "THIRD-PARTY-NOTICES.md").unlink() - - assert check_bundle.main([str(bundle)]) == 1 - - output = capsys.readouterr().out - assert output.startswith("::error::") - assert "THIRD-PARTY-NOTICES.md" in output - - def test_bundled_build_tooling_is_annotated_as_an_error( - self, - bundle: Path, - capsys: pytest.CaptureFixture[str], - ) -> None: - _install_launcher(bundle) - (bundle / check_bundle.INTERNAL_DIRECTORY / "PIL").mkdir(parents=True) - - assert check_bundle.main([str(bundle)]) == 1 - - output = capsys.readouterr().out - assert output.startswith("::error::") - assert "PIL" in output - - def test_a_missing_launcher_is_annotated_as_an_error( - self, - bundle: Path, - capsys: pytest.CaptureFixture[str], - ) -> None: - assert check_bundle.main([str(bundle)]) == 1 - assert capsys.readouterr().out.startswith("::error::") - - def test_a_launcher_that_fails_propagates_its_status( - self, - bundle: Path, - monkeypatch: pytest.MonkeyPatch, - ) -> None: - _install_launcher(bundle) - _stub_run(monkeypatch, returncode=3) - - assert check_bundle.main([str(bundle)]) == 3 - - def test_the_launcher_runs_only_once_the_notices_are_present( - self, - bundle: Path, - monkeypatch: pytest.MonkeyPatch, - ) -> None: - _install_launcher(bundle) - (bundle / "LICENSE").unlink() - commands = _stub_run(monkeypatch, returncode=0) - - check_bundle.main([str(bundle)]) - - assert commands == [] diff --git a/tests/unit/scripts/ci/checks/test_version_tag.py b/tests/unit/scripts/ci/checks/test_version_tag.py deleted file mode 100644 index 77725fb10..000000000 --- a/tests/unit/scripts/ci/checks/test_version_tag.py +++ /dev/null @@ -1,139 +0,0 @@ -from dataclasses import dataclass - -import pytest - -from tests.suite.base import BaseTestSuite -from tests.suite.case import BaseRegularTestCase -from tests.suite.scripts import load_script - -check_version_tag = load_script("ci/checks/version_tag.py") - - -class TestVersionFromTag(BaseTestSuite): - @dataclass(frozen=True, kw_only=True) - class TestCase(BaseRegularTestCase): - tag: str - expected: str - - test_cases = ( - TestCase( - label="release_tag", - tag="v0.3.0", - expected="0.3.0", - ), - TestCase( - label="prerelease_tag", - tag="v0.3.0.dev1", - expected="0.3.0.dev1", - ), - TestCase( - label="release_candidate", - tag="v1.0.0rc2", - expected="1.0.0rc2", - ), - TestCase( - label="bare_version", - tag="0.3.0", - expected="0.3.0", - ), - TestCase( - label="single_prefix_is_dropped", - tag="vv0.3.0", - expected="v0.3.0", - ), - ) - - @pytest.mark.parametrize( - "test_case", - test_cases, - ids=lambda test_case: test_case.label, - ) - def test_version_from_tag(self, test_case: TestCase) -> None: - assert check_version_tag.version_from_tag(test_case.tag) == test_case.expected - - -class TestTagNamesVersion(BaseTestSuite): - @dataclass(frozen=True, kw_only=True) - class TestCase(BaseRegularTestCase): - tag: str - project_version: str - expected: bool - - test_cases = ( - TestCase( - label="tag_matches", - tag="v0.3.0", - project_version="0.3.0", - expected=True, - ), - TestCase( - label="prerelease_matches", - tag="v0.3.0.dev1", - project_version="0.3.0.dev1", - expected=True, - ), - TestCase( - label="patch_differs", - tag="v0.3.1", - project_version="0.3.0", - expected=False, - ), - TestCase( - label="project_ahead_of_tag", - tag="v0.3.0", - project_version="0.4.0", - expected=False, - ), - TestCase( - label="prerelease_against_release", - tag="v0.3.0", - project_version="0.3.0.dev1", - expected=False, - ), - ) - - @pytest.mark.parametrize( - "test_case", - test_cases, - ids=lambda test_case: test_case.label, - ) - def test_tag_names_version( - self, - test_case: TestCase, - ) -> None: - result = check_version_tag.tag_names_version( - tag=test_case.tag, - project_version=test_case.project_version, - ) - - assert result is test_case.expected - - -class TestMain: - def test_matching_version_succeeds( - self, - capsys: pytest.CaptureFixture[str], - ) -> None: - assert ( - check_version_tag.main( - ["--tag", "v0.3.0", "--project-version", "0.3.0"], - ) - == 0 - ) - assert "matches" in capsys.readouterr().out - - def test_mismatched_version_is_annotated_as_an_error( - self, - capsys: pytest.CaptureFixture[str], - ) -> None: - assert ( - check_version_tag.main( - ["--tag", "v0.3.1", "--project-version", "0.3.0"], - ) - == 1 - ) - - output = capsys.readouterr().out - assert output.startswith("::error::") - assert "v0.3.1" in output - assert "0.3.0" in output diff --git a/tests/unit/scripts/ci/test_zip_bundle.py b/tests/unit/scripts/test_archive_bundle.py similarity index 57% rename from tests/unit/scripts/ci/test_zip_bundle.py rename to tests/unit/scripts/test_archive_bundle.py index a5c40633d..1d85dafc8 100644 --- a/tests/unit/scripts/ci/test_zip_bundle.py +++ b/tests/unit/scripts/test_archive_bundle.py @@ -4,11 +4,15 @@ import pytest +from bootstrap.layout import BUNDLES, DISTRIBUTION +from bootstrap.project import read_project +from tests.suite.bootstrap import PROJECT_NAME, PROJECT_VERSION, write_project from tests.suite.scripts import load_script -zip_bundle = load_script("ci/zip_bundle.py") +archive_bundle = load_script("archive_bundle.py") -ROOT = "sampletones-v0.3.0-windows-x86_64" +LABEL = "windows-x86_64" +ROOT = f"{PROJECT_NAME}-v{PROJECT_VERSION}-{LABEL}" LAUNCHER = "sampletones.exe" LIBRARY = "_internal/python312.dll" @@ -37,14 +41,14 @@ def _names(archive: Path) -> List[str]: class TestWriteArchive: def test_every_entry_sits_under_the_root_directory(self, bundle: Path, archive: Path) -> None: - zip_bundle.write_archive(bundle, archive, root=ROOT) + archive_bundle.write_archive(bundle, archive, root=ROOT) names = _names(archive) assert names assert all(name.startswith(f"{ROOT}/") for name in names) def test_bundle_contents_are_archived(self, bundle: Path, archive: Path) -> None: - zip_bundle.write_archive(bundle, archive, root=ROOT) + archive_bundle.write_archive(bundle, archive, root=ROOT) names = _names(archive) assert f"{ROOT}/{LAUNCHER}" in names @@ -52,18 +56,18 @@ def test_bundle_contents_are_archived(self, bundle: Path, archive: Path) -> None assert f"{ROOT}/{NOTICES}" in names def test_empty_directories_are_kept(self, bundle: Path, archive: Path) -> None: - zip_bundle.write_archive(bundle, archive, root=ROOT) + archive_bundle.write_archive(bundle, archive, root=ROOT) assert f"{ROOT}/_internal/empty/" in _names(archive) def test_file_contents_survive_the_round_trip(self, bundle: Path, archive: Path) -> None: - zip_bundle.write_archive(bundle, archive, root=ROOT) + archive_bundle.write_archive(bundle, archive, root=ROOT) with zipfile.ZipFile(archive) as written: assert written.read(f"{ROOT}/{LIBRARY}") == b"library" def test_missing_target_directory_is_created(self, bundle: Path, archive: Path) -> None: - zip_bundle.write_archive(bundle, archive, root=ROOT) + archive_bundle.write_archive(bundle, archive, root=ROOT) assert archive.is_file() @@ -71,14 +75,14 @@ def test_repeated_runs_archive_the_same_order(self, bundle: Path, tmp_path: Path first = tmp_path / "first.zip" second = tmp_path / "second.zip" - zip_bundle.write_archive(bundle, first, root=ROOT) - zip_bundle.write_archive(bundle, second, root=ROOT) + archive_bundle.write_archive(bundle, first, root=ROOT) + archive_bundle.write_archive(bundle, second, root=ROOT) assert _names(first) == _names(second) def test_source_directory_is_left_in_place(self, bundle: Path, archive: Path) -> None: """Archiving reads the bundle where it lies, so the build output keeps its own name.""" - zip_bundle.write_archive(bundle, archive, root=ROOT) + archive_bundle.write_archive(bundle, archive, root=ROOT) assert bundle.is_dir() assert (bundle / LAUNCHER).is_file() @@ -91,25 +95,40 @@ def test_archives_while_a_handle_is_held_inside_the_bundle(self, bundle: Path, a """ with (bundle / LIBRARY).open("rb"): - zip_bundle.write_archive(bundle, archive, root=ROOT) + archive_bundle.write_archive(bundle, archive, root=ROOT) assert f"{ROOT}/{LIBRARY}" in _names(archive) -class TestMain: - def test_reports_success_for_a_built_bundle(self, bundle: Path, archive: Path) -> None: - assert zip_bundle.main([str(bundle), str(archive), "--root", ROOT]) == 0 - assert archive.is_file() +class TestArchiveRoot: + def test_the_root_names_the_project_its_version_and_the_platform(self, tmp_path: Path) -> None: + assert archive_bundle.archive_root(read_project(write_project(tmp_path)), LABEL) == ROOT - def test_reports_a_missing_bundle_directory(self, tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: - absent = tmp_path / "bin" / "sampletones" - assert zip_bundle.main([str(absent), str(tmp_path / "out.zip"), "--root", ROOT]) == 1 +class TestMain: + def test_the_release_bundle_is_archived_into_the_bundles_directory( + self, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + ) -> None: + root = write_project(tmp_path / "repository") + source = root / DISTRIBUTION / PROJECT_NAME + source.mkdir(parents=True) + (source / LAUNCHER).write_bytes(b"MZ") + monkeypatch.setattr(archive_bundle, "repository_root", lambda: root) + + assert archive_bundle.main(["--label", LABEL]) == 0 + assert _names(root / BUNDLES / f"{ROOT}.zip") == [f"{ROOT}/{LAUNCHER}"] + + def test_a_missing_bundle_is_annotated_and_writes_no_archive( + self, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + ) -> None: + root = write_project(tmp_path) + monkeypatch.setattr(archive_bundle, "repository_root", lambda: root) + + assert archive_bundle.main(["--label", LABEL]) == 1 assert "::error::" in capsys.readouterr().out - - def test_a_missing_bundle_writes_no_archive(self, tmp_path: Path) -> None: - archive = tmp_path / "out.zip" - - zip_bundle.main([str(tmp_path / "absent"), str(archive), "--root", ROOT]) - - assert not archive.exists() + assert not (root / BUNDLES).exists() diff --git a/tests/unit/scripts/test_build_environment.py b/tests/unit/scripts/test_build_environment.py index dd047fd2f..e8766fa57 100644 --- a/tests/unit/scripts/test_build_environment.py +++ b/tests/unit/scripts/test_build_environment.py @@ -1,28 +1,27 @@ import pytest +from bootstrap.platforms import macos from bootstrap.platforms.linux import Linux from bootstrap.platforms.macos import MacOS from tests.suite.scripts import load_script build_environment = load_script("build_environment.py") +PORTAUDIO_PREFIX = "/opt/homebrew/opt/portaudio" + class TestMain: - def test_macos_prints_the_flags_one_per_line( + def test_the_platform_s_flags_are_printed_one_per_line( self, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], ) -> None: monkeypatch.setattr(build_environment, "current_platform", MacOS) - monkeypatch.setattr(build_environment, "portaudio_prefix", lambda: "/opt/homebrew/opt/portaudio") + monkeypatch.setattr(macos, "homebrew_prefix", lambda package: PORTAUDIO_PREFIX) monkeypatch.setattr(build_environment.running, "machine", lambda: "arm64") assert build_environment.main([]) == 0 - assert capsys.readouterr().out.splitlines() == [ - "CFLAGS=-I/opt/homebrew/opt/portaudio/include", - "LDFLAGS=-L/opt/homebrew/opt/portaudio/lib", - "ARCHFLAGS=-arch arm64", - ] + assert capsys.readouterr().out.splitlines() == list(MacOS().build_flags(machine="arm64")) def test_linux_prints_nothing( self, diff --git a/tests/unit/scripts/test_bundle.py b/tests/unit/scripts/test_bundle.py index 51faf5ae3..cb4422d80 100644 --- a/tests/unit/scripts/test_bundle.py +++ b/tests/unit/scripts/test_bundle.py @@ -1,147 +1,157 @@ from pathlib import Path -from typing import Sequence +from typing import Callable, Sequence import pytest +from bootstrap.layout import BUILD_ENVIRONMENT, BUILD_TOOLS, DISTRIBUTION, NOTICES, RELEASE_HOOK from bootstrap.platforms.linux import Linux from bootstrap.platforms.macos import MacOS -from tests.suite.bootstrap import RecordingRunner +from bootstrap.platforms.windows import Windows +from bootstrap.project import BUILD_EXTRA, GPU_EXTRA, read_project +from tests.suite.bootstrap import PROJECT_NAME, RecordingRunner, write_project from tests.suite.scripts import load_script bundle = load_script("bundle.py") +def _repository(tmp_path: Path) -> Path: + for notice in NOTICES: + (tmp_path / notice).write_text(notice) + + return write_project(tmp_path) + + class TestPyInstallerCommand: - def test_a_release_is_a_directory_with_the_runtime_hook(self) -> None: + def test_a_release_is_a_directory_with_the_runtime_hook(self, tmp_path: Path) -> None: + project = read_project(_repository(tmp_path)) options = bundle.BundleOptions(release=True, gpu=False) - command = bundle.pyinstaller_command(Path("python"), Linux(), options) + command = bundle.pyinstaller_command(Path("python"), Linux().bundling(), project, options) assert command[:3] == ["python", "-m", "PyInstaller"] assert "--onedir" in command - assert "--runtime-hook" in command - assert command[command.index("--runtime-hook") + 1] == bundle.RELEASE_HOOK - assert command[-1] == bundle.ENTRY + assert command[command.index("--runtime-hook") + 1] == RELEASE_HOOK + assert command[-1] == project.entry_script - def test_a_development_bundle_is_one_file_without_the_hook(self) -> None: - options = bundle.BundleOptions(release=False, gpu=False) + def test_a_development_bundle_is_one_file_without_the_hook(self, tmp_path: Path) -> None: + project = read_project(_repository(tmp_path)) - command = bundle.pyinstaller_command(Path("python"), Linux(), options) + command = bundle.pyinstaller_command( + Path("python"), + Linux().bundling(), + project, + bundle.BundleOptions(release=False, gpu=False), + ) assert "--onefile" in command assert "--runtime-hook" not in command - def test_the_data_and_the_exclusions_ride_along(self) -> None: - command = bundle.pyinstaller_command(Path("python"), Linux(), bundle.BundleOptions(release=False, gpu=False)) + def test_every_package_brings_its_data_and_the_build_tools_stay_out(self, tmp_path: Path) -> None: + project = read_project(_repository(tmp_path)) + + command = bundle.pyinstaller_command( + Path("python"), + Windows().bundling(), + project, + bundle.BundleOptions(release=False, gpu=False), + ) - data = [command[index + 1] for index, flag in enumerate(command) if flag == "--add-data"] - assert data == [f"{source}:{destination}" for source, destination in bundle.DATA] - assert command[command.index("--exclude-module") + 1] == "PIL" - assert command[command.index("--icon") + 1] == Linux().icon + collected = [command[index + 1] for index, flag in enumerate(command) if flag == "--collect-data"] + excluded = [command[index + 1] for index, flag in enumerate(command) if flag == "--exclude-module"] + assert collected == list(project.packages) + assert excluded == list(BUILD_TOOLS) + assert command[command.index("--icon") + 1] == Windows().bundling().icon + assert command[command.index("--name") + 1] == project.name class TestExtras: def test_the_build_extra_is_always_installed_and_gpu_on_request(self) -> None: - assert bundle.extras(bundle.BundleOptions(release=False, gpu=False)) == ("build",) - assert bundle.extras(bundle.BundleOptions(release=False, gpu=True)) == ("build", "gpu") + assert bundle.extras(bundle.BundleOptions(release=False, gpu=False)) == (BUILD_EXTRA,) + assert bundle.extras(bundle.BundleOptions(release=False, gpu=True)) == (BUILD_EXTRA, GPU_EXTRA) class TestRemovePrevious: def test_a_previous_file_and_directory_are_removed(self, tmp_path: Path) -> None: - (tmp_path / "sampletones").mkdir() - (tmp_path / "sampletones.exe").write_text("") + (tmp_path / PROJECT_NAME).mkdir() + (tmp_path / f"{PROJECT_NAME}.exe").write_text("") - bundle.remove_previous(tmp_path) + bundle.remove_previous(tmp_path, Windows().bundling(), PROJECT_NAME) assert list(tmp_path.iterdir()) == [] -def _repository(tmp_path: Path) -> Path: - for notice in bundle.NOTICES: - (tmp_path / notice).write_text(notice) +def _leave_behind(root: Path, launcher: Path) -> Callable[[Sequence[str]], None]: + """What a build leaves on disk: the environment's interpreter, and the launcher where PyInstaller writes it.""" + + def on_run(command: Sequence[str]) -> None: + if "venv" in command: + python = Linux().interpreter(root / BUILD_ENVIRONMENT) + python.parent.mkdir(parents=True) + python.write_text("") - return tmp_path + if "PyInstaller" in command: + launcher.parent.mkdir(parents=True, exist_ok=True) + launcher.write_text("") + + return on_run class TestBuildBundle: def test_a_release_runs_every_step_in_order_and_places_the_notices(self, tmp_path: Path) -> None: root = _repository(tmp_path) - platform = Linux() - options = bundle.BundleOptions(release=True, gpu=False) - launcher = platform.launcher(root / bundle.DISTRIBUTION, release=True) + launcher = Linux().bundling().launcher(root / DISTRIBUTION, name=PROJECT_NAME, release=True) + runner = RecordingRunner({}, _leave_behind(root, launcher)) - def leave_behind(command: Sequence[str]) -> None: - if "venv" in command: - platform.interpreter(root / ".venv-build").parent.mkdir(parents=True) - platform.interpreter(root / ".venv-build").write_text("") - - if "PyInstaller" in command: - launcher.parent.mkdir(parents=True) - launcher.write_text("") - - runner = RecordingRunner({}, leave_behind) - - built = bundle.build_bundle(root, platform, options, runner=runner, environment={}) + built = bundle.build_bundle( + root, Linux(), bundle.BundleOptions(release=True, gpu=False), runner=runner, environment={} + ) assert built == launcher - assert runner.lines[0].endswith(".venv-build") + assert runner.lines[0].endswith(BUILD_ENVIRONMENT) assert "pip install --upgrade pip" in runner.lines[1] - assert ".[build]" in runner.lines[2] + assert f".[{BUILD_EXTRA}]" in runner.lines[2] assert "import pyaudio" in runner.lines[3] assert "import tkinter" in runner.lines[4] assert "PyInstaller" in runner.lines[5] - assert runner.lines[6] == f"{launcher} self-check" - assert all((launcher.parent / notice).read_text() == notice for notice in bundle.NOTICES) + assert runner.lines[6] == f"{launcher} {bundle.SELF_CHECK}" + assert all((launcher.parent / notice).read_text() == notice for notice in NOTICES) def test_a_bundle_pyinstaller_never_wrote_is_reported(self, tmp_path: Path) -> None: root = _repository(tmp_path) - platform = Linux() - - def leave_behind(command: Sequence[str]) -> None: - if "venv" in command: - platform.interpreter(root / ".venv-build").parent.mkdir(parents=True) - platform.interpreter(root / ".venv-build").write_text("") + elsewhere = tmp_path / "elsewhere" / PROJECT_NAME with pytest.raises(SystemExit, match="produced no executable"): bundle.build_bundle( root, - platform, + Linux(), bundle.BundleOptions(release=False, gpu=False), - runner=RecordingRunner({}, leave_behind), + runner=RecordingRunner({}, _leave_behind(root, elsewhere)), environment={}, ) def test_a_launcher_failing_its_self_check_fails_the_build(self, tmp_path: Path) -> None: root = _repository(tmp_path) - platform = Linux() - launcher = platform.launcher(root / bundle.DISTRIBUTION, release=False) - - def leave_behind(command: Sequence[str]) -> None: - if "venv" in command: - platform.interpreter(root / ".venv-build").parent.mkdir(parents=True) - platform.interpreter(root / ".venv-build").write_text("") + launcher = Linux().bundling().launcher(root / DISTRIBUTION, name=PROJECT_NAME, release=False) - if "PyInstaller" in command: - launcher.parent.mkdir(parents=True, exist_ok=True) - launcher.write_text("") - - with pytest.raises(SystemExit, match="self-check"): + with pytest.raises(SystemExit, match=bundle.SELF_CHECK): bundle.build_bundle( root, - platform, + Linux(), bundle.BundleOptions(release=False, gpu=False), - runner=RecordingRunner({"self-check": 1}, leave_behind), + runner=RecordingRunner({bundle.SELF_CHECK: 1}, _leave_behind(root, launcher)), environment={}, ) + def test_a_system_without_bundles_is_told_to_run_from_source_before_anything_runs(self, tmp_path: Path) -> None: + runner = RecordingRunner({}, None) -class TestMain: - def test_a_system_without_bundles_is_told_to_run_from_source( - self, - monkeypatch: pytest.MonkeyPatch, - capsys: pytest.CaptureFixture[str], - ) -> None: - monkeypatch.setattr(bundle, "current_platform", MacOS) + with pytest.raises(SystemExit, match="make setup"): + bundle.build_bundle( + _repository(tmp_path), + MacOS(), + bundle.BundleOptions(release=False, gpu=False), + runner=runner, + environment={}, + ) - assert bundle.main([]) == 1 - assert "make setup" in capsys.readouterr().err + assert runner.lines == [] diff --git a/tests/unit/scripts/test_formatting.py b/tests/unit/scripts/test_formatting.py index 01a62ba3c..691ee31b1 100644 --- a/tests/unit/scripts/test_formatting.py +++ b/tests/unit/scripts/test_formatting.py @@ -1,3 +1,5 @@ +from pathlib import Path + import pytest from tests.suite.bootstrap import RecordingRunner @@ -6,34 +8,29 @@ formatting = load_script("formatting.py") -class TestMain: - def test_isort_runs_before_black_over_the_three_trees( - self, - monkeypatch: pytest.MonkeyPatch, - capsys: pytest.CaptureFixture[str], - ) -> None: - runner = RecordingRunner({}, None) - monkeypatch.setattr(formatting, "run", runner) +class TestFormattedPaths: + def test_the_three_trees_are_formatted_by_default(self) -> None: + assert formatting.formatted_paths(()) == formatting.FORMATTED_TREES - assert formatting.main([]) == 0 - assert runner.lines == [ - "uv run python -m isort src tests scripts", - "uv run python -m black src tests scripts", - ] - assert "Code formatting complete." in capsys.readouterr().out + def test_named_paths_replace_the_trees(self) -> None: + assert formatting.formatted_paths(("scripts/lint.py",)) == ("scripts/lint.py",) - def test_named_paths_replace_the_trees(self, monkeypatch: pytest.MonkeyPatch) -> None: + +class TestFormatCode: + def test_isort_runs_before_black(self, tmp_path: Path) -> None: runner = RecordingRunner({}, None) - monkeypatch.setattr(formatting, "run", runner) - assert formatting.main(["scripts/lint.py"]) == 0 - assert all(line.endswith(" scripts/lint.py") for line in runner.lines) + formatting.format_code(tmp_path, formatting.FORMATTED_TREES, runner=runner, environment={}) + + assert runner.lines == [ + " ".join((*formatting.ISORT, *formatting.FORMATTED_TREES)), + " ".join((*formatting.BLACK, *formatting.FORMATTED_TREES)), + ] - def test_a_failing_formatter_stops_the_run(self, monkeypatch: pytest.MonkeyPatch) -> None: + def test_a_failing_formatter_stops_the_run(self, tmp_path: Path) -> None: runner = RecordingRunner({"isort": 1}, None) - monkeypatch.setattr(formatting, "run", runner) with pytest.raises(SystemExit, match="isort"): - formatting.main([]) + formatting.format_code(tmp_path, formatting.FORMATTED_TREES, runner=runner, environment={}) assert len(runner.lines) == 1 diff --git a/tests/unit/scripts/test_hooks.py b/tests/unit/scripts/test_hooks.py index bbd9a0529..1364dcdbb 100644 --- a/tests/unit/scripts/test_hooks.py +++ b/tests/unit/scripts/test_hooks.py @@ -1,4 +1,4 @@ -import pytest +from pathlib import Path from tests.suite.bootstrap import RecordingRunner from tests.suite.scripts import load_script @@ -6,15 +6,12 @@ hooks = load_script("hooks.py") -class TestMain: - def test_both_hook_stages_are_installed( - self, - monkeypatch: pytest.MonkeyPatch, - capsys: pytest.CaptureFixture[str], - ) -> None: +class TestInstallHooks: + def test_both_hook_stages_are_installed_from_the_repository(self, tmp_path: Path) -> None: runner = RecordingRunner({}, None) - monkeypatch.setattr(hooks, "run", runner) - assert hooks.main([]) == 0 - assert runner.lines == ["uv run pre-commit install --hook-type pre-commit --hook-type pre-push"] - assert "Pre-commit hooks installed." in capsys.readouterr().out + hooks.install_hooks(tmp_path, runner=runner, environment={}) + + assert runner.lines == [" ".join(hooks.INSTALL_HOOKS)] + assert "--hook-type pre-commit --hook-type pre-push" in runner.lines[0] + assert runner.commands[0].cwd == tmp_path diff --git a/tests/unit/scripts/test_lint.py b/tests/unit/scripts/test_lint.py index 6f3f1587d..5f517e2de 100644 --- a/tests/unit/scripts/test_lint.py +++ b/tests/unit/scripts/test_lint.py @@ -1,3 +1,5 @@ +from pathlib import Path + import pytest from tests.suite.bootstrap import RecordingRunner @@ -10,8 +12,8 @@ class TestLinters: def test_mypy_reads_its_configuration_and_pylint_sweeps_the_trees(self) -> None: mypy, pylint = lint.linters(()) - assert mypy.command == ("uv", "run", "python", "-m", "mypy") - assert pylint.command == ("uv", "run", "python", "-m", "pylint", "src", "scripts") + assert mypy.command == ("uv", "run", "python", "-m", lint.MYPY) + assert pylint.command == ("uv", "run", "python", "-m", lint.PYLINT, *lint.LINTED_TREES) def test_named_paths_reach_both(self) -> None: mypy, pylint = lint.linters(("scripts/lint.py",)) @@ -20,36 +22,29 @@ def test_named_paths_reach_both(self) -> None: assert pylint.command[-1] == "scripts/lint.py" -class TestMain: - def test_both_linters_run_by_default( - self, - monkeypatch: pytest.MonkeyPatch, - capsys: pytest.CaptureFixture[str], - ) -> None: - runner = RecordingRunner({}, None) - monkeypatch.setattr(lint, "run", runner) +class TestChosenLinters: + def test_both_linters_run_by_default(self) -> None: + assert [linter.name for linter in lint.chosen_linters((), mypy=False, pylint=False)] == [lint.MYPY, lint.PYLINT] + + def test_a_flag_picks_one_linter(self) -> None: + assert [linter.name for linter in lint.chosen_linters((), mypy=False, pylint=True)] == [lint.PYLINT] - assert lint.main([]) == 0 - assert [line.split()[-1] for line in runner.lines[:1]] == ["mypy"] - assert "pylint" in runner.lines[1] - assert "All linting checks passed." in capsys.readouterr().out - def test_a_flag_picks_one_linter(self, monkeypatch: pytest.MonkeyPatch) -> None: +class TestLintCode: + def test_every_linter_passing_is_reported(self, tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: runner = RecordingRunner({}, None) - monkeypatch.setattr(lint, "run", runner) - assert lint.main(["--pylint"]) == 0 - assert len(runner.lines) == 1 - assert "pylint src scripts" in runner.lines[0] + assert lint.lint_code(tmp_path, lint.linters(()), runner=runner, environment={}) == 0 + assert len(runner.lines) == 2 + assert "All linting checks passed." in capsys.readouterr().out def test_a_failing_linter_stops_nothing_and_is_named( self, - monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, capsys: pytest.CaptureFixture[str], ) -> None: - runner = RecordingRunner({"mypy": 1}, None) - monkeypatch.setattr(lint, "run", runner) + runner = RecordingRunner({lint.MYPY: 1}, None) - assert lint.main([]) == 1 + assert lint.lint_code(tmp_path, lint.linters(()), runner=runner, environment={}) == 1 assert len(runner.lines) == 2 - assert "Linting failed: mypy." in capsys.readouterr().out + assert f"Linting failed: {lint.MYPY}." in capsys.readouterr().out diff --git a/tests/unit/scripts/test_run_tests.py b/tests/unit/scripts/test_run_tests.py index b0be40d0f..99e47f061 100644 --- a/tests/unit/scripts/test_run_tests.py +++ b/tests/unit/scripts/test_run_tests.py @@ -1,4 +1,5 @@ from dataclasses import dataclass +from pathlib import Path from typing import Tuple import pytest @@ -38,20 +39,19 @@ def test_the_benchmarks_run_serial_uncovered_and_show_their_readings(self) -> No assert "-n" not in command -class TestMain: - def test_the_named_pass_runs_alone(self, monkeypatch: pytest.MonkeyPatch) -> None: +class TestRunPass: + def test_the_pass_runs_from_the_repository(self, tmp_path: Path) -> None: runner = RecordingRunner({}, None) - monkeypatch.setattr(run_tests, "run", runner) + chosen = run_tests.planned_passes(run_tests.DEFAULT_WORKERS)[run_tests.DOCTESTS] - assert run_tests.main([run_tests.DOCTESTS]) == 0 - assert runner.lines == [ - " ".join(run_tests.planned_passes(run_tests.DEFAULT_WORKERS)[run_tests.DOCTESTS].command) - ] + assert run_tests.run_pass(chosen, tmp_path, runner=runner, environment={}) == 0 + assert runner.lines == [" ".join(chosen.command)] + assert runner.commands[0].cwd == tmp_path - def test_the_status_is_pytest_s_own(self, monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr(run_tests, "run", RecordingRunner({"pytest": 5}, None)) + def test_the_status_is_pytest_s_own(self, tmp_path: Path) -> None: + chosen = run_tests.planned_passes("auto")[run_tests.SUITE] - assert run_tests.main([run_tests.SUITE, "--workers", "auto"]) == 5 + assert run_tests.run_pass(chosen, tmp_path, runner=RecordingRunner({"pytest": 5}, None), environment={}) == 5 class TestRefusedPasses(BaseTestSuite): @@ -65,16 +65,8 @@ class TestCase(BaseRegularTestCase): ) @pytest.mark.parametrize("test_case", test_cases, ids=lambda test_case: test_case.label) - def test_a_pass_outside_the_three_is_refused_before_anything_runs( - self, - monkeypatch: pytest.MonkeyPatch, - test_case: TestCase, - ) -> None: - runner = RecordingRunner({}, None) - monkeypatch.setattr(run_tests, "run", runner) - + def test_a_pass_outside_the_three_is_refused(self, test_case: TestCase) -> None: with pytest.raises(SystemExit) as exit_info: run_tests.main(test_case.argv) assert exit_info.value.code == 2 - assert runner.lines == [] diff --git a/tests/unit/scripts/test_setup_environment.py b/tests/unit/scripts/test_setup_environment.py index c665ecbfb..63fe93d75 100644 --- a/tests/unit/scripts/test_setup_environment.py +++ b/tests/unit/scripts/test_setup_environment.py @@ -1,5 +1,11 @@ +from pathlib import Path + import pytest +from bootstrap.platforms import macos +from bootstrap.platforms.linux import Linux +from bootstrap.platforms.macos import ARCHFLAGS, MacOS +from bootstrap.project import DEVELOPMENT_GROUP, GPU_CUDA11_EXTRA, GPU_EXTRA from tests.suite.bootstrap import RecordingRunner from tests.suite.scripts import load_script @@ -8,30 +14,25 @@ class TestGpuExtra: def test_zero_keeps_the_cpu_backend(self) -> None: - assert setup_environment.gpu_extra("0", system="Linux") is None + assert setup_environment.gpu_extra(setup_environment.GPU_OFF, Linux(), {}) is None def test_a_named_extra_is_taken_as_given(self) -> None: - assert setup_environment.gpu_extra("gpu-cuda11", system="Linux") == "gpu-cuda11" + assert setup_environment.gpu_extra(GPU_CUDA11_EXTRA, Linux(), {}) == GPU_CUDA11_EXTRA def test_auto_on_macos_keeps_the_cpu_backend(self) -> None: - assert setup_environment.gpu_extra("auto", system="Darwin") is None + assert setup_environment.gpu_extra(setup_environment.GPU_AUTO, MacOS(), {}) is None class TestMain: def test_a_gpu_choice_outside_the_extras_is_refused_before_anything_runs( self, - monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], ) -> None: - runner = RecordingRunner({}, None) - monkeypatch.setattr(setup_environment, "run", runner) - with pytest.raises(SystemExit) as exit_info: setup_environment.main(["--gpu", "1"]) refusal = capsys.readouterr().err assert exit_info.value.code == 2 - assert runner.lines == [] assert all(choice in refusal for choice in setup_environment.GPU_CHOICES) @@ -39,28 +40,49 @@ class TestSetupCommands: def test_the_cpu_backend_synchronizes_and_installs_the_bare_package(self) -> None: commands = setup_environment.setup_commands(None) - assert len(commands) == 2 - assert commands[0] == ["uv", "sync", "--group", "dev"] - assert commands[1] == ["uv", "tool", "install", "--force", "."] + assert commands == [ + ["uv", "sync", "--group", DEVELOPMENT_GROUP], + ["uv", "tool", "install", "--force", "."], + ] def test_a_gpu_extra_reaches_both_installs(self) -> None: - commands = setup_environment.setup_commands("gpu") + commands = setup_environment.setup_commands(GPU_EXTRA) - assert commands[0] == ["uv", "sync", "--group", "dev", "--extra", "gpu"] - assert commands[1] == ["uv", "tool", "install", "--force", ".[gpu]"] + assert commands[0] == ["uv", "sync", "--group", DEVELOPMENT_GROUP, "--extra", GPU_EXTRA] + assert commands[1] == ["uv", "tool", "install", "--force", f".[{GPU_EXTRA}]"] -class TestSetupEnvironmentVariables: - def test_macos_pins_the_native_architecture(self) -> None: - variables = setup_environment.setup_environment_variables( - {"PATH": "/usr/bin"}, system="Darwin", machine="arm64" +class TestSetUpEnvironment: + def test_macos_runs_the_commands_on_its_native_architecture( + self, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + monkeypatch.setattr(macos.shutil, "which", lambda name: None) + runner = RecordingRunner({}, None) + + setup_environment.set_up_environment( + tmp_path, + MacOS(), + None, + machine="arm64", + runner=runner, + environment={"PATH": "/usr/bin"}, ) - assert variables == {"PATH": "/usr/bin", "ARCHFLAGS": "-arch arm64"} + assert runner.lines == [" ".join(command) for command in setup_environment.setup_commands(None)] + assert all(recorded.environment[ARCHFLAGS] == "-arch arm64" for recorded in runner.commands) + + def test_linux_passes_the_variables_through(self, tmp_path: Path) -> None: + runner = RecordingRunner({}, None) - def test_other_systems_pass_the_variables_through(self) -> None: - variables = setup_environment.setup_environment_variables( - {"PATH": "/usr/bin"}, system="Linux", machine="x86_64" + setup_environment.set_up_environment( + tmp_path, + Linux(), + GPU_EXTRA, + machine="x86_64", + runner=runner, + environment={"PATH": "/usr/bin"}, ) - assert variables == {"PATH": "/usr/bin"} + assert all(recorded.environment == {"PATH": "/usr/bin"} for recorded in runner.commands) diff --git a/tests/unit/scripts/test_system_dependencies.py b/tests/unit/scripts/test_system_dependencies.py index 01ed5bbcb..d4194a8ce 100644 --- a/tests/unit/scripts/test_system_dependencies.py +++ b/tests/unit/scripts/test_system_dependencies.py @@ -1,3 +1,5 @@ +from pathlib import Path + import pytest from bootstrap.platforms import macos @@ -10,42 +12,45 @@ system_dependencies = load_script("system_dependencies.py") -class TestMain: - def test_windows_has_nothing_to_install( - self, - monkeypatch: pytest.MonkeyPatch, - capsys: pytest.CaptureFixture[str], - ) -> None: - monkeypatch.setattr(system_dependencies, "current_platform", Windows) +class TestInstallSystemPackages: + def test_windows_has_nothing_to_install(self, tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: + runner = RecordingRunner({}, None) - assert system_dependencies.main([]) == 0 + assert system_dependencies.install_system_packages(tmp_path, Windows(), runner=runner, environment={}) == 0 + assert runner.lines == [] assert "Nothing to install" in capsys.readouterr().out - def test_linux_runs_the_apt_commands_in_order(self, monkeypatch: pytest.MonkeyPatch) -> None: + def test_linux_runs_the_apt_commands_in_order(self, tmp_path: Path) -> None: runner = RecordingRunner({}, None) - monkeypatch.setattr(system_dependencies, "current_platform", Linux) - monkeypatch.setattr(system_dependencies, "run", runner) - assert system_dependencies.main([]) == 0 - assert runner.lines[0] == "sudo apt-get update" - assert runner.lines[1].startswith("sudo apt-get install -y") + assert system_dependencies.install_system_packages(tmp_path, Linux(), runner=runner, environment={}) == 0 + assert runner.lines == [" ".join(command) for command in Linux().system_packages()] - def test_macos_with_homebrew_installs_portaudio(self, monkeypatch: pytest.MonkeyPatch) -> None: + def test_macos_with_homebrew_installs_portaudio(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: runner = RecordingRunner({}, None) - monkeypatch.setattr(system_dependencies, "current_platform", MacOS) - monkeypatch.setattr(system_dependencies, "run", runner) monkeypatch.setattr(macos.shutil, "which", lambda name: f"/opt/homebrew/bin/{name}") - assert system_dependencies.main([]) == 0 + assert system_dependencies.install_system_packages(tmp_path, MacOS(), runner=runner, environment={}) == 0 assert runner.lines == [" ".join(command) for command in MacOS().system_packages()] def test_macos_without_homebrew_is_refused( self, + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], ) -> None: - monkeypatch.setattr(system_dependencies, "current_platform", MacOS) + runner = RecordingRunner({}, None) monkeypatch.setattr(macos.shutil, "which", lambda name: None) - assert system_dependencies.main([]) == 1 + assert system_dependencies.install_system_packages(tmp_path, MacOS(), runner=runner, environment={}) == 1 + assert runner.lines == [] assert "Homebrew" in capsys.readouterr().err + + def test_a_failing_install_stops_the_script(self, tmp_path: Path) -> None: + with pytest.raises(SystemExit, match="apt-get update"): + system_dependencies.install_system_packages( + tmp_path, + Linux(), + runner=RecordingRunner({"update": 100}, None), + environment={}, + ) diff --git a/tests/unit/scripts/test_verify_bundle.py b/tests/unit/scripts/test_verify_bundle.py new file mode 100644 index 000000000..5bc6d90fe --- /dev/null +++ b/tests/unit/scripts/test_verify_bundle.py @@ -0,0 +1,105 @@ +from pathlib import Path + +import pytest + +from bootstrap.layout import BUILD_TOOLS, DISTRIBUTION, NOTICES +from bootstrap.platforms.linux import Linux +from bootstrap.platforms.macos import MacOS +from tests.suite.bootstrap import PROJECT_NAME, RecordingRunner, write_project +from tests.suite.scripts import load_script + +verify_bundle = load_script("verify_bundle.py") + + +@pytest.fixture(name="root") +def root_fixture(tmp_path: Path) -> Path: + """A repository holding a release bundle with its notices and its launcher.""" + launcher = Linux().bundling().launcher(tmp_path / DISTRIBUTION, name=PROJECT_NAME, release=True) + launcher.parent.mkdir(parents=True) + launcher.write_bytes(b"launcher") + for name in NOTICES: + (launcher.parent / name).write_text(name) + + return write_project(tmp_path) + + +def _bundle(root: Path) -> Path: + return root / DISTRIBUTION / PROJECT_NAME + + +class TestMissingNotices: + def test_a_complete_bundle_lacks_nothing(self, root: Path) -> None: + assert verify_bundle.missing_notices(_bundle(root)) == [] + + def test_every_absent_notice_is_reported(self, root: Path) -> None: + for name in NOTICES[1:]: + (_bundle(root) / name).unlink() + + assert verify_bundle.missing_notices(_bundle(root)) == list(NOTICES[1:]) + + def test_a_notice_directory_counts_as_absent(self, root: Path) -> None: + (_bundle(root) / NOTICES[0]).unlink() + (_bundle(root) / NOTICES[0]).mkdir() + + assert verify_bundle.missing_notices(_bundle(root)) == [NOTICES[0]] + + +class TestCarriedBuildTools: + def test_an_application_bundle_holds_to_its_notices(self, root: Path) -> None: + assert verify_bundle.carried_build_tools(_bundle(root)) == [] + + def test_a_build_tool_beside_the_application_is_reported(self, root: Path) -> None: + (_bundle(root) / verify_bundle.INTERNAL_DIRECTORY / BUILD_TOOLS[0]).mkdir(parents=True) + + assert verify_bundle.carried_build_tools(_bundle(root)) == [BUILD_TOOLS[0]] + + def test_a_build_tool_beside_the_launcher_is_reported(self, root: Path) -> None: + (_bundle(root) / BUILD_TOOLS[0]).mkdir() + + assert verify_bundle.carried_build_tools(_bundle(root)) == [BUILD_TOOLS[0]] + + +class TestBundleFailures: + def test_a_complete_bundle_passes_once_its_launcher_starts(self, root: Path) -> None: + runner = RecordingRunner({}, None) + + assert verify_bundle.bundle_failures(root, Linux(), runner=runner, environment={}) == [] + assert runner.lines == [f"{_bundle(root) / PROJECT_NAME} {verify_bundle.VERSION_FLAG}"] + + def test_a_missing_notice_is_named(self, root: Path) -> None: + (_bundle(root) / NOTICES[1]).unlink() + + failures = verify_bundle.bundle_failures(root, Linux(), runner=RecordingRunner({}, None), environment={}) + + assert len(failures) == 1 + assert NOTICES[1] in failures[0] + + def test_bundled_build_tooling_is_named(self, root: Path) -> None: + (_bundle(root) / verify_bundle.INTERNAL_DIRECTORY / BUILD_TOOLS[0]).mkdir(parents=True) + + failures = verify_bundle.bundle_failures(root, Linux(), runner=RecordingRunner({}, None), environment={}) + + assert [BUILD_TOOLS[0] in failure for failure in failures] == [True] + + def test_a_missing_launcher_is_named_and_nothing_runs(self, root: Path) -> None: + (_bundle(root) / PROJECT_NAME).unlink() + runner = RecordingRunner({}, None) + + failures = verify_bundle.bundle_failures(root, Linux(), runner=runner, environment={}) + + assert [("offers no launcher" in failure) for failure in failures] == [True] + assert runner.lines == [] + + def test_a_launcher_that_fails_is_named_with_its_status(self, root: Path) -> None: + failures = verify_bundle.bundle_failures( + root, + Linux(), + runner=RecordingRunner({verify_bundle.VERSION_FLAG: 3}, None), + environment={}, + ) + + assert [("status 3" in failure) for failure in failures] == [True] + + def test_a_system_without_bundles_is_refused(self, root: Path) -> None: + with pytest.raises(SystemExit, match="make setup"): + verify_bundle.bundle_failures(root, MacOS(), runner=RecordingRunner({}, None), environment={}) diff --git a/tests/unit/scripts/test_verify_version_tag.py b/tests/unit/scripts/test_verify_version_tag.py new file mode 100644 index 000000000..30ef81774 --- /dev/null +++ b/tests/unit/scripts/test_verify_version_tag.py @@ -0,0 +1,43 @@ +from dataclasses import dataclass + +import pytest + +from bootstrap.layout import repository_root +from bootstrap.project import read_project +from tests.suite.base import BaseTestSuite +from tests.suite.case import BaseRegularTestCase +from tests.suite.scripts import load_script + +verify_version_tag = load_script("verify_version_tag.py") + + +class TestVersionFromTag(BaseTestSuite): + @dataclass(frozen=True, kw_only=True) + class TestCase(BaseRegularTestCase): + tag: str + expected: str + + test_cases = ( + TestCase(label="release_tag", tag="v0.3.0", expected="0.3.0"), + TestCase(label="prerelease_tag", tag="v0.3.0.dev1", expected="0.3.0.dev1"), + TestCase(label="release_candidate", tag="v1.0.0rc2", expected="1.0.0rc2"), + TestCase(label="bare_version", tag="0.3.0", expected="0.3.0"), + TestCase(label="single_prefix_is_dropped", tag="vv0.3.0", expected="v0.3.0"), + ) + + @pytest.mark.parametrize("test_case", test_cases, ids=lambda test_case: test_case.label) + def test_version_from_tag(self, test_case: TestCase) -> None: + assert verify_version_tag.version_from_tag(test_case.tag) == test_case.expected + + +class TestMain: + def test_the_tag_naming_the_project_version_passes(self) -> None: + version = read_project(repository_root()).version + + assert verify_version_tag.main(["--tag", f"{verify_version_tag.TAG_PREFIX}{version}"]) == 0 + + def test_a_tag_naming_another_version_is_annotated_as_an_error(self, capsys: pytest.CaptureFixture[str]) -> None: + version = read_project(repository_root()).version + + assert verify_version_tag.main(["--tag", f"{verify_version_tag.TAG_PREFIX}{version}.post9"]) == 1 + assert capsys.readouterr().out.startswith("::error::") From cb526e17d2a686da5b8919acaf12b60a57f18f28 Mon Sep 17 00:00:00 2001 From: JakimPL Date: Sun, 13 Sep 2026 20:42:14 +0200 Subject: [PATCH 27/36] Corrected: the stale facts in the user documentation --- README.md | 82 ++++-------------------------- docs/api/index.md | 19 ++++--- docs/guide/command-line.md | 80 ++++++++++++++++++----------- docs/guide/converting.md | 6 +-- docs/guide/installation.md | 101 +++++++++++++++++++------------------ docs/guide/sequencer.md | 14 ++--- 6 files changed, 136 insertions(+), 166 deletions(-) diff --git a/README.md b/README.md index 4098bb697..defacb2cb 100644 --- a/README.md +++ b/README.md @@ -37,79 +37,19 @@ It supports: ## Installation -### Standalone bundle +You can install _SampleToNES_ in three ways: -The easiest way to use _SampleToNES_ is to download the release package for your platform from the -[releases page](https://github.com/JakimPL/SampleToNES/releases), extract it, and run -`sampletones` from inside the extracted folder. On Linux you may need `chmod +x sampletones`. +- **Download a release** for Windows or Linux from the [releases page](https://github.com/JakimPL/SampleToNES/releases), extract it, and start `sampletones`. +- **Install from PyPI** on Windows, macOS or Linux. You need Python 3.12 or newer: -### Requirements + ```sh + uv tool install sampletones # or: pipx install sampletones + sampletones + ``` -- Windows, macOS, or Linux -- Python 3.12 or newer (https://www.python.org/downloads/) +- **Run from source** with Python 3.12 or newer and [uv](https://docs.astral.sh/uv/): `make system-deps`, `make setup`, then `make run`. -### From PyPI - -The quickest way to get the `sampletones` command. Because _SampleToNES_ is an application -rather than a library, installing it into its own isolated environment is recommended: - -```sh -uv tool install sampletones # or: pipx install sampletones -sampletones # launch the GUI -``` - -A plain `pip install sampletones` into an active virtual environment works too. - -To install with GPU support, request the `gpu` extra (see [GPU acceleration](#gpu-acceleration)): - -```sh -uv tool install "sampletones[gpu]" -``` - -On Linux and macOS, audio playback and file dialogs rely on system libraries that come from -the platform's package manager. Install them first: - -```sh -sudo apt-get install libportaudio2 libasound2 python3-tk # Debian/Ubuntu -brew install portaudio # macOS -``` -### Building the executable yourself - -You only need Python 3.12. - -#### Windows - -1. Install Python 3.12. -2. Double-click `install.bat`. It builds `bin\sampletones.exe`. -3. Double-click `bin\sampletones.exe` to start. - -#### Linux - -1. Install the audio and file-dialog system packages: `make system-deps` (or run `python3 scripts/system_dependencies.py`). -2. Install Python 3.12, then run `./install.sh` in a terminal. It builds a `bin/sampletones` executable. -3. Run `./bin/sampletones` to start. - -### Run from source - -For development. Requires [uv](https://docs.astral.sh/uv/) (and, on Linux, the system packages from the Linux steps above): - -```sh -make setup # create the environment and install the sampletones command -make run # run the app -``` - -To update the global command after pulling new changes, re-run `make setup` (or `uv tool install --force .`). - -### GPU acceleration - -_SampleToNES_ can use an NVIDIA GPU (via [_CuPy_](https://cupy.dev/) and CUDA) to speed up instruction-library generation and reconstruction. `make setup` detects your NVIDIA driver and installs the matching CuPy build automatically: - -```sh -make setup # installs GPU support when a supported driver is present -make setup GPU=0 # forces the CPU (NumPy) backend -``` - -A current NVIDIA driver is all you need — the CUDA components ship with the CuPy build, on Linux and Windows alike. On macOS the app runs on the CPU. +An NVIDIA graphics card can speed up conversion. The [installation guide](https://github.com/JakimPL/SampleToNES/blob/main/docs/guide/installation.md) covers the system libraries each way needs, GPU support, and building a standalone app yourself. ## Usage @@ -132,11 +72,11 @@ sampletones convert --config -o # reconstruct sampletones library --config # generate an instruction library ``` -Run `sampletones --help` for the commands and `sampletones --help` for a command's options. +Run `sampletones --help` for the commands and `sampletones --help` for a command's options. The [command-line guide](https://github.com/JakimPL/SampleToNES/blob/main/docs/guide/command-line.md) explains them. ## Documentation -Internals — the reconstruction algorithms, file formats, the Python API, and developer notes — live in [`docs/`](https://github.com/JakimPL/SampleToNES/tree/main/docs). +The [user guide](https://github.com/JakimPL/SampleToNES/tree/main/docs/guide) explains how to use the app, step by step. The rest of [`docs/`](https://github.com/JakimPL/SampleToNES/tree/main/docs) covers the reconstruction algorithms, the file formats, the Python API and the developer notes. ## License diff --git a/docs/api/index.md b/docs/api/index.md index 81819275e..2fa75e32f 100644 --- a/docs/api/index.md +++ b/docs/api/index.md @@ -1,8 +1,11 @@ # Python API -This page is for using _SampleToNES_ as a library from your own Python code. Everything you need is re-exported from the top-level `sampletones` package — its facade — so `from sampletones import ...` is the whole public surface. Consult this page when you want to render instructions, generate a library, or run and export a reconstruction outside the application. +This page is for using _SampleToNES_ as a library in your own Python code. Use it to render instructions, generate a library, or run and export a reconstruction outside the application. -A few lower-level helpers used in the examples (`write_wave`, `generate_library`) are not part of the facade; they are imported from `sampletones_core` with their full path, as shown. +Names come from two packages: + +- The names in the table below come from `sampletones`: `from sampletones import ...`. +- The examples also use a few helpers from `sampletones_core`: `write_wave`, `generate_library`, `DEFAULT_CHANNELS`, and the FamiTracker instrument writers. Import them with the full path each example shows. ## Public surface @@ -121,15 +124,19 @@ reconstruction = Reconstruction.load("reconstruction.stn") ### Export instruments -`Reconstruction.export` returns the per-channel [features](../formats/instruction-libraries.md), one entry per channel, and each set saves as a FamiTracker `.fti` instrument: +`Reconstruction.export` returns the [envelopes](../formats/famitracker.md#b-the-2a03-instrument) of each channel. `build_instrument` turns one channel's envelopes into a FamiTracker instrument, and `write_fti` saves it as an `.fti` file: ```python from sampletones import Reconstruction +from sampletones_core.formats.famitracker.builder import build_instrument +from sampletones_core.formats.famitracker.instrument import write_fti +from sampletones_core.formats.famitracker.specification.instruments import STANDALONE_INSTRUMENT_INDEX reconstruction = Reconstruction.load("reconstruction.stn") -for name, features in reconstruction.export().items(): - features.save(f"{name}.fti", instrument_name=str(name)) +for channel, features in reconstruction.export().items(): + instrument = build_instrument(STANDALONE_INSTRUMENT_INDEX, channel.value, features) + write_fti(f"{channel.value}.fti", instrument) ``` -This writes one `.fti` per channel. A complete FamiTracker `.ftm` module is assembled from a project in the application, not from a single reconstruction — see [FamiTracker formats](../formats/famitracker.md). +This writes one `.fti` per channel, named after the channel. A complete FamiTracker `.ftm` module is assembled from a project in the application, not from a single reconstruction — see [FamiTracker formats](../formats/famitracker.md). diff --git a/docs/guide/command-line.md b/docs/guide/command-line.md index 22072399f..0d5df76ce 100644 --- a/docs/guide/command-line.md +++ b/docs/guide/command-line.md @@ -1,42 +1,62 @@ # Command line -You can run _SampleToNES_ from a terminal: to reconstruct without opening the interface, to -generate a library, or to open a file directly in the app. Every operation is a named command, -and `sampletones` alone starts the interface. +You can run _SampleToNES_ from a terminal. Use it to convert recordings without opening the +window, to build an instruction library, or to open a file in the app. -The command is `sampletones` when installed from source; a standalone build is the executable you -made (`./bin/sampletones` on Linux, `bin\sampletones.exe` on Windows). +## Getting the command + +How you run the command depends on how you installed _SampleToNES_: + +- **Release download**: run `sampletones` in the extracted folder. On Windows the file is + `sampletones.exe`. +- **PyPI**: `uv tool install sampletones` or `pipx install sampletones` puts `sampletones` on your + path. +- **From source**: `make setup` puts `sampletones` on your path. A standalone build you make + yourself is `./bin/sampletones` on Linux and `bin\sampletones.exe` on Windows. + +[Installation](installation.md) explains each way. ## Commands -| Command | Purpose | +| Command | What it does | | --- | --- | -| `sampletones`, `sampletones run` | start the interface | -| `sampletones open ` | start the interface with a `.stp` project, a `.stn` reconstruction or an `.ins` library loaded | -| `sampletones convert ...` | reconstruct recordings into a `.stn` file, or every recording under one folder | -| `sampletones library` | build the instruction library for a configuration, then exit | -| `sampletones self-check` | verify that this build's imports, bundled resources and configuration files are all usable, then exit | -| `sampletones --version` | print the version | +| `sampletones` or `sampletones run` | Starts the app. | +| `sampletones open ` | Starts the app with a `.stp` project, a `.stn` reconstruction or an `.ins` library open. | +| `sampletones convert ...` | Converts recordings into one `.stn` reconstruction, or converts every recording in a folder. | +| `sampletones library` | Builds the instruction library for a configuration, then exits. | +| `sampletones self-check` | Checks that the app can start: its code, its fonts and icons, and its configuration files. | +| `sampletones --version` or `sampletones -v` | Prints the version. | + +Add `--help` to any command to see its options. -Every command lists its options with `--help`. +`sampletones --help` also lists commands for developing _SampleToNES_, such as `check` and +`codec`. They run from a copy of the source code, and [Tooling](../development/tooling.md) +describes them. + +## Options + +- `--config ` or `-c ` uses a configuration file. It works with `run`, `open`, + `convert` and `library`. Without it, the app uses your saved configuration, or the built-in + defaults if you have not saved one. +- `--output ` or `-o ` sets where `convert` saves the reconstruction. Without it, the + reconstruction goes to the reconstructions folder of your configuration. +- `--channels ` sets the channels `convert` may use, for example `--channels pulse1,pulse2`. + Without it, `convert` uses pulse 1, triangle and noise. +- `--stems ` gives `convert` a stems file, which it needs to mix several recordings. + +A `convert` command takes either `--channels` or `--stems`. ## Common tasks -* **Reconstruct a file** — `sampletones convert input.wav -o output.stn` -* **Reconstruct a folder** — `sampletones convert path/to/folder` reconstructs every audio - file inside it, into the reconstructions folder your configuration names. -* **Choose the channels** — add `--channels pulse1,pulse2` to reconstruct onto those two - alone; without it a run uses pulse 1, triangle, and noise. -* **Mix several recordings into one reconstruction** — `sampletones convert bass.wav lead.wav - --stems stems.json`. The stems file describes the same setup the interface's stems list - builds: one entry per recording, in order, each naming the channels it may occupy and the - ones it bends. The command prints which recording plays under which stem before it starts. +- **Convert one file**: `sampletones convert input.wav -o output.stn` +- **Convert a folder**: `sampletones convert path/to/folder`. Every recording in the folder + becomes its own reconstruction, saved in your reconstructions folder. +- **Mix several recordings into one reconstruction**: + `sampletones convert bass.wav lead.wav --stems stems.json`. The stems file has one entry per + recording, in the same order. Each entry lists the channels the recording may use and the + channels it bends. The command prints which recording uses which entry before it starts. [Reconstructions](../formats/reconstructions.md) shows the file. -* **Use a specific configuration** — add `--config my-config.json` to `run`, `open`, `convert` - or `library`; otherwise your saved configuration is used (`config.json`, or built-in defaults - if you have not saved one yet). -* **Generate a library and exit** — `sampletones library --config my-config.json` - -GPU acceleration is selected at setup, not per run: `make setup` detects a supported -NVIDIA driver and installs the matching build (`make setup GPU=0` forces the CPU -backend) — see [Installation](installation.md). +- **Build a library**: `sampletones library --config my-config.json` + +GPU support is chosen when you install _SampleToNES_. [Installation](installation.md) explains +how. diff --git a/docs/guide/converting.md b/docs/guide/converting.md index ce11aacb6..162a24084 100644 --- a/docs/guide/converting.md +++ b/docs/guide/converting.md @@ -33,11 +33,11 @@ A folder represents all the recordings inside it. Its checkbox shows their chann Click the checkbox to change the channel for all recordings in the folder. To change the channel for one recording, open the folder first: click the marker next to the folder name, or double-click the folder name. Each recording inside then has its own checkboxes. -The **Source settings** card has two more settings. **Drive** sets how hard the channels are pushed. It applies to the whole conversion. +The **Source settings** card has **Drive**, which sets how hard the channels are pushed. It applies to the whole conversion. -Below it, the card shows the name of the recording you selected in the list and a checkbox for each of its channels. **Pulse 1**, **Pulse 2**, and **Triangle** also have a **bend** checkbox. This tunes each note to the recording's exact pitch. Noise has no bend. +Below **Drive**, the card shows the name of the recording you selected in the list and a checkbox for each of its channels. **Pulse 1**, **Pulse 2**, and **Triangle** also have a **bend** checkbox. This tunes each note to the recording's exact pitch. Noise has no bend. -**Channels per source** limits how many channels one recording can use at the same time, from 1 to 4. Set it to 1 to make each recording use one channel. +**Channels per source** is on the **Converter** card, below the list. It limits how many channels one recording can use at the same time, from 1 to 4. Set it to 1 to make each recording use one channel. ## One reconstruction each, or one mix from all diff --git a/docs/guide/installation.md b/docs/guide/installation.md index 5b888a5d2..d21538a81 100644 --- a/docs/guide/installation.md +++ b/docs/guide/installation.md @@ -1,77 +1,80 @@ # Installation -There are two ways to run _SampleToNES_: a **standalone build** (the easiest, and -all most people need) and **running from source** (for development, and on macOS). -GPU acceleration is optional. +You can install _SampleToNES_ in three ways: -## Requirements +- **Download a release.** This is the easiest way on Windows and Linux. +- **Install from PyPI.** This works on Windows, macOS and Linux. +- **Run from source.** Use this to change the code or to build the app yourself. -Everyone needs: +GPU acceleration is optional. The last section of this page explains it. -- A supported operating system: Windows, macOS, or Linux -- [Python 3.12 or newer](https://www.python.org/downloads/) +## Download a release + +1. Open the [releases page](https://github.com/JakimPL/SampleToNES/releases). +2. Download the file for your system: Windows or Linux. +3. Extract the file. +4. Start `sampletones` in the extracted folder. On Windows, double-click `sampletones.exe`. -Some setups need a little more — each is covered in the relevant section below: +On Linux, you may need to make the file executable first: `chmod +x sampletones`. -- **On Linux**, a few system packages are required to build or run: the Tk - file-dialog and PortAudio audio libraries. Install them with `make system-deps`. -- **On macOS**, audio playback is compiled against PortAudio on install, so the - library comes from [Homebrew](https://brew.sh): `make system-deps` installs it. - Tk and the graphics libraries arrive with the official Python installer. -- **On Windows**, the official Python installer and the packaged dependencies - cover everything. -- **Running from source** also needs [uv](https://docs.astral.sh/uv/). -- **GPU acceleration** needs an NVIDIA GPU with a current driver. The matching CuPy - build is installed for you, so the driver is all you need — on Linux and Windows alike. +## Install from PyPI -## Standalone build +You need [Python 3.12 or newer](https://www.python.org/downloads/). -A ready-to-run executable built on your machine. You only need Python 3.12. +On Linux and macOS, install the audio and file dialog libraries first: -### Windows +```sh +sudo apt-get install libportaudio2 libasound2 python3-tk # Debian and Ubuntu +brew install portaudio # macOS +``` -1. Install Python 3.12. -2. Double-click `install.bat`. It builds `bin\sampletones.exe`. -3. Double-click `bin\sampletones.exe` to start. +Then install _SampleToNES_ in an environment of its own, and start it: -### Linux +```sh +uv tool install sampletones # or: pipx install sampletones +sampletones +``` -1. Install the audio and file-dialog system packages: `make system-deps` (or run - `python3 scripts/system_dependencies.py`). -2. Install Python 3.12, then run `./install.sh` in a terminal. It builds a - `bin/sampletones` executable. -3. Run `./bin/sampletones` to start. +You can also run `pip install sampletones` inside an active virtual environment. ## Run from source -For development, and the way to run on macOS. Requires [uv](https://docs.astral.sh/uv/) -— and, on Linux and macOS, the system packages from the requirements above: +You need: + +- [Python 3.12 or newer](https://www.python.org/downloads/) +- [uv](https://docs.astral.sh/uv/) +- `make` + +Get the code and set it up: ```sh -make system-deps # Linux and macOS: install the system libraries -make setup # create the environment and install the sampletones command -make run # run the app +git clone https://github.com/JakimPL/SampleToNES.git +cd SampleToNES +make system-deps # Linux and macOS: installs the system libraries +make setup # creates the environment and installs the sampletones command +make run # starts the app ``` -To update the global command after pulling new changes, re-run `make setup`. +After you pull new changes, run `make setup` again. -## GPU acceleration +### Build a standalone app -_SampleToNES_ can use an NVIDIA GPU (via [CuPy](https://cupy.dev/) and CUDA) to -speed up instruction-library generation and reconstruction. `make setup` detects -your NVIDIA driver and installs the matching CuPy build automatically: +On Windows and Linux, you can build a standalone app from the source code: -```sh -make setup # installs GPU support when a supported driver is present -make setup GPU=0 # forces the CPU (NumPy) backend -``` +- **Windows**: double-click `install.bat`. It builds `bin\sampletones.exe`. +- **Linux**: run `make system-deps`, then `./install.sh`. It builds `bin/sampletones`. + +## GPU acceleration + +_SampleToNES_ can use an NVIDIA graphics card to build libraries and convert recordings faster. It +needs an NVIDIA card with a current driver, on Windows or Linux. On macOS, the app uses the CPU. -A current NVIDIA driver is all you need: the CUDA components ship with the CuPy -build, on Linux and Windows alike. Detection reads the driver's CUDA version — -version 12 and newer use the default build, version 11 uses a legacy build. On -macOS, _SampleToNES_ runs on the CPU. +- **From source**: `make setup` checks your NVIDIA driver and installs the matching GPU support. + `make setup GPU=0` installs the app without GPU support. +- **From PyPI**: add the `gpu` extra, `uv tool install "sampletones[gpu]"`. If your driver supports + CUDA 11 only, use the `gpu-cuda11` extra instead. --- -Once it runs, [Getting started](getting-started.md) walks through your first +Once the app runs, [Getting started](getting-started.md) walks you through your first reconstruction and your first song. diff --git a/docs/guide/sequencer.md b/docs/guide/sequencer.md index 728c8f69b..a85d38ca2 100644 --- a/docs/guide/sequencer.md +++ b/docs/guide/sequencer.md @@ -111,7 +111,7 @@ Both grids take a **selection** — a rectangle of cells you copy, cut, paste, a delete in one go. Hold `Shift` and press the arrow keys to reach out from the cursor, or drag the pointer across the cells; `Shift`+click carries the selection to the cell you click. Dragging past the edge of a grid scrolls it along, so a selection -can run further than the screen shows. Any plain move, and `Escape`, puts the +can run further than the screen shows. Any plain move, and `Esc`, puts the selection away again. | Key | Action | @@ -186,10 +186,10 @@ throughout the tab: | `Shift+Space` | Play from the start | | `Ctrl+Space` | Play from the frame currently shown | | `Ctrl+Shift+Space` | Play from the cursor's row in the pattern grid | -| `Escape` | Stop | +| `Esc` | Stop | | `Ctrl+L` | **Loop song** — start the song over each time it reaches the end | -`Escape` silences everything, including a sample preview. The same commands sit on +`Esc` silences everything, including a sample preview. The same commands sit on the **Playback** menu and the transport buttons. ## Following the playhead @@ -246,8 +246,8 @@ after voices exist re-times how they all play back, so it asks **Change NES frequency** first (with a **Don't ask again** option). The project's title, author, and comment — which carry into the exported module — -are set in **Project properties**, from the button or **File ▸ Project -properties...**, along with the meter the song is counted in. +are set in **File ▸ Project properties...**, along with the meter the song is +counted in. **First highlight** and **Second highlight** are that meter: how many rows make a beat, and how many make a bar. The tracker tints the row that opens each one. The @@ -267,8 +267,8 @@ Every change is undoable. The **History** panel on the right shows the stack, wi **Undo** and **Redo** (also on the **Edit** menu); click any entry to jump straight to that point. -When the song is ready, **Export as FamiTracker module** (or **File ▸ Export -FamiTracker module...**) writes the `.ftm`. See +When the song is ready, **File ▸ Export ▸ FamiTracker module...** writes the +`.ftm`. See [FamiTracker export](../formats/famitracker.md) for what the module contains and the limits it respects. From 9def965c807b416fe216176bcaa159463f9a2e2b Mon Sep 17 00:00:00 2001 From: JakimPL Date: Sun, 13 Sep 2026 20:46:03 +0200 Subject: [PATCH 28/36] Rewrote: the user guide in plain language --- docs/guide/command-line.md | 6 +- docs/guide/configuration.md | 66 +++-- docs/guide/converting.md | 20 +- docs/guide/files.md | 68 +++-- docs/guide/getting-started.md | 9 +- docs/guide/installation.md | 2 +- docs/guide/interface.md | 4 +- docs/guide/reconstruction.md | 14 +- docs/guide/sequencer.md | 450 ++++++++++++++++------------------ docs/index.md | 24 +- 10 files changed, 311 insertions(+), 352 deletions(-) diff --git a/docs/guide/command-line.md b/docs/guide/command-line.md index 0d5df76ce..f2022c903 100644 --- a/docs/guide/command-line.md +++ b/docs/guide/command-line.md @@ -1,7 +1,7 @@ # Command line -You can run _SampleToNES_ from a terminal. Use it to convert recordings without opening the -window, to build an instruction library, or to open a file in the app. +You can run _SampleToNES_ from a terminal. Use it to convert recordings in the terminal, to build +an instruction library, or to open a file in the app. ## Getting the command @@ -37,7 +37,7 @@ describes them. - `--config ` or `-c ` uses a configuration file. It works with `run`, `open`, `convert` and `library`. Without it, the app uses your saved configuration, or the built-in - defaults if you have not saved one. + defaults until you save one. - `--output ` or `-o ` sets where `convert` saves the reconstruction. Without it, the reconstruction goes to the reconstructions folder of your configuration. - `--channels ` sets the channels `convert` may use, for example `--channels pulse1,pulse2`. diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index 4f5cf0edc..2d2fc1724 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -1,51 +1,41 @@ # Configuration -_SampleToNES_ reconstructs audio according to a **generation configuration**: the -sample rate, the NES frequency, how the audio is analyzed, and how candidates are -scored. The settings you change most often are on the **Main** tab. The rest are -in the configuration file, for when you want to go deeper. - -Channels are not part of the configuration. The NES has four sound channels — -**Pulse 1**, **Pulse 2**, **Triangle**, and **Noise** — and you choose which of -them each recording uses in the converter's list, every time you set up a +_SampleToNES_ converts audio with a **generation configuration**. The configuration sets the sample +rate, the NES frequency, how the audio is analyzed and how sounds are compared. The settings you +change most often are on the **Main** tab. The configuration file has the rest. + +You choose the channels each recording uses in the converter's list, each time you set up a conversion. See [choosing which channels a recording uses](converting.md#choosing-which-channels-a-recording-uses). -## From the interface +## Settings on the Main tab -Three cards on the **Main** tab hold the everyday settings: +Three cards on the **Main** tab have the everyday settings: -- **General settings** — **Normalize audio** and **Quantize audio**, and the - **Sample rate** and **NES frequency** that a library is built for. -- **Source settings** — **Drive**, which sets how hard the channels are pushed, - and the channels and bends of the recording you selected in the converter's - list. -- **Advanced settings** — **Method** and **Feature scaling**, which set how the - audio's frequency content is measured and weighted (see [Reconstruction - algorithms](../concepts/reconstruction.md)); the **Workers** count; and the - instruction library and output folders. Choose **View ▸ Show advanced - settings** to display this card. +- **General settings**: **Normalize audio**, **Quantize audio**, and the **Sample rate** and **NES + frequency** a library is built for. +- **Source settings**: **Drive**, which sets how hard the channels are pushed, and the channels and + bends of the recording you selected in the converter's list. +- **Advanced settings**: **Method** and **Feature scaling**, which set how the app measures the + frequencies in the audio, the number of **Workers**, and the library and output folders. Choose + **View ▸ Show advanced settings** to show this card. [Reconstruction + algorithms](../concepts/reconstruction.md) explains **Method** and **Feature scaling**. -Changing any of these updates your configuration, which is saved to `config.json` -(see [Where your files live](files.md)). +The app saves your changes to `config.json`. See [Where your files live](files.md). -## In the configuration file +## The configuration file -The configuration holds more than the interface shows. You can edit the finer -controls directly in `config.json`: the -[selector](../concepts/reconstruction.md) (greedy or Viterbi), the phase aligner, -the scoring weights and distance metric, the number of candidates kept per frame, -and so on. The [configuration file -reference](../formats/configuration.md) lists every section and key, and -[Reconstruction algorithms](../concepts/reconstruction.md) explains what they do -and lists the defaults. +`config.json` has more settings than the **Main** tab shows. For example, you can change: -You can also load and save whole configurations from the **Reconstruction** menu -(**Load generation settings...** and **Save generation settings...**), or point -the app at one on the [command line](command-line.md) with `--config`. +- the [selector](../concepts/reconstruction.md), greedy or Viterbi +- the phase aligner +- the scoring weights and the distance measure +- the number of candidates kept for each frame -## Deployment settings +The [configuration file reference](../formats/configuration.md) lists every setting. +[Reconstruction algorithms](../concepts/reconstruction.md) explains what each one does, and lists +the defaults. -Two settings — the log level and strict history checking — are decided when the -application is packaged, so they are not part of your configuration. They exist -for development and support. +To load or save a whole configuration, use **Reconstruction ▸ Load generation settings...** and +**Reconstruction ▸ Save generation settings...**. On the [command line](command-line.md), add +`--config` to a command. diff --git a/docs/guide/converting.md b/docs/guide/converting.md index 162a24084..68a6e85bd 100644 --- a/docs/guide/converting.md +++ b/docs/guide/converting.md @@ -15,17 +15,17 @@ The **Converter** card lists the recordings a conversion uses. Add them from the Turn on **Playback ▸ Autoplay** (`Ctrl+P`) to play a recording with a single click. This lets you listen through a folder before adding anything from it. With Autoplay off, right-click a recording and choose **Play**. -Adding a folder opens a small window while the folder is read. **Stop** ends the search and leaves the list as it was. A folder with no recordings inside it says so and adds nothing. +Adding a folder opens a small window while the app reads the folder. **Stop** ends the search and keeps the list as it was. If the folder has no recordings, the window says so and the list stays the same. **x** removes a row from the list. Removing a folder removes every recording in it. -Click a row to select it. The **Source settings** card then shows that recording. Right-clicking a row selects it as well. Press `Del` to remove the selected row. Closing a folder that holds the selected recording clears the selection. +Click a row to select it. The **Source settings** card then shows that recording. Right-clicking a row selects it as well. Press `Del` to remove the selected row. Closing a folder that contains the selected recording clears the selection. ## Choosing which channels a recording uses The NES has four sound channels: **Pulse 1**, **Pulse 2**, **Triangle**, and **Noise**. Every recording in the list has a checkbox for each channel. Check the channels that the recording may use. Press `1` to `4` to switch a channel on or off for the selected row. -A folder represents all the recordings inside it. Its checkbox shows their channel assignments: +A folder's checkboxes show the channels of all the recordings inside it: - checked if all recordings use the channel, - filled with the channel's color if only some recordings use it, @@ -41,23 +41,23 @@ Below **Drive**, the card shows the name of the recording you selected in the li ## One reconstruction each, or one mix from all -**Output**, at the top of the **Converter** card, decides what the conversion produces: +**Output**, at the top of the **Converter** card, sets what the conversion makes: - **One per recording** — each recording in the list becomes its own reconstruction. Recordings added with a folder are saved in a matching folder structure. - **One from all** — all recordings are mixed into a single reconstruction. Folders are replaced by the recordings inside them. A mix can hold up to eight recordings. If you switch to **One from all** with more than eight recordings in the list, a dialog asks which ones to mix. The same dialog opens when you add a folder with more recordings than the mix has room for. -The dialog lists the same rows as the card and shows how many recordings you have selected. Double-click a row to hear the recording. **Add** becomes available when your selection fits. A full mix cannot accept more recordings, so uncheck one before checking another. +The dialog lists the same rows as the card and shows how many recordings you selected. Double-click a row to hear the recording. **Add** becomes available when your selection fits. When the mix is full, uncheck a recording before you check another one. -Once a mix has two or more recordings, the rows are grouped into **levels**. A level decides which recordings choose their channels first. Everything on level 1 is given channels before anything on level 2. This lets a lead melody take the channels it needs before a background part does. +When a mix has two or more recordings, the rows are grouped into **levels**. Levels set which recordings get their channels first. Recordings on level 1 get channels before recordings on level 2. This lets a lead melody take the channels it needs before a background part does. Drag a row onto another row to put them on the same level. Drag it into the gap between levels to give it a level of its own. You can also right-click a row to use the same commands. -**Order** decides how the levels take turns: +**Order** sets how the levels take turns: - **Round robin** — every level gets a turn in each round. -- **Strict** — one level is filled before the next one chooses. +- **Strict** — one level gets all its channels before the next level chooses. ## Running a conversion @@ -67,13 +67,13 @@ Click the button under **Output** to start the conversion. The button's label te When the conversion finishes, click **Load** to open the result on the **Reconstruction** tab, where you can [listen to it and export it](reconstruction.md). After a conversion of several recordings, the button reads **Open** instead. -The first conversion with a given set of settings builds the [instruction library](../concepts/instruction-library.md) it needs. This takes a while. Later conversions with the same settings reuse the library. +The first conversion with new settings builds the [instruction library](../concepts/instruction-library.md) for those settings. This takes a while. Later conversions with the same settings use the same library. ## The instruction library The **Instructions** tab (`F4`) builds and browses the [instruction library](../concepts/instruction-library.md), the catalog of NES tones that a conversion searches. -A conversion builds the library it needs on its own, so you rarely need to go there. The tab is useful for building a library before a long session and for exploring what your settings can produce. +A conversion builds the library it needs by itself, so you rarely need this tab. Use it to build a library before a long session, or to explore the sounds your settings can make. Select an instruction to see its **Waveform** and **Spectrum**. This lets you see and hear a single NES tone on its own. diff --git a/docs/guide/files.md b/docs/guide/files.md index 7c15b2c5b..ec0483e27 100644 --- a/docs/guide/files.md +++ b/docs/guide/files.md @@ -1,58 +1,54 @@ # Where your files live -_SampleToNES_ keeps your work in a **SampleToNES** folder inside your documents -folder: +_SampleToNES_ saves your work in a **SampleToNES** folder inside your documents folder: - Windows: `C:\Users\\Documents\SampleToNES` - macOS: `/Users//Documents/SampleToNES` - Linux: `/home//Documents/SampleToNES` -Inside it: +The folder contains: -- `instructions/` — the generated [instruction libraries](../formats/instruction-libraries.md) (`.ins`) -- `reconstructions/` — saved [reconstructions](../formats/reconstructions.md) (`.stn`) -- `projects/` — saved [projects](../formats/projects.md) (`.stp`) -- `config.json` — your generation [configuration](configuration.md) +- `instructions/`: your [instruction libraries](../formats/instruction-libraries.md) (`.ins`) +- `reconstructions/`: your [reconstructions](../formats/reconstructions.md) (`.stn`) +- `projects/`: your [projects](../formats/projects.md) (`.stp`) +- `config.json`: your [configuration](configuration.md) -You can point the library and output folders elsewhere from the **Main** tab's -**Advanced settings**, or from the right-click menu in its **Filesystem** browser. +To use other folders for libraries and reconstructions, open **Advanced settings** on the **Main** +tab, or right-click in the **Filesystem** browser. ## File types -| Type | What it is | Where it lives | +| Type | What it is | Where it is saved | | --- | --- | --- | | `.ins` | [instruction library](../formats/instruction-libraries.md) | `instructions/` | | `.stn` | [reconstruction](../formats/reconstructions.md) | `reconstructions/` | | `.stp` | [project](../formats/projects.md) | `projects/` | -| `.fti` | FamiTracker instrument (exported) | wherever you choose | -| `.ftm` | FamiTracker module (exported) | wherever you choose | -| `.json` | Bitphase instrument preset (exported) | wherever you choose | -| `.btp` | Bitphase project (exported) | wherever you choose | -| `.nsf` | NES sound file (exported) | wherever you choose | +| `.fti` | FamiTracker instrument | where you choose | +| `.ftm` | FamiTracker module | where you choose | +| `.json` | Bitphase instrument preset | where you choose | +| `.btp` | Bitphase project | where you choose | +| `.nsf` | NES sound file | where you choose | -The `.fti` and `.ftm` files are what you load into -[FamiTracker](../formats/famitracker.md), `.json` and `.btp` are what -[Bitphase](../formats/bitphase.md) reads, and an `.nsf` plays on its own in a NES -sound player or on the console; the rest are _SampleToNES_'s own formats. +The last five types are exports: -## Exported files +- Open `.fti` and `.ftm` files in [FamiTracker](../formats/famitracker.md). +- Open `.json` and `.btp` files in [Bitphase](../formats/bitphase.md). +- Play `.nsf` files in an NSF player or on a NES. -The extension names what an export is written for: `.fti` and `.ftm` go to -FamiTracker, `.json` and `.btp` to Bitphase, and `.nsf` to a NES sound player. The -save dialog offers the file types that fit what you are exporting and fills in the -extension of the type it is set to. Exporting one channel offers all three, so -switching the type there switches the target; typing an extension yourself picks it -directly. +## Naming exported files -What you name in the dialog also names what a tracker lists: +The save dialog lists the file types that fit your export, and adds the extension of the type you +pick. When you export one channel, the dialog lists all three instrument types, so you can pick the +program there. You can also type the extension yourself. -| Export | You name | What is written | +The name you type also names the instrument in the tracker: + +| Export | The name you type | What the app saves | | --- | --- | --- | -| **Instruments** panel ▸ **Export instrument...** | the file | that file, its instrument carrying the name you gave | -| **Reconstruction ▸ Export instruments** | the batch | one file per channel beside that name, each named ` (channel)`, or a single file at that name for `.nsf` | -| **File ▸ Export** | the file | that file, holding the whole song | - -So exporting a `Kick` reconstruction to FamiTracker instruments writes -`Kick (pulse1).fti`, `Kick (triangle).fti`, and one file for every other channel the -reconstruction uses, all in the folder you chose. An `.nsf` gathers every channel into -one program, so it is the single file you named. +| **Instruments** panel ▸ **Export instrument...** | the file | one file, and the instrument inside it has the same name | +| **Reconstruction ▸ Export instruments** | the set | one file per channel, each named ` (channel)`. For `.nsf`, one file with the name you typed | +| **File ▸ Export** | the file | one file with the whole song | + +For example, exporting a reconstruction named `Kick` to FamiTracker instruments saves +`Kick (pulse1).fti`, `Kick (triangle).fti`, and one file for each other channel the reconstruction +uses. An `.nsf` export saves every channel in one file. diff --git a/docs/guide/getting-started.md b/docs/guide/getting-started.md index 04a9530bf..f3dcd0a7d 100644 --- a/docs/guide/getting-started.md +++ b/docs/guide/getting-started.md @@ -1,8 +1,7 @@ # Getting started -Two quick paths through _SampleToNES_: turning a sound into FamiTracker -instruments, and building a whole song. Both assume it is already -[installed](installation.md). +This page shows two quick ways to start: turning a sound into FamiTracker instruments, and building +a song. First, [install](installation.md) _SampleToNES_. ## Reconstruct a sound into FamiTracker instruments @@ -10,8 +9,8 @@ instruments, and building a whole song. Both assume it is already 2. In the **Filesystem** browser, double-click an audio file (WAV, MP3, FLAC, OGG, AIFF, or AU) — or Ctrl-click a folder, to reconstruct every audio file inside it. -3. Optionally click the recording in the list and choose which channels it uses - under **Source settings**, and adjust **General settings**. Each recording +3. If you want, click the recording in the list and check the channels it may use + under **Source settings**, and change **General settings**. Each recording needs at least one channel. 4. Click the button under **Output** to start the conversion. The first time you convert with a given set of settings, the [instruction diff --git a/docs/guide/installation.md b/docs/guide/installation.md index d21538a81..74cefbf82 100644 --- a/docs/guide/installation.md +++ b/docs/guide/installation.md @@ -70,7 +70,7 @@ _SampleToNES_ can use an NVIDIA graphics card to build libraries and convert rec needs an NVIDIA card with a current driver, on Windows or Linux. On macOS, the app uses the CPU. - **From source**: `make setup` checks your NVIDIA driver and installs the matching GPU support. - `make setup GPU=0` installs the app without GPU support. + `make setup GPU=0` installs the app for the CPU alone. - **From PyPI**: add the `gpu` extra, `uv tool install "sampletones[gpu]"`. If your driver supports CUDA 11 only, use the `gpu-cuda11` extra instead. diff --git a/docs/guide/interface.md b/docs/guide/interface.md index 1458e8dfd..f81e8af86 100644 --- a/docs/guide/interface.md +++ b/docs/guide/interface.md @@ -9,7 +9,7 @@ _SampleToNES_ has four tabs. Use `F1` to `F4` to switch between them: You usually work through the tabs in this order. Convert your files on **Main**. When the conversion finishes, click **Load** to open the result on **Reconstruction**. From there, **Add to Sequencer** adds the reconstruction to a song. -The **Instructions** tab is optional. It lets you explore individual _instructions_ — the unit blocks produced by the NES sound processor. +The **Instructions** tab is optional. Use it to explore single _instructions_: the smallest sounds the NES sound chip makes. ## The menus @@ -26,7 +26,7 @@ Each menu covers one kind of work: Two items are easy to miss: - **View ▸ Show advanced settings** shows the **Advanced settings** card on the **Main** tab. It contains the generation method, feature scaling, worker count, and library and output folders. [Configuration](configuration.md) explains each one. -- **Playback ▸ Audio settings...** chooses the device, sample rate, and buffer size you listen through. These are separate from **Sample rate** and **NES frequency** on the **Main** tab. Those settings decide how the audio is reconstructed. +- **Playback ▸ Audio settings...** chooses the device, sample rate, and buffer size you listen through. **Sample rate** and **NES frequency** on the **Main** tab are different settings: they set how the audio is converted. Project properties belong to a project and are covered in the [sequencer guide](sequencer.md). diff --git a/docs/guide/reconstruction.md b/docs/guide/reconstruction.md index 1b597ea1c..1ded88432 100644 --- a/docs/guide/reconstruction.md +++ b/docs/guide/reconstruction.md @@ -18,17 +18,17 @@ If the reconstruction you have open has unsaved changes, the app asks whether to To keep frequently used reconstructions within reach, right-click a reconstruction or a folder and choose **Mark as favorite**. Check **Favorites only** to show only those items. -The **Source** card switches playback between **Reconstruction** and **Original audio**, so you can compare the two. The **Waveform** card shows each channel and has a checkbox for each one. Keys `1` to `4` toggle the same checkboxes. +The **Source** card switches playback between **Reconstruction** and **Original audio**, so you can compare the two. The **Waveform** card has a checkbox for each channel. Keys `1` to `4` switch the same checkboxes. ## Hearing what each recording contributed The **Stems** card lists the recordings used to build a reconstruction. It groups them by the level each one was given. Every row has a checkbox for each channel that the recording used, and the checkbox at the front toggles all of them. -Uncheck a channel to silence it in the waveform, playback, original audio, and WAV export. This lets you hear what one recording contributed, channel by channel. These checkboxes only change what you hear. They do not change anything that is saved. +Uncheck a channel to silence it in the waveform, in playback, in the original audio and in a WAV export. This lets you hear what one recording added, channel by channel. The saved reconstruction keeps every channel, whatever you uncheck. **Collapse levels** shows the whole list as one table. -**x** at the end of a row removes that recording from the reconstruction after asking you to confirm. The recording becomes silent and its row disappears, and the change is written when you save the reconstruction. A reconstruction keeps at least one recording, so the last row's **x** is disabled. +**x** at the end of a row removes that recording from the reconstruction. The app asks you to confirm first. The recording goes silent, its row disappears, and the change is saved when you save the reconstruction. A reconstruction needs at least one recording, so the **x** of the last row is disabled. ## Editing instruments @@ -42,11 +42,11 @@ Each channel has its own set of sequences: Type `|` before a value to mark where the sequence repeats while a note is held. `15 14 | 12 10` plays the attack once and then loops the last two values. -Clear a sequence to leave that setting to the channel. For example, an instrument with an empty volume sequence plays at whatever volume the channel is set to. Each channel shows how many bytes its instrument takes on the NES, so you can see what an edit costs. +Clear a sequence to use the channel's own setting. For example, an instrument with an empty volume sequence plays at the volume the channel is set to. Each channel shows how many bytes its instrument takes on the NES, so you can see how much space an edit uses. -You can also edit a hand-written **instrument** here — a voice with no recording behind it, described in the [sequencer guide](sequencer.md#voices-samples-and-instruments). Right-click it in the **Voices** list and choose **Edit**. +You can also edit an **instrument** here, a voice you write by hand. The [sequencer guide](sequencer.md#voices-samples-and-instruments) describes instruments. Right-click one in the **Voices** list and choose **Edit**. -Since an instrument has no recording, the panel shows **Audition** instead of the pitch steppers. Choose **Pulse**, **Triangle**, or **Noise**, and the note keys then play the instrument on that sound generator. +For an instrument, the panel shows **Audition** in place of the pitch steppers. Choose **Pulse**, **Triangle** or **Noise**, and the note keys play the instrument on that sound generator. ## Exporting @@ -57,6 +57,6 @@ You export a reconstruction from the **Reconstruction** menu: - **Export instruments ▸ NSF program...** writes a single `.nsf` file that plays the whole reconstruction on a NES. - **Export to WAV...** renders the audio using the channel and stem checkboxes you have set. -**Export instrument...** in the **Instruments** panel writes only the channel you are looking at, in whichever format you choose in the save dialog. See [where your files live](files.md#exported-files). +**Export instrument...** in the **Instruments** panel writes only the channel you are looking at, in whichever format you choose in the save dialog. See [where your files live](files.md#naming-exported-files). To use a reconstruction in a song, right-click it and choose **Add to Sequencer**. diff --git a/docs/guide/sequencer.md b/docs/guide/sequencer.md index a85d38ca2..fc6984124 100644 --- a/docs/guide/sequencer.md +++ b/docs/guide/sequencer.md @@ -1,307 +1,281 @@ # The sequencer -The **Sequencer** tab is a tracker: it arranges voices into a song across the four -NES channels and exports it as a FamiTracker -[module](../formats/famitracker.md) (`.ftm`). It works on a -[project](../formats/projects.md), so start one with **File ▸ New project** (or -open an existing `.stp`). The pattern grid and order sit in the center, a browser -for pulling in reconstructions on the left, and the module settings, voice list, -and undo history on the right. +The **Sequencer** tab (`F3`) is a tracker. You use it to arrange voices into a song on the four NES +channels. You can export the song as a FamiTracker [module](../formats/famitracker.md) (`.ftm`), +play it back, or render it to audio. + +The sequencer works on a [project](../formats/projects.md). Choose **File ▸ New project** to start +one, or open an existing `.stp` file. ## Voices: samples and instruments -A song is built from **voices**, and there are two kinds. A **sample** is a -reconstruction brought in as something a row can play. An **instrument** is written by -hand — envelopes with no recording behind them — for the melodies and basses you write -yourself. Both sit in the **Voices** list on the right, numbered together, and a -mark at the front of each row says which kind it is. The mark carries a color as well -as a shape — amber for a sample, magenta for an instrument — and the same two colors -name a voice in the pattern grid and in the history, so the two kinds read apart -wherever one is named. - -Four ways bring a voice in, and the **Voice** menu holds all four: - -| Way in | What arrives | -|--------|--------------| -| **New instrument** | An instrument holding a note at full volume, ready to place and hear | -| **Add sample from file...** | A reconstruction saved anywhere on disk, as a sample | +A song plays **voices**. There are two kinds: + +- A **sample** is a reconstruction that the song plays. +- An **instrument** is a sound you write by hand. Use instruments for melodies and bass lines. + +Both kinds are in the **Voices** list, numbered together. + +The **Voice** menu adds a voice in four ways: + +| Menu item | What it adds | +|-----------|--------------| +| **New instrument** | An instrument that plays a note at full volume, ready to place and hear | +| **Add sample from file...** | A reconstruction file from anywhere on your disk, as a sample | | **Import instrument...** | A FamiTracker instrument file (`.fti`), as an instrument | -| **Add to Sequencer** | The reconstruction the **Reconstruction** tab holds, as a sample | - -The first three also sit at the top of the **Voices** list, and on the list's own -menu — right-click below the rows to reach it. **Add to Sequencer** is on the -**Browser** on the Sequencer tab (right-click a reconstruction) and on the -**Reconstruction** tab. If a reconstruction was made at a different NES frequency -than the project and the project already has voices, _SampleToNES_ warns with -**Different NES frequency**; **Add anyway** adds it regardless. - -A new instrument starts out holding a note at full volume, so you can place it and -hear it straight away; give it the sound you want on the **Reconstruction** tab -(right-click ▸ **Edit**). See [editing instruments](reconstruction.md#editing-instruments). -An imported `.fti` arrives with the volume, arpeggio, and duty-cycle envelopes the -file states, and **Instrument imported** names anything the file held on a tracker's -own terms that the voice leaves behind — see [reading an instrument +| **Add to Sequencer** | The reconstruction open on the **Reconstruction** tab, as a sample | + +The first three items are also at the top of the **Voices** list. Right-click below the rows of +the list to open the same menu. + +**Add to Sequencer** is also on the **Reconstruction** tab. On the Sequencer tab, right-click a +reconstruction in the **Browser** to find it. + +If a reconstruction uses a different NES frequency than the project, the app shows **Different +NES frequency**. Click **Add anyway** to add it. + +To change how a new instrument sounds, right-click it and choose **Edit**. The instrument opens on +the **Reconstruction** tab. See [editing instruments](reconstruction.md#editing-instruments). + +An imported `.fti` file keeps its volume, arpeggio and duty cycle sequences. **Instrument +imported** lists any other settings of the file, which the instrument leaves out. See [reading an instrument file](../formats/famitracker.md#c-reading-an-instrument-file). -**New instrument from ▸ _channel_** on a sample's menu writes what one of its channels -plays into an instrument of its own, so a recorded part becomes envelopes you edit by -hand. +A sample's right-click menu has **New instrument from**. Choose a channel to copy what the sample +plays on that channel into a new instrument you can edit. -Right-click any voice to **Edit**, **Rename**, **Duplicate**, **Remove**, or reorder it. -**Export instrument...** writes the voice out as a `.fti` another tracker reads — a -sample holds one instrument per channel it plays, so it asks which. The **Edit** menu -carries the same actions for the voice you have picked. The right-click menu also names -how much room the voice takes on the NES — a sample's total and then each channel it -plays, and an instrument's single figure. The figures are in bytes, and they count what a -FamiTracker export saves. Removing a voice that patterns still use asks first, because it -clears every row that references it. +Right-click any voice to use these commands: -Hovering a row says what that voice is in one line: its name, whether it is a sample or -an instrument, the channels a sample plays, and the room it takes. +- **Edit**, **Rename**, **Duplicate** and **Remove**. +- Commands that move the voice up or down the list. +- **Export instrument...**, which saves the voice as an `.fti` file. A sample has one instrument + per channel, so the app asks which channel to export. + +The **Edit** menu has the same commands for the voice you selected. + +The right-click menu also shows how many bytes the voice takes on the NES. For a sample, it shows +the total and the size of each channel. + +If patterns still use a voice, removing the voice asks you first. Removing it clears every row +that uses it. + +Hover over a voice to see its name, its kind, its channels and its size. ## Writing a pattern -The **Tracker** grid is the pattern editor. Each row is one step in time; the -columns are the **Sample** and the four channels — **Pulse 1**, **Pulse 2**, -**Triangle**, **Noise** — each carrying a voice, a pitch, and a volume. Click a cell -and type its value. Right-clicking a cell opens the rest of the operations — **Set -voice**, **Note off**, **Clear cell** and **Clear row**, transpose and volume -adjustments, **Play from here** to audition from the cursor row, and **Play from -this frame** to start at the top of the shown frame. - -The **Sample** column places a sample across every channel its reconstruction -covers, and clears the rest of the row. It takes samples alone: an instrument sounds -on the one channel that names it, so **Set voice** lists instruments there grayed out, -and a number typed over one leaves the cell reading what it held. Name an instrument -in the channel column you want it on. - -A cell holding a voice wears that voice's color — amber for a sample, magenta for an -instrument — while an empty cell, a cut, and a `?` where the **Sample** column's -channels disagree read in a plain gray. Silencing a channel dims its cells and keeps -those colors, so a muted column stays as readable as the rest. - -## Reading and typing a pitch - -A pitch cell holds one number, and it reads in the terms of the voice the channel is -carrying. A sample was converted at a pitch of its own, so its cells read as steps -from it — `+00` plays it as recorded, `+0C` an octave up. An instrument sounds at -whatever note a row names it with, so its cells read as the notes they sound — `C-4`, -`A#3`. -A row that only bends a note reads the same way as the row that started it. - -Type a note into an instrument's cell piano-style: the bottom two rows of the keyboard are -one octave (`Z` `S` `X` `D` `C` …) and the two above them the next (`Q` `2` `W` `3` -`E` …). **Octave** above the grid says where the bottom row opens. The keys work on -a sample's cell too, writing the step that reaches the note you pressed. The noise -channel selects one of sixteen periods rather than a note, so its cells are typed as -a signed value. +The **Tracker** grid is the pattern editor. Each row is one step in time. The grid has a +**Sample** column and a column for each channel: **Pulse 1**, **Pulse 2**, **Triangle** and +**Noise**. Each channel column has a voice, a pitch and a volume. + +Click a cell and type its value. + +Right-click a cell for more commands: + +- **Set voice** chooses the voice for the cell. +- **Note off** stops the note. +- **Clear cell** and **Clear row** empty the cell or the whole row. +- Transpose and volume commands change the pitch and volume. +- **Play from here** plays from the row you clicked. +- **Play from this frame** plays from the top of the frame on screen. + +The **Sample** column places a sample on every channel the sample uses, and clears the other +channels of the row. The **Sample** column takes samples only. To place an instrument, use the +column of the channel you want it on. + +A `?` in the **Sample** column means the channels of that row play different voices. + +## Typing a pitch + +A pitch cell shows a number. What the number means depends on the voice: + +- For a sample, the number is a step from the pitch the sample was recorded at. `+00` plays the + sample as recorded. `+0C` plays it one octave higher. +- For an instrument, the number is a note, such as `C-4` or `A#3`. + +Type notes on your keyboard like a piano: + +- The bottom two rows of keys play one octave: `Z` `S` `X` `D` `C` and so on. +- The two rows above them play the next octave: `Q` `2` `W` `3` `E` and so on. + +**Octave**, above the grid, sets the octave of the bottom row. The same keys work in a sample's +cell and type the step that plays the note you pressed. + +The noise channel has sixteen sounds in place of notes. Type a noise cell as a signed step, such as +`+03` or `-02`. ## Arranging the song -A song plays a sequence of patterns, and the **Order** grid sets that sequence — -one column per position, with a row for the master and each channel. Type an entry -to place a pattern, or right-click a frame for the rest: **Duplicate** repeats the -frame with the patterns it already plays, **Clone** gives the copy patterns of its -own so you can change it on its own, and **Insert frame**, **Clear frame**, -**Remove**, the moves, and **Play from this frame** do what they say. +A song plays patterns in a sequence. The **Order** grid sets that sequence. Each column is one +position in the song. The grid has a row for the **Master** and a row for each channel. + +Type a pattern number in the **Order** grid to place a pattern. Right-click a frame for more +commands: + +- **Duplicate** repeats the frame with the same patterns. +- **Clone** repeats the frame with new copies of its patterns, so you can change them separately. +- **Insert frame**, **Clear frame** and **Remove** add, empty and delete frames. +- **Play from this frame** plays from that frame. + +## Selecting, copying and pasting + +You can select a block of cells in both grids, then copy, cut, paste or delete it. -## Working on a block +To select cells: -Both grids take a **selection** — a rectangle of cells you copy, cut, paste, and -delete in one go. Hold `Shift` and press the arrow keys to reach out from the -cursor, or drag the pointer across the cells; `Shift`+click carries the selection to -the cell you click. Dragging past the edge of a grid scrolls it along, so a selection -can run further than the screen shows. Any plain move, and `Esc`, puts the -selection away again. +- Hold `Shift` and press the arrow keys. +- Drag across the cells with the mouse. +- `Shift`+click a cell to extend the selection to it. + +Dragging past the edge of a grid scrolls the grid. Any plain arrow key, or `Esc`, clears the +selection. | Key | Action | |-----|--------| -| `Shift`+arrows | Reach the selection out a cell at a time | -| `Shift+Home` / `Shift+End` | Reach it to the first or the last row (tracker) or position (order) | +| `Shift`+arrows | Extend the selection by one cell | +| `Shift+Home` / `Shift+End` | Extend the selection to the first or last row (tracker) or position (order) | | `Ctrl+A` | Select the whole frame, or the whole order | -| `Ctrl+Shift+A` | Select the column you are in (tracker), or your channel's row (order) | -| `Ctrl+Alt+A` | Select the subcolumn you are in (tracker) | +| `Ctrl+Shift+A` | Select your column (tracker), or your channel's row (order) | +| `Ctrl+Alt+A` | Select your subcolumn (tracker) | | `Ctrl+C` | Copy | -| `Ctrl+X` | Cut — copy, then empty what was selected | -| `Ctrl+V` | Paste, starting at the cursor | +| `Ctrl+X` | Cut | +| `Ctrl+V` | Paste at the cursor | | `Del` | Empty the selection | -Copy, cut, paste and delete act on the cell the cursor stands on when nothing is -selected, so copying one cell needs no selection first. All four sit on each grid's -right-click menu: raised inside a selection they act on the whole of it, raised -anywhere else on the cell you clicked. Each grid keeps its own copy, so a tracker -block pastes into the tracker and an order block into the order. - -The **Select** keys work from the cell you are on and reach the whole length of the -grid. They sit on the right-click menu too. +With nothing selected, copy, cut, paste and delete work on the cell under the cursor. The same +commands are on each grid's right-click menu. Right-click inside a selection to use them on the +whole selection. -A paste is anchored: the block starts at the cell you paste onto and lands the rest -down and to the right of it. +Each grid has its own copy. A block copied in the tracker pastes into the tracker, and a block +copied in the order pastes into the order. -In the **Tracker**, a block keeps the kinds of the cells it came from — a transpose -lands in a transpose, a volume in a volume, whichever column you paste onto — and -whatever reaches past the last row or the last column is left out. A cell reading -`?`, where the **Sample** column's channels disagree, passes over its target and -leaves what was there; an empty cell empties it. +A paste starts at the cursor and fills down and to the right: -In the **Order**, a block pasted past the last frame grows the song to hold it, and -one reaching past the **Noise** row stops there. The **Master** row copies the index -its channels share and reads `?` when they differ, which pasted leaves each channel -as it was. +- In the **Tracker**, each cell keeps its kind. A volume pastes into a volume column, wherever you + paste. Cells past the last row or column are dropped. A `?` cell leaves the target cell as it + was. +- In the **Order**, a paste past the last frame adds frames to the song. A paste stops at the + **Noise** row. -Emptying cells keeps the rows and frames they sit in, and every block action is one -step in the history, so a single **Undo** takes it all back. +Emptying cells keeps the rows and frames. Every copy, cut, paste and delete is one step in the +history, so one **Undo** reverses it. -A copy also goes to your desktop's clipboard as plain text, so a block carries between -two open windows of _SampleToNES_ — copy in one, paste in the other — and you can paste -one into a message to show someone what you wrote. Anything else on the clipboard -leaves you with the last block you copied here. Notes travel by their number in the -**Voices** list, so a block pasted into another project plays whichever voice holds -that number there. +A copy also goes to your clipboard as text. You can paste a block into another open window of +_SampleToNES_, or into a message. Voices are copied by their number in the **Voices** list, so in +another project the block plays the voices with those numbers. -## Transposing and shading +## Transpose and volume -In the **Tracker**, transpose and volume move whatever the selection covers, so a -run of rows nudges together. +In the **Tracker**, transpose and volume keys change every cell in the selection. | Key | Action | |-----|--------| -| `Ctrl+Up` / `Ctrl+Down` | Transpose a semitone | -| `Ctrl+Shift+Up` / `Ctrl+Shift+Down` | Transpose an octave | -| `Alt+Up` / `Alt+Down` | Volume a step | -| `Alt+Shift+Up` / `Alt+Shift+Down` | Volume four steps | +| `Ctrl+Up` / `Ctrl+Down` | Transpose by a semitone | +| `Ctrl+Shift+Up` / `Ctrl+Shift+Down` | Transpose by an octave | +| `Alt+Up` / `Alt+Down` | Change the volume by one step | +| `Alt+Shift+Up` / `Alt+Shift+Down` | Change the volume by four steps | -Control carries pitch, Alt carries volume, and Shift makes the step the bigger one. -With nothing selected they act on the cell the cursor stands on, and the same -commands sit on the right-click menu with these keys beside them. +With nothing selected, the keys change the cell under the cursor. The same commands are on the +right-click menu. ## Playing the song -The transport below the grid plays the song, and the keyboard drives playback -throughout the tab: +The transport under the grid plays the song. These keys work anywhere on the tab: | Key | Action | |-----|--------| -| `Space` | Play, or pause and resume what is playing | +| `Space` | Play, or pause and resume | | `Shift+Space` | Play from the start | -| `Ctrl+Space` | Play from the frame currently shown | -| `Ctrl+Shift+Space` | Play from the cursor's row in the pattern grid | +| `Ctrl+Space` | Play from the frame on screen | +| `Ctrl+Shift+Space` | Play from the cursor's row | | `Esc` | Stop | -| `Ctrl+L` | **Loop song** — start the song over each time it reaches the end | +| `Ctrl+L` | **Loop song**: start the song again when it ends | -`Esc` silences everything, including a sample preview. The same commands sit on -the **Playback** menu and the transport buttons. +`Esc` also stops a sample preview. The same commands are on the **Playback** menu. -## Following the playhead +## Following the playback -**Playback ▸ Follow playback** chooses how far the view travels with the sounding -row. Each mode carries a key of its own, so you can change your mind while the song -plays, and the choice is remembered for the next time you launch: +**Playback ▸ Follow playback** chooses whether the view moves with the song. The app remembers your +choice. -| Mode | Key | Where the view goes | +| Mode | Key | What the view does | |------|-----|---------------------| -| **Follow rows** | `Ctrl+F` | Scrolls the pattern grid to keep the sounding row on screen, and shows the frame being played | -| **Follow patterns** | `Ctrl+Shift+F` | Shows the frame being played, and leaves the scroll where you put it | -| **Don't follow** | `Ctrl+Alt+F` | Holds the view where you put it | - -The **Order** grid marks the frame being played under every mode, and the tracker -marks the sounding row of the frame it shows — so a held view still shows the -playhead each time the song passes through the frame you are editing. -**Follow rows** is the one that moves the grid while you play, which is what makes -the other two the modes to type in: they hold the view still under your cursor while -the song runs. - -## Listening to one channel at a time - -Channel names are switches. Click **Triangle** at the top of the tracker to silence -that channel: its name grays, its column and its row in the **Order** grid go -neutral, and its notes dim — still readable, still editable, just not sounding. -Click the name again to bring it back. The same click works on the channel's name -in the **Order** grid, and both grids show every change, so a channel looks the same -wherever you see it. +| **Follow rows** | `Ctrl+F` | Scrolls the tracker to the playing row, and shows the playing frame | +| **Follow patterns** | `Ctrl+Shift+F` | Shows the playing frame, and keeps your scroll position | +| **Don't follow** | `Ctrl+Alt+F` | Stays where you put it | + +In every mode, the **Order** grid marks the playing frame, and the tracker marks the playing row. +Use **Follow patterns** or **Don't follow** to type while the song plays. + +## Muting channels + +Click a channel's name at the top of the tracker to mute the channel. Click the name again to +unmute it. The channel names in the **Order** grid work the same way. | Gesture | Action | |---------|--------| -| Click a channel's name | Silence it, or bring it back | -| `Ctrl`+click a channel's name | Solo it — silence the other three; `Ctrl`+click again returns the mix you had | -| Click **Sample** (tracker) or **Master** (order) | Silence every channel, or bring them all back | -| Right-click any name | The same actions as a menu | - -The **Playback ▸ Channels** submenu carries the same mix: a check marks each channel -that sounds, and **Unmute all channels** returns the whole set. `1` to `4` do the -same from the keyboard, one key per channel, wherever the grids are not holding your -cursor — inside them the digits enter values. - -Muting is for listening only. The song keeps every channel, so saving, exporting a -module, and undo all work on the full arrangement, and a mute survives undo and -redo. Toggling during playback is heard within about a quarter second. Opening, -creating, or closing a project starts a fresh listening session with every channel -audible. +| Click a channel's name | Mute or unmute the channel | +| `Ctrl`+click a channel's name | Solo the channel. `Ctrl`+click again to restore the previous mix | +| Click **Sample** (tracker) or **Master** (order) | Mute or unmute every channel | +| Right-click a name | The same commands as a menu | + +**Playback ▸ Channels** shows which channels play. **Unmute all channels** unmutes all four. Keys +`1` to `4` mute and unmute a channel when the cursor is outside the grids. Inside the grids, the +digit keys type values. + +Muting changes only what you hear. Saving, exporting, rendering and undo use every channel. Opening, +creating or closing a project unmutes all channels. ## Timing and properties -Set the song's timing in **Module options** on the right: **Rows** per pattern, -**Tempo**, **Speed**, and the **NES frequency**. Changing the **NES frequency** -after voices exist re-times how they all play back, so it asks **Change NES -frequency** first (with a **Don't ask again** option). +**Module options** sets the song's timing: **Rows** per pattern, **Tempo**, **Speed** and **NES +frequency**. If the project has voices, changing **NES frequency** changes how they play, so the +app asks **Change NES frequency** first. Check **Don't ask again** to skip the question. + +**File ▸ Project properties...** sets the title, the author and the comment. The exported module +includes them. -The project's title, author, and comment — which carry into the exported module — -are set in **File ▸ Project properties...**, along with the meter the song is -counted in. +**Project properties** also sets the meter: -**First highlight** and **Second highlight** are that meter: how many rows make a -beat, and how many make a bar. The tracker tints the row that opens each one. The -bar divided by the beat is how many beats you hear in a bar, so the default 4 and -16 give four beats of four rows — common time. Waltz time keeps the four-row beat -and shortens the bar to 12, for three beats. The beat is what the tempo counts, so -the two together say how fast the song is felt as well as how it looks. +- **First highlight** is the number of rows in a beat. +- **Second highlight** is the number of rows in a bar. -The meter also places the song's timing. Most tempos ask for a row length the engine -can only reach on average, so the rows of a bar differ a little: the meter gives the -extra time to the row that opens the bar, then to the row that opens each beat, which -keeps the beat audible where you expect it. +The tracker marks the first row of each beat and each bar. The defaults, 4 and 16, give four beats +of four rows in a bar. For waltz time, set **Second highlight** to 12, which gives three beats. + +The tempo counts beats, so the meter also changes how fast the song feels. [Tempo as a +groove](../formats/bitphase.md#d-tempo-as-a-groove) explains how the app spreads a tempo over the +rows. ## Undo and export -Every change is undoable. The **History** panel on the right shows the stack, with -**Undo** and **Redo** (also on the **Edit** menu); click any entry to jump straight -to that point. +You can undo every change. The **History** panel lists your changes. **Undo** and **Redo** are also +on the **Edit** menu. Click a change in the **History** panel to go back to that point. -When the song is ready, **File ▸ Export ▸ FamiTracker module...** writes the -`.ftm`. See -[FamiTracker export](../formats/famitracker.md) for what the module contains and -the limits it respects. +To export the song: -**File ▸ Export ▸ NSF program...** writes the song as an `.nsf` instead: a program -the console itself plays, carrying its own player, so it needs no tracker to sound. -The console holds one program in 32 KB, so a long song can outgrow it — the export -says so rather than writing a file that plays part of itself. See -[NSF export](../formats/nsf.md) for what the file holds, and -[song compression](../concepts/compression.md) for how a song of minutes is fitted -into that space. +- **File ▸ Export ▸ FamiTracker module...** saves an `.ftm` file. See [FamiTracker + export](../formats/famitracker.md) for its contents and limits. +- **File ▸ Export ▸ Bitphase project...** saves a `.btp` file. +- **File ▸ Export ▸ NSF program...** saves an `.nsf` file, which the NES or an NSF player plays + directly. An NSF program has room for 32 KB, so the app tells you when a song is too long. See [NSF + export](../formats/nsf.md) and [song compression](../concepts/compression.md). ## Rendering to audio -A module is for a tracker. To get a file anyone can play, use **File ▸ Render -song...** (`Ctrl+Shift+E`), which writes the whole song as audio. - -The dialog holds the choices: +**File ▸ Render song...** (`Ctrl+Shift+E`) saves the whole song as an audio file that any player +opens. | Setting | What it does | |---------|--------------| -| **Format** | **WAV** for the full-quality file, **MP3** for a smaller one | -| **Sample rate** | How many samples a second the file holds; 44100 Hz is the usual choice | -| **Bit depth** (WAV) | How finely each sample is stored. 16-bit PCM is the usual choice; 8-bit is there for the crunch the NES itself has | -| **Bitrate** (MP3) | How much the file spends per second — higher sounds better and takes more room. What is on offer depends on the sample rate, so the list follows when you change it | -| **Normalize peak** | Lifts the whole song so its loudest moment reaches full scale, keeping the balance between channels as it was | -| **File** | Where it is written. **Browse...** opens the save dialog, clicking the path shows where the file is going in your file manager, and the folder you pick is offered again next time | - -**Length** tells you how long the file will be before you start. **Render** begins, -and a bar reports how far it has got; **Cancel** stops it and leaves the file -unwritten. When it finishes, _SampleToNES_ shows the file it wrote — click the path -to open its folder. - -A render takes the song itself, once through, with every channel sounding: muting -and **Loop song** are for listening and stay out of the file. It is one of the long -jobs that run alone, so the item is unavailable while a conversion or a library -generation is going, and those wait for a render in the same way. +| **Format** | **WAV** for full quality, **MP3** for a smaller file | +| **Sample rate** | Samples per second. 44100 Hz is the usual choice | +| **Bit depth** (WAV) | 16-bit PCM is the usual choice. 8-bit sounds rougher, like the NES | +| **Bitrate** (MP3) | A higher bitrate sounds better and makes a larger file. The choices depend on the sample rate | +| **Normalize peak** | Makes the song louder until its loudest moment reaches full volume, keeping the balance between channels | +| **File** | Where the file is saved. **Browse...** opens the save dialog. Click the path to open its folder | + +**Length** shows how long the file will be. Click **Render** to start. **Cancel** stops the render +and saves no file. When the render finishes, click the path to open its folder. + +A render plays the song once, with every channel, whatever you muted or looped. While a render +runs, conversions and library builds wait, and a render waits for them in the same way. diff --git a/docs/index.md b/docs/index.md index ee95a56f9..08611ae56 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,11 +1,11 @@ # SampleToNES documentation -_SampleToNES_ approximates an audio sample using only the sound channels of the -NES's 2A03 chip — two pulse waves, a triangle and noise — and lets you arrange -the results into a song and export them to [FamiTracker](glossary.md#famitracker), -to [Bitphase](glossary.md#bitphase), or as an `.nsf` program the console itself plays. -This is the documentation for using it, understanding how it works, and building -on it. +_SampleToNES_ rebuilds an audio sample from the sound channels of the NES's 2A03 chip: two pulse +waves, a triangle and noise. You can arrange the results into a song. You can export them to +[FamiTracker](glossary.md#famitracker), to [Bitphase](glossary.md#bitphase), or as an `.nsf` program +the NES plays. + +This documentation explains how to use the app, how it works, and how to build on it. The sections below are grouped by what you want to do. They assume different starting points: the guide needs no prior knowledge, the concepts and formats @@ -16,7 +16,7 @@ sections are written for programmers. The [**guide**](guide/) walks through the application from installation onward. -- [Installation](guide/installation.md) — the standalone build, running from source, and GPU acceleration. +- [Installation](guide/installation.md) — a release download, PyPI, running from source, and GPU acceleration. - [Getting started](guide/getting-started.md) — your first reconstruction and your first song. - [The interface](guide/interface.md) — the four tabs, the menus, and the keyboard shortcuts. - [Converting audio](guide/converting.md) — the Main tab: gathering recordings, choosing channels, and running a conversion. @@ -57,16 +57,16 @@ worked examples. ## Development -The [**development**](development/) section is for contributors. The documents at its top -span the whole repository; the documents about the graphical application and about a release -each have a directory of their own. +The [**development**](development/) section is for contributors. The documents at its top cover +the whole repository. The documents about the graphical application and about releases have +directories of their own. - [Architecture](development/architecture.md) — the application's layers and the contracts between them. - [Package layers](development/packages.md) — the packages the repository divides into, and the order they import each other in. - [Tooling](development/tooling.md) — the `sampletones` command, the tools package and the bootstrap scripts: what each runs on and what it may import. - [Coding guidelines](development/guidelines.md) — conventions for the codebase. - [Console player](development/player.md) — the 6502 driver an `.nsf` carries, the codec that fits a song beside it, and how both are verified. -- [Progress](development/progress.md) — how a long operation says how far it has come, in one process and across the pool's workers. +- [Progress](development/progress.md) — how a long operation reports its progress, in one process and across the pool's workers. - [Bugs and to-dos](development/bugs-and-todos.md) — the working ledger of known gaps. ### The application @@ -83,7 +83,7 @@ each have a directory of their own. ### Releases -- [Data compatibility](development/release/compatibility.md) — the upgrades that bring a file an older version saved up to the current format as it loads. +- [Data compatibility](development/release/compatibility.md) — how a file saved by an older version is upgraded to the current format when it loads. - [Dependencies](development/release/dependencies.md) — the libraries _SampleToNES_ builds on. ## Glossary From 2097dc445454d473f319679c3b47ed9cce0549bf Mon Sep 17 00:00:00 2001 From: JakimPL Date: Sun, 13 Sep 2026 21:07:38 +0200 Subject: [PATCH 29/36] Fixed: the Windows path in the open command's refusal test --- tests/unit/sampletones/commands/test_open.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/unit/sampletones/commands/test_open.py b/tests/unit/sampletones/commands/test_open.py index ddd373808..264f622f1 100644 --- a/tests/unit/sampletones/commands/test_open.py +++ b/tests/unit/sampletones/commands/test_open.py @@ -1,3 +1,4 @@ +import re from pathlib import Path import pytest @@ -44,7 +45,7 @@ def test_a_recording_is_pointed_at_convert(self, monkeypatch: pytest.MonkeyPatch monkeypatch.setattr(LAUNCHER, application) path = _file(tmp_path, "song.wav") - with pytest.raises(SystemExit, match=f"sampletones convert {path}"): + with pytest.raises(SystemExit, match=re.escape(f"sampletones convert {path}")): dispatch(COMMANDS, ["open", str(path)]) assert application.starts == [] From 1dc9ba06e2210eeca1bd912b38cf5438eaa681ae Mon Sep 17 00:00:00 2001 From: JakimPL Date: Sun, 13 Sep 2026 21:25:46 +0200 Subject: [PATCH 30/36] Improved: compression progress indicator --- docs/development/architecture.md | 2 +- .../layout/settings/__init__.py | 2 +- .../layout/settings/export/__init__.py | 0 .../layout/settings/{ => export}/export.py | 2 + .../layout/settings/export/indicator.py | 13 +++++ .../ui/elements/layout/centered.py | 26 ++++++++++ .../ui/elements/plus_minus_buttons.py | 7 ++- .../ui/panels/dialogs/export.py | 38 +++++++++----- .../view_model/shared/export.py | 8 +++ .../layout/settings/export.yaml | 3 ++ .../ui/elements/layout/test_centered.py | 52 +++++++++++++++++++ .../ui/panels/dialogs/test_export.py | 5 ++ .../view_model/shared/test_export.py | 6 +++ 13 files changed, 148 insertions(+), 16 deletions(-) create mode 100644 src/sampletones_application/layout/settings/export/__init__.py rename src/sampletones_application/layout/settings/{ => export}/export.py (59%) create mode 100644 src/sampletones_application/layout/settings/export/indicator.py create mode 100644 src/sampletones_application/ui/elements/layout/centered.py create mode 100644 tests/unit/sampletones_application/ui/elements/layout/test_centered.py diff --git a/docs/development/architecture.md b/docs/development/architecture.md index 6fdee6c11..2bbfd250a 100644 --- a/docs/development/architecture.md +++ b/docs/development/architecture.md @@ -166,7 +166,7 @@ They read the source as an AST through the source layer in `sampletones_tools/ch | Path | Role | |------|------| | `ui/elements/` | Reusable low-level widgets: `GUIPanel` (the panel base class), `GUIWindow` (modal variant), buttons, tables, graphs, trees, fonts, the status bar, and `MenuSection` — a run of menu items restated each time its menu is opened | -| `ui/elements/layout/` | Reusable layout primitives: `TabColumns` (the tab column scaffold), the `card()` context manager and the `well()` inset region, driven declaratively by tab coordinators | +| `ui/elements/layout/` | Reusable layout primitives: `TabColumns` (the tab column scaffold), the `card()` context manager and the `well()` inset region, driven declaratively by tab coordinators, and `centered()`, which stands content in the middle of the width it is offered | | `ui/panels/` | Domain-level composite panels, organized by feature area | | `ui/themes/` | DPG themes and per-widget style helpers | | `ui/resources/` | Icons and image resources loaded at startup | diff --git a/src/sampletones_application/layout/settings/__init__.py b/src/sampletones_application/layout/settings/__init__.py index f95124389..c23a3763b 100644 --- a/src/sampletones_application/layout/settings/__init__.py +++ b/src/sampletones_application/layout/settings/__init__.py @@ -2,7 +2,7 @@ from sampletones_application.layout.settings.audio import AudioSettingsLayout from sampletones_application.layout.settings.display import DisplaySettingsLayout -from sampletones_application.layout.settings.export import ExportSettingsLayout +from sampletones_application.layout.settings.export.export import ExportSettingsLayout from sampletones_application.layout.settings.keybindings import KeybindingsSettingsLayout from sampletones_application.layout.settings.render import RenderSettingsLayout diff --git a/src/sampletones_application/layout/settings/export/__init__.py b/src/sampletones_application/layout/settings/export/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/sampletones_application/layout/settings/export.py b/src/sampletones_application/layout/settings/export/export.py similarity index 59% rename from src/sampletones_application/layout/settings/export.py rename to src/sampletones_application/layout/settings/export/export.py index 5c4d289a9..ffdb77547 100644 --- a/src/sampletones_application/layout/settings/export.py +++ b/src/sampletones_application/layout/settings/export/export.py @@ -1,7 +1,9 @@ from pydantic import BaseModel from sampletones_application.layout.primitives import Dimensions +from sampletones_application.layout.settings.export.indicator import LoadingIndicatorLayout class ExportSettingsLayout(BaseModel, extra="forbid", frozen=True): window: Dimensions + indicator: LoadingIndicatorLayout diff --git a/src/sampletones_application/layout/settings/export/indicator.py b/src/sampletones_application/layout/settings/export/indicator.py new file mode 100644 index 000000000..f4099966a --- /dev/null +++ b/src/sampletones_application/layout/settings/export/indicator.py @@ -0,0 +1,13 @@ +from pydantic import BaseModel, Field + + +class LoadingIndicatorLayout(BaseModel, extra="forbid", frozen=True): + """The turning ring's size, stated as factors of the font it is drawn in. + + DearPyGui draws the ring ``radius × font size × (1 − thickness / 4)`` pixels across and pads its + box by the frame padding above and below, the same padding it sets text beside the ring down by. + The ring sits level with a line of that font when ``radius × (1 − thickness / 4)`` equals one. + """ + + radius: float = Field(gt=0.0) + thickness: float = Field(gt=0.0, lt=4.0) diff --git a/src/sampletones_application/ui/elements/layout/centered.py b/src/sampletones_application/ui/elements/layout/centered.py new file mode 100644 index 000000000..6ed5bcb03 --- /dev/null +++ b/src/sampletones_application/ui/elements/layout/centered.py @@ -0,0 +1,26 @@ +from contextlib import contextmanager +from typing import Final, Iterator + +import dearpygui.dearpygui as dpg + +EQUAL_SHARE: Final[float] = 1.0 + + +@contextmanager +def centered() -> Iterator[None]: + """Stand what the block builds in the middle of the width it is offered. + + The content sits in a column fitted to it, flanked by two columns sharing the rest of the width + equally. The fitted column follows the content from frame to frame, so a label that grows or + shrinks while it stands stays centered. + """ + with dpg.table(header_row=False): + dpg.add_table_column(width_stretch=True, init_width_or_weight=EQUAL_SHARE) + dpg.add_table_column(width_fixed=True) + dpg.add_table_column(width_stretch=True, init_width_or_weight=EQUAL_SHARE) + with dpg.table_row(): + dpg.add_spacer() + with dpg.group(): + yield + + dpg.add_spacer() diff --git a/src/sampletones_application/ui/elements/plus_minus_buttons.py b/src/sampletones_application/ui/elements/plus_minus_buttons.py index b355f4363..0f2dbc79c 100644 --- a/src/sampletones_application/ui/elements/plus_minus_buttons.py +++ b/src/sampletones_application/ui/elements/plus_minus_buttons.py @@ -90,7 +90,12 @@ def delete(self) -> None: """Removes the buttons and, when hold-repeat is armed, the shared mouse handler.""" self._clear_existing_items() - def _build(self, *, increment_enabled: bool, decrement_enabled: bool) -> None: + def _build( + self, + *, + increment_enabled: bool, + decrement_enabled: bool, + ) -> None: self._clear_existing_items() increment_leads = self._order is PlusMinusOrder.PLUS_FIRST with dpg.table( diff --git a/src/sampletones_application/ui/panels/dialogs/export.py b/src/sampletones_application/ui/panels/dialogs/export.py index 25f53c6f9..6a97b5b95 100644 --- a/src/sampletones_application/ui/panels/dialogs/export.py +++ b/src/sampletones_application/ui/panels/dialogs/export.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, Optional +from typing import Any, Dict, Final, Optional import dearpygui.dearpygui as dpg @@ -20,6 +20,7 @@ from sampletones_application.ui.elements.dialog import GUIDialogWindow from sampletones_application.ui.elements.fonts.font import Font from sampletones_application.ui.elements.fonts.registry import FontRegistry +from sampletones_application.ui.elements.layout.centered import centered from sampletones_application.utils.gui.dialog_navigation import FocusStop from sampletones_application.utils.gui.dpg import dpg_configure_item, dpg_set_value from sampletones_application.utils.gui.keyboard import KeyRouter @@ -30,6 +31,8 @@ from sampletones_core.exports.stage import ExportStage from sampletones_shared.types.callback import VoidCallback +RING_STYLE: Final[int] = 1 + class GUIExportWindow(GUIDialogWindow): """Modal report over an export while it runs. @@ -55,6 +58,7 @@ def __init__( ) -> None: self._language_manager = language_manager self._text_colors = text_colors + self._indicator = layout.export.indicator self._view_model: SongExportViewModel = SongExportViewModel.idle() self.on_cancel: Optional[VoidCallback] = None @@ -138,23 +142,27 @@ def _create_working(self) -> None: """The reading of a stage whose length the data decides: what it holds, and that it turns. A bar would have to state a fraction of something, and this stage travels toward nothing, - so what stands here is the figure it does know beside a symbol that keeps moving. + so what stands here is the figure it does know beside a symbol that keeps moving. The pair + stands centered in the window, and both are drawn in the figure's font, which is the font + the ring takes its size from, so the ring sits level with the line beside it. """ with dpg.group( tag=TAG_SETTINGS_EXPORT_GROUP_WORKING, show=False, - horizontal=True, ): - dpg.add_loading_indicator( - style=1, - radius=2.0, - thickness=1.5, - ) - dpg.add_text("", tag=TAG_SETTINGS_EXPORT_TEXT_FIGURE) - FontRegistry.bind_to_item( - TAG_SETTINGS_EXPORT_TEXT_FIGURE, - Font.MONO_SMALL, - ) + with centered(): + with dpg.group(horizontal=True): + dpg.add_loading_indicator( + style=RING_STYLE, + radius=self._indicator.radius, + thickness=self._indicator.thickness, + ) + dpg.add_text("", tag=TAG_SETTINGS_EXPORT_TEXT_FIGURE) + + FontRegistry.bind_to_item( + TAG_SETTINGS_EXPORT_GROUP_WORKING, + Font.MONO_SMALL, + ) def _create_cancel(self) -> None: GUIButton( @@ -182,6 +190,10 @@ def _render(self) -> None: show=view_model.working_visible, ) dpg_set_value(TAG_SETTINGS_EXPORT_TEXT_FIGURE, view_model.figure) + dpg_configure_item( + TAG_SETTINGS_EXPORT_TEXT_FIGURE, + show=view_model.figure_visible, + ) dpg_configure_item( TAG_SETTINGS_EXPORT_BUTTON_CANCEL, enabled=view_model.cancel_enabled, diff --git a/src/sampletones_application/view_model/shared/export.py b/src/sampletones_application/view_model/shared/export.py index 64aece0e0..dee7cf5b7 100644 --- a/src/sampletones_application/view_model/shared/export.py +++ b/src/sampletones_application/view_model/shared/export.py @@ -78,6 +78,14 @@ def working_visible(self) -> bool: """Whether the turning indicator stands, which is how a stage without an end reads.""" return not self.traveling + @property + def figure_visible(self) -> bool: + """Whether the figure stands beside the turning indicator, which words to state earn. + + A stage with nothing to state leaves the indicator alone, centered on its own width. + """ + return bool(self.figure) + @property def progress_overlay(self) -> str: """The percentage label rendered over the progress bar, derived from the fraction.""" diff --git a/src/sampletones_config/layout/settings/export.yaml b/src/sampletones_config/layout/settings/export.yaml index 6843f19d3..beb55ad60 100644 --- a/src/sampletones_config/layout/settings/export.yaml +++ b/src/sampletones_config/layout/settings/export.yaml @@ -1,3 +1,6 @@ window: width: 460 height: 120 +indicator: + radius: 1.6 + thickness: 1.5 diff --git a/tests/unit/sampletones_application/ui/elements/layout/test_centered.py b/tests/unit/sampletones_application/ui/elements/layout/test_centered.py new file mode 100644 index 000000000..5f68d5f7a --- /dev/null +++ b/tests/unit/sampletones_application/ui/elements/layout/test_centered.py @@ -0,0 +1,52 @@ +from typing import Final, Iterator, List + +import dearpygui.dearpygui as dpg +import pytest + +from sampletones_application.ui.elements.layout.centered import centered + +CONTENT_TAG: Final[str] = "test.centered.text.content" +COLUMN_SLOT: Final[int] = 0 +CELL_SLOT: Final[int] = 1 +MIDDLE: Final[int] = 1 + + +@pytest.fixture(name="dpg_context") +def dpg_context_fixture() -> Iterator[None]: + dpg.create_context() + try: + yield + finally: + dpg.destroy_context() + + +def build() -> int: + """Builds one centered item and returns the table standing around it.""" + with dpg.window(): + with centered(): + dpg.add_text("content", tag=CONTENT_TAG) + + group = dpg.get_item_parent(CONTENT_TAG) + row = dpg.get_item_parent(group) + return dpg.get_item_parent(row) + + +class TestCentered: + """``centered`` stands the block's content in a fitted column between two equal ones.""" + + def test_the_content_sits_in_the_middle_cell(self, dpg_context: None) -> None: + table = build() + row = dpg.get_item_children(table, CELL_SLOT)[0] + cells: List[int] = dpg.get_item_children(row, CELL_SLOT) + assert cells.index(dpg.get_item_parent(CONTENT_TAG)) == MIDDLE + + def test_the_middle_column_fits_its_content(self, dpg_context: None) -> None: + columns: List[int] = dpg.get_item_children(build(), COLUMN_SLOT) + assert dpg.get_item_configuration(columns[MIDDLE])["width_fixed"] + + def test_the_flanking_columns_share_the_rest_equally(self, dpg_context: None) -> None: + columns: List[int] = dpg.get_item_children(build(), COLUMN_SLOT) + left = dpg.get_item_configuration(columns[0]) + right = dpg.get_item_configuration(columns[-1]) + assert left["width_stretch"] and right["width_stretch"] + assert left["init_width_or_weight"] == right["init_width_or_weight"] diff --git a/tests/unit/sampletones_application/ui/panels/dialogs/test_export.py b/tests/unit/sampletones_application/ui/panels/dialogs/test_export.py index b6c2ef49c..6789c261e 100644 --- a/tests/unit/sampletones_application/ui/panels/dialogs/test_export.py +++ b/tests/unit/sampletones_application/ui/panels/dialogs/test_export.py @@ -116,6 +116,11 @@ def test_a_stage_without_an_end_shows_what_it_holds(self, window: GUIExportWindo def test_the_figure_reaches_the_reader(self, window: GUIExportWindow) -> None: render(window, traveling=False, figure=SIZE) assert dpg.get_value(TAG_SETTINGS_EXPORT_TEXT_FIGURE) == SIZE + assert shown(TAG_SETTINGS_EXPORT_TEXT_FIGURE) + + def test_a_stage_with_nothing_to_state_leaves_the_indicator_alone(self, window: GUIExportWindow) -> None: + render(window, traveling=False, figure="") + assert not shown(TAG_SETTINGS_EXPORT_TEXT_FIGURE) class TestStoppingARun: diff --git a/tests/unit/sampletones_application/view_model/shared/test_export.py b/tests/unit/sampletones_application/view_model/shared/test_export.py index f744e4825..d239e86d0 100644 --- a/tests/unit/sampletones_application/view_model/shared/test_export.py +++ b/tests/unit/sampletones_application/view_model/shared/test_export.py @@ -75,6 +75,12 @@ def test_a_traveling_stage_hides_the_turning_symbol(self) -> None: def test_a_stage_without_an_end_shows_the_turning_symbol(self) -> None: assert view_model(traveling=False, figure=SIZE).working_visible is True + def test_a_stage_with_words_to_state_shows_its_figure(self) -> None: + assert view_model(traveling=False, figure=SIZE).figure_visible is True + + def test_a_stage_with_nothing_to_state_leaves_the_indicator_alone(self) -> None: + assert view_model(traveling=False, figure="").figure_visible is False + def test_a_bar_is_labeled_with_the_share_it_has_covered(self) -> None: assert view_model(progress=HALFWAY).progress_overlay == "50%" From 387e930929d6e781aaf2963e7125acb834ddc3b9 Mon Sep 17 00:00:00 2001 From: JakimPL Date: Sun, 13 Sep 2026 21:36:34 +0200 Subject: [PATCH 31/36] Fixed: ruff issues --- .../ui/panels/dialogs/export.py | 24 +++++++------- .../ui/panels/main/converter/setup.py | 23 +++++++------ .../reconstruction/instruments/instruments.py | 32 ++++++++++--------- .../utils/palette/colors/written.py | 9 ++++-- src/sampletones_core/features/__init__.py | 4 +-- src/sampletones_core/generators/__init__.py | 4 +-- .../headless/conversion/request.py | 6 ++-- .../parallelization/channel/process.py | 2 +- .../parallelization/progress.py | 6 ++-- src/sampletones_shared/utils/validation.py | 2 +- .../checks/import_boundary.py | 0 src/sampletones_tools/checks/language_keys.py | 0 .../checks/palette_colors.py | 0 .../checks/rendered_literals.py | 0 .../checks/shortcut_actions.py | 0 src/sampletones_tools/checks/tag_names.py | 0 src/sampletones_tools/checks/unused_tags.py | 0 .../codec/study/accounting/pairs.py | 3 +- .../codec/study/accounting/runs.py | 4 +-- .../codec/study/report/run.py | 6 ++-- src/sampletones_tools/samples/render.py | 0 src/sampletones_tools/synthesis/frequency.py | 4 ++- 22 files changed, 75 insertions(+), 54 deletions(-) mode change 100755 => 100644 src/sampletones_tools/checks/import_boundary.py mode change 100755 => 100644 src/sampletones_tools/checks/language_keys.py mode change 100755 => 100644 src/sampletones_tools/checks/palette_colors.py mode change 100755 => 100644 src/sampletones_tools/checks/rendered_literals.py mode change 100755 => 100644 src/sampletones_tools/checks/shortcut_actions.py mode change 100755 => 100644 src/sampletones_tools/checks/tag_names.py mode change 100755 => 100644 src/sampletones_tools/checks/unused_tags.py mode change 100755 => 100644 src/sampletones_tools/samples/render.py diff --git a/src/sampletones_application/ui/panels/dialogs/export.py b/src/sampletones_application/ui/panels/dialogs/export.py index 6a97b5b95..5725fda24 100644 --- a/src/sampletones_application/ui/panels/dialogs/export.py +++ b/src/sampletones_application/ui/panels/dialogs/export.py @@ -146,18 +146,20 @@ def _create_working(self) -> None: stands centered in the window, and both are drawn in the figure's font, which is the font the ring takes its size from, so the ring sits level with the line beside it. """ - with dpg.group( - tag=TAG_SETTINGS_EXPORT_GROUP_WORKING, - show=False, + with ( + dpg.group( + tag=TAG_SETTINGS_EXPORT_GROUP_WORKING, + show=False, + ), + centered(), + dpg.group(horizontal=True), ): - with centered(): - with dpg.group(horizontal=True): - dpg.add_loading_indicator( - style=RING_STYLE, - radius=self._indicator.radius, - thickness=self._indicator.thickness, - ) - dpg.add_text("", tag=TAG_SETTINGS_EXPORT_TEXT_FIGURE) + dpg.add_loading_indicator( + style=RING_STYLE, + radius=self._indicator.radius, + thickness=self._indicator.thickness, + ) + dpg.add_text("", tag=TAG_SETTINGS_EXPORT_TEXT_FIGURE) FontRegistry.bind_to_item( TAG_SETTINGS_EXPORT_GROUP_WORKING, diff --git a/src/sampletones_application/ui/panels/main/converter/setup.py b/src/sampletones_application/ui/panels/main/converter/setup.py index 01c50114f..c9a0da3cf 100644 --- a/src/sampletones_application/ui/panels/main/converter/setup.py +++ b/src/sampletones_application/ui/panels/main/converter/setup.py @@ -102,15 +102,20 @@ def create_controls(self) -> None: ) FontRegistry.bind_to_item(cap_input, Font.MONO) - with dpg.group(tag=TAG_MAIN_CONVERTER_GROUP_ORDER, show=False): - with labeled_field(self._language_manager["main.converter.label.hierarchy_mode"], self._label_width): - dpg.add_combo( - items=list(self._hierarchy_labels.values()), - tag=TAG_MAIN_CONVERTER_COMBO_HIERARCHY_MODE, - width=self._input_width, - default_value=self._hierarchy_labels[DEFAULT_STEMS_HIERARCHY_MODE], - callback=self._on_hierarchy_mode_edited, - ) + with ( + dpg.group(tag=TAG_MAIN_CONVERTER_GROUP_ORDER, show=False), + labeled_field( + self._language_manager["main.converter.label.hierarchy_mode"], + self._label_width, + ), + ): + dpg.add_combo( + items=list(self._hierarchy_labels.values()), + tag=TAG_MAIN_CONVERTER_COMBO_HIERARCHY_MODE, + width=self._input_width, + default_value=self._hierarchy_labels[DEFAULT_STEMS_HIERARCHY_MODE], + callback=self._on_hierarchy_mode_edited, + ) dpg.bind_item_handler_registry(TAG_MAIN_CONVERTER_INPUT_CHANNEL_CAP, self._settings_handler_tag) self._attach_tooltips() diff --git a/src/sampletones_application/ui/panels/reconstruction/instruments/instruments.py b/src/sampletones_application/ui/panels/reconstruction/instruments/instruments.py index 05ae2b8c7..b84b9312e 100644 --- a/src/sampletones_application/ui/panels/reconstruction/instruments/instruments.py +++ b/src/sampletones_application/ui/panels/reconstruction/instruments/instruments.py @@ -537,24 +537,26 @@ def _create_audition_selector(self, window_tag: str) -> None: means choosing which one reads it. The choice belongs to the reader listening rather than to the voice, so it stays on the panel and reaches no document. """ - with dpg.group( - tag=self.audition_group_tag, - parent=window_tag, - show=False, - ): - with labeled_field( + with ( + dpg.group( + tag=self.audition_group_tag, + parent=window_tag, + show=False, + ), + labeled_field( self._language_manager["reconstructions.instruments.label.audition"], self._pitch_stepper_style.dimensions.label_width, parent=self.audition_group_tag, - ): - dpg.add_radio_button( - items=[self._generator_labels[generator_name] for generator_name in GeneratorName], - tag=self.audition_tag, - default_value=self._generator_labels[AUDITION_GENERATOR], - callback=self._on_audition_generator_changed, - horizontal=True, - ) - FontRegistry.bind_to_item(self.audition_tag, Font.REGULAR_SMALL) + ), + ): + dpg.add_radio_button( + items=[self._generator_labels[generator_name] for generator_name in GeneratorName], + tag=self.audition_tag, + default_value=self._generator_labels[AUDITION_GENERATOR], + callback=self._on_audition_generator_changed, + horizontal=True, + ) + FontRegistry.bind_to_item(self.audition_tag, Font.REGULAR_SMALL) self._status_bar.bind_to_item( self.audition_tag, diff --git a/src/sampletones_application/utils/palette/colors/written.py b/src/sampletones_application/utils/palette/colors/written.py index 0684e5753..e782ad793 100644 --- a/src/sampletones_application/utils/palette/colors/written.py +++ b/src/sampletones_application/utils/palette/colors/written.py @@ -5,7 +5,10 @@ from sampletones_application.utils.palette.colors.base import BaseColor from sampletones_application.utils.palette.colors.literal import LiteralColor from sampletones_application.utils.palette.colors.named import NamedColor -from sampletones_application.utils.palette.reference import PaletteReference, is_reference +from sampletones_application.utils.palette.reference import ( + PaletteReference, + is_reference, +) from sampletones_application.utils.palette.source import PaletteSource from sampletones_shared.utils.color import parse_hex_color @@ -47,7 +50,9 @@ def _written_color(value: object, info: ValidationInfo) -> BaseColor: return value if not isinstance(value, str): - raise ValueError(f"A color is written as a palette reference or a hex literal, got {type(value)}") + raise ValueError( # noqa: TRY004 + f"A color is written as a palette reference or a hex literal, got {type(value)}" + ) text = value.strip() color: BaseColor diff --git a/src/sampletones_core/features/__init__.py b/src/sampletones_core/features/__init__.py index 5b51dd5ac..0a9762813 100644 --- a/src/sampletones_core/features/__init__.py +++ b/src/sampletones_core/features/__init__.py @@ -22,19 +22,19 @@ __all__ = [ "BEND_FEATURES", "CHANNEL_FEATURE_DEFAULTS", + "CHANNEL_GENERATOR_KIND", "FEATURE_DIMENSION_ORDER", "GENERATOR_CHANNEL_KINDS", "GENERATOR_FEATURE_RANGES", - "CHANNEL_GENERATOR_KIND", "RESTING_REFERENCE_PERIOD", "RESTING_REFERENCE_PITCH", "FeatureRange", "channel_reference", "feature_range", "generator_channel", - "speaks_in_periods", "resting_held_features", "resting_reference", + "speaks_in_periods", "supported_features", "supports", "transposed_reference", diff --git a/src/sampletones_core/generators/__init__.py b/src/sampletones_core/generators/__init__.py index 92183c2f6..614cb6130 100644 --- a/src/sampletones_core/generators/__init__.py +++ b/src/sampletones_core/generators/__init__.py @@ -30,11 +30,11 @@ __all__ = [ "CHANNEL_CLASSES", + "CLASS_NAME_TO_GENERATOR_MAP", "GENERATOR_CLASS_MAP", + "GENERATOR_TO_CLASS_NAME_MAP", "GENERATOR_TO_INSTRUCTION_MAP", "INSTRUCTION_TO_GENERATOR_MAP", - "CLASS_NAME_TO_GENERATOR_MAP", - "GENERATOR_TO_CLASS_NAME_MAP", "MIXER_LEVELS", "Generator", "GeneratorClass", diff --git a/src/sampletones_core/headless/conversion/request.py b/src/sampletones_core/headless/conversion/request.py index 47f816ee4..73ddf2420 100644 --- a/src/sampletones_core/headless/conversion/request.py +++ b/src/sampletones_core/headless/conversion/request.py @@ -11,7 +11,9 @@ ordered_channels, ) from sampletones_core.reconstructions.converter.paths import is_audio_file -from sampletones_core.reconstructions.reconstructor.stems.configs.config import StemsConfig +from sampletones_core.reconstructions.reconstructor.stems.configs.config import ( + StemsConfig, +) from sampletones_shared.paths.extensions import EXT_FILES_AUDIO from sampletones_shared.utils.serialization import load_json from sampletones_shared.utils.text import listed_items @@ -55,7 +57,7 @@ def load_stems(path: Path) -> StemsConfig: loaded = load_json(path) if not isinstance(loaded, dict): - raise ValueError(f"Stems file {path} must hold a mapping, got {type(loaded).__name__}.") + raise ValueError(f"Stems file {path} must hold a mapping, got {type(loaded).__name__}.") # noqa: TRY004 return StemsConfig.model_validate(loaded) diff --git a/src/sampletones_core/parallelization/channel/process.py b/src/sampletones_core/parallelization/channel/process.py index 340745d83..bf4426466 100644 --- a/src/sampletones_core/parallelization/channel/process.py +++ b/src/sampletones_core/parallelization/channel/process.py @@ -54,7 +54,7 @@ class ProcessProgressChannel: def __init__(self) -> None: self._manager: SyncManager = multiprocessing.get_context(SPAWN_CONTEXT).Manager() - self._reports: "queue.Queue[TaskReport]" = self._manager.Queue() + self._reports: queue.Queue[TaskReport] = self._manager.Queue() self._withdrawn: threading.Event = self._manager.Event() def reporter(self, index: int) -> StepReporter: diff --git a/src/sampletones_core/parallelization/progress.py b/src/sampletones_core/parallelization/progress.py index 19bf09379..6a1a9a443 100644 --- a/src/sampletones_core/parallelization/progress.py +++ b/src/sampletones_core/parallelization/progress.py @@ -1,6 +1,6 @@ from collections import deque from time import monotonic -from typing import Deque, Final, Optional, Tuple, Union +from typing import Deque, Final, Optional, Tuple ESTIMATION_MEASUREMENTS_SAMPLES: Final[float] = 0.05 @@ -15,7 +15,7 @@ class ETAEstimator: def __init__( self, - total: Union[int, float], + total: float, ems: float = ESTIMATION_MEASUREMENTS_SAMPLES, ) -> None: self._total = total @@ -23,7 +23,7 @@ def __init__( self._samples_window: Deque[Tuple[float, float]] = deque(maxlen=self._ems) self._processed_items: float = 0.0 - def update(self, completed_items: Union[int, float]) -> Optional[float]: + def update(self, completed_items: float) -> Optional[float]: now = monotonic() self._processed_items = completed_items self._samples_window.append((now, completed_items)) diff --git a/src/sampletones_shared/utils/validation.py b/src/sampletones_shared/utils/validation.py index 9e6cb67b9..0374b91bf 100644 --- a/src/sampletones_shared/utils/validation.py +++ b/src/sampletones_shared/utils/validation.py @@ -153,7 +153,7 @@ def _existing_prefix(container: Any, location: Location) -> Location: node: Any = container length = 0 for index, key in enumerate(location): - if isinstance(node, Mapping) and key in node: + if isinstance(node, Mapping) and key in node: # noqa: SIM114 node = node[key] elif isinstance(node, list) and isinstance(key, int) and -len(node) <= key < len(node): node = node[key] diff --git a/src/sampletones_tools/checks/import_boundary.py b/src/sampletones_tools/checks/import_boundary.py old mode 100755 new mode 100644 diff --git a/src/sampletones_tools/checks/language_keys.py b/src/sampletones_tools/checks/language_keys.py old mode 100755 new mode 100644 diff --git a/src/sampletones_tools/checks/palette_colors.py b/src/sampletones_tools/checks/palette_colors.py old mode 100755 new mode 100644 diff --git a/src/sampletones_tools/checks/rendered_literals.py b/src/sampletones_tools/checks/rendered_literals.py old mode 100755 new mode 100644 diff --git a/src/sampletones_tools/checks/shortcut_actions.py b/src/sampletones_tools/checks/shortcut_actions.py old mode 100755 new mode 100644 diff --git a/src/sampletones_tools/checks/tag_names.py b/src/sampletones_tools/checks/tag_names.py old mode 100755 new mode 100644 diff --git a/src/sampletones_tools/checks/unused_tags.py b/src/sampletones_tools/checks/unused_tags.py old mode 100755 new mode 100644 diff --git a/src/sampletones_tools/codec/study/accounting/pairs.py b/src/sampletones_tools/codec/study/accounting/pairs.py index 03918d148..1e00cdbaf 100644 --- a/src/sampletones_tools/codec/study/accounting/pairs.py +++ b/src/sampletones_tools/codec/study/accounting/pairs.py @@ -1,3 +1,4 @@ +import itertools from typing import Final, Sequence from sampletones_player.specification.compression import TokenTag @@ -25,7 +26,7 @@ def set_holds(tokens: Sequence[ReadToken]) -> Finding: Finding: The bytes such pairs take, and the bytes a two-byte token would spare. """ finding = NOTHING - for token, following in zip(tokens, tokens[1:]): + for token, following in itertools.pairwise(tokens): if token.tag is TokenTag.LITERAL and len(token.payload) == SINGLE_VALUE and following.tag is TokenTag.HOLD: now = token.size + following.size finding += Finding(now, now - SET_HOLD_SIZE) diff --git a/src/sampletones_tools/codec/study/accounting/runs.py b/src/sampletones_tools/codec/study/accounting/runs.py index d4b9200b5..bbbb19fca 100644 --- a/src/sampletones_tools/codec/study/accounting/runs.py +++ b/src/sampletones_tools/codec/study/accounting/runs.py @@ -1,4 +1,4 @@ -from itertools import groupby +from itertools import groupby, pairwise from typing import Tuple from sampletones_player.specification.binary import BYTE_VALUES @@ -28,5 +28,5 @@ def ramps(data: bytes) -> Tuple[int, ...]: Returns: Tuple[int, ...]: The length of every ramp of at least two values. """ - steps = tuple((following - value) % BYTE_VALUES for value, following in zip(data, data[1:])) + steps = tuple((following - value) % BYTE_VALUES for value, following in pairwise(data)) return tuple(len(list(group)) + 1 for step, group in groupby(steps) if step != 0) diff --git a/src/sampletones_tools/codec/study/report/run.py b/src/sampletones_tools/codec/study/report/run.py index d7e963b3f..407290c3f 100644 --- a/src/sampletones_tools/codec/study/report/run.py +++ b/src/sampletones_tools/codec/study/report/run.py @@ -124,8 +124,10 @@ def _header( "", f"Commit: {commit_hash()}", f"Date: {datetime.now(UTC).isoformat(timespec='seconds')}", - f"Manifest: {MANIFEST_JSON}, {len(manifest.projects)} projects lengthened to " - f"{manifest.lengthen_seconds} s, {len(manifest.reconstructions)} reconstruction sources", + ( + f"Manifest: {MANIFEST_JSON}, {len(manifest.projects)} projects lengthened to " + f"{manifest.lengthen_seconds} s, {len(manifest.reconstructions)} reconstruction sources" + ), f"Variants: {named}", "", "Each accounting share is the saving a hypothesis would reach, as a share of the whole song block.", diff --git a/src/sampletones_tools/samples/render.py b/src/sampletones_tools/samples/render.py old mode 100755 new mode 100644 diff --git a/src/sampletones_tools/synthesis/frequency.py b/src/sampletones_tools/synthesis/frequency.py index 9ab12a4df..f6a8fb700 100644 --- a/src/sampletones_tools/synthesis/frequency.py +++ b/src/sampletones_tools/synthesis/frequency.py @@ -18,7 +18,9 @@ def _require_hertz(value: Any) -> Any: ValueError: If the value is an integer. """ if isinstance(value, int) and not isinstance(value, bool): - raise ValueError("An integer frequency specification is a MIDI pitch and must lie in the pitch range") + raise ValueError( # noqa: TRY004 + "An integer frequency specification is a MIDI pitch and must lie in the pitch range" + ) return value From 41b6a85b88f15b8b2792df2d54e8f5651904813f Mon Sep 17 00:00:00 2001 From: JakimPL Date: Sun, 13 Sep 2026 21:46:52 +0200 Subject: [PATCH 32/36] Moved: the mark's data into its config package --- docs/development/application/config-organization.md | 5 +++-- docs/development/release/dependencies.md | 2 +- src/sampletones_tools/assets/mark/config/__init__.py | 0 src/sampletones_tools/assets/mark/{ => config}/mark.yaml | 0 src/sampletones_tools/assets/mark/{ => config}/template.svg | 0 src/sampletones_tools/assets/mark/paths.py | 6 +++--- src/sampletones_tools/assets/mark/specification/__init__.py | 2 +- 7 files changed, 8 insertions(+), 7 deletions(-) create mode 100644 src/sampletones_tools/assets/mark/config/__init__.py rename src/sampletones_tools/assets/mark/{ => config}/mark.yaml (100%) rename src/sampletones_tools/assets/mark/{ => config}/template.svg (100%) diff --git a/docs/development/application/config-organization.md b/docs/development/application/config-organization.md index 06181ad48..ea2ea0c64 100644 --- a/docs/development/application/config-organization.md +++ b/docs/development/application/config-organization.md @@ -39,8 +39,9 @@ So the data carries the values and the consumer carries the meaning, and the two on their own terms. Data only a developer tool reads ships with that tool, beside the schema reading it: the -calibration tuning sits in `sampletones_tools/calibration/config/` and the synthetic corpus in -`sampletones_tools/corpus/config/`, each placed by `package_directory`. `sampletones_config` +calibration tuning sits in `sampletones_tools/calibration/config/`, the synthetic corpus in +`sampletones_tools/corpus/config/` and the application mark in `sampletones_tools/assets/mark/config/`, +each placed by `package_directory`. `sampletones_config` holds what the application reads and the import boundaries, which state the repository's package layers for every tree and which [package layers](../packages.md) refers to. diff --git a/docs/development/release/dependencies.md b/docs/development/release/dependencies.md index 50f79d118..4196d8e8e 100644 --- a/docs/development/release/dependencies.md +++ b/docs/development/release/dependencies.md @@ -54,7 +54,7 @@ Dialogs open through the XDG desktop portal (`org.freedesktop.portal.FileChooser ## Application icon The icon suite in `src/sampletones_assets/icons` is generated from the mark declared in -`src/sampletones_tools/assets/mark`: `mark.yaml` carries the geometry, colors and rasterization +`src/sampletones_tools/assets/mark/config`: `mark.yaml` carries the geometry, colors and rasterization settings, validated as a `Mark`, and `template.svg` is the vector the rendered geometry fills. The tools package writes the whole suite — the vector `sampletones.svg` and the rasters the application ships, `sampletones.png` and the multi-resolution `sampletones.ico` — and `uv run sampletones icons` diff --git a/src/sampletones_tools/assets/mark/config/__init__.py b/src/sampletones_tools/assets/mark/config/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/sampletones_tools/assets/mark/mark.yaml b/src/sampletones_tools/assets/mark/config/mark.yaml similarity index 100% rename from src/sampletones_tools/assets/mark/mark.yaml rename to src/sampletones_tools/assets/mark/config/mark.yaml diff --git a/src/sampletones_tools/assets/mark/template.svg b/src/sampletones_tools/assets/mark/config/template.svg similarity index 100% rename from src/sampletones_tools/assets/mark/template.svg rename to src/sampletones_tools/assets/mark/config/template.svg diff --git a/src/sampletones_tools/assets/mark/paths.py b/src/sampletones_tools/assets/mark/paths.py index b1c3afd27..3b93d6c5e 100644 --- a/src/sampletones_tools/assets/mark/paths.py +++ b/src/sampletones_tools/assets/mark/paths.py @@ -3,6 +3,6 @@ from sampletones_shared.paths.package import package_directory -MARK_DIRECTORY: Final[Path] = package_directory("sampletones_tools.assets.mark") -MARK_PATH: Final[Path] = MARK_DIRECTORY / "mark.yaml" -TEMPLATE_PATH: Final[Path] = MARK_DIRECTORY / "template.svg" +MARK_CONFIG_DIRECTORY: Final[Path] = package_directory("sampletones_tools.assets.mark.config") +MARK_PATH: Final[Path] = MARK_CONFIG_DIRECTORY / "mark.yaml" +TEMPLATE_PATH: Final[Path] = MARK_CONFIG_DIRECTORY / "template.svg" diff --git a/src/sampletones_tools/assets/mark/specification/__init__.py b/src/sampletones_tools/assets/mark/specification/__init__.py index a2f386226..4b4b2239f 100644 --- a/src/sampletones_tools/assets/mark/specification/__init__.py +++ b/src/sampletones_tools/assets/mark/specification/__init__.py @@ -30,7 +30,7 @@ def load(cls) -> Self: """Load the packaged mark definition. Returns: - The mark validated from `sampletones_tools/assets/mark/mark.yaml`. + The mark validated from `sampletones_tools/assets/mark/config/mark.yaml`. Raises: TypeError: If the definition file holds anything other than a mapping. From 0091300bff399a16cd3fff21ef2ef7e205031e66 Mon Sep 17 00:00:00 2001 From: JakimPL Date: Sun, 13 Sep 2026 21:55:49 +0200 Subject: [PATCH 33/36] Hardened: the scripts' interpreter gate, CI conditions and release facts --- .github/workflows/ci.yml | 13 ++-- .gitignore | 1 + docs/development/tooling.md | 14 +++-- pyproject.toml | 1 - scripts/archive_bundle.py | 38 ++++++++--- scripts/bootstrap/__init__.py | 3 + scripts/bootstrap/cuda.py | 29 ++------- scripts/bootstrap/interpreter.py | 14 +++-- scripts/bootstrap/layout.py | 3 +- scripts/bootstrap/passes.py | 28 +++++++-- scripts/bootstrap/platforms/linux.py | 15 ++--- scripts/bootstrap/platforms/macos.py | 63 ++++++++++--------- scripts/bootstrap/platforms/protocol.py | 4 +- scripts/bootstrap/platforms/unix.py | 9 +++ scripts/bootstrap/platforms/windows.py | 4 +- scripts/bootstrap/project.py | 6 ++ scripts/bundle.py | 4 +- scripts/formatting.py | 4 +- scripts/lint.py | 4 +- scripts/run_tests.py | 41 +++--------- scripts/verify_bundle.py | 2 +- scripts/verify_version_tag.py | 31 ++++++--- tests/suite/bootstrap.py | 13 ++-- .../scripts/bootstrap/platforms/test_linux.py | 4 +- .../scripts/bootstrap/platforms/test_macos.py | 10 +-- .../bootstrap/platforms/test_windows.py | 2 +- tests/unit/scripts/bootstrap/test_cuda.py | 6 +- .../scripts/bootstrap/test_interpreter.py | 34 ++++++++-- tests/unit/scripts/bootstrap/test_passes.py | 19 +++++- tests/unit/scripts/bootstrap/test_project.py | 25 +++++--- tests/unit/scripts/test_archive_bundle.py | 33 +++++----- tests/unit/scripts/test_build_environment.py | 3 +- tests/unit/scripts/test_run_tests.py | 22 +------ tests/unit/scripts/test_setup_environment.py | 8 +-- tests/unit/scripts/test_verify_version_tag.py | 25 +++++++- 35 files changed, 310 insertions(+), 225 deletions(-) create mode 100644 scripts/bootstrap/platforms/unix.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9169e3305..6787bc557 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -52,6 +52,8 @@ jobs: matrix: os: [ubuntu-latest, windows-latest, macos-latest] python: ["3.12", "3.13"] + env: + PYTHON: ${{ matrix.os == 'windows-latest' && 'python' || 'python3' }} steps: - uses: actions/checkout@v7 @@ -72,18 +74,19 @@ jobs: python3 scripts/build_environment.py >> "$GITHUB_ENV" - name: Install the development environment + id: environment run: uv sync --group dev - name: Run the doctests shell: bash - run: ${{ runner.os == 'Windows' && 'python' || 'python3' }} scripts/run_tests.py doctests + run: $PYTHON scripts/run_tests.py doctests - name: Run the test suite with coverage - if: ${{ !cancelled() }} + if: ${{ !cancelled() && steps.environment.outcome == 'success' }} shell: bash - run: ${{ runner.os == 'Windows' && 'python' || 'python3' }} scripts/run_tests.py suite --workers auto + run: $PYTHON scripts/run_tests.py suite --workers auto - name: Run the benchmarks - if: ${{ !cancelled() }} + if: ${{ !cancelled() && steps.environment.outcome == 'success' }} shell: bash - run: ${{ runner.os == 'Windows' && 'python' || 'python3' }} scripts/run_tests.py benchmarks + run: $PYTHON scripts/run_tests.py benchmarks diff --git a/.gitignore b/.gitignore index 10e71d195..5d9bf85c5 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,7 @@ dist/ wheels/ /bin/ /build/ +/bundles/ sampletones !src/sampletones diff --git a/docs/development/tooling.md b/docs/development/tooling.md index 05cb4f251..7476745cc 100644 --- a/docs/development/tooling.md +++ b/docs/development/tooling.md @@ -19,10 +19,11 @@ command's name says what it does, in plain words. runs on the system interpreter, before or beside the environment: it creates the environment, installs system packages, builds the standalone bundle, cleans the tree, and runs the tests, the linters and the formatters the environment provides. It imports the standard library and the other -bootstrap modules, nothing else, so it runs on a machine that has Python and nothing more, and it -installs nothing into the interpreter it runs on: every package a build installs lands in -`.venv-build`, a virtual environment of its own, and pip is told to refuse any interpreter outside -one. System packages are a step of their own, `make system-deps`, the only one that asks for +bootstrap modules, nothing else, so it runs on a machine that has Python 3.12 or newer and nothing +more. Importing `scripts/bootstrap/` checks that version before anything else, so an older +interpreter is told the version and where to download it. A bootstrap script installs nothing into +the interpreter it runs on: every package a build installs lands in `.venv-build`, a virtual +environment of its own, and pip is told to refuse any interpreter outside one. System packages are a step of their own, `make system-deps`, the only one that asks for administrator rights. The import boundary check holds the tree to the rule: `sampletones_config/boundaries/standalone.yaml` names the scripts, and an import beyond the standard library and the tree fails the hook. @@ -165,8 +166,9 @@ reason. - `venv_build.py`, `preflight.py`: the build environment and the installs into it, and the preflight of the build interpreter. -Every script's work is a function taking the runner and the variables; `main` parses the arguments -and passes in the real ones, and the tests pass a `RecordingRunner`. +Every script's work is a function taking what it reads: the repository root, the platform, and for +a script running commands the runner and the variables. `main` parses the arguments and passes in +the real ones, and the tests pass a temporary repository and a `RecordingRunner`. ## Who governs what diff --git a/pyproject.toml b/pyproject.toml index 5912fdbc7..0b2680fc8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -119,7 +119,6 @@ profile = "black" line_length = 120 known_first_party = [ "bootstrap", - "ci", "sampletones", "sampletones_application", "sampletones_assets", diff --git a/scripts/archive_bundle.py b/scripts/archive_bundle.py index 171653c27..19a586392 100644 --- a/scripts/archive_bundle.py +++ b/scripts/archive_bundle.py @@ -5,6 +5,8 @@ from typing import Final, List, Sequence from bootstrap.layout import BUNDLES, DISTRIBUTION, repository_root +from bootstrap.platforms.factory import current_platform +from bootstrap.platforms.protocol import Platform from bootstrap.project import Project, read_project ARCHIVE_COMPRESSION: Final[int] = zipfile.ZIP_DEFLATED @@ -15,13 +17,13 @@ def archive_root(project: Project, label: str) -> str: """The directory every archived entry sits under, which names the archive too. Args: - project: The project, whose name and version lead the name. + project: The project, whose name and release tag lead the name. label: The platform the bundle was built for, such as ``windows-x86_64``. Returns: str: The name, such as ``sampletones-v0.3.0-windows-x86_64``. """ - return f"{project.name}-v{project.version}-{label}" + return f"{project.name}-{project.tag}-{label}" def bundle_entries(source: Path) -> List[Path]: @@ -50,25 +52,41 @@ def write_archive(source: Path, archive: Path, *, root: str) -> List[Path]: return entries -def main(argv: Sequence[str]) -> int: - """Archives the release bundle into the bundles directory, named by the version and the platform.""" - parser = argparse.ArgumentParser(description="Archive the release bundle under a versioned root.") - parser.add_argument("--label", required=True, help="the platform the bundle was built for, such as windows-x86_64") - arguments = parser.parse_args(list(argv)) +def archive_release(root: Path, platform: Platform, *, label: str) -> int: + """Archives the release bundle built under ``root`` into its bundles directory. + + Args: + root: The repository the bundle was built in. + platform: The system the bundle was built for, which places the bundle's directory. + label: The platform's name in the archive, such as ``windows-x86_64``. + + Returns: + int: ``0`` once the archive is written, ``1`` where the release bundle is missing. - root = repository_root() + Raises: + SystemExit: If the system builds no bundle. + """ project = read_project(root) - source = root / DISTRIBUTION / project.name + source = platform.bundling().launcher(root / DISTRIBUTION, name=project.name, release=True).parent if not source.is_dir(): print(f"::error::Bundle directory {source} is missing") return 1 - name = archive_root(project, arguments.label) + name = archive_root(project, label) archive = root / BUNDLES / f"{name}{ARCHIVE_SUFFIX}" entries = write_archive(source, archive, root=name) print(f"Archived {len(entries)} entries from {source} into {archive}") return 0 +def main(argv: Sequence[str]) -> int: + """Archives the release bundle into the bundles directory, named by the version and the platform.""" + parser = argparse.ArgumentParser(description="Archive the release bundle under a versioned root.") + parser.add_argument("--label", required=True, help="the platform the bundle was built for, such as windows-x86_64") + arguments = parser.parse_args(list(argv)) + + return archive_release(repository_root(), current_platform(), label=arguments.label) + + if __name__ == "__main__": raise SystemExit(main(sys.argv[1:])) diff --git a/scripts/bootstrap/__init__.py b/scripts/bootstrap/__init__.py index e69de29bb..e29480584 100644 --- a/scripts/bootstrap/__init__.py +++ b/scripts/bootstrap/__init__.py @@ -0,0 +1,3 @@ +from bootstrap.interpreter import REQUIRED_VERSION, require_python + +require_python(REQUIRED_VERSION) diff --git a/scripts/bootstrap/cuda.py b/scripts/bootstrap/cuda.py index 309469fd3..ec43e3868 100644 --- a/scripts/bootstrap/cuda.py +++ b/scripts/bootstrap/cuda.py @@ -17,19 +17,13 @@ @dataclass(frozen=True, kw_only=True) class CudaDetection: - """The NVIDIA driver capability observed on the host and the CuPy extra it maps to. + """The CuPy extra the host's NVIDIA driver maps to, and what was found to choose it. Attributes: - system: The system the detection ran on. - nvidia_smi: The driver's ``nvidia-smi``, where one was found. - cuda_version: The newest CUDA version the driver supports, as ``(major, minor)``. extra: The optional-dependency extra matching the driver, or ``None`` for the CPU backend. reason: One line saying what was found and what it selects. """ - system: str - nvidia_smi: Optional[Path] - cuda_version: Optional[Tuple[int, int]] extra: Optional[str] reason: str @@ -133,31 +127,16 @@ def detect(platform: Platform, environment: Mapping[str, str]) -> CudaDetection: Returns: CudaDetection: What was found and the extra it selects. """ - if not platform.cuda: - return CudaDetection( - system=platform.name, - nvidia_smi=None, - cuda_version=None, - extra=None, - reason=f"NVIDIA CUDA runs on Linux and Windows; on {platform.name}, keeping the CPU (NumPy) backend.", - ) + if platform.cpu_backend_reason is not None: + return CudaDetection(extra=None, reason=platform.cpu_backend_reason) nvidia_smi = find_nvidia_smi(platform, environment) if nvidia_smi is None: return CudaDetection( - system=platform.name, - nvidia_smi=None, - cuda_version=None, extra=None, reason="No NVIDIA driver detected (nvidia-smi is absent); keeping the CPU (NumPy) backend.", ) cuda_version = query_driver_cuda_version(nvidia_smi) extra = select_extra(cuda_version) - return CudaDetection( - system=platform.name, - nvidia_smi=nvidia_smi, - cuda_version=cuda_version, - extra=extra, - reason=_describe(cuda_version=cuda_version, extra=extra), - ) + return CudaDetection(extra=extra, reason=_describe(cuda_version=cuda_version, extra=extra)) diff --git a/scripts/bootstrap/interpreter.py b/scripts/bootstrap/interpreter.py index 902800d2f..f1ae54587 100644 --- a/scripts/bootstrap/interpreter.py +++ b/scripts/bootstrap/interpreter.py @@ -8,15 +8,16 @@ def require_python(version: Tuple[int, int]) -> None: """Holds the interpreter running the script to ``version`` or newer. + Importing ``bootstrap`` runs it first, so an older interpreter is told the version it needs + before a script reaches the standard library that version brings. + Args: - version: The oldest major and minor version the operation runs on. + version: The oldest major and minor version the scripts run on. Raises: SystemExit: If the interpreter is older, naming where a newer one is downloaded. """ - running = sys.version_info - if (running.major, running.minor) >= version: - print(f"Detected Python version: {running.major}.{running.minor}.{running.micro}") + if tuple(sys.version_info[:2]) >= version: return major, minor = version @@ -24,3 +25,8 @@ def require_python(version: Tuple[int, int]) -> None: f"ERROR: Python {major}.{minor} or newer is required.\n" f"Please install Python {major}.{minor}+ from {DOWNLOADS}" ) + + +def running_version() -> str: + """The version of the interpreter running the script, as ``major.minor.micro``.""" + return ".".join(str(part) for part in sys.version_info[:3]) diff --git a/scripts/bootstrap/layout.py b/scripts/bootstrap/layout.py index 30d91ac04..68f1acf1e 100644 --- a/scripts/bootstrap/layout.py +++ b/scripts/bootstrap/layout.py @@ -4,13 +4,14 @@ REPOSITORY_ROOT: Final[Path] = Path(__file__).resolve().parents[2] PROJECT_FILE: Final[str] = "pyproject.toml" SOURCE_DIRECTORY: Final[str] = "src" +BENCHMARKS_DIRECTORY: Final[str] = "tests/benchmarks" DISTRIBUTION: Final[str] = "bin" BUNDLES: Final[str] = "bundles" BUILD_ENVIRONMENT: Final[str] = ".venv-build" RELEASE_HOOK: Final[str] = "scripts/runtime_hooks/release_environment.py" NOTICES: Final[Tuple[str, ...]] = ("LICENSE", "THIRD-PARTY-NOTICES.md", "THIRD-PARTY-LICENSES.txt") BUILD_TOOLS: Final[Tuple[str, ...]] = ("PIL",) -CLEAN_ARTIFACTS: Final[Tuple[str, ...]] = (DISTRIBUTION, "build", "dist", "htmlcov", ".coverage") +CLEAN_ARTIFACTS: Final[Tuple[str, ...]] = (DISTRIBUTION, BUNDLES, "build", "dist", "htmlcov", ".coverage") CLEAN_PATTERNS: Final[Tuple[str, ...]] = ("*.spec",) CACHE_DIRECTORIES: Final[Tuple[str, ...]] = ("__pycache__",) CACHE_DIRECTORY_SUFFIXES: Final[Tuple[str, ...]] = (".egg-info",) diff --git a/scripts/bootstrap/passes.py b/scripts/bootstrap/passes.py index 4fe91b1e5..ef21e7e2c 100644 --- a/scripts/bootstrap/passes.py +++ b/scripts/bootstrap/passes.py @@ -7,7 +7,7 @@ @dataclass(frozen=True) class Pass: - """One step of a run that reports every failure at once: named, announced, run as one command. + """One named command a script announces and runs from the repository root. Attributes: name: What the step is called in the report and on the command line. @@ -20,6 +20,28 @@ class Pass: command: Tuple[str, ...] +def run_pass( + current: Pass, + *, + root: Path, + runner: Runner, + environment: Mapping[str, str], +) -> int: + """Announces one pass and runs its command from the repository. + + Args: + current: The pass. + root: The repository, which the command runs in. + runner: What runs the command. + environment: The variables the command sees. + + Returns: + int: The status the command exited with. + """ + print(current.announcement) + return runner(current.command, cwd=root, environment=environment, quiet=False) + + def run_passes( passes: Sequence[Pass], *, @@ -43,9 +65,7 @@ def run_passes( """ failed: List[str] = [] for current in passes: - print(current.announcement) - status = runner(current.command, cwd=root, environment=environment, quiet=False) - if status != 0: + if run_pass(current, root=root, runner=runner, environment=environment) != 0: failed.append(current.name) return failed diff --git a/scripts/bootstrap/platforms/linux.py b/scripts/bootstrap/platforms/linux.py index 324b07f2f..2c0691ac2 100644 --- a/scripts/bootstrap/platforms/linux.py +++ b/scripts/bootstrap/platforms/linux.py @@ -2,9 +2,9 @@ from typing import Dict, Final, Mapping, Optional, Sequence, Tuple from bootstrap.platforms.bundling import Bundling +from bootstrap.platforms.unix import unix_interpreter LINUX: Final[str] = "Linux" -POSIX_INTERPRETER: Final[Tuple[str, str]] = ("bin", "python") PNG_ICON: Final[str] = "src/sampletones_assets/icons/sampletones.png" SYSTEM_PACKAGES: Final[Tuple[str, ...]] = ( "libportaudio2", @@ -39,24 +39,19 @@ ) -def posix_interpreter(environment: Path) -> Path: - """The interpreter a POSIX virtual environment at ``environment`` runs.""" - return environment.joinpath(*POSIX_INTERPRETER) - - class Linux: - """A Debian-based Linux: packages through apt, a launcher without an extension.""" + """A Debian-based Linux: packages through apt, and a launcher named after the project alone.""" @property def name(self) -> str: return LINUX @property - def cuda(self) -> bool: - return True + def cpu_backend_reason(self) -> Optional[str]: + return None def interpreter(self, environment: Path) -> Path: - return posix_interpreter(environment) + return unix_interpreter(environment) def bundling(self) -> Bundling: return BUNDLING diff --git a/scripts/bootstrap/platforms/macos.py b/scripts/bootstrap/platforms/macos.py index 2c3b02474..bd6d64c95 100644 --- a/scripts/bootstrap/platforms/macos.py +++ b/scripts/bootstrap/platforms/macos.py @@ -4,7 +4,7 @@ from typing import Dict, Final, Mapping, Optional, Sequence from bootstrap.platforms.bundling import Bundling -from bootstrap.platforms.linux import posix_interpreter +from bootstrap.platforms.unix import unix_interpreter DARWIN: Final[str] = "Darwin" HOMEBREW: Final[str] = "brew" @@ -21,36 +21,15 @@ "\n" "See docs/guide/installation.md for the full steps." ) - - -def homebrew_prefix(package: str) -> str: - """Where Homebrew installed ``package``, or empty where Homebrew or the package is absent.""" - if shutil.which(HOMEBREW) is None: - return "" - - completed = subprocess.run( - [HOMEBREW, "--prefix", package], - capture_output=True, - text=True, - check=False, - ) - if completed.returncode != 0: - return "" - - return completed.stdout.strip() - - -def architecture_flag(machine: str) -> str: - """The flag compiling an extension for the machine's own architecture alone.""" - return f"-arch {machine}" +CPU_BACKEND: Final[str] = "NVIDIA CUDA is available on Linux and Windows; on macOS, keeping the CPU (NumPy) backend." class MacOS: """macOS: PortAudio through Homebrew, and the application run from source. Homebrew's PortAudio carries the machine's own architecture while a python.org interpreter - compiles for both, so the setup and a build pin ``ARCHFLAGS`` to the native one. NVIDIA CUDA - has no driver on the system, so the CPU backend is the one it runs. + compiles for both, so the setup and a build pin ``ARCHFLAGS`` to the native one. The CPU backend + is the one it runs, as NVIDIA ships CUDA for Linux and Windows. """ @property @@ -58,11 +37,33 @@ def name(self) -> str: return DARWIN @property - def cuda(self) -> bool: - return False + def cpu_backend_reason(self) -> Optional[str]: + return CPU_BACKEND + + @staticmethod + def homebrew_prefix(package: str) -> str: + """Where Homebrew installed ``package``, or empty where Homebrew or the package is absent.""" + if shutil.which(HOMEBREW) is None: + return "" + + completed = subprocess.run( + [HOMEBREW, "--prefix", package], + capture_output=True, + text=True, + check=False, + ) + if completed.returncode != 0: + return "" + + return completed.stdout.strip() + + @staticmethod + def architecture_flag(machine: str) -> str: + """The flag compiling an extension for the machine's own architecture alone.""" + return f"-arch {machine}" def interpreter(self, environment: Path) -> Path: - return posix_interpreter(environment) + return unix_interpreter(environment) def bundling(self) -> Bundling: raise SystemExit(NO_BUNDLE) @@ -80,7 +81,7 @@ def system_packages(self) -> Sequence[Sequence[str]]: return ((HOMEBREW, "install", PORTAUDIO),) def setup_variables(self, base: Mapping[str, str], *, machine: str) -> Dict[str, str]: - return {**base, ARCHFLAGS: architecture_flag(machine)} + return {**base, ARCHFLAGS: self.architecture_flag(machine)} def build_flags(self, *, machine: str) -> Sequence[str]: """The flags compiling audio playback against Homebrew's PortAudio on the native architecture. @@ -88,7 +89,7 @@ def build_flags(self, *, machine: str) -> Sequence[str]: Raises: SystemExit: If Homebrew reports no PortAudio prefix. """ - prefix = homebrew_prefix(PORTAUDIO) + prefix = self.homebrew_prefix(PORTAUDIO) if not prefix: raise SystemExit( "ERROR: Homebrew is required to locate the PortAudio headers and library.\n" @@ -98,7 +99,7 @@ def build_flags(self, *, machine: str) -> Sequence[str]: return ( f"CFLAGS=-I{prefix}/include", f"LDFLAGS=-L{prefix}/lib", - f"{ARCHFLAGS}={architecture_flag(machine)}", + f"{ARCHFLAGS}={self.architecture_flag(machine)}", ) def nvidia_smi_locations(self, environment: Mapping[str, str]) -> Sequence[Path]: diff --git a/scripts/bootstrap/platforms/protocol.py b/scripts/bootstrap/platforms/protocol.py index 469e5d880..e5bfdf9c0 100644 --- a/scripts/bootstrap/platforms/protocol.py +++ b/scripts/bootstrap/platforms/protocol.py @@ -12,8 +12,8 @@ def name(self) -> str: """The name ``platform.system()`` reports for the system.""" @property - def cuda(self) -> bool: - """Whether an NVIDIA driver with CUDA can run on the system.""" + def cpu_backend_reason(self) -> Optional[str]: + """Why the system runs the CPU backend whatever its hardware, or ``None`` where CUDA can run.""" def interpreter(self, environment: Path) -> Path: """The interpreter a virtual environment at ``environment`` runs. diff --git a/scripts/bootstrap/platforms/unix.py b/scripts/bootstrap/platforms/unix.py new file mode 100644 index 000000000..b6e97c559 --- /dev/null +++ b/scripts/bootstrap/platforms/unix.py @@ -0,0 +1,9 @@ +from pathlib import Path +from typing import Final, Tuple + +UNIX_INTERPRETER: Final[Tuple[str, str]] = ("bin", "python") + + +def unix_interpreter(environment: Path) -> Path: + """The interpreter a virtual environment at ``environment`` runs on Linux and macOS.""" + return environment.joinpath(*UNIX_INTERPRETER) diff --git a/scripts/bootstrap/platforms/windows.py b/scripts/bootstrap/platforms/windows.py index 41e8d7382..9f67f0fba 100644 --- a/scripts/bootstrap/platforms/windows.py +++ b/scripts/bootstrap/platforms/windows.py @@ -34,8 +34,8 @@ def name(self) -> str: return WINDOWS @property - def cuda(self) -> bool: - return True + def cpu_backend_reason(self) -> Optional[str]: + return None def interpreter(self, environment: Path) -> Path: return environment.joinpath(*WINDOWS_INTERPRETER) diff --git a/scripts/bootstrap/project.py b/scripts/bootstrap/project.py index bc8a6ef02..2428c7cbc 100644 --- a/scripts/bootstrap/project.py +++ b/scripts/bootstrap/project.py @@ -14,6 +14,7 @@ ENTRY_SEPARATOR: Final[str] = ":" MODULE_SEPARATOR: Final[str] = "." MODULE_SUFFIX: Final[str] = ".py" +TAG_PREFIX: Final[str] = "v" @dataclass(frozen=True) @@ -41,6 +42,11 @@ def entry_script(self) -> str: """The entry module as a path from the repository root, which PyInstaller starts from.""" return f"{SOURCE_DIRECTORY}/{self.entry_module.replace(MODULE_SEPARATOR, '/')}{MODULE_SUFFIX}" + @property + def tag(self) -> str: + """The tag a release of this version is published under, such as ``v0.3.0``.""" + return f"{TAG_PREFIX}{self.version}" + def parse_project(document: Dict[str, Any]) -> Project: """The project a parsed ``pyproject.toml`` states, held to the names the scripts rely on. diff --git a/scripts/bundle.py b/scripts/bundle.py index 0cb5f4471..88b2a94a0 100644 --- a/scripts/bundle.py +++ b/scripts/bundle.py @@ -7,7 +7,7 @@ from typing import Final, List, Mapping, Sequence, Tuple from bootstrap.files import remove_path -from bootstrap.interpreter import REQUIRED_VERSION, require_python +from bootstrap.interpreter import running_version from bootstrap.layout import BUILD_TOOLS, DISTRIBUTION, NOTICES, RELEASE_HOOK, repository_root from bootstrap.platforms.bundling import Bundling from bootstrap.platforms.factory import current_platform @@ -197,7 +197,7 @@ def main(argv: Sequence[str]) -> int: arguments = parser.parse_args(list(argv)) options = BundleOptions(release=arguments.release, gpu=arguments.gpu) - require_python(REQUIRED_VERSION) + print(f"Detected Python version: {running_version()}") build_bundle( repository_root(), current_platform(), diff --git a/scripts/formatting.py b/scripts/formatting.py index 410d7f8a1..82407162c 100644 --- a/scripts/formatting.py +++ b/scripts/formatting.py @@ -4,10 +4,10 @@ from pathlib import Path from typing import Final, Mapping, Sequence, Tuple -from bootstrap.layout import repository_root +from bootstrap.layout import SOURCE_DIRECTORY, repository_root from bootstrap.processes import Runner, expect_success, run -FORMATTED_TREES: Final[Tuple[str, ...]] = ("src", "tests", "scripts") +FORMATTED_TREES: Final[Tuple[str, ...]] = (SOURCE_DIRECTORY, "tests", "scripts") ISORT: Final[Tuple[str, ...]] = ("uv", "run", "python", "-m", "isort") BLACK: Final[Tuple[str, ...]] = ("uv", "run", "python", "-m", "black") diff --git a/scripts/lint.py b/scripts/lint.py index ab5a6be06..55061cffd 100644 --- a/scripts/lint.py +++ b/scripts/lint.py @@ -4,13 +4,13 @@ from pathlib import Path from typing import Final, Mapping, Sequence, Set, Tuple -from bootstrap.layout import repository_root +from bootstrap.layout import SOURCE_DIRECTORY, repository_root from bootstrap.passes import Pass, run_passes from bootstrap.processes import Runner, run MYPY: Final[str] = "mypy" PYLINT: Final[str] = "pylint" -LINTED_TREES: Final[Tuple[str, ...]] = ("src", "scripts") +LINTED_TREES: Final[Tuple[str, ...]] = (SOURCE_DIRECTORY, "scripts") def linters(paths: Sequence[str]) -> Tuple[Pass, ...]: diff --git a/scripts/run_tests.py b/scripts/run_tests.py index 6da4bc94e..f1d213741 100644 --- a/scripts/run_tests.py +++ b/scripts/run_tests.py @@ -1,12 +1,11 @@ import argparse import os import sys -from pathlib import Path -from typing import Dict, Final, Mapping, Sequence, Tuple +from typing import Dict, Final, Sequence, Tuple -from bootstrap.layout import repository_root -from bootstrap.passes import Pass -from bootstrap.processes import Runner, run +from bootstrap.layout import BENCHMARKS_DIRECTORY, SOURCE_DIRECTORY, repository_root +from bootstrap.passes import Pass, run_pass +from bootstrap.processes import run SUITE: Final[str] = "suite" DOCTESTS: Final[str] = "doctests" @@ -33,48 +32,26 @@ def planned_passes(workers: str) -> Dict[str, Pass]: Pass( SUITE, "Running pytest with coverage...", - (*PYTEST, "-n", workers, "--cov", "--ignore=tests/benchmarks"), + (*PYTEST, "-n", workers, "--cov", f"--ignore={BENCHMARKS_DIRECTORY}"), ), Pass( DOCTESTS, "Running doctests...", - (*PYTEST, "src/", "--doctest-modules", "--no-cov"), + (*PYTEST, SOURCE_DIRECTORY, "--doctest-modules", "--no-cov"), ), Pass( BENCHMARKS, "Running benchmarks...", - (*PYTEST, "tests/benchmarks", "--no-cov", "-s"), + (*PYTEST, BENCHMARKS_DIRECTORY, "--no-cov", "-s"), ), ) return {current.name: current for current in passes} -def run_pass( - chosen: Pass, - root: Path, - *, - runner: Runner, - environment: Mapping[str, str], -) -> int: - """Announces one pass and runs it from the repository root. - - Args: - chosen: The pass. - root: The repository. - runner: What runs the pass. - environment: The variables pytest sees. - - Returns: - int: The status pytest exited with. - """ - print(chosen.announcement) - return runner(chosen.command, cwd=root, environment=environment, quiet=False) - - def main(argv: Sequence[str]) -> int: """Runs one pass of the tests and exits with the status pytest gave it.""" parser = argparse.ArgumentParser(description="Run one pass of the SampleToNES tests.") - parser.add_argument("name", choices=(SUITE, DOCTESTS, BENCHMARKS), help="the pass to run") + parser.add_argument("name", choices=tuple(planned_passes(DEFAULT_WORKERS)), help="the pass to run") parser.add_argument( "--workers", default=DEFAULT_WORKERS, @@ -84,7 +61,7 @@ def main(argv: Sequence[str]) -> int: return run_pass( planned_passes(arguments.workers)[arguments.name], - repository_root(), + root=repository_root(), runner=run, environment=os.environ, ) diff --git a/scripts/verify_bundle.py b/scripts/verify_bundle.py index d60e25cbe..daeb751d9 100644 --- a/scripts/verify_bundle.py +++ b/scripts/verify_bundle.py @@ -39,7 +39,7 @@ def bundle_failures( ) -> List[str]: """What keeps the release bundle under ``root`` from shipping, each as a line of its own. - The bundle ships its notices, carries none of the build-time tooling they leave out, and + The bundle ships its notices, carries the application and its runtime dependencies alone, and offers a launcher that starts. Args: diff --git a/scripts/verify_version_tag.py b/scripts/verify_version_tag.py index 513848b66..ac1599f25 100644 --- a/scripts/verify_version_tag.py +++ b/scripts/verify_version_tag.py @@ -1,11 +1,9 @@ import argparse import sys -from typing import Final, Sequence +from typing import Optional, Sequence from bootstrap.layout import repository_root -from bootstrap.project import read_project - -TAG_PREFIX: Final[str] = "v" +from bootstrap.project import TAG_PREFIX, Project, read_project def version_from_tag(tag: str) -> str: @@ -13,6 +11,22 @@ def version_from_tag(tag: str) -> str: return tag.removeprefix(TAG_PREFIX) +def tag_failure(tag: str, project: Project) -> Optional[str]: + """What keeps a release tag from publishing the project, or ``None`` for a tag naming its version. + + Args: + tag: The release tag being built, such as ``v0.3.0``. + project: The project the tag publishes. + + Returns: + Optional[str]: The failure as one line, or ``None``. + """ + if version_from_tag(tag) == project.version: + return None + + return f"Tag {tag} names a version other than the project version {project.version}" + + def main(argv: Sequence[str]) -> int: """Confirms a release tag names the version ``pyproject.toml`` records.""" parser = argparse.ArgumentParser(description="Compare a release tag against the project version.") @@ -20,12 +34,13 @@ def main(argv: Sequence[str]) -> int: arguments = parser.parse_args(list(argv)) tag: str = arguments.tag - version = read_project(repository_root()).version - if version_from_tag(tag) != version: - print(f"::error::Tag {tag} names a version other than the project version {version}") + project = read_project(repository_root()) + failure = tag_failure(tag, project) + if failure is not None: + print(f"::error::{failure}") return 1 - print(f"Version {version} matches tag {tag}") + print(f"Version {project.version} matches tag {tag}") return 0 diff --git a/tests/suite/bootstrap.py b/tests/suite/bootstrap.py index 0c17d4b38..bb7ff5258 100644 --- a/tests/suite/bootstrap.py +++ b/tests/suite/bootstrap.py @@ -2,6 +2,9 @@ from pathlib import Path from typing import Callable, Dict, Final, List, Mapping, Optional, Sequence, Tuple +from bootstrap.layout import PROJECT_FILE +from bootstrap.project import BUILD_EXTRA, DEVELOPMENT_GROUP, GPU_CUDA11_EXTRA, GPU_EXTRA + PROJECT_NAME: Final[str] = "sampletones" PROJECT_VERSION: Final[str] = "0.3.0" PROJECT_DOCUMENT: Final[str] = f""" @@ -13,12 +16,12 @@ {PROJECT_NAME} = "{PROJECT_NAME}.__main__:main" [project.optional-dependencies] -build = ["pyinstaller"] -gpu = ["cupy-cuda12x"] -gpu-cuda11 = ["cupy-cuda11x"] +{BUILD_EXTRA} = ["pyinstaller"] +{GPU_EXTRA} = ["cupy-cuda12x"] +{GPU_CUDA11_EXTRA} = ["cupy-cuda11x"] [dependency-groups] -dev = ["pytest"] +{DEVELOPMENT_GROUP} = ["pytest"] [tool.hatch.build.targets.wheel] packages = ["src/{PROJECT_NAME}", "src/{PROJECT_NAME}_core"] @@ -97,5 +100,5 @@ def lines(self) -> List[str]: def write_project(root: Path) -> Path: """Writes a ``pyproject.toml`` under ``root`` stating what the scripts read, and answers with ``root``.""" root.mkdir(parents=True, exist_ok=True) - (root / "pyproject.toml").write_text(PROJECT_DOCUMENT, encoding="utf-8") + (root / PROJECT_FILE).write_text(PROJECT_DOCUMENT, encoding="utf-8") return root diff --git a/tests/unit/scripts/bootstrap/platforms/test_linux.py b/tests/unit/scripts/bootstrap/platforms/test_linux.py index cc237c4d9..c7af9df0d 100644 --- a/tests/unit/scripts/bootstrap/platforms/test_linux.py +++ b/tests/unit/scripts/bootstrap/platforms/test_linux.py @@ -13,7 +13,7 @@ def test_a_bundle_launcher_carries_no_extension(self) -> None: def test_apt_is_updated_before_the_packages_are_installed(self) -> None: commands = Linux().system_packages() - assert [command[:2] for command in commands] == [("sudo", "apt-get"), ("sudo", "apt-get")] + assert [command[:3] for command in commands] == [("sudo", "apt-get", "update"), ("sudo", "apt-get", "install")] assert "portaudio19-dev" in commands[1] assert "python3-tk" in commands[1] assert Linux().missing_package_manager() is None @@ -23,5 +23,5 @@ def test_the_setup_and_the_build_take_the_variables_as_given(self) -> None: assert Linux().build_flags(machine="x86_64") == () def test_the_driver_is_looked_up_on_the_path_alone(self) -> None: - assert Linux().cuda + assert Linux().cpu_backend_reason is None assert Linux().nvidia_smi_locations({"SystemRoot": "C:/Windows"}) == () diff --git a/tests/unit/scripts/bootstrap/platforms/test_macos.py b/tests/unit/scripts/bootstrap/platforms/test_macos.py index fddbb26ad..4a2ea00ae 100644 --- a/tests/unit/scripts/bootstrap/platforms/test_macos.py +++ b/tests/unit/scripts/bootstrap/platforms/test_macos.py @@ -3,7 +3,7 @@ import pytest from bootstrap.platforms import macos -from bootstrap.platforms.macos import ARCHFLAGS, HOMEBREW, HOMEBREW_SITE, PORTAUDIO, MacOS +from bootstrap.platforms.macos import ARCHFLAGS, CPU_BACKEND, HOMEBREW, HOMEBREW_SITE, PORTAUDIO, MacOS PORTAUDIO_PREFIX = "/opt/homebrew/opt/portaudio" @@ -38,7 +38,7 @@ def test_the_setup_pins_the_native_architecture(self) -> None: assert variables == {"PATH": "/usr/bin", ARCHFLAGS: "-arch arm64"} def test_a_build_compiles_against_homebrew_s_portaudio(self, monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr(macos, "homebrew_prefix", lambda package: PORTAUDIO_PREFIX) + monkeypatch.setattr(MacOS, "homebrew_prefix", staticmethod(lambda package: PORTAUDIO_PREFIX)) assert MacOS().build_flags(machine="arm64") == ( f"CFLAGS=-I{PORTAUDIO_PREFIX}/include", @@ -47,13 +47,13 @@ def test_a_build_compiles_against_homebrew_s_portaudio(self, monkeypatch: pytest ) def test_a_build_without_homebrew_s_portaudio_is_refused(self, monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr(macos, "homebrew_prefix", lambda package: "") + monkeypatch.setattr(MacOS, "homebrew_prefix", staticmethod(lambda package: "")) with pytest.raises(SystemExit, match="Homebrew"): MacOS().build_flags(machine="arm64") def test_the_cpu_backend_is_the_one_it_runs(self) -> None: - assert not MacOS().cuda + assert MacOS().cpu_backend_reason == CPU_BACKEND assert MacOS().nvidia_smi_locations({}) == () @@ -61,4 +61,4 @@ class TestHomebrewPrefix: def test_without_homebrew_the_prefix_is_empty(self, monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(macos.shutil, "which", lambda name: None) - assert macos.homebrew_prefix(PORTAUDIO) == "" + assert MacOS.homebrew_prefix(PORTAUDIO) == "" diff --git a/tests/unit/scripts/bootstrap/platforms/test_windows.py b/tests/unit/scripts/bootstrap/platforms/test_windows.py index 6ddc30017..77fc8fb3a 100644 --- a/tests/unit/scripts/bootstrap/platforms/test_windows.py +++ b/tests/unit/scripts/bootstrap/platforms/test_windows.py @@ -23,5 +23,5 @@ def test_the_setup_and_the_build_take_the_variables_as_given(self) -> None: def test_the_driver_s_fixed_locations_are_read_from_the_variables_that_are_set(self) -> None: locations = Windows().nvidia_smi_locations({"SystemRoot": "C:/Windows"}) - assert Windows().cuda + assert Windows().cpu_backend_reason is None assert locations == (Path("C:/Windows", "System32", "nvidia-smi.exe"),) diff --git a/tests/unit/scripts/bootstrap/test_cuda.py b/tests/unit/scripts/bootstrap/test_cuda.py index 998f09dbb..cb565bfa9 100644 --- a/tests/unit/scripts/bootstrap/test_cuda.py +++ b/tests/unit/scripts/bootstrap/test_cuda.py @@ -211,7 +211,7 @@ def test_macos_keeps_cpu(self) -> None: detection = cuda.detect(MacOS(), {}) assert detection.extra is None - assert detection.cuda_version is None + assert detection.reason == MacOS().cpu_backend_reason def test_no_driver_keeps_cpu(self, monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(cuda.shutil, "which", lambda name: None) @@ -219,7 +219,7 @@ def test_no_driver_keeps_cpu(self, monkeypatch: pytest.MonkeyPatch) -> None: detection = cuda.detect(Linux(), {}) assert detection.extra is None - assert detection.nvidia_smi is None + assert cuda.NVIDIA_SMI in detection.reason def test_selects_gpu_for_cuda12_driver(self, monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(cuda.shutil, "which", lambda name: "/usr/bin/nvidia-smi") @@ -227,5 +227,5 @@ def test_selects_gpu_for_cuda12_driver(self, monkeypatch: pytest.MonkeyPatch) -> detection = cuda.detect(Linux(), {}) - assert detection.cuda_version == (12, 4) assert detection.extra == GPU_EXTRA + assert "12.4" in detection.reason diff --git a/tests/unit/scripts/bootstrap/test_interpreter.py b/tests/unit/scripts/bootstrap/test_interpreter.py index d9b698abc..687dd7f65 100644 --- a/tests/unit/scripts/bootstrap/test_interpreter.py +++ b/tests/unit/scripts/bootstrap/test_interpreter.py @@ -1,16 +1,42 @@ +import subprocess +import sys +from typing import Final + import pytest -from bootstrap.interpreter import DOWNLOADS, require_python +from bootstrap.interpreter import DOWNLOADS, REQUIRED_VERSION, require_python, running_version +from sampletones_tools.checks.paths import SCRIPTS_ROOT + +OLDER_INTERPRETER: Final[str] = "(3, 10, 0, 'final', 0)" class TestRequirePython: - def test_an_interpreter_at_least_the_version_passes(self, capsys: pytest.CaptureFixture[str]) -> None: + def test_an_interpreter_at_least_the_version_passes(self) -> None: require_python((3, 8)) - assert "Detected Python version" in capsys.readouterr().out - def test_an_older_interpreter_is_refused_with_the_download_site(self) -> None: with pytest.raises(SystemExit) as refused: require_python((99, 0)) assert DOWNLOADS in str(refused.value) + + def test_the_running_version_reads_major_minor_and_micro(self) -> None: + assert running_version() == ".".join(str(part) for part in sys.version_info[:3]) + + +class TestBootstrapGate: + def test_importing_a_helper_on_an_older_interpreter_names_the_required_version(self) -> None: + probe = f"import sys; sys.version_info = {OLDER_INTERPRETER}; import bootstrap.project" + + completed = subprocess.run( + [sys.executable, "-c", probe], + cwd=SCRIPTS_ROOT, + capture_output=True, + text=True, + check=False, + ) + + major, minor = REQUIRED_VERSION + assert completed.returncode == 1 + assert f"Python {major}.{minor} or newer is required" in completed.stderr + assert DOWNLOADS in completed.stderr diff --git a/tests/unit/scripts/bootstrap/test_passes.py b/tests/unit/scripts/bootstrap/test_passes.py index 43f5fe507..0703b4e07 100644 --- a/tests/unit/scripts/bootstrap/test_passes.py +++ b/tests/unit/scripts/bootstrap/test_passes.py @@ -2,7 +2,7 @@ import pytest -from bootstrap.passes import Pass, run_passes +from bootstrap.passes import Pass, run_pass, run_passes from tests.suite.bootstrap import RecordingRunner PASSES = ( @@ -12,6 +12,23 @@ ) +class TestRunPass: + def test_the_pass_is_announced_and_run_from_the_repository( + self, + capsys: pytest.CaptureFixture[str], + tmp_path: Path, + ) -> None: + runner = RecordingRunner({}, None) + + assert run_pass(PASSES[0], root=tmp_path, runner=runner, environment={}) == 0 + assert runner.lines == ["first command"] + assert runner.commands[0].cwd == tmp_path + assert capsys.readouterr().out == "First...\n" + + def test_the_status_is_the_command_s_own(self, tmp_path: Path) -> None: + assert run_pass(PASSES[1], root=tmp_path, runner=RecordingRunner({"second": 5}, None), environment={}) == 5 + + class TestRunPasses: def test_every_pass_runs_and_the_failed_ones_are_named(self, capsys: pytest.CaptureFixture[str]) -> None: runner = RecordingRunner({"second": 1}, None) diff --git a/tests/unit/scripts/bootstrap/test_project.py b/tests/unit/scripts/bootstrap/test_project.py index 6ef73c8ad..482df6f7c 100644 --- a/tests/unit/scripts/bootstrap/test_project.py +++ b/tests/unit/scripts/bootstrap/test_project.py @@ -3,8 +3,16 @@ import pytest -from bootstrap.layout import repository_root -from bootstrap.project import NAMED_EXTRAS, NAMED_GROUPS, parse_project, read_project +from bootstrap.layout import SOURCE_DIRECTORY, repository_root +from bootstrap.project import ( + DEVELOPMENT_GROUP, + GPU_CUDA11_EXTRA, + NAMED_EXTRAS, + NAMED_GROUPS, + TAG_PREFIX, + parse_project, + read_project, +) from tests.suite.bootstrap import PROJECT_DOCUMENT, PROJECT_NAME, PROJECT_VERSION, write_project @@ -15,7 +23,7 @@ def test_the_repository_states_every_name_the_scripts_install(self) -> None: assert set(NAMED_EXTRAS) <= set(project.extras) assert set(NAMED_GROUPS) <= set(project.groups) assert (repository_root() / project.entry_script).is_file() - assert all((repository_root() / "src" / package).is_dir() for package in project.packages) + assert all((repository_root() / SOURCE_DIRECTORY / package).is_dir() for package in project.packages) def test_the_facts_are_read_from_the_file(self, tmp_path: Path) -> None: project = read_project(write_project(tmp_path)) @@ -23,23 +31,24 @@ def test_the_facts_are_read_from_the_file(self, tmp_path: Path) -> None: assert project.name == PROJECT_NAME assert project.version == PROJECT_VERSION assert project.entry_module == f"{PROJECT_NAME}.__main__" - assert project.entry_script == f"src/{PROJECT_NAME}/__main__.py" + assert project.entry_script == f"{SOURCE_DIRECTORY}/{PROJECT_NAME}/__main__.py" assert project.packages == (PROJECT_NAME, f"{PROJECT_NAME}_core") + assert project.tag == f"{TAG_PREFIX}{PROJECT_VERSION}" class TestParseProject: def test_a_missing_extra_is_refused_by_name(self) -> None: document = tomllib.loads(PROJECT_DOCUMENT) - del document["project"]["optional-dependencies"]["gpu-cuda11"] + del document["project"]["optional-dependencies"][GPU_CUDA11_EXTRA] - with pytest.raises(SystemExit, match="gpu-cuda11"): + with pytest.raises(SystemExit, match=GPU_CUDA11_EXTRA): parse_project(document) def test_a_missing_group_is_refused_by_name(self) -> None: document = tomllib.loads(PROJECT_DOCUMENT) - del document["dependency-groups"]["dev"] + del document["dependency-groups"][DEVELOPMENT_GROUP] - with pytest.raises(SystemExit, match="dev"): + with pytest.raises(SystemExit, match=DEVELOPMENT_GROUP): parse_project(document) def test_a_project_without_its_command_is_refused(self) -> None: diff --git a/tests/unit/scripts/test_archive_bundle.py b/tests/unit/scripts/test_archive_bundle.py index 1d85dafc8..47bfd7a46 100644 --- a/tests/unit/scripts/test_archive_bundle.py +++ b/tests/unit/scripts/test_archive_bundle.py @@ -5,14 +5,16 @@ import pytest from bootstrap.layout import BUNDLES, DISTRIBUTION -from bootstrap.project import read_project +from bootstrap.platforms.macos import MacOS +from bootstrap.platforms.windows import Windows +from bootstrap.project import TAG_PREFIX, read_project from tests.suite.bootstrap import PROJECT_NAME, PROJECT_VERSION, write_project from tests.suite.scripts import load_script archive_bundle = load_script("archive_bundle.py") LABEL = "windows-x86_64" -ROOT = f"{PROJECT_NAME}-v{PROJECT_VERSION}-{LABEL}" +ROOT = f"{PROJECT_NAME}-{TAG_PREFIX}{PROJECT_VERSION}-{LABEL}" LAUNCHER = "sampletones.exe" LIBRARY = "_internal/python312.dll" @@ -105,30 +107,27 @@ def test_the_root_names_the_project_its_version_and_the_platform(self, tmp_path: assert archive_bundle.archive_root(read_project(write_project(tmp_path)), LABEL) == ROOT -class TestMain: - def test_the_release_bundle_is_archived_into_the_bundles_directory( - self, - monkeypatch: pytest.MonkeyPatch, - tmp_path: Path, - ) -> None: +class TestArchiveRelease: + def test_the_release_bundle_is_archived_into_the_bundles_directory(self, tmp_path: Path) -> None: root = write_project(tmp_path / "repository") - source = root / DISTRIBUTION / PROJECT_NAME - source.mkdir(parents=True) - (source / LAUNCHER).write_bytes(b"MZ") - monkeypatch.setattr(archive_bundle, "repository_root", lambda: root) + launcher = Windows().bundling().launcher(root / DISTRIBUTION, name=PROJECT_NAME, release=True) + launcher.parent.mkdir(parents=True) + launcher.write_bytes(b"MZ") - assert archive_bundle.main(["--label", LABEL]) == 0 - assert _names(root / BUNDLES / f"{ROOT}.zip") == [f"{ROOT}/{LAUNCHER}"] + assert archive_bundle.archive_release(root, Windows(), label=LABEL) == 0 + assert _names(root / BUNDLES / f"{ROOT}.zip") == [f"{ROOT}/{launcher.name}"] def test_a_missing_bundle_is_annotated_and_writes_no_archive( self, - monkeypatch: pytest.MonkeyPatch, tmp_path: Path, capsys: pytest.CaptureFixture[str], ) -> None: root = write_project(tmp_path) - monkeypatch.setattr(archive_bundle, "repository_root", lambda: root) - assert archive_bundle.main(["--label", LABEL]) == 1 + assert archive_bundle.archive_release(root, Windows(), label=LABEL) == 1 assert "::error::" in capsys.readouterr().out assert not (root / BUNDLES).exists() + + def test_a_system_without_bundles_is_refused(self, tmp_path: Path) -> None: + with pytest.raises(SystemExit, match="make setup"): + archive_bundle.archive_release(write_project(tmp_path), MacOS(), label=LABEL) diff --git a/tests/unit/scripts/test_build_environment.py b/tests/unit/scripts/test_build_environment.py index e8766fa57..738d962a1 100644 --- a/tests/unit/scripts/test_build_environment.py +++ b/tests/unit/scripts/test_build_environment.py @@ -1,6 +1,5 @@ import pytest -from bootstrap.platforms import macos from bootstrap.platforms.linux import Linux from bootstrap.platforms.macos import MacOS from tests.suite.scripts import load_script @@ -17,7 +16,7 @@ def test_the_platform_s_flags_are_printed_one_per_line( capsys: pytest.CaptureFixture[str], ) -> None: monkeypatch.setattr(build_environment, "current_platform", MacOS) - monkeypatch.setattr(macos, "homebrew_prefix", lambda package: PORTAUDIO_PREFIX) + monkeypatch.setattr(MacOS, "homebrew_prefix", staticmethod(lambda package: PORTAUDIO_PREFIX)) monkeypatch.setattr(build_environment.running, "machine", lambda: "arm64") assert build_environment.main([]) == 0 diff --git a/tests/unit/scripts/test_run_tests.py b/tests/unit/scripts/test_run_tests.py index 99e47f061..35d9fbd73 100644 --- a/tests/unit/scripts/test_run_tests.py +++ b/tests/unit/scripts/test_run_tests.py @@ -1,11 +1,10 @@ from dataclasses import dataclass -from pathlib import Path from typing import Tuple import pytest +from bootstrap.layout import BENCHMARKS_DIRECTORY from tests.suite.base import BaseTestSuite -from tests.suite.bootstrap import RecordingRunner from tests.suite.case import BaseRegularTestCase from tests.suite.scripts import load_script @@ -22,7 +21,7 @@ def test_every_pass_is_found_under_its_name(self) -> None: def test_the_suite_is_covered_across_the_workers_and_leaves_the_benchmarks_out(self) -> None: command = run_tests.planned_passes("auto")[run_tests.SUITE].command - assert command[-4:] == ("-n", "auto", "--cov", "--ignore=tests/benchmarks") + assert command[-4:] == ("-n", "auto", "--cov", f"--ignore={BENCHMARKS_DIRECTORY}") def test_the_doctests_read_the_sources_uncovered(self) -> None: command = run_tests.planned_passes(run_tests.DEFAULT_WORKERS)[run_tests.DOCTESTS].command @@ -33,27 +32,12 @@ def test_the_doctests_read_the_sources_uncovered(self) -> None: def test_the_benchmarks_run_serial_uncovered_and_show_their_readings(self) -> None: command = run_tests.planned_passes(run_tests.DEFAULT_WORKERS)[run_tests.BENCHMARKS].command - assert "tests/benchmarks" in command + assert BENCHMARKS_DIRECTORY in command assert "--no-cov" in command assert "-s" in command assert "-n" not in command -class TestRunPass: - def test_the_pass_runs_from_the_repository(self, tmp_path: Path) -> None: - runner = RecordingRunner({}, None) - chosen = run_tests.planned_passes(run_tests.DEFAULT_WORKERS)[run_tests.DOCTESTS] - - assert run_tests.run_pass(chosen, tmp_path, runner=runner, environment={}) == 0 - assert runner.lines == [" ".join(chosen.command)] - assert runner.commands[0].cwd == tmp_path - - def test_the_status_is_pytest_s_own(self, tmp_path: Path) -> None: - chosen = run_tests.planned_passes("auto")[run_tests.SUITE] - - assert run_tests.run_pass(chosen, tmp_path, runner=RecordingRunner({"pytest": 5}, None), environment={}) == 5 - - class TestRefusedPasses(BaseTestSuite): @dataclass(frozen=True, kw_only=True) class TestCase(BaseRegularTestCase): diff --git a/tests/unit/scripts/test_setup_environment.py b/tests/unit/scripts/test_setup_environment.py index 63fe93d75..7de7fdf3a 100644 --- a/tests/unit/scripts/test_setup_environment.py +++ b/tests/unit/scripts/test_setup_environment.py @@ -2,7 +2,6 @@ import pytest -from bootstrap.platforms import macos from bootstrap.platforms.linux import Linux from bootstrap.platforms.macos import ARCHFLAGS, MacOS from bootstrap.project import DEVELOPMENT_GROUP, GPU_CUDA11_EXTRA, GPU_EXTRA @@ -53,12 +52,7 @@ def test_a_gpu_extra_reaches_both_installs(self) -> None: class TestSetUpEnvironment: - def test_macos_runs_the_commands_on_its_native_architecture( - self, - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, - ) -> None: - monkeypatch.setattr(macos.shutil, "which", lambda name: None) + def test_macos_runs_the_commands_on_its_native_architecture(self, tmp_path: Path) -> None: runner = RecordingRunner({}, None) setup_environment.set_up_environment( diff --git a/tests/unit/scripts/test_verify_version_tag.py b/tests/unit/scripts/test_verify_version_tag.py index 30ef81774..351a7baa7 100644 --- a/tests/unit/scripts/test_verify_version_tag.py +++ b/tests/unit/scripts/test_verify_version_tag.py @@ -1,10 +1,12 @@ from dataclasses import dataclass +from pathlib import Path import pytest from bootstrap.layout import repository_root -from bootstrap.project import read_project +from bootstrap.project import TAG_PREFIX, read_project from tests.suite.base import BaseTestSuite +from tests.suite.bootstrap import write_project from tests.suite.case import BaseRegularTestCase from tests.suite.scripts import load_script @@ -30,14 +32,31 @@ def test_version_from_tag(self, test_case: TestCase) -> None: assert verify_version_tag.version_from_tag(test_case.tag) == test_case.expected +class TestTagFailure: + def test_the_tag_naming_the_project_version_passes(self, tmp_path: Path) -> None: + project = read_project(write_project(tmp_path)) + + assert verify_version_tag.tag_failure(project.tag, project) is None + + def test_a_tag_naming_another_version_names_both(self, tmp_path: Path) -> None: + project = read_project(write_project(tmp_path)) + tag = f"{project.tag}.post9" + + failure = verify_version_tag.tag_failure(tag, project) + + assert failure is not None + assert tag in failure + assert project.version in failure + + class TestMain: def test_the_tag_naming_the_project_version_passes(self) -> None: version = read_project(repository_root()).version - assert verify_version_tag.main(["--tag", f"{verify_version_tag.TAG_PREFIX}{version}"]) == 0 + assert verify_version_tag.main(["--tag", f"{TAG_PREFIX}{version}"]) == 0 def test_a_tag_naming_another_version_is_annotated_as_an_error(self, capsys: pytest.CaptureFixture[str]) -> None: version = read_project(repository_root()).version - assert verify_version_tag.main(["--tag", f"{verify_version_tag.TAG_PREFIX}{version}.post9"]) == 1 + assert verify_version_tag.main(["--tag", f"{TAG_PREFIX}{version}.post9"]) == 1 assert capsys.readouterr().out.startswith("::error::") From 597a3c4fd8a3c29988198d1faa3b2a6da8be4510 Mon Sep 17 00:00:00 2001 From: JakimPL Date: Sun, 13 Sep 2026 22:16:34 +0200 Subject: [PATCH 34/36] Tidied: the study's sources, the light paths and the tools' names --- docs/development/tooling.md | 2 +- src/sampletones/commands/convert.py | 2 +- src/sampletones/commands/open.py | 2 +- src/sampletones/dispatcher.py | 4 +- .../logic/main/sources/scan.py | 3 +- .../ui/resources/loader.py | 34 +++---- .../conversion/{console.py => pairing.py} | 0 .../headless/conversion/request.py | 3 +- .../converter/paths/__init__.py | 2 - .../reconstructions/converter/paths/utils.py | 6 +- src/sampletones_shared/paths/extensions.py | 6 ++ src/sampletones_shared/paths/package.py | 6 +- src/sampletones_shared/utils/validation.py | 2 +- src/sampletones_tools/calibration/report.py | 2 +- src/sampletones_tools/calibration/session.py | 4 +- .../checks/commands/palette_colors.py | 4 +- src/sampletones_tools/checks/language_keys.py | 4 +- .../checks/palette_colors.py | 4 +- .../checks/shortcut_actions.py | 4 +- .../checks/source/packages.py | 2 +- src/sampletones_tools/checks/tag_names.py | 4 +- src/sampletones_tools/checks/unused_tags.py | 4 +- src/sampletones_tools/codec/command.py | 24 +++-- src/sampletones_tools/codec/study/manifest.py | 17 +++- src/sampletones_tools/codec/study/plan.py | 35 +++++++ .../codec/study/report/run.py | 19 ++-- src/sampletones_tools/codec/study/session.py | 48 +++------- .../player/assembler/builder.py | 9 +- .../player/assembler/layout.py | 9 +- tests/integration/tooling/test_codec_study.py | 75 +++++++++++++++ tests/suite/files.py | 9 ++ tests/unit/sampletones/commands/test_open.py | 30 ++++++ tests/unit/sampletones/test_dispatcher.py | 7 +- .../{test_console.py => test_pairing.py} | 2 +- .../calibration/test_command.py | 5 +- .../calibration/test_report.py | 56 +++++++++++ .../calibration/test_session.py | 5 +- .../checks/commands/__init__.py | 0 .../checks/commands/test_palette_colors.py | 59 ++++++++++++ .../checks/source/test_packages.py | 19 ++-- .../codec/study/test_manifest.py | 36 +++++++ .../codec/study/test_plan.py | 32 +++++++ .../codec/study/test_session.py | 35 +++---- .../sampletones_tools/codec/test_command.py | 94 +++++++++++-------- .../player/test_song_include.py | 7 +- 45 files changed, 535 insertions(+), 201 deletions(-) rename src/sampletones_core/headless/conversion/{console.py => pairing.py} (100%) create mode 100644 src/sampletones_tools/codec/study/plan.py create mode 100644 tests/integration/tooling/test_codec_study.py create mode 100644 tests/suite/files.py rename tests/unit/sampletones_core/headless/conversion/{test_console.py => test_pairing.py} (95%) create mode 100644 tests/unit/sampletones_tools/calibration/test_report.py create mode 100644 tests/unit/sampletones_tools/checks/commands/__init__.py create mode 100644 tests/unit/sampletones_tools/checks/commands/test_palette_colors.py create mode 100644 tests/unit/sampletones_tools/codec/study/test_manifest.py create mode 100644 tests/unit/sampletones_tools/codec/study/test_plan.py diff --git a/docs/development/tooling.md b/docs/development/tooling.md index 7476745cc..0e9b677fb 100644 --- a/docs/development/tooling.md +++ b/docs/development/tooling.md @@ -125,7 +125,7 @@ command `make setup` installs is a wheel and refuses the guarded ones the same w imports pillow, NumPy and the like inside `run`, and a test imports the registry in a subprocess and asserts that only the command, registry and package modules of the tools load and no heavy library does, since a startup failure in any tool module would break every invocation, the GUI included. A -command reports a refused value in one line: `describe_failure` in +command reports each refused value on a line of its own: `describe_failure` in `sampletones_shared/utils/validation.py` renders a validation error the way a person reads it. The editable install puts `src/` on the path whole, so a checkout sees the tools package whatever the wheel lists; hatchling's `dev-mode-exact` stays off for that diff --git a/src/sampletones/commands/convert.py b/src/sampletones/commands/convert.py index fd765d242..59d909229 100644 --- a/src/sampletones/commands/convert.py +++ b/src/sampletones/commands/convert.py @@ -55,7 +55,7 @@ def run(arguments: Namespace) -> int: ) from sampletones_core.headless.config import load_config - from sampletones_core.headless.conversion.console import pairing_lines + from sampletones_core.headless.conversion.pairing import pairing_lines from sampletones_core.headless.conversion.request import ( ConversionRequest, channels_named, diff --git a/src/sampletones/commands/open.py b/src/sampletones/commands/open.py index b9d92871c..9457c97bd 100644 --- a/src/sampletones/commands/open.py +++ b/src/sampletones/commands/open.py @@ -32,11 +32,11 @@ def run(arguments: Namespace) -> int: """ given = OpenArguments(path=arguments.path, config=arguments.config) - from sampletones_core.reconstructions.converter.paths import is_audio_file from sampletones_shared.paths.extensions import ( EXT_FILE_LIBRARY, EXT_FILE_PROJECT, EXT_FILE_RECONSTRUCTION, + is_audio_file, ) if not given.path.is_file(): diff --git a/src/sampletones/dispatcher.py b/src/sampletones/dispatcher.py index 5a33dd278..b0d85872a 100644 --- a/src/sampletones/dispatcher.py +++ b/src/sampletones/dispatcher.py @@ -9,10 +9,10 @@ DEFAULT_COMMAND: Final[str] = "run" COMMAND_FIELD: Final[str] = "command" COMMAND_METAVAR: Final[str] = "" -DEVELOPER_GUIDE: Final[str] = "docs/development/tooling.md" EPILOG: Final[str] = ( f"Run '{PROGRAM} {COMMAND_METAVAR} --help' for a command's options. The commands for developing " - f"SampleToNES run from a checkout as 'uv run {PROGRAM} {COMMAND_METAVAR}'; {DEVELOPER_GUIDE} lists them." + "SampleToNES follow the ones for using it; one that needs the repository says so and names the " + f"'uv run {PROGRAM} {COMMAND_METAVAR}' line to run from a checkout." ) diff --git a/src/sampletones_application/logic/main/sources/scan.py b/src/sampletones_application/logic/main/sources/scan.py index 43d08b7b7..56e1328fb 100644 --- a/src/sampletones_application/logic/main/sources/scan.py +++ b/src/sampletones_application/logic/main/sources/scan.py @@ -3,7 +3,8 @@ from typing import Callable, Final, List, Optional, Tuple from sampletones_application.utils.parallelization.thread import concurrent -from sampletones_core.reconstructions.converter.paths import is_audio_file, walk_entries +from sampletones_core.reconstructions.converter.paths import walk_entries +from sampletones_shared.paths.extensions import is_audio_file from sampletones_shared.types.callback import PathCallback, VoidCallback from sampletones_shared.utils.callbacks import CallbackMixin diff --git a/src/sampletones_application/ui/resources/loader.py b/src/sampletones_application/ui/resources/loader.py index 35dad7f1c..327a1fa7c 100644 --- a/src/sampletones_application/ui/resources/loader.py +++ b/src/sampletones_application/ui/resources/loader.py @@ -1,33 +1,33 @@ -from importlib.resources import files -from importlib.resources.abc import Traversable from pathlib import Path -from typing import Union +from typing import Final +from sampletones_shared.paths.package import package_directory from sampletones_shared.types.path import Pathlike +ASSETS_PACKAGE: Final[str] = "sampletones_assets" + class ResourceLoader: + """Reads the files of one directory of the assets package, where the package lies in every copy.""" + def __init__(self, resource_directory: Pathlike) -> None: self.resource_directory = Path(resource_directory) - def _get_package_path(self, resource_name: str) -> Union[Path, Traversable]: - package_name = f"sampletones_assets.{self.resource_directory.name}" - return files(package_name).joinpath(resource_name) + def _get_package_path(self, resource_name: str) -> Path: + package = f"{ASSETS_PACKAGE}.{self.resource_directory.name}" + return package_directory(package) / resource_name def get_path(self, resource_name: str) -> str: - resource_path = self._get_package_path(resource_name) + """The resource's location, which a widget loading it by path reads. - if isinstance(resource_path, Path): - if not resource_path.exists(): - raise FileNotFoundError(f"Resource not found: '{resource_path}'") - else: - try: - resource_path.read_bytes() - except FileNotFoundError as exception: - raise FileNotFoundError(f"Resource not found: '{resource_path}'") from exception + Raises: + FileNotFoundError: If the directory holds no file of that name. + """ + resource_path = self._get_package_path(resource_name) + if not resource_path.is_file(): + raise FileNotFoundError(f"Resource not found: '{resource_path}'") return str(resource_path) def get_bytes(self, resource_name: str) -> bytes: - resource_path = self._get_package_path(resource_name) - return resource_path.read_bytes() + return self._get_package_path(resource_name).read_bytes() diff --git a/src/sampletones_core/headless/conversion/console.py b/src/sampletones_core/headless/conversion/pairing.py similarity index 100% rename from src/sampletones_core/headless/conversion/console.py rename to src/sampletones_core/headless/conversion/pairing.py diff --git a/src/sampletones_core/headless/conversion/request.py b/src/sampletones_core/headless/conversion/request.py index 73ddf2420..8317568a2 100644 --- a/src/sampletones_core/headless/conversion/request.py +++ b/src/sampletones_core/headless/conversion/request.py @@ -10,11 +10,10 @@ bending_channels, ordered_channels, ) -from sampletones_core.reconstructions.converter.paths import is_audio_file from sampletones_core.reconstructions.reconstructor.stems.configs.config import ( StemsConfig, ) -from sampletones_shared.paths.extensions import EXT_FILES_AUDIO +from sampletones_shared.paths.extensions import EXT_FILES_AUDIO, is_audio_file from sampletones_shared.utils.serialization import load_json from sampletones_shared.utils.text import listed_items diff --git a/src/sampletones_core/reconstructions/converter/paths/__init__.py b/src/sampletones_core/reconstructions/converter/paths/__init__.py index 700793310..515e92e0d 100644 --- a/src/sampletones_core/reconstructions/converter/paths/__init__.py +++ b/src/sampletones_core/reconstructions/converter/paths/__init__.py @@ -8,7 +8,6 @@ get_output_path, get_relative_path, group_output_path, - is_audio_file, walk_audio_files, walk_entries, ) @@ -21,7 +20,6 @@ "get_output_path", "get_relative_path", "group_output_path", - "is_audio_file", "walk_audio_files", "walk_entries", ] diff --git a/src/sampletones_core/reconstructions/converter/paths/utils.py b/src/sampletones_core/reconstructions/converter/paths/utils.py index f974267b7..e6b70cb00 100644 --- a/src/sampletones_core/reconstructions/converter/paths/utils.py +++ b/src/sampletones_core/reconstructions/converter/paths/utils.py @@ -10,6 +10,7 @@ from sampletones_shared.paths.extensions import ( EXT_FILE_RECONSTRUCTION, EXT_FILES_AUDIO, + is_audio_file, ) from sampletones_shared.utils.system.paths import to_path @@ -94,11 +95,6 @@ def walk_entries(input_directory: Path) -> Iterator[Path]: return input_directory.rglob("*") -def is_audio_file(path: Path, extensions: Tuple[str, ...] = EXT_FILES_AUDIO) -> bool: - """Whether a path names a recording a run converts.""" - return path.is_file() and path.suffix.lower() in extensions - - def walk_audio_files( input_directory: Path, extensions: Tuple[str, ...] = EXT_FILES_AUDIO, diff --git a/src/sampletones_shared/paths/extensions.py b/src/sampletones_shared/paths/extensions.py index 783e3e5d2..7e8f5642b 100644 --- a/src/sampletones_shared/paths/extensions.py +++ b/src/sampletones_shared/paths/extensions.py @@ -1,3 +1,4 @@ +from pathlib import Path from typing import Final, Tuple EXT_FILE_JSON: Final[str] = ".json" @@ -23,3 +24,8 @@ EXT_FILE_AIFF, EXT_FILE_AU, ) + + +def is_audio_file(path: Path, extensions: Tuple[str, ...] = EXT_FILES_AUDIO) -> bool: + """Whether a path names a recording a run converts.""" + return path.is_file() and path.suffix.lower() in extensions diff --git a/src/sampletones_shared/paths/package.py b/src/sampletones_shared/paths/package.py index a6f6daa41..6b967884f 100644 --- a/src/sampletones_shared/paths/package.py +++ b/src/sampletones_shared/paths/package.py @@ -7,9 +7,8 @@ def package_directory(package: str) -> Path: The import system's own record of where the package lives places it, which holds in a checkout, in an installed copy and in a bundle. PyInstaller unpacks a package's collected data - into a directory of the package's name, which a package the application never imports as code - reaches as a namespace package, so the location is read from the spec whatever kind of - package answers. + into a directory of the package's name, which a data package reaches as a namespace package, + so the location is read from the spec whatever kind of package answers. Args: package: The package's dotted name. @@ -19,6 +18,7 @@ def package_directory(package: str) -> Path: Raises: FileNotFoundError: If no package of that name is importable. + ModuleNotFoundError: If a package the dotted name passes through is absent. """ spec = find_spec(package) if spec is None or spec.submodule_search_locations is None: diff --git a/src/sampletones_shared/utils/validation.py b/src/sampletones_shared/utils/validation.py index 0374b91bf..48cedf11d 100644 --- a/src/sampletones_shared/utils/validation.py +++ b/src/sampletones_shared/utils/validation.py @@ -55,7 +55,7 @@ def describe_failure(error: ValueError) -> str: Renders a refused value the way a command line reports it, one line per problem. A reason a validator raised keeps its own words, and a broken constraint is named by the - field it binds, so a person reads what to change in place of Pydantic's diagnostic layout. + field it binds, so a person reads what to change and where. Args: error: The failure, a Pydantic validation error or a plain ``ValueError``. diff --git a/src/sampletones_tools/calibration/report.py b/src/sampletones_tools/calibration/report.py index d104199f8..7628a378f 100644 --- a/src/sampletones_tools/calibration/report.py +++ b/src/sampletones_tools/calibration/report.py @@ -7,8 +7,8 @@ from sampletones_shared.utils.tables import Table from sampletones_tools.calibration.runner import CalibrationRow -CSV_COLUMNS: Final[Tuple[str, ...]] = ("variant", "item", "category", "referee", "score") VARIANT_COLUMN: Final[str] = "variant" +CSV_COLUMNS: Final[Tuple[str, ...]] = (VARIANT_COLUMN, "item", "category", "referee", "score") OVERALL_COLUMN: Final[str] = "overall" diff --git a/src/sampletones_tools/calibration/session.py b/src/sampletones_tools/calibration/session.py index ec37c02d4..d405fae4b 100644 --- a/src/sampletones_tools/calibration/session.py +++ b/src/sampletones_tools/calibration/session.py @@ -19,7 +19,7 @@ DEFAULT_METHODS: Final[Tuple[SpectrumMethod, ...]] = (SpectrumMethod.FFT, SpectrumMethod.CQT) DEFAULT_PERCEPTUAL_EXPONENTS: Final[Tuple[float, ...]] = (1.0,) BASE_BLEND: Final[Tuple[float, ...]] = () -OUTPUT_DIRECTORY: Final[str] = "calibration" +OUTPUT_ROOT: Final[Path] = USER_PATH_DOCUMENTS / "calibration" CORPUS_DIRECTORY: Final[str] = "corpus" CSV_REPORT: Final[str] = "report.csv" MARKDOWN_REPORT: Final[str] = "report.md" @@ -66,7 +66,7 @@ def floats_named(stated: Optional[str], default: Sequence[float]) -> List[float] def default_output() -> Path: """A timestamped run directory under the user's calibration documents.""" - return stamped_run_directory(USER_PATH_DOCUMENTS / OUTPUT_DIRECTORY) + return stamped_run_directory(OUTPUT_ROOT) class CalibrationRequest(BaseModel): diff --git a/src/sampletones_tools/checks/commands/palette_colors.py b/src/sampletones_tools/checks/commands/palette_colors.py index 22c495e63..a85c6e5e5 100644 --- a/src/sampletones_tools/checks/commands/palette_colors.py +++ b/src/sampletones_tools/checks/commands/palette_colors.py @@ -8,7 +8,7 @@ NAME: Final[str] = "palette-colors" HELP: Final[str] = "hold every color to a palette token until it is drawn with" PACKAGE_HELP: Final[str] = "package whose color reads to check; without it, the application package" -CONFIG_HELP: Final[str] = ( +CONFIG_DIRECTORY_HELP: Final[str] = ( "shipped configuration package whose colors must name palette tokens; without it, the shipped one" ) PALETTES_HELP: Final[str] = "directory holding the palettes, where color values belong; without it, the shipped one" @@ -25,7 +25,7 @@ class PaletteColorsArguments: def configure(parser: ArgumentParser) -> None: parser.add_argument("--package", type=Path, default=None, help=PACKAGE_HELP) - parser.add_argument("--config-directory", type=Path, default=None, help=CONFIG_HELP) + parser.add_argument("--config-directory", type=Path, default=None, help=CONFIG_DIRECTORY_HELP) parser.add_argument("--palettes", type=Path, default=None, help=PALETTES_HELP) diff --git a/src/sampletones_tools/checks/language_keys.py b/src/sampletones_tools/checks/language_keys.py index 0653965bb..d8c4f7f3c 100644 --- a/src/sampletones_tools/checks/language_keys.py +++ b/src/sampletones_tools/checks/language_keys.py @@ -26,12 +26,12 @@ from sampletones_tools.checks.source.index import source_index from sampletones_tools.checks.source.lookups import LookupSite, tree_lookups from sampletones_tools.checks.source.modules import discover_modules, module_name -from sampletones_tools.checks.source.packages import package_directory +from sampletones_tools.checks.source.packages import source_package_directory from sampletones_tools.checks.source.values import EnumMembers, EnumTable EnumPredicate = Callable[[object], bool] -APPLICATION_PACKAGE: Final[Path] = package_directory("sampletones_application") +APPLICATION_PACKAGE: Final[Path] = source_package_directory("sampletones_application") RECEIVER_TYPE: Final[str] = "LanguageManager" ELEMENT_BASE: Final[str] = AbstractElement.__name__ diff --git a/src/sampletones_tools/checks/palette_colors.py b/src/sampletones_tools/checks/palette_colors.py index 1f3604e43..a756dcdf1 100644 --- a/src/sampletones_tools/checks/palette_colors.py +++ b/src/sampletones_tools/checks/palette_colors.py @@ -10,9 +10,9 @@ from sampletones_shared.logger import logger from sampletones_tools.checks.source.modules import SourceModule, discover_modules from sampletones_tools.checks.source.nodes import terminal_name -from sampletones_tools.checks.source.packages import package_directory +from sampletones_tools.checks.source.packages import source_package_directory -APPLICATION_PACKAGE: Final[Path] = package_directory("sampletones_application") +APPLICATION_PACKAGE: Final[Path] = source_package_directory("sampletones_application") HEX_COLOR: Final[re.Pattern[str]] = re.compile(r"[\"']#[0-9a-fA-F]{6}(?:[0-9a-fA-F]{2})?[\"']") diff --git a/src/sampletones_tools/checks/shortcut_actions.py b/src/sampletones_tools/checks/shortcut_actions.py index 92373030f..b44550f0a 100644 --- a/src/sampletones_tools/checks/shortcut_actions.py +++ b/src/sampletones_tools/checks/shortcut_actions.py @@ -28,11 +28,11 @@ ShortcutId, ) from sampletones_tools.checks.source.modules import SourceModule, parse_module -from sampletones_tools.checks.source.packages import package_directory +from sampletones_tools.checks.source.packages import source_package_directory SHORTCUTS_MODULE: Final[Path] = Path(ids_module.__file__) ELEMENTS_MODULE: Final[Path] = Path(settings_module.__file__) -SHELL_MODULE: Final[Path] = package_directory("sampletones_application") / "shell.py" +SHELL_MODULE: Final[Path] = source_package_directory("sampletones_application") / "shell.py" SCHEME_ENCODING: Final[str] = "utf-8" SCHEME_BINDINGS_FIELD: Final[str] = "bindings" diff --git a/src/sampletones_tools/checks/source/packages.py b/src/sampletones_tools/checks/source/packages.py index f19a3899a..4762e1816 100644 --- a/src/sampletones_tools/checks/source/packages.py +++ b/src/sampletones_tools/checks/source/packages.py @@ -3,7 +3,7 @@ from sampletones_shared.paths.source import SOURCE_ROOT -def package_directory(name: str, *parts: str) -> Path: +def source_package_directory(name: str, *parts: str) -> Path: """The directory a package occupies, named by path rather than by import. A source check reads the tree it checks, so taking a package from the source root keeps the diff --git a/src/sampletones_tools/checks/tag_names.py b/src/sampletones_tools/checks/tag_names.py index 2c34de76c..8d3ddb4c2 100644 --- a/src/sampletones_tools/checks/tag_names.py +++ b/src/sampletones_tools/checks/tag_names.py @@ -24,9 +24,9 @@ parse_module, ) from sampletones_tools.checks.source.nodes import terminal_name -from sampletones_tools.checks.source.packages import package_directory +from sampletones_tools.checks.source.packages import source_package_directory -TAGS_PACKAGE: Final[Path] = package_directory("sampletones_application", "tags") +TAGS_PACKAGE: Final[Path] = source_package_directory("sampletones_application", "tags") TAG_PREFIX: Final[str] = "TAG" TAG_NAME_CLASS: Final[str] = "TagName" diff --git a/src/sampletones_tools/checks/unused_tags.py b/src/sampletones_tools/checks/unused_tags.py index 8c0287de2..9cc7e1ee8 100644 --- a/src/sampletones_tools/checks/unused_tags.py +++ b/src/sampletones_tools/checks/unused_tags.py @@ -7,10 +7,10 @@ from sampletones_tools.checks.paths import SCRIPTS_ROOT from sampletones_tools.checks.source.constants import module_constants from sampletones_tools.checks.source.modules import SourceModule, discover_modules -from sampletones_tools.checks.source.packages import package_directory +from sampletones_tools.checks.source.packages import source_package_directory from sampletones_tools.checks.source.references import count_identifier_loads -TAGS_PACKAGE: Final[Path] = package_directory("sampletones_application", "tags") +TAGS_PACKAGE: Final[Path] = source_package_directory("sampletones_application", "tags") REFERENCE_ROOTS: Final[Tuple[Path, ...]] = ( SOURCE_ROOT, REPOSITORY_ROOT / "tests", diff --git a/src/sampletones_tools/codec/command.py b/src/sampletones_tools/codec/command.py index 6e029b369..29d0634f8 100644 --- a/src/sampletones_tools/codec/command.py +++ b/src/sampletones_tools/codec/command.py @@ -31,6 +31,10 @@ "variants every song is encoded under, comma separated; without it every one, and the baseline always runs" ) DEFAULT_LENGTHEN_SECONDS: Final[int] = 180 +NO_SOURCE: Final[str] = ( + "The study reads the files it is given: name a project with --project, a stem file or a directory of " + "stems with --reconstruction, or a manifest a run wrote with --manifest." +) @dataclass(frozen=True) @@ -103,16 +107,16 @@ def _study(given: StudyArguments) -> int: """Measures the sources a run names. Raises: - SystemExit: If the run names neither a source nor a manifest, the manifest is missing or - broken, the lengthening is below one second, or a variant is unknown. + SystemExit: If the run names neither a source nor a manifest, a source or the manifest is + missing, the manifest is broken, the lengthening is below one second, or a variant is + unknown. """ + if given.manifest is None and not given.projects and not given.reconstructions: + raise SystemExit(NO_SOURCE) + from sampletones_shared.utils.validation import describe_failure - from sampletones_tools.codec.study.session import ( - resolve_manifest, - run_study, - study_variants, - variant_names, - ) + from sampletones_tools.codec.study.plan import plan_study + from sampletones_tools.codec.study.session import resolve_manifest, run_study, variant_names try: manifest = resolve_manifest( @@ -122,11 +126,11 @@ def _study(given: StudyArguments) -> int: lengthen_seconds=given.lengthen, variants=variant_names(given.variants), ) - variants = study_variants(manifest.variants) + plan = plan_study(manifest) except ValueError as error: raise SystemExit(describe_failure(error)) from error - run_study(manifest, variants, given.output) + run_study(plan, given.output) return 0 diff --git a/src/sampletones_tools/codec/study/manifest.py b/src/sampletones_tools/codec/study/manifest.py index 76db22ec8..026a242c5 100644 --- a/src/sampletones_tools/codec/study/manifest.py +++ b/src/sampletones_tools/codec/study/manifest.py @@ -3,10 +3,7 @@ from pydantic import BaseModel, ConfigDict, Field, model_validator -NO_SOURCE: Final[str] = ( - "The study reads the files it is given: name a project with --project, a stem file or a directory of " - "stems with --reconstruction, or a manifest a run wrote with --manifest." -) +NAMES_A_SOURCE: Final[str] = "A study measures at least one project or reconstruction." class StudySource(BaseModel): @@ -22,6 +19,16 @@ class StudySource(BaseModel): label: str path: Path + @model_validator(mode="after") + def _names_a_file_or_directory(self) -> Self: + """Raises: + ValueError: If nothing stands at the path. + """ + if not self.path.exists(): + raise ValueError(f"No file at {self.path}.") + + return self + @classmethod def at(cls, path: Path) -> Self: """A source labeled by its own name: a file's stem, or a directory's name. @@ -58,7 +65,7 @@ def _names_a_source(self) -> Self: ValueError: If the manifest names neither a project nor a reconstruction. """ if not self.projects and not self.reconstructions: - raise ValueError(NO_SOURCE) + raise ValueError(NAMES_A_SOURCE) return self diff --git a/src/sampletones_tools/codec/study/plan.py b/src/sampletones_tools/codec/study/plan.py new file mode 100644 index 000000000..f46c7e3c3 --- /dev/null +++ b/src/sampletones_tools/codec/study/plan.py @@ -0,0 +1,35 @@ +from dataclasses import dataclass +from typing import Tuple + +from sampletones_tools.codec.study.manifest import StudyManifest +from sampletones_tools.codec.study.variants.baselines import Baselines +from sampletones_tools.codec.study.variants.registry import selected_variants +from sampletones_tools.codec.study.variants.variant import Variant + + +@dataclass(frozen=True) +class StudyPlan: + """What one run measures, and the variants its manifest names, so the two stay in step. + + Attributes: + manifest: What the run reads, which the run saves to repeat it. + variants: The variants the manifest's names select, the baseline first. + """ + + manifest: StudyManifest + variants: Tuple[Variant, ...] + + +def plan_study(manifest: StudyManifest) -> StudyPlan: + """The run a manifest describes, its variant names selected over production encodings of their own. + + Args: + manifest: What the run reads. + + Returns: + StudyPlan: The manifest beside the variants it names. + + Raises: + ValueError: If a name is registered to no variant. + """ + return StudyPlan(manifest=manifest, variants=selected_variants(manifest.variants, Baselines())) diff --git a/src/sampletones_tools/codec/study/report/run.py b/src/sampletones_tools/codec/study/report/run.py index 407290c3f..6d54cab46 100644 --- a/src/sampletones_tools/codec/study/report/run.py +++ b/src/sampletones_tools/codec/study/report/run.py @@ -11,6 +11,7 @@ from sampletones_tools.codec.study.accounting import rows as accounting from sampletones_tools.codec.study.manifest import StudyManifest from sampletones_tools.codec.study.measure import Measurement +from sampletones_tools.codec.study.plan import StudyPlan from sampletones_tools.codec.study.report import aggregate from sampletones_tools.codec.study.report import rows as songs from sampletones_tools.codec.study.report import verdicts @@ -18,7 +19,7 @@ from sampletones_tools.codec.study.variants.variant import Variant from sampletones_tools.runs import stamped_run_directory -DEFAULT_OUTPUT_ROOT: Final[Path] = USER_PATH_DOCUMENTS / "compression" +OUTPUT_ROOT: Final[Path] = USER_PATH_DOCUMENTS / "compression" REPORT_CSV: Final[str] = "report.csv" REPORT_MARKDOWN: Final[str] = "report.md" ACCOUNTING_CSV: Final[str] = "accounting.csv" @@ -36,7 +37,7 @@ def run_directory(output: Optional[Path]) -> Path: Returns: Path: The directory. """ - directory = output or stamped_run_directory(DEFAULT_OUTPUT_ROOT) + directory = output or stamped_run_directory(OUTPUT_ROOT) directory.mkdir(parents=True, exist_ok=True) return directory @@ -66,8 +67,7 @@ def commit_hash() -> str: def write_run( directory: Path, - manifest: StudyManifest, - variants: Sequence[Variant], + plan: StudyPlan, measurements: Sequence[Measurement], derived: Sequence[Measurement], ) -> None: @@ -75,8 +75,7 @@ def write_run( Args: directory: The run's directory. - manifest: What the run read. - variants: The variants every song was encoded under. + plan: What the run read and the variants every song was encoded under. measurements: Every song under every variant, in the order measured. derived: The strategy-depth measurements drawn from those, reported beside them and left out of the accounting, which reads each written encoding once. @@ -84,20 +83,20 @@ def write_run( reported = (*measurements, *derived) song_rows = [songs.study_row(measurement) for measurement in reported] group_rows = aggregate.group_rows(reported, BASELINE_NAME) - verdict_rows = verdicts.verdict_rows(group_rows, measurements, variants) + verdict_rows = verdicts.verdict_rows(group_rows, measurements, plan.variants) accounting_rows = [ accounting.account(measurement, compressed) for measurement, compressed in _written(measurements) ] song_table = Table(columns=songs.COLUMNS, rows=tuple(row.cells for row in song_rows)) verdict_table = Table(columns=verdicts.COLUMNS, rows=tuple(row.cells for row in verdict_rows)) - manifest.save(directory / MANIFEST_JSON) + plan.manifest.save(directory / MANIFEST_JSON) song_table.write_csv(directory / REPORT_CSV) verdict_table.write_csv(directory / VERDICTS_CSV) Table(columns=accounting.COLUMNS, rows=tuple(row.cells for row in accounting_rows)).write_csv( directory / ACCOUNTING_CSV ) - lines = _header(manifest, variants) - lines.extend(("## Variants", "", *_variants_table(variants).markdown_lines(), "")) + lines = _header(plan.manifest, plan.variants) + lines.extend(("## Variants", "", *_variants_table(plan.variants).markdown_lines(), "")) lines.extend(("## Verdicts", "", verdicts.RULE, "", *verdict_table.markdown_lines(), "")) group_table = Table(columns=aggregate.COLUMNS, rows=tuple(row.cells for row in group_rows)) lines.extend(("## Groups", "", *group_table.markdown_lines(), "")) diff --git a/src/sampletones_tools/codec/study/session.py b/src/sampletones_tools/codec/study/session.py index 61935f1de..a98c1fd87 100644 --- a/src/sampletones_tools/codec/study/session.py +++ b/src/sampletones_tools/codec/study/session.py @@ -1,16 +1,15 @@ from pathlib import Path -from typing import List, Optional, Sequence, Tuple +from typing import List, Optional, Tuple from sampletones_shared.logger import logger from sampletones_shared.utils.text import listed_items from sampletones_tools.codec.study.corpus.build import build_corpus -from sampletones_tools.codec.study.manifest import NO_SOURCE, StudyManifest, StudySource +from sampletones_tools.codec.study.manifest import StudyManifest, StudySource from sampletones_tools.codec.study.measure import Measurement, measure +from sampletones_tools.codec.study.plan import StudyPlan from sampletones_tools.codec.study.report.run import run_directory, write_run -from sampletones_tools.codec.study.variants.baselines import Baselines -from sampletones_tools.codec.study.variants.registry import EVERY_VARIANT, selected_variants +from sampletones_tools.codec.study.variants.registry import EVERY_VARIANT from sampletones_tools.codec.study.variants.strategy import STRATEGY_ORDER, depth_measurements -from sampletones_tools.codec.study.variants.variant import Variant def variant_names(stated: Optional[str]) -> Tuple[str, ...]: @@ -41,12 +40,9 @@ def resolve_manifest( variants: The names of the variants every song is encoded under, unless a manifest states them. Raises: - ValueError: If neither a manifest nor a source is named, no manifest stands at the path, - or the manifest breaks one of its bounds. + ValueError: If no manifest stands at the path, a source names nothing, or the manifest + names no source or breaks one of its bounds. """ - if path is None and not projects and not reconstructions: - raise ValueError(NO_SOURCE) - if path is not None and not path.is_file(): raise ValueError(f"No manifest at {path}.") @@ -62,29 +58,14 @@ def resolve_manifest( ) -def study_variants(names: Sequence[str]) -> Tuple[Variant, ...]: - """The variants a run encodes under, the baseline first, over production encodings of their own. - - Args: - names: The names a manifest states, or ``all``. - - Raises: - ValueError: If a name is registered to no variant. - """ - return selected_variants(names, Baselines()) - +def run_study(plan: StudyPlan, output: Optional[Path]) -> Path: + """Encodes every song of the plan's manifest under every variant it names and writes the run. -def run_study( - manifest: StudyManifest, - variants: Sequence[Variant], - output: Optional[Path], -) -> Path: - """Encodes every song of the manifest under every variant and writes the run. + The corpus is read before the run's directory is made, so a source that fails to read leaves + the output untouched. Args: - manifest: What is measured. - variants: The variants every song is encoded under, as ``study_variants`` selects them - from the manifest's names. + plan: What is measured and the variants it is encoded under. output: The directory the run writes into, or ``None`` for a stamped one under the documents. @@ -94,12 +75,12 @@ def run_study( Raises: ValueError: If a variant writes a song as streams that play back differently. """ + corpus = build_corpus(plan.manifest) directory = run_directory(output) - corpus = build_corpus(manifest) measurements: List[Measurement] = [] for song in corpus: - for variant in variants: + for variant in plan.variants: if not variant.applies(song): continue @@ -116,8 +97,7 @@ def run_study( write_run( directory, - manifest, - variants, + plan, measurements, depth_measurements(measurements, STRATEGY_ORDER), ) diff --git a/src/sampletones_tools/player/assembler/builder.py b/src/sampletones_tools/player/assembler/builder.py index eef811e62..b171534b0 100644 --- a/src/sampletones_tools/player/assembler/builder.py +++ b/src/sampletones_tools/player/assembler/builder.py @@ -1,4 +1,3 @@ -from importlib.resources import as_file from pathlib import Path from tempfile import TemporaryDirectory from typing import Final, List @@ -9,11 +8,11 @@ from sampletones_shared.exceptions import DriverBuildError from sampletones_tools.player.assembler.labels import read_addresses from sampletones_tools.player.assembler.layout import ( + ASSEMBLY_DIRECTORY, INCLUDE_DIRECTORY, LINKER_CONFIGURATION, SOURCE_DIRECTORY, SOURCE_NAMES, - assembly, ) from sampletones_tools.player.assembler.toolchain import Toolchain @@ -41,12 +40,12 @@ def build_driver(destination: Path) -> DriverImage: built to answer at. """ toolchain = Toolchain.locate() - with as_file(assembly()) as sources, TemporaryDirectory() as directory: + with TemporaryDirectory() as directory: work_directory = Path(directory) - objects = assemble_sources(toolchain, sources, work_directory) + objects = assemble_sources(toolchain, ASSEMBLY_DIRECTORY, work_directory) assembled = work_directory / DRIVER_CODE_NAME labels = work_directory / LABELS_NAME - toolchain.link(sources / LINKER_CONFIGURATION, objects, assembled, labels) + toolchain.link(ASSEMBLY_DIRECTORY / LINKER_CONFIGURATION, objects, assembled, labels) image = DriverImage( code=assembled.read_bytes(), addresses=read_addresses(labels), diff --git a/src/sampletones_tools/player/assembler/layout.py b/src/sampletones_tools/player/assembler/layout.py index 04d461cc8..73f560912 100644 --- a/src/sampletones_tools/player/assembler/layout.py +++ b/src/sampletones_tools/player/assembler/layout.py @@ -1,9 +1,8 @@ -from importlib.resources import files -from importlib.resources.abc import Traversable from pathlib import Path from typing import Final, Tuple from sampletones_player.specification.driver import DRIVER_BINARY_DIRECTORY +from sampletones_shared.paths.package import package_directory from sampletones_shared.paths.source import SOURCE_ROOT ASSEMBLY_PACKAGE: Final[str] = "sampletones_tools.player.assembly" @@ -12,8 +11,4 @@ LINKER_CONFIGURATION: Final[str] = "nsf.cfg" SOURCE_NAMES: Final[Tuple[str, ...]] = ("driver.s", "clock.s", "channels.s") BINARY_DIRECTORY: Final[Path] = SOURCE_ROOT / "sampletones_player" / "driver" / DRIVER_BINARY_DIRECTORY - - -def assembly() -> Traversable: - """The assembly sources, their includes and the linker configuration, read from the package.""" - return files(ASSEMBLY_PACKAGE) +ASSEMBLY_DIRECTORY: Final[Path] = package_directory(ASSEMBLY_PACKAGE) diff --git a/tests/integration/tooling/test_codec_study.py b/tests/integration/tooling/test_codec_study.py new file mode 100644 index 000000000..add93f6cf --- /dev/null +++ b/tests/integration/tooling/test_codec_study.py @@ -0,0 +1,75 @@ +import csv +from pathlib import Path +from typing import Final, List + +import pytest + +from sampletones_core.project.container import ProjectContainer +from sampletones_core.project.project import Project +from sampletones_shared.exceptions.project import NotAValidArchiveError +from sampletones_shared.utils.tables import Table +from sampletones_tools.codec.study.manifest import StudyManifest, StudySource +from sampletones_tools.codec.study.plan import StudyPlan, plan_study +from sampletones_tools.codec.study.report import rows as songs +from sampletones_tools.codec.study.report import verdicts +from sampletones_tools.codec.study.report.run import MANIFEST_JSON, REPORT_CSV, REPORT_MARKDOWN, VERDICTS_CSV +from sampletones_tools.codec.study.session import run_study + +VARIANT: Final[str] = "wide-hold" +LENGTHEN_SECONDS: Final[int] = 2 + + +def _read_csv(path: Path) -> List[List[str]]: + with path.open(newline="", encoding="utf-8") as handle: + return list(csv.reader(handle)) + + +@pytest.fixture(name="plan") +def plan_fixture(integration_project: Project, tmp_path: Path) -> StudyPlan: + project = tmp_path / "synthetic.stp" + ProjectContainer.save(integration_project, project) + return plan_study( + StudyManifest( + projects=(StudySource.at(project),), + reconstructions=(), + lengthen_seconds=LENGTHEN_SECONDS, + variants=(VARIANT,), + ) + ) + + +class TestStudyRun: + def test_a_run_writes_its_tables_and_the_manifest_that_repeats_it(self, plan: StudyPlan, tmp_path: Path) -> None: + directory = run_study(plan, tmp_path / "run") + + assert StudyManifest.load(directory / MANIFEST_JSON) == plan.manifest + + header, *measured = _read_csv(directory / REPORT_CSV) + assert tuple(header) == songs.COLUMNS + assert {row[header.index("variant")] for row in measured} >= {variant.name for variant in plan.variants} + + verdict_header, *verdict_rows = _read_csv(directory / VERDICTS_CSV) + assert tuple(verdict_header) == verdicts.COLUMNS + + markdown = (directory / REPORT_MARKDOWN).read_text(encoding="utf-8").split("\n") + table = Table(columns=verdicts.COLUMNS, rows=tuple(tuple(row) for row in verdict_rows)).markdown_lines() + start = markdown.index(table[0]) + assert markdown[start : start + len(table)] == table + + def test_a_source_that_fails_to_read_leaves_the_output_untouched(self, tmp_path: Path) -> None: + broken = tmp_path / "broken.stp" + broken.write_bytes(b"") + output = tmp_path / "run" + plan = plan_study( + StudyManifest( + projects=(StudySource.at(broken),), + reconstructions=(), + lengthen_seconds=LENGTHEN_SECONDS, + variants=(VARIANT,), + ) + ) + + with pytest.raises(NotAValidArchiveError): + run_study(plan, output) + + assert not output.exists() diff --git a/tests/suite/files.py b/tests/suite/files.py new file mode 100644 index 000000000..e90e20d9c --- /dev/null +++ b/tests/suite/files.py @@ -0,0 +1,9 @@ +from pathlib import Path + + +def empty_file(directory: Path, name: str) -> Path: + """Writes an empty file called ``name`` under ``directory``, which a check that reads only names and presence accepts.""" + path = directory / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(b"") + return path diff --git a/tests/unit/sampletones/commands/test_open.py b/tests/unit/sampletones/commands/test_open.py index 264f622f1..91d3434ba 100644 --- a/tests/unit/sampletones/commands/test_open.py +++ b/tests/unit/sampletones/commands/test_open.py @@ -1,13 +1,28 @@ +import json import re +import subprocess +import sys from pathlib import Path +from typing import Final, List, Tuple import pytest from sampletones.commands.registry import COMMANDS from sampletones.dispatcher import dispatch from tests.suite.commands import RecordedApplication +from tests.suite.files import empty_file LAUNCHER = "sampletones.run.run_application" +ENGINE_MODULES: Final[Tuple[str, ...]] = ("numpy", "scipy", "sampletones_core.reconstructions") +REFUSAL_PROBE: Final[str] = """ +import json, sys +from sampletones.commands.registry import COMMANDS +from sampletones.dispatcher import dispatch +try: + dispatch(COMMANDS, ["open", sys.argv[1]]) +except SystemExit: + print(json.dumps(sorted(sys.modules))) +""" def _file(tmp_path: Path, name: str) -> Path: @@ -59,3 +74,18 @@ def test_a_file_of_another_kind_is_refused(self, tmp_path: Path) -> None: def test_a_missing_file_is_refused(self, tmp_path: Path) -> None: with pytest.raises(SystemExit, match="No file at"): dispatch(COMMANDS, ["open", str(tmp_path / "absent.stp")]) + + +class TestRefusingARecording: + """Pointing a recording at convert is a check on its name, which a fresh interpreter makes without the engine.""" + + def test_the_refusal_loads_no_engine(self, tmp_path: Path) -> None: + completed = subprocess.run( + [sys.executable, "-c", REFUSAL_PROBE, str(empty_file(tmp_path, "song.wav"))], + check=True, + capture_output=True, + text=True, + ) + + modules: List[str] = json.loads(completed.stdout) + assert set(modules) & set(ENGINE_MODULES) == set() diff --git a/tests/unit/sampletones/test_dispatcher.py b/tests/unit/sampletones/test_dispatcher.py index c24b7cf9e..0a55f6eb3 100644 --- a/tests/unit/sampletones/test_dispatcher.py +++ b/tests/unit/sampletones/test_dispatcher.py @@ -3,7 +3,7 @@ import pytest -from sampletones.dispatcher import DEFAULT_COMMAND, DEVELOPER_GUIDE, build_parser, dispatch +from sampletones.dispatcher import COMMAND_METAVAR, DEFAULT_COMMAND, PROGRAM, build_parser, dispatch from sampletones_shared.application import SAMPLETONES_NAME_VERSION from sampletones_shared.command import Command @@ -35,11 +35,10 @@ def test_the_version_is_a_flag_of_the_entry(self, capsys: pytest.CaptureFixture[ assert leaving.value.code == 0 assert SAMPLETONES_NAME_VERSION in capsys.readouterr().out - def test_the_listing_says_where_the_developer_commands_run_and_what_lists_them(self) -> None: + def test_the_listing_names_the_line_a_developer_command_runs_from_a_checkout(self) -> None: listing = " ".join(build_parser((_command("first", []),)).format_help().split()) - assert "uv run sampletones" in listing - assert DEVELOPER_GUIDE in listing + assert f"uv run {PROGRAM} {COMMAND_METAVAR}" in listing def test_every_command_is_listed_with_its_help(self) -> None: parser = build_parser((_command("first", []), _command("second", []))) diff --git a/tests/unit/sampletones_core/headless/conversion/test_console.py b/tests/unit/sampletones_core/headless/conversion/test_pairing.py similarity index 95% rename from tests/unit/sampletones_core/headless/conversion/test_console.py rename to tests/unit/sampletones_core/headless/conversion/test_pairing.py index 1fa89df21..02e68c549 100644 --- a/tests/unit/sampletones_core/headless/conversion/test_console.py +++ b/tests/unit/sampletones_core/headless/conversion/test_pairing.py @@ -1,7 +1,7 @@ from pathlib import Path from sampletones_core.constants.enums import DEFAULT_CHANNELS -from sampletones_core.headless.conversion.console import describe_stem, pairing_lines +from sampletones_core.headless.conversion.pairing import describe_stem, pairing_lines from sampletones_core.headless.conversion.request import ConversionRequest, classic_setup from tests.unit.sampletones_core.headless.conversion.stems import recording, two_stems diff --git a/tests/unit/sampletones_tools/calibration/test_command.py b/tests/unit/sampletones_tools/calibration/test_command.py index 3e80f8d4b..e1977983e 100644 --- a/tests/unit/sampletones_tools/calibration/test_command.py +++ b/tests/unit/sampletones_tools/calibration/test_command.py @@ -7,8 +7,7 @@ from sampletones.dispatcher import dispatch from sampletones_core.configs import Config from sampletones_core.constants.enums import DEFAULT_CHANNELS, ChannelName, SpectrumMethod -from sampletones_shared.paths.user import USER_PATH_DOCUMENTS -from sampletones_tools.calibration.session import OUTPUT_DIRECTORY, CalibrationRequest +from sampletones_tools.calibration.session import OUTPUT_ROOT, CalibrationRequest CALIBRATE = "sampletones_tools.calibration.session.calibrate" LOADER = "sampletones_core.headless.config.load_config" @@ -68,7 +67,7 @@ def test_without_options_the_run_sweeps_both_methods_into_the_documents( assert request.perceptual_exponents == [1.0] assert request.temporal_weights == [] assert request.channels == list(DEFAULT_CHANNELS) - assert request.output.parent == USER_PATH_DOCUMENTS / OUTPUT_DIRECTORY + assert request.output.parent == OUTPUT_ROOT def test_an_unknown_method_is_refused(self, calibration: RecordedCalibration) -> None: with pytest.raises(SystemExit, match="Unknown spectrum method"): diff --git a/tests/unit/sampletones_tools/calibration/test_report.py b/tests/unit/sampletones_tools/calibration/test_report.py new file mode 100644 index 000000000..5998f3d0f --- /dev/null +++ b/tests/unit/sampletones_tools/calibration/test_report.py @@ -0,0 +1,56 @@ +import csv +from pathlib import Path +from typing import Final, List + +from sampletones_shared.utils.tables import Table +from sampletones_tools.calibration.report import CSV_COLUMNS, OVERALL_COLUMN, VARIANT_COLUMN, write_csv, write_markdown +from sampletones_tools.calibration.runner import CalibrationRow + +ROWS: Final[List[CalibrationRow]] = [ + CalibrationRow(variant="fft", item="tone-a", category="tones", referee="spectral", score=0.25), + CalibrationRow(variant="fft", item="tone-b", category="tones", referee="spectral", score=0.75), + CalibrationRow(variant="fft", item="hiss", category="noise", referee="spectral", score=1.0), + CalibrationRow(variant="cqt", item="tone-a", category="tones", referee="spectral", score=0.5), + CalibrationRow(variant="cqt", item="hiss", category="noise", referee="spectral", score=0.5), + CalibrationRow(variant="fft", item="tone-a", category="tones", referee="envelope", score=2.0), +] + + +class TestWriteCsv: + def test_every_row_reads_back_under_the_columns(self, tmp_path: Path) -> None: + path = tmp_path / "report.csv" + + write_csv(ROWS, path) + + with path.open(newline="", encoding="utf-8") as handle: + header, *cells = list(csv.reader(handle)) + assert tuple(header) == CSV_COLUMNS + assert [(row[0], row[1], row[2], row[3], float(row[4])) for row in cells] == [ + (row.variant, row.item, row.category, row.referee, row.score) for row in ROWS + ] + + +class TestWriteMarkdown: + def test_each_referee_pivots_the_mean_of_every_variant_by_category(self, tmp_path: Path) -> None: + path = tmp_path / "report.md" + + write_markdown(ROWS, path) + + lines = path.read_text(encoding="utf-8").split("\n") + spectral = Table( + columns=(VARIANT_COLUMN, "tones", "noise", OVERALL_COLUMN), + rows=(("fft", "0.500", "1.000", "0.667"), ("cqt", "0.500", "0.500", "0.500")), + ) + envelope = Table(columns=(VARIANT_COLUMN, "tones", OVERALL_COLUMN), rows=(("fft", "2.000", "2.000"),)) + assert lines == [ + "# Calibration report", + "", + "## spectral", + "", + *spectral.markdown_lines(), + "", + "## envelope", + "", + *envelope.markdown_lines(), + "", + ] diff --git a/tests/unit/sampletones_tools/calibration/test_session.py b/tests/unit/sampletones_tools/calibration/test_session.py index 85d4da3db..06596c88e 100644 --- a/tests/unit/sampletones_tools/calibration/test_session.py +++ b/tests/unit/sampletones_tools/calibration/test_session.py @@ -5,9 +5,8 @@ from sampletones_core.configs import Config from sampletones_core.constants.enums import ChannelName, SpectrumMethod -from sampletones_shared.paths.user import USER_PATH_DOCUMENTS from sampletones_tools.calibration.session import ( - OUTPUT_DIRECTORY, + OUTPUT_ROOT, CalibrationRequest, default_output, floats_named, @@ -45,7 +44,7 @@ class TestDefaultOutput: def test_a_run_lands_in_a_timestamped_directory_under_the_documents(self) -> None: output = default_output() - assert output.parent == USER_PATH_DOCUMENTS / OUTPUT_DIRECTORY + assert output.parent == OUTPUT_ROOT assert datetime.strptime(output.name, RUN_STAMP) diff --git a/tests/unit/sampletones_tools/checks/commands/__init__.py b/tests/unit/sampletones_tools/checks/commands/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/unit/sampletones_tools/checks/commands/test_palette_colors.py b/tests/unit/sampletones_tools/checks/commands/test_palette_colors.py new file mode 100644 index 000000000..2fcedacaa --- /dev/null +++ b/tests/unit/sampletones_tools/checks/commands/test_palette_colors.py @@ -0,0 +1,59 @@ +from pathlib import Path +from typing import Final, List, Tuple + +import pytest + +from sampletones.commands.registry import COMMANDS +from sampletones.dispatcher import dispatch +from sampletones_application.paths import PALETTES_DIRECTORY +from sampletones_shared.paths.resources import CONFIG_DIRECTORY +from sampletones_tools.checks import palette_colors +from sampletones_tools.checks.palette_colors import APPLICATION_PACKAGE, ColorFinding + +GUARD: Final[str] = "sampletones_tools.checkout.require_checkout" + + +class RecordedCheck: + def __init__(self) -> None: + self.checked: List[Tuple[Path, Path, Path]] = [] + + def __call__(self, package: Path, config: Path, palettes: Path) -> List[ColorFinding]: + self.checked.append((package, config, palettes)) + return [] + + +@pytest.fixture(name="check") +def check_fixture(monkeypatch: pytest.MonkeyPatch) -> RecordedCheck: + recorded = RecordedCheck() + guarded: List[str] = [] + monkeypatch.setattr(palette_colors, "check_colors", recorded) + monkeypatch.setattr(GUARD, guarded.append) + return recorded + + +class TestPaletteColorsCommand: + def test_the_shipped_trees_are_checked_when_none_is_named(self, check: RecordedCheck) -> None: + assert dispatch(COMMANDS, ["check", "palette-colors"]) == 0 + assert check.checked == [(APPLICATION_PACKAGE, CONFIG_DIRECTORY, PALETTES_DIRECTORY)] + + def test_the_trees_named_are_the_ones_checked(self, check: RecordedCheck, tmp_path: Path) -> None: + package = tmp_path / "package" + config = tmp_path / "config" + palettes = tmp_path / "palettes" + + status = dispatch( + COMMANDS, + [ + "check", + "palette-colors", + "--package", + str(package), + "--config-directory", + str(config), + "--palettes", + str(palettes), + ], + ) + + assert status == 0 + assert check.checked == [(package, config, palettes)] diff --git a/tests/unit/sampletones_tools/checks/source/test_packages.py b/tests/unit/sampletones_tools/checks/source/test_packages.py index 7859a88e5..9ee79c658 100644 --- a/tests/unit/sampletones_tools/checks/source/test_packages.py +++ b/tests/unit/sampletones_tools/checks/source/test_packages.py @@ -1,31 +1,34 @@ import pytest from sampletones_shared.paths.source import SOURCE_ROOT -from sampletones_tools.checks.source.packages import package_directory +from sampletones_tools.checks.source.packages import source_package_directory SHARED_PACKAGE = "sampletones_shared" APPLICATION_PACKAGE = "sampletones_application" -class TestPackageDirectory: +class TestSourcePackageDirectory: def test_a_top_level_package_sits_under_the_source_root(self) -> None: - assert package_directory(SHARED_PACKAGE) == SOURCE_ROOT / SHARED_PACKAGE + assert source_package_directory(SHARED_PACKAGE) == SOURCE_ROOT / SHARED_PACKAGE def test_a_subpackage_is_named_part_by_part(self) -> None: - assert package_directory(SHARED_PACKAGE, "utils", "system") == SOURCE_ROOT / SHARED_PACKAGE / "utils" / "system" + assert ( + source_package_directory(SHARED_PACKAGE, "utils", "system") + == SOURCE_ROOT / SHARED_PACKAGE / "utils" / "system" + ) def test_the_answer_is_a_directory_a_sweep_reads_under(self) -> None: """A package resource resolves to `__init__.py`, which a sweep reads nothing under.""" - assert package_directory(APPLICATION_PACKAGE, "tags").is_dir() + assert source_package_directory(APPLICATION_PACKAGE, "tags").is_dir() def test_a_package_the_source_root_holds_no_directory_for_raises(self) -> None: with pytest.raises(NotADirectoryError): - package_directory("sampletones_absent") + source_package_directory("sampletones_absent") def test_a_module_named_as_a_package_raises(self) -> None: with pytest.raises(NotADirectoryError): - package_directory(SHARED_PACKAGE, "paths.py") + source_package_directory(SHARED_PACKAGE, "paths.py") def test_the_report_names_the_path_it_looked_at(self) -> None: with pytest.raises(NotADirectoryError, match="sampletones_absent"): - package_directory("sampletones_absent") + source_package_directory("sampletones_absent") diff --git a/tests/unit/sampletones_tools/codec/study/test_manifest.py b/tests/unit/sampletones_tools/codec/study/test_manifest.py new file mode 100644 index 000000000..76802717d --- /dev/null +++ b/tests/unit/sampletones_tools/codec/study/test_manifest.py @@ -0,0 +1,36 @@ +from pathlib import Path + +import pytest + +from sampletones_tools.codec.study.manifest import NAMES_A_SOURCE, StudyManifest, StudySource +from tests.suite.files import empty_file + + +class TestStudySource: + def test_a_file_is_labeled_by_its_stem_and_a_directory_by_its_name(self, tmp_path: Path) -> None: + project = empty_file(tmp_path, "one.stp") + + assert StudySource.at(project) == StudySource(label="one", path=project) + assert StudySource.at(tmp_path) == StudySource(label=tmp_path.name, path=tmp_path) + + def test_a_source_naming_nothing_is_refused(self, tmp_path: Path) -> None: + with pytest.raises(ValueError, match="No file at"): + StudySource.at(tmp_path / "absent.stp") + + +class TestStudyManifest: + def test_a_manifest_naming_no_source_is_refused(self) -> None: + with pytest.raises(ValueError, match=NAMES_A_SOURCE): + StudyManifest(projects=(), reconstructions=(), lengthen_seconds=45, variants=("wide-hold",)) + + def test_a_saved_manifest_loads_as_it_was_written(self, tmp_path: Path) -> None: + written = StudyManifest( + projects=(StudySource.at(empty_file(tmp_path, "one.stp")),), + reconstructions=(StudySource.at(tmp_path),), + lengthen_seconds=45, + variants=("wide-hold",), + ) + path = tmp_path / "manifest.json" + written.save(path) + + assert StudyManifest.load(path) == written diff --git a/tests/unit/sampletones_tools/codec/study/test_plan.py b/tests/unit/sampletones_tools/codec/study/test_plan.py new file mode 100644 index 000000000..8b63028ae --- /dev/null +++ b/tests/unit/sampletones_tools/codec/study/test_plan.py @@ -0,0 +1,32 @@ +from pathlib import Path +from typing import Tuple + +import pytest + +from sampletones_tools.codec.study.manifest import StudyManifest, StudySource +from sampletones_tools.codec.study.plan import plan_study +from sampletones_tools.codec.study.variants.production import BASELINE_NAME +from tests.suite.files import empty_file + + +def _manifest(directory: Path, variants: Tuple[str, ...]) -> StudyManifest: + return StudyManifest( + projects=(StudySource.at(empty_file(directory, "one.stp")),), + reconstructions=(), + lengthen_seconds=45, + variants=variants, + ) + + +class TestPlanStudy: + def test_the_variants_are_the_ones_the_manifest_names_after_the_baseline(self, tmp_path: Path) -> None: + manifest = _manifest(tmp_path, ("wide-hold",)) + + plan = plan_study(manifest) + + assert plan.manifest == manifest + assert [variant.name for variant in plan.variants] == [BASELINE_NAME, "wide-hold"] + + def test_an_unknown_variant_is_refused(self, tmp_path: Path) -> None: + with pytest.raises(ValueError, match="No variant is called bogus"): + plan_study(_manifest(tmp_path, ("bogus",))) diff --git a/tests/unit/sampletones_tools/codec/study/test_session.py b/tests/unit/sampletones_tools/codec/study/test_session.py index 4e250ea6a..b9a645832 100644 --- a/tests/unit/sampletones_tools/codec/study/test_session.py +++ b/tests/unit/sampletones_tools/codec/study/test_session.py @@ -2,9 +2,10 @@ import pytest -from sampletones_tools.codec.study.manifest import StudyManifest, StudySource +from sampletones_tools.codec.study.manifest import NAMES_A_SOURCE, StudyManifest, StudySource from sampletones_tools.codec.study.session import resolve_manifest, variant_names from sampletones_tools.codec.study.variants.registry import EVERY_VARIANT +from tests.suite.files import empty_file class TestVariantNames: @@ -16,22 +17,26 @@ def test_names_are_read_in_order_with_their_spaces_stripped(self) -> None: class TestResolveManifest: - def test_sources_named_outright_are_measured(self) -> None: + def test_sources_named_outright_are_measured(self, tmp_path: Path) -> None: + project = empty_file(tmp_path / "songs", "one.stp") + stems = tmp_path / "stems" / "two" + stems.mkdir(parents=True) + manifest = resolve_manifest( None, - projects=(Path("songs/one.stp"),), - reconstructions=(Path("stems/two"),), + projects=(project,), + reconstructions=(stems,), lengthen_seconds=30, variants=("wide-hold",), ) - assert manifest.projects == (StudySource(label="one", path=Path("songs/one.stp")),) - assert manifest.reconstructions == (StudySource(label="two", path=Path("stems/two")),) + assert manifest.projects == (StudySource(label="one", path=project),) + assert manifest.reconstructions == (StudySource(label="two", path=stems),) assert manifest.lengthen_seconds == 30 assert manifest.variants == ("wide-hold",) def test_a_run_naming_no_source_and_no_manifest_is_refused(self) -> None: - with pytest.raises(ValueError, match="reads the files it is given"): + with pytest.raises(ValueError, match=NAMES_A_SOURCE): resolve_manifest( None, projects=(), @@ -52,7 +57,7 @@ def test_a_manifest_missing_from_its_path_is_refused(self, tmp_path: Path) -> No def test_a_manifest_file_is_measured_as_it_stands(self, tmp_path: Path) -> None: written = StudyManifest( - projects=(StudySource(label="one", path=Path("songs/one.stp")),), + projects=(StudySource.at(empty_file(tmp_path, "one.stp")),), reconstructions=(), lengthen_seconds=45, variants=("wide-hold",), @@ -73,26 +78,22 @@ def test_a_manifest_file_is_measured_as_it_stands(self, tmp_path: Path) -> None: def test_sources_named_outright_replace_a_manifest_s_own_and_keep_its_sweep(self, tmp_path: Path) -> None: path = tmp_path / "manifest.json" StudyManifest( - projects=(StudySource(label="one", path=Path("songs/one.stp")),), + projects=(StudySource.at(empty_file(tmp_path, "one.stp")),), reconstructions=(), lengthen_seconds=45, variants=("wide-hold",), ).save(path) + stems = tmp_path / "stems" / "two" + stems.mkdir(parents=True) manifest = resolve_manifest( path, projects=(), - reconstructions=(Path("stems/two"),), + reconstructions=(stems,), lengthen_seconds=30, variants=(EVERY_VARIANT,), ) assert manifest.projects == () - assert manifest.reconstructions == (StudySource(label="two", path=Path("stems/two")),) + assert manifest.reconstructions == (StudySource(label="two", path=stems),) assert (manifest.lengthen_seconds, manifest.variants) == (45, ("wide-hold",)) - - -class TestStudyManifest: - def test_a_manifest_naming_no_source_is_refused(self) -> None: - with pytest.raises(ValueError, match="reads the files it is given"): - StudyManifest(projects=(), reconstructions=(), lengthen_seconds=45, variants=("wide-hold",)) diff --git a/tests/unit/sampletones_tools/codec/test_command.py b/tests/unit/sampletones_tools/codec/test_command.py index 49d8edc5b..cc4919054 100644 --- a/tests/unit/sampletones_tools/codec/test_command.py +++ b/tests/unit/sampletones_tools/codec/test_command.py @@ -1,19 +1,20 @@ +import re from dataclasses import dataclass from pathlib import Path -from typing import Final, List, Optional, Sequence, Tuple +from typing import Final, List, Optional, Tuple import pytest from sampletones.commands.registry import COMMANDS from sampletones.dispatcher import dispatch -from sampletones_tools.codec.command import DEFAULT_LENGTHEN_SECONDS +from sampletones_tools.codec.command import DEFAULT_LENGTHEN_SECONDS, NO_SOURCE from sampletones_tools.codec.report.session import CompressionReport -from sampletones_tools.codec.study.manifest import StudyManifest +from sampletones_tools.codec.study.plan import StudyPlan from sampletones_tools.codec.study.variants.production import BASELINE_NAME from sampletones_tools.codec.study.variants.registry import EVERY_VARIANT -from sampletones_tools.codec.study.variants.variant import Variant from tests.suite.base import BaseTestSuite from tests.suite.case import BaseRegularTestCase +from tests.suite.files import empty_file RUNNER: Final[str] = "sampletones_tools.codec.study.session.run_study" REPORTER: Final[str] = "sampletones_tools.codec.report.session.run_report" @@ -21,13 +22,33 @@ class RecordedStudy: def __init__(self) -> None: - self.runs: List[Tuple[StudyManifest, Tuple[str, ...], Optional[Path]]] = [] + self.runs: List[Tuple[StudyPlan, Optional[Path]]] = [] - def __call__(self, manifest: StudyManifest, variants: Sequence[Variant], output: Optional[Path]) -> Path: - self.runs.append((manifest, tuple(variant.name for variant in variants), output)) + def __call__(self, plan: StudyPlan, output: Optional[Path]) -> Path: + self.runs.append((plan, output)) return output if output is not None else Path("run") +@dataclass(frozen=True) +class StudySources: + project: Path + reconstruction: Path + + +@pytest.fixture(name="study") +def study_fixture(monkeypatch: pytest.MonkeyPatch) -> RecordedStudy: + recorded = RecordedStudy() + monkeypatch.setattr(RUNNER, recorded) + return recorded + + +@pytest.fixture(name="sources") +def sources_fixture(tmp_path: Path) -> StudySources: + reconstruction = tmp_path / "stems" / "two" + reconstruction.mkdir(parents=True) + return StudySources(project=empty_file(tmp_path / "songs", "one.stp"), reconstruction=reconstruction) + + class TestCodecReport: def test_the_report_is_written_into_the_output_and_its_tables_are_printed( self, @@ -65,21 +86,19 @@ def test_the_output_is_required(self) -> None: class TestCodecStudy: def test_the_sources_and_the_sweep_are_read_from_the_options( self, - monkeypatch: pytest.MonkeyPatch, + study: RecordedStudy, + sources: StudySources, tmp_path: Path, ) -> None: - study = RecordedStudy() - monkeypatch.setattr(RUNNER, study) - status = dispatch( COMMANDS, [ "codec", "study", "--project", - "songs/one.stp", + str(sources.project), "--reconstruction", - "stems/two", + str(sources.reconstruction), "--variants", "wide-hold", "--lengthen", @@ -90,31 +109,26 @@ def test_the_sources_and_the_sweep_are_read_from_the_options( ) assert status == 0 - manifest, variants, output = study.runs[0] - assert [source.path for source in manifest.projects] == [Path("songs/one.stp")] - assert [source.path for source in manifest.reconstructions] == [Path("stems/two")] - assert manifest.variants == ("wide-hold",) - assert variants == (BASELINE_NAME, "wide-hold") - assert manifest.lengthen_seconds == 30 + plan, output = study.runs[0] + assert [source.path for source in plan.manifest.projects] == [sources.project] + assert [source.path for source in plan.manifest.reconstructions] == [sources.reconstruction] + assert plan.manifest.variants == ("wide-hold",) + assert [variant.name for variant in plan.variants] == [BASELINE_NAME, "wide-hold"] + assert plan.manifest.lengthen_seconds == 30 assert output == tmp_path - def test_a_run_naming_no_source_is_refused_with_the_way_to_name_one(self, monkeypatch: pytest.MonkeyPatch) -> None: - study = RecordedStudy() - monkeypatch.setattr(RUNNER, study) - - with pytest.raises(SystemExit, match="reads the files it is given"): + def test_a_run_naming_no_source_is_refused_with_the_way_to_name_one(self, study: RecordedStudy) -> None: + with pytest.raises(SystemExit, match=re.escape(NO_SOURCE)): dispatch(COMMANDS, ["codec", "study"]) + assert all(flag in NO_SOURCE for flag in ("--project", "--reconstruction", "--manifest")) assert study.runs == [] - def test_the_lengthening_defaults_to_its_constant(self, monkeypatch: pytest.MonkeyPatch) -> None: - study = RecordedStudy() - monkeypatch.setattr(RUNNER, study) - - assert dispatch(COMMANDS, ["codec", "study", "--project", "songs/one.stp"]) == 0 - manifest, _, output = study.runs[0] - assert manifest.lengthen_seconds == DEFAULT_LENGTHEN_SECONDS - assert manifest.variants == (EVERY_VARIANT,) + def test_the_lengthening_defaults_to_its_constant(self, study: RecordedStudy, sources: StudySources) -> None: + assert dispatch(COMMANDS, ["codec", "study", "--project", str(sources.project)]) == 0 + plan, output = study.runs[0] + assert plan.manifest.lengthen_seconds == DEFAULT_LENGTHEN_SECONDS + assert plan.manifest.variants == (EVERY_VARIANT,) assert output is None def test_an_action_is_required(self) -> None: @@ -132,21 +146,25 @@ class TestCase(BaseRegularTestCase): test_cases = ( TestCase(label="an unknown variant", argv=("--variants", "bogus"), refusal="No variant is called bogus"), - TestCase(label="no lengthening", argv=("--lengthen", "0"), refusal="lengthen_seconds: Input should be"), - TestCase(label="a missing manifest", argv=("--manifest", "absent.json"), refusal="No manifest at"), + TestCase(label="no lengthening", argv=("--lengthen", "0"), refusal="lengthen_seconds"), + TestCase(label="a missing manifest", argv=("--manifest", "{directory}/absent.json"), refusal="No manifest at"), + TestCase(label="a missing project", argv=("--project", "{directory}/absent.stp"), refusal="No file at"), + TestCase(label="a broken manifest", argv=("--manifest", "{directory}/broken.json"), refusal="JSON"), ) @pytest.mark.parametrize("test_case", test_cases, ids=lambda test_case: test_case.label) def test_a_run_asking_the_impossible_is_refused_in_one_line_before_it_starts( self, - monkeypatch: pytest.MonkeyPatch, + study: RecordedStudy, + sources: StudySources, + tmp_path: Path, test_case: TestCase, ) -> None: - study = RecordedStudy() - monkeypatch.setattr(RUNNER, study) + (tmp_path / "broken.json").write_text("{", encoding="utf-8") + argv = [argument.format(directory=tmp_path) for argument in test_case.argv] with pytest.raises(SystemExit, match=test_case.refusal) as leaving: - dispatch(COMMANDS, ["codec", "study", "--project", "songs/one.stp", *test_case.argv]) + dispatch(COMMANDS, ["codec", "study", "--project", str(sources.project), *argv]) assert len(str(leaving.value).splitlines()) == 1 assert study.runs == [] diff --git a/tests/unit/sampletones_tools/player/test_song_include.py b/tests/unit/sampletones_tools/player/test_song_include.py index 6daddf079..7576b0141 100644 --- a/tests/unit/sampletones_tools/player/test_song_include.py +++ b/tests/unit/sampletones_tools/player/test_song_include.py @@ -1,5 +1,4 @@ import ast -from importlib.resources.abc import Traversable from pathlib import Path from typing import Dict, Final @@ -31,7 +30,7 @@ TIMER_TABLE_OFFSET, TOTAL_TICKS_OFFSET, ) -from sampletones_tools.player.assembler.layout import INCLUDE_DIRECTORY, assembly +from sampletones_tools.player.assembler.layout import ASSEMBLY_DIRECTORY, INCLUDE_DIRECTORY SONG_INCLUDE: Final[str] = "song.inc" HEXADECIMAL_MARKER: Final[str] = "$" @@ -82,7 +81,7 @@ def _value(node: ast.expr, defined: Dict[str, int]) -> int: raise ValueError(f"an equate reads {ast.dump(node)}, which the include holds no form for") -def read_equates(path: Traversable) -> Dict[str, int]: +def read_equates(path: Path) -> Dict[str, int]: """Reads the constants an assembly include states, each over the ones stated before it. The driver and the exporter read one song block, so what the assembly believes about the @@ -109,7 +108,7 @@ def read_equates(path: Traversable) -> Dict[str, int]: @pytest.fixture(name="equates", scope="module") def equates_fixture() -> Dict[str, int]: - return read_equates(assembly() / INCLUDE_DIRECTORY / SONG_INCLUDE) + return read_equates(ASSEMBLY_DIRECTORY / INCLUDE_DIRECTORY / SONG_INCLUDE) class TestTheDriverReadsTheBlockTheExporterWrites: From 0d257e7337f9570a5e48ce4cc3ac0976cbf42d53 Mon Sep 17 00:00:00 2001 From: JakimPL Date: Sun, 13 Sep 2026 22:22:19 +0200 Subject: [PATCH 35/36] Sharpened: the tests the review found weak --- .../unit/sampletones/commands/test_convert.py | 25 ++++++++----------- tests/unit/sampletones/commands/test_open.py | 12 +++------ .../headless/conversion/stems.py | 8 ------ .../headless/conversion/test_pairing.py | 14 +++++------ .../headless/conversion/test_request.py | 15 +++++------ .../headless/conversion/test_runners.py | 4 +-- .../sampletones_shared/paths/test_package.py | 6 ++++- 7 files changed, 35 insertions(+), 49 deletions(-) diff --git a/tests/unit/sampletones/commands/test_convert.py b/tests/unit/sampletones/commands/test_convert.py index 78ce418c5..c98d8509e 100644 --- a/tests/unit/sampletones/commands/test_convert.py +++ b/tests/unit/sampletones/commands/test_convert.py @@ -13,6 +13,7 @@ from sampletones_core.reconstructions.reconstructor.stems.configs.entry import StemEntry from sampletones_core.reconstructions.reconstructor.stems.configs.hierarchy import StemsHierarchy from sampletones_core.reconstructions.reconstructor.stems.configs.settings import StemSettings +from tests.suite.files import empty_file RECONSTRUCTION = "sampletones_core.headless.conversion.runners.reconstruct" LOADER = "sampletones_core.headless.config.load_config" @@ -35,12 +36,6 @@ def reconstruction_fixture(monkeypatch: pytest.MonkeyPatch) -> RecordedReconstru return recorded -def _recording(tmp_path: Path, name: str) -> Path: - path = tmp_path / name - path.write_bytes(b"") - return path - - def _two_stems() -> StemsConfig: return StemsConfig( entries=[ @@ -59,7 +54,7 @@ def _two_stems() -> StemsConfig: class TestConvert: def test_the_channels_named_become_one_stem(self, reconstruction: RecordedReconstruction, tmp_path: Path) -> None: - source = _recording(tmp_path, "song.wav") + source = empty_file(tmp_path, "song.wav") output = tmp_path / "song.stn" status = dispatch(COMMANDS, ["convert", str(source), "--channels", "pulse1,pulse2", "-o", str(output)]) @@ -75,7 +70,7 @@ def test_without_channels_the_usual_three_are_used( reconstruction: RecordedReconstruction, tmp_path: Path, ) -> None: - source = _recording(tmp_path, "song.wav") + source = empty_file(tmp_path, "song.wav") assert dispatch(COMMANDS, ["convert", str(source)]) == 0 assert reconstruction.requests[0].stems == classic_setup(DEFAULT_CHANNELS) @@ -87,8 +82,8 @@ def test_a_stems_file_pairs_its_entries_with_the_sources_in_order( tmp_path: Path, capsys: pytest.CaptureFixture[str], ) -> None: - bass = _recording(tmp_path, "bass.wav") - lead = _recording(tmp_path, "lead.wav") + bass = empty_file(tmp_path, "bass.wav") + lead = empty_file(tmp_path, "lead.wav") stems = _two_stems() setup = tmp_path / "stems.json" setup.write_text(json.dumps(stems.model_dump(mode="json")), encoding="utf-8") @@ -100,7 +95,7 @@ def test_a_stems_file_pairs_its_entries_with_the_sources_in_order( assert "lead.wav: stem 1 on triangle, bending triangle" in printed def test_a_setup_pairing_wrong_is_refused(self, reconstruction: RecordedReconstruction, tmp_path: Path) -> None: - source = _recording(tmp_path, "song.wav") + source = empty_file(tmp_path, "song.wav") setup = tmp_path / "stems.json" setup.write_text(json.dumps(_two_stems().model_dump(mode="json")), encoding="utf-8") @@ -110,7 +105,7 @@ def test_a_setup_pairing_wrong_is_refused(self, reconstruction: RecordedReconstr assert reconstruction.requests == [] def test_an_unknown_channel_is_refused(self, reconstruction: RecordedReconstruction, tmp_path: Path) -> None: - source = _recording(tmp_path, "song.wav") + source = empty_file(tmp_path, "song.wav") with pytest.raises(SystemExit, match="Unknown channel 'pulse3'"): dispatch(COMMANDS, ["convert", str(source), "--channels", "pulse3"]) @@ -127,7 +122,7 @@ def test_a_missing_source_is_refused_in_one_line( assert reconstruction.requests == [] def test_a_project_is_refused_as_no_recording(self, reconstruction: RecordedReconstruction, tmp_path: Path) -> None: - project = _recording(tmp_path, "song.stp") + project = empty_file(tmp_path, "song.stp") with pytest.raises(SystemExit, match="is no recording") as leaving: dispatch(COMMANDS, ["convert", str(project)]) @@ -136,7 +131,7 @@ def test_a_project_is_refused_as_no_recording(self, reconstruction: RecordedReco assert reconstruction.requests == [] def test_a_missing_stems_file_is_refused(self, reconstruction: RecordedReconstruction, tmp_path: Path) -> None: - source = _recording(tmp_path, "song.wav") + source = empty_file(tmp_path, "song.wav") with pytest.raises(SystemExit, match="No stems file at"): dispatch(COMMANDS, ["convert", str(source), "--stems", str(tmp_path / "absent.json")]) @@ -144,7 +139,7 @@ def test_a_missing_stems_file_is_refused(self, reconstruction: RecordedReconstru assert reconstruction.requests == [] def test_channels_and_stems_exclude_each_other(self, tmp_path: Path) -> None: - source = _recording(tmp_path, "song.wav") + source = empty_file(tmp_path, "song.wav") with pytest.raises(SystemExit) as leaving: dispatch(COMMANDS, ["convert", str(source), "--channels", "pulse1", "--stems", "stems.json"]) diff --git a/tests/unit/sampletones/commands/test_open.py b/tests/unit/sampletones/commands/test_open.py index 91d3434ba..48569d622 100644 --- a/tests/unit/sampletones/commands/test_open.py +++ b/tests/unit/sampletones/commands/test_open.py @@ -25,12 +25,6 @@ """ -def _file(tmp_path: Path, name: str) -> Path: - path = tmp_path / name - path.write_bytes(b"") - return path - - class TestOpen: @pytest.mark.parametrize( ("name", "field"), @@ -45,7 +39,7 @@ def test_a_file_is_loaded_by_its_kind( ) -> None: application = RecordedApplication() monkeypatch.setattr(LAUNCHER, application) - path = _file(tmp_path, name) + path = empty_file(tmp_path, name) assert dispatch(COMMANDS, ["open", str(path), "--config", "custom.json"]) == 0 start = application.starts[0] @@ -58,7 +52,7 @@ def test_a_file_is_loaded_by_its_kind( def test_a_recording_is_pointed_at_convert(self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: application = RecordedApplication() monkeypatch.setattr(LAUNCHER, application) - path = _file(tmp_path, "song.wav") + path = empty_file(tmp_path, "song.wav") with pytest.raises(SystemExit, match=re.escape(f"sampletones convert {path}")): dispatch(COMMANDS, ["open", str(path)]) @@ -66,7 +60,7 @@ def test_a_recording_is_pointed_at_convert(self, monkeypatch: pytest.MonkeyPatch assert application.starts == [] def test_a_file_of_another_kind_is_refused(self, tmp_path: Path) -> None: - path = _file(tmp_path, "notes.txt") + path = empty_file(tmp_path, "notes.txt") with pytest.raises(SystemExit, match="neither"): dispatch(COMMANDS, ["open", str(path)]) diff --git a/tests/unit/sampletones_core/headless/conversion/stems.py b/tests/unit/sampletones_core/headless/conversion/stems.py index 89728ff60..895d652d7 100644 --- a/tests/unit/sampletones_core/headless/conversion/stems.py +++ b/tests/unit/sampletones_core/headless/conversion/stems.py @@ -1,5 +1,3 @@ -from pathlib import Path - from sampletones_core.constants.enums import ChannelName from sampletones_core.reconstructions.reconstructor.stems.configs.config import StemsConfig from sampletones_core.reconstructions.reconstructor.stems.configs.entry import StemEntry @@ -7,12 +5,6 @@ from sampletones_core.reconstructions.reconstructor.stems.configs.settings import StemSettings -def recording(tmp_path: Path, name: str) -> Path: - path = tmp_path / name - path.write_bytes(b"") - return path - - def two_stems() -> StemsConfig: return StemsConfig( entries=[ diff --git a/tests/unit/sampletones_core/headless/conversion/test_pairing.py b/tests/unit/sampletones_core/headless/conversion/test_pairing.py index 02e68c549..77e13facb 100644 --- a/tests/unit/sampletones_core/headless/conversion/test_pairing.py +++ b/tests/unit/sampletones_core/headless/conversion/test_pairing.py @@ -3,7 +3,8 @@ from sampletones_core.constants.enums import DEFAULT_CHANNELS from sampletones_core.headless.conversion.pairing import describe_stem, pairing_lines from sampletones_core.headless.conversion.request import ConversionRequest, classic_setup -from tests.unit.sampletones_core.headless.conversion.stems import recording, two_stems +from tests.suite.files import empty_file +from tests.unit.sampletones_core.headless.conversion.stems import two_stems class TestDescribeStem: @@ -16,8 +17,8 @@ def test_a_stem_names_its_channels_and_its_bends(self) -> None: class TestPairingLines: def test_recordings_pair_with_the_entries_in_order(self, tmp_path: Path) -> None: - bass = recording(tmp_path, "bass.wav") - drums = recording(tmp_path, "drums.wav") + bass = empty_file(tmp_path, "bass.wav") + drums = empty_file(tmp_path, "drums.wav") request = ConversionRequest(sources=(bass, drums), stems=two_stems(), output_path=None) @@ -27,8 +28,7 @@ def test_recordings_pair_with_the_entries_in_order(self, tmp_path: Path) -> None ] def test_a_directory_names_the_one_stem_every_recording_plays_under(self, tmp_path: Path) -> None: - request = ConversionRequest(sources=(tmp_path,), stems=classic_setup(DEFAULT_CHANNELS), output_path=None) + setup = classic_setup(DEFAULT_CHANNELS) + request = ConversionRequest(sources=(tmp_path,), stems=setup, output_path=None) - assert pairing_lines(request) == [ - f"{tmp_path.name}/: every recording under stem 0 on pulse1, triangle, noise, bending pulse1, triangle" - ] + assert pairing_lines(request) == [f"{tmp_path.name}/: every recording under {describe_stem(setup.entries[0])}"] diff --git a/tests/unit/sampletones_core/headless/conversion/test_request.py b/tests/unit/sampletones_core/headless/conversion/test_request.py index 9f73bdb60..968cbbdfe 100644 --- a/tests/unit/sampletones_core/headless/conversion/test_request.py +++ b/tests/unit/sampletones_core/headless/conversion/test_request.py @@ -10,7 +10,8 @@ classic_setup, load_stems, ) -from tests.unit.sampletones_core.headless.conversion.stems import recording, two_stems +from tests.suite.files import empty_file +from tests.unit.sampletones_core.headless.conversion.stems import two_stems class TestChannelsNamed: @@ -62,9 +63,9 @@ def test_a_mapping_that_is_no_setup_is_refused(self, tmp_path: Path) -> None: class TestConversionRequest: - def test_recordings_pair_with_the_entries_in_order(self, tmp_path: Path) -> None: - bass = recording(tmp_path, "bass.wav") - drums = recording(tmp_path, "drums.wav") + def test_recordings_name_no_directory(self, tmp_path: Path) -> None: + bass = empty_file(tmp_path, "bass.wav") + drums = empty_file(tmp_path, "drums.wav") request = ConversionRequest(sources=(bass, drums), stems=two_stems(), output_path=None) @@ -87,7 +88,7 @@ def test_the_sources_are_classified_once_when_the_request_is_made(self, tmp_path def test_a_count_mismatch_is_refused(self, tmp_path: Path) -> None: with pytest.raises(ValueError, match="1 sources for 2 stems"): - ConversionRequest(sources=(recording(tmp_path, "bass.wav"),), stems=two_stems(), output_path=None) + ConversionRequest(sources=(empty_file(tmp_path, "bass.wav"),), stems=two_stems(), output_path=None) def test_a_directory_under_several_stems_is_refused(self, tmp_path: Path) -> None: with pytest.raises(ValueError, match="under one stem; the setup holds 2"): @@ -95,7 +96,7 @@ def test_a_directory_under_several_stems_is_refused(self, tmp_path: Path) -> Non def test_a_directory_among_recordings_is_refused(self, tmp_path: Path) -> None: with pytest.raises(ValueError, match="one directory alone"): - ConversionRequest(sources=(recording(tmp_path, "bass.wav"), tmp_path), stems=two_stems(), output_path=None) + ConversionRequest(sources=(empty_file(tmp_path, "bass.wav"), tmp_path), stems=two_stems(), output_path=None) def test_an_output_path_for_a_directory_is_refused(self, tmp_path: Path) -> None: with pytest.raises(ValueError, match="an output path names the one file"): @@ -116,7 +117,7 @@ def test_a_missing_source_is_refused_by_its_path(self, tmp_path: Path) -> None: def test_a_file_other_than_a_recording_is_refused(self, tmp_path: Path) -> None: with pytest.raises(ValueError, match="is no recording"): ConversionRequest( - sources=(recording(tmp_path, "song.stp"),), + sources=(empty_file(tmp_path, "song.stp"),), stems=classic_setup(DEFAULT_CHANNELS), output_path=None, ) diff --git a/tests/unit/sampletones_core/headless/conversion/test_runners.py b/tests/unit/sampletones_core/headless/conversion/test_runners.py index 6361f68aa..e78e719b5 100644 --- a/tests/unit/sampletones_core/headless/conversion/test_runners.py +++ b/tests/unit/sampletones_core/headless/conversion/test_runners.py @@ -9,7 +9,7 @@ from sampletones_core.headless.conversion.request import ConversionRequest, classic_setup from sampletones_core.headless.conversion.runners import reconstruct from sampletones_core.reconstructions.reconstructor.stems.configs.config import StemsConfig -from tests.unit.sampletones_core.headless.conversion.stems import recording +from tests.suite.files import empty_file class TestReconstruct: @@ -23,7 +23,7 @@ def reconstruct_sources( calls.append((sources, output_path)) monkeypatch.setattr(runners, "reconstruct_sources", reconstruct_sources) - source = recording(tmp_path, "song.wav") + source = empty_file(tmp_path, "song.wav") request = ConversionRequest( sources=(source,), stems=classic_setup(DEFAULT_CHANNELS), output_path=tmp_path / "x.stn" ) diff --git a/tests/unit/sampletones_shared/paths/test_package.py b/tests/unit/sampletones_shared/paths/test_package.py index 07337a8bd..07658997f 100644 --- a/tests/unit/sampletones_shared/paths/test_package.py +++ b/tests/unit/sampletones_shared/paths/test_package.py @@ -3,6 +3,7 @@ import pytest import sampletones_config +import sampletones_tools.corpus.config from sampletones_shared.paths.package import package_directory @@ -14,7 +15,10 @@ def test_the_directory_holds_the_package_s_modules_and_data(self) -> None: assert (directory / "application").is_dir() def test_a_nested_package_is_placed_by_its_own_location(self) -> None: - assert package_directory("sampletones_tools.corpus.config").name == "config" + assert ( + package_directory("sampletones_tools.corpus.config") + == Path(sampletones_tools.corpus.config.__file__).parent + ) def test_a_directory_of_data_alone_is_placed_as_a_namespace_package( self, From 599dcccf69d806cf0f8612e96879c33261523cc3 Mon Sep 17 00:00:00 2001 From: JakimPL Date: Sun, 13 Sep 2026 22:22:42 +0200 Subject: [PATCH 36/36] Corrected: the development documents the review found stale --- docs/development/packages.md | 8 +++---- docs/development/tooling.md | 45 +++++++++++++++++++----------------- docs/guide/installation.md | 4 +--- 3 files changed, 29 insertions(+), 28 deletions(-) diff --git a/docs/development/packages.md b/docs/development/packages.md index c48f177f0..41f974222 100644 --- a/docs/development/packages.md +++ b/docs/development/packages.md @@ -44,7 +44,7 @@ graph TD | Package | Purpose | May import | |---------|---------|------------| | `sampletones_shared` | Facts and helpers any package holds: constants, exception families, paths, the logger, the array backend, and the command type the entry and the tools share | — | -| `sampletones_config` | The shipped YAML — layout, palettes, themes, keybindings, language, calibration, and these boundaries themselves — reached as package data rather than by import | — | +| `sampletones_config` | The shipped YAML — layout, palettes, themes, keybindings, language, behavior, deployment, and these boundaries themselves — reached as package data rather than by import | — | | `sampletones_assets` | The application icons and the bundled fonts, reached as package data | — | | `sampletones_core` | The reconstruction engine, the project model, playing a song out into instructions, and the tracker export formats | `sampletones_shared` | | `sampletones_player` | The NES player: the register model, the re-clocking schedule, the 6502 driver and the NSF file | `sampletones_shared`, `sampletones_core` | @@ -110,9 +110,9 @@ includes and the linker configuration in `sampletones_tools/player/assembly/`, r data, to produce the committed `driver/binary/driver.bin`; `uv run sampletones driver` runs it, and the tests rebuild the sources and hold the committed image to them wherever cc65 is installed. `sampletones_tools/player/trace/` holds `RegisterTrace`, what the driver is expected -to write call by call, which the emulator tests hold the assembled driver to. The application -ships the binary alone. The toolchain the build needs is described in -[`dependencies.md`](release/dependencies.md). +to write call by call, which the emulator tests hold the assembled driver to. Exporting reads the +assembled binary; the wheel carries the assembly sources beside it, inside the tools package. The +toolchain the build needs is described in [`dependencies.md`](release/dependencies.md). --- diff --git a/docs/development/tooling.md b/docs/development/tooling.md index 0e9b677fb..695cbc12d 100644 --- a/docs/development/tooling.md +++ b/docs/development/tooling.md @@ -59,6 +59,20 @@ target: a target names the script that does the work and passes its flag. The tw the root, `install.sh` and `install.bat`, exist for the double-click path and call the same bundle script. +**8. A developer command works from what it is given, in every copy of the program.** The wheel +and the bundle carry the tools package, so every developer command exists wherever `sampletones` +is installed, and each one reaches files the same way in all of them: + +- *Inputs* arrive on the command line or in a file a run wrote, so a run starts from what the person + running it has. +- *Outputs* go where `--output` (`-o`) names. A measurement given no `-o` writes a timestamped + directory under the user's Documents; `codec report` and the sample emitters take `-o` always. +- *The repository* is reached through the checkout guard. A command that reads or writes the + repository, or needs a development dependency, runs from a checkout; so `driver` and `icons` + rewrite the files the package ships from a checkout, and a measurement runs anywhere. +- *Package data* is read from the package it ships in, which holds in a checkout, in the wheel and + in the bundle. + ## The commands `src/sampletones/` is the entry package. `dispatcher.py` builds one parser over the commands and @@ -67,8 +81,9 @@ them. A command is a frozen `Command` (`sampletones_shared/command.py`): its nam help, the function adding its options to a parser, and the function running it over the parsed arguments. Each command turns its arguments into a frozen record, field by field, before it works; a command with actions, such as `codec`, reads the action first and builds the record that action takes. -A command writing files names their directory `--output` (`-o`), and an option naming an input says -what it reads, so `--config` is a configuration file wherever it appears. +A command writing files names where they go `--output` (`-o`): a file for `convert`, a directory for +the others. An option naming an input says what it reads, so `--config` is a configuration file +wherever it appears. | Command | What it does | |---|---| @@ -100,25 +115,13 @@ The developer commands, listed by `sampletones_tools/registry.py` and run as `src/sampletones_tools/` holds every tool the running application does not use, in subpackages by subject, and `sampletones_tools/registry.py` lists the developer commands they offer. `sampletones/commands/registry.py` appends them to the user commands, which is the one import of -the tools package; [package layers](packages.md) holds the edge. The wheel and the bundle carry the -package, so every command exists in every copy of the program, and four rules decide what a -developer command does there: - -- **The checkout guard.** `sampletones_tools/checkout.py` holds `require_checkout(command)`: the - repository root must hold `pyproject.toml` beside `src/`, or the command exits naming - `uv run sampletones ` in a checkout. Every developer command that reads or writes the - repository calls it first, and so does one that needs a development dependency, as `icons` needs - Pillow. A command that measures the code on this machine runs anywhere. -- **No default derived from the repository.** An emitter takes a required `--output`; a measurement - defaults to the user's Documents. Nothing a developer command writes lands beside an installed - package. -- **Package data is read from the package.** `package_directory` in - `sampletones_shared/paths/package.py` places a package from the import system's own record, so - the same path holds in a checkout, in the wheel and in the bundle, where PyInstaller unpacks each - package's data beside its modules. -- **A tool reads the files it is given.** What a tool measures or converts arrives on its command - line or in a file a run wrote; the code names no file on one machine, so every run starts from - what the person running it has. +the tools package; [package layers](packages.md) holds the edge. Two helpers carry out principle 8: + +- `sampletones_tools/checkout.py` holds `require_checkout(command)`: the repository root holds + `pyproject.toml` beside `src/`, or the command exits naming `uv run sampletones ` in a + checkout. `icons` calls it for Pillow, a development dependency, as well as for the repository. +- `package_directory` in `sampletones_shared/paths/package.py` places a package from the import + system's own record, where PyInstaller unpacks each package's data beside its modules. Developer commands are run as `uv run sampletones ` from a checkout; the `sampletones` command `make setup` installs is a wheel and refuses the guarded ones the same way. A command module diff --git a/docs/guide/installation.md b/docs/guide/installation.md index 74cefbf82..4e2236393 100644 --- a/docs/guide/installation.md +++ b/docs/guide/installation.md @@ -57,9 +57,7 @@ make run # starts the app After you pull new changes, run `make setup` again. -### Build a standalone app - -On Windows and Linux, you can build a standalone app from the source code: +On Windows and Linux, you can also build a standalone app from the source code: - **Windows**: double-click `install.bat`. It builds `bin\sampletones.exe`. - **Linux**: run `make system-deps`, then `./install.sh`. It builds `bin/sampletones`.