diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 687508232..6787bc557 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,7 +27,7 @@ jobs: enable-cache: true - name: Install system libraries - run: bash scripts/linux/build/dependencies.sh + run: python3 scripts/system_dependencies.py - name: Install the development environment run: uv sync --group dev @@ -52,6 +52,8 @@ jobs: matrix: os: [ubuntu-latest, windows-latest, macos-latest] python: ["3.12", "3.13"] + env: + PYTHON: ${{ matrix.os == 'windows-latest' && 'python' || 'python3' }} steps: - uses: actions/checkout@v7 @@ -63,22 +65,28 @@ jobs: - name: Install system libraries (Linux) if: runner.os == 'Linux' - run: bash scripts/linux/build/dependencies.sh + run: python3 scripts/system_dependencies.py - name: Install system libraries (macOS) if: runner.os == 'macOS' run: | - bash scripts/macos/build/dependencies.sh - bash scripts/macos/build/build_env.sh >> "$GITHUB_ENV" + python3 scripts/system_dependencies.py + python3 scripts/build_environment.py >> "$GITHUB_ENV" - name: Install the development environment + id: environment run: uv sync --group dev - - name: Run doctests - run: uv run python -m pytest src/ --doctest-modules --no-cov + - name: Run the doctests + shell: bash + run: $PYTHON scripts/run_tests.py doctests - - name: Run the unit and integration suites with coverage - run: uv run python -m pytest -n auto --cov --ignore=tests/benchmarks + - name: Run the test suite with coverage + if: ${{ !cancelled() && steps.environment.outcome == 'success' }} + shell: bash + run: $PYTHON scripts/run_tests.py suite --workers auto - name: Run the benchmarks - run: uv run python -m pytest tests/benchmarks --no-cov + if: ${{ !cancelled() && steps.environment.outcome == 'success' }} + shell: bash + run: $PYTHON scripts/run_tests.py benchmarks diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml index f94d55538..bd22d7aa4 100644 --- a/.github/workflows/workflow.yml +++ b/.github/workflows/workflow.yml @@ -32,10 +32,7 @@ jobs: - name: Verify tag matches project version if: startsWith(github.ref, 'refs/tags/v') - run: | - python3 scripts/ci/checks/version_tag.py \ - --tag "$GITHUB_REF_NAME" \ - --project-version "$(uv version --short)" + run: python3 scripts/verify_version_tag.py --tag "$GITHUB_REF_NAME" - name: Build sdist and wheel run: uv build @@ -78,8 +75,8 @@ jobs: - name: Install PortAudio (macOS) if: runner.os == 'macOS' run: | - bash repository/scripts/macos/build/dependencies.sh - bash repository/scripts/macos/build/build_env.sh >> "$GITHUB_ENV" + python3 repository/scripts/system_dependencies.py + python3 repository/scripts/build_environment.py >> "$GITHUB_ENV" - name: Install the wheel and check the entry point shell: bash @@ -87,10 +84,19 @@ jobs: python -m pip install --upgrade pip python -m pip install dist/*.whl sampletones --version + sampletones --help - name: Check the wheel carries every resource it needs at startup shell: bash - run: sampletones --self-check + run: sampletones self-check + + - name: Check a developer command refuses to run outside a checkout + shell: bash + run: | + if refusal=$(sampletones check import-boundary --all 2>&1); then + exit 1 + fi + grep "runs from a checkout" <<< "$refusal" bundle: name: Standalone bundle (${{ matrix.platform }}) @@ -114,38 +120,19 @@ jobs: - name: Install system libraries (Linux) if: runner.os == 'Linux' - run: bash scripts/linux/build/dependencies.sh + run: python scripts/system_dependencies.py - - name: Create the build environment the bundle scripts expect + - name: Build the bundle shell: bash - run: | - python -m venv .venv-build - if [ "$RUNNER_OS" = "Windows" ]; then - venv_python=.venv-build/Scripts/python.exe - else - venv_python=.venv-build/bin/python - fi - "$venv_python" -m pip install --upgrade pip - "$venv_python" -m pip install ".[build]" --group assets - - - name: Build the bundle (Linux) - if: runner.os == 'Linux' - run: bash scripts/linux/build/build.sh --release - - - name: Build the bundle (Windows) - if: runner.os == 'Windows' - shell: cmd - run: scripts\windows\build\build.bat --release + run: python scripts/bundle.py --release - name: Check the bundle runs and carries its notices shell: bash - run: python scripts/ci/checks/bundle.py bin/sampletones + run: python scripts/verify_bundle.py - name: Zip the bundle shell: bash - run: | - name="sampletones-${GITHUB_REF_NAME}-${{ matrix.platform }}" - python scripts/ci/zip_bundle.py bin/sampletones "bundles/${name}.zip" --root "${name}" + run: python scripts/archive_bundle.py --label "${{ matrix.platform }}" - uses: actions/upload-artifact@v7 with: diff --git a/.gitignore b/.gitignore index 585ea73dc..5d9bf85c5 100644 --- a/.gitignore +++ b/.gitignore @@ -13,10 +13,11 @@ dist/ wheels/ /bin/ /build/ +/bundles/ sampletones !src/sampletones -!tests/sampletones +!tests/unit/sampletones *.pyc *.pyo diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a8eee20e8..a3d32077d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -35,7 +35,7 @@ repos: hooks: - id: import-boundary name: import boundary - entry: uv run scripts/checks/import_boundary.py --all + entry: uv run sampletones check import-boundary --all language: system types: [python] pass_filenames: false @@ -43,7 +43,7 @@ repos: - id: unused-tags name: unused tags - entry: uv run scripts/checks/unused_tags.py + entry: uv run sampletones check unused-tags language: system types: [python] pass_filenames: false @@ -51,7 +51,7 @@ repos: - id: rendered-literals name: rendered literals - entry: uv run scripts/checks/rendered_literals.py + entry: uv run sampletones check rendered-literals language: system files: ^tests/.*\.py$ pass_filenames: false @@ -59,7 +59,7 @@ repos: - id: tag-names name: tag names - entry: uv run scripts/checks/tag_names.py --all + entry: uv run sampletones check tag-names --all language: system types: [python] pass_filenames: false @@ -67,7 +67,7 @@ repos: - id: palette-colors name: palette colors - entry: uv run scripts/checks/palette_colors.py + entry: uv run sampletones check palette-colors language: system files: (^src/sampletones_application/.*\.py|^src/sampletones_config/.*\.yaml)$ pass_filenames: false @@ -75,7 +75,7 @@ repos: - id: language-keys name: language keys - entry: uv run scripts/checks/language_keys.py + entry: uv run sampletones check language-keys language: system files: (\.py|^src/sampletones_config/lang/.*\.yaml)$ pass_filenames: false @@ -83,7 +83,7 @@ repos: - id: shortcut-actions name: shortcut actions - entry: uv run scripts/checks/shortcut_actions.py + entry: uv run sampletones check shortcut-actions language: system files: (\.py|^src/sampletones_config/keybindings/.*\.yaml)$ pass_filenames: false @@ -109,16 +109,16 @@ repos: - id: icons name: icons - entry: uv run python scripts/assets/icons.py + entry: uv run sampletones icons language: system - files: ^src/sampletones_assets/(icons|mark)/ + files: ^src/(sampletones_assets/icons|sampletones_tools/assets/mark)/ pass_filenames: false verbose: true stages: - pre-push - - id: pytest - name: pytest + - id: test + name: test entry: make test language: system types: @@ -126,3 +126,22 @@ repos: stages: - pre-push pass_filenames: false + + - id: test-docs + name: test docs + entry: make test-docs + language: system + files: ^src/.*\.py$ + stages: + - pre-push + pass_filenames: false + + - id: benchmarks + name: benchmarks + entry: make benchmarks + language: system + types: + - python + stages: + - pre-push + pass_filenames: false diff --git a/CHANGELOG.md b/CHANGELOG.md index b30d05826..89b4fcdac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,9 +3,9 @@ ## v0.3.2 * Added NSF player and export. -* Added stems conversion: mix several recordings into one reconstruction -* Matched each stem against its own recording, so a stem plays what was recorded on it -* Added a per-source channel cap +* Added stems conversion: mix several recordings into one reconstruction. +* Matched each stem against its own recording, so a stem plays what was recorded on it. +* Added a per-source channel cap. * Bumped the reconstruction data-version to `2.2` with backward compatibility for `2.1`. ## v0.3.1 [2026-08-18] diff --git a/LICENSE b/LICENSE index 5918803cf..119ae7fc2 100644 --- a/LICENSE +++ b/LICENSE @@ -24,7 +24,7 @@ SOFTWARE. The MIT license above covers the SampleToNES source code and the application icons under `src/sampletones_assets/icons/`, which are drawn from the mark -declared in `src/sampletones_assets/mark/`. +declared in `src/sampletones_tools/assets/mark/`. Font files bundled under `src/sampletones_assets/fonts/` are the work of third parties and remain under their own licenses (SIL Open Font License 1.1 and the diff --git a/Makefile b/Makefile index 6b75d682a..a2e9dee94 100644 --- a/Makefile +++ b/Makefile @@ -1,173 +1,70 @@ -.PHONY: help setup install build release system-deps run clean pre-commit test benchmarks \ - ftm-samples nsf-samples nsf-render compression-report icons player check-import-boundary check-tag-names check-unused-tags check-rendered-literals \ - check-language-keys check-palette-colors check-shortcut-actions calibration lint pylint mypy format +.PHONY: help setup install system-deps build release run clean pre-commit test test-docs benchmarks lint format ifeq ($(OS),Windows_NT) -ifeq ($(MSYSTEM),) -UNAME_S := Windows +PYTHON := python else -UNAME_S := $(shell uname -s) -endif -else -UNAME_S := $(shell uname -s) -endif - -ifeq ($(UNAME_S),Windows) - SCRIPTS_DIR := scripts/windows - SCRIPT_EXT := .bat - RUN_SCRIPT := - BUILD_SCRIPT := install.bat - EXECUTABLE := sampletones.exe - PYTHON := python -else - SCRIPTS_DIR := scripts/linux - SCRIPT_EXT := .sh - RUN_SCRIPT := bash - BUILD_SCRIPT := ./install.sh - EXECUTABLE := sampletones - PYTHON := python3 -endif - -ifeq ($(UNAME_S),Windows) -script = $(subst /,\,$(SCRIPTS_DIR)/$(1)$(SCRIPT_EXT)) -else -script = $(RUN_SCRIPT) $(SCRIPTS_DIR)/$(1)$(SCRIPT_EXT) +PYTHON := python3 endif -ifeq ($(UNAME_S),Windows) +ifeq ($(OS)$(MSYSTEM),Windows_NT) Q := else Q := " endif -BUILD_COMMAND := $(RUN_SCRIPT) $(BUILD_SCRIPT) -RELEASE_COMMAND := $(RUN_SCRIPT) $(BUILD_SCRIPT) --release -SYSTEM_DEPS_COMMAND := bash scripts/linux/build/dependencies.sh -SETUP_ENV := - -ifeq ($(UNAME_S),Darwin) - MACOS_NO_BUNDLE := bash scripts/macos/build/no_bundle.sh - BUILD_COMMAND := $(MACOS_NO_BUNDLE) 'make build' - RELEASE_COMMAND := $(MACOS_NO_BUNDLE) 'make release' - SYSTEM_DEPS_COMMAND := bash scripts/macos/build/dependencies.sh - SETUP_ENV := ARCHFLAGS="-arch $(shell uname -m)" -endif - GPU ?= auto -GPU_EXTRA := -ifeq ($(filter 0,$(GPU)),) -ifneq ($(filter setup,$(MAKECMDGOALS)),) - GPU_EXTRA := $(shell $(PYTHON) scripts/detect_cuda.py --extra) -endif -endif help: @echo $(Q)Available targets:$(Q) @echo $(Q) make setup - Set up development environment (uv); GPU auto-detected, GPU=0 forces CPU$(Q) @echo $(Q) make pre-commit - Install pre-commit hooks$(Q) - @echo $(Q) make system-deps - Install system packages required to build and run (Debian-based, or Homebrew on macOS)$(Q) + @echo $(Q) make system-deps - Install system packages required to build and run (apt on Debian-based Linux, Homebrew on macOS)$(Q) @echo $(Q) make build - Compile standalone executable (development deployment config: DEBUG, strict history)$(Q) @echo $(Q) make release - Compile standalone executable with the release deployment config (INFO, self-healing history)$(Q) - @echo $(Q) make test - Run unit tests with coverage$(Q) - @echo $(Q) make benchmarks - Run the measured-duration suite on its own$(Q) - @echo $(Q) make ftm-samples - Emit example .ftm files to build/ftm via the integration suite$(Q) - @echo $(Q) make nsf-samples - Emit example .nsf files to build/nsf via the integration suite$(Q) - @echo $(Q) make nsf-render - Render the .nsf files in build/nsf to waves with ffmpeg$(Q) - @echo $(Q) make compression-report - Measure the song codec into build/compression$(Q) - @echo $(Q) make icons - Generate the icon suite into src/sampletones_assets/icons$(Q) - @echo $(Q) make player - Assemble the NES player driver with cc65$(Q) - @echo $(Q) make calibration - Score the reconstruction corpus; the report lands in Documents/SampleToNES/calibration$(Q) + @echo $(Q) make test - Run the test suite with coverage$(Q) + @echo $(Q) make test-docs - Run the doctests$(Q) + @echo $(Q) make benchmarks - Run the measured-duration suite$(Q) @echo $(Q) make clean - Remove build artifacts and cache files$(Q) - @echo $(Q) make lint - Run linting (pylint, mypy)$(Q) + @echo $(Q) make lint - Run mypy and pylint (ARGS=--mypy or ARGS=--pylint for one of them)$(Q) @echo $(Q) make format - Auto-format code (isort, black)$(Q) @echo $(Q) make run - Run SampleToNES application$(Q) setup: - $(SETUP_ENV) uv sync --group dev $(if $(GPU_EXTRA),--extra $(GPU_EXTRA),) - $(MAKE) icons - $(SETUP_ENV) uv tool install --force $(if $(GPU_EXTRA),".[$(GPU_EXTRA)]",.) + $(PYTHON) scripts/setup_environment.py --gpu $(GPU) install: $(MAKE) setup $(MAKE) build build: - $(BUILD_COMMAND) + $(PYTHON) scripts/bundle.py release: - $(RELEASE_COMMAND) + $(PYTHON) scripts/bundle.py --release system-deps: - $(SYSTEM_DEPS_COMMAND) + $(PYTHON) scripts/system_dependencies.py run: uv run sampletones clean: - $(call script,build/clean) + $(PYTHON) scripts/clean.py pre-commit: - $(call script,dev/pre_commit) + $(PYTHON) scripts/hooks.py test: - $(call script,dev/tests) - -benchmarks: - uv run python -m pytest tests/benchmarks --no-cov -s - -ftm-samples: export SAMPLETONES_FTM_OUTPUT_DIR := build/ftm -ftm-samples: - uv run python -m pytest tests/integration/famitracker + $(PYTHON) scripts/run_tests.py suite -nsf-samples: export SAMPLETONES_NSF_OUTPUT_DIR := build/nsf -nsf-samples: - uv run python -m pytest tests/integration/nsf +test-docs: + $(PYTHON) scripts/run_tests.py doctests -nsf-render: nsf-samples - uv run scripts/nsf_render.py - -compression-report: export SAMPLETONES_COMPRESSION_OUTPUT_DIR := build/compression -compression-report: - uv run python -m pytest tests/integration/nsf/test_compression_report.py - -icons: - uv run --group assets python scripts/assets/icons.py - -player: - uv run scripts/player.py - -check-import-boundary: - uv run scripts/checks/import_boundary.py --all - -check-tag-names: - uv run scripts/checks/tag_names.py --all - -check-unused-tags: - uv run scripts/checks/unused_tags.py - -check-rendered-literals: - uv run scripts/checks/rendered_literals.py - -check-language-keys: - uv run scripts/checks/language_keys.py - -check-palette-colors: - uv run scripts/checks/palette_colors.py - -check-shortcut-actions: - uv run scripts/checks/shortcut_actions.py - -calibration: - uv run scripts/calibration.py +benchmarks: + $(PYTHON) scripts/run_tests.py benchmarks lint: - $(call script,dev/lint) - -pylint: - $(call script,dev/pylint) - -mypy: - $(call script,dev/mypy) + $(PYTHON) scripts/lint.py $(ARGS) format: - $(call script,dev/format) + $(PYTHON) scripts/formatting.py diff --git a/README.md b/README.md index 170ba6561..defacb2cb 100644 --- a/README.md +++ b/README.md @@ -37,79 +37,19 @@ It supports: ## Installation -### Standalone bundle +You can install _SampleToNES_ in three ways: -The easiest way to use _SampleToNES_ is to download the release package for your platform from the -[releases page](https://github.com/JakimPL/SampleToNES/releases), extract it, and run -`sampletones` from inside the extracted folder. On Linux you may need `chmod +x sampletones`. +- **Download a release** for Windows or Linux from the [releases page](https://github.com/JakimPL/SampleToNES/releases), extract it, and start `sampletones`. +- **Install from PyPI** on Windows, macOS or Linux. You need Python 3.12 or newer: -### Requirements + ```sh + uv tool install sampletones # or: pipx install sampletones + sampletones + ``` -- Windows, macOS, or Linux -- Python 3.12 or newer (https://www.python.org/downloads/) +- **Run from source** with Python 3.12 or newer and [uv](https://docs.astral.sh/uv/): `make system-deps`, `make setup`, then `make run`. -### From PyPI - -The quickest way to get the `sampletones` command. Because _SampleToNES_ is an application -rather than a library, installing it into its own isolated environment is recommended: - -```sh -uv tool install sampletones # or: pipx install sampletones -sampletones # launch the GUI -``` - -A plain `pip install sampletones` into an active virtual environment works too. - -To install with GPU support, request the `gpu` extra (see [GPU acceleration](#gpu-acceleration)): - -```sh -uv tool install "sampletones[gpu]" -``` - -On Linux and macOS, audio playback and file dialogs rely on system libraries that come from -the platform's package manager. Install them first: - -```sh -sudo apt-get install libportaudio2 libasound2 python3-tk # Debian/Ubuntu -brew install portaudio # macOS -``` -### Building the executable yourself - -You only need Python 3.12. - -#### Windows - -1. Install Python 3.12. -2. Double-click `install.bat`. It builds `bin\sampletones.exe`. -3. Double-click `bin\sampletones.exe` to start. - -#### Linux - -1. Install the audio and file-dialog system packages: `make system-deps` (or run `./scripts/linux/build/dependencies.sh`). -2. Install Python 3.12, then run `./install.sh` in a terminal. It builds a `bin/sampletones` executable. -3. Run `./bin/sampletones` to start. - -### Run from source - -For development. Requires [uv](https://docs.astral.sh/uv/) (and, on Linux, the system packages from the Linux steps above): - -```sh -make setup # create the environment and install the sampletones command -make run # run the app -``` - -To update the global command after pulling new changes, re-run `make setup` (or `uv tool install --force .`). - -### GPU acceleration - -_SampleToNES_ can use an NVIDIA GPU (via [_CuPy_](https://cupy.dev/) and CUDA) to speed up instruction-library generation and reconstruction. `make setup` detects your NVIDIA driver and installs the matching CuPy build automatically: - -```sh -make setup # installs GPU support when a supported driver is present -make setup GPU=0 # forces the CPU (NumPy) backend -``` - -A current NVIDIA driver is all you need — the CUDA components ship with the CuPy build, on Linux and Windows alike. On macOS the app runs on the CPU. +An NVIDIA graphics card can speed up conversion. The [installation guide](https://github.com/JakimPL/SampleToNES/blob/main/docs/guide/installation.md) covers the system libraries each way needs, GPU support, and building a standalone app yourself. ## Usage @@ -123,19 +63,20 @@ Your configuration, instruction libraries (`.ins`), and reconstructions (`.stn`) ### Command line -You can run without the GUI to use a custom config, generate an instruction library, or reconstruct a file: +Every operation is a named command, and `sampletones` alone starts the interface: ```sh -sampletones --config # run with a custom config -sampletones --generate --config # generate an instruction library -sampletones --config --output # reconstruct an audio file +sampletones run --config # start with a custom config +sampletones open # start with a project, reconstruction or library loaded +sampletones convert --config -o # reconstruct a recording without the GUI +sampletones library --config # generate an instruction library ``` -Run `sampletones --help` for all options. +Run `sampletones --help` for the commands and `sampletones --help` for a command's options. The [command-line guide](https://github.com/JakimPL/SampleToNES/blob/main/docs/guide/command-line.md) explains them. ## Documentation -Internals — the reconstruction algorithms, file formats, the Python API, and developer notes — live in [`docs/`](https://github.com/JakimPL/SampleToNES/tree/main/docs). +The [user guide](https://github.com/JakimPL/SampleToNES/tree/main/docs/guide) explains how to use the app, step by step. The rest of [`docs/`](https://github.com/JakimPL/SampleToNES/tree/main/docs) covers the reconstruction algorithms, the file formats, the Python API and the developer notes. ## License diff --git a/THIRD-PARTY-NOTICES.md b/THIRD-PARTY-NOTICES.md index 99cd24b32..f702953e4 100644 --- a/THIRD-PARTY-NOTICES.md +++ b/THIRD-PARTY-NOTICES.md @@ -112,7 +112,7 @@ CUDA components from their publishers straight to your machine. ## Build-time tooling -The application icons are drawn by `sampletones_assets.mark` and rasterized with +The application icons are drawn by `sampletones_tools.assets.mark` and rasterized with [Pillow](https://pypi.org/project/Pillow/), which is under the [MIT-CMU license](https://github.com/python-pillow/Pillow/blob/main/LICENSE). Pillow belongs to the `assets` dependency group alone, so `pip`/`uv` installs it on the machine that diff --git a/docs/api/index.md b/docs/api/index.md index a2752db75..2fa75e32f 100644 --- a/docs/api/index.md +++ b/docs/api/index.md @@ -1,8 +1,11 @@ # Python API -This page is for using _SampleToNES_ as a library from your own Python code. Everything you need is re-exported from the top-level `sampletones` package — its facade — so `from sampletones import ...` is the whole public surface. Consult this page when you want to render instructions, generate a library, or run and export a reconstruction outside the application. +This page is for using _SampleToNES_ as a library in your own Python code. Use it to render instructions, generate a library, or run and export a reconstruction outside the application. -A few lower-level helpers used in the examples (`write_wave`, `generate_library`) are not part of the facade; they are imported from `sampletones_core` with their full path, as shown. +Names come from two packages: + +- The names in the table below come from `sampletones`: `from sampletones import ...`. +- The examples also use a few helpers from `sampletones_core`: `write_wave`, `generate_library`, `DEFAULT_CHANNELS`, and the FamiTracker instrument writers. Import them with the full path each example shows. ## Public surface @@ -77,13 +80,13 @@ A reconstruction searches an [instruction library](../formats/instruction-librar ```python from sampletones import Config -from sampletones_core.scripts.library import generate_library +from sampletones_core.headless.library import generate_library config = Config.load("config.json") generate_library(config) # renders every instruction and writes the .ins library ``` -The same step is reached from the application's _Instructions_ tab, or on the command line with `sampletones --generate --config config.json`. +The same step is reached from the application's _Instructions_ tab, or on the command line with `sampletones library --config config.json`. ### Reconstruct a sample @@ -121,15 +124,19 @@ reconstruction = Reconstruction.load("reconstruction.stn") ### Export instruments -`Reconstruction.export` returns the per-channel [features](../formats/instruction-libraries.md), one entry per channel, and each set saves as a FamiTracker `.fti` instrument: +`Reconstruction.export` returns the [envelopes](../formats/famitracker.md#b-the-2a03-instrument) of each channel. `build_instrument` turns one channel's envelopes into a FamiTracker instrument, and `write_fti` saves it as an `.fti` file: ```python from sampletones import Reconstruction +from sampletones_core.formats.famitracker.builder import build_instrument +from sampletones_core.formats.famitracker.instrument import write_fti +from sampletones_core.formats.famitracker.specification.instruments import STANDALONE_INSTRUMENT_INDEX reconstruction = Reconstruction.load("reconstruction.stn") -for name, features in reconstruction.export().items(): - features.save(f"{name}.fti", instrument_name=str(name)) +for channel, features in reconstruction.export().items(): + instrument = build_instrument(STANDALONE_INSTRUMENT_INDEX, channel.value, features) + write_fti(f"{channel.value}.fti", instrument) ``` -This writes one `.fti` per channel. A complete FamiTracker `.ftm` module is assembled from a project in the application, not from a single reconstruction — see [FamiTracker formats](../formats/famitracker.md). +This writes one `.fti` per channel, named after the channel. A complete FamiTracker `.ftm` module is assembled from a project in the application, not from a single reconstruction — see [FamiTracker formats](../formats/famitracker.md). diff --git a/docs/concepts/calibration.md b/docs/concepts/calibration.md index 37d63ef4a..dfff8c513 100644 --- a/docs/concepts/calibration.md +++ b/docs/concepts/calibration.md @@ -6,13 +6,13 @@ spectral/temporal blend, the Viterbi transition weights — and their best value an empirical question. **Calibration** answers it with a repeatable experiment: reconstruct a fixed probe corpus under several candidate configurations and let independent judges score the results, so configuration decisions rest on measured -quality and targeted listening. The harness lives in `sampletones_core.calibration` -and runs as a script: +quality and targeted listening. The harness lives in `sampletones_tools.calibration` +and runs as a developer command from a checkout: ``` -python scripts/calibration.py [--config ] [--methods fft,cqt] +uv run sampletones calibration [--config ] [--methods fft,cqt] [--perceptual-exponents 0.5,1.0] [--temporal-weights 0.1,0.3] - [--channels pulse1,triangle,noise] + [--channels pulse1,triangle,noise] [--output ] ``` The base configuration comes from `--config` when given; otherwise the saved @@ -28,10 +28,10 @@ It has three moving parts: signals in six categories: steady tones across the pitch range, pulse timbres of several duty cycles, white and dark noise, tone-plus-noise mixes, percussive transients (snare, kick, pluck) and a crescendo probing the dynamic - range. Each probe is a `sampletones_synthesis` voice — oscillators, envelopes + range. Each probe is a `sampletones_tools.synthesis` voice — oscillators, envelopes and filters composed from one shared, exactly-rendered configuration vocabulary — built from the probe families in - `sampletones_config/calibration/corpus.yaml`. Each category isolates one kind of decision the criterion must get + `sampletones_tools/calibration/config/corpus.yaml`. Each category isolates one kind of decision the criterion must get right — pitch, timbre, noise balance, attack sharpness, level tracking — and the fixed seed makes every run bit-identical, so scores are comparable across runs and code changes. @@ -51,7 +51,7 @@ It has three moving parts: resolves with a single frame length. Band energies are floored at a fixed audibility range below the reference's loudest band, so the score reflects audible content and holds steady under a common gain. Its tuning is a - `RefereeConfig` loaded from `sampletones_config/calibration/referee.yaml`. + `RefereeConfig` loaded from `sampletones_tools/calibration/config/referee.yaml`. When the [zimtohrli](https://github.com/google/zimtohrli) binary is installed it joins automatically as a second, psychoacoustic referee. diff --git a/docs/concepts/compression.md b/docs/concepts/compression.md index 9fab475a1..95889a817 100644 --- a/docs/concepts/compression.md +++ b/docs/concepts/compression.md @@ -255,7 +255,7 @@ minutes at 60 Hz**, against the 49 seconds a record per tick reaches. Encoding i about two seconds; decoding it costs the console around twenty instructions per plane per tick, comfortably inside a video frame. -`make compression-report` writes this table over a corpus of songs, and the format's +`uv run sampletones codec report` writes this table over a corpus of songs, and the format's constants are settled from it. Two of them were settled against expectation: splitting the duty cycle out of the control byte into a plane of its own **costs** 14 %, because volume and duty turn over together and a split pays two opcodes for what one covers; @@ -264,11 +264,13 @@ transposition it makes possible. ## 7. Limitations -- **The search struggles on dense reconstructions.** A reconstruction whose planes turn - over at nearly every tick offers enormous numbers of candidates, and past a cap the - search stops gathering and earns nothing for that song. It bites at lengths where the - song already exceeds the program area, so it costs a diagnosis rather than a song, but - a dense export of two minutes is stored materially larger than the same song at one. +- **The search reads a dense plane only so far.** A reconstruction whose planes turn over + at nearly every tick offers enormous numbers of candidates, and a round gathers a fixed + number of them, shared among the planes by what each has to offer. A plane that spells out + little is read whole; a dense one is read as far as its share reaches, and the figures + beyond that point go unsearched. It bites at lengths where the song already exceeds the + program area, so it costs a diagnosis rather than a song, but a dense export of two + minutes is stored materially larger than the same song at one. - **The dictionary holds 255 phrases.** A project of roughly 30 to 60 samples fills it from its instruments alone, at which point the search has no room left to work in and further samples compete for slots on measured value. @@ -303,6 +305,7 @@ Where things live: | the dictionary, its entries and its pruning | `sampletones_player.compression.dictionary` | | phrases the instruments offer | `sampletones_player.compression.seeds` | | phrases the search earns | `sampletones_player.compression.search` | +| how much work the search spends | `sampletones_player.compression.budget` | | what a phrase plays against a plane | `sampletones_player.compression.matches` | | encoding, and the decoder the driver is held to | `sampletones_player.compression.encode`, `.decode` | | the opcode layout and its bounds | `sampletones_player.specification.compression` | diff --git a/docs/concepts/instruction-library.md b/docs/concepts/instruction-library.md index c0890acd3..374bd6dc8 100644 --- a/docs/concepts/instruction-library.md +++ b/docs/concepts/instruction-library.md @@ -52,7 +52,7 @@ representation so their spectra are directly comparable. ## Generating and exploring Generate a library from the _Instructions_ tab, or on the command line with -`sampletones --generate`. Generation renders every instruction and stores its +`sampletones library`. Generation renders every instruction and stores its waveform and spectrum; a configuration that has no library yet is also built automatically the first time a reconstruction needs it. Regenerating a library that already exists replaces it. diff --git a/docs/concepts/reconstruction.md b/docs/concepts/reconstruction.md index 35c522f40..3d56895a8 100644 --- a/docs/concepts/reconstruction.md +++ b/docs/concepts/reconstruction.md @@ -406,5 +406,5 @@ Package map: | audio I/O and level | `sampletones_core.audio` | | tracker export | `sampletones_core.exporters` | | pitch refinement | `sampletones_core.reconstructions.reconstructor.refinement` | -| criterion calibration | `sampletones_core.calibration` | -| analytic waveform synthesis | `sampletones_synthesis` | +| criterion calibration | `sampletones_tools.calibration` | +| analytic waveform synthesis | `sampletones_tools.synthesis` | diff --git a/docs/concepts/stems.md b/docs/concepts/stems.md index f6d377f93..633cd7a2d 100644 --- a/docs/concepts/stems.md +++ b/docs/concepts/stems.md @@ -255,7 +255,7 @@ silences them. channels a stem newly reaches, and closing the document empties the card. 6. **Listening choices stay out of the document.** The ticked set is session state, like every choice that shapes what is heard — see - [Playback](../development/playback.md). Saving the reconstruction records + [Playback](../development/application/playback.md). Saving the reconstruction records the assignment, never the selection. So is the banding: collapsing the levels changes how the card draws, never what it describes. 7. **Removing a recording edits the document.** Where a box steers listening, diff --git a/docs/development/browser.md b/docs/development/application/browser.md similarity index 100% rename from docs/development/browser.md rename to docs/development/application/browser.md diff --git a/docs/development/config-organization.md b/docs/development/application/config-organization.md similarity index 89% rename from docs/development/config-organization.md rename to docs/development/application/config-organization.md index d2994b588..ea2ea0c64 100644 --- a/docs/development/config-organization.md +++ b/docs/development/application/config-organization.md @@ -10,8 +10,8 @@ is read; use it as the reference when adding or moving a value. It sits alongsid first: - **Shipped configuration** — the `sampletones_config` YAML package: layout, theme, - palettes, keybindings, language, behavior, deployment, calibration, and the import - boundaries. *(This document.)* + palettes, keybindings, language, behavior, deployment, and the import boundaries. + *(This document.)* - **Runtime user preferences** — mutable state persisted to the user profile (`sampletones_application/config`, e.g. `PlaybackConfig`, `ShortcutsConfig`, `ApplicationState`), governed by that package. @@ -32,17 +32,23 @@ empty `__init__.py`. Each schema lives with its reader: - `sampletones_application` owns the layout, theme, palettes, keybindings, language, behavior, and deployment schemas. -- `sampletones_core` owns the calibration schemas. -- `sampletones_shared` owns the import-boundary schemas and the loader primitives - (`load_yaml_model`, `load_yaml_model_dir`). +- `sampletones_tools` owns the import-boundary schemas. +- `sampletones_shared` owns the loader primitives (`load_yaml_model`, `load_yaml_model_dir`). So the data carries the values and the consumer carries the meaning, and the two evolve on their own terms. +Data only a developer tool reads ships with that tool, beside the schema reading it: the +calibration tuning sits in `sampletones_tools/calibration/config/`, the synthetic corpus in +`sampletones_tools/corpus/config/` and the application mark in `sampletones_tools/assets/mark/config/`, +each placed by `package_directory`. `sampletones_config` +holds what the application reads and the import boundaries, which state the repository's +package layers for every tree and which [package layers](../packages.md) refers to. + ### 2. The top level is organized by domain `sampletones_config` has one top-level directory per schema family and its loader: -`application`, `behavior`, `boundaries`, `calibration`, `keybindings`, `lang`, `layout`, +`application`, `behavior`, `boundaries`, `keybindings`, `lang`, `layout`, `palettes`, `theme`. Each domain owns its schema and its load path (see [Domains](#domains)). A new domain is a new top-level directory with its own schema owner and loader. @@ -132,8 +138,7 @@ each value sits in the tree stays in the factory. |--------|-----------|--------------|------------------| | Application | `application/` | `DeploymentConfig` (`sampletones_application/config/deployment/`) | `DeploymentConfig.load()`, with `SAMPLETONES_*` env overrides | | Behavior | `behavior/` | `BehaviorConfig` (`sampletones_application/layout/behavior.py`) | folded into `LayoutConfig.behavior` by `load_layout_config` | -| Boundaries | `boundaries/` | `ImportBoundaryRules` (`sampletones_shared/meta/import_boundary/configs/`) | `ImportBoundaryRules.load()` | -| Calibration | `calibration/` | `CorpusConfig`, `RefereeConfig` (`sampletones_core/calibration/config/`) | each model's own `.load()` | +| Boundaries | `boundaries/` | `ImportBoundaryRules` (`sampletones_tools/checks/boundary/configs/`) | `ImportBoundaryRules.load()` | | Keybindings | `keybindings/` | `ShortcutScheme` (`sampletones_application/utils/gui/shortcuts/`) | `ShortcutCatalog.load()`, indexed by scheme name | | Language | `lang/` | `LanguageManager` (`sampletones_application/categories/`) | flat string map keyed `page.panel.text_type.element`, each key validated at load | | Layout | `layout/` | `LayoutConfig` (`sampletones_application/layout/config.py`) | `load_layout_config` (`layout/loader.py`) | @@ -169,12 +174,12 @@ sites across `application.py` and the tab coordinators that read it, and the ~10 that import `SchedulingBehavior` as a type. Boundaries is the domain a developer tool reads. It states the layer graphs the packages -divide into, the imports each part of the application stays clear of, and the spellings a -tree keeps out, and `scripts/checks/import_boundary.py` runs it over the source tree on every -commit. A declaration draws on the named prefix groups `general.yaml` holds, so a set several +divide into, the imports each part of the application stays clear of, the spellings a tree +keeps out, and the standard-library rule the bootstrap scripts under `scripts/` hold to, and +`sampletones check import-boundary` runs it over the source and scripts trees on every commit. A declaration draws on the named prefix groups `general.yaml` holds, so a set several rules reach for is written once and each rule names it, and a name reaching no group is -refused as the domain is read. The bundle carries the domain because `--add-data` copies -`sampletones_config` whole — the terms `calibration/` already ships on. +refused as the domain is read. The bundle carries the domain with the rest of +`sampletones_config`. --- @@ -200,6 +205,6 @@ Three load mechanisms serve the three grouping schemes: file on disk. `ShortcutCatalog.load()` reads `keybindings/` the same way, keyed by `ShortcutScheme.name`. -Deployment, calibration and the boundaries each load through a bespoke `.load()` +Deployment and the boundaries each load through a bespoke `.load()` classmethod over the same low-level primitives in `sampletones_shared/utils/serialization.py` — the one module that calls `yaml.safe_load`. diff --git a/docs/development/keyboard.md b/docs/development/application/keyboard.md similarity index 99% rename from docs/development/keyboard.md rename to docs/development/application/keyboard.md index 5572905cc..9c3b6aeff 100644 --- a/docs/development/keyboard.md +++ b/docs/development/application/keyboard.md @@ -6,7 +6,7 @@ declared and shown. It governs `utils/gui/keyboard/`, `utils/gui/shortcuts/`, th `sampletones_config/keybindings/`, and the menu surfaces that print an action. Consult it when giving a panel keys of its own, adding a shortcut, or putting an action on a menu. -The design truths it realizes are principles 12 and 14 of [`architecture.md`](architecture.md): +The design truths it realizes are principles 12 and 14 of [`architecture.md`](../architecture.md): one dispatcher owns the keyboard, and an action is declared once. This document holds the mechanism behind both. @@ -136,7 +136,7 @@ holds every shipped scheme to the same action set. ## Actions Declaring an action is a chain of four links, and the `shortcut-actions` check holds every one of -them (see [`architecture.md`](architecture.md) § Enforcement): +them (see [`architecture.md`](../architecture.md) § Enforcement): | Link | Where | What it states | |------|-------|----------------| diff --git a/docs/development/palette.md b/docs/development/application/palette.md similarity index 94% rename from docs/development/palette.md rename to docs/development/application/palette.md index 854cf7785..8ec3ed2be 100644 --- a/docs/development/palette.md +++ b/docs/development/application/palette.md @@ -4,7 +4,7 @@ This document describes how a color is written, composed, and handed to DearPyGu palette change costs. It governs `utils/palette/` and `utils/gui/palette/`. Consult it when adding a color the interface draws with, or a shade the interface derives from one. -The design truth it realizes is principle 13 of [`architecture.md`](architecture.md): a color is a +The design truth it realizes is principle 13 of [`architecture.md`](../architecture.md): a color is a token, resolved where it is drawn. This document holds the mechanism. --- @@ -37,4 +37,4 @@ DearPyGui holds as table state. The `palette-colors` hook holds all three rules — an attribute assigned a resolved `rgba`, a theme color filled outside the palette bindings, and a hex literal in the shipped configuration outside -`palettes/` (see [`architecture.md`](architecture.md) § Enforcement). +`palettes/` (see [`architecture.md`](../architecture.md) § Enforcement). diff --git a/docs/development/playback.md b/docs/development/application/playback.md similarity index 99% rename from docs/development/playback.md rename to docs/development/application/playback.md index c7c5b9f4e..0061ae77b 100644 --- a/docs/development/playback.md +++ b/docs/development/application/playback.md @@ -3,7 +3,7 @@ This document governs sound across the application: what may be heard, who decides, and what each transport command means. Consult it when adding audio a user can start, a surface that starts it, or a control over what is heard. The contracts here bind every tab and every player. It complements -`docs/development/architecture.md`, `docs/development/keyboard.md` (which owns the keyboard-routing +`docs/development/architecture.md`, `docs/development/application/keyboard.md` (which owns the keyboard-routing layer) and `docs/development/guidelines.md`. --- diff --git a/docs/development/render-thread.md b/docs/development/application/render-thread.md similarity index 99% rename from docs/development/render-thread.md rename to docs/development/application/render-thread.md index 9049d3eb2..2f04b15b0 100644 --- a/docs/development/render-thread.md +++ b/docs/development/application/render-thread.md @@ -6,7 +6,7 @@ its context, and what each crossing costs. It governs `utils/gui/render_thread.p a worker thread has something to show, when a gesture rebuilds widgets, or when work needs a frame to have been drawn first. -The design truth it realizes is principle 6 of [`architecture.md`](architecture.md): DearPyGui's +The design truth it realizes is principle 6 of [`architecture.md`](../architecture.md): DearPyGui's context belongs to the render thread. This document holds the mechanism. --- diff --git a/docs/development/sequencer-blocks.md b/docs/development/application/sequencer-blocks.md similarity index 98% rename from docs/development/sequencer-blocks.md rename to docs/development/application/sequencer-blocks.md index f8b779cfe..314167f89 100644 --- a/docs/development/sequencer-blocks.md +++ b/docs/development/application/sequencer-blocks.md @@ -7,8 +7,8 @@ and both grids — the tracker's pattern rows and the order's frames — carry t This document states the rules those gestures follow, how a block leaves the app as text, how a selection is drawn, and how a grid's actions reach the menus and the keyboard that fire them. The layering they sit in is -[Architecture](architecture.md); the conventions the code is held to are the -[coding guidelines](guidelines.md). +[Architecture](../architecture.md); the conventions the code is held to are the +[coding guidelines](../guidelines.md). ## Three vocabularies, kept apart @@ -123,7 +123,7 @@ SampleToNES/1 order rows=1 positions=0..1 The form and its reading live in `logic/sequencer/clipboard/`, which deals in blocks and strings alone; the desktop's clipboard is reached through `utils/gui/clipboard.py::TextClipboard`, one more piece of external behavior standing behind -a protocol ([Architecture](architecture.md), principle 11). The sequencer coordinator wires +a protocol ([Architecture](../architecture.md), principle 11). The sequencer coordinator wires the two. **A field prints what the grid prints in its cell**, which is what carries the three states @@ -172,7 +172,7 @@ needs no selection made first, and a menu raised inside a selection acts on the One builder means an action added to a grid appears at every door, and the accelerator **Edit** prints is the one that grid answers to, since a binding is declared once and every -reader of it reads that entry ([Architecture](architecture.md), principle 12). +reader of it reads that entry ([Architecture](../architecture.md), principle 12). `EditRouter` (`coordinators/edit/`) is the menu-side counterpart of the `KeyRouter` the keyboard runs through. Each surface states whether it owns the editing gestures at this diff --git a/docs/development/undo.md b/docs/development/application/undo.md similarity index 98% rename from docs/development/undo.md rename to docs/development/application/undo.md index 21fc240fc..1321705ed 100644 --- a/docs/development/undo.md +++ b/docs/development/application/undo.md @@ -87,7 +87,7 @@ cursor and repaints rows in place via an index-keyed diff. values, so an edit path reaching the project outside a transaction raises `UntrackedMutationError` at once — in a development run and in the test suite alike, since several tests build the whole application and read that file. A user build takes the opposite -values from `scripts/release_env_hook.py`, so a gap that reaches a release is healed into an +values from `scripts/runtime_hooks/release_environment.py`, so a gap that reaches a release is healed into an `UNTRACKED` entry rather than shown to the user. The gap therefore surfaces where it can be fixed and stays quiet where it cannot. diff --git a/docs/development/vocabularies.md b/docs/development/application/vocabularies.md similarity index 99% rename from docs/development/vocabularies.md rename to docs/development/application/vocabularies.md index 8b8f4690c..1181ff4ac 100644 --- a/docs/development/vocabularies.md +++ b/docs/development/application/vocabularies.md @@ -6,7 +6,7 @@ are held to the source whole-tree by a pre-commit hook, and both keep in one pla otherwise be restated at every use. Consult this document when adding a string the reader sees, or a widget another module reaches. -The design truths it realizes are principles 8 and 9 of [`architecture.md`](architecture.md): all +The design truths it realizes are principles 8 and 9 of [`architecture.md`](../architecture.md): all display text comes from `LanguageManager`, and `tags/` holds only DPG identifiers. This document holds the grammar behind both. diff --git a/docs/development/architecture.md b/docs/development/architecture.md index 79d215183..2bbfd250a 100644 --- a/docs/development/architecture.md +++ b/docs/development/architecture.md @@ -2,7 +2,7 @@ This document describes the design of `sampletones_application` — the GUI front-end of _SampleToNES_. It is prescriptive: it states the contracts each layer must honor, in the form they are enforced, and the rationale behind them. Use it as the reference when deciding where new code belongs. -Concrete classes and modules appear throughout as **examples** that anchor a rule; the rules bind every instance, named or not. Known deviations from these contracts are tracked in `docs/development/bugs-and-todos.md`. Coding-level rules live in `docs/development/guidelines.md`; the undo subsystem has its own design document, `docs/development/undo.md`, the audio transport has `docs/development/playback.md`, the reconstruction browser has `docs/development/browser.md`, the YAML configuration package has `docs/development/config-organization.md`, how a long operation says how far it has come has `docs/development/progress.md`, the keyboard and the actions it reaches have `docs/development/keyboard.md`, the identifier vocabularies have `docs/development/vocabularies.md`, colors and palettes have `docs/development/palette.md`, and the packages the repository divides into have `docs/development/packages.md`. +Concrete classes and modules appear throughout as **examples** that anchor a rule; the rules bind every instance, named or not. Known deviations from these contracts are tracked in `docs/development/bugs-and-todos.md`. Coding-level rules live in `docs/development/guidelines.md`; the undo subsystem has its own design document, `docs/development/application/undo.md`, the audio transport has `docs/development/application/playback.md`, the reconstruction browser has `docs/development/application/browser.md`, the YAML configuration package has `docs/development/application/config-organization.md`, how a long operation says how far it has come has `docs/development/progress.md`, the keyboard and the actions it reaches have `docs/development/application/keyboard.md`, the identifier vocabularies have `docs/development/application/vocabularies.md`, colors and palettes have `docs/development/application/palette.md`, and the packages the repository divides into have `docs/development/packages.md`. --- @@ -41,7 +41,7 @@ These principles govern every structural decision in the codebase. Each layer imports only from the layers below it. Coordinators, at the top, reach every layer they orchestrate. The UI layer knows only view models and shared utilities. Logic owns domain state and produces view models. Services, at the bottom of the application stack, know only the core libraries and thread-safe utilities — a service is driven through a logic-side `Protocol` and reports through its result-contract types, so even logic reaches a service only through inversion. -The load-bearing prohibitions: nothing in `logic/` or `services/` imports `ui/` or `coordinators/`, and `services/` imports neither `logic/` nor `view_model/`. The authoritative import matrix is the **May import / Must not import** pair in each Layer Reference section below; the boundary script enforces it (see Enforcement). +The load-bearing prohibitions: nothing in `logic/` or `services/` imports `ui/` or `coordinators/`, and `services/` imports neither `logic/` nor `view_model/`. The authoritative import matrix is the **May import / Must not import** pair in each Layer Reference section below; the boundary check enforces it (see Enforcement). ### 2. DPG stays in the visual layers @@ -67,7 +67,7 @@ This decouples widget construction (which happens during `create_panel()`) from ### 6. DearPyGui's context belongs to the render thread -The thread that created the DearPyGui context is the only one that may build, configure, or destroy an item, and an item freed from another thread is freed with no Python thread state — a crash rather than a glitch. So work reaching the interface from anywhere else arrives on that thread first, through a crossing named for what it carries: a background result is queued for the render loop to drain, a worker's touch of a widget goes through `on_render_thread`, and a gesture DearPyGui gathered is run at the top of a frame. A crossing that would hold the frames up puts its waiting on a thread of its own, and work that needs a drawn frame names the frame it is picked up on. The four crossings, the helpers that make them, and the hazard each one answers are in [`render-thread.md`](render-thread.md). +The thread that created the DearPyGui context is the only one that may build, configure, or destroy an item, and an item freed from another thread is freed with no Python thread state — a crash rather than a glitch. So work reaching the interface from anywhere else arrives on that thread first, through a crossing named for what it carries: a background result is queued for the render loop to drain, a worker's touch of a widget goes through `on_render_thread`, and a gesture DearPyGui gathered is run at the top of a frame. A crossing that would hold the frames up puts its waiting on a thread of its own, and work that needs a drawn frame names the frame it is picked up on. The four crossings, the helpers that make them, and the hazard each one answers are in [`render-thread.md`](application/render-thread.md). ### 7. Construction flows from the composition root @@ -77,11 +77,11 @@ The thread that created the DearPyGui context is the only one that may build, co ### 8. All display text comes from `LanguageManager` -Every user-visible string is looked up on `LanguageManager` by the key the language file spells — `page.panel.text_type.element` — and resolves at the point of use, so a language change takes effect on the next read. `en.yaml` is a flat map keyed exactly this way, which makes the text system the single source of truth, lets a reader hold a key against the language file by eye, and enables future localization. A lookup states its key in a form the `language-keys` hook can read, so every key the code spells names an entry and every entry the file holds is reached. The grammar, the forms a lookup takes, and where each element enum lives are in [`vocabularies.md`](vocabularies.md). Log messages are developer-facing and exempt. +Every user-visible string is looked up on `LanguageManager` by the key the language file spells — `page.panel.text_type.element` — and resolves at the point of use, so a language change takes effect on the next read. `en.yaml` is a flat map keyed exactly this way, which makes the text system the single source of truth, lets a reader hold a key against the language file by eye, and enables future localization. A lookup states its key in a form the `language-keys` hook can read, so every key the code spells names an entry and every entry the file holds is reached. The grammar, the forms a lookup takes, and where each element enum lives are in [`vocabularies.md`](application/vocabularies.md). Log messages are developer-facing and exempt. ### 9. `tags/` holds only DPG identifiers -The `tags/` package contains only DPG widget string identifiers: `TAG_*` whole tags, and `SUF_*`/`PRE_*` fragments that compose into them. Dimensions, colors, timings, and display strings live in YAML configuration loaded at startup (`layout/`). Every tag reaches its final spelling through one composer, and a constant's name states the tag it composes, which the `tag-names` hook holds it to. The composer, the `TagName` spelling, and the rules a fragment follows are in [`vocabularies.md`](vocabularies.md). +The `tags/` package contains only DPG widget string identifiers: `TAG_*` whole tags, and `SUF_*`/`PRE_*` fragments that compose into them. Dimensions, colors, timings, and display strings live in YAML configuration loaded at startup (`layout/`). Every tag reaches its final spelling through one composer, and a constant's name states the tag it composes, which the `tag-names` hook holds it to. The composer, the `TagName` spelling, and the rules a fragment follows are in [`vocabularies.md`](application/vocabularies.md). ### 10. Exclusive operations expose a lifecycle-accurate active state @@ -98,7 +98,7 @@ A new exclusive operation joins by contributing its `is_active` to the authority Where behavior depends on the operating system, the desktop environment, or an external command-line tool, that variation is expressed as a `Protocol` with one implementation per target, chosen by a runtime factory — never as platform branches scattered through the callers. The factory probes availability (`locate_program`) and environment (`System.current()`, `XDG_CURRENT_DESKTOP`) and returns the implementation that fits; callers depend only on the Protocol and read identically on every platform. -Each tool's quirks stay sealed inside its own implementation and are named in that class's docstring, where a reader meets them beside the code they explain; the guarantee callers depend on — that a saved file carries one of the offered extensions — is enforced once in the API layer above every backend. `utils/file_dialogs/` applies this to native file dialogs: a `FileDialogBackend` Protocol in `protocol.py`, with desktop-portal, `kdialog`, `zenity`, and `tkinter` implementations under `backends/`, selected by `select_file_dialog_backend()`. `sampletones_core/calibration/referee/` follows the same shape with its `build_referees()` factory. +Each tool's quirks stay sealed inside its own implementation and are named in that class's docstring, where a reader meets them beside the code they explain; the guarantee callers depend on — that a saved file carries one of the offered extensions — is enforced once in the API layer above every backend. `utils/file_dialogs/` applies this to native file dialogs: a `FileDialogBackend` Protocol in `protocol.py`, with desktop-portal, `kdialog`, `zenity`, and `tkinter` implementations under `backends/`, selected by `select_file_dialog_backend()`. `sampletones_tools/calibration/referee/` follows the same shape with its `build_referees()` factory. Ordering the implementations is part of the factory's job: where several are available, the one that expresses the most wins. A save offering several file types is answered by the portal because it alone reports which type was chosen, so an export names its format in the type selector; a backend answering with a name alone leaves the extension to be read from the name, and the API layer settles it either way. @@ -108,17 +108,17 @@ DearPyGui gives every key handler the same global reach, so priority and consume A binding is declared once and read by everyone who prints or fires it: `ShortcutId` names the action together with the category that answers it, and the scheme under `sampletones_config/keybindings/` decides the combination, so a printed key and the handler behind it stay in step by construction. -The router is constructed at the composition root and injected into every consumer (principle 7). The scopes, the focus query, the modal stack, the key vocabulary, and how a scheme is chosen, layered, and edited are in [`keyboard.md`](keyboard.md). +The router is constructed at the composition root and injected into every consumer (principle 7). The scopes, the focus query, the modal stack, the key vocabulary, and how a scheme is chosen, layered, and edited are in [`keyboard.md`](application/keyboard.md). ### 13. A color is a token, resolved where it is drawn -A color is written as a palette token and stays one until it reaches DearPyGui. `BaseColor` (`utils/palette/colors/`) carries what was written, and its `rgba` property answers with the palette active at the moment of the read, so whoever holds the color follows a palette swap. Every annotation names `BaseColor`; the read happens where the value is handed to a widget, and what a consumer keeps is the token. What DearPyGui has already taken a copy of is registered with `PaletteBindings` rather than remembered by whoever set it, so a palette change is one switch. The `palette-colors` hook holds all three rules (see Enforcement); the color forms and the switch itself are in [`palette.md`](palette.md). +A color is written as a palette token and stays one until it reaches DearPyGui. `BaseColor` (`utils/palette/colors/`) carries what was written, and its `rgba` property answers with the palette active at the moment of the read, so whoever holds the color follows a palette swap. Every annotation names `BaseColor`; the read happens where the value is handed to a widget, and what a consumer keeps is the token. What DearPyGui has already taken a copy of is registered with `PaletteBindings` rather than remembered by whoever set it, so a palette change is one switch. The `palette-colors` hook holds all three rules (see Enforcement); the color forms and the switch itself are in [`palette.md`](application/palette.md). ### 14. An action is declared once; whoever shows it prints it An **action** is one `ShortcutId` — the name a key press, a menu item, and a context item all reach one behavior by. Declaring one is a chain of four links: the action and the category that answers it, its keys in every shipped scheme, the one call it makes, and the label the keybindings editor lists it by. The `shortcut-actions` check holds every link (see Enforcement). -A menu item is a view of an action: `ShortcutManager.add_menu_item(shortcut_id, ...)` takes both the accelerator and the call from the action and keeps the item under it, so a rebind re-prints the key already on screen. A set of actions several menus show is stated by one builder belonging to whoever owns them, and each door decides where to print it. A menu whose contents follow a selection states them when it is opened. The four links, the kinds of action that state their call differently, and the mechanism behind a restated menu are in [`keyboard.md`](keyboard.md). +A menu item is a view of an action: `ShortcutManager.add_menu_item(shortcut_id, ...)` takes both the accelerator and the call from the action and keeps the item under it, so a rebind re-prints the key already on screen. A set of actions several menus show is stated by one builder belonging to whoever owns them, and each door decides where to print it. A menu whose contents follow a selection states them when it is opened. The four links, the kinds of action that state their call differently, and the mechanism behind a restated menu are in [`keyboard.md`](application/keyboard.md). --- @@ -126,20 +126,20 @@ A menu item is a view of an action: `ShortcutManager.add_menu_item(shortcut_id, Two mechanisms keep the codebase aligned with this document. -**Import-expressible contracts are enforced by a check.** `sampletones_config/boundaries/rules.yaml` states one rule per layer, mirroring the **Must not import** lists in the Layer Reference; the Layer Reference is the source of truth, and a divergence between it and the configuration is itself a defect. The same domain holds the order the repository's packages import each other in, and the layering inside `sampletones_player`, both declared as layer tables in `docs/development/packages.md`. `sampletones_config/boundaries/` declares what the boundaries are, `sampletones_shared/meta/import_boundary/` holds how they are read and reported, and `scripts/checks/import_boundary.py` (a pre-commit hook, also run via `make check-import-boundary`) runs them over the source tree. A rule names the prefixes it reaches through the groups `boundaries/general.yaml` declares, so the interface several layers stay clear of is written once and each rule names it. Where a layer may consume another layer's data contract while its implementation stays out of reach (logic and the service result types), the rule names the contracts group that stays in reach. The hook audits the entire source tree on every commit (`--all`), so strengthening a rule surfaces violations in files a commit never touched. That property sets the working idiom for structural refactors: turn the stricter rule on first, and let the failing hook enumerate the remaining work. +**Import-expressible contracts are enforced by a check.** `sampletones_config/boundaries/rules.yaml` states one rule per layer, mirroring the **Must not import** lists in the Layer Reference; the Layer Reference is the source of truth, and a divergence between it and the configuration is itself a defect. The same domain holds the order the repository's packages import each other in, and the layering inside `sampletones_player`, both declared as layer tables in `docs/development/packages.md`. `sampletones_config/boundaries/` declares what the boundaries are, `sampletones_tools/checks/boundary/` holds how they are read and reported, and `sampletones check import-boundary` (a pre-commit hook, run as `uv run sampletones check import-boundary --all`) runs them over the source and scripts trees. A rule names the prefixes it reaches through the groups `boundaries/general.yaml` declares, so the interface several layers stay clear of is written once and each rule names it. Where a layer may consume another layer's data contract while its implementation stays out of reach (logic and the service result types), the rule names the contracts group that stays in reach. The hook audits the entire source tree on every commit (`--all`), so strengthening a rule surfaces violations in files a commit never touched. That property sets the working idiom for structural refactors: turn the stricter rule on first, and let the failing hook enumerate the remaining work. -**The identifier vocabularies, the declarations that complete them, and the shapes a case may not take are enforced the same way.** Further scripts under `scripts/checks/` run whole-tree as pre-commit hooks, each also available as a `make check-*` target: +**The identifier vocabularies, the declarations that complete them, and the shapes a case may not take are enforced the same way.** Further checks, each a `sampletones check ` command under `sampletones_tools/checks/`, run whole-tree as pre-commit hooks: -| Hook | Script | What it holds | -|------|--------|---------------| -| `language-keys` | `language_keys.py` | Code and `en.yaml` against each other, in both directions: a literal key names an entry, every entry is reached by some lookup, and a lookup states values the check can read (principle 8) | -| `tag-names` | `tag_names.py` | A tag constant's name against the tag it composes (principle 9) | -| `unused-tags` | `unused_tags.py` | Every `TAG_*`/`SUF_*`/`PRE_*` the `tags/` package declares against the reads of it across `src/`, `tests/`, and `scripts/`, where an import alone stands at no reads | -| `palette-colors` | `palette_colors.py` | A color as a token up to the moment it is drawn with: an attribute assigned a resolved `rgba`, a theme color filled outside the palette bindings, and a hex literal in the shipped configuration outside `palettes/` (principle 13) | -| `shortcut-actions` | `shortcut_actions.py` | Every action against the links it needs: a combination in every shipped scheme, a name the keybindings editor lists it by, and — for an application-scope action — the call it makes, whether its own binding or a family (principle 14) | -| `rendered-literals` | `rendered_literals.py` | A case against the text it renders: an equality holding `str(...)` or an f-string against a written-out string pins whatever the platform or the build decided (`guidelines.md` § Tests) | +| Hook | Command | What it holds | +|------|---------|---------------| +| `language-keys` | `check language-keys` | Code and `en.yaml` against each other, in both directions: a literal key names an entry, every entry is reached by some lookup, and a lookup states values the check can read (principle 8) | +| `tag-names` | `check tag-names --all` | A tag constant's name against the tag it composes (principle 9) | +| `unused-tags` | `check unused-tags` | Every `TAG_*`/`SUF_*`/`PRE_*` the `tags/` package declares against the reads of it across `src/`, `tests/`, and `scripts/`, where an import alone stands at no reads | +| `palette-colors` | `check palette-colors` | A color as a token up to the moment it is drawn with: an attribute assigned a resolved `rgba`, a theme color filled outside the palette bindings, and a hex literal in the shipped configuration outside `palettes/` (principle 13) | +| `shortcut-actions` | `check shortcut-actions` | Every action against the links it needs: a combination in every shipped scheme, a name the keybindings editor lists it by, and — for an application-scope action — the call it makes, whether its own binding or a family (principle 14) | +| `rendered-literals` | `check rendered-literals` | A case against the text it renders: an equality holding `str(...)` or an f-string against a written-out string pins whatever the platform or the build decided (`guidelines.md` § Tests) | -They read the source as an AST through the shared layer in `sampletones_shared/meta/source/`, which discovers modules, resolves the receiver a subscript sits on, and expands an enum-annotated key part to its members; the palette and shortcut checks read the shipped YAML beside it. That layer derives each package directory from its own location and reports a root it finds nothing at, so a check that sweeps nothing fails loudly where it would otherwise pass clean. Because the checks are global by nature — a dead entry and an unread fragment are both absences — the hooks pass whole-tree rather than filenames. +They read the source as an AST through the source layer in `sampletones_tools/checks/source/`, which discovers modules, resolves the receiver a subscript sits on, and expands an enum-annotated key part to its members; the palette and shortcut checks read the shipped YAML beside it. That layer derives each package directory from its own location and reports a root it finds nothing at, so a check that sweeps nothing fails loudly where it would otherwise pass clean. Because the checks are global by nature — a dead entry and an unread fragment are both absences — the hooks pass whole-tree rather than filenames. **Behavioral contracts are enforced by review.** Contracts a grep cannot see — where state lives, which methods touch DPG, how errors travel — are upheld in code review against this document. A change that alters one of them lands with the edit stating the new contract, and one that knowingly leaves a distance behind lands with an entry in `docs/development/bugs-and-todos.md § Architecture` — `guidelines.md` § Documents holds that rule. The ledger, not the codebase, is the memory of what is currently out of line. @@ -166,7 +166,7 @@ They read the source as an AST through the shared layer in `sampletones_shared/m | Path | Role | |------|------| | `ui/elements/` | Reusable low-level widgets: `GUIPanel` (the panel base class), `GUIWindow` (modal variant), buttons, tables, graphs, trees, fonts, the status bar, and `MenuSection` — a run of menu items restated each time its menu is opened | -| `ui/elements/layout/` | Reusable layout primitives: `TabColumns` (the tab column scaffold), the `card()` context manager and the `well()` inset region, driven declaratively by tab coordinators | +| `ui/elements/layout/` | Reusable layout primitives: `TabColumns` (the tab column scaffold), the `card()` context manager and the `well()` inset region, driven declaratively by tab coordinators, and `centered()`, which stands content in the middle of the width it is offered | | `ui/panels/` | Domain-level composite panels, organized by feature area | | `ui/themes/` | DPG themes and per-widget style helpers | | `ui/resources/` | Icons and image resources loaded at startup | @@ -207,9 +207,9 @@ They read the source as an AST through the shared layer in `sampletones_shared/m *Logic objects* (e.g. `ConverterLogic`) orchestrate multi-step workflows within a feature area. They subscribe to services and translate service results into view model updates. -`logic/history/` implements the session-scoped undo engine (`HistoryManager`); its invariants and mechanics are documented in `docs/development/undo.md`. +`logic/history/` implements the session-scoped undo engine (`HistoryManager`); its invariants and mechanics are documented in `docs/development/application/undo.md`. -`logic/reconstruction/browser/` builds the tree of reconstructions both browser tabs render (`BrowserManager`); its pipeline, node vocabulary and shaping rules are documented in `docs/development/browser.md`. +`logic/reconstruction/browser/` builds the tree of reconstructions both browser tabs render (`BrowserManager`); its pipeline, node vocabulary and shaping rules are documented in `docs/development/application/browser.md`. **Contracts:** - Logic classes produce view models and may therefore import `view_model/`; they import neither `ui/` nor `coordinators/`. @@ -245,7 +245,7 @@ They read the source as an AST through the shared layer in `sampletones_shared/m There are two coordinator kinds: -*Domain coordinators* manage a cross-cutting concern that spans the whole application lifecycle — e.g. `ProjectCoordinator` (project file I/O, save confirmations), `PlaybackRouter` (the single transport over the shared output device, acting on the active tab's source or the engaged one — see `docs/development/playback.md`), or `EditRouter` (the single edit surface behind the menu bar's Edit menu, which shows the actions of the grid holding the cursor — see `docs/development/sequencer-blocks.md`). +*Domain coordinators* manage a cross-cutting concern that spans the whole application lifecycle — e.g. `ProjectCoordinator` (project file I/O, save confirmations), `PlaybackRouter` (the single transport over the shared output device, acting on the active tab's source or the engaged one — see `docs/development/application/playback.md`), or `EditRouter` (the single edit surface behind the menu bar's Edit menu, which shows the actions of the grid holding the cursor — see `docs/development/application/sequencer-blocks.md`). *Tab coordinators* own everything for one tab: they instantiate its panels, logic objects, and tab-scoped services, wire their callbacks together, and provide `create_tab()` — the single method that builds the DPG widget tree for that tab. Tab coordinators present a narrow public API of intent-level methods (`set_input_path`, `display_reconstruction`, …) and keep their panels and logic objects private. diff --git a/docs/development/guidelines.md b/docs/development/guidelines.md index 3ca1c3ccc..811a92124 100644 --- a/docs/development/guidelines.md +++ b/docs/development/guidelines.md @@ -2,7 +2,7 @@ These rules govern the Python in this repository. They complement `docs/development/architecture.md` (ownership and layering) and -`docs/development/config-organization.md` (configuration). +`docs/development/application/config-organization.md` (configuration). ## General @@ -19,7 +19,7 @@ These rules govern the Python in this repository. They complement 1. Prefer `pathlib.Path` over `os.path`. 1. Separate function options with `*`, and choose positional arguments intentionally. 1. Change internal APIs, configs, and data shapes freely; preserve backward compatibility only when the user explicitly asks. -1. Move a stored data version once per release. The version a build writes between releases is still being written, so a further change to that format extends the upgrade step already pending — one step carries the whole distance from the version the last release shipped. See `compatibility.md`. +1. Move a stored data version once per release. The version a build writes between releases is still being written, so a further change to that format extends the upgrade step already pending — one step carries the whole distance from the version the last release shipped. See [data compatibility](release/compatibility.md). 1. Run `pre-commit` on new files after each change. ## Ownership @@ -75,6 +75,7 @@ These rules govern the Python in this repository. They complement ## Documents 1. A document under `docs/` explains a subsystem to someone about to change it. Open by stating what it governs and when to consult it, so a reader learns in one paragraph whether they are in the right place. +1. A development document sits beside what it governs. The top of `docs/development/` holds what spans the repository's packages; `application/` holds what governs the graphical application alone, and `release/` what a release ships and keeps compatible. `docs/index.md` lists every document under the heading of its directory. 1. Lead with principles, then mechanics. A principle is a design truth you reason from; state the principles first, and let concrete conventions and reference tables follow as the way each principle is realized. 1. Keep principles, conventions, and descriptions distinct. A principle is a reason; a convention is a handy mechanic that serves it; a description is a fact about how something works. A convention promoted to a principle, or a principle buried in a description, misleads the reader about what is load-bearing. 1. Prefer a few strong principles to many narrow rules. When several rules are facets of one idea, state the idea once and derive them. A growing list of ad-hoc rules signals a principle that has gone unstated. @@ -96,7 +97,7 @@ These rules govern the Python in this repository. They complement 1. A test file mirrors the ownership of the code it exercises. 1. When functionality moves between packages, move its direct unit tests in the same change. -1. **A test whose assertion is a measured duration lives in `tests/benchmarks/`.** The gated suite runs across six workers and under coverage, which multiplies the cost of the code being measured, so those tests run in a pass of their own — serial and uncovered — where the reading is the code's own cost. `make test` runs that pass after the covered one, and `make benchmarks` runs it alone. +1. **A test whose assertion is a measured duration lives in `tests/benchmarks/`.** The gated suite runs across six workers and under coverage, which multiplies the cost of the code being measured, so those tests run in a pass of their own — serial and uncovered — where the reading is the code's own cost. `make test` runs the covered suite, and `make benchmarks` runs the measured pass. 1. Parametrize tests that share a body, using a test-case dataclass. 1. Test case classes and cases themselves should be defined inside the testing class, unless these objects are shared between test classes. A suite inherits from `BaseTestSuite` and names its case class `TestCase`, which inherits from `BaseRegularTestCase`, or from `BaseAutolabelTestCase` where the case derives its own label. The parametrized argument carries the case as `test_case`. 1. For a multi-step scenario, use a test-scenario suite class — a series of functions with assertions. diff --git a/docs/development/packages.md b/docs/development/packages.md index f246dd5a8..41f974222 100644 --- a/docs/development/packages.md +++ b/docs/development/packages.md @@ -17,22 +17,24 @@ has come — inside one process and across the pool's workers — is [`progress. ```mermaid graph TD ENTRY["sampletones\n(entry point)"] + TOOLS["sampletones_tools\n(developer tools)"] APP["sampletones_application\n(GUI)"] PLAYER["sampletones_player\n(NES player)"] CORE["sampletones_core\n(reconstruction engine)"] - SYNTH["sampletones_synthesis\n(waveform synthesis)"] - ASSETS["sampletones_assets\n(mark and fonts)"] + ASSETS["sampletones_assets\n(icons and fonts)"] SHARED["sampletones_shared\n(facts and helpers)"] CONFIG["sampletones_config\n(shipped YAML)"] ENTRY --> APP ENTRY --> CORE + ENTRY --> TOOLS + TOOLS --> APP + TOOLS --> PLAYER + TOOLS --> CORE + TOOLS --> SHARED APP --> PLAYER APP --> CORE PLAYER --> CORE - CORE --> SYNTH - ASSETS --> SHARED - SYNTH --> SHARED CORE --> SHARED PLAYER --> SHARED APP --> SHARED @@ -41,17 +43,23 @@ graph TD | Package | Purpose | May import | |---------|---------|------------| -| `sampletones_shared` | Facts and helpers any package holds: constants, exception families, paths, the logger, the array backend, the source layer the checks read the tree through, and the schema these boundaries are declared in | — | -| `sampletones_config` | The shipped YAML — layout, palettes, themes, keybindings, language, calibration, and these boundaries themselves — reached as package data rather than by import | — | -| `sampletones_assets` | The application mark and the bundled fonts, with the code that draws the mark | `sampletones_shared` | -| `sampletones_synthesis` | Analytic waveform synthesis: oscillators, envelopes, layers and voices | `sampletones_shared` | -| `sampletones_core` | The reconstruction engine, the project model, playing a song out into instructions, and the tracker export formats | `sampletones_shared`, `sampletones_synthesis` | +| `sampletones_shared` | Facts and helpers any package holds: constants, exception families, paths, the logger, the array backend, and the command type the entry and the tools share | — | +| `sampletones_config` | The shipped YAML — layout, palettes, themes, keybindings, language, behavior, deployment, and these boundaries themselves — reached as package data rather than by import | — | +| `sampletones_assets` | The application icons and the bundled fonts, reached as package data | — | +| `sampletones_core` | The reconstruction engine, the project model, playing a song out into instructions, and the tracker export formats | `sampletones_shared` | | `sampletones_player` | The NES player: the register model, the re-clocking schedule, the 6502 driver and the NSF file | `sampletones_shared`, `sampletones_core` | | `sampletones_application` | The DearPyGui front end | `sampletones_shared`, `sampletones_core`, `sampletones_player` | -| `sampletones` | The command-line entry point and the startup self-check | `sampletones_shared`, `sampletones_core`, `sampletones_application` | +| `sampletones_tools` | Everything a developer runs and the application does not: analytic waveform synthesis, the calibration harness, the driver toolchain and the register trace, the mark the icons are drawn from, the source checks, the synthetic corpus with its sample emitters, and the developer commands that run them | `sampletones_shared`, `sampletones_core`, `sampletones_player`, `sampletones_application` | +| `sampletones` | The command-line entry: the dispatcher, the commands and the startup self-check | `sampletones_shared`, `sampletones_core`, `sampletones_application`, `sampletones_tools` | Third-party imports are the package author's own choice and stand outside this table. +**The tools package is reached from the command line alone.** `sampletones_tools` holds what a +developer runs and the application never imports: the developer commands and the libraries behind +them. `sampletones` is its one importer, appending the developer commands to the user commands, so +the wheel and the bundle carry the tools and no shipped package depends on them. +[Tooling](tooling.md) states what a tool is and what a developer command does in an installed copy. + **The reconstruction engine stands below the console player.** A reconstruction is produced, saved and exported to a tracker with `sampletones_player` absent from the process, which is what lets the player's format move while the engine holds still. The consequence is that an export backend @@ -91,19 +99,20 @@ them. | `compression/` | The planes a song separates into, the dictionary its tokens name, and the codec that reads them both ways | `specification/`, `registers/` | | `song.py` | `Song` — the compressed planes, the timer table, the schedule and the loop point as one value | `clock/`, `registers/`, `compression/` | | `builder.py` | The song a reconstruction or an export request plays as, its instructions encoded, its planes compressed and its rate scheduled | `song.py`, `registers/`, `clock/`, `compression/` | -| `trace/` | `RegisterTrace` — what the driver is expected to write, call by call | `song.py`, `specification/` | | `nsf/` | The song block, the header and the `.nsf` file the console loads | `song.py`, `specification/`, `compression/`, `driver/` | | `driver/` | The assembled 6502 driver and the addresses its build reports | `specification/` | -| `driver/assembler/` | The cc65 build: the layout, the toolchain, the linker map reader and the builder | `driver/`, `specification/` | | `export.py` | `NSFBackend` — the export seam answered in `.nsf` files, holding the driver every one of them carries and saying which stage a run is in | `builder.py`, `nsf/`, `driver/`, `compression/` | -### The build toolchain is a developer tool +### The toolchain and the oracle live with the tools -`driver/assembler/` runs `ca65` and `ld65` over `driver/assembly/` to produce the committed -`driver/binary/driver.bin`. It is reached from `scripts/player.py` and from the tests, and the wheel -carries the binary alone — so a module of the shipped tree that imported it would break an installed -copy, and no unit above declares it. The developer toolchain it needs is described in -[`dependencies.md`](dependencies.md). +`sampletones_tools/player/assembler/` runs `ca65` and `ld65` over the assembly sources, their +includes and the linker configuration in `sampletones_tools/player/assembly/`, read as package +data, to produce the committed `driver/binary/driver.bin`; `uv run sampletones driver` runs it, +and the tests rebuild the sources and hold the committed image to them wherever cc65 is +installed. `sampletones_tools/player/trace/` holds `RegisterTrace`, what the driver is expected +to write call by call, which the emulator tests hold the assembled driver to. Exporting reads the +assembled binary; the wheel carries the assembly sources beside it, inside the tools package. The +toolchain the build needs is described in [`dependencies.md`](release/dependencies.md). --- @@ -112,15 +121,26 @@ copy, and no unit above declares it. The developer toolchain it needs is describ `sampletones_config/boundaries/graphs.yaml` declares both graphs as layer tables — each unit and the units it may import — and the rule the check runs derives from them: every unit a table leaves out is out of reach, so an edge is declared before it is taken. The hook audits the whole source tree on -every commit (`make check-import-boundary`), which means adding an edge to a table is how a new +every commit (`uv run sampletones check import-boundary --all`), which means adding an edge to a table is how a new dependency is opened, and removing one enumerates the work of closing it. +Five token rules hold the shipped packages to the tools edge a second way: a module of +`sampletones_application`, `sampletones_core`, `sampletones_player`, `sampletones_shared` or +`sampletones_assets` that spells `sampletones_tools` at all is reported, so the edge is closed in +words as well as in imports. + A graph answers for its own well-formedness as it is read: a unit reaching a unit the graph leaves undeclared is refused, and so is a graph whose units reach themselves, since a unit's layers state a level only where the units stand in an order. Three parts share the work. `sampletones_config/boundaries/` states what the boundaries are. -`sampletones_shared/meta/import_boundary/` validates that statement and holds the mechanism — +`sampletones_tools/checks/boundary/` validates that statement and holds the mechanism — reading a module line by line, resolving a unit to the modules it owns, deriving a rule from a graph -and reporting what crosses it — beside the source layer the other checks read the tree through. -`scripts/checks/import_boundary.py` runs them over a source tree and prints what they find. +and reporting what crosses it — beside the source layer the other checks read the tree through, +`sampletones_tools/checks/source/`. `sampletones check import-boundary` runs them over the source +and scripts trees and prints what they find. + +The scripts tree is held to a rule of its own, `boundaries/standalone.yaml`. A bootstrap script +runs on the system interpreter, so it imports the standard library and the scripts tree itself, +and a name in that tree that stands in for a standard-library module is reported too, since the +tree sits on the import path. [Tooling](tooling.md) states the principle. diff --git a/docs/development/player.md b/docs/development/player.md index b3fcf59db..65ff6b590 100644 --- a/docs/development/player.md +++ b/docs/development/player.md @@ -3,7 +3,7 @@ This document governs `sampletones_player`: the 6502 driver an exported `.nsf` carries, the codec that fits a song into the console's program area, and the chain that holds both to what the application plays. Read it before changing the assembly under -`driver/assembly/`, anything under `compression/`, or the way a song is built in +`sampletones_tools/player/assembly/`, anything under `compression/`, or the way a song is built in `builder.py`. The byte layout the two sides meet on is [the NSF format](../formats/nsf.md); where the package sits among the others is [package layers](packages.md). @@ -38,9 +38,20 @@ planes it writes, and every row playing it becomes a token naming that entry. Se what the samples leave uncovered. **Every layer earns its place on measured ground.** Each stage of the codec can be switched -off on its own, and `make compression-report` writes what each one saves across a corpus of +off on its own, and `uv run sampletones codec report` writes what each one saves across a corpus of songs. The format's constants are settled from that report rather than from argument. +**A change to the codec is measured before it is built.** `uv run sampletones codec study` reads the +projects and stems named on its command line, encodes every song under every candidate change, +and writes the sizes, the times, a verdict per candidate and the manifest that repeats the run +under `Documents/SampleToNES/compression`. A candidate is one of two things. A new way of choosing +tokens is encoded and played back by the production codec itself. A new token grammar is +priced in bytes by a study parser, which first has to reproduce the production parser's +bytes on today's grammar. The rule is printed in the report: a candidate earns a production +layer when it saves 3% over the projects or 5% over the reconstructions and grows no song by +more than 1%. The study lives under `sampletones_tools/codec/study`, outside the shipped +packages. + ## The song a file carries `Song` is the compressed song: the dictionary, one token stream per plane, the timer table, @@ -140,14 +151,15 @@ The chain runs from the register values upward, and each link is held on its own |---|---| | The codec is lossless | every encoding decodes to the planes it was written from, over a corpus | | The codec is safe | a plane the codec finds nothing in stays within its literal bound | -| The ratio | `make compression-report` — bytes per tick and ticks that fit, per layer | +| The ratio | `uv run sampletones codec report` — bytes per tick and ticks that fit, per layer | +| What a change would save | `uv run sampletones codec study` — the projects and stems it is given, under every candidate change, with a verdict each | | The byte layout | a hand-built song serializes to expected bytes | | The assembly agrees with the specification | the include's equates are read and compared field by field | | The driver behaves | the assembled image on a 6502 emulator against `RegisterTrace.from_song`, over several rates and over songs that repeat | | The driver's arithmetic | a song stating a bend outright, held to the divider each tick is meant to sound at | | The audio | a captured trace re-rendered against the reconstruction's own approximation | | The whole export | a project exported, played on the emulator, and read back as the instructions the sequencer sounds | -| Listening | `make nsf-samples` then `make nsf-render`, or any NSF player | +| Listening | `uv run sampletones nsf samples -o build/nsf` then `uv run sampletones nsf render --directory build/nsf`, or any NSF player | | Speed | `make benchmarks` — the encoder's own cost on the shapes that scale worst | The audio comparison is the one that catches a mistake the trace would let through: the @@ -156,9 +168,9 @@ the reconstruction was built as. ## Building the driver -`make player` assembles the sources with cc65 and writes `driver/binary/driver.bin`, which -is committed beside them — exporting an `.nsf` needs no assembler, and the wheel carries the -binary alone. +`uv run sampletones driver` assembles the sources under `sampletones_tools/player/assembly/` +with cc65 and writes `sampletones_player/driver/binary/driver.bin`, which is committed — +exporting an `.nsf` needs no assembler, and the application ships the binary alone. The link line names our own configuration and our own object files, with the CPU stated outright. That is the guardrail that keeps the shipped image entirely ours: reaching for a @@ -167,4 +179,4 @@ package distributes. A build also holds the linker's own labels against the addr exporter states without one, so the committed image and the header describing it cannot drift apart. -Installing cc65 is covered in [dependencies](dependencies.md). +Installing cc65 is covered in [dependencies](release/dependencies.md). diff --git a/docs/development/compatibility.md b/docs/development/release/compatibility.md similarity index 97% rename from docs/development/compatibility.md rename to docs/development/release/compatibility.md index fca02ffdc..c2ffba193 100644 --- a/docs/development/compatibility.md +++ b/docs/development/release/compatibility.md @@ -9,9 +9,9 @@ The upgrades live in `sampletones_core/compatibility` and run at the load boundary of each format, before deserialization. The formats' own documents describe their stored shape and versioning: -- [`formats/reconstructions.md`](../formats/reconstructions.md) -- [`formats/instruction-libraries.md`](../formats/instruction-libraries.md) -- [`formats/projects.md`](../formats/projects.md) +- [`formats/reconstructions.md`](../../formats/reconstructions.md) +- [`formats/instruction-libraries.md`](../../formats/instruction-libraries.md) +- [`formats/projects.md`](../../formats/projects.md) ## Principles diff --git a/docs/development/dependencies.md b/docs/development/release/dependencies.md similarity index 70% rename from docs/development/dependencies.md rename to docs/development/release/dependencies.md index 103d68085..4196d8e8e 100644 --- a/docs/development/dependencies.md +++ b/docs/development/release/dependencies.md @@ -12,7 +12,7 @@ The core depends on common Python packages: * `librosa` * `cupy` (optional; enables the GPU backend, with the build selected for your NVIDIA driver) -See [GPU acceleration](../guide/installation.md#gpu-acceleration) for enabling it. +See [GPU acceleration](../../guide/installation.md#gpu-acceleration) for enabling it. ## Serialization @@ -20,9 +20,9 @@ Instruction libraries and reconstructions are serialized with [MessagePack](http ## Audio playback -Playback goes through PortAudio, reached with the `pyaudio` package. PyPI carries `pyaudio` wheels for Windows, so Linux and macOS compile it on install and need the PortAudio headers and library on the machine. Linux takes them from the distribution packages listed in `scripts/linux/build/dependencies.sh`; macOS takes them from Homebrew through `scripts/macos/build/dependencies.sh`. +Playback goes through PortAudio, reached with the `pyaudio` package. PyPI carries `pyaudio` wheels for Windows, so Linux and macOS compile it on install and need the PortAudio headers and library on the machine. `scripts/system_dependencies.py` installs them: the distribution packages through apt on Linux, PortAudio through Homebrew on macOS. -Compiling on macOS also depends on the interpreter's architecture. The python.org installer ships a universal2 build, which compiles extensions for both Apple Silicon and Intel, while Homebrew's `libportaudio` carries the machine's own architecture. Pinning `ARCHFLAGS` to `uname -m` settles it on the native one: `make setup` sets it directly, and the CI workflows take it from `scripts/macos/build/build_env.sh`, which reports it as a `KEY=VALUE` line alongside the PortAudio prefix for a Homebrew installed outside its usual place. +Compiling on macOS also depends on the interpreter's architecture. The python.org installer ships a universal2 build, which compiles extensions for both Apple Silicon and Intel, while Homebrew's `libportaudio` carries the machine's own architecture. Pinning `ARCHFLAGS` to `uname -m` settles it on the native one: `make setup` sets it directly, and the CI workflows take it from `scripts/build_environment.py`, which reports it as a `KEY=VALUE` line alongside the PortAudio prefix for a Homebrew installed outside its usual place. ## Audio rendering @@ -53,32 +53,34 @@ Dialogs open through the XDG desktop portal (`org.freedesktop.portal.FileChooser ## Application icon -The icon suite in `src/sampletones_assets/icons` is generated from the mark declared beside it in -`src/sampletones_assets/mark`: `mark.yaml` carries the geometry, colors and rasterization +The icon suite in `src/sampletones_assets/icons` is generated from the mark declared in +`src/sampletones_tools/assets/mark/config`: `mark.yaml` carries the geometry, colors and rasterization settings, validated as a `Mark`, and `template.svg` is the vector the rendered geometry fills. The -package writes the whole suite — the vector `sampletones.svg` and the rasters the application -ships, `sampletones.png` and the multi-resolution `sampletones.ico` — and `scripts/assets/icons.py` +tools package writes the whole suite — the vector `sampletones.svg` and the rasters the application +ships, `sampletones.png` and the multi-resolution `sampletones.ico` — and `uv run sampletones icons` points it at the directory the icons are shipped from. Rasterization uses Pillow, declared in the -`assets` dependency group. +`assets` dependency group, which the `dev` group includes. The whole suite is committed, so a plain checkout carries the icons the application opens its window -with, and every wheel, bundle and test run finds them where they lie. `make icons` writes them again -from the mark, and the `icons` pre-push hook writes them for a push that touches either directory, -holding the committed files to what the mark describes. CI runs that same hook. +with, and every wheel, bundle and test run finds them where they lie. `uv run sampletones icons` +writes them again from the mark, and the `icons` pre-push hook writes them for a push that touches +either directory, holding the committed files to what the mark describes. CI runs that same hook. -Pillow is a build-time tool, and the bundle scripts pass `--exclude-module PIL` to hold it to that: +Pillow is a developer tool the build environment never installs, and the bundle script passes +`--exclude-module PIL` besides, to hold it to that: `pygments`, which arrives with `rich`, offers an image formatter that imports Pillow where it is installed, and PyInstaller follows that import into the bundle. The application reads its icons as files, so the exclusion spares every bundle Pillow's extension modules and the imaging libraries -that come with them. `scripts/ci/checks/bundle.py` holds the release bundles to it. +that come with them. `scripts/verify_bundle.py` holds the release bundles to it. ## NES player driver -The player that runs on the console is 6502 assembly, and `src/sampletones_player/driver` holds it -in three parts: `assembly/` carries the sources, their includes and the linker configuration, -`binary/` carries the assembled `driver.bin`, and `assembler/` carries the Python that turns one -into the other. `make player` runs `scripts/player.py` over that package, so the build behaves the -same on every system the project supports. +The player that runs on the console is 6502 assembly, held in three parts: +`src/sampletones_tools/player/assembly/` carries the sources, their includes and the linker +configuration, `src/sampletones_tools/player/assembler/` carries the Python that assembles them, +and `src/sampletones_player/driver/binary/` carries the assembled `driver.bin` the application +ships. `uv run sampletones driver` runs the build, so it behaves the same on every system the +project supports. Assembling needs `ca65` and `ld65` from [cc65](https://cc65.github.io/) — on Debian and Ubuntu, `sudo apt install cc65`, and a build names the equivalent for whichever system it runs on when the @@ -88,9 +90,10 @@ The assembled `driver.bin` is committed, so a checkout carries the player and ex needs no assembler. A jump table leads the image, which fixes the addresses an NSF header names whatever the driver's length, so the exporter states them from `specification/driver.py` and a build holds the linker's own labels to them before it writes anything. Editing the assembly means -running `make player` again and committing what it writes; the driver's test suite rebuilds the +running `uv run sampletones driver` again and committing what it writes; the driver's test suite rebuilds the sources and holds the committed image to them wherever cc65 is installed. The wheel carries the -assembled image alone, which is all an installed copy reads. +assembled image, which is what exporting reads, and the assembly sources inside the tools package, +so `sampletones driver -o DIR` assembles the driver from an installed copy too. cc65 is distributed under the zlib license, and the driver stays clear of it: the link line names our own object files and our own `nsf.cfg`, so nothing of cc65's start-up code or libraries reaches @@ -108,10 +111,10 @@ is a developer dependency, outside both the wheel and the bundles, and its BSD l project's own terms untouched. Listening to a real APU needs [ffmpeg](https://ffmpeg.org/) carrying the `libgme` demuxer, which -is a build option rather than a given: `make nsf-render` asks the installed ffmpeg which demuxers -it holds and names this system's install command before it decodes anything. It exports the example -files and renders each one to a wave beside it, its length read out of the song block the file -carries. That is an ear rather than a gate: the register trace is what the driver answers to, and +is a build option rather than a given: `uv run sampletones nsf render` asks the installed ffmpeg which demuxers +it holds and names this system's install command before it decodes anything. `nsf samples -o DIR` +writes the example files, and `nsf render --directory DIR` renders each one to a wave beside it, its +length read out of the song block the file carries. That is an ear rather than a gate: the register trace is what the driver answers to, and the wave is what a person listens to. ### The player's tools @@ -120,20 +123,20 @@ Three tools serve the player, each reached by one command: | Tool | Run by | Installed with | Reaches | | --- | --- | --- | --- | -| cc65 (`ca65`, `ld65`) | `make player` | the system's package manager | the machine assembling the driver | +| cc65 (`ca65`, `ld65`) | `sampletones driver` | the system's package manager | the machine assembling the driver | | py65 | `make test` | `uv sync --group dev` | the `dev` dependency group | -| ffmpeg with `libgme` | `make nsf-render` | the system's package manager | the machine listening to an export | +| ffmpeg with `libgme` | `sampletones nsf render` | the system's package manager | the machine listening to an export | -`scripts/linux/build/dependencies.sh` and its macOS counterpart carry what building and running the -application needs, and the workflows install the `dev` group, so py65 is the one of the three CI +`scripts/system_dependencies.py` carries what building and running the application needs, and the +workflows install the `dev` group, so py65 is the one of the three CI reaches — the suite verifies the driver through it alone. cc65 and ffmpeg stay on the machine of -whoever runs `make player` or `make nsf-render`, and a workflow that assembles the driver or renders +whoever runs `sampletones driver` or `sampletones nsf render`, and a workflow that assembles the driver or renders a wave is what would put them in those scripts. The application itself calls neither: an export is written by the package's own code, from the committed `driver.bin`. ## Linux (standalone executable) -Building a standalone executable on Linux needs the PortAudio, Tk and OpenGL/X11 system packages. Install them with `make system-deps` (or run `scripts/linux/build/dependencies.sh`), which holds the full list. +Building a standalone executable on Linux needs the PortAudio, Tk and OpenGL/X11 system packages. Install them with `make system-deps` (or run `python3 scripts/system_dependencies.py`), which holds the full list. PortAudio is required. Tk backs the file dialogs where neither a portal nor a desktop tool answers, and `make release` requires it so the shipped executable stays self-contained. diff --git a/docs/development/tooling.md b/docs/development/tooling.md new file mode 100644 index 000000000..695cbc12d --- /dev/null +++ b/docs/development/tooling.md @@ -0,0 +1,196 @@ +# Tooling + +This document governs how the repository is run: the `sampletones` command and what it offers, the +tools package behind its developer commands, the scripts under `scripts/`, and the `Makefile`. Read +it before adding a command, a tool, a script or a make target. Which packages may import which is +[package layers](packages.md); the libraries and tools the scripts reach for are +[dependencies](release/dependencies.md). + +## Principles + +**1. One door.** Everything a person runs by hand is a `sampletones` command with its own parser +and help, always named: `sampletones` alone is `sampletones run`, a file is opened with +`sampletones open `, a recording is converted with `sampletones convert `. A +command's name says what it does, in plain words. + +**2. Three kinds of runnable code, told apart by who runs them and what they may import.** The +*application* is what a user installs. A *tool* runs inside the project environment, through +`uv run`: it may import any package, and only the command line reaches it. A *bootstrap script* +runs on the system interpreter, before or beside the environment: it creates the environment, +installs system packages, builds the standalone bundle, cleans the tree, and runs the tests, the +linters and the formatters the environment provides. It imports the standard library and the other +bootstrap modules, nothing else, so it runs on a machine that has Python 3.12 or newer and nothing +more. Importing `scripts/bootstrap/` checks that version before anything else, so an older +interpreter is told the version and where to download it. A bootstrap script installs nothing into +the interpreter it runs on: every package a build installs lands in `.venv-build`, a virtual +environment of its own, and pip is told to refuse any interpreter outside one. System packages are a step of their own, `make system-deps`, the only one that asks for +administrator rights. The import boundary check holds the tree to the rule: +`sampletones_config/boundaries/standalone.yaml` names the scripts, and an import beyond the +standard library and the tree fails the hook. + +**3. What earns a place on the command.** An operation is a *user command* when its input and +output are the user's own files and it needs nothing beyond the installed package. It is a +*developer command* when it reads or writes the repository or measures the code on this machine, +so it needs a checkout and the project environment. It is a *bootstrap script* when it must run +without the environment. A test is run by pytest and is never wrapped in a command; a tool the +tests exercise is a function they call. + +**4. A tool is a library with a thin face.** The work is a function that takes values and returns +values; the command module parses the arguments, calls it and prints. A command module imports the +standard library, the command type and its constants at module level, and its implementation +inside `run`, so listing the commands loads no tool. A bootstrap script is shaped the same way: +pure functions assemble the commands and take the decisions, `main` wires in the real runner and +the real environment, and the tests call the functions with a runner that records what it was +asked to run, so a build is verified without building. + +**5. The import graph is declared once and checked.** `sampletones` is the top of the graph, and +the bootstrap tree is a root of its own under the standard-library rule. [Package layers](packages.md) +holds the graph and its enforcement. + +**6. Tests mirror the tree.** `tests/unit//` mirrors `src//` and +`tests/unit/scripts/` mirrors `scripts/`. Code and its tests move in one change. + +**7. One script per operation, the same on every system.** What differs between systems (the +launcher's name and extension, the icon, the package manager, the compiler flags audio playback +needs) sits behind one `Platform` protocol with an implementation per system, chosen by a factory +from `platform.system()`. A script never branches on the operating system itself, and a system the +project does not build on is refused by name. The Makefile is the developer's index, one line per +target: a target names the script that does the work and passes its flag. The two shell files at +the root, `install.sh` and `install.bat`, exist for the double-click path and call the same bundle +script. + +**8. A developer command works from what it is given, in every copy of the program.** The wheel +and the bundle carry the tools package, so every developer command exists wherever `sampletones` +is installed, and each one reaches files the same way in all of them: + +- *Inputs* arrive on the command line or in a file a run wrote, so a run starts from what the person + running it has. +- *Outputs* go where `--output` (`-o`) names. A measurement given no `-o` writes a timestamped + directory under the user's Documents; `codec report` and the sample emitters take `-o` always. +- *The repository* is reached through the checkout guard. A command that reads or writes the + repository, or needs a development dependency, runs from a checkout; so `driver` and `icons` + rewrite the files the package ships from a checkout, and a measurement runs anywhere. +- *Package data* is read from the package it ships in, which holds in a checkout, in the wheel and + in the bundle. + +## The commands + +`src/sampletones/` is the entry package. `dispatcher.py` builds one parser over the commands and +runs the one named; `commands/` holds one module per command and `commands/registry.py` lists +them. A command is a frozen `Command` (`sampletones_shared/command.py`): its name, one line of +help, the function adding its options to a parser, and the function running it over the parsed +arguments. Each command turns its arguments into a frozen record, field by field, before it works; a +command with actions, such as `codec`, reads the action first and builds the record that action takes. +A command writing files names where they go `--output` (`-o`): a file for `convert`, a directory for +the others. An option naming an input says what it reads, so `--config` is a configuration file +wherever it appears. + +| Command | What it does | +|---|---| +| `run [--config FILE]` | Starts the application | +| `open PATH [--config FILE]` | Starts the application with a `.stp` project, a `.stn` reconstruction or an `.ins` library loaded; a recording is refused with the `convert` line to run instead | +| `convert SOURCE... [-o FILE] [--config FILE] [--channels LIST \| --stems FILE]` | Reconstructs recordings into one `.stn` file, or every recording under one directory file by file. `--stems` names a JSON file holding the setup the `.stn` record stores, its entries paired with the sources in order; the pairing is printed before the run, and a missing source, a file other than a recording or a missing stems file is refused first | +| `library [--config FILE]` | Generates the instruction library for a configuration | +| `self-check` | Verifies that the build's imports, bundled resources and configuration files are usable | + +`--version` and `--help` are flags of the entry itself. The headless runs behind `convert` and +`library` live in `sampletones_core/headless/`, where the calibration reuses them. + +The developer commands, listed by `sampletones_tools/registry.py` and run as +`uv run sampletones ` from a checkout: + +| Command | What it does | +|---|---| +| `calibration [--config FILE] [-o DIR] [--methods LIST] [--perceptual-exponents LIST] [--temporal-weights LIST] [--channels LIST]` | Reconstructs the calibration corpus under every variant of the sweep, scores it with every referee, and writes the reports; without `-o` the run lands in a timestamped directory under Documents/SampleToNES/calibration | +| `check [options]` | Holds the tree to one of its checks: `import-boundary`, `language-keys`, `palette-colors`, `rendered-literals`, `shortcut-actions`, `tag-names`, `unused-tags`; each is a pre-commit hook, and [architecture](architecture.md#enforcement) says what each holds. Needs a checkout | +| `btp samples -o DIR`, `ftm samples -o DIR`, `nsf samples -o DIR` | Builds the synthetic corpus and writes it as example files: the arrangement as two Bitphase documents (at its tempo and as a groove), the arrangement as a FamiTracker module, or each sample and the arrangement as `.nsf` programs | +| `codec report -o DIR` | Compresses the synthetic corpus under every layer of the codec and writes the report the format's constants are settled from, as CSV and Markdown | +| `codec study [--manifest FILE] [--project FILE]... [--reconstruction PATH]... [-o DIR] [--lengthen SECONDS] [--variants LIST]` | Encodes the projects and stems it is given, or the ones a manifest names, under every candidate change to the codec and writes the sizes, the times, a verdict per candidate and the manifest that repeats the run; without `-o` the run lands under Documents/SampleToNES/compression | +| `driver [-o DIR]` | Assembles the NES player driver with cc65 and prints the layout the build produced; without `-o` it writes the driver the package ships, which needs a checkout | +| `icons [-o DIR]` | Writes the icon suite from the mark, into `-o` or over the icons the package ships. Needs a checkout | +| `nsf render --directory DIR [--tail SECONDS]` | Renders every exported `.nsf` file in the directory to a wave beside it, through ffmpeg's libgme demuxer | + +## The tools package + +`src/sampletones_tools/` holds every tool the running application does not use, in subpackages by +subject, and `sampletones_tools/registry.py` lists the developer commands they offer. +`sampletones/commands/registry.py` appends them to the user commands, which is the one import of +the tools package; [package layers](packages.md) holds the edge. Two helpers carry out principle 8: + +- `sampletones_tools/checkout.py` holds `require_checkout(command)`: the repository root holds + `pyproject.toml` beside `src/`, or the command exits naming `uv run sampletones ` in a + checkout. `icons` calls it for Pillow, a development dependency, as well as for the repository. +- `package_directory` in `sampletones_shared/paths/package.py` places a package from the import + system's own record, where PyInstaller unpacks each package's data beside its modules. + +Developer commands are run as `uv run sampletones ` from a checkout; the `sampletones` +command `make setup` installs is a wheel and refuses the guarded ones the same way. A command module +imports pillow, NumPy and the like inside `run`, and a test imports the registry in a subprocess and +asserts that only the command, registry and package modules of the tools load and no heavy library +does, since a startup failure in any tool module would break every invocation, the GUI included. A +command reports each refused value on a line of its own: `describe_failure` in +`sampletones_shared/utils/validation.py` renders a validation error the way a person reads it. The +editable install puts `src/` on the path whole, so a checkout +sees the tools package whatever the wheel lists; hatchling's `dev-mode-exact` stays off for that +reason. + +## The bootstrap scripts + +| Script | Target | What it does | +|---|---|---| +| `bundle.py` | `make build`, `make release` | Creates `.venv-build`, installs the package with the `build` extra, checks the interpreter carries PortAudio (and Tk, for a release), writes the bundle with PyInstaller, runs its self-check, and copies the notices beside a release. Every package the wheel carries brings its data files at its own package path, so the frozen application finds them where an installed one does | +| `setup_environment.py` | `make setup` | Reads the NVIDIA driver, synchronizes the development environment with the matching GPU extra, installs the global `sampletones` command | +| `system_dependencies.py` | `make system-deps` | Installs the system packages: apt on Debian-based Linux, Homebrew on macOS, nothing on Windows | +| `build_environment.py` | CI | Prints the compiler flags a macOS build exports, one `KEY=VALUE` per line | +| `clean.py` | `make clean` | Removes the build outputs, the coverage reports and the bytecode caches | +| `run_tests.py` | `make test`, `make test-docs`, `make benchmarks` | Runs one pass of the tests, named on its command line: `suite`, the covered suite across six workers (`--workers` sets the count); `doctests`; or `benchmarks`, serial and uncovered. Each pass is a target, a pre-push hook and a CI step of its own, so a failure names its pass | +| `lint.py` | `make lint` | Runs mypy over the files `pyproject.toml` configures and pylint over `src/` and `scripts/`; `--mypy` or `--pylint` picks one, and named paths narrow both | +| `formatting.py` | `make format` | Runs isort, then black, over `src/`, `tests/` and `scripts/`, or over the paths named | +| `hooks.py` | `make pre-commit` | Installs the git hooks pre-commit runs at commit and at push | +| `runtime_hooks/release_environment.py` | build input | The PyInstaller runtime hook that gives a release bundle its deployment defaults | +| `verify_version_tag.py` | the release workflow | Holds the release tag to the version `pyproject.toml` records | +| `verify_bundle.py` | the release workflow | Holds the release bundle to its notices, keeps the build tools out of it, and starts its launcher | +| `archive_bundle.py` | the release workflow | Zips the release bundle into `bundles/`, named by the version and the `--label` of the platform | + +`scripts/bootstrap/` holds what they share, one fact in one place: + +- `layout.py`: the repository root and every path and list a script names: `bin`, `bundles`, + `.venv-build`, the runtime hook, the notices, the build tools a bundle leaves out, and what + `make clean` removes. +- `project.py`: what `pyproject.toml` states, read with `tomllib`: the name, the version, the + entry module, the wheel's packages, and the extras and groups the scripts install, which it + holds the file to. +- `platforms/`: what differs between systems, behind `Platform`, with `Bundling` holding what a + bundle takes on a system that builds one. +- `cuda.py`: the NVIDIA driver's CUDA version and the CuPy extra it selects. +- `interpreter.py`, `processes.py`, `passes.py`, `files.py`: the interpreter version check, + running a command and holding it to success, a run of named passes that reports every failure + at once, and removing a file or a tree. +- `venv_build.py`, `preflight.py`: the build environment and the installs into it, and the + preflight of the build interpreter. + +Every script's work is a function taking what it reads: the repository root, the platform, and for +a script running commands the runner and the variables. `main` parses the arguments and passes in +the real ones, and the tests pass a temporary repository and a `RecordingRunner`. + +## Who governs what + +| Concern | Owner | +|---|---| +| Which commands the entry offers | `src/sampletones/commands/registry.py` | +| Which developer commands exist | `src/sampletones_tools/registry.py` | +| Which checks the tree is held to | `src/sampletones_tools/checks/registry.py` | +| Whether a command runs outside a checkout | `src/sampletones_tools/checkout.py` | +| The synthetic corpus the emitters and the integration tests share | `src/sampletones_tools/corpus/` | +| What a command is | `src/sampletones_shared/command.py` | +| What a bootstrap script may import | `sampletones_config/boundaries/standalone.yaml` | +| What differs between systems | `scripts/bootstrap/platforms/` | +| Where a script finds a path, a notice or a clean target | `scripts/bootstrap/layout.py` | +| What the scripts read from `pyproject.toml` | `scripts/bootstrap/project.py` | +| Where a build installs | `scripts/bootstrap/venv_build.py` | +| What a bundle has to carry before it is built | `scripts/bootstrap/preflight.py` | +| The PyInstaller invocation | `scripts/bundle.py` | +| The test passes and the command each runs | `scripts/run_tests.py` | +| What `make lint` and `make format` sweep | `scripts/lint.py`, `scripts/formatting.py` | +| The GPU extra a machine gets | `scripts/bootstrap/cuda.py` | +| What a release is held to | `scripts/verify_version_tag.py`, `scripts/verify_bundle.py` | diff --git a/docs/formats/instruction-libraries.md b/docs/formats/instruction-libraries.md index bfa6fdd01..3d2629163 100644 --- a/docs/formats/instruction-libraries.md +++ b/docs/formats/instruction-libraries.md @@ -8,7 +8,7 @@ it is built, see [Instruction library](../concepts/instruction-library.md); this page documents the file. Libraries are generated from the _Instructions_ tab (or with -`sampletones --generate`) and stored in the documents folder. +`sampletones library`) and stored in the documents folder. ## Contents diff --git a/docs/formats/nsf.md b/docs/formats/nsf.md index bc9ae9733..de3838c94 100644 --- a/docs/formats/nsf.md +++ b/docs/formats/nsf.md @@ -16,7 +16,7 @@ and the song that driver plays. Every constant named here has a counterpart under `sampletones_player/specification/`, and the assembly reads the same figures from -`driver/assembly/include/song.inc`. The two are held against each other by a test, so a +`sampletones_tools/player/assembly/include/song.inc`. The two are held against each other by a test, so a change made in one file and forgotten in the other is reported by name. ## A. The file diff --git a/docs/formats/projects.md b/docs/formats/projects.md index d11b4d613..7da88b12d 100644 --- a/docs/formats/projects.md +++ b/docs/formats/projects.md @@ -70,7 +70,7 @@ _SampleToNES_ requires that version to match the one it supports and declines an incompatible file rather than misreading it. A file written at a version the upgrade chain reaches is migrated in memory to the current shape before deserialization (see -[Data compatibility](../development/compatibility.md)). Unknown or extra fields +[Data compatibility](../development/release/compatibility.md)). Unknown or extra fields within a matching version are ignored, which leaves room for the format to grow. The current format version is 1.1. It gathers the pool under `voices`, each record diff --git a/docs/formats/reconstructions.md b/docs/formats/reconstructions.md index 489991cee..718a99f20 100644 --- a/docs/formats/reconstructions.md +++ b/docs/formats/reconstructions.md @@ -59,6 +59,21 @@ to the player, which is the record a channel edited down to empty envelopes reac as well. A file naming a stream for the channels it plays alone reads as the whole four, with the rest coming back standing by. +The stems setup is also what `sampletones convert --stems` reads, written as JSON with the +same fields: one entry per recording, in the order the recordings are given, each naming the +channels it may occupy and the ones it bends; a hierarchy listing the stem ids by precedence +level; and a channel cap. Two recordings, the first on the pulses and the second on the rest: + +```json +{ + "entries": [ + {"id": 0, "settings": {"channels": ["pulse1", "pulse2"], "bends": ["pulse1", "pulse2"]}}, + {"id": 1, "settings": {"channels": ["triangle", "noise"], "bends": ["triangle"]}} + ], + "hierarchy": {"levels": [[0], [1]]} +} +``` + ## Detached reconstructions A reconstruction normally remembers the path to its source audio. Embedding one @@ -74,7 +89,7 @@ _SampleToNES_ requires that version to match the one it supports and declines a file written by an incompatible version rather than misreading it. A file written at a version the upgrade chain reaches is migrated in memory to the current shape before deserialization (see -[Data compatibility](../development/compatibility.md)); the application version +[Data compatibility](../development/release/compatibility.md)); the application version is stored alongside the data version, for reference. The current data version is 2.2. Version 2.2 renamed the per-channel stream and diff --git a/docs/guide/command-line.md b/docs/guide/command-line.md index 76a2d5912..f2022c903 100644 --- a/docs/guide/command-line.md +++ b/docs/guide/command-line.md @@ -1,42 +1,62 @@ # Command line -You can run _SampleToNES_ from a terminal — to reconstruct without opening the -interface, to generate a library, or to open a file directly in the app. Run it -with no arguments to launch the GUI as usual. +You can run _SampleToNES_ from a terminal. Use it to convert recordings in the terminal, to build +an instruction library, or to open a file in the app. -The command is `sampletones` when installed from source; a standalone build is the -executable you made (`./bin/sampletones` on Linux, `bin\sampletones.exe` on Windows). +## Getting the command -## Common tasks +How you run the command depends on how you installed _SampleToNES_: -* **Launch the interface** — `sampletones` -* **Reconstruct a file** — `sampletones input.wav -o output.stn` -* **Reconstruct a folder** — `sampletones path/to/folder` reconstructs every audio - file inside it. -* **Choose the channels** — add `--channels pulse1,pulse2` to reconstruct onto those two - alone; without it a run uses pulse 1, triangle, and noise. -* **Open a file in the app** — `sampletones song.stp` opens the interface preloaded - with it; a `.stn` reconstruction or `.ins` library works the same way. -* **Use a specific configuration** — add `--config my-config.json`; otherwise your - saved configuration is used (`config.json`, or built-in defaults if you have not - saved one yet). -* **Generate a library and exit** — `sampletones --generate --config my-config.json` -* **Check the version** — `sampletones --version` -* **Check that a build works** — `sampletones --self-check` +- **Release download**: run `sampletones` in the extracted folder. On Windows the file is + `sampletones.exe`. +- **PyPI**: `uv tool install sampletones` or `pipx install sampletones` puts `sampletones` on your + path. +- **From source**: `make setup` puts `sampletones` on your path. A standalone build you make + yourself is `./bin/sampletones` on Linux and `bin\sampletones.exe` on Windows. -## Options +[Installation](installation.md) explains each way. -| Option | Purpose | +## Commands + +| Command | What it does | | --- | --- | -| `path` | (positional) an audio file or folder to reconstruct, or a `.stn` / `.ins` / `.stp` file to open in the app. Omit it to launch the interface. | -| `--output`, `-o` | output path for a reconstruction | -| `--channels` | channels the reconstruction may use, comma separated (default: `pulse1,triangle,noise`) | -| `--config`, `-c` | path to a configuration `.json` (default: your saved `config.json`) | -| `--generate`, `-g` | build the instruction library for the configuration, then exit | -| `--version`, `-v` | print the version and exit | -| `--self-check` | verify that this build's imports, bundled resources, and configuration files are all usable, then exit | -| `--help`, `-h` | show the full option list | - -GPU acceleration is selected at setup, not per run: `make setup` detects a supported -NVIDIA driver and installs the matching build (`make setup GPU=0` forces the CPU -backend) — see [Installation](installation.md). +| `sampletones` or `sampletones run` | Starts the app. | +| `sampletones open ` | Starts the app with a `.stp` project, a `.stn` reconstruction or an `.ins` library open. | +| `sampletones convert ...` | Converts recordings into one `.stn` reconstruction, or converts every recording in a folder. | +| `sampletones library` | Builds the instruction library for a configuration, then exits. | +| `sampletones self-check` | Checks that the app can start: its code, its fonts and icons, and its configuration files. | +| `sampletones --version` or `sampletones -v` | Prints the version. | + +Add `--help` to any command to see its options. + +`sampletones --help` also lists commands for developing _SampleToNES_, such as `check` and +`codec`. They run from a copy of the source code, and [Tooling](../development/tooling.md) +describes them. + +## Options + +- `--config ` or `-c ` uses a configuration file. It works with `run`, `open`, + `convert` and `library`. Without it, the app uses your saved configuration, or the built-in + defaults until you save one. +- `--output ` or `-o ` sets where `convert` saves the reconstruction. Without it, the + reconstruction goes to the reconstructions folder of your configuration. +- `--channels ` sets the channels `convert` may use, for example `--channels pulse1,pulse2`. + Without it, `convert` uses pulse 1, triangle and noise. +- `--stems ` gives `convert` a stems file, which it needs to mix several recordings. + +A `convert` command takes either `--channels` or `--stems`. + +## Common tasks + +- **Convert one file**: `sampletones convert input.wav -o output.stn` +- **Convert a folder**: `sampletones convert path/to/folder`. Every recording in the folder + becomes its own reconstruction, saved in your reconstructions folder. +- **Mix several recordings into one reconstruction**: + `sampletones convert bass.wav lead.wav --stems stems.json`. The stems file has one entry per + recording, in the same order. Each entry lists the channels the recording may use and the + channels it bends. The command prints which recording uses which entry before it starts. + [Reconstructions](../formats/reconstructions.md) shows the file. +- **Build a library**: `sampletones library --config my-config.json` + +GPU support is chosen when you install _SampleToNES_. [Installation](installation.md) explains +how. diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index 4f5cf0edc..2d2fc1724 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -1,51 +1,41 @@ # Configuration -_SampleToNES_ reconstructs audio according to a **generation configuration**: the -sample rate, the NES frequency, how the audio is analyzed, and how candidates are -scored. The settings you change most often are on the **Main** tab. The rest are -in the configuration file, for when you want to go deeper. - -Channels are not part of the configuration. The NES has four sound channels — -**Pulse 1**, **Pulse 2**, **Triangle**, and **Noise** — and you choose which of -them each recording uses in the converter's list, every time you set up a +_SampleToNES_ converts audio with a **generation configuration**. The configuration sets the sample +rate, the NES frequency, how the audio is analyzed and how sounds are compared. The settings you +change most often are on the **Main** tab. The configuration file has the rest. + +You choose the channels each recording uses in the converter's list, each time you set up a conversion. See [choosing which channels a recording uses](converting.md#choosing-which-channels-a-recording-uses). -## From the interface +## Settings on the Main tab -Three cards on the **Main** tab hold the everyday settings: +Three cards on the **Main** tab have the everyday settings: -- **General settings** — **Normalize audio** and **Quantize audio**, and the - **Sample rate** and **NES frequency** that a library is built for. -- **Source settings** — **Drive**, which sets how hard the channels are pushed, - and the channels and bends of the recording you selected in the converter's - list. -- **Advanced settings** — **Method** and **Feature scaling**, which set how the - audio's frequency content is measured and weighted (see [Reconstruction - algorithms](../concepts/reconstruction.md)); the **Workers** count; and the - instruction library and output folders. Choose **View ▸ Show advanced - settings** to display this card. +- **General settings**: **Normalize audio**, **Quantize audio**, and the **Sample rate** and **NES + frequency** a library is built for. +- **Source settings**: **Drive**, which sets how hard the channels are pushed, and the channels and + bends of the recording you selected in the converter's list. +- **Advanced settings**: **Method** and **Feature scaling**, which set how the app measures the + frequencies in the audio, the number of **Workers**, and the library and output folders. Choose + **View ▸ Show advanced settings** to show this card. [Reconstruction + algorithms](../concepts/reconstruction.md) explains **Method** and **Feature scaling**. -Changing any of these updates your configuration, which is saved to `config.json` -(see [Where your files live](files.md)). +The app saves your changes to `config.json`. See [Where your files live](files.md). -## In the configuration file +## The configuration file -The configuration holds more than the interface shows. You can edit the finer -controls directly in `config.json`: the -[selector](../concepts/reconstruction.md) (greedy or Viterbi), the phase aligner, -the scoring weights and distance metric, the number of candidates kept per frame, -and so on. The [configuration file -reference](../formats/configuration.md) lists every section and key, and -[Reconstruction algorithms](../concepts/reconstruction.md) explains what they do -and lists the defaults. +`config.json` has more settings than the **Main** tab shows. For example, you can change: -You can also load and save whole configurations from the **Reconstruction** menu -(**Load generation settings...** and **Save generation settings...**), or point -the app at one on the [command line](command-line.md) with `--config`. +- the [selector](../concepts/reconstruction.md), greedy or Viterbi +- the phase aligner +- the scoring weights and the distance measure +- the number of candidates kept for each frame -## Deployment settings +The [configuration file reference](../formats/configuration.md) lists every setting. +[Reconstruction algorithms](../concepts/reconstruction.md) explains what each one does, and lists +the defaults. -Two settings — the log level and strict history checking — are decided when the -application is packaged, so they are not part of your configuration. They exist -for development and support. +To load or save a whole configuration, use **Reconstruction ▸ Load generation settings...** and +**Reconstruction ▸ Save generation settings...**. On the [command line](command-line.md), add +`--config` to a command. diff --git a/docs/guide/converting.md b/docs/guide/converting.md index ce11aacb6..68a6e85bd 100644 --- a/docs/guide/converting.md +++ b/docs/guide/converting.md @@ -15,17 +15,17 @@ The **Converter** card lists the recordings a conversion uses. Add them from the Turn on **Playback ▸ Autoplay** (`Ctrl+P`) to play a recording with a single click. This lets you listen through a folder before adding anything from it. With Autoplay off, right-click a recording and choose **Play**. -Adding a folder opens a small window while the folder is read. **Stop** ends the search and leaves the list as it was. A folder with no recordings inside it says so and adds nothing. +Adding a folder opens a small window while the app reads the folder. **Stop** ends the search and keeps the list as it was. If the folder has no recordings, the window says so and the list stays the same. **x** removes a row from the list. Removing a folder removes every recording in it. -Click a row to select it. The **Source settings** card then shows that recording. Right-clicking a row selects it as well. Press `Del` to remove the selected row. Closing a folder that holds the selected recording clears the selection. +Click a row to select it. The **Source settings** card then shows that recording. Right-clicking a row selects it as well. Press `Del` to remove the selected row. Closing a folder that contains the selected recording clears the selection. ## Choosing which channels a recording uses The NES has four sound channels: **Pulse 1**, **Pulse 2**, **Triangle**, and **Noise**. Every recording in the list has a checkbox for each channel. Check the channels that the recording may use. Press `1` to `4` to switch a channel on or off for the selected row. -A folder represents all the recordings inside it. Its checkbox shows their channel assignments: +A folder's checkboxes show the channels of all the recordings inside it: - checked if all recordings use the channel, - filled with the channel's color if only some recordings use it, @@ -33,31 +33,31 @@ A folder represents all the recordings inside it. Its checkbox shows their chann Click the checkbox to change the channel for all recordings in the folder. To change the channel for one recording, open the folder first: click the marker next to the folder name, or double-click the folder name. Each recording inside then has its own checkboxes. -The **Source settings** card has two more settings. **Drive** sets how hard the channels are pushed. It applies to the whole conversion. +The **Source settings** card has **Drive**, which sets how hard the channels are pushed. It applies to the whole conversion. -Below it, the card shows the name of the recording you selected in the list and a checkbox for each of its channels. **Pulse 1**, **Pulse 2**, and **Triangle** also have a **bend** checkbox. This tunes each note to the recording's exact pitch. Noise has no bend. +Below **Drive**, the card shows the name of the recording you selected in the list and a checkbox for each of its channels. **Pulse 1**, **Pulse 2**, and **Triangle** also have a **bend** checkbox. This tunes each note to the recording's exact pitch. Noise has no bend. -**Channels per source** limits how many channels one recording can use at the same time, from 1 to 4. Set it to 1 to make each recording use one channel. +**Channels per source** is on the **Converter** card, below the list. It limits how many channels one recording can use at the same time, from 1 to 4. Set it to 1 to make each recording use one channel. ## One reconstruction each, or one mix from all -**Output**, at the top of the **Converter** card, decides what the conversion produces: +**Output**, at the top of the **Converter** card, sets what the conversion makes: - **One per recording** — each recording in the list becomes its own reconstruction. Recordings added with a folder are saved in a matching folder structure. - **One from all** — all recordings are mixed into a single reconstruction. Folders are replaced by the recordings inside them. A mix can hold up to eight recordings. If you switch to **One from all** with more than eight recordings in the list, a dialog asks which ones to mix. The same dialog opens when you add a folder with more recordings than the mix has room for. -The dialog lists the same rows as the card and shows how many recordings you have selected. Double-click a row to hear the recording. **Add** becomes available when your selection fits. A full mix cannot accept more recordings, so uncheck one before checking another. +The dialog lists the same rows as the card and shows how many recordings you selected. Double-click a row to hear the recording. **Add** becomes available when your selection fits. When the mix is full, uncheck a recording before you check another one. -Once a mix has two or more recordings, the rows are grouped into **levels**. A level decides which recordings choose their channels first. Everything on level 1 is given channels before anything on level 2. This lets a lead melody take the channels it needs before a background part does. +When a mix has two or more recordings, the rows are grouped into **levels**. Levels set which recordings get their channels first. Recordings on level 1 get channels before recordings on level 2. This lets a lead melody take the channels it needs before a background part does. Drag a row onto another row to put them on the same level. Drag it into the gap between levels to give it a level of its own. You can also right-click a row to use the same commands. -**Order** decides how the levels take turns: +**Order** sets how the levels take turns: - **Round robin** — every level gets a turn in each round. -- **Strict** — one level is filled before the next one chooses. +- **Strict** — one level gets all its channels before the next level chooses. ## Running a conversion @@ -67,13 +67,13 @@ Click the button under **Output** to start the conversion. The button's label te When the conversion finishes, click **Load** to open the result on the **Reconstruction** tab, where you can [listen to it and export it](reconstruction.md). After a conversion of several recordings, the button reads **Open** instead. -The first conversion with a given set of settings builds the [instruction library](../concepts/instruction-library.md) it needs. This takes a while. Later conversions with the same settings reuse the library. +The first conversion with new settings builds the [instruction library](../concepts/instruction-library.md) for those settings. This takes a while. Later conversions with the same settings use the same library. ## The instruction library The **Instructions** tab (`F4`) builds and browses the [instruction library](../concepts/instruction-library.md), the catalog of NES tones that a conversion searches. -A conversion builds the library it needs on its own, so you rarely need to go there. The tab is useful for building a library before a long session and for exploring what your settings can produce. +A conversion builds the library it needs by itself, so you rarely need this tab. Use it to build a library before a long session, or to explore the sounds your settings can make. Select an instruction to see its **Waveform** and **Spectrum**. This lets you see and hear a single NES tone on its own. diff --git a/docs/guide/files.md b/docs/guide/files.md index 7c15b2c5b..ec0483e27 100644 --- a/docs/guide/files.md +++ b/docs/guide/files.md @@ -1,58 +1,54 @@ # Where your files live -_SampleToNES_ keeps your work in a **SampleToNES** folder inside your documents -folder: +_SampleToNES_ saves your work in a **SampleToNES** folder inside your documents folder: - Windows: `C:\Users\\Documents\SampleToNES` - macOS: `/Users//Documents/SampleToNES` - Linux: `/home//Documents/SampleToNES` -Inside it: +The folder contains: -- `instructions/` — the generated [instruction libraries](../formats/instruction-libraries.md) (`.ins`) -- `reconstructions/` — saved [reconstructions](../formats/reconstructions.md) (`.stn`) -- `projects/` — saved [projects](../formats/projects.md) (`.stp`) -- `config.json` — your generation [configuration](configuration.md) +- `instructions/`: your [instruction libraries](../formats/instruction-libraries.md) (`.ins`) +- `reconstructions/`: your [reconstructions](../formats/reconstructions.md) (`.stn`) +- `projects/`: your [projects](../formats/projects.md) (`.stp`) +- `config.json`: your [configuration](configuration.md) -You can point the library and output folders elsewhere from the **Main** tab's -**Advanced settings**, or from the right-click menu in its **Filesystem** browser. +To use other folders for libraries and reconstructions, open **Advanced settings** on the **Main** +tab, or right-click in the **Filesystem** browser. ## File types -| Type | What it is | Where it lives | +| Type | What it is | Where it is saved | | --- | --- | --- | | `.ins` | [instruction library](../formats/instruction-libraries.md) | `instructions/` | | `.stn` | [reconstruction](../formats/reconstructions.md) | `reconstructions/` | | `.stp` | [project](../formats/projects.md) | `projects/` | -| `.fti` | FamiTracker instrument (exported) | wherever you choose | -| `.ftm` | FamiTracker module (exported) | wherever you choose | -| `.json` | Bitphase instrument preset (exported) | wherever you choose | -| `.btp` | Bitphase project (exported) | wherever you choose | -| `.nsf` | NES sound file (exported) | wherever you choose | +| `.fti` | FamiTracker instrument | where you choose | +| `.ftm` | FamiTracker module | where you choose | +| `.json` | Bitphase instrument preset | where you choose | +| `.btp` | Bitphase project | where you choose | +| `.nsf` | NES sound file | where you choose | -The `.fti` and `.ftm` files are what you load into -[FamiTracker](../formats/famitracker.md), `.json` and `.btp` are what -[Bitphase](../formats/bitphase.md) reads, and an `.nsf` plays on its own in a NES -sound player or on the console; the rest are _SampleToNES_'s own formats. +The last five types are exports: -## Exported files +- Open `.fti` and `.ftm` files in [FamiTracker](../formats/famitracker.md). +- Open `.json` and `.btp` files in [Bitphase](../formats/bitphase.md). +- Play `.nsf` files in an NSF player or on a NES. -The extension names what an export is written for: `.fti` and `.ftm` go to -FamiTracker, `.json` and `.btp` to Bitphase, and `.nsf` to a NES sound player. The -save dialog offers the file types that fit what you are exporting and fills in the -extension of the type it is set to. Exporting one channel offers all three, so -switching the type there switches the target; typing an extension yourself picks it -directly. +## Naming exported files -What you name in the dialog also names what a tracker lists: +The save dialog lists the file types that fit your export, and adds the extension of the type you +pick. When you export one channel, the dialog lists all three instrument types, so you can pick the +program there. You can also type the extension yourself. -| Export | You name | What is written | +The name you type also names the instrument in the tracker: + +| Export | The name you type | What the app saves | | --- | --- | --- | -| **Instruments** panel ▸ **Export instrument...** | the file | that file, its instrument carrying the name you gave | -| **Reconstruction ▸ Export instruments** | the batch | one file per channel beside that name, each named ` (channel)`, or a single file at that name for `.nsf` | -| **File ▸ Export** | the file | that file, holding the whole song | - -So exporting a `Kick` reconstruction to FamiTracker instruments writes -`Kick (pulse1).fti`, `Kick (triangle).fti`, and one file for every other channel the -reconstruction uses, all in the folder you chose. An `.nsf` gathers every channel into -one program, so it is the single file you named. +| **Instruments** panel ▸ **Export instrument...** | the file | one file, and the instrument inside it has the same name | +| **Reconstruction ▸ Export instruments** | the set | one file per channel, each named ` (channel)`. For `.nsf`, one file with the name you typed | +| **File ▸ Export** | the file | one file with the whole song | + +For example, exporting a reconstruction named `Kick` to FamiTracker instruments saves +`Kick (pulse1).fti`, `Kick (triangle).fti`, and one file for each other channel the reconstruction +uses. An `.nsf` export saves every channel in one file. diff --git a/docs/guide/getting-started.md b/docs/guide/getting-started.md index 04a9530bf..f3dcd0a7d 100644 --- a/docs/guide/getting-started.md +++ b/docs/guide/getting-started.md @@ -1,8 +1,7 @@ # Getting started -Two quick paths through _SampleToNES_: turning a sound into FamiTracker -instruments, and building a whole song. Both assume it is already -[installed](installation.md). +This page shows two quick ways to start: turning a sound into FamiTracker instruments, and building +a song. First, [install](installation.md) _SampleToNES_. ## Reconstruct a sound into FamiTracker instruments @@ -10,8 +9,8 @@ instruments, and building a whole song. Both assume it is already 2. In the **Filesystem** browser, double-click an audio file (WAV, MP3, FLAC, OGG, AIFF, or AU) — or Ctrl-click a folder, to reconstruct every audio file inside it. -3. Optionally click the recording in the list and choose which channels it uses - under **Source settings**, and adjust **General settings**. Each recording +3. If you want, click the recording in the list and check the channels it may use + under **Source settings**, and change **General settings**. Each recording needs at least one channel. 4. Click the button under **Output** to start the conversion. The first time you convert with a given set of settings, the [instruction diff --git a/docs/guide/installation.md b/docs/guide/installation.md index 74efb5882..4e2236393 100644 --- a/docs/guide/installation.md +++ b/docs/guide/installation.md @@ -1,77 +1,78 @@ # Installation -There are two ways to run _SampleToNES_: a **standalone build** (the easiest, and -all most people need) and **running from source** (for development, and on macOS). -GPU acceleration is optional. +You can install _SampleToNES_ in three ways: -## Requirements +- **Download a release.** This is the easiest way on Windows and Linux. +- **Install from PyPI.** This works on Windows, macOS and Linux. +- **Run from source.** Use this to change the code or to build the app yourself. -Everyone needs: +GPU acceleration is optional. The last section of this page explains it. -- A supported operating system: Windows, macOS, or Linux -- [Python 3.12 or newer](https://www.python.org/downloads/) +## Download a release -Some setups need a little more — each is covered in the relevant section below: +1. Open the [releases page](https://github.com/JakimPL/SampleToNES/releases). +2. Download the file for your system: Windows or Linux. +3. Extract the file. +4. Start `sampletones` in the extracted folder. On Windows, double-click `sampletones.exe`. -- **On Linux**, a few system packages are required to build or run: the Tk - file-dialog and PortAudio audio libraries. Install them with `make system-deps`. -- **On macOS**, audio playback is compiled against PortAudio on install, so the - library comes from [Homebrew](https://brew.sh): `make system-deps` installs it. - Tk and the graphics libraries arrive with the official Python installer. -- **On Windows**, the official Python installer and the packaged dependencies - cover everything. -- **Running from source** also needs [uv](https://docs.astral.sh/uv/). -- **GPU acceleration** needs an NVIDIA GPU with a current driver. The matching CuPy - build is installed for you, so the driver is all you need — on Linux and Windows alike. +On Linux, you may need to make the file executable first: `chmod +x sampletones`. -## Standalone build +## Install from PyPI -A ready-to-run executable built on your machine. You only need Python 3.12. +You need [Python 3.12 or newer](https://www.python.org/downloads/). -### Windows +On Linux and macOS, install the audio and file dialog libraries first: -1. Install Python 3.12. -2. Double-click `install.bat`. It builds `bin\sampletones.exe`. -3. Double-click `bin\sampletones.exe` to start. +```sh +sudo apt-get install libportaudio2 libasound2 python3-tk # Debian and Ubuntu +brew install portaudio # macOS +``` -### Linux +Then install _SampleToNES_ in an environment of its own, and start it: -1. Install the audio and file-dialog system packages: `make system-deps` (or run - `./scripts/linux/build/dependencies.sh`). -2. Install Python 3.12, then run `./install.sh` in a terminal. It builds a - `bin/sampletones` executable. -3. Run `./bin/sampletones` to start. +```sh +uv tool install sampletones # or: pipx install sampletones +sampletones +``` + +You can also run `pip install sampletones` inside an active virtual environment. ## Run from source -For development, and the way to run on macOS. Requires [uv](https://docs.astral.sh/uv/) -— and, on Linux and macOS, the system packages from the requirements above: +You need: + +- [Python 3.12 or newer](https://www.python.org/downloads/) +- [uv](https://docs.astral.sh/uv/) +- `make` + +Get the code and set it up: ```sh -make system-deps # Linux and macOS: install the system libraries -make setup # create the environment and install the sampletones command -make run # run the app +git clone https://github.com/JakimPL/SampleToNES.git +cd SampleToNES +make system-deps # Linux and macOS: installs the system libraries +make setup # creates the environment and installs the sampletones command +make run # starts the app ``` -To update the global command after pulling new changes, re-run `make setup`. +After you pull new changes, run `make setup` again. -## GPU acceleration +On Windows and Linux, you can also build a standalone app from the source code: -_SampleToNES_ can use an NVIDIA GPU (via [CuPy](https://cupy.dev/) and CUDA) to -speed up instruction-library generation and reconstruction. `make setup` detects -your NVIDIA driver and installs the matching CuPy build automatically: +- **Windows**: double-click `install.bat`. It builds `bin\sampletones.exe`. +- **Linux**: run `make system-deps`, then `./install.sh`. It builds `bin/sampletones`. -```sh -make setup # installs GPU support when a supported driver is present -make setup GPU=0 # forces the CPU (NumPy) backend -``` +## GPU acceleration + +_SampleToNES_ can use an NVIDIA graphics card to build libraries and convert recordings faster. It +needs an NVIDIA card with a current driver, on Windows or Linux. On macOS, the app uses the CPU. -A current NVIDIA driver is all you need: the CUDA components ship with the CuPy -build, on Linux and Windows alike. Detection reads the driver's CUDA version — -version 12 and newer use the default build, version 11 uses a legacy build. On -macOS, _SampleToNES_ runs on the CPU. +- **From source**: `make setup` checks your NVIDIA driver and installs the matching GPU support. + `make setup GPU=0` installs the app for the CPU alone. +- **From PyPI**: add the `gpu` extra, `uv tool install "sampletones[gpu]"`. If your driver supports + CUDA 11 only, use the `gpu-cuda11` extra instead. --- -Once it runs, [Getting started](getting-started.md) walks through your first +Once the app runs, [Getting started](getting-started.md) walks you through your first reconstruction and your first song. diff --git a/docs/guide/interface.md b/docs/guide/interface.md index 1458e8dfd..f81e8af86 100644 --- a/docs/guide/interface.md +++ b/docs/guide/interface.md @@ -9,7 +9,7 @@ _SampleToNES_ has four tabs. Use `F1` to `F4` to switch between them: You usually work through the tabs in this order. Convert your files on **Main**. When the conversion finishes, click **Load** to open the result on **Reconstruction**. From there, **Add to Sequencer** adds the reconstruction to a song. -The **Instructions** tab is optional. It lets you explore individual _instructions_ — the unit blocks produced by the NES sound processor. +The **Instructions** tab is optional. Use it to explore single _instructions_: the smallest sounds the NES sound chip makes. ## The menus @@ -26,7 +26,7 @@ Each menu covers one kind of work: Two items are easy to miss: - **View ▸ Show advanced settings** shows the **Advanced settings** card on the **Main** tab. It contains the generation method, feature scaling, worker count, and library and output folders. [Configuration](configuration.md) explains each one. -- **Playback ▸ Audio settings...** chooses the device, sample rate, and buffer size you listen through. These are separate from **Sample rate** and **NES frequency** on the **Main** tab. Those settings decide how the audio is reconstructed. +- **Playback ▸ Audio settings...** chooses the device, sample rate, and buffer size you listen through. **Sample rate** and **NES frequency** on the **Main** tab are different settings: they set how the audio is converted. Project properties belong to a project and are covered in the [sequencer guide](sequencer.md). diff --git a/docs/guide/reconstruction.md b/docs/guide/reconstruction.md index 1b597ea1c..1ded88432 100644 --- a/docs/guide/reconstruction.md +++ b/docs/guide/reconstruction.md @@ -18,17 +18,17 @@ If the reconstruction you have open has unsaved changes, the app asks whether to To keep frequently used reconstructions within reach, right-click a reconstruction or a folder and choose **Mark as favorite**. Check **Favorites only** to show only those items. -The **Source** card switches playback between **Reconstruction** and **Original audio**, so you can compare the two. The **Waveform** card shows each channel and has a checkbox for each one. Keys `1` to `4` toggle the same checkboxes. +The **Source** card switches playback between **Reconstruction** and **Original audio**, so you can compare the two. The **Waveform** card has a checkbox for each channel. Keys `1` to `4` switch the same checkboxes. ## Hearing what each recording contributed The **Stems** card lists the recordings used to build a reconstruction. It groups them by the level each one was given. Every row has a checkbox for each channel that the recording used, and the checkbox at the front toggles all of them. -Uncheck a channel to silence it in the waveform, playback, original audio, and WAV export. This lets you hear what one recording contributed, channel by channel. These checkboxes only change what you hear. They do not change anything that is saved. +Uncheck a channel to silence it in the waveform, in playback, in the original audio and in a WAV export. This lets you hear what one recording added, channel by channel. The saved reconstruction keeps every channel, whatever you uncheck. **Collapse levels** shows the whole list as one table. -**x** at the end of a row removes that recording from the reconstruction after asking you to confirm. The recording becomes silent and its row disappears, and the change is written when you save the reconstruction. A reconstruction keeps at least one recording, so the last row's **x** is disabled. +**x** at the end of a row removes that recording from the reconstruction. The app asks you to confirm first. The recording goes silent, its row disappears, and the change is saved when you save the reconstruction. A reconstruction needs at least one recording, so the **x** of the last row is disabled. ## Editing instruments @@ -42,11 +42,11 @@ Each channel has its own set of sequences: Type `|` before a value to mark where the sequence repeats while a note is held. `15 14 | 12 10` plays the attack once and then loops the last two values. -Clear a sequence to leave that setting to the channel. For example, an instrument with an empty volume sequence plays at whatever volume the channel is set to. Each channel shows how many bytes its instrument takes on the NES, so you can see what an edit costs. +Clear a sequence to use the channel's own setting. For example, an instrument with an empty volume sequence plays at the volume the channel is set to. Each channel shows how many bytes its instrument takes on the NES, so you can see how much space an edit uses. -You can also edit a hand-written **instrument** here — a voice with no recording behind it, described in the [sequencer guide](sequencer.md#voices-samples-and-instruments). Right-click it in the **Voices** list and choose **Edit**. +You can also edit an **instrument** here, a voice you write by hand. The [sequencer guide](sequencer.md#voices-samples-and-instruments) describes instruments. Right-click one in the **Voices** list and choose **Edit**. -Since an instrument has no recording, the panel shows **Audition** instead of the pitch steppers. Choose **Pulse**, **Triangle**, or **Noise**, and the note keys then play the instrument on that sound generator. +For an instrument, the panel shows **Audition** in place of the pitch steppers. Choose **Pulse**, **Triangle** or **Noise**, and the note keys play the instrument on that sound generator. ## Exporting @@ -57,6 +57,6 @@ You export a reconstruction from the **Reconstruction** menu: - **Export instruments ▸ NSF program...** writes a single `.nsf` file that plays the whole reconstruction on a NES. - **Export to WAV...** renders the audio using the channel and stem checkboxes you have set. -**Export instrument...** in the **Instruments** panel writes only the channel you are looking at, in whichever format you choose in the save dialog. See [where your files live](files.md#exported-files). +**Export instrument...** in the **Instruments** panel writes only the channel you are looking at, in whichever format you choose in the save dialog. See [where your files live](files.md#naming-exported-files). To use a reconstruction in a song, right-click it and choose **Add to Sequencer**. diff --git a/docs/guide/sequencer.md b/docs/guide/sequencer.md index 728c8f69b..fc6984124 100644 --- a/docs/guide/sequencer.md +++ b/docs/guide/sequencer.md @@ -1,307 +1,281 @@ # The sequencer -The **Sequencer** tab is a tracker: it arranges voices into a song across the four -NES channels and exports it as a FamiTracker -[module](../formats/famitracker.md) (`.ftm`). It works on a -[project](../formats/projects.md), so start one with **File ▸ New project** (or -open an existing `.stp`). The pattern grid and order sit in the center, a browser -for pulling in reconstructions on the left, and the module settings, voice list, -and undo history on the right. +The **Sequencer** tab (`F3`) is a tracker. You use it to arrange voices into a song on the four NES +channels. You can export the song as a FamiTracker [module](../formats/famitracker.md) (`.ftm`), +play it back, or render it to audio. + +The sequencer works on a [project](../formats/projects.md). Choose **File ▸ New project** to start +one, or open an existing `.stp` file. ## Voices: samples and instruments -A song is built from **voices**, and there are two kinds. A **sample** is a -reconstruction brought in as something a row can play. An **instrument** is written by -hand — envelopes with no recording behind them — for the melodies and basses you write -yourself. Both sit in the **Voices** list on the right, numbered together, and a -mark at the front of each row says which kind it is. The mark carries a color as well -as a shape — amber for a sample, magenta for an instrument — and the same two colors -name a voice in the pattern grid and in the history, so the two kinds read apart -wherever one is named. - -Four ways bring a voice in, and the **Voice** menu holds all four: - -| Way in | What arrives | -|--------|--------------| -| **New instrument** | An instrument holding a note at full volume, ready to place and hear | -| **Add sample from file...** | A reconstruction saved anywhere on disk, as a sample | +A song plays **voices**. There are two kinds: + +- A **sample** is a reconstruction that the song plays. +- An **instrument** is a sound you write by hand. Use instruments for melodies and bass lines. + +Both kinds are in the **Voices** list, numbered together. + +The **Voice** menu adds a voice in four ways: + +| Menu item | What it adds | +|-----------|--------------| +| **New instrument** | An instrument that plays a note at full volume, ready to place and hear | +| **Add sample from file...** | A reconstruction file from anywhere on your disk, as a sample | | **Import instrument...** | A FamiTracker instrument file (`.fti`), as an instrument | -| **Add to Sequencer** | The reconstruction the **Reconstruction** tab holds, as a sample | - -The first three also sit at the top of the **Voices** list, and on the list's own -menu — right-click below the rows to reach it. **Add to Sequencer** is on the -**Browser** on the Sequencer tab (right-click a reconstruction) and on the -**Reconstruction** tab. If a reconstruction was made at a different NES frequency -than the project and the project already has voices, _SampleToNES_ warns with -**Different NES frequency**; **Add anyway** adds it regardless. - -A new instrument starts out holding a note at full volume, so you can place it and -hear it straight away; give it the sound you want on the **Reconstruction** tab -(right-click ▸ **Edit**). See [editing instruments](reconstruction.md#editing-instruments). -An imported `.fti` arrives with the volume, arpeggio, and duty-cycle envelopes the -file states, and **Instrument imported** names anything the file held on a tracker's -own terms that the voice leaves behind — see [reading an instrument +| **Add to Sequencer** | The reconstruction open on the **Reconstruction** tab, as a sample | + +The first three items are also at the top of the **Voices** list. Right-click below the rows of +the list to open the same menu. + +**Add to Sequencer** is also on the **Reconstruction** tab. On the Sequencer tab, right-click a +reconstruction in the **Browser** to find it. + +If a reconstruction uses a different NES frequency than the project, the app shows **Different +NES frequency**. Click **Add anyway** to add it. + +To change how a new instrument sounds, right-click it and choose **Edit**. The instrument opens on +the **Reconstruction** tab. See [editing instruments](reconstruction.md#editing-instruments). + +An imported `.fti` file keeps its volume, arpeggio and duty cycle sequences. **Instrument +imported** lists any other settings of the file, which the instrument leaves out. See [reading an instrument file](../formats/famitracker.md#c-reading-an-instrument-file). -**New instrument from ▸ _channel_** on a sample's menu writes what one of its channels -plays into an instrument of its own, so a recorded part becomes envelopes you edit by -hand. +A sample's right-click menu has **New instrument from**. Choose a channel to copy what the sample +plays on that channel into a new instrument you can edit. -Right-click any voice to **Edit**, **Rename**, **Duplicate**, **Remove**, or reorder it. -**Export instrument...** writes the voice out as a `.fti` another tracker reads — a -sample holds one instrument per channel it plays, so it asks which. The **Edit** menu -carries the same actions for the voice you have picked. The right-click menu also names -how much room the voice takes on the NES — a sample's total and then each channel it -plays, and an instrument's single figure. The figures are in bytes, and they count what a -FamiTracker export saves. Removing a voice that patterns still use asks first, because it -clears every row that references it. +Right-click any voice to use these commands: -Hovering a row says what that voice is in one line: its name, whether it is a sample or -an instrument, the channels a sample plays, and the room it takes. +- **Edit**, **Rename**, **Duplicate** and **Remove**. +- Commands that move the voice up or down the list. +- **Export instrument...**, which saves the voice as an `.fti` file. A sample has one instrument + per channel, so the app asks which channel to export. + +The **Edit** menu has the same commands for the voice you selected. + +The right-click menu also shows how many bytes the voice takes on the NES. For a sample, it shows +the total and the size of each channel. + +If patterns still use a voice, removing the voice asks you first. Removing it clears every row +that uses it. + +Hover over a voice to see its name, its kind, its channels and its size. ## Writing a pattern -The **Tracker** grid is the pattern editor. Each row is one step in time; the -columns are the **Sample** and the four channels — **Pulse 1**, **Pulse 2**, -**Triangle**, **Noise** — each carrying a voice, a pitch, and a volume. Click a cell -and type its value. Right-clicking a cell opens the rest of the operations — **Set -voice**, **Note off**, **Clear cell** and **Clear row**, transpose and volume -adjustments, **Play from here** to audition from the cursor row, and **Play from -this frame** to start at the top of the shown frame. - -The **Sample** column places a sample across every channel its reconstruction -covers, and clears the rest of the row. It takes samples alone: an instrument sounds -on the one channel that names it, so **Set voice** lists instruments there grayed out, -and a number typed over one leaves the cell reading what it held. Name an instrument -in the channel column you want it on. - -A cell holding a voice wears that voice's color — amber for a sample, magenta for an -instrument — while an empty cell, a cut, and a `?` where the **Sample** column's -channels disagree read in a plain gray. Silencing a channel dims its cells and keeps -those colors, so a muted column stays as readable as the rest. - -## Reading and typing a pitch - -A pitch cell holds one number, and it reads in the terms of the voice the channel is -carrying. A sample was converted at a pitch of its own, so its cells read as steps -from it — `+00` plays it as recorded, `+0C` an octave up. An instrument sounds at -whatever note a row names it with, so its cells read as the notes they sound — `C-4`, -`A#3`. -A row that only bends a note reads the same way as the row that started it. - -Type a note into an instrument's cell piano-style: the bottom two rows of the keyboard are -one octave (`Z` `S` `X` `D` `C` …) and the two above them the next (`Q` `2` `W` `3` -`E` …). **Octave** above the grid says where the bottom row opens. The keys work on -a sample's cell too, writing the step that reaches the note you pressed. The noise -channel selects one of sixteen periods rather than a note, so its cells are typed as -a signed value. +The **Tracker** grid is the pattern editor. Each row is one step in time. The grid has a +**Sample** column and a column for each channel: **Pulse 1**, **Pulse 2**, **Triangle** and +**Noise**. Each channel column has a voice, a pitch and a volume. + +Click a cell and type its value. + +Right-click a cell for more commands: + +- **Set voice** chooses the voice for the cell. +- **Note off** stops the note. +- **Clear cell** and **Clear row** empty the cell or the whole row. +- Transpose and volume commands change the pitch and volume. +- **Play from here** plays from the row you clicked. +- **Play from this frame** plays from the top of the frame on screen. + +The **Sample** column places a sample on every channel the sample uses, and clears the other +channels of the row. The **Sample** column takes samples only. To place an instrument, use the +column of the channel you want it on. + +A `?` in the **Sample** column means the channels of that row play different voices. + +## Typing a pitch + +A pitch cell shows a number. What the number means depends on the voice: + +- For a sample, the number is a step from the pitch the sample was recorded at. `+00` plays the + sample as recorded. `+0C` plays it one octave higher. +- For an instrument, the number is a note, such as `C-4` or `A#3`. + +Type notes on your keyboard like a piano: + +- The bottom two rows of keys play one octave: `Z` `S` `X` `D` `C` and so on. +- The two rows above them play the next octave: `Q` `2` `W` `3` `E` and so on. + +**Octave**, above the grid, sets the octave of the bottom row. The same keys work in a sample's +cell and type the step that plays the note you pressed. + +The noise channel has sixteen sounds in place of notes. Type a noise cell as a signed step, such as +`+03` or `-02`. ## Arranging the song -A song plays a sequence of patterns, and the **Order** grid sets that sequence — -one column per position, with a row for the master and each channel. Type an entry -to place a pattern, or right-click a frame for the rest: **Duplicate** repeats the -frame with the patterns it already plays, **Clone** gives the copy patterns of its -own so you can change it on its own, and **Insert frame**, **Clear frame**, -**Remove**, the moves, and **Play from this frame** do what they say. +A song plays patterns in a sequence. The **Order** grid sets that sequence. Each column is one +position in the song. The grid has a row for the **Master** and a row for each channel. + +Type a pattern number in the **Order** grid to place a pattern. Right-click a frame for more +commands: + +- **Duplicate** repeats the frame with the same patterns. +- **Clone** repeats the frame with new copies of its patterns, so you can change them separately. +- **Insert frame**, **Clear frame** and **Remove** add, empty and delete frames. +- **Play from this frame** plays from that frame. + +## Selecting, copying and pasting + +You can select a block of cells in both grids, then copy, cut, paste or delete it. -## Working on a block +To select cells: -Both grids take a **selection** — a rectangle of cells you copy, cut, paste, and -delete in one go. Hold `Shift` and press the arrow keys to reach out from the -cursor, or drag the pointer across the cells; `Shift`+click carries the selection to -the cell you click. Dragging past the edge of a grid scrolls it along, so a selection -can run further than the screen shows. Any plain move, and `Escape`, puts the -selection away again. +- Hold `Shift` and press the arrow keys. +- Drag across the cells with the mouse. +- `Shift`+click a cell to extend the selection to it. + +Dragging past the edge of a grid scrolls the grid. Any plain arrow key, or `Esc`, clears the +selection. | Key | Action | |-----|--------| -| `Shift`+arrows | Reach the selection out a cell at a time | -| `Shift+Home` / `Shift+End` | Reach it to the first or the last row (tracker) or position (order) | +| `Shift`+arrows | Extend the selection by one cell | +| `Shift+Home` / `Shift+End` | Extend the selection to the first or last row (tracker) or position (order) | | `Ctrl+A` | Select the whole frame, or the whole order | -| `Ctrl+Shift+A` | Select the column you are in (tracker), or your channel's row (order) | -| `Ctrl+Alt+A` | Select the subcolumn you are in (tracker) | +| `Ctrl+Shift+A` | Select your column (tracker), or your channel's row (order) | +| `Ctrl+Alt+A` | Select your subcolumn (tracker) | | `Ctrl+C` | Copy | -| `Ctrl+X` | Cut — copy, then empty what was selected | -| `Ctrl+V` | Paste, starting at the cursor | +| `Ctrl+X` | Cut | +| `Ctrl+V` | Paste at the cursor | | `Del` | Empty the selection | -Copy, cut, paste and delete act on the cell the cursor stands on when nothing is -selected, so copying one cell needs no selection first. All four sit on each grid's -right-click menu: raised inside a selection they act on the whole of it, raised -anywhere else on the cell you clicked. Each grid keeps its own copy, so a tracker -block pastes into the tracker and an order block into the order. - -The **Select** keys work from the cell you are on and reach the whole length of the -grid. They sit on the right-click menu too. +With nothing selected, copy, cut, paste and delete work on the cell under the cursor. The same +commands are on each grid's right-click menu. Right-click inside a selection to use them on the +whole selection. -A paste is anchored: the block starts at the cell you paste onto and lands the rest -down and to the right of it. +Each grid has its own copy. A block copied in the tracker pastes into the tracker, and a block +copied in the order pastes into the order. -In the **Tracker**, a block keeps the kinds of the cells it came from — a transpose -lands in a transpose, a volume in a volume, whichever column you paste onto — and -whatever reaches past the last row or the last column is left out. A cell reading -`?`, where the **Sample** column's channels disagree, passes over its target and -leaves what was there; an empty cell empties it. +A paste starts at the cursor and fills down and to the right: -In the **Order**, a block pasted past the last frame grows the song to hold it, and -one reaching past the **Noise** row stops there. The **Master** row copies the index -its channels share and reads `?` when they differ, which pasted leaves each channel -as it was. +- In the **Tracker**, each cell keeps its kind. A volume pastes into a volume column, wherever you + paste. Cells past the last row or column are dropped. A `?` cell leaves the target cell as it + was. +- In the **Order**, a paste past the last frame adds frames to the song. A paste stops at the + **Noise** row. -Emptying cells keeps the rows and frames they sit in, and every block action is one -step in the history, so a single **Undo** takes it all back. +Emptying cells keeps the rows and frames. Every copy, cut, paste and delete is one step in the +history, so one **Undo** reverses it. -A copy also goes to your desktop's clipboard as plain text, so a block carries between -two open windows of _SampleToNES_ — copy in one, paste in the other — and you can paste -one into a message to show someone what you wrote. Anything else on the clipboard -leaves you with the last block you copied here. Notes travel by their number in the -**Voices** list, so a block pasted into another project plays whichever voice holds -that number there. +A copy also goes to your clipboard as text. You can paste a block into another open window of +_SampleToNES_, or into a message. Voices are copied by their number in the **Voices** list, so in +another project the block plays the voices with those numbers. -## Transposing and shading +## Transpose and volume -In the **Tracker**, transpose and volume move whatever the selection covers, so a -run of rows nudges together. +In the **Tracker**, transpose and volume keys change every cell in the selection. | Key | Action | |-----|--------| -| `Ctrl+Up` / `Ctrl+Down` | Transpose a semitone | -| `Ctrl+Shift+Up` / `Ctrl+Shift+Down` | Transpose an octave | -| `Alt+Up` / `Alt+Down` | Volume a step | -| `Alt+Shift+Up` / `Alt+Shift+Down` | Volume four steps | +| `Ctrl+Up` / `Ctrl+Down` | Transpose by a semitone | +| `Ctrl+Shift+Up` / `Ctrl+Shift+Down` | Transpose by an octave | +| `Alt+Up` / `Alt+Down` | Change the volume by one step | +| `Alt+Shift+Up` / `Alt+Shift+Down` | Change the volume by four steps | -Control carries pitch, Alt carries volume, and Shift makes the step the bigger one. -With nothing selected they act on the cell the cursor stands on, and the same -commands sit on the right-click menu with these keys beside them. +With nothing selected, the keys change the cell under the cursor. The same commands are on the +right-click menu. ## Playing the song -The transport below the grid plays the song, and the keyboard drives playback -throughout the tab: +The transport under the grid plays the song. These keys work anywhere on the tab: | Key | Action | |-----|--------| -| `Space` | Play, or pause and resume what is playing | +| `Space` | Play, or pause and resume | | `Shift+Space` | Play from the start | -| `Ctrl+Space` | Play from the frame currently shown | -| `Ctrl+Shift+Space` | Play from the cursor's row in the pattern grid | -| `Escape` | Stop | -| `Ctrl+L` | **Loop song** — start the song over each time it reaches the end | +| `Ctrl+Space` | Play from the frame on screen | +| `Ctrl+Shift+Space` | Play from the cursor's row | +| `Esc` | Stop | +| `Ctrl+L` | **Loop song**: start the song again when it ends | -`Escape` silences everything, including a sample preview. The same commands sit on -the **Playback** menu and the transport buttons. +`Esc` also stops a sample preview. The same commands are on the **Playback** menu. -## Following the playhead +## Following the playback -**Playback ▸ Follow playback** chooses how far the view travels with the sounding -row. Each mode carries a key of its own, so you can change your mind while the song -plays, and the choice is remembered for the next time you launch: +**Playback ▸ Follow playback** chooses whether the view moves with the song. The app remembers your +choice. -| Mode | Key | Where the view goes | +| Mode | Key | What the view does | |------|-----|---------------------| -| **Follow rows** | `Ctrl+F` | Scrolls the pattern grid to keep the sounding row on screen, and shows the frame being played | -| **Follow patterns** | `Ctrl+Shift+F` | Shows the frame being played, and leaves the scroll where you put it | -| **Don't follow** | `Ctrl+Alt+F` | Holds the view where you put it | - -The **Order** grid marks the frame being played under every mode, and the tracker -marks the sounding row of the frame it shows — so a held view still shows the -playhead each time the song passes through the frame you are editing. -**Follow rows** is the one that moves the grid while you play, which is what makes -the other two the modes to type in: they hold the view still under your cursor while -the song runs. - -## Listening to one channel at a time - -Channel names are switches. Click **Triangle** at the top of the tracker to silence -that channel: its name grays, its column and its row in the **Order** grid go -neutral, and its notes dim — still readable, still editable, just not sounding. -Click the name again to bring it back. The same click works on the channel's name -in the **Order** grid, and both grids show every change, so a channel looks the same -wherever you see it. +| **Follow rows** | `Ctrl+F` | Scrolls the tracker to the playing row, and shows the playing frame | +| **Follow patterns** | `Ctrl+Shift+F` | Shows the playing frame, and keeps your scroll position | +| **Don't follow** | `Ctrl+Alt+F` | Stays where you put it | + +In every mode, the **Order** grid marks the playing frame, and the tracker marks the playing row. +Use **Follow patterns** or **Don't follow** to type while the song plays. + +## Muting channels + +Click a channel's name at the top of the tracker to mute the channel. Click the name again to +unmute it. The channel names in the **Order** grid work the same way. | Gesture | Action | |---------|--------| -| Click a channel's name | Silence it, or bring it back | -| `Ctrl`+click a channel's name | Solo it — silence the other three; `Ctrl`+click again returns the mix you had | -| Click **Sample** (tracker) or **Master** (order) | Silence every channel, or bring them all back | -| Right-click any name | The same actions as a menu | - -The **Playback ▸ Channels** submenu carries the same mix: a check marks each channel -that sounds, and **Unmute all channels** returns the whole set. `1` to `4` do the -same from the keyboard, one key per channel, wherever the grids are not holding your -cursor — inside them the digits enter values. - -Muting is for listening only. The song keeps every channel, so saving, exporting a -module, and undo all work on the full arrangement, and a mute survives undo and -redo. Toggling during playback is heard within about a quarter second. Opening, -creating, or closing a project starts a fresh listening session with every channel -audible. +| Click a channel's name | Mute or unmute the channel | +| `Ctrl`+click a channel's name | Solo the channel. `Ctrl`+click again to restore the previous mix | +| Click **Sample** (tracker) or **Master** (order) | Mute or unmute every channel | +| Right-click a name | The same commands as a menu | + +**Playback ▸ Channels** shows which channels play. **Unmute all channels** unmutes all four. Keys +`1` to `4` mute and unmute a channel when the cursor is outside the grids. Inside the grids, the +digit keys type values. + +Muting changes only what you hear. Saving, exporting, rendering and undo use every channel. Opening, +creating or closing a project unmutes all channels. ## Timing and properties -Set the song's timing in **Module options** on the right: **Rows** per pattern, -**Tempo**, **Speed**, and the **NES frequency**. Changing the **NES frequency** -after voices exist re-times how they all play back, so it asks **Change NES -frequency** first (with a **Don't ask again** option). +**Module options** sets the song's timing: **Rows** per pattern, **Tempo**, **Speed** and **NES +frequency**. If the project has voices, changing **NES frequency** changes how they play, so the +app asks **Change NES frequency** first. Check **Don't ask again** to skip the question. + +**File ▸ Project properties...** sets the title, the author and the comment. The exported module +includes them. -The project's title, author, and comment — which carry into the exported module — -are set in **Project properties**, from the button or **File ▸ Project -properties...**, along with the meter the song is counted in. +**Project properties** also sets the meter: -**First highlight** and **Second highlight** are that meter: how many rows make a -beat, and how many make a bar. The tracker tints the row that opens each one. The -bar divided by the beat is how many beats you hear in a bar, so the default 4 and -16 give four beats of four rows — common time. Waltz time keeps the four-row beat -and shortens the bar to 12, for three beats. The beat is what the tempo counts, so -the two together say how fast the song is felt as well as how it looks. +- **First highlight** is the number of rows in a beat. +- **Second highlight** is the number of rows in a bar. -The meter also places the song's timing. Most tempos ask for a row length the engine -can only reach on average, so the rows of a bar differ a little: the meter gives the -extra time to the row that opens the bar, then to the row that opens each beat, which -keeps the beat audible where you expect it. +The tracker marks the first row of each beat and each bar. The defaults, 4 and 16, give four beats +of four rows in a bar. For waltz time, set **Second highlight** to 12, which gives three beats. + +The tempo counts beats, so the meter also changes how fast the song feels. [Tempo as a +groove](../formats/bitphase.md#d-tempo-as-a-groove) explains how the app spreads a tempo over the +rows. ## Undo and export -Every change is undoable. The **History** panel on the right shows the stack, with -**Undo** and **Redo** (also on the **Edit** menu); click any entry to jump straight -to that point. +You can undo every change. The **History** panel lists your changes. **Undo** and **Redo** are also +on the **Edit** menu. Click a change in the **History** panel to go back to that point. -When the song is ready, **Export as FamiTracker module** (or **File ▸ Export -FamiTracker module...**) writes the `.ftm`. See -[FamiTracker export](../formats/famitracker.md) for what the module contains and -the limits it respects. +To export the song: -**File ▸ Export ▸ NSF program...** writes the song as an `.nsf` instead: a program -the console itself plays, carrying its own player, so it needs no tracker to sound. -The console holds one program in 32 KB, so a long song can outgrow it — the export -says so rather than writing a file that plays part of itself. See -[NSF export](../formats/nsf.md) for what the file holds, and -[song compression](../concepts/compression.md) for how a song of minutes is fitted -into that space. +- **File ▸ Export ▸ FamiTracker module...** saves an `.ftm` file. See [FamiTracker + export](../formats/famitracker.md) for its contents and limits. +- **File ▸ Export ▸ Bitphase project...** saves a `.btp` file. +- **File ▸ Export ▸ NSF program...** saves an `.nsf` file, which the NES or an NSF player plays + directly. An NSF program has room for 32 KB, so the app tells you when a song is too long. See [NSF + export](../formats/nsf.md) and [song compression](../concepts/compression.md). ## Rendering to audio -A module is for a tracker. To get a file anyone can play, use **File ▸ Render -song...** (`Ctrl+Shift+E`), which writes the whole song as audio. - -The dialog holds the choices: +**File ▸ Render song...** (`Ctrl+Shift+E`) saves the whole song as an audio file that any player +opens. | Setting | What it does | |---------|--------------| -| **Format** | **WAV** for the full-quality file, **MP3** for a smaller one | -| **Sample rate** | How many samples a second the file holds; 44100 Hz is the usual choice | -| **Bit depth** (WAV) | How finely each sample is stored. 16-bit PCM is the usual choice; 8-bit is there for the crunch the NES itself has | -| **Bitrate** (MP3) | How much the file spends per second — higher sounds better and takes more room. What is on offer depends on the sample rate, so the list follows when you change it | -| **Normalize peak** | Lifts the whole song so its loudest moment reaches full scale, keeping the balance between channels as it was | -| **File** | Where it is written. **Browse...** opens the save dialog, clicking the path shows where the file is going in your file manager, and the folder you pick is offered again next time | - -**Length** tells you how long the file will be before you start. **Render** begins, -and a bar reports how far it has got; **Cancel** stops it and leaves the file -unwritten. When it finishes, _SampleToNES_ shows the file it wrote — click the path -to open its folder. - -A render takes the song itself, once through, with every channel sounding: muting -and **Loop song** are for listening and stay out of the file. It is one of the long -jobs that run alone, so the item is unavailable while a conversion or a library -generation is going, and those wait for a render in the same way. +| **Format** | **WAV** for full quality, **MP3** for a smaller file | +| **Sample rate** | Samples per second. 44100 Hz is the usual choice | +| **Bit depth** (WAV) | 16-bit PCM is the usual choice. 8-bit sounds rougher, like the NES | +| **Bitrate** (MP3) | A higher bitrate sounds better and makes a larger file. The choices depend on the sample rate | +| **Normalize peak** | Makes the song louder until its loudest moment reaches full volume, keeping the balance between channels | +| **File** | Where the file is saved. **Browse...** opens the save dialog. Click the path to open its folder | + +**Length** shows how long the file will be. Click **Render** to start. **Cancel** stops the render +and saves no file. When the render finishes, click the path to open its folder. + +A render plays the song once, with every channel, whatever you muted or looped. While a render +runs, conversions and library builds wait, and a render waits for them in the same way. diff --git a/docs/index.md b/docs/index.md index 58b11407e..08611ae56 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,11 +1,11 @@ # SampleToNES documentation -_SampleToNES_ approximates an audio sample using only the sound channels of the -NES's 2A03 chip — two pulse waves, a triangle and noise — and lets you arrange -the results into a song and export them to [FamiTracker](glossary.md#famitracker), -to [Bitphase](glossary.md#bitphase), or as an `.nsf` program the console itself plays. -This is the documentation for using it, understanding how it works, and building -on it. +_SampleToNES_ rebuilds an audio sample from the sound channels of the NES's 2A03 chip: two pulse +waves, a triangle and noise. You can arrange the results into a song. You can export them to +[FamiTracker](glossary.md#famitracker), to [Bitphase](glossary.md#bitphase), or as an `.nsf` program +the NES plays. + +This documentation explains how to use the app, how it works, and how to build on it. The sections below are grouped by what you want to do. They assume different starting points: the guide needs no prior knowledge, the concepts and formats @@ -16,7 +16,7 @@ sections are written for programmers. The [**guide**](guide/) walks through the application from installation onward. -- [Installation](guide/installation.md) — the standalone build, running from source, and GPU acceleration. +- [Installation](guide/installation.md) — a release download, PyPI, running from source, and GPU acceleration. - [Getting started](guide/getting-started.md) — your first reconstruction and your first song. - [The interface](guide/interface.md) — the four tabs, the menus, and the keyboard shortcuts. - [Converting audio](guide/converting.md) — the Main tab: gathering recordings, choosing channels, and running a conversion. @@ -57,25 +57,35 @@ worked examples. ## Development -The [**development**](development/) section is for contributors. +The [**development**](development/) section is for contributors. The documents at its top cover +the whole repository. The documents about the graphical application and about releases have +directories of their own. - [Architecture](development/architecture.md) — the application's layers and the contracts between them. - [Package layers](development/packages.md) — the packages the repository divides into, and the order they import each other in. -- [Undo engine](development/undo.md) — the design of the undo/redo subsystem. -- [Sequencer blocks](development/sequencer-blocks.md) — the rules copy, cut, paste and delete follow on both grids. -- [Keyboard and actions](development/keyboard.md) — how a press reaches behavior, and how an action is declared and shown. -- [Identifier vocabularies](development/vocabularies.md) — the keys display text is looked up by, and the tags DearPyGui knows a widget by. -- [Colors and palettes](development/palette.md) — how a color is written, composed, and handed to DearPyGui. -- [The render thread](development/render-thread.md) — how work reaches DearPyGui from another thread, and what each crossing costs. -- [Playback](development/playback.md) — the audio transport shared by every view, and rendering the song to a file. -- [Progress](development/progress.md) — how a long operation says how far it has come, in one process and across the pool's workers. -- [Console player](development/player.md) — the 6502 driver an `.nsf` carries, the codec that fits a song beside it, and how both are verified. -- [Reconstruction browser](development/browser.md) — how a reconstructions directory becomes the tree both browser tabs render, and what narrows it. -- [Configuration](development/config-organization.md) — how the YAML configuration package is laid out. +- [Tooling](development/tooling.md) — the `sampletones` command, the tools package and the bootstrap scripts: what each runs on and what it may import. - [Coding guidelines](development/guidelines.md) — conventions for the codebase. -- [Dependencies](development/dependencies.md) — the libraries _SampleToNES_ builds on. +- [Console player](development/player.md) — the 6502 driver an `.nsf` carries, the codec that fits a song beside it, and how both are verified. +- [Progress](development/progress.md) — how a long operation reports its progress, in one process and across the pool's workers. - [Bugs and to-dos](development/bugs-and-todos.md) — the working ledger of known gaps. +### The application + +- [Undo engine](development/application/undo.md) — the design of the undo/redo subsystem. +- [Sequencer blocks](development/application/sequencer-blocks.md) — the rules copy, cut, paste and delete follow on both grids. +- [Keyboard and actions](development/application/keyboard.md) — how a press reaches behavior, and how an action is declared and shown. +- [Identifier vocabularies](development/application/vocabularies.md) — the keys display text is looked up by, and the tags DearPyGui knows a widget by. +- [Colors and palettes](development/application/palette.md) — how a color is written, composed, and handed to DearPyGui. +- [The render thread](development/application/render-thread.md) — how work reaches DearPyGui from another thread, and what each crossing costs. +- [Playback](development/application/playback.md) — the audio transport shared by every view, and rendering the song to a file. +- [Reconstruction browser](development/application/browser.md) — how a reconstructions directory becomes the tree both browser tabs render, and what narrows it. +- [Configuration](development/application/config-organization.md) — how the YAML configuration package is laid out. + +### Releases + +- [Data compatibility](development/release/compatibility.md) — how a file saved by an older version is upgraded to the current format when it loads. +- [Dependencies](development/release/dependencies.md) — the libraries _SampleToNES_ builds on. + ## Glossary The [glossary](glossary.md) defines the recurring terms — NES hardware, the diff --git a/install.bat b/install.bat index 1c375bb96..4d4861042 100644 --- a/install.bat +++ b/install.bat @@ -1,13 +1,7 @@ @echo off setlocal -set SCRIPT_DIR=%~dp0 - -call "%SCRIPT_DIR%scripts\windows\lib\root.bat" || exit /b - -call "%SCRIPT_DIR%scripts\windows\build\python.bat" || exit /b -call "%SCRIPT_DIR%scripts\windows\build\venv.bat" || exit /b -call "%SCRIPT_DIR%scripts\windows\build\sampletones.bat" %* || exit /b -call "%SCRIPT_DIR%scripts\windows\build\build.bat" %* || exit /b - +python "%~dp0scripts\bundle.py" %* +set STATUS=%ERRORLEVEL% pause +exit /b %STATUS% diff --git a/install.sh b/install.sh index 73eea2e26..2bdbee4e3 100755 --- a/install.sh +++ b/install.sh @@ -1,16 +1,3 @@ #!/usr/bin/env bash -set -e - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" - -if [[ "$(uname -s)" == "Darwin" ]]; then - exec bash "${SCRIPT_DIR}/scripts/macos/build/no_bundle.sh" "./install.sh" -fi - -source "${SCRIPT_DIR}/scripts/linux/lib/root.sh" - -source "${SCRIPT_DIR}/scripts/linux/build/python.sh" -source "${SCRIPT_DIR}/scripts/linux/build/venv.sh" -bash "${SCRIPT_DIR}/scripts/linux/build/sampletones.sh" "$@" -bash "${SCRIPT_DIR}/scripts/linux/build/build.sh" "$@" +exec python3 "$(dirname "$0")/scripts/bundle.py" "$@" diff --git a/pyproject.toml b/pyproject.toml index 20448235e..0b2680fc8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -99,10 +99,6 @@ build-backend = "hatchling.build" conflicts = [[{ extra = "gpu" }, { extra = "gpu-cuda11" }]] [tool.hatch.build.targets.wheel] -exclude = [ - "src/sampletones_player/driver/assembler", - "src/sampletones_player/driver/assembly", -] packages = [ "src/sampletones", "src/sampletones_application", @@ -111,7 +107,7 @@ packages = [ "src/sampletones_core", "src/sampletones_player", "src/sampletones_shared", - "src/sampletones_synthesis", + "src/sampletones_tools", ] [tool.black] @@ -122,6 +118,7 @@ target-version = ["py312"] profile = "black" line_length = 120 known_first_party = [ + "bootstrap", "sampletones", "sampletones_application", "sampletones_assets", @@ -129,12 +126,12 @@ known_first_party = [ "sampletones_core", "sampletones_player", "sampletones_shared", - "sampletones_synthesis", + "sampletones_tools", ] [tool.pytest.ini_options] addopts = "--import-mode=importlib" -pythonpath = ["."] +pythonpath = [".", "scripts"] [tool.coverage.run] source = [ @@ -143,7 +140,7 @@ source = [ "sampletones_core", "sampletones_player", "sampletones_shared", - "sampletones_synthesis", + "sampletones_tools", ] [tool.coverage.report] @@ -160,7 +157,8 @@ files = [ "src/sampletones_core", "src/sampletones_player", "src/sampletones_shared", - "src/sampletones_synthesis", + "src/sampletones_tools", + "scripts", ] exclude = ["tests"] disallow_subclassing_any = true diff --git a/run.bat b/run.bat deleted file mode 100644 index 170ec8572..000000000 --- a/run.bat +++ /dev/null @@ -1,2 +0,0 @@ -@echo off -uv run sampletones %* diff --git a/run.sh b/run.sh deleted file mode 100755 index 1716dccd6..000000000 --- a/run.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env bash -set -e -uv run sampletones "$@" diff --git a/scripts/archive_bundle.py b/scripts/archive_bundle.py new file mode 100644 index 000000000..19a586392 --- /dev/null +++ b/scripts/archive_bundle.py @@ -0,0 +1,92 @@ +import argparse +import sys +import zipfile +from pathlib import Path +from typing import Final, List, Sequence + +from bootstrap.layout import BUNDLES, DISTRIBUTION, repository_root +from bootstrap.platforms.factory import current_platform +from bootstrap.platforms.protocol import Platform +from bootstrap.project import Project, read_project + +ARCHIVE_COMPRESSION: Final[int] = zipfile.ZIP_DEFLATED +ARCHIVE_SUFFIX: Final[str] = ".zip" + + +def archive_root(project: Project, label: str) -> str: + """The directory every archived entry sits under, which names the archive too. + + Args: + project: The project, whose name and release tag lead the name. + label: The platform the bundle was built for, such as ``windows-x86_64``. + + Returns: + str: The name, such as ``sampletones-v0.3.0-windows-x86_64``. + """ + return f"{project.name}-{project.tag}-{label}" + + +def bundle_entries(source: Path) -> List[Path]: + """Every file and directory inside a built bundle, ordered so repeated runs archive alike.""" + return sorted(source.rglob("*")) + + +def archive_name(path: Path, *, source: Path, root: str) -> str: + """The location a bundle path takes inside the archive, gathered under a single root directory.""" + return f"{root}/{path.relative_to(source).as_posix()}" + + +def write_archive(source: Path, archive: Path, *, root: str) -> List[Path]: + """Archives a built bundle directory, placing every entry under ``root``. + + Each entry is read where it lies, which keeps the archive available while a virus scanner or a + process that ran the executable holds a handle inside the directory, and carries over the + permission bits that let the launcher run once the archive is extracted. + """ + archive.parent.mkdir(parents=True, exist_ok=True) + entries = bundle_entries(source) + with zipfile.ZipFile(archive, "w", ARCHIVE_COMPRESSION) as bundle: + for path in entries: + bundle.write(path, archive_name(path, source=source, root=root)) + + return entries + + +def archive_release(root: Path, platform: Platform, *, label: str) -> int: + """Archives the release bundle built under ``root`` into its bundles directory. + + Args: + root: The repository the bundle was built in. + platform: The system the bundle was built for, which places the bundle's directory. + label: The platform's name in the archive, such as ``windows-x86_64``. + + Returns: + int: ``0`` once the archive is written, ``1`` where the release bundle is missing. + + Raises: + SystemExit: If the system builds no bundle. + """ + project = read_project(root) + source = platform.bundling().launcher(root / DISTRIBUTION, name=project.name, release=True).parent + if not source.is_dir(): + print(f"::error::Bundle directory {source} is missing") + return 1 + + name = archive_root(project, label) + archive = root / BUNDLES / f"{name}{ARCHIVE_SUFFIX}" + entries = write_archive(source, archive, root=name) + print(f"Archived {len(entries)} entries from {source} into {archive}") + return 0 + + +def main(argv: Sequence[str]) -> int: + """Archives the release bundle into the bundles directory, named by the version and the platform.""" + parser = argparse.ArgumentParser(description="Archive the release bundle under a versioned root.") + parser.add_argument("--label", required=True, help="the platform the bundle was built for, such as windows-x86_64") + arguments = parser.parse_args(list(argv)) + + return archive_release(repository_root(), current_platform(), label=arguments.label) + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) diff --git a/scripts/assets/icons.py b/scripts/assets/icons.py deleted file mode 100755 index c6484eb94..000000000 --- a/scripts/assets/icons.py +++ /dev/null @@ -1,46 +0,0 @@ -#!/usr/bin/env python3 - -""" -Writes the application icon suite from the packaged mark definition. - -The mark, its template and the code drawing them live in `sampletones_assets/mark`; this -script points them at the directory the icons are shipped from. - -Usage: - python scripts/assets/icons.py # write the suite into src/sampletones_assets/icons -""" - -import argparse -import sys -from pathlib import Path -from typing import Final, Sequence - -from sampletones_assets.mark.specification import Mark -from sampletones_assets.mark.suite import write_icon_suite - -REPOSITORY_ROOT: Final[Path] = Path(__file__).resolve().parents[2] -ICONS_DIRECTORY: Final[Path] = REPOSITORY_ROOT / "src" / "sampletones_assets" / "icons" - - -def main(argv: Sequence[str]) -> int: - """Writes the icon suite and reports each file it produced.""" - - parser = argparse.ArgumentParser( - description="Write the application icon suite from the mark definition.", - ) - parser.add_argument( - "--directory", - type=Path, - default=ICONS_DIRECTORY, - help="directory receiving the icon files", - ) - arguments = parser.parse_args(list(argv)) - - for path in write_icon_suite(arguments.directory, Mark.load()): - print(f"Wrote {path}") - - return 0 - - -if __name__ == "__main__": - raise SystemExit(main(sys.argv[1:])) diff --git a/scripts/bootstrap/__init__.py b/scripts/bootstrap/__init__.py new file mode 100644 index 000000000..e29480584 --- /dev/null +++ b/scripts/bootstrap/__init__.py @@ -0,0 +1,3 @@ +from bootstrap.interpreter import REQUIRED_VERSION, require_python + +require_python(REQUIRED_VERSION) diff --git a/scripts/bootstrap/cuda.py b/scripts/bootstrap/cuda.py new file mode 100644 index 000000000..ec43e3868 --- /dev/null +++ b/scripts/bootstrap/cuda.py @@ -0,0 +1,142 @@ +import re +import shutil +import subprocess +from dataclasses import dataclass +from pathlib import Path +from typing import Final, Mapping, Optional, Sequence, Tuple + +from bootstrap.platforms.protocol import Platform +from bootstrap.project import GPU_CUDA11_EXTRA, GPU_EXTRA + +NVIDIA_SMI: Final[str] = "nvidia-smi" +CUDA12_MAJOR: Final[int] = 12 +CUDA11_MAJOR: Final[int] = 11 +CUDA_VERSION_PATTERN: Final[re.Pattern[str]] = re.compile(r"CUDA Version\s*:?\s*(\d+)\.(\d+)") +QUERY_ARGUMENTS: Final[Tuple[Tuple[str, ...], ...]] = ((), ("-q",)) + + +@dataclass(frozen=True, kw_only=True) +class CudaDetection: + """The CuPy extra the host's NVIDIA driver maps to, and what was found to choose it. + + Attributes: + extra: The optional-dependency extra matching the driver, or ``None`` for the CPU backend. + reason: One line saying what was found and what it selects. + """ + + extra: Optional[str] + reason: str + + +def find_nvidia_smi(platform: Platform, environment: Mapping[str, str]) -> Optional[Path]: + """The driver's ``nvidia-smi``: on ``PATH``, or at a location the system's driver installs it. + + Args: + platform: The system, which names the locations off ``PATH``. + environment: The variables those locations are read from. + + Returns: + Optional[Path]: The executable, or ``None`` where no NVIDIA driver is installed. + """ + located = shutil.which(NVIDIA_SMI) + if located is not None: + return Path(located) + + return next((candidate for candidate in platform.nvidia_smi_locations(environment) if candidate.exists()), None) + + +def _run_nvidia_smi(nvidia_smi: Path, arguments: Sequence[str]) -> Optional[str]: + try: + completed = subprocess.run( + [str(nvidia_smi), *arguments], + capture_output=True, + text=True, + check=False, + ) + except (OSError, subprocess.SubprocessError): + return None + + if completed.returncode != 0: + return None + + return completed.stdout + completed.stderr + + +def query_driver_cuda_version(nvidia_smi: Path) -> Optional[Tuple[int, int]]: + """The newest CUDA version the driver supports, as ``(major, minor)``. + + ``nvidia-smi`` reports the driver's newest supported CUDA version, which governs CuPy wheel + selection: CuPy wheels bind to the driver, whatever CUDA Toolkit the system carries. The + default table carries the value, and ``-q`` provides a structured fallback for formats that + omit it. + """ + for arguments in QUERY_ARGUMENTS: + output = _run_nvidia_smi(nvidia_smi, arguments) + if output is None: + continue + + match = CUDA_VERSION_PATTERN.search(output) + if match is not None: + return int(match.group(1)), int(match.group(2)) + + return None + + +def select_extra(cuda_version: Optional[Tuple[int, int]]) -> Optional[str]: + """The optional-dependency extra matching a driver's CUDA version. + + CUDA 12 and newer drivers map to the ``gpu`` extra (``cupy-cuda12x``): a CUDA 12 wheel runs on + every 12.x driver through Enhanced Compatibility and on newer drivers through backward + compatibility. CUDA 11 drivers map to the legacy ``gpu-cuda11`` extra. Older drivers map to + ``None``, keeping the CPU backend. + """ + if cuda_version is None: + return None + + major = cuda_version[0] + if major >= CUDA12_MAJOR: + return GPU_EXTRA + + if major == CUDA11_MAJOR: + return GPU_CUDA11_EXTRA + + return None + + +def _describe(*, cuda_version: Optional[Tuple[int, int]], extra: Optional[str]) -> str: + if cuda_version is None: + return "NVIDIA driver detected, though its CUDA version was unreadable; keeping the CPU (NumPy) backend." + + major, minor = cuda_version + if extra is None: + return ( + f"Detected CUDA {major}.{minor}, which predates the supported CuPy builds; " + "keeping the CPU (NumPy) backend." + ) + + return f"Detected an NVIDIA driver supporting CUDA {major}.{minor}; selecting the '{extra}' extra." + + +def detect(platform: Platform, environment: Mapping[str, str]) -> CudaDetection: + """The CUDA capability of the host and the CuPy extra that matches it. + + Args: + platform: The system the detection runs on. + environment: The variables the driver's locations are read from. + + Returns: + CudaDetection: What was found and the extra it selects. + """ + if platform.cpu_backend_reason is not None: + return CudaDetection(extra=None, reason=platform.cpu_backend_reason) + + nvidia_smi = find_nvidia_smi(platform, environment) + if nvidia_smi is None: + return CudaDetection( + extra=None, + reason="No NVIDIA driver detected (nvidia-smi is absent); keeping the CPU (NumPy) backend.", + ) + + cuda_version = query_driver_cuda_version(nvidia_smi) + extra = select_extra(cuda_version) + return CudaDetection(extra=extra, reason=_describe(cuda_version=cuda_version, extra=extra)) diff --git a/scripts/bootstrap/files.py b/scripts/bootstrap/files.py new file mode 100644 index 000000000..d38000af7 --- /dev/null +++ b/scripts/bootstrap/files.py @@ -0,0 +1,22 @@ +import shutil +from pathlib import Path + + +def remove_path(path: Path) -> bool: + """Removes what stands at ``path``, a directory with everything under it or a single file. + + Args: + path: The file or directory. + + Returns: + bool: Whether anything stood there. + """ + if path.is_dir(): + shutil.rmtree(path) + return True + + if path.exists(): + path.unlink() + return True + + return False diff --git a/scripts/bootstrap/interpreter.py b/scripts/bootstrap/interpreter.py new file mode 100644 index 000000000..f1ae54587 --- /dev/null +++ b/scripts/bootstrap/interpreter.py @@ -0,0 +1,32 @@ +import sys +from typing import Final, Tuple + +REQUIRED_VERSION: Final[Tuple[int, int]] = (3, 12) +DOWNLOADS: Final[str] = "https://www.python.org/downloads/" + + +def require_python(version: Tuple[int, int]) -> None: + """Holds the interpreter running the script to ``version`` or newer. + + Importing ``bootstrap`` runs it first, so an older interpreter is told the version it needs + before a script reaches the standard library that version brings. + + Args: + version: The oldest major and minor version the scripts run on. + + Raises: + SystemExit: If the interpreter is older, naming where a newer one is downloaded. + """ + if tuple(sys.version_info[:2]) >= version: + return + + major, minor = version + raise SystemExit( + f"ERROR: Python {major}.{minor} or newer is required.\n" + f"Please install Python {major}.{minor}+ from {DOWNLOADS}" + ) + + +def running_version() -> str: + """The version of the interpreter running the script, as ``major.minor.micro``.""" + return ".".join(str(part) for part in sys.version_info[:3]) diff --git a/scripts/bootstrap/layout.py b/scripts/bootstrap/layout.py new file mode 100644 index 000000000..68f1acf1e --- /dev/null +++ b/scripts/bootstrap/layout.py @@ -0,0 +1,32 @@ +from pathlib import Path +from typing import Final, Tuple + +REPOSITORY_ROOT: Final[Path] = Path(__file__).resolve().parents[2] +PROJECT_FILE: Final[str] = "pyproject.toml" +SOURCE_DIRECTORY: Final[str] = "src" +BENCHMARKS_DIRECTORY: Final[str] = "tests/benchmarks" +DISTRIBUTION: Final[str] = "bin" +BUNDLES: Final[str] = "bundles" +BUILD_ENVIRONMENT: Final[str] = ".venv-build" +RELEASE_HOOK: Final[str] = "scripts/runtime_hooks/release_environment.py" +NOTICES: Final[Tuple[str, ...]] = ("LICENSE", "THIRD-PARTY-NOTICES.md", "THIRD-PARTY-LICENSES.txt") +BUILD_TOOLS: Final[Tuple[str, ...]] = ("PIL",) +CLEAN_ARTIFACTS: Final[Tuple[str, ...]] = (DISTRIBUTION, BUNDLES, "build", "dist", "htmlcov", ".coverage") +CLEAN_PATTERNS: Final[Tuple[str, ...]] = ("*.spec",) +CACHE_DIRECTORIES: Final[Tuple[str, ...]] = ("__pycache__",) +CACHE_DIRECTORY_SUFFIXES: Final[Tuple[str, ...]] = (".egg-info",) +CACHE_FILE_SUFFIXES: Final[Tuple[str, ...]] = (".pyc",) +ENVIRONMENTS: Final[Tuple[str, ...]] = (".git", ".venv", BUILD_ENVIRONMENT) + + +def repository_root() -> Path: + """The repository the scripts belong to, which every build and clean-up runs against. + + Raises: + FileNotFoundError: If the scripts lie outside a SampleToNES checkout. + """ + project = REPOSITORY_ROOT / PROJECT_FILE + if not project.is_file() or not (REPOSITORY_ROOT / SOURCE_DIRECTORY).is_dir(): + raise FileNotFoundError(f"SampleToNES project root not found (expected {project} beside {SOURCE_DIRECTORY})") + + return REPOSITORY_ROOT diff --git a/scripts/bootstrap/passes.py b/scripts/bootstrap/passes.py new file mode 100644 index 000000000..ef21e7e2c --- /dev/null +++ b/scripts/bootstrap/passes.py @@ -0,0 +1,71 @@ +from dataclasses import dataclass +from pathlib import Path +from typing import List, Mapping, Sequence, Tuple + +from bootstrap.processes import Runner + + +@dataclass(frozen=True) +class Pass: + """One named command a script announces and runs from the repository root. + + Attributes: + name: What the step is called in the report and on the command line. + announcement: The line printed as the step starts. + command: The program and its arguments, run from the repository root. + """ + + name: str + announcement: str + command: Tuple[str, ...] + + +def run_pass( + current: Pass, + *, + root: Path, + runner: Runner, + environment: Mapping[str, str], +) -> int: + """Announces one pass and runs its command from the repository. + + Args: + current: The pass. + root: The repository, which the command runs in. + runner: What runs the command. + environment: The variables the command sees. + + Returns: + int: The status the command exited with. + """ + print(current.announcement) + return runner(current.command, cwd=root, environment=environment, quiet=False) + + +def run_passes( + passes: Sequence[Pass], + *, + root: Path, + runner: Runner, + environment: Mapping[str, str], +) -> List[str]: + """Runs every pass in order and names the ones that failed. + + Every pass runs whatever the earlier ones reported, so one run shows everything that is + wrong. + + Args: + passes: The steps, in order. + root: The repository, which every command runs in. + runner: What runs the commands. + environment: The variables the commands see. + + Returns: + List[str]: The names of the passes that exited with a failure, in order. + """ + failed: List[str] = [] + for current in passes: + if run_pass(current, root=root, runner=runner, environment=environment) != 0: + failed.append(current.name) + + return failed diff --git a/src/sampletones_assets/mark/__init__.py b/scripts/bootstrap/platforms/__init__.py similarity index 100% rename from src/sampletones_assets/mark/__init__.py rename to scripts/bootstrap/platforms/__init__.py diff --git a/scripts/bootstrap/platforms/bundling.py b/scripts/bootstrap/platforms/bundling.py new file mode 100644 index 000000000..02eaad2ac --- /dev/null +++ b/scripts/bootstrap/platforms/bundling.py @@ -0,0 +1,38 @@ +from dataclasses import dataclass +from pathlib import Path + + +@dataclass(frozen=True) +class Bundling: + """What building a standalone bundle takes on one system. + + Attributes: + icon: The icon file the bundle is stamped with, relative to the repository. + executable_suffix: What the launcher's file name ends with. + pyaudio_advice: How to supply PortAudio to the build interpreter. + tkinter_advice: How to supply Tk to the build interpreter for a release bundle. + tkinter_warning: What a development bundle built without Tk does about file dialogs. + """ + + icon: str + executable_suffix: str + pyaudio_advice: str + tkinter_advice: str + tkinter_warning: str + + def launcher(self, distribution: Path, *, name: str, release: bool) -> Path: + """The executable PyInstaller writes under ``distribution``. + + Args: + distribution: The directory the bundle is written into. + name: The bundle's name. + release: Whether the bundle is a release, which is a directory beside its launcher. + + Returns: + Path: The launcher. + """ + executable = f"{name}{self.executable_suffix}" + if release: + return distribution / name / executable + + return distribution / executable diff --git a/scripts/bootstrap/platforms/factory.py b/scripts/bootstrap/platforms/factory.py new file mode 100644 index 000000000..8b5d4df71 --- /dev/null +++ b/scripts/bootstrap/platforms/factory.py @@ -0,0 +1,37 @@ +import platform +from typing import Dict, Final + +from bootstrap.platforms.linux import LINUX, Linux +from bootstrap.platforms.macos import DARWIN, MacOS +from bootstrap.platforms.protocol import Platform +from bootstrap.platforms.windows import WINDOWS, Windows + +PLATFORMS: Final[Dict[str, Platform]] = { + LINUX: Linux(), + WINDOWS: Windows(), + DARWIN: MacOS(), +} + + +def platform_named(system: str) -> Platform: + """The platform a system name selects. + + Args: + system: The name ``platform.system()`` reports. + + Returns: + Platform: The platform. + + Raises: + SystemExit: If SampleToNES supports no system of that name. + """ + chosen = PLATFORMS.get(system) + if chosen is None: + raise SystemExit(f"ERROR: {system} is not a system SampleToNES builds on; supported: {', '.join(PLATFORMS)}.") + + return chosen + + +def current_platform() -> Platform: + """The platform the script runs on.""" + return platform_named(platform.system()) diff --git a/scripts/bootstrap/platforms/linux.py b/scripts/bootstrap/platforms/linux.py new file mode 100644 index 000000000..2c0691ac2 --- /dev/null +++ b/scripts/bootstrap/platforms/linux.py @@ -0,0 +1,78 @@ +from pathlib import Path +from typing import Dict, Final, Mapping, Optional, Sequence, Tuple + +from bootstrap.platforms.bundling import Bundling +from bootstrap.platforms.unix import unix_interpreter + +LINUX: Final[str] = "Linux" +PNG_ICON: Final[str] = "src/sampletones_assets/icons/sampletones.png" +SYSTEM_PACKAGES: Final[Tuple[str, ...]] = ( + "libportaudio2", + "libasound-dev", + "libpulse-dev", + "portaudio19-dev", + "python3-tk", + "tk-dev", + "tcl-dev", + "libgl1", + "libegl1", + "libx11-6", + "libx11-xcb1", + "libxcursor1", + "libxi6", + "libxinerama1", + "libxrandr2", + "libxrender1", + "libxxf86vm1", +) +BUNDLING: Final[Bundling] = Bundling( + icon=PNG_ICON, + executable_suffix="", + pyaudio_advice=( + "Run 'make system-deps' to install the PortAudio packages, then 'make build' to reinstall the dependencies." + ), + tkinter_advice="Run 'make system-deps' to install python3-tk, then build again.", + tkinter_warning=( + "This bundle opens file dialogs through zenity or kdialog, which the machine running it has to " + "provide. Run 'make system-deps' to install python3-tk and carry Tk as a self-contained fallback." + ), +) + + +class Linux: + """A Debian-based Linux: packages through apt, and a launcher named after the project alone.""" + + @property + def name(self) -> str: + return LINUX + + @property + def cpu_backend_reason(self) -> Optional[str]: + return None + + def interpreter(self, environment: Path) -> Path: + return unix_interpreter(environment) + + def bundling(self) -> Bundling: + return BUNDLING + + def missing_package_manager(self) -> Optional[str]: + return None + + def system_packages(self) -> Sequence[Sequence[str]]: + return ( + ("sudo", "apt-get", "update"), + ("sudo", "apt-get", "install", "-y", *SYSTEM_PACKAGES), + ) + + def setup_variables(self, base: Mapping[str, str], *, machine: str) -> Dict[str, str]: + del machine + return dict(base) + + def build_flags(self, *, machine: str) -> Sequence[str]: + del machine + return () + + def nvidia_smi_locations(self, environment: Mapping[str, str]) -> Sequence[Path]: + del environment + return () diff --git a/scripts/bootstrap/platforms/macos.py b/scripts/bootstrap/platforms/macos.py new file mode 100644 index 000000000..bd6d64c95 --- /dev/null +++ b/scripts/bootstrap/platforms/macos.py @@ -0,0 +1,107 @@ +import shutil +import subprocess +from pathlib import Path +from typing import Dict, Final, Mapping, Optional, Sequence + +from bootstrap.platforms.bundling import Bundling +from bootstrap.platforms.unix import unix_interpreter + +DARWIN: Final[str] = "Darwin" +HOMEBREW: Final[str] = "brew" +HOMEBREW_SITE: Final[str] = "https://brew.sh" +PORTAUDIO: Final[str] = "portaudio" +ARCHFLAGS: Final[str] = "ARCHFLAGS" +NO_BUNDLE: Final[str] = ( + "ERROR: a standalone bundle is built on Linux and Windows.\n" + "On macOS, SampleToNES runs from source:\n" + "\n" + " make system-deps\n" + " make setup\n" + " make run\n" + "\n" + "See docs/guide/installation.md for the full steps." +) +CPU_BACKEND: Final[str] = "NVIDIA CUDA is available on Linux and Windows; on macOS, keeping the CPU (NumPy) backend." + + +class MacOS: + """macOS: PortAudio through Homebrew, and the application run from source. + + Homebrew's PortAudio carries the machine's own architecture while a python.org interpreter + compiles for both, so the setup and a build pin ``ARCHFLAGS`` to the native one. The CPU backend + is the one it runs, as NVIDIA ships CUDA for Linux and Windows. + """ + + @property + def name(self) -> str: + return DARWIN + + @property + def cpu_backend_reason(self) -> Optional[str]: + return CPU_BACKEND + + @staticmethod + def homebrew_prefix(package: str) -> str: + """Where Homebrew installed ``package``, or empty where Homebrew or the package is absent.""" + if shutil.which(HOMEBREW) is None: + return "" + + completed = subprocess.run( + [HOMEBREW, "--prefix", package], + capture_output=True, + text=True, + check=False, + ) + if completed.returncode != 0: + return "" + + return completed.stdout.strip() + + @staticmethod + def architecture_flag(machine: str) -> str: + """The flag compiling an extension for the machine's own architecture alone.""" + return f"-arch {machine}" + + def interpreter(self, environment: Path) -> Path: + return unix_interpreter(environment) + + def bundling(self) -> Bundling: + raise SystemExit(NO_BUNDLE) + + def missing_package_manager(self) -> Optional[str]: + if shutil.which(HOMEBREW) is not None: + return None + + return ( + "ERROR: Homebrew is required to install the macOS system dependencies.\n" + f"Install it from {HOMEBREW_SITE}, then run this script again." + ) + + def system_packages(self) -> Sequence[Sequence[str]]: + return ((HOMEBREW, "install", PORTAUDIO),) + + def setup_variables(self, base: Mapping[str, str], *, machine: str) -> Dict[str, str]: + return {**base, ARCHFLAGS: self.architecture_flag(machine)} + + def build_flags(self, *, machine: str) -> Sequence[str]: + """The flags compiling audio playback against Homebrew's PortAudio on the native architecture. + + Raises: + SystemExit: If Homebrew reports no PortAudio prefix. + """ + prefix = self.homebrew_prefix(PORTAUDIO) + if not prefix: + raise SystemExit( + "ERROR: Homebrew is required to locate the PortAudio headers and library.\n" + "Run 'make system-deps' first." + ) + + return ( + f"CFLAGS=-I{prefix}/include", + f"LDFLAGS=-L{prefix}/lib", + f"{ARCHFLAGS}={self.architecture_flag(machine)}", + ) + + def nvidia_smi_locations(self, environment: Mapping[str, str]) -> Sequence[Path]: + del environment + return () diff --git a/scripts/bootstrap/platforms/protocol.py b/scripts/bootstrap/platforms/protocol.py new file mode 100644 index 000000000..e5bfdf9c0 --- /dev/null +++ b/scripts/bootstrap/platforms/protocol.py @@ -0,0 +1,73 @@ +from pathlib import Path +from typing import Dict, Mapping, Optional, Protocol, Sequence + +from bootstrap.platforms.bundling import Bundling + + +class Platform(Protocol): + """What building and running SampleToNES has to know about the system it happens on.""" + + @property + def name(self) -> str: + """The name ``platform.system()`` reports for the system.""" + + @property + def cpu_backend_reason(self) -> Optional[str]: + """Why the system runs the CPU backend whatever its hardware, or ``None`` where CUDA can run.""" + + def interpreter(self, environment: Path) -> Path: + """The interpreter a virtual environment at ``environment`` runs. + + Args: + environment: The virtual environment's directory. + + Returns: + Path: The interpreter. + """ + + def bundling(self) -> Bundling: + """What building a standalone bundle takes on the system. + + Returns: + Bundling: The icon, the launcher's name and the advice a build gives. + + Raises: + SystemExit: If the system builds no bundle, naming how SampleToNES runs there. + """ + + def missing_package_manager(self) -> Optional[str]: + """What stands in the way of installing system packages, or ``None`` where nothing does.""" + + def system_packages(self) -> Sequence[Sequence[str]]: + """The commands that install the system packages the application needs, in order.""" + + def setup_variables(self, base: Mapping[str, str], *, machine: str) -> Dict[str, str]: + """The variables the development setup runs under: ``base`` and what the system adds to it. + + Args: + base: The caller's variables. + machine: The processor architecture ``platform.machine()`` reports. + + Returns: + Dict[str, str]: The variables. + """ + + def build_flags(self, *, machine: str) -> Sequence[str]: + """The ``KEY=VALUE`` lines a build exports so audio playback compiles against PortAudio. + + Args: + machine: The processor architecture ``platform.machine()`` reports. + + Returns: + Sequence[str]: The lines, empty where the build needs none. + """ + + def nvidia_smi_locations(self, environment: Mapping[str, str]) -> Sequence[Path]: + """Where the NVIDIA driver installs ``nvidia-smi`` outside ``PATH``. + + Args: + environment: The variables the locations are read from. + + Returns: + Sequence[Path]: The candidates, in the order they are tried. + """ diff --git a/scripts/bootstrap/platforms/unix.py b/scripts/bootstrap/platforms/unix.py new file mode 100644 index 000000000..b6e97c559 --- /dev/null +++ b/scripts/bootstrap/platforms/unix.py @@ -0,0 +1,9 @@ +from pathlib import Path +from typing import Final, Tuple + +UNIX_INTERPRETER: Final[Tuple[str, str]] = ("bin", "python") + + +def unix_interpreter(environment: Path) -> Path: + """The interpreter a virtual environment at ``environment`` runs on Linux and macOS.""" + return environment.joinpath(*UNIX_INTERPRETER) diff --git a/scripts/bootstrap/platforms/windows.py b/scripts/bootstrap/platforms/windows.py new file mode 100644 index 000000000..9f67f0fba --- /dev/null +++ b/scripts/bootstrap/platforms/windows.py @@ -0,0 +1,65 @@ +from pathlib import Path +from typing import Dict, Final, Mapping, Optional, Sequence, Tuple + +from bootstrap.platforms.bundling import Bundling + +WINDOWS: Final[str] = "Windows" +WINDOWS_INTERPRETER: Final[Tuple[str, str]] = ("Scripts", "python.exe") +ICO_ICON: Final[str] = "src/sampletones_assets/icons/sampletones.ico" +EXECUTABLE_SUFFIX: Final[str] = ".exe" +NVIDIA_SMI_LOCATIONS: Final[Tuple[Tuple[str, str], ...]] = ( + ("SystemRoot", "System32/nvidia-smi.exe"), + ("ProgramFiles", "NVIDIA Corporation/NVSMI/nvidia-smi.exe"), +) +BUNDLING: Final[Bundling] = Bundling( + icon=ICO_ICON, + executable_suffix=EXECUTABLE_SUFFIX, + pyaudio_advice="Run install.bat from the project root to reinstall the dependencies.", + tkinter_advice="Install Python from python.org or the Microsoft Store, which both include Tk, then build again.", + tkinter_warning=( + "This bundle opens no file dialogs. Install Python from python.org or the Microsoft Store to include Tk." + ), +) + + +class Windows: + """Windows: the official Python installer carries what the application needs. + + The NVIDIA driver installs ``nvidia-smi.exe`` into fixed system locations that ``PATH`` may + leave out, so CUDA detection tries those too. + """ + + @property + def name(self) -> str: + return WINDOWS + + @property + def cpu_backend_reason(self) -> Optional[str]: + return None + + def interpreter(self, environment: Path) -> Path: + return environment.joinpath(*WINDOWS_INTERPRETER) + + def bundling(self) -> Bundling: + return BUNDLING + + def missing_package_manager(self) -> Optional[str]: + return None + + def system_packages(self) -> Sequence[Sequence[str]]: + return () + + def setup_variables(self, base: Mapping[str, str], *, machine: str) -> Dict[str, str]: + del machine + return dict(base) + + def build_flags(self, *, machine: str) -> Sequence[str]: + del machine + return () + + def nvidia_smi_locations(self, environment: Mapping[str, str]) -> Sequence[Path]: + return tuple( + Path(environment[variable]) / relative + for variable, relative in NVIDIA_SMI_LOCATIONS + if environment.get(variable) + ) diff --git a/scripts/bootstrap/preflight.py b/scripts/bootstrap/preflight.py new file mode 100644 index 000000000..1757f71f7 --- /dev/null +++ b/scripts/bootstrap/preflight.py @@ -0,0 +1,84 @@ +from pathlib import Path +from typing import Final, Mapping + +from bootstrap.platforms.bundling import Bundling +from bootstrap.processes import Runner + +PYAUDIO: Final[str] = "pyaudio" +TKINTER: Final[str] = "tkinter" + + +def can_import( + python: Path, + module: str, + *, + runner: Runner, + cwd: Path, + environment: Mapping[str, str], +) -> bool: + """Whether the interpreter at ``python`` imports ``module``. + + Args: + python: The interpreter. + module: The module's name. + runner: What runs the probe. + cwd: The directory the probe runs in. + environment: The variables the probe sees. + + Returns: + bool: Whether the import succeeds. + """ + status = runner( + (str(python), "-c", f"import {module}"), + cwd=cwd, + environment=environment, + quiet=True, + ) + return status == 0 + + +def check_build_interpreter( + python: Path, + bundling: Bundling, + *, + release: bool, + runner: Runner, + cwd: Path, + environment: Mapping[str, str], +) -> None: + """Holds the build interpreter to what a bundle has to carry. + + Audio playback is required of every bundle. Tk is required of a release bundle, so the + shipped executable opens file dialogs on its own; a development bundle built without it is + warned about what it leans on instead. + + Args: + python: The build environment's interpreter. + bundling: What building a bundle takes on the system, whose advice a refusal gives. + release: Whether the bundle is a release. + runner: What runs the probes. + cwd: The directory the probes run in. + environment: The variables the probes see. + + Raises: + SystemExit: If the interpreter cannot play audio, or a release lacks Tk. + """ + print("Checking the build environment...") + if not can_import(python, PYAUDIO, runner=runner, cwd=cwd, environment=environment): + raise SystemExit( + "ERROR: the build interpreter cannot import pyaudio, so the bundle would carry no audio playback.\n" + f"{bundling.pyaudio_advice}" + ) + + print(f"{PYAUDIO}: available") + if can_import(python, TKINTER, runner=runner, cwd=cwd, environment=environment): + print(f"{TKINTER}: available") + return + + if release: + raise SystemExit( + "ERROR: the build interpreter cannot import tkinter, so a release bundle would depend on the " + f"machine running it for file dialogs.\n{bundling.tkinter_advice}" + ) + + print(f"WARNING: the build interpreter cannot import tkinter. {bundling.tkinter_warning}") diff --git a/scripts/bootstrap/processes.py b/scripts/bootstrap/processes.py new file mode 100644 index 000000000..6d40f468a --- /dev/null +++ b/scripts/bootstrap/processes.py @@ -0,0 +1,74 @@ +import shlex +import subprocess +import sys +from pathlib import Path +from typing import Mapping, Protocol, Sequence + + +class Runner(Protocol): + """Runs a command to completion and answers with its exit status.""" + + def __call__( + self, + command: Sequence[str], + *, + cwd: Path, + environment: Mapping[str, str], + quiet: bool, + ) -> int: ... + + +def run( + command: Sequence[str], + *, + cwd: Path, + environment: Mapping[str, str], + quiet: bool, +) -> int: + """Runs a command in ``cwd`` under ``environment`` and answers with its exit status. + + A quiet run keeps the command's output to itself, which is what a probe asks for. Whatever + the script printed reaches the terminal before the command's own output, so a log read + through a pipe keeps the announcements ahead of what they announce. + + Args: + command: The program and its arguments. + cwd: The directory the command runs in. + environment: The variables the command sees. + quiet: Whether the command's output is captured instead of shown. + + Returns: + int: The command's exit status. + """ + sys.stdout.flush() + completed = subprocess.run( + list(command), + cwd=str(cwd), + env=dict(environment), + check=False, + capture_output=quiet, + ) + return completed.returncode + + +def expect_success( + runner: Runner, + command: Sequence[str], + *, + cwd: Path, + environment: Mapping[str, str], +) -> None: + """Runs a command the operation depends on. + + Args: + runner: What runs the command. + command: The program and its arguments. + cwd: The directory the command runs in. + environment: The variables the command sees. + + Raises: + SystemExit: If the command exits with a status other than zero. + """ + status = runner(command, cwd=cwd, environment=environment, quiet=False) + if status != 0: + raise SystemExit(f"ERROR: {shlex.join(command)} exited with status {status}") diff --git a/scripts/bootstrap/project.py b/scripts/bootstrap/project.py new file mode 100644 index 000000000..2428c7cbc --- /dev/null +++ b/scripts/bootstrap/project.py @@ -0,0 +1,94 @@ +import tomllib +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Dict, Final, Tuple + +from bootstrap.layout import PROJECT_FILE, SOURCE_DIRECTORY + +BUILD_EXTRA: Final[str] = "build" +GPU_EXTRA: Final[str] = "gpu" +GPU_CUDA11_EXTRA: Final[str] = "gpu-cuda11" +DEVELOPMENT_GROUP: Final[str] = "dev" +NAMED_EXTRAS: Final[Tuple[str, ...]] = (BUILD_EXTRA, GPU_EXTRA, GPU_CUDA11_EXTRA) +NAMED_GROUPS: Final[Tuple[str, ...]] = (DEVELOPMENT_GROUP,) +ENTRY_SEPARATOR: Final[str] = ":" +MODULE_SEPARATOR: Final[str] = "." +MODULE_SUFFIX: Final[str] = ".py" +TAG_PREFIX: Final[str] = "v" + + +@dataclass(frozen=True) +class Project: + """What ``pyproject.toml`` states that a build, a setup and a release gate read. + + Attributes: + name: The distribution's name, which the command and the bundle carry too. + version: The version a release is tagged with. + entry_module: The module the command runs, read from ``[project.scripts]``. + packages: The import packages the wheel carries. + extras: The optional-dependency extras. + groups: The dependency groups. + """ + + name: str + version: str + entry_module: str + packages: Tuple[str, ...] + extras: Tuple[str, ...] + groups: Tuple[str, ...] + + @property + def entry_script(self) -> str: + """The entry module as a path from the repository root, which PyInstaller starts from.""" + return f"{SOURCE_DIRECTORY}/{self.entry_module.replace(MODULE_SEPARATOR, '/')}{MODULE_SUFFIX}" + + @property + def tag(self) -> str: + """The tag a release of this version is published under, such as ``v0.3.0``.""" + return f"{TAG_PREFIX}{self.version}" + + +def parse_project(document: Dict[str, Any]) -> Project: + """The project a parsed ``pyproject.toml`` states, held to the names the scripts rely on. + + Args: + document: The file as ``tomllib`` reads it. + + Returns: + Project: The project. + + Raises: + SystemExit: If the file lacks the command, an extra or a group the scripts name. + """ + table = document["project"] + name: str = table["name"] + scripts: Dict[str, str] = table.get("scripts", {}) + if name not in scripts: + raise SystemExit(f"ERROR: {PROJECT_FILE} names no '{name}' command under [project.scripts].") + + project = Project( + name=name, + version=table["version"], + entry_module=scripts[name].split(ENTRY_SEPARATOR, 1)[0], + packages=tuple( + Path(package).name for package in document["tool"]["hatch"]["build"]["targets"]["wheel"]["packages"] + ), + extras=tuple(table.get("optional-dependencies", {})), + groups=tuple(document.get("dependency-groups", {})), + ) + missing = [extra for extra in NAMED_EXTRAS if extra not in project.extras] + missing.extend(group for group in NAMED_GROUPS if group not in project.groups) + if missing: + raise SystemExit(f"ERROR: {PROJECT_FILE} lacks what the scripts install: {', '.join(missing)}.") + + return project + + +def read_project(root: Path) -> Project: + """The project the repository at ``root`` states. + + Raises: + SystemExit: If the file lacks the command, an extra or a group the scripts name. + """ + with (root / PROJECT_FILE).open("rb") as handle: + return parse_project(tomllib.load(handle)) diff --git a/scripts/bootstrap/venv_build.py b/scripts/bootstrap/venv_build.py new file mode 100644 index 000000000..7aab0b423 --- /dev/null +++ b/scripts/bootstrap/venv_build.py @@ -0,0 +1,86 @@ +import sys +from pathlib import Path +from typing import Dict, Final, Mapping, Sequence + +from bootstrap.layout import BUILD_ENVIRONMENT +from bootstrap.platforms.protocol import Platform +from bootstrap.processes import Runner, expect_success + +PIP_REQUIRE_VIRTUALENV: Final[str] = "PIP_REQUIRE_VIRTUALENV" + + +def ensure_build_venv( + root: Path, + platform: Platform, + *, + runner: Runner, + environment: Mapping[str, str], +) -> Path: + """The interpreter of the virtual environment a bundle is built in, created under ``root`` where it is missing. + + Every package a build installs lands here, so the interpreter running the script stays as + it was found. An environment is present once its interpreter is; a directory an interrupted + creation left behind is cleared and created again. + + Args: + root: The repository. + platform: The system, which places the interpreter inside the environment. + runner: What runs the command creating the environment. + environment: The variables the command sees. + + Returns: + Path: The environment's interpreter. + """ + directory = root / BUILD_ENVIRONMENT + python = platform.interpreter(directory) + if python.is_file(): + print("Virtual environment already exists.") + return python + + print("Creating virtual environment...") + expect_success( + runner, + (sys.executable, "-m", "venv", "--clear", str(directory)), + cwd=root, + environment=environment, + ) + print("Virtual environment created.") + return python + + +def install( + root: Path, + python: Path, + *, + extras: Sequence[str], + runner: Runner, + environment: Mapping[str, str], +) -> None: + """Installs the package with ``extras`` into the environment ``python`` runs. + + Pip is told to refuse any interpreter outside a virtual environment, so an install reaches + the build environment alone. + + Args: + root: The repository, which is the package installed. + python: The build environment's interpreter. + extras: The optional-dependency extras installed with the package. + runner: What runs the commands. + environment: The variables the commands see. + """ + guarded: Dict[str, str] = {**environment, PIP_REQUIRE_VIRTUALENV: "1"} + print("Installing dependencies...") + expect_success( + runner, + (str(python), "-m", "pip", "install", "--upgrade", "pip"), + cwd=root, + environment=guarded, + ) + print(f"Installing with extras: {','.join(extras)}") + expect_success( + runner, + (str(python), "-m", "pip", "install", f".[{','.join(extras)}]"), + cwd=root, + environment=guarded, + ) + print("sampletones Python package installed successfully.") diff --git a/scripts/build_environment.py b/scripts/build_environment.py new file mode 100644 index 000000000..e749fd293 --- /dev/null +++ b/scripts/build_environment.py @@ -0,0 +1,21 @@ +import argparse +import platform as running +import sys +from typing import Sequence + +from bootstrap.platforms.factory import current_platform + + +def main(argv: Sequence[str]) -> int: + """Prints the variables a build exports so audio playback compiles, one ``KEY=VALUE`` per line.""" + parser = argparse.ArgumentParser(description="Print the build environment audio playback compiles under.") + parser.parse_args(list(argv)) + + for line in current_platform().build_flags(machine=running.machine()): + print(line) + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) diff --git a/scripts/bundle.py b/scripts/bundle.py new file mode 100644 index 000000000..88b2a94a0 --- /dev/null +++ b/scripts/bundle.py @@ -0,0 +1,212 @@ +import argparse +import os +import shutil +import sys +from dataclasses import dataclass +from pathlib import Path +from typing import Final, List, Mapping, Sequence, Tuple + +from bootstrap.files import remove_path +from bootstrap.interpreter import running_version +from bootstrap.layout import BUILD_TOOLS, DISTRIBUTION, NOTICES, RELEASE_HOOK, repository_root +from bootstrap.platforms.bundling import Bundling +from bootstrap.platforms.factory import current_platform +from bootstrap.platforms.protocol import Platform +from bootstrap.preflight import check_build_interpreter +from bootstrap.processes import Runner, expect_success, run +from bootstrap.project import BUILD_EXTRA, GPU_EXTRA, Project, read_project +from bootstrap.venv_build import ensure_build_venv, install + +SELF_CHECK: Final[str] = "self-check" + + +@dataclass(frozen=True) +class BundleOptions: + """How a bundle is built. + + Attributes: + release: Whether the bundle is a release: a directory beside its launcher, carrying the + release deployment configuration and the notices. + gpu: Whether the bundle carries GPU support. + """ + + release: bool + gpu: bool + + +def extras(options: BundleOptions) -> Tuple[str, ...]: + """The optional-dependency extras a bundle is built with.""" + if options.gpu: + return (BUILD_EXTRA, GPU_EXTRA) + + return (BUILD_EXTRA,) + + +def pyinstaller_command( + python: Path, + bundling: Bundling, + project: Project, + options: BundleOptions, +) -> List[str]: + """The PyInstaller invocation that writes the bundle. + + Every package the wheel carries brings its data files along at its own package path, so the + frozen application reads them through ``importlib.resources`` the way an installed one does. + + Args: + python: The build environment's interpreter. + bundling: What building a bundle takes on the system. + project: The project the bundle is built from. + options: How the bundle is built. + + Returns: + List[str]: The command, run from the repository root. + """ + command = [ + str(python), + "-m", + "PyInstaller", + "--name", + project.name, + "--onedir" if options.release else "--onefile", + "--noconfirm", + "--distpath", + DISTRIBUTION, + "--icon", + bundling.icon, + ] + for package in project.packages: + command.extend(("--collect-data", package)) + + command.extend(("--copy-metadata", project.name)) + for module in BUILD_TOOLS: + command.extend(("--exclude-module", module)) + + if options.release: + command.extend(("--runtime-hook", RELEASE_HOOK)) + + command.append(project.entry_script) + return command + + +def remove_previous(distribution: Path, bundling: Bundling, name: str) -> None: + """Removes what an earlier build left under ``distribution``, a release's directory or a single file.""" + for previous in ( + bundling.launcher(distribution, name=name, release=True).parent, + bundling.launcher(distribution, name=name, release=False), + ): + if remove_path(previous): + print(f"Removed the previous artifact: {previous}") + + +def copy_notices(root: Path, bundle: Path) -> None: + """Places the license and notice files beside a release bundle's launcher.""" + for notice in NOTICES: + shutil.copyfile(root / notice, bundle / notice) + + print(f"Bundled notices: {', '.join(NOTICES)}") + + +def build_bundle( + root: Path, + platform: Platform, + options: BundleOptions, + *, + runner: Runner, + environment: Mapping[str, str], +) -> Path: + """Builds the standalone bundle in a build environment of its own and verifies it starts. + + Args: + root: The repository. + platform: The system the bundle is built on and for. + options: How the bundle is built. + runner: What runs the commands. + environment: The variables the commands see. + + Returns: + Path: The launcher the bundle offers. + + Raises: + SystemExit: If the system builds no bundle, a step fails, or PyInstaller produced no launcher. + """ + bundling = platform.bundling() + project = read_project(root) + if options.release: + print("Release build: onedir bundle, injecting release deployment configuration") + + python = ensure_build_venv( + root, + platform, + runner=runner, + environment=environment, + ) + install( + root, + python, + extras=extras(options), + runner=runner, + environment=environment, + ) + check_build_interpreter( + python, + bundling, + release=options.release, + runner=runner, + cwd=root, + environment=environment, + ) + distribution = root / DISTRIBUTION + remove_previous(distribution, bundling, project.name) + print("Building executable...") + expect_success( + runner, + pyinstaller_command(python, bundling, project, options), + cwd=root, + environment=environment, + ) + launcher = bundling.launcher(distribution, name=project.name, release=options.release) + if not launcher.is_file(): + raise SystemExit(f"Build failed: PyInstaller produced no executable at {launcher}.") + + print("Verifying the bundle...") + status = runner((str(launcher), SELF_CHECK), cwd=root, environment=environment, quiet=False) + if status != 0: + raise SystemExit(f"Build failed: {launcher} did not pass its self-check.") + + if options.release: + copy_notices(root, launcher.parent) + + print(f"Build complete: {launcher}") + return launcher + + +def main(argv: Sequence[str]) -> int: + """Builds the standalone bundle, as a development build or a release.""" + parser = argparse.ArgumentParser(description="Build the standalone SampleToNES bundle.") + parser.add_argument( + "--release", + action="store_true", + help="build a release: a directory bundle with the release deployment configuration and the notices", + ) + parser.add_argument( + "--gpu", + action="store_true", + help="build with GPU support", + ) + arguments = parser.parse_args(list(argv)) + options = BundleOptions(release=arguments.release, gpu=arguments.gpu) + + print(f"Detected Python version: {running_version()}") + build_bundle( + repository_root(), + current_platform(), + options, + runner=run, + environment=os.environ, + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) diff --git a/scripts/calibration.py b/scripts/calibration.py deleted file mode 100644 index 4bf8d57d0..000000000 --- a/scripts/calibration.py +++ /dev/null @@ -1,109 +0,0 @@ -import argparse -from datetime import UTC, datetime -from pathlib import Path -from typing import Final - -from sampletones_core.calibration.config.corpus import CorpusConfig -from sampletones_core.calibration.corpus.synthesis import build_corpus -from sampletones_core.calibration.corpus.writer import write_corpus -from sampletones_core.calibration.referee.factory import build_referees -from sampletones_core.calibration.report import write_csv, write_markdown -from sampletones_core.calibration.runner import build_variants, evaluate_variants -from sampletones_core.configs import Config -from sampletones_core.constants.enums import ( - DEFAULT_CHANNELS, - ChannelName, - SpectrumMethod, -) -from sampletones_shared.logger import logger -from sampletones_shared.paths.user import USER_PATH_DOCUMENTS - -DEFAULT_OUTPUT_ROOT: Final[Path] = USER_PATH_DOCUMENTS / "calibration" -DEFAULT_METHODS: Final[str] = f"{SpectrumMethod.FFT.value},{SpectrumMethod.CQT.value}" -DEFAULT_PERCEPTUAL_EXPONENTS: Final[str] = "1.0" -DEFAULT_CHANNEL_NAMES: Final[str] = ",".join(generator.value for generator in DEFAULT_CHANNELS) - - -def main() -> None: - parser = argparse.ArgumentParser( - description="Reconstruct the calibration corpus and score it with referees.", - ) - parser.add_argument( - "--config", - type=Path, - default=None, - help="Base configuration path.", - ) - parser.add_argument( - "--output", - type=Path, - default=None, - help="Output directory of the run.", - ) - parser.add_argument( - "--methods", - type=str, - default=DEFAULT_METHODS, - help="Comma-separated spectrum methods.", - ) - parser.add_argument( - "--perceptual-exponents", - type=str, - default=DEFAULT_PERCEPTUAL_EXPONENTS, - help="Comma-separated values of metric.perceptual_exponent.", - ) - parser.add_argument( - "--temporal-weights", - type=str, - default="", - help="Comma-separated values of weights.temporal_loss_weight; empty keeps the base blend.", - ) - parser.add_argument( - "--channels", - type=str, - default=DEFAULT_CHANNEL_NAMES, - help="Comma-separated channels every variant reconstructs with.", - ) - arguments = parser.parse_args() - - channels = [ChannelName(name.strip()) for name in arguments.channels.split(",") if name.strip()] - if not channels: - parser.error("--channels requires at least one channel name") - - base = Config.load(arguments.config) if arguments.config else Config.default() - base = base.model_copy( - update={ - "generation": base.generation.model_copy( - update={"channels": channels}, - ) - }, - ) - output = arguments.output or DEFAULT_OUTPUT_ROOT / datetime.now(UTC).strftime("run-%Y%m%d-%H%M%S") - output.mkdir(parents=True, exist_ok=True) - - methods = [SpectrumMethod(name.strip()) for name in arguments.methods.split(",") if name.strip()] - if not methods: - parser.error("--methods requires at least one spectrum method") - - exponents = [float(value) for value in arguments.perceptual_exponents.split(",") if value.strip()] - temporal_weights = [float(value) for value in arguments.temporal_weights.split(",") if value.strip()] - - sample_rate = base.library.sample_rate - items = build_corpus(sample_rate, config=CorpusConfig.load()) - item_paths = write_corpus(items, output / "corpus", sample_rate) - referees = build_referees(sample_rate) - variants = build_variants(base, methods, exponents, temporal_weights) - - channel_names = ", ".join(channel.value for channel in channels) - logger.info( - f"Evaluating {len(variants)} variants x {len(items)} items x {len(referees)} referees on {channel_names}" - ) - rows = evaluate_variants(variants, items, item_paths, referees) - - write_csv(rows, output / "report.csv") - write_markdown(rows, output / "report.md") - logger.info(f"Report written to {output}") - - -if __name__ == "__main__": - main() diff --git a/scripts/checks/import_boundary.py b/scripts/checks/import_boundary.py deleted file mode 100755 index c906623b0..000000000 --- a/scripts/checks/import_boundary.py +++ /dev/null @@ -1,86 +0,0 @@ -#!/usr/bin/env python3 - -""" -Enforces the import boundaries the source tree is layered by. - -A layer graph names a tree of modules, the units it divides into, and the units each one may -import; every other unit is out of reach, so an edge across the graph is declared before it is -taken. The packages under `src/` are one such graph — `sampletones_core` sits below -`sampletones_player`, which is what keeps the reconstruction engine clear of the console player — -and the player's own subpackages are another, where `driver/assembler/` reaches the cc65 toolchain -and stays outside the wheel, so no shipped module imports it. - -Boundary rules state a contract the other way round, by the import prefixes a directory stays clear -of, and carry the contract modules exempt from those prefixes: a layer may consume another layer's -data contract (e.g. a service's result types) while its implementation modules stay out of reach. -`sampletones_application` is layered that way. - -Token rules additionally forbid a regex within a file glob, enforcing contracts a prefix cannot -express — e.g. that panels never compose a column suffix (`SUF_PANEL_*`) or parent into another -panel's container. - -`sampletones_config/boundaries/` declares what the boundaries are and -`sampletones_shared/meta/import_boundary/` holds how they are read and reported; this script runs -them over a source tree and prints what they find. - -Usage: - python scripts/checks/import_boundary.py [files...] # check specific files - python scripts/checks/import_boundary.py --all # run all rules against the source tree -""" - -import argparse -import sys -from pathlib import Path -from typing import List, Sequence - -from sampletones_shared.meta.import_boundary.check import check_boundaries -from sampletones_shared.meta.import_boundary.configs.rules import ImportBoundaryRules -from sampletones_shared.paths.source import SOURCE_ROOT - - -def main(argv: Sequence[str]) -> int: - """Report every import and token the layer boundaries forbid.""" - parser = argparse.ArgumentParser( - description="Check the import boundaries the source tree is layered by.", - ) - parser.add_argument( - "files", - nargs="*", - type=Path, - help="modules to check", - ) - parser.add_argument( - "--all", - action="store_true", - help=f"check every module under {SOURCE_ROOT.name}/ instead of named files", - ) - parser.add_argument( - "--source", - type=Path, - default=SOURCE_ROOT, - help="source root the rule roots are named within", - ) - arguments = parser.parse_args(list(argv)) - - files: List[Path] = arguments.files - selection = None if arguments.all else {path.resolve() for path in files} - boundaries = ImportBoundaryRules.load() - violations = check_boundaries( - arguments.source, - boundaries.boundary_rules(), - boundaries.tokens, - selection, - ) - if not violations: - return 0 - - print("Layer boundary violation(s) found:", file=sys.stderr) - for kind, location in violations: - print(f" [forbidden: {kind}] {location}", file=sys.stderr) - - print(f"\nFound {len(violations)} violation(s) in total.", file=sys.stderr) - return 1 - - -if __name__ == "__main__": - raise SystemExit(main(sys.argv[1:])) diff --git a/scripts/ci/checks/bundle.py b/scripts/ci/checks/bundle.py deleted file mode 100644 index acef01234..000000000 --- a/scripts/ci/checks/bundle.py +++ /dev/null @@ -1,83 +0,0 @@ -import argparse -import platform -import subprocess -import sys -from pathlib import Path -from typing import Final, List, Sequence - -WINDOWS: Final[str] = "Windows" -WINDOWS_LAUNCHER: Final[str] = "sampletones.exe" -POSIX_LAUNCHER: Final[str] = "sampletones" -VERSION_FLAG: Final[str] = "--version" - -REQUIRED_NOTICES: Final[Sequence[str]] = ( - "LICENSE", - "THIRD-PARTY-NOTICES.md", - "THIRD-PARTY-LICENSES.txt", -) - -INTERNAL_DIRECTORY: Final[str] = "_internal" -BUILD_TOOLS: Final[Sequence[str]] = ("PIL",) - - -def launcher_path(bundle: Path, *, system: str) -> Path: - """The executable a built bundle offers on the platform it was built for.""" - name = WINDOWS_LAUNCHER if system == WINDOWS else POSIX_LAUNCHER - return bundle / name - - -def missing_notices(bundle: Path) -> List[str]: - """The license and notice files a release bundle must ship that are absent from it.""" - return [name for name in REQUIRED_NOTICES if not (bundle / name).is_file()] - - -def carried_build_tools(bundle: Path) -> List[str]: - """The build-time packages found in a bundle, which the notices place on the build machine. - - A bundle carries the application and its runtime dependencies. Tooling that only draws the - assets belongs to the machine that builds it, so finding it here means the notices describe - a different set of components than the bundle ships. - """ - directories = (bundle, bundle / INTERNAL_DIRECTORY) - return [name for name in BUILD_TOOLS if any((directory / name).is_dir() for directory in directories)] - - -def main(argv: Sequence[str]) -> int: - """Confirm a built bundle ships its notices, holds to them, and that its launcher starts.""" - parser = argparse.ArgumentParser( - description="Verify a built bundle before it is archived.", - ) - parser.add_argument( - "bundle", - type=Path, - help="the built bundle directory, such as bin/sampletones", - ) - arguments = parser.parse_args(list(argv)) - - bundle: Path = arguments.bundle - absent = missing_notices(bundle) - if absent: - print(f"::error::Bundle {bundle} is missing {', '.join(absent)}") - return 1 - - carried = carried_build_tools(bundle) - if carried: - print(f"::error::Bundle {bundle} carries build-time tooling its notices leave out: {', '.join(carried)}") - return 1 - - launcher = launcher_path(bundle, system=platform.system()) - if not launcher.is_file(): - print(f"::error::Bundle {bundle} offers no launcher at {launcher}") - return 1 - - completed = subprocess.run([str(launcher), VERSION_FLAG], check=False) - if completed.returncode != 0: - print(f"::error::Launcher {launcher} exited with status {completed.returncode}") - return completed.returncode - - print(f"Bundle {bundle} ships its notices and its launcher starts") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main(sys.argv[1:])) diff --git a/scripts/ci/checks/version_tag.py b/scripts/ci/checks/version_tag.py deleted file mode 100644 index 47d4438c7..000000000 --- a/scripts/ci/checks/version_tag.py +++ /dev/null @@ -1,46 +0,0 @@ -import argparse -import sys -from typing import Final, Sequence - -TAG_PREFIX: Final[str] = "v" - - -def version_from_tag(tag: str) -> str: - """The project version a release tag names, read from the tag with its ``v`` prefix dropped.""" - return tag.removeprefix(TAG_PREFIX) - - -def tag_names_version(*, tag: str, project_version: str) -> bool: - """Whether a release tag names the version recorded in the project metadata.""" - return version_from_tag(tag) == project_version - - -def main(argv: Sequence[str]) -> int: - """Confirm a release tag and the project metadata agree on the version being released.""" - parser = argparse.ArgumentParser( - description="Compare a release tag against the project version.", - ) - parser.add_argument( - "--tag", - required=True, - help="the release tag being built, such as v0.3.0", - ) - parser.add_argument( - "--project-version", - required=True, - help="the version recorded in pyproject.toml", - ) - arguments = parser.parse_args(list(argv)) - - tag: str = arguments.tag - project_version: str = arguments.project_version - if not tag_names_version(tag=tag, project_version=project_version): - print(f"::error::Tag {tag} names a version other than the pyproject version {project_version}") - return 1 - - print(f"Version {project_version} matches tag {tag}") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main(sys.argv[1:])) diff --git a/scripts/ci/zip_bundle.py b/scripts/ci/zip_bundle.py deleted file mode 100644 index f8a523bef..000000000 --- a/scripts/ci/zip_bundle.py +++ /dev/null @@ -1,56 +0,0 @@ -import argparse -import sys -import zipfile -from pathlib import Path -from typing import Final, List, Sequence - -ARCHIVE_COMPRESSION: Final[int] = zipfile.ZIP_DEFLATED - - -def bundle_entries(source: Path) -> List[Path]: - """Every file and directory inside a built bundle, ordered so repeated runs archive alike.""" - return sorted(source.rglob("*")) - - -def archive_name(path: Path, *, source: Path, root: str) -> str: - """The location a bundle path takes inside the archive, gathered under a single root directory.""" - return f"{root}/{path.relative_to(source).as_posix()}" - - -def write_archive(source: Path, archive: Path, *, root: str) -> List[Path]: - """Archive a built bundle directory, placing every entry under ``root``. - - Each entry is read where it lies, which keeps the archive available while a virus scanner or a - process that ran the executable holds a handle inside the directory, and carries over the - permission bits that let the launcher run once the archive is extracted. - """ - archive.parent.mkdir(parents=True, exist_ok=True) - entries = bundle_entries(source) - with zipfile.ZipFile(archive, "w", ARCHIVE_COMPRESSION) as bundle: - for path in entries: - bundle.write(path, archive_name(path, source=source, root=root)) - - return entries - - -def main(argv: Sequence[str]) -> int: - """Archive a built bundle directory under a versioned root directory.""" - parser = argparse.ArgumentParser(description="Archive a built bundle directory under a versioned root.") - parser.add_argument("source", type=Path, help="the built bundle directory, such as bin/sampletones") - parser.add_argument("archive", type=Path, help="the path of the zip file to write") - parser.add_argument("--root", required=True, help="the directory name every archived entry sits under") - arguments = parser.parse_args(list(argv)) - - source: Path = arguments.source - archive: Path = arguments.archive - if not source.is_dir(): - print(f"::error::Bundle directory {source} is missing") - return 1 - - entries = write_archive(source, archive, root=arguments.root) - print(f"Archived {len(entries)} entries from {source} into {archive}") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main(sys.argv[1:])) diff --git a/scripts/clean.py b/scripts/clean.py new file mode 100644 index 000000000..08a690ee6 --- /dev/null +++ b/scripts/clean.py @@ -0,0 +1,56 @@ +import argparse +import sys +from pathlib import Path +from typing import Sequence + +from bootstrap.files import remove_path +from bootstrap.layout import ( + CACHE_DIRECTORIES, + CACHE_DIRECTORY_SUFFIXES, + CACHE_FILE_SUFFIXES, + CLEAN_ARTIFACTS, + CLEAN_PATTERNS, + ENVIRONMENTS, + repository_root, +) + + +def remove_artifacts(root: Path) -> None: + """Removes the build outputs and the coverage reports under ``root``.""" + for name in CLEAN_ARTIFACTS: + remove_path(root / name) + + for pattern in CLEAN_PATTERNS: + for path in root.glob(pattern): + remove_path(path) + + +def remove_caches(root: Path) -> None: + """Removes the bytecode caches and packaging leftovers under ``root``, the environments left alone.""" + for directory, subdirectories, files in root.walk(): + subdirectories[:] = [name for name in subdirectories if name not in ENVIRONMENTS] + for name in list(subdirectories): + if name in CACHE_DIRECTORIES or name.endswith(CACHE_DIRECTORY_SUFFIXES): + remove_path(directory / name) + subdirectories.remove(name) + + for name in files: + if name.endswith(CACHE_FILE_SUFFIXES): + remove_path(directory / name) + + +def main(argv: Sequence[str]) -> int: + """Removes the build artifacts and cache files from the repository.""" + parser = argparse.ArgumentParser(description="Remove build artifacts and cache files.") + parser.parse_args(list(argv)) + + root = repository_root() + print("Removing build artifacts and temporary files...") + remove_artifacts(root) + remove_caches(root) + print("Cleaned build artifacts and temporary files.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) diff --git a/scripts/detect_cuda.py b/scripts/detect_cuda.py deleted file mode 100644 index 9aa905228..000000000 --- a/scripts/detect_cuda.py +++ /dev/null @@ -1,235 +0,0 @@ -import argparse -import os -import platform -import re -import shutil -import subprocess -import sys -from dataclasses import dataclass -from pathlib import Path -from typing import Final, List, Optional, Sequence, Tuple - -NVIDIA_SMI: Final[str] = "nvidia-smi" - -DARWIN: Final[str] = "Darwin" -WINDOWS: Final[str] = "Windows" - -EXTRA_GPU: Final[str] = "gpu" -EXTRA_GPU_CUDA11: Final[str] = "gpu-cuda11" - -CUDA12_MAJOR: Final[int] = 12 -CUDA11_MAJOR: Final[int] = 11 - -CUDA_VERSION_PATTERN: Final[re.Pattern[str]] = re.compile(r"CUDA Version\s*:?\s*(\d+)\.(\d+)") - -WINDOWS_NVIDIA_SMI_SUBPATHS: Final[Sequence[Tuple[str, str]]] = ( - ("SystemRoot", "System32/nvidia-smi.exe"), - ("ProgramFiles", "NVIDIA Corporation/NVSMI/nvidia-smi.exe"), -) - - -@dataclass(frozen=True, kw_only=True) -class CudaDetection: - """The NVIDIA driver capability observed on the host and the CuPy extra it maps to.""" - - system: str - nvidia_smi: Optional[Path] - cuda_version: Optional[Tuple[int, int]] - extra: Optional[str] - reason: str - - -def _windows_nvidia_smi_candidates() -> List[Path]: - """Return the fixed Windows locations where the driver installs ``nvidia-smi.exe``.""" - candidates: List[Path] = [] - for variable, relative in WINDOWS_NVIDIA_SMI_SUBPATHS: - root = os.environ.get(variable) - if root: - candidates.append(Path(root) / relative) - - return candidates - - -def find_nvidia_smi(*, system: str) -> Optional[Path]: - """Return the path to ``nvidia-smi`` when an NVIDIA driver is installed. - - ``nvidia-smi`` ships with the driver and lands on ``PATH`` on Linux and Windows. On Windows - it may also sit in a fixed system location off ``PATH``, so probe those as a fallback. - """ - located = shutil.which(NVIDIA_SMI) - if located is not None: - return Path(located) - - if system == WINDOWS: - for candidate in _windows_nvidia_smi_candidates(): - if candidate.exists(): - return candidate - return None - - -def _run_nvidia_smi( - nvidia_smi: Path, - arguments: Sequence[str], -) -> Optional[str]: - """Return the combined output of an ``nvidia-smi`` invocation that succeeds.""" - try: - completed = subprocess.run( - [str(nvidia_smi), *arguments], - capture_output=True, - text=True, - check=False, - ) - except (OSError, subprocess.SubprocessError): - return None - - if completed.returncode != 0: - return None - - return completed.stdout + completed.stderr - - -def query_driver_cuda_version(nvidia_smi: Path) -> Optional[Tuple[int, int]]: - """Return the maximum CUDA version the driver supports, as ``(major, minor)``. - - ``nvidia-smi`` reports the driver's maximum supported CUDA version, which governs CuPy wheel - selection: CuPy wheels bind to the driver rather than to a system CUDA Toolkit. The default - table carries the value, and ``-q`` provides a structured fallback for formats that omit it. - """ - for arguments in ((), ("-q",)): - output = _run_nvidia_smi(nvidia_smi, arguments) - if output is None: - continue - match = CUDA_VERSION_PATTERN.search(output) - if match is not None: - return int(match.group(1)), int(match.group(2)) - - return None - - -def select_extra(cuda_version: Optional[Tuple[int, int]]) -> Optional[str]: - """Return the optional-dependency extra matching a driver's CUDA version. - - CUDA 12 and newer drivers map to the ``gpu`` extra (``cupy-cuda12x``): a CUDA 12 wheel runs on - every 12.x driver through Enhanced Compatibility and on newer drivers through backward - compatibility. CUDA 11 drivers map to the legacy ``gpu-cuda11`` extra. Older drivers map to - ``None``, keeping the CPU backend. - """ - if cuda_version is None: - return None - - major = cuda_version[0] - if major >= CUDA12_MAJOR: - return EXTRA_GPU - - if major == CUDA11_MAJOR: - return EXTRA_GPU_CUDA11 - - return None - - -def _describe( - *, - cuda_version: Optional[Tuple[int, int]], - extra: Optional[str], -) -> str: - """Return a one-line summary of a driver-present detection outcome.""" - if cuda_version is None: - return "NVIDIA driver detected, though its CUDA version was unreadable; keeping the CPU (NumPy) backend." - major, minor = cuda_version - if extra is None: - return ( - f"Detected CUDA {major}.{minor}, which predates the supported CuPy builds; " - "keeping the CPU (NumPy) backend." - ) - return f"Detected an NVIDIA driver supporting CUDA {major}.{minor}; selecting the '{extra}' extra." - - -def detect(*, system: str) -> CudaDetection: - """Return the CUDA capability of the host and the CuPy extra that matches it. - - macOS keeps the CPU backend, since NVIDIA CUDA is available only on Linux and Windows. On - those systems the driver's ``nvidia-smi`` supplies the CUDA version that drives selection. - """ - if system == DARWIN: - return CudaDetection( - system=system, - nvidia_smi=None, - cuda_version=None, - extra=None, - reason="NVIDIA CUDA is available on Linux and Windows; on macOS, keeping the CPU (NumPy) backend.", - ) - - nvidia_smi = find_nvidia_smi(system=system) - if nvidia_smi is None: - return CudaDetection( - system=system, - nvidia_smi=None, - cuda_version=None, - extra=None, - reason="No NVIDIA driver detected (nvidia-smi is absent); keeping the CPU (NumPy) backend.", - ) - - cuda_version = query_driver_cuda_version(nvidia_smi) - extra = select_extra(cuda_version) - return CudaDetection( - system=system, - nvidia_smi=nvidia_smi, - cuda_version=cuda_version, - extra=extra, - reason=_describe(cuda_version=cuda_version, extra=extra), - ) - - -def _format_version(cuda_version: Optional[Tuple[int, int]]) -> str: - """Return a human-readable CUDA version, or ``unknown`` when it was unreadable.""" - if cuda_version is None: - return "unknown" - - major, minor = cuda_version - return f"{major}.{minor}" - - -def _print_report(detection: CudaDetection) -> None: - """Print the full detection outcome to stdout for a human reader.""" - nvidia_smi = str(detection.nvidia_smi) if detection.nvidia_smi is not None else "not found" - extra = detection.extra if detection.extra is not None else "none (CPU)" - lines = [ - f"System: {detection.system}", - f"nvidia-smi: {nvidia_smi}", - f"Driver CUDA: {_format_version(detection.cuda_version)}", - f"Selected extra: {extra}", - f"Summary: {detection.reason}", - ] - print("\n".join(lines)) - - -def main(argv: Sequence[str]) -> int: - """Report the detected CUDA capability, or print the matching extra for install scripts. - - Under ``--extra`` the chosen extra name is written to stdout (empty when the CPU backend is - kept) so a Makefile can capture it, and the summary is written to stderr. Without it, a full - report is written to stdout. - """ - parser = argparse.ArgumentParser( - description="Select the CuPy extra matching the local NVIDIA driver.", - ) - parser.add_argument( - "--extra", - action="store_true", - help="print the chosen optional-dependency extra to stdout (empty when the CPU backend is kept)", - ) - arguments = parser.parse_args(list(argv)) - detection = detect(system=platform.system()) - - if arguments.extra: - print(detection.reason, file=sys.stderr) - if detection.extra is not None: - print(detection.extra) - else: - _print_report(detection) - - return 0 - - -if __name__ == "__main__": - raise SystemExit(main(sys.argv[1:])) diff --git a/scripts/formatting.py b/scripts/formatting.py new file mode 100644 index 000000000..82407162c --- /dev/null +++ b/scripts/formatting.py @@ -0,0 +1,61 @@ +import argparse +import os +import sys +from pathlib import Path +from typing import Final, Mapping, Sequence, Tuple + +from bootstrap.layout import SOURCE_DIRECTORY, repository_root +from bootstrap.processes import Runner, expect_success, run + +FORMATTED_TREES: Final[Tuple[str, ...]] = (SOURCE_DIRECTORY, "tests", "scripts") +ISORT: Final[Tuple[str, ...]] = ("uv", "run", "python", "-m", "isort") +BLACK: Final[Tuple[str, ...]] = ("uv", "run", "python", "-m", "black") + + +def format_code( + root: Path, + paths: Sequence[str], + *, + runner: Runner, + environment: Mapping[str, str], +) -> None: + """Sorts the imports and formats the code under ``paths``, isort first so black settles the result. + + Args: + root: The repository, which the formatters run in. + paths: The files or directories formatted. + runner: What runs the formatters. + environment: The variables the formatters see. + + Raises: + SystemExit: If a formatter fails, before the next one runs. + """ + print("Formatting imports with isort...") + expect_success(runner, (*ISORT, *paths), cwd=root, environment=environment) + print("Formatting code with black...") + expect_success(runner, (*BLACK, *paths), cwd=root, environment=environment) + + +def formatted_paths(named: Sequence[str]) -> Tuple[str, ...]: + """The paths a run formats: the ones named, or the source, test and script trees.""" + return tuple(named) or FORMATTED_TREES + + +def main(argv: Sequence[str]) -> int: + """Formats the source, test and script trees, or the paths named.""" + parser = argparse.ArgumentParser(description="Format the SampleToNES code with isort and black.") + parser.add_argument("paths", nargs="*", help="files or directories to format in place of the whole trees") + arguments = parser.parse_args(list(argv)) + + format_code( + repository_root(), + formatted_paths(tuple(arguments.paths)), + runner=run, + environment=os.environ, + ) + print("Code formatting complete.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) diff --git a/scripts/hooks.py b/scripts/hooks.py new file mode 100644 index 000000000..96fb13c01 --- /dev/null +++ b/scripts/hooks.py @@ -0,0 +1,48 @@ +import argparse +import os +import sys +from pathlib import Path +from typing import Final, Mapping, Sequence, Tuple + +from bootstrap.layout import repository_root +from bootstrap.processes import Runner, expect_success, run + +INSTALL_HOOKS: Final[Tuple[str, ...]] = ( + "uv", + "run", + "pre-commit", + "install", + "--hook-type", + "pre-commit", + "--hook-type", + "pre-push", +) + + +def install_hooks( + root: Path, + *, + runner: Runner, + environment: Mapping[str, str], +) -> None: + """Installs the git hooks pre-commit runs at commit and at push. + + Raises: + SystemExit: If pre-commit fails to install them. + """ + print("Installing pre-commit hooks...") + expect_success(runner, INSTALL_HOOKS, cwd=root, environment=environment) + print("Pre-commit hooks installed.") + + +def main(argv: Sequence[str]) -> int: + """Installs the git hooks pre-commit runs at commit and at push.""" + parser = argparse.ArgumentParser(description="Install the pre-commit hooks.") + parser.parse_args(list(argv)) + + install_hooks(repository_root(), runner=run, environment=os.environ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) diff --git a/scripts/lint.py b/scripts/lint.py new file mode 100644 index 000000000..55061cffd --- /dev/null +++ b/scripts/lint.py @@ -0,0 +1,86 @@ +import argparse +import os +import sys +from pathlib import Path +from typing import Final, Mapping, Sequence, Set, Tuple + +from bootstrap.layout import SOURCE_DIRECTORY, repository_root +from bootstrap.passes import Pass, run_passes +from bootstrap.processes import Runner, run + +MYPY: Final[str] = "mypy" +PYLINT: Final[str] = "pylint" +LINTED_TREES: Final[Tuple[str, ...]] = (SOURCE_DIRECTORY, "scripts") + + +def linters(paths: Sequence[str]) -> Tuple[Pass, ...]: + """Mypy and pylint, in that order, over ``paths``. + + Without paths, mypy reads the files ``pyproject.toml`` configures and pylint sweeps the + source and script trees. + """ + return ( + Pass( + MYPY, + "Running type checking with mypy...", + ("uv", "run", "python", "-m", MYPY, *paths), + ), + Pass( + PYLINT, + "Running linting with pylint...", + ("uv", "run", "python", "-m", PYLINT, *(paths or LINTED_TREES)), + ), + ) + + +def lint_code( + root: Path, + passes: Sequence[Pass], + *, + runner: Runner, + environment: Mapping[str, str], +) -> int: + """Runs every linter asked for and names the ones that failed. + + Args: + root: The repository, which the linters run in. + passes: The linters, in order. + runner: What runs them. + environment: The variables they see. + + Returns: + int: 0 where every linter passed, 1 otherwise. + """ + failed = run_passes(passes, root=root, runner=runner, environment=environment) + if failed: + print(f"Linting failed: {', '.join(failed)}.") + return 1 + + print("All linting checks passed.") + return 0 + + +def chosen_linters(paths: Sequence[str], *, mypy: bool, pylint: bool) -> Tuple[Pass, ...]: + """The linters a run asks for: the ones flagged, or both where none is.""" + chosen: Set[str] = {name for name, wanted in ((MYPY, mypy), (PYLINT, pylint)) if wanted} + return tuple(linter for linter in linters(paths) if not chosen or linter.name in chosen) + + +def main(argv: Sequence[str]) -> int: + """Type checks and lints the code, and reports which linter failed.""" + parser = argparse.ArgumentParser(description="Type check and lint the SampleToNES code.") + parser.add_argument("--mypy", action="store_true", help="run mypy alone") + parser.add_argument("--pylint", action="store_true", help="run pylint alone") + parser.add_argument("paths", nargs="*", help="files or directories to lint in place of the whole trees") + arguments = parser.parse_args(list(argv)) + + return lint_code( + repository_root(), + chosen_linters(tuple(arguments.paths), mypy=arguments.mypy, pylint=arguments.pylint), + runner=run, + environment=os.environ, + ) + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) diff --git a/scripts/linux/build/build.sh b/scripts/linux/build/build.sh deleted file mode 100755 index 198b0acf7..000000000 --- a/scripts/linux/build/build.sh +++ /dev/null @@ -1,71 +0,0 @@ -#!/usr/bin/env bash - -set -e - -SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) -. "$SCRIPT_DIR/../lib/root.sh" - -PROJECT_DIR=$(CDPATH= cd -- "$SCRIPT_DIR/../../.." && pwd) -VENV_DIR="$PROJECT_DIR/.venv-build" -VENV_PY="$VENV_DIR/bin/python" - -RELEASE=0 -BUNDLE_ARGS=(--onefile) -RELEASE_HOOK_ARGS=() -for arg in "$@"; do - if [[ "$arg" == "--release" ]]; then - echo "Release build: onedir bundle, injecting release deployment configuration" - RELEASE=1 - BUNDLE_ARGS=(--onedir) - RELEASE_HOOK_ARGS=(--runtime-hook scripts/release_env_hook.py) - fi -done - -if [[ "${RELEASE}" == "1" ]]; then - EXECUTABLE="bin/sampletones/sampletones" -else - EXECUTABLE="bin/sampletones" -fi - -bash "$SCRIPT_DIR/preflight.sh" "$@" -bash "$SCRIPT_DIR/icons.sh" - -if [[ -e "${PROJECT_DIR}/bin/sampletones" ]]; then - echo "Removing the previous artifact: ./bin/sampletones" - rm -rf "${PROJECT_DIR}/bin/sampletones" -fi - -echo "Building executable..." -"$VENV_PY" -m PyInstaller --name sampletones \ - "${BUNDLE_ARGS[@]}" \ - --noconfirm \ - --distpath ./bin \ - --icon "src/sampletones_assets/icons/sampletones.png" \ - --add-data "src/sampletones_assets/icons:assets/icons" \ - --add-data "src/sampletones_assets/fonts:assets/fonts" \ - --add-data "src/sampletones_config:config" \ - --add-data "src/sampletones_player/driver/binary:sampletones_player/driver/binary" \ - --copy-metadata sampletones \ - --exclude-module PIL \ - "${RELEASE_HOOK_ARGS[@]}" \ - "src/sampletones/__main__.py" - -if [[ ! -x "${EXECUTABLE}" ]]; then - echo "Build failed: PyInstaller produced no executable at ./${EXECUTABLE}." >&2 - exit 1 -fi - -echo "Verifying the bundle..." -if ! "${PROJECT_DIR}/${EXECUTABLE}" --self-check; then - echo "Build failed: ./${EXECUTABLE} did not pass its self-check." >&2 - exit 1 -fi - -if [[ "${RELEASE}" == "1" ]]; then - for notice in LICENSE THIRD-PARTY-NOTICES.md THIRD-PARTY-LICENSES.txt; do - cp "${notice}" "bin/sampletones/${notice}" - done - echo "Bundled notices: LICENSE, THIRD-PARTY-NOTICES.md, THIRD-PARTY-LICENSES.txt" -fi - -echo "Build complete: ./${EXECUTABLE}" diff --git a/scripts/linux/build/clean.sh b/scripts/linux/build/clean.sh deleted file mode 100755 index 932a5fe58..000000000 --- a/scripts/linux/build/clean.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "${BASH_SOURCE[0]}")/../lib/root.sh" - -echo "Removing build artifacts and temporary files..." -rm -rf bin/ build/ dist/ *.spec htmlcov/ .coverage -find . -type d -name "__pycache__" -exec rm -rf {} + 2>/dev/null || true -find . -type d -name "*.egg-info" -exec rm -rf {} + 2>/dev/null || true -find . -type f -name "*.pyc" -delete -echo "Cleaned build artifacts and temporary files." diff --git a/scripts/linux/build/dependencies.sh b/scripts/linux/build/dependencies.sh deleted file mode 100755 index 7b1758276..000000000 --- a/scripts/linux/build/dependencies.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env bash - -set -e - -PACKAGES=( - # PortAudio: audio playback through pyaudio - libportaudio2 - libasound-dev - libpulse-dev - portaudio19-dev - # Tk: file dialogs where kdialog and zenity are absent - python3-tk - tk-dev - tcl-dev - # OpenGL and X11: the window DearPyGui opens through GLFW - libgl1 - libegl1 - libx11-6 - libx11-xcb1 - libxcursor1 - libxi6 - libxinerama1 - libxrandr2 - libxrender1 - libxxf86vm1 -) - -echo "Installing system dependencies (requires sudo)" -sudo apt-get update -sudo apt-get install -y "${PACKAGES[@]}" -echo "System dependencies installed." diff --git a/scripts/linux/build/icons.sh b/scripts/linux/build/icons.sh deleted file mode 100644 index df7bf35b8..000000000 --- a/scripts/linux/build/icons.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env bash - -set -e - -SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) -. "$SCRIPT_DIR/../lib/root.sh" - -PROJECT_DIR=$(CDPATH= cd -- "$SCRIPT_DIR/../../.." && pwd) -VENV_PY="$PROJECT_DIR/.venv-build/bin/python" - -echo "Generating the icon suite..." -"$VENV_PY" scripts/assets/icons.py diff --git a/scripts/linux/build/preflight.sh b/scripts/linux/build/preflight.sh deleted file mode 100755 index 5008bde05..000000000 --- a/scripts/linux/build/preflight.sh +++ /dev/null @@ -1,53 +0,0 @@ -#!/usr/bin/env bash - -set -e - -SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) -. "$SCRIPT_DIR/../lib/root.sh" - -PROJECT_DIR=$(CDPATH= cd -- "$SCRIPT_DIR/../../.." && pwd) -VENV_DIR="$PROJECT_DIR/.venv-build" -VENV_PY="$VENV_DIR/bin/python" - -RELEASE=0 -for arg in "$@"; do - if [[ "$arg" == "--release" ]]; then - RELEASE=1 - fi -done - -can_import() { - "$VENV_PY" -c "import $1" >/dev/null 2>&1 -} - -echo "Checking the build environment..." - -if [[ ! -x "$VENV_PY" ]]; then - echo "ERROR: build interpreter not found at ${VENV_PY}." >&2 - echo "Run './install.sh' from the project root to create the build environment." >&2 - exit 1 -fi - -if ! can_import pyaudio; then - echo "ERROR: the build interpreter cannot import pyaudio, so the bundle would carry no audio playback." >&2 - echo "Run 'make system-deps' to install the PortAudio packages, then './install.sh' to reinstall dependencies." >&2 - exit 1 -fi - -echo "pyaudio: available" - -if can_import tkinter; then - echo "tkinter: available" - exit 0 -fi - -if [[ "${RELEASE}" == "1" ]]; then - echo "ERROR: the build interpreter cannot import tkinter, so a release bundle would depend on the" >&2 - echo "target machine providing zenity or kdialog for file dialogs." >&2 - echo "Run 'make system-deps' to install python3-tk, then build again." >&2 - exit 1 -fi - -echo "WARNING: the build interpreter cannot import tkinter." -echo "This bundle opens file dialogs through zenity or kdialog, which the machine running it has to provide." -echo "Run 'make system-deps' to install python3-tk and carry Tk as a self-contained fallback." diff --git a/scripts/linux/build/python.sh b/scripts/linux/build/python.sh deleted file mode 100755 index fec272e0a..000000000 --- a/scripts/linux/build/python.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env bash - -set -e - -PYVER=$(python3 --version 2>&1 | awk '{print $2}') -echo "Detected Python version: $PYVER" -MAJOR=$(echo $PYVER | cut -d. -f1) -MINOR=$(echo $PYVER | cut -d. -f2) -if [ "$MAJOR" -lt 3 ] || { [ "$MAJOR" -eq 3 ] && [ "$MINOR" -lt 12 ]; }; then - echo - echo "ERROR: Python 3.12 or newer is required." - echo "Please install Python 3.12+ from https://www.python.org/downloads/" - exit 1 -fi diff --git a/scripts/linux/build/sampletones.sh b/scripts/linux/build/sampletones.sh deleted file mode 100755 index d5accc984..000000000 --- a/scripts/linux/build/sampletones.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/env bash - -set -e - -SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) -. "$SCRIPT_DIR/../lib/root.sh" - -PROJECT_DIR=$(CDPATH= cd -- "$SCRIPT_DIR/../../.." && pwd) -VENV_DIR="$PROJECT_DIR/.venv-build" -VENV_PY="$VENV_DIR/bin/python" - -EXTRAS=("build") -for arg in "$@"; do - case $arg in - --gpu) - EXTRAS+=("gpu") - ;; - esac -done - -echo "Installing dependencies..." -"$VENV_PY" -m pip install --upgrade pip - -EXTRAS_STR=$(IFS=,; echo "${EXTRAS[*]}") -echo "Installing with extras: $EXTRAS_STR" -"$VENV_PY" -m pip install ".[$EXTRAS_STR]" --group assets - -echo "sampletones Python package installed successfully." diff --git a/scripts/linux/build/venv.sh b/scripts/linux/build/venv.sh deleted file mode 100755 index 29b7ad326..000000000 --- a/scripts/linux/build/venv.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env bash - -set -e - -source "$(dirname "${BASH_SOURCE[0]}")/../lib/root.sh" - -if [[ -d ".venv-build" ]]; then - echo "Virtual environment already exists." -else - echo "Creating virtual environment..." - python3 -m venv .venv-build - echo "Virtual environment created." -fi - -return 0 diff --git a/scripts/linux/dev/format.sh b/scripts/linux/dev/format.sh deleted file mode 100755 index 16bcc7d03..000000000 --- a/scripts/linux/dev/format.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env bash - -set -e - -echo "Formatting imports with isort..." -uv run python -m isort src/ tests/ - -echo "Formatting code with black..." -uv run python -m black src/ tests/ - -echo "Code formatting complete." diff --git a/scripts/linux/dev/lint.sh b/scripts/linux/dev/lint.sh deleted file mode 100755 index 6ac482b7a..000000000 --- a/scripts/linux/dev/lint.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env bash - -set +e - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" - -bash "${SCRIPT_DIR}/mypy.sh" -MYPY_EXIT=$? - -bash "${SCRIPT_DIR}/pylint.sh" -PYLINT_EXIT=$? - -if [[ $MYPY_EXIT -ne 0 ]] || [[ $PYLINT_EXIT -ne 0 ]]; then - echo "Linting failed." - exit 1 -fi - -echo "All linting checks passed." -exit 0 diff --git a/scripts/linux/dev/mypy.sh b/scripts/linux/dev/mypy.sh deleted file mode 100755 index 409357be7..000000000 --- a/scripts/linux/dev/mypy.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env bash - -set +e - -echo "Running type checking with mypy..." -if [ $# -eq 0 ]; then - uv run python -m mypy -else - uv run python -m mypy "$@" -fi -exit $? diff --git a/scripts/linux/dev/pre_commit.sh b/scripts/linux/dev/pre_commit.sh deleted file mode 100755 index bb30b8619..000000000 --- a/scripts/linux/dev/pre_commit.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env bash - -set -e - -echo "Installing pre-commit hooks..." -uv run pre-commit install -uv run pre-commit install --hook-type pre-commit --hook-type pre-push -echo "Pre-commit hooks installed successfully." diff --git a/scripts/linux/dev/pylint.sh b/scripts/linux/dev/pylint.sh deleted file mode 100755 index 6a4ccad62..000000000 --- a/scripts/linux/dev/pylint.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env bash - -set +e - -echo "Running linting with pylint..." -if [ $# -eq 0 ]; then - uv run python -m pylint src/sampletones -else - uv run python -m pylint "$@" -fi -exit $? diff --git a/scripts/linux/dev/tests.sh b/scripts/linux/dev/tests.sh deleted file mode 100755 index 46d4f865a..000000000 --- a/scripts/linux/dev/tests.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env bash - -set +e - -echo "Running doctests..." -uv run python -m pytest src/ --doctest-modules --no-cov -DOCTEST_EXIT=$? - -echo "Running benchmarks..." -uv run python -m pytest tests/benchmarks --no-cov -BENCHMARK_EXIT=$? - -echo "Running pytest with coverage..." -uv run python -m pytest -n 6 --cov --ignore=tests/benchmarks -PYTEST_EXIT=$? - -if [[ $DOCTEST_EXIT -ne 0 ]] || [[ $PYTEST_EXIT -ne 0 ]] || [[ $BENCHMARK_EXIT -ne 0 ]]; then - echo "Tests failed." - exit 1 -fi - -echo "All tests passed." -exit 0 diff --git a/scripts/linux/lib/root.sh b/scripts/linux/lib/root.sh deleted file mode 100755 index a5859c2eb..000000000 --- a/scripts/linux/lib/root.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env bash - -_PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)" - -if [[ ! -d "${_PROJECT_ROOT}/src/sampletones" ]]; then - echo "ERROR: SampleToNES project root not found (expected ${_PROJECT_ROOT}/src/sampletones)." >&2 - exit 1 -fi - -cd "${_PROJECT_ROOT}" diff --git a/scripts/macos/build/build_env.sh b/scripts/macos/build/build_env.sh deleted file mode 100755 index f6d195827..000000000 --- a/scripts/macos/build/build_env.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env bash - -set -e - -if ! command -v brew >/dev/null 2>&1; then - echo "ERROR: Homebrew is required to locate the PortAudio headers and library." >&2 - echo "Run scripts/macos/build/dependencies.sh first." >&2 - exit 1 -fi - -PORTAUDIO_PREFIX=$(brew --prefix portaudio) - -echo "CFLAGS=-I${PORTAUDIO_PREFIX}/include" -echo "LDFLAGS=-L${PORTAUDIO_PREFIX}/lib" -echo "ARCHFLAGS=-arch $(uname -m)" diff --git a/scripts/macos/build/dependencies.sh b/scripts/macos/build/dependencies.sh deleted file mode 100755 index ca5049082..000000000 --- a/scripts/macos/build/dependencies.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env bash - -set -e - -PACKAGES=( - portaudio -) - -if ! command -v brew >/dev/null 2>&1; then - echo "ERROR: Homebrew is required to install the macOS system dependencies." >&2 - echo "Install it from https://brew.sh, then run this script again." >&2 - exit 1 -fi - -echo "Installing system dependencies through Homebrew" -brew install "${PACKAGES[@]}" -echo "System dependencies installed." diff --git a/scripts/macos/build/no_bundle.sh b/scripts/macos/build/no_bundle.sh deleted file mode 100755 index b6cfb4ec7..000000000 --- a/scripts/macos/build/no_bundle.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env bash - -set -e - -OPERATION="${1:-this command}" - -echo "ERROR: ${OPERATION} supports Linux and Windows." >&2 -echo "On macOS, SampleToNES runs from source:" >&2 -echo >&2 -echo " make system-deps" >&2 -echo " make setup" >&2 -echo " make run" >&2 -echo >&2 -echo "See docs/guide/installation.md for the full steps." >&2 -exit 1 diff --git a/scripts/player.py b/scripts/player.py deleted file mode 100755 index d96e2847c..000000000 --- a/scripts/player.py +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env python3 - -import argparse -import sys -from pathlib import Path -from typing import Sequence - -from sampletones_player.driver.assembler.builder import build_driver -from sampletones_player.driver.assembler.layout import BINARY_DIRECTORY -from sampletones_player.specification.driver import DRIVER_CODE_NAME -from sampletones_shared.exceptions import DriverBuildError - - -def main(argv: Sequence[str]) -> int: - """Assembles the NES player driver and reports the layout the build produced.""" - - parser = argparse.ArgumentParser( - description="Assemble the NES player driver with cc65.", - ) - parser.add_argument( - "--directory", - type=Path, - default=BINARY_DIRECTORY, - help="directory receiving the assembled driver", - ) - arguments = parser.parse_args(list(argv)) - - try: - image = build_driver(arguments.directory) - except DriverBuildError as error: - print(error, file=sys.stderr) - return 1 - - addresses = image.addresses - print(f"{DRIVER_CODE_NAME} {len(image.code)} bytes, ${addresses.load:04X}-${addresses.song - 1:04X}") - print(f"init ${addresses.init:04X}") - print(f"play ${addresses.play:04X}") - print(f"song ${addresses.song:04X}") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main(sys.argv[1:])) diff --git a/scripts/run_tests.py b/scripts/run_tests.py new file mode 100644 index 000000000..f1d213741 --- /dev/null +++ b/scripts/run_tests.py @@ -0,0 +1,71 @@ +import argparse +import os +import sys +from typing import Dict, Final, Sequence, Tuple + +from bootstrap.layout import BENCHMARKS_DIRECTORY, SOURCE_DIRECTORY, repository_root +from bootstrap.passes import Pass, run_pass +from bootstrap.processes import run + +SUITE: Final[str] = "suite" +DOCTESTS: Final[str] = "doctests" +BENCHMARKS: Final[str] = "benchmarks" +DEFAULT_WORKERS: Final[str] = "6" +PYTEST: Final[Tuple[str, ...]] = ("uv", "run", "python", "-m", "pytest") + + +def planned_passes(workers: str) -> Dict[str, Pass]: + """The passes a test run is made of, by name: the covered suite, the doctests, the benchmarks. + + Each pass is a target and a hook of its own, so a failure names the pass it belongs to. The + covered suite runs across ``workers`` pytest workers. The benchmarks run serial, uncovered + and with their output shown, so a measured duration is the code's own cost and its reading + reaches the terminal. + + Args: + workers: The worker count for the covered suite, or ``auto`` for one per processor. + + Returns: + Dict[str, Pass]: Every pass, under its name. + """ + passes = ( + Pass( + SUITE, + "Running pytest with coverage...", + (*PYTEST, "-n", workers, "--cov", f"--ignore={BENCHMARKS_DIRECTORY}"), + ), + Pass( + DOCTESTS, + "Running doctests...", + (*PYTEST, SOURCE_DIRECTORY, "--doctest-modules", "--no-cov"), + ), + Pass( + BENCHMARKS, + "Running benchmarks...", + (*PYTEST, BENCHMARKS_DIRECTORY, "--no-cov", "-s"), + ), + ) + return {current.name: current for current in passes} + + +def main(argv: Sequence[str]) -> int: + """Runs one pass of the tests and exits with the status pytest gave it.""" + parser = argparse.ArgumentParser(description="Run one pass of the SampleToNES tests.") + parser.add_argument("name", choices=tuple(planned_passes(DEFAULT_WORKERS)), help="the pass to run") + parser.add_argument( + "--workers", + default=DEFAULT_WORKERS, + help="pytest workers for the covered suite: a count, or auto for one per processor", + ) + arguments = parser.parse_args(list(argv)) + + return run_pass( + planned_passes(arguments.workers)[arguments.name], + root=repository_root(), + runner=run, + environment=os.environ, + ) + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) diff --git a/scripts/release_env_hook.py b/scripts/runtime_hooks/release_environment.py similarity index 100% rename from scripts/release_env_hook.py rename to scripts/runtime_hooks/release_environment.py diff --git a/scripts/setup_environment.py b/scripts/setup_environment.py new file mode 100644 index 000000000..e1358e9ba --- /dev/null +++ b/scripts/setup_environment.py @@ -0,0 +1,126 @@ +import argparse +import os +import platform as running +import sys +from pathlib import Path +from typing import Final, List, Mapping, Optional, Sequence, Tuple + +from bootstrap.cuda import detect +from bootstrap.layout import repository_root +from bootstrap.platforms.factory import current_platform +from bootstrap.platforms.protocol import Platform +from bootstrap.processes import Runner, expect_success, run +from bootstrap.project import DEVELOPMENT_GROUP, GPU_CUDA11_EXTRA, GPU_EXTRA, read_project + +GPU_AUTO: Final[str] = "auto" +GPU_OFF: Final[str] = "0" +GPU_CHOICES: Final[Tuple[str, ...]] = ( + GPU_AUTO, + GPU_OFF, + GPU_EXTRA, + GPU_CUDA11_EXTRA, +) +DEFAULT_GPU: Final[str] = GPU_AUTO + + +def gpu_extra( + choice: str, + platform: Platform, + environment: Mapping[str, str], +) -> Optional[str]: + """The optional-dependency extra a GPU choice selects. + + Args: + choice: ``auto`` to read the NVIDIA driver, ``0`` for the CPU backend, or an extra's name. + platform: The system the setup runs on. + environment: The variables the driver's locations are read from. + + Returns: + Optional[str]: The extra, or ``None`` for the CPU backend. + """ + if choice == GPU_OFF: + return None + + if choice == GPU_AUTO: + detection = detect(platform, environment) + print(detection.reason, file=sys.stderr) + return detection.extra + + return choice + + +def setup_commands(extra: Optional[str]) -> List[List[str]]: + """The commands that create the development environment and install the global command. + + Args: + extra: The GPU extra installed with the package, or ``None`` for the CPU backend. + + Returns: + List[List[str]]: The commands, in order. + """ + synchronize = ["uv", "sync", "--group", DEVELOPMENT_GROUP] + package = "." + if extra is not None: + synchronize.extend(("--extra", extra)) + package = f".[{extra}]" + + return [ + synchronize, + ["uv", "tool", "install", "--force", package], + ] + + +def set_up_environment( + root: Path, + platform: Platform, + extra: Optional[str], + *, + machine: str, + runner: Runner, + environment: Mapping[str, str], +) -> None: + """Synchronizes the development environment and installs the global ``sampletones`` command. + + Args: + root: The repository. + platform: The system the setup runs on, which adds the variables the build needs. + extra: The GPU extra installed with the package, or ``None`` for the CPU backend. + machine: The processor architecture ``platform.machine()`` reports. + runner: What runs the commands. + environment: The caller's variables. + + Raises: + SystemExit: If a command fails. + """ + variables = platform.setup_variables(environment, machine=machine) + for command in setup_commands(extra): + expect_success(runner, command, cwd=root, environment=variables) + + +def main(argv: Sequence[str]) -> int: + """Creates the development environment and installs the global ``sampletones`` command.""" + parser = argparse.ArgumentParser(description="Set up the SampleToNES development environment.") + parser.add_argument( + "--gpu", + default=DEFAULT_GPU, + choices=GPU_CHOICES, + help="auto to match the NVIDIA driver, 0 for the CPU backend, or the GPU extra to install", + ) + arguments = parser.parse_args(list(argv)) + + root = repository_root() + read_project(root) + platform = current_platform() + set_up_environment( + root, + platform, + gpu_extra(arguments.gpu, platform, os.environ), + machine=running.machine(), + runner=run, + environment=os.environ, + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) diff --git a/scripts/system_dependencies.py b/scripts/system_dependencies.py new file mode 100644 index 000000000..a8071a5a8 --- /dev/null +++ b/scripts/system_dependencies.py @@ -0,0 +1,66 @@ +import argparse +import os +import sys +from pathlib import Path +from typing import Mapping, Sequence + +from bootstrap.layout import repository_root +from bootstrap.platforms.factory import current_platform +from bootstrap.platforms.protocol import Platform +from bootstrap.processes import Runner, expect_success, run + + +def install_system_packages( + root: Path, + platform: Platform, + *, + runner: Runner, + environment: Mapping[str, str], +) -> int: + """Installs the system packages building and running the application needs. + + Args: + root: The repository, which the commands run in. + platform: The system, which names its package manager and packages. + runner: What runs the commands. + environment: The variables the commands see. + + Returns: + int: The exit status: 1 where the package manager is missing, 0 otherwise. + + Raises: + SystemExit: If an install command fails. + """ + missing = platform.missing_package_manager() + if missing is not None: + print(missing, file=sys.stderr) + return 1 + + commands = platform.system_packages() + if not commands: + print(f"Nothing to install on {platform.name}: the Python installer carries what the application needs.") + return 0 + + print("Installing system dependencies...") + for command in commands: + expect_success(runner, command, cwd=root, environment=environment) + + print("System dependencies installed.") + return 0 + + +def main(argv: Sequence[str]) -> int: + """Installs the system packages building and running the application needs on this machine.""" + parser = argparse.ArgumentParser(description="Install the system packages SampleToNES needs.") + parser.parse_args(list(argv)) + + return install_system_packages( + repository_root(), + current_platform(), + runner=run, + environment=os.environ, + ) + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) diff --git a/scripts/verify_bundle.py b/scripts/verify_bundle.py new file mode 100644 index 000000000..daeb751d9 --- /dev/null +++ b/scripts/verify_bundle.py @@ -0,0 +1,97 @@ +import argparse +import os +import sys +from pathlib import Path +from typing import Final, List, Mapping, Sequence + +from bootstrap.layout import BUILD_TOOLS, DISTRIBUTION, NOTICES, repository_root +from bootstrap.platforms.factory import current_platform +from bootstrap.platforms.protocol import Platform +from bootstrap.processes import Runner, run +from bootstrap.project import read_project + +VERSION_FLAG: Final[str] = "--version" +INTERNAL_DIRECTORY: Final[str] = "_internal" + + +def missing_notices(bundle: Path) -> List[str]: + """The license and notice files a release bundle ships that are absent from it.""" + return [name for name in NOTICES if not (bundle / name).is_file()] + + +def carried_build_tools(bundle: Path) -> List[str]: + """The build-time packages found in a bundle, which the notices place on the build machine. + + A bundle carries the application and its runtime dependencies. Tooling that draws the assets + belongs to the machine that builds it, so finding it here means the notices describe a + different set of components than the bundle ships. + """ + directories = (bundle, bundle / INTERNAL_DIRECTORY) + return [name for name in BUILD_TOOLS if any((directory / name).is_dir() for directory in directories)] + + +def bundle_failures( + root: Path, + platform: Platform, + *, + runner: Runner, + environment: Mapping[str, str], +) -> List[str]: + """What keeps the release bundle under ``root`` from shipping, each as a line of its own. + + The bundle ships its notices, carries the application and its runtime dependencies alone, and + offers a launcher that starts. + + Args: + root: The repository the bundle was built in. + platform: The system the bundle was built for. + runner: What runs the launcher. + environment: The variables the launcher sees. + + Returns: + List[str]: The failures, empty for a bundle ready to archive. + + Raises: + SystemExit: If the system builds no bundle. + """ + name = read_project(root).name + launcher = platform.bundling().launcher(root / DISTRIBUTION, name=name, release=True) + bundle = launcher.parent + failures: List[str] = [] + absent = missing_notices(bundle) + if absent: + failures.append(f"Bundle {bundle} is missing {', '.join(absent)}") + + carried = carried_build_tools(bundle) + if carried: + failures.append(f"Bundle {bundle} carries build-time tooling its notices leave out: {', '.join(carried)}") + + if not launcher.is_file(): + failures.append(f"Bundle {bundle} offers no launcher at {launcher}") + return failures + + status = runner((str(launcher), VERSION_FLAG), cwd=root, environment=environment, quiet=False) + if status != 0: + failures.append(f"Launcher {launcher} exited with status {status}") + + return failures + + +def main(argv: Sequence[str]) -> int: + """Confirms the release bundle ships its notices, holds to them, and that its launcher starts.""" + parser = argparse.ArgumentParser(description="Verify the release bundle before it is archived.") + parser.parse_args(list(argv)) + + failures = bundle_failures(repository_root(), current_platform(), runner=run, environment=os.environ) + for failure in failures: + print(f"::error::{failure}") + + if failures: + return 1 + + print("The bundle ships its notices and its launcher starts") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) diff --git a/scripts/verify_version_tag.py b/scripts/verify_version_tag.py new file mode 100644 index 000000000..ac1599f25 --- /dev/null +++ b/scripts/verify_version_tag.py @@ -0,0 +1,48 @@ +import argparse +import sys +from typing import Optional, Sequence + +from bootstrap.layout import repository_root +from bootstrap.project import TAG_PREFIX, Project, read_project + + +def version_from_tag(tag: str) -> str: + """The project version a release tag names, read from the tag with its ``v`` prefix dropped.""" + return tag.removeprefix(TAG_PREFIX) + + +def tag_failure(tag: str, project: Project) -> Optional[str]: + """What keeps a release tag from publishing the project, or ``None`` for a tag naming its version. + + Args: + tag: The release tag being built, such as ``v0.3.0``. + project: The project the tag publishes. + + Returns: + Optional[str]: The failure as one line, or ``None``. + """ + if version_from_tag(tag) == project.version: + return None + + return f"Tag {tag} names a version other than the project version {project.version}" + + +def main(argv: Sequence[str]) -> int: + """Confirms a release tag names the version ``pyproject.toml`` records.""" + parser = argparse.ArgumentParser(description="Compare a release tag against the project version.") + parser.add_argument("--tag", required=True, help="the release tag being built, such as v0.3.0") + arguments = parser.parse_args(list(argv)) + + tag: str = arguments.tag + project = read_project(repository_root()) + failure = tag_failure(tag, project) + if failure is not None: + print(f"::error::{failure}") + return 1 + + print(f"Version {project.version} matches tag {tag}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) diff --git a/scripts/windows/build/build.bat b/scripts/windows/build/build.bat deleted file mode 100644 index f2567962c..000000000 --- a/scripts/windows/build/build.bat +++ /dev/null @@ -1,81 +0,0 @@ -@echo off -setlocal EnableExtensions - -set "SCRIPT_DIR=%~dp0" -call "%SCRIPT_DIR%\..\lib\root.bat" || exit /b 1 - -set "PROJECT_DIR=%SCRIPT_DIR%..\..\.." -set "VENV_DIR=%PROJECT_DIR%\.venv-build" -set "VENV_PY=%VENV_DIR%\Scripts\python.exe" - -set RELEASE=0 -set BUNDLE_MODE=--onefile -set RELEASE_HOOK= -:parse_args -if "%~1"=="" goto build -if "%~1"=="--release" ( - echo Release build: onedir bundle, injecting release deployment configuration - set RELEASE=1 - set BUNDLE_MODE=--onedir - set RELEASE_HOOK=--runtime-hook scripts\release_env_hook.py -) -shift -goto parse_args - -:build -if "%RELEASE%"=="1" ( - set EXECUTABLE=bin\sampletones\sampletones.exe -) else ( - set EXECUTABLE=bin\sampletones.exe -) - -call "%SCRIPT_DIR%preflight.bat" %* || exit /b 1 -call "%SCRIPT_DIR%icons.bat" || exit /b 1 - -if exist "bin\sampletones.exe" ( - echo Removing the previous artifact: bin\sampletones.exe - del /Q "bin\sampletones.exe" || exit /b 1 -) - -if exist "bin\sampletones\" ( - echo Removing the previous artifact: bin\sampletones - rmdir /S /Q "bin\sampletones" || exit /b 1 -) - -echo Building executable... -"%VENV_PY%" -m PyInstaller --name sampletones ^ - %BUNDLE_MODE% ^ - --noconfirm ^ - --distpath bin ^ - --icon "src\sampletones_assets\icons\sampletones.ico" ^ - --add-data "src\sampletones_assets\icons;assets\icons" ^ - --add-data "src\sampletones_assets\fonts;assets\fonts" ^ - --add-data "src\sampletones_config;config" ^ - --add-data "src\sampletones_player\driver\binary;sampletones_player\driver\binary" ^ - --copy-metadata sampletones ^ - --exclude-module PIL ^ - %RELEASE_HOOK% ^ - "src\sampletones\__main__.py" || exit /b - -if not exist "%EXECUTABLE%" ( - echo Build failed: PyInstaller produced no executable at %EXECUTABLE%.>&2 - exit /b 1 -) - -echo Verifying the bundle... -"%EXECUTABLE%" --self-check -if errorlevel 1 ( - echo Build failed: %EXECUTABLE% did not pass its self-check.>&2 - exit /b 1 -) - -if "%RELEASE%"=="1" ( - copy /Y LICENSE bin\sampletones\LICENSE >nul || exit /b 1 - copy /Y THIRD-PARTY-NOTICES.md bin\sampletones\THIRD-PARTY-NOTICES.md >nul || exit /b 1 - copy /Y THIRD-PARTY-LICENSES.txt bin\sampletones\THIRD-PARTY-LICENSES.txt >nul || exit /b 1 - echo Bundled notices: LICENSE, THIRD-PARTY-NOTICES.md, THIRD-PARTY-LICENSES.txt -) - -echo Build complete: .\%EXECUTABLE% - -exit /b 0 diff --git a/scripts/windows/build/clean.bat b/scripts/windows/build/clean.bat deleted file mode 100644 index a41b8bf8d..000000000 --- a/scripts/windows/build/clean.bat +++ /dev/null @@ -1,18 +0,0 @@ -@echo off -setlocal - -call "%~dp0..\lib\root.bat" || exit /b 1 - -echo Cleaning build artifacts... -if exist bin rmdir /s /q bin 2>nul -if exist build rmdir /s /q build 2>nul -if exist dist rmdir /s /q dist 2>nul -if exist htmlcov rmdir /s /q htmlcov 2>nul -if exist .coverage del /q .coverage 2>nul -if exist *.spec del /q *.spec 2>nul -for /d /r . %%d in (__pycache__) do @if exist "%%d" rmdir /s /q "%%d" 2>nul -for /d /r . %%d in (*.egg-info) do @if exist "%%d" rmdir /s /q "%%d" 2>nul -del /s /q *.pyc 2>nul - -echo Clean complete. -exit /b 0 diff --git a/scripts/windows/build/icons.bat b/scripts/windows/build/icons.bat deleted file mode 100644 index d4ee7f14a..000000000 --- a/scripts/windows/build/icons.bat +++ /dev/null @@ -1,14 +0,0 @@ -@echo off -setlocal EnableExtensions - -set "SCRIPT_DIR=%~dp0" -call "%SCRIPT_DIR%\..\lib\root.bat" || exit /b 1 - -set "PROJECT_DIR=%SCRIPT_DIR%..\..\.." -set "VENV_DIR=%PROJECT_DIR%\.venv-build" -set "VENV_PY=%VENV_DIR%\Scripts\python.exe" - -echo Generating the icon suite... -"%VENV_PY%" scripts\assets\icons.py || exit /b 1 - -exit /b 0 diff --git a/scripts/windows/build/preflight.bat b/scripts/windows/build/preflight.bat deleted file mode 100644 index 6cb6a7467..000000000 --- a/scripts/windows/build/preflight.bat +++ /dev/null @@ -1,51 +0,0 @@ -@echo off -setlocal EnableExtensions - -set "SCRIPT_DIR=%~dp0" -call "%SCRIPT_DIR%\..\lib\root.bat" || exit /b 1 - -set "PROJECT_DIR=%SCRIPT_DIR%..\..\.." -set "VENV_DIR=%PROJECT_DIR%\.venv-build" -set "VENV_PY=%VENV_DIR%\Scripts\python.exe" - -set RELEASE=0 -:parse_args -if "%~1"=="" goto check -if "%~1"=="--release" set RELEASE=1 -shift -goto parse_args - -:check -echo Checking the build environment... - -if not exist "%VENV_PY%" ( - echo ERROR: build interpreter not found at %VENV_PY%.>&2 - echo Run install.bat from the project root to create the build environment.>&2 - exit /b 1 -) - -"%VENV_PY%" -c "import pyaudio" >nul 2>&1 -if errorlevel 1 ( - echo ERROR: the build interpreter cannot import pyaudio, so the bundle would carry no audio playback.>&2 - echo Run install.bat from the project root to reinstall the dependencies.>&2 - exit /b 1 -) - -echo pyaudio: available - -"%VENV_PY%" -c "import tkinter" >nul 2>&1 -if errorlevel 1 goto missing_tkinter - -echo tkinter: available -exit /b 0 - -:missing_tkinter -if "%RELEASE%"=="1" ( - echo ERROR: the build interpreter cannot import tkinter, so a release bundle would open no file dialogs.>&2 - echo Install Python from python.org or the Microsoft Store, which both include Tk, then build again.>&2 - exit /b 1 -) - -echo WARNING: the build interpreter cannot import tkinter. -echo This bundle opens no file dialogs. Install Python from python.org or the Microsoft Store to include Tk. -exit /b 0 diff --git a/scripts/windows/build/python.bat b/scripts/windows/build/python.bat deleted file mode 100644 index 381804ed4..000000000 --- a/scripts/windows/build/python.bat +++ /dev/null @@ -1,22 +0,0 @@ -@echo off -setlocal - -for /f "tokens=2 delims= " %%v in ('python --version 2^>^&1') do set PYTHON_VERSION=%%v -echo Detected Python version: %PYTHON_VERSION% - -for /f "tokens=1,2 delims=." %%a in ("%PYTHON_VERSION%") do set PY_MAJOR=%%a& set PY_MINOR=%%b - -set /a VERSION_OK=0 -if %PY_MAJOR% GEQ 3 ( - if %PY_MINOR% GEQ 12 set VERSION_OK=1 -) - -if not %VERSION_OK%==1 ( - echo. - echo ERROR: Python 3.12 or newer is required. - echo Please install Python 3.12+ from https://www.python.org/downloads/ - pause - exit /b 1 -) - -exit /b 0 diff --git a/scripts/windows/build/sampletones.bat b/scripts/windows/build/sampletones.bat deleted file mode 100644 index 902ea3726..000000000 --- a/scripts/windows/build/sampletones.bat +++ /dev/null @@ -1,28 +0,0 @@ -@echo off -setlocal enabledelayedexpansion -setlocal EnableExtensions - -set "SCRIPT_DIR=%~dp0" -call "%SCRIPT_DIR%\..\lib\root.bat" || exit /b 1 - -set "PROJECT_DIR=%SCRIPT_DIR%..\..\.." -set "VENV_DIR=%PROJECT_DIR%\.venv-build" -set "VENV_PY=%VENV_DIR%\Scripts\python.exe" - -set "EXTRAS=build" - -:parse_args -if "%~1"=="" goto install -if "%~1"=="--gpu" set "EXTRAS=!EXTRAS!,gpu" -shift -goto parse_args - -:install -echo Installing dependencies... -"%VENV_PY%" -m pip install --upgrade pip - -echo Installing with extras: !EXTRAS! -"%VENV_PY%" -m pip install ".[!EXTRAS!]" --group assets || exit /b 1 - -echo sampletones Python package installed successfully. -exit /b 0 diff --git a/scripts/windows/build/venv.bat b/scripts/windows/build/venv.bat deleted file mode 100644 index 85f4d6307..000000000 --- a/scripts/windows/build/venv.bat +++ /dev/null @@ -1,14 +0,0 @@ -@echo off -setlocal - -call "%~dp0..\lib\root.bat" || exit /b 1 - -if exist ".venv-build" ( - echo Virtual environment already exists. -) else ( - echo Creating virtual environment... - python -m venv .venv-build || exit /b - echo Virtual environment created. -) - -exit /b 0 diff --git a/scripts/windows/dev/format.bat b/scripts/windows/dev/format.bat deleted file mode 100644 index cd1717a4c..000000000 --- a/scripts/windows/dev/format.bat +++ /dev/null @@ -1,11 +0,0 @@ -@echo off -setlocal - -echo Formatting imports with isort... -uv run python -m isort src/ tests/ || exit /b - -echo Formatting code with black... -uv run python -m black src/ tests/ || exit /b - -echo Code formatting complete. -exit /b 0 diff --git a/scripts/windows/dev/lint.bat b/scripts/windows/dev/lint.bat deleted file mode 100644 index fba418f24..000000000 --- a/scripts/windows/dev/lint.bat +++ /dev/null @@ -1,23 +0,0 @@ -@echo off -setlocal - -set SCRIPT_DIR=%~dp0 - -call "%SCRIPT_DIR%mypy.bat" -set MYPY_EXIT=%ERRORLEVEL% - -call "%SCRIPT_DIR%pylint.bat" -set PYLINT_EXIT=%ERRORLEVEL% - -if not %MYPY_EXIT%==0 ( - echo Linting failed. - exit /b 1 -) - -if not %PYLINT_EXIT%==0 ( - echo Linting failed. - exit /b 1 -) - -echo All linting checks passed. -exit /b 0 diff --git a/scripts/windows/dev/mypy.bat b/scripts/windows/dev/mypy.bat deleted file mode 100644 index 686c7716f..000000000 --- a/scripts/windows/dev/mypy.bat +++ /dev/null @@ -1,10 +0,0 @@ -@echo off -setlocal - -echo Running type checking with mypy... -if "%~1"=="" ( - uv run python -m mypy -) else ( - uv run python -m mypy %* -) -exit /b %ERRORLEVEL% diff --git a/scripts/windows/dev/pre_commit.bat b/scripts/windows/dev/pre_commit.bat deleted file mode 100644 index deebc171d..000000000 --- a/scripts/windows/dev/pre_commit.bat +++ /dev/null @@ -1,9 +0,0 @@ -@echo off -setlocal - -echo Installing pre-commit hooks... -uv run pre-commit install || exit /b -uv run pre-commit install --hook-type pre-commit --hook-type pre-push || exit /b -echo Pre-commit hooks installed successfully. - -exit /b 0 diff --git a/scripts/windows/dev/pylint.bat b/scripts/windows/dev/pylint.bat deleted file mode 100644 index 5e6aa0fa0..000000000 --- a/scripts/windows/dev/pylint.bat +++ /dev/null @@ -1,10 +0,0 @@ -@echo off -setlocal - -echo Running linting with pylint... -if "%~1"=="" ( - uv run python -m pylint src/sampletones -) else ( - uv run python -m pylint %* -) -exit /b %ERRORLEVEL% diff --git a/scripts/windows/dev/tests.bat b/scripts/windows/dev/tests.bat deleted file mode 100644 index fd0d1659b..000000000 --- a/scripts/windows/dev/tests.bat +++ /dev/null @@ -1,32 +0,0 @@ -@echo off -setlocal - -echo Running doctests... -uv run python -m pytest src/ --doctest-modules --no-cov -set DOCTEST_EXIT=%ERRORLEVEL% - -echo Running benchmarks... -uv run python -m pytest tests/benchmarks --no-cov -set BENCHMARK_EXIT=%ERRORLEVEL% - -echo Running pytest with coverage... -uv run python -m pytest -n 6 --cov --ignore=tests/benchmarks -set PYTEST_EXIT=%ERRORLEVEL% - -if not %DOCTEST_EXIT%==0 ( - echo Tests failed. - exit /b 1 -) - -if not %PYTEST_EXIT%==0 ( - echo Tests failed. - exit /b 1 -) - -if not %BENCHMARK_EXIT%==0 ( - echo Tests failed. - exit /b 1 -) - -echo All tests passed. -exit /b 0 diff --git a/scripts/windows/lib/root.bat b/scripts/windows/lib/root.bat deleted file mode 100644 index f9952477a..000000000 --- a/scripts/windows/lib/root.bat +++ /dev/null @@ -1,7 +0,0 @@ -if not exist "%~dp0..\..\..\src\sampletones\" ( - echo ERROR: SampleToNES project root not found.>&2 - exit /b 1 -) - -cd /d "%~dp0..\..\.." || exit /b 1 -exit /b 0 diff --git a/src/sampletones/__init__.py b/src/sampletones/__init__.py index 60d9ed3cc..a059dd93b 100644 --- a/src/sampletones/__init__.py +++ b/src/sampletones/__init__.py @@ -74,9 +74,9 @@ def __getattr__(name: str) -> Any: __all__ = [ + "ChannelName", "Config", "Generator", - "ChannelName", "Instruction", "InstructionLibrary", "NoiseGenerator", diff --git a/src/sampletones/__main__.py b/src/sampletones/__main__.py index 234e5e91d..c17e0f4ba 100644 --- a/src/sampletones/__main__.py +++ b/src/sampletones/__main__.py @@ -1,224 +1,15 @@ -import argparse import multiprocessing -from argparse import RawTextHelpFormatter -from dataclasses import dataclass -from pathlib import Path -from typing import TYPE_CHECKING, Optional +import sys -from sampletones_shared.paths.extensions import EXT_FILES_AUDIO +from sampletones.commands.registry import COMMANDS +from sampletones.dispatcher import dispatch -if TYPE_CHECKING: - from typing import List - from sampletones_core.configs import Config - from sampletones_core.constants.enums import ChannelName - -HELP_PATH = """Path to either: - * audio file path/directory to reconstruct - * reconstruction .stn file to load a reconstruction - * instructions library .ins file to load a library""" - -HELP_OUTPUT = """Output path for reconstruction.""" - -HELP_CHANNELS = """Channels the reconstruction may use, comma separated - (pulse1, pulse2, triangle, noise; default: pulse1,triangle,noise)""" - -HELP_CONFIG = """Path to a configuration .json file - (if not provided, default configuration will be used)""" - -HELP_GENERATE = """Generate library data for given configuration - (using default one if not provided)""" - -HELP_HELP = """Show this help message and exit""" - -HELP_VERSION = "Show application version information" - -HELP_SELF_CHECK = """Verify that the imports, bundled resources - and configuration files this build ships are all usable""" - - -@dataclass(frozen=True) -class ProgramArguments: - path: Optional[Path] = None - output: Optional[Path] = None - config: Optional[Path] = None - channels: Optional[str] = None - - help: bool = False - version: bool = False - generate: bool = False - self_check: bool = False - - -def _load_config(config_path: Optional[Path]) -> "Config": - from sampletones_core.configs import Config - - return Config.load(config_path) if config_path else Config.default() - - -def _channels(stated: Optional[str]) -> "List[ChannelName]": - """The channels a run hands out: the ones named on the command line, or the usual three. - - Raises: - SystemExit: If a name is not one of the channels the hardware has. - """ - from sampletones_core.constants.enums import DEFAULT_CHANNELS, ChannelName - - if stated is None: - return list(DEFAULT_CHANNELS) - - names = [name.strip() for name in stated.split(",") if name.strip()] - try: - return [ChannelName(name) for name in names] - except ValueError as exception: - raise SystemExit(f"Unknown channel in --channels: {exception}") from exception - - -def main() -> None: - parser = argparse.ArgumentParser( - prog="SampleToNES", - add_help=False, - formatter_class=RawTextHelpFormatter, - ) - parser.add_argument( - "path", - nargs="?", - default=None, - help=HELP_PATH, - ) - parser.add_argument( - "--output", - "-o", - type=Path, - default=None, - help=HELP_OUTPUT, - ) - parser.add_argument( - "--channels", - type=str, - default=None, - help=HELP_CHANNELS, - ) - parser.add_argument( - "--config", - "-c", - type=Path, - default=None, - help=HELP_CONFIG, - ) - parser.add_argument( - "--generate", - "-g", - action="store_true", - help=HELP_GENERATE, - ) - parser.add_argument( - "--help", - "-h", - action="store_true", - help=HELP_HELP, - ) - parser.add_argument( - "--version", - "-v", - action="store_true", - help=HELP_VERSION, - ) - parser.add_argument( - "--self-check", - action="store_true", - help=HELP_SELF_CHECK, - ) - args: ProgramArguments = ProgramArguments(**vars(parser.parse_args())) - - if args.help: - parser.print_help() - return None - - if args.version: - from sampletones_shared.application import ( - SAMPLETONES_NAME_VERSION, - ) - - return print(SAMPLETONES_NAME_VERSION) - - if args.self_check: - from sampletones.self_check import run_self_check - - raise SystemExit(run_self_check()) - - from sampletones_shared.array import report_array_backend - - report_array_backend() - - config_path = Path(args.config) if args.config else None - output_path = Path(args.output) if args.output else None - - from sampletones_shared.paths.extensions import ( - EXT_FILE_LIBRARY, - EXT_FILE_PROJECT, - EXT_FILE_RECONSTRUCTION, - ) - - if args.generate: - from sampletones_core.scripts.library import generate_library - - config = _load_config(config_path) - return generate_library(config) - - project_path: Optional[Path] = None - library_path: Optional[Path] = None - reconstruction_path: Optional[Path] = None - - if args.path: - path = Path(args.path) - if path.is_file(): - suffix = path.suffix.lower() - if suffix == EXT_FILE_PROJECT: - project_path = path - - elif suffix == EXT_FILE_RECONSTRUCTION: - reconstruction_path = path - - elif suffix == EXT_FILE_LIBRARY: - library_path = path - - elif suffix in EXT_FILES_AUDIO: - from sampletones_core.scripts.reconstruction import ( - reconstruct_file, - ) - - config = _load_config(config_path) - return reconstruct_file(path, config, _channels(args.channels), output_path) - - else: - raise RuntimeError( - f"Unsupported file extension, only audio ({', '.join(EXT_FILES_AUDIO)})," - f"{EXT_FILE_RECONSTRUCTION} reconstruction, " - f"and {EXT_FILE_LIBRARY} library files are supported." - ) - - elif path.is_dir(): - from sampletones_core.scripts.reconstruction import ( - reconstruct_directory, - ) - - config = _load_config(config_path) - return reconstruct_directory(path, config, _channels(args.channels)) - - else: - raise RuntimeError("Unsupported path type or file extension.") - - from sampletones.run import run_application - - return run_application( - config_path, - library_path=library_path, - reconstruction_path=reconstruction_path, - project_path=project_path, - ) +def main() -> int: + """Runs the command named on the command line, which is what the ``sampletones`` entry does.""" + return dispatch(COMMANDS, sys.argv[1:]) if __name__ == "__main__": multiprocessing.freeze_support() - main() + raise SystemExit(main()) diff --git a/src/sampletones_core/scripts/__init__.py b/src/sampletones/commands/__init__.py similarity index 100% rename from src/sampletones_core/scripts/__init__.py rename to src/sampletones/commands/__init__.py diff --git a/src/sampletones/commands/convert.py b/src/sampletones/commands/convert.py new file mode 100644 index 000000000..59d909229 --- /dev/null +++ b/src/sampletones/commands/convert.py @@ -0,0 +1,88 @@ +from argparse import ArgumentParser, Namespace +from dataclasses import dataclass +from pathlib import Path +from typing import Final, Optional, Tuple + +from sampletones_shared.command import Command +from sampletones_shared.options import add_config_option + +NAME: Final[str] = "convert" +HELP: Final[str] = "reconstruct recordings into a .stn file" +SOURCES_HELP: Final[str] = "recordings mixed into one reconstruction, or one directory converted file by file" +OUTPUT_HELP: Final[str] = ( + "where the reconstruction is written; without it, the configuration's reconstructions directory" +) +CHANNELS_HELP: Final[str] = ( + "channels the reconstruction may use, comma separated: pulse1, pulse2, triangle, noise; " + "without it pulse1, triangle and noise" +) +STEMS_HELP: Final[str] = "a JSON file holding a stems setup; its entries pair with the sources in order" + + +@dataclass(frozen=True) +class ConvertArguments: + """What a conversion is given: the sources, where the result goes, and how the channels are handed out.""" + + sources: Tuple[Path, ...] + output: Optional[Path] + config: Optional[Path] + channels: Optional[str] + stems: Optional[Path] + + +def configure(parser: ArgumentParser) -> None: + parser.add_argument("sources", nargs="+", type=Path, help=SOURCES_HELP) + parser.add_argument("--output", "-o", type=Path, default=None, help=OUTPUT_HELP) + add_config_option(parser) + setup = parser.add_mutually_exclusive_group() + setup.add_argument("--channels", type=str, default=None, help=CHANNELS_HELP) + setup.add_argument("--stems", type=Path, default=None, help=STEMS_HELP) + + +def run(arguments: Namespace) -> int: + """Reconstructs the sources under the stems setup the options describe. + + Raises: + SystemExit: If a source is missing or no recording, a channel is unknown, the stems file + is missing or no setup, or the sources and the setup pair up wrong. + """ + given = ConvertArguments( + sources=tuple(arguments.sources), + output=arguments.output, + config=arguments.config, + channels=arguments.channels, + stems=arguments.stems, + ) + + from sampletones_core.headless.config import load_config + from sampletones_core.headless.conversion.pairing import pairing_lines + from sampletones_core.headless.conversion.request import ( + ConversionRequest, + channels_named, + classic_setup, + load_stems, + ) + from sampletones_core.headless.conversion.runners import reconstruct + from sampletones_shared.array import report_array_backend + from sampletones_shared.utils.validation import describe_failure + + try: + stems = load_stems(given.stems) if given.stems is not None else classic_setup(channels_named(given.channels)) + request = ConversionRequest(sources=given.sources, stems=stems, output_path=given.output) + except ValueError as error: + raise SystemExit(describe_failure(error)) from error + + for line in pairing_lines(request): + print(line) + + report_array_backend() + reconstruct(request, load_config(given.config)) + return 0 + + +CONVERT: Final[Command] = Command( + name=NAME, + help=HELP, + configure=configure, + run=run, +) diff --git a/src/sampletones/commands/library.py b/src/sampletones/commands/library.py new file mode 100644 index 000000000..09308690f --- /dev/null +++ b/src/sampletones/commands/library.py @@ -0,0 +1,42 @@ +from argparse import ArgumentParser, Namespace +from dataclasses import dataclass +from pathlib import Path +from typing import Final, Optional + +from sampletones_shared.command import Command +from sampletones_shared.options import add_config_option + +NAME: Final[str] = "library" +HELP: Final[str] = "generate the instruction library for a configuration" + + +@dataclass(frozen=True) +class LibraryArguments: + """What a library generation is given: the configuration the library is built for, if any.""" + + config: Optional[Path] + + +def configure(parser: ArgumentParser) -> None: + add_config_option(parser) + + +def run(arguments: Namespace) -> int: + """Generates the instruction library for the configuration.""" + given = LibraryArguments(config=arguments.config) + + from sampletones_core.headless.config import load_config + from sampletones_core.headless.library import generate_library + from sampletones_shared.array import report_array_backend + + report_array_backend() + generate_library(load_config(given.config)) + return 0 + + +LIBRARY: Final[Command] = Command( + name=NAME, + help=HELP, + configure=configure, + run=run, +) diff --git a/src/sampletones/commands/open.py b/src/sampletones/commands/open.py new file mode 100644 index 000000000..9457c97bd --- /dev/null +++ b/src/sampletones/commands/open.py @@ -0,0 +1,73 @@ +from argparse import ArgumentParser, Namespace +from dataclasses import dataclass +from pathlib import Path +from typing import Final, Optional + +from sampletones_shared.command import Command +from sampletones_shared.options import add_config_option + +NAME: Final[str] = "open" +HELP: Final[str] = "start the application with a project, reconstruction or library loaded" +PATH_HELP: Final[str] = "a .stp project, a .stn reconstruction or an .ins library" + + +@dataclass(frozen=True) +class OpenArguments: + """What an opening run is given: the file to load and the configuration to start with.""" + + path: Path + config: Optional[Path] + + +def configure(parser: ArgumentParser) -> None: + parser.add_argument("path", type=Path, help=PATH_HELP) + add_config_option(parser) + + +def run(arguments: Namespace) -> int: + """Starts the application with the file loaded. + + Raises: + SystemExit: If the path names no file, a recording, or a file of another kind. + """ + given = OpenArguments(path=arguments.path, config=arguments.config) + + from sampletones_shared.paths.extensions import ( + EXT_FILE_LIBRARY, + EXT_FILE_PROJECT, + EXT_FILE_RECONSTRUCTION, + is_audio_file, + ) + + if not given.path.is_file(): + raise SystemExit(f"No file at {given.path}.") + + if is_audio_file(given.path): + raise SystemExit(f"{given.path} is a recording; run: sampletones convert {given.path}") + + suffix = given.path.suffix.lower() + if suffix not in (EXT_FILE_PROJECT, EXT_FILE_RECONSTRUCTION, EXT_FILE_LIBRARY): + raise SystemExit( + f"{given.path} is neither a {EXT_FILE_PROJECT} project, a {EXT_FILE_RECONSTRUCTION} " + f"reconstruction nor an {EXT_FILE_LIBRARY} library." + ) + + from sampletones.run import run_application + from sampletones_shared.array import report_array_backend + + report_array_backend() + run_application( + given.config, + project_path=given.path if suffix == EXT_FILE_PROJECT else None, + reconstruction_path=given.path if suffix == EXT_FILE_RECONSTRUCTION else None, + library_path=given.path if suffix == EXT_FILE_LIBRARY else None, + ) + return 0 + + +OPEN: Final[Command] = Command( + name=NAME, + help=HELP, + configure=configure, + run=run, +) diff --git a/src/sampletones/commands/registry.py b/src/sampletones/commands/registry.py new file mode 100644 index 000000000..540865cce --- /dev/null +++ b/src/sampletones/commands/registry.py @@ -0,0 +1,12 @@ +from typing import Final, Tuple + +from sampletones.commands.convert import CONVERT +from sampletones.commands.library import LIBRARY +from sampletones.commands.open import OPEN +from sampletones.commands.run import RUN +from sampletones.commands.self_check import SELF_CHECK +from sampletones_shared.command import Command +from sampletones_tools.registry import DEVELOPER_COMMANDS + +USER_COMMANDS: Final[Tuple[Command, ...]] = (RUN, OPEN, CONVERT, LIBRARY, SELF_CHECK) +COMMANDS: Final[Tuple[Command, ...]] = (*USER_COMMANDS, *DEVELOPER_COMMANDS) diff --git a/src/sampletones/commands/run.py b/src/sampletones/commands/run.py new file mode 100644 index 000000000..9f5a3e1d1 --- /dev/null +++ b/src/sampletones/commands/run.py @@ -0,0 +1,41 @@ +from argparse import ArgumentParser, Namespace +from dataclasses import dataclass +from pathlib import Path +from typing import Final, Optional + +from sampletones_shared.command import Command +from sampletones_shared.options import add_config_option + +NAME: Final[str] = "run" +HELP: Final[str] = "start the application" + + +@dataclass(frozen=True) +class RunArguments: + """What a run of the application is given: the configuration it starts with, if any.""" + + config: Optional[Path] + + +def configure(parser: ArgumentParser) -> None: + add_config_option(parser) + + +def run(arguments: Namespace) -> int: + """Starts the application.""" + given = RunArguments(config=arguments.config) + + from sampletones.run import run_application + from sampletones_shared.array import report_array_backend + + report_array_backend() + run_application(given.config) + return 0 + + +RUN: Final[Command] = Command( + name=NAME, + help=HELP, + configure=configure, + run=run, +) diff --git a/src/sampletones/commands/self_check.py b/src/sampletones/commands/self_check.py new file mode 100644 index 000000000..a9957caa0 --- /dev/null +++ b/src/sampletones/commands/self_check.py @@ -0,0 +1,28 @@ +from argparse import ArgumentParser, Namespace +from typing import Final + +from sampletones_shared.command import Command + +NAME: Final[str] = "self-check" +HELP: Final[str] = "verify that this build's imports, bundled resources and configuration files are usable" + + +def configure(parser: ArgumentParser) -> None: + del parser + + +def run(arguments: Namespace) -> int: + """Runs every startup check and answers with the status a packaged build is held to.""" + del arguments + + from sampletones.self_check import run_self_check + + return run_self_check() + + +SELF_CHECK: Final[Command] = Command( + name=NAME, + help=HELP, + configure=configure, + run=run, +) diff --git a/src/sampletones/dispatcher.py b/src/sampletones/dispatcher.py new file mode 100644 index 000000000..b0d85872a --- /dev/null +++ b/src/sampletones/dispatcher.py @@ -0,0 +1,73 @@ +from argparse import ArgumentParser +from typing import Final, Sequence + +from sampletones_shared.application import SAMPLETONES_NAME_VERSION +from sampletones_shared.command import Command + +PROGRAM: Final[str] = "sampletones" +DESCRIPTION: Final[str] = "SampleToNES turns recordings into NES instruments and plays them back." +DEFAULT_COMMAND: Final[str] = "run" +COMMAND_FIELD: Final[str] = "command" +COMMAND_METAVAR: Final[str] = "" +EPILOG: Final[str] = ( + f"Run '{PROGRAM} {COMMAND_METAVAR} --help' for a command's options. The commands for developing " + "SampleToNES follow the ones for using it; one that needs the repository says so and names the " + f"'uv run {PROGRAM} {COMMAND_METAVAR}' line to run from a checkout." +) + + +def build_parser(commands: Sequence[Command]) -> ArgumentParser: + """The parser over ``commands``: one subcommand each, with the version and the help as flags. + + Args: + commands: The commands on offer, listed in the help in this order. + + Returns: + ArgumentParser: The parser the entry runs. + + Raises: + ValueError: If two commands share a name. + """ + names = [command.name for command in commands] + repeated = sorted({name for name in names if names.count(name) > 1}) + if repeated: + raise ValueError(f"the commands share a name: {', '.join(repeated)}") + + parser = ArgumentParser( + prog=PROGRAM, + description=DESCRIPTION, + epilog=EPILOG, + ) + parser.add_argument( + "--version", + "-v", + action="version", + version=SAMPLETONES_NAME_VERSION, + ) + subparsers = parser.add_subparsers( + dest=COMMAND_FIELD, + metavar=COMMAND_METAVAR, + required=True, + ) + + for command in commands: + subparser = subparsers.add_parser(command.name, help=command.help, description=command.help) + command.configure(subparser) + + return parser + + +def dispatch(commands: Sequence[Command], argv: Sequence[str]) -> int: + """Runs the command ``argv`` names, and the default command when it names none. + + Args: + commands: The commands on offer. + argv: The arguments after the program name. + + Returns: + int: The exit status the command answers with. + """ + parser = build_parser(commands) + arguments = parser.parse_args(list(argv) or [DEFAULT_COMMAND]) + command = next(command for command in commands if command.name == arguments.command) + return command.run(arguments) diff --git a/src/sampletones/self_check.py b/src/sampletones/self_check.py index 30fe3b407..9841e8f94 100644 --- a/src/sampletones/self_check.py +++ b/src/sampletones/self_check.py @@ -115,7 +115,10 @@ def _check_language() -> str: def _check_resources() -> str: from sampletones_application.ui.resources.items import FontResource, IconResource - from sampletones_application.ui.resources.resources import get_font_path, get_icon_path + from sampletones_application.ui.resources.resources import ( + get_font_path, + get_icon_path, + ) for font in FontResource: get_font_path(font) @@ -138,7 +141,9 @@ def _check_export_backends() -> str: def _check_file_dialog_backend() -> str: - from sampletones_application.utils.file_dialogs.selection import select_file_dialog_backend + from sampletones_application.utils.file_dialogs.selection import ( + select_file_dialog_backend, + ) return type(select_file_dialog_backend()).__name__ @@ -168,7 +173,10 @@ def run_self_check() -> int: try: detail = check.run() except CHECK_FAILURES as exception: - print(f"{FAILURE_PREFIX} {check.name}: {type(exception).__name__}: {exception}", file=sys.stderr) + print( + f"{FAILURE_PREFIX} {check.name}: {type(exception).__name__}: {exception}", + file=sys.stderr, + ) return FAILURE_STATUS print(f"{SUCCESS_PREFIX} {check.name}: {detail}") diff --git a/src/sampletones_application/config/deployment/deployment.py b/src/sampletones_application/config/deployment/deployment.py index a89c411f8..d537bc7b5 100644 --- a/src/sampletones_application/config/deployment/deployment.py +++ b/src/sampletones_application/config/deployment/deployment.py @@ -23,7 +23,7 @@ class DeploymentConfig(BaseModel, frozen=True): development values — verbose logging, and strict history so a missing transaction is reported the moment it happens rather than healed in silence — since the source tree is where an edit is written and where that report is worth having. A release build injects the - user-facing values through ``scripts/release_env_hook.py``, so a shipped artifact is quiet + user-facing values through ``scripts/runtime_hooks/release_environment.py``, so a shipped artifact is quiet and self-healing whatever the tree it was built from said. The ``SAMPLETONES_LOG_LEVEL`` and ``SAMPLETONES_STRICT_HISTORY`` environment variables set diff --git a/src/sampletones_application/coordinators/playback/router.py b/src/sampletones_application/coordinators/playback/router.py index 194460f3d..ab1206d27 100644 --- a/src/sampletones_application/coordinators/playback/router.py +++ b/src/sampletones_application/coordinators/playback/router.py @@ -13,7 +13,7 @@ class PlaybackRouter: Every command acts on one *target*: the active tab's own source when that tab has something to play, and otherwise the intentional source currently engaged (owning the device). Preview sounds belong to no source and answer only to Stop. The full contract is documented in - ``docs/development/playback.md``. + ``docs/development/application/playback.md``. It is stateless: the target is recomputed from the live sources on each call, so the transport stays in step with tab changes and playback transitions automatically. diff --git a/src/sampletones_application/layout/settings/__init__.py b/src/sampletones_application/layout/settings/__init__.py index f95124389..c23a3763b 100644 --- a/src/sampletones_application/layout/settings/__init__.py +++ b/src/sampletones_application/layout/settings/__init__.py @@ -2,7 +2,7 @@ from sampletones_application.layout.settings.audio import AudioSettingsLayout from sampletones_application.layout.settings.display import DisplaySettingsLayout -from sampletones_application.layout.settings.export import ExportSettingsLayout +from sampletones_application.layout.settings.export.export import ExportSettingsLayout from sampletones_application.layout.settings.keybindings import KeybindingsSettingsLayout from sampletones_application.layout.settings.render import RenderSettingsLayout diff --git a/src/sampletones_player/driver/assembler/__init__.py b/src/sampletones_application/layout/settings/export/__init__.py similarity index 100% rename from src/sampletones_player/driver/assembler/__init__.py rename to src/sampletones_application/layout/settings/export/__init__.py diff --git a/src/sampletones_application/layout/settings/export.py b/src/sampletones_application/layout/settings/export/export.py similarity index 59% rename from src/sampletones_application/layout/settings/export.py rename to src/sampletones_application/layout/settings/export/export.py index 5c4d289a9..ffdb77547 100644 --- a/src/sampletones_application/layout/settings/export.py +++ b/src/sampletones_application/layout/settings/export/export.py @@ -1,7 +1,9 @@ from pydantic import BaseModel from sampletones_application.layout.primitives import Dimensions +from sampletones_application.layout.settings.export.indicator import LoadingIndicatorLayout class ExportSettingsLayout(BaseModel, extra="forbid", frozen=True): window: Dimensions + indicator: LoadingIndicatorLayout diff --git a/src/sampletones_application/layout/settings/export/indicator.py b/src/sampletones_application/layout/settings/export/indicator.py new file mode 100644 index 000000000..f4099966a --- /dev/null +++ b/src/sampletones_application/layout/settings/export/indicator.py @@ -0,0 +1,13 @@ +from pydantic import BaseModel, Field + + +class LoadingIndicatorLayout(BaseModel, extra="forbid", frozen=True): + """The turning ring's size, stated as factors of the font it is drawn in. + + DearPyGui draws the ring ``radius × font size × (1 − thickness / 4)`` pixels across and pads its + box by the frame padding above and below, the same padding it sets text beside the ring down by. + The ring sits level with a line of that font when ``radius × (1 − thickness / 4)`` equals one. + """ + + radius: float = Field(gt=0.0) + thickness: float = Field(gt=0.0, lt=4.0) diff --git a/src/sampletones_application/logic/main/sources/scan.py b/src/sampletones_application/logic/main/sources/scan.py index 43d08b7b7..56e1328fb 100644 --- a/src/sampletones_application/logic/main/sources/scan.py +++ b/src/sampletones_application/logic/main/sources/scan.py @@ -3,7 +3,8 @@ from typing import Callable, Final, List, Optional, Tuple from sampletones_application.utils.parallelization.thread import concurrent -from sampletones_core.reconstructions.converter.paths import is_audio_file, walk_entries +from sampletones_core.reconstructions.converter.paths import walk_entries +from sampletones_shared.paths.extensions import is_audio_file from sampletones_shared.types.callback import PathCallback, VoidCallback from sampletones_shared.utils.callbacks import CallbackMixin diff --git a/src/sampletones_application/ui/elements/layout/centered.py b/src/sampletones_application/ui/elements/layout/centered.py new file mode 100644 index 000000000..6ed5bcb03 --- /dev/null +++ b/src/sampletones_application/ui/elements/layout/centered.py @@ -0,0 +1,26 @@ +from contextlib import contextmanager +from typing import Final, Iterator + +import dearpygui.dearpygui as dpg + +EQUAL_SHARE: Final[float] = 1.0 + + +@contextmanager +def centered() -> Iterator[None]: + """Stand what the block builds in the middle of the width it is offered. + + The content sits in a column fitted to it, flanked by two columns sharing the rest of the width + equally. The fitted column follows the content from frame to frame, so a label that grows or + shrinks while it stands stays centered. + """ + with dpg.table(header_row=False): + dpg.add_table_column(width_stretch=True, init_width_or_weight=EQUAL_SHARE) + dpg.add_table_column(width_fixed=True) + dpg.add_table_column(width_stretch=True, init_width_or_weight=EQUAL_SHARE) + with dpg.table_row(): + dpg.add_spacer() + with dpg.group(): + yield + + dpg.add_spacer() diff --git a/src/sampletones_application/ui/elements/plus_minus_buttons.py b/src/sampletones_application/ui/elements/plus_minus_buttons.py index b355f4363..0f2dbc79c 100644 --- a/src/sampletones_application/ui/elements/plus_minus_buttons.py +++ b/src/sampletones_application/ui/elements/plus_minus_buttons.py @@ -90,7 +90,12 @@ def delete(self) -> None: """Removes the buttons and, when hold-repeat is armed, the shared mouse handler.""" self._clear_existing_items() - def _build(self, *, increment_enabled: bool, decrement_enabled: bool) -> None: + def _build( + self, + *, + increment_enabled: bool, + decrement_enabled: bool, + ) -> None: self._clear_existing_items() increment_leads = self._order is PlusMinusOrder.PLUS_FIRST with dpg.table( diff --git a/src/sampletones_application/ui/panels/dialogs/export.py b/src/sampletones_application/ui/panels/dialogs/export.py index 25f53c6f9..5725fda24 100644 --- a/src/sampletones_application/ui/panels/dialogs/export.py +++ b/src/sampletones_application/ui/panels/dialogs/export.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, Optional +from typing import Any, Dict, Final, Optional import dearpygui.dearpygui as dpg @@ -20,6 +20,7 @@ from sampletones_application.ui.elements.dialog import GUIDialogWindow from sampletones_application.ui.elements.fonts.font import Font from sampletones_application.ui.elements.fonts.registry import FontRegistry +from sampletones_application.ui.elements.layout.centered import centered from sampletones_application.utils.gui.dialog_navigation import FocusStop from sampletones_application.utils.gui.dpg import dpg_configure_item, dpg_set_value from sampletones_application.utils.gui.keyboard import KeyRouter @@ -30,6 +31,8 @@ from sampletones_core.exports.stage import ExportStage from sampletones_shared.types.callback import VoidCallback +RING_STYLE: Final[int] = 1 + class GUIExportWindow(GUIDialogWindow): """Modal report over an export while it runs. @@ -55,6 +58,7 @@ def __init__( ) -> None: self._language_manager = language_manager self._text_colors = text_colors + self._indicator = layout.export.indicator self._view_model: SongExportViewModel = SongExportViewModel.idle() self.on_cancel: Optional[VoidCallback] = None @@ -138,23 +142,29 @@ def _create_working(self) -> None: """The reading of a stage whose length the data decides: what it holds, and that it turns. A bar would have to state a fraction of something, and this stage travels toward nothing, - so what stands here is the figure it does know beside a symbol that keeps moving. + so what stands here is the figure it does know beside a symbol that keeps moving. The pair + stands centered in the window, and both are drawn in the figure's font, which is the font + the ring takes its size from, so the ring sits level with the line beside it. """ - with dpg.group( - tag=TAG_SETTINGS_EXPORT_GROUP_WORKING, - show=False, - horizontal=True, + with ( + dpg.group( + tag=TAG_SETTINGS_EXPORT_GROUP_WORKING, + show=False, + ), + centered(), + dpg.group(horizontal=True), ): dpg.add_loading_indicator( - style=1, - radius=2.0, - thickness=1.5, + style=RING_STYLE, + radius=self._indicator.radius, + thickness=self._indicator.thickness, ) dpg.add_text("", tag=TAG_SETTINGS_EXPORT_TEXT_FIGURE) - FontRegistry.bind_to_item( - TAG_SETTINGS_EXPORT_TEXT_FIGURE, - Font.MONO_SMALL, - ) + + FontRegistry.bind_to_item( + TAG_SETTINGS_EXPORT_GROUP_WORKING, + Font.MONO_SMALL, + ) def _create_cancel(self) -> None: GUIButton( @@ -182,6 +192,10 @@ def _render(self) -> None: show=view_model.working_visible, ) dpg_set_value(TAG_SETTINGS_EXPORT_TEXT_FIGURE, view_model.figure) + dpg_configure_item( + TAG_SETTINGS_EXPORT_TEXT_FIGURE, + show=view_model.figure_visible, + ) dpg_configure_item( TAG_SETTINGS_EXPORT_BUTTON_CANCEL, enabled=view_model.cancel_enabled, diff --git a/src/sampletones_application/ui/panels/main/converter/setup.py b/src/sampletones_application/ui/panels/main/converter/setup.py index 01c50114f..c9a0da3cf 100644 --- a/src/sampletones_application/ui/panels/main/converter/setup.py +++ b/src/sampletones_application/ui/panels/main/converter/setup.py @@ -102,15 +102,20 @@ def create_controls(self) -> None: ) FontRegistry.bind_to_item(cap_input, Font.MONO) - with dpg.group(tag=TAG_MAIN_CONVERTER_GROUP_ORDER, show=False): - with labeled_field(self._language_manager["main.converter.label.hierarchy_mode"], self._label_width): - dpg.add_combo( - items=list(self._hierarchy_labels.values()), - tag=TAG_MAIN_CONVERTER_COMBO_HIERARCHY_MODE, - width=self._input_width, - default_value=self._hierarchy_labels[DEFAULT_STEMS_HIERARCHY_MODE], - callback=self._on_hierarchy_mode_edited, - ) + with ( + dpg.group(tag=TAG_MAIN_CONVERTER_GROUP_ORDER, show=False), + labeled_field( + self._language_manager["main.converter.label.hierarchy_mode"], + self._label_width, + ), + ): + dpg.add_combo( + items=list(self._hierarchy_labels.values()), + tag=TAG_MAIN_CONVERTER_COMBO_HIERARCHY_MODE, + width=self._input_width, + default_value=self._hierarchy_labels[DEFAULT_STEMS_HIERARCHY_MODE], + callback=self._on_hierarchy_mode_edited, + ) dpg.bind_item_handler_registry(TAG_MAIN_CONVERTER_INPUT_CHANNEL_CAP, self._settings_handler_tag) self._attach_tooltips() diff --git a/src/sampletones_application/ui/panels/reconstruction/instruments/instruments.py b/src/sampletones_application/ui/panels/reconstruction/instruments/instruments.py index 05ae2b8c7..b84b9312e 100644 --- a/src/sampletones_application/ui/panels/reconstruction/instruments/instruments.py +++ b/src/sampletones_application/ui/panels/reconstruction/instruments/instruments.py @@ -537,24 +537,26 @@ def _create_audition_selector(self, window_tag: str) -> None: means choosing which one reads it. The choice belongs to the reader listening rather than to the voice, so it stays on the panel and reaches no document. """ - with dpg.group( - tag=self.audition_group_tag, - parent=window_tag, - show=False, - ): - with labeled_field( + with ( + dpg.group( + tag=self.audition_group_tag, + parent=window_tag, + show=False, + ), + labeled_field( self._language_manager["reconstructions.instruments.label.audition"], self._pitch_stepper_style.dimensions.label_width, parent=self.audition_group_tag, - ): - dpg.add_radio_button( - items=[self._generator_labels[generator_name] for generator_name in GeneratorName], - tag=self.audition_tag, - default_value=self._generator_labels[AUDITION_GENERATOR], - callback=self._on_audition_generator_changed, - horizontal=True, - ) - FontRegistry.bind_to_item(self.audition_tag, Font.REGULAR_SMALL) + ), + ): + dpg.add_radio_button( + items=[self._generator_labels[generator_name] for generator_name in GeneratorName], + tag=self.audition_tag, + default_value=self._generator_labels[AUDITION_GENERATOR], + callback=self._on_audition_generator_changed, + horizontal=True, + ) + FontRegistry.bind_to_item(self.audition_tag, Font.REGULAR_SMALL) self._status_bar.bind_to_item( self.audition_tag, diff --git a/src/sampletones_application/ui/resources/loader.py b/src/sampletones_application/ui/resources/loader.py index 5c61dc3d2..327a1fa7c 100644 --- a/src/sampletones_application/ui/resources/loader.py +++ b/src/sampletones_application/ui/resources/loader.py @@ -1,39 +1,33 @@ -import sys -from importlib.resources import files -from importlib.resources.abc import Traversable from pathlib import Path -from typing import Union +from typing import Final +from sampletones_shared.paths.package import package_directory from sampletones_shared.types.path import Pathlike +ASSETS_PACKAGE: Final[str] = "sampletones_assets" + class ResourceLoader: + """Reads the files of one directory of the assets package, where the package lies in every copy.""" + def __init__(self, resource_directory: Pathlike) -> None: self.resource_directory = Path(resource_directory) - def _get_package_path(self, resource_name: str) -> Union[Path, Traversable]: - if getattr(sys, "frozen", False): - base_path = Path(sys._MEIPASS) # type: ignore[attr-defined] # pylint: disable=protected-access - resource_type = self.resource_directory.name - return base_path / "assets" / resource_type / resource_name - - package_name = f"sampletones_assets.{self.resource_directory.name}" - return files(package_name).joinpath(resource_name) + def _get_package_path(self, resource_name: str) -> Path: + package = f"{ASSETS_PACKAGE}.{self.resource_directory.name}" + return package_directory(package) / resource_name def get_path(self, resource_name: str) -> str: - resource_path = self._get_package_path(resource_name) + """The resource's location, which a widget loading it by path reads. - if isinstance(resource_path, Path): - if not resource_path.exists(): - raise FileNotFoundError(f"Resource not found: '{resource_path}'") - else: - try: - resource_path.read_bytes() - except FileNotFoundError as exception: - raise FileNotFoundError(f"Resource not found: '{resource_path}'") from exception + Raises: + FileNotFoundError: If the directory holds no file of that name. + """ + resource_path = self._get_package_path(resource_name) + if not resource_path.is_file(): + raise FileNotFoundError(f"Resource not found: '{resource_path}'") return str(resource_path) def get_bytes(self, resource_name: str) -> bytes: - resource_path = self._get_package_path(resource_name) - return resource_path.read_bytes() + return self._get_package_path(resource_name).read_bytes() diff --git a/src/sampletones_application/utils/palette/colors/written.py b/src/sampletones_application/utils/palette/colors/written.py index 0684e5753..e782ad793 100644 --- a/src/sampletones_application/utils/palette/colors/written.py +++ b/src/sampletones_application/utils/palette/colors/written.py @@ -5,7 +5,10 @@ from sampletones_application.utils.palette.colors.base import BaseColor from sampletones_application.utils.palette.colors.literal import LiteralColor from sampletones_application.utils.palette.colors.named import NamedColor -from sampletones_application.utils.palette.reference import PaletteReference, is_reference +from sampletones_application.utils.palette.reference import ( + PaletteReference, + is_reference, +) from sampletones_application.utils.palette.source import PaletteSource from sampletones_shared.utils.color import parse_hex_color @@ -47,7 +50,9 @@ def _written_color(value: object, info: ValidationInfo) -> BaseColor: return value if not isinstance(value, str): - raise ValueError(f"A color is written as a palette reference or a hex literal, got {type(value)}") + raise ValueError( # noqa: TRY004 + f"A color is written as a palette reference or a hex literal, got {type(value)}" + ) text = value.strip() color: BaseColor diff --git a/src/sampletones_application/view_model/shared/export.py b/src/sampletones_application/view_model/shared/export.py index 64aece0e0..dee7cf5b7 100644 --- a/src/sampletones_application/view_model/shared/export.py +++ b/src/sampletones_application/view_model/shared/export.py @@ -78,6 +78,14 @@ def working_visible(self) -> bool: """Whether the turning indicator stands, which is how a stage without an end reads.""" return not self.traveling + @property + def figure_visible(self) -> bool: + """Whether the figure stands beside the turning indicator, which words to state earn. + + A stage with nothing to state leaves the indicator alone, centered on its own width. + """ + return bool(self.figure) + @property def progress_overlay(self) -> str: """The percentage label rendered over the progress bar, derived from the fraction.""" diff --git a/src/sampletones_assets/mark/paths.py b/src/sampletones_assets/mark/paths.py deleted file mode 100644 index 4b9decd97..000000000 --- a/src/sampletones_assets/mark/paths.py +++ /dev/null @@ -1,7 +0,0 @@ -from importlib.resources import files -from pathlib import Path -from typing import Final - -MARK_DIRECTORY: Final[Path] = Path(str(files("sampletones_assets.mark"))) -MARK_PATH: Final[Path] = MARK_DIRECTORY / "mark.yaml" -TEMPLATE_PATH: Final[Path] = MARK_DIRECTORY / "template.svg" diff --git a/src/sampletones_config/README.md b/src/sampletones_config/README.md index 767fb77f6..204682e19 100644 --- a/src/sampletones_config/README.md +++ b/src/sampletones_config/README.md @@ -8,8 +8,8 @@ programmatic role is to be importable so consumers can resolve its directory The schema that validates each file lives in the **consuming** package: - `sampletones_application` — layout, theme, palettes, language, behavior, deployment. -- `sampletones_core` — calibration. -- `sampletones_shared` — the import boundaries and the loader primitives. +- `sampletones_tools` — the import boundaries. +- `sampletones_shared` — the loader primitives. The data package must not import a schema, and a schema package must not inline data. @@ -19,8 +19,7 @@ The data package must not import a schema, and a schema package must not inline |-----------|---------|--------------| | `application/` | Deployment-time environment knobs | `DeploymentConfig` | | `behavior/` | Non-visual runtime behavior | `BehaviorConfig` | -| `boundaries/` | The imports the source tree is held to | `ImportBoundaryRules` | -| `calibration/` | DSP calibration tuning | `CorpusConfig`, `RefereeConfig` | +| `boundaries/` | The imports the source and scripts trees are held to | `ImportBoundaryRules` | | `keybindings/` | The key combinations each named action answers | `ShortcutScheme` | | `lang/` | Interface strings (i18n) | `LanguageManager` | | `layout/` | UI geometry, dimensions, fonts | `LayoutConfig` | @@ -29,4 +28,4 @@ The data package must not import a schema, and a schema package must not inline The rules for where a value belongs, how the directories nest, and how each domain is loaded are prescriptive and documented in -[`docs/development/config-organization.md`](../../docs/development/config-organization.md). +[`docs/development/application/config-organization.md`](../../docs/development/application/config-organization.md). diff --git a/src/sampletones_config/boundaries/graphs.yaml b/src/sampletones_config/boundaries/graphs.yaml index eda8d30e2..593e11c7e 100644 --- a/src/sampletones_config/boundaries/graphs.yaml +++ b/src/sampletones_config/boundaries/graphs.yaml @@ -2,12 +2,12 @@ packages: layers: sampletones_shared: [] sampletones_config: [] - sampletones_assets: [sampletones_shared] - sampletones_synthesis: [sampletones_shared] - sampletones_core: [sampletones_shared, sampletones_synthesis] + sampletones_assets: [] + sampletones_core: [sampletones_shared] sampletones_player: [sampletones_shared, sampletones_core] sampletones_application: [sampletones_shared, sampletones_core, sampletones_player] - sampletones: [sampletones_shared, sampletones_core, sampletones_application] + sampletones_tools: [sampletones_shared, sampletones_core, sampletones_player, sampletones_application] + sampletones: [sampletones_shared, sampletones_core, sampletones_application, sampletones_tools] player: root: sampletones_player @@ -20,8 +20,6 @@ player: compression: [specification, registers] song.py: [clock, registers, compression] builder.py: [song.py, registers, clock, compression] - trace: [song.py, specification] nsf: [song.py, specification, compression, driver] export.py: [builder.py, nsf, driver, compression] driver: [specification] - driver/assembler: [driver, specification] diff --git a/src/sampletones_config/boundaries/standalone.yaml b/src/sampletones_config/boundaries/standalone.yaml new file mode 100644 index 000000000..599690650 --- /dev/null +++ b/src/sampletones_config/boundaries/standalone.yaml @@ -0,0 +1,5 @@ +- pattern: "**/*.py" + reserved: [build, test, tests] + message: >- + a bootstrap script runs on the system interpreter, so it imports the standard library and + the scripts tree alone; code that needs the project environment is a tool, run through uv diff --git a/src/sampletones_config/boundaries/tokens.yaml b/src/sampletones_config/boundaries/tokens.yaml index f144f828a..0eeed6346 100644 --- a/src/sampletones_config/boundaries/tokens.yaml +++ b/src/sampletones_config/boundaries/tokens.yaml @@ -19,3 +19,38 @@ ui/panels must not bind a structural depth theme (TAG_GLOBAL_THEME_PANEL_SURFACE/GROUND); only the layout primitives own depth (TabColumns binds the column, card() binds the card), and a panel binds only semantic themes + +- root: sampletones_application + pattern: "**/*.py" + forbidden: '\bsampletones_tools\b' + message: >- + sampletones_application is shipped code and names no tool; the command line is the one importer of + sampletones_tools + +- root: sampletones_core + pattern: "**/*.py" + forbidden: '\bsampletones_tools\b' + message: >- + sampletones_core is shipped code and names no tool; the command line is the one importer of + sampletones_tools + +- root: sampletones_player + pattern: "**/*.py" + forbidden: '\bsampletones_tools\b' + message: >- + sampletones_player is shipped code and names no tool; the command line is the one importer of + sampletones_tools + +- root: sampletones_shared + pattern: "**/*.py" + forbidden: '\bsampletones_tools\b' + message: >- + sampletones_shared is shipped code and names no tool; the command line is the one importer of + sampletones_tools + +- root: sampletones_assets + pattern: "**/*.py" + forbidden: '\bsampletones_tools\b' + message: >- + sampletones_assets is shipped code and names no tool; the command line is the one importer of + sampletones_tools diff --git a/src/sampletones_config/layout/settings/export.yaml b/src/sampletones_config/layout/settings/export.yaml index 6843f19d3..beb55ad60 100644 --- a/src/sampletones_config/layout/settings/export.yaml +++ b/src/sampletones_config/layout/settings/export.yaml @@ -1,3 +1,6 @@ window: width: 460 height: 120 +indicator: + radius: 1.6 + thickness: 1.5 diff --git a/src/sampletones_core/calibration/__init__.py b/src/sampletones_core/calibration/__init__.py deleted file mode 100644 index e63aed347..000000000 --- a/src/sampletones_core/calibration/__init__.py +++ /dev/null @@ -1,37 +0,0 @@ -from .config.corpus import CorpusConfig -from .config.referee import RefereeConfig -from .corpus.item import CorpusItem -from .corpus.synthesis import build_corpus -from .corpus.writer import write_corpus -from .referee.auditory import MultiResolutionAuditoryReferee -from .referee.factory import build_referees -from .referee.protocol import Referee -from .referee.zimtohrli import ZimtohrliReferee, find_zimtohrli -from .report import write_csv, write_markdown -from .runner import ( - CalibrationRow, - CalibrationVariant, - build_variants, - ensure_library, - evaluate_variants, -) - -__all__ = [ - "CalibrationRow", - "CalibrationVariant", - "CorpusConfig", - "CorpusItem", - "MultiResolutionAuditoryReferee", - "Referee", - "RefereeConfig", - "ZimtohrliReferee", - "build_corpus", - "build_referees", - "build_variants", - "ensure_library", - "evaluate_variants", - "find_zimtohrli", - "write_corpus", - "write_csv", - "write_markdown", -] diff --git a/src/sampletones_core/features/__init__.py b/src/sampletones_core/features/__init__.py index 5b51dd5ac..0a9762813 100644 --- a/src/sampletones_core/features/__init__.py +++ b/src/sampletones_core/features/__init__.py @@ -22,19 +22,19 @@ __all__ = [ "BEND_FEATURES", "CHANNEL_FEATURE_DEFAULTS", + "CHANNEL_GENERATOR_KIND", "FEATURE_DIMENSION_ORDER", "GENERATOR_CHANNEL_KINDS", "GENERATOR_FEATURE_RANGES", - "CHANNEL_GENERATOR_KIND", "RESTING_REFERENCE_PERIOD", "RESTING_REFERENCE_PITCH", "FeatureRange", "channel_reference", "feature_range", "generator_channel", - "speaks_in_periods", "resting_held_features", "resting_reference", + "speaks_in_periods", "supported_features", "supports", "transposed_reference", diff --git a/src/sampletones_core/generators/__init__.py b/src/sampletones_core/generators/__init__.py index 92183c2f6..614cb6130 100644 --- a/src/sampletones_core/generators/__init__.py +++ b/src/sampletones_core/generators/__init__.py @@ -30,11 +30,11 @@ __all__ = [ "CHANNEL_CLASSES", + "CLASS_NAME_TO_GENERATOR_MAP", "GENERATOR_CLASS_MAP", + "GENERATOR_TO_CLASS_NAME_MAP", "GENERATOR_TO_INSTRUCTION_MAP", "INSTRUCTION_TO_GENERATOR_MAP", - "CLASS_NAME_TO_GENERATOR_MAP", - "GENERATOR_TO_CLASS_NAME_MAP", "MIXER_LEVELS", "Generator", "GeneratorClass", diff --git a/src/sampletones_player/trace/__init__.py b/src/sampletones_core/headless/__init__.py similarity index 100% rename from src/sampletones_player/trace/__init__.py rename to src/sampletones_core/headless/__init__.py diff --git a/src/sampletones_core/headless/config.py b/src/sampletones_core/headless/config.py new file mode 100644 index 000000000..e1d807512 --- /dev/null +++ b/src/sampletones_core/headless/config.py @@ -0,0 +1,9 @@ +from pathlib import Path +from typing import Optional + +from sampletones_core.configs import Config + + +def load_config(path: Optional[Path]) -> Config: + """The configuration a headless run uses: the file named, or the saved one with the defaults behind it.""" + return Config.load(path) if path is not None else Config.default() diff --git a/src/sampletones_shared/meta/import_boundary/__init__.py b/src/sampletones_core/headless/conversion/__init__.py similarity index 100% rename from src/sampletones_shared/meta/import_boundary/__init__.py rename to src/sampletones_core/headless/conversion/__init__.py diff --git a/src/sampletones_core/headless/conversion/pairing.py b/src/sampletones_core/headless/conversion/pairing.py new file mode 100644 index 000000000..41b6fbd5f --- /dev/null +++ b/src/sampletones_core/headless/conversion/pairing.py @@ -0,0 +1,21 @@ +from typing import List + +from sampletones_core.headless.conversion.request import ConversionRequest +from sampletones_core.reconstructions.reconstructor.stems.configs.entry import StemEntry + + +def describe_stem(entry: StemEntry) -> str: + """One stem as the pairing names it: its id, the channels it may occupy and the ones it bends.""" + channels = ", ".join(channel.value for channel in entry.settings.channels) + bends = ", ".join(channel.value for channel in entry.settings.bends) + bending = f", bending {bends}" if bends else "" + return f"stem {entry.id} on {channels}{bending}" + + +def pairing_lines(request: ConversionRequest) -> List[str]: + """One line per source naming the stem it plays under, in the order they pair.""" + directory = request.directory + if directory is not None: + return [f"{directory.name}/: every recording under {describe_stem(request.stems.entries[0])}"] + + return [f"{source.name}: {describe_stem(entry)}" for source, entry in zip(request.sources, request.stems.entries)] diff --git a/src/sampletones_core/headless/conversion/request.py b/src/sampletones_core/headless/conversion/request.py new file mode 100644 index 000000000..8317568a2 --- /dev/null +++ b/src/sampletones_core/headless/conversion/request.py @@ -0,0 +1,145 @@ +from functools import cached_property +from pathlib import Path +from typing import List, Optional, Self, Sequence, Tuple + +from pydantic import BaseModel, ConfigDict, Field, model_validator + +from sampletones_core.constants.enums import ( + DEFAULT_CHANNELS, + ChannelName, + bending_channels, + ordered_channels, +) +from sampletones_core.reconstructions.reconstructor.stems.configs.config import ( + StemsConfig, +) +from sampletones_shared.paths.extensions import EXT_FILES_AUDIO, is_audio_file +from sampletones_shared.utils.serialization import load_json +from sampletones_shared.utils.text import listed_items + + +def channels_named(stated: Optional[str]) -> List[ChannelName]: + """The channels a run hands out: the ones named, comma separated, or the usual three. + + Raises: + ValueError: If a name is none of the channels the hardware has. + """ + if stated is None: + return list(DEFAULT_CHANNELS) + + channels: List[ChannelName] = [] + for name in listed_items(stated): + try: + channels.append(ChannelName(name)) + except ValueError as error: + known = ", ".join(channel.value for channel in ChannelName) + raise ValueError(f"Unknown channel {name!r}; the channels are {known}.") from error + + return channels + + +def classic_setup(channels: Sequence[ChannelName]) -> StemsConfig: + """The setup a single-source conversion runs under: one stem over the channels it was given.""" + ordered = ordered_channels(frozenset(channels)) + return StemsConfig.single_entry(ordered, bending_channels(ordered)) + + +def load_stems(path: Path) -> StemsConfig: + """The stems setup a JSON file holds, validated the way the ``.stn`` record is. + + Raises: + ValueError: If no file stands at the path, the file holds anything other than a JSON + mapping, or the mapping is no stems setup. + """ + if not path.is_file(): + raise ValueError(f"No stems file at {path}.") + + loaded = load_json(path) + if not isinstance(loaded, dict): + raise ValueError(f"Stems file {path} must hold a mapping, got {type(loaded).__name__}.") # noqa: TRY004 + + return StemsConfig.model_validate(loaded) + + +class ConversionRequest(BaseModel): + """What a headless conversion is asked for: the sources, the stems setup and where the result goes. + + The i-th source plays under the i-th entry of the setup. One directory stands for every + recording under it, each converted alone under the setup's one stem, into the + configuration's reconstructions directory. + + Attributes: + sources: The recordings, or one directory of them. + stems: The setup handing the channels out. + output_path: The file the reconstruction of the recordings is written to, or ``None`` + for the configuration's own directory. + """ + + model_config = ConfigDict(frozen=True) + + sources: Tuple[Path, ...] = Field(min_length=1) + stems: StemsConfig + output_path: Optional[Path] + + @cached_property + def directory(self) -> Optional[Path]: + """The one directory the request converts file by file, or ``None`` for recordings. + + The sources are classified once, when the request is validated, and every later reading + answers from that. + """ + if len(self.sources) == 1 and self.sources[0].is_dir(): + return self.sources[0] + + return None + + @model_validator(mode="after") + def _sources_are_recordings_or_one_directory(self) -> Self: + """Raises: + ValueError: If a directory stands among several sources, or a source names nothing or + a file other than a recording. + """ + if self.directory is not None: + return self + + for source in self.sources: + if source.is_dir(): + raise ValueError("Sources are recordings, or one directory alone.") + + if not source.exists(): + raise ValueError(f"No file at {source}.") + + if not is_audio_file(source): + raise ValueError(f"{source} is no recording; a source is a {', '.join(EXT_FILES_AUDIO)} file.") + + return self + + @model_validator(mode="after") + def _entries_pair_with_sources(self) -> Self: + """Raises: + ValueError: If the setup holds a different number of stems than there are sources. + """ + entries = len(self.stems.entries) + if self.directory is not None and entries != 1: + raise ValueError(f"A directory is converted file by file under one stem; the setup holds {entries}.") + + if self.directory is None and entries != len(self.sources): + raise ValueError( + f"{len(self.sources)} sources for {entries} stems; a setup pairs one stem with each source, in order." + ) + + return self + + @model_validator(mode="after") + def _output_names_the_one_file(self) -> Self: + """Raises: + ValueError: If an output path is given for a directory, whose reconstructions land in + the configuration's directory. + """ + if self.directory is not None and self.output_path is not None: + raise ValueError( + "A directory's reconstructions land in the configuration's reconstructions directory; " + "an output path names the one file recordings are mixed into." + ) + + return self diff --git a/src/sampletones_core/scripts/reconstruction.py b/src/sampletones_core/headless/conversion/runners.py similarity index 51% rename from src/sampletones_core/scripts/reconstruction.py rename to src/sampletones_core/headless/conversion/runners.py index c61d73725..5ec8ced89 100644 --- a/src/sampletones_core/scripts/reconstruction.py +++ b/src/sampletones_core/headless/conversion/runners.py @@ -1,14 +1,11 @@ from pathlib import Path -from typing import Final, Optional, Sequence, Tuple +from typing import Final, Optional, Tuple from tqdm import tqdm from sampletones_core.configs import Config -from sampletones_core.constants.enums import ( - ChannelName, - bending_channels, - ordered_channels, -) +from sampletones_core.headless.conversion.request import ConversionRequest +from sampletones_core.headless.library import generate_library from sampletones_core.library import InstructionLibrary from sampletones_core.parallelization import TaskProgress, TaskStatus from sampletones_core.reconstructions import Reconstructor @@ -18,38 +15,51 @@ ReconstructionConverter, reconstruct_job, ) -from sampletones_core.reconstructions.converter.paths import get_output_path +from sampletones_core.reconstructions.converter.paths import group_output_path from sampletones_core.reconstructions.progress import ReconstructionProgress from sampletones_core.reconstructions.reconstructor.stems.configs.config import StemsConfig -from sampletones_core.scripts.library import generate_library from sampletones_shared.logger import logger, null_logger BAR_STEPS: Final[int] = 1000 -def reconstruct_file( - input_path: Path, +def reconstruct(request: ConversionRequest, config: Config) -> None: + """Builds what the request asks for: one reconstruction of the recordings, or one per file of the directory.""" + directory = request.directory + if directory is not None: + reconstruct_directory(directory, config, request.stems) + return + + reconstruct_sources(request.sources, config, request.stems, request.output_path) + + +def reconstruct_sources( + sources: Tuple[Path, ...], config: Config, - channels: Sequence[ChannelName], - output_path: Optional[Path] = None, + stems: StemsConfig, + output_path: Optional[Path], ) -> None: + """Mixes the recordings into one reconstruction and writes it, showing the progress as a bar. + + A file already standing at the output path is kept, and the run says so. + + Args: + sources: The recordings, one per stem of the setup. + config: The configuration selecting the library and the matching settings. + stems: The setup handing the channels out. + output_path: The file written, or ``None`` for the configuration's own directory. + """ if output_path is None: - output_path = get_output_path(config, input_path, frozenset(channels)) + output_path = group_output_path(config, sources, stems.covered_channels) if output_path.exists(): - logger.info(f"Reconstructing file {input_path} exists, skipping") + logger.info(f"Reconstruction {output_path} exists, skipping") return - if input_path.is_dir(): - raise IsADirectoryError(f"Expected a file path, got directory path: {input_path}") - - logger.info(f"Starting reconstruction for file {input_path}") - job = ConversionJob( - sources=(input_path,), - stems=_classic_setup(channels), - output_path=output_path, - ) - progress_bar = tqdm(total=BAR_STEPS, desc=f"Reconstructing {input_path.name}", unit="step") + names = ", ".join(source.name for source in sources) + logger.info(f"Starting reconstruction of {names}") + job = ConversionJob(sources=sources, stems=stems, output_path=output_path) + progress_bar = tqdm(total=BAR_STEPS, desc=f"Reconstructing {output_path.stem}", unit="step") def on_progress(progress: ReconstructionProgress) -> bool: progress_bar.set_postfix_str(progress.stage) @@ -57,7 +67,7 @@ def on_progress(progress: ReconstructionProgress) -> bool: return True try: - reconstruct_job((Reconstructor(config, frozenset(channels)), job, on_progress)) + reconstruct_job((Reconstructor(config, stems.covered_channels), job, on_progress)) finally: progress_bar.close() @@ -65,30 +75,33 @@ def on_progress(progress: ReconstructionProgress) -> bool: def reconstruct_directory( - input_path: Path, + directory: Path, config: Config, - channels: Sequence[ChannelName], - output_path: Optional[Path] = None, + stems: StemsConfig, ) -> None: - if output_path is None: - output_path = get_output_path(config, input_path, frozenset(channels)) + """Reconstructs every recording under the directory, each alone under the setup's stem. - if not input_path.is_dir(): - raise NotADirectoryError(f"Expected a directory path, got file path: {input_path}") + The library the configuration names is generated first where it is missing. The results + mirror the directory's tree inside the configuration's reconstructions directory. + Args: + directory: The directory of recordings. + config: The configuration selecting the library and the matching settings. + stems: The one-stem setup every recording is converted under. + """ library = InstructionLibrary.from_config(config) if not library.exists(config): logger.warning("Library does not exist for the given configuration, generating a new library") generate_library(config) - progress_bar = tqdm(total=0, desc=f"Reconstructing {input_path.name}", unit="file") + progress_bar = tqdm(total=0, desc=f"Reconstructing {directory.name}", unit="file") def on_start() -> None: progress_bar.disable = False - logger.info(f"Starting reconstruction for directory {input_path}") + logger.info(f"Starting reconstruction for directory {directory}") - def on_completed(_written: Tuple[Path, ...]) -> None: - logger.info(f"Reconstruction directory saved to {output_path}") + def on_completed(written: Tuple[Path, ...]) -> None: + logger.info(f"Reconstructed {len(written)} files from {directory}") progress_bar.close() def on_progress( @@ -106,7 +119,7 @@ def on_progress( progress_bar.update(delta) if task_progress.current_item: - progress_bar.set_description(f"{input_path.name}: {task_progress.current_item}") + progress_bar.set_description(f"{directory.name}: {task_progress.current_item}") if task_status in ( TaskStatus.COMPLETED, @@ -124,7 +137,7 @@ def on_error(_exception: Exception) -> None: converter = ReconstructionConverter( config, - DirectoryConversion(directory=input_path, stems=_classic_setup(channels)), + DirectoryConversion(directory=directory, stems=stems), logger=null_logger, ) @@ -143,9 +156,3 @@ def on_error(_exception: Exception) -> None: logger.info("Reconstruction interrupted by user") finally: progress_bar.close() - - -def _classic_setup(channels: Sequence[ChannelName]) -> StemsConfig: - """The setup a single-source conversion runs under: one stem over the channels it was given.""" - ordered = ordered_channels(frozenset(channels)) - return StemsConfig.single_entry(ordered, bending_channels(ordered)) diff --git a/src/sampletones_core/scripts/library.py b/src/sampletones_core/headless/library.py similarity index 100% rename from src/sampletones_core/scripts/library.py rename to src/sampletones_core/headless/library.py diff --git a/src/sampletones_core/parallelization/channel/process.py b/src/sampletones_core/parallelization/channel/process.py index 340745d83..bf4426466 100644 --- a/src/sampletones_core/parallelization/channel/process.py +++ b/src/sampletones_core/parallelization/channel/process.py @@ -54,7 +54,7 @@ class ProcessProgressChannel: def __init__(self) -> None: self._manager: SyncManager = multiprocessing.get_context(SPAWN_CONTEXT).Manager() - self._reports: "queue.Queue[TaskReport]" = self._manager.Queue() + self._reports: queue.Queue[TaskReport] = self._manager.Queue() self._withdrawn: threading.Event = self._manager.Event() def reporter(self, index: int) -> StepReporter: diff --git a/src/sampletones_core/parallelization/progress.py b/src/sampletones_core/parallelization/progress.py index 19bf09379..6a1a9a443 100644 --- a/src/sampletones_core/parallelization/progress.py +++ b/src/sampletones_core/parallelization/progress.py @@ -1,6 +1,6 @@ from collections import deque from time import monotonic -from typing import Deque, Final, Optional, Tuple, Union +from typing import Deque, Final, Optional, Tuple ESTIMATION_MEASUREMENTS_SAMPLES: Final[float] = 0.05 @@ -15,7 +15,7 @@ class ETAEstimator: def __init__( self, - total: Union[int, float], + total: float, ems: float = ESTIMATION_MEASUREMENTS_SAMPLES, ) -> None: self._total = total @@ -23,7 +23,7 @@ def __init__( self._samples_window: Deque[Tuple[float, float]] = deque(maxlen=self._ems) self._processed_items: float = 0.0 - def update(self, completed_items: Union[int, float]) -> Optional[float]: + def update(self, completed_items: float) -> Optional[float]: now = monotonic() self._processed_items = completed_items self._samples_window.append((now, completed_items)) diff --git a/src/sampletones_core/reconstructions/converter/paths/__init__.py b/src/sampletones_core/reconstructions/converter/paths/__init__.py index 700793310..515e92e0d 100644 --- a/src/sampletones_core/reconstructions/converter/paths/__init__.py +++ b/src/sampletones_core/reconstructions/converter/paths/__init__.py @@ -8,7 +8,6 @@ get_output_path, get_relative_path, group_output_path, - is_audio_file, walk_audio_files, walk_entries, ) @@ -21,7 +20,6 @@ "get_output_path", "get_relative_path", "group_output_path", - "is_audio_file", "walk_audio_files", "walk_entries", ] diff --git a/src/sampletones_core/reconstructions/converter/paths/utils.py b/src/sampletones_core/reconstructions/converter/paths/utils.py index f974267b7..e6b70cb00 100644 --- a/src/sampletones_core/reconstructions/converter/paths/utils.py +++ b/src/sampletones_core/reconstructions/converter/paths/utils.py @@ -10,6 +10,7 @@ from sampletones_shared.paths.extensions import ( EXT_FILE_RECONSTRUCTION, EXT_FILES_AUDIO, + is_audio_file, ) from sampletones_shared.utils.system.paths import to_path @@ -94,11 +95,6 @@ def walk_entries(input_directory: Path) -> Iterator[Path]: return input_directory.rglob("*") -def is_audio_file(path: Path, extensions: Tuple[str, ...] = EXT_FILES_AUDIO) -> bool: - """Whether a path names a recording a run converts.""" - return path.is_file() and path.suffix.lower() in extensions - - def walk_audio_files( input_directory: Path, extensions: Tuple[str, ...] = EXT_FILES_AUDIO, diff --git a/src/sampletones_player/compression/budget.py b/src/sampletones_player/compression/budget.py new file mode 100644 index 000000000..bc2fbb87f --- /dev/null +++ b/src/sampletones_player/compression/budget.py @@ -0,0 +1,61 @@ +from typing import Final, List, Sequence, Tuple + +from pydantic import BaseModel, ConfigDict, Field + + +class SearchBudget(BaseModel): + """How much work the search spends on the phrases a song's own planes repeat. + + A round gathers candidate windows from what the parse still spells out, ranks them, and + confirms the best few by parsing the song again with each one added. Each figure here + bounds one of those steps, so the budget is what an export trades against the bytes the + search earns. + + Attributes: + candidate_entries: The candidate windows one round gathers over every plane together, + shared among the planes by what each has to offer. + rounds: The rounds the search runs, each adding at most one phrase. + confirmed_candidates: The best-ranked candidates a round parses the song again with. + """ + + model_config = ConfigDict(extra="forbid", frozen=True) + + candidate_entries: int = Field(ge=1) + rounds: int = Field(ge=0) + confirmed_candidates: int = Field(ge=1) + + +DEFAULT_SEARCH_BUDGET: Final[SearchBudget] = SearchBudget( + candidate_entries=200_000, + rounds=64, + confirmed_candidates=3, +) + + +def shares( + demands: Sequence[int], + total: int, +) -> Tuple[int, ...]: + """Divides ``total`` among claimants, meeting small demands in full and splitting what + remains evenly among the larger ones. + + Claimants are served from the smallest demand upward, each taking the lesser of its demand + and an even share of what is left, so a claimant asking for little leaves its surplus to the + rest and a claimant asking for much is held to the same share as its peers. + + Args: + demands: What each claimant would take, given the room. + total: What there is to share. + + Returns: + Tuple[int, ...]: What each claimant receives, in the order the demands were given. + """ + granted: List[int] = [0] * len(demands) + remaining = total + pending = len(demands) + for claimant in sorted(range(len(demands)), key=lambda index: demands[index]): + granted[claimant] = min(demands[claimant], remaining // pending) + remaining -= granted[claimant] + pending -= 1 + + return tuple(granted) diff --git a/src/sampletones_player/compression/encode.py b/src/sampletones_player/compression/encode.py index 1cf52ccfe..708ebdc2f 100644 --- a/src/sampletones_player/compression/encode.py +++ b/src/sampletones_player/compression/encode.py @@ -2,6 +2,7 @@ from typing import Dict, Final, FrozenSet, Iterable, Sequence, Tuple from sampletones_player.compression.admit import admit_seeds +from sampletones_player.compression.budget import DEFAULT_SEARCH_BUDGET, SearchBudget from sampletones_player.compression.compressed import CompressedPlanes from sampletones_player.compression.dictionary.phrase import Phrase from sampletones_player.compression.dictionary.prune import prune @@ -119,6 +120,7 @@ def encode_planes( *, options: CodecOptions, boundaries: FrozenSet[int], + budget: SearchBudget = DEFAULT_SEARCH_BUDGET, report: CodecReporter = silent_reporter, ) -> CompressedPlanes: """Compresses a song's planes into the dictionary and streams the driver reads. @@ -135,6 +137,7 @@ def encode_planes( seeds: The phrases the song's instruments offer. options: Which of the codec's layers the encoding is built from. boundaries: The ticks a token starts on, beyond the first tick of the song. + budget: How much work the search spends beyond the phrases the instruments seed. report: Hears what the run holds each time it looks up, and answers whether it goes on. Returns: @@ -170,6 +173,7 @@ def encode_planes( options, entries, monitor, + budget, ) table, parses = _settle( diff --git a/src/sampletones_player/compression/search.py b/src/sampletones_player/compression/search.py index 830e350cf..5ea60b2a9 100644 --- a/src/sampletones_player/compression/search.py +++ b/src/sampletones_player/compression/search.py @@ -1,5 +1,6 @@ -from typing import Dict, Final, FrozenSet, List, NamedTuple, Sequence +from typing import Callable, Dict, Final, FrozenSet, Iterator, List, NamedTuple, Sequence, Tuple +from sampletones_player.compression.budget import SearchBudget, shares from sampletones_player.compression.dictionary.phrase import Phrase, phrase_entry_size from sampletones_player.compression.dictionary.table import PhraseTable, phrase_table from sampletones_player.compression.matches.cache import MatchCache @@ -14,9 +15,6 @@ MIN_CANDIDATE_LENGTH: Final[int] = 3 MAX_CANDIDATE_LENGTH: Final[int] = 48 -MAX_CANDIDATE_ENTRIES: Final[int] = 200_000 -MAX_SEARCH_ROUNDS: Final[int] = 64 -CONFIRMED_CANDIDATES: Final[int] = 3 MIN_OCCURRENCES: Final[int] = 2 UNSHIFTED_OCCURRENCE_TRANSPOSE: Final[int] = 0 SHIFTED_OCCURRENCE_TRANSPOSE: Final[int] = 1 @@ -49,28 +47,58 @@ def _residue_spans(parse: Parse) -> List[_Span]: return spans +def _windows(parse: Parse) -> Iterator[Tuple[int, int]]: + """Each position the parse still spells out, beside the longest candidate starting there.""" + for span in _residue_spans(parse): + for position in range(span.start, span.end): + yield position, min(MAX_CANDIDATE_LENGTH, span.end - position) + + +def _entries(longest: int) -> int: + return max(0, longest - MIN_CANDIDATE_LENGTH + 1) + + +def _demand(parse: Parse) -> int: + return sum(_entries(longest) for _, longest in _windows(parse)) + + +def _gather_plane( + plane: int, + index: PlaneIndex, + parse: Parse, + allowance: int, + gather: Callable[[bytes, List[_Occurrence]], List[_Occurrence]], +) -> None: + differences = index.differences + entries = 0 + for position, longest in _windows(parse): + if entries >= allowance: + return + + occurrence = _Occurrence(plane=plane, position=position) + for length in range(MIN_CANDIDATE_LENGTH, longest + 1): + gather(differences[position : position + length - 1], []).append(occurrence) + + entries += _entries(longest) + + def _candidates( indices: Sequence[PlaneIndex], parses: Sequence[Parse], monitor: CodecMonitor, + entries: int, ) -> Dict[bytes, List[_Occurrence]]: + """Every candidate shape the planes still spell out, beside where each occurs. + + The entries a round gathers are shared among the planes by what each has to offer, so a + plane that spells out little is read whole and a dense one is read as far as its share + reaches, and every plane is heard from. + """ found: Dict[bytes, List[_Occurrence]] = {} - gather = found.setdefault - entries = 0 - for plane, (index, parse) in enumerate(zip(indices, parses)): + allowances = shares([_demand(parse) for parse in parses], entries) + for plane, (index, parse, allowance) in enumerate(zip(indices, parses, allowances)): monitor.poll() - differences = index.differences - for span in _residue_spans(parse): - if entries > MAX_CANDIDATE_ENTRIES: - return found - - for position in range(span.start, span.end): - longest = min(MAX_CANDIDATE_LENGTH, span.end - position) - occurrence = _Occurrence(plane=plane, position=position) - for length in range(MIN_CANDIDATE_LENGTH, longest + 1): - gather(differences[position : position + length - 1], []).append(occurrence) - - entries += max(0, longest - MIN_CANDIDATE_LENGTH + 1) + _gather_plane(plane, index, parse, allowance, found.setdefault) return found @@ -112,9 +140,10 @@ def _ranked( parses: Sequence[Parse], phrase_id: int, monitor: CodecMonitor, + budget: SearchBudget, ) -> List[_Candidate]: ranked: List[_Candidate] = [] - for key, occurrences in _candidates(indices, parses, monitor).items(): + for key, occurrences in _candidates(indices, parses, monitor, budget.candidate_entries).items(): if len(occurrences) < MIN_OCCURRENCES: continue @@ -130,7 +159,7 @@ def _ranked( ranked.append(_Candidate(gain=gain, body=body)) ranked.sort(key=lambda candidate: (-candidate.gain, candidate.body)) - return ranked[:CONFIRMED_CANDIDATES] + return ranked[: budget.confirmed_candidates] def _total(table: PhraseTable, parses: Sequence[Parse]) -> int: @@ -143,6 +172,7 @@ def search_phrases( options: CodecOptions, boundaries: FrozenSet[int], monitor: CodecMonitor, + budget: SearchBudget, ) -> PhraseTable: """Fills the dictionary with the phrases the song's own planes repeat. @@ -161,6 +191,7 @@ def search_phrases( options: Which of the codec's layers the encoding is built from. boundaries: The ticks a token starts on. monitor: Carries the run's reckoning of itself onward. + budget: How many candidates a round gathers and confirms, and how many rounds run. Returns: PhraseTable: The seeded phrases alongside the ones the search earned. @@ -172,12 +203,12 @@ def search_phrases( parses = parse_planes(cache, table, options, boundaries, monitor) total = _total(table, parses) monitor.reached(len(table), total) - for _ in range(MAX_SEARCH_ROUNDS): + for _ in range(budget.rounds): if len(table) == MAX_PHRASE_IDS: return table settled = False - for candidate in _ranked(indices, parses, len(table), monitor): + for candidate in _ranked(indices, parses, len(table), monitor, budget): offered = Phrase(body=candidate.body) enlarged = phrase_table(table.phrases + (offered,)) trial = parse_planes_offered( diff --git a/src/sampletones_player/compression/song.py b/src/sampletones_player/compression/song.py index 7a38582a7..df0f34741 100644 --- a/src/sampletones_player/compression/song.py +++ b/src/sampletones_player/compression/song.py @@ -1,5 +1,6 @@ from typing import FrozenSet, Optional, Sequence +from sampletones_player.compression.budget import DEFAULT_SEARCH_BUDGET, SearchBudget from sampletones_player.compression.compressed import CompressedPlanes from sampletones_player.compression.decode import decode_planes from sampletones_player.compression.dictionary.phrase import Phrase @@ -26,6 +27,7 @@ def compress_song( *, seeds: Sequence[Phrase], loop_tick: Optional[int] = None, + budget: SearchBudget = DEFAULT_SEARCH_BUDGET, report: CodecReporter = silent_reporter, ) -> CompressedPlanes: """Compresses a song's four register streams into the dictionary and streams a file carries. @@ -39,6 +41,7 @@ def compress_song( pitches: The timer each pitch sounds at, which is what turns a timer into an index. seeds: The phrases the song's instruments offer the dictionary. loop_tick: The tick the song returns to once it ends, or ``None`` where it stops there. + budget: How much work the search spends beyond the phrases the instruments seed. report: Hears what the codec holds each time it looks up, and answers whether it goes on. Returns: @@ -53,6 +56,7 @@ def compress_song( seeds, options=EVERY_LAYER, boundaries=_entries(loop_tick), + budget=budget, report=report, ) diff --git a/src/sampletones_player/driver/assembler/layout.py b/src/sampletones_player/driver/assembler/layout.py deleted file mode 100644 index 6e312bbb7..000000000 --- a/src/sampletones_player/driver/assembler/layout.py +++ /dev/null @@ -1,14 +0,0 @@ -from pathlib import Path -from typing import Final, Tuple - -from sampletones_player.specification.driver import DRIVER_BINARY_DIRECTORY -from sampletones_shared.paths.source import SOURCE_ROOT - -DRIVER_DIRECTORY: Final[Path] = SOURCE_ROOT / "sampletones_player" / "driver" -ASSEMBLY_DIRECTORY: Final[Path] = DRIVER_DIRECTORY / "assembly" -INCLUDE_DIRECTORY: Final[Path] = ASSEMBLY_DIRECTORY / "include" -SOURCE_DIRECTORY: Final[Path] = ASSEMBLY_DIRECTORY / "source" -LINKER_CONFIGURATION: Final[Path] = ASSEMBLY_DIRECTORY / "nsf.cfg" -BINARY_DIRECTORY: Final[Path] = DRIVER_DIRECTORY / DRIVER_BINARY_DIRECTORY - -SOURCE_NAMES: Final[Tuple[str, ...]] = ("driver.s", "clock.s", "channels.s") diff --git a/src/sampletones_player/driver/image.py b/src/sampletones_player/driver/image.py index f16292b57..79cbe1d22 100644 --- a/src/sampletones_player/driver/image.py +++ b/src/sampletones_player/driver/image.py @@ -16,7 +16,7 @@ class DriverImage(BaseModel): """The assembled player, paired with the addresses it lays out. - The 6502 program that plays a song is written in assembly, built once by ``make player`` and + The 6502 program that plays a song is written in assembly, built once by ``sampletones driver`` and committed beside its sources, so exporting an NSF needs no assembler. Pairing the bytes with their addresses is what lets the exporter name the routines in an NSF header and place the song where the driver looks for it. diff --git a/src/sampletones_synthesis/py.typed b/src/sampletones_player/py.typed similarity index 100% rename from src/sampletones_synthesis/py.typed rename to src/sampletones_player/py.typed diff --git a/src/sampletones_shared/command.py b/src/sampletones_shared/command.py new file mode 100644 index 000000000..a9490fc22 --- /dev/null +++ b/src/sampletones_shared/command.py @@ -0,0 +1,23 @@ +from argparse import ArgumentParser, Namespace +from dataclasses import dataclass +from typing import Callable + + +@dataclass(frozen=True) +class Command: + """One named operation the ``sampletones`` entry runs: its options and what it does. + + A command module declares one of these at import time and keeps its implementation behind + ``run``, so listing the commands loads nothing a command needs to do its work. + + Attributes: + name: The word that selects the command on the command line. + help: One line saying what the command does, shown in the command list. + configure: Adds the command's arguments to the parser it is given. + run: Performs the command over the parsed arguments and answers with the exit status. + """ + + name: str + help: str + configure: Callable[[ArgumentParser], None] + run: Callable[[Namespace], int] diff --git a/src/sampletones_shared/options.py b/src/sampletones_shared/options.py new file mode 100644 index 000000000..9de454668 --- /dev/null +++ b/src/sampletones_shared/options.py @@ -0,0 +1,10 @@ +from argparse import ArgumentParser +from pathlib import Path +from typing import Final + +CONFIG_HELP: Final[str] = "a configuration .json file; without it, the saved configuration or the built-in defaults" + + +def add_config_option(parser: ArgumentParser) -> None: + """Adds the ``--config`` option the commands reading a configuration share.""" + parser.add_argument("--config", "-c", type=Path, default=None, help=CONFIG_HELP) diff --git a/src/sampletones_shared/paths/extensions.py b/src/sampletones_shared/paths/extensions.py index 783e3e5d2..7e8f5642b 100644 --- a/src/sampletones_shared/paths/extensions.py +++ b/src/sampletones_shared/paths/extensions.py @@ -1,3 +1,4 @@ +from pathlib import Path from typing import Final, Tuple EXT_FILE_JSON: Final[str] = ".json" @@ -23,3 +24,8 @@ EXT_FILE_AIFF, EXT_FILE_AU, ) + + +def is_audio_file(path: Path, extensions: Tuple[str, ...] = EXT_FILES_AUDIO) -> bool: + """Whether a path names a recording a run converts.""" + return path.is_file() and path.suffix.lower() in extensions diff --git a/src/sampletones_shared/paths/package.py b/src/sampletones_shared/paths/package.py new file mode 100644 index 000000000..6b967884f --- /dev/null +++ b/src/sampletones_shared/paths/package.py @@ -0,0 +1,27 @@ +from importlib.util import find_spec +from pathlib import Path + + +def package_directory(package: str) -> Path: + """The directory an import package's files lie in, its data files among them. + + The import system's own record of where the package lives places it, which holds in a + checkout, in an installed copy and in a bundle. PyInstaller unpacks a package's collected data + into a directory of the package's name, which a data package reaches as a namespace package, + so the location is read from the spec whatever kind of package answers. + + Args: + package: The package's dotted name. + + Returns: + Path: The directory. + + Raises: + FileNotFoundError: If no package of that name is importable. + ModuleNotFoundError: If a package the dotted name passes through is absent. + """ + spec = find_spec(package) + if spec is None or spec.submodule_search_locations is None: + raise FileNotFoundError(f"No package {package} is importable to place its directory by") + + return Path(next(iter(spec.submodule_search_locations))) diff --git a/src/sampletones_shared/paths/resources.py b/src/sampletones_shared/paths/resources.py index 9da9292dd..2e415f2ff 100644 --- a/src/sampletones_shared/paths/resources.py +++ b/src/sampletones_shared/paths/resources.py @@ -1,15 +1,9 @@ -import sys -from importlib.resources import files from pathlib import Path -from typing import Final, Optional +from typing import Final -_BUNDLE_ROOT: Final[Optional[str]] = getattr(sys, "_MEIPASS", None) +from sampletones_shared.paths.package import package_directory -CONFIG_DIRECTORY: Final[Path] = ( - Path(_BUNDLE_ROOT) / "config" if _BUNDLE_ROOT is not None else Path(str(files("sampletones_config"))) -) - -ASSETS_DIRECTORY: Final[str] = "assets" +CONFIG_DIRECTORY: Final[Path] = package_directory("sampletones_config") ICON_DIRECTORY: Final[str] = "icons" ICON_WIN_FILENAME: Final[str] = "sampletones.ico" diff --git a/src/sampletones_shared/utils/tables.py b/src/sampletones_shared/utils/tables.py new file mode 100644 index 000000000..76370119c --- /dev/null +++ b/src/sampletones_shared/utils/tables.py @@ -0,0 +1,59 @@ +import csv +from pathlib import Path +from typing import Final, List, Self, Sequence, Tuple + +from pydantic import BaseModel, ConfigDict, model_validator + +MARKDOWN_RULE: Final[str] = "---" +MARKDOWN_SEPARATOR: Final[str] = " | " + + +class Table(BaseModel): + """A report's table: a header and rows of cells already written as text. + + A report builds a table once and writes it both ways, as CSV another tool reads and as the + lines of a Markdown document a person reads, so the two copies carry the same cells. + + Attributes: + columns: The header row. + rows: The rows, each with one cell per column. + """ + + model_config = ConfigDict(frozen=True) + + columns: Tuple[str, ...] + rows: Tuple[Tuple[str, ...], ...] + + @model_validator(mode="after") + def _rows_fill_the_columns(self) -> Self: + """Raises: + ValueError: If a row holds a different number of cells than there are columns. + """ + for index, row in enumerate(self.rows): + if len(row) != len(self.columns): + raise ValueError(f"Row {index} holds {len(row)} cells for {len(self.columns)} columns.") + + return self + + def write_csv(self, path: Path) -> None: + """Writes the table as UTF-8 CSV, the header first. + + Args: + path: Where the table is written. + """ + with path.open("w", encoding="utf-8", newline="") as handle: + writer = csv.writer(handle) + writer.writerow(self.columns) + writer.writerows(self.rows) + + def markdown_lines(self) -> List[str]: + """The table as the lines of a Markdown document: the header, the rule and one line per row.""" + return [ + self._markdown_line(self.columns), + "|" + "|".join(MARKDOWN_RULE for _ in self.columns) + "|", + *(self._markdown_line(row) for row in self.rows), + ] + + @staticmethod + def _markdown_line(cells: Sequence[str]) -> str: + return f"| {MARKDOWN_SEPARATOR.join(cells)} |" diff --git a/src/sampletones_shared/utils/text.py b/src/sampletones_shared/utils/text.py index 1abc9f9e8..d1954b5bc 100644 --- a/src/sampletones_shared/utils/text.py +++ b/src/sampletones_shared/utils/text.py @@ -1,9 +1,10 @@ import re -from typing import Final, Tuple, TypeAlias +from typing import Final, List, Tuple, TypeAlias NaturalSortKey: TypeAlias = Tuple[Tuple[int, str], ...] _DIGIT_RUN_PATTERN: Final[re.Pattern[str]] = re.compile(r"(\d+)") +LIST_SEPARATOR: Final[str] = "," def natural_sort_key(text: str) -> NaturalSortKey: @@ -30,3 +31,23 @@ def natural_sort_key(text: str) -> NaturalSortKey: (int(part), "") if part.isdecimal() else (0, part.casefold()) for part in _DIGIT_RUN_PATTERN.split(text) ) return tokens + ((0, text),) + + +def listed_items(stated: str) -> List[str]: + """ + Reads a comma-separated list the way a command line states one. + + Each item keeps its own words with the spaces around it stripped, and an empty item, as a + trailing comma leaves, is dropped. + + Args: + stated: The list as written. + + Returns: + The items, in the order written. + + Examples: + >>> listed_items("pulse1, triangle,,noise,") + ['pulse1', 'triangle', 'noise'] + """ + return [item.strip() for item in stated.split(LIST_SEPARATOR) if item.strip()] diff --git a/src/sampletones_shared/utils/validation.py b/src/sampletones_shared/utils/validation.py index f4d15912f..48cedf11d 100644 --- a/src/sampletones_shared/utils/validation.py +++ b/src/sampletones_shared/utils/validation.py @@ -3,12 +3,14 @@ import copy from collections.abc import Mapping from dataclasses import dataclass -from typing import Any, Generic, List, Optional, Tuple, Type, TypeVar, Union +from typing import Any, Final, Generic, List, Optional, Tuple, Type, TypeVar, Union from pydantic import BaseModel, ValidationError +from pydantic_core import ErrorDetails ModelTypeT = TypeVar("ModelTypeT", bound=BaseModel) Location = Tuple[Union[str, int], ...] +VALUE_ERROR: Final[str] = "value_error" @dataclass(frozen=True) @@ -48,6 +50,35 @@ def flatten_location(location: Location) -> str: return "".join(parts) +def describe_failure(error: ValueError) -> str: + """ + Renders a refused value the way a command line reports it, one line per problem. + + A reason a validator raised keeps its own words, and a broken constraint is named by the + field it binds, so a person reads what to change and where. + + Args: + error: The failure, a Pydantic validation error or a plain ``ValueError``. + + Returns: + str: The reasons, one per line. + """ + match error: + case ValidationError(): + return "\n".join(_describe_detail(detail) for detail in error.errors()) + case _: + return str(error) + + +def _describe_detail(detail: ErrorDetails) -> str: + context = detail.get("ctx") + if detail["type"] == VALUE_ERROR and context is not None: + return str(context["error"]) + + location = flatten_location(tuple(detail["loc"])) + return f"{location}: {detail['msg']}" if location else detail["msg"] + + def validate_with_recovery( model_class: Type[ModelTypeT], data: Mapping[str, Any], @@ -122,7 +153,7 @@ def _existing_prefix(container: Any, location: Location) -> Location: node: Any = container length = 0 for index, key in enumerate(location): - if isinstance(node, Mapping) and key in node: + if isinstance(node, Mapping) and key in node: # noqa: SIM114 node = node[key] elif isinstance(node, list) and isinstance(key, int) and -len(node) <= key < len(node): node = node[key] diff --git a/src/sampletones_shared/meta/import_boundary/configs/__init__.py b/src/sampletones_tools/__init__.py similarity index 100% rename from src/sampletones_shared/meta/import_boundary/configs/__init__.py rename to src/sampletones_tools/__init__.py diff --git a/src/sampletones_shared/meta/source/__init__.py b/src/sampletones_tools/assets/__init__.py similarity index 100% rename from src/sampletones_shared/meta/source/__init__.py rename to src/sampletones_tools/assets/__init__.py diff --git a/src/sampletones_tools/assets/command.py b/src/sampletones_tools/assets/command.py new file mode 100644 index 000000000..85c9cffc0 --- /dev/null +++ b/src/sampletones_tools/assets/command.py @@ -0,0 +1,54 @@ +from argparse import ArgumentParser, Namespace +from dataclasses import dataclass +from pathlib import Path +from typing import Final, Optional + +from sampletones_shared.command import Command + +NAME: Final[str] = "icons" +HELP: Final[str] = "write the application icon suite from the mark" +OUTPUT_HELP: Final[str] = "the directory receiving the icon files; without it, the icons the package ships" + + +@dataclass(frozen=True) +class IconsArguments: + """What an icon suite run is given: where the files go, if anywhere but the package.""" + + output: Optional[Path] + + +def configure(parser: ArgumentParser) -> None: + parser.add_argument("--output", "-o", type=Path, default=None, help=OUTPUT_HELP) + + +def run(arguments: Namespace) -> int: + """Writes the icon suite and reports each file produced. + + The suite is rasterized with Pillow, which the project environment carries, so the command + runs from a checkout wherever it writes. + + Raises: + SystemExit: If the run happens outside a checkout. + """ + given = IconsArguments(output=arguments.output) + + from sampletones_tools.checkout import require_checkout + + require_checkout(NAME) + + from sampletones_tools.assets.mark.specification import Mark + from sampletones_tools.assets.mark.suite import write_icon_suite + from sampletones_tools.assets.paths import ICONS_DIRECTORY + + for path in write_icon_suite(given.output if given.output is not None else ICONS_DIRECTORY, Mark.load()): + print(f"Wrote {path}") + + return 0 + + +ICONS: Final[Command] = Command( + name=NAME, + help=HELP, + configure=configure, + run=run, +) diff --git a/src/sampletones_shared/meta/source/bindings/__init__.py b/src/sampletones_tools/assets/mark/__init__.py similarity index 100% rename from src/sampletones_shared/meta/source/bindings/__init__.py rename to src/sampletones_tools/assets/mark/__init__.py diff --git a/tests/integration/assets/__init__.py b/src/sampletones_tools/assets/mark/config/__init__.py similarity index 100% rename from tests/integration/assets/__init__.py rename to src/sampletones_tools/assets/mark/config/__init__.py diff --git a/src/sampletones_assets/mark/mark.yaml b/src/sampletones_tools/assets/mark/config/mark.yaml similarity index 100% rename from src/sampletones_assets/mark/mark.yaml rename to src/sampletones_tools/assets/mark/config/mark.yaml diff --git a/src/sampletones_assets/mark/template.svg b/src/sampletones_tools/assets/mark/config/template.svg similarity index 100% rename from src/sampletones_assets/mark/template.svg rename to src/sampletones_tools/assets/mark/config/template.svg diff --git a/src/sampletones_assets/mark/geometry.py b/src/sampletones_tools/assets/mark/geometry.py similarity index 86% rename from src/sampletones_assets/mark/geometry.py rename to src/sampletones_tools/assets/mark/geometry.py index 5fd855bdd..e268dd3b9 100644 --- a/src/sampletones_assets/mark/geometry.py +++ b/src/sampletones_tools/assets/mark/geometry.py @@ -2,8 +2,8 @@ from dataclasses import dataclass from typing import List -from sampletones_assets.mark.specification.point import CubicCurve, Point -from sampletones_assets.mark.specification.waves import MarkSine, MarkSquare +from sampletones_tools.assets.mark.specification.point import CubicCurve, Point +from sampletones_tools.assets.mark.specification.waves import MarkSine, MarkSquare @dataclass(frozen=True) @@ -38,8 +38,20 @@ def _cubic_point( progress: float, ) -> Point: return Point( - x=_cubic_coordinate(start.x, curve.control_start.x, curve.control_end.x, curve.end.x, progress), - y=_cubic_coordinate(start.y, curve.control_start.y, curve.control_end.y, curve.end.y, progress), + x=_cubic_coordinate( + start.x, + curve.control_start.x, + curve.control_end.x, + curve.end.x, + progress, + ), + y=_cubic_coordinate( + start.y, + curve.control_start.y, + curve.control_end.y, + curve.end.y, + progress, + ), ) diff --git a/src/sampletones_tools/assets/mark/paths.py b/src/sampletones_tools/assets/mark/paths.py new file mode 100644 index 000000000..3b93d6c5e --- /dev/null +++ b/src/sampletones_tools/assets/mark/paths.py @@ -0,0 +1,8 @@ +from pathlib import Path +from typing import Final + +from sampletones_shared.paths.package import package_directory + +MARK_CONFIG_DIRECTORY: Final[Path] = package_directory("sampletones_tools.assets.mark.config") +MARK_PATH: Final[Path] = MARK_CONFIG_DIRECTORY / "mark.yaml" +TEMPLATE_PATH: Final[Path] = MARK_CONFIG_DIRECTORY / "template.svg" diff --git a/src/sampletones_assets/mark/raster.py b/src/sampletones_tools/assets/mark/raster.py similarity index 85% rename from src/sampletones_assets/mark/raster.py rename to src/sampletones_tools/assets/mark/raster.py index e7d388bd9..ccd244c20 100644 --- a/src/sampletones_assets/mark/raster.py +++ b/src/sampletones_tools/assets/mark/raster.py @@ -2,10 +2,10 @@ from PIL import Image, ImageDraw -from sampletones_assets.mark.geometry import sine_points, square_rectangles -from sampletones_assets.mark.specification import Mark from sampletones_shared.types.application import ColorRGBA from sampletones_shared.utils.color import parse_hex_color, with_alpha_fraction +from sampletones_tools.assets.mark.geometry import sine_points, square_rectangles +from sampletones_tools.assets.mark.specification import Mark TRANSPARENT: Final[ColorRGBA] = (0, 0, 0, 0) OPAQUE: Final[int] = 255 @@ -55,7 +55,11 @@ def _background(self) -> Image.Image: size = (self.canvas, self.canvas) top = Image.new("RGB", size, self.mark.colors.background.top) bottom = Image.new("RGB", size, self.mark.colors.background.bottom) - shaded = Image.composite(bottom, top, Image.linear_gradient("L").resize(size)) + shaded = Image.composite( + bottom, + top, + Image.linear_gradient("L").resize(size), + ) background = Image.new("RGBA", size, TRANSPARENT) background.paste(shaded, mask=self._frame_mask()) @@ -77,7 +81,10 @@ def _draw_sine(self, draw: ImageDraw.ImageDraw) -> None: what holds the outline smooth along its whole sweep. """ radius = self.mark.waves.width * self.scale / 2 - for point in sine_points(self.mark.waves.sine, self.mark.render.curve_samples): + for point in sine_points( + self.mark.waves.sine, + self.mark.render.curve_samples, + ): center_x, center_y = point.x * self.scale, point.y * self.scale draw.ellipse( ( @@ -90,7 +97,10 @@ def _draw_sine(self, draw: ImageDraw.ImageDraw) -> None: ) def _draw_square(self, draw: ImageDraw.ImageDraw) -> None: - for rectangle in square_rectangles(self.mark.waves.square, self.mark.waves.width): + for rectangle in square_rectangles( + self.mark.waves.square, + self.mark.waves.width, + ): draw.rectangle( ( round(rectangle.left * self.scale), @@ -113,4 +123,7 @@ def _rim(self) -> Image.Image: return overlay def _rim_color(self) -> ColorRGBA: - return with_alpha_fraction(parse_hex_color(self.mark.colors.rim), self.mark.frame.rim.opacity) + return with_alpha_fraction( + parse_hex_color(self.mark.colors.rim), + self.mark.frame.rim.opacity, + ) diff --git a/src/sampletones_assets/mark/specification/__init__.py b/src/sampletones_tools/assets/mark/specification/__init__.py similarity index 73% rename from src/sampletones_assets/mark/specification/__init__.py rename to src/sampletones_tools/assets/mark/specification/__init__.py index 6e78e8710..4b4b2239f 100644 --- a/src/sampletones_assets/mark/specification/__init__.py +++ b/src/sampletones_tools/assets/mark/specification/__init__.py @@ -2,12 +2,12 @@ from pydantic import BaseModel, Field -from sampletones_assets.mark.paths import MARK_PATH -from sampletones_assets.mark.specification.colors import MarkColors -from sampletones_assets.mark.specification.frame import MarkFrame -from sampletones_assets.mark.specification.render import MarkRender -from sampletones_assets.mark.specification.waves import MarkWaves from sampletones_shared.utils.serialization import load_yaml_model +from sampletones_tools.assets.mark.paths import MARK_PATH +from sampletones_tools.assets.mark.specification.colors import MarkColors +from sampletones_tools.assets.mark.specification.frame import MarkFrame +from sampletones_tools.assets.mark.specification.render import MarkRender +from sampletones_tools.assets.mark.specification.waves import MarkWaves class Mark(BaseModel, extra="forbid", frozen=True): @@ -30,7 +30,7 @@ def load(cls) -> Self: """Load the packaged mark definition. Returns: - The mark validated from `sampletones_assets/mark/mark.yaml`. + The mark validated from `sampletones_tools/assets/mark/config/mark.yaml`. Raises: TypeError: If the definition file holds anything other than a mapping. diff --git a/src/sampletones_assets/mark/specification/colors.py b/src/sampletones_tools/assets/mark/specification/colors.py similarity index 100% rename from src/sampletones_assets/mark/specification/colors.py rename to src/sampletones_tools/assets/mark/specification/colors.py diff --git a/src/sampletones_assets/mark/specification/frame.py b/src/sampletones_tools/assets/mark/specification/frame.py similarity index 100% rename from src/sampletones_assets/mark/specification/frame.py rename to src/sampletones_tools/assets/mark/specification/frame.py diff --git a/src/sampletones_assets/mark/specification/point.py b/src/sampletones_tools/assets/mark/specification/point.py similarity index 100% rename from src/sampletones_assets/mark/specification/point.py rename to src/sampletones_tools/assets/mark/specification/point.py diff --git a/src/sampletones_assets/mark/specification/render.py b/src/sampletones_tools/assets/mark/specification/render.py similarity index 100% rename from src/sampletones_assets/mark/specification/render.py rename to src/sampletones_tools/assets/mark/specification/render.py diff --git a/src/sampletones_assets/mark/specification/waves.py b/src/sampletones_tools/assets/mark/specification/waves.py similarity index 96% rename from src/sampletones_assets/mark/specification/waves.py rename to src/sampletones_tools/assets/mark/specification/waves.py index b97a1c35f..cb5b6f9b3 100644 --- a/src/sampletones_assets/mark/specification/waves.py +++ b/src/sampletones_tools/assets/mark/specification/waves.py @@ -3,7 +3,7 @@ from pydantic import BaseModel, Field, PositiveFloat, model_validator -from sampletones_assets.mark.specification.point import CubicCurve, Point +from sampletones_tools.assets.mark.specification.point import CubicCurve, Point class MarkSine(BaseModel, extra="forbid", frozen=True): diff --git a/src/sampletones_assets/mark/suite.py b/src/sampletones_tools/assets/mark/suite.py similarity index 78% rename from src/sampletones_assets/mark/suite.py rename to src/sampletones_tools/assets/mark/suite.py index a464b50a0..a6bd27c00 100644 --- a/src/sampletones_assets/mark/suite.py +++ b/src/sampletones_tools/assets/mark/suite.py @@ -3,14 +3,14 @@ from PIL import Image -from sampletones_assets.mark.raster import MarkRaster -from sampletones_assets.mark.specification import Mark -from sampletones_assets.mark.vector import render_vector from sampletones_shared.paths.resources import ( ICON_UNIX_FILENAME, ICON_VECTOR_FILENAME, ICON_WIN_FILENAME, ) +from sampletones_tools.assets.mark.raster import MarkRaster +from sampletones_tools.assets.mark.specification import Mark +from sampletones_tools.assets.mark.vector import render_vector def _resized(master: Image.Image, size: int) -> Image.Image: @@ -62,6 +62,14 @@ def write_icon_suite(directory: Path, mark: Mark) -> List[Path]: return [ _write_vector(directory / ICON_VECTOR_FILENAME, mark), - _write_raster(directory / ICON_UNIX_FILENAME, master, mark.render.raster_size), - _write_windows_icon(directory / ICON_WIN_FILENAME, master, mark.render.windows_sizes), + _write_raster( + directory / ICON_UNIX_FILENAME, + master, + mark.render.raster_size, + ), + _write_windows_icon( + directory / ICON_WIN_FILENAME, + master, + mark.render.windows_sizes, + ), ] diff --git a/src/sampletones_assets/mark/vector.py b/src/sampletones_tools/assets/mark/vector.py similarity index 89% rename from src/sampletones_assets/mark/vector.py rename to src/sampletones_tools/assets/mark/vector.py index f9ba412c9..0f8cded6e 100644 --- a/src/sampletones_assets/mark/vector.py +++ b/src/sampletones_tools/assets/mark/vector.py @@ -2,10 +2,10 @@ from string import Template from typing import Dict -from sampletones_assets.mark.paths import TEMPLATE_PATH -from sampletones_assets.mark.specification import Mark -from sampletones_assets.mark.specification.point import Point -from sampletones_assets.mark.specification.waves import MarkSine, MarkSquare +from sampletones_tools.assets.mark.paths import TEMPLATE_PATH +from sampletones_tools.assets.mark.specification import Mark +from sampletones_tools.assets.mark.specification.point import Point +from sampletones_tools.assets.mark.specification.waves import MarkSine, MarkSquare def _number(value: float) -> str: diff --git a/src/sampletones_tools/assets/paths.py b/src/sampletones_tools/assets/paths.py new file mode 100644 index 000000000..4d4c72e04 --- /dev/null +++ b/src/sampletones_tools/assets/paths.py @@ -0,0 +1,7 @@ +from pathlib import Path +from typing import Final + +from sampletones_shared.paths.resources import ICON_DIRECTORY +from sampletones_shared.paths.source import SOURCE_ROOT + +ICONS_DIRECTORY: Final[Path] = SOURCE_ROOT / "sampletones_assets" / ICON_DIRECTORY diff --git a/tests/unit/sampletones_assets/__init__.py b/src/sampletones_tools/calibration/__init__.py similarity index 100% rename from tests/unit/sampletones_assets/__init__.py rename to src/sampletones_tools/calibration/__init__.py diff --git a/src/sampletones_tools/calibration/command.py b/src/sampletones_tools/calibration/command.py new file mode 100644 index 000000000..237eb1208 --- /dev/null +++ b/src/sampletones_tools/calibration/command.py @@ -0,0 +1,94 @@ +from argparse import ArgumentParser, Namespace +from dataclasses import dataclass +from pathlib import Path +from typing import Final, Optional + +from sampletones_shared.command import Command +from sampletones_shared.options import add_config_option + +NAME: Final[str] = "calibration" +HELP: Final[str] = "score the reconstruction corpus under candidate configurations" +OUTPUT_HELP: Final[str] = ( + "the directory the run writes into; without it, a timestamped directory under Documents/SampleToNES/calibration" +) +METHODS_HELP: Final[str] = "spectrum methods to evaluate, comma separated; without it fft and cqt" +EXPONENTS_HELP: Final[str] = "values of metric.perceptual_exponent to evaluate, comma separated; without it 1.0" +WEIGHTS_HELP: Final[str] = ( + "values of weights.temporal_loss_weight to evaluate, comma separated; without it the base blend" +) +CHANNELS_HELP: Final[str] = ( + "channels every variant reconstructs with, comma separated; without it pulse1, triangle and noise" +) + + +@dataclass(frozen=True) +class CalibrationArguments: + """What a calibration run is given, as written on the command line.""" + + config: Optional[Path] + output: Optional[Path] + methods: Optional[str] + perceptual_exponents: Optional[str] + temporal_weights: Optional[str] + channels: Optional[str] + + +def configure(parser: ArgumentParser) -> None: + add_config_option(parser) + parser.add_argument("--output", "-o", type=Path, default=None, help=OUTPUT_HELP) + parser.add_argument("--methods", type=str, default=None, help=METHODS_HELP) + parser.add_argument("--perceptual-exponents", type=str, default=None, help=EXPONENTS_HELP) + parser.add_argument("--temporal-weights", type=str, default=None, help=WEIGHTS_HELP) + parser.add_argument("--channels", type=str, default=None, help=CHANNELS_HELP) + + +def run(arguments: Namespace) -> int: + """Runs the calibration the options describe. + + Raises: + SystemExit: If a method, a value or a channel is unknown. + """ + given = CalibrationArguments( + config=arguments.config, + output=arguments.output, + methods=arguments.methods, + perceptual_exponents=arguments.perceptual_exponents, + temporal_weights=arguments.temporal_weights, + channels=arguments.channels, + ) + + from sampletones_core.headless.config import load_config + from sampletones_core.headless.conversion.request import channels_named + from sampletones_shared.utils.validation import describe_failure + from sampletones_tools.calibration.session import ( + BASE_BLEND, + DEFAULT_PERCEPTUAL_EXPONENTS, + CalibrationRequest, + calibrate, + default_output, + floats_named, + methods_named, + ) + + try: + request = CalibrationRequest( + base=load_config(given.config), + output=given.output if given.output is not None else default_output(), + methods=methods_named(given.methods), + perceptual_exponents=floats_named(given.perceptual_exponents, DEFAULT_PERCEPTUAL_EXPONENTS), + temporal_weights=floats_named(given.temporal_weights, BASE_BLEND), + channels=channels_named(given.channels), + ) + except ValueError as error: + raise SystemExit(describe_failure(error)) from error + + calibrate(request) + return 0 + + +CALIBRATION: Final[Command] = Command( + name=NAME, + help=HELP, + configure=configure, + run=run, +) diff --git a/src/sampletones_core/calibration/config/__init__.py b/src/sampletones_tools/calibration/config/__init__.py similarity index 100% rename from src/sampletones_core/calibration/config/__init__.py rename to src/sampletones_tools/calibration/config/__init__.py diff --git a/src/sampletones_core/calibration/config/corpus.py b/src/sampletones_tools/calibration/config/corpus.py similarity index 54% rename from src/sampletones_core/calibration/config/corpus.py rename to src/sampletones_tools/calibration/config/corpus.py index b9fab70c5..1eb2e1307 100644 --- a/src/sampletones_core/calibration/config/corpus.py +++ b/src/sampletones_tools/calibration/config/corpus.py @@ -2,8 +2,8 @@ from pydantic import BaseModel, Field -from sampletones_core.calibration.paths import CORPUS_CONFIG_PATH from sampletones_shared.utils.serialization import load_yaml_model +from sampletones_tools.calibration.paths import CORPUS_CONFIG_PATH from .mix import MixConfig from .noise import NoiseConfig @@ -18,23 +18,43 @@ class CorpusConfig(BaseModel, frozen=True): Every probe is synthesized at unit scale and multiplied by the corpus amplitude, so the per-class parameters compose under one loudness - convention. Values are loaded from the packaged `calibration/corpus.yaml`, + convention. Values are loaded from the `corpus.yaml` beside this model, so corpus content stays reproducible across calibration runs while remaining adjustable in one place. """ - seed: int = Field(ge=0, description="Seed of the random generator behind the stochastic probes.") - item_seconds: float = Field(gt=0.0, description="Duration of every corpus item in seconds.") - amplitude: float = Field(gt=0.0, le=1.0, description="Scale applied to every unit-level probe.") + seed: int = Field( + ge=0, + description="Seed of the random generator behind the stochastic probes.", + ) + item_seconds: float = Field( + gt=0.0, + description="Duration of every corpus item in seconds.", + ) + amplitude: float = Field( + gt=0.0, + le=1.0, + description="Scale applied to every unit-level probe.", + ) reference_frequency: float = Field( gt=0.0, description="Anchor tone in Hz shared by the mix, pluck, and crescendo probes.", ) - tone: ToneConfig = Field(description="Steady sine probes.") - timbre: TimbreConfig = Field(description="Pulse-wave probes.") - noise: NoiseConfig = Field(description="Broadband noise probes.") - mix: MixConfig = Field(description="Tone-plus-noise probes.") - transient: TransientConfig = Field(description="Percussive probes.") + tone: ToneConfig = Field( + description="Steady sine probes.", + ) + timbre: TimbreConfig = Field( + description="Pulse-wave probes.", + ) + noise: NoiseConfig = Field( + description="Broadband noise probes.", + ) + mix: MixConfig = Field( + description="Tone-plus-noise probes.", + ) + transient: TransientConfig = Field( + description="Percussive probes.", + ) @classmethod def load(cls) -> Self: @@ -42,7 +62,7 @@ def load(cls) -> Self: Load the packaged corpus tuning. Returns: - The corpus configuration validated from `sampletones_config/calibration/corpus.yaml`. + The corpus configuration validated from `sampletones_tools/calibration/config/corpus.yaml`. Raises: TypeError: If the configuration file holds anything other than a mapping. diff --git a/src/sampletones_config/calibration/corpus.yaml b/src/sampletones_tools/calibration/config/corpus.yaml similarity index 100% rename from src/sampletones_config/calibration/corpus.yaml rename to src/sampletones_tools/calibration/config/corpus.yaml diff --git a/src/sampletones_core/calibration/config/mix.py b/src/sampletones_tools/calibration/config/mix.py similarity index 100% rename from src/sampletones_core/calibration/config/mix.py rename to src/sampletones_tools/calibration/config/mix.py diff --git a/src/sampletones_core/calibration/config/noise.py b/src/sampletones_tools/calibration/config/noise.py similarity index 100% rename from src/sampletones_core/calibration/config/noise.py rename to src/sampletones_tools/calibration/config/noise.py diff --git a/src/sampletones_core/calibration/config/referee.py b/src/sampletones_tools/calibration/config/referee.py similarity index 66% rename from src/sampletones_core/calibration/config/referee.py rename to src/sampletones_tools/calibration/config/referee.py index 9e4e2e65a..d70bc9d5d 100644 --- a/src/sampletones_core/calibration/config/referee.py +++ b/src/sampletones_tools/calibration/config/referee.py @@ -2,8 +2,8 @@ from pydantic import BaseModel, Field, PositiveInt -from sampletones_core.calibration.paths import REFEREE_CONFIG_PATH from sampletones_shared.utils.serialization import load_yaml_model +from sampletones_tools.calibration.paths import REFEREE_CONFIG_PATH class RefereeConfig(BaseModel, frozen=True): @@ -19,10 +19,20 @@ class RefereeConfig(BaseModel, frozen=True): min_length=1, description="STFT window sizes covering the time-frequency trade-off.", ) - hop_divisor: PositiveInt = Field(description="Window size divided by this gives the STFT hop.") - band_count: PositiveInt = Field(description="Number of ERB-spaced aggregation bands.") - low_frequency: float = Field(gt=0.0, description="Lower bound of the band axis in Hz.") - energy_floor: float = Field(gt=0.0, description="Absolute floor keeping silent band energies finite.") + hop_divisor: PositiveInt = Field( + description="Window size divided by this gives the STFT hop.", + ) + band_count: PositiveInt = Field( + description="Number of ERB-spaced aggregation bands.", + ) + low_frequency: float = Field( + gt=0.0, + description="Lower bound of the band axis in Hz.", + ) + energy_floor: float = Field( + gt=0.0, + description="Absolute floor keeping silent band energies finite.", + ) audibility_range_decibels: float = Field( gt=0.0, description="Audible range below the reference's loudest band; quieter content saturates.", @@ -34,7 +44,7 @@ def load(cls) -> Self: Load the packaged referee tuning. Returns: - The referee configuration validated from `sampletones_config/calibration/referee.yaml`. + The referee configuration validated from `sampletones_tools/calibration/config/referee.yaml`. Raises: TypeError: If the configuration file holds anything other than a mapping. diff --git a/src/sampletones_config/calibration/referee.yaml b/src/sampletones_tools/calibration/config/referee.yaml similarity index 100% rename from src/sampletones_config/calibration/referee.yaml rename to src/sampletones_tools/calibration/config/referee.yaml diff --git a/src/sampletones_core/calibration/config/timbre.py b/src/sampletones_tools/calibration/config/timbre.py similarity index 86% rename from src/sampletones_core/calibration/config/timbre.py rename to src/sampletones_tools/calibration/config/timbre.py index 0febd84ae..86c31f775 100644 --- a/src/sampletones_core/calibration/config/timbre.py +++ b/src/sampletones_tools/calibration/config/timbre.py @@ -2,7 +2,7 @@ from pydantic import BaseModel, Field -from sampletones_synthesis.oscillators.pulse import DutyCycle +from sampletones_tools.synthesis.oscillators.pulse import DutyCycle class TimbreConfig(BaseModel, frozen=True): diff --git a/src/sampletones_core/calibration/config/tone.py b/src/sampletones_tools/calibration/config/tone.py similarity index 100% rename from src/sampletones_core/calibration/config/tone.py rename to src/sampletones_tools/calibration/config/tone.py diff --git a/src/sampletones_core/calibration/config/transient.py b/src/sampletones_tools/calibration/config/transient.py similarity index 100% rename from src/sampletones_core/calibration/config/transient.py rename to src/sampletones_tools/calibration/config/transient.py diff --git a/src/sampletones_core/calibration/corpus/__init__.py b/src/sampletones_tools/calibration/corpus/__init__.py similarity index 100% rename from src/sampletones_core/calibration/corpus/__init__.py rename to src/sampletones_tools/calibration/corpus/__init__.py diff --git a/src/sampletones_core/calibration/corpus/item.py b/src/sampletones_tools/calibration/corpus/item.py similarity index 100% rename from src/sampletones_core/calibration/corpus/item.py rename to src/sampletones_tools/calibration/corpus/item.py diff --git a/src/sampletones_core/calibration/corpus/synthesis.py b/src/sampletones_tools/calibration/corpus/synthesis.py similarity index 81% rename from src/sampletones_core/calibration/corpus/synthesis.py rename to src/sampletones_tools/calibration/corpus/synthesis.py index b8815670a..e24450e6b 100644 --- a/src/sampletones_core/calibration/corpus/synthesis.py +++ b/src/sampletones_tools/calibration/corpus/synthesis.py @@ -3,21 +3,23 @@ import numpy as np from sampletones_core.audio.processing import clip_audio -from sampletones_core.calibration.config.corpus import CorpusConfig -from sampletones_synthesis.envelopes.exponential_decay import ExponentialDecayEnvelope -from sampletones_synthesis.envelopes.linear_attack import LinearAttackEnvelope -from sampletones_synthesis.envelopes.linear_ramp import LinearRampEnvelope -from sampletones_synthesis.envelopes.types import EnvelopeUnion -from sampletones_synthesis.oscillators.exponential_glide import ( +from sampletones_tools.calibration.config.corpus import CorpusConfig +from sampletones_tools.synthesis.envelopes.exponential_decay import ( + ExponentialDecayEnvelope, +) +from sampletones_tools.synthesis.envelopes.linear_attack import LinearAttackEnvelope +from sampletones_tools.synthesis.envelopes.linear_ramp import LinearRampEnvelope +from sampletones_tools.synthesis.envelopes.types import EnvelopeUnion +from sampletones_tools.synthesis.oscillators.exponential_glide import ( ExponentialGlideOscillator, ) -from sampletones_synthesis.oscillators.pulse import PulseOscillator -from sampletones_synthesis.oscillators.sine import SineOscillator -from sampletones_synthesis.oscillators.types import OscillatorUnion -from sampletones_synthesis.oscillators.walk_noise import WalkNoiseOscillator -from sampletones_synthesis.oscillators.white_noise import WhiteNoiseOscillator -from sampletones_synthesis.voice.layer import Layer -from sampletones_synthesis.voice.voice import Voice +from sampletones_tools.synthesis.oscillators.pulse import PulseOscillator +from sampletones_tools.synthesis.oscillators.sine import SineOscillator +from sampletones_tools.synthesis.oscillators.types import OscillatorUnion +from sampletones_tools.synthesis.oscillators.walk_noise import WalkNoiseOscillator +from sampletones_tools.synthesis.oscillators.white_noise import WhiteNoiseOscillator +from sampletones_tools.synthesis.voice.layer import Layer +from sampletones_tools.synthesis.voice.voice import Voice from .item import CorpusItem @@ -193,8 +195,16 @@ def _transient_probes(config: CorpusConfig) -> List[Probe]: "transient", _voice(config, _layer(WhiteNoiseOscillator(kind="white_noise"), snare_decay)), ), - ("transient-kick", "transient", _voice(config, _layer(kick_glide, kick_decay))), - ("transient-pluck", "transient", _voice(config, _layer(reference_tone, pluck_attack, pluck_decay))), + ( + "transient-kick", + "transient", + _voice(config, _layer(kick_glide, kick_decay)), + ), + ( + "transient-pluck", + "transient", + _voice(config, _layer(reference_tone, pluck_attack, pluck_decay)), + ), ] @@ -228,7 +238,12 @@ def _voice(config: CorpusConfig, *layers: Layer) -> Voice: ) -def _item(name: str, category: str, audio: np.ndarray, config: CorpusConfig) -> CorpusItem: +def _item( + name: str, + category: str, + audio: np.ndarray, + config: CorpusConfig, +) -> CorpusItem: """ Assemble a corpus item from a unit-scale probe. diff --git a/src/sampletones_core/calibration/corpus/writer.py b/src/sampletones_tools/calibration/corpus/writer.py similarity index 100% rename from src/sampletones_core/calibration/corpus/writer.py rename to src/sampletones_tools/calibration/corpus/writer.py diff --git a/src/sampletones_core/calibration/paths.py b/src/sampletones_tools/calibration/paths.py similarity index 56% rename from src/sampletones_core/calibration/paths.py rename to src/sampletones_tools/calibration/paths.py index 751bbb533..2e407c1ea 100644 --- a/src/sampletones_core/calibration/paths.py +++ b/src/sampletones_tools/calibration/paths.py @@ -1,8 +1,8 @@ from pathlib import Path from typing import Final -from sampletones_shared.paths.resources import CONFIG_DIRECTORY +from sampletones_shared.paths.package import package_directory -CALIBRATION_CONFIG_DIRECTORY: Final[Path] = CONFIG_DIRECTORY / "calibration" +CALIBRATION_CONFIG_DIRECTORY: Final[Path] = package_directory("sampletones_tools.calibration.config") REFEREE_CONFIG_PATH: Final[Path] = CALIBRATION_CONFIG_DIRECTORY / "referee.yaml" CORPUS_CONFIG_PATH: Final[Path] = CALIBRATION_CONFIG_DIRECTORY / "corpus.yaml" diff --git a/src/sampletones_core/calibration/referee/__init__.py b/src/sampletones_tools/calibration/referee/__init__.py similarity index 100% rename from src/sampletones_core/calibration/referee/__init__.py rename to src/sampletones_tools/calibration/referee/__init__.py diff --git a/src/sampletones_core/calibration/referee/auditory.py b/src/sampletones_tools/calibration/referee/auditory.py similarity index 88% rename from src/sampletones_core/calibration/referee/auditory.py rename to src/sampletones_tools/calibration/referee/auditory.py index f5f0e3046..aebb873cd 100644 --- a/src/sampletones_core/calibration/referee/auditory.py +++ b/src/sampletones_tools/calibration/referee/auditory.py @@ -3,7 +3,7 @@ import numpy as np from scipy.signal import stft -from sampletones_core.calibration.config.referee import RefereeConfig +from sampletones_tools.calibration.config.referee import RefereeConfig ERB_RATE_SCALE: Final[float] = 21.4 ERB_RATE_FACTOR: Final[float] = 4.37e-3 @@ -64,7 +64,12 @@ def score(self, reference: np.ndarray, estimate: np.ndarray) -> float: return float(np.mean(distances)) - def _band_energy(self, audio: np.ndarray, window_size: int, band_matrix: np.ndarray) -> np.ndarray: + def _band_energy( + self, + audio: np.ndarray, + window_size: int, + band_matrix: np.ndarray, + ) -> np.ndarray: hop = window_size // self.config.hop_divisor _, _, spectrum = stft( audio.astype(np.float64), @@ -77,7 +82,12 @@ def _band_energy(self, audio: np.ndarray, window_size: int, band_matrix: np.ndar return band_energy @classmethod - def _band_matrix(cls, sample_rate: int, window_size: int, config: RefereeConfig) -> np.ndarray: + def _band_matrix( + cls, + sample_rate: int, + window_size: int, + config: RefereeConfig, + ) -> np.ndarray: """ Rectangular aggregation matrix from STFT bins onto ERB-spaced bands. @@ -93,7 +103,11 @@ def _band_matrix(cls, sample_rate: int, window_size: int, config: RefereeConfig) config.band_count + 1, ) bin_rates = cls._erb_rate(frequencies) - band_indices = np.clip(np.searchsorted(edges_rate, bin_rates, side="right") - 1, 0, config.band_count - 1) + band_indices = np.clip( + np.searchsorted(edges_rate, bin_rates, side="right") - 1, + 0, + config.band_count - 1, + ) matrix = np.zeros((config.band_count, frequencies.shape[0])) matrix[band_indices, np.arange(frequencies.shape[0])] = 1.0 diff --git a/src/sampletones_core/calibration/referee/factory.py b/src/sampletones_tools/calibration/referee/factory.py similarity index 92% rename from src/sampletones_core/calibration/referee/factory.py rename to src/sampletones_tools/calibration/referee/factory.py index e6310e878..7c9fb8885 100644 --- a/src/sampletones_core/calibration/referee/factory.py +++ b/src/sampletones_tools/calibration/referee/factory.py @@ -1,6 +1,6 @@ from typing import List -from sampletones_core.calibration.config.referee import RefereeConfig +from sampletones_tools.calibration.config.referee import RefereeConfig from .auditory import MultiResolutionAuditoryReferee from .protocol import Referee diff --git a/src/sampletones_core/calibration/referee/protocol.py b/src/sampletones_tools/calibration/referee/protocol.py similarity index 100% rename from src/sampletones_core/calibration/referee/protocol.py rename to src/sampletones_tools/calibration/referee/protocol.py diff --git a/src/sampletones_core/calibration/referee/zimtohrli.py b/src/sampletones_tools/calibration/referee/zimtohrli.py similarity index 100% rename from src/sampletones_core/calibration/referee/zimtohrli.py rename to src/sampletones_tools/calibration/referee/zimtohrli.py diff --git a/src/sampletones_core/calibration/report.py b/src/sampletones_tools/calibration/report.py similarity index 52% rename from src/sampletones_core/calibration/report.py rename to src/sampletones_tools/calibration/report.py index f9a9ceae2..7628a378f 100644 --- a/src/sampletones_core/calibration/report.py +++ b/src/sampletones_tools/calibration/report.py @@ -1,11 +1,15 @@ -import csv from collections import defaultdict from pathlib import Path -from typing import Dict, Iterable, List, Tuple +from typing import Dict, Final, Iterable, List, Tuple import numpy as np -from .runner import CalibrationRow +from sampletones_shared.utils.tables import Table +from sampletones_tools.calibration.runner import CalibrationRow + +VARIANT_COLUMN: Final[str] = "variant" +CSV_COLUMNS: Final[Tuple[str, ...]] = (VARIANT_COLUMN, "item", "category", "referee", "score") +OVERALL_COLUMN: Final[str] = "overall" def write_csv(rows: List[CalibrationRow], path: Path) -> None: @@ -16,11 +20,10 @@ def write_csv(rows: List[CalibrationRow], path: Path) -> None: rows: Scored rows from the runner. path: Target CSV path. """ - with path.open("w", newline="") as handle: - writer = csv.writer(handle) - writer.writerow(["variant", "item", "category", "referee", "score"]) - for row in rows: - writer.writerow([row.variant, row.item, row.category, row.referee, f"{row.score:.6f}"]) + Table( + columns=CSV_COLUMNS, + rows=tuple((row.variant, row.item, row.category, row.referee, f"{row.score:.6f}") for row in rows), + ).write_csv(path) def write_markdown(rows: List[CalibrationRow], path: Path) -> None: @@ -33,25 +36,23 @@ def write_markdown(rows: List[CalibrationRow], path: Path) -> None: path: Target markdown path. """ lines: List[str] = ["# Calibration report", ""] - for referee in _ordered(row.referee for row in rows): referee_rows = [row for row in rows if row.referee == referee] - categories = _ordered(row.category for row in referee_rows) - variants = _ordered(row.variant for row in referee_rows) - means = _mean_scores(referee_rows) - - lines.append(f"## {referee}") - lines.append("") - lines.append("| variant | " + " | ".join(categories) + " | overall |") - lines.append("|---" * (len(categories) + 2) + "|") - for variant in variants: - cells = [f"{means.get((variant, category), float('nan')):.3f}" for category in categories] - overall = np.mean([row.score for row in referee_rows if row.variant == variant]) - lines.append(f"| {variant} | " + " | ".join(cells) + f" | {float(overall):.3f} |") - - lines.append("") - - path.write_text("\n".join(lines)) + lines.extend((f"## {referee}", "", *_referee_table(referee_rows).markdown_lines(), "")) + + path.write_text("\n".join(lines), encoding="utf-8") + + +def _referee_table(rows: List[CalibrationRow]) -> Table: + categories = _ordered(row.category for row in rows) + means = _mean_scores(rows) + cells: List[Tuple[str, ...]] = [] + for variant in _ordered(row.variant for row in rows): + scores = [f"{means.get((variant, category), float('nan')):.3f}" for category in categories] + overall = np.mean([row.score for row in rows if row.variant == variant]) + cells.append((variant, *scores, f"{float(overall):.3f}")) + + return Table(columns=(VARIANT_COLUMN, *categories, OVERALL_COLUMN), rows=tuple(cells)) def _mean_scores(rows: List[CalibrationRow]) -> Dict[Tuple[str, str], float]: diff --git a/src/sampletones_core/calibration/runner.py b/src/sampletones_tools/calibration/runner.py similarity index 98% rename from src/sampletones_core/calibration/runner.py rename to src/sampletones_tools/calibration/runner.py index c1388eca6..46bccb2c1 100644 --- a/src/sampletones_core/calibration/runner.py +++ b/src/sampletones_tools/calibration/runner.py @@ -11,9 +11,9 @@ SpectrumMethod, ) from sampletones_core.fft import Window +from sampletones_core.headless.library import generate_library from sampletones_core.library import InstructionLibrary from sampletones_core.reconstructions import Reconstructor -from sampletones_core.scripts.library import generate_library from sampletones_shared.logger import logger from .corpus.item import CorpusItem diff --git a/src/sampletones_tools/calibration/session.py b/src/sampletones_tools/calibration/session.py new file mode 100644 index 000000000..d405fae4b --- /dev/null +++ b/src/sampletones_tools/calibration/session.py @@ -0,0 +1,129 @@ +from pathlib import Path +from typing import Final, List, Optional, Sequence, Tuple + +from pydantic import BaseModel, ConfigDict, Field + +from sampletones_core.configs import Config +from sampletones_core.constants.enums import ChannelName, SpectrumMethod +from sampletones_shared.logger import logger +from sampletones_shared.paths.user import USER_PATH_DOCUMENTS +from sampletones_shared.utils.text import listed_items +from sampletones_tools.calibration.config.corpus import CorpusConfig +from sampletones_tools.calibration.corpus.synthesis import build_corpus +from sampletones_tools.calibration.corpus.writer import write_corpus +from sampletones_tools.calibration.referee.factory import build_referees +from sampletones_tools.calibration.report import write_csv, write_markdown +from sampletones_tools.calibration.runner import build_variants, evaluate_variants +from sampletones_tools.runs import stamped_run_directory + +DEFAULT_METHODS: Final[Tuple[SpectrumMethod, ...]] = (SpectrumMethod.FFT, SpectrumMethod.CQT) +DEFAULT_PERCEPTUAL_EXPONENTS: Final[Tuple[float, ...]] = (1.0,) +BASE_BLEND: Final[Tuple[float, ...]] = () +OUTPUT_ROOT: Final[Path] = USER_PATH_DOCUMENTS / "calibration" +CORPUS_DIRECTORY: Final[str] = "corpus" +CSV_REPORT: Final[str] = "report.csv" +MARKDOWN_REPORT: Final[str] = "report.md" + + +def methods_named(stated: Optional[str]) -> List[SpectrumMethod]: + """The spectrum methods a run evaluates: the ones named, comma separated, or FFT and CQT. + + Raises: + ValueError: If a name is none of the spectrum methods. + """ + if stated is None: + return list(DEFAULT_METHODS) + + methods: List[SpectrumMethod] = [] + for name in listed_items(stated): + try: + methods.append(SpectrumMethod(name)) + except ValueError as error: + known = ", ".join(method.value for method in SpectrumMethod) + raise ValueError(f"Unknown spectrum method {name!r}; the methods are {known}.") from error + + return methods + + +def floats_named(stated: Optional[str], default: Sequence[float]) -> List[float]: + """The values a sweep takes: the ones named, comma separated, or ``default``. + + Raises: + ValueError: If a value is no number. + """ + if stated is None: + return list(default) + + values: List[float] = [] + for value in listed_items(stated): + try: + values.append(float(value)) + except ValueError as error: + raise ValueError(f"Not a number: {value!r}.") from error + + return values + + +def default_output() -> Path: + """A timestamped run directory under the user's calibration documents.""" + return stamped_run_directory(OUTPUT_ROOT) + + +class CalibrationRequest(BaseModel): + """What a calibration run is asked for: the base configuration, the sweep and where it writes. + + Attributes: + base: The configuration every value the sweep leaves untouched comes from. + output: The directory the corpus and the reports are written into. + methods: The spectrum methods evaluated. + perceptual_exponents: The values of ``metric.perceptual_exponent`` evaluated. + temporal_weights: The values of ``weights.temporal_loss_weight`` evaluated; empty keeps + the base blend. + channels: The channels every variant reconstructs with. + """ + + model_config = ConfigDict(frozen=True) + + base: Config + output: Path + methods: List[SpectrumMethod] = Field(min_length=1) + perceptual_exponents: List[float] = Field(min_length=1) + temporal_weights: List[float] + channels: List[ChannelName] = Field(min_length=1) + + def pinned_base(self) -> Config: + """The base configuration reconstructing with the channels the run pins.""" + generation = self.base.generation.model_copy(update={"channels": self.channels}) + return self.base.model_copy(update={"generation": generation}) + + +def calibrate(request: CalibrationRequest) -> Path: + """Reconstructs the corpus under every variant, scores it with every referee and writes the reports. + + Returns: + Path: The directory holding the corpus, ``report.csv`` and ``report.md``. + """ + base = request.pinned_base() + request.output.mkdir(parents=True, exist_ok=True) + + sample_rate = base.library.sample_rate + items = build_corpus(sample_rate, config=CorpusConfig.load()) + item_paths = write_corpus(items, request.output / CORPUS_DIRECTORY, sample_rate) + referees = build_referees(sample_rate) + variants = build_variants( + base, + request.methods, + request.perceptual_exponents, + request.temporal_weights, + ) + + channel_names = ", ".join(channel.value for channel in request.channels) + logger.info( + f"Evaluating {len(variants)} variants x {len(items)} items x {len(referees)} referees on {channel_names}" + ) + rows = evaluate_variants(variants, items, item_paths, referees) + + write_csv(rows, request.output / CSV_REPORT) + write_markdown(rows, request.output / MARKDOWN_REPORT) + logger.info(f"Report written to {request.output}") + return request.output diff --git a/src/sampletones_tools/checkout.py b/src/sampletones_tools/checkout.py new file mode 100644 index 000000000..ea09b3193 --- /dev/null +++ b/src/sampletones_tools/checkout.py @@ -0,0 +1,32 @@ +from pathlib import Path +from typing import Final + +from sampletones_shared.paths.source import REPOSITORY_ROOT + +PROJECT_FILE: Final[str] = "pyproject.toml" +SOURCE_DIRECTORY: Final[str] = "src" +CHECKOUT_ADVICE: Final[str] = ( + "This command needs the repository and its project environment, so it runs from a checkout: " + "clone SampleToNES, run 'make setup', then 'uv run sampletones {command}'." +) + + +def is_checkout(root: Path) -> bool: + """Whether ``root`` is a SampleToNES checkout: the project file beside the source tree.""" + return (root / PROJECT_FILE).is_file() and (root / SOURCE_DIRECTORY).is_dir() + + +def require_checkout(command: str) -> None: + """Holds a developer command to a checkout, where the repository and the project environment are. + + An installed copy, from the wheel or the bundle, carries the package without the repository + around it or the development dependencies, so the refusal names the way to run the command there. + + Args: + command: The command line the advice names, such as ``check import-boundary --all``. + + Raises: + SystemExit: If the package runs outside a checkout. + """ + if not is_checkout(REPOSITORY_ROOT): + raise SystemExit(CHECKOUT_ADVICE.format(command=command)) diff --git a/tests/unit/sampletones_assets/mark/__init__.py b/src/sampletones_tools/checks/__init__.py similarity index 100% rename from tests/unit/sampletones_assets/mark/__init__.py rename to src/sampletones_tools/checks/__init__.py diff --git a/tests/unit/sampletones_core/calibration/__init__.py b/src/sampletones_tools/checks/boundary/__init__.py similarity index 100% rename from tests/unit/sampletones_core/calibration/__init__.py rename to src/sampletones_tools/checks/boundary/__init__.py diff --git a/src/sampletones_shared/meta/import_boundary/check.py b/src/sampletones_tools/checks/boundary/check.py similarity index 83% rename from src/sampletones_shared/meta/import_boundary/check.py rename to src/sampletones_tools/checks/boundary/check.py index c5604faec..8781c7c70 100644 --- a/src/sampletones_shared/meta/import_boundary/check.py +++ b/src/sampletones_tools/checks/boundary/check.py @@ -1,11 +1,11 @@ from pathlib import Path from typing import List, Optional, Sequence, Set -from sampletones_shared.meta.import_boundary.rule import BoundaryRule -from sampletones_shared.meta.import_boundary.scope import rule_modules -from sampletones_shared.meta.import_boundary.token import TokenRule -from sampletones_shared.meta.import_boundary.violation import Violation -from sampletones_shared.meta.source.modules import source_paths +from sampletones_tools.checks.boundary.rule import BoundaryRule +from sampletones_tools.checks.boundary.scope import rule_modules +from sampletones_tools.checks.boundary.token import TokenRule +from sampletones_tools.checks.boundary.violation import Violation +from sampletones_tools.checks.source.modules import source_paths def check_boundaries( diff --git a/tests/unit/sampletones_core/calibration/config/__init__.py b/src/sampletones_tools/checks/boundary/configs/__init__.py similarity index 100% rename from tests/unit/sampletones_core/calibration/config/__init__.py rename to src/sampletones_tools/checks/boundary/configs/__init__.py diff --git a/src/sampletones_shared/meta/import_boundary/configs/declaration.py b/src/sampletones_tools/checks/boundary/configs/declaration.py similarity index 93% rename from src/sampletones_shared/meta/import_boundary/configs/declaration.py rename to src/sampletones_tools/checks/boundary/configs/declaration.py index 5957647a3..addfc9c5b 100644 --- a/src/sampletones_shared/meta/import_boundary/configs/declaration.py +++ b/src/sampletones_tools/checks/boundary/configs/declaration.py @@ -2,8 +2,8 @@ from pydantic import BaseModel, ConfigDict -from sampletones_shared.meta.import_boundary.configs.general import GeneralBoundaries -from sampletones_shared.meta.import_boundary.rule import BoundaryRule +from sampletones_tools.checks.boundary.configs.general import GeneralBoundaries +from sampletones_tools.checks.boundary.rule import BoundaryRule class BoundaryDeclaration(BaseModel): diff --git a/src/sampletones_shared/meta/import_boundary/configs/general.py b/src/sampletones_tools/checks/boundary/configs/general.py similarity index 100% rename from src/sampletones_shared/meta/import_boundary/configs/general.py rename to src/sampletones_tools/checks/boundary/configs/general.py diff --git a/src/sampletones_shared/meta/import_boundary/configs/paths.py b/src/sampletones_tools/checks/boundary/configs/paths.py similarity index 100% rename from src/sampletones_shared/meta/import_boundary/configs/paths.py rename to src/sampletones_tools/checks/boundary/configs/paths.py diff --git a/src/sampletones_shared/meta/import_boundary/configs/rules.py b/src/sampletones_tools/checks/boundary/configs/rules.py similarity index 71% rename from src/sampletones_shared/meta/import_boundary/configs/rules.py rename to src/sampletones_tools/checks/boundary/configs/rules.py index b4fc43a46..89784d669 100644 --- a/src/sampletones_shared/meta/import_boundary/configs/rules.py +++ b/src/sampletones_tools/checks/boundary/configs/rules.py @@ -2,29 +2,32 @@ from pydantic import BaseModel, ConfigDict, model_validator -from sampletones_shared.meta.import_boundary.configs.declaration import BoundaryDeclaration -from sampletones_shared.meta.import_boundary.configs.general import GeneralBoundaries -from sampletones_shared.meta.import_boundary.configs.paths import BOUNDARIES_DIRECTORY -from sampletones_shared.meta.import_boundary.graph import LayerGraph -from sampletones_shared.meta.import_boundary.rule import BoundaryRule -from sampletones_shared.meta.import_boundary.token import TokenRule from sampletones_shared.utils.serialization import load_yaml_model_dir +from sampletones_tools.checks.boundary.configs.declaration import BoundaryDeclaration +from sampletones_tools.checks.boundary.configs.general import GeneralBoundaries +from sampletones_tools.checks.boundary.configs.paths import BOUNDARIES_DIRECTORY +from sampletones_tools.checks.boundary.graph import LayerGraph +from sampletones_tools.checks.boundary.rule import BoundaryRule +from sampletones_tools.checks.boundary.standalone import StandaloneRule +from sampletones_tools.checks.boundary.token import TokenRule class ImportBoundaryRules(BaseModel): """Every boundary the source tree is held to, as the shipped configuration states it. - The declaration comes in three forms, each a fragment of its own. A layer graph names a tree + The declaration comes in four forms, each a fragment of its own. A layer graph names a tree of modules and what each unit may import, and amounts to one rule per unit. A boundary declaration names one directory and the imports it stays clear of. A token rule names a - spelling a tree keeps out. The general vocabulary holds the prefix sets the declarations draw - on, so a set several of them reach for is written once. + spelling a tree keeps out. A standalone rule names the scripts that run on the system + interpreter and holds them to the standard library. The general vocabulary holds the prefix + sets the declarations draw on, so a set several of them reach for is written once. Attributes: general: The names the declarations are written in. graphs: Each layer graph the source tree divides into, under the name the documents give it. rules: The boundaries written directly. tokens: The spellings kept out of the trees they name. + standalone: The scripts held to the standard library and the tree they sit in. """ model_config = ConfigDict(extra="forbid", frozen=True) @@ -33,6 +36,7 @@ class ImportBoundaryRules(BaseModel): graphs: Dict[str, LayerGraph] rules: Tuple[BoundaryDeclaration, ...] tokens: Tuple[TokenRule, ...] + standalone: Tuple[StandaloneRule, ...] @model_validator(mode="after") def _validate_every_named_group_is_declared(self) -> Self: diff --git a/src/sampletones_shared/meta/import_boundary/graph.py b/src/sampletones_tools/checks/boundary/graph.py similarity index 96% rename from src/sampletones_shared/meta/import_boundary/graph.py rename to src/sampletones_tools/checks/boundary/graph.py index dede55c51..9842a9a54 100644 --- a/src/sampletones_shared/meta/import_boundary/graph.py +++ b/src/sampletones_tools/checks/boundary/graph.py @@ -2,8 +2,8 @@ from pydantic import BaseModel, ConfigDict, model_validator -from sampletones_shared.meta.import_boundary.rule import BoundaryRule -from sampletones_shared.meta.import_boundary.units import ( +from sampletones_tools.checks.boundary.rule import BoundaryRule +from sampletones_tools.checks.boundary.units import ( nested_globs, unit_glob, unit_prefix, diff --git a/src/sampletones_shared/meta/import_boundary/imports.py b/src/sampletones_tools/checks/boundary/imports.py similarity index 91% rename from src/sampletones_shared/meta/import_boundary/imports.py rename to src/sampletones_tools/checks/boundary/imports.py index 5a4f50508..83a41277e 100644 --- a/src/sampletones_shared/meta/import_boundary/imports.py +++ b/src/sampletones_tools/checks/boundary/imports.py @@ -1,7 +1,7 @@ import re from typing import Final, Optional -from sampletones_shared.meta.source.modules import MODULE_SEPARATOR +from sampletones_tools.checks.source.modules import MODULE_SEPARATOR IMPORT_PATTERN: Final[re.Pattern[str]] = re.compile(r"^\s*(import|from)\s+([\w.]+)") diff --git a/src/sampletones_shared/meta/import_boundary/lines.py b/src/sampletones_tools/checks/boundary/lines.py similarity index 91% rename from src/sampletones_shared/meta/import_boundary/lines.py rename to src/sampletones_tools/checks/boundary/lines.py index 3b6f2d28a..3bdd39d96 100644 --- a/src/sampletones_shared/meta/import_boundary/lines.py +++ b/src/sampletones_tools/checks/boundary/lines.py @@ -1,7 +1,7 @@ from pathlib import Path from typing import Iterator, Tuple -from sampletones_shared.meta.source.modules import SOURCE_ENCODING +from sampletones_tools.checks.source.modules import SOURCE_ENCODING def numbered_lines(path: Path) -> Iterator[Tuple[int, str]]: diff --git a/src/sampletones_shared/meta/import_boundary/rule.py b/src/sampletones_tools/checks/boundary/rule.py similarity index 91% rename from src/sampletones_shared/meta/import_boundary/rule.py rename to src/sampletones_tools/checks/boundary/rule.py index 8ab77c3d5..5a4e1339b 100644 --- a/src/sampletones_shared/meta/import_boundary/rule.py +++ b/src/sampletones_tools/checks/boundary/rule.py @@ -3,12 +3,12 @@ from pydantic import BaseModel, ConfigDict -from sampletones_shared.meta.import_boundary.imports import ( +from sampletones_tools.checks.boundary.imports import ( imported_module, matches_prefix, ) -from sampletones_shared.meta.import_boundary.lines import numbered_lines -from sampletones_shared.meta.import_boundary.violation import Violation +from sampletones_tools.checks.boundary.lines import numbered_lines +from sampletones_tools.checks.boundary.violation import Violation class BoundaryRule(BaseModel): diff --git a/src/sampletones_shared/meta/import_boundary/scope.py b/src/sampletones_tools/checks/boundary/scope.py similarity index 100% rename from src/sampletones_shared/meta/import_boundary/scope.py rename to src/sampletones_tools/checks/boundary/scope.py diff --git a/src/sampletones_tools/checks/boundary/standalone.py b/src/sampletones_tools/checks/boundary/standalone.py new file mode 100644 index 000000000..bd7ab4b1c --- /dev/null +++ b/src/sampletones_tools/checks/boundary/standalone.py @@ -0,0 +1,145 @@ +import sys +from pathlib import Path +from typing import Final, List, Optional, Sequence, Set, Tuple + +from pydantic import BaseModel, ConfigDict + +from sampletones_tools.checks.boundary.imports import imported_module +from sampletones_tools.checks.boundary.lines import numbered_lines +from sampletones_tools.checks.boundary.scope import rule_modules +from sampletones_tools.checks.boundary.units import MODULE_SUFFIX +from sampletones_tools.checks.boundary.violation import Violation +from sampletones_tools.checks.source.modules import ( + MODULE_SEPARATOR, + PACKAGE_INITIALIZER, + SOURCE_PATTERN, + source_paths, +) + +SHADOWING: Final[str] = "stands in for a module the tree sits beside" + + +def local_names(root: Path) -> Set[str]: + """The names a script under ``root`` reaches in the tree itself: its modules and its packages. + + A script runs with its tree on the import path, so a module beside it and a package holding + an initializer are reached by their bare names. + + Args: + root: The tree the scripts sit in. + + Returns: + Set[str]: The importable names the tree offers. + """ + modules = {path.stem for path in root.glob(SOURCE_PATTERN)} + packages = {path.name for path in root.iterdir() if (path / f"{PACKAGE_INITIALIZER}{MODULE_SUFFIX}").is_file()} + return modules | packages + + +class StandaloneRule(BaseModel): + """One tree of scripts that run on the system interpreter, and what they may import. + + A script the rule reaches imports the standard library and the tree it sits in, so it runs + on a machine that has Python and nothing more. The tree sits on the import path beside the + standard library and the repository's own packages, so a name in it that stands in for one + of theirs is reported as well. + + Attributes: + pattern: Glob naming the scripts the rule reaches, written against the tree's root. + reserved: Names the tree keeps clear of beyond the standard library's. + message: What the rule holds, printed where a script imports past it. + """ + + model_config = ConfigDict(extra="forbid", frozen=True) + + pattern: str + reserved: Tuple[str, ...] = () + message: str + + def violations(self, path: Path, local: Set[str]) -> List[Violation]: + """Every import one script takes beyond the standard library and the tree. + + A relative import stays inside the tree by construction, and a name the tree offers is + reached the way a script reaches it when run from the tree. + + Args: + path: Script to read. + local: The names the tree offers, as `local_names` reads them. + + Returns: + List[Violation]: The imports the rule reports, in line order. + + Raises: + OSError: If the script cannot be read. + """ + violations: List[Violation] = [] + for line_number, line in numbered_lines(path): + module = imported_module(line) + if module is None: + continue + + top = module.split(MODULE_SEPARATOR, 1)[0] + if top and top not in sys.stdlib_module_names and top not in local: + violations.append(Violation.at(self.message, path, line_number, line)) + + return violations + + def shadowing(self, root: Path, paths: Sequence[Path]) -> List[Violation]: + """Every name in the tree that stands in for a standard-library or reserved module. + + A name is reported once, at the first script that carries it, since a directory is + named by every script under it. + + Args: + root: The tree the scripts sit in, resolved. + paths: The scripts the rule reaches, resolved and in path order. + + Returns: + List[Violation]: One violation per offending name, in the order the tree is read. + """ + taken = set(sys.stdlib_module_names) | set(self.reserved) + seen: Set[str] = set() + violations: List[Violation] = [] + for path in paths: + for part in path.relative_to(root).with_suffix("").parts: + if part in taken and part not in seen: + seen.add(part) + violations.append( + Violation( + kind=f"{part} {SHADOWING}", + location=str(path), + ) + ) + + return violations + + +def check_standalone( + root: Path, + rules: Sequence[StandaloneRule], + selection: Optional[Set[Path]], +) -> List[Violation]: + """Every import and name the rules forbid under a tree of standalone scripts. + + Args: + root: The tree the rules are written against. + rules: What the scripts may import. + selection: Resolved paths to narrow the check to, or `None` to check the whole tree. + + Returns: + List[Violation]: What the rules report, the shadowing names before the imports. + + Raises: + NotADirectoryError: If the root names no directory. + FileNotFoundError: If the root holds no module to read. + """ + tree = root.resolve() + swept = {path.resolve() for path in source_paths([tree])} + local = local_names(tree) + violations: List[Violation] = [] + for rule in rules: + paths = rule_modules(tree, rule.pattern, (), swept, selection) + violations.extend(rule.shadowing(tree, paths)) + violations.extend(violation for path in paths for violation in rule.violations(path, local)) + + return violations diff --git a/src/sampletones_shared/meta/import_boundary/token.py b/src/sampletones_tools/checks/boundary/token.py similarity index 92% rename from src/sampletones_shared/meta/import_boundary/token.py rename to src/sampletones_tools/checks/boundary/token.py index 51891e26d..e99660945 100644 --- a/src/sampletones_shared/meta/import_boundary/token.py +++ b/src/sampletones_tools/checks/boundary/token.py @@ -4,8 +4,8 @@ from pydantic import BaseModel, ConfigDict, field_validator -from sampletones_shared.meta.import_boundary.lines import numbered_lines -from sampletones_shared.meta.import_boundary.violation import Violation +from sampletones_tools.checks.boundary.lines import numbered_lines +from sampletones_tools.checks.boundary.violation import Violation class TokenRule(BaseModel): diff --git a/src/sampletones_shared/meta/import_boundary/units.py b/src/sampletones_tools/checks/boundary/units.py similarity index 94% rename from src/sampletones_shared/meta/import_boundary/units.py rename to src/sampletones_tools/checks/boundary/units.py index c87b06d0a..a594604b5 100644 --- a/src/sampletones_shared/meta/import_boundary/units.py +++ b/src/sampletones_tools/checks/boundary/units.py @@ -1,6 +1,6 @@ from typing import Final, Iterable, Tuple -from sampletones_shared.meta.source.modules import MODULE_SEPARATOR +from sampletones_tools.checks.source.modules import MODULE_SEPARATOR MODULE_SUFFIX: Final[str] = ".py" PATH_SEPARATOR: Final[str] = "/" diff --git a/src/sampletones_shared/meta/import_boundary/violation.py b/src/sampletones_tools/checks/boundary/violation.py similarity index 100% rename from src/sampletones_shared/meta/import_boundary/violation.py rename to src/sampletones_tools/checks/boundary/violation.py diff --git a/src/sampletones_tools/checks/command.py b/src/sampletones_tools/checks/command.py new file mode 100644 index 000000000..156138c2e --- /dev/null +++ b/src/sampletones_tools/checks/command.py @@ -0,0 +1,48 @@ +from argparse import ArgumentParser, Namespace +from typing import Final + +from sampletones_shared.command import Command +from sampletones_tools.checks.registry import GATES + +NAME: Final[str] = "check" +HELP: Final[str] = "hold the repository to one of its source checks" +GATE_FIELD: Final[str] = "gate" +GATE_METAVAR: Final[str] = "" + + +def configure(parser: ArgumentParser) -> None: + gates = parser.add_subparsers( + dest=GATE_FIELD, + metavar=GATE_METAVAR, + required=True, + ) + for gate in GATES: + gate.configure( + gates.add_parser( + gate.name, + help=gate.help, + description=gate.help, + ) + ) + + +def run(arguments: Namespace) -> int: + """Runs the check named, from a checkout, and answers with its status. + + Raises: + SystemExit: If the package runs outside a checkout. + """ + gate = next(gate for gate in GATES if gate.name == arguments.gate) + + from sampletones_tools.checkout import require_checkout + + require_checkout(f"{NAME} {gate.name}") + return gate.run(arguments) + + +CHECK: Final[Command] = Command( + name=NAME, + help=HELP, + configure=configure, + run=run, +) diff --git a/tests/unit/sampletones_core/calibration/corpus/__init__.py b/src/sampletones_tools/checks/commands/__init__.py similarity index 100% rename from tests/unit/sampletones_core/calibration/corpus/__init__.py rename to src/sampletones_tools/checks/commands/__init__.py diff --git a/src/sampletones_tools/checks/commands/import_boundary.py b/src/sampletones_tools/checks/commands/import_boundary.py new file mode 100644 index 000000000..285a71f95 --- /dev/null +++ b/src/sampletones_tools/checks/commands/import_boundary.py @@ -0,0 +1,60 @@ +from argparse import ArgumentParser, Namespace +from dataclasses import dataclass +from pathlib import Path +from typing import Final, Optional, Tuple + +from sampletones_shared.command import Command + +NAME: Final[str] = "import-boundary" +HELP: Final[str] = "hold the source and scripts trees to the declared import boundaries" +FILES_HELP: Final[str] = "modules to check" +ALL_HELP: Final[str] = "check every module under the source and scripts trees instead of named files" +SOURCE_HELP: Final[str] = "source root the rule roots are named within; without it, the repository's src" +SCRIPTS_HELP: Final[str] = "scripts tree the standalone rules are written against; without it, the repository's scripts" + + +@dataclass(frozen=True) +class ImportBoundaryArguments: + """What a boundary check is given: the modules or the whole trees, and where the trees are.""" + + files: Tuple[Path, ...] + everything: bool + source: Optional[Path] + scripts: Optional[Path] + + +def configure(parser: ArgumentParser) -> None: + parser.add_argument("files", nargs="*", type=Path, help=FILES_HELP) + parser.add_argument("--all", action="store_true", help=ALL_HELP) + parser.add_argument("--source", type=Path, default=None, help=SOURCE_HELP) + parser.add_argument("--scripts", type=Path, default=None, help=SCRIPTS_HELP) + + +def run(arguments: Namespace) -> int: + """Reports every import and token the boundaries forbid.""" + given = ImportBoundaryArguments( + files=tuple(arguments.files), + everything=arguments.all, + source=arguments.source, + scripts=arguments.scripts, + ) + + from sampletones_shared.paths.source import SOURCE_ROOT + from sampletones_tools.checks.import_boundary import check_imports, report + from sampletones_tools.checks.paths import SCRIPTS_ROOT + + selection = None if given.everything else {path.resolve() for path in given.files} + violations = check_imports( + given.source if given.source is not None else SOURCE_ROOT, + given.scripts if given.scripts is not None else SCRIPTS_ROOT, + selection, + ) + return report(violations) + + +IMPORT_BOUNDARY: Final[Command] = Command( + name=NAME, + help=HELP, + configure=configure, + run=run, +) diff --git a/src/sampletones_tools/checks/commands/language_keys.py b/src/sampletones_tools/checks/commands/language_keys.py new file mode 100644 index 000000000..0c46a3151 --- /dev/null +++ b/src/sampletones_tools/checks/commands/language_keys.py @@ -0,0 +1,60 @@ +from argparse import ArgumentParser, Namespace +from dataclasses import dataclass +from pathlib import Path +from typing import Final, Optional + +from sampletones_shared.command import Command + +NAME: Final[str] = "language-keys" +HELP: Final[str] = "hold the code and the language file to each other, in both directions" +SOURCE_HELP: Final[str] = "root of the sources to read; without it, the repository's src" +LANGUAGE_FILE_HELP: Final[str] = "language file to check against; without it, the shipped en.yaml" + + +@dataclass(frozen=True) +class LanguageKeysArguments: + """What a language key check is given: the sources and the language file.""" + + source: Optional[Path] + language_file: Optional[Path] + + +def configure(parser: ArgumentParser) -> None: + parser.add_argument( + "--source", + type=Path, + default=None, + help=SOURCE_HELP, + ) + parser.add_argument( + "--language-file", + type=Path, + default=None, + help=LANGUAGE_FILE_HELP, + ) + + +def run(arguments: Namespace) -> int: + """Reports every disagreement between the language file and the lookups reading it.""" + given = LanguageKeysArguments( + source=arguments.source, + language_file=arguments.language_file, + ) + + from sampletones_application.paths import LANG_EN + from sampletones_shared.paths.source import SOURCE_ROOT + from sampletones_tools.checks.language_keys import check_language_keys, report + + findings = check_language_keys( + given.source if given.source is not None else SOURCE_ROOT, + given.language_file if given.language_file is not None else LANG_EN, + ) + return report(findings) + + +LANGUAGE_KEYS: Final[Command] = Command( + name=NAME, + help=HELP, + configure=configure, + run=run, +) diff --git a/src/sampletones_tools/checks/commands/palette_colors.py b/src/sampletones_tools/checks/commands/palette_colors.py new file mode 100644 index 000000000..a85c6e5e5 --- /dev/null +++ b/src/sampletones_tools/checks/commands/palette_colors.py @@ -0,0 +1,61 @@ +from argparse import ArgumentParser, Namespace +from dataclasses import dataclass +from pathlib import Path +from typing import Final, Optional + +from sampletones_shared.command import Command + +NAME: Final[str] = "palette-colors" +HELP: Final[str] = "hold every color to a palette token until it is drawn with" +PACKAGE_HELP: Final[str] = "package whose color reads to check; without it, the application package" +CONFIG_DIRECTORY_HELP: Final[str] = ( + "shipped configuration package whose colors must name palette tokens; without it, the shipped one" +) +PALETTES_HELP: Final[str] = "directory holding the palettes, where color values belong; without it, the shipped one" + + +@dataclass(frozen=True) +class PaletteColorsArguments: + """What a palette check is given: the package, the configuration and the palettes.""" + + package: Optional[Path] + config_directory: Optional[Path] + palettes: Optional[Path] + + +def configure(parser: ArgumentParser) -> None: + parser.add_argument("--package", type=Path, default=None, help=PACKAGE_HELP) + parser.add_argument("--config-directory", type=Path, default=None, help=CONFIG_DIRECTORY_HELP) + parser.add_argument("--palettes", type=Path, default=None, help=PALETTES_HELP) + + +def run(arguments: Namespace) -> int: + """Reports every color the application stores resolved or the configuration writes out.""" + given = PaletteColorsArguments( + package=arguments.package, + config_directory=arguments.config_directory, + palettes=arguments.palettes, + ) + + from sampletones_application.paths import PALETTES_DIRECTORY + from sampletones_shared.paths.resources import CONFIG_DIRECTORY + from sampletones_tools.checks.palette_colors import ( + APPLICATION_PACKAGE, + check_colors, + report, + ) + + findings = check_colors( + given.package if given.package is not None else APPLICATION_PACKAGE, + given.config_directory if given.config_directory is not None else CONFIG_DIRECTORY, + given.palettes if given.palettes is not None else PALETTES_DIRECTORY, + ) + return report(findings) + + +PALETTE_COLORS: Final[Command] = Command( + name=NAME, + help=HELP, + configure=configure, + run=run, +) diff --git a/src/sampletones_tools/checks/commands/rendered_literals.py b/src/sampletones_tools/checks/commands/rendered_literals.py new file mode 100644 index 000000000..67dd0a045 --- /dev/null +++ b/src/sampletones_tools/checks/commands/rendered_literals.py @@ -0,0 +1,49 @@ +from argparse import ArgumentParser, Namespace +from dataclasses import dataclass +from pathlib import Path +from typing import Final, Tuple + +from sampletones_shared.command import Command + +NAME: Final[str] = "rendered-literals" +HELP: Final[str] = "hold every case to comparing values, never the text it rendered" +TESTS_HELP: Final[str] = "directory of cases to read, repeatable; without it, the repository's tests" + + +@dataclass(frozen=True) +class RenderedLiteralsArguments: + """What a rendered literal check is given: the directories of cases.""" + + roots: Tuple[Path, ...] + + +def configure(parser: ArgumentParser) -> None: + parser.add_argument( + "--tests", + type=Path, + action="append", + dest="roots", + default=[], + help=TESTS_HELP, + ) + + +def run(arguments: Namespace) -> int: + """Reports every case comparing text it rendered with a literal it spelled out.""" + given = RenderedLiteralsArguments(roots=tuple(arguments.roots)) + + from sampletones_tools.checks.rendered_literals import ( + TEST_ROOTS, + check_cases, + report, + ) + + return report(check_cases(given.roots or TEST_ROOTS)) + + +RENDERED_LITERALS: Final[Command] = Command( + name=NAME, + help=HELP, + configure=configure, + run=run, +) diff --git a/src/sampletones_tools/checks/commands/shortcut_actions.py b/src/sampletones_tools/checks/commands/shortcut_actions.py new file mode 100644 index 000000000..4d20fc24b --- /dev/null +++ b/src/sampletones_tools/checks/commands/shortcut_actions.py @@ -0,0 +1,28 @@ +from argparse import ArgumentParser, Namespace +from typing import Final + +from sampletones_shared.command import Command + +NAME: Final[str] = "shortcut-actions" +HELP: Final[str] = "hold every action to its keys, its name and the call it makes" + + +def configure(parser: ArgumentParser) -> None: + del parser + + +def run(arguments: Namespace) -> int: + """Reports every action left short of a combination, a name, or the call it makes.""" + del arguments + + from sampletones_tools.checks.shortcut_actions import check, report + + return report(check()) + + +SHORTCUT_ACTIONS: Final[Command] = Command( + name=NAME, + help=HELP, + configure=configure, + run=run, +) diff --git a/src/sampletones_tools/checks/commands/tag_names.py b/src/sampletones_tools/checks/commands/tag_names.py new file mode 100644 index 000000000..bc4640f41 --- /dev/null +++ b/src/sampletones_tools/checks/commands/tag_names.py @@ -0,0 +1,44 @@ +from argparse import ArgumentParser, Namespace +from dataclasses import dataclass +from pathlib import Path +from typing import Final, Tuple + +from sampletones_shared.command import Command + +NAME: Final[str] = "tag-names" +HELP: Final[str] = "hold every tag constant's name to the tag it composes" +FILES_HELP: Final[str] = "modules to check" +ALL_HELP: Final[str] = "check every module of the tags package instead of named files" + + +@dataclass(frozen=True) +class TagNamesArguments: + """What a tag name check is given: the modules, or the whole tags package.""" + + files: Tuple[Path, ...] + everything: bool + + +def configure(parser: ArgumentParser) -> None: + parser.add_argument("files", nargs="*", type=Path, help=FILES_HELP) + parser.add_argument("--all", action="store_true", help=ALL_HELP) + + +def run(arguments: Namespace) -> int: + """Reports any tag constant whose name departs from the tag it composes.""" + given = TagNamesArguments( + files=tuple(arguments.files), + everything=arguments.all, + ) + + from sampletones_tools.checks.tag_names import check_tags, report + + return report(check_tags(given.files, given.everything)) + + +TAG_NAMES: Final[Command] = Command( + name=NAME, + help=HELP, + configure=configure, + run=run, +) diff --git a/src/sampletones_tools/checks/commands/unused_tags.py b/src/sampletones_tools/checks/commands/unused_tags.py new file mode 100644 index 000000000..eaa545740 --- /dev/null +++ b/src/sampletones_tools/checks/commands/unused_tags.py @@ -0,0 +1,60 @@ +from argparse import ArgumentParser, Namespace +from dataclasses import dataclass +from pathlib import Path +from typing import Final, Optional, Tuple + +from sampletones_shared.command import Command + +NAME: Final[str] = "unused-tags" +HELP: Final[str] = "hold every declared tag fragment to a read somewhere in the repository" +TAGS_HELP: Final[str] = "package declaring the tag fragments; without it, the application's tags package" +REFERENCE_ROOT_HELP: Final[str] = "directory to count reads in, repeatable; without it, src, tests and scripts" + + +@dataclass(frozen=True) +class UnusedTagsArguments: + """What an unused tag check is given: the tags package and the directories read for references.""" + + tags: Optional[Path] + reference_roots: Tuple[Path, ...] + + +def configure(parser: ArgumentParser) -> None: + parser.add_argument("--tags", type=Path, default=None, help=TAGS_HELP) + parser.add_argument( + "--reference-root", + type=Path, + action="append", + dest="reference_roots", + default=[], + help=REFERENCE_ROOT_HELP, + ) + + +def run(arguments: Namespace) -> int: + """Reports every tag fragment the repository declares and never reads.""" + given = UnusedTagsArguments( + tags=arguments.tags, + reference_roots=tuple(arguments.reference_roots), + ) + + from sampletones_tools.checks.unused_tags import ( + REFERENCE_ROOTS, + TAGS_PACKAGE, + check_reads, + report, + ) + + unread = check_reads( + given.tags if given.tags is not None else TAGS_PACKAGE, + given.reference_roots or REFERENCE_ROOTS, + ) + return report(unread) + + +UNUSED_TAGS: Final[Command] = Command( + name=NAME, + help=HELP, + configure=configure, + run=run, +) diff --git a/src/sampletones_tools/checks/import_boundary.py b/src/sampletones_tools/checks/import_boundary.py new file mode 100644 index 000000000..8501fe349 --- /dev/null +++ b/src/sampletones_tools/checks/import_boundary.py @@ -0,0 +1,39 @@ +import sys +from pathlib import Path +from typing import List, Optional, Sequence, Set + +from sampletones_tools.checks.boundary.check import check_boundaries +from sampletones_tools.checks.boundary.configs.rules import ImportBoundaryRules +from sampletones_tools.checks.boundary.standalone import check_standalone +from sampletones_tools.checks.boundary.violation import Violation + + +def check_imports(source: Path, scripts: Path, selection: Optional[Set[Path]]) -> List[Violation]: + """Every import and token the shipped boundaries forbid under the source and scripts trees. + + Args: + source: Source root the rule roots are named within. + scripts: Scripts tree the standalone rules are written against. + selection: Resolved paths to narrow the check to, or `None` to check both trees whole. + + Returns: + List[Violation]: What the rules report, the source tree's before the scripts tree's. + """ + boundaries = ImportBoundaryRules.load() + return [ + *check_boundaries(source, boundaries.boundary_rules(), boundaries.tokens, selection), + *check_standalone(scripts, boundaries.standalone, selection), + ] + + +def report(violations: Sequence[Violation]) -> int: + """Prints every violation on the standard error stream and answers with the exit status.""" + if not violations: + return 0 + + print("Layer boundary violation(s) found:", file=sys.stderr) + for kind, location in violations: + print(f" [forbidden: {kind}] {location}", file=sys.stderr) + + print(f"\nFound {len(violations)} violation(s) in total.", file=sys.stderr) + return 1 diff --git a/scripts/checks/language_keys.py b/src/sampletones_tools/checks/language_keys.py old mode 100755 new mode 100644 similarity index 75% rename from scripts/checks/language_keys.py rename to src/sampletones_tools/checks/language_keys.py index d6a55601c..d8c4f7f3c --- a/scripts/checks/language_keys.py +++ b/src/sampletones_tools/checks/language_keys.py @@ -1,49 +1,37 @@ -#!/usr/bin/env python3 - -""" -Checks the language file and the lookups that read it against each other, in both directions. - -Every lookup on a `LanguageManager` states a key, so the check reads each one and holds it against -`en.yaml`. A key spelled entirely from literals must name an entry, and every entry must be reachable -from some lookup — a key part arriving in a variable stands for each member of the enum it is -annotated with, which is why a dynamic part names a concrete element enum. An element enum is found -by what it derives from, so one lives wherever its domain lives and the check reads it there. - -Three things are reported: - broken lookup — a literal key the language file holds no entry for - unresolved part — a key part the check reads no values from - unreached entry — an entry no lookup asks for - -Usage: - python scripts/checks/language_keys.py -""" - -import argparse import importlib import inspect import sys from enum import Enum, EnumMeta from pathlib import Path from types import ModuleType -from typing import Callable, Dict, Final, List, Mapping, NamedTuple, Optional, Sequence, Type +from typing import ( + Callable, + Dict, + Final, + List, + Mapping, + NamedTuple, + Optional, + Sequence, + Type, +) import yaml from sampletones_application.categories import hierarchy from sampletones_application.categories.abstract import AbstractElement -from sampletones_application.paths import LANG_EN from sampletones_application.tags.compose import TAG_SEPARATOR -from sampletones_shared.meta.source.classes import declared_subclasses -from sampletones_shared.meta.source.index import source_index -from sampletones_shared.meta.source.lookups import LookupSite, tree_lookups -from sampletones_shared.meta.source.modules import discover_modules, module_name -from sampletones_shared.meta.source.packages import package_directory -from sampletones_shared.meta.source.values import EnumMembers, EnumTable from sampletones_shared.paths.source import SOURCE_ROOT +from sampletones_tools.checks.source.classes import declared_subclasses +from sampletones_tools.checks.source.index import source_index +from sampletones_tools.checks.source.lookups import LookupSite, tree_lookups +from sampletones_tools.checks.source.modules import discover_modules, module_name +from sampletones_tools.checks.source.packages import source_package_directory +from sampletones_tools.checks.source.values import EnumMembers, EnumTable EnumPredicate = Callable[[object], bool] -APPLICATION_PACKAGE: Final[Path] = package_directory("sampletones_application") +APPLICATION_PACKAGE: Final[Path] = source_package_directory("sampletones_application") RECEIVER_TYPE: Final[str] = "LanguageManager" ELEMENT_BASE: Final[str] = AbstractElement.__name__ @@ -151,7 +139,10 @@ def entry_key(line: str) -> Optional[str]: def entry_lines(path: Path) -> Dict[str, int]: """Where each key sits in the language file, read line by line.""" lines: Dict[str, int] = {} - for number, line in enumerate(path.read_text(encoding=LANGUAGE_ENCODING).splitlines(), start=FIRST_LINE): + for number, line in enumerate( + path.read_text(encoding=LANGUAGE_ENCODING).splitlines(), + start=FIRST_LINE, + ): key = entry_key(line) if key is not None: lines.setdefault(key, number) @@ -179,7 +170,10 @@ def language_entries(path: Path) -> Dict[str, int]: return {str(key): lines.get(str(key), FIRST_LINE) for key in raw} -def broken_lookups(sites: Sequence[LookupSite], entries: Mapping[str, int]) -> List[Finding]: +def broken_lookups( + sites: Sequence[LookupSite], + entries: Mapping[str, int], +) -> List[Finding]: """Every literal key a lookup asks for that the language file holds no entry for. Args: @@ -275,26 +269,8 @@ def check_language_keys(source: Path, language_file: Path) -> List[Finding]: ] -def main(argv: Sequence[str]) -> int: - """Report every disagreement between the language file and the lookups reading it.""" - parser = argparse.ArgumentParser( - description="Check language keys against the language file, in both directions.", - ) - parser.add_argument( - "--source", - type=Path, - default=SOURCE_ROOT, - help="root of the sources to read", - ) - parser.add_argument( - "--language-file", - type=Path, - default=LANG_EN, - help="language file to check against", - ) - arguments = parser.parse_args(list(argv)) - - findings = check_language_keys(arguments.source, arguments.language_file) +def report(findings: Sequence[Finding]) -> int: + """Prints every finding on the standard error stream and answers with the exit status.""" if not findings: return 0 @@ -304,7 +280,3 @@ def main(argv: Sequence[str]) -> int: print(f"\nFound {len(findings)} language key finding(s).", file=sys.stderr) return 1 - - -if __name__ == "__main__": - raise SystemExit(main(sys.argv[1:])) diff --git a/scripts/checks/palette_colors.py b/src/sampletones_tools/checks/palette_colors.py old mode 100755 new mode 100644 similarity index 72% rename from scripts/checks/palette_colors.py rename to src/sampletones_tools/checks/palette_colors.py index c3fa9e4cb..a756dcdf1 --- a/scripts/checks/palette_colors.py +++ b/src/sampletones_tools/checks/palette_colors.py @@ -1,19 +1,3 @@ -#!/usr/bin/env python3 - -""" -Checks that a color stays a palette token until the moment it is drawn with. - -`BaseColor.rgba` answers with the palette active right now, so a consumer that holds the -token follows a palette swap and one that stores the answer keeps the shade it read at -construction. The check reports the three ways that contract is lost: an attribute assigned the -resolved value, a theme color filled outside the palette bindings that record it, and a color -written into the shipped configuration as a literal instead of a palette token. - -Usage: - python scripts/checks/palette_colors.py # check the source tree and the config package -""" - -import argparse import ast import logging import re @@ -23,14 +7,12 @@ from pathlib import Path from typing import Final, List, NamedTuple, Tuple, Union -from sampletones_application.paths import PALETTES_DIRECTORY from sampletones_shared.logger import logger -from sampletones_shared.meta.source.modules import SourceModule, discover_modules -from sampletones_shared.meta.source.nodes import terminal_name -from sampletones_shared.meta.source.packages import package_directory -from sampletones_shared.paths.resources import CONFIG_DIRECTORY +from sampletones_tools.checks.source.modules import SourceModule, discover_modules +from sampletones_tools.checks.source.nodes import terminal_name +from sampletones_tools.checks.source.packages import source_package_directory -APPLICATION_PACKAGE: Final[Path] = package_directory("sampletones_application") +APPLICATION_PACKAGE: Final[Path] = source_package_directory("sampletones_application") HEX_COLOR: Final[re.Pattern[str]] = re.compile(r"[\"']#[0-9a-fA-F]{6}(?:[0-9a-fA-F]{2})?[\"']") @@ -195,54 +177,35 @@ def find_literal_colors(package: Path, palettes: Path) -> List[ColorFinding]: return [finding for path in paths if palettes not in path.parents for finding in literal_colors(path)] -def main(argv: Sequence[str]) -> int: - """Report every color the application stores resolved or the configuration writes out.""" +def check_colors(package: Path, config: Path, palettes: Path) -> List[ColorFinding]: + """Every color the application stores resolved or the configuration writes out. + + Args: + package: Package whose color reads are checked. + config: Shipped configuration package whose colors must name palette tokens. + palettes: Directory holding the palettes, where color values belong. + Returns: + List[ColorFinding]: The detached colors of the package, then the literal colors of the + configuration. + """ logger.set_level(level=logging.ERROR) bindings_module, theme_color_helper = dpg_module_helper() - - parser = argparse.ArgumentParser( - description="Check that a color stays a palette token until it is drawn with.", - ) - parser.add_argument( - "--package", - type=Path, - default=APPLICATION_PACKAGE, - help="package whose color reads to check", - ) - parser.add_argument( - "--config", - type=Path, - default=CONFIG_DIRECTORY, - help="shipped configuration package whose colors must name palette tokens", - ) - parser.add_argument( - "--palettes", - type=Path, - default=PALETTES_DIRECTORY, - help="directory holding the palettes, where color values belong", - ) - arguments = parser.parse_args(list(argv)) - findings = find_detached_colors( - arguments.package, + package, bindings_module=bindings_module, theme_color_helper=theme_color_helper, ) - findings.extend( - find_literal_colors( - arguments.config, - arguments.palettes, - ) - ) + findings.extend(find_literal_colors(config, palettes)) + return findings + +def report(findings: Sequence[ColorFinding]) -> int: + """Prints every finding on the standard error stream and answers with the exit status.""" if not findings: return 0 - print( - "Color(s) that stop following the active palette:", - file=sys.stderr, - ) + print("Color(s) that stop following the active palette:", file=sys.stderr) for location, message in findings: print(f" {location}: {message}", file=sys.stderr) @@ -251,7 +214,3 @@ def main(argv: Sequence[str]) -> int: file=sys.stderr, ) return 1 - - -if __name__ == "__main__": - raise SystemExit(main(sys.argv[1:])) diff --git a/src/sampletones_tools/checks/paths.py b/src/sampletones_tools/checks/paths.py new file mode 100644 index 000000000..becbb4168 --- /dev/null +++ b/src/sampletones_tools/checks/paths.py @@ -0,0 +1,6 @@ +from pathlib import Path +from typing import Final + +from sampletones_shared.paths.source import REPOSITORY_ROOT + +SCRIPTS_ROOT: Final[Path] = REPOSITORY_ROOT / "scripts" diff --git a/src/sampletones_tools/checks/registry.py b/src/sampletones_tools/checks/registry.py new file mode 100644 index 000000000..b1cc2af33 --- /dev/null +++ b/src/sampletones_tools/checks/registry.py @@ -0,0 +1,20 @@ +from typing import Final, Tuple + +from sampletones_shared.command import Command +from sampletones_tools.checks.commands.import_boundary import IMPORT_BOUNDARY +from sampletones_tools.checks.commands.language_keys import LANGUAGE_KEYS +from sampletones_tools.checks.commands.palette_colors import PALETTE_COLORS +from sampletones_tools.checks.commands.rendered_literals import RENDERED_LITERALS +from sampletones_tools.checks.commands.shortcut_actions import SHORTCUT_ACTIONS +from sampletones_tools.checks.commands.tag_names import TAG_NAMES +from sampletones_tools.checks.commands.unused_tags import UNUSED_TAGS + +GATES: Final[Tuple[Command, ...]] = ( + IMPORT_BOUNDARY, + LANGUAGE_KEYS, + PALETTE_COLORS, + RENDERED_LITERALS, + SHORTCUT_ACTIONS, + TAG_NAMES, + UNUSED_TAGS, +) diff --git a/scripts/checks/rendered_literals.py b/src/sampletones_tools/checks/rendered_literals.py old mode 100755 new mode 100644 similarity index 63% rename from scripts/checks/rendered_literals.py rename to src/sampletones_tools/checks/rendered_literals.py index 954acc18b..c198fa957 --- a/scripts/checks/rendered_literals.py +++ b/src/sampletones_tools/checks/rendered_literals.py @@ -1,31 +1,10 @@ -#!/usr/bin/env python3 - -""" -Checks that no case holds text it rendered itself against a literal it spelled out. - -Rendering a value and comparing the result with a written-out string pins whatever decided that -rendering. `str(path)` reads one way on Windows and another elsewhere, and a formatted setting reads -whatever the build ships, so a case written that way passes where it was authored and fails where it -is run next. Comparing the values instead — a `Path` against a `Path`, a setting against the -configuration it comes from — holds on every platform and survives every tuning. - -Equality is what pins a rendering entire, so that is what the check reads; a case holding a -fragment picks one clear of anything a platform decides. Every hit therefore reads under one of two -rules in `guidelines.md`: a case assumes no one platform, or a shipped value is a choice rather than -a contract. - -Usage: - python scripts/checks/rendered_literals.py -""" - -import argparse import ast import sys from pathlib import Path from typing import Final, Iterator, List, NamedTuple, Sequence, Tuple, Type -from sampletones_shared.meta.source.modules import SourceModule, discover_modules from sampletones_shared.paths.source import REPOSITORY_ROOT +from sampletones_tools.checks.source.modules import SourceModule, discover_modules TEST_ROOTS: Final[Tuple[Path, ...]] = (REPOSITORY_ROOT / "tests",) @@ -107,28 +86,25 @@ def findings(modules: Sequence[SourceModule]) -> List[Finding]: return [finding for module in modules for finding in module_findings(module)] -def main(argv: Sequence[str]) -> int: - """Report every case comparing text it rendered with a literal it spelled out.""" - parser = argparse.ArgumentParser( - description="Check that no case holds rendered text against a spelled-out literal.", - ) - parser.add_argument( - "--tests", - type=Path, - action="append", - dest="roots", - help="directory of cases to read, repeatable", - ) - arguments = parser.parse_args(list(argv)) +def check_cases(roots: Sequence[Path]) -> List[Finding]: + """Every case under the roots comparing text it rendered with a literal it spelled out.""" + return findings(discover_modules(roots)) - roots: Tuple[Path, ...] = tuple(arguments.roots or TEST_ROOTS) - found = findings(discover_modules(roots)) + +def report(found: Sequence[Finding]) -> int: + """Prints every finding on the standard error stream and answers with the exit status.""" if not found: return 0 - print("Case(s) holding rendered text against a spelled-out literal:", file=sys.stderr) + print( + "Case(s) holding rendered text against a spelled-out literal:", + file=sys.stderr, + ) for location, rendering in found: - print(f" {location}: {rendering} compared with a written-out string", file=sys.stderr) + print( + f" {location}: {rendering} compared with a written-out string", + file=sys.stderr, + ) print( f"\nFound {len(found)} such comparison(s). Compare the values rather than their text: " @@ -136,7 +112,3 @@ def main(argv: Sequence[str]) -> int: file=sys.stderr, ) return 1 - - -if __name__ == "__main__": - raise SystemExit(main(sys.argv[1:])) diff --git a/scripts/checks/shortcut_actions.py b/src/sampletones_tools/checks/shortcut_actions.py old mode 100755 new mode 100644 similarity index 78% rename from scripts/checks/shortcut_actions.py rename to src/sampletones_tools/checks/shortcut_actions.py index b8ff915ed..b44550f0a --- a/scripts/checks/shortcut_actions.py +++ b/src/sampletones_tools/checks/shortcut_actions.py @@ -1,29 +1,3 @@ -#!/usr/bin/env python3 - -""" -Checks that every action a key or a menu item reaches is declared the whole way through. - -An action is one `ShortcutId`, and naming it is the first of the links it needs: every shipped -keybinding scheme states the combination that fires it, `KeybindingActionElements` names it so the -keybindings editor can list it, and an application-scope action states the call it makes — either -its own entry in the shell's binding map, or membership of a family that maps a whole enum onto -one call. - -The scheme a build loads validates itself as it loads, so this check covers what that validation -cannot reach: the schemes a platform other than this one ships, the editor's vocabulary, and the -call behind a menu item, which otherwise goes missing until the application is constructed. - -Four things are reported: - unanswered action — an action a shipped scheme states no combination for - unnamed action — a rebindable action `KeybindingActionElements` has no member for - uncalled action — an application-scope action no binding and no family answers - stale entry — a name a scheme or the editor states that no action carries - -Usage: - python scripts/checks/shortcut_actions.py -""" - -import argparse import ast import sys from pathlib import Path @@ -53,12 +27,12 @@ ShortcutCategory, ShortcutId, ) -from sampletones_shared.meta.source.modules import SourceModule, parse_module -from sampletones_shared.meta.source.packages import package_directory +from sampletones_tools.checks.source.modules import SourceModule, parse_module +from sampletones_tools.checks.source.packages import source_package_directory SHORTCUTS_MODULE: Final[Path] = Path(ids_module.__file__) ELEMENTS_MODULE: Final[Path] = Path(settings_module.__file__) -SHELL_MODULE: Final[Path] = package_directory("sampletones_application") / "shell.py" +SHELL_MODULE: Final[Path] = source_package_directory("sampletones_application") / "shell.py" SCHEME_ENCODING: Final[str] = "utf-8" SCHEME_BINDINGS_FIELD: Final[str] = "bindings" @@ -234,14 +208,8 @@ def check() -> List[Finding]: ] -def main(argv: Sequence[str]) -> int: - """Report every action left short of a combination, a name, or the call it makes.""" - parser = argparse.ArgumentParser( - description="Check that every action is declared the whole way through.", - ) - parser.parse_args(list(argv)) - - findings = check() +def report(findings: Sequence[Finding]) -> int: + """Prints every finding on the standard error stream and answers with the exit status.""" if not findings: return 0 @@ -254,7 +222,3 @@ def main(argv: Sequence[str]) -> int: file=sys.stderr, ) return 1 - - -if __name__ == "__main__": - raise SystemExit(main(sys.argv[1:])) diff --git a/tests/unit/sampletones_core/calibration/referee/__init__.py b/src/sampletones_tools/checks/source/__init__.py similarity index 100% rename from tests/unit/sampletones_core/calibration/referee/__init__.py rename to src/sampletones_tools/checks/source/__init__.py diff --git a/src/sampletones_shared/meta/source/annotations.py b/src/sampletones_tools/checks/source/annotations.py similarity index 97% rename from src/sampletones_shared/meta/source/annotations.py rename to src/sampletones_tools/checks/source/annotations.py index 5994feb7e..e71278bfe 100644 --- a/src/sampletones_shared/meta/source/annotations.py +++ b/src/sampletones_tools/checks/source/annotations.py @@ -1,7 +1,7 @@ import ast from typing import Final, FrozenSet, Optional, Tuple -from sampletones_shared.meta.source.nodes import slice_elements, terminal_name +from sampletones_tools.checks.source.nodes import slice_elements, terminal_name ANNOTATION_WRAPPERS: Final[FrozenSet[str]] = frozenset({"Annotated", "ClassVar", "Final", "Optional"}) diff --git a/tests/unit/sampletones_player/driver/assembler/__init__.py b/src/sampletones_tools/checks/source/bindings/__init__.py similarity index 100% rename from tests/unit/sampletones_player/driver/assembler/__init__.py rename to src/sampletones_tools/checks/source/bindings/__init__.py diff --git a/src/sampletones_shared/meta/source/bindings/containers.py b/src/sampletones_tools/checks/source/bindings/containers.py similarity index 96% rename from src/sampletones_shared/meta/source/bindings/containers.py rename to src/sampletones_tools/checks/source/bindings/containers.py index 0e9d81e5e..75c3d1bf8 100644 --- a/src/sampletones_shared/meta/source/bindings/containers.py +++ b/src/sampletones_tools/checks/source/bindings/containers.py @@ -1,8 +1,8 @@ import ast from typing import Dict, Final, FrozenSet, Mapping, NamedTuple, Optional, Tuple -from sampletones_shared.meta.source.annotations import annotation_item_types -from sampletones_shared.meta.source.nodes import expression_spelling +from sampletones_tools.checks.source.annotations import annotation_item_types +from sampletones_tools.checks.source.nodes import expression_spelling KEY_ACCESSOR: Final[str] = "keys" VALUE_ACCESSOR: Final[str] = "values" diff --git a/src/sampletones_shared/meta/source/bindings/environment.py b/src/sampletones_tools/checks/source/bindings/environment.py similarity index 100% rename from src/sampletones_shared/meta/source/bindings/environment.py rename to src/sampletones_tools/checks/source/bindings/environment.py diff --git a/src/sampletones_shared/meta/source/bindings/scopes.py b/src/sampletones_tools/checks/source/bindings/scopes.py similarity index 94% rename from src/sampletones_shared/meta/source/bindings/scopes.py rename to src/sampletones_tools/checks/source/bindings/scopes.py index 38362f18c..95d3af019 100644 --- a/src/sampletones_shared/meta/source/bindings/scopes.py +++ b/src/sampletones_tools/checks/source/bindings/scopes.py @@ -2,21 +2,21 @@ from dataclasses import dataclass from typing import Dict, List, NamedTuple, Optional -from sampletones_shared.meta.source.bindings.containers import ( +from sampletones_tools.checks.source.bindings.containers import ( ItemTypes, container_item_types, iterated_container, iterated_types, ) -from sampletones_shared.meta.source.bindings.environment import TypeEnvironment -from sampletones_shared.meta.source.bindings.statements import ( +from sampletones_tools.checks.source.bindings.environment import TypeEnvironment +from sampletones_tools.checks.source.bindings.statements import ( AliasStatement, LoopStatement, Statement, TypeStatement, read_statement, ) -from sampletones_shared.meta.source.nodes import expression_spelling, is_attribute_spelling, nested_scopes, own_nodes +from sampletones_tools.checks.source.nodes import expression_spelling, is_attribute_spelling, nested_scopes, own_nodes @dataclass(frozen=True) diff --git a/src/sampletones_shared/meta/source/bindings/statements.py b/src/sampletones_tools/checks/source/bindings/statements.py similarity index 95% rename from src/sampletones_shared/meta/source/bindings/statements.py rename to src/sampletones_tools/checks/source/bindings/statements.py index cadd9a9a2..7d4d55182 100644 --- a/src/sampletones_shared/meta/source/bindings/statements.py +++ b/src/sampletones_tools/checks/source/bindings/statements.py @@ -1,8 +1,8 @@ import ast from typing import NamedTuple, Optional, Union -from sampletones_shared.meta.source.annotations import annotation_type_name -from sampletones_shared.meta.source.nodes import expression_spelling, terminal_name +from sampletones_tools.checks.source.annotations import annotation_type_name +from sampletones_tools.checks.source.nodes import expression_spelling, terminal_name class TypeStatement(NamedTuple): diff --git a/src/sampletones_shared/meta/source/classes.py b/src/sampletones_tools/checks/source/classes.py similarity index 93% rename from src/sampletones_shared/meta/source/classes.py rename to src/sampletones_tools/checks/source/classes.py index 02bac0a34..f4f1483b8 100644 --- a/src/sampletones_shared/meta/source/classes.py +++ b/src/sampletones_tools/checks/source/classes.py @@ -1,7 +1,7 @@ import ast from typing import List -from sampletones_shared.meta.source.nodes import terminal_name +from sampletones_tools.checks.source.nodes import terminal_name def declared_subclasses(tree: ast.Module, base: str) -> List[str]: diff --git a/src/sampletones_shared/meta/source/constants.py b/src/sampletones_tools/checks/source/constants.py similarity index 100% rename from src/sampletones_shared/meta/source/constants.py rename to src/sampletones_tools/checks/source/constants.py diff --git a/src/sampletones_shared/meta/source/index.py b/src/sampletones_tools/checks/source/index.py similarity index 85% rename from src/sampletones_shared/meta/source/index.py rename to src/sampletones_tools/checks/source/index.py index b61822453..8c670ac28 100644 --- a/src/sampletones_shared/meta/source/index.py +++ b/src/sampletones_tools/checks/source/index.py @@ -2,9 +2,9 @@ from dataclasses import dataclass from typing import Dict, Iterable, Mapping, Tuple -from sampletones_shared.meta.source.bindings.containers import container_item_types -from sampletones_shared.meta.source.constants import module_constants -from sampletones_shared.meta.source.modules import SourceModule +from sampletones_tools.checks.source.bindings.containers import container_item_types +from sampletones_tools.checks.source.constants import module_constants +from sampletones_tools.checks.source.modules import SourceModule @dataclass(frozen=True) diff --git a/src/sampletones_shared/meta/source/lookups.py b/src/sampletones_tools/checks/source/lookups.py similarity index 84% rename from src/sampletones_shared/meta/source/lookups.py rename to src/sampletones_tools/checks/source/lookups.py index 6266cd6d9..43d8abc7b 100644 --- a/src/sampletones_shared/meta/source/lookups.py +++ b/src/sampletones_tools/checks/source/lookups.py @@ -3,11 +3,15 @@ from itertools import product from typing import Iterable, List, Sequence, Tuple -from sampletones_shared.meta.source.bindings.scopes import module_scopes -from sampletones_shared.meta.source.index import SourceIndex -from sampletones_shared.meta.source.modules import SourceModule -from sampletones_shared.meta.source.subscripts import SubscriptSite, find_subscripts -from sampletones_shared.meta.source.values import EnumTable, ResolvedValues, ValueResolver +from sampletones_tools.checks.source.bindings.scopes import module_scopes +from sampletones_tools.checks.source.index import SourceIndex +from sampletones_tools.checks.source.modules import SourceModule +from sampletones_tools.checks.source.subscripts import SubscriptSite, find_subscripts +from sampletones_tools.checks.source.values import ( + EnumTable, + ResolvedValues, + ValueResolver, +) @dataclass(frozen=True) @@ -29,7 +33,10 @@ def resolved(self) -> bool: return not self.unresolved_parts -def composed_values(resolutions: Sequence[ResolvedValues], separator: str) -> Tuple[str, ...]: +def composed_values( + resolutions: Sequence[ResolvedValues], + separator: str, +) -> Tuple[str, ...]: """Every string a lookup's resolved parts spell together. A lookup of one part spells that part's values. A lookup of several parts spells one string per @@ -107,7 +114,18 @@ def module_lookups( enums=enums, constants=index.constants, ) - sites.extend(lookup_site(module, site, resolver, separator) for site in find_subscripts(scope.node, receivers)) + sites.extend( + lookup_site( + module, + site, + resolver, + separator, + ) + for site in find_subscripts( + scope.node, + receivers, + ) + ) return sites diff --git a/src/sampletones_shared/meta/source/modules.py b/src/sampletones_tools/checks/source/modules.py similarity index 98% rename from src/sampletones_shared/meta/source/modules.py rename to src/sampletones_tools/checks/source/modules.py index d2b5efd7e..bdb2a23ac 100644 --- a/src/sampletones_shared/meta/source/modules.py +++ b/src/sampletones_tools/checks/source/modules.py @@ -3,7 +3,7 @@ from pathlib import Path from typing import Final, Iterable, List -from sampletones_shared.meta.source.nodes import PositionedNode +from sampletones_tools.checks.source.nodes import PositionedNode SOURCE_PATTERN: Final[str] = "*.py" SOURCE_ENCODING: Final[str] = "utf-8-sig" diff --git a/src/sampletones_shared/meta/source/nodes.py b/src/sampletones_tools/checks/source/nodes.py similarity index 100% rename from src/sampletones_shared/meta/source/nodes.py rename to src/sampletones_tools/checks/source/nodes.py diff --git a/src/sampletones_shared/meta/source/packages.py b/src/sampletones_tools/checks/source/packages.py similarity index 93% rename from src/sampletones_shared/meta/source/packages.py rename to src/sampletones_tools/checks/source/packages.py index f19a3899a..4762e1816 100644 --- a/src/sampletones_shared/meta/source/packages.py +++ b/src/sampletones_tools/checks/source/packages.py @@ -3,7 +3,7 @@ from sampletones_shared.paths.source import SOURCE_ROOT -def package_directory(name: str, *parts: str) -> Path: +def source_package_directory(name: str, *parts: str) -> Path: """The directory a package occupies, named by path rather than by import. A source check reads the tree it checks, so taking a package from the source root keeps the diff --git a/src/sampletones_shared/meta/source/references.py b/src/sampletones_tools/checks/source/references.py similarity index 100% rename from src/sampletones_shared/meta/source/references.py rename to src/sampletones_tools/checks/source/references.py diff --git a/src/sampletones_shared/meta/source/subscripts.py b/src/sampletones_tools/checks/source/subscripts.py similarity index 94% rename from src/sampletones_shared/meta/source/subscripts.py rename to src/sampletones_tools/checks/source/subscripts.py index d942ea362..4121b15e5 100644 --- a/src/sampletones_shared/meta/source/subscripts.py +++ b/src/sampletones_tools/checks/source/subscripts.py @@ -2,7 +2,7 @@ from dataclasses import dataclass from typing import Collection, List, Tuple -from sampletones_shared.meta.source.nodes import expression_spelling, own_nodes, slice_elements +from sampletones_tools.checks.source.nodes import expression_spelling, own_nodes, slice_elements @dataclass(frozen=True) diff --git a/src/sampletones_shared/meta/source/values.py b/src/sampletones_tools/checks/source/values.py similarity index 96% rename from src/sampletones_shared/meta/source/values.py rename to src/sampletones_tools/checks/source/values.py index 4abaf4cb7..6a1c1afe2 100644 --- a/src/sampletones_shared/meta/source/values.py +++ b/src/sampletones_tools/checks/source/values.py @@ -2,8 +2,8 @@ from dataclasses import dataclass from typing import Final, FrozenSet, Mapping, Tuple -from sampletones_shared.meta.source.bindings.environment import TypeEnvironment -from sampletones_shared.meta.source.nodes import expression_spelling, terminal_name +from sampletones_tools.checks.source.bindings.environment import TypeEnvironment +from sampletones_tools.checks.source.nodes import expression_spelling, terminal_name EnumMembers = Mapping[str, str] EnumTable = Mapping[str, EnumMembers] diff --git a/scripts/checks/tag_names.py b/src/sampletones_tools/checks/tag_names.py old mode 100755 new mode 100644 similarity index 73% rename from scripts/checks/tag_names.py rename to src/sampletones_tools/checks/tag_names.py index e2757858d..8d3ddb4c2 --- a/scripts/checks/tag_names.py +++ b/src/sampletones_tools/checks/tag_names.py @@ -1,34 +1,32 @@ -#!/usr/bin/env python3 - -""" -Checks that every tag constant is named after the tag it composes. - -`TAG_GLOBAL_WINDOW_MAIN = TagName(Page.GLOBAL, Panel.IMPLICIT, Widget.WINDOW, "main")` composes the -tag `global.window.main`, so its name is that tag upper-cased with each separator turned into an -underscore, behind the `TAG_` prefix. Reading the name therefore states the tag, and reading the tag -states where its constant lives. - -Usage: - python scripts/checks/tag_names.py [files...] # check the given modules - python scripts/checks/tag_names.py --all # check every module of the tags package -""" - -import argparse import ast import sys from enum import StrEnum from pathlib import Path -from typing import Dict, Final, List, NamedTuple, Optional, Sequence, Tuple, Type, TypeVar +from typing import ( + Dict, + Final, + List, + NamedTuple, + Optional, + Sequence, + Tuple, + Type, + TypeVar, +) from sampletones_application.categories.hierarchy import Page, Panel, Widget from sampletones_application.categories.key.tag import TagName from sampletones_application.tags.compose import TAG_SEPARATOR -from sampletones_shared.meta.source.constants import ModuleConstant, module_constants -from sampletones_shared.meta.source.modules import SourceModule, discover_modules, parse_module -from sampletones_shared.meta.source.nodes import terminal_name -from sampletones_shared.meta.source.packages import package_directory +from sampletones_tools.checks.source.constants import ModuleConstant, module_constants +from sampletones_tools.checks.source.modules import ( + SourceModule, + discover_modules, + parse_module, +) +from sampletones_tools.checks.source.nodes import terminal_name +from sampletones_tools.checks.source.packages import source_package_directory -TAGS_PACKAGE: Final[Path] = package_directory("sampletones_application", "tags") +TAGS_PACKAGE: Final[Path] = source_package_directory("sampletones_application", "tags") TAG_PREFIX: Final[str] = "TAG" TAG_NAME_CLASS: Final[str] = "TagName" @@ -98,7 +96,10 @@ def call_arguments(call: ast.Call) -> Optional[Dict[str, ast.expr]]: return arguments -def hierarchy_member(node: ast.expr, enum: Type[EnumMember]) -> Optional[EnumMember]: +def hierarchy_member( + node: ast.expr, + enum: Type[EnumMember], +) -> Optional[EnumMember]: """The hierarchy member an argument names, such as `Page.GLOBAL`. Args: @@ -144,7 +145,10 @@ def composed_tag(arguments: Dict[str, ast.expr]) -> Optional[TagName]: return TagName(page, panel, widget, element) -def check_constant(module: SourceModule, constant: ModuleConstant) -> Optional[TagFinding]: +def check_constant( + module: SourceModule, + constant: ModuleConstant, +) -> Optional[TagFinding]: """Checks one constant of a tags module, where it is bound to a tag. Args: @@ -202,27 +206,14 @@ def check_modules(modules: Sequence[SourceModule]) -> List[TagFinding]: return [finding for module in modules for finding in check_module(module)] -def main(argv: Sequence[str]) -> int: - """Report any tag constant whose name departs from the tag it composes.""" - parser = argparse.ArgumentParser( - description="Check that tag constant names state the tags they compose.", - ) - parser.add_argument( - "files", - nargs="*", - type=Path, - help="modules to check", - ) - parser.add_argument( - "--all", - action="store_true", - help=f"check every module under {TAGS_PACKAGE.name}/ instead of named files", - ) - arguments = parser.parse_args(list(argv)) - - files: List[Path] = arguments.files - modules = discover_modules([TAGS_PACKAGE]) if arguments.all else [parse_module(path) for path in files] - findings = check_modules(modules) +def check_tags(files: Sequence[Path], everything: bool) -> List[TagFinding]: + """The findings over the named modules, or over the whole tags package when ``everything``.""" + modules = discover_modules([TAGS_PACKAGE]) if everything else [parse_module(path) for path in files] + return check_modules(modules) + + +def report(findings: Sequence[TagFinding]) -> int: + """Prints every finding on the standard error stream and answers with the exit status.""" if not findings: return 0 @@ -232,7 +223,3 @@ def main(argv: Sequence[str]) -> int: print(f"\nFound {len(findings)} tag name(s) to fix.", file=sys.stderr) return 1 - - -if __name__ == "__main__": - raise SystemExit(main(sys.argv[1:])) diff --git a/scripts/checks/unused_tags.py b/src/sampletones_tools/checks/unused_tags.py old mode 100755 new mode 100644 similarity index 58% rename from scripts/checks/unused_tags.py rename to src/sampletones_tools/checks/unused_tags.py index c2a87b5ce..9cc7e1ee8 --- a/scripts/checks/unused_tags.py +++ b/src/sampletones_tools/checks/unused_tags.py @@ -1,34 +1,20 @@ -#!/usr/bin/env python3 - -""" -Checks that every tag fragment the tags package declares is read somewhere. - -A tag or a suffix nobody reads still reads as part of the interface vocabulary, so it invites a -second constant for the same widget. The check counts reads across the sources, the tests, and the -scripts: a fragment feeding another fragment's value counts as read, while an import alone does not, -which is what makes a re-exported yet unread fragment visible. - -Usage: - python scripts/checks/unused_tags.py -""" - -import argparse import sys from collections import Counter from pathlib import Path from typing import Dict, Final, List, NamedTuple, Sequence, Tuple -from sampletones_shared.meta.source.constants import module_constants -from sampletones_shared.meta.source.modules import SourceModule, discover_modules -from sampletones_shared.meta.source.packages import package_directory -from sampletones_shared.meta.source.references import count_identifier_loads from sampletones_shared.paths.source import REPOSITORY_ROOT, SOURCE_ROOT +from sampletones_tools.checks.paths import SCRIPTS_ROOT +from sampletones_tools.checks.source.constants import module_constants +from sampletones_tools.checks.source.modules import SourceModule, discover_modules +from sampletones_tools.checks.source.packages import source_package_directory +from sampletones_tools.checks.source.references import count_identifier_loads -TAGS_PACKAGE: Final[Path] = package_directory("sampletones_application", "tags") +TAGS_PACKAGE: Final[Path] = source_package_directory("sampletones_application", "tags") REFERENCE_ROOTS: Final[Tuple[Path, ...]] = ( SOURCE_ROOT, REPOSITORY_ROOT / "tests", - REPOSITORY_ROOT / "scripts", + SCRIPTS_ROOT, ) FRAGMENT_PREFIXES: Final[Tuple[str, ...]] = ("TAG_", "SUF_", "PRE_") @@ -98,31 +84,15 @@ def unread_fragments( return [fragment for fragment in fragments if counts.get(fragment.name, 0) == 0] -def main(argv: Sequence[str]) -> int: - """Report every tag fragment the repository declares and never reads.""" - parser = argparse.ArgumentParser( - description="Check that every declared tag fragment is read somewhere.", - ) - parser.add_argument( - "--tags", - type=Path, - default=TAGS_PACKAGE, - help="package declaring the tag fragments", - ) - parser.add_argument( - "--reference-root", - type=Path, - action="append", - dest="reference_roots", - help="directory to count reads in, repeatable", - ) - arguments = parser.parse_args(list(argv)) - - tags: Path = arguments.tags - reference_roots: Tuple[Path, ...] = tuple(arguments.reference_roots or REFERENCE_ROOTS) +def check_reads(tags: Path, reference_roots: Sequence[Path]) -> List[Fragment]: + """Every tag fragment the package declares and the reference roots never read.""" fragments = declared_fragments(discover_modules([tags])) counts = reference_counts(discover_modules(reference_roots)) - unread = unread_fragments(fragments, counts) + return unread_fragments(fragments, counts) + + +def report(unread: Sequence[Fragment]) -> int: + """Prints every unread fragment on the standard error stream and answers with the exit status.""" if not unread: return 0 @@ -132,7 +102,3 @@ def main(argv: Sequence[str]) -> int: print(f"\nFound {len(unread)} unread tag fragment(s).", file=sys.stderr) return 1 - - -if __name__ == "__main__": - raise SystemExit(main(sys.argv[1:])) diff --git a/tests/unit/sampletones_player/trace/__init__.py b/src/sampletones_tools/codec/__init__.py similarity index 100% rename from tests/unit/sampletones_player/trace/__init__.py rename to src/sampletones_tools/codec/__init__.py diff --git a/src/sampletones_tools/codec/command.py b/src/sampletones_tools/codec/command.py new file mode 100644 index 000000000..29d0634f8 --- /dev/null +++ b/src/sampletones_tools/codec/command.py @@ -0,0 +1,142 @@ +from argparse import ArgumentParser, Namespace +from dataclasses import dataclass +from pathlib import Path +from typing import Final, Optional, Tuple + +from sampletones_shared.command import Command + +NAME: Final[str] = "codec" +HELP: Final[str] = "measure the song codec on the synthetic corpus or on the songs named" +ACTION_FIELD: Final[str] = "action" +ACTION_METAVAR: Final[str] = "" +REPORT: Final[str] = "report" +REPORT_HELP: Final[str] = "compress the synthetic corpus under every layer of the codec and write the report tables" +REPORT_OUTPUT_HELP: Final[str] = "the directory the report is written into, created when missing" +STUDY: Final[str] = "study" +STUDY_HELP: Final[str] = "encode the projects and stems named under every candidate change, with a verdict each" +MANIFEST_HELP: Final[str] = ( + "a manifest a run wrote, measured again; its lengthening and variants stand in for the options below" +) +PROJECT_HELP: Final[str] = ( + "a project file to measure, repeatable; a run names at least one project, reconstruction or manifest" +) +RECONSTRUCTION_HELP: Final[str] = ( + "a stem file, or a directory of stems, to measure, repeatable; named sources replace a manifest's own" +) +OUTPUT_HELP: Final[str] = ( + "the directory the run writes into; without it, a timestamped directory under Documents/SampleToNES/compression" +) +LENGTHEN_HELP: Final[str] = "seconds each project's lengthened copy lasts" +VARIANTS_HELP: Final[str] = ( + "variants every song is encoded under, comma separated; without it every one, and the baseline always runs" +) +DEFAULT_LENGTHEN_SECONDS: Final[int] = 180 +NO_SOURCE: Final[str] = ( + "The study reads the files it is given: name a project with --project, a stem file or a directory of " + "stems with --reconstruction, or a manifest a run wrote with --manifest." +) + + +@dataclass(frozen=True) +class ReportArguments: + """What a report run is given: the directory the tables are written into.""" + + output: Path + + +@dataclass(frozen=True) +class StudyArguments: + """What a study run is given, as written on the command line.""" + + manifest: Optional[Path] + projects: Tuple[Path, ...] + reconstructions: Tuple[Path, ...] + output: Optional[Path] + lengthen: int + variants: Optional[str] + + +def configure(parser: ArgumentParser) -> None: + actions = parser.add_subparsers(dest=ACTION_FIELD, metavar=ACTION_METAVAR, required=True) + report = actions.add_parser(REPORT, help=REPORT_HELP, description=REPORT_HELP) + report.add_argument("--output", "-o", type=Path, required=True, help=REPORT_OUTPUT_HELP) + study = actions.add_parser(STUDY, help=STUDY_HELP, description=STUDY_HELP) + study.add_argument("--manifest", type=Path, default=None, help=MANIFEST_HELP) + study.add_argument("--project", type=Path, action="append", dest="projects", default=[], help=PROJECT_HELP) + study.add_argument( + "--reconstruction", + type=Path, + action="append", + dest="reconstructions", + default=[], + help=RECONSTRUCTION_HELP, + ) + study.add_argument("--output", "-o", type=Path, default=None, help=OUTPUT_HELP) + study.add_argument("--lengthen", type=int, default=DEFAULT_LENGTHEN_SECONDS, help=LENGTHEN_HELP) + study.add_argument("--variants", type=str, default=None, help=VARIANTS_HELP) + + +def run(arguments: Namespace) -> int: + """Measures the codec the way the action describes.""" + if arguments.action == REPORT: + return _report(ReportArguments(output=arguments.output)) + + return _study( + StudyArguments( + manifest=arguments.manifest, + projects=tuple(arguments.projects), + reconstructions=tuple(arguments.reconstructions), + output=arguments.output, + lengthen=arguments.lengthen, + variants=arguments.variants, + ) + ) + + +def _report(given: ReportArguments) -> int: + from sampletones_tools.codec.report.session import run_report + + report = run_report(given.output) + for path in (report.csv_path, report.markdown_path): + print(f"Wrote {path}") + + return 0 + + +def _study(given: StudyArguments) -> int: + """Measures the sources a run names. + + Raises: + SystemExit: If the run names neither a source nor a manifest, a source or the manifest is + missing, the manifest is broken, the lengthening is below one second, or a variant is + unknown. + """ + if given.manifest is None and not given.projects and not given.reconstructions: + raise SystemExit(NO_SOURCE) + + from sampletones_shared.utils.validation import describe_failure + from sampletones_tools.codec.study.plan import plan_study + from sampletones_tools.codec.study.session import resolve_manifest, run_study, variant_names + + try: + manifest = resolve_manifest( + given.manifest, + projects=given.projects, + reconstructions=given.reconstructions, + lengthen_seconds=given.lengthen, + variants=variant_names(given.variants), + ) + plan = plan_study(manifest) + except ValueError as error: + raise SystemExit(describe_failure(error)) from error + + run_study(plan, given.output) + return 0 + + +CODEC: Final[Command] = Command( + name=NAME, + help=HELP, + configure=configure, + run=run, +) diff --git a/tests/unit/sampletones_shared/meta/import_boundary/__init__.py b/src/sampletones_tools/codec/report/__init__.py similarity index 100% rename from tests/unit/sampletones_shared/meta/import_boundary/__init__.py rename to src/sampletones_tools/codec/report/__init__.py diff --git a/tests/integration/nsf/corpus.py b/src/sampletones_tools/codec/report/corpus.py similarity index 90% rename from tests/integration/nsf/corpus.py rename to src/sampletones_tools/codec/report/corpus.py index db7d107d7..f220720a4 100644 --- a/tests/integration/nsf/corpus.py +++ b/src/sampletones_tools/codec/report/corpus.py @@ -30,7 +30,7 @@ from sampletones_player.compression.seeds import phrases_from_project from sampletones_player.song import Song from sampletones_shared.music import Tuning -from tests.integration.nsf.songs import RECORD_BYTES_PER_TICK, lengthened +from sampletones_tools.codec.report.songs import RECORD_BYTES_PER_TICK, lengthened ARRANGEMENT: Final[str] = "arrangement" LONG_ARRANGEMENT: Final[str] = "arrangement, three minutes" @@ -81,20 +81,20 @@ def _sample_project( def sample_entries( - instrument_catalog: Dict[str, Sample], + catalog: Dict[str, Sample], settings: ProjectSettings, ) -> Tuple[CorpusEntry, ...]: """Each catalog sample as a song of its own, played at the tuning it was reconstructed at. Args: - instrument_catalog: The samples the integration suite reads. + catalog: The samples of the synthetic corpus, by name. settings: The project settings a sample is seeded under. Returns: Tuple[CorpusEntry, ...]: One entry per sample, in catalog order. """ entries: List[CorpusEntry] = [] - for name, sample in instrument_catalog.items(): + for name, sample in catalog.items(): tuning = sample.reconstruction.config.library.tuning entries.append( CorpusEntry( @@ -225,21 +225,24 @@ def reconstruction_entry(name: str, seconds: int) -> CorpusEntry: ) -def build_corpus( - instrument_catalog: Dict[str, Sample], - integration_project: Project, +def corpus_entries( + catalog: Dict[str, Sample], + project: Project, ) -> Tuple[CorpusEntry, ...]: - """The songs the codec is measured on: each sample alone, and the arrangement at two lengths. + """The songs the codec is measured on: each sample alone, the arrangement at two lengths, and + a minute of dense reconstruction. Args: - instrument_catalog: The samples the integration suite reads. - integration_project: The arrangement those samples are played in. + catalog: The samples of the synthetic corpus, by name. + project: The arrangement those samples are played in. Returns: - Tuple[CorpusEntry, ...]: The samples first, then the arrangement, then the long one. + Tuple[CorpusEntry, ...]: The samples first, then the arrangement, the long one, and the + reconstruction. """ return ( - *sample_entries(instrument_catalog, integration_project.settings), - arrangement_entry(ARRANGEMENT, integration_project), - arrangement_entry(LONG_ARRANGEMENT, lengthened_arrangement(integration_project, TARGET_SECONDS)), + *sample_entries(catalog, project.settings), + arrangement_entry(ARRANGEMENT, project), + arrangement_entry(LONG_ARRANGEMENT, lengthened_arrangement(project, TARGET_SECONDS)), + reconstruction_entry(RECONSTRUCTION, RECONSTRUCTION_SECONDS), ) diff --git a/src/sampletones_tools/codec/report/encoding.py b/src/sampletones_tools/codec/report/encoding.py new file mode 100644 index 000000000..414c27ba7 --- /dev/null +++ b/src/sampletones_tools/codec/report/encoding.py @@ -0,0 +1,264 @@ +from dataclasses import dataclass +from time import process_time +from typing import Final, List, Sequence, Tuple + +from sampletones_player.compression.compressed import CompressedPlanes +from sampletones_player.compression.dictionary.table import phrase_table +from sampletones_player.compression.encode import STREAM_START, encode_planes +from sampletones_player.compression.matches.cache import MatchCache +from sampletones_player.compression.matches.index import PlaneIndex +from sampletones_player.compression.matches.matcher import PhraseMatcher +from sampletones_player.compression.options import CodecOptions +from sampletones_player.compression.parse.plane import parse_plane +from sampletones_player.compression.planes.song import SongPlanes +from sampletones_player.registers.streams import ChannelStreams +from sampletones_player.specification.registers import DUTY_CYCLE_SHIFT +from sampletones_tools.codec.report.corpus import CorpusEntry +from sampletones_tools.codec.report.rows import ReportRow + +LITERALS: Final[str] = "literals" +HOLDS: Final[str] = "holds" +INSTRUMENTS: Final[str] = "instruments" +TRANSPOSITION: Final[str] = "transposition" +SEARCH: Final[str] = "search" +RECORDS: Final[str] = "records" +REGISTER_PLANES: Final[str] = "register planes" +SPLIT_CONTROL: Final[str] = "split control" +CONTROL_LEVEL_MASK: Final[int] = 0x3F +HOLDS_OPTIONS: Final[CodecOptions] = CodecOptions( + holds=True, + phrases=False, + transposition=False, + search=False, +) + +PLANE_VARIANTS: Final[Tuple[Tuple[str, CodecOptions], ...]] = ( + ( + LITERALS, + CodecOptions( + holds=False, + phrases=False, + transposition=False, + search=False, + ), + ), + (HOLDS, HOLDS_OPTIONS), + ( + INSTRUMENTS, + CodecOptions( + holds=True, + phrases=True, + transposition=False, + search=False, + ), + ), + ( + TRANSPOSITION, + CodecOptions( + holds=True, + phrases=True, + transposition=True, + search=False, + ), + ), + ( + SEARCH, + CodecOptions( + holds=True, + phrases=True, + transposition=True, + search=True, + ), + ), +) + + +@dataclass(frozen=True) +class Encoding: + """One corpus song compressed under one variant of the codec. + + Attributes: + entry: The song compressed. + variant: The layers the codec was switched on with. + planes: The planes the song separates into. + compressed: What the encoder wrote. + seconds: The processor time the encoding took. + """ + + entry: CorpusEntry + variant: str + planes: SongPlanes + compressed: CompressedPlanes + seconds: float + + @property + def size(self) -> int: + """The bytes the dictionary, the streams and the pitch table take together.""" + return self.compressed.size + len(self.entry.pitches.data) + + @property + def streams(self) -> int: + """The bytes the token streams take, which is the part that grows with the song.""" + return sum(len(stream) for stream in self.compressed.streams) + + +def encode_corpus(entries: Sequence[CorpusEntry]) -> Tuple[Encoding, ...]: + """Compresses every corpus song under every variant of the codec, timing each encoding. + + Args: + entries: The songs to compress. + + Returns: + Tuple[Encoding, ...]: The encodings, song by song, each in the order of the variants. + """ + encodings: List[Encoding] = [] + for entry in entries: + planes = entry.planes + for variant, options in PLANE_VARIANTS: + started = process_time() + compressed = encode_planes( + planes, + entry.seeds, + options=options, + boundaries=frozenset(), + ) + encodings.append( + Encoding( + entry=entry, + variant=variant, + planes=planes, + compressed=compressed, + seconds=process_time() - started, + ) + ) + + return tuple(encodings) + + +def report_rows( + entries: Sequence[CorpusEntry], + encodings: Sequence[Encoding], + space: int, +) -> Tuple[ReportRow, ...]: + """The report's measurements: per song, the baselines and then each of its encodings. + + Args: + entries: The songs measured. + encodings: Their encodings, as `encode_corpus` returns them. + space: The program area a song is written into. + + Returns: + Tuple[ReportRow, ...]: The rows, in the order the report prints them. + """ + rows: List[ReportRow] = [] + for entry in entries: + rows.extend(_baseline_rows(entry, space)) + rows.extend(_encoded_row(encoding, space) for encoding in encodings if encoding.entry is entry) + + return tuple(rows) + + +def _encoded_row(encoding: Encoding, space: int) -> ReportRow: + entry = encoding.entry + return ReportRow( + corpus=entry.name, + variant=encoding.variant, + ticks=entry.song.ticks, + size=encoding.size, + variable=encoding.streams, + phrases=len(encoding.compressed.phrases), + dictionary=encoding.compressed.phrases.size, + seconds=encoding.seconds, + records=entry.records, + space=space, + ) + + +def _baseline_rows(entry: CorpusEntry, space: int) -> Tuple[ReportRow, ...]: + return ( + ReportRow( + corpus=entry.name, + variant=RECORDS, + ticks=entry.song.ticks, + size=entry.records, + variable=entry.records, + phrases=0, + dictionary=0, + seconds=0.0, + records=entry.records, + space=space, + ), + _measured_row( + entry, + REGISTER_PLANES, + _register_planes(entry.song.streams), + 0, + space, + ), + _measured_row( + entry, + SPLIT_CONTROL, + _split_control_planes(entry.planes), + len(entry.pitches.data), + space, + ), + ) + + +def _measured_row( + entry: CorpusEntry, + variant: str, + planes: Sequence[bytes], + fixed: int, + space: int, +) -> ReportRow: + started = process_time() + coded = _coded_size(planes, HOLDS_OPTIONS) + return ReportRow( + corpus=entry.name, + variant=variant, + ticks=entry.song.ticks, + size=fixed + coded, + variable=coded, + phrases=0, + dictionary=0, + seconds=process_time() - started, + records=entry.records, + space=space, + ) + + +def _register_planes(streams: ChannelStreams) -> Tuple[bytes, ...]: + return tuple( + bytes(tick.values[register] for tick in stream) + for stream in streams.padded + for register in range(len(stream[0].values)) + ) + + +def _split_control_planes(planes: SongPlanes) -> Tuple[bytes, ...]: + split: List[bytes] = [] + for channels in planes.ordered: + if channels in (planes.pulse1, planes.pulse2): + split.append(bytes(control >> DUTY_CYCLE_SHIFT for control in channels.control)) + split.append(bytes(control & CONTROL_LEVEL_MASK for control in channels.control)) + else: + split.append(channels.control) + + split.append(channels.value) + + return tuple(split) + + +def _coded_size(planes: Sequence[bytes], options: CodecOptions) -> int: + cache = MatchCache(PlaneIndex.from_plane(plane) for plane in planes) + table = phrase_table(()) + entries = frozenset({STREAM_START}) + return sum( + parse_plane( + PhraseMatcher(table, plane, cache), + options, + entries, + ).size + for plane in range(len(planes)) + ) diff --git a/tests/integration/nsf/report.py b/src/sampletones_tools/codec/report/rows.py similarity index 75% rename from tests/integration/nsf/report.py rename to src/sampletones_tools/codec/report/rows.py index c689926fe..408272fec 100644 --- a/tests/integration/nsf/report.py +++ b/src/sampletones_tools/codec/report/rows.py @@ -1,8 +1,9 @@ -import csv from dataclasses import dataclass from pathlib import Path from typing import Final, Sequence, Tuple +from sampletones_shared.utils.tables import Table + COLUMNS: Final[Tuple[str, ...]] = ( "corpus", "variant", @@ -81,26 +82,17 @@ def cells(self) -> Tuple[str, ...]: ) -def write_csv(rows: Sequence[ReportRow], path: Path) -> None: - """Writes the measurements as a table another tool reads. - - Args: - rows: The measurements, in the order they are reported. - path: Where the table is written. - """ - with path.open("w", encoding="utf-8", newline="") as handle: - writer = csv.writer(handle) - writer.writerow(COLUMNS) - for row in rows: - writer.writerow(row.cells) +def report_table(rows: Sequence[ReportRow]) -> Table: + """The measurements as the report's table, in the order they are reported.""" + return Table(columns=COLUMNS, rows=tuple(row.cells for row in rows)) -def write_markdown(rows: Sequence[ReportRow], path: Path, state: int) -> None: - """Writes the measurements as a table a reader reads. +def write_markdown(table: Table, path: Path, state: int) -> None: + """Writes the measurements as a document a reader reads. Args: - rows: The measurements, in the order they are reported. - path: Where the table is written. + table: The measurements. + path: Where the document is written. state: The zero-page bytes the decoder's plane state takes. """ lines = [ @@ -108,8 +100,6 @@ def write_markdown(rows: Sequence[ReportRow], path: Path, state: int) -> None: "", f"Decoder state: {state} bytes of zero page.", "", - "| " + " | ".join(COLUMNS) + " |", - "|" + "|".join("---" for _ in COLUMNS) + "|", + *table.markdown_lines(), ] - lines.extend("| " + " | ".join(row.cells) + " |" for row in rows) path.write_text("\n".join(lines) + "\n", encoding="utf-8") diff --git a/src/sampletones_tools/codec/report/session.py b/src/sampletones_tools/codec/report/session.py new file mode 100644 index 000000000..5a8d66019 --- /dev/null +++ b/src/sampletones_tools/codec/report/session.py @@ -0,0 +1,78 @@ +from dataclasses import dataclass +from pathlib import Path +from typing import Final, Sequence, Tuple + +from sampletones_player.driver.image import DriverImage +from sampletones_player.specification.compression import PLANE_COUNT, PLANE_STATE_SIZE +from sampletones_tools.codec.report.corpus import CorpusEntry, corpus_entries +from sampletones_tools.codec.report.encoding import Encoding, encode_corpus, report_rows +from sampletones_tools.codec.report.rows import report_table, write_markdown +from sampletones_tools.codec.report.songs import available_bytes +from sampletones_tools.corpus.build import build_synthetic_corpus + +CSV_FILENAME: Final[str] = "report.csv" +MARKDOWN_FILENAME: Final[str] = "report.md" + + +@dataclass(frozen=True) +class CompressionReport: + """What a report run measured, and the tables it wrote. + + Attributes: + entries: The songs measured. + encodings: Every song compressed under every variant of the codec. + csv_path: The table another tool reads. + markdown_path: The table a reader reads. + """ + + entries: Tuple[CorpusEntry, ...] + encodings: Tuple[Encoding, ...] + csv_path: Path + markdown_path: Path + + +def write_report( + entries: Sequence[CorpusEntry], + encodings: Sequence[Encoding], + space: int, + output: Path, +) -> Tuple[Path, Path]: + """Writes the measurements as a table another tool reads and a table a reader reads. + + Args: + entries: The songs measured. + encodings: Their encodings. + space: The program area a song is written into. + output: The directory the two tables are written into, created when missing. + + Returns: + Tuple[Path, Path]: The CSV table and the Markdown table. + """ + table = report_table(report_rows(entries, encodings, space)) + output.mkdir(parents=True, exist_ok=True) + csv_path = output / CSV_FILENAME + markdown_path = output / MARKDOWN_FILENAME + table.write_csv(csv_path) + write_markdown(table, markdown_path, PLANE_COUNT * PLANE_STATE_SIZE) + return csv_path, markdown_path + + +def run_report(output: Path) -> CompressionReport: + """Builds the synthetic corpus, compresses it under every variant and writes the report. + + Args: + output: The directory the report is written into. + + Returns: + CompressionReport: The songs, their encodings and the tables written. + """ + corpus = build_synthetic_corpus() + entries = corpus_entries(corpus.catalog, corpus.project) + encodings = encode_corpus(entries) + csv_path, markdown_path = write_report(entries, encodings, available_bytes(DriverImage.load()), output) + return CompressionReport( + entries=entries, + encodings=encodings, + csv_path=csv_path, + markdown_path=markdown_path, + ) diff --git a/tests/integration/nsf/songs.py b/src/sampletones_tools/codec/report/songs.py similarity index 88% rename from tests/integration/nsf/songs.py rename to src/sampletones_tools/codec/report/songs.py index db464f778..01499c2cb 100644 --- a/tests/integration/nsf/songs.py +++ b/src/sampletones_tools/codec/report/songs.py @@ -15,8 +15,8 @@ def available_bytes(driver_image: DriverImage) -> int: def lengthened(project: Project, frames: int) -> Project: """``project`` with its order repeated to ``frames`` positions, over the same samples. - The song is copied rather than edited so the session's own project keeps the arrangement - every other case reads. + The lengthened song is a copy of its own, so ``project`` keeps its arrangement for every other + reader. """ longer = Project.create( rows_per_pattern=project.song.rows_per_pattern, diff --git a/tests/unit/sampletones_shared/meta/import_boundary/configs/__init__.py b/src/sampletones_tools/codec/study/__init__.py similarity index 100% rename from tests/unit/sampletones_shared/meta/import_boundary/configs/__init__.py rename to src/sampletones_tools/codec/study/__init__.py diff --git a/tests/unit/sampletones_shared/meta/source/__init__.py b/src/sampletones_tools/codec/study/accounting/__init__.py similarity index 100% rename from tests/unit/sampletones_shared/meta/source/__init__.py rename to src/sampletones_tools/codec/study/accounting/__init__.py diff --git a/src/sampletones_tools/codec/study/accounting/coincident.py b/src/sampletones_tools/codec/study/accounting/coincident.py new file mode 100644 index 000000000..408058e11 --- /dev/null +++ b/src/sampletones_tools/codec/study/accounting/coincident.py @@ -0,0 +1,37 @@ +from typing import Final, Mapping, Sequence + +from sampletones_player.compression.planes.order import PlaneOrder +from sampletones_player.specification.compression import OPCODE_SIZE +from sampletones_tools.codec.study.accounting.finding import NOTHING, Finding +from sampletones_tools.codec.study.accounting.tokens import ReadToken + +CONTROL_SUFFIX: Final[str] = "_control" +VALUE_SUFFIX: Final[str] = "_value" + + +def coincident_starts(tokens: Mapping[str, Sequence[ReadToken]]) -> Finding: + """What one opcode for a channel's control and value would spare where both start a token + on the same tick (H5). + + A note usually changes its timbre and its pitch on the same tick, so the two planes pay two + opcodes for one moment. One opcode announcing both is the bound stated here: a byte per + coincidence, before the cost of telling the two apart. + + Args: + tokens: Every plane's tokens, under the plane's name in the song block. + + Returns: + Finding: The opcode bytes at coincident starts, and the bytes one opcode would spare. + """ + finding = NOTHING + for name in PlaneOrder.names(): + if not name.endswith(CONTROL_SUFFIX): + continue + + channel = name.removesuffix(CONTROL_SUFFIX) + control = {token.tick for token in tokens[name]} + value = {token.tick for token in tokens[f"{channel}{VALUE_SUFFIX}"]} + shared = len(control & value) + finding += Finding(2 * OPCODE_SIZE * shared, OPCODE_SIZE * shared) + + return finding diff --git a/src/sampletones_tools/codec/study/accounting/dictionary.py b/src/sampletones_tools/codec/study/accounting/dictionary.py new file mode 100644 index 000000000..1307bd4b2 --- /dev/null +++ b/src/sampletones_tools/codec/study/accounting/dictionary.py @@ -0,0 +1,66 @@ +from collections import Counter +from typing import Dict, Final, Iterable + +from sampletones_player.compression.dictionary.table import PhraseTable +from sampletones_player.specification.compression import PHRASE_COUNT_SIZE +from sampletones_tools.codec.study.accounting.finding import NOTHING, Finding +from sampletones_tools.codec.study.accounting.runs import runs +from sampletones_tools.codec.study.accounting.tokens import ReadToken + +REPEATED: Final[int] = 2 +MIN_RLE_RUN: Final[int] = 3 +RLE_RUN_SIZE: Final[int] = 2 +DEFAULT_COUNT_SIZE: Final[int] = 1 + + +def plateaus(table: PhraseTable) -> Finding: + """What run-length coding inside phrase bodies would spare on the plateaus they hold (H2). + + A phrase is stored verbatim, a value resting for several ticks spelled out once per tick. + A run coded as its value and its count takes two bytes however long it rests. + + Args: + table: The dictionary. + + Returns: + Finding: The body bytes sitting in a repeat, and the bytes run-length coding would spare. + """ + finding = NOTHING + for phrase in table.phrases: + for run in runs(phrase.body): + if run >= REPEATED: + finding += Finding(run - 1, max(0, run - RLE_RUN_SIZE) if run >= MIN_RLE_RUN else 0) + + return finding + + +def default_counts( + table: PhraseTable, + tokens: Iterable[ReadToken], +) -> Finding: + """What a default count stored with each phrase would spare on the count bytes (H4). + + Every phrase token carries the ticks it plays for. A phrase played mostly for one length + could state that length once in its body, and a token playing it that long could leave the + count byte out. + + Args: + table: The dictionary. + tokens: Every token of every plane. + + Returns: + Finding: The count bytes phrase tokens carry, and the bytes a default would spare. + """ + counts: Dict[int, Counter[int]] = {phrase_id: Counter() for phrase_id in range(len(table))} + for token in tokens: + if token.phrase_id is not None: + counts[token.phrase_id][token.ticks] += 1 + + targeted = PHRASE_COUNT_SIZE * sum(sum(counter.values()) for counter in counts.values()) + saving = 0 + for counter in counts.values(): + if counter: + (_, modal), *_ = counter.most_common(1) + saving += max(0, PHRASE_COUNT_SIZE * modal - DEFAULT_COUNT_SIZE) + + return Finding(targeted, saving) diff --git a/src/sampletones_tools/codec/study/accounting/finding.py b/src/sampletones_tools/codec/study/accounting/finding.py new file mode 100644 index 000000000..4b04d4351 --- /dev/null +++ b/src/sampletones_tools/codec/study/accounting/finding.py @@ -0,0 +1,24 @@ +from __future__ import annotations + +from typing import Final, NamedTuple + + +class Finding(NamedTuple): + """What one hypothesis reaches in one encoding. + + Attributes: + targeted: The bytes the hypothesis acts upon, as the encoding writes them today. + saving: The bytes it would spare under the pricing the hypothesis states. + """ + + targeted: int + saving: int + + def __add__(self, other: object) -> Finding: + if not isinstance(other, Finding): + return NotImplemented + + return Finding(self.targeted + other.targeted, self.saving + other.saving) + + +NOTHING: Final[Finding] = Finding(0, 0) diff --git a/src/sampletones_tools/codec/study/accounting/fixed.py b/src/sampletones_tools/codec/study/accounting/fixed.py new file mode 100644 index 000000000..ea098b36f --- /dev/null +++ b/src/sampletones_tools/codec/study/accounting/fixed.py @@ -0,0 +1,63 @@ +from dataclasses import dataclass +from typing import Final + +from sampletones_player.compression.compressed import CompressedPlanes +from sampletones_player.specification.binary import WORD_SIZE +from sampletones_player.specification.compression import ( + PHRASE_TABLE_ENTRY_SIZE, + PLANE_COUNT, +) +from sampletones_player.specification.song import SONG_HEADER_SIZE +from sampletones_tools.codec.study.accounting.finding import Finding +from sampletones_tools.codec.study.corpus.song import StudySong + +TIMER_SIZE: Final[int] = WORD_SIZE + + +@dataclass(frozen=True) +class FixedOverheads: + """The bytes a song block pays before any tick is described (H7). + + Attributes: + header: The song header. + pitch_table: The pitch table as written, every pitch the format can name. + pitch_table_used: The pitch table cut at the highest pitch the song sounds. + loop_entries: The re-entry offsets written whether or not the song loops. + phrase_offsets: The table of offsets a sequential walk over the phrases could replace. + """ + + header: int + pitch_table: int + pitch_table_used: int + loop_entries: int + phrase_offsets: int + + @property + def finding(self) -> Finding: + """The fixed bytes, and what trimming the table, the entries and the offsets would spare.""" + targeted = self.header + self.pitch_table + self.phrase_offsets + saving = (self.pitch_table - self.pitch_table_used) + self.loop_entries + self.phrase_offsets + return Finding(targeted, saving) + + +def fixed_overheads( + song: StudySong, + compressed: CompressedPlanes, +) -> FixedOverheads: + """Accounts for the bytes a song block pays regardless of its length. + + Args: + song: The song, for the pitches it sounds. + compressed: The encoding, for the phrases it holds. + + Returns: + FixedOverheads: The fixed bytes, part by part. + """ + highest = max(max(channel.value) for channel in (song.planes.pulse1, song.planes.pulse2, song.planes.triangle)) + return FixedOverheads( + header=SONG_HEADER_SIZE, + pitch_table=len(song.pitches.data), + pitch_table_used=TIMER_SIZE * (highest + 1), + loop_entries=WORD_SIZE * PLANE_COUNT, + phrase_offsets=PHRASE_TABLE_ENTRY_SIZE * len(compressed.phrases), + ) diff --git a/src/sampletones_tools/codec/study/accounting/pairs.py b/src/sampletones_tools/codec/study/accounting/pairs.py new file mode 100644 index 000000000..1e00cdbaf --- /dev/null +++ b/src/sampletones_tools/codec/study/accounting/pairs.py @@ -0,0 +1,58 @@ +import itertools +from typing import Final, Sequence + +from sampletones_player.specification.compression import TokenTag +from sampletones_tools.codec.study.accounting.finding import NOTHING, Finding +from sampletones_tools.codec.study.accounting.runs import ramps +from sampletones_tools.codec.study.accounting.tokens import ReadToken + +SET_HOLD_SIZE: Final[int] = 2 +SINGLE_VALUE: Final[int] = 1 +RAMP_TOKEN_SIZE: Final[int] = 3 +MIN_RAMP_LENGTH: Final[int] = 3 + + +def set_holds(tokens: Sequence[ReadToken]) -> Finding: + """What a set-then-hold token would spare on a single value followed by a hold (H3). + + A plane that steps to a value and rests there pays a literal of one value and a hold, three + bytes. A token carrying the value and the count together takes two bytes on an opcode of its + own, which is the bound stated here; on the free escape it takes three and spares nothing. + + Args: + tokens: The tokens of one plane's stream. + + Returns: + Finding: The bytes such pairs take, and the bytes a two-byte token would spare. + """ + finding = NOTHING + for token, following in itertools.pairwise(tokens): + if token.tag is TokenTag.LITERAL and len(token.payload) == SINGLE_VALUE and following.tag is TokenTag.HOLD: + now = token.size + following.size + finding += Finding(now, now - SET_HOLD_SIZE) + + return finding + + +def ramps_in_literals(tokens: Sequence[ReadToken]) -> Finding: + """What a ramp token would spare on constant-step runs inside literals (H6). + + A fade or a slide spells every step out. A token naming the first value, the step and the + count takes three bytes however long the ramp runs. + + Args: + tokens: The tokens of one plane's stream. + + Returns: + Finding: The payload bytes lying in ramps, and the bytes ramp tokens would spare. + """ + finding = NOTHING + for token in tokens: + if token.tag is not TokenTag.LITERAL: + continue + + for length in ramps(token.payload): + if length >= MIN_RAMP_LENGTH: + finding += Finding(length, max(0, length - RAMP_TOKEN_SIZE)) + + return finding diff --git a/src/sampletones_tools/codec/study/accounting/rows.py b/src/sampletones_tools/codec/study/accounting/rows.py new file mode 100644 index 000000000..ff0e9c832 --- /dev/null +++ b/src/sampletones_tools/codec/study/accounting/rows.py @@ -0,0 +1,142 @@ +from dataclasses import dataclass +from typing import Dict, Final, List, Tuple + +from sampletones_player.compression.compressed import CompressedPlanes +from sampletones_player.compression.planes.order import PlaneOrder +from sampletones_tools.codec.study.accounting.coincident import coincident_starts +from sampletones_tools.codec.study.accounting.dictionary import default_counts, plateaus +from sampletones_tools.codec.study.accounting.finding import NOTHING, Finding +from sampletones_tools.codec.study.accounting.fixed import fixed_overheads +from sampletones_tools.codec.study.accounting.pairs import ramps_in_literals, set_holds +from sampletones_tools.codec.study.accounting.shares import PlaneShares, plane_shares +from sampletones_tools.codec.study.accounting.tokens import ReadToken, read_tokens +from sampletones_tools.codec.study.measure import Measurement + +HYPOTHESES: Final[Tuple[Tuple[str, str], ...]] = ( + ("H1", "hold chains"), + ("H2", "plateaus in bodies"), + ("H3", "set-then-hold"), + ("H4", "default counts"), + ("H5", "coincident starts"), + ("H6", "ramps in literals"), + ("H7", "fixed tables"), +) +LEADING_COLUMNS: Final[Tuple[str, ...]] = ( + "group", + "song", + "variant", + "block", + "dictionary", + "streams", + "hold opcodes", + "literal opcodes", + "literal payload", + "phrase tokens", + "idle planes", + "idle bytes", + "bend bytes", +) +COLUMNS: Final[Tuple[str, ...]] = ( + *LEADING_COLUMNS, + *(f"{label} {column}" for label, _ in HYPOTHESES for column in ("targeted", "saving", "share")), +) + + +@dataclass(frozen=True) +class AccountingRow: + """Where one encoding's bytes go, and what each hypothesis would reach in it. + + Attributes: + group: Which kind of song it is. + song: The song's name. + variant: The variant the encoding was built by. + block: The bytes the whole song block takes. + dictionary: The bytes the dictionary takes. + streams: The bytes the streams take together. + planes: Every plane's shares, in the order the song block writes them. + findings: What each hypothesis reaches, in the order of ``HYPOTHESES``. + """ + + group: str + song: str + variant: str + block: int + dictionary: int + streams: int + planes: Tuple[PlaneShares, ...] + findings: Tuple[Finding, ...] + + @property + def idle_planes(self) -> Tuple[PlaneShares, ...]: + """The planes keeping one value throughout the song.""" + return tuple(plane for plane in self.planes if plane.idle) + + @property + def cells(self) -> Tuple[str, ...]: + """The row as the table prints it, column by column.""" + leading = ( + self.group, + self.song, + self.variant, + f"{self.block}", + f"{self.dictionary}", + f"{self.streams}", + f"{sum(plane.hold_opcodes for plane in self.planes)}", + f"{sum(plane.literal_opcodes for plane in self.planes)}", + f"{sum(plane.literal_payload for plane in self.planes)}", + f"{sum(plane.phrase_bytes for plane in self.planes)}", + f"{len(self.idle_planes)}", + f"{sum(plane.stream for plane in self.idle_planes)}", + f"{sum(plane.stream for plane in self.planes if plane.bend)}", + ) + findings: List[str] = [] + for finding in self.findings: + findings.extend((f"{finding.targeted}", f"{finding.saving}", self.share(finding.saving))) + + return (*leading, *findings) + + def share(self, saving: int) -> str: + """``saving`` as a percentage of the whole block.""" + return f"{100.0 * saving / self.block:.1f}%" + + +def account( + measurement: Measurement, + compressed: CompressedPlanes, +) -> AccountingRow: + """Reads one encoding back and states what every hypothesis would reach in it. + + Args: + measurement: The encoding under the song and variant it belongs to. + compressed: Its streams as the driver reads them. + + Returns: + AccountingRow: The shares and the findings. + """ + tokens: Dict[str, Tuple[ReadToken, ...]] = { + name: read_tokens(stream) for name, stream in zip(PlaneOrder.names(), compressed.streams) + } + planes = tuple( + plane_shares(name, plane, tokens[name]) + for name, plane in zip(PlaneOrder.names(), measurement.song.planes.planes) + ) + every_token = [token for name in PlaneOrder.names() for token in tokens[name]] + findings = ( + sum((plane.hold_chains for plane in planes), NOTHING), + plateaus(compressed.phrases), + sum((set_holds(tokens[name]) for name in PlaneOrder.names()), NOTHING), + default_counts(compressed.phrases, every_token), + coincident_starts(tokens), + sum((ramps_in_literals(tokens[name]) for name in PlaneOrder.names()), NOTHING), + fixed_overheads(measurement.song, compressed).finding, + ) + return AccountingRow( + group=measurement.song.group.value, + song=measurement.song.name, + variant=measurement.variant, + block=measurement.block, + dictionary=measurement.dictionary, + streams=measurement.streams, + planes=planes, + findings=findings, + ) diff --git a/src/sampletones_tools/codec/study/accounting/runs.py b/src/sampletones_tools/codec/study/accounting/runs.py new file mode 100644 index 000000000..bbbb19fca --- /dev/null +++ b/src/sampletones_tools/codec/study/accounting/runs.py @@ -0,0 +1,32 @@ +from itertools import groupby, pairwise +from typing import Tuple + +from sampletones_player.specification.binary import BYTE_VALUES + + +def runs(data: bytes) -> Tuple[int, ...]: + """The lengths of the runs of one value ``data`` is made of, in order. + + Args: + data: The bytes to read. + + Returns: + Tuple[int, ...]: One length per run; they sum to the length of ``data``. + """ + return tuple(len(list(group)) for _, group in groupby(data)) + + +def ramps(data: bytes) -> Tuple[int, ...]: + """The lengths of the constant-step runs ``data`` is made of, steps of zero left aside. + + A ramp is a series stepping by the same amount from one value to the next, a fade or a + slide, and its length counts the values it covers. + + Args: + data: The bytes to read. + + Returns: + Tuple[int, ...]: The length of every ramp of at least two values. + """ + steps = tuple((following - value) % BYTE_VALUES for value, following in pairwise(data)) + return tuple(len(list(group)) + 1 for step, group in groupby(steps) if step != 0) diff --git a/src/sampletones_tools/codec/study/accounting/shares.py b/src/sampletones_tools/codec/study/accounting/shares.py new file mode 100644 index 000000000..9f722363a --- /dev/null +++ b/src/sampletones_tools/codec/study/accounting/shares.py @@ -0,0 +1,116 @@ +from dataclasses import dataclass +from math import ceil +from typing import Final, List, Sequence + +from sampletones_player.specification.compression import ( + MAX_HOLD_TICKS, + OPCODE_SIZE, + TokenTag, +) +from sampletones_tools.codec.study.accounting.finding import NOTHING, Finding +from sampletones_tools.codec.study.accounting.tokens import ReadToken + +WIDE_HOLD_SIZE: Final[int] = 2 +WIDE_HOLD_UNITS: Final[int] = MAX_HOLD_TICKS +BEND_SUFFIX: Final[str] = "_bend" + + +@dataclass(frozen=True) +class PlaneShares: + """Where one plane's stream bytes go, token kind by token kind. + + Attributes: + name: The plane's name in the song block. + stream: The bytes the stream takes. + hold_opcodes: The bytes spent on hold opcodes. + literal_opcodes: The bytes spent on literal opcodes. + literal_payload: The bytes spent on the values literals carry. + phrase_bytes: The bytes spent on phrase tokens. + idle: Whether the plane keeps one value throughout the song. + bend: Whether the plane is a bend plane. + hold_chains: What a wide hold would reach (H1): runs of consecutive holds. + """ + + name: str + stream: int + hold_opcodes: int + literal_opcodes: int + literal_payload: int + phrase_bytes: int + idle: bool + bend: bool + hold_chains: Finding + + +def plane_shares( + name: str, + plane: bytes, + tokens: Sequence[ReadToken], +) -> PlaneShares: + """Accounts for one plane's stream. + + Args: + name: The plane's name in the song block. + plane: The values the plane plays. + tokens: The tokens the plane's stream was read back as. + + Returns: + PlaneShares: The bytes by token kind, and what a wide hold would reach. + """ + return PlaneShares( + name=name, + stream=sum(token.size for token in tokens), + hold_opcodes=sum(token.size for token in tokens if token.tag is TokenTag.HOLD), + literal_opcodes=sum(OPCODE_SIZE for token in tokens if token.tag is TokenTag.LITERAL), + literal_payload=sum(len(token.payload) for token in tokens), + phrase_bytes=sum(token.size for token in tokens if token.names_a_phrase), + idle=len(set(plane)) == 1, + bend=name.endswith(BEND_SUFFIX), + hold_chains=hold_chains(tokens), + ) + + +def hold_chains(tokens: Sequence[ReadToken]) -> Finding: + """What a wide hold would spare on runs of consecutive holds (H1). + + A hold covers at most ``MAX_HOLD_TICKS``, so a plane resting longer pays an opcode per that + many ticks. A wide hold on the free escape takes two bytes and counts full holds instead of + ticks, so a chain is priced as the wide holds its full holds need and one plain hold for + the ticks left over. + + Args: + tokens: The tokens of one plane's stream. + + Returns: + Finding: The hold opcodes in chains, and the bytes wide holds would spare on them. + """ + finding = NOTHING + for chain in _chains(tokens): + full, rest = divmod(sum(token.ticks for token in chain), MAX_HOLD_TICKS) + if full == 0: + continue + + now = OPCODE_SIZE * len(chain) + wide = WIDE_HOLD_SIZE * ceil(full / WIDE_HOLD_UNITS) + (OPCODE_SIZE if rest else 0) + finding += Finding(now, max(0, now - wide)) + + return finding + + +def _chains(tokens: Sequence[ReadToken]) -> List[List[ReadToken]]: + chains: List[List[ReadToken]] = [] + current: List[ReadToken] = [] + for token in tokens: + if token.tag is TokenTag.HOLD: + current.append(token) + continue + + if len(current) > 1: + chains.append(current) + + current = [] + + if len(current) > 1: + chains.append(current) + + return chains diff --git a/src/sampletones_tools/codec/study/accounting/tokens.py b/src/sampletones_tools/codec/study/accounting/tokens.py new file mode 100644 index 000000000..73deed04f --- /dev/null +++ b/src/sampletones_tools/codec/study/accounting/tokens.py @@ -0,0 +1,105 @@ +from typing import List, NamedTuple, Optional, Tuple + +from sampletones_player.compression.tokens.span import token_span +from sampletones_player.specification.compression import ( + OPCODE_SIZE, + PHRASE_COUNT_SIZE, + PHRASE_ESCAPE_SIZE, + PHRASE_ID_ESCAPE, + TOKEN_OPERAND_MASK, + TOKEN_TAG_MASK, + TokenTag, +) + + +class ReadToken(NamedTuple): + """One token read back from a written stream, with everything the accounting asks of it. + + Attributes: + tag: What kind of token it is. + tick: The tick the token starts on. + size: The bytes the token takes. + ticks: The ticks the token covers. + payload: The values a literal carries, empty for every other kind. + phrase_id: The phrase a phrase token names, ``None`` for every other kind. + transpose: The shift a transposed phrase is played at, zero otherwise. + """ + + tag: TokenTag + tick: int + size: int + ticks: int + payload: bytes + phrase_id: Optional[int] + transpose: int + + @property + def names_a_phrase(self) -> bool: + """Whether the token plays a phrase from the dictionary.""" + return self.phrase_id is not None + + +def _phrase_operands( + stream: bytes, + position: int, + operand: int, + *, + transposed: bool, +) -> Tuple[int, int]: + after = position + OPCODE_SIZE + phrase_id = operand + if operand == PHRASE_ID_ESCAPE: + phrase_id = stream[after] + after += PHRASE_ESCAPE_SIZE + + transpose = stream[after + PHRASE_COUNT_SIZE] if transposed else 0 + return phrase_id, transpose + + +def read_tokens(stream: bytes) -> Tuple[ReadToken, ...]: + """Reads a plane's stream back as the tokens it was written from. + + Args: + stream: The plane's token stream. + + Returns: + Tuple[ReadToken, ...]: The tokens in the order they are read. + """ + tokens: List[ReadToken] = [] + position = 0 + tick = 0 + while position < len(stream): + span = token_span(stream, position) + tag = TokenTag(stream[position] & TOKEN_TAG_MASK) + operand = stream[position] & TOKEN_OPERAND_MASK + payload = b"" + phrase_id: Optional[int] = None + transpose = 0 + match tag: + case TokenTag.LITERAL: + payload = stream[position + OPCODE_SIZE : position + span.size] + case TokenTag.PHRASE | TokenTag.TRANSPOSED_PHRASE: + phrase_id, transpose = _phrase_operands( + stream, + position, + operand, + transposed=tag is TokenTag.TRANSPOSED_PHRASE, + ) + case TokenTag.HOLD: + pass + + tokens.append( + ReadToken( + tag=tag, + tick=tick, + size=span.size, + ticks=span.ticks, + payload=payload, + phrase_id=phrase_id, + transpose=transpose, + ) + ) + position += span.size + tick += span.ticks + + return tuple(tokens) diff --git a/tests/unit/sampletones_shared/meta/source/bindings/__init__.py b/src/sampletones_tools/codec/study/corpus/__init__.py similarity index 100% rename from tests/unit/sampletones_shared/meta/source/bindings/__init__.py rename to src/sampletones_tools/codec/study/corpus/__init__.py diff --git a/src/sampletones_tools/codec/study/corpus/build.py b/src/sampletones_tools/codec/study/corpus/build.py new file mode 100644 index 000000000..aec03a639 --- /dev/null +++ b/src/sampletones_tools/codec/study/corpus/build.py @@ -0,0 +1,43 @@ +from typing import List, Tuple + +from sampletones_shared.logger import logger +from sampletones_tools.codec.study.corpus.projects import lengthened_song, project_song +from sampletones_tools.codec.study.corpus.reconstructions import reconstruction_paths, reconstruction_song +from sampletones_tools.codec.study.corpus.song import StudySong +from sampletones_tools.codec.study.manifest import StudyManifest, StudySource + + +def build_corpus(manifest: StudyManifest) -> Tuple[StudySong, ...]: + """Reads every song the manifest names. + + Args: + manifest: What the run reads. + + Returns: + Tuple[StudySong, ...]: The projects as they stand, then each one lengthened, then every + stem in manifest order. + """ + songs: List[StudySong] = [] + for source in manifest.projects: + logger.info(f"Reading project {source.path}") + songs.append(project_song(source.path)) + + for source in manifest.projects: + logger.info(f"Lengthening project {source.path} to {manifest.lengthen_seconds} s") + songs.append(lengthened_song(source.path, manifest.lengthen_seconds)) + + for source in manifest.reconstructions: + songs.extend(_reconstructions(source)) + + return tuple(songs) + + +def _reconstructions(source: StudySource) -> List[StudySong]: + paths = reconstruction_paths(source.path) + songs: List[StudySong] = [] + for path in paths: + logger.info(f"Reading stem {path}") + name = source.label if len(paths) == 1 else f"{source.label}/{path.stem}" + songs.append(reconstruction_song(name, path)) + + return songs diff --git a/src/sampletones_tools/codec/study/corpus/projects.py b/src/sampletones_tools/codec/study/corpus/projects.py new file mode 100644 index 000000000..9897231eb --- /dev/null +++ b/src/sampletones_tools/codec/study/corpus/projects.py @@ -0,0 +1,94 @@ +from math import ceil +from pathlib import Path + +from sampletones_core.performance import song_instructions +from sampletones_core.project.container import ProjectContainer +from sampletones_core.project.project import Project +from sampletones_core.project.tuning import tuning_from_project +from sampletones_core.timers.utils import get_timer_table +from sampletones_core.timing import SongTiming +from sampletones_player.builder import streams_from_instructions +from sampletones_player.compression.pitch import PitchTable +from sampletones_player.compression.planes.separate import planes_from_streams +from sampletones_player.compression.seeds import phrases_from_project +from sampletones_shared.utils.progress import silent_reporter +from sampletones_tools.codec.study.corpus.song import SongGroup, StudySong + + +def project_song(path: Path) -> StudySong: + """Reads a project file as the song its arrangement plays. + + Args: + path: The project file. + + Returns: + StudySong: The song, seeded with the phrases the project's instruments offer. + """ + return _song(path.stem, SongGroup.PROJECT, path, ProjectContainer.load(path)) + + +def lengthened_song( + path: Path, + seconds: int, +) -> StudySong: + """Reads a project file as its arrangement repeated until the song lasts ``seconds``. + + A project on disk is a few seconds of patterns, and an export is measured against a song of + minutes, so the whole order is played through as many times as that takes. + + Args: + path: The project file. + seconds: How long the song is to last. + + Returns: + StudySong: The song, seeded with the phrases the project's instruments offer. + """ + project = ProjectContainer.load(path) + return _song( + f"{path.stem} ({seconds} s)", + SongGroup.LONG_PROJECT, + path, + _repeated(project, seconds), + ) + + +def _song( + name: str, + group: SongGroup, + source: Path, + project: Project, +) -> StudySong: + tuning = tuning_from_project(project) + pitches = PitchTable.from_tuning(tuning) + streams = streams_from_instructions( + song_instructions(project, silent_reporter), + get_timer_table(tuning), + ) + return StudySong( + name=name, + group=group, + source=source, + planes=planes_from_streams(streams, pitches), + seeds=phrases_from_project(project, tuning), + pitches=pitches, + ) + + +def _repeated( + project: Project, + seconds: int, +) -> Project: + groove = SongTiming.from_project(project).groove() + repetitions = max(1, ceil(seconds * project.settings.nes_frequency / groove.total_ticks)) + longer = Project.create( + rows_per_pattern=project.song.rows_per_pattern, + settings=project.settings, + ) + for voice in project.voices: + longer.voices.append(voice) + + longer.song = project.song.model_copy(deep=True) + for _ in range(repetitions - 1): + longer.song.order.extend(dict(frame) for frame in project.song.order) + + return longer diff --git a/src/sampletones_tools/codec/study/corpus/reconstructions.py b/src/sampletones_tools/codec/study/corpus/reconstructions.py new file mode 100644 index 000000000..e7fd84b49 --- /dev/null +++ b/src/sampletones_tools/codec/study/corpus/reconstructions.py @@ -0,0 +1,61 @@ +from pathlib import Path +from typing import Final, Tuple + +from sampletones_core.reconstructions import Reconstruction +from sampletones_core.timers.utils import get_timer_table +from sampletones_player.builder import streams_from_instructions +from sampletones_player.compression.dictionary.phrase import Phrase +from sampletones_player.compression.pitch import PitchTable +from sampletones_player.compression.planes.separate import planes_from_streams +from sampletones_tools.codec.study.corpus.song import SongGroup, StudySong + +STEM_SUFFIX: Final[str] = ".stn" +NO_SEEDS: Final[Tuple[Phrase, ...]] = () + + +def reconstruction_paths(path: Path) -> Tuple[Path, ...]: + """The stem files ``path`` names: the file itself, or every stem under a directory. + + Args: + path: A stem file or a directory holding stems. + + Returns: + Tuple[Path, ...]: The stem files, a directory's in sorted order. + """ + if path.is_dir(): + return tuple(sorted(path.rglob(f"*{STEM_SUFFIX}"))) + + return (path,) + + +def reconstruction_song( + name: str, + path: Path, +) -> StudySong: + """Reads a stem file as the song the console plays it as. + + A reconstruction sounds each of its slices once, so the song offers the dictionary nothing + and the search fills it from what the streams themselves repeat. + + Args: + name: What the song is called in a report. + path: The stem file. + + Returns: + StudySong: The song, at the tuning and rate the stem was reconstructed at. + """ + reconstruction = Reconstruction.load(path) + tuning = reconstruction.config.tuning + pitches = PitchTable.from_tuning(tuning) + streams = streams_from_instructions( + reconstruction.instructions, + get_timer_table(tuning), + ) + return StudySong( + name=name, + group=SongGroup.RECONSTRUCTION, + source=path, + planes=planes_from_streams(streams, pitches), + seeds=NO_SEEDS, + pitches=pitches, + ) diff --git a/src/sampletones_tools/codec/study/corpus/song.py b/src/sampletones_tools/codec/study/corpus/song.py new file mode 100644 index 000000000..473655eef --- /dev/null +++ b/src/sampletones_tools/codec/study/corpus/song.py @@ -0,0 +1,49 @@ +from dataclasses import dataclass +from enum import StrEnum +from pathlib import Path +from typing import Tuple + +from sampletones_player.compression.dictionary.phrase import Phrase +from sampletones_player.compression.pitch import PitchTable +from sampletones_player.compression.planes.song import SongPlanes + + +class SongGroup(StrEnum): + """Which kind of song a corpus entry is, which is what the report aggregates over. + + Attributes: + PROJECT: A project as it stands on disk, a few seconds of patterns. + LONG_PROJECT: A project with its order repeated to the length an export is measured at. + RECONSTRUCTION: A stem reconstructed from audio, its planes turning over at nearly + every tick. + """ + + PROJECT = "project" + LONG_PROJECT = "project-long" + RECONSTRUCTION = "reconstruction" + + +@dataclass(frozen=True) +class StudySong: + """One song the study measures the codec on. + + Attributes: + name: What the song is called in a report. + group: Which kind of song it is. + source: The file the song was read from. + planes: The planes the codec compresses. + seeds: The phrases the song's instruments offer the dictionary. + pitches: The timer each pitch of the song sounds at. + """ + + name: str + group: SongGroup + source: Path + planes: SongPlanes + seeds: Tuple[Phrase, ...] + pitches: PitchTable + + @property + def ticks(self) -> int: + """The ticks the song lasts.""" + return self.planes.ticks diff --git a/src/sampletones_tools/codec/study/manifest.py b/src/sampletones_tools/codec/study/manifest.py new file mode 100644 index 000000000..026a242c5 --- /dev/null +++ b/src/sampletones_tools/codec/study/manifest.py @@ -0,0 +1,90 @@ +from pathlib import Path +from typing import Final, Self, Tuple + +from pydantic import BaseModel, ConfigDict, Field, model_validator + +NAMES_A_SOURCE: Final[str] = "A study measures at least one project or reconstruction." + + +class StudySource(BaseModel): + """One file or directory the corpus is read from. + + Attributes: + label: What songs read from the source are called in a report. + path: The file, or the directory whose stems are read. + """ + + model_config = ConfigDict(extra="forbid", frozen=True) + + label: str + path: Path + + @model_validator(mode="after") + def _names_a_file_or_directory(self) -> Self: + """Raises: + ValueError: If nothing stands at the path. + """ + if not self.path.exists(): + raise ValueError(f"No file at {self.path}.") + + return self + + @classmethod + def at(cls, path: Path) -> Self: + """A source labeled by its own name: a file's stem, or a directory's name. + + Args: + path: The file or directory. + + Returns: + Self: The source under that label. + """ + return cls(label=path.name if path.is_dir() else path.stem, path=path) + + +class StudyManifest(BaseModel): + """What one run of the study reads and measures. + + Attributes: + projects: The project files, each measured as it stands and at the lengthened duration. + reconstructions: The stem files and directories of stems. + lengthen_seconds: How long each project's lengthened copy lasts. + variants: The names of the variants every song is encoded under. + """ + + model_config = ConfigDict(extra="forbid", frozen=True) + + projects: Tuple[StudySource, ...] + reconstructions: Tuple[StudySource, ...] + lengthen_seconds: int = Field(ge=1) + variants: Tuple[str, ...] = Field(min_length=1) + + @model_validator(mode="after") + def _names_a_source(self) -> Self: + """Raises: + ValueError: If the manifest names neither a project nor a reconstruction. + """ + if not self.projects and not self.reconstructions: + raise ValueError(NAMES_A_SOURCE) + + return self + + @classmethod + def load(cls, path: Path) -> Self: + """Reads a manifest a run wrote, or one written by hand. + + Args: + path: The manifest file, as JSON. + + Returns: + Self: The manifest. + """ + return cls.model_validate_json(path.read_text(encoding="utf-8")) + + def save(self, path: Path) -> None: + """Writes the manifest beside a run's report, so the run can be repeated. + + Args: + path: Where the manifest is written, as JSON. + """ + path.write_text(self.model_dump_json(indent=2), encoding="utf-8") diff --git a/src/sampletones_tools/codec/study/measure.py b/src/sampletones_tools/codec/study/measure.py new file mode 100644 index 000000000..e116fcec4 --- /dev/null +++ b/src/sampletones_tools/codec/study/measure.py @@ -0,0 +1,132 @@ +from dataclasses import dataclass +from typing import Callable, Optional, Tuple + +from sampletones_player.compression.compressed import CompressedPlanes +from sampletones_player.compression.decode import decode_planes +from sampletones_player.specification.song import SONG_HEADER_SIZE +from sampletones_tools.codec.study.corpus.song import StudySong + + +@dataclass(frozen=True) +class Encoding: + """What a variant wrote a song as, priced in the bytes the song block counts. + + A variant of the encoder writes the grammar the driver reads today, and its streams are + kept as the bytes they are. A variant of the grammar is priced at token level, its streams + stated as the bytes each would take, so the two kinds sit in one report on the same terms. + + Attributes: + phrases: The phrases the dictionary holds. + dictionary: The bytes the dictionary takes. + streams: The bytes each plane's stream takes, in the order the song block writes them. + seconds: The processor time the encoding took. + lossless: Whether the streams play back to the planes they were written from. + written: The streams as the driver reads them, where the variant writes its grammar. + """ + + phrases: int + dictionary: int + streams: Tuple[int, ...] + seconds: float + lossless: bool + written: Optional[CompressedPlanes] + + +Encoder = Callable[[StudySong], Encoding] + + +@dataclass(frozen=True) +class Measurement: + """One song encoded under one variant. + + Attributes: + song: The song encoded. + variant: The name of the variant the encoding was built by. + encoding: What the variant wrote, and what writing it cost. + """ + + song: StudySong + variant: str + encoding: Encoding + + @property + def block(self) -> int: + """The bytes the whole song block takes: header, pitch table, dictionary and streams.""" + return SONG_HEADER_SIZE + len(self.song.pitches.data) + self.dictionary + self.streams + + @property + def dictionary(self) -> int: + """The bytes the dictionary takes.""" + return self.encoding.dictionary + + @property + def streams(self) -> int: + """The bytes the token streams take together.""" + return sum(self.encoding.streams) + + @property + def phrases(self) -> int: + """The phrases the dictionary holds.""" + return self.encoding.phrases + + @property + def seconds(self) -> float: + """The processor time the encoding took.""" + return self.encoding.seconds + + @property + def lossless(self) -> bool: + """Whether the streams play back to the planes they were written from.""" + return self.encoding.lossless + + @property + def bytes_per_tick(self) -> float: + """The bytes each tick of the song costs, the whole block counted.""" + return self.block / self.song.ticks + + +def production_encoding( + song: StudySong, + compressed: CompressedPlanes, + seconds: float, +) -> Encoding: + """What the production codec wrote a song as, played back through the production decoder. + + Args: + song: The song encoded. + compressed: The dictionary and the streams the codec wrote. + seconds: The processor time the encoding took. + + Returns: + Encoding: The encoding, its streams kept as written. + """ + return Encoding( + phrases=len(compressed.phrases), + dictionary=compressed.phrases.size, + streams=tuple(len(stream) for stream in compressed.streams), + seconds=seconds, + lossless=decode_planes(compressed) == song.planes, + written=compressed, + ) + + +def measure( + song: StudySong, + variant: str, + encode: Encoder, +) -> Measurement: + """Encodes a song under a variant. + + Args: + song: The song to encode. + variant: The name of the variant. + encode: What the variant writes the song as. + + Returns: + Measurement: The encoding under the song and variant it belongs to. + """ + return Measurement( + song=song, + variant=variant, + encoding=encode(song), + ) diff --git a/src/sampletones_tools/codec/study/plan.py b/src/sampletones_tools/codec/study/plan.py new file mode 100644 index 000000000..f46c7e3c3 --- /dev/null +++ b/src/sampletones_tools/codec/study/plan.py @@ -0,0 +1,35 @@ +from dataclasses import dataclass +from typing import Tuple + +from sampletones_tools.codec.study.manifest import StudyManifest +from sampletones_tools.codec.study.variants.baselines import Baselines +from sampletones_tools.codec.study.variants.registry import selected_variants +from sampletones_tools.codec.study.variants.variant import Variant + + +@dataclass(frozen=True) +class StudyPlan: + """What one run measures, and the variants its manifest names, so the two stay in step. + + Attributes: + manifest: What the run reads, which the run saves to repeat it. + variants: The variants the manifest's names select, the baseline first. + """ + + manifest: StudyManifest + variants: Tuple[Variant, ...] + + +def plan_study(manifest: StudyManifest) -> StudyPlan: + """The run a manifest describes, its variant names selected over production encodings of their own. + + Args: + manifest: What the run reads. + + Returns: + StudyPlan: The manifest beside the variants it names. + + Raises: + ValueError: If a name is registered to no variant. + """ + return StudyPlan(manifest=manifest, variants=selected_variants(manifest.variants, Baselines())) diff --git a/tests/unit/sampletones_synthesis/__init__.py b/src/sampletones_tools/codec/study/report/__init__.py similarity index 100% rename from tests/unit/sampletones_synthesis/__init__.py rename to src/sampletones_tools/codec/study/report/__init__.py diff --git a/src/sampletones_tools/codec/study/report/aggregate.py b/src/sampletones_tools/codec/study/report/aggregate.py new file mode 100644 index 000000000..512042d87 --- /dev/null +++ b/src/sampletones_tools/codec/study/report/aggregate.py @@ -0,0 +1,112 @@ +from dataclasses import dataclass +from typing import Dict, Final, List, Sequence, Tuple + +from sampletones_tools.codec.study.corpus.song import SongGroup +from sampletones_tools.codec.study.measure import Measurement + +COLUMNS: Final[Tuple[str, ...]] = ( + "group", + "variant", + "songs", + "ticks", + "block", + "bytes per tick", + "change", + "worst song", + "seconds", +) + + +@dataclass(frozen=True) +class GroupRow: + """One kind of song under one variant, summed over the songs of that kind. + + Attributes: + group: Which kind of song it is. + variant: The variant the encodings were built by. + songs: How many songs the group holds. + ticks: The ticks the songs last together. + block: The bytes the song blocks take together. + baseline: The bytes the same songs take under the baseline. + worst: The largest growth any one song shows against its baseline, as a ratio. + seconds: The processor time the encodings took together. + """ + + group: str + variant: str + songs: int + ticks: int + block: int + baseline: int + worst: float + seconds: float + + @property + def change(self) -> float: + """How the group's bytes stand against the baseline, as a ratio; negative is smaller.""" + return self.block / self.baseline - 1.0 + + @property + def cells(self) -> Tuple[str, ...]: + """The row as the table prints it, column by column.""" + return ( + self.group, + self.variant, + f"{self.songs}", + f"{self.ticks}", + f"{self.block}", + f"{self.block / self.ticks:.3f}", + f"{100.0 * self.change:+.1f}%", + f"{100.0 * self.worst:+.1f}%", + f"{self.seconds:.2f}", + ) + + +def group_rows( + measurements: Sequence[Measurement], + baseline: str, +) -> Tuple[GroupRow, ...]: + """Sums the measurements over each kind of song, variant by variant. + + Args: + measurements: Every song under every variant. + baseline: The name of the variant the others are held against. + + Returns: + Tuple[GroupRow, ...]: One row per kind of song and variant, kinds in their declared + order and variants in the order they were measured. + """ + reference: Dict[str, int] = { + measurement.song.name: measurement.block for measurement in measurements if measurement.variant == baseline + } + variants = list(dict.fromkeys(measurement.variant for measurement in measurements)) + rows: List[GroupRow] = [] + for group in SongGroup: + for variant in variants: + chosen = [ + measurement + for measurement in measurements + if measurement.song.group is group and measurement.variant == variant + ] + if chosen: + rows.append(_group_row(group, variant, chosen, reference)) + + return tuple(rows) + + +def _group_row( + group: SongGroup, + variant: str, + chosen: Sequence[Measurement], + reference: Dict[str, int], +) -> GroupRow: + return GroupRow( + group=group.value, + variant=variant, + songs=len(chosen), + ticks=sum(measurement.song.ticks for measurement in chosen), + block=sum(measurement.block for measurement in chosen), + baseline=sum(reference[measurement.song.name] for measurement in chosen), + worst=max(measurement.block / reference[measurement.song.name] - 1.0 for measurement in chosen), + seconds=sum(measurement.seconds for measurement in chosen), + ) diff --git a/src/sampletones_tools/codec/study/report/rows.py b/src/sampletones_tools/codec/study/report/rows.py new file mode 100644 index 000000000..38188afaa --- /dev/null +++ b/src/sampletones_tools/codec/study/report/rows.py @@ -0,0 +1,87 @@ +from dataclasses import dataclass +from typing import Final, Tuple + +from sampletones_tools.codec.study.measure import Measurement + +COLUMNS: Final[Tuple[str, ...]] = ( + "group", + "song", + "variant", + "ticks", + "block", + "bytes per tick", + "dictionary", + "streams", + "phrases", + "seconds", + "lossless", +) + + +@dataclass(frozen=True) +class StudyRow: + """One song measured under one variant, as the report prints it. + + Attributes: + group: Which kind of song it is. + song: The song's name. + variant: The variant the encoding was built by. + ticks: The ticks the song lasts. + block: The bytes the whole song block takes. + dictionary: The bytes the dictionary takes. + streams: The bytes the streams take together. + phrases: The phrases the dictionary holds. + seconds: The processor time the encoding took. + lossless: Whether the encoding plays back to its planes. + """ + + group: str + song: str + variant: str + ticks: int + block: int + dictionary: int + streams: int + phrases: int + seconds: float + lossless: bool + + @property + def cells(self) -> Tuple[str, ...]: + """The row as the table prints it, column by column.""" + return ( + self.group, + self.song, + self.variant, + f"{self.ticks}", + f"{self.block}", + f"{self.block / self.ticks:.3f}", + f"{self.dictionary}", + f"{self.streams}", + f"{self.phrases}", + f"{self.seconds:.2f}", + "yes" if self.lossless else "no", + ) + + +def study_row(measurement: Measurement) -> StudyRow: + """The report row of one measurement. + + Args: + measurement: The encoding and its cost. + + Returns: + StudyRow: The row. + """ + return StudyRow( + group=measurement.song.group.value, + song=measurement.song.name, + variant=measurement.variant, + ticks=measurement.song.ticks, + block=measurement.block, + dictionary=measurement.dictionary, + streams=measurement.streams, + phrases=measurement.phrases, + seconds=measurement.seconds, + lossless=measurement.lossless, + ) diff --git a/src/sampletones_tools/codec/study/report/run.py b/src/sampletones_tools/codec/study/report/run.py new file mode 100644 index 000000000..6d54cab46 --- /dev/null +++ b/src/sampletones_tools/codec/study/report/run.py @@ -0,0 +1,159 @@ +import subprocess +from datetime import UTC, datetime +from pathlib import Path +from typing import Final, Iterator, List, Optional, Sequence, Tuple + +from sampletones_player.compression.compressed import CompressedPlanes +from sampletones_shared.paths.source import REPOSITORY_ROOT +from sampletones_shared.paths.user import USER_PATH_DOCUMENTS +from sampletones_shared.utils.tables import Table +from sampletones_tools.checkout import is_checkout +from sampletones_tools.codec.study.accounting import rows as accounting +from sampletones_tools.codec.study.manifest import StudyManifest +from sampletones_tools.codec.study.measure import Measurement +from sampletones_tools.codec.study.plan import StudyPlan +from sampletones_tools.codec.study.report import aggregate +from sampletones_tools.codec.study.report import rows as songs +from sampletones_tools.codec.study.report import verdicts +from sampletones_tools.codec.study.variants.production import BASELINE_NAME +from sampletones_tools.codec.study.variants.variant import Variant +from sampletones_tools.runs import stamped_run_directory + +OUTPUT_ROOT: Final[Path] = USER_PATH_DOCUMENTS / "compression" +REPORT_CSV: Final[str] = "report.csv" +REPORT_MARKDOWN: Final[str] = "report.md" +ACCOUNTING_CSV: Final[str] = "accounting.csv" +VERDICTS_CSV: Final[str] = "verdicts.csv" +MANIFEST_JSON: Final[str] = "manifest.json" +UNKNOWN_COMMIT: Final[str] = "unknown" + + +def run_directory(output: Optional[Path]) -> Path: + """The directory a run writes into, created where it is missing. + + Args: + output: The directory asked for, or ``None`` for a stamped one under the documents. + + Returns: + Path: The directory. + """ + directory = output or stamped_run_directory(OUTPUT_ROOT) + directory.mkdir(parents=True, exist_ok=True) + return directory + + +def commit_hash() -> str: + """The short hash of the commit the checkout stands at, which dates the run's report. + + An installed copy has no checkout of its own around it, and a machine may run without git, so + either records ``unknown``. + """ + if not is_checkout(REPOSITORY_ROOT): + return UNKNOWN_COMMIT + + try: + completed = subprocess.run( + ["git", "rev-parse", "--short", "HEAD"], + capture_output=True, + text=True, + check=False, + cwd=REPOSITORY_ROOT, + ) + except FileNotFoundError: + return UNKNOWN_COMMIT + + return completed.stdout.strip() or UNKNOWN_COMMIT + + +def write_run( + directory: Path, + plan: StudyPlan, + measurements: Sequence[Measurement], + derived: Sequence[Measurement], +) -> None: + """Writes a run's report, its verdicts, its accounting and the manifest that reproduces it. + + Args: + directory: The run's directory. + plan: What the run read and the variants every song was encoded under. + measurements: Every song under every variant, in the order measured. + derived: The strategy-depth measurements drawn from those, reported beside them and + left out of the accounting, which reads each written encoding once. + """ + reported = (*measurements, *derived) + song_rows = [songs.study_row(measurement) for measurement in reported] + group_rows = aggregate.group_rows(reported, BASELINE_NAME) + verdict_rows = verdicts.verdict_rows(group_rows, measurements, plan.variants) + accounting_rows = [ + accounting.account(measurement, compressed) for measurement, compressed in _written(measurements) + ] + song_table = Table(columns=songs.COLUMNS, rows=tuple(row.cells for row in song_rows)) + verdict_table = Table(columns=verdicts.COLUMNS, rows=tuple(row.cells for row in verdict_rows)) + plan.manifest.save(directory / MANIFEST_JSON) + song_table.write_csv(directory / REPORT_CSV) + verdict_table.write_csv(directory / VERDICTS_CSV) + Table(columns=accounting.COLUMNS, rows=tuple(row.cells for row in accounting_rows)).write_csv( + directory / ACCOUNTING_CSV + ) + lines = _header(plan.manifest, plan.variants) + lines.extend(("## Variants", "", *_variants_table(plan.variants).markdown_lines(), "")) + lines.extend(("## Verdicts", "", verdicts.RULE, "", *verdict_table.markdown_lines(), "")) + group_table = Table(columns=aggregate.COLUMNS, rows=tuple(row.cells for row in group_rows)) + lines.extend(("## Groups", "", *group_table.markdown_lines(), "")) + lines.extend(("## Songs", "", *song_table.markdown_lines(), "")) + lines.extend(("## Accounting", "", *_accounting_table(accounting_rows).markdown_lines(), "")) + (directory / REPORT_MARKDOWN).write_text("\n".join(lines), encoding="utf-8") + + +def _written(measurements: Sequence[Measurement]) -> Iterator[Tuple[Measurement, CompressedPlanes]]: + """The measurements whose streams the driver reads as they stand, beside those streams.""" + for measurement in measurements: + written = measurement.encoding.written + if written is not None: + yield measurement, written + + +def _header( + manifest: StudyManifest, + variants: Sequence[Variant], +) -> List[str]: + named = ", ".join(f"{variant.name} ({variant.kind.value})" for variant in variants) + return [ + "# Compression study", + "", + f"Commit: {commit_hash()}", + f"Date: {datetime.now(UTC).isoformat(timespec='seconds')}", + ( + f"Manifest: {MANIFEST_JSON}, {len(manifest.projects)} projects lengthened to " + f"{manifest.lengthen_seconds} s, {len(manifest.reconstructions)} reconstruction sources" + ), + f"Variants: {named}", + "", + "Each accounting share is the saving a hypothesis would reach, as a share of the whole song block.", + "", + ] + + +def _variants_table(variants: Sequence[Variant]) -> Table: + columns = ("variant", "hypothesis", "kind", "driver") + cells = tuple((variant.name, variant.hypothesis, variant.kind.value, variant.note) for variant in variants) + return Table(columns=columns, rows=cells) + + +def _accounting_table(rows: Sequence[accounting.AccountingRow]) -> Table: + columns = ("group", "song", "variant", "block", "dictionary", "idle bytes", "bend bytes") + labels = tuple(f"{label} {name}" for label, name in accounting.HYPOTHESES) + cells = tuple( + ( + row.group, + row.song, + row.variant, + f"{row.block}", + row.share(row.dictionary), + row.share(sum(plane.stream for plane in row.idle_planes)), + row.share(sum(plane.stream for plane in row.planes if plane.bend)), + *(row.share(finding.saving) for finding in row.findings), + ) + for row in rows + ) + return Table(columns=(*columns, *labels), rows=cells) diff --git a/src/sampletones_tools/codec/study/report/verdicts.py b/src/sampletones_tools/codec/study/report/verdicts.py new file mode 100644 index 000000000..8dbcbe677 --- /dev/null +++ b/src/sampletones_tools/codec/study/report/verdicts.py @@ -0,0 +1,180 @@ +from dataclasses import dataclass +from enum import StrEnum +from typing import Dict, Final, List, Optional, Sequence, Tuple + +from sampletones_tools.codec.study.corpus.song import SongGroup +from sampletones_tools.codec.study.measure import Measurement +from sampletones_tools.codec.study.report.aggregate import GroupRow +from sampletones_tools.codec.study.variants.variant import Variant, VariantKind + +PROJECT_BAR: Final[float] = 0.03 +RECONSTRUCTION_BAR: Final[float] = 0.05 +REGRESSION_BAR: Final[float] = 0.01 +PROJECT_GROUPS: Final[Tuple[SongGroup, ...]] = (SongGroup.PROJECT, SongGroup.LONG_PROJECT) +UNMEASURED: Final[str] = "" +COLUMNS: Final[Tuple[str, ...]] = ( + "variant", + "hypothesis", + "kind", + "projects", + "reconstructions", + "worst song", + "verdict", +) +RULE: Final[str] = ( + f"A variant graduates when it saves {100 * PROJECT_BAR:.0f}% over the projects, short and " + f"lengthened together, or {100 * RECONSTRUCTION_BAR:.0f}% over the reconstructions, with no song " + f"growing by more than {100 * REGRESSION_BAR:.0f}%. A variant priced at token level is provisional " + "until its production layer confirms the figure." +) + + +class Verdict(StrEnum): + """What the decision rule says of a variant. + + Attributes: + GRADUATES: The variant saves enough, harms no song, and its bytes are the driver's. + PROVISIONAL: The variant saves enough and harms no song, priced at token level. + REJECTED: The variant saves too little, or grows a song past the bar. + """ + + GRADUATES = "graduates" + PROVISIONAL = "provisional" + REJECTED = "rejected" + + +@dataclass(frozen=True) +class VerdictRow: + """One variant judged by the decision rule. + + Attributes: + variant: The variant's name. + hypothesis: The hypothesis the variant measures. + kind: What the variant changes. + projects: How the projects' bytes stand against the baseline, short and lengthened + together, as a ratio; ``None`` where the variant measured none. + reconstructions: The same over the reconstructions. + worst: The largest growth any one song shows against its baseline, as a ratio. + verdict: What the rule says. + """ + + variant: str + hypothesis: str + kind: VariantKind + projects: Optional[float] + reconstructions: Optional[float] + worst: float + verdict: Verdict + + @property + def cells(self) -> Tuple[str, ...]: + """The row as the table prints it, column by column.""" + return ( + self.variant, + self.hypothesis, + self.kind.value, + _percent(self.projects), + _percent(self.reconstructions), + f"{100.0 * self.worst:+.1f}%", + self.verdict.value, + ) + + +def _percent(change: Optional[float]) -> str: + return UNMEASURED if change is None else f"{100.0 * change:+.1f}%" + + +def _change(rows: Sequence[GroupRow]) -> Optional[float]: + if not rows: + return None + + return sum(row.block for row in rows) / sum(row.baseline for row in rows) - 1.0 + + +def _earned( + projects: Optional[float], + reconstructions: Optional[float], +) -> bool: + over_projects = projects is not None and projects <= -PROJECT_BAR + over_reconstructions = reconstructions is not None and reconstructions <= -RECONSTRUCTION_BAR + return over_projects or over_reconstructions + + +def judge( + projects: Optional[float], + reconstructions: Optional[float], + worst: float, + *, + priced: bool, +) -> Verdict: + """Applies the decision rule to one variant's aggregates. + + Args: + projects: The change over the projects, short and lengthened together. + reconstructions: The change over the reconstructions. + worst: The largest growth any one song shows. + priced: Whether the variant's bytes were priced at token level. + + Returns: + Verdict: What the rule says. + """ + if worst > REGRESSION_BAR or not _earned(projects, reconstructions): + return Verdict.REJECTED + + return Verdict.PROVISIONAL if priced else Verdict.GRADUATES + + +def verdict_rows( + group_rows: Sequence[GroupRow], + measurements: Sequence[Measurement], + variants: Sequence[Variant], +) -> Tuple[VerdictRow, ...]: + """Judges every variant the run measured, the baseline excepted. + + Args: + group_rows: The measurements summed over each kind of song, variant by variant. + measurements: Every song under every variant. + variants: The variants the run encoded under. + + Returns: + Tuple[VerdictRow, ...]: One row per measured variant, in the order given. + """ + by_variant: Dict[str, List[GroupRow]] = {} + for row in group_rows: + by_variant.setdefault(row.variant, []).append(row) + + priced = {measurement.variant for measurement in measurements if measurement.encoding.written is None} + rows: List[VerdictRow] = [] + for variant in variants: + if variant.kind is VariantKind.BASELINE or variant.name not in by_variant: + continue + + rows.append( + _verdict_row( + variant, + by_variant[variant.name], + priced=variant.name in priced, + ) + ) + + return tuple(rows) + + +def _verdict_row( + variant: Variant, + rows: Sequence[GroupRow], + *, + priced: bool, +) -> VerdictRow: + projects = _change([row for row in rows if SongGroup(row.group) in PROJECT_GROUPS]) + reconstructions = _change([row for row in rows if SongGroup(row.group) is SongGroup.RECONSTRUCTION]) + worst = max(row.worst for row in rows) + return VerdictRow( + variant=variant.name, + hypothesis=variant.hypothesis, + kind=variant.kind, + projects=projects, + reconstructions=reconstructions, + worst=worst, + verdict=judge(projects, reconstructions, worst, priced=priced), + ) diff --git a/tests/unit/sampletones_synthesis/envelopes/__init__.py b/src/sampletones_tools/codec/study/sandbox/__init__.py similarity index 100% rename from tests/unit/sampletones_synthesis/envelopes/__init__.py rename to src/sampletones_tools/codec/study/sandbox/__init__.py diff --git a/src/sampletones_tools/codec/study/sandbox/context.py b/src/sampletones_tools/codec/study/sandbox/context.py new file mode 100644 index 000000000..60051a41f --- /dev/null +++ b/src/sampletones_tools/codec/study/sandbox/context.py @@ -0,0 +1,43 @@ +from dataclasses import dataclass +from typing import Tuple + +from sampletones_player.compression.matches.index import PlaneIndex +from sampletones_player.compression.matches.matcher import PhraseMatcher +from sampletones_player.compression.parse.boundaries import Boundaries +from sampletones_player.specification.compression import INITIAL_PLANE_VALUE +from sampletones_tools.codec.study.sandbox.costs import Costs + + +@dataclass(frozen=True) +class PlaneContext: + """One plane as a grammar reads it: its values, the phrases it may play, and the terms. + + Attributes: + index: The plane, its steps and its runs. + matcher: Which phrases the plane plays at a tick, and for how long. + boundaries: The ticks a token starts on, read either way from every tick. + transposition: Whether a phrase may play at a shift. + defaults: The default count of each phrase, by id, zero where the phrase has none. + costs: The bytes each token takes. + """ + + index: PlaneIndex + matcher: PhraseMatcher + boundaries: Boundaries + transposition: bool + defaults: Tuple[int, ...] + costs: Costs + + def value_before(self, position: int) -> int: + """The value the plane holds as ``position`` is reached, which a hold there keeps. + + The driver seeds every plane before the first token, so the stream's first tick is + reached holding that value like any other. + + Args: + position: The tick. + + Returns: + int: The value. + """ + return self.index.plane[position - 1] if position > 0 else INITIAL_PLANE_VALUE diff --git a/src/sampletones_tools/codec/study/sandbox/costs.py b/src/sampletones_tools/codec/study/sandbox/costs.py new file mode 100644 index 000000000..23d3f1703 --- /dev/null +++ b/src/sampletones_tools/codec/study/sandbox/costs.py @@ -0,0 +1,93 @@ +from dataclasses import dataclass +from typing import Final + +from sampletones_player.compression.dictionary.table import PhraseTable +from sampletones_player.compression.tokens.sizes import hold_size +from sampletones_player.specification.compression import ( + CHEAP_PHRASE_IDS, + OPCODE_SIZE, + PHRASE_COUNT_SIZE, + PHRASE_ESCAPE_SIZE, + TRANSPOSE_SIZE, +) + +VALUE_SIZE: Final[int] = 1 +DEFAULT_COUNT_SIZE: Final[int] = 1 +DEFAULT_FLAG_BITS: Final[int] = 1 + + +@dataclass(frozen=True) +class Costs: + """The bytes each token takes under one grammar. + + A token on the free escape is a phrase opcode followed by a count of zero, which the codec + as it stands never writes, so the opcode's operand is what such a token carries: the sizes + it counts are as many as the cheap ids a phrase opcode names. + + Attributes: + opcode: The bytes an opcode takes. + hold: The bytes a hold takes. + wide_hold: The bytes a wide hold takes. + set_hold: The bytes a set-hold takes. + phrase_count: The bytes a phrase token's count takes. + phrase_escape: The bytes naming a phrase beyond the cheap ids takes. + transpose: The bytes a shift takes. + operands: The operand values below the escape: the cheap ids a phrase opcode names, and + the sizes a token on the escape counts. + default_entry: The bytes a table entry grows by to carry the phrase's default count. + """ + + opcode: int + hold: int + wide_hold: int + set_hold: int + phrase_count: int + phrase_escape: int + transpose: int + operands: int + default_entry: int + + def literal(self, length: int) -> int: + """The bytes a literal of ``length`` values takes.""" + return self.opcode + length + + def phrase( + self, + phrase_id: int, + transpose: int, + *, + default: bool, + ) -> int: + """The bytes a token playing a phrase takes. + + Args: + phrase_id: Position the phrase takes in the table. + transpose: The shift every byte of the phrase is played at. + default: Whether the token plays the phrase's default count and carries none. + + Returns: + int: The bytes the token takes. + """ + escape = self.phrase_escape if phrase_id >= self.operands else 0 + count = 0 if default else self.phrase_count + shift = self.transpose if transpose else 0 + return self.opcode + escape + count + shift + + def dictionary(self, table: PhraseTable) -> int: + """The bytes ``table`` takes in the song block under this grammar.""" + return table.size + self.default_entry * len(table) + + +PRODUCTION_COSTS: Final[Costs] = Costs( + opcode=OPCODE_SIZE, + hold=hold_size(), + wide_hold=OPCODE_SIZE + PHRASE_COUNT_SIZE, + set_hold=OPCODE_SIZE + PHRASE_COUNT_SIZE + VALUE_SIZE, + phrase_count=PHRASE_COUNT_SIZE, + phrase_escape=PHRASE_ESCAPE_SIZE, + transpose=TRANSPOSE_SIZE, + operands=CHEAP_PHRASE_IDS, + default_entry=0, +) +SET_HOLD_BOUND: Final[int] = OPCODE_SIZE + VALUE_SIZE +DEFAULT_COUNT_OPERANDS: Final[int] = (CHEAP_PHRASE_IDS + 1) // (1 << DEFAULT_FLAG_BITS) - 1 diff --git a/src/sampletones_tools/codec/study/sandbox/decode.py b/src/sampletones_tools/codec/study/sandbox/decode.py new file mode 100644 index 000000000..720064704 --- /dev/null +++ b/src/sampletones_tools/codec/study/sandbox/decode.py @@ -0,0 +1,53 @@ +from typing import Sequence + +from sampletones_player.compression.dictionary.table import PhraseTable +from sampletones_player.specification.binary import BYTE_VALUES +from sampletones_player.specification.compression import INITIAL_PLANE_VALUE +from sampletones_tools.codec.study.sandbox.tokens import Hold, Literal, Play, SetHold, StudyToken, WideHold + + +def _played( + token: Play, + table: PhraseTable, +) -> bytes: + body = table[token.phrase_id].body + last = len(body) - 1 + return bytes((body[min(offset, last)] + token.transpose) % BYTE_VALUES for offset in range(token.ticks)) + + +def play_tokens( + tokens: Sequence[StudyToken], + table: PhraseTable, + ticks: int, +) -> bytes: + """Plays a plane's tokens back into the values they write, tick by tick. + + This is the reading a driver of the grammar would perform, stated over tokens: a hold + keeps the value reached, a phrase played past its end holds its final value onward, and + every plane opens on the value the driver seeds it to. + + Args: + tokens: The tokens the plane is written as, in the order they are read. + table: The dictionary the tokens name. + ticks: The ticks the song lasts. + + Returns: + bytes: The values the plane writes, one per tick. + """ + values = bytearray() + current = INITIAL_PLANE_VALUE + for token in tokens: + match token: + case Hold() | WideHold(): + played = bytes([current]) * token.ticks + case SetHold(): + played = bytes([token.value]) * token.ticks + case Literal(): + played = token.values + case Play(): + played = _played(token, table) + + current = played[-1] + values.extend(played) + + return bytes(values[:ticks]) diff --git a/src/sampletones_tools/codec/study/sandbox/defaults.py b/src/sampletones_tools/codec/study/sandbox/defaults.py new file mode 100644 index 000000000..34c697823 --- /dev/null +++ b/src/sampletones_tools/codec/study/sandbox/defaults.py @@ -0,0 +1,44 @@ +from collections import Counter +from typing import Final, List, Sequence, Tuple + +from sampletones_tools.codec.study.sandbox.parse import StudyParse +from sampletones_tools.codec.study.sandbox.tokens import Play + +NO_DEFAULT: Final[int] = 0 + + +def no_defaults(phrases: int) -> Tuple[int, ...]: + """A default count for none of ``phrases`` phrases.""" + return (NO_DEFAULT,) * phrases + + +def modal_counts( + parses: Sequence[StudyParse], + phrases: int, +) -> Tuple[int, ...]: + """The count each phrase is played at most often across the parses (H4). + + A phrase's default count is the one its tokens would leave unstated most often, so it is + read off the parse as it stands. A phrase the parses never play keeps no default. + + Args: + parses: Every plane's parse. + phrases: The phrases the table holds. + + Returns: + Tuple[int, ...]: The default count of each phrase, by id, zero where it has none. + """ + counters: List[Counter[int]] = [Counter() for _ in range(phrases)] + for parse in parses: + for token in parse.tokens: + if isinstance(token, Play): + counters[token.phrase_id][token.ticks] += 1 + + return tuple(_modal(counter) for counter in counters) + + +def _modal(counter: Counter[int]) -> int: + if not counter: + return NO_DEFAULT + + return min(counter, key=lambda ticks: (-counter[ticks], ticks)) diff --git a/tests/unit/sampletones_synthesis/filters/__init__.py b/src/sampletones_tools/codec/study/sandbox/edges/__init__.py similarity index 100% rename from tests/unit/sampletones_synthesis/filters/__init__.py rename to src/sampletones_tools/codec/study/sandbox/edges/__init__.py diff --git a/src/sampletones_tools/codec/study/sandbox/edges/generator.py b/src/sampletones_tools/codec/study/sandbox/edges/generator.py new file mode 100644 index 000000000..33a080306 --- /dev/null +++ b/src/sampletones_tools/codec/study/sandbox/edges/generator.py @@ -0,0 +1,53 @@ +from typing import Iterable, Protocol + +from sampletones_tools.codec.study.sandbox.context import PlaneContext +from sampletones_tools.codec.study.sandbox.shortest import Shortest +from sampletones_tools.codec.study.sandbox.tokens import StudyToken + + +class EdgeGenerator(Protocol): + """Offers every token of one kind that may start at a tick of a plane.""" + + def __call__( + self, + context: PlaneContext, + shortest: Shortest[StudyToken], + position: int, + reach: int, + *, + holdable: bool, + ) -> None: + """Relaxes the edges of the tokens starting at ``position``. + + Args: + context: The plane and the terms the grammar reads it on. + shortest: The search the edges join. + position: The tick the tokens start on. + reach: The most ticks a token may cover from there. + holdable: Whether a token may keep the value the plane holds as the tick is reached. + """ + + +def offered_lengths( + longest: int, + *, + least: int, + every: bool, +) -> Iterable[int]: + """The lengths a token kind is offered at from one tick. + + The codec as it stands offers the longest token alone, and a grammar asking for every + length offers each one from ``least`` up to it. + + Args: + longest: The most ticks the token may cover. + least: The fewest ticks a token of the kind covers. + every: Whether every length is offered. + + Returns: + Iterable[int]: The lengths, the longest one last. + """ + if every: + return range(least, longest + 1) + + return (longest,) diff --git a/src/sampletones_tools/codec/study/sandbox/edges/holds.py b/src/sampletones_tools/codec/study/sandbox/edges/holds.py new file mode 100644 index 000000000..d8f703298 --- /dev/null +++ b/src/sampletones_tools/codec/study/sandbox/edges/holds.py @@ -0,0 +1,79 @@ +from typing import Final + +from sampletones_player.specification.compression import MAX_HOLD_TICKS +from sampletones_tools.codec.study.sandbox.context import PlaneContext +from sampletones_tools.codec.study.sandbox.edges.generator import EdgeGenerator, offered_lengths +from sampletones_tools.codec.study.sandbox.shortest import Shortest +from sampletones_tools.codec.study.sandbox.tokens import Hold, StudyToken, WideHold + +LEAST_HOLD_TICKS: Final[int] = 1 +LEAST_WIDE_BLOCKS: Final[int] = 1 + + +def hold_edges(*, every_length: bool) -> EdgeGenerator: + """Holds, as the codec offers them: over the value the plane holds as the tick is reached. + + Args: + every_length: Whether a hold of every length is offered beside the longest one. + + Returns: + EdgeGenerator: The generator. + """ + + def relax( + context: PlaneContext, + shortest: Shortest[StudyToken], + position: int, + reach: int, + *, + holdable: bool, + ) -> None: + index = context.index + if not holdable or index.plane[position] != context.value_before(position): + return + + longest = min(MAX_HOLD_TICKS, index.runs[position], reach) + cost = shortest.costs[position] + context.costs.hold + for ticks in offered_lengths(longest, least=LEAST_HOLD_TICKS, every=every_length): + if shortest.improves(position + ticks, cost): + shortest.relax(position, position + ticks, cost, Hold(ticks=ticks)) + + return relax + + +def wide_hold_edges(*, every_length: bool) -> EdgeGenerator: + """Wide holds (H1): whole blocks of a plain hold's longest reach, counted by one token. + + A wide hold rides the free escape, so it counts as many blocks as the escape has operand + values, and a run shorter than a block is left to a plain hold. + + Args: + every_length: Whether a wide hold of every block count is offered beside the longest. + + Returns: + EdgeGenerator: The generator. + """ + + def relax( + context: PlaneContext, + shortest: Shortest[StudyToken], + position: int, + reach: int, + *, + holdable: bool, + ) -> None: + index = context.index + if not holdable or index.plane[position] != context.value_before(position): + return + + longest = min(context.costs.operands, index.runs[position] // MAX_HOLD_TICKS, reach // MAX_HOLD_TICKS) + if longest < LEAST_WIDE_BLOCKS: + return + + cost = shortest.costs[position] + context.costs.wide_hold + for blocks in offered_lengths(longest, least=LEAST_WIDE_BLOCKS, every=every_length): + end = position + blocks * MAX_HOLD_TICKS + if shortest.improves(end, cost): + shortest.relax(position, end, cost, WideHold(blocks=blocks)) + + return relax diff --git a/src/sampletones_tools/codec/study/sandbox/edges/literals.py b/src/sampletones_tools/codec/study/sandbox/edges/literals.py new file mode 100644 index 000000000..8c19c1950 --- /dev/null +++ b/src/sampletones_tools/codec/study/sandbox/edges/literals.py @@ -0,0 +1,29 @@ +from sampletones_player.compression.parse.literals import LiteralWindow +from sampletones_tools.codec.study.sandbox.context import PlaneContext +from sampletones_tools.codec.study.sandbox.shortest import Shortest +from sampletones_tools.codec.study.sandbox.tokens import Literal, StudyToken + + +def relax_literal( + context: PlaneContext, + shortest: Shortest[StudyToken], + window: LiteralWindow, + position: int, + earliest: int, +) -> None: + """Spelling the values out reaches ``position`` from wherever that costs least. + + Literals are the one token every grammar keeps, priced as the codec prices them: an opcode + and the values, from the start the window finds cheapest. + + Args: + context: The plane and the terms the grammar reads it on. + shortest: The search the edge joins. + window: The starts a literal ending here may take, the cheapest kept at the front. + position: The tick the literal ends at. + earliest: The earliest tick the literal may start on. + """ + start = window.cheapest(position, earliest) + cost = shortest.costs[start] + context.costs.literal(position - start) + if shortest.improves(position, cost): + shortest.relax(start, position, cost, Literal(values=context.index.plane[start:position])) diff --git a/src/sampletones_tools/codec/study/sandbox/edges/phrases.py b/src/sampletones_tools/codec/study/sandbox/edges/phrases.py new file mode 100644 index 000000000..8b5791611 --- /dev/null +++ b/src/sampletones_tools/codec/study/sandbox/edges/phrases.py @@ -0,0 +1,57 @@ +from sampletones_player.specification.compression import MAX_PHRASE_TICKS +from sampletones_tools.codec.study.sandbox.context import PlaneContext +from sampletones_tools.codec.study.sandbox.edges.generator import EdgeGenerator +from sampletones_tools.codec.study.sandbox.shortest import Shortest +from sampletones_tools.codec.study.sandbox.tokens import Play, StudyToken + + +def phrase_edges(*, defaults: bool) -> EdgeGenerator: + """Phrases, as the codec offers them: each one the plane plays from the tick, played whole. + + With defaults on, a phrase played at least its default count from the tick is offered at + that count too, as the token that carries no count of its own (H4). + + Args: + defaults: Whether the default-count token is offered. + + Returns: + EdgeGenerator: The generator. + """ + + def relax( + context: PlaneContext, + shortest: Shortest[StudyToken], + position: int, + reach: int, + *, + holdable: bool, + ) -> None: + del holdable + cost = shortest.costs[position] + costs = context.costs + for phrase_id, ticks, transpose in context.matcher.matches( + position, + min(MAX_PHRASE_TICKS, reach), + transposition=context.transposition, + ): + stated = cost + costs.phrase(phrase_id, transpose, default=False) + if shortest.improves(position + ticks, stated): + shortest.relax( + position, + position + ticks, + stated, + Play(phrase_id=phrase_id, ticks=ticks, transpose=transpose, default=False), + ) + + count = context.defaults[phrase_id] if defaults else 0 + if 0 < count <= ticks: + unstated = cost + costs.phrase(phrase_id, transpose, default=True) + if shortest.improves(position + count, unstated): + shortest.relax( + position, + position + count, + unstated, + Play(phrase_id=phrase_id, ticks=count, transpose=transpose, default=True), + ) + + return relax diff --git a/src/sampletones_tools/codec/study/sandbox/edges/set_hold.py b/src/sampletones_tools/codec/study/sandbox/edges/set_hold.py new file mode 100644 index 000000000..17a62f2e7 --- /dev/null +++ b/src/sampletones_tools/codec/study/sandbox/edges/set_hold.py @@ -0,0 +1,45 @@ +from typing import Final + +from sampletones_tools.codec.study.sandbox.context import PlaneContext +from sampletones_tools.codec.study.sandbox.edges.generator import EdgeGenerator, offered_lengths +from sampletones_tools.codec.study.sandbox.shortest import Shortest +from sampletones_tools.codec.study.sandbox.tokens import SetHold, StudyToken + +LEAST_SET_HOLD_TICKS: Final[int] = 2 + + +def set_hold_edges(*, every_length: bool) -> EdgeGenerator: + """Set-holds (H3): one token taking a value and keeping it for a run of ticks. + + The token names its value, so it starts anywhere a run does, the stream's first tick and a + loop entry included. A run of one tick is a literal's, which costs the same and carries + more. + + Args: + every_length: Whether a set-hold of every length is offered beside the longest one. + + Returns: + EdgeGenerator: The generator. + """ + + def relax( + context: PlaneContext, + shortest: Shortest[StudyToken], + position: int, + reach: int, + *, + holdable: bool, + ) -> None: + del holdable + index = context.index + longest = min(context.costs.operands, index.runs[position], reach) + if longest < LEAST_SET_HOLD_TICKS: + return + + value = index.plane[position] + cost = shortest.costs[position] + context.costs.set_hold + for ticks in offered_lengths(longest, least=LEAST_SET_HOLD_TICKS, every=every_length): + if shortest.improves(position + ticks, cost): + shortest.relax(position, position + ticks, cost, SetHold(value=value, ticks=ticks)) + + return relax diff --git a/src/sampletones_tools/codec/study/sandbox/encode.py b/src/sampletones_tools/codec/study/sandbox/encode.py new file mode 100644 index 000000000..ff6279c8d --- /dev/null +++ b/src/sampletones_tools/codec/study/sandbox/encode.py @@ -0,0 +1,80 @@ +from time import process_time +from typing import Final, Sequence, Tuple + +from sampletones_tools.codec.study.measure import Encoding +from sampletones_tools.codec.study.sandbox.defaults import modal_counts, no_defaults +from sampletones_tools.codec.study.sandbox.grammar import Grammar +from sampletones_tools.codec.study.sandbox.parse import StudyParse, parse_plane +from sampletones_tools.codec.study.sandbox.reference import Reference +from sampletones_tools.codec.study.sandbox.verify import plays_back + +DEFAULT_ROUNDS: Final[int] = 3 + + +def encode_grammar( + reference: Reference, + grammar: Grammar, +) -> Encoding: + """Prices a song under a grammar, over the dictionary the production codec settled on. + + Args: + reference: The song, its dictionary and its matches. + grammar: The grammar. + + Returns: + Encoding: The bytes each stream would take, timed, and whether the tokens play back. + """ + started = process_time() + parses = _parses(reference, grammar) + seconds = process_time() - started + return Encoding( + phrases=len(reference.table), + dictionary=grammar.costs.dictionary(reference.table), + streams=tuple(parse.size for parse in parses), + seconds=seconds, + lossless=plays_back(parses, reference.table, reference.song.planes), + written=None, + ) + + +def _parse_all( + reference: Reference, + grammar: Grammar, + defaults: Sequence[int], +) -> Tuple[StudyParse, ...]: + return tuple(parse_plane(context, grammar) for context in reference.contexts(grammar.costs, defaults)) + + +def _streams(parses: Sequence[StudyParse]) -> int: + return sum(parse.size for parse in parses) + + +def _parses( + reference: Reference, + grammar: Grammar, +) -> Tuple[StudyParse, ...]: + """Every plane under the grammar, its default counts settled where it carries them. + + A default count is read off the parse it serves, so the first reading takes the counts the + baseline parse plays each phrase at, and each further round reads them off the parse that + used them, stopping where they stand still or stop paying. + """ + phrases = len(reference.table) + if not grammar.default_counts: + return _parse_all(reference, grammar, no_defaults(phrases)) + + defaults = modal_counts(reference.baseline, phrases) + parses = _parse_all(reference, grammar, defaults) + for _ in range(DEFAULT_ROUNDS - 1): + refined = modal_counts(parses, phrases) + if refined == defaults: + break + + trial = _parse_all(reference, grammar, refined) + if _streams(trial) >= _streams(parses): + break + + defaults = refined + parses = trial + + return parses diff --git a/src/sampletones_tools/codec/study/sandbox/grammar.py b/src/sampletones_tools/codec/study/sandbox/grammar.py new file mode 100644 index 000000000..1827e2924 --- /dev/null +++ b/src/sampletones_tools/codec/study/sandbox/grammar.py @@ -0,0 +1,40 @@ +from dataclasses import dataclass +from typing import Final + +from sampletones_tools.codec.study.sandbox.costs import PRODUCTION_COSTS, Costs + + +@dataclass(frozen=True) +class Grammar: + """One token grammar the sandbox prices a plane under. + + The baseline grammar is the codec as it stands, and every other one switches a change on + over it, so what a change earns is read as the difference between the two on the same plane + and the same dictionary. + + Attributes: + every_length: Whether a hold of every length is offered beside the longest one (H9a). + start_hold: Whether a hold may open a stream, over the value the driver seeds every + plane to (H9b). + wide_hold: Whether a wide hold covers blocks of a plain hold's longest reach at once (H1). + set_hold: Whether one token sets a value and holds it (H3). + default_counts: Whether a phrase carries a default count a token may leave unstated (H4). + costs: The bytes each token takes. + """ + + every_length: bool + start_hold: bool + wide_hold: bool + set_hold: bool + default_counts: bool + costs: Costs + + +BASELINE_GRAMMAR: Final[Grammar] = Grammar( + every_length=False, + start_hold=False, + wide_hold=False, + set_hold=False, + default_counts=False, + costs=PRODUCTION_COSTS, +) diff --git a/src/sampletones_tools/codec/study/sandbox/parse.py b/src/sampletones_tools/codec/study/sandbox/parse.py new file mode 100644 index 000000000..b6b5c416d --- /dev/null +++ b/src/sampletones_tools/codec/study/sandbox/parse.py @@ -0,0 +1,130 @@ +from dataclasses import dataclass +from typing import List, Sequence, Tuple + +from sampletones_player.compression.parse.literals import LiteralWindow +from sampletones_player.specification.compression import MAX_LITERAL_BYTES +from sampletones_tools.codec.study.sandbox.context import PlaneContext +from sampletones_tools.codec.study.sandbox.edges.generator import EdgeGenerator +from sampletones_tools.codec.study.sandbox.edges.holds import ( + hold_edges, + wide_hold_edges, +) +from sampletones_tools.codec.study.sandbox.edges.literals import relax_literal +from sampletones_tools.codec.study.sandbox.edges.phrases import phrase_edges +from sampletones_tools.codec.study.sandbox.edges.set_hold import set_hold_edges +from sampletones_tools.codec.study.sandbox.grammar import Grammar +from sampletones_tools.codec.study.sandbox.shortest import Shortest +from sampletones_tools.codec.study.sandbox.tokens import StudyToken + + +@dataclass(frozen=True) +class StudyParse: + """A plane read as the tokens of one grammar, alongside what each of its prefixes costs. + + Attributes: + tokens: The tokens the plane is written as, in the order they are read. + costs: The bytes each prefix of the plane takes, the whole plane's cost last. + """ + + tokens: Tuple[StudyToken, ...] + costs: Tuple[int, ...] + + @property + def size(self) -> int: + """The bytes the plane's token stream takes.""" + return self.costs[-1] + + +def generators(grammar: Grammar) -> Tuple[EdgeGenerator, ...]: + """The token kinds a grammar offers from a tick, each as the edges it adds to the search. + + Args: + grammar: The grammar. + + Returns: + Tuple[EdgeGenerator, ...]: The generators, holds first and phrases last. + """ + chosen: List[EdgeGenerator] = [hold_edges(every_length=grammar.every_length)] + if grammar.wide_hold: + chosen.append(wide_hold_edges(every_length=grammar.every_length)) + + if grammar.set_hold: + chosen.append(set_hold_edges(every_length=grammar.every_length)) + + chosen.append(phrase_edges(defaults=grammar.default_counts)) + return tuple(chosen) + + +def _relax_forward( + offered: Sequence[EdgeGenerator], + context: PlaneContext, + shortest: Shortest[StudyToken], + position: int, + reach: int, + *, + holdable: bool, +) -> None: + for generator in offered: + generator( + context, + shortest, + position, + reach, + holdable=holdable, + ) + + +def parse_plane( + context: PlaneContext, + grammar: Grammar, +) -> StudyParse: + """Reads a plane as the cheapest token stream ``grammar`` allows. + + This is the production parse with the token kinds plugged in: literals reach every tick + from their cheapest start, and the grammar's other kinds are offered forward from each tick + a token may start on. + + Args: + context: The plane and the terms the grammar reads it on. + grammar: The grammar. + + Returns: + StudyParse: The tokens the plane is written as, and what each of its prefixes costs. + """ + ticks = context.index.ticks + entries = context.boundaries + previous = entries.previous + following = entries.following + offered = generators(grammar) + shortest: Shortest[StudyToken] = Shortest.across(ticks) + window = LiteralWindow(shortest.costs) + _relax_forward( + offered, + context, + shortest, + 0, + following[0], + holdable=grammar.start_hold, + ) + for position in range(1, ticks + 1): + relax_literal( + context, + shortest, + window, + position, + max(position - MAX_LITERAL_BYTES, previous[position]), + ) + if position < ticks: + _relax_forward( + offered, + context, + shortest, + position, + following[position] - position, + holdable=position not in entries.entries, + ) + + return StudyParse( + tokens=shortest.walk(ticks), + costs=tuple(shortest.costs), + ) diff --git a/src/sampletones_tools/codec/study/sandbox/reference.py b/src/sampletones_tools/codec/study/sandbox/reference.py new file mode 100644 index 000000000..a02973524 --- /dev/null +++ b/src/sampletones_tools/codec/study/sandbox/reference.py @@ -0,0 +1,108 @@ +from dataclasses import dataclass +from typing import Final, FrozenSet, Sequence, Tuple + +from sampletones_player.compression.compressed import CompressedPlanes +from sampletones_player.compression.dictionary.table import PhraseTable +from sampletones_player.compression.encode import STREAM_START +from sampletones_player.compression.matches.cache import MatchCache +from sampletones_player.compression.matches.index import PlaneIndex +from sampletones_player.compression.matches.matcher import PhraseMatcher +from sampletones_player.compression.options import EVERY_LAYER +from sampletones_player.compression.parse.boundaries import Boundaries +from sampletones_tools.codec.study.corpus.song import StudySong +from sampletones_tools.codec.study.sandbox.context import PlaneContext +from sampletones_tools.codec.study.sandbox.costs import PRODUCTION_COSTS, Costs +from sampletones_tools.codec.study.sandbox.defaults import no_defaults +from sampletones_tools.codec.study.sandbox.grammar import BASELINE_GRAMMAR +from sampletones_tools.codec.study.sandbox.parse import StudyParse, parse_plane +from sampletones_tools.codec.study.sandbox.verify import verify_baseline + +STREAM_ENTRIES: Final[FrozenSet[int]] = frozenset({STREAM_START}) + + +@dataclass(frozen=True) +class Reference: + """One song as every grammar is priced against it. + + The dictionary is the one the production codec settled on, held fixed across the grammars: + what a grammar earns is read on the streams alone, and how the search and the settling + would answer a new grammar is left to its production layer to measure. + + Attributes: + song: The song. + table: The dictionary the production codec settled on. + cache: What each phrase plays against each plane, shared by every grammar's parse. + baseline: Every plane under the baseline grammar, held to the production streams. + """ + + song: StudySong + table: PhraseTable + cache: MatchCache + baseline: Tuple[StudyParse, ...] + + def contexts( + self, + costs: Costs, + defaults: Sequence[int], + ) -> Tuple[PlaneContext, ...]: + """Every plane of the song under one grammar's terms. + + Args: + costs: The bytes each token takes. + defaults: The default count of each phrase, by id. + + Returns: + Tuple[PlaneContext, ...]: One context per plane, in song-block order. + """ + return _contexts(self.cache, self.table, costs, defaults) + + +def _contexts( + cache: MatchCache, + table: PhraseTable, + costs: Costs, + defaults: Sequence[int], +) -> Tuple[PlaneContext, ...]: + contexts = [] + for plane, index in enumerate(cache.indices): + contexts.append( + PlaneContext( + index=index, + matcher=PhraseMatcher(table, plane, cache), + boundaries=Boundaries.across(index.ticks, STREAM_ENTRIES), + transposition=EVERY_LAYER.transposition, + defaults=tuple(defaults), + costs=costs, + ) + ) + + return tuple(contexts) + + +def reference( + song: StudySong, + compressed: CompressedPlanes, +) -> Reference: + """Reads a song into what every grammar is priced against, proving the parser on the way. + + Args: + song: The song. + compressed: The production encoding of the song. + + Returns: + Reference: The song, its dictionary, its matches and its baseline parse. + + Raises: + ValueError: If the baseline grammar prices a plane differently from the codec's stream. + """ + cache = MatchCache(PlaneIndex.from_plane(plane) for plane in song.planes.planes) + table = compressed.phrases + contexts = _contexts(cache, table, PRODUCTION_COSTS, no_defaults(len(table))) + baseline = tuple(parse_plane(context, BASELINE_GRAMMAR) for context in contexts) + verify_baseline(baseline, compressed) + return Reference( + song=song, + table=table, + cache=cache, + baseline=baseline, + ) diff --git a/src/sampletones_tools/codec/study/sandbox/shortest.py b/src/sampletones_tools/codec/study/sandbox/shortest.py new file mode 100644 index 000000000..8bfa57f44 --- /dev/null +++ b/src/sampletones_tools/codec/study/sandbox/shortest.py @@ -0,0 +1,86 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import Final, Generic, List, Optional, Tuple, TypeVar + +UNREACHED: Final[int] = -1 + +Token = TypeVar("Token") + + +@dataclass +class Shortest(Generic[Token]): + """The cheapest way found so far of reaching each tick of a plane, over any token kind. + + Every token is an edge from the tick it starts on to the tick after the ones it covers, and + its cost is the bytes it takes, so the cheapest path across the plane is its encoding. This + is the production search stated over whichever tokens a grammar offers. + + Attributes: + costs: The bytes reaching each tick takes, ticks not yet reached holding ``UNREACHED``. + origins: The tick each one was reached from. + tokens: The token each tick was reached by. + """ + + costs: List[int] + origins: List[int] + tokens: List[Optional[Token]] + + @classmethod + def across(cls, ticks: int) -> Shortest[Token]: + """Opens a search over a plane of ``ticks`` ticks, its first tick free to reach.""" + return cls( + costs=[0] + [UNREACHED] * ticks, + origins=[0] * (ticks + 1), + tokens=[None] * (ticks + 1), + ) + + def improves(self, end: int, cost: int) -> bool: + """Whether reaching ``end`` for ``cost`` beats what it has been reached for so far.""" + return self.costs[end] == UNREACHED or cost < self.costs[end] + + def relax( + self, + start: int, + end: int, + cost: int, + token: Token, + ) -> None: + """Takes a token as the way to reach ``end`` where it is the cheapest one found. + + Args: + start: The tick the token starts on. + end: The tick following the ones the token covers. + cost: The bytes reaching ``end`` through this token takes. + token: The token covering the ticks between the two. + """ + if self.costs[end] != UNREACHED and self.costs[end] <= cost: + return + + self.costs[end] = cost + self.origins[end] = start + self.tokens[end] = token + + def walk(self, ticks: int) -> Tuple[Token, ...]: + """Reads the tokens of the cheapest path back from ``ticks`` to the plane's first tick. + + Args: + ticks: The tick the path ends at, which is the ticks the plane covers. + + Returns: + Tuple[Token, ...]: The tokens, in the order they are read. + + Raises: + ValueError: If a tick along the path was never reached. + """ + read: List[Token] = [] + position = ticks + while position > 0: + token = self.tokens[position] + if token is None: + raise ValueError(f"tick {position} of the plane was left unreachable") + + read.append(token) + position = self.origins[position] + + return tuple(reversed(read)) diff --git a/src/sampletones_tools/codec/study/sandbox/tokens.py b/src/sampletones_tools/codec/study/sandbox/tokens.py new file mode 100644 index 000000000..4accffcb2 --- /dev/null +++ b/src/sampletones_tools/codec/study/sandbox/tokens.py @@ -0,0 +1,59 @@ +from dataclasses import dataclass +from typing import Union + +from sampletones_player.specification.compression import MAX_HOLD_TICKS + + +@dataclass(frozen=True) +class Hold: + """The plane keeps the value it reached, for ``ticks`` ticks.""" + + ticks: int + + +@dataclass(frozen=True) +class WideHold: + """The plane keeps the value it reached for ``blocks`` blocks of a plain hold's longest reach (H1).""" + + blocks: int + + @property + def ticks(self) -> int: + """The ticks the token covers.""" + return self.blocks * MAX_HOLD_TICKS + + +@dataclass(frozen=True) +class SetHold: + """The plane takes ``value`` and keeps it for ``ticks`` ticks (H3).""" + + value: int + ticks: int + + +@dataclass(frozen=True) +class Literal: + """The plane takes the values verbatim, one per tick.""" + + values: bytes + + @property + def ticks(self) -> int: + """The ticks the token covers.""" + return len(self.values) + + +@dataclass(frozen=True) +class Play: + """The plane plays a phrase from the table, shifted by ``transpose``, for ``ticks`` ticks. + + A default play covers the phrase's own default count and carries no count of its own (H4). + """ + + phrase_id: int + ticks: int + transpose: int + default: bool + + +StudyToken = Union[Hold, WideHold, SetHold, Literal, Play] diff --git a/src/sampletones_tools/codec/study/sandbox/verify.py b/src/sampletones_tools/codec/study/sandbox/verify.py new file mode 100644 index 000000000..a4728a5e3 --- /dev/null +++ b/src/sampletones_tools/codec/study/sandbox/verify.py @@ -0,0 +1,51 @@ +from typing import Sequence + +from sampletones_player.compression.compressed import CompressedPlanes +from sampletones_player.compression.dictionary.table import PhraseTable +from sampletones_player.compression.planes.order import PlaneOrder +from sampletones_player.compression.planes.song import SongPlanes +from sampletones_tools.codec.study.sandbox.decode import play_tokens +from sampletones_tools.codec.study.sandbox.parse import StudyParse + + +def verify_baseline( + parses: Sequence[StudyParse], + compressed: CompressedPlanes, +) -> None: + """Holds the sandbox's baseline grammar to the streams the production codec wrote. + + The sandbox prices every grammar on its own parser, so the parser is proven on the one + grammar whose answer is known: each plane under the baseline grammar costs exactly the + bytes the production stream takes, or the study is reading the planes wrong. + + Args: + parses: Every plane under the baseline grammar. + compressed: The production encoding of the same planes over the same dictionary. + + Raises: + ValueError: If a plane is priced differently from the stream the codec wrote. + """ + for name, parse, stream in zip(PlaneOrder.names(), parses, compressed.streams): + if parse.size != len(stream): + raise ValueError( + f"the sandbox reads {name} as {parse.size} bytes where the codec wrote {len(stream)}; " + "the baseline grammar has parted from the production parser" + ) + + +def plays_back( + parses: Sequence[StudyParse], + table: PhraseTable, + planes: SongPlanes, +) -> bool: + """Whether every plane's tokens play back to the plane they were written from. + + Args: + parses: Every plane's parse, in the order the song block writes them. + table: The dictionary the tokens name. + planes: The planes the parses were read from. + + Returns: + bool: Whether the encoding is lossless. + """ + return all(play_tokens(parse.tokens, table, planes.ticks) == plane for parse, plane in zip(parses, planes.planes)) diff --git a/src/sampletones_tools/codec/study/session.py b/src/sampletones_tools/codec/study/session.py new file mode 100644 index 000000000..a98c1fd87 --- /dev/null +++ b/src/sampletones_tools/codec/study/session.py @@ -0,0 +1,105 @@ +from pathlib import Path +from typing import List, Optional, Tuple + +from sampletones_shared.logger import logger +from sampletones_shared.utils.text import listed_items +from sampletones_tools.codec.study.corpus.build import build_corpus +from sampletones_tools.codec.study.manifest import StudyManifest, StudySource +from sampletones_tools.codec.study.measure import Measurement, measure +from sampletones_tools.codec.study.plan import StudyPlan +from sampletones_tools.codec.study.report.run import run_directory, write_run +from sampletones_tools.codec.study.variants.registry import EVERY_VARIANT +from sampletones_tools.codec.study.variants.strategy import STRATEGY_ORDER, depth_measurements + + +def variant_names(stated: Optional[str]) -> Tuple[str, ...]: + """The variants a run encodes under: the ones named, comma separated, or every one.""" + if stated is None: + return (EVERY_VARIANT,) + + return tuple(listed_items(stated)) + + +def resolve_manifest( + path: Optional[Path], + *, + projects: Tuple[Path, ...], + reconstructions: Tuple[Path, ...], + lengthen_seconds: int, + variants: Tuple[str, ...], +) -> StudyManifest: + """The manifest a run measures: the sources named outright, or the ones a manifest file states. + + Sources named outright stand in for a manifest's own, while its lengthening and variants stay. + + Args: + path: A manifest a run wrote, or ``None`` to measure the sources named outright. + projects: Project files to measure. + reconstructions: Stem files, or directories of stems, to measure. + lengthen_seconds: How long each project's lengthened copy lasts, unless a manifest states it. + variants: The names of the variants every song is encoded under, unless a manifest states them. + + Raises: + ValueError: If no manifest stands at the path, a source names nothing, or the manifest + names no source or breaks one of its bounds. + """ + if path is not None and not path.is_file(): + raise ValueError(f"No manifest at {path}.") + + manifest = StudyManifest.load(path) if path is not None else None + if manifest is not None and not projects and not reconstructions: + return manifest + + return StudyManifest( + projects=tuple(StudySource.at(project) for project in projects), + reconstructions=tuple(StudySource.at(reconstruction) for reconstruction in reconstructions), + lengthen_seconds=manifest.lengthen_seconds if manifest is not None else lengthen_seconds, + variants=manifest.variants if manifest is not None else variants, + ) + + +def run_study(plan: StudyPlan, output: Optional[Path]) -> Path: + """Encodes every song of the plan's manifest under every variant it names and writes the run. + + The corpus is read before the run's directory is made, so a source that fails to read leaves + the output untouched. + + Args: + plan: What is measured and the variants it is encoded under. + output: The directory the run writes into, or ``None`` for a stamped one under the + documents. + + Returns: + Path: The directory holding the report, the accounting and the verdicts. + + Raises: + ValueError: If a variant writes a song as streams that play back differently. + """ + corpus = build_corpus(plan.manifest) + directory = run_directory(output) + + measurements: List[Measurement] = [] + for song in corpus: + for variant in plan.variants: + if not variant.applies(song): + continue + + logger.info(f"Encoding {song.name} ({song.ticks} ticks) under {variant.name}") + measurement = measure(song, variant.name, variant.encode) + if not measurement.lossless: + raise ValueError(f"{variant.name} wrote {song.name} as streams that play back differently") + + logger.info( + f" {measurement.block} bytes, {measurement.bytes_per_tick:.3f} bytes per tick, " + f"{measurement.phrases} phrases, {measurement.seconds:.1f} s" + ) + measurements.append(measurement) + + write_run( + directory, + plan, + measurements, + depth_measurements(measurements, STRATEGY_ORDER), + ) + logger.info(f"Report written to {directory}") + return directory diff --git a/tests/unit/sampletones_synthesis/oscillators/__init__.py b/src/sampletones_tools/codec/study/variants/__init__.py similarity index 100% rename from tests/unit/sampletones_synthesis/oscillators/__init__.py rename to src/sampletones_tools/codec/study/variants/__init__.py diff --git a/src/sampletones_tools/codec/study/variants/baselines.py b/src/sampletones_tools/codec/study/variants/baselines.py new file mode 100644 index 000000000..c47e54475 --- /dev/null +++ b/src/sampletones_tools/codec/study/variants/baselines.py @@ -0,0 +1,91 @@ +from time import process_time +from typing import Dict + +from sampletones_player.compression.compressed import CompressedPlanes +from sampletones_tools.codec.study.corpus.song import StudySong +from sampletones_tools.codec.study.measure import Encoding, production_encoding +from sampletones_tools.codec.study.sandbox.reference import Reference, reference +from sampletones_tools.codec.study.variants.production import BASELINE_NAME, compress_baseline +from sampletones_tools.codec.study.variants.variant import Variant, VariantKind + + +class Baselines: + """The production encoding of each song, taken once and read by every variant built on it. + + A run measures a song under every variant before moving to the next, so one song's + encoding is kept at a time: the baseline variant writes it, and the sandbox reads the + dictionary and the planes' matches from there in place of encoding the song again. A + song asked for before its baseline ran is encoded on the spot. + """ + + def __init__(self) -> None: + self._compressed: Dict[str, CompressedPlanes] = {} + self._references: Dict[str, Reference] = {} + + def encode(self, song: StudySong) -> Encoding: + """Encodes a song as the export does today, keeping the result for the sandbox. + + Args: + song: The song to encode. + + Returns: + Encoding: The encoding, its streams kept as written. + """ + started = process_time() + compressed = compress_baseline(song) + seconds = process_time() - started + self._compressed = {song.name: compressed} + self._references = {} + return production_encoding(song, compressed, seconds) + + def compressed(self, song: StudySong) -> CompressedPlanes: + """The production encoding of ``song``. + + Args: + song: The song. + + Returns: + CompressedPlanes: The dictionary and the streams the codec wrote. + """ + remembered = self._compressed.get(song.name) + if remembered is None: + remembered = compress_baseline(song) + self._compressed = {song.name: remembered} + self._references = {} + + return remembered + + def reference(self, song: StudySong) -> Reference: + """What every grammar prices ``song`` against. + + Args: + song: The song. + + Returns: + Reference: The song, its dictionary, its matches and its baseline parse. + """ + remembered = self._references.get(song.name) + if remembered is None: + remembered = reference(song, self.compressed(song)) + self._references = {song.name: remembered} + + return remembered + + +def baseline_variant(baselines: Baselines) -> Variant: + """The codec as it stands, which every other variant is held against. + + Args: + baselines: Where the encodings are kept for the variants built on them. + + Returns: + Variant: The baseline. + """ + return Variant( + name=BASELINE_NAME, + hypothesis="", + kind=VariantKind.BASELINE, + note="", + encode=baselines.encode, + needs_seeds=False, + ) diff --git a/src/sampletones_tools/codec/study/variants/production.py b/src/sampletones_tools/codec/study/variants/production.py new file mode 100644 index 000000000..e483c40e9 --- /dev/null +++ b/src/sampletones_tools/codec/study/variants/production.py @@ -0,0 +1,198 @@ +from time import process_time +from typing import Callable, Final, Sequence, Tuple + +from sampletones_player.compression.budget import DEFAULT_SEARCH_BUDGET, SearchBudget +from sampletones_player.compression.compressed import CompressedPlanes +from sampletones_player.compression.dictionary.phrase import Phrase +from sampletones_player.compression.encode import encode_planes +from sampletones_player.compression.options import EVERY_LAYER +from sampletones_tools.codec.study.corpus.song import StudySong +from sampletones_tools.codec.study.measure import Encoder, Encoding, production_encoding +from sampletones_tools.codec.study.variants.seeds import split, trimmed, whole_and_split +from sampletones_tools.codec.study.variants.variant import Variant, VariantKind + +SeedTransform = Callable[[Sequence[Phrase]], Tuple[Phrase, ...]] + +BASELINE_NAME: Final[str] = "baseline" +NO_LOOP_BOUNDARIES: Final[frozenset[int]] = frozenset() +SEED_SPLITTING: Final[str] = "H2'" +SEARCH_BUDGET: Final[str] = "H8" +DRIVER_UNCHANGED: Final[str] = "driver unchanged" +SPLIT_THRESHOLDS: Final[Tuple[int, ...]] = (2, 3, 4, 6, 8) +LIGHT_ENTRIES: Final[int] = 50_000 +WIDE_ENTRIES: Final[int] = 1_000_000 +LONG_ROUNDS: Final[int] = 255 +LIGHT_CONFIRMED: Final[int] = 1 +DEEP_CONFIRMED: Final[int] = 8 + + +def compress( + song: StudySong, + *, + seeds: Sequence[Phrase], + budget: SearchBudget, +) -> CompressedPlanes: + """Compresses a song as the export does, every layer on, over the seeds and budget given. + + Args: + song: The song to compress. + seeds: The phrases offered to the dictionary. + budget: How much work the search spends. + + Returns: + CompressedPlanes: The dictionary and the token streams. + """ + return encode_planes( + song.planes, + seeds, + options=EVERY_LAYER, + boundaries=NO_LOOP_BOUNDARIES, + budget=budget, + ) + + +def compress_baseline(song: StudySong) -> CompressedPlanes: + """Compresses a song as the export does today. + + Args: + song: The song to compress. + + Returns: + CompressedPlanes: The dictionary and the token streams. + """ + return compress(song, seeds=song.seeds, budget=DEFAULT_SEARCH_BUDGET) + + +def encode_production( + song: StudySong, + *, + seeds: Sequence[Phrase], + budget: SearchBudget, +) -> Encoding: + """Encodes a song as the export does, timing the run and playing the result back. + + Args: + song: The song to encode. + seeds: The phrases offered to the dictionary. + budget: How much work the search spends. + + Returns: + Encoding: The encoding, its streams kept as written. + """ + started = process_time() + compressed = compress(song, seeds=seeds, budget=budget) + return production_encoding(song, compressed, process_time() - started) + + +def seed_encoder(transform: SeedTransform) -> Encoder: + """An encoder offering the dictionary the song's seeds after ``transform``. + + Args: + transform: What the seeds become before they are offered. + + Returns: + Encoder: The encoder, searching at the default budget. + """ + + def encode(song: StudySong) -> Encoding: + return encode_production( + song, + seeds=transform(song.seeds), + budget=DEFAULT_SEARCH_BUDGET, + ) + + return encode + + +def budget_encoder(budget: SearchBudget) -> Encoder: + """An encoder searching under ``budget`` over the song's own seeds. + + Args: + budget: How much work the search spends. + + Returns: + Encoder: The encoder. + """ + + def encode(song: StudySong) -> Encoding: + return encode_production(song, seeds=song.seeds, budget=budget) + + return encode + + +def _seed_variant( + name: str, + transform: SeedTransform, +) -> Variant: + return Variant( + name=name, + hypothesis=SEED_SPLITTING, + kind=VariantKind.ENCODER, + note=DRIVER_UNCHANGED, + encode=seed_encoder(transform), + needs_seeds=True, + ) + + +def _budget_variant( + name: str, + budget: SearchBudget, +) -> Variant: + return Variant( + name=name, + hypothesis=SEARCH_BUDGET, + kind=VariantKind.ENCODER, + note=DRIVER_UNCHANGED, + encode=budget_encoder(budget), + needs_seeds=False, + ) + + +def _splitter(threshold: int) -> SeedTransform: + return lambda seeds: split(seeds, threshold) + + +def _both(threshold: int) -> SeedTransform: + return lambda seeds: whole_and_split(seeds, threshold) + + +SEED_VARIANTS: Final[Tuple[Variant, ...]] = ( + _seed_variant("seeds-trimmed", trimmed), + *(_seed_variant(f"seeds-split-{threshold}", _splitter(threshold)) for threshold in SPLIT_THRESHOLDS), + *(_seed_variant(f"seeds-both-{threshold}", _both(threshold)) for threshold in SPLIT_THRESHOLDS), +) + +BUDGET_VARIANTS: Final[Tuple[Variant, ...]] = ( + _budget_variant( + "search-light", + SearchBudget( + candidate_entries=LIGHT_ENTRIES, + rounds=DEFAULT_SEARCH_BUDGET.rounds, + confirmed_candidates=LIGHT_CONFIRMED, + ), + ), + _budget_variant( + "search-long", + SearchBudget( + candidate_entries=DEFAULT_SEARCH_BUDGET.candidate_entries, + rounds=LONG_ROUNDS, + confirmed_candidates=DEFAULT_SEARCH_BUDGET.confirmed_candidates, + ), + ), + _budget_variant( + "search-wide", + SearchBudget( + candidate_entries=WIDE_ENTRIES, + rounds=DEFAULT_SEARCH_BUDGET.rounds, + confirmed_candidates=DEFAULT_SEARCH_BUDGET.confirmed_candidates, + ), + ), + _budget_variant( + "search-deep", + SearchBudget( + candidate_entries=WIDE_ENTRIES, + rounds=LONG_ROUNDS, + confirmed_candidates=DEEP_CONFIRMED, + ), + ), +) diff --git a/src/sampletones_tools/codec/study/variants/registry.py b/src/sampletones_tools/codec/study/variants/registry.py new file mode 100644 index 000000000..5b300cf29 --- /dev/null +++ b/src/sampletones_tools/codec/study/variants/registry.py @@ -0,0 +1,56 @@ +from typing import Dict, Final, Sequence, Tuple + +from sampletones_tools.codec.study.variants.baselines import Baselines, baseline_variant +from sampletones_tools.codec.study.variants.production import BASELINE_NAME, BUDGET_VARIANTS, SEED_VARIANTS +from sampletones_tools.codec.study.variants.sandbox import grammar_variants +from sampletones_tools.codec.study.variants.variant import Variant + +EVERY_VARIANT: Final[str] = "all" + + +def variants(baselines: Baselines) -> Dict[str, Variant]: + """Every variant a run may encode under, by name, the baseline first. + + Args: + baselines: Where the production encodings are kept for the variants built on them. + + Returns: + Dict[str, Variant]: The variants, in the order a run encodes them. + """ + every = ( + baseline_variant(baselines), + *SEED_VARIANTS, + *BUDGET_VARIANTS, + *grammar_variants(baselines), + ) + return {variant.name: variant for variant in every} + + +def selected_variants( + names: Sequence[str], + baselines: Baselines, +) -> Tuple[Variant, ...]: + """The variants a run encodes every song under, the baseline always first. + + Args: + names: The names the run asks for, or ``all`` for every registered variant. + baselines: Where the production encodings are kept for the variants built on them. + + Returns: + Tuple[Variant, ...]: The baseline, then the named variants in the order given. + + Raises: + ValueError: If a name is registered to no variant. + """ + registered = variants(baselines) + if EVERY_VARIANT in names: + return tuple(registered.values()) + + unknown = [name for name in names if name not in registered] + if unknown: + raise ValueError( + f"No variant is called {', '.join(unknown)}; the variants are {', '.join(registered)} or {EVERY_VARIANT}." + ) + + chosen = [registered[name] for name in names if name != BASELINE_NAME] + return (registered[BASELINE_NAME], *chosen) diff --git a/src/sampletones_tools/codec/study/variants/sandbox.py b/src/sampletones_tools/codec/study/variants/sandbox.py new file mode 100644 index 000000000..bb63a6769 --- /dev/null +++ b/src/sampletones_tools/codec/study/variants/sandbox.py @@ -0,0 +1,181 @@ +from dataclasses import replace +from typing import Final, NamedTuple, Tuple + +from sampletones_tools.codec.study.corpus.song import StudySong +from sampletones_tools.codec.study.measure import Encoder, Encoding +from sampletones_tools.codec.study.sandbox.costs import ( + DEFAULT_COUNT_OPERANDS, + DEFAULT_COUNT_SIZE, + PRODUCTION_COSTS, + SET_HOLD_BOUND, + Costs, +) +from sampletones_tools.codec.study.sandbox.encode import encode_grammar +from sampletones_tools.codec.study.sandbox.grammar import BASELINE_GRAMMAR, Grammar +from sampletones_tools.codec.study.variants.baselines import Baselines +from sampletones_tools.codec.study.variants.variant import Variant, VariantKind + +WIDE_HOLD: Final[str] = "H1" +SET_HOLD: Final[str] = "H3" +DEFAULT_COUNT: Final[str] = "H4" +HOLD_EDGES: Final[str] = "H9" +DRIVER_UNCHANGED: Final[str] = "driver unchanged" +WIDE_HOLD_NOTE: Final[str] = ( + "a phrase opcode with a zero count; the block count can sit in the idle shift field and " + "reload in the token fetch, the tick path unchanged" +) +SET_HOLD_NOTE: Final[str] = ( + "a transposed-phrase opcode with a zero count and the value behind it, three bytes; one compare in the token fetch" +) +SET_HOLD_BOUND_NOTE: Final[str] = ( + "a two-byte set-hold needs an opcode of its own, so this bounds what any reallocation of the opcode space reaches" +) +DEFAULT_COUNT_NOTE: Final[str] = ( + "one operand bit says the count is the phrase's own, leaving 31 cheap ids; each table entry " + "grows by a byte, read once when a phrase is set" +) +COMBINED_NOTE: Final[str] = "every change the free escape carries at once, the escape counting 31 blocks and 31 ticks" + +DEFAULT_COUNT_COSTS: Final[Costs] = replace( + PRODUCTION_COSTS, + operands=DEFAULT_COUNT_OPERANDS, + default_entry=DEFAULT_COUNT_SIZE, +) +SET_HOLD_BOUND_COSTS: Final[Costs] = replace( + PRODUCTION_COSTS, + set_hold=SET_HOLD_BOUND, +) + + +class GrammarVariant(NamedTuple): + """One grammar the sandbox prices, and how the report names it. + + Attributes: + name: What the variant is called in a report. + hypothesis: The hypothesis the variant measures, by its label in the plan. + kind: What the variant changes. + note: What the driver would have to do. + grammar: The grammar. + """ + + name: str + hypothesis: str + kind: VariantKind + note: str + grammar: Grammar + + +GRAMMAR_VARIANTS: Final[Tuple[GrammarVariant, ...]] = ( + GrammarVariant( + name="start-hold", + hypothesis=HOLD_EDGES, + kind=VariantKind.ENCODER, + note=f"{DRIVER_UNCHANGED}: every plane is seeded before its first token, so a hold may open a stream", + grammar=replace(BASELINE_GRAMMAR, start_hold=True), + ), + GrammarVariant( + name="every-hold", + hypothesis=HOLD_EDGES, + kind=VariantKind.ENCODER, + note=DRIVER_UNCHANGED, + grammar=replace(BASELINE_GRAMMAR, every_length=True), + ), + GrammarVariant( + name="wide-hold", + hypothesis=WIDE_HOLD, + kind=VariantKind.FORMAT, + note=WIDE_HOLD_NOTE, + grammar=replace(BASELINE_GRAMMAR, wide_hold=True), + ), + GrammarVariant( + name="wide-hold+start-hold", + hypothesis=f"{WIDE_HOLD}+{HOLD_EDGES}", + kind=VariantKind.FORMAT, + note=WIDE_HOLD_NOTE, + grammar=replace(BASELINE_GRAMMAR, wide_hold=True, start_hold=True), + ), + GrammarVariant( + name="set-hold-3", + hypothesis=SET_HOLD, + kind=VariantKind.FORMAT, + note=SET_HOLD_NOTE, + grammar=replace(BASELINE_GRAMMAR, set_hold=True), + ), + GrammarVariant( + name="set-hold-2", + hypothesis=SET_HOLD, + kind=VariantKind.FORMAT, + note=SET_HOLD_BOUND_NOTE, + grammar=replace(BASELINE_GRAMMAR, set_hold=True, costs=SET_HOLD_BOUND_COSTS), + ), + GrammarVariant( + name="default-count", + hypothesis=DEFAULT_COUNT, + kind=VariantKind.FORMAT, + note=DEFAULT_COUNT_NOTE, + grammar=replace(BASELINE_GRAMMAR, default_counts=True, costs=DEFAULT_COUNT_COSTS), + ), + GrammarVariant( + name="wide-hold+default-count", + hypothesis=f"{WIDE_HOLD}+{DEFAULT_COUNT}", + kind=VariantKind.FORMAT, + note=f"{WIDE_HOLD_NOTE}; {DEFAULT_COUNT_NOTE}", + grammar=replace(BASELINE_GRAMMAR, wide_hold=True, default_counts=True, costs=DEFAULT_COUNT_COSTS), + ), + GrammarVariant( + name="escape-grammar", + hypothesis=f"{WIDE_HOLD}+{SET_HOLD}+{DEFAULT_COUNT}+{HOLD_EDGES}", + kind=VariantKind.FORMAT, + note=COMBINED_NOTE, + grammar=replace( + BASELINE_GRAMMAR, + start_hold=True, + wide_hold=True, + set_hold=True, + default_counts=True, + costs=DEFAULT_COUNT_COSTS, + ), + ), +) + + +def grammar_encoder( + baselines: Baselines, + grammar: Grammar, +) -> Encoder: + """An encoder pricing a song under ``grammar`` over its production dictionary. + + Args: + baselines: Where the production encodings are kept. + grammar: The grammar. + + Returns: + Encoder: The encoder. + """ + + def encode(song: StudySong) -> Encoding: + return encode_grammar(baselines.reference(song), grammar) + + return encode + + +def grammar_variants(baselines: Baselines) -> Tuple[Variant, ...]: + """Every grammar the sandbox prices, as the variants a run encodes under. + + Args: + baselines: Where the production encodings are kept. + + Returns: + Tuple[Variant, ...]: The variants, in the order of ``GRAMMAR_VARIANTS``. + """ + return tuple( + Variant( + name=entry.name, + hypothesis=entry.hypothesis, + kind=entry.kind, + note=entry.note, + encode=grammar_encoder(baselines, entry.grammar), + needs_seeds=False, + ) + for entry in GRAMMAR_VARIANTS + ) diff --git a/src/sampletones_tools/codec/study/variants/seeds.py b/src/sampletones_tools/codec/study/variants/seeds.py new file mode 100644 index 000000000..d0eed4053 --- /dev/null +++ b/src/sampletones_tools/codec/study/variants/seeds.py @@ -0,0 +1,92 @@ +from typing import Final, List, Sequence, Tuple + +from sampletones_player.compression.dictionary.phrase import Phrase + +MIN_PIECE_LENGTH: Final[int] = 2 + + +def trimmed(seeds: Sequence[Phrase]) -> Tuple[Phrase, ...]: + """Every seed with its trailing plateau cut to one value. + + A token playing a phrase past its end holds the final value onward, so the ticks a body + spends resting on its last value are ticks the token covers for free. + + Args: + seeds: The phrases the song's instruments offer. + + Returns: + Tuple[Phrase, ...]: The same phrases, each ending where its last value first appears. + """ + return tuple(Phrase(body=_trimmed_body(seed.body)) for seed in seeds) + + +def split( + seeds: Sequence[Phrase], + threshold: int, +) -> Tuple[Phrase, ...]: + """Every seed cut at the plateaus of at least ``threshold`` ticks it holds. + + A plateau inside a body is stored one byte per tick, where a token's count would cover the + rest for free. Cutting the body where a plateau begins keeps the plateau's first value at + the end of one piece, so the token playing that piece holds it, and the next piece starts + where the body moves again. + + Args: + seeds: The phrases the song's instruments offer. + threshold: The ticks a plateau rests for before the body is cut there. + + Returns: + Tuple[Phrase, ...]: The pieces, each of at least two values, in the order they are cut. + """ + return tuple(Phrase(body=piece) for seed in seeds for piece in _pieces(seed.body, threshold)) + + +def whole_and_split( + seeds: Sequence[Phrase], + threshold: int, +) -> Tuple[Phrase, ...]: + """Every seed as it stands, followed by its pieces, for the encoder to weigh against each other. + + Args: + seeds: The phrases the song's instruments offer. + threshold: The ticks a plateau rests for before the body is cut there. + + Returns: + Tuple[Phrase, ...]: The whole phrases, then the pieces. + """ + return (*seeds, *split(seeds, threshold)) + + +def _trimmed_body(body: bytes) -> bytes: + end = len(body) + while end > 1 and body[end - 1] == body[end - 2]: + end -= 1 + + return body[:end] + + +def _pieces( + body: bytes, + threshold: int, +) -> List[bytes]: + pieces: List[bytes] = [] + current = bytearray() + start = 0 + while start < len(body): + end = start + while end < len(body) and body[end] == body[start]: + end += 1 + + if end - start >= threshold: + current.append(body[start]) + pieces.append(bytes(current)) + current = bytearray() + else: + current.extend(body[start:end]) + + start = end + + if current: + pieces.append(bytes(current)) + + return [piece for piece in pieces if len(piece) >= MIN_PIECE_LENGTH] diff --git a/src/sampletones_tools/codec/study/variants/strategy.py b/src/sampletones_tools/codec/study/variants/strategy.py new file mode 100644 index 000000000..211783f66 --- /dev/null +++ b/src/sampletones_tools/codec/study/variants/strategy.py @@ -0,0 +1,71 @@ +from dataclasses import replace +from typing import Dict, Final, List, Optional, Sequence, Tuple + +from sampletones_tools.codec.study.measure import Measurement + +STRATEGY_ORDER: Final[Tuple[str, ...]] = ( + "baseline", + "seeds-trimmed", + "seeds-both-4", + "seeds-split-4", + "search-long", + "search-wide", + "search-deep", +) +DEPTH_PREFIX: Final[str] = "depth-" + + +def depth_measurements( + measurements: Sequence[Measurement], + order: Sequence[str], +) -> Tuple[Measurement, ...]: + """What an export trying the first ``d`` strategies and keeping the smallest would write. + + An export option choosing how deep to search is modeled as a list of encoder-side + strategies tried in order: at each depth a song is written by whichever of the strategies so + far made it smallest, and the time is what all of them took together. A strategy a song + offers nothing to keeps the song where the previous depth left it. + + Args: + measurements: Every song under every variant the run measured. + order: The strategies, in the order an export would try them. + + Returns: + Tuple[Measurement, ...]: One measurement per song and depth, named ``depth-d``, in song + order then depth order. + """ + present = [name for name in order if any(measurement.variant == name for measurement in measurements)] + derived: List[Measurement] = [] + for name in dict.fromkeys(measurement.song.name for measurement in measurements): + by_variant: Dict[str, Measurement] = { + measurement.variant: measurement for measurement in measurements if measurement.song.name == name + } + derived.extend(_song_depths(by_variant, present)) + + return tuple(derived) + + +def _song_depths( + by_variant: Dict[str, Measurement], + present: Sequence[str], +) -> List[Measurement]: + depths: List[Measurement] = [] + best: Optional[Measurement] = None + seconds = 0.0 + for depth, name in enumerate(present, start=1): + measured = by_variant.get(name) + if measured is not None: + seconds += measured.seconds + if best is None or measured.block < best.block: + best = measured + + if best is not None: + depths.append( + Measurement( + song=best.song, + variant=f"{DEPTH_PREFIX}{depth}", + encoding=replace(best.encoding, seconds=seconds), + ) + ) + + return depths diff --git a/src/sampletones_tools/codec/study/variants/variant.py b/src/sampletones_tools/codec/study/variants/variant.py new file mode 100644 index 000000000..47ea3754e --- /dev/null +++ b/src/sampletones_tools/codec/study/variants/variant.py @@ -0,0 +1,51 @@ +from dataclasses import dataclass +from enum import StrEnum + +from sampletones_tools.codec.study.corpus.song import StudySong +from sampletones_tools.codec.study.measure import Encoder + + +class VariantKind(StrEnum): + """What a variant changes to earn its bytes. + + Attributes: + BASELINE: The production codec as it stands. + ENCODER: A change to how the encoder chooses, readable by the driver as it stands. + FORMAT: A change to the token grammar, which the driver has to learn. + """ + + BASELINE = "baseline" + ENCODER = "encoder" + FORMAT = "format" + + +@dataclass(frozen=True) +class Variant: + """One way of encoding a song the study prices against the baseline. + + Attributes: + name: What the variant is called in a report. + hypothesis: The hypothesis the variant measures, by its label in the plan. + kind: What the variant changes. + note: What the driver would have to do, where the variant changes the format. + encode: What the variant writes a song as. + needs_seeds: Whether the variant changes anything only where a song offers seeds. + """ + + name: str + hypothesis: str + kind: VariantKind + note: str + encode: Encoder + needs_seeds: bool + + def applies(self, song: StudySong) -> bool: + """Whether encoding ``song`` under this variant can differ from the baseline. + + Args: + song: The song. + + Returns: + bool: Whether the variant is worth measuring on it. + """ + return bool(song.seeds) or not self.needs_seeds diff --git a/tests/unit/sampletones_synthesis/voice/__init__.py b/src/sampletones_tools/corpus/__init__.py similarity index 100% rename from tests/unit/sampletones_synthesis/voice/__init__.py rename to src/sampletones_tools/corpus/__init__.py diff --git a/src/sampletones_tools/corpus/build.py b/src/sampletones_tools/corpus/build.py new file mode 100644 index 000000000..1eb460bf4 --- /dev/null +++ b/src/sampletones_tools/corpus/build.py @@ -0,0 +1,66 @@ +from dataclasses import dataclass +from pathlib import Path +from tempfile import TemporaryDirectory +from typing import Dict + +from sampletones_core.project.project import Project +from sampletones_core.project.settings import ProjectSettings +from sampletones_core.project.voices.sample import Sample +from sampletones_tools.corpus.catalog import CatalogSpec, build_catalog +from sampletones_tools.corpus.module import ModuleConfig +from sampletones_tools.corpus.song import SongSpec, build_song +from sampletones_tools.corpus.synth import SynthConfig + + +@dataclass(frozen=True) +class Corpus: + """The synthetic corpus: the reconstructed samples and the arrangement that plays them. + + Attributes: + catalog: The samples, by name. + project: The arrangement, carrying the samples as its voices. + """ + + catalog: Dict[str, Sample] + project: Project + + +def build_project( + catalog: Dict[str, Sample], + module_config: ModuleConfig, + song_spec: SongSpec, +) -> Project: + """The arrangement playing the catalog under the module's identity and playback settings.""" + settings = ProjectSettings( + tempo=module_config.tempo, + speed=module_config.speed, + nes_frequency=module_config.nes_frequency, + ) + project = Project.create( + title=module_config.title, + author=module_config.author, + settings=settings, + ) + for sample in catalog.values(): + project.voices.append(sample) + + project.song = build_song(song_spec, catalog) + return project + + +def build_synthetic_corpus() -> Corpus: + """Renders, reconstructs and arranges the corpus the package describes. + + The recordings are rendered into a temporary directory of the build's own, which is gone once + their reconstructions are made. + + Returns: + Corpus: The samples and the arrangement. + """ + with TemporaryDirectory() as recordings: + catalog = build_catalog(CatalogSpec.load(), SynthConfig.load(), tmp_dir=Path(recordings)) + + return Corpus( + catalog=catalog, + project=build_project(catalog, ModuleConfig.load(), SongSpec.load()), + ) diff --git a/src/sampletones_tools/corpus/catalog.py b/src/sampletones_tools/corpus/catalog.py new file mode 100644 index 000000000..a8bf761a9 --- /dev/null +++ b/src/sampletones_tools/corpus/catalog.py @@ -0,0 +1,218 @@ +from pathlib import Path +from typing import Dict, Final, FrozenSet, List, Self, Sequence, Tuple + +import numpy as np +from pydantic import BaseModel, ConfigDict + +from sampletones_core.audio import write_wave +from sampletones_core.audio.processing import normalize +from sampletones_core.configs import Config, InstructionsLibraryConfig +from sampletones_core.constants.enums import ChannelName, GeneratorClassName, SpectrumMethod +from sampletones_core.fft import Window +from sampletones_core.fft.features import get_feature_extractor +from sampletones_core.generators import get_generators_by_channels +from sampletones_core.instructions import InstructionUnion +from sampletones_core.library import InstructionLibrary, InstructionLibraryData +from sampletones_core.library.creator.creation import generate_instruction +from sampletones_core.project.voices.sample import Sample +from sampletones_core.reconstructions import Reconstruction, Reconstructor +from sampletones_shared.types.path import Pathlike +from sampletones_shared.utils.serialization import load_yaml_model +from sampletones_tools.corpus.paths import CATALOG_CONFIG_PATH +from sampletones_tools.corpus.synth import SynthConfig + +INSTRUCTIONS_PER_GENERATOR: Final[int] = 48 +CHANNELS: Final[List[ChannelName]] = [ + ChannelName.PULSE1, + ChannelName.TRIANGLE, + ChannelName.NOISE, +] + + +class ReconstructionSettings(BaseModel): + """The settings the one in-memory library every instrument is matched against is built with.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + spectrum_method: SpectrumMethod + transformation_gamma: int + instructions_per_generator: int + + +class InstrumentSpec(BaseModel): + """One instrument of the catalog: the voice it is rendered from and the channels it covers.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + name: str + synth: str + channels: List[ChannelName] + + +class CatalogSpec(BaseModel): + """The reconstructed sample catalog: the library settings and the instruments built under them.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + reconstruction: ReconstructionSettings + instruments: List[InstrumentSpec] + + @classmethod + def load(cls) -> Self: + """The catalog the package ships.""" + return load_yaml_model(CATALOG_CONFIG_PATH, cls) + + +def build_mini_library( + config: Config, + *, + per_generator: int = INSTRUCTIONS_PER_GENERATOR, +) -> InstructionLibrary: + """Builds a small in-memory instruction library covering pulse/triangle/noise. + + Candidates are sampled with an even stride across each channel's instruction + space so pitch, volume and period are represented, rather than a biased prefix. + """ + window = Window.from_config(config) + extractor = get_feature_extractor(config, window) + generators = { + generator.class_name(): generator for generator in get_generators_by_channels(config, CHANNELS).values() + } + sampled: List[Tuple[GeneratorClassName, InstructionUnion]] = [ + (class_name, instruction) + for class_name, generator in generators.items() + for instruction in _evenly_sampled(generator.get_possible_instructions(), per_generator) + ] + data = dict( + generate_instruction(generators, class_name, instruction, extractor) for class_name, instruction in sampled + ) + + library = InstructionLibrary() + library.data[library.create_key(config, window)] = InstructionLibraryData.create(config, data) + return library + + +def _evenly_sampled(candidates: Sequence[InstructionUnion], count: int) -> List[InstructionUnion]: + stride = max(1, len(candidates) // count) + return list(candidates[::stride][:count]) + + +def reconstruct_sample( + audio: np.ndarray, + config: Config, + library: InstructionLibrary, + channels: FrozenSet[ChannelName], + *, + tmp_dir: Pathlike, + name: str, +) -> Reconstruction: + """Runs the real reconstruction pipeline on ``audio`` via a temp WAV. + + Raises: + ValueError: If the reconstructor returns no reconstruction for the recording. + """ + path = Path(tmp_dir) / f"{name}.wav" + write_wave(path, config.library.sample_rate, audio) + reconstruction = Reconstructor(config, channels, library=library)(path) + if reconstruction is None: + raise ValueError(f"Reconstruction of '{name}' produced no result") + + return reconstruction + + +def make_sample( + name: str, + audio: np.ndarray, + config: Config, + library: InstructionLibrary, + *, + tmp_dir: Pathlike, + expected_slices: FrozenSet[ChannelName], +) -> Sample: + """Reconstructs ``audio`` into a `Sample` playing exactly the expected channels. + + Raises: + ValueError: If the reconstruction plays other channels than ``expected_slices``. + """ + reconstruction = reconstruct_sample( + audio, + config, + library, + expected_slices, + tmp_dir=tmp_dir, + name=name, + ) + played = frozenset(reconstruction.playing_channels) + if played != expected_slices: + raise ValueError(f"Sample '{name}' covers {set(played)}, expected {set(expected_slices)}") + + return Sample(name=name, reconstruction=reconstruction) + + +def build_catalog( + spec: CatalogSpec, + synth_config: SynthConfig, + *, + tmp_dir: Pathlike, +) -> Dict[str, Sample]: + """Builds the reconstructed sample catalog the spec describes. + + The spec fixes the reconstruction settings (spectrum method, gamma, library size) shared by + one in-memory library, and per instrument the voice to render and the channel slices to + cover. Voice parameters come from ``synth_config``. + + Args: + spec: The catalog: the library settings and the instruments. + synth_config: The voices the instruments are rendered from. + tmp_dir: Where the rendered recordings are written before they are reconstructed. + + Returns: + Dict[str, Sample]: The samples, by name, in the order the spec lists them. + """ + settings = spec.reconstruction + library_config = InstructionsLibraryConfig( + spectrum_method=settings.spectrum_method, + transformation_gamma=settings.transformation_gamma, + ) + sample_rate = library_config.sample_rate + library = build_mini_library( + Config(library=library_config), + per_generator=settings.instructions_per_generator, + ) + + catalog: Dict[str, Sample] = {} + for instrument in spec.instruments: + config = Config(library=library_config) + audio = _render_instrument(synth_config, instrument.synth, sample_rate=sample_rate) + catalog[instrument.name] = make_sample( + instrument.name, + audio, + config, + library, + tmp_dir=tmp_dir, + expected_slices=frozenset(instrument.channels), + ) + + return catalog + + +def _render_instrument( + synth_config: SynthConfig, + name: str, + *, + sample_rate: int, +) -> np.ndarray: + """Renders a named voice at peak level 1.0. + + A fresh channel seeded from the synth configuration keeps every instrument reproducible + independently of catalog order. + """ + voice = synth_config.voices[name] + generator = np.random.default_rng(synth_config.seed) + instrument: np.ndarray = normalize( + voice.render( + sample_rate=sample_rate, + generator=generator, + ) + ) + return instrument diff --git a/src/sampletones_tools/corpus/config/__init__.py b/src/sampletones_tools/corpus/config/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/integration/config/module.yaml b/src/sampletones_tools/corpus/config/module.yaml similarity index 100% rename from tests/integration/config/module.yaml rename to src/sampletones_tools/corpus/config/module.yaml diff --git a/tests/integration/config/reconstruction.yaml b/src/sampletones_tools/corpus/config/reconstruction.yaml similarity index 100% rename from tests/integration/config/reconstruction.yaml rename to src/sampletones_tools/corpus/config/reconstruction.yaml diff --git a/tests/integration/config/song.yaml b/src/sampletones_tools/corpus/config/song.yaml similarity index 100% rename from tests/integration/config/song.yaml rename to src/sampletones_tools/corpus/config/song.yaml diff --git a/tests/integration/config/synth.yaml b/src/sampletones_tools/corpus/config/synth.yaml similarity index 100% rename from tests/integration/config/synth.yaml rename to src/sampletones_tools/corpus/config/synth.yaml diff --git a/src/sampletones_tools/corpus/module.py b/src/sampletones_tools/corpus/module.py new file mode 100644 index 000000000..c8796d2fc --- /dev/null +++ b/src/sampletones_tools/corpus/module.py @@ -0,0 +1,23 @@ +from typing import Self + +from pydantic import BaseModel, ConfigDict + +from sampletones_shared.utils.serialization import load_yaml_model +from sampletones_tools.corpus.paths import MODULE_CONFIG_PATH + + +class ModuleConfig(BaseModel): + """Module identity and playback settings that shape the exported files.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + title: str + author: str + tempo: int + speed: int + nes_frequency: int + + @classmethod + def load(cls) -> Self: + """The module settings the package ships.""" + return load_yaml_model(MODULE_CONFIG_PATH, cls) diff --git a/src/sampletones_tools/corpus/paths.py b/src/sampletones_tools/corpus/paths.py new file mode 100644 index 000000000..92c1cf78f --- /dev/null +++ b/src/sampletones_tools/corpus/paths.py @@ -0,0 +1,10 @@ +from pathlib import Path +from typing import Final + +from sampletones_shared.paths.package import package_directory + +CONFIG_DIRECTORY: Final[Path] = package_directory("sampletones_tools.corpus.config") +SYNTH_CONFIG_PATH: Final[Path] = CONFIG_DIRECTORY / "synth.yaml" +CATALOG_CONFIG_PATH: Final[Path] = CONFIG_DIRECTORY / "reconstruction.yaml" +MODULE_CONFIG_PATH: Final[Path] = CONFIG_DIRECTORY / "module.yaml" +SONG_PATH: Final[Path] = CONFIG_DIRECTORY / "song.yaml" diff --git a/src/sampletones_tools/corpus/song.py b/src/sampletones_tools/corpus/song.py new file mode 100644 index 000000000..5ae846682 --- /dev/null +++ b/src/sampletones_tools/corpus/song.py @@ -0,0 +1,132 @@ +from typing import Dict, List, Mapping, Optional, Self, Sequence + +from pydantic import BaseModel, ConfigDict + +from sampletones_core.constants.enums import ChannelName +from sampletones_core.project.patterns.channel import Channel +from sampletones_core.project.patterns.pattern import Pattern +from sampletones_core.project.patterns.row import Row +from sampletones_core.project.song import Song +from sampletones_core.project.voices.note_off import NoteOff +from sampletones_core.project.voices.note_on import NoteOn +from sampletones_core.project.voices.sample import Sample +from sampletones_shared.utils.serialization import load_yaml_model +from sampletones_tools.corpus.paths import SONG_PATH + + +class RowSpec(BaseModel): + """One written row of a pattern: what it plays, or that it releases the note. + + Attributes: + row: The row's index in its pattern. + sample: The sample the row plays, or ``None`` for a row that plays none. + transpose: The semitones the row transposes the sample by, or ``None`` to leave it. + volume: The volume the row sets, or ``None`` to leave it. + off: Whether the row releases the note. + """ + + model_config = ConfigDict(frozen=True, extra="forbid") + + row: int + sample: Optional[str] = None + transpose: Optional[int] = None + volume: Optional[int] = None + off: bool = False + + +class ChannelSpec(BaseModel): + """The patterns one channel holds, each as the rows written into it.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + patterns: Dict[int, List[RowSpec]] + + +class SongSpec(BaseModel): + """The arrangement: the pattern length, the order and the channels' patterns.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + rows_per_pattern: int + order: List[Dict[ChannelName, int]] + channels: Dict[ChannelName, ChannelSpec] + + @classmethod + def load(cls) -> Self: + """The arrangement the package ships.""" + return load_yaml_model(SONG_PATH, cls) + + +def _order( + frames: Sequence[Mapping[ChannelName, int]], +) -> List[Dict[ChannelName, Optional[int]]]: + return [{channel: frame.get(channel) for channel in ChannelName.items()} for frame in frames] + + +def _row(spec: RowSpec, channel: ChannelName, samples_by_name: Mapping[str, Sample]) -> Row: + if spec.off: + return Row(command=NoteOff(), volume=spec.volume) + + if spec.sample is None: + return Row(transpose=spec.transpose, volume=spec.volume) + + sample = samples_by_name[spec.sample] + if channel not in sample.reconstruction.playing_channels: + raise ValueError(f"Sample '{spec.sample}' has no '{channel.value}' slice for the {channel.value} channel") + + return Row( + command=NoteOn(voice_id=sample.id), + transpose=spec.transpose, + volume=spec.volume, + ) + + +def _pattern( + row_specs: Sequence[RowSpec], + rows_per_pattern: int, + channel: ChannelName, + samples_by_name: Mapping[str, Sample], +) -> Pattern: + rows = [Row() for _ in range(rows_per_pattern)] + for spec in row_specs: + rows[spec.row] = _row(spec, channel, samples_by_name) + + return Pattern(rows=rows) + + +def _channels( + channel_specs: Mapping[ChannelName, ChannelSpec], + rows_per_pattern: int, + samples_by_name: Mapping[str, Sample], +) -> Dict[ChannelName, Channel]: + channels: Dict[ChannelName, Channel] = {} + for channel, spec in channel_specs.items(): + patterns = { + index: _pattern( + row_specs, + rows_per_pattern, + channel, + samples_by_name, + ) + for index, row_specs in spec.patterns.items() + } + channels[channel] = Channel(name=channel, patterns=patterns) + + for channel in ChannelName.items(): + channels.setdefault(channel, Channel(name=channel, patterns={})) + + return channels + + +def build_song(spec: SongSpec, samples_by_name: Mapping[str, Sample]) -> Song: + """The song the spec describes, playing the named samples. + + Raises: + KeyError: If a row names a sample the catalog lacks. + ValueError: If a row plays a sample on a channel the sample has no slice for. + """ + return Song( + rows_per_pattern=spec.rows_per_pattern, + order=_order(spec.order), + channels=_channels(spec.channels, spec.rows_per_pattern, samples_by_name), + ) diff --git a/src/sampletones_tools/corpus/synth.py b/src/sampletones_tools/corpus/synth.py new file mode 100644 index 000000000..f7fab5a43 --- /dev/null +++ b/src/sampletones_tools/corpus/synth.py @@ -0,0 +1,21 @@ +from typing import Dict, Self + +from pydantic import BaseModel, ConfigDict + +from sampletones_shared.utils.serialization import load_yaml_model +from sampletones_tools.corpus.paths import SYNTH_CONFIG_PATH +from sampletones_tools.synthesis.voice.voice import Voice + + +class SynthConfig(BaseModel): + """The named synthesizer voices the corpus is rendered from and their shared noise seed.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + seed: int + voices: Dict[str, Voice] + + @classmethod + def load(cls) -> Self: + """The voices the package ships.""" + return load_yaml_model(SYNTH_CONFIG_PATH, cls) diff --git a/src/sampletones_tools/player/__init__.py b/src/sampletones_tools/player/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/sampletones_tools/player/assembler/__init__.py b/src/sampletones_tools/player/assembler/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/sampletones_player/driver/assembler/builder.py b/src/sampletones_tools/player/assembler/builder.py similarity index 83% rename from src/sampletones_player/driver/assembler/builder.py rename to src/sampletones_tools/player/assembler/builder.py index aae212b2c..b171534b0 100644 --- a/src/sampletones_player/driver/assembler/builder.py +++ b/src/sampletones_tools/player/assembler/builder.py @@ -3,17 +3,18 @@ from typing import Final, List from sampletones_player.driver.addresses import DriverAddresses -from sampletones_player.driver.assembler.labels import read_addresses -from sampletones_player.driver.assembler.layout import ( +from sampletones_player.driver.image import DriverImage +from sampletones_player.specification.driver import DRIVER_CODE_NAME +from sampletones_shared.exceptions import DriverBuildError +from sampletones_tools.player.assembler.labels import read_addresses +from sampletones_tools.player.assembler.layout import ( + ASSEMBLY_DIRECTORY, INCLUDE_DIRECTORY, LINKER_CONFIGURATION, SOURCE_DIRECTORY, SOURCE_NAMES, ) -from sampletones_player.driver.assembler.toolchain import Toolchain -from sampletones_player.driver.image import DriverImage -from sampletones_player.specification.driver import DRIVER_CODE_NAME -from sampletones_shared.exceptions import DriverBuildError +from sampletones_tools.player.assembler.toolchain import Toolchain LABELS_NAME: Final[str] = "driver.labels" OBJECT_SUFFIX: Final[str] = ".o" @@ -41,10 +42,10 @@ def build_driver(destination: Path) -> DriverImage: toolchain = Toolchain.locate() with TemporaryDirectory() as directory: work_directory = Path(directory) - objects = assemble_sources(toolchain, work_directory) + objects = assemble_sources(toolchain, ASSEMBLY_DIRECTORY, work_directory) assembled = work_directory / DRIVER_CODE_NAME labels = work_directory / LABELS_NAME - toolchain.link(LINKER_CONFIGURATION, objects, assembled, labels) + toolchain.link(ASSEMBLY_DIRECTORY / LINKER_CONFIGURATION, objects, assembled, labels) image = DriverImage( code=assembled.read_bytes(), addresses=read_addresses(labels), @@ -56,11 +57,13 @@ def build_driver(destination: Path) -> DriverImage: return image -def assemble_sources(toolchain: Toolchain, work_directory: Path) -> List[Path]: +def assemble_sources(toolchain: Toolchain, sources: Path, work_directory: Path) -> List[Path]: """Assembles every source the driver is built from. Args: toolchain: The cc65 programs the build runs. + sources: The assembly package as a directory: the sources, their includes and the + linker configuration. work_directory: The directory receiving the object files. Returns: @@ -72,7 +75,7 @@ def assemble_sources(toolchain: Toolchain, work_directory: Path) -> List[Path]: objects: List[Path] = [] for name in SOURCE_NAMES: object_file = (work_directory / name).with_suffix(OBJECT_SUFFIX) - toolchain.assemble(SOURCE_DIRECTORY / name, INCLUDE_DIRECTORY, object_file) + toolchain.assemble(sources / SOURCE_DIRECTORY / name, sources / INCLUDE_DIRECTORY, object_file) objects.append(object_file) return objects diff --git a/src/sampletones_player/driver/assembler/labels.py b/src/sampletones_tools/player/assembler/labels.py similarity index 100% rename from src/sampletones_player/driver/assembler/labels.py rename to src/sampletones_tools/player/assembler/labels.py diff --git a/src/sampletones_tools/player/assembler/layout.py b/src/sampletones_tools/player/assembler/layout.py new file mode 100644 index 000000000..73f560912 --- /dev/null +++ b/src/sampletones_tools/player/assembler/layout.py @@ -0,0 +1,14 @@ +from pathlib import Path +from typing import Final, Tuple + +from sampletones_player.specification.driver import DRIVER_BINARY_DIRECTORY +from sampletones_shared.paths.package import package_directory +from sampletones_shared.paths.source import SOURCE_ROOT + +ASSEMBLY_PACKAGE: Final[str] = "sampletones_tools.player.assembly" +INCLUDE_DIRECTORY: Final[str] = "include" +SOURCE_DIRECTORY: Final[str] = "source" +LINKER_CONFIGURATION: Final[str] = "nsf.cfg" +SOURCE_NAMES: Final[Tuple[str, ...]] = ("driver.s", "clock.s", "channels.s") +BINARY_DIRECTORY: Final[Path] = SOURCE_ROOT / "sampletones_player" / "driver" / DRIVER_BINARY_DIRECTORY +ASSEMBLY_DIRECTORY: Final[Path] = package_directory(ASSEMBLY_PACKAGE) diff --git a/src/sampletones_tools/player/assembler/report.py b/src/sampletones_tools/player/assembler/report.py new file mode 100644 index 000000000..f7afd1155 --- /dev/null +++ b/src/sampletones_tools/player/assembler/report.py @@ -0,0 +1,15 @@ +from typing import List + +from sampletones_player.driver.image import DriverImage +from sampletones_player.specification.driver import DRIVER_CODE_NAME + + +def layout_lines(image: DriverImage) -> List[str]: + """The layout a build produced, one line per figure a reader checks against the header.""" + addresses = image.addresses + return [ + f"{DRIVER_CODE_NAME} {len(image.code)} bytes, ${addresses.load:04X}-${addresses.song - 1:04X}", + f"init ${addresses.init:04X}", + f"play ${addresses.play:04X}", + f"song ${addresses.song:04X}", + ] diff --git a/src/sampletones_player/driver/assembler/toolchain.py b/src/sampletones_tools/player/assembler/toolchain.py similarity index 100% rename from src/sampletones_player/driver/assembler/toolchain.py rename to src/sampletones_tools/player/assembler/toolchain.py diff --git a/src/sampletones_tools/player/assembly/__init__.py b/src/sampletones_tools/player/assembly/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/sampletones_player/driver/assembly/include/nes.inc b/src/sampletones_tools/player/assembly/include/nes.inc similarity index 100% rename from src/sampletones_player/driver/assembly/include/nes.inc rename to src/sampletones_tools/player/assembly/include/nes.inc diff --git a/src/sampletones_player/driver/assembly/include/song.inc b/src/sampletones_tools/player/assembly/include/song.inc similarity index 100% rename from src/sampletones_player/driver/assembly/include/song.inc rename to src/sampletones_tools/player/assembly/include/song.inc diff --git a/src/sampletones_player/driver/assembly/nsf.cfg b/src/sampletones_tools/player/assembly/nsf.cfg similarity index 100% rename from src/sampletones_player/driver/assembly/nsf.cfg rename to src/sampletones_tools/player/assembly/nsf.cfg diff --git a/src/sampletones_player/driver/assembly/source/channels.s b/src/sampletones_tools/player/assembly/source/channels.s similarity index 100% rename from src/sampletones_player/driver/assembly/source/channels.s rename to src/sampletones_tools/player/assembly/source/channels.s diff --git a/src/sampletones_player/driver/assembly/source/clock.s b/src/sampletones_tools/player/assembly/source/clock.s similarity index 100% rename from src/sampletones_player/driver/assembly/source/clock.s rename to src/sampletones_tools/player/assembly/source/clock.s diff --git a/src/sampletones_player/driver/assembly/source/driver.s b/src/sampletones_tools/player/assembly/source/driver.s similarity index 100% rename from src/sampletones_player/driver/assembly/source/driver.s rename to src/sampletones_tools/player/assembly/source/driver.s diff --git a/src/sampletones_tools/player/command.py b/src/sampletones_tools/player/command.py new file mode 100644 index 000000000..620d4ccb4 --- /dev/null +++ b/src/sampletones_tools/player/command.py @@ -0,0 +1,60 @@ +from argparse import ArgumentParser, Namespace +from dataclasses import dataclass +from pathlib import Path +from typing import Final, Optional + +from sampletones_shared.command import Command + +NAME: Final[str] = "driver" +HELP: Final[str] = "assemble the NES player driver with cc65" +OUTPUT_HELP: Final[str] = "the directory receiving the assembled driver; without it, the driver the package ships" + + +@dataclass(frozen=True) +class DriverArguments: + """What a driver build is given: where the image goes, if anywhere but the package.""" + + output: Optional[Path] + + +def configure(parser: ArgumentParser) -> None: + parser.add_argument("--output", "-o", type=Path, default=None, help=OUTPUT_HELP) + + +def run(arguments: Namespace) -> int: + """Assembles the driver and prints the layout the build produced. + + Writing the driver the package ships needs a checkout, since that is where the package is. + + Raises: + SystemExit: If the build runs outside a checkout without an output of its own, or fails. + """ + given = DriverArguments(output=arguments.output) + + from sampletones_tools.checkout import require_checkout + + if given.output is None: + require_checkout(NAME) + + from sampletones_shared.exceptions.player import DriverBuildError + from sampletones_tools.player.assembler.builder import build_driver + from sampletones_tools.player.assembler.layout import BINARY_DIRECTORY + from sampletones_tools.player.assembler.report import layout_lines + + try: + image = build_driver(given.output if given.output is not None else BINARY_DIRECTORY) + except DriverBuildError as error: + raise SystemExit(str(error)) from error + + for line in layout_lines(image): + print(line) + + return 0 + + +DRIVER: Final[Command] = Command( + name=NAME, + help=HELP, + configure=configure, + run=run, +) diff --git a/src/sampletones_tools/player/trace/__init__.py b/src/sampletones_tools/player/trace/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/sampletones_player/trace/trace.py b/src/sampletones_tools/player/trace/trace.py similarity index 98% rename from src/sampletones_player/trace/trace.py rename to src/sampletones_tools/player/trace/trace.py index cd9bd185b..e9cccfdc3 100644 --- a/src/sampletones_player/trace/trace.py +++ b/src/sampletones_tools/player/trace/trace.py @@ -21,7 +21,7 @@ SILENCED_REGISTER, SWEEP_DISABLED, ) -from sampletones_player.trace.write import RegisterWrite +from sampletones_tools.player.trace.write import RegisterWrite FIRST_TICK: Final[int] = 0 diff --git a/src/sampletones_player/trace/write.py b/src/sampletones_tools/player/trace/write.py similarity index 100% rename from src/sampletones_player/trace/write.py rename to src/sampletones_tools/player/trace/write.py diff --git a/src/sampletones_tools/py.typed b/src/sampletones_tools/py.typed new file mode 100644 index 000000000..e69de29bb diff --git a/src/sampletones_tools/registry.py b/src/sampletones_tools/registry.py new file mode 100644 index 000000000..27995f58a --- /dev/null +++ b/src/sampletones_tools/registry.py @@ -0,0 +1,22 @@ +from typing import Final, Tuple + +from sampletones_shared.command import Command +from sampletones_tools.assets.command import ICONS +from sampletones_tools.calibration.command import CALIBRATION +from sampletones_tools.checks.command import CHECK +from sampletones_tools.codec.command import CODEC +from sampletones_tools.player.command import DRIVER +from sampletones_tools.samples.commands.btp import BTP +from sampletones_tools.samples.commands.ftm import FTM +from sampletones_tools.samples.commands.nsf import NSF + +DEVELOPER_COMMANDS: Final[Tuple[Command, ...]] = ( + BTP, + CALIBRATION, + CHECK, + CODEC, + DRIVER, + FTM, + ICONS, + NSF, +) diff --git a/src/sampletones_tools/runs.py b/src/sampletones_tools/runs.py new file mode 100644 index 000000000..25e2a5b3e --- /dev/null +++ b/src/sampletones_tools/runs.py @@ -0,0 +1,19 @@ +from datetime import UTC, datetime +from pathlib import Path +from typing import Final + +RUN_STAMP: Final[str] = "run-%Y%m%d-%H%M%S" + + +def stamped_run_directory(root: Path) -> Path: + """The directory a measurement run lands in under ``root``, named by the moment it starts. + + The stamp is UTC and sorts by name, so the runs under one root read in the order they ran. + + Args: + root: The directory the runs of one measurement share. + + Returns: + Path: The run's directory, which the run creates. + """ + return root / datetime.now(UTC).strftime(RUN_STAMP) diff --git a/src/sampletones_tools/samples/__init__.py b/src/sampletones_tools/samples/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/sampletones_tools/samples/bitphase.py b/src/sampletones_tools/samples/bitphase.py new file mode 100644 index 000000000..d8debf72c --- /dev/null +++ b/src/sampletones_tools/samples/bitphase.py @@ -0,0 +1,39 @@ +from pathlib import Path +from typing import Final, List + +from sampletones_core.formats.bitphase.btp import write_btp +from sampletones_core.formats.bitphase.builder import project_to_bitphase +from sampletones_core.project.project import Project +from sampletones_tools.corpus.build import Corpus + +DOCUMENT_FILENAME: Final[str] = "drums.btp" +GROOVE_DOCUMENT_FILENAME: Final[str] = "drums-groove.btp" +GROOVE_TEMPO: Final[int] = 210 + + +def at_tempo(project: Project, tempo: int) -> Project: + """The same project played at another tempo, leaving the given project as it is.""" + return Project( + metadata=project.metadata, + info=project.info, + settings=project.settings.model_copy(update={"tempo": tempo}), + voices=project.voices, + song=project.song, + ) + + +def write_samples(corpus: Corpus, output: Path) -> List[Path]: + """Writes the corpus arrangement as two Bitphase documents: at its own tempo and as a groove. + + Args: + corpus: The samples and the arrangement. + output: The directory the documents are written into. + + Returns: + List[Path]: The document at the song's tempo, then the one carrying a groove. + """ + document = output / DOCUMENT_FILENAME + write_btp(document, project_to_bitphase(corpus.project)) + groove_document = output / GROOVE_DOCUMENT_FILENAME + write_btp(groove_document, project_to_bitphase(at_tempo(corpus.project, GROOVE_TEMPO))) + return [document, groove_document] diff --git a/src/sampletones_tools/samples/commands/__init__.py b/src/sampletones_tools/samples/commands/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/sampletones_tools/samples/commands/btp.py b/src/sampletones_tools/samples/commands/btp.py new file mode 100644 index 000000000..fb7dfb124 --- /dev/null +++ b/src/sampletones_tools/samples/commands/btp.py @@ -0,0 +1,45 @@ +from argparse import ArgumentParser, Namespace +from typing import Final + +from sampletones_shared.command import Command +from sampletones_tools.samples.commands.options import ( + ACTION_FIELD, + ACTION_METAVAR, + SAMPLES, + SamplesArguments, + add_output_option, + print_written, +) + +NAME: Final[str] = "btp" +HELP: Final[str] = "write example Bitphase documents (.btp) from the synthetic corpus" +SAMPLES_HELP: Final[str] = "write the corpus arrangement as Bitphase documents (.btp), at its own tempo and as a groove" + + +def configure(parser: ArgumentParser) -> None: + actions = parser.add_subparsers( + dest=ACTION_FIELD, + metavar=ACTION_METAVAR, + required=True, + ) + add_output_option( + actions.add_parser(SAMPLES, help=SAMPLES_HELP, description=SAMPLES_HELP), + ) + + +def run(arguments: Namespace) -> int: + """Builds the corpus, writes its arrangement and prints each file written.""" + given = SamplesArguments(output=arguments.output) + + from sampletones_tools.samples.bitphase import write_samples + from sampletones_tools.samples.emit import emit_samples + + return print_written(emit_samples(given.output, write_samples)) + + +BTP: Final[Command] = Command( + name=NAME, + help=HELP, + configure=configure, + run=run, +) diff --git a/src/sampletones_tools/samples/commands/ftm.py b/src/sampletones_tools/samples/commands/ftm.py new file mode 100644 index 000000000..aacbc783e --- /dev/null +++ b/src/sampletones_tools/samples/commands/ftm.py @@ -0,0 +1,45 @@ +from argparse import ArgumentParser, Namespace +from typing import Final + +from sampletones_shared.command import Command +from sampletones_tools.samples.commands.options import ( + ACTION_FIELD, + ACTION_METAVAR, + SAMPLES, + SamplesArguments, + add_output_option, + print_written, +) + +NAME: Final[str] = "ftm" +HELP: Final[str] = "write an example FamiTracker module (.ftm) from the synthetic corpus" +SAMPLES_HELP: Final[str] = "write the corpus arrangement as a FamiTracker module (.ftm)" + + +def configure(parser: ArgumentParser) -> None: + actions = parser.add_subparsers( + dest=ACTION_FIELD, + metavar=ACTION_METAVAR, + required=True, + ) + add_output_option( + actions.add_parser(SAMPLES, help=SAMPLES_HELP, description=SAMPLES_HELP), + ) + + +def run(arguments: Namespace) -> int: + """Builds the corpus, writes its arrangement and prints each file written.""" + given = SamplesArguments(output=arguments.output) + + from sampletones_tools.samples.emit import emit_samples + from sampletones_tools.samples.famitracker import write_samples + + return print_written(emit_samples(given.output, write_samples)) + + +FTM: Final[Command] = Command( + name=NAME, + help=HELP, + configure=configure, + run=run, +) diff --git a/src/sampletones_tools/samples/commands/nsf.py b/src/sampletones_tools/samples/commands/nsf.py new file mode 100644 index 000000000..7671686d9 --- /dev/null +++ b/src/sampletones_tools/samples/commands/nsf.py @@ -0,0 +1,106 @@ +from argparse import ArgumentParser, Namespace +from dataclasses import dataclass +from pathlib import Path +from typing import Final + +from sampletones_shared.command import Command +from sampletones_tools.samples.commands.options import ( + ACTION_FIELD, + ACTION_METAVAR, + SAMPLES, + SamplesArguments, + add_output_option, + print_written, +) + +NAME: Final[str] = "nsf" +HELP: Final[str] = "write example .nsf files from the synthetic corpus and render .nsf files to waves" +SAMPLES_HELP: Final[str] = "write each corpus sample and the corpus arrangement as .nsf files" +RENDER: Final[str] = "render" +RENDER_HELP: Final[str] = "render the .nsf files in a directory to waves through ffmpeg's libgme demuxer" +DIRECTORY_HELP: Final[str] = "the directory holding the exported .nsf files" +TAIL_HELP: Final[str] = "seconds kept past the end of each song" +DEFAULT_TAIL_SECONDS: Final[float] = 0.5 + + +@dataclass(frozen=True) +class RenderArguments: + """What a render is given: the directory of exported files and the tail each wave keeps.""" + + directory: Path + tail: float + + +def configure(parser: ArgumentParser) -> None: + actions = parser.add_subparsers( + dest=ACTION_FIELD, + metavar=ACTION_METAVAR, + required=True, + ) + samples = actions.add_parser( + SAMPLES, + help=SAMPLES_HELP, + description=SAMPLES_HELP, + ) + add_output_option(samples) + render = actions.add_parser( + RENDER, + help=RENDER_HELP, + description=RENDER_HELP, + ) + render.add_argument( + "--directory", + type=Path, + required=True, + help=DIRECTORY_HELP, + ) + render.add_argument( + "--tail", + type=float, + default=DEFAULT_TAIL_SECONDS, + help=TAIL_HELP, + ) + + +def run(arguments: Namespace) -> int: + """Writes the example files or renders exported ones, as the action names.""" + if arguments.action == SAMPLES: + return _write_samples(SamplesArguments(output=arguments.output)) + + return _render( + RenderArguments(directory=arguments.directory, tail=arguments.tail), + ) + + +def _write_samples(given: SamplesArguments) -> int: + from sampletones_tools.samples.emit import emit_samples + from sampletones_tools.samples.nsf import write_samples + + return print_written(emit_samples(given.output, write_samples)) + + +def _render(given: RenderArguments) -> int: + """Renders the exported files and prints each wave written. + + Raises: + SystemExit: If ffmpeg is unusable or rejects a file. + """ + from sampletones_tools.samples.render import RenderingError, render_directory + + try: + rendered = render_directory(given.directory, given.tail) + except RenderingError as error: + raise SystemExit(str(error)) from error + + for wave in rendered: + print(f"{wave.destination} {wave.seconds:.3f} s") + + return 0 + + +NSF: Final[Command] = Command( + name=NAME, + help=HELP, + configure=configure, + run=run, +) diff --git a/src/sampletones_tools/samples/commands/options.py b/src/sampletones_tools/samples/commands/options.py new file mode 100644 index 000000000..1b4343472 --- /dev/null +++ b/src/sampletones_tools/samples/commands/options.py @@ -0,0 +1,29 @@ +from argparse import ArgumentParser +from dataclasses import dataclass +from pathlib import Path +from typing import Final, Sequence + +ACTION_FIELD: Final[str] = "action" +ACTION_METAVAR: Final[str] = "" +SAMPLES: Final[str] = "samples" +OUTPUT_HELP: Final[str] = "the directory the files are written into, created when missing" + + +@dataclass(frozen=True) +class SamplesArguments: + """What an emitter run is given: the directory its files are written into.""" + + output: Path + + +def add_output_option(parser: ArgumentParser) -> None: + """Adds the required output directory an emitter writes into.""" + parser.add_argument("--output", "-o", type=Path, required=True, help=OUTPUT_HELP) + + +def print_written(paths: Sequence[Path]) -> int: + """Prints each file an emitter wrote and returns the command's success status.""" + for path in paths: + print(f"Wrote {path}") + + return 0 diff --git a/src/sampletones_tools/samples/emit.py b/src/sampletones_tools/samples/emit.py new file mode 100644 index 000000000..d334d1255 --- /dev/null +++ b/src/sampletones_tools/samples/emit.py @@ -0,0 +1,21 @@ +from pathlib import Path +from typing import Callable, List + +from sampletones_tools.corpus.build import Corpus, build_synthetic_corpus + +Emitter = Callable[[Corpus, Path], List[Path]] + + +def emit_samples(output: Path, emitter: Emitter) -> List[Path]: + """Builds the synthetic corpus and hands it to an emitter writing into ``output``. + + Args: + output: The directory the files are written into, created when missing. + emitter: The writer of one format. + + Returns: + List[Path]: The files the emitter wrote. + """ + corpus = build_synthetic_corpus() + output.mkdir(parents=True, exist_ok=True) + return emitter(corpus, output) diff --git a/src/sampletones_tools/samples/famitracker.py b/src/sampletones_tools/samples/famitracker.py new file mode 100644 index 000000000..a3dba6459 --- /dev/null +++ b/src/sampletones_tools/samples/famitracker.py @@ -0,0 +1,22 @@ +from pathlib import Path +from typing import Final, List + +from sampletones_core.formats.famitracker.export import write_ftm +from sampletones_tools.corpus.build import Corpus + +MODULE_FILENAME: Final[str] = "drums.ftm" + + +def write_samples(corpus: Corpus, output: Path) -> List[Path]: + """Writes the corpus arrangement as one FamiTracker module. + + Args: + corpus: The samples and the arrangement. + output: The directory the module is written into. + + Returns: + List[Path]: The module written. + """ + destination = output / MODULE_FILENAME + write_ftm(destination, corpus.project) + return [destination] diff --git a/src/sampletones_tools/samples/nsf.py b/src/sampletones_tools/samples/nsf.py new file mode 100644 index 000000000..ca577b01c --- /dev/null +++ b/src/sampletones_tools/samples/nsf.py @@ -0,0 +1,49 @@ +from pathlib import Path +from typing import Final, List + +from sampletones_core.exports.request import ProjectExport +from sampletones_player.builder import song_from_reconstruction +from sampletones_player.driver.image import DriverImage +from sampletones_player.export import NSFBackend +from sampletones_player.nsf.file import write_nsf +from sampletones_player.nsf.information import NSFInformation +from sampletones_shared.paths.extensions import EXT_FILE_NSF +from sampletones_tools.corpus.build import Corpus + +SONG_NAME: Final[str] = "song" + + +def exported_information(name: str, author: str) -> NSFInformation: + """The header text an exported sample carries: its own name, by the author of the arrangement.""" + return NSFInformation(title=name, artist=author) + + +def write_samples(corpus: Corpus, output: Path) -> List[Path]: + """Writes each corpus sample as a program of its own, then the arrangement through the backend. + + Args: + corpus: The samples and the arrangement. + output: The directory the files are written into. + + Returns: + List[Path]: The files written, the samples first and the arrangement last. + """ + image = DriverImage.load() + written: List[Path] = [] + for name, sample in corpus.catalog.items(): + destination = output / f"{name}{EXT_FILE_NSF}" + write_nsf( + destination, + song_from_reconstruction(sample.reconstruction, loop_tick=None), + exported_information(name, corpus.project.info.author), + image, + ) + written.append(destination) + + arrangement = output / f"{SONG_NAME}{EXT_FILE_NSF}" + NSFBackend().write_project( + arrangement, + ProjectExport(project=corpus.project), + ) + written.append(arrangement) + return written diff --git a/scripts/nsf_render.py b/src/sampletones_tools/samples/render.py old mode 100755 new mode 100644 similarity index 66% rename from scripts/nsf_render.py rename to src/sampletones_tools/samples/render.py index 922c58407..8da4f30da --- a/scripts/nsf_render.py +++ b/src/sampletones_tools/samples/render.py @@ -1,11 +1,8 @@ -#!/usr/bin/env python3 - -import argparse import struct import subprocess -import sys +from dataclasses import dataclass from pathlib import Path -from typing import Dict, Final, List, Sequence +from typing import Dict, Final, List from sampletones_player.driver.image import DriverImage from sampletones_player.specification.clock import ( @@ -18,6 +15,7 @@ STEP_WHOLE_OFFSET, TOTAL_TICKS_OFFSET, ) +from sampletones_shared.exceptions import SampleToNESError from sampletones_shared.paths.extensions import EXT_FILE_NSF, EXT_FILE_WAVE from sampletones_shared.utils.system.programs import ( locate_program, @@ -25,8 +23,6 @@ ) from sampletones_shared.utils.system.system import System -SAMPLES_DIRECTORY: Final[Path] = Path("build") / "nsf" -TAIL_SECONDS: Final[float] = 0.5 FFMPEG: Final[str] = "ffmpeg" GME_FORMAT: Final[str] = "libgme" WORD: Final[str] = " bool: """Whether the installed ffmpeg carries the demuxer an exported file is read through. @@ -60,6 +75,24 @@ def decodes_exports() -> bool: return GME_FORMAT in reported.stdout +def require_renderer() -> None: + """Holds a render to an ffmpeg that decodes exported files. + + Raises: + RenderingError: If ffmpeg is absent, naming how this system installs it, or carries no + libgme demuxer. + """ + if locate_program(FFMPEG) is None: + raise RenderingError( + missing_program_message(FFMPEG, RENDER_PURPOSE, INSTALL_HINTS), + ) + + if not decodes_exports(): + raise RenderingError( + f"{FFMPEG} reports no {GME_FORMAT} demuxer; rendering needs a build made with --enable-libgme" + ) + + def song_seconds(data: bytes, code_length: int) -> float: """How long the song in an exported file lasts, read out of the block behind the driver. @@ -110,62 +143,41 @@ def render(source: Path, destination: Path, seconds: float) -> None: ) -def main(argv: Sequence[str]) -> int: - """Renders every exported file in a directory to a wave beside it.""" +def render_directory(directory: Path, tail_seconds: float) -> List[RenderedWave]: + """Renders every exported file in a directory to a wave beside it. - parser = argparse.ArgumentParser( - description="Render exported .nsf files to waves with ffmpeg's libgme demuxer.", - ) - parser.add_argument( - "--directory", - type=Path, - default=SAMPLES_DIRECTORY, - help="directory holding the exported .nsf files", - ) - parser.add_argument( - "--tail", - type=float, - default=TAIL_SECONDS, - help="seconds to keep past the end of each song", - ) - arguments = parser.parse_args(list(argv)) - - if locate_program(FFMPEG) is None: - print(missing_program_message(FFMPEG, RENDER_PURPOSE, INSTALL_HINTS), file=sys.stderr) - return 1 + Args: + directory: The directory holding the `.nsf` files. + tail_seconds: How much to keep past the end of each song. - if not decodes_exports(): - print( - f"{FFMPEG} reports no {GME_FORMAT} demuxer; rendering needs a build made with --enable-libgme", - file=sys.stderr, - ) - return 1 + Returns: + List[RenderedWave]: The waves written, in path order. - sources: List[Path] = sorted(arguments.directory.glob(f"*{EXT_FILE_NSF}")) + Raises: + RenderingError: If ffmpeg is unusable, the directory holds no exported file, or ffmpeg + rejects one. + """ + require_renderer() + sources = sorted(directory.glob(f"*{EXT_FILE_NSF}")) if not sources: - print( - f"no {EXT_FILE_NSF} files in {arguments.directory}; run make nsf-samples first", - file=sys.stderr, - ) - return 1 + raise RenderingError(f"no {EXT_FILE_NSF} files in {directory}") code_length = len(DriverImage.load().code) + rendered: List[RenderedWave] = [] for source in sources: destination = source.with_suffix(EXT_FILE_WAVE) - seconds = song_seconds(source.read_bytes(), code_length) + arguments.tail + seconds = song_seconds(source.read_bytes(), code_length) + tail_seconds try: render(source, destination, seconds) except subprocess.CalledProcessError as error: - print( - f"{FFMPEG} rejected {source}: exit status {error.returncode}", - file=sys.stderr, - ) - return 1 - - print(f"{destination} {seconds:.3f} s") - - return 0 + raise RenderingError(f"{FFMPEG} rejected {source}: exit status {error.returncode}") from error + rendered.append( + RenderedWave( + source=source, + destination=destination, + seconds=seconds, + ) + ) -if __name__ == "__main__": - raise SystemExit(main(sys.argv[1:])) + return rendered diff --git a/src/sampletones_synthesis/__init__.py b/src/sampletones_tools/synthesis/__init__.py similarity index 100% rename from src/sampletones_synthesis/__init__.py rename to src/sampletones_tools/synthesis/__init__.py diff --git a/src/sampletones_synthesis/envelopes/__init__.py b/src/sampletones_tools/synthesis/envelopes/__init__.py similarity index 100% rename from src/sampletones_synthesis/envelopes/__init__.py rename to src/sampletones_tools/synthesis/envelopes/__init__.py diff --git a/src/sampletones_synthesis/envelopes/exponential_decay.py b/src/sampletones_tools/synthesis/envelopes/exponential_decay.py similarity index 100% rename from src/sampletones_synthesis/envelopes/exponential_decay.py rename to src/sampletones_tools/synthesis/envelopes/exponential_decay.py diff --git a/src/sampletones_synthesis/envelopes/linear_attack.py b/src/sampletones_tools/synthesis/envelopes/linear_attack.py similarity index 100% rename from src/sampletones_synthesis/envelopes/linear_attack.py rename to src/sampletones_tools/synthesis/envelopes/linear_attack.py diff --git a/src/sampletones_synthesis/envelopes/linear_ramp.py b/src/sampletones_tools/synthesis/envelopes/linear_ramp.py similarity index 100% rename from src/sampletones_synthesis/envelopes/linear_ramp.py rename to src/sampletones_tools/synthesis/envelopes/linear_ramp.py diff --git a/src/sampletones_synthesis/envelopes/types.py b/src/sampletones_tools/synthesis/envelopes/types.py similarity index 100% rename from src/sampletones_synthesis/envelopes/types.py rename to src/sampletones_tools/synthesis/envelopes/types.py diff --git a/src/sampletones_synthesis/filters/__init__.py b/src/sampletones_tools/synthesis/filters/__init__.py similarity index 100% rename from src/sampletones_synthesis/filters/__init__.py rename to src/sampletones_tools/synthesis/filters/__init__.py diff --git a/src/sampletones_synthesis/filters/butterworth_highpass.py b/src/sampletones_tools/synthesis/filters/butterworth_highpass.py similarity index 100% rename from src/sampletones_synthesis/filters/butterworth_highpass.py rename to src/sampletones_tools/synthesis/filters/butterworth_highpass.py diff --git a/src/sampletones_synthesis/filters/types.py b/src/sampletones_tools/synthesis/filters/types.py similarity index 100% rename from src/sampletones_synthesis/filters/types.py rename to src/sampletones_tools/synthesis/filters/types.py diff --git a/src/sampletones_synthesis/frequency.py b/src/sampletones_tools/synthesis/frequency.py similarity index 91% rename from src/sampletones_synthesis/frequency.py rename to src/sampletones_tools/synthesis/frequency.py index 9ab12a4df..f6a8fb700 100644 --- a/src/sampletones_synthesis/frequency.py +++ b/src/sampletones_tools/synthesis/frequency.py @@ -18,7 +18,9 @@ def _require_hertz(value: Any) -> Any: ValueError: If the value is an integer. """ if isinstance(value, int) and not isinstance(value, bool): - raise ValueError("An integer frequency specification is a MIDI pitch and must lie in the pitch range") + raise ValueError( # noqa: TRY004 + "An integer frequency specification is a MIDI pitch and must lie in the pitch range" + ) return value diff --git a/src/sampletones_synthesis/oscillators/__init__.py b/src/sampletones_tools/synthesis/oscillators/__init__.py similarity index 100% rename from src/sampletones_synthesis/oscillators/__init__.py rename to src/sampletones_tools/synthesis/oscillators/__init__.py diff --git a/src/sampletones_synthesis/oscillators/exponential_glide.py b/src/sampletones_tools/synthesis/oscillators/exponential_glide.py similarity index 95% rename from src/sampletones_synthesis/oscillators/exponential_glide.py rename to src/sampletones_tools/synthesis/oscillators/exponential_glide.py index 31254045a..fef55d880 100644 --- a/src/sampletones_synthesis/oscillators/exponential_glide.py +++ b/src/sampletones_tools/synthesis/oscillators/exponential_glide.py @@ -3,7 +3,7 @@ import numpy as np from pydantic import BaseModel, ConfigDict, Field -from sampletones_synthesis.frequency import FrequencySpec, resolve_frequency +from sampletones_tools.synthesis.frequency import FrequencySpec, resolve_frequency class ExponentialGlideOscillator(BaseModel): diff --git a/src/sampletones_synthesis/oscillators/geometric_sweep.py b/src/sampletones_tools/synthesis/oscillators/geometric_sweep.py similarity index 95% rename from src/sampletones_synthesis/oscillators/geometric_sweep.py rename to src/sampletones_tools/synthesis/oscillators/geometric_sweep.py index db207f3c8..8fdd899c7 100644 --- a/src/sampletones_synthesis/oscillators/geometric_sweep.py +++ b/src/sampletones_tools/synthesis/oscillators/geometric_sweep.py @@ -3,7 +3,7 @@ import numpy as np from pydantic import BaseModel, ConfigDict -from sampletones_synthesis.frequency import FrequencySpec, resolve_frequency +from sampletones_tools.synthesis.frequency import FrequencySpec, resolve_frequency class GeometricSweepOscillator(BaseModel): diff --git a/src/sampletones_synthesis/oscillators/pulse.py b/src/sampletones_tools/synthesis/oscillators/pulse.py similarity index 93% rename from src/sampletones_synthesis/oscillators/pulse.py rename to src/sampletones_tools/synthesis/oscillators/pulse.py index 8bb5e90d4..bfaca5790 100644 --- a/src/sampletones_synthesis/oscillators/pulse.py +++ b/src/sampletones_tools/synthesis/oscillators/pulse.py @@ -3,7 +3,7 @@ import numpy as np from pydantic import BaseModel, ConfigDict, Field -from sampletones_synthesis.frequency import FrequencySpec, resolve_frequency +from sampletones_tools.synthesis.frequency import FrequencySpec, resolve_frequency DutyCycle = Annotated[float, Field(gt=0.0, lt=1.0)] diff --git a/src/sampletones_synthesis/oscillators/sine.py b/src/sampletones_tools/synthesis/oscillators/sine.py similarity index 91% rename from src/sampletones_synthesis/oscillators/sine.py rename to src/sampletones_tools/synthesis/oscillators/sine.py index 3e2e96ed3..9abeb685e 100644 --- a/src/sampletones_synthesis/oscillators/sine.py +++ b/src/sampletones_tools/synthesis/oscillators/sine.py @@ -3,7 +3,7 @@ import numpy as np from pydantic import BaseModel, ConfigDict -from sampletones_synthesis.frequency import FrequencySpec, resolve_frequency +from sampletones_tools.synthesis.frequency import FrequencySpec, resolve_frequency class SineOscillator(BaseModel): diff --git a/src/sampletones_synthesis/oscillators/types.py b/src/sampletones_tools/synthesis/oscillators/types.py similarity index 100% rename from src/sampletones_synthesis/oscillators/types.py rename to src/sampletones_tools/synthesis/oscillators/types.py diff --git a/src/sampletones_synthesis/oscillators/walk_noise.py b/src/sampletones_tools/synthesis/oscillators/walk_noise.py similarity index 100% rename from src/sampletones_synthesis/oscillators/walk_noise.py rename to src/sampletones_tools/synthesis/oscillators/walk_noise.py diff --git a/src/sampletones_synthesis/oscillators/white_noise.py b/src/sampletones_tools/synthesis/oscillators/white_noise.py similarity index 100% rename from src/sampletones_synthesis/oscillators/white_noise.py rename to src/sampletones_tools/synthesis/oscillators/white_noise.py diff --git a/src/sampletones_synthesis/protocols.py b/src/sampletones_tools/synthesis/protocols.py similarity index 100% rename from src/sampletones_synthesis/protocols.py rename to src/sampletones_tools/synthesis/protocols.py diff --git a/src/sampletones_synthesis/voice/__init__.py b/src/sampletones_tools/synthesis/voice/__init__.py similarity index 100% rename from src/sampletones_synthesis/voice/__init__.py rename to src/sampletones_tools/synthesis/voice/__init__.py diff --git a/src/sampletones_synthesis/voice/layer.py b/src/sampletones_tools/synthesis/voice/layer.py similarity index 90% rename from src/sampletones_synthesis/voice/layer.py rename to src/sampletones_tools/synthesis/voice/layer.py index 31b8429d7..5ea85b401 100644 --- a/src/sampletones_synthesis/voice/layer.py +++ b/src/sampletones_tools/synthesis/voice/layer.py @@ -3,8 +3,8 @@ import numpy as np from pydantic import BaseModel, ConfigDict, Field -from sampletones_synthesis.envelopes.types import EnvelopeUnion -from sampletones_synthesis.oscillators.types import OscillatorUnion +from sampletones_tools.synthesis.envelopes.types import EnvelopeUnion +from sampletones_tools.synthesis.oscillators.types import OscillatorUnion class Layer(BaseModel): diff --git a/src/sampletones_synthesis/voice/voice.py b/src/sampletones_tools/synthesis/voice/voice.py similarity index 97% rename from src/sampletones_synthesis/voice/voice.py rename to src/sampletones_tools/synthesis/voice/voice.py index 968502c5a..89187ebcd 100644 --- a/src/sampletones_synthesis/voice/voice.py +++ b/src/sampletones_tools/synthesis/voice/voice.py @@ -3,7 +3,7 @@ import numpy as np from pydantic import BaseModel, ConfigDict, Field -from sampletones_synthesis.filters.types import FilterUnion +from sampletones_tools.synthesis.filters.types import FilterUnion from .layer import Layer diff --git a/tests/benchmarks/conftest.py b/tests/benchmarks/conftest.py index 3c2949f8d..aaa5d2942 100644 --- a/tests/benchmarks/conftest.py +++ b/tests/benchmarks/conftest.py @@ -1,15 +1,11 @@ from tests.integration.conftest import ( - audio_directory, instrument_catalog, integration_project, - module_config, - synth_config, + synthetic_corpus, ) __all__ = [ - "audio_directory", "instrument_catalog", "integration_project", - "module_config", - "synth_config", + "synthetic_corpus", ] diff --git a/tests/benchmarks/test_compression.py b/tests/benchmarks/test_compression.py index 083074ef8..0753900f3 100644 --- a/tests/benchmarks/test_compression.py +++ b/tests/benchmarks/test_compression.py @@ -6,7 +6,7 @@ from sampletones_core.project.project import Project from sampletones_player.compression.encode import encode_planes from sampletones_player.compression.options import EVERY_LAYER -from tests.integration.nsf.corpus import ( +from sampletones_tools.codec.report.corpus import ( LONG_ARRANGEMENT, RECONSTRUCTION, RECONSTRUCTION_SECONDS, diff --git a/tests/integration/assets/module_config.py b/tests/integration/assets/module_config.py deleted file mode 100644 index 3b52384d5..000000000 --- a/tests/integration/assets/module_config.py +++ /dev/null @@ -1,20 +0,0 @@ -from pydantic import BaseModel, ConfigDict - -from sampletones_shared.types.path import Pathlike -from sampletones_shared.utils.serialization import load_yaml - - -class ModuleConfig(BaseModel): - """Module identity and playback settings that shape the exported ``.ftm``.""" - - model_config = ConfigDict(frozen=True, extra="forbid") - - title: str - author: str - tempo: int - speed: int - nes_frequency: int - - -def load_module_config(path: Pathlike) -> ModuleConfig: - return ModuleConfig.model_validate(load_yaml(path)) diff --git a/tests/integration/assets/reconstruction.py b/tests/integration/assets/reconstruction.py deleted file mode 100644 index a03457c13..000000000 --- a/tests/integration/assets/reconstruction.py +++ /dev/null @@ -1,240 +0,0 @@ -from pathlib import Path -from typing import Any, Dict, Final, FrozenSet, List, Tuple - -import numpy as np - -from sampletones_core.audio import write_wave -from sampletones_core.audio.processing import normalize -from sampletones_core.configs import Config, InstructionsLibraryConfig -from sampletones_core.configs.generation import GenerationConfig -from sampletones_core.constants.enums import ( - DEFAULT_CHANNELS, - ChannelName, - HierarchyMode, - SpectrumMethod, - bending_channels, -) -from sampletones_core.fft import Window -from sampletones_core.fft.features import get_feature_extractor -from sampletones_core.generators import get_generators_by_channels -from sampletones_core.instructions import InstructionUnion -from sampletones_core.library import ( - InstructionLibrary, - InstructionLibraryData, - InstructionLibraryFragment, -) -from sampletones_core.project.voices.sample import Sample -from sampletones_core.reconstructions import Reconstruction, Reconstructor -from sampletones_core.reconstructions.reconstructor.stems.configs.config import StemsConfig -from sampletones_core.reconstructions.reconstructor.stems.configs.entry import StemEntry -from sampletones_core.reconstructions.reconstructor.stems.configs.hierarchy import StemsHierarchy -from sampletones_core.reconstructions.reconstructor.stems.configs.settings import StemSettings -from sampletones_shared.types.path import Pathlike -from sampletones_shared.utils.serialization import load_yaml -from tests.integration.assets.synth_config import SynthConfig - -INSTRUCTIONS_PER_GENERATOR: Final[int] = 48 -CHANNELS: Final[List[ChannelName]] = [ - ChannelName.PULSE1, - ChannelName.TRIANGLE, - ChannelName.NOISE, -] - -STEM_A_ID: Final[int] = 0 -STEM_B_ID: Final[int] = 1 -STEM_C_ID: Final[int] = 2 -THREE_STEM_CHANNELS: Final[List[ChannelName]] = [ - ChannelName.PULSE1, - ChannelName.PULSE2, - ChannelName.TRIANGLE, - ChannelName.NOISE, -] -THREE_STEM_ENTRY_CHANNELS: Final[Dict[int, List[ChannelName]]] = { - STEM_A_ID: [ChannelName.PULSE1, ChannelName.TRIANGLE, ChannelName.NOISE], - STEM_B_ID: [ChannelName.PULSE2, ChannelName.TRIANGLE], - STEM_C_ID: [ChannelName.PULSE1, ChannelName.NOISE], -} -STEM_RECORDING_DURATION_SECONDS: Final[float] = 0.5 - - -def three_stem_config() -> StemsConfig: - """Builds the three-stem setup the stems tests share. - - Stems a (pulse 1, triangle, noise) and b (pulse 2, triangle) pick on the first - hierarchy level, stem c (pulse 1, noise) on the second. - """ - return StemsConfig( - entries=[ - StemEntry(id=stem_id, settings=StemSettings(channels=channels, bends=bending_channels(channels))) - for stem_id, channels in THREE_STEM_ENTRY_CHANNELS.items() - ], - hierarchy=StemsHierarchy( - levels=[[STEM_A_ID, STEM_B_ID], [STEM_C_ID]], - mode=HierarchyMode.STRICT, - ), - channel_cap=1, - ) - - -def three_stem_reconstruction_config() -> Config: - """Builds a reconstruction config for the three-stem example.""" - return Config() - - -def write_three_stem_recordings( - config: Config, - tmp_dir: Pathlike, -) -> Tuple[Path, Path, Path]: - """Writes three distinct stem recordings a, b, c and returns their paths in order.""" - sample_rate = config.library.sample_rate - count = int(sample_rate * STEM_RECORDING_DURATION_SECONDS) - time = np.arange(count) / sample_rate - recordings = { - "a": 0.5 * np.sin(2 * np.pi * 440.0 * time), - "b": 0.4 * np.sin(2 * np.pi * 220.0 * time), - "c": np.random.default_rng(93).uniform(-0.3, 0.3, count), - } - - paths: List[Path] = [] - for name, audio in recordings.items(): - path = Path(tmp_dir) / f"stem_{name}.wav" - write_wave(path, sample_rate, audio) - paths.append(path) - - return paths[0], paths[1], paths[2] - - -def build_mini_library( - config: Config, - *, - per_generator: int = INSTRUCTIONS_PER_GENERATOR, -) -> InstructionLibrary: - """Builds a small in-memory instruction library covering pulse/triangle/noise. - - Candidates are sampled with an even stride across each channel's instruction - space so pitch, volume and period are represented, rather than a biased prefix. - """ - window = Window.from_config(config) - extractor = get_feature_extractor(config, window) - channels = get_generators_by_channels(config, CHANNELS) - - data: Dict[InstructionUnion, InstructionLibraryFragment[Any]] = {} - for channel in channels.values(): - candidates = list(channel.get_possible_instructions()) - stride = max(1, len(candidates) // per_generator) - for instruction in candidates[::stride][:per_generator]: - data[instruction] = InstructionLibraryFragment.create(channel, instruction, extractor) - - library = InstructionLibrary() - library.data[library.create_key(config, window)] = InstructionLibraryData.create(config, data) - return library - - -def reconstruct_sample( - audio: np.ndarray, - config: Config, - library: InstructionLibrary, - channels: FrozenSet[ChannelName], - *, - tmp_dir: Pathlike, - name: str, -) -> Reconstruction: - """Runs the real reconstruction pipeline on ``audio`` via a temp WAV.""" - path = Path(tmp_dir) / f"{name}.wav" - write_wave(path, config.library.sample_rate, audio) - reconstruction = Reconstructor(config, channels, library=library)(path) - if reconstruction is None: - raise AssertionError(f"Reconstruction of '{name}' produced no result") - - return reconstruction - - -def make_sample( - name: str, - audio: np.ndarray, - config: Config, - library: InstructionLibrary, - *, - tmp_dir: Pathlike, - expected_slices: FrozenSet[ChannelName], - loop: bool = False, -) -> Sample: - """Reconstructs ``audio`` into a `Sample`, asserting the channels it plays.""" - reconstruction = reconstruct_sample( - audio, - config, - library, - expected_slices, - tmp_dir=tmp_dir, - name=name, - ) - played = frozenset(reconstruction.playing_channels) - if played != expected_slices: - raise AssertionError(f"Sample '{name}' covers {set(played)}, expected {set(expected_slices)}") - - return Sample(name=name, reconstruction=reconstruction) - - -def load_instrument_catalog( - spec_path: Pathlike, - synth_config: SynthConfig, - *, - tmp_dir: Pathlike, -) -> Dict[str, Sample]: - """Builds the reconstructed sample catalog described by a JSON spec. - - The spec fixes the reconstruction settings (spectrum method, gamma, library - size) shared by one in-memory library, and per instrument the synth to run and - the channel slices to cover. Synth parameters come from ``synth_config``. - Returns a name -> `Sample` mapping. - """ - spec = load_yaml(spec_path) - settings = spec["reconstruction"] - library_config = InstructionsLibraryConfig( - spectrum_method=SpectrumMethod(settings["spectrum_method"]), - transformation_gamma=settings["transformation_gamma"], - ) - sample_rate = library_config.sample_rate - library = build_mini_library( - Config(library=library_config), - per_generator=settings["instructions_per_generator"], - ) - - catalog: Dict[str, Sample] = {} - for entry in spec["instruments"]: - channels = [ChannelName(name) for name in entry["channels"]] - config = Config(library=library_config) - audio = _render_instrument(synth_config, entry["synth"], sample_rate=sample_rate) - catalog[entry["name"]] = make_sample( - entry["name"], - audio, - config, - library, - tmp_dir=tmp_dir, - expected_slices=frozenset(channels), - ) - - return catalog - - -def _render_instrument( - synth_config: SynthConfig, - name: str, - *, - sample_rate: int, -) -> np.ndarray: - """ - Render a named voice at peak level 1.0. - - A fresh channel seeded from the synth configuration keeps every instrument - reproducible independently of catalog order. - """ - voice = synth_config.voices[name] - generator = np.random.default_rng(synth_config.seed) - instrument: np.ndarray = normalize( - voice.render( - sample_rate=sample_rate, - generator=generator, - ) - ) - return instrument diff --git a/tests/integration/assets/song_loader.py b/tests/integration/assets/song_loader.py deleted file mode 100644 index e8f48294c..000000000 --- a/tests/integration/assets/song_loader.py +++ /dev/null @@ -1,94 +0,0 @@ -from typing import Any, Dict, List, Mapping, Optional - -from sampletones_core.constants.enums import ChannelName -from sampletones_core.project.patterns.channel import Channel -from sampletones_core.project.patterns.pattern import Pattern -from sampletones_core.project.patterns.row import Row -from sampletones_core.project.song import Song -from sampletones_core.project.voices.note_off import NoteOff -from sampletones_core.project.voices.note_on import NoteOn -from sampletones_core.project.voices.sample import Sample -from sampletones_shared.types.path import Pathlike -from sampletones_shared.utils.serialization import load_yaml - -RowSpec = Dict[str, Any] - - -def _order( - order_specs: List[Dict[str, int]], -) -> List[Dict[ChannelName, Optional[int]]]: - frames: List[Dict[ChannelName, Optional[int]]] = [] - for spec in order_specs: - frames.append({channel: spec.get(channel.value) for channel in ChannelName.items()}) - - return frames - - -def _row(spec: RowSpec, channel: ChannelName, samples_by_name: Mapping[str, Sample]) -> Row: - transpose = spec.get("transpose") - volume = spec.get("volume") - - if spec.get("off"): - return Row(command=NoteOff(), volume=volume) - - sample_name = spec.get("sample") - if sample_name is None: - return Row(transpose=transpose, volume=volume) - - sample = samples_by_name[sample_name] - if channel not in sample.reconstruction.instructions: - raise ValueError(f"Sample '{sample_name}' has no '{channel.value}' slice for the {channel.value} channel") - - command = NoteOn(voice_id=sample.id) - return Row(command=command, transpose=transpose, volume=volume) - - -def _pattern( - row_specs: List[RowSpec], - rows_per_pattern: int, - channel: ChannelName, - samples_by_name: Mapping[str, Sample], -) -> Pattern: - rows = [Row() for _ in range(rows_per_pattern)] - for spec in row_specs: - rows[spec["row"]] = _row(spec, channel, samples_by_name) - - return Pattern(rows=rows) - - -def _channels( - channels_spec: Dict[str, Dict[str, Any]], - rows_per_pattern: int, - samples_by_name: Mapping[str, Sample], -) -> Dict[ChannelName, Channel]: - channels: Dict[ChannelName, Channel] = {} - for name, spec in channels_spec.items(): - channel = ChannelName(name) - patterns = { - int(index): _pattern( - row_specs, - rows_per_pattern, - channel, - samples_by_name, - ) - for index, row_specs in spec["patterns"].items() - } - channels[channel] = Channel( - name=channel, - patterns=patterns, - ) - - for channel in ChannelName.items(): - channels.setdefault(channel, Channel(name=channel, patterns={})) - - return channels - - -def load_song(path: Pathlike, samples_by_name: Mapping[str, Sample]) -> Song: - document = load_yaml(path) - rows_per_pattern = document["rows_per_pattern"] - return Song( - rows_per_pattern=rows_per_pattern, - order=_order(document["order"]), - channels=_channels(document["channels"], rows_per_pattern, samples_by_name), - ) diff --git a/tests/integration/assets/synth_config.py b/tests/integration/assets/synth_config.py deleted file mode 100644 index 1cff14660..000000000 --- a/tests/integration/assets/synth_config.py +++ /dev/null @@ -1,20 +0,0 @@ -from typing import Dict - -from pydantic import BaseModel, ConfigDict - -from sampletones_shared.types.path import Pathlike -from sampletones_shared.utils.serialization import load_yaml_model -from sampletones_synthesis.voice.voice import Voice - - -class SynthConfig(BaseModel): - """The named synthesizer voices of the integration suite and their shared noise seed.""" - - model_config = ConfigDict(frozen=True, extra="forbid") - - seed: int - voices: Dict[str, Voice] - - -def load_synth_config(path: Pathlike) -> SynthConfig: - return load_yaml_model(path, SynthConfig) diff --git a/tests/integration/bitphase/conftest.py b/tests/integration/bitphase/conftest.py index d1dd846e8..0007eda69 100644 --- a/tests/integration/bitphase/conftest.py +++ b/tests/integration/bitphase/conftest.py @@ -1,29 +1,17 @@ from pathlib import Path -from typing import Optional import pytest -from tests.integration.output import resolve_output_directory, resolve_output_path -from tests.integration.paths import ( - BTP_OUTPUT_ENV, - DOCUMENT_FILENAME, - GROOVE_DOCUMENT_FILENAME, -) - - -@pytest.fixture(scope="session") -def btp_output_dir() -> Optional[Path]: - """The persistent output directory ``SAMPLETONES_BTP_OUTPUT_DIR`` names.""" - return resolve_output_directory(BTP_OUTPUT_ENV) +from sampletones_tools.samples.bitphase import DOCUMENT_FILENAME, GROOVE_DOCUMENT_FILENAME @pytest.fixture -def document_path(btp_output_dir: Optional[Path], tmp_path: Path) -> Path: +def document_path(tmp_path: Path) -> Path: """Where a produced ``.btp`` is written.""" - return resolve_output_path(btp_output_dir, tmp_path, DOCUMENT_FILENAME) + return tmp_path / DOCUMENT_FILENAME @pytest.fixture -def groove_document_path(btp_output_dir: Optional[Path], tmp_path: Path) -> Path: +def groove_document_path(tmp_path: Path) -> Path: """Where the document carrying a groove is written, beside the one at the song's own tempo.""" - return resolve_output_path(btp_output_dir, tmp_path, GROOVE_DOCUMENT_FILENAME) + return tmp_path / GROOVE_DOCUMENT_FILENAME diff --git a/tests/integration/bitphase/test_btp_pipeline.py b/tests/integration/bitphase/test_btp_pipeline.py index 85ad771ae..fc96efa83 100644 --- a/tests/integration/bitphase/test_btp_pipeline.py +++ b/tests/integration/bitphase/test_btp_pipeline.py @@ -45,6 +45,7 @@ ) from sampletones_core.project.project import Project from sampletones_core.timing import Meter, RowRate, calculate_groove +from sampletones_tools.samples.bitphase import GROOVE_TEMPO, at_tempo from tests.suite.bitphase import ( BITPHASE_NO_EFFECTS, LoadedEffect, @@ -56,7 +57,6 @@ ) EXPECTED_INSTRUMENT_COUNT: Final[int] = 5 -GROOVE_TEMPO: Final[int] = 210 PLAYED_CHANNELS: Final[List[int]] = [ int(ChannelIndex.SQUARE1), int(ChannelIndex.SQUARE2), @@ -75,17 +75,6 @@ def note_index(note: LoadedNote) -> int: return note.name - int(NoteName.C) + (note.octave - FIRST_OCTAVE) * NOTE_RANGE -def at_tempo(project: Project, tempo: int) -> Project: - """The same project played at another tempo, leaving the session-wide fixture as it is.""" - return Project( - metadata=project.metadata, - info=project.info, - settings=project.settings.model_copy(update={"tempo": tempo}), - voices=project.voices, - song=project.song, - ) - - @pytest.fixture def document(integration_project: Project, document_path: Path) -> LoadedProject: write_btp(document_path, project_to_bitphase(integration_project)) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 8d6b920c8..9b5566dac 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -1,56 +1,23 @@ -from pathlib import Path from typing import Dict import pytest from sampletones_core.project.project import Project -from sampletones_core.project.settings import ProjectSettings from sampletones_core.project.voices.sample import Sample -from sampletones_core.structures import IdentifiedCollection -from tests.integration.assets.module_config import ModuleConfig, load_module_config -from tests.integration.assets.reconstruction import load_instrument_catalog -from tests.integration.assets.song_loader import load_song -from tests.integration.assets.synth_config import SynthConfig, load_synth_config -from tests.integration.paths import ( - MODULE_CONFIG_PATH, - RECONSTRUCTION_CONFIG_PATH, - SONG_PATH, - SYNTH_CONFIG_PATH, -) +from sampletones_tools.corpus.build import Corpus, build_synthetic_corpus @pytest.fixture(scope="session") -def audio_directory(tmp_path_factory: pytest.TempPathFactory) -> Path: - return tmp_path_factory.mktemp("integration_audio") +def synthetic_corpus() -> Corpus: + """The corpus the emitters and the report are built from, built once for the session.""" + return build_synthetic_corpus() @pytest.fixture(scope="session") -def synth_config() -> SynthConfig: - return load_synth_config(SYNTH_CONFIG_PATH) +def instrument_catalog(synthetic_corpus: Corpus) -> Dict[str, Sample]: + return synthetic_corpus.catalog @pytest.fixture(scope="session") -def module_config() -> ModuleConfig: - return load_module_config(MODULE_CONFIG_PATH) - - -@pytest.fixture(scope="session") -def instrument_catalog(audio_directory: Path, synth_config: SynthConfig) -> Dict[str, Sample]: - return load_instrument_catalog(RECONSTRUCTION_CONFIG_PATH, synth_config, tmp_dir=audio_directory) - - -@pytest.fixture(scope="session") -def integration_project(instrument_catalog: Dict[str, Sample], module_config: ModuleConfig) -> Project: - voices: IdentifiedCollection[Sample] = IdentifiedCollection() - for sample in instrument_catalog.values(): - voices.append(sample) - - settings = ProjectSettings( - tempo=module_config.tempo, - speed=module_config.speed, - nes_frequency=module_config.nes_frequency, - ) - project = Project.create(title=module_config.title, author=module_config.author, settings=settings) - project.voices = voices - project.song = load_song(SONG_PATH, instrument_catalog) - return project +def integration_project(synthetic_corpus: Corpus) -> Project: + return synthetic_corpus.project diff --git a/tests/integration/famitracker/conftest.py b/tests/integration/famitracker/conftest.py index b10b93fe7..1ffa52897 100644 --- a/tests/integration/famitracker/conftest.py +++ b/tests/integration/famitracker/conftest.py @@ -1,19 +1,11 @@ from pathlib import Path -from typing import Optional import pytest -from tests.integration.output import resolve_output_directory, resolve_output_path -from tests.integration.paths import FTM_OUTPUT_ENV, MODULE_FILENAME - - -@pytest.fixture(scope="session") -def ftm_output_dir() -> Optional[Path]: - """The persistent output directory ``SAMPLETONES_FTM_OUTPUT_DIR`` names.""" - return resolve_output_directory(FTM_OUTPUT_ENV) +from sampletones_tools.samples.famitracker import MODULE_FILENAME @pytest.fixture -def module_path(ftm_output_dir: Optional[Path], tmp_path: Path) -> Path: +def module_path(tmp_path: Path) -> Path: """Where a produced ``.ftm`` is written.""" - return resolve_output_path(ftm_output_dir, tmp_path, MODULE_FILENAME) + return tmp_path / MODULE_FILENAME diff --git a/tests/integration/nsf/conftest.py b/tests/integration/nsf/conftest.py index 1a2567f14..9c166fd90 100644 --- a/tests/integration/nsf/conftest.py +++ b/tests/integration/nsf/conftest.py @@ -1,5 +1,5 @@ from pathlib import Path -from typing import Dict, Final, Optional +from typing import Dict, Final import pytest @@ -8,26 +8,14 @@ from sampletones_player.driver.image import DriverImage from sampletones_player.song import Song from sampletones_shared.paths.extensions import EXT_FILE_NSF -from tests.integration.output import resolve_output_directory, resolve_output_path -from tests.integration.paths import NSF_OUTPUT_ENV EXPORTED_SAMPLE: Final[str] = "lead" -@pytest.fixture(scope="session") -def nsf_output_dir() -> Optional[Path]: - """The persistent output directory ``SAMPLETONES_NSF_OUTPUT_DIR`` names.""" - return resolve_output_directory(NSF_OUTPUT_ENV) - - @pytest.fixture -def nsf_paths( - nsf_output_dir: Optional[Path], - tmp_path: Path, - instrument_catalog: Dict[str, Sample], -) -> Dict[str, Path]: +def nsf_paths(tmp_path: Path, instrument_catalog: Dict[str, Sample]) -> Dict[str, Path]: """Where each sample's produced ``.nsf`` is written.""" - return {name: resolve_output_path(nsf_output_dir, tmp_path, f"{name}{EXT_FILE_NSF}") for name in instrument_catalog} + return {name: tmp_path / f"{name}{EXT_FILE_NSF}" for name in instrument_catalog} @pytest.fixture(scope="session") diff --git a/tests/integration/nsf/console/instructions.py b/tests/integration/nsf/console/instructions.py index c82f8bbf1..0fdb700c3 100644 --- a/tests/integration/nsf/console/instructions.py +++ b/tests/integration/nsf/console/instructions.py @@ -22,7 +22,7 @@ TRIANGLE_COUNTER_CONTROL, TRIANGLE_SOUNDING_RELOAD, ) -from sampletones_player.trace.trace import RegisterTrace +from sampletones_tools.player.trace.trace import RegisterTrace from tests.integration.nsf.console.machine import register_file TRIANGLE_SOUNDING: Final[int] = TRIANGLE_COUNTER_CONTROL | TRIANGLE_SOUNDING_RELOAD diff --git a/tests/integration/nsf/console/machine.py b/tests/integration/nsf/console/machine.py index afa2a1c56..87edc8778 100644 --- a/tests/integration/nsf/console/machine.py +++ b/tests/integration/nsf/console/machine.py @@ -9,8 +9,8 @@ APU_FRAME_COUNTER, FIRST_CHANNEL_REGISTER, ) -from sampletones_player.trace.trace import RegisterTrace -from sampletones_player.trace.write import RegisterWrite +from sampletones_tools.player.trace.trace import RegisterTrace +from sampletones_tools.player.trace.write import RegisterWrite RETURN_SENTINEL: Final[int] = 0xFFF0 STACK_PAGE: Final[int] = 0x0100 diff --git a/tests/integration/nsf/console/session.py b/tests/integration/nsf/console/session.py index 356116ee6..7f64b337c 100644 --- a/tests/integration/nsf/console/session.py +++ b/tests/integration/nsf/console/session.py @@ -4,7 +4,7 @@ from sampletones_player.nsf.file import nsf_to_bytes from sampletones_player.nsf.information import NSFInformation from sampletones_player.song import Song -from sampletones_player.trace.trace import RegisterTrace +from sampletones_tools.player.trace.trace import RegisterTrace from tests.integration.nsf.console.machine import Console TRAILING_CALLS: Final[int] = 2 diff --git a/tests/integration/nsf/exports.py b/tests/integration/nsf/exports.py deleted file mode 100644 index c7041fafb..000000000 --- a/tests/integration/nsf/exports.py +++ /dev/null @@ -1,10 +0,0 @@ -from typing import Final - -from sampletones_player.nsf.information import NSFInformation - -ARTIST: Final[str] = "Integration" - - -def exported_information(name: str) -> NSFInformation: - """The header text an exported sample carries.""" - return NSFInformation(title=name, artist=ARTIST) diff --git a/tests/integration/nsf/header.py b/tests/integration/nsf/header.py new file mode 100644 index 000000000..6f93dbf76 --- /dev/null +++ b/tests/integration/nsf/header.py @@ -0,0 +1,8 @@ +from sampletones_player.nsf.information import NSFInformation +from sampletones_tools.corpus.module import ModuleConfig +from sampletones_tools.samples.nsf import exported_information + + +def sample_information(name: str) -> NSFInformation: + """The header text ``nsf samples`` gives a corpus sample: its name, by the corpus's author.""" + return exported_information(name, ModuleConfig.load().author) diff --git a/tests/integration/nsf/test_backend.py b/tests/integration/nsf/test_backend.py index a080aa36a..6f516b180 100644 --- a/tests/integration/nsf/test_backend.py +++ b/tests/integration/nsf/test_backend.py @@ -1,5 +1,5 @@ from pathlib import Path -from typing import Dict, Final, List, Optional +from typing import Dict, Final, List import numpy as np import pytest @@ -36,7 +36,6 @@ captured_run, play_calls_reaching, ) -from tests.integration.output import resolve_output_path ChannelInstructions = Dict[ChannelName, List[InstructionUnion]] @@ -204,7 +203,6 @@ def project_song(integration_project: Project) -> Song: def project_file( backend: NSFBackend, integration_project: Project, - nsf_output_dir: Optional[Path], tmp_path_factory: pytest.TempPathFactory, ) -> Path: """The whole arrangement written through the backend the application registers. @@ -212,11 +210,7 @@ def project_file( It joins the samples in the emitted artifacts, so the arrangement can be listened to beside the instruments it is built from. """ - destination = resolve_output_path( - nsf_output_dir, - tmp_path_factory.mktemp("nsf-project"), - f"{PROJECT_ARTIFACT}{EXT_FILE_NSF}", - ) + destination = tmp_path_factory.mktemp("nsf-project") / f"{PROJECT_ARTIFACT}{EXT_FILE_NSF}" backend.write_project(destination, ProjectExport(project=integration_project)) return destination diff --git a/tests/integration/nsf/test_compression_report.py b/tests/integration/nsf/test_compression_report.py index a7863419d..ad0d08041 100644 --- a/tests/integration/nsf/test_compression_report.py +++ b/tests/integration/nsf/test_compression_report.py @@ -1,229 +1,43 @@ -from dataclasses import dataclass +import csv from math import ceil -from pathlib import Path -from time import process_time -from typing import Dict, Final, List, Optional, Sequence, Tuple +from typing import Dict, List, Tuple import pytest -from sampletones_core.project.project import Project -from sampletones_core.project.voices.sample import Sample -from sampletones_player.compression.compressed import CompressedPlanes from sampletones_player.compression.decode import decode_planes -from sampletones_player.compression.dictionary.table import phrase_table -from sampletones_player.compression.encode import STREAM_START, encode_planes -from sampletones_player.compression.matches.cache import MatchCache -from sampletones_player.compression.matches.index import PlaneIndex -from sampletones_player.compression.matches.matcher import PhraseMatcher -from sampletones_player.compression.options import CodecOptions -from sampletones_player.compression.parse.plane import parse_plane from sampletones_player.compression.planes.rebuild import streams_from_planes -from sampletones_player.compression.planes.song import SongPlanes from sampletones_player.driver.image import DriverImage -from sampletones_player.registers.streams import ChannelStreams -from sampletones_player.specification.compression import ( - MAX_LITERAL_BYTES, - PLANE_COUNT, - PLANE_STATE_SIZE, -) -from sampletones_player.specification.registers import DUTY_CYCLE_SHIFT +from sampletones_player.specification.compression import MAX_LITERAL_BYTES from sampletones_player.specification.song import SONG_HEADER_SIZE -from tests.integration.nsf.corpus import LONG_ARRANGEMENT, CorpusEntry, build_corpus -from tests.integration.nsf.report import ReportRow, write_csv, write_markdown -from tests.integration.nsf.songs import available_bytes -from tests.integration.output import resolve_output_directory, resolve_output_path -from tests.integration.paths import COMPRESSION_OUTPUT_ENV - -LITERALS: Final[str] = "literals" -HOLDS: Final[str] = "holds" -SEARCH: Final[str] = "search" -RECORDS: Final[str] = "records" -REGISTER_PLANES: Final[str] = "register planes" -SPLIT_CONTROL: Final[str] = "split control" -CONTROL_LEVEL_MASK: Final[int] = 0x3F - -PLANE_VARIANTS: Final[Tuple[Tuple[str, CodecOptions], ...]] = ( - (LITERALS, CodecOptions(holds=False, phrases=False, transposition=False, search=False)), - (HOLDS, CodecOptions(holds=True, phrases=False, transposition=False, search=False)), - ("instruments", CodecOptions(holds=True, phrases=True, transposition=False, search=False)), - ("transposition", CodecOptions(holds=True, phrases=True, transposition=True, search=False)), - (SEARCH, CodecOptions(holds=True, phrases=True, transposition=True, search=True)), +from sampletones_shared.utils.tables import Table +from sampletones_tools.codec.report import session +from sampletones_tools.codec.report.corpus import LONG_ARRANGEMENT +from sampletones_tools.codec.report.encoding import ( + LITERALS, + PLANE_VARIANTS, + RECORDS, + REGISTER_PLANES, + SEARCH, + SPLIT_CONTROL, + Encoding, ) - -CSV_FILENAME: Final[str] = "report.csv" -MARKDOWN_FILENAME: Final[str] = "report.md" - - -@dataclass(frozen=True) -class Encoding: - """One corpus song compressed under one variant of the codec.""" - - entry: CorpusEntry - variant: str - planes: SongPlanes - compressed: CompressedPlanes - seconds: float - - @property - def size(self) -> int: - """The bytes the dictionary, the streams and the pitch table take together.""" - return self.compressed.size + len(self.entry.pitches.data) - - @property - def streams(self) -> int: - """The bytes the token streams take, which is the part that grows with the song.""" - return sum(len(stream) for stream in self.compressed.streams) - - -def _register_planes(streams: ChannelStreams) -> Tuple[bytes, ...]: - return tuple( - bytes(tick.values[register] for tick in stream) - for stream in streams.padded - for register in range(len(stream[0].values)) - ) - - -def _split_control_planes(planes: SongPlanes) -> Tuple[bytes, ...]: - split: List[bytes] = [] - for channels in planes.ordered: - if channels in (planes.pulse1, planes.pulse2): - split.append(bytes(control >> DUTY_CYCLE_SHIFT for control in channels.control)) - split.append(bytes(control & CONTROL_LEVEL_MASK for control in channels.control)) - else: - split.append(channels.control) - - split.append(channels.value) - - return tuple(split) - - -def _coded_size(planes: Sequence[bytes], options: CodecOptions) -> int: - cache = MatchCache(PlaneIndex.from_plane(plane) for plane in planes) - table = phrase_table(()) - entries = frozenset({STREAM_START}) - return sum( - parse_plane( - PhraseMatcher(table, plane, cache), - options, - entries, - ).size - for plane in range(len(planes)) - ) +from sampletones_tools.codec.report.session import CompressionReport, run_report +from sampletones_tools.codec.report.songs import available_bytes +from sampletones_tools.corpus.build import Corpus @pytest.fixture(scope="module") -def corpus( - instrument_catalog: Dict[str, Sample], - integration_project: Project, -) -> Tuple[CorpusEntry, ...]: - """The songs the report measures: each sample alone, and the arrangement at two lengths.""" - return build_corpus(instrument_catalog, integration_project) +def report(synthetic_corpus: Corpus, tmp_path_factory: pytest.TempPathFactory) -> CompressionReport: + """The report ``codec report`` writes, over the session's corpus in place of a build of its own.""" + with pytest.MonkeyPatch.context() as monkeypatch: + monkeypatch.setattr(session, "build_synthetic_corpus", lambda: synthetic_corpus) + return run_report(tmp_path_factory.mktemp("compression_report")) @pytest.fixture(scope="module") -def encodings(corpus: Tuple[CorpusEntry, ...]) -> Tuple[Encoding, ...]: +def encodings(report: CompressionReport) -> Tuple[Encoding, ...]: """Every corpus song compressed under every variant of the codec.""" - encoded: List[Encoding] = [] - for entry in corpus: - planes = entry.planes - for variant, options in PLANE_VARIANTS: - started = process_time() - compressed = encode_planes( - planes, - entry.seeds, - options=options, - boundaries=frozenset(), - ) - encoded.append( - Encoding( - entry=entry, - variant=variant, - planes=planes, - compressed=compressed, - seconds=process_time() - started, - ) - ) - - return tuple(encoded) - - -def _measured_row( - entry: CorpusEntry, - variant: str, - planes: Sequence[bytes], - fixed: int, - space: int, -) -> ReportRow: - _, holds = PLANE_VARIANTS[1] - started = process_time() - coded = _coded_size(planes, holds) - return ReportRow( - corpus=entry.name, - variant=variant, - ticks=entry.song.ticks, - size=fixed + coded, - variable=coded, - phrases=0, - dictionary=0, - seconds=process_time() - started, - records=entry.records, - space=space, - ) - - -def _baseline_rows(entry: CorpusEntry, space: int) -> Tuple[ReportRow, ...]: - return ( - ReportRow( - corpus=entry.name, - variant=RECORDS, - ticks=entry.song.ticks, - size=entry.records, - variable=entry.records, - phrases=0, - dictionary=0, - seconds=0.0, - records=entry.records, - space=space, - ), - _measured_row(entry, REGISTER_PLANES, _register_planes(entry.song.streams), 0, space), - _measured_row( - entry, - SPLIT_CONTROL, - _split_control_planes(entry.planes), - len(entry.pitches.data), - space, - ), - ) - - -def _rows( - corpus: Sequence[CorpusEntry], - encodings: Sequence[Encoding], - space: int, -) -> Tuple[ReportRow, ...]: - rows: List[ReportRow] = [] - for entry in corpus: - rows.extend(_baseline_rows(entry, space)) - for encoding in encodings: - if encoding.entry is not entry: - continue - - rows.append( - ReportRow( - corpus=entry.name, - variant=encoding.variant, - ticks=entry.song.ticks, - size=encoding.size, - variable=encoding.streams, - phrases=len(encoding.compressed.phrases), - dictionary=encoding.compressed.phrases.size, - seconds=encoding.seconds, - records=entry.records, - space=space, - ) - ) - - return tuple(rows) + return report.encodings class TestTheCodecAnswersWithTheSongItWasGiven: @@ -297,24 +111,19 @@ def test_every_layer_undercuts_a_record_per_tick( class TestTheReportStatesWhatEachLayerSaves: """The measurements the format's constants are settled from.""" - def test_the_report_is_written( - self, - corpus: Tuple[CorpusEntry, ...], - encodings: Tuple[Encoding, ...], - driver_image: DriverImage, - compression_output_dir: Optional[Path], - tmp_path: Path, - ) -> None: - rows = _rows(corpus, encodings, available_bytes(driver_image)) - csv_path = resolve_output_path(compression_output_dir, tmp_path, CSV_FILENAME) - markdown_path = resolve_output_path(compression_output_dir, tmp_path, MARKDOWN_FILENAME) - write_csv(rows, csv_path) - write_markdown(rows, markdown_path, PLANE_COUNT * PLANE_STATE_SIZE) - assert csv_path.read_text(encoding="utf-8").count("\n") == len(rows) + 1 - assert markdown_path.exists() - - -@pytest.fixture(scope="session") -def compression_output_dir() -> Optional[Path]: - """The persistent output directory ``SAMPLETONES_COMPRESSION_OUTPUT_DIR`` names.""" - return resolve_output_directory(COMPRESSION_OUTPUT_ENV) + def test_the_report_holds_the_baselines_then_every_variant_per_song(self, report: CompressionReport) -> None: + with report.csv_path.open(encoding="utf-8", newline="") as handle: + header, *rows = list(csv.reader(handle)) + + variants: Dict[str, List[str]] = {} + for row in rows: + variants.setdefault(row[header.index("corpus")], []).append(row[header.index("variant")]) + + assert list(variants) == [entry.name for entry in report.entries] + assert all( + listed == [RECORDS, REGISTER_PLANES, SPLIT_CONTROL, *(name for name, _ in PLANE_VARIANTS)] + for listed in variants.values() + ) + table = Table(columns=tuple(header), rows=tuple(tuple(row) for row in rows)).markdown_lines() + document = report.markdown_path.read_text(encoding="utf-8").splitlines() + assert document[document.index(table[0]) : document.index(table[0]) + len(table)] == table diff --git a/tests/integration/nsf/test_driver_audio.py b/tests/integration/nsf/test_driver_audio.py index 02018b35c..14efacc1c 100644 --- a/tests/integration/nsf/test_driver_audio.py +++ b/tests/integration/nsf/test_driver_audio.py @@ -13,7 +13,7 @@ from sampletones_player.registers.playable import playable from tests.integration.nsf.console.instructions import instructions_from_trace, sounded_approximation from tests.integration.nsf.console.session import captured_trace -from tests.integration.nsf.exports import exported_information +from tests.integration.nsf.header import sample_information ChannelInstructions = Dict[ChannelName, List[InstructionUnion]] @@ -21,7 +21,7 @@ def played_by_console(sample: Sample) -> ChannelInstructions: """The per-tick instructions the console sounds, read back out of the registers it wrote.""" song = song_from_reconstruction(sample.reconstruction, loop_tick=None) - trace = captured_trace(song, exported_information(sample.name)) + trace = captured_trace(song, sample_information(sample.name)) return instructions_from_trace(trace, get_timer_table(sample.reconstruction.config.tuning)) diff --git a/tests/integration/nsf/test_driver_bend.py b/tests/integration/nsf/test_driver_bend.py index 1db9a4803..3e600fc6d 100644 --- a/tests/integration/nsf/test_driver_bend.py +++ b/tests/integration/nsf/test_driver_bend.py @@ -7,11 +7,11 @@ from sampletones_player.song import Song from sampletones_player.specification.binary import BYTE_VALUES from sampletones_player.specification.registers import PULSE1_TIMER_HIGH, TIMER_HIGH_SHIFT -from sampletones_player.trace.trace import RegisterTrace +from sampletones_tools.player.trace.trace import RegisterTrace from tests.integration.nsf.console.instructions import channel_values, timer_value from tests.integration.nsf.console.machine import register_file from tests.integration.nsf.console.session import captured_trace, play_calls_covering -from tests.integration.nsf.exports import exported_information +from tests.integration.nsf.header import sample_information from tests.suite.base import BaseTestSuite from tests.suite.case import BaseAutolabelTestCase from tests.suite.player import PLAYER_PITCHES, bent_song @@ -35,7 +35,7 @@ def sounded_dividers(song: Song) -> List[int]: Returns: List[int]: One divider per tick the song covers, in order. """ - trace = captured_trace(song, exported_information(SONG_NAME)) + trace = captured_trace(song, sample_information(SONG_NAME)) dividers = [] for registers in register_file(trace): values = channel_values(registers, ChannelName.PULSE1) @@ -96,7 +96,7 @@ def test_the_console_sounds_the_divider_the_bend_states(self, test_case: TestCas @pytest.mark.parametrize("test_case", test_cases, ids=lambda case: case.label) def test_the_model_states_the_writes_the_driver_makes(self, test_case: TestCase) -> None: song = bent_song(test_case.pitch_index, test_case.bends, NTSC_RATE) - trace = captured_trace(song, exported_information(SONG_NAME)) + trace = captured_trace(song, sample_information(SONG_NAME)) assert trace == RegisterTrace.from_song(song, play_calls_covering(song)) @@ -127,7 +127,7 @@ def test_the_high_half_is_written_wherever_the_bend_moves_it(self, crossing: Son halves = [(divider + bend) >> TIMER_HIGH_SHIFT for bend in self.BENDS] crossings = sum(1 for earlier, later in zip(halves, halves[1:]) if earlier != later) - trace = captured_trace(crossing, exported_information(SONG_NAME)) + trace = captured_trace(crossing, sample_information(SONG_NAME)) written = [write for writes in trace.play_calls for write in writes if write.address == PULSE1_TIMER_HIGH] assert crossings assert len(written) == crossings diff --git a/tests/integration/nsf/test_driver_trace.py b/tests/integration/nsf/test_driver_trace.py index 8640103d5..abf98d12e 100644 --- a/tests/integration/nsf/test_driver_trace.py +++ b/tests/integration/nsf/test_driver_trace.py @@ -11,7 +11,7 @@ from sampletones_player.specification.binary import WORD_SIZE from sampletones_player.specification.nsf import PROGRAM_SIZE from sampletones_player.specification.song import STEP_FRACTION_OFFSET, STEP_WHOLE_OFFSET -from sampletones_player.trace.trace import RegisterTrace +from sampletones_tools.player.trace.trace import RegisterTrace from tests.integration.nsf.console.session import ( TRAILING_CALLS, captured_trace, @@ -19,7 +19,7 @@ play_calls_covering, play_calls_reaching, ) -from tests.integration.nsf.exports import exported_information +from tests.integration.nsf.header import sample_information from tests.suite.base import BaseTestSuite from tests.suite.case import BaseAutolabelTestCase @@ -33,7 +33,7 @@ @pytest.fixture def trace(song: Song, sample: Sample) -> RegisterTrace: """Every APU write the assembled driver makes over a full run of the sample.""" - return captured_trace(song, exported_information(sample.name)) + return captured_trace(song, sample_information(sample.name)) @pytest.fixture @@ -101,7 +101,7 @@ def test_the_driver_writes_what_the_model_states( reclocked = sample.reconstruction.with_nes_frequency(test_case.expected) song = song_from_reconstruction(reclocked, loop_tick=None) - trace = captured_trace(song, exported_information(sample.name)) + trace = captured_trace(song, sample_information(sample.name)) assert trace == RegisterTrace.from_song(song, play_calls_covering(song)) @pytest.mark.parametrize("test_case", test_cases, ids=lambda case: case.label) @@ -166,7 +166,7 @@ def test_the_driver_writes_what_the_model_states( covered = song.ticks + test_case.expected * (song.ticks - song.loop_tick) calls = play_calls_reaching(song, covered) - trace = captured_trace_over(song, exported_information(sample.name), calls) + trace = captured_trace_over(song, sample_information(sample.name), calls) assert trace == RegisterTrace.from_song(song, calls) @pytest.mark.parametrize("test_case", test_cases, ids=lambda case: case.label) @@ -181,5 +181,5 @@ def test_the_run_keeps_sounding_past_the_songs_end( covered = song.ticks + test_case.expected * (song.ticks - song.loop_tick) calls = play_calls_reaching(song, covered) - trace = captured_trace_over(song, exported_information(sample.name), calls) + trace = captured_trace_over(song, sample_information(sample.name), calls) assert any(writes for writes in trace.play_calls[-TRAILING_CALLS:]) diff --git a/tests/integration/nsf/test_nsf_pipeline.py b/tests/integration/nsf/test_nsf_pipeline.py index fda31e4cf..98d739f48 100644 --- a/tests/integration/nsf/test_nsf_pipeline.py +++ b/tests/integration/nsf/test_nsf_pipeline.py @@ -29,7 +29,7 @@ TOTAL_TICKS_OFFSET, ) from sampletones_shared.paths.extensions import EXT_FILE_RECONSTRUCTION -from tests.integration.nsf.exports import exported_information +from tests.integration.nsf.header import sample_information def song_block(data: bytes, image: DriverImage) -> bytes: @@ -52,7 +52,7 @@ def exported( driver_image: DriverImage, ) -> bytes: destination = nsf_paths[sample.name] - write_nsf(destination, song, exported_information(sample.name), driver_image) + write_nsf(destination, song, sample_information(sample.name), driver_image) return destination.read_bytes() @@ -67,7 +67,7 @@ def test_every_sample_reaches_a_playable_file( ) -> None: for name, sample in instrument_catalog.items(): song = song_from_reconstruction(sample.reconstruction, loop_tick=None) - write_nsf(nsf_paths[name], song, exported_information(name), driver_image) + write_nsf(nsf_paths[name], song, sample_information(name), driver_image) assert nsf_paths[name].read_bytes()[: len(NSF_MAGIC)] == NSF_MAGIC def test_the_file_carries_the_shipped_driver(self, exported: bytes, driver_image: DriverImage) -> None: @@ -148,7 +148,7 @@ def test_the_round_trip_leaves_the_exported_bytes_alone( sample.reconstruction.save(stored) reloaded = song_from_reconstruction(Reconstruction.load(stored), loop_tick=None) - information = exported_information(sample.name) + information = sample_information(sample.name) before = tmp_path / "before.nsf" after = tmp_path / "after.nsf" write_nsf(before, song, information, driver_image) diff --git a/tests/integration/nsf/test_song_export.py b/tests/integration/nsf/test_song_export.py index 7f566958c..09d4e78d1 100644 --- a/tests/integration/nsf/test_song_export.py +++ b/tests/integration/nsf/test_song_export.py @@ -13,7 +13,7 @@ TOTAL_TICKS_OFFSET, ) from sampletones_shared.exceptions import SongTooLargeError -from tests.integration.nsf.songs import RECORD_BYTES_PER_TICK, available_bytes +from sampletones_tools.codec.report.songs import RECORD_BYTES_PER_TICK, available_bytes def read_word(data: bytes, offset: int) -> int: diff --git a/tests/integration/output.py b/tests/integration/output.py deleted file mode 100644 index 423b89d30..000000000 --- a/tests/integration/output.py +++ /dev/null @@ -1,49 +0,0 @@ -import os -import shutil -from pathlib import Path -from typing import Optional - -from tests.integration.paths import REPO_ROOT - - -def resolve_output_directory(variable: str) -> Optional[Path]: - """Reads the persistent output directory an environment variable names. - - Emission is opt-in so an ordinary (and parallel) run writes only to ``tmp_path``. - A named directory is cleaned once per session, so each run leaves a fresh set of - files there. - - Args: - variable: Environment variable naming the directory. - - Returns: - Optional[Path]: The prepared directory, or ``None`` while emission is unasked for. - """ - configured = os.environ.get(variable) - if not configured: - return None - - directory = Path(configured) - if not directory.is_absolute(): - directory = REPO_ROOT / directory - - if directory.exists(): - shutil.rmtree(directory) - - directory.mkdir(parents=True, exist_ok=True) - return directory - - -def resolve_output_path(output_directory: Optional[Path], tmp_path: Path, filename: str) -> Path: - """Locates a produced file: the persistent directory where one is named, else ``tmp_path``. - - Args: - output_directory: The persistent directory, or ``None`` while emission is unasked for. - tmp_path: The test's own temporary directory. - filename: Name the produced file carries. - - Returns: - Path: Where the file is written. - """ - base = output_directory if output_directory is not None else tmp_path - return base / filename diff --git a/tests/integration/paths.py b/tests/integration/paths.py deleted file mode 100644 index c27997891..000000000 --- a/tests/integration/paths.py +++ /dev/null @@ -1,30 +0,0 @@ -from pathlib import Path -from typing import Final - - -def _repo_root() -> Path: - for parent in Path(__file__).resolve().parents: - if (parent / "pyproject.toml").exists(): - return parent - - raise RuntimeError("Could not locate the repository root") - - -REPO_ROOT: Final[Path] = _repo_root() -INTEGRATION_ROOT: Final[Path] = Path(__file__).parent - -CONFIG_DIRECTORY: Final[Path] = INTEGRATION_ROOT / "config" -SYNTH_CONFIG_PATH: Final[Path] = CONFIG_DIRECTORY / "synth.yaml" -RECONSTRUCTION_CONFIG_PATH: Final[Path] = CONFIG_DIRECTORY / "reconstruction.yaml" -MODULE_CONFIG_PATH: Final[Path] = CONFIG_DIRECTORY / "module.yaml" -SONG_PATH: Final[Path] = CONFIG_DIRECTORY / "song.yaml" - -MODULE_FILENAME: Final[str] = "drums.ftm" -FTM_OUTPUT_ENV: Final[str] = "SAMPLETONES_FTM_OUTPUT_DIR" - -DOCUMENT_FILENAME: Final[str] = "drums.btp" -GROOVE_DOCUMENT_FILENAME: Final[str] = "drums-groove.btp" -BTP_OUTPUT_ENV: Final[str] = "SAMPLETONES_BTP_OUTPUT_DIR" - -NSF_OUTPUT_ENV: Final[str] = "SAMPLETONES_NSF_OUTPUT_DIR" -COMPRESSION_OUTPUT_ENV: Final[str] = "SAMPLETONES_COMPRESSION_OUTPUT_DIR" diff --git a/tests/integration/reconstruction/test_conversion_jobs.py b/tests/integration/reconstruction/test_conversion_jobs.py index 488621ee4..672505513 100644 --- a/tests/integration/reconstruction/test_conversion_jobs.py +++ b/tests/integration/reconstruction/test_conversion_jobs.py @@ -21,14 +21,14 @@ from sampletones_core.reconstructions.stage import ReconstructionStage from sampletones_shared.exceptions import OperationCanceled from sampletones_shared.utils.progress import silent_reporter -from tests.integration.assets.reconstruction import ( +from sampletones_tools.corpus.catalog import build_mini_library +from tests.suite.progress import FIRST_REPORT, RecordingReporter, reported_stages +from tests.suite.stems import ( THREE_STEM_CHANNELS, - build_mini_library, three_stem_config, three_stem_reconstruction_config, write_three_stem_recordings, ) -from tests.suite.progress import FIRST_REPORT, RecordingReporter, reported_stages def _writing_to(config: Config, directory: Path) -> Config: diff --git a/tests/integration/reconstruction/test_decoding.py b/tests/integration/reconstruction/test_decoding.py index cda43bd2a..66b110c6b 100644 --- a/tests/integration/reconstruction/test_decoding.py +++ b/tests/integration/reconstruction/test_decoding.py @@ -13,7 +13,7 @@ ) from sampletones_core.instructions import InstructionUnion from sampletones_core.reconstructions import Reconstruction, Reconstructor -from tests.integration.assets.reconstruction import build_mini_library +from sampletones_tools.corpus.catalog import build_mini_library _DURATION_SECONDS: Final[float] = 1.0 _LOWER_TONE: Final[float] = 440.0 diff --git a/tests/integration/reconstruction/test_stems_reconstruction.py b/tests/integration/reconstruction/test_stems_reconstruction.py index ef8df05ba..ac927745f 100644 --- a/tests/integration/reconstruction/test_stems_reconstruction.py +++ b/tests/integration/reconstruction/test_stems_reconstruction.py @@ -21,13 +21,13 @@ from sampletones_core.reconstructions.reconstructor.stems.configs.entry import StemEntry from sampletones_core.reconstructions.reconstructor.stems.configs.hierarchy import StemsHierarchy from sampletones_core.reconstructions.reconstructor.stems.configs.settings import StemSettings -from tests.integration.assets.reconstruction import ( +from sampletones_tools.corpus.catalog import build_mini_library +from tests.suite.stems import ( STEM_A_ID, STEM_B_ID, STEM_C_ID, STEM_RECORDING_DURATION_SECONDS, THREE_STEM_CHANNELS, - build_mini_library, three_stem_config, three_stem_reconstruction_config, write_three_stem_recordings, diff --git a/tests/integration/samples/__init__.py b/tests/integration/samples/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/integration/samples/test_emitters.py b/tests/integration/samples/test_emitters.py new file mode 100644 index 000000000..99e8bc6f4 --- /dev/null +++ b/tests/integration/samples/test_emitters.py @@ -0,0 +1,53 @@ +from pathlib import Path + +import pytest + +from sampletones_core.formats.bitphase.specification.channels import CHANNEL_LABELS +from sampletones_player.specification.nsf import ARTIST_OFFSET, NSF_MAGIC, STRING_FIELD_SIZE +from sampletones_shared.paths.extensions import EXT_FILE_NSF +from sampletones_tools.corpus.build import Corpus +from sampletones_tools.samples import bitphase, emit, famitracker, nsf +from sampletones_tools.samples.emit import emit_samples +from tests.suite.bitphase import parse_btp +from tests.suite.famitracker import parse_ftm + + +def _artist(data: bytes) -> str: + return data[ARTIST_OFFSET : ARTIST_OFFSET + STRING_FIELD_SIZE].split(b"\x00", 1)[0].decode("utf-8") + + +@pytest.fixture(name="corpus") +def corpus_fixture(monkeypatch: pytest.MonkeyPatch, synthetic_corpus: Corpus) -> Corpus: + """The session's corpus, which the samples commands hand their emitter in place of a build of their own.""" + monkeypatch.setattr(emit, "build_synthetic_corpus", lambda: synthetic_corpus) + return synthetic_corpus + + +class TestTheEmittersWriteTheCorpus: + """Each format's emitter writes the files a player of that format opens.""" + + def test_the_nsf_emitter_writes_every_sample_then_the_arrangement(self, corpus: Corpus, tmp_path: Path) -> None: + written = emit_samples(tmp_path, nsf.write_samples) + + assert [path.name for path in written] == [ + *(f"{name}{EXT_FILE_NSF}" for name in corpus.catalog), + f"{nsf.SONG_NAME}{EXT_FILE_NSF}", + ] + assert all(path.read_bytes()[: len(NSF_MAGIC)] == NSF_MAGIC for path in written) + assert all(_artist(path.read_bytes()) == corpus.project.info.author for path in written) + + def test_the_famitracker_emitter_writes_one_module(self, corpus: Corpus, tmp_path: Path) -> None: + written = emit_samples(tmp_path, famitracker.write_samples) + + assert written == [tmp_path / famitracker.MODULE_FILENAME] + assert parse_ftm(written[0].read_bytes()).instruments + + def test_the_bitphase_emitter_writes_the_song_at_its_tempo_and_as_a_groove( + self, + corpus: Corpus, + tmp_path: Path, + ) -> None: + written = emit_samples(tmp_path, bitphase.write_samples) + + assert written == [tmp_path / bitphase.DOCUMENT_FILENAME, tmp_path / bitphase.GROOVE_DOCUMENT_FILENAME] + assert all(parse_btp(path.read_bytes(), list(CHANNEL_LABELS)).songs for path in written) diff --git a/tests/integration/tooling/test_check_commands.py b/tests/integration/tooling/test_check_commands.py index a265cea4b..69f756c6f 100644 --- a/tests/integration/tooling/test_check_commands.py +++ b/tests/integration/tooling/test_check_commands.py @@ -1,69 +1,53 @@ from pathlib import Path -from typing import Dict, Final, List, Optional +from typing import Dict, Final, List import yaml +from sampletones.commands.registry import COMMANDS +from sampletones.dispatcher import build_parser from sampletones_shared.paths.source import REPOSITORY_ROOT +from sampletones_tools.checks.registry import GATES PRE_COMMIT_CONFIG: Final[Path] = REPOSITORY_ROOT / ".pre-commit-config.yaml" -MAKEFILE: Final[Path] = REPOSITORY_ROOT / "Makefile" FILE_ENCODING: Final[str] = "utf-8" LOCAL_REPOSITORY: Final[str] = "local" -CHECK_SCRIPTS: Final[str] = "scripts/checks/" -TARGET_PREFIX: Final[str] = "check-" -SCRIPT_SUFFIX: Final[str] = ".py" -RECIPE_PREFIX: Final[str] = "\t" -TARGET_SUFFIX: Final[str] = ":" +ENTRY_PREFIX: Final[str] = "uv run sampletones " +CHECK_PREFIX: Final[str] = f"{ENTRY_PREFIX}check " def check_hooks() -> List[Dict[str, object]]: - """Every local hook running one of the check scripts, as the configuration declares it.""" + """Every local hook running one of the checks, as the configuration declares it.""" config = yaml.safe_load(PRE_COMMIT_CONFIG.read_text(encoding=FILE_ENCODING)) return [ hook for repository in config["repos"] if repository["repo"] == LOCAL_REPOSITORY for hook in repository["hooks"] - if CHECK_SCRIPTS in str(hook["entry"]) + if str(hook["entry"]).startswith(CHECK_PREFIX) ] -def check_targets() -> Dict[str, str]: - """The command each `check-*` target of the Makefile runs, keyed by target name.""" - targets: Dict[str, str] = {} - target: Optional[str] = None - for line in MAKEFILE.read_text(encoding=FILE_ENCODING).splitlines(): - if line.startswith(TARGET_PREFIX) and line.endswith(TARGET_SUFFIX): - target = line.removesuffix(TARGET_SUFFIX) - elif target is not None and line.startswith(RECIPE_PREFIX): - targets[target] = line.strip() - target = None - - return targets - - -def script_path(entry: str) -> Path: - """The check script an entry runs, taken from the words the entry is written with.""" - return REPOSITORY_ROOT / next(word for word in entry.split() if word.endswith(SCRIPT_SUFFIX)) +def hook_gate(hook: Dict[str, object]) -> str: + """The check a hook runs, taken from the words the entry is written with.""" + return str(hook["entry"]).removeprefix(CHECK_PREFIX).split()[0] class TestCheckHooks: - def test_the_configuration_declares_a_hook_for_every_check_script(self) -> None: - scripts = {path.name for path in (REPOSITORY_ROOT / CHECK_SCRIPTS).glob(f"*{SCRIPT_SUFFIX}")} + def test_the_configuration_declares_a_hook_for_every_check(self) -> None: + hooks = check_hooks() + + assert GATES + assert {hook_gate(hook) for hook in hooks} == {gate.name for gate in GATES} + assert len(hooks) == len(GATES) - assert {script_path(str(hook["entry"])).name for hook in check_hooks()} == scripts + def test_every_check_hook_parses_as_the_entry_runs_it(self) -> None: + """A hook's entry is a command line the entry accepts, so a renamed option fails here.""" + parser = build_parser(COMMANDS) - def test_every_check_hook_names_a_script_that_is_there(self) -> None: - assert all(script_path(str(hook["entry"])).is_file() for hook in check_hooks()) + for hook in check_hooks(): + parser.parse_args(str(hook["entry"]).removeprefix(ENTRY_PREFIX).split()) def test_every_check_hook_sweeps_the_whole_tree(self) -> None: """A hook handed the staged files checks the staged subset, which passes what it never reads.""" assert all(hook["pass_filenames"] is False for hook in check_hooks()) - - -class TestCheckTargets: - def test_each_hook_and_its_make_target_run_the_same_command(self) -> None: - commands = {f"{TARGET_PREFIX}{hook['id']}": str(hook["entry"]) for hook in check_hooks()} - - assert check_targets() == commands diff --git a/tests/integration/tooling/test_codec_study.py b/tests/integration/tooling/test_codec_study.py new file mode 100644 index 000000000..add93f6cf --- /dev/null +++ b/tests/integration/tooling/test_codec_study.py @@ -0,0 +1,75 @@ +import csv +from pathlib import Path +from typing import Final, List + +import pytest + +from sampletones_core.project.container import ProjectContainer +from sampletones_core.project.project import Project +from sampletones_shared.exceptions.project import NotAValidArchiveError +from sampletones_shared.utils.tables import Table +from sampletones_tools.codec.study.manifest import StudyManifest, StudySource +from sampletones_tools.codec.study.plan import StudyPlan, plan_study +from sampletones_tools.codec.study.report import rows as songs +from sampletones_tools.codec.study.report import verdicts +from sampletones_tools.codec.study.report.run import MANIFEST_JSON, REPORT_CSV, REPORT_MARKDOWN, VERDICTS_CSV +from sampletones_tools.codec.study.session import run_study + +VARIANT: Final[str] = "wide-hold" +LENGTHEN_SECONDS: Final[int] = 2 + + +def _read_csv(path: Path) -> List[List[str]]: + with path.open(newline="", encoding="utf-8") as handle: + return list(csv.reader(handle)) + + +@pytest.fixture(name="plan") +def plan_fixture(integration_project: Project, tmp_path: Path) -> StudyPlan: + project = tmp_path / "synthetic.stp" + ProjectContainer.save(integration_project, project) + return plan_study( + StudyManifest( + projects=(StudySource.at(project),), + reconstructions=(), + lengthen_seconds=LENGTHEN_SECONDS, + variants=(VARIANT,), + ) + ) + + +class TestStudyRun: + def test_a_run_writes_its_tables_and_the_manifest_that_repeats_it(self, plan: StudyPlan, tmp_path: Path) -> None: + directory = run_study(plan, tmp_path / "run") + + assert StudyManifest.load(directory / MANIFEST_JSON) == plan.manifest + + header, *measured = _read_csv(directory / REPORT_CSV) + assert tuple(header) == songs.COLUMNS + assert {row[header.index("variant")] for row in measured} >= {variant.name for variant in plan.variants} + + verdict_header, *verdict_rows = _read_csv(directory / VERDICTS_CSV) + assert tuple(verdict_header) == verdicts.COLUMNS + + markdown = (directory / REPORT_MARKDOWN).read_text(encoding="utf-8").split("\n") + table = Table(columns=verdicts.COLUMNS, rows=tuple(tuple(row) for row in verdict_rows)).markdown_lines() + start = markdown.index(table[0]) + assert markdown[start : start + len(table)] == table + + def test_a_source_that_fails_to_read_leaves_the_output_untouched(self, tmp_path: Path) -> None: + broken = tmp_path / "broken.stp" + broken.write_bytes(b"") + output = tmp_path / "run" + plan = plan_study( + StudyManifest( + projects=(StudySource.at(broken),), + reconstructions=(), + lengthen_seconds=LENGTHEN_SECONDS, + variants=(VARIANT,), + ) + ) + + with pytest.raises(NotAValidArchiveError): + run_study(plan, output) + + assert not output.exists() diff --git a/tests/suite/bootstrap.py b/tests/suite/bootstrap.py new file mode 100644 index 000000000..bb7ff5258 --- /dev/null +++ b/tests/suite/bootstrap.py @@ -0,0 +1,104 @@ +from dataclasses import dataclass +from pathlib import Path +from typing import Callable, Dict, Final, List, Mapping, Optional, Sequence, Tuple + +from bootstrap.layout import PROJECT_FILE +from bootstrap.project import BUILD_EXTRA, DEVELOPMENT_GROUP, GPU_CUDA11_EXTRA, GPU_EXTRA + +PROJECT_NAME: Final[str] = "sampletones" +PROJECT_VERSION: Final[str] = "0.3.0" +PROJECT_DOCUMENT: Final[str] = f""" +[project] +name = "{PROJECT_NAME}" +version = "{PROJECT_VERSION}" + +[project.scripts] +{PROJECT_NAME} = "{PROJECT_NAME}.__main__:main" + +[project.optional-dependencies] +{BUILD_EXTRA} = ["pyinstaller"] +{GPU_EXTRA} = ["cupy-cuda12x"] +{GPU_CUDA11_EXTRA} = ["cupy-cuda11x"] + +[dependency-groups] +{DEVELOPMENT_GROUP} = ["pytest"] + +[tool.hatch.build.targets.wheel] +packages = ["src/{PROJECT_NAME}", "src/{PROJECT_NAME}_core"] +""" + + +@dataclass(frozen=True) +class RecordedCommand: + """One command a recording runner was asked to run. + + Attributes: + command: The program and its arguments. + cwd: The directory it was to run in. + environment: The variables it was to see. + quiet: Whether its output was to be captured. + """ + + command: Tuple[str, ...] + cwd: Path + environment: Dict[str, str] + quiet: bool + + @property + def line(self) -> str: + """The command as one line.""" + return " ".join(self.command) + + +class RecordingRunner: + """A runner that records every command and answers with the status a test assigned it. + + A status is assigned by a fragment of the command line, so a probe such as ``import pyaudio`` + can be made to fail while everything else succeeds. A callback runs on every command, which + lets a test leave behind the files a real command would have written. + """ + + def __init__( + self, + statuses: Mapping[str, int], + on_run: Optional[Callable[[Sequence[str]], None]], + ) -> None: + self.commands: List[RecordedCommand] = [] + self._statuses = dict(statuses) + self._on_run = on_run + + def __call__( + self, + command: Sequence[str], + *, + cwd: Path, + environment: Mapping[str, str], + quiet: bool, + ) -> int: + recorded = RecordedCommand( + command=tuple(command), + cwd=cwd, + environment=dict(environment), + quiet=quiet, + ) + self.commands.append(recorded) + if self._on_run is not None: + self._on_run(command) + + for fragment, status in self._statuses.items(): + if fragment in recorded.line: + return status + + return 0 + + @property + def lines(self) -> List[str]: + """Every recorded command as one line, in the order run.""" + return [recorded.line for recorded in self.commands] + + +def write_project(root: Path) -> Path: + """Writes a ``pyproject.toml`` under ``root`` stating what the scripts read, and answers with ``root``.""" + root.mkdir(parents=True, exist_ok=True) + (root / PROJECT_FILE).write_text(PROJECT_DOCUMENT, encoding="utf-8") + return root diff --git a/tests/suite/commands.py b/tests/suite/commands.py new file mode 100644 index 000000000..c10ba206a --- /dev/null +++ b/tests/suite/commands.py @@ -0,0 +1,26 @@ +from pathlib import Path +from typing import Dict, List, Optional + + +class RecordedApplication: + """A stand-in for the application launcher that records what each start was given.""" + + def __init__(self) -> None: + self.starts: List[Dict[str, Optional[Path]]] = [] + + def __call__( + self, + config_path: Optional[Path] = None, + *, + library_path: Optional[Path] = None, + reconstruction_path: Optional[Path] = None, + project_path: Optional[Path] = None, + ) -> None: + self.starts.append( + { + "config": config_path, + "library": library_path, + "reconstruction": reconstruction_path, + "project": project_path, + } + ) diff --git a/tests/suite/files.py b/tests/suite/files.py new file mode 100644 index 000000000..e90e20d9c --- /dev/null +++ b/tests/suite/files.py @@ -0,0 +1,9 @@ +from pathlib import Path + + +def empty_file(directory: Path, name: str) -> Path: + """Writes an empty file called ``name`` under ``directory``, which a check that reads only names and presence accepts.""" + path = directory / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(b"") + return path diff --git a/tests/suite/scripts.py b/tests/suite/scripts.py index 413fa9d7a..26e6393ee 100644 --- a/tests/suite/scripts.py +++ b/tests/suite/scripts.py @@ -1,7 +1,7 @@ import importlib.util from types import ModuleType -from sampletones_shared.paths.source import REPOSITORY_ROOT +from sampletones_tools.checks.paths import SCRIPTS_ROOT def load_script(relative_path: str) -> ModuleType: @@ -10,7 +10,7 @@ def load_script(relative_path: str) -> ModuleType: The scripts under ``scripts/`` are entry points invoked by path from workflows, hooks and the Makefile, so importing them the same way keeps a test exercising the module the tooling runs. """ - path = REPOSITORY_ROOT / "scripts" / relative_path + path = SCRIPTS_ROOT / relative_path spec = importlib.util.spec_from_file_location(path.stem, path) assert spec is not None and spec.loader is not None diff --git a/tests/suite/source.py b/tests/suite/source.py index 86a038f75..60f9078d3 100644 --- a/tests/suite/source.py +++ b/tests/suite/source.py @@ -3,8 +3,8 @@ from textwrap import dedent from typing import Iterable, Set -from sampletones_shared.meta.source.bindings.scopes import Scope -from sampletones_shared.meta.source.modules import source_paths +from sampletones_tools.checks.source.bindings.scopes import Scope +from sampletones_tools.checks.source.modules import source_paths def parse_source(source: str) -> ast.Module: diff --git a/tests/suite/stems.py b/tests/suite/stems.py index 3dd8ba7dc..8bbcff6e1 100644 --- a/tests/suite/stems.py +++ b/tests/suite/stems.py @@ -1,10 +1,37 @@ -from typing import List, Mapping, Sequence +from pathlib import Path +from typing import Dict, Final, List, Mapping, Sequence, Tuple +import numpy as np + +from sampletones_core.audio import write_wave +from sampletones_core.configs import Config from sampletones_core.constants.algorithm import DEFAULT_STEMS_CHANNEL_CAP -from sampletones_core.constants.enums import ChannelName, bending_channels +from sampletones_core.constants.enums import ChannelName, HierarchyMode, bending_channels from sampletones_core.instructions import InstructionUnion from sampletones_core.reconstructions.reconstruction.stems.channel_assignment import ChannelAssignment from sampletones_core.reconstructions.reconstruction.stems.data import StemsData +from sampletones_core.reconstructions.reconstructor.stems.configs.config import StemsConfig +from sampletones_core.reconstructions.reconstructor.stems.configs.entry import StemEntry +from sampletones_core.reconstructions.reconstructor.stems.configs.hierarchy import StemsHierarchy +from sampletones_core.reconstructions.reconstructor.stems.configs.settings import StemSettings +from sampletones_shared.types.path import Pathlike + +STEM_A_ID: Final[int] = 0 +STEM_B_ID: Final[int] = 1 +STEM_C_ID: Final[int] = 2 +THREE_STEM_CHANNELS: Final[List[ChannelName]] = [ + ChannelName.PULSE1, + ChannelName.PULSE2, + ChannelName.TRIANGLE, + ChannelName.NOISE, +] +THREE_STEM_ENTRY_CHANNELS: Final[Dict[int, List[ChannelName]]] = { + STEM_A_ID: [ChannelName.PULSE1, ChannelName.TRIANGLE, ChannelName.NOISE], + STEM_B_ID: [ChannelName.PULSE2, ChannelName.TRIANGLE], + STEM_C_ID: [ChannelName.PULSE1, ChannelName.NOISE], +} +STEM_RECORDING_DURATION_SECONDS: Final[float] = 0.5 +RECORDING_SEED: Final[int] = 93 def single_entry_stems_data( @@ -22,3 +49,50 @@ def single_entry_stems_data( bending_channels(channels), assignments, ) + + +def three_stem_config() -> StemsConfig: + """Builds the three-stem setup the stems tests share. + + Stems a (pulse 1, triangle, noise) and b (pulse 2, triangle) pick on the first + hierarchy level, stem c (pulse 1, noise) on the second. + """ + return StemsConfig( + entries=[ + StemEntry(id=stem_id, settings=StemSettings(channels=channels, bends=bending_channels(channels))) + for stem_id, channels in THREE_STEM_ENTRY_CHANNELS.items() + ], + hierarchy=StemsHierarchy( + levels=[[STEM_A_ID, STEM_B_ID], [STEM_C_ID]], + mode=HierarchyMode.STRICT, + ), + channel_cap=1, + ) + + +def three_stem_reconstruction_config() -> Config: + """Builds a reconstruction config for the three-stem example.""" + return Config() + + +def write_three_stem_recordings( + config: Config, + tmp_dir: Pathlike, +) -> Tuple[Path, Path, Path]: + """Writes three distinct stem recordings a, b, c and returns their paths in order.""" + sample_rate = config.library.sample_rate + count = int(sample_rate * STEM_RECORDING_DURATION_SECONDS) + time = np.arange(count) / sample_rate + recordings = { + "a": 0.5 * np.sin(2 * np.pi * 440.0 * time), + "b": 0.4 * np.sin(2 * np.pi * 220.0 * time), + "c": np.random.default_rng(RECORDING_SEED).uniform(-0.3, 0.3, count), + } + + paths: List[Path] = [] + for name, audio in recordings.items(): + path = Path(tmp_dir) / f"stem_{name}.wav" + write_wave(path, sample_rate, audio) + paths.append(path) + + return paths[0], paths[1], paths[2] diff --git a/tests/unit/sampletones/commands/test_convert.py b/tests/unit/sampletones/commands/test_convert.py new file mode 100644 index 000000000..c98d8509e --- /dev/null +++ b/tests/unit/sampletones/commands/test_convert.py @@ -0,0 +1,147 @@ +import json +from pathlib import Path +from typing import List + +import pytest + +from sampletones.commands.registry import COMMANDS +from sampletones.dispatcher import dispatch +from sampletones_core.configs import Config +from sampletones_core.constants.enums import DEFAULT_CHANNELS, ChannelName +from sampletones_core.headless.conversion.request import ConversionRequest, classic_setup +from sampletones_core.reconstructions.reconstructor.stems.configs.config import StemsConfig +from sampletones_core.reconstructions.reconstructor.stems.configs.entry import StemEntry +from sampletones_core.reconstructions.reconstructor.stems.configs.hierarchy import StemsHierarchy +from sampletones_core.reconstructions.reconstructor.stems.configs.settings import StemSettings +from tests.suite.files import empty_file + +RECONSTRUCTION = "sampletones_core.headless.conversion.runners.reconstruct" +LOADER = "sampletones_core.headless.config.load_config" + + +class RecordedReconstruction: + def __init__(self) -> None: + self.requests: List[ConversionRequest] = [] + + def __call__(self, request: ConversionRequest, config: Config) -> None: + del config + self.requests.append(request) + + +@pytest.fixture(name="reconstruction") +def reconstruction_fixture(monkeypatch: pytest.MonkeyPatch) -> RecordedReconstruction: + recorded = RecordedReconstruction() + monkeypatch.setattr(RECONSTRUCTION, recorded) + monkeypatch.setattr(LOADER, lambda path: Config()) + return recorded + + +def _two_stems() -> StemsConfig: + return StemsConfig( + entries=[ + StemEntry( + id=0, + settings=StemSettings(channels=[ChannelName.PULSE1, ChannelName.PULSE2], bends=[ChannelName.PULSE1]), + ), + StemEntry( + id=1, + settings=StemSettings(channels=[ChannelName.TRIANGLE], bends=[ChannelName.TRIANGLE]), + ), + ], + hierarchy=StemsHierarchy(levels=[[0], [1]]), + ) + + +class TestConvert: + def test_the_channels_named_become_one_stem(self, reconstruction: RecordedReconstruction, tmp_path: Path) -> None: + source = empty_file(tmp_path, "song.wav") + output = tmp_path / "song.stn" + + status = dispatch(COMMANDS, ["convert", str(source), "--channels", "pulse1,pulse2", "-o", str(output)]) + + assert status == 0 + request = reconstruction.requests[0] + assert request.sources == (source,) + assert request.stems == classic_setup([ChannelName.PULSE1, ChannelName.PULSE2]) + assert request.output_path == output + + def test_without_channels_the_usual_three_are_used( + self, + reconstruction: RecordedReconstruction, + tmp_path: Path, + ) -> None: + source = empty_file(tmp_path, "song.wav") + + assert dispatch(COMMANDS, ["convert", str(source)]) == 0 + assert reconstruction.requests[0].stems == classic_setup(DEFAULT_CHANNELS) + assert reconstruction.requests[0].output_path is None + + def test_a_stems_file_pairs_its_entries_with_the_sources_in_order( + self, + reconstruction: RecordedReconstruction, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + ) -> None: + bass = empty_file(tmp_path, "bass.wav") + lead = empty_file(tmp_path, "lead.wav") + stems = _two_stems() + setup = tmp_path / "stems.json" + setup.write_text(json.dumps(stems.model_dump(mode="json")), encoding="utf-8") + + assert dispatch(COMMANDS, ["convert", str(bass), str(lead), "--stems", str(setup)]) == 0 + assert reconstruction.requests[0].stems == stems + printed = capsys.readouterr().out + assert "bass.wav: stem 0 on pulse1, pulse2, bending pulse1" in printed + assert "lead.wav: stem 1 on triangle, bending triangle" in printed + + def test_a_setup_pairing_wrong_is_refused(self, reconstruction: RecordedReconstruction, tmp_path: Path) -> None: + source = empty_file(tmp_path, "song.wav") + setup = tmp_path / "stems.json" + setup.write_text(json.dumps(_two_stems().model_dump(mode="json")), encoding="utf-8") + + with pytest.raises(SystemExit, match="1 sources for 2 stems"): + dispatch(COMMANDS, ["convert", str(source), "--stems", str(setup)]) + + assert reconstruction.requests == [] + + def test_an_unknown_channel_is_refused(self, reconstruction: RecordedReconstruction, tmp_path: Path) -> None: + source = empty_file(tmp_path, "song.wav") + + with pytest.raises(SystemExit, match="Unknown channel 'pulse3'"): + dispatch(COMMANDS, ["convert", str(source), "--channels", "pulse3"]) + + assert reconstruction.requests == [] + + def test_a_missing_source_is_refused_in_one_line( + self, reconstruction: RecordedReconstruction, tmp_path: Path + ) -> None: + with pytest.raises(SystemExit) as leaving: + dispatch(COMMANDS, ["convert", str(tmp_path / "absent.wav")]) + + assert str(leaving.value) == f"No file at {tmp_path / 'absent.wav'}." + assert reconstruction.requests == [] + + def test_a_project_is_refused_as_no_recording(self, reconstruction: RecordedReconstruction, tmp_path: Path) -> None: + project = empty_file(tmp_path, "song.stp") + + with pytest.raises(SystemExit, match="is no recording") as leaving: + dispatch(COMMANDS, ["convert", str(project)]) + + assert len(str(leaving.value).splitlines()) == 1 + assert reconstruction.requests == [] + + def test_a_missing_stems_file_is_refused(self, reconstruction: RecordedReconstruction, tmp_path: Path) -> None: + source = empty_file(tmp_path, "song.wav") + + with pytest.raises(SystemExit, match="No stems file at"): + dispatch(COMMANDS, ["convert", str(source), "--stems", str(tmp_path / "absent.json")]) + + assert reconstruction.requests == [] + + def test_channels_and_stems_exclude_each_other(self, tmp_path: Path) -> None: + source = empty_file(tmp_path, "song.wav") + + with pytest.raises(SystemExit) as leaving: + dispatch(COMMANDS, ["convert", str(source), "--channels", "pulse1", "--stems", "stems.json"]) + + assert leaving.value.code == 2 diff --git a/tests/unit/sampletones/commands/test_library.py b/tests/unit/sampletones/commands/test_library.py new file mode 100644 index 000000000..02fbd2088 --- /dev/null +++ b/tests/unit/sampletones/commands/test_library.py @@ -0,0 +1,29 @@ +from pathlib import Path +from typing import List + +import pytest + +from sampletones.commands.registry import COMMANDS +from sampletones.dispatcher import dispatch +from sampletones_core.configs import Config + +GENERATOR = "sampletones_core.headless.library.generate_library" +LOADER = "sampletones_core.headless.config.load_config" + + +class TestLibrary: + def test_the_library_is_generated_for_the_configuration_named(self, monkeypatch: pytest.MonkeyPatch) -> None: + generated: List[Config] = [] + loaded: List[Path] = [] + configuration = Config() + + def load_config(path: Path) -> Config: + loaded.append(path) + return configuration + + monkeypatch.setattr(GENERATOR, generated.append) + monkeypatch.setattr(LOADER, load_config) + + assert dispatch(COMMANDS, ["library", "--config", "custom.json"]) == 0 + assert loaded == [Path("custom.json")] + assert generated == [configuration] diff --git a/tests/unit/sampletones/commands/test_open.py b/tests/unit/sampletones/commands/test_open.py new file mode 100644 index 000000000..48569d622 --- /dev/null +++ b/tests/unit/sampletones/commands/test_open.py @@ -0,0 +1,85 @@ +import json +import re +import subprocess +import sys +from pathlib import Path +from typing import Final, List, Tuple + +import pytest + +from sampletones.commands.registry import COMMANDS +from sampletones.dispatcher import dispatch +from tests.suite.commands import RecordedApplication +from tests.suite.files import empty_file + +LAUNCHER = "sampletones.run.run_application" +ENGINE_MODULES: Final[Tuple[str, ...]] = ("numpy", "scipy", "sampletones_core.reconstructions") +REFUSAL_PROBE: Final[str] = """ +import json, sys +from sampletones.commands.registry import COMMANDS +from sampletones.dispatcher import dispatch +try: + dispatch(COMMANDS, ["open", sys.argv[1]]) +except SystemExit: + print(json.dumps(sorted(sys.modules))) +""" + + +class TestOpen: + @pytest.mark.parametrize( + ("name", "field"), + [("song.stp", "project"), ("song.stn", "reconstruction"), ("library.ins", "library")], + ) + def test_a_file_is_loaded_by_its_kind( + self, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + name: str, + field: str, + ) -> None: + application = RecordedApplication() + monkeypatch.setattr(LAUNCHER, application) + path = empty_file(tmp_path, name) + + assert dispatch(COMMANDS, ["open", str(path), "--config", "custom.json"]) == 0 + start = application.starts[0] + assert start[field] == path + assert start["config"] == Path("custom.json") + assert [key for key, value in start.items() if value is None] == [ + key for key in ("library", "reconstruction", "project") if key != field + ] + + def test_a_recording_is_pointed_at_convert(self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + application = RecordedApplication() + monkeypatch.setattr(LAUNCHER, application) + path = empty_file(tmp_path, "song.wav") + + with pytest.raises(SystemExit, match=re.escape(f"sampletones convert {path}")): + dispatch(COMMANDS, ["open", str(path)]) + + assert application.starts == [] + + def test_a_file_of_another_kind_is_refused(self, tmp_path: Path) -> None: + path = empty_file(tmp_path, "notes.txt") + + with pytest.raises(SystemExit, match="neither"): + dispatch(COMMANDS, ["open", str(path)]) + + def test_a_missing_file_is_refused(self, tmp_path: Path) -> None: + with pytest.raises(SystemExit, match="No file at"): + dispatch(COMMANDS, ["open", str(tmp_path / "absent.stp")]) + + +class TestRefusingARecording: + """Pointing a recording at convert is a check on its name, which a fresh interpreter makes without the engine.""" + + def test_the_refusal_loads_no_engine(self, tmp_path: Path) -> None: + completed = subprocess.run( + [sys.executable, "-c", REFUSAL_PROBE, str(empty_file(tmp_path, "song.wav"))], + check=True, + capture_output=True, + text=True, + ) + + modules: List[str] = json.loads(completed.stdout) + assert set(modules) & set(ENGINE_MODULES) == set() diff --git a/tests/unit/sampletones/commands/test_registry.py b/tests/unit/sampletones/commands/test_registry.py new file mode 100644 index 000000000..c112f5557 --- /dev/null +++ b/tests/unit/sampletones/commands/test_registry.py @@ -0,0 +1,79 @@ +import json +import subprocess +import sys +from importlib.util import find_spec +from typing import Final, List, Tuple + +import pytest + +from sampletones.commands.registry import COMMANDS, USER_COMMANDS +from sampletones.dispatcher import DEFAULT_COMMAND, build_parser +from sampletones_tools.registry import DEVELOPER_COMMANDS + +TOOLS_PACKAGE: Final[str] = "sampletones_tools" +FACE_MODULES: Final[Tuple[str, ...]] = ("command", "registry") +FACE_PACKAGE: Final[str] = "commands" +HEAVY_MODULES: Final[Tuple[str, ...]] = ( + "numpy", + "scipy", + "librosa", + "cupy", + "PIL", + "py65", + "pytest", + "dearpygui", + "sampletones_shared.paths.user", +) +PROBE: Final[str] = "import json, sys, sampletones.commands.registry; print(json.dumps(sorted(sys.modules)))" + + +def _is_face(module: str) -> bool: + """Whether a tools module is one a command list reads: a package initializer, a command, a + registry, or a module under ``commands``.""" + spec = find_spec(module) + parts = module.split(".") + return ( + (spec is not None and spec.submodule_search_locations is not None) + or parts[-1] in FACE_MODULES + or FACE_PACKAGE in parts[:-1] + ) + + +@pytest.fixture(scope="module") +def loaded_modules() -> List[str]: + """Every module a fresh interpreter holds once it lists the commands.""" + completed = subprocess.run( + [sys.executable, "-c", PROBE], + check=True, + capture_output=True, + text=True, + ) + modules: List[str] = json.loads(completed.stdout) + return modules + + +class TestRegistry: + def test_the_user_commands_are_the_ones_the_guide_names(self) -> None: + assert [command.name for command in USER_COMMANDS] == ["run", "open", "convert", "library", "self-check"] + + def test_the_commands_are_the_user_ones_followed_by_the_developer_ones(self) -> None: + assert COMMANDS == (*USER_COMMANDS, *DEVELOPER_COMMANDS) + + def test_the_default_command_is_registered(self) -> None: + assert DEFAULT_COMMAND in {command.name for command in COMMANDS} + + def test_the_registry_builds_one_parser(self) -> None: + assert build_parser(COMMANDS).format_help() + + +class TestListingTheCommands: + """A module a tool needs, loaded with the command list, would slow and could break every invocation, the GUI included.""" + + def test_only_the_faces_of_the_tools_are_loaded(self, loaded_modules: List[str]) -> None: + tools = [module for module in loaded_modules if module.startswith(f"{TOOLS_PACKAGE}.")] + + assert tools + assert [module for module in tools if not _is_face(module)] == [] + + def test_no_heavy_module_is_loaded(self, loaded_modules: List[str]) -> None: + assert set(loaded_modules) & set(HEAVY_MODULES) == set() diff --git a/tests/unit/sampletones/commands/test_run.py b/tests/unit/sampletones/commands/test_run.py new file mode 100644 index 000000000..72a9f347d --- /dev/null +++ b/tests/unit/sampletones/commands/test_run.py @@ -0,0 +1,28 @@ +from pathlib import Path + +import pytest + +from sampletones.commands.registry import COMMANDS +from sampletones.dispatcher import dispatch +from tests.suite.commands import RecordedApplication + +LAUNCHER = "sampletones.run.run_application" + + +class TestRun: + def test_no_arguments_start_the_application_with_the_saved_configuration( + self, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + application = RecordedApplication() + monkeypatch.setattr(LAUNCHER, application) + + assert dispatch(COMMANDS, []) == 0 + assert application.starts == [{"config": None, "library": None, "reconstruction": None, "project": None}] + + def test_a_configuration_reaches_the_application(self, monkeypatch: pytest.MonkeyPatch) -> None: + application = RecordedApplication() + monkeypatch.setattr(LAUNCHER, application) + + assert dispatch(COMMANDS, ["run", "--config", "custom.json"]) == 0 + assert application.starts[0]["config"] == Path("custom.json") diff --git a/tests/unit/sampletones/commands/test_self_check.py b/tests/unit/sampletones/commands/test_self_check.py new file mode 100644 index 000000000..2748135e4 --- /dev/null +++ b/tests/unit/sampletones/commands/test_self_check.py @@ -0,0 +1,13 @@ +import pytest + +from sampletones.commands.registry import COMMANDS +from sampletones.dispatcher import dispatch + +CHECK = "sampletones.self_check.run_self_check" + + +class TestSelfCheck: + def test_the_status_is_the_checks_own(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(CHECK, lambda: 7) + + assert dispatch(COMMANDS, ["self-check"]) == 7 diff --git a/tests/unit/sampletones/test_dispatcher.py b/tests/unit/sampletones/test_dispatcher.py new file mode 100644 index 000000000..0a55f6eb3 --- /dev/null +++ b/tests/unit/sampletones/test_dispatcher.py @@ -0,0 +1,69 @@ +from argparse import ArgumentParser, Namespace +from typing import List, Tuple + +import pytest + +from sampletones.dispatcher import COMMAND_METAVAR, DEFAULT_COMMAND, PROGRAM, build_parser, dispatch +from sampletones_shared.application import SAMPLETONES_NAME_VERSION +from sampletones_shared.command import Command + + +def _command(name: str, calls: List[Tuple[str, bool]]) -> Command: + def configure(parser: ArgumentParser) -> None: + parser.add_argument("--flag", action="store_true") + + def run(arguments: Namespace) -> int: + calls.append((name, arguments.flag)) + return 3 + + return Command(name=name, help=f"{name} help", configure=configure, run=run) + + +class TestBuildParser: + def test_two_commands_sharing_a_name_are_refused(self) -> None: + calls: List[Tuple[str, bool]] = [] + + with pytest.raises(ValueError, match="share a name: twin"): + build_parser((_command("twin", calls), _command("twin", calls))) + + def test_the_version_is_a_flag_of_the_entry(self, capsys: pytest.CaptureFixture[str]) -> None: + parser = build_parser((_command(DEFAULT_COMMAND, []),)) + + with pytest.raises(SystemExit) as leaving: + parser.parse_args(["--version"]) + + assert leaving.value.code == 0 + assert SAMPLETONES_NAME_VERSION in capsys.readouterr().out + + def test_the_listing_names_the_line_a_developer_command_runs_from_a_checkout(self) -> None: + listing = " ".join(build_parser((_command("first", []),)).format_help().split()) + + assert f"uv run {PROGRAM} {COMMAND_METAVAR}" in listing + + def test_every_command_is_listed_with_its_help(self) -> None: + parser = build_parser((_command("first", []), _command("second", []))) + + listing = parser.format_help() + + assert "first help" in listing + assert "second help" in listing + + +class TestDispatch: + def test_no_arguments_run_the_default_command(self) -> None: + calls: List[Tuple[str, bool]] = [] + + assert dispatch((_command(DEFAULT_COMMAND, calls), _command("other", calls)), []) == 3 + assert calls == [(DEFAULT_COMMAND, False)] + + def test_a_named_command_runs_with_its_arguments(self) -> None: + calls: List[Tuple[str, bool]] = [] + + assert dispatch((_command(DEFAULT_COMMAND, calls), _command("other", calls)), ["other", "--flag"]) == 3 + assert calls == [("other", True)] + + def test_an_unknown_command_is_refused(self) -> None: + with pytest.raises(SystemExit) as leaving: + dispatch((_command(DEFAULT_COMMAND, []),), ["absent"]) + + assert leaving.value.code == 2 diff --git a/tests/unit/sampletones/test_self_check.py b/tests/unit/sampletones/test_self_check.py new file mode 100644 index 000000000..ac2476f7d --- /dev/null +++ b/tests/unit/sampletones/test_self_check.py @@ -0,0 +1,40 @@ +from unittest.mock import patch + +import pytest + +from sampletones.self_check import CHECKS, FAILURE_STATUS, SUCCESS_STATUS, run_self_check +from sampletones_shared.exceptions import FileDialogUnavailableError + +RESOURCES_MODULE = "sampletones_application.ui.resources.resources" +SELECTION_MODULE = "sampletones_application.utils.file_dialogs.selection" + + +class TestRunSelfCheck: + def test_source_checkout_passes(self, capsys: pytest.CaptureFixture[str]) -> None: + status = run_self_check() + output = capsys.readouterr().out + + assert status == SUCCESS_STATUS + for check in CHECKS: + assert check.name in output + + def test_missing_resource_fails(self, capsys: pytest.CaptureFixture[str]) -> None: + with patch(f"{RESOURCES_MODULE}.get_font_path", side_effect=FileNotFoundError("Resource not found")): + status = run_self_check() + + captured = capsys.readouterr() + + assert status == FAILURE_STATUS + assert "resources" in captured.err + assert "FileNotFoundError" in captured.err + assert "file dialog backend" not in captured.out + + def test_unavailable_dialog_backend_fails(self, capsys: pytest.CaptureFixture[str]) -> None: + with patch( + f"{SELECTION_MODULE}.select_file_dialog_backend", + side_effect=FileDialogUnavailableError("No file dialog backend is available."), + ): + status = run_self_check() + + assert status == FAILURE_STATUS + assert "file dialog backend" in capsys.readouterr().err diff --git a/tests/unit/sampletones_application/ui/elements/layout/test_centered.py b/tests/unit/sampletones_application/ui/elements/layout/test_centered.py new file mode 100644 index 000000000..5f68d5f7a --- /dev/null +++ b/tests/unit/sampletones_application/ui/elements/layout/test_centered.py @@ -0,0 +1,52 @@ +from typing import Final, Iterator, List + +import dearpygui.dearpygui as dpg +import pytest + +from sampletones_application.ui.elements.layout.centered import centered + +CONTENT_TAG: Final[str] = "test.centered.text.content" +COLUMN_SLOT: Final[int] = 0 +CELL_SLOT: Final[int] = 1 +MIDDLE: Final[int] = 1 + + +@pytest.fixture(name="dpg_context") +def dpg_context_fixture() -> Iterator[None]: + dpg.create_context() + try: + yield + finally: + dpg.destroy_context() + + +def build() -> int: + """Builds one centered item and returns the table standing around it.""" + with dpg.window(): + with centered(): + dpg.add_text("content", tag=CONTENT_TAG) + + group = dpg.get_item_parent(CONTENT_TAG) + row = dpg.get_item_parent(group) + return dpg.get_item_parent(row) + + +class TestCentered: + """``centered`` stands the block's content in a fitted column between two equal ones.""" + + def test_the_content_sits_in_the_middle_cell(self, dpg_context: None) -> None: + table = build() + row = dpg.get_item_children(table, CELL_SLOT)[0] + cells: List[int] = dpg.get_item_children(row, CELL_SLOT) + assert cells.index(dpg.get_item_parent(CONTENT_TAG)) == MIDDLE + + def test_the_middle_column_fits_its_content(self, dpg_context: None) -> None: + columns: List[int] = dpg.get_item_children(build(), COLUMN_SLOT) + assert dpg.get_item_configuration(columns[MIDDLE])["width_fixed"] + + def test_the_flanking_columns_share_the_rest_equally(self, dpg_context: None) -> None: + columns: List[int] = dpg.get_item_children(build(), COLUMN_SLOT) + left = dpg.get_item_configuration(columns[0]) + right = dpg.get_item_configuration(columns[-1]) + assert left["width_stretch"] and right["width_stretch"] + assert left["init_width_or_weight"] == right["init_width_or_weight"] diff --git a/tests/unit/sampletones_application/ui/panels/dialogs/test_export.py b/tests/unit/sampletones_application/ui/panels/dialogs/test_export.py index b6c2ef49c..6789c261e 100644 --- a/tests/unit/sampletones_application/ui/panels/dialogs/test_export.py +++ b/tests/unit/sampletones_application/ui/panels/dialogs/test_export.py @@ -116,6 +116,11 @@ def test_a_stage_without_an_end_shows_what_it_holds(self, window: GUIExportWindo def test_the_figure_reaches_the_reader(self, window: GUIExportWindow) -> None: render(window, traveling=False, figure=SIZE) assert dpg.get_value(TAG_SETTINGS_EXPORT_TEXT_FIGURE) == SIZE + assert shown(TAG_SETTINGS_EXPORT_TEXT_FIGURE) + + def test_a_stage_with_nothing_to_state_leaves_the_indicator_alone(self, window: GUIExportWindow) -> None: + render(window, traveling=False, figure="") + assert not shown(TAG_SETTINGS_EXPORT_TEXT_FIGURE) class TestStoppingARun: diff --git a/tests/unit/sampletones_application/view_model/shared/test_export.py b/tests/unit/sampletones_application/view_model/shared/test_export.py index f744e4825..d239e86d0 100644 --- a/tests/unit/sampletones_application/view_model/shared/test_export.py +++ b/tests/unit/sampletones_application/view_model/shared/test_export.py @@ -75,6 +75,12 @@ def test_a_traveling_stage_hides_the_turning_symbol(self) -> None: def test_a_stage_without_an_end_shows_the_turning_symbol(self) -> None: assert view_model(traveling=False, figure=SIZE).working_visible is True + def test_a_stage_with_words_to_state_shows_its_figure(self) -> None: + assert view_model(traveling=False, figure=SIZE).figure_visible is True + + def test_a_stage_with_nothing_to_state_leaves_the_indicator_alone(self) -> None: + assert view_model(traveling=False, figure="").figure_visible is False + def test_a_bar_is_labeled_with_the_share_it_has_covered(self) -> None: assert view_model(progress=HALFWAY).progress_overlay == "50%" diff --git a/tests/unit/sampletones_core/headless/conversion/stems.py b/tests/unit/sampletones_core/headless/conversion/stems.py new file mode 100644 index 000000000..895d652d7 --- /dev/null +++ b/tests/unit/sampletones_core/headless/conversion/stems.py @@ -0,0 +1,21 @@ +from sampletones_core.constants.enums import ChannelName +from sampletones_core.reconstructions.reconstructor.stems.configs.config import StemsConfig +from sampletones_core.reconstructions.reconstructor.stems.configs.entry import StemEntry +from sampletones_core.reconstructions.reconstructor.stems.configs.hierarchy import StemsHierarchy +from sampletones_core.reconstructions.reconstructor.stems.configs.settings import StemSettings + + +def two_stems() -> StemsConfig: + return StemsConfig( + entries=[ + StemEntry( + id=0, + settings=StemSettings(channels=[ChannelName.PULSE1, ChannelName.PULSE2], bends=[ChannelName.PULSE1]), + ), + StemEntry( + id=1, + settings=StemSettings(channels=[ChannelName.NOISE], bends=[]), + ), + ], + hierarchy=StemsHierarchy(levels=[[0, 1]]), + ) diff --git a/tests/unit/sampletones_core/headless/conversion/test_pairing.py b/tests/unit/sampletones_core/headless/conversion/test_pairing.py new file mode 100644 index 000000000..77e13facb --- /dev/null +++ b/tests/unit/sampletones_core/headless/conversion/test_pairing.py @@ -0,0 +1,34 @@ +from pathlib import Path + +from sampletones_core.constants.enums import DEFAULT_CHANNELS +from sampletones_core.headless.conversion.pairing import describe_stem, pairing_lines +from sampletones_core.headless.conversion.request import ConversionRequest, classic_setup +from tests.suite.files import empty_file +from tests.unit.sampletones_core.headless.conversion.stems import two_stems + + +class TestDescribeStem: + def test_a_stem_names_its_channels_and_its_bends(self) -> None: + first, second = two_stems().entries + + assert describe_stem(first) == "stem 0 on pulse1, pulse2, bending pulse1" + assert describe_stem(second) == "stem 1 on noise" + + +class TestPairingLines: + def test_recordings_pair_with_the_entries_in_order(self, tmp_path: Path) -> None: + bass = empty_file(tmp_path, "bass.wav") + drums = empty_file(tmp_path, "drums.wav") + + request = ConversionRequest(sources=(bass, drums), stems=two_stems(), output_path=None) + + assert pairing_lines(request) == [ + "bass.wav: stem 0 on pulse1, pulse2, bending pulse1", + "drums.wav: stem 1 on noise", + ] + + def test_a_directory_names_the_one_stem_every_recording_plays_under(self, tmp_path: Path) -> None: + setup = classic_setup(DEFAULT_CHANNELS) + request = ConversionRequest(sources=(tmp_path,), stems=setup, output_path=None) + + assert pairing_lines(request) == [f"{tmp_path.name}/: every recording under {describe_stem(setup.entries[0])}"] diff --git a/tests/unit/sampletones_core/headless/conversion/test_request.py b/tests/unit/sampletones_core/headless/conversion/test_request.py new file mode 100644 index 000000000..968cbbdfe --- /dev/null +++ b/tests/unit/sampletones_core/headless/conversion/test_request.py @@ -0,0 +1,127 @@ +import json +from pathlib import Path + +import pytest + +from sampletones_core.constants.enums import DEFAULT_CHANNELS, ChannelName +from sampletones_core.headless.conversion.request import ( + ConversionRequest, + channels_named, + classic_setup, + load_stems, +) +from tests.suite.files import empty_file +from tests.unit.sampletones_core.headless.conversion.stems import two_stems + + +class TestChannelsNamed: + def test_nothing_named_is_the_usual_three(self) -> None: + assert channels_named(None) == list(DEFAULT_CHANNELS) + + def test_names_are_read_in_order_with_their_spaces_stripped(self) -> None: + assert channels_named("noise, pulse1") == [ChannelName.NOISE, ChannelName.PULSE1] + + def test_an_unknown_name_is_refused_with_the_known_ones(self) -> None: + with pytest.raises(ValueError, match="Unknown channel 'pulse3'; the channels are pulse1, pulse2"): + channels_named("pulse1,pulse3") + + +class TestClassicSetup: + def test_one_stem_holds_the_channels_in_order_and_bends_the_toned_ones(self) -> None: + setup = classic_setup([ChannelName.NOISE, ChannelName.PULSE1]) + + assert len(setup.entries) == 1 + assert setup.entries[0].settings.channels == [ChannelName.PULSE1, ChannelName.NOISE] + assert setup.entries[0].settings.bends == [ChannelName.PULSE1] + + +class TestLoadStems: + def test_a_setup_written_as_json_reads_back(self, tmp_path: Path) -> None: + stems = two_stems() + path = tmp_path / "stems.json" + path.write_text(json.dumps(stems.model_dump(mode="json")), encoding="utf-8") + + assert load_stems(path) == stems + + def test_a_file_holding_no_mapping_is_refused(self, tmp_path: Path) -> None: + path = tmp_path / "stems.json" + path.write_text("[]", encoding="utf-8") + + with pytest.raises(ValueError, match="must hold a mapping"): + load_stems(path) + + def test_a_missing_file_is_refused_by_its_path(self, tmp_path: Path) -> None: + with pytest.raises(ValueError, match="No stems file at"): + load_stems(tmp_path / "absent.json") + + def test_a_mapping_that_is_no_setup_is_refused(self, tmp_path: Path) -> None: + path = tmp_path / "stems.json" + path.write_text(json.dumps({"entries": [{"id": 0, "settings": {"channels": ["pulse9"], "bends": []}}]})) + + with pytest.raises(ValueError): + load_stems(path) + + +class TestConversionRequest: + def test_recordings_name_no_directory(self, tmp_path: Path) -> None: + bass = empty_file(tmp_path, "bass.wav") + drums = empty_file(tmp_path, "drums.wav") + + request = ConversionRequest(sources=(bass, drums), stems=two_stems(), output_path=None) + + assert request.directory is None + assert request.sources == (bass, drums) + + def test_a_directory_is_converted_file_by_file_under_one_stem(self, tmp_path: Path) -> None: + request = ConversionRequest(sources=(tmp_path,), stems=classic_setup(DEFAULT_CHANNELS), output_path=None) + + assert request.directory == tmp_path + + def test_the_sources_are_classified_once_when_the_request_is_made(self, tmp_path: Path) -> None: + directory = tmp_path / "recordings" + directory.mkdir() + request = ConversionRequest(sources=(directory,), stems=classic_setup(DEFAULT_CHANNELS), output_path=None) + + directory.rmdir() + + assert request.directory == directory + + def test_a_count_mismatch_is_refused(self, tmp_path: Path) -> None: + with pytest.raises(ValueError, match="1 sources for 2 stems"): + ConversionRequest(sources=(empty_file(tmp_path, "bass.wav"),), stems=two_stems(), output_path=None) + + def test_a_directory_under_several_stems_is_refused(self, tmp_path: Path) -> None: + with pytest.raises(ValueError, match="under one stem; the setup holds 2"): + ConversionRequest(sources=(tmp_path,), stems=two_stems(), output_path=None) + + def test_a_directory_among_recordings_is_refused(self, tmp_path: Path) -> None: + with pytest.raises(ValueError, match="one directory alone"): + ConversionRequest(sources=(empty_file(tmp_path, "bass.wav"), tmp_path), stems=two_stems(), output_path=None) + + def test_an_output_path_for_a_directory_is_refused(self, tmp_path: Path) -> None: + with pytest.raises(ValueError, match="an output path names the one file"): + ConversionRequest( + sources=(tmp_path,), + stems=classic_setup(DEFAULT_CHANNELS), + output_path=tmp_path / "out.stn", + ) + + def test_a_missing_source_is_refused_by_its_path(self, tmp_path: Path) -> None: + with pytest.raises(ValueError, match="No file at"): + ConversionRequest( + sources=(tmp_path / "absent.wav",), + stems=classic_setup(DEFAULT_CHANNELS), + output_path=None, + ) + + def test_a_file_other_than_a_recording_is_refused(self, tmp_path: Path) -> None: + with pytest.raises(ValueError, match="is no recording"): + ConversionRequest( + sources=(empty_file(tmp_path, "song.stp"),), + stems=classic_setup(DEFAULT_CHANNELS), + output_path=None, + ) + + def test_no_source_is_refused(self) -> None: + with pytest.raises(ValueError): + ConversionRequest(sources=(), stems=classic_setup(DEFAULT_CHANNELS), output_path=None) diff --git a/tests/unit/sampletones_core/headless/conversion/test_runners.py b/tests/unit/sampletones_core/headless/conversion/test_runners.py new file mode 100644 index 000000000..e78e719b5 --- /dev/null +++ b/tests/unit/sampletones_core/headless/conversion/test_runners.py @@ -0,0 +1,47 @@ +from pathlib import Path +from typing import List, Tuple + +import pytest + +from sampletones_core.configs import Config +from sampletones_core.constants.enums import DEFAULT_CHANNELS +from sampletones_core.headless.conversion import runners +from sampletones_core.headless.conversion.request import ConversionRequest, classic_setup +from sampletones_core.headless.conversion.runners import reconstruct +from sampletones_core.reconstructions.reconstructor.stems.configs.config import StemsConfig +from tests.suite.files import empty_file + + +class TestReconstruct: + def test_recordings_are_mixed_into_one_file(self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + calls: List[Tuple[Tuple[Path, ...], Path]] = [] + + def reconstruct_sources( + sources: Tuple[Path, ...], config: Config, stems: StemsConfig, output_path: Path + ) -> None: + del config, stems + calls.append((sources, output_path)) + + monkeypatch.setattr(runners, "reconstruct_sources", reconstruct_sources) + source = empty_file(tmp_path, "song.wav") + request = ConversionRequest( + sources=(source,), stems=classic_setup(DEFAULT_CHANNELS), output_path=tmp_path / "x.stn" + ) + + reconstruct(request, Config()) + + assert calls == [((source,), tmp_path / "x.stn")] + + def test_a_directory_is_walked(self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + walked: List[Path] = [] + + def reconstruct_directory(directory: Path, config: Config, stems: StemsConfig) -> None: + del config, stems + walked.append(directory) + + monkeypatch.setattr(runners, "reconstruct_directory", reconstruct_directory) + request = ConversionRequest(sources=(tmp_path,), stems=classic_setup(DEFAULT_CHANNELS), output_path=None) + + reconstruct(request, Config()) + + assert walked == [tmp_path] diff --git a/tests/unit/sampletones_core/headless/test_config.py b/tests/unit/sampletones_core/headless/test_config.py new file mode 100644 index 000000000..f37588181 --- /dev/null +++ b/tests/unit/sampletones_core/headless/test_config.py @@ -0,0 +1,15 @@ +from pathlib import Path + +from sampletones_core.configs import Config +from sampletones_core.headless.config import load_config + + +class TestLoadConfig: + def test_a_file_named_is_read(self, tmp_path: Path) -> None: + path = tmp_path / "config.json" + path.write_text("{}", encoding="utf-8") + + assert load_config(path) == Config() + + def test_nothing_named_is_the_saved_configuration(self) -> None: + assert load_config(None) == Config.default() diff --git a/tests/unit/sampletones_player/compression/test_budget.py b/tests/unit/sampletones_player/compression/test_budget.py new file mode 100644 index 000000000..e39a02db0 --- /dev/null +++ b/tests/unit/sampletones_player/compression/test_budget.py @@ -0,0 +1,61 @@ +from dataclasses import dataclass +from typing import Tuple + +import pytest +from pydantic import ValidationError + +from sampletones_player.compression.budget import DEFAULT_SEARCH_BUDGET, SearchBudget, shares +from tests.suite.base import BaseTestSuite +from tests.suite.case import BaseRegularTestCase + + +class TestSharesMeetSmallDemandsAndSplitTheRest(BaseTestSuite): + """A claimant asking for little is served whole; the larger ones split what remains evenly.""" + + @dataclass(frozen=True, kw_only=True) + class TestCase(BaseRegularTestCase): + demands: Tuple[int, ...] + total: int + expected: Tuple[int, ...] + + test_cases = ( + TestCase(label="every_demand_fits", demands=(5, 5, 5), total=30, expected=(5, 5, 5)), + TestCase(label="a_small_demand_leaves_its_surplus", demands=(100, 1, 100), total=41, expected=(20, 1, 20)), + TestCase(label="a_zero_demand_takes_nothing", demands=(0, 7), total=3, expected=(0, 3)), + TestCase(label="equal_demands_split_evenly", demands=(9, 9), total=8, expected=(4, 4)), + TestCase(label="a_total_short_of_one_each", demands=(9, 9, 9), total=2, expected=(0, 1, 1)), + TestCase(label="no_claimants", demands=(), total=10, expected=()), + ) + + @pytest.mark.parametrize( + "test_case", + test_cases, + ids=lambda test_case: test_case.label, + ) + def test_shares(self, test_case: TestCase) -> None: + assert shares(test_case.demands, test_case.total) == test_case.expected + + def test_the_total_is_never_exceeded(self) -> None: + demands = (30, 70, 10, 90) + for total in range(0, 220, 7): + assert sum(shares(demands, total)) <= total + + +class TestTheBudgetBoundsEveryStep: + """Each figure is at least what one round can act on, and the default states them all.""" + + def test_the_default_gathers_ranks_and_confirms(self) -> None: + assert DEFAULT_SEARCH_BUDGET.candidate_entries >= 1 + assert DEFAULT_SEARCH_BUDGET.rounds >= 1 + assert DEFAULT_SEARCH_BUDGET.confirmed_candidates >= 1 + + def test_a_round_gathers_at_least_one_entry(self) -> None: + with pytest.raises(ValidationError): + SearchBudget(candidate_entries=0, rounds=1, confirmed_candidates=1) + + def test_a_round_confirms_at_least_one_candidate(self) -> None: + with pytest.raises(ValidationError): + SearchBudget(candidate_entries=1, rounds=1, confirmed_candidates=0) + + def test_no_rounds_is_a_budget(self) -> None: + assert SearchBudget(candidate_entries=1, rounds=0, confirmed_candidates=1).rounds == 0 diff --git a/tests/unit/sampletones_player/compression/test_search.py b/tests/unit/sampletones_player/compression/test_search.py new file mode 100644 index 000000000..7a8d7ccab --- /dev/null +++ b/tests/unit/sampletones_player/compression/test_search.py @@ -0,0 +1,71 @@ +from random import Random +from typing import Final, FrozenSet + +import pytest + +from sampletones_player.compression.budget import SearchBudget +from sampletones_player.compression.dictionary.table import PhraseTable, phrase_table +from sampletones_player.compression.matches.cache import MatchCache +from sampletones_player.compression.matches.index import PlaneIndex +from sampletones_player.compression.options import EVERY_LAYER +from sampletones_player.compression.progress.monitor import CodecMonitor +from sampletones_player.compression.search import search_phrases +from sampletones_player.specification.binary import BYTE_VALUES +from sampletones_shared.utils.progress import silent_reporter + +STREAM_START: Final[FrozenSet[int]] = frozenset({0}) +FIGURE: Final[bytes] = b"\x10\x18\x14\x22\x1c\x30\x11\x19\x15\x23\x1d\x31\x12\x1a\x16\x24" +FIGURE_REPEATS: Final[int] = 6 +CROWDED_TICKS: Final[int] = 600 +CROWDED_SEED: Final[int] = 20260912 +REPEATING: Final[bytes] = FIGURE * FIGURE_REPEATS +NARROW_ENTRIES: Final[int] = 8_000 + + +def _crowded() -> bytes: + """A plane of unrelated values, which offers many candidates and repeats none of them.""" + random = Random(CROWDED_SEED) + return bytes(random.randrange(BYTE_VALUES) for _ in range(CROWDED_TICKS)) + + +def _searched(budget: SearchBudget) -> PhraseTable: + cache = MatchCache([PlaneIndex.from_plane(_crowded()), PlaneIndex.from_plane(REPEATING)]) + return search_phrases( + cache, + phrase_table(()), + EVERY_LAYER, + STREAM_START, + CodecMonitor(silent_reporter), + budget, + ) + + +@pytest.fixture(name="narrow") +def narrow_fixture() -> SearchBudget: + """A round too small for the crowded plane alone, and ample for the repeating one.""" + return SearchBudget(candidate_entries=NARROW_ENTRIES, rounds=64, confirmed_candidates=3) + + +class TestEveryPlaneIsHeardFrom: + """A crowded plane takes its share of a round and leaves the rest to the planes after it.""" + + def test_a_plane_after_a_crowded_one_earns_its_phrase(self, narrow: SearchBudget) -> None: + table = _searched(narrow) + assert any(phrase.body in REPEATING for phrase in table.phrases) + + def test_the_crowded_plane_earns_nothing(self, narrow: SearchBudget) -> None: + crowded = _crowded() + table = _searched(narrow) + assert all(phrase.body not in crowded for phrase in table.phrases) + + +class TestTheRoundsBoundWhatTheSearchEarns: + """Each round adds at most one phrase, so the rounds cap the dictionary the search fills.""" + + def test_no_rounds_leaves_the_table_as_seeded(self) -> None: + table = _searched(SearchBudget(candidate_entries=NARROW_ENTRIES, rounds=0, confirmed_candidates=3)) + assert len(table) == 0 + + def test_one_round_adds_one_phrase(self) -> None: + table = _searched(SearchBudget(candidate_entries=NARROW_ENTRIES, rounds=1, confirmed_candidates=3)) + assert len(table) == 1 diff --git a/tests/unit/sampletones_shared/paths/test_package.py b/tests/unit/sampletones_shared/paths/test_package.py new file mode 100644 index 000000000..07658997f --- /dev/null +++ b/tests/unit/sampletones_shared/paths/test_package.py @@ -0,0 +1,37 @@ +from pathlib import Path + +import pytest + +import sampletones_config +import sampletones_tools.corpus.config +from sampletones_shared.paths.package import package_directory + + +class TestPackageDirectory: + def test_the_directory_holds_the_package_s_modules_and_data(self) -> None: + directory = package_directory("sampletones_config") + + assert directory == Path(sampletones_config.__file__).parent + assert (directory / "application").is_dir() + + def test_a_nested_package_is_placed_by_its_own_location(self) -> None: + assert ( + package_directory("sampletones_tools.corpus.config") + == Path(sampletones_tools.corpus.config.__file__).parent + ) + + def test_a_directory_of_data_alone_is_placed_as_a_namespace_package( + self, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + ) -> None: + """A bundle unpacks a package the application never imports as code this way.""" + (tmp_path / "unpacked_data").mkdir() + (tmp_path / "unpacked_data" / "values.yaml").write_text("") + monkeypatch.syspath_prepend(str(tmp_path)) + + assert package_directory("unpacked_data") == tmp_path / "unpacked_data" + + def test_an_absent_package_is_refused(self) -> None: + with pytest.raises(FileNotFoundError, match="sampletones_absent"): + package_directory("sampletones_absent") diff --git a/tests/unit/sampletones_shared/paths/test_source.py b/tests/unit/sampletones_shared/paths/test_source.py index 6d5f1ec83..3b55bd6a7 100644 --- a/tests/unit/sampletones_shared/paths/test_source.py +++ b/tests/unit/sampletones_shared/paths/test_source.py @@ -16,6 +16,3 @@ def test_the_source_root_is_where_this_package_lives(self) -> None: class TestRepositoryRoot: def test_the_repository_root_holds_the_project_file(self) -> None: assert (REPOSITORY_ROOT / PROJECT_FILE).is_file() - - def test_the_repository_root_holds_the_scripts_the_checks_run_from(self) -> None: - assert (REPOSITORY_ROOT / "scripts" / "checks").is_dir() diff --git a/tests/unit/sampletones_shared/test_command.py b/tests/unit/sampletones_shared/test_command.py new file mode 100644 index 000000000..c929c6c9b --- /dev/null +++ b/tests/unit/sampletones_shared/test_command.py @@ -0,0 +1,31 @@ +from argparse import ArgumentParser, Namespace +from dataclasses import FrozenInstanceError + +import pytest + +from sampletones_shared.command import Command + + +def _configure(parser: ArgumentParser) -> None: + parser.add_argument("--flag", action="store_true") + + +def _run(arguments: Namespace) -> int: + return 3 if arguments.flag else 0 + + +class TestCommand: + def test_a_command_carries_its_parser_and_its_work(self) -> None: + command = Command(name="probe", help="probe the thing", configure=_configure, run=_run) + parser = ArgumentParser() + + command.configure(parser) + + assert command.run(parser.parse_args(["--flag"])) == 3 + assert command.run(parser.parse_args([])) == 0 + + def test_a_command_is_settled_once_declared(self) -> None: + command = Command(name="probe", help="probe the thing", configure=_configure, run=_run) + + with pytest.raises(FrozenInstanceError): + command.name = "other" # type: ignore[misc] diff --git a/tests/unit/sampletones_shared/utils/test_tables.py b/tests/unit/sampletones_shared/utils/test_tables.py new file mode 100644 index 000000000..3147ed78e --- /dev/null +++ b/tests/unit/sampletones_shared/utils/test_tables.py @@ -0,0 +1,39 @@ +import csv +from pathlib import Path + +import pytest + +from sampletones_shared.utils.tables import Table + + +@pytest.fixture(name="table") +def table_fixture() -> Table: + return Table( + columns=("song", "bytes"), + rows=(("Theme, reprise", "120"), ("Café", "7")), + ) + + +class TestTable: + def test_the_csv_reads_back_cell_for_cell(self, table: Table, tmp_path: Path) -> None: + path = tmp_path / "report.csv" + + table.write_csv(path) + + with path.open(encoding="utf-8", newline="") as handle: + assert [tuple(row) for row in csv.reader(handle)] == [table.columns, *table.rows] + + def test_the_markdown_is_the_header_the_rule_and_a_line_per_row(self, table: Table) -> None: + assert table.markdown_lines() == [ + "| song | bytes |", + "|---|---|", + "| Theme, reprise | 120 |", + "| Café | 7 |", + ] + + def test_a_table_without_rows_is_its_header_and_rule(self) -> None: + assert Table(columns=("song",), rows=()).markdown_lines() == ["| song |", "|---|"] + + def test_a_row_short_of_the_columns_is_refused(self) -> None: + with pytest.raises(ValueError, match="Row 1 holds 1 cells for 2 columns"): + Table(columns=("song", "bytes"), rows=(("one", "1"), ("two",))) diff --git a/tests/unit/sampletones_shared/utils/test_text.py b/tests/unit/sampletones_shared/utils/test_text.py index c5b960799..a15c73d10 100644 --- a/tests/unit/sampletones_shared/utils/test_text.py +++ b/tests/unit/sampletones_shared/utils/test_text.py @@ -2,7 +2,7 @@ import pytest -from sampletones_shared.utils.text import natural_sort_key +from sampletones_shared.utils.text import LIST_SEPARATOR, listed_items, natural_sort_key class TestNumbers: @@ -51,3 +51,17 @@ def test_one_name_reaches_one_key(self) -> None: def test_a_name_the_reader_alone_can_spell(self) -> None: """A digit-like glyph outside the decimal digits is text, and the key states it as text.""" assert sorted(["m²", "m1"], key=natural_sort_key) == ["m1", "m²"] + + +class TestListedItems: + def test_items_keep_their_order_with_the_spaces_around_them_stripped(self) -> None: + assert listed_items(" fft ,cqt") == ["fft", "cqt"] + + def test_empty_items_are_dropped(self) -> None: + assert listed_items(f"pulse1{LIST_SEPARATOR}{LIST_SEPARATOR} {LIST_SEPARATOR}noise{LIST_SEPARATOR}") == [ + "pulse1", + "noise", + ] + + def test_an_empty_statement_lists_nothing(self) -> None: + assert listed_items("") == [] diff --git a/tests/unit/sampletones_shared/utils/test_validation.py b/tests/unit/sampletones_shared/utils/test_validation.py index fa867de34..c0ac56f3c 100644 --- a/tests/unit/sampletones_shared/utils/test_validation.py +++ b/tests/unit/sampletones_shared/utils/test_validation.py @@ -5,13 +5,14 @@ from typing import Any, Dict, List, Tuple import pytest -from pydantic import BaseModel, ConfigDict, Field, ValidationError +from pydantic import BaseModel, ConfigDict, Field, ValidationError, field_validator from sampletones_application.config.session.application.config import ApplicationConfig from sampletones_application.config.session.state.state import ApplicationState from sampletones_core.configs import Config from sampletones_shared.utils.validation import ( Location, + describe_failure, flatten_location, validate_with_recovery, ) @@ -209,6 +210,40 @@ def test_flatten(self, test_case: TestCase) -> None: assert flatten_location(test_case.location) == test_case.expected +class Refusing(BaseModel): + model_config = ConfigDict(extra="forbid") + + leaf: Leaf + count: int = Field(ge=1) + + @field_validator("count") + @classmethod + def _count_is_odd(cls, count: int) -> int: + if count % 2 == 0: + raise ValueError(f"The count {count} is even.") + + return count + + +class TestDescribeFailure: + def test_a_plain_value_error_reads_as_its_message(self) -> None: + assert describe_failure(ValueError("No file at song.wav.")) == "No file at song.wav." + + def test_a_raised_reason_keeps_its_own_words(self) -> None: + with pytest.raises(ValidationError) as failure: + Refusing(leaf=Leaf(), count=2) + + assert describe_failure(failure.value) == "The count 2 is even." + + def test_each_broken_constraint_is_one_line_naming_its_field(self) -> None: + with pytest.raises(ValidationError) as failure: + Refusing.model_validate({"leaf": {"value": 11}, "count": 0}) + + lines = describe_failure(failure.value).splitlines() + + assert [line.split(":")[0] for line in lines] == ["leaf.value", "count"] + + class TestRecoveryOnRealModels: def test_config_keeps_valid_settings_and_drops_incompatible(self) -> None: raw = { diff --git a/tests/unit/sampletones_tools/assets/mark/__init__.py b/tests/unit/sampletones_tools/assets/mark/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/unit/sampletones_assets/mark/test_geometry.py b/tests/unit/sampletones_tools/assets/mark/test_geometry.py similarity index 94% rename from tests/unit/sampletones_assets/mark/test_geometry.py rename to tests/unit/sampletones_tools/assets/mark/test_geometry.py index 78279edb4..fb92405ee 100644 --- a/tests/unit/sampletones_assets/mark/test_geometry.py +++ b/tests/unit/sampletones_tools/assets/mark/test_geometry.py @@ -3,8 +3,8 @@ import pytest -from sampletones_assets.mark.geometry import Rectangle, sine_points, square_rectangles -from sampletones_assets.mark.specification import Mark +from sampletones_tools.assets.mark.geometry import Rectangle, sine_points, square_rectangles +from sampletones_tools.assets.mark.specification import Mark SAMPLES: Final[int] = 5 diff --git a/tests/unit/sampletones_assets/mark/test_raster.py b/tests/unit/sampletones_tools/assets/mark/test_raster.py similarity index 93% rename from tests/unit/sampletones_assets/mark/test_raster.py rename to tests/unit/sampletones_tools/assets/mark/test_raster.py index 91b5ae74c..acef28217 100644 --- a/tests/unit/sampletones_assets/mark/test_raster.py +++ b/tests/unit/sampletones_tools/assets/mark/test_raster.py @@ -2,9 +2,9 @@ import pytest -from sampletones_assets.mark.raster import MarkRaster -from sampletones_assets.mark.specification import Mark from sampletones_shared.utils.color import parse_hex_color +from sampletones_tools.assets.mark.raster import MarkRaster +from sampletones_tools.assets.mark.specification import Mark CORNER: Final[Tuple[int, int]] = (0, 0) ALPHA: Final[int] = 3 diff --git a/tests/unit/sampletones_assets/mark/test_specification.py b/tests/unit/sampletones_tools/assets/mark/test_specification.py similarity index 98% rename from tests/unit/sampletones_assets/mark/test_specification.py rename to tests/unit/sampletones_tools/assets/mark/test_specification.py index a3f51d363..9048f54d0 100644 --- a/tests/unit/sampletones_assets/mark/test_specification.py +++ b/tests/unit/sampletones_tools/assets/mark/test_specification.py @@ -4,7 +4,7 @@ import pytest from pydantic import ValidationError -from sampletones_assets.mark.specification import Mark +from sampletones_tools.assets.mark.specification import Mark from tests.suite.case import BaseRegularTestCase VALID_FRAME: Final[Dict[str, Any]] = { diff --git a/tests/unit/sampletones_assets/mark/test_suite.py b/tests/unit/sampletones_tools/assets/mark/test_suite.py similarity index 94% rename from tests/unit/sampletones_assets/mark/test_suite.py rename to tests/unit/sampletones_tools/assets/mark/test_suite.py index b0a8f3e3c..97821fcf3 100644 --- a/tests/unit/sampletones_assets/mark/test_suite.py +++ b/tests/unit/sampletones_tools/assets/mark/test_suite.py @@ -3,13 +3,13 @@ import pytest from PIL import Image -from sampletones_assets.mark.specification import Mark -from sampletones_assets.mark.suite import write_icon_suite from sampletones_shared.paths.resources import ( ICON_UNIX_FILENAME, ICON_VECTOR_FILENAME, ICON_WIN_FILENAME, ) +from sampletones_tools.assets.mark.specification import Mark +from sampletones_tools.assets.mark.suite import write_icon_suite RGBA_MODE = "RGBA" ICO_SIZES_KEY = "sizes" diff --git a/tests/unit/sampletones_assets/mark/test_vector.py b/tests/unit/sampletones_tools/assets/mark/test_vector.py similarity index 93% rename from tests/unit/sampletones_assets/mark/test_vector.py rename to tests/unit/sampletones_tools/assets/mark/test_vector.py index 108abb064..e73925b68 100644 --- a/tests/unit/sampletones_assets/mark/test_vector.py +++ b/tests/unit/sampletones_tools/assets/mark/test_vector.py @@ -1,9 +1,9 @@ from importlib.resources import files from pathlib import Path -from sampletones_assets.mark.specification import Mark -from sampletones_assets.mark.vector import render_vector from sampletones_shared.paths.resources import ICON_VECTOR_FILENAME +from sampletones_tools.assets.mark.specification import Mark +from sampletones_tools.assets.mark.vector import render_vector PLACEHOLDER_PREFIX = "$" REPLACEMENT_COLOR = "#010203" diff --git a/tests/unit/sampletones_tools/assets/test_command.py b/tests/unit/sampletones_tools/assets/test_command.py new file mode 100644 index 000000000..723388def --- /dev/null +++ b/tests/unit/sampletones_tools/assets/test_command.py @@ -0,0 +1,53 @@ +from pathlib import Path +from typing import Final, List, Tuple + +import pytest + +from sampletones.commands.registry import COMMANDS +from sampletones.dispatcher import dispatch +from sampletones_tools.assets.mark.specification import Mark +from sampletones_tools.assets.paths import ICONS_DIRECTORY + +WRITER: Final[str] = "sampletones_tools.assets.mark.suite.write_icon_suite" +GUARD: Final[str] = "sampletones_tools.checkout.require_checkout" + + +class RecordedSuite: + def __init__(self) -> None: + self.writes: List[Tuple[Path, Mark]] = [] + + def __call__(self, directory: Path, mark: Mark) -> List[Path]: + self.writes.append((directory, mark)) + return [directory / "sampletones.svg"] + + +class TestIcons: + def test_without_a_directory_the_shipped_icons_are_written_from_a_checkout( + self, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], + ) -> None: + suite = RecordedSuite() + guarded: List[str] = [] + monkeypatch.setattr(WRITER, suite) + monkeypatch.setattr(GUARD, guarded.append) + + assert dispatch(COMMANDS, ["icons"]) == 0 + assert [directory for directory, _ in suite.writes] == [ICONS_DIRECTORY] + assert suite.writes[0][1] == Mark.load() + assert guarded == ["icons"] + assert f"Wrote {ICONS_DIRECTORY / 'sampletones.svg'}" in capsys.readouterr().out + + def test_a_directory_of_its_own_is_written_from_a_checkout_too( + self, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + ) -> None: + suite = RecordedSuite() + guarded: List[str] = [] + monkeypatch.setattr(WRITER, suite) + monkeypatch.setattr(GUARD, guarded.append) + + assert dispatch(COMMANDS, ["icons", "--output", str(tmp_path)]) == 0 + assert [directory for directory, _ in suite.writes] == [tmp_path] + assert guarded == ["icons"] diff --git a/tests/unit/sampletones_tools/calibration/__init__.py b/tests/unit/sampletones_tools/calibration/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/unit/sampletones_tools/calibration/config/__init__.py b/tests/unit/sampletones_tools/calibration/config/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/unit/sampletones_core/calibration/config/test_corpus.py b/tests/unit/sampletones_tools/calibration/config/test_corpus.py similarity index 98% rename from tests/unit/sampletones_core/calibration/config/test_corpus.py rename to tests/unit/sampletones_tools/calibration/config/test_corpus.py index b09e60af6..4d2dae139 100644 --- a/tests/unit/sampletones_core/calibration/config/test_corpus.py +++ b/tests/unit/sampletones_tools/calibration/config/test_corpus.py @@ -4,7 +4,7 @@ import pytest from pydantic import ValidationError -from sampletones_core.calibration.config.corpus import CorpusConfig +from sampletones_tools.calibration.config.corpus import CorpusConfig from tests.suite.case import BaseRegularTestCase VALID_TRANSIENT: Final[Dict[str, Any]] = { diff --git a/tests/unit/sampletones_core/calibration/config/test_referee.py b/tests/unit/sampletones_tools/calibration/config/test_referee.py similarity index 96% rename from tests/unit/sampletones_core/calibration/config/test_referee.py rename to tests/unit/sampletones_tools/calibration/config/test_referee.py index 21120e5ad..eb5659cf3 100644 --- a/tests/unit/sampletones_core/calibration/config/test_referee.py +++ b/tests/unit/sampletones_tools/calibration/config/test_referee.py @@ -4,7 +4,7 @@ import pytest from pydantic import ValidationError -from sampletones_core.calibration.config.referee import RefereeConfig +from sampletones_tools.calibration.config.referee import RefereeConfig from tests.suite.case import BaseRegularTestCase VALID_FIELDS: Final[Dict[str, Any]] = { diff --git a/tests/unit/sampletones_tools/calibration/corpus/__init__.py b/tests/unit/sampletones_tools/calibration/corpus/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/unit/sampletones_core/calibration/corpus/conftest.py b/tests/unit/sampletones_tools/calibration/corpus/conftest.py similarity index 68% rename from tests/unit/sampletones_core/calibration/corpus/conftest.py rename to tests/unit/sampletones_tools/calibration/corpus/conftest.py index db4080c0d..6a89d2862 100644 --- a/tests/unit/sampletones_core/calibration/corpus/conftest.py +++ b/tests/unit/sampletones_tools/calibration/corpus/conftest.py @@ -2,9 +2,9 @@ import pytest -from sampletones_core.calibration.config.corpus import CorpusConfig -from sampletones_core.calibration.corpus.item import CorpusItem -from sampletones_core.calibration.corpus.synthesis import build_corpus +from sampletones_tools.calibration.config.corpus import CorpusConfig +from sampletones_tools.calibration.corpus.item import CorpusItem +from sampletones_tools.calibration.corpus.synthesis import build_corpus SAMPLE_RATE: Final[int] = 22050 diff --git a/tests/unit/sampletones_core/calibration/corpus/test_synthesis.py b/tests/unit/sampletones_tools/calibration/corpus/test_synthesis.py similarity index 87% rename from tests/unit/sampletones_core/calibration/corpus/test_synthesis.py rename to tests/unit/sampletones_tools/calibration/corpus/test_synthesis.py index 38350b1cc..45cd11514 100644 --- a/tests/unit/sampletones_core/calibration/corpus/test_synthesis.py +++ b/tests/unit/sampletones_tools/calibration/corpus/test_synthesis.py @@ -2,9 +2,9 @@ import numpy as np -from sampletones_core.calibration.config.corpus import CorpusConfig -from sampletones_core.calibration.corpus.item import CorpusItem -from sampletones_core.calibration.corpus.synthesis import build_corpus +from sampletones_tools.calibration.config.corpus import CorpusConfig +from sampletones_tools.calibration.corpus.item import CorpusItem +from sampletones_tools.calibration.corpus.synthesis import build_corpus EXPECTED_CATEGORIES: Final[FrozenSet[str]] = frozenset({"tone", "timbre", "noise", "mix", "transient", "dynamics"}) diff --git a/tests/unit/sampletones_core/calibration/corpus/test_writer.py b/tests/unit/sampletones_tools/calibration/corpus/test_writer.py similarity index 78% rename from tests/unit/sampletones_core/calibration/corpus/test_writer.py rename to tests/unit/sampletones_tools/calibration/corpus/test_writer.py index 0520366e3..9d1873d65 100644 --- a/tests/unit/sampletones_core/calibration/corpus/test_writer.py +++ b/tests/unit/sampletones_tools/calibration/corpus/test_writer.py @@ -1,8 +1,8 @@ from pathlib import Path from typing import List -from sampletones_core.calibration.corpus.item import CorpusItem -from sampletones_core.calibration.corpus.writer import write_corpus +from sampletones_tools.calibration.corpus.item import CorpusItem +from sampletones_tools.calibration.corpus.writer import write_corpus class TestWriteCorpus: diff --git a/tests/unit/sampletones_tools/calibration/referee/__init__.py b/tests/unit/sampletones_tools/calibration/referee/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/unit/sampletones_core/calibration/referee/test_auditory.py b/tests/unit/sampletones_tools/calibration/referee/test_auditory.py similarity index 94% rename from tests/unit/sampletones_core/calibration/referee/test_auditory.py rename to tests/unit/sampletones_tools/calibration/referee/test_auditory.py index 7581f7804..63a20e856 100644 --- a/tests/unit/sampletones_core/calibration/referee/test_auditory.py +++ b/tests/unit/sampletones_tools/calibration/referee/test_auditory.py @@ -3,8 +3,8 @@ import numpy as np import pytest -from sampletones_core.calibration.config.referee import RefereeConfig -from sampletones_core.calibration.referee.auditory import MultiResolutionAuditoryReferee +from sampletones_tools.calibration.config.referee import RefereeConfig +from sampletones_tools.calibration.referee.auditory import MultiResolutionAuditoryReferee SAMPLE_RATE: Final[int] = 22050 SIGNAL_SECONDS: Final[float] = 1.0 diff --git a/tests/unit/sampletones_tools/calibration/test_command.py b/tests/unit/sampletones_tools/calibration/test_command.py new file mode 100644 index 000000000..e1977983e --- /dev/null +++ b/tests/unit/sampletones_tools/calibration/test_command.py @@ -0,0 +1,82 @@ +from pathlib import Path +from typing import List + +import pytest + +from sampletones.commands.registry import COMMANDS +from sampletones.dispatcher import dispatch +from sampletones_core.configs import Config +from sampletones_core.constants.enums import DEFAULT_CHANNELS, ChannelName, SpectrumMethod +from sampletones_tools.calibration.session import OUTPUT_ROOT, CalibrationRequest + +CALIBRATE = "sampletones_tools.calibration.session.calibrate" +LOADER = "sampletones_core.headless.config.load_config" + + +class RecordedCalibration: + def __init__(self) -> None: + self.requests: List[CalibrationRequest] = [] + + def __call__(self, request: CalibrationRequest) -> Path: + self.requests.append(request) + return request.output + + +@pytest.fixture(name="calibration") +def calibration_fixture(monkeypatch: pytest.MonkeyPatch) -> RecordedCalibration: + recorded = RecordedCalibration() + monkeypatch.setattr(CALIBRATE, recorded) + monkeypatch.setattr(LOADER, lambda path: Config()) + return recorded + + +class TestCalibration: + def test_the_sweep_is_read_from_the_options(self, calibration: RecordedCalibration, tmp_path: Path) -> None: + status = dispatch( + COMMANDS, + [ + "calibration", + "--methods", + "fft", + "--perceptual-exponents", + "0.5,1", + "--temporal-weights", + "0.25", + "--channels", + "pulse1", + "-o", + str(tmp_path / "run"), + ], + ) + + assert status == 0 + request = calibration.requests[0] + assert request.methods == [SpectrumMethod.FFT] + assert request.perceptual_exponents == [0.5, 1.0] + assert request.temporal_weights == [0.25] + assert request.channels == [ChannelName.PULSE1] + assert request.output == tmp_path / "run" + + def test_without_options_the_run_sweeps_both_methods_into_the_documents( + self, + calibration: RecordedCalibration, + ) -> None: + assert dispatch(COMMANDS, ["calibration"]) == 0 + request = calibration.requests[0] + assert request.methods == [SpectrumMethod.FFT, SpectrumMethod.CQT] + assert request.perceptual_exponents == [1.0] + assert request.temporal_weights == [] + assert request.channels == list(DEFAULT_CHANNELS) + assert request.output.parent == OUTPUT_ROOT + + def test_an_unknown_method_is_refused(self, calibration: RecordedCalibration) -> None: + with pytest.raises(SystemExit, match="Unknown spectrum method"): + dispatch(COMMANDS, ["calibration", "--methods", "dct"]) + + assert calibration.requests == [] + + def test_a_value_that_is_no_number_is_refused(self, calibration: RecordedCalibration) -> None: + with pytest.raises(SystemExit, match="Not a number"): + dispatch(COMMANDS, ["calibration", "--perceptual-exponents", "high"]) + + assert calibration.requests == [] diff --git a/tests/unit/sampletones_tools/calibration/test_report.py b/tests/unit/sampletones_tools/calibration/test_report.py new file mode 100644 index 000000000..5998f3d0f --- /dev/null +++ b/tests/unit/sampletones_tools/calibration/test_report.py @@ -0,0 +1,56 @@ +import csv +from pathlib import Path +from typing import Final, List + +from sampletones_shared.utils.tables import Table +from sampletones_tools.calibration.report import CSV_COLUMNS, OVERALL_COLUMN, VARIANT_COLUMN, write_csv, write_markdown +from sampletones_tools.calibration.runner import CalibrationRow + +ROWS: Final[List[CalibrationRow]] = [ + CalibrationRow(variant="fft", item="tone-a", category="tones", referee="spectral", score=0.25), + CalibrationRow(variant="fft", item="tone-b", category="tones", referee="spectral", score=0.75), + CalibrationRow(variant="fft", item="hiss", category="noise", referee="spectral", score=1.0), + CalibrationRow(variant="cqt", item="tone-a", category="tones", referee="spectral", score=0.5), + CalibrationRow(variant="cqt", item="hiss", category="noise", referee="spectral", score=0.5), + CalibrationRow(variant="fft", item="tone-a", category="tones", referee="envelope", score=2.0), +] + + +class TestWriteCsv: + def test_every_row_reads_back_under_the_columns(self, tmp_path: Path) -> None: + path = tmp_path / "report.csv" + + write_csv(ROWS, path) + + with path.open(newline="", encoding="utf-8") as handle: + header, *cells = list(csv.reader(handle)) + assert tuple(header) == CSV_COLUMNS + assert [(row[0], row[1], row[2], row[3], float(row[4])) for row in cells] == [ + (row.variant, row.item, row.category, row.referee, row.score) for row in ROWS + ] + + +class TestWriteMarkdown: + def test_each_referee_pivots_the_mean_of_every_variant_by_category(self, tmp_path: Path) -> None: + path = tmp_path / "report.md" + + write_markdown(ROWS, path) + + lines = path.read_text(encoding="utf-8").split("\n") + spectral = Table( + columns=(VARIANT_COLUMN, "tones", "noise", OVERALL_COLUMN), + rows=(("fft", "0.500", "1.000", "0.667"), ("cqt", "0.500", "0.500", "0.500")), + ) + envelope = Table(columns=(VARIANT_COLUMN, "tones", OVERALL_COLUMN), rows=(("fft", "2.000", "2.000"),)) + assert lines == [ + "# Calibration report", + "", + "## spectral", + "", + *spectral.markdown_lines(), + "", + "## envelope", + "", + *envelope.markdown_lines(), + "", + ] diff --git a/tests/unit/sampletones_core/calibration/test_runner.py b/tests/unit/sampletones_tools/calibration/test_runner.py similarity index 95% rename from tests/unit/sampletones_core/calibration/test_runner.py rename to tests/unit/sampletones_tools/calibration/test_runner.py index 2eb447e31..329c1dbcb 100644 --- a/tests/unit/sampletones_core/calibration/test_runner.py +++ b/tests/unit/sampletones_tools/calibration/test_runner.py @@ -3,9 +3,9 @@ import pytest -from sampletones_core.calibration.runner import build_variants from sampletones_core.configs import Config from sampletones_core.constants.enums import SpectrumMethod +from sampletones_tools.calibration.runner import build_variants METHODS: Final[List[SpectrumMethod]] = [SpectrumMethod.FFT, SpectrumMethod.CQT] EXPONENTS: Final[List[float]] = [1.0] diff --git a/tests/unit/sampletones_tools/calibration/test_session.py b/tests/unit/sampletones_tools/calibration/test_session.py new file mode 100644 index 000000000..06596c88e --- /dev/null +++ b/tests/unit/sampletones_tools/calibration/test_session.py @@ -0,0 +1,73 @@ +from datetime import datetime +from pathlib import Path + +import pytest + +from sampletones_core.configs import Config +from sampletones_core.constants.enums import ChannelName, SpectrumMethod +from sampletones_tools.calibration.session import ( + OUTPUT_ROOT, + CalibrationRequest, + default_output, + floats_named, + methods_named, +) +from sampletones_tools.runs import RUN_STAMP + + +class TestMethodsNamed: + def test_nothing_named_is_fft_and_cqt(self) -> None: + assert methods_named(None) == [SpectrumMethod.FFT, SpectrumMethod.CQT] + + def test_names_are_read_in_order(self) -> None: + assert methods_named("cqt, fft") == [SpectrumMethod.CQT, SpectrumMethod.FFT] + + def test_an_unknown_method_is_refused_with_the_known_ones(self) -> None: + with pytest.raises(ValueError, match="Unknown spectrum method 'dct'; the methods are"): + methods_named("fft,dct") + + +class TestFloatsNamed: + def test_nothing_named_is_the_default(self) -> None: + assert floats_named(None, (1.0,)) == [1.0] + assert floats_named(None, ()) == [] + + def test_values_are_read_in_order(self) -> None: + assert floats_named("0.5, 1", ()) == [0.5, 1.0] + + def test_a_value_that_is_no_number_is_refused(self) -> None: + with pytest.raises(ValueError, match="Not a number: 'high'"): + floats_named("0.5,high", ()) + + +class TestDefaultOutput: + def test_a_run_lands_in_a_timestamped_directory_under_the_documents(self) -> None: + output = default_output() + + assert output.parent == OUTPUT_ROOT + assert datetime.strptime(output.name, RUN_STAMP) + + +class TestCalibrationRequest: + def test_the_base_is_pinned_to_the_channels(self, tmp_path: Path) -> None: + request = CalibrationRequest( + base=Config(), + output=tmp_path, + methods=[SpectrumMethod.FFT], + perceptual_exponents=[1.0], + temporal_weights=[], + channels=[ChannelName.PULSE1], + ) + + assert request.pinned_base().generation.channels == [ChannelName.PULSE1] + + def test_an_empty_sweep_is_refused(self, tmp_path: Path) -> None: + with pytest.raises(ValueError): + CalibrationRequest( + base=Config(), + output=tmp_path, + methods=[], + perceptual_exponents=[1.0], + temporal_weights=[], + channels=[ChannelName.PULSE1], + ) diff --git a/tests/unit/sampletones_tools/checks/boundary/__init__.py b/tests/unit/sampletones_tools/checks/boundary/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/unit/sampletones_tools/checks/boundary/configs/__init__.py b/tests/unit/sampletones_tools/checks/boundary/configs/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/unit/sampletones_shared/meta/import_boundary/configs/test_declaration.py b/tests/unit/sampletones_tools/checks/boundary/configs/test_declaration.py similarity index 90% rename from tests/unit/sampletones_shared/meta/import_boundary/configs/test_declaration.py rename to tests/unit/sampletones_tools/checks/boundary/configs/test_declaration.py index de73889d2..333b71165 100644 --- a/tests/unit/sampletones_shared/meta/import_boundary/configs/test_declaration.py +++ b/tests/unit/sampletones_tools/checks/boundary/configs/test_declaration.py @@ -1,7 +1,7 @@ from typing import Final -from sampletones_shared.meta.import_boundary.configs.declaration import BoundaryDeclaration -from sampletones_shared.meta.import_boundary.configs.general import GeneralBoundaries +from sampletones_tools.checks.boundary.configs.declaration import BoundaryDeclaration +from sampletones_tools.checks.boundary.configs.general import GeneralBoundaries ROOT: Final[str] = "package" PATTERN: Final[str] = "logic/**/*.py" diff --git a/tests/unit/sampletones_shared/meta/import_boundary/configs/test_general.py b/tests/unit/sampletones_tools/checks/boundary/configs/test_general.py similarity index 91% rename from tests/unit/sampletones_shared/meta/import_boundary/configs/test_general.py rename to tests/unit/sampletones_tools/checks/boundary/configs/test_general.py index 51b85a3c6..c1b86c2ea 100644 --- a/tests/unit/sampletones_shared/meta/import_boundary/configs/test_general.py +++ b/tests/unit/sampletones_tools/checks/boundary/configs/test_general.py @@ -2,7 +2,7 @@ import pytest -from sampletones_shared.meta.import_boundary.configs.general import GeneralBoundaries +from sampletones_tools.checks.boundary.configs.general import GeneralBoundaries GENERAL: Final[GeneralBoundaries] = GeneralBoundaries( groups={ diff --git a/tests/unit/sampletones_shared/meta/import_boundary/configs/test_rules.py b/tests/unit/sampletones_tools/checks/boundary/configs/test_rules.py similarity index 66% rename from tests/unit/sampletones_shared/meta/import_boundary/configs/test_rules.py rename to tests/unit/sampletones_tools/checks/boundary/configs/test_rules.py index 58e9acf81..2b9602d7c 100644 --- a/tests/unit/sampletones_shared/meta/import_boundary/configs/test_rules.py +++ b/tests/unit/sampletones_tools/checks/boundary/configs/test_rules.py @@ -5,14 +5,16 @@ import pytest from pydantic import ValidationError -from sampletones_shared.meta.import_boundary.check import check_boundaries -from sampletones_shared.meta.import_boundary.configs.declaration import BoundaryDeclaration -from sampletones_shared.meta.import_boundary.configs.general import GeneralBoundaries -from sampletones_shared.meta.import_boundary.configs.rules import ImportBoundaryRules -from sampletones_shared.meta.import_boundary.graph import reached_units -from sampletones_shared.meta.import_boundary.rule import BoundaryRule -from sampletones_shared.meta.import_boundary.scope import rule_modules from sampletones_shared.paths.source import SOURCE_ROOT +from sampletones_tools.checks.boundary.check import check_boundaries +from sampletones_tools.checks.boundary.configs.declaration import BoundaryDeclaration +from sampletones_tools.checks.boundary.configs.general import GeneralBoundaries +from sampletones_tools.checks.boundary.configs.rules import ImportBoundaryRules +from sampletones_tools.checks.boundary.graph import reached_units +from sampletones_tools.checks.boundary.rule import BoundaryRule +from sampletones_tools.checks.boundary.scope import rule_modules +from sampletones_tools.checks.boundary.standalone import check_standalone +from sampletones_tools.checks.paths import SCRIPTS_ROOT from tests.suite.source import swept_paths, write_module BOUNDARIES: Final[ImportBoundaryRules] = ImportBoundaryRules.load() @@ -20,15 +22,16 @@ APPLICATION: Final[str] = "sampletones_application" CORE: Final[str] = "sampletones_core" PLAYER: Final[str] = "sampletones_player" -ASSEMBLER: Final[str] = "sampletones_player.driver.assembler" +TOOLS: Final[str] = "sampletones_tools" +ENTRY: Final[str] = "sampletones" VISUAL_IMPORT: Final[str] = "import dearpygui.dearpygui as dpg\n" CONTRACT_IMPORT: Final[str] = "from sampletones_application.services.result import ServiceResult\n" PLAIN_IMPORT: Final[str] = "from sampletones_core.project.project import Project\n" PLAYER_IMPORT: Final[str] = "from sampletones_player.song import Song\n" -ASSEMBLER_IMPORT: Final[str] = "from sampletones_player.driver.assembler.builder import build_driver\n" -DRIVER_IMPORT: Final[str] = "from sampletones_player.driver.image import DriverImage\n" +TOOLS_IMPORT: Final[str] = "from sampletones_tools.registry import DEVELOPER_COMMANDS\n" PANEL_SUFFIX: Final[str] = "def build() -> None:\n dpg.add_group(parent=SUF_PANEL_LEFT)\n" +THIRD_PARTY_IMPORT: Final[str] = "import numpy\n" def reached_modules(rule: BoundaryRule) -> List[Path]: @@ -64,9 +67,17 @@ def test_the_reconstruction_engine_stays_clear_of_the_console_player(self) -> No def test_the_console_player_reads_the_reconstruction_engine(self) -> None: assert CORE in self.LAYERS[PLAYER] - def test_the_synthesis_package_stands_below_the_reconstruction_engine(self) -> None: - """Equal temperament sits in `sampletones_shared`, so synthesis reaches no engine module.""" - assert CORE not in reached_units(self.LAYERS, "sampletones_synthesis") + def test_the_entry_is_the_one_importer_of_the_tools(self) -> None: + """The wheel carries the tools, and the command line is where a developer reaches them.""" + assert {unit for unit, layers in self.LAYERS.items() if TOOLS in layers} == {ENTRY} + + def test_no_shipped_package_reaches_the_tools(self) -> None: + shipped = set(self.LAYERS) - {ENTRY, TOOLS} + + assert all(TOOLS not in reached_units(self.LAYERS, package) for package in shipped) + + def test_the_tools_reach_the_application(self) -> None: + assert APPLICATION in self.LAYERS[TOOLS] class TestPlayerGraph: @@ -77,10 +88,6 @@ class TestPlayerGraph: def test_the_specification_is_the_layer_everything_stands_on(self) -> None: assert self.LAYERS["specification"] == () - def test_the_build_toolchain_is_reached_from_no_shipped_module(self) -> None: - """`driver/assembler/` stays outside the wheel, so an import of it breaks an installed copy.""" - assert all("driver/assembler" not in layers for layers in self.LAYERS.values()) - def test_the_driver_is_reached_through_the_file_that_writes_the_nsf(self) -> None: assert "driver" in self.LAYERS["nsf"] @@ -108,6 +115,7 @@ def test_a_declaration_naming_no_declared_group_is_refused(self) -> None: ), ), tokens=(), + standalone=(), ) @@ -135,20 +143,37 @@ def test_the_console_player_reading_the_engine_is_left_alone(self, tmp_path: Pat assert reported(tmp_path) == [] - def test_a_shipped_module_reaching_the_build_toolchain_is_reported(self, tmp_path: Path) -> None: - write_module(tmp_path / PLAYER / "nsf", "file.py", ASSEMBLER_IMPORT) + def test_a_panel_composing_a_column_suffix_is_reported(self, tmp_path: Path) -> None: + write_module(tmp_path / APPLICATION / "ui" / "panels", "left.py", PANEL_SUFFIX) + + assert len(reported(tmp_path)) == 1 + + def test_a_shipped_module_naming_the_tools_is_reported_by_the_graph_and_by_name(self, tmp_path: Path) -> None: + write_module(tmp_path / CORE / "formats", "tooling.py", TOOLS_IMPORT) + + kinds = reported(tmp_path) - assert reported(tmp_path) == [ASSEMBLER] + assert TOOLS in kinds + assert any("names no tool" in kind for kind in kinds) - def test_the_build_toolchain_reads_the_driver_it_assembles(self, tmp_path: Path) -> None: - write_module(tmp_path / PLAYER / "driver" / "assembler", "builder.py", DRIVER_IMPORT) + def test_the_entry_naming_the_tools_is_left_alone(self, tmp_path: Path) -> None: + write_module(tmp_path / ENTRY / "commands", "registry.py", TOOLS_IMPORT) assert reported(tmp_path) == [] - def test_a_panel_composing_a_column_suffix_is_reported(self, tmp_path: Path) -> None: - write_module(tmp_path / APPLICATION / "ui" / "panels", "left.py", PANEL_SUFFIX) - assert len(reported(tmp_path)) == 1 +class TestStandaloneRules: + """The scripts that run on the system interpreter, held to the standard library.""" + + def test_the_scripts_tree_holds_to_the_rule(self) -> None: + assert check_standalone(SCRIPTS_ROOT, BOUNDARIES.standalone, None) == [] + + def test_a_bootstrap_script_reaching_a_third_party_package_is_reported(self, tmp_path: Path) -> None: + write_module(tmp_path, "bundle.py", THIRD_PARTY_IMPORT) + + reported = check_standalone(tmp_path, BOUNDARIES.standalone, None) + + assert [violation.kind for violation in reported] == [rule.message for rule in BOUNDARIES.standalone] class TestRuleCoverage: @@ -162,3 +187,8 @@ def test_every_boundary_rule_reaches_a_module(self) -> None: def test_every_token_rule_reaches_a_module(self) -> None: assert all(list((SOURCE_ROOT / rule.root).glob(rule.pattern)) for rule in BOUNDARIES.tokens) + + def test_every_standalone_rule_reaches_a_script(self) -> None: + swept = swept_paths(SCRIPTS_ROOT) + + assert all(rule_modules(SCRIPTS_ROOT, rule.pattern, (), swept, None) for rule in BOUNDARIES.standalone) diff --git a/tests/unit/sampletones_shared/meta/import_boundary/test_check.py b/tests/unit/sampletones_tools/checks/boundary/test_check.py similarity index 94% rename from tests/unit/sampletones_shared/meta/import_boundary/test_check.py rename to tests/unit/sampletones_tools/checks/boundary/test_check.py index 7cd39473c..59554dc72 100644 --- a/tests/unit/sampletones_shared/meta/import_boundary/test_check.py +++ b/tests/unit/sampletones_tools/checks/boundary/test_check.py @@ -3,9 +3,9 @@ import pytest -from sampletones_shared.meta.import_boundary.check import check_boundaries -from sampletones_shared.meta.import_boundary.rule import BoundaryRule -from sampletones_shared.meta.import_boundary.token import TokenRule +from sampletones_tools.checks.boundary.check import check_boundaries +from sampletones_tools.checks.boundary.rule import BoundaryRule +from sampletones_tools.checks.boundary.token import TokenRule from tests.suite.source import write_module FORBIDDEN: Final[str] = "from other_package.module import Thing\n" diff --git a/tests/unit/sampletones_shared/meta/import_boundary/test_graph.py b/tests/unit/sampletones_tools/checks/boundary/test_graph.py similarity index 97% rename from tests/unit/sampletones_shared/meta/import_boundary/test_graph.py rename to tests/unit/sampletones_tools/checks/boundary/test_graph.py index d6857ef97..f7006ffef 100644 --- a/tests/unit/sampletones_shared/meta/import_boundary/test_graph.py +++ b/tests/unit/sampletones_tools/checks/boundary/test_graph.py @@ -3,7 +3,7 @@ import pytest from pydantic import ValidationError -from sampletones_shared.meta.import_boundary.graph import LayerGraph, reached_units +from sampletones_tools.checks.boundary.graph import LayerGraph, reached_units GRAPH: Final[LayerGraph] = LayerGraph( root="package", diff --git a/tests/unit/sampletones_shared/meta/import_boundary/test_imports.py b/tests/unit/sampletones_tools/checks/boundary/test_imports.py similarity index 95% rename from tests/unit/sampletones_shared/meta/import_boundary/test_imports.py rename to tests/unit/sampletones_tools/checks/boundary/test_imports.py index 681e2c086..bb9bd8bdc 100644 --- a/tests/unit/sampletones_shared/meta/import_boundary/test_imports.py +++ b/tests/unit/sampletones_tools/checks/boundary/test_imports.py @@ -3,7 +3,7 @@ import pytest -from sampletones_shared.meta.import_boundary.imports import imported_module, matches_prefix +from sampletones_tools.checks.boundary.imports import imported_module, matches_prefix from tests.suite.base import BaseTestSuite from tests.suite.case import BaseAutolabelTestCase diff --git a/tests/unit/sampletones_shared/meta/import_boundary/test_rule.py b/tests/unit/sampletones_tools/checks/boundary/test_rule.py similarity index 97% rename from tests/unit/sampletones_shared/meta/import_boundary/test_rule.py rename to tests/unit/sampletones_tools/checks/boundary/test_rule.py index a6058a404..bf398cb4a 100644 --- a/tests/unit/sampletones_shared/meta/import_boundary/test_rule.py +++ b/tests/unit/sampletones_tools/checks/boundary/test_rule.py @@ -1,7 +1,7 @@ from pathlib import Path from typing import Final -from sampletones_shared.meta.import_boundary.rule import BoundaryRule +from sampletones_tools.checks.boundary.rule import BoundaryRule from tests.suite.source import write_module FORBIDDEN: Final[str] = "from other_package.module import Thing\n" diff --git a/tests/unit/sampletones_shared/meta/import_boundary/test_scope.py b/tests/unit/sampletones_tools/checks/boundary/test_scope.py similarity index 97% rename from tests/unit/sampletones_shared/meta/import_boundary/test_scope.py rename to tests/unit/sampletones_tools/checks/boundary/test_scope.py index 92050e731..94b58a597 100644 --- a/tests/unit/sampletones_shared/meta/import_boundary/test_scope.py +++ b/tests/unit/sampletones_tools/checks/boundary/test_scope.py @@ -1,7 +1,7 @@ from pathlib import Path from typing import Final -from sampletones_shared.meta.import_boundary.scope import rule_modules +from sampletones_tools.checks.boundary.scope import rule_modules from tests.suite.source import swept_paths, write_module LOGIC_PATTERN: Final[str] = "logic/**/*.py" diff --git a/tests/unit/sampletones_tools/checks/boundary/test_standalone.py b/tests/unit/sampletones_tools/checks/boundary/test_standalone.py new file mode 100644 index 000000000..6057c838a --- /dev/null +++ b/tests/unit/sampletones_tools/checks/boundary/test_standalone.py @@ -0,0 +1,125 @@ +from pathlib import Path +from typing import Final, List + +import pytest + +from sampletones_tools.checks.boundary.standalone import ( + StandaloneRule, + check_standalone, + local_names, +) +from tests.suite.source import write_module + +MESSAGE: Final[str] = "a bootstrap script imports the standard library and the tree alone" + +RULE: Final[StandaloneRule] = StandaloneRule( + pattern="**/*.py", + reserved=("tests",), + message=MESSAGE, +) + +THIRD_PARTY: Final[str] = "import numpy\n" +REACHABLE: Final[str] = ( + "import argparse\n" + "from pathlib import Path\n" + "from bootstrap.processes import run\n" + "import detect_cuda\n" + "from . import sibling\n" +) + + +def _tree(root: Path) -> None: + write_module(root / "bootstrap", "__init__.py", "") + write_module(root / "bootstrap", "processes.py", "import subprocess\n") + write_module(root, "detect_cuda.py", "import re\n") + + +def kinds(root: Path) -> List[str]: + """What the rule reports over a tree a test builds.""" + return [violation.kind for violation in check_standalone(root, (RULE,), None)] + + +class TestLocalNames: + def test_the_modules_and_the_packages_of_the_tree_are_its_names(self, tmp_path: Path) -> None: + _tree(tmp_path) + (tmp_path / "runtime_hooks").mkdir() + + assert local_names(tmp_path) == {"bootstrap", "detect_cuda"} + + +class TestStandaloneViolations: + """The imports of one script that reach past the standard library and the tree.""" + + def test_a_third_party_import_is_reported_with_the_rules_message(self, tmp_path: Path) -> None: + path = write_module(tmp_path, "bundle.py", THIRD_PARTY) + + assert [violation.kind for violation in RULE.violations(path, set())] == [MESSAGE] + + def test_the_standard_library_and_the_tree_stay_reachable(self, tmp_path: Path) -> None: + path = write_module(tmp_path, "bundle.py", REACHABLE) + + assert RULE.violations(path, {"bootstrap", "detect_cuda"}) == [] + + def test_the_report_names_the_line_the_import_sits_on(self, tmp_path: Path) -> None: + path = write_module(tmp_path, "bundle.py", f"import sys\n{THIRD_PARTY}") + + assert RULE.violations(path, set())[0].location == f"{path}:2: import numpy" + + +class TestShadowing: + """A name in the tree that stands in for a module the tree sits beside.""" + + def test_a_standard_library_name_is_reported_once_at_its_first_script(self, tmp_path: Path) -> None: + first = write_module(tmp_path / "venv", "first.py", "") + write_module(tmp_path / "venv", "second.py", "") + + reported = RULE.shadowing(tmp_path, [first, tmp_path / "venv" / "second.py"]) + + assert [violation.kind for violation in reported] == ["venv stands in for a module the tree sits beside"] + assert reported[0].location == str(first) + + def test_a_reserved_name_is_reported(self, tmp_path: Path) -> None: + path = write_module(tmp_path, "tests.py", "") + + assert [violation.kind for violation in RULE.shadowing(tmp_path, [path])] == [ + "tests stands in for a module the tree sits beside", + ] + + def test_a_name_of_the_trees_own_is_left_alone(self, tmp_path: Path) -> None: + path = write_module(tmp_path / "bootstrap", "venv_build.py", "") + + assert RULE.shadowing(tmp_path, [path]) == [] + + +class TestCheckStandalone: + """Every rule read over one tree, from the sweep to the report.""" + + def test_a_clean_tree_reports_nothing(self, tmp_path: Path) -> None: + _tree(tmp_path) + write_module(tmp_path, "bundle.py", REACHABLE) + + assert kinds(tmp_path) == [] + + def test_a_script_in_a_subdirectory_is_held_to_the_rule(self, tmp_path: Path) -> None: + _tree(tmp_path) + write_module(tmp_path / "ci", "archive.py", THIRD_PARTY) + + assert kinds(tmp_path) == [MESSAGE] + + def test_the_shadowing_names_lead_the_imports(self, tmp_path: Path) -> None: + _tree(tmp_path) + write_module(tmp_path, "bundle.py", THIRD_PARTY) + write_module(tmp_path, "platform.py", "") + + assert kinds(tmp_path) == ["platform stands in for a module the tree sits beside", MESSAGE] + + def test_a_selection_narrows_the_check(self, tmp_path: Path) -> None: + _tree(tmp_path) + checked = write_module(tmp_path, "checked.py", THIRD_PARTY) + write_module(tmp_path, "other.py", THIRD_PARTY) + + assert len(check_standalone(tmp_path, (RULE,), {checked.resolve()})) == 1 + + def test_a_tree_holding_no_script_stops_the_check(self, tmp_path: Path) -> None: + with pytest.raises(FileNotFoundError): + check_standalone(tmp_path, (RULE,), None) diff --git a/tests/unit/sampletones_shared/meta/import_boundary/test_token.py b/tests/unit/sampletones_tools/checks/boundary/test_token.py similarity index 96% rename from tests/unit/sampletones_shared/meta/import_boundary/test_token.py rename to tests/unit/sampletones_tools/checks/boundary/test_token.py index d1b3018a9..ec2881921 100644 --- a/tests/unit/sampletones_shared/meta/import_boundary/test_token.py +++ b/tests/unit/sampletones_tools/checks/boundary/test_token.py @@ -4,7 +4,7 @@ import pytest from pydantic import ValidationError -from sampletones_shared.meta.import_boundary.token import TokenRule +from sampletones_tools.checks.boundary.token import TokenRule from tests.suite.source import write_module MESSAGE: Final[str] = "a panel receives its parent through create_panel(parent)" diff --git a/tests/unit/sampletones_shared/meta/import_boundary/test_units.py b/tests/unit/sampletones_tools/checks/boundary/test_units.py similarity index 92% rename from tests/unit/sampletones_shared/meta/import_boundary/test_units.py rename to tests/unit/sampletones_tools/checks/boundary/test_units.py index d824a4adf..a3df988a2 100644 --- a/tests/unit/sampletones_shared/meta/import_boundary/test_units.py +++ b/tests/unit/sampletones_tools/checks/boundary/test_units.py @@ -1,4 +1,4 @@ -from sampletones_shared.meta.import_boundary.units import nested_globs, unit_glob, unit_prefix +from sampletones_tools.checks.boundary.units import nested_globs, unit_glob, unit_prefix PLAYER = "sampletones_player" diff --git a/tests/unit/sampletones_shared/meta/import_boundary/test_violation.py b/tests/unit/sampletones_tools/checks/boundary/test_violation.py similarity index 92% rename from tests/unit/sampletones_shared/meta/import_boundary/test_violation.py rename to tests/unit/sampletones_tools/checks/boundary/test_violation.py index af8e8a0f5..2b4939a8f 100644 --- a/tests/unit/sampletones_shared/meta/import_boundary/test_violation.py +++ b/tests/unit/sampletones_tools/checks/boundary/test_violation.py @@ -1,6 +1,6 @@ from pathlib import Path -from sampletones_shared.meta.import_boundary.violation import Violation +from sampletones_tools.checks.boundary.violation import Violation class TestViolationLocation: diff --git a/tests/unit/sampletones_tools/checks/commands/__init__.py b/tests/unit/sampletones_tools/checks/commands/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/unit/sampletones_tools/checks/commands/test_palette_colors.py b/tests/unit/sampletones_tools/checks/commands/test_palette_colors.py new file mode 100644 index 000000000..2fcedacaa --- /dev/null +++ b/tests/unit/sampletones_tools/checks/commands/test_palette_colors.py @@ -0,0 +1,59 @@ +from pathlib import Path +from typing import Final, List, Tuple + +import pytest + +from sampletones.commands.registry import COMMANDS +from sampletones.dispatcher import dispatch +from sampletones_application.paths import PALETTES_DIRECTORY +from sampletones_shared.paths.resources import CONFIG_DIRECTORY +from sampletones_tools.checks import palette_colors +from sampletones_tools.checks.palette_colors import APPLICATION_PACKAGE, ColorFinding + +GUARD: Final[str] = "sampletones_tools.checkout.require_checkout" + + +class RecordedCheck: + def __init__(self) -> None: + self.checked: List[Tuple[Path, Path, Path]] = [] + + def __call__(self, package: Path, config: Path, palettes: Path) -> List[ColorFinding]: + self.checked.append((package, config, palettes)) + return [] + + +@pytest.fixture(name="check") +def check_fixture(monkeypatch: pytest.MonkeyPatch) -> RecordedCheck: + recorded = RecordedCheck() + guarded: List[str] = [] + monkeypatch.setattr(palette_colors, "check_colors", recorded) + monkeypatch.setattr(GUARD, guarded.append) + return recorded + + +class TestPaletteColorsCommand: + def test_the_shipped_trees_are_checked_when_none_is_named(self, check: RecordedCheck) -> None: + assert dispatch(COMMANDS, ["check", "palette-colors"]) == 0 + assert check.checked == [(APPLICATION_PACKAGE, CONFIG_DIRECTORY, PALETTES_DIRECTORY)] + + def test_the_trees_named_are_the_ones_checked(self, check: RecordedCheck, tmp_path: Path) -> None: + package = tmp_path / "package" + config = tmp_path / "config" + palettes = tmp_path / "palettes" + + status = dispatch( + COMMANDS, + [ + "check", + "palette-colors", + "--package", + str(package), + "--config-directory", + str(config), + "--palettes", + str(palettes), + ], + ) + + assert status == 0 + assert check.checked == [(package, config, palettes)] diff --git a/tests/unit/sampletones_tools/checks/source/__init__.py b/tests/unit/sampletones_tools/checks/source/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/unit/sampletones_tools/checks/source/bindings/__init__.py b/tests/unit/sampletones_tools/checks/source/bindings/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/unit/sampletones_shared/meta/source/bindings/test_containers.py b/tests/unit/sampletones_tools/checks/source/bindings/test_containers.py similarity index 98% rename from tests/unit/sampletones_shared/meta/source/bindings/test_containers.py rename to tests/unit/sampletones_tools/checks/source/bindings/test_containers.py index b66c9d09e..c99070413 100644 --- a/tests/unit/sampletones_shared/meta/source/bindings/test_containers.py +++ b/tests/unit/sampletones_tools/checks/source/bindings/test_containers.py @@ -1,7 +1,7 @@ import ast from typing import Dict, Final, List, Optional, Tuple -from sampletones_shared.meta.source.bindings.containers import ( +from sampletones_tools.checks.source.bindings.containers import ( container_item_types, iterated_container, iterated_types, diff --git a/tests/unit/sampletones_shared/meta/source/bindings/test_environment.py b/tests/unit/sampletones_tools/checks/source/bindings/test_environment.py similarity index 94% rename from tests/unit/sampletones_shared/meta/source/bindings/test_environment.py rename to tests/unit/sampletones_tools/checks/source/bindings/test_environment.py index 79bc0bb9c..a088ae164 100644 --- a/tests/unit/sampletones_shared/meta/source/bindings/test_environment.py +++ b/tests/unit/sampletones_tools/checks/source/bindings/test_environment.py @@ -1,6 +1,6 @@ from typing import Final -from sampletones_shared.meta.source.bindings.environment import TypeEnvironment +from sampletones_tools.checks.source.bindings.environment import TypeEnvironment ENVIRONMENT: Final[TypeEnvironment] = TypeEnvironment( types={ diff --git a/tests/unit/sampletones_shared/meta/source/bindings/test_scopes.py b/tests/unit/sampletones_tools/checks/source/bindings/test_scopes.py similarity index 97% rename from tests/unit/sampletones_shared/meta/source/bindings/test_scopes.py rename to tests/unit/sampletones_tools/checks/source/bindings/test_scopes.py index aa4b6d2aa..f9b7c6a82 100644 --- a/tests/unit/sampletones_shared/meta/source/bindings/test_scopes.py +++ b/tests/unit/sampletones_tools/checks/source/bindings/test_scopes.py @@ -1,7 +1,7 @@ from typing import Final, List, Mapping, Tuple -from sampletones_shared.meta.source.bindings.environment import TypeEnvironment -from sampletones_shared.meta.source.bindings.scopes import Scope, module_scopes +from sampletones_tools.checks.source.bindings.environment import TypeEnvironment +from sampletones_tools.checks.source.bindings.scopes import Scope, module_scopes from tests.suite.source import parse_source, scope_named PANEL_SOURCE: Final[str] = """ diff --git a/tests/unit/sampletones_shared/meta/source/bindings/test_statements.py b/tests/unit/sampletones_tools/checks/source/bindings/test_statements.py similarity index 98% rename from tests/unit/sampletones_shared/meta/source/bindings/test_statements.py rename to tests/unit/sampletones_tools/checks/source/bindings/test_statements.py index 8aa1043fe..6b1467ea5 100644 --- a/tests/unit/sampletones_shared/meta/source/bindings/test_statements.py +++ b/tests/unit/sampletones_tools/checks/source/bindings/test_statements.py @@ -1,7 +1,7 @@ import ast from typing import Optional, Type -from sampletones_shared.meta.source.bindings.statements import ( +from sampletones_tools.checks.source.bindings.statements import ( AliasStatement, LoopStatement, Statement, diff --git a/tests/unit/sampletones_shared/meta/source/test_annotations.py b/tests/unit/sampletones_tools/checks/source/test_annotations.py similarity index 99% rename from tests/unit/sampletones_shared/meta/source/test_annotations.py rename to tests/unit/sampletones_tools/checks/source/test_annotations.py index 36b3de5a7..c00b4c58e 100644 --- a/tests/unit/sampletones_shared/meta/source/test_annotations.py +++ b/tests/unit/sampletones_tools/checks/source/test_annotations.py @@ -4,7 +4,7 @@ import pytest -from sampletones_shared.meta.source.annotations import ( +from sampletones_tools.checks.source.annotations import ( annotation_item_types, annotation_type_name, unwrap_annotation, diff --git a/tests/unit/sampletones_shared/meta/source/test_classes.py b/tests/unit/sampletones_tools/checks/source/test_classes.py similarity index 96% rename from tests/unit/sampletones_shared/meta/source/test_classes.py rename to tests/unit/sampletones_tools/checks/source/test_classes.py index 00603297f..4174ab6ed 100644 --- a/tests/unit/sampletones_shared/meta/source/test_classes.py +++ b/tests/unit/sampletones_tools/checks/source/test_classes.py @@ -1,6 +1,6 @@ from typing import Final, List -from sampletones_shared.meta.source.classes import declared_subclasses +from sampletones_tools.checks.source.classes import declared_subclasses from tests.suite.source import parse_source CLASSES_SOURCE: Final[str] = """ diff --git a/tests/unit/sampletones_shared/meta/source/test_constants.py b/tests/unit/sampletones_tools/checks/source/test_constants.py similarity index 96% rename from tests/unit/sampletones_shared/meta/source/test_constants.py rename to tests/unit/sampletones_tools/checks/source/test_constants.py index 1c76fadd1..6ea61c404 100644 --- a/tests/unit/sampletones_shared/meta/source/test_constants.py +++ b/tests/unit/sampletones_tools/checks/source/test_constants.py @@ -1,7 +1,7 @@ import ast from typing import Dict, Final, List -from sampletones_shared.meta.source.constants import ModuleConstant, module_constants +from sampletones_tools.checks.source.constants import ModuleConstant, module_constants from tests.suite.source import parse_source CONSTANTS_SOURCE: Final[str] = """ diff --git a/tests/unit/sampletones_shared/meta/source/test_index.py b/tests/unit/sampletones_tools/checks/source/test_index.py similarity index 92% rename from tests/unit/sampletones_shared/meta/source/test_index.py rename to tests/unit/sampletones_tools/checks/source/test_index.py index 7e50e97c0..4828305b6 100644 --- a/tests/unit/sampletones_shared/meta/source/test_index.py +++ b/tests/unit/sampletones_tools/checks/source/test_index.py @@ -2,8 +2,8 @@ from pathlib import Path from typing import Final -from sampletones_shared.meta.source.index import SourceIndex, source_index -from sampletones_shared.meta.source.modules import SourceModule +from sampletones_tools.checks.source.index import SourceIndex, source_index +from sampletones_tools.checks.source.modules import SourceModule from tests.suite.source import parse_source TAGS_SOURCE: Final[str] = """ diff --git a/tests/unit/sampletones_shared/meta/source/test_lookups.py b/tests/unit/sampletones_tools/checks/source/test_lookups.py similarity index 96% rename from tests/unit/sampletones_shared/meta/source/test_lookups.py rename to tests/unit/sampletones_tools/checks/source/test_lookups.py index 1db807dbc..613a8da8c 100644 --- a/tests/unit/sampletones_shared/meta/source/test_lookups.py +++ b/tests/unit/sampletones_tools/checks/source/test_lookups.py @@ -4,15 +4,15 @@ import pytest -from sampletones_shared.meta.source.index import source_index -from sampletones_shared.meta.source.lookups import ( +from sampletones_tools.checks.source.index import source_index +from sampletones_tools.checks.source.lookups import ( LookupSite, composed_values, module_lookups, tree_lookups, ) -from sampletones_shared.meta.source.modules import SourceModule -from sampletones_shared.meta.source.values import UNRESOLVED, EnumTable, ResolvedValues +from sampletones_tools.checks.source.modules import SourceModule +from sampletones_tools.checks.source.values import UNRESOLVED, EnumTable, ResolvedValues from tests.suite.base import BaseTestSuite from tests.suite.case import BaseRegularTestCase from tests.suite.source import parse_source diff --git a/tests/unit/sampletones_shared/meta/source/test_modules.py b/tests/unit/sampletones_tools/checks/source/test_modules.py similarity index 97% rename from tests/unit/sampletones_shared/meta/source/test_modules.py rename to tests/unit/sampletones_tools/checks/source/test_modules.py index 777586ccb..bd5cfa7f8 100644 --- a/tests/unit/sampletones_shared/meta/source/test_modules.py +++ b/tests/unit/sampletones_tools/checks/source/test_modules.py @@ -5,7 +5,7 @@ import pytest -from sampletones_shared.meta.source.modules import ( +from sampletones_tools.checks.source.modules import ( discover_modules, is_visible, module_name, @@ -52,7 +52,7 @@ def test_a_location_names_the_path_and_the_line(self, tmp_path: Path) -> None: class TestIsVisible: def test_a_plain_path_is_visible(self) -> None: - assert is_visible(Path("src/sampletones_shared/meta/source/modules.py")) + assert is_visible(Path("src/sampletones_tools/checks/source/modules.py")) def test_a_hidden_directory_hides_the_path(self) -> None: assert not is_visible(Path(".venv/lib/python3.12/ast.py")) diff --git a/tests/unit/sampletones_shared/meta/source/test_nodes.py b/tests/unit/sampletones_tools/checks/source/test_nodes.py similarity index 98% rename from tests/unit/sampletones_shared/meta/source/test_nodes.py rename to tests/unit/sampletones_tools/checks/source/test_nodes.py index 1d68f1b39..fc939274b 100644 --- a/tests/unit/sampletones_shared/meta/source/test_nodes.py +++ b/tests/unit/sampletones_tools/checks/source/test_nodes.py @@ -1,7 +1,7 @@ import ast from typing import Final, Iterable, List -from sampletones_shared.meta.source.nodes import ( +from sampletones_tools.checks.source.nodes import ( expression_spelling, is_attribute_spelling, nested_scopes, diff --git a/tests/unit/sampletones_shared/meta/source/test_packages.py b/tests/unit/sampletones_tools/checks/source/test_packages.py similarity index 57% rename from tests/unit/sampletones_shared/meta/source/test_packages.py rename to tests/unit/sampletones_tools/checks/source/test_packages.py index e18e8b853..9ee79c658 100644 --- a/tests/unit/sampletones_shared/meta/source/test_packages.py +++ b/tests/unit/sampletones_tools/checks/source/test_packages.py @@ -1,31 +1,34 @@ import pytest -from sampletones_shared.meta.source.packages import package_directory from sampletones_shared.paths.source import SOURCE_ROOT +from sampletones_tools.checks.source.packages import source_package_directory SHARED_PACKAGE = "sampletones_shared" APPLICATION_PACKAGE = "sampletones_application" -class TestPackageDirectory: +class TestSourcePackageDirectory: def test_a_top_level_package_sits_under_the_source_root(self) -> None: - assert package_directory(SHARED_PACKAGE) == SOURCE_ROOT / SHARED_PACKAGE + assert source_package_directory(SHARED_PACKAGE) == SOURCE_ROOT / SHARED_PACKAGE def test_a_subpackage_is_named_part_by_part(self) -> None: - assert package_directory(SHARED_PACKAGE, "meta", "source") == SOURCE_ROOT / SHARED_PACKAGE / "meta" / "source" + assert ( + source_package_directory(SHARED_PACKAGE, "utils", "system") + == SOURCE_ROOT / SHARED_PACKAGE / "utils" / "system" + ) def test_the_answer_is_a_directory_a_sweep_reads_under(self) -> None: """A package resource resolves to `__init__.py`, which a sweep reads nothing under.""" - assert package_directory(APPLICATION_PACKAGE, "tags").is_dir() + assert source_package_directory(APPLICATION_PACKAGE, "tags").is_dir() def test_a_package_the_source_root_holds_no_directory_for_raises(self) -> None: with pytest.raises(NotADirectoryError): - package_directory("sampletones_absent") + source_package_directory("sampletones_absent") def test_a_module_named_as_a_package_raises(self) -> None: with pytest.raises(NotADirectoryError): - package_directory(SHARED_PACKAGE, "paths.py") + source_package_directory(SHARED_PACKAGE, "paths.py") def test_the_report_names_the_path_it_looked_at(self) -> None: with pytest.raises(NotADirectoryError, match="sampletones_absent"): - package_directory("sampletones_absent") + source_package_directory("sampletones_absent") diff --git a/tests/unit/sampletones_shared/meta/source/test_references.py b/tests/unit/sampletones_tools/checks/source/test_references.py similarity index 95% rename from tests/unit/sampletones_shared/meta/source/test_references.py rename to tests/unit/sampletones_tools/checks/source/test_references.py index 50456eef8..195d98aa6 100644 --- a/tests/unit/sampletones_shared/meta/source/test_references.py +++ b/tests/unit/sampletones_tools/checks/source/test_references.py @@ -1,6 +1,6 @@ from typing import Dict, Final -from sampletones_shared.meta.source.references import count_identifier_loads +from sampletones_tools.checks.source.references import count_identifier_loads from tests.suite.source import parse_source REFERENCES_SOURCE: Final[str] = """ diff --git a/tests/unit/sampletones_shared/meta/source/test_subscripts.py b/tests/unit/sampletones_tools/checks/source/test_subscripts.py similarity index 93% rename from tests/unit/sampletones_shared/meta/source/test_subscripts.py rename to tests/unit/sampletones_tools/checks/source/test_subscripts.py index 5285a216a..88437fc37 100644 --- a/tests/unit/sampletones_shared/meta/source/test_subscripts.py +++ b/tests/unit/sampletones_tools/checks/source/test_subscripts.py @@ -1,7 +1,7 @@ from typing import Final, List -from sampletones_shared.meta.source.bindings.scopes import module_scopes -from sampletones_shared.meta.source.subscripts import SubscriptSite, find_subscripts +from sampletones_tools.checks.source.bindings.scopes import module_scopes +from sampletones_tools.checks.source.subscripts import SubscriptSite, find_subscripts from tests.suite.source import parse_source, scope_named RECEIVER_TYPE: Final[str] = "LanguageManager" diff --git a/tests/unit/sampletones_shared/meta/source/test_values.py b/tests/unit/sampletones_tools/checks/source/test_values.py similarity index 97% rename from tests/unit/sampletones_shared/meta/source/test_values.py rename to tests/unit/sampletones_tools/checks/source/test_values.py index 733ad0e94..025ab8349 100644 --- a/tests/unit/sampletones_shared/meta/source/test_values.py +++ b/tests/unit/sampletones_tools/checks/source/test_values.py @@ -4,8 +4,8 @@ import pytest -from sampletones_shared.meta.source.bindings.environment import TypeEnvironment -from sampletones_shared.meta.source.values import ( +from sampletones_tools.checks.source.bindings.environment import TypeEnvironment +from sampletones_tools.checks.source.values import ( UNRESOLVED, EnumTable, ResolvedValues, diff --git a/tests/unit/sampletones_tools/checks/test_command.py b/tests/unit/sampletones_tools/checks/test_command.py new file mode 100644 index 000000000..6b3d52989 --- /dev/null +++ b/tests/unit/sampletones_tools/checks/test_command.py @@ -0,0 +1,61 @@ +from argparse import ArgumentParser, Namespace +from typing import Final, List + +import pytest + +from sampletones.commands.registry import COMMANDS +from sampletones.dispatcher import build_parser, dispatch +from sampletones_shared.command import Command +from sampletones_tools.checks import command +from sampletones_tools.checks.registry import GATES + +GUARD: Final[str] = "sampletones_tools.checkout.require_checkout" + + +def _gate(name: str, calls: List[str]) -> Command: + def configure(parser: ArgumentParser) -> None: + parser.add_argument("--flag", action="store_true") + + def run(arguments: Namespace) -> int: + calls.append(f"{name} {arguments.flag}") + return 3 + + return Command(name=name, help=f"{name} help", configure=configure, run=run) + + +class TestCheck: + def test_every_gate_is_listed(self, capsys: pytest.CaptureFixture[str]) -> None: + with pytest.raises(SystemExit) as leaving: + build_parser(COMMANDS).parse_args(["check", "--help"]) + + assert leaving.value.code == 0 + listing = capsys.readouterr().out + assert GATES + assert all(gate.name in listing for gate in GATES) + + def test_the_gate_named_runs_from_a_checkout(self, monkeypatch: pytest.MonkeyPatch) -> None: + calls: List[str] = [] + guarded: List[str] = [] + monkeypatch.setattr(command, "GATES", (_gate("probe", calls),)) + monkeypatch.setattr(GUARD, guarded.append) + + assert dispatch(COMMANDS, ["check", "probe", "--flag"]) == 3 + assert calls == ["probe True"] + assert guarded == ["check probe"] + + def test_an_unknown_check_is_refused(self) -> None: + with pytest.raises(SystemExit) as leaving: + dispatch(COMMANDS, ["check", "absent"]) + + assert leaving.value.code == 2 + + def test_a_check_is_required(self) -> None: + with pytest.raises(SystemExit) as leaving: + dispatch(COMMANDS, ["check"]) + + assert leaving.value.code == 2 + + def test_every_gate_has_a_name_of_its_own(self) -> None: + names = [gate.name for gate in GATES] + + assert len(set(names)) == len(names) diff --git a/tests/unit/sampletones_tools/checks/test_import_boundary.py b/tests/unit/sampletones_tools/checks/test_import_boundary.py new file mode 100644 index 000000000..c5f372b05 --- /dev/null +++ b/tests/unit/sampletones_tools/checks/test_import_boundary.py @@ -0,0 +1,71 @@ +from pathlib import Path +from typing import Final + +import pytest + +from sampletones.commands.registry import COMMANDS +from sampletones.dispatcher import dispatch +from sampletones_tools.checks import import_boundary as check_import_boundary +from tests.suite.source import write_module + +APPLICATION: Final[str] = "sampletones_application" + +VISUAL_IMPORT: Final[str] = "import dearpygui.dearpygui as dpg\n" +PLAIN_IMPORT: Final[str] = "from sampletones_core.project.project import Project\n" +THIRD_PARTY_IMPORT: Final[str] = "import numpy\n" + + +class TestMain: + def test_the_repository_holds_its_import_boundaries(self) -> None: + assert dispatch(COMMANDS, ["check", "import-boundary", "--all"]) == 0 + + def test_a_forbidden_import_is_reported_where_it_sits( + self, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + ) -> None: + path = write_module(tmp_path / APPLICATION / "logic", "direct.py", VISUAL_IMPORT) + + exit_code = dispatch(COMMANDS, ["check", "import-boundary", "--all", "--source", str(tmp_path)]) + + assert exit_code == 1 + error = capsys.readouterr().err + assert f"{path}:1" in error + assert "dearpygui" in error + + def test_a_bootstrap_script_reaching_past_the_standard_library_is_reported( + self, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + ) -> None: + write_module(tmp_path / "src" / APPLICATION / "logic", "clean.py", PLAIN_IMPORT) + path = write_module(tmp_path / "scripts", "bundle.py", THIRD_PARTY_IMPORT) + + exit_code = dispatch( + COMMANDS, + [ + "check", + "import-boundary", + "--all", + "--source", + str(tmp_path / "src"), + "--scripts", + str(tmp_path / "scripts"), + ], + ) + + assert exit_code == 1 + error = capsys.readouterr().err + assert f"{path}:1" in error + assert "system interpreter" in error + + def test_named_files_narrow_the_run_to_themselves( + self, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + ) -> None: + write_module(tmp_path / APPLICATION / "logic", "reported.py", VISUAL_IMPORT) + clean = write_module(tmp_path / APPLICATION / "logic", "clean.py", PLAIN_IMPORT) + + assert dispatch(COMMANDS, ["check", "import-boundary", str(clean), "--source", str(tmp_path)]) == 0 + assert capsys.readouterr().err == "" diff --git a/tests/unit/scripts/checks/test_language_keys.py b/tests/unit/sampletones_tools/checks/test_language_keys.py similarity index 94% rename from tests/unit/scripts/checks/test_language_keys.py rename to tests/unit/sampletones_tools/checks/test_language_keys.py index 3e74ffab2..6f47d2b5b 100644 --- a/tests/unit/scripts/checks/test_language_keys.py +++ b/tests/unit/sampletones_tools/checks/test_language_keys.py @@ -3,15 +3,15 @@ import pytest +from sampletones.commands.registry import COMMANDS +from sampletones.dispatcher import dispatch from sampletones_application.categories.elements.global_ import DialogElements from sampletones_application.logic.history.action import HistoryAction from sampletones_application.paths import LANG_EN -from sampletones_shared.meta.source.lookups import LookupSite -from sampletones_shared.meta.source.modules import source_paths -from sampletones_shared.meta.source.values import EnumTable -from tests.suite.scripts import load_script - -check_language_keys = load_script("checks/language_keys.py") +from sampletones_tools.checks import language_keys as check_language_keys +from sampletones_tools.checks.source.lookups import LookupSite +from sampletones_tools.checks.source.modules import source_paths +from sampletones_tools.checks.source.values import EnumTable ENUMS: Final[EnumTable] = check_language_keys.enum_table() @@ -251,7 +251,7 @@ def test_the_language_file_is_there_to_read(self) -> None: class TestMain: def test_the_repository_and_its_language_file_agree(self) -> None: - assert check_language_keys.main([]) == 0 + assert dispatch(COMMANDS, ["check", "language-keys"]) == 0 def test_a_disagreement_is_reported_by_kind( self, @@ -261,7 +261,9 @@ def test_a_disagreement_is_reported_by_kind( source = source_tree(tmp_path, LOOKUP_SOURCE.format(key=ABSENT_KEY)) entries = language_file(tmp_path, f'{OK_KEY}: "OK"\n') - exit_code = check_language_keys.main(["--source", str(source), "--language-file", str(entries)]) + exit_code = dispatch( + COMMANDS, ["check", "language-keys", "--source", str(source), "--language-file", str(entries)] + ) assert exit_code == 1 error = capsys.readouterr().err diff --git a/tests/unit/scripts/checks/test_palette_colors.py b/tests/unit/sampletones_tools/checks/test_palette_colors.py similarity index 95% rename from tests/unit/scripts/checks/test_palette_colors.py rename to tests/unit/sampletones_tools/checks/test_palette_colors.py index 9e13f2873..c0bc27174 100644 --- a/tests/unit/scripts/checks/test_palette_colors.py +++ b/tests/unit/sampletones_tools/checks/test_palette_colors.py @@ -4,14 +4,12 @@ from pytest import fixture from sampletones_application.paths import PALETTES_DIRECTORY -from sampletones_shared.meta.source.modules import SourceModule, source_paths from sampletones_shared.paths.resources import CONFIG_DIRECTORY -from scripts.checks.palette_colors import dpg_module_helper -from tests.suite.scripts import load_script +from sampletones_tools.checks import palette_colors as check_palette_colors +from sampletones_tools.checks.palette_colors import dpg_module_helper +from sampletones_tools.checks.source.modules import SourceModule, source_paths from tests.suite.source import parse_source -check_palette_colors = load_script("checks/palette_colors.py") - PANEL_MODULE: Final[Path] = Path("ui/panel.py") PANEL_SOURCE: Final[str] = """ diff --git a/tests/unit/sampletones_tools/checks/test_paths.py b/tests/unit/sampletones_tools/checks/test_paths.py new file mode 100644 index 000000000..d94e64cef --- /dev/null +++ b/tests/unit/sampletones_tools/checks/test_paths.py @@ -0,0 +1,6 @@ +from sampletones_tools.checks.paths import SCRIPTS_ROOT + + +class TestScriptsRoot: + def test_the_scripts_root_holds_the_bootstrap_tree(self) -> None: + assert (SCRIPTS_ROOT / "bootstrap" / "__init__.py").is_file() diff --git a/tests/unit/scripts/checks/test_rendered_literals.py b/tests/unit/sampletones_tools/checks/test_rendered_literals.py similarity index 91% rename from tests/unit/scripts/checks/test_rendered_literals.py rename to tests/unit/sampletones_tools/checks/test_rendered_literals.py index e38c0c0f0..833df5d15 100644 --- a/tests/unit/scripts/checks/test_rendered_literals.py +++ b/tests/unit/sampletones_tools/checks/test_rendered_literals.py @@ -4,14 +4,14 @@ import pytest -from sampletones_shared.meta.source.modules import SourceModule +from sampletones.commands.registry import COMMANDS +from sampletones.dispatcher import dispatch +from sampletones_tools.checks import rendered_literals as check_rendered_literals +from sampletones_tools.checks.source.modules import SourceModule from tests.suite.base import BaseTestSuite from tests.suite.case import BaseRegularTestCase -from tests.suite.scripts import load_script from tests.suite.source import parse_source -check_rendered_literals = load_script("checks/rendered_literals.py") - CASE_MODULE: Final[Path] = Path("tests/unit/test_card.py") @@ -99,7 +99,7 @@ def test_it_is_passed_over(self, test_case: TestCase) -> None: class TestMain: def test_no_case_in_the_repository_holds_rendered_text_against_a_literal(self) -> None: - assert check_rendered_literals.main([]) == 0 + assert dispatch(COMMANDS, ["check", "rendered-literals"]) == 0 def test_a_case_that_does_is_reported_where_it_sits( self, @@ -114,7 +114,7 @@ def test_a_case_that_does_is_reported_where_it_sits( encoding="utf-8", ) - exit_code = check_rendered_literals.main(["--tests", str(cases)]) + exit_code = dispatch(COMMANDS, ["check", "rendered-literals", "--tests", str(cases)]) assert exit_code == 1 assert f"{module}:2" in capsys.readouterr().err diff --git a/tests/unit/scripts/checks/test_shortcut_actions.py b/tests/unit/sampletones_tools/checks/test_shortcut_actions.py similarity index 96% rename from tests/unit/scripts/checks/test_shortcut_actions.py rename to tests/unit/sampletones_tools/checks/test_shortcut_actions.py index 569d95c5a..1c821a66a 100644 --- a/tests/unit/scripts/checks/test_shortcut_actions.py +++ b/tests/unit/sampletones_tools/checks/test_shortcut_actions.py @@ -7,12 +7,10 @@ ShortcutCategory, ShortcutId, ) -from sampletones_shared.meta.source.modules import SourceModule -from tests.suite.scripts import load_script +from sampletones_tools.checks import shortcut_actions as check_shortcut_actions +from sampletones_tools.checks.source.modules import SourceModule from tests.suite.source import parse_source -check_shortcut_actions = load_script("checks/shortcut_actions.py") - SCHEME: Final[Path] = Path("keybindings/default.yaml") BOUND_ACTION: Final[ShortcutId] = ShortcutId.NEW_PROJECT FAMILY_ACTION: Final[ShortcutId] = ShortcutId.EXPORT_PROJECT_FAMITRACKER diff --git a/tests/unit/scripts/checks/test_tag_names.py b/tests/unit/sampletones_tools/checks/test_tag_names.py similarity index 93% rename from tests/unit/scripts/checks/test_tag_names.py rename to tests/unit/sampletones_tools/checks/test_tag_names.py index 3b03f4fcb..a9b7d51b4 100644 --- a/tests/unit/scripts/checks/test_tag_names.py +++ b/tests/unit/sampletones_tools/checks/test_tag_names.py @@ -4,17 +4,17 @@ import pytest +from sampletones.commands.registry import COMMANDS +from sampletones.dispatcher import dispatch from sampletones_application.categories.hierarchy import Page, Panel, Widget from sampletones_application.categories.key.tag import TagName -from sampletones_shared.meta.source.modules import SourceModule, source_paths from sampletones_shared.paths.source import SOURCE_ROOT +from sampletones_tools.checks import tag_names as check_tag_names +from sampletones_tools.checks.source.modules import SourceModule, source_paths from tests.suite.base import BaseTestSuite from tests.suite.case import BaseRegularTestCase -from tests.suite.scripts import load_script from tests.suite.source import parse_source -check_tag_names = load_script("checks/tag_names.py") - MODULE_PATH: Final[Path] = Path("src/sampletones_application/tags/general.py") WINDOW_TAG: Final[str] = 'TagName(Page.GLOBAL, Panel.IMPLICIT, Widget.WINDOW, "main")' @@ -157,7 +157,7 @@ def test_the_tags_package_sits_under_the_source_root(self) -> None: class TestMain: def test_the_tags_package_names_its_tags_after_them(self) -> None: - assert check_tag_names.main(["--all"]) == 0 + assert dispatch(COMMANDS, ["check", "tag-names", "--all"]) == 0 def test_a_mistyped_tag_is_reported_where_it_sits( self, @@ -167,7 +167,7 @@ def test_a_mistyped_tag_is_reported_where_it_sits( module = tmp_path / "tags.py" module.write_text(f"TAG_GLOBAL_WINDOW_MIAN = {WINDOW_TAG}\n", encoding="utf-8") - assert check_tag_names.main([str(module)]) == 1 + assert dispatch(COMMANDS, ["check", "tag-names", str(module)]) == 1 error = capsys.readouterr().err assert f"{module}:1" in error @@ -181,5 +181,5 @@ def test_a_well_named_module_reports_nothing( module = tmp_path / "tags.py" module.write_text(f"TAG_GLOBAL_WINDOW_MAIN = {WINDOW_TAG}\n", encoding="utf-8") - assert check_tag_names.main([str(module)]) == 0 + assert dispatch(COMMANDS, ["check", "tag-names", str(module)]) == 0 assert capsys.readouterr().err == "" diff --git a/tests/unit/scripts/checks/test_unused_tags.py b/tests/unit/sampletones_tools/checks/test_unused_tags.py similarity index 91% rename from tests/unit/scripts/checks/test_unused_tags.py rename to tests/unit/sampletones_tools/checks/test_unused_tags.py index 1a6b033f5..633fa86a8 100644 --- a/tests/unit/scripts/checks/test_unused_tags.py +++ b/tests/unit/sampletones_tools/checks/test_unused_tags.py @@ -3,13 +3,13 @@ import pytest -from sampletones_shared.meta.source.modules import SourceModule, source_paths +from sampletones.commands.registry import COMMANDS +from sampletones.dispatcher import dispatch from sampletones_shared.paths.source import SOURCE_ROOT -from tests.suite.scripts import load_script +from sampletones_tools.checks import unused_tags as check_unused_tags +from sampletones_tools.checks.source.modules import SourceModule, source_paths from tests.suite.source import parse_source -check_unused_tags = load_script("checks/unused_tags.py") - TAGS_MODULE: Final[Path] = Path("tags/general.py") PANEL_MODULE: Final[Path] = Path("ui/panel.py") @@ -102,7 +102,7 @@ def test_the_tags_package_sits_under_the_source_root(self) -> None: class TestMain: def test_the_repository_reads_every_fragment_it_declares(self) -> None: - assert check_unused_tags.main([]) == 0 + assert dispatch(COMMANDS, ["check", "unused-tags"]) == 0 def test_an_unread_fragment_is_reported_where_it_sits( self, @@ -116,7 +116,7 @@ def test_an_unread_fragment_is_reported_where_it_sits( sources.mkdir() (sources / "panel.py").write_text("show(TAG_READ)\n", encoding="utf-8") - exit_code = check_unused_tags.main(["--tags", str(tags), "--reference-root", str(sources)]) + exit_code = dispatch(COMMANDS, ["check", "unused-tags", "--tags", str(tags), "--reference-root", str(sources)]) assert exit_code == 1 error = capsys.readouterr().err diff --git a/tests/unit/sampletones_tools/codec/study/report/test_run.py b/tests/unit/sampletones_tools/codec/study/report/test_run.py new file mode 100644 index 000000000..dbdf563ec --- /dev/null +++ b/tests/unit/sampletones_tools/codec/study/report/test_run.py @@ -0,0 +1,38 @@ +import subprocess +from typing import List, Sequence + +import pytest + +from sampletones_tools.codec.study.report import run +from sampletones_tools.codec.study.report.run import UNKNOWN_COMMIT, commit_hash + + +class TestCommitHash: + def test_a_copy_outside_a_checkout_records_unknown_without_asking_git( + self, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + asked: List[Sequence[str]] = [] + monkeypatch.setattr(run, "is_checkout", lambda root: False) + monkeypatch.setattr(run.subprocess, "run", lambda command, **options: asked.append(command)) + + assert commit_hash() == UNKNOWN_COMMIT + assert asked == [] + + def test_a_machine_without_git_records_unknown(self, monkeypatch: pytest.MonkeyPatch) -> None: + def missing_git(command: Sequence[str], **options: object) -> subprocess.CompletedProcess[str]: + raise FileNotFoundError(command[0]) + + monkeypatch.setattr(run, "is_checkout", lambda root: True) + monkeypatch.setattr(run.subprocess, "run", missing_git) + + assert commit_hash() == UNKNOWN_COMMIT + + def test_a_checkout_records_what_git_reports(self, monkeypatch: pytest.MonkeyPatch) -> None: + def reporting_git(command: Sequence[str], **options: object) -> subprocess.CompletedProcess[str]: + return subprocess.CompletedProcess(command, 0, stdout="abc1234\n", stderr="") + + monkeypatch.setattr(run, "is_checkout", lambda root: True) + monkeypatch.setattr(run.subprocess, "run", reporting_git) + + assert commit_hash() == "abc1234" diff --git a/tests/unit/sampletones_tools/codec/study/test_accounting.py b/tests/unit/sampletones_tools/codec/study/test_accounting.py new file mode 100644 index 000000000..f22839b39 --- /dev/null +++ b/tests/unit/sampletones_tools/codec/study/test_accounting.py @@ -0,0 +1,252 @@ +from dataclasses import dataclass +from typing import Dict, Final, Sequence, Tuple + +import pytest + +from sampletones_player.compression.dictionary.phrase import Phrase +from sampletones_player.compression.dictionary.table import phrase_table +from sampletones_player.compression.encode import emit +from sampletones_player.compression.planes.order import PlaneOrder +from sampletones_player.compression.tokens.hold import HoldToken +from sampletones_player.compression.tokens.literal import LiteralToken +from sampletones_player.compression.tokens.phrase import PhraseToken +from sampletones_player.compression.tokens.types import TokenUnion +from sampletones_player.specification.compression import MAX_HOLD_TICKS, TokenTag +from sampletones_tools.codec.study.accounting.coincident import coincident_starts +from sampletones_tools.codec.study.accounting.dictionary import default_counts, plateaus +from sampletones_tools.codec.study.accounting.finding import Finding +from sampletones_tools.codec.study.accounting.pairs import ramps_in_literals, set_holds +from sampletones_tools.codec.study.accounting.runs import ramps, runs +from sampletones_tools.codec.study.accounting.shares import hold_chains +from sampletones_tools.codec.study.accounting.tokens import ReadToken, read_tokens +from tests.suite.base import BaseTestSuite +from tests.suite.case import BaseRegularTestCase + +ESCAPED_PHRASE_ID: Final[int] = 70 +REST: Final[int] = 32 + + +class TestReadTokensReadsBackWhatEmitWrote: + """Every token comes back with its tag, where it starts, what it takes and what it carries.""" + + tokens: Final[Tuple[TokenUnion, ...]] = ( + HoldToken(ticks=3), + LiteralToken(values=b"\x01\x02"), + PhraseToken(phrase_id=2, ticks=4, transpose=0), + PhraseToken(phrase_id=ESCAPED_PHRASE_ID, ticks=2, transpose=3), + ) + + def test_the_tokens_come_back_in_order(self) -> None: + read = read_tokens(emit(self.tokens)) + + assert [token.tag for token in read] == [ + TokenTag.HOLD, + TokenTag.LITERAL, + TokenTag.PHRASE, + TokenTag.TRANSPOSED_PHRASE, + ] + assert [token.tick for token in read] == [0, 3, 5, 9] + assert [token.ticks for token in read] == [3, 2, 4, 2] + assert [token.size for token in read] == [token.size for token in self.tokens] + + def test_the_operands_come_back(self) -> None: + read = read_tokens(emit(self.tokens)) + + assert [token.payload for token in read] == [b"", b"\x01\x02", b"", b""] + assert [token.phrase_id for token in read] == [None, None, 2, ESCAPED_PHRASE_ID] + assert [token.transpose for token in read] == [0, 0, 0, 3] + + +class TestRuns(BaseTestSuite): + @dataclass(frozen=True, kw_only=True) + class TestCase(BaseRegularTestCase): + data: bytes + expected: Tuple[int, ...] + + test_cases = ( + TestCase(label="empty", data=b"", expected=()), + TestCase(label="one value", data=b"\x07", expected=(1,)), + TestCase(label="two plateaus", data=b"\x01\x01\x01\x02\x02", expected=(3, 2)), + TestCase(label="no repeat", data=b"\x01\x02\x03", expected=(1, 1, 1)), + ) + + @pytest.mark.parametrize("test_case", test_cases, ids=lambda test_case: test_case.label) + def test_the_run_lengths_cover_the_data(self, test_case: TestCase) -> None: + assert runs(test_case.data) == test_case.expected + assert sum(test_case.expected) == len(test_case.data) + + +class TestRamps(BaseTestSuite): + @dataclass(frozen=True, kw_only=True) + class TestCase(BaseRegularTestCase): + data: bytes + expected: Tuple[int, ...] + + test_cases = ( + TestCase(label="a rise", data=b"\x01\x02\x03\x04\x05", expected=(5,)), + TestCase(label="a plateau steps nowhere", data=b"\x05\x05\x05", expected=()), + TestCase(label="a rise across the byte wraps", data=b"\xfe\xff\x00\x01", expected=(4,)), + TestCase(label="two steps of different size", data=b"\x01\x02\x04", expected=(2, 2)), + TestCase(label="a fall after a rest", data=b"\x09\x09\x08\x07", expected=(3,)), + ) + + @pytest.mark.parametrize("test_case", test_cases, ids=lambda test_case: test_case.label) + def test_the_constant_step_runs_are_found(self, test_case: TestCase) -> None: + assert ramps(test_case.data) == test_case.expected + + +class TestHoldChains(BaseTestSuite): + """A wide hold pays two bytes for up to sixty-four full holds, and one for the ticks left.""" + + @dataclass(frozen=True, kw_only=True) + class TestCase(BaseRegularTestCase): + holds: Tuple[int, ...] + expected: Finding + + test_cases = ( + TestCase(label="one hold is no chain", holds=(MAX_HOLD_TICKS,), expected=Finding(0, 0)), + TestCase(label="a full hold and a rest break even", holds=(MAX_HOLD_TICKS, REST), expected=Finding(2, 0)), + TestCase( + label="three full holds and a rest spare one", + holds=(MAX_HOLD_TICKS, MAX_HOLD_TICKS, MAX_HOLD_TICKS, REST), + expected=Finding(4, 1), + ), + TestCase( + label="sixty-four full holds become one wide hold", + holds=(MAX_HOLD_TICKS,) * MAX_HOLD_TICKS, + expected=Finding(MAX_HOLD_TICKS, MAX_HOLD_TICKS - 2), + ), + TestCase( + label="sixty-five full holds need two wide holds", + holds=(MAX_HOLD_TICKS,) * (MAX_HOLD_TICKS + 1), + expected=Finding(MAX_HOLD_TICKS + 1, MAX_HOLD_TICKS + 1 - 4), + ), + ) + + @pytest.mark.parametrize("test_case", test_cases, ids=lambda test_case: test_case.label) + def test_the_chain_is_priced_against_wide_holds(self, test_case: TestCase) -> None: + stream = emit([HoldToken(ticks=ticks) for ticks in test_case.holds]) + + assert hold_chains(read_tokens(stream)) == test_case.expected + + def test_a_literal_breaks_the_chain(self) -> None: + stream = emit( + [ + HoldToken(ticks=MAX_HOLD_TICKS), + LiteralToken(values=b"\x01"), + HoldToken(ticks=MAX_HOLD_TICKS), + HoldToken(ticks=MAX_HOLD_TICKS), + ] + ) + + assert hold_chains(read_tokens(stream)) == Finding(2, 0) + + +class TestSetHolds(BaseTestSuite): + @dataclass(frozen=True, kw_only=True) + class TestCase(BaseRegularTestCase): + tokens: Tuple[TokenUnion, ...] + expected: Finding + + test_cases = ( + TestCase( + label="a value then a rest is a pair", + tokens=(LiteralToken(values=b"\x05"), HoldToken(ticks=10)), + expected=Finding(3, 1), + ), + TestCase( + label="two values then a rest is no pair", + tokens=(LiteralToken(values=b"\x05\x06"), HoldToken(ticks=10)), + expected=Finding(0, 0), + ), + TestCase( + label="a rest then a value is no pair", + tokens=(HoldToken(ticks=10), LiteralToken(values=b"\x05")), + expected=Finding(0, 0), + ), + TestCase( + label="every pair counts", + tokens=( + LiteralToken(values=b"\x05"), + HoldToken(ticks=10), + LiteralToken(values=b"\x06"), + HoldToken(ticks=10), + ), + expected=Finding(6, 2), + ), + ) + + @pytest.mark.parametrize("test_case", test_cases, ids=lambda test_case: test_case.label) + def test_the_pairs_are_priced_at_two_bytes(self, test_case: TestCase) -> None: + assert set_holds(read_tokens(emit(list(test_case.tokens)))) == test_case.expected + + +class TestRampsInLiterals(BaseTestSuite): + @dataclass(frozen=True, kw_only=True) + class TestCase(BaseRegularTestCase): + values: bytes + expected: Finding + + test_cases = ( + TestCase(label="a rise of five", values=b"\x01\x02\x03\x04\x05", expected=Finding(5, 2)), + TestCase(label="a rise of three breaks even", values=b"\x01\x02\x03", expected=Finding(3, 0)), + TestCase(label="a rise of two is no ramp", values=b"\x01\x02", expected=Finding(0, 0)), + TestCase(label="a plateau is no ramp", values=b"\x05\x05\x05\x05", expected=Finding(0, 0)), + ) + + @pytest.mark.parametrize("test_case", test_cases, ids=lambda test_case: test_case.label) + def test_the_ramps_are_priced_at_three_bytes(self, test_case: TestCase) -> None: + assert ramps_in_literals(read_tokens(emit([LiteralToken(values=test_case.values)]))) == test_case.expected + + def test_a_ramp_inside_a_hold_is_none(self) -> None: + assert ramps_in_literals(read_tokens(emit([HoldToken(ticks=5)]))) == Finding(0, 0) + + +def _starting_at(ticks: Sequence[int]) -> Tuple[ReadToken, ...]: + return tuple( + ReadToken( + tag=TokenTag.HOLD, + tick=tick, + size=1, + ticks=1, + payload=b"", + phrase_id=None, + transpose=0, + ) + for tick in ticks + ) + + +class TestCoincidentStarts: + def test_a_channel_counts_the_ticks_both_its_planes_start_a_token_on(self) -> None: + tokens: Dict[str, Tuple[ReadToken, ...]] = {name: _starting_at((0,)) for name in PlaneOrder.names()} + tokens["pulse1_control"] = _starting_at((0, 5)) + tokens["pulse1_value"] = _starting_at((0, 5, 9)) + tokens["noise_value"] = _starting_at((0, 7)) + + assert coincident_starts(tokens) == Finding(10, 5) + + +class TestPlateausInBodies: + def test_the_repeats_beyond_a_runs_first_byte_are_targeted(self) -> None: + table = phrase_table((Phrase(body=b"\x01\x01\x01\x02\x02"), Phrase(body=b"\x03\x04"))) + + assert plateaus(table) == Finding(3, 1) + + +class TestDefaultCounts: + def test_the_modal_count_of_each_phrase_spares_its_bytes_but_one(self) -> None: + table = phrase_table((Phrase(body=b"\x01\x02"), Phrase(body=b"\x03\x04"))) + tokens = read_tokens( + emit( + [ + PhraseToken(phrase_id=0, ticks=4, transpose=0), + PhraseToken(phrase_id=0, ticks=4, transpose=0), + PhraseToken(phrase_id=0, ticks=4, transpose=2), + PhraseToken(phrase_id=0, ticks=6, transpose=0), + PhraseToken(phrase_id=1, ticks=2, transpose=0), + ] + ) + ) + + assert default_counts(table, tokens) == Finding(5, 2) diff --git a/tests/unit/sampletones_tools/codec/study/test_manifest.py b/tests/unit/sampletones_tools/codec/study/test_manifest.py new file mode 100644 index 000000000..76802717d --- /dev/null +++ b/tests/unit/sampletones_tools/codec/study/test_manifest.py @@ -0,0 +1,36 @@ +from pathlib import Path + +import pytest + +from sampletones_tools.codec.study.manifest import NAMES_A_SOURCE, StudyManifest, StudySource +from tests.suite.files import empty_file + + +class TestStudySource: + def test_a_file_is_labeled_by_its_stem_and_a_directory_by_its_name(self, tmp_path: Path) -> None: + project = empty_file(tmp_path, "one.stp") + + assert StudySource.at(project) == StudySource(label="one", path=project) + assert StudySource.at(tmp_path) == StudySource(label=tmp_path.name, path=tmp_path) + + def test_a_source_naming_nothing_is_refused(self, tmp_path: Path) -> None: + with pytest.raises(ValueError, match="No file at"): + StudySource.at(tmp_path / "absent.stp") + + +class TestStudyManifest: + def test_a_manifest_naming_no_source_is_refused(self) -> None: + with pytest.raises(ValueError, match=NAMES_A_SOURCE): + StudyManifest(projects=(), reconstructions=(), lengthen_seconds=45, variants=("wide-hold",)) + + def test_a_saved_manifest_loads_as_it_was_written(self, tmp_path: Path) -> None: + written = StudyManifest( + projects=(StudySource.at(empty_file(tmp_path, "one.stp")),), + reconstructions=(StudySource.at(tmp_path),), + lengthen_seconds=45, + variants=("wide-hold",), + ) + path = tmp_path / "manifest.json" + written.save(path) + + assert StudyManifest.load(path) == written diff --git a/tests/unit/sampletones_tools/codec/study/test_plan.py b/tests/unit/sampletones_tools/codec/study/test_plan.py new file mode 100644 index 000000000..8b63028ae --- /dev/null +++ b/tests/unit/sampletones_tools/codec/study/test_plan.py @@ -0,0 +1,32 @@ +from pathlib import Path +from typing import Tuple + +import pytest + +from sampletones_tools.codec.study.manifest import StudyManifest, StudySource +from sampletones_tools.codec.study.plan import plan_study +from sampletones_tools.codec.study.variants.production import BASELINE_NAME +from tests.suite.files import empty_file + + +def _manifest(directory: Path, variants: Tuple[str, ...]) -> StudyManifest: + return StudyManifest( + projects=(StudySource.at(empty_file(directory, "one.stp")),), + reconstructions=(), + lengthen_seconds=45, + variants=variants, + ) + + +class TestPlanStudy: + def test_the_variants_are_the_ones_the_manifest_names_after_the_baseline(self, tmp_path: Path) -> None: + manifest = _manifest(tmp_path, ("wide-hold",)) + + plan = plan_study(manifest) + + assert plan.manifest == manifest + assert [variant.name for variant in plan.variants] == [BASELINE_NAME, "wide-hold"] + + def test_an_unknown_variant_is_refused(self, tmp_path: Path) -> None: + with pytest.raises(ValueError, match="No variant is called bogus"): + plan_study(_manifest(tmp_path, ("bogus",))) diff --git a/tests/unit/sampletones_tools/codec/study/test_sandbox.py b/tests/unit/sampletones_tools/codec/study/test_sandbox.py new file mode 100644 index 000000000..5fc9699e3 --- /dev/null +++ b/tests/unit/sampletones_tools/codec/study/test_sandbox.py @@ -0,0 +1,397 @@ +from dataclasses import dataclass, replace +from pathlib import Path +from random import Random +from typing import Final, FrozenSet, List, Sequence, Tuple + +import pytest + +from sampletones_player.compression.dictionary.phrase import Phrase +from sampletones_player.compression.dictionary.table import PhraseTable, phrase_table +from sampletones_player.compression.encode import STREAM_START +from sampletones_player.compression.matches.cache import MatchCache +from sampletones_player.compression.matches.index import PlaneIndex +from sampletones_player.compression.matches.matcher import PhraseMatcher +from sampletones_player.compression.options import EVERY_LAYER +from sampletones_player.compression.parse.boundaries import Boundaries +from sampletones_player.compression.parse.plane import parse_plane as parse_production +from sampletones_player.compression.pitch import PitchTable +from sampletones_player.compression.planes.order import PlaneOrder +from sampletones_player.compression.planes.song import SongPlanes +from sampletones_player.specification.compression import ( + CHEAP_PHRASE_IDS, + MAX_HOLD_TICKS, + PLANE_COUNT, +) +from sampletones_shared.music import Tuning +from sampletones_tools.codec.study.corpus.song import SongGroup, StudySong +from sampletones_tools.codec.study.sandbox.context import PlaneContext +from sampletones_tools.codec.study.sandbox.costs import PRODUCTION_COSTS, SET_HOLD_BOUND, Costs +from sampletones_tools.codec.study.sandbox.decode import play_tokens +from sampletones_tools.codec.study.sandbox.defaults import modal_counts, no_defaults +from sampletones_tools.codec.study.sandbox.edges.generator import offered_lengths +from sampletones_tools.codec.study.sandbox.encode import encode_grammar +from sampletones_tools.codec.study.sandbox.grammar import BASELINE_GRAMMAR, Grammar +from sampletones_tools.codec.study.sandbox.parse import StudyParse, parse_plane +from sampletones_tools.codec.study.sandbox.reference import reference +from sampletones_tools.codec.study.sandbox.tokens import Hold, Literal, Play, SetHold, StudyToken, WideHold +from sampletones_tools.codec.study.sandbox.verify import verify_baseline +from sampletones_tools.codec.study.variants.production import compress_baseline +from sampletones_tools.codec.study.variants.sandbox import DEFAULT_COUNT_COSTS, GRAMMAR_VARIANTS, GrammarVariant +from tests.suite.base import BaseTestSuite +from tests.suite.case import BaseRegularTestCase + +ENTRIES: Final[FrozenSet[int]] = frozenset({STREAM_START}) +FIGURES: Final[Tuple[bytes, ...]] = (b"\x0a\x0c\x0f\x0f\x0f\x0f\x0f\x0f", b"\x03\x04\x05\x06") +TABLE: Final[PhraseTable] = phrase_table(Phrase(body=body) for body in FIGURES) +NO_TABLE: Final[PhraseTable] = phrase_table(()) +SONG_TICKS: Final[int] = 200 +SEED: Final[int] = 20260912 +WIDE_HOLD: Final[Grammar] = replace(BASELINE_GRAMMAR, wide_hold=True) +START_HOLD: Final[Grammar] = replace(BASELINE_GRAMMAR, start_hold=True) +EVERY_HOLD: Final[Grammar] = replace(BASELINE_GRAMMAR, every_length=True) +SET_HOLD_ESCAPE: Final[Grammar] = replace(BASELINE_GRAMMAR, set_hold=True) +SET_HOLD_REALLOCATED: Final[Grammar] = replace( + BASELINE_GRAMMAR, set_hold=True, costs=replace(PRODUCTION_COSTS, set_hold=SET_HOLD_BOUND) +) +DEFAULT_COUNTS: Final[Grammar] = replace(BASELINE_GRAMMAR, default_counts=True, costs=DEFAULT_COUNT_COSTS) + + +def _figures_plane(random: Random, ticks: int) -> bytes: + values = bytearray() + while len(values) < ticks: + figure = random.choice(FIGURES) + shift = random.randrange(20) + values.extend((value + shift) % 256 for value in figure) + values.extend([values[-1]] * random.randrange(7)) + + return bytes(values[:ticks]) + + +def _runs_plane(random: Random, ticks: int) -> bytes: + values = bytearray() + while len(values) < ticks: + values.extend([random.randrange(16)] * random.randint(1, 40)) + + return bytes(values[:ticks]) + + +def _dense_plane(random: Random, ticks: int) -> bytes: + return bytes(random.randrange(50) for _ in range(ticks)) + + +def _song(ticks: int) -> StudySong: + random = Random(SEED) + planes: List[bytes] = [] + for plane in range(PLANE_COUNT): + if plane % 3 == 2: + planes.append(bytes(ticks)) + elif plane % 3 == 0: + planes.append(_runs_plane(random, ticks)) + else: + planes.append(_figures_plane(random, ticks)) + + return StudySong( + name="song", + group=SongGroup.PROJECT, + source=Path("song.stp"), + planes=SongPlanes.from_order(PlaneOrder.across(planes)), + seeds=tuple(Phrase(body=body) for body in FIGURES), + pitches=PitchTable.from_tuning(Tuning()), + ) + + +def _context( + plane: bytes, + table: PhraseTable, + costs: Costs, + defaults: Sequence[int], +) -> PlaneContext: + cache = MatchCache([PlaneIndex.from_plane(plane)]) + return PlaneContext( + index=cache.index(0), + matcher=PhraseMatcher(table, 0, cache), + boundaries=Boundaries.across(len(plane), ENTRIES), + transposition=EVERY_LAYER.transposition, + defaults=tuple(defaults), + costs=costs, + ) + + +def _parse( + plane: bytes, + table: PhraseTable, + grammar: Grammar, + defaults: Sequence[int], +) -> StudyParse: + return parse_plane(_context(plane, table, grammar.costs, defaults), grammar) + + +def _production_size(plane: bytes, table: PhraseTable) -> int: + cache = MatchCache([PlaneIndex.from_plane(plane)]) + return parse_production(PhraseMatcher(table, 0, cache), EVERY_LAYER, ENTRIES).size + + +class TestTheBaselineGrammarReadsAsTheCodec(BaseTestSuite): + @dataclass(frozen=True, kw_only=True) + class TestCase(BaseRegularTestCase): + plane: bytes + + test_cases = ( + TestCase(label="an idle plane", plane=bytes(SONG_TICKS)), + TestCase(label="a ramp", plane=bytes(range(SONG_TICKS))), + TestCase(label="two runs", plane=bytes([5] * 10 + [7] * 10)), + TestCase(label="figures at several pitches", plane=_figures_plane(Random(SEED), SONG_TICKS)), + TestCase(label="runs of many lengths", plane=_runs_plane(Random(SEED), SONG_TICKS)), + TestCase(label="dense values", plane=_dense_plane(Random(SEED), SONG_TICKS)), + ) + + @pytest.mark.parametrize("test_case", test_cases, ids=lambda test_case: test_case.label) + def test_the_plane_costs_what_the_production_parser_charges(self, test_case: TestCase) -> None: + parse = _parse(test_case.plane, TABLE, BASELINE_GRAMMAR, no_defaults(len(TABLE))) + + assert parse.size == _production_size(test_case.plane, TABLE) + + @pytest.mark.parametrize("test_case", test_cases, ids=lambda test_case: test_case.label) + def test_the_tokens_play_back(self, test_case: TestCase) -> None: + parse = _parse(test_case.plane, TABLE, BASELINE_GRAMMAR, no_defaults(len(TABLE))) + + assert play_tokens(parse.tokens, TABLE, len(test_case.plane)) == test_case.plane + + +class TestPlayTokens(BaseTestSuite): + @dataclass(frozen=True, kw_only=True) + class TestCase(BaseRegularTestCase): + tokens: Tuple[StudyToken, ...] + expected: bytes + + test_cases = ( + TestCase( + label="a hold keeps the value a literal reached", + tokens=(Literal(values=b"\x07"), Hold(ticks=3)), + expected=b"\x07\x07\x07\x07", + ), + TestCase( + label="a hold opening the stream keeps the seeded value", + tokens=(Hold(ticks=2),), + expected=b"\x00\x00", + ), + TestCase( + label="a wide hold counts blocks of the longest plain hold", + tokens=(Literal(values=b"\x09"), WideHold(blocks=2)), + expected=b"\x09" * (1 + 2 * MAX_HOLD_TICKS), + ), + TestCase( + label="a set-hold takes its value and keeps it", + tokens=(SetHold(value=5, ticks=3), Hold(ticks=1)), + expected=b"\x05\x05\x05\x05", + ), + TestCase( + label="a phrase played past its end holds its final value", + tokens=(Play(phrase_id=1, ticks=6, transpose=0, default=False),), + expected=b"\x03\x04\x05\x06\x06\x06", + ), + TestCase( + label="a shifted phrase plays every value shifted", + tokens=(Play(phrase_id=1, ticks=2, transpose=10, default=True),), + expected=b"\x0d\x0e", + ), + ) + + @pytest.mark.parametrize("test_case", test_cases, ids=lambda test_case: test_case.label) + def test_the_values_are_played(self, test_case: TestCase) -> None: + assert play_tokens(test_case.tokens, TABLE, len(test_case.expected)) == test_case.expected + + def test_the_song_end_cuts_the_final_token(self) -> None: + assert play_tokens((Literal(values=b"\x01"), Hold(ticks=5)), NO_TABLE, 3) == b"\x01\x01\x01" + + +class TestGrammarsPriceAPlane(BaseTestSuite): + @dataclass(frozen=True, kw_only=True) + class TestCase(BaseRegularTestCase): + plane: bytes + table: PhraseTable + grammar: Grammar + defaults: Tuple[int, ...] + expected: int + + idle = bytes(SONG_TICKS) + two_runs = bytes([5] * 10 + [7] * 10) + run_then_figure = bytes([5] * 6 + [9] * 5) + figure_table = phrase_table((Phrase(body=b"\x05\x05\x09"),)) + repeated = bytes(b"\x01\x02\x03\x03" * 3) + repeated_table = phrase_table((Phrase(body=b"\x01\x02\x03"),)) + test_cases = ( + TestCase( + label="an idle plane pays a literal and a hold per block", + plane=idle, + table=NO_TABLE, + grammar=BASELINE_GRAMMAR, + defaults=(), + expected=6, + ), + TestCase( + label="a hold may open the stream over the seeded value", + plane=idle, + table=NO_TABLE, + grammar=START_HOLD, + defaults=(), + expected=4, + ), + TestCase( + label="a wide hold covers the full blocks at once", + plane=idle, + table=NO_TABLE, + grammar=WIDE_HOLD, + defaults=(), + expected=5, + ), + TestCase( + label="a wide hold opening the stream costs two bytes and a remainder", + plane=idle, + table=NO_TABLE, + grammar=replace(WIDE_HOLD, start_hold=True), + defaults=(), + expected=3, + ), + TestCase( + label="two runs pay a literal and a hold each", + plane=two_runs, + table=NO_TABLE, + grammar=BASELINE_GRAMMAR, + defaults=(), + expected=6, + ), + TestCase( + label="a three-byte set-hold breaks even with a literal and a hold", + plane=two_runs, + table=NO_TABLE, + grammar=SET_HOLD_ESCAPE, + defaults=(), + expected=6, + ), + TestCase( + label="a two-byte set-hold spares the hold", + plane=two_runs, + table=NO_TABLE, + grammar=SET_HOLD_REALLOCATED, + defaults=(), + expected=4, + ), + TestCase( + label="the longest hold alone overshoots where a figure begins", + plane=run_then_figure, + table=figure_table, + grammar=BASELINE_GRAMMAR, + defaults=(0,), + expected=6, + ), + TestCase( + label="a shorter hold lets the figure start where it matches", + plane=run_then_figure, + table=figure_table, + grammar=EVERY_HOLD, + defaults=(0,), + expected=5, + ), + TestCase( + label="a phrase played at its default count carries no count", + plane=repeated, + table=repeated_table, + grammar=DEFAULT_COUNTS, + defaults=(4,), + expected=3, + ), + TestCase( + label="a phrase played at another count carries it", + plane=repeated, + table=repeated_table, + grammar=DEFAULT_COUNTS, + defaults=(3,), + expected=6, + ), + ) + + @pytest.mark.parametrize("test_case", test_cases, ids=lambda test_case: test_case.label) + def test_the_plane_costs_what_the_grammar_charges(self, test_case: TestCase) -> None: + parse = _parse(test_case.plane, test_case.table, test_case.grammar, test_case.defaults) + + assert parse.size == test_case.expected + assert play_tokens(parse.tokens, test_case.table, len(test_case.plane)) == test_case.plane + + +class TestCosts: + def test_a_phrase_beyond_the_cheap_ids_pays_the_escape(self) -> None: + assert PRODUCTION_COSTS.phrase(CHEAP_PHRASE_IDS - 1, 0, default=False) == 2 + assert PRODUCTION_COSTS.phrase(CHEAP_PHRASE_IDS, 0, default=False) == 3 + assert PRODUCTION_COSTS.phrase(0, 3, default=False) == 3 + + def test_default_counts_halve_the_cheap_ids_and_widen_the_entries(self) -> None: + assert DEFAULT_COUNT_COSTS.phrase(30, 0, default=True) == 1 + assert DEFAULT_COUNT_COSTS.phrase(31, 0, default=True) == 2 + assert DEFAULT_COUNT_COSTS.dictionary(TABLE) == TABLE.size + len(TABLE) + + def test_offered_lengths_are_the_longest_alone_or_every_one(self) -> None: + assert tuple(offered_lengths(5, least=2, every=False)) == (5,) + assert tuple(offered_lengths(5, least=2, every=True)) == (2, 3, 4, 5) + + +class TestModalCounts: + def test_the_count_played_most_often_is_the_default(self) -> None: + parse = StudyParse( + tokens=( + Play(phrase_id=0, ticks=4, transpose=0, default=False), + Play(phrase_id=0, ticks=8, transpose=2, default=False), + Play(phrase_id=0, ticks=4, transpose=0, default=True), + Hold(ticks=1), + ), + costs=(0, 2, 5, 6, 7), + ) + + assert modal_counts((parse,), 2) == (4, 0) + + def test_a_tie_takes_the_shorter_count(self) -> None: + parse = StudyParse( + tokens=( + Play(phrase_id=0, ticks=8, transpose=0, default=False), + Play(phrase_id=0, ticks=4, transpose=0, default=False), + ), + costs=(0, 2, 4), + ) + + assert modal_counts((parse,), 1) == (4,) + + +class TestTheReferenceHoldsTheSandboxToTheCodec: + song = _song(SONG_TICKS) + compressed = compress_baseline(song) + + def test_the_baseline_parse_costs_what_the_codec_wrote(self) -> None: + read = reference(self.song, self.compressed) + + assert [parse.size for parse in read.baseline] == [len(stream) for stream in self.compressed.streams] + + def test_a_stream_priced_differently_stops_the_study(self) -> None: + read = reference(self.song, self.compressed) + tampered = self.compressed.model_copy( + update={"streams": PlaneOrder.across([stream + b"\x00" for stream in self.compressed.streams])} + ) + + with pytest.raises(ValueError, match="parted from the production parser"): + verify_baseline(read.baseline, tampered) + + @pytest.mark.parametrize("entry", GRAMMAR_VARIANTS, ids=lambda entry: entry.name) + def test_every_grammar_plays_back_and_adds_no_bytes_over_a_small_table(self, entry: GrammarVariant) -> None: + read = reference(self.song, self.compressed) + + encoding = encode_grammar(read, entry.grammar) + + assert encoding.lossless + assert encoding.written is None + assert sum(encoding.streams) <= sum(len(stream) for stream in self.compressed.streams) + + def test_wide_holds_opening_the_stream_reduce_an_idle_plane_to_three_bytes(self) -> None: + read = reference(self.song, self.compressed) + + encoding = encode_grammar(read, replace(WIDE_HOLD, start_hold=True)) + + assert [encoding.streams[plane] for plane in range(2, PLANE_COUNT, 3)] == [3, 3, 3] diff --git a/tests/unit/sampletones_tools/codec/study/test_session.py b/tests/unit/sampletones_tools/codec/study/test_session.py new file mode 100644 index 000000000..b9a645832 --- /dev/null +++ b/tests/unit/sampletones_tools/codec/study/test_session.py @@ -0,0 +1,99 @@ +from pathlib import Path + +import pytest + +from sampletones_tools.codec.study.manifest import NAMES_A_SOURCE, StudyManifest, StudySource +from sampletones_tools.codec.study.session import resolve_manifest, variant_names +from sampletones_tools.codec.study.variants.registry import EVERY_VARIANT +from tests.suite.files import empty_file + + +class TestVariantNames: + def test_nothing_named_is_every_variant(self) -> None: + assert variant_names(None) == (EVERY_VARIANT,) + + def test_names_are_read_in_order_with_their_spaces_stripped(self) -> None: + assert variant_names("wide-hold, default-count") == ("wide-hold", "default-count") + + +class TestResolveManifest: + def test_sources_named_outright_are_measured(self, tmp_path: Path) -> None: + project = empty_file(tmp_path / "songs", "one.stp") + stems = tmp_path / "stems" / "two" + stems.mkdir(parents=True) + + manifest = resolve_manifest( + None, + projects=(project,), + reconstructions=(stems,), + lengthen_seconds=30, + variants=("wide-hold",), + ) + + assert manifest.projects == (StudySource(label="one", path=project),) + assert manifest.reconstructions == (StudySource(label="two", path=stems),) + assert manifest.lengthen_seconds == 30 + assert manifest.variants == ("wide-hold",) + + def test_a_run_naming_no_source_and_no_manifest_is_refused(self) -> None: + with pytest.raises(ValueError, match=NAMES_A_SOURCE): + resolve_manifest( + None, + projects=(), + reconstructions=(), + lengthen_seconds=30, + variants=(EVERY_VARIANT,), + ) + + def test_a_manifest_missing_from_its_path_is_refused(self, tmp_path: Path) -> None: + with pytest.raises(ValueError, match="No manifest at"): + resolve_manifest( + tmp_path / "absent.json", + projects=(), + reconstructions=(), + lengthen_seconds=30, + variants=(EVERY_VARIANT,), + ) + + def test_a_manifest_file_is_measured_as_it_stands(self, tmp_path: Path) -> None: + written = StudyManifest( + projects=(StudySource.at(empty_file(tmp_path, "one.stp")),), + reconstructions=(), + lengthen_seconds=45, + variants=("wide-hold",), + ) + path = tmp_path / "manifest.json" + written.save(path) + + manifest = resolve_manifest( + path, + projects=(), + reconstructions=(), + lengthen_seconds=30, + variants=(EVERY_VARIANT,), + ) + + assert manifest == written + + def test_sources_named_outright_replace_a_manifest_s_own_and_keep_its_sweep(self, tmp_path: Path) -> None: + path = tmp_path / "manifest.json" + StudyManifest( + projects=(StudySource.at(empty_file(tmp_path, "one.stp")),), + reconstructions=(), + lengthen_seconds=45, + variants=("wide-hold",), + ).save(path) + stems = tmp_path / "stems" / "two" + stems.mkdir(parents=True) + + manifest = resolve_manifest( + path, + projects=(), + reconstructions=(stems,), + lengthen_seconds=30, + variants=(EVERY_VARIANT,), + ) + + assert manifest.projects == () + assert manifest.reconstructions == (StudySource(label="two", path=stems),) + assert (manifest.lengthen_seconds, manifest.variants) == (45, ("wide-hold",)) diff --git a/tests/unit/sampletones_tools/codec/study/test_variants.py b/tests/unit/sampletones_tools/codec/study/test_variants.py new file mode 100644 index 000000000..32e842639 --- /dev/null +++ b/tests/unit/sampletones_tools/codec/study/test_variants.py @@ -0,0 +1,152 @@ +from dataclasses import dataclass +from pathlib import Path +from typing import Final, Tuple + +import pytest + +from sampletones_player.compression.compressed import CompressedPlanes +from sampletones_player.compression.dictionary.phrase import Phrase +from sampletones_player.compression.dictionary.table import phrase_table +from sampletones_player.compression.encode import emit +from sampletones_player.compression.pitch import PitchTable +from sampletones_player.compression.planes.order import PlaneOrder +from sampletones_player.compression.planes.song import SongPlanes +from sampletones_player.compression.tokens.hold import HoldToken +from sampletones_player.compression.tokens.literal import LiteralToken +from sampletones_player.specification.compression import PLANE_COUNT +from sampletones_shared.music import Tuning +from sampletones_tools.codec.study.corpus.song import SongGroup, StudySong +from sampletones_tools.codec.study.measure import Measurement, production_encoding +from sampletones_tools.codec.study.variants.seeds import split, trimmed, whole_and_split +from sampletones_tools.codec.study.variants.strategy import DEPTH_PREFIX, depth_measurements +from tests.suite.base import BaseTestSuite +from tests.suite.case import BaseRegularTestCase + +TICKS: Final[int] = 4 +BASELINE: Final[str] = "baseline" +TRIMMED: Final[str] = "seeds-trimmed" +WIDE: Final[str] = "search-wide" +ORDER: Final[Tuple[str, ...]] = (BASELINE, TRIMMED, "seeds-split-4", WIDE) + + +class TestTrimmed(BaseTestSuite): + @dataclass(frozen=True, kw_only=True) + class TestCase(BaseRegularTestCase): + body: bytes + expected: bytes + + test_cases = ( + TestCase( + label="a trailing plateau ends at its first value", body=b"\x01\x02\x03\x03\x03", expected=b"\x01\x02\x03" + ), + TestCase(label="a body of one value keeps one", body=b"\x05\x05", expected=b"\x05"), + TestCase(label="a body moving to its end stays", body=b"\x01\x02", expected=b"\x01\x02"), + TestCase(label="a plateau inside the body stays", body=b"\x01\x01\x02", expected=b"\x01\x01\x02"), + ) + + @pytest.mark.parametrize("test_case", test_cases, ids=lambda test_case: test_case.label) + def test_the_tail_is_cut(self, test_case: TestCase) -> None: + assert trimmed((Phrase(body=test_case.body),)) == (Phrase(body=test_case.expected),) + + +class TestSplit(BaseTestSuite): + @dataclass(frozen=True, kw_only=True) + class TestCase(BaseRegularTestCase): + body: bytes + threshold: int + expected: Tuple[bytes, ...] + + test_cases = ( + TestCase( + label="plateaus of three cut the body", + body=b"\x01\x02\x02\x02\x03\x04\x04\x04\x04\x05", + threshold=3, + expected=(b"\x01\x02", b"\x03\x04"), + ), + TestCase( + label="a threshold above every plateau keeps the body whole", + body=b"\x01\x02\x02\x02\x03", + threshold=4, + expected=(b"\x01\x02\x02\x02\x03",), + ), + TestCase( + label="plateaus of two cut too", + body=b"\x01\x01\x02\x03\x03\x04", + threshold=2, + expected=(b"\x02\x03",), + ), + TestCase( + label="a piece of one value is dropped", + body=b"\x01\x02\x02\x02\x03", + threshold=3, + expected=(b"\x01\x02",), + ), + ) + + @pytest.mark.parametrize("test_case", test_cases, ids=lambda test_case: test_case.label) + def test_the_pieces_end_where_a_plateau_begins(self, test_case: TestCase) -> None: + assert split((Phrase(body=test_case.body),), test_case.threshold) == tuple( + Phrase(body=piece) for piece in test_case.expected + ) + + def test_whole_and_split_offers_both(self) -> None: + seeds = (Phrase(body=b"\x01\x02\x02\x02\x03\x04"),) + + assert whole_and_split(seeds, 3) == (*seeds, Phrase(body=b"\x01\x02"), Phrase(body=b"\x03\x04")) + + +def _song() -> StudySong: + return StudySong( + name="song", + group=SongGroup.PROJECT, + source=Path("song.stp"), + planes=SongPlanes.from_order(PlaneOrder.across([bytes(TICKS)] * PLANE_COUNT)), + seeds=(), + pitches=PitchTable.from_tuning(Tuning()), + ) + + +def _measurement( + song: StudySong, + variant: str, + *, + spelled_out: bool, + seconds: float, +) -> Measurement: + stream = emit([LiteralToken(values=bytes(TICKS))] if spelled_out else [HoldToken(ticks=TICKS)]) + compressed = CompressedPlanes( + phrases=phrase_table(()), + streams=PlaneOrder.across([stream] * PLANE_COUNT), + ticks=TICKS, + ) + return Measurement( + song=song, + variant=variant, + encoding=production_encoding(song, compressed, seconds), + ) + + +class TestDepthMeasurements: + def test_each_depth_keeps_the_smallest_so_far_and_sums_the_time(self) -> None: + song = _song() + measurements = ( + _measurement(song, BASELINE, spelled_out=True, seconds=1.0), + _measurement(song, TRIMMED, spelled_out=False, seconds=2.0), + _measurement(song, WIDE, spelled_out=True, seconds=4.0), + ) + + depths = depth_measurements(measurements, ORDER) + + assert [depth.variant for depth in depths] == [f"{DEPTH_PREFIX}{depth}" for depth in (1, 2, 3)] + assert [depth.block for depth in depths] == [ + measurements[0].block, + measurements[1].block, + measurements[1].block, + ] + assert [depth.seconds for depth in depths] == [1.0, 3.0, 7.0] + + def test_a_strategy_no_song_was_measured_under_takes_no_depth(self) -> None: + song = _song() + measurements = (_measurement(song, BASELINE, spelled_out=True, seconds=1.0),) + + assert [depth.variant for depth in depth_measurements(measurements, ORDER)] == [f"{DEPTH_PREFIX}1"] diff --git a/tests/unit/sampletones_tools/codec/study/test_verdicts.py b/tests/unit/sampletones_tools/codec/study/test_verdicts.py new file mode 100644 index 000000000..e4530544f --- /dev/null +++ b/tests/unit/sampletones_tools/codec/study/test_verdicts.py @@ -0,0 +1,195 @@ +from dataclasses import dataclass, replace +from pathlib import Path +from typing import Final, Optional, Tuple + +import pytest + +from sampletones_player.compression.pitch import PitchTable +from sampletones_player.compression.planes.order import PlaneOrder +from sampletones_player.compression.planes.song import SongPlanes +from sampletones_player.specification.compression import PLANE_COUNT +from sampletones_shared.music import Tuning +from sampletones_tools.codec.study.corpus.song import SongGroup, StudySong +from sampletones_tools.codec.study.measure import Encoding, Measurement +from sampletones_tools.codec.study.report.aggregate import group_rows +from sampletones_tools.codec.study.report.verdicts import Verdict, judge, verdict_rows +from sampletones_tools.codec.study.variants.variant import Variant, VariantKind +from tests.suite.base import BaseTestSuite +from tests.suite.case import BaseRegularTestCase + +TICKS: Final[int] = 4 +BASELINE: Final[str] = "baseline" +STREAM: Final[int] = 100 + + +class TestJudge(BaseTestSuite): + @dataclass(frozen=True, kw_only=True) + class TestCase(BaseRegularTestCase): + projects: Optional[float] + reconstructions: Optional[float] + worst: float + priced: bool + expected: Verdict + + test_cases = ( + TestCase( + label="saving enough over the projects graduates", + projects=-0.03, + reconstructions=0.0, + worst=0.0, + priced=False, + expected=Verdict.GRADUATES, + ), + TestCase( + label="saving enough over the reconstructions graduates", + projects=0.0, + reconstructions=-0.05, + worst=-0.02, + priced=False, + expected=Verdict.GRADUATES, + ), + TestCase( + label="a token-level price is provisional", + projects=-0.1, + reconstructions=-0.1, + worst=0.0, + priced=True, + expected=Verdict.PROVISIONAL, + ), + TestCase( + label="a song growing past the bar rejects the variant", + projects=-0.1, + reconstructions=-0.1, + worst=0.011, + priced=False, + expected=Verdict.REJECTED, + ), + TestCase( + label="saving too little rejects the variant", + projects=-0.029, + reconstructions=-0.049, + worst=0.0, + priced=False, + expected=Verdict.REJECTED, + ), + TestCase( + label="a group the variant left unmeasured counts for nothing", + projects=None, + reconstructions=-0.05, + worst=0.0, + priced=False, + expected=Verdict.GRADUATES, + ), + ) + + @pytest.mark.parametrize("test_case", test_cases, ids=lambda test_case: test_case.label) + def test_the_rule_answers(self, test_case: TestCase) -> None: + verdict = judge( + test_case.projects, + test_case.reconstructions, + test_case.worst, + priced=test_case.priced, + ) + + assert verdict is test_case.expected + + +def _song(name: str, group: SongGroup) -> StudySong: + return StudySong( + name=name, + group=group, + source=Path(f"{name}.stp"), + planes=SongPlanes.from_order(PlaneOrder.across([bytes(TICKS)] * PLANE_COUNT)), + seeds=(), + pitches=PitchTable.from_tuning(Tuning()), + ) + + +def _measurement( + song: StudySong, + variant: str, + streams: int, +) -> Measurement: + return Measurement( + song=song, + variant=variant, + encoding=Encoding( + phrases=0, + dictionary=0, + streams=(streams,) + (0,) * (PLANE_COUNT - 1), + seconds=0.0, + lossless=True, + written=None, + ), + ) + + +def _variant(name: str, kind: VariantKind) -> Variant: + return Variant( + name=name, + hypothesis="H", + kind=kind, + note="", + encode=lambda song: Encoding( + phrases=0, + dictionary=0, + streams=(), + seconds=0.0, + lossless=True, + written=None, + ), + needs_seeds=False, + ) + + +class TestVerdictRows: + short = _song("short", SongGroup.PROJECT) + long = _song("long", SongGroup.LONG_PROJECT) + stem = _song("stem", SongGroup.RECONSTRUCTION) + variants: Tuple[Variant, ...] = ( + _variant(BASELINE, VariantKind.BASELINE), + _variant("shrinks", VariantKind.FORMAT), + _variant("projects-only", VariantKind.ENCODER), + ) + + def test_the_projects_are_judged_together_and_the_baseline_is_left_out(self) -> None: + measurements = ( + _measurement(self.short, BASELINE, STREAM), + _measurement(self.long, BASELINE, STREAM), + _measurement(self.stem, BASELINE, STREAM), + _measurement(self.short, "shrinks", STREAM), + _measurement(self.long, "shrinks", STREAM - 60), + _measurement(self.stem, "shrinks", STREAM - 10), + _measurement(self.short, "projects-only", STREAM - 40), + _measurement(self.long, "projects-only", STREAM - 40), + ) + baseline = 2 * measurements[0].block + + rows = verdict_rows(group_rows(measurements, BASELINE), measurements, self.variants) + + assert [row.variant for row in rows] == ["shrinks", "projects-only"] + assert rows[0].projects == pytest.approx(-60 / baseline) + assert rows[0].reconstructions == pytest.approx(-10 / measurements[2].block) + assert rows[0].worst == pytest.approx(0.0) + assert rows[0].verdict is Verdict.PROVISIONAL + assert rows[1].reconstructions is None + assert rows[1].cells[4] == "" + + def test_a_variant_the_run_left_out_takes_no_row(self) -> None: + measurements = (_measurement(self.short, BASELINE, STREAM), _measurement(self.short, "shrinks", STREAM)) + + rows = verdict_rows(group_rows(measurements, BASELINE), measurements, self.variants) + + assert [row.variant for row in rows] == ["shrinks"] + assert rows[0].verdict is Verdict.REJECTED + + def test_a_written_encoding_graduates_outright(self) -> None: + written = replace(self.variants[2], kind=VariantKind.ENCODER) + measurements = ( + _measurement(self.short, BASELINE, STREAM), + _measurement(self.short, written.name, STREAM - 50), + ) + + rows = verdict_rows(group_rows(measurements, BASELINE), measurements, (self.variants[0], written)) + + assert rows[0].verdict is Verdict.PROVISIONAL diff --git a/tests/unit/sampletones_tools/codec/test_command.py b/tests/unit/sampletones_tools/codec/test_command.py new file mode 100644 index 000000000..cc4919054 --- /dev/null +++ b/tests/unit/sampletones_tools/codec/test_command.py @@ -0,0 +1,170 @@ +import re +from dataclasses import dataclass +from pathlib import Path +from typing import Final, List, Optional, Tuple + +import pytest + +from sampletones.commands.registry import COMMANDS +from sampletones.dispatcher import dispatch +from sampletones_tools.codec.command import DEFAULT_LENGTHEN_SECONDS, NO_SOURCE +from sampletones_tools.codec.report.session import CompressionReport +from sampletones_tools.codec.study.plan import StudyPlan +from sampletones_tools.codec.study.variants.production import BASELINE_NAME +from sampletones_tools.codec.study.variants.registry import EVERY_VARIANT +from tests.suite.base import BaseTestSuite +from tests.suite.case import BaseRegularTestCase +from tests.suite.files import empty_file + +RUNNER: Final[str] = "sampletones_tools.codec.study.session.run_study" +REPORTER: Final[str] = "sampletones_tools.codec.report.session.run_report" + + +class RecordedStudy: + def __init__(self) -> None: + self.runs: List[Tuple[StudyPlan, Optional[Path]]] = [] + + def __call__(self, plan: StudyPlan, output: Optional[Path]) -> Path: + self.runs.append((plan, output)) + return output if output is not None else Path("run") + + +@dataclass(frozen=True) +class StudySources: + project: Path + reconstruction: Path + + +@pytest.fixture(name="study") +def study_fixture(monkeypatch: pytest.MonkeyPatch) -> RecordedStudy: + recorded = RecordedStudy() + monkeypatch.setattr(RUNNER, recorded) + return recorded + + +@pytest.fixture(name="sources") +def sources_fixture(tmp_path: Path) -> StudySources: + reconstruction = tmp_path / "stems" / "two" + reconstruction.mkdir(parents=True) + return StudySources(project=empty_file(tmp_path / "songs", "one.stp"), reconstruction=reconstruction) + + +class TestCodecReport: + def test_the_report_is_written_into_the_output_and_its_tables_are_printed( + self, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + ) -> None: + outputs: List[Path] = [] + + def run_report(output: Path) -> CompressionReport: + outputs.append(output) + return CompressionReport( + entries=(), + encodings=(), + csv_path=output / "report.csv", + markdown_path=output / "report.md", + ) + + monkeypatch.setattr(REPORTER, run_report) + + assert dispatch(COMMANDS, ["codec", "report", "-o", str(tmp_path)]) == 0 + assert outputs == [tmp_path] + assert capsys.readouterr().out.splitlines() == [ + f"Wrote {tmp_path / 'report.csv'}", + f"Wrote {tmp_path / 'report.md'}", + ] + + def test_the_output_is_required(self) -> None: + with pytest.raises(SystemExit) as leaving: + dispatch(COMMANDS, ["codec", "report"]) + + assert leaving.value.code == 2 + + +class TestCodecStudy: + def test_the_sources_and_the_sweep_are_read_from_the_options( + self, + study: RecordedStudy, + sources: StudySources, + tmp_path: Path, + ) -> None: + status = dispatch( + COMMANDS, + [ + "codec", + "study", + "--project", + str(sources.project), + "--reconstruction", + str(sources.reconstruction), + "--variants", + "wide-hold", + "--lengthen", + "30", + "-o", + str(tmp_path), + ], + ) + + assert status == 0 + plan, output = study.runs[0] + assert [source.path for source in plan.manifest.projects] == [sources.project] + assert [source.path for source in plan.manifest.reconstructions] == [sources.reconstruction] + assert plan.manifest.variants == ("wide-hold",) + assert [variant.name for variant in plan.variants] == [BASELINE_NAME, "wide-hold"] + assert plan.manifest.lengthen_seconds == 30 + assert output == tmp_path + + def test_a_run_naming_no_source_is_refused_with_the_way_to_name_one(self, study: RecordedStudy) -> None: + with pytest.raises(SystemExit, match=re.escape(NO_SOURCE)): + dispatch(COMMANDS, ["codec", "study"]) + + assert all(flag in NO_SOURCE for flag in ("--project", "--reconstruction", "--manifest")) + assert study.runs == [] + + def test_the_lengthening_defaults_to_its_constant(self, study: RecordedStudy, sources: StudySources) -> None: + assert dispatch(COMMANDS, ["codec", "study", "--project", str(sources.project)]) == 0 + plan, output = study.runs[0] + assert plan.manifest.lengthen_seconds == DEFAULT_LENGTHEN_SECONDS + assert plan.manifest.variants == (EVERY_VARIANT,) + assert output is None + + def test_an_action_is_required(self) -> None: + with pytest.raises(SystemExit) as leaving: + dispatch(COMMANDS, ["codec"]) + + assert leaving.value.code == 2 + + +class TestRefusedStudies(BaseTestSuite): + @dataclass(frozen=True, kw_only=True) + class TestCase(BaseRegularTestCase): + argv: Tuple[str, ...] + refusal: str + + test_cases = ( + TestCase(label="an unknown variant", argv=("--variants", "bogus"), refusal="No variant is called bogus"), + TestCase(label="no lengthening", argv=("--lengthen", "0"), refusal="lengthen_seconds"), + TestCase(label="a missing manifest", argv=("--manifest", "{directory}/absent.json"), refusal="No manifest at"), + TestCase(label="a missing project", argv=("--project", "{directory}/absent.stp"), refusal="No file at"), + TestCase(label="a broken manifest", argv=("--manifest", "{directory}/broken.json"), refusal="JSON"), + ) + + @pytest.mark.parametrize("test_case", test_cases, ids=lambda test_case: test_case.label) + def test_a_run_asking_the_impossible_is_refused_in_one_line_before_it_starts( + self, + study: RecordedStudy, + sources: StudySources, + tmp_path: Path, + test_case: TestCase, + ) -> None: + (tmp_path / "broken.json").write_text("{", encoding="utf-8") + argv = [argument.format(directory=tmp_path) for argument in test_case.argv] + + with pytest.raises(SystemExit, match=test_case.refusal) as leaving: + dispatch(COMMANDS, ["codec", "study", "--project", str(sources.project), *argv]) + + assert len(str(leaving.value).splitlines()) == 1 + assert study.runs == [] diff --git a/tests/unit/sampletones_tools/corpus/test_build.py b/tests/unit/sampletones_tools/corpus/test_build.py new file mode 100644 index 000000000..8fcbd8d65 --- /dev/null +++ b/tests/unit/sampletones_tools/corpus/test_build.py @@ -0,0 +1,35 @@ +from typing import Dict + +from sampletones_core.constants.enums import ChannelName +from sampletones_core.project.voices.sample import Sample +from sampletones_tools.corpus.build import build_project +from sampletones_tools.corpus.module import ModuleConfig +from sampletones_tools.corpus.song import ChannelSpec, RowSpec, SongSpec +from tests.suite.performance import make_pulse_reconstruction, make_triangle_reconstruction + +MODULE: ModuleConfig = ModuleConfig(title="Demo", author="Someone", tempo=125, speed=3, nes_frequency=50) + + +def _catalog() -> Dict[str, Sample]: + return { + "lead": Sample(name="lead", reconstruction=make_pulse_reconstruction()), + "bass": Sample(name="bass", reconstruction=make_triangle_reconstruction()), + } + + +class TestBuildProject: + def test_the_project_carries_the_module_the_voices_and_the_song(self) -> None: + catalog = _catalog() + spec = SongSpec( + rows_per_pattern=2, + order=[{ChannelName.TRIANGLE: 0}], + channels={ChannelName.TRIANGLE: ChannelSpec(patterns={0: [RowSpec(row=0, sample="bass")]})}, + ) + + project = build_project(catalog, MODULE, spec) + + assert (project.info.title, project.info.author) == ("Demo", "Someone") + assert (project.settings.tempo, project.settings.speed, project.settings.nes_frequency) == (125, 3, 50) + assert [voice.name for voice in project.voices] == ["lead", "bass"] + assert project.song.rows_per_pattern == 2 + assert project.song.channels[ChannelName.TRIANGLE].patterns[0].rows[0].volume is None diff --git a/tests/unit/sampletones_tools/corpus/test_catalog.py b/tests/unit/sampletones_tools/corpus/test_catalog.py new file mode 100644 index 000000000..9b0e86efa --- /dev/null +++ b/tests/unit/sampletones_tools/corpus/test_catalog.py @@ -0,0 +1,13 @@ +from sampletones_tools.corpus.catalog import CatalogSpec +from sampletones_tools.corpus.synth import SynthConfig + + +class TestCatalogSpec: + def test_every_shipped_instrument_is_rendered_from_a_voice_the_synthesizer_holds(self) -> None: + voices = SynthConfig.load().voices + + instruments = CatalogSpec.load().instruments + + assert instruments + assert all(instrument.synth in voices for instrument in instruments) + assert all(instrument.channels for instrument in instruments) diff --git a/tests/unit/sampletones_tools/corpus/test_module.py b/tests/unit/sampletones_tools/corpus/test_module.py new file mode 100644 index 000000000..a5146c346 --- /dev/null +++ b/tests/unit/sampletones_tools/corpus/test_module.py @@ -0,0 +1,9 @@ +from sampletones_tools.corpus.module import ModuleConfig + + +class TestModuleConfig: + def test_the_shipped_module_loads_from_the_package(self) -> None: + module = ModuleConfig.load() + + assert module.title + assert module.tempo > 0 diff --git a/tests/unit/sampletones_tools/corpus/test_song.py b/tests/unit/sampletones_tools/corpus/test_song.py new file mode 100644 index 000000000..6644d0b91 --- /dev/null +++ b/tests/unit/sampletones_tools/corpus/test_song.py @@ -0,0 +1,74 @@ +from typing import Dict + +import pytest + +from sampletones_core.constants.enums import ChannelName +from sampletones_core.project.voices.note_off import NoteOff +from sampletones_core.project.voices.note_on import NoteOn +from sampletones_core.project.voices.sample import Sample +from sampletones_tools.corpus.song import ChannelSpec, RowSpec, SongSpec, build_song +from tests.suite.performance import make_noise_reconstruction, make_pulse_reconstruction + +ROWS_PER_PATTERN: int = 4 + + +def _catalog() -> Dict[str, Sample]: + return { + "lead": Sample(name="lead", reconstruction=make_pulse_reconstruction()), + "hihat": Sample(name="hihat", reconstruction=make_noise_reconstruction()), + } + + +def _spec(channel: ChannelName, rows: Dict[int, RowSpec]) -> SongSpec: + return SongSpec( + rows_per_pattern=ROWS_PER_PATTERN, + order=[{channel: 0}], + channels={channel: ChannelSpec(patterns={0: list(rows.values())})}, + ) + + +class TestSongSpec: + def test_the_shipped_arrangement_loads_from_the_package(self) -> None: + spec = SongSpec.load() + + assert spec.rows_per_pattern > 0 + assert spec.order + assert set(spec.channels) <= set(ChannelName.items()) + + +class TestBuildSong: + def test_every_written_row_reaches_its_pattern(self) -> None: + catalog = _catalog() + spec = _spec( + ChannelName.PULSE1, + { + 0: RowSpec(row=0, sample="lead", transpose=12, volume=10), + 2: RowSpec(row=2, off=True), + 3: RowSpec(row=3, volume=4), + }, + ) + + song = build_song(spec, catalog) + + rows = song.channels[ChannelName.PULSE1].patterns[0].rows + assert len(rows) == ROWS_PER_PATTERN + assert rows[0].command == NoteOn(voice_id=catalog["lead"].id) + assert (rows[0].transpose, rows[0].volume) == (12, 10) + assert rows[1].command is None + assert isinstance(rows[2].command, NoteOff) + assert (rows[3].command, rows[3].volume) == (None, 4) + + def test_the_order_names_every_channel_and_every_channel_is_present(self) -> None: + song = build_song(_spec(ChannelName.NOISE, {0: RowSpec(row=0, sample="hihat")}), _catalog()) + + assert song.order == [{channel: 0 if channel == ChannelName.NOISE else None for channel in ChannelName.items()}] + assert set(song.channels) == set(ChannelName.items()) + assert song.channels[ChannelName.PULSE1].patterns == {} + + def test_a_sample_the_catalog_lacks_is_refused(self) -> None: + with pytest.raises(KeyError, match="snare"): + build_song(_spec(ChannelName.NOISE, {0: RowSpec(row=0, sample="snare")}), _catalog()) + + def test_a_sample_on_a_channel_it_has_no_slice_for_is_refused(self) -> None: + with pytest.raises(ValueError, match="Sample 'lead' has no 'noise' slice"): + build_song(_spec(ChannelName.NOISE, {0: RowSpec(row=0, sample="lead")}), _catalog()) diff --git a/tests/unit/sampletones_tools/player/assembler/__init__.py b/tests/unit/sampletones_tools/player/assembler/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/unit/sampletones_player/driver/assembler/test_builder.py b/tests/unit/sampletones_tools/player/assembler/test_builder.py similarity index 93% rename from tests/unit/sampletones_player/driver/assembler/test_builder.py rename to tests/unit/sampletones_tools/player/assembler/test_builder.py index 0c8479146..fea7fbb4a 100644 --- a/tests/unit/sampletones_player/driver/assembler/test_builder.py +++ b/tests/unit/sampletones_tools/player/assembler/test_builder.py @@ -5,8 +5,6 @@ import pytest from sampletones_player.driver.addresses import DriverAddresses -from sampletones_player.driver.assembler.builder import build_driver, verify_addresses -from sampletones_player.driver.assembler.toolchain import ASSEMBLER from sampletones_player.driver.image import DriverImage from sampletones_player.specification.driver import ( DRIVER_CODE_NAME, @@ -15,6 +13,8 @@ LOAD_ADDRESS, ) from sampletones_shared.exceptions import DriverBuildError +from sampletones_tools.player.assembler.builder import build_driver, verify_addresses +from sampletones_tools.player.assembler.toolchain import ASSEMBLER from tests.suite.base import BaseTestSuite DISPLACEMENT: Final[int] = 0x0100 @@ -57,7 +57,7 @@ class TestTheDriverBuild(BaseTestSuite): """The committed driver against the sources it is built from.""" def test_the_committed_driver_matches_its_sources(self, built_driver: DriverImage) -> None: - message = f"{DRIVER_CODE_NAME} is behind its sources: run `make player`" + message = f"{DRIVER_CODE_NAME} is behind its sources: run `uv run sampletones driver`" assert built_driver.code == DriverImage.load().code, message def test_the_linker_lays_the_driver_out_where_it_is_declared(self, built_driver: DriverImage) -> None: diff --git a/tests/unit/sampletones_player/driver/assembler/test_labels.py b/tests/unit/sampletones_tools/player/assembler/test_labels.py similarity index 97% rename from tests/unit/sampletones_player/driver/assembler/test_labels.py rename to tests/unit/sampletones_tools/player/assembler/test_labels.py index 411d50765..0fa438e84 100644 --- a/tests/unit/sampletones_player/driver/assembler/test_labels.py +++ b/tests/unit/sampletones_tools/player/assembler/test_labels.py @@ -3,7 +3,9 @@ import pytest -from sampletones_player.driver.assembler.labels import ( +from sampletones_player.specification.driver import INIT_ADDRESS, LOAD_ADDRESS, PLAY_ADDRESS +from sampletones_shared.exceptions import DriverBuildError +from sampletones_tools.player.assembler.labels import ( INIT_SYMBOL, LOAD_SYMBOL, PLAY_SYMBOL, @@ -11,8 +13,6 @@ read_addresses, read_labels, ) -from sampletones_player.specification.driver import INIT_ADDRESS, LOAD_ADDRESS, PLAY_ADDRESS -from sampletones_shared.exceptions import DriverBuildError from tests.suite.base import BaseTestSuite LABELS_NAME: Final[str] = "driver.labels" diff --git a/tests/unit/sampletones_tools/player/assembler/test_report.py b/tests/unit/sampletones_tools/player/assembler/test_report.py new file mode 100644 index 000000000..95932fb7a --- /dev/null +++ b/tests/unit/sampletones_tools/player/assembler/test_report.py @@ -0,0 +1,19 @@ +from typing import Final + +from sampletones_player.driver.addresses import DriverAddresses +from sampletones_player.driver.image import DriverImage +from sampletones_player.specification.driver import JUMP_ABSOLUTE_OPCODE +from sampletones_tools.player.assembler.report import layout_lines + +RETURN_OPCODE: Final[int] = 0x60 +CODE: Final[bytes] = bytes((JUMP_ABSOLUTE_OPCODE, 0x00, 0x80, JUMP_ABSOLUTE_OPCODE, 0x00, 0x80, RETURN_OPCODE)) + + +class TestLayoutLines: + def test_the_figures_a_reader_checks_against_the_header(self) -> None: + image = DriverImage(code=CODE, addresses=DriverAddresses.for_code(len(CODE))) + + lines = layout_lines(image) + + assert lines[0] == "driver.bin 7 bytes, $8000-$8006" + assert lines[1:] == ["init $8000", "play $8003", "song $8007"] diff --git a/tests/unit/sampletones_player/driver/assembler/test_toolchain.py b/tests/unit/sampletones_tools/player/assembler/test_toolchain.py similarity index 96% rename from tests/unit/sampletones_player/driver/assembler/test_toolchain.py rename to tests/unit/sampletones_tools/player/assembler/test_toolchain.py index 89602b787..4ee145f24 100644 --- a/tests/unit/sampletones_player/driver/assembler/test_toolchain.py +++ b/tests/unit/sampletones_tools/player/assembler/test_toolchain.py @@ -5,14 +5,14 @@ import pytest -from sampletones_player.driver.assembler.toolchain import ( +from sampletones_shared.exceptions import DriverBuildError, ToolchainMissingError +from sampletones_shared.utils.system.system import System +from sampletones_tools.player.assembler.toolchain import ( ASSEMBLER, INSTALL_HINTS, LINKER, Toolchain, ) -from sampletones_shared.exceptions import DriverBuildError, ToolchainMissingError -from sampletones_shared.utils.system.system import System from tests.suite.base import BaseTestSuite FAILING_PROGRAM: Final[str] = "import sys; sys.stderr.write('boom'); sys.exit(1)" diff --git a/tests/unit/sampletones_tools/player/test_command.py b/tests/unit/sampletones_tools/player/test_command.py new file mode 100644 index 000000000..c9b7d56e6 --- /dev/null +++ b/tests/unit/sampletones_tools/player/test_command.py @@ -0,0 +1,65 @@ +from pathlib import Path +from typing import Final, List + +import pytest + +from sampletones.commands.registry import COMMANDS +from sampletones.dispatcher import dispatch +from sampletones_player.driver.addresses import DriverAddresses +from sampletones_player.driver.image import DriverImage +from sampletones_player.specification.driver import JUMP_ABSOLUTE_OPCODE +from sampletones_shared.exceptions.player import DriverBuildError +from sampletones_tools.player.assembler.layout import BINARY_DIRECTORY + +BUILDER: Final[str] = "sampletones_tools.player.assembler.builder.build_driver" +GUARD: Final[str] = "sampletones_tools.checkout.require_checkout" +RETURN_OPCODE: Final[int] = 0x60 +CODE: Final[bytes] = bytes((JUMP_ABSOLUTE_OPCODE, 0x00, 0x80, JUMP_ABSOLUTE_OPCODE, 0x00, 0x80, RETURN_OPCODE)) +IMAGE: Final[DriverImage] = DriverImage(code=CODE, addresses=DriverAddresses.for_code(len(CODE))) + + +class RecordedBuild: + def __init__(self) -> None: + self.destinations: List[Path] = [] + + def __call__(self, destination: Path) -> DriverImage: + self.destinations.append(destination) + return IMAGE + + +def _refuse(command: str) -> None: + raise AssertionError(f"the checkout guard ran for {command}") + + +class TestDriver: + def test_without_a_directory_the_shipped_driver_is_written_from_a_checkout( + self, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], + ) -> None: + build = RecordedBuild() + guarded: List[str] = [] + monkeypatch.setattr(BUILDER, build) + monkeypatch.setattr(GUARD, guarded.append) + + assert dispatch(COMMANDS, ["driver"]) == 0 + assert build.destinations == [BINARY_DIRECTORY] + assert guarded == ["driver"] + assert "driver.bin 7 bytes" in capsys.readouterr().out + + def test_a_directory_of_its_own_needs_no_checkout(self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + build = RecordedBuild() + monkeypatch.setattr(BUILDER, build) + monkeypatch.setattr(GUARD, _refuse) + + assert dispatch(COMMANDS, ["driver", "--output", str(tmp_path)]) == 0 + assert build.destinations == [tmp_path] + + def test_a_failing_build_is_reported(self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + def fail(destination: Path) -> DriverImage: + raise DriverBuildError(f"ca65 failed: boom in {destination}") + + monkeypatch.setattr(BUILDER, fail) + + with pytest.raises(SystemExit, match="ca65 failed"): + dispatch(COMMANDS, ["driver", "--output", str(tmp_path)]) diff --git a/tests/unit/sampletones_player/driver/test_song_include.py b/tests/unit/sampletones_tools/player/test_song_include.py similarity index 96% rename from tests/unit/sampletones_player/driver/test_song_include.py rename to tests/unit/sampletones_tools/player/test_song_include.py index 042e4b300..7576b0141 100644 --- a/tests/unit/sampletones_player/driver/test_song_include.py +++ b/tests/unit/sampletones_tools/player/test_song_include.py @@ -6,7 +6,6 @@ from sampletones_player.compression.pitch import PITCH_COUNT from sampletones_player.compression.planes.order import PlaneOrder -from sampletones_player.driver.assembler.layout import INCLUDE_DIRECTORY from sampletones_player.specification.binary import WORD_SIZE from sampletones_player.specification.compression import ( OPCODE_SIZE, @@ -31,6 +30,7 @@ TIMER_TABLE_OFFSET, TOTAL_TICKS_OFFSET, ) +from sampletones_tools.player.assembler.layout import ASSEMBLY_DIRECTORY, INCLUDE_DIRECTORY SONG_INCLUDE: Final[str] = "song.inc" HEXADECIMAL_MARKER: Final[str] = "$" @@ -108,7 +108,7 @@ def read_equates(path: Path) -> Dict[str, int]: @pytest.fixture(name="equates", scope="module") def equates_fixture() -> Dict[str, int]: - return read_equates(INCLUDE_DIRECTORY / SONG_INCLUDE) + return read_equates(ASSEMBLY_DIRECTORY / INCLUDE_DIRECTORY / SONG_INCLUDE) class TestTheDriverReadsTheBlockTheExporterWrites: diff --git a/tests/unit/sampletones_tools/player/trace/__init__.py b/tests/unit/sampletones_tools/player/trace/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/unit/sampletones_player/trace/test_trace.py b/tests/unit/sampletones_tools/player/trace/test_trace.py similarity index 98% rename from tests/unit/sampletones_player/trace/test_trace.py rename to tests/unit/sampletones_tools/player/trace/test_trace.py index f800fd3bc..f585cc9a4 100644 --- a/tests/unit/sampletones_player/trace/test_trace.py +++ b/tests/unit/sampletones_tools/player/trace/test_trace.py @@ -22,8 +22,8 @@ SWEEP_DISABLED, TRIANGLE_TIMER_HIGH, ) -from sampletones_player.trace.trace import RegisterTrace -from sampletones_player.trace.write import RegisterWrite +from sampletones_tools.player.trace.trace import RegisterTrace +from sampletones_tools.player.trace.write import RegisterWrite from tests.suite.base import BaseTestSuite from tests.suite.case import BaseAutolabelTestCase from tests.suite.player import ( diff --git a/tests/unit/sampletones_tools/samples/commands/test_btp.py b/tests/unit/sampletones_tools/samples/commands/test_btp.py new file mode 100644 index 000000000..02c5801b2 --- /dev/null +++ b/tests/unit/sampletones_tools/samples/commands/test_btp.py @@ -0,0 +1,43 @@ +from pathlib import Path +from typing import Final, List, Tuple + +import pytest + +from sampletones.commands.registry import COMMANDS +from sampletones.dispatcher import dispatch +from sampletones_tools.samples.bitphase import write_samples +from sampletones_tools.samples.emit import Emitter + +EMITTER: Final[str] = "sampletones_tools.samples.emit.emit_samples" + + +class TestBtpSamples: + def test_the_bitphase_emitter_writes_into_the_output_and_every_file_is_printed( + self, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + ) -> None: + calls: List[Tuple[Path, Emitter]] = [] + + def emit_samples(output: Path, emitter: Emitter) -> List[Path]: + calls.append((output, emitter)) + return [output / "drums.btp"] + + monkeypatch.setattr(EMITTER, emit_samples) + + assert dispatch(COMMANDS, ["btp", "samples", "-o", str(tmp_path)]) == 0 + assert calls == [(tmp_path, write_samples)] + assert capsys.readouterr().out.splitlines() == [f"Wrote {tmp_path / 'drums.btp'}"] + + def test_the_output_is_required(self) -> None: + with pytest.raises(SystemExit) as leaving: + dispatch(COMMANDS, ["btp", "samples"]) + + assert leaving.value.code == 2 + + def test_an_action_is_required(self) -> None: + with pytest.raises(SystemExit) as leaving: + dispatch(COMMANDS, ["btp"]) + + assert leaving.value.code == 2 diff --git a/tests/unit/sampletones_tools/samples/commands/test_ftm.py b/tests/unit/sampletones_tools/samples/commands/test_ftm.py new file mode 100644 index 000000000..21d0b8c81 --- /dev/null +++ b/tests/unit/sampletones_tools/samples/commands/test_ftm.py @@ -0,0 +1,43 @@ +from pathlib import Path +from typing import Final, List, Tuple + +import pytest + +from sampletones.commands.registry import COMMANDS +from sampletones.dispatcher import dispatch +from sampletones_tools.samples.emit import Emitter +from sampletones_tools.samples.famitracker import write_samples + +EMITTER: Final[str] = "sampletones_tools.samples.emit.emit_samples" + + +class TestFtmSamples: + def test_the_famitracker_emitter_writes_into_the_output_and_every_file_is_printed( + self, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + ) -> None: + calls: List[Tuple[Path, Emitter]] = [] + + def emit_samples(output: Path, emitter: Emitter) -> List[Path]: + calls.append((output, emitter)) + return [output / "drums.ftm"] + + monkeypatch.setattr(EMITTER, emit_samples) + + assert dispatch(COMMANDS, ["ftm", "samples", "-o", str(tmp_path)]) == 0 + assert calls == [(tmp_path, write_samples)] + assert capsys.readouterr().out.splitlines() == [f"Wrote {tmp_path / 'drums.ftm'}"] + + def test_the_output_is_required(self) -> None: + with pytest.raises(SystemExit) as leaving: + dispatch(COMMANDS, ["ftm", "samples"]) + + assert leaving.value.code == 2 + + def test_an_action_is_required(self) -> None: + with pytest.raises(SystemExit) as leaving: + dispatch(COMMANDS, ["ftm"]) + + assert leaving.value.code == 2 diff --git a/tests/unit/sampletones_tools/samples/commands/test_nsf.py b/tests/unit/sampletones_tools/samples/commands/test_nsf.py new file mode 100644 index 000000000..1ef0970c2 --- /dev/null +++ b/tests/unit/sampletones_tools/samples/commands/test_nsf.py @@ -0,0 +1,83 @@ +from pathlib import Path +from typing import Final, List, Tuple + +import pytest + +from sampletones.commands.registry import COMMANDS +from sampletones.dispatcher import dispatch +from sampletones_tools.samples.emit import Emitter +from sampletones_tools.samples.nsf import write_samples +from sampletones_tools.samples.render import RenderedWave, RenderingError + +RENDERER: Final[str] = "sampletones_tools.samples.render.render_directory" +EMITTER: Final[str] = "sampletones_tools.samples.emit.emit_samples" + + +class TestNsfSamples: + def test_the_nsf_emitter_writes_into_the_output_and_every_file_is_printed( + self, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + ) -> None: + calls: List[Tuple[Path, Emitter]] = [] + + def emit_samples(output: Path, emitter: Emitter) -> List[Path]: + calls.append((output, emitter)) + return [output / "kick.nsf", output / "song.nsf"] + + monkeypatch.setattr(EMITTER, emit_samples) + + assert dispatch(COMMANDS, ["nsf", "samples", "--output", str(tmp_path)]) == 0 + assert calls == [(tmp_path, write_samples)] + assert capsys.readouterr().out.splitlines() == [ + f"Wrote {tmp_path / 'kick.nsf'}", + f"Wrote {tmp_path / 'song.nsf'}", + ] + + def test_the_output_is_required(self) -> None: + with pytest.raises(SystemExit) as leaving: + dispatch(COMMANDS, ["nsf", "samples"]) + + assert leaving.value.code == 2 + + +class TestNsfRender: + def test_every_wave_written_is_printed( + self, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + ) -> None: + calls: List[Tuple[Path, float]] = [] + + def render_directory(directory: Path, tail_seconds: float) -> List[RenderedWave]: + calls.append((directory, tail_seconds)) + return [RenderedWave(source=directory / "a.nsf", destination=directory / "a.wav", seconds=1.5)] + + monkeypatch.setattr(RENDERER, render_directory) + + assert dispatch(COMMANDS, ["nsf", "render", "--directory", str(tmp_path)]) == 0 + assert calls == [(tmp_path, 0.5)] + assert f"{tmp_path / 'a.wav'} 1.500 s" in capsys.readouterr().out + + def test_the_directory_is_required(self) -> None: + with pytest.raises(SystemExit) as leaving: + dispatch(COMMANDS, ["nsf", "render"]) + + assert leaving.value.code == 2 + + def test_an_action_is_required(self) -> None: + with pytest.raises(SystemExit) as leaving: + dispatch(COMMANDS, ["nsf"]) + + assert leaving.value.code == 2 + + def test_a_rendering_failure_is_reported(self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + def fail(directory: Path, tail_seconds: float) -> List[RenderedWave]: + raise RenderingError(f"ffmpeg is missing for {directory} at {tail_seconds}") + + monkeypatch.setattr(RENDERER, fail) + + with pytest.raises(SystemExit, match="ffmpeg is missing"): + dispatch(COMMANDS, ["nsf", "render", "--directory", str(tmp_path)]) diff --git a/tests/unit/sampletones_tools/samples/test_render.py b/tests/unit/sampletones_tools/samples/test_render.py new file mode 100644 index 000000000..132ece2c3 --- /dev/null +++ b/tests/unit/sampletones_tools/samples/test_render.py @@ -0,0 +1,44 @@ +import struct +from pathlib import Path + +import pytest + +from sampletones_player.specification.clock import FIXED_POINT_SCALE, NTSC_FRAME_RATE +from sampletones_player.specification.nsf import HEADER_SIZE +from sampletones_player.specification.song import ( + STEP_FRACTION_OFFSET, + STEP_WHOLE_OFFSET, + TOTAL_TICKS_OFFSET, +) +from sampletones_tools.samples import render +from sampletones_tools.samples.render import RenderingError, render_directory, song_seconds + +CODE_LENGTH = 10 + + +def _exported(ticks: int, whole: int, fraction: int) -> bytes: + block = bytearray(max(TOTAL_TICKS_OFFSET, STEP_FRACTION_OFFSET, STEP_WHOLE_OFFSET) + 2) + struct.pack_into(" None: + data = _exported(ticks=600, whole=1, fraction=0) + + assert song_seconds(data, CODE_LENGTH) == pytest.approx(600 / float(NTSC_FRAME_RATE)) + + def test_a_fractional_step_takes_fewer_calls(self) -> None: + data = _exported(ticks=600, whole=1, fraction=FIXED_POINT_SCALE // 2) + + assert song_seconds(data, CODE_LENGTH) == pytest.approx(400 / float(NTSC_FRAME_RATE)) + + +class TestRenderDirectory: + def test_a_directory_without_exports_is_refused(self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + monkeypatch.setattr(render, "require_renderer", lambda: None) + + with pytest.raises(RenderingError, match="no .nsf files"): + render_directory(tmp_path, 0.5) diff --git a/tests/unit/sampletones_tools/synthesis/__init__.py b/tests/unit/sampletones_tools/synthesis/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/unit/sampletones_synthesis/conftest.py b/tests/unit/sampletones_tools/synthesis/conftest.py similarity index 100% rename from tests/unit/sampletones_synthesis/conftest.py rename to tests/unit/sampletones_tools/synthesis/conftest.py diff --git a/tests/unit/sampletones_tools/synthesis/envelopes/__init__.py b/tests/unit/sampletones_tools/synthesis/envelopes/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/unit/sampletones_synthesis/envelopes/test_envelopes.py b/tests/unit/sampletones_tools/synthesis/envelopes/test_envelopes.py similarity index 87% rename from tests/unit/sampletones_synthesis/envelopes/test_envelopes.py rename to tests/unit/sampletones_tools/synthesis/envelopes/test_envelopes.py index 2984acaf7..957c37ba3 100644 --- a/tests/unit/sampletones_synthesis/envelopes/test_envelopes.py +++ b/tests/unit/sampletones_tools/synthesis/envelopes/test_envelopes.py @@ -3,9 +3,9 @@ import numpy as np import pytest -from sampletones_synthesis.envelopes.exponential_decay import ExponentialDecayEnvelope -from sampletones_synthesis.envelopes.linear_attack import LinearAttackEnvelope -from sampletones_synthesis.envelopes.linear_ramp import LinearRampEnvelope +from sampletones_tools.synthesis.envelopes.exponential_decay import ExponentialDecayEnvelope +from sampletones_tools.synthesis.envelopes.linear_attack import LinearAttackEnvelope +from sampletones_tools.synthesis.envelopes.linear_ramp import LinearRampEnvelope TIME_CONSTANT_SECONDS: Final[float] = 0.25 ATTACK_SECONDS: Final[float] = 0.005 diff --git a/tests/unit/sampletones_tools/synthesis/filters/__init__.py b/tests/unit/sampletones_tools/synthesis/filters/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/unit/sampletones_synthesis/filters/test_butterworth_highpass.py b/tests/unit/sampletones_tools/synthesis/filters/test_butterworth_highpass.py similarity index 90% rename from tests/unit/sampletones_synthesis/filters/test_butterworth_highpass.py rename to tests/unit/sampletones_tools/synthesis/filters/test_butterworth_highpass.py index 90e2623fa..3c4ddc7ae 100644 --- a/tests/unit/sampletones_synthesis/filters/test_butterworth_highpass.py +++ b/tests/unit/sampletones_tools/synthesis/filters/test_butterworth_highpass.py @@ -2,7 +2,7 @@ import numpy as np -from sampletones_synthesis.filters.butterworth_highpass import ButterworthHighpassFilter +from sampletones_tools.synthesis.filters.butterworth_highpass import ButterworthHighpassFilter CUTOFF_HZ: Final[float] = 2000.0 ORDER: Final[int] = 4 diff --git a/tests/unit/sampletones_tools/synthesis/oscillators/__init__.py b/tests/unit/sampletones_tools/synthesis/oscillators/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/unit/sampletones_synthesis/oscillators/test_noise.py b/tests/unit/sampletones_tools/synthesis/oscillators/test_noise.py similarity index 89% rename from tests/unit/sampletones_synthesis/oscillators/test_noise.py rename to tests/unit/sampletones_tools/synthesis/oscillators/test_noise.py index 727df82a5..23fa37c32 100644 --- a/tests/unit/sampletones_synthesis/oscillators/test_noise.py +++ b/tests/unit/sampletones_tools/synthesis/oscillators/test_noise.py @@ -3,8 +3,8 @@ import numpy as np import pytest -from sampletones_synthesis.oscillators.walk_noise import WalkNoiseOscillator -from sampletones_synthesis.oscillators.white_noise import WhiteNoiseOscillator +from sampletones_tools.synthesis.oscillators.walk_noise import WalkNoiseOscillator +from sampletones_tools.synthesis.oscillators.white_noise import WhiteNoiseOscillator SEED: Final[int] = 99 diff --git a/tests/unit/sampletones_synthesis/oscillators/test_pulse.py b/tests/unit/sampletones_tools/synthesis/oscillators/test_pulse.py similarity index 93% rename from tests/unit/sampletones_synthesis/oscillators/test_pulse.py rename to tests/unit/sampletones_tools/synthesis/oscillators/test_pulse.py index bb4c3c707..c8d6f7c11 100644 --- a/tests/unit/sampletones_synthesis/oscillators/test_pulse.py +++ b/tests/unit/sampletones_tools/synthesis/oscillators/test_pulse.py @@ -3,7 +3,7 @@ import numpy as np import pytest -from sampletones_synthesis.oscillators.pulse import PulseOscillator +from sampletones_tools.synthesis.oscillators.pulse import PulseOscillator FREQUENCY: Final[float] = 220.0 DUTY_CYCLES: Final[Tuple[float, ...]] = (0.125, 0.25, 0.5) diff --git a/tests/unit/sampletones_synthesis/oscillators/test_sine.py b/tests/unit/sampletones_tools/synthesis/oscillators/test_sine.py similarity index 95% rename from tests/unit/sampletones_synthesis/oscillators/test_sine.py rename to tests/unit/sampletones_tools/synthesis/oscillators/test_sine.py index d015852f2..cd5393e30 100644 --- a/tests/unit/sampletones_synthesis/oscillators/test_sine.py +++ b/tests/unit/sampletones_tools/synthesis/oscillators/test_sine.py @@ -3,7 +3,7 @@ import numpy as np import pytest -from sampletones_synthesis.oscillators.sine import SineOscillator +from sampletones_tools.synthesis.oscillators.sine import SineOscillator FREQUENCY: Final[float] = 440.0 diff --git a/tests/unit/sampletones_synthesis/oscillators/test_sweeps.py b/tests/unit/sampletones_tools/synthesis/oscillators/test_sweeps.py similarity index 90% rename from tests/unit/sampletones_synthesis/oscillators/test_sweeps.py rename to tests/unit/sampletones_tools/synthesis/oscillators/test_sweeps.py index 49c1a351a..25667dfa0 100644 --- a/tests/unit/sampletones_synthesis/oscillators/test_sweeps.py +++ b/tests/unit/sampletones_tools/synthesis/oscillators/test_sweeps.py @@ -5,10 +5,10 @@ import pytest from scipy.integrate import cumulative_trapezoid -from sampletones_synthesis.oscillators.exponential_glide import ExponentialGlideOscillator -from sampletones_synthesis.oscillators.geometric_sweep import GeometricSweepOscillator -from sampletones_synthesis.oscillators.sine import SineOscillator -from sampletones_synthesis.protocols import Oscillator +from sampletones_tools.synthesis.oscillators.exponential_glide import ExponentialGlideOscillator +from sampletones_tools.synthesis.oscillators.geometric_sweep import GeometricSweepOscillator +from sampletones_tools.synthesis.oscillators.sine import SineOscillator +from sampletones_tools.synthesis.protocols import Oscillator FREQUENCY_START: Final[float] = 392.0 FREQUENCY_END: Final[float] = 44.0 diff --git a/tests/unit/sampletones_synthesis/test_frequency.py b/tests/unit/sampletones_tools/synthesis/test_frequency.py similarity index 95% rename from tests/unit/sampletones_synthesis/test_frequency.py rename to tests/unit/sampletones_tools/synthesis/test_frequency.py index 314c0b5b2..b408ae398 100644 --- a/tests/unit/sampletones_synthesis/test_frequency.py +++ b/tests/unit/sampletones_tools/synthesis/test_frequency.py @@ -4,7 +4,7 @@ import pytest from pydantic import TypeAdapter, ValidationError -from sampletones_synthesis.frequency import FrequencySpec, resolve_frequency +from sampletones_tools.synthesis.frequency import FrequencySpec, resolve_frequency FREQUENCY_SPEC_ADAPTER: Final[TypeAdapter[Any]] = TypeAdapter(FrequencySpec) diff --git a/tests/unit/sampletones_synthesis/test_unions.py b/tests/unit/sampletones_tools/synthesis/test_unions.py similarity index 79% rename from tests/unit/sampletones_synthesis/test_unions.py rename to tests/unit/sampletones_tools/synthesis/test_unions.py index 1b70adaa3..21a6bb4e7 100644 --- a/tests/unit/sampletones_synthesis/test_unions.py +++ b/tests/unit/sampletones_tools/synthesis/test_unions.py @@ -4,17 +4,17 @@ import pytest from pydantic import BaseModel, TypeAdapter, ValidationError -from sampletones_synthesis.envelopes.exponential_decay import ExponentialDecayEnvelope -from sampletones_synthesis.envelopes.linear_attack import LinearAttackEnvelope -from sampletones_synthesis.envelopes.linear_ramp import LinearRampEnvelope -from sampletones_synthesis.envelopes.types import EnvelopeUnion -from sampletones_synthesis.oscillators.exponential_glide import ExponentialGlideOscillator -from sampletones_synthesis.oscillators.geometric_sweep import GeometricSweepOscillator -from sampletones_synthesis.oscillators.pulse import PulseOscillator -from sampletones_synthesis.oscillators.sine import SineOscillator -from sampletones_synthesis.oscillators.types import OscillatorUnion -from sampletones_synthesis.oscillators.walk_noise import WalkNoiseOscillator -from sampletones_synthesis.oscillators.white_noise import WhiteNoiseOscillator +from sampletones_tools.synthesis.envelopes.exponential_decay import ExponentialDecayEnvelope +from sampletones_tools.synthesis.envelopes.linear_attack import LinearAttackEnvelope +from sampletones_tools.synthesis.envelopes.linear_ramp import LinearRampEnvelope +from sampletones_tools.synthesis.envelopes.types import EnvelopeUnion +from sampletones_tools.synthesis.oscillators.exponential_glide import ExponentialGlideOscillator +from sampletones_tools.synthesis.oscillators.geometric_sweep import GeometricSweepOscillator +from sampletones_tools.synthesis.oscillators.pulse import PulseOscillator +from sampletones_tools.synthesis.oscillators.sine import SineOscillator +from sampletones_tools.synthesis.oscillators.types import OscillatorUnion +from sampletones_tools.synthesis.oscillators.walk_noise import WalkNoiseOscillator +from sampletones_tools.synthesis.oscillators.white_noise import WhiteNoiseOscillator OSCILLATOR_ADAPTER: Final[TypeAdapter[Any]] = TypeAdapter(OscillatorUnion) ENVELOPE_ADAPTER: Final[TypeAdapter[Any]] = TypeAdapter(EnvelopeUnion) diff --git a/tests/unit/sampletones_tools/synthesis/voice/__init__.py b/tests/unit/sampletones_tools/synthesis/voice/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/unit/sampletones_synthesis/voice/test_layer.py b/tests/unit/sampletones_tools/synthesis/voice/test_layer.py similarity index 86% rename from tests/unit/sampletones_synthesis/voice/test_layer.py rename to tests/unit/sampletones_tools/synthesis/voice/test_layer.py index 92ee232e1..70237aa10 100644 --- a/tests/unit/sampletones_synthesis/voice/test_layer.py +++ b/tests/unit/sampletones_tools/synthesis/voice/test_layer.py @@ -3,10 +3,10 @@ import numpy as np import pytest -from sampletones_synthesis.envelopes.exponential_decay import ExponentialDecayEnvelope -from sampletones_synthesis.envelopes.linear_attack import LinearAttackEnvelope -from sampletones_synthesis.oscillators.sine import SineOscillator -from sampletones_synthesis.voice.layer import Layer +from sampletones_tools.synthesis.envelopes.exponential_decay import ExponentialDecayEnvelope +from sampletones_tools.synthesis.envelopes.linear_attack import LinearAttackEnvelope +from sampletones_tools.synthesis.oscillators.sine import SineOscillator +from sampletones_tools.synthesis.voice.layer import Layer FREQUENCY: Final[float] = 220.0 TIME_CONSTANT_SECONDS: Final[float] = 0.3 diff --git a/tests/unit/sampletones_synthesis/voice/test_voice.py b/tests/unit/sampletones_tools/synthesis/voice/test_voice.py similarity index 94% rename from tests/unit/sampletones_synthesis/voice/test_voice.py rename to tests/unit/sampletones_tools/synthesis/voice/test_voice.py index 6323253a1..e21cd3213 100644 --- a/tests/unit/sampletones_synthesis/voice/test_voice.py +++ b/tests/unit/sampletones_tools/synthesis/voice/test_voice.py @@ -4,9 +4,9 @@ import pytest from pydantic import ValidationError -from sampletones_synthesis.oscillators.sine import SineOscillator -from sampletones_synthesis.voice.layer import Layer -from sampletones_synthesis.voice.voice import Voice +from sampletones_tools.synthesis.oscillators.sine import SineOscillator +from sampletones_tools.synthesis.voice.layer import Layer +from sampletones_tools.synthesis.voice.voice import Voice DURATION_SECONDS: Final[float] = 0.5 LOW_FREQUENCY: Final[float] = 220.0 diff --git a/tests/unit/sampletones_tools/test_checkout.py b/tests/unit/sampletones_tools/test_checkout.py new file mode 100644 index 000000000..9cdaf98e1 --- /dev/null +++ b/tests/unit/sampletones_tools/test_checkout.py @@ -0,0 +1,36 @@ +from pathlib import Path + +import pytest + +from sampletones_tools import checkout +from sampletones_tools.checkout import is_checkout, require_checkout + +COMMAND = "check import-boundary --all" + + +class TestIsCheckout: + def test_the_project_file_beside_the_source_tree_makes_a_checkout(self, tmp_path: Path) -> None: + (tmp_path / "pyproject.toml").write_text("", encoding="utf-8") + (tmp_path / "src").mkdir() + + assert is_checkout(tmp_path) + + def test_an_installed_copy_is_no_checkout(self, tmp_path: Path) -> None: + (tmp_path / "site-packages").mkdir() + + assert not is_checkout(tmp_path) + + +class TestRequireCheckout: + def test_the_repository_passes(self) -> None: + require_checkout(COMMAND) + + def test_outside_a_checkout_the_command_is_refused_with_the_way_to_run_it( + self, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + ) -> None: + monkeypatch.setattr(checkout, "REPOSITORY_ROOT", tmp_path) + + with pytest.raises(SystemExit, match=f"uv run sampletones {COMMAND}"): + require_checkout(COMMAND) diff --git a/tests/unit/sampletones_tools/test_registry.py b/tests/unit/sampletones_tools/test_registry.py new file mode 100644 index 000000000..b01cdc8d5 --- /dev/null +++ b/tests/unit/sampletones_tools/test_registry.py @@ -0,0 +1,10 @@ +from sampletones.commands.registry import USER_COMMANDS +from sampletones_tools.registry import DEVELOPER_COMMANDS + + +class TestDeveloperCommands: + def test_every_developer_command_has_a_name_of_its_own(self) -> None: + names = [command.name for command in DEVELOPER_COMMANDS] + + assert len(set(names)) == len(names) + assert not set(names) & {command.name for command in USER_COMMANDS} diff --git a/tests/unit/sampletones_tools/test_runs.py b/tests/unit/sampletones_tools/test_runs.py new file mode 100644 index 000000000..d54d92994 --- /dev/null +++ b/tests/unit/sampletones_tools/test_runs.py @@ -0,0 +1,15 @@ +from datetime import UTC, datetime +from pathlib import Path + +from sampletones_tools.runs import RUN_STAMP, stamped_run_directory + + +class TestStampedRunDirectory: + def test_a_run_lands_under_the_root_named_by_its_start(self, tmp_path: Path) -> None: + before = datetime.now(UTC).replace(microsecond=0) + + directory = stamped_run_directory(tmp_path) + + started = datetime.strptime(directory.name, RUN_STAMP).replace(tzinfo=UTC) + assert directory.parent == tmp_path + assert before <= started <= datetime.now(UTC) diff --git a/tests/unit/scripts/bootstrap/platforms/test_bundling.py b/tests/unit/scripts/bootstrap/platforms/test_bundling.py new file mode 100644 index 000000000..172169618 --- /dev/null +++ b/tests/unit/scripts/bootstrap/platforms/test_bundling.py @@ -0,0 +1,50 @@ +from dataclasses import dataclass +from pathlib import Path + +import pytest + +from bootstrap.platforms.bundling import Bundling +from tests.suite.base import BaseTestSuite +from tests.suite.case import BaseRegularTestCase + +NAME = "sampletones" + + +def _bundling(executable_suffix: str) -> Bundling: + return Bundling( + icon="icon.png", + executable_suffix=executable_suffix, + pyaudio_advice="", + tkinter_advice="", + tkinter_warning="", + ) + + +class TestLauncher(BaseTestSuite): + @dataclass(frozen=True, kw_only=True) + class TestCase(BaseRegularTestCase): + suffix: str + release: bool + expected: Path + + test_cases = ( + TestCase(label="a development bundle is one file", suffix="", release=False, expected=Path("bin", NAME)), + TestCase( + label="a release is a directory beside its launcher", + suffix="", + release=True, + expected=Path("bin", NAME, NAME), + ), + TestCase( + label="the suffix ends the launcher's name", + suffix=".exe", + release=True, + expected=Path("bin", NAME, f"{NAME}.exe"), + ), + ) + + @pytest.mark.parametrize("test_case", test_cases, ids=lambda test_case: test_case.label) + def test_the_launcher_lies_where_pyinstaller_writes_it(self, test_case: TestCase) -> None: + launcher = _bundling(test_case.suffix).launcher(Path("bin"), name=NAME, release=test_case.release) + + assert launcher == test_case.expected diff --git a/tests/unit/scripts/bootstrap/platforms/test_factory.py b/tests/unit/scripts/bootstrap/platforms/test_factory.py new file mode 100644 index 000000000..0f25c0cab --- /dev/null +++ b/tests/unit/scripts/bootstrap/platforms/test_factory.py @@ -0,0 +1,30 @@ +from dataclasses import dataclass + +import pytest + +from bootstrap.platforms.factory import platform_named +from bootstrap.platforms.linux import LINUX +from bootstrap.platforms.macos import DARWIN +from bootstrap.platforms.windows import WINDOWS +from tests.suite.base import BaseTestSuite +from tests.suite.case import BaseRegularTestCase + + +class TestPlatformNamed(BaseTestSuite): + @dataclass(frozen=True, kw_only=True) + class TestCase(BaseRegularTestCase): + system: str + + test_cases = ( + TestCase(label="Linux", system=LINUX), + TestCase(label="Windows", system=WINDOWS), + TestCase(label="macOS", system=DARWIN), + ) + + @pytest.mark.parametrize("test_case", test_cases, ids=lambda test_case: test_case.label) + def test_the_system_name_selects_its_platform(self, test_case: TestCase) -> None: + assert platform_named(test_case.system).name == test_case.system + + def test_an_unknown_system_is_refused_by_name(self) -> None: + with pytest.raises(SystemExit, match="Plan 9"): + platform_named("Plan 9") diff --git a/tests/unit/scripts/bootstrap/platforms/test_linux.py b/tests/unit/scripts/bootstrap/platforms/test_linux.py new file mode 100644 index 000000000..c7af9df0d --- /dev/null +++ b/tests/unit/scripts/bootstrap/platforms/test_linux.py @@ -0,0 +1,27 @@ +from pathlib import Path + +from bootstrap.platforms.linux import Linux + + +class TestLinux: + def test_a_virtual_environment_runs_bin_python(self) -> None: + assert Linux().interpreter(Path(".venv-build")) == Path(".venv-build", "bin", "python") + + def test_a_bundle_launcher_carries_no_extension(self) -> None: + assert Linux().bundling().launcher(Path("bin"), name="sampletones", release=False) == Path("bin", "sampletones") + + def test_apt_is_updated_before_the_packages_are_installed(self) -> None: + commands = Linux().system_packages() + + assert [command[:3] for command in commands] == [("sudo", "apt-get", "update"), ("sudo", "apt-get", "install")] + assert "portaudio19-dev" in commands[1] + assert "python3-tk" in commands[1] + assert Linux().missing_package_manager() is None + + def test_the_setup_and_the_build_take_the_variables_as_given(self) -> None: + assert Linux().setup_variables({"PATH": "/usr/bin"}, machine="x86_64") == {"PATH": "/usr/bin"} + assert Linux().build_flags(machine="x86_64") == () + + def test_the_driver_is_looked_up_on_the_path_alone(self) -> None: + assert Linux().cpu_backend_reason is None + assert Linux().nvidia_smi_locations({"SystemRoot": "C:/Windows"}) == () diff --git a/tests/unit/scripts/bootstrap/platforms/test_macos.py b/tests/unit/scripts/bootstrap/platforms/test_macos.py new file mode 100644 index 000000000..4a2ea00ae --- /dev/null +++ b/tests/unit/scripts/bootstrap/platforms/test_macos.py @@ -0,0 +1,64 @@ +from pathlib import Path + +import pytest + +from bootstrap.platforms import macos +from bootstrap.platforms.macos import ARCHFLAGS, CPU_BACKEND, HOMEBREW, HOMEBREW_SITE, PORTAUDIO, MacOS + +PORTAUDIO_PREFIX = "/opt/homebrew/opt/portaudio" + + +class TestMacOS: + def test_a_virtual_environment_runs_bin_python(self) -> None: + assert MacOS().interpreter(Path(".venv-build")) == Path(".venv-build", "bin", "python") + + def test_a_bundle_is_refused_with_the_way_to_run_from_source(self) -> None: + with pytest.raises(SystemExit, match="make setup"): + MacOS().bundling() + + def test_portaudio_is_installed_through_homebrew(self) -> None: + assert MacOS().system_packages() == ((HOMEBREW, "install", PORTAUDIO),) + + def test_with_homebrew_the_package_manager_is_present(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(macos.shutil, "which", lambda name: f"/opt/homebrew/bin/{name}") + + assert MacOS().missing_package_manager() is None + + def test_without_homebrew_the_refusal_names_where_to_get_it(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(macos.shutil, "which", lambda name: None) + + refusal = MacOS().missing_package_manager() + + assert refusal is not None + assert HOMEBREW_SITE in refusal + + def test_the_setup_pins_the_native_architecture(self) -> None: + variables = MacOS().setup_variables({"PATH": "/usr/bin"}, machine="arm64") + + assert variables == {"PATH": "/usr/bin", ARCHFLAGS: "-arch arm64"} + + def test_a_build_compiles_against_homebrew_s_portaudio(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(MacOS, "homebrew_prefix", staticmethod(lambda package: PORTAUDIO_PREFIX)) + + assert MacOS().build_flags(machine="arm64") == ( + f"CFLAGS=-I{PORTAUDIO_PREFIX}/include", + f"LDFLAGS=-L{PORTAUDIO_PREFIX}/lib", + f"{ARCHFLAGS}=-arch arm64", + ) + + def test_a_build_without_homebrew_s_portaudio_is_refused(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(MacOS, "homebrew_prefix", staticmethod(lambda package: "")) + + with pytest.raises(SystemExit, match="Homebrew"): + MacOS().build_flags(machine="arm64") + + def test_the_cpu_backend_is_the_one_it_runs(self) -> None: + assert MacOS().cpu_backend_reason == CPU_BACKEND + assert MacOS().nvidia_smi_locations({}) == () + + +class TestHomebrewPrefix: + def test_without_homebrew_the_prefix_is_empty(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(macos.shutil, "which", lambda name: None) + + assert MacOS.homebrew_prefix(PORTAUDIO) == "" diff --git a/tests/unit/scripts/bootstrap/platforms/test_windows.py b/tests/unit/scripts/bootstrap/platforms/test_windows.py new file mode 100644 index 000000000..77fc8fb3a --- /dev/null +++ b/tests/unit/scripts/bootstrap/platforms/test_windows.py @@ -0,0 +1,27 @@ +from pathlib import Path + +from bootstrap.platforms.windows import Windows + + +class TestWindows: + def test_a_virtual_environment_runs_scripts_python(self) -> None: + assert Windows().interpreter(Path(".venv-build")) == Path(".venv-build", "Scripts", "python.exe") + + def test_a_bundle_launcher_carries_the_executable_extension(self) -> None: + launcher = Windows().bundling().launcher(Path("bin"), name="sampletones", release=True) + + assert launcher == Path("bin", "sampletones", "sampletones.exe") + + def test_the_installer_carries_every_system_package(self) -> None: + assert Windows().system_packages() == () + assert Windows().missing_package_manager() is None + + def test_the_setup_and_the_build_take_the_variables_as_given(self) -> None: + assert Windows().setup_variables({"PATH": "C:/Python"}, machine="AMD64") == {"PATH": "C:/Python"} + assert Windows().build_flags(machine="AMD64") == () + + def test_the_driver_s_fixed_locations_are_read_from_the_variables_that_are_set(self) -> None: + locations = Windows().nvidia_smi_locations({"SystemRoot": "C:/Windows"}) + + assert Windows().cpu_backend_reason is None + assert locations == (Path("C:/Windows", "System32", "nvidia-smi.exe"),) diff --git a/tests/unit/scripts/test_detect_cuda.py b/tests/unit/scripts/bootstrap/test_cuda.py similarity index 66% rename from tests/unit/scripts/test_detect_cuda.py rename to tests/unit/scripts/bootstrap/test_cuda.py index 60012eb6d..cb565bfa9 100644 --- a/tests/unit/scripts/test_detect_cuda.py +++ b/tests/unit/scripts/bootstrap/test_cuda.py @@ -7,11 +7,13 @@ import pytest +from bootstrap import cuda +from bootstrap.platforms.linux import Linux +from bootstrap.platforms.macos import MacOS +from bootstrap.platforms.windows import Windows +from bootstrap.project import GPU_CUDA11_EXTRA, GPU_EXTRA from tests.suite.base import BaseTestSuite from tests.suite.case import BaseRegularTestCase -from tests.suite.scripts import load_script - -detect_cuda = load_script("detect_cuda.py") def _completed( @@ -42,32 +44,32 @@ class TestCase(BaseRegularTestCase): TestCase( label="cuda_12_0_selects_gpu", cuda_version=(12, 0), - expected="gpu", + expected=GPU_EXTRA, ), TestCase( label="cuda_12_9_selects_gpu", cuda_version=(12, 9), - expected="gpu", + expected=GPU_EXTRA, ), TestCase( label="cuda_13_0_selects_gpu", cuda_version=(13, 0), - expected="gpu", + expected=GPU_EXTRA, ), TestCase( label="cuda_14_2_selects_gpu", cuda_version=(14, 2), - expected="gpu", + expected=GPU_EXTRA, ), TestCase( label="cuda_11_8_selects_legacy", cuda_version=(11, 8), - expected="gpu-cuda11", + expected=GPU_CUDA11_EXTRA, ), TestCase( label="cuda_11_0_selects_legacy", cuda_version=(11, 0), - expected="gpu-cuda11", + expected=GPU_CUDA11_EXTRA, ), TestCase( label="cuda_10_2_keeps_cpu", @@ -87,7 +89,7 @@ class TestCase(BaseRegularTestCase): ids=lambda test_case: test_case.label, ) def test_select_extra(self, test_case: TestCase) -> None: - assert detect_cuda.select_extra(test_case.cuda_version) == test_case.expected + assert cuda.select_extra(test_case.cuda_version) == test_case.expected class TestQueryDriverCudaVersion(BaseTestSuite): @@ -135,8 +137,8 @@ def fake_run( ) -> subprocess.CompletedProcess[str]: return _completed(test_case.output) - monkeypatch.setattr(detect_cuda.subprocess, "run", fake_run) - assert detect_cuda.query_driver_cuda_version(Path("nvidia-smi")) == test_case.expected + monkeypatch.setattr(cuda.subprocess, "run", fake_run) + assert cuda.query_driver_cuda_version(Path("nvidia-smi")) == test_case.expected def test_falls_back_to_query_flag( self, @@ -149,8 +151,8 @@ def fake_run( output = QUERY_OUTPUT_CUDA11 if "-q" in command else NO_VERSION_OUTPUT return _completed(output) - monkeypatch.setattr(detect_cuda.subprocess, "run", fake_run) - assert detect_cuda.query_driver_cuda_version(Path("nvidia-smi")) == (11, 8) + monkeypatch.setattr(cuda.subprocess, "run", fake_run) + assert cuda.query_driver_cuda_version(Path("nvidia-smi")) == (11, 8) def test_missing_executable_keeps_cpu( self, @@ -162,8 +164,8 @@ def fake_run( ) -> subprocess.CompletedProcess[str]: raise OSError("nvidia-smi is not executable") - monkeypatch.setattr(detect_cuda.subprocess, "run", fake_run) - assert detect_cuda.query_driver_cuda_version(Path("nvidia-smi")) is None + monkeypatch.setattr(cuda.subprocess, "run", fake_run) + assert cuda.query_driver_cuda_version(Path("nvidia-smi")) is None def test_nonzero_return_code_keeps_cpu( self, @@ -175,50 +177,55 @@ def fake_run( ) -> subprocess.CompletedProcess[str]: return _completed(TABLE_OUTPUT_CUDA12, returncode=9) - monkeypatch.setattr(detect_cuda.subprocess, "run", fake_run) - assert detect_cuda.query_driver_cuda_version(Path("nvidia-smi")) is None + monkeypatch.setattr(cuda.subprocess, "run", fake_run) + assert cuda.query_driver_cuda_version(Path("nvidia-smi")) is None class TestFindNvidiaSmi: def test_uses_path_when_present(self, monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr( - detect_cuda.shutil, - "which", - lambda name: "/usr/bin/nvidia-smi", - ) - assert detect_cuda.find_nvidia_smi(system="Linux") == Path("/usr/bin/nvidia-smi") + monkeypatch.setattr(cuda.shutil, "which", lambda name: "/usr/bin/nvidia-smi") + + assert cuda.find_nvidia_smi(Linux(), {}) == Path("/usr/bin/nvidia-smi") def test_absent_on_linux(self, monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr(detect_cuda.shutil, "which", lambda name: None) - assert detect_cuda.find_nvidia_smi(system="Linux") is None + monkeypatch.setattr(cuda.shutil, "which", lambda name: None) + + assert cuda.find_nvidia_smi(Linux(), {}) is None + + def test_a_fixed_windows_location_stands_in_for_the_path( + self, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + ) -> None: + monkeypatch.setattr(cuda.shutil, "which", lambda name: None) + (tmp_path / "System32").mkdir() + (tmp_path / "System32" / "nvidia-smi.exe").write_bytes(b"") + + assert ( + cuda.find_nvidia_smi(Windows(), {"SystemRoot": str(tmp_path)}) == tmp_path / "System32" / "nvidia-smi.exe" + ) class TestDetect: def test_macos_keeps_cpu(self) -> None: - detection = detect_cuda.detect(system="Darwin") + detection = cuda.detect(MacOS(), {}) + assert detection.extra is None - assert detection.cuda_version is None + assert detection.reason == MacOS().cpu_backend_reason def test_no_driver_keeps_cpu(self, monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr(detect_cuda.shutil, "which", lambda name: None) - detection = detect_cuda.detect(system="Linux") + monkeypatch.setattr(cuda.shutil, "which", lambda name: None) + + detection = cuda.detect(Linux(), {}) + assert detection.extra is None - assert detection.nvidia_smi is None + assert cuda.NVIDIA_SMI in detection.reason - def test_selects_gpu_for_cuda12_driver( - self, - monkeypatch: pytest.MonkeyPatch, - ) -> None: - monkeypatch.setattr( - detect_cuda.shutil, - "which", - lambda name: "/usr/bin/nvidia-smi", - ) - monkeypatch.setattr( - detect_cuda.subprocess, - "run", - lambda command, **_: _completed(TABLE_OUTPUT_CUDA12), - ) - detection = detect_cuda.detect(system="Linux") - assert detection.cuda_version == (12, 4) - assert detection.extra == "gpu" + def test_selects_gpu_for_cuda12_driver(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(cuda.shutil, "which", lambda name: "/usr/bin/nvidia-smi") + monkeypatch.setattr(cuda.subprocess, "run", lambda command, **_: _completed(TABLE_OUTPUT_CUDA12)) + + detection = cuda.detect(Linux(), {}) + + assert detection.extra == GPU_EXTRA + assert "12.4" in detection.reason diff --git a/tests/unit/scripts/bootstrap/test_files.py b/tests/unit/scripts/bootstrap/test_files.py new file mode 100644 index 000000000..d20fafc60 --- /dev/null +++ b/tests/unit/scripts/bootstrap/test_files.py @@ -0,0 +1,23 @@ +from pathlib import Path + +from bootstrap.files import remove_path + + +class TestRemovePath: + def test_a_directory_goes_with_everything_under_it(self, tmp_path: Path) -> None: + directory = tmp_path / "bin" + (directory / "nested").mkdir(parents=True) + (directory / "nested" / "file").write_text("") + + assert remove_path(directory) + assert not directory.exists() + + def test_a_file_goes(self, tmp_path: Path) -> None: + file = tmp_path / "sampletones.spec" + file.write_text("") + + assert remove_path(file) + assert not file.exists() + + def test_an_absent_path_is_reported_as_nothing_removed(self, tmp_path: Path) -> None: + assert not remove_path(tmp_path / "absent") diff --git a/tests/unit/scripts/bootstrap/test_interpreter.py b/tests/unit/scripts/bootstrap/test_interpreter.py new file mode 100644 index 000000000..687dd7f65 --- /dev/null +++ b/tests/unit/scripts/bootstrap/test_interpreter.py @@ -0,0 +1,42 @@ +import subprocess +import sys +from typing import Final + +import pytest + +from bootstrap.interpreter import DOWNLOADS, REQUIRED_VERSION, require_python, running_version +from sampletones_tools.checks.paths import SCRIPTS_ROOT + +OLDER_INTERPRETER: Final[str] = "(3, 10, 0, 'final', 0)" + + +class TestRequirePython: + def test_an_interpreter_at_least_the_version_passes(self) -> None: + require_python((3, 8)) + + def test_an_older_interpreter_is_refused_with_the_download_site(self) -> None: + with pytest.raises(SystemExit) as refused: + require_python((99, 0)) + + assert DOWNLOADS in str(refused.value) + + def test_the_running_version_reads_major_minor_and_micro(self) -> None: + assert running_version() == ".".join(str(part) for part in sys.version_info[:3]) + + +class TestBootstrapGate: + def test_importing_a_helper_on_an_older_interpreter_names_the_required_version(self) -> None: + probe = f"import sys; sys.version_info = {OLDER_INTERPRETER}; import bootstrap.project" + + completed = subprocess.run( + [sys.executable, "-c", probe], + cwd=SCRIPTS_ROOT, + capture_output=True, + text=True, + check=False, + ) + + major, minor = REQUIRED_VERSION + assert completed.returncode == 1 + assert f"Python {major}.{minor} or newer is required" in completed.stderr + assert DOWNLOADS in completed.stderr diff --git a/tests/unit/scripts/bootstrap/test_layout.py b/tests/unit/scripts/bootstrap/test_layout.py new file mode 100644 index 000000000..a8308884e --- /dev/null +++ b/tests/unit/scripts/bootstrap/test_layout.py @@ -0,0 +1,9 @@ +from bootstrap.layout import PROJECT_FILE, SOURCE_DIRECTORY, repository_root + + +class TestRepositoryRoot: + def test_the_root_holds_the_project_beside_the_sources(self) -> None: + root = repository_root() + + assert (root / PROJECT_FILE).is_file() + assert (root / SOURCE_DIRECTORY / "sampletones" / "__main__.py").is_file() diff --git a/tests/unit/scripts/bootstrap/test_passes.py b/tests/unit/scripts/bootstrap/test_passes.py new file mode 100644 index 000000000..0703b4e07 --- /dev/null +++ b/tests/unit/scripts/bootstrap/test_passes.py @@ -0,0 +1,47 @@ +from pathlib import Path + +import pytest + +from bootstrap.passes import Pass, run_pass, run_passes +from tests.suite.bootstrap import RecordingRunner + +PASSES = ( + Pass("first", "First...", ("first", "command")), + Pass("second", "Second...", ("second", "command")), + Pass("third", "Third...", ("third", "command")), +) + + +class TestRunPass: + def test_the_pass_is_announced_and_run_from_the_repository( + self, + capsys: pytest.CaptureFixture[str], + tmp_path: Path, + ) -> None: + runner = RecordingRunner({}, None) + + assert run_pass(PASSES[0], root=tmp_path, runner=runner, environment={}) == 0 + assert runner.lines == ["first command"] + assert runner.commands[0].cwd == tmp_path + assert capsys.readouterr().out == "First...\n" + + def test_the_status_is_the_command_s_own(self, tmp_path: Path) -> None: + assert run_pass(PASSES[1], root=tmp_path, runner=RecordingRunner({"second": 5}, None), environment={}) == 5 + + +class TestRunPasses: + def test_every_pass_runs_and_the_failed_ones_are_named(self, capsys: pytest.CaptureFixture[str]) -> None: + runner = RecordingRunner({"second": 1}, None) + + failed = run_passes(PASSES, root=Path("/repository"), runner=runner, environment={"PATH": "/usr/bin"}) + + assert failed == ["second"] + assert runner.lines == ["first command", "second command", "third command"] + assert all(recorded.cwd == Path("/repository") for recorded in runner.commands) + assert all(recorded.environment == {"PATH": "/usr/bin"} for recorded in runner.commands) + assert capsys.readouterr().out == "First...\nSecond...\nThird...\n" + + def test_a_clean_run_names_nothing(self) -> None: + runner = RecordingRunner({}, None) + + assert run_passes(PASSES, root=Path("/repository"), runner=runner, environment={}) == [] diff --git a/tests/unit/scripts/bootstrap/test_preflight.py b/tests/unit/scripts/bootstrap/test_preflight.py new file mode 100644 index 000000000..08f061205 --- /dev/null +++ b/tests/unit/scripts/bootstrap/test_preflight.py @@ -0,0 +1,85 @@ +from pathlib import Path + +import pytest + +from bootstrap.platforms.linux import Linux +from bootstrap.preflight import can_import, check_build_interpreter +from tests.suite.bootstrap import RecordingRunner + + +@pytest.fixture +def python(tmp_path: Path) -> Path: + return tmp_path / "python" + + +class TestCanImport: + def test_the_probe_is_quiet_and_answers_the_status(self, python: Path, tmp_path: Path) -> None: + runner = RecordingRunner({"import tkinter": 1}, None) + + assert can_import(python, "pyaudio", runner=runner, cwd=tmp_path, environment={}) + assert not can_import(python, "tkinter", runner=runner, cwd=tmp_path, environment={}) + assert all(recorded.quiet for recorded in runner.commands) + + +class TestCheckBuildInterpreter: + def test_an_interpreter_without_audio_playback_is_refused_with_the_platform_s_advice( + self, + python: Path, + tmp_path: Path, + ) -> None: + with pytest.raises(SystemExit, match="make system-deps"): + check_build_interpreter( + python, + Linux().bundling(), + release=False, + runner=RecordingRunner({"import pyaudio": 1}, None), + cwd=tmp_path, + environment={}, + ) + + def test_a_release_without_tk_is_refused(self, python: Path, tmp_path: Path) -> None: + with pytest.raises(SystemExit, match="tkinter"): + check_build_interpreter( + python, + Linux().bundling(), + release=True, + runner=RecordingRunner({"import tkinter": 1}, None), + cwd=tmp_path, + environment={}, + ) + + def test_a_development_bundle_without_tk_is_warned( + self, + python: Path, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + ) -> None: + check_build_interpreter( + python, + Linux().bundling(), + release=False, + runner=RecordingRunner({"import tkinter": 1}, None), + cwd=tmp_path, + environment={}, + ) + + assert "WARNING" in capsys.readouterr().out + + def test_an_interpreter_carrying_both_passes( + self, + python: Path, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + ) -> None: + check_build_interpreter( + python, + Linux().bundling(), + release=True, + runner=RecordingRunner({}, None), + cwd=tmp_path, + environment={}, + ) + + output = capsys.readouterr().out + assert "pyaudio: available" in output + assert "tkinter: available" in output diff --git a/tests/unit/scripts/bootstrap/test_processes.py b/tests/unit/scripts/bootstrap/test_processes.py new file mode 100644 index 000000000..1267cadbe --- /dev/null +++ b/tests/unit/scripts/bootstrap/test_processes.py @@ -0,0 +1,52 @@ +import os +import sys +from pathlib import Path + +import pytest + +from bootstrap.processes import expect_success, run +from tests.suite.bootstrap import RecordingRunner + + +class TestRun: + def test_the_status_is_the_command_s_own(self, tmp_path: Path) -> None: + status = run( + (sys.executable, "-c", "raise SystemExit(3)"), + cwd=tmp_path, + environment={}, + quiet=True, + ) + + assert status == 3 + + def test_what_the_script_printed_leads_the_command_s_output( + self, + tmp_path: Path, + capfd: pytest.CaptureFixture[str], + ) -> None: + print("before") + run( + (sys.executable, "-c", "print('inside')"), + cwd=tmp_path, + environment=os.environ, + quiet=False, + ) + + assert capfd.readouterr().out.splitlines() == ["before", "inside"] + + +class TestExpectSuccess: + def test_a_succeeding_command_passes(self, tmp_path: Path) -> None: + runner = RecordingRunner({}, None) + + expect_success(runner, ("echo", "hello"), cwd=tmp_path, environment={"KEY": "value"}) + + assert runner.lines == ["echo hello"] + assert runner.commands[0].environment == {"KEY": "value"} + assert not runner.commands[0].quiet + + def test_a_failing_command_stops_the_script_naming_it(self, tmp_path: Path) -> None: + runner = RecordingRunner({"echo": 2}, None) + + with pytest.raises(SystemExit, match="echo hello exited with status 2"): + expect_success(runner, ("echo", "hello"), cwd=tmp_path, environment={}) diff --git a/tests/unit/scripts/bootstrap/test_project.py b/tests/unit/scripts/bootstrap/test_project.py new file mode 100644 index 000000000..482df6f7c --- /dev/null +++ b/tests/unit/scripts/bootstrap/test_project.py @@ -0,0 +1,59 @@ +import tomllib +from pathlib import Path + +import pytest + +from bootstrap.layout import SOURCE_DIRECTORY, repository_root +from bootstrap.project import ( + DEVELOPMENT_GROUP, + GPU_CUDA11_EXTRA, + NAMED_EXTRAS, + NAMED_GROUPS, + TAG_PREFIX, + parse_project, + read_project, +) +from tests.suite.bootstrap import PROJECT_DOCUMENT, PROJECT_NAME, PROJECT_VERSION, write_project + + +class TestReadProject: + def test_the_repository_states_every_name_the_scripts_install(self) -> None: + project = read_project(repository_root()) + + assert set(NAMED_EXTRAS) <= set(project.extras) + assert set(NAMED_GROUPS) <= set(project.groups) + assert (repository_root() / project.entry_script).is_file() + assert all((repository_root() / SOURCE_DIRECTORY / package).is_dir() for package in project.packages) + + def test_the_facts_are_read_from_the_file(self, tmp_path: Path) -> None: + project = read_project(write_project(tmp_path)) + + assert project.name == PROJECT_NAME + assert project.version == PROJECT_VERSION + assert project.entry_module == f"{PROJECT_NAME}.__main__" + assert project.entry_script == f"{SOURCE_DIRECTORY}/{PROJECT_NAME}/__main__.py" + assert project.packages == (PROJECT_NAME, f"{PROJECT_NAME}_core") + assert project.tag == f"{TAG_PREFIX}{PROJECT_VERSION}" + + +class TestParseProject: + def test_a_missing_extra_is_refused_by_name(self) -> None: + document = tomllib.loads(PROJECT_DOCUMENT) + del document["project"]["optional-dependencies"][GPU_CUDA11_EXTRA] + + with pytest.raises(SystemExit, match=GPU_CUDA11_EXTRA): + parse_project(document) + + def test_a_missing_group_is_refused_by_name(self) -> None: + document = tomllib.loads(PROJECT_DOCUMENT) + del document["dependency-groups"][DEVELOPMENT_GROUP] + + with pytest.raises(SystemExit, match=DEVELOPMENT_GROUP): + parse_project(document) + + def test_a_project_without_its_command_is_refused(self) -> None: + document = tomllib.loads(PROJECT_DOCUMENT) + del document["project"]["scripts"] + + with pytest.raises(SystemExit, match=r"\[project.scripts\]"): + parse_project(document) diff --git a/tests/unit/scripts/bootstrap/test_venv_build.py b/tests/unit/scripts/bootstrap/test_venv_build.py new file mode 100644 index 000000000..ce8bdc9de --- /dev/null +++ b/tests/unit/scripts/bootstrap/test_venv_build.py @@ -0,0 +1,60 @@ +import sys +from pathlib import Path + +from bootstrap.layout import BUILD_ENVIRONMENT +from bootstrap.platforms.linux import Linux +from bootstrap.venv_build import ensure_build_venv, install +from tests.suite.bootstrap import RecordingRunner + + +class TestEnsureBuildVenv: + def test_a_missing_environment_is_created_by_the_running_interpreter(self, tmp_path: Path) -> None: + runner = RecordingRunner({}, None) + + python = ensure_build_venv(tmp_path, Linux(), runner=runner, environment={}) + + assert python == Linux().interpreter(tmp_path / BUILD_ENVIRONMENT) + assert runner.lines == [f"{sys.executable} -m venv --clear {tmp_path / BUILD_ENVIRONMENT}"] + + def test_an_environment_with_its_interpreter_is_kept(self, tmp_path: Path) -> None: + python = Linux().interpreter(tmp_path / BUILD_ENVIRONMENT) + python.parent.mkdir(parents=True) + python.write_text("") + runner = RecordingRunner({}, None) + + assert ensure_build_venv(tmp_path, Linux(), runner=runner, environment={}) == python + assert runner.lines == [] + + def test_a_directory_an_interrupted_creation_left_is_created_again(self, tmp_path: Path) -> None: + (tmp_path / BUILD_ENVIRONMENT).mkdir() + runner = RecordingRunner({}, None) + + ensure_build_venv(tmp_path, Linux(), runner=runner, environment={}) + + assert runner.lines == [f"{sys.executable} -m venv --clear {tmp_path / BUILD_ENVIRONMENT}"] + + +class TestInstall: + def test_pip_is_upgraded_then_the_package_installed_with_its_extras(self, tmp_path: Path) -> None: + runner = RecordingRunner({}, None) + python = tmp_path / "python" + + install( + tmp_path, + python, + extras=("build", "gpu"), + runner=runner, + environment={"PATH": "/usr/bin"}, + ) + + assert runner.lines == [ + f"{python} -m pip install --upgrade pip", + f"{python} -m pip install .[build,gpu]", + ] + + def test_every_install_refuses_an_interpreter_outside_a_virtual_environment(self, tmp_path: Path) -> None: + runner = RecordingRunner({}, None) + + install(tmp_path, tmp_path / "python", extras=("build",), runner=runner, environment={}) + + assert all(recorded.environment["PIP_REQUIRE_VIRTUALENV"] == "1" for recorded in runner.commands) diff --git a/tests/unit/scripts/checks/test_import_boundary.py b/tests/unit/scripts/checks/test_import_boundary.py deleted file mode 100644 index 2e0798f3b..000000000 --- a/tests/unit/scripts/checks/test_import_boundary.py +++ /dev/null @@ -1,44 +0,0 @@ -from pathlib import Path -from typing import Final - -import pytest - -from tests.suite.scripts import load_script -from tests.suite.source import write_module - -check_import_boundary = load_script("checks/import_boundary.py") - -APPLICATION: Final[str] = "sampletones_application" - -VISUAL_IMPORT: Final[str] = "import dearpygui.dearpygui as dpg\n" -PLAIN_IMPORT: Final[str] = "from sampletones_core.project.project import Project\n" - - -class TestMain: - def test_the_repository_holds_its_import_boundaries(self) -> None: - assert check_import_boundary.main(["--all"]) == 0 - - def test_a_forbidden_import_is_reported_where_it_sits( - self, - tmp_path: Path, - capsys: pytest.CaptureFixture[str], - ) -> None: - path = write_module(tmp_path / APPLICATION / "logic", "direct.py", VISUAL_IMPORT) - - exit_code = check_import_boundary.main(["--all", "--source", str(tmp_path)]) - - assert exit_code == 1 - error = capsys.readouterr().err - assert f"{path}:1" in error - assert "dearpygui" in error - - def test_named_files_narrow_the_run_to_themselves( - self, - tmp_path: Path, - capsys: pytest.CaptureFixture[str], - ) -> None: - write_module(tmp_path / APPLICATION / "logic", "reported.py", VISUAL_IMPORT) - clean = write_module(tmp_path / APPLICATION / "logic", "clean.py", PLAIN_IMPORT) - - assert check_import_boundary.main([str(clean), "--source", str(tmp_path)]) == 0 - assert capsys.readouterr().err == "" diff --git a/tests/unit/scripts/ci/checks/test_bundle.py b/tests/unit/scripts/ci/checks/test_bundle.py deleted file mode 100644 index 06e339dcb..000000000 --- a/tests/unit/scripts/ci/checks/test_bundle.py +++ /dev/null @@ -1,198 +0,0 @@ -import subprocess -from dataclasses import dataclass -from pathlib import Path -from typing import Any, List, Sequence - -import pytest - -from tests.suite.base import BaseTestSuite -from tests.suite.case import BaseRegularTestCase -from tests.suite.scripts import load_script - -check_bundle = load_script("ci/checks/bundle.py") - -NOTICES = ("LICENSE", "THIRD-PARTY-NOTICES.md", "THIRD-PARTY-LICENSES.txt") - - -@pytest.fixture -def bundle(tmp_path: Path) -> Path: - source = tmp_path / "bin" / "sampletones" - source.mkdir(parents=True) - for name in NOTICES: - (source / name).write_text(name) - - return source - - -def _install_launcher(bundle: Path) -> Path: - launcher: Path = check_bundle.launcher_path( - bundle, - system=check_bundle.platform.system(), - ) - launcher.write_bytes(b"launcher") - return launcher - - -def _stub_run( - monkeypatch: pytest.MonkeyPatch, - *, - returncode: int, -) -> List[Sequence[str]]: - commands: List[Sequence[str]] = [] - - def fake_run( - command: Sequence[str], - **_: Any, - ) -> subprocess.CompletedProcess[str]: - commands.append(command) - return subprocess.CompletedProcess( - args=list(command), - returncode=returncode, - ) - - monkeypatch.setattr(check_bundle.subprocess, "run", fake_run) - return commands - - -class TestLauncherPath(BaseTestSuite): - @dataclass(frozen=True, kw_only=True) - class TestCase(BaseRegularTestCase): - system: str - expected: str - - test_cases = ( - TestCase( - label="windows_launcher_carries_an_extension", - system="Windows", - expected="sampletones.exe", - ), - TestCase( - label="linux_launcher", - system="Linux", - expected="sampletones", - ), - TestCase( - label="macos_launcher", - system="Darwin", - expected="sampletones", - ), - ) - - @pytest.mark.parametrize( - "test_case", - test_cases, - ids=lambda test_case: test_case.label, - ) - def test_launcher_path( - self, - test_case: "TestLauncherPath.TestCase", - tmp_path: Path, - ) -> None: - assert check_bundle.launcher_path(tmp_path, system=test_case.system).name == test_case.expected - - -class TestMissingNotices: - def test_a_complete_bundle_lacks_nothing(self, bundle: Path) -> None: - assert check_bundle.missing_notices(bundle) == [] - - def test_every_absent_notice_is_reported(self, bundle: Path) -> None: - (bundle / "LICENSE").unlink() - (bundle / "THIRD-PARTY-LICENSES.txt").unlink() - - assert check_bundle.missing_notices(bundle) == [ - "LICENSE", - "THIRD-PARTY-LICENSES.txt", - ] - - def test_a_notice_directory_counts_as_absent(self, bundle: Path) -> None: - (bundle / "LICENSE").unlink() - (bundle / "LICENSE").mkdir() - - assert check_bundle.missing_notices(bundle) == ["LICENSE"] - - -class TestCarriedBuildTools: - def test_an_application_bundle_holds_to_its_notices(self, bundle: Path) -> None: - assert check_bundle.carried_build_tools(bundle) == [] - - def test_a_build_tool_beside_the_application_is_reported(self, bundle: Path) -> None: - (bundle / check_bundle.INTERNAL_DIRECTORY / "PIL").mkdir(parents=True) - - assert check_bundle.carried_build_tools(bundle) == ["PIL"] - - def test_a_build_tool_beside_the_launcher_is_reported(self, bundle: Path) -> None: - (bundle / "PIL").mkdir() - - assert check_bundle.carried_build_tools(bundle) == ["PIL"] - - -class TestMain: - def test_a_complete_bundle_passes( - self, - bundle: Path, - monkeypatch: pytest.MonkeyPatch, - ) -> None: - launcher = _install_launcher(bundle) - commands = _stub_run(monkeypatch, returncode=0) - - assert check_bundle.main([str(bundle)]) == 0 - assert commands == [[str(launcher), "--version"]] - - def test_a_missing_notice_is_annotated_as_an_error( - self, - bundle: Path, - capsys: pytest.CaptureFixture[str], - ) -> None: - _install_launcher(bundle) - (bundle / "THIRD-PARTY-NOTICES.md").unlink() - - assert check_bundle.main([str(bundle)]) == 1 - - output = capsys.readouterr().out - assert output.startswith("::error::") - assert "THIRD-PARTY-NOTICES.md" in output - - def test_bundled_build_tooling_is_annotated_as_an_error( - self, - bundle: Path, - capsys: pytest.CaptureFixture[str], - ) -> None: - _install_launcher(bundle) - (bundle / check_bundle.INTERNAL_DIRECTORY / "PIL").mkdir(parents=True) - - assert check_bundle.main([str(bundle)]) == 1 - - output = capsys.readouterr().out - assert output.startswith("::error::") - assert "PIL" in output - - def test_a_missing_launcher_is_annotated_as_an_error( - self, - bundle: Path, - capsys: pytest.CaptureFixture[str], - ) -> None: - assert check_bundle.main([str(bundle)]) == 1 - assert capsys.readouterr().out.startswith("::error::") - - def test_a_launcher_that_fails_propagates_its_status( - self, - bundle: Path, - monkeypatch: pytest.MonkeyPatch, - ) -> None: - _install_launcher(bundle) - _stub_run(monkeypatch, returncode=3) - - assert check_bundle.main([str(bundle)]) == 3 - - def test_the_launcher_runs_only_once_the_notices_are_present( - self, - bundle: Path, - monkeypatch: pytest.MonkeyPatch, - ) -> None: - _install_launcher(bundle) - (bundle / "LICENSE").unlink() - commands = _stub_run(monkeypatch, returncode=0) - - check_bundle.main([str(bundle)]) - - assert commands == [] diff --git a/tests/unit/scripts/ci/checks/test_version_tag.py b/tests/unit/scripts/ci/checks/test_version_tag.py deleted file mode 100644 index 77725fb10..000000000 --- a/tests/unit/scripts/ci/checks/test_version_tag.py +++ /dev/null @@ -1,139 +0,0 @@ -from dataclasses import dataclass - -import pytest - -from tests.suite.base import BaseTestSuite -from tests.suite.case import BaseRegularTestCase -from tests.suite.scripts import load_script - -check_version_tag = load_script("ci/checks/version_tag.py") - - -class TestVersionFromTag(BaseTestSuite): - @dataclass(frozen=True, kw_only=True) - class TestCase(BaseRegularTestCase): - tag: str - expected: str - - test_cases = ( - TestCase( - label="release_tag", - tag="v0.3.0", - expected="0.3.0", - ), - TestCase( - label="prerelease_tag", - tag="v0.3.0.dev1", - expected="0.3.0.dev1", - ), - TestCase( - label="release_candidate", - tag="v1.0.0rc2", - expected="1.0.0rc2", - ), - TestCase( - label="bare_version", - tag="0.3.0", - expected="0.3.0", - ), - TestCase( - label="single_prefix_is_dropped", - tag="vv0.3.0", - expected="v0.3.0", - ), - ) - - @pytest.mark.parametrize( - "test_case", - test_cases, - ids=lambda test_case: test_case.label, - ) - def test_version_from_tag(self, test_case: TestCase) -> None: - assert check_version_tag.version_from_tag(test_case.tag) == test_case.expected - - -class TestTagNamesVersion(BaseTestSuite): - @dataclass(frozen=True, kw_only=True) - class TestCase(BaseRegularTestCase): - tag: str - project_version: str - expected: bool - - test_cases = ( - TestCase( - label="tag_matches", - tag="v0.3.0", - project_version="0.3.0", - expected=True, - ), - TestCase( - label="prerelease_matches", - tag="v0.3.0.dev1", - project_version="0.3.0.dev1", - expected=True, - ), - TestCase( - label="patch_differs", - tag="v0.3.1", - project_version="0.3.0", - expected=False, - ), - TestCase( - label="project_ahead_of_tag", - tag="v0.3.0", - project_version="0.4.0", - expected=False, - ), - TestCase( - label="prerelease_against_release", - tag="v0.3.0", - project_version="0.3.0.dev1", - expected=False, - ), - ) - - @pytest.mark.parametrize( - "test_case", - test_cases, - ids=lambda test_case: test_case.label, - ) - def test_tag_names_version( - self, - test_case: TestCase, - ) -> None: - result = check_version_tag.tag_names_version( - tag=test_case.tag, - project_version=test_case.project_version, - ) - - assert result is test_case.expected - - -class TestMain: - def test_matching_version_succeeds( - self, - capsys: pytest.CaptureFixture[str], - ) -> None: - assert ( - check_version_tag.main( - ["--tag", "v0.3.0", "--project-version", "0.3.0"], - ) - == 0 - ) - assert "matches" in capsys.readouterr().out - - def test_mismatched_version_is_annotated_as_an_error( - self, - capsys: pytest.CaptureFixture[str], - ) -> None: - assert ( - check_version_tag.main( - ["--tag", "v0.3.1", "--project-version", "0.3.0"], - ) - == 1 - ) - - output = capsys.readouterr().out - assert output.startswith("::error::") - assert "v0.3.1" in output - assert "0.3.0" in output diff --git a/tests/unit/scripts/ci/test_zip_bundle.py b/tests/unit/scripts/test_archive_bundle.py similarity index 54% rename from tests/unit/scripts/ci/test_zip_bundle.py rename to tests/unit/scripts/test_archive_bundle.py index a5c40633d..47bfd7a46 100644 --- a/tests/unit/scripts/ci/test_zip_bundle.py +++ b/tests/unit/scripts/test_archive_bundle.py @@ -4,11 +4,17 @@ import pytest +from bootstrap.layout import BUNDLES, DISTRIBUTION +from bootstrap.platforms.macos import MacOS +from bootstrap.platforms.windows import Windows +from bootstrap.project import TAG_PREFIX, read_project +from tests.suite.bootstrap import PROJECT_NAME, PROJECT_VERSION, write_project from tests.suite.scripts import load_script -zip_bundle = load_script("ci/zip_bundle.py") +archive_bundle = load_script("archive_bundle.py") -ROOT = "sampletones-v0.3.0-windows-x86_64" +LABEL = "windows-x86_64" +ROOT = f"{PROJECT_NAME}-{TAG_PREFIX}{PROJECT_VERSION}-{LABEL}" LAUNCHER = "sampletones.exe" LIBRARY = "_internal/python312.dll" @@ -37,14 +43,14 @@ def _names(archive: Path) -> List[str]: class TestWriteArchive: def test_every_entry_sits_under_the_root_directory(self, bundle: Path, archive: Path) -> None: - zip_bundle.write_archive(bundle, archive, root=ROOT) + archive_bundle.write_archive(bundle, archive, root=ROOT) names = _names(archive) assert names assert all(name.startswith(f"{ROOT}/") for name in names) def test_bundle_contents_are_archived(self, bundle: Path, archive: Path) -> None: - zip_bundle.write_archive(bundle, archive, root=ROOT) + archive_bundle.write_archive(bundle, archive, root=ROOT) names = _names(archive) assert f"{ROOT}/{LAUNCHER}" in names @@ -52,18 +58,18 @@ def test_bundle_contents_are_archived(self, bundle: Path, archive: Path) -> None assert f"{ROOT}/{NOTICES}" in names def test_empty_directories_are_kept(self, bundle: Path, archive: Path) -> None: - zip_bundle.write_archive(bundle, archive, root=ROOT) + archive_bundle.write_archive(bundle, archive, root=ROOT) assert f"{ROOT}/_internal/empty/" in _names(archive) def test_file_contents_survive_the_round_trip(self, bundle: Path, archive: Path) -> None: - zip_bundle.write_archive(bundle, archive, root=ROOT) + archive_bundle.write_archive(bundle, archive, root=ROOT) with zipfile.ZipFile(archive) as written: assert written.read(f"{ROOT}/{LIBRARY}") == b"library" def test_missing_target_directory_is_created(self, bundle: Path, archive: Path) -> None: - zip_bundle.write_archive(bundle, archive, root=ROOT) + archive_bundle.write_archive(bundle, archive, root=ROOT) assert archive.is_file() @@ -71,14 +77,14 @@ def test_repeated_runs_archive_the_same_order(self, bundle: Path, tmp_path: Path first = tmp_path / "first.zip" second = tmp_path / "second.zip" - zip_bundle.write_archive(bundle, first, root=ROOT) - zip_bundle.write_archive(bundle, second, root=ROOT) + archive_bundle.write_archive(bundle, first, root=ROOT) + archive_bundle.write_archive(bundle, second, root=ROOT) assert _names(first) == _names(second) def test_source_directory_is_left_in_place(self, bundle: Path, archive: Path) -> None: """Archiving reads the bundle where it lies, so the build output keeps its own name.""" - zip_bundle.write_archive(bundle, archive, root=ROOT) + archive_bundle.write_archive(bundle, archive, root=ROOT) assert bundle.is_dir() assert (bundle / LAUNCHER).is_file() @@ -91,25 +97,37 @@ def test_archives_while_a_handle_is_held_inside_the_bundle(self, bundle: Path, a """ with (bundle / LIBRARY).open("rb"): - zip_bundle.write_archive(bundle, archive, root=ROOT) + archive_bundle.write_archive(bundle, archive, root=ROOT) assert f"{ROOT}/{LIBRARY}" in _names(archive) -class TestMain: - def test_reports_success_for_a_built_bundle(self, bundle: Path, archive: Path) -> None: - assert zip_bundle.main([str(bundle), str(archive), "--root", ROOT]) == 0 - assert archive.is_file() +class TestArchiveRoot: + def test_the_root_names_the_project_its_version_and_the_platform(self, tmp_path: Path) -> None: + assert archive_bundle.archive_root(read_project(write_project(tmp_path)), LABEL) == ROOT - def test_reports_a_missing_bundle_directory(self, tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: - absent = tmp_path / "bin" / "sampletones" - assert zip_bundle.main([str(absent), str(tmp_path / "out.zip"), "--root", ROOT]) == 1 - assert "::error::" in capsys.readouterr().out +class TestArchiveRelease: + def test_the_release_bundle_is_archived_into_the_bundles_directory(self, tmp_path: Path) -> None: + root = write_project(tmp_path / "repository") + launcher = Windows().bundling().launcher(root / DISTRIBUTION, name=PROJECT_NAME, release=True) + launcher.parent.mkdir(parents=True) + launcher.write_bytes(b"MZ") - def test_a_missing_bundle_writes_no_archive(self, tmp_path: Path) -> None: - archive = tmp_path / "out.zip" + assert archive_bundle.archive_release(root, Windows(), label=LABEL) == 0 + assert _names(root / BUNDLES / f"{ROOT}.zip") == [f"{ROOT}/{launcher.name}"] - zip_bundle.main([str(tmp_path / "absent"), str(archive), "--root", ROOT]) + def test_a_missing_bundle_is_annotated_and_writes_no_archive( + self, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + ) -> None: + root = write_project(tmp_path) + + assert archive_bundle.archive_release(root, Windows(), label=LABEL) == 1 + assert "::error::" in capsys.readouterr().out + assert not (root / BUNDLES).exists() - assert not archive.exists() + def test_a_system_without_bundles_is_refused(self, tmp_path: Path) -> None: + with pytest.raises(SystemExit, match="make setup"): + archive_bundle.archive_release(write_project(tmp_path), MacOS(), label=LABEL) diff --git a/tests/unit/scripts/test_build_environment.py b/tests/unit/scripts/test_build_environment.py new file mode 100644 index 000000000..738d962a1 --- /dev/null +++ b/tests/unit/scripts/test_build_environment.py @@ -0,0 +1,33 @@ +import pytest + +from bootstrap.platforms.linux import Linux +from bootstrap.platforms.macos import MacOS +from tests.suite.scripts import load_script + +build_environment = load_script("build_environment.py") + +PORTAUDIO_PREFIX = "/opt/homebrew/opt/portaudio" + + +class TestMain: + def test_the_platform_s_flags_are_printed_one_per_line( + self, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], + ) -> None: + monkeypatch.setattr(build_environment, "current_platform", MacOS) + monkeypatch.setattr(MacOS, "homebrew_prefix", staticmethod(lambda package: PORTAUDIO_PREFIX)) + monkeypatch.setattr(build_environment.running, "machine", lambda: "arm64") + + assert build_environment.main([]) == 0 + assert capsys.readouterr().out.splitlines() == list(MacOS().build_flags(machine="arm64")) + + def test_linux_prints_nothing( + self, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], + ) -> None: + monkeypatch.setattr(build_environment, "current_platform", Linux) + + assert build_environment.main([]) == 0 + assert capsys.readouterr().out == "" diff --git a/tests/unit/scripts/test_bundle.py b/tests/unit/scripts/test_bundle.py new file mode 100644 index 000000000..cb4422d80 --- /dev/null +++ b/tests/unit/scripts/test_bundle.py @@ -0,0 +1,157 @@ +from pathlib import Path +from typing import Callable, Sequence + +import pytest + +from bootstrap.layout import BUILD_ENVIRONMENT, BUILD_TOOLS, DISTRIBUTION, NOTICES, RELEASE_HOOK +from bootstrap.platforms.linux import Linux +from bootstrap.platforms.macos import MacOS +from bootstrap.platforms.windows import Windows +from bootstrap.project import BUILD_EXTRA, GPU_EXTRA, read_project +from tests.suite.bootstrap import PROJECT_NAME, RecordingRunner, write_project +from tests.suite.scripts import load_script + +bundle = load_script("bundle.py") + + +def _repository(tmp_path: Path) -> Path: + for notice in NOTICES: + (tmp_path / notice).write_text(notice) + + return write_project(tmp_path) + + +class TestPyInstallerCommand: + def test_a_release_is_a_directory_with_the_runtime_hook(self, tmp_path: Path) -> None: + project = read_project(_repository(tmp_path)) + options = bundle.BundleOptions(release=True, gpu=False) + + command = bundle.pyinstaller_command(Path("python"), Linux().bundling(), project, options) + + assert command[:3] == ["python", "-m", "PyInstaller"] + assert "--onedir" in command + assert command[command.index("--runtime-hook") + 1] == RELEASE_HOOK + assert command[-1] == project.entry_script + + def test_a_development_bundle_is_one_file_without_the_hook(self, tmp_path: Path) -> None: + project = read_project(_repository(tmp_path)) + + command = bundle.pyinstaller_command( + Path("python"), + Linux().bundling(), + project, + bundle.BundleOptions(release=False, gpu=False), + ) + + assert "--onefile" in command + assert "--runtime-hook" not in command + + def test_every_package_brings_its_data_and_the_build_tools_stay_out(self, tmp_path: Path) -> None: + project = read_project(_repository(tmp_path)) + + command = bundle.pyinstaller_command( + Path("python"), + Windows().bundling(), + project, + bundle.BundleOptions(release=False, gpu=False), + ) + + collected = [command[index + 1] for index, flag in enumerate(command) if flag == "--collect-data"] + excluded = [command[index + 1] for index, flag in enumerate(command) if flag == "--exclude-module"] + assert collected == list(project.packages) + assert excluded == list(BUILD_TOOLS) + assert command[command.index("--icon") + 1] == Windows().bundling().icon + assert command[command.index("--name") + 1] == project.name + + +class TestExtras: + def test_the_build_extra_is_always_installed_and_gpu_on_request(self) -> None: + assert bundle.extras(bundle.BundleOptions(release=False, gpu=False)) == (BUILD_EXTRA,) + assert bundle.extras(bundle.BundleOptions(release=False, gpu=True)) == (BUILD_EXTRA, GPU_EXTRA) + + +class TestRemovePrevious: + def test_a_previous_file_and_directory_are_removed(self, tmp_path: Path) -> None: + (tmp_path / PROJECT_NAME).mkdir() + (tmp_path / f"{PROJECT_NAME}.exe").write_text("") + + bundle.remove_previous(tmp_path, Windows().bundling(), PROJECT_NAME) + + assert list(tmp_path.iterdir()) == [] + + +def _leave_behind(root: Path, launcher: Path) -> Callable[[Sequence[str]], None]: + """What a build leaves on disk: the environment's interpreter, and the launcher where PyInstaller writes it.""" + + def on_run(command: Sequence[str]) -> None: + if "venv" in command: + python = Linux().interpreter(root / BUILD_ENVIRONMENT) + python.parent.mkdir(parents=True) + python.write_text("") + + if "PyInstaller" in command: + launcher.parent.mkdir(parents=True, exist_ok=True) + launcher.write_text("") + + return on_run + + +class TestBuildBundle: + def test_a_release_runs_every_step_in_order_and_places_the_notices(self, tmp_path: Path) -> None: + root = _repository(tmp_path) + launcher = Linux().bundling().launcher(root / DISTRIBUTION, name=PROJECT_NAME, release=True) + runner = RecordingRunner({}, _leave_behind(root, launcher)) + + built = bundle.build_bundle( + root, Linux(), bundle.BundleOptions(release=True, gpu=False), runner=runner, environment={} + ) + + assert built == launcher + assert runner.lines[0].endswith(BUILD_ENVIRONMENT) + assert "pip install --upgrade pip" in runner.lines[1] + assert f".[{BUILD_EXTRA}]" in runner.lines[2] + assert "import pyaudio" in runner.lines[3] + assert "import tkinter" in runner.lines[4] + assert "PyInstaller" in runner.lines[5] + assert runner.lines[6] == f"{launcher} {bundle.SELF_CHECK}" + assert all((launcher.parent / notice).read_text() == notice for notice in NOTICES) + + def test_a_bundle_pyinstaller_never_wrote_is_reported(self, tmp_path: Path) -> None: + root = _repository(tmp_path) + elsewhere = tmp_path / "elsewhere" / PROJECT_NAME + + with pytest.raises(SystemExit, match="produced no executable"): + bundle.build_bundle( + root, + Linux(), + bundle.BundleOptions(release=False, gpu=False), + runner=RecordingRunner({}, _leave_behind(root, elsewhere)), + environment={}, + ) + + def test_a_launcher_failing_its_self_check_fails_the_build(self, tmp_path: Path) -> None: + root = _repository(tmp_path) + launcher = Linux().bundling().launcher(root / DISTRIBUTION, name=PROJECT_NAME, release=False) + + with pytest.raises(SystemExit, match=bundle.SELF_CHECK): + bundle.build_bundle( + root, + Linux(), + bundle.BundleOptions(release=False, gpu=False), + runner=RecordingRunner({bundle.SELF_CHECK: 1}, _leave_behind(root, launcher)), + environment={}, + ) + + def test_a_system_without_bundles_is_told_to_run_from_source_before_anything_runs(self, tmp_path: Path) -> None: + runner = RecordingRunner({}, None) + + with pytest.raises(SystemExit, match="make setup"): + bundle.build_bundle( + _repository(tmp_path), + MacOS(), + bundle.BundleOptions(release=False, gpu=False), + runner=runner, + environment={}, + ) + + assert runner.lines == [] diff --git a/tests/unit/scripts/test_clean.py b/tests/unit/scripts/test_clean.py new file mode 100644 index 000000000..5d65aa15e --- /dev/null +++ b/tests/unit/scripts/test_clean.py @@ -0,0 +1,46 @@ +from pathlib import Path + +from tests.suite.scripts import load_script + +clean = load_script("clean.py") + + +def _tree(root: Path) -> None: + for directory in ( + "bin", + "build", + "dist", + "htmlcov", + "src/__pycache__", + "src/sampletones.egg-info", + ".venv/__pycache__", + ): + (root / directory).mkdir(parents=True) + + for file in (".coverage", "sampletones.spec", "src/module.pyc", "src/module.py", ".venv/cached.pyc"): + (root / file).write_text("") + + +class TestRemoveArtifacts: + def test_the_build_outputs_and_reports_go(self, tmp_path: Path) -> None: + _tree(tmp_path) + + clean.remove_artifacts(tmp_path) + + assert not any((tmp_path / name).exists() for name in ("bin", "build", "dist", "htmlcov", ".coverage")) + assert not (tmp_path / "sampletones.spec").exists() + assert (tmp_path / "src" / "module.py").exists() + + +class TestRemoveCaches: + def test_the_caches_go_and_the_environments_stay(self, tmp_path: Path) -> None: + _tree(tmp_path) + + clean.remove_caches(tmp_path) + + assert not (tmp_path / "src" / "__pycache__").exists() + assert not (tmp_path / "src" / "sampletones.egg-info").exists() + assert not (tmp_path / "src" / "module.pyc").exists() + assert (tmp_path / "src" / "module.py").exists() + assert (tmp_path / ".venv" / "__pycache__").exists() + assert (tmp_path / ".venv" / "cached.pyc").exists() diff --git a/tests/unit/scripts/test_formatting.py b/tests/unit/scripts/test_formatting.py new file mode 100644 index 000000000..691ee31b1 --- /dev/null +++ b/tests/unit/scripts/test_formatting.py @@ -0,0 +1,36 @@ +from pathlib import Path + +import pytest + +from tests.suite.bootstrap import RecordingRunner +from tests.suite.scripts import load_script + +formatting = load_script("formatting.py") + + +class TestFormattedPaths: + def test_the_three_trees_are_formatted_by_default(self) -> None: + assert formatting.formatted_paths(()) == formatting.FORMATTED_TREES + + def test_named_paths_replace_the_trees(self) -> None: + assert formatting.formatted_paths(("scripts/lint.py",)) == ("scripts/lint.py",) + + +class TestFormatCode: + def test_isort_runs_before_black(self, tmp_path: Path) -> None: + runner = RecordingRunner({}, None) + + formatting.format_code(tmp_path, formatting.FORMATTED_TREES, runner=runner, environment={}) + + assert runner.lines == [ + " ".join((*formatting.ISORT, *formatting.FORMATTED_TREES)), + " ".join((*formatting.BLACK, *formatting.FORMATTED_TREES)), + ] + + def test_a_failing_formatter_stops_the_run(self, tmp_path: Path) -> None: + runner = RecordingRunner({"isort": 1}, None) + + with pytest.raises(SystemExit, match="isort"): + formatting.format_code(tmp_path, formatting.FORMATTED_TREES, runner=runner, environment={}) + + assert len(runner.lines) == 1 diff --git a/tests/unit/scripts/test_hooks.py b/tests/unit/scripts/test_hooks.py new file mode 100644 index 000000000..1364dcdbb --- /dev/null +++ b/tests/unit/scripts/test_hooks.py @@ -0,0 +1,17 @@ +from pathlib import Path + +from tests.suite.bootstrap import RecordingRunner +from tests.suite.scripts import load_script + +hooks = load_script("hooks.py") + + +class TestInstallHooks: + def test_both_hook_stages_are_installed_from_the_repository(self, tmp_path: Path) -> None: + runner = RecordingRunner({}, None) + + hooks.install_hooks(tmp_path, runner=runner, environment={}) + + assert runner.lines == [" ".join(hooks.INSTALL_HOOKS)] + assert "--hook-type pre-commit --hook-type pre-push" in runner.lines[0] + assert runner.commands[0].cwd == tmp_path diff --git a/tests/unit/scripts/test_lint.py b/tests/unit/scripts/test_lint.py new file mode 100644 index 000000000..5f517e2de --- /dev/null +++ b/tests/unit/scripts/test_lint.py @@ -0,0 +1,50 @@ +from pathlib import Path + +import pytest + +from tests.suite.bootstrap import RecordingRunner +from tests.suite.scripts import load_script + +lint = load_script("lint.py") + + +class TestLinters: + def test_mypy_reads_its_configuration_and_pylint_sweeps_the_trees(self) -> None: + mypy, pylint = lint.linters(()) + + assert mypy.command == ("uv", "run", "python", "-m", lint.MYPY) + assert pylint.command == ("uv", "run", "python", "-m", lint.PYLINT, *lint.LINTED_TREES) + + def test_named_paths_reach_both(self) -> None: + mypy, pylint = lint.linters(("scripts/lint.py",)) + + assert mypy.command[-1] == "scripts/lint.py" + assert pylint.command[-1] == "scripts/lint.py" + + +class TestChosenLinters: + def test_both_linters_run_by_default(self) -> None: + assert [linter.name for linter in lint.chosen_linters((), mypy=False, pylint=False)] == [lint.MYPY, lint.PYLINT] + + def test_a_flag_picks_one_linter(self) -> None: + assert [linter.name for linter in lint.chosen_linters((), mypy=False, pylint=True)] == [lint.PYLINT] + + +class TestLintCode: + def test_every_linter_passing_is_reported(self, tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: + runner = RecordingRunner({}, None) + + assert lint.lint_code(tmp_path, lint.linters(()), runner=runner, environment={}) == 0 + assert len(runner.lines) == 2 + assert "All linting checks passed." in capsys.readouterr().out + + def test_a_failing_linter_stops_nothing_and_is_named( + self, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + ) -> None: + runner = RecordingRunner({lint.MYPY: 1}, None) + + assert lint.lint_code(tmp_path, lint.linters(()), runner=runner, environment={}) == 1 + assert len(runner.lines) == 2 + assert f"Linting failed: {lint.MYPY}." in capsys.readouterr().out diff --git a/tests/unit/scripts/test_run_tests.py b/tests/unit/scripts/test_run_tests.py new file mode 100644 index 000000000..35d9fbd73 --- /dev/null +++ b/tests/unit/scripts/test_run_tests.py @@ -0,0 +1,56 @@ +from dataclasses import dataclass +from typing import Tuple + +import pytest + +from bootstrap.layout import BENCHMARKS_DIRECTORY +from tests.suite.base import BaseTestSuite +from tests.suite.case import BaseRegularTestCase +from tests.suite.scripts import load_script + +run_tests = load_script("run_tests.py") + + +class TestPlannedPasses: + def test_every_pass_is_found_under_its_name(self) -> None: + passes = run_tests.planned_passes(run_tests.DEFAULT_WORKERS) + + assert all(name == current.name for name, current in passes.items()) + assert set(passes) == {run_tests.SUITE, run_tests.DOCTESTS, run_tests.BENCHMARKS} + + def test_the_suite_is_covered_across_the_workers_and_leaves_the_benchmarks_out(self) -> None: + command = run_tests.planned_passes("auto")[run_tests.SUITE].command + + assert command[-4:] == ("-n", "auto", "--cov", f"--ignore={BENCHMARKS_DIRECTORY}") + + def test_the_doctests_read_the_sources_uncovered(self) -> None: + command = run_tests.planned_passes(run_tests.DEFAULT_WORKERS)[run_tests.DOCTESTS].command + + assert "--doctest-modules" in command + assert "--no-cov" in command + + def test_the_benchmarks_run_serial_uncovered_and_show_their_readings(self) -> None: + command = run_tests.planned_passes(run_tests.DEFAULT_WORKERS)[run_tests.BENCHMARKS].command + + assert BENCHMARKS_DIRECTORY in command + assert "--no-cov" in command + assert "-s" in command + assert "-n" not in command + + +class TestRefusedPasses(BaseTestSuite): + @dataclass(frozen=True, kw_only=True) + class TestCase(BaseRegularTestCase): + argv: Tuple[str, ...] + + test_cases = ( + TestCase(label="a missing pass", argv=()), + TestCase(label="an unknown pass", argv=("everything",)), + ) + + @pytest.mark.parametrize("test_case", test_cases, ids=lambda test_case: test_case.label) + def test_a_pass_outside_the_three_is_refused(self, test_case: TestCase) -> None: + with pytest.raises(SystemExit) as exit_info: + run_tests.main(test_case.argv) + + assert exit_info.value.code == 2 diff --git a/tests/unit/scripts/test_setup_environment.py b/tests/unit/scripts/test_setup_environment.py new file mode 100644 index 000000000..7de7fdf3a --- /dev/null +++ b/tests/unit/scripts/test_setup_environment.py @@ -0,0 +1,82 @@ +from pathlib import Path + +import pytest + +from bootstrap.platforms.linux import Linux +from bootstrap.platforms.macos import ARCHFLAGS, MacOS +from bootstrap.project import DEVELOPMENT_GROUP, GPU_CUDA11_EXTRA, GPU_EXTRA +from tests.suite.bootstrap import RecordingRunner +from tests.suite.scripts import load_script + +setup_environment = load_script("setup_environment.py") + + +class TestGpuExtra: + def test_zero_keeps_the_cpu_backend(self) -> None: + assert setup_environment.gpu_extra(setup_environment.GPU_OFF, Linux(), {}) is None + + def test_a_named_extra_is_taken_as_given(self) -> None: + assert setup_environment.gpu_extra(GPU_CUDA11_EXTRA, Linux(), {}) == GPU_CUDA11_EXTRA + + def test_auto_on_macos_keeps_the_cpu_backend(self) -> None: + assert setup_environment.gpu_extra(setup_environment.GPU_AUTO, MacOS(), {}) is None + + +class TestMain: + def test_a_gpu_choice_outside_the_extras_is_refused_before_anything_runs( + self, + capsys: pytest.CaptureFixture[str], + ) -> None: + with pytest.raises(SystemExit) as exit_info: + setup_environment.main(["--gpu", "1"]) + + refusal = capsys.readouterr().err + assert exit_info.value.code == 2 + assert all(choice in refusal for choice in setup_environment.GPU_CHOICES) + + +class TestSetupCommands: + def test_the_cpu_backend_synchronizes_and_installs_the_bare_package(self) -> None: + commands = setup_environment.setup_commands(None) + + assert commands == [ + ["uv", "sync", "--group", DEVELOPMENT_GROUP], + ["uv", "tool", "install", "--force", "."], + ] + + def test_a_gpu_extra_reaches_both_installs(self) -> None: + commands = setup_environment.setup_commands(GPU_EXTRA) + + assert commands[0] == ["uv", "sync", "--group", DEVELOPMENT_GROUP, "--extra", GPU_EXTRA] + assert commands[1] == ["uv", "tool", "install", "--force", f".[{GPU_EXTRA}]"] + + +class TestSetUpEnvironment: + def test_macos_runs_the_commands_on_its_native_architecture(self, tmp_path: Path) -> None: + runner = RecordingRunner({}, None) + + setup_environment.set_up_environment( + tmp_path, + MacOS(), + None, + machine="arm64", + runner=runner, + environment={"PATH": "/usr/bin"}, + ) + + assert runner.lines == [" ".join(command) for command in setup_environment.setup_commands(None)] + assert all(recorded.environment[ARCHFLAGS] == "-arch arm64" for recorded in runner.commands) + + def test_linux_passes_the_variables_through(self, tmp_path: Path) -> None: + runner = RecordingRunner({}, None) + + setup_environment.set_up_environment( + tmp_path, + Linux(), + GPU_EXTRA, + machine="x86_64", + runner=runner, + environment={"PATH": "/usr/bin"}, + ) + + assert all(recorded.environment == {"PATH": "/usr/bin"} for recorded in runner.commands) diff --git a/tests/unit/scripts/test_system_dependencies.py b/tests/unit/scripts/test_system_dependencies.py new file mode 100644 index 000000000..d4194a8ce --- /dev/null +++ b/tests/unit/scripts/test_system_dependencies.py @@ -0,0 +1,56 @@ +from pathlib import Path + +import pytest + +from bootstrap.platforms import macos +from bootstrap.platforms.linux import Linux +from bootstrap.platforms.macos import MacOS +from bootstrap.platforms.windows import Windows +from tests.suite.bootstrap import RecordingRunner +from tests.suite.scripts import load_script + +system_dependencies = load_script("system_dependencies.py") + + +class TestInstallSystemPackages: + def test_windows_has_nothing_to_install(self, tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: + runner = RecordingRunner({}, None) + + assert system_dependencies.install_system_packages(tmp_path, Windows(), runner=runner, environment={}) == 0 + assert runner.lines == [] + assert "Nothing to install" in capsys.readouterr().out + + def test_linux_runs_the_apt_commands_in_order(self, tmp_path: Path) -> None: + runner = RecordingRunner({}, None) + + assert system_dependencies.install_system_packages(tmp_path, Linux(), runner=runner, environment={}) == 0 + assert runner.lines == [" ".join(command) for command in Linux().system_packages()] + + def test_macos_with_homebrew_installs_portaudio(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + runner = RecordingRunner({}, None) + monkeypatch.setattr(macos.shutil, "which", lambda name: f"/opt/homebrew/bin/{name}") + + assert system_dependencies.install_system_packages(tmp_path, MacOS(), runner=runner, environment={}) == 0 + assert runner.lines == [" ".join(command) for command in MacOS().system_packages()] + + def test_macos_without_homebrew_is_refused( + self, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], + ) -> None: + runner = RecordingRunner({}, None) + monkeypatch.setattr(macos.shutil, "which", lambda name: None) + + assert system_dependencies.install_system_packages(tmp_path, MacOS(), runner=runner, environment={}) == 1 + assert runner.lines == [] + assert "Homebrew" in capsys.readouterr().err + + def test_a_failing_install_stops_the_script(self, tmp_path: Path) -> None: + with pytest.raises(SystemExit, match="apt-get update"): + system_dependencies.install_system_packages( + tmp_path, + Linux(), + runner=RecordingRunner({"update": 100}, None), + environment={}, + ) diff --git a/tests/unit/scripts/test_verify_bundle.py b/tests/unit/scripts/test_verify_bundle.py new file mode 100644 index 000000000..5bc6d90fe --- /dev/null +++ b/tests/unit/scripts/test_verify_bundle.py @@ -0,0 +1,105 @@ +from pathlib import Path + +import pytest + +from bootstrap.layout import BUILD_TOOLS, DISTRIBUTION, NOTICES +from bootstrap.platforms.linux import Linux +from bootstrap.platforms.macos import MacOS +from tests.suite.bootstrap import PROJECT_NAME, RecordingRunner, write_project +from tests.suite.scripts import load_script + +verify_bundle = load_script("verify_bundle.py") + + +@pytest.fixture(name="root") +def root_fixture(tmp_path: Path) -> Path: + """A repository holding a release bundle with its notices and its launcher.""" + launcher = Linux().bundling().launcher(tmp_path / DISTRIBUTION, name=PROJECT_NAME, release=True) + launcher.parent.mkdir(parents=True) + launcher.write_bytes(b"launcher") + for name in NOTICES: + (launcher.parent / name).write_text(name) + + return write_project(tmp_path) + + +def _bundle(root: Path) -> Path: + return root / DISTRIBUTION / PROJECT_NAME + + +class TestMissingNotices: + def test_a_complete_bundle_lacks_nothing(self, root: Path) -> None: + assert verify_bundle.missing_notices(_bundle(root)) == [] + + def test_every_absent_notice_is_reported(self, root: Path) -> None: + for name in NOTICES[1:]: + (_bundle(root) / name).unlink() + + assert verify_bundle.missing_notices(_bundle(root)) == list(NOTICES[1:]) + + def test_a_notice_directory_counts_as_absent(self, root: Path) -> None: + (_bundle(root) / NOTICES[0]).unlink() + (_bundle(root) / NOTICES[0]).mkdir() + + assert verify_bundle.missing_notices(_bundle(root)) == [NOTICES[0]] + + +class TestCarriedBuildTools: + def test_an_application_bundle_holds_to_its_notices(self, root: Path) -> None: + assert verify_bundle.carried_build_tools(_bundle(root)) == [] + + def test_a_build_tool_beside_the_application_is_reported(self, root: Path) -> None: + (_bundle(root) / verify_bundle.INTERNAL_DIRECTORY / BUILD_TOOLS[0]).mkdir(parents=True) + + assert verify_bundle.carried_build_tools(_bundle(root)) == [BUILD_TOOLS[0]] + + def test_a_build_tool_beside_the_launcher_is_reported(self, root: Path) -> None: + (_bundle(root) / BUILD_TOOLS[0]).mkdir() + + assert verify_bundle.carried_build_tools(_bundle(root)) == [BUILD_TOOLS[0]] + + +class TestBundleFailures: + def test_a_complete_bundle_passes_once_its_launcher_starts(self, root: Path) -> None: + runner = RecordingRunner({}, None) + + assert verify_bundle.bundle_failures(root, Linux(), runner=runner, environment={}) == [] + assert runner.lines == [f"{_bundle(root) / PROJECT_NAME} {verify_bundle.VERSION_FLAG}"] + + def test_a_missing_notice_is_named(self, root: Path) -> None: + (_bundle(root) / NOTICES[1]).unlink() + + failures = verify_bundle.bundle_failures(root, Linux(), runner=RecordingRunner({}, None), environment={}) + + assert len(failures) == 1 + assert NOTICES[1] in failures[0] + + def test_bundled_build_tooling_is_named(self, root: Path) -> None: + (_bundle(root) / verify_bundle.INTERNAL_DIRECTORY / BUILD_TOOLS[0]).mkdir(parents=True) + + failures = verify_bundle.bundle_failures(root, Linux(), runner=RecordingRunner({}, None), environment={}) + + assert [BUILD_TOOLS[0] in failure for failure in failures] == [True] + + def test_a_missing_launcher_is_named_and_nothing_runs(self, root: Path) -> None: + (_bundle(root) / PROJECT_NAME).unlink() + runner = RecordingRunner({}, None) + + failures = verify_bundle.bundle_failures(root, Linux(), runner=runner, environment={}) + + assert [("offers no launcher" in failure) for failure in failures] == [True] + assert runner.lines == [] + + def test_a_launcher_that_fails_is_named_with_its_status(self, root: Path) -> None: + failures = verify_bundle.bundle_failures( + root, + Linux(), + runner=RecordingRunner({verify_bundle.VERSION_FLAG: 3}, None), + environment={}, + ) + + assert [("status 3" in failure) for failure in failures] == [True] + + def test_a_system_without_bundles_is_refused(self, root: Path) -> None: + with pytest.raises(SystemExit, match="make setup"): + verify_bundle.bundle_failures(root, MacOS(), runner=RecordingRunner({}, None), environment={}) diff --git a/tests/unit/scripts/test_verify_version_tag.py b/tests/unit/scripts/test_verify_version_tag.py new file mode 100644 index 000000000..351a7baa7 --- /dev/null +++ b/tests/unit/scripts/test_verify_version_tag.py @@ -0,0 +1,62 @@ +from dataclasses import dataclass +from pathlib import Path + +import pytest + +from bootstrap.layout import repository_root +from bootstrap.project import TAG_PREFIX, read_project +from tests.suite.base import BaseTestSuite +from tests.suite.bootstrap import write_project +from tests.suite.case import BaseRegularTestCase +from tests.suite.scripts import load_script + +verify_version_tag = load_script("verify_version_tag.py") + + +class TestVersionFromTag(BaseTestSuite): + @dataclass(frozen=True, kw_only=True) + class TestCase(BaseRegularTestCase): + tag: str + expected: str + + test_cases = ( + TestCase(label="release_tag", tag="v0.3.0", expected="0.3.0"), + TestCase(label="prerelease_tag", tag="v0.3.0.dev1", expected="0.3.0.dev1"), + TestCase(label="release_candidate", tag="v1.0.0rc2", expected="1.0.0rc2"), + TestCase(label="bare_version", tag="0.3.0", expected="0.3.0"), + TestCase(label="single_prefix_is_dropped", tag="vv0.3.0", expected="v0.3.0"), + ) + + @pytest.mark.parametrize("test_case", test_cases, ids=lambda test_case: test_case.label) + def test_version_from_tag(self, test_case: TestCase) -> None: + assert verify_version_tag.version_from_tag(test_case.tag) == test_case.expected + + +class TestTagFailure: + def test_the_tag_naming_the_project_version_passes(self, tmp_path: Path) -> None: + project = read_project(write_project(tmp_path)) + + assert verify_version_tag.tag_failure(project.tag, project) is None + + def test_a_tag_naming_another_version_names_both(self, tmp_path: Path) -> None: + project = read_project(write_project(tmp_path)) + tag = f"{project.tag}.post9" + + failure = verify_version_tag.tag_failure(tag, project) + + assert failure is not None + assert tag in failure + assert project.version in failure + + +class TestMain: + def test_the_tag_naming_the_project_version_passes(self) -> None: + version = read_project(repository_root()).version + + assert verify_version_tag.main(["--tag", f"{TAG_PREFIX}{version}"]) == 0 + + def test_a_tag_naming_another_version_is_annotated_as_an_error(self, capsys: pytest.CaptureFixture[str]) -> None: + version = read_project(repository_root()).version + + assert verify_version_tag.main(["--tag", f"{TAG_PREFIX}{version}.post9"]) == 1 + assert capsys.readouterr().out.startswith("::error::")