From 8c987bef3d04550b20684e2274110f30bd12454c Mon Sep 17 00:00:00 2001 From: Myles Penner Date: Wed, 15 Oct 2025 15:58:39 -0700 Subject: [PATCH 1/5] Add documentation validation This commit introduces documentation validation system that ensures our published CLI examples remain executable and up to date. Summary of Changes: - `ci/select-doc-pages.py`: builds an ordered execution plan from changed snippet files and their declared dependencies (`# @depends:`). - `ci/run-doc-pages.py`: extracts and runs (or dry-runs) `[docs-exec:*]` blocks from snippet files in sequential order. - `[docs-view:*]` blocks: used for display in documentation via `literalinclude`, ensuring examples in the docs match what CI validates. - Unit tests for both selector and runner using pytest. - Updated GitHub Actions workflow for automatic snippet validation. - Added `ci/README.md` explaining the framework, usage, and conventions. --- .../workflows/docs-validation-unit-tests.yml | 48 +++++ .github/workflows/docs-validation.yml | 175 +++++++++++++++ ci/README.md | 154 ++++++++++++++ ci/run-doc-pages.py | 201 ++++++++++++++++++ ci/select-doc-pages.py | 151 +++++++++++++ docs/how-to/features/caas.rst | 16 +- docs/how-to/features/images-sync.rst | 14 +- docs/how-to/features/instance-recovery.rst | 14 +- docs/how-to/features/ldap.rst | 44 ++-- docs/how-to/features/load-balancer.rst | 14 +- docs/how-to/features/observability.rst | 35 +-- docs/how-to/features/orchestration.rst | 7 +- .../how-to/features/resource-optimization.rst | 14 +- docs/how-to/features/secrets.rst | 16 +- docs/how-to/features/telemetry.rst | 14 +- docs/how-to/features/validation.rst | 30 +-- docs/how-to/snippets/caas.task.sh | 18 ++ docs/how-to/snippets/images-sync.task.sh | 18 ++ .../how-to/snippets/instance-recovery.task.sh | 18 ++ docs/how-to/snippets/ldap.task.sh | 40 ++++ docs/how-to/snippets/load-balancer.task.sh | 17 ++ docs/how-to/snippets/observability.task.sh | 33 +++ docs/how-to/snippets/orchestration.task.sh | 18 ++ .../snippets/resource-optimization.task.sh | 18 ++ docs/how-to/snippets/secrets.task.sh | 18 ++ docs/how-to/snippets/telemetry.task.sh | 18 ++ docs/how-to/snippets/validation.task.sh | 18 ++ docs/tutorial/get-started-with-openstack.rst | 50 +++-- .../get-started-with-openstack.task.sh | 48 +++++ tests/conftest.py | 35 +++ tests/snippets/base-fixture.task.sh | 8 + tests/snippets/fixture-missing-end.task.sh | 4 + tests/snippets/fixture-multi-blocks.task.sh | 8 + tests/snippets/fixture-no-blocks.task.sh | 4 + tests/snippets/fixture-relative-dep.task.sh | 8 + tests/snippets/fixture-with-dep.task.sh | 6 + tests/test_run_doc_pages.py | 71 +++++++ tests/test_select_doc_pages.py | 43 ++++ 38 files changed, 1348 insertions(+), 118 deletions(-) create mode 100644 .github/workflows/docs-validation-unit-tests.yml create mode 100644 .github/workflows/docs-validation.yml create mode 100644 ci/README.md create mode 100644 ci/run-doc-pages.py create mode 100644 ci/select-doc-pages.py create mode 100644 docs/how-to/snippets/caas.task.sh create mode 100644 docs/how-to/snippets/images-sync.task.sh create mode 100644 docs/how-to/snippets/instance-recovery.task.sh create mode 100644 docs/how-to/snippets/ldap.task.sh create mode 100644 docs/how-to/snippets/load-balancer.task.sh create mode 100644 docs/how-to/snippets/observability.task.sh create mode 100644 docs/how-to/snippets/orchestration.task.sh create mode 100644 docs/how-to/snippets/resource-optimization.task.sh create mode 100644 docs/how-to/snippets/secrets.task.sh create mode 100644 docs/how-to/snippets/telemetry.task.sh create mode 100644 docs/how-to/snippets/validation.task.sh create mode 100755 docs/tutorial/snippets/get-started-with-openstack.task.sh create mode 100644 tests/conftest.py create mode 100644 tests/snippets/base-fixture.task.sh create mode 100644 tests/snippets/fixture-missing-end.task.sh create mode 100644 tests/snippets/fixture-multi-blocks.task.sh create mode 100644 tests/snippets/fixture-no-blocks.task.sh create mode 100644 tests/snippets/fixture-relative-dep.task.sh create mode 100644 tests/snippets/fixture-with-dep.task.sh create mode 100644 tests/test_run_doc_pages.py create mode 100644 tests/test_select_doc_pages.py diff --git a/.github/workflows/docs-validation-unit-tests.yml b/.github/workflows/docs-validation-unit-tests.yml new file mode 100644 index 0000000..975ed85 --- /dev/null +++ b/.github/workflows/docs-validation-unit-tests.yml @@ -0,0 +1,48 @@ +name: Documentation Unit Tests + +on: + push: + branches: [ main, docs-testing ] + paths: + - "ci/**" + pull_request: + branches: [ main ] + paths: + - "ci/**" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: docs-unit-tests-${{ github.ref }} + cancel-in-progress: true + +jobs: + unit-tests: + name: Pytest (selector & executor) + runs-on: ubuntu-latest + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + python-version: ["3.12"] + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Install test dependencies + run: | + python -m pip install pytest requests-mock anyio + + - name: Run pytest + env: + PYTHONUNBUFFERED: "1" + run: | + pytest -q --maxfail=1 diff --git a/.github/workflows/docs-validation.yml b/.github/workflows/docs-validation.yml new file mode 100644 index 0000000..5a5af78 --- /dev/null +++ b/.github/workflows/docs-validation.yml @@ -0,0 +1,175 @@ +name: Documentation Functional Testing + +on: + push: + branches: + - main + - docs-testing + paths: + - "tutorial/**/*.task.sh" + - "how-to/**/*.task.sh" + pull_request: + branches: + - main + paths: + - "tutorial/**/*.task.sh" + - "how-to/**/*.task.sh" + # Manual trigger + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: docs-validation-${{ github.ref }} + cancel-in-progress: true + +defaults: + run: + shell: bash + +env: + CI: "true" + PYTHONUNBUFFERED: "1" + PIP_DISABLE_PIP_VERSION_CHECK: "1" + # Dry-run switch. Set to "1" to print without executing. + DOCS_DRY_RUN: "1" + +jobs: + lint-snippets: + name: Lint snippet shell files + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Discover snippet files + id: discover + run: | + find tutorial how-to -type f -name '*.task.sh' -print0 2>/dev/null > all_snippets.null + COUNT=$(tr -cd '\0' < all_snippets.null | wc -c) + echo "count=${COUNT}" >> "$GITHUB_OUTPUT" + + # Summary for GitHub output + { + echo "### Lint target discovery" + if [[ "$COUNT" -gt 0 ]]; then + echo "Found **$COUNT** snippet files to lint." + else + echo "_No snippet files found._" + fi + } >> "$GITHUB_STEP_SUMMARY" + + - name: Set up Python (for bashate) + if: steps.discover.outputs.count > 0 + uses: actions/setup-python@v5 + with: + python-version: "3.13" + + - name: Install bashate + if: steps.discover.outputs.count > 0 + run: python -m pip install --upgrade pip bashate + + - name: Lint with bashate (ignore long lines) + if: steps.discover.outputs.count > 0 + run: | + xargs -0 -r bashate -i E006 < all_snippets.null + + plan-and-run: + name: Plan & run docs tests + needs: lint-snippets + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Determine changed snippet files + id: changed + uses: tj-actions/changed-files@v45 + with: + files: | + tutorial/**/*.task.sh + how-to/**/*.task.sh + + - name: Write changed list to file + id: write-changed + if: steps.changed.outputs.all_changed_files != '' + run: | + # No quotes around the variable allows the shell to perform word splitting + printf '%s\n' ${{ steps.changed.outputs.all_changed_files }} > changed_snippets.txt + + COUNT=$(wc -l < changed_snippets.txt | tr -d ' ') + echo "count=${COUNT}" >> "$GITHUB_OUTPUT" + + { + echo "### Changed snippets" + echo + echo "**${COUNT} file(s) changed:**" + echo + sed 's/^/- /' changed_snippets.txt + } >> "$GITHUB_STEP_SUMMARY" + + - name: Short-circuit if nothing to run + if: steps.write-changed.outputs.count == null || steps.write-changed.outputs.count == '0' + run: echo "No .task.sh files changed. Skipping subsequent steps." + + - name: Set up Python (executor/selector) + if: steps.write-changed.outputs.count > 0 + uses: actions/setup-python@v5 + with: + python-version: "3.13" + + - name: Build and display execution plan + if: steps.write-changed.outputs.count > 0 + id: plan + run: | + python ci/select-doc-pages.py --changed-file-list changed_snippets.txt --out plan.txt + PLAN_COUNT=$(wc -l < plan.txt | tr -d ' ') + echo "count=${PLAN_COUNT}" >> "$GITHUB_OUTPUT" + + echo "--- Execution Plan (${PLAN_COUNT} steps) ---" + nl -ba plan.txt || true + echo "-----------------------------------" + + { + echo "### Execution plan" + if [[ "$PLAN_COUNT" -gt 0 ]]; then + echo "Selector produced a plan with **${PLAN_COUNT}** item(s)." + echo '```' + cat plan.txt + echo '```' + else + echo "_Selector produced an empty plan._" + fi + } >> "$GITHUB_STEP_SUMMARY" + + - name: Fail if plan is unexpectedly empty + if: steps.write-changed.outputs.count > 0 && steps.plan.outputs.count == '0' + run: | + echo "::error::Selector produced an empty plan, but there were changed snippets." + exit 1 + + - name: Dry-run plan validation + if: env.DOCS_DRY_RUN == '1' && steps.plan.outputs.count > 0 + run: | + echo "Executing dry-run validation of the plan..." + python ci/run-doc-pages.py --plan plan.txt + + - name: Execute plan + if: env.DOCS_DRY_RUN != '1' && steps.plan.outputs.count > 0 + run: | + python ci/run-doc-pages.py --plan plan.txt 2>&1 | tee run.log + + - name: Upload artifacts (plan & logs) + if: always() && steps.write-changed.outputs.count > 0 + uses: actions/upload-artifact@v4 + with: + name: docs-validation-artifacts-${{ github.run_id }} + path: | + plan.txt + changed_snippets.txt + run.log + if-no-files-found: ignore \ No newline at end of file diff --git a/ci/README.md b/ci/README.md new file mode 100644 index 0000000..4ae5562 --- /dev/null +++ b/ci/README.md @@ -0,0 +1,154 @@ +--- +orphan: true +--- + +# Documentation Testing Framework + +## Purpose + +The goal is to ensure that every command shown in our documentation matches the software. +Each tutorial or how-to snippet contains **both display and execution blocks**: + +- `[docs-view:]` — what appears in the rendered documentation (human-readable example) +- `[docs-exec:]` — what CI executes (machine-runnable equivalent) + +Both appear in the same `.task.sh` snippet file, so you only maintain one source of truth. + +--- + +## Block Types + +### `[docs-view:]` — for rendered docs +Used with the Sphinx `literalinclude` directive. +Contains clean, readable shell commands. + +Example: +```bash +# [docs-view:enable-secrets] +sunbeam enable secrets +# [docs-view:enable-secrets-end] +``` + +### `[docs-exec:]` +Used for **execution** in CI or local validation runs. +It can include wrappers, environment variables, or anything needed to actually make it work. + +Example: +```bash +# [docs-exec:enable-secrets] +sg snap_daemon 'sunbeam enable secrets' +# [docs-exec:enable-secrets-end] +``` + +> The `docs-view` block is what appears in the docs. +> The `docs-exec` block is what actually runs — but they should mirror each other closely. + +## Creating or Editing a Snippet + +Create a new file like `how-to/snippets/secrets.task.sh`: + + +```bash +# [docs-view:enable-secrets] +sunbeam enable secrets +# [docs-view:enable-secrets-end] + +# [docs-exec:enable-secrets] +sg snap_daemon 'sunbeam enable secrets' +# [docs-exec:enable-secrets-end] + +# [docs-view:disable-secrets] +sunbeam disable secrets +# [docs-view:disable-secrets-end] + +# [docs-exec:disable-secrets] +sg snap_daemon 'sunbeam disable secrets' +# [docs-exec:disable-secrets-end] +``` + +**Rules** +- Use `# [docs-exec:]` … `# [docs-exec:-end]` for every runnable block. +- Blocks must not overlap or nest. +- Keep commands non-interactive and idempotent where possible. + +--- + +## Including `[docs-view]` Blocks in RST Files + +To include a `docs-view` block in the rendered documentation, use `literalinclude` with `start-after` and `end-before` markers: + +```rst +.. literalinclude:: ../snippets/ldap.task.sh + :language: bash + :start-after: [docs-view:enable-ldap] + :end-before: [docs-view:enable-ldap-end] +``` + +**Tips** +- Always point to your `*.task.sh` file in the appropriate `snippets/` directory. +- Make sure your markers (`[docs-view:NAME]` and `[docs-view:NAME-end]`) match exactly. +- Sphinx will automatically include only the lines between those markers. + +--- + +## Declaring Dependencies + +If one snippet must run after another (e.g., a feature enable depends on sunbeam being deployed), add a `# @depends:` line near the top pointing to the prerequisite `*.task.sh` script: + +```bash +# @depends: tutorial/snippets/get-started-with-openstack.task.sh +``` + +**Notes** +- Colon after `@depends` is **required**. +- Paths must be **repo-relative**. +- Multiple dependencies are allowed (one per line). +- The selector resolves them such that dependencies always appear **before** dependents in the plan. + +--- + +## Running Locally + +```bash +# Build a plan from changed snippets +printf '%s\n' how-to/snippets/secrets.task.sh > changed_snippets.txt +python ci/select-doc-pages.py --changed-file-list changed_snippets.txt --out plan.txt + +# Preview what would run +DOCS_DRY_RUN=1 python ci/run-doc-pages.py --plan plan.txt + +# Actually execute (be careful!) +DOCS_DRY_RUN=0 python ci/run-doc-pages.py --plan plan.txt +``` + +--- + +## CI Workflows + +There are two workflows using this system: + +### 1. **Documentation Functional Testing** +- Triggers when `*.task.sh` files change. +- Uses the selector to build `plan.txt`. +- Runs the runner in dry-run mode to verify parsing and order. +- Artifacts: `plan.txt`, `changed_snippets.txt`, `run.log`. + +### 2. **Documentation Unit Tests** +- Triggers when `ci/**` changes. +- Runs `pytest` on the selector/runner themselves. + +--- + +## Adding Certificates, DNS, or Other Setup + +If a how-to requires additional setup (e.g., certificates, DNS records), wrap those commands in their own `[docs-exec:*]` blocks inside the same `.task.sh` file: + +```bash +# [docs-exec:generate-cert] +openssl req -x509 -nodes -newkey rsa:2048 \ + -keyout /tmp/demo.key -out /tmp/demo.crt \ + -subj "/CN=demo.internal" -days 365 +# [docs-exec:generate-cert-end] +``` + +These commands will appear in the printed (or executed) plan just like any other snippet. \ No newline at end of file diff --git a/ci/run-doc-pages.py b/ci/run-doc-pages.py new file mode 100644 index 0000000..aae5ae7 --- /dev/null +++ b/ci/run-doc-pages.py @@ -0,0 +1,201 @@ +#!/usr/bin/env python3 +""" +run-doc-pages.py + +Executor for docs snippets plan. This script extracts and runs shell commands +found within special comment blocks (`[docs-exec:] ... [docs-exec:-end]`) +in scripts. If no such blocks are found in a script, the step fails. + +Usage: + python ci/run-doc-pages.py --plan plan.txt + +Environment: + DOCS_DRY_RUN=1 # Print the plan, show the extracted bash, and skip execution. + +Plan format: newline-delimited file where each line is a path to a *.task.sh script. +""" + +from __future__ import annotations + +import argparse +import os +import re +import sys +from dataclasses import dataclass, field +from pathlib import Path +from subprocess import run # nosec B404 + + +def parse_args() -> argparse.Namespace: + p = argparse.ArgumentParser(description="Run docs snippet plan sequentially.") + p.add_argument( + "--plan", + required=True, + type=Path, + help="Path to a newline-delimited file of scripts to run (execution order).", + ) + return p.parse_args() + + +def read_plan(plan_path: Path) -> list[Path]: + if not plan_path.is_file(): + raise FileNotFoundError(f"Plan file not found: {plan_path}") + lines = plan_path.read_text(encoding="utf-8", errors="ignore").splitlines() + return [Path(p.strip()) for p in lines if p.strip() and not p.lstrip().startswith("#")] + + +def require_files_exist(paths: list[Path]) -> None: + missing = [str(p.resolve()) for p in paths if not p.resolve().is_file()] + if missing: + raise FileNotFoundError("Script(s) in plan not found:\n - " + "\n - ".join(missing)) + + +EXEC_START = re.compile(r'^\s*#\s*\[docs-exec:([^\]]+)\]\s*$') +EXEC_END = r'^\s*#\s*\[docs-exec:%s-end\]\s*$' +HEADER = "#!/usr/bin/env bash\nset -euo pipefail\n" + + +@dataclass +class ExtractedScript: + text: str + used_exec_blocks: bool + block_names: list[str] + errors: list[str] = field(default_factory=list) + + +def build_ci_text_from_exec_blocks(script_path: Path) -> ExtractedScript: + """Parse script and concatenate all [docs-exec:]...[docs-exec:-end] blocks.""" + try: + src_lines = script_path.read_text(encoding="utf-8", errors="ignore").splitlines(True) + except OSError as e: + raise OSError(f"Failed to read script file {script_path}: {e}") from e + + out_chunks: list[str] = [] + names: list[str] = [] + errors: list[str] = [] + + it = iter(src_lines) + for line in it: + m = EXEC_START.match(line) + if not m: + continue + + name = m.group(1).strip() + names.append(name) + end_re = re.compile(EXEC_END % re.escape(name)) + + block: list[str] = [] + for block_line in it: + if end_re.match(block_line): + break + block.append(block_line) + else: + errors.append(f"Missing [docs-exec:{name}-end] in {script_path}") + continue + + chunk = "".join(block) + if not chunk.endswith("\n"): + chunk += "\n" + out_chunks.append(chunk) + + if names: + return ExtractedScript( + text=HEADER + "".join(out_chunks), + used_exec_blocks=True, + block_names=names, + errors=errors, + ) + + return ExtractedScript(text="", used_exec_blocks=False, block_names=[], errors=errors) + + +def run_script_text(text: str, cwd: Path) -> int: + """Execute bash by piping the given text to stdin.""" + completed = run( # nosec B603 + ["bash"], input=text, text=True, cwd=str(cwd), check=False + ) + return completed.returncode + + +def main() -> int: + args = parse_args() + scripts = read_plan(args.plan) + + if not scripts: + print("Plan is empty. Nothing to run.") + return 0 + + require_files_exist(scripts) + dry_run = os.environ.get("DOCS_DRY_RUN") == "1" + + print("\n--- Documentation Test Execution ---") + print(f"Total steps: {len(scripts)}\n") + + # Phase 1: Parse and validate all files + extracted: list[tuple[Path, ExtractedScript]] = [] + print("[PLAN] Execution plan:") + had_structural_errors = False + for i, script in enumerate(scripts, start=1): + ex = build_ci_text_from_exec_blocks(script) + extracted.append((script, ex)) + using = f"docs-exec: {', '.join(ex.block_names)}" if ex.used_exec_blocks else "NO docs-exec FOUND" + print(f" [{i:02d}] {script} | {using}") + for err in ex.errors: + print(f"ERROR: {err}", file=sys.stderr) + had_structural_errors = True + print("") + + if had_structural_errors: + print("[run-doc-pages] Aborting due to structural errors in docs-exec blocks.", file=sys.stderr) + return 1 + + # Phase 2: Process the validated plan + if dry_run: + print("[DRY RUN] Printing all assembled scripts (no execution):\n") + for i, (script, ex) in enumerate(extracted, start=1): + print(f"----- BEGIN SCRIPT [{i}] {script} -----") + if ex.used_exec_blocks: + sys.stdout.write(ex.text) + else: + print("# (no [docs-exec:*] blocks found - this would be skipped in a real run)") + print(f"----- END SCRIPT [{i}] {script} -----\n") + + print("[DRY RUN] Completed printing scripts. Nothing executed.") + else: + for i, (script, ex) in enumerate(extracted, start=1): + print(f"==> [Step {i}/{len(scripts)}] {script}") + + if not ex.used_exec_blocks: + print(" -> No [docs-exec:*] blocks found. Skipping execution for this file.") + print(f"\n--- [Step {i}/{len(scripts)}] SKIPPED: {script} ---\n") + continue # Move to the next script in the plan + + sect_disp = ", ".join(ex.block_names) + print(f" using: docs-exec blocks -> {sect_disp}\n") + + rc = run_script_text(ex.text, cwd=script.parent) + if rc != 0: + print(f"\n[run-doc-pages] FAILURE at step {i}: {script} exited with code {rc}") + return rc + + print(f"\n--- [Step {i}/{len(scripts)}] SUCCESS: {script} ---\n") + + print("All steps in the execution plan completed successfully.") + + return 0 + + +if __name__ == "__main__": + try: + sys.exit(main()) + except KeyboardInterrupt: + print("\n[run-doc-pages] Interrupted by user.", file=sys.stderr) + sys.exit(130) + except FileNotFoundError as e: + # Configuration error (e.g., plan file not found) + print(f"[run-doc-pages] CONFIGURATION ERROR: {e}", file=sys.stderr) + sys.exit(2) + except Exception as exc: + # Unexpected runtime error + print(f"[run-doc-pages] UNEXPECTED ERROR: {exc}", file=sys.stderr) + sys.exit(1) diff --git a/ci/select-doc-pages.py b/ci/select-doc-pages.py new file mode 100644 index 0000000..917dfdd --- /dev/null +++ b/ci/select-doc-pages.py @@ -0,0 +1,151 @@ +#!/usr/bin/env python3 +""" +Builds an ordered execution plan of snippet scripts to run for docs CI. + +Usage: + python ci/select-doc-pages.py --changed-file-list changed.txt --out plan.txt [--repo-root .] + +Dependency additions: + In any *.task.sh file, add lines like: + # @depends: tutorial/snippets/get-started-with-openstack.task.sh +""" +from __future__ import annotations + +import argparse +from dataclasses import dataclass, field +from pathlib import Path +import re +import sys +from typing import List + +def parse_args() -> argparse.Namespace: + p = argparse.ArgumentParser(description="Select doc snippet pages to run.") + p.add_argument("--changed-file-list", required=True, type=Path) + p.add_argument("--out", required=True, type=Path) + p.add_argument("--repo-root", default=".", type=Path) + return p.parse_args() + + +def normalize_path(path: Path, repo_root: Path) -> str: + try: + return path.resolve().relative_to(repo_root.resolve()).as_posix() + except ValueError: + return path.resolve().as_posix() + + +@dataclass +class ExecutionPlan: + """Represents the set of scripts to be executed.""" + scripts: List[Path] = field(default_factory=list) + repo_root: Path = Path(".") + _depends_re = re.compile(r"^\s*#\s*@depends:\s+(.+?)\s*$") + + @classmethod + def from_changed_files(cls, path: Path, repo_root: Path) -> "ExecutionPlan": + """Create a plan from a file listing changed paths.""" + if not path.is_file(): + raise FileNotFoundError(f"Changed file list not found: {path}") + + lines = path.read_text(encoding="utf-8", errors="ignore").splitlines() + # Filter comments and empty lines + task_strings = [ + ln.strip() for ln in lines if ln.strip() and not ln.strip().startswith("#") + ] + # Treat each changed entry as repo-relative and resolve to absolute + task_paths = [(repo_root / Path(p)).resolve() for p in task_strings if p.endswith(".task.sh")] + + # Preserve order, drop duplicates + unique_paths = list(dict.fromkeys(task_paths)) + return cls(scripts=unique_paths, repo_root=repo_root) + + def validate_paths_exist(self) -> None: + """Ensure all scripts in the plan exist on disk.""" + missing = [str(p) for p in self.scripts if not p.is_file()] + if missing: + joined = "\n - ".join(missing) + raise FileNotFoundError(f"The following snippet files do not exist:\n - {joined}") + + def _resolve_dep(self, raw_dep: str, relative_to: Path) -> Path: + """Resolve a dependency path based on its format.""" + s = raw_dep.strip().strip("'\"") + if s.startswith("/"): + return Path(s).resolve() + if s.startswith("./") or s.startswith("../"): + return (relative_to.parent / s).resolve() + return (self.repo_root / s).resolve() + + def _parse_direct_depends(self, script: Path) -> List[Path]: + """Parse '# @depends:' lines from a single script file.""" + try: + text = script.read_text(encoding="utf-8", errors="ignore") + except FileNotFoundError: + return [] + + deps: List[Path] = [] + for line in text.splitlines(): + if m := self._depends_re.match(line): + raw = m.group(1) + # Only consider *.task.sh dependencies + if raw.strip().strip("'\"").endswith(".task.sh"): + deps.append(self._resolve_dep(raw, relative_to=script)) + + return list(dict.fromkeys(deps)) # De-dupe deps from same file + + def expand_dependencies(self) -> None: + """ + Rebuilds the script list, inserting direct dependencies before each script. + The final list is de-duplicated. + """ + expanded_list: List[Path] = [] + for script in self.scripts: + # Add dependencies first, then the script + expanded_list.extend(self._parse_direct_depends(script)) + expanded_list.append(script) + + self.scripts = list(dict.fromkeys(expanded_list)) + + def write(self, out_path: Path) -> None: + """Write the final, normalized plan to a file.""" + out_path.parent.mkdir(parents=True, exist_ok=True) + # Resolve all paths to be absolute + resolved_scripts = [p.resolve() for p in self.scripts] + normalized_paths = [normalize_path(p, self.repo_root) for p in resolved_scripts] + out_path.write_text("\n".join(normalized_paths) + "\n", encoding="utf-8") + + +def main() -> int: + args = parse_args() + + try: + plan = ExecutionPlan.from_changed_files( + path=args.changed_file_list, repo_root=args.repo_root + ) + except FileNotFoundError as e: + print(f"[select-doc-pages] ERROR: {e}", file=sys.stderr) + return 2 + + if not plan.scripts: + print("No *.task.sh files changed; writing empty plan.", file=sys.stderr) + plan.write(args.out) + return 0 + + # Validate initial changed files + plan.validate_paths_exist() + + # Expand with dependencies + plan.expand_dependencies() + + # Check paths to ensure all dependencies exist on disk + plan.validate_paths_exist() + + plan.write(args.out) + + return 0 + + +if __name__ == "__main__": + try: + sys.exit(main()) + except Exception as exc: + print(f"[select-doc-pages] ERROR: {exc}", file=sys.stderr) + sys.exit(1) \ No newline at end of file diff --git a/docs/how-to/features/caas.rst b/docs/how-to/features/caas.rst index 84a0e7a..f5a9d2f 100644 --- a/docs/how-to/features/caas.rst +++ b/docs/how-to/features/caas.rst @@ -10,9 +10,10 @@ Enabling CaaS To enable CaaS, run the following command: -:: - - sunbeam enable caas +.. literalinclude:: ../snippets/caas.task.sh + :language: bash + :start-after: [docs-view:enable-caas] + :end-before: [docs-view:enable-caas-end] Use the OpenStack CLI to manage container infrastructures. See the upstream `Magnum`_ documentation for details. @@ -30,9 +31,10 @@ Disabling CaaS To disable CaaS, run the following command: -:: - - sunbeam disable caas +.. literalinclude:: ../snippets/caas.task.sh + :language: bash + :start-after: [docs-view:disable-caas] + :end-before: [docs-view:disable-caas-end] Usage ----- @@ -70,7 +72,7 @@ Create a cluster template using the following command: --labels octavia_lb_algorithm=SOURCE_IP_PORT \ --network-driver cilium \ --coe kubernetes - + Sample output: .. terminal:: diff --git a/docs/how-to/features/images-sync.rst b/docs/how-to/features/images-sync.rst index 25bd87a..7d2f416 100644 --- a/docs/how-to/features/images-sync.rst +++ b/docs/how-to/features/images-sync.rst @@ -9,18 +9,20 @@ Enable Images Sync To enable Images Sync, run the following command: -:: - - sunbeam enable images-sync +.. literalinclude:: ../snippets/images-sync.task.sh + :language: bash + :start-after: [docs-view:enable-images-sync] + :end-before: [docs-view:enable-images-sync-end] Disable Images Sync ------------------- To disable Images Sync, run the following command: -:: - - sunbeam disable images-sync +.. literalinclude:: ../snippets/images-sync.task.sh + :language: bash + :start-after: [docs-view:disable-images-sync] + :end-before: [docs-view:disable-images-sync-end] .. caution:: **Caution**: Disabling Images Sync will **not** remove images that have been diff --git a/docs/how-to/features/instance-recovery.rst b/docs/how-to/features/instance-recovery.rst index 2734e1e..fa9ef0f 100644 --- a/docs/how-to/features/instance-recovery.rst +++ b/docs/how-to/features/instance-recovery.rst @@ -9,9 +9,10 @@ Enabling Instance Recovery To enable Instance Recovery, run the following command: -:: - - sunbeam enable instance-recovery +.. literalinclude:: ../snippets/instance-recovery.task.sh + :language: bash + :start-after: [docs-view:enable-instance-recovery] + :end-before: [docs-view:enable-instance-recovery-end] .. note:: For MAAS provider, ensure to reserve an IP on storage subnet with label `-storage-ippool`, @@ -27,9 +28,10 @@ Disabling Instance Recovery To disable Instance Recovery, run the following command: -:: - - sunbeam disable instance-recovery +.. literalinclude:: ../snippets/instance-recovery.task.sh + :language: bash + :start-after: [docs-view:disable-instance-recovery] + :end-before: [docs-view:disable-instance-recovery-end] Encrypted disk recovery ----------------------- diff --git a/docs/how-to/features/ldap.rst b/docs/how-to/features/ldap.rst index e3a7a14..76a492f 100644 --- a/docs/how-to/features/ldap.rst +++ b/docs/how-to/features/ldap.rst @@ -11,18 +11,20 @@ Enabling LDAP To enable the LDAP feature, run the following command: -:: - - sunbeam enable ldap +.. literalinclude:: ../snippets/ldap.task.sh + :language: bash + :start-after: [docs-view:enable-ldap] + :end-before: [docs-view:enable-ldap-end] Disabling LDAP -------------- To disable the LDAP feature, run the following command: -:: - - sunbeam disable ldap +.. literalinclude:: ../snippets/ldap.task.sh + :language: bash + :start-after: [docs-view:disable-ldap] + :end-before: [docs-view:disable-ldap-end] Usage ----- @@ -63,11 +65,10 @@ LDAP servers. 3. Use the ``sunbeam ldap add-domain`` command to set up the domain, adding the ``--ca-cert-file`` option if TLS is in use: -.. code:: text - - sunbeam ldap add-domain \ - --domain-config-file ./dom1.yaml \ - --ca-cert-file ./dom1.cert dom1 +.. literalinclude:: ../snippets/ldap.task.sh + :language: bash + :start-after: [docs-view:ldap-add] + :end-before: [docs-view:ldap-add-end] 4. A new LDAP-backed domain will be created in Keystone. Verify this with the native ``openstack`` CLI: @@ -89,27 +90,30 @@ Updating a domain To update an LDAP domain the process is similar to adding one: -:: - - sunbeam ldap update-domain --domain-config-file ./dom1.yaml --ca-cert-file ./dom1.cert dom1 +.. literalinclude:: ../snippets/ldap.task.sh + :language: bash + :start-after: [docs-view:ldap-update] + :end-before: [docs-view:ldap-update-end] Listing domains ~~~~~~~~~~~~~~~ To list LDAP domains: -:: - - sunbeam ldap list-domains +.. literalinclude:: ../snippets/ldap.task.sh + :language: bash + :start-after: [docs-view:ldap-list] + :end-before: [docs-view:ldap-list-end] Removing a domain ~~~~~~~~~~~~~~~~~ To remove an LDAP domain: -:: - - sunbeam ldap remove-domain +.. literalinclude:: ../snippets/ldap.task.sh + :language: bash + :start-after: [docs-view:ldap-remove] + :end-before: [docs-view:ldap-remove-end] .. important:: Since configuration (e.g. OpenStack projects) could have been made to the diff --git a/docs/how-to/features/load-balancer.rst b/docs/how-to/features/load-balancer.rst index e2c3a4f..9aa8f4e 100644 --- a/docs/how-to/features/load-balancer.rst +++ b/docs/how-to/features/load-balancer.rst @@ -22,9 +22,10 @@ Enabling Load Balancer To enable Load Balancer, run the following command: -:: - - sunbeam enable loadbalancer +.. literalinclude:: ../snippets/load-balancer.task.sh + :language: bash + :start-after: [docs-view:enable-load-balancer] + :end-before: [docs-view:enable-load-balancer-end] This enables Octavia with the OVN provider. Use the OpenStack CLI to manage load balancers. See the upstream `Octavia documentation @@ -207,9 +208,10 @@ Disabling Load Balancer To disable Load Balancer, run the following command: -:: - - sunbeam disable loadbalancer +.. literalinclude:: ../snippets/load-balancer.task.sh + :language: bash + :start-after: [docs-view:disable-load-balancer] + :end-before: [docs-view:disable-load-balancer-end] Usage ----- diff --git a/docs/how-to/features/observability.rst b/docs/how-to/features/observability.rst index c22b35f..517692d 100644 --- a/docs/how-to/features/observability.rst +++ b/docs/how-to/features/observability.rst @@ -27,9 +27,10 @@ observability related offers. To enable Sunbeam observability integration, run the following command: -:: - - sunbeam enable observability external CONTROLLER GRAFANA_DASHBOARD_OFFER_URL PROMETHEUS_RECEIVE_REMOTE_WRITE_OFFER_URL LOKI_LOGGING_OFFER_URL +.. literalinclude:: ../snippets/observability.task.sh + :language: bash + :start-after: [docs-view:enable-observability-external] + :end-before: [docs-view:enable-observability-external-end] ``CONTROLLER`` is the name of external Juju controller that hosts COS. @@ -46,9 +47,10 @@ Disabling Observability To disable Observability, run the following command: -:: - - sunbeam disable observability external +.. literalinclude:: ../snippets/observability.task.sh + :language: bash + :start-after: [docs-view:disable-observability-external] + :end-before: [docs-view:disable-observability-external-end] Deploy COS in Canonical OpenStack --------------------------------- @@ -60,9 +62,10 @@ Enabling Observability To enable Observability, run the following command: -:: - - sunbeam enable observability embedded +.. literalinclude:: ../snippets/observability.task.sh + :language: bash + :start-after: [docs-view:enable-observability-embedded] + :end-before: [docs-view:enable-observability-embedded-end] .. note:: Storage size for the embedded observability stack can be configured using @@ -78,18 +81,20 @@ Disabling Observability To disable Observability, run the following command: -:: - - sunbeam disable observability embedded +.. literalinclude:: ../snippets/observability.task.sh + :language: bash + :start-after: [docs-view:disable-observability-embedded] + :end-before: [docs-view:disable-observability-embedded-end] Retrieve Grafana dashboard URL ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ To get the URL of the dashboard use the ``dashboard url`` command: -:: - - sunbeam observability dashboard url +.. literalinclude:: ../snippets/observability.task.sh + :language: bash + :start-after: [docs-view:observability-dashboard-url] + :end-before: [docs-view:observability-dashboard-url-end] Sample output: diff --git a/docs/how-to/features/orchestration.rst b/docs/how-to/features/orchestration.rst index 4be92f6..9f158b7 100644 --- a/docs/how-to/features/orchestration.rst +++ b/docs/how-to/features/orchestration.rst @@ -9,9 +9,10 @@ Enabling Orchestration To enable Orchestration, run the following command: -:: - - sunbeam enable orchestration +.. literalinclude:: ../snippets/orchestration.task.sh + :language: bash + :start-after: [docs-view:enable-orchestration] + :end-before: [docs-view:enable-orchestration-end] Use OpenStack CLI to manage orchestration stacks. See the upstream `Heat documentation `__ diff --git a/docs/how-to/features/resource-optimization.rst b/docs/how-to/features/resource-optimization.rst index 35a6f49..348c4e5 100644 --- a/docs/how-to/features/resource-optimization.rst +++ b/docs/how-to/features/resource-optimization.rst @@ -9,18 +9,20 @@ Enabling Resource Optimization To enable Resource Optimization, run the following command: -:: - - sunbeam enable resource-optimization +.. literalinclude:: ../snippets/resource-optimization.task.sh + :language: bash + :start-after: [docs-view:enable-resource-optimization] + :end-before: [docs-view:enable-resource-optimization-end] Disabling Resource Optimization ------------------------------- To disable Resource Optimization, run the following command: -:: - - sunbeam disable resource-optimization +.. literalinclude:: ../snippets/resource-optimization.task.sh + :language: bash + :start-after: [docs-view:disable-resource-optimization] + :end-before: [docs-view:disable-resource-optimization-end] Usage ----- diff --git a/docs/how-to/features/secrets.rst b/docs/how-to/features/secrets.rst index 9664dfb..a73df42 100644 --- a/docs/how-to/features/secrets.rst +++ b/docs/how-to/features/secrets.rst @@ -8,9 +8,10 @@ Enabling Secrets To enable Secrets, run the following command: -:: - - sunbeam enable secrets +.. literalinclude:: ../snippets/secrets.task.sh + :language: bash + :start-after: [docs-view:enable-secrets] + :end-before: [docs-view:enable-secrets-end] The openstack CLI can now be used to manage Secrets. See the upstream `Barbican CLI`_ documentation for details. @@ -25,9 +26,10 @@ Disabling Secrets To disable Secrets, run the following command: -:: - - sunbeam disable secrets +.. literalinclude:: ../snippets/secrets.task.sh + :language: bash + :start-after: [docs-view:disable-secrets] + :end-before: [docs-view:disable-secrets-end] Usage ----- @@ -41,7 +43,7 @@ Verify if a user belongs to this role with (admin rights needed): openstack role assignment list --user --role creator +----------------------------------+----------------------------------+-------+----------------------------------+--------+--------+-----------+ - | Role | User | Group | Project | Domain | System | Inherited | + | Role | User | Group | Project | Domain | System | Inherited | +----------------------------------+----------------------------------+-------+----------------------------------+--------+--------+-----------+ | 3ef18094c76a403291ccf727851616ae | 4f2e8ef6b897403fb9865123b7b57a34 | | 3e5bb39a247b471494e051ae8d0530fb | | | False | +----------------------------------+----------------------------------+-------+----------------------------------+--------+--------+-----------+ diff --git a/docs/how-to/features/telemetry.rst b/docs/how-to/features/telemetry.rst index 40ef6da..3554f3f 100644 --- a/docs/how-to/features/telemetry.rst +++ b/docs/how-to/features/telemetry.rst @@ -9,9 +9,10 @@ Enabling Telemetry To enable Telemetry, run the following command: -:: - - sunbeam enable telemetry +.. literalinclude:: ../snippets/telemetry.task.sh + :language: bash + :start-after: [docs-view:enable-telemetry] + :end-before: [docs-view:enable-telemetry-end] Use the OpenStack CLI to create and manage alarms. See the upstream `Aodh @@ -23,9 +24,10 @@ Disabling Telemetry To disable Telemetry, run the following command: -:: - - sunbeam disable telemetry +.. literalinclude:: ../snippets/telemetry.task.sh + :language: bash + :start-after: [docs-view:disable-telemetry] + :end-before: [docs-view:disable-telemetry-end] This will terminate the application but not remove it from the model. To do that, run the following: diff --git a/docs/how-to/features/validation.rst b/docs/how-to/features/validation.rst index 5302c4f..d8d2ff3 100644 --- a/docs/how-to/features/validation.rst +++ b/docs/how-to/features/validation.rst @@ -14,9 +14,10 @@ Enable Validation To enable Validation, run the following command: -:: - - sunbeam enable validation +.. literalinclude:: ../snippets/validation.task.sh + :language: bash + :start-after: [docs-view:enable-validation] + :end-before: [docs-view:enable-validation-end] .. note :: @@ -29,9 +30,10 @@ Disable Validation To disable Validation, run the following command: -:: - - sunbeam disable validation +.. literalinclude:: ../snippets/validation.task.sh + :language: bash + :start-after: [docs-view:disable-validation] + :end-before: [docs-view:disable-validation-end] Usage ----- @@ -61,14 +63,14 @@ Sample output: :: - Available profiles - Name Description - ──────────────────────────────────────────────────────────────────────────────────────── - refstack Tests that are part of the RefStack project https://refstack.openstack.org/ - quick A short list of tests for quick validation - smoke Tests tagged as "smoke" - all All tests (very large number, not usually recommended) - + Available profiles + Name Description + ──────────────────────────────────────────────────────────────────────────────────────── + refstack Tests that are part of the RefStack project https://refstack.openstack.org/ + quick A short list of tests for quick validation + smoke Tests tagged as "smoke" + all All tests (very large number, not usually recommended) + A summary of the validation result will be printed out to the screen upon completion of the command: diff --git a/docs/how-to/snippets/caas.task.sh b/docs/how-to/snippets/caas.task.sh new file mode 100644 index 0000000..bf78f21 --- /dev/null +++ b/docs/how-to/snippets/caas.task.sh @@ -0,0 +1,18 @@ +# @depends: tutorial/snippets/get-started-with-openstack.task.sh + +# [docs-view:enable-caas] +sunbeam enable caas +# [docs-view:enable-caas-end] + +# [docs-exec:enable-caas] +sg snap_daemon 'sunbeam enable caas' +# [docs-exec:enable-caas-end] + + +# [docs-view:disable-caas] +sunbeam disable caas +# [docs-view:disable-caas-end] + +# [docs-exec:disable-caas] +sg snap_daemon 'sunbeam disable caas' +# [docs-exec:disable-caas-end] diff --git a/docs/how-to/snippets/images-sync.task.sh b/docs/how-to/snippets/images-sync.task.sh new file mode 100644 index 0000000..9cf1b9d --- /dev/null +++ b/docs/how-to/snippets/images-sync.task.sh @@ -0,0 +1,18 @@ +# @depends: tutorial/snippets/get-started-with-openstack.task.sh + +# [docs-view:enable-images-sync] +sunbeam enable images-sync +# [docs-view:enable-images-sync-end] + +# [docs-exec:enable-images-sync] +sg snap_daemon 'sunbeam enable images-sync' +# [docs-exec:enable-images-sync-end] + + +# [docs-view:disable-images-sync] +sunbeam disable images-sync +# [docs-view:disable-images-sync-end] + +# [docs-exec:disable-images-sync] +sg snap_daemon 'sunbeam disable images-sync' +# [docs-exec:disable-images-sync-end] diff --git a/docs/how-to/snippets/instance-recovery.task.sh b/docs/how-to/snippets/instance-recovery.task.sh new file mode 100644 index 0000000..0f40858 --- /dev/null +++ b/docs/how-to/snippets/instance-recovery.task.sh @@ -0,0 +1,18 @@ +# @depends: tutorial/snippets/get-started-with-openstack.task.sh + +# [docs-view:enable-instance-recovery] +sunbeam enable instance-recovery +# [docs-view:enable-instance-recovery-end] + +# [docs-exec:enable-instance-recovery] +sg snap_daemon 'sunbeam enable instance-recovery' +# [docs-exec:enable-instance-recovery-end] + + +# [docs-view:disable-instance-recovery] +sunbeam disable instance-recovery +# [docs-view:disable-instance-recovery-end] + +# [docs-exec:disable-instance-recovery] +sg snap_daemon 'sunbeam disable instance-recovery' +# [docs-exec:disable-instance-recovery-end] \ No newline at end of file diff --git a/docs/how-to/snippets/ldap.task.sh b/docs/how-to/snippets/ldap.task.sh new file mode 100644 index 0000000..27fb236 --- /dev/null +++ b/docs/how-to/snippets/ldap.task.sh @@ -0,0 +1,40 @@ +# @depends: tutorial/snippets/get-started-with-openstack.task.sh + +# [docs-view:enable-ldap] +sunbeam enable ldap +# [docs-view:enable-ldap-end] + +# [docs-exec:enable-ldap] +sg snap_daemon 'sunbeam enable ldap' +# [docs-exec:enable-ldap-end] + + +# [docs-view:disable-ldap] +sunbeam disable ldap +# [docs-view:disable-ldap-end] + +# [docs-exec:disable-ldap] +sg snap_daemon 'sunbeam disable ldap' +# [docs-exec:disable-ldap-end] + + +# [docs-view:ldap-add] +sunbeam ldap add-domain \ + --domain-config-file ./dom1.yaml \ + --ca-cert-file ./dom1.cert dom1 +# [docs-view:ldap-add-end] + + +# [docs-view:ldap-update] +sunbeam ldap update-domain --domain-config-file ./dom1.yaml --ca-cert-file ./dom1.cert dom1 +# [docs-view:ldap-update-end] + + +# [docs-view:ldap-list] +sunbeam ldap list-domains +# [docs-view:ldap-list-end] + + +# [docs-view:ldap-remove] +sunbeam ldap remove-domain '' +# [docs-view:ldap-remove-end] diff --git a/docs/how-to/snippets/load-balancer.task.sh b/docs/how-to/snippets/load-balancer.task.sh new file mode 100644 index 0000000..1308f1d --- /dev/null +++ b/docs/how-to/snippets/load-balancer.task.sh @@ -0,0 +1,17 @@ +# @depends: tutorial/snippets/get-started-with-openstack.task.sh + +# [docs-view:enable-load-balancer] +sunbeam enable load-balancer +# [docs-view:enable-load-balancer-end] + +# [docs-exec:enable-load-balancer] +sg snap_daemon 'sunbeam enable load-balancer' +# [docs-exec:enable-load-balancer-end] + +# [docs-view:disable-load-balancer] +sunbeam disable load-balancer +# [docs-view:disable-load-balancer-end] + +# [docs-exec:disable-load-balancer] +sg snap_daemon 'sunbeam disable load-balancer' +# [docs-exec:disable-load-balancer-end] diff --git a/docs/how-to/snippets/observability.task.sh b/docs/how-to/snippets/observability.task.sh new file mode 100644 index 0000000..af5af13 --- /dev/null +++ b/docs/how-to/snippets/observability.task.sh @@ -0,0 +1,33 @@ +# @depends: tutorial/snippets/get-started-with-openstack.task.sh + +# [docs-view:enable-observability-external] +sunbeam enable observability external CONTROLLER GRAFANA_DASHBOARD_OFFER_URL PROMETHEUS_RECEIVE_REMOTE_WRITE_OFFER_URL LOKI_LOGGING_OFFER_URL +# [docs-view:enable-observability-external-end] + + +# [docs-view:disable-observability-external] +sunbeam disable observability external +# [docs-view:disable-observability-external-end] + + +# [docs-view:enable-observability-embedded] +sunbeam enable observability embedded +# [docs-view:enable-observability-embedded-end] + +# [docs-exec:enable-observability-embedded] +sg snap_daemon 'sunbeam enable observability embedded' +# [docs-exec:enable-observability-embedded-end] + + +# [docs-view:disable-observability-embedded] +sunbeam disable observability embedded +# [docs-view:disable-observability-embedded-end] + +# [docs-exec:disable-observability-embedded] +sg snap_daemon 'sunbeam disable observability embedded' +# [docs-exec:disable-observability-embedded-end] + + +# [docs-view:observability-dashboard-url] +sunbeam observability dashboard-url +# [docs-view:observability-dashboard-url-end] \ No newline at end of file diff --git a/docs/how-to/snippets/orchestration.task.sh b/docs/how-to/snippets/orchestration.task.sh new file mode 100644 index 0000000..111392c --- /dev/null +++ b/docs/how-to/snippets/orchestration.task.sh @@ -0,0 +1,18 @@ +# @depends: tutorial/snippets/get-started-with-openstack.task.sh + +# [docs-view:enable-orchestration] +sunbeam enable orchestration +# [docs-view:enable-orchestration-end] + +# [docs-exec:enable-orchestration] +sg snap_daemon 'sunbeam enable orchestration' +# [docs-exec:enable-orchestration-end] + + +# [docs-view:disable-orchestration] +sunbeam disable orchestration +# [docs-view:disable-orchestration-end] + +# [docs-exec:disable-orchestration] +sg snap_daemon 'sunbeam disable orchestration' +# [docs-exec:disable-orchestration-end] diff --git a/docs/how-to/snippets/resource-optimization.task.sh b/docs/how-to/snippets/resource-optimization.task.sh new file mode 100644 index 0000000..3deac00 --- /dev/null +++ b/docs/how-to/snippets/resource-optimization.task.sh @@ -0,0 +1,18 @@ +# @depends: tutorial/snippets/get-started-with-openstack.task.sh + +# [docs-view:enable-resource-optimization] +sunbeam enable resource-optimization +# [docs-view:enable-resource-optimization-end] + +# [docs-exec:enable-resource-optimization] +sg snap_daemon 'sunbeam enable resource-optimization' +# [docs-exec:enable-resource-optimization-end] + + +# [docs-view:disable-resource-optimization] +sunbeam disable resource-optimization +# [docs-view:disable-resource-optimization-end] + +# [docs-exec:disable-resource-optimization] +sg snap_daemon 'sunbeam disable resource-optimization' +# [docs-exec:disable-resource-optimization-end] \ No newline at end of file diff --git a/docs/how-to/snippets/secrets.task.sh b/docs/how-to/snippets/secrets.task.sh new file mode 100644 index 0000000..b6650bc --- /dev/null +++ b/docs/how-to/snippets/secrets.task.sh @@ -0,0 +1,18 @@ +# @depends: tutorial/snippets/get-started-with-openstack.task.sh + +# [docs-view:enable-secrets] +sunbeam enable secrets +# [docs-view:enable-secrets-end] + +# [docs-exec:enable-secrets] +sg snap_daemon 'sunbeam enable secrets' +# [docs-exec:enable-secrets-end] + + +# [docs-view:disable-secrets] +sunbeam disable secrets +# [docs-view:disable-secrets-end] + +# [docs-exec:disable-secrets] +sg snap_daemon 'sunbeam disable secrets' +# [docs-exec:disable-secrets-end] \ No newline at end of file diff --git a/docs/how-to/snippets/telemetry.task.sh b/docs/how-to/snippets/telemetry.task.sh new file mode 100644 index 0000000..1daeb23 --- /dev/null +++ b/docs/how-to/snippets/telemetry.task.sh @@ -0,0 +1,18 @@ +# @depends: tutorial/snippets/get-started-with-openstack.task.sh + +# [docs-view:enable-telemetry] +sunbeam enable telemetry +# [docs-view:enable-telemetry-end] + +# [docs-exec:enable-telemetry] +sg snap_daemon 'sunbeam enable telemetry' +# [docs-exec:enable-telemetry-end] + + +# [docs-view:disable-telemetry] +sunbeam disable telemetry +# [docs-view:disable-telemetry-end] + +# [docs-exec:disable-telemetry] +sg snap_daemon 'sunbeam disable telemetry' +# [docs-exec:disable-telemetry-end] \ No newline at end of file diff --git a/docs/how-to/snippets/validation.task.sh b/docs/how-to/snippets/validation.task.sh new file mode 100644 index 0000000..68040af --- /dev/null +++ b/docs/how-to/snippets/validation.task.sh @@ -0,0 +1,18 @@ +# @depends: tutorial/snippets/get-started-with-openstack.task.sh + +# [docs-view:enable-validation] +sunbeam enable validation +# [docs-view:enable-validation-end] + +# [docs-exec:enable-validation] +sg snap_daemon 'sunbeam enable validation' +# [docs-exec:enable-validation-end] + + +# [docs-view:disable-validation] +sunbeam disable validation +# [docs-view:disable-validation-end] + +# [docs-exec:disable-validation] +sg snap_daemon 'sunbeam disable validation' +# [docs-exec:disable-validation-end] \ No newline at end of file diff --git a/docs/tutorial/get-started-with-openstack.rst b/docs/tutorial/get-started-with-openstack.rst index 277e6d6..f8a93a6 100644 --- a/docs/tutorial/get-started-with-openstack.rst +++ b/docs/tutorial/get-started-with-openstack.rst @@ -50,9 +50,10 @@ abstracting its complexity from operators. To install the ``openstack`` snap, execute the following terminal command: -.. code-block :: text - - sudo snap install openstack +.. literalinclude:: snippets/get-started-with-openstack.task.sh + :language: bash + :start-after: [docs-view:installation] + :end-before: [docs-view:installation-end] Prepare the machine ------------------- @@ -70,14 +71,15 @@ In order to facilitate this process, Sunbeam can generate a script that you can and execute step by step: .. code-block :: text - + sunbeam prepare-node-script --bootstrap However, if you simply want to execute all those commands at once, you can also pipe them directly to Bash instead: -.. code-block :: text - - sunbeam prepare-node-script --bootstrap | bash -x && newgrp snap_daemon +.. literalinclude:: snippets/get-started-with-openstack.task.sh + :language: bash + :start-after: [docs-view:prepare-node-script] + :end-before: [docs-view:prepare-node-script-end] Bootstrap the cloud ------------------- @@ -99,9 +101,10 @@ a while to complete. In principle, Sunbeam orchestrates the following actions in To bootstrap the cloud for sample usage, execute the following command: -.. code-block :: text - - sunbeam cluster bootstrap --accept-defaults --role control,compute,storage +.. literalinclude:: snippets/get-started-with-openstack.task.sh + :language: bash + :start-after: [docs-view:bootstrap] + :end-before: [docs-view:bootstrap-end] .. important:: @@ -112,7 +115,7 @@ To bootstrap the cloud for sample usage, execute the following command: Once it completes, you should be able to see the following message on your screen: .. code-block :: text - + Node has been bootstrapped with roles: storage, control, compute .. note :: @@ -138,9 +141,10 @@ We will explore in :doc:`another tutorial` how th To configure the cloud for sample usage, execute the following command: -.. code-block :: text - - sunbeam configure --accept-defaults --openrc demo-openrc +.. literalinclude:: snippets/get-started-with-openstack.task.sh + :language: bash + :start-after: [docs-view:configure] + :end-before: [docs-view:configure-end] Once it completes, you should be able to see the following message on your screen: @@ -159,14 +163,15 @@ The best way to verify whether Canonical OpenStack has been deployed successfull In order to launch a test VM, execute the following command: -.. code-block :: text - - sunbeam launch ubuntu --name test +.. literalinclude:: snippets/get-started-with-openstack.task.sh + :language: bash + :start-after: [docs-view:launch] + :end-before: [docs-view:launch-end] Sample output: .. code-block :: text - + Launching an OpenStack instance ... Access instance with `ssh -i /home/ubuntu/snap/openstack/584/sunbeam ubuntu@10.20.20.94` @@ -174,14 +179,15 @@ Sample output: You should now be able to connect to your VM over SSH using the provided command: -.. code-block :: text - - ssh -i /home/ubuntu/.config/openstack/sunbeam ubuntu@10.20.20.200 +.. literalinclude:: snippets/get-started-with-openstack.task.sh + :language: bash + :start-after: [docs-view:ssh] + :end-before: [docs-view:ssh-end] That's it. You're now connected to the VM. You can use regular shell commands to execute various tasks: .. code-block :: text - + $ uptime 10:54:29 up 1 min, 1 user, load average: 0.00, 0.00, 0.00 diff --git a/docs/tutorial/snippets/get-started-with-openstack.task.sh b/docs/tutorial/snippets/get-started-with-openstack.task.sh new file mode 100755 index 0000000..d422907 --- /dev/null +++ b/docs/tutorial/snippets/get-started-with-openstack.task.sh @@ -0,0 +1,48 @@ +# [docs-view:installation] +sudo snap install openstack +# [docs-view:installation-end] + +# [docs-exec:installation] +sudo snap install openstack +# [docs-exec:installation-end] + + +# [docs-view:prepare-node-script] +sunbeam prepare-node-script --bootstrap | bash -x && newgrp snap_daemon +# [docs-view:prepare-node-script-end] + +# [docs-exec:prepare-node-script] +sunbeam prepare-node-script --bootstrap | bash -x +# [docs-exec:prepare-node-script-end] + + +# [docs-view:bootstrap] +sunbeam cluster bootstrap --accept-defaults --role control,compute,storage +# [docs-view:bootstrap-end] + +# [docs-exec:bootstrap] +sg snap_daemon 'sunbeam cluster bootstrap --accept-defaults --role control,compute' +# [docs-exec:bootstrap-end] + + +# [docs-view:configure] +sunbeam configure --accept-defaults --openrc demo-openrc +# [docs-view:configure-end] + +# [docs-exec:configure] +sg snap_daemon 'sunbeam configure --accept-defaults --openrc demo-openrc' +# [docs-exec:configure-end] + + +# [docs-view:launch] +sunbeam launch ubuntu --name test +# [docs-view:launch-end] + +# [docs-exec:launch] +sg snap_daemon 'sunbeam launch ubuntu --name test' +# [docs-exec:launch-end] + + +# [docs-view:ssh] +ssh -i /home/ubuntu/.config/openstack/sunbeam ubuntu@10.20.20.200 +# [docs-view:ssh-end] diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..c10d4c5 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,35 @@ +import os +import shutil +import subprocess +from pathlib import Path +import pytest + +REPO_ROOT = Path(__file__).resolve().parents[1] + +@pytest.fixture +def tmp_repo(tmp_path: Path) -> Path: + """Copies snippets and ci/ scripts into an isolated temp repo.""" + src = REPO_ROOT / "tests" / "snippets" + dst = tmp_path + shutil.copytree(src, dst / "tests/snippets", dirs_exist_ok=True) + (dst / "ci").mkdir(exist_ok=True) + for name in ["select-doc-pages.py", "run-doc-pages.py"]: + shutil.copy2(REPO_ROOT / "ci" / name, dst / "ci" / name) + return dst + +def run_cmd(args, cwd: Path, env=None): + env_vars = os.environ.copy() + if env: + env_vars.update(env) + return subprocess.run( + args, + cwd=cwd, + env=env_vars, + text=True, + capture_output=True, + check=False, + ) + +def write_file(path: Path, content: str): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") diff --git a/tests/snippets/base-fixture.task.sh b/tests/snippets/base-fixture.task.sh new file mode 100644 index 0000000..801293f --- /dev/null +++ b/tests/snippets/base-fixture.task.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +set -euo pipefail +# [docs-exec:install] +echo "install base" +# [docs-exec:install-end] +# [docs-exec:bootstrap] +echo "bootstrap base" +# [docs-exec:bootstrap-end] diff --git a/tests/snippets/fixture-missing-end.task.sh b/tests/snippets/fixture-missing-end.task.sh new file mode 100644 index 0000000..dcfdb6a --- /dev/null +++ b/tests/snippets/fixture-missing-end.task.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +set -euo pipefail +# [docs-exec:oops] +echo "missing closing tag" diff --git a/tests/snippets/fixture-multi-blocks.task.sh b/tests/snippets/fixture-multi-blocks.task.sh new file mode 100644 index 0000000..a111f56 --- /dev/null +++ b/tests/snippets/fixture-multi-blocks.task.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +set -euo pipefail +# [docs-exec:first] +echo "block one" +# [docs-exec:first-end] +# [docs-exec:second] +echo "block two" +# [docs-exec:second-end] diff --git a/tests/snippets/fixture-no-blocks.task.sh b/tests/snippets/fixture-no-blocks.task.sh new file mode 100644 index 0000000..0eb9366 --- /dev/null +++ b/tests/snippets/fixture-no-blocks.task.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +set -euo pipefail +# just a file with no docs-exec markers +echo "this file should trigger the no-blocks path" diff --git a/tests/snippets/fixture-relative-dep.task.sh b/tests/snippets/fixture-relative-dep.task.sh new file mode 100644 index 0000000..d323e94 --- /dev/null +++ b/tests/snippets/fixture-relative-dep.task.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +set -euo pipefail + +# @depends ./base-fixture.task.sh + +# [docs-exec:relative] +echo "relative dependency example" +# [docs-exec:relative-end] diff --git a/tests/snippets/fixture-with-dep.task.sh b/tests/snippets/fixture-with-dep.task.sh new file mode 100644 index 0000000..41019cc --- /dev/null +++ b/tests/snippets/fixture-with-dep.task.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +set -euo pipefail +# @depends: tests/snippets/base-fixture.task.sh +# [docs-exec:enable] +echo "enable feature that depends on base" +# [docs-exec:enable-end] diff --git a/tests/test_run_doc_pages.py b/tests/test_run_doc_pages.py new file mode 100644 index 0000000..b0fca78 --- /dev/null +++ b/tests/test_run_doc_pages.py @@ -0,0 +1,71 @@ +from pathlib import Path +from conftest import run_cmd, write_file + +def test_dry_run_prints_assembled_scripts(tmp_repo: Path): + """ + Checks that in dry-run mode, the script correctly assembles and prints + the content of multiple docs-exec blocks from multiple files. + """ + plan = tmp_repo / "plan.txt" + write_file(plan, + "tests/snippets/base-fixture.task.sh\n" + "tests/snippets/fixture-multi-blocks.task.sh\n") + + rc = run_cmd( + ["python3", "ci/run-doc-pages.py", "--plan", str(plan)], + cwd=tmp_repo, + env={"DOCS_DRY_RUN": "1"}, # Explicitly set dry-run mode + ) + + assert rc.returncode == 0, rc.stderr + out = rc.stdout + + # Check for content + assert "install base" in out + assert "block one" in out + + # Check that the final line of output is the correct dry-run message + last_line = out.strip().splitlines()[-1] + assert "Nothing executed" in last_line + +def test_dry_run_fails_on_structural_error(tmp_repo: Path): + """ + Checks that the script fails (even in dry-run) if it finds a malformed + block, because parsing happens before the dry-run check. + """ + plan = tmp_repo / "plan.txt" + write_file(plan, "tests/snippets/fixture-missing-end.task.sh\n") + + rc = run_cmd( + ["python3", "ci/run-doc-pages.py", "--plan", str(plan)], + cwd=tmp_repo, + env={"DOCS_DRY_RUN": "1"}, # Explicitly set dry-run mode + ) + + # Structural errors should cause a failure with exit code 1. + assert rc.returncode == 1 + + # Check for the specific error messages + assert "Missing [docs-exec:oops-end]" in rc.stderr + assert "Aborting due to structural errors" in rc.stderr + +def test_dry_run_handles_no_blocks_gracefully(tmp_repo: Path): + """ + Checks that in dry-run mode, a file with no blocks is reported correctly + and the process SUCCEEDS. + """ + plan = tmp_repo / "plan.txt" + write_file(plan, "tests/snippets/fixture-no-blocks.task.sh\n") + + rc = run_cmd( + ["python3", "ci/run-doc-pages.py", "--plan", str(plan)], + cwd=tmp_repo, + env={"DOCS_DRY_RUN": "1"}, # Explicitly set dry-run mode + ) + + # A dry run should always succeed if there are no structural errors. + assert rc.returncode == 0, rc.stderr + + # It should print the correct output for the file with no blocks. + assert "NO docs-exec FOUND" in rc.stdout + assert "(no [docs-exec:*] blocks found - this would be skipped in a real run)" in rc.stdout \ No newline at end of file diff --git a/tests/test_select_doc_pages.py b/tests/test_select_doc_pages.py new file mode 100644 index 0000000..f0a326b --- /dev/null +++ b/tests/test_select_doc_pages.py @@ -0,0 +1,43 @@ +from pathlib import Path +from conftest import run_cmd, write_file + +def test_dependency_inserted_before_changed(tmp_repo: Path): + changed = tmp_repo / "changed.txt" + write_file(changed, "tests/snippets/fixture-with-dep.task.sh\n") + plan = tmp_repo / "plan.txt" + + rc = run_cmd( + ["python3", "ci/select-doc-pages.py", + "--changed-file-list", str(changed), + "--out", str(plan), + "--repo-root", str(tmp_repo)], + cwd=tmp_repo, + ) + assert rc.returncode == 0, rc.stderr + lines = plan.read_text().strip().splitlines() + # Dep must appear first + assert lines == [ + "tests/snippets/base-fixture.task.sh", + "tests/snippets/fixture-with-dep.task.sh", + ] + +def test_dedup_when_both_changed(tmp_repo: Path): + changed = tmp_repo / "changed.txt" + write_file(changed, + "tests/snippets/base-fixture.task.sh\n" + "tests/snippets/fixture-with-dep.task.sh\n") + plan = tmp_repo / "plan.txt" + + rc = run_cmd( + ["python3", "ci/select-doc-pages.py", + "--changed-file-list", str(changed), + "--out", str(plan), + "--repo-root", str(tmp_repo)], + cwd=tmp_repo, + ) + assert rc.returncode == 0 + lines = plan.read_text().strip().splitlines() + assert lines == [ + "tests/snippets/base-fixture.task.sh", + "tests/snippets/fixture-with-dep.task.sh", + ] From 7a2fadc64d481a65dc3e8e133609f72a71143c78 Mon Sep 17 00:00:00 2001 From: Myles Penner Date: Tue, 21 Oct 2025 15:25:41 -0700 Subject: [PATCH 2/5] Fix syntax and address review comments --- .../workflows/docs-validation-unit-tests.yml | 2 - .github/workflows/docs-validation.yml | 42 +++---------------- ci/README.md | 15 ------- ci/run-doc-pages.py | 28 ++++++------- docs/Makefile | 9 +++- .../how-to/snippets/instance-recovery.task.sh | 2 +- docs/how-to/snippets/observability.task.sh | 2 +- .../snippets/resource-optimization.task.sh | 2 +- docs/how-to/snippets/secrets.task.sh | 2 +- docs/how-to/snippets/telemetry.task.sh | 2 +- docs/how-to/snippets/validation.task.sh | 2 +- tests/test_run_doc_pages.py | 9 ++-- 12 files changed, 37 insertions(+), 80 deletions(-) diff --git a/.github/workflows/docs-validation-unit-tests.yml b/.github/workflows/docs-validation-unit-tests.yml index 975ed85..bf914ad 100644 --- a/.github/workflows/docs-validation-unit-tests.yml +++ b/.github/workflows/docs-validation-unit-tests.yml @@ -2,11 +2,9 @@ name: Documentation Unit Tests on: push: - branches: [ main, docs-testing ] paths: - "ci/**" pull_request: - branches: [ main ] paths: - "ci/**" workflow_dispatch: diff --git a/.github/workflows/docs-validation.yml b/.github/workflows/docs-validation.yml index 5a5af78..cc241ad 100644 --- a/.github/workflows/docs-validation.yml +++ b/.github/workflows/docs-validation.yml @@ -2,15 +2,10 @@ name: Documentation Functional Testing on: push: - branches: - - main - - docs-testing paths: - "tutorial/**/*.task.sh" - "how-to/**/*.task.sh" pull_request: - branches: - - main paths: - "tutorial/**/*.task.sh" - "how-to/**/*.task.sh" @@ -29,7 +24,6 @@ defaults: shell: bash env: - CI: "true" PYTHONUNBUFFERED: "1" PIP_DISABLE_PIP_VERSION_CHECK: "1" # Dry-run switch. Set to "1" to print without executing. @@ -43,37 +37,13 @@ jobs: - name: Checkout uses: actions/checkout@v4 - - name: Discover snippet files - id: discover - run: | - find tutorial how-to -type f -name '*.task.sh' -print0 2>/dev/null > all_snippets.null - COUNT=$(tr -cd '\0' < all_snippets.null | wc -c) - echo "count=${COUNT}" >> "$GITHUB_OUTPUT" - - # Summary for GitHub output - { - echo "### Lint target discovery" - if [[ "$COUNT" -gt 0 ]]; then - echo "Found **$COUNT** snippet files to lint." - else - echo "_No snippet files found._" - fi - } >> "$GITHUB_STEP_SUMMARY" - - - name: Set up Python (for bashate) - if: steps.discover.outputs.count > 0 + - name: Set up Python uses: actions/setup-python@v5 with: - python-version: "3.13" + python-version: "3.12" - - name: Install bashate - if: steps.discover.outputs.count > 0 - run: python -m pip install --upgrade pip bashate - - - name: Lint with bashate (ignore long lines) - if: steps.discover.outputs.count > 0 - run: | - xargs -0 -r bashate -i E006 < all_snippets.null + - name: Lint shell snippets + run: make lint plan-and-run: name: Plan & run docs tests @@ -120,7 +90,7 @@ jobs: if: steps.write-changed.outputs.count > 0 uses: actions/setup-python@v5 with: - python-version: "3.13" + python-version: "3.12" - name: Build and display execution plan if: steps.write-changed.outputs.count > 0 @@ -156,7 +126,7 @@ jobs: if: env.DOCS_DRY_RUN == '1' && steps.plan.outputs.count > 0 run: | echo "Executing dry-run validation of the plan..." - python ci/run-doc-pages.py --plan plan.txt + python ci/run-doc-pages.py --plan plan.txt --dry-run - name: Execute plan if: env.DOCS_DRY_RUN != '1' && steps.plan.outputs.count > 0 diff --git a/ci/README.md b/ci/README.md index 4ae5562..8f3ce6c 100644 --- a/ci/README.md +++ b/ci/README.md @@ -107,21 +107,6 @@ If one snippet must run after another (e.g., a feature enable depends on sunbeam --- -## Running Locally - -```bash -# Build a plan from changed snippets -printf '%s\n' how-to/snippets/secrets.task.sh > changed_snippets.txt -python ci/select-doc-pages.py --changed-file-list changed_snippets.txt --out plan.txt - -# Preview what would run -DOCS_DRY_RUN=1 python ci/run-doc-pages.py --plan plan.txt - -# Actually execute (be careful!) -DOCS_DRY_RUN=0 python ci/run-doc-pages.py --plan plan.txt -``` - ---- ## CI Workflows diff --git a/ci/run-doc-pages.py b/ci/run-doc-pages.py index aae5ae7..4dcff83 100644 --- a/ci/run-doc-pages.py +++ b/ci/run-doc-pages.py @@ -4,13 +4,14 @@ Executor for docs snippets plan. This script extracts and runs shell commands found within special comment blocks (`[docs-exec:] ... [docs-exec:-end]`) -in scripts. If no such blocks are found in a script, the step fails. +in scripts. If no such blocks are found in a script, the step is skipped. Usage: + # For a real execution python ci/run-doc-pages.py --plan plan.txt -Environment: - DOCS_DRY_RUN=1 # Print the plan, show the extracted bash, and skip execution. + # To print the assembled scripts without running them + python ci/run-doc-pages.py --plan plan.txt --dry-run Plan format: newline-delimited file where each line is a path to a *.task.sh script. """ @@ -18,13 +19,15 @@ from __future__ import annotations import argparse -import os import re import sys from dataclasses import dataclass, field from pathlib import Path from subprocess import run # nosec B404 +EXEC_START = re.compile(r'^\s*#\s*\[docs-exec:([^\]]+)\]\s*$') +EXEC_END = r'^\s*#\s*\[docs-exec:%s-end\]\s*$' +HEADER = "#!/usr/bin/env bash\nset -euo pipefail\n" def parse_args() -> argparse.Namespace: p = argparse.ArgumentParser(description="Run docs snippet plan sequentially.") @@ -34,6 +37,11 @@ def parse_args() -> argparse.Namespace: type=Path, help="Path to a newline-delimited file of scripts to run (execution order).", ) + p.add_argument( + "--dry-run", + action="store_true", + help="Print the execution plan and assembled scripts without running them.", + ) return p.parse_args() @@ -50,11 +58,6 @@ def require_files_exist(paths: list[Path]) -> None: raise FileNotFoundError("Script(s) in plan not found:\n - " + "\n - ".join(missing)) -EXEC_START = re.compile(r'^\s*#\s*\[docs-exec:([^\]]+)\]\s*$') -EXEC_END = r'^\s*#\s*\[docs-exec:%s-end\]\s*$' -HEADER = "#!/usr/bin/env bash\nset -euo pipefail\n" - - @dataclass class ExtractedScript: text: str @@ -65,10 +68,7 @@ class ExtractedScript: def build_ci_text_from_exec_blocks(script_path: Path) -> ExtractedScript: """Parse script and concatenate all [docs-exec:]...[docs-exec:-end] blocks.""" - try: - src_lines = script_path.read_text(encoding="utf-8", errors="ignore").splitlines(True) - except OSError as e: - raise OSError(f"Failed to read script file {script_path}: {e}") from e + src_lines = script_path.read_text(encoding="utf-8", errors="ignore").splitlines(True) out_chunks: list[str] = [] names: list[str] = [] @@ -126,7 +126,7 @@ def main() -> int: return 0 require_files_exist(scripts) - dry_run = os.environ.get("DOCS_DRY_RUN") == "1" + dry_run = args.dry_run print("\n--- Documentation Test Execution ---") print(f"Total steps: {len(scripts)}\n") diff --git a/docs/Makefile b/docs/Makefile index d207fb9..e8d1656 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -34,6 +34,7 @@ help: @echo "* clean full environment: make clean" @echo "* check links: make linkcheck" @echo "* check markdown: make lint-md" + @echo "* check shell snippets: make lint-sh" @echo "* check spelling: make spelling" @echo "* check spelling (without building again): make spellcheck" @echo "* check inclusive language: make woke" @@ -47,7 +48,7 @@ help: .PHONY: help full-help html epub pdf linkcheck spelling spellcheck woke \ vale pa11y run serve install pa11y-install \ vale-install pdf-prep pdf-prep-force clean clean-doc \ - update lint-md + update lint-md lint-sh full-help: $(DOCS_VENVDIR) @. $(DOCS_VENV); $(SPHINX_BUILD) -M help "$(DOCS_SOURCEDIR)" "$(DOCS_BUILDDIR)" $(SPHINX_OPTS) $(O) @@ -129,6 +130,12 @@ lint-md: pymarkdownlnt-install echo "pymarkdownlnt exited with code $$status"; \ exit $$status; +lint-sh: install + @echo "Linting shell snippets with bashate..." + @. $(DOCS_VENV); pip show bashate > /dev/null || (echo "--> Installing bashate..."; pip install bashate) + @. $(DOCS_VENV); find tutorial how-to -type f -name '*.task.sh' -print0 2>/dev/null | xargs -0 -r bashate -i E006 + @echo "Linting finished successfully." + vale-install: install @. $(DOCS_VENV); test -f $(VALE_CONFIG) || python3 $(DEV_DIR)/get_vale_conf.py @echo '.Name=="Canonical.400-Enforce-inclusive-terms"' > $(DEV_DIR)/styles/woke.filter diff --git a/docs/how-to/snippets/instance-recovery.task.sh b/docs/how-to/snippets/instance-recovery.task.sh index 0f40858..4a593a3 100644 --- a/docs/how-to/snippets/instance-recovery.task.sh +++ b/docs/how-to/snippets/instance-recovery.task.sh @@ -15,4 +15,4 @@ sunbeam disable instance-recovery # [docs-exec:disable-instance-recovery] sg snap_daemon 'sunbeam disable instance-recovery' -# [docs-exec:disable-instance-recovery-end] \ No newline at end of file +# [docs-exec:disable-instance-recovery-end] diff --git a/docs/how-to/snippets/observability.task.sh b/docs/how-to/snippets/observability.task.sh index af5af13..65df7cf 100644 --- a/docs/how-to/snippets/observability.task.sh +++ b/docs/how-to/snippets/observability.task.sh @@ -30,4 +30,4 @@ sg snap_daemon 'sunbeam disable observability embedded' # [docs-view:observability-dashboard-url] sunbeam observability dashboard-url -# [docs-view:observability-dashboard-url-end] \ No newline at end of file +# [docs-view:observability-dashboard-url-end] diff --git a/docs/how-to/snippets/resource-optimization.task.sh b/docs/how-to/snippets/resource-optimization.task.sh index 3deac00..7889f0d 100644 --- a/docs/how-to/snippets/resource-optimization.task.sh +++ b/docs/how-to/snippets/resource-optimization.task.sh @@ -15,4 +15,4 @@ sunbeam disable resource-optimization # [docs-exec:disable-resource-optimization] sg snap_daemon 'sunbeam disable resource-optimization' -# [docs-exec:disable-resource-optimization-end] \ No newline at end of file +# [docs-exec:disable-resource-optimization-end] diff --git a/docs/how-to/snippets/secrets.task.sh b/docs/how-to/snippets/secrets.task.sh index b6650bc..9501dd6 100644 --- a/docs/how-to/snippets/secrets.task.sh +++ b/docs/how-to/snippets/secrets.task.sh @@ -15,4 +15,4 @@ sunbeam disable secrets # [docs-exec:disable-secrets] sg snap_daemon 'sunbeam disable secrets' -# [docs-exec:disable-secrets-end] \ No newline at end of file +# [docs-exec:disable-secrets-end] diff --git a/docs/how-to/snippets/telemetry.task.sh b/docs/how-to/snippets/telemetry.task.sh index 1daeb23..0b9a74e 100644 --- a/docs/how-to/snippets/telemetry.task.sh +++ b/docs/how-to/snippets/telemetry.task.sh @@ -15,4 +15,4 @@ sunbeam disable telemetry # [docs-exec:disable-telemetry] sg snap_daemon 'sunbeam disable telemetry' -# [docs-exec:disable-telemetry-end] \ No newline at end of file +# [docs-exec:disable-telemetry-end] diff --git a/docs/how-to/snippets/validation.task.sh b/docs/how-to/snippets/validation.task.sh index 68040af..be12739 100644 --- a/docs/how-to/snippets/validation.task.sh +++ b/docs/how-to/snippets/validation.task.sh @@ -15,4 +15,4 @@ sunbeam disable validation # [docs-exec:disable-validation] sg snap_daemon 'sunbeam disable validation' -# [docs-exec:disable-validation-end] \ No newline at end of file +# [docs-exec:disable-validation-end] diff --git a/tests/test_run_doc_pages.py b/tests/test_run_doc_pages.py index b0fca78..34cf46f 100644 --- a/tests/test_run_doc_pages.py +++ b/tests/test_run_doc_pages.py @@ -12,9 +12,8 @@ def test_dry_run_prints_assembled_scripts(tmp_repo: Path): "tests/snippets/fixture-multi-blocks.task.sh\n") rc = run_cmd( - ["python3", "ci/run-doc-pages.py", "--plan", str(plan)], + ["python3", "ci/run-doc-pages.py", "--plan", str(plan), "--dry-run"], cwd=tmp_repo, - env={"DOCS_DRY_RUN": "1"}, # Explicitly set dry-run mode ) assert rc.returncode == 0, rc.stderr @@ -37,9 +36,8 @@ def test_dry_run_fails_on_structural_error(tmp_repo: Path): write_file(plan, "tests/snippets/fixture-missing-end.task.sh\n") rc = run_cmd( - ["python3", "ci/run-doc-pages.py", "--plan", str(plan)], + ["python3", "ci/run-doc-pages.py", "--plan", str(plan), "--dry-run"], cwd=tmp_repo, - env={"DOCS_DRY_RUN": "1"}, # Explicitly set dry-run mode ) # Structural errors should cause a failure with exit code 1. @@ -58,9 +56,8 @@ def test_dry_run_handles_no_blocks_gracefully(tmp_repo: Path): write_file(plan, "tests/snippets/fixture-no-blocks.task.sh\n") rc = run_cmd( - ["python3", "ci/run-doc-pages.py", "--plan", str(plan)], + ["python3", "ci/run-doc-pages.py", "--plan", str(plan), "--dry-run"], cwd=tmp_repo, - env={"DOCS_DRY_RUN": "1"}, # Explicitly set dry-run mode ) # A dry run should always succeed if there are no structural errors. From 5e9914f6c898ea93cb1d93e1ba9eee0beedb3ad1 Mon Sep 17 00:00:00 2001 From: Myles Penner Date: Wed, 22 Oct 2025 15:19:35 -0700 Subject: [PATCH 3/5] Always print plan regardless of dry-run mode --- ci/run-doc-pages.py | 22 ++++++++++++---------- tests/test_run_doc_pages.py | 27 ++++++++++++++++++++++++++- 2 files changed, 38 insertions(+), 11 deletions(-) diff --git a/ci/run-doc-pages.py b/ci/run-doc-pages.py index 4dcff83..0c8ea5e 100644 --- a/ci/run-doc-pages.py +++ b/ci/run-doc-pages.py @@ -149,19 +149,21 @@ def main() -> int: print("[run-doc-pages] Aborting due to structural errors in docs-exec blocks.", file=sys.stderr) return 1 - # Phase 2: Process the validated plan - if dry_run: - print("[DRY RUN] Printing all assembled scripts (no execution):\n") - for i, (script, ex) in enumerate(extracted, start=1): - print(f"----- BEGIN SCRIPT [{i}] {script} -----") - if ex.used_exec_blocks: - sys.stdout.write(ex.text) - else: - print("# (no [docs-exec:*] blocks found - this would be skipped in a real run)") - print(f"----- END SCRIPT [{i}] {script} -----\n") + # Phase 2: Always print assembled scripts for visibility + print("\n[PLAN PREVIEW] Printing all assembled scripts:\n") + for i, (script, ex) in enumerate(extracted, start=1): + print(f"----- BEGIN SCRIPT [{i}] {script} -----") + if ex.used_exec_blocks: + sys.stdout.write(ex.text) + else: + print("# (no [docs-exec:*] blocks found - will be skipped)") + print(f"----- END SCRIPT [{i}] {script} -----\n") + # Phase 3: Execute if not in dry-run mode + if dry_run: print("[DRY RUN] Completed printing scripts. Nothing executed.") else: + print("\n[EXECUTION] Starting real execution of plan...\n") for i, (script, ex) in enumerate(extracted, start=1): print(f"==> [Step {i}/{len(scripts)}] {script}") diff --git a/tests/test_run_doc_pages.py b/tests/test_run_doc_pages.py index 34cf46f..75ca60d 100644 --- a/tests/test_run_doc_pages.py +++ b/tests/test_run_doc_pages.py @@ -65,4 +65,29 @@ def test_dry_run_handles_no_blocks_gracefully(tmp_repo: Path): # It should print the correct output for the file with no blocks. assert "NO docs-exec FOUND" in rc.stdout - assert "(no [docs-exec:*] blocks found - this would be skipped in a real run)" in rc.stdout \ No newline at end of file + assert "(no [docs-exec:*] blocks found - will be skipped)" in rc.stdout + +def test_plan_preview_always_shown(tmp_repo: Path): + """ + Checks that the plan preview is shown in both dry-run and execution modes. + """ + plan = tmp_repo / "plan.txt" + write_file(plan, "tests/snippets/base-fixture.task.sh\n") + + # Test dry-run mode + rc_dry = run_cmd( + ["python3", "ci/run-doc-pages.py", "--plan", str(plan), "--dry-run"], + cwd=tmp_repo, + ) + assert rc_dry.returncode == 0, rc_dry.stderr + assert "[PLAN PREVIEW]" in rc_dry.stdout + assert "BEGIN SCRIPT" in rc_dry.stdout + assert "install base" in rc_dry.stdout + assert "[DRY RUN]" in rc_dry.stdout + assert "Nothing executed" in rc_dry.stdout + + # Test execution mode (without actually executing by checking output format) + # Note: We're testing dry-run here as a proxy since we don't want to + # actually execute scripts in unit tests, but we verify the structure + # is correct by checking that execution mode would show [EXECUTION] + assert "[EXECUTION]" not in rc_dry.stdout # Should not appear in dry-run mode \ No newline at end of file From 42169fb3d8210587b99f78f9208b0af580dda130 Mon Sep 17 00:00:00 2001 From: Myles Penner Date: Thu, 23 Oct 2025 06:55:50 -0700 Subject: [PATCH 4/5] Update README.md --- ci/README.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/ci/README.md b/ci/README.md index 8f3ce6c..ff6ba53 100644 --- a/ci/README.md +++ b/ci/README.md @@ -1,7 +1,3 @@ ---- -orphan: true ---- - # Documentation Testing Framework ## Purpose From c048674b5b5aa8b54062a83a52ac8836df07af25 Mon Sep 17 00:00:00 2001 From: Heather Lanigan Date: Tue, 18 Aug 2026 15:01:25 -0400 Subject: [PATCH 5/5] fix: update rockcraft link per link check --- docs/explanation/architecture.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/explanation/architecture.rst b/docs/explanation/architecture.rst index e363f87..0fc7b7a 100644 --- a/docs/explanation/architecture.rst +++ b/docs/explanation/architecture.rst @@ -50,7 +50,7 @@ Canonical OpenStack is distributed in the form of three types of artifacts. Rocks ^^^^^ -`Rocks `_ are `OCI-compliant container images `_ that are built to be secure and stable by design. Canonical OpenStack uses Rocks for the purpose of hosting OpenStack's control plane services as well as some other software components (i.e. the observability stack). +`Rocks `_ are `OCI-compliant container images `_ that are built to be secure and stable by design. Canonical OpenStack uses Rocks for the purpose of hosting OpenStack's control plane services as well as some other software components (i.e. the observability stack). Snaps ^^^^^