From 94d3d4358d9ed1ca81771defc56cfd71016304d3 Mon Sep 17 00:00:00 2001 From: dayland Date: Fri, 24 Jul 2026 10:03:29 +0100 Subject: [PATCH 1/2] Authoring-assist: promote to first-class tool with non-blocking PR advisory MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Promotes the authoring-assist prototype to a first-class BCQuality feature that reviews knowledge-article front-matter and proposes missing routing `signals:` (raise triggers and `effect: suppress` suppressors). The suggestion is ADVISORY ONLY: it never fails a build; authors apply it via a normal PR under existing R29 + CI + CODEOWNERS review. Feature: - tools/Suggest-ArticleSignals.ps1: suggestion engine hardened with a stable JSON contract (schemaVersion/toolVersion) and a deterministic, effect-sensitive per-proposal suggestionId, plus -ChangedFiles scoping for PR-diff runs. Runs self-contained: the routing seed is now OPTIONAL (built-in domain map), so this does not depend on the separate routing-index tuning thread. - tools/New-AuthoringAssistComment.ps1: renders one upsertable, feedback-instrumented advisory comment (stable anchor + aa:meta/aa:article markers carrying suggestionIds + reaction footer) for the downstream acceptance-measurement work. - .github/workflows/authoring-assist*.yml: unprivileged pull_request intake + trusted workflow_run runner (review/publish split) + in-repo self-test. - tools/Test-SuggestArticleSignals.ps1: 23-check smoke test (contract, determinism, scoping, suppressor/prohibition, renderer markers, seed-free graceful degradation). Dormant schema support: - .github/scripts/validate_frontmatter.py: adds R29, validating the optional `signals` block shape (bare token or mapping with token + optional pattern/domain/effect in {raise,suppress}). Reserved & dormant โ€” no production pipeline consumes it; the existing PR Reviewer only reads `effect: suppress` behind its BCQ_INDEX_V2 flag and falls back safely when absent. Existing rules (incl. R24 skill-id uniqueness, R27, R28) are unchanged. Excludes the routing-index tuning thread entirely (separate work). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 75d852d6-18a6-4c58-986f-ed5ab16618fa --- .github/scripts/validate_frontmatter.py | 36 +- .github/workflows/authoring-assist-runner.yml | 242 +++++++++ .../workflows/authoring-assist-selftest.yml | 33 ++ .github/workflows/authoring-assist.yml | 50 ++ tools/New-AuthoringAssistComment.ps1 | 168 ++++++ tools/Suggest-ArticleSignals.ps1 | 500 ++++++++++++++++++ tools/Test-SuggestArticleSignals.ps1 | 133 +++++ tools/authoring-assist.md | 152 ++++++ 8 files changed, 1313 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/authoring-assist-runner.yml create mode 100644 .github/workflows/authoring-assist-selftest.yml create mode 100644 .github/workflows/authoring-assist.yml create mode 100644 tools/New-AuthoringAssistComment.ps1 create mode 100644 tools/Suggest-ArticleSignals.ps1 create mode 100644 tools/Test-SuggestArticleSignals.ps1 create mode 100644 tools/authoring-assist.md diff --git a/.github/scripts/validate_frontmatter.py b/.github/scripts/validate_frontmatter.py index dd3ef4e..701ceea 100644 --- a/.github/scripts/validate_frontmatter.py +++ b/.github/scripts/validate_frontmatter.py @@ -34,6 +34,11 @@ "bc-version", "domain", "keywords", "technologies", "countries", "application-area", } +# Optional knowledge keys (do not trigger the R02 closed-key-set error). +# `signals`: OPTIONAL routing-signal declarations. RESERVED & DORMANT โ€” the shape +# is validated (R29 below) so authors and the authoring-assist advisory can +# populate it, but no production pipeline consumes it yet. See tools/authoring-assist.md. +KNOWLEDGE_OPTIONAL_KEYS = {"signals"} ACTION_SKILL_REQUIRED_KEYS = { "kind", "id", "version", "title", "description", "inputs", "outputs", } @@ -203,7 +208,7 @@ def validate_knowledge(path: Path, parsed: Parsed, report: Report) -> None: # R02 required keys, no extras, none empty missing = KNOWLEDGE_REQUIRED_KEYS - fm.keys() - extras = fm.keys() - KNOWLEDGE_REQUIRED_KEYS + extras = fm.keys() - KNOWLEDGE_REQUIRED_KEYS - KNOWLEDGE_OPTIONAL_KEYS if missing: report.error(path, "R02", f"missing required frontmatter keys: {sorted(missing)}", 1) if extras: @@ -213,6 +218,35 @@ def validate_knowledge(path: Path, parsed: Parsed, report: Report) -> None: if v is None or v == "" or v == []: report.error(path, "R02", f"frontmatter key '{k}' must not be empty", 1) + # R29 optional routing `signals` block. Each entry is either a bare token + # string or a mapping with a required 'token' and optional 'pattern'/'domain'/ + # 'effect' (all non-empty strings; effect in {raise, suppress}). Reserved & + # dormant: validated for shape only so the block stays well-formed for the + # authoring-assist advisory; no production pipeline consumes it yet. + if "signals" in fm: + sigs = fm["signals"] + if not isinstance(sigs, list) or not sigs: + report.error(path, "R29", "signals must be a non-empty list", 1) + else: + for entry in sigs: + if isinstance(entry, str): + if not entry.strip(): + report.error(path, "R29", "signals token string must not be empty", 1) + elif isinstance(entry, dict): + tok = entry.get("token") + if not isinstance(tok, str) or not tok.strip(): + report.error(path, "R29", "signals entry must have a non-empty 'token'", 1) + for opt in ("pattern", "domain"): + if opt in entry and (not isinstance(entry[opt], str) or not entry[opt].strip()): + report.error(path, "R29", f"signals '{opt}' must be a non-empty string", 1) + if "effect" in entry and entry["effect"] not in ("raise", "suppress"): + report.error(path, "R29", "signals 'effect' must be 'raise' or 'suppress'", 1) + unknown = set(entry.keys()) - {"token", "pattern", "domain", "effect"} + if unknown: + report.error(path, "R29", f"signals entry has unknown keys: {sorted(unknown)}", 1) + else: + report.error(path, "R29", "signals entry must be a string or a mapping", 1) + # R03 bc-version if "bc-version" in fm: _, err = expand_bc_version(fm["bc-version"]) diff --git a/.github/workflows/authoring-assist-runner.yml b/.github/workflows/authoring-assist-runner.yml new file mode 100644 index 0000000..6fc1d54 --- /dev/null +++ b/.github/workflows/authoring-assist-runner.yml @@ -0,0 +1,242 @@ +name: Authoring-assist advisory runner + +# Privileged companion to `Authoring-assist advisory`. Runs in the trusted +# workflow_run context so it can post a comment on fork PRs. Two jobs: +# review - read-only; resolves the PR, reads the PR-head article content, runs +# the (trusted-base) suggestion tool + renderer, uploads the comment. +# publish - holds pull-requests:write; upserts the single advisory comment. +# The suggestion tool is a deterministic PowerShell script that only PARSES +# article text (it never executes AL or PR content), and it runs from the trusted +# base checkout with the trusted routing seed, so only inert article text comes +# from the PR head. The advisory is NON-BLOCKING and is never a required check. + +on: + workflow_run: + workflows: + - Authoring-assist advisory + types: + - completed + +concurrency: + group: authoring-assist-runner-${{ github.event.workflow_run.id }} + cancel-in-progress: true + +permissions: {} + +jobs: + review: + if: github.event.workflow_run.event == 'pull_request' && github.event.workflow_run.conclusion == 'success' + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + pull-requests: read + defaults: + run: + shell: pwsh + outputs: + pr_number: ${{ steps.pr.outputs.number }} + head_sha: ${{ steps.pr.outputs.head_sha }} + has_report: ${{ steps.run.outputs.has_report }} + steps: + - name: Checkout trusted base tools + uses: actions/checkout@v4 + with: + ref: refs/heads/main + path: trusted + fetch-depth: 1 + # The tool reads untrusted PR-head article text; keep no token in + # .git/config. Public content is fetched unauthenticated below. + persist-credentials: false + + - name: Resolve PR details + id: pr + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + $headers = @{ + Accept = 'application/vnd.github+json' + Authorization = "Bearer $env:GITHUB_TOKEN" + 'User-Agent' = 'bcquality-authoring-assist' + } + $expectedHeadSha = '${{ github.event.workflow_run.head_sha }}' + $prNumber = '${{ github.event.workflow_run.pull_requests[0].number }}' + + if (-not $prNumber) { + $headOwner = '${{ github.event.workflow_run.head_repository.owner.login }}' + $headBranch = '${{ github.event.workflow_run.head_branch }}' + if (-not $headOwner -or -not $headBranch) { + throw 'No PR number in workflow_run payload and insufficient head metadata for fallback.' + } + $encodedHead = [System.Uri]::EscapeDataString("${headOwner}:$headBranch") + $searchUri = "https://api.github.com/repos/${{ github.repository }}/pulls?state=open&head=$encodedHead&per_page=30" + $candidates = Invoke-RestMethod -Uri $searchUri -Headers $headers -Method GET + $matching = @($candidates | Where-Object { $_.head.sha -eq $expectedHeadSha }) + if ($matching.Count -eq 1) { $prNumber = [string]$matching[0].number } + elseif ($matching.Count -gt 1) { throw 'Fallback PR lookup returned multiple matches.' } + } + if (-not $prNumber) { throw 'No pull request number resolved.' } + + $pr = Invoke-RestMethod -Uri "https://api.github.com/repos/${{ github.repository }}/pulls/$prNumber" -Headers $headers -Method GET + if ($expectedHeadSha -and $pr.head.sha -ne $expectedHeadSha) { + throw 'Resolved PR head SHA does not match workflow_run head SHA.' + } + + "number=$($pr.number)" >> $env:GITHUB_OUTPUT + "head_sha=$($pr.head.sha)" >> $env:GITHUB_OUTPUT + "base_ref=$($pr.base.ref)" >> $env:GITHUB_OUTPUT + + - name: List changed knowledge articles + id: changed + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + $headers = @{ + Accept = 'application/vnd.github+json' + Authorization = "Bearer $env:GITHUB_TOKEN" + 'User-Agent' = 'bcquality-authoring-assist' + } + $n = '${{ steps.pr.outputs.number }}' + $changed = [System.Collections.Generic.List[string]]::new() + for ($page = 1; $page -le 30; $page++) { + $uri = "https://api.github.com/repos/${{ github.repository }}/pulls/$n/files?per_page=100&page=$page" + $files = Invoke-RestMethod -Uri $uri -Headers $headers -Method GET + if (-not $files -or @($files).Count -eq 0) { break } + foreach ($f in $files) { + if ($f.status -in @('added','modified','renamed','changed') ` + -and $f.filename -match '(^|/)knowledge/' ` + -and $f.filename -match '\.md$') { + $changed.Add($f.filename) | Out-Null + } + } + if (@($files).Count -lt 100) { break } + } + $outFile = Join-Path $env:GITHUB_WORKSPACE 'changed-articles.txt' + ($changed | Sort-Object -Unique) -join "`n" | Set-Content -LiteralPath $outFile -Encoding UTF8 + Write-Host "Changed knowledge articles: $($changed.Count)" + "count=$($changed.Count)" >> $env:GITHUB_OUTPUT + + - name: Fetch PR head content + if: steps.changed.outputs.count != '0' + run: | + $root = Join-Path $env:GITHUB_WORKSPACE 'head' + New-Item -ItemType Directory -Force -Path $root | Out-Null + git -C $root init -q + git -C $root remote add origin "https://github.com/${{ github.repository }}.git" + # PR head commits (incl. forks) are replicated under refs/pull/N/head. + git -C $root fetch --depth=1 origin "refs/pull/${{ steps.pr.outputs.number }}/head" + if ($LASTEXITCODE -ne 0) { throw "git fetch of PR head failed (exit $LASTEXITCODE)" } + git -C $root checkout -q FETCH_HEAD + + - name: Generate suggestions and render comment + id: run + if: steps.changed.outputs.count != '0' + env: + AUTHORING_ASSIST_DOC_URL: ${{ vars.AUTHORING_ASSIST_DOC_URL }} + run: | + $out = Join-Path $env:GITHUB_WORKSPACE 'advisory-output' + New-Item -ItemType Directory -Force -Path $out | Out-Null + + $changed = @(Get-Content -LiteralPath (Join-Path $env:GITHUB_WORKSPACE 'changed-articles.txt') | + Where-Object { $_ -and $_.Trim() }) + + $tool = Join-Path $env:GITHUB_WORKSPACE 'trusted/tools/Suggest-ArticleSignals.ps1' + $renderer = Join-Path $env:GITHUB_WORKSPACE 'trusted/tools/New-AuthoringAssistComment.ps1' + # Routing seed is optional (owned by the routing-index tuning thread). + # Pass it through only if that thread has landed one in trusted/tools; + # otherwise the tool runs self-contained. + $seed = Join-Path $env:GITHUB_WORKSPACE 'trusted/tools/routing-seed.json' + $headRoot = Join-Path $env:GITHUB_WORKSPACE 'head' + $reportPath = Join-Path $out 'report.json' + $commentPath = Join-Path $out 'comment.md' + + # Trusted logic over head article text; enrich with the trusted seed + # only when present. Only inert article text comes from head. + $seedArgs = if (Test-Path -LiteralPath $seed) { @('-SeedPath', $seed) } else { @() } + & $tool -BCQualityRoot $headRoot @seedArgs -ChangedFiles $changed -AsJson | + Set-Content -LiteralPath $reportPath -Encoding UTF8 + + $repoUrl = "https://github.com/${{ github.repository }}" + & $renderer -JsonPath $reportPath ` + -RepoUrl $repoUrl ` + -Ref '${{ steps.pr.outputs.head_sha }}' ` + -DocUrl $env:AUTHORING_ASSIST_DOC_URL | + Set-Content -LiteralPath $commentPath -Encoding UTF8 + + $report = Get-Content -LiteralPath $reportPath -Raw | ConvertFrom-Json + "article_count=$($report.articleCount)" >> $env:GITHUB_OUTPUT + "has_report=true" >> $env:GITHUB_OUTPUT + Write-Host "Articles with suggestions: $($report.articleCount)" + + - name: Upload advisory output + if: steps.run.outputs.has_report == 'true' + uses: actions/upload-artifact@v4 + with: + name: authoring-assist-output-${{ steps.pr.outputs.number }} + path: ${{ github.workspace }}/advisory-output + if-no-files-found: warn + + publish: + needs: review + if: ${{ needs.review.result == 'success' && needs.review.outputs.has_report == 'true' && needs.review.outputs.pr_number != '' }} + runs-on: ubuntu-latest + permissions: + pull-requests: write + issues: write + defaults: + run: + shell: pwsh + steps: + - name: Download advisory output + uses: actions/download-artifact@v4 + with: + name: authoring-assist-output-${{ needs.review.outputs.pr_number }} + path: ${{ github.workspace }}/advisory-output + github-token: ${{ github.token }} + run-id: ${{ github.event.workflow_run.id }} + + - name: Upsert advisory comment + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + $anchor = '' + $body = Get-Content -LiteralPath (Join-Path $env:GITHUB_WORKSPACE 'advisory-output/comment.md') -Raw + $report = Get-Content -LiteralPath (Join-Path $env:GITHUB_WORKSPACE 'advisory-output/report.json') -Raw | ConvertFrom-Json + + $headers = @{ + Accept = 'application/vnd.github+json' + Authorization = "Bearer $env:GITHUB_TOKEN" + 'User-Agent' = 'bcquality-authoring-assist' + } + $repo = '${{ github.repository }}' + $n = '${{ needs.review.outputs.pr_number }}' + + # Find an existing advisory comment (idempotent update-in-place). + $existing = $null + for ($page = 1; $page -le 20; $page++) { + $uri = "https://api.github.com/repos/$repo/issues/$n/comments?per_page=100&page=$page" + $comments = Invoke-RestMethod -Uri $uri -Headers $headers -Method GET + if (-not $comments -or @($comments).Count -eq 0) { break } + $hit = $comments | Where-Object { $_.body -and $_.body.Contains($anchor) } | Select-Object -First 1 + if ($hit) { $existing = $hit; break } + if (@($comments).Count -lt 100) { break } + } + + # Do not create a brand-new "no suggestions" comment; only update an + # existing one to reflect that state. + if (-not $existing -and [int]$report.articleCount -eq 0) { + Write-Host 'No suggestions and no existing advisory comment; nothing to post.' + return + } + + $payload = @{ body = $body } | ConvertTo-Json -Depth 4 + $bytes = [System.Text.Encoding]::UTF8.GetBytes($payload) + if ($existing) { + $uri = "https://api.github.com/repos/$repo/issues/comments/$($existing.id)" + Invoke-RestMethod -Uri $uri -Headers $headers -Method PATCH -Body $bytes -ContentType 'application/json' | Out-Null + Write-Host "Updated advisory comment $($existing.id)." + } else { + $uri = "https://api.github.com/repos/$repo/issues/$n/comments" + Invoke-RestMethod -Uri $uri -Headers $headers -Method POST -Body $bytes -ContentType 'application/json' | Out-Null + Write-Host 'Created advisory comment.' + } diff --git a/.github/workflows/authoring-assist-selftest.yml b/.github/workflows/authoring-assist-selftest.yml new file mode 100644 index 0000000..44382af --- /dev/null +++ b/.github/workflows/authoring-assist-selftest.yml @@ -0,0 +1,33 @@ +name: Authoring-assist self-test + +# Regression-guards the authoring-assist feature (suggestion engine + JSON +# contract + comment renderer) against this repo's own corpus. Runs on changes +# to the tool, renderer, the optional routing seed, or the test itself. + +on: + pull_request: + branches: [main] + paths: + - 'tools/Suggest-ArticleSignals.ps1' + - 'tools/New-AuthoringAssistComment.ps1' + - 'tools/Test-SuggestArticleSignals.ps1' + - 'tools/routing-seed.json' + - '.github/workflows/authoring-assist*.yml' + push: + branches: [main] + paths: + - 'tools/Suggest-ArticleSignals.ps1' + - 'tools/New-AuthoringAssistComment.ps1' + - 'tools/Test-SuggestArticleSignals.ps1' + - 'tools/routing-seed.json' + +jobs: + selftest: + runs-on: ubuntu-latest + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Run authoring-assist smoke test + shell: pwsh + run: ./tools/Test-SuggestArticleSignals.ps1 -BCQualityRoot . diff --git a/.github/workflows/authoring-assist.yml b/.github/workflows/authoring-assist.yml new file mode 100644 index 0000000..6ad01a2 --- /dev/null +++ b/.github/workflows/authoring-assist.yml @@ -0,0 +1,50 @@ +name: Authoring-assist advisory + +# Unprivileged intake. Fires on PRs that add/modify knowledge articles and +# captures PR metadata as an artifact. All privileged work (reading the PR head, +# generating suggestions, posting the advisory comment) happens in the companion +# `Authoring-assist advisory runner` workflow, which runs in the trusted +# `workflow_run` context. This split lets the advisory comment on fork PRs +# without ever exposing a write-scoped token to a job that reads untrusted PR +# content. The advisory is NON-BLOCKING: it only suggests, never fails a check. + +on: + pull_request: + branches: [main] + types: [opened, reopened, synchronize, ready_for_review] + paths: + - '**/knowledge/**/*.md' + +concurrency: + group: authoring-assist-${{ github.event.pull_request.number }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + intake: + if: github.event.pull_request.draft == false + runs-on: ubuntu-latest + defaults: + run: + shell: pwsh + steps: + - name: Save PR metadata + run: | + $outputDir = Join-Path $env:GITHUB_WORKSPACE 'advisory-input' + New-Item -Path $outputDir -ItemType Directory -Force | Out-Null + + @{ + prNumber = "${{ github.event.pull_request.number }}" + headSha = "${{ github.event.pull_request.head.sha }}" + baseRef = "${{ github.event.pull_request.base.ref }}" + repository = "${{ github.repository }}" + } | ConvertTo-Json -Depth 5 | Set-Content -Path (Join-Path $outputDir 'pr-metadata.json') -Encoding UTF8 + + - name: Upload advisory input + uses: actions/upload-artifact@v4 + with: + name: authoring-assist-input-${{ github.event.pull_request.number }} + path: ${{ github.workspace }}/advisory-input + if-no-files-found: error diff --git a/tools/New-AuthoringAssistComment.ps1 b/tools/New-AuthoringAssistComment.ps1 new file mode 100644 index 0000000..a29d88f --- /dev/null +++ b/tools/New-AuthoringAssistComment.ps1 @@ -0,0 +1,168 @@ +<# +.SYNOPSIS + Renders the JSON output of Suggest-ArticleSignals.ps1 into a single, + non-blocking Markdown PR advisory comment with embedded feedback metadata. + +.DESCRIPTION + The automated authoring-assist advisory posts ONE consolidated comment per + PR (update-in-place) built from the per-article suggestions. Each article + section carries HTML metadata markers so downstream feedback tooling (the + BC-ALAgentsInternal harvester) can correlate reactions and applied/not-applied + outcomes back to a specific suggestion via its stable `suggestionId`. + + The comment is ADVISORY ONLY: it never fails a build, it only suggests + front-matter an author eyeballs and applies via a normal PR. R29 + (validate_frontmatter.py), CI, and CODEOWNERS remain the gate. + + Markers emitted (stable contract โ€” do not change without bumping the tool + schema and updating the harvester): + + top anchor (find/update) + + + +.PARAMETER JsonPath + Path to a file containing the `Suggest-ArticleSignals.ps1 -AsJson` output. + If omitted, JSON is read from stdin. + +.PARAMETER RepoUrl + Base repo URL used to build article permalinks (e.g. + https://github.com/microsoft/BCQuality). Optional; when omitted, article + paths are rendered as inline code without links. + +.PARAMETER Ref + Git ref/SHA the suggestions were computed against, used in permalinks and + surfaced in the footer for provenance. Optional. + +.PARAMETER DocUrl + URL surfaced in the feedback footer ("reply with why"). Optional. + +.OUTPUTS + Writes the Markdown comment body to stdout. +#> +[CmdletBinding()] +param( + [string] $JsonPath, + [string] $RepoUrl, + [string] $Ref, + [string] $DocUrl +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +$Anchor = '' + +if ($JsonPath) { + if (-not (Test-Path -LiteralPath $JsonPath)) { throw "JSON file not found: $JsonPath" } + $raw = Get-Content -LiteralPath $JsonPath -Raw +} else { + $raw = [Console]::In.ReadToEnd() +} +if ([string]::IsNullOrWhiteSpace($raw)) { throw 'No JSON input provided (empty file/stdin).' } + +$data = $raw | ConvertFrom-Json + +$reports = @() +if ($data.PSObject.Properties.Name -contains 'reports' -and $data.reports) { + $reports = @($data.reports) +} + +function Format-Provenance { + param([object] $Proposal) + $tags = [System.Collections.Generic.List[string]]::new() + if ($Proposal.keywordBacked) { $tags.Add('keyword-backed') | Out-Null } + if ($Proposal.inBad) { $tags.Add('anti-pattern sample') | Out-Null } + if ($Proposal.proseOnly) { $tags.Add('prose-only') | Out-Null } + if ($tags.Count -eq 0) { $tags.Add('sample') | Out-Null } + return ($tags -join ', ') +} + +function Format-SignalsBlock { + param([object] $Report) + $lines = [System.Collections.Generic.List[string]]::new() + $lines.Add('signals:') | Out-Null + foreach ($p in $Report.proposals) { + if ($Report.effect -eq 'suppress') { + $lines.Add(" - token: $($p.token)") | Out-Null + $lines.Add(' effect: suppress') | Out-Null + } else { + $lines.Add(" - $($p.token)") | Out-Null + } + } + return ($lines -join "`n") +} + +$body = [System.Collections.Generic.List[string]]::new() +$body.Add($Anchor) | Out-Null + +$schema = if ($data.PSObject.Properties.Name -contains 'schemaVersion') { $data.schemaVersion } else { 'unknown' } +$toolVer = if ($data.PSObject.Properties.Name -contains 'toolVersion') { $data.toolVersion } else { 'unknown' } +$generated = if ($data.PSObject.Properties.Name -contains 'generatedAt') { $data.generatedAt } else { '' } + +$body.Add(('' -f $schema, $toolVer, $generated, @($reports).Count)) | Out-Null +$body.Add('') | Out-Null +$body.Add('## ๐Ÿงญ Authoring-assist advisory') | Out-Null +$body.Add('') | Out-Null + +if (@($reports).Count -eq 0) { + $body.Add('No routing head-matter suggestions for the knowledge articles changed in this PR. โœ…') | Out-Null + $body.Add('') | Out-Null + $body.Add('Advisory only โ€” never blocks a merge. Generated by `tools/Suggest-ArticleSignals.ps1`.') | Out-Null + return ($body -join "`n") +} + +$body.Add(("This PR changes knowledge articles whose routing head-matter could be improved. " + + "These are **non-blocking suggestions** โ€” paste the proposed ``signals:`` block into the " + + "article's front-matter if it looks right. R29 + CODEOWNERS still review the change.")) | Out-Null +$body.Add('') | Out-Null + +foreach ($r in ($reports | Sort-Object domain, path)) { + $effect = $r.effect + $badge = if ($r.suppressor) { '๐Ÿ›ก๏ธ suppressor' } else { '๐ŸŽฏ trigger' } + + $suggestionIds = @($r.proposals | ForEach-Object { $_.suggestionId }) -join ',' + $body.Add(('' -f ` + $r.path, $r.domain, $effect, ([string]$r.suppressor).ToLowerInvariant(), $suggestionIds)) | Out-Null + + $title = if ($RepoUrl -and $Ref) { "[``$($r.path)``]($RepoUrl/blob/$Ref/$($r.path))" } else { "``$($r.path)``" } + $body.Add("### $title") | Out-Null + $body.Add("**Domain:** ``$($r.domain)``  ยท  $badge  ยท  ``effect: $effect``") | Out-Null + $body.Add('') | Out-Null + + if (@($r.proposals).Count -gt 0) { + $body.Add('Proposed front-matter:') | Out-Null + $body.Add('') | Out-Null + $body.Add('```yaml') | Out-Null + $body.Add((Format-SignalsBlock -Report $r)) | Out-Null + $body.Add('```') | Out-Null + $body.Add('') | Out-Null + foreach ($p in $r.proposals) { + $body.Add(("- ``$($p.token)`` โ€” {0} ยท {1}" -f (Format-Provenance -Proposal $p), $p.note)) | Out-Null + } + $body.Add('') | Out-Null + } + + if ($r.mismatch) { + $body.Add("> โš ๏ธ **Domain check:** $($r.mismatch)") | Out-Null + $body.Add('') | Out-Null + } + if ($r.applicability) { + $body.Add("> โš ๏ธ **Applicability:** $($r.applicability)") | Out-Null + $body.Add('') | Out-Null + } + if (@($r.keywordSuggestions).Count -gt 0) { + $kw = @($r.keywordSuggestions | ForEach-Object { '`' + $_ + '`' }) -join ', ' + $body.Add("> ๐Ÿ’ก **Keywords:** consider adding $kw") | Out-Null + $body.Add('') | Out-Null + } +} + +$replyLink = if ($DocUrl) { " - reply with why" } else { '' } +$provenance = if ($Ref) { " against ``$Ref``" } else { '' } +$body.Add('---') | Out-Null +$body.Add("๐Ÿ‘ useful ยท โค๏ธ especially valuable ยท ๐Ÿ‘Ž wrong$replyLink") | Out-Null +$body.Add("Advisory only โ€” never blocks a merge. Generated by ``tools/Suggest-ArticleSignals.ps1``$provenance. Suggestions carry stable ids for feedback; applying them is optional and human-reviewed.") | Out-Null + +return ($body -join "`n") diff --git a/tools/Suggest-ArticleSignals.ps1 b/tools/Suggest-ArticleSignals.ps1 new file mode 100644 index 0000000..49094dc --- /dev/null +++ b/tools/Suggest-ArticleSignals.ps1 @@ -0,0 +1,500 @@ +<# +.SYNOPSIS + Authoring-assist (PROTOTYPE): reviews the routing-relevant front-matter of + BCQuality knowledge articles. It proposes `signals:` (trigger AND suppressor), + flags declared-`domain` vs sample-content mismatches, cross-checks + `technologies`, and suggests `keywords` โ€” all suggestion-only. + +.DESCRIPTION + Routing quality is capped by how well each article's front-matter describes + the code it catches. Today most articles declare no explicit `signals:` block + and several review domains have zero detection signals, so a large slice of + knowledge is only reachable via the catch-all pass. This tool lowers the + authoring cost of closing that gap. For each article it reads the co-located + `good`/`bad` `.al` samples AND the article prose (inline code + fenced `al` + blocks), extracts candidate AL constructs, and emits: + + * A proposed `signals:` block. For a normal article these are RAISE + triggers. For a "this-is-not-a-violation" article (detected from the + title/prose) they are `effect: suppress` entries, which DAMPEN the + domain score instead of raising it. + * A domain-mismatch flag when the samples look like another domain. + * An applicability note when the samples use a technology (e.g. JavaScript + control add-ins) not declared in `technologies`. + * Keyword suggestions for strong constructs missing from `keywords`. + + It is OFFLINE and OPEN: no feedback data, deterministic, ships in the repo so + the community can run it. It SUGGESTS ONLY โ€” it never edits an article. The + output is a review report (Markdown by default, JSON with -AsJson) an author + or maintainer eyeballs; the front-matter validator (R29), CI, and CODEOWNERS + review remain the gate. This is the on-demand prototype the team agreed to + before formalizing an automated advisory PR check. + +.PARAMETER BCQualityRoot + Content root to scan. Defaults to the clone root (parent of tools/). + +.PARAMETER Path + Optional filter: only articles whose repo-relative path CONTAINS this string + (e.g. 'knowledge/style' or a single article file name). Enables on-demand, + per-PR-style scoping. Omit to scan every article. + +.PARAMETER ChangedFiles + Optional exact-match filter for the automated PR advisory: one or more + repo-relative article paths (forward-slash, e.g. + 'microsoft/knowledge/performance/avoid-commit-inside-loops.md'). Only these + articles are reported. Non-'.md' entries and paths outside a knowledge tree + are ignored, so the workflow can pass the raw changed-file list verbatim. + Combine with -AsJson to feed the comment renderer. Ignored when empty. + +.PARAMETER Top + Max suggested signals per article (default 3). + +.PARAMETER OnlyGaps + Only report articles that either have NO existing signal coverage for their + domain, are a suppressor candidate, or trigger a domain-mismatch/applicability + flag โ€” i.e. where the suggestion adds the most value. Off = report every + article that yields a suggestion. + +.PARAMETER NoProse + Ignore article prose; use only the `.al` samples as the construct source. + +.PARAMETER AsJson + Emit a machine-readable JSON report instead of Markdown (for a future + automated PR check to consume). + +.PARAMETER SeedPath + OPTIONAL routing seed (domain-normalization + the existing signal catalog to + de-dupe against and to power domain-mismatch detection). When supplied it must + exist. When omitted, tools/routing-seed.json is used only if present; + otherwise the tool runs seed-free (self-contained) with a built-in + domain-normalization map and no catalog de-dup/mismatch. The routing-index + tuning thread owns the seed; authoring-assist works with or without it. + +.OUTPUTS + Writes the report to stdout. Returns nothing. +#> +[CmdletBinding()] +param( + [string] $BCQualityRoot, + [string] $Path, + [string[]] $ChangedFiles, + [int] $Top = 3, + [switch] $OnlyGaps, + [switch] $NoProse, + [switch] $AsJson, + [string] $SeedPath +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +# JSON output contract. Bump SchemaVersion only for breaking shape changes; the +# comment renderer and the BC-ALAgentsInternal harvester key off it. ToolVersion +# tracks behavioral changes to ranking/extraction and feeds each suggestionId so +# a re-run after a tool change yields distinct, attributable ids. +$SchemaVersion = '1.0' +$ToolVersion = '1.0.0' + +# Stable, attributable identity for a single proposal. Deterministic across runs +# and machines: sha256 over the normalized article path + token + effect + tool +# version, truncated to 16 hex chars. Used to correlate PR-comment feedback +# (reactions, applied/not-applied) back to the exact suggestion that produced it. +function Get-SuggestionId { + param([string] $ArticlePath, [string] $Token, [string] $Effect) + $material = "$ArticlePath|$Token|$Effect|$ToolVersion" + $sha = [System.Security.Cryptography.SHA256]::Create() + try { + $bytes = $sha.ComputeHash([System.Text.Encoding]::UTF8.GetBytes($material)) + } finally { + $sha.Dispose() + } + return (($bytes[0..7] | ForEach-Object { $_.ToString('x2') }) -join '') +} + +if (-not $BCQualityRoot) { $BCQualityRoot = (Resolve-Path (Join-Path $PSScriptRoot '..')).Path } +$BCQualityRoot = (Resolve-Path -LiteralPath $BCQualityRoot).Path + +# Default canonical-domain taxonomy. Kept in-tool so authoring-assist is +# self-contained and does NOT depend on the routing-index tuning thread. When a +# routing seed IS present (supplied via -SeedPath or dropped at +# tools/routing-seed.json by the routing-index work), it overrides/extends this +# map AND unlocks the seed-catalog features below (de-dup against already-routed +# tokens + domain-mismatch detection). Absent a seed, the tool still normalizes +# domains and proposes signals from each article's own samples/prose. +$domainNorm = @{ + 'error-handling' = 'Error Handling' + 'performance' = 'Performance' + 'privacy' = 'Privacy' + 'security' = 'Security' + 'style' = 'Style' + 'testing' = 'Testing' + 'ui' = 'Accessibility' + 'upgrade' = 'Upgrade' + 'breaking-changes' = 'Breaking Changes' + 'events' = 'Events' + 'interfaces' = 'Interfaces' + 'web-services' = 'Web Services' + 'telemetry' = 'Privacy' +} + +# Resolve an OPTIONAL routing seed. Explicit -SeedPath must exist; otherwise the +# conventional tools/routing-seed.json is used only if present. No seed is not an +# error โ€” seed-catalog features (alreadyRoutes de-dup, domain-mismatch) are then +# simply inactive. +if ($SeedPath) { + if (-not (Test-Path -LiteralPath $SeedPath)) { throw "Routing seed not found: $SeedPath" } +} else { + $defaultSeed = Join-Path $PSScriptRoot 'routing-seed.json' + if (Test-Path -LiteralPath $defaultSeed) { $SeedPath = $defaultSeed } +} + +# Existing seed tokens by canonical domain, so we only propose NEW value. +$seedByToken = @{} # lowercased construct -> canonical domain it already routes +if ($SeedPath) { + $seed = Get-Content -LiteralPath $SeedPath -Raw | ConvertFrom-Json + foreach ($p in $seed.'domain-normalization'.PSObject.Properties) { $domainNorm[$p.Name] = $p.Value } + foreach ($s in $seed.signals) { + # Derive the bare words a seed pattern is likely to match (best effort) so a + # candidate construct already covered by the catalog is recognised. + foreach ($w in ([regex]::Matches($s.pattern, '[A-Za-z][A-Za-z0-9]{2,}') | ForEach-Object { $_.Value })) { + $lw = $w.ToLowerInvariant() + if (-not $seedByToken.ContainsKey($lw)) { $seedByToken[$lw] = $s.domain } + } + $seedByToken[$s.token.ToLowerInvariant()] = $s.domain + } +} + +function ConvertTo-CanonicalDomain { + param([string] $Domain) + if ([string]::IsNullOrWhiteSpace($Domain)) { return '' } + $d = $Domain.Trim() + if ($domainNorm.ContainsKey($d)) { return $domainNorm[$d] } + return $d +} + +function Get-RelativePath { + param([string] $Root, [string] $Full) + return (($Full.Substring($Root.Length).TrimStart([char]'/', [char]'\')) -replace '\\', '/') +} + +# Structural / common AL vocabulary that carries no routing signal โ€” never +# propose these. Object kinds, primitive types, control-flow, and the sample +# scaffolding tables/fields we see repeatedly. +$stop = @{} +@( + 'record','codeunit','page','report','query','xmlport','enum','interface','table','profile', + 'tableextension','pageextension','reportextension','enumextension','permissionset','entitlement', + 'procedure','trigger','begin','end','var','local','internal','protected','exit','then','else','repeat','until','case', + 'text','integer','boolean','decimal','code','date','datetime','time','guid','option','biginteger','duration','char','byte','label', + 'customer','item','vendor','salesheader','salesline','glentry','username','name','value','field','fields', + 'uppercase','lowercase','format','copystr','strsubstno','strlen','maxstrlen','abs','round','power', + 'get','next','insert','modify','delete','init','sourcetable','pagetype','list','card','document','worksheet', + 'true','false','array','temporary','count','isempty','testfield','fieldno','recordid','tabledata', + 'action','group','field','part','area','layout','actions','usercontrol','cuegroup','repeater' +) | ForEach-Object { $stop[$_] = $true } + +function Get-ArticleFrontmatter { + param([string[]] $Lines) + if (-not $Lines -or @($Lines).Count -lt 1 -or $Lines[0].Trim() -ne '---') { return $null } + $fmEnd = -1 + for ($i = 1; $i -lt $Lines.Count; $i++) { if ($Lines[$i].Trim() -eq '---') { $fmEnd = $i; break } } + if ($fmEnd -lt 0) { return $null } + $domain = ''; $keywords = @(); $technologies = @(); $hasSignals = $false + for ($i = 1; $i -lt $fmEnd; $i++) { + if ($Lines[$i] -match '^\s*domain\s*:\s*(.+?)\s*$') { $domain = $Matches[1].Trim() } + elseif ($Lines[$i] -match '^\s*keywords\s*:\s*\[(.*)\]\s*$') { + $inner = $Matches[1].Trim() + if ($inner) { $keywords = @($inner -split '\s*,\s*' | ForEach-Object { $_.Trim() }) } + } + elseif ($Lines[$i] -match '^\s*technologies\s*:\s*\[(.*)\]\s*$') { + $inner = $Matches[1].Trim() + if ($inner) { $technologies = @($inner -split '\s*,\s*' | ForEach-Object { $_.Trim().ToLowerInvariant() }) } + } + elseif ($Lines[$i] -match '^\s*signals\s*:') { $hasSignals = $true } + } + return [pscustomobject]@{ domain = $domain; keywords = $keywords; technologies = $technologies; hasSignals = $hasSignals; fmEnd = $fmEnd } +} + +# A "suppressor" article exists to say a construct is NOT a violation of its +# domain (e.g. page-display-is-not-a-privacy-concern). Its constructs should be +# proposed as effect: suppress, not raise. Detected from the title/basename and +# prose โ€” but NOT prohibitions ("do-not-add-X" means X *is* a violation). +function Test-SuppressorArticle { + param([string] $BaseName, [string] $Body) + $bn = $BaseName.ToLowerInvariant() + if ($bn -match '^(do-not|dont|do-nt|avoid|never|no-inline|prevent|disallow|must-not)') { return $false } + $suppressName = $bn -match '(is-not-a|is-not-an|not-a-violation|not-a-[a-z-]*concern|not-an-[a-z-]*issue|need-no|need-not|-allowed(-|$)|allowed-on|is-fine|is-acceptable|no-[a-z-]*concern|exempt)' + if ($suppressName) { return $true } + # Prose phrasing as a weaker secondary signal (first ~40 lines of body). + $head = ($Body -split "`n" | Select-Object -First 40) -join "`n" + if ($head -match '(?i)\b(is not a (violation|concern|problem|privacy)|not a violation|no .{0,20}concern|need no |perfectly (fine|acceptable)|is acceptable here|not flagged)\b') { + # Guard against prohibition prose. + if ($head -notmatch '(?i)\b(do not|don''t|avoid|never|must not|should not)\b') { return $true } + } + return $false +} + +# Extract candidate AL constructs from ARTICLE PROSE: fenced ```al code blocks +# (same shape as samples) and inline `code` spans that look like AL constructs. +# Prose is a weaker source than a .al sample โ€” used to reach the articles that +# ship no samples, and to corroborate sample constructs. +function Get-ProseConstructs { + param([string] $Body) + $out = @{} + if (-not $Body) { return $out } + # Fenced al/pascal code blocks. + foreach ($m in [regex]::Matches($Body, '(?s)```(?:al|pascal)?\s*(.*?)```')) { + foreach ($kv in (Get-SampleConstructs -Text $m.Groups[1].Value -IsBad:$false).GetEnumerator()) { + if (-not $out.ContainsKey($kv.Key)) { $out[$kv.Key] = 0 } + $out[$kv.Key] += $kv.Value.count + } + } + # Inline code spans that are a single AL-construct-shaped token. + foreach ($m in [regex]::Matches($Body, '`([A-Za-z][A-Za-z0-9]*)\s*(?:\(\))?`')) { + $tok = $m.Groups[1].Value + if ($tok -notmatch '^[A-Z][A-Za-z0-9]{2,}$') { continue } + if ($stop.ContainsKey($tok.ToLowerInvariant())) { continue } + if (-not $out.ContainsKey($tok)) { $out[$tok] = 0 } + $out[$tok] += 1 + } + return $out +} + +# Extract candidate AL constructs from sample text: method calls (Foo(), +# property assignments (Foo =), and attributes ([Foo]). Returns a hashtable of +# construct -> @{ count; inBad } so anti-pattern (bad-sample) constructs rank up. +function Get-SampleConstructs { + param([string] $Text, [bool] $IsBad) + $out = @{} + if (-not $Text) { return $out } + # Identifiers the sample itself DECLARES (object name, procedures, triggers) + # are sample-local scaffolding, not reusable routing constructs โ€” exclude. + $defined = @{} + foreach ($dm in [regex]::Matches($Text, '(?im)^\s*(?:procedure|trigger)\s+([A-Za-z][A-Za-z0-9]*)')) { + $defined[$dm.Groups[1].Value.ToLowerInvariant()] = $true + } + foreach ($dm in [regex]::Matches($Text, '(?im)^\s*(?:codeunit|page|report|query|table|xmlport|enum|interface|codeunit)\s+\d+\s+"?([A-Za-z][A-Za-z0-9 ]*)')) { + foreach ($w in ($dm.Groups[1].Value -split '\s+')) { if ($w) { $defined[$w.ToLowerInvariant()] = $true } } + } + $patterns = @( + '\b([A-Z][A-Za-z0-9]{2,})\s*\(', # method call + '(?m)^\s*([A-Z][A-Za-z0-9]{2,})\s*=', # property assignment + '\[([A-Z][A-Za-z0-9]{2,})' # attribute + ) + foreach ($pat in $patterns) { + foreach ($m in [regex]::Matches($Text, $pat)) { + $tok = $m.Groups[1].Value + if ($stop.ContainsKey($tok.ToLowerInvariant())) { continue } + if ($defined.ContainsKey($tok.ToLowerInvariant())) { continue } + if (-not $out.ContainsKey($tok)) { $out[$tok] = @{ count = 0; inBad = $false } } + $out[$tok].count++ + if ($IsBad) { $out[$tok].inBad = $true } + } + } + return $out +} + +# ---- Scan articles --------------------------------------------------------- +$reports = [System.Collections.Generic.List[object]]::new() + +# Normalize the optional exact-match changed-file set (automated advisory path). +# Accept the workflow's raw changed-file list: keep only knowledge '*.md' files, +# normalize to forward slashes, and match case-insensitively against each +# article's repo-relative path. +$changedSet = $null +if ($ChangedFiles -and @($ChangedFiles).Count -gt 0) { + $changedSet = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::OrdinalIgnoreCase) + foreach ($cf in $ChangedFiles) { + if ([string]::IsNullOrWhiteSpace($cf)) { continue } + $norm = ($cf.Trim() -replace '\\', '/').TrimStart('/') + if ($norm -notmatch '\.md$') { continue } + if ($norm -notmatch '(^|/)knowledge/') { continue } + [void]$changedSet.Add($norm) + } + if ($changedSet.Count -eq 0) { $changedSet = [System.Collections.Generic.HashSet[string]]::new() } +} +foreach ($layerDir in @('microsoft', 'community', 'custom')) { + $kbRoot = Join-Path $BCQualityRoot (Join-Path $layerDir 'knowledge') + if (-not (Test-Path $kbRoot)) { continue } + Get-ChildItem -LiteralPath $kbRoot -Recurse -File -Filter '*.md' -ErrorAction SilentlyContinue | + Sort-Object FullName | + ForEach-Object { + $rel = Get-RelativePath -Root $BCQualityRoot -Full $_.FullName + if ($Path -and ($rel -notlike "*$Path*")) { return } + if ($null -ne $changedSet -and -not $changedSet.Contains($rel)) { return } + + $lines = Get-Content -LiteralPath $_.FullName -ErrorAction SilentlyContinue + $fm = Get-ArticleFrontmatter -Lines $lines + if (-not $fm) { return } + $canonDomain = ConvertTo-CanonicalDomain $fm.domain + $body = ($lines | Select-Object -Skip ($fm.fmEnd + 1)) -join "`n" + $isSuppressor = Test-SuppressorArticle -BaseName $_.BaseName -Body $body + + # Collect constructs from co-located samples (good + bad). + $constructs = @{} + $sampleCount = 0 + foreach ($sample in Get-ChildItem -LiteralPath $_.DirectoryName -Filter "$($_.BaseName)*.al" -ErrorAction SilentlyContinue) { + $sampleCount++ + $isBad = $sample.Name -match '\.bad\.al$' + $text = Get-Content -LiteralPath $sample.FullName -Raw -ErrorAction SilentlyContinue + foreach ($kv in (Get-SampleConstructs -Text $text -IsBad:$isBad).GetEnumerator()) { + if (-not $constructs.ContainsKey($kv.Key)) { $constructs[$kv.Key] = @{ count = 0; inBad = $false; prose = $false; sample = $false } } + $constructs[$kv.Key].count += $kv.Value.count + $constructs[$kv.Key].sample = $true + if ($kv.Value.inBad) { $constructs[$kv.Key].inBad = $true } + } + } + + # Corroborate / fill gaps from prose (fenced al + inline code spans). + if (-not $NoProse) { + foreach ($kv in (Get-ProseConstructs -Body $body).GetEnumerator()) { + if (-not $constructs.ContainsKey($kv.Key)) { $constructs[$kv.Key] = @{ count = 0; inBad = $false; prose = $false; sample = $false } } + $constructs[$kv.Key].count += $kv.Value + $constructs[$kv.Key].prose = $true + } + } + + # Applicability cross-check: samples/prose that use JavaScript / control + # add-ins but `technologies` omits javascript (small but concrete). + $applicability = $null + $jsUse = @($constructs.Keys | Where-Object { $_ -in @('ControlAddIn','StartupScript','RecreateControls','Scripts') }).Count -gt 0 + if (-not $jsUse -and ($body -match '(?i)\bcontrol\s*add-?in\b|\.js\b|javascript')) { $jsUse = $true } + if ($jsUse -and ($fm.technologies -notcontains 'javascript')) { + $applicability = "samples/prose reference a JavaScript control add-in but 'technologies' does not list 'javascript'" + } + + # Keyword set (normalized) for relatedness scoring. + $kwset = @{} + foreach ($k in $fm.keywords) { $kwset[($k -replace '-', '').ToLowerInvariant()] = $true } + + # Score + classify candidates. + $cands = foreach ($c in $constructs.Keys) { + $lc = $c.ToLowerInvariant() + $seedDomain = if ($seedByToken.ContainsKey($lc)) { $seedByToken[$lc] } else { $null } + $score = [double]$constructs[$c].count + if ($constructs[$c].inBad) { $score += 2 } # anti-pattern construct + if ($kwset.ContainsKey($lc)) { $score += 3 } # backed by a declared keyword + if ($constructs[$c].sample) { $score += 1 } # seen in an actual sample + [pscustomobject]@{ + token = $c; score = $score; inBad = $constructs[$c].inBad + keywordBacked = $kwset.ContainsKey($lc) + proseOnly = ($constructs[$c].prose -and -not $constructs[$c].sample) + seedDomain = $seedDomain + alreadyRoutes = ($seedDomain -eq $canonDomain) + } + } + $cands = @($cands | Sort-Object -Property @{ Expression = 'score'; Descending = $true }, @{ Expression = 'token'; Descending = $false }) + + # Domain-mismatch flag: among candidates already known to the seed, + # what domain dominates? If it disagrees with the declared domain, + # the article's code looks like it belongs elsewhere. Skipped for + # suppressor articles, which deliberately reference other domains. + $seedMatched = @($cands | Where-Object { $_.seedDomain }) + $mismatch = $null + if (-not $isSuppressor -and $seedMatched.Count -ge 2) { + $byDom = $seedMatched | Group-Object seedDomain | Sort-Object Count -Descending + $dominant = $byDom[0] + if ($dominant.Name -ne $canonDomain -and $dominant.Count -ge 2) { + $mismatch = "declared domain '$canonDomain' but $($dominant.Count) sample construct(s) route to '$($dominant.Name)' ($([string]::Join(', ', ($dominant.Group | ForEach-Object { $_.token } | Select-Object -First 4))))" + } + } + + # Proposals: for a suppressor, the highest-value constructs (they say + # "seeing this here is fine") become effect: suppress. For a normal + # article, the highest-value NEW triggers = not already routed here. + $effect = if ($isSuppressor) { 'suppress' } else { 'raise' } + $proposals = @(if ($isSuppressor) { + @($cands | Select-Object -First $Top) + } else { + @($cands | Where-Object { -not $_.alreadyRoutes } | Select-Object -First $Top) + }) + + # Keyword suggestions: strong proposed tokens not already a keyword. + $keywordSuggestions = @($proposals | Where-Object { -not $_.keywordBacked } | ForEach-Object { $_.token } | Select-Object -First 3) + + $domainHasSeed = ($seedByToken.Values -contains $canonDomain) + $isGap = (-not $domainHasSeed) -or ($mismatch) -or ($applicability) -or ($isSuppressor) -or (-not $fm.hasSignals -and $proposals.Count -gt 0) + + if ($proposals.Count -eq 0 -and -not $mismatch -and -not $applicability) { return } + if ($OnlyGaps -and -not $isGap) { return } + + $reports.Add([pscustomobject]@{ + path = $rel; layer = $layerDir; domain = $canonDomain; rawDomain = $fm.domain + hasSignals = $fm.hasSignals; domainHasSeed = $domainHasSeed + suppressor = $isSuppressor; effect = $effect + sampleCount = $sampleCount + mismatch = $mismatch + applicability = $applicability + keywordSuggestions = $keywordSuggestions + proposals = @($proposals | ForEach-Object { + [pscustomobject]@{ + suggestionId = Get-SuggestionId -ArticlePath $rel -Token $_.token -Effect $effect + token = $_.token + pattern = '\b' + [regex]::Escape($_.token) + '\b' + effect = $effect + keywordBacked = $_.keywordBacked + inBad = $_.inBad + proseOnly = $_.proseOnly + note = if ($_.seedDomain) { "also seen in seed domain '$($_.seedDomain)'" } else { 'new construct' } + } + }) + }) | Out-Null + } +} + +# ---- Emit ------------------------------------------------------------------ +if ($AsJson) { + [pscustomobject]@{ + schemaVersion = $SchemaVersion + toolVersion = $ToolVersion + generatedAt = (Get-Date).ToUniversalTime().ToString('o') + root = $BCQualityRoot + filter = $Path + articleCount = $reports.Count + reports = @($reports) + } | ConvertTo-Json -Depth 8 + return +} + +$flagged = @($reports | Where-Object { $_.mismatch }).Count +$zeroDomain = @($reports | Where-Object { -not $_.domainHasSeed }).Count +$suppressors = @($reports | Where-Object { $_.suppressor }).Count +$applic = @($reports | Where-Object { $_.applicability }).Count +Write-Host "" +Write-Host "BCQuality authoring-assist (prototype) โ€” SUGGESTIONS ONLY, nothing written." -ForegroundColor Cyan +Write-Host ("Articles with suggestions: {0} Suppressor articles: {1} Domain-mismatch: {2} Applicability: {3} Zero-signal domain: {4}" -f $reports.Count, $suppressors, $flagged, $applic, $zeroDomain) +if ($Path) { Write-Host ("Filter: *{0}*" -f $Path) } +Write-Host ("-" * 78) + +foreach ($r in ($reports | Sort-Object domain, path)) { + Write-Host "" + $title = if ($r.suppressor) { "$($r.path) [SUPPRESSOR]" } else { $r.path } + Write-Host $title -ForegroundColor White + $domNote = if (-not $r.domainHasSeed) { " [zero-signal domain]" } else { "" } + Write-Host (" domain: {0}{1} existing signals: {2} samples: {3}" -f $r.domain, $domNote, $(if ($r.hasSignals) { 'yes' } else { 'none' }), $r.sampleCount) + if ($r.mismatch) { Write-Host (" โš  domain check: {0}" -f $r.mismatch) -ForegroundColor Yellow } + if ($r.applicability) { Write-Host (" โš  applicability: {0}" -f $r.applicability) -ForegroundColor Yellow } + if ($r.proposals.Count -gt 0) { + $kind = if ($r.suppressor) { "SUPPRESS (this construct is NOT a violation here)" } else { "RAISE" } + Write-Host (" proposed signals [{0}]: front-matter block ->" -f $kind) -ForegroundColor Green + Write-Host " signals:" + foreach ($p in $r.proposals) { + $why = @() + if ($p.keywordBacked) { $why += 'keyword-backed' } + if ($p.inBad) { $why += 'anti-pattern sample' } + if ($p.proseOnly) { $why += 'from prose' } + $whyStr = if ($why) { " # " + ($why -join ', ') } else { "" } + if ($r.suppressor) { + Write-Host (" - token: {0}{1}" -f $p.token, $whyStr) + Write-Host " effect: suppress" + } else { + Write-Host (" - {0}{1}" -f $p.token, $whyStr) + } + } + } + if ($r.keywordSuggestions.Count -gt 0) { + Write-Host (" keyword idea: consider adding to 'keywords': {0}" -f ([string]::Join(', ', @($r.keywordSuggestions | ForEach-Object { ($_ -creplace '(? +[CmdletBinding()] +param( + [string] $BCQualityRoot, + [string] $SeedPath +) +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +if (-not $BCQualityRoot) { $BCQualityRoot = (Resolve-Path (Join-Path $PSScriptRoot '..')).Path } +$tool = Join-Path $PSScriptRoot 'Suggest-ArticleSignals.ps1' +$renderer = Join-Path $PSScriptRoot 'New-AuthoringAssistComment.ps1' + +# The routing seed is OPTIONAL (owned by the routing-index tuning thread). Detect +# whether one is available so the seed-only features (domain-mismatch, catalog +# de-dup) are exercised when present and asserted dormant when absent. +if (-not $SeedPath) { + $defaultSeed = Join-Path $PSScriptRoot 'routing-seed.json' + if (Test-Path -LiteralPath $defaultSeed) { $SeedPath = $defaultSeed } +} +$seedPresent = [bool]$SeedPath +$seedArgs = if ($seedPresent) { @('-SeedPath', $SeedPath) } else { @() } + +$pass = 0; $fail = 0 +function Check($name, [bool]$cond) { + if ($cond) { Write-Host " PASS $name" -ForegroundColor Green; $script:pass++ } + else { Write-Host " FAIL $name" -ForegroundColor Red; $script:fail++ } +} + +Write-Host 'Test-SuggestArticleSignals' -ForegroundColor Cyan +Check 'tool exists' (Test-Path $tool) +Check 'renderer exists' (Test-Path $renderer) + +# 1) Known keyword-backed anti-pattern construct is proposed as a raise trigger. +$j = & $tool -BCQualityRoot $BCQualityRoot -Path 'avoid-commit-inside-loops' -AsJson | ConvertFrom-Json +$commit = $j.reports | Where-Object { $_.path -like '*avoid-commit-inside-loops*' } +Check 'commit-in-loops article reported' ($null -ne $commit) +Check 'proposes Commit signal' (@($commit.proposals | Where-Object token -eq 'Commit').Count -eq 1) + +# 2) JSON contract: schema + tool version present. +Check 'json carries schemaVersion' (-not [string]::IsNullOrWhiteSpace($j.schemaVersion)) +Check 'json carries toolVersion' (-not [string]::IsNullOrWhiteSpace($j.toolVersion)) + +# 3) suggestionId present + deterministic across runs. +$firstId = @($commit.proposals)[0].suggestionId +Check 'proposal carries suggestionId' (-not [string]::IsNullOrWhiteSpace($firstId)) +$j2 = & $tool -BCQualityRoot $BCQualityRoot -Path 'avoid-commit-inside-loops' -AsJson | ConvertFrom-Json +$firstId2 = @(($j2.reports | Where-Object { $_.path -like '*avoid-commit-inside-loops*' }).proposals)[0].suggestionId +Check 'suggestionId is deterministic' ($firstId -eq $firstId2) + +# 4) Zero-signal Style domain yields a real property signal. +$style = & $tool -BCQualityRoot $BCQualityRoot -Path 'api-page-version-format' -AsJson | ConvertFrom-Json +$sr = $style.reports | Where-Object { $_.path -like '*api-page-version-format*' } +Check 'style article marked zero-signal domain' ($sr.domainHasSeed -eq $false) +Check 'proposes APIVersion signal' (@($sr.proposals | Where-Object token -eq 'APIVersion').Count -ge 1) + +# 5) Suppressor detection: effect suppress + suggestionId differs from a raise id +# for the same token (effect participates in the id). +$supp = & $tool -BCQualityRoot $BCQualityRoot -Path 'page-display-is-not-a-privacy-concern' -AsJson | ConvertFrom-Json +$sp = $supp.reports | Where-Object { $_.path -like '*page-display-is-not-a-privacy-concern*' } +Check 'suppressor article detected' ($sp.suppressor -eq $true) +Check 'suppressor effect is suppress' ($sp.effect -eq 'suppress') +Check 'suppressor proposals carry effect suppress' (@($sp.proposals | Where-Object { $_.effect -eq 'suppress' }).Count -ge 1) + +# 6) Prohibition is NOT mis-classified as a suppressor. +$prohib = & $tool -BCQualityRoot $BCQualityRoot -Path 'do-not-add-ishandled-to-an-existing-event' -AsJson | ConvertFrom-Json +$pr = $prohib.reports | Where-Object { $_.path -like '*do-not-add-ishandled*' } +Check 'prohibition article is not a suppressor' ($null -eq $pr -or $pr.suppressor -eq $false) + +# 7) Domain-mismatch is a seed-only enrichment. With a seed present it must fire +# on a known cross-domain article; seed-free it must stay dormant (null) while +# the corpus scan still succeeds โ€” proving graceful degradation. +$full = & $tool -BCQualityRoot $BCQualityRoot @seedArgs -AsJson | ConvertFrom-Json +Check 'reports >100 articles corpus-wide' ($full.articleCount -gt 100) +$mm = $full.reports | Where-Object { $_.path -like '*avoid-raising-events-inside-try-functions*' -and $_.mismatch } +if ($seedPresent) { + Check 'events/try-function article flagged as domain-mismatch (seeded)' ($null -ne $mm) +} else { + Check 'domain-mismatch dormant without a seed' ($null -eq $mm) + Check 'seed-free corpus still marks every domain zero-signal' (@($full.reports | Where-Object { $_.domainHasSeed }).Count -eq 0) +} + +# 8) -ChangedFiles exact-match scoping: only the listed knowledge md is reported; +# non-md and non-existent entries are ignored. +$target = $commit.path +$cf = & $tool -BCQualityRoot $BCQualityRoot -ChangedFiles @($target, 'README.md', 'microsoft/knowledge/does-not-exist.md') -AsJson | ConvertFrom-Json +Check 'ChangedFiles reports exactly the one changed article' ($cf.articleCount -eq 1 -and $cf.reports[0].path -eq $target) + +# 9) Renderer: emits anchor + markers + a paste-ready signals block. +$tmp = Join-Path ([System.IO.Path]::GetTempPath()) ('aa-selftest-' + [guid]::NewGuid().ToString('N') + '.json') +try { + & $tool -BCQualityRoot $BCQualityRoot -ChangedFiles @($target) -AsJson | Set-Content -LiteralPath $tmp -Encoding UTF8 + $md = & $renderer -JsonPath $tmp + Check 'renderer emits advisory anchor' ($md -match '') + Check 'renderer emits aa:meta marker' ($md -match '') -and ($mdEmpty -match 'No routing head-matter suggestions')) +} finally { + if (Test-Path -LiteralPath $emptyTmp) { Remove-Item -LiteralPath $emptyTmp -Force } +} + +Write-Host '' +Write-Host ("Result: {0} passed, {1} failed" -f $pass, $fail) -ForegroundColor $(if ($fail) { 'Red' } else { 'Green' }) +if ($fail) { exit 1 } diff --git a/tools/authoring-assist.md b/tools/authoring-assist.md new file mode 100644 index 0000000..194b35d --- /dev/null +++ b/tools/authoring-assist.md @@ -0,0 +1,152 @@ +# Authoring-assist: `Suggest-ArticleSignals.ps1` + +> **Status:** first-class BCQuality tool. It **suggests only** โ€” it never edits an +> article. It runs on demand from the CLI *and* as a **non-blocking advisory** on +> every PR that adds or modifies a knowledge article (see +> [Automated advisory](#automated-advisory-ci)). Suggestions are human-reviewed; +> R29 + CI + CODEOWNERS remain the gate. + +## Why + +Routing quality is capped by how well each knowledge article's front-matter +describes the code it catches. The orchestrator scores a PR diff against +per-article `signals:`; an article +with no signal for its domain is only reachable via the expensive catch-all pass. + +At prototype time **0 of ~207 articles declared an explicit `signals:` block** and +**5 review domains had zero detection signals** (Style, Breaking Changes, +Interfaces, Testing, appsource โ‰ˆ 22% of the corpus). Those gaps won't be closed by +hand at scale, and hand-authoring `signals:` correctly is exactly the precision- +critical work we don't want to guess at. This tool lowers that authoring cost. + +## What it does + +For each article it reads the article's own co-located `good`/`bad` `.al` samples +**and its prose** (inline `` `code` `` spans + fenced ```` ```al ```` blocks), plus +its front-matter, extracts candidate AL constructs, and emits: + +1. **Proposed `signals:` block** โ€” the highest-value constructs from the article's + own samples/prose. Constructs are + ranked by: keyword-backed (+3), appears in the `.bad.al` anti-pattern sample + (+2), seen in an actual sample (+1), frequency. Sample-local scaffolding + (procedure/object names) and structural AL vocabulary (types, control-flow, + object kinds) are filtered out. When an **optional** routing seed is present + (`-SeedPath`, or `tools/routing-seed.json` if the routing-index thread has + landed one), constructs already routed to the article's domain are de-duped + out; without a seed every strong construct is offered. +2. **Suppressor (`effect: suppress`) proposals** โ€” when an article exists to say a + construct is **not** a violation of its domain (detected from the title/basename + and prose, e.g. `page-display-is-not-a-privacy-concern`), its constructs are + proposed as `effect: suppress` (mapping form) instead of raise. A suppress hit + *dampens* the domain score in the orchestrator rather than raising it, directly + attacking the over-firing (precision) problem. Prohibitions (`do-not-โ€ฆ`, + `avoid-โ€ฆ`) are explicitly **excluded** โ€” those describe a real violation. +3. **Domain-mismatch flag** *(seed-only; dormant without a seed)* โ€” when โ‰ฅ2 of an + article's sample constructs are known + to the seed and they route to a *different* domain than the article declares + (e.g. an `events/` article whose sample is all `TryFunction`/`Error`). A soft + advisory prompting the author to confirm the domain or add a cross-domain signal. + Skipped for suppressor articles, which deliberately reference other domains. +4. **Applicability note** โ€” flags samples/prose that use a JavaScript control add-in + when `technologies` omits `javascript`. Small but concrete; kept minimal because + the technologies vocabulary is tiny (`al`/`javascript`). +5. **Keyword suggestions** โ€” strong proposed constructs not already present in the + article's `keywords` list, surfaced as an advisory "consider adding". + +**Prose mining** matters for the ~17% of articles that ship no `.al` samples at all +(including most suppressor articles): their constructs are recovered from the prose +and marked `proseOnly` so a reviewer knows the weaker provenance. Use `-NoProse` to +restrict the source to samples only. + +It is **deterministic, offline, and open**: no feedback/telemetry data, so the +community can run it against `community/` and `custom/` layers too. + +## Usage + +```powershell +# Whole corpus, human-readable report +tools/Suggest-ArticleSignals.ps1 + +# On-demand, scoped to a folder or a single article (path substring match) +tools/Suggest-ArticleSignals.ps1 -Path knowledge/style +tools/Suggest-ArticleSignals.ps1 -Path avoid-commit-inside-loops + +# Exact-match scoping used by the automated advisory (repo-relative paths) +tools/Suggest-ArticleSignals.ps1 -ChangedFiles microsoft/knowledge/performance/avoid-commit-inside-loops.md + +# Only the high-value cases (zero-signal domains, suppressors, mismatch/applicability) +tools/Suggest-ArticleSignals.ps1 -OnlyGaps + +# Samples-only (ignore article prose) +tools/Suggest-ArticleSignals.ps1 -NoProse + +# Machine-readable (the advisory + feedback contract) +tools/Suggest-ArticleSignals.ps1 -AsJson + +# Render a JSON report into the PR advisory comment body +tools/Suggest-ArticleSignals.ps1 -ChangedFiles -AsJson > report.json +tools/New-AuthoringAssistComment.ps1 -JsonPath report.json +``` + +Key parameters: `-Top ` (max signals per article, default 3), `-ChangedFiles` +(exact repo-relative paths), `-NoProse`, `-BCQualityRoot`, `-SeedPath`. + +## Automated advisory (CI) + +On every PR that touches `**/knowledge/**/*.md`, a **non-blocking** advisory +comment proposes the missing routing head-matter for the changed articles. It +never fails a check; applying a suggestion is optional and human-reviewed. + +Three workflows implement it: + +| Workflow | Role | +| --- | --- | +| `.github/workflows/authoring-assist.yml` | Unprivileged intake on `pull_request` (path-filtered); saves PR metadata. | +| `.github/workflows/authoring-assist-runner.yml` | Trusted `workflow_run` companion. A read-only `review` job runs the **trusted-base** tool + renderer over the PR-head article text (enriched with the trusted routing seed only if one is present); a separate `publish` job holds `pull-requests: write` and upserts the single advisory comment. | +| `.github/workflows/authoring-assist-selftest.yml` | Runs `Test-SuggestArticleSignals.ps1` against this repo on changes to the tool/renderer. | + +The `pull_request` โ†’ `workflow_run` split lets the advisory comment on **fork +PRs** without exposing a write-scoped token to the job that reads untrusted PR +content. The tool is a deterministic parser โ€” it never executes AL or PR content. + +## JSON contract & feedback metadata + +`-AsJson` is a stable contract (`schemaVersion`, `toolVersion`). Every proposal +carries a `suggestionId` โ€” a deterministic hash of `articlePath + token + effect ++ toolVersion` โ€” so feedback can be correlated back to the exact suggestion. + +`New-AuthoringAssistComment.ps1` renders that JSON into the advisory comment and +embeds machine-readable markers: + +```html + top anchor (idempotent upsert) + + +``` + +Each comment also carries a reaction footer (๐Ÿ‘ useful ยท โค๏ธ especially valuable ยท +๐Ÿ‘Ž wrong). These markers + reactions are the input to the **feedback harvester** +in `BC-ALAgentsInternal`, which measures suggestion **acceptance** (did the author +apply the proposed `signals:`?) and sentiment. BCQuality emits only the +deterministic markers; the harvest/aggregation is internal and reads BCQuality via +the GitHub API โ€” a downward dependency that keeps the graph acyclic. + +## Guardrails (why it only suggests) + +`domain`/`signals` are precision-critical and human-reviewed. Silent auto-population +would inject the same noise we're trying to remove and would bypass CODEOWNERS +review. So the tool emits a proposal an author eyeballs and adds via a normal PR; +[`validate_frontmatter.py` R29](../.github/scripts/validate_frontmatter.py) + CI + +human review remain the gate. Both proposed bare-token (raise) and mapping-form +(`effect: suppress`) blocks satisfy R29 as-is. + +## Relationship to the feedback-driven engine (Tier-2) + +This is the **offline/static (Tier-1)** engine: it proposes signals from an +article's own samples and prose. A complementary **feedback-driven (Tier-2)** +engine in `BC-ALAgentsInternal` closes the loop โ€” it harvests the advisory +comments' feedback metadata (acceptance + reactions, keyed by `suggestionId`) and +mines aggregate PR signals to refine suggestions. It emits aggregate suggestions +only and reads BCQuality via the GitHub API, so the dependency still points +downward (no cycle). From c0d0418f65475a05efd3680e256ca3c42c51be1b Mon Sep 17 00:00:00 2001 From: dayland Date: Fri, 24 Jul 2026 11:59:53 +0100 Subject: [PATCH 2/2] Authoring-assist runner: download advisory artifact from the current run The review and publish jobs run in the same workflow_run; the artifact is intra-run. Downloading with run-id set to the intake run (github.event.workflow_run.id) looked in the wrong run and failed with 'Artifact not found'. Drop run-id/github-token so download-artifact resolves the artifact uploaded earlier in this run. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 75d852d6-18a6-4c58-986f-ed5ab16618fa --- .github/workflows/authoring-assist-runner.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/authoring-assist-runner.yml b/.github/workflows/authoring-assist-runner.yml index 6fc1d54..2f49481 100644 --- a/.github/workflows/authoring-assist-runner.yml +++ b/.github/workflows/authoring-assist-runner.yml @@ -192,8 +192,6 @@ jobs: with: name: authoring-assist-output-${{ needs.review.outputs.pr_number }} path: ${{ github.workspace }}/advisory-output - github-token: ${{ github.token }} - run-id: ${{ github.event.workflow_run.id }} - name: Upsert advisory comment env: