From 00d991f1b5a3eccde5944c7f76e0a4e1ea2be51c Mon Sep 17 00:00:00 2001 From: Hashem Aldhaheri <158606+aenawi@users.noreply.github.com> Date: Sat, 5 Sep 2026 15:55:38 +0400 Subject: [PATCH] =?UTF-8?q?feat(ooxml,cli):=20DOCX=20writer=20=E2=80=94=20?= =?UTF-8?q?M3=203.6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `mirsam repair report.docx fixed.docx` works, and `repair` stops turning a Word document away as a readable format without a writer. M3 is closed. `word.rs` is WordprocessingML's repair vocabulary — what `rewrite.rs` is for DrawingML and `sheet.rs` for SpreadsheetML — and it is the third vocabulary built on `token.rs` without a Word element name entering it. Every mutation is a scaffold call with a name in it: `w:pPr/w:bidi`, `w:pPr/w:jc`, `w:rPr/w:lang@w:bidi`, `w:rPr/w:rFonts@w:cs`, `w:pPr/w:numPr` and `w:tblPr/w:bidiVisual`, each placed by rank against the schema sequence its parent is, because `CT_PPrBase` and `CT_RPr` are `xsd:sequence` and a correct element in the wrong position is a document Word offers to repair. The writer needs no inherited direction, and that is Word's doing rather than a shortcut. 1.2's hardest correction was that lowering `Start` onto DrawingML's physical `algn` without knowing the paragraph's direction reproduces the defect being repaired — which is why `rewrite::Inherited` exists and why `PptxDocument::apply` re-runs its scanner over every part it touches. `w:jc` is evaluated against the paragraph's own `w:bidi`, so `Start` is written `start` whichever way the paragraph runs and stays the start edge if the direction changes afterwards. The whole inheritance pass is absent from this adapter, not skipped in it. Two refusals, and both are the format's. A physical edge is not something a Word paragraph can state, which is the refusal the conformance suite already records for the reading side; `supports` answers it for the writing side, so `SetAlignment(Left)` is listed in `repairs.skipped` rather than lowered onto whichever relative edge happens to land left today. And a typed bullet becomes a `w:numPr` pointing into the numbering part, so it can only join a list the document already defines — `package.rs` replaces the entries a package holds, and a numbering part that is not there is a part, a content-type override and a relationship, none of which is an edit to the paragraph the finding named. `bullet_list` prefers a definition whose first level draws the marker the author typed, falls back to any bulleted one, and refuses a `w:num` that overrides its abstract at level zero. Three smaller decisions, each a defect if taken the other way. `` is on, so a left-to-right repair writes `w:val="0"` and never an empty element: creating one to mean off would turn the repair into the defect it was sent to fix. Filling `@w:cs` removes `@w:cstheme` beside it, because the theme reference is what Word renders and `@w:cs` the value it caches for consumers that do not implement themes — writing into the cache and leaving the reference standing is a repair that reports success and changes nothing a reader sees. And a `w:t` whose text this repair changed is marked `xml:space="preserve"` where the result has an edge of whitespace, because stripping a typed bullet leaves the space behind it and Word would otherwise collapse a space nobody asked to delete; only the runs actually rewritten are marked, so the rest of the paragraph still comes out byte for byte. `token.rs` gained the ability to step over a caller-named set of elements, and Word is why. A `w:txbxContent` nests whole paragraphs inside a run, and each is a unit with its own ordinal, its own text and its own repairs — offsets the domain computed for the outer paragraph index the outer paragraph's runs alone. An `mc:Fallback` is stepped over for the reader's reason: it spells out the same paragraphs as the `mc:Choice` beside it, and a rewriter that counted both would put paragraph 2's repair on the fallback's copy of paragraph 1, a report and a document that had come apart while both still looked right. It also gained `remove_attribute`, the counterpart `set_attribute` needed once a repair had to stop a second attribute contradicting the one being written. The corpus is where it shows, which is what the recorded refusal was for. Both `.docx` reports lost their `refused` line and gained a repair report; `quarterly-review.docx` now carries a `changed_parts` diff naming exactly the tags above, and its `after` audit is empty — every fixable finding in a real Word document cleared. `make validate-fixtures` accepts the repaired copy against the ECMA-376 transitional schemas, and a second `repair` of that output stages nothing and reproduces it byte for byte. The CLI case that proved the refusal moved to `.html`, the format that still gives one, rather than being deleted with the adapter that outgrew it. Not run: "Word opens the repaired document without offering to repair it", the M1 application check in its Word form, cannot run in CI. Schema validity is the strongest claim a machine can make here and is what the acceptance rests on. It is not the same claim. --- AGENTS.md | 41 +- CHANGELOG.md | 34 + README.md | 32 +- crates/mirsam-cli/src/main.rs | 3 + crates/mirsam-cli/tests/cli.rs | 67 +- crates/mirsam-cli/tests/golden.rs | 12 +- crates/mirsam-ooxml/src/docx.rs | 255 +++++- crates/mirsam-ooxml/src/lib.rs | 13 +- crates/mirsam-ooxml/src/rewrite.rs | 9 +- crates/mirsam-ooxml/src/token.rs | 182 +++- crates/mirsam-ooxml/src/word.rs | 686 +++++++++++++++ crates/mirsam-ooxml/tests/word.rs | 790 ++++++++++++++++++ docs/ARCHITECTURE.md | 15 +- docs/PLAN.md | 89 +- docs/ROADMAP.md | 19 +- .../quarterly-review-correct.expected.json | 48 +- tests/fixtures/quarterly-review.expected.json | 383 ++++++++- 17 files changed, 2583 insertions(+), 95 deletions(-) create mode 100644 crates/mirsam-ooxml/src/word.rs create mode 100644 crates/mirsam-ooxml/tests/word.rs diff --git a/AGENTS.md b/AGENTS.md index dc859b6..cc18992 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -11,7 +11,7 @@ binary: no Python, no Node, no Office, no runtime of any kind on the target machine. `audit` reads `.pptx`, `.docx`, `.xlsx` and `.html`/`.htm`. `repair` writes -`.pptx` and `.xlsx`: the other two are refused as readable formats without a +`.pptx`, `.docx` and `.xlsx`: `.html` is refused as a readable format without a writer, which is a usage error (exit `2`) and not a claim that the document was not understood. @@ -176,6 +176,41 @@ through untouched, and mechanically so: `xl/workbook.xml` is not a part a repair plan can name, and a cell repair edits `@s` or the content of a `` and nothing beside it. +### Repairing a Word document, where two things are refused + +A `.docx` repair writes `w:pPr/w:bidi`, `w:pPr/w:jc`, `w:rPr/w:lang@w:bidi`, +`w:rPr/w:rFonts@w:cs`, `w:pPr/w:numPr` and `w:tblPr/w:bidiVisual`, and every +one of them lands in its schema-sequence position — WordprocessingML's +property elements are `xsd:sequence`, so a correct element written in the +wrong place is a document Word offers to repair. + +Three things about it are worth knowing before reading a repair report. + +- **`w:jc` is written relative, so no direction is resolved to write one.** + The PowerPoint writer has to know the direction a paragraph inherits before + it can put a `Start` down, because `a:pPr/@algn` names physical edges. Word's + does not: `start` is the start edge whichever way the paragraph runs. The + same fact is why **a physical edge is refused** — `SetAlignment(Left)` and + `SetAlignment(Right)` appear in `repairs.skipped`, never as a repair that + failed, because Word has no spelling for "the left of the page whatever the + direction". No Word unit ever produces one, so this is a refusal you will not + see in practice; it is stated so the writer cannot start inventing one. +- **`--convert-bullets` needs a list the document already defines.** Word has + no per-paragraph bullet: a list is a `w:numPr` pointing into the numbering + part. The repair points at a bulleted definition the document already + carries, preferring one that draws the marker the author typed, and + otherwise the fix is **skipped**. A repair cannot add the numbering part + itself — that is a part, a content-type override and a relationship, none of + which is an edit to the paragraph the finding named. +- **`--font` clears `@w:cstheme` on the runs it fills.** `@w:cstheme` is what + Word renders and `@w:cs` the value it caches beside it, so writing the + typeface into the cache and leaving a theme reference standing would change + what the file says and not what a reader sees. + +A paragraph inside a text box is a unit of its own and is repaired as one; the +paragraph anchoring the box is not repaired through it, and an `mc:Fallback` +is neither counted nor edited — the same two regions the reader steps over. + ### HTML, where the direction is usually not in the document An `.html` page states direction in three places and the adapter reads all of @@ -380,7 +415,9 @@ share a stem: the report is named for it. A report for a format the tool reads but cannot write holds the refusal `repair` gave and the exit code it used, in place of a repair report — so the -day a writer lands, it shows up as a diff on a real document. +day a writer lands, it shows up as a diff on a real document. That is what the +two `.docx` reports did when M3's writer landed, and `.html` is what carries +the refusal now. **The shaping fixtures are fonts, and they are generated too.** The five under `crates/mirsam-core/tests/fonts/` are written byte by byte by diff --git a/CHANGELOG.md b/CHANGELOG.md index 70a1633..95b0278 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,40 @@ Working towards byte-preserving `repair` for PPTX. See ### Added +- **The DOCX writer.** `mirsam repair report.docx fixed.docx` works, and + `repair` stops turning a Word document away as a readable format without a + writer. A repair writes `w:pPr/w:bidi`, `w:pPr/w:jc`, `w:rPr/w:lang@w:bidi`, + `w:rPr/w:rFonts@w:cs`, `w:pPr/w:numPr` and `w:tblPr/w:bidiVisual`, each + placed by rank against the schema sequence its parent is — WordprocessingML's + property elements are `xsd:sequence`, and a correct element in the wrong + position is a document Word offers to repair. `word.rs` is the third + vocabulary built on the same token-rewrite scaffold, and `token.rs` did not + have to learn a Word element name to carry it. + + **The writer needs no inherited direction, unlike PowerPoint's.** + DrawingML's `algn` names physical edges, so lowering a `Start` there without + knowing the paragraph's direction reproduces the defect being repaired. + Word's `w:jc` is evaluated against the paragraph's own `w:bidi`, so `start` + is the start edge whichever way the paragraph runs — the inheritance pass is + absent from this adapter rather than skipped in it. + + **Two repairs are refused, and both refusals are the format's.** Word has no + spelling for a *physical* edge, which is the refusal the conformance suite + already records for the reading side, so `SetAlignment(Left)` appears in + `repairs.skipped` rather than being lowered onto whichever relative edge + lands left today. And a typed bullet becomes a `w:numPr` pointing into the + numbering part, so `--convert-bullets` can only join a list the document + already defines — preferring one that draws the marker the author typed — + and is skipped otherwise. Filling `@w:cs` also clears `@w:cstheme` beside + it, because the theme reference is what Word renders and leaving it standing + would be a repair that changes what the file says and not what a reader + sees. See [`docs/PLAN.md`](docs/PLAN.md) §3.6. + + The two corpus Word documents recorded the refusal this replaces, so the + change arrives as a repair report and a part-level diff on + `quarterly-review.docx` — whose `after` audit is now empty — rather than as + a test somebody remembered to update. + - **The XLSX adapter, and the second format `repair` can write.** `mirsam audit book.xlsx` reads a workbook and `mirsam repair book.xlsx fixed.xlsx` writes one. A cell is a paragraph and the worksheet around it is the diff --git a/README.md b/README.md index 1b86298..c0ab2f8 100644 --- a/README.md +++ b/README.md @@ -4,9 +4,8 @@ finds right-to-left, bidirectional and typography defects in PowerPoint, Word, Excel and HTML files, *proves* each one by resolving what the text will actually look like when rendered, and repairs them without touching a byte it -was not asked to. PowerPoint and Excel are repaired; Word and HTML are -audit-only so far, and the PDF adapter is scheduled; see -[Roadmap](docs/ROADMAP.md). +was not asked to. PowerPoint, Word and Excel are repaired; HTML is audit-only +so far, and the PDF adapter is scheduled; see [Roadmap](docs/ROADMAP.md). > **mirsam** (مَرْسَم) — an atelier; the room where things are properly drawn. > From the root ر-س-م: to draw, to inscribe, to render. @@ -154,12 +153,12 @@ so an agent can act on it without opening PowerPoint. See [`AGENTS.md`](AGENTS.m **v0.1 “Steppe Eagle” — audit only, PPTX only.** Byte-preserving `repair` for PPTX has landed on `main` and ships as v0.2. Three formats have landed since. -DOCX and HTML are **readers**: `mirsam audit report.docx` works, and `mirsam -audit page.html` reads `dir`, `lang` and the part of CSS that decides direction -— including a stylesheet the page links by a relative path. `repair` refuses -both and says why, because neither writer is built yet. A page linking a -stylesheet on a server is audited without it and the report names what it did -not read, because this tool makes no network calls. + +HTML is a **reader**: `mirsam audit page.html` reads `dir`, `lang` and the part +of CSS that decides direction — including a stylesheet the page links by a +relative path. `repair` refuses it and says why, because its writer is not +built. A page linking a stylesheet on a server is audited without it and the +report names what it did not read, because this tool makes no network calls. XLSX is the second format with a **writer**: `mirsam repair book.xlsx fixed.xlsx` sets a sheet's `rightToLeft` and a cell's reading order and @@ -167,10 +166,17 @@ alignment, and leaves every formula and defined name exactly as it found them — a workbook's formatting is shared between its cells, so a repair appends a format record and repoints the one cell rather than editing what forty cells are using. A formula's cached result is not judged and is named in the report, -for the reason a stylesheet on a server is. The PDF adapter is specified and -scheduled; see [`docs/ROADMAP.md`](docs/ROADMAP.md). The tool reports what it -can actually verify and nothing more — a discipline inherited from its prior -art. +for the reason a stylesheet on a server is. + +DOCX is the third: `mirsam repair report.docx fixed.docx` writes `w:bidi`, +`w:jc`, the complex-script language and font slots, a table's `w:bidiVisual` +and a real list in place of a typed bullet — each in the schema position Word +requires. Two repairs are refused rather than approximated, and both refusals +are the format's: Word has no way to state a *physical* edge, and a typed +bullet can only join a list the document already defines. The PDF adapter is +specified and scheduled; see [`docs/ROADMAP.md`](docs/ROADMAP.md). The tool +reports what it can actually verify and nothing more — a discipline inherited +from its prior art. ## Design diff --git a/crates/mirsam-cli/src/main.rs b/crates/mirsam-cli/src/main.rs index 864ada2..68fe069 100644 --- a/crates/mirsam-cli/src/main.rs +++ b/crates/mirsam-cli/src/main.rs @@ -302,6 +302,9 @@ fn open_for_repair(path: &Path) -> Result> { "pptx" => Ok(Box::new( PptxDocument::open(path).with_context(|| format!("opening {}", path.display()))?, )), + "docx" => Ok(Box::new( + DocxDocument::open(path).with_context(|| format!("opening {}", path.display()))?, + )), "xlsx" => Ok(Box::new( XlsxDocument::open(path).with_context(|| format!("opening {}", path.display()))?, )), diff --git a/crates/mirsam-cli/tests/cli.rs b/crates/mirsam-cli/tests/cli.rs index de29786..79183a3 100644 --- a/crates/mirsam-cli/tests/cli.rs +++ b/crates/mirsam-cli/tests/cli.rs @@ -766,21 +766,76 @@ fn a_correctly_marked_word_document_exits_zero() { #[test] fn repairing_a_readable_but_unwritable_format_says_so_rather_than_denying_it() { - // `docx` is not an unknown extension — `audit` reads it. Reporting it as + // `html` is not an unknown extension — `audit` reads it. Reporting it as // one would be the tool contradicting itself between two commands, so the // refusal names the real reason and stays a usage error. - let scratch = Scratch::new("docx-repair"); - let input = docx(&scratch, "in.docx", ""); - let out = run(&["repair", &input, &scratch.path("out.docx")]); + // + // This case was written against `.docx`, which is the format that used to + // answer it. Word gained a writer, so the claim moved to the format that + // still has none rather than being deleted with the adapter that outgrew + // it: what is under test is the refusal, not which format gives it. + let scratch = Scratch::new("html-repair"); + let input = scratch.file( + "page.html", + "

ارتفع الأداء بنسبة 25%

".as_bytes(), + ); + let out = run(&["repair", &input, &scratch.path("out.html")]); - assert_eq!(out.code, exit::USAGE); + assert_eq!(out.code, exit::USAGE, "stderr:\n{}", out.stderr); assert!( out.stderr.contains("audited") && out.stderr.contains("not yet repaired"), "stderr:\n{}", out.stderr ); // And nothing was written. - assert!(!Path::new(&scratch.path("out.docx")).exists()); + assert!(!Path::new(&scratch.path("out.html")).exists()); +} + +#[test] +fn a_word_document_is_repaired_and_the_repaired_copy_audits_clean() { + // The other side of the case above: `repair` no longer turns a `.docx` + // away, and what it writes is a document the same binary reads back with + // the finding gone. + let scratch = Scratch::new("docx-repaired"); + let input = docx( + &scratch, + "broken.docx", + r#""#, + ); + let output = scratch.path("fixed.docx"); + + let out = run(&["repair", &input, &output, "--format", "json"]); + assert_eq!(out.code, exit::OK, "stderr:\n{}", out.stderr); + + let json = parse_json(&out); + assert_eq!(json["format"], "docx"); + assert!( + !json["repairs"]["applied"].as_array().unwrap().is_empty(), + "{json:#?}" + ); + assert!( + json["repairs"]["skipped"].as_array().unwrap().is_empty(), + "nothing in this document is beyond WordprocessingML:\n{json:#?}" + ); + assert!( + rule_ids(&json["before"]["diagnostics"]).contains(&"direction-mismatch".to_string()), + "{json:#?}" + ); + assert!( + !rule_ids(&json["after"]["diagnostics"]).contains(&"direction-mismatch".to_string()), + "{json:#?}" + ); + + // The input is never modified, and the direction landed on the paragraph. + let repaired = std::fs::read(&output).unwrap(); + let mut archive = zip::ZipArchive::new(std::io::Cursor::new(repaired)).unwrap(); + let mut document = String::new(); + std::io::Read::read_to_string( + &mut archive.by_name("word/document.xml").unwrap(), + &mut document, + ) + .unwrap(); + assert!(document.contains(r#""#), "{document}"); } #[test] diff --git a/crates/mirsam-cli/tests/golden.rs b/crates/mirsam-cli/tests/golden.rs index aaed77b..bd13b28 100644 --- a/crates/mirsam-cli/tests/golden.rs +++ b/crates/mirsam-cli/tests/golden.rs @@ -13,12 +13,16 @@ //! //! ## A format the tool reads but cannot write //! -//! `repair` refuses a `.docx`, and refuses it as a *readable format without a +//! `repair` refuses an `.html`, and refuses it as a *readable format without a //! writer* rather than as an unknown extension. That refusal is recorded here //! rather than worked around: the report holds the message and the exit code -//! it produced, so the day the Word writer lands, the corpus shows the change -//! as a diff on a real document instead of as a test somebody remembered to -//! update. +//! it produced, so the day the writer lands, the corpus shows the change as a +//! diff on a real document instead of as a test somebody remembered to update. +//! +//! It did exactly that for Word. The two `.docx` reports held that refusal +//! until M3's writer landed, and the change arrived as a repair report and a +//! part-level diff on `quarterly-review.docx` — which is the review this +//! paragraph exists to force. //! //! A diff is a change in behaviour. When it is intended, regenerate the //! reports and commit them with the change that caused them, so the diff is diff --git a/crates/mirsam-ooxml/src/docx.rs b/crates/mirsam-ooxml/src/docx.rs index e7c1aa8..d432301 100644 --- a/crates/mirsam-ooxml/src/docx.rs +++ b/crates/mirsam-ooxml/src/docx.rs @@ -71,29 +71,41 @@ //! caller holding one part and no package — an unstated property stays //! `Unset` rather than being guessed at. //! -//! ## Reading only, for now +//! ## Writing //! -//! [`DocumentWriter`] is deliberately not implemented. `DocumentReader` and -//! `DocumentWriter` are separate ports precisely so an adapter can arrive one -//! half at a time, and claiming a repair this crate cannot yet express in -//! WordprocessingML would be the kind of inferred capability `AGENTS.md` -//! forbids. +//! [`DocumentWriter`] lands the repairs, and [`crate::word`] is where every +//! element name they are written with lives — this module groups them by the +//! part and the paragraph a unit id names and hands the rest over. //! +//! Two of the shape it takes are worth reading before the code. It needs no +//! map of inherited directions, unlike the PowerPoint adapter: `w:jc` is +//! relative to the paragraph's own direction, so there is nothing to lower a +//! `Start` against. And a typed bullet is converted by pointing the paragraph +//! at a list the document already defines, so [`supports`] answers *no* for +//! `ConvertLiteralBullet` on a document that defines none — a `w:numPr` naming +//! a list that does not exist is a document Word offers to repair. +//! +//! [`supports`]: mirsam_core::ports::DocumentWriter::supports //! [`DocumentWriter`]: mirsam_core::ports::DocumentWriter //! [ECMA-376]: https://ecma-international.org/publications-and-standards/standards/ecma-376/ //! [MS-OE376]: https://learn.microsoft.com/en-us/openspecs/office_standards/ms-oe376/26ecf09a-0f0b-4574-9907-ebd1ddf3015f use crate::inherit::{ThemeFont, ThemeScript}; -use crate::package::Package; +use crate::package::{Edits, Package}; +use crate::rels::RelationshipGraph; use crate::style::{StyleSheet, theme_reference}; use crate::token::is_true; +use crate::word; use mirsam_core::error::{Error, Result}; -use mirsam_core::ports::DocumentReader; +use mirsam_core::ports::{DocumentReader, DocumentWriter}; use mirsam_core::text::{ Alignment, Bullet, Direction, Location, Properties, Resolved, TextUnit, UnitKind, }; +use mirsam_core::{Fix, Repair, UnitId}; use quick_xml::events::{BytesStart, Event}; use quick_xml::{Reader, XmlVersion}; +use std::cell::RefCell; +use std::collections::BTreeMap; use std::path::{Path, PathBuf}; /// Alignment values WordprocessingML understands, as *Word* reads them. @@ -157,6 +169,34 @@ fn table_id(part: &str, index: usize) -> String { format!("{part}#tbl{index}") } +/// What a unit id this adapter issued points at. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Target { + Paragraph(usize), + Table(usize), +} + +/// Recover the part and target from an id this adapter issued. +/// +/// `#` cannot occur in an OPC part name, so the last `#` is unambiguous, and +/// `tbl` is tried before `p` because `p` is one letter. The same shapes the +/// PowerPoint adapter issues parse the same way here; an id from any other +/// adapter is refused rather than half-understood, because a repair that +/// landed on "whatever paragraph 3 of this part is" would be a repair the +/// report never asked for. +fn parse_unit_id(id: &UnitId) -> Option<(&str, Target)> { + let (part, rest) = id.0.rsplit_once('#')?; + let ordinal = |digits: &str| digits.parse::().ok().filter(|n| *n > 0); + + let target = if let Some(n) = rest.strip_prefix("tbl") { + Target::Table(ordinal(n)?) + } else { + Target::Paragraph(ordinal(rest.strip_prefix('p')?)?) + }; + + (!part.is_empty()).then_some((part, target)) +} + /// Accumulates a table while the rows, cells and paragraphs inside it are /// being parsed. /// @@ -588,15 +628,27 @@ impl PartScan { } } -/// A Word package opened for auditing. +/// A Word package opened for auditing and repair. pub struct DocxDocument { package: Package, + /// Parts rewritten by [`DocumentWriter::apply`], awaiting + /// [`DocumentWriter::write`]. Everything else is copied raw. + edits: Edits, + /// The list each typed marker converts to, answered once per marker. + /// + /// [`DocumentWriter::supports`] takes `&self` and is asked the same + /// question for every literal bullet in the document; reading and parsing + /// `word/numbering.xml` once per finding would be a cost that grows with + /// the defect count for an answer that cannot change. + bullets: RefCell>>, } impl DocxDocument { pub fn open(path: impl Into) -> Result { Ok(Self { package: Package::open(path)?, + edits: Edits::new(), + bullets: RefCell::new(BTreeMap::new()), }) } @@ -634,6 +686,52 @@ impl DocxDocument { StyleSheet::read(&self.package) } + /// One part's XML, taking a staged edit over the package's own bytes so + /// that repairs applied in two rounds compose rather than the second round + /// discarding the first. + fn part_text(&self, part: &str) -> Result { + match self.edits.get(part) { + Some(bytes) => { + String::from_utf8(bytes.clone()).map_err(|e| Error::Format(format!("{part}: {e}"))) + } + None => self.package.read_text(part), + } + } + + /// The `w:numId` a paragraph joins when a typed `marker` is converted, or + /// `None` when this document defines no bulleted list. + /// + /// The numbering part is reached by the relationship pointing at it rather + /// than by its conventional path, for the reason [`crate::style`] gives + /// about `styles.xml`: a package is free to store it anywhere, and a + /// writer that hard-codes `word/numbering.xml` refuses a repair it could + /// have made on a document that stores it elsewhere. + /// + /// An unreadable or malformed numbering part answers `None`. That is the + /// same answer as "there is no list here", and it is the right one: the + /// repair is reported as *not made* rather than attempted against a part + /// this adapter could not understand. + fn bullet_list(&self, marker: char) -> Option { + if let Some(known) = self.bullets.borrow().get(&marker) { + return known.clone(); + } + let found = self.numbering_part().and_then(|part| { + let xml = self.package.read_text(&part).ok()?; + word::bullet_list(&part, &xml, marker).ok().flatten() + }); + self.bullets.borrow_mut().insert(marker, found.clone()); + found + } + + /// The part this document defines its lists in. + fn numbering_part(&self) -> Option { + let graph = RelationshipGraph::read(&self.package).ok()?; + let document = graph.office_document()?.to_string(); + graph + .first_part_of_kind(&document, "numbering") + .map(str::to_string) + } + /// Parse one `word/**/*.xml` part into text units. /// /// Direction, alignment, the font slots and the language are recorded as @@ -724,13 +822,108 @@ impl DocumentReader for DocxDocument { let styles = self.styles()?; let mut units = Vec::new(); for part in self.text_parts()? { - let xml = self.package.read_text(&part)?; + let xml = self.part_text(&part)?; units.extend(Self::scan_part(&part, &xml, Some(&styles))?); } Ok(units) } } +impl DocumentWriter for DocxDocument { + /// Whether WordprocessingML can express `fix`. + /// + /// Two refusals, and both are the format's rather than this adapter's. + /// + /// A **physical edge** is not something a Word paragraph can state: + /// `w:jc`'s values are evaluated against the paragraph's own `w:bidi`, so + /// there is no value meaning "the left of the page whatever the direction" + /// to write [`Alignment::Left`] as. This is the writing half of the + /// refusal the conformance suite states for reading, and it is why no + /// Word unit ever comes back carrying one. + /// + /// A **typed bullet** is converted by pointing the paragraph at a list, + /// and the list has to be one this document already defines. A repair + /// cannot supply it: [`crate::package`] replaces the entries a package + /// holds, and a numbering part that is not there needs a part, a + /// content-type override and a relationship, none of which is an edit to + /// the paragraph the finding named. + fn supports(&self, fix: &Fix) -> bool { + match fix { + Fix::SetAlignment(Alignment::Left | Alignment::Right) => false, + Fix::ConvertLiteralBullet { marker } => self.bullet_list(*marker).is_some(), + _ => true, + } + } + + /// Stage repairs against the parts they name. + /// + /// Repairs are grouped by part and then by paragraph, so a part is read + /// and rewritten once however many paragraphs it carries. Nothing is + /// staged unless every part succeeds: a failure half-way through must not + /// leave a document that is partly repaired and reports otherwise. + /// + /// There is no inheritance pass here, and its absence is Word's doing. + /// The PowerPoint adapter has to re-run its scanner over each part to + /// learn the direction every paragraph takes from its container, because + /// `a:pPr/@algn` names physical edges and a `Start` cannot be written + /// without one. `w:jc` is already relative to the paragraph's own + /// direction, so the rewriter needs nothing the paragraph does not carry. + fn apply(&mut self, repairs: &[Repair]) -> Result { + let mut by_part: BTreeMap = BTreeMap::new(); + let mut bullets = word::Bullets::new(); + + for repair in repairs { + let Some((part, target)) = parse_unit_id(&repair.unit) else { + return Err(Error::Format(format!( + "{}: not a unit this adapter produced", + repair.unit + ))); + }; + // Resolved here rather than in the rewriter, which holds no + // package and so cannot see the numbering part. `supports` has + // already answered this question for every repair the caller + // staged; a `Fix` that reached `apply` anyway is refused rather + // than written as a `w:numPr` naming nothing. + if let Fix::ConvertLiteralBullet { marker } = &repair.fix { + let Some(num_id) = self.bullet_list(*marker) else { + return Err(Error::Format(format!( + "{part}: cannot {}; this document defines no bulleted list \ + for a paragraph to join", + repair.fix + ))); + }; + bullets.insert(*marker, num_id); + } + + let plan = by_part.entry(part.to_string()).or_default(); + match target { + Target::Paragraph(index) => plan.paragraphs.entry(index).or_default(), + Target::Table(index) => plan.tables.entry(index).or_default(), + } + .push(repair.fix.clone()); + } + + let mut staged = Edits::new(); + let mut applied = 0usize; + for (part, plan) in &by_part { + // A part edited by an earlier call is edited again from its staged + // bytes, so repairs applied in two rounds compose. + let xml = self.part_text(part)?; + let rewritten = word::apply_plan(part, &xml, plan, &bullets)?; + applied += plan.len(); + staged.insert(part.clone(), rewritten.into_bytes()); + } + + self.edits.extend(staged); + Ok(applied) + } + + fn write(&mut self, dest: &Path) -> Result<()> { + self.package.rewrite(dest, &self.edits)?; + Ok(()) + } +} + /// Parse an in-memory part into every unit this adapter produces for it. /// /// Exposed for tests and for callers that already hold the XML. There is no @@ -744,3 +937,45 @@ pub fn scan_xml(part: &str, xml: &str) -> Result> { pub fn scan_xml_with(part: &str, xml: &str, styles: Option<&StyleSheet>) -> Result> { DocxDocument::scan_part(part, xml, styles) } + +#[cfg(test)] +mod unit_id_tests { + use super::*; + + #[test] + fn a_unit_id_round_trips_through_its_own_parser() { + let part = "word/document.xml"; + assert_eq!( + parse_unit_id(&UnitId(unit_id(part, 3))), + Some((part, Target::Paragraph(3))) + ); + assert_eq!( + parse_unit_id(&UnitId(table_id(part, 2))), + Some((part, Target::Table(2))) + ); + } + + #[test] + fn an_id_this_adapter_did_not_issue_is_rejected() { + for foreign in [ + "", + "document", + "#p1", + "word/document.xml#p0", + "x#px", + "x#p-1", + "x#tbl0", + "x#tblx", + "x#t1", + // The containers the PowerPoint adapter issues and this one does + // not: a Word paragraph is not laid out in columns, and a chart in + // a Word document is a part the DrawingML reader answers for. + "x#cols1", + "x#catax1", + "x#legend1", + "x#dlbls1", + ] { + assert_eq!(parse_unit_id(&UnitId(foreign.into())), None, "{foreign:?}"); + } + } +} diff --git a/crates/mirsam-ooxml/src/lib.rs b/crates/mirsam-ooxml/src/lib.rs index 1caf808..d2af42b 100644 --- a/crates/mirsam-ooxml/src/lib.rs +++ b/crates/mirsam-ooxml/src/lib.rs @@ -23,12 +23,12 @@ //! inside a part. Everything else names elements, and an element name belongs //! to exactly one format: [`rewrite`] is DrawingML's repair vocabulary, //! [`pptx`] its reader, [`inherit`] the layout/master chain it resolves, -//! [`chart`] the chart parts a deck references, and [`docx`] and [`style`] -//! WordprocessingML's reader and its own style chain — which share -//! [`package`] and [`token`] with the rest and name not one DrawingML -//! element. The theme is the exception both formats really do share: a -//! `a:fontScheme` is DrawingML wherever it is stored, so [`inherit`] reads -//! Word's theme part unchanged. +//! [`chart`] the chart parts a deck references, and [`docx`], [`style`] and +//! [`word`] WordprocessingML's reader, its own style chain and its repair +//! vocabulary — which share [`package`] and [`token`] with the rest and name +//! not one DrawingML element. The theme is the exception both formats really +//! do share: a `a:fontScheme` is DrawingML wherever it is stored, so +//! [`inherit`] reads Word's theme part unchanged. //! //! SpreadsheetML is the third vocabulary, and it reuses those same two shared //! modules and nothing else: [`xlsx`] is Excel's reader and writer, @@ -54,6 +54,7 @@ pub mod rewrite; pub mod sheet; pub mod style; pub mod token; +pub mod word; pub mod workbook; pub mod xlsx; diff --git a/crates/mirsam-ooxml/src/rewrite.rs b/crates/mirsam-ooxml/src/rewrite.rs index 79f7b5b..cc6144b 100644 --- a/crates/mirsam-ooxml/src/rewrite.rs +++ b/crates/mirsam-ooxml/src/rewrite.rs @@ -11,9 +11,12 @@ //! of its own: every mutation below is a scaffold call with a DrawingML name //! in it. //! -//! That split is what makes a second format cheap. A WordprocessingML repair -//! is the same scaffold with `w:p`, `w:pPr/w:bidi` and `w:jc` in place of the -//! names here; nothing in [`crate::token`] has to learn about it. +//! That split is what makes a second format cheap, and [`crate::word`] is the +//! claim collected: the same scaffold with `w:p`, `w:pPr/w:bidi` and `w:jc` in +//! place of the names here, and not a line of [`crate::token`] moved to carry +//! it. It came out simpler than this module in one respect — see +//! [`Inherited`], whose Word counterpart does not exist because `w:jc` is +//! already relative to the paragraph's own direction. use crate::chart::ChartText; use crate::token::is_true; diff --git a/crates/mirsam-ooxml/src/token.rs b/crates/mirsam-ooxml/src/token.rs index 3936415..3d80944 100644 --- a/crates/mirsam-ooxml/src/token.rs +++ b/crates/mirsam-ooxml/src/token.rs @@ -6,7 +6,8 @@ //! splices attribute values in their raw bytes, inserts children at a position //! a caller-supplied schema sequence decides, and reads and rewrites run text. //! Which elements and attributes those are is the *vocabulary*, and the -//! vocabulary lives with its format — [`crate::rewrite`] holds DrawingML's. +//! vocabulary lives with its format — [`crate::rewrite`] holds DrawingML's, +//! [`crate::word`] WordprocessingML's and [`crate::sheet`] SpreadsheetML's. //! //! Nothing here names an element. `a:t` and `w:t` are both "the element whose //! content is run text", and a caller says which it means; `a:pPr` and `w:pPr` @@ -14,6 +15,14 @@ //! sequence. That is the whole of the separation, and it is why a second //! adapter reuses this file rather than copying it. //! +//! *Which* elements a scan should step over is a vocabulary question too, so +//! the `_outside` functions take the names from the caller. DrawingML never +//! needs them — an `a:p` cannot contain another. WordprocessingML does: +//! [`crate::word`] passes `w:txbxContent`, because a text box nests whole +//! paragraphs that are units in their own right, and `mc:Fallback`, because +//! [`crate::docx`] does not read one and a rewriter that counted it would +//! number the paragraphs differently from the report it is applying. +//! //! Two rules follow from the byte-preservation guarantee, and they shape all //! the code here. //! @@ -54,10 +63,14 @@ pub fn is_true(value: &str) -> bool { // ------------------------------------------------------------ raw attributes -/// One attribute's name and the byte range of its value inside a tag's content. +/// One attribute's name and the byte ranges it occupies inside a tag's content. struct RawAttribute { name: String, + /// The value alone, inside its quotes: what [`set_attribute`] splices. value: std::ops::Range, + /// The whole attribute, from the first byte of its name to its closing + /// quote: what [`remove_attribute`] cuts out. + span: std::ops::Range, } /// Scan a start tag's raw content into attribute names and value ranges. @@ -112,11 +125,15 @@ fn scan_attributes(content: &str) -> Vec { while i < content.len() && content[i] != quote { i += 1; } + let value = value_start..i; + i += 1; // past the closing quote out.push(RawAttribute { name, - value: value_start..i, + value, + // Clamped: an unterminated value leaves `i` past the end, and a + // malformed tag must not make a caller's slice panic. + span: name_start..i.min(content.len()), }); - i += 1; // past the closing quote } out @@ -167,6 +184,36 @@ pub fn set_attribute(tag: &BytesStart, name: &str, value: &str) -> BytesStart<'s BytesStart::from_content(replaced, tag.name().0.len()) } +/// Delete an attribute and the whitespace that separated it, leaving every +/// other byte of the tag untouched. A tag that does not carry it is returned +/// as it was. +/// +/// The counterpart to [`set_attribute`], and needed for the same reason: some +/// repairs are only true if a second attribute stops contradicting the one +/// being written. WordprocessingML's `w:rFonts` is the case in point — Word +/// renders `@w:cstheme` and keeps `@w:cs` beside it as a cached answer, so +/// writing the typeface into `@w:cs` and leaving a theme reference standing +/// would change what the file *says* and not what a reader *sees*. +pub fn remove_attribute(tag: &BytesStart, name: &str) -> BytesStart<'static> { + let content: &str = tag; + let Some(found) = scan_attributes(content) + .into_iter() + .find(|a| a.name == name) + else { + return tag.clone().into_owned(); + }; + + // Take the whitespace before the attribute with it, so removing the only + // attribute leaves `` rather than ``. + let start = content[..found.span.start] + .bytes() + .rposition(|b| !b.is_ascii_whitespace()) + .map_or(0, |i| i + 1); + let kept = format!("{}{}", &content[..start], &content[found.span.end..]); + + BytesStart::from_content(kept, tag.name().0.len()) +} + // --------------------------------------------------------------- event stream /// Read a part into an owned event stream. @@ -269,13 +316,52 @@ pub fn element_ranges( events: &[Event<'static>], name: &str, ) -> Vec<(usize, std::ops::Range)> { + element_ranges_outside(events, name, &[]) +} + +/// [`element_ranges`], skipping the whole of any element named in `excluded`. +/// +/// A rewriter must number the elements it edits exactly as the scanner that +/// produced the report numbered them, and a scanner that does not read some +/// region of the part does not count what is in it. DrawingML has never +/// needed this; WordprocessingML does, because its reader steps over an +/// `mc:Fallback` — the fallback and the `mc:Choice` beside it spell out the +/// same paragraphs, and counting both would put a repair on the wrong one. +/// +/// The container's own start tag, at index 0 of a slice, is not a candidate +/// for exclusion: a caller passing a paragraph and excluding paragraphs means +/// the ones nested inside it. +pub fn element_ranges_outside( + events: &[Event<'static>], + name: &str, + excluded: &[&str], +) -> Vec<(usize, std::ops::Range)> { + element_starts_outside(events, name, excluded) + .into_iter() + .enumerate() + .map(|(n, i)| (n + 1, element_range(events, i))) + .collect() +} + +/// The index of every element named `name`, in document order, skipping the +/// whole of any element named in `excluded`. See [`element_ranges_outside`]. +pub fn element_starts_outside( + events: &[Event<'static>], + name: &str, + excluded: &[&str], +) -> Vec { let mut out = Vec::new(); - let mut index = 0usize; - for i in 0..events.len() { - if name_of(&events[i]).as_deref() == Some(name) { - index += 1; - out.push((index, element_range(events, i))); + let mut i = 0usize; + while i < events.len() { + match name_of(&events[i]).as_deref() { + Some(found) if i > 0 && excluded.contains(&found) => { + i = element_range(events, i).end; + continue; + } + Some(found) if found == name => out.push(i), + _ => {} } + i += 1; } out } @@ -406,20 +492,31 @@ pub fn text_content_ranges( events: &[Event<'static>], text_element: &str, ) -> Vec> { - let mut out = Vec::new(); - let mut start = None; - for (i, event) in events.iter().enumerate() { - match event { - Event::Start(e) if e.name().0 == text_element => start = Some(i + 1), - Event::End(e) if e.name().0 == text_element => { - if let Some(from) = start.take() { - out.push(from..i); - } - } - _ => {} - } - } - out + text_content_ranges_outside(events, text_element, &[]) +} + +/// [`text_content_ranges`], skipping the whole of any element named in +/// `excluded` — a caller's way of saying which runs are *this* unit's own. +/// +/// DrawingML has no use for it, because an `a:p` cannot contain another one. A +/// WordprocessingML paragraph can: a text box is a `w:txbxContent` inside a +/// run, and the paragraphs in it are units in their own right, with their own +/// text and their own repairs. An offset the domain computed for the outer +/// paragraph indexes the outer paragraph's runs alone, so a repair that +/// reached the inner ones would delete from a string nobody scanned. +pub fn text_content_ranges_outside( + events: &[Event<'static>], + text_element: &str, + excluded: &[&str], +) -> Vec> { + element_starts_outside(events, text_element, excluded) + .into_iter() + // A self-closing `` holds no content and so has no range; the + // element range of one is a single event, and `end - 1` would be the + // start tag itself. + .filter(|&i| matches!(events[i], Event::Start(_))) + .map(|i| i + 1..element_range(events, i).end - 1) + .collect() } /// Resolve a run of content events to logical-order text. @@ -486,9 +583,20 @@ pub fn replace_content( /// untouched — character references and all. pub fn map_runs(events: &mut Vec>, text_element: &str, f: impl Fn(&str) -> String) { let ranges = text_content_ranges(events, text_element); - let before = read_runs(events, &ranges); + map_runs_in(events, &ranges, f); +} + +/// [`map_runs`], over a caller-chosen set of content ranges — the ones +/// [`text_content_ranges_outside`] answered with, where some of the element's +/// runs belong to a unit inside it rather than to the unit being repaired. +pub fn map_runs_in( + events: &mut Vec>, + ranges: &[std::ops::Range], + f: impl Fn(&str) -> String, +) { + let before = read_runs(events, ranges); let after: Vec = before.iter().map(|text| f(text)).collect(); - replace_content(events, &ranges, &before, &after); + replace_content(events, ranges, &before, &after); } /// Remove the characters at the given offsets into the element's text. @@ -498,7 +606,17 @@ pub fn map_runs(events: &mut Vec>, text_element: &str, f: impl Fn /// to front so that removing one does not move the next. pub fn remove_at_offsets(events: &mut Vec>, text_element: &str, offsets: &[usize]) { let ranges = text_content_ranges(events, text_element); - let before = read_runs(events, &ranges); + remove_at_offsets_in(events, &ranges, offsets); +} + +/// [`remove_at_offsets`], over a caller-chosen set of content ranges. See +/// [`map_runs_in`] for when a caller has to choose them. +pub fn remove_at_offsets_in( + events: &mut Vec>, + ranges: &[std::ops::Range], + offsets: &[usize], +) { + let before = read_runs(events, ranges); let mut runs = before.clone(); let mut sorted: Vec = offsets.to_vec(); @@ -520,13 +638,23 @@ pub fn remove_at_offsets(events: &mut Vec>, text_element: &str, o } } - replace_content(events, &ranges, &before, &runs); + replace_content(events, ranges, &before, &runs); } /// Strip a leading marker character, and the whitespace after it, from the /// start of the element's first run. pub fn strip_leading_marker(events: &mut Vec>, text_element: &str, marker: char) { let ranges = text_content_ranges(events, text_element); + strip_leading_marker_in(events, &ranges, marker); +} + +/// [`strip_leading_marker`], over a caller-chosen set of content ranges. See +/// [`map_runs_in`] for when a caller has to choose them. +pub fn strip_leading_marker_in( + events: &mut Vec>, + ranges: &[std::ops::Range], + marker: char, +) { let Some(first) = ranges.first().cloned() else { return; }; diff --git a/crates/mirsam-ooxml/src/word.rs b/crates/mirsam-ooxml/src/word.rs new file mode 100644 index 0000000..726d0fc --- /dev/null +++ b/crates/mirsam-ooxml/src/word.rs @@ -0,0 +1,686 @@ +//! WordprocessingML's repair vocabulary: which element and which attribute +//! each `Fix` lands on, and where a created element goes. +//! +//! [`crate::rewrite`] is this module's opposite number for DrawingML and +//! [`crate::sheet`] Excel's; the three share [`crate::token`] and +//! [`crate::package`] and not one element name. The guarantee is the same at +//! both scales: a part no repair touches survives byte for byte, and inside a +//! part that *is* rewritten, a token no repair addresses comes out as it went +//! in. +//! +//! ## Word's alignment is already direction-relative, so nothing is lowered +//! +//! `a:pPr/@algn` names physical edges, so the PowerPoint rewriter has to be +//! told the direction each paragraph reads in before it can write a `Start` +//! down — that is what [`crate::rewrite::Inherited`] is for, and getting it +//! wrong reproduces the defect being repaired. +//! +//! **This module needs no such map, and that is Word's doing rather than a +//! shortcut.** `w:jc`'s values are evaluated against the paragraph's own +//! `w:bidi` ([MS-OE376] Part 4 §2.3.1.13, note b), so `Start` is written as +//! `start` whichever way the paragraph runs and stays the start edge if the +//! direction is changed afterwards. The same fact read from the other side is +//! why [`crate::docx`] never produces `Alignment::Left`: a physical edge is +//! not a thing WordprocessingML can say, so [`Alignment::Left`] and +//! [`Alignment::Right`] are *refused* here rather than lowered onto the edge +//! they happen to mean today. The conformance suite states that refusal for +//! the reading side, and this is its writing half. +//! +//! ## `` is on, so left-to-right must be written out +//! +//! An `ST_OnOff` element with no `w:val` is true, which is the form Word +//! writes far more often than the explicit one. A repair setting a paragraph +//! left-to-right therefore writes `w:val="0"` and never an empty element: +//! creating `` to mean *off* would turn the repair into the defect. +//! +//! ## A typed bullet becomes a list this document already has +//! +//! DrawingML answers `ConvertLiteralBullet` with `a:buChar`, an attribute on +//! the paragraph. Word has no such thing: a list is a `w:numPr` pointing into +//! `word/numbering.xml`, and a paragraph cannot carry the definition itself. +//! [`bullet_list`] finds a definition the document already stores — preferring +//! one that draws the marker the author typed — and the repair points at it. +//! A document with no bulleted list at all is *refused*, because +//! [`crate::package`] rewrites the entries a package has and creating +//! `word/numbering.xml`, its content-type override and its relationship is +//! three new parts rather than an edit. +//! +//! ## Which runs belong to the paragraph being repaired +//! +//! A `w:txbxContent` nests whole paragraphs inside a run, and each of them is +//! a unit in its own right with its own text, its own ordinal and its own +//! repairs. An `mc:Fallback` spells out the same paragraphs as the +//! `mc:Choice` beside it and [`crate::docx`] reads neither it nor them. So +//! both are stepped over here — when the elements are counted, so a repair +//! lands on the paragraph the report named, and when a paragraph's runs are +//! edited, so an offset the domain computed cannot reach text nobody scanned. +//! +//! [MS-OE376]: https://learn.microsoft.com/en-us/openspecs/office_standards/ms-oe376/26ecf09a-0f0b-4574-9907-ebd1ddf3015f + +use crate::token::{ + self, child_or_insert, edit_tag, element_range, element_ranges_outside, find_direct_child, + get_attribute, remove_attribute, set_attribute, +}; +use mirsam_core::Fix; +use mirsam_core::error::{Error, Result}; +use mirsam_core::script; +use mirsam_core::text::{Alignment, Direction}; +use quick_xml::events::Event; +use std::collections::BTreeMap; +use std::ops::Range; + +/// Repairs for one part, keyed by paragraph index. +/// +/// The index counts every `w:p` in the part, 1-based, exactly as +/// [`crate::docx`] counts them — including the paragraphs that produced no +/// text unit, and excluding everything inside an `mc:Fallback`, so the two +/// numberings cannot drift. +pub type PartFixes = BTreeMap>; + +/// Repairs for one part's tables, keyed by table index: every `w:tbl` in the +/// part, 1-based, as the scanner counts them. +pub type TableFixes = BTreeMap>; + +/// The list a typed marker is converted to, by the marker that was typed. +/// +/// Supplied by the adapter, which is the half of this that can read +/// `word/numbering.xml`; see [`bullet_list`]. +pub type Bullets = BTreeMap; + +/// Everything to change in one part. +#[derive(Debug, Default, Clone)] +pub struct PartPlan { + pub paragraphs: PartFixes, + pub tables: TableFixes, +} + +impl PartPlan { + /// How many repairs the plan carries. + pub fn len(&self) -> usize { + [&self.paragraphs, &self.tables] + .into_iter() + .flat_map(|fixes| fixes.values()) + .map(Vec::len) + .sum() + } + + pub fn is_empty(&self) -> bool { + self.len() == 0 + } +} + +// ---------------------------------------------------------------- schema order + +/// `CT_PPr`, in schema sequence order: `CT_PPrBase` followed by the paragraph +/// mark's run properties, the section properties and the revision record. +const PPR_ORDER: &[&str] = &[ + "w:pStyle", + "w:keepNext", + "w:keepLines", + "w:pageBreakBefore", + "w:framePr", + "w:widowControl", + "w:numPr", + "w:suppressLineNumbers", + "w:pBdr", + "w:shd", + "w:tabs", + "w:suppressAutoHyphens", + "w:kinsoku", + "w:wordWrap", + "w:overflowPunct", + "w:topLinePunct", + "w:autoSpaceDE", + "w:autoSpaceDN", + "w:bidi", + "w:adjustRightInd", + "w:snapToGrid", + "w:spacing", + "w:ind", + "w:contextualSpacing", + "w:mirrorIndents", + "w:suppressOverlap", + "w:jc", + "w:textDirection", + "w:textAlignment", + "w:textboxTightWrap", + "w:outlineLvl", + "w:divId", + "w:cnfStyle", + "w:rPr", + "w:sectPr", + "w:pPrChange", +]; + +/// `CT_RPr`, in schema sequence order, with the four revision markers +/// `CT_ParaRPr` puts in front of the same sequence. +/// +/// One list for both, because the paragraph mark's run properties and a run's +/// carry the same elements in the same order; the markers simply never occur +/// in a `w:rPr` that is not a paragraph mark's, and an element that never +/// occurs cannot be ranked wrongly. +const RPR_ORDER: &[&str] = &[ + "w:ins", + "w:del", + "w:moveFrom", + "w:moveTo", + "w:rStyle", + "w:rFonts", + "w:b", + "w:bCs", + "w:i", + "w:iCs", + "w:caps", + "w:smallCaps", + "w:strike", + "w:dstrike", + "w:outline", + "w:shadow", + "w:emboss", + "w:imprint", + "w:noProof", + "w:snapToGrid", + "w:vanish", + "w:webHidden", + "w:color", + "w:spacing", + "w:w", + "w:kern", + "w:position", + "w:sz", + "w:szCs", + "w:highlight", + "w:u", + "w:effect", + "w:bdr", + "w:shd", + "w:fitText", + "w:vertAlign", + "w:rtl", + "w:cs", + "w:em", + "w:lang", + "w:eastAsianLayout", + "w:specVanish", + "w:oMath", + "w:rPrChange", +]; + +/// `CT_TblPrBase`, in schema sequence order, with the revision record +/// `CT_TblPr` adds after it. +const TBLPR_ORDER: &[&str] = &[ + "w:tblStyle", + "w:tblpPr", + "w:tblOverlap", + "w:bidiVisual", + "w:tblStyleRowBandSize", + "w:tblStyleColBandSize", + "w:tblW", + "w:jc", + "w:tblCellSpacing", + "w:tblInd", + "w:tblBorders", + "w:shd", + "w:tblLayout", + "w:tblCellMar", + "w:tblLook", + "w:tblCaption", + "w:tblDescription", + "w:tblPrChange", +]; + +/// `CT_Tbl`: the properties, then the grid, then the rows. +const TBL_ORDER: &[&str] = &["w:tblPr", "w:tblGrid", "w:tr"]; + +/// `CT_NumPr`, in schema sequence order. +const NUMPR_ORDER: &[&str] = &["w:ilvl", "w:numId", "w:numberingChange", "w:ins"]; + +/// `CT_P`: `w:pPr` precedes every run. +const P_ORDER: &[&str] = &["w:pPr"]; + +/// `CT_R`: `w:rPr` precedes the run's content. +const R_ORDER: &[&str] = &["w:rPr"]; + +/// The element a run's characters live in. +const W_T: &str = "w:t"; + +/// Regions of a part whose paragraphs are not the paragraph being repaired: +/// the text boxes that nest whole units inside a run, and the fallback +/// [`crate::docx`] steps over. +const NESTED: &[&str] = &["w:txbxContent", "mc:Fallback"]; + +/// Regions of a part that [`crate::docx`] does not count, and so regions this +/// module must not count either. +/// +/// A `w:txbxContent` is missing on purpose: the paragraphs in a text box *are* +/// counted by the scanner and are units of their own, and only their text and +/// their run properties belong to the inner paragraph rather than the outer. +const UNCOUNTED: &[&str] = &["mc:Fallback"]; + +// ------------------------------------------------------------------- lexemes + +/// An `ST_OnOff` value, written out in full. +/// +/// Never the empty element: `` is *on*, so a left-to-right repair has +/// to say `w:val="0"` or it writes the opposite of what it was asked for. +fn on_off(direction: Direction) -> &'static str { + if direction == Direction::Rtl { + "1" + } else { + "0" + } +} + +/// Lower an alignment onto `w:jc`, which names the paragraph's own start and +/// end edges rather than the page's. +/// +/// `None` for a physical edge. Word evaluates `left` and `right` against the +/// paragraph's `w:bidi`, so there is no value here that means "the left of the +/// page whatever the direction"; writing the relative spelling that happens to +/// land left today would be answering a different question from the one asked, +/// and one that comes apart the moment the direction changes. +fn word_alignment(alignment: Alignment) -> Option<&'static str> { + Some(match alignment { + Alignment::Start => "start", + Alignment::End => "end", + Alignment::Center => "center", + Alignment::Justify => "both", + Alignment::Distributed => "distribute", + Alignment::Left | Alignment::Right => return None, + }) +} + +// ------------------------------------------------------------------ numbering + +/// One attribute of the tag at `at`. +fn attribute(events: &[Event<'static>], at: usize, name: &str) -> Option { + match &events[at] { + Event::Start(tag) | Event::Empty(tag) => get_attribute(tag, name), + _ => None, + } +} + +/// The direct children of the element at `at`, as absolute indices. +fn children(events: &[Event<'static>], at: usize) -> Vec { + let range = element_range(events, at); + token::direct_children(&events[range.clone()]) + .into_iter() + .map(|i| range.start + i) + .collect() +} + +/// The `w:numId` of a list in `word/numbering.xml` that draws a bullet, +/// preferring one whose first level draws `marker` itself. +/// +/// A repair converting a typed glyph into a real list has to point at a +/// definition, and this is the only honest place to get one: the definitions +/// the document already carries. Preferring the marker the author typed keeps +/// the repaired page looking like the page they were aiming at; falling back +/// to any bulleted list keeps the repair possible when it does not, because +/// the defect being fixed is the glyph in the text and not which glyph it is. +/// +/// `None` when the document defines no bulleted list, which is a refusal +/// rather than a licence to invent one: a `w:num` this module wrote would name +/// an `w:abstractNum` it also wrote, and neither is a repair to a paragraph. +/// +/// A `w:lvlOverride` inside a `w:num` is not read. It can restate a level's +/// format, and a definition this function accepted on the strength of its +/// abstract while an override said otherwise would be a list drawing something +/// else — so only a `w:num` whose abstract says `bullet` and which overrides +/// nothing at level zero is offered. +pub fn bullet_list(part: &str, xml: &str, marker: char) -> Result> { + let events = token::read_events(part, xml)?; + + // Abstract definitions whose first level draws a bullet, and the glyph + // each of them draws. + let mut abstracts: BTreeMap> = BTreeMap::new(); + for at in token::element_starts_outside(&events, "w:abstractNum", &[]) { + let Some(id) = attribute(&events, at, "w:abstractNumId") else { + continue; + }; + let Some(level) = children(&events, at).into_iter().find(|&i| { + token::name_of(&events[i]).as_deref() == Some("w:lvl") + && attribute(&events, i, "w:ilvl").as_deref() == Some("0") + }) else { + continue; + }; + let format = find_direct_child(&events, level, "w:numFmt") + .and_then(|i| attribute(&events, i, "w:val")); + if format.as_deref() != Some("bullet") { + continue; + } + let text = find_direct_child(&events, level, "w:lvlText") + .and_then(|i| attribute(&events, i, "w:val")); + abstracts.insert(id, text); + } + + // The lists pointing at them, in document order, with the ones that draw + // the typed marker first. + let typed = marker.to_string(); + let mut exact = None; + let mut any = None; + for at in token::element_starts_outside(&events, "w:num", &[]) { + let Some(num_id) = attribute(&events, at, "w:numId") else { + continue; + }; + if children(&events, at) + .into_iter() + .any(|i| token::name_of(&events[i]).as_deref() == Some("w:lvlOverride")) + { + continue; + } + let Some(abstract_id) = find_direct_child(&events, at, "w:abstractNumId") + .and_then(|i| attribute(&events, i, "w:val")) + else { + continue; + }; + let Some(drawn) = abstracts.get(&abstract_id) else { + continue; + }; + let slot = if drawn.as_deref() == Some(typed.as_str()) { + &mut exact + } else { + &mut any + }; + if slot.is_none() { + *slot = Some(num_id); + } + } + + Ok(exact.or(any)) +} + +// ------------------------------------------------------------------- editing + +/// The paragraph's `w:pPr`, created in schema position if it has none. +fn ensure_ppr(para: &mut Vec>) -> usize { + child_or_insert(para, 0, P_ORDER, "w:pPr") +} + +/// The content ranges of the runs that are this paragraph's own. +fn own_text(para: &[Event<'static>]) -> Vec> { + token::text_content_ranges_outside(para, W_T, NESTED) +} + +/// Apply `f` to every run-property element the paragraph owns, creating +/// `w:rPr` for runs that lack one. +/// +/// Runs are visited back to front: creating a `w:rPr` shifts every index after +/// it, and the ones still to be visited must not move. The paragraph mark's +/// own properties come last, because `w:pPr` sits in front of every run and so +/// does not move when one of them gains a child — and they are edited only +/// where they already exist, matching how the DrawingML rewriter treats an +/// `a:endParaRPr`: the mark is the pilcrow's formatting, and a paragraph that +/// never stated any is not made to. +fn for_each_run_property( + para: &mut Vec>, + mut f: impl FnMut(&mut Vec>, usize), +) { + for at in token::element_starts_outside(para, "w:r", NESTED) + .into_iter() + .rev() + { + let rpr = child_or_insert(para, at, R_ORDER, "w:rPr"); + f(para, rpr); + } + + if let Some(ppr) = find_direct_child(para, 0, "w:pPr") + && let Some(rpr) = find_direct_child(para, ppr, "w:rPr") + { + f(para, rpr); + } +} + +/// Mark the runs whose text changed as significant-whitespace, where it now +/// is. +/// +/// WordprocessingML is the one vocabulary here where this matters. A `w:t` +/// without `xml:space="preserve"` has its leading and trailing whitespace +/// collapsed away, so a repair that strips a typed `•` and leaves the space +/// behind it — or deletes a control character that was standing in front of +/// one — would silently lose a space it never meant to touch. Only runs this +/// repair actually rewrote are marked, so nothing else in the paragraph moves. +fn preserve_space(para: &mut Vec>, before: &[String]) { + let ranges = own_text(para); + if ranges.len() != before.len() { + return; + } + let after = token::read_runs(para, &ranges); + for (i, range) in ranges.iter().enumerate() { + let changed = before[i] != after[i]; + let padded = + after[i].starts_with(char::is_whitespace) || after[i].ends_with(char::is_whitespace); + if changed && padded && range.start > 0 { + edit_tag(para, range.start - 1, |tag| { + set_attribute(tag, "xml:space", "preserve") + }); + } + } +} + +/// Apply every repair for one paragraph. +fn apply_to_paragraph( + part: &str, + para: &mut Vec>, + fixes: &[Fix], + bullets: &Bullets, +) -> Result<()> { + // Text first, on the same terms the DrawingML rewriter states: these + // replace text events in place, so they neither move nor are moved by the + // structural edits that follow; deletions come before anything else + // whatever order the fixes arrived in, because both kinds carry byte + // offsets into the text as it was scanned; and both sets are deleted in + // one pass, because they index the same original string. + let text_before = token::read_runs(para, &own_text(para)); + + let deletions: Vec = fixes + .iter() + .filter_map(|fix| match fix { + Fix::RemoveControls(offsets) | Fix::RemoveTatweel(offsets) => Some(offsets), + _ => None, + }) + .flatten() + .copied() + .collect(); + if !deletions.is_empty() { + let ranges = own_text(para); + token::remove_at_offsets_in(para, &ranges, &deletions); + } + if fixes.contains(&Fix::NormalizePresentationForms) { + let ranges = own_text(para); + token::map_runs_in(para, &ranges, script::normalize_presentation_forms); + } + for fix in fixes { + if let Fix::ConvertLiteralBullet { marker } = fix { + let ranges = own_text(para); + token::strip_leading_marker_in(para, &ranges, *marker); + } + } + preserve_space(para, &text_before); + + for fix in fixes { + match fix { + Fix::SetDirection(direction) => { + let ppr = ensure_ppr(para); + let bidi = child_or_insert(para, ppr, PPR_ORDER, "w:bidi"); + edit_tag(para, bidi, |tag| { + set_attribute(tag, "w:val", on_off(*direction)) + }); + } + + Fix::SetAlignment(alignment) => { + let Some(value) = word_alignment(*alignment) else { + return Err(Error::Format(format!( + "{part}: cannot {fix}; w:jc names the paragraph's own start \ + and end edges, and Word has no spelling for a physical one" + ))); + }; + let ppr = ensure_ppr(para); + let jc = child_or_insert(para, ppr, PPR_ORDER, "w:jc"); + edit_tag(para, jc, |tag| set_attribute(tag, "w:val", value)); + } + + // The complex-script language, `w:lang/@w:bidi`. `@w:val` is the + // Latin one and is left exactly as it was, for the reason + // [`crate::docx`] reads the one and not the other: Arabic tagged + // `en-US` in `@w:val` and `ar-SA` in `@w:bidi` is correctly + // tagged, and this repair has nothing to say about the Latin. + Fix::SetLanguage(tag_value) => { + for_each_run_property(para, |events, at| { + let lang = child_or_insert(events, at, RPR_ORDER, "w:lang"); + edit_tag(events, lang, |tag| set_attribute(tag, "w:bidi", tag_value)); + }); + } + + // `w:rFonts/@w:cs`, and the theme reference beside it goes. + // `@w:cstheme` is what Word renders and `@w:cs` the resolved value + // it caches for consumers that do not implement themes; writing + // the typeface into the cache and leaving the reference standing + // would change what the file says and not what a reader sees. + Fix::SetComplexFont(typeface) => { + for_each_run_property(para, |events, at| { + let fonts = child_or_insert(events, at, RPR_ORDER, "w:rFonts"); + edit_tag(events, fonts, |tag| set_attribute(tag, "w:cs", typeface)); + edit_tag(events, fonts, |tag| remove_attribute(tag, "w:cstheme")); + }); + } + + // The marker itself was stripped above; what is left is to point + // the paragraph at a list. The indent comes from the numbering + // definition's own `w:lvl/w:pPr/w:ind`, so unlike DrawingML there + // is nothing for the paragraph to state about it. + Fix::ConvertLiteralBullet { marker } => { + let Some(num_id) = bullets.get(marker) else { + return Err(Error::Format(format!( + "{part}: cannot {fix}; this document defines no bulleted list \ + for a paragraph to join" + ))); + }; + let ppr = ensure_ppr(para); + let numpr = child_or_insert(para, ppr, PPR_ORDER, "w:numPr"); + let level = child_or_insert(para, numpr, NUMPR_ORDER, "w:ilvl"); + edit_tag(para, level, |tag| set_attribute(tag, "w:val", "0")); + let num = child_or_insert(para, numpr, NUMPR_ORDER, "w:numId"); + edit_tag(para, num, |tag| set_attribute(tag, "w:val", num_id)); + } + + Fix::RemoveControls(_) | Fix::RemoveTatweel(_) | Fix::NormalizePresentationForms => {} + } + } + + Ok(()) +} + +/// Apply every repair for one table, whose `w:tbl` starts at `at`. +/// +/// A table has one property of its own this tool reasons about — its +/// direction, `w:tblPr/w:bidiVisual`, which decides which side the first cell +/// in the file is displayed on. Anything else a plan names on a table is a +/// mistake upstream and is refused rather than guessed at. +fn apply_to_table( + part: &str, + events: &mut Vec>, + at: usize, + fixes: &[Fix], +) -> Result<()> { + for fix in fixes { + match fix { + Fix::SetDirection(direction) => { + let tblpr = child_or_insert(events, at, TBL_ORDER, "w:tblPr"); + let bidi = child_or_insert(events, tblpr, TBLPR_ORDER, "w:bidiVisual"); + edit_tag(events, bidi, |tag| { + set_attribute(tag, "w:val", on_off(*direction)) + }); + } + other => { + return Err(Error::Format(format!( + "{part}: cannot {other} on a table; only its direction is a table's own" + ))); + } + } + } + Ok(()) +} + +/// Every `w:p` in the part, counted as [`crate::docx`] counts them. +fn paragraph_ranges(events: &[Event<'static>]) -> Vec<(usize, Range)> { + element_ranges_outside(events, "w:p", UNCOUNTED) +} + +/// Every `w:tbl` in the part, on the same terms. +fn table_ranges(events: &[Event<'static>]) -> Vec<(usize, Range)> { + element_ranges_outside(events, "w:tbl", UNCOUNTED) +} + +/// The event range of the `index`th element of a kind, or the error that says +/// the report and the document have come apart. +fn locate( + part: &str, + ranges: Vec<(usize, Range)>, + index: usize, + what: &str, +) -> Result> { + ranges + .into_iter() + .find(|(i, _)| *i == index) + .map(|(_, range)| range) + .ok_or_else(|| { + Error::Format(format!( + "{part}: no {what} {index}; the document and the report disagree" + )) + }) +} + +/// Apply repairs to a part's paragraphs, leaving every token they do not +/// address untouched. +pub fn apply(part: &str, xml: &str, fixes: &PartFixes) -> Result { + let plan = PartPlan { + paragraphs: fixes.clone(), + ..Default::default() + }; + apply_plan(part, xml, &plan, &Bullets::new()) +} + +/// Apply a whole part's plan — paragraphs and tables — leaving every token it +/// does not address untouched. +/// +/// `bullets` names the list each typed marker is converted to; see +/// [`bullet_list`]. There is no inherited direction to pass: `w:jc` is +/// relative to the paragraph's own, so nothing here has to know it. +pub fn apply_plan(part: &str, xml: &str, plan: &PartPlan, bullets: &Bullets) -> Result { + if plan.is_empty() { + return token::passthrough(part, xml); + } + + let mut events = token::read_events(part, xml)?; + + // Every index the plan names, checked before anything is edited, so a plan + // that does not fit the document is refused whole rather than half-applied. + for index in plan.paragraphs.keys() { + locate(part, paragraph_ranges(&events), *index, "paragraph")?; + } + for index in plan.tables.keys() { + locate(part, table_ranges(&events), *index, "table")?; + } + + // Back to front: splicing a paragraph changes every index after it. The + // ranges are taken afresh each time rather than once, because a paragraph + // in a text box sits *inside* another paragraph, and repairing the inner + // one moves the end of the outer one's range. + for index in plan.paragraphs.keys().rev() { + let range = locate(part, paragraph_ranges(&events), *index, "paragraph")?; + let mut para: Vec> = events[range.clone()].to_vec(); + apply_to_paragraph(part, &mut para, &plan.paragraphs[index], bullets)?; + events.splice(range, para); + } + + // Tables after paragraphs, and again back to front: a paragraph edit + // inside a table moved its range, and a `w:tblPr` created on one table + // moves every table after it. + for index in plan.tables.keys().rev() { + let range = locate(part, table_ranges(&events), *index, "table")?; + apply_to_table(part, &mut events, range.start, &plan.tables[index])?; + } + + token::write_events(part, &events) +} diff --git a/crates/mirsam-ooxml/tests/word.rs b/crates/mirsam-ooxml/tests/word.rs new file mode 100644 index 0000000..0352d35 --- /dev/null +++ b/crates/mirsam-ooxml/tests/word.rs @@ -0,0 +1,790 @@ +//! WordprocessingML's repair vocabulary, and the `DocumentWriter` port over it. +//! +//! The first half is `rewrite.rs`'s opposite number: one test per `Fix`, each +//! asserting the **whole** rewritten part. Full-string equality is deliberate +//! for the reason stated there — "the diff contains exactly the intended change +//! and nothing else" is only actually asserted by a test that any unintended +//! byte fails. +//! +//! The second half is `writer.rs`'s: a repair planned against a unit id reaches +//! the paragraph it names, through the part it lives in, and nothing else in +//! the package moves. + +use mirsam_core::error::Error; +use mirsam_core::text::{Alignment, Direction}; +use mirsam_core::{DocumentReader, DocumentWriter, Fix, Repair, UnitId}; +use mirsam_ooxml::DocxDocument; +use mirsam_ooxml::word::{Bullets, PartFixes, PartPlan, apply, apply_plan, bullet_list}; +use std::io::{Read, Write}; +use std::path::{Path, PathBuf}; + +const PART: &str = "word/document.xml"; + +fn rewrite(xml: &str, fixes: Vec) -> String { + let mut part = PartFixes::new(); + part.insert(1, fixes); + apply(PART, xml, &part).expect("rewrite failed") +} + +fn assert_rewrite(input: &str, fixes: Vec, expected: &str) { + assert_eq!(rewrite(input, fixes), expected); +} + +/// The error a rewrite refused with. +fn refusal(xml: &str, fixes: Vec) -> String { + let mut part = PartFixes::new(); + part.insert(1, fixes); + match apply(PART, xml, &part) { + Err(Error::Format(message)) => message, + other => panic!("expected a refusal, got {other:?}"), + } +} + +// ------------------------------------------------------------------ direction + +#[test] +fn set_direction_creates_a_ppr_when_the_paragraph_has_none() { + // `w:pPr` is first in CT_P, before any run. + assert_rewrite( + r#"مرحبا"#, + vec![Fix::SetDirection(Direction::Rtl)], + r#"مرحبا"#, + ); +} + +#[test] +fn left_to_right_is_written_out_because_an_empty_bidi_element_means_on() { + // The whole reason `w:val` is always written. `` is *true*, so a + // repair setting a paragraph left-to-right by creating an empty element + // would write the opposite of what it was asked for. + assert_rewrite( + r#"hi"#, + vec![Fix::SetDirection(Direction::Ltr)], + r#"hi"#, + ); +} + +#[test] +fn set_direction_lands_in_schema_position_among_the_children_already_there() { + // CT_PPrBase is a sequence: `w:pStyle` precedes `w:bidi`, `w:bidi` + // precedes `w:jc`, and the paragraph mark's `w:rPr` comes after all three. + // An element in the wrong place is a document Word offers to repair. + assert_rewrite( + concat!( + r#""#, + r#""#, + r#"مرحبا"#, + ), + vec![Fix::SetDirection(Direction::Rtl)], + concat!( + r#""#, + r#""#, + r#"مرحبا"#, + ), + ); +} + +#[test] +fn set_direction_replaces_the_value_and_leaves_neighbouring_attributes_alone() { + // `w:val='0'` is single-quoted on the neighbour. Rebuilding the tag from + // parsed attributes would re-quote it — a change the repair never asked for. + assert_rewrite( + r#""#, + vec![Fix::SetDirection(Direction::Rtl)], + r#""#, + ); +} + +// ------------------------------------------------------------------ alignment + +#[test] +fn alignment_is_written_relative_because_word_reads_it_relative() { + // The claim that makes this rewriter simpler than DrawingML's: `w:jc` is + // evaluated against the paragraph's own `w:bidi`, so `Start` is `start` + // whichever way the paragraph runs and there is nothing to lower it + // against. Two paragraphs, opposite directions, one value. + for direction in ["0", "1"] { + assert_rewrite( + &format!(r#""#), + vec![Fix::SetAlignment(Alignment::Start)], + &format!( + r#""# + ), + ); + } +} + +#[test] +fn start_and_end_are_opposite_edges() { + let paragraph = r#""#; + assert!(rewrite(paragraph, vec![Fix::SetAlignment(Alignment::Start)]).contains(r#""start""#)); + assert!(rewrite(paragraph, vec![Fix::SetAlignment(Alignment::End)]).contains(r#""end""#)); +} + +#[test] +fn a_physical_edge_is_refused_rather_than_lowered_onto_the_edge_it_means_today() { + // Word has no spelling for "the left of the page whatever the direction", + // which is the refusal the conformance suite states for the reading side. + // Writing `end` on a right-to-left paragraph would land left today and + // move the moment somebody changed the direction — a different claim from + // the one the fix makes. + for physical in [Alignment::Left, Alignment::Right] { + let message = refusal( + r#""#, + vec![Fix::SetAlignment(physical)], + ); + assert!(message.contains("physical"), "{message}"); + } +} + +#[test] +fn every_alignment_word_can_state_is_one_this_adapter_reads_back() { + // The two halves have to agree: a value written here that the reader + // lowered onto something else would make a repair look like it failed. + for (alignment, expected) in [ + (Alignment::Start, Alignment::Start), + (Alignment::End, Alignment::End), + (Alignment::Center, Alignment::Center), + (Alignment::Justify, Alignment::Justify), + (Alignment::Distributed, Alignment::Distributed), + ] { + let written = rewrite( + r#"مرحبا"#, + vec![Fix::SetAlignment(alignment)], + ); + let units = mirsam_ooxml::docx::scan_xml(PART, &written).expect("scan failed"); + assert_eq!( + units[0].props.alignment.effective().copied(), + Some(expected), + "{alignment:?} in {written}" + ); + } +} + +// ------------------------------------------------------------------- language + +#[test] +fn set_language_writes_the_complex_script_slot_of_every_run() { + assert_rewrite( + r#"مرحباألف"#, + vec![Fix::SetLanguage("ar-SA".into())], + concat!( + r#"مرحبا"#, + r#"ألف"#, + ), + ); +} + +#[test] +fn set_language_leaves_the_latin_tag_exactly_as_it_was() { + // `@w:val` is the Latin language and `@w:bidi` the complex-script one. + // Arabic tagged `en-US` in the first and `ar-SA` in the second is + // correctly tagged, and this repair has nothing to say about the Latin. + assert_rewrite( + r#"مرحبا"#, + vec![Fix::SetLanguage("ar-SA".into())], + concat!( + r#""#, + r#"مرحبا"#, + ), + ); +} + +#[test] +fn the_paragraph_mark_is_edited_where_it_exists() { + assert_rewrite( + concat!( + r#""#, + r#"مرحبا"#, + ), + vec![Fix::SetLanguage("ar-SA".into())], + concat!( + r#""#, + r#"مرحبا"#, + ), + ); +} + +#[test] +fn the_paragraph_mark_is_not_created_where_it_does_not_exist() { + // The mark is the pilcrow's own formatting, and a paragraph that never + // stated any is not made to — the same restraint the DrawingML rewriter + // shows an absent `a:endParaRPr`. + let written = rewrite( + r#"مرحبا"#, + vec![Fix::SetLanguage("ar-SA".into())], + ); + assert_eq!( + written, + concat!( + r#""#, + r#"مرحبا"#, + ) + ); +} + +// ----------------------------------------------------------------------- font + +#[test] +fn set_complex_font_fills_the_slot_beside_the_latin_one() { + assert_rewrite( + r#"مرحبا"#, + vec![Fix::SetComplexFont("Dubai".into())], + concat!( + r#""#, + r#"مرحبا"#, + ), + ); +} + +#[test] +fn set_complex_font_removes_the_theme_reference_it_would_otherwise_contradict() { + // `@w:cstheme` is what Word renders and `@w:cs` the value it caches beside + // it. Writing the typeface into the cache and leaving the reference + // standing would change what the file says and not what a reader sees — + // a repair that reports success and fixes nothing. + assert_rewrite( + concat!( + r#""#, + r#"مرحبا"#, + ), + vec![Fix::SetComplexFont("Dubai".into())], + concat!( + r#""#, + r#"مرحبا"#, + ), + ); +} + +#[test] +fn set_complex_font_creates_the_run_properties_in_schema_position() { + // `w:rPr` is first in CT_R, and `w:rFonts` second in CT_RPr. + assert_rewrite( + r#"مرحبا"#, + vec![Fix::SetComplexFont("Dubai".into())], + r#"مرحبا"#, + ); +} + +// ----------------------------------------------------------------------- text + +#[test] +fn remove_controls_deletes_exactly_the_offsets_it_was_given() { + // U+202B RIGHT-TO-LEFT EMBEDDING at 0, U+202C POP at the end. + let text = "\u{202b}مرحبا\u{202c}"; + let last = text.len() - "\u{202c}".len(); + assert_rewrite( + &format!(r#"{text}"#), + vec![Fix::RemoveControls(vec![0, last])], + r#"مرحبا"#, + ); +} + +#[test] +fn normalizing_presentation_forms_leaves_a_run_that_has_none_verbatim() { + // The second run holds a character reference. A run the mapping does not + // change is not rewritten at all, so `م` survives as it was written + // rather than being resolved into the letter it names. + assert_rewrite( + r#"ﺍم"#, + vec![Fix::NormalizePresentationForms], + r#"ام"#, + ); +} + +#[test] +fn stripping_a_marker_that_leaves_whitespace_marks_the_run_as_significant() { + // WordprocessingML collapses a `w:t`'s leading and trailing whitespace + // unless the run says otherwise, so a repair that exposes a space has to + // say so or it silently deletes one it never meant to touch. + let mut plan = PartPlan::default(); + plan.paragraphs + .insert(1, vec![Fix::ConvertLiteralBullet { marker: '•' }]); + let bullets = Bullets::from([('•', "1".to_string())]); + + let written = apply_plan( + PART, + r#" • نمو"#, + &plan, + &bullets, + ) + .expect("rewrite failed"); + + assert_eq!( + written, + concat!( + r#""#, + r#" نمو"#, + ) + ); +} + +#[test] +fn a_run_this_repair_did_not_rewrite_keeps_the_tag_it_had() { + // The other half of the case above: only runs whose text actually changed + // are marked, so the second run's tag comes out byte for byte. + let mut plan = PartPlan::default(); + plan.paragraphs + .insert(1, vec![Fix::ConvertLiteralBullet { marker: '•' }]); + let bullets = Bullets::from([('•', "1".to_string())]); + + let written = apply_plan( + PART, + r#"• نمو في قطاع"#, + &plan, + &bullets, + ) + .expect("rewrite failed"); + + assert!(written.contains(r#" في قطاع"#), "{written}"); +} + +// -------------------------------------------------------------------- bullets + +#[test] +fn converting_a_typed_bullet_points_the_paragraph_at_a_list_the_document_defines() { + // DrawingML answers this with an attribute on the paragraph. Word cannot: + // a list is a reference into the numbering part, and the indent comes from + // the definition rather than from anything written here. + let mut plan = PartPlan::default(); + plan.paragraphs + .insert(1, vec![Fix::ConvertLiteralBullet { marker: '•' }]); + let bullets = Bullets::from([('•', "3".to_string())]); + + let written = apply_plan( + PART, + r#"• نمو"#, + &plan, + &bullets, + ) + .expect("rewrite failed"); + + assert_eq!( + written, + concat!( + r#""#, + r#"نمو"#, + ) + ); +} + +#[test] +fn a_typed_bullet_with_no_list_to_join_is_refused_rather_than_pointed_at_nothing() { + let message = refusal( + r#"• نمو"#, + vec![Fix::ConvertLiteralBullet { marker: '•' }], + ); + assert!(message.contains("no bulleted list"), "{message}"); +} + +const NUMBERING: &str = concat!( + r#""#, + // 0: a numbered list, which is not a bullet. + r#""#, + r#""#, + // 1: a bullet drawing a square. + r#""#, + r#""#, + // 2: a bullet drawing the marker people actually type. + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, +); + +#[test] +fn the_list_chosen_is_one_that_draws_the_marker_the_author_typed() { + assert_eq!( + bullet_list("word/numbering.xml", NUMBERING, '•').unwrap(), + Some("3".to_string()) + ); +} + +#[test] +fn any_bullet_list_will_do_when_none_draws_that_marker() { + // The defect being repaired is the glyph sitting in the text, not which + // glyph it is, so a marker no definition draws still gets converted — to + // the first bulleted list in the document, in document order. + assert_eq!( + bullet_list("word/numbering.xml", NUMBERING, '-').unwrap(), + Some("2".to_string()) + ); +} + +#[test] +fn a_numbered_list_is_not_a_bullet_list() { + let only_decimal = concat!( + r#""#, + r#""#, + r#""#, + ); + assert_eq!( + bullet_list("word/numbering.xml", only_decimal, '•').unwrap(), + None + ); +} + +#[test] +fn a_list_that_overrides_its_definition_is_not_offered() { + // A `w:lvlOverride` can restate the level's format, so a list accepted on + // the strength of its abstract while the override said otherwise would be + // a list drawing something else entirely. + let overridden = concat!( + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + ); + assert_eq!( + bullet_list("word/numbering.xml", overridden, '•').unwrap(), + None + ); +} + +// ------------------------------------------------------- what is not this unit + +#[test] +fn a_paragraph_in_a_text_box_is_not_rewritten_by_the_paragraph_around_it() { + // A `w:txbxContent` nests whole paragraphs inside a run. The inner one is + // a unit of its own, with its own text and its own ordinal, and its runs + // are not the outer paragraph's to edit — an offset the domain computed + // for the outer paragraph indexes the outer paragraph's runs alone. + let part = concat!( + r#""#, + r#"ﺍ"#, + r#"ﺍ"#, + ); + + let mut fixes = PartFixes::new(); + fixes.insert(1, vec![Fix::NormalizePresentationForms]); + let written = apply(PART, part, &fixes).expect("rewrite failed"); + + assert_eq!( + written, + concat!( + r#""#, + r#"ﺍ"#, + r#"ا"#, + ) + ); +} + +#[test] +fn a_nested_paragraph_is_repaired_under_its_own_ordinal() { + // The other side of the case above. The outer paragraph opens first, so it + // is paragraph 1 and the one inside the box is paragraph 2 — exactly how + // the reader numbers them. + let part = concat!( + r#""#, + r#"مرحبا"#, + r#""#, + ); + + let mut fixes = PartFixes::new(); + fixes.insert(2, vec![Fix::SetDirection(Direction::Rtl)]); + let written = apply(PART, part, &fixes).expect("rewrite failed"); + + assert_eq!( + written, + concat!( + r#""#, + r#"مرحبا"#, + r#""#, + ) + ); +} + +#[test] +fn paragraphs_in_a_fallback_are_not_counted_because_the_reader_does_not_read_them() { + // `mc:Fallback` spells out the same paragraph as the `mc:Choice` beside + // it, and the reader skips it. A rewriter that counted both would put + // paragraph 2's repair on the fallback's copy of paragraph 1 — a report + // and a document that had come apart while both looked right. + let part = concat!( + r#""#, + r#"مرحبا"#, + r#"مرحبا"#, + r#"ألف"#, + ); + + let mut fixes = PartFixes::new(); + fixes.insert(2, vec![Fix::SetDirection(Direction::Rtl)]); + let written = apply(PART, part, &fixes).expect("rewrite failed"); + + assert_eq!( + written, + concat!( + r#""#, + r#"مرحبا"#, + r#"مرحبا"#, + r#""#, + r#"ألف"#, + ) + ); +} + +// --------------------------------------------------------------------- tables + +fn table_plan(fixes: Vec) -> PartPlan { + let mut plan = PartPlan::default(); + plan.tables.insert(1, fixes); + plan +} + +#[test] +fn a_table_direction_lands_between_the_style_and_the_width() { + // CT_TblPrBase is a sequence: `w:tblStyle`, then `w:bidiVisual`, then + // `w:tblW`. + let written = apply_plan( + PART, + concat!( + r#""#, + r#""#, + ), + &table_plan(vec![Fix::SetDirection(Direction::Rtl)]), + &Bullets::new(), + ) + .expect("rewrite failed"); + + assert_eq!( + written, + concat!( + r#""#, + r#""#, + ) + ); +} + +#[test] +fn a_table_with_no_properties_gets_them_before_the_grid() { + let written = apply_plan( + PART, + r#""#, + &table_plan(vec![Fix::SetDirection(Direction::Rtl)]), + &Bullets::new(), + ) + .expect("rewrite failed"); + + assert_eq!( + written, + concat!( + r#""#, + r#""#, + ) + ); +} + +#[test] +fn anything_but_a_direction_on_a_table_is_refused() { + // A table's one property this tool reasons about is its column order. + // Anything else a plan names on one is a mistake upstream, and guessing + // at it would put a change in the document that no finding asked for. + let error = apply_plan( + PART, + r#""#, + &table_plan(vec![Fix::SetLanguage("ar-SA".into())]), + &Bullets::new(), + ); + let Err(Error::Format(message)) = error else { + panic!("a language on a table should be refused"); + }; + assert!(message.contains("only its direction"), "{message}"); +} + +#[test] +fn a_cells_paragraph_and_its_table_are_both_repaired_in_one_pass() { + // The composition the ordering exists for: editing the paragraph moves the + // table's range, so the table's own repair is located again afterwards. + let mut plan = table_plan(vec![Fix::SetDirection(Direction::Rtl)]); + plan.paragraphs + .insert(1, vec![Fix::SetDirection(Direction::Rtl)]); + + let written = apply_plan( + PART, + r#"مرحبا"#, + &plan, + &Bullets::new(), + ) + .expect("rewrite failed"); + + assert_eq!( + written, + concat!( + r#""#, + r#"مرحبا"#, + r#""#, + ) + ); +} + +// ---------------------------------------------------------------- the part + +#[test] +fn an_empty_plan_reproduces_the_part() { + let part = r#"مرحبا"#; + assert_eq!( + apply(PART, part, &PartFixes::new()).expect("rewrite failed"), + part + ); +} + +#[test] +fn a_plan_naming_a_paragraph_the_part_does_not_have_is_refused() { + let mut fixes = PartFixes::new(); + fixes.insert(9, vec![Fix::SetDirection(Direction::Rtl)]); + let Err(Error::Format(message)) = apply(PART, r#""#, &fixes) else { + panic!("a plan that does not fit the document should be refused"); + }; + assert!(message.contains("no paragraph 9"), "{message}"); +} + +// ------------------------------------------------------- the DocumentWriter port + +fn fixture(name: &str) -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../tests/fixtures") + .join(name) +} + +/// A scratch directory that removes itself. +struct Scratch(PathBuf); + +impl Scratch { + fn new(tag: &str) -> Self { + let dir = std::env::temp_dir().join(format!("mirsam-word-{tag}-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + Self(dir) + } + fn join(&self, name: &str) -> PathBuf { + self.0.join(name) + } +} + +impl Drop for Scratch { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } +} + +/// Every entry's name and raw compressed bytes, in package order, never +/// decompressed. +fn raw_entries(path: &Path) -> Vec<(String, Vec)> { + let file = std::fs::File::open(path).unwrap(); + let mut archive = zip::ZipArchive::new(std::io::BufReader::new(file)).unwrap(); + (0..archive.len()) + .map(|i| { + let mut entry = archive.by_index_raw(i).unwrap(); + let mut raw = Vec::new(); + entry.read_to_end(&mut raw).unwrap(); + (entry.name().to_string(), raw) + }) + .collect() +} + +#[test] +fn a_repair_reaches_the_paragraph_it_names_and_no_other_entry_moves() { + let scratch = Scratch::new("one-paragraph"); + let output = scratch.join("repaired.docx"); + + let mut document = DocxDocument::open(fixture("quarterly-review.docx")).unwrap(); + let staged = document + .apply(&[Repair::new( + &UnitId(format!("{PART}#p3")), + Fix::SetDirection(Direction::Rtl), + )]) + .unwrap(); + assert_eq!(staged, 1); + document.write(&output).unwrap(); + + let before = raw_entries(&fixture("quarterly-review.docx")); + let after = raw_entries(&output); + assert_eq!( + before.iter().map(|(n, _)| n).collect::>(), + after.iter().map(|(n, _)| n).collect::>(), + "the repair added or removed an entry" + ); + for ((name, was), (_, now)) in before.iter().zip(&after) { + if name == PART { + assert_ne!(was, now, "{name} carried the repair and did not change"); + } else { + assert_eq!(was, now, "{name} is not the part the repair named"); + } + } + + let mut repaired = DocxDocument::open(&output).unwrap(); + let units = repaired.scan().unwrap(); + let unit = units + .iter() + .find(|u| u.id.0 == format!("{PART}#p3")) + .unwrap(); + assert_eq!( + unit.props.direction.effective().copied(), + Some(Direction::Rtl) + ); +} + +#[test] +fn a_unit_id_this_adapter_did_not_issue_is_refused_and_stages_nothing() { + let mut document = DocxDocument::open(fixture("quarterly-review.docx")).unwrap(); + let error = document.apply(&[Repair::new( + &UnitId("ppt/slides/slide1.xml#cols1".into()), + Fix::SetDirection(Direction::Rtl), + )]); + let Err(Error::Format(message)) = error else { + panic!("an id from another adapter should be refused"); + }; + assert!( + message.contains("not a unit this adapter produced"), + "{message}" + ); +} + +#[test] +fn a_physical_edge_is_not_a_repair_this_format_supports() { + let document = DocxDocument::open(fixture("quarterly-review.docx")).unwrap(); + assert!(!document.supports(&Fix::SetAlignment(Alignment::Left))); + assert!(!document.supports(&Fix::SetAlignment(Alignment::Right))); + assert!(document.supports(&Fix::SetAlignment(Alignment::Start))); +} + +#[test] +fn a_typed_bullet_is_supported_exactly_where_the_document_defines_a_list() { + let with_list = DocxDocument::open(fixture("quarterly-review.docx")).unwrap(); + assert!(with_list.supports(&Fix::ConvertLiteralBullet { marker: '•' })); + + let scratch = Scratch::new("no-list"); + let without = scratch.join("plain.docx"); + write_minimal_package(&without); + let without = DocxDocument::open(&without).unwrap(); + assert!(!without.supports(&Fix::ConvertLiteralBullet { marker: '•' })); + // And everything else still is: the refusal is about the list, not about + // the document. + assert!(without.supports(&Fix::SetDirection(Direction::Rtl))); +} + +/// A Word package with one paragraph and no numbering part. +fn write_minimal_package(path: &Path) { + let file = std::fs::File::create(path).unwrap(); + let mut zip = zip::ZipWriter::new(file); + let options = zip::write::SimpleFileOptions::default(); + + zip.start_file("[Content_Types].xml", options).unwrap(); + zip.write_all( + br#""#, + ) + .unwrap(); + zip.start_file(PART, options).unwrap(); + zip.write_all( + concat!( + r#""#, + r#"• نمو"#, + ) + .as_bytes(), + ) + .unwrap(); + zip.finish().unwrap(); +} diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 8866c64..0c241f8 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -198,8 +198,13 @@ crates/ legend or a set of data labels draws, which are not paragraphs and which no DrawingML pass can see docx.rs WordprocessingML vocabulary: paragraphs and the properties - the rules judge. DocumentReader only — Word's writer is - not built, and the split ports are what let it wait + the rules judge, and the tables around them. + DocumentReader and DocumentWriter + word.rs WordprocessingML's repair vocabulary. What rewrite.rs is + for DrawingML, over a format whose w:jc is already + direction-relative — so unlike PowerPoint's, nothing here + has to be told which way a paragraph reads before it can + write a Start down style.rs Word's own chain, which is not a walk between parts: docDefaults, the styles a paragraph and its runs name, and the w:basedOn above each — all in one word/styles.xml. @@ -249,9 +254,9 @@ crates/ directory and a few kilobytes, not half a gigabyte of outlines — with ttf-parser decoding the records mirsam-cli/ driving adapter — argument parsing and rendering only - tests/golden.rs the golden corpus: every .pptx, .docx and .html under - tests/fixtures/ against its committed report of what the - binary finds, repairs and writes — or, for a format it + tests/golden.rs the golden corpus: every .pptx, .docx, .xlsx and .html + under tests/fixtures/ against its committed report of what + the binary finds, repairs and writes — or, for a format it reads but cannot write, of the refusal it gives instead ``` diff --git a/docs/PLAN.md b/docs/PLAN.md index 926d97d..dccca4f 100644 --- a/docs/PLAN.md +++ b/docs/PLAN.md @@ -886,12 +886,13 @@ document that never inherits anything would let a broken `StyleSheet` pass: contradicts the Arabic under it, and `RtlTable` supplies a column order the table does not state. -**`repair` refuses a `.docx`, and the corpus now records the refusal rather -than working around it.** A report for a readable format with no writer holds -the sentence the binary printed and the exit code it used — `2`, a usage -error, because the audit above it read the file perfectly well. The day the -Word writer lands, that shows up as a diff on a real document instead of as a -test somebody remembered to update. The three OPC-level invariants in +**`repair` refused a `.docx` when this item landed, and the corpus recorded +the refusal rather than working around it.** A report for a readable format +with no writer holds the sentence the binary printed and the exit code it used +— `2`, a usage error, because the audit above it read the file perfectly well. +That is exactly how §3.6 arrived: a repair report and a part-level diff on a +real document, rather than a test somebody remembered to update. The three +OPC-level invariants in `corpus_packages.rs` — content types, relationships resolving, ASCII item names — now run over every corpus document rather than the decks alone, which is the package layer being held to the guarantee the second format reuses it @@ -903,6 +904,82 @@ regenerates a corpus that now holds documents of both formats. The five committed reports changed in one line each — the key naming the file is `document` rather than `deck`, because the corpus is no longer only decks. +### 3.6 DOCX writer `[x]` +`DocumentWriter` on `DocxDocument`, and `mirsam repair report.docx fixed.docx` +stops refusing. `word.rs` is WordprocessingML's repair vocabulary — what +`rewrite.rs` is for DrawingML and `sheet.rs` for SpreadsheetML — and it is the +third module built on `token.rs` without a line of it moving, which is the +claim 3.1 made and this is the second test of. + +*Shape.* `docx.rs` groups repairs by the part and the ordinal a unit id names +and hands each part to `word::apply_plan`. Every mutation there is a `token.rs` +call with a WordprocessingML name in it: `w:pPr/w:bidi`, `w:pPr/w:jc`, +`w:rPr/w:lang@w:bidi`, `w:rPr/w:rFonts@w:cs`, `w:pPr/w:numPr` and +`w:tblPr/w:bidiVisual`, each placed by rank against the schema sequence its +parent is, because `CT_PPrBase` and `CT_RPr` are `xsd:sequence` and a correct +element in the wrong position is a document Word offers to repair. + +**The writer needs no inherited direction, and that is Word's doing rather +than a shortcut.** 1.2's hardest correction was that lowering `Start` onto +DrawingML's physical `algn` without knowing the paragraph's direction +reproduces the defect being repaired, which is why `rewrite::Inherited` exists +and why `PptxDocument::apply` re-runs its scanner over every part it touches. +`w:jc` is evaluated against the paragraph's own `w:bidi` ([MS-OE376] Part 4 +§2.3.1.13, note b), so `Start` is written `start` whichever way the paragraph +runs, and there is nothing here to resolve. The whole inheritance pass is +absent from this adapter, not skipped in it. + +**Two refusals, both the format's.** A *physical* edge is not something a Word +paragraph can state, which is the refusal the conformance suite already +records for the reading side; `supports` answers it for the writing side, so +`SetAlignment(Left)` is listed in `repairs.skipped` rather than lowered onto +whichever relative edge lands left today. And a typed bullet becomes a +`w:numPr` pointing into the numbering part, so it can only join a list the +document already defines — `package.rs` replaces the entries a package holds, +and a numbering part that is not there is a part, a content-type override and +a relationship, none of which is an edit to the paragraph the finding named. +`bullet_list` prefers a definition whose first level draws the marker the +author typed, falls back to any bulleted one, and refuses a list whose `w:num` +overrides its abstract at level zero. + +**Three smaller decisions, each a defect if taken the other way.** +`` is *on*, so a left-to-right repair writes `w:val="0"` and never an +empty element — creating one to mean *off* would turn the repair into the +defect it was sent to fix. Filling `@w:cs` **removes** `@w:cstheme` beside it: +the theme reference is what Word renders and `@w:cs` the value it caches for +consumers that do not implement themes, so writing into the cache and leaving +the reference standing is a repair that reports success and changes nothing a +reader sees. And a `w:t` whose text this repair changed is marked +`xml:space="preserve"` where the result has an edge of whitespace, because +stripping a typed `•` leaves the space behind it and Word would otherwise +collapse a space nobody asked to delete — only the runs actually rewritten are +marked, so the rest of the paragraph still comes out byte for byte. + +**What is not this paragraph.** `token.rs` gained the ability to take a +caller-chosen set of elements to *step over*, and Word is why: a +`w:txbxContent` nests whole paragraphs inside a run, and each is a unit with +its own ordinal, its own text and its own repairs. Offsets the domain computed +for the outer paragraph index the outer paragraph's runs alone. An +`mc:Fallback` is stepped over for the reader's reason — it spells out the same +paragraphs as the `mc:Choice` beside it, and a rewriter that counted both +would put paragraph 2's repair on the fallback's copy of paragraph 1, a report +and a document that had come apart while both looked right. + +*Acceptance:* met. `crates/mirsam-ooxml/tests/word.rs`, thirty-eight cases — +the rewriter asserted on the **whole** part, as 1.2's are, plus the +`DocumentWriter` port over a real package. The corpus is where it shows: both +`.docx` reports lost their `refused` line and gained a repair report, +`quarterly-review.docx` now has a `changed_parts` diff naming exactly the tags +above, and its `after` audit is empty — every fixable finding in a real Word +document cleared. `make validate-fixtures` accepts the repaired copy against +the ECMA-376 transitional schemas, and a second `repair` of that output stages +nothing and reproduces it byte for byte. + +*Not run.* "Word opens the repaired document without offering to repair it" is +the M1 application check in its Word form, and it cannot run in CI. Schema +validity is the strongest claim a machine can make here and is what the +acceptance above rests on; it is not the same claim. + --- ## M4 — Shaping `[x]` diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index a7c05e3..f6de3b4 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -91,13 +91,22 @@ DOCX reuses the OOXML package layer; only the vocabulary differs no line of `mirsam-core` changed to make DOCX pass, which was the question the suite existed to answer. Two Word documents joined the golden corpus. See [`PLAN.md`](PLAN.md) §3.5. -- The writer. Until it exists `repair` refuses a `.docx` as a readable format - without one, rather than as an unknown extension. The corpus records that - refusal, so the day it lands shows up as a diff on a real document. +- **Landed:** the writer. `mirsam repair report.docx fixed.docx` works, and + the corpus recorded the refusal it replaced, so it arrived as a diff on a + real document rather than as a test somebody remembered to update. The + interesting result is what the writer did *not* need: PowerPoint's rewriter + must be told which way each paragraph reads before it can write a `Start`, + because `algn` names physical edges, and Word's `w:jc` is already relative — + so the whole inheritance pass is absent from this adapter rather than + skipped in it. Two repairs are refused, and both refusals are the format's: + Word cannot state a physical edge, and a typed bullet can only join a list + the document already defines. See [`PLAN.md`](PLAN.md) §3.6. - Numbering and fields. -**Done when** the writer lands and `repair` stops refusing. The abstraction -question — does one suite hold both adapters unchanged — is answered. +**Done when** the writer lands and `repair` stops refusing. **Met.** The +abstraction question — does one suite hold both adapters unchanged — is +answered, and so is its writing half: three vocabularies now sit on one +token-rewrite scaffold. --- diff --git a/tests/fixtures/quarterly-review-correct.expected.json b/tests/fixtures/quarterly-review-correct.expected.json index e72925f..9fb6555 100644 --- a/tests/fixtures/quarterly-review-correct.expected.json +++ b/tests/fixtures/quarterly-review-correct.expected.json @@ -20,15 +20,55 @@ "warnings": 0 } }, - "changed_parts": null, + "changed_parts": {}, "document": "quarterly-review-correct.docx", "exit_codes": { "audit": 0, "audit --strict": 0, - "repair": 2, - "repair --strict": 2 + "repair": 0, + "repair --strict": 0 }, "repair": { - "refused": ".docx can be audited but not yet repaired; `mirsam audit` reads it, and the writer is scheduled in docs/ROADMAP.md" + "after": { + "diagnostics": [], + "summary": { + "arabic_units": 14, + "errors": 0, + "mixed_units": 3, + "notes": 0, + "units_scanned": 17, + "warnings": 0 + } + }, + "before": { + "diagnostics": [], + "summary": { + "arabic_units": 14, + "errors": 0, + "mixed_units": 3, + "notes": 0, + "units_scanned": 17, + "warnings": 0 + } + }, + "blocking": false, + "file": "quarterly-review-correct.docx", + "fonts": { + "checked": false + }, + "format": "docx", + "options": { + "align": true, + "complex_font": "Dubai", + "convert_bullets": true, + "language": "ar-SA", + "strip_tatweel": true + }, + "output": "quarterly-review-correct.repaired.docx", + "repairs": { + "applied": [], + "skipped": [] + }, + "strict": false } } diff --git a/tests/fixtures/quarterly-review.expected.json b/tests/fixtures/quarterly-review.expected.json index 95cf10c..9b63af6 100644 --- a/tests/fixtures/quarterly-review.expected.json +++ b/tests/fixtures/quarterly-review.expected.json @@ -244,15 +244,390 @@ "warnings": 5 } }, - "changed_parts": null, + "changed_parts": { + "word/document.xml": [ + "+", + "+", + "+", + "+", + "+", + "+", + "-", + "+", + "+", + "+", + "+", + "-", + "+", + "-", + "+", + "+", + "+", + "+", + "+", + "-• نمو في قطاع التجزئة", + "+نمو في قطاع التجزئة", + "-‫ارتفع الأداء بنسبة 25%‬", + "+ارتفع الأداء بنسبة 25%", + "-ﺍﻟﺘﻘﺮﻳﺮ ﺍﻟﻔﺼﻠﻲ", + "+التقرير الفصلي", + "+" + ] + }, "document": "quarterly-review.docx", "exit_codes": { "audit": 1, "audit --strict": 1, - "repair": 2, - "repair --strict": 2 + "repair": 0, + "repair --strict": 0 }, "repair": { - "refused": ".docx can be audited but not yet repaired; `mirsam audit` reads it, and the writer is scheduled in docs/ROADMAP.md" + "after": { + "diagnostics": [], + "summary": { + "arabic_units": 14, + "errors": 0, + "mixed_units": 3, + "notes": 0, + "units_scanned": 17, + "warnings": 0 + } + }, + "before": { + "diagnostics": [ + { + "evidence": { + "inherited_from": null, + "logical": "‫ارتفع الأداء بنسبة 25%‬", + "offenders": [ + "POP DIRECTIONAL FORMATTING", + "RIGHT-TO-LEFT EMBEDDING" + ], + "visual_declared": null, + "visual_expected": null + }, + "fixable": true, + "location": { + "container": null, + "paragraph": 7, + "part": "word/document.xml" + }, + "message": "2 explicit bidi control(s) embedded in text; set container direction instead", + "rule": "bidi-control", + "severity": "error", + "unit": "word/document.xml#p7" + }, + { + "evidence": { + "inherited_from": null, + "logical": "ارتفع الأداء بنسبة 25% في Q4 2026.", + "offenders": [], + "visual_declared": "يف %25 ةبسنب ءادألا عفترا Q4 2026.", + "visual_expected": ".Q4 2026 يف %25 ةبسنب ءادألا عفترا" + }, + "fixable": true, + "location": { + "container": null, + "paragraph": 3, + "part": "word/document.xml" + }, + "message": "renders as ltr but reads as rtl; visual order differs from the logical text", + "rule": "direction-mismatch", + "severity": "error", + "unit": "word/document.xml#p3" + }, + { + "evidence": { + "inherited_from": null, + "logical": "ﺍﻟﺘﻘﺮﻳﺮ ﺍﻟﻔﺼﻠﻲ", + "offenders": [ + "U+FE8D", + "U+FE98", + "U+FEAE", + "U+FEBC", + "U+FED4", + "U+FED8", + "U+FEDF", + "U+FEE0", + "U+FEF2", + "U+FEF3" + ], + "visual_declared": null, + "visual_expected": null + }, + "fixable": true, + "location": { + "container": null, + "paragraph": 8, + "part": "word/document.xml" + }, + "message": "13 pre-shaped presentation form(s); text is not searchable or reflowable", + "rule": "presentation-forms", + "severity": "error", + "unit": "word/document.xml#p8" + }, + { + "evidence": { + "inherited_from": null, + "logical": null, + "offenders": [], + "visual_declared": null, + "visual_expected": null + }, + "fixable": true, + "location": { + "container": null, + "paragraph": 5, + "part": "word/document.xml" + }, + "message": "Latin font specified without a complex-script font; Arabic will be substituted", + "rule": "complex-font-missing", + "severity": "warning", + "unit": "word/document.xml#p5" + }, + { + "evidence": { + "inherited_from": null, + "logical": "المؤشر\nالربع الثالث\nالربع الرابع\nالإيرادات\n12.4\n15.5", + "offenders": [], + "visual_declared": null, + "visual_expected": null + }, + "fixable": true, + "location": { + "container": null, + "paragraph": null, + "part": "word/document.xml" + }, + "message": "table declares no direction; its cells read rtl, so the columns run the wrong way", + "rule": "container-direction", + "severity": "warning", + "unit": "word/document.xml#tbl1" + }, + { + "evidence": { + "inherited_from": "word/styles.xml style[EnglishBody]/pPr@bidi", + "logical": null, + "offenders": [], + "visual_declared": null, + "visual_expected": null + }, + "fixable": true, + "location": { + "container": null, + "paragraph": 4, + "part": "word/document.xml" + }, + "message": "no base direction of its own; inherits ltr, but this reads as rtl", + "rule": "direction-unset", + "severity": "warning", + "unit": "word/document.xml#p4" + }, + { + "evidence": { + "inherited_from": null, + "logical": null, + "offenders": [], + "visual_declared": null, + "visual_expected": null + }, + "fixable": true, + "location": { + "container": null, + "paragraph": 2, + "part": "word/document.xml" + }, + "message": "Arabic text tagged as none; expected an ar-* language tag", + "rule": "language-missing", + "severity": "warning", + "unit": "word/document.xml#p2" + }, + { + "evidence": { + "inherited_from": null, + "logical": null, + "offenders": [], + "visual_declared": null, + "visual_expected": null + }, + "fixable": true, + "location": { + "container": null, + "paragraph": 6, + "part": "word/document.xml" + }, + "message": "paragraph begins with a typed '•'; use a native list instead", + "rule": "literal-bullet", + "severity": "warning", + "unit": "word/document.xml#p6" + }, + { + "evidence": { + "inherited_from": null, + "logical": "التقرير الفصلي للربع الرابع", + "offenders": [], + "visual_declared": null, + "visual_expected": null + }, + "fixable": true, + "location": { + "container": null, + "paragraph": 2, + "part": "word/document.xml" + }, + "message": "no alignment declared; a left-to-right layout places this on the left edge", + "rule": "alignment-unset", + "severity": "note", + "unit": "word/document.xml#p2" + }, + { + "evidence": { + "inherited_from": null, + "logical": "ارتفع الأداء بنسبة 25% في Q4 2026.", + "offenders": [], + "visual_declared": null, + "visual_expected": null + }, + "fixable": true, + "location": { + "container": null, + "paragraph": 3, + "part": "word/document.xml" + }, + "message": "no alignment declared; a left-to-right layout places this on the left edge", + "rule": "alignment-unset", + "severity": "note", + "unit": "word/document.xml#p3" + }, + { + "evidence": { + "inherited_from": null, + "logical": "التقرير الفصلي للربع الرابع", + "offenders": [], + "visual_declared": null, + "visual_expected": null + }, + "fixable": true, + "location": { + "container": null, + "paragraph": 4, + "part": "word/document.xml" + }, + "message": "no alignment declared; a left-to-right layout places this on the left edge", + "rule": "alignment-unset", + "severity": "note", + "unit": "word/document.xml#p4" + } + ], + "summary": { + "arabic_units": 13, + "errors": 3, + "mixed_units": 3, + "notes": 3, + "units_scanned": 17, + "warnings": 5 + } + }, + "blocking": false, + "file": "quarterly-review.docx", + "fonts": { + "checked": false + }, + "format": "docx", + "options": { + "align": true, + "complex_font": "Dubai", + "convert_bullets": true, + "language": "ar-SA", + "strip_tatweel": true + }, + "output": "quarterly-review.repaired.docx", + "repairs": { + "applied": [ + { + "fix": { + "kind": "set_alignment", + "value": "start" + }, + "unit": "word/document.xml#p2" + }, + { + "fix": { + "kind": "set_language", + "value": "ar-SA" + }, + "unit": "word/document.xml#p2" + }, + { + "fix": { + "kind": "set_direction", + "value": "rtl" + }, + "unit": "word/document.xml#p3" + }, + { + "fix": { + "kind": "set_alignment", + "value": "start" + }, + "unit": "word/document.xml#p3" + }, + { + "fix": { + "kind": "set_direction", + "value": "rtl" + }, + "unit": "word/document.xml#p4" + }, + { + "fix": { + "kind": "set_alignment", + "value": "start" + }, + "unit": "word/document.xml#p4" + }, + { + "fix": { + "kind": "set_complex_font", + "value": "Dubai" + }, + "unit": "word/document.xml#p5" + }, + { + "fix": { + "kind": "convert_literal_bullet", + "value": { + "marker": "•" + } + }, + "unit": "word/document.xml#p6" + }, + { + "fix": { + "kind": "remove_controls", + "value": [ + 0, + 41 + ] + }, + "unit": "word/document.xml#p7" + }, + { + "fix": { + "kind": "normalize_presentation_forms" + }, + "unit": "word/document.xml#p8" + }, + { + "fix": { + "kind": "set_direction", + "value": "rtl" + }, + "unit": "word/document.xml#tbl1" + } + ], + "skipped": [] + }, + "strict": false } }