From 581802cc4d501d969b0591de1e86c6a62b17a8c6 Mon Sep 17 00:00:00 2001 From: Duc-Tam Nguyen Date: Thu, 6 Aug 2026 10:38:45 +0700 Subject: [PATCH] Keep the doctype so saved pages are not quirks mode (issue #16) kage serialises a rendered page as the outerHTML of . A doctype is a sibling of rather than a child, so it was never in that string and every page kage has ever written came out without one. A document with no doctype is quirks mode in every browser. The box model reverts to the pre-CSS2 IE one and line-height, table cell inheritance and vertical-align all change, so the saved copy lays out differently from the original. It also weakens the that ensureCharset guarantees: in standards mode that declaration is authoritative, while in quirks mode a browser is readier to fall back to its locale encoding and mojibake the text. That is the symptom reported in issue #16, and a webview or e-reader with no encoding menu has no way back from it. Read the doctype from the DOM after the render and put it back. It is reproduced exactly rather than replaced with , because the string itself selects the rendering mode: HTML 4.01 Transitional is standards mode with its system identifier and quirks mode without it. A page that genuinely had no doctype on the live web still gets none, so it keeps rendering the way its author saw it. The parts come from an untrusted page and land at the top of a file we write, so the source form is assembled in Go and anything that could close the token early or carry markup is dropped. The banner comment now goes after the doctype so the doctype is the first thing in the file. --- CHANGELOG.md | 10 ++++ browser/pool.go | 102 +++++++++++++++++++++++++++++++++++++- browser/pool_test.go | 98 ++++++++++++++++++++++++++++++++++++ sanitize/sanitize.go | 14 ++++-- sanitize/sanitize_test.go | 47 ++++++++++++++++++ 5 files changed, 266 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9a3458e..fcd0731 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,16 @@ All notable changes to kage are recorded here. The format follows ## [Unreleased] +### Fixed + +- Saved pages keep their `` instead of rendering in quirks mode ([#16](https://github.com/tamnd/kage/issues/16)). + kage serialises a rendered page as the outerHTML of ``, and a doctype is a sibling of `` rather than a child, so it was never in that string and every page kage has ever written came out without one. + A document with no doctype is quirks mode in every browser: the box model reverts to the pre-CSS2 IE one and `line-height`, table cell inheritance and `vertical-align` all change, so the saved copy laid out differently from the original, and the `` declaration lost its authority, leaving a reader free to fall back to its locale encoding and mojibake every multibyte character. + That is the encoding problem reported in #16, and a webview or e-reader with no encoding menu has no way back from it. + The doctype is now read from the DOM and reproduced exactly rather than replaced with ``, because the string itself selects the rendering mode: HTML 4.01 Transitional is standards mode with its system identifier and quirks mode without it. + A page that genuinely had no doctype on the live web still gets none, so it keeps rendering the way its author saw it. +- The `cloned by kage` banner comment is written after the doctype rather than before it, so the doctype stays the first thing in the file. + ## [0.3.11] - 2026-08-01 ### Fixed diff --git a/browser/pool.go b/browser/pool.go index 74c5d9a..baadda9 100644 --- a/browser/pool.go +++ b/browser/pool.go @@ -7,6 +7,7 @@ package browser import ( "context" + "encoding/json" "fmt" "os" "runtime" @@ -17,6 +18,7 @@ import ( "github.com/go-rod/rod/lib/proto" "github.com/tamnd/kage/internal/rod" "github.com/tamnd/kage/internal/stealth" + "golang.org/x/net/html" ) // Options configure a Pool. @@ -145,12 +147,17 @@ func (p *Pool) Render(ctx context.Context, rawURL string) (RenderResult, error) settle(page, p.opts.Settle) } - html, err := page.HTML() + doc, err := page.HTML() if err != nil { return RenderResult{}, fmt.Errorf("serialise %s: %w", rawURL, err) } + // page.HTML() is the outerHTML of , which cannot contain the doctype, + // so put it back (issue #16). + if dt := pageDoctype(page); dt != "" { + doc = dt + "\n" + doc + } - res := RenderResult{HTML: html, FinalURL: rawURL} + res := RenderResult{HTML: doc, FinalURL: rawURL} if info, err := page.Info(); err == nil && info != nil { res.FinalURL = info.URL res.Title = info.Title @@ -158,6 +165,97 @@ func (p *Pool) Render(ctx context.Context, rawURL string) (RenderResult, error) return res, nil } +// doctypeJS reads the parts of document.doctype. It returns them as a JSON +// array rather than a ready-made string so the source form is assembled in Go, +// where a hostile page cannot influence it. +const doctypeJS = `() => { + const d = document.doctype; + return d ? JSON.stringify([d.name, d.publicId, d.systemId]) : ""; +}` + +// pageDoctype returns the document's doctype in source form, or "" when the +// page has none or Chrome will not say. +// +// Chrome's serialisation of a page is the outerHTML of , and a doctype is +// a sibling of rather than a child, so it is never in that string. Left +// alone, every page kage saves comes out with no doctype and every browser +// renders it in quirks mode: the box model reverts to the pre-CSS2 IE one, so +// the saved copy lays out differently from the original, and the +// declaration loses its authority, so a reader is free to fall back to its +// locale encoding and mojibake the text. A reader with no encoding menu, a +// webview or an e-reader, has no way back from that (issue #16). +// +// The doctype is reproduced exactly rather than replaced with , +// because the string itself selects the rendering mode: HTML 4.01 Transitional +// is standards mode with its system identifier and quirks mode without it. A +// page that was genuinely quirks mode on the live web keeps no doctype and so +// keeps rendering the way its author saw it. +func pageDoctype(page *rod.Page) string { + obj, err := page.Eval(doctypeJS) + if err != nil || obj == nil { + return "" + } + var parts []string + if err := json.Unmarshal([]byte(obj.Value.Str()), &parts); err != nil || len(parts) != 3 { + return "" + } + return renderDoctype(parts[0], parts[1], parts[2]) +} + +// renderDoctype rebuilds the source form of a doctype from its DOM parts with +// the same renderer that writes the saved page, so the two always agree. +// +// The parts arrive from an untrusted page and land at the very top of a file we +// write, so anything that does not look like a doctype a parser produced is +// dropped rather than escaped. x/net/html quotes the identifiers but does not +// escape a quote inside one, and it writes the name verbatim. +func renderDoctype(name, publicID, systemID string) string { + if !validDoctypeName(name) || !validDoctypeID(publicID) || !validDoctypeID(systemID) { + return "" + } + n := &html.Node{Type: html.DoctypeNode, Data: name} + if publicID != "" { + n.Attr = append(n.Attr, html.Attribute{Key: "public", Val: publicID}) + } + if systemID != "" { + n.Attr = append(n.Attr, html.Attribute{Key: "system", Val: systemID}) + } + var b strings.Builder + if err := html.Render(&b, n); err != nil { + return "" + } + return b.String() +} + +// validDoctypeName accepts the name of a doctype: ASCII letters only. In +// practice it is always "html", but "math" and "svg" are legal too. +func validDoctypeName(name string) bool { + if name == "" || len(name) > 32 { + return false + } + for _, r := range name { + if (r < 'a' || r > 'z') && (r < 'A' || r > 'Z') { + return false + } + } + return true +} + +// validDoctypeID accepts a public or system identifier: printable ASCII with no +// quote or angle bracket, which is every identifier any real doctype uses and +// nothing that could close the token early. +func validDoctypeID(id string) bool { + if len(id) > 256 { + return false + } + for _, r := range id { + if r < 0x20 || r > 0x7e || r == '"' || r == '\'' || r == '<' || r == '>' { + return false + } + } + return true +} + // getBrowser lazily connects to or launches Chrome. func (p *Pool) getBrowser() (*rod.Browser, error) { p.mu.Lock() diff --git a/browser/pool_test.go b/browser/pool_test.go index 1fd1b89..176d9e2 100644 --- a/browser/pool_test.go +++ b/browser/pool_test.go @@ -247,3 +247,101 @@ func TestRenderRoutesNonHTML(t *testing.T) { } } } + +func TestRenderDoctype(t *testing.T) { + cases := []struct { + name, doctype, public, system, want string + }{ + {"html5", "html", "", "", ""}, + { + "html401 transitional", "html", + "-//W3C//DTD HTML 4.01 Transitional//EN", + "http://www.w3.org/TR/html4/loose.dtd", + ``, + }, + { + // No system identifier: the difference between standards mode and + // quirks mode for this doctype, so it has to survive verbatim. + "html401 no system", "html", + "-//W3C//DTD HTML 4.01//EN", "", + ``, + }, + { + "xhtml", "html", + "-//W3C//DTD XHTML 1.0 Strict//EN", + "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd", + ``, + }, + {"legacy compat", "html", "", "about:legacy-compat", ``}, + + // A page can patch its own DOM, and whatever comes back is written to the + // top of a file we save, so anything that could close the token early or + // carry markup is dropped rather than escaped. + {"no name", "", "", "", ""}, + {"name with markup", "html>