diff --git a/.github/workflows/idric-core.yml b/.github/workflows/idric-core.yml
index 0d3bffd..ce0fbd5 100644
--- a/.github/workflows/idric-core.yml
+++ b/.github/workflows/idric-core.yml
@@ -29,6 +29,9 @@ jobs:
- name: Install system dependencies through Grease
run: sh bin/ci_browser_foundation.grease install-dependencies
+ - name: Exercise ICU search and bounded network pre-decision
+ run: sh bin/ci_browser_foundation.grease exercise-prepaint-network
+
- name: Restore Idric compiler
id: idric-cache
uses: actions/cache/restore@v4
diff --git a/bin/ci_browser_foundation.grease b/bin/ci_browser_foundation.grease
index b3df9fe..f44975d 100755
--- a/bin/ci_browser_foundation.grease
+++ b/bin/ci_browser_foundation.grease
@@ -8,7 +8,7 @@ pdf_harvester_ref=${PDF_HARVESTER_REF:-77d85dc6f7e89d109cb9e06f42706bc34f00e4a3}
install_dependencies() {
sudo apt-get update
- sudo apt-get install -y chezscheme curl xmlstarlet busybox poppler-utils
+ sudo apt-get install -y chezscheme curl xmlstarlet busybox poppler-utils jq
}
build_idric() {
@@ -70,6 +70,16 @@ exercise_workbench() {
grep -Fx 'resident-at-10=10' /tmp/ib-workbench.txt
}
+exercise_prepaint_network() {
+ cd "$repository_root"
+ sh -n bin/icu_search.grease
+ sh -n bin/prepaint_predecision.grease
+ sh -n tests/test_icu_search.grease
+ sh -n tests/test_prepaint_predecision.grease
+ sh tests/test_icu_search.grease
+ sh tests/test_prepaint_predecision.grease
+}
+
build_information_programs() {
test -x "$idric_prefix/bin/idris2" || build_idric
cd "$repository_root/src"
@@ -100,6 +110,17 @@ exercise_information() {
grep -Fx 'phone-api-row=True' /tmp/ib-information-smoke.txt
grep -Fx 'phone-lines=8' /tmp/ib-information-smoke.txt
grep -Fx 'phone-width-bounded=True' /tmp/ib-information-smoke.txt
+ grep -Fx 'article-first-paragraph=True' /tmp/ib-information-smoke.txt
+ grep -Fx 'article-second-paragraph=True' /tmp/ib-information-smoke.txt
+ grep -Fx 'article-href=True' /tmp/ib-information-smoke.txt
+ grep -Fx 'article-inline-href=True' /tmp/ib-information-smoke.txt
+ grep -Fx 'article-inline-prefix=True' /tmp/ib-information-smoke.txt
+ grep -Fx 'article-inline-suffix=True' /tmp/ib-information-smoke.txt
+ grep -Fx 'article-linked-image=True' /tmp/ib-information-smoke.txt
+ grep -Fx 'article-image-empty-link=False' /tmp/ib-information-smoke.txt
+ grep -Fx 'plain-first-url=True' /tmp/ib-information-smoke.txt
+ grep -Fx 'plain-second-url=True' /tmp/ib-information-smoke.txt
+ grep -Fx 'cheap-url-rejects-relative=False' /tmp/ib-information-smoke.txt
sh -n bin/prepaint_arxiv_progressively.grease
sh -n tests/test_arxiv_progressive_prepaint.grease
sh tests/test_arxiv_progressive_prepaint.grease
@@ -149,11 +170,12 @@ case "${1:-}" in
exercise-information) exercise_information ;;
exercise-core) exercise_core ;;
exercise-workbench) exercise_workbench ;;
+ exercise-prepaint-network) exercise_prepaint_network ;;
exercise-scientific-media) exercise_scientific_media ;;
exercise-live-arxiv) exercise_live_arxiv ;;
exercise-live-arxiv-prepaint) exercise_live_arxiv_prepaint ;;
*)
- printf 'usage: %s {install-dependencies|build-idric|verify-pdf-harvester|exercise-information|exercise-core|exercise-workbench|exercise-scientific-media|exercise-live-arxiv|exercise-live-arxiv-prepaint}\n' "$0" >&2
+ printf 'usage: %s {install-dependencies|build-idric|verify-pdf-harvester|exercise-information|exercise-core|exercise-workbench|exercise-prepaint-network|exercise-scientific-media|exercise-live-arxiv|exercise-live-arxiv-prepaint}\n' "$0" >&2
exit 2
;;
esac
diff --git a/bin/icu_search.grease b/bin/icu_search.grease
new file mode 100755
index 0000000..cd037cd
--- /dev/null
+++ b/bin/icu_search.grease
@@ -0,0 +1,18 @@
+#!/bin/sh
+set -eu
+
+if test "$#" -eq 0; then
+ printf 'usage: %s SEARCH WORDS...\n' "$0" >&2
+ exit 2
+fi
+
+icu_command=${IB_ICU:-icu}
+jq_command=${IB_JQ:-jq}
+
+# With the ordinary space IFS, "$*" is the shell operation the original tiny
+# search command was remembering. PS1 is only the interactive prompt. jq owns
+# UTF-8 form encoding here; replacing %20 with + gives the familiar query form.
+query=$*
+encoded_query=$(printf '%s' "$query" | "$jq_command" -sRr @uri | sed 's/%20/+/g')
+
+exec "$icu_command" get "https://www.google.com/search?q=$encoded_query"
diff --git a/bin/prepaint_predecision.grease b/bin/prepaint_predecision.grease
new file mode 100755
index 0000000..6852304
--- /dev/null
+++ b/bin/prepaint_predecision.grease
@@ -0,0 +1,118 @@
+#!/bin/sh
+set -eu
+
+if test "$#" -ne 1; then
+ printf 'usage: %s URL\n' "$0" >&2
+ exit 2
+fi
+
+input_url=$1
+case $input_url in
+ http://*) authority_and_rest=${input_url#http://} ;;
+ https://*) authority_and_rest=${input_url#https://} ;;
+ *)
+ printf 'predecision: URL must begin with http:// or https://\n' >&2
+ exit 2
+ ;;
+esac
+
+authority=${authority_and_rest%%/*}
+authority=${authority%%\?*}
+authority=${authority%%\#*}
+case $authority in
+ ''|*@*|'['*)
+ printf 'predecision: unsupported or empty URL authority\n' >&2
+ exit 2
+ ;;
+esac
+host=${authority%%:*}
+
+dig_command=${IB_DIG:-dig}
+traceroute_command=${IB_TRACEROUTE:-traceroute}
+timeout_command=${IB_TIMEOUT:-timeout}
+
+work_directory=$(mktemp -d)
+cleanup() {
+ rm -rf "$work_directory"
+}
+trap cleanup EXIT HUP INT TERM
+
+dig_status=unavailable
+if command -v "$dig_command" >/dev/null 2>&1; then
+ if "$timeout_command" 3 "$dig_command" +tries=1 +time=1 +stats \
+ "$host" A "$host" NS > "$work_directory/dig.txt" 2>&1
+ then
+ dig_status=ok
+ else
+ dig_status=failed
+ fi
+else
+ : > "$work_directory/dig.txt"
+fi
+
+route_status=unavailable
+if command -v "$traceroute_command" >/dev/null 2>&1; then
+ if "$timeout_command" 10 "$traceroute_command" -n -m 8 -w 1 -q 1 \
+ "$host" > "$work_directory/traceroute.txt" 2>&1
+ then
+ route_status=ok
+ else
+ route_status=failed
+ fi
+else
+ : > "$work_directory/traceroute.txt"
+fi
+
+dns_milliseconds=$(awk '/Query time:/ { print $(NF - 1); exit }' "$work_directory/dig.txt")
+test -n "$dns_milliseconds" || dns_milliseconds=unknown
+address_records=$(awk '$4 == "A" || $4 == "AAAA" { amount += 1 } END { print amount + 0 }' \
+ "$work_directory/dig.txt")
+namespace_records=$(awk '$4 == "NS" { amount += 1 } END { print amount + 0 }' \
+ "$work_directory/dig.txt")
+route_hops=$(awk '/^[[:space:]]*[0-9]+[[:space:]]/ { hop = $1 } END { if (hop == "") print 0; else print hop }' \
+ "$work_directory/traceroute.txt")
+route_last_milliseconds=$(awk '
+ /^[[:space:]]*[0-9]+[[:space:]]/ {
+ for (field = 1; field <= NF; field += 1) {
+ if ($field == "ms" && field > 1) last = $(field - 1)
+ }
+ }
+ END { if (last == "") print "unknown"; else print last }
+' "$work_directory/traceroute.txt")
+
+known_asset_count=${IB_KNOWN_ASSET_COUNT:-unknown}
+known_asset_bytes=${IB_KNOWN_ASSET_BYTES:-unknown}
+assumed_bytes_per_second=${IB_ASSUMED_BYTES_PER_SECOND:-unknown}
+estimated_transfer_seconds=unknown
+decision=measure-assets-before-heavy-renderer
+
+case $known_asset_bytes:$assumed_bytes_per_second in
+ *[!0-9:]*|:*|*:0) ;;
+ *)
+ estimated_transfer_seconds=$((
+ (known_asset_bytes + assumed_bytes_per_second - 1) / assumed_bytes_per_second
+ ))
+ if test "$estimated_transfer_seconds" -ge 300; then
+ decision=keep-prepaint-unless-user-escalates
+ else
+ decision=prepaint-then-bounded-escalation
+ fi
+ ;;
+esac
+
+printf 'predecision\t1\n'
+printf 'url\t%s\n' "$input_url"
+printf 'host\t%s\n' "$host"
+printf 'dig-status\t%s\n' "$dig_status"
+printf 'dns-ms\t%s\n' "$dns_milliseconds"
+printf 'address-records\t%s\n' "$address_records"
+printf 'namespace-records\t%s\n' "$namespace_records"
+printf 'traceroute-status\t%s\n' "$route_status"
+printf 'route-hops\t%s\n' "$route_hops"
+printf 'route-last-ms\t%s\n' "$route_last_milliseconds"
+printf 'known-asset-count\t%s\n' "$known_asset_count"
+printf 'known-asset-bytes\t%s\n' "$known_asset_bytes"
+printf 'assumed-bytes-per-second\t%s\n' "$assumed_bytes_per_second"
+printf 'estimated-transfer-seconds\t%s\n' "$estimated_transfer_seconds"
+printf 'decision\t%s\n' "$decision"
+printf 'route-warning\t%s\n' 'DNS and traceroute are diagnostics, not page-load predictions.'
diff --git a/docs/prepaint-policy-boundary.md b/docs/prepaint-policy-boundary.md
new file mode 100644
index 0000000..f800643
--- /dev/null
+++ b/docs/prepaint-policy-boundary.md
@@ -0,0 +1,107 @@
+# Page pre-paint, browser policy, and workbench diagnostics
+
+The page pre-painter and the developer workbench answer different questions.
+They must not be merged into one screen merely because both can be rendered as
+dark text and rows.
+
+## Page pre-paint
+
+The page surface contains information that came from the selected resource:
+
+- article paragraphs and headings;
+- links and their resolved targets;
+- page forms and controls when the browser can represent their action honestly;
+- table/data rows;
+- already-fetched images, unchanged;
+- a small browser-owned search/navigation control outside page content.
+
+The page surface does not contain fixture-universe counts, hot/warm/cold tab
+graphs, deployment topology invented by a test, DNS traces, route hops, cache
+classification, or storage totals. A real page may itself contain a topology
+image; that is ordinary page content. The bundled APK sample should not use such
+an image because it falsely suggests that IB adds the diagnostic to every page.
+
+Opening a plain UTF-8 file is a real pre-paint path. Ordinary prose is paintable
+without an `ib-prepaint` envelope. A file containing one absolute HTTP(S) URL per
+line becomes a link list. A saved HTML file still belongs to the Idriç HTML
+extractor; the Android display harness must not grow a second HTML parser.
+
+The HTML extractor retains links nested inside article paragraphs and retains
+an enclosing anchor on an image as a linked-image item. A remote `src` is only a
+fetch candidate: the background ICU path must validate the response and replace
+it with an already-fetched source before the Android harness paints the image.
+
+## Browser policy and the pre-decision probe
+
+Before starting a heavyweight renderer, browser policy can inspect bounded
+evidence:
+
+- whether a cached pre-paint is already useful;
+- known resource count and known/declared bytes;
+- prior measurements for the site and device;
+- DNS lookup state;
+- a deliberately sampled route diagnostic;
+- whether the page clearly requires JavaScript or another missing capability.
+
+`bin/prepaint_predecision.grease` is a first shell boundary for this evidence.
+It gives `dig` three seconds and asks for A plus NS data. It gives `traceroute`
+ten seconds, eight hops, and one probe per hop. Missing, filtered, or failed
+diagnostics remain data; they do not block the cheap paint.
+
+DNS time and traceroute hop latency are not estimates of total page-load time.
+They do not reveal asset count, transfer bandwidth, server computation, browser
+main-thread work, JavaScript work, decoding, or layout. The probe therefore
+labels them as diagnostics. Its only initial duration estimate is the explicit
+lower bound:
+
+```text
+ceil(known asset bytes / assumed measured bytes per second)
+```
+
+At five minutes or more, the stub keeps the useful pre-paint and requires a user
+or later policy decision before heavyweight escalation. A future policy should
+use observed per-device/site throughput instead of inventing a network rate.
+
+This probe is opt-in or sampled. `dig` and especially `traceroute` must not run on
+every navigation.
+
+## Developer workbench / inspector
+
+The workbench is where browser-owned facts and diagnostic controls belong:
+
+- logical-page universe and hot/warm/cold counts;
+- resident renderer working set and memory estimates;
+- cache/storage classification and bounded inspection;
+- resource graph size and known byte totals;
+- DNS and route probe results;
+- the pre-decision evidence and reason for allowing or deferring escalation;
+- controls for eviction, cache clearing, simulated pressure, and replay.
+
+These facts may explain why a page stayed in pre-paint mode, but they are not the
+page. A small status affordance can link from the page to the inspector without
+injecting the diagnostic graph into page content.
+
+## Search handoff
+
+Search is browser chrome, not a fake form copied out of a workbench fixture:
+
+```text
+query words
+ -> application/x-www-form-urlencoded query (spaces become +)
+ -> https://www.google.com/search?q=...
+ -> icu get URL
+ -> Idriç text/link/image extraction
+ -> disposable pre-paint
+ -> Android display
+```
+
+`bin/icu_search.grease` records the shell boundary. In the old shell idiom,
+`$*` is the expansion that joins positional parameters using the first `IFS`
+character; `PS1` is only the interactive prompt. The current stub uses `jq` for
+UTF-8 percent encoding and then changes encoded spaces to `+`. ICU, not curl or
+WebView, performs the GET.
+
+The standalone display APK has no Internet permission and cannot execute the
+host ICU binary. It may show and copy the exact request so the interaction is
+testable, but it must not pretend that local request construction is a fetched
+result. The integrated browser shell will own the executable ICU/Android bridge.
diff --git a/docs/url-recognition-strategy.txt b/docs/url-recognition-strategy.txt
new file mode 100644
index 0000000..a8b19c6
--- /dev/null
+++ b/docs/url-recognition-strategy.txt
@@ -0,0 +1,114 @@
+IB URL recognition strategy — staged, readable, and replaceable
+================================================================
+
+This file deliberately keeps two recognizers instead of one enormous regular
+expression.
+
+
+1. Cheap foreground recognizer
+------------------------------
+
+Purpose: turn an ordinary text file containing one URL per line into useful
+links without delaying first paint.
+
+Current rule, written as words:
+
+* begin at the start of the trimmed line;
+* find exactly "http://" or "https://";
+* require a nonempty authority before slash, question mark, or fragment;
+* reject whitespace, controls, non-ASCII bytes, and user information;
+* accept the remainder as a candidate, not as proof that the resource exists.
+
+This intentionally mirrors the current ICU input boundary. It will miss valid
+internationalized or unusual URLs. False negatives are acceptable in the first
+paint because the background recognizer can add links later.
+
+
+2. Authoritative recognizer
+---------------------------
+
+Do not validate the full web with a copied mega-regex. Parse components.
+
+The complete generic syntax is the collected ABNF in RFC 3986 Appendix A:
+
+https://www.rfc-editor.org/rfc/rfc3986.html#appendix-A
+
+That grammar covers scheme, authority, user information, host forms including
+IP literals, port, all path variants, query, fragment, percent encoding,
+reserved characters, and unreserved characters. Scheme-specific rules still
+apply after generic parsing. RFC 3986 itself warns that generic syntax is a
+superset; parsing a shape is not the same as proving it is dereferenceable.
+
+IB's current authoritative HTTP subset should call the same Idriç URL parser as
+ICU rather than independently drifting. Today ICU intentionally supports only
+http/https, visible ASCII targets, no user information, and no IPv6 literal.
+When ICU grows IDNA/IPv6/IRI handling, IB should consume that typed result.
+
+
+3. Background enrichment
+------------------------
+
+After plain text is already visible, a low-priority pass may:
+
+* scan URLs embedded inside prose rather than occupying a whole line;
+* trim sentence punctuation using surrounding-text rules;
+* resolve HTML href and src values against the fetched base URL;
+* preserve an anchor's label and absolute target;
+* preserve an image's alternate text, caption, source URL, and link target;
+* ask ICU to fetch the candidate;
+* classify an image from response Content-Type plus decoded magic, not merely a
+ filename extension;
+* replace a remote image reference with an already-fetched local/content source
+ in a later pre-paint revision.
+
+An `` is a link even when it wraps text. An `` wrapping an image
+must retain both behaviors: paint the fetched image as an image and keep its
+navigation target. `InformationItem.Image` now carries source, alternate text,
+caption/title, and the enclosing link target. The sibling Android interchange
+accepts the same optional linked-image field. Until ICU has fetched and checked
+the bytes, the source remains a remote reference; only the later revision may
+replace it with a local/content source and claim that it is paintable.
+
+
+4. Readable-regex references
+----------------------------
+
+The remembered Ruby project is VerbalExpressions. Its URL example literally
+uses names such as start_of_line, find, maybe, anything_but, and end_of_line:
+
+https://github.com/ryan-endacott/verbal_expressions
+
+It is MIT licensed and useful as a readable sketching vocabulary. Its small URL
+example is not a complete URL validator; it essentially accepts http(s), an
+optional www, and then non-space text. Keep it as a strategy/prototyping
+reference, not as IB's source of truth.
+
+Ruby's maintained URI module is a better executable reference for component
+parsing and exposes parser/regular-expression facilities:
+
+https://github.com/ruby/uri
+
+Neither Ruby dependency belongs on the foreground phone path. A slow comparison
+tool or fixture oracle is acceptable while the Idriç/ICU parser is developed.
+
+
+5. Test ladder
+--------------
+
+Keep separate fixtures for:
+
+* cheap obvious http and https lines;
+* relative href resolution against a base URL;
+* punctuation around a URL in prose;
+* query, fragment, percent encoding, and an explicit port;
+* IPv4, IPv6 literals, IDNA, and Unicode IRI input;
+* malformed percent escapes, spaces, controls, and empty hosts;
+* user information that must not leak into painted text;
+* a text URL that resolves to HTML;
+* a text URL that resolves to an image despite having no image extension;
+* an image-looking extension whose response is not an image;
+* an anchor wrapping an image.
+
+The cheap recognizer may deliberately fail advanced positive cases. The
+authoritative/background path may not silently convert malformed or dangerous
+input into a clickable target.
diff --git a/src/IB/Information.idric b/src/IB/Information.idric
index 337d535..ad0cf0e 100644
--- a/src/IB/Information.idric
+++ b/src/IB/Information.idric
@@ -7,6 +7,7 @@ data InformationItem
= Heading String
| TextBlock String
| Link String String
+ | Image String String String String
| TableRow (List String)
| Form String String
@@ -36,7 +37,7 @@ data Capture
| CaptureTitle
| CaptureHeading String
| CaptureText String
- | CaptureLink String
+ | CaptureLink String Bool Capture
| CaptureForm String
| CaptureCell String
@@ -158,6 +159,47 @@ starts_with_chars (_ :: _) [] = False
starts_with_chars (wanted :: wanted_rest) (value :: rest) =
wanted == value && starts_with_chars wanted_rest rest
+strip_prefix_chars : List Char → List Char → Maybe (List Char)
+strip_prefix_chars [] values = Just values
+strip_prefix_chars (_ :: _) [] = Nothing
+strip_prefix_chars (wanted :: wanted_rest) (value :: rest) =
+ if wanted == value
+ then strip_prefix_chars wanted_rest rest
+ else Nothing
+
+visible_http_character : Char → Bool
+visible_http_character value = ord value > 32 && ord value < 127
+
+authority_end : Char → Bool
+authority_end value = value == '/' || value == '?' || value == '#'
+
+safe_authority_tail : List Char → Bool
+safe_authority_tail [] = True
+safe_authority_tail (value :: rest) =
+ if authority_end value
+ then True
+ else visible_http_character value && value /= '@' && safe_authority_tail rest
+
+safe_authority : List Char → Bool
+safe_authority [] = False
+safe_authority ('[' :: _) = False
+safe_authority (value :: rest) =
+ not (authority_end value) && visible_http_character value && value /= '@' &&
+ safe_authority_tail rest
+
+http_tail : String → Maybe (List Char)
+http_tail text =
+ case strip_prefix_chars (unpack "http://") (unpack text) of
+ Just rest ⇒ Just rest
+ Nothing ⇒ strip_prefix_chars (unpack "https://") (unpack text)
+
+public export
+cheap_http_url : String → Bool
+cheap_http_url text =
+ case http_tail text of
+ Nothing ⇒ False
+ Just rest ⇒ safe_authority rest
+
drop_chars : Nat → List Char → List Char
drop_chars Z values = values
drop_chars (S amount) [] = []
@@ -216,7 +258,7 @@ capture_target NoCapture = ""
capture_target CaptureTitle = "title"
capture_target (CaptureHeading name) = name
capture_target (CaptureText name) = name
-capture_target (CaptureLink _) = "a"
+capture_target (CaptureLink _ _ _) = "a"
capture_target (CaptureForm _) = "form"
capture_target (CaptureCell name) = name
@@ -231,8 +273,10 @@ finish_capture (CaptureHeading _) text title items row =
if text == "" then (title, items, row) else (title, Heading text :: items, row)
finish_capture (CaptureText _) text title items row =
if text == "" then (title, items, row) else (title, TextBlock text :: items, row)
-finish_capture (CaptureLink href) text title items row =
- if text == "" && href == "" then (title, items, row) else (title, Link text href :: items, row)
+finish_capture (CaptureLink href has_image parent) text title items row =
+ if text == "" && (href == "" || has_image)
+ then (title, items, row)
+ else (title, Link text href :: items, row)
finish_capture (CaptureForm action) text title items row =
if text == "" && action == "" then (title, items, row) else (title, Form text action :: items, row)
finish_capture (CaptureCell _) text title items Nothing = (title, items, Nothing)
@@ -250,6 +294,32 @@ add_standalone_text text Nothing items =
let cleaned = clean_text text in
if cleaned == "" then items else TextBlock cleaned :: items
+capture_parent : Capture → Capture
+capture_parent (CaptureLink _ _ parent) = parent
+capture_parent _ = NoCapture
+
+capture_link : Capture → String
+capture_link (CaptureLink href _ _) = href
+capture_link _ = ""
+
+mark_link_image : Capture → Capture
+mark_link_image (CaptureLink href _ parent) = CaptureLink href True parent
+mark_link_image capture = capture
+
+image_from_tag : String → String → Maybe InformationItem
+image_from_tag href tag =
+ let source = attribute_or_empty "src" tag in
+ if source == ""
+ then Nothing
+ else Just (Image source (attribute_or_empty "alt" tag)
+ (attribute_or_empty "title" tag) href)
+
+add_image : String → String → List InformationItem → List InformationItem
+add_image href tag items =
+ case image_from_tag href tag of
+ Nothing ⇒ items
+ Just image ⇒ image :: items
+
update_skip : String → Bool → SkipState → SkipState
update_skip name closing NotSkipping = NotSkipping
update_skip name closing (Skipping target depth) =
@@ -291,7 +361,11 @@ collect_information (HtmlTag tag :: rest) skip capture chars row title items =
else if (name == "p" || name == "li") && not closing
then collect_information rest NotSkipping (CaptureText name) [] row title items
else if name == "a" && not closing
- then collect_information rest NotSkipping (CaptureLink (attribute_or_empty "href" tag)) [] row title items
+ then collect_information rest NotSkipping
+ (CaptureLink (attribute_or_empty "href" tag) False NoCapture) [] row title items
+ else if name == "img" && not closing
+ then collect_information rest NotSkipping NoCapture [] row title
+ (add_image "" tag items)
else if name == "form" && not closing
then collect_information rest NotSkipping (CaptureForm (attribute_or_empty "action" tag)) [] row title items
else if name == "tr" && not closing
@@ -307,9 +381,24 @@ collect_information (HtmlTag tag :: rest) skip capture chars row title items =
if closing && name == capture_target capture
then let text = clean_text (pack (reverse chars)) in
let (next_title, next_items, next_row) = finish_capture capture text title items row in
- collect_information rest NotSkipping NoCapture [] next_row next_title next_items
+ collect_information rest NotSkipping (capture_parent capture) [] next_row next_title next_items
else if not closing && skipped_tag name
then collect_information rest (Skipping name Z) capture chars row title items
+ else if name == "a" && not closing
+ then case capture of
+ CaptureLink _ _ _ ⇒ collect_information rest NotSkipping capture chars row title items
+ _ ⇒
+ let text = clean_text (pack (reverse chars)) in
+ let (next_title, next_items, next_row) = finish_capture capture text title items row in
+ collect_information rest NotSkipping
+ (CaptureLink (attribute_or_empty "href" tag) False capture)
+ [] next_row next_title next_items
+ else if name == "img" && not closing
+ then let next_capture = mark_link_image capture in
+ let text = clean_text (pack (reverse chars)) in
+ let (next_title, next_items, next_row) = finish_capture next_capture text title items row in
+ collect_information rest NotSkipping next_capture [] next_row next_title
+ (add_image (capture_link capture) tag next_items)
else collect_information rest NotSkipping capture chars row title items
public export
@@ -318,6 +407,31 @@ extract_html requested_url resolved_url html =
let (title, items) = collect_information (tokenize html) NotSkipping NoCapture [] Nothing "" [] in
View requested_url resolved_url title items
+finish_plain_line : List Char → List InformationItem → List InformationItem
+finish_plain_line chars items =
+ let line = clean_text (pack (reverse chars)) in
+ if line == ""
+ then items
+ else if cheap_http_url line
+ then Link line line :: items
+ else TextBlock line :: items
+
+plain_text_items : List Char → List Char → List InformationItem → List InformationItem
+plain_text_items [] chars items = reverse (finish_plain_line chars items)
+plain_text_items ('\n' :: rest) chars items =
+ plain_text_items rest [] (finish_plain_line chars items)
+plain_text_items (value :: rest) chars items =
+ plain_text_items rest (value :: chars) items
+
+public export
+prepaint_text : String → String → InformationView
+prepaint_text title body = View "" "" title (plain_text_items (unpack body) [] [])
+
+public export
+prepaint_full_text : String → String → String → InformationView
+prepaint_full_text requested_url resolved_url html =
+ extract_html requested_url resolved_url html
+
public export
has_table_row : List String → InformationView → Bool
has_table_row wanted (View _ _ _ items) = go items
@@ -336,6 +450,16 @@ has_link wanted_text wanted_href (View _ _ _ items) = go items
go (Link text href :: rest) = (text == wanted_text && href == wanted_href) || go rest
go (_ :: rest) = go rest
+public export
+has_image : String → String → String → InformationView → Bool
+has_image wanted_alt wanted_source wanted_href (View _ _ _ items) = go items
+ where
+ go : List InformationItem → Bool
+ go [] = False
+ go (Image source alt caption href :: rest) =
+ (alt == wanted_alt && source == wanted_source && href == wanted_href) || go rest
+ go (_ :: rest) = go rest
+
public export
has_form : String → InformationView → Bool
has_form wanted_action (View _ _ _ items) = go items
@@ -354,6 +478,7 @@ item_text : InformationItem → String
item_text (Heading text) = text
item_text (TextBlock text) = text
item_text (Link text href) = text ++ " " ++ href
+item_text (Image source alt caption href) = alt ++ " " ++ source ++ " " ++ caption ++ " " ++ href
item_text (TableRow cells) = join_cells cells
item_text (Form text action) = text ++ " " ++ action
@@ -387,6 +512,9 @@ render_item : Nat → InformationItem → String
render_item width (Heading text) = clip_text width ("# " ++ text)
render_item width (TextBlock text) = clip_text width text
render_item width (Link text href) = clip_text width ("link: " ++ text ++ " -> " ++ href)
+render_item width (Image source alt caption href) =
+ clip_text width ("image: " ++ alt ++ " -> " ++ source ++
+ if href == "" then "" else " [link " ++ href ++ "]")
render_item width (TableRow cells) = clip_text width ("| " ++ join_with " | " cells ++ " |")
render_item width (Form text action) = clip_text width ("form: " ++ text ++ " -> " ++ action)
diff --git a/src/InformationSmoke.idric b/src/InformationSmoke.idric
index 8d04202..5565112 100644
--- a/src/InformationSmoke.idric
+++ b/src/InformationSmoke.idric
@@ -35,6 +35,23 @@ shell_only =
"" ++
"