Add readable multi-model filesystem vector views

.github/workflows/embedding-models.yml

@@ -0,0 +1,47 @@
+name: Pinned embedding model views
+
+on:
+  pull_request:
+    paths:
+      - 'models/embedding/**'
+      - 'bin/ib_embedding_model.grease'
+      - 'bin/ib_vector_view.grease'
+      - 'experiments/embedding-models/**'
+      - 'native/vector-index/**'
+      - 'tests/fixtures/embedding-*.tsv'
+      - 'tests/live_embedding_models.sh'
+      - '.github/workflows/embedding-models.yml'
+
+permissions:
+  contents: read
+
+jobs:
+  end-to-end:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+          cache: pip
+          cache-dependency-path: experiments/embedding-models/requirements.txt
+
+      - name: Install pinned comparison adapter
+        run: pip install -r experiments/embedding-models/requirements.txt
+
+      - name: Compile filesystem index
+        run: make -C native/vector-index
+
+      - uses: actions/cache@v4
+        with:
+          path: ${{ runner.temp }}/ib-embedding-models
+          key: ib-embedding-models-${{ hashFiles('models/embedding/*.model') }}
+
+      - name: Fetch, verify, embed, index, and retrieve with both model views
+        run: |
+          mkdir -p '${{ runner.temp }}/ib-embedding-models'
+          tests/live_embedding_models.sh \
+            native/vector-index/ib-vector-index \
+            python \
+            '${{ runner.temp }}/ib-embedding-models'

.github/workflows/idric-core.yml

@@ -26,6 +26,12 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
+      - name: Exercise filesystem category and hot views
+        run: sh bin/ci_browser_foundation.grease exercise-filesystem-views
+
+      - name: Exercise saved-tab text question mock
+        run: sh bin/ci_browser_foundation.grease exercise-tab-qa
+
       - name: Install system dependencies through Grease
         run: sh bin/ci_browser_foundation.grease install-dependencies
 
@@ -56,6 +62,9 @@ jobs:
       - name: Exercise developer workbench
         run: sh bin/ci_browser_foundation.grease exercise-workbench
 
+      - name: Exercise durable file store across process restart
+        run: sh bin/ci_browser_foundation.grease exercise-file-store
+
   scientific-media:
     runs-on: ubuntu-latest
     env:

.github/workflows/vector-index.yml

@@ -0,0 +1,86 @@
+name: Filesystem vector index
+
+on:
+  pull_request:
+    paths:
+      - 'native/vector-index/**'
+      - 'tests/vector-index-smoke.sh'
+      - 'src/IB/VectorIndex.idric'
+      - 'src/IB/Storage.idric'
+      - 'docs/vector-index.md'
+      - '.github/workflows/vector-index.yml'
+
+permissions:
+  contents: read
+
+jobs:
+  host:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Compile warning-clean C99 backend
+        run: make -C native/vector-index
+
+      - name: Exercise build, persistence, validation, and exact query
+        run: make -C native/vector-index test
+
+      - name: Exercise the 10,000 URL scale with Float32 storage
+        run: |
+          index_root="$(mktemp -d)/views/organizing-the-information/vector-spaces/scale-model/pages"
+          awk 'BEGIN {
+            for (row = 0; row < 10000; row++) {
+              printf "url-%d\t", row
+              for (column = 0; column < 384; column++)
+                printf "%s0.01", column == 0 ? "" : " "
+              printf "\n"
+            }
+          }' | native/vector-index/ib-vector-index build "$index_root" 384 cosine | tee /tmp/vector-scale-build.txt
+          grep -Fx 'count=10000' /tmp/vector-scale-build.txt
+          text_file="$(sed -n 's/^vectors_text //p' "$index_root/format.txt")"
+          cache_file="$(sed -n 's/^vectors_cache //p' "$index_root/format.txt")"
+          test "$(stat -c %s "$index_root/$text_file")" = 61440000
+          test "$(stat -c %s "$index_root/$cache_file")" = 15360000
+          awk 'BEGIN {
+            for (column = 0; column < 384; column++)
+              printf "%s0.01", column == 0 ? "" : " "
+            printf "\n"
+          }' | native/vector-index/ib-vector-index query "$index_root" 3 > /tmp/vector-scale-query.txt
+          test "$(wc -l < /tmp/vector-scale-query.txt)" = 3
+          sed -n '1s/\t.*//p' /tmp/vector-scale-query.txt | grep -Fx 'url-0'
+
+  android:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        include:
+          - abi: arm64-v8a
+            compiler: aarch64-linux-android24-clang
+            file_architecture: ARM aarch64
+          - abi: armeabi-v7a
+            compiler: armv7a-linux-androideabi24-clang
+            file_architecture: ARM
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: android-actions/setup-android@v3
+
+      - name: Install pinned Android NDK
+        run: sdkmanager 'ndk;27.2.12479018'
+
+      - name: Cross-compile small Android executable
+        run: |
+          toolchain="$ANDROID_SDK_ROOT/ndk/27.2.12479018/toolchains/llvm/prebuilt/linux-x86_64/bin"
+          mkdir -p build/android
+          "$toolchain/${{ matrix.compiler }}" \
+            -O2 -Wall -Wextra -Werror -Wpedantic -std=c99 \
+            native/vector-index/ib_vector_index.c -lm \
+            -o build/android/ib-vector-index
+          "$toolchain/llvm-strip" build/android/ib-vector-index
+          file build/android/ib-vector-index | grep -F '${{ matrix.file_architecture }}'
+          test "$(stat -c %s build/android/ib-vector-index)" -lt 100000
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: ib-vector-index-android-${{ matrix.abi }}
+          path: build/android/ib-vector-index

.gitignore

@@ -0,0 +1,4 @@
+/build/
+/native/vector-index/ib-vector-index
+__pycache__/
+*.pyc

README.md

@@ -15,9 +15,13 @@ The browser core owns resource, tab, event, and task identity; sleeping and waki
 - `docs/prefetch-and-reading.md` — durable investigation frontiers, disposable fetches, and `~/reading`
 - `docs/tab-categorization.md` — overlapping personal categories and adaptive refinement
 - `docs/inference-and-learning.md` — configured-model proposals, explicit hyperplanes, ensembles, and human supervision
+- `docs/filesystem-views.md` — category links, `_active`, and the requested `hot/` presentation set
 - `docs/storage-model.md` — identity levels and canonical, proposed, and derived state
 - `docs/developer-workbench.md` — fixture and memory-pressure harness
+- `docs/vector-index.md` — readable multi-model vector views and rebuildable Float32 query caches
+- `docs/tab-qa-mock.md` — console questions over saved reading pages with replaceable processing stages
 - `experiments/category-hyperplanes/README.md` — disposable embedding and explicit affine-separator probe
+- `experiments/embedding-models/README.md` — pinned tiny ONNX models and an end-to-end filesystem-index comparison
 
 ## Implementation languages
 

bin/ask_saved_pages.grease

@@ -0,0 +1,30 @@
+#!/bin/sh
+set -eu
+
+repository_root=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)
+export IB_TAB_QA_REPOSITORY_ROOT="$repository_root"
+. "$repository_root/lib/tab_qa.grease"
+
+case "${1:-}" in
+  --index)
+    test "$#" -eq 1 || exit 2
+    ib_tab_qa_build_index
+    ;;
+  --inspect)
+    test "$#" -eq 1 || exit 2
+    ib_tab_qa_inspect
+    ;;
+  --help)
+    printf '%s\n' \
+      "usage: $0 [--index|--inspect|QUESTION]" \
+      'With no argument, read one question from the console.'
+    ;;
+  '')
+    printf 'question> ' >&2
+    IFS= read -r question || exit 1
+    ib_tab_qa_answer "$question"
+    ;;
+  *)
+    ib_tab_qa_answer "$*"
+    ;;
+esac

bin/ci_browser_foundation.grease

@@ -42,6 +42,7 @@ exercise_core() {
   grep -Fx 'host-count=2' /tmp/ib-smoke.txt
   grep -Fx 'canonical=canonical' /tmp/ib-smoke.txt
   grep -Fx 'cache-fake=cache' /tmp/ib-smoke.txt
+  grep -Fx 'views=derived' /tmp/ib-smoke.txt
   grep -Fx 'secret=secret' /tmp/ib-smoke.txt
   grep -Fx 'readable-files=2' /tmp/ib-smoke.txt
   grep -Fx 'canonical-files=2' /tmp/ib-smoke.txt
@@ -54,6 +55,36 @@ exercise_core() {
   grep -Fx 'temporary-needs-language-model=True' /tmp/ib-smoke.txt
   grep -Fx 'prefetch-bounded=2' /tmp/ib-smoke.txt
   grep -Fx 'prefetch-reading=1' /tmp/ib-smoke.txt
+  grep -Fx 'vector-backend=flat-f32-exact' /tmp/ib-smoke.txt
+  grep -Fx 'vector-scalar=f32' /tmp/ib-smoke.txt
+  grep -Fx 'vector-contract=2' /tmp/ib-smoke.txt
+  grep -Fx 'vector-directory=organizing-the-information/vector-spaces/test-model/pages' /tmp/ib-smoke.txt
+  grep -Fx 'vector-format-readable=True' /tmp/ib-smoke.txt
+  grep -Fx 'vector-model-readable=True' /tmp/ib-smoke.txt
+  grep -Fx 'vector-text-readable=True' /tmp/ib-smoke.txt
+  grep -Fx 'vector-bytes-readable=False' /tmp/ib-smoke.txt
+}
+
+exercise_filesystem_views() {
+  cd "$repository_root"
+  sh -n lib/filesystem_views.grease
+  sh -n bin/ib_views.grease
+  sh -n tests/fixtures/mock_documentation_synthesizer.grease
+  sh -n tests/test_filesystem_views.grease
+  sh tests/test_filesystem_views.grease
+}
+
+exercise_tab_qa() {
+  cd "$repository_root"
+  sh -n lib/tab_qa.grease
+  sh -n bin/ask_saved_pages.grease
+  sh -n bin/mock_tab_model.grease
+  sh -n bin/mock_tab_qa_reducer.grease
+  sh -n tests/test_mock_tab_model.grease
+  sh -n tests/test_tab_qa_pipeline.grease
+  make -C native/vector-index
+  sh tests/test_mock_tab_model.grease
+  sh tests/test_tab_qa_pipeline.grease
 }
 
 exercise_workbench() {
@@ -82,6 +113,24 @@ exercise_workbench() {
   grep -Fx 'resident-at-10=10' /tmp/ib-workbench.txt
 }
 
+exercise_file_store() {
+  cd "$repository_root"
+  "$idric_prefix/bin/idris2" --source-dir src src/FileStoreSmoke.idric -o ib-file-store-smoke \
+    2>&1 | tee /tmp/idric-file-store-compile.txt
+  test -x ./build/exec/ib-file-store-smoke
+  ! grep -q '^Error:' /tmp/idric-file-store-compile.txt
+
+  store_root="$(mktemp -d)/state"
+  ./build/exec/ib-file-store-smoke write "$store_root" | tee /tmp/ib-file-store-write.txt
+  grep -Fx 'write=ok' /tmp/ib-file-store-write.txt
+  grep -Fx 'unsafe-path=rejected' /tmp/ib-file-store-write.txt
+  grep -Fx 'noncanonical-path=rejected' /tmp/ib-file-store-write.txt
+  ./build/exec/ib-file-store-smoke read "$store_root" | tee /tmp/ib-file-store-read.txt
+  grep -Fx 'manifest-survived=True' /tmp/ib-file-store-read.txt
+  grep -Fx 'history-survived=True' /tmp/ib-file-store-read.txt
+  grep -Fx 'history-lines=2' /tmp/ib-file-store-read.txt
+}
+
 build_information_programs() {
   test -x "$idric_prefix/bin/idris2" || build_idric
   cd "$repository_root/src"
@@ -161,11 +210,14 @@ case "${1:-}" in
   exercise-information) exercise_information ;;
   exercise-core) exercise_core ;;
   exercise-workbench) exercise_workbench ;;
+  exercise-file-store) exercise_file_store ;;
+  exercise-filesystem-views) exercise_filesystem_views ;;
+  exercise-tab-qa) exercise_tab_qa ;;
   exercise-scientific-media) exercise_scientific_media ;;
   exercise-live-arxiv) exercise_live_arxiv ;;
   exercise-live-arxiv-prepaint) exercise_live_arxiv_prepaint ;;
   *)
-    printf 'usage: %s {install-dependencies|build-idric|verify-pdf-harvester|exercise-information|exercise-core|exercise-workbench|exercise-scientific-media|exercise-live-arxiv|exercise-live-arxiv-prepaint}\n' "$0" >&2
+    printf 'usage: %s {install-dependencies|build-idric|verify-pdf-harvester|exercise-information|exercise-core|exercise-workbench|exercise-file-store|exercise-filesystem-views|exercise-tab-qa|exercise-scientific-media|exercise-live-arxiv|exercise-live-arxiv-prepaint}\n' "$0" >&2
     exit 2
     ;;
 esac