diff --git a/.github/workflows/embedding-models.yml b/.github/workflows/embedding-models.yml new file mode 100644 index 0000000..3f90579 --- /dev/null +++ b/.github/workflows/embedding-models.yml @@ -0,0 +1,47 @@ +name: Pinned embedding model views + +on: + pull_request: + paths: + - 'models/embedding/**' + - 'bin/ib_embedding_model.grease' + - 'bin/ib_vector_view.grease' + - 'experiments/embedding-models/**' + - 'native/vector-index/**' + - 'tests/fixtures/embedding-*.tsv' + - 'tests/live_embedding_models.sh' + - '.github/workflows/embedding-models.yml' + +permissions: + contents: read + +jobs: + end-to-end: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + cache: pip + cache-dependency-path: experiments/embedding-models/requirements.txt + + - name: Install pinned comparison adapter + run: pip install -r experiments/embedding-models/requirements.txt + + - name: Compile filesystem index + run: make -C native/vector-index + + - uses: actions/cache@v4 + with: + path: ${{ runner.temp }}/ib-embedding-models + key: ib-embedding-models-${{ hashFiles('models/embedding/*.model') }} + + - name: Fetch, verify, embed, index, and retrieve with both model views + run: | + mkdir -p '${{ runner.temp }}/ib-embedding-models' + tests/live_embedding_models.sh \ + native/vector-index/ib-vector-index \ + python \ + '${{ runner.temp }}/ib-embedding-models' diff --git a/.github/workflows/idric-core.yml b/.github/workflows/idric-core.yml index 0d3bffd..37d3dce 100644 --- a/.github/workflows/idric-core.yml +++ b/.github/workflows/idric-core.yml @@ -26,6 +26,12 @@ jobs: steps: - uses: actions/checkout@v4 + - name: Exercise filesystem category and hot views + run: sh bin/ci_browser_foundation.grease exercise-filesystem-views + + - name: Exercise saved-tab text question mock + run: sh bin/ci_browser_foundation.grease exercise-tab-qa + - name: Install system dependencies through Grease run: sh bin/ci_browser_foundation.grease install-dependencies @@ -56,6 +62,9 @@ jobs: - name: Exercise developer workbench run: sh bin/ci_browser_foundation.grease exercise-workbench + - name: Exercise durable file store across process restart + run: sh bin/ci_browser_foundation.grease exercise-file-store + scientific-media: runs-on: ubuntu-latest env: diff --git a/.github/workflows/vector-index.yml b/.github/workflows/vector-index.yml new file mode 100644 index 0000000..86ee944 --- /dev/null +++ b/.github/workflows/vector-index.yml @@ -0,0 +1,86 @@ +name: Filesystem vector index + +on: + pull_request: + paths: + - 'native/vector-index/**' + - 'tests/vector-index-smoke.sh' + - 'src/IB/VectorIndex.idric' + - 'src/IB/Storage.idric' + - 'docs/vector-index.md' + - '.github/workflows/vector-index.yml' + +permissions: + contents: read + +jobs: + host: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Compile warning-clean C99 backend + run: make -C native/vector-index + + - name: Exercise build, persistence, validation, and exact query + run: make -C native/vector-index test + + - name: Exercise the 10,000 URL scale with Float32 storage + run: | + index_root="$(mktemp -d)/views/organizing-the-information/vector-spaces/scale-model/pages" + awk 'BEGIN { + for (row = 0; row < 10000; row++) { + printf "url-%d\t", row + for (column = 0; column < 384; column++) + printf "%s0.01", column == 0 ? "" : " " + printf "\n" + } + }' | native/vector-index/ib-vector-index build "$index_root" 384 cosine | tee /tmp/vector-scale-build.txt + grep -Fx 'count=10000' /tmp/vector-scale-build.txt + text_file="$(sed -n 's/^vectors_text //p' "$index_root/format.txt")" + cache_file="$(sed -n 's/^vectors_cache //p' "$index_root/format.txt")" + test "$(stat -c %s "$index_root/$text_file")" = 61440000 + test "$(stat -c %s "$index_root/$cache_file")" = 15360000 + awk 'BEGIN { + for (column = 0; column < 384; column++) + printf "%s0.01", column == 0 ? "" : " " + printf "\n" + }' | native/vector-index/ib-vector-index query "$index_root" 3 > /tmp/vector-scale-query.txt + test "$(wc -l < /tmp/vector-scale-query.txt)" = 3 + sed -n '1s/\t.*//p' /tmp/vector-scale-query.txt | grep -Fx 'url-0' + + android: + runs-on: ubuntu-latest + strategy: + matrix: + include: + - abi: arm64-v8a + compiler: aarch64-linux-android24-clang + file_architecture: ARM aarch64 + - abi: armeabi-v7a + compiler: armv7a-linux-androideabi24-clang + file_architecture: ARM + steps: + - uses: actions/checkout@v4 + + - uses: android-actions/setup-android@v3 + + - name: Install pinned Android NDK + run: sdkmanager 'ndk;27.2.12479018' + + - name: Cross-compile small Android executable + run: | + toolchain="$ANDROID_SDK_ROOT/ndk/27.2.12479018/toolchains/llvm/prebuilt/linux-x86_64/bin" + mkdir -p build/android + "$toolchain/${{ matrix.compiler }}" \ + -O2 -Wall -Wextra -Werror -Wpedantic -std=c99 \ + native/vector-index/ib_vector_index.c -lm \ + -o build/android/ib-vector-index + "$toolchain/llvm-strip" build/android/ib-vector-index + file build/android/ib-vector-index | grep -F '${{ matrix.file_architecture }}' + test "$(stat -c %s build/android/ib-vector-index)" -lt 100000 + + - uses: actions/upload-artifact@v4 + with: + name: ib-vector-index-android-${{ matrix.abi }} + path: build/android/ib-vector-index diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..904dab3 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +/build/ +/native/vector-index/ib-vector-index +__pycache__/ +*.pyc diff --git a/README.md b/README.md index 75ce748..d170526 100644 --- a/README.md +++ b/README.md @@ -15,9 +15,13 @@ The browser core owns resource, tab, event, and task identity; sleeping and waki - `docs/prefetch-and-reading.md` — durable investigation frontiers, disposable fetches, and `~/reading` - `docs/tab-categorization.md` — overlapping personal categories and adaptive refinement - `docs/inference-and-learning.md` — configured-model proposals, explicit hyperplanes, ensembles, and human supervision +- `docs/filesystem-views.md` — category links, `_active`, and the requested `hot/` presentation set - `docs/storage-model.md` — identity levels and canonical, proposed, and derived state - `docs/developer-workbench.md` — fixture and memory-pressure harness +- `docs/vector-index.md` — readable multi-model vector views and rebuildable Float32 query caches +- `docs/tab-qa-mock.md` — console questions over saved reading pages with replaceable processing stages - `experiments/category-hyperplanes/README.md` — disposable embedding and explicit affine-separator probe +- `experiments/embedding-models/README.md` — pinned tiny ONNX models and an end-to-end filesystem-index comparison ## Implementation languages diff --git a/bin/ask_saved_pages.grease b/bin/ask_saved_pages.grease new file mode 100755 index 0000000..42e6a6c --- /dev/null +++ b/bin/ask_saved_pages.grease @@ -0,0 +1,30 @@ +#!/bin/sh +set -eu + +repository_root=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) +export IB_TAB_QA_REPOSITORY_ROOT="$repository_root" +. "$repository_root/lib/tab_qa.grease" + +case "${1:-}" in + --index) + test "$#" -eq 1 || exit 2 + ib_tab_qa_build_index + ;; + --inspect) + test "$#" -eq 1 || exit 2 + ib_tab_qa_inspect + ;; + --help) + printf '%s\n' \ + "usage: $0 [--index|--inspect|QUESTION]" \ + 'With no argument, read one question from the console.' + ;; + '') + printf 'question> ' >&2 + IFS= read -r question || exit 1 + ib_tab_qa_answer "$question" + ;; + *) + ib_tab_qa_answer "$*" + ;; +esac diff --git a/bin/ci_browser_foundation.grease b/bin/ci_browser_foundation.grease index 6eba5d3..545b23d 100755 --- a/bin/ci_browser_foundation.grease +++ b/bin/ci_browser_foundation.grease @@ -42,6 +42,7 @@ exercise_core() { grep -Fx 'host-count=2' /tmp/ib-smoke.txt grep -Fx 'canonical=canonical' /tmp/ib-smoke.txt grep -Fx 'cache-fake=cache' /tmp/ib-smoke.txt + grep -Fx 'views=derived' /tmp/ib-smoke.txt grep -Fx 'secret=secret' /tmp/ib-smoke.txt grep -Fx 'readable-files=2' /tmp/ib-smoke.txt grep -Fx 'canonical-files=2' /tmp/ib-smoke.txt @@ -54,6 +55,36 @@ exercise_core() { grep -Fx 'temporary-needs-language-model=True' /tmp/ib-smoke.txt grep -Fx 'prefetch-bounded=2' /tmp/ib-smoke.txt grep -Fx 'prefetch-reading=1' /tmp/ib-smoke.txt + grep -Fx 'vector-backend=flat-f32-exact' /tmp/ib-smoke.txt + grep -Fx 'vector-scalar=f32' /tmp/ib-smoke.txt + grep -Fx 'vector-contract=2' /tmp/ib-smoke.txt + grep -Fx 'vector-directory=organizing-the-information/vector-spaces/test-model/pages' /tmp/ib-smoke.txt + grep -Fx 'vector-format-readable=True' /tmp/ib-smoke.txt + grep -Fx 'vector-model-readable=True' /tmp/ib-smoke.txt + grep -Fx 'vector-text-readable=True' /tmp/ib-smoke.txt + grep -Fx 'vector-bytes-readable=False' /tmp/ib-smoke.txt +} + +exercise_filesystem_views() { + cd "$repository_root" + sh -n lib/filesystem_views.grease + sh -n bin/ib_views.grease + sh -n tests/fixtures/mock_documentation_synthesizer.grease + sh -n tests/test_filesystem_views.grease + sh tests/test_filesystem_views.grease +} + +exercise_tab_qa() { + cd "$repository_root" + sh -n lib/tab_qa.grease + sh -n bin/ask_saved_pages.grease + sh -n bin/mock_tab_model.grease + sh -n bin/mock_tab_qa_reducer.grease + sh -n tests/test_mock_tab_model.grease + sh -n tests/test_tab_qa_pipeline.grease + make -C native/vector-index + sh tests/test_mock_tab_model.grease + sh tests/test_tab_qa_pipeline.grease } exercise_workbench() { @@ -82,6 +113,24 @@ exercise_workbench() { grep -Fx 'resident-at-10=10' /tmp/ib-workbench.txt } +exercise_file_store() { + cd "$repository_root" + "$idric_prefix/bin/idris2" --source-dir src src/FileStoreSmoke.idric -o ib-file-store-smoke \ + 2>&1 | tee /tmp/idric-file-store-compile.txt + test -x ./build/exec/ib-file-store-smoke + ! grep -q '^Error:' /tmp/idric-file-store-compile.txt + + store_root="$(mktemp -d)/state" + ./build/exec/ib-file-store-smoke write "$store_root" | tee /tmp/ib-file-store-write.txt + grep -Fx 'write=ok' /tmp/ib-file-store-write.txt + grep -Fx 'unsafe-path=rejected' /tmp/ib-file-store-write.txt + grep -Fx 'noncanonical-path=rejected' /tmp/ib-file-store-write.txt + ./build/exec/ib-file-store-smoke read "$store_root" | tee /tmp/ib-file-store-read.txt + grep -Fx 'manifest-survived=True' /tmp/ib-file-store-read.txt + grep -Fx 'history-survived=True' /tmp/ib-file-store-read.txt + grep -Fx 'history-lines=2' /tmp/ib-file-store-read.txt +} + build_information_programs() { test -x "$idric_prefix/bin/idris2" || build_idric cd "$repository_root/src" @@ -161,11 +210,14 @@ case "${1:-}" in exercise-information) exercise_information ;; exercise-core) exercise_core ;; exercise-workbench) exercise_workbench ;; + exercise-file-store) exercise_file_store ;; + exercise-filesystem-views) exercise_filesystem_views ;; + exercise-tab-qa) exercise_tab_qa ;; exercise-scientific-media) exercise_scientific_media ;; exercise-live-arxiv) exercise_live_arxiv ;; exercise-live-arxiv-prepaint) exercise_live_arxiv_prepaint ;; *) - printf 'usage: %s {install-dependencies|build-idric|verify-pdf-harvester|exercise-information|exercise-core|exercise-workbench|exercise-scientific-media|exercise-live-arxiv|exercise-live-arxiv-prepaint}\n' "$0" >&2 + printf 'usage: %s {install-dependencies|build-idric|verify-pdf-harvester|exercise-information|exercise-core|exercise-workbench|exercise-file-store|exercise-filesystem-views|exercise-tab-qa|exercise-scientific-media|exercise-live-arxiv|exercise-live-arxiv-prepaint}\n' "$0" >&2 exit 2 ;; esac diff --git a/bin/ib_embedding_model.grease b/bin/ib_embedding_model.grease new file mode 100755 index 0000000..e3ebdb7 --- /dev/null +++ b/bin/ib_embedding_model.grease @@ -0,0 +1,174 @@ +#!/bin/sh +set -eu + +usage() { + printf '%s\n' 'usage:' >&2 + printf '%s\n' ' ib_embedding_model.grease describe MODEL_MANIFEST' >&2 + printf '%s\n' ' ib_embedding_model.grease fetch MODEL_MANIFEST MODEL_DIRECTORY' >&2 + printf '%s\n' ' ib_embedding_model.grease verify MODEL_MANIFEST MODEL_DIRECTORY' >&2 + exit 2 +} + +fail() { + printf 'ib-embedding-model: %s\n' "$*" >&2 + return 1 +} + +check_manifest() { + awk ' + function bad(message) { + print "ib-embedding-model: " message > "/dev/stderr" + exit 1 + } + NR == 1 { + if ($0 != "ib-embedding-model 1") bad("unsupported manifest header") + next + } + $1 == "artifact" { + if (NF != 4) bad("artifact rows require path, byte count, and SHA-256") + if ($2 !~ /^[A-Za-z0-9._-]+(\/[A-Za-z0-9._-]+)*$/ || + $2 ~ /(^|\/)\.\.?(\/|$)/) bad("unsafe artifact path") + if ($3 !~ /^[0-9]+$/ || $3 == 0) bad("invalid artifact byte count") + if (length($4) != 64 || $4 !~ /^[0-9a-f]+$/) bad("invalid artifact SHA-256") + if (artifact[$2]++) bad("duplicate artifact path") + artifacts++ + next + } + $1 ~ /^(slug|repository|revision|license|runtime|weight_precision|dimensions|max_tokens|pooling|normalization|token_policy|padding|query_prefix|document_prefix|tokenizer_file|onnx_file|onnx_output)$/ { + if (NF != 2) bad("manifest fields require one value") + if (seen[$1]++) bad("duplicate manifest field " $1) + value[$1] = $2 + next + } + { bad("unknown manifest row " NR) } + END { + if (NR == 0) bad("empty manifest") + required = "slug repository revision license runtime weight_precision dimensions max_tokens pooling normalization token_policy padding query_prefix document_prefix tokenizer_file onnx_file onnx_output" + count = split(required, names, " ") + for (i = 1; i <= count; i++) if (!seen[names[i]]) bad("missing field " names[i]) + if (value["slug"] !~ /^[a-z0-9][a-z0-9._-]*$/) bad("unsafe slug") + if (value["repository"] !~ /^[A-Za-z0-9._-]+\/[A-Za-z0-9._-]+$/) bad("invalid repository") + if (length(value["revision"]) != 40 || value["revision"] !~ /^[0-9a-f]+$/) bad("revision must be a commit SHA") + if (value["dimensions"] !~ /^[0-9]+$/ || value["dimensions"] == 0) bad("invalid dimensions") + if (value["max_tokens"] !~ /^[0-9]+$/ || value["max_tokens"] == 0) bad("invalid max_tokens") + if (!artifact[value["tokenizer_file"]]) bad("tokenizer_file is not an artifact") + if (!artifact[value["onnx_file"]]) bad("onnx_file is not an artifact") + if (artifacts == 0) bad("manifest has no artifacts") + } + ' "$1" +} + +field() { + awk -v wanted="$2" '$1 == wanted { print $2 }' "$1" +} + +verify_directory() { + verify_manifest=$1 + verify_directory_name=$2 + test -d "$verify_directory_name" || fail "model directory is missing: $verify_directory_name" + + while IFS=' ' read -r marker relative_path expected_bytes expected_sha256 + do + test "$marker" = artifact || continue + artifact_path=$verify_directory_name/$relative_path + test -f "$artifact_path" || fail "model artifact is missing: $relative_path" + actual_bytes=$(wc -c < "$artifact_path" | tr -d ' ') + test "$actual_bytes" = "$expected_bytes" || + fail "model artifact has the wrong size: $relative_path" + actual_sha256=$(sha256sum "$artifact_path" | awk '{print $1}') + test "$actual_sha256" = "$expected_sha256" || + fail "model artifact has the wrong SHA-256: $relative_path" + done < "$verify_manifest" + + if test -f "$verify_directory_name/model.txt"; then + cmp -s "$verify_manifest" "$verify_directory_name/model.txt" || + fail 'installed model.txt differs from the requested manifest' + fi +} + +describe_manifest() { + describe_file=$1 + printf 'slug=%s\n' "$(field "$describe_file" slug)" + printf 'repository=%s\n' "$(field "$describe_file" repository)" + printf 'revision=%s\n' "$(field "$describe_file" revision)" + printf 'runtime=%s\n' "$(field "$describe_file" runtime)" + printf 'weight_precision=%s\n' "$(field "$describe_file" weight_precision)" + printf 'dimensions=%s\n' "$(field "$describe_file" dimensions)" + awk '$1 == "artifact" { bytes += $3; files++ } + END { printf "artifact_files=%d\nartifact_bytes=%.0f\n", files, bytes }' \ + "$describe_file" +} + +fetch_model() { + fetch_manifest=$1 + fetch_directory=$2 + case "$fetch_directory" in + ''|/) fail "unsafe model directory: $fetch_directory"; return 1 ;; + esac + command -v sha256sum >/dev/null 2>&1 || fail 'missing command: sha256sum' + if test -e "$fetch_directory"; then + verify_directory "$fetch_manifest" "$fetch_directory" + printf '%s\n' 'fetch=already-present' 'verify=ok' + return 0 + fi + + command -v curl >/dev/null 2>&1 || fail 'missing command: curl' + fetch_parent=$(dirname "$fetch_directory") + fetch_name=$(basename "$fetch_directory") + mkdir -p "$fetch_parent" + fetch_temporary=$(mktemp -d "$fetch_parent/.${fetch_name}.download.XXXXXX") + trap 'rm -rf "$fetch_temporary"' EXIT HUP INT TERM + repository=$(field "$fetch_manifest" repository) + revision=$(field "$fetch_manifest" revision) + + while IFS=' ' read -r marker relative_path expected_bytes expected_sha256 + do + test "$marker" = artifact || continue + artifact_path=$fetch_temporary/$relative_path + mkdir -p "$(dirname "$artifact_path")" + curl --proto '=https' --tlsv1.2 --retry 3 --retry-all-errors \ + --connect-timeout 30 -fsSL \ + "https://huggingface.co/$repository/resolve/$revision/$relative_path?download=true" \ + -o "$artifact_path" + actual_bytes=$(wc -c < "$artifact_path" | tr -d ' ') + test "$actual_bytes" = "$expected_bytes" || + fail "downloaded artifact has the wrong size: $relative_path" + actual_sha256=$(sha256sum "$artifact_path" | awk '{print $1}') + test "$actual_sha256" = "$expected_sha256" || + fail "downloaded artifact has the wrong SHA-256: $relative_path" + done < "$fetch_manifest" + + cp "$fetch_manifest" "$fetch_temporary/model.txt" + verify_directory "$fetch_manifest" "$fetch_temporary" + if ! mv -T "$fetch_temporary" "$fetch_directory"; then + fail "cannot install model directory: $fetch_directory" + return 1 + fi + trap - EXIT HUP INT TERM + printf '%s\n' 'fetch=ok' 'verify=ok' +} + +test "$#" -gt 0 || usage +command_name=$1 +shift + +case "$command_name" in + describe) + test "$#" -eq 1 || usage + check_manifest "$1" + describe_manifest "$1" + ;; + fetch) + test "$#" -eq 2 || usage + check_manifest "$1" + fetch_model "$1" "$2" + ;; + verify) + test "$#" -eq 2 || usage + check_manifest "$1" + command -v sha256sum >/dev/null 2>&1 || fail 'missing command: sha256sum' + verify_directory "$1" "$2" + printf '%s\n' 'verify=ok' + ;; + *) usage ;; +esac diff --git a/bin/ib_vector_view.grease b/bin/ib_vector_view.grease new file mode 100755 index 0000000..91fd179 --- /dev/null +++ b/bin/ib_vector_view.grease @@ -0,0 +1,81 @@ +#!/bin/sh +set -eu + +repository_root=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) + +usage() { + printf '%s\n' \ + 'usage: ib_vector_view.grease build VIEW_ROOT COLLECTION MODEL_MANIFEST cosine|dot [VECTOR_PROGRAM]' >&2 + exit 2 +} + +fail() { + printf 'ib-vector-view: %s\n' "$*" >&2 + return 1 +} + +safe_component() { + case "$2" in + ''|.|..|*/*) fail "unsafe $1: $2"; return 1 ;; + *' +'*) fail "$1 contains a newline"; return 1 ;; + esac +} + +install_model_manifest() { + install_source=$1 + install_directory=$2 + install_target=$install_directory/embedding-model.txt + + if test -e "$install_target" || test -L "$install_target"; then + if ! test -f "$install_target" || test -L "$install_target"; then + fail "embedding model record is not a regular file: $install_target" + return 1 + fi + if ! cmp -s "$install_source" "$install_target"; then + fail 'model slug is already bound to a different immutable manifest' + return 1 + fi + return 0 + fi + + install_temporary=$(mktemp "$install_directory/.embedding-model.XXXXXX") + trap 'rm -f "$install_temporary"' EXIT HUP INT TERM + cp "$install_source" "$install_temporary" + mv -T "$install_temporary" "$install_target" + trap - EXIT HUP INT TERM +} + +test "$#" -gt 0 || usage +command_name=$1 +shift +test "$command_name" = build || usage +test "$#" -eq 4 || test "$#" -eq 5 || usage + +view_root=$1 +collection=$2 +model_manifest=$3 +metric=$4 +vector_program=${5:-ib-vector-index} + +case "$view_root" in + ''|/) fail "unsafe view root: $view_root"; exit 1 ;; +esac +safe_component collection "$collection" +case "$metric" in cosine|dot) ;; *) fail "unsupported metric: $metric"; exit 1 ;; esac + +"$repository_root/bin/ib_embedding_model.grease" describe "$model_manifest" >/dev/null +slug=$(awk '$1 == "slug" { print $2 }' "$model_manifest") +dimensions=$(awk '$1 == "dimensions" { print $2 }' "$model_manifest") +vector_spaces=$view_root/organizing-the-information/vector-spaces +model_view_directory=$vector_spaces/$slug +index_directory=$model_view_directory/$collection + +sh "$repository_root/bin/ib_views.grease" init "$view_root" +test ! -L "$model_view_directory" || + fail "model vector-space directory is a symbolic link: $model_view_directory" +mkdir -p "$model_view_directory" +test ! -L "$index_directory" || fail "vector view is a symbolic link: $index_directory" +mkdir -p "$index_directory" +install_model_manifest "$model_manifest" "$model_view_directory" +"$vector_program" build "$index_directory" "$dimensions" "$metric" diff --git a/bin/ib_views.grease b/bin/ib_views.grease new file mode 100644 index 0000000..e876056 --- /dev/null +++ b/bin/ib_views.grease @@ -0,0 +1,63 @@ +#!/bin/sh +set -eu + +repository_root=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) +. "$repository_root/lib/filesystem_views.grease" + +usage() { + printf '%s\n' 'usage:' >&2 + printf '%s\n' ' ib_views.grease init VIEW_ROOT' >&2 + printf '%s\n' ' ib_views.grease category-add VIEW_ROOT STATE_ROOT CATEGORY {tab|resource} ID' >&2 + printf '%s\n' ' ib_views.grease category-remove VIEW_ROOT CATEGORY {tab|resource} ID' >&2 + printf '%s\n' ' ib_views.grease category-activate VIEW_ROOT CATEGORY' >&2 + printf '%s\n' ' ib_views.grease category-deactivate VIEW_ROOT CATEGORY' >&2 + printf '%s\n' ' ib_views.grease category-memberships VIEW_ROOT STATE_ROOT {tab|resource} ID' >&2 + printf '%s\n' ' ib_views.grease hot-add VIEW_ROOT PRESENTATION_ROOT SLOT RELATIVE_TARGET' >&2 + printf '%s\n' ' ib_views.grease hot-remove VIEW_ROOT SLOT' >&2 + printf '%s\n' ' ib_views.grease prune-broken VIEW_ROOT' >&2 + exit 2 +} + +command_name=${1:-} +test "$#" -gt 0 || usage +shift + +case "$command_name" in + init) + test "$#" -eq 1 || usage + ib_views_initialize "$1" + ;; + category-add) + test "$#" -eq 5 || usage + ib_category_add "$1" "$2" "$3" "$4" "$5" + ;; + category-remove) + test "$#" -eq 4 || usage + ib_category_remove "$1" "$2" "$3" "$4" + ;; + category-activate) + test "$#" -eq 2 || usage + ib_category_activate "$1" "$2" + ;; + category-deactivate) + test "$#" -eq 2 || usage + ib_category_deactivate "$1" "$2" + ;; + category-memberships) + test "$#" -eq 4 || usage + ib_category_memberships "$1" "$2" "$3" "$4" + ;; + hot-add) + test "$#" -eq 4 || usage + ib_hot_add "$1" "$2" "$3" "$4" + ;; + hot-remove) + test "$#" -eq 2 || usage + ib_hot_remove "$1" "$2" + ;; + prune-broken) + test "$#" -eq 1 || usage + ib_views_prune_broken_links "$1" + ;; + *) usage ;; +esac diff --git a/bin/mock_tab_model.grease b/bin/mock_tab_model.grease new file mode 100755 index 0000000..d478ac4 --- /dev/null +++ b/bin/mock_tab_model.grease @@ -0,0 +1,116 @@ +#!/bin/sh +set -eu + +model_name=mock-token-i8-v1 +dimensions=32 + +inspect_model() { + printf '%s\n' \ + "model=$model_name" \ + 'available=True' \ + 'model-kind=deterministic-token-test-double' \ + 'llm=False' \ + 'quantization=int8-counts' \ + "dimensions=$dimensions" \ + 'tasks=embedding,extractive-answer' +} + +embed_text() { + awk -v dimensions="$dimensions" ' + function token_slot(token, alphabet, character, hash, index_in_alphabet, position) { + alphabet = "abcdefghijklmnopqrstuvwxyz0123456789" + hash = 0 + for (position = 1; position <= length(token); position++) { + character = substr(token, position, 1) + index_in_alphabet = index(alphabet, character) + hash = (hash * 37 + index_in_alphabet) % dimensions + } + return hash + 1 + } + { + text = tolower($0) + gsub(/[^[:alnum:]]+/, " ", text) + token_count = split(text, tokens, / +/) + for (token_index = 1; token_index <= token_count; token_index++) { + token = tokens[token_index] + if (length(token) < 2) + continue + slot = token_slot(token) + if (values[slot] < 127) + values[slot]++ + } + } + END { + for (slot = 1; slot <= dimensions; slot++) + printf "%s%d", slot == 1 ? "" : " ", values[slot] + 0 + printf "\n" + } + ' +} + +answer_from_document() { + question=$1 + document=$2 + test -s "$document" || { + printf 'mock tab model: source document is empty: %s\n' "$document" >&2 + return 1 + } + + awk -v question="$question" ' + BEGIN { + normalized_question = tolower(question) + gsub(/[^[:alnum:]]+/, " ", normalized_question) + question_count = split(normalized_question, question_tokens, / +/) + for (question_index = 1; question_index <= question_count; question_index++) + if (length(question_tokens[question_index]) >= 2) + wanted[question_tokens[question_index]] = 1 + } + NF { + if (first_line == "") + first_line = $0 + normalized_line = tolower($0) + gsub(/[^[:alnum:]]+/, " ", normalized_line) + line_count = split(normalized_line, line_tokens, / +/) + score = 0 + for (line_index = 1; line_index <= line_count; line_index++) { + token = line_tokens[line_index] + if (token in wanted && seen_on_line[token] != NR) { + score++ + seen_on_line[token] = NR + } + } + if (best_line == "" || score > best_score) { + best_line = $0 + best_score = score + } + } + END { + if (best_line != "") + print best_line + else if (first_line != "") + print first_line + } + ' "$document" +} + +case "${1:-}" in + inspect) + test "$#" -eq 1 || exit 2 + inspect_model + ;; + embed) + test "$#" -eq 1 || exit 2 + embed_text + ;; + answer) + test "$#" -eq 3 || { + printf 'usage: %s answer QUESTION DOCUMENT\n' "$0" >&2 + exit 2 + } + answer_from_document "$2" "$3" + ;; + *) + printf 'usage: %s {inspect|embed|answer QUESTION DOCUMENT}\n' "$0" >&2 + exit 2 + ;; +esac diff --git a/bin/mock_tab_qa_reducer.grease b/bin/mock_tab_qa_reducer.grease new file mode 100755 index 0000000..5619dc3 --- /dev/null +++ b/bin/mock_tab_qa_reducer.grease @@ -0,0 +1,68 @@ +#!/bin/sh +set -eu + +reducer_name=mock-single-member-weighted-reducer-v1 + +inspect_reducer() { + printf '%s\n' \ + "reducer=$reducer_name" \ + 'available=True' \ + 'ensemble=single-member-test-double' \ + 'member-count=1' \ + 'xgboost=False' +} + +reduce_candidate() { + retrieval_score=$1 + source_document=$2 + report=${IB_TAB_QA_REDUCER_REPORT:?IB_TAB_QA_REDUCER_REPORT must name the reducer report} + candidate=$(sed -n '1p') + + test -n "$candidate" || { + printf 'mock reducer: model candidate is empty\n' >&2 + return 1 + } + test -s "$source_document" || { + printf 'mock reducer: source document is unavailable\n' >&2 + return 1 + } + awk -v value="$retrieval_score" 'BEGIN { + numeric = value ~ /^[-+]?([0-9]+([.][0-9]*)?|[.][0-9]+)([eE][-+]?[0-9]+)?$/ + exit !numeric + }' || { + printf 'mock reducer: retrieval score is not numeric\n' >&2 + return 1 + } + + printf '%s\n' \ + "reducer=$reducer_name" \ + 'ensemble-stage-ran=True' \ + 'member-count=1' \ + 'checks-run=3' \ + 'candidate-nonempty=True' \ + 'source-readable=True' \ + 'score-numeric=True' \ + "aggregate-score=$retrieval_score" > "$report" + + # This one-member reducer deliberately preserves the model text. A later + # voting, bagging, rank-aggregation, or XGBoost adapter occupies this stage. + printf '%s\n' "$candidate" +} + +case "${1:-}" in + inspect) + test "$#" -eq 1 || exit 2 + inspect_reducer + ;; + reduce) + test "$#" -eq 3 || { + printf 'usage: %s reduce RETRIEVAL_SCORE SOURCE_DOCUMENT < candidate.txt\n' "$0" >&2 + exit 2 + } + reduce_candidate "$2" "$3" + ;; + *) + printf 'usage: %s {inspect|reduce RETRIEVAL_SCORE SOURCE_DOCUMENT}\n' "$0" >&2 + exit 2 + ;; +esac diff --git a/docs/developer-workbench.md b/docs/developer-workbench.md index 0a74344..0c5ef9c 100644 --- a/docs/developer-workbench.md +++ b/docs/developer-workbench.md @@ -50,6 +50,8 @@ Possible states: - **cold** — durable metadata or task graph only; - **never visited** — fixture exists but has no browsing state. +The filesystem `views/hot` set is a request to favor presentation targets for immediate use. It is not the same measurement as resident renderer tabs: a text or synthesized Markdown presentation may be hot without any page renderer, and memory pressure may temporarily make requested-hot differ from actually resident. See `docs/filesystem-views.md`. + Opening a cold URL may promote a logical tab into the renderer working set. Memory pressure or working-set limits demote another renderer without losing browser-owned tab, event, task, or organization state. Prefetched documents remain cold or warm; prefetching seven documentation pages must not create seven live renderer sessions. diff --git a/docs/filesystem-views.md b/docs/filesystem-views.md new file mode 100644 index 0000000..1a8b614 --- /dev/null +++ b/docs/filesystem-views.md @@ -0,0 +1,77 @@ +# Filesystem views and presentation working sets + +IB uses ordinary directories and symbolic links as inspectable control and organization surfaces. The names around the larger work area remain provisional; this contract does not depend on calling it a forge, workbench, or anything else. + +## Keep three axes separate + +Category membership, present attention, and renderer residency answer different questions. + +- A category says why a resource or tab may matter. Categories overlap. +- `_active` says which category neighborhoods belong on the present work surface. Activating one category does not wake all of its members. +- `hot/` says which already-available presentation targets should be favored for immediate use. It is a requested working set, not proof that the bytes are currently resident under every memory-pressure condition. + +Actual residency remains a measured runtime state. A later inspector can report requested-hot versus resident without making a symbolic link pretend to be RAM. + +Removing a link from `hot/` is a demotion of attention, not deletion or freezing. The presentation can remain in the reading corpus, task state, categories, and search indexes and become hot again without reconstructing its identity. There is no separate `warm/` directory in this first slice because “warm” may mean several different facts—saved readable material, disposable fetched bytes, serialized renderer state, or actual cache residency—and those must not be collapsed into one link. + +## Sketch + +```text +browser-owned state/ human-readable presentation root/ + tabs/01T-CAMPAIGN/ aws-composite/ + resources/01R-RULES/ view.md + sources.tsv + +views/ + retrieved-from-the-web/ + text/ + images/ + + organizing-the-information/ + categories/ + critical-role/ + tab-01T-CAMPAIGN -> .../state/tabs/01T-CAMPAIGN + campaign-4/ + tab-01T-CAMPAIGN -> .../state/tabs/01T-CAMPAIGN + vector-spaces/ + potion-base-2m/ + embedding-model.txt + pages/ + format.txt + vectors-.txt + + _active/ + campaign-4 -> .../views/organizing-the-information/categories/campaign-4 + + hot/ + aws-iam -> .../presentation-root/aws-composite +``` + +`retrieved-from-the-web` names source-derived text and image views without pretending that a fetched corporate page is the browser's sovereign output. `organizing-the-information` groups ways of looking across those and other browser objects: overlapping categories and one or more model-specific vector spaces. The repeated category link does not duplicate the tab. The hot entry does not need to point at that tab or at any corporation's preferred page. It may point at a source-backed text file, Markdown file, HTML file, or a presentation bundle containing a view, local images, and provenance. A synthesis made from twenty documentation pages is one legitimate hot presentation while its twenty source identities and edges remain intact. + +Markdown is only a cheap current presentation format. It is not canonical browser state and does not constrain later renderers. + +## The page is input, not sovereign output + +Fetched HTML, PDFs, structured responses, screenshots, and saved pages are source material. IB may extract, combine, rank, or summarize them into the representation that answers the user's task fastest. The original representations and source references remain available; a derived presentation does not rewrite them and does not masquerade as an original page. + +This supports the documentation case directly: acquire the relevant parts of a documentation set once, pass selected immutable representations through a cheap local producer, and pre-paint the resulting text, Markdown, HTML, or local-image bundle without opening twenty corporate page interfaces. + +A hot local presentation should be paintable without a network round trip. The intended interaction is console-fast immediate content followed by richer replacement only when it adds something useful; this filesystem slice supplies the pointer handoff but does not claim the frontend or RAM loader yet. + +## First executable slice + +`lib/filesystem_views.grease` now owns the filesystem-facing primitives: + +- initialize `retrieved-from-the-web/{text,images}`, `organizing-the-information/{categories,vector-spaces}`, `_active`, and `hot` under `views/`; +- add one tab or resource to any number of category directories; +- activate or deactivate a category independently of membership; +- list reverse category membership for a typed object link; +- publish or replace a hot presentation link; +- remove only symbolic links, never their targets; +- prune broken view links; +- reject traversal, missing targets, target-root escapes, reserved names, and collisions with ordinary files. + +`bin/ib_views.grease` exposes the same operations as small shell commands. Materialized links use resolved targets and can be rebuilt when storage moves. A deterministic mock producer proves that a twenty-source composite bundle can become hot without depending on a language model or a renderer in the test. + +This slice creates views only. It does not itself load hot targets into RAM, execute an embedding model in the phone-resident browsing loop, store canonical category assertions, import hand-edited links as assertion events, or rebuild all projections from durable Idriç records. The initial models are now pinned and exercised end to end by `experiments/embedding-models/`; moving the selected inference adapter behind the Android/runtime boundary remains separate work. The Grease layer must not fabricate those claims. diff --git a/docs/idric-implementation.md b/docs/idric-implementation.md index f1b11e2..033c160 100644 --- a/docs/idric-implementation.md +++ b/docs/idric-implementation.md @@ -9,6 +9,8 @@ The initial source modules deliberately keep the executable boundary small: - `IB.History` owns normalized history values, import ordering, and stable newest-first chronology. - `IB.Index` builds transparent rebuildable list indices without collapsing duplicate visits. - `IB.Storage` classifies schema-shaped paths and defines which records may be generically inspected. +- `IB.FileStore` performs the first real browser-owned file I/O: it creates the store/tab directories, reads and writes tab manifests, appends history records, rejects paths outside the canonical schema, and reports filesystem failures. +- `IB.VectorIndex` owns the replaceable vector-backend specification and lowers it to the versioned text-stream process contract. The first backend is the filesystem-native exact `float32` tool in `native/vector-index`. - `IB.Inspect` summarizes physical rows without following or interpreting renderer state. - `IB.ScientificMedia` owns HTML-before-PDF source preference and the evidence order for image naming. @@ -27,10 +29,12 @@ This audit leaves `IB.History`, `IB.Index`, `IB.Storage`, `IB.Inspect`, and the The previous large `run:` blocks in GitHub Actions are now delegated to `bin/ci_browser_foundation.grease`. YAML still selects actions and cache behavior because GitHub requires that format; the actual command sequences live in Grease. +`IB.FileStore` deliberately does not yet claim crash-safe replacement. Its manifest write uses the ordinary Idriç file API, and its history write uses append mode. Atomic replacement, flushing, and restart recovery need a narrow native boundary plus fault-injection tests; they are the next durability layer, not properties of this first file-I/O slice. + ## Native boundary A future Android inspector should ask native glue only for bounded metadata/read operations that need kernel enforcement. In particular, rooted no-follow opens belong at the Linux/Android boundary. Classification, visibility, paging policy, and presentation data remain Idriç decisions. ## Build -CI builds the current `isomorphisms/Idric` compiler and compiles `src/Smoke.idric`. The smoke executable exercises history ordering, duplicate preservation, indices, storage classification, protected reads, inspector readability, and scientific-media policy. Grease tests exercise the OS-facing media pipeline separately. +CI builds the current `isomorphisms/Idric` compiler and compiles `src/Smoke.idric` and `src/FileStoreSmoke.idric`. The ordinary smoke executable exercises history ordering, duplicate preservation, indices, storage classification, protected reads, inspector readability, and scientific-media policy. The file-store smoke runs as two separate processes: the first creates a tab and appends two history records, and the second reloads the same files to prove they survive a process restart. Grease tests exercise operating-system-facing paths separately. diff --git a/docs/inference-and-learning.md b/docs/inference-and-learning.md index 41bdaac..4af6604 100644 --- a/docs/inference-and-learning.md +++ b/docs/inference-and-learning.md @@ -133,9 +133,16 @@ Current baselines and evaluation ideas: - ensembles when disagreement is useful; - measurable improvement after a few nearby corrections. +Pinned initial embedding views: + +- `potion-base-2m`, a 64-coordinate static Model2Vec view for extremely cheap broad indexing; +- `mxbai-embed-xsmall-v1-int8`, a 384-coordinate transformer view used as an independent retrieval challenger; +- both remain replaceable, retain immutable model/file provenance, and may coexist rather than forcing one representation. + Still open: -- the embedding model and feature weights; +- which model views are active by default on each device and how their evidence is ensembled; +- feature weights beyond the frozen embedding output; - the online update rule; - acceptance thresholds and which proposals require explicit review; - the exact proposal serialization; diff --git a/docs/storage-model.md b/docs/storage-model.md index abf5623..9ffeade 100644 --- a/docs/storage-model.md +++ b/docs/storage-model.md @@ -49,7 +49,9 @@ state/ ... ``` -This is a conceptual separation, not a frozen on-disk grammar. In particular, the category, task, proposal, and assertion paths are not implemented on `main` yet. +This is not a frozen on-disk grammar. `IB.FileStore` implements the first slice: store and tab directories, `tab.txt`, and `history.log` are created and accessed by Idriç. The category, task, proposal, and assertion paths remain design boundaries rather than claims about the current runtime. + +Rebuildable filesystem projections currently live under a separate `views/` root. Source-derived text and images live below `retrieved-from-the-web/`; cross-object categories and model-specific vector spaces live below `organizing-the-information/`; `_active` and `hot/` remain short control surfaces at the root. The Grease slice does not yet provide the durable Idriç assertion records required to reconstruct accepted human organization after deleting every view. ## Stable event identity @@ -110,6 +112,8 @@ authoritative user assertions and corrections -------------------------------+ A malformed or malicious model output cannot change canonical history, source files, explicit assertions, or secret storage. See `docs/inference-and-learning.md`. +The current file-store slice appends complete caller-supplied records and preserves duplicates and ordering. It does not yet parse or validate the record grammar. It also does not yet provide `fsync`, atomic manifest replacement, or journal recovery; those guarantees require a small native filesystem boundary and fault-injection tests. + ## Derived artifacts An index or derived artifact should be reconstructible from canonical records and durable decisions whenever practical. @@ -139,6 +143,8 @@ Clearing cache may remove response bytes retained only for prefetch, extracted i It must not remove tabs, stable history events, task roots and frontiers, explicit assertions and corrections, proposal decisions, accepted category definitions, material deliberately promoted to the reading corpus, or the accepted or last-complete summary record required to resume a retained task. Missing presentation artifacts may be rebuilt without rerunning a model in a way that loses human decisions. +The first vector implementation follows the same rule without SQLite: model-specific directories under `views/organizing-the-information/vector-spaces/` retain IDs and fixed-width row-major vectors as plain text. A little-endian Float32 sidecar accelerates exact scans but can be deleted and recreated from the text. The backend-neutral streaming command contract permits another disposable cache implementation without changing canonical records or readable vector views. See `vector-index.md`. + ## Sync Sync operates on browser-owned identities, events, tasks, assertions, proposals, decisions, and selected snapshots rather than a renderer profile directory. Multiple frontends or machines may share the durable corpus while maintaining separate live renderers and caches. diff --git a/docs/tab-categorization.md b/docs/tab-categorization.md index 2a4be68..0c44881 100644 --- a/docs/tab-categorization.md +++ b/docs/tab-categorization.md @@ -51,15 +51,17 @@ state/ events/ visits.log -categories/ - algebraic-topology/ - tab-01T... -> ../../state/tabs/01T... +views/ + organizing-the-information/ + categories/ + algebraic-topology/ + tab-01T... -> ../../../../state/tabs/01T... - algebraic-topology-1950s/ - tab-01T... -> ../../state/tabs/01T... + algebraic-topology-1950s/ + tab-01T... -> ../../../../state/tabs/01T... - Serre/ - tab-01T... -> ../../state/tabs/01T... + Serre/ + tab-01T... -> ../../../../state/tabs/01T... ``` The same tab can therefore appear in `algebraic-topology`, `algebraic-topology-1950s`, `Serre`, and any other useful view without duplicating the underlying record. @@ -94,18 +96,26 @@ Once a narrower name is unambiguous and useful to the user, redundant umbrella p Categories can remain durable without all of them being active in the current workbench. -One representation may use an underscore-prefixed control directory such as `_active` as a small set of pointers to category views currently in use: +One representation uses `views/_active` as a small set of pointers to category views currently in use: ```text -categories/ +views/ + organizing-the-information/ + categories/ + algebraic-topology-1950s/ + algebraic-topology-1970s/ + campaign-4/ + _active/ - algebraic-topology-1950s -> ../algebraic-topology-1950s - algebraic-topology-1970s -> ../algebraic-topology-1970s - campaign-4 -> ../campaign-4 + algebraic-topology-1950s -> ../organizing-the-information/categories/algebraic-topology-1950s + algebraic-topology-1970s -> ../organizing-the-information/categories/algebraic-topology-1970s + campaign-4 -> ../organizing-the-information/categories/campaign-4 ``` Removing a link from `_active` neither deletes nor freezes the category. It removes it from the present attention surface while leaving older interests searchable and recoverable. `_active` is not the renderer's hot-tab working set: activating a category must not wake every tab or fetch every resource in it. +The separate `views/hot` control surface points at presentation targets that should receive immediate working-set attention. A hot target may be a text, Markdown, or HTML file, or a local bundle synthesized from several resources; it need not be the original fetched page or a tab record. Requested-hot and actually resident remain distinct under memory pressure. See `docs/filesystem-views.md`. + ## Derived, rebuildable organization Category directories and active-view links are materialized projections. They should be cheap to rebuild, inspect, or replace. Category definitions, user-authored or accepted memberships and corrections, active-category choices, and proposal decisions are durable browser-owned metadata. A direct filesystem edit must be imported as an assertion or attention event before a later rebuild. Rebuilding a projection must not mean rerunning a model and losing human work. @@ -142,7 +152,11 @@ Private account, authentication, messaging, password-reset, and token-bearing UR ## Implementation status -The current `IB.History` and `IB.Index` slices preserve repeated URL rows and their input order. That normalized order is not yet a stable event identity across imports, merges, restarts, or sync. Category definitions, proposal records, accepted-membership storage, filesystem projections, reverse-membership inspection, and `_active` are not implemented on `main` yet. The current storage inspector also does not follow category symlinks. This note defines the boundary for that future work; its example directories are not a claim about the present schema. +The current `IB.History` and `IB.Index` slices preserve repeated URL rows and their input order. That normalized order is not yet a stable event identity across imports, merges, restarts, or sync. + +The first Grease filesystem slice can materialize typed tab/resource links into overlapping category directories, manage `_active`, list reverse membership, publish presentation files or bundles into `views/hot`, and prune broken links. Its deterministic fixture makes a twenty-source Markdown composite hot without a renderer or language model. See `docs/filesystem-views.md`. + +Category definitions, proposal records, and accepted-membership storage are not implemented yet, so these projections cannot yet be rebuilt from durable Idriç assertions. The current storage inspector classifies `views/` as derived but does not follow category symlinks. The Grease slice also does not claim that a requested-hot presentation has actually been loaded into RAM. ## Commitment level @@ -160,7 +174,8 @@ Current heuristics and baselines: - roughly 5–10 relevant objects as a category-promotion signal; - one affine inclusion scorer and separate threshold per category using positives, explicit negatives, and unlabeled material correctly; - over-inclusion when the alternative is failed retrieval; -- `_active` as a filesystem-shaped working-set control. +- `_active` as a filesystem-shaped category-attention control; +- `views/hot` as a separate requested presentation working set. Still open: diff --git a/docs/tab-qa-mock.md b/docs/tab-qa-mock.md new file mode 100644 index 0000000..5144a44 --- /dev/null +++ b/docs/tab-qa-mock.md @@ -0,0 +1,84 @@ +# Saved-tab question-and-answer mock + +This is the first executable vertical slice of the ChatGPT-like text frontend. +It is a console prompt, not the eventual phone UI, and it answers only from +plain-text documents deliberately present below `~/reading`. + +The stages are explicit: + +1. `mock-token-i8-v1` tokenizes text into a 32-component, bounded 8-bit-count + test vector and selects one source line as an extractive answer. +2. `ib-vector-index` stores one exact cosine index in the existing model-view + tree. Authoritative fixed-width text and its rebuildable Float32 cache are + two query paths over that index. Each query can record whether its dot + products used a GPU; the current C backend records `False` and `compute=cpu`. +3. `mock-single-member-weighted-reducer-v1` is a one-member ensemble test + double. It runs three validations, records an aggregate score, and preserves + the candidate. This is the replaceable slot for voting, bagging, rank + aggregation, XGBoost, or another reducer; it does not claim to run XGBoost. +4. `ib_tab_qa_null_render` is an identity post-processing step before the text + is returned. + +The bundled model is intentionally not described as an LLM. It is a tiny, +queryable deterministic mock that exercises token, embedding, retrieval, +model-output, reducer, and renderer boundaries without a model download. +`IB_TAB_QA_MODEL_COMMAND` can name a later adapter implementing `inspect`, +`embed`, and `answer`. `IB_TAB_QA_REDUCER_COMMAND` can replace the one-member +reducer. The two pinned Hugging Face embedding manifests remain available for +real model views; this mock writes `model-adapter.txt`, never a misleading +`embedding-model.txt` manifest. A model name is bound to one exact adapter +record, including the adapter command's SHA-256, so changed adapter code cannot +silently reuse old vectors. An adapter with external model or vocabulary files +must report their immutable hashes from `inspect` as part of the same record. + +## Filesystem boundaries + +Saved source text stays below `${IB_READING_DIR:-~/reading}`. The rebuildable +index defaults to: + +```text +${XDG_DATA_HOME:-~/.local/share}/ib/views/ + organizing-the-information/vector-spaces/mock-token-i8-v1/reading/ +``` + +Successful questions are appended as one directory per exchange below the +separate requested folder: + +```text +~/questions and answers about tabs that the user has visited/ + -/ + question.txt + model-candidate.txt + reduced-answer.txt + answer.txt + source.tsv + vector-query.tsv + vector-format.txt + reducer.tsv + indexing.tsv + model.tsv + reducer-model.tsv + reading-source.tsv + pipeline.tsv +``` + +The exchange preserves the raw model candidate, reduction evidence, selected +source identity and content hash, upstream reading-source record, exact vector +generation, adapter and reducer command hashes, vector execution report, and +final response separately. It snapshots the selected document for processing +and fails closed if those bytes differ from the indexed corpus record. The +snapshot is temporary: canonical reading text is neither moved nor duplicated +into the Q&A store. The command rejects a Q&A root equal to or nested with the +reading root. + +## Run the mock + +```text +make -C native/vector-index +sh bin/ask_saved_pages.grease --index +sh bin/ask_saved_pages.grease "What do feed-forward networks do to input space?" +``` + +Running the last command without a question displays `question>` and reads one +line from the console. The index is reused until `--index` is run again; corpus +change detection is deliberately left for the next slice. diff --git a/docs/vector-index.md b/docs/vector-index.md new file mode 100644 index 0000000..c26a2cb --- /dev/null +++ b/docs/vector-index.md @@ -0,0 +1,100 @@ +# Readable filesystem vector spaces + +IB's first vector backend is a flat exact scan whose source of truth is ordinary fixed-width text. It is deliberately a file tool, not a database. A row-major little-endian Float32 sidecar is a rebuildable query cache, not the only copy of the vectors. + +Several embedding models may materialize independent ways of organizing the same collection. Categories and vector spaces are siblings below `organizing-the-information`; no model is promoted to the one true representation. + +## Boundary + +Canonical URLs, visits, extracted text, and document identities remain ordinary inspectable browser records. Embeddings and their index are derived state and can be deleted and rebuilt. + +Idriç owns the index specification: collection, embedding model, dimensions, metric, and selected backend. `IB.VectorIndex` lowers that specification to a versioned process contract. Grease or thin platform glue may run the selected program. The initial program is `ib-vector-index`, implemented in C99 so the same source builds for Linux and the Android NDK. + +A replacement backend must implement the same standard-input/standard-output contract: + +```text +BACKEND build INDEX_DIRECTORY DIMENSIONS cosine|dot < rows.tsv +BACKEND query INDEX_DIRECTORY RESULT_COUNT < vector.txt +BACKEND query-text INDEX_DIRECTORY RESULT_COUNT < vector.txt +BACKEND column INDEX_DIRECTORY ONE_BASED_COORDINATE +BACKEND compile-cache INDEX_DIRECTORY +BACKEND check INDEX_DIRECTORY +BACKEND inspect INDEX_DIRECTORY +``` + +Build input has one row per line: + +```text +document-id0.1 -0.2 0.3 ... +``` + +Query input is one space-separated vector. Query output is score-descending text: + +```text +document-id0.8125 +``` + +`query` memory-maps the Float32 cache. `query-text` performs the same exact scan by parsing the readable file and works with the cache removed. `column` uses the fixed-width layout to print one coordinate across every ID. `compile-cache` atomically recreates the Float32 file from text. A later exact or approximate program can occupy the same process boundary without changing canonical browser state or callers that stream rows and queries. + +When `IB_VECTOR_QUERY_REPORT` names a file, either query command also writes a +small execution report containing the compute device, storage path, metric, +dimensions, and number of row dot products. The current portable C scan reports +`dot-product-used-gpu=False`; a future accelerated backend must report its own +execution rather than letting callers infer it. + +## Files + +One model-specific view lives below the derived view root: + +```text +views/organizing-the-information/vector-spaces// + embedding-model.txt + / + format.txt + ids-.txt + vectors-.txt + vectors-.f32 +``` + +`embedding-model.txt` is the exact pinned model manifest copied once beside its collections; the materializer refuses to bind the same slug to different model facts. Each collection's `format.txt` is the atomic pointer to one immutable vector generation. It records contract version 2, backend, metric, dimensions, row count, text encoding, text slot width, and current filenames. IDs and authoritative vectors remain text. The `.f32` file is row-major little-endian IEEE 754 Float32 derived from that text. + +Every scalar occupies exactly 16 ASCII bytes: a 15-byte signed scientific number with eight digits after the decimal point, followed by a space or the row-ending newline. For example: + +```text ++1.00000000e+00 +0.00000000e+00 -2.50000000e-01 +``` + +The format uses nine significant decimal digits, enough to round-trip a finite Float32. For zero-based row `r`, zero-based coordinate `c`, and `D` dimensions, the scalar begins at: + +```text +(r * D + c) * 16 bytes +``` + +There is therefore no auxiliary transpose index to understand or rebuild. A direct seek finds any scalar; `column` applies that arithmetic across rows. + +New builds write new generation files and replace `format.txt` last. A failed build therefore cannot make a partial generation current. Old generation files may be removed during serialized index maintenance after active queries finish. + +`check` validates every text slot and proves the cache contains the identical Float32 values. Ordinary cached queries only check structure and file sizes before scanning, so they do not pay for a redundant text parse. Deleting the cache does not lose the vector view: `query-text` continues to work and `compile-cache` restores it. + +## Metric and precision + +`cosine` normalizes stored and query vectors once and then uses a dot product. Zero, NaN, and infinite vectors are rejected. `dot` stores the supplied values without normalization. + +Similarity accumulation and the cache use 32-bit float. The text encoding preserves each stored Float32 exactly while remaining inspectable and interoperable. The rebuildable boundary lets a later backend use another cache representation without migrating the readable vectors. + +## Initial model views + +The initial pair is intentionally heterogeneous: + +| View | Mechanism | Pinned inference file | Coordinates | Purpose | +| --- | --- | ---: | ---: | --- | +| `potion-base-2m` | static token lookup, mean, normalize | 7,563,349-byte ONNX | 64 | very cheap always-on semantic view | +| `mxbai-embed-xsmall-v1-int8` | INT8 transformer, mean pool, normalize | 24,448,010-byte ONNX | 384 | retrieval-oriented challenger | + +Their manifests under `models/embedding/` pin the Hugging Face repository, immutable commit, preprocessing, every required file's size, and every required file's SHA-256. `bin/ib_embedding_model.grease` fetches and verifies those artifacts. `bin/ib_vector_view.grease` binds a manifest to its readable vector directory and invokes the backend with the manifest's dimensions. The disposable ONNX adapter under `experiments/embedding-models/` has run both models through this index contract; it is evidence for model selection, not a new Python browser runtime dependency. + +At 10,000 rows, the 64-coordinate view occupies 10.24 MB as readable text plus a 2.56 MB cache. The 384-coordinate view occupies 61.44 MB as text plus a 15.36 MB cache. One 384-coordinate exact query performs 3.84 million coordinate products. These sizes remain small enough that a graph database would add opaque persistent state and approximate behavior before this corpus needs either. + +## Growth path + +The text scan is the correctness reference for the Float32 cache and for any approximate replacement. Add an ANN cache only after measurements on the phone show exact-query latency or corpus size is actually a problem. Such a cache must remain disposable and reproducible from the readable vectors. diff --git a/experiments/embedding-models/README.md b/experiments/embedding-models/README.md new file mode 100644 index 0000000..36838cc --- /dev/null +++ b/experiments/embedding-models/README.md @@ -0,0 +1,33 @@ +# Pinned ONNX embedding comparison + +This disposable experiment proves that the two checked-in model descriptions can produce normalized vector rows for the filesystem index. Python is not an IB runtime layer; the model, vector-index, and process contracts are intentionally independent of this comparison adapter. + +The two deliberately different views are: + +| Slug | Mechanism | Weights fetched | Output | Role | +| --- | --- | ---: | ---: | --- | +| `potion-base-2m` | static token lookup and mean | 7.56 MB ONNX | 64 | cheapest always-available semantic view | +| `mxbai-embed-xsmall-v1-int8` | six-layer transformer, INT8 ONNX | 24.45 MB ONNX | 384 | slower retrieval-quality challenger | + +Every required file is tied to an immutable Hugging Face commit, byte count, and SHA-256 in `models/embedding/*.model`. Fetching is explicit and model weights are not committed to this repository. + +```text +bin/ib_embedding_model.grease fetch \ + models/embedding/potion-base-2m.model build/models/potion-base-2m + +python3 -m venv build/embedding-venv +build/embedding-venv/bin/pip install -r experiments/embedding-models/requirements.txt +build/embedding-venv/bin/python experiments/embedding-models/embed_onnx.py \ + --model-manifest models/embedding/potion-base-2m.model \ + --model-directory build/models/potion-base-2m \ + --input tests/fixtures/embedding-corpus.tsv \ + --output build/potion-rows.tsv \ + --npz build/potion-vectors.npz \ + --provenance build/potion-run.json + +bin/ib_vector_view.grease build \ + build/views pages models/embedding/potion-base-2m.model cosine \ + native/vector-index/ib-vector-index < build/potion-rows.tsv +``` + +The NumPy archive is accepted by the category-hyperplane probe. The row TSV is accepted directly by the C99 index. `provenance` records the exact manifest, source text, artifacts, preprocessing policy, and inference-library versions. diff --git a/experiments/embedding-models/embed_onnx.py b/experiments/embedding-models/embed_onnx.py new file mode 100755 index 0000000..db60d6e --- /dev/null +++ b/experiments/embedding-models/embed_onnx.py @@ -0,0 +1,451 @@ +#!/usr/bin/env python3 +"""Produce reproducible IB vector rows from a pinned local ONNX model. + +This is an experiment adapter, not a browser implementation layer. It keeps +model comparison executable while the selected inference path is moved behind +IB's native/Grease boundary. +""" + +from __future__ import annotations + +import argparse +import csv +import hashlib +import json +import os +import re +import tempfile +from dataclasses import dataclass +from pathlib import Path +from typing import TextIO + +# Official ONNX Runtime wheels enable cross-platform telemetry by default. +# The browser experiment neither needs nor permits it. +os.environ.setdefault("ORT_DISABLE_TELEMETRY", "1") + +import numpy as np +import onnxruntime as ort +import tokenizers +from tokenizers import Tokenizer + + +SHA256_PATTERN = re.compile(r"[0-9a-f]{64}") +REVISION_PATTERN = re.compile(r"[0-9a-f]{40}") +SAFE_PATH_PATTERN = re.compile(r"[A-Za-z0-9._-]+(?:/[A-Za-z0-9._-]+)*") +REQUIRED_FIELDS = { + "slug", + "repository", + "revision", + "license", + "runtime", + "weight_precision", + "dimensions", + "max_tokens", + "pooling", + "normalization", + "token_policy", + "padding", + "query_prefix", + "document_prefix", + "tokenizer_file", + "onnx_file", + "onnx_output", +} + + +@dataclass(frozen=True) +class Artifact: + relative_path: str + byte_count: int + sha256: str + + +@dataclass(frozen=True) +class ModelManifest: + path: Path + fields: dict[str, str] + artifacts: tuple[Artifact, ...] + + def value(self, name: str) -> str: + return self.fields[name] + + @property + def dimensions(self) -> int: + return int(self.value("dimensions")) + + @property + def max_tokens(self) -> int: + return int(self.value("max_tokens")) + + +def sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for block in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def safe_relative_path(value: str) -> bool: + return bool(SAFE_PATH_PATTERN.fullmatch(value)) and all( + part not in {".", ".."} for part in value.split("/") + ) + + +def load_manifest(path: Path) -> ModelManifest: + lines = path.read_text(encoding="utf-8").splitlines() + if not lines or lines[0] != "ib-embedding-model 1": + raise ValueError("unsupported embedding-model manifest header") + fields: dict[str, str] = {} + artifacts: list[Artifact] = [] + artifact_paths: set[str] = set() + for line_number, line in enumerate(lines[1:], start=2): + parts = line.split(" ") + if parts[0] == "artifact": + if len(parts) != 4 or not safe_relative_path(parts[1]): + raise ValueError(f"manifest line {line_number}: invalid artifact row") + try: + byte_count = int(parts[2]) + except ValueError as error: + raise ValueError( + f"manifest line {line_number}: invalid artifact byte count" + ) from error + if byte_count <= 0 or not SHA256_PATTERN.fullmatch(parts[3]): + raise ValueError(f"manifest line {line_number}: invalid artifact facts") + if parts[1] in artifact_paths: + raise ValueError(f"manifest line {line_number}: duplicate artifact") + artifact_paths.add(parts[1]) + artifacts.append(Artifact(parts[1], byte_count, parts[3])) + elif len(parts) == 2 and parts[0] in REQUIRED_FIELDS: + if parts[0] in fields: + raise ValueError(f"manifest line {line_number}: duplicate field") + fields[parts[0]] = parts[1] + else: + raise ValueError(f"manifest line {line_number}: unknown or malformed row") + missing = REQUIRED_FIELDS - fields.keys() + if missing: + raise ValueError(f"manifest is missing fields: {', '.join(sorted(missing))}") + if not REVISION_PATTERN.fullmatch(fields["revision"]): + raise ValueError("manifest revision must be a full commit SHA") + if not re.fullmatch(r"[a-z0-9][a-z0-9._-]*", fields["slug"]): + raise ValueError("manifest slug is unsafe") + if not re.fullmatch( + r"[A-Za-z0-9._-]+/[A-Za-z0-9._-]+", fields["repository"] + ): + raise ValueError("manifest repository is invalid") + if not artifacts: + raise ValueError("manifest has no artifacts") + if fields["tokenizer_file"] not in artifact_paths: + raise ValueError("manifest tokenizer_file is not a pinned artifact") + if fields["onnx_file"] not in artifact_paths: + raise ValueError("manifest onnx_file is not a pinned artifact") + if int(fields["dimensions"]) <= 0 or int(fields["max_tokens"]) <= 0: + raise ValueError("manifest dimensions and max_tokens must be positive") + if fields["normalization"] != "l2": + raise ValueError("this adapter requires l2-normalized output") + if fields["query_prefix"] != "none" or fields["document_prefix"] != "none": + raise ValueError("this adapter does not implement input prefixes") + expected_policy = { + "model2vec-onnx": { + "pooling": "static-token-mean", + "token_policy": "exclude-special-and-unknown", + "padding": "none", + }, + "sentence-transformer-onnx": { + "pooling": "attention-mask-mean", + "token_policy": "include-special", + "padding": "dynamic-right", + }, + } + if fields["runtime"] not in expected_policy: + raise ValueError(f"unsupported model runtime: {fields['runtime']}") + for name, expected in expected_policy[fields["runtime"]].items(): + if fields[name] != expected: + raise ValueError( + f"manifest {name}={fields[name]!r} is incompatible with " + f"runtime {fields['runtime']!r}" + ) + return ModelManifest(path, fields, tuple(artifacts)) + + +def verify_model(manifest: ModelManifest, model_directory: Path) -> None: + for artifact in manifest.artifacts: + path = model_directory / artifact.relative_path + if not path.is_file(): + raise ValueError(f"model artifact is missing: {artifact.relative_path}") + if path.stat().st_size != artifact.byte_count: + raise ValueError(f"model artifact has the wrong size: {artifact.relative_path}") + if sha256(path) != artifact.sha256: + raise ValueError(f"model artifact has the wrong SHA-256: {artifact.relative_path}") + + +def read_inputs(path: Path) -> tuple[list[str], list[str]]: + row_ids: list[str] = [] + texts: list[str] = [] + with path.open("r", encoding="utf-8", newline="") as stream: + reader = csv.DictReader(stream, delimiter="\t") + if reader.fieldnames != ["id", "text"]: + raise ValueError("input TSV header must be exactly: idtext") + for line_number, row in enumerate(reader, start=2): + if None in row or not row["id"] or not row["text"]: + raise ValueError(f"input TSV row {line_number}: empty or extra field") + if "\t" in row["id"] or "\n" in row["id"] or "\r" in row["id"]: + raise ValueError(f"input TSV row {line_number}: unsafe id") + row_ids.append(row["id"]) + texts.append(row["text"]) + if not row_ids: + raise ValueError("input TSV has no data rows") + if len(row_ids) != len(set(row_ids)): + raise ValueError("input TSV ids must be unique") + return row_ids, texts + + +def session_for( + manifest: ModelManifest, model_directory: Path, threads: int +) -> ort.InferenceSession: + options = ort.SessionOptions() + options.execution_mode = ort.ExecutionMode.ORT_SEQUENTIAL + options.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL + options.intra_op_num_threads = threads + options.inter_op_num_threads = 1 + options.log_severity_level = 3 + return ort.InferenceSession( + str(model_directory / manifest.value("onnx_file")), + sess_options=options, + providers=["CPUExecutionProvider"], + ) + + +def encode_model2vec( + session: ort.InferenceSession, + tokenizer: Tokenizer, + texts: list[str], + max_tokens: int, + output_name: str, +) -> np.ndarray: + tokenizer.no_padding() + tokenizer.no_truncation() + unknown_id = tokenizer.token_to_id("[UNK]") + batches: list[list[int]] = [] + for encoding in tokenizer.encode_batch(texts, add_special_tokens=False): + token_ids = encoding.ids + if unknown_id is not None: + token_ids = [token_id for token_id in token_ids if token_id != unknown_id] + token_ids = token_ids[:max_tokens] + if not token_ids: + raise ValueError("Model2Vec input produced no known tokens") + batches.append(token_ids) + offsets = np.asarray( + np.cumsum([0] + [len(token_ids) for token_ids in batches[:-1]]), + dtype=np.int64, + ) + input_ids = np.asarray( + [token_id for token_ids in batches for token_id in token_ids], dtype=np.int64 + ) + return session.run( + [output_name], {"input_ids": input_ids, "offsets": offsets} + )[0] + + +def encode_sentence_transformer( + session: ort.InferenceSession, + tokenizer: Tokenizer, + texts: list[str], + max_tokens: int, + output_name: str, +) -> np.ndarray: + tokenizer.no_padding() + tokenizer.no_truncation() + tokenizer.enable_truncation(max_length=max_tokens) + pad_id = tokenizer.token_to_id("[PAD]") + if pad_id is None: + raise ValueError("sentence-transformer tokenizer has no [PAD] token") + tokenizer.enable_padding(direction="right", pad_id=pad_id, pad_token="[PAD]") + encodings = tokenizer.encode_batch(texts, add_special_tokens=True) + input_ids = np.asarray([encoding.ids for encoding in encodings], dtype=np.int64) + attention_mask = np.asarray( + [encoding.attention_mask for encoding in encodings], dtype=np.int64 + ) + return session.run( + [output_name], + {"input_ids": input_ids, "attention_mask": attention_mask}, + )[0] + + +def normalize_rows(vectors: np.ndarray, dimensions: int) -> np.ndarray: + vectors = np.asarray(vectors, dtype=np.float32) + if vectors.ndim != 2 or vectors.shape[1] != dimensions: + raise ValueError( + f"model emitted shape {vectors.shape!r}, expected (*, {dimensions})" + ) + if not np.isfinite(vectors).all(): + raise ValueError("model emitted a non-finite value") + norms = np.linalg.norm(vectors, axis=1, keepdims=True) + if np.any(norms == 0): + raise ValueError("model emitted a zero vector") + return np.asarray(vectors / norms, dtype=np.float32) + + +def encode( + manifest: ModelManifest, + model_directory: Path, + texts: list[str], + batch_size: int, + threads: int, +) -> np.ndarray: + tokenizer = Tokenizer.from_file(str(model_directory / manifest.value("tokenizer_file"))) + session = session_for(manifest, model_directory, threads) + outputs: list[np.ndarray] = [] + for start in range(0, len(texts), batch_size): + batch = texts[start : start + batch_size] + if manifest.value("runtime") == "model2vec-onnx": + vectors = encode_model2vec( + session, + tokenizer, + batch, + manifest.max_tokens, + manifest.value("onnx_output"), + ) + elif manifest.value("runtime") == "sentence-transformer-onnx": + vectors = encode_sentence_transformer( + session, + tokenizer, + batch, + manifest.max_tokens, + manifest.value("onnx_output"), + ) + else: + raise ValueError(f"unsupported model runtime: {manifest.value('runtime')}") + outputs.append(normalize_rows(vectors, manifest.dimensions)) + return np.concatenate(outputs, axis=0) + + +def write_rows(stream: TextIO, row_ids: list[str], vectors: np.ndarray) -> None: + for row_id, vector in zip(row_ids, vectors, strict=True): + values = " ".join(f"{float(value):+.9e}" for value in vector) + stream.write(f"{row_id}\t{values}\n") + + +def atomic_text_output(path: Path, row_ids: list[str], vectors: np.ndarray) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + descriptor, temporary_name = tempfile.mkstemp( + prefix=f".{path.name}.", suffix=".tmp", dir=path.parent + ) + try: + with os.fdopen(descriptor, "w", encoding="utf-8", newline="") as stream: + write_rows(stream, row_ids, vectors) + stream.flush() + os.fsync(stream.fileno()) + os.replace(temporary_name, path) + except BaseException: + try: + os.unlink(temporary_name) + except FileNotFoundError: + pass + raise + + +def provenance( + manifest: ModelManifest, + input_path: Path, + row_count: int, + batch_size: int, + threads: int, +) -> dict[str, object]: + return { + "format": "ib-embedding-run 1", + "model": { + "slug": manifest.value("slug"), + "repository": manifest.value("repository"), + "revision": manifest.value("revision"), + "runtime": manifest.value("runtime"), + "weight_precision": manifest.value("weight_precision"), + "dimensions": manifest.dimensions, + "max_tokens": manifest.max_tokens, + "pooling": manifest.value("pooling"), + "normalization": manifest.value("normalization"), + "token_policy": manifest.value("token_policy"), + "padding": manifest.value("padding"), + "query_prefix": manifest.value("query_prefix"), + "document_prefix": manifest.value("document_prefix"), + }, + "manifest_sha256": sha256(manifest.path), + "input_sha256": sha256(input_path), + "row_count": row_count, + "adapter": {"batch_size": batch_size, "threads": threads}, + "artifacts": [ + { + "path": artifact.relative_path, + "bytes": artifact.byte_count, + "sha256": artifact.sha256, + } + for artifact in manifest.artifacts + ], + "software": { + "numpy": np.__version__, + "onnxruntime": ort.__version__, + "tokenizers": tokenizers.__version__, + }, + } + + +def parse_arguments() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--model-manifest", type=Path, required=True) + parser.add_argument("--model-directory", type=Path, required=True) + parser.add_argument("--input", type=Path, required=True) + parser.add_argument("--output", default="-", help="vector rows TSV, or - for stdout") + parser.add_argument("--npz", type=Path) + parser.add_argument("--provenance", type=Path) + parser.add_argument("--batch-size", type=int, default=64) + parser.add_argument("--threads", type=int, default=1) + return parser.parse_args() + + +def main() -> None: + arguments = parse_arguments() + if arguments.batch_size <= 0 or arguments.threads <= 0: + raise ValueError("batch size and thread count must be positive") + manifest = load_manifest(arguments.model_manifest) + verify_model(manifest, arguments.model_directory) + row_ids, texts = read_inputs(arguments.input) + vectors = encode( + manifest, + arguments.model_directory, + texts, + arguments.batch_size, + arguments.threads, + ) + if arguments.output == "-": + import sys + + write_rows(sys.stdout, row_ids, vectors) + else: + atomic_text_output(Path(arguments.output), row_ids, vectors) + if arguments.npz: + arguments.npz.parent.mkdir(parents=True, exist_ok=True) + np.savez( + arguments.npz, + ids=np.asarray(row_ids, dtype=str), + vectors=vectors, + ) + if arguments.provenance: + arguments.provenance.parent.mkdir(parents=True, exist_ok=True) + arguments.provenance.write_text( + json.dumps( + provenance( + manifest, + arguments.input, + len(row_ids), + arguments.batch_size, + arguments.threads, + ), + indent=2, + ) + + "\n", + encoding="utf-8", + ) + + +if __name__ == "__main__": + main() diff --git a/experiments/embedding-models/requirements.txt b/experiments/embedding-models/requirements.txt new file mode 100644 index 0000000..db9c692 --- /dev/null +++ b/experiments/embedding-models/requirements.txt @@ -0,0 +1,3 @@ +numpy==2.3.5 +onnxruntime==1.29.0 +tokenizers==0.23.1 diff --git a/lib/filesystem_views.grease b/lib/filesystem_views.grease new file mode 100644 index 0000000..2a4a90a --- /dev/null +++ b/lib/filesystem_views.grease @@ -0,0 +1,335 @@ +# Grease source. This layer materializes filesystem control surfaces over +# browser-owned state and human-readable presentation artifacts. The links +# are rebuildable views; they do not become canonical browser records. + +ib_views_fail() { + printf '%s\n' "$*" >&2 + return 1 +} + +ib_views_require_command() { + command -v "$1" >/dev/null 2>&1 || ib_views_fail "missing command: $1" +} + +ib_views_validate_component() ( + ib_views_component_label=$1 + ib_views_component_value=$2 + + case "$ib_views_component_value" in + ''|.|..|*/*) + ib_views_fail "unsafe $ib_views_component_label: $ib_views_component_value" + return 1 + ;; + *' +'*) + ib_views_fail "$ib_views_component_label contains a newline" + return 1 + ;; + esac +) + +ib_views_validate_relative_path() ( + ib_views_relative_path=$1 + + case "$ib_views_relative_path" in + ''|.|/*|./*|*/./*|*/.|..|../*|*/../*|*/..) + ib_views_fail "unsafe relative target: $ib_views_relative_path" + return 1 + ;; + *' +'*) + ib_views_fail 'relative target contains a newline' + return 1 + ;; + esac +) + +ib_views_existing_target() ( + ib_views_allowed_root=$1 + ib_views_relative_target=$2 + + ib_views_require_command realpath || return 1 + ib_views_validate_relative_path "$ib_views_relative_target" || return 1 + test -d "$ib_views_allowed_root" || { + ib_views_fail "target root is not a directory: $ib_views_allowed_root" + return 1 + } + + ib_views_physical_root=$(realpath -e "$ib_views_allowed_root") || return 1 + ib_views_physical_target=$(realpath -e "$ib_views_allowed_root/$ib_views_relative_target") || { + ib_views_fail "view target does not exist: $ib_views_relative_target" + return 1 + } + + case "$ib_views_physical_target" in + "$ib_views_physical_root"/*) ;; + *) + ib_views_fail "view target escapes its allowed root: $ib_views_relative_target" + return 1 + ;; + esac + + printf '%s\n' "$ib_views_physical_target" +) + +ib_views_atomic_link() ( + ib_views_link_target=$1 + ib_views_link_path=$2 + ib_views_link_directory=$(dirname "$ib_views_link_path") + ib_views_link_name=$(basename "$ib_views_link_path") + + ib_views_require_command ln || return 1 + ib_views_require_command mktemp || return 1 + ib_views_require_command mv || return 1 + ib_views_require_command readlink || return 1 + mkdir -p "$ib_views_link_directory" + + if test -e "$ib_views_link_path" || test -L "$ib_views_link_path"; then + test -L "$ib_views_link_path" || { + ib_views_fail "refusing to replace non-link view entry: $ib_views_link_path" + return 1 + } + if test "$(readlink "$ib_views_link_path")" = "$ib_views_link_target"; then + printf '%s\n' "$ib_views_link_path" + return 0 + fi + fi + + ib_views_temporary_directory=$(mktemp -d "$ib_views_link_directory/.${ib_views_link_name}.link.XXXXXX") || return 1 + ib_views_temporary_link="$ib_views_temporary_directory/candidate" + if ! ln -s "$ib_views_link_target" "$ib_views_temporary_link"; then + rmdir "$ib_views_temporary_directory" + return 1 + fi + if ! mv -Tf "$ib_views_temporary_link" "$ib_views_link_path"; then + rm -f "$ib_views_temporary_link" + rmdir "$ib_views_temporary_directory" + return 1 + fi + rmdir "$ib_views_temporary_directory" + + printf '%s\n' "$ib_views_link_path" +) + +ib_views_remove_link() ( + ib_views_link_path=$1 + + if ! test -e "$ib_views_link_path" && ! test -L "$ib_views_link_path"; then + return 0 + fi + test -L "$ib_views_link_path" || { + ib_views_fail "refusing to remove non-link view entry: $ib_views_link_path" + return 1 + } + rm -f "$ib_views_link_path" +) + +ib_views_initialize() ( + ib_views_root=$1 + + case "$ib_views_root" in + ''|/) + ib_views_fail "unsafe view root: $ib_views_root" + return 1 + ;; + esac + mkdir -p "$ib_views_root" + for ib_views_relative_directory in \ + retrieved-from-the-web \ + organizing-the-information \ + _active \ + hot \ + retrieved-from-the-web/text \ + retrieved-from-the-web/images \ + organizing-the-information/categories \ + organizing-the-information/vector-spaces + do + ib_views_directory="$ib_views_root/$ib_views_relative_directory" + test ! -L "$ib_views_directory" || { + ib_views_fail "view directory must not be a symbolic link: $ib_views_directory" + return 1 + } + if test -e "$ib_views_directory" && ! test -d "$ib_views_directory"; then + ib_views_fail "view directory path is not a directory: $ib_views_directory" + return 1 + fi + mkdir -p "$ib_views_directory" + done +) + +ib_category_add() ( + ib_views_root=$1 + ib_views_state_root=$2 + ib_views_category=$3 + ib_views_object_kind=$4 + ib_views_object_id=$5 + + ib_views_initialize "$ib_views_root" || return 1 + ib_views_validate_component category "$ib_views_category" || return 1 + test "$ib_views_category" != _active || { + ib_views_fail '_active is reserved for active-category links' + return 1 + } + ib_views_validate_component object-id "$ib_views_object_id" || return 1 + + case "$ib_views_object_kind" in + tab) ib_views_object_relative_path="tabs/$ib_views_object_id" ;; + resource) ib_views_object_relative_path="resources/$ib_views_object_id" ;; + *) + ib_views_fail "unsupported category target kind: $ib_views_object_kind" + return 1 + ;; + esac + + ib_views_object_target=$( + ib_views_existing_target "$ib_views_state_root" "$ib_views_object_relative_path" + ) || return 1 + + ib_views_category_directory="$ib_views_root/organizing-the-information/categories/$ib_views_category" + test ! -L "$ib_views_category_directory" || { + ib_views_fail "category directory must not be a symbolic link: $ib_views_category" + return 1 + } + if test -e "$ib_views_category_directory" && ! test -d "$ib_views_category_directory"; then + ib_views_fail "category path is not a directory: $ib_views_category" + return 1 + fi + mkdir -p "$ib_views_category_directory" + + ib_views_atomic_link \ + "$ib_views_object_target" \ + "$ib_views_category_directory/$ib_views_object_kind-$ib_views_object_id" +) + +ib_category_remove() ( + ib_views_root=$1 + ib_views_category=$2 + ib_views_object_kind=$3 + ib_views_object_id=$4 + + ib_views_initialize "$ib_views_root" || return 1 + ib_views_validate_component category "$ib_views_category" || return 1 + test "$ib_views_category" != _active || { + ib_views_fail '_active is reserved for active-category links' + return 1 + } + ib_views_validate_component object-id "$ib_views_object_id" || return 1 + case "$ib_views_object_kind" in + tab|resource) ;; + *) + ib_views_fail "unsupported category target kind: $ib_views_object_kind" + return 1 + ;; + esac + + ib_views_category_directory="$ib_views_root/organizing-the-information/categories/$ib_views_category" + test ! -L "$ib_views_category_directory" || { + ib_views_fail "category directory must not be a symbolic link: $ib_views_category" + return 1 + } + ib_views_remove_link \ + "$ib_views_category_directory/$ib_views_object_kind-$ib_views_object_id" +) + +ib_category_activate() ( + ib_views_root=$1 + ib_views_category=$2 + + ib_views_initialize "$ib_views_root" || return 1 + ib_views_validate_component category "$ib_views_category" || return 1 + test "$ib_views_category" != _active || { + ib_views_fail '_active cannot activate itself' + return 1 + } + + ib_views_category_directory="$ib_views_root/organizing-the-information/categories/$ib_views_category" + test -d "$ib_views_category_directory" && test ! -L "$ib_views_category_directory" || { + ib_views_fail "category does not exist: $ib_views_category" + return 1 + } + ib_views_category_target=$(realpath -e "$ib_views_category_directory") || return 1 + + ib_views_atomic_link \ + "$ib_views_category_target" \ + "$ib_views_root/_active/$ib_views_category" +) + +ib_category_deactivate() ( + ib_views_root=$1 + ib_views_category=$2 + + ib_views_initialize "$ib_views_root" || return 1 + ib_views_validate_component category "$ib_views_category" || return 1 + ib_views_remove_link "$ib_views_root/_active/$ib_views_category" +) + +ib_category_memberships() ( + ib_views_root=$1 + ib_views_state_root=$2 + ib_views_object_kind=$3 + ib_views_object_id=$4 + + ib_views_initialize "$ib_views_root" || return 1 + ib_views_validate_component object-id "$ib_views_object_id" || return 1 + case "$ib_views_object_kind" in + tab) ib_views_object_relative_path="tabs/$ib_views_object_id" ;; + resource) ib_views_object_relative_path="resources/$ib_views_object_id" ;; + *) + ib_views_fail "unsupported category target kind: $ib_views_object_kind" + return 1 + ;; + esac + ib_views_expected_target=$( + ib_views_existing_target "$ib_views_state_root" "$ib_views_object_relative_path" + ) || return 1 + + for ib_views_category_directory in "$ib_views_root/organizing-the-information/categories/"*; do + test -d "$ib_views_category_directory" || continue + test ! -L "$ib_views_category_directory" || continue + ib_views_category=$(basename "$ib_views_category_directory") + ib_views_member_path="$ib_views_category_directory/$ib_views_object_kind-$ib_views_object_id" + test -L "$ib_views_member_path" || continue + test "$(readlink "$ib_views_member_path")" = "$ib_views_expected_target" || continue + printf '%s\n' "$ib_views_category" + done +) + +ib_hot_add() ( + ib_views_root=$1 + ib_views_presentation_root=$2 + ib_views_slot=$3 + ib_views_presentation_relative_path=$4 + + ib_views_initialize "$ib_views_root" || return 1 + ib_views_validate_component hot-slot "$ib_views_slot" || return 1 + ib_views_presentation_target=$( + ib_views_existing_target \ + "$ib_views_presentation_root" \ + "$ib_views_presentation_relative_path" + ) || return 1 + + ib_views_atomic_link \ + "$ib_views_presentation_target" \ + "$ib_views_root/hot/$ib_views_slot" +) + +ib_hot_remove() ( + ib_views_root=$1 + ib_views_slot=$2 + + ib_views_initialize "$ib_views_root" || return 1 + ib_views_validate_component hot-slot "$ib_views_slot" || return 1 + ib_views_remove_link "$ib_views_root/hot/$ib_views_slot" +) + +ib_views_prune_broken_links() ( + ib_views_root=$1 + + ib_views_initialize "$ib_views_root" || return 1 + ib_views_require_command find || return 1 + find "$ib_views_root" -type l -print | while IFS= read -r ib_views_link_path; do + test -e "$ib_views_link_path" && continue + printf '%s\n' "$ib_views_link_path" + rm -f "$ib_views_link_path" + done +) diff --git a/lib/tab_qa.grease b/lib/tab_qa.grease new file mode 100644 index 0000000..de9c713 --- /dev/null +++ b/lib/tab_qa.grease @@ -0,0 +1,398 @@ +# Grease source. This is the first console-sized vertical slice of the +# text-and-action workbench over material deliberately saved in ~/reading. + +ib_tab_qa_fail() { + printf 'tab Q&A: %s\n' "$1" >&2 + return 1 +} + +ib_tab_qa_reading_root() { + if test -n "${IB_READING_DIR:-}"; then + printf '%s\n' "$IB_READING_DIR" + else + test -n "${HOME:-}" || return 1 + printf '%s/reading\n' "$HOME" + fi +} + +ib_tab_qa_root() { + if test -n "${IB_TAB_QA_DIR:-}"; then + printf '%s\n' "$IB_TAB_QA_DIR" + else + test -n "${HOME:-}" || return 1 + printf '%s/questions and answers about tabs that the user has visited\n' "$HOME" + fi +} + +ib_tab_qa_views_root() { + if test -n "${IB_VIEWS_DIR:-}"; then + printf '%s\n' "$IB_VIEWS_DIR" + else + test -n "${HOME:-}" || return 1 + data_home=${XDG_DATA_HOME:-$HOME/.local/share} + printf '%s/ib/views\n' "$data_home" + fi +} + +ib_tab_qa_model_command() { + printf '%s\n' "${IB_TAB_QA_MODEL_COMMAND:-$IB_TAB_QA_REPOSITORY_ROOT/bin/mock_tab_model.grease}" +} + +ib_tab_qa_reducer_command() { + printf '%s\n' "${IB_TAB_QA_REDUCER_COMMAND:-$IB_TAB_QA_REPOSITORY_ROOT/bin/mock_tab_qa_reducer.grease}" +} + +ib_tab_qa_vector_program() { + printf '%s\n' "${IB_VECTOR_INDEX_PROGRAM:-$IB_TAB_QA_REPOSITORY_ROOT/native/vector-index/ib-vector-index}" +} + +ib_tab_qa_metadata_value() { + metadata_key=$1 + sed -n "s/^${metadata_key}=//p" | sed -n '1p' +} + +ib_tab_qa_sha256() { + command -v sha256sum >/dev/null 2>&1 || { + ib_tab_qa_fail 'sha256sum is unavailable' + return 1 + } + checksum_line=$(sha256sum "$1") || return 1 + checksum=${checksum_line%% *} + case "$checksum" in + ''|*[!0-9a-f]*) ib_tab_qa_fail "invalid SHA-256 for: $1"; return 1 ;; + esac + test "${#checksum}" -eq 64 || { + ib_tab_qa_fail "invalid SHA-256 for: $1" + return 1 + } + printf '%s\n' "$checksum" +} + +ib_tab_qa_safe_component() { + case "$1" in + ''|.|..|*[!A-Za-z0-9._-]*) return 1 ;; + *) return 0 ;; + esac +} + +ib_tab_qa_safe_relative_document() { + relative_document=$1 + tab_character=$(printf '\t') + case "$relative_document" in + ''|/*|.|..|../*|*/../*|*/..|*"$tab_character"*|*' +'*) return 1 ;; + *) return 0 ;; + esac +} + +ib_tab_qa_separate_roots() { + command -v realpath >/dev/null 2>&1 || return 1 + first_root=$(realpath -m "$1") || return 1 + second_root=$(realpath -m "$2") || return 1 + case "$first_root/" in + "$second_root/"*) return 1 ;; + esac + case "$second_root/" in + "$first_root/"*) return 1 ;; + esac + return 0 +} + +ib_tab_qa_model_metadata() { + model_command=$(ib_tab_qa_model_command) || return 1 + test -x "$model_command" || { + ib_tab_qa_fail "model is unavailable: $model_command" + return 1 + } + model_report=$("$model_command" inspect) || return 1 + model_command_sha256=$(ib_tab_qa_sha256 "$model_command") || return 1 + printf '%s\nadapter-command-sha256=%s\n' "$model_report" "$model_command_sha256" +} + +ib_tab_qa_reducer_metadata() { + reducer_command=$(ib_tab_qa_reducer_command) || return 1 + test -x "$reducer_command" || { + ib_tab_qa_fail "reducer is unavailable: $reducer_command" + return 1 + } + reducer_report=$("$reducer_command" inspect) || return 1 + reducer_command_sha256=$(ib_tab_qa_sha256 "$reducer_command") || return 1 + printf '%s\nreducer-command-sha256=%s\n' "$reducer_report" "$reducer_command_sha256" +} + +ib_tab_qa_model_name() { + model_metadata=$(ib_tab_qa_model_metadata) || return 1 + model_name=$(printf '%s\n' "$model_metadata" | ib_tab_qa_metadata_value model) + ib_tab_qa_safe_component "$model_name" || { + ib_tab_qa_fail "model reported an unsafe name: $model_name" + return 1 + } + printf '%s\n' "$model_name" +} + +ib_tab_qa_index_directory() { + views_root=$(ib_tab_qa_views_root) || return 1 + model_name=$(ib_tab_qa_model_name) || return 1 + printf '%s/organizing-the-information/vector-spaces/%s/reading\n' \ + "$views_root" "$model_name" +} + +ib_tab_qa_build_index() ( + build_work= + cleanup() { + test -z "$build_work" || rm -rf "$build_work" + } + trap cleanup EXIT HUP INT TERM + + reading_root=$(ib_tab_qa_reading_root) || return 1 + views_root=$(ib_tab_qa_views_root) || return 1 + index_directory=$(ib_tab_qa_index_directory) || return 1 + model_command=$(ib_tab_qa_model_command) || return 1 + vector_program=$(ib_tab_qa_vector_program) || return 1 + + test -d "$reading_root" || ib_tab_qa_fail "reading directory is missing: $reading_root" || return 1 + test -x "$model_command" || ib_tab_qa_fail "model is unavailable: $model_command" || return 1 + test -x "$vector_program" || ib_tab_qa_fail "vector backend is unavailable: $vector_program" || return 1 + command -v find >/dev/null 2>&1 || ib_tab_qa_fail 'find is unavailable' || return 1 + command -v sha256sum >/dev/null 2>&1 || ib_tab_qa_fail 'sha256sum is unavailable' || return 1 + + model_metadata=$(ib_tab_qa_model_metadata) || return 1 + model_available=$(printf '%s\n' "$model_metadata" | ib_tab_qa_metadata_value available) + model_name=$(printf '%s\n' "$model_metadata" | ib_tab_qa_metadata_value model) + model_dimensions=$(printf '%s\n' "$model_metadata" | ib_tab_qa_metadata_value dimensions) + test "$model_available" = True || ib_tab_qa_fail "model did not report available: $model_command" || return 1 + ib_tab_qa_safe_component "$model_name" || ib_tab_qa_fail "model reported an unsafe name: $model_name" || return 1 + case "$model_dimensions" in + ''|*[!0-9]*) ib_tab_qa_fail 'model dimensions are not a positive integer' || return 1 ;; + esac + test "$model_dimensions" -gt 0 || ib_tab_qa_fail 'model dimensions must be positive' || return 1 + + sh "$IB_TAB_QA_REPOSITORY_ROOT/bin/ib_views.grease" init "$views_root" + model_view_directory=$(dirname "$index_directory") + test ! -L "$model_view_directory" || ib_tab_qa_fail "model view is a symbolic link: $model_view_directory" || return 1 + mkdir -p "$model_view_directory" + adapter_record="$model_view_directory/model-adapter.txt" + if test -e "$adapter_record" || test -L "$adapter_record"; then + test -f "$adapter_record" && test ! -L "$adapter_record" || \ + ib_tab_qa_fail "model adapter record is not a regular file: $adapter_record" || return 1 + recorded_model_metadata=$(cat "$adapter_record") + test "$recorded_model_metadata" = "$model_metadata" || \ + ib_tab_qa_fail "model name is already bound to different adapter facts: $model_name" || return 1 + else + adapter_temporary=$(mktemp "$model_view_directory/.model-adapter.XXXXXX") || return 1 + printf '%s\n' "$model_metadata" > "$adapter_temporary" + mv -T "$adapter_temporary" "$adapter_record" + fi + + build_work=$(mktemp -d) || return 1 + document_paths="$build_work/document-paths.txt" + vector_rows="$build_work/vector-rows.tsv" + corpus_rows="$build_work/corpus.tsv" + build_report="$build_work/build-report.txt" + : > "$vector_rows" + : > "$corpus_rows" + find "$reading_root" -type f -name document.txt -print | LC_ALL=C sort > "$document_paths" + + document_count=0 + while IFS= read -r document_path; do + test -n "$document_path" || continue + document_id=${document_path#"${reading_root%/}"/} + ib_tab_qa_safe_relative_document "$document_id" || { + ib_tab_qa_fail "unsafe reading document identity: $document_id" + return 1 + } + test -s "$document_path" || ib_tab_qa_fail "reading document is empty: $document_id" || return 1 + document_vector=$("$model_command" embed < "$document_path") || return 1 + vector_dimensions=$(printf '%s\n' "$document_vector" | awk 'NR == 1 { print NF }') + test "$vector_dimensions" = "$model_dimensions" || { + ib_tab_qa_fail "model emitted $vector_dimensions dimensions, expected $model_dimensions" + return 1 + } + document_sha256=$(ib_tab_qa_sha256 "$document_path") || return 1 + document_bytes=$(wc -c < "$document_path" | tr -d '[:space:]') + printf '%s\t%s\n' "$document_id" "$document_vector" >> "$vector_rows" + printf '%s\t%s\t%s\n' "$document_id" "$document_sha256" "$document_bytes" >> "$corpus_rows" + document_count=$((document_count + 1)) + done < "$document_paths" + + test "$document_count" -gt 0 || ib_tab_qa_fail "no document.txt files exist below $reading_root" || return 1 + "$vector_program" build "$index_directory" "$model_dimensions" cosine \ + < "$vector_rows" > "$build_report" + "$vector_program" check "$index_directory" >/dev/null + + corpus_temporary=$(mktemp "$index_directory/.corpus.XXXXXX") || return 1 + cp "$corpus_rows" "$corpus_temporary" + mv -T "$corpus_temporary" "$index_directory/corpus.tsv" + indexing_temporary=$(mktemp "$index_directory/.indexing.XXXXXX") || return 1 + printf '%s\n' \ + 'index-count=1' \ + 'index-1=flat-f32-exact/cosine' \ + 'query-path-count=2' \ + 'query-path-1=readable-fixed-width-text' \ + 'query-path-2=rebuildable-float32-cache' \ + 'collection=reading' \ + "model=$model_name" \ + "dimensions=$model_dimensions" \ + "document-count=$document_count" > "$indexing_temporary" + mv -T "$indexing_temporary" "$index_directory/indexing.tsv" + + sed -n '1,20p' "$build_report" + sed -n '1,20p' "$index_directory/indexing.tsv" +) + +ib_tab_qa_ensure_index() { + index_directory=$(ib_tab_qa_index_directory) || return 1 + adapter_record=$(dirname "$index_directory")/model-adapter.txt + current_model_metadata=$(ib_tab_qa_model_metadata) || return 1 + if test -s "$index_directory/format.txt" && + test -s "$index_directory/corpus.tsv" && + test -s "$index_directory/indexing.tsv" && + test -f "$adapter_record" && test ! -L "$adapter_record" && + test "$(cat "$adapter_record")" = "$current_model_metadata"; then + return 0 + fi + ib_tab_qa_build_index >/dev/null +} + +ib_tab_qa_null_render() { + # This is a mock standing in for any further post-processing we want to do + # before returning the text to the user. Its current behavior is identity. + cat +} + +ib_tab_qa_inspect() { + reading_root=$(ib_tab_qa_reading_root) || return 1 + qa_root=$(ib_tab_qa_root) || return 1 + views_root=$(ib_tab_qa_views_root) || return 1 + index_directory=$(ib_tab_qa_index_directory) || return 1 + pinned_manifest_count=$(find "$IB_TAB_QA_REPOSITORY_ROOT/models/embedding" -type f -name '*.model' | wc -l | tr -d '[:space:]') + printf '%s\n' \ + 'frontend=console-text-stub' \ + "reading-root=$reading_root" \ + "qa-root=$qa_root" \ + "views-root=$views_root" \ + "index-directory=$index_directory" \ + "pinned-embedding-manifest-count=$pinned_manifest_count" + ib_tab_qa_model_metadata + ib_tab_qa_reducer_metadata + if test -s "$index_directory/format.txt"; then + printf 'index-available=True\n' + sed -n '/^index-count=/p; /^query-path-count=/p' "$index_directory/indexing.tsv" + else + printf 'index-available=False\n' + fi +} + +ib_tab_qa_answer() ( + answer_work= + exchange_partial= + cleanup() { + test -z "$answer_work" || rm -rf "$answer_work" + test -z "$exchange_partial" || rm -rf "$exchange_partial" + } + trap cleanup EXIT HUP INT TERM + + question=$1 + test -n "$question" || ib_tab_qa_fail 'question is empty' || return 1 + + reading_root=$(ib_tab_qa_reading_root) || return 1 + qa_root=$(ib_tab_qa_root) || return 1 + ib_tab_qa_separate_roots "$reading_root" "$qa_root" || { + ib_tab_qa_fail 'question-and-answer storage must be separate from the reading corpus' + return 1 + } + ib_tab_qa_ensure_index || return 1 + index_directory=$(ib_tab_qa_index_directory) || return 1 + model_command=$(ib_tab_qa_model_command) || return 1 + reducer_command=$(ib_tab_qa_reducer_command) || return 1 + vector_program=$(ib_tab_qa_vector_program) || return 1 + test -x "$reducer_command" || ib_tab_qa_fail "reducer is unavailable: $reducer_command" || return 1 + + answer_work=$(mktemp -d) || return 1 + printf '%s\n' "$question" | "$model_command" embed > "$answer_work/question-vector.txt" + IB_VECTOR_QUERY_REPORT="$answer_work/vector-query.tsv" \ + "$vector_program" query "$index_directory" 1 \ + < "$answer_work/question-vector.txt" > "$answer_work/results.tsv" + + result_line=$(sed -n '1p' "$answer_work/results.tsv") + test -n "$result_line" || ib_tab_qa_fail 'vector query returned no saved page' || return 1 + source_id=$(printf '%s\n' "$result_line" | awk -F '\t' '{ print $1 }') + retrieval_score=$(printf '%s\n' "$result_line" | awk -F '\t' '{ print $2 }') + ib_tab_qa_safe_relative_document "$source_id" || ib_tab_qa_fail "vector index returned unsafe identity: $source_id" || return 1 + source_document="${reading_root%/}/$source_id" + test -s "$source_document" || ib_tab_qa_fail "retrieved source is unavailable: $source_id" || return 1 + + corpus_record=$(awk -F '\t' -v wanted="$source_id" ' + $1 == wanted { record = $0; matches++ } + END { if (matches == 1) print record; else exit 1 } + ' "$index_directory/corpus.tsv") || { + ib_tab_qa_fail "retrieved source has no unique index provenance: $source_id" + return 1 + } + indexed_source_sha256=$(printf '%s\n' "$corpus_record" | awk -F '\t' '{ print $2 }') + indexed_source_bytes=$(printf '%s\n' "$corpus_record" | awk -F '\t' '{ print $3 }') + cp "$source_document" "$answer_work/source-document.txt" + answer_source_sha256=$(ib_tab_qa_sha256 "$answer_work/source-document.txt") || return 1 + answer_source_bytes=$(wc -c < "$answer_work/source-document.txt" | tr -d '[:space:]') + if test "$answer_source_sha256" != "$indexed_source_sha256" || + test "$answer_source_bytes" != "$indexed_source_bytes"; then + ib_tab_qa_fail "reading source changed since index build: $source_id (run --index)" + return 1 + fi + + reading_source_record=${source_document%/*}/source.tsv + reading_source_present=False + reading_source_sha256= + if test -f "$reading_source_record" && test ! -L "$reading_source_record"; then + cp "$reading_source_record" "$answer_work/reading-source.tsv" + reading_source_sha256=$(ib_tab_qa_sha256 "$answer_work/reading-source.tsv") || return 1 + reading_source_present=True + fi + + "$model_command" answer "$question" "$answer_work/source-document.txt" > "$answer_work/model-candidate.txt" + IB_TAB_QA_REDUCER_REPORT="$answer_work/reducer.tsv" \ + "$reducer_command" reduce "$retrieval_score" "$answer_work/source-document.txt" \ + < "$answer_work/model-candidate.txt" > "$answer_work/reduced-answer.txt" + ib_tab_qa_null_render < "$answer_work/reduced-answer.txt" > "$answer_work/answer.txt" + test -s "$answer_work/answer.txt" || ib_tab_qa_fail 'post-processing returned an empty response' || return 1 + + mkdir -p "$qa_root" + exchange_partial=$(mktemp -d "$qa_root/.partial.XXXXXX") || return 1 + printf '%s\n' "$question" > "$exchange_partial/question.txt" + cp "$answer_work/model-candidate.txt" "$exchange_partial/model-candidate.txt" + cp "$answer_work/reduced-answer.txt" "$exchange_partial/reduced-answer.txt" + cp "$answer_work/answer.txt" "$exchange_partial/answer.txt" + cp "$answer_work/vector-query.tsv" "$exchange_partial/vector-query.tsv" + cp "$answer_work/reducer.tsv" "$exchange_partial/reducer.tsv" + cp "$index_directory/indexing.tsv" "$exchange_partial/indexing.tsv" + cp "$index_directory/format.txt" "$exchange_partial/vector-format.txt" + ib_tab_qa_model_metadata > "$exchange_partial/model.tsv" + ib_tab_qa_reducer_metadata > "$exchange_partial/reducer-model.tsv" + if test "$reading_source_present" = True; then + cp "$answer_work/reading-source.tsv" "$exchange_partial/reading-source.tsv" + fi + { + printf 'document-id\t%s\n' "$source_id" + printf 'retrieval-score\t%s\n' "$retrieval_score" + printf 'document-sha256\t%s\n' "$answer_source_sha256" + printf 'document-bytes\t%s\n' "$answer_source_bytes" + printf 'reading-source-present\t%s\n' "$reading_source_present" + printf 'reading-source-sha256\t%s\n' "$reading_source_sha256" + } > "$exchange_partial/source.tsv" + printf '%s\n' \ + '1=vector-retrieval' \ + '2=model-answer' \ + '3=mock-ensemble-reducer' \ + '4=null-identity-renderer' > "$exchange_partial/pipeline.tsv" + + exchange_base=$(date -u '+%Y%m%dT%H%M%SZ')-$$ + exchange_directory="$qa_root/$exchange_base" + exchange_suffix=0 + while test -e "$exchange_directory"; do + exchange_suffix=$((exchange_suffix + 1)) + exchange_directory="$qa_root/$exchange_base-$exchange_suffix" + done + mv "$exchange_partial" "$exchange_directory" + exchange_partial= + cat "$answer_work/answer.txt" +) diff --git a/models/embedding/mxbai-embed-xsmall-v1-int8.model b/models/embedding/mxbai-embed-xsmall-v1-int8.model new file mode 100644 index 0000000..f6c1aa1 --- /dev/null +++ b/models/embedding/mxbai-embed-xsmall-v1-int8.model @@ -0,0 +1,25 @@ +ib-embedding-model 1 +slug mxbai-embed-xsmall-v1-int8 +repository mixedbread-ai/mxbai-embed-xsmall-v1 +revision b0561d9a97e6b298da39f0ef3e7d3cf153b1b29a +license Apache-2.0 +runtime sentence-transformer-onnx +weight_precision int8 +dimensions 384 +max_tokens 128 +pooling attention-mask-mean +normalization l2 +token_policy include-special +padding dynamic-right +query_prefix none +document_prefix none +tokenizer_file tokenizer.json +onnx_file onnx/model_int8.onnx +onnx_output sentence_embedding +artifact config.json 675 55f755d351fd04b0fef37760e07e195eb47e15f7aed6fc42d9be3dde3d38bca4 +artifact tokenizer.json 711661 da0e79933b9ed51798a3ae27893d3c5fa4a201126cef75586296df9b4d2c62a0 +artifact tokenizer_config.json 1433 bd2e06a5b20fd1b13ca988bedc8763d332d242381b4fbc98f8fead4524158f79 +artifact special_tokens_map.json 695 5d5b662e421ea9fac075174bb0688ee0d9431699900b90662acd44b2a350503a +artifact modules.json 229 8f4b264b80206c830bebbdcae377e137925650a433b689343a63bdc9b3145460 +artifact 1_Pooling/config.json 296 a19c83805e1ce4174f3fbfec4ac8d3b8dbae0c958f8fd51b80937eb33e0c5335 +artifact onnx/model_int8.onnx 24448010 952f996d8cf46c311ee8654a750fa942b71c8b94aabe69d043dbb2bcaff5528e diff --git a/models/embedding/potion-base-2m.model b/models/embedding/potion-base-2m.model new file mode 100644 index 0000000..07d4215 --- /dev/null +++ b/models/embedding/potion-base-2m.model @@ -0,0 +1,21 @@ +ib-embedding-model 1 +slug potion-base-2m +repository minishlab/potion-base-2M +revision 389b9f64be5aa4ae7a6bc6fe95ef20ce485ae5da +license MIT +runtime model2vec-onnx +weight_precision f32 +dimensions 64 +max_tokens 128 +pooling static-token-mean +normalization l2 +token_policy exclude-special-and-unknown +padding none +query_prefix none +document_prefix none +tokenizer_file tokenizer.json +onnx_file onnx/model.onnx +onnx_output embeddings +artifact config.json 200 b2a89173391ca774c2d7323090a993a9a1553faa5b40eb37bb7cec6685fbea47 +artifact tokenizer.json 683666 e67e803f624fb4d67dea1c730d06e1067e1b14d830e2c2202569e3ef0f70bb50 +artifact onnx/model.onnx 7563349 92d4a7576de7d39055924b4d9d3979c8c3b2de272010f76f99e2ac14c7b2e5a8 diff --git a/native/vector-index/Makefile b/native/vector-index/Makefile new file mode 100644 index 0000000..7355f7e --- /dev/null +++ b/native/vector-index/Makefile @@ -0,0 +1,17 @@ +CC ?= cc +CFLAGS ?= -O2 +CPPFLAGS ?= +WARNINGS = -Wall -Wextra -Werror -Wpedantic + +.PHONY: all clean test + +all: ib-vector-index + +ib-vector-index: ib_vector_index.c + $(CC) $(CPPFLAGS) $(CFLAGS) $(WARNINGS) -std=c99 $< -lm -o $@ + +test: ib-vector-index + sh ../../tests/vector-index-smoke.sh ./ib-vector-index + +clean: + rm -f ib-vector-index diff --git a/native/vector-index/ib_vector_index.c b/native/vector-index/ib_vector_index.c new file mode 100644 index 0000000..413819e --- /dev/null +++ b/native/vector-index/ib_vector_index.c @@ -0,0 +1,1261 @@ +#define _POSIX_C_SOURCE 200809L + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define FORMAT_HEADER "ib-vector-index 2" +#define BACKEND_NAME "flat-f32-exact" +#define PATH_BUFFER 4096 +#define TEXT_SCALAR_BYTES 15 +#define TEXT_SLOT_BYTES 16 + +enum metric_kind { + METRIC_DOT, + METRIC_COSINE +}; + +struct manifest { + enum metric_kind metric; + size_t dimensions; + size_t count; + char ids_file[256]; + char vectors_text_file[256]; + char vectors_cache_file[256]; +}; + +struct match { + float score; + size_t row; + char *id; +}; + +static int fail(const char *message) { + fprintf(stderr, "ib-vector-index: %s\n", message); + return 1; +} + +static int fail_path(const char *action, const char *path) { + fprintf(stderr, "ib-vector-index: %s %s: %s\n", action, path, + strerror(errno)); + return 1; +} + +static int path_join(char *out, size_t capacity, const char *directory, + const char *name) { + int written = snprintf(out, capacity, "%s/%s", directory, name); + if (written < 0 || (size_t)written >= capacity) { + fprintf(stderr, "ib-vector-index: path is too long: %s/%s\n", directory, + name); + return 0; + } + return 1; +} + +static int plain_filename(const char *name) { + return name[0] != '\0' && strcmp(name, ".") != 0 && + strcmp(name, "..") != 0 && strchr(name, '/') == NULL; +} + +static int make_directories(const char *path) { + char copy[PATH_BUFFER]; + size_t length = strlen(path); + size_t start = 1; + + if (length == 0 || length >= sizeof(copy)) { + return fail("index directory is empty or too long"); + } + + memcpy(copy, path, length + 1); + if (copy[length - 1] == '/' && length > 1) { + copy[length - 1] = '\0'; + } + if (copy[0] != '/') { + start = 0; + } + + for (size_t i = start; copy[i] != '\0'; ++i) { + if (copy[i] != '/') { + continue; + } + copy[i] = '\0'; + if (copy[0] != '\0' && mkdir(copy, 0700) != 0 && errno != EEXIST) { + return fail_path("cannot create", copy); + } + copy[i] = '/'; + } + + if (mkdir(copy, 0700) != 0 && errno != EEXIST) { + return fail_path("cannot create", copy); + } + return 0; +} + +static int flush_file(FILE *file, const char *path) { + if (fflush(file) != 0) { + return fail_path("cannot flush", path); + } + if (fsync(fileno(file)) != 0) { + return fail_path("cannot synchronize", path); + } + return 0; +} + +static FILE *create_exclusive(const char *path) { + int descriptor = open(path, O_WRONLY | O_CREAT | O_EXCL, 0600); + if (descriptor < 0) { + return NULL; + } + FILE *file = fdopen(descriptor, "wb"); + if (file == NULL) { + int saved_errno = errno; + close(descriptor); + unlink(path); + errno = saved_errno; + } + return file; +} + +static int parse_size(const char *text, size_t *value) { + char *end = NULL; + unsigned long long parsed; + + errno = 0; + parsed = strtoull(text, &end, 10); + if (errno != 0 || end == text || *end != '\0' || parsed == 0 || + parsed > SIZE_MAX) { + return 0; + } + *value = (size_t)parsed; + return 1; +} + +static const char *metric_text(enum metric_kind metric) { + return metric == METRIC_COSINE ? "cosine" : "dot"; +} + +static int parse_metric(const char *text, enum metric_kind *metric) { + if (strcmp(text, "dot") == 0) { + *metric = METRIC_DOT; + return 1; + } + if (strcmp(text, "cosine") == 0) { + *metric = METRIC_COSINE; + return 1; + } + return 0; +} + +static int host_is_little_endian(void) { + const uint16_t one = 1; + return *((const unsigned char *)&one) == 1; +} + +static int write_float32_le(FILE *file, float value) { + unsigned char bytes[4]; + uint32_t bits; + memcpy(&bits, &value, sizeof(bits)); + bytes[0] = (unsigned char)(bits & 0xffu); + bytes[1] = (unsigned char)((bits >> 8u) & 0xffu); + bytes[2] = (unsigned char)((bits >> 16u) & 0xffu); + bytes[3] = (unsigned char)((bits >> 24u) & 0xffu); + return fwrite(bytes, sizeof(bytes), 1, file) == 1; +} + +static int write_float32_text(FILE *file, float value, int last_in_row) { + char slot[TEXT_SLOT_BYTES + 1]; + int written = snprintf(slot, sizeof(slot), "%+.8e%c", (double)value, + last_in_row ? '\n' : ' '); + return written == TEXT_SLOT_BYTES && + fwrite(slot, TEXT_SLOT_BYTES, 1, file) == 1; +} + +static int read_float32_text(const unsigned char *slot, float *value) { + char number[TEXT_SCALAR_BYTES + 1]; + char *end = NULL; + memcpy(number, slot, TEXT_SCALAR_BYTES); + number[TEXT_SCALAR_BYTES] = '\0'; + errno = 0; + *value = strtof(number, &end); + return errno == 0 && end == number + TEXT_SCALAR_BYTES && isfinite(*value); +} + +static float read_float32_le(const unsigned char *bytes) { + uint32_t bits = (uint32_t)bytes[0] | ((uint32_t)bytes[1] << 8u) | + ((uint32_t)bytes[2] << 16u) | + ((uint32_t)bytes[3] << 24u); + float value; + memcpy(&value, &bits, sizeof(value)); + return value; +} + +static int normalize(float *values, size_t dimensions) { + float squared_norm = 0.0f; + for (size_t i = 0; i < dimensions; ++i) { + squared_norm += values[i] * values[i]; + } + if (!(squared_norm > 0.0f) || !isfinite(squared_norm)) { + return 0; + } + float scale = 1.0f / sqrtf(squared_norm); + for (size_t i = 0; i < dimensions; ++i) { + values[i] *= scale; + } + return 1; +} + +static int parse_vector(char *text, size_t dimensions, float *values) { + char *cursor = text; + for (size_t i = 0; i < dimensions; ++i) { + char *end = NULL; + while (*cursor == ' ' || *cursor == '\t') { + ++cursor; + } + errno = 0; + values[i] = strtof(cursor, &end); + if (errno != 0 || end == cursor || !isfinite(values[i])) { + return 0; + } + cursor = end; + } + while (*cursor == ' ' || *cursor == '\t') { + ++cursor; + } + return *cursor == '\0'; +} + +static void trim_line_end(char *line) { + size_t length = strlen(line); + while (length > 0 && (line[length - 1] == '\n' || line[length - 1] == '\r')) { + line[--length] = '\0'; + } +} + +static int write_manifest_file(const char *path, const struct manifest *manifest) { + FILE *file = fopen(path, "wb"); + if (file == NULL) { + return fail_path("cannot open", path); + } + int bad = fprintf(file, + FORMAT_HEADER "\n" + "backend " BACKEND_NAME "\n" + "scalar f32\n" + "byte_order little\n" + "metric %s\n" + "dimensions %zu\n" + "count %zu\n" + "ids %s\n" + "vectors_text %s\n" + "text_scalar scientific-e8\n" + "text_slot_bytes %d\n" + "vectors_cache %s\n", + metric_text(manifest->metric), manifest->dimensions, + manifest->count, manifest->ids_file, + manifest->vectors_text_file, TEXT_SLOT_BYTES, + manifest->vectors_cache_file) < 0; + if (!bad && flush_file(file, path) != 0) { + bad = 1; + } + if (fclose(file) != 0) { + bad = 1; + } + return bad ? fail("cannot write index manifest") : 0; +} + +static int manifest_field(char *line, const char *name, char **value) { + size_t length = strlen(name); + if (strncmp(line, name, length) != 0 || line[length] != ' ') { + return 0; + } + *value = line + length + 1; + return **value != '\0'; +} + +static int read_manifest(const char *directory, struct manifest *manifest) { + char path[PATH_BUFFER]; + char *line = NULL; + size_t capacity = 0; + ssize_t length; + int line_number = 0; + int seen_backend = 0, seen_scalar = 0, seen_order = 0; + int seen_metric = 0, seen_dimensions = 0, seen_count = 0; + int seen_ids = 0, seen_vectors_text = 0, seen_text_scalar = 0; + int seen_text_slot_bytes = 0, seen_vectors_cache = 0; + + memset(manifest, 0, sizeof(*manifest)); + if (!path_join(path, sizeof(path), directory, "format.txt")) { + return 1; + } + FILE *file = fopen(path, "rb"); + if (file == NULL) { + return fail_path("cannot open", path); + } + + while ((length = getline(&line, &capacity, file)) >= 0) { + char *value = NULL; + (void)length; + ++line_number; + trim_line_end(line); + if (line_number == 1) { + if (strcmp(line, FORMAT_HEADER) != 0) { + goto invalid; + } + } else if (manifest_field(line, "backend", &value)) { + if (seen_backend) { + goto invalid; + } + seen_backend = strcmp(value, BACKEND_NAME) == 0; + } else if (manifest_field(line, "scalar", &value)) { + if (seen_scalar) { + goto invalid; + } + seen_scalar = strcmp(value, "f32") == 0; + } else if (manifest_field(line, "byte_order", &value)) { + if (seen_order) { + goto invalid; + } + seen_order = strcmp(value, "little") == 0; + } else if (manifest_field(line, "metric", &value)) { + if (seen_metric) { + goto invalid; + } + seen_metric = parse_metric(value, &manifest->metric); + } else if (manifest_field(line, "dimensions", &value)) { + if (seen_dimensions) { + goto invalid; + } + seen_dimensions = parse_size(value, &manifest->dimensions); + } else if (manifest_field(line, "count", &value)) { + if (seen_count) { + goto invalid; + } + char *end = NULL; + unsigned long long parsed; + errno = 0; + parsed = strtoull(value, &end, 10); + seen_count = errno == 0 && end != value && *end == '\0' && + parsed <= SIZE_MAX; + manifest->count = (size_t)parsed; + } else if (manifest_field(line, "ids", &value)) { + if (seen_ids) { + goto invalid; + } + seen_ids = plain_filename(value) && + snprintf(manifest->ids_file, sizeof(manifest->ids_file), "%s", + value) < (int)sizeof(manifest->ids_file); + } else if (manifest_field(line, "vectors_text", &value)) { + if (seen_vectors_text) { + goto invalid; + } + seen_vectors_text = + plain_filename(value) && + snprintf(manifest->vectors_text_file, + sizeof(manifest->vectors_text_file), "%s", value) < + (int)sizeof(manifest->vectors_text_file); + } else if (manifest_field(line, "text_scalar", &value)) { + if (seen_text_scalar) { + goto invalid; + } + seen_text_scalar = strcmp(value, "scientific-e8") == 0; + } else if (manifest_field(line, "text_slot_bytes", &value)) { + if (seen_text_slot_bytes) { + goto invalid; + } + size_t parsed = 0; + seen_text_slot_bytes = parse_size(value, &parsed) && + parsed == TEXT_SLOT_BYTES; + } else if (manifest_field(line, "vectors_cache", &value)) { + if (seen_vectors_cache) { + goto invalid; + } + seen_vectors_cache = + plain_filename(value) && + snprintf(manifest->vectors_cache_file, + sizeof(manifest->vectors_cache_file), "%s", value) < + (int)sizeof(manifest->vectors_cache_file); + } else { + goto invalid; + } + } + + free(line); + if (ferror(file)) { + fclose(file); + return fail_path("cannot read", path); + } + fclose(file); + if (line_number != 12 || !seen_backend || !seen_scalar || !seen_order || + !seen_metric || !seen_dimensions || !seen_count || !seen_ids || + !seen_vectors_text || !seen_text_scalar || !seen_text_slot_bytes || + !seen_vectors_cache) { + return fail("index manifest is incomplete or unsupported"); + } + if (strcmp(manifest->ids_file, manifest->vectors_text_file) == 0 || + strcmp(manifest->ids_file, manifest->vectors_cache_file) == 0 || + strcmp(manifest->vectors_text_file, manifest->vectors_cache_file) == 0) { + return fail("index manifest data filenames must be distinct"); + } + return 0; + +invalid: + free(line); + fclose(file); + return fail("index manifest has an unsupported format"); +} + +static int checked_value_count(const struct manifest *manifest, size_t *values) { + if (manifest->count != 0 && manifest->dimensions > SIZE_MAX / manifest->count) { + return 0; + } + *values = manifest->count * manifest->dimensions; + return 1; +} + +static int checked_vector_bytes(const struct manifest *manifest, size_t *bytes) { + size_t values; + if (!checked_value_count(manifest, &values) || + values > SIZE_MAX / sizeof(float)) { + return 0; + } + *bytes = values * sizeof(float); + return 1; +} + +static int checked_text_bytes(const struct manifest *manifest, size_t *bytes) { + size_t values; + if (!checked_value_count(manifest, &values) || + values > SIZE_MAX / TEXT_SLOT_BYTES) { + return 0; + } + *bytes = values * TEXT_SLOT_BYTES; + return 1; +} + +static int validate_text_vectors(const char *path, + const struct manifest *manifest, + size_t expected_bytes) { + int descriptor = -1; + const unsigned char *mapped = MAP_FAILED; + int status = 1; + + if (expected_bytes == 0) { + return 0; + } + descriptor = open(path, O_RDONLY); + if (descriptor < 0) { + return fail_path("cannot open", path); + } + mapped = mmap(NULL, expected_bytes, PROT_READ, MAP_PRIVATE, descriptor, 0); + if (mapped == MAP_FAILED) { + fail_path("cannot map", path); + goto cleanup; + } + size_t values = expected_bytes / TEXT_SLOT_BYTES; + for (size_t index = 0; index < values; ++index) { + const unsigned char *slot = mapped + index * TEXT_SLOT_BYTES; + float value; + unsigned char expected_delimiter = + (index + 1) % manifest->dimensions == 0 ? '\n' : ' '; + if (slot[TEXT_SCALAR_BYTES] != expected_delimiter || + !read_float32_text(slot, &value)) { + fail("plain-text vector file has an invalid fixed-width slot"); + goto cleanup; + } + } + status = 0; + +cleanup: + if (mapped != MAP_FAILED) { + munmap((void *)mapped, expected_bytes); + } + close(descriptor); + return status; +} + +static int validate_cache_vectors(const char *text_path, const char *cache_path, + const struct manifest *manifest, + size_t text_bytes, size_t cache_bytes) { + int text_descriptor = -1, cache_descriptor = -1; + const unsigned char *text = MAP_FAILED, *cache = MAP_FAILED; + int status = 1; + + if (text_bytes == 0) { + return 0; + } + text_descriptor = open(text_path, O_RDONLY); + if (text_descriptor < 0) { + return fail_path("cannot open", text_path); + } + cache_descriptor = open(cache_path, O_RDONLY); + if (cache_descriptor < 0) { + fail_path("cannot open", cache_path); + goto cleanup; + } + text = mmap(NULL, text_bytes, PROT_READ, MAP_PRIVATE, text_descriptor, 0); + if (text == MAP_FAILED) { + fail_path("cannot map", text_path); + goto cleanup; + } + cache = mmap(NULL, cache_bytes, PROT_READ, MAP_PRIVATE, cache_descriptor, 0); + if (cache == MAP_FAILED) { + fail_path("cannot map", cache_path); + goto cleanup; + } + + size_t values = text_bytes / TEXT_SLOT_BYTES; + for (size_t index = 0; index < values; ++index) { + const unsigned char *slot = text + index * TEXT_SLOT_BYTES; + float text_value, cache_value; + uint32_t text_bits, cache_bits; + unsigned char expected_delimiter = + (index + 1) % manifest->dimensions == 0 ? '\n' : ' '; + if (slot[TEXT_SCALAR_BYTES] != expected_delimiter || + !read_float32_text(slot, &text_value)) { + fail("plain-text vector file has an invalid fixed-width slot"); + goto cleanup; + } + cache_value = read_float32_le(cache + index * sizeof(float)); + memcpy(&text_bits, &text_value, sizeof(text_bits)); + memcpy(&cache_bits, &cache_value, sizeof(cache_bits)); + if (text_bits != cache_bits) { + fail("Float32 cache does not match the plain-text vectors"); + goto cleanup; + } + } + status = 0; + +cleanup: + if (cache != MAP_FAILED) { + munmap((void *)cache, cache_bytes); + } + if (text != MAP_FAILED) { + munmap((void *)text, text_bytes); + } + if (cache_descriptor >= 0) { + close(cache_descriptor); + } + if (text_descriptor >= 0) { + close(text_descriptor); + } + return status; +} + +static int check_index(const char *directory, struct manifest *manifest, + int announce, int require_cache, int validate_values) { + char ids_path[PATH_BUFFER], text_path[PATH_BUFFER], cache_path[PATH_BUFFER]; + struct stat text_stat, cache_stat; + size_t expected_text_bytes, expected_cache_bytes; + size_t ids = 0; + char *line = NULL; + size_t capacity = 0; + + if (read_manifest(directory, manifest) != 0) { + return 1; + } + if (!path_join(ids_path, sizeof(ids_path), directory, manifest->ids_file) || + !path_join(text_path, sizeof(text_path), directory, + manifest->vectors_text_file) || + !path_join(cache_path, sizeof(cache_path), directory, + manifest->vectors_cache_file)) { + return 1; + } + if (!checked_text_bytes(manifest, &expected_text_bytes) || + !checked_vector_bytes(manifest, &expected_cache_bytes)) { + return fail("index dimensions overflow file size"); + } + if (stat(text_path, &text_stat) != 0) { + return fail_path("cannot inspect", text_path); + } + if (text_stat.st_size < 0 || + (uintmax_t)text_stat.st_size != expected_text_bytes) { + return fail("plain-text vector size does not match the manifest"); + } + if (require_cache) { + if (stat(cache_path, &cache_stat) != 0) { + return fail_path("cannot inspect", cache_path); + } + if (cache_stat.st_size < 0 || + (uintmax_t)cache_stat.st_size != expected_cache_bytes) { + return fail("Float32 cache size does not match the manifest"); + } + } + if (validate_values) { + int invalid = require_cache + ? validate_cache_vectors(text_path, cache_path, manifest, + expected_text_bytes, + expected_cache_bytes) + : validate_text_vectors(text_path, manifest, + expected_text_bytes); + if (invalid) { + return 1; + } + } + + FILE *ids_file = fopen(ids_path, "rb"); + if (ids_file == NULL) { + return fail_path("cannot open", ids_path); + } + while (getline(&line, &capacity, ids_file) >= 0) { + trim_line_end(line); + if (line[0] == '\0' || strchr(line, '\t') != NULL) { + free(line); + fclose(ids_file); + return fail("ID file contains an empty or tabbed ID"); + } + ++ids; + } + free(line); + if (ferror(ids_file)) { + fclose(ids_file); + return fail_path("cannot read", ids_path); + } + fclose(ids_file); + if (ids != manifest->count) { + return fail("ID count does not match the manifest"); + } + + if (announce) { + printf("check=ok\nbackend=%s\nscalar=f32\ntext_slot_bytes=%d\nmetric=%s\ndimensions=%zu\ncount=%zu\n", + BACKEND_NAME, TEXT_SLOT_BYTES, metric_text(manifest->metric), + manifest->dimensions, manifest->count); + } + return 0; +} + +static int build_index(const char *directory, const char *dimension_text, + const char *metric_name) { + struct manifest manifest; + char generation[96]; + char ids_path[PATH_BUFFER], text_path[PATH_BUFFER], cache_path[PATH_BUFFER]; + char manifest_path[PATH_BUFFER], manifest_temp[PATH_BUFFER]; + char *line = NULL; + size_t line_capacity = 0; + ssize_t line_length; + float *values = NULL; + FILE *ids_file = NULL, *text_file = NULL, *cache_file = NULL; + int status = 1; + + memset(&manifest, 0, sizeof(manifest)); + if (!parse_size(dimension_text, &manifest.dimensions)) { + return fail("dimensions must be a positive integer"); + } + if (manifest.dimensions > SIZE_MAX / sizeof(*values)) { + return fail("dimensions are too large for this process"); + } + if (!parse_metric(metric_name, &manifest.metric)) { + return fail("metric must be cosine or dot"); + } + if (make_directories(directory) != 0) { + return 1; + } + + struct timespec now; + if (clock_gettime(CLOCK_REALTIME, &now) != 0) { + return fail_path("cannot read clock for", directory); + } + snprintf(generation, sizeof(generation), "%lld-%09ld-%ld", + (long long)now.tv_sec, now.tv_nsec, (long)getpid()); + snprintf(manifest.ids_file, sizeof(manifest.ids_file), "ids-%s.txt", + generation); + snprintf(manifest.vectors_text_file, sizeof(manifest.vectors_text_file), + "vectors-%s.txt", generation); + snprintf(manifest.vectors_cache_file, sizeof(manifest.vectors_cache_file), + "vectors-%s.f32", generation); + + if (!path_join(ids_path, sizeof(ids_path), directory, manifest.ids_file) || + !path_join(text_path, sizeof(text_path), directory, + manifest.vectors_text_file) || + !path_join(cache_path, sizeof(cache_path), directory, + manifest.vectors_cache_file) || + !path_join(manifest_path, sizeof(manifest_path), directory, "format.txt") || + snprintf(manifest_temp, sizeof(manifest_temp), "%s.tmp.%ld", manifest_path, + (long)getpid()) >= (int)sizeof(manifest_temp)) { + return 1; + } + + ids_file = create_exclusive(ids_path); + if (ids_file == NULL) { + return fail_path("cannot create", ids_path); + } + text_file = create_exclusive(text_path); + if (text_file == NULL) { + fail_path("cannot create", text_path); + goto cleanup; + } + cache_file = create_exclusive(cache_path); + if (cache_file == NULL) { + fail_path("cannot create", cache_path); + goto cleanup; + } + values = malloc(manifest.dimensions * sizeof(*values)); + if (values == NULL) { + fail("out of memory while reading vectors"); + goto cleanup; + } + + while ((line_length = getline(&line, &line_capacity, stdin)) >= 0) { + char *tab; + (void)line_length; + trim_line_end(line); + if (line[0] == '\0') { + continue; + } + tab = strchr(line, '\t'); + if (tab == NULL || tab == line) { + fail("each input row must be ID, tab, then vector values"); + goto cleanup; + } + *tab = '\0'; + if (strchr(tab + 1, '\n') != NULL || + !parse_vector(tab + 1, manifest.dimensions, values)) { + fail("an input row has the wrong vector dimension or a non-finite value"); + goto cleanup; + } + if (manifest.metric == METRIC_COSINE && + !normalize(values, manifest.dimensions)) { + fail("cosine vectors must have a finite, nonzero norm"); + goto cleanup; + } + if (fprintf(ids_file, "%s\n", line) < 0) { + fail_path("cannot write", ids_path); + goto cleanup; + } + for (size_t i = 0; i < manifest.dimensions; ++i) { + if (!write_float32_text(text_file, values[i], + i + 1 == manifest.dimensions)) { + fail_path("cannot write", text_path); + goto cleanup; + } + if (!write_float32_le(cache_file, values[i])) { + fail_path("cannot write", cache_path); + goto cleanup; + } + } + if (manifest.count == SIZE_MAX) { + fail("too many vector rows"); + goto cleanup; + } + ++manifest.count; + } + if (ferror(stdin)) { + fail("cannot read vector rows from standard input"); + goto cleanup; + } + if (flush_file(ids_file, ids_path) != 0 || + flush_file(text_file, text_path) != 0 || + flush_file(cache_file, cache_path) != 0) { + goto cleanup; + } + int ids_close_failed = fclose(ids_file) != 0; + ids_file = NULL; + int text_close_failed = fclose(text_file) != 0; + text_file = NULL; + int cache_close_failed = fclose(cache_file) != 0; + cache_file = NULL; + if (ids_close_failed || text_close_failed || cache_close_failed) { + fail("cannot close new index files"); + goto cleanup; + } + + if (write_manifest_file(manifest_temp, &manifest) != 0) { + goto cleanup; + } + if (rename(manifest_temp, manifest_path) != 0) { + fail_path("cannot install", manifest_path); + goto cleanup; + } + + printf("build=ok\nbackend=%s\nscalar=f32\nmetric=%s\ndimensions=%zu\ncount=%zu\n", + BACKEND_NAME, metric_text(manifest.metric), manifest.dimensions, + manifest.count); + status = 0; + +cleanup: + free(values); + free(line); + if (ids_file != NULL) { + fclose(ids_file); + } + if (text_file != NULL) { + fclose(text_file); + } + if (cache_file != NULL) { + fclose(cache_file); + } + if (status != 0) { + unlink(ids_path); + unlink(text_path); + unlink(cache_path); + unlink(manifest_temp); + } + return status; +} + +static float dot_product(const unsigned char *stored, const float *query, + size_t dimensions) { + float score = 0.0f; + if (host_is_little_endian()) { + for (size_t i = 0; i < dimensions; ++i) { + float value; + memcpy(&value, stored + i * sizeof(float), sizeof(value)); + score += value * query[i]; + } + } else { + for (size_t i = 0; i < dimensions; ++i) { + score += read_float32_le(stored + i * sizeof(float)) * query[i]; + } + } + return score; +} + +static int dot_product_text(const unsigned char *stored, const float *query, + size_t dimensions, float *score) { + *score = 0.0f; + for (size_t i = 0; i < dimensions; ++i) { + float value; + const unsigned char *slot = stored + i * TEXT_SLOT_BYTES; + unsigned char expected_delimiter = i + 1 == dimensions ? '\n' : ' '; + if (slot[TEXT_SCALAR_BYTES] != expected_delimiter || + !read_float32_text(slot, &value)) { + return 0; + } + *score += value * query[i]; + } + return 1; +} + +static int write_query_report(const char *path, const struct manifest *manifest, + int use_text, size_t dot_products) { + if (path == NULL || path[0] == '\0') { + return 0; + } + FILE *file = fopen(path, "wb"); + if (file == NULL) { + return fail_path("cannot open query report", path); + } + int bad = fprintf(file, + "operation=exact-dot-product-scan\n" + "compute=cpu\n" + "dot-product-used-gpu=False\n" + "storage=%s\n" + "metric=%s\n" + "dimensions=%zu\n" + "dot-products=%zu\n", + use_text ? "readable-text" : "float32-cache", + metric_text(manifest->metric), manifest->dimensions, + dot_products) < 0; + if (!bad && flush_file(file, path) != 0) { + bad = 1; + } + if (fclose(file) != 0) { + bad = 1; + } + return bad ? fail("cannot write query report") : 0; +} + +static int match_before(float score, size_t row, const struct match *other) { + return score > other->score || (score == other->score && row < other->row); +} + +static void consider_match(struct match *matches, size_t *used, size_t limit, + float score, size_t row, const char *id) { + size_t position = 0; + while (position < *used && !match_before(score, row, &matches[position])) { + ++position; + } + if (position >= limit) { + return; + } + size_t new_used = *used < limit ? *used + 1 : *used; + if (*used == limit) { + free(matches[limit - 1].id); + } + for (size_t i = new_used - 1; i > position; --i) { + matches[i] = matches[i - 1]; + } + matches[position].score = score; + matches[position].row = row; + matches[position].id = strdup(id); + if (matches[position].id == NULL) { + fail("out of memory while retaining matches"); + exit(1); + } + *used = new_used; +} + +static int query_index(const char *directory, const char *limit_text, + int use_text) { + struct manifest manifest; + char ids_path[PATH_BUFFER], data_path[PATH_BUFFER]; + char *line = NULL; + size_t line_capacity = 0, limit, used = 0, data_bytes = 0; + float *query = NULL; + struct match *matches = NULL; + FILE *ids_file = NULL; + int data_fd = -1; + unsigned char *mapped = MAP_FAILED; + int status = 1; + const char *query_report_path = getenv("IB_VECTOR_QUERY_REPORT"); + + if (!parse_size(limit_text, &limit)) { + return fail("result count must be a positive integer"); + } + if (check_index(directory, &manifest, 0, !use_text, 0) != 0) { + return 1; + } + if (limit > manifest.count) { + limit = manifest.count; + } + query = malloc(manifest.dimensions * sizeof(*query)); + if (query == NULL) { + return fail("out of memory while reading query"); + } + if (getline(&line, &line_capacity, stdin) < 0) { + fail("query vector is missing on standard input"); + goto cleanup; + } + trim_line_end(line); + if (!parse_vector(line, manifest.dimensions, query)) { + fail("query has the wrong vector dimension or a non-finite value"); + goto cleanup; + } + if (manifest.metric == METRIC_COSINE && + !normalize(query, manifest.dimensions)) { + fail("cosine query must have a finite, nonzero norm"); + goto cleanup; + } + while (getline(&line, &line_capacity, stdin) >= 0) { + trim_line_end(line); + if (line[0] != '\0') { + fail("query accepts exactly one vector"); + goto cleanup; + } + } + + if (limit == 0) { + status = write_query_report(query_report_path, &manifest, use_text, 0); + goto cleanup; + } + matches = calloc(limit, sizeof(*matches)); + if (matches == NULL) { + fail("out of memory while allocating matches"); + goto cleanup; + } + int valid_size = use_text ? checked_text_bytes(&manifest, &data_bytes) + : checked_vector_bytes(&manifest, &data_bytes); + const char *data_file = use_text ? manifest.vectors_text_file + : manifest.vectors_cache_file; + if (!valid_size || + !path_join(ids_path, sizeof(ids_path), directory, manifest.ids_file) || + !path_join(data_path, sizeof(data_path), directory, data_file)) { + goto cleanup; + } + ids_file = fopen(ids_path, "rb"); + if (ids_file == NULL) { + fail_path("cannot open", ids_path); + goto cleanup; + } + data_fd = open(data_path, O_RDONLY); + if (data_fd < 0) { + fail_path("cannot open", data_path); + goto cleanup; + } + mapped = mmap(NULL, data_bytes, PROT_READ, MAP_PRIVATE, data_fd, 0); + if (mapped == MAP_FAILED) { + fail_path("cannot map", data_path); + goto cleanup; + } + + for (size_t row = 0; row < manifest.count; ++row) { + if (getline(&line, &line_capacity, ids_file) < 0) { + fail("ID file ended during query"); + goto cleanup; + } + trim_line_end(line); + size_t scalar_bytes = use_text ? TEXT_SLOT_BYTES : sizeof(float); + const unsigned char *stored = + mapped + row * manifest.dimensions * scalar_bytes; + float score; + if (use_text) { + if (!dot_product_text(stored, query, manifest.dimensions, &score)) { + fail("plain-text vector file changed during query"); + goto cleanup; + } + } else { + score = dot_product(stored, query, manifest.dimensions); + } + if (!isfinite(score)) { + fail("similarity score overflowed or the cache is corrupt"); + goto cleanup; + } + if (manifest.metric == METRIC_COSINE) { + if (score > 1.0f) { + score = 1.0f; + } else if (score < -1.0f) { + score = -1.0f; + } + } + consider_match(matches, &used, limit, score, row, line); + } + for (size_t i = 0; i < used; ++i) { + printf("%s\t%.9g\n", matches[i].id, (double)matches[i].score); + } + status = write_query_report(query_report_path, &manifest, use_text, + manifest.count); + +cleanup: + if (mapped != MAP_FAILED) { + munmap(mapped, data_bytes); + } + if (data_fd >= 0) { + close(data_fd); + } + if (ids_file != NULL) { + fclose(ids_file); + } + if (matches != NULL) { + for (size_t i = 0; i < used; ++i) { + free(matches[i].id); + } + } + free(matches); + free(query); + free(line); + return status; +} + +static int compile_cache(const char *directory) { + struct manifest manifest; + char text_path[PATH_BUFFER], cache_path[PATH_BUFFER], temporary_path[PATH_BUFFER]; + size_t text_bytes = 0, cache_bytes = 0; + int text_fd = -1; + const unsigned char *mapped = MAP_FAILED; + FILE *cache_file = NULL; + int status = 1; + + if (check_index(directory, &manifest, 0, 0, 1) != 0 || + !checked_text_bytes(&manifest, &text_bytes) || + !checked_vector_bytes(&manifest, &cache_bytes) || + !path_join(text_path, sizeof(text_path), directory, + manifest.vectors_text_file) || + !path_join(cache_path, sizeof(cache_path), directory, + manifest.vectors_cache_file) || + snprintf(temporary_path, sizeof(temporary_path), "%s.tmp.%ld", cache_path, + (long)getpid()) >= (int)sizeof(temporary_path)) { + return 1; + } + cache_file = create_exclusive(temporary_path); + if (cache_file == NULL) { + return fail_path("cannot create", temporary_path); + } + if (text_bytes != 0) { + text_fd = open(text_path, O_RDONLY); + if (text_fd < 0) { + fail_path("cannot open", text_path); + goto cleanup; + } + mapped = mmap(NULL, text_bytes, PROT_READ, MAP_PRIVATE, text_fd, 0); + if (mapped == MAP_FAILED) { + fail_path("cannot map", text_path); + goto cleanup; + } + } + + size_t values = cache_bytes / sizeof(float); + for (size_t index = 0; index < values; ++index) { + float value; + if (!read_float32_text(mapped + index * TEXT_SLOT_BYTES, &value) || + !write_float32_le(cache_file, value)) { + fail("cannot compile Float32 cache from plain-text vectors"); + goto cleanup; + } + } + if (flush_file(cache_file, temporary_path) != 0) { + goto cleanup; + } + if (fclose(cache_file) != 0) { + cache_file = NULL; + fail_path("cannot close", temporary_path); + goto cleanup; + } + cache_file = NULL; + if (rename(temporary_path, cache_path) != 0) { + fail_path("cannot install", cache_path); + goto cleanup; + } + printf("compile-cache=ok\nbytes=%zu\n", cache_bytes); + status = 0; + +cleanup: + if (mapped != MAP_FAILED) { + munmap((void *)mapped, text_bytes); + } + if (text_fd >= 0) { + close(text_fd); + } + if (cache_file != NULL) { + fclose(cache_file); + } + if (status != 0) { + unlink(temporary_path); + } + return status; +} + +static int print_column(const char *directory, const char *coordinate_text) { + struct manifest manifest; + char ids_path[PATH_BUFFER], text_path[PATH_BUFFER]; + char *line = NULL; + size_t line_capacity = 0, coordinate, text_bytes = 0; + FILE *ids_file = NULL; + int text_fd = -1; + const unsigned char *mapped = MAP_FAILED; + int status = 1; + + if (!parse_size(coordinate_text, &coordinate)) { + return fail("coordinate must be a positive one-based integer"); + } + if (check_index(directory, &manifest, 0, 0, 0) != 0) { + return 1; + } + if (coordinate > manifest.dimensions) { + return fail("coordinate exceeds the vector dimensions"); + } + if (!checked_text_bytes(&manifest, &text_bytes) || + !path_join(ids_path, sizeof(ids_path), directory, manifest.ids_file) || + !path_join(text_path, sizeof(text_path), directory, + manifest.vectors_text_file)) { + return 1; + } + ids_file = fopen(ids_path, "rb"); + if (ids_file == NULL) { + return fail_path("cannot open", ids_path); + } + if (text_bytes != 0) { + text_fd = open(text_path, O_RDONLY); + if (text_fd < 0) { + fail_path("cannot open", text_path); + goto cleanup; + } + mapped = mmap(NULL, text_bytes, PROT_READ, MAP_PRIVATE, text_fd, 0); + if (mapped == MAP_FAILED) { + fail_path("cannot map", text_path); + goto cleanup; + } + } + + size_t coordinate_index = coordinate - 1; + for (size_t row = 0; row < manifest.count; ++row) { + if (getline(&line, &line_capacity, ids_file) < 0) { + fail("ID file ended during column scan"); + goto cleanup; + } + trim_line_end(line); + size_t slot_index = row * manifest.dimensions + coordinate_index; + const unsigned char *slot = mapped + slot_index * TEXT_SLOT_BYTES; + float value; + unsigned char expected_delimiter = + coordinate == manifest.dimensions ? '\n' : ' '; + if (slot[TEXT_SCALAR_BYTES] != expected_delimiter || + !read_float32_text(slot, &value)) { + fail("plain-text vector file has an invalid selected coordinate"); + goto cleanup; + } + printf("%s\t%.*s\n", line, TEXT_SCALAR_BYTES, (const char *)slot); + } + status = 0; + +cleanup: + if (mapped != MAP_FAILED) { + munmap((void *)mapped, text_bytes); + } + if (text_fd >= 0) { + close(text_fd); + } + if (ids_file != NULL) { + fclose(ids_file); + } + free(line); + return status; +} + +static int inspect_index(const char *directory) { + struct manifest manifest; + if (check_index(directory, &manifest, 0, 0, 0) != 0) { + return 1; + } + printf(FORMAT_HEADER "\n" + "backend " BACKEND_NAME "\n" + "scalar f32\n" + "byte_order little\n" + "metric %s\n" + "dimensions %zu\n" + "count %zu\n" + "ids %s\n" + "vectors_text %s\n" + "text_scalar scientific-e8\n" + "text_slot_bytes %d\n" + "vectors_cache %s\n", + metric_text(manifest.metric), manifest.dimensions, manifest.count, + manifest.ids_file, manifest.vectors_text_file, TEXT_SLOT_BYTES, + manifest.vectors_cache_file); + return 0; +} + +static void usage(FILE *out) { + fprintf(out, + "usage:\n" + " ib-vector-index build INDEX_DIRECTORY DIMENSIONS cosine|dot < rows.tsv\n" + " ib-vector-index query INDEX_DIRECTORY RESULT_COUNT < vector.txt\n" + " ib-vector-index query-text INDEX_DIRECTORY RESULT_COUNT < vector.txt\n" + " ib-vector-index column INDEX_DIRECTORY ONE_BASED_COORDINATE\n" + " ib-vector-index compile-cache INDEX_DIRECTORY\n" + " ib-vector-index check INDEX_DIRECTORY\n" + " ib-vector-index inspect INDEX_DIRECTORY\n\n" + "build rows are: IDnumber number ...\n" + "query results are: IDscore\n"); +} + +int main(int argc, char **argv) { + if (sizeof(float) != 4 || FLT_RADIX != 2 || FLT_MANT_DIG != 24 || + FLT_MAX_EXP != 128) { + return fail("this backend requires IEEE 754 binary32 float"); + } + if (argc == 6 && strcmp(argv[1], "build") == 0) { + return fail("build received too many arguments"); + } + if (argc == 5 && strcmp(argv[1], "build") == 0) { + return build_index(argv[2], argv[3], argv[4]); + } + if (argc == 4 && strcmp(argv[1], "query") == 0) { + return query_index(argv[2], argv[3], 0); + } + if (argc == 4 && strcmp(argv[1], "query-text") == 0) { + return query_index(argv[2], argv[3], 1); + } + if (argc == 4 && strcmp(argv[1], "column") == 0) { + return print_column(argv[2], argv[3]); + } + if (argc == 3 && strcmp(argv[1], "compile-cache") == 0) { + return compile_cache(argv[2]); + } + if (argc == 3 && strcmp(argv[1], "check") == 0) { + struct manifest manifest; + return check_index(argv[2], &manifest, 1, 1, 1); + } + if (argc == 3 && strcmp(argv[1], "inspect") == 0) { + return inspect_index(argv[2]); + } + usage(stderr); + return 2; +} diff --git a/src/FileStoreSmoke.idric b/src/FileStoreSmoke.idric new file mode 100644 index 0000000..e08a4ba --- /dev/null +++ b/src/FileStoreSmoke.idric @@ -0,0 +1,61 @@ +module FileStoreSmoke + +import IB.FileStore + +import Data.String +import System + +%default total + +manifest : String +manifest = "id T1\nstate sleeping\ncurrent_history 2\n" + +first_history : String +first_history = "1 2026-08-24T17:00:00Z request https://example.test/first" + +second_history : String +second_history = "2 2026-08-24T17:01:00Z request https://example.test/second" + +fail_with : String → IO () +fail_with message = die ("error=" ++ message) + +write_fixture : String → IO () +write_fixture root = do + Right () ← write_tab_manifest root "T1" manifest + | Left error ⇒ fail_with (show error) + Right () ← append_tab_history root "T1" first_history + | Left error ⇒ fail_with (show error) + Right () ← append_tab_history root "T1" second_history + | Left error ⇒ fail_with (show error) + path_check ← read_canonical_text root "../outside" + case path_check of + Left (InvalidRelativePath _) ⇒ putStrLn "unsafe-path=rejected" + Left error ⇒ fail_with (show error) + Right _ ⇒ fail_with "unsafe path was readable" + kind_check ← read_canonical_text root "cache/fake" + case kind_check of + Left (NonCanonicalPath _) ⇒ do + putStrLn "write=ok" + putStrLn "noncanonical-path=rejected" + Left error ⇒ fail_with (show error) + Right _ ⇒ fail_with "noncanonical path was readable" + +covering +read_fixture : String → IO () +read_fixture root = do + Right saved_manifest ← read_tab_manifest root "T1" + | Left error ⇒ fail_with (show error) + Right saved_history ← read_tab_history root "T1" + | Left error ⇒ fail_with (show error) + putStrLn ("manifest-survived=" ++ show (saved_manifest == manifest)) + putStrLn ("history-survived=" ++ show (saved_history == first_history ++ "\n" ++ second_history ++ "\n")) + putStrLn ("history-lines=" ++ show (length (lines saved_history))) + +covering +main : IO () +main = do + arguments ← getArgs + case arguments of + [_, "write", root] ⇒ write_fixture root + [_, "read", root] ⇒ read_fixture root + _ ⇒ die "usage: ib-file-store-smoke write|read STORE_ROOT" diff --git a/src/IB/FileStore.idric b/src/IB/FileStore.idric new file mode 100644 index 0000000..8b4b4da --- /dev/null +++ b/src/IB/FileStore.idric @@ -0,0 +1,149 @@ +module IB.FileStore + +import IB.Storage + +import System.Directory +import System.File + +%default total + +public export +data StoreError + = InvalidRoot + | InvalidRelativePath String + | NonCanonicalPath String + | FileFailure String FileError + +file_error_text : FileError → String +file_error_text (GenericFileError number) = "filesystem error " ++ show number +file_error_text FileReadError = "file read error" +file_error_text FileWriteError = "file write error" +file_error_text FileNotFound = "file not found" +file_error_text PermissionDenied = "permission denied" +file_error_text FileExists = "file exists" + +public export +Show StoreError where + show InvalidRoot = "store root must not be empty" + show (InvalidRelativePath path) = "unsafe store path: " ++ path + show (NonCanonicalPath path) = "not a canonical store path: " ++ path + show (FileFailure path error) = path ++ ": " ++ file_error_text error + +canonical_kind : storage_kind → Bool +canonical_kind canonical = True +canonical_kind _ = False + +store_path : String → String → String +store_path root relative = root ++ "/" ++ relative + +create_if_missing : String → IO (Either StoreError ()) +create_if_missing path = do + result ← createDir path + case result of + Right () ⇒ pure (Right ()) + Left FileExists ⇒ pure (Right ()) + Left error ⇒ pure (Left (FileFailure path error)) + +public export +prepare_store : String → IO (Either StoreError ()) +prepare_store "" = pure (Left InvalidRoot) +prepare_store root = do + Right () ← create_if_missing root + | Left error ⇒ pure (Left error) + Right () ← create_if_missing (store_path root "tabs") + | Left error ⇒ pure (Left error) + Right () ← create_if_missing (store_path root "snapshots") + | Left error ⇒ pure (Left error) + Right () ← create_if_missing (store_path root "indexes") + | Left error ⇒ pure (Left error) + create_if_missing (store_path root "cache") + +valid_tab_id : String → Bool +valid_tab_id tab_id = + tab_id /= "." && safe_relative_path tab_id && path_parts tab_id == [tab_id] + +public export +prepare_tab : String → String → IO (Either StoreError ()) +prepare_tab root tab_id = + if valid_tab_id tab_id + then do + Right () ← prepare_store root + | Left error ⇒ pure (Left error) + create_if_missing (store_path root ("tabs/" ++ tab_id)) + else pure (Left (InvalidRelativePath tab_id)) + +check_canonical_path : String → Either StoreError () +check_canonical_path path = + if not (safe_relative_path path) + then Left (InvalidRelativePath path) + else if canonical_kind (classify_path path) + then Right () + else Left (NonCanonicalPath path) + +check_store_path : String → String → Either StoreError () +check_store_path "" relative = Left InvalidRoot +check_store_path root relative = check_canonical_path relative + +public export +covering +read_canonical_text : String → String → IO (Either StoreError String) +read_canonical_text root relative = + case check_store_path root relative of + Left error ⇒ pure (Left error) + Right () ⇒ do + result ← readFile (store_path root relative) + case result of + Left error ⇒ pure (Left (FileFailure (store_path root relative) error)) + Right text ⇒ pure (Right text) + +public export +write_canonical_text : String → String → String → IO (Either StoreError ()) +write_canonical_text root relative text = + case check_store_path root relative of + Left error ⇒ pure (Left error) + Right () ⇒ do + result ← writeFile (store_path root relative) text + case result of + Left error ⇒ pure (Left (FileFailure (store_path root relative) error)) + Right () ⇒ pure (Right ()) + +public export +append_canonical_text : String → String → String → IO (Either StoreError ()) +append_canonical_text root relative text = + case check_store_path root relative of + Left error ⇒ pure (Left error) + Right () ⇒ do + result ← appendFile (store_path root relative) text + case result of + Left error ⇒ pure (Left (FileFailure (store_path root relative) error)) + Right () ⇒ pure (Right ()) + +public export +write_tab_manifest : String → String → String → IO (Either StoreError ()) +write_tab_manifest root tab_id text = do + Right () ← prepare_tab root tab_id + | Left error ⇒ pure (Left error) + write_canonical_text root ("tabs/" ++ tab_id ++ "/tab.txt") text + +public export +covering +read_tab_manifest : String → String → IO (Either StoreError String) +read_tab_manifest root tab_id = + if valid_tab_id tab_id + then read_canonical_text root ("tabs/" ++ tab_id ++ "/tab.txt") + else pure (Left (InvalidRelativePath tab_id)) + +public export +append_tab_history : String → String → String → IO (Either StoreError ()) +append_tab_history root tab_id line = do + Right () ← prepare_tab root tab_id + | Left error ⇒ pure (Left error) + append_canonical_text root ("tabs/" ++ tab_id ++ "/history.log") (line ++ "\n") + +public export +covering +read_tab_history : String → String → IO (Either StoreError String) +read_tab_history root tab_id = + if valid_tab_id tab_id + then read_canonical_text root ("tabs/" ++ tab_id ++ "/history.log") + else pure (Left (InvalidRelativePath tab_id)) diff --git a/src/IB/Storage.idric b/src/IB/Storage.idric index 07a6328..fad953a 100644 --- a/src/IB/Storage.idric +++ b/src/IB/Storage.idric @@ -64,6 +64,7 @@ classify_path path = then secret else case parts of "indexes" :: _ ⇒ derived + "views" :: _ ⇒ derived "snapshots" :: _ ⇒ snapshot "cache" :: _ ⇒ cache "caches" :: _ ⇒ cache @@ -79,11 +80,31 @@ transparent_index name = elem name [ "queries.tsv", "terms.tsv", "summary.json" ] +char_prefix : List Char → List Char → Bool +char_prefix [] _ = True +char_prefix _ [] = False +char_prefix (wanted :: more_wanted) (actual :: more_actual) = + wanted == actual && char_prefix more_wanted more_actual + +begins : String → String → Bool +begins opening value = char_prefix (unpack opening) (unpack value) + +finishes : String → String → Bool +finishes closing value = + char_prefix (reverse (unpack closing)) (reverse (unpack value)) + +vector_text_metadata : List String → Bool +vector_text_metadata ["views", "organizing-the-information", "vector-spaces", _, _, "format.txt"] = True +vector_text_metadata ["views", "organizing-the-information", "vector-spaces", _, "embedding-model.txt"] = True +vector_text_metadata ["views", "organizing-the-information", "vector-spaces", _, _, name] = + (begins "ids-" name || begins "vectors-" name) && finishes ".txt" name +vector_text_metadata _ = False + public export transparent_path : String → Bool transparent_path path = let parts = path_parts path in - parts == ["visits.jsonl"] || is_tab_record parts || + parts == ["visits.jsonl"] || is_tab_record parts || vector_text_metadata parts || case parts of ["indexes", name] ⇒ transparent_index name _ ⇒ False diff --git a/src/IB/VectorIndex.idric b/src/IB/VectorIndex.idric new file mode 100644 index 0000000..0ee813e --- /dev/null +++ b/src/IB/VectorIndex.idric @@ -0,0 +1,128 @@ +module IB.VectorIndex + +import IB.Storage + +%default total + +public export +choice vector_metric one_of + cosine + dot_product + +public export +vector_metric_text : vector_metric → String +vector_metric_text cosine = "cosine" +vector_metric_text dot_product = "dot" + +public export +record VectorBackend where + constructor Backend + backend_name : String + backend_program : String + scalar_name : String + contract_version : Nat + +public export +flat_f32_exact : VectorBackend +flat_f32_exact = Backend "flat-f32-exact" "ib-vector-index" "f32" 2 + +public export +record VectorIndexSpec where + constructor IndexSpec + backend : VectorBackend + collection : String + embedding_model : String + dimensions : Nat + metric : vector_metric + +safe_component : String → Bool +safe_component value = + value /= "." && value /= ".." && safe_relative_path value && + path_parts value == [value] + +public export +vector_index_relative_directory : VectorIndexSpec → Maybe String +vector_index_relative_directory spec = + if safe_component (collection spec) && safe_component (embedding_model spec) + then Just ("organizing-the-information/vector-spaces/" ++ + embedding_model spec ++ "/" ++ collection spec) + else Nothing + +public export +data BackendCommand = Run String (List String) + +public export +command_program : BackendCommand → String +command_program (Run program _) = program + +public export +command_arguments : BackendCommand → List String +command_arguments (Run _ arguments) = arguments + +index_directory : String → VectorIndexSpec → Maybe String +index_directory "" _ = Nothing +index_directory view_root spec = + case vector_index_relative_directory spec of + Nothing ⇒ Nothing + Just relative ⇒ Just (view_root ++ "/" ++ relative) + +public export +build_command : String → VectorIndexSpec → Maybe BackendCommand +build_command view_root spec = + case index_directory view_root spec of + Nothing ⇒ Nothing + Just directory ⇒ + if dimensions spec == 0 + then Nothing + else Just (Run + (backend_program (backend spec)) + [ "build" + , directory + , show (dimensions spec) + , vector_metric_text (metric spec) + ]) + +public export +query_command : String → VectorIndexSpec → Nat → Maybe BackendCommand +query_command view_root spec result_count = + case index_directory view_root spec of + Nothing ⇒ Nothing + Just directory ⇒ + if dimensions spec == 0 || result_count == 0 + then Nothing + else Just (Run + (backend_program (backend spec)) + ["query", directory, show result_count]) + +public export +query_text_command : String → VectorIndexSpec → Nat → Maybe BackendCommand +query_text_command view_root spec result_count = + case index_directory view_root spec of + Nothing ⇒ Nothing + Just directory ⇒ + if dimensions spec == 0 || result_count == 0 + then Nothing + else Just (Run + (backend_program (backend spec)) + ["query-text", directory, show result_count]) + +public export +compile_cache_command : String → VectorIndexSpec → Maybe BackendCommand +compile_cache_command view_root spec = + case index_directory view_root spec of + Nothing ⇒ Nothing + Just directory ⇒ Just (Run + (backend_program (backend spec)) + ["compile-cache", directory]) + +public export +column_command : String → VectorIndexSpec → Nat → Maybe BackendCommand +column_command view_root spec coordinate = + case index_directory view_root spec of + Nothing ⇒ Nothing + Just directory ⇒ + if dimensions spec == 0 || coordinate == 0 || coordinate > dimensions spec + then Nothing + else Just (Run + (backend_program (backend spec)) + ["column", directory, show coordinate]) diff --git a/src/Smoke.idric b/src/Smoke.idric index e2fd73c..e1d8b7e 100644 --- a/src/Smoke.idric +++ b/src/Smoke.idric @@ -7,6 +7,7 @@ import IB.Inspect import IB.DisplayRepair import IB.ScientificMedia import IB.Prefetch +import IB.VectorIndex %default total @@ -30,6 +31,10 @@ first_or : Nat → List Nat → Nat first_or fallback [] = fallback first_or fallback (value :: _) = value +maybe_string : String → Maybe String → String +maybe_string fallback Nothing = fallback +maybe_string _ (Just value) = value + main : IO () main = do let raw = ingest_raw_lines ["# ignored", " https://example.test/a ", "https://example.test/a", "https://other.test/b"] @@ -43,12 +48,14 @@ main = do ] let copy_repair = copy_button "code-1" "git status" code_block let prefetch = bounded_prefetch 2 sample_prefetch_targets + let vector_spec = IndexSpec flat_f32_exact "pages" "test-model" 384 cosine putStrLn ("ingest=" ++ show (length raw)) putStrLn ("duplicate-url-count=" ++ show (row_count "https://example.test/a" (url_rows indices))) putStrLn ("newest-order=" ++ show (first_or 99 (index_chronology indices))) putStrLn ("host-count=" ++ show (row_count "example.test" (host_rows indices))) putStrLn ("canonical=" ++ storage_kind_text (classify_path "visits.jsonl")) putStrLn ("cache-fake=" ++ storage_kind_text (classify_path "cache/visits.jsonl")) + putStrLn ("views=" ++ storage_kind_text (classify_path "views/organizing-the-information/categories/math/tab-T1")) putStrLn ("secret=" ++ storage_kind_text (classify_path "renderer/passwords")) putStrLn ("readable-files=" ++ show (readable_file_count files)) putStrLn ("canonical-files=" ++ show (files_of_kind canonical files)) @@ -61,3 +68,11 @@ main = do putStrLn ("temporary-needs-language-model=" ++ show (needs_language_model_image_name temporary_image_name)) putStrLn ("prefetch-bounded=" ++ show (length prefetch)) putStrLn ("prefetch-reading=" ++ show (reading_target_count prefetch)) + putStrLn ("vector-backend=" ++ backend_name (backend vector_spec)) + putStrLn ("vector-scalar=" ++ scalar_name (backend vector_spec)) + putStrLn ("vector-contract=" ++ show (contract_version (backend vector_spec))) + putStrLn ("vector-directory=" ++ maybe_string "invalid" (vector_index_relative_directory vector_spec)) + putStrLn ("vector-format-readable=" ++ show (readable_path "views/organizing-the-information/vector-spaces/test-model/pages/format.txt")) + putStrLn ("vector-model-readable=" ++ show (readable_path "views/organizing-the-information/vector-spaces/test-model/embedding-model.txt")) + putStrLn ("vector-text-readable=" ++ show (readable_path "views/organizing-the-information/vector-spaces/test-model/pages/vectors-1.txt")) + putStrLn ("vector-bytes-readable=" ++ show (readable_path "views/organizing-the-information/vector-spaces/test-model/pages/vectors-1.f32")) diff --git a/tests/fixtures/embedding-corpus.tsv b/tests/fixtures/embedding-corpus.tsv new file mode 100644 index 0000000..e393318 --- /dev/null +++ b/tests/fixtures/embedding-corpus.tsv @@ -0,0 +1,4 @@ +id text +symplectic-page Symplectic geometry studies smooth spaces equipped with a closed nondegenerate differential two-form. +woodworking-page Choose a torque wrench and screwdriver set for repairing workshop tools and furniture. +bread-page A baker explains sourdough fermentation, shaping a loaf, and baking bread in a hot oven. diff --git a/tests/fixtures/embedding-query.tsv b/tests/fixtures/embedding-query.tsv new file mode 100644 index 0000000..fb13c1d --- /dev/null +++ b/tests/fixtures/embedding-query.tsv @@ -0,0 +1,2 @@ +id text +query What mathematical subject studies closed nondegenerate two-forms on smooth manifolds? diff --git a/tests/fixtures/mock_documentation_synthesizer.grease b/tests/fixtures/mock_documentation_synthesizer.grease new file mode 100644 index 0000000..17acff7 --- /dev/null +++ b/tests/fixtures/mock_documentation_synthesizer.grease @@ -0,0 +1,37 @@ +#!/bin/sh +set -eu + +output_directory=${1:-} +test -n "$output_directory" || { + printf 'usage: %s OUTPUT_DIRECTORY SOURCE...\n' "$0" >&2 + exit 2 +} +shift +test "$#" -gt 0 || { + printf 'mock synthesizer needs at least one source\n' >&2 + exit 2 +} + +mkdir -p "$output_directory" +view_partial="$output_directory/.view.md.$$" +sources_partial="$output_directory/.sources.tsv.$$" +rm -f "$view_partial" "$sources_partial" + +printf 'ordinal\tsource\n' > "$sources_partial" +{ + printf '%s\n\n' '# Mock documentation synthesis' + printf '%s\n\n' 'Deterministic fixture output; no language model ran.' + source_ordinal=0 + for source_path in "$@"; do + source_ordinal=$((source_ordinal + 1)) + test -f "$source_path" + printf '%s\n\n' '---' + printf '## Source %s\n\n' "$source_ordinal" + sed -n '1p' "$source_path" + printf '\n' + printf '%s\t%s\n' "$source_ordinal" "$source_path" >> "$sources_partial" + done +} > "$view_partial" + +mv "$view_partial" "$output_directory/view.md" +mv "$sources_partial" "$output_directory/sources.tsv" diff --git a/tests/fixtures/reading/arxiv/1901.09021/document.txt b/tests/fixtures/reading/arxiv/1901.09021/document.txt new file mode 100644 index 0000000..e449eb0 --- /dev/null +++ b/tests/fixtures/reading/arxiv/1901.09021/document.txt @@ -0,0 +1,3 @@ +Tiny color blocks +Figure 1 shows tiny red, green, blue, and white color blocks. +The figure is embedded in the PDF fixture used by the scientific reading path. diff --git a/tests/fixtures/reading/arxiv/1901.09021/source.tsv b/tests/fixtures/reading/arxiv/1901.09021/source.tsv new file mode 100644 index 0000000..725a695 --- /dev/null +++ b/tests/fixtures/reading/arxiv/1901.09021/source.tsv @@ -0,0 +1 @@ +pdf 1901.09021 fixture://arxiv_site/pdf/1901.09021 diff --git a/tests/fixtures/reading/arxiv/2203.11355/document.txt b/tests/fixtures/reading/arxiv/2203.11355/document.txt new file mode 100644 index 0000000..54fb221 --- /dev/null +++ b/tests/fixtures/reading/arxiv/2203.11355/document.txt @@ -0,0 +1,5 @@ +Origami in N dimensions: How feed-forward networks manufacture linear separability +Feed-forward networks fold input space into representations whose classes can be separated. +The paper studies the geometry of the folding process. +Figure 1. Linear regions along a one dimensional input. +Activation space folding. diff --git a/tests/fixtures/reading/arxiv/2203.11355/source.tsv b/tests/fixtures/reading/arxiv/2203.11355/source.tsv new file mode 100644 index 0000000..4e2175c --- /dev/null +++ b/tests/fixtures/reading/arxiv/2203.11355/source.tsv @@ -0,0 +1 @@ +html 2203.11355 fixture://arxiv_site/html/2203.11355 diff --git a/tests/live_embedding_models.sh b/tests/live_embedding_models.sh new file mode 100755 index 0000000..90667d8 --- /dev/null +++ b/tests/live_embedding_models.sh @@ -0,0 +1,62 @@ +#!/bin/sh +set -eu + +repository_root=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) +program=${1:?usage: live_embedding_models.sh VECTOR_PROGRAM PYTHON MODEL_ROOT} +python=${2:?usage: live_embedding_models.sh VECTOR_PROGRAM PYTHON MODEL_ROOT} +model_root=${3:?usage: live_embedding_models.sh VECTOR_PROGRAM PYTHON MODEL_ROOT} +temporary=$(mktemp -d) +trap 'rm -rf "$temporary"' EXIT HUP INT TERM + +for manifest in \ + "$repository_root/models/embedding/potion-base-2m.model" \ + "$repository_root/models/embedding/mxbai-embed-xsmall-v1-int8.model" +do + slug=$(awk '$1 == "slug" { print $2 }' "$manifest") + dimensions=$(awk '$1 == "dimensions" { print $2 }' "$manifest") + model_directory=$model_root/$slug + index_directory=$temporary/views/organizing-the-information/vector-spaces/$slug/pages + + "$repository_root/bin/ib_embedding_model.grease" fetch \ + "$manifest" "$model_directory" > "$temporary/$slug-fetch.txt" + "$python" "$repository_root/experiments/embedding-models/embed_onnx.py" \ + --model-manifest "$manifest" \ + --model-directory "$model_directory" \ + --input "$repository_root/tests/fixtures/embedding-corpus.tsv" \ + --output "$temporary/$slug-rows.tsv" \ + --npz "$temporary/$slug-vectors.npz" \ + --provenance "$temporary/$slug-run.json" + "$repository_root/bin/ib_vector_view.grease" build \ + "$temporary/views" pages "$manifest" cosine "$program" \ + < "$temporary/$slug-rows.tsv" > "$temporary/$slug-build.txt" + cmp "$manifest" \ + "$temporary/views/organizing-the-information/vector-spaces/$slug/embedding-model.txt" + if test "$slug" = potion-base-2m; then + sed 's/^license MIT$/license changed-license/' "$manifest" \ + > "$temporary/conflicting-model.model" + if "$repository_root/bin/ib_vector_view.grease" build \ + "$temporary/views" pages "$temporary/conflicting-model.model" cosine "$program" \ + /dev/null 2> "$temporary/conflicting-model.txt"; then + echo 'vector view unexpectedly rebound a model slug to different facts' >&2 + exit 1 + fi + grep -F 'different immutable manifest' "$temporary/conflicting-model.txt" >/dev/null + fi + "$program" check "$index_directory" > "$temporary/$slug-check.txt" + + "$python" "$repository_root/experiments/embedding-models/embed_onnx.py" \ + --model-manifest "$manifest" \ + --model-directory "$model_directory" \ + --input "$repository_root/tests/fixtures/embedding-query.tsv" \ + --output "$temporary/$slug-query-row.tsv" + cut -f2- "$temporary/$slug-query-row.tsv" > "$temporary/$slug-query.txt" + "$program" query "$index_directory" 1 \ + < "$temporary/$slug-query.txt" > "$temporary/$slug-result.tsv" + "$program" query-text "$index_directory" 1 \ + < "$temporary/$slug-query.txt" > "$temporary/$slug-text-result.tsv" + cmp "$temporary/$slug-result.tsv" "$temporary/$slug-text-result.tsv" + sed -n '1s/\t.*//p' "$temporary/$slug-result.tsv" | grep -Fx symplectic-page + grep -F "\"dimensions\": $dimensions" "$temporary/$slug-run.json" >/dev/null + test -s "$temporary/$slug-vectors.npz" + printf 'embedding-model=%s end-to-end=ok\n' "$slug" +done diff --git a/tests/test_filesystem_views.grease b/tests/test_filesystem_views.grease new file mode 100644 index 0000000..2862109 --- /dev/null +++ b/tests/test_filesystem_views.grease @@ -0,0 +1,147 @@ +#!/bin/sh +set -eu + +repository_root=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) +. "$repository_root/lib/filesystem_views.grease" + +work_directory=$(mktemp -d) +cleanup() { + rm -rf "$work_directory" +} +trap cleanup EXIT HUP INT TERM + +view_root="$work_directory/views" +state_root="$work_directory/state" +presentation_root="$work_directory/reading" +mkdir -p \ + "$state_root/tabs/01T-CAMPAIGN" \ + "$state_root/resources/01R-RULES" \ + "$presentation_root/sources" \ + "$presentation_root/plain" \ + "$presentation_root/html" +printf '%s\n' 'id 01T-CAMPAIGN' > "$state_root/tabs/01T-CAMPAIGN/tab.txt" +printf '%s\n' 'id 01R-RULES' > "$state_root/resources/01R-RULES/resource.txt" +printf '%s\n' 'plain prepaint' > "$presentation_root/plain/view.txt" +printf '%s\n' '

fast local prepaint

' > "$presentation_root/html/view.html" + +ib_views_initialize "$view_root" +ib_views_initialize "$view_root" +test -d "$view_root/retrieved-from-the-web/text" +test -d "$view_root/retrieved-from-the-web/images" +test -d "$view_root/organizing-the-information/categories" +test -d "$view_root/organizing-the-information/vector-spaces" +test -d "$view_root/_active" +test -d "$view_root/hot" + +ib_category_add "$view_root" "$state_root" critical-role tab 01T-CAMPAIGN >/dev/null +ib_category_add "$view_root" "$state_root" campaign-4 tab 01T-CAMPAIGN >/dev/null +ib_category_add "$view_root" "$state_root" campaign-4 tab 01T-CAMPAIGN >/dev/null +ib_category_add "$view_root" "$state_root" rules-reference resource 01R-RULES >/dev/null + +tab_target=$(realpath "$state_root/tabs/01T-CAMPAIGN") +test "$(readlink "$view_root/organizing-the-information/categories/critical-role/tab-01T-CAMPAIGN")" = "$tab_target" +test "$(readlink "$view_root/organizing-the-information/categories/campaign-4/tab-01T-CAMPAIGN")" = "$tab_target" +test "$(ib_category_memberships "$view_root" "$state_root" tab 01T-CAMPAIGN)" = "$(printf '%s\n' campaign-4 critical-role)" + +ib_category_activate "$view_root" campaign-4 >/dev/null +test -L "$view_root/_active/campaign-4" +test -d "$view_root/_active/campaign-4" +ib_category_deactivate "$view_root" campaign-4 +test ! -L "$view_root/_active/campaign-4" +test -L "$view_root/organizing-the-information/categories/campaign-4/tab-01T-CAMPAIGN" + +set -- +source_ordinal=1 +while test "$source_ordinal" -le 20; do + source_path="$presentation_root/sources/page-$source_ordinal.txt" + printf 'documentation page %s\nbody not needed by the fake\n' "$source_ordinal" > "$source_path" + set -- "$@" "$source_path" + source_ordinal=$((source_ordinal + 1)) +done + +sh "$repository_root/tests/fixtures/mock_documentation_synthesizer.grease" \ + "$presentation_root/aws-composite" "$@" +test "$(grep -c '^## Source ' "$presentation_root/aws-composite/view.md")" -eq 20 +test "$(wc -l < "$presentation_root/aws-composite/sources.tsv")" -eq 21 + +ib_hot_add "$view_root" "$presentation_root" aws-iam aws-composite >/dev/null +ib_hot_add "$view_root" "$presentation_root" quick.txt plain/view.txt >/dev/null +ib_hot_add "$view_root" "$presentation_root" quick.html html/view.html >/dev/null +test -d "$view_root/hot/aws-iam" +test -f "$view_root/hot/aws-iam/view.md" +test -f "$view_root/hot/quick.txt" +test -f "$view_root/hot/quick.html" +grep -F 'documentation page 20' "$view_root/hot/aws-iam/view.md" >/dev/null + +printf '%s\n' '# Replacement presentation' > "$presentation_root/replacement.md" +ib_hot_add "$view_root" "$presentation_root" aws-iam replacement.md >/dev/null +test "$(readlink "$view_root/hot/aws-iam")" = "$(realpath "$presentation_root/replacement.md")" +test -f "$presentation_root/aws-composite/view.md" + +printf '%s\n' '# Race A' > "$presentation_root/race-a.md" +printf '%s\n' '# Race B' > "$presentation_root/race-b.md" +ib_hot_add "$view_root" "$presentation_root" race race-a.md >/dev/null & +race_a_pid=$! +ib_hot_add "$view_root" "$presentation_root" race race-b.md >/dev/null & +race_b_pid=$! +wait "$race_a_pid" +wait "$race_b_pid" +race_target=$(readlink "$view_root/hot/race") +case "$race_target" in + "$(realpath "$presentation_root/race-a.md")"|"$(realpath "$presentation_root/race-b.md")") ;; + *) + printf '%s\n' 'concurrent hot publication left an invalid target' >&2 + exit 1 + ;; +esac +test -f "$view_root/hot/race" + +printf '%s\n' 'do not clobber me' > "$view_root/hot/manual" +if ib_hot_add "$view_root" "$presentation_root" manual replacement.md >/dev/null 2>&1; then + printf '%s\n' 'hot materialization replaced a regular file' >&2 + exit 1 +fi +grep -Fx 'do not clobber me' "$view_root/hot/manual" >/dev/null + +mkdir -p "$work_directory/outside" +printf '%s\n' 'outside allowed presentation root' > "$work_directory/outside/view.md" +ln -s "$work_directory/outside/view.md" "$presentation_root/escape.md" +if ib_hot_add "$view_root" "$presentation_root" escape escape.md >/dev/null 2>&1; then + printf '%s\n' 'hot materialization accepted an escaping target' >&2 + exit 1 +fi +if ib_category_add "$view_root" "$state_root" '../escape' tab 01T-CAMPAIGN >/dev/null 2>&1; then + printf '%s\n' 'category materialization accepted an unsafe name' >&2 + exit 1 +fi +if ib_category_add "$view_root" "$state_root" _active tab 01T-CAMPAIGN >/dev/null 2>&1; then + printf '%s\n' 'category materialization accepted the reserved control name' >&2 + exit 1 +fi +ln -s "$work_directory/outside" "$state_root/tabs/01T-ESCAPE" +if ib_category_add "$view_root" "$state_root" escaped-tab tab 01T-ESCAPE >/dev/null 2>&1; then + printf '%s\n' 'category materialization accepted an escaping object target' >&2 + exit 1 +fi + +printf '%s\n' 'soon stale' > "$presentation_root/stale.md" +ib_hot_add "$view_root" "$presentation_root" stale stale.md >/dev/null +rm -f "$presentation_root/stale.md" +pruned=$(ib_views_prune_broken_links "$view_root") +test "$pruned" = "$view_root/hot/stale" +test ! -L "$view_root/hot/stale" + +ib_category_remove "$view_root" critical-role tab 01T-CAMPAIGN +test ! -L "$view_root/organizing-the-information/categories/critical-role/tab-01T-CAMPAIGN" +test -f "$state_root/tabs/01T-CAMPAIGN/tab.txt" +ib_hot_remove "$view_root" quick.txt +test ! -L "$view_root/hot/quick.txt" +test -f "$presentation_root/plain/view.txt" + +cli_view_root="$work_directory/cli-views" +sh "$repository_root/bin/ib_views.grease" init "$cli_view_root" +sh "$repository_root/bin/ib_views.grease" \ + category-add "$cli_view_root" "$state_root" rules-reference resource 01R-RULES >/dev/null +test -L "$cli_view_root/organizing-the-information/categories/rules-reference/resource-01R-RULES" + +printf '%s\n' 'filesystem view tests: ok' diff --git a/tests/test_mock_tab_model.grease b/tests/test_mock_tab_model.grease new file mode 100755 index 0000000..0992f1f --- /dev/null +++ b/tests/test_mock_tab_model.grease @@ -0,0 +1,34 @@ +#!/bin/sh +set -eu + +repository_root=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) +model="$repository_root/bin/mock_tab_model.grease" +document="$repository_root/tests/fixtures/reading/arxiv/2203.11355/document.txt" + +metadata=$("$model" inspect) +printf '%s\n' "$metadata" | grep -Fx 'model=mock-token-i8-v1' +printf '%s\n' "$metadata" | grep -Fx 'available=True' +printf '%s\n' "$metadata" | grep -Fx 'llm=False' +printf '%s\n' "$metadata" | grep -Fx 'quantization=int8-counts' +printf '%s\n' "$metadata" | grep -Fx 'dimensions=32' + +vector=$(printf '%s\n' 'feed-forward networks fold input space' | "$model" embed) +printf '%s\n' "$vector" | awk ' + NF != 32 { exit 1 } + { + nonzero = 0 + for (field = 1; field <= NF; field++) { + if ($field !~ /^[0-9]+$/ || $field < 0 || $field > 127) + exit 1 + if ($field > 0) + nonzero = 1 + } + exit !nonzero + } +' + +answer=$("$model" answer 'What do feed-forward networks do to input space?' "$document") +printf '%s\n' "$answer" | grep -Fx \ + 'Feed-forward networks fold input space into representations whose classes can be separated.' + +printf 'mock tab model tests: ok\n' diff --git a/tests/test_tab_qa_pipeline.grease b/tests/test_tab_qa_pipeline.grease new file mode 100755 index 0000000..238bc26 --- /dev/null +++ b/tests/test_tab_qa_pipeline.grease @@ -0,0 +1,171 @@ +#!/bin/sh +set -eu + +repository_root=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) +work_directory=$(mktemp -d) +cleanup() { + rm -rf "$work_directory" +} +trap cleanup EXIT HUP INT TERM + +export HOME="$work_directory/home" +export IB_READING_DIR="$work_directory/reading" +export IB_TAB_QA_DIR="$work_directory/questions and answers about tabs that the user has visited" +export IB_VIEWS_DIR="$work_directory/views" +export IB_VECTOR_INDEX_PROGRAM="$repository_root/native/vector-index/ib-vector-index" +index_directory="$IB_VIEWS_DIR/organizing-the-information/vector-spaces/mock-token-i8-v1/reading" +model_view_directory=$(dirname "$index_directory") +mkdir -p "$HOME" "$IB_READING_DIR" +cp -R "$repository_root/tests/fixtures/reading/." "$IB_READING_DIR/" + +make -C "$repository_root/native/vector-index" >/dev/null + +document_count=$(find "$IB_READING_DIR" -type f -name document.txt | wc -l | tr -d '[:space:]') +test "$document_count" = 2 +find "$IB_READING_DIR" -type f -name document.txt -exec test -s {} \; +grep -F 'Origami in N dimensions' \ + "$repository_root/tests/fixtures/arxiv_site/html/2203.11355" >/dev/null +grep -aF 'Figure 1. Tiny color blocks' \ + "$repository_root/tests/fixtures/arxiv_site/pdf/1901.09021" >/dev/null +grep -F 'Feed-forward networks fold input space' \ + "$IB_READING_DIR/arxiv/2203.11355/document.txt" >/dev/null +grep -F 'tiny red, green, blue, and white color blocks' \ + "$IB_READING_DIR/arxiv/1901.09021/document.txt" >/dev/null +awk -F '\t' 'NR == 1 && $1 == "html" && $3 == "fixture://arxiv_site/html/2203.11355" { found = 1 } END { exit !found }' \ + "$IB_READING_DIR/arxiv/2203.11355/source.tsv" +awk -F '\t' 'NR == 1 && $1 == "pdf" && $3 == "fixture://arxiv_site/pdf/1901.09021" { found = 1 } END { exit !found }' \ + "$IB_READING_DIR/arxiv/1901.09021/source.tsv" + +inspect_before=$(sh "$repository_root/bin/ask_saved_pages.grease" --inspect) +printf '%s\n' "$inspect_before" | grep -Fx 'frontend=console-text-stub' +pinned_manifest_count=$(printf '%s\n' "$inspect_before" | sed -n 's/^pinned-embedding-manifest-count=//p') +test "$pinned_manifest_count" -ge 1 +printf '%s\n' "$inspect_before" | grep -Fx 'available=True' +printf '%s\n' "$inspect_before" | grep -Fx 'llm=False' +printf '%s\n' "$inspect_before" | grep -Fx 'ensemble=single-member-test-double' +printf '%s\n' "$inspect_before" | grep -Fx 'index-available=False' + +sh "$repository_root/bin/ask_saved_pages.grease" --index > "$work_directory/index-build.txt" +grep -Fx 'build=ok' "$work_directory/index-build.txt" +grep -Fx 'count=2' "$work_directory/index-build.txt" +grep -Fx 'index-count=1' "$work_directory/index-build.txt" +grep -Fx 'query-path-count=2' "$work_directory/index-build.txt" +grep -Fx 'query-path-1=readable-fixed-width-text' "$work_directory/index-build.txt" +grep -Fx 'query-path-2=rebuildable-float32-cache' "$work_directory/index-build.txt" +grep -Fx 'document-count=2' "$work_directory/index-build.txt" +test -s "$index_directory/format.txt" +test -s "$index_directory/corpus.tsv" +test -s "$model_view_directory/model-adapter.txt" +grep -E '^adapter-command-sha256=[0-9a-f]{64}$' "$model_view_directory/model-adapter.txt" >/dev/null +test ! -e "$model_view_directory/embedding-model.txt" +test "$(wc -l < "$index_directory/corpus.tsv" | tr -d '[:space:]')" = 2 +vectors_text=$(sed -n 's/^vectors_text //p' "$index_directory/format.txt") +vectors_cache=$(sed -n 's/^vectors_cache //p' "$index_directory/format.txt") +test -s "$index_directory/$vectors_text" +test -s "$index_directory/$vectors_cache" +"$IB_VECTOR_INDEX_PROGRAM" check "$index_directory" >/dev/null +inspect_after=$(sh "$repository_root/bin/ask_saved_pages.grease" --inspect) +printf '%s\n' "$inspect_after" | grep -Fx 'index-available=True' +printf '%s\n' "$inspect_after" | grep -Fx 'index-count=1' +printf '%s\n' "$inspect_after" | grep -Fx 'query-path-count=2' + +changed_model="$work_directory/changed-model.grease" +cp "$repository_root/bin/mock_tab_model.grease" "$changed_model" +printf '%s\n' '# behavior-bearing adapter bytes changed' >> "$changed_model" +chmod +x "$changed_model" +if IB_TAB_QA_MODEL_COMMAND="$changed_model" \ + sh "$repository_root/bin/ask_saved_pages.grease" --index \ + > "$work_directory/changed-model.txt" 2>&1; then + printf 'changed adapter code silently reused the model name\n' >&2 + exit 1 +fi +grep -F 'model name is already bound to different adapter facts' \ + "$work_directory/changed-model.txt" >/dev/null + +question='What do feed-forward networks do to input space?' +printf '%s\n' "$question" | "$repository_root/bin/mock_tab_model.grease" embed \ + > "$work_directory/question-vector.txt" +IB_VECTOR_QUERY_REPORT="$work_directory/cached-query.tsv" \ + "$IB_VECTOR_INDEX_PROGRAM" query "$index_directory" 2 \ + < "$work_directory/question-vector.txt" > "$work_directory/cached-results.tsv" +IB_VECTOR_QUERY_REPORT="$work_directory/readable-query.tsv" \ + "$IB_VECTOR_INDEX_PROGRAM" query-text "$index_directory" 2 \ + < "$work_directory/question-vector.txt" > "$work_directory/readable-results.tsv" +cmp "$work_directory/cached-results.tsv" "$work_directory/readable-results.tsv" +grep -Fx 'dot-product-used-gpu=False' "$work_directory/cached-query.tsv" +grep -Fx 'compute=cpu' "$work_directory/cached-query.tsv" +grep -Fx 'storage=float32-cache' "$work_directory/cached-query.tsv" +grep -Fx 'dot-products=2' "$work_directory/cached-query.tsv" +grep -Fx 'dot-product-used-gpu=False' "$work_directory/readable-query.tsv" +grep -Fx 'storage=readable-text' "$work_directory/readable-query.tsv" + +response=$(printf '%s\n' "$question" | \ + sh "$repository_root/bin/ask_saved_pages.grease" 2> "$work_directory/prompt.txt") +grep -Fx 'question> ' "$work_directory/prompt.txt" +test -n "$response" +printf '%s\n' "$response" | grep -Fx \ + 'Feed-forward networks fold input space into representations whose classes can be separated.' + +test "$IB_TAB_QA_DIR" != "$IB_READING_DIR" +exchange_count=$(find "$IB_TAB_QA_DIR" -mindepth 1 -maxdepth 1 -type d ! -name '.partial.*' | wc -l | tr -d '[:space:]') +test "$exchange_count" = 1 +exchange_directory=$(find "$IB_TAB_QA_DIR" -mindepth 1 -maxdepth 1 -type d ! -name '.partial.*' | sed -n '1p') +test -s "$exchange_directory/question.txt" +test -s "$exchange_directory/answer.txt" +test -s "$exchange_directory/model-candidate.txt" +test -s "$exchange_directory/reduced-answer.txt" +test -s "$exchange_directory/source.tsv" +test -s "$exchange_directory/vector-format.txt" +test -s "$exchange_directory/reading-source.tsv" +grep -Fx 'dot-product-used-gpu=False' "$exchange_directory/vector-query.tsv" +grep -Fx 'compute=cpu' "$exchange_directory/vector-query.tsv" +grep -Fx 'storage=float32-cache' "$exchange_directory/vector-query.tsv" +grep -Fx 'dot-products=2' "$exchange_directory/vector-query.tsv" +grep -Fx 'reducer=mock-single-member-weighted-reducer-v1' "$exchange_directory/reducer.tsv" +grep -Fx 'ensemble-stage-ran=True' "$exchange_directory/reducer.tsv" +grep -Fx 'member-count=1' "$exchange_directory/reducer.tsv" +grep -Fx 'checks-run=3' "$exchange_directory/reducer.tsv" +grep -Fx 'candidate-nonempty=True' "$exchange_directory/reducer.tsv" +grep -E '^aggregate-score=[-+0-9.eE]+$' "$exchange_directory/reducer.tsv" >/dev/null +awk -F '\t' '$1 == "document-id" && $2 == "arxiv/2203.11355/document.txt" { found = 1 } END { exit !found }' \ + "$exchange_directory/source.tsv" +awk -F '\t' '$1 == "document-sha256" && $2 ~ /^[0-9a-f]{64}$/ { found = 1 } END { exit !found }' \ + "$exchange_directory/source.tsv" +awk -F '\t' '$1 == "reading-source-present" && $2 == "True" { found = 1 } END { exit !found }' \ + "$exchange_directory/source.tsv" +awk -F '\t' '$1 == "reading-source-sha256" && $2 ~ /^[0-9a-f]{64}$/ { found = 1 } END { exit !found }' \ + "$exchange_directory/source.tsv" +cmp "$exchange_directory/reading-source.tsv" "$IB_READING_DIR/arxiv/2203.11355/source.tsv" +grep -E '^adapter-command-sha256=[0-9a-f]{64}$' "$exchange_directory/model.tsv" >/dev/null +grep -E '^reducer-command-sha256=[0-9a-f]{64}$' "$exchange_directory/reducer-model.tsv" >/dev/null +cmp "$exchange_directory/reduced-answer.txt" "$exchange_directory/answer.txt" +grep -Fx '4=null-identity-renderer' "$exchange_directory/pipeline.tsv" +grep -Fx 'index-count=1' "$exchange_directory/indexing.tsv" +test "$(find "$IB_READING_DIR" -type f -name document.txt | wc -l | tr -d '[:space:]')" = 2 + +printf '%s\n' 'changed after indexing' >> "$IB_READING_DIR/arxiv/2203.11355/document.txt" +if sh "$repository_root/bin/ask_saved_pages.grease" "$question" \ + > "$work_directory/stale-source.txt" 2>&1; then + printf 'changed reading source was unexpectedly accepted\n' >&2 + exit 1 +fi +grep -F 'reading source changed since index build' "$work_directory/stale-source.txt" >/dev/null +test "$(find "$IB_TAB_QA_DIR" -mindepth 1 -maxdepth 1 -type d ! -name '.partial.*' | wc -l | tr -d '[:space:]')" = 1 + +if IB_TAB_QA_DIR="$IB_READING_DIR/qa" \ + sh "$repository_root/bin/ask_saved_pages.grease" "$question" \ + > "$work_directory/nested-root.txt" 2>&1; then + printf 'nested Q&A root was unexpectedly accepted\n' >&2 + exit 1 +fi +grep -F 'question-and-answer storage must be separate' "$work_directory/nested-root.txt" >/dev/null + +if IB_TAB_QA_DIR="$IB_READING_DIR/../reading/qa" \ + sh "$repository_root/bin/ask_saved_pages.grease" "$question" \ + > "$work_directory/normalized-nested-root.txt" 2>&1; then + printf 'normalized nested Q&A root was unexpectedly accepted\n' >&2 + exit 1 +fi +grep -F 'question-and-answer storage must be separate' "$work_directory/normalized-nested-root.txt" >/dev/null + +printf 'tab Q&A pipeline tests: ok\n' diff --git a/tests/vector-index-smoke.sh b/tests/vector-index-smoke.sh new file mode 100644 index 0000000..c01a17c --- /dev/null +++ b/tests/vector-index-smoke.sh @@ -0,0 +1,153 @@ +#!/bin/sh +set -eu + +program=${1:?usage: vector-index-smoke.sh PROGRAM} +temporary=$(mktemp -d) +trap 'rm -rf "$temporary"' EXIT HUP INT TERM +index=$temporary/views/organizing-the-information/vector-spaces/test-model/pages + +printf '%s\n' \ + 'book-page 1 0 0' \ + 'tool-page 0 1 0' \ + 'mixed-page 1 1 0' | + "$program" build "$index" 3 cosine > "$temporary/build.txt" + +grep -Fx 'build=ok' "$temporary/build.txt" +grep -Fx 'backend=flat-f32-exact' "$temporary/build.txt" +grep -Fx 'scalar=f32' "$temporary/build.txt" +grep -Fx 'metric=cosine' "$temporary/build.txt" +grep -Fx 'dimensions=3' "$temporary/build.txt" +grep -Fx 'count=3' "$temporary/build.txt" + +"$program" check "$index" > "$temporary/check.txt" +grep -Fx 'check=ok' "$temporary/check.txt" +grep -Fx 'text_slot_bytes=16' "$temporary/check.txt" + +printf '%s\n' '0.9 0.1 0' | + IB_VECTOR_QUERY_REPORT="$temporary/cache-query-report.txt" \ + "$program" query "$index" 2 > "$temporary/results.txt" +sed -n '1s/ .*//p' "$temporary/results.txt" | grep -Fx 'book-page' +sed -n '2s/ .*//p' "$temporary/results.txt" | grep -Fx 'mixed-page' +grep -Fx 'operation=exact-dot-product-scan' "$temporary/cache-query-report.txt" +grep -Fx 'compute=cpu' "$temporary/cache-query-report.txt" +grep -Fx 'dot-product-used-gpu=False' "$temporary/cache-query-report.txt" +grep -Fx 'storage=float32-cache' "$temporary/cache-query-report.txt" +grep -Fx 'dimensions=3' "$temporary/cache-query-report.txt" +grep -Fx 'dot-products=3' "$temporary/cache-query-report.txt" + +printf '%s\n' '0.9 0.1 0' | + IB_VECTOR_QUERY_REPORT="$temporary/text-query-report.txt" \ + "$program" query-text "$index" 2 > "$temporary/text-results.txt" +cmp "$temporary/results.txt" "$temporary/text-results.txt" +grep -Fx 'dot-product-used-gpu=False' "$temporary/text-query-report.txt" +grep -Fx 'storage=readable-text' "$temporary/text-query-report.txt" + +"$program" column "$index" 1 > "$temporary/first-coordinate.txt" +grep -Fx 'book-page +1.00000000e+00' "$temporary/first-coordinate.txt" +grep -Fx 'tool-page +0.00000000e+00' "$temporary/first-coordinate.txt" + +cp "$index/format.txt" "$temporary/format-before-failed-build.txt" +printf '%s\n' 'broken-page 1 0' | + if "$program" build "$index" 3 cosine > /dev/null 2> "$temporary/failed-build.txt"; then + echo 'wrong-dimension build unexpectedly succeeded' >&2 + exit 1 + fi +cmp "$temporary/format-before-failed-build.txt" "$index/format.txt" +printf '%s\n' '0.9 0.1 0' | + "$program" query "$index" 1 | sed -n '1s/ .*//p' | grep -Fx 'book-page' + +vectors_text=$(sed -n 's/^vectors_text //p' "$index/format.txt") +vectors_cache=$(sed -n 's/^vectors_cache //p' "$index/format.txt") +test -n "$vectors_text" +test -n "$vectors_cache" +test "$(wc -c < "$index/$vectors_text" | tr -d ' ')" = 144 +test "$(wc -c < "$index/$vectors_cache" | tr -d ' ')" = 36 +third_coordinate=$(dd if="$index/$vectors_text" bs=1 skip=32 count=15 2>/dev/null) +test "$third_coordinate" = '+0.00000000e+00' + +mv "$index/$vectors_cache" "$temporary/original-cache.f32" +printf '%s\n' '0.9 0.1 0' | + "$program" query-text "$index" 1 | sed -n '1s/ .*//p' | grep -Fx 'book-page' +if "$program" check "$index" >/dev/null 2>"$temporary/missing-cache.txt"; then + echo 'check unexpectedly accepted a missing Float32 cache' >&2 + exit 1 +fi +"$program" compile-cache "$index" > "$temporary/compile-cache.txt" +grep -Fx 'compile-cache=ok' "$temporary/compile-cache.txt" +cmp "$temporary/original-cache.f32" "$index/$vectors_cache" + +cp "$index/$vectors_cache" "$temporary/cache-before-corruption.f32" +printf '\001' | dd of="$index/$vectors_cache" bs=1 seek=0 conv=notrunc 2>/dev/null +if "$program" check "$index" >/dev/null 2> "$temporary/cache-corruption.txt"; then + echo 'check unexpectedly accepted a cache value that differs from text' >&2 + exit 1 +fi +grep -F 'cache does not match' "$temporary/cache-corruption.txt" +"$program" compile-cache "$index" >/dev/null +cmp "$temporary/cache-before-corruption.f32" "$index/$vectors_cache" + +cp "$index/$vectors_text" "$temporary/text-before-corruption.txt" +printf 'X' | dd of="$index/$vectors_text" bs=1 seek=15 conv=notrunc 2>/dev/null +if "$program" check "$index" >/dev/null 2> "$temporary/text-corruption.txt"; then + echo 'check unexpectedly accepted a malformed fixed-width text slot' >&2 + exit 1 +fi +grep -F 'invalid fixed-width slot' "$temporary/text-corruption.txt" +cp "$temporary/text-before-corruption.txt" "$index/$vectors_text" +"$program" check "$index" >/dev/null + +cp "$index/format.txt" "$temporary/strict-format.txt" +printf '%s\n' 'unknown extra' >> "$index/format.txt" +if "$program" inspect "$index" >/dev/null 2> "$temporary/extra-manifest-row.txt"; then + echo 'inspect unexpectedly accepted an unknown manifest row' >&2 + exit 1 +fi +grep -F 'unsupported format' "$temporary/extra-manifest-row.txt" +cp "$temporary/strict-format.txt" "$index/format.txt" + +printf '%s\n' '1 0' | + if "$program" query "$index" 1 > /dev/null 2> "$temporary/wrong-dimension.txt"; then + echo 'wrong-dimension query unexpectedly succeeded' >&2 + exit 1 + fi +grep -F 'wrong vector dimension' "$temporary/wrong-dimension.txt" + +printf '%s\n' '0.9 0.1 0' '' '1 0 0' | + if "$program" query "$index" 1 > /dev/null 2> "$temporary/extra-query.txt"; then + echo 'query unexpectedly accepted a second nonempty vector' >&2 + exit 1 + fi +grep -F 'exactly one vector' "$temporary/extra-query.txt" + +printf '%s\n' 'zero 0 0 0' | + if "$program" build "$temporary/zero-index" 3 cosine > /dev/null 2> "$temporary/zero.txt"; then + echo 'zero cosine vector unexpectedly succeeded' >&2 + exit 1 + fi +grep -F 'nonzero norm' "$temporary/zero.txt" + +"$program" build "$temporary/empty-index" 2 cosine "$temporary/empty-build.txt" +grep -Fx 'count=0' "$temporary/empty-build.txt" +"$program" check "$temporary/empty-index" >/dev/null +printf '%s\n' '1 0' | + "$program" query-text "$temporary/empty-index" 1 \ + > "$temporary/empty-results.txt" +test ! -s "$temporary/empty-results.txt" +empty_cache=$(sed -n 's/^vectors_cache //p' "$temporary/empty-index/format.txt") +rm -f "$temporary/empty-index/$empty_cache" +"$program" compile-cache "$temporary/empty-index" >/dev/null +test -f "$temporary/empty-index/$empty_cache" +test ! -s "$temporary/empty-index/$empty_cache" + +printf '%s\n' 'large 3e38' | + "$program" build "$temporary/dot-overflow-index" 1 dot >/dev/null +printf '%s\n' '3e38' | + if "$program" query "$temporary/dot-overflow-index" 1 \ + >/dev/null 2> "$temporary/dot-overflow.txt"; then + echo 'dot query unexpectedly emitted a non-finite score' >&2 + exit 1 + fi +grep -F 'score overflowed' "$temporary/dot-overflow.txt" + +printf '%s\n' 'vector-index-smoke=ok'