From f0eaf87bc630a9b90380b8d6678a31a768446f39 Mon Sep 17 00:00:00 2001 From: luoshuai990529 <58510855+luoshuai990529@users.noreply.github.com> Date: Wed, 11 Nov 2026 21:56:25 +0800 Subject: [PATCH] fix(fnos): support legacy x86 CPUs and recover interrupted bootstrap --- .github/workflows/fnos-release.yml | 20 ++ .../sag_api/fnos/knowledge_compatibility.py | 131 +++++++++++- apps/api/sag_api/fnos/knowledge_upgrade.py | 17 +- .../tests/test_fnos_empty_engine_bootstrap.py | 186 ++++++++++++++++++ apps/api/tests/test_fnos_supervisor.py | 42 ++++ .../tests/test_fnos_vendor_dependencies.py | 72 +++++++ docs/fnos/x86-cpu-compatibility-2026-11-11.md | 102 ++++++++++ scripts/build-fnos-native-vendor.mjs | 25 ++- scripts/fnos-native-cpu-smoke.py | 142 +++++++++++++ scripts/fnos-x86-vendor-requirements.txt | 5 + scripts/tests/fnos-native-vendor.test.mjs | 78 ++++++++ scripts/validate-fnos-native-vendor.py | 75 +++++++ 12 files changed, 882 insertions(+), 13 deletions(-) create mode 100644 apps/api/tests/test_fnos_empty_engine_bootstrap.py create mode 100644 apps/api/tests/test_fnos_vendor_dependencies.py create mode 100644 docs/fnos/x86-cpu-compatibility-2026-11-11.md create mode 100644 scripts/fnos-native-cpu-smoke.py create mode 100644 scripts/fnos-x86-vendor-requirements.txt create mode 100644 scripts/tests/fnos-native-vendor.test.mjs create mode 100644 scripts/validate-fnos-native-vendor.py diff --git a/.github/workflows/fnos-release.yml b/.github/workflows/fnos-release.yml index 2741c889..669180f8 100644 --- a/.github/workflows/fnos-release.yml +++ b/.github/workflows/fnos-release.yml @@ -142,6 +142,26 @@ jobs: node scripts/build-fnos-native-vendor.mjs \ --platform linux/amd64 \ --output "$RUNNER_TEMP/vendor" + + - name: Verify x86 package storage without AVX instructions + shell: bash + run: | + set -euo pipefail + sudo apt-get update + sudo apt-get install -y --no-install-recommends qemu-user + uv python install 3.12 + python312="$(uv python find 3.12)" + export PYTHONPATH="$RUNNER_TEMP/vendor:$GITHUB_WORKSPACE/apps/api" + export TIKTOKEN_CACHE_DIR="$RUNNER_TEMP/tiktoken-cache" + qemu-x86_64 -cpu Nehalem "$python312" scripts/fnos-native-cpu-smoke.py \ + --data-dir "$RUNNER_TEMP/cpu-smoke" --mode write --require-no-avx + qemu-x86_64 -cpu Nehalem "$python312" scripts/fnos-native-cpu-smoke.py \ + --data-dir "$RUNNER_TEMP/cpu-smoke" --mode reopen --require-no-avx + + - name: Package verified Native x86 inputs + shell: bash + run: | + set -euo pipefail node scripts/build-fnos-native-package.mjs \ --platform x86 \ --vendor "$RUNNER_TEMP/vendor" \ diff --git a/apps/api/sag_api/fnos/knowledge_compatibility.py b/apps/api/sag_api/fnos/knowledge_compatibility.py index 739db9be..a191daf3 100644 --- a/apps/api/sag_api/fnos/knowledge_compatibility.py +++ b/apps/api/sag_api/fnos/knowledge_compatibility.py @@ -127,7 +127,104 @@ def columns(table: str) -> set[str]: raise ValueError(UNKNOWN_KNOWLEDGE) -def classify_workspace(workspace: Path, marker: object, *, has_sources: bool, has_documents: bool = False) -> str: +def _only_empty_lancedb(directory: Path) -> bool: + """Recognize the exact directory-only residue of SDK ensure_local_dirs(). + + Reuse it in place: classification must not remove directories or files. + Nested directories, sidecars and links are not evidence of an empty engine. + """ + entries = list(directory.iterdir()) + return ( + len(entries) == 1 + and entries[0].name == "lancedb" + and not entries[0].is_symlink() + and entries[0].is_dir() + and not any(entries[0].iterdir()) + ) + + +def recoverable_bootstrap_uploads(workspace: Path, marker: object, records: dict[str, list[dict]]) -> bool: + """Admit only never-indexed uploads whose durable first-process jobs can replay. + + This is a read-only exception to the partial-restore guard. Existing results, + replacement jobs, missing originals or ambiguous metadata remain protected. + """ + current = workspace / "engine-v0.13-clean" + if ( + not committed_marker(marker) + or current.is_symlink() + or not current.is_dir() + or not _only_empty_lancedb(current) + ): + return False + try: + documents, sources, jobs = (records[name] for name in ("documents", "sources", "jobs")) + if not documents or any(source["chunk_count"] != 0 or source["event_count"] != 0 for source in sources): + return False + source_ids = {source["id"] for source in sources} + uploads = workspace / "uploads" + if uploads.is_symlink() or not uploads.is_dir(): + return False + uploads_root = uploads.resolve(strict=True) + for document in documents: + source_id = document["source_id"] + if ( + source_id not in source_ids + or Path(source_id).name != source_id + or source_id in {".", ".."} + or document["status"] not in {"PENDING", "LOADING"} + or document["chunk_count"] != 0 + or document["event_count"] != 0 + or document["token_usage"] != 0 + or document["sag_source_id"] is not None + or document["knowledge_state"] not in {None, "queued", "running"} + ): + return False + path = Path(document["storage_path"]) + if not path.is_absolute() or path.parent.resolve(strict=True) != uploads_root / source_id: + return False + # Permit the package's trusted /var/apps alias, but no link within + # uploads, including the source directory or the original itself. + ancestor = path + while ancestor.resolve(strict=True) != uploads_root: + if ancestor.is_symlink() or ancestor == ancestor.parent: + return False + ancestor = ancestor.parent + descriptor = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK) + with os.fdopen(descriptor, "rb") as stream: + info = os.fstat(stream.fileno()) + if not stat.S_ISREG(info.st_mode) or info.st_size != document["size_bytes"]: + return False + active = [job for job in jobs if job["document_id"] == document["id"]] + if not active: + return False + for job in active: + if ( + job["type"] != "PROCESS_DOCUMENT" + or job["status"] not in {"QUEUED", "RUNNING"} + or job["source_id"] != source_id + or job["finished_at"] is not None + ): + return False + raw_payload = job["payload_json"] + payload = json.loads(raw_payload) if isinstance(raw_payload, str) else raw_payload + if not isinstance(payload, dict): + return False + scheduler = payload.get("_scheduler", {}) + if not isinstance(scheduler, dict) or scheduler.get("blocked_reason"): + return False + checkpoint = payload.get("process_checkpoint", {}) + if not isinstance(checkpoint, dict) or any(checkpoint.values()): + return False + return True + except (KeyError, OSError, TypeError, ValueError): + return False + + +def classify_workspace( + workspace: Path, marker: object, *, has_sources: bool, has_documents: bool = False, + recoverable_uploads: bool = False, +) -> str: """Classify without using a missing app marker as evidence of incompatibility.""" current = workspace / "engine-v0.13-clean" legacy = workspace / "engine" @@ -141,9 +238,16 @@ def classify_workspace(workspace: Path, marker: object, *, has_sources: bool, ha if engine_format(current / "sag.db") != "compatible": raise ValueError(UNKNOWN_KNOWLEDGE) return "compatible" - if current.exists() and (not current.is_dir() or any(current.iterdir())): - raise ValueError(UNKNOWN_KNOWLEDGE) - if committed_marker(marker) and has_documents: + if current.exists(): + if not current.is_dir(): + raise ValueError(UNKNOWN_KNOWLEDGE) + if any(current.iterdir()) and ((has_documents and not recoverable_uploads) or not _only_empty_lancedb(current)): + # The SDK creates lancedb before its first SQLite connection. An + # interrupted bootstrap may resume only when no documents could + # lose an index, or intact original uploads have verified replay + # jobs and no index results. Keep marker/legacy checks below intact. + raise ValueError(UNKNOWN_KNOWLEDGE) + if committed_marker(marker) and has_documents and not recoverable_uploads: # A partial restore must not pair old document registrations with a # fresh empty index merely because a migration marker survived. raise ValueError(UNKNOWN_KNOWLEDGE) @@ -207,7 +311,24 @@ def classify_tenant(workspace: Path) -> str: marker = json.loads(row[0]) if row else None has_sources = bool(connection.execute("SELECT count(*) FROM sources").fetchone()[0]) has_documents = bool(connection.execute("SELECT count(*) FROM documents").fetchone()[0]) - return classify_workspace(workspace, marker, has_sources=has_sources, has_documents=has_documents) + recoverable = False + current = workspace / "engine-v0.13-clean" + if ( + has_documents and committed_marker(marker) and current.is_dir() + and not (current / "sag.db").exists() and _only_empty_lancedb(current) + ): + records = {} + for table in ("documents", "sources", "jobs"): + cursor = connection.execute(f"SELECT * FROM {table}") + columns = [item[0] for item in cursor.description] + records[table] = [ + {name: row[index] for index, name in enumerate(columns)} for row in cursor.fetchall() + ] + recoverable = recoverable_bootstrap_uploads(workspace, marker, records) + return classify_workspace( + workspace, marker, has_sources=has_sources, has_documents=has_documents, + recoverable_uploads=recoverable, + ) except (OSError, sqlite3.Error, json.JSONDecodeError): raise ValueError(UNKNOWN_KNOWLEDGE) from None diff --git a/apps/api/sag_api/fnos/knowledge_upgrade.py b/apps/api/sag_api/fnos/knowledge_upgrade.py index 9bef485e..d3e3b806 100644 --- a/apps/api/sag_api/fnos/knowledge_upgrade.py +++ b/apps/api/sag_api/fnos/knowledge_upgrade.py @@ -2,7 +2,7 @@ from pathlib import Path -from sqlalchemy import delete, func, select +from sqlalchemy import delete, func, select, text from sqlalchemy.ext.asyncio import AsyncSession from sag_api.db.models import ( @@ -19,7 +19,7 @@ UniversePartition, ) from sag_api.enums import BindingTargetType -from sag_api.fnos.knowledge_compatibility import classify_workspace, committed_marker +from sag_api.fnos.knowledge_compatibility import classify_workspace, committed_marker, recoverable_bootstrap_uploads _UPGRADE_KEY = "fnos_knowledge_engine_0_13" @@ -41,11 +41,22 @@ async def reset_legacy_knowledge( the complete tenant metadata before allowing install or upgrade to proceed. """ marker = await session.scalar(select(Setting).where(Setting.scope == "global", Setting.key == _UPGRADE_KEY)) + has_documents = bool(await session.scalar(select(func.count(Document.id)))) + recoverable = False + if has_documents and not (legacy_engine_dir.parent / "engine-v0.13-clean/sag.db").exists(): + records = {} + for table in ("documents", "sources", "jobs"): + rows = await session.execute(text(f"SELECT * FROM {table}")) + records[table] = [dict(row) for row in rows.mappings()] + recoverable = recoverable_bootstrap_uploads( + legacy_engine_dir.parent, marker.value if marker is not None else None, records, + ) state = classify_workspace( legacy_engine_dir.parent, marker.value if marker is not None else None, has_sources=bool(await session.scalar(select(func.count(Source.id)))), - has_documents=bool(await session.scalar(select(func.count(Document.id)))), + has_documents=has_documents, + recoverable_uploads=recoverable, ) if state == "compatible" and marker is not None and committed_marker(marker.value): return 0 diff --git a/apps/api/tests/test_fnos_empty_engine_bootstrap.py b/apps/api/tests/test_fnos_empty_engine_bootstrap.py new file mode 100644 index 00000000..ba9b40ca --- /dev/null +++ b/apps/api/tests/test_fnos_empty_engine_bootstrap.py @@ -0,0 +1,186 @@ +"""Recover an interrupted first engine bootstrap without clearing knowledge.""" + +import pytest + +from sag_api.fnos.knowledge_compatibility import UNKNOWN_KNOWLEDGE, classify_workspace + +COMMITTED = {"engine": "0.13.0", "knowledge_reset": True} + + +@pytest.mark.parametrize("marker,has_sources,expected", [ + (None, False, "fresh"), + (COMMITTED, False, "compatible"), + (COMMITTED, True, "compatible"), +]) +def test_empty_lancedb_bootstrap_can_resume_without_deleting_directories(tmp_path, marker, has_sources, expected): + directory = tmp_path / "engine-v0.13-clean" / "lancedb" + directory.mkdir(parents=True) + + for _ in range(2): + assert classify_workspace(tmp_path, marker, has_sources=has_sources) == expected + assert directory.is_dir() + assert list(directory.iterdir()) == [] + + +@pytest.mark.parametrize("marker", [None, COMMITTED]) +def test_registered_documents_prevent_empty_bootstrap_recovery(tmp_path, marker): + (tmp_path / "engine-v0.13-clean" / "lancedb").mkdir(parents=True) + with pytest.raises(ValueError, match=UNKNOWN_KNOWLEDGE): + classify_workspace(tmp_path, marker, has_sources=True, has_documents=True) + + +@pytest.mark.parametrize("defect", ["vector-file", "vector-subdirectory", "unknown-file", "unknown-directory", + "database-sidecar", "corrupt-database", "vector-symlink", "vector-file-path"]) +def test_unknown_or_populated_bootstrap_is_preserved_and_rejected(tmp_path, defect): + current = tmp_path / "engine-v0.13-clean" + vector = current / "lancedb" + vector.mkdir(parents=True) + if defect == "vector-file": + (vector / "data.lance").write_bytes(b"existing knowledge") + elif defect == "vector-subdirectory": + (vector / "data.lance").mkdir() + elif defect == "unknown-file": + (current / "unknown").write_bytes(b"keep") + elif defect == "unknown-directory": + (current / "unknown").mkdir() + elif defect == "database-sidecar": + (current / "sag.db-wal").write_bytes(b"uncheckpointed data") + elif defect == "corrupt-database": + (current / "sag.db").write_bytes(b"damaged database") + elif defect == "vector-symlink": + vector.rmdir() + target = tmp_path / "outside" + target.mkdir() + vector.symlink_to(target, target_is_directory=True) + elif defect == "vector-file-path": + vector.rmdir() + vector.write_bytes(b"keep") + before = {str(p.relative_to(current)): p.read_bytes() for p in current.rglob("*") if p.is_file()} + + with pytest.raises(ValueError, match=UNKNOWN_KNOWLEDGE): + classify_workspace(tmp_path, COMMITTED, has_sources=True) + + assert {str(p.relative_to(current)): p.read_bytes() for p in current.rglob("*") if p.is_file()} == before + assert vector.exists() + + +@pytest.mark.asyncio +async def test_real_sdk_interruption_before_database_creation_is_recoverable(tmp_path, monkeypatch): + from zleap.sag import DataEngine + from zleap.sag.core import resources + + from sag_api.core.config import settings + from sag_api.sag.config_builder import build_engine_config + + current = tmp_path / "engine-v0.13-clean" + config = build_engine_config(settings.model_copy(update={"data_dir": str(current)})) + engine = DataEngine(config, data_source_id="bootstrap-regression", health_check=False) + + def interrupted(*_args, **_kwargs): + raise RuntimeError("interrupted bootstrap") + + with monkeypatch.context() as patch: + patch.setattr(resources, "build_container", interrupted) + with pytest.raises(Exception, match="interrupted bootstrap"): + await engine.init_schema() + + assert (current / "lancedb").is_dir() + assert not (current / "sag.db").exists() + assert classify_workspace(tmp_path, COMMITTED, has_sources=True) == "compatible" + + await engine.init_schema() + assert (current / "sag.db").is_file() + assert classify_workspace(tmp_path, COMMITTED, has_sources=True) == "compatible" + await engine.aclose() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("job_status", ["QUEUED", "RUNNING"]) +@pytest.mark.parametrize("defect", [None, "ready-document", "source-chunks", "document-chunks", "checkpoint", + "no-job", "finished-job", "wrong-source-job", "truncated-upload", "missing-upload", + "outside-upload", "symlink-upload", "symlink-parent", "nonempty-vector", + "blocked-job"]) +async def test_first_upload_recovery_uses_durable_job_and_intact_original(tmp_path, job_status, defect): + from sqlalchemy import select + from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine + + from sag_api.db.base import Base + from sag_api.db.models import Document, Job, Setting, Source + from sag_api.enums import DocumentStatus, JobStatus, JobType + from sag_api.fnos.knowledge_compatibility import classify_tenant + from sag_api.fnos.knowledge_upgrade import reset_legacy_knowledge + + (tmp_path / "meta").mkdir() + db = create_async_engine(f"sqlite+aiosqlite:///{tmp_path / 'meta/sag.db'}") + async with db.begin() as connection: + await connection.run_sync(Base.metadata.create_all) + sessions = async_sessionmaker(db, expire_on_commit=False) + vector = tmp_path / "engine-v0.13-clean/lancedb" + vector.mkdir(parents=True) + if defect == "nonempty-vector": + (vector / "existing").write_text("keep") + original = tmp_path / "uploads/source/doc.txt" + original.parent.mkdir(parents=True) + original.write_bytes(b"original") + storage_path = original + if defect == "truncated-upload": + original.write_bytes(b"short") + elif defect == "missing-upload": + original.unlink() + elif defect == "outside-upload": + storage_path = tmp_path / "outside.txt" + storage_path.write_bytes(b"original") + elif defect in {"symlink-upload", "symlink-parent"}: + target = tmp_path / "outside" + target.mkdir() + (target / "doc.txt").write_bytes(b"original") + original.unlink() + if defect == "symlink-upload": + original.symlink_to(target / "doc.txt") + else: + original.parent.rmdir() + original.parent.symlink_to(target, target_is_directory=True) + try: + async with sessions() as session: + source = Source(id="source", name="First", sag_source_config_id="first", + chunk_count=1 if defect == "source-chunks" else 0) + session.add(source) + await session.flush() + document = Document(id="doc", source_id=source.id, filename="doc.txt", storage_path=str(storage_path), + size_bytes=8, status=DocumentStatus.READY if defect == "ready-document" + else (DocumentStatus.LOADING if job_status == "RUNNING" else DocumentStatus.PENDING), + chunk_count=1 if defect == "document-chunks" else 0) + session.add_all([document, Setting(scope="global", key="fnos_knowledge_engine_0_13", value=COMMITTED), + Setting(scope="global", key="model_config", value={"llm_api_key": "preserved-test-key"})]) + await session.flush() + if defect != "no-job": + session.add(Job(type=JobType.PROCESS_DOCUMENT, + status=JobStatus.SUCCEEDED if defect == "finished-job" else JobStatus[job_status], + source_id=None if defect == "wrong-source-job" else source.id, document_id=document.id, + payload={ + "checkpoint": {"process_checkpoint": {"chunk_ids": ["already-indexed"]}}, + "blocked-job": {"_scheduler": {"blocked_reason": "source_maintenance"}}, + }.get(defect, {}))) + await session.commit() + if defect is None: + assert classify_tenant(tmp_path) == "compatible" + assert await reset_legacy_knowledge(session, tmp_path / "engine") == 0 + assert await session.get(Document, document.id) is not None + assert (await session.scalar(select(Setting).where(Setting.key == "model_config"))).value == { + "llm_api_key": "preserved-test-key" + } + assert original.read_bytes() == b"original" + from sag_api.jobs.inproc import InProcessAsyncQueue + + queue = InProcessAsyncQueue(sessions, None) + await queue._recover() + assert queue._queue.qsize() == 1 + async with sessions() as recovered: + assert (await recovered.scalar(select(Job))).status == JobStatus.QUEUED + else: + with pytest.raises(ValueError, match=UNKNOWN_KNOWLEDGE): + classify_tenant(tmp_path) + with pytest.raises(ValueError, match=UNKNOWN_KNOWLEDGE): + await reset_legacy_knowledge(session, tmp_path / "engine") + finally: + await db.dispose() diff --git a/apps/api/tests/test_fnos_supervisor.py b/apps/api/tests/test_fnos_supervisor.py index aa3fae1d..c3914dea 100644 --- a/apps/api/tests/test_fnos_supervisor.py +++ b/apps/api/tests/test_fnos_supervisor.py @@ -234,6 +234,48 @@ def clock() -> Clock: return Clock() +@pytest.mark.asyncio +async def test_supervisor_starts_tenant_with_empty_lancedb_bootstrap(tmp_path): + import json + import sqlite3 + + from sag_api.fnos.supervisor import WorkerSupervisor + from sag_api.fnos.workspace import WorkspacePaths + + identity = GatewayIdentity(1000, "Alice", False) + paths = WorkspacePaths.for_identity(tmp_path / "data", tmp_path / "temp", identity) + paths.prepare() + with sqlite3.connect(paths.database_file) as connection: + connection.executescript(""" + CREATE TABLE settings (scope TEXT, key TEXT, value_json TEXT); + CREATE TABLE sources (id TEXT); + CREATE TABLE documents (id TEXT); + INSERT INTO sources VALUES ('first-source'); + """) + connection.execute("INSERT INTO settings VALUES (?, ?, ?)", ( + "global", "fnos_knowledge_engine_0_13", + json.dumps({"engine": "0.13.0", "knowledge_reset": True}), + )) + connection.execute("INSERT INTO settings VALUES (?, ?, ?)", ( + "global", "model_config", json.dumps({"llm_api_key": "test-key-preserved"}), + )) + vector = paths.engine_dir / "lancedb" + vector.mkdir() + metadata_before = paths.database_file.read_bytes() + vector_inode = vector.stat().st_ino + factory = FakeFactory() + supervisor = WorkerSupervisor(paths.data_root, paths.temp_root, process_factory=factory) + try: + lease = await supervisor.acquire(identity) + assert factory.spawn_count == 1 + await lease.release() + finally: + await supervisor.close() + assert paths.database_file.read_bytes() == metadata_before + assert vector.stat().st_ino == vector_inode + assert list(vector.iterdir()) == [] + + @pytest.fixture def supervisor(tmp_path: Path, clock: Clock): from sag_api.fnos.supervisor import WorkerSupervisor diff --git a/apps/api/tests/test_fnos_vendor_dependencies.py b/apps/api/tests/test_fnos_vendor_dependencies.py new file mode 100644 index 00000000..1a57f829 --- /dev/null +++ b/apps/api/tests/test_fnos_vendor_dependencies.py @@ -0,0 +1,72 @@ +"""Validate target wheel metadata without importing target native extensions.""" + +import shutil +import subprocess +import sys +from pathlib import Path + +import pytest + +import packaging + +CHECKER = Path(__file__).resolve().parents[3] / "scripts/validate-fnos-native-vendor.py" + + +def distribution(root, name, version, *requirements): + directory = root / f"{name.replace('-', '_')}-{version}.dist-info" + directory.mkdir() + (directory / "METADATA").write_text( + f"Metadata-Version: 2.1\nName: {name}\nVersion: {version}\n" + + "".join(f"Requires-Dist: {value}\n" for value in requirements) + ) + + +@pytest.fixture +def vendor(tmp_path): + shutil.copytree(Path(packaging.__file__).parent, tmp_path / "packaging") + distribution(tmp_path, "packaging", "26.0") + distribution(tmp_path, "zleap-sag", "0.15.0", "lancedb>=0.16") + distribution(tmp_path, "lancedb-compat", "0.41.0", "lance-namespace>=0.13.0", "packaging>=23") + distribution(tmp_path, "lance-namespace", "0.13.0", "lance-namespace-urllib3-client==0.13.0") + distribution(tmp_path, "lance-namespace-urllib3-client", "0.13.0") + return tmp_path + + +def check(vendor): + return subprocess.run([sys.executable, str(CHECKER), "--vendor", str(vendor), "--platform", "linux/amd64"], + capture_output=True, text=True) + + +def test_official_compat_satisfies_only_sdk_lancedb_requirement(vendor): + assert check(vendor).returncode == 0 + + +@pytest.mark.parametrize("defect", ["standard-package", "missing-package", "wrong-version", "unapproved-alias", + "target-linux", "target-x86", "target-python", "sdk-version-mismatch"]) +def test_missing_incompatible_or_unapproved_target_dependencies_fail(vendor, defect): + if defect == "standard-package": + distribution(vendor, "lancedb", "0.34.0") + elif defect == "missing-package": + distribution(vendor, "dependent", "1.0", "absent>=1") + elif defect == "wrong-version": + distribution(vendor, "dependent", "1.0", "packaging<20") + elif defect == "unapproved-alias": + distribution(vendor, "dependent", "1.0", "lancedb>=0.16") + elif defect == "target-linux": + distribution(vendor, "dependent", "1.0", "absent; sys_platform == 'linux'") + elif defect == "target-x86": + distribution(vendor, "dependent", "1.0", "absent; platform_machine == 'x86_64'") + elif defect == "target-python": + distribution(vendor, "dependent", "1.0", "absent; python_version >= '3.12'") + elif defect == "sdk-version-mismatch": + (vendor / "zleap_sag-0.15.0.dist-info/METADATA").write_text( + "Name: zleap-sag\nVersion: 0.15.0\nRequires-Dist: lancedb<0.40\n" + ) + assert check(vendor).returncode != 0 + + +def test_host_platform_and_unselected_extras_do_not_require_target_packages(vendor): + distribution(vendor, "dependent", "1.0", "absent; sys_platform == 'darwin'", + "absent; platform_machine == 'aarch64'", "absent; python_version < '3.12'", + "absent; extra == 'dev'") + assert check(vendor).returncode == 0 diff --git a/docs/fnos/x86-cpu-compatibility-2026-11-11.md b/docs/fnos/x86-cpu-compatibility-2026-11-11.md new file mode 100644 index 00000000..f51fc95d --- /dev/null +++ b/docs/fnos/x86-cpu-compatibility-2026-11-11.md @@ -0,0 +1,102 @@ +# fnOS x86 CPU 兼容与初始化恢复验收 + +日期:2026-11-11(Asia/Shanghai)。范围:fnOS Native x86;SDK 保持 `zleap-sag==0.15.0`。 + +## 故障链与包内实物 + +用户报告的非法指令位于 LanceDB 原生扩展。现有 `1.5.13-fnos` 实机安装目录的 +`lancedb/_lancedb.abi3.so` SHA-256 为 +`7bef83b94bef1b2b39432ade0a7108b834a1c3dd03388ff1fdf502cd51ac6f07`, +与从当前 frozen lock 导出构建的 Linux x86 vendor 完全一致,分发版本是 `lancedb==0.34.0`。 + +[官方 Python README](https://github.com/lancedb/lancedb/blob/main/python/README.md) +说明标准 x86 wheel 采用 Haswell 基线,需要 AVX2、FMA、F16C; +[官方问题 #3324](https://github.com/lancedb/lancedb/issues/3324) 记录了旧 CPU 的 SIGILL。 +本轮以同一份 0.34.0 vendor 在 QEMU Nehalem 下执行导入,进程退出码为 132, +输出 `uncaught target signal 4 (Illegal instruction)`。这与用户证据一致。 + +SDK 在第一次关系数据库连接之前先调用 `ensure_local_dirs()` 创建 `lancedb/`, +随后加载向量原生库。SIGILL 会终止整个 Worker,不能被 Python 异常处理恢复; +因此可能留下没有引擎 `sag.db` 的空目录。旧目录判定又阻止下一次启动,最终表现为 +`/auth/me` 返回初始化失败 503。空目录是崩溃残留,CPU 指令不兼容才是本次首次故障原因。 + +## 依赖方案 + +官方 [lancedb-compat](https://pypi.org/project/lancedb-compat/) 使用 x86-64-v2 基线和运行时 SIMD 分派。 +它与标准分发占用相同导入命名空间,不能混装。 + +- PyPI 没有 `lancedb-compat==0.34.0`。 +- 实物检查发现 0.36.0、0.37.1、0.38.0、0.40.0 的 Python 入口仍查询标准分发的版本元数据, + 单独安装不能正常导入;问题见 [#3950](https://github.com/lancedb/lancedb/issues/3950)。 +- 0.41.0 官方完整 wheel 已从原生模块读取版本,包含 [#3973](https://github.com/lancedb/lancedb/pull/3973) + 所述修复;本次不修改它的 Python 源码、原生 `.so` 或分发元数据。 + +x86 vendor 精确固定 `lancedb-compat==0.41.0` 及其要求的 namespace/client 0.13.0, +官方完整 wheel URL 和 SHA 记录在 `scripts/fnos-x86-vendor-requirements.txt`。 +其余依赖保留 `apps/api/uv.lock` 的版本和哈希。导出时排除被替代的三个标准分发, +以 `--no-deps --require-hashes` 一次安装到空目录,避免 SDK 重新拉入标准 LanceDB。 +构建后按 Python 3.12/Linux/x86 marker 核验完整依赖闭包;仅 SDK 的 `lancedb>=0.16` +要求允许由官方 compat 分发满足,并检查版本范围。不伪造 `lancedb.dist-info`。 +ARM 构建行为与主锁文件均未改变。 + +## 无 AVX 环境验证 + +宿主为 ARM64 Docker VM;在 ARM Debian 容器内显式使用 `qemu-x86_64 7.2.22 -cpu Nehalem`, +运行 Linux x86 CPython 3.12.16。测试读取 CPUID 指令本身,不使用会反映宿主的 `/proc/cpuinfo`。 + +```json +{"avx":false,"avx2":false,"f16c":false,"fma":false,"popcnt":true,"sse3":true,"sse4_1":true,"sse4_2":true,"ssse3":true} +``` + +已验证: + +| 场景 | 结果 | +| --- | --- | +| 标准 0.34.0 / Nehalem 导入负例 | SIGILL,退出 132 | +| 官方未修改 compat 0.41.0 / Nehalem | 导入、建表、文本与向量写入、向量检索通过 | +| SDK 0.15 + 应用 EngineManager / Nehalem | 初始化、分块、关系与向量入库、应用批量向量检索通过,检索到真实文档内容 | +| 独立新进程重新打开 compat 创建的数据 / Nehalem | 初始化与相同文档检索通过 | +| 标准 0.34.0 / Haswell 创建 SDK 数据 | 文档入库与应用检索通过 | +| compat 0.41.0 / Nehalem 打开上述 0.34.0 数据 | 原有文档检索通过 | + +测试脚本为 `scripts/fnos-native-cpu-smoke.py`,使用隔离数据目录和确定性的本地 embedding HTTP fixture, +不发送真实资料或密钥到模型平台。每次 write/reopen 都检查检索结果中确实包含文档内容, +不是仅检查接口返回成功。模型质量、商业服务密钥与计费不属于该 fixture 的验证范围。 + +发布工作流在打包之前使用 QEMU 执行 write/reopen;脚本要求 CPUID 中 AVX/AVX2/FMA/F16C 全部关闭, +普通 CPU 上通过不会被视作旧 CPU 验收。新基线仍要求 x86-64-v2;不宣称兼容低于该基线的 CPU。 + +## 初始化中断恢复保护 + +仅复用已识别的空 `lancedb/` 目录,不删除文件、不清空数据库。 +无已登记文档时保留原有 marker 和旧引擎格式检查;已有首次上传文档时,必须同时确认: +初始化标记已提交、文档和信源没有索引成果、原始上传文件路径安全且大小匹配、 +有可重放的 PROCESS_DOCUMENT QUEUED/RUNNING 任务且没有索引 checkpoint。 +已有成功文档、丢失原文件、未知文件、非空索引、符号链接和损坏数据库仍会拒绝自动恢复。 + +测试覆盖 SDK 自己创建目录后被中断、恢复检查、实际 SDK 后续建库,以及任务队列把 RUNNING +恢复为 QUEUED 并重新入队。旧版生命周期脚本可能使用系统 Python,因此独立判定器保留 Python 3.9 可执行性。 + +## 交付边界 + +完整本地验证:后端 1075 passed / 5 skipped,前端 554 passed,i18n/typecheck/lint/build 通过; +`SAG_FNPACK_TESTS=1` 的全部 fnOS 合约 196 passed / 14 skipped(Linux `/proc` 专属项在 macOS 跳过)。 +源码 lint 和 diff 检查通过。此前发现的独立脚本 Python 3.9 兼容问题已修正并完成整套复跑。 + +实机隔离验收:FN-EVO4-9C3D,x86_64,内核 `6.18.18.c1032-trim`,使用系统提供的 Python 3.12/Node 22, +以 `sag` 包用户运行实际 FPK 解包内容,持久化目录与服务 socket 均放在独立临时树。 +SDK 自己创建残留目录后,实际 `main upgrade-knowledge` 完成冷备且元数据库逐字节不变; +两轮 `main start/stop` 的健康检查、`auth/me`、信源列表和聊天页均返回 200。 +实际 `install_callback` 对保留的空目录故障数据也通过,并生成冷备。 +这是实机隔离生命周期验收,不是应用中心对现有 SAG 的覆盖安装验收;原有 SAG 安装和知识数据未被替换。 + +本轮 FPK 使用正式版本格式 `1.5.14-fnos`,由共享版本解析器根据已发布的 fnOS 版本推导。 +PR 合并和 GitHub Release 发布是独立动作;本地生成包不等于已通过远端 Delivery 发布。 +FPK 大小 302,813,340 字节,SHA-256: +`6e38809d2bc23667087a9f5f8f3707674c47d550330ab70a0da7ca0949b1758d`。 +包内判定器与修复源码一致,原生库哈希与官方 compat wheel 一致,120 个 vendor 分发版本与 QEMU 验证环境一致。 + +故障用户建议在应用中心停止 SAG,卸载时明确选择保留数据,再手动安装此 FPK。 +新包的保留数据安装检查已在隔离树验证;不要选择删除应用数据,也不要自行清空索引目录。 +不宣称旧安装脚本一定允许直接覆盖升级:本轮 CLI `install-fpk` 对已安装应用返回 installed,未执行覆盖替换。 +无 AVX 模拟验证不能替代用户实际设备安装确认,但能直接验证此次非法指令的修复路径。 diff --git a/scripts/build-fnos-native-vendor.mjs b/scripts/build-fnos-native-vendor.mjs index 03be70d5..7d6b4a27 100644 --- a/scripts/build-fnos-native-vendor.mjs +++ b/scripts/build-fnos-native-vendor.mjs @@ -1,15 +1,21 @@ #!/usr/bin/env node -import { cp, lstat, mkdir, mkdtemp, readdir, readFile, rename, rm, writeFile } from "node:fs/promises"; +import { appendFile, lstat, mkdir, mkdtemp, readdir, readFile, rm, writeFile } from "node:fs/promises"; import { spawnSync } from "node:child_process"; import os from "node:os"; import path from "node:path"; import { createHash } from "node:crypto"; -import { fileURLToPath } from "node:url"; +import { fileURLToPath, pathToFileURL } from "node:url"; const root = fileURLToPath(new URL("..", import.meta.url)); const api = path.join(root, "apps/api"); const platforms = { "linux/amd64": "x86_64-unknown-linux-gnu", "linux/arm64": "aarch64-unknown-linux-gnu" }; const fail = (message) => { throw new Error(`fnos-native-vendor: ${message}`); }; +export function vendorDependencyPlan(platform) { + return platform === "linux/amd64" + ? { excluded: ["lancedb", "lance-namespace", "lance-namespace-urllib3-client"], + installFlags: ["--no-deps", "--require-hashes"], keepHashes: true } + : { excluded: [], installFlags: [], keepHashes: false }; +} function command(bin, args, options) { const result = spawnSync(bin, args, { encoding: "utf8", ...options }); if (result.status !== 0) fail(`${bin}: ${(result.stderr || result.stdout).trim()}`); } async function files(dir, base = dir) { const out = []; for (const entry of await readdir(dir, { withFileTypes: true })) { const child = path.join(dir, entry.name); if (entry.isDirectory()) out.push(...await files(child, base)); else if (entry.isFile()) out.push(path.relative(base, child)); else fail(`unsupported entry ${child}`); } return out; } async function pruneDevelopmentFiles(dir) { for (const entry of await readdir(dir, { withFileTypes: true })) { const child = path.join(dir, entry.name); if (entry.isDirectory()) { if (["test", "tests", "__pycache__"].includes(entry.name)) await rm(child, { recursive: true, force: true }); else await pruneDevelopmentFiles(child); } else if (entry.isFile() && entry.name.endsWith(".pyc")) await rm(child, { force: true }); } } @@ -22,8 +28,15 @@ async function main() { const temp = await mkdtemp(path.join(os.tmpdir(), "sag-native-vendor-")); try { const requirements = path.join(temp, "requirements.txt"); - command("uv", ["export", "--frozen", "--no-dev", "--no-hashes", "--no-emit-project", "--output-file", requirements], { cwd: api }); - command("uv", ["pip", "install", "--target", target, "--python-version", "3.12", "--python-platform", platforms[platform], "--only-binary", ":all:", "-r", requirements]); + const plan = vendorDependencyPlan(platform); + command("uv", ["export", "--frozen", "--no-dev", ...(!plan.keepHashes ? ["--no-hashes"] : []), + "--no-emit-project", ...plan.excluded.flatMap((name) => ["--no-emit-package", name]), + "--output-file", requirements], { cwd: api }); + if (plan.keepHashes) await appendFile(requirements, `\n${await readFile(path.join(root, "scripts/fnos-x86-vendor-requirements.txt"), "utf8")}`); + command("uv", ["pip", "install", "--target", target, "--python-version", "3.12", "--python-platform", platforms[platform], + "--only-binary", ":all:", ...plan.installFlags, "-r", requirements]); + if (plan.keepHashes) command(process.env.PYTHON ?? "python3", [path.join(root, "scripts/validate-fnos-native-vendor.py"), + "--vendor", target, "--platform", platform]); await pruneDevelopmentFiles(target); const entries = await files(target); if (entries.some((entry) => /(^|\/)(__pycache__|tests?)(\/|$)|\.pyc$|\.(dll|dylib)$/i.test(entry))) fail("vendor contains forbidden development or host files"); @@ -31,4 +44,6 @@ async function main() { await writeFile(path.join(target, "vendor-manifest.json"), `${JSON.stringify(manifest, null, 2)}\n`); } finally { await rm(temp, { recursive: true, force: true }); } } -main().catch((error) => { process.stderr.write(`${error.message}\n`); process.exitCode = 1; }); +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + main().catch((error) => { process.stderr.write(`${error.message}\n`); process.exitCode = 1; }); +} diff --git a/scripts/fnos-native-cpu-smoke.py b/scripts/fnos-native-cpu-smoke.py new file mode 100644 index 00000000..3edec0c0 --- /dev/null +++ b/scripts/fnos-native-cpu-smoke.py @@ -0,0 +1,142 @@ +"""Exercise the shipped SDK/storage under an explicitly constrained x86 CPU. + +Run write and reopen in separate processes. Embeddings come from a deterministic +loopback fixture; this tests native storage compatibility, not model quality. +""" + +import argparse +import asyncio +import ctypes +import json +import mmap +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from importlib.metadata import version +from pathlib import Path + + +def cpu_features() -> dict[str, bool]: + # SysV x86-64: cpuid(leaf, subleaf, uint32[4]); preserve callee-owned RBX. + # /proc/cpuinfo reports the ARM host under qemu-user, so query CPUID itself. + code = bytes.fromhex( + "53 49 89 d0 89 f8 89 f1 0f a2 41 89 00 41 89 58 04 41 89 48 08 41 89 50 0c 5b c3" + ) + with mmap.mmap(-1, len(code), prot=mmap.PROT_READ | mmap.PROT_WRITE | mmap.PROT_EXEC) as memory: + memory.write(code) + call = ctypes.CFUNCTYPE(None, ctypes.c_uint32, ctypes.c_uint32, ctypes.POINTER(ctypes.c_uint32))( + ctypes.addressof(ctypes.c_char.from_buffer(memory)) + ) + first, seventh = (ctypes.c_uint32 * 4)(), (ctypes.c_uint32 * 4)() + call(1, 0, first) + call(7, 0, seventh) + result = {name: bool(first[2] & (1 << bit)) for name, bit in { + "sse3": 0, "ssse3": 9, "sse4_1": 19, "sse4_2": 20, "popcnt": 23, + "avx": 28, "fma": 12, "f16c": 29, + }.items()} + result["avx2"] = bool(seventh[1] & (1 << 5)) + return result + + +class EmbeddingFixture(BaseHTTPRequestHandler): + def log_message(self, *_args): + pass + + def do_POST(self): + if self.path != "/v1/embeddings": + self.send_error(404) + return + body = json.loads(self.rfile.read(int(self.headers["Content-Length"]))) + inputs = body["input"] + if not isinstance(inputs, list): + inputs = [inputs] + payload = { + "object": "list", "model": "cpu-test", + "data": [ + {"object": "embedding", "index": index, "embedding": [1.0] + [0.0] * 31} + for index in range(len(inputs)) + ], + "usage": {"prompt_tokens": 1, "total_tokens": 1}, + } + encoded = json.dumps(payload).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(encoded))) + self.end_headers() + self.wfile.write(encoded) + + +async def exercise(args, endpoint: str) -> None: + import lancedb + + from sag_api.core.config import Settings + from sag_api.db.models import Source + from sag_api.sag.engine_manager import EngineManager + from zleap.sag.pipeline import SourceDescriptor + + assert version("zleap-sag") == "0.15.0" + assert lancedb.__version__ == args.expected_version + print(json.dumps({"sdk": version("zleap-sag"), "lancedb": lancedb.__version__}), flush=True) + settings = Settings( + _env_file=None, data_dir=str(args.data_dir), + llm_api_key="cpu-test", llm_model="openai/cpu-test", + embedding_api_key="cpu-test", embedding_model="cpu-test", + embedding_dimensions=32, embedding_base_url=endpoint, + ) + manager = EngineManager(settings) + source = Source( + id="cpu-source", name="CPU compatibility", sag_source_config_id="cpu-datasource", config={}, + ) + try: + slot = await manager._slot("cpu-datasource", source) + print("SDK_INIT_OK", flush=True) + if args.mode == "write": + descriptor = SourceDescriptor( + data_source_id="cpu-datasource", source_id="cpu-document", source_type="text", title="CPU test", + ) + chunks = await slot.engine.chunk_text( + "SAG compatibility document about the blue ocean project.", descriptor, + ) + indexed = await slot.engine.index(chunks, descriptor) + assert indexed.relation_status == "succeeded" + assert indexed.vector_status == "succeeded" + assert indexed.chunk_ids + print("SDK_DOCUMENT_INDEX_OK", flush=True) + # Use the same batch-vector path as the fnOS application, including + # embedding generation, data_source_id filters, and chunk hydration. + result = await manager._search._search_chunk_vectors( + [("cpu-datasource", source)], "blue ocean", top_k=3, requested_sources=1, + ) + assert result.sections, "No document was retrieved" + assert any("blue ocean" in item.content for item in result.sections) + assert any(item.source_id == "cpu-document" for item in result.sections) + print(json.dumps({"retrieved": len(result.sections), "mode": args.mode}), flush=True) + finally: + await manager.aclose_all() + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--data-dir", type=Path, required=True) + parser.add_argument("--mode", choices=["write", "reopen"], required=True) + parser.add_argument("--expected-version", default="0.41.0") + parser.add_argument("--require-no-avx", action="store_true") + args = parser.parse_args() + if args.mode == "write" and args.data_dir.exists(): + parser.error("write requires a new isolated data directory") + if args.mode == "reopen" and not (args.data_dir / "sag.db").is_file(): + parser.error("reopen requires the previously created engine database") + features = cpu_features() + print(json.dumps({"cpuid": features}, sort_keys=True), flush=True) + if args.require_no_avx: + assert not any(features[name] for name in ("avx", "avx2", "fma", "f16c")), features + with ThreadingHTTPServer(("127.0.0.1", 0), EmbeddingFixture) as server: + threading.Thread(target=server.serve_forever, daemon=True).start() + try: + asyncio.run(exercise(args, f"http://127.0.0.1:{server.server_port}/v1")) + finally: + server.shutdown() + print("CPU_COMPATIBILITY_PASS", flush=True) + + +if __name__ == "__main__": + main() diff --git a/scripts/fnos-x86-vendor-requirements.txt b/scripts/fnos-x86-vendor-requirements.txt new file mode 100644 index 00000000..50969b95 --- /dev/null +++ b/scripts/fnos-x86-vendor-requirements.txt @@ -0,0 +1,5 @@ +# Official PyPI wheels for the fnOS x86 baseline without AVX2/FMA/F16C. +# Keep the rest of apps/api/uv.lock unchanged; install this overlay once with --no-deps --require-hashes. +lancedb-compat @ https://files.pythonhosted.org/packages/31/fc/c285f11c645ee357aaec1cf2b790857ee053381875ca6de26529b496309b/lancedb_compat-0.41.0-cp310-abi3-manylinux_2_28_x86_64.whl --hash=sha256:66c26c78d2e8e8fb32cfc041d21b6d289e35682f8a84b3a28ed6f131ebfdf491 +lance-namespace @ https://files.pythonhosted.org/packages/e7/47/cfa33cca1ba7c749fd2918cfc5c8ded788f378cc6424d23e4fead5a14125/lance_namespace-0.13.0-py3-none-any.whl --hash=sha256:438c7b17aef421c21c138196e715f2510d62f07e865372047f87c1e75e618c7a +lance-namespace-urllib3-client @ https://files.pythonhosted.org/packages/60/4c/7b8f0712a7fe1b8655b711342bc4989696635a79e7027ee56ba3cae23e99/lance_namespace_urllib3_client-0.13.0-py3-none-any.whl --hash=sha256:fb361eb4f6c7f2d1f9e92809609657b73e38241393e9c4516e12d6b6ee643a0a diff --git a/scripts/tests/fnos-native-vendor.test.mjs b/scripts/tests/fnos-native-vendor.test.mjs new file mode 100644 index 00000000..fbdd6548 --- /dev/null +++ b/scripts/tests/fnos-native-vendor.test.mjs @@ -0,0 +1,78 @@ +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { chmod, mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import test from "node:test"; + +test("x86 vendor uses a hashed compat-only dependency overlay without changing ARM", async () => { + const { vendorDependencyPlan } = await import("../build-fnos-native-vendor.mjs"); + const x86 = vendorDependencyPlan("linux/amd64"); + assert.deepEqual(x86.excluded, ["lancedb", "lance-namespace", "lance-namespace-urllib3-client"]); + assert.deepEqual(x86.installFlags, ["--no-deps", "--require-hashes"]); + assert.equal(x86.keepHashes, true); + const arm = vendorDependencyPlan("linux/arm64"); + assert.deepEqual(arm.excluded, []); + assert.deepEqual(arm.installFlags, []); + assert.equal(arm.keepHashes, false); +}); + +test("x86 compatibility overlay pins complete official wheels and their PyPI hashes", async () => { + const text = await readFile(new URL("../fnos-x86-vendor-requirements.txt", import.meta.url), "utf8"); + const rows = text.split("\n").filter((line) => line && !line.startsWith("#")); + assert.equal(rows.length, 3); + assert.ok(rows.every((line) => / @ https:\/\/files\.pythonhosted\.org\/.*\.whl --hash=sha256:[a-f0-9]{64}$/.test(line))); + assert.match(rows[0], /^lancedb-compat @ .*lancedb_compat-0\.41\.0-.*x86_64\.whl /); + assert.match(rows[1], /^lance-namespace @ .*lance_namespace-0\.13\.0-py3-none-any\.whl /); + assert.match(rows[2], /^lance-namespace-urllib3-client @ .*0\.13\.0-py3-none-any\.whl /); +}); + +for (const platform of ["linux/amd64", "linux/arm64"]) { + test(`${platform} builder installs once and validates only the x86 replacement`, async (t) => { + const directory = await mkdtemp(path.join(os.tmpdir(), "fnos-vendor-contract-")); + t.after(() => rm(directory, { recursive: true, force: true })); + const bin = path.join(directory, "bin"); + await mkdir(bin); + const log = path.join(directory, "calls.jsonl"); + const requirements = path.join(directory, "installed.txt"); + for (const command of ["uv", "python"]) { + const script = path.join(bin, command); + await writeFile(script, `#!${process.execPath} +const fs = require("node:fs"); +const args = process.argv.slice(2); +fs.appendFileSync(process.env.CALLS, JSON.stringify({command: ${JSON.stringify(command)}, args}) + "\\n"); +if (args[0] === "export") fs.writeFileSync(args[args.indexOf("--output-file") + 1], "base==1 --hash=sha256:abc\\n"); +if (args[0] === "pip") fs.copyFileSync(args[args.indexOf("-r") + 1], process.env.INSTALLED_REQUIREMENTS); +`); + await chmod(script, 0o755); + } + const result = spawnSync(process.execPath, [new URL("../build-fnos-native-vendor.mjs", import.meta.url).pathname, + "--platform", platform, "--output", path.join(directory, "vendor")], { + env: { ...process.env, PATH: `${bin}:${process.env.PATH}`, PYTHON: path.join(bin, "python"), + CALLS: log, INSTALLED_REQUIREMENTS: requirements }, encoding: "utf8", + }); + assert.equal(result.status, 0, result.stderr); + const calls = (await readFile(log, "utf8")).trim().split("\n").map(JSON.parse); + assert.equal(calls.filter((item) => item.args[0] === "pip").length, 1); + const exported = calls.find((item) => item.args[0] === "export").args; + const installed = calls.find((item) => item.args[0] === "pip").args; + const text = await readFile(requirements, "utf8"); + assert.ok(exported.includes("--frozen")); + if (platform === "linux/amd64") { + assert.equal(exported.includes("--no-hashes"), false); + for (const name of ["lancedb", "lance-namespace", "lance-namespace-urllib3-client"]) { + assert.ok(exported.some((value, index) => value === "--no-emit-package" && exported[index + 1] === name)); + } + assert.ok(installed.includes("--no-deps") && installed.includes("--require-hashes")); + assert.match(text, /lancedb-compat @/); + assert.equal(calls.filter((item) => item.command === "python").length, 1); + } else { + assert.ok(exported.includes("--no-hashes")); + assert.equal(exported.includes("--no-emit-package"), false); + assert.equal(installed.includes("--no-deps"), false); + assert.doesNotMatch(text, /lancedb-compat/); + assert.equal(calls.filter((item) => item.command === "python").length, 0); + } + assert.ok(text.startsWith("base==1 --hash=sha256:abc\n")); + }); +} diff --git a/scripts/validate-fnos-native-vendor.py b/scripts/validate-fnos-native-vendor.py new file mode 100644 index 00000000..01484945 --- /dev/null +++ b/scripts/validate-fnos-native-vendor.py @@ -0,0 +1,75 @@ +"""Check native vendor dependency closure for the target, without loading its .so files.""" + +from __future__ import annotations + +import argparse +import importlib.metadata +import sys +from pathlib import Path + + +def validate(directory: Path, platform: str) -> None: + # packaging is a pure-Python dependency already present in the target. Do + # not require build-host packages or import any cross-compiled native code. + sys.path.insert(0, str(directory)) + from packaging.requirements import Requirement + from packaging.specifiers import SpecifierSet + from packaging.utils import canonicalize_name + + installed = {} + for distribution in importlib.metadata.distributions(path=[str(directory)]): + name = canonicalize_name(distribution.metadata["Name"]) + if name in installed: + raise ValueError(f"duplicate vendor distribution: {name}") + installed[name] = distribution + if platform == "linux/amd64": + if "lancedb" in installed: + raise ValueError("x86 vendor must contain only lancedb-compat, not the AVX2 lancedb distribution") + for name, version in { + "lancedb-compat": "0.41.0", + "lance-namespace": "0.13.0", + "lance-namespace-urllib3-client": "0.13.0", + }.items(): + if name not in installed or installed[name].version != version: + raise ValueError(f"x86 vendor requires {name}=={version}") + if "zleap-sag" not in installed: + raise ValueError("x86 vendor requires zleap-sag metadata") + environment = { + "implementation_name": "cpython", "implementation_version": "3.12.0", "os_name": "posix", + "platform_machine": "x86_64" if platform == "linux/amd64" else "aarch64", + "platform_python_implementation": "CPython", "platform_release": "", "platform_system": "Linux", + "platform_version": "", "python_full_version": "3.12.0", "python_version": "3.12", + "sys_platform": "linux", "extra": "", + } + for name, distribution in sorted(installed.items()): + requires_python = distribution.metadata.get("Requires-Python") + if requires_python and not SpecifierSet(requires_python).contains(environment["python_full_version"]): + raise ValueError(f"{name} requires Python {requires_python}, target is Python 3.12") + for value in distribution.requires or []: + requirement = Requirement(value) + if requirement.marker is not None and not requirement.marker.evaluate(environment): + continue + required = canonicalize_name(requirement.name) + # Only this SDK dependency may be fulfilled by the official + # compat distribution sharing the lancedb import namespace. + if platform == "linux/amd64" and name == "zleap-sag" and required == "lancedb": + required = "lancedb-compat" + dependency = installed.get(required) + if dependency is None or not requirement.specifier.contains(dependency.version, prereleases=True): + raise ValueError(f"{name} has unsatisfied target dependency: {requirement}") + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--vendor", required=True, type=Path) + parser.add_argument("--platform", required=True, choices=("linux/amd64", "linux/arm64")) + args = parser.parse_args() + validate(args.vendor.resolve(strict=True), args.platform) + print("fnOS target vendor dependency closure verified") + + +if __name__ == "__main__": + try: + main() + except Exception as error: + sys.exit(f"fnos-native-vendor: {error}")