From abab612eab3d41a8be289fc12f67be68959ae32e Mon Sep 17 00:00:00 2001 From: cms-cms <1303573631@qq.com> Date: Wed, 16 Sep 2026 17:41:36 +0800 Subject: [PATCH 01/25] test: cover embedded fulltext first pass and memtable release --- tests/integration_tests/EMBEDDED_FULLTEXT.md | 50 +++ .../embedded_fulltext_support.py | 366 ++++++++++++++++++ .../test_embedded_fulltext_stability.py | 69 ++++ .../test_embedded_fulltext_harness.py | 132 +++++++ 4 files changed, 617 insertions(+) create mode 100644 tests/integration_tests/EMBEDDED_FULLTEXT.md create mode 100644 tests/integration_tests/embedded_fulltext_support.py create mode 100644 tests/integration_tests/test_embedded_fulltext_stability.py create mode 100644 tests/unit_tests/test_embedded_fulltext_harness.py diff --git a/tests/integration_tests/EMBEDDED_FULLTEXT.md b/tests/integration_tests/EMBEDDED_FULLTEXT.md new file mode 100644 index 00000000..5bd18287 --- /dev/null +++ b/tests/integration_tests/EMBEDDED_FULLTEXT.md @@ -0,0 +1,50 @@ +# Embedded full-text stability (seekdb #1382) + +Run the same tests natively on Linux x86_64 and macOS arm64 with a selected +`pylibseekdb` wheel and this PySeekDB source checkout: + +```sh +SEEKDB_TEST_DATA_ROOT=/absolute/path/on/local-disk/fts-data \ +SEEKDB_TEST_ARTIFACT_ROOT=/absolute/path/to/artifacts \ +python -m pytest tests/integration_tests/test_embedded_fulltext_stability.py -v -s +``` + +Use real local disk, not a Linux tmpfs mount that rejects direct I/O. Each case +uses a new subprocess and database directory; it does not reopen an already +warmed database. The two cases participate in the existing `-k embedded` suite. + +* `first-pass`: 24,515 deterministic synthetic documents, supplied 8-dimensional + embeddings (no model/network download), SDK upsert and refresh_index, followed + immediately by four passes of 300 public `collection.hybrid_search` full-text + queries (`knn=None`). Pass zero is included in the verdict. No warm-up query, + sleeps or retries erase an initial failure. +* `scan-freeze`: a separate table with a full-text index and **no vector index**, + 24,515 preloaded documents, eight independently connected scanning threads, + and one writer inserting 32 rows then executing `ALTER SYSTEM MINOR FREEZE` + every approximately 120 ms. A barrier starts the 75-second window only when + all connections are ready. Each scanner must execute successfully and at + least one freeze must complete; all errors, not just 4016, fail the test. + +This preserves the original API failure surface and the accelerated scan/release +trigger separately. It is **not** a byte-identical replay of the original +SciFact/384-dimensional embedding benchmark, and a single passing stress run +does not establish that a probabilistic bug is absent. + +`summary.json` records runtime/version, counts, first failure and extracted trace +IDs. `errors.jsonl` preserves up to 200 failures, while the total is unbounded. +The first error immediately triggers bounded database-log context collection; +it does not wait for the pressure window to end. Log scans use a total byte +budget and include tails of rotated files larger than 256 MiB. Inventories +explicitly report scan/output truncation. `last_trace_id()` is best-effort on +the failing connection, not a guaranteed mapping to an SDK internal statement. + +`max_syslog_file_count=50` bounds the normal rotated set; it is not a guarantee +of long-term retention. Successful cases remove only their generated database +directory after their worker exits and diagnostic collection completes; failed +cases retain it. Evidence lives outside that directory. CI should archive it +before any later retention cleanup. + +For pre/post-fix comparisons, hold this test SHA and bindings source SHA fixed, +record the actual SeekDB REVISION inside each wheel, and change only the kernel +source revision. Old-build failures must retain a failed result. Do not add an +xfail or suppress 4016 to make the comparison green. diff --git a/tests/integration_tests/embedded_fulltext_support.py b/tests/integration_tests/embedded_fulltext_support.py new file mode 100644 index 00000000..76e8887a --- /dev/null +++ b/tests/integration_tests/embedded_fulltext_support.py @@ -0,0 +1,366 @@ +"""Isolated, dependency-free harness; pyseekdb is imported only in the worker. + +Synthetic deterministic documents (not the original SciFact benchmark). The +scan/freezer uses the same SQL shape as the Linux issue #1382 reproducer, without +a vector index. Public hybrid_search coverage lives in the separate first pass. +""" + +from __future__ import annotations + +import argparse +import collections +import importlib.metadata +import json +import platform +import re +import threading +import time +import traceback +from pathlib import Path + +ROWS = 24515 +WORKERS = 8 +SECONDS = 75 +INTERVAL = 0.120 +WRITE_BATCH = 32 +QUERIES = 300 +PASSES = 4 +TEXT = "the database retrieval benchmark" +TABLE = "`c$v2$16d82ab3afe311f186fe000000000000`" +SEARCH = ( + f"SELECT `_id`, MATCH(`document`) AGAINST('{TEXT}') AS relevance FROM {TABLE} " + f"WHERE MATCH(`document`) AGAINST('{TEXT}') ORDER BY relevance DESC LIMIT 10" +) +TRACE = re.compile(r"\bY[A-Za-z0-9]+-[A-Za-z0-9]+-\d+-\d+\b") + + +def document(ordinal): + """Every row matches the broad query while cohorts supply varied queries.""" + return ( + f"{TEXT} exercises full text postings during concurrent memtable release " + f"cohort{ordinal % QUERIES:03d} document ordinal{ordinal:08d}" + ) + + +def error_details(error): + """Retain wrapper/cause messages and only report explicit numeric codes.""" + chain, seen, codes = [], set(), [] + while error is not None and id(error) not in seen: + seen.add(id(error)) + chain.append(f"{type(error).__name__}: {error}") + if error.args and isinstance(error.args[0], int): + codes.append(error.args[0]) + codes.extend(int(value) for value in re.findall(r"\bcode[=:]\s*(-?\d+)\b", str(error))) + error = error.__cause__ or error.__context__ + text = "\n".join(chain) + return {"exception_chain": chain, "error_codes": sorted(set(codes)), "trace_ids": sorted(set(TRACE.findall(text)))} + + +def capture_logs(root, evidence, label, traces=(), budget=128 * 1024 * 1024, output_limit=2 * 1024 * 1024): + """Read bounded tails by total bytes, never discard a large rotated file.""" + patterns = [value.encode() for value in traces] + [ + b"ret=-4016", + b"read_barrier_", + b"data/schema type does not match", + b"failed to get next row from memtable scanner", + b"ob_memtable_key.h", + ] + files = sorted( + (p for p in root.rglob("seekdb.log*") if p.is_file() and not p.is_symlink()), + key=lambda p: p.stat().st_mtime, + reverse=True, + ) + inventory, written = [], 0 + with (evidence / f"{label}-db-context.log").open("wb") as output: + for path in files: + size = path.stat().st_size + count = min(size, budget) + inventory.append({"path": str(path), "size": size, "read_bytes": count}) + if not count: + continue + with path.open("rb") as stream: + stream.seek(size - count) + previous, following, remaining = collections.deque(maxlen=5), 0, count + while remaining > 0: + line = stream.readline(min(65536, remaining)) + if not line: + break + remaining -= len(line) + if any(pattern in line for pattern in patterns): + # Preserve the matching line before optional context so + # a huge preceding SQL line cannot consume the budget. + context = f"\nFILE={path}\n".encode() + line + b"[preceding context]\n" + b"".join(previous) + part = context[: max(0, output_limit - written)] + output.write(part) + written += len(part) + following = 6 + if following: + part = line[: max(0, output_limit - written)] + output.write(part) + written += len(part) + following -= 1 + previous.append(line) + budget -= count + (evidence / f"{label}-log-inventory.json").write_text( + json.dumps( + { + "files": inventory, + "context_bytes": written, + "context_capped": written >= output_limit, + "scan_capped": any(item["read_bytes"] < item["size"] for item in inventory), + }, + indent=2, + ) + ) + + +class Recorder: + """Flush errors immediately and snapshot the first failure before recycling.""" + + def __init__(self, root, evidence): + self.root, self.evidence = root, evidence + self.lock = threading.Lock() + self.errors = 0 + self.traces = set() + self.first = None + + def error(self, error, phase, client=None, **details): + event = {"time": time.time(), "phase": phase, **details, **error_details(error)} + if not event["trace_ids"] and client is not None: + try: + # Immediately on the failing worker's own connection; mark it + # as best-effort, not proof of a particular internal SDK SQL. + rows = client._server._execute("SELECT last_trace_id() FROM DUAL") + event["trace_ids"] = TRACE.findall(str(rows)) + event["trace_source"] = "same_connection_last_trace_id_best_effort" + except Exception as lookup_error: + event["trace_lookup_error"] = str(lookup_error) + with self.lock: + self.errors += 1 + self.traces.update(event["trace_ids"]) + first = self.first is None + if first: + self.first = event + # Keep a bounded sample but never lose the total failure count. + if self.errors <= 200: + with (self.evidence / "errors.jsonl").open("a") as stream: + stream.write(json.dumps(event) + "\n") + if first: + (self.evidence / "first-error.json").write_text(json.dumps(event, indent=2)) + if first: + print(f"[FTS][FIRST_ERROR] {json.dumps(event)}", flush=True) + capture_logs(self.root, self.evidence, "early", event["trace_ids"]) + + +def validate_report(report, scenario): + """A zero worker exit alone is insufficient with native binding teardown.""" + assert report.get("scenario") == scenario, report + assert report.get("completed") is True and report.get("errors") == 0, report + assert report.get("rows") == ROWS, report + if scenario == "first-pass": + assert report.get("first_pass_successes") == QUERIES, report + assert report.get("fts_successes") == QUERIES * PASSES, report + else: + counts = report.get("worker_successes", []) + assert len(counts) == WORKERS and all(value > 0 for value in counts), report + assert report.get("freeze_successes", 0) > 0, report + assert report.get("elapsed_seconds", 0) >= SECONDS, report + + +def first_pass(client, recorder, report): + import pyseekdb + + collection = client.create_collection( + "fts_first_pass", + configuration=pyseekdb.HNSWConfiguration(dimension=8, distance="cosine"), + embedding_function=None, + ) + vector = [float(i + 1) / 8 for i in range(8)] + for start in range(0, ROWS, 100): + indexes = range(start, min(start + 100, ROWS)) + collection.upsert( + ids=[f"doc-{i}" for i in indexes], + documents=[document(i) for i in indexes], + embeddings=[vector for _ in indexes], + metadatas=[{"ordinal": i} for i in indexes], + ) + if start % 1000 == 0: + print(f"[FTS] ingest={min(start + 100, ROWS)}/{ROWS}", flush=True) + assert collection.count() == ROWS + collection.refresh_index() + report.update(rows=ROWS, first_pass_successes=0, fts_successes=0) + # No FTS warm-up, sleep, retry or reopen before pass zero. + for pass_id in range(PASSES): + for query_id in range(QUERIES): + try: + result = collection.hybrid_search( + query={"where_document": {"$contains": f"{TEXT} cohort{query_id:03d}"}, "n_results": 10}, + knn=None, + n_results=10, + include=["documents", "metadatas"], + ) + assert result.get("ids") and result["ids"][0], result + report["fts_successes"] += 1 + if pass_id == 0: + report["first_pass_successes"] += 1 + except Exception as error: + recorder.error( + error, + "hybrid_search", + client, + pass_id=pass_id, + query_id=query_id, + query_text=f"{TEXT} cohort{query_id:03d}", + ) + print(f"[FTS] pass={pass_id} successes={report['fts_successes']} errors={recorder.errors}", flush=True) + + +def insert_batch(client, start, size): + # IDs/documents are generated here, not user-controlled SQL inputs. + values = ",".join(f"('doc-{i}', '{document(i)}', '{{\"ordinal\":{i}}}')" for i in range(start, start + size)) + client._server._execute(f"INSERT INTO {TABLE} (`_id`, `document`, `metadata`) VALUES {values}") + + +def scan_freeze(client, make_client, recorder, report): + client._server._execute( + f"CREATE TABLE {TABLE} (`__pk_increment` BIGINT NOT NULL AUTO_INCREMENT PRIMARY KEY, " + "`_id` VARCHAR(128) NOT NULL, `document` LONGTEXT NOT NULL, `metadata` JSON NOT NULL, " + "UNIQUE KEY uk_id (`_id`), FULLTEXT KEY idx_document (`document`) WITH PARSER space)" + ) + for start in range(0, ROWS, 100): + insert_batch(client, start, min(100, ROWS - start)) + if start % 1000 == 0: + print(f"[FTS] ingest={min(start + 100, ROWS)}/{ROWS}", flush=True) + counts = client._server._execute(f"SELECT COUNT(*) AS row_count FROM {TABLE}") + count = counts[0]["row_count"] if isinstance(counts[0], dict) else counts[0][0] + assert count == ROWS, counts + schema = client._server._execute(f"SHOW CREATE TABLE {TABLE}") + assert "FULLTEXT" in str(schema).upper() and "VECTOR" not in str(schema).upper(), schema + report.update( + rows=ROWS, + schema=str(schema), + search_sql=SEARCH, + write_batch_size=WRITE_BATCH, + worker_successes=[0] * WORKERS, + freeze_successes=0, + ) + stop = threading.Event() + deadline = [0.0] + started = [0.0] + + def start_window(): + started[0] = time.monotonic() + deadline[0] = started[0] + SECONDS + + barrier = threading.Barrier(WORKERS + 2, action=start_window) + + def worker(number): + connection = None + try: + connection = make_client() + connection._server._execute("SELECT 1") + barrier.wait(timeout=60) + while time.monotonic() < deadline[0] and not stop.is_set(): + started = time.monotonic() + try: + if number == WORKERS: + start = ROWS + report["freeze_successes"] * WRITE_BATCH + insert_batch(connection, start, WRITE_BATCH) + connection._server._execute("ALTER SYSTEM MINOR FREEZE") + report["freeze_successes"] += 1 + else: + rows = connection._server._execute(SEARCH) + assert rows, "full-text scan returned no rows" + report["worker_successes"][number] += 1 + except Exception as error: + recorder.error(error, "minor-freeze" if number == WORKERS else "scan", connection, worker=number) + if number == WORKERS: + stop.set() # No valid pressure if writes/freezes cannot execute. + if number == WORKERS: + stop.wait(max(0, INTERVAL - (time.monotonic() - started))) + except Exception as error: + recorder.error(error, "worker-setup", connection, worker=number) + barrier.abort() + stop.set() + finally: + if connection is not None: + try: + connection.close() + except Exception as error: + recorder.error(error, "worker-close", worker=number) + + threads = [threading.Thread(target=worker, args=(i,), daemon=True) for i in range(WORKERS + 1)] + for thread in threads: + thread.start() + try: + barrier.wait(timeout=60) + while not stop.wait(min(5, max(0, deadline[0] - time.monotonic()))) and time.monotonic() < deadline[0]: + print( + f"[FTS] scans={sum(report['worker_successes'])} freezes={report['freeze_successes']} " + f"errors={recorder.errors}", + flush=True, + ) + finally: + for thread in threads: + thread.join(timeout=65) + assert not any(thread.is_alive() for thread in threads), "FTS worker did not drain" + report["elapsed_seconds"] = time.monotonic() - started[0] + + +def run(scenario, root, evidence): + import pyseekdb + + report = { + "scenario": scenario, + "completed": False, + "platform": platform.platform(), + "pyseekdb": importlib.metadata.version("pyseekdb"), + "pylibseekdb": importlib.metadata.version("pylibseekdb"), + "corpus": "synthetic deterministic / 24515 documents", + "workers": WORKERS, + "duration_seconds": SECONDS, + "freeze_interval_seconds": INTERVAL, + } + recorder = Recorder(root, evidence) + client = None + try: + db_dir = root / "database" + assert not db_dir.exists(), "fresh database required" + + def make_client(): + return pyseekdb.Client(path=str(db_dir), database="test") + + client = make_client() + report["database_version"] = str(client._server._execute("SELECT VERSION()")) + client._server._execute("ALTER SYSTEM SET max_syslog_file_count = 50") + print(f"[FTS] start {json.dumps(report)}", flush=True) + if scenario == "first-pass": + first_pass(client, recorder, report) + else: + scan_freeze(client, make_client, recorder, report) + report["completed"] = True + except Exception as error: + recorder.error(error, "scenario", client) + traceback.print_exc() + finally: + if client is not None: + try: + client.close() + except Exception as error: + recorder.error(error, "close") + try: + capture_logs(root, evidence, "final", recorder.traces) + except Exception as error: + recorder.error(error, "diagnostics") + report.update(errors=recorder.errors, first_error=recorder.first, trace_ids=sorted(recorder.traces)) + (evidence / "summary.json").write_text(json.dumps(report, indent=2)) + validate_report(report, scenario) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--scenario", choices=["first-pass", "scan-freeze"], required=True) + parser.add_argument("--root", type=Path, required=True) + parser.add_argument("--evidence", type=Path, required=True) + options = parser.parse_args() + run(options.scenario, options.root, options.evidence) diff --git a/tests/integration_tests/test_embedded_fulltext_stability.py b/tests/integration_tests/test_embedded_fulltext_stability.py new file mode 100644 index 00000000..292412a3 --- /dev/null +++ b/tests/integration_tests/test_embedded_fulltext_stability.py @@ -0,0 +1,69 @@ +"""Fresh-ingestion and memtable-release regressions for seekdb issue #1382. + +Each case runs in a fresh process and database directory, including with legacy +process-global pylibseekdb. Do not rerun failed queries to obtain a passing result. +""" + +import json +import os +import shutil +import subprocess +import sys +import tempfile +import time +from pathlib import Path + +import pytest +from embedded_fulltext_support import validate_report + + +@pytest.mark.parametrize("scenario", ["first-pass", "scan-freeze"]) +def test_embedded_fulltext_stability(scenario): + """Exercise the public SDK first pass and an accelerated kernel stress case.""" + repo = Path(__file__).resolve().parents[2] + data_root = Path(os.environ.get("SEEKDB_TEST_DATA_ROOT", repo / ".seekdb-test-data")) + data_root.mkdir(parents=True, exist_ok=True) + root = Path(tempfile.mkdtemp(prefix=f"fts-{scenario}-", dir=data_root)) + evidence_root = Path(os.environ.get("SEEKDB_TEST_ARTIFACT_ROOT", data_root / "artifacts")) + evidence = evidence_root / root.name + evidence.mkdir(parents=True, exist_ok=False) + command = [ + sys.executable, + str(Path(__file__).with_name("embedded_fulltext_support.py")), + "--scenario", + scenario, + "--root", + str(root), + "--evidence", + str(evidence), + ] + # A whole-case timeout includes ingestion and connection setup; the pressure + # window itself is 75 seconds, starting only after every worker is ready. + with (evidence / "worker.log").open("w") as log: + # Fixed interpreter/helper and generated paths, never shell input. + process = subprocess.Popen(command, stdout=log, stderr=subprocess.STDOUT) # noqa: S603 + started = time.monotonic() + try: + with (evidence / "worker.log").open() as progress: + while process.poll() is None: + if time.monotonic() - started > 900: + raise TimeoutError(f"FTS worker exceeded 900s; evidence={evidence}") + # Stream bounded real output rather than hiding all ingestion + # and pressure activity inside pytest until the case ends. + text = progress.read(16384) + if text: + print(text, end="", flush=True) + time.sleep(1) + print(progress.read(), end="", flush=True) + finally: + if process.poll() is None: + process.kill() + process.wait(timeout=10) + report_path = evidence / "summary.json" + assert report_path.is_file(), f"worker produced no terminal report; evidence={evidence}" + report = json.loads(report_path.read_text()) + validate_report(report, scenario) + assert process.returncode == 0, f"worker rc={process.returncode}; evidence={evidence}" + # Remove only our generated DB after the process has exited and evidence has + # been collected. Failed cases retain their DB for investigation. + shutil.rmtree(root) diff --git a/tests/unit_tests/test_embedded_fulltext_harness.py b/tests/unit_tests/test_embedded_fulltext_harness.py new file mode 100644 index 00000000..8a29fcfc --- /dev/null +++ b/tests/unit_tests/test_embedded_fulltext_harness.py @@ -0,0 +1,132 @@ +"""Contracts for the regression harness; these do not require a database.""" + +import importlib.util +import json +from pathlib import Path + +import pytest + +spec = importlib.util.spec_from_file_location( + "fts_support", Path(__file__).parents[1] / "integration_tests/embedded_fulltext_support.py" +) +fts = importlib.util.module_from_spec(spec) +spec.loader.exec_module(fts) + + +def report(scenario): + return { + "scenario": scenario, + "completed": True, + "errors": 0, + "rows": fts.ROWS, + "first_pass_successes": fts.QUERIES, + "fts_successes": fts.QUERIES * fts.PASSES, + "worker_successes": [1] * fts.WORKERS, + "freeze_successes": 1, + "elapsed_seconds": fts.SECONDS, + } + + +@pytest.mark.parametrize("scenario", ["first-pass", "scan-freeze"]) +def test_valid_report(scenario): + fts.validate_report(report(scenario), scenario) + + +@pytest.mark.parametrize( + "scenario,field,value", + [ + ("first-pass", "first_pass_successes", 299), + ("first-pass", "fts_successes", 1199), + ("first-pass", "errors", 1), + ("first-pass", "completed", False), + ("scan-freeze", "errors", 1), + ("scan-freeze", "worker_successes", [0] * 8), + ("scan-freeze", "worker_successes", [1] * 7), + ("scan-freeze", "freeze_successes", 0), + ("scan-freeze", "elapsed_seconds", 74), + ("scan-freeze", "rows", 0), + ], +) +def test_failed_or_incomplete_work_cannot_pass(scenario, field, value): + value_report = report(scenario) + value_report[field] = value + with pytest.raises(AssertionError): + fts.validate_report(value_report, scenario) + + +def test_wrapped_numeric_error_is_not_inferred_from_prose(): + error = RuntimeError("Failed to execute query") + error.__cause__ = RuntimeError("execute failed: code=4016") + assert fts.error_details(error)["error_codes"] == [4016] + assert fts.error_details(RuntimeError("internal error"))["error_codes"] == [] + + +def test_large_rotated_log_uses_total_tail_budget(tmp_path): + log = tmp_path / "seekdb.log.20260916" + with log.open("wb") as stream: + stream.seek(300 * 1024 * 1024) + stream.write(b"\nread_barrier_:true ret=-4016\n") + evidence = tmp_path / "evidence" + evidence.mkdir() + fts.capture_logs(tmp_path, evidence, "early", budget=4096, output_limit=1024) + assert "ret=-4016" in (evidence / "early-db-context.log").read_text() + inventory = json.loads((evidence / "early-log-inventory.json").read_text()) + assert inventory["files"][0]["read_bytes"] == 4096 + assert inventory["files"][0]["size"] > 256 * 1024 * 1024 + assert inventory["scan_capped"] + assert (evidence / "early-db-context.log").stat().st_size <= 1024 + + +def test_first_failure_is_captured_immediately_and_never_reset(tmp_path): + evidence = tmp_path / "evidence" + evidence.mkdir() + (tmp_path / "seekdb.log").write_text("ret=-4016 data/schema type does not match\n") + recorder = fts.Recorder(tmp_path, evidence) + recorder.error(RuntimeError("code=4016"), "scan") + assert (evidence / "first-error.json").exists() + assert (evidence / "early-db-context.log").exists() + assert not (evidence / "summary.json").exists() # before the end of the workload + recorder.error(RuntimeError("code=4016"), "scan") + assert recorder.errors == 2 + assert recorder.first["error_codes"] == [4016] + + +def test_scan_freeze_uses_separate_connections_and_no_vector_index(tmp_path, monkeypatch): + # Shorten only this harness unit test, not the real integration defaults. + for key, value in (("ROWS", 20), ("WORKERS", 2), ("SECONDS", 0.03), ("INTERVAL", 0.005)): + monkeypatch.setattr(fts, key, value) + clients = [] + + class Client: + def __init__(self): + self._server = self + self.commands = [] + self.closed = False + + def _execute(self, sql): + self.commands.append(sql) + if sql.startswith("SHOW CREATE"): + return [("fixture", "CREATE TABLE fixture (... FULLTEXT ...)")] + if sql.startswith("SELECT COUNT"): + return [(20,)] + return [("doc-1", 1.0)] + + def close(self): + self.closed = True + + def make_client(): + client = Client() + clients.append(client) + return client + + evidence = tmp_path / "evidence" + evidence.mkdir() + recorder = fts.Recorder(tmp_path, evidence) + value_report = {"scenario": "scan-freeze", "completed": True, "errors": 0} + setup = Client() + fts.scan_freeze(setup, make_client, recorder, value_report) + fts.validate_report(value_report, "scan-freeze") + assert recorder.errors == 0 + assert len(clients) == 3 and all(client.closed for client in clients) + assert "VECTOR" not in setup.commands[0].upper() + assert sum("ALTER SYSTEM MINOR FREEZE" in client.commands for client in clients) == 1 From d85e947b31e7b33ce98e5ab176be00bd23e6cb7a Mon Sep 17 00:00:00 2001 From: cms-cms <1303573631@qq.com> Date: Thu, 17 Sep 2026 17:19:32 +0800 Subject: [PATCH 02/25] test: prioritize failure traces and stabilize fake scan scheduling --- .../embedded_fulltext_support.py | 98 ++++++++++++------- .../test_embedded_fulltext_harness.py | 51 ++++++++++ 2 files changed, 115 insertions(+), 34 deletions(-) diff --git a/tests/integration_tests/embedded_fulltext_support.py b/tests/integration_tests/embedded_fulltext_support.py index 76e8887a..0350c563 100644 --- a/tests/integration_tests/embedded_fulltext_support.py +++ b/tests/integration_tests/embedded_fulltext_support.py @@ -57,10 +57,17 @@ def error_details(error): def capture_logs(root, evidence, label, traces=(), budget=128 * 1024 * 1024, output_limit=2 * 1024 * 1024): - """Read bounded tails by total bytes, never discard a large rotated file.""" - patterns = [value.encode() for value in traces] + [ - b"ret=-4016", - b"read_barrier_", + """Scan once under a byte budget; prioritize errors over routine context. + + Each priority buffer is bounded by output_limit (at most 4x that limit in + memory). A full low-priority buffer must not stop the scan: the failing + trace can occur later in a large, noisy rotated log. + """ + if budget < 0 or output_limit <= 0: + raise ValueError("invalid diagnostic byte budget") + trace_patterns = [value.encode() for value in traces if value] + error_patterns = [ + b"-4016", b"data/schema type does not match", b"failed to get next row from memtable scanner", b"ob_memtable_key.h", @@ -70,43 +77,66 @@ def capture_logs(root, evidence, label, traces=(), budget=128 * 1024 * 1024, out key=lambda p: p.stat().st_mtime, reverse=True, ) - inventory, written = [], 0 + inventory = [] + priorities = ("trace-error", "trace", "error", "memtable-context") + buffers = [bytearray() for _ in priorities] + matched = [0] * len(priorities) + capped = [False] * len(priorities) + + def append(priority, content): + available = output_limit - len(buffers[priority]) + if len(content) > available: + capped[priority] = True + buffers[priority].extend(content[:available]) + + for path in files: + size = path.stat().st_size + count = min(size, budget) + inventory.append({"path": str(path), "size": size, "read_bytes": count}) + if not count: + continue + with path.open("rb") as stream: + stream.seek(size - count) + previous, following, remaining = collections.deque(maxlen=5), 0, count + context_priority = 3 + while remaining > 0: + line = stream.readline(min(65536, remaining)) + if not line: + break + remaining -= len(line) + is_trace = any(pattern in line for pattern in trace_patterns) + is_error = any(pattern in line for pattern in error_patterns) + priority = ( + 0 + if is_trace and is_error + else 1 if is_trace else 2 if is_error else 3 if b"read_barrier_:true" in line else None + ) + if priority is not None: + # Preserve the matching line before optional context so + # a huge preceding SQL line cannot consume the budget. + context = f"\nFILE={path}\n".encode() + line + b"[preceding context]\n" + b"".join(previous) + matched[priority] += 1 + append(priority, context) + following, context_priority = 5, priority + elif following: + append(context_priority, line) + following -= 1 + previous.append(line) + budget -= count + written = 0 with (evidence / f"{label}-db-context.log").open("wb") as output: - for path in files: - size = path.stat().st_size - count = min(size, budget) - inventory.append({"path": str(path), "size": size, "read_bytes": count}) - if not count: - continue - with path.open("rb") as stream: - stream.seek(size - count) - previous, following, remaining = collections.deque(maxlen=5), 0, count - while remaining > 0: - line = stream.readline(min(65536, remaining)) - if not line: - break - remaining -= len(line) - if any(pattern in line for pattern in patterns): - # Preserve the matching line before optional context so - # a huge preceding SQL line cannot consume the budget. - context = f"\nFILE={path}\n".encode() + line + b"[preceding context]\n" + b"".join(previous) - part = context[: max(0, output_limit - written)] - output.write(part) - written += len(part) - following = 6 - if following: - part = line[: max(0, output_limit - written)] - output.write(part) - written += len(part) - following -= 1 - previous.append(line) - budget -= count + for content in buffers: + part = content[: max(0, output_limit - written)] + output.write(part) + written += len(part) (evidence / f"{label}-log-inventory.json").write_text( json.dumps( { "files": inventory, "context_bytes": written, "context_capped": written >= output_limit, + "priority_matches": dict(zip(priorities, matched, strict=True)), + "priority_capped": dict(zip(priorities, capped, strict=True)), "scan_capped": any(item["read_bytes"] < item["size"] for item in inventory), }, indent=2, diff --git a/tests/unit_tests/test_embedded_fulltext_harness.py b/tests/unit_tests/test_embedded_fulltext_harness.py index 8a29fcfc..9d7e5e5c 100644 --- a/tests/unit_tests/test_embedded_fulltext_harness.py +++ b/tests/unit_tests/test_embedded_fulltext_harness.py @@ -2,7 +2,10 @@ import importlib.util import json +import threading +import time from pathlib import Path +from types import SimpleNamespace import pytest @@ -91,11 +94,57 @@ def test_first_failure_is_captured_immediately_and_never_reset(tmp_path): assert recorder.first["error_codes"] == [4016] +def test_early_trace_error_wins_over_noise_and_other_errors(tmp_path): + trace = "YB427F000001-00065BA8AA327DFF-0-0" + # All lower-priority buffers fill before the failing trace is encountered. + noise = ( + "read_barrier_:false ordinary INFO\n" * 200 + + "read_barrier_:true release_head_memtable_\n" * 200 + + "ret=-4016 unrelated session\n" * 200 + + f"[{trace}] normal statement context\n" * 200 + ) + failure = f"[{trace}] send_error_packet ob_error=-4016\n" + (tmp_path / "seekdb.log").write_text(noise + failure) + evidence = tmp_path / "evidence" + evidence.mkdir() + recorder = fts.Recorder(tmp_path, evidence) + # Exercise the real immediate-error entry point, not just final collection. + original = fts.capture_logs + from unittest.mock import patch + + with patch.object(fts, "capture_logs", side_effect=lambda *args: original(*args, output_limit=1024)): + recorder.error(RuntimeError(f"code=4016 [{trace}]"), "scan") + context = (evidence / "early-db-context.log").read_text() + assert failure in context + assert context.index(failure) < context.index("[preceding context]") + assert not (evidence / "summary.json").exists() + inventory = json.loads((evidence / "early-log-inventory.json").read_text()) + assert inventory["priority_matches"]["trace-error"] == 1 + assert inventory["priority_capped"]["memtable-context"] + assert inventory["context_bytes"] <= 1024 + + +def test_false_read_barrier_alone_does_not_consume_output(tmp_path): + (tmp_path / "seekdb.log").write_text("read_barrier_:false ordinary INFO\n" * 100) + evidence = tmp_path / "evidence" + evidence.mkdir() + fts.capture_logs(tmp_path, evidence, "early", output_limit=128) + assert (evidence / "early-db-context.log").read_bytes() == b"" + + def test_scan_freeze_uses_separate_connections_and_no_vector_index(tmp_path, monkeypatch): # Shorten only this harness unit test, not the real integration defaults. for key, value in (("ROWS", 20), ("WORKERS", 2), ("SECONDS", 0.03), ("INTERVAL", 0.005)): monkeypatch.setattr(fts, key, value) clients = [] + # Fake queries do no I/O and can monopolize the GIL for the entire 30ms + # window. Model one completed round deterministically instead of depending + # on OS scheduling. This clock is local to the fake-client unit test only. + first_round = threading.Event() + round_barrier = threading.Barrier(fts.WORKERS + 1, action=first_round.set) + monkeypatch.setattr( + fts, "time", SimpleNamespace(time=time.time, monotonic=lambda: fts.SECONDS if first_round.is_set() else 0.0) + ) class Client: def __init__(self): @@ -105,6 +154,8 @@ def __init__(self): def _execute(self, sql): self.commands.append(sql) + if sql == fts.SEARCH or sql == "ALTER SYSTEM MINOR FREEZE": + round_barrier.wait(timeout=5) if sql.startswith("SHOW CREATE"): return [("fixture", "CREATE TABLE fixture (... FULLTEXT ...)")] if sql.startswith("SELECT COUNT"): From 4a5d0d0c4f2f2c0d5fb37daa934a38097ff5165b Mon Sep 17 00:00:00 2001 From: cms-cms <1303573631@qq.com> Date: Thu, 17 Sep 2026 21:03:21 +0800 Subject: [PATCH 03/25] ci: verify embedded regression with a pinned fixed source wheel --- .github/embedded-source.json | 7 ++ .github/scripts/build-embedded-wheel.sh | 82 ++++++++++++++++ .github/scripts/embedded_wheel_manifest.py | 73 +++++++++++++++ .github/workflows/ci.yml | 76 ++++++++++++++- .gitignore | 1 + tests/integration_tests/EMBEDDED_FULLTEXT.md | 24 +++++ .../embedded_fulltext_support.py | 11 ++- .../test_embedded_fulltext_harness.py | 7 ++ .../test_embedded_wheel_manifest.py | 93 +++++++++++++++++++ 9 files changed, 370 insertions(+), 4 deletions(-) create mode 100644 .github/embedded-source.json create mode 100644 .github/scripts/build-embedded-wheel.sh create mode 100644 .github/scripts/embedded_wheel_manifest.py create mode 100644 tests/unit_tests/test_embedded_wheel_manifest.py diff --git a/.github/embedded-source.json b/.github/embedded-source.json new file mode 100644 index 00000000..a3f33a1c --- /dev/null +++ b/.github/embedded-source.json @@ -0,0 +1,7 @@ +{ + "seekdb_sha": "8a8d6f5a8dd48cdbf859305b3f7eb050149a5bd1", + "bindings_sha": "5e8eece63205dccfa9500f98893c91d1f50eb4f1", + "fix_commit": "48b80f5193bcac0afafba4d4820aebdf1103bc69", + "platform": "linux-x86_64", + "python": "3.11" +} diff --git a/.github/scripts/build-embedded-wheel.sh b/.github/scripts/build-embedded-wheel.sh new file mode 100644 index 00000000..def7d376 --- /dev/null +++ b/.github/scripts/build-embedded-wheel.sh @@ -0,0 +1,82 @@ +#!/usr/bin/env bash +# CI-local Linux wheel: not a portable manylinux release or a PyPI publication. +set -Eeuo pipefail +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +: "${RUNNER_TEMP:?use an isolated GitHub runner temporary directory}" +: "${EMBEDDED_WHEEL_DIR:?wheel output/cache directory is required}" +[[ "$(uname -s)/$(uname -m)" == Linux/x86_64 ]] +PYTHON="$(command -v python3)" +"$PYTHON" -c 'import sys; assert sys.version_info[:2] == (3, 11)' +read -r SEEKDB_SHA BINDINGS_SHA FIX_COMMIT < <( + "$PYTHON" -c 'import json, sys; p=json.load(open(sys.argv[1])); print(p["seekdb_sha"], p["bindings_sha"], p["fix_commit"])' \ + "$REPO_ROOT/.github/embedded-source.json" +) +[[ "$SEEKDB_SHA" =~ ^[0-9a-f]{40}$ && "$BINDINGS_SHA" =~ ^[0-9a-f]{40}$ && "$FIX_COMMIT" =~ ^[0-9a-f]{40}$ ]] +BUILD_ROOT="$(mktemp -d "$RUNNER_TEMP/embedded-source.XXXXXX")" +mkdir -p "$EMBEDDED_WHEEL_DIR" +[[ -z "$(ls -A "$EMBEDDED_WHEEL_DIR")" ]] || { echo 'refusing to overwrite wheel evidence' >&2; exit 1; } + +checkout_exact() { + local repository="$1" revision="$2" target="$3" + git init -q "$target" + git -C "$target" remote add origin "$repository" + git -C "$target" config remote.origin.promisor true + git -C "$target" config remote.origin.partialclonefilter blob:none + # Full ancestry is needed to prove the kernel contains the required fix. + # Historical source blobs are not: fetch them lazily for the exact checkout. + git -C "$target" fetch --filter=blob:none --no-tags origin "$revision" + git -C "$target" checkout --detach FETCH_HEAD + [[ "$(git -C "$target" rev-parse HEAD)" == "$revision" ]] +} +checkout_exact https://github.com/oceanbase/seekdb.git "$SEEKDB_SHA" "$BUILD_ROOT/seekdb" +git -C "$BUILD_ROOT/seekdb" merge-base --is-ancestor "$FIX_COMMIT" HEAD +checkout_exact https://github.com/oceanbase/seekdb-bindings.git "$BINDINGS_SHA" "$BUILD_ROOT/bindings" +git -C "$BUILD_ROOT/bindings" submodule update --init --depth=1 deps/mariadb-connector-c + +# Match the upstream seekdb-bindings Ubuntu source-build recipe. These are +# packaging/client tools, not dependencies of the seekdb compilation target. +DEP_PROFILE="$BUILD_ROOT/seekdb/deps/init/oceanbase.el9.x86_64.deps" +sed -i -E '/^[^#]*(target=(obshell|obdeploy)|(lib)?obclient-)/d' "$DEP_PROFILE" + +# Install the source's exact Rust toolchain once, before parallel compilation; +# never let two Rust targets race while auto-installing the same components. +RUST_TOOLCHAIN="$("$PYTHON" -c 'import sys,tomllib; print(tomllib.load(open(sys.argv[1],"rb"))["toolchain"]["channel"])' \ + "$BUILD_ROOT/seekdb/rust/rust-toolchain.toml")" +[[ "$RUST_TOOLCHAIN" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] +export RUSTUP_HOME="$BUILD_ROOT/rustup" CARGO_HOME="$BUILD_ROOT/cargo" +rustup toolchain install "$RUST_TOOLCHAIN" --profile minimal --no-self-update +export RUSTUP_TOOLCHAIN="$RUST_TOOLCHAIN" RUSTUP_AUTO_INSTALL=0 +rustup run "$RUST_TOOLCHAIN" cargo --version +BUILD_JOBS="$(nproc)" +(( BUILD_JOBS <= 3 )) || BUILD_JOBS=3 +MEMORY_JOBS="$(awk '/MemTotal:/ {n=int(($2/1024-3072)/4096); print (n>0?n:1)}' /proc/meminfo)" +(( BUILD_JOBS <= MEMORY_JOBS )) || BUILD_JOBS="$MEMORY_JOBS" +export CMAKE_BUILD_PARALLEL_LEVEL="$BUILD_JOBS" CARGO_BUILD_JOBS="$BUILD_JOBS" +( + cd "$BUILD_ROOT/seekdb" + ./build.sh release --init -DBUILD_EMBED_MODE=ON --make -j"$BUILD_JOBS" +) +mapfile -t binaries < <(find "$BUILD_ROOT/seekdb/build_release" -type f -name seekdb -executable) +[[ "${#binaries[@]}" == 1 ]] +SEEKDB_BIN="${binaries[0]}" +strip --strip-debug --strip-unneeded "$SEEKDB_BIN" +"$SEEKDB_BIN" -V 2>&1 | tee "$EMBEDDED_WHEEL_DIR/seekdb-version.txt" +grep -F "$SEEKDB_SHA" "$EMBEDDED_WHEEL_DIR/seekdb-version.txt" + +# Do not call the upstream wheel target (it selects an unqualified `python`). +# Build only the driver, then build a CPython 3.11 wheel with this exact venv. +"$PYTHON" -m venv "$BUILD_ROOT/venv" +BUILD_PYTHON="$BUILD_ROOT/venv/bin/python" +"$BUILD_PYTHON" -m pip install --disable-pip-version-check \ + 'pip==25.3' 'build==1.3.0' 'cmake==3.31.6' 'nanobind==3.0.0' 'scikit-build-core==0.11.6' 'ninja==1.11.1.4' +export PATH="$BUILD_ROOT/venv/bin:$PATH" +cmake -S "$BUILD_ROOT/bindings" -B "$BUILD_ROOT/bindings/build" \ + -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTING=OFF -DSEEKDB_BUILD_PYTHON=OFF \ + -DSEEKDB_BIN="$SEEKDB_BIN" +cmake --build "$BUILD_ROOT/bindings/build" --target seekdb --parallel "$BUILD_JOBS" +"$BUILD_PYTHON" -m pip wheel --no-deps --no-build-isolation \ + --config-settings="cmake.define.Python_EXECUTABLE=$BUILD_PYTHON" \ + --config-settings="cmake.define.Python_FIND_VIRTUALENV=ONLY" \ + --wheel-dir "$EMBEDDED_WHEEL_DIR" "$BUILD_ROOT/bindings/python" +"$PYTHON" "$REPO_ROOT/.github/scripts/embedded_wheel_manifest.py" create \ + "$REPO_ROOT/.github/embedded-source.json" "$EMBEDDED_WHEEL_DIR" "$SEEKDB_BIN" diff --git a/.github/scripts/embedded_wheel_manifest.py b/.github/scripts/embedded_wheel_manifest.py new file mode 100644 index 00000000..39a38854 --- /dev/null +++ b/.github/scripts/embedded_wheel_manifest.py @@ -0,0 +1,73 @@ +"""Bind a CI-local wheel to pinned sources and verify the installed payload.""" + +import hashlib +import importlib.metadata +import json +import platform +import subprocess +import sys +import zipfile +from pathlib import Path + + +def digest(path): + with path.open("rb") as stream: + return hashlib.file_digest(stream, "sha256").hexdigest() + + +def wheel_payload(wheel): + with zipfile.ZipFile(wheel) as archive, archive.open("pylibseekdb/seekdb") as stream: + return hashlib.file_digest(stream, "sha256").hexdigest() + + +def verify(pin, directory): + manifest = json.loads((directory / "manifest.json").read_text()) + if manifest["source"] != pin: + raise ValueError("cached wheel source identity mismatch") + if platform.system() != "Linux" or platform.machine() != "x86_64" or sys.version_info[:2] != (3, 11): + raise ValueError("wheel requires Linux x86_64 CPython 3.11") + wheels = list(directory.glob("*.whl")) + if len(wheels) != 1 or wheels[0].name != manifest["wheel"]: + raise ValueError("expected exactly the manifest wheel") + if digest(wheels[0]) != manifest["wheel_sha256"]: + raise ValueError("wheel checksum mismatch") + if wheel_payload(wheels[0]) != manifest["binary_sha256"]: + raise ValueError("wheel contains a different SeekDB binary") + version = (directory / "seekdb-version.txt").read_text() + if pin["seekdb_sha"] not in version or version != manifest["revision_output"]: + raise ValueError("SeekDB full REVISION mismatch") + return manifest + + +def main(): + mode, pin_path, directory_name, *rest = sys.argv[1:] + pin = json.loads(Path(pin_path).read_text()) + directory = Path(directory_name) + if mode == "create": + (binary_name,) = rest + (wheel,) = directory.glob("*.whl") + manifest = { + "source": pin, + "wheel": wheel.name, + "wheel_sha256": digest(wheel), + "binary_sha256": digest(Path(binary_name)), + "revision_output": (directory / "seekdb-version.txt").read_text(), + } + (directory / "manifest.json").write_text(json.dumps(manifest, indent=2) + "\n") + elif mode not in {"verify", "installed"}: + raise ValueError(f"unknown verification mode: {mode}") + manifest = verify(pin, directory) + if mode == "installed": + binary = Path(importlib.metadata.distribution("pylibseekdb").locate_file("pylibseekdb/seekdb")) + if digest(binary) != manifest["binary_sha256"]: + raise ValueError("installed SeekDB differs from source-built wheel (dependency resync?)") + # The executable is the verified local wheel payload, never shell input. + version = subprocess.check_output([str(binary), "-V"], text=True) # noqa: S603 + if pin["seekdb_sha"] not in version: + raise ValueError("installed SeekDB does not report pinned full REVISION") + print(f"[EMBEDDED_WHEEL] installed={binary}") + print(json.dumps(manifest, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7fc585d3..a359699b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -100,7 +100,11 @@ jobs: bash .github/scripts/check-pytest-summary.sh pytest.log integration-test: - runs-on: ubuntu-latest + # Keep the CI-local source wheel on the same native Ubuntu image as its tests. + runs-on: ${{ matrix.test_mode == 'embedded' && 'ubuntu-22.04' || 'ubuntu-latest' }} + timeout-minutes: 180 + env: + EMBEDDED_WHEEL_DIR: ${{ github.workspace }}/.embedded-wheel strategy: fail-fast: false # Allow other jobs to continue if one fails matrix: @@ -137,6 +141,54 @@ jobs: with: python-version: '3.11' + - name: Restore exact embedded source wheel + if: matrix.test_mode == 'embedded' + id: embedded-wheel + uses: actions/cache/restore@v4 + with: + path: .embedded-wheel + # Deliberately no restore-keys: another source/build recipe is not reusable. + key: embedded-source-ubuntu22-x86_64-cp311-${{ hashFiles('.github/embedded-source.json', '.github/scripts/build-embedded-wheel.sh', '.github/scripts/embedded_wheel_manifest.py') }} + + - name: Install embedded source build dependencies + if: matrix.test_mode == 'embedded' && steps.embedded-wheel.outputs.cache-hit != 'true' + run: | + sudo apt-get update + sudo apt-get install -y git wget rpm rpm2cpio cpio make build-essential binutils m4 libtool-bin libncurses5 libaio1 libssl-dev zlib1g-dev + + - name: Build embedded wheel from pinned fixed sources + if: matrix.test_mode == 'embedded' && steps.embedded-wheel.outputs.cache-hit != 'true' + run: bash .github/scripts/build-embedded-wheel.sh + + - name: Verify and install the exact embedded source wheel + if: matrix.test_mode == 'embedded' + run: | + set -euo pipefail + # Needed even when the build cache was restored on a fresh runner. + sudo apt-get update + sudo apt-get install -y libaio1 + python .github/scripts/embedded_wheel_manifest.py verify .github/embedded-source.json "$EMBEDDED_WHEEL_DIR" + uv pip install --python .venv/bin/python --no-deps --reinstall "$EMBEDDED_WHEEL_DIR"/*.whl + .venv/bin/python .github/scripts/embedded_wheel_manifest.py installed .github/embedded-source.json "$EMBEDDED_WHEEL_DIR" + + - name: Save verified embedded source wheel + if: matrix.test_mode == 'embedded' && steps.embedded-wheel.outputs.cache-hit != 'true' + uses: actions/cache/save@v4 + with: + path: .embedded-wheel + key: ${{ steps.embedded-wheel.outputs.cache-primary-key }} + + - name: Record embedded wheel provenance + if: matrix.test_mode == 'embedded' + uses: actions/upload-artifact@v4 + with: + name: embedded-wheel-provenance + include-hidden-files: true + path: | + .embedded-wheel/manifest.json + .embedded-wheel/seekdb-version.txt + if-no-files-found: error + - name: Start OceanBase container if: matrix.test_mode == 'oceanbase' uses: oceanbase/setup-oceanbase-ce@v1 @@ -171,10 +223,28 @@ jobs: log_level=INFO fi set -o pipefail - uv run pytest tests/integration_tests/ -v --log-cli-level=${log_level} \ + # Do not let uv replace the verified source wheel with uv.lock's release. + uv run --no-sync pytest tests/integration_tests/ -v --log-cli-level=${log_level} \ -k "${{ matrix.test_mode }}" | tee pytest.log bash .github/scripts/check-pytest-summary.sh pytest.log + - name: Preserve embedded full-text failure evidence + if: failure() && matrix.test_mode == 'embedded' + uses: actions/upload-artifact@v4 + with: + name: embedded-fulltext-failure-${{ github.run_id }}-${{ github.run_attempt }} + # Only harness reports and bounded log excerpts, never database files. + include-hidden-files: true + path: | + .seekdb-test-data/artifacts/fts-*/summary.json + .seekdb-test-data/artifacts/fts-*/first-error.json + .seekdb-test-data/artifacts/fts-*/errors.jsonl + .seekdb-test-data/artifacts/fts-*/*-db-context.log + .seekdb-test-data/artifacts/fts-*/*-log-inventory.json + .seekdb-test-data/artifacts/fts-*/worker.log + if-no-files-found: warn + retention-days: 14 + - name: Run examples for ${{ matrix.test_mode }} env: MODE: ${{ matrix.test_mode }} @@ -191,7 +261,7 @@ jobs: fi echo "Testing ${example_file} in ${MODE} mode" - if uv run python "$example_file"; then + if uv run --no-sync python "$example_file"; then echo "${example_file} passed" else status=$? diff --git a/.gitignore b/.gitignore index 3c774da3..19c58691 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,4 @@ ENV/ env.bak/ venv.bak/ demo/rag/data/ +.embedded-wheel/ diff --git a/tests/integration_tests/EMBEDDED_FULLTEXT.md b/tests/integration_tests/EMBEDDED_FULLTEXT.md index 5bd18287..247626db 100644 --- a/tests/integration_tests/EMBEDDED_FULLTEXT.md +++ b/tests/integration_tests/EMBEDDED_FULLTEXT.md @@ -48,3 +48,27 @@ For pre/post-fix comparisons, hold this test SHA and bindings source SHA fixed, record the actual SeekDB REVISION inside each wheel, and change only the kernel source revision. Old-build failures must retain a failed result. Do not add an xfail or suppress 4016 to make the comparison green. + +## GitHub PR gate and released wheels + +The GitHub embedded job builds a **CI-local Linux x86_64 / CPython 3.11** wheel +from the full SeekDB and bindings commits in `.github/embedded-source.json`. +The pinned SeekDB commit contains the #1384 fix; the build verifies that ancestry. +This wheel is not published and does not claim portable manylinux compatibility. +The kernel build uses at most three workers, further limited by available RAM. + +The cache key includes both source pins, the build/verification scripts, Ubuntu +22.04 and CPython 3.11. No approximate cache key is accepted. On every restore, +CI verifies the manifest, wheel and embedded binary SHA-256 and full REVISION. +After installation it verifies the actual installed binary again. Integration +tests and examples use `uv run --no-sync` so the lock file cannot silently +replace the tested wheel with the old PyPI release. All embedded tests still run; +neither these two regressions nor database errors are skipped or marked xfail. + +The lock file remains unchanged for the other jobs. This source-built gate does +**not** prove that the published wheel is fixed: run 35204502666 used the locked +`pylibseekdb 1.3.0` and failed both new scenarios with explicit 4016 errors (one +first-pass error and 72 scan errors, whose first error was 4016). A released wheel +containing #1384 can replace the temporary source-build path after verification. +Failed full-text reports and bounded database-log excerpts are uploaded as +artifacts; whole database directories are not uploaded. diff --git a/tests/integration_tests/embedded_fulltext_support.py b/tests/integration_tests/embedded_fulltext_support.py index 0350c563..4e94c609 100644 --- a/tests/integration_tests/embedded_fulltext_support.py +++ b/tests/integration_tests/embedded_fulltext_support.py @@ -51,6 +51,9 @@ def error_details(error): if error.args and isinstance(error.args[0], int): codes.append(error.args[0]) codes.extend(int(value) for value in re.findall(r"\bcode[=:]\s*(-?\d+)\b", str(error))) + # Released bindings may expose a RuntimeError rather than numeric args, + # e.g. "execute sql failed OB_ERR_UNEXPECTED(4016): %s". + codes.extend(int(value) for value in re.findall(r"\bOB_[A-Z0-9_]+\((-?\d+)\)", str(error))) error = error.__cause__ or error.__context__ text = "\n".join(chain) return {"exception_chain": chain, "error_codes": sorted(set(codes)), "trace_ids": sorted(set(TRACE.findall(text)))} @@ -109,7 +112,13 @@ def append(priority, content): priority = ( 0 if is_trace and is_error - else 1 if is_trace else 2 if is_error else 3 if b"read_barrier_:true" in line else None + else 1 + if is_trace + else 2 + if is_error + else 3 + if b"read_barrier_:true" in line + else None ) if priority is not None: # Preserve the matching line before optional context so diff --git a/tests/unit_tests/test_embedded_fulltext_harness.py b/tests/unit_tests/test_embedded_fulltext_harness.py index 9d7e5e5c..7d2db014 100644 --- a/tests/unit_tests/test_embedded_fulltext_harness.py +++ b/tests/unit_tests/test_embedded_fulltext_harness.py @@ -64,6 +64,13 @@ def test_wrapped_numeric_error_is_not_inferred_from_prose(): assert fts.error_details(RuntimeError("internal error"))["error_codes"] == [] +def test_released_binding_symbolic_error_preserves_numeric_code(): + error = RuntimeError("Failed to execute query") + error.__cause__ = RuntimeError("execute sql failed OB_ERR_UNEXPECTED(4016): %s") + assert fts.error_details(error)["error_codes"] == [4016] + assert fts.error_details(RuntimeError("query ordinal4016 returned no rows"))["error_codes"] == [] + + def test_large_rotated_log_uses_total_tail_budget(tmp_path): log = tmp_path / "seekdb.log.20260916" with log.open("wb") as stream: diff --git a/tests/unit_tests/test_embedded_wheel_manifest.py b/tests/unit_tests/test_embedded_wheel_manifest.py new file mode 100644 index 00000000..929fb5db --- /dev/null +++ b/tests/unit_tests/test_embedded_wheel_manifest.py @@ -0,0 +1,93 @@ +"""The CI source wheel must not silently fall back to a released binary.""" + +import importlib.util +import json +import zipfile +from pathlib import Path +from types import SimpleNamespace + +import pytest + +spec = importlib.util.spec_from_file_location( + "wheel_manifest", Path(__file__).parents[2] / ".github/scripts/embedded_wheel_manifest.py" +) +wheel_manifest = importlib.util.module_from_spec(spec) +spec.loader.exec_module(wheel_manifest) + + +@pytest.fixture +def bundle(tmp_path, monkeypatch): + pin = {"seekdb_sha": "a" * 40, "bindings_sha": "b" * 40, "fix_commit": "c" * 40} + pin_path = tmp_path / "source.json" + pin_path.write_text(json.dumps(pin)) + binary = tmp_path / "seekdb" + binary.write_bytes(b"the fixed binary") + wheel = tmp_path / "pylibseekdb-0.1.0-cp311-cp311-linux_x86_64.whl" + with zipfile.ZipFile(wheel, "w") as archive: + archive.writestr("pylibseekdb/seekdb", binary.read_bytes()) + (tmp_path / "seekdb-version.txt").write_text(f"REVISION: {pin['seekdb_sha']}\n") + monkeypatch.setattr(wheel_manifest.platform, "system", lambda: "Linux") + monkeypatch.setattr(wheel_manifest.platform, "machine", lambda: "x86_64") + monkeypatch.setattr( + wheel_manifest, + "sys", + SimpleNamespace(version_info=(3, 11), argv=["manifest", "create", str(pin_path), str(tmp_path), str(binary)]), + ) + wheel_manifest.main() + return pin, tmp_path, binary, wheel + + +def test_exact_wheel_and_revision_pass(bundle): + pin, directory, binary, _ = bundle + assert wheel_manifest.verify(pin, directory)["binary_sha256"] == wheel_manifest.digest(binary) + + +def test_wrong_source_cannot_reuse_cache(bundle): + pin, directory, _, _ = bundle + with pytest.raises(ValueError, match="source identity"): + wheel_manifest.verify({**pin, "seekdb_sha": "d" * 40}, directory) + + +def test_corrupt_wheel_is_rejected(bundle): + pin, directory, _, wheel = bundle + with wheel.open("ab") as stream: + stream.write(b"corruption") + with pytest.raises(ValueError, match="checksum"): + wheel_manifest.verify(pin, directory) + + +def test_replaced_binary_is_rejected_even_with_updated_wheel_hash(bundle): + pin, directory, _, wheel = bundle + with zipfile.ZipFile(wheel, "w") as archive: + archive.writestr("pylibseekdb/seekdb", b"old released binary") + manifest_path = directory / "manifest.json" + manifest = json.loads(manifest_path.read_text()) + manifest["wheel_sha256"] = wheel_manifest.digest(wheel) + manifest_path.write_text(json.dumps(manifest)) + with pytest.raises(ValueError, match="different SeekDB"): + wheel_manifest.verify(pin, directory) + + +def test_short_revision_is_not_enough(bundle): + pin, directory, _, _ = bundle + (directory / "seekdb-version.txt").write_text(f"REVISION: {pin['seekdb_sha'][:8]}") + with pytest.raises(ValueError, match="full REVISION"): + wheel_manifest.verify(pin, directory) + + +def test_dependency_resync_is_detected(bundle, monkeypatch): + _, directory, binary, _ = bundle + binary.write_bytes(b"old released binary installed by uv sync") + monkeypatch.setattr( + wheel_manifest.importlib.metadata, "distribution", lambda _: SimpleNamespace(locate_file=lambda _: binary) + ) + wheel_manifest.sys.argv = ["manifest", "installed", str(directory / "source.json"), str(directory)] + with pytest.raises(ValueError, match="dependency resync"): + wheel_manifest.main() + + +def test_incompatible_host_is_rejected(bundle, monkeypatch): + pin, directory, _, _ = bundle + monkeypatch.setattr(wheel_manifest.platform, "machine", lambda: "aarch64") + with pytest.raises(ValueError, match="Linux x86_64"): + wheel_manifest.verify(pin, directory) From 0b8cbff5e8d52e9d0f8ec8f80ca13230c4f72f8f Mon Sep 17 00:00:00 2001 From: cms-cms <1303573631@qq.com> Date: Fri, 18 Sep 2026 10:18:32 +0800 Subject: [PATCH 04/25] ci: capture SeekDB revision from stderr --- .github/scripts/embedded_wheel_manifest.py | 4 ++- .../test_embedded_wheel_manifest.py | 32 +++++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/.github/scripts/embedded_wheel_manifest.py b/.github/scripts/embedded_wheel_manifest.py index 39a38854..07c1c38b 100644 --- a/.github/scripts/embedded_wheel_manifest.py +++ b/.github/scripts/embedded_wheel_manifest.py @@ -62,7 +62,9 @@ def main(): if digest(binary) != manifest["binary_sha256"]: raise ValueError("installed SeekDB differs from source-built wheel (dependency resync?)") # The executable is the verified local wheel payload, never shell input. - version = subprocess.check_output([str(binary), "-V"], text=True) # noqa: S603 + version = subprocess.check_output( # noqa: S603 + [str(binary), "-V"], text=True, stderr=subprocess.STDOUT + ) if pin["seekdb_sha"] not in version: raise ValueError("installed SeekDB does not report pinned full REVISION") print(f"[EMBEDDED_WHEEL] installed={binary}") diff --git a/tests/unit_tests/test_embedded_wheel_manifest.py b/tests/unit_tests/test_embedded_wheel_manifest.py index 929fb5db..4f5641fc 100644 --- a/tests/unit_tests/test_embedded_wheel_manifest.py +++ b/tests/unit_tests/test_embedded_wheel_manifest.py @@ -86,6 +86,38 @@ def test_dependency_resync_is_detected(bundle, monkeypatch): wheel_manifest.main() +def test_installed_revision_from_stderr_is_verified(bundle, monkeypatch): + pin, directory, binary, _ = bundle + monkeypatch.setattr( + wheel_manifest.importlib.metadata, "distribution", lambda _: SimpleNamespace(locate_file=lambda _: binary) + ) + + def version_output(command, *, text, stderr): + assert command == [str(binary), "-V"] + assert text is True + assert stderr is wheel_manifest.subprocess.STDOUT + return f"REVISION: 1-{pin['seekdb_sha']}\n" + + monkeypatch.setattr(wheel_manifest.subprocess, "check_output", version_output) + wheel_manifest.sys.argv = ["manifest", "installed", str(directory / "source.json"), str(directory)] + wheel_manifest.main() + + +def test_installed_wrong_revision_is_rejected(bundle, monkeypatch): + _, directory, binary, _ = bundle + monkeypatch.setattr( + wheel_manifest.importlib.metadata, "distribution", lambda _: SimpleNamespace(locate_file=lambda _: binary) + ) + monkeypatch.setattr( + wheel_manifest.subprocess, + "check_output", + lambda *_, **__: f"REVISION: 1-{'d' * 40}\n", + ) + wheel_manifest.sys.argv = ["manifest", "installed", str(directory / "source.json"), str(directory)] + with pytest.raises(ValueError, match="pinned full REVISION"): + wheel_manifest.main() + + def test_incompatible_host_is_rejected(bundle, monkeypatch): pin, directory, _, _ = bundle monkeypatch.setattr(wheel_manifest.platform, "machine", lambda: "aarch64") From 257f99a5cd8ec25e49a4edb77c42c136a3615c1b Mon Sep 17 00:00:00 2001 From: cms-cms <1303573631@qq.com> Date: Fri, 18 Sep 2026 11:39:19 +0800 Subject: [PATCH 05/25] ci: stabilize default embedding model downloads --- .github/workflows/ci.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a359699b..93f10e9c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -141,6 +141,13 @@ jobs: with: python-version: '3.11' + - name: Restore default ONNX model cache + if: matrix.test_mode == 'oceanbase' + uses: actions/cache@v4 + with: + path: ~/.cache/pyseekdb/onnx_models + key: pyseekdb-onnx-all-MiniLM-L6-v2-v1 + - name: Restore exact embedded source wheel if: matrix.test_mode == 'embedded' id: embedded-wheel @@ -212,6 +219,9 @@ jobs: - name: Run integration tests for ${{ matrix.test_mode }} env: CI: 1 + # GitHub-hosted runners can reach the canonical endpoint reliably; + # avoid making the suite depend on the China mirror's DNS state. + HF_ENDPOINT: https://huggingface.co SEEKDB_TEST_DATA_ROOT: ${{ github.workspace }}/.seekdb-test-data OB_PORT: 10000 SERVER_PORT: 2881 From d8d40385e66409a6b55d9c2de53f73f0641f35b1 Mon Sep 17 00:00:00 2001 From: cms-cms <1303573631@qq.com> Date: Sun, 20 Sep 2026 20:56:20 +0800 Subject: [PATCH 06/25] test: make embedding dependencies deterministic --- pyproject.toml | 13 + .../mnn_embedding_function.py | 7 +- ...t_huggingface_sparse_embedding_function.py | 26 +- .../unit_tests/test_mnn_embedding_function.py | 8 + ...sentence_transformer_embedding_function.py | 16 ++ uv.lock | 230 +++++------------- 6 files changed, 108 insertions(+), 192 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 4219de97..207b8c50 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,11 +46,24 @@ unit-tests = [ "voyageai", "sentence-transformers>=5.0", "huggingface-hub>=0.26", + # Declare torch directly so Linux CI resolves the CPU wheel instead of the + # default PyPI CUDA dependency set pulled indirectly by transformers. + "torch==2.9.1", # pylibseekdb 1.3.x requires an ABI=1 native runtime to be loaded first # in embedded integration environments. This is not a runtime dependency. "onnxruntime>=1.19.0", ] +[tool.uv.sources] +torch = [ + { index = "pytorch-cpu", marker = "sys_platform == 'linux' and platform_machine == 'x86_64'" }, +] + +[[tool.uv.index]] +name = "pytorch-cpu" +url = "https://download.pytorch.org/whl/cpu" +explicit = true + [tool.uv.workspace] members = ["demo/rag"] diff --git a/src/pyseekdb/utils/embedding_functions/mnn_embedding_function.py b/src/pyseekdb/utils/embedding_functions/mnn_embedding_function.py index 1fd75ee2..b017b8b7 100644 --- a/src/pyseekdb/utils/embedding_functions/mnn_embedding_function.py +++ b/src/pyseekdb/utils/embedding_functions/mnn_embedding_function.py @@ -84,11 +84,8 @@ def __init__( self.hf_revision = hf_revision self._dimension = dimension self._expected_sha256 = dict(expected_sha256 or {}) - self.download_path = ( - download_path - if download_path is not None - else Path.home() / ".cache" / "pyseekdb" / "mnn_models" / model_name - ) + cache_root = Path(os.environ.get("PYSEEKDB_CACHE_DIR", Path.home() / ".cache" / "pyseekdb")) + self.download_path = download_path if download_path is not None else cache_root / "mnn_models" / model_name # These imports are intentionally lazy so importing pyseekdb does not # initialize the native MNN runtime until embeddings are requested. diff --git a/tests/unit_tests/test_huggingface_sparse_embedding_function.py b/tests/unit_tests/test_huggingface_sparse_embedding_function.py index 9549de6a..5d19ab71 100644 --- a/tests/unit_tests/test_huggingface_sparse_embedding_function.py +++ b/tests/unit_tests/test_huggingface_sparse_embedding_function.py @@ -41,15 +41,26 @@ def _sparse_encoder_available() -> bool: @pytest.fixture(autouse=True) -def _clear_model_cache(): - """Clear the class-level model cache before each test.""" +def mock_sparse_encoder(): + """Keep unit tests deterministic and leave real model loading to integration tests.""" from pyseekdb.utils.embedding_functions.huggingface_sparse_embedding_function import ( HuggingFaceSparseEmbeddingFunction, ) saved = HuggingFaceSparseEmbeddingFunction.models.copy() HuggingFaceSparseEmbeddingFunction.models.clear() - yield + with patch("sentence_transformers.SparseEncoder") as mock_encoder: + model = MagicMock() + + def encode(documents): + vector = np.zeros(8, dtype=np.float32) + vector[2] = 0.75 + return [vector.copy() for _ in documents] + + model.encode_document.side_effect = encode + model.encode_query.side_effect = encode + mock_encoder.return_value = model + yield mock_encoder, model HuggingFaceSparseEmbeddingFunction.models.clear() HuggingFaceSparseEmbeddingFunction.models.update(saved) @@ -113,15 +124,6 @@ def test_different_models_loaded_separately(self): HuggingFaceSparseEmbeddingFunction(model_name="naver/splade-v3-distilbert") -@pytest.fixture -def mock_sparse_encoder(): - """Patch SparseEncoder so tests get a mock model; yields (MockEncoder, mock_instance).""" - with patch("sentence_transformers.SparseEncoder") as MockEncoder: - mock_instance = MagicMock() - MockEncoder.return_value = mock_instance - yield (MockEncoder, mock_instance) - - @pytest.mark.skipif( not _sparse_encoder_available(), reason="sentence-transformers with SparseEncoder is not available", diff --git a/tests/unit_tests/test_mnn_embedding_function.py b/tests/unit_tests/test_mnn_embedding_function.py index e644cda0..eb2d21f9 100644 --- a/tests/unit_tests/test_mnn_embedding_function.py +++ b/tests/unit_tests/test_mnn_embedding_function.py @@ -57,6 +57,14 @@ def test_local_model_does_not_require_huggingface_metadata(tmp_path: Path) -> No assert ef.hf_revision == "main" +def test_default_cache_honors_pyseekdb_cache_dir(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("PYSEEKDB_CACHE_DIR", str(tmp_path)) + + ef = MnnEmbeddingFunction("test-model", "org/test-model", 3) + + assert ef.download_path == tmp_path / "mnn_models" / "test-model" + + def test_hf_endpoint_must_use_https(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: ef = _make_embedding_function(tmp_path) monkeypatch.setenv("HF_ENDPOINT", "http://example.invalid") diff --git a/tests/unit_tests/test_sentence_transformer_embedding_function.py b/tests/unit_tests/test_sentence_transformer_embedding_function.py index df6914a2..2ea3ab35 100644 --- a/tests/unit_tests/test_sentence_transformer_embedding_function.py +++ b/tests/unit_tests/test_sentence_transformer_embedding_function.py @@ -10,7 +10,9 @@ """ import importlib.util +from unittest.mock import MagicMock, patch +import numpy as np import pytest from pyseekdb.utils.embedding_functions import SentenceTransformerEmbeddingFunction @@ -44,6 +46,20 @@ def is_sentence_transformers_available() -> bool: return importlib.util.find_spec("sentence_transformers") is not None +@pytest.fixture(autouse=True) +def mock_sentence_transformer(): + """Persistence unit tests must not download real embedding models.""" + saved = SentenceTransformerEmbeddingFunction.models.copy() + SentenceTransformerEmbeddingFunction.models.clear() + with patch("sentence_transformers.SentenceTransformer") as mock_class: + model = MagicMock() + model.encode.side_effect = lambda documents, **_kwargs: np.zeros((len(documents), 3), dtype=np.float32) + mock_class.return_value = model + yield mock_class, model + SentenceTransformerEmbeddingFunction.models.clear() + SentenceTransformerEmbeddingFunction.models.update(saved) + + @pytest.mark.skipif( not is_sentence_transformers_available(), reason="sentence-transformers is not available on this system", diff --git a/uv.lock b/uv.lock index 7e235cba..d3b8b2aa 100644 --- a/uv.lock +++ b/uv.lock @@ -2,10 +2,14 @@ version = 1 revision = 3 requires-python = ">=3.11, <4" resolution-markers = [ - "python_full_version >= '3.14'", - "python_full_version == '3.13.*'", - "python_full_version == '3.12.*'", - "python_full_version < '3.12'", + "python_full_version >= '3.14' and platform_machine == 'x86_64' and sys_platform == 'linux'", + "(python_full_version >= '3.14' and platform_machine != 'x86_64') or (python_full_version >= '3.14' and sys_platform != 'linux')", + "python_full_version == '3.13.*' and platform_machine == 'x86_64' and sys_platform == 'linux'", + "(python_full_version == '3.13.*' and platform_machine != 'x86_64') or (python_full_version == '3.13.*' and sys_platform != 'linux')", + "python_full_version == '3.12.*' and platform_machine == 'x86_64' and sys_platform == 'linux'", + "(python_full_version == '3.12.*' and platform_machine != 'x86_64') or (python_full_version == '3.12.*' and sys_platform != 'linux')", + "python_full_version < '3.12' and platform_machine == 'x86_64' and sys_platform == 'linux'", + "(python_full_version < '3.12' and platform_machine != 'x86_64') or (python_full_version < '3.12' and sys_platform != 'linux')", ] [manifest] @@ -1790,140 +1794,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/de/e5/b7d20451657664b07986c2f6e3be564433f5dcaf3482d68eaecd79afaf03/numpy-2.4.2-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:be71bf1edb48ebbbf7f6337b5bfd2f895d1902f6335a5830b20141fc126ffba0", size = 12502577, upload-time = "2026-01-31T23:13:07.08Z" }, ] -[[package]] -name = "nvidia-cublas-cu12" -version = "12.8.4.1" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/dc/61/e24b560ab2e2eaeb3c839129175fb330dfcfc29e5203196e5541a4c44682/nvidia_cublas_cu12-12.8.4.1-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:8ac4e771d5a348c551b2a426eda6193c19aa630236b418086020df5ba9667142", size = 594346921, upload-time = "2025-03-07T01:44:31.254Z" }, -] - -[[package]] -name = "nvidia-cuda-cupti-cu12" -version = "12.8.90" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f8/02/2adcaa145158bf1a8295d83591d22e4103dbfd821bcaf6f3f53151ca4ffa/nvidia_cuda_cupti_cu12-12.8.90-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ea0cb07ebda26bb9b29ba82cda34849e73c166c18162d3913575b0c9db9a6182", size = 10248621, upload-time = "2025-03-07T01:40:21.213Z" }, -] - -[[package]] -name = "nvidia-cuda-nvrtc-cu12" -version = "12.8.93" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/05/6b/32f747947df2da6994e999492ab306a903659555dddc0fbdeb9d71f75e52/nvidia_cuda_nvrtc_cu12-12.8.93-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:a7756528852ef889772a84c6cd89d41dfa74667e24cca16bb31f8f061e3e9994", size = 88040029, upload-time = "2025-03-07T01:42:13.562Z" }, -] - -[[package]] -name = "nvidia-cuda-runtime-cu12" -version = "12.8.90" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/0d/9b/a997b638fcd068ad6e4d53b8551a7d30fe8b404d6f1804abf1df69838932/nvidia_cuda_runtime_cu12-12.8.90-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:adade8dcbd0edf427b7204d480d6066d33902cab2a4707dcfc48a2d0fd44ab90", size = 954765, upload-time = "2025-03-07T01:40:01.615Z" }, -] - -[[package]] -name = "nvidia-cudnn-cu12" -version = "9.10.2.21" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "nvidia-cublas-cu12" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/ba/51/e123d997aa098c61d029f76663dedbfb9bc8dcf8c60cbd6adbe42f76d049/nvidia_cudnn_cu12-9.10.2.21-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:949452be657fa16687d0930933f032835951ef0892b37d2d53824d1a84dc97a8", size = 706758467, upload-time = "2025-06-06T21:54:08.597Z" }, -] - -[[package]] -name = "nvidia-cufft-cu12" -version = "11.3.3.83" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "nvidia-nvjitlink-cu12" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/1f/13/ee4e00f30e676b66ae65b4f08cb5bcbb8392c03f54f2d5413ea99a5d1c80/nvidia_cufft_cu12-11.3.3.83-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4d2dd21ec0b88cf61b62e6b43564355e5222e4a3fb394cac0db101f2dd0d4f74", size = 193118695, upload-time = "2025-03-07T01:45:27.821Z" }, -] - -[[package]] -name = "nvidia-cufile-cu12" -version = "1.13.1.3" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/bb/fe/1bcba1dfbfb8d01be8d93f07bfc502c93fa23afa6fd5ab3fc7c1df71038a/nvidia_cufile_cu12-1.13.1.3-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1d069003be650e131b21c932ec3d8969c1715379251f8d23a1860554b1cb24fc", size = 1197834, upload-time = "2025-03-07T01:45:50.723Z" }, -] - -[[package]] -name = "nvidia-curand-cu12" -version = "10.3.9.90" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/fb/aa/6584b56dc84ebe9cf93226a5cde4d99080c8e90ab40f0c27bda7a0f29aa1/nvidia_curand_cu12-10.3.9.90-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:b32331d4f4df5d6eefa0554c565b626c7216f87a06a4f56fab27c3b68a830ec9", size = 63619976, upload-time = "2025-03-07T01:46:23.323Z" }, -] - -[[package]] -name = "nvidia-cusolver-cu12" -version = "11.7.3.90" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "nvidia-cublas-cu12" }, - { name = "nvidia-cusparse-cu12" }, - { name = "nvidia-nvjitlink-cu12" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/85/48/9a13d2975803e8cf2777d5ed57b87a0b6ca2cc795f9a4f59796a910bfb80/nvidia_cusolver_cu12-11.7.3.90-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:4376c11ad263152bd50ea295c05370360776f8c3427b30991df774f9fb26c450", size = 267506905, upload-time = "2025-03-07T01:47:16.273Z" }, -] - -[[package]] -name = "nvidia-cusparse-cu12" -version = "12.5.8.93" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "nvidia-nvjitlink-cu12" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/c2/f5/e1854cb2f2bcd4280c44736c93550cc300ff4b8c95ebe370d0aa7d2b473d/nvidia_cusparse_cu12-12.5.8.93-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1ec05d76bbbd8b61b06a80e1eaf8cf4959c3d4ce8e711b65ebd0443bb0ebb13b", size = 288216466, upload-time = "2025-03-07T01:48:13.779Z" }, -] - -[[package]] -name = "nvidia-cusparselt-cu12" -version = "0.7.1" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/56/79/12978b96bd44274fe38b5dde5cfb660b1d114f70a65ef962bcbbed99b549/nvidia_cusparselt_cu12-0.7.1-py3-none-manylinux2014_x86_64.whl", hash = "sha256:f1bb701d6b930d5a7cea44c19ceb973311500847f81b634d802b7b539dc55623", size = 287193691, upload-time = "2025-02-26T00:15:44.104Z" }, -] - -[[package]] -name = "nvidia-nccl-cu12" -version = "2.27.5" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/6e/89/f7a07dc961b60645dbbf42e80f2bc85ade7feb9a491b11a1e973aa00071f/nvidia_nccl_cu12-2.27.5-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ad730cf15cb5d25fe849c6e6ca9eb5b76db16a80f13f425ac68d8e2e55624457", size = 322348229, upload-time = "2025-06-26T04:11:28.385Z" }, -] - -[[package]] -name = "nvidia-nvjitlink-cu12" -version = "12.8.93" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f6/74/86a07f1d0f42998ca31312f998bd3b9a7eff7f52378f4f270c8679c77fb9/nvidia_nvjitlink_cu12-12.8.93-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:81ff63371a7ebd6e6451970684f916be2eab07321b73c9d244dc2b4da7f73b88", size = 39254836, upload-time = "2025-03-07T01:49:55.661Z" }, -] - -[[package]] -name = "nvidia-nvshmem-cu12" -version = "3.3.20" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/3b/6c/99acb2f9eb85c29fc6f3a7ac4dccfd992e22666dd08a642b303311326a97/nvidia_nvshmem_cu12-3.3.20-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d00f26d3f9b2e3c3065be895e3059d6479ea5c638a3f38c9fec49b1b9dd7c1e5", size = 124657145, upload-time = "2025-08-04T20:25:19.995Z" }, -] - -[[package]] -name = "nvidia-nvtx-cu12" -version = "12.8.90" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a2/eb/86626c1bbc2edb86323022371c39aa48df6fd8b0a1647bc274577f72e90b/nvidia_nvtx_cu12-12.8.90-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:5b17e2001cc0d751a5bc2c6ec6d26ad95913324a4adb86788c944f8ce9ba441f", size = 89954, upload-time = "2025-03-07T01:42:44.131Z" }, -] - [[package]] name = "onnxruntime" version = "1.30.0" @@ -2673,6 +2543,8 @@ unit-tests = [ { name = "onnxruntime" }, { name = "openai" }, { name = "sentence-transformers" }, + { name = "torch", version = "2.9.1", source = { registry = "https://pypi.org/simple" }, marker = "platform_machine != 'x86_64' or sys_platform != 'linux'" }, + { name = "torch", version = "2.9.1+cpu", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, { name = "voyageai" }, ] @@ -2707,6 +2579,8 @@ unit-tests = [ { name = "onnxruntime", specifier = ">=1.19.0" }, { name = "openai" }, { name = "sentence-transformers", specifier = ">=5.0" }, + { name = "torch", marker = "platform_machine != 'x86_64' or sys_platform != 'linux'", specifier = "==2.9.1" }, + { name = "torch", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'", specifier = "==2.9.1", index = "https://download.pytorch.org/whl/cpu" }, { name = "voyageai" }, ] @@ -3406,7 +3280,8 @@ dependencies = [ { name = "huggingface-hub" }, { name = "scikit-learn" }, { name = "scipy" }, - { name = "torch" }, + { name = "torch", version = "2.9.1", source = { registry = "https://pypi.org/simple" }, marker = "platform_machine != 'x86_64' or sys_platform != 'linux'" }, + { name = "torch", version = "2.9.1+cpu", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, { name = "tqdm" }, { name = "transformers" }, { name = "typing-extensions" }, @@ -3769,30 +3644,20 @@ wheels = [ name = "torch" version = "2.9.1" source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "(python_full_version >= '3.14' and platform_machine != 'x86_64') or (python_full_version >= '3.14' and sys_platform != 'linux')", + "(python_full_version == '3.13.*' and platform_machine != 'x86_64') or (python_full_version == '3.13.*' and sys_platform != 'linux')", + "(python_full_version == '3.12.*' and platform_machine != 'x86_64') or (python_full_version == '3.12.*' and sys_platform != 'linux')", + "(python_full_version < '3.12' and platform_machine != 'x86_64') or (python_full_version < '3.12' and sys_platform != 'linux')", +] dependencies = [ - { name = "filelock" }, - { name = "fsspec" }, - { name = "jinja2" }, - { name = "networkx" }, - { name = "nvidia-cublas-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "nvidia-cuda-cupti-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "nvidia-cuda-nvrtc-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "nvidia-cuda-runtime-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "nvidia-cudnn-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "nvidia-cufft-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "nvidia-cufile-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "nvidia-curand-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "nvidia-cusolver-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "nvidia-cusparse-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "nvidia-cusparselt-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "nvidia-nccl-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "nvidia-nvjitlink-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "nvidia-nvshmem-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "nvidia-nvtx-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "setuptools", marker = "python_full_version >= '3.12'" }, - { name = "sympy" }, - { name = "triton", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "typing-extensions" }, + { name = "filelock", marker = "platform_machine != 'x86_64' or sys_platform != 'linux'" }, + { name = "fsspec", marker = "platform_machine != 'x86_64' or sys_platform != 'linux'" }, + { name = "jinja2", marker = "platform_machine != 'x86_64' or sys_platform != 'linux'" }, + { name = "networkx", marker = "platform_machine != 'x86_64' or sys_platform != 'linux'" }, + { name = "setuptools", marker = "(python_full_version >= '3.12' and platform_machine != 'x86_64') or (python_full_version >= '3.12' and sys_platform != 'linux')" }, + { name = "sympy", marker = "platform_machine != 'x86_64' or sys_platform != 'linux'" }, + { name = "typing-extensions", marker = "platform_machine != 'x86_64' or sys_platform != 'linux'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/15/db/c064112ac0089af3d2f7a2b5bfbabf4aa407a78b74f87889e524b91c5402/torch-2.9.1-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:62b3fd888277946918cba4478cf849303da5359f0fb4e3bfb86b0533ba2eaf8d", size = 104220430, upload-time = "2025-11-12T15:20:31.705Z" }, @@ -3821,6 +3686,34 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/db/2b/f7818f6ec88758dfd21da46b6cd46af9d1b3433e53ddbb19ad1e0da17f9b/torch-2.9.1-cp314-cp314t-win_amd64.whl", hash = "sha256:c88d3299ddeb2b35dcc31753305612db485ab6f1823e37fb29451c8b2732b87e", size = 111163659, upload-time = "2025-11-12T15:23:20.009Z" }, ] +[[package]] +name = "torch" +version = "2.9.1+cpu" +source = { registry = "https://download.pytorch.org/whl/cpu" } +resolution-markers = [ + "python_full_version >= '3.14' and platform_machine == 'x86_64' and sys_platform == 'linux'", + "python_full_version == '3.13.*' and platform_machine == 'x86_64' and sys_platform == 'linux'", + "python_full_version == '3.12.*' and platform_machine == 'x86_64' and sys_platform == 'linux'", + "python_full_version < '3.12' and platform_machine == 'x86_64' and sys_platform == 'linux'", +] +dependencies = [ + { name = "filelock", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "fsspec", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "jinja2", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "networkx", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "setuptools", marker = "python_full_version >= '3.12' and platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "sympy", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "typing-extensions", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, +] +wheels = [ + { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.9.1%2Bcpu-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:3de2adb9b4443dc9210ef1f1b16da3647ace53553166d6360bbbd7edd6f16e4d", upload-time = "2026-01-23T15:12:58Z" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.9.1%2Bcpu-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:7417d8c565f219d3455654cb431c6d892a3eb40246055e14d645422de13b9ea1", upload-time = "2026-01-23T15:13:04Z" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.9.1%2Bcpu-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:39b3dff6d8fba240ae0d1bede4ca11c2531ae3b47329206512d99e17907ff74b", upload-time = "2026-01-23T15:13:12Z" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.9.1%2Bcpu-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:031a597147fa81b1e6d79ccf1ad3ccc7fafa27941d6cf26ff5caaa384fb20e92", upload-time = "2026-01-23T15:13:19Z" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.9.1%2Bcpu-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:88adf5157db5da1d54b1c9fe4a6c1d20ceef00e75d854e206a87dbf69e3037dc", upload-time = "2026-01-23T15:13:23Z" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.9.1%2Bcpu-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:5b688445f928f13563b7418b17c57e97bf955ab559cf73cd8f2b961f8572dbb3", upload-time = "2026-01-23T15:13:29Z" }, +] + [[package]] name = "tornado" version = "6.5.7" @@ -3871,19 +3764,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/6a/6b/2f416568b3c4c91c96e5a365d164f8a4a4a88030aa8ab4644181fdadce97/transformers-4.57.3-py3-none-any.whl", hash = "sha256:c77d353a4851b1880191603d36acb313411d3577f6e2897814f333841f7003f4", size = 11993463, upload-time = "2025-11-25T15:51:26.493Z" }, ] -[[package]] -name = "triton" -version = "3.5.1" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b0/72/ec90c3519eaf168f22cb1757ad412f3a2add4782ad3a92861c9ad135d886/triton-3.5.1-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:61413522a48add32302353fdbaaf92daaaab06f6b5e3229940d21b5207f47579", size = 170425802, upload-time = "2025-11-11T17:40:53.209Z" }, - { url = "https://files.pythonhosted.org/packages/f2/50/9a8358d3ef58162c0a415d173cfb45b67de60176e1024f71fbc4d24c0b6d/triton-3.5.1-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d2c6b915a03888ab931a9fd3e55ba36785e1fe70cbea0b40c6ef93b20fc85232", size = 170470207, upload-time = "2025-11-11T17:41:00.253Z" }, - { url = "https://files.pythonhosted.org/packages/27/46/8c3bbb5b0a19313f50edcaa363b599e5a1a5ac9683ead82b9b80fe497c8d/triton-3.5.1-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f3f4346b6ebbd4fad18773f5ba839114f4826037c9f2f34e0148894cd5dd3dba", size = 170470410, upload-time = "2025-11-11T17:41:06.319Z" }, - { url = "https://files.pythonhosted.org/packages/37/92/e97fcc6b2c27cdb87ce5ee063d77f8f26f19f06916aa680464c8104ef0f6/triton-3.5.1-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0b4d2c70127fca6a23e247f9348b8adde979d2e7a20391bfbabaac6aebc7e6a8", size = 170579924, upload-time = "2025-11-11T17:41:12.455Z" }, - { url = "https://files.pythonhosted.org/packages/a4/e6/c595c35e5c50c4bc56a7bac96493dad321e9e29b953b526bbbe20f9911d0/triton-3.5.1-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d0637b1efb1db599a8e9dc960d53ab6e4637db7d4ab6630a0974705d77b14b60", size = 170480488, upload-time = "2025-11-11T17:41:18.222Z" }, - { url = "https://files.pythonhosted.org/packages/16/b5/b0d3d8b901b6a04ca38df5e24c27e53afb15b93624d7fd7d658c7cd9352a/triton-3.5.1-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bac7f7d959ad0f48c0e97d6643a1cc0fd5786fe61cb1f83b537c6b2d54776478", size = 170582192, upload-time = "2025-11-11T17:41:23.963Z" }, -] - [[package]] name = "typing-extensions" version = "4.15.0" From 477e07c5ef6b401b0ac96958e6f32dc094bca69e Mon Sep 17 00:00:00 2001 From: cms-cms <1303573631@qq.com> Date: Sun, 20 Sep 2026 21:00:09 +0800 Subject: [PATCH 07/25] test: allow sparse fallback output override --- tests/unit_tests/test_huggingface_sparse_embedding_function.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/unit_tests/test_huggingface_sparse_embedding_function.py b/tests/unit_tests/test_huggingface_sparse_embedding_function.py index 5d19ab71..8b1de1b3 100644 --- a/tests/unit_tests/test_huggingface_sparse_embedding_function.py +++ b/tests/unit_tests/test_huggingface_sparse_embedding_function.py @@ -194,6 +194,7 @@ def test_numpy_array_fallback(self, mock_sparse_encoder): arr = np.zeros(100, dtype=np.float32) arr[7] = 0.6 arr[99] = 0.1 + mock_model.encode_document.side_effect = None mock_model.encode_document.return_value = [arr] ef = HuggingFaceSparseEmbeddingFunction() From ec16471811c5f72882e41530f32035b8bb2dddac Mon Sep 17 00:00:00 2001 From: cms-cms <1303573631@qq.com> Date: Sun, 20 Sep 2026 21:29:10 +0800 Subject: [PATCH 08/25] ci: validate embedded tests with fork timeout fix --- .github/embedded-source.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/embedded-source.json b/.github/embedded-source.json index a3f33a1c..3084230f 100644 --- a/.github/embedded-source.json +++ b/.github/embedded-source.json @@ -1,5 +1,5 @@ { - "seekdb_sha": "8a8d6f5a8dd48cdbf859305b3f7eb050149a5bd1", + "seekdb_sha": "c60ac64913009d393c6434bc04e8284530b83e12", "bindings_sha": "5e8eece63205dccfa9500f98893c91d1f50eb4f1", "fix_commit": "48b80f5193bcac0afafba4d4820aebdf1103bc69", "platform": "linux-x86_64", From 19a26a59e442c993b458e834fab6d106e79a7325 Mon Sep 17 00:00:00 2001 From: cms-cms <1303573631@qq.com> Date: Sun, 20 Sep 2026 22:46:32 +0800 Subject: [PATCH 09/25] ci: validate embedded fork with ddl deadline fix --- .github/embedded-source.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/embedded-source.json b/.github/embedded-source.json index 3084230f..713bf611 100644 --- a/.github/embedded-source.json +++ b/.github/embedded-source.json @@ -1,5 +1,5 @@ { - "seekdb_sha": "c60ac64913009d393c6434bc04e8284530b83e12", + "seekdb_sha": "ae50a9c845d263c077dc880dfc7477d5a82039c4", "bindings_sha": "5e8eece63205dccfa9500f98893c91d1f50eb4f1", "fix_commit": "48b80f5193bcac0afafba4d4820aebdf1103bc69", "platform": "linux-x86_64", From 8bdee2a6b4c3a65ac2f3a8cb81eadffae52ec6bf Mon Sep 17 00:00:00 2001 From: cms-cms <1303573631@qq.com> Date: Mon, 21 Sep 2026 04:32:04 +0800 Subject: [PATCH 10/25] fix: prefer source-built seekdb runtime --- src/pyseekdb/client/__init__.py | 6 +-- src/pyseekdb/client/client_seekdb_embedded.py | 44 +++++++++++-------- tests/integration_tests/conftest.py | 9 ++-- .../embedded_fulltext_support.py | 6 ++- ...t_get_or_create_collection_multiprocess.py | 7 +-- .../test_embedded_client_lifecycle.py | 30 +++++++++++++ 6 files changed, 70 insertions(+), 32 deletions(-) diff --git a/src/pyseekdb/client/__init__.py b/src/pyseekdb/client/__init__.py index 03827b28..a242e869 100644 --- a/src/pyseekdb/client/__init__.py +++ b/src/pyseekdb/client/__init__.py @@ -12,7 +12,7 @@ - Returns: _AdminClientProxy (database operations only) All factories use the underlying ServerAPI implementations: -- SeekdbEmbeddedClient - Local seekdb (requires pylibseekdb, Linux only) +- SeekdbEmbeddedClient - Local seekdb (requires seekdb or legacy pylibseekdb) - RemoteServerClient - Remote server via pymysql (supports both seekdb Server and OceanBase Server) """ @@ -136,7 +136,7 @@ def _create_server_client( return SeekdbEmbeddedClient(path=default_path, database=database, **kwargs) raise ValueError( - "Default embedded mode is not available because pylibseekdb could not be imported. " + "Default embedded mode is not available because a SeekDB Python binding could not be imported. " "Please provide host/port parameters to use RemoteServerClient." ) @@ -145,7 +145,7 @@ def __getattr__(name: str) -> Any: """ Lazily expose optional embedded client symbols. - This avoids importing pylibseekdb during `import pyseekdb`, which can crash + This avoids importing a native embedded binding during `import pyseekdb`, which can crash on unsupported interpreter/platform combinations (e.g. some Python 3.14 CI environments). """ if name == "SeekdbEmbeddedClient": diff --git a/src/pyseekdb/client/client_seekdb_embedded.py b/src/pyseekdb/client/client_seekdb_embedded.py index 42efc8ea..31836431 100644 --- a/src/pyseekdb/client/client_seekdb_embedded.py +++ b/src/pyseekdb/client/client_seekdb_embedded.py @@ -1,8 +1,9 @@ """ Embedded mode client - based on seekdb -Note: Only available when pylibseekdb is installed (Linux only) +Available when the current ``seekdb`` binding or legacy ``pylibseekdb`` is installed. """ +import importlib import logging import os import threading @@ -12,25 +13,30 @@ import pymysql from pymysql.cursors import DictCursor -# Try to import pylibseekdb - it may not be available on all platforms -try: - import pylibseekdb as seekdb # type: ignore[import-not-found] - - _PYLIBSEEKDB_AVAILABLE = True -except ImportError: - seekdb = None # type: ignore[assignment] - _PYLIBSEEKDB_AVAILABLE = False - from .admin_client import DEFAULT_TENANT from .client_base import BaseClient from .database import Database from .sql_utils import render_sql_with_params + +def _load_embedded_runtime() -> tuple[Any | None, str | None]: + """Prefer the current ``seekdb`` binding, with legacy compatibility.""" + for distribution in ("seekdb", "pylibseekdb"): + try: + return importlib.import_module(distribution), distribution + except ImportError: + pass + return None, None + + +seekdb, _EMBEDDED_RUNTIME_DISTRIBUTION = _load_embedded_runtime() +_PYLIBSEEKDB_AVAILABLE = seekdb is not None + logger = logging.getLogger(__name__) class _NativeEmbeddedBackend: - """Connect through pylibseekdb's native Python connection wrapper.""" + """Connect through the embedded binding's native connection wrapper.""" supports_dbapi_cursor = False @@ -54,7 +60,7 @@ class _PyMySQLEmbeddedBackend: @staticmethod def connect(instance: Any, database: str, connection_kwargs: dict[str, Any]) -> pymysql.Connection: if instance is None: - raise RuntimeError("pylibseekdb returned no SeekdbInstance for a PyMySQL embedded connection") + raise RuntimeError("the embedded binding returned no SeekdbInstance for a PyMySQL connection") options = dict(instance.connection_options()) kwargs = dict(connection_kwargs) @@ -80,7 +86,7 @@ def is_connection_open(connection: Any) -> bool: def _create_embedded_backend() -> _NativeEmbeddedBackend | _PyMySQLEmbeddedBackend: - """Select the safest connection backend from pylibseekdb's capabilities.""" + """Select the safest connection backend exposed by the installed binding.""" instance_type = getattr(seekdb, "SeekdbInstance", None) if instance_type is not None and callable(getattr(instance_type, "connection_options", None)): return _PyMySQLEmbeddedBackend() @@ -90,7 +96,7 @@ def _create_embedded_backend() -> _NativeEmbeddedBackend | _PyMySQLEmbeddedBacke class SeekdbEmbeddedClient(BaseClient): """Embedded seekdb client (lazy connection) - Note: Only available on Linux platforms. pylibseekdb dependency is Linux-only. + Availability follows the installed embedded binding and its supported platform. """ def __init__(self, path: str = "./seekdb.db", database: str = "test", **kwargs): @@ -102,13 +108,13 @@ def __init__(self, path: str = "./seekdb.db", database: str = "test", **kwargs): database: database name Raises: - RuntimeError: If pylibseekdb is not available + RuntimeError: If no embedded binding is available """ - # Check if pylibseekdb is available + # Check if an embedded binding is available. if not _PYLIBSEEKDB_AVAILABLE or seekdb is None: raise RuntimeError( - "Embedded Client is not available because pylibseekdb is not available. " - "Please install pylibseekdb (Linux only) or use RemoteServerClient (host/port) instead." + "Embedded Client is not available because no SeekDB Python binding was found. " + "Please install seekdb (or legacy pylibseekdb), or use RemoteServerClient (host/port) instead." ) self.path = os.path.abspath(path) @@ -169,7 +175,7 @@ def _ensure_connection(self) -> Any: return self._connection def _cleanup(self) -> None: - """Close the connection and its owning pylibseekdb instance.""" + """Close the connection and its owning embedded runtime instance.""" with self._connection_lock: connection = self._connection instance = self._instance diff --git a/tests/integration_tests/conftest.py b/tests/integration_tests/conftest.py index 5eda1f80..0d771584 100644 --- a/tests/integration_tests/conftest.py +++ b/tests/integration_tests/conftest.py @@ -4,6 +4,7 @@ """ import contextlib +import importlib.util import os import sys from pathlib import Path @@ -47,9 +48,7 @@ # ==================== Client Factory Functions ==================== def create_embedded_client(): """Create an embedded client instance.""" - try: - import pylibseekdb # noqa: F401 - except ImportError: + if not any(importlib.util.find_spec(name) for name in ("seekdb", "pylibseekdb")): pytest.skip("seekdb embedded package is not installed") return pyseekdb.Client(path=SEEKDB_PATH, database=SEEKDB_DATABASE) @@ -100,9 +99,7 @@ def create_oceanbase_client(): # ==================== AdminClient Factory Functions ==================== def create_embedded_admin_client(): """Create an embedded admin client instance.""" - try: - import pylibseekdb # noqa: F401 - except ImportError: + if not any(importlib.util.find_spec(name) for name in ("seekdb", "pylibseekdb")): pytest.skip("seekdb embedded package is not installed") return pyseekdb.AdminClient(path=SEEKDB_PATH) diff --git a/tests/integration_tests/embedded_fulltext_support.py b/tests/integration_tests/embedded_fulltext_support.py index 4e94c609..19514013 100644 --- a/tests/integration_tests/embedded_fulltext_support.py +++ b/tests/integration_tests/embedded_fulltext_support.py @@ -10,6 +10,7 @@ import argparse import collections import importlib.metadata +import importlib.util import json import platform import re @@ -349,12 +350,15 @@ def worker(number): def run(scenario, root, evidence): import pyseekdb + runtime_distribution = "seekdb" if importlib.util.find_spec("seekdb") else "pylibseekdb" + report = { "scenario": scenario, "completed": False, "platform": platform.platform(), "pyseekdb": importlib.metadata.version("pyseekdb"), - "pylibseekdb": importlib.metadata.version("pylibseekdb"), + "embedded_runtime": runtime_distribution, + "embedded_runtime_version": importlib.metadata.version(runtime_distribution), "corpus": "synthetic deterministic / 24515 documents", "workers": WORKERS, "duration_seconds": SECONDS, diff --git a/tests/integration_tests/test_get_or_create_collection_multiprocess.py b/tests/integration_tests/test_get_or_create_collection_multiprocess.py index 248a5028..8b4d24f0 100644 --- a/tests/integration_tests/test_get_or_create_collection_multiprocess.py +++ b/tests/integration_tests/test_get_or_create_collection_multiprocess.py @@ -146,15 +146,16 @@ def _refresh_collection(client_config: dict[str, Any], collection_name: str) -> def _require_embedded_pylibseekdb() -> None: """Require embedded pylibseekdb.""" + runtime_distribution = "seekdb" if importlib.util.find_spec("seekdb") else "pylibseekdb" try: - import pylibseekdb # noqa: F401 + importlib.import_module(runtime_distribution) except ImportError: pytest.skip("seekdb embedded package is not installed") try: - installed_version = Version(importlib.metadata.version("pylibseekdb")) + installed_version = Version(importlib.metadata.version(runtime_distribution)) except importlib.metadata.PackageNotFoundError: - pytest.skip("pylibseekdb is not installed") + pytest.skip("seekdb embedded package is not installed") if installed_version < MIN_PYLIBSEEKDB_VERSION: pytest.skip( diff --git a/tests/unit_tests/test_embedded_client_lifecycle.py b/tests/unit_tests/test_embedded_client_lifecycle.py index 748d87fd..d1053b6c 100644 --- a/tests/unit_tests/test_embedded_client_lifecycle.py +++ b/tests/unit_tests/test_embedded_client_lifecycle.py @@ -133,9 +133,11 @@ def connect(self, *, database: str, autocommit: bool) -> _FakeConnection: def embedded_module(monkeypatch: pytest.MonkeyPatch): """Import the embedded module without loading the native pylibseekdb extension.""" module_name = "pyseekdb.client.client_seekdb_embedded" + monkeypatch.setitem(sys.modules, "seekdb", ModuleType("seekdb")) monkeypatch.setitem(sys.modules, "pylibseekdb", ModuleType("pylibseekdb")) sys.modules.pop(module_name, None) module = importlib.import_module(module_name) + assert module._EMBEDDED_RUNTIME_DISTRIBUTION == "seekdb" yield module sys.modules.pop(module_name, None) @@ -145,6 +147,34 @@ def _install_fake_seekdb(embedded_module: Any, monkeypatch: pytest.MonkeyPatch, monkeypatch.setattr(embedded_module, "_PYLIBSEEKDB_AVAILABLE", True) +def test_embedded_runtime_prefers_current_seekdb_package(monkeypatch, embedded_module) -> None: + current = ModuleType("seekdb") + legacy = ModuleType("pylibseekdb") + modules = {"seekdb": current, "pylibseekdb": legacy} + monkeypatch.setattr(embedded_module.importlib, "import_module", modules.__getitem__) + + runtime, distribution = embedded_module._load_embedded_runtime() + + assert runtime is current + assert distribution == "seekdb" + + +def test_embedded_runtime_falls_back_to_legacy_package(monkeypatch, embedded_module) -> None: + legacy = ModuleType("pylibseekdb") + + def import_runtime(name: str) -> ModuleType: + if name == "seekdb": + raise ImportError(name) + return legacy + + monkeypatch.setattr(embedded_module.importlib, "import_module", import_runtime) + + runtime, distribution = embedded_module._load_embedded_runtime() + + assert runtime is legacy + assert distribution == "pylibseekdb" + + def test_clients_use_and_close_their_own_seekdb_instances(tmp_path, monkeypatch, embedded_module) -> None: fake_seekdb = _FakeInstanceApi() _install_fake_seekdb(embedded_module, monkeypatch, fake_seekdb) From 171c4f3df0a355c0e91a65483a21c5608e9e5c92 Mon Sep 17 00:00:00 2001 From: cms-cms <1303573631@qq.com> Date: Mon, 21 Sep 2026 15:58:20 +0800 Subject: [PATCH 11/25] ci: validate embedded tests with latest seekdb fix --- .github/embedded-source.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/embedded-source.json b/.github/embedded-source.json index 713bf611..d3ba9f38 100644 --- a/.github/embedded-source.json +++ b/.github/embedded-source.json @@ -1,5 +1,5 @@ { - "seekdb_sha": "ae50a9c845d263c077dc880dfc7477d5a82039c4", + "seekdb_sha": "f8bed6bcc6ac3e89157c26f69c0c3b430fc93116", "bindings_sha": "5e8eece63205dccfa9500f98893c91d1f50eb4f1", "fix_commit": "48b80f5193bcac0afafba4d4820aebdf1103bc69", "platform": "linux-x86_64", From c874667912aa9c5c8110a30e0c518b639abba2e3 Mon Sep 17 00:00:00 2001 From: cms-cms <1303573631@qq.com> Date: Mon, 21 Sep 2026 18:03:58 +0800 Subject: [PATCH 12/25] ci: validate embedded tests with combined seekdb fixes --- .github/embedded-source.json | 3 ++- .github/scripts/build-embedded-wheel.sh | 7 ++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/embedded-source.json b/.github/embedded-source.json index d3ba9f38..b4aa48d7 100644 --- a/.github/embedded-source.json +++ b/.github/embedded-source.json @@ -1,5 +1,6 @@ { - "seekdb_sha": "f8bed6bcc6ac3e89157c26f69c0c3b430fc93116", + "seekdb_repository": "https://github.com/cms-cms/seekdb.git", + "seekdb_sha": "3f3b11ab9d6115ee448c2f0f693dfbe7f927c2ea", "bindings_sha": "5e8eece63205dccfa9500f98893c91d1f50eb4f1", "fix_commit": "48b80f5193bcac0afafba4d4820aebdf1103bc69", "platform": "linux-x86_64", diff --git a/.github/scripts/build-embedded-wheel.sh b/.github/scripts/build-embedded-wheel.sh index def7d376..7f469273 100644 --- a/.github/scripts/build-embedded-wheel.sh +++ b/.github/scripts/build-embedded-wheel.sh @@ -7,10 +7,11 @@ REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" [[ "$(uname -s)/$(uname -m)" == Linux/x86_64 ]] PYTHON="$(command -v python3)" "$PYTHON" -c 'import sys; assert sys.version_info[:2] == (3, 11)' -read -r SEEKDB_SHA BINDINGS_SHA FIX_COMMIT < <( - "$PYTHON" -c 'import json, sys; p=json.load(open(sys.argv[1])); print(p["seekdb_sha"], p["bindings_sha"], p["fix_commit"])' \ +read -r SEEKDB_REPOSITORY SEEKDB_SHA BINDINGS_SHA FIX_COMMIT < <( + "$PYTHON" -c 'import json, sys; p=json.load(open(sys.argv[1])); print(p["seekdb_repository"], p["seekdb_sha"], p["bindings_sha"], p["fix_commit"])' \ "$REPO_ROOT/.github/embedded-source.json" ) +[[ "$SEEKDB_REPOSITORY" == "https://github.com/cms-cms/seekdb.git" ]] [[ "$SEEKDB_SHA" =~ ^[0-9a-f]{40}$ && "$BINDINGS_SHA" =~ ^[0-9a-f]{40}$ && "$FIX_COMMIT" =~ ^[0-9a-f]{40}$ ]] BUILD_ROOT="$(mktemp -d "$RUNNER_TEMP/embedded-source.XXXXXX")" mkdir -p "$EMBEDDED_WHEEL_DIR" @@ -28,7 +29,7 @@ checkout_exact() { git -C "$target" checkout --detach FETCH_HEAD [[ "$(git -C "$target" rev-parse HEAD)" == "$revision" ]] } -checkout_exact https://github.com/oceanbase/seekdb.git "$SEEKDB_SHA" "$BUILD_ROOT/seekdb" +checkout_exact "$SEEKDB_REPOSITORY" "$SEEKDB_SHA" "$BUILD_ROOT/seekdb" git -C "$BUILD_ROOT/seekdb" merge-base --is-ancestor "$FIX_COMMIT" HEAD checkout_exact https://github.com/oceanbase/seekdb-bindings.git "$BINDINGS_SHA" "$BUILD_ROOT/bindings" git -C "$BUILD_ROOT/bindings" submodule update --init --depth=1 deps/mariadb-connector-c From aa70f5321e724683c8e65aa1925359f9eed1e90b Mon Sep 17 00:00:00 2001 From: cms-cms <1303573631@qq.com> Date: Mon, 21 Sep 2026 18:18:37 +0800 Subject: [PATCH 13/25] refactor: keep pyseekdb pr product-only --- .github/embedded-source.json | 8 - .github/scripts/build-embedded-wheel.sh | 83 ---- .github/scripts/embedded_wheel_manifest.py | 75 ---- .github/workflows/ci.yml | 86 +--- .gitignore | 1 - pyproject.toml | 13 - .../mnn_embedding_function.py | 7 +- tests/integration_tests/EMBEDDED_FULLTEXT.md | 74 ---- tests/integration_tests/conftest.py | 9 +- .../embedded_fulltext_support.py | 409 ------------------ .../test_embedded_fulltext_stability.py | 69 --- ...t_get_or_create_collection_multiprocess.py | 7 +- .../test_embedded_client_lifecycle.py | 30 -- .../test_embedded_fulltext_harness.py | 190 -------- .../test_embedded_wheel_manifest.py | 125 ------ ...t_huggingface_sparse_embedding_function.py | 27 +- .../unit_tests/test_mnn_embedding_function.py | 8 - ...sentence_transformer_embedding_function.py | 16 - uv.lock | 230 +++++++--- 19 files changed, 204 insertions(+), 1263 deletions(-) delete mode 100644 .github/embedded-source.json delete mode 100644 .github/scripts/build-embedded-wheel.sh delete mode 100644 .github/scripts/embedded_wheel_manifest.py delete mode 100644 tests/integration_tests/EMBEDDED_FULLTEXT.md delete mode 100644 tests/integration_tests/embedded_fulltext_support.py delete mode 100644 tests/integration_tests/test_embedded_fulltext_stability.py delete mode 100644 tests/unit_tests/test_embedded_fulltext_harness.py delete mode 100644 tests/unit_tests/test_embedded_wheel_manifest.py diff --git a/.github/embedded-source.json b/.github/embedded-source.json deleted file mode 100644 index b4aa48d7..00000000 --- a/.github/embedded-source.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "seekdb_repository": "https://github.com/cms-cms/seekdb.git", - "seekdb_sha": "3f3b11ab9d6115ee448c2f0f693dfbe7f927c2ea", - "bindings_sha": "5e8eece63205dccfa9500f98893c91d1f50eb4f1", - "fix_commit": "48b80f5193bcac0afafba4d4820aebdf1103bc69", - "platform": "linux-x86_64", - "python": "3.11" -} diff --git a/.github/scripts/build-embedded-wheel.sh b/.github/scripts/build-embedded-wheel.sh deleted file mode 100644 index 7f469273..00000000 --- a/.github/scripts/build-embedded-wheel.sh +++ /dev/null @@ -1,83 +0,0 @@ -#!/usr/bin/env bash -# CI-local Linux wheel: not a portable manylinux release or a PyPI publication. -set -Eeuo pipefail -REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" -: "${RUNNER_TEMP:?use an isolated GitHub runner temporary directory}" -: "${EMBEDDED_WHEEL_DIR:?wheel output/cache directory is required}" -[[ "$(uname -s)/$(uname -m)" == Linux/x86_64 ]] -PYTHON="$(command -v python3)" -"$PYTHON" -c 'import sys; assert sys.version_info[:2] == (3, 11)' -read -r SEEKDB_REPOSITORY SEEKDB_SHA BINDINGS_SHA FIX_COMMIT < <( - "$PYTHON" -c 'import json, sys; p=json.load(open(sys.argv[1])); print(p["seekdb_repository"], p["seekdb_sha"], p["bindings_sha"], p["fix_commit"])' \ - "$REPO_ROOT/.github/embedded-source.json" -) -[[ "$SEEKDB_REPOSITORY" == "https://github.com/cms-cms/seekdb.git" ]] -[[ "$SEEKDB_SHA" =~ ^[0-9a-f]{40}$ && "$BINDINGS_SHA" =~ ^[0-9a-f]{40}$ && "$FIX_COMMIT" =~ ^[0-9a-f]{40}$ ]] -BUILD_ROOT="$(mktemp -d "$RUNNER_TEMP/embedded-source.XXXXXX")" -mkdir -p "$EMBEDDED_WHEEL_DIR" -[[ -z "$(ls -A "$EMBEDDED_WHEEL_DIR")" ]] || { echo 'refusing to overwrite wheel evidence' >&2; exit 1; } - -checkout_exact() { - local repository="$1" revision="$2" target="$3" - git init -q "$target" - git -C "$target" remote add origin "$repository" - git -C "$target" config remote.origin.promisor true - git -C "$target" config remote.origin.partialclonefilter blob:none - # Full ancestry is needed to prove the kernel contains the required fix. - # Historical source blobs are not: fetch them lazily for the exact checkout. - git -C "$target" fetch --filter=blob:none --no-tags origin "$revision" - git -C "$target" checkout --detach FETCH_HEAD - [[ "$(git -C "$target" rev-parse HEAD)" == "$revision" ]] -} -checkout_exact "$SEEKDB_REPOSITORY" "$SEEKDB_SHA" "$BUILD_ROOT/seekdb" -git -C "$BUILD_ROOT/seekdb" merge-base --is-ancestor "$FIX_COMMIT" HEAD -checkout_exact https://github.com/oceanbase/seekdb-bindings.git "$BINDINGS_SHA" "$BUILD_ROOT/bindings" -git -C "$BUILD_ROOT/bindings" submodule update --init --depth=1 deps/mariadb-connector-c - -# Match the upstream seekdb-bindings Ubuntu source-build recipe. These are -# packaging/client tools, not dependencies of the seekdb compilation target. -DEP_PROFILE="$BUILD_ROOT/seekdb/deps/init/oceanbase.el9.x86_64.deps" -sed -i -E '/^[^#]*(target=(obshell|obdeploy)|(lib)?obclient-)/d' "$DEP_PROFILE" - -# Install the source's exact Rust toolchain once, before parallel compilation; -# never let two Rust targets race while auto-installing the same components. -RUST_TOOLCHAIN="$("$PYTHON" -c 'import sys,tomllib; print(tomllib.load(open(sys.argv[1],"rb"))["toolchain"]["channel"])' \ - "$BUILD_ROOT/seekdb/rust/rust-toolchain.toml")" -[[ "$RUST_TOOLCHAIN" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] -export RUSTUP_HOME="$BUILD_ROOT/rustup" CARGO_HOME="$BUILD_ROOT/cargo" -rustup toolchain install "$RUST_TOOLCHAIN" --profile minimal --no-self-update -export RUSTUP_TOOLCHAIN="$RUST_TOOLCHAIN" RUSTUP_AUTO_INSTALL=0 -rustup run "$RUST_TOOLCHAIN" cargo --version -BUILD_JOBS="$(nproc)" -(( BUILD_JOBS <= 3 )) || BUILD_JOBS=3 -MEMORY_JOBS="$(awk '/MemTotal:/ {n=int(($2/1024-3072)/4096); print (n>0?n:1)}' /proc/meminfo)" -(( BUILD_JOBS <= MEMORY_JOBS )) || BUILD_JOBS="$MEMORY_JOBS" -export CMAKE_BUILD_PARALLEL_LEVEL="$BUILD_JOBS" CARGO_BUILD_JOBS="$BUILD_JOBS" -( - cd "$BUILD_ROOT/seekdb" - ./build.sh release --init -DBUILD_EMBED_MODE=ON --make -j"$BUILD_JOBS" -) -mapfile -t binaries < <(find "$BUILD_ROOT/seekdb/build_release" -type f -name seekdb -executable) -[[ "${#binaries[@]}" == 1 ]] -SEEKDB_BIN="${binaries[0]}" -strip --strip-debug --strip-unneeded "$SEEKDB_BIN" -"$SEEKDB_BIN" -V 2>&1 | tee "$EMBEDDED_WHEEL_DIR/seekdb-version.txt" -grep -F "$SEEKDB_SHA" "$EMBEDDED_WHEEL_DIR/seekdb-version.txt" - -# Do not call the upstream wheel target (it selects an unqualified `python`). -# Build only the driver, then build a CPython 3.11 wheel with this exact venv. -"$PYTHON" -m venv "$BUILD_ROOT/venv" -BUILD_PYTHON="$BUILD_ROOT/venv/bin/python" -"$BUILD_PYTHON" -m pip install --disable-pip-version-check \ - 'pip==25.3' 'build==1.3.0' 'cmake==3.31.6' 'nanobind==3.0.0' 'scikit-build-core==0.11.6' 'ninja==1.11.1.4' -export PATH="$BUILD_ROOT/venv/bin:$PATH" -cmake -S "$BUILD_ROOT/bindings" -B "$BUILD_ROOT/bindings/build" \ - -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTING=OFF -DSEEKDB_BUILD_PYTHON=OFF \ - -DSEEKDB_BIN="$SEEKDB_BIN" -cmake --build "$BUILD_ROOT/bindings/build" --target seekdb --parallel "$BUILD_JOBS" -"$BUILD_PYTHON" -m pip wheel --no-deps --no-build-isolation \ - --config-settings="cmake.define.Python_EXECUTABLE=$BUILD_PYTHON" \ - --config-settings="cmake.define.Python_FIND_VIRTUALENV=ONLY" \ - --wheel-dir "$EMBEDDED_WHEEL_DIR" "$BUILD_ROOT/bindings/python" -"$PYTHON" "$REPO_ROOT/.github/scripts/embedded_wheel_manifest.py" create \ - "$REPO_ROOT/.github/embedded-source.json" "$EMBEDDED_WHEEL_DIR" "$SEEKDB_BIN" diff --git a/.github/scripts/embedded_wheel_manifest.py b/.github/scripts/embedded_wheel_manifest.py deleted file mode 100644 index 07c1c38b..00000000 --- a/.github/scripts/embedded_wheel_manifest.py +++ /dev/null @@ -1,75 +0,0 @@ -"""Bind a CI-local wheel to pinned sources and verify the installed payload.""" - -import hashlib -import importlib.metadata -import json -import platform -import subprocess -import sys -import zipfile -from pathlib import Path - - -def digest(path): - with path.open("rb") as stream: - return hashlib.file_digest(stream, "sha256").hexdigest() - - -def wheel_payload(wheel): - with zipfile.ZipFile(wheel) as archive, archive.open("pylibseekdb/seekdb") as stream: - return hashlib.file_digest(stream, "sha256").hexdigest() - - -def verify(pin, directory): - manifest = json.loads((directory / "manifest.json").read_text()) - if manifest["source"] != pin: - raise ValueError("cached wheel source identity mismatch") - if platform.system() != "Linux" or platform.machine() != "x86_64" or sys.version_info[:2] != (3, 11): - raise ValueError("wheel requires Linux x86_64 CPython 3.11") - wheels = list(directory.glob("*.whl")) - if len(wheels) != 1 or wheels[0].name != manifest["wheel"]: - raise ValueError("expected exactly the manifest wheel") - if digest(wheels[0]) != manifest["wheel_sha256"]: - raise ValueError("wheel checksum mismatch") - if wheel_payload(wheels[0]) != manifest["binary_sha256"]: - raise ValueError("wheel contains a different SeekDB binary") - version = (directory / "seekdb-version.txt").read_text() - if pin["seekdb_sha"] not in version or version != manifest["revision_output"]: - raise ValueError("SeekDB full REVISION mismatch") - return manifest - - -def main(): - mode, pin_path, directory_name, *rest = sys.argv[1:] - pin = json.loads(Path(pin_path).read_text()) - directory = Path(directory_name) - if mode == "create": - (binary_name,) = rest - (wheel,) = directory.glob("*.whl") - manifest = { - "source": pin, - "wheel": wheel.name, - "wheel_sha256": digest(wheel), - "binary_sha256": digest(Path(binary_name)), - "revision_output": (directory / "seekdb-version.txt").read_text(), - } - (directory / "manifest.json").write_text(json.dumps(manifest, indent=2) + "\n") - elif mode not in {"verify", "installed"}: - raise ValueError(f"unknown verification mode: {mode}") - manifest = verify(pin, directory) - if mode == "installed": - binary = Path(importlib.metadata.distribution("pylibseekdb").locate_file("pylibseekdb/seekdb")) - if digest(binary) != manifest["binary_sha256"]: - raise ValueError("installed SeekDB differs from source-built wheel (dependency resync?)") - # The executable is the verified local wheel payload, never shell input. - version = subprocess.check_output( # noqa: S603 - [str(binary), "-V"], text=True, stderr=subprocess.STDOUT - ) - if pin["seekdb_sha"] not in version: - raise ValueError("installed SeekDB does not report pinned full REVISION") - print(f"[EMBEDDED_WHEEL] installed={binary}") - print(json.dumps(manifest, indent=2)) - - -if __name__ == "__main__": - main() diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 93f10e9c..7fc585d3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -100,11 +100,7 @@ jobs: bash .github/scripts/check-pytest-summary.sh pytest.log integration-test: - # Keep the CI-local source wheel on the same native Ubuntu image as its tests. - runs-on: ${{ matrix.test_mode == 'embedded' && 'ubuntu-22.04' || 'ubuntu-latest' }} - timeout-minutes: 180 - env: - EMBEDDED_WHEEL_DIR: ${{ github.workspace }}/.embedded-wheel + runs-on: ubuntu-latest strategy: fail-fast: false # Allow other jobs to continue if one fails matrix: @@ -141,61 +137,6 @@ jobs: with: python-version: '3.11' - - name: Restore default ONNX model cache - if: matrix.test_mode == 'oceanbase' - uses: actions/cache@v4 - with: - path: ~/.cache/pyseekdb/onnx_models - key: pyseekdb-onnx-all-MiniLM-L6-v2-v1 - - - name: Restore exact embedded source wheel - if: matrix.test_mode == 'embedded' - id: embedded-wheel - uses: actions/cache/restore@v4 - with: - path: .embedded-wheel - # Deliberately no restore-keys: another source/build recipe is not reusable. - key: embedded-source-ubuntu22-x86_64-cp311-${{ hashFiles('.github/embedded-source.json', '.github/scripts/build-embedded-wheel.sh', '.github/scripts/embedded_wheel_manifest.py') }} - - - name: Install embedded source build dependencies - if: matrix.test_mode == 'embedded' && steps.embedded-wheel.outputs.cache-hit != 'true' - run: | - sudo apt-get update - sudo apt-get install -y git wget rpm rpm2cpio cpio make build-essential binutils m4 libtool-bin libncurses5 libaio1 libssl-dev zlib1g-dev - - - name: Build embedded wheel from pinned fixed sources - if: matrix.test_mode == 'embedded' && steps.embedded-wheel.outputs.cache-hit != 'true' - run: bash .github/scripts/build-embedded-wheel.sh - - - name: Verify and install the exact embedded source wheel - if: matrix.test_mode == 'embedded' - run: | - set -euo pipefail - # Needed even when the build cache was restored on a fresh runner. - sudo apt-get update - sudo apt-get install -y libaio1 - python .github/scripts/embedded_wheel_manifest.py verify .github/embedded-source.json "$EMBEDDED_WHEEL_DIR" - uv pip install --python .venv/bin/python --no-deps --reinstall "$EMBEDDED_WHEEL_DIR"/*.whl - .venv/bin/python .github/scripts/embedded_wheel_manifest.py installed .github/embedded-source.json "$EMBEDDED_WHEEL_DIR" - - - name: Save verified embedded source wheel - if: matrix.test_mode == 'embedded' && steps.embedded-wheel.outputs.cache-hit != 'true' - uses: actions/cache/save@v4 - with: - path: .embedded-wheel - key: ${{ steps.embedded-wheel.outputs.cache-primary-key }} - - - name: Record embedded wheel provenance - if: matrix.test_mode == 'embedded' - uses: actions/upload-artifact@v4 - with: - name: embedded-wheel-provenance - include-hidden-files: true - path: | - .embedded-wheel/manifest.json - .embedded-wheel/seekdb-version.txt - if-no-files-found: error - - name: Start OceanBase container if: matrix.test_mode == 'oceanbase' uses: oceanbase/setup-oceanbase-ce@v1 @@ -219,9 +160,6 @@ jobs: - name: Run integration tests for ${{ matrix.test_mode }} env: CI: 1 - # GitHub-hosted runners can reach the canonical endpoint reliably; - # avoid making the suite depend on the China mirror's DNS state. - HF_ENDPOINT: https://huggingface.co SEEKDB_TEST_DATA_ROOT: ${{ github.workspace }}/.seekdb-test-data OB_PORT: 10000 SERVER_PORT: 2881 @@ -233,28 +171,10 @@ jobs: log_level=INFO fi set -o pipefail - # Do not let uv replace the verified source wheel with uv.lock's release. - uv run --no-sync pytest tests/integration_tests/ -v --log-cli-level=${log_level} \ + uv run pytest tests/integration_tests/ -v --log-cli-level=${log_level} \ -k "${{ matrix.test_mode }}" | tee pytest.log bash .github/scripts/check-pytest-summary.sh pytest.log - - name: Preserve embedded full-text failure evidence - if: failure() && matrix.test_mode == 'embedded' - uses: actions/upload-artifact@v4 - with: - name: embedded-fulltext-failure-${{ github.run_id }}-${{ github.run_attempt }} - # Only harness reports and bounded log excerpts, never database files. - include-hidden-files: true - path: | - .seekdb-test-data/artifacts/fts-*/summary.json - .seekdb-test-data/artifacts/fts-*/first-error.json - .seekdb-test-data/artifacts/fts-*/errors.jsonl - .seekdb-test-data/artifacts/fts-*/*-db-context.log - .seekdb-test-data/artifacts/fts-*/*-log-inventory.json - .seekdb-test-data/artifacts/fts-*/worker.log - if-no-files-found: warn - retention-days: 14 - - name: Run examples for ${{ matrix.test_mode }} env: MODE: ${{ matrix.test_mode }} @@ -271,7 +191,7 @@ jobs: fi echo "Testing ${example_file} in ${MODE} mode" - if uv run --no-sync python "$example_file"; then + if uv run python "$example_file"; then echo "${example_file} passed" else status=$? diff --git a/.gitignore b/.gitignore index 19c58691..3c774da3 100644 --- a/.gitignore +++ b/.gitignore @@ -19,4 +19,3 @@ ENV/ env.bak/ venv.bak/ demo/rag/data/ -.embedded-wheel/ diff --git a/pyproject.toml b/pyproject.toml index 207b8c50..4219de97 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,24 +46,11 @@ unit-tests = [ "voyageai", "sentence-transformers>=5.0", "huggingface-hub>=0.26", - # Declare torch directly so Linux CI resolves the CPU wheel instead of the - # default PyPI CUDA dependency set pulled indirectly by transformers. - "torch==2.9.1", # pylibseekdb 1.3.x requires an ABI=1 native runtime to be loaded first # in embedded integration environments. This is not a runtime dependency. "onnxruntime>=1.19.0", ] -[tool.uv.sources] -torch = [ - { index = "pytorch-cpu", marker = "sys_platform == 'linux' and platform_machine == 'x86_64'" }, -] - -[[tool.uv.index]] -name = "pytorch-cpu" -url = "https://download.pytorch.org/whl/cpu" -explicit = true - [tool.uv.workspace] members = ["demo/rag"] diff --git a/src/pyseekdb/utils/embedding_functions/mnn_embedding_function.py b/src/pyseekdb/utils/embedding_functions/mnn_embedding_function.py index b017b8b7..1fd75ee2 100644 --- a/src/pyseekdb/utils/embedding_functions/mnn_embedding_function.py +++ b/src/pyseekdb/utils/embedding_functions/mnn_embedding_function.py @@ -84,8 +84,11 @@ def __init__( self.hf_revision = hf_revision self._dimension = dimension self._expected_sha256 = dict(expected_sha256 or {}) - cache_root = Path(os.environ.get("PYSEEKDB_CACHE_DIR", Path.home() / ".cache" / "pyseekdb")) - self.download_path = download_path if download_path is not None else cache_root / "mnn_models" / model_name + self.download_path = ( + download_path + if download_path is not None + else Path.home() / ".cache" / "pyseekdb" / "mnn_models" / model_name + ) # These imports are intentionally lazy so importing pyseekdb does not # initialize the native MNN runtime until embeddings are requested. diff --git a/tests/integration_tests/EMBEDDED_FULLTEXT.md b/tests/integration_tests/EMBEDDED_FULLTEXT.md deleted file mode 100644 index 247626db..00000000 --- a/tests/integration_tests/EMBEDDED_FULLTEXT.md +++ /dev/null @@ -1,74 +0,0 @@ -# Embedded full-text stability (seekdb #1382) - -Run the same tests natively on Linux x86_64 and macOS arm64 with a selected -`pylibseekdb` wheel and this PySeekDB source checkout: - -```sh -SEEKDB_TEST_DATA_ROOT=/absolute/path/on/local-disk/fts-data \ -SEEKDB_TEST_ARTIFACT_ROOT=/absolute/path/to/artifacts \ -python -m pytest tests/integration_tests/test_embedded_fulltext_stability.py -v -s -``` - -Use real local disk, not a Linux tmpfs mount that rejects direct I/O. Each case -uses a new subprocess and database directory; it does not reopen an already -warmed database. The two cases participate in the existing `-k embedded` suite. - -* `first-pass`: 24,515 deterministic synthetic documents, supplied 8-dimensional - embeddings (no model/network download), SDK upsert and refresh_index, followed - immediately by four passes of 300 public `collection.hybrid_search` full-text - queries (`knn=None`). Pass zero is included in the verdict. No warm-up query, - sleeps or retries erase an initial failure. -* `scan-freeze`: a separate table with a full-text index and **no vector index**, - 24,515 preloaded documents, eight independently connected scanning threads, - and one writer inserting 32 rows then executing `ALTER SYSTEM MINOR FREEZE` - every approximately 120 ms. A barrier starts the 75-second window only when - all connections are ready. Each scanner must execute successfully and at - least one freeze must complete; all errors, not just 4016, fail the test. - -This preserves the original API failure surface and the accelerated scan/release -trigger separately. It is **not** a byte-identical replay of the original -SciFact/384-dimensional embedding benchmark, and a single passing stress run -does not establish that a probabilistic bug is absent. - -`summary.json` records runtime/version, counts, first failure and extracted trace -IDs. `errors.jsonl` preserves up to 200 failures, while the total is unbounded. -The first error immediately triggers bounded database-log context collection; -it does not wait for the pressure window to end. Log scans use a total byte -budget and include tails of rotated files larger than 256 MiB. Inventories -explicitly report scan/output truncation. `last_trace_id()` is best-effort on -the failing connection, not a guaranteed mapping to an SDK internal statement. - -`max_syslog_file_count=50` bounds the normal rotated set; it is not a guarantee -of long-term retention. Successful cases remove only their generated database -directory after their worker exits and diagnostic collection completes; failed -cases retain it. Evidence lives outside that directory. CI should archive it -before any later retention cleanup. - -For pre/post-fix comparisons, hold this test SHA and bindings source SHA fixed, -record the actual SeekDB REVISION inside each wheel, and change only the kernel -source revision. Old-build failures must retain a failed result. Do not add an -xfail or suppress 4016 to make the comparison green. - -## GitHub PR gate and released wheels - -The GitHub embedded job builds a **CI-local Linux x86_64 / CPython 3.11** wheel -from the full SeekDB and bindings commits in `.github/embedded-source.json`. -The pinned SeekDB commit contains the #1384 fix; the build verifies that ancestry. -This wheel is not published and does not claim portable manylinux compatibility. -The kernel build uses at most three workers, further limited by available RAM. - -The cache key includes both source pins, the build/verification scripts, Ubuntu -22.04 and CPython 3.11. No approximate cache key is accepted. On every restore, -CI verifies the manifest, wheel and embedded binary SHA-256 and full REVISION. -After installation it verifies the actual installed binary again. Integration -tests and examples use `uv run --no-sync` so the lock file cannot silently -replace the tested wheel with the old PyPI release. All embedded tests still run; -neither these two regressions nor database errors are skipped or marked xfail. - -The lock file remains unchanged for the other jobs. This source-built gate does -**not** prove that the published wheel is fixed: run 35204502666 used the locked -`pylibseekdb 1.3.0` and failed both new scenarios with explicit 4016 errors (one -first-pass error and 72 scan errors, whose first error was 4016). A released wheel -containing #1384 can replace the temporary source-build path after verification. -Failed full-text reports and bounded database-log excerpts are uploaded as -artifacts; whole database directories are not uploaded. diff --git a/tests/integration_tests/conftest.py b/tests/integration_tests/conftest.py index 0d771584..5eda1f80 100644 --- a/tests/integration_tests/conftest.py +++ b/tests/integration_tests/conftest.py @@ -4,7 +4,6 @@ """ import contextlib -import importlib.util import os import sys from pathlib import Path @@ -48,7 +47,9 @@ # ==================== Client Factory Functions ==================== def create_embedded_client(): """Create an embedded client instance.""" - if not any(importlib.util.find_spec(name) for name in ("seekdb", "pylibseekdb")): + try: + import pylibseekdb # noqa: F401 + except ImportError: pytest.skip("seekdb embedded package is not installed") return pyseekdb.Client(path=SEEKDB_PATH, database=SEEKDB_DATABASE) @@ -99,7 +100,9 @@ def create_oceanbase_client(): # ==================== AdminClient Factory Functions ==================== def create_embedded_admin_client(): """Create an embedded admin client instance.""" - if not any(importlib.util.find_spec(name) for name in ("seekdb", "pylibseekdb")): + try: + import pylibseekdb # noqa: F401 + except ImportError: pytest.skip("seekdb embedded package is not installed") return pyseekdb.AdminClient(path=SEEKDB_PATH) diff --git a/tests/integration_tests/embedded_fulltext_support.py b/tests/integration_tests/embedded_fulltext_support.py deleted file mode 100644 index 19514013..00000000 --- a/tests/integration_tests/embedded_fulltext_support.py +++ /dev/null @@ -1,409 +0,0 @@ -"""Isolated, dependency-free harness; pyseekdb is imported only in the worker. - -Synthetic deterministic documents (not the original SciFact benchmark). The -scan/freezer uses the same SQL shape as the Linux issue #1382 reproducer, without -a vector index. Public hybrid_search coverage lives in the separate first pass. -""" - -from __future__ import annotations - -import argparse -import collections -import importlib.metadata -import importlib.util -import json -import platform -import re -import threading -import time -import traceback -from pathlib import Path - -ROWS = 24515 -WORKERS = 8 -SECONDS = 75 -INTERVAL = 0.120 -WRITE_BATCH = 32 -QUERIES = 300 -PASSES = 4 -TEXT = "the database retrieval benchmark" -TABLE = "`c$v2$16d82ab3afe311f186fe000000000000`" -SEARCH = ( - f"SELECT `_id`, MATCH(`document`) AGAINST('{TEXT}') AS relevance FROM {TABLE} " - f"WHERE MATCH(`document`) AGAINST('{TEXT}') ORDER BY relevance DESC LIMIT 10" -) -TRACE = re.compile(r"\bY[A-Za-z0-9]+-[A-Za-z0-9]+-\d+-\d+\b") - - -def document(ordinal): - """Every row matches the broad query while cohorts supply varied queries.""" - return ( - f"{TEXT} exercises full text postings during concurrent memtable release " - f"cohort{ordinal % QUERIES:03d} document ordinal{ordinal:08d}" - ) - - -def error_details(error): - """Retain wrapper/cause messages and only report explicit numeric codes.""" - chain, seen, codes = [], set(), [] - while error is not None and id(error) not in seen: - seen.add(id(error)) - chain.append(f"{type(error).__name__}: {error}") - if error.args and isinstance(error.args[0], int): - codes.append(error.args[0]) - codes.extend(int(value) for value in re.findall(r"\bcode[=:]\s*(-?\d+)\b", str(error))) - # Released bindings may expose a RuntimeError rather than numeric args, - # e.g. "execute sql failed OB_ERR_UNEXPECTED(4016): %s". - codes.extend(int(value) for value in re.findall(r"\bOB_[A-Z0-9_]+\((-?\d+)\)", str(error))) - error = error.__cause__ or error.__context__ - text = "\n".join(chain) - return {"exception_chain": chain, "error_codes": sorted(set(codes)), "trace_ids": sorted(set(TRACE.findall(text)))} - - -def capture_logs(root, evidence, label, traces=(), budget=128 * 1024 * 1024, output_limit=2 * 1024 * 1024): - """Scan once under a byte budget; prioritize errors over routine context. - - Each priority buffer is bounded by output_limit (at most 4x that limit in - memory). A full low-priority buffer must not stop the scan: the failing - trace can occur later in a large, noisy rotated log. - """ - if budget < 0 or output_limit <= 0: - raise ValueError("invalid diagnostic byte budget") - trace_patterns = [value.encode() for value in traces if value] - error_patterns = [ - b"-4016", - b"data/schema type does not match", - b"failed to get next row from memtable scanner", - b"ob_memtable_key.h", - ] - files = sorted( - (p for p in root.rglob("seekdb.log*") if p.is_file() and not p.is_symlink()), - key=lambda p: p.stat().st_mtime, - reverse=True, - ) - inventory = [] - priorities = ("trace-error", "trace", "error", "memtable-context") - buffers = [bytearray() for _ in priorities] - matched = [0] * len(priorities) - capped = [False] * len(priorities) - - def append(priority, content): - available = output_limit - len(buffers[priority]) - if len(content) > available: - capped[priority] = True - buffers[priority].extend(content[:available]) - - for path in files: - size = path.stat().st_size - count = min(size, budget) - inventory.append({"path": str(path), "size": size, "read_bytes": count}) - if not count: - continue - with path.open("rb") as stream: - stream.seek(size - count) - previous, following, remaining = collections.deque(maxlen=5), 0, count - context_priority = 3 - while remaining > 0: - line = stream.readline(min(65536, remaining)) - if not line: - break - remaining -= len(line) - is_trace = any(pattern in line for pattern in trace_patterns) - is_error = any(pattern in line for pattern in error_patterns) - priority = ( - 0 - if is_trace and is_error - else 1 - if is_trace - else 2 - if is_error - else 3 - if b"read_barrier_:true" in line - else None - ) - if priority is not None: - # Preserve the matching line before optional context so - # a huge preceding SQL line cannot consume the budget. - context = f"\nFILE={path}\n".encode() + line + b"[preceding context]\n" + b"".join(previous) - matched[priority] += 1 - append(priority, context) - following, context_priority = 5, priority - elif following: - append(context_priority, line) - following -= 1 - previous.append(line) - budget -= count - written = 0 - with (evidence / f"{label}-db-context.log").open("wb") as output: - for content in buffers: - part = content[: max(0, output_limit - written)] - output.write(part) - written += len(part) - (evidence / f"{label}-log-inventory.json").write_text( - json.dumps( - { - "files": inventory, - "context_bytes": written, - "context_capped": written >= output_limit, - "priority_matches": dict(zip(priorities, matched, strict=True)), - "priority_capped": dict(zip(priorities, capped, strict=True)), - "scan_capped": any(item["read_bytes"] < item["size"] for item in inventory), - }, - indent=2, - ) - ) - - -class Recorder: - """Flush errors immediately and snapshot the first failure before recycling.""" - - def __init__(self, root, evidence): - self.root, self.evidence = root, evidence - self.lock = threading.Lock() - self.errors = 0 - self.traces = set() - self.first = None - - def error(self, error, phase, client=None, **details): - event = {"time": time.time(), "phase": phase, **details, **error_details(error)} - if not event["trace_ids"] and client is not None: - try: - # Immediately on the failing worker's own connection; mark it - # as best-effort, not proof of a particular internal SDK SQL. - rows = client._server._execute("SELECT last_trace_id() FROM DUAL") - event["trace_ids"] = TRACE.findall(str(rows)) - event["trace_source"] = "same_connection_last_trace_id_best_effort" - except Exception as lookup_error: - event["trace_lookup_error"] = str(lookup_error) - with self.lock: - self.errors += 1 - self.traces.update(event["trace_ids"]) - first = self.first is None - if first: - self.first = event - # Keep a bounded sample but never lose the total failure count. - if self.errors <= 200: - with (self.evidence / "errors.jsonl").open("a") as stream: - stream.write(json.dumps(event) + "\n") - if first: - (self.evidence / "first-error.json").write_text(json.dumps(event, indent=2)) - if first: - print(f"[FTS][FIRST_ERROR] {json.dumps(event)}", flush=True) - capture_logs(self.root, self.evidence, "early", event["trace_ids"]) - - -def validate_report(report, scenario): - """A zero worker exit alone is insufficient with native binding teardown.""" - assert report.get("scenario") == scenario, report - assert report.get("completed") is True and report.get("errors") == 0, report - assert report.get("rows") == ROWS, report - if scenario == "first-pass": - assert report.get("first_pass_successes") == QUERIES, report - assert report.get("fts_successes") == QUERIES * PASSES, report - else: - counts = report.get("worker_successes", []) - assert len(counts) == WORKERS and all(value > 0 for value in counts), report - assert report.get("freeze_successes", 0) > 0, report - assert report.get("elapsed_seconds", 0) >= SECONDS, report - - -def first_pass(client, recorder, report): - import pyseekdb - - collection = client.create_collection( - "fts_first_pass", - configuration=pyseekdb.HNSWConfiguration(dimension=8, distance="cosine"), - embedding_function=None, - ) - vector = [float(i + 1) / 8 for i in range(8)] - for start in range(0, ROWS, 100): - indexes = range(start, min(start + 100, ROWS)) - collection.upsert( - ids=[f"doc-{i}" for i in indexes], - documents=[document(i) for i in indexes], - embeddings=[vector for _ in indexes], - metadatas=[{"ordinal": i} for i in indexes], - ) - if start % 1000 == 0: - print(f"[FTS] ingest={min(start + 100, ROWS)}/{ROWS}", flush=True) - assert collection.count() == ROWS - collection.refresh_index() - report.update(rows=ROWS, first_pass_successes=0, fts_successes=0) - # No FTS warm-up, sleep, retry or reopen before pass zero. - for pass_id in range(PASSES): - for query_id in range(QUERIES): - try: - result = collection.hybrid_search( - query={"where_document": {"$contains": f"{TEXT} cohort{query_id:03d}"}, "n_results": 10}, - knn=None, - n_results=10, - include=["documents", "metadatas"], - ) - assert result.get("ids") and result["ids"][0], result - report["fts_successes"] += 1 - if pass_id == 0: - report["first_pass_successes"] += 1 - except Exception as error: - recorder.error( - error, - "hybrid_search", - client, - pass_id=pass_id, - query_id=query_id, - query_text=f"{TEXT} cohort{query_id:03d}", - ) - print(f"[FTS] pass={pass_id} successes={report['fts_successes']} errors={recorder.errors}", flush=True) - - -def insert_batch(client, start, size): - # IDs/documents are generated here, not user-controlled SQL inputs. - values = ",".join(f"('doc-{i}', '{document(i)}', '{{\"ordinal\":{i}}}')" for i in range(start, start + size)) - client._server._execute(f"INSERT INTO {TABLE} (`_id`, `document`, `metadata`) VALUES {values}") - - -def scan_freeze(client, make_client, recorder, report): - client._server._execute( - f"CREATE TABLE {TABLE} (`__pk_increment` BIGINT NOT NULL AUTO_INCREMENT PRIMARY KEY, " - "`_id` VARCHAR(128) NOT NULL, `document` LONGTEXT NOT NULL, `metadata` JSON NOT NULL, " - "UNIQUE KEY uk_id (`_id`), FULLTEXT KEY idx_document (`document`) WITH PARSER space)" - ) - for start in range(0, ROWS, 100): - insert_batch(client, start, min(100, ROWS - start)) - if start % 1000 == 0: - print(f"[FTS] ingest={min(start + 100, ROWS)}/{ROWS}", flush=True) - counts = client._server._execute(f"SELECT COUNT(*) AS row_count FROM {TABLE}") - count = counts[0]["row_count"] if isinstance(counts[0], dict) else counts[0][0] - assert count == ROWS, counts - schema = client._server._execute(f"SHOW CREATE TABLE {TABLE}") - assert "FULLTEXT" in str(schema).upper() and "VECTOR" not in str(schema).upper(), schema - report.update( - rows=ROWS, - schema=str(schema), - search_sql=SEARCH, - write_batch_size=WRITE_BATCH, - worker_successes=[0] * WORKERS, - freeze_successes=0, - ) - stop = threading.Event() - deadline = [0.0] - started = [0.0] - - def start_window(): - started[0] = time.monotonic() - deadline[0] = started[0] + SECONDS - - barrier = threading.Barrier(WORKERS + 2, action=start_window) - - def worker(number): - connection = None - try: - connection = make_client() - connection._server._execute("SELECT 1") - barrier.wait(timeout=60) - while time.monotonic() < deadline[0] and not stop.is_set(): - started = time.monotonic() - try: - if number == WORKERS: - start = ROWS + report["freeze_successes"] * WRITE_BATCH - insert_batch(connection, start, WRITE_BATCH) - connection._server._execute("ALTER SYSTEM MINOR FREEZE") - report["freeze_successes"] += 1 - else: - rows = connection._server._execute(SEARCH) - assert rows, "full-text scan returned no rows" - report["worker_successes"][number] += 1 - except Exception as error: - recorder.error(error, "minor-freeze" if number == WORKERS else "scan", connection, worker=number) - if number == WORKERS: - stop.set() # No valid pressure if writes/freezes cannot execute. - if number == WORKERS: - stop.wait(max(0, INTERVAL - (time.monotonic() - started))) - except Exception as error: - recorder.error(error, "worker-setup", connection, worker=number) - barrier.abort() - stop.set() - finally: - if connection is not None: - try: - connection.close() - except Exception as error: - recorder.error(error, "worker-close", worker=number) - - threads = [threading.Thread(target=worker, args=(i,), daemon=True) for i in range(WORKERS + 1)] - for thread in threads: - thread.start() - try: - barrier.wait(timeout=60) - while not stop.wait(min(5, max(0, deadline[0] - time.monotonic()))) and time.monotonic() < deadline[0]: - print( - f"[FTS] scans={sum(report['worker_successes'])} freezes={report['freeze_successes']} " - f"errors={recorder.errors}", - flush=True, - ) - finally: - for thread in threads: - thread.join(timeout=65) - assert not any(thread.is_alive() for thread in threads), "FTS worker did not drain" - report["elapsed_seconds"] = time.monotonic() - started[0] - - -def run(scenario, root, evidence): - import pyseekdb - - runtime_distribution = "seekdb" if importlib.util.find_spec("seekdb") else "pylibseekdb" - - report = { - "scenario": scenario, - "completed": False, - "platform": platform.platform(), - "pyseekdb": importlib.metadata.version("pyseekdb"), - "embedded_runtime": runtime_distribution, - "embedded_runtime_version": importlib.metadata.version(runtime_distribution), - "corpus": "synthetic deterministic / 24515 documents", - "workers": WORKERS, - "duration_seconds": SECONDS, - "freeze_interval_seconds": INTERVAL, - } - recorder = Recorder(root, evidence) - client = None - try: - db_dir = root / "database" - assert not db_dir.exists(), "fresh database required" - - def make_client(): - return pyseekdb.Client(path=str(db_dir), database="test") - - client = make_client() - report["database_version"] = str(client._server._execute("SELECT VERSION()")) - client._server._execute("ALTER SYSTEM SET max_syslog_file_count = 50") - print(f"[FTS] start {json.dumps(report)}", flush=True) - if scenario == "first-pass": - first_pass(client, recorder, report) - else: - scan_freeze(client, make_client, recorder, report) - report["completed"] = True - except Exception as error: - recorder.error(error, "scenario", client) - traceback.print_exc() - finally: - if client is not None: - try: - client.close() - except Exception as error: - recorder.error(error, "close") - try: - capture_logs(root, evidence, "final", recorder.traces) - except Exception as error: - recorder.error(error, "diagnostics") - report.update(errors=recorder.errors, first_error=recorder.first, trace_ids=sorted(recorder.traces)) - (evidence / "summary.json").write_text(json.dumps(report, indent=2)) - validate_report(report, scenario) - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument("--scenario", choices=["first-pass", "scan-freeze"], required=True) - parser.add_argument("--root", type=Path, required=True) - parser.add_argument("--evidence", type=Path, required=True) - options = parser.parse_args() - run(options.scenario, options.root, options.evidence) diff --git a/tests/integration_tests/test_embedded_fulltext_stability.py b/tests/integration_tests/test_embedded_fulltext_stability.py deleted file mode 100644 index 292412a3..00000000 --- a/tests/integration_tests/test_embedded_fulltext_stability.py +++ /dev/null @@ -1,69 +0,0 @@ -"""Fresh-ingestion and memtable-release regressions for seekdb issue #1382. - -Each case runs in a fresh process and database directory, including with legacy -process-global pylibseekdb. Do not rerun failed queries to obtain a passing result. -""" - -import json -import os -import shutil -import subprocess -import sys -import tempfile -import time -from pathlib import Path - -import pytest -from embedded_fulltext_support import validate_report - - -@pytest.mark.parametrize("scenario", ["first-pass", "scan-freeze"]) -def test_embedded_fulltext_stability(scenario): - """Exercise the public SDK first pass and an accelerated kernel stress case.""" - repo = Path(__file__).resolve().parents[2] - data_root = Path(os.environ.get("SEEKDB_TEST_DATA_ROOT", repo / ".seekdb-test-data")) - data_root.mkdir(parents=True, exist_ok=True) - root = Path(tempfile.mkdtemp(prefix=f"fts-{scenario}-", dir=data_root)) - evidence_root = Path(os.environ.get("SEEKDB_TEST_ARTIFACT_ROOT", data_root / "artifacts")) - evidence = evidence_root / root.name - evidence.mkdir(parents=True, exist_ok=False) - command = [ - sys.executable, - str(Path(__file__).with_name("embedded_fulltext_support.py")), - "--scenario", - scenario, - "--root", - str(root), - "--evidence", - str(evidence), - ] - # A whole-case timeout includes ingestion and connection setup; the pressure - # window itself is 75 seconds, starting only after every worker is ready. - with (evidence / "worker.log").open("w") as log: - # Fixed interpreter/helper and generated paths, never shell input. - process = subprocess.Popen(command, stdout=log, stderr=subprocess.STDOUT) # noqa: S603 - started = time.monotonic() - try: - with (evidence / "worker.log").open() as progress: - while process.poll() is None: - if time.monotonic() - started > 900: - raise TimeoutError(f"FTS worker exceeded 900s; evidence={evidence}") - # Stream bounded real output rather than hiding all ingestion - # and pressure activity inside pytest until the case ends. - text = progress.read(16384) - if text: - print(text, end="", flush=True) - time.sleep(1) - print(progress.read(), end="", flush=True) - finally: - if process.poll() is None: - process.kill() - process.wait(timeout=10) - report_path = evidence / "summary.json" - assert report_path.is_file(), f"worker produced no terminal report; evidence={evidence}" - report = json.loads(report_path.read_text()) - validate_report(report, scenario) - assert process.returncode == 0, f"worker rc={process.returncode}; evidence={evidence}" - # Remove only our generated DB after the process has exited and evidence has - # been collected. Failed cases retain their DB for investigation. - shutil.rmtree(root) diff --git a/tests/integration_tests/test_get_or_create_collection_multiprocess.py b/tests/integration_tests/test_get_or_create_collection_multiprocess.py index 8b4d24f0..248a5028 100644 --- a/tests/integration_tests/test_get_or_create_collection_multiprocess.py +++ b/tests/integration_tests/test_get_or_create_collection_multiprocess.py @@ -146,16 +146,15 @@ def _refresh_collection(client_config: dict[str, Any], collection_name: str) -> def _require_embedded_pylibseekdb() -> None: """Require embedded pylibseekdb.""" - runtime_distribution = "seekdb" if importlib.util.find_spec("seekdb") else "pylibseekdb" try: - importlib.import_module(runtime_distribution) + import pylibseekdb # noqa: F401 except ImportError: pytest.skip("seekdb embedded package is not installed") try: - installed_version = Version(importlib.metadata.version(runtime_distribution)) + installed_version = Version(importlib.metadata.version("pylibseekdb")) except importlib.metadata.PackageNotFoundError: - pytest.skip("seekdb embedded package is not installed") + pytest.skip("pylibseekdb is not installed") if installed_version < MIN_PYLIBSEEKDB_VERSION: pytest.skip( diff --git a/tests/unit_tests/test_embedded_client_lifecycle.py b/tests/unit_tests/test_embedded_client_lifecycle.py index d1053b6c..748d87fd 100644 --- a/tests/unit_tests/test_embedded_client_lifecycle.py +++ b/tests/unit_tests/test_embedded_client_lifecycle.py @@ -133,11 +133,9 @@ def connect(self, *, database: str, autocommit: bool) -> _FakeConnection: def embedded_module(monkeypatch: pytest.MonkeyPatch): """Import the embedded module without loading the native pylibseekdb extension.""" module_name = "pyseekdb.client.client_seekdb_embedded" - monkeypatch.setitem(sys.modules, "seekdb", ModuleType("seekdb")) monkeypatch.setitem(sys.modules, "pylibseekdb", ModuleType("pylibseekdb")) sys.modules.pop(module_name, None) module = importlib.import_module(module_name) - assert module._EMBEDDED_RUNTIME_DISTRIBUTION == "seekdb" yield module sys.modules.pop(module_name, None) @@ -147,34 +145,6 @@ def _install_fake_seekdb(embedded_module: Any, monkeypatch: pytest.MonkeyPatch, monkeypatch.setattr(embedded_module, "_PYLIBSEEKDB_AVAILABLE", True) -def test_embedded_runtime_prefers_current_seekdb_package(monkeypatch, embedded_module) -> None: - current = ModuleType("seekdb") - legacy = ModuleType("pylibseekdb") - modules = {"seekdb": current, "pylibseekdb": legacy} - monkeypatch.setattr(embedded_module.importlib, "import_module", modules.__getitem__) - - runtime, distribution = embedded_module._load_embedded_runtime() - - assert runtime is current - assert distribution == "seekdb" - - -def test_embedded_runtime_falls_back_to_legacy_package(monkeypatch, embedded_module) -> None: - legacy = ModuleType("pylibseekdb") - - def import_runtime(name: str) -> ModuleType: - if name == "seekdb": - raise ImportError(name) - return legacy - - monkeypatch.setattr(embedded_module.importlib, "import_module", import_runtime) - - runtime, distribution = embedded_module._load_embedded_runtime() - - assert runtime is legacy - assert distribution == "pylibseekdb" - - def test_clients_use_and_close_their_own_seekdb_instances(tmp_path, monkeypatch, embedded_module) -> None: fake_seekdb = _FakeInstanceApi() _install_fake_seekdb(embedded_module, monkeypatch, fake_seekdb) diff --git a/tests/unit_tests/test_embedded_fulltext_harness.py b/tests/unit_tests/test_embedded_fulltext_harness.py deleted file mode 100644 index 7d2db014..00000000 --- a/tests/unit_tests/test_embedded_fulltext_harness.py +++ /dev/null @@ -1,190 +0,0 @@ -"""Contracts for the regression harness; these do not require a database.""" - -import importlib.util -import json -import threading -import time -from pathlib import Path -from types import SimpleNamespace - -import pytest - -spec = importlib.util.spec_from_file_location( - "fts_support", Path(__file__).parents[1] / "integration_tests/embedded_fulltext_support.py" -) -fts = importlib.util.module_from_spec(spec) -spec.loader.exec_module(fts) - - -def report(scenario): - return { - "scenario": scenario, - "completed": True, - "errors": 0, - "rows": fts.ROWS, - "first_pass_successes": fts.QUERIES, - "fts_successes": fts.QUERIES * fts.PASSES, - "worker_successes": [1] * fts.WORKERS, - "freeze_successes": 1, - "elapsed_seconds": fts.SECONDS, - } - - -@pytest.mark.parametrize("scenario", ["first-pass", "scan-freeze"]) -def test_valid_report(scenario): - fts.validate_report(report(scenario), scenario) - - -@pytest.mark.parametrize( - "scenario,field,value", - [ - ("first-pass", "first_pass_successes", 299), - ("first-pass", "fts_successes", 1199), - ("first-pass", "errors", 1), - ("first-pass", "completed", False), - ("scan-freeze", "errors", 1), - ("scan-freeze", "worker_successes", [0] * 8), - ("scan-freeze", "worker_successes", [1] * 7), - ("scan-freeze", "freeze_successes", 0), - ("scan-freeze", "elapsed_seconds", 74), - ("scan-freeze", "rows", 0), - ], -) -def test_failed_or_incomplete_work_cannot_pass(scenario, field, value): - value_report = report(scenario) - value_report[field] = value - with pytest.raises(AssertionError): - fts.validate_report(value_report, scenario) - - -def test_wrapped_numeric_error_is_not_inferred_from_prose(): - error = RuntimeError("Failed to execute query") - error.__cause__ = RuntimeError("execute failed: code=4016") - assert fts.error_details(error)["error_codes"] == [4016] - assert fts.error_details(RuntimeError("internal error"))["error_codes"] == [] - - -def test_released_binding_symbolic_error_preserves_numeric_code(): - error = RuntimeError("Failed to execute query") - error.__cause__ = RuntimeError("execute sql failed OB_ERR_UNEXPECTED(4016): %s") - assert fts.error_details(error)["error_codes"] == [4016] - assert fts.error_details(RuntimeError("query ordinal4016 returned no rows"))["error_codes"] == [] - - -def test_large_rotated_log_uses_total_tail_budget(tmp_path): - log = tmp_path / "seekdb.log.20260916" - with log.open("wb") as stream: - stream.seek(300 * 1024 * 1024) - stream.write(b"\nread_barrier_:true ret=-4016\n") - evidence = tmp_path / "evidence" - evidence.mkdir() - fts.capture_logs(tmp_path, evidence, "early", budget=4096, output_limit=1024) - assert "ret=-4016" in (evidence / "early-db-context.log").read_text() - inventory = json.loads((evidence / "early-log-inventory.json").read_text()) - assert inventory["files"][0]["read_bytes"] == 4096 - assert inventory["files"][0]["size"] > 256 * 1024 * 1024 - assert inventory["scan_capped"] - assert (evidence / "early-db-context.log").stat().st_size <= 1024 - - -def test_first_failure_is_captured_immediately_and_never_reset(tmp_path): - evidence = tmp_path / "evidence" - evidence.mkdir() - (tmp_path / "seekdb.log").write_text("ret=-4016 data/schema type does not match\n") - recorder = fts.Recorder(tmp_path, evidence) - recorder.error(RuntimeError("code=4016"), "scan") - assert (evidence / "first-error.json").exists() - assert (evidence / "early-db-context.log").exists() - assert not (evidence / "summary.json").exists() # before the end of the workload - recorder.error(RuntimeError("code=4016"), "scan") - assert recorder.errors == 2 - assert recorder.first["error_codes"] == [4016] - - -def test_early_trace_error_wins_over_noise_and_other_errors(tmp_path): - trace = "YB427F000001-00065BA8AA327DFF-0-0" - # All lower-priority buffers fill before the failing trace is encountered. - noise = ( - "read_barrier_:false ordinary INFO\n" * 200 - + "read_barrier_:true release_head_memtable_\n" * 200 - + "ret=-4016 unrelated session\n" * 200 - + f"[{trace}] normal statement context\n" * 200 - ) - failure = f"[{trace}] send_error_packet ob_error=-4016\n" - (tmp_path / "seekdb.log").write_text(noise + failure) - evidence = tmp_path / "evidence" - evidence.mkdir() - recorder = fts.Recorder(tmp_path, evidence) - # Exercise the real immediate-error entry point, not just final collection. - original = fts.capture_logs - from unittest.mock import patch - - with patch.object(fts, "capture_logs", side_effect=lambda *args: original(*args, output_limit=1024)): - recorder.error(RuntimeError(f"code=4016 [{trace}]"), "scan") - context = (evidence / "early-db-context.log").read_text() - assert failure in context - assert context.index(failure) < context.index("[preceding context]") - assert not (evidence / "summary.json").exists() - inventory = json.loads((evidence / "early-log-inventory.json").read_text()) - assert inventory["priority_matches"]["trace-error"] == 1 - assert inventory["priority_capped"]["memtable-context"] - assert inventory["context_bytes"] <= 1024 - - -def test_false_read_barrier_alone_does_not_consume_output(tmp_path): - (tmp_path / "seekdb.log").write_text("read_barrier_:false ordinary INFO\n" * 100) - evidence = tmp_path / "evidence" - evidence.mkdir() - fts.capture_logs(tmp_path, evidence, "early", output_limit=128) - assert (evidence / "early-db-context.log").read_bytes() == b"" - - -def test_scan_freeze_uses_separate_connections_and_no_vector_index(tmp_path, monkeypatch): - # Shorten only this harness unit test, not the real integration defaults. - for key, value in (("ROWS", 20), ("WORKERS", 2), ("SECONDS", 0.03), ("INTERVAL", 0.005)): - monkeypatch.setattr(fts, key, value) - clients = [] - # Fake queries do no I/O and can monopolize the GIL for the entire 30ms - # window. Model one completed round deterministically instead of depending - # on OS scheduling. This clock is local to the fake-client unit test only. - first_round = threading.Event() - round_barrier = threading.Barrier(fts.WORKERS + 1, action=first_round.set) - monkeypatch.setattr( - fts, "time", SimpleNamespace(time=time.time, monotonic=lambda: fts.SECONDS if first_round.is_set() else 0.0) - ) - - class Client: - def __init__(self): - self._server = self - self.commands = [] - self.closed = False - - def _execute(self, sql): - self.commands.append(sql) - if sql == fts.SEARCH or sql == "ALTER SYSTEM MINOR FREEZE": - round_barrier.wait(timeout=5) - if sql.startswith("SHOW CREATE"): - return [("fixture", "CREATE TABLE fixture (... FULLTEXT ...)")] - if sql.startswith("SELECT COUNT"): - return [(20,)] - return [("doc-1", 1.0)] - - def close(self): - self.closed = True - - def make_client(): - client = Client() - clients.append(client) - return client - - evidence = tmp_path / "evidence" - evidence.mkdir() - recorder = fts.Recorder(tmp_path, evidence) - value_report = {"scenario": "scan-freeze", "completed": True, "errors": 0} - setup = Client() - fts.scan_freeze(setup, make_client, recorder, value_report) - fts.validate_report(value_report, "scan-freeze") - assert recorder.errors == 0 - assert len(clients) == 3 and all(client.closed for client in clients) - assert "VECTOR" not in setup.commands[0].upper() - assert sum("ALTER SYSTEM MINOR FREEZE" in client.commands for client in clients) == 1 diff --git a/tests/unit_tests/test_embedded_wheel_manifest.py b/tests/unit_tests/test_embedded_wheel_manifest.py deleted file mode 100644 index 4f5641fc..00000000 --- a/tests/unit_tests/test_embedded_wheel_manifest.py +++ /dev/null @@ -1,125 +0,0 @@ -"""The CI source wheel must not silently fall back to a released binary.""" - -import importlib.util -import json -import zipfile -from pathlib import Path -from types import SimpleNamespace - -import pytest - -spec = importlib.util.spec_from_file_location( - "wheel_manifest", Path(__file__).parents[2] / ".github/scripts/embedded_wheel_manifest.py" -) -wheel_manifest = importlib.util.module_from_spec(spec) -spec.loader.exec_module(wheel_manifest) - - -@pytest.fixture -def bundle(tmp_path, monkeypatch): - pin = {"seekdb_sha": "a" * 40, "bindings_sha": "b" * 40, "fix_commit": "c" * 40} - pin_path = tmp_path / "source.json" - pin_path.write_text(json.dumps(pin)) - binary = tmp_path / "seekdb" - binary.write_bytes(b"the fixed binary") - wheel = tmp_path / "pylibseekdb-0.1.0-cp311-cp311-linux_x86_64.whl" - with zipfile.ZipFile(wheel, "w") as archive: - archive.writestr("pylibseekdb/seekdb", binary.read_bytes()) - (tmp_path / "seekdb-version.txt").write_text(f"REVISION: {pin['seekdb_sha']}\n") - monkeypatch.setattr(wheel_manifest.platform, "system", lambda: "Linux") - monkeypatch.setattr(wheel_manifest.platform, "machine", lambda: "x86_64") - monkeypatch.setattr( - wheel_manifest, - "sys", - SimpleNamespace(version_info=(3, 11), argv=["manifest", "create", str(pin_path), str(tmp_path), str(binary)]), - ) - wheel_manifest.main() - return pin, tmp_path, binary, wheel - - -def test_exact_wheel_and_revision_pass(bundle): - pin, directory, binary, _ = bundle - assert wheel_manifest.verify(pin, directory)["binary_sha256"] == wheel_manifest.digest(binary) - - -def test_wrong_source_cannot_reuse_cache(bundle): - pin, directory, _, _ = bundle - with pytest.raises(ValueError, match="source identity"): - wheel_manifest.verify({**pin, "seekdb_sha": "d" * 40}, directory) - - -def test_corrupt_wheel_is_rejected(bundle): - pin, directory, _, wheel = bundle - with wheel.open("ab") as stream: - stream.write(b"corruption") - with pytest.raises(ValueError, match="checksum"): - wheel_manifest.verify(pin, directory) - - -def test_replaced_binary_is_rejected_even_with_updated_wheel_hash(bundle): - pin, directory, _, wheel = bundle - with zipfile.ZipFile(wheel, "w") as archive: - archive.writestr("pylibseekdb/seekdb", b"old released binary") - manifest_path = directory / "manifest.json" - manifest = json.loads(manifest_path.read_text()) - manifest["wheel_sha256"] = wheel_manifest.digest(wheel) - manifest_path.write_text(json.dumps(manifest)) - with pytest.raises(ValueError, match="different SeekDB"): - wheel_manifest.verify(pin, directory) - - -def test_short_revision_is_not_enough(bundle): - pin, directory, _, _ = bundle - (directory / "seekdb-version.txt").write_text(f"REVISION: {pin['seekdb_sha'][:8]}") - with pytest.raises(ValueError, match="full REVISION"): - wheel_manifest.verify(pin, directory) - - -def test_dependency_resync_is_detected(bundle, monkeypatch): - _, directory, binary, _ = bundle - binary.write_bytes(b"old released binary installed by uv sync") - monkeypatch.setattr( - wheel_manifest.importlib.metadata, "distribution", lambda _: SimpleNamespace(locate_file=lambda _: binary) - ) - wheel_manifest.sys.argv = ["manifest", "installed", str(directory / "source.json"), str(directory)] - with pytest.raises(ValueError, match="dependency resync"): - wheel_manifest.main() - - -def test_installed_revision_from_stderr_is_verified(bundle, monkeypatch): - pin, directory, binary, _ = bundle - monkeypatch.setattr( - wheel_manifest.importlib.metadata, "distribution", lambda _: SimpleNamespace(locate_file=lambda _: binary) - ) - - def version_output(command, *, text, stderr): - assert command == [str(binary), "-V"] - assert text is True - assert stderr is wheel_manifest.subprocess.STDOUT - return f"REVISION: 1-{pin['seekdb_sha']}\n" - - monkeypatch.setattr(wheel_manifest.subprocess, "check_output", version_output) - wheel_manifest.sys.argv = ["manifest", "installed", str(directory / "source.json"), str(directory)] - wheel_manifest.main() - - -def test_installed_wrong_revision_is_rejected(bundle, monkeypatch): - _, directory, binary, _ = bundle - monkeypatch.setattr( - wheel_manifest.importlib.metadata, "distribution", lambda _: SimpleNamespace(locate_file=lambda _: binary) - ) - monkeypatch.setattr( - wheel_manifest.subprocess, - "check_output", - lambda *_, **__: f"REVISION: 1-{'d' * 40}\n", - ) - wheel_manifest.sys.argv = ["manifest", "installed", str(directory / "source.json"), str(directory)] - with pytest.raises(ValueError, match="pinned full REVISION"): - wheel_manifest.main() - - -def test_incompatible_host_is_rejected(bundle, monkeypatch): - pin, directory, _, _ = bundle - monkeypatch.setattr(wheel_manifest.platform, "machine", lambda: "aarch64") - with pytest.raises(ValueError, match="Linux x86_64"): - wheel_manifest.verify(pin, directory) diff --git a/tests/unit_tests/test_huggingface_sparse_embedding_function.py b/tests/unit_tests/test_huggingface_sparse_embedding_function.py index 8b1de1b3..9549de6a 100644 --- a/tests/unit_tests/test_huggingface_sparse_embedding_function.py +++ b/tests/unit_tests/test_huggingface_sparse_embedding_function.py @@ -41,26 +41,15 @@ def _sparse_encoder_available() -> bool: @pytest.fixture(autouse=True) -def mock_sparse_encoder(): - """Keep unit tests deterministic and leave real model loading to integration tests.""" +def _clear_model_cache(): + """Clear the class-level model cache before each test.""" from pyseekdb.utils.embedding_functions.huggingface_sparse_embedding_function import ( HuggingFaceSparseEmbeddingFunction, ) saved = HuggingFaceSparseEmbeddingFunction.models.copy() HuggingFaceSparseEmbeddingFunction.models.clear() - with patch("sentence_transformers.SparseEncoder") as mock_encoder: - model = MagicMock() - - def encode(documents): - vector = np.zeros(8, dtype=np.float32) - vector[2] = 0.75 - return [vector.copy() for _ in documents] - - model.encode_document.side_effect = encode - model.encode_query.side_effect = encode - mock_encoder.return_value = model - yield mock_encoder, model + yield HuggingFaceSparseEmbeddingFunction.models.clear() HuggingFaceSparseEmbeddingFunction.models.update(saved) @@ -124,6 +113,15 @@ def test_different_models_loaded_separately(self): HuggingFaceSparseEmbeddingFunction(model_name="naver/splade-v3-distilbert") +@pytest.fixture +def mock_sparse_encoder(): + """Patch SparseEncoder so tests get a mock model; yields (MockEncoder, mock_instance).""" + with patch("sentence_transformers.SparseEncoder") as MockEncoder: + mock_instance = MagicMock() + MockEncoder.return_value = mock_instance + yield (MockEncoder, mock_instance) + + @pytest.mark.skipif( not _sparse_encoder_available(), reason="sentence-transformers with SparseEncoder is not available", @@ -194,7 +192,6 @@ def test_numpy_array_fallback(self, mock_sparse_encoder): arr = np.zeros(100, dtype=np.float32) arr[7] = 0.6 arr[99] = 0.1 - mock_model.encode_document.side_effect = None mock_model.encode_document.return_value = [arr] ef = HuggingFaceSparseEmbeddingFunction() diff --git a/tests/unit_tests/test_mnn_embedding_function.py b/tests/unit_tests/test_mnn_embedding_function.py index eb2d21f9..e644cda0 100644 --- a/tests/unit_tests/test_mnn_embedding_function.py +++ b/tests/unit_tests/test_mnn_embedding_function.py @@ -57,14 +57,6 @@ def test_local_model_does_not_require_huggingface_metadata(tmp_path: Path) -> No assert ef.hf_revision == "main" -def test_default_cache_honors_pyseekdb_cache_dir(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setenv("PYSEEKDB_CACHE_DIR", str(tmp_path)) - - ef = MnnEmbeddingFunction("test-model", "org/test-model", 3) - - assert ef.download_path == tmp_path / "mnn_models" / "test-model" - - def test_hf_endpoint_must_use_https(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: ef = _make_embedding_function(tmp_path) monkeypatch.setenv("HF_ENDPOINT", "http://example.invalid") diff --git a/tests/unit_tests/test_sentence_transformer_embedding_function.py b/tests/unit_tests/test_sentence_transformer_embedding_function.py index 2ea3ab35..df6914a2 100644 --- a/tests/unit_tests/test_sentence_transformer_embedding_function.py +++ b/tests/unit_tests/test_sentence_transformer_embedding_function.py @@ -10,9 +10,7 @@ """ import importlib.util -from unittest.mock import MagicMock, patch -import numpy as np import pytest from pyseekdb.utils.embedding_functions import SentenceTransformerEmbeddingFunction @@ -46,20 +44,6 @@ def is_sentence_transformers_available() -> bool: return importlib.util.find_spec("sentence_transformers") is not None -@pytest.fixture(autouse=True) -def mock_sentence_transformer(): - """Persistence unit tests must not download real embedding models.""" - saved = SentenceTransformerEmbeddingFunction.models.copy() - SentenceTransformerEmbeddingFunction.models.clear() - with patch("sentence_transformers.SentenceTransformer") as mock_class: - model = MagicMock() - model.encode.side_effect = lambda documents, **_kwargs: np.zeros((len(documents), 3), dtype=np.float32) - mock_class.return_value = model - yield mock_class, model - SentenceTransformerEmbeddingFunction.models.clear() - SentenceTransformerEmbeddingFunction.models.update(saved) - - @pytest.mark.skipif( not is_sentence_transformers_available(), reason="sentence-transformers is not available on this system", diff --git a/uv.lock b/uv.lock index d3b8b2aa..7e235cba 100644 --- a/uv.lock +++ b/uv.lock @@ -2,14 +2,10 @@ version = 1 revision = 3 requires-python = ">=3.11, <4" resolution-markers = [ - "python_full_version >= '3.14' and platform_machine == 'x86_64' and sys_platform == 'linux'", - "(python_full_version >= '3.14' and platform_machine != 'x86_64') or (python_full_version >= '3.14' and sys_platform != 'linux')", - "python_full_version == '3.13.*' and platform_machine == 'x86_64' and sys_platform == 'linux'", - "(python_full_version == '3.13.*' and platform_machine != 'x86_64') or (python_full_version == '3.13.*' and sys_platform != 'linux')", - "python_full_version == '3.12.*' and platform_machine == 'x86_64' and sys_platform == 'linux'", - "(python_full_version == '3.12.*' and platform_machine != 'x86_64') or (python_full_version == '3.12.*' and sys_platform != 'linux')", - "python_full_version < '3.12' and platform_machine == 'x86_64' and sys_platform == 'linux'", - "(python_full_version < '3.12' and platform_machine != 'x86_64') or (python_full_version < '3.12' and sys_platform != 'linux')", + "python_full_version >= '3.14'", + "python_full_version == '3.13.*'", + "python_full_version == '3.12.*'", + "python_full_version < '3.12'", ] [manifest] @@ -1794,6 +1790,140 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/de/e5/b7d20451657664b07986c2f6e3be564433f5dcaf3482d68eaecd79afaf03/numpy-2.4.2-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:be71bf1edb48ebbbf7f6337b5bfd2f895d1902f6335a5830b20141fc126ffba0", size = 12502577, upload-time = "2026-01-31T23:13:07.08Z" }, ] +[[package]] +name = "nvidia-cublas-cu12" +version = "12.8.4.1" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/dc/61/e24b560ab2e2eaeb3c839129175fb330dfcfc29e5203196e5541a4c44682/nvidia_cublas_cu12-12.8.4.1-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:8ac4e771d5a348c551b2a426eda6193c19aa630236b418086020df5ba9667142", size = 594346921, upload-time = "2025-03-07T01:44:31.254Z" }, +] + +[[package]] +name = "nvidia-cuda-cupti-cu12" +version = "12.8.90" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f8/02/2adcaa145158bf1a8295d83591d22e4103dbfd821bcaf6f3f53151ca4ffa/nvidia_cuda_cupti_cu12-12.8.90-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ea0cb07ebda26bb9b29ba82cda34849e73c166c18162d3913575b0c9db9a6182", size = 10248621, upload-time = "2025-03-07T01:40:21.213Z" }, +] + +[[package]] +name = "nvidia-cuda-nvrtc-cu12" +version = "12.8.93" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/05/6b/32f747947df2da6994e999492ab306a903659555dddc0fbdeb9d71f75e52/nvidia_cuda_nvrtc_cu12-12.8.93-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:a7756528852ef889772a84c6cd89d41dfa74667e24cca16bb31f8f061e3e9994", size = 88040029, upload-time = "2025-03-07T01:42:13.562Z" }, +] + +[[package]] +name = "nvidia-cuda-runtime-cu12" +version = "12.8.90" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0d/9b/a997b638fcd068ad6e4d53b8551a7d30fe8b404d6f1804abf1df69838932/nvidia_cuda_runtime_cu12-12.8.90-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:adade8dcbd0edf427b7204d480d6066d33902cab2a4707dcfc48a2d0fd44ab90", size = 954765, upload-time = "2025-03-07T01:40:01.615Z" }, +] + +[[package]] +name = "nvidia-cudnn-cu12" +version = "9.10.2.21" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "nvidia-cublas-cu12" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/ba/51/e123d997aa098c61d029f76663dedbfb9bc8dcf8c60cbd6adbe42f76d049/nvidia_cudnn_cu12-9.10.2.21-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:949452be657fa16687d0930933f032835951ef0892b37d2d53824d1a84dc97a8", size = 706758467, upload-time = "2025-06-06T21:54:08.597Z" }, +] + +[[package]] +name = "nvidia-cufft-cu12" +version = "11.3.3.83" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "nvidia-nvjitlink-cu12" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/1f/13/ee4e00f30e676b66ae65b4f08cb5bcbb8392c03f54f2d5413ea99a5d1c80/nvidia_cufft_cu12-11.3.3.83-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4d2dd21ec0b88cf61b62e6b43564355e5222e4a3fb394cac0db101f2dd0d4f74", size = 193118695, upload-time = "2025-03-07T01:45:27.821Z" }, +] + +[[package]] +name = "nvidia-cufile-cu12" +version = "1.13.1.3" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/bb/fe/1bcba1dfbfb8d01be8d93f07bfc502c93fa23afa6fd5ab3fc7c1df71038a/nvidia_cufile_cu12-1.13.1.3-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1d069003be650e131b21c932ec3d8969c1715379251f8d23a1860554b1cb24fc", size = 1197834, upload-time = "2025-03-07T01:45:50.723Z" }, +] + +[[package]] +name = "nvidia-curand-cu12" +version = "10.3.9.90" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fb/aa/6584b56dc84ebe9cf93226a5cde4d99080c8e90ab40f0c27bda7a0f29aa1/nvidia_curand_cu12-10.3.9.90-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:b32331d4f4df5d6eefa0554c565b626c7216f87a06a4f56fab27c3b68a830ec9", size = 63619976, upload-time = "2025-03-07T01:46:23.323Z" }, +] + +[[package]] +name = "nvidia-cusolver-cu12" +version = "11.7.3.90" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "nvidia-cublas-cu12" }, + { name = "nvidia-cusparse-cu12" }, + { name = "nvidia-nvjitlink-cu12" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/85/48/9a13d2975803e8cf2777d5ed57b87a0b6ca2cc795f9a4f59796a910bfb80/nvidia_cusolver_cu12-11.7.3.90-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:4376c11ad263152bd50ea295c05370360776f8c3427b30991df774f9fb26c450", size = 267506905, upload-time = "2025-03-07T01:47:16.273Z" }, +] + +[[package]] +name = "nvidia-cusparse-cu12" +version = "12.5.8.93" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "nvidia-nvjitlink-cu12" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/c2/f5/e1854cb2f2bcd4280c44736c93550cc300ff4b8c95ebe370d0aa7d2b473d/nvidia_cusparse_cu12-12.5.8.93-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1ec05d76bbbd8b61b06a80e1eaf8cf4959c3d4ce8e711b65ebd0443bb0ebb13b", size = 288216466, upload-time = "2025-03-07T01:48:13.779Z" }, +] + +[[package]] +name = "nvidia-cusparselt-cu12" +version = "0.7.1" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/56/79/12978b96bd44274fe38b5dde5cfb660b1d114f70a65ef962bcbbed99b549/nvidia_cusparselt_cu12-0.7.1-py3-none-manylinux2014_x86_64.whl", hash = "sha256:f1bb701d6b930d5a7cea44c19ceb973311500847f81b634d802b7b539dc55623", size = 287193691, upload-time = "2025-02-26T00:15:44.104Z" }, +] + +[[package]] +name = "nvidia-nccl-cu12" +version = "2.27.5" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6e/89/f7a07dc961b60645dbbf42e80f2bc85ade7feb9a491b11a1e973aa00071f/nvidia_nccl_cu12-2.27.5-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ad730cf15cb5d25fe849c6e6ca9eb5b76db16a80f13f425ac68d8e2e55624457", size = 322348229, upload-time = "2025-06-26T04:11:28.385Z" }, +] + +[[package]] +name = "nvidia-nvjitlink-cu12" +version = "12.8.93" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f6/74/86a07f1d0f42998ca31312f998bd3b9a7eff7f52378f4f270c8679c77fb9/nvidia_nvjitlink_cu12-12.8.93-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:81ff63371a7ebd6e6451970684f916be2eab07321b73c9d244dc2b4da7f73b88", size = 39254836, upload-time = "2025-03-07T01:49:55.661Z" }, +] + +[[package]] +name = "nvidia-nvshmem-cu12" +version = "3.3.20" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3b/6c/99acb2f9eb85c29fc6f3a7ac4dccfd992e22666dd08a642b303311326a97/nvidia_nvshmem_cu12-3.3.20-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d00f26d3f9b2e3c3065be895e3059d6479ea5c638a3f38c9fec49b1b9dd7c1e5", size = 124657145, upload-time = "2025-08-04T20:25:19.995Z" }, +] + +[[package]] +name = "nvidia-nvtx-cu12" +version = "12.8.90" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a2/eb/86626c1bbc2edb86323022371c39aa48df6fd8b0a1647bc274577f72e90b/nvidia_nvtx_cu12-12.8.90-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:5b17e2001cc0d751a5bc2c6ec6d26ad95913324a4adb86788c944f8ce9ba441f", size = 89954, upload-time = "2025-03-07T01:42:44.131Z" }, +] + [[package]] name = "onnxruntime" version = "1.30.0" @@ -2543,8 +2673,6 @@ unit-tests = [ { name = "onnxruntime" }, { name = "openai" }, { name = "sentence-transformers" }, - { name = "torch", version = "2.9.1", source = { registry = "https://pypi.org/simple" }, marker = "platform_machine != 'x86_64' or sys_platform != 'linux'" }, - { name = "torch", version = "2.9.1+cpu", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, { name = "voyageai" }, ] @@ -2579,8 +2707,6 @@ unit-tests = [ { name = "onnxruntime", specifier = ">=1.19.0" }, { name = "openai" }, { name = "sentence-transformers", specifier = ">=5.0" }, - { name = "torch", marker = "platform_machine != 'x86_64' or sys_platform != 'linux'", specifier = "==2.9.1" }, - { name = "torch", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'", specifier = "==2.9.1", index = "https://download.pytorch.org/whl/cpu" }, { name = "voyageai" }, ] @@ -3280,8 +3406,7 @@ dependencies = [ { name = "huggingface-hub" }, { name = "scikit-learn" }, { name = "scipy" }, - { name = "torch", version = "2.9.1", source = { registry = "https://pypi.org/simple" }, marker = "platform_machine != 'x86_64' or sys_platform != 'linux'" }, - { name = "torch", version = "2.9.1+cpu", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "torch" }, { name = "tqdm" }, { name = "transformers" }, { name = "typing-extensions" }, @@ -3644,20 +3769,30 @@ wheels = [ name = "torch" version = "2.9.1" source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "(python_full_version >= '3.14' and platform_machine != 'x86_64') or (python_full_version >= '3.14' and sys_platform != 'linux')", - "(python_full_version == '3.13.*' and platform_machine != 'x86_64') or (python_full_version == '3.13.*' and sys_platform != 'linux')", - "(python_full_version == '3.12.*' and platform_machine != 'x86_64') or (python_full_version == '3.12.*' and sys_platform != 'linux')", - "(python_full_version < '3.12' and platform_machine != 'x86_64') or (python_full_version < '3.12' and sys_platform != 'linux')", -] dependencies = [ - { name = "filelock", marker = "platform_machine != 'x86_64' or sys_platform != 'linux'" }, - { name = "fsspec", marker = "platform_machine != 'x86_64' or sys_platform != 'linux'" }, - { name = "jinja2", marker = "platform_machine != 'x86_64' or sys_platform != 'linux'" }, - { name = "networkx", marker = "platform_machine != 'x86_64' or sys_platform != 'linux'" }, - { name = "setuptools", marker = "(python_full_version >= '3.12' and platform_machine != 'x86_64') or (python_full_version >= '3.12' and sys_platform != 'linux')" }, - { name = "sympy", marker = "platform_machine != 'x86_64' or sys_platform != 'linux'" }, - { name = "typing-extensions", marker = "platform_machine != 'x86_64' or sys_platform != 'linux'" }, + { name = "filelock" }, + { name = "fsspec" }, + { name = "jinja2" }, + { name = "networkx" }, + { name = "nvidia-cublas-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "nvidia-cuda-cupti-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "nvidia-cuda-nvrtc-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "nvidia-cuda-runtime-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "nvidia-cudnn-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "nvidia-cufft-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "nvidia-cufile-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "nvidia-curand-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "nvidia-cusolver-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "nvidia-cusparse-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "nvidia-cusparselt-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "nvidia-nccl-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "nvidia-nvjitlink-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "nvidia-nvshmem-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "nvidia-nvtx-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "setuptools", marker = "python_full_version >= '3.12'" }, + { name = "sympy" }, + { name = "triton", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "typing-extensions" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/15/db/c064112ac0089af3d2f7a2b5bfbabf4aa407a78b74f87889e524b91c5402/torch-2.9.1-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:62b3fd888277946918cba4478cf849303da5359f0fb4e3bfb86b0533ba2eaf8d", size = 104220430, upload-time = "2025-11-12T15:20:31.705Z" }, @@ -3686,34 +3821,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/db/2b/f7818f6ec88758dfd21da46b6cd46af9d1b3433e53ddbb19ad1e0da17f9b/torch-2.9.1-cp314-cp314t-win_amd64.whl", hash = "sha256:c88d3299ddeb2b35dcc31753305612db485ab6f1823e37fb29451c8b2732b87e", size = 111163659, upload-time = "2025-11-12T15:23:20.009Z" }, ] -[[package]] -name = "torch" -version = "2.9.1+cpu" -source = { registry = "https://download.pytorch.org/whl/cpu" } -resolution-markers = [ - "python_full_version >= '3.14' and platform_machine == 'x86_64' and sys_platform == 'linux'", - "python_full_version == '3.13.*' and platform_machine == 'x86_64' and sys_platform == 'linux'", - "python_full_version == '3.12.*' and platform_machine == 'x86_64' and sys_platform == 'linux'", - "python_full_version < '3.12' and platform_machine == 'x86_64' and sys_platform == 'linux'", -] -dependencies = [ - { name = "filelock", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "fsspec", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "jinja2", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "networkx", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "setuptools", marker = "python_full_version >= '3.12' and platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "sympy", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "typing-extensions", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, -] -wheels = [ - { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.9.1%2Bcpu-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:3de2adb9b4443dc9210ef1f1b16da3647ace53553166d6360bbbd7edd6f16e4d", upload-time = "2026-01-23T15:12:58Z" }, - { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.9.1%2Bcpu-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:7417d8c565f219d3455654cb431c6d892a3eb40246055e14d645422de13b9ea1", upload-time = "2026-01-23T15:13:04Z" }, - { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.9.1%2Bcpu-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:39b3dff6d8fba240ae0d1bede4ca11c2531ae3b47329206512d99e17907ff74b", upload-time = "2026-01-23T15:13:12Z" }, - { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.9.1%2Bcpu-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:031a597147fa81b1e6d79ccf1ad3ccc7fafa27941d6cf26ff5caaa384fb20e92", upload-time = "2026-01-23T15:13:19Z" }, - { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.9.1%2Bcpu-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:88adf5157db5da1d54b1c9fe4a6c1d20ceef00e75d854e206a87dbf69e3037dc", upload-time = "2026-01-23T15:13:23Z" }, - { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.9.1%2Bcpu-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:5b688445f928f13563b7418b17c57e97bf955ab559cf73cd8f2b961f8572dbb3", upload-time = "2026-01-23T15:13:29Z" }, -] - [[package]] name = "tornado" version = "6.5.7" @@ -3764,6 +3871,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/6a/6b/2f416568b3c4c91c96e5a365d164f8a4a4a88030aa8ab4644181fdadce97/transformers-4.57.3-py3-none-any.whl", hash = "sha256:c77d353a4851b1880191603d36acb313411d3577f6e2897814f333841f7003f4", size = 11993463, upload-time = "2025-11-25T15:51:26.493Z" }, ] +[[package]] +name = "triton" +version = "3.5.1" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b0/72/ec90c3519eaf168f22cb1757ad412f3a2add4782ad3a92861c9ad135d886/triton-3.5.1-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:61413522a48add32302353fdbaaf92daaaab06f6b5e3229940d21b5207f47579", size = 170425802, upload-time = "2025-11-11T17:40:53.209Z" }, + { url = "https://files.pythonhosted.org/packages/f2/50/9a8358d3ef58162c0a415d173cfb45b67de60176e1024f71fbc4d24c0b6d/triton-3.5.1-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d2c6b915a03888ab931a9fd3e55ba36785e1fe70cbea0b40c6ef93b20fc85232", size = 170470207, upload-time = "2025-11-11T17:41:00.253Z" }, + { url = "https://files.pythonhosted.org/packages/27/46/8c3bbb5b0a19313f50edcaa363b599e5a1a5ac9683ead82b9b80fe497c8d/triton-3.5.1-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f3f4346b6ebbd4fad18773f5ba839114f4826037c9f2f34e0148894cd5dd3dba", size = 170470410, upload-time = "2025-11-11T17:41:06.319Z" }, + { url = "https://files.pythonhosted.org/packages/37/92/e97fcc6b2c27cdb87ce5ee063d77f8f26f19f06916aa680464c8104ef0f6/triton-3.5.1-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0b4d2c70127fca6a23e247f9348b8adde979d2e7a20391bfbabaac6aebc7e6a8", size = 170579924, upload-time = "2025-11-11T17:41:12.455Z" }, + { url = "https://files.pythonhosted.org/packages/a4/e6/c595c35e5c50c4bc56a7bac96493dad321e9e29b953b526bbbe20f9911d0/triton-3.5.1-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d0637b1efb1db599a8e9dc960d53ab6e4637db7d4ab6630a0974705d77b14b60", size = 170480488, upload-time = "2025-11-11T17:41:18.222Z" }, + { url = "https://files.pythonhosted.org/packages/16/b5/b0d3d8b901b6a04ca38df5e24c27e53afb15b93624d7fd7d658c7cd9352a/triton-3.5.1-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bac7f7d959ad0f48c0e97d6643a1cc0fd5786fe61cb1f83b537c6b2d54776478", size = 170582192, upload-time = "2025-11-11T17:41:23.963Z" }, +] + [[package]] name = "typing-extensions" version = "4.15.0" From 9eabb42fe617fc0de672edd1a32b457f259aa6aa Mon Sep 17 00:00:00 2001 From: cms-cms <1303573631@qq.com> Date: Mon, 21 Sep 2026 20:54:44 +0800 Subject: [PATCH 14/25] test: cover embedded fulltext stability --- .../embedded_fulltext_support.py | 409 ++++++++++++++++++ .../test_embedded_fulltext_stability.py | 69 +++ 2 files changed, 478 insertions(+) create mode 100644 tests/integration_tests/embedded_fulltext_support.py create mode 100644 tests/integration_tests/test_embedded_fulltext_stability.py diff --git a/tests/integration_tests/embedded_fulltext_support.py b/tests/integration_tests/embedded_fulltext_support.py new file mode 100644 index 00000000..19514013 --- /dev/null +++ b/tests/integration_tests/embedded_fulltext_support.py @@ -0,0 +1,409 @@ +"""Isolated, dependency-free harness; pyseekdb is imported only in the worker. + +Synthetic deterministic documents (not the original SciFact benchmark). The +scan/freezer uses the same SQL shape as the Linux issue #1382 reproducer, without +a vector index. Public hybrid_search coverage lives in the separate first pass. +""" + +from __future__ import annotations + +import argparse +import collections +import importlib.metadata +import importlib.util +import json +import platform +import re +import threading +import time +import traceback +from pathlib import Path + +ROWS = 24515 +WORKERS = 8 +SECONDS = 75 +INTERVAL = 0.120 +WRITE_BATCH = 32 +QUERIES = 300 +PASSES = 4 +TEXT = "the database retrieval benchmark" +TABLE = "`c$v2$16d82ab3afe311f186fe000000000000`" +SEARCH = ( + f"SELECT `_id`, MATCH(`document`) AGAINST('{TEXT}') AS relevance FROM {TABLE} " + f"WHERE MATCH(`document`) AGAINST('{TEXT}') ORDER BY relevance DESC LIMIT 10" +) +TRACE = re.compile(r"\bY[A-Za-z0-9]+-[A-Za-z0-9]+-\d+-\d+\b") + + +def document(ordinal): + """Every row matches the broad query while cohorts supply varied queries.""" + return ( + f"{TEXT} exercises full text postings during concurrent memtable release " + f"cohort{ordinal % QUERIES:03d} document ordinal{ordinal:08d}" + ) + + +def error_details(error): + """Retain wrapper/cause messages and only report explicit numeric codes.""" + chain, seen, codes = [], set(), [] + while error is not None and id(error) not in seen: + seen.add(id(error)) + chain.append(f"{type(error).__name__}: {error}") + if error.args and isinstance(error.args[0], int): + codes.append(error.args[0]) + codes.extend(int(value) for value in re.findall(r"\bcode[=:]\s*(-?\d+)\b", str(error))) + # Released bindings may expose a RuntimeError rather than numeric args, + # e.g. "execute sql failed OB_ERR_UNEXPECTED(4016): %s". + codes.extend(int(value) for value in re.findall(r"\bOB_[A-Z0-9_]+\((-?\d+)\)", str(error))) + error = error.__cause__ or error.__context__ + text = "\n".join(chain) + return {"exception_chain": chain, "error_codes": sorted(set(codes)), "trace_ids": sorted(set(TRACE.findall(text)))} + + +def capture_logs(root, evidence, label, traces=(), budget=128 * 1024 * 1024, output_limit=2 * 1024 * 1024): + """Scan once under a byte budget; prioritize errors over routine context. + + Each priority buffer is bounded by output_limit (at most 4x that limit in + memory). A full low-priority buffer must not stop the scan: the failing + trace can occur later in a large, noisy rotated log. + """ + if budget < 0 or output_limit <= 0: + raise ValueError("invalid diagnostic byte budget") + trace_patterns = [value.encode() for value in traces if value] + error_patterns = [ + b"-4016", + b"data/schema type does not match", + b"failed to get next row from memtable scanner", + b"ob_memtable_key.h", + ] + files = sorted( + (p for p in root.rglob("seekdb.log*") if p.is_file() and not p.is_symlink()), + key=lambda p: p.stat().st_mtime, + reverse=True, + ) + inventory = [] + priorities = ("trace-error", "trace", "error", "memtable-context") + buffers = [bytearray() for _ in priorities] + matched = [0] * len(priorities) + capped = [False] * len(priorities) + + def append(priority, content): + available = output_limit - len(buffers[priority]) + if len(content) > available: + capped[priority] = True + buffers[priority].extend(content[:available]) + + for path in files: + size = path.stat().st_size + count = min(size, budget) + inventory.append({"path": str(path), "size": size, "read_bytes": count}) + if not count: + continue + with path.open("rb") as stream: + stream.seek(size - count) + previous, following, remaining = collections.deque(maxlen=5), 0, count + context_priority = 3 + while remaining > 0: + line = stream.readline(min(65536, remaining)) + if not line: + break + remaining -= len(line) + is_trace = any(pattern in line for pattern in trace_patterns) + is_error = any(pattern in line for pattern in error_patterns) + priority = ( + 0 + if is_trace and is_error + else 1 + if is_trace + else 2 + if is_error + else 3 + if b"read_barrier_:true" in line + else None + ) + if priority is not None: + # Preserve the matching line before optional context so + # a huge preceding SQL line cannot consume the budget. + context = f"\nFILE={path}\n".encode() + line + b"[preceding context]\n" + b"".join(previous) + matched[priority] += 1 + append(priority, context) + following, context_priority = 5, priority + elif following: + append(context_priority, line) + following -= 1 + previous.append(line) + budget -= count + written = 0 + with (evidence / f"{label}-db-context.log").open("wb") as output: + for content in buffers: + part = content[: max(0, output_limit - written)] + output.write(part) + written += len(part) + (evidence / f"{label}-log-inventory.json").write_text( + json.dumps( + { + "files": inventory, + "context_bytes": written, + "context_capped": written >= output_limit, + "priority_matches": dict(zip(priorities, matched, strict=True)), + "priority_capped": dict(zip(priorities, capped, strict=True)), + "scan_capped": any(item["read_bytes"] < item["size"] for item in inventory), + }, + indent=2, + ) + ) + + +class Recorder: + """Flush errors immediately and snapshot the first failure before recycling.""" + + def __init__(self, root, evidence): + self.root, self.evidence = root, evidence + self.lock = threading.Lock() + self.errors = 0 + self.traces = set() + self.first = None + + def error(self, error, phase, client=None, **details): + event = {"time": time.time(), "phase": phase, **details, **error_details(error)} + if not event["trace_ids"] and client is not None: + try: + # Immediately on the failing worker's own connection; mark it + # as best-effort, not proof of a particular internal SDK SQL. + rows = client._server._execute("SELECT last_trace_id() FROM DUAL") + event["trace_ids"] = TRACE.findall(str(rows)) + event["trace_source"] = "same_connection_last_trace_id_best_effort" + except Exception as lookup_error: + event["trace_lookup_error"] = str(lookup_error) + with self.lock: + self.errors += 1 + self.traces.update(event["trace_ids"]) + first = self.first is None + if first: + self.first = event + # Keep a bounded sample but never lose the total failure count. + if self.errors <= 200: + with (self.evidence / "errors.jsonl").open("a") as stream: + stream.write(json.dumps(event) + "\n") + if first: + (self.evidence / "first-error.json").write_text(json.dumps(event, indent=2)) + if first: + print(f"[FTS][FIRST_ERROR] {json.dumps(event)}", flush=True) + capture_logs(self.root, self.evidence, "early", event["trace_ids"]) + + +def validate_report(report, scenario): + """A zero worker exit alone is insufficient with native binding teardown.""" + assert report.get("scenario") == scenario, report + assert report.get("completed") is True and report.get("errors") == 0, report + assert report.get("rows") == ROWS, report + if scenario == "first-pass": + assert report.get("first_pass_successes") == QUERIES, report + assert report.get("fts_successes") == QUERIES * PASSES, report + else: + counts = report.get("worker_successes", []) + assert len(counts) == WORKERS and all(value > 0 for value in counts), report + assert report.get("freeze_successes", 0) > 0, report + assert report.get("elapsed_seconds", 0) >= SECONDS, report + + +def first_pass(client, recorder, report): + import pyseekdb + + collection = client.create_collection( + "fts_first_pass", + configuration=pyseekdb.HNSWConfiguration(dimension=8, distance="cosine"), + embedding_function=None, + ) + vector = [float(i + 1) / 8 for i in range(8)] + for start in range(0, ROWS, 100): + indexes = range(start, min(start + 100, ROWS)) + collection.upsert( + ids=[f"doc-{i}" for i in indexes], + documents=[document(i) for i in indexes], + embeddings=[vector for _ in indexes], + metadatas=[{"ordinal": i} for i in indexes], + ) + if start % 1000 == 0: + print(f"[FTS] ingest={min(start + 100, ROWS)}/{ROWS}", flush=True) + assert collection.count() == ROWS + collection.refresh_index() + report.update(rows=ROWS, first_pass_successes=0, fts_successes=0) + # No FTS warm-up, sleep, retry or reopen before pass zero. + for pass_id in range(PASSES): + for query_id in range(QUERIES): + try: + result = collection.hybrid_search( + query={"where_document": {"$contains": f"{TEXT} cohort{query_id:03d}"}, "n_results": 10}, + knn=None, + n_results=10, + include=["documents", "metadatas"], + ) + assert result.get("ids") and result["ids"][0], result + report["fts_successes"] += 1 + if pass_id == 0: + report["first_pass_successes"] += 1 + except Exception as error: + recorder.error( + error, + "hybrid_search", + client, + pass_id=pass_id, + query_id=query_id, + query_text=f"{TEXT} cohort{query_id:03d}", + ) + print(f"[FTS] pass={pass_id} successes={report['fts_successes']} errors={recorder.errors}", flush=True) + + +def insert_batch(client, start, size): + # IDs/documents are generated here, not user-controlled SQL inputs. + values = ",".join(f"('doc-{i}', '{document(i)}', '{{\"ordinal\":{i}}}')" for i in range(start, start + size)) + client._server._execute(f"INSERT INTO {TABLE} (`_id`, `document`, `metadata`) VALUES {values}") + + +def scan_freeze(client, make_client, recorder, report): + client._server._execute( + f"CREATE TABLE {TABLE} (`__pk_increment` BIGINT NOT NULL AUTO_INCREMENT PRIMARY KEY, " + "`_id` VARCHAR(128) NOT NULL, `document` LONGTEXT NOT NULL, `metadata` JSON NOT NULL, " + "UNIQUE KEY uk_id (`_id`), FULLTEXT KEY idx_document (`document`) WITH PARSER space)" + ) + for start in range(0, ROWS, 100): + insert_batch(client, start, min(100, ROWS - start)) + if start % 1000 == 0: + print(f"[FTS] ingest={min(start + 100, ROWS)}/{ROWS}", flush=True) + counts = client._server._execute(f"SELECT COUNT(*) AS row_count FROM {TABLE}") + count = counts[0]["row_count"] if isinstance(counts[0], dict) else counts[0][0] + assert count == ROWS, counts + schema = client._server._execute(f"SHOW CREATE TABLE {TABLE}") + assert "FULLTEXT" in str(schema).upper() and "VECTOR" not in str(schema).upper(), schema + report.update( + rows=ROWS, + schema=str(schema), + search_sql=SEARCH, + write_batch_size=WRITE_BATCH, + worker_successes=[0] * WORKERS, + freeze_successes=0, + ) + stop = threading.Event() + deadline = [0.0] + started = [0.0] + + def start_window(): + started[0] = time.monotonic() + deadline[0] = started[0] + SECONDS + + barrier = threading.Barrier(WORKERS + 2, action=start_window) + + def worker(number): + connection = None + try: + connection = make_client() + connection._server._execute("SELECT 1") + barrier.wait(timeout=60) + while time.monotonic() < deadline[0] and not stop.is_set(): + started = time.monotonic() + try: + if number == WORKERS: + start = ROWS + report["freeze_successes"] * WRITE_BATCH + insert_batch(connection, start, WRITE_BATCH) + connection._server._execute("ALTER SYSTEM MINOR FREEZE") + report["freeze_successes"] += 1 + else: + rows = connection._server._execute(SEARCH) + assert rows, "full-text scan returned no rows" + report["worker_successes"][number] += 1 + except Exception as error: + recorder.error(error, "minor-freeze" if number == WORKERS else "scan", connection, worker=number) + if number == WORKERS: + stop.set() # No valid pressure if writes/freezes cannot execute. + if number == WORKERS: + stop.wait(max(0, INTERVAL - (time.monotonic() - started))) + except Exception as error: + recorder.error(error, "worker-setup", connection, worker=number) + barrier.abort() + stop.set() + finally: + if connection is not None: + try: + connection.close() + except Exception as error: + recorder.error(error, "worker-close", worker=number) + + threads = [threading.Thread(target=worker, args=(i,), daemon=True) for i in range(WORKERS + 1)] + for thread in threads: + thread.start() + try: + barrier.wait(timeout=60) + while not stop.wait(min(5, max(0, deadline[0] - time.monotonic()))) and time.monotonic() < deadline[0]: + print( + f"[FTS] scans={sum(report['worker_successes'])} freezes={report['freeze_successes']} " + f"errors={recorder.errors}", + flush=True, + ) + finally: + for thread in threads: + thread.join(timeout=65) + assert not any(thread.is_alive() for thread in threads), "FTS worker did not drain" + report["elapsed_seconds"] = time.monotonic() - started[0] + + +def run(scenario, root, evidence): + import pyseekdb + + runtime_distribution = "seekdb" if importlib.util.find_spec("seekdb") else "pylibseekdb" + + report = { + "scenario": scenario, + "completed": False, + "platform": platform.platform(), + "pyseekdb": importlib.metadata.version("pyseekdb"), + "embedded_runtime": runtime_distribution, + "embedded_runtime_version": importlib.metadata.version(runtime_distribution), + "corpus": "synthetic deterministic / 24515 documents", + "workers": WORKERS, + "duration_seconds": SECONDS, + "freeze_interval_seconds": INTERVAL, + } + recorder = Recorder(root, evidence) + client = None + try: + db_dir = root / "database" + assert not db_dir.exists(), "fresh database required" + + def make_client(): + return pyseekdb.Client(path=str(db_dir), database="test") + + client = make_client() + report["database_version"] = str(client._server._execute("SELECT VERSION()")) + client._server._execute("ALTER SYSTEM SET max_syslog_file_count = 50") + print(f"[FTS] start {json.dumps(report)}", flush=True) + if scenario == "first-pass": + first_pass(client, recorder, report) + else: + scan_freeze(client, make_client, recorder, report) + report["completed"] = True + except Exception as error: + recorder.error(error, "scenario", client) + traceback.print_exc() + finally: + if client is not None: + try: + client.close() + except Exception as error: + recorder.error(error, "close") + try: + capture_logs(root, evidence, "final", recorder.traces) + except Exception as error: + recorder.error(error, "diagnostics") + report.update(errors=recorder.errors, first_error=recorder.first, trace_ids=sorted(recorder.traces)) + (evidence / "summary.json").write_text(json.dumps(report, indent=2)) + validate_report(report, scenario) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--scenario", choices=["first-pass", "scan-freeze"], required=True) + parser.add_argument("--root", type=Path, required=True) + parser.add_argument("--evidence", type=Path, required=True) + options = parser.parse_args() + run(options.scenario, options.root, options.evidence) diff --git a/tests/integration_tests/test_embedded_fulltext_stability.py b/tests/integration_tests/test_embedded_fulltext_stability.py new file mode 100644 index 00000000..292412a3 --- /dev/null +++ b/tests/integration_tests/test_embedded_fulltext_stability.py @@ -0,0 +1,69 @@ +"""Fresh-ingestion and memtable-release regressions for seekdb issue #1382. + +Each case runs in a fresh process and database directory, including with legacy +process-global pylibseekdb. Do not rerun failed queries to obtain a passing result. +""" + +import json +import os +import shutil +import subprocess +import sys +import tempfile +import time +from pathlib import Path + +import pytest +from embedded_fulltext_support import validate_report + + +@pytest.mark.parametrize("scenario", ["first-pass", "scan-freeze"]) +def test_embedded_fulltext_stability(scenario): + """Exercise the public SDK first pass and an accelerated kernel stress case.""" + repo = Path(__file__).resolve().parents[2] + data_root = Path(os.environ.get("SEEKDB_TEST_DATA_ROOT", repo / ".seekdb-test-data")) + data_root.mkdir(parents=True, exist_ok=True) + root = Path(tempfile.mkdtemp(prefix=f"fts-{scenario}-", dir=data_root)) + evidence_root = Path(os.environ.get("SEEKDB_TEST_ARTIFACT_ROOT", data_root / "artifacts")) + evidence = evidence_root / root.name + evidence.mkdir(parents=True, exist_ok=False) + command = [ + sys.executable, + str(Path(__file__).with_name("embedded_fulltext_support.py")), + "--scenario", + scenario, + "--root", + str(root), + "--evidence", + str(evidence), + ] + # A whole-case timeout includes ingestion and connection setup; the pressure + # window itself is 75 seconds, starting only after every worker is ready. + with (evidence / "worker.log").open("w") as log: + # Fixed interpreter/helper and generated paths, never shell input. + process = subprocess.Popen(command, stdout=log, stderr=subprocess.STDOUT) # noqa: S603 + started = time.monotonic() + try: + with (evidence / "worker.log").open() as progress: + while process.poll() is None: + if time.monotonic() - started > 900: + raise TimeoutError(f"FTS worker exceeded 900s; evidence={evidence}") + # Stream bounded real output rather than hiding all ingestion + # and pressure activity inside pytest until the case ends. + text = progress.read(16384) + if text: + print(text, end="", flush=True) + time.sleep(1) + print(progress.read(), end="", flush=True) + finally: + if process.poll() is None: + process.kill() + process.wait(timeout=10) + report_path = evidence / "summary.json" + assert report_path.is_file(), f"worker produced no terminal report; evidence={evidence}" + report = json.loads(report_path.read_text()) + validate_report(report, scenario) + assert process.returncode == 0, f"worker rc={process.returncode}; evidence={evidence}" + # Remove only our generated DB after the process has exited and evidence has + # been collected. Failed cases retain their DB for investigation. + shutil.rmtree(root) From 6bf8473685303877bffc598d3855943011ece460 Mon Sep 17 00:00:00 2001 From: cms-cms <1303573631@qq.com> Date: Mon, 21 Sep 2026 21:37:11 +0800 Subject: [PATCH 15/25] ci: test embedded regressions with fixed seekdb runtime --- .github/embedded-source.json | 8 +++ .github/scripts/build-embedded-wheel.sh | 83 ++++++++++++++++++++++ .github/scripts/embedded_wheel_manifest.py | 75 +++++++++++++++++++ .github/workflows/ci.yml | 57 ++++++++++++++- 4 files changed, 221 insertions(+), 2 deletions(-) create mode 100644 .github/embedded-source.json create mode 100755 .github/scripts/build-embedded-wheel.sh create mode 100644 .github/scripts/embedded_wheel_manifest.py diff --git a/.github/embedded-source.json b/.github/embedded-source.json new file mode 100644 index 00000000..b4aa48d7 --- /dev/null +++ b/.github/embedded-source.json @@ -0,0 +1,8 @@ +{ + "seekdb_repository": "https://github.com/cms-cms/seekdb.git", + "seekdb_sha": "3f3b11ab9d6115ee448c2f0f693dfbe7f927c2ea", + "bindings_sha": "5e8eece63205dccfa9500f98893c91d1f50eb4f1", + "fix_commit": "48b80f5193bcac0afafba4d4820aebdf1103bc69", + "platform": "linux-x86_64", + "python": "3.11" +} diff --git a/.github/scripts/build-embedded-wheel.sh b/.github/scripts/build-embedded-wheel.sh new file mode 100755 index 00000000..7f469273 --- /dev/null +++ b/.github/scripts/build-embedded-wheel.sh @@ -0,0 +1,83 @@ +#!/usr/bin/env bash +# CI-local Linux wheel: not a portable manylinux release or a PyPI publication. +set -Eeuo pipefail +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +: "${RUNNER_TEMP:?use an isolated GitHub runner temporary directory}" +: "${EMBEDDED_WHEEL_DIR:?wheel output/cache directory is required}" +[[ "$(uname -s)/$(uname -m)" == Linux/x86_64 ]] +PYTHON="$(command -v python3)" +"$PYTHON" -c 'import sys; assert sys.version_info[:2] == (3, 11)' +read -r SEEKDB_REPOSITORY SEEKDB_SHA BINDINGS_SHA FIX_COMMIT < <( + "$PYTHON" -c 'import json, sys; p=json.load(open(sys.argv[1])); print(p["seekdb_repository"], p["seekdb_sha"], p["bindings_sha"], p["fix_commit"])' \ + "$REPO_ROOT/.github/embedded-source.json" +) +[[ "$SEEKDB_REPOSITORY" == "https://github.com/cms-cms/seekdb.git" ]] +[[ "$SEEKDB_SHA" =~ ^[0-9a-f]{40}$ && "$BINDINGS_SHA" =~ ^[0-9a-f]{40}$ && "$FIX_COMMIT" =~ ^[0-9a-f]{40}$ ]] +BUILD_ROOT="$(mktemp -d "$RUNNER_TEMP/embedded-source.XXXXXX")" +mkdir -p "$EMBEDDED_WHEEL_DIR" +[[ -z "$(ls -A "$EMBEDDED_WHEEL_DIR")" ]] || { echo 'refusing to overwrite wheel evidence' >&2; exit 1; } + +checkout_exact() { + local repository="$1" revision="$2" target="$3" + git init -q "$target" + git -C "$target" remote add origin "$repository" + git -C "$target" config remote.origin.promisor true + git -C "$target" config remote.origin.partialclonefilter blob:none + # Full ancestry is needed to prove the kernel contains the required fix. + # Historical source blobs are not: fetch them lazily for the exact checkout. + git -C "$target" fetch --filter=blob:none --no-tags origin "$revision" + git -C "$target" checkout --detach FETCH_HEAD + [[ "$(git -C "$target" rev-parse HEAD)" == "$revision" ]] +} +checkout_exact "$SEEKDB_REPOSITORY" "$SEEKDB_SHA" "$BUILD_ROOT/seekdb" +git -C "$BUILD_ROOT/seekdb" merge-base --is-ancestor "$FIX_COMMIT" HEAD +checkout_exact https://github.com/oceanbase/seekdb-bindings.git "$BINDINGS_SHA" "$BUILD_ROOT/bindings" +git -C "$BUILD_ROOT/bindings" submodule update --init --depth=1 deps/mariadb-connector-c + +# Match the upstream seekdb-bindings Ubuntu source-build recipe. These are +# packaging/client tools, not dependencies of the seekdb compilation target. +DEP_PROFILE="$BUILD_ROOT/seekdb/deps/init/oceanbase.el9.x86_64.deps" +sed -i -E '/^[^#]*(target=(obshell|obdeploy)|(lib)?obclient-)/d' "$DEP_PROFILE" + +# Install the source's exact Rust toolchain once, before parallel compilation; +# never let two Rust targets race while auto-installing the same components. +RUST_TOOLCHAIN="$("$PYTHON" -c 'import sys,tomllib; print(tomllib.load(open(sys.argv[1],"rb"))["toolchain"]["channel"])' \ + "$BUILD_ROOT/seekdb/rust/rust-toolchain.toml")" +[[ "$RUST_TOOLCHAIN" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] +export RUSTUP_HOME="$BUILD_ROOT/rustup" CARGO_HOME="$BUILD_ROOT/cargo" +rustup toolchain install "$RUST_TOOLCHAIN" --profile minimal --no-self-update +export RUSTUP_TOOLCHAIN="$RUST_TOOLCHAIN" RUSTUP_AUTO_INSTALL=0 +rustup run "$RUST_TOOLCHAIN" cargo --version +BUILD_JOBS="$(nproc)" +(( BUILD_JOBS <= 3 )) || BUILD_JOBS=3 +MEMORY_JOBS="$(awk '/MemTotal:/ {n=int(($2/1024-3072)/4096); print (n>0?n:1)}' /proc/meminfo)" +(( BUILD_JOBS <= MEMORY_JOBS )) || BUILD_JOBS="$MEMORY_JOBS" +export CMAKE_BUILD_PARALLEL_LEVEL="$BUILD_JOBS" CARGO_BUILD_JOBS="$BUILD_JOBS" +( + cd "$BUILD_ROOT/seekdb" + ./build.sh release --init -DBUILD_EMBED_MODE=ON --make -j"$BUILD_JOBS" +) +mapfile -t binaries < <(find "$BUILD_ROOT/seekdb/build_release" -type f -name seekdb -executable) +[[ "${#binaries[@]}" == 1 ]] +SEEKDB_BIN="${binaries[0]}" +strip --strip-debug --strip-unneeded "$SEEKDB_BIN" +"$SEEKDB_BIN" -V 2>&1 | tee "$EMBEDDED_WHEEL_DIR/seekdb-version.txt" +grep -F "$SEEKDB_SHA" "$EMBEDDED_WHEEL_DIR/seekdb-version.txt" + +# Do not call the upstream wheel target (it selects an unqualified `python`). +# Build only the driver, then build a CPython 3.11 wheel with this exact venv. +"$PYTHON" -m venv "$BUILD_ROOT/venv" +BUILD_PYTHON="$BUILD_ROOT/venv/bin/python" +"$BUILD_PYTHON" -m pip install --disable-pip-version-check \ + 'pip==25.3' 'build==1.3.0' 'cmake==3.31.6' 'nanobind==3.0.0' 'scikit-build-core==0.11.6' 'ninja==1.11.1.4' +export PATH="$BUILD_ROOT/venv/bin:$PATH" +cmake -S "$BUILD_ROOT/bindings" -B "$BUILD_ROOT/bindings/build" \ + -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTING=OFF -DSEEKDB_BUILD_PYTHON=OFF \ + -DSEEKDB_BIN="$SEEKDB_BIN" +cmake --build "$BUILD_ROOT/bindings/build" --target seekdb --parallel "$BUILD_JOBS" +"$BUILD_PYTHON" -m pip wheel --no-deps --no-build-isolation \ + --config-settings="cmake.define.Python_EXECUTABLE=$BUILD_PYTHON" \ + --config-settings="cmake.define.Python_FIND_VIRTUALENV=ONLY" \ + --wheel-dir "$EMBEDDED_WHEEL_DIR" "$BUILD_ROOT/bindings/python" +"$PYTHON" "$REPO_ROOT/.github/scripts/embedded_wheel_manifest.py" create \ + "$REPO_ROOT/.github/embedded-source.json" "$EMBEDDED_WHEEL_DIR" "$SEEKDB_BIN" diff --git a/.github/scripts/embedded_wheel_manifest.py b/.github/scripts/embedded_wheel_manifest.py new file mode 100644 index 00000000..07c1c38b --- /dev/null +++ b/.github/scripts/embedded_wheel_manifest.py @@ -0,0 +1,75 @@ +"""Bind a CI-local wheel to pinned sources and verify the installed payload.""" + +import hashlib +import importlib.metadata +import json +import platform +import subprocess +import sys +import zipfile +from pathlib import Path + + +def digest(path): + with path.open("rb") as stream: + return hashlib.file_digest(stream, "sha256").hexdigest() + + +def wheel_payload(wheel): + with zipfile.ZipFile(wheel) as archive, archive.open("pylibseekdb/seekdb") as stream: + return hashlib.file_digest(stream, "sha256").hexdigest() + + +def verify(pin, directory): + manifest = json.loads((directory / "manifest.json").read_text()) + if manifest["source"] != pin: + raise ValueError("cached wheel source identity mismatch") + if platform.system() != "Linux" or platform.machine() != "x86_64" or sys.version_info[:2] != (3, 11): + raise ValueError("wheel requires Linux x86_64 CPython 3.11") + wheels = list(directory.glob("*.whl")) + if len(wheels) != 1 or wheels[0].name != manifest["wheel"]: + raise ValueError("expected exactly the manifest wheel") + if digest(wheels[0]) != manifest["wheel_sha256"]: + raise ValueError("wheel checksum mismatch") + if wheel_payload(wheels[0]) != manifest["binary_sha256"]: + raise ValueError("wheel contains a different SeekDB binary") + version = (directory / "seekdb-version.txt").read_text() + if pin["seekdb_sha"] not in version or version != manifest["revision_output"]: + raise ValueError("SeekDB full REVISION mismatch") + return manifest + + +def main(): + mode, pin_path, directory_name, *rest = sys.argv[1:] + pin = json.loads(Path(pin_path).read_text()) + directory = Path(directory_name) + if mode == "create": + (binary_name,) = rest + (wheel,) = directory.glob("*.whl") + manifest = { + "source": pin, + "wheel": wheel.name, + "wheel_sha256": digest(wheel), + "binary_sha256": digest(Path(binary_name)), + "revision_output": (directory / "seekdb-version.txt").read_text(), + } + (directory / "manifest.json").write_text(json.dumps(manifest, indent=2) + "\n") + elif mode not in {"verify", "installed"}: + raise ValueError(f"unknown verification mode: {mode}") + manifest = verify(pin, directory) + if mode == "installed": + binary = Path(importlib.metadata.distribution("pylibseekdb").locate_file("pylibseekdb/seekdb")) + if digest(binary) != manifest["binary_sha256"]: + raise ValueError("installed SeekDB differs from source-built wheel (dependency resync?)") + # The executable is the verified local wheel payload, never shell input. + version = subprocess.check_output( # noqa: S603 + [str(binary), "-V"], text=True, stderr=subprocess.STDOUT + ) + if pin["seekdb_sha"] not in version: + raise ValueError("installed SeekDB does not report pinned full REVISION") + print(f"[EMBEDDED_WHEEL] installed={binary}") + print(json.dumps(manifest, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7fc585d3..9b0182b7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -100,7 +100,11 @@ jobs: bash .github/scripts/check-pytest-summary.sh pytest.log integration-test: - runs-on: ubuntu-latest + # Build and test the CI-local embedded wheel on the same native image. + runs-on: ${{ matrix.test_mode == 'embedded' && 'ubuntu-22.04' || 'ubuntu-latest' }} + timeout-minutes: 180 + env: + EMBEDDED_WHEEL_DIR: ${{ github.workspace }}/.embedded-wheel strategy: fail-fast: false # Allow other jobs to continue if one fails matrix: @@ -137,6 +141,53 @@ jobs: with: python-version: '3.11' + - name: Restore exact embedded source wheel + if: matrix.test_mode == 'embedded' + id: embedded-wheel + uses: actions/cache/restore@v4 + with: + path: .embedded-wheel + # Never reuse a wheel built from another source or build recipe. + key: embedded-source-ubuntu22-x86_64-cp311-${{ hashFiles('.github/embedded-source.json', '.github/scripts/build-embedded-wheel.sh', '.github/scripts/embedded_wheel_manifest.py') }} + + - name: Install embedded source build dependencies + if: matrix.test_mode == 'embedded' && steps.embedded-wheel.outputs.cache-hit != 'true' + run: | + sudo apt-get update + sudo apt-get install -y git wget rpm rpm2cpio cpio make build-essential binutils m4 libtool-bin libncurses5 libaio1 libssl-dev zlib1g-dev + + - name: Build embedded wheel from pinned fixed sources + if: matrix.test_mode == 'embedded' && steps.embedded-wheel.outputs.cache-hit != 'true' + run: bash .github/scripts/build-embedded-wheel.sh + + - name: Verify and install the exact embedded source wheel + if: matrix.test_mode == 'embedded' + run: | + set -euo pipefail + sudo apt-get update + sudo apt-get install -y libaio1 + python .github/scripts/embedded_wheel_manifest.py verify .github/embedded-source.json "$EMBEDDED_WHEEL_DIR" + uv pip install --python .venv/bin/python --no-deps --reinstall "$EMBEDDED_WHEEL_DIR"/*.whl + .venv/bin/python .github/scripts/embedded_wheel_manifest.py installed .github/embedded-source.json "$EMBEDDED_WHEEL_DIR" + + - name: Save verified embedded source wheel + if: matrix.test_mode == 'embedded' && steps.embedded-wheel.outputs.cache-hit != 'true' + uses: actions/cache/save@v4 + with: + path: .embedded-wheel + key: ${{ steps.embedded-wheel.outputs.cache-primary-key }} + + - name: Record embedded wheel provenance + if: matrix.test_mode == 'embedded' + uses: actions/upload-artifact@v4 + with: + name: embedded-wheel-provenance + include-hidden-files: true + path: | + .embedded-wheel/manifest.json + .embedded-wheel/seekdb-version.txt + if-no-files-found: error + - name: Start OceanBase container if: matrix.test_mode == 'oceanbase' uses: oceanbase/setup-oceanbase-ce@v1 @@ -171,7 +222,9 @@ jobs: log_level=INFO fi set -o pipefail - uv run pytest tests/integration_tests/ -v --log-cli-level=${log_level} \ + # Keep the verified source wheel; a normal uv sync would reinstall + # the released pylibseekdb from uv.lock. + uv run --no-sync pytest tests/integration_tests/ -v --log-cli-level=${log_level} \ -k "${{ matrix.test_mode }}" | tee pytest.log bash .github/scripts/check-pytest-summary.sh pytest.log From 458e592246c54c2cfdd88e20cb0864652f7277fb Mon Sep 17 00:00:00 2001 From: cms-cms <1303573631@qq.com> Date: Tue, 22 Sep 2026 11:59:54 +0800 Subject: [PATCH 16/25] ci: validate fork fix from updated seekdb source --- .github/embedded-source.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/embedded-source.json b/.github/embedded-source.json index b4aa48d7..9dacf748 100644 --- a/.github/embedded-source.json +++ b/.github/embedded-source.json @@ -1,6 +1,6 @@ { "seekdb_repository": "https://github.com/cms-cms/seekdb.git", - "seekdb_sha": "3f3b11ab9d6115ee448c2f0f693dfbe7f927c2ea", + "seekdb_sha": "69fc8e6bae590ade24495c4661e3d5b05341716a", "bindings_sha": "5e8eece63205dccfa9500f98893c91d1f50eb4f1", "fix_commit": "48b80f5193bcac0afafba4d4820aebdf1103bc69", "platform": "linux-x86_64", From fa94bb00d1b85caef35227df1eff5e12f1bab769 Mon Sep 17 00:00:00 2001 From: cms-cms <1303573631@qq.com> Date: Tue, 22 Sep 2026 13:52:45 +0800 Subject: [PATCH 17/25] ci: retain embedded diagnostics on failure --- .github/workflows/ci.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9b0182b7..14810488 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -228,6 +228,17 @@ jobs: -k "${{ matrix.test_mode }}" | tee pytest.log bash .github/scripts/check-pytest-summary.sh pytest.log + - name: Upload embedded database diagnostics + if: failure() && matrix.test_mode == 'embedded' + uses: actions/upload-artifact@v4 + with: + name: embedded-database-diagnostics + include-hidden-files: true + path: | + .seekdb-test-data/ + pytest.log + if-no-files-found: warn + - name: Run examples for ${{ matrix.test_mode }} env: MODE: ${{ matrix.test_mode }} From 695fb15059a08e7445204118fb0269b08821af8f Mon Sep 17 00:00:00 2001 From: cms-cms <1303573631@qq.com> Date: Tue, 22 Sep 2026 14:13:13 +0800 Subject: [PATCH 18/25] ci: capture embedded diagnostics for fork analysis --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 14810488..54799c26 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -229,7 +229,7 @@ jobs: bash .github/scripts/check-pytest-summary.sh pytest.log - name: Upload embedded database diagnostics - if: failure() && matrix.test_mode == 'embedded' + if: always() && matrix.test_mode == 'embedded' uses: actions/upload-artifact@v4 with: name: embedded-database-diagnostics From 5dfa29d0f2d6142b350bcdd1492f5d0aecd1177e Mon Sep 17 00:00:00 2001 From: cms-cms <1303573631@qq.com> Date: Tue, 22 Sep 2026 14:43:14 +0800 Subject: [PATCH 19/25] ci: validate active transaction fork fix --- .github/embedded-source.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/embedded-source.json b/.github/embedded-source.json index 9dacf748..ef1c7ccd 100644 --- a/.github/embedded-source.json +++ b/.github/embedded-source.json @@ -1,6 +1,6 @@ { "seekdb_repository": "https://github.com/cms-cms/seekdb.git", - "seekdb_sha": "69fc8e6bae590ade24495c4661e3d5b05341716a", + "seekdb_sha": "3d32ddde64bae917728225ca6a84e00f00aa1177", "bindings_sha": "5e8eece63205dccfa9500f98893c91d1f50eb4f1", "fix_commit": "48b80f5193bcac0afafba4d4820aebdf1103bc69", "platform": "linux-x86_64", From 57da5af9388f4be98cd4d8681fd9ca7df75edbb1 Mon Sep 17 00:00:00 2001 From: cms-cms <1303573631@qq.com> Date: Tue, 22 Sep 2026 17:37:42 +0800 Subject: [PATCH 20/25] ci: validate latest seekdb fork fix --- .github/embedded-source.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/embedded-source.json b/.github/embedded-source.json index ef1c7ccd..4b990f25 100644 --- a/.github/embedded-source.json +++ b/.github/embedded-source.json @@ -1,6 +1,6 @@ { "seekdb_repository": "https://github.com/cms-cms/seekdb.git", - "seekdb_sha": "3d32ddde64bae917728225ca6a84e00f00aa1177", + "seekdb_sha": "39a6560054811f30d3cad3f6fb67aced5162d893", "bindings_sha": "5e8eece63205dccfa9500f98893c91d1f50eb4f1", "fix_commit": "48b80f5193bcac0afafba4d4820aebdf1103bc69", "platform": "linux-x86_64", From 40bc2366a84c5e18f292d84a998bd0f4fe2addbc Mon Sep 17 00:00:00 2001 From: cms-cms <1303573631@qq.com> Date: Tue, 22 Sep 2026 17:47:00 +0800 Subject: [PATCH 21/25] ci: validate complete seekdb fork fix --- .github/embedded-source.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/embedded-source.json b/.github/embedded-source.json index 4b990f25..d1104f96 100644 --- a/.github/embedded-source.json +++ b/.github/embedded-source.json @@ -1,6 +1,6 @@ { "seekdb_repository": "https://github.com/cms-cms/seekdb.git", - "seekdb_sha": "39a6560054811f30d3cad3f6fb67aced5162d893", + "seekdb_sha": "b7c3883ff1564863c2f4e23275f89cb54548a994", "bindings_sha": "5e8eece63205dccfa9500f98893c91d1f50eb4f1", "fix_commit": "48b80f5193bcac0afafba4d4820aebdf1103bc69", "platform": "linux-x86_64", From 8c581ce2f296e966f03ce18ede20914e9a316bc9 Mon Sep 17 00:00:00 2001 From: cms-cms <1303573631@qq.com> Date: Tue, 22 Sep 2026 19:42:57 +0800 Subject: [PATCH 22/25] ci: validate open-transaction fork fix --- .github/embedded-source.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/embedded-source.json b/.github/embedded-source.json index d1104f96..0982563f 100644 --- a/.github/embedded-source.json +++ b/.github/embedded-source.json @@ -1,6 +1,6 @@ { "seekdb_repository": "https://github.com/cms-cms/seekdb.git", - "seekdb_sha": "b7c3883ff1564863c2f4e23275f89cb54548a994", + "seekdb_sha": "a459e66553491e96f7ad7f564b15adfd117f7686", "bindings_sha": "5e8eece63205dccfa9500f98893c91d1f50eb4f1", "fix_commit": "48b80f5193bcac0afafba4d4820aebdf1103bc69", "platform": "linux-x86_64", From d3efb1b7b2138e0d6a082dcf7c27410313611c82 Mon Sep 17 00:00:00 2001 From: cms-cms <1303573631@qq.com> Date: Tue, 22 Sep 2026 21:58:12 +0800 Subject: [PATCH 23/25] ci: validate committed-log fork watermark --- .github/embedded-source.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/embedded-source.json b/.github/embedded-source.json index 0982563f..13545c0a 100644 --- a/.github/embedded-source.json +++ b/.github/embedded-source.json @@ -1,6 +1,6 @@ { "seekdb_repository": "https://github.com/cms-cms/seekdb.git", - "seekdb_sha": "a459e66553491e96f7ad7f564b15adfd117f7686", + "seekdb_sha": "d3c13ac2b8c48b5b9100195be047b8babb4e7a85", "bindings_sha": "5e8eece63205dccfa9500f98893c91d1f50eb4f1", "fix_commit": "48b80f5193bcac0afafba4d4820aebdf1103bc69", "platform": "linux-x86_64", From 62c5a21efb6f6fe61fc55f6911681ef2471b3ee8 Mon Sep 17 00:00:00 2001 From: cms-cms <1303573631@qq.com> Date: Wed, 23 Sep 2026 00:03:23 +0800 Subject: [PATCH 24/25] ci: validate fork pre-drain ordering --- .github/embedded-source.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/embedded-source.json b/.github/embedded-source.json index 13545c0a..41755189 100644 --- a/.github/embedded-source.json +++ b/.github/embedded-source.json @@ -1,6 +1,6 @@ { "seekdb_repository": "https://github.com/cms-cms/seekdb.git", - "seekdb_sha": "d3c13ac2b8c48b5b9100195be047b8babb4e7a85", + "seekdb_sha": "b255464c3df78d1b8f769bcb71448e5737940fcc", "bindings_sha": "5e8eece63205dccfa9500f98893c91d1f50eb4f1", "fix_commit": "48b80f5193bcac0afafba4d4820aebdf1103bc69", "platform": "linux-x86_64", From d6f71e55269f559b8a7beadf21db311374cdef67 Mon Sep 17 00:00:00 2001 From: cms-cms <1303573631@qq.com> Date: Thu, 24 Sep 2026 19:10:12 +0800 Subject: [PATCH 25/25] test: detect current embedded seekdb runtime --- tests/integration_tests/conftest.py | 22 +++++++++++------- ...t_get_or_create_collection_multiprocess.py | 23 +++++++++++-------- 2 files changed, 27 insertions(+), 18 deletions(-) diff --git a/tests/integration_tests/conftest.py b/tests/integration_tests/conftest.py index 5eda1f80..e1d207b1 100644 --- a/tests/integration_tests/conftest.py +++ b/tests/integration_tests/conftest.py @@ -4,6 +4,7 @@ """ import contextlib +import importlib import os import sys from pathlib import Path @@ -45,12 +46,20 @@ # ==================== Client Factory Functions ==================== +def require_embedded_runtime(): + """Require the current seekdb binding or its legacy pylibseekdb name.""" + for module_name in ("seekdb", "pylibseekdb"): + try: + importlib.import_module(module_name) + return + except ImportError: + continue + pytest.skip("seekdb embedded package is not installed") + + def create_embedded_client(): """Create an embedded client instance.""" - try: - import pylibseekdb # noqa: F401 - except ImportError: - pytest.skip("seekdb embedded package is not installed") + require_embedded_runtime() return pyseekdb.Client(path=SEEKDB_PATH, database=SEEKDB_DATABASE) @@ -100,10 +109,7 @@ def create_oceanbase_client(): # ==================== AdminClient Factory Functions ==================== def create_embedded_admin_client(): """Create an embedded admin client instance.""" - try: - import pylibseekdb # noqa: F401 - except ImportError: - pytest.skip("seekdb embedded package is not installed") + require_embedded_runtime() return pyseekdb.AdminClient(path=SEEKDB_PATH) diff --git a/tests/integration_tests/test_get_or_create_collection_multiprocess.py b/tests/integration_tests/test_get_or_create_collection_multiprocess.py index 248a5028..a2379618 100644 --- a/tests/integration_tests/test_get_or_create_collection_multiprocess.py +++ b/tests/integration_tests/test_get_or_create_collection_multiprocess.py @@ -144,22 +144,25 @@ def _refresh_collection(client_config: dict[str, Any], collection_name: str) -> collection.refresh_index() -def _require_embedded_pylibseekdb() -> None: - """Require embedded pylibseekdb.""" +def _require_embedded_seekdb() -> None: + """Require the current seekdb binding or its legacy pylibseekdb name.""" + distribution = "seekdb" try: - import pylibseekdb # noqa: F401 + importlib.import_module(distribution) except ImportError: - pytest.skip("seekdb embedded package is not installed") + distribution = "pylibseekdb" + try: + importlib.import_module(distribution) + except ImportError: + pytest.skip("seekdb embedded package is not installed") try: - installed_version = Version(importlib.metadata.version("pylibseekdb")) + installed_version = Version(importlib.metadata.version(distribution)) except importlib.metadata.PackageNotFoundError: - pytest.skip("pylibseekdb is not installed") + pytest.skip(f"{distribution} is not installed") if installed_version < MIN_PYLIBSEEKDB_VERSION: - pytest.skip( - f"embedded multiprocess tests require pylibseekdb >= {MIN_PYLIBSEEKDB_VERSION}, got {installed_version}" - ) + pytest.skip(f"embedded multiprocess tests require seekdb >= {MIN_PYLIBSEEKDB_VERSION}, got {installed_version}") def _run_processes( @@ -486,7 +489,7 @@ def _build_client_config(mode: str) -> tuple[dict[str, Any], Path | None, Any]: temp_db_path: Path | None = None if mode == "embedded": - _require_embedded_pylibseekdb() + _require_embedded_seekdb() SEEKDB_TEST_DATA_ROOT.mkdir(parents=True, exist_ok=True) temp_db_path = Path(tempfile.mkdtemp(prefix="seekdb-mp-", dir=SEEKDB_TEST_DATA_ROOT)) client_config = {"mode": "embedded", "path": str(temp_db_path), "database": database}