From 720e3a34387d826f78baff2ba9065223f38aab35 Mon Sep 17 00:00:00 2001 From: fanng Date: Mon, 14 Sep 2026 12:20:05 +0900 Subject: [PATCH 1/3] chore: pin pylance 8.0.0 --- AGENTS.md | 2 +- README.md | 18 ++++++++++------- docs/daft-tuning.md | 4 +++- multimodal_toolkit/workflow/manage.py | 29 ++++++++++++++------------- pyproject.toml | 2 +- tests/workflow/test_index_manage.py | 9 ++++----- uv.lock | 28 +++++++++++++------------- 7 files changed, 49 insertions(+), 43 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 2d811ae..b7c9fad 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -28,7 +28,7 @@ tests/ # pytest；按 audio/image/storage/workflow 分目录 ## 硬性架构约定 1. **媒体隔离**：audio 和 image 各自拥有独立的 package 和 workflow 入口，**不要**把共享 Stage 参数化成"通用管线"。可共享的只有 `storage/` 层 infra 和 `config.py` 的辅助函数。 -2. **数据 API 优先级**：所有数据操作优先用 **Daft**；Daft 做不到再用 **lance-ray**；**pylance 直接调用是最后手段**（目前锁在 <8，blob compaction 被禁用）。 +2. **数据 API 优先级**：所有数据操作优先用 **Daft**；Daft 做不到再用 **lance-ray**；**pylance 直接调用是最后手段**。当前固定为 pylance 8.0.0；blob compaction 仍需在单独改动中完成 lance-ray 兼容性和对象存储回归验证后再启用。 3. **行不丢弃**：analyze 阶段 manifest 里每个条目对应输出一行，失败用 `status` 列标记（`download_failed` / `decode_failed` / ...），分数结论置 null。合规场景必须能区分"内容有问题"和"根本没处理"。 4. **配置只走环境变量**，经 `config.py` 的 `env_int`/`env_bool`/`env_choice` 等辅助函数读取。注意：Daft 原生只读少数 `DAFT_*` 环境变量，新增 Daft 执行参数必须在 `storage/io.py::configure_daft_runner()` 里显式传给 `daft.set_execution_config()`，否则是死配置；S3 参数同理要传进 `daft_io_config()` 的 `S3Config`。 5. **写出受控**：`write_lance` 必须传 `LANCE_MAX_ROWS_PER_FILE` / `LANCE_MAX_BYTES_PER_FILE`；JSONL 写出前用 `coalesce_for_write()` 收敛分区，避免小文件。 diff --git a/README.md b/README.md index 1d8150c..e1abd7b 100644 --- a/README.md +++ b/README.md @@ -410,8 +410,8 @@ python -m multimodal_toolkit.workflow.manage \ |------|------|------| | Daft | 0.7.15 | 主执行引擎 | | daft-lance | 0.4.0 | `read_lance`、`write_lance`、`take_blobs`、`create_scalar_index`、`compact_files` | -| pylance | 7.0.0 | Lance dataset、blob v2、ANN scanner、delete、cleanup | -| lance-ray | 0.4.2 | 向量索引创建；写回路径暂缓 | +| pylance | 8.0.0 | Lance dataset、blob v2、ANN scanner、delete、cleanup | +| lance-ray | 0.4.2 | 向量索引创建；与 pylance 8 的分布式提交兼容性见下方限制 | | Ray | 2.55.1 | 由 lance-ray 引入；除非 `USE_RAY=1`，否则 Daft 走 native runner | Daft 默认 runner 是 `native`（本地多线程）。设置 `USE_RAY=1` 可把 Daft 相关步骤切到 Ray。 @@ -455,8 +455,12 @@ confidence 与 reason 字段在本地行中为 null。VLM 调用失败时仍保 **本地 Lance URI 已做端到端验证。** S3 上的 Lance 表读写由底层库覆盖，但在本 POC 中应作为一个独立的验证项对待。 -**blob v2 资产表的 compaction 暂时禁用。** -当前 `lance-ray 0.4.2` 的向量索引创建依赖 pylance 7.x，而 pylance 8.0.0 虽然修复了 blob v2 -compaction，却改动了 `lance-ray 0.4.2` 仍在调用的分布式索引 commit API。项目暂时锁定 -`pylance<8.0.0`，Stage 5 在删除行之后跳过 compaction。等升级到 pylance 8.x 并配套兼容的 -lance-ray 版本后再重新开启 blob v2 compaction。 +**pylance 8.0.0 与 lance-ray 0.4.2 存在已知的分布式索引限制。** +pylance 8.0.0 修复了 blob v2 compaction，但移除了 `lance-ray 0.4.2` 在分布式向量索引提交阶段 +调用的 `create_index_segment_builder()`。Stage 3 会捕获该错误并回退到 pylance 本地建索引， +所以索引功能仍可用，但大表会失去 Ray 分布式建索引的资源扩展能力。等配套兼容的 lance-ray +版本可用后，应移除这条回退限制。 + +**blob v2 资产表的 compaction 仍暂时禁用。** +本次依赖升级不改变 Stage 5 的表管理行为；删除行之后仍跳过 compaction。后续启用时需单独覆盖 +lance-ray、本地 Lance URI 和 MinIO/S3 的 Blob v2 回归验证，避免把版本对齐和行为变更混在一起。 diff --git a/docs/daft-tuning.md b/docs/daft-tuning.md index a4b4803..1990b52 100644 --- a/docs/daft-tuning.md +++ b/docs/daft-tuning.md @@ -112,7 +112,9 @@ CPU/网络 在下载下一批图片 | `LANCE_MAX_BYTES_PER_FILE` | `512MB` | 单个 Lance 数据文件最大字节数。 | | `DAFT_JSON_TARGET_FILESIZE` | `128MB` | JSONL 单文件目标大小。 | -为什么要管：碎文件会拖慢后续所有读取，而我们的 pylance 7 禁用了 blob compaction，写坏了没有事后补救，只能在写入侧控制。默认值一般不用动。 +为什么要管：碎文件会拖慢后续所有读取。虽然 pylance 已升级到 8.0.0，Stage 5 仍要等 +lance-ray 和对象存储路径完成独立回归验证后才会开启 blob compaction；在此之前仍应从写入侧 +控制文件布局。默认值一般不用动。 ### 3.7 观测（长任务建议必开） diff --git a/multimodal_toolkit/workflow/manage.py b/multimodal_toolkit/workflow/manage.py index 1a6091e..47b1631 100644 --- a/multimodal_toolkit/workflow/manage.py +++ b/multimodal_toolkit/workflow/manage.py @@ -1,15 +1,15 @@ -"""Stage 5: manage the lance asset table — delete rows by ingest_time range. +"""Stage 5：按 ingest_time 范围删除 Lance 资产表中的行。 -API priority (lance_ray preferred for table management): - delete → pylance ds.delete() (no Daft/lance-ray equivalent) - compact → disabled for blob v2 tables until pylance 8 is supported - cleanup → pylance ds.cleanup_old_versions() (no alternative) +表管理仍遵循项目的数据 API 优先级： + delete → pylance ds.delete() （Daft/lance-ray 没有等价 API） + compact → 暂不执行；升级 pylance 后仍需单独验证 lance-ray 和对象存储路径 + cleanup → pylance ds.cleanup_old_versions() （没有替代 API） - --before DATE delete rows where ingest_time < DATE - --after DATE delete rows where ingest_time > DATE + --before DATE 删除 ingest_time < DATE 的行 + --after DATE 删除 ingest_time > DATE 的行 -DATE format: ISO 8601, e.g. 2025-01-01 or 2025-01-01T00:00:00 -At least one bound must be provided; both can be combined for a date range. +DATE 使用 ISO 8601 格式，例如 2025-01-01 或 2025-01-01T00:00:00。 +至少传一个边界；两个边界可以组合成日期范围。 """ from __future__ import annotations @@ -35,16 +35,17 @@ def delete_by_date( clauses.append(f"ingest_time > timestamp '{after}'") filter_str = " AND ".join(clauses) - # delete: pylance only (no Daft/lance-ray equivalent) + # 删除目前只有 pylance 提供 API；这里保留直接调用，不能为了统一入口而绕过 + # Lance 自身的事务提交语义。 ds = lance.dataset(lance_uri, storage_options=lance_storage_options(lance_uri)) ds.delete(filter_str) print(f"[ok] deleted rows where: {filter_str}") - # Blob v2 tables cannot be compacted safely on pylance 7.x - # (lance-format/lance#7071). Keep compaction disabled until the project - # can move to pylance 8.x together with a compatible lance-ray release. - print("[warn] compaction skipped; enable after upgrading to pylance 8.x") + # pylance 8.0.0 已修复 Blob v2 compaction 的解码问题，但版本升级本身不等于 + # 分布式 compaction 链路已经验证。这里继续跳过，待后续改动同时覆盖 lance-ray、 + # 本地 Lance URI 和 MinIO/S3 后再开启，避免依赖升级悄悄扩大表管理行为。 + print("[warn] compaction skipped; enable after validating lance-ray and object storage") ds.cleanup_old_versions() diff --git a/pyproject.toml b/pyproject.toml index 44316a1..b4f614a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,7 +19,7 @@ dependencies = [ "pillow", "psutil", "pyarrow", - "pylance<8.0.0", + "pylance==8.0.0", "python-dotenv", "ray", "soundfile", diff --git a/tests/workflow/test_index_manage.py b/tests/workflow/test_index_manage.py index 8f67849..47b8589 100644 --- a/tests/workflow/test_index_manage.py +++ b/tests/workflow/test_index_manage.py @@ -1,9 +1,8 @@ -"""Tests for workflow/index.py and workflow/manage.py — local lance tables. +"""在本地 Lance 表上验证 workflow/index.py 和 workflow/manage.py。 -build_embedding_index is marked `ray` and excluded from the default run (see -pyproject) — run it explicitly with `pytest -m ray`. delete_by_date does not -compact while the project is pinned to pylance 7.x, so those tests stay in the -default suite. +build_embedding_index 的真实 Ray 路径不属于默认测试；需要时用 `pytest -m ray` +显式执行。pylance 已升级到 8.0.0，但 delete_by_date 仍有意不执行 compaction， +因此删除测试继续留在默认测试集。 """ from __future__ import annotations diff --git a/uv.lock b/uv.lock index cef60ab..0abe688 100644 --- a/uv.lock +++ b/uv.lock @@ -691,19 +691,19 @@ wheels = [ [[package]] name = "lance-namespace" -version = "0.7.7" +version = "0.8.6" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "lance-namespace-urllib3-client" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/06/5c/9822af615fc1bd3ee1073994696c739aecde377be32435ec3303aed1bc5d/lance_namespace-0.7.7.tar.gz", hash = "sha256:d00b525f2e26993a6c61668e798bca6c808605ab8a79f29f86a1a1af92d91ae2", size = 10754, upload-time = "2026-05-20T17:32:59.45Z" } +sdist = { url = "https://files.pythonhosted.org/packages/af/12/f7ab93b29be3edbf5fc3610714bf2d06088e7f4524bfb38dfd6852458b08/lance_namespace-0.8.6.tar.gz", hash = "sha256:18232e721c8188145f4ec9389cc2dfbeeabf54a619d94885ea1b3375bee9f4af", size = 11529, upload-time = "2026-06-12T17:36:41.651Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/11/43/186acc1156da20c351db196e2b6241b2453b16dc1b4cc8e0a626667ca471/lance_namespace-0.7.7-py3-none-any.whl", hash = "sha256:477a7ca6b5e1f673a2c9ba52f42d6e8e3ff7c27a601392a21eb90fba98d0309b", size = 12581, upload-time = "2026-05-20T17:32:57.389Z" }, + { url = "https://files.pythonhosted.org/packages/a0/1b/5b1668ee2dc8910965f390640359112a31157092fcf8e000b89c79b58708/lance_namespace-0.8.6-py3-none-any.whl", hash = "sha256:571eae34f9aad70e5b05020416c2860889b9ec82993ccd0eb015e7b39c3ea309", size = 13383, upload-time = "2026-06-12T17:36:43.456Z" }, ] [[package]] name = "lance-namespace-urllib3-client" -version = "0.7.7" +version = "0.8.6" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "pydantic" }, @@ -711,9 +711,9 @@ dependencies = [ { name = "typing-extensions" }, { name = "urllib3" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/07/95/38ab81ccc1e09beeecd8ddfc61b8bc73831dc5053db1e3f9021f64a4896b/lance_namespace_urllib3_client-0.7.7.tar.gz", hash = "sha256:4d8c066628c17c6a10cf643b51a7f7ae1bfb8a614d9cc54a5af38a4ba2b4b102", size = 202930, upload-time = "2026-05-20T17:32:58.308Z" } +sdist = { url = "https://files.pythonhosted.org/packages/c7/80/fb224b4a89c1c1638cde949cb6cce6c3aca7759effbfea46a3d9c3960b21/lance_namespace_urllib3_client-0.8.6.tar.gz", hash = "sha256:b6fb1d306e74a7576e5309919020be744527de484a63dbf5eed10f8b368548df", size = 228772, upload-time = "2026-06-12T17:36:42.609Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/35/96/5483e48e40433b1d078183c15a92c99e59a156041b0260e7f18ee34e7c08/lance_namespace_urllib3_client-0.7.7-py3-none-any.whl", hash = "sha256:9221c3e00fd89f0c811953d94b32d2ea527765280460a174f5872dc8a74c0ed6", size = 334767, upload-time = "2026-05-20T17:32:55.883Z" }, + { url = "https://files.pythonhosted.org/packages/c5/90/1e27de15cd1b16785a1c7312beb0a59e75c8344a815f600f58173a565bd1/lance_namespace_urllib3_client-0.8.6-py3-none-any.whl", hash = "sha256:9d78249c3fb15aa3d15d668f78f04a275af3d08d800a7027492f37996ac4968b", size = 369950, upload-time = "2026-06-12T17:36:40.438Z" }, ] [[package]] @@ -927,7 +927,7 @@ requires-dist = [ { name = "pillow" }, { name = "psutil" }, { name = "pyarrow" }, - { name = "pylance", specifier = "<8.0.0" }, + { name = "pylance", specifier = "==8.0.0" }, { name = "python-dotenv" }, { name = "ray" }, { name = "soundfile" }, @@ -1473,7 +1473,7 @@ wheels = [ [[package]] name = "pylance" -version = "7.0.0" +version = "8.0.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "lance-namespace" }, @@ -1481,12 +1481,12 @@ dependencies = [ { name = "pyarrow" }, ] wheels = [ - { url = "https://files.pythonhosted.org/packages/ac/ad/2f64921bf346e7075aef24a72595db44821724a3d89a9a92dd24e79632aa/pylance-7.0.0-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:98422021975be76e72b1572f41b8c9abb3bee5bdc9bfa5e9ce731110a65ed4d1", size = 62134146, upload-time = "2026-05-27T21:59:37.459Z" }, - { url = "https://files.pythonhosted.org/packages/73/1c/c5a01bee0160b55d9a98895cbd33091d038f0a0995b121ab72e629008d02/pylance-7.0.0-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4bec86ee5b6fbd8bfc493e653f0a1fba0303cfe5492b9b46fc25ab908edc7183", size = 65373684, upload-time = "2026-05-27T22:04:01.584Z" }, - { url = "https://files.pythonhosted.org/packages/eb/da/1fe8b8f7dbfe734d76af76acc994fc360a0d0c79a4874ef69f5a72a58fe3/pylance-7.0.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:881491432c53184e52f8d1db8d5f872f39a03f36fb104bec77b33d379519d8b5", size = 69458555, upload-time = "2026-05-27T22:16:50.567Z" }, - { url = "https://files.pythonhosted.org/packages/76/f0/dd505cf3fd0226ab9d94759acd713125af1d3bfacfd80bbd52e3b9f89509/pylance-7.0.0-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:18453999e7fff4f76b16d6b7882c9df0628bd142ff95e2461bd7dd5ee3fe0af3", size = 65394430, upload-time = "2026-05-27T22:05:30.923Z" }, - { url = "https://files.pythonhosted.org/packages/17/ba/2357b81034f28eb00790e258ed140289a6a887a7468ca9df6349fd186b27/pylance-7.0.0-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:04a58051d408c60fe76d41a220dcaf8fea8fb6d1aa0ca78a709b60bc3cc8d19a", size = 69473470, upload-time = "2026-05-27T22:17:18.935Z" }, - { url = "https://files.pythonhosted.org/packages/1f/ec/5c00b6303a67d787f9475141832cbdc513d674ac3dcaeef8a7b169905e65/pylance-7.0.0-cp39-abi3-win_amd64.whl", hash = "sha256:467d4864af047eaab4e1370e2f1e88e2c6f507c079874421116cb41d78bc3629", size = 74792863, upload-time = "2026-05-27T22:19:23.875Z" }, + { url = "https://files.pythonhosted.org/packages/3b/99/03db9f4f723353502d2b5f5d8a0eb9a30f0dc41a8cc036ac5fa486471223/pylance-8.0.0-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:769cb38e43a533fdffb87a804043f6a37793eb77d4acbc14cffe9e45dc8bd90b", size = 67235437, upload-time = "2026-07-01T15:09:09.519Z" }, + { url = "https://files.pythonhosted.org/packages/48/af/8f8f8b9c70200e3ab99d8785c26ba691d0ff475cae27f7620618dfc8f3b6/pylance-8.0.0-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:597277a993d159dfcf58230958f1c5f6edebca8693fbd3502129be951ec6927e", size = 70558570, upload-time = "2026-07-01T15:13:50.95Z" }, + { url = "https://files.pythonhosted.org/packages/ca/4c/b10b02a7891753ec1d4a846b8cc001fbafb4bf7c48c2bda5303cf57c1b8a/pylance-8.0.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2c8823134a09ff311b0a963227bdfc160c686910944a373af357cd5b3fa70b5c", size = 74747275, upload-time = "2026-07-01T15:27:09.817Z" }, + { url = "https://files.pythonhosted.org/packages/4c/56/09602b664d8036aa0105ffdb94189c23f9fde4ce71963f4a98f05fda526d/pylance-8.0.0-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:28c98c6ae99edc76622e2af331e99da4d57c9f44c32ebba2872a8e60dee9f7f8", size = 70578246, upload-time = "2026-07-01T15:14:53.143Z" }, + { url = "https://files.pythonhosted.org/packages/4b/1a/9f49850dfee7b1e763f358ff7477a1aa27228d1e40f907020f13ea13b3e1/pylance-8.0.0-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:0b779a7465424de04975cedc774b1eadff8638e0f96f6f19245e9bea332e1e62", size = 74708654, upload-time = "2026-07-01T15:27:54.326Z" }, + { url = "https://files.pythonhosted.org/packages/77/5d/1876b43e680b9afb0d827af032bd72d3f70a9eaf68cb8f5f4286b20a613e/pylance-8.0.0-cp39-abi3-win_amd64.whl", hash = "sha256:ea2e35722f546eb2e056eafb1cc39ced5f353157653516a1472f1127bfa92eec", size = 80511928, upload-time = "2026-07-01T15:28:56.53Z" }, ] [[package]] From 8b2e01bca622dce9f4072f6b5378c812a7a1fda4 Mon Sep 17 00:00:00 2001 From: fanng Date: Mon, 14 Sep 2026 15:07:00 +0900 Subject: [PATCH 2/3] feat: upgrade lance ray and pylance --- AGENTS.md | 4 +- README.md | 18 +++---- docs/daft-tuning.md | 2 +- multimodal_toolkit/workflow/index.py | 17 ++----- multimodal_toolkit/workflow/manage.py | 2 +- pyproject.toml | 9 +++- tests/workflow/test_index_manage.py | 37 ++------------ uv.lock | 72 ++++++++------------------- 8 files changed, 47 insertions(+), 114 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index b7c9fad..b49ad71 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -28,7 +28,7 @@ tests/ # pytest；按 audio/image/storage/workflow 分目录 ## 硬性架构约定 1. **媒体隔离**：audio 和 image 各自拥有独立的 package 和 workflow 入口，**不要**把共享 Stage 参数化成"通用管线"。可共享的只有 `storage/` 层 infra 和 `config.py` 的辅助函数。 -2. **数据 API 优先级**：所有数据操作优先用 **Daft**；Daft 做不到再用 **lance-ray**；**pylance 直接调用是最后手段**。当前固定为 pylance 8.0.0；blob compaction 仍需在单独改动中完成 lance-ray 兼容性和对象存储回归验证后再启用。 +2. **数据 API 优先级**：所有数据操作优先用 **Daft**；Daft 做不到再用 **lance-ray**；**pylance 直接调用是最后手段**。当前固定为 lance-ray 0.5.0 与 pylance 9.0.0；blob compaction 仍需在单独改动中完成对象存储回归验证后再启用。 3. **行不丢弃**：analyze 阶段 manifest 里每个条目对应输出一行，失败用 `status` 列标记（`download_failed` / `decode_failed` / ...），分数结论置 null。合规场景必须能区分"内容有问题"和"根本没处理"。 4. **配置只走环境变量**，经 `config.py` 的 `env_int`/`env_bool`/`env_choice` 等辅助函数读取。注意：Daft 原生只读少数 `DAFT_*` 环境变量，新增 Daft 执行参数必须在 `storage/io.py::configure_daft_runner()` 里显式传给 `daft.set_execution_config()`，否则是死配置；S3 参数同理要传进 `daft_io_config()` 的 `S3Config`。 5. **写出受控**：`write_lance` 必须传 `LANCE_MAX_ROWS_PER_FILE` / `LANCE_MAX_BYTES_PER_FILE`；JSONL 写出前用 `coalesce_for_write()` 收敛分区，避免小文件。 @@ -43,7 +43,7 @@ USE_RAY=1 ... # 切换 Ray runner（默认 native） ``` - 本地依赖 MinIO（默认 `http://127.0.0.1:9000`，minioadmin/minioadmin），配置见 `.env` / README Setup 小节。 -- venv 里的 lance-ray 是本地 dev checkout（版本号 0.4.2 与 PyPI 同名但内容不同），不要"顺手升级"它。 +- lance-ray 与 pylance 必须按 `pyproject.toml` 和 `uv.lock` 固定的配套版本安装，不能只升级其中一个。 ## 提交与 PR diff --git a/README.md b/README.md index e1abd7b..3917169 100644 --- a/README.md +++ b/README.md @@ -66,7 +66,7 @@ Manifest (parquet / jsonl / csv) │ └──▶ Stage 5 — workflow/manage.py pylance ds.delete() : --before / --after - lance_ray.compact_files : delete 之后自动执行 + compaction : 当前暂不执行 ``` ### 引擎分工 @@ -74,9 +74,9 @@ Manifest (parquet / jsonl / csv) | 引擎 | 用于 | 原因 | |----------------|-----------------------------------------------------------------------------|-------------------------------------------------------------| | **Daft** | manifest 读取、S3 下载、ASR/LLM 流水线、Lance 写入（Stage 1 & 2）、标量与 ANN 查询 | 主引擎；API 稳定 | -| **lance_ray** | IVF_PQ 向量索引创建、`compact_files` | Lance 表管理的首选；可用分布式 Ray worker | +| **lance_ray** | IVF_PQ 向量索引创建、`compact_files` | 分布式索引与表维护能力；本项目暂未启用 compaction | | **pylance** | ZONEMAP 标量索引、行删除、`cleanup_old_versions` | ZONEMAP：lance_ray 依赖未发布代码；delete：只有这一个 API | -| **daft_lance** | lance_ray 不可用时 `compact_files` 的兜底 | 不用于建索引；Daft 优先原则只适用于数据处理 | +| **daft_lance** | `read_lance`、`write_lance` 等 Daft 数据操作 | 不用于建索引；Daft 优先原则只适用于数据处理 | ## 环境准备 @@ -410,8 +410,8 @@ python -m multimodal_toolkit.workflow.manage \ |------|------|------| | Daft | 0.7.15 | 主执行引擎 | | daft-lance | 0.4.0 | `read_lance`、`write_lance`、`take_blobs`、`create_scalar_index`、`compact_files` | -| pylance | 8.0.0 | Lance dataset、blob v2、ANN scanner、delete、cleanup | -| lance-ray | 0.4.2 | 向量索引创建；与 pylance 8 的分布式提交兼容性见下方限制 | +| pylance | 9.0.0 | Lance dataset、blob v2、ANN scanner、delete、cleanup | +| lance-ray | 0.5.0 | 分布式向量索引创建与分布式 compaction API | | Ray | 2.55.1 | 由 lance-ray 引入；除非 `USE_RAY=1`，否则 Daft 走 native runner | Daft 默认 runner 是 `native`（本地多线程）。设置 `USE_RAY=1` 可把 Daft 相关步骤切到 Ray。 @@ -455,11 +455,9 @@ confidence 与 reason 字段在本地行中为 null。VLM 调用失败时仍保 **本地 Lance URI 已做端到端验证。** S3 上的 Lance 表读写由底层库覆盖，但在本 POC 中应作为一个独立的验证项对待。 -**pylance 8.0.0 与 lance-ray 0.4.2 存在已知的分布式索引限制。** -pylance 8.0.0 修复了 blob v2 compaction，但移除了 `lance-ray 0.4.2` 在分布式向量索引提交阶段 -调用的 `create_index_segment_builder()`。Stage 3 会捕获该错误并回退到 pylance 本地建索引， -所以索引功能仍可用，但大表会失去 Ray 分布式建索引的资源扩展能力。等配套兼容的 lance-ray -版本可用后，应移除这条回退限制。 +**分布式向量索引失败会直接报错。** +Stage 3 固定使用 lance-ray 0.5.0 的分布式提交链路，不再回退到 pylance 单机建索引。这样调用方 +能明确获知 Ray、对象存储凭据或索引提交失败，避免大表在驱动进程静默退化为单机执行。 **blob v2 资产表的 compaction 仍暂时禁用。** 本次依赖升级不改变 Stage 5 的表管理行为；删除行之后仍跳过 compaction。后续启用时需单独覆盖 diff --git a/docs/daft-tuning.md b/docs/daft-tuning.md index 1990b52..4d37129 100644 --- a/docs/daft-tuning.md +++ b/docs/daft-tuning.md @@ -112,7 +112,7 @@ CPU/网络 在下载下一批图片 | `LANCE_MAX_BYTES_PER_FILE` | `512MB` | 单个 Lance 数据文件最大字节数。 | | `DAFT_JSON_TARGET_FILESIZE` | `128MB` | JSONL 单文件目标大小。 | -为什么要管：碎文件会拖慢后续所有读取。虽然 pylance 已升级到 8.0.0，Stage 5 仍要等 +为什么要管：碎文件会拖慢后续所有读取。虽然 pylance 已升级到 9.0.0，Stage 5 仍要等 lance-ray 和对象存储路径完成独立回归验证后才会开启 blob compaction；在此之前仍应从写入侧 控制文件布局。默认值一般不用动。 diff --git a/multimodal_toolkit/workflow/index.py b/multimodal_toolkit/workflow/index.py index 965d066..3ce22b2 100644 --- a/multimodal_toolkit/workflow/index.py +++ b/multimodal_toolkit/workflow/index.py @@ -42,20 +42,9 @@ def build_embedding_index( ) if index_type in ("IVF_PQ", "IVF_HNSW_PQ"): kwargs["num_sub_vectors"] = num_sub_vectors - try: - lance_ray.create_index(lance_uri, **kwargs) - except Exception as exc: - print(f"[warn] lance_ray index build failed; falling back to pylance: {exc}") - fallback_kwargs: dict = dict( - index_type=index_type, - replace=True, - num_partitions=num_partitions, - sample_rate=sample_rate, - storage_options=storage_options, - ) - if index_type in ("IVF_PQ", "IVF_HNSW_PQ"): - fallback_kwargs["num_sub_vectors"] = num_sub_vectors - ds.create_index(column, **fallback_kwargs) + # 向量索引必须由 lance-ray 的分布式提交链路完成。不能静默回退到 pylance + # 单机建索引，否则大表会在驱动进程失去并行能力且调用方无法感知执行模式变化。 + lance_ray.create_index(lance_uri, **kwargs) print(f"[ok] built {index_type} index on {column} ({num_partitions} partitions): {lance_uri}") diff --git a/multimodal_toolkit/workflow/manage.py b/multimodal_toolkit/workflow/manage.py index 47b1631..2613691 100644 --- a/multimodal_toolkit/workflow/manage.py +++ b/multimodal_toolkit/workflow/manage.py @@ -42,7 +42,7 @@ def delete_by_date( print(f"[ok] deleted rows where: {filter_str}") - # pylance 8.0.0 已修复 Blob v2 compaction 的解码问题，但版本升级本身不等于 + # pylance 9.0.0 已修复 Blob v2 compaction 的解码问题，但版本升级本身不等于 # 分布式 compaction 链路已经验证。这里继续跳过，待后续改动同时覆盖 lance-ray、 # 本地 Lance URI 和 MinIO/S3 后再开启，避免依赖升级悄悄扩大表管理行为。 print("[warn] compaction skipped; enable after validating lance-ray and object storage") diff --git a/pyproject.toml b/pyproject.toml index b4f614a..1f986cd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,7 +10,7 @@ dependencies = [ "getdaft", "httpx", "insightface", - "lance-ray", + "lance-ray==0.5.0", "modelscope", "numpy", "onnxruntime", @@ -19,7 +19,7 @@ dependencies = [ "pillow", "psutil", "pyarrow", - "pylance==8.0.0", + "pylance==9.0.0", "python-dotenv", "ray", "soundfile", @@ -28,6 +28,11 @@ dependencies = [ "transformers", ] +[dependency-groups] +dev = [ + "pytest>=9.1.1", +] + [tool.pytest.ini_options] markers = [ "ray: hard dependency on a local Ray cluster; excluded by default, run with `pytest -m ray`", diff --git a/tests/workflow/test_index_manage.py b/tests/workflow/test_index_manage.py index 47b8589..59acd71 100644 --- a/tests/workflow/test_index_manage.py +++ b/tests/workflow/test_index_manage.py @@ -1,7 +1,7 @@ """在本地 Lance 表上验证 workflow/index.py 和 workflow/manage.py。 build_embedding_index 的真实 Ray 路径不属于默认测试；需要时用 `pytest -m ray` -显式执行。pylance 已升级到 8.0.0，但 delete_by_date 仍有意不执行 compaction， +显式执行。pylance 已升级到 9.0.0，但 delete_by_date 仍有意不执行 compaction， 因此删除测试继续留在默认测试集。 """ from __future__ import annotations @@ -91,41 +91,14 @@ def fake_create_index(uri, **kwargs): ] -def test_build_embedding_index_falls_back_to_pylance(monkeypatch): - calls = [] - - class FakeSchema: - names = ["audio_embedding"] - - class FakeDataset: - schema = FakeSchema() - - def create_index(self, column, **kwargs): - calls.append((column, kwargs)) - +def test_build_embedding_index_propagates_lance_ray_failure(monkeypatch, lance_uri): def fake_create_index(uri, **kwargs): - raise RuntimeError("ray worker failed") + raise RuntimeError("distributed index failed") - monkeypatch.setattr( - "multimodal_toolkit.workflow.index.lance.dataset", - lambda *args, **kwargs: FakeDataset(), - ) monkeypatch.setattr("multimodal_toolkit.workflow.index.lance_ray.create_index", fake_create_index) - build_embedding_index("table.lance", num_partitions=1, sample_rate=2, index_type="IVF_FLAT") - - assert calls == [ - ( - "audio_embedding", - { - "index_type": "IVF_FLAT", - "replace": True, - "num_partitions": 1, - "sample_rate": 2, - "storage_options": None, - }, - ) - ] + with pytest.raises(RuntimeError, match="distributed index failed"): + build_embedding_index(lance_uri, num_partitions=1, sample_rate=2, index_type="IVF_FLAT") def test_build_embedding_index_missing_column(lance_uri_no_embedding): diff --git a/uv.lock b/uv.lock index 0abe688..8541e25 100644 --- a/uv.lock +++ b/uv.lock @@ -162,30 +162,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, ] -[[package]] -name = "coverage" -version = "7.14.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/b4/91/0a7c28934e50d8ac9a7b117712d176f2953c3170bccced5eaacfa3e96175/coverage-7.14.3.tar.gz", hash = "sha256:1a7563a443f3d53fdeb040ec8c9f7466aed7ca3dc5891aa09d3ca3625fa4387f", size = 924398, upload-time = "2026-06-22T23:10:25.584Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/bd/b0/8a911f6ffe6974dac4df95b468ab9a2899d0e59f0f99a489afeec39f00bc/coverage-7.14.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3d74ff26299c4879ce3a4d826f9d3d4d556fd285fde7bbce3c0ef5a8ab1cec24", size = 220672, upload-time = "2026-06-22T23:08:26.621Z" }, - { url = "https://files.pythonhosted.org/packages/36/16/0fc0cb52538783dbbae0934b834f5a58fd5354380ee6cad4a07b15dc845d/coverage-7.14.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:96150a9cf3468ea20f0bc5d0e21b3df8972c31480ef90fa7614b773cc6429665", size = 221035, upload-time = "2026-06-22T23:08:28.372Z" }, - { url = "https://files.pythonhosted.org/packages/77/e2/421ccfbb48335ac49e93301478cf5d623b0c2bf1c0cadd8e2b2fc6c0c710/coverage-7.14.3-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:27d07a46500ba23515b838dbcf52512026af04090755cf6cc64166d88c9b9a1a", size = 252540, upload-time = "2026-06-22T23:08:30.226Z" }, - { url = "https://files.pythonhosted.org/packages/06/c2/05b8c890097c61a7f4406b35396b997a635200ded0339eda83dfbe526c5f/coverage-7.14.3-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:621e13c6108234d7960aaf5762ab5c3c00f33c30c15af06dcbff0c73bf112727", size = 255274, upload-time = "2026-06-22T23:08:31.876Z" }, - { url = "https://files.pythonhosted.org/packages/dc/be/b6d9efe447f8ba3c3c854195f326bd64c54b907d936cd2fdebf8767ec72e/coverage-7.14.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4b60ca6d8af70473491a15a343cbabab2e8f9ea66a4376e81c7aa24876a6f977", size = 256389, upload-time = "2026-06-22T23:08:33.843Z" }, - { url = "https://files.pythonhosted.org/packages/d4/3c/f26e50acc429e608bc534ac06f0a3c169019c798178ec5e9de3dbc0df9c9/coverage-7.14.3-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c90a7cdd5e380e1ce02f19792e2ac2fbfbf177e35a27e69fd3e873b30d895c0c", size = 258648, upload-time = "2026-06-22T23:08:35.481Z" }, - { url = "https://files.pythonhosted.org/packages/9e/a2/01c1fabf816c8e1dae197e258edf878a3d3ddc86fbda34b76e5794277d8f/coverage-7.14.3-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5d788e5fd55347eef06ca0732c77d04a264de67e8ff24631270cdff3767a60cf", size = 252949, upload-time = "2026-06-22T23:08:37.562Z" }, - { url = "https://files.pythonhosted.org/packages/89/c6/941166dd79c31fd44a13063780ae8d552eee0089a0a0930b9bdb7df554ed/coverage-7.14.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:62c7f79db2851c95ef020e5d28b97afde3daf9f7febcd35b53e05638f729063f", size = 254310, upload-time = "2026-06-22T23:08:39.174Z" }, - { url = "https://files.pythonhosted.org/packages/10/31/80b1fd028201a961033ce95be3cd1e39e521b3762e6b4a1ac1616cb291e7/coverage-7.14.3-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:90f7608aeb5d9b60b523b9fb2a4ee1973867cc4865a3f26fe6c7577073b70205", size = 252453, upload-time = "2026-06-22T23:08:40.84Z" }, - { url = "https://files.pythonhosted.org/packages/5f/85/c3d9addd94c4b524f3f4af0232075f5fe7170ce99a1386edff803e5934db/coverage-7.14.3-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:1e3b91f9c4740aeb571ecf82e5e8d8e4ab62d34fcb5a5d4e5baa38c6f7d2857c", size = 256522, upload-time = "2026-06-22T23:08:42.494Z" }, - { url = "https://files.pythonhosted.org/packages/91/14/e5a0575f73795af3a7a9ae13dadf812e17d32422896839987dc3f86947e1/coverage-7.14.3-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:c946099774a7699de03cbd0ff0a64e21aed4525eed9d959adde4afe6d15758ef", size = 252023, upload-time = "2026-06-22T23:08:44.243Z" }, - { url = "https://files.pythonhosted.org/packages/38/9b/9652ee531937ce3b8a63a8896885b2b4a2d56adc30e53c9540c666286d88/coverage-7.14.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:16b206e521feb8b7133a45754643dead0538489cf8b783b90cf5f4e3299625fd", size = 253893, upload-time = "2026-06-22T23:08:46.113Z" }, - { url = "https://files.pythonhosted.org/packages/b1/05/42678841c8c38e4b08bdfc48269f5a16dfbf5806000fe6a89b4cece3c691/coverage-7.14.3-cp312-cp312-win32.whl", hash = "sha256:ea3169c7116eb6cdf7608c6c7da9ecfcb3da40688e3a510fac2d1d2bafd6dc35", size = 222734, upload-time = "2026-06-22T23:08:47.858Z" }, - { url = "https://files.pythonhosted.org/packages/df/87/07a4fcee55177a25f1b52331a8e92cf4f2c53b1a9c75ce2981fd59c684ad/coverage-7.14.3-cp312-cp312-win_amd64.whl", hash = "sha256:7ea52fc08f007bcc494d4bb3df3851e95843d881860ba38fe2c64dc100db5e7d", size = 223266, upload-time = "2026-06-22T23:08:49.494Z" }, - { url = "https://files.pythonhosted.org/packages/aa/34/2b8b66a989282ea7b370beb49f50bab29470dc30bb0b03935b6b802782f7/coverage-7.14.3-cp312-cp312-win_arm64.whl", hash = "sha256:8cec0ad652ec57790970d817490105bd917d783c2f7b38d6b58a0ca312e1a336", size = 222655, upload-time = "2026-06-22T23:08:51.766Z" }, - { url = "https://files.pythonhosted.org/packages/eb/e3/a0aa32bfa3a081951f60a23bc0e7b512891ef0eecda1153cf1d8ba36c6b1/coverage-7.14.3-py3-none-any.whl", hash = "sha256:fb7e18afb6e903c1a92401a2f0501ac277dca527bb9ca6fe1f691a8a0026a0e8", size = 212469, upload-time = "2026-06-22T23:10:23.405Z" }, -] - [[package]] name = "crcmod" version = "1.7" @@ -718,20 +694,18 @@ wheels = [ [[package]] name = "lance-ray" -version = "0.4.2" +version = "0.5.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "lance-namespace" }, { name = "packaging" }, { name = "pyarrow" }, { name = "pylance" }, - { name = "pytest" }, - { name = "pytest-cov" }, { name = "ray", extra = ["data"] }, ] -sdist = { url = "https://files.pythonhosted.org/packages/3f/6d/5d7047d6680bb71e5f1d83342fb58c6b7b376ea8ba30142fe9771d2d657f/lance_ray-0.4.2.tar.gz", hash = "sha256:c1d32ca787b29e1efc41d68d5802ead6229371e680d7135fa52f2f4f2f011d0c", size = 60459, upload-time = "2026-05-12T03:37:32.619Z" } +sdist = { url = "https://files.pythonhosted.org/packages/48/5c/7eaeace1eff30543873bc191e4bf0d1a8ffc5b604168b7ad3e652c3d8e7e/lance_ray-0.5.0.tar.gz", hash = "sha256:9e90530e4fa6560fc69c9179373f1f92d837650d0f9bb2b72aa98456059a0121", size = 89875, upload-time = "2026-07-24T23:30:24.171Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/2c/b8/4b993529f2d5d90a30ab9373254081723b9ad9e2caa9bd359b78a34f7243/lance_ray-0.4.2-py3-none-any.whl", hash = "sha256:aed853718177783a371788fba4a0bfd84621ae577665f1fc9c359fc4a541863e", size = 42869, upload-time = "2026-05-12T03:37:31.502Z" }, + { url = "https://files.pythonhosted.org/packages/95/ba/32d66fbd8bc90372f4185c3d0fe90a9d4cba3b22cde5edb79d1c22b0fa6b/lance_ray-0.5.0-py3-none-any.whl", hash = "sha256:fe7ba364063defd209bbaba0f8abef54b6ddaae2665ab887b281d571cc122575", size = 58921, upload-time = "2026-07-24T23:30:23.229Z" }, ] [[package]] @@ -911,6 +885,11 @@ dependencies = [ { name = "transformers" }, ] +[package.dev-dependencies] +dev = [ + { name = "pytest" }, +] + [package.metadata] requires-dist = [ { name = "daft-lance", specifier = ">=0.4.0" }, @@ -918,7 +897,7 @@ requires-dist = [ { name = "getdaft" }, { name = "httpx" }, { name = "insightface" }, - { name = "lance-ray" }, + { name = "lance-ray", specifier = "==0.5.0" }, { name = "modelscope" }, { name = "numpy" }, { name = "onnxruntime" }, @@ -927,7 +906,7 @@ requires-dist = [ { name = "pillow" }, { name = "psutil" }, { name = "pyarrow" }, - { name = "pylance", specifier = "==8.0.0" }, + { name = "pylance", specifier = "==9.0.0" }, { name = "python-dotenv" }, { name = "ray" }, { name = "soundfile" }, @@ -936,6 +915,9 @@ requires-dist = [ { name = "transformers" }, ] +[package.metadata.requires-dev] +dev = [{ name = "pytest", specifier = ">=9.1.1" }] + [[package]] name = "narwhals" version = "2.22.1" @@ -1473,7 +1455,7 @@ wheels = [ [[package]] name = "pylance" -version = "8.0.0" +version = "9.0.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "lance-namespace" }, @@ -1481,12 +1463,12 @@ dependencies = [ { name = "pyarrow" }, ] wheels = [ - { url = "https://files.pythonhosted.org/packages/3b/99/03db9f4f723353502d2b5f5d8a0eb9a30f0dc41a8cc036ac5fa486471223/pylance-8.0.0-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:769cb38e43a533fdffb87a804043f6a37793eb77d4acbc14cffe9e45dc8bd90b", size = 67235437, upload-time = "2026-07-01T15:09:09.519Z" }, - { url = "https://files.pythonhosted.org/packages/48/af/8f8f8b9c70200e3ab99d8785c26ba691d0ff475cae27f7620618dfc8f3b6/pylance-8.0.0-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:597277a993d159dfcf58230958f1c5f6edebca8693fbd3502129be951ec6927e", size = 70558570, upload-time = "2026-07-01T15:13:50.95Z" }, - { url = "https://files.pythonhosted.org/packages/ca/4c/b10b02a7891753ec1d4a846b8cc001fbafb4bf7c48c2bda5303cf57c1b8a/pylance-8.0.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2c8823134a09ff311b0a963227bdfc160c686910944a373af357cd5b3fa70b5c", size = 74747275, upload-time = "2026-07-01T15:27:09.817Z" }, - { url = "https://files.pythonhosted.org/packages/4c/56/09602b664d8036aa0105ffdb94189c23f9fde4ce71963f4a98f05fda526d/pylance-8.0.0-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:28c98c6ae99edc76622e2af331e99da4d57c9f44c32ebba2872a8e60dee9f7f8", size = 70578246, upload-time = "2026-07-01T15:14:53.143Z" }, - { url = "https://files.pythonhosted.org/packages/4b/1a/9f49850dfee7b1e763f358ff7477a1aa27228d1e40f907020f13ea13b3e1/pylance-8.0.0-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:0b779a7465424de04975cedc774b1eadff8638e0f96f6f19245e9bea332e1e62", size = 74708654, upload-time = "2026-07-01T15:27:54.326Z" }, - { url = "https://files.pythonhosted.org/packages/77/5d/1876b43e680b9afb0d827af032bd72d3f70a9eaf68cb8f5f4286b20a613e/pylance-8.0.0-cp39-abi3-win_amd64.whl", hash = "sha256:ea2e35722f546eb2e056eafb1cc39ced5f353157653516a1472f1127bfa92eec", size = 80511928, upload-time = "2026-07-01T15:28:56.53Z" }, + { url = "https://files.pythonhosted.org/packages/e9/be/45733acd64801991852aac8e658601fd8fc12f76ceb81e57fca690896b90/pylance-9.0.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:8257213501d3298c5b6a344d60938e4bbe4de9f00cd3265371a56d1dc3dd15ca", size = 68377982, upload-time = "2026-07-24T16:53:45.247Z" }, + { url = "https://files.pythonhosted.org/packages/d0/3e/1ef707cb215cc7268c63ad84a91344ad6313d3984b343eeb19a9b708698e/pylance-9.0.0-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:804eedfa1fda2e703cca8580c76f0b44a1b849b725a8908c06f8acfca811732f", size = 71844362, upload-time = "2026-07-24T16:56:07.6Z" }, + { url = "https://files.pythonhosted.org/packages/c8/fb/a499e5c53ddb75c7de44100fd2bb1f7cc735966200d20292eed7af9ef552/pylance-9.0.0-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0a0b75595e3766c1d5f4c90abdc52337b4de60f1a52d123a4f8c4e5bcbbbfa8f", size = 75663088, upload-time = "2026-07-24T17:10:31.283Z" }, + { url = "https://files.pythonhosted.org/packages/e9/80/0714e09f64a68dbdf62558955e737a7436df763b9834a6aba506861b5352/pylance-9.0.0-cp310-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:d2f69c5c390ae3710c35a429905fe15f769951777fafb1b72a359e035ec121f7", size = 71866858, upload-time = "2026-07-24T16:56:35.937Z" }, + { url = "https://files.pythonhosted.org/packages/4b/3c/78d3a6d6ca0d843b7c3ac0c30d9cd2cf4635b0c2cabe6ec66583d5bfc1a1/pylance-9.0.0-cp310-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:836268a7832d62f3d5ccbe1c3fca239621971d90297bcfff14a70b3cb6842aa8", size = 75642656, upload-time = "2026-07-24T17:13:11.879Z" }, + { url = "https://files.pythonhosted.org/packages/cf/c1/dc9c9a31e171530ec0add024d922488c046437d32a7087c29e254a7eacc7/pylance-9.0.0-cp310-abi3-win_amd64.whl", hash = "sha256:96441d27a5ed3805300388ccf8f31835cd280c98751f80b6a1a11dcd6808fc43", size = 81668288, upload-time = "2026-07-24T17:05:38.707Z" }, ] [[package]] @@ -1521,20 +1503,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl", hash = "sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c", size = 386536, upload-time = "2026-06-19T10:58:31.347Z" }, ] -[[package]] -name = "pytest-cov" -version = "7.1.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "coverage" }, - { name = "pluggy" }, - { name = "pytest" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/b1/51/a849f96e117386044471c8ec2bd6cfebacda285da9525c9106aeb28da671/pytest_cov-7.1.0.tar.gz", hash = "sha256:30674f2b5f6351aa09702a9c8c364f6a01c27aae0c1366ae8016160d1efc56b2", size = 55592, upload-time = "2026-03-21T20:11:16.284Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/9d/7a/d968e294073affff457b041c2be9868a40c1c71f4a35fcc1e45e5493067b/pytest_cov-7.1.0-py3-none-any.whl", hash = "sha256:a0461110b7865f9a271aa1b51e516c9a95de9d696734a2f71e3e78f46e1d4678", size = 22876, upload-time = "2026-03-21T20:11:14.438Z" }, -] - [[package]] name = "python-dateutil" version = "2.9.0.post0" From 0bd15a40cf33007f362131a5e37a0aaf70d87181 Mon Sep 17 00:00:00 2001 From: fanng Date: Mon, 14 Sep 2026 15:32:33 +0900 Subject: [PATCH 3/3] feat: compact tables after deletion --- AGENTS.md | 2 +- README.md | 11 ++--- docs/daft-tuning.md | 5 +-- multimodal_toolkit/workflow/manage.py | 18 ++++++--- tests/workflow/test_index_manage.py | 58 ++++++++++++++++++++++++--- 5 files changed, 75 insertions(+), 19 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index b49ad71..27dd290 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -28,7 +28,7 @@ tests/ # pytest；按 audio/image/storage/workflow 分目录 ## 硬性架构约定 1. **媒体隔离**：audio 和 image 各自拥有独立的 package 和 workflow 入口，**不要**把共享 Stage 参数化成"通用管线"。可共享的只有 `storage/` 层 infra 和 `config.py` 的辅助函数。 -2. **数据 API 优先级**：所有数据操作优先用 **Daft**；Daft 做不到再用 **lance-ray**；**pylance 直接调用是最后手段**。当前固定为 lance-ray 0.5.0 与 pylance 9.0.0；blob compaction 仍需在单独改动中完成对象存储回归验证后再启用。 +2. **数据 API 优先级**：所有数据操作优先用 **Daft**；Daft 做不到再用 **lance-ray**；**pylance 直接调用是最后手段**。当前固定为 lance-ray 0.5.0 与 pylance 9.0.0；Blob v2 表删除后由 lance-ray 执行分布式 compaction，涉及该路径的改动必须覆盖本地和对象存储回归验证。 3. **行不丢弃**：analyze 阶段 manifest 里每个条目对应输出一行，失败用 `status` 列标记（`download_failed` / `decode_failed` / ...），分数结论置 null。合规场景必须能区分"内容有问题"和"根本没处理"。 4. **配置只走环境变量**，经 `config.py` 的 `env_int`/`env_bool`/`env_choice` 等辅助函数读取。注意：Daft 原生只读少数 `DAFT_*` 环境变量，新增 Daft 执行参数必须在 `storage/io.py::configure_daft_runner()` 里显式传给 `daft.set_execution_config()`，否则是死配置；S3 参数同理要传进 `daft_io_config()` 的 `S3Config`。 5. **写出受控**：`write_lance` 必须传 `LANCE_MAX_ROWS_PER_FILE` / `LANCE_MAX_BYTES_PER_FILE`；JSONL 写出前用 `coalesce_for_write()` 收敛分区，避免小文件。 diff --git a/README.md b/README.md index 3917169..2c45308 100644 --- a/README.md +++ b/README.md @@ -66,7 +66,7 @@ Manifest (parquet / jsonl / csv) │ └──▶ Stage 5 — workflow/manage.py pylance ds.delete() : --before / --after - compaction : 当前暂不执行 + lance_ray.compact_files : delete 之后自动执行 ``` ### 引擎分工 @@ -74,7 +74,7 @@ Manifest (parquet / jsonl / csv) | 引擎 | 用于 | 原因 | |----------------|-----------------------------------------------------------------------------|-------------------------------------------------------------| | **Daft** | manifest 读取、S3 下载、ASR/LLM 流水线、Lance 写入（Stage 1 & 2）、标量与 ANN 查询 | 主引擎；API 稳定 | -| **lance_ray** | IVF_PQ 向量索引创建、`compact_files` | 分布式索引与表维护能力；本项目暂未启用 compaction | +| **lance_ray** | IVF_PQ 向量索引创建、`compact_files` | 分布式索引与删除后的表维护 | | **pylance** | ZONEMAP 标量索引、行删除、`cleanup_old_versions` | ZONEMAP：lance_ray 依赖未发布代码；delete：只有这一个 API | | **daft_lance** | `read_lance`、`write_lance` 等 Daft 数据操作 | 不用于建索引；Daft 优先原则只适用于数据处理 | @@ -459,6 +459,7 @@ S3 上的 Lance 表读写由底层库覆盖，但在本 POC 中应作为一个 Stage 3 固定使用 lance-ray 0.5.0 的分布式提交链路，不再回退到 pylance 单机建索引。这样调用方 能明确获知 Ray、对象存储凭据或索引提交失败，避免大表在驱动进程静默退化为单机执行。 -**blob v2 资产表的 compaction 仍暂时禁用。** -本次依赖升级不改变 Stage 5 的表管理行为；删除行之后仍跳过 compaction。后续启用时需单独覆盖 -lance-ray、本地 Lance URI 和 MinIO/S3 的 Blob v2 回归验证，避免把版本对齐和行为变更混在一起。 +**删除后会执行 Blob v2 compaction。** +Stage 5 先提交删除，再使用 lance-ray 0.5.0 的默认 compaction 选项压实文件和删除向量，最后清理旧版本。 +0.4.x 中为规避 `compaction_options=None` 缺陷而传入空字典的 workaround 已移除。对象存储部署仍应在 +发布前验证 worker 凭据、共享 Lance URI 和 Blob v2 读取。 diff --git a/docs/daft-tuning.md b/docs/daft-tuning.md index 4d37129..467568b 100644 --- a/docs/daft-tuning.md +++ b/docs/daft-tuning.md @@ -112,9 +112,8 @@ CPU/网络 在下载下一批图片 | `LANCE_MAX_BYTES_PER_FILE` | `512MB` | 单个 Lance 数据文件最大字节数。 | | `DAFT_JSON_TARGET_FILESIZE` | `128MB` | JSONL 单文件目标大小。 | -为什么要管：碎文件会拖慢后续所有读取。虽然 pylance 已升级到 9.0.0，Stage 5 仍要等 -lance-ray 和对象存储路径完成独立回归验证后才会开启 blob compaction；在此之前仍应从写入侧 -控制文件布局。默认值一般不用动。 +为什么要管：碎文件会拖慢后续所有读取。Stage 5 会在删除后压实 Blob v2 表，但写入侧控制仍能 +降低 compaction 的资源消耗和文件膨胀；对象存储环境还应验证 worker 的共享 URI 与凭据。默认值一般不用动。 ### 3.7 观测（长任务建议必开） diff --git a/multimodal_toolkit/workflow/manage.py b/multimodal_toolkit/workflow/manage.py index 2613691..257844d 100644 --- a/multimodal_toolkit/workflow/manage.py +++ b/multimodal_toolkit/workflow/manage.py @@ -2,7 +2,7 @@ 表管理仍遵循项目的数据 API 优先级： delete → pylance ds.delete() （Daft/lance-ray 没有等价 API） - compact → 暂不执行；升级 pylance 后仍需单独验证 lance-ray 和对象存储路径 + compact → lance_ray.compact_files() （删除后回收小文件与删除向量） cleanup → pylance ds.cleanup_old_versions() （没有替代 API） --before DATE 删除 ingest_time < DATE 的行 @@ -16,6 +16,7 @@ import argparse import lance +import lance_ray from ..storage.io import lance_storage_options @@ -42,10 +43,17 @@ def delete_by_date( print(f"[ok] deleted rows where: {filter_str}") - # pylance 9.0.0 已修复 Blob v2 compaction 的解码问题，但版本升级本身不等于 - # 分布式 compaction 链路已经验证。这里继续跳过，待后续改动同时覆盖 lance-ray、 - # 本地 Lance URI 和 MinIO/S3 后再开启，避免依赖升级悄悄扩大表管理行为。 - print("[warn] compaction skipped; enable after validating lance-ray and object storage") + # lance-ray 0.5.0 在未传 compaction_options 时会构造默认选项；不再需要 0.4.x + # 为绕过 None 传参缺陷而显式传入空字典。压实必须在删除提交之后执行，才能把删除 + # 向量和小文件合并成新的 fragment。 + metrics = lance_ray.compact_files( + lance_uri, + storage_options=lance_storage_options(lance_uri), + ) + if metrics is None: + print(f"[ok] no compaction needed: {lance_uri}") + else: + print(f"[ok] compacted: {lance_uri}") ds.cleanup_old_versions() diff --git a/tests/workflow/test_index_manage.py b/tests/workflow/test_index_manage.py index 59acd71..bbd5720 100644 --- a/tests/workflow/test_index_manage.py +++ b/tests/workflow/test_index_manage.py @@ -1,8 +1,8 @@ """在本地 Lance 表上验证 workflow/index.py 和 workflow/manage.py。 build_embedding_index 的真实 Ray 路径不属于默认测试；需要时用 `pytest -m ray` -显式执行。pylance 已升级到 9.0.0，但 delete_by_date 仍有意不执行 compaction， -因此删除测试继续留在默认测试集。 +显式执行。delete_by_date 会执行分布式 compaction，因此删除测试使用共享的 local_ray +fixture，并同时覆盖普通表和 Blob v2 表。 """ from __future__ import annotations @@ -49,6 +49,37 @@ def lance_uri() -> str: return uri +@pytest.fixture() +def lance_uri_blob() -> str: + """按资产表的写入方式创建多 fragment 的 Blob v2 表。 + + 每批 100 行分开 append，确保删除后 compaction 至少有多个 fragment 可以合并； + 这能覆盖 pylance 9 与 lance-ray 0.5 的真实 Blob v2 表维护链路。 + """ + import daft + from daft import col + + tmp = tempfile.mkdtemp() + uri = str(pathlib.Path(tmp) / "table_blob.lance") + for batch in range(3): + times = [datetime(2024, 1, batch + 1, tzinfo=timezone.utc)] * 100 + df = daft.from_pydict( + { + "doc_id": [f"blob_{batch}_{row:04d}" for row in range(100)], + "ingest_time": times, + "blob": [b"x" * 100] * 100, + } + ).with_column( + "ingest_time", col("ingest_time").cast(daft.DataType.timestamp("us", "UTC")) + ) + df.write_lance( + uri, + mode="create" if batch == 0 else "append", + blob_columns=["blob"], + ) + return uri + + @pytest.fixture() def lance_uri_no_embedding() -> str: tmp = tempfile.mkdtemp() @@ -120,18 +151,35 @@ def test_delete_requires_a_bound(lance_uri): delete_by_date(lance_uri) -def test_delete_before(lance_uri): +def test_delete_before(lance_uri, local_ray): delete_by_date(lance_uri, before="2024-03-01") assert lance.dataset(lance_uri).count_rows() == 200 # 2024-01-01 rows gone -def test_delete_after(lance_uri): +def test_delete_after(lance_uri, local_ray): delete_by_date(lance_uri, after="2024-09-01") assert lance.dataset(lance_uri).count_rows() == 200 # 2024-12-01 rows gone -def test_delete_window(lance_uri): +def test_delete_window(lance_uri, local_ray): # Outside 2024-03-01 .. 2024-09-01 survives: keeps Jan and Dec rows. delete_by_date(lance_uri, before="2024-09-01", after="2024-03-01") remaining = lance.dataset(lance_uri).count_rows() assert remaining == 200 + + +def test_delete_and_compact_blob_v2_table(lance_uri_blob, local_ray): + """删除后必须能压实 Blob v2 表，并保持剩余 blob 可读。""" + from multimodal_toolkit.storage.blob import validate_blob_v2 + + # Daft 的执行配置会影响每次 append 实际写出的 fragment 数；这里只约束 + # compaction 的核心语义：压实前确实有多个 fragment，压实后数量必须减少。 + fragments_before = len(lance.dataset(lance_uri_blob).get_fragments()) + assert fragments_before > 1 + delete_by_date(lance_uri_blob, before="2024-01-02") + + dataset = lance.dataset(lance_uri_blob) + assert dataset.count_rows() == 200 + assert len(dataset.get_fragments()) < fragments_before + assert len(dataset.take_blobs("blob", indices=[0])[0].read()) == 100 + validate_blob_v2(lance_uri_blob, "blob")