From 506a80e8c2e65ac2c6deb90e06c73f555a1e30c7 Mon Sep 17 00:00:00 2001 From: lin <1609924163@qq.com> Date: Sun, 19 Jul 2026 13:11:40 +0800 Subject: [PATCH 1/6] Clarify request model usage and harden native drag handling --- README.md | 6 +- docs/features.md | 2 +- docs/manual-verification.md | 2 +- docs/no-prompt-revision/README.md | 6 +- docs/privacy-policy.md | 2 +- docs/troubleshooting.md | 20 +-- docs/usage.md | 7 +- gio-client/internal/ui/layout_controls.go | 30 ++-- gio-client/internal/ui/upstream_models.go | 6 +- .../internal/ui/upstream_models_test.go | 6 +- go-cli/pkg/client/payload.go | 1 - go-cli/pkg/client/payload_test.go | 4 +- image-studio/backend/dialogs.go | 12 ++ image-studio/backend/mask_composite.go | 153 ++++++++++++++++++ image-studio/backend/mask_composite_test.go | 118 ++++++++++++++ image-studio/backend/native_drag_windows.go | 26 ++- image-studio/backend/service.go | 34 +++- image-studio/backend/types.go | 1 + .../src/components/canvas/CanvasStage.tsx | 2 +- .../src/components/panel/FAQModal.tsx | 20 +-- .../components/panel/ResultDetailDrawer.tsx | 2 +- .../panel/UpstreamProfileEditor.tsx | 51 +++--- .../frontend/src/lib/maskComposite.ts | 110 +++++++++++++ .../upstream/AndroidUpstreamProfileForm.tsx | 43 ++--- .../frontend/src/platform/runtime/host.ts | 38 ++++- .../frontend/src/state/studioStore.shared.ts | 38 ++++- .../frontend/src/state/studioStore.ts | 33 ++-- image-studio/frontend/src/styles/_canvas.css | 8 + .../frontend/test/requestModel.test.mjs | 4 +- shared/kernel/requestModel.js | 1 - 30 files changed, 652 insertions(+), 134 deletions(-) create mode 100644 image-studio/backend/mask_composite.go create mode 100644 image-studio/backend/mask_composite_test.go create mode 100644 image-studio/frontend/src/lib/maskComposite.ts diff --git a/README.md b/README.md index 3708d89d0..fa652cdc3 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ Image Studio 面向 OpenAI 兼容图像上游，重点解决长时间图像推理在 Cloudflare / Nginx 后面容易遇到的 524/504 断连问题。Responses API 模式支持 `HTTP SSE` 与 `WebSocket mode` 两种传输；Images API 模式则兼容标准 `/v1/images/generations` 与 `/v1/images/edits`。 -项目不内置任何默认上游。首次启动需要你自己填写 BASE_URL、API Key、文本模型与图像模型。 +项目不内置任何默认上游。首次启动需要你自己填写 BASE_URL、API Key 与请求模型 ID。 当前没有独立部署的在线 Web 版。仓库里的浏览器预览主要用于前端调试和 target platform 预览，不等同于可直接对外提供服务的 SaaS Web 端。 @@ -28,7 +28,7 @@ Image Studio 面向 OpenAI 兼容图像上游，重点解决长时间图像推 - 抢先体验当前分支的最新改动:到 [DR-lin-eng/Image-Studio Actions · release.yml](https://github.com/DR-lin-eng/Image-Studio/actions/workflows/release.yml) 下载最近一次成功构建的 artifact。 Windows 上这类 CI `exe` 如果没有签名，可能会被 Win11 Smart App Control / SmartScreen 拦截，因此只建议用于内部测试。 - 各平台安装包区别、命名规则和选择建议见 [docs/packages.md](./docs/packages.md)。 -2. 首次启动后打开「上游配置」，填写 API 形态、BASE_URL、API Key、文本模型 ID、图像模型 ID。 +2. 首次启动后打开「上游配置」，填写 API 形态、BASE_URL、API Key、请求模型 ID。 3. 根据上游能力选择 API 形态 - Responses API:更适合长推理、抗 524/504。 - Images API:更适合只提供标准图像接口的兼容上游。 @@ -43,7 +43,7 @@ Image Studio 面向 OpenAI 兼容图像上游，重点解决长时间图像推 提 Issue 前建议先做这几步: -1. 在当前 profile 里点一次「测试连接」，确认 `BASE_URL`、`API Key`、文本模型 ID、图像模型 ID 真实可用。 +1. 在当前 profile 里点一次「测试连接」，确认 `BASE_URL`、`API Key`、请求模型 ID 真实可用。 2. 对照 [docs/troubleshooting.md](./docs/troubleshooting.md) 自查 `524/504`、`401/403`、`model not found`、多参考图/蒙版不生效、Android 保存目录行为等常见非软件问题。 3. 从历史详情或 raw 响应里确认真实 HTTP 状态码和上游报错，不要只看页面 toast。 4. 如果同样的 `BASE_URL + Key + 模型 ID` 在 curl、Postman 或上游自带调试页里也失败，优先联系你的上游服务商，而不是提交本仓库 Issue。 diff --git a/docs/features.md b/docs/features.md index abeecbbe8..8d9c3b36f 100644 --- a/docs/features.md +++ b/docs/features.md @@ -48,7 +48,7 @@ ## 设置 -- 上游配置:API 形态、BASE_URL、API Key、文本模型 ID、图像模型 ID、连接测试。 +- 上游配置:API 形态、BASE_URL、API Key、请求模型 ID、连接测试。Responses API 和 Images API 都只填写一个请求模型 ID。 - API Key: - 桌面端使用系统安全存储(Keychain / Credential Manager / Secret Service)。 - Android 壳层使用应用私有 SharedPreferences。 diff --git a/docs/manual-verification.md b/docs/manual-verification.md index f2f2d4bfa..e33cdcaf7 100644 --- a/docs/manual-verification.md +++ b/docs/manual-verification.md @@ -75,7 +75,7 @@ node scripts/init-manual-verification.mjs custom "my regression check" ### 前置条件 - 至少一条真实可用上游，且能稳定返回图像 -- 已知正常的 `BASE_URL` / `API Key` / 文本模型 ID / 图像模型 ID +- 已知正常的 `BASE_URL` / `API Key` / 请求模型 ID - 如果是 Android: - 已接入真机或模拟器 - 可先执行： diff --git a/docs/no-prompt-revision/README.md b/docs/no-prompt-revision/README.md index afe9eb728..04a35ff02 100644 --- a/docs/no-prompt-revision/README.md +++ b/docs/no-prompt-revision/README.md @@ -1,14 +1,14 @@ # 原始提示词传递说明 -Image Studio 现在默认要求 Responses API 文本模型不要改写用户输入的 prompt。 +Image Studio 现在默认要求 Responses API 请求模型不要改写用户输入的 prompt。 -它用于 Responses API 模式。Image Studio 会在 `/v1/responses` 请求顶层加入一段 `instructions`，要求文本模型把用户 prompt 原样传给 `image_generation`，不要重写、扩写、润色或调整措辞。界面不再提供关闭或开启该行为的按钮；所有 Responses 生成请求都按这个策略发送。 +它用于 Responses API 模式。Image Studio 会在 `/v1/responses` 请求顶层加入一段 `instructions`，要求请求模型把用户 prompt 原样传给 `image_generation`，不要重写、扩写、润色或调整措辞。界面不再提供关闭或开启该行为的按钮；所有 Responses 生成请求都按这个策略发送。 ## 适合什么场景 - 你已经精修过 prompt，希望图像模型尽量逐字执行。 - prompt 里有固定格式、专有名词、镜头参数、构图要求或中英文混排内容。 -- 你想减少 Responses API 文本模型二次发挥导致的风格漂移。 +- 你想减少 Responses API 请求模型二次发挥导致的风格漂移。 ## 功能边界 diff --git a/docs/privacy-policy.md b/docs/privacy-policy.md index 8fccb04b0..0018fa1f4 100644 --- a/docs/privacy-policy.md +++ b/docs/privacy-policy.md @@ -14,7 +14,7 @@ Image Studio 在本地可能处理以下数据： -- 上游配置与偏好设置：例如 `BASE_URL`、API 形态、文本/图像模型 ID、请求策略、并发限制、代理模式与代理地址、主题、字号、输出格式、预设、自定义比例、输出目录、受信任输出目录、是否保留日志、忽略的版本号、完成提示音配置等。 +- 上游配置与偏好设置：例如 `BASE_URL`、API 形态、请求模型 ID、请求策略、并发限制、代理模式与代理地址、主题、字号、输出格式、预设、自定义比例、输出目录、受信任输出目录、是否保留日志、忽略的版本号、完成提示音配置等。 - API Key： - 桌面版默认使用系统安全存储，不写入普通 JSON 配置。 - Android 版保存在应用私有 `SharedPreferences` 中。 diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index d48a9297d..621d17f24 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -9,9 +9,9 @@ 先做这组最短排查: -1. 在当前 profile 里点一次「测试连接」，确认 `BASE_URL`、`API Key`、文本模型 ID、图像模型 ID 至少能通过基础校验。 +1. 在当前 profile 里点一次「测试连接」，确认 `BASE_URL`、`API Key`、请求模型 ID 至少能通过基础校验。 2. 确认自己选对了 API 形态: - - `Responses API` 需要文本模型可用，并且上游真正实现了 `/v1/responses` 与 SSE。 + - `Responses API` 需要所填请求模型可用，并且上游真正实现了 `/v1/responses` 与 SSE。 - `Images API` 需要上游真正实现 `/v1/images/generations` 或 `/v1/images/edits`。 3. 打开历史详情或 raw 响应，先看真实 `HTTP status` 和上游错误 message，不要只看页面上的 toast。 4. 如果同样的 `BASE_URL + Key + 模型 ID` 在 curl、Postman 或上游自带调试页里也失败，优先联系上游服务商。 @@ -20,13 +20,13 @@ ### 这些情况通常不是软件 bug - `401 / 403 / model not found`: - 常见原因是 Key 没权限、模型 ID 填错、账号没开通对应模型、IP 白名单未放行，或上游把图像模型和文本模型分了不同权限组。 + 常见原因是 Key 没权限、请求模型 ID 填错、账号没开通对应模型或 IP 白名单未放行。 - `524 / 504 / 5xx`: 常见原因是 Cloudflare / Nginx / relay 网关超时、上游排队过久、服务商限流、上游临时故障，不一定是本地程序崩溃。 - 多参考图、蒙版、`seed`、`negative_prompt` 不生效: 常见原因是 relay 只“接受字段”但没有真正透传，或目标模型本身就不支持这些能力。 - `Responses API` 一直失败: - 常见原因是上游根本没实现 `/v1/responses`、会缓冲 SSE，或者 Key 只有 image-only 权限。 + 常见原因是上游根本没实现 `/v1/responses`、会缓冲 SSE，或者 Key 没有所填请求模型的权限。 - `Responses WebSocket mode` 握手失败: 如果 raw 日志或上游报错里出现 `WebSocket upgrade required (Upgrade: websocket)`，说明当前链路把 WebSocket 请求降成了普通 HTTP，请求没有真正完成 Upgrade。常见原因是中转站 / 反向代理 / 网关没有正确放行 `Upgrade: websocket`，这种情况应直接切回 `HTTP SSE` 或修上游代理配置。 - Android 看不到“目录”或保存位置和桌面端不一样: @@ -54,7 +54,7 @@ - 应用版本: release 版本号或 Actions artifact 对应 commit / 构建日期 - API 形态: `Responses API` 或 `Images API` - `BASE_URL` 类型: 官方 / OpenAI 兼容中转 / 自建网关 -- 文本模型 ID 与图像模型 ID +- 请求模型 ID - 最短复现步骤 - raw 响应中的 `HTTP status`、错误 message 或截图 @@ -96,7 +96,7 @@ log/ 处理顺序: 1. 如果当前是 Images API，优先切到 Responses API。 -2. 确认 key 有文本模型权限，例如默认 `gpt-5.5`。 +2. 确认 key 有所填请求模型的权限。 3. 降低质量或尺寸，缩短单次推理时间。 4. 从历史项查看 raw 响应，确认是 Cloudflare 524/504、上游 JSON 5xx，还是模型权限错误。 5. 如果上游本身不支持 SSE 或会缓冲 SSE，换上游或走 Images API。 @@ -145,13 +145,13 @@ websocket handshake failed: HTTP 400: WebSocket upgrade required (Upgrade: webso Responses API: -- key 没有文本模型权限。 -- 文本模型 ID 或图像模型 ID 在该上游不可用。 -- key 绑到了 image-only 分组，但 Responses API 需要文本模型来调用 `image_generation` 工具。 +- key 没有请求模型权限。 +- 请求模型 ID 在该上游不可用。 +- 上游虽然列出了该模型，但没有允许它通过 Responses API 调用。 Images API: -- 图像模型 ID 不存在。 +- 请求模型 ID 不存在。 - key 没有 image endpoint 权限。 - 上游只实现了 `/v1/chat/completions`，没有实现 `/v1/images/generations` 或 `/v1/images/edits`。 diff --git a/docs/usage.md b/docs/usage.md index 1d2b18f63..52316a4e0 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -9,9 +9,8 @@ 1. API 形态:Responses API 或 Images API。 2. BASE_URL:你自己的 OpenAI 兼容中转站地址。 3. API Key。 -4. 文本模型 ID:Responses API 与 prompt 优化会用到，默认 `gpt-5.5`。 -5. 图像模型 ID:两种 API 形态都会用到，默认 `gpt-image-2`。 -6. 测试连接:保存前建议先点一次，确认当前 profile 能真正走通。 +4. 请求模型 ID:Responses API 直接把它写入请求的 `model` 字段，支持上游允许的文本或图像模型；Images API 也把它作为请求的 `model` 字段。 +5. 测试连接:保存前建议先点一次，确认当前 profile 能真正走通。 本应用不内置任何默认上游，也不会向除你配置的 BASE_URL 以外的生成服务发请求。 @@ -25,7 +24,7 @@ - 图像推理容易超过 100 秒。 - 上游在 Cloudflare / Nginx 后面，常见 524/504。 -- 你的 key 有文本模型权限，例如默认的 `gpt-5.5`。 +- 你的 key 有所填请求模型的权限，例如默认的 `gpt-5.5`，或上游支持的图像模型。 特点: diff --git a/gio-client/internal/ui/layout_controls.go b/gio-client/internal/ui/layout_controls.go index b1166d052..017010474 100644 --- a/gio-client/internal/ui/layout_controls.go +++ b/gio-client/internal/ui/layout_controls.go @@ -1043,10 +1043,10 @@ func (a *App) layoutSettingsHelpModal(gtx layout.Context) layout.Dimensions { return a.helpInfoCard(gtx, "模型 ID 怎么填?", "", func(gtx layout.Context) layout.Dimensions { return layout.Flex{Axis: layout.Vertical, Gap: gtx.Dp(unit.Dp(8))}.Layout(gtx, layout.Rigid(func(gtx layout.Context) layout.Dimensions { - return a.label(gtx, "Responses API 会同时用到文本模型 ID 与图像模型 ID；Images API 只读取图像模型 ID。", unit.Sp(11), fluent.textMuted, font.Normal) + return a.label(gtx, "Responses API 与 Images API 都只填写一个请求模型 ID；Responses API 可使用上游支持的文本或图像模型。", unit.Sp(11), fluent.textMuted, font.Normal) }), layout.Rigid(func(gtx layout.Context) layout.Dimensions { - return a.label(gtx, "留空时默认文本模型是 gpt-5.5，默认图像模型是 gpt-image-2。", unit.Sp(11), fluent.textMuted, font.Normal) + return a.label(gtx, "Responses API 留空默认 gpt-5.5；Images API 留空默认 gpt-image-2。", unit.Sp(11), fluent.textMuted, font.Normal) }), ) }) @@ -2866,7 +2866,7 @@ func (a *App) layoutSettingsEditorPane(gtx layout.Context, snap snapshot) layout }, a.api, a.apiButtons, 2, func(value string) { a.api = value }) }), layout.Rigid(func(gtx layout.Context) layout.Dimensions { - hint := "需要 key 绑定到拥有 gpt-5.5 模型的分组。SSE 保活可防 Cloudflare 524。" + hint := "需要 key 拥有所填请求模型的权限。SSE 保活可防 Cloudflare 524。" if a.api == string(client.APIModeImages) { hint = "使用标准 Images API，key 走 image-2 / image API 分组，兼容性最广。" } @@ -2976,11 +2976,11 @@ func (a *App) layoutSettingsEditorPane(gtx layout.Context, snap snapshot) layout })) if a.api == string(client.APIModeResponses) { rows = append(rows, layout.Rigid(func(gtx layout.Context) layout.Dimensions { - return a.technicalField(gtx, "文本模型 ID", &a.textModelInput, client.TextModel, unit.Dp(40)) + return a.technicalField(gtx, "请求模型 ID", &a.textModelInput, client.TextModel, unit.Dp(40)) })) if len(probeTextModels) > 0 { rows = append(rows, layout.Rigid(func(gtx layout.Context) layout.Dimensions { - return a.layoutProbeModelSuggestions(gtx, "推荐文本模型", "text", probeTextModels, strings.TrimSpace(a.textModelInput.Text()), func(id string) { + return a.layoutProbeModelSuggestions(gtx, "推荐请求模型", "text", probeTextModels, strings.TrimSpace(a.textModelInput.Text()), func(id string) { a.textModelInput.SetText(strings.TrimSpace(id)) }) })) @@ -2994,17 +2994,17 @@ func (a *App) layoutSettingsEditorPane(gtx layout.Context, snap snapshot) layout return a.label(gtx, "仅 Responses API 生效，会写入 reasoning.effort。推理越高通常越稳，但延迟也会更长。", unit.Sp(10), fluent.textDim, font.Normal) })) } - rows = append(rows, - layout.Rigid(func(gtx layout.Context) layout.Dimensions { - return a.technicalField(gtx, "图像模型 ID", &a.imageModelInput, client.ImageModel, unit.Dp(40)) - }), - layout.Rigid(func(gtx layout.Context) layout.Dimensions { - return a.technicalField(gtx, "并发数量限制", &a.concurrencyLimitInput, "留空 = 不限制", unit.Dp(40)) - }), - ) - if len(probeImageModels) > 0 { + if a.api == string(client.APIModeImages) { + rows = append(rows, layout.Rigid(func(gtx layout.Context) layout.Dimensions { + return a.technicalField(gtx, "请求模型 ID", &a.imageModelInput, client.ImageModel, unit.Dp(40)) + })) + } + rows = append(rows, layout.Rigid(func(gtx layout.Context) layout.Dimensions { + return a.technicalField(gtx, "并发数量限制", &a.concurrencyLimitInput, "留空 = 不限制", unit.Dp(40)) + })) + if a.api == string(client.APIModeImages) && len(probeImageModels) > 0 { rows = append(rows, layout.Rigid(func(gtx layout.Context) layout.Dimensions { - return a.layoutProbeModelSuggestions(gtx, "推荐图像模型", "image", probeImageModels, strings.TrimSpace(a.imageModelInput.Text()), func(id string) { + return a.layoutProbeModelSuggestions(gtx, "推荐请求模型", "image", probeImageModels, strings.TrimSpace(a.imageModelInput.Text()), func(id string) { a.imageModelInput.SetText(strings.TrimSpace(id)) }) })) diff --git a/gio-client/internal/ui/upstream_models.go b/gio-client/internal/ui/upstream_models.go index 256965aee..d6b690e39 100644 --- a/gio-client/internal/ui/upstream_models.go +++ b/gio-client/internal/ui/upstream_models.go @@ -61,9 +61,9 @@ func preferredProbeModels(apiMode string, models []kernel.UpstreamModelDescripto } return nil, image } - if len(text) == 0 { - text = all - } + // Responses accepts any supported upstream model as the request model; + // keep the suggestions unclassified so image-capable models are not hidden. + text = all if len(image) == 0 { image = all } diff --git a/gio-client/internal/ui/upstream_models_test.go b/gio-client/internal/ui/upstream_models_test.go index 36c2c043d..7c2313fbb 100644 --- a/gio-client/internal/ui/upstream_models_test.go +++ b/gio-client/internal/ui/upstream_models_test.go @@ -6,14 +6,14 @@ import ( "image-studio/gio-client/internal/kernel" ) -func TestPreferredProbeModelsSplitsTextAndImage(t *testing.T) { +func TestPreferredProbeModelsUsesAllModelsForResponses(t *testing.T) { text, image := preferredProbeModels("responses", []kernel.UpstreamModelDescriptor{ {ID: "gpt-5.5"}, {ID: "gpt-image-2"}, {ID: "gpt-5.5"}, }) - if len(text) != 1 || text[0].ID != "gpt-5.5" { - t.Fatalf("text=%v want [gpt-5.5]", text) + if len(text) != 2 || text[0].ID != "gpt-5.5" || text[1].ID != "gpt-image-2" { + t.Fatalf("text=%v want all request models", text) } if len(image) != 1 || image[0].ID != "gpt-image-2" { t.Fatalf("image=%v want [gpt-image-2]", image) diff --git a/go-cli/pkg/client/payload.go b/go-cli/pkg/client/payload.go index 1704e189b..213414b98 100644 --- a/go-cli/pkg/client/payload.go +++ b/go-cli/pkg/client/payload.go @@ -65,7 +65,6 @@ func BuildPayload(opts Options) ([]byte, error) { } tool := map[string]any{ "type": "image_generation", - "model": imgModel, "action": action, "size": size, "quality": quality, diff --git a/go-cli/pkg/client/payload_test.go b/go-cli/pkg/client/payload_test.go index 3e6433c21..0dec3b244 100644 --- a/go-cli/pkg/client/payload_test.go +++ b/go-cli/pkg/client/payload_test.go @@ -44,8 +44,8 @@ func TestBuildPayloadUsesSizeAndQuality(t *testing.T) { if tool["moderation"] != "low" { t.Errorf("moderation = %v, want low", tool["moderation"]) } - if tool["model"] != "gpt-image-2" { - t.Errorf("model = %v, want gpt-image-2", tool["model"]) + if _, ok := tool["model"]; ok { + t.Errorf("Responses image_generation tool unexpectedly contains a second model: %v", tool["model"]) } if tool["action"] != "generate" { t.Errorf("action = %v, want generate", tool["action"]) diff --git a/image-studio/backend/dialogs.go b/image-studio/backend/dialogs.go index 4b7e35f6e..e2877b899 100644 --- a/image-studio/backend/dialogs.go +++ b/image-studio/backend/dialogs.go @@ -25,6 +25,18 @@ func (s *Service) BeginNativeFileDrag(path string) error { if err != nil { return err } + s.nativeDragMu.Lock() + if s.nativeDragActive { + s.nativeDragMu.Unlock() + return errors.New("已有文件正在拖动，请先松开鼠标或按 Esc 取消") + } + s.nativeDragActive = true + s.nativeDragMu.Unlock() + defer func() { + s.nativeDragMu.Lock() + s.nativeDragActive = false + s.nativeDragMu.Unlock() + }() if s.ctx != nil { runtime.EventsEmit(s.ctx, "native-file-drag", allowed) } diff --git a/image-studio/backend/mask_composite.go b/image-studio/backend/mask_composite.go new file mode 100644 index 000000000..f5817d4bb --- /dev/null +++ b/image-studio/backend/mask_composite.go @@ -0,0 +1,153 @@ +package backend + +import ( + "bytes" + "encoding/base64" + "errors" + "fmt" + "image" + "image/color" + "image/draw" + "image/png" + "strings" + + xdraw "golang.org/x/image/draw" +) + +// normalizeEditMaskB64 converts imported black/white masks and painted masks +// to OpenAI's alpha convention: transparent pixels are editable and opaque +// pixels protect the source image. The returned PNG always matches the first +// edit source dimensions. +func normalizeEditMaskB64(maskB64, sourcePath string) (string, error) { + maskBytes, err := base64.StdEncoding.DecodeString(strings.TrimSpace(maskB64)) + if err != nil { + return "", fmt.Errorf("蒙版图片 base64 无效: %w", err) + } + if len(maskBytes) == 0 { + return "", errors.New("蒙版图片为空") + } + mask, _, err := image.Decode(bytes.NewReader(maskBytes)) + if err != nil { + return "", fmt.Errorf("解析蒙版图片失败: %w", err) + } + source, err := loadImage(sourcePath) + if err != nil { + return "", fmt.Errorf("读取蒙版源图失败: %w", err) + } + + dims := source.Bounds().Size() + if dims.X <= 0 || dims.Y <= 0 { + return "", errors.New("蒙版源图尺寸无效") + } + scaled := image.NewNRGBA(image.Rect(0, 0, dims.X, dims.Y)) + copyOrScaleMask(scaled, mask) + + hasTransparency := maskHasTransparency(scaled) + + normalized := image.NewNRGBA(scaled.Bounds()) + for y := 0; y < dims.Y; y++ { + for x := 0; x < dims.X; x++ { + pixel := scaled.NRGBAAt(x, y) + alpha := pixel.A + if !hasTransparency { + // Opaque black/white masks are accepted as a convenience. Match the + // canvas UI: white is the painted/editable selection, black is kept. + luma := (uint32(pixel.R)*299 + uint32(pixel.G)*587 + uint32(pixel.B)*114 + 500) / 1000 + alpha = uint8(0xff - luma) + } + normalized.SetNRGBA(x, y, color.NRGBA{A: alpha}) + } + } + + var encoded bytes.Buffer + if err := png.Encode(&encoded, normalized); err != nil { + return "", fmt.Errorf("编码蒙版 PNG 失败: %w", err) + } + return base64.StdEncoding.EncodeToString(encoded.Bytes()), nil +} + +func maskHasTransparency(mask *image.NRGBA) bool { + for y := mask.Bounds().Min.Y; y < mask.Bounds().Max.Y; y++ { + for x := mask.Bounds().Min.X; x < mask.Bounds().Max.X; x++ { + if mask.NRGBAAt(x, y).A < 0xff { + return true + } + } + } + return false +} + +// compositeMaskedEditB64 enforces the mask locally after generation. Pixels +// protected by the mask come from the original source, so model drift outside +// the editable selection cannot leak into the saved result. +func compositeMaskedEditB64(sourcePath, normalizedMaskB64, generatedB64 string) (string, error) { + source, err := loadImage(sourcePath) + if err != nil { + return "", fmt.Errorf("读取蒙版源图失败: %w", err) + } + mask, err := decodeBase64Image(normalizedMaskB64, "蒙版") + if err != nil { + return "", err + } + generated, err := decodeBase64Image(generatedB64, "生成结果") + if err != nil { + return "", err + } + + bounds := image.Rect(0, 0, source.Bounds().Dx(), source.Bounds().Dy()) + if bounds.Empty() { + return "", errors.New("蒙版源图尺寸无效") + } + sourceRGBA := image.NewNRGBA(bounds) + draw.Draw(sourceRGBA, bounds, source, source.Bounds().Min, draw.Src) + maskRGBA := image.NewNRGBA(bounds) + copyOrScaleMask(maskRGBA, mask) + generatedRGBA := image.NewNRGBA(bounds) + xdraw.CatmullRom.Scale(generatedRGBA, bounds, generated, generated.Bounds(), draw.Src, nil) + + result := image.NewNRGBA(bounds) + for y := 0; y < bounds.Dy(); y++ { + for x := 0; x < bounds.Dx(); x++ { + keep := uint32(maskRGBA.NRGBAAt(x, y).A) + edit := uint32(0xff) - keep + srcPixel := sourceRGBA.NRGBAAt(x, y) + generatedPixel := generatedRGBA.NRGBAAt(x, y) + result.SetNRGBA(x, y, color.NRGBA{ + R: blendMaskChannel(srcPixel.R, generatedPixel.R, keep, edit), + G: blendMaskChannel(srcPixel.G, generatedPixel.G, keep, edit), + B: blendMaskChannel(srcPixel.B, generatedPixel.B, keep, edit), + A: blendMaskChannel(srcPixel.A, generatedPixel.A, keep, edit), + }) + } + } + + var encoded bytes.Buffer + if err := png.Encode(&encoded, result); err != nil { + return "", fmt.Errorf("编码蒙版合成结果失败: %w", err) + } + return base64.StdEncoding.EncodeToString(encoded.Bytes()), nil +} + +func copyOrScaleMask(dst *image.NRGBA, src image.Image) { + if dst.Bounds().Dx() == src.Bounds().Dx() && dst.Bounds().Dy() == src.Bounds().Dy() { + draw.Draw(dst, dst.Bounds(), src, src.Bounds().Min, draw.Src) + return + } + xdraw.ApproxBiLinear.Scale(dst, dst.Bounds(), src, src.Bounds(), draw.Src, nil) +} + +func decodeBase64Image(value, label string) (image.Image, error) { + raw, err := base64.StdEncoding.DecodeString(strings.TrimSpace(value)) + if err != nil { + return nil, fmt.Errorf("%s base64 无效: %w", label, err) + } + decoded, _, err := image.Decode(bytes.NewReader(raw)) + if err != nil { + return nil, fmt.Errorf("解析%s失败: %w", label, err) + } + return decoded, nil +} + +func blendMaskChannel(source, generated uint8, keep, edit uint32) uint8 { + return uint8((uint32(source)*keep + uint32(generated)*edit + 127) / 255) +} diff --git a/image-studio/backend/mask_composite_test.go b/image-studio/backend/mask_composite_test.go new file mode 100644 index 000000000..cd438a978 --- /dev/null +++ b/image-studio/backend/mask_composite_test.go @@ -0,0 +1,118 @@ +package backend + +import ( + "bytes" + "encoding/base64" + "image" + "image/color" + "image/png" + "os" + "path/filepath" + "testing" +) + +func TestNormalizeEditMaskB64ConvertsWhiteSelectionToTransparentAlpha(t *testing.T) { + dir := t.TempDir() + sourcePath := filepath.Join(dir, "source.png") + writeMaskTestPNG(t, sourcePath, image.NewUniform(color.NRGBA{R: 20, G: 30, B: 40, A: 255}), image.Rect(0, 0, 2, 1)) + + mask := image.NewNRGBA(image.Rect(0, 0, 2, 1)) + mask.SetNRGBA(0, 0, color.NRGBA{A: 255}) + mask.SetNRGBA(1, 0, color.NRGBA{R: 255, G: 255, B: 255, A: 255}) + maskB64 := encodeMaskTestPNG(t, mask) + + normalizedB64, err := normalizeEditMaskB64(maskB64, sourcePath) + if err != nil { + t.Fatal(err) + } + normalized := decodeMaskTestPNG(t, normalizedB64) + if got := color.NRGBAModel.Convert(normalized.At(0, 0)).(color.NRGBA).A; got != 255 { + t.Fatalf("protected black alpha=%d want 255", got) + } + if got := color.NRGBAModel.Convert(normalized.At(1, 0)).(color.NRGBA).A; got != 0 { + t.Fatalf("editable white alpha=%d want 0", got) + } +} + +func TestCompositeMaskedEditB64KeepsProtectedPixels(t *testing.T) { + dir := t.TempDir() + sourcePath := filepath.Join(dir, "source.png") + source := image.NewNRGBA(image.Rect(0, 0, 2, 1)) + source.SetNRGBA(0, 0, color.NRGBA{R: 200, A: 255}) + source.SetNRGBA(1, 0, color.NRGBA{G: 200, A: 255}) + writeMaskTestPNG(t, sourcePath, source, source.Bounds()) + + mask := image.NewNRGBA(source.Bounds()) + mask.SetNRGBA(0, 0, color.NRGBA{A: 255}) + mask.SetNRGBA(1, 0, color.NRGBA{A: 0}) + generated := image.NewNRGBA(source.Bounds()) + generated.SetNRGBA(0, 0, color.NRGBA{B: 220, A: 255}) + generated.SetNRGBA(1, 0, color.NRGBA{B: 220, A: 255}) + + resultB64, err := compositeMaskedEditB64(sourcePath, encodeMaskTestPNG(t, mask), encodeMaskTestPNG(t, generated)) + if err != nil { + t.Fatal(err) + } + result := decodeMaskTestPNG(t, resultB64) + if got := color.NRGBAModel.Convert(result.At(0, 0)).(color.NRGBA); got != source.NRGBAAt(0, 0) { + t.Fatalf("protected pixel=%v want source %v", got, source.NRGBAAt(0, 0)) + } + if got := color.NRGBAModel.Convert(result.At(1, 0)).(color.NRGBA); got != generated.NRGBAAt(1, 0) { + t.Fatalf("editable pixel=%v want generated %v", got, generated.NRGBAAt(1, 0)) + } +} + +func TestNormalizeEditMaskB64PreservesFullyTransparentFullEditMask(t *testing.T) { + dir := t.TempDir() + sourcePath := filepath.Join(dir, "source.png") + writeMaskTestPNG(t, sourcePath, image.NewUniform(color.NRGBA{A: 255}), image.Rect(0, 0, 1, 1)) + mask := image.NewNRGBA(image.Rect(0, 0, 1, 1)) + + normalizedB64, err := normalizeEditMaskB64(encodeMaskTestPNG(t, mask), sourcePath) + if err != nil { + t.Fatal(err) + } + normalized := decodeMaskTestPNG(t, normalizedB64) + if got := color.NRGBAModel.Convert(normalized.At(0, 0)).(color.NRGBA).A; got != 0 { + t.Fatalf("full edit alpha=%d want 0", got) + } +} + +func encodeMaskTestPNG(t *testing.T, img image.Image) string { + t.Helper() + var buf bytes.Buffer + if err := png.Encode(&buf, img); err != nil { + t.Fatal(err) + } + return base64.StdEncoding.EncodeToString(buf.Bytes()) +} + +func decodeMaskTestPNG(t *testing.T, value string) image.Image { + t.Helper() + raw, err := base64.StdEncoding.DecodeString(value) + if err != nil { + t.Fatal(err) + } + img, err := png.Decode(bytes.NewReader(raw)) + if err != nil { + t.Fatal(err) + } + return img +} + +func writeMaskTestPNG(t *testing.T, path string, src image.Image, bounds image.Rectangle) { + t.Helper() + img := image.NewNRGBA(image.Rect(0, 0, bounds.Dx(), bounds.Dy())) + for y := 0; y < bounds.Dy(); y++ { + for x := 0; x < bounds.Dx(); x++ { + img.Set(x, y, src.At(bounds.Min.X+x, bounds.Min.Y+y)) + } + } + var buf bytes.Buffer + if err := png.Encode(&buf, img); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, buf.Bytes(), 0o600); err != nil { + t.Fatal(err) + } +} diff --git a/image-studio/backend/native_drag_windows.go b/image-studio/backend/native_drag_windows.go index cf94a9089..06de40636 100644 --- a/image-studio/backend/native_drag_windows.go +++ b/image-studio/backend/native_drag_windows.go @@ -8,6 +8,7 @@ import ( "strings" "sync/atomic" "syscall" + "time" "unsafe" "golang.org/x/sys/windows" @@ -23,6 +24,8 @@ const ( dropEffectCopy uintptr = 0x00000001 mouseKeyStateLeftButtonDown uintptr = 0x00000001 globalAllocMoveableZeroInit uintptr = 0x00000042 + virtualKeyLeftButton uintptr = 0x01 + nativeDragMaxDuration = 30 * time.Second cfHDrop uint16 = 15 dvAspectContent uint32 = 1 tymedHGlobal uint32 = 1 @@ -32,6 +35,7 @@ var ( ole32 = windows.NewLazySystemDLL("ole32.dll") shell32 = windows.NewLazySystemDLL("shell32.dll") kernel32 = windows.NewLazySystemDLL("kernel32.dll") + user32 = windows.NewLazySystemDLL("user32.dll") procOleInitialize = ole32.NewProc("OleInitialize") procOleUninitialize = ole32.NewProc("OleUninitialize") procDoDragDrop = ole32.NewProc("DoDragDrop") @@ -42,6 +46,7 @@ var ( procGlobalLock = kernel32.NewProc("GlobalLock") procGlobalUnlock = kernel32.NewProc("GlobalUnlock") procGlobalFree = kernel32.NewProc("GlobalFree") + procGetAsyncKeyState = user32.NewProc("GetAsyncKeyState") iidIUnknown = windows.GUID{Data1: 0x00000000, Data2: 0x0000, Data3: 0x0000, Data4: [8]byte{0xc0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x46}} iidIDataObj = windows.GUID{Data1: 0x0000010e, Data2: 0x0000, Data3: 0x0000, Data4: [8]byte{0xc0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x46}} @@ -57,8 +62,9 @@ type dropSourceVtbl struct { } type dropSource struct { - lpVtbl *dropSourceVtbl - refs uint32 + lpVtbl *dropSourceVtbl + refs uint32 + startedUnix int64 } var nativeDropSourceVtbl = dropSourceVtbl{ @@ -114,7 +120,7 @@ func beginNativeFileDrag(path string) error { return err } - source := &dropSource{lpVtbl: &nativeDropSourceVtbl, refs: 1} + source := &dropSource{lpVtbl: &nativeDropSourceVtbl, refs: 1, startedUnix: time.Now().UnixNano()} var effect uintptr hr, _, _ = procDoDragDrop.Call( dataObject, @@ -234,13 +240,23 @@ func dropSourceRelease(this uintptr) uintptr { return uintptr(next) } -func dropSourceQueryContinueDrag(_ uintptr, escapePressed uintptr, keyState uintptr) uintptr { +func dropSourceQueryContinueDrag(this uintptr, escapePressed uintptr, keyState uintptr) uintptr { if escapePressed != 0 { return dragDropSCancel } - if keyState&mouseKeyStateLeftButtonDown == 0 { + if this != 0 { + source := (*dropSource)(unsafe.Pointer(this)) + if source.startedUnix > 0 && time.Since(time.Unix(0, source.startedUnix)) >= nativeDragMaxDuration { + return dragDropSCancel + } + } + asyncState, _, _ := procGetAsyncKeyState.Call(virtualKeyLeftButton) + if keyState&mouseKeyStateLeftButtonDown == 0 || uint16(asyncState)&0x8000 == 0 { return dragDropSDrop } + // Some third-party file managers poll IDropSource extremely aggressively. + // Yield briefly so a stalled target cannot pin a CPU core while hovering. + time.Sleep(time.Millisecond) return sOK } diff --git a/image-studio/backend/service.go b/image-studio/backend/service.go index 7965c7293..58716a67a 100644 --- a/image-studio/backend/service.go +++ b/image-studio/backend/service.go @@ -35,6 +35,8 @@ type Service struct { ctx context.Context mu sync.Mutex + nativeDragMu sync.Mutex + nativeDragActive bool jobs map[string]*job runningByAPIMode map[string]int outputDir string // 用户自定义输出目录;空时回退到 defaultOutputDir() @@ -431,7 +433,14 @@ func (s *Service) runJob(ctx context.Context, jobID string, opts GenerateOptions PartialImages: clientOpts.PartialImages, } } + originalEditSourcePath := "" if mode == client.ModeEdit { + for _, candidate := range opts.collectPaths() { + if strings.TrimSpace(candidate) != "" { + originalEditSourcePath = strings.TrimSpace(candidate) + break + } + } paths, cleanup, prepErr := prepareUploadSourcePaths(opts.collectPaths()) if prepErr != nil { s.emitError(jobID, prepErr) @@ -442,6 +451,17 @@ func (s *Service) runJob(ctx context.Context, jobID string, opts GenerateOptions if fallbackClientOpts != nil { fallbackClientOpts.ImagePaths = paths } + if strings.TrimSpace(opts.MaskB64) != "" { + normalizedMask, maskErr := normalizeEditMaskB64(opts.MaskB64, originalEditSourcePath) + if maskErr != nil { + s.emitError(jobID, maskErr) + return + } + clientOpts.MaskB64 = normalizedMask + if fallbackClientOpts != nil { + fallbackClientOpts.MaskB64 = normalizedMask + } + } // Responses API 仍需 data URL(走 input_image 形态); // Images API 直接 multipart 上传文件,跳过 base64 编码节省往返开销。 if apiMode == client.APIModeResponses || (fallbackClientOpts != nil && fallbackClientOpts.APIMode == client.APIModeResponses) { @@ -569,8 +589,19 @@ func (s *Service) runJob(ctx context.Context, jobID string, opts GenerateOptions s.emitErrorWithRaw(jobID, err, rawPath) return } + effectiveOutputFormat := opts.OutputFormat + if mode == client.ModeEdit && strings.TrimSpace(clientOpts.MaskB64) != "" && len(clientOpts.ImagePaths) > 0 { + composited, compositeErr := compositeMaskedEditB64(originalEditSourcePath, clientOpts.MaskB64, result.ImageB64) + if compositeErr != nil { + s.emitErrorWithRaw(jobID, fmt.Errorf("保护蒙版外原图失败: %w", compositeErr), rawPath) + return + } + result.ImageB64 = composited + effectiveOutputFormat = "png" + logFn("已在本地合成蒙版结果，蒙版外区域使用原图像素。") + } - imageName := buildImageName(mode, opts.Prompt, timestamp, opts.OutputFormat) + imageName := buildImageName(mode, opts.Prompt, timestamp, effectiveOutputFormat) savedPath := filepath.Join(imagesDir, imageName) absSaved, werr := writeBase64PNG(result.ImageB64, savedPath) if werr != nil { @@ -605,6 +636,7 @@ func (s *Service) runJob(ctx context.Context, jobID string, opts GenerateOptions RawPath: absRaw, Mode: string(mode), Prompt: opts.Prompt, + OutputFormat: effectiveOutputFormat, }) } diff --git a/image-studio/backend/types.go b/image-studio/backend/types.go index cec9baa8f..6ebca49a9 100644 --- a/image-studio/backend/types.go +++ b/image-studio/backend/types.go @@ -192,6 +192,7 @@ type ResultPayload struct { RawPath string `json:"rawPath"` // raw SSE dump location Mode string `json:"mode"` Prompt string `json:"prompt"` + OutputFormat string `json:"outputFormat,omitempty"` } // PreviewPayload is emitted as `preview:` while Responses API streams diff --git a/image-studio/frontend/src/components/canvas/CanvasStage.tsx b/image-studio/frontend/src/components/canvas/CanvasStage.tsx index fc94f9d9f..6b2c66df3 100644 --- a/image-studio/frontend/src/components/canvas/CanvasStage.tsx +++ b/image-studio/frontend/src/components/canvas/CanvasStage.tsx @@ -464,7 +464,7 @@ export function CanvasStage() { <>
{!currentImage && !showingResultGrid && } diff --git a/image-studio/frontend/src/components/panel/FAQModal.tsx b/image-studio/frontend/src/components/panel/FAQModal.tsx index 0eb280809..c07f748a7 100644 --- a/image-studio/frontend/src/components/panel/FAQModal.tsx +++ b/image-studio/frontend/src/components/panel/FAQModal.tsx @@ -36,19 +36,19 @@ export function FAQModal({ open, onClose }: { open: boolean; onClose: () => void image_generation 工具触发生图，支持 HTTP SSE 与 WebSocket mode 两种传输。 能防 Cloudflare 524/504 超时(图像推理常常超过 100 秒)。
- Key 要绑「拥有 gpt-5.5 模型的分组」(中转站后台通常叫「余额分组」或「套餐分组」), - 不要选 image-2 分组。 + Key 需要拥有请求模型的权限(中转站后台通常按「余额分组」或「套餐分组」授权)。 + 请求模型可以是该 Responses 上游支持的文本或图像模型。
  • Images API · 兼容广:标准 OpenAI /v1/images/generations(文生图)+ /v1/images/edits(图生图,multipart 上传)。一次性 JSON 响应,无 SSE 保活, 长推理上 CF 超时风险更高。几乎所有 OpenAI 兼容中转站都开了这两个端点。
    - Key 可使用标准的 image-2 / image API 分组,不需要 gpt-5.5 权限。 + Key 需要拥有请求模型及 Images 端点权限。

    • - 选哪个?看你 key 绑的是哪个分组。 + 选哪个?看上游实际支持的端点和请求模型。 两边都有的话优先 Responses(SSE 更抗 524)。

    @@ -96,23 +96,19 @@ export function FAQModal({ open, onClose }: { open: boolean; onClose: () => void

    - 能换其他文本 / 图像模型吗? + 能换其他请求模型吗?

    - 可以。在「🔧 上游配置」里填即可,不同 API 形态用到的字段不一样: + 可以。在「🔧 上游配置」里填写请求模型 ID 即可:

    • - Responses API 用两个模型 ID: -
        -
      • 文本模型 ID(默认 gpt-5.5):承担推理 + 调用 image_generation 工具
        • -
      • 图像模型 ID(默认 gpt-image-2):工具实际用哪个图像模型出图
        • -
      + Responses API 只用一个请求模型 ID，直接写入请求的 model 字段。它可以是文本模型，也可以是上游支持的图像模型；应用不会再要求分别配置两种模型。
    • Images API 只用一个模型 ID:
        -
      • 图像模型 ID(默认 gpt-image-2):通常直接传给 /v1/images/generations 的 model 字段。Google 官方主机上的 gemini-3.1-flash-image 会按官方 Interactions API 请求；第三方 Gemini / Imagen 中转仍使用非流式 Images 兼容请求。文本模型 ID 在此模式下不读。
        • +
      • 请求模型 ID(通常为 gpt-image-2):直接传给 /v1/images/generations 的 model 字段。Google 官方主机上的 gemini-3.1-flash-image 会按官方 Interactions API 请求；第三方 Gemini / Imagen 中转仍使用非流式 Images 兼容请求。
    diff --git a/image-studio/frontend/src/components/panel/ResultDetailDrawer.tsx b/image-studio/frontend/src/components/panel/ResultDetailDrawer.tsx index e3b30b558..d5d87870b 100644 --- a/image-studio/frontend/src/components/panel/ResultDetailDrawer.tsx +++ b/image-studio/frontend/src/components/panel/ResultDetailDrawer.tsx @@ -128,7 +128,7 @@ export function ResultDetailDrawer() { {detail.revisedPrompt && (
    优化后提示词} - hint="Responses API 模式下文本模型可能会重写你的提示词。" + hint="Responses API 模式下请求模型可能会重写你的提示词。" > {detail.revisedPrompt}
    diff --git a/image-studio/frontend/src/components/panel/UpstreamProfileEditor.tsx b/image-studio/frontend/src/components/panel/UpstreamProfileEditor.tsx index e77492ad9..7351fb75f 100644 --- a/image-studio/frontend/src/components/panel/UpstreamProfileEditor.tsx +++ b/image-studio/frontend/src/components/panel/UpstreamProfileEditor.tsx @@ -75,6 +75,9 @@ export function UpstreamProfileEditor({ const selectedRequestPolicy = requestPolicyOptions.find((option) => option.id === draft.requestPolicy) ?? requestPolicyOptions[0]; const selectedReasoningEffort = REASONING_EFFORT_OPTIONS.find((option) => option.value === draft.reasoningEffort) ?? REASONING_EFFORT_OPTIONS[0]; const preferredModels = modelCatalog ? preferredModelsForAPIMode(modelCatalog, draft.apiMode) : null; + const requestModels = modelCatalog + ? (draft.apiMode === "responses" ? modelCatalog.all : preferredModels?.image ?? []) + : []; const fallbackCandidates = profiles.filter((profile) => profile.id !== draft.id && profile.baseURL.trim()); return ( @@ -124,7 +127,7 @@ export function UpstreamProfileEditor({
    {draft.apiMode === "responses" - ? "需要 key 绑定到「拥有 gpt-5.5 模型的分组」。SSE 保活可防 Cloudflare 524。" + ? "需要 key 拥有所填请求模型的权限。SSE 保活可防 Cloudflare 524。" : "使用标准 Images API,key 用 image-2 / image API 分组,兼容性最广。"} @@ -283,7 +286,7 @@ export function UpstreamProfileEditor({ - + - {preferredModels && preferredModels.text.length > 0 ? ( + {requestModels.length > 0 ? ( onPatchDraft({ textModelID: id })} @@ -332,25 +335,27 @@ export function UpstreamProfileEditor({ ) : null} - - onPatchDraft({ imageModelID: e.target.value })} - spellCheck={false} - className={`focus-ring w-full min-w-0 border border-black/[0.08] bg-[var(--surface)] px-3 py-2 text-sm text-zinc-900 placeholder:text-zinc-400 dark:border-white/[0.08] dark:text-zinc-100 dark:placeholder:text-zinc-500 font-mono-token ${usesFluentUI ? "rounded-[10px]" : "rounded-[14px]"}`} - /> - {preferredModels && preferredModels.image.length > 0 ? ( - onPatchDraft({ imageModelID: id })} + {draft.apiMode === "images" ? ( + + onPatchDraft({ imageModelID: e.target.value })} + spellCheck={false} + className={`focus-ring w-full min-w-0 border border-black/[0.08] bg-[var(--surface)] px-3 py-2 text-sm text-zinc-900 placeholder:text-zinc-400 dark:border-white/[0.08] dark:text-zinc-100 dark:placeholder:text-zinc-500 font-mono-token ${usesFluentUI ? "rounded-[10px]" : "rounded-[14px]"}`} /> - ) : null} - + {requestModels.length > 0 ? ( + onPatchDraft({ imageModelID: id })} + /> + ) : null} + + ) : null} { + const [source, mask] = await Promise.all([ + loadImage(sourceDataURL), + loadImage(dataURLFromBase64(maskB64)), + ]); + const canvas = makeCanvas(source.naturalWidth, source.naturalHeight); + const context = requireContext(canvas); + context.drawImage(mask, 0, 0, canvas.width, canvas.height); + const pixels = context.getImageData(0, 0, canvas.width, canvas.height); + let hasTransparency = false; + for (let offset = 3; offset < pixels.data.length; offset += 4) { + const alpha = pixels.data[offset]; + if (alpha < 255) hasTransparency = true; + } + if (!hasTransparency) { + for (let offset = 0; offset < pixels.data.length; offset += 4) { + const luma = Math.round( + pixels.data[offset] * 0.299 + + pixels.data[offset + 1] * 0.587 + + pixels.data[offset + 2] * 0.114, + ); + pixels.data[offset] = 0; + pixels.data[offset + 1] = 0; + pixels.data[offset + 2] = 0; + pixels.data[offset + 3] = 255 - luma; + } + context.putImageData(pixels, 0, 0); + } + return { sourceDataURL, maskB64: stripDataURL(canvas.toDataURL("image/png")) }; +} + +export async function compositeMaskedEdit( + prepared: PreparedMaskComposite, + generatedB64: string, +): Promise { + const [source, generated, mask] = await Promise.all([ + loadImage(prepared.sourceDataURL), + loadImage(dataURLFromBase64(generatedB64)), + loadImage(dataURLFromBase64(prepared.maskB64)), + ]); + const width = source.naturalWidth; + const height = source.naturalHeight; + const maskCanvas = makeCanvas(width, height); + const maskContext = requireContext(maskCanvas); + maskContext.drawImage(mask, 0, 0, width, height); + const maskPixels = maskContext.getImageData(0, 0, width, height).data; + + const sourceCanvas = makeCanvas(width, height); + const sourceContext = requireContext(sourceCanvas); + sourceContext.drawImage(source, 0, 0, width, height); + const sourcePixels = sourceContext.getImageData(0, 0, width, height).data; + + const generatedCanvas = makeCanvas(width, height); + const generatedContext = requireContext(generatedCanvas); + generatedContext.drawImage(generated, 0, 0, width, height); + const generatedPixels = generatedContext.getImageData(0, 0, width, height).data; + + const result = generatedContext.createImageData(width, height); + for (let offset = 0; offset < result.data.length; offset += 4) { + const keep = maskPixels[offset + 3]; + const edit = 255 - keep; + for (let channel = 0; channel < 4; channel++) { + result.data[offset + channel] = Math.round( + (sourcePixels[offset + channel] * keep + generatedPixels[offset + channel] * edit) / 255, + ); + } + } + generatedContext.putImageData(result, 0, 0); + return stripDataURL(generatedCanvas.toDataURL("image/png")); +} + +function makeCanvas(width: number, height: number): HTMLCanvasElement { + if (!Number.isFinite(width) || !Number.isFinite(height) || width <= 0 || height <= 0) { + throw new Error("蒙版源图尺寸无效"); + } + const canvas = document.createElement("canvas"); + canvas.width = width; + canvas.height = height; + return canvas; +} + +function requireContext(canvas: HTMLCanvasElement): CanvasRenderingContext2D { + const context = canvas.getContext("2d", { willReadFrequently: true }); + if (!context) throw new Error("无法创建蒙版合成画布"); + return context; +} + +function loadImage(dataURL: string): Promise { + return new Promise((resolve, reject) => { + const image = new Image(); + image.onload = () => resolve(image); + image.onerror = () => reject(new Error("无法解码蒙版或源图片")); + image.src = dataURL; + }); +} + +function stripDataURL(value: string): string { + const comma = value.indexOf(","); + return comma >= 0 ? value.slice(comma + 1) : value; +} diff --git a/image-studio/frontend/src/platform/android/upstream/AndroidUpstreamProfileForm.tsx b/image-studio/frontend/src/platform/android/upstream/AndroidUpstreamProfileForm.tsx index a5dd26486..7f5cff942 100644 --- a/image-studio/frontend/src/platform/android/upstream/AndroidUpstreamProfileForm.tsx +++ b/image-studio/frontend/src/platform/android/upstream/AndroidUpstreamProfileForm.tsx @@ -59,6 +59,9 @@ export function AndroidUpstreamProfileForm({ const isActive = draft.id === activeProfileId; const busy = saving || isTestingKey; const preferredModels = modelCatalog ? preferredModelsForAPIMode(modelCatalog, draft.apiMode) : null; + const requestModels = modelCatalog + ? (draft.apiMode === "responses" ? modelCatalog.all : preferredModels?.image ?? []) + : []; return (
    @@ -187,7 +190,7 @@ export function AndroidUpstreamProfileForm({
    - + - {preferredModels && preferredModels.text.length > 0 ? ( + {requestModels.length > 0 ? ( onPatchDraft({ textModelID: id })} /> @@ -223,23 +226,25 @@ export function AndroidUpstreamProfileForm({ ) : null} - - onPatchDraft({ imageModelID: event.target.value })} - placeholder="留空 = 默认 gpt-image-2" - className="focus-ring android-upstream-input font-mono-token" - spellCheck={false} - /> - {preferredModels && preferredModels.image.length > 0 ? ( - onPatchDraft({ imageModelID: id })} + {draft.apiMode === "images" ? ( + + onPatchDraft({ imageModelID: event.target.value })} + placeholder="留空 = 默认 gpt-image-2" + className="focus-ring android-upstream-input font-mono-token" + spellCheck={false} /> - ) : null} - + {requestModels.length > 0 ? ( + onPatchDraft({ imageModelID: id })} + /> + ) : null} + + ) : null}
    diff --git a/image-studio/frontend/src/platform/runtime/host.ts b/image-studio/frontend/src/platform/runtime/host.ts index 940472c66..9cccad5a8 100644 --- a/image-studio/frontend/src/platform/runtime/host.ts +++ b/image-studio/frontend/src/platform/runtime/host.ts @@ -18,7 +18,10 @@ import { registerVirtualText, releaseVirtualPath, rotateVirtualImage, + sourceToDataURL, } from "../../lib/virtualHostStore.ts"; +import { compositeMaskedEdit, normalizeMaskForSource, type PreparedMaskComposite } from "../../lib/maskComposite.ts"; +import { dataURLFromBase64 } from "../../lib/images.ts"; import { browserStoredAPIKey, fileNameFromPath, @@ -146,12 +149,26 @@ async function startRemoteJob(options: GenerateOptionsLike): Promise { try { + let preparedMask: PreparedMaskComposite | null = null; + let remoteOptions = options; + if (options.mode === "edit" && options.maskB64) { + const firstSource = options.sourceImages?.[0]; + let sourceDataURL = await sourceToDataURL(firstSource); + const sourcePath = firstSource?.path || options.imagePaths?.[0] || options.imagePath; + if (!sourceDataURL && sourcePath) { + const sourceB64 = await ReadImageAsBase64(sourcePath); + sourceDataURL = dataURLFromBase64(sourceB64); + } + if (!sourceDataURL) throw new RemoteKernelError("蒙版任务无法读取第一张源图"); + preparedMask = await normalizeMaskForSource(sourceDataURL, options.maskB64); + remoteOptions = { ...options, maskB64: preparedMask.maskB64 }; + } const result = await runRemoteImageJob({ payload: { - ...options, - requestPolicy: normalizeRequestPolicy(options.requestPolicy), - imagesNewAPICompat: options.imagesNewAPICompat === true, - allowInsecureConnection: options.allowInsecureConnection === true, - }, sourceImages: options.sourceImages }, { + ...remoteOptions, + requestPolicy: normalizeRequestPolicy(remoteOptions.requestPolicy), + imagesNewAPICompat: remoteOptions.imagesNewAPICompat === true, + allowInsecureConnection: remoteOptions.allowInsecureConnection === true, + }, sourceImages: remoteOptions.sourceImages }, { signal: controller.signal, onLog: (line) => emitLocalEvent(`log:${jobId}`, line), onProgress: (stage, elapsed, bytes) => emitLocalEvent(`progress:${jobId}`, { stage, elapsed, bytes }), @@ -164,18 +181,23 @@ async function startRemoteJob(options: GenerateOptionsLike): Promise { @@ -198,6 +209,21 @@ export function imageDims(b64: string): { w: number; h: number } | null { return getImageDimensionsFromBase64(b64); } +export async function loadImageDims(b64: string): Promise<{ w: number; h: number } | null> { + if (!b64) return null; + const fast = detectImageMimeTypeFromBase64(b64) === "image/png" ? imageDims(b64) : null; + if (fast) return fast; + if (typeof Image === "undefined") return null; + return new Promise((resolve) => { + const image = new Image(); + image.onload = () => resolve(image.naturalWidth > 0 && image.naturalHeight > 0 + ? { w: image.naturalWidth, h: image.naturalHeight } + : null); + image.onerror = () => resolve(null); + image.src = tempDataURLFromB64(b64); + }); +} + export function augmentPromptWithAnnotations( prompt: string, annotations: Annotation[], diff --git a/image-studio/frontend/src/state/studioStore.ts b/image-studio/frontend/src/state/studioStore.ts index 4ebb10a9b..c9b05a9f9 100644 --- a/image-studio/frontend/src/state/studioStore.ts +++ b/image-studio/frontend/src/state/studioStore.ts @@ -164,6 +164,7 @@ import { genId, imageDims, loadModeConfig, + loadImageDims, loadStoredActiveProfileId, loadStoredAIProfileId, loadStoredProfiles, @@ -1263,6 +1264,26 @@ export const useStudioStore = create((set, get) => ({ jobsCompleted: 0, runningJobs: [], }; + const importedMaskDataURL = s.maskDataURL && s.maskDataURL !== "__PENDING_MASK__" ? s.maskDataURL : null; + let maskTargetDims = s.currentImage?.imageB64 ? await loadImageDims(s.currentImage.imageB64) : null; + if (s.mode === "edit" && s.strokes.length > 0 && !maskTargetDims && editSourcePaths[0]) { + const sourceB64 = await ReadImageAsBase64(editSourcePaths[0]).catch(() => ""); + maskTargetDims = await loadImageDims(sourceB64); + } + if (s.mode === "edit" && s.strokes.length > 0 && !maskTargetDims) { + set({ + errorMessage: "无法读取源图真实尺寸，未提交蒙版任务。请重新导入源图后再试。", + errorCanRetry: false, + errorRawPath: null, + }); + return; + } + const maskDataURL = s.mode === "edit" + ? (s.strokes.length > 0 + ? buildMaskPNGDataURL(s.strokes, maskTargetDims) + : importedMaskDataURL) + : null; + const maskB64 = maskDataURL ? stripDataURLPrefix(maskDataURL) : ""; set({ ...runPatch, batchCount, @@ -1280,14 +1301,6 @@ export const useStudioStore = create((set, get) => ({ resultGridOpen: requestedJobCount > 1, }), }); - - const importedMaskDataURL = s.maskDataURL && s.maskDataURL !== "__PENDING_MASK__" ? s.maskDataURL : null; - const maskDataURL = s.mode === "edit" - ? (s.strokes.length > 0 - ? buildMaskPNGDataURL(s.strokes, s.currentImage?.imageB64 ? imageDims(s.currentImage.imageB64) : null) - : importedMaskDataURL) - : null; - const maskB64 = maskDataURL ? stripDataURLPrefix(maskDataURL) : ""; let augmentedPrompt = augmentPromptWithAnnotations(s.prompt, s.annotations, s.currentImage?.imageB64 ? imageDims(s.currentImage.imageB64) : null); // Append style chip suffix if the user picked one (other than "全部"). const styleSuffix = STYLE_SUFFIXES[s.styleTag]; @@ -2544,7 +2557,9 @@ async function launchOneJob( mode: r.mode as Mode, size: snapshot.size, quality: snapshot.quality, - outputFormat: snapshot.outputFormat, + outputFormat: r.outputFormat === "png" || r.outputFormat === "jpeg" || r.outputFormat === "webp" + ? r.outputFormat + : snapshot.outputFormat, parentId, createdAt: Date.now(), seed: payload.seed || undefined, diff --git a/image-studio/frontend/src/styles/_canvas.css b/image-studio/frontend/src/styles/_canvas.css index 8e9f4a157..f4043eae0 100644 --- a/image-studio/frontend/src/styles/_canvas.css +++ b/image-studio/frontend/src/styles/_canvas.css @@ -292,6 +292,14 @@ html[data-target-platform="macos"] .source-strip { will-change: background-position; } +/* The Konva canvas fully covers the checkerboard once an image is present. + Stop the hidden background animation so an idle/backgrounded WebView does + not keep repainting the whole desktop canvas. */ +.stage-host.has-image { + animation: none; + will-change: auto; +} + .stream-preview-overlay { position: absolute; top: 12px; diff --git a/image-studio/frontend/test/requestModel.test.mjs b/image-studio/frontend/test/requestModel.test.mjs index 5119006fa..097b80b32 100644 --- a/image-studio/frontend/test/requestModel.test.mjs +++ b/image-studio/frontend/test/requestModel.test.mjs @@ -40,9 +40,11 @@ test("Responses payload defaults partial_images to streaming preview count", () quality: "low", outputFormat: "png", imageModelID: "gpt-image-2", - textModelID: "gpt-5.5", + textModelID: "gpt-image-1.5", requestPolicy: "openai", }, []); + assert.equal(payload.model, "gpt-image-1.5"); + assert.equal("model" in payload.tools[0], false); assert.equal(payload.tools[0].partial_images, DEFAULT_PARTIAL_IMAGES); assert.equal(payload.reasoning.effort, DEFAULT_REASONING_EFFORT); }); diff --git a/shared/kernel/requestModel.js b/shared/kernel/requestModel.js index 858227a42..e9ad19121 100644 --- a/shared/kernel/requestModel.js +++ b/shared/kernel/requestModel.js @@ -372,7 +372,6 @@ export function buildResponsesImageTool(payload, sourceDataURLs, options = {}) { const partialImages = payload.disablePreview ? 0 : normalizePartialImages(payload.partialImages); const tool = { type: "image_generation", - model: normalizeImageModel(payload.imageModelID), action: sourceDataURLs.length > 0 ? "edit" : "generate", size, quality, From d27fbd694cf3310c0e5ce3e62a3ea69a69dfb626 Mon Sep 17 00:00:00 2001 From: lin <1609924163@qq.com> Date: Tue, 21 Jul 2026 12:45:51 +0800 Subject: [PATCH 2/6] Add provider-aware upstream probing and profile persistence --- .../android/AndroidImageStudioBridge.kt | 41 ++++++--- gio-client/internal/compat/state.go | 5 ++ gio-client/internal/kernel/probe.go | 44 ++++++++-- gio-client/internal/kernel/runner.go | 15 +++- gio-client/internal/kernel/runner_test.go | 31 +++++++ gio-client/internal/ui/app.go | 6 ++ gio-client/internal/ui/codex_config.go | 2 + gio-client/internal/ui/layout_controls.go | 37 +++++++- gio-client/internal/ui/run_job.go | 2 + gio-client/internal/ui/ui_state.go | 22 +++++ .../internal/ui/upstream_config_transfer.go | 7 ++ go-cli/pkg/client/client.go | 4 +- go-cli/pkg/client/grok_images.go | 87 +++++++++++++++++++ go-cli/pkg/client/images_api.go | 23 ++++- go-cli/pkg/client/images_api_test.go | 83 ++++++++++++++++++ go-cli/pkg/client/types.go | 14 +++ go-cli/pkg/client/url.go | 51 ++++++++--- go-cli/pkg/client/url_test.go | 16 ++++ image-studio/backend/probe.go | 43 ++++++--- image-studio/backend/probe_test.go | 31 +++++++ image-studio/backend/service.go | 2 + image-studio/backend/types.go | 5 +- .../frontend/src/app/dev/previewData.ts | 1 + .../components/panel/UpstreamConfigModal.tsx | 42 ++++++--- .../panel/UpstreamProfileEditor.tsx | 67 +++++++++++--- .../components/panel/UpstreamProfileList.tsx | 2 +- image-studio/frontend/src/lib/profiles.ts | 18 +++- .../src/lib/upstreamConfigTransfer.ts | 12 ++- .../upstream/AndroidUpstreamEmptyState.tsx | 10 +-- .../upstream/AndroidUpstreamProfileForm.tsx | 37 ++++++-- .../upstream/AndroidUpstreamProfileRail.tsx | 3 +- .../upstream/useAndroidUpstreamConfig.ts | 31 ++++++- .../frontend/src/platform/runtime/host.ts | 2 + .../src/platform/runtime/hostTypes.ts | 5 +- .../platform/runtime/remote-kernel/common.ts | 19 ++-- .../platform/runtime/remote-kernel/images.ts | 6 +- .../platform/runtime/remote-kernel/index.ts | 5 +- .../runtime/remote-kernel/requestPayloads.ts | 52 ++++++++++- .../platform/runtime/remote-kernel/types.ts | 4 +- .../src/state/studioStore.profiles.ts | 14 ++- .../frontend/src/state/studioStore.ts | 13 ++- .../frontend/src/state/studioStore.types.ts | 3 + image-studio/frontend/src/types/domain.ts | 3 + .../frontend/test/remoteKernel.test.mjs | 50 +++++++++++ image-studio/frontend/wailsjs/go/models.ts | 4 + shared/compat-go/state.go | 7 ++ shared/compat-go/state_test.go | 3 +- shared/kernel/requestModel.d.ts | 7 +- shared/kernel/requestModel.js | 29 +++++-- 49 files changed, 892 insertions(+), 128 deletions(-) create mode 100644 go-cli/pkg/client/grok_images.go diff --git a/android-shell/app/src/main/java/top/gptcodex/imagestudio/android/AndroidImageStudioBridge.kt b/android-shell/app/src/main/java/top/gptcodex/imagestudio/android/AndroidImageStudioBridge.kt index d3a27436c..bb670c60b 100644 --- a/android-shell/app/src/main/java/top/gptcodex/imagestudio/android/AndroidImageStudioBridge.kt +++ b/android-shell/app/src/main/java/top/gptcodex/imagestudio/android/AndroidImageStudioBridge.kt @@ -595,18 +595,24 @@ class AndroidImageStudioBridge( val apiKey = payload.optString("apiKey").trim() val proxyMode = payload.optString("proxyMode", "system") val proxyUrl = payload.optString("proxyURL", "") + val provider = payload.optString("provider", "openai").lowercase(Locale.US).let { + if (it == "google" || it == "grok") it else "openai" + } val apiMode = payload.optString("apiMode", "responses") val responsesTransport = payload.optString("responsesTransport", "sse") if (apiKey.isBlank()) throw IllegalArgumentException("API Key 不能为空") thread(name = "image-studio-probe-${requestId.take(12)}") { try { - val connection = openHttpConnection(openAIEndpoint(baseUrl, "models"), proxyMode, proxyUrl, allowInsecureConnection).apply { + val endpointLabel = if (provider == "google") "/v1beta/models" else "/v1/models" + val modelsEndpoint = if (provider == "google") googleEndpoint(baseUrl, "models") else openAIEndpoint(baseUrl, "models") + val connection = openHttpConnection(modelsEndpoint, proxyMode, proxyUrl, allowInsecureConnection).apply { requestMethod = "GET" instanceFollowRedirects = true connectTimeout = 20_000 readTimeout = 20_000 doInput = true - setRequestProperty("Authorization", "Bearer $apiKey") + if (provider == "google") setRequestProperty("X-Goog-Api-Key", apiKey) + else setRequestProperty("Authorization", "Bearer $apiKey") setRequestProperty("Accept", "application/json") setRequestProperty("User-Agent", "image-studio-android") } @@ -617,30 +623,30 @@ class AndroidImageStudioBridge( } ?: "" connection.disconnect() if (status !in 200..299) { - throw IllegalStateException("上游 /v1/models 返回 $status${summarizeProbeBody(body).let { if (it.isBlank()) "" else ": $it" }}") + throw IllegalStateException("上游 $endpointLabel 返回 $status${summarizeProbeBody(body).let { if (it.isBlank()) "" else ": $it" }}") } val parsed = JSONObject(body) - if (!parsed.has("data") || parsed.isNull("data")) { - throw IllegalStateException("上游 /v1/models 响应缺少 data 数组") - } - val data = parsed.optJSONArray("data") ?: throw IllegalStateException("上游 /v1/models 响应缺少 data 数组") + val data = if (provider == "google") parsed.optJSONArray("models") + ?: throw IllegalStateException("上游 /v1beta/models 响应缺少 models 数组") + else parsed.optJSONArray("data") + ?: throw IllegalStateException("上游 /v1/models 响应缺少 data 数组") val models = mutableListOf>() for (index in 0 until data.length()) { val item = data.optJSONObject(index) ?: continue - val id = item.optString("id").trim() + val id = if (provider == "google") item.optString("name").trim().removePrefix("models/") else item.optString("id").trim() if (id.isBlank()) continue models += mapOf( "id" to id, - "object" to item.optString("object").trim(), - "ownedBy" to item.optString("owned_by").trim(), - "displayName" to item.optString("name").trim(), + "object" to if (provider == "google") "model" else item.optString("object").trim(), + "ownedBy" to if (provider == "google") "google" else item.optString("owned_by").trim(), + "displayName" to if (provider == "google") item.optString("displayName").trim() else item.optString("name").trim(), ) } val result = mutableMapOf( "modelCount" to data.length(), "models" to models, ) - if (apiMode == "responses" && responsesTransport == "websocket") { + if (provider == "openai" && apiMode == "responses" && responsesTransport == "websocket") { result["responsesTransport"] = "websocket" val probePayload = JSONObject() .put("type", "response.create") @@ -994,6 +1000,17 @@ class AndroidImageStudioBridge( return if (basePath.endsWith("/openai")) "$cleaned/$path" else "$cleaned/v1/$path" } + private fun googleEndpoint(baseUrl: String, endpointPath: String): String { + var cleaned = baseUrl.trim().trimEnd('/') + val path = endpointPath.trim().trim('/') + cleaned = when { + cleaned.endsWith("/v1beta/openai", ignoreCase = true) -> cleaned.dropLast("/openai".length) + cleaned.endsWith("/v1beta", ignoreCase = true) -> cleaned + else -> "$cleaned/v1beta" + } + return if (path.isBlank()) cleaned else "$cleaned/$path" + } + private fun isProbeLoopbackHost(host: String): Boolean { val lower = host.lowercase(Locale.US).trim('[', ']') return lower == "localhost" || lower.endsWith(".localhost") || lower == "127.0.0.1" || lower == "::1" || lower == "0:0:0:0:0:0:0:1" diff --git a/gio-client/internal/compat/state.go b/gio-client/internal/compat/state.go index 56e8f8a8f..2e3c5e9d3 100644 --- a/gio-client/internal/compat/state.go +++ b/gio-client/internal/compat/state.go @@ -106,7 +106,11 @@ func ConfigFromState(cfg kernel.Config, state shared.State) kernel.Config { cfg.BaseURL = profile.BaseURL cfg.TextModelID = profile.TextModelID cfg.ImageModelID = profile.ImageModelID + cfg.Provider = client.NormalizeProvider(client.Provider(profile.Provider)) cfg.APIMode = normaliseAPIMode(profile.APIMode) + if cfg.Provider != client.ProviderOpenAI { + cfg.APIMode = client.APIModeImages + } cfg.ResponsesTransport = client.ResponsesTransport(normalizeProfileResponsesTransport(profile.ResponsesTransport)) cfg.FallbackProfileID = strings.TrimSpace(profile.FallbackProfileID) cfg.RequestPolicy = normalisePolicy(profile.RequestPolicy) @@ -234,6 +238,7 @@ func UpsertConfig(state shared.State, cfg kernel.Config) shared.State { profile := shared.UpstreamProfile{ ID: profileID, Name: nextDefaultProfileName(state.Profiles), + Provider: string(client.NormalizeProvider(cfg.Provider)), APIMode: string(normaliseAPIMode(string(cfg.APIMode))), ResponsesTransport: normalizeProfileResponsesTransport(string(cfg.ResponsesTransport)), RequestPolicy: string(normalisePolicy(string(cfg.RequestPolicy))), diff --git a/gio-client/internal/kernel/probe.go b/gio-client/internal/kernel/probe.go index a913063fd..7afc449b5 100644 --- a/gio-client/internal/kernel/probe.go +++ b/gio-client/internal/kernel/probe.go @@ -42,11 +42,22 @@ func ProbeUpstream(ctx context.Context, cfg Config) (ProbeResult, error) { ctx, cancel := context.WithTimeout(ctx, probeUpstreamTimeout) defer cancel() - req, err := http.NewRequestWithContext(ctx, http.MethodGet, baseURL+"/v1/models", nil) + provider := client.NormalizeProvider(cfg.Provider) + modelsEndpoint := client.OpenAIAPIEndpoint(baseURL, "models") + endpointLabel := "/v1/models" + if provider == client.ProviderGoogle { + modelsEndpoint = client.GoogleAPIEndpoint(baseURL, "models") + endpointLabel = "/v1beta/models" + } + req, err := http.NewRequestWithContext(ctx, http.MethodGet, modelsEndpoint, nil) if err != nil { return ProbeResult{}, fmt.Errorf("构造测活请求失败: %w", err) } - req.Header.Set("Authorization", "Bearer "+apiKey) + if provider == client.ProviderGoogle { + req.Header.Set("X-Goog-Api-Key", apiKey) + } else { + req.Header.Set("Authorization", "Bearer "+apiKey) + } req.Header.Set("Accept", "application/json") req.Header.Set("User-Agent", client.UserAgent()) @@ -72,9 +83,9 @@ func ProbeUpstream(ctx context.Context, cfg Config) (ProbeResult, error) { summary = readErr.Error() } if summary != "" { - return ProbeResult{}, fmt.Errorf("上游 /v1/models 返回 %d: %s", resp.StatusCode, summary) + return ProbeResult{}, fmt.Errorf("上游 %s 返回 %d: %s", endpointLabel, resp.StatusCode, summary) } - return ProbeResult{}, fmt.Errorf("上游 /v1/models 返回 %d", resp.StatusCode) + return ProbeResult{}, fmt.Errorf("上游 %s 返回 %d", endpointLabel, resp.StatusCode) } if readErr != nil { return ProbeResult{}, fmt.Errorf("读取上游响应失败: %w", readErr) @@ -87,12 +98,27 @@ func ProbeUpstream(ctx context.Context, cfg Config) (ProbeResult, error) { OwnedBy string `json:"owned_by"` DisplayName string `json:"name"` } `json:"data"` + Models []struct { + Name string `json:"name"` + DisplayName string `json:"displayName"` + } `json:"models"` } if err := json.Unmarshal(body, &parsed); err != nil { - return ProbeResult{}, fmt.Errorf("上游 /v1/models 返回的 JSON 无效: %w", err) - } - if parsed.Data == nil { - return ProbeResult{}, fmt.Errorf("上游 /v1/models 响应缺少 data 数组") + return ProbeResult{}, fmt.Errorf("上游 %s 返回的 JSON 无效: %w", endpointLabel, err) + } + if provider == client.ProviderGoogle { + for _, item := range parsed.Models { + parsed.Data = append(parsed.Data, struct { + ID string `json:"id"` + Object string `json:"object"` + OwnedBy string `json:"owned_by"` + DisplayName string `json:"name"` + }{ + ID: strings.TrimPrefix(strings.TrimSpace(item.Name), "models/"), Object: "model", OwnedBy: "google", DisplayName: item.DisplayName, + }) + } + } else if parsed.Data == nil { + return ProbeResult{}, fmt.Errorf("上游 %s 响应缺少 data 数组", endpointLabel) } models := make([]UpstreamModelDescriptor, 0, len(parsed.Data)) for _, item := range parsed.Data { @@ -111,7 +137,7 @@ func ProbeUpstream(ctx context.Context, cfg Config) (ProbeResult, error) { ModelCount: len(parsed.Data), Models: models, } - if cfg.APIMode == client.APIModeResponses && + if provider == client.ProviderOpenAI && cfg.APIMode == client.APIModeResponses && client.NormalizeProxyTransportValue(strings.TrimSpace(string(cfg.ResponsesTransport))) == string(client.ResponsesTransportWebSocket) { result.ResponsesTransport = string(client.ResponsesTransportWebSocket) if wsErr := client.ProbeResponsesWebSocket(ctx, client.ProbeResponsesWebSocketOptions{ diff --git a/gio-client/internal/kernel/runner.go b/gio-client/internal/kernel/runner.go index b98112a90..67bbaf06b 100644 --- a/gio-client/internal/kernel/runner.go +++ b/gio-client/internal/kernel/runner.go @@ -30,6 +30,7 @@ type Config struct { ImageModelID string Prompt string Mode client.Mode + Provider client.Provider APIMode client.APIMode RequestPolicy client.RequestPolicy ResponsesTransport client.ResponsesTransport @@ -72,6 +73,7 @@ type FallbackProfileConfig struct { BaseURL string TextModelID string ImageModelID string + Provider client.Provider APIMode client.APIMode ResponsesTransport client.ResponsesTransport RequestPolicy client.RequestPolicy @@ -107,6 +109,7 @@ func DefaultConfig() Config { TextModelID: client.TextModel, ImageModelID: client.ImageModel, Mode: client.ModeGenerate, + Provider: client.ProviderOpenAI, APIMode: client.APIModeResponses, RequestPolicy: client.RequestPolicyOpenAI, ResponsesTransport: client.ResponsesTransportSSE, @@ -206,6 +209,7 @@ func (Runner) Run(ctx context.Context, cfg Config, cb Callbacks) (Result, error) fallbackCfg.BaseURL = fallback.BaseURL fallbackCfg.TextModelID = fallback.TextModelID fallbackCfg.ImageModelID = fallback.ImageModelID + fallbackCfg.Provider = fallback.Provider fallbackCfg.APIMode = fallback.APIMode fallbackCfg.ResponsesTransport = fallback.ResponsesTransport fallbackCfg.RequestPolicy = fallback.RequestPolicy @@ -274,7 +278,10 @@ func normalizeConfig(cfg Config) Config { if cfg.Mode != client.ModeEdit { cfg.Mode = client.ModeGenerate } - if cfg.APIMode != client.APIModeImages { + cfg.Provider = client.NormalizeProvider(cfg.Provider) + if cfg.Provider != client.ProviderOpenAI || cfg.APIMode == client.APIModeImages { + cfg.APIMode = client.APIModeImages + } else { cfg.APIMode = client.APIModeResponses } if cfg.RequestPolicy != client.RequestPolicyCompat { @@ -343,8 +350,11 @@ func normalizeFallbackProfileConfig(value *FallbackProfileConfig) *FallbackProfi next.BaseURL = strings.TrimSpace(next.BaseURL) next.TextModelID = strings.TrimSpace(next.TextModelID) next.ImageModelID = strings.TrimSpace(next.ImageModelID) + next.Provider = client.NormalizeProvider(next.Provider) next.RequestPolicy = normalizeRequestPolicy(next.RequestPolicy) - if next.APIMode != client.APIModeImages { + if next.Provider != client.ProviderOpenAI || next.APIMode == client.APIModeImages { + next.APIMode = client.APIModeImages + } else { next.APIMode = client.APIModeResponses } if next.ResponsesTransport != client.ResponsesTransportWebSocket { @@ -385,6 +395,7 @@ func runSingleConfig(ctx context.Context, cfg Config, cb Callbacks, imagesDir st ImageModelID: cfg.ImageModelID, Prompt: cfg.Prompt, Mode: cfg.Mode, + Provider: cfg.Provider, APIMode: cfg.APIMode, ResponsesTransport: cfg.ResponsesTransport, RequestPolicy: cfg.RequestPolicy, diff --git a/gio-client/internal/kernel/runner_test.go b/gio-client/internal/kernel/runner_test.go index 856fc64cd..190aee9f7 100644 --- a/gio-client/internal/kernel/runner_test.go +++ b/gio-client/internal/kernel/runner_test.go @@ -156,6 +156,37 @@ func TestProbeUpstreamReturnsModelCount(t *testing.T) { } } +func TestProbeUpstreamGoogleProviderUsesNativeModelsEndpoint(t *testing.T) { + var gotPath string + var gotGoogleKey string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotPath = r.URL.Path + gotGoogleKey = r.Header.Get("X-Goog-Api-Key") + if auth := r.Header.Get("Authorization"); auth != "" { + t.Fatalf("Google probe must not send Authorization: %q", auth) + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"models":[{"name":"models/gemini-3.1-flash-image","displayName":"Nano Banana 2"}]}`)) + })) + defer server.Close() + + result, err := ProbeUpstream(context.Background(), Config{ + APIKey: "google-key", + BaseURL: server.URL, + Provider: client.ProviderGoogle, + APIMode: client.APIModeImages, + }) + if err != nil { + t.Fatalf("Google probe returned error: %v", err) + } + if gotPath != "/v1beta/models" || gotGoogleKey != "google-key" { + t.Fatalf("path=%q X-Goog-Api-Key=%q", gotPath, gotGoogleKey) + } + if result.ModelCount != 1 || len(result.Models) != 1 || result.Models[0].ID != "gemini-3.1-flash-image" || result.Models[0].OwnedBy != "google" { + t.Fatalf("unexpected Google models: %+v", result) + } +} + func TestProbeUpstreamSkipsModelItemsWithoutID(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") diff --git a/gio-client/internal/ui/app.go b/gio-client/internal/ui/app.go index e1a79c693..3f46fe202 100644 --- a/gio-client/internal/ui/app.go +++ b/gio-client/internal/ui/app.go @@ -25,6 +25,7 @@ import ( "gioui.org/unit" "gioui.org/widget" "gioui.org/widget/material" + "github.com/yuanhua/image-gptcodex/pkg/client" "github.com/yuanhua/image-gptcodex/pkg/promptimport" ) @@ -239,6 +240,7 @@ type App struct { customSizeHeightInput widget.Editor mode string + provider string api string size string quality string @@ -283,6 +285,7 @@ type App struct { batchCount int modeButtons []widget.Clickable + providerButtons []widget.Clickable apiButtons []widget.Clickable sizeButtons []widget.Clickable aspectButtons []widget.Clickable @@ -908,6 +911,7 @@ func New() *App { th: th, runner: kernel.Runner{}, mode: string(cfg.Mode), + provider: string(client.NormalizeProvider(cfg.Provider)), api: string(cfg.APIMode), size: cfg.Size, quality: cfg.Quality, @@ -1007,6 +1011,7 @@ func New() *App { workflowPreviewButtons: make([]widget.Clickable, len(partialPreviewChoices)), pruneGeneralHistoryButtons: make([]widget.Clickable, 2), modeButtons: make([]widget.Clickable, len(modeChoices)), + providerButtons: make([]widget.Clickable, 3), apiButtons: make([]widget.Clickable, len(apiChoices)), sizeButtons: make([]widget.Clickable, len(sizeChoices)), aspectButtons: make([]widget.Clickable, len(aspectChoices)+len(compatState.Settings.CustomAspectRatios)+24), @@ -1220,6 +1225,7 @@ func (a *App) configureEditors(cfg kernel.Config) { a.baseURLInput.SetText(cfg.BaseURL) a.textModelInput.SetText(cfg.TextModelID) a.imageModelInput.SetText(cfg.ImageModelID) + a.provider = string(client.NormalizeProvider(cfg.Provider)) a.outputDirInput.SetText(cfg.OutputDir) a.partialImagesInput.SetText(strconv.Itoa(cfg.PartialImages)) a.outputCompressionInput.SetText(strconv.Itoa(cfg.OutputCompression)) diff --git a/gio-client/internal/ui/codex_config.go b/gio-client/internal/ui/codex_config.go index 7af8f543e..3c77af2e3 100644 --- a/gio-client/internal/ui/codex_config.go +++ b/gio-client/internal/ui/codex_config.go @@ -305,6 +305,7 @@ func (a *App) applyCodexConfigSync(imported codexAPIConfig) error { continue } state.Profiles[i].Name = name + state.Profiles[i].Provider = string(client.ProviderOpenAI) state.Profiles[i].APIMode = string(client.APIModeResponses) state.Profiles[i].RequestPolicy = string(client.RequestPolicyOpenAI) state.Profiles[i].ImagesNewAPICompat = false @@ -319,6 +320,7 @@ func (a *App) applyCodexConfigSync(imported codexAPIConfig) error { state.Profiles = append(state.Profiles, sharedCompat.UpstreamProfile{ ID: profileID, Name: name, + Provider: string(client.ProviderOpenAI), APIMode: string(client.APIModeResponses), RequestPolicy: string(client.RequestPolicyOpenAI), ImagesNewAPICompat: false, diff --git a/gio-client/internal/ui/layout_controls.go b/gio-client/internal/ui/layout_controls.go index 017010474..927125787 100644 --- a/gio-client/internal/ui/layout_controls.go +++ b/gio-client/internal/ui/layout_controls.go @@ -2860,12 +2860,31 @@ func (a *App) layoutSettingsEditorPane(gtx layout.Context, snap snapshot) layout return a.field(gtx, "名称", &a.profileNameInput, "配置1", unit.Dp(40)) }), layout.Rigid(func(gtx layout.Context) layout.Dimensions { + return a.layoutSettingsOptionCards(gtx, "服务商请求方式", []settingsOptionChoice{ + {Title: "OpenAI / 兼容", Detail: "Responses 或 Images", Value: string(client.ProviderOpenAI)}, + {Title: "Google Gemini", Detail: "Interactions 原生协议", Value: string(client.ProviderGoogle)}, + {Title: "Grok / xAI", Detail: "Imagine 原生协议", Value: string(client.ProviderGrok)}, + }, a.provider, a.providerButtons, 3, func(value string) { + a.provider = normalizeProfileProvider(value) + if a.provider != string(client.ProviderOpenAI) { + a.api = string(client.APIModeImages) + a.policy = string(client.RequestPolicyOpenAI) + } + }) + }), + layout.Rigid(func(gtx layout.Context) layout.Dimensions { + if a.provider != string(client.ProviderOpenAI) { + return a.label(gtx, "服务商决定请求路径、认证头与请求体；第三方 OpenAI 兼容中转请选择 OpenAI / 兼容。", unit.Sp(10), fluent.textDim, font.Normal) + } return a.layoutSettingsOptionCards(gtx, "API 形态", []settingsOptionChoice{ {Title: "Responses API", Detail: "SSE 保活，更适合长推理", Value: string(client.APIModeResponses)}, {Title: "Images API", Detail: "标准 generations / edits", Value: string(client.APIModeImages)}, }, a.api, a.apiButtons, 2, func(value string) { a.api = value }) }), layout.Rigid(func(gtx layout.Context) layout.Dimensions { + if a.provider != string(client.ProviderOpenAI) { + return layout.Dimensions{} + } hint := "需要 key 拥有所填请求模型的权限。SSE 保活可防 Cloudflare 524。" if a.api == string(client.APIModeImages) { hint = "使用标准 Images API，key 走 image-2 / image API 分组，兼容性最广。" @@ -2873,7 +2892,7 @@ func (a *App) layoutSettingsEditorPane(gtx layout.Context, snap snapshot) layout return a.label(gtx, hint, unit.Sp(10), fluent.textDim, font.Normal) }), layout.Rigid(func(gtx layout.Context) layout.Dimensions { - if a.api != string(client.APIModeResponses) { + if a.provider != string(client.ProviderOpenAI) || a.api != string(client.APIModeResponses) { return layout.Dimensions{} } return a.layoutSettingsOptionCards(gtx, "Responses 传输", responsesTransportChoices, a.responsesTransport, a.responsesTransportButtons, 2, func(value string) { @@ -2881,25 +2900,37 @@ func (a *App) layoutSettingsEditorPane(gtx layout.Context, snap snapshot) layout }) }), layout.Rigid(func(gtx layout.Context) layout.Dimensions { - if a.api != string(client.APIModeResponses) { + if a.provider != string(client.ProviderOpenAI) || a.api != string(client.APIModeResponses) { return layout.Dimensions{} } return a.label(gtx, "WebSocket mode 需要上游 / 代理正确转发 Upgrade: websocket；若握手失败，请求层会自动回退到 HTTP SSE，并把原因写进 Raw response。", unit.Sp(10), fluent.textDim, font.Normal) }), layout.Rigid(func(gtx layout.Context) layout.Dimensions { + if a.provider != string(client.ProviderOpenAI) { + return layout.Dimensions{} + } return a.layoutSettingsOptionCards(gtx, "参数策略", []settingsOptionChoice{ {Title: "OpenAI 标准", Detail: "只发送官方公开字段", Value: string(client.RequestPolicyOpenAI)}, {Title: "兼容中转扩展", Detail: "附带 seed / negative_prompt", Value: string(client.RequestPolicyCompat)}, }, a.policy, a.policyButtons, 1, func(value string) { a.policy = value }) }), layout.Rigid(func(gtx layout.Context) layout.Dimensions { + if a.provider != string(client.ProviderOpenAI) { + return layout.Dimensions{} + } return a.label(gtx, "OpenAI 标准更适合直连 OpenAI；兼容中转扩展会额外发送 relay 常见扩展字段。", unit.Sp(10), fluent.textDim, font.Normal) }), layout.Rigid(func(gtx layout.Context) layout.Dimensions { return a.technicalField(gtx, "上游 BASE_URL", &a.baseURLInput, "https://example.com", unit.Dp(40)) }), layout.Rigid(func(gtx layout.Context) layout.Dimensions { - return a.label(gtx, "中转站只填根地址；Google 官方 Images 配置填写 https://generativelanguage.googleapis.com/v1beta/openai。", unit.Sp(10), fluent.textDim, font.Normal) + hint := "中转站只填根地址，应用自动拼接 OpenAI API 路径。" + if a.provider == string(client.ProviderGoogle) { + hint = "Google 官方填写 https://generativelanguage.googleapis.com，应用自动拼接 /v1beta/interactions。" + } else if a.provider == string(client.ProviderGrok) { + hint = "xAI 官方填写 https://api.x.ai，应用自动拼接 Imagine API 路径。" + } + return a.label(gtx, hint, unit.Sp(10), fluent.textDim, font.Normal) }), layout.Rigid(func(gtx layout.Context) layout.Dimensions { active := a.allowInsecureConnection diff --git a/gio-client/internal/ui/run_job.go b/gio-client/internal/ui/run_job.go index bd9ff7c48..5c4f96224 100644 --- a/gio-client/internal/ui/run_job.go +++ b/gio-client/internal/ui/run_job.go @@ -598,6 +598,7 @@ func (a *App) currentConfig() kernel.Config { ImageModelID: a.imageModelInput.Text(), Prompt: prompt, Mode: client.Mode(a.mode), + Provider: client.Provider(a.provider), APIMode: client.APIMode(a.api), RequestPolicy: client.RequestPolicy(a.policy), ResponsesTransport: client.ResponsesTransport(a.responsesTransport), @@ -769,6 +770,7 @@ func resolveFallbackProfileConfig(state sharedCompat.State, fallbackProfileID st BaseURL: strings.TrimSpace(profile.BaseURL), TextModelID: strings.TrimSpace(profile.TextModelID), ImageModelID: strings.TrimSpace(profile.ImageModelID), + Provider: client.Provider(normalizeProfileProvider(profile.Provider)), APIMode: client.APIMode(normalizeProfileAPIMode(profile.APIMode)), ResponsesTransport: client.ResponsesTransport(normalizeProfileResponsesTransport(profile.ResponsesTransport)), RequestPolicy: client.RequestPolicy(normalizeProfilePolicy(profile.RequestPolicy)), diff --git a/gio-client/internal/ui/ui_state.go b/gio-client/internal/ui/ui_state.go index 28d3f5502..21de4c807 100644 --- a/gio-client/internal/ui/ui_state.go +++ b/gio-client/internal/ui/ui_state.go @@ -1395,6 +1395,17 @@ func normalizeProfileAPIMode(mode string) string { return string(client.APIModeResponses) } +func normalizeProfileProvider(value string) string { + switch strings.ToLower(strings.TrimSpace(value)) { + case string(client.ProviderGoogle): + return string(client.ProviderGoogle) + case string(client.ProviderGrok): + return string(client.ProviderGrok) + default: + return string(client.ProviderOpenAI) + } +} + func normalizeProfilePolicy(policy string) string { if strings.TrimSpace(policy) == string(client.RequestPolicyCompat) { return string(client.RequestPolicyCompat) @@ -1447,7 +1458,11 @@ func (a *App) settingsDraftReady() bool { func (a *App) applySettingsProfileDraft(state sharedCompat.State, profile sharedCompat.UpstreamProfile) { a.settingsSelectedProfileID = profile.ID a.profileNameInput.SetText(strings.TrimSpace(profile.Name)) + a.provider = normalizeProfileProvider(profile.Provider) a.api = normalizeProfileAPIMode(profile.APIMode) + if a.provider != string(client.ProviderOpenAI) { + a.api = string(client.APIModeImages) + } a.policy = normalizeProfilePolicy(profile.RequestPolicy) a.responsesTransport = normalizeProfileResponsesTransport(profile.ResponsesTransport) a.reasoningEffort = normalizeReasoningEffort(profile.ReasoningEffort) @@ -1584,6 +1599,7 @@ func (a *App) loadSettingsProfileDraft(profileID string) error { a.protectStreamPreview = kernel.DefaultConfig().ProtectStreamPreview } a.imagesNewAPICompat = false + a.provider = string(client.ProviderOpenAI) a.api = string(client.APIModeResponses) a.policy = string(client.RequestPolicyOpenAI) a.apiKeyVisible = false @@ -1678,7 +1694,11 @@ func (a *App) saveSettingsSelection() error { } } state.Profiles[i].Name = name + state.Profiles[i].Provider = normalizeProfileProvider(a.provider) state.Profiles[i].APIMode = normalizeProfileAPIMode(a.api) + if state.Profiles[i].Provider != string(client.ProviderOpenAI) { + state.Profiles[i].APIMode = string(client.APIModeImages) + } state.Profiles[i].RequestPolicy = normalizeProfilePolicy(a.policy) state.Profiles[i].ResponsesTransport = normalizeProfileResponsesTransport(a.responsesTransport) state.Profiles[i].ReasoningEffort = normalizeReasoningEffort(a.reasoningEffort) @@ -1802,6 +1822,7 @@ func (a *App) createSettingsProfile(apiMode string) error { profile = sharedCompat.UpstreamProfile{ ID: nextProfileID(state.Profiles, now), Name: nextProfileName(state.Profiles), + Provider: string(client.ProviderOpenAI), APIMode: normalizeProfileAPIMode(apiMode), ResponsesTransport: string(client.ResponsesTransportSSE), RequestPolicy: string(client.RequestPolicyOpenAI), @@ -2070,6 +2091,7 @@ func (a *App) createBlankProfileWithMode(apiMode string) { profile = sharedCompat.UpstreamProfile{ ID: nextProfileID(state.Profiles, now), Name: nextProfileName(state.Profiles), + Provider: string(client.ProviderOpenAI), APIMode: apiMode, ResponsesTransport: string(client.ResponsesTransportSSE), RequestPolicy: string(client.RequestPolicyOpenAI), diff --git a/gio-client/internal/ui/upstream_config_transfer.go b/gio-client/internal/ui/upstream_config_transfer.go index d0a1056eb..c8a83d623 100644 --- a/gio-client/internal/ui/upstream_config_transfer.go +++ b/gio-client/internal/ui/upstream_config_transfer.go @@ -19,6 +19,7 @@ import ( type upstreamConfigExportProfile struct { ID string `json:"id"` Name string `json:"name"` + Provider string `json:"provider"` APIMode string `json:"apiMode"` ResponsesTransport string `json:"responsesTransport"` RequestPolicy string `json:"requestPolicy"` @@ -78,6 +79,7 @@ func buildUpstreamConfigExportFile(state sharedCompat.State) (upstreamConfigExpo profiles = append(profiles, upstreamConfigExportProfile{ ID: profile.ID, Name: strings.TrimSpace(profile.Name), + Provider: normalizeProfileProvider(profile.Provider), APIMode: normalizeProfileAPIMode(profile.APIMode), ResponsesTransport: normalizeProfileResponsesTransport(profile.ResponsesTransport), RequestPolicy: normalizeProfilePolicy(profile.RequestPolicy), @@ -181,6 +183,7 @@ func parseExportProfile(raw any) (upstreamConfigExportProfile, bool) { return upstreamConfigExportProfile{ ID: strings.TrimSpace(id), Name: strings.TrimSpace(name), + Provider: normalizeProfileProvider(stringValue(source["provider"])), APIMode: normalizeProfileAPIMode(stringValue(source["apiMode"])), ResponsesTransport: normalizeProfileResponsesTransport(stringValue(source["responsesTransport"])), RequestPolicy: normalizeProfilePolicy(stringValue(source["requestPolicy"])), @@ -213,6 +216,7 @@ func parseNewAPIChannelConnTemplate(input map[string]any) (parsedUpstreamConfigI Profiles: []upstreamConfigExportProfile{{ ID: profileID, Name: buildTemplateProfileName("NewAPI", baseURL, ""), + Provider: string(client.ProviderOpenAI), APIMode: string(client.APIModeResponses), ResponsesTransport: string(client.ResponsesTransportSSE), RequestPolicy: string(client.RequestPolicyOpenAI), @@ -262,6 +266,7 @@ func parseOpenCodeProviderTemplate(input map[string]any) (parsedUpstreamConfigIm profiles = append(profiles, upstreamConfigExportProfile{ ID: nextTemplateProfileID(index), Name: buildTemplateProfileName("OpenCode", baseURL, providerName), + Provider: string(client.ProviderOpenAI), APIMode: apiMode, ResponsesTransport: string(client.ResponsesTransportSSE), RequestPolicy: string(client.RequestPolicyOpenAI), @@ -386,6 +391,7 @@ func (a *App) applyParsedUpstreamConfigImport(parsed parsedUpstreamConfigImport) if existingIndex, ok := existingByName[strings.TrimSpace(incoming.Name)]; ok { profile := state.Profiles[existingIndex] profile.Name = strings.TrimSpace(incoming.Name) + profile.Provider = normalizeProfileProvider(incoming.Provider) profile.APIMode = normalizeProfileAPIMode(incoming.APIMode) profile.ResponsesTransport = normalizeProfileResponsesTransport(incoming.ResponsesTransport) profile.RequestPolicy = normalizeProfilePolicy(incoming.RequestPolicy) @@ -411,6 +417,7 @@ func (a *App) applyParsedUpstreamConfigImport(parsed parsedUpstreamConfigImport) profile := sharedCompat.UpstreamProfile{ ID: actualID, Name: strings.TrimSpace(incoming.Name), + Provider: normalizeProfileProvider(incoming.Provider), APIMode: normalizeProfileAPIMode(incoming.APIMode), ResponsesTransport: normalizeProfileResponsesTransport(incoming.ResponsesTransport), RequestPolicy: normalizeProfilePolicy(incoming.RequestPolicy), diff --git a/go-cli/pkg/client/client.go b/go-cli/pkg/client/client.go index 673500625..9961c5a47 100644 --- a/go-cli/pkg/client/client.go +++ b/go-cli/pkg/client/client.go @@ -146,7 +146,7 @@ func RequestAndExtractWithRetriesAndPartial( onProgress func(stage string, elapsed int, bytes int64), onPartial func(PartialImage), ) (ImageResult, string, error) { - if opts.APIMode == APIModeImages { + if EffectiveAPIMode(opts.Provider, opts.APIMode) == APIModeImages { return imagesAPIWithRetries(ctx, opts, outputDir, timestamp, onLog, onProgress, onPartial) } return responsesAPIWithRetries(ctx, transport, opts, outputDir, timestamp, onLog, onProgress, onPartial) @@ -160,7 +160,7 @@ func RequestAndExtractWithRetriesAndPartialInMemory( onProgress func(stage string, elapsed int, bytes int64), onPartial func(PartialImage), ) (ImageResult, string, error) { - if opts.APIMode == APIModeImages { + if EffectiveAPIMode(opts.Provider, opts.APIMode) == APIModeImages { return imagesAPIWithRetriesInMemory(ctx, opts, onLog, onProgress, onPartial) } return responsesAPIWithRetriesInMemory(ctx, transport, opts, onLog, onProgress, onPartial) diff --git a/go-cli/pkg/client/grok_images.go b/go-cli/pkg/client/grok_images.go new file mode 100644 index 000000000..e66fe96a7 --- /dev/null +++ b/go-cli/pkg/client/grok_images.go @@ -0,0 +1,87 @@ +package client + +import ( + "encoding/json" + "fmt" + "math" + "strings" +) + +var grokAspectRatios = []string{ + "1:1", "16:9", "9:16", "4:3", "3:4", "3:2", "2:3", "2:1", "1:2", "19.5:9", "9:19.5", "20:9", "9:20", +} + +func buildGrokImagePayload(opts Options, paths []string, model, size string) ([]byte, error) { + if strings.TrimSpace(opts.MaskB64) != "" { + return nil, fmt.Errorf("Grok Imagine 不支持 OpenAI mask 参数；请清除蒙版后重试") + } + payload := map[string]any{ + "model": model, + "prompt": opts.Prompt, + "response_format": "b64_json", + } + if ratio, resolution := grokImageDimensions(size); ratio != "" { + payload["aspect_ratio"] = ratio + payload["resolution"] = resolution + } + if opts.Mode == ModeEdit { + if len(paths) == 0 { + return nil, fmt.Errorf("Grok Imagine 图生图需要至少一张源图") + } + images := make([]map[string]string, 0, len(paths)) + for _, path := range paths { + dataURL, err := ImageFileToDataURL(path) + if err != nil { + return nil, err + } + images = append(images, map[string]string{ + "type": "image_url", + "url": dataURL, + }) + } + if len(images) == 1 { + payload["image"] = images[0] + } else { + payload["image"] = images + } + } + return json.Marshal(payload) +} + +func grokImageDimensions(size string) (string, string) { + width, height, ok := parseGoogleInteractionSize(size) + if !ok { + return "", "" + } + maxSide := width + if height > maxSide { + maxSide = height + } + resolution := "1k" + if maxSide >= 1536 { + resolution = "2k" + } + return closestGrokAspectRatio(float64(width) / float64(height)), resolution +} + +func closestGrokAspectRatio(target float64) string { + best := "1:1" + bestDistance := math.Inf(1) + for _, candidate := range grokAspectRatios { + parts := strings.Split(candidate, ":") + left, _ := parseAspectNumber(parts[0]) + right, _ := parseAspectNumber(parts[1]) + distance := math.Abs(math.Log((left / right) / target)) + if distance < bestDistance { + best = candidate + bestDistance = distance + } + } + return best +} + +func parseAspectNumber(value string) (float64, error) { + var out float64 + _, err := fmt.Sscanf(value, "%f", &out) + return out, err +} diff --git a/go-cli/pkg/client/images_api.go b/go-cli/pkg/client/images_api.go index bbb19fd51..daa6e60fe 100644 --- a/go-cli/pkg/client/images_api.go +++ b/go-cli/pkg/client/images_api.go @@ -261,7 +261,10 @@ func RequestImagesAPIWithPartial( } includeExtended := shouldSendExtendedImageParameters(opts.RequestPolicy) useNewAPICompat := shouldUseImagesNonStreamingCompat(model, opts.ImagesNewAPICompat) - useGoogleInteractions := shouldUseGoogleNativeInteractions(baseURL, model) + provider := NormalizeProvider(opts.Provider) + useGoogleInteractions := provider == ProviderGoogle || + (provider == ProviderOpenAI && shouldUseGoogleNativeInteractions(baseURL, model)) + useGrokImagine := provider == ProviderGrok var ( url string @@ -287,6 +290,22 @@ func RequestImagesAPIWithPartial( } body = bytes.NewReader(payload) contentType = "application/json" + } else if useGrokImagine { + paths := []string(nil) + if opts.Mode == ModeEdit { + paths = opts.imageSourcePathsForEdit() + } + payload, err := buildGrokImagePayload(opts, paths, model, size) + if err != nil { + return ImageResult{}, err + } + endpoint := "images/generations" + if opts.Mode == ModeEdit { + endpoint = "images/edits" + } + url = openAIAPIEndpoint(baseURL, endpoint) + body = bytes.NewReader(payload) + contentType = "application/json" } else if opts.Mode == ModeEdit { paths := opts.imageSourcePathsForEdit() if len(paths) == 0 { @@ -372,6 +391,8 @@ func RequestImagesAPIWithPartial( progressStage := "等待 Images API 返回(无 SSE 保活)" if useGoogleInteractions { progressStage = "等待 Google Interactions 返回(无 SSE 保活)" + } else if useGrokImagine { + progressStage = "等待 Grok Imagine 返回(无 SSE 保活)" } // Progress ticker — Images API has no streaming so we just tick elapsed time. stopProgress := make(chan struct{}) diff --git a/go-cli/pkg/client/images_api_test.go b/go-cli/pkg/client/images_api_test.go index f4d9a7f83..b593da6e1 100644 --- a/go-cli/pkg/client/images_api_test.go +++ b/go-cli/pkg/client/images_api_test.go @@ -599,3 +599,86 @@ func TestRequestImagesAPISendsDalle3Style(t *testing.T) { t.Fatalf("request body missing style=natural: %s", requestBody) } } + +func TestRequestImagesAPIExplicitGoogleProviderUsesInteractionsOnCustomBase(t *testing.T) { + finalB64 := base64.StdEncoding.EncodeToString(fakePNG) + var requestPath string + var googleKey string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestPath = r.URL.Path + googleKey = r.Header.Get("X-Goog-Api-Key") + w.Header().Set("Content-Type", "application/json") + fmt.Fprintf(w, `{"status":"completed","output_image":{"type":"image","data":%q,"mime_type":"image/png"}}`, finalB64) + })) + defer srv.Close() + + result, err := RequestImagesAPIWithPartial(context.Background(), Options{ + APIKey: "google-key", Provider: ProviderGoogle, BaseURL: srv.URL, + Prompt: "cat", ImageModelID: "gemini-image-model", APIMode: APIModeResponses, + }, &bytes.Buffer{}, nil, nil) + if err != nil { + t.Fatal(err) + } + if requestPath != "/v1beta/interactions" || googleKey != "google-key" { + t.Fatalf("path=%q google key=%q", requestPath, googleKey) + } + if result.ImageB64 != finalB64 || result.SourceEvent != "google_interactions" { + t.Fatalf("result=%+v", result) + } +} + +func TestRequestImagesAPIGrokProviderUsesNativeJSONContracts(t *testing.T) { + finalB64 := base64.StdEncoding.EncodeToString([]byte("final")) + dir := t.TempDir() + source := filepath.Join(dir, "source.png") + if err := os.WriteFile(source, fakePNG, 0o600); err != nil { + t.Fatal(err) + } + tests := []struct { + name string + mode Mode + paths []string + wantPath string + }{ + {name: "generation", mode: ModeGenerate, wantPath: "/v1/images/generations"}, + {name: "edit", mode: ModeEdit, paths: []string{source}, wantPath: "/v1/images/edits"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var requestBody []byte + var requestPath, contentType, authorization string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestPath = r.URL.Path + contentType = r.Header.Get("Content-Type") + authorization = r.Header.Get("Authorization") + requestBody, _ = io.ReadAll(r.Body) + w.Header().Set("Content-Type", "application/json") + fmt.Fprintf(w, `{"data":[{"b64_json":%q}]}`, finalB64) + })) + defer srv.Close() + + _, err := RequestImagesAPIWithPartial(context.Background(), Options{ + APIKey: "xai-key", Provider: ProviderGrok, BaseURL: srv.URL, + Prompt: "cat", ImageModelID: "grok-imagine-image", Mode: tt.mode, + ImagePaths: tt.paths, Size: "2048x1152", APIMode: APIModeResponses, + }, &bytes.Buffer{}, nil, nil) + if err != nil { + t.Fatal(err) + } + if requestPath != tt.wantPath || contentType != "application/json" || authorization != "Bearer xai-key" { + t.Fatalf("path=%q content-type=%q auth=%q", requestPath, contentType, authorization) + } + var payload map[string]any + if err := json.Unmarshal(requestBody, &payload); err != nil { + t.Fatal(err) + } + if payload["aspect_ratio"] != "16:9" || payload["resolution"] != "2k" || payload["response_format"] != "b64_json" { + t.Fatalf("payload=%s", requestBody) + } + _, hasImage := payload["image"] + if hasImage != (tt.mode == ModeEdit) { + t.Fatalf("image presence=%v payload=%s", hasImage, requestBody) + } + }) + } +} diff --git a/go-cli/pkg/client/types.go b/go-cli/pkg/client/types.go index 357206ee3..327e27f8f 100644 --- a/go-cli/pkg/client/types.go +++ b/go-cli/pkg/client/types.go @@ -116,6 +116,16 @@ const ( APIModeImages APIMode = "images" ) +// Provider selects the upstream vendor contract. Empty values retain the +// historical OpenAI-compatible behavior. +type Provider string + +const ( + ProviderOpenAI Provider = "openai" + ProviderGoogle Provider = "google" + ProviderGrok Provider = "grok" +) + type ResponsesTransport string const ( @@ -162,6 +172,10 @@ type Options struct { // Empty string is treated as APIModeResponses for back-compat. APIMode APIMode + // Provider selects request paths, authentication, and payload semantics. + // Google and Grok use their native image APIs and therefore imply Images. + Provider Provider + // ResponsesTransport selects how Responses API requests are transported. // Empty string is treated as ResponsesTransportSSE for back-compat. ResponsesTransport ResponsesTransport diff --git a/go-cli/pkg/client/url.go b/go-cli/pkg/client/url.go index 5f3b5f5b6..cc6d0bea0 100644 --- a/go-cli/pkg/client/url.go +++ b/go-cli/pkg/client/url.go @@ -58,6 +58,44 @@ func OpenAIAPIEndpoint(baseURL, endpointPath string) string { return cleaned + "/v1/" + path } +// NormalizeProvider keeps old configurations on the OpenAI-compatible path. +func NormalizeProvider(provider Provider) Provider { + switch Provider(strings.ToLower(strings.TrimSpace(string(provider)))) { + case ProviderGoogle: + return ProviderGoogle + case ProviderGrok: + return ProviderGrok + default: + return ProviderOpenAI + } +} + +// EffectiveAPIMode returns the only valid request shape for the selected +// provider. Native Google and Grok image APIs do not use OpenAI Responses. +func EffectiveAPIMode(provider Provider, apiMode APIMode) APIMode { + if NormalizeProvider(provider) != ProviderOpenAI || apiMode == APIModeImages { + return APIModeImages + } + return APIModeResponses +} + +// GoogleAPIEndpoint accepts an official root, the legacy /v1beta/openai base, +// or a Google-compatible relay prefix without duplicating version segments. +func GoogleAPIEndpoint(baseURL, endpointPath string) string { + cleaned := strings.TrimRight(strings.TrimSpace(baseURL), "/") + path := strings.Trim(strings.TrimSpace(endpointPath), "/") + lower := strings.ToLower(cleaned) + if strings.HasSuffix(lower, "/v1beta/openai") { + cleaned = cleaned[:len(cleaned)-len("/openai")] + } else if !strings.HasSuffix(lower, "/v1beta") { + cleaned += "/v1beta" + } + if path == "" { + return cleaned + } + return cleaned + "/" + path +} + func openAIAPIEndpoint(baseURL, endpointPath string) string { return OpenAIAPIEndpoint(baseURL, endpointPath) } @@ -88,18 +126,11 @@ func shouldUseGoogleNativeInteractions(baseURL, model string) bool { } func googleInteractionsEndpoint(baseURL string) (string, error) { - if !isOfficialGoogleGeminiBaseURL(baseURL) { - return "", fmt.Errorf("Google Interactions 仅支持官方 generativelanguage.googleapis.com 主机") - } - u, err := url.Parse(strings.TrimRight(strings.TrimSpace(baseURL), "/")) - if err != nil { + endpoint := GoogleAPIEndpoint(baseURL, "interactions") + if _, err := url.Parse(endpoint); err != nil { return "", err } - u.Path = "/v1beta/interactions" - u.RawPath = "" - u.RawQuery = "" - u.Fragment = "" - return u.String(), nil + return endpoint, nil } func isLoopbackHost(host string) bool { diff --git a/go-cli/pkg/client/url_test.go b/go-cli/pkg/client/url_test.go index 8dc8d1753..dc439d7bf 100644 --- a/go-cli/pkg/client/url_test.go +++ b/go-cli/pkg/client/url_test.go @@ -85,3 +85,19 @@ func TestGoogleInteractionsEndpointIsNarrowToOfficialNanoBanana2(t *testing.T) { t.Fatal("non-Nano-Banana-2 model must stay on the documented OpenAI-compatible path") } } + +func TestProviderRoutingDefaultsAndNativeModes(t *testing.T) { + t.Parallel() + if got := NormalizeProvider(""); got != ProviderOpenAI { + t.Fatalf("provider=%q", got) + } + if got := EffectiveAPIMode(ProviderGoogle, APIModeResponses); got != APIModeImages { + t.Fatalf("google mode=%q", got) + } + if got := EffectiveAPIMode(ProviderGrok, APIModeResponses); got != APIModeImages { + t.Fatalf("grok mode=%q", got) + } + if got := GoogleAPIEndpoint("https://generativelanguage.googleapis.com/v1beta/openai", "models"); got != "https://generativelanguage.googleapis.com/v1beta/models" { + t.Fatalf("google endpoint=%q", got) + } +} diff --git a/image-studio/backend/probe.go b/image-studio/backend/probe.go index b03f4c812..57961561d 100644 --- a/image-studio/backend/probe.go +++ b/image-studio/backend/probe.go @@ -17,7 +17,11 @@ const probeUpstreamTimeout = 20 * time.Second const probeUpstreamMaxBody = 1 << 20 type modelsListProbeResponse struct { - Data []upstreamModelDescriptorJSON `json:"data"` + Data []upstreamModelDescriptorJSON `json:"data"` + Models []struct { + Name string `json:"name"` + DisplayName string `json:"displayName"` + } `json:"models"` } type upstreamModelDescriptorJSON struct { @@ -46,11 +50,22 @@ func probeUpstream(parent context.Context, opts ProbeUpstreamOptions) (ProbeUpst ctx, cancel := context.WithTimeout(parent, probeUpstreamTimeout) defer cancel() - req, err := http.NewRequestWithContext(ctx, http.MethodGet, client.OpenAIAPIEndpoint(baseURL, "models"), nil) + provider := client.NormalizeProvider(client.Provider(opts.Provider)) + modelsEndpoint := client.OpenAIAPIEndpoint(baseURL, "models") + endpointLabel := "/v1/models" + if provider == client.ProviderGoogle { + modelsEndpoint = client.GoogleAPIEndpoint(baseURL, "models") + endpointLabel = "/v1beta/models" + } + req, err := http.NewRequestWithContext(ctx, http.MethodGet, modelsEndpoint, nil) if err != nil { return ProbeUpstreamResult{}, fmt.Errorf("构造测活请求失败: %w", err) } - req.Header.Set("Authorization", "Bearer "+apiKey) + if provider == client.ProviderGoogle { + req.Header.Set("X-Goog-Api-Key", apiKey) + } else { + req.Header.Set("Authorization", "Bearer "+apiKey) + } req.Header.Set("Accept", "application/json") req.Header.Set("User-Agent", client.UserAgent()) @@ -72,9 +87,9 @@ func probeUpstream(parent context.Context, opts ProbeUpstreamOptions) (ProbeUpst summary = readErr.Error() } if summary != "" { - return ProbeUpstreamResult{}, fmt.Errorf("上游 /v1/models 返回 %d: %s", resp.StatusCode, summary) + return ProbeUpstreamResult{}, fmt.Errorf("上游 %s 返回 %d: %s", endpointLabel, resp.StatusCode, summary) } - return ProbeUpstreamResult{}, fmt.Errorf("上游 /v1/models 返回 %d", resp.StatusCode) + return ProbeUpstreamResult{}, fmt.Errorf("上游 %s 返回 %d", endpointLabel, resp.StatusCode) } if readErr != nil { return ProbeUpstreamResult{}, fmt.Errorf("读取上游响应失败: %w", readErr) @@ -82,10 +97,18 @@ func probeUpstream(parent context.Context, opts ProbeUpstreamOptions) (ProbeUpst var parsed modelsListProbeResponse if err := json.Unmarshal(body, &parsed); err != nil { - return ProbeUpstreamResult{}, fmt.Errorf("上游 /v1/models 返回的 JSON 无效: %w", err) - } - if parsed.Data == nil { - return ProbeUpstreamResult{}, fmt.Errorf("上游 /v1/models 响应缺少 data 数组") + return ProbeUpstreamResult{}, fmt.Errorf("上游 %s 返回的 JSON 无效: %w", endpointLabel, err) + } + if provider == client.ProviderGoogle { + for _, item := range parsed.Models { + id := strings.TrimPrefix(strings.TrimSpace(item.Name), "models/") + if id == "" { + continue + } + parsed.Data = append(parsed.Data, upstreamModelDescriptorJSON{ID: id, Object: "model", OwnedBy: "google", DisplayName: item.DisplayName}) + } + } else if parsed.Data == nil { + return ProbeUpstreamResult{}, fmt.Errorf("上游 %s 响应缺少 data 数组", endpointLabel) } models := make([]UpstreamModelDescriptor, 0, len(parsed.Data)) for _, item := range parsed.Data { @@ -104,7 +127,7 @@ func probeUpstream(parent context.Context, opts ProbeUpstreamOptions) (ProbeUpst ModelCount: len(parsed.Data), Models: models, } - if strings.TrimSpace(opts.APIMode) == string(client.APIModeResponses) && + if provider == client.ProviderOpenAI && strings.TrimSpace(opts.APIMode) == string(client.APIModeResponses) && client.NormalizeProxyTransportValue(strings.TrimSpace(opts.ResponsesTransport)) == string(client.ResponsesTransportWebSocket) { result.ResponsesTransport = string(client.ResponsesTransportWebSocket) if wsErr := client.ProbeResponsesWebSocket(ctx, client.ProbeResponsesWebSocketOptions{ diff --git a/image-studio/backend/probe_test.go b/image-studio/backend/probe_test.go index 0bd0f1e6f..cf5835e0e 100644 --- a/image-studio/backend/probe_test.go +++ b/image-studio/backend/probe_test.go @@ -49,6 +49,37 @@ func TestProbeUpstreamRequestsModelsFromBackend(t *testing.T) { } } +func TestProbeUpstreamGoogleProviderUsesNativeModelsEndpoint(t *testing.T) { + var gotPath string + var gotGoogleKey string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotPath = r.URL.Path + gotGoogleKey = r.Header.Get("X-Goog-Api-Key") + if auth := r.Header.Get("Authorization"); auth != "" { + t.Fatalf("Google probe must not send Authorization: %q", auth) + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"models":[{"name":"models/gemini-3.1-flash-image","displayName":"Nano Banana 2"}]}`)) + })) + defer srv.Close() + + got, err := probeUpstream(context.Background(), ProbeUpstreamOptions{ + APIKey: "google-key", + BaseURL: srv.URL, + Provider: "google", + APIMode: "images", + }) + if err != nil { + t.Fatalf("Google probe returned error: %v", err) + } + if gotPath != "/v1beta/models" || gotGoogleKey != "google-key" { + t.Fatalf("path=%q X-Goog-Api-Key=%q", gotPath, gotGoogleKey) + } + if got.ModelCount != 1 || len(got.Models) != 1 || got.Models[0].ID != "gemini-3.1-flash-image" || got.Models[0].OwnedBy != "google" { + t.Fatalf("unexpected Google models: %+v", got) + } +} + func TestProbeUpstreamSkipsModelItemsWithoutID(t *testing.T) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") diff --git a/image-studio/backend/service.go b/image-studio/backend/service.go index 58716a67a..562a66e3c 100644 --- a/image-studio/backend/service.go +++ b/image-studio/backend/service.go @@ -380,6 +380,7 @@ func (s *Service) runJob(ctx context.Context, jobID string, opts GenerateOptions TextModelID: opts.TextModelID, ImageModelID: opts.ImageModelID, ReasoningEffort: opts.ReasoningEffort, + Provider: client.Provider(strings.TrimSpace(opts.Provider)), Proxy: client.ProxyConfig{Mode: opts.ProxyMode, URL: opts.ProxyURL}, APIMode: apiMode, ResponsesTransport: client.ResponsesTransport(strings.TrimSpace(opts.ResponsesTransport)), @@ -420,6 +421,7 @@ func (s *Service) runJob(ctx context.Context, jobID string, opts GenerateOptions TextModelID: strings.TrimSpace(opts.FallbackProfile.TextModelID), ImageModelID: strings.TrimSpace(opts.FallbackProfile.ImageModelID), ReasoningEffort: strings.TrimSpace(opts.FallbackProfile.ReasoningEffort), + Provider: client.Provider(strings.TrimSpace(opts.FallbackProfile.Provider)), Proxy: client.ProxyConfig{Mode: opts.ProxyMode, URL: opts.ProxyURL}, APIMode: fallbackAPIMode, ResponsesTransport: client.ResponsesTransport(strings.TrimSpace(opts.FallbackProfile.ResponsesTransport)), diff --git a/image-studio/backend/types.go b/image-studio/backend/types.go index 6ebca49a9..ffb68e96b 100644 --- a/image-studio/backend/types.go +++ b/image-studio/backend/types.go @@ -41,7 +41,8 @@ type GenerateOptions struct { TextModelID string `json:"textModelID"` // overrides the default text model ImageModelID string `json:"imageModelID"` // overrides the default image model ReasoningEffort string `json:"reasoningEffort"` - APIMode string `json:"apiMode"` // "responses" (default) | "images" + Provider string `json:"provider,omitempty"` // "openai" (default) | "google" | "grok" + APIMode string `json:"apiMode"` // "responses" (default) | "images" ResponsesTransport string `json:"responsesTransport,omitempty"` RequestPolicy string `json:"requestPolicy"` // "openai" (default) | "compat" // ImagesNewAPICompat 开启后仅影响 Images API 请求: @@ -77,6 +78,7 @@ type FallbackProfileOptions struct { TextModelID string `json:"textModelID"` ImageModelID string `json:"imageModelID"` ReasoningEffort string `json:"reasoningEffort"` + Provider string `json:"provider,omitempty"` APIMode string `json:"apiMode"` ResponsesTransport string `json:"responsesTransport,omitempty"` RequestPolicy string `json:"requestPolicy"` @@ -106,6 +108,7 @@ type ProbeUpstreamOptions struct { BaseURL string `json:"baseURL"` ProxyMode string `json:"proxyMode"` ProxyURL string `json:"proxyURL"` + Provider string `json:"provider,omitempty"` APIMode string `json:"apiMode,omitempty"` ResponsesTransport string `json:"responsesTransport,omitempty"` AllowInsecureConnection bool `json:"allowInsecureConnection,omitempty"` diff --git a/image-studio/frontend/src/app/dev/previewData.ts b/image-studio/frontend/src/app/dev/previewData.ts index f74ff84ca..a10e5cd1a 100644 --- a/image-studio/frontend/src/app/dev/previewData.ts +++ b/image-studio/frontend/src/app/dev/previewData.ts @@ -166,6 +166,7 @@ function buildPreviewProfile(now: number): UpstreamProfile { return { id: "preview-profile", name: "Preview Responses", + provider: "openai", apiMode: "responses", responsesTransport: "sse", requestPolicy: "openai", diff --git a/image-studio/frontend/src/components/panel/UpstreamConfigModal.tsx b/image-studio/frontend/src/components/panel/UpstreamConfigModal.tsx index 982e5ec61..57947aa35 100644 --- a/image-studio/frontend/src/components/panel/UpstreamConfigModal.tsx +++ b/image-studio/frontend/src/components/panel/UpstreamConfigModal.tsx @@ -12,7 +12,7 @@ import { probeCurrentUpstream, } from "../../platform/runtime/host"; import { genProfileId, keyringUserFor } from "../../lib/profiles"; -import type { APIMode, RequestPolicy, UpstreamProfile } from "../../types/domain"; +import type { APIMode, RequestPolicy, UpstreamProfile, UpstreamProvider } from "../../types/domain"; import { FAQModal } from "./FAQModal"; import { UpstreamProfileEditor } from "./UpstreamProfileEditor"; import { UpstreamProfileList } from "./UpstreamProfileList"; @@ -102,7 +102,13 @@ export function UpstreamConfigModal({ function patchDraft(patch: Partial) { if (!draft) return; - setDraft({ ...draft, ...patch }); + const provider = patch.provider ?? draft.provider ?? "openai"; + setDraft({ + ...draft, + ...patch, + provider, + apiMode: provider === "openai" ? patch.apiMode ?? draft.apiMode : "images", + }); } function codexProfileName(provider: string) { @@ -122,6 +128,7 @@ export function UpstreamConfigModal({ if (existing) { await updateProfile(existing.id, { name, + provider: "openai", apiMode: "responses", requestPolicy: "openai", imagesNewAPICompat: false, @@ -137,6 +144,7 @@ export function UpstreamConfigModal({ } else { syncedId = await createProfile({ name, + provider: "openai", apiMode: "responses", requestPolicy: "openai", baseURL: imported.baseURL, @@ -225,8 +233,9 @@ export function UpstreamConfigModal({ } } - async function handleNew(apiMode: APIMode = "responses") { + async function handleNew(provider: UpstreamProvider = "openai", apiMode: APIMode = provider === "openai" ? "responses" : "images") { const id = await createProfile({ + provider, apiMode, requestPolicy: "openai", setActive: profiles.length === 0, // 第一个自动 active,后续手动切 @@ -254,6 +263,7 @@ export function UpstreamConfigModal({ if (!draft) return; await updateProfile(draft.id, { name: draft.name, + provider: draft.provider, apiMode: draft.apiMode, responsesTransport: draft.responsesTransport ?? "sse", requestPolicy: draft.requestPolicy, @@ -326,6 +336,8 @@ export function UpstreamConfigModal({ draft.apiMode, draft.responsesTransport ?? "sse", draft.allowInsecureConnection === true, + undefined, + draft.provider, ); const catalog = buildUpstreamModelCatalog(result.models ?? []); setModelCatalog(catalog); @@ -405,16 +417,22 @@ export function UpstreamConfigModal({ {([ { - id: "responses" as APIMode, - title: "Responses API", - sub: "首选。支持 SSE 保活，长任务更稳。", - note: "适合 GPT 图像链路和提示词优化。", + id: "openai" as UpstreamProvider, + title: "OpenAI / 兼容中转", + sub: "支持 Responses 与 Images API。", + note: "适合 OpenAI、NewAPI 和兼容中转。", + }, + { + id: "google" as UpstreamProvider, + title: "Google Gemini", + sub: "原生 /v1beta/interactions。", + note: "使用 x-goog-api-key 与 Google payload。", }, { - id: "images" as APIMode, - title: "Images API", - sub: "兼容性更广，接标准 generations / edits。", - note: "适合只想尽快接上常规生图接口。", + id: "grok" as UpstreamProvider, + title: "Grok / xAI", + sub: "原生 Imagine generations / edits。", + note: "图生图使用 JSON，不使用 multipart。", }, ]).map((item) => ( - 通过宿主侧请求 /v1/models 获取模型列表，避免浏览器跨域或 WebView 差异影响结果。 + 通过宿主侧请求 {draft.provider === "google" ? /v1beta/models : /v1/models} 获取模型列表，避免浏览器跨域或 WebView 差异影响结果。 {modelCatalogError ? {modelCatalogError} : null} - {draft.apiMode === "responses" ? ( + {draft.provider === "openai" && draft.apiMode === "responses" ? ( <> onPatchDraft({ imageModelID: e.target.value })} spellCheck={false} className={`focus-ring w-full min-w-0 border border-black/[0.08] bg-[var(--surface)] px-3 py-2 text-sm text-zinc-900 placeholder:text-zinc-400 dark:border-white/[0.08] dark:text-zinc-100 dark:placeholder:text-zinc-500 font-mono-token ${usesFluentUI ? "rounded-[10px]" : "rounded-[14px]"}`} @@ -379,7 +414,7 @@ export function UpstreamProfileEditor({ {fallbackCandidates.map((profile) => ( ))} @@ -388,7 +423,7 @@ export function UpstreamProfileEditor({ - {draft.apiMode === "images" ? ( + {draft.provider === "openai" && draft.apiMode === "images" ? (
    ) : null}
    diff --git a/image-studio/frontend/src/components/panel/UpstreamProfileList.tsx b/image-studio/frontend/src/components/panel/UpstreamProfileList.tsx index 03b940bc0..c3a3e1cd3 100644 --- a/image-studio/frontend/src/components/panel/UpstreamProfileList.tsx +++ b/image-studio/frontend/src/components/panel/UpstreamProfileList.tsx @@ -75,7 +75,7 @@ export function UpstreamProfileList({ {isActive ? 生图 : null} {isAI ? AI : null} - {p.apiMode === "responses" ? "R" : "I"} + {(p.provider ?? "openai") === "openai" ? (p.apiMode === "responses" ? "R" : "I") : p.provider === "google" ? "G" : "X"} ); diff --git a/image-studio/frontend/src/lib/profiles.ts b/image-studio/frontend/src/lib/profiles.ts index 5cd5e2bd0..376b74299 100644 --- a/image-studio/frontend/src/lib/profiles.ts +++ b/image-studio/frontend/src/lib/profiles.ts @@ -1,4 +1,4 @@ -import type { APIMode, ReasoningEffortValue, RequestPolicy, ResponsesTransport, UpstreamProfile } from "../types/domain"; +import type { APIMode, ReasoningEffortValue, RequestPolicy, ResponsesTransport, UpstreamProfile, UpstreamProvider } from "../types/domain"; function normalizeReasoningEffort(value: unknown): ReasoningEffortValue { return value === "low" || value === "medium" || value === "high" || value === "xhigh" @@ -10,6 +10,17 @@ export function normalizeResponsesTransport(value: unknown): ResponsesTransport return value === "websocket" ? "websocket" : "sse"; } +export function normalizeUpstreamProvider(value: unknown): UpstreamProvider { + if (value === "google" || value === "grok") return value; + return "openai"; +} + +export function upstreamProviderLabel(provider: UpstreamProvider): string { + if (provider === "google") return "Google Gemini"; + if (provider === "grok") return "Grok / xAI"; + return "OpenAI / 兼容中转"; +} + // localStorage 键名规范: // gptcodex.profiles —— UpstreamProfile[] JSON(无 apiKey,key 在 keyring) // gptcodex.activeProfileId —— 当前 active profile 的 id @@ -57,7 +68,8 @@ export function tryParseProfile(raw: unknown): UpstreamProfile | null { const o = raw as Record; const id = typeof o.id === "string" ? o.id : ""; const name = typeof o.name === "string" ? o.name : ""; - const apiMode = o.apiMode === "images" ? "images" : "responses"; + const provider = normalizeUpstreamProvider(o.provider); + const apiMode = provider !== "openai" || o.apiMode === "images" ? "images" : "responses"; const responsesTransport = normalizeResponsesTransport(o.responsesTransport); const requestPolicy = o.requestPolicy === "compat" ? "compat" : "openai"; const imagesNewAPICompat = o.imagesNewAPICompat === true; @@ -75,6 +87,7 @@ export function tryParseProfile(raw: unknown): UpstreamProfile | null { return { id, name, + provider, apiMode, responsesTransport, requestPolicy, @@ -145,6 +158,7 @@ export function makeBlankProfile(apiMode: APIMode = "responses", profiles: Upstr return { id: genProfileId(), name: nextDefaultProfileName(profiles), + provider: "openai", apiMode, responsesTransport: "sse", requestPolicy: "openai", diff --git a/image-studio/frontend/src/lib/upstreamConfigTransfer.ts b/image-studio/frontend/src/lib/upstreamConfigTransfer.ts index d020cbcd2..dc85eece4 100644 --- a/image-studio/frontend/src/lib/upstreamConfigTransfer.ts +++ b/image-studio/frontend/src/lib/upstreamConfigTransfer.ts @@ -1,10 +1,12 @@ -import type { APIMode, RequestPolicy, ReasoningEffortValue, ResponsesTransport, UpstreamProfile } from "../types/domain"; +import type { APIMode, RequestPolicy, ReasoningEffortValue, ResponsesTransport, UpstreamProfile, UpstreamProvider } from "../types/domain"; +import { normalizeUpstreamProvider } from "./profiles.ts"; import { buildUpstreamModelCatalog } from "./upstreamModels.ts"; import { cleanBaseURL } from "./security.ts"; export type UpstreamConfigExportProfile = { id: string; name: string; + provider?: UpstreamProvider; apiMode: APIMode; responsesTransport: ResponsesTransport; requestPolicy: RequestPolicy; @@ -39,6 +41,7 @@ export type UpstreamConfigImportActions = { getProfiles: () => UpstreamProfile[]; createProfile: (input: { name?: string; + provider?: UpstreamProvider; apiMode: APIMode; responsesTransport?: ResponsesTransport; baseURL?: string; @@ -71,6 +74,7 @@ function toProfileSnapshot(input: UpstreamConfigExportProfile, actualId: string) return { id: actualId, name: input.name, + provider: input.provider ?? "openai", apiMode: input.apiMode, responsesTransport: input.responsesTransport ?? "sse", requestPolicy: input.requestPolicy, @@ -152,10 +156,12 @@ function parseExportProfile(raw: unknown): UpstreamConfigExportProfile | null { const id = typeof source.id === "string" ? source.id.trim() : ""; const name = typeof source.name === "string" ? source.name.trim() : ""; if (!id || !name) return null; + const provider = normalizeUpstreamProvider(source.provider); return { id, name, - apiMode: normalizeAPIMode(source.apiMode), + ...(source.provider !== undefined ? { provider } : {}), + apiMode: provider === "openai" ? normalizeAPIMode(source.apiMode) : "images", responsesTransport: normalizeResponsesTransport(source.responsesTransport), requestPolicy: normalizeRequestPolicy(source.requestPolicy), imagesNewAPICompat: source.imagesNewAPICompat === true, @@ -328,6 +334,7 @@ function buildProfilePatch( ): Partial> & { apiKey?: string } { return { name: incoming.name, + provider: incoming.provider ?? "openai", apiMode: incoming.apiMode, responsesTransport: incoming.responsesTransport ?? "sse", requestPolicy: incoming.requestPolicy, @@ -366,6 +373,7 @@ export async function applyParsedUpstreamConfigImport( const newId = await actions.createProfile({ name: incoming.name, + provider: incoming.provider ?? "openai", apiMode: incoming.apiMode, responsesTransport: incoming.responsesTransport ?? "sse", requestPolicy: incoming.requestPolicy, diff --git a/image-studio/frontend/src/platform/android/upstream/AndroidUpstreamEmptyState.tsx b/image-studio/frontend/src/platform/android/upstream/AndroidUpstreamEmptyState.tsx index 02b96af04..4163e515a 100644 --- a/image-studio/frontend/src/platform/android/upstream/AndroidUpstreamEmptyState.tsx +++ b/image-studio/frontend/src/platform/android/upstream/AndroidUpstreamEmptyState.tsx @@ -1,12 +1,12 @@ import { Boxes, ClipboardPaste, Plus, RadioTower } from "lucide-react"; -import type { APIMode } from "../../../types/domain"; -import { ANDROID_API_MODE_OPTIONS } from "./useAndroidUpstreamConfig"; +import type { UpstreamProvider } from "../../../types/domain"; +import { ANDROID_PROVIDER_OPTIONS } from "./useAndroidUpstreamConfig"; export function AndroidUpstreamEmptyState({ onCreate, onQuickImport, }: { - onCreate: (apiMode: APIMode) => void | Promise; + onCreate: (provider: UpstreamProvider) => void | Promise; onQuickImport: () => void | Promise; }) { return ( @@ -29,10 +29,10 @@ export function AndroidUpstreamEmptyState({ - {ANDROID_API_MODE_OPTIONS.map((option) => ( + {ANDROID_PROVIDER_OPTIONS.map((option) => ( + ))} + + + + {draft.provider === "openai" ?
    {ANDROID_API_MODE_OPTIONS.map((option) => (
    -     + : null} - + {draft.provider === "openai" ?
    {ANDROID_REQUEST_POLICY_OPTIONS.map((option) => (
    -     +     : null} - + {modelCatalogError}

    : null}     - {draft.apiMode === "responses" ? ( + {draft.provider === "openai" && draft.apiMode === "responses" ? ( <>
    @@ -232,7 +253,7 @@ export function AndroidUpstreamProfileForm({ type="text" value={draft.imageModelID} onChange={(event) => onPatchDraft({ imageModelID: event.target.value })} - placeholder="留空 = 默认 gpt-image-2" + placeholder={draft.provider === "google" ? "例如 gemini-3.1-flash-image" : draft.provider === "grok" ? "例如 grok-imagine-image" : "留空 = 默认 gpt-image-2"} className="focus-ring android-upstream-input font-mono-token" spellCheck={false} /> @@ -274,7 +295,7 @@ export function AndroidUpstreamProfileForm({
    - {draft.apiMode === "images" ? ( + {draft.provider === "openai" && draft.apiMode === "images" ? ( {profile.name || "未命名配置"} - {profile.apiMode === "responses" ? "Responses" : "Images"} + {upstreamProviderLabel(profile.provider ?? "openai")} {profile.baseURL ? ` · ${profile.baseURL}` : " · 未填写地址"} diff --git a/image-studio/frontend/src/platform/android/upstream/useAndroidUpstreamConfig.ts b/image-studio/frontend/src/platform/android/upstream/useAndroidUpstreamConfig.ts index 64924792e..02ab1a6fe 100644 --- a/image-studio/frontend/src/platform/android/upstream/useAndroidUpstreamConfig.ts +++ b/image-studio/frontend/src/platform/android/upstream/useAndroidUpstreamConfig.ts @@ -2,7 +2,7 @@ import { useEffect, useMemo, useState } from "react"; import { GetStoredAPIKey, probeCurrentUpstream } from "../../runtime/host"; import { keyringUserFor } from "../../../lib/profiles"; import { useStudioStore } from "../../../state/studioStore"; -import type { APIMode, ReasoningEffortValue, RequestPolicy, UpstreamProfile } from "../../../types/domain"; +import type { APIMode, ReasoningEffortValue, RequestPolicy, UpstreamProfile, UpstreamProvider } from "../../../types/domain"; import { buildUpstreamModelCatalog, type UpstreamModelCatalog } from "../../../lib/upstreamModels"; import { applyParsedUpstreamConfigImport, @@ -18,6 +18,16 @@ export const ANDROID_API_MODE_OPTIONS: Array<{ { id: "images", title: "Images", meta: "标准图像接口" }, ]; +export const ANDROID_PROVIDER_OPTIONS: Array<{ + id: UpstreamProvider; + title: string; + meta: string; +}> = [ + { id: "openai", title: "OpenAI / 兼容", meta: "Responses 或 Images" }, + { id: "google", title: "Google Gemini", meta: "Interactions 原生协议" }, + { id: "grok", title: "Grok / xAI", meta: "Imagine 原生协议" }, +]; + export const ANDROID_REQUEST_POLICY_OPTIONS: Array<{ id: RequestPolicy; title: string; @@ -122,12 +132,22 @@ export function useAndroidUpstreamConfig(open: boolean) { && !saving; function patchDraft(patch: Partial) { - setDraft((current) => (current ? { ...current, ...patch } : current)); + setDraft((current) => { + if (!current) return current; + const provider = patch.provider ?? current.provider ?? "openai"; + return { + ...current, + ...patch, + provider, + apiMode: provider === "openai" ? patch.apiMode ?? current.apiMode : "images", + }; + }); } - async function handleNew(apiMode: APIMode = "responses") { + async function handleNew(provider: UpstreamProvider = "openai") { const id = await createProfile({ - apiMode, + provider, + apiMode: provider === "openai" ? "responses" : "images", requestPolicy: "openai", setActive: profiles.length === 0, }); @@ -158,6 +178,7 @@ export function useAndroidUpstreamConfig(open: boolean) { try { const ok = await updateProfile(draft.id, { name: draft.name, + provider: draft.provider, apiMode: draft.apiMode, responsesTransport: draft.responsesTransport ?? "sse", requestPolicy: draft.requestPolicy, @@ -227,6 +248,8 @@ export function useAndroidUpstreamConfig(open: boolean) { draft.apiMode, draft.responsesTransport ?? "sse", draft.allowInsecureConnection === true, + undefined, + draft.provider, ); const catalog = buildUpstreamModelCatalog(result.models ?? []); setModelCatalog(catalog); diff --git a/image-studio/frontend/src/platform/runtime/host.ts b/image-studio/frontend/src/platform/runtime/host.ts index 9cccad5a8..683b0f07c 100644 --- a/image-studio/frontend/src/platform/runtime/host.ts +++ b/image-studio/frontend/src/platform/runtime/host.ts @@ -786,6 +786,7 @@ export async function probeCurrentUpstream( responsesTransport = "sse", allowInsecureConnection = false, signal?: AbortSignal, + provider: ProbeUpstreamOptionsLike["provider"] = "openai", ): Promise { if (signal?.aborted) throw new DOMException("Aborted", "AbortError"); const options: ProbeUpstreamOptionsLike = { @@ -795,6 +796,7 @@ export async function probeCurrentUpstream( proxyURL, apiMode, responsesTransport, + ...(provider !== "openai" ? { provider } : {}), ...(allowInsecureConnection ? { allowInsecureConnection: true } : {}), }; if (hasServiceMethod("ProbeUpstream")) { diff --git a/image-studio/frontend/src/platform/runtime/hostTypes.ts b/image-studio/frontend/src/platform/runtime/hostTypes.ts index 02d372a0b..6bed14f80 100644 --- a/image-studio/frontend/src/platform/runtime/hostTypes.ts +++ b/image-studio/frontend/src/platform/runtime/hostTypes.ts @@ -1,4 +1,4 @@ -import type { RequestPolicy } from "../../types/domain"; +import type { RequestPolicy, UpstreamProvider } from "../../types/domain"; export type GenerateOptionsLike = { apiKey: string; @@ -22,6 +22,7 @@ export type GenerateOptionsLike = { textModelID: string; imageModelID: string; reasoningEffort?: string; + provider?: UpstreamProvider; proxyMode?: string; proxyURL?: string; apiMode: string; @@ -38,6 +39,7 @@ export type GenerateOptionsLike = { textModelID: string; imageModelID: string; reasoningEffort?: string; + provider?: UpstreamProvider; apiMode: string; responsesTransport?: string; requestPolicy: RequestPolicy; @@ -83,6 +85,7 @@ export type ProbeUpstreamOptionsLike = { proxyMode?: string; proxyURL?: string; apiMode?: string; + provider?: UpstreamProvider; responsesTransport?: string; allowInsecureConnection?: boolean; }; diff --git a/image-studio/frontend/src/platform/runtime/remote-kernel/common.ts b/image-studio/frontend/src/platform/runtime/remote-kernel/common.ts index cd1b5a3c2..2a96ced66 100644 --- a/image-studio/frontend/src/platform/runtime/remote-kernel/common.ts +++ b/image-studio/frontend/src/platform/runtime/remote-kernel/common.ts @@ -44,17 +44,24 @@ export function normalizeBaseURL(raw: string): string { export function validateRemoteBaseURL(raw: string, allowInsecureConnection: boolean): string { const normalized = normalizeBaseURL(raw); - let parsed: URL; + let protocol = ""; + let hostname = ""; try { - parsed = new URL(normalized); + if (typeof URL !== "function") throw new TypeError("URL constructor unavailable"); + const parsed = new URL(normalized); + protocol = parsed.protocol; + hostname = parsed.hostname; } catch { - throw new Error("BASE_URL 必须包含协议和主机，例如 https://example.com"); + const fallback = /^(https?):\/\/(\[[^\]]+\]|[^/?#:]+)(?::\d+)?(?:[/?#]|$)/i.exec(normalized); + if (!fallback) throw new Error("BASE_URL 必须包含协议和主机，例如 https://example.com"); + protocol = `${fallback[1].toLowerCase()}:`; + hostname = fallback[2]; } - if (parsed.protocol === "https:") return normalized; - if (parsed.protocol !== "http:") { + if (protocol === "https:") return normalized; + if (protocol !== "http:") { throw new Error("BASE_URL 仅支持 http:// 或 https://"); } - const host = parsed.hostname.toLowerCase().replace(/^\[|\]$/g, ""); + const host = hostname.toLowerCase().replace(/^\[|\]$/g, ""); const loopback = host === "localhost" || host.endsWith(".localhost") || host === "127.0.0.1" diff --git a/image-studio/frontend/src/platform/runtime/remote-kernel/images.ts b/image-studio/frontend/src/platform/runtime/remote-kernel/images.ts index ba57b1aa1..a12193a9a 100644 --- a/image-studio/frontend/src/platform/runtime/remote-kernel/images.ts +++ b/image-studio/frontend/src/platform/runtime/remote-kernel/images.ts @@ -325,7 +325,7 @@ export async function requestImagesOnce( const built = await buildImagesRequestBody(request, sourceDataURLs); const allowEmptyKeepAlive = request.payload.imagesNewAPICompat === true; const startedAt = Date.now(); - const protocolLabel = built.protocol === "google-interactions" ? "Google Interactions" : "Images API"; + const protocolLabel = built.protocol === "google-interactions" ? "Google Interactions" : built.protocol === "grok-images" ? "Grok Imagine" : "Images API"; callbacks.onLog?.(`[${protocolLabel}] 第 ${attempt}/${maxAttempts} 次请求...`); callbacks.onProgress?.(`等待 ${protocolLabel} 返回(无 SSE 保活)`, 0, 0); const ticker = globalThis.setInterval(() => { @@ -351,7 +351,7 @@ export async function requestImagesOnce( built.url, "POST", { - ...(built.protocol === "openai-images" ? { Authorization: `Bearer ${request.payload.apiKey}` } : {}), + ...(built.protocol !== "google-interactions" ? { Authorization: `Bearer ${request.payload.apiKey}` } : {}), Accept: "text/event-stream, application/json", ...(built.headers ?? {}), }, @@ -411,7 +411,7 @@ export async function requestImagesOnce( const response = await fetch(built.url, { method: "POST", headers: { - ...(built.protocol === "openai-images" ? { Authorization: `Bearer ${request.payload.apiKey}` } : {}), + ...(built.protocol !== "google-interactions" ? { Authorization: `Bearer ${request.payload.apiKey}` } : {}), Accept: "text/event-stream, application/json", ...(built.headers ?? {}), }, diff --git a/image-studio/frontend/src/platform/runtime/remote-kernel/index.ts b/image-studio/frontend/src/platform/runtime/remote-kernel/index.ts index f15875e8a..e5fec6e49 100644 --- a/image-studio/frontend/src/platform/runtime/remote-kernel/index.ts +++ b/image-studio/frontend/src/platform/runtime/remote-kernel/index.ts @@ -3,10 +3,10 @@ import { MAX_AUTO_RETRY_COUNT, MAX_ATTEMPTS, buildPromptOptimizePayload, + effectiveAPIMode, extractInvalidSize, fileNameFromPath, isRetryableRaw, - normalizeAPIMode, normalizeAutoRetryCount, normalizeBaseURL, repairSizeForOpenAI, @@ -54,6 +54,7 @@ export async function runRemoteImageJob( textModelID: request.payload.fallbackProfile.textModelID || request.payload.textModelID, imageModelID: request.payload.fallbackProfile.imageModelID || request.payload.imageModelID, reasoningEffort: request.payload.fallbackProfile.reasoningEffort || request.payload.reasoningEffort, + provider: request.payload.fallbackProfile.provider || request.payload.provider, apiMode: request.payload.fallbackProfile.apiMode || request.payload.apiMode, responsesTransport: request.payload.fallbackProfile.responsesTransport || request.payload.responsesTransport, requestPolicy: request.payload.fallbackProfile.requestPolicy || request.payload.requestPolicy, @@ -77,7 +78,7 @@ export async function runRemoteImageJob( if (activeRequest.payload.allowInsecureConnection && !shouldUseAndroidNativeHTTP()) { throw new RemoteKernelError("允许不安全连接需要桌面本地内核或 Android 原生内核；浏览器不能绕过 HTTPS 证书校验"); } - const apiMode = normalizeAPIMode(activeRequest.payload.apiMode); + const apiMode = effectiveAPIMode(activeRequest.payload.provider || "openai", activeRequest.payload.apiMode); if (apiMode === "images") { return await requestImagesOnce(activeRequest, attempt, maxAttempts, callbacks); } diff --git a/image-studio/frontend/src/platform/runtime/remote-kernel/requestPayloads.ts b/image-studio/frontend/src/platform/runtime/remote-kernel/requestPayloads.ts index bab0f800a..558441868 100644 --- a/image-studio/frontend/src/platform/runtime/remote-kernel/requestPayloads.ts +++ b/image-studio/frontend/src/platform/runtime/remote-kernel/requestPayloads.ts @@ -7,6 +7,7 @@ import { normalizeBackground, normalizeImageStyle, normalizeInputFidelity, + normalizeUpstreamProvider, normalizeOutputCompression, normalizePartialImages, normalizeModeration, @@ -27,12 +28,32 @@ import { import { normalizeBaseURL, normalizeImageModel } from "./common.ts"; import { RemoteKernelError, type RemoteGeneratePayload, type RemoteJobRequest } from "./types.ts"; -export type ImagesRequestProtocol = "openai-images" | "google-interactions"; +export type ImagesRequestProtocol = "openai-images" | "google-interactions" | "grok-images"; const MAX_MASK_IMAGE_BYTES = 50 * 1024 * 1024; const GOOGLE_INTERACTION_ASPECT_RATIOS = [ "1:8", "1:4", "2:3", "3:4", "4:5", "1:1", "5:4", "4:3", "3:2", "16:9", "21:9", "4:1", "8:1", ] as const; +const GROK_ASPECT_RATIOS = [ + "1:1", "16:9", "9:16", "4:3", "3:4", "3:2", "2:3", "2:1", "1:2", "19.5:9", "9:19.5", "20:9", "9:20", +] as const; + +function grokImageDimensions(size: string): { aspect_ratio: string; resolution: string } | null { + const matched = /^(\d+)x(\d+)$/i.exec(size.trim()); + if (!matched) return null; + const width = Number(matched[1]); + const height = Number(matched[2]); + if (!Number.isFinite(width) || !Number.isFinite(height) || width <= 0 || height <= 0) return null; + const targetRatio = width / height; + const aspectRatio = GROK_ASPECT_RATIOS.reduce((best, candidate) => { + const [left, right] = candidate.split(":").map(Number); + const [bestLeft, bestRight] = best.split(":").map(Number); + return Math.abs(Math.log((left / right) / targetRatio)) < Math.abs(Math.log((bestLeft / bestRight) / targetRatio)) + ? candidate + : best; + }, "1:1" as (typeof GROK_ASPECT_RATIOS)[number]); + return { aspect_ratio: aspectRatio, resolution: Math.max(width, height) >= 1536 ? "2k" : "1k" }; +} function googleInteractionResponseFormat(size: string, outputFormat: string): Record { const format: Record = { type: "image", delivery: "inline" }; @@ -102,8 +123,9 @@ export async function buildImagesRequestBody( const includeExtended = shouldSendExtendedImageParameters(request.payload.requestPolicy); const partialImages = request.payload.disablePreview ? 0 : normalizePartialImages(request.payload.partialImages); const useNewAPICompat = shouldUseImagesNewAPICompat(request.payload); + const provider = normalizeUpstreamProvider(request.payload.provider || "openai"); - if (shouldUseGoogleNativeInteractions(baseURL, imageModel)) { + if (shouldUseGoogleNativeInteractions(baseURL, imageModel, provider)) { if (mode === "edit" && sourceDataURLs.length === 0) { throw new RemoteKernelError("Google Interactions 图生图需要至少一张源图"); } @@ -126,6 +148,32 @@ export async function buildImagesRequestBody( }; } + if (provider === "grok") { + if (mode === "edit" && sourceDataURLs.length === 0) { + throw new RemoteKernelError("Grok Imagine 图生图需要至少一张源图"); + } + if (request.payload.maskB64) { + throw new RemoteKernelError("Grok Imagine 不支持 OpenAI mask 参数；请清除蒙版后重试"); + } + const dimensions = grokImageDimensions(size); + const payload: Record = { + model: imageModel, + prompt: request.payload.prompt, + response_format: "b64_json", + ...(dimensions ?? {}), + }; + if (mode === "edit") { + const images = sourceDataURLs.map((url) => ({ type: "image_url", url })); + payload.image = images.length === 1 ? images[0] : images; + } + return { + url: openAIAPIEndpoint(baseURL, mode === "edit" ? "images/edits" : "images/generations"), + protocol: "grok-images", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(payload), + }; + } + if (mode === "edit") { if (sourceDataURLs.length === 0) { throw new RemoteKernelError("图生图模式需要至少一张源图(请先添加参考图)"); diff --git a/image-studio/frontend/src/platform/runtime/remote-kernel/types.ts b/image-studio/frontend/src/platform/runtime/remote-kernel/types.ts index 47b85ed57..252b8b429 100644 --- a/image-studio/frontend/src/platform/runtime/remote-kernel/types.ts +++ b/image-studio/frontend/src/platform/runtime/remote-kernel/types.ts @@ -1,4 +1,4 @@ -import type { RequestPolicy } from "../../../types/domain"; +import type { RequestPolicy, UpstreamProvider } from "../../../types/domain"; export type KernelImageSource = { path?: string; @@ -30,6 +30,7 @@ export type RemoteGeneratePayload = { textModelID: string; imageModelID: string; reasoningEffort?: string; + provider?: UpstreamProvider; proxyMode?: string; proxyURL?: string; apiMode: string; @@ -47,6 +48,7 @@ export type RemoteGeneratePayload = { textModelID: string; imageModelID: string; reasoningEffort?: string; + provider?: UpstreamProvider; apiMode: string; responsesTransport?: string; requestPolicy: RequestPolicy; diff --git a/image-studio/frontend/src/state/studioStore.profiles.ts b/image-studio/frontend/src/state/studioStore.profiles.ts index f68ce285d..c206848d0 100644 --- a/image-studio/frontend/src/state/studioStore.profiles.ts +++ b/image-studio/frontend/src/state/studioStore.profiles.ts @@ -3,7 +3,7 @@ import { GetStoredAPIKey, SetStoredAPIKey, } from "../platform/runtime/host"; -import type { APIMode, ReasoningEffortValue, RequestPolicy, UpstreamProfile } from "../types/domain"; +import type { APIMode, ReasoningEffortValue, RequestPolicy, UpstreamProfile, UpstreamProvider } from "../types/domain"; import type { StudioState } from "./studioStore.types"; import { duplicateProfile as cloneProfile, @@ -28,6 +28,7 @@ export function createProfileActions(store: StateAdapter) { return { async createProfile(input: { name?: string; + provider?: UpstreamProvider; apiMode: APIMode; responsesTransport?: UpstreamProfile["responsesTransport"]; baseURL?: string; @@ -46,7 +47,8 @@ export function createProfileActions(store: StateAdapter) { const profile: UpstreamProfile = { id, name: input.name?.trim() || nextDefaultProfileName(list), - apiMode: input.apiMode, + provider: input.provider ?? "openai", + apiMode: input.provider && input.provider !== "openai" ? "images" : input.apiMode, responsesTransport: normalizeResponsesTransport(input.apiMode === "responses" ? input.responsesTransport : "sse"), requestPolicy: input.requestPolicy ?? "openai", imagesNewAPICompat: input.imagesNewAPICompat === true, @@ -86,7 +88,10 @@ export function createProfileActions(store: StateAdapter) { const next: UpstreamProfile = { ...current, name: patch.name !== undefined ? patch.name.trim() : current.name, - apiMode: patch.apiMode ?? current.apiMode, + provider: patch.provider ?? current.provider ?? "openai", + apiMode: (patch.provider ?? current.provider ?? "openai") !== "openai" + ? "images" + : patch.apiMode ?? current.apiMode, responsesTransport: patch.responsesTransport !== undefined ? normalizeResponsesTransport(patch.responsesTransport) : normalizeResponsesTransport(current.responsesTransport), @@ -118,6 +123,7 @@ export function createProfileActions(store: StateAdapter) { if (id === store.getState().activeProfileId) { const apiKey = patch.apiKey !== undefined ? patch.apiKey.trim() : store.getState().apiKey; store.setState({ + provider: next.provider, apiMode: next.apiMode, responsesTransport: next.responsesTransport ?? "sse", requestPolicy: next.requestPolicy, @@ -159,6 +165,7 @@ export function createProfileActions(store: StateAdapter) { textModelID: "", imageModelID: "", reasoningEffort: "xhigh", + provider: "openai", apiMode: "responses", responsesTransport: "sse", requestPolicy: "openai", @@ -198,6 +205,7 @@ export function createProfileActions(store: StateAdapter) { store.setState({ profiles: nextProfiles, activeProfileId: id, + provider: profile.provider ?? "openai", apiMode: profile.apiMode, responsesTransport: profile.responsesTransport ?? "sse", requestPolicy: profile.requestPolicy, diff --git a/image-studio/frontend/src/state/studioStore.ts b/image-studio/frontend/src/state/studioStore.ts index c9b05a9f9..c141d635e 100644 --- a/image-studio/frontend/src/state/studioStore.ts +++ b/image-studio/frontend/src/state/studioStore.ts @@ -641,6 +641,7 @@ export const useStudioStore = create((set, get) => ({ textModelID: "", imageModelID: "", reasoningEffort: "xhigh", + provider: "openai", proxyMode: "system", proxyURL: "", apiMode: "responses", @@ -1185,6 +1186,7 @@ export const useStudioStore = create((set, get) => ({ } } const activeProfile = s.profiles.find((p) => p.id === s.activeProfileId); + const provider = activeProfile?.provider ?? s.provider; const responsesTransport = activeProfile?.responsesTransport ?? s.responsesTransport; if (activeProfile?.allowInsecureConnection && s.kernelRuntimeMode === "remote" && !runtimePlatform.isAndroid) { set({ @@ -1373,6 +1375,7 @@ export const useStudioStore = create((set, get) => ({ textModelID: s.textModelID, imageModelID: s.imageModelID, reasoningEffort: s.reasoningEffort, + provider, proxyMode: s.proxyMode, proxyURL: s.proxyURL, responsesTransport: s.responsesTransport, @@ -1391,6 +1394,7 @@ export const useStudioStore = create((set, get) => ({ textModelID: fallbackProfile.textModelID, imageModelID: fallbackProfile.imageModelID, reasoningEffort: fallbackProfile.reasoningEffort, + provider: fallbackProfile.provider ?? "openai", apiMode: fallbackProfile.apiMode, responsesTransport: fallbackProfile.responsesTransport ?? "sse", requestPolicy: fallbackProfile.requestPolicy, @@ -1783,6 +1787,7 @@ export const useStudioStore = create((set, get) => ({ synth.push({ id, name: "Responses · 默认", + provider: "openai", apiMode: "responses", responsesTransport: "sse", requestPolicy: "openai", @@ -1804,6 +1809,7 @@ export const useStudioStore = create((set, get) => ({ synth.push({ id, name: "Images · 默认", + provider: "openai", apiMode: "images", responsesTransport: "sse", requestPolicy: "openai", @@ -1846,7 +1852,8 @@ export const useStudioStore = create((set, get) => ({ aiProfileId = aiProfile?.id ?? ""; persistAIProfileId(aiProfileId); } - const apiMode: APIMode = activeProfile?.apiMode ?? "responses"; + const provider = activeProfile?.provider ?? "openai"; + const apiMode: APIMode = provider === "openai" ? activeProfile?.apiMode ?? "responses" : "images"; const responsesTransport = activeProfile?.responsesTransport ?? "sse"; const requestPolicy: RequestPolicy = activeProfile?.requestPolicy ?? "openai"; const imagesNewAPICompat = activeProfile?.imagesNewAPICompat === true; @@ -1925,7 +1932,7 @@ export const useStudioStore = create((set, get) => ({ historyHasMore, historyLoading: false, historyCursorBeforeDayStart: initialHistoryPage.nextCursor?.beforeDayStart ?? null, - apiMode, responsesTransport, requestPolicy, imagesNewAPICompat, baseURL, textModelID, imageModelID, reasoningEffort, kernelRuntimeMode, noPromptRevision, + provider, apiMode, responsesTransport, requestPolicy, imagesNewAPICompat, baseURL, textModelID, imageModelID, reasoningEffort, kernelRuntimeMode, noPromptRevision, proxyMode: proxyConfig.mode, proxyURL: proxyConfig.url, outputFormat, @@ -2222,6 +2229,8 @@ export const useStudioStore = create((set, get) => ({ s.apiMode, s.responsesTransport, activeProfile?.allowInsecureConnection === true, + undefined, + activeProfile?.provider ?? "openai", ); set({ isTestingKey: false }); if (result.responsesTransport === "websocket" && result.responsesTransportOK === false) { diff --git a/image-studio/frontend/src/state/studioStore.types.ts b/image-studio/frontend/src/state/studioStore.types.ts index ae92fd1f3..73912b814 100644 --- a/image-studio/frontend/src/state/studioStore.types.ts +++ b/image-studio/frontend/src/state/studioStore.types.ts @@ -34,6 +34,7 @@ import type { ThemeMode, Toast, UpstreamProfile, + UpstreamProvider, Workspace, } from "../types/domain"; import type { RunningJobMeta } from "./workspaceRuntime"; @@ -104,6 +105,7 @@ export interface StudioState { proxyURL: string; imageModelID: string; reasoningEffort: import("../types/domain").ReasoningEffortValue; + provider: UpstreamProvider; apiMode: APIMode; responsesTransport: import("../types/domain").ResponsesTransport; requestPolicy: RequestPolicy; @@ -175,6 +177,7 @@ export interface StudioState { clearError: () => void; createProfile: (input: { name?: string; + provider?: UpstreamProvider; apiMode: APIMode; responsesTransport?: import("../types/domain").ResponsesTransport; baseURL?: string; diff --git a/image-studio/frontend/src/types/domain.ts b/image-studio/frontend/src/types/domain.ts index eabd9175f..e1746bec5 100644 --- a/image-studio/frontend/src/types/domain.ts +++ b/image-studio/frontend/src/types/domain.ts @@ -6,6 +6,7 @@ export type Mode = "generate" | "edit"; // 上游 API 形态 —— Responses (`/v1/responses` + SSE) 或标准 Images API。 // 老代码里以前是顶层全局二选一,v0.1.6 起降级成 profile 的字段。 export type APIMode = "responses" | "images"; +export type UpstreamProvider = "openai" | "google" | "grok"; export type RequestPolicy = "openai" | "compat"; export type ReasoningEffortValue = "low" | "medium" | "high" | "xhigh"; export type ResponsesTransport = "sse" | "websocket"; @@ -20,6 +21,8 @@ export type AutoRetryCountValue = number; export interface UpstreamProfile { id: string; name: string; + // 决定实际请求路径、认证头与 payload。旧配置缺省按 OpenAI 兼容协议处理。 + provider: UpstreamProvider; apiMode: APIMode; responsesTransport?: ResponsesTransport; requestPolicy: RequestPolicy; diff --git a/image-studio/frontend/test/remoteKernel.test.mjs b/image-studio/frontend/test/remoteKernel.test.mjs index e06f6012e..ec6cc7a15 100644 --- a/image-studio/frontend/test/remoteKernel.test.mjs +++ b/image-studio/frontend/test/remoteKernel.test.mjs @@ -319,6 +319,56 @@ test("runRemoteImageJob emits Responses API partial image previews", async () => }); }); +test("remote payload builder routes explicit Google provider to Interactions", async () => { + const payloads = await import(`../src/platform/runtime/remote-kernel/requestPayloads.ts?google-provider=${Date.now()}-${Math.random()}`); + const built = await payloads.buildImagesRequestBody({ + payload: { + provider: "google", + apiKey: "google-key", + mode: "generate", + prompt: "cat", + size: "2048x1152", + quality: "auto", + outputFormat: "png", + baseURL: "https://google-relay.example", + imageModelID: "gemini-image-model", + requestPolicy: "openai", + negativePrompt: "", + maskB64: "", + }, + }, []); + assert.equal(built.protocol, "google-interactions"); + assert.equal(built.url, "https://google-relay.example/v1beta/interactions"); + assert.equal(built.headers["x-goog-api-key"], "google-key"); +}); + +test("remote payload builder uses Grok JSON edit contract", async () => { + const payloads = await import(`../src/platform/runtime/remote-kernel/requestPayloads.ts?grok-provider=${Date.now()}-${Math.random()}`); + const built = await payloads.buildImagesRequestBody({ + payload: { + provider: "grok", + apiKey: "xai-key", + mode: "edit", + prompt: "restyle", + size: "2048x1152", + quality: "auto", + outputFormat: "png", + baseURL: "https://api.x.ai", + imageModelID: "grok-imagine-image", + requestPolicy: "openai", + negativePrompt: "", + maskB64: "", + }, + }, ["data:image/png;base64,YWJj"]); + const body = JSON.parse(built.body); + assert.equal(built.protocol, "grok-images"); + assert.equal(built.url, "https://api.x.ai/v1/images/edits"); + assert.equal(built.headers["Content-Type"], "application/json"); + assert.equal(body.aspect_ratio, "16:9"); + assert.equal(body.resolution, "2k"); + assert.deepEqual(body.image, { type: "image_url", url: "data:image/png;base64,YWJj" }); +}); + test("runRemoteImageJob retries when Responses API only returns partial previews", async () => { let calls = 0; const partials = []; diff --git a/image-studio/frontend/wailsjs/go/models.ts b/image-studio/frontend/wailsjs/go/models.ts index ee29b80c9..66a64b766 100755 --- a/image-studio/frontend/wailsjs/go/models.ts +++ b/image-studio/frontend/wailsjs/go/models.ts @@ -23,6 +23,7 @@ export namespace backend { textModelID: string; imageModelID: string; reasoningEffort: string; + provider: string; proxyMode: string; proxyURL: string; apiMode: string; @@ -65,6 +66,7 @@ export namespace backend { this.textModelID = source["textModelID"]; this.imageModelID = source["imageModelID"]; this.reasoningEffort = source["reasoningEffort"]; + this.provider = source["provider"]; this.proxyMode = source["proxyMode"]; this.proxyURL = source["proxyURL"]; this.apiMode = source["apiMode"]; @@ -215,6 +217,7 @@ export namespace backend { baseURL: string; proxyMode: string; proxyURL: string; + provider: string; apiMode: string; responsesTransport: string; allowInsecureConnection: boolean; @@ -229,6 +232,7 @@ export namespace backend { this.baseURL = source["baseURL"]; this.proxyMode = source["proxyMode"]; this.proxyURL = source["proxyURL"]; + this.provider = source["provider"]; this.apiMode = source["apiMode"]; this.responsesTransport = source["responsesTransport"]; this.allowInsecureConnection = source["allowInsecureConnection"]; diff --git a/shared/compat-go/state.go b/shared/compat-go/state.go index c955198fe..832511dde 100644 --- a/shared/compat-go/state.go +++ b/shared/compat-go/state.go @@ -74,6 +74,7 @@ type CompletionNotificationSettings struct { type UpstreamProfile struct { ID string `json:"id"` Name string `json:"name"` + Provider string `json:"provider,omitempty"` APIMode string `json:"apiMode"` ResponsesTransport string `json:"responsesTransport,omitempty"` RequestPolicy string `json:"requestPolicy"` @@ -240,6 +241,12 @@ func Normalize(state State) State { state.Profiles = []UpstreamProfile{} } for i := range state.Profiles { + switch state.Profiles[i].Provider { + case "google", "grok": + state.Profiles[i].APIMode = "images" + default: + state.Profiles[i].Provider = "openai" + } if state.Profiles[i].ReasoningEffort == "" { state.Profiles[i].ReasoningEffort = "xhigh" } diff --git a/shared/compat-go/state_test.go b/shared/compat-go/state_test.go index ce41c3540..69a43c421 100644 --- a/shared/compat-go/state_test.go +++ b/shared/compat-go/state_test.go @@ -87,6 +87,7 @@ func TestSaveLoadRoundTrip(t *testing.T) { Profiles: []UpstreamProfile{{ ID: "p1", Name: "配置1", + Provider: "google", APIMode: "responses", RequestPolicy: "openai", BaseURL: "https://upstream.example", @@ -141,7 +142,7 @@ func TestSaveLoadRoundTrip(t *testing.T) { if preset.StyleTag != "anime" || preset.Background != "transparent" || preset.InputFidelity != "high" || preset.ImageStyle != "vivid" || preset.Moderation != "auto" || preset.KernelRuntimeMode != "remote" || preset.BatchCount != 4 { t.Fatalf("preset fields not preserved: %#v", preset) } - if len(loaded.Profiles) != 1 || loaded.Profiles[0].BaseURL != "https://upstream.example" || loaded.Profiles[0].ReasoningEffort != "xhigh" { + if len(loaded.Profiles) != 1 || loaded.Profiles[0].BaseURL != "https://upstream.example" || loaded.Profiles[0].ReasoningEffort != "xhigh" || loaded.Profiles[0].Provider != "google" || loaded.Profiles[0].APIMode != "images" { t.Fatalf("profiles not preserved: %#v", loaded.Profiles) } if len(loaded.History) != 1 || loaded.History[0].SavedPath != "/tmp/images/cat.png" { diff --git a/shared/kernel/requestModel.d.ts b/shared/kernel/requestModel.d.ts index d9eb7fd2c..85c1fa934 100644 --- a/shared/kernel/requestModel.d.ts +++ b/shared/kernel/requestModel.d.ts @@ -10,6 +10,7 @@ export const DEFAULT_IMAGE_STYLE: string; export const DEFAULT_MODERATION: string; export const DEFAULT_REASONING_EFFORT: string; export const DEFAULT_REQUEST_POLICY: string; +export const DEFAULT_UPSTREAM_PROVIDER: string; export const DEFAULT_PARTIAL_IMAGES: number; export const DEFAULT_AUTO_RETRY_COUNT: number; export const MAX_AUTO_RETRY_COUNT: number; @@ -23,6 +24,7 @@ export const MAX_OPENAI_IMAGE_PIXELS: number; export const MAX_OPENAI_IMAGE_ASPECT_RATIO: number; export type RequestPolicy = "openai" | "compat"; +export type UpstreamProvider = "openai" | "google" | "grok"; export type ReasoningEffort = "low" | "medium" | "high" | "xhigh"; export type SharedImageRequestPayload = { @@ -52,9 +54,12 @@ export type SharedImageRequestPayload = { export function normalizeBaseURL(raw: string): string; export function isVersionedOpenAICompatibilityBaseURL(raw: string): boolean; export function openAIAPIEndpoint(baseURL: string, endpointPath: string): string; +export function normalizeUpstreamProvider(provider: string): UpstreamProvider; +export function effectiveAPIMode(provider: string, apiMode: string): "responses" | "images"; +export function googleAPIEndpoint(baseURL: string, endpointPath: string): string; export function isOfficialGoogleGeminiBaseURL(raw: string): boolean; export function isGoogleNativeNanoBanana2Model(imageModelID: string): boolean; -export function shouldUseGoogleNativeInteractions(baseURL: string, imageModelID: string): boolean; +export function shouldUseGoogleNativeInteractions(baseURL: string, imageModelID: string, provider?: string): boolean; export function googleInteractionsEndpoint(baseURL: string): string; export function normalizeAPIMode(apiMode: string): "responses" | "images"; export function normalizeRequestPolicy(requestPolicy: string): RequestPolicy; diff --git a/shared/kernel/requestModel.js b/shared/kernel/requestModel.js index e9ad19121..52cd16a12 100644 --- a/shared/kernel/requestModel.js +++ b/shared/kernel/requestModel.js @@ -10,6 +10,7 @@ export const DEFAULT_IMAGE_STYLE = "default"; export const DEFAULT_MODERATION = "low"; export const DEFAULT_REASONING_EFFORT = "xhigh"; export const DEFAULT_REQUEST_POLICY = "openai"; +export const DEFAULT_UPSTREAM_PROVIDER = "openai"; export const DEFAULT_PARTIAL_IMAGES = 1; export const DEFAULT_AUTO_RETRY_COUNT = 5; export const MAX_AUTO_RETRY_COUNT = 10; @@ -48,6 +49,22 @@ export function openAIAPIEndpoint(baseURL, endpointPath) { return `${normalized}/v1/${path}`; } +export function normalizeUpstreamProvider(provider) { + return provider === "google" || provider === "grok" ? provider : DEFAULT_UPSTREAM_PROVIDER; +} + +export function effectiveAPIMode(provider, apiMode) { + return normalizeUpstreamProvider(provider) === "openai" ? normalizeAPIMode(apiMode) : "images"; +} + +export function googleAPIEndpoint(baseURL, endpointPath) { + let normalized = normalizeBaseURL(baseURL); + const path = String(endpointPath || "").trim().replace(/^\/+|\/+$/g, ""); + if (/\/v1beta\/openai$/i.test(normalized)) normalized = normalized.replace(/\/openai$/i, ""); + else if (!/\/v1beta$/i.test(normalized)) normalized += "/v1beta"; + return path ? `${normalized}/${path}` : normalized; +} + export function isOfficialGoogleGeminiBaseURL(raw) { try { const parsed = new URL(normalizeBaseURL(raw)); @@ -61,17 +78,13 @@ export function isGoogleNativeNanoBanana2Model(imageModelID) { return normalizeImageModel(imageModelID).toLowerCase() === "gemini-3.1-flash-image"; } -export function shouldUseGoogleNativeInteractions(baseURL, imageModelID) { - return isOfficialGoogleGeminiBaseURL(baseURL) && isGoogleNativeNanoBanana2Model(imageModelID); +export function shouldUseGoogleNativeInteractions(baseURL, imageModelID, provider = "openai") { + return normalizeUpstreamProvider(provider) === "google" || + (isOfficialGoogleGeminiBaseURL(baseURL) && isGoogleNativeNanoBanana2Model(imageModelID)); } export function googleInteractionsEndpoint(baseURL) { - if (!isOfficialGoogleGeminiBaseURL(baseURL)) return ""; - const parsed = new URL(normalizeBaseURL(baseURL)); - parsed.pathname = "/v1beta/interactions"; - parsed.search = ""; - parsed.hash = ""; - return parsed.toString(); + return googleAPIEndpoint(baseURL, "interactions"); } export function normalizeAPIMode(apiMode) { From c49519cfb1a230b6fbcacd451f2b6717f2200af0 Mon Sep 17 00:00:00 2001 From: lin <1609924163@qq.com> Date: Wed, 29 Jul 2026 11:00:27 +0800 Subject: [PATCH 3/6] Refactor image workflows and improve application state handling --- gio-client/internal/compat/state.go | 6 +- gio-client/internal/compat/state_test.go | 14 + gio-client/internal/kernel/mask_composite.go | 205 ++++++++++++ .../internal/kernel/mask_composite_test.go | 237 ++++++++++++++ gio-client/internal/kernel/runner.go | 25 +- gio-client/internal/ui/canvas_mask.go | 38 ++- gio-client/internal/ui/canvas_mask_test.go | 43 ++- gio-client/internal/ui/run_job.go | 8 +- go-cli/go.mod | 5 +- go-cli/go.sum | 4 + go-cli/pkg/client/client.go | 18 + go-cli/pkg/client/client_test.go | 29 ++ go-cli/pkg/client/images_api.go | 138 +++++--- go-cli/pkg/client/images_api_test.go | 166 +++++++++- go-cli/pkg/client/mask_input.go | 202 ++++++++++++ go-cli/pkg/client/mask_test_helpers_test.go | 64 ++++ go-cli/pkg/client/payload.go | 7 +- go-cli/pkg/client/payload_test.go | 53 ++- go-cli/pkg/client/types.go | 4 +- .../src/app/components/PlatformWorkspace.tsx | 3 +- .../src/components/canvas/CanvasStage.tsx | 181 +++++----- .../components/canvas/MaskOverlayLayer.tsx | 125 +++++++ .../src/components/canvas/Toolbar.tsx | 50 ++- .../canvas/toolbarActionSections.tsx | 46 ++- .../components/canvas/useCanvasShortcuts.ts | 16 +- image-studio/frontend/src/lib/images.ts | 6 + .../frontend/src/lib/maskComposite.ts | 38 ++- image-studio/frontend/src/lib/maskEditor.ts | 187 +++++++++++ .../android/canvas/AndroidCanvasStage.tsx | 179 ++++++---- .../android/canvas/AndroidCanvasWorkspace.tsx | 77 ++++- .../frontend/src/platform/runtime/host.ts | 16 +- .../runtime/remote-kernel/requestPayloads.ts | 90 ++++- .../platform/runtime/remote-kernel/types.ts | 2 + .../frontend/src/state/studioStore.images.ts | 14 + .../frontend/src/state/studioStore.media.ts | 4 + .../frontend/src/state/studioStore.shared.ts | 43 +-- .../frontend/src/state/studioStore.ts | 309 ++++++++++++++++-- .../frontend/src/state/studioStore.types.ts | 6 + .../src/state/studioStore.workspaces.ts | 9 + .../frontend/test/imageSources.test.mjs | 13 + .../frontend/test/maskEditor.test.mjs | 59 ++++ image-studio/frontend/test/platform.test.mjs | 6 + .../frontend/test/remoteKernel.test.mjs | 121 ++++++- .../frontend/test/requestModel.test.mjs | 19 ++ shared/kernel/requestModel.js | 22 +- 45 files changed, 2548 insertions(+), 359 deletions(-) create mode 100644 gio-client/internal/kernel/mask_composite.go create mode 100644 gio-client/internal/kernel/mask_composite_test.go create mode 100644 go-cli/go.sum create mode 100644 go-cli/pkg/client/mask_input.go create mode 100644 go-cli/pkg/client/mask_test_helpers_test.go create mode 100644 image-studio/frontend/src/components/canvas/MaskOverlayLayer.tsx create mode 100644 image-studio/frontend/src/lib/maskEditor.ts create mode 100644 image-studio/frontend/test/maskEditor.test.mjs diff --git a/gio-client/internal/compat/state.go b/gio-client/internal/compat/state.go index 2e3c5e9d3..799cc2102 100644 --- a/gio-client/internal/compat/state.go +++ b/gio-client/internal/compat/state.go @@ -297,6 +297,10 @@ func UpsertConfig(state shared.State, cfg kernel.Config) shared.State { } func HistoryItemFromRun(cfg kernel.Config, result kernel.Result, elapsedSec float64, previewOnlyResult bool) shared.HistoryItem { + outputFormat := strings.TrimSpace(result.OutputFormat) + if outputFormat == "" { + outputFormat = cfg.OutputFormat + } item := shared.HistoryItem{ ID: randomID(), Prompt: cfg.Prompt, @@ -304,7 +308,7 @@ func HistoryItemFromRun(cfg kernel.Config, result kernel.Result, elapsedSec floa Mode: string(cfg.Mode), Size: cfg.Size, Quality: cfg.Quality, - OutputFormat: cfg.OutputFormat, + OutputFormat: outputFormat, CreatedAt: time.Now().UnixMilli(), Seed: cfg.Seed, NegativePrompt: cfg.NegativePrompt, diff --git a/gio-client/internal/compat/state_test.go b/gio-client/internal/compat/state_test.go index e08afb28c..e2113193a 100644 --- a/gio-client/internal/compat/state_test.go +++ b/gio-client/internal/compat/state_test.go @@ -178,6 +178,20 @@ func TestHistoryItemFromRunPreviewOnlyRemoteKeepsRawAndDefersSavedPaths(t *testi } } +func TestHistoryItemFromRunPrefersEffectiveResultFormat(t *testing.T) { + item := HistoryItemFromRun(kernel.Config{ + Prompt: "masked edit", + Mode: client.ModeEdit, + OutputFormat: "jpeg", + }, kernel.Result{ + SavedPath: "/tmp/images/masked.png", + OutputFormat: "png", + }, 1, false) + if item.OutputFormat != "png" { + t.Fatalf("history output format=%q want effective PNG result", item.OutputFormat) + } +} + func TestSaveConfigAndHistoryWithPreviewModeStoresHistoryFullAndHydratesImageB64(t *testing.T) { root := t.TempDir() origStable := StableDataRootForTest() diff --git a/gio-client/internal/kernel/mask_composite.go b/gio-client/internal/kernel/mask_composite.go new file mode 100644 index 000000000..b1def5638 --- /dev/null +++ b/gio-client/internal/kernel/mask_composite.go @@ -0,0 +1,205 @@ +package kernel + +import ( + "bytes" + "encoding/base64" + "errors" + "fmt" + "image" + "image/color" + "image/draw" + "image/png" + "os" + "strings" + + "github.com/yuanhua/image-gptcodex/pkg/client" + xdraw "golang.org/x/image/draw" +) + +type preparedEditMask struct { + source *image.NRGBA + mask *image.NRGBA +} + +func prepareEditMask(cfg *Config) (*preparedEditMask, error) { + if cfg == nil || cfg.Mode != client.ModeEdit || strings.TrimSpace(cfg.MaskB64) == "" { + return nil, nil + } + source, err := firstEditSourceImage(*cfg) + if err != nil { + return nil, fmt.Errorf("读取蒙版源图失败: %w", err) + } + sourceNRGBA := copyImageToNRGBA(source) + if sourceNRGBA.Bounds().Empty() { + return nil, errors.New("蒙版源图尺寸无效") + } + normalizedB64, normalizedMask, err := normalizeEditMaskB64(cfg.MaskB64, sourceNRGBA.Bounds().Size()) + if err != nil { + return nil, err + } + cfg.MaskB64 = normalizedB64 + return &preparedEditMask{source: sourceNRGBA, mask: normalizedMask}, nil +} + +func firstEditSourceImage(cfg Config) (image.Image, error) { + if paths := normalizeSourcePaths(cfg.SourcePaths); len(paths) > 0 { + file, err := os.Open(paths[0]) + if err != nil { + return nil, err + } + defer file.Close() + img, _, err := image.Decode(file) + if err != nil { + return nil, fmt.Errorf("解析首张源图失败: %w", err) + } + return img, nil + } + if dataURLs := normalizeSourceImageDataURLs(cfg.SourceImageDataURLs); len(dataURLs) > 0 { + return decodeImageDataURL(dataURLs[0]) + } + return nil, errors.New("图生图模式需要至少一张源图") +} + +func decodeImageDataURL(value string) (image.Image, error) { + value = strings.TrimSpace(value) + comma := strings.IndexByte(value, ',') + if !strings.HasPrefix(value, "data:") || comma < 0 || !strings.Contains(value[:comma], ";base64") { + return nil, errors.New("首张源图不是有效的 base64 data URL") + } + raw, err := base64.StdEncoding.DecodeString(value[comma+1:]) + if err != nil { + return nil, fmt.Errorf("首张源图 base64 无效: %w", err) + } + img, _, err := image.Decode(bytes.NewReader(raw)) + if err != nil { + return nil, fmt.Errorf("解析首张源图失败: %w", err) + } + return img, nil +} + +// normalizeEditMaskB64 converts legacy opaque black/white masks and alpha +// masks to the OpenAI convention: transparent pixels are editable and opaque +// pixels protect the source. The returned PNG matches the source dimensions. +func normalizeEditMaskB64(maskB64 string, dims image.Point) (string, *image.NRGBA, error) { + if dims.X <= 0 || dims.Y <= 0 { + return "", nil, errors.New("蒙版源图尺寸无效") + } + mask, err := decodeBase64Image(maskB64, "蒙版") + if err != nil { + return "", nil, err + } + + bounds := image.Rect(0, 0, dims.X, dims.Y) + scaled := image.NewNRGBA(bounds) + if mask.Bounds().Size() == dims { + draw.Draw(scaled, bounds, mask, mask.Bounds().Min, draw.Src) + } else { + xdraw.ApproxBiLinear.Scale(scaled, bounds, mask, mask.Bounds(), draw.Src, nil) + } + + hasTransparency := maskHasTransparency(scaled) + normalized := image.NewNRGBA(bounds) + for y := bounds.Min.Y; y < bounds.Max.Y; y++ { + for x := bounds.Min.X; x < bounds.Max.X; x++ { + pixel := scaled.NRGBAAt(x, y) + alpha := pixel.A + if !hasTransparency { + luma := (uint32(pixel.R)*299 + uint32(pixel.G)*587 + uint32(pixel.B)*114 + 500) / 1000 + alpha = uint8(0xff - luma) + } + normalized.SetNRGBA(x, y, color.NRGBA{A: alpha}) + } + } + + encoded, err := encodePNGBase64(normalized) + if err != nil { + return "", nil, fmt.Errorf("编码蒙版 PNG 失败: %w", err) + } + return encoded, normalized, nil +} + +func compositeMaskedEditB64(prepared *preparedEditMask, generatedB64 string) (string, error) { + if prepared == nil || prepared.source == nil || prepared.mask == nil { + return "", errors.New("蒙版合成状态无效") + } + generated, err := decodeBase64Image(generatedB64, "生成结果") + if err != nil { + return "", err + } + bounds := prepared.source.Bounds() + generatedNRGBA := image.NewNRGBA(bounds) + xdraw.CatmullRom.Scale(generatedNRGBA, bounds, generated, generated.Bounds(), draw.Src, nil) + + result := image.NewNRGBA(bounds) + for y := bounds.Min.Y; y < bounds.Max.Y; y++ { + for x := bounds.Min.X; x < bounds.Max.X; x++ { + keep := uint32(prepared.mask.NRGBAAt(x, y).A) + edit := uint32(0xff) - keep + sourcePixel := prepared.source.NRGBAAt(x, y) + generatedPixel := generatedNRGBA.NRGBAAt(x, y) + result.SetNRGBA(x, y, color.NRGBA{ + R: blendMaskChannel(sourcePixel.R, generatedPixel.R, keep, edit), + G: blendMaskChannel(sourcePixel.G, generatedPixel.G, keep, edit), + B: blendMaskChannel(sourcePixel.B, generatedPixel.B, keep, edit), + A: blendMaskChannel(sourcePixel.A, generatedPixel.A, keep, edit), + }) + } + } + encoded, err := encodePNGBase64(result) + if err != nil { + return "", fmt.Errorf("编码蒙版合成结果失败: %w", err) + } + return encoded, nil +} + +func decodeBase64Image(value, label string) (image.Image, error) { + value = strings.TrimSpace(value) + if comma := strings.IndexByte(value, ','); strings.HasPrefix(value, "data:") && comma >= 0 { + value = value[comma+1:] + } + raw, err := base64.StdEncoding.DecodeString(value) + if err != nil { + return nil, fmt.Errorf("%s base64 无效: %w", label, err) + } + if len(raw) == 0 { + return nil, fmt.Errorf("%s图片为空", label) + } + img, _, err := image.Decode(bytes.NewReader(raw)) + if err != nil { + return nil, fmt.Errorf("解析%s失败: %w", label, err) + } + return img, nil +} + +func encodePNGBase64(img image.Image) (string, error) { + var buf bytes.Buffer + if err := png.Encode(&buf, img); err != nil { + return "", err + } + return base64.StdEncoding.EncodeToString(buf.Bytes()), nil +} + +func copyImageToNRGBA(src image.Image) *image.NRGBA { + if src == nil || src.Bounds().Empty() { + return image.NewNRGBA(image.Rectangle{}) + } + bounds := image.Rect(0, 0, src.Bounds().Dx(), src.Bounds().Dy()) + dst := image.NewNRGBA(bounds) + draw.Draw(dst, bounds, src, src.Bounds().Min, draw.Src) + return dst +} + +func maskHasTransparency(mask *image.NRGBA) bool { + for y := mask.Bounds().Min.Y; y < mask.Bounds().Max.Y; y++ { + for x := mask.Bounds().Min.X; x < mask.Bounds().Max.X; x++ { + if mask.NRGBAAt(x, y).A < 0xff { + return true + } + } + } + return false +} + +func blendMaskChannel(source, generated uint8, keep, edit uint32) uint8 { + return uint8((uint32(source)*keep + uint32(generated)*edit + 127) / 255) +} diff --git a/gio-client/internal/kernel/mask_composite_test.go b/gio-client/internal/kernel/mask_composite_test.go new file mode 100644 index 000000000..4107891cb --- /dev/null +++ b/gio-client/internal/kernel/mask_composite_test.go @@ -0,0 +1,237 @@ +package kernel + +import ( + "bytes" + "context" + "encoding/base64" + "fmt" + "image" + "image/color" + "image/png" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/yuanhua/image-gptcodex/pkg/client" +) + +func TestNormalizeEditMaskB64ScalesOpaqueBlackWhiteMask(t *testing.T) { + legacy := image.NewNRGBA(image.Rect(0, 0, 2, 1)) + legacy.SetNRGBA(0, 0, color.NRGBA{A: 0xff}) + legacy.SetNRGBA(1, 0, color.NRGBA{R: 0xff, G: 0xff, B: 0xff, A: 0xff}) + + normalizedB64, normalized, err := normalizeEditMaskB64(encodeKernelTestPNG(t, legacy), image.Pt(4, 2)) + if err != nil { + t.Fatal(err) + } + if normalized.Bounds().Size() != image.Pt(4, 2) { + t.Fatalf("normalized bounds=%v want 4x2", normalized.Bounds()) + } + if alpha := normalized.NRGBAAt(0, 0).A; alpha != 0xff { + t.Fatalf("black legacy pixel alpha=%d want opaque protection", alpha) + } + if alpha := normalized.NRGBAAt(3, 1).A; alpha != 0 { + t.Fatalf("white legacy pixel alpha=%d want transparent edit selection", alpha) + } + decoded := decodeKernelTestPNG(t, normalizedB64) + if decoded.Bounds().Size() != image.Pt(4, 2) { + t.Fatalf("encoded normalized bounds=%v want 4x2", decoded.Bounds()) + } +} + +func TestNormalizeEditMaskB64PreservesAlphaSemantics(t *testing.T) { + alphaMask := image.NewNRGBA(image.Rect(0, 0, 1, 1)) + alphaMask.SetNRGBA(0, 0, color.NRGBA{R: 0xff, G: 0x80, B: 0x20, A: 0x40}) + + _, normalized, err := normalizeEditMaskB64(encodeKernelTestPNG(t, alphaMask), image.Pt(2, 2)) + if err != nil { + t.Fatal(err) + } + for y := 0; y < 2; y++ { + for x := 0; x < 2; x++ { + pixel := normalized.NRGBAAt(x, y) + if pixel.A != 0x40 || pixel.R != 0 || pixel.G != 0 || pixel.B != 0 { + t.Fatalf("normalized pixel (%d,%d)=%v want RGB zero and alpha 64", x, y, pixel) + } + } + } +} + +func TestPrepareEditMaskUsesFirstSourceDataURLDimensions(t *testing.T) { + source := image.NewNRGBA(image.Rect(0, 0, 3, 2)) + mask := image.NewNRGBA(image.Rect(0, 0, 1, 1)) + mask.SetNRGBA(0, 0, color.NRGBA{R: 0xff, G: 0xff, B: 0xff, A: 0xff}) + cfg := Config{ + Mode: client.ModeEdit, + SourceImageDataURLs: []string{"data:image/png;base64," + encodeKernelTestPNG(t, source)}, + MaskB64: encodeKernelTestPNG(t, mask), + } + + prepared, err := prepareEditMask(&cfg) + if err != nil { + t.Fatal(err) + } + if prepared == nil || prepared.mask.Bounds().Size() != image.Pt(3, 2) { + t.Fatalf("prepared mask=%v want source dimensions 3x2", prepared) + } + if decoded := decodeKernelTestPNG(t, cfg.MaskB64); decoded.Bounds().Size() != image.Pt(3, 2) { + t.Fatalf("request mask bounds=%v want source dimensions 3x2", decoded.Bounds()) + } +} + +func TestCompositeMaskedEditB64RestoresProtectedPixels(t *testing.T) { + source := image.NewNRGBA(image.Rect(0, 0, 2, 1)) + source.SetNRGBA(0, 0, color.NRGBA{R: 0xff, A: 0xff}) + source.SetNRGBA(1, 0, color.NRGBA{B: 0xff, A: 0xff}) + mask := image.NewNRGBA(source.Bounds()) + mask.SetNRGBA(0, 0, color.NRGBA{A: 0xff}) + mask.SetNRGBA(1, 0, color.NRGBA{A: 0}) + generated := image.NewNRGBA(source.Bounds()) + generated.SetNRGBA(0, 0, color.NRGBA{G: 0xff, A: 0xff}) + generated.SetNRGBA(1, 0, color.NRGBA{R: 0xff, G: 0xff, A: 0xff}) + + resultB64, err := compositeMaskedEditB64(&preparedEditMask{source: source, mask: mask}, encodeKernelTestPNG(t, generated)) + if err != nil { + t.Fatal(err) + } + result := decodeKernelTestPNG(t, resultB64) + assertKernelPixel(t, result, 0, 0, color.NRGBA{R: 0xff, A: 0xff}) + assertKernelPixel(t, result, 1, 0, color.NRGBA{R: 0xff, G: 0xff, A: 0xff}) +} + +func TestRunnerRunNormalizesAndCompositesMaskedEditBeforeSave(t *testing.T) { + source := image.NewNRGBA(image.Rect(0, 0, 2, 1)) + source.SetNRGBA(0, 0, color.NRGBA{R: 0xff, A: 0xff}) + source.SetNRGBA(1, 0, color.NRGBA{B: 0xff, A: 0xff}) + sourcePath := filepath.Join(t.TempDir(), "source.png") + writeKernelTestPNG(t, sourcePath, source) + + legacyMask := image.NewNRGBA(source.Bounds()) + legacyMask.SetNRGBA(0, 0, color.NRGBA{A: 0xff}) + legacyMask.SetNRGBA(1, 0, color.NRGBA{R: 0xff, G: 0xff, B: 0xff, A: 0xff}) + generated := image.NewNRGBA(source.Bounds()) + generated.SetNRGBA(0, 0, color.NRGBA{G: 0xff, A: 0xff}) + generated.SetNRGBA(1, 0, color.NRGBA{R: 0xff, G: 0xff, A: 0xff}) + generatedB64 := encodeKernelTestPNG(t, generated) + + type capturedMask struct { + img image.Image + err error + } + captured := make(chan capturedMask, 1) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/v1/images/edits" { + captured <- capturedMask{err: fmt.Errorf("unexpected path %s", r.URL.Path)} + http.NotFound(w, r) + return + } + if err := r.ParseMultipartForm(4 << 20); err != nil { + captured <- capturedMask{err: err} + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + file, _, err := r.FormFile("mask") + if err != nil { + captured <- capturedMask{err: err} + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + mask, decodeErr := png.Decode(file) + _ = file.Close() + captured <- capturedMask{img: mask, err: decodeErr} + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"data":[{"b64_json":"` + generatedB64 + `"}]}`)) + })) + defer server.Close() + + result, err := (Runner{}).Run(context.Background(), Config{ + APIKey: "sk-mask-test", + BaseURL: server.URL, + Prompt: "replace selected pixel", + Mode: client.ModeEdit, + APIMode: client.APIModeImages, + OutputDir: t.TempDir(), + OutputFormat: "jpeg", + SourcePaths: []string{sourcePath}, + MaskB64: encodeKernelTestPNG(t, legacyMask), + AutoRetryEnabled: false, + }, Callbacks{}) + if err != nil { + t.Fatalf("Run: %v", err) + } + uploaded := <-captured + if uploaded.err != nil { + t.Fatalf("capture uploaded mask: %v", uploaded.err) + } + if uploaded.img.Bounds().Size() != source.Bounds().Size() { + t.Fatalf("uploaded mask bounds=%v want %v", uploaded.img.Bounds(), source.Bounds()) + } + if alpha := color.NRGBAModel.Convert(uploaded.img.At(0, 0)).(color.NRGBA).A; alpha != 0xff { + t.Fatalf("uploaded protected alpha=%d want 255", alpha) + } + if alpha := color.NRGBAModel.Convert(uploaded.img.At(1, 0)).(color.NRGBA).A; alpha != 0 { + t.Fatalf("uploaded editable alpha=%d want 0", alpha) + } + if result.OutputFormat != "png" || strings.ToLower(filepath.Ext(result.SavedPath)) != ".png" { + t.Fatalf("masked result format=%q path=%q want PNG", result.OutputFormat, result.SavedPath) + } + savedFile, err := os.Open(result.SavedPath) + if err != nil { + t.Fatal(err) + } + saved, _, err := image.Decode(savedFile) + _ = savedFile.Close() + if err != nil { + t.Fatal(err) + } + assertKernelPixel(t, saved, 0, 0, color.NRGBA{R: 0xff, A: 0xff}) + assertKernelPixel(t, saved, 1, 0, color.NRGBA{R: 0xff, G: 0xff, A: 0xff}) +} + +func encodeKernelTestPNG(t *testing.T, img image.Image) string { + t.Helper() + var buf bytes.Buffer + if err := png.Encode(&buf, img); err != nil { + t.Fatal(err) + } + return base64.StdEncoding.EncodeToString(buf.Bytes()) +} + +func decodeKernelTestPNG(t *testing.T, rawB64 string) image.Image { + t.Helper() + raw, err := base64.StdEncoding.DecodeString(rawB64) + if err != nil { + t.Fatal(err) + } + img, err := png.Decode(bytes.NewReader(raw)) + if err != nil { + t.Fatal(err) + } + return img +} + +func writeKernelTestPNG(t *testing.T, path string, img image.Image) { + t.Helper() + file, err := os.Create(path) + if err != nil { + t.Fatal(err) + } + if err := png.Encode(file, img); err != nil { + _ = file.Close() + t.Fatal(err) + } + if err := file.Close(); err != nil { + t.Fatal(err) + } +} + +func assertKernelPixel(t *testing.T, img image.Image, x, y int, want color.NRGBA) { + t.Helper() + got := color.NRGBAModel.Convert(img.At(x, y)).(color.NRGBA) + if got != want { + t.Fatalf("pixel (%d,%d)=%v want %v", x, y, got, want) + } +} diff --git a/gio-client/internal/kernel/runner.go b/gio-client/internal/kernel/runner.go index 67bbaf06b..6d32373ca 100644 --- a/gio-client/internal/kernel/runner.go +++ b/gio-client/internal/kernel/runner.go @@ -95,6 +95,7 @@ type Result struct { RawPath string RawText string ImageB64 string + OutputFormat string RevisedPrompt string SourceEvent string } @@ -183,6 +184,10 @@ func (Runner) Run(ctx context.Context, cfg Config, cb Callbacks) (Result, error) if cfg.Mode == client.ModeEdit && len(cfg.SourcePaths) == 0 && len(cfg.SourceImageDataURLs) == 0 { return Result{}, fmt.Errorf("图生图模式需要至少一张源图") } + preparedMask, err := prepareEditMask(&cfg) + if err != nil { + return Result{}, err + } logDir := filepath.Join(cfg.OutputDir, "log") if !cfg.PreviewOnlyResult { if err := os.MkdirAll(cfg.OutputDir, 0o700); err != nil { @@ -225,7 +230,7 @@ func (Runner) Run(ctx context.Context, cfg Config, cb Callbacks) (Result, error) if idx > 0 { nonNilLog(cb.Log)("主上游自动重试失败，切换到备用上游再试一次...") } - result, err := runSingleConfig(ctx, variant, cb, imagesDir, logDir, idx) + result, err := runSingleConfig(ctx, variant, cb, imagesDir, logDir, idx, preparedMask) if err == nil { return result, nil } @@ -378,7 +383,7 @@ func normalizeRequestPolicy(policy client.RequestPolicy) client.RequestPolicy { return client.RequestPolicyOpenAI } -func runSingleConfig(ctx context.Context, cfg Config, cb Callbacks, imagesDir string, logDir string, variantIndex int) (Result, error) { +func runSingleConfig(ctx context.Context, cfg Config, cb Callbacks, imagesDir string, logDir string, variantIndex int, preparedMask *preparedEditMask) (Result, error) { proxy, err := client.NormalizeProxyConfig(cfg.ProxyMode, cfg.ProxyURL) if err != nil { return Result{}, err @@ -462,20 +467,31 @@ func runSingleConfig(ctx context.Context, cfg Config, cb Callbacks, imagesDir st if reqErr != nil { return Result{RawPath: rawPath, RawText: rawText}, reqErr } + finalOutputFormat := cfg.OutputFormat + if preparedMask != nil { + composited, compositeErr := compositeMaskedEditB64(preparedMask, result.ImageB64) + if compositeErr != nil { + return Result{RawPath: rawPath, RawText: rawText}, fmt.Errorf("保护蒙版外原图失败: %w", compositeErr) + } + result.ImageB64 = composited + finalOutputFormat = "png" + nonNilLog(cb.Log)("已在本地合成蒙版结果，蒙版外区域使用原图像素。") + } if cfg.PreviewOnlyResult { return Result{ RawPath: rawPath, RawText: rawText, ImageB64: result.ImageB64, + OutputFormat: finalOutputFormat, RevisedPrompt: result.RevisedPrompt, SourceEvent: result.SourceEvent, }, nil } - imageName := buildImageName(cfg.Mode, cfg.Prompt, timestamp, cfg.OutputFormat) + imageName := buildImageName(cfg.Mode, cfg.Prompt, timestamp, finalOutputFormat) savedPath, err := saveImage(result.ImageB64, filepath.Join(imagesDir, imageName)) if err != nil { - return Result{RawPath: rawPath, ImageB64: result.ImageB64}, err + return Result{RawPath: rawPath, ImageB64: result.ImageB64, OutputFormat: finalOutputFormat}, err } previewPath := "" thumbPath := "" @@ -498,6 +514,7 @@ func runSingleConfig(ctx context.Context, cfg Config, cb Callbacks, imagesDir st RawPath: rawPath, RawText: rawText, ImageB64: result.ImageB64, + OutputFormat: finalOutputFormat, RevisedPrompt: result.RevisedPrompt, SourceEvent: result.SourceEvent, }, nil diff --git a/gio-client/internal/ui/canvas_mask.go b/gio-client/internal/ui/canvas_mask.go index 1710a8933..287e7b2a0 100644 --- a/gio-client/internal/ui/canvas_mask.go +++ b/gio-client/internal/ui/canvas_mask.go @@ -6,6 +6,7 @@ import ( "fmt" "image" "image/color" + "image/draw" "image/png" "math" "strings" @@ -306,7 +307,7 @@ func (a *App) redoLatestCanvasAction() { } } -func drawMaskDot(img *image.Gray, center f32.Point, radius float32, value uint8) { +func drawMaskDot(img *image.NRGBA, center f32.Point, radius float32, alpha uint8) { if img == nil || radius <= 0 { return } @@ -322,25 +323,25 @@ func drawMaskDot(img *image.Gray, center f32.Point, radius float32, value uint8) if dx*dx+dy*dy > r2 { continue } - img.SetGray(x, y, color.Gray{Y: value}) + img.SetNRGBA(x, y, color.NRGBA{A: alpha}) } } } -func rasterizeMaskStroke(img *image.Gray, stroke canvasMaskStroke, dims image.Point) { +func rasterizeMaskStroke(img *image.NRGBA, stroke canvasMaskStroke, dims image.Point) { if img == nil || len(stroke.Points) == 0 { return } - value := uint8(255) + alpha := uint8(0) if stroke.Erase { - value = 0 + alpha = 0xff } radius := denormalizeCanvasBrushSize(stroke.SizeNorm, dims) / 2 if radius < 0.5 { radius = 0.5 } last := denormalizeCanvasMaskPoint(stroke.Points[0], dims) - drawMaskDot(img, last, radius, value) + drawMaskDot(img, last, radius, alpha) for _, point := range stroke.Points[1:] { current := denormalizeCanvasMaskPoint(point, dims) dx := current.X - last.X @@ -351,7 +352,7 @@ func rasterizeMaskStroke(img *image.Gray, stroke canvasMaskStroke, dims image.Po } for step := 1; step <= steps; step++ { t := float32(step) / float32(steps) - drawMaskDot(img, f32.Pt(last.X+dx*t, last.Y+dy*t), radius, value) + drawMaskDot(img, f32.Pt(last.X+dx*t, last.Y+dy*t), radius, alpha) } last = current } @@ -361,18 +362,15 @@ func buildCanvasMaskB64(strokes []canvasMaskStroke, dims image.Point) string { if len(strokes) == 0 || dims.X <= 0 || dims.Y <= 0 { return "" } - img := image.NewGray(image.Rect(0, 0, dims.X, dims.Y)) - hasWhite := false + img := image.NewNRGBA(image.Rect(0, 0, dims.X, dims.Y)) + draw.Draw(img, img.Bounds(), image.NewUniform(color.NRGBA{A: 0xff}), image.Point{}, draw.Src) for _, stroke := range strokes { if len(stroke.Points) == 0 { continue } - if !stroke.Erase { - hasWhite = true - } rasterizeMaskStroke(img, stroke, dims) } - if !hasWhite { + if !maskImageHasEditablePixels(img) { return "" } var buf bytes.Buffer @@ -382,6 +380,20 @@ func buildCanvasMaskB64(strokes []canvasMaskStroke, dims image.Point) string { return base64.StdEncoding.EncodeToString(buf.Bytes()) } +func maskImageHasEditablePixels(img *image.NRGBA) bool { + if img == nil { + return false + } + for y := img.Bounds().Min.Y; y < img.Bounds().Max.Y; y++ { + for x := img.Bounds().Min.X; x < img.Bounds().Max.X; x++ { + if img.NRGBAAt(x, y).A < 0xff { + return true + } + } + } + return false +} + func paintCanvasMaskStroke(gtx layout.Context, stroke canvasMaskStroke, origin image.Point, displaySize image.Point) { if len(stroke.Points) == 0 || displaySize.X <= 0 || displaySize.Y <= 0 { return diff --git a/gio-client/internal/ui/canvas_mask_test.go b/gio-client/internal/ui/canvas_mask_test.go index 6d15f6486..007663e07 100644 --- a/gio-client/internal/ui/canvas_mask_test.go +++ b/gio-client/internal/ui/canvas_mask_test.go @@ -25,7 +25,7 @@ func decodeMaskImage(t *testing.T, rawB64 string) image.Image { return img } -func TestBuildCanvasMaskB64ReturnsPNGWhenPaintStrokePresent(t *testing.T) { +func TestBuildCanvasMaskB64UsesAlphaMaskSemantics(t *testing.T) { maskB64 := buildCanvasMaskB64([]canvasMaskStroke{{ Points: []f32.Point{f32.Pt(0.1, 0.1), f32.Pt(0.9, 0.9)}, SizeNorm: 0.1, @@ -38,17 +38,36 @@ func TestBuildCanvasMaskB64ReturnsPNGWhenPaintStrokePresent(t *testing.T) { if img.Bounds().Dx() != 64 || img.Bounds().Dy() != 64 { t.Fatalf("mask bounds=%v want 64x64", img.Bounds()) } - hasWhite := false - for y := 0; y < img.Bounds().Dy() && !hasWhite; y++ { - for x := 0; x < img.Bounds().Dx(); x++ { - if color.GrayModel.Convert(img.At(x, y)).(color.Gray).Y > 0 { - hasWhite = true - break - } - } - } - if !hasWhite { - t.Fatal("mask should contain white painted area") + if alpha := color.NRGBAModel.Convert(img.At(32, 32)).(color.NRGBA).A; alpha != 0 { + t.Fatalf("painted pixel alpha=%d want transparent editable pixel", alpha) + } + if alpha := color.NRGBAModel.Convert(img.At(63, 0)).(color.NRGBA).A; alpha != 0xff { + t.Fatalf("unpainted pixel alpha=%d want opaque protected pixel", alpha) + } +} + +func TestBuildCanvasMaskB64EraseRestoresProtection(t *testing.T) { + maskB64 := buildCanvasMaskB64([]canvasMaskStroke{ + {Points: []f32.Point{f32.Pt(0.5, 0.5)}, SizeNorm: 0.5}, + {Points: []f32.Point{f32.Pt(0.5, 0.5)}, SizeNorm: 0.2, Erase: true}, + }, image.Pt(20, 20)) + img := decodeMaskImage(t, maskB64) + if alpha := color.NRGBAModel.Convert(img.At(10, 10)).(color.NRGBA).A; alpha != 0xff { + t.Fatalf("erased center alpha=%d want opaque protected pixel", alpha) + } + if alpha := color.NRGBAModel.Convert(img.At(6, 10)).(color.NRGBA).A; alpha != 0 { + t.Fatalf("painted ring alpha=%d want transparent editable pixel", alpha) + } +} + +func TestBuildCanvasMaskB64ReturnsEmptyWithoutEditablePixels(t *testing.T) { + maskB64 := buildCanvasMaskB64([]canvasMaskStroke{{ + Points: []f32.Point{f32.Pt(0.5, 0.5)}, + SizeNorm: 0.2, + Erase: true, + }}, image.Pt(20, 20)) + if maskB64 != "" { + t.Fatal("erase-only mask should not submit a fully protected image") } } diff --git a/gio-client/internal/ui/run_job.go b/gio-client/internal/ui/run_job.go index 5c4f96224..7b56fdf66 100644 --- a/gio-client/internal/ui/run_job.go +++ b/gio-client/internal/ui/run_job.go @@ -377,12 +377,16 @@ func (a *App) startRunWithConfig(cfg kernel.Config, total int, workflowWorkspace }) previewOnlyRemote := normalizeKernelRuntimeMode(kernelRuntimeMode) == "remote" if previewOnlyRemote && strings.TrimSpace(res.SavedPath) == "" && strings.TrimSpace(res.ImageB64) != "" { + resultFormat := strings.TrimSpace(res.OutputFormat) + if resultFormat == "" { + resultFormat = jobCfg.OutputFormat + } res.SavedPath = registerVirtualImage(res.ImageB64, suggestedSaveNameForHistoryItem(sharedCompat.HistoryItem{ Prompt: jobCfg.Prompt, Mode: string(jobCfg.Mode), - OutputFormat: jobCfg.OutputFormat, + OutputFormat: resultFormat, CreatedAt: time.Now().UnixMilli(), - }), jobCfg.OutputFormat) + }), resultFormat) } if previewOnlyRemote && strings.TrimSpace(res.RawText) != "" { res.RawPath = registerVirtualText(res.RawText, fmt.Sprintf("raw-response-%d-%d.txt", i+1, time.Now().UnixNano())) diff --git a/go-cli/go.mod b/go-cli/go.mod index 658827041..8bf515919 100644 --- a/go-cli/go.mod +++ b/go-cli/go.mod @@ -4,4 +4,7 @@ go 1.25.5 toolchain go1.26.3 -require github.com/gorilla/websocket v1.5.3 +require ( + github.com/gorilla/websocket v1.5.3 + golang.org/x/image v0.41.0 +) diff --git a/go-cli/go.sum b/go-cli/go.sum new file mode 100644 index 000000000..61393affb --- /dev/null +++ b/go-cli/go.sum @@ -0,0 +1,4 @@ +github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= +github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +golang.org/x/image v0.41.0 h1:8wS72eGJMJaBxK6okTzd4WaXumUlTVlb753MlsSvTCo= +golang.org/x/image v0.41.0/go.mod h1:uIc348UZMSvS5Z65CVZ7iDPaNobNFEPeJ4kbqTOszmA= diff --git a/go-cli/pkg/client/client.go b/go-cli/pkg/client/client.go index 9961c5a47..29dd7b04b 100644 --- a/go-cli/pkg/client/client.go +++ b/go-cli/pkg/client/client.go @@ -442,6 +442,9 @@ func imagesAPIWithRetries( } continue } + if errors.Is(reqErr, ErrNoImageInResponse) { + return ImageResult{}, lastPath, errors.New(describeImagesNoResult(raw)) + } // 同上,raw 路径靠返回值带,不再嵌进 error message。 return ImageResult{}, lastPath, reqErr } @@ -499,12 +502,27 @@ func imagesAPIWithRetriesInMemory( } continue } + if errors.Is(reqErr, ErrNoImageInResponse) { + return ImageResult{}, raw, errors.New(describeImagesNoResult(raw)) + } return ImageResult{}, raw, reqErr } return ImageResult{}, lastRaw, fmt.Errorf("多次请求后仍未成功:%w", lastErr) } +func describeImagesNoResult(raw string) string { + reason := DescribeProblem(raw) + switch reason { + case "接口返回为空。": + return "Images API 返回为空，未收到最终图片。" + case "接口已返回内容,但没有发现 image_generation_call.result。": + return "Images API 已返回内容，但没有发现 b64_json 或 url 图片。请查看原始响应日志确认中转站返回格式。" + default: + return reason + } +} + func repairSizeRetryOptions(opts Options, raw string, reqErr error) (Options, bool, string) { if extractInvalidSize(raw) == "" && (reqErr == nil || extractInvalidSize(reqErr.Error()) == "") { return opts, false, "" diff --git a/go-cli/pkg/client/client_test.go b/go-cli/pkg/client/client_test.go index bdb0ff2a7..934ee773a 100644 --- a/go-cli/pkg/client/client_test.go +++ b/go-cli/pkg/client/client_test.go @@ -464,6 +464,35 @@ func TestImagesAPIWithRetriesSkipsOuterRetryWhenWorkerAlreadyRetried(t *testing. } } +func TestImagesAPIWithRetriesExplainsMissingImageResult(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"data":[]}`)) + })) + defer srv.Close() + + _, _, err := RequestAndExtractWithRetries( + context.Background(), nil, + Options{ + APIKey: "sk-test", + Prompt: "p", + Size: "1024x1024", + Quality: "auto", + BaseURL: srv.URL, + APIMode: APIModeImages, + AutoRetryEnabled: func() *bool { value := false; return &value }(), + }, + t.TempDir(), "20260728-100002", + nil, nil, + ) + if err == nil || !strings.Contains(err.Error(), "b64_json 或 url") { + t.Fatalf("err = %v, want actionable missing-image explanation", err) + } + if strings.Contains(err.Error(), ErrNoImageInResponse.Error()) { + t.Fatalf("err leaked internal sentinel: %v", err) + } +} + func TestRequestAndExtractWithRetriesRepairsInvalidSizeAndRetriesOnce(t *testing.T) { pngB64 := base64.StdEncoding.EncodeToString([]byte("\x89PNG\r\n\x1a\nfixed")) hits := 0 diff --git a/go-cli/pkg/client/images_api.go b/go-cli/pkg/client/images_api.go index daa6e60fe..3ab5f85a5 100644 --- a/go-cli/pkg/client/images_api.go +++ b/go-cli/pkg/client/images_api.go @@ -128,10 +128,30 @@ type imagesAPIResponse struct { type imageStreamExtractor struct { partialB64 string final ImageResult + finalURL string hasFinal bool onPartial func(PartialImage) } +func (e *imageStreamExtractor) captureDatum(d imagesAPIDatum) bool { + if strings.TrimSpace(d.B64JSON) != "" { + e.final = imageResultFromImagesDatum(d) + e.finalURL = "" + e.hasFinal = true + return true + } + if strings.TrimSpace(d.URL) != "" { + e.final = ImageResult{ + RevisedPrompt: d.RevisedPrompt, + SourceEvent: "images_api_url", + } + e.finalURL = strings.TrimSpace(d.URL) + e.hasFinal = true + return true + } + return false +} + func (e *imageStreamExtractor) consume(line string) bool { stripped := strings.TrimSpace(line) if stripped == "" { @@ -163,39 +183,36 @@ func (e *imageStreamExtractor) consume(line string) bool { } return true case "image_generation.completed", "image_edit.completed": - if b64, ok := ev["b64_json"].(string); ok && b64 != "" { - e.final = ImageResult{ImageB64: b64, SourceEvent: "images_api"} - e.hasFinal = true + datum := imagesAPIDatum{} + if b64, ok := ev["b64_json"].(string); ok { + datum.B64JSON = b64 + } + if rawURL, ok := ev["url"].(string); ok { + datum.URL = rawURL + } + if revisedPrompt, ok := ev["revised_prompt"].(string); ok { + datum.RevisedPrompt = revisedPrompt + } + if e.captureDatum(datum) { return true } case "error": return true } - if ev["object"] == "image.generation.result" || ev["object"] == "image.edit.result" { - b, err := json.Marshal(ev) - if err == nil { - if result, err := parseImagesAPIResponseBytes(b, 200); err == nil { - e.final = result - e.hasFinal = true - return true - } - } - } if b, err := json.Marshal(ev); err == nil { - if result, err := parseImagesAPIResponseBytes(b, http.StatusOK); err == nil { - e.final = result - e.hasFinal = true + var parsed imagesAPIResponse + if json.Unmarshal(b, &parsed) == nil && len(parsed.Data) > 0 && e.captureDatum(parsed.Data[0]) { return true } } return true } -func (e *imageStreamExtractor) result() (ImageResult, bool) { +func (e *imageStreamExtractor) result() (ImageResult, string, bool) { if e.hasFinal { - return e.final, true + return e.final, e.finalURL, true } - return ImageResult{}, false + return ImageResult{}, "", false } // RequestImagesAPI executes a single (no-retry) request against the standard @@ -223,6 +240,9 @@ func RequestImagesAPIWithPartial( if strings.TrimSpace(opts.Prompt) == "" { return ImageResult{}, ErrEmptyPrompt } + if strings.TrimSpace(opts.MaskB64) != "" && opts.Mode != ModeEdit { + return ImageResult{}, errors.New("蒙版仅支持图生图模式") + } baseURL := strings.TrimSpace(opts.BaseURL) if baseURL == "" { @@ -440,15 +460,23 @@ func RequestImagesAPIWithPartial( } } if err := scanner.Err(); err != nil { - if result, ok := extractor.result(); ok && result.ImageB64 != "" { - return result, nil + if result, resultURL, ok := extractor.result(); ok { + if resultURL != "" { + return downloadImagesAPIURL(ctx, httpClient, resultURL, result.RevisedPrompt, onProgress, startedAt) + } + if result.ImageB64 != "" { + return result, nil + } } return ImageResult{}, fmt.Errorf("read Images API stream: %w", err) } if resp.StatusCode/100 != 2 { return ImageResult{}, fmt.Errorf("上游返回 HTTP %d", resp.StatusCode) } - if result, ok := extractor.result(); ok { + if result, resultURL, ok := extractor.result(); ok { + if resultURL != "" { + return downloadImagesAPIURL(ctx, httpClient, resultURL, result.RevisedPrompt, onProgress, startedAt) + } return result, nil } return ImageResult{}, ErrNoImageInResponse @@ -725,53 +753,43 @@ func writeDataURLToTemp(dataURL string) (string, error) { } // buildEditsMultipart constructs the multipart/form-data body for /v1/images/edits. -// 多张源图按 image[] / image[1] / ... 形式串联 —— 不同中转站对多图编辑支持不一, -// 仅第一张是 OpenAI 官方接受的最小可用形态,其余作为兼容性 best-effort。 +// OpenAI's multi-image contract uses repeated image[] fields for every source; +// the singular image field is retained only for a one-image edit. func buildEditsMultipart( paths []string, maskB64, prompt, model, size, quality, outputFormat, background string, outputCompression int, inputFidelity, moderation, userIdentifier, negativePrompt string, seed int64, requestPolicy RequestPolicy, partialImages int, useNewAPICompat bool, ) (*bytes.Buffer, string, error) { buf := &bytes.Buffer{} w := multipart.NewWriter(buf) + var maskPair *openAIImagesMaskPair + if strings.TrimSpace(maskB64) != "" { + if len(paths) == 0 || strings.TrimSpace(paths[0]) == "" { + return nil, "", errors.New("蒙版任务需要至少一张源图") + } + prepared, err := prepareOpenAIImagesMaskPair(paths[0], maskB64) + if err != nil { + return nil, "", err + } + maskPair = &prepared + } for i, p := range paths { fieldName := "image" - if i > 0 { - // Some relays accept multiple `image` fields, others want image[] — - // we send both to maximise compatibility. The extra field is cheap. + if len(paths) > 1 { fieldName = "image[]" } + if i == 0 && maskPair != nil { + if err := writeMultipartBytes(w, fieldName, "source.png", "image/png", maskPair.sourcePNG); err != nil { + return nil, "", fmt.Errorf("attach source.png: %w", err) + } + continue + } if err := writeMultipartFile(w, fieldName, p); err != nil { return nil, "", fmt.Errorf("attach %s: %w", filepath.Base(p), err) } } - if strings.TrimSpace(maskB64) != "" { - raw, err := base64.StdEncoding.DecodeString(maskB64) - if err != nil { - return nil, "", fmt.Errorf("蒙版图片 base64 无效:%w", err) - } - if len(raw) == 0 { - return nil, "", errors.New("蒙版图片为空") - } - if len(raw) > MaxInputImageBytes { - return nil, "", fmt.Errorf("蒙版图片过大(%dB > %dB 上限)", len(raw), MaxInputImageBytes) - } - // Preserve the actual PNG/JPEG/WebP type for compatible relays instead - // of relabeling arbitrary bytes as PNG. Official OpenAI users can still - // supply a PNG mask when their selected model requires it. - maskMimeType := detectImageMimeTypeFromBytes(raw) - if strings.TrimSpace(maskMimeType) == "" { - return nil, "", errors.New("蒙版图片不是支持的 PNG/JPEG/WebP 格式") - } - maskExt := imageExtensionForMimeType(maskMimeType) - h := make(textproto.MIMEHeader) - h.Set("Content-Disposition", fmt.Sprintf(`form-data; name="mask"; filename="mask.%s"`, maskExt)) - h.Set("Content-Type", maskMimeType) - fw, err := w.CreatePart(h) - if err != nil { - return nil, "", err - } - if _, err := fw.Write(raw); err != nil { + if maskPair != nil { + if err := writeMultipartBytes(w, "mask", "mask.png", "image/png", maskPair.maskPNG); err != nil { return nil, "", err } } @@ -843,6 +861,18 @@ func writeMultipartFile(w *multipart.Writer, fieldName, path string) error { return err } +func writeMultipartBytes(w *multipart.Writer, fieldName, filename, mimeType string, data []byte) error { + h := make(textproto.MIMEHeader) + h.Set("Content-Disposition", fmt.Sprintf(`form-data; name="%s"; filename="%s"`, fieldName, filename)) + h.Set("Content-Type", mimeType) + part, err := w.CreatePart(h) + if err != nil { + return err + } + _, err = part.Write(data) + return err +} + func mimeForPath(p string) string { ext := strings.ToLower(filepath.Ext(p)) if m, ok := SupportedImageMime[ext]; ok { diff --git a/go-cli/pkg/client/images_api_test.go b/go-cli/pkg/client/images_api_test.go index b593da6e1..0e7389a6f 100644 --- a/go-cli/pkg/client/images_api_test.go +++ b/go-cli/pkg/client/images_api_test.go @@ -7,6 +7,8 @@ import ( "encoding/json" "errors" "fmt" + "image" + "image/png" "io" "mime" "mime/multipart" @@ -179,6 +181,45 @@ func TestRequestImagesAPIDownloadsURLOnlyResponse(t *testing.T) { } } +func TestRequestImagesAPIDownloadsURLOnlySSEResponse(t *testing.T) { + imageBytes := []byte{0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00, 0x01} + var imageURL string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/v1/images/generations": + w.Header().Set("Content-Type", "text/event-stream") + fmt.Fprintf(w, "data: {\"data\":[{\"url\":%q,\"revised_prompt\":\"cat revised from stream\"}]}\n\n", imageURL) + case "/image.png": + w.Header().Set("Content-Type", "image/png") + _, _ = w.Write(imageBytes) + default: + http.NotFound(w, r) + } + })) + defer srv.Close() + imageURL = srv.URL + "/image.png" + + res, err := RequestImagesAPIWithPartial(context.Background(), Options{ + APIKey: "sk-test", + Prompt: "cat", + BaseURL: srv.URL, + APIMode: APIModeImages, + ImageModelID: "relay-image-model", + }, &bytes.Buffer{}, nil, nil) + if err != nil { + t.Fatal(err) + } + if res.ImageB64 != base64.StdEncoding.EncodeToString(imageBytes) { + t.Fatalf("ImageB64 = %q", res.ImageB64) + } + if res.RevisedPrompt != "cat revised from stream" { + t.Fatalf("RevisedPrompt = %q", res.RevisedPrompt) + } + if res.SourceEvent != "images_api_url" { + t.Fatalf("SourceEvent = %q", res.SourceEvent) + } +} + func TestRequestImagesAPIGeminiModelUsesNonStreamingCompatEndpoint(t *testing.T) { finalB64 := base64.StdEncoding.EncodeToString([]byte("final")) var requestBody []byte @@ -333,13 +374,13 @@ func TestRequestImagesAPIWithRetriesRetriesWhenOnlyPartialPreviewArrives(t *test func TestBuildEditsMultipartSetsMaskMimeType(t *testing.T) { dir := t.TempDir() src := filepath.Join(dir, "source.png") - if err := os.WriteFile(src, fakePNG, 0o644); err != nil { + if err := os.WriteFile(src, testPNGBytes(t, 2, 1), 0o644); err != nil { t.Fatal(err) } buf, contentType, err := buildEditsMultipart( []string{src}, - base64.StdEncoding.EncodeToString(fakePNG), + base64.StdEncoding.EncodeToString(testAlphaMaskPNGBytes(t, 2, 1)), "edit this", "gpt-image-2", "1024x1024", @@ -415,13 +456,71 @@ func TestBuildEditsMultipartSetsMaskMimeType(t *testing.T) { } } -func TestBuildEditsMultipartDetectsJPEGMaskMimeType(t *testing.T) { +func TestBuildEditsMultipartUsesArrayFieldForEveryMultiImageSource(t *testing.T) { dir := t.TempDir() - src := filepath.Join(dir, "source.png") - if err := os.WriteFile(src, fakePNG, 0o644); err != nil { + paths := []string{ + filepath.Join(dir, "first.png"), + filepath.Join(dir, "second.png"), + } + for _, path := range paths { + if err := os.WriteFile(path, fakePNG, 0o644); err != nil { + t.Fatal(err) + } + } + + buf, contentType, err := buildEditsMultipart( + paths, + "", + "edit this", + "gpt-image-2", + "1024x1024", + "auto", + "png", + "auto", + 100, + "auto", + "low", + "", + "", + 0, + RequestPolicyOpenAI, + DefaultPartialImages, + false, + ) + if err != nil { + t.Fatal(err) + } + _, params, err := mime.ParseMediaType(contentType) + if err != nil { + t.Fatal(err) + } + reader := multipart.NewReader(buf, params["boundary"]) + var imageFields []string + for { + part, err := reader.NextPart() + if err == io.EOF { + break + } + if err != nil { + t.Fatal(err) + } + if part.FileName() != "" { + imageFields = append(imageFields, part.FormName()) + } + _, _ = io.Copy(io.Discard, part) + } + if got, want := strings.Join(imageFields, ","), "image[],image[]"; got != want { + t.Fatalf("image multipart fields = %q, want %q", got, want) + } +} + +func TestBuildEditsMultipartCanonicalizesJPEGSourceAndMaskToSameSizePNG(t *testing.T) { + dir := t.TempDir() + src := filepath.Join(dir, "source.jpg") + if err := os.WriteFile(src, testJPEGBytes(t, 3, 2), 0o644); err != nil { t.Fatal(err) } - jpegMask := base64.StdEncoding.EncodeToString([]byte{0xff, 0xd8, 0xff, 0xdb, 0x00, 0x43}) + jpegMask := base64.StdEncoding.EncodeToString(testJPEGBytes(t, 1, 1)) buf, contentType, err := buildEditsMultipart( []string{src}, @@ -451,6 +550,8 @@ func TestBuildEditsMultipartDetectsJPEGMaskMimeType(t *testing.T) { t.Fatal(err) } reader := multipart.NewReader(buf, params["boundary"]) + foundSource := false + foundMask := false for { part, err := reader.NextPart() if err == io.EOF { @@ -459,18 +560,57 @@ func TestBuildEditsMultipartDetectsJPEGMaskMimeType(t *testing.T) { if err != nil { t.Fatal(err) } - if part.FormName() == "mask" { - if got := part.Header.Get("Content-Type"); got != "image/jpeg" { - t.Fatalf("mask content-type = %q, want image/jpeg", got) + if part.FormName() == "image" || part.FormName() == "mask" { + if got := part.Header.Get("Content-Type"); got != "image/png" { + t.Fatalf("%s content-type = %q, want image/png", part.FormName(), got) } - if got := part.FileName(); got != "mask.jpg" { - t.Fatalf("mask filename = %q, want mask.jpg", got) + wantName := "source.png" + if part.FormName() == "mask" { + wantName = "mask.png" + foundMask = true + } else { + foundSource = true } - return + if got := part.FileName(); got != wantName { + t.Fatalf("%s filename = %q, want %q", part.FormName(), got, wantName) + } + img, err := png.Decode(part) + if err != nil { + t.Fatalf("decode %s PNG: %v", part.FormName(), err) + } + if got := img.Bounds().Size(); got != image.Pt(3, 2) { + t.Fatalf("%s size = %v, want 3x2", part.FormName(), got) + } + continue } _, _ = io.Copy(io.Discard, part) } - t.Fatal("expected mask part in multipart body") + if !foundSource || !foundMask { + t.Fatalf("canonical pair missing: source=%v mask=%v", foundSource, foundMask) + } +} + +func TestRequestImagesAPIRejectsMaskOutsideEditBeforeNetwork(t *testing.T) { + hits := 0 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + hits++ + http.Error(w, "unexpected request", http.StatusInternalServerError) + })) + defer srv.Close() + + _, err := RequestImagesAPIWithPartial(context.Background(), Options{ + APIKey: "sk-test", + BaseURL: srv.URL, + Prompt: "cat", + Mode: ModeGenerate, + MaskB64: base64.StdEncoding.EncodeToString(testAlphaMaskPNGBytes(t, 1, 1)), + }, &bytes.Buffer{}, nil, nil) + if err == nil || !strings.Contains(err.Error(), "图生图") { + t.Fatalf("err = %v, want edit-mode validation", err) + } + if hits != 0 { + t.Fatalf("server received %d requests, want 0", hits) + } } func TestBuildEditsMultipartOmitsMaskWhenEmpty(t *testing.T) { diff --git a/go-cli/pkg/client/mask_input.go b/go-cli/pkg/client/mask_input.go new file mode 100644 index 000000000..5c2997e41 --- /dev/null +++ b/go-cli/pkg/client/mask_input.go @@ -0,0 +1,202 @@ +package client + +import ( + "bytes" + "encoding/base64" + "errors" + "fmt" + "image" + "image/color" + "image/draw" + _ "image/jpeg" + "image/png" + "os" + "strings" + + xdraw "golang.org/x/image/draw" + _ "golang.org/x/image/webp" +) + +type openAIImagesMaskPair struct { + sourcePNG []byte + maskPNG []byte +} + +func validateResponsesMaskInput(mode Mode, sourceDataURLs []string, maskB64 string) error { + if strings.TrimSpace(maskB64) == "" { + return nil + } + if mode != ModeEdit { + return errors.New("蒙版仅支持图生图模式") + } + if len(sourceDataURLs) == 0 || strings.TrimSpace(sourceDataURLs[0]) == "" { + return errors.New("蒙版任务需要至少一张源图") + } + if _, _, err := decodeImageDataURLWithinLimit(sourceDataURLs[0], "首张源图"); err != nil { + return err + } + maskRaw, err := decodeBase64WithinLimit(maskB64, "蒙版图片") + if err != nil { + return err + } + _, _, err = decodeSupportedImage(maskRaw, "蒙版图片") + return err +} + +func prepareOpenAIImagesMaskPair(sourcePath, maskB64 string) (openAIImagesMaskPair, error) { + sourceRaw, err := readImageFileWithinLimit(sourcePath, "首张源图") + if err != nil { + return openAIImagesMaskPair{}, err + } + source, _, err := decodeSupportedImage(sourceRaw, "首张源图") + if err != nil { + return openAIImagesMaskPair{}, err + } + dims := source.Bounds().Size() + if dims.X <= 0 || dims.Y <= 0 { + return openAIImagesMaskPair{}, errors.New("首张源图尺寸无效") + } + + maskRaw, err := decodeBase64WithinLimit(maskB64, "蒙版图片") + if err != nil { + return openAIImagesMaskPair{}, err + } + mask, _, err := decodeSupportedImage(maskRaw, "蒙版图片") + if err != nil { + return openAIImagesMaskPair{}, err + } + normalizedMask := normalizeMaskToSize(mask, dims) + + sourcePNG, err := encodePNGWithinLimit(source, "首张源图") + if err != nil { + return openAIImagesMaskPair{}, err + } + maskPNG, err := encodePNGWithinLimit(normalizedMask, "蒙版图片") + if err != nil { + return openAIImagesMaskPair{}, err + } + return openAIImagesMaskPair{sourcePNG: sourcePNG, maskPNG: maskPNG}, nil +} + +func readImageFileWithinLimit(path, label string) ([]byte, error) { + path = strings.TrimSpace(path) + if path == "" { + return nil, fmt.Errorf("%s为空", label) + } + info, err := os.Stat(path) + if err != nil { + return nil, fmt.Errorf("读取%s失败: %w", label, err) + } + if info.IsDir() { + return nil, fmt.Errorf("%s不是图片文件", label) + } + if info.Size() <= 0 { + return nil, fmt.Errorf("%s为空", label) + } + if info.Size() > MaxInputImageBytes { + return nil, fmt.Errorf("%s过大(%dB > %dB 上限)", label, info.Size(), MaxInputImageBytes) + } + raw, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("读取%s失败: %w", label, err) + } + if len(raw) > MaxInputImageBytes { + return nil, fmt.Errorf("%s过大(%dB > %dB 上限)", label, len(raw), MaxInputImageBytes) + } + return raw, nil +} + +func decodeBase64WithinLimit(value, label string) ([]byte, error) { + encoded := strings.TrimSpace(value) + if encoded == "" { + return nil, fmt.Errorf("%s为空", label) + } + if base64.StdEncoding.DecodedLen(len(encoded)) > MaxInputImageBytes+2 { + return nil, fmt.Errorf("%s超过 50MB 上限", label) + } + raw, err := base64.StdEncoding.DecodeString(encoded) + if err != nil { + return nil, fmt.Errorf("%s base64 无效: %w", label, err) + } + if len(raw) == 0 { + return nil, fmt.Errorf("%s为空", label) + } + if len(raw) > MaxInputImageBytes { + return nil, fmt.Errorf("%s过大(%dB > %dB 上限)", label, len(raw), MaxInputImageBytes) + } + return raw, nil +} + +func decodeImageDataURLWithinLimit(value, label string) ([]byte, image.Image, error) { + value = strings.TrimSpace(value) + comma := strings.IndexByte(value, ',') + if !strings.HasPrefix(value, "data:image/") || comma < 0 || !strings.Contains(value[:comma], ";base64") { + return nil, nil, fmt.Errorf("%s不是有效的 base64 图片 data URL", label) + } + raw, err := decodeBase64WithinLimit(value[comma+1:], label) + if err != nil { + return nil, nil, err + } + img, _, err := decodeSupportedImage(raw, label) + return raw, img, err +} + +func decodeSupportedImage(raw []byte, label string) (image.Image, string, error) { + mimeType := detectImageMimeTypeFromBytes(raw) + if mimeType == "" { + return nil, "", fmt.Errorf("%s不是支持的 PNG/JPEG/WebP 图片", label) + } + img, _, err := image.Decode(bytes.NewReader(raw)) + if err != nil { + return nil, "", fmt.Errorf("解析%s失败: %w", label, err) + } + if img.Bounds().Dx() <= 0 || img.Bounds().Dy() <= 0 { + return nil, "", fmt.Errorf("%s尺寸无效", label) + } + return img, mimeType, nil +} + +func normalizeMaskToSize(mask image.Image, dims image.Point) *image.NRGBA { + bounds := image.Rect(0, 0, dims.X, dims.Y) + scaled := image.NewNRGBA(bounds) + if mask.Bounds().Size() == dims { + draw.Draw(scaled, bounds, mask, mask.Bounds().Min, draw.Src) + } else { + xdraw.ApproxBiLinear.Scale(scaled, bounds, mask, mask.Bounds(), draw.Src, nil) + } + + hasTransparency := false + for y := bounds.Min.Y; y < bounds.Max.Y && !hasTransparency; y++ { + for x := bounds.Min.X; x < bounds.Max.X; x++ { + if scaled.NRGBAAt(x, y).A < 0xff { + hasTransparency = true + break + } + } + } + + normalized := image.NewNRGBA(bounds) + for y := bounds.Min.Y; y < bounds.Max.Y; y++ { + for x := bounds.Min.X; x < bounds.Max.X; x++ { + pixel := scaled.NRGBAAt(x, y) + alpha := pixel.A + if !hasTransparency { + luma := (uint32(pixel.R)*299 + uint32(pixel.G)*587 + uint32(pixel.B)*114 + 500) / 1000 + alpha = uint8(0xff - luma) + } + normalized.SetNRGBA(x, y, color.NRGBA{A: alpha}) + } + } + return normalized +} + +func encodePNGWithinLimit(img image.Image, label string) ([]byte, error) { + var encoded bytes.Buffer + if err := png.Encode(&encoded, img); err != nil { + return nil, fmt.Errorf("编码%s PNG 失败: %w", label, err) + } + if encoded.Len() > MaxInputImageBytes { + return nil, fmt.Errorf("%s转换为 PNG 后过大(%dB > %dB 上限)", label, encoded.Len(), MaxInputImageBytes) + } + return encoded.Bytes(), nil +} diff --git a/go-cli/pkg/client/mask_test_helpers_test.go b/go-cli/pkg/client/mask_test_helpers_test.go new file mode 100644 index 000000000..78bef5b6c --- /dev/null +++ b/go-cli/pkg/client/mask_test_helpers_test.go @@ -0,0 +1,64 @@ +package client + +import ( + "bytes" + "image" + "image/color" + "image/jpeg" + "image/png" + "testing" +) + +func testPNGBytes(t *testing.T, width, height int) []byte { + t.Helper() + img := image.NewNRGBA(image.Rect(0, 0, width, height)) + for y := 0; y < height; y++ { + for x := 0; x < width; x++ { + img.SetNRGBA(x, y, color.NRGBA{ + R: uint8(40 + x*30), + G: uint8(60 + y*30), + B: 120, + A: 0xff, + }) + } + } + var buf bytes.Buffer + if err := png.Encode(&buf, img); err != nil { + t.Fatal(err) + } + return buf.Bytes() +} + +func testAlphaMaskPNGBytes(t *testing.T, width, height int) []byte { + t.Helper() + img := image.NewNRGBA(image.Rect(0, 0, width, height)) + for y := 0; y < height; y++ { + for x := 0; x < width; x++ { + alpha := uint8(0xff) + if x == width-1 { + alpha = 0 + } + img.SetNRGBA(x, y, color.NRGBA{A: alpha}) + } + } + var buf bytes.Buffer + if err := png.Encode(&buf, img); err != nil { + t.Fatal(err) + } + return buf.Bytes() +} + +func testJPEGBytes(t *testing.T, width, height int) []byte { + t.Helper() + img := image.NewRGBA(image.Rect(0, 0, width, height)) + for y := 0; y < height; y++ { + for x := 0; x < width; x++ { + img.SetRGBA(x, y, color.RGBA{R: uint8(80 + x*20), G: uint8(40 + y*20), B: 100, A: 0xff}) + } + } + var buf bytes.Buffer + if err := jpeg.Encode(&buf, img, &jpeg.Options{Quality: 90}); err != nil { + t.Fatal(err) + } + return buf.Bytes() +} diff --git a/go-cli/pkg/client/payload.go b/go-cli/pkg/client/payload.go index 213414b98..39c77fd7d 100644 --- a/go-cli/pkg/client/payload.go +++ b/go-cli/pkg/client/payload.go @@ -49,6 +49,9 @@ func BuildPayload(opts Options) ([]byte, error) { } action := "generate" imageURLs := opts.EffectiveImageDataURLs() + if err := validateResponsesMaskInput(opts.Mode, imageURLs, opts.MaskB64); err != nil { + return nil, err + } for _, url := range imageURLs { content = append(content, map[string]any{ "type": "input_image", @@ -83,9 +86,9 @@ func BuildPayload(opts Options) ([]byte, error) { if supportsImageModeration(imgModel) { tool["moderation"] = moderation } - if opts.MaskB64 != "" { + if strings.TrimSpace(opts.MaskB64) != "" { tool["input_image_mask"] = map[string]any{ - "image_url": imageDataURLFromBase64(opts.MaskB64, ""), + "image_url": imageDataURLFromBase64(strings.TrimSpace(opts.MaskB64), ""), } } if includeExtended && opts.Seed != 0 { diff --git a/go-cli/pkg/client/payload_test.go b/go-cli/pkg/client/payload_test.go index 0dec3b244..c58272f8c 100644 --- a/go-cli/pkg/client/payload_test.go +++ b/go-cli/pkg/client/payload_test.go @@ -345,21 +345,41 @@ func TestBuildPayloadOmitsMaskWhenEmpty(t *testing.T) { } func TestBuildPayloadIncludesMaskWhenSet(t *testing.T) { - raw, _ := BuildPayload(Options{Prompt: "x", MaskB64: "AAAA"}) + maskB64 := base64.StdEncoding.EncodeToString(testAlphaMaskPNGBytes(t, 2, 1)) + sourceB64 := base64.StdEncoding.EncodeToString(testPNGBytes(t, 2, 1)) + raw, err := BuildPayload(Options{ + Prompt: "x", + Mode: ModeEdit, + ImageDataURLs: []string{"data:image/png;base64," + sourceB64}, + MaskB64: maskB64, + }) + if err != nil { + t.Fatal(err) + } v := mustDecodePayload(t, raw) tool := v["tools"].([]any)[0].(map[string]any) mask, ok := tool["input_image_mask"].(map[string]any) if !ok { t.Fatalf("expected input_image_mask object, got %T", tool["input_image_mask"]) } - if mask["image_url"] != "data:image/png;base64,AAAA" { - t.Errorf("input_image_mask.image_url = %v, want data:image/png;base64,AAAA", mask["image_url"]) + want := "data:image/png;base64," + maskB64 + if mask["image_url"] != want { + t.Errorf("input_image_mask.image_url = %v, want %s", mask["image_url"], want) } } func TestBuildPayloadDetectsMaskMimeTypeFromBase64(t *testing.T) { - jpegMask := base64.StdEncoding.EncodeToString([]byte{0xff, 0xd8, 0xff, 0xdb, 0x00, 0x43}) - raw, _ := BuildPayload(Options{Prompt: "x", MaskB64: jpegMask}) + jpegMask := base64.StdEncoding.EncodeToString(testJPEGBytes(t, 1, 1)) + sourceB64 := base64.StdEncoding.EncodeToString(testPNGBytes(t, 1, 1)) + raw, err := BuildPayload(Options{ + Prompt: "x", + Mode: ModeEdit, + ImageDataURLs: []string{"data:image/png;base64," + sourceB64}, + MaskB64: jpegMask, + }) + if err != nil { + t.Fatal(err) + } v := mustDecodePayload(t, raw) tool := v["tools"].([]any)[0].(map[string]any) mask := tool["input_image_mask"].(map[string]any) @@ -369,6 +389,29 @@ func TestBuildPayloadDetectsMaskMimeTypeFromBase64(t *testing.T) { } } +func TestBuildPayloadRejectsMaskWithoutEditSource(t *testing.T) { + maskB64 := base64.StdEncoding.EncodeToString(testAlphaMaskPNGBytes(t, 1, 1)) + if _, err := BuildPayload(Options{Prompt: "x", Mode: ModeGenerate, MaskB64: maskB64}); err == nil || !strings.Contains(err.Error(), "图生图") { + t.Fatalf("err = %v, want edit-mode validation", err) + } + if _, err := BuildPayload(Options{Prompt: "x", Mode: ModeEdit, MaskB64: maskB64}); err == nil || !strings.Contains(err.Error(), "源图") { + t.Fatalf("err = %v, want source validation", err) + } +} + +func TestBuildPayloadRejectsInvalidMaskImage(t *testing.T) { + sourceB64 := base64.StdEncoding.EncodeToString(testPNGBytes(t, 1, 1)) + _, err := BuildPayload(Options{ + Prompt: "x", + Mode: ModeEdit, + ImageDataURLs: []string{"data:image/png;base64," + sourceB64}, + MaskB64: base64.StdEncoding.EncodeToString([]byte("not-an-image")), + }) + if err == nil || !strings.Contains(err.Error(), "不是支持的") { + t.Fatalf("err = %v, want invalid-image validation", err) + } +} + func TestImageFileToDataURLEncodesPNG(t *testing.T) { dir := t.TempDir() src := filepath.Join(dir, "source.png") diff --git a/go-cli/pkg/client/types.go b/go-cli/pkg/client/types.go index 327e27f8f..907b3c232 100644 --- a/go-cli/pkg/client/types.go +++ b/go-cli/pkg/client/types.go @@ -194,7 +194,9 @@ type Options struct { // certificate verification for this upstream. It must remain opt-in. AllowInsecureConnection bool - MaskB64 string // optional, reserved for Phase 3 GUI; omitted from payload when empty + // MaskB64 is a raw base64 image for edit requests. Images API uploads are + // normalized with the first source into a same-size PNG pair. + MaskB64 string // Seed pins the random source so users can reproduce a result. 0 means // "let the model pick", and the field is then omitted from the payload. diff --git a/image-studio/frontend/src/app/components/PlatformWorkspace.tsx b/image-studio/frontend/src/app/components/PlatformWorkspace.tsx index 983ff56e3..7849f8c95 100644 --- a/image-studio/frontend/src/app/components/PlatformWorkspace.tsx +++ b/image-studio/frontend/src/app/components/PlatformWorkspace.tsx @@ -29,13 +29,12 @@ export function PlatformWorkspace({ androidView: AndroidView; onChangeAndroidView: (value: AndroidView) => void; }) { - const { isAndroid, isAndroidPad, targetPlatform, androidWidthClass } = usePlatform(); + const { isAndroid, isAndroidPad, androidWidthClass } = usePlatform(); return ( }> {isAndroid ? ( (null); const imageLayerRef = useRef(null); - const maskLayerRef = useRef(null); const roRef = useRef(null); const currentImageURL = historyFullSrc(currentImage, null); const image = useImageFromSource(currentImage?.imageBlob ?? null, currentImage?.imageB64, currentImageURL); - const importedMaskURL = maskDataURL && maskDataURL !== "__PENDING_MASK__" ? maskDataURL : null; - const importedMaskImage = useImageFromSource(null, undefined, importedMaskURL); + const importedMaskImage = useImageFromSource(null, undefined, maskDataURL); useEffect(() => { if (!currentImage?.previewOnly) return; @@ -194,23 +194,60 @@ export function CanvasStage() { // store (so we don't spam zustand on every mousemove). Forces re-render via // a tick counter when the in-progress stroke needs to redraw. const drawingRef = useRef<{ active: boolean; current: Stroke | null }>({ active: false, current: null }); + const previousImageIDRef = useRef(currentImage?.id); + const previousCanvasResetTickRef = useRef(canvasViewResetTick); const [, setDrawingTick] = useState(0); + const [maskCursor, setMaskCursor] = useState<{ x: number; y: number } | null>(null); // Annotation drag state. const [drag, setDrag] = useState(null); const [canvasMenu, setCanvasMenu] = useState(null); - // When the displayed image identity changes, clear the user's manual view - // and per-image canvas state. This guarantees the new image starts at fit. - // canvasViewResetTick 触发同样的重置 —— 用于 旋转 / 翻转 / 裁剪 这些「就地编辑」 - // 操作:currentImage.id 没变(就是原来那张),但底图尺寸 / 坐标已变,残留的 pan/zoom - // 与蒙版坐标系都失效了。 + useEffect(() => { + const imageChanged = previousImageIDRef.current !== currentImage?.id; + previousImageIDRef.current = currentImage?.id; + setUserView(null); + if (imageChanged) { + useStudioStore.setState({ + maskDataURL: null, + maskTargetPath: null, + strokes: [], + annotations: [], + selectedAnnotationId: null, + undoStack: [], + redoStack: [], + }); + } + drawingRef.current = { active: false, current: null }; + setMaskCursor(null); + }, [currentImage?.id]); + + // Transform actions keep the history id but change the underlying pixels + // and coordinate system. Skip the initial mount so layout remounts such as + // Android fullscreen do not destroy an existing draft. + useEffect(() => { + const canvasChanged = previousCanvasResetTickRef.current !== canvasViewResetTick; + previousCanvasResetTickRef.current = canvasViewResetTick; + if (!canvasChanged) return; + setUserView(null); + useStudioStore.setState({ + maskDataURL: null, + maskTargetPath: null, + strokes: [], + annotations: [], + selectedAnnotationId: null, + undoStack: [], + redoStack: [], + }); + drawingRef.current = { active: false, current: null }; + setMaskCursor(null); + }, [canvasViewResetTick]); + useEffect(() => { setUserView(null); - setMaskDataURL(null); drawingRef.current = { active: false, current: null }; - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [currentImage?.id, canvasViewResetTick]); + setMaskCursor(null); + }, [fullscreen]); // setView is the only writer of userView. Treat any explicit pan/zoom as a // user override; auto-recenter happens by resetting to null elsewhere. @@ -251,13 +288,17 @@ export function CanvasStage() { }; } + function isInsideImage(point: { x: number; y: number }): boolean { + return !!image && point.x >= 0 && point.y >= 0 && point.x <= image.width && point.y <= image.height; + } + // In-progress freehand annotation buffer (kept in ref to keep mousemove cheap). const freehandRef = useRef(null); function onMouseDown(e: Konva.KonvaEventObject) { if (!image) return; const local = stagePointerToImageCoord(); - if (!local) return; + if (!local || !isInsideImage(local)) return; if (effectiveTool === "mask") { drawingRef.current = { active: true, current: { points: [local.x, local.y], size: brushSize, erase: brushMode === "erase" } }; } else if (effectiveTool === "annotate") { @@ -310,6 +351,12 @@ export function CanvasStage() { if (!image) return; const local = stagePointerToImageCoord(); if (!local) return; + const insideImage = isInsideImage(local); + setMaskCursor(effectiveTool === "mask" && insideImage ? local : null); + if (!insideImage) { + if (effectiveTool === "mask" && drawingRef.current.active) finishMaskStroke(); + return; + } if (effectiveTool === "mask" && drawingRef.current.active && drawingRef.current.current) { drawingRef.current.current.points.push(local.x, local.y); setDrawingTick((n) => n + 1); @@ -322,10 +369,8 @@ export function CanvasStage() { } function onMouseUp() { - if (effectiveTool === "mask" && drawingRef.current.active && drawingRef.current.current) { - const finished = drawingRef.current.current; - drawingRef.current = { active: false, current: null }; - pushStroke(finished); + if (effectiveTool === "mask" && drawingRef.current.active) { + finishMaskStroke(); } else if (effectiveTool === "annotate" && annotationKind === "freehand" && freehandRef.current) { const pts = freehandRef.current; freehandRef.current = null; @@ -369,21 +414,16 @@ export function CanvasStage() { } } - // Keep the store flag in sync so submit can cheaply know whether any mask - // exists, but defer the expensive PNG export until the user actually submits. - useEffect(() => { - const hasImportedMask = !!maskDataURL && maskDataURL !== "__PENDING_MASK__"; - if (!image) { - setMaskDataURL(null); - return; - } - if (strokes.length === 0) { - if (!hasImportedMask) setMaskDataURL(null); - return; - } - const hasWhite = strokes.some((s) => !s.erase); - setMaskDataURL(hasWhite ? "__PENDING_MASK__" : null); - }, [strokes, image, maskDataURL, setMaskDataURL]); + function finishMaskStroke() { + const finished = drawingRef.current.current; + drawingRef.current = { active: false, current: null }; + if (finished) pushStroke(finished); + } + + function onMouseLeave() { + setMaskCursor(null); + onMouseUp(); + } function resetView() { setUserView(null); @@ -399,6 +439,7 @@ export function CanvasStage() { useCanvasShortcuts({ brushSize, + brushMode, canNavigateBatchResults, cancel, compareB, @@ -427,11 +468,16 @@ export function CanvasStage() { resetView, selectedAnnotationId, setBrushSize: (value) => setField("brushSize", value), + setBrushMode: (value) => setField("brushMode", value), setCompareB, setErrorMessage: (value) => setField("errorMessage", value), toggleFullscreen, setSelectedAnnotationId: (value) => setField("selectedAnnotationId", value), - setTool: (value) => setField("tool", value), + setTool: (value) => { + if (value === "mask") void activateMaskTool(); + else setField("tool", value); + }, + tool, undo, }); @@ -465,7 +511,13 @@ export function CanvasStage() {
    {!currentImage && !showingResultGrid && } {streamPreview && currentImage && !showingLiveBatchGrid ? ( @@ -526,7 +578,7 @@ export function CanvasStage() { onMouseDown={onMouseDown} onMouseMove={onMouseMove} onMouseUp={onMouseUp} - onMouseLeave={onMouseUp} + onMouseLeave={onMouseLeave} onDblClick={onStageDblClick} onContextMenu={openCanvasMenu} > @@ -534,47 +586,24 @@ export function CanvasStage() { {image && } - - {image && importedMaskImage ? ( - - ) : null} - {strokes.map((s, i) => ( - - ))} - {drawingRef.current.current && ( - - )} - + {image ? ( + + ) : null} {annotations.map((a) => ( diff --git a/image-studio/frontend/src/components/canvas/MaskOverlayLayer.tsx b/image-studio/frontend/src/components/canvas/MaskOverlayLayer.tsx new file mode 100644 index 000000000..077b026e1 --- /dev/null +++ b/image-studio/frontend/src/components/canvas/MaskOverlayLayer.tsx @@ -0,0 +1,125 @@ +import { useEffect, useRef } from "react"; +import type Konva from "konva"; +import { Circle, Image as KonvaImage, Layer, Line, Rect } from "react-konva"; +import type { Stroke } from "../../state/studioStore.types"; + +type Point = { x: number; y: number }; + +type MaskOverlayLayerProps = { + image: HTMLImageElement; + baseMaskImage: HTMLImageElement | null; + strokes: Stroke[]; + draft: Stroke | null; + visible: boolean; + opacity: number; + cursor: Point | null; + showCursor: boolean; + brushSize: number; + brushMode: "paint" | "erase"; + viewScale: number; +}; + +const MASK_COLOR = "#4d7cff"; + +export function MaskOverlayLayer({ + image, + baseMaskImage, + strokes, + draft, + visible, + opacity, + cursor, + showCursor, + brushSize, + brushMode, + viewScale, +}: MaskOverlayLayerProps) { + const maskLayerRef = useRef(null); + const clampedOpacity = Math.max(0.1, Math.min(0.9, opacity)); + const clip = { + clipX: 0, + clipY: 0, + clipWidth: image.width, + clipHeight: image.height, + }; + useEffect(() => { + const canvas = maskLayerRef.current?.getCanvas()._canvas; + if (!canvas) return; + canvas.style.opacity = String(clampedOpacity); + return () => { + canvas.style.opacity = ""; + }; + }, [clampedOpacity]); + return ( + <> + + {baseMaskImage ? ( + <> + + + + ) : null} + {strokes.map((stroke, index) => ( + + ))} + {draft ? : null} + + + {showCursor && cursor ? ( + <> + + + + ) : null} + + + ); +} + +function MaskStrokeShape({ stroke }: { stroke: Stroke }) { + const operation = stroke.erase ? "destination-out" : "source-over"; + if (stroke.points.length === 2) { + return ( + + ); + } + return ( + + ); +} diff --git a/image-studio/frontend/src/components/canvas/Toolbar.tsx b/image-studio/frontend/src/components/canvas/Toolbar.tsx index 06059b84b..825f24eda 100644 --- a/image-studio/frontend/src/components/canvas/Toolbar.tsx +++ b/image-studio/frontend/src/components/canvas/Toolbar.tsx @@ -1,4 +1,4 @@ -import { useState } from "react"; +import { useEffect, useState } from "react"; import { Crop, FlipHorizontal, FlipVertical, RotateCcw, RotateCw } from "lucide-react"; import { useStudioStore } from "../../state/studioStore"; import { usePlatform } from "../../platform/context"; @@ -15,12 +15,14 @@ import { Sep, ToolBtn, ToolbarGroup } from "./toolbarPrimitives"; export function Toolbar() { const { currentImage, tool, brushSize, brushMode, + maskDataURL, maskVisible, maskOpacity, strokes, annotationKind, annotationColor, annotations, selectedAnnotationId, fullscreen, batchResults, resultGridOpen, openResultGrid, closeResultGrid, setField, toggleFullscreen, saveCurrentImageAs, - importMaskImage, resetMask, clearAnnotations, + activateMaskTool, importMaskImage, fillMask, invertMask, resetMask, clearAnnotations, + provider, batchProcess, undoStack, redoStack, undo, redo, rotateCurrent, flipCurrent, cropToRect, openResultDetail, @@ -29,6 +31,21 @@ export function Toolbar() { const { isAndroidPhone, isMac, usesFluentUI, usesAppleUI } = usePlatform(); const [mobileAdjustOpen, setMobileAdjustOpen] = useState(false); const hasImage = !!currentImage; + const hasMask = !!maskDataURL || strokes.some((stroke) => !stroke.erase); + const maskDisabledTitle = provider === "google" + ? "Google Interactions 不支持 OpenAI 蒙版参数" + : provider === "grok" + ? "Grok Imagine 不支持 OpenAI 蒙版参数" + : batchProcess.enabled + ? "批量目录处理不支持单图蒙版" + : undefined; + useEffect(() => { + if (tool === "mask" && maskDisabledTitle) setField("tool", "pan"); + }, [maskDisabledTitle, setField, tool]); + const setCanvasTool = (nextTool: "pan" | "mask" | "annotate") => { + if (nextTool === "mask") void activateMaskTool(); + else setField("tool", nextTool); + }; const showBatchGridToggle = batchResults.length > 1; if (isMac && !hasImage && !showBatchGridToggle) return null; const showToolSection = hasImage; @@ -55,10 +72,17 @@ export function Toolbar() { showViewReset brushMode={brushMode} brushSize={brushSize} + maskVisible={maskVisible} + maskOpacity={maskOpacity} + hasMask={hasMask} annotationKind={annotationKind} annotationColor={annotationColor} onSetBrushMode={(mode) => setField("brushMode", mode)} onSetBrushSize={(size) => setField("brushSize", size)} + onToggleMaskVisible={() => setField("maskVisible", !maskVisible)} + onSetMaskOpacity={(opacity) => setField("maskOpacity", opacity)} + onFillMask={fillMask} + onInvertMask={invertMask} onImportMask={importMaskImage} onResetMask={resetMask} onSetAnnotationKind={(kind) => setField("annotationKind", kind)} @@ -104,9 +128,11 @@ export function Toolbar() { setField("tool", nextTool)} + onSetTool={setCanvasTool} onUndo={undo} onRedo={redo} /> @@ -120,10 +146,17 @@ export function Toolbar() { showViewReset={showViewReset} brushMode={brushMode} brushSize={brushSize} + maskVisible={maskVisible} + maskOpacity={maskOpacity} + hasMask={hasMask} annotationKind={annotationKind} annotationColor={annotationColor} onSetBrushMode={(mode) => setField("brushMode", mode)} onSetBrushSize={(size) => setField("brushSize", size)} + onToggleMaskVisible={() => setField("maskVisible", !maskVisible)} + onSetMaskOpacity={(opacity) => setField("maskOpacity", opacity)} + onFillMask={fillMask} + onInvertMask={invertMask} onImportMask={importMaskImage} onResetMask={resetMask} onSetAnnotationKind={(kind) => setField("annotationKind", kind)} @@ -171,9 +204,11 @@ export function Toolbar() { setField("tool", nextTool)} + onSetTool={setCanvasTool} onUndo={undo} onRedo={redo} /> @@ -187,10 +222,17 @@ export function Toolbar() { showViewReset={showViewReset} brushMode={brushMode} brushSize={brushSize} + maskVisible={maskVisible} + maskOpacity={maskOpacity} + hasMask={hasMask} annotationKind={annotationKind} annotationColor={annotationColor} onSetBrushMode={(mode) => setField("brushMode", mode)} onSetBrushSize={(size) => setField("brushSize", size)} + onToggleMaskVisible={() => setField("maskVisible", !maskVisible)} + onSetMaskOpacity={(opacity) => setField("maskOpacity", opacity)} + onFillMask={fillMask} + onInvertMask={invertMask} onImportMask={importMaskImage} onResetMask={resetMask} onSetAnnotationKind={(kind) => setField("annotationKind", kind)} diff --git a/image-studio/frontend/src/components/canvas/toolbarActionSections.tsx b/image-studio/frontend/src/components/canvas/toolbarActionSections.tsx index 8fe9b394f..07d854444 100644 --- a/image-studio/frontend/src/components/canvas/toolbarActionSections.tsx +++ b/image-studio/frontend/src/components/canvas/toolbarActionSections.tsx @@ -1,6 +1,6 @@ import { - Brush, Crop, Eraser, FlipHorizontal, FlipVertical, Hand, - Info, MoveRight, Pencil, RotateCcw, RotateCw, Save, Square, + Brush, Contrast, Crop, Eraser, Eye, EyeOff, FlipHorizontal, FlipVertical, Hand, + Info, MoveRight, PaintBucket, Pencil, RotateCcw, RotateCw, Save, Square, Trash2, Maximize, Minimize, Type as TypeIcon, Upload, } from "lucide-react"; import { ANNOTATION_COLORS } from "../../types/domain"; @@ -19,6 +19,8 @@ import { export function BaseToolSection({ hasImage, tool, + maskDisabled = false, + maskDisabledTitle, undoDisabled, redoDisabled, onSetTool, @@ -27,6 +29,8 @@ export function BaseToolSection({ }: { hasImage: boolean; tool: "pan" | "mask" | "annotate"; + maskDisabled?: boolean; + maskDisabledTitle?: string; undoDisabled: boolean; redoDisabled: boolean; onSetTool: (tool: "pan" | "mask" | "annotate") => void; @@ -40,7 +44,7 @@ export function BaseToolSection({ onSetTool("pan")} title="拖动 / 缩放 (1)" label={isMac ? "拖动" : undefined}> - onSetTool("mask")} title="蒙版画笔 (2)" label={isMac ? "蒙版" : undefined}> + onSetTool("mask")} title={maskDisabledTitle || "蒙版画笔 (2)"} label={isMac ? "蒙版" : undefined}> onSetTool("annotate")} title="画框标注 (3)" label={isMac ? "标注" : undefined}> @@ -62,10 +66,17 @@ export function ContextualSection({ showViewReset, brushMode, brushSize, + maskVisible, + maskOpacity, + hasMask, annotationKind, annotationColor, onSetBrushMode, onSetBrushSize, + onToggleMaskVisible, + onSetMaskOpacity, + onFillMask, + onInvertMask, onImportMask, onResetMask, onSetAnnotationKind, @@ -78,10 +89,17 @@ export function ContextualSection({ showViewReset: boolean; brushMode: "paint" | "erase"; brushSize: number; + maskVisible: boolean; + maskOpacity: number; + hasMask: boolean; annotationKind: "rect" | "arrow" | "freehand" | "text"; annotationColor: string; onSetBrushMode: (mode: "paint" | "erase") => void; onSetBrushSize: (size: number) => void; + onToggleMaskVisible: () => void; + onSetMaskOpacity: (opacity: number) => void; + onFillMask: () => void | Promise; + onInvertMask: () => void | Promise; onImportMask: () => void | Promise; onResetMask: () => void; onSetAnnotationKind: (kind: "rect" | "arrow" | "freehand" | "text") => void; @@ -103,6 +121,15 @@ export function ContextualSection({ void onImportMask()} title="导入蒙版图片" label={isMac ? "导入" : undefined}> + void onFillMask()} title="全选整张图为可编辑区域" label={isMac ? "全选" : undefined}> + + + void onInvertMask()} title="反选可编辑区域" label={isMac ? "反选" : undefined}> + + + + {maskVisible ? : } + 大小 {brushSize} - + 叠加 + onSetMaskOpacity(Number(event.target.value) / 100)} + className="w-16 accent-[var(--accent)] disabled:opacity-40" + /> + {Math.round(maskOpacity * 100)}% + 清空 diff --git a/image-studio/frontend/src/components/canvas/useCanvasShortcuts.ts b/image-studio/frontend/src/components/canvas/useCanvasShortcuts.ts index ef1dee7e5..5fda1381e 100644 --- a/image-studio/frontend/src/components/canvas/useCanvasShortcuts.ts +++ b/image-studio/frontend/src/components/canvas/useCanvasShortcuts.ts @@ -5,6 +5,7 @@ type Tool = "pan" | "mask" | "annotate"; type UseCanvasShortcutsArgs = { brushSize: number; + brushMode: "paint" | "erase"; canNavigateBatchResults?: boolean; cancel: () => void; compareB: HistoryItem | null; @@ -19,16 +20,19 @@ type UseCanvasShortcutsArgs = { resetView: () => void; selectedAnnotationId: string | null; setBrushSize: (value: number) => void; + setBrushMode: (value: "paint" | "erase") => void; setCompareB: (item: HistoryItem | null) => void; setErrorMessage: (value: string | null) => void; toggleFullscreen: () => void | Promise; setSelectedAnnotationId: (value: string | null) => void; setTool: (value: Tool) => void; + tool: Tool; undo: () => void; }; export function useCanvasShortcuts({ brushSize, + brushMode, canNavigateBatchResults = false, cancel, compareB, @@ -43,11 +47,13 @@ export function useCanvasShortcuts({ resetView, selectedAnnotationId, setBrushSize, + setBrushMode, setCompareB, setErrorMessage, toggleFullscreen, setSelectedAnnotationId, setTool, + tool, undo, }: UseCanvasShortcutsArgs) { useEffect(() => { @@ -121,7 +127,12 @@ export function useCanvasShortcuts({ return; } } - if (k === "[" || k === "]") { + if (tool === "mask" && k === "x") { + e.preventDefault(); + setBrushMode(brushMode === "paint" ? "erase" : "paint"); + return; + } + if (tool === "mask" && (k === "[" || k === "]")) { e.preventDefault(); const delta = k === "[" ? -5 : 5; setBrushSize(Math.max(5, Math.min(120, brushSize + delta))); @@ -132,6 +143,7 @@ export function useCanvasShortcuts({ return () => window.removeEventListener("keydown", onKeyDown); }, [ brushSize, + brushMode, canNavigateBatchResults, cancel, compareB, @@ -146,11 +158,13 @@ export function useCanvasShortcuts({ resetView, selectedAnnotationId, setBrushSize, + setBrushMode, setCompareB, setErrorMessage, toggleFullscreen, setSelectedAnnotationId, setTool, + tool, undo, ]); } diff --git a/image-studio/frontend/src/lib/images.ts b/image-studio/frontend/src/lib/images.ts index 23bdb6112..2f78f4b92 100644 --- a/image-studio/frontend/src/lib/images.ts +++ b/image-studio/frontend/src/lib/images.ts @@ -301,3 +301,9 @@ export function blobSourceToURL(source: { imageBlob?: Blob | null; imageB64?: st if (source.imageB64) return URL.createObjectURL(base64ToBlob(source.imageB64)); return null; } + +export function prioritizeImageSource(sources: T[], target: T): T[] { + const targetPath = target.path.trim(); + if (!targetPath) return sources; + return [target, ...sources.filter((source) => source.path.trim() !== targetPath)]; +} diff --git a/image-studio/frontend/src/lib/maskComposite.ts b/image-studio/frontend/src/lib/maskComposite.ts index ad811d17c..2a5c28827 100644 --- a/image-studio/frontend/src/lib/maskComposite.ts +++ b/image-studio/frontend/src/lib/maskComposite.ts @@ -2,17 +2,28 @@ import { dataURLFromBase64 } from "./images.ts"; export type PreparedMaskComposite = { sourceDataURL: string; + sourceUploadDataURL: string; maskB64: string; }; +export const MAX_MASK_INPUT_BYTES = 50 * 1024 * 1024; + export async function normalizeMaskForSource( sourceDataURL: string, maskB64: string, ): Promise { + assertImageDataURLWithinLimit(sourceDataURL, "蒙版源图"); + const normalizedInputMaskB64 = assertBase64WithinLimit(maskB64, "蒙版图片"); const [source, mask] = await Promise.all([ loadImage(sourceDataURL), - loadImage(dataURLFromBase64(maskB64)), + loadImage(dataURLFromBase64(normalizedInputMaskB64)), ]); + const sourceCanvas = makeCanvas(source.naturalWidth, source.naturalHeight); + const sourceContext = requireContext(sourceCanvas); + sourceContext.drawImage(source, 0, 0, sourceCanvas.width, sourceCanvas.height); + const sourceUploadDataURL = sourceCanvas.toDataURL("image/png"); + assertImageDataURLWithinLimit(sourceUploadDataURL, "蒙版源图 PNG"); + const canvas = makeCanvas(source.naturalWidth, source.naturalHeight); const context = requireContext(canvas); context.drawImage(mask, 0, 0, canvas.width, canvas.height); @@ -36,7 +47,9 @@ export async function normalizeMaskForSource( } context.putImageData(pixels, 0, 0); } - return { sourceDataURL, maskB64: stripDataURL(canvas.toDataURL("image/png")) }; + const normalizedMaskB64 = stripDataURL(canvas.toDataURL("image/png")); + assertBase64WithinLimit(normalizedMaskB64, "蒙版 PNG"); + return { sourceDataURL, sourceUploadDataURL, maskB64: normalizedMaskB64 }; } export async function compositeMaskedEdit( @@ -108,3 +121,24 @@ function stripDataURL(value: string): string { const comma = value.indexOf(","); return comma >= 0 ? value.slice(comma + 1) : value; } + +export function assertBase64WithinLimit(value: string, label: string): string { + const encoded = String(value || "").replace(/\s+/g, ""); + if (!encoded) throw new Error(`${label}为空`); + if (!/^[A-Za-z0-9+/]*={0,2}$/.test(encoded) || encoded.length % 4 === 1) { + throw new Error(`${label} base64 无效`); + } + const padding = encoded.endsWith("==") ? 2 : encoded.endsWith("=") ? 1 : 0; + const decodedBytes = Math.floor((encoded.length * 3) / 4) - padding; + if (decodedBytes <= 0) throw new Error(`${label}为空`); + if (decodedBytes > MAX_MASK_INPUT_BYTES) { + throw new Error(`${label}过大(${decodedBytes}B > ${MAX_MASK_INPUT_BYTES}B 上限)`); + } + return encoded; +} + +export function assertImageDataURLWithinLimit(value: string, label: string): { mimeType: string; b64: string } { + const match = /^data:(image\/(?:png|jpeg|webp));base64,(.+)$/is.exec(String(value || "").trim()); + if (!match) throw new Error(`${label}不是支持的 PNG/JPEG/WebP base64 data URL`); + return { mimeType: match[1].toLowerCase(), b64: assertBase64WithinLimit(match[2], label) }; +} diff --git a/image-studio/frontend/src/lib/maskEditor.ts b/image-studio/frontend/src/lib/maskEditor.ts new file mode 100644 index 000000000..efb1e60bc --- /dev/null +++ b/image-studio/frontend/src/lib/maskEditor.ts @@ -0,0 +1,187 @@ +export type MaskDimensions = { + w: number; + h: number; +}; + +export type MaskStrokeInput = { + points: number[]; + size: number; + erase?: boolean; +}; + +export type RenderMaskOptions = { + dimensions: MaskDimensions; + strokes: MaskStrokeInput[]; + baseMaskDataURL?: string | null; + invert?: boolean; + allowEmpty?: boolean; +}; + +export function normalizeMaskPixels(pixels: Uint8ClampedArray): boolean { + let hasTransparency = false; + for (let offset = 3; offset < pixels.length; offset += 4) { + if (pixels[offset] < 255) { + hasTransparency = true; + break; + } + } + + let hasEditablePixels = false; + for (let offset = 0; offset < pixels.length; offset += 4) { + if (!hasTransparency) { + const luma = Math.round( + pixels[offset] * 0.299 + + pixels[offset + 1] * 0.587 + + pixels[offset + 2] * 0.114, + ); + pixels[offset + 3] = 255 - luma; + } + pixels[offset] = 0; + pixels[offset + 1] = 0; + pixels[offset + 2] = 0; + if (pixels[offset + 3] < 255) hasEditablePixels = true; + } + return hasEditablePixels; +} + +export function invertMaskPixels(pixels: Uint8ClampedArray): boolean { + let hasEditablePixels = false; + for (let offset = 0; offset < pixels.length; offset += 4) { + pixels[offset] = 0; + pixels[offset + 1] = 0; + pixels[offset + 2] = 0; + pixels[offset + 3] = 255 - pixels[offset + 3]; + if (pixels[offset + 3] < 255) hasEditablePixels = true; + } + return hasEditablePixels; +} + +export async function normalizeImportedMaskDataURL(dataURL: string): Promise { + const image = await loadImage(dataURL); + return renderMaskPNGDataURL({ + dimensions: { w: image.naturalWidth, h: image.naturalHeight }, + strokes: [], + baseMaskDataURL: dataURL, + }); +} + +export async function createFullEditableMaskDataURL(dimensions: MaskDimensions): Promise { + const dataURL = await renderMaskPNGDataURL({ dimensions, strokes: [], invert: true }); + if (!dataURL) throw new Error("无法创建全选蒙版"); + return dataURL; +} + +export async function renderMaskPNGDataURL({ + dimensions, + strokes, + baseMaskDataURL, + invert = false, + allowEmpty = false, +}: RenderMaskOptions): Promise { + const canvas = makeCanvas(dimensions); + const context = requireContext(canvas); + + if (baseMaskDataURL) { + const baseMask = await loadImage(baseMaskDataURL); + context.clearRect(0, 0, canvas.width, canvas.height); + context.drawImage(baseMask, 0, 0, canvas.width, canvas.height); + const basePixels = context.getImageData(0, 0, canvas.width, canvas.height); + normalizeMaskPixels(basePixels.data); + context.putImageData(basePixels, 0, 0); + } else { + context.fillStyle = "#000"; + context.fillRect(0, 0, canvas.width, canvas.height); + } + + context.lineCap = "round"; + context.lineJoin = "round"; + for (const stroke of strokes) drawStroke(context, stroke); + context.globalCompositeOperation = "source-over"; + + const finalPixels = context.getImageData(0, 0, canvas.width, canvas.height); + const hasEditablePixels = invert + ? invertMaskPixels(finalPixels.data) + : hasTransparentPixel(finalPixels.data); + if (!hasEditablePixels && !allowEmpty) return null; + if (invert) context.putImageData(finalPixels, 0, 0); + return canvas.toDataURL("image/png"); +} + +export async function loadImageDimensionsFromSource(source: string): Promise { + if (!source.trim()) return null; + try { + const image = await loadImage(source); + if (image.naturalWidth <= 0 || image.naturalHeight <= 0) return null; + return { w: image.naturalWidth, h: image.naturalHeight }; + } catch { + return null; + } +} + +function drawStroke(context: CanvasRenderingContext2D, stroke: MaskStrokeInput) { + const points = finitePointPairs(stroke.points); + if (points.length === 0) return; + const size = Number.isFinite(stroke.size) ? Math.max(1, stroke.size) : 1; + context.globalCompositeOperation = stroke.erase ? "source-over" : "destination-out"; + context.fillStyle = "#000"; + context.strokeStyle = "#000"; + context.lineWidth = size; + + if (points.length === 1) { + context.beginPath(); + context.arc(points[0].x, points[0].y, size / 2, 0, Math.PI * 2); + context.fill(); + return; + } + + context.beginPath(); + context.moveTo(points[0].x, points[0].y); + for (let index = 1; index < points.length; index += 1) { + context.lineTo(points[index].x, points[index].y); + } + context.stroke(); +} + +function finitePointPairs(points: number[]): Array<{ x: number; y: number }> { + const result: Array<{ x: number; y: number }> = []; + for (let index = 0; index + 1 < points.length; index += 2) { + const x = points[index]; + const y = points[index + 1]; + if (Number.isFinite(x) && Number.isFinite(y)) result.push({ x, y }); + } + return result; +} + +function hasTransparentPixel(pixels: Uint8ClampedArray): boolean { + for (let offset = 3; offset < pixels.length; offset += 4) { + if (pixels[offset] < 255) return true; + } + return false; +} + +function makeCanvas(dimensions: MaskDimensions): HTMLCanvasElement { + const width = Math.round(dimensions.w); + const height = Math.round(dimensions.h); + if (!Number.isFinite(width) || !Number.isFinite(height) || width <= 0 || height <= 0) { + throw new Error("蒙版源图尺寸无效"); + } + const canvas = document.createElement("canvas"); + canvas.width = width; + canvas.height = height; + return canvas; +} + +function requireContext(canvas: HTMLCanvasElement): CanvasRenderingContext2D { + const context = canvas.getContext("2d", { willReadFrequently: true }); + if (!context) throw new Error("无法创建蒙版编辑画布"); + return context; +} + +function loadImage(source: string): Promise { + return new Promise((resolve, reject) => { + const image = new Image(); + image.onload = () => resolve(image); + image.onerror = () => reject(new Error("无法解码蒙版图片")); + image.src = source; + }); +} diff --git a/image-studio/frontend/src/platform/android/canvas/AndroidCanvasStage.tsx b/image-studio/frontend/src/platform/android/canvas/AndroidCanvasStage.tsx index 2efd228fa..73d22fe76 100644 --- a/image-studio/frontend/src/platform/android/canvas/AndroidCanvasStage.tsx +++ b/image-studio/frontend/src/platform/android/canvas/AndroidCanvasStage.tsx @@ -13,6 +13,7 @@ import { useCanvasShortcuts } from "../../../components/canvas/useCanvasShortcut import { historyFullSrc, orderedNavigationItemsForCurrent, sortHistoryItemsByCreatedAtAsc } from "../../../lib/images"; import { streamPreviewItemsFromPreviews } from "../../../state/studioStore.streamPreview"; import { vibrateForPlatform } from "../bridge"; +import { MaskOverlayLayer } from "../../../components/canvas/MaskOverlayLayer"; type ViewState = { scale: number; x: number; y: number }; type PinchState = { @@ -31,8 +32,8 @@ export function AndroidCanvasStage() { annotationKind, annotationColor, selectedAnnotationId, annotations, addAnnotation, removeAnnotation, - setMaskDataURL, maskDataURL, + maskVisible, maskOpacity, activateMaskTool, strokes, pushStroke, undo, redo, compareB, compareSplit, setCompareSplit, setCompareB, @@ -43,6 +44,7 @@ export function AndroidCanvasStage() { jobsTotal, jobsCompleted, activeWorkspaceId, + fullscreen, toggleFullscreen, history, batchResults, resultGridOpen, selectBatchResult, closeResultGrid, @@ -91,8 +93,7 @@ export function AndroidCanvasStage() { const currentImageURL = historyFullSrc(currentImage, null); const image = useImageFromSource(currentImage?.imageBlob ?? null, currentImage?.imageB64, currentImageURL); - const importedMaskURL = maskDataURL && maskDataURL !== "__PENDING_MASK__" ? maskDataURL : null; - const importedMaskImage = useImageFromSource(null, undefined, importedMaskURL); + const importedMaskImage = useImageFromSource(null, undefined, maskDataURL); useEffect(() => { if (!currentImage?.previewOnly) return; @@ -227,25 +228,78 @@ export function AndroidCanvasStage() { } const drawingRef = useRef<{ active: boolean; current: Stroke | null }>({ active: false, current: null }); + const previousImageIDRef = useRef(currentImage?.id); + const previousCanvasResetTickRef = useRef(canvasViewResetTick); const freehandRef = useRef(null); const [, setDrawingTick] = useState(0); + const [maskCursor, setMaskCursor] = useState<{ x: number; y: number } | null>(null); const [drag, setDrag] = useState(null); + useEffect(() => { + const imageChanged = previousImageIDRef.current !== currentImage?.id; + previousImageIDRef.current = currentImage?.id; + setUserView(null); + if (imageChanged) { + useStudioStore.setState({ + maskDataURL: null, + maskTargetPath: null, + strokes: [], + annotations: [], + selectedAnnotationId: null, + undoStack: [], + redoStack: [], + }); + } + drawingRef.current = { active: false, current: null }; + freehandRef.current = null; + setMaskCursor(null); + setDrag(null); + pinchRef.current = null; + setPinching(false); + }, [currentImage?.id]); + + useEffect(() => { + const canvasChanged = previousCanvasResetTickRef.current !== canvasViewResetTick; + previousCanvasResetTickRef.current = canvasViewResetTick; + if (!canvasChanged) return; + setUserView(null); + useStudioStore.setState({ + maskDataURL: null, + maskTargetPath: null, + strokes: [], + annotations: [], + selectedAnnotationId: null, + undoStack: [], + redoStack: [], + }); + drawingRef.current = { active: false, current: null }; + freehandRef.current = null; + setMaskCursor(null); + setDrag(null); + pinchRef.current = null; + setPinching(false); + }, [canvasViewResetTick]); + useEffect(() => { setUserView(null); - setMaskDataURL(null); drawingRef.current = { active: false, current: null }; freehandRef.current = null; + setMaskCursor(null); setDrag(null); pinchRef.current = null; setPinching(false); - }, [currentImage?.id, canvasViewResetTick, setMaskDataURL]); + }, [fullscreen]); + + function isInsideImage(point: { x: number; y: number }): boolean { + return !!image && point.x >= 0 && point.y >= 0 && point.x <= image.width && point.y <= image.height; + } function beginPointer(e: Konva.KonvaEventObject) { if (!image || pinching) return; const local = stagePointerToImageCoord(); - if (!local) return; + if (!local || !isInsideImage(local)) return; if (effectiveTool === "mask") { + setMaskCursor(local); drawingRef.current = { active: true, current: { points: [local.x, local.y], size: brushSize, erase: brushMode === "erase" } }; vibrateForPlatform(4); } else if (effectiveTool === "annotate") { @@ -278,6 +332,12 @@ export function AndroidCanvasStage() { if (!image || pinching) return; const local = stagePointerToImageCoord(); if (!local) return; + const insideImage = isInsideImage(local); + setMaskCursor(effectiveTool === "mask" && insideImage ? local : null); + if (!insideImage) { + if (effectiveTool === "mask" && drawingRef.current.active) finishMaskStroke(); + return; + } if (effectiveTool === "mask" && drawingRef.current.active && drawingRef.current.current) { drawingRef.current.current.points.push(local.x, local.y); setDrawingTick((n) => n + 1); @@ -291,10 +351,9 @@ export function AndroidCanvasStage() { function endPointer() { if (pinching) return; - if (effectiveTool === "mask" && drawingRef.current.active && drawingRef.current.current) { - const finished = drawingRef.current.current; - drawingRef.current = { active: false, current: null }; - pushStroke(finished); + setMaskCursor(null); + if (effectiveTool === "mask" && drawingRef.current.active) { + finishMaskStroke(); } else if (effectiveTool === "annotate" && annotationKind === "freehand" && freehandRef.current) { const pts = freehandRef.current; freehandRef.current = null; @@ -338,6 +397,17 @@ export function AndroidCanvasStage() { } } + function finishMaskStroke() { + const finished = drawingRef.current.current; + drawingRef.current = { active: false, current: null }; + if (finished) pushStroke(finished); + } + + function leavePointer() { + setMaskCursor(null); + endPointer(); + } + function touchPoint(touch: Touch) { const rect = stageRef.current?.container().getBoundingClientRect(); return { @@ -361,6 +431,7 @@ export function AndroidCanvasStage() { const b = touchPoint(e.evt.touches[1]); drawingRef.current = { active: false, current: null }; freehandRef.current = null; + setMaskCursor(null); setDrag(null); pinchRef.current = { distance: distance(a, b), @@ -407,20 +478,6 @@ export function AndroidCanvasStage() { zoomAround(pointer, direction > 0 ? oldScale * factor : oldScale / factor); } - useEffect(() => { - const hasImportedMask = !!maskDataURL && maskDataURL !== "__PENDING_MASK__"; - if (!image) { - setMaskDataURL(null); - return; - } - if (strokes.length === 0) { - if (!hasImportedMask) setMaskDataURL(null); - return; - } - const hasWhite = strokes.some((s) => !s.erase); - setMaskDataURL(hasWhite ? "__PENDING_MASK__" : null); - }, [strokes, image, maskDataURL, setMaskDataURL]); - useEffect(() => { (window as any).__canvasResetView = resetView; (window as any).__androidCanvasResetView = resetView; @@ -436,6 +493,7 @@ export function AndroidCanvasStage() { useCanvasShortcuts({ brushSize, + brushMode, canNavigateBatchResults, cancel, compareB, @@ -464,11 +522,16 @@ export function AndroidCanvasStage() { resetView, selectedAnnotationId, setBrushSize: (value) => setField("brushSize", value), + setBrushMode: (value) => setField("brushMode", value), setCompareB, setErrorMessage: (value) => setField("errorMessage", value), toggleFullscreen, setSelectedAnnotationId: (value) => setField("selectedAnnotationId", value), - setTool: (value) => setField("tool", value), + setTool: (value) => { + if (value === "mask") void activateMaskTool(); + else setField("tool", value); + }, + tool, undo, }); @@ -476,7 +539,13 @@ export function AndroidCanvasStage() {
    {showingResultGrid ? ( : null} - - {image && importedMaskImage ? ( - - ) : null} - {strokes.map((s, i) => ( - - ))} - {drawingRef.current.current ? ( - - ) : null} - + {image ? ( + + ) : null} {annotations.map((a) => ( diff --git a/image-studio/frontend/src/platform/android/canvas/AndroidCanvasWorkspace.tsx b/image-studio/frontend/src/platform/android/canvas/AndroidCanvasWorkspace.tsx index 220267b33..bd3bc4709 100644 --- a/image-studio/frontend/src/platform/android/canvas/AndroidCanvasWorkspace.tsx +++ b/image-studio/frontend/src/platform/android/canvas/AndroidCanvasWorkspace.tsx @@ -2,8 +2,11 @@ import { ArrowRight, Brush, CheckCircle2, + Contrast, Crop, Eraser, + Eye, + EyeOff, Expand, FlipHorizontal, FlipVertical, @@ -16,6 +19,7 @@ import { ZoomIn, ZoomOut, Pencil, + PaintBucket, RotateCcw, RotateCw, Save, @@ -49,6 +53,8 @@ export function AndroidCanvasWorkspace() { currentImage, mode, sources, + provider, + batchProcess, isRunning, progress, streamPreview, @@ -59,6 +65,10 @@ export function AndroidCanvasWorkspace() { tool, brushMode, brushSize, + maskDataURL, + maskVisible, + maskOpacity, + strokes, annotationKind, annotationColor, annotations, @@ -72,7 +82,10 @@ export function AndroidCanvasWorkspace() { setField, undo, redo, + activateMaskTool, importMaskImage, + fillMask, + invertMask, resetMask, clearAnnotations, saveCurrentImageAs, @@ -95,6 +108,8 @@ export function AndroidCanvasWorkspace() { const { isAndroidPad, androidOrientation } = usePlatform(); const [sourceOpen, setSourceOpen] = useState(true); const hasImage = !!currentImage; + const hasMask = !!maskDataURL || strokes.some((stroke) => !stroke.erase); + const maskDisabled = providerDoesNotSupportMask(provider) || batchProcess.enabled; const hasSources = mode === "edit" && sources.length > 0; const selRect = annotations.find((a) => a.id === selectedAnnotationId && a.kind === "rect"); const cropAction = selRect && selRect.width && selRect.height @@ -109,6 +124,10 @@ export function AndroidCanvasWorkspace() { if (hasSources) setSourceOpen(true); }, [hasSources]); + useEffect(() => { + if (tool === "mask" && maskDisabled) setField("tool", "pan"); + }, [maskDisabled, setField, tool]); + const runAction = (action: () => void | Promise, vibration = 8) => { vibrateForPlatform(vibration); void action(); @@ -242,7 +261,11 @@ export function AndroidCanvasWorkspace() { runAction(() => setField("tool", next), 8)} + maskDisabled={maskDisabled} + onChange={(next) => runAction(async () => { + if (next === "mask") await activateMaskTool(); + else setField("tool", next); + }, 8)} />
    runAction(undo, 6)}> @@ -257,9 +280,16 @@ export function AndroidCanvasWorkspace() { runAction(() => setField("brushMode", next), 5)} onSetBrushSize={(next) => setField("brushSize", next)} + onToggleMaskVisible={() => runAction(() => setField("maskVisible", !maskVisible), 4)} + onSetMaskOpacity={(next) => setField("maskOpacity", next)} onImportMask={() => runAction(importMaskImage, 8)} + onFillMask={() => runAction(fillMask, 8)} + onInvertMask={() => runAction(invertMask, 8)} onResetMask={() => runAction(resetMask, 6)} /> ) : null} @@ -443,10 +473,12 @@ function AndroidCanvasHeader({ function AndroidToolSegment({ value, disabled, + maskDisabled, onChange, }: { value: CanvasTool; disabled: boolean; + maskDisabled: boolean; onChange: (value: CanvasTool) => void; }) { return ( @@ -454,7 +486,7 @@ function AndroidToolSegment({ onChange("pan")}> - onChange("mask")}> + onChange("mask")}> onChange("annotate")}> @@ -467,16 +499,30 @@ function AndroidToolSegment({ function AndroidMaskControls({ brushMode, brushSize, + maskVisible, + maskOpacity, + hasMask, onSetBrushMode, onSetBrushSize, + onToggleMaskVisible, + onSetMaskOpacity, onImportMask, + onFillMask, + onInvertMask, onResetMask, }: { brushMode: BrushMode; brushSize: number; + maskVisible: boolean; + maskOpacity: number; + hasMask: boolean; onSetBrushMode: (value: BrushMode) => void; onSetBrushSize: (value: number) => void; + onToggleMaskVisible: () => void; + onSetMaskOpacity: (value: number) => void; onImportMask: () => void; + onFillMask: () => void; + onInvertMask: () => void; onResetMask: () => void; }) { return ( @@ -491,7 +537,16 @@ function AndroidMaskControls({ -
    @@ -506,10 +561,26 @@ function AndroidMaskControls({ /> {brushSize} +
    ); } +function providerDoesNotSupportMask(provider: "openai" | "google" | "grok") { + return provider === "google" || provider === "grok"; +} + function AndroidAnnotationControls({ annotationKind, annotationColor, diff --git a/image-studio/frontend/src/platform/runtime/host.ts b/image-studio/frontend/src/platform/runtime/host.ts index 683b0f07c..14aef3113 100644 --- a/image-studio/frontend/src/platform/runtime/host.ts +++ b/image-studio/frontend/src/platform/runtime/host.ts @@ -163,12 +163,16 @@ async function startRemoteJob(options: GenerateOptionsLike): Promise emitLocalEvent(`log:${jobId}`, line), onProgress: (stage, elapsed, bytes) => emitLocalEvent(`progress:${jobId}`, { stage, elapsed, bytes }), diff --git a/image-studio/frontend/src/platform/runtime/remote-kernel/requestPayloads.ts b/image-studio/frontend/src/platform/runtime/remote-kernel/requestPayloads.ts index 558441868..15f552679 100644 --- a/image-studio/frontend/src/platform/runtime/remote-kernel/requestPayloads.ts +++ b/image-studio/frontend/src/platform/runtime/remote-kernel/requestPayloads.ts @@ -2,6 +2,11 @@ import { detectImageMimeTypeFromBase64, imageExtensionForMimeType, } from "../../../lib/images.ts"; +import { + assertBase64WithinLimit, + assertImageDataURLWithinLimit, + normalizeMaskForSource, +} from "../../../lib/maskComposite.ts"; import { buildResponsesPayload as buildSharedResponsesPayload, normalizeBackground, @@ -30,7 +35,6 @@ import { RemoteKernelError, type RemoteGeneratePayload, type RemoteJobRequest } export type ImagesRequestProtocol = "openai-images" | "google-interactions" | "grok-images"; -const MAX_MASK_IMAGE_BYTES = 50 * 1024 * 1024; const GOOGLE_INTERACTION_ASPECT_RATIOS = [ "1:8", "1:4", "2:3", "3:4", "4:5", "1:1", "5:4", "4:3", "3:2", "16:9", "21:9", "4:1", "8:1", ] as const; @@ -124,12 +128,17 @@ export async function buildImagesRequestBody( const partialImages = request.payload.disablePreview ? 0 : normalizePartialImages(request.payload.partialImages); const useNewAPICompat = shouldUseImagesNewAPICompat(request.payload); const provider = normalizeUpstreamProvider(request.payload.provider || "openai"); + const maskB64 = String(request.payload.maskB64 || "").trim(); + + if (maskB64 && mode !== "edit") { + throw new RemoteKernelError("蒙版仅支持图生图模式"); + } if (shouldUseGoogleNativeInteractions(baseURL, imageModel, provider)) { if (mode === "edit" && sourceDataURLs.length === 0) { throw new RemoteKernelError("Google Interactions 图生图需要至少一张源图"); } - if (request.payload.maskB64) { + if (maskB64) { throw new RemoteKernelError("Google Interactions 不支持 OpenAI mask 参数；请清除蒙版，或改用支持 /v1/images/edits 的中转站"); } return { @@ -152,7 +161,7 @@ export async function buildImagesRequestBody( if (mode === "edit" && sourceDataURLs.length === 0) { throw new RemoteKernelError("Grok Imagine 图生图需要至少一张源图"); } - if (request.payload.maskB64) { + if (maskB64) { throw new RemoteKernelError("Grok Imagine 不支持 OpenAI mask 参数；请清除蒙版后重试"); } const dimensions = grokImageDimensions(size); @@ -178,23 +187,34 @@ export async function buildImagesRequestBody( if (sourceDataURLs.length === 0) { throw new RemoteKernelError("图生图模式需要至少一张源图(请先添加参考图)"); } + let uploadSourceDataURLs = sourceDataURLs; + let uploadMaskB64 = maskB64; + if (maskB64) { + let prepared = request.preparedMask; + if (!prepared || prepared.maskB64.trim() !== maskB64) { + try { + prepared = await normalizeMaskForSource(sourceDataURLs[0], maskB64); + } catch (error) { + throw new RemoteKernelError(`准备蒙版 PNG 失败: ${String((error as any)?.message || error)}`); + } + } + const sourceUpload = assertPreparedPNGDataURL(prepared.sourceUploadDataURL, "蒙版首张源图"); + uploadMaskB64 = assertPreparedPNGBase64(prepared.maskB64, "蒙版图片"); + uploadSourceDataURLs = [sourceUpload, ...sourceDataURLs.slice(1)]; + } + const form = new FormData(); - for (let i = 0; i < sourceDataURLs.length; i++) { - const dataURL = sourceDataURLs[i]; + for (let i = 0; i < uploadSourceDataURLs.length; i++) { + const dataURL = uploadSourceDataURLs[i]; const payload = dataURL.slice(dataURL.indexOf(",") + 1); const mimeType = dataURL.slice(5, dataURL.indexOf(";")) || "image/png"; const ext = imageExtensionForMimeType(mimeType); - form.append(i === 0 ? "image" : "image[]", new Blob([Uint8Array.from(atob(payload), (ch) => ch.charCodeAt(0))], { type: mimeType }), `source-${i + 1}.${ext}`); + const fieldName = uploadSourceDataURLs.length > 1 ? "image[]" : "image"; + form.append(fieldName, new Blob([Uint8Array.from(atob(payload), (ch) => ch.charCodeAt(0))], { type: mimeType }), `source-${i + 1}.${ext}`); } - if (request.payload.maskB64) { - const maskMime = detectImageMimeTypeFromBase64(request.payload.maskB64); - if (!maskMime) throw new RemoteKernelError("蒙版图片不是支持的 PNG/JPEG/WebP 格式"); - const maskBytes = Uint8Array.from(atob(request.payload.maskB64), (ch) => ch.charCodeAt(0)); - if (maskBytes.byteLength > MAX_MASK_IMAGE_BYTES) { - throw new RemoteKernelError(`蒙版图片过大(${maskBytes.byteLength}B > ${MAX_MASK_IMAGE_BYTES}B 上限)`); - } - const ext = imageExtensionForMimeType(maskMime); - form.append("mask", new Blob([maskBytes], { type: maskMime }), `mask.${ext}`); + if (uploadMaskB64) { + const maskBytes = decodeCheckedBase64(uploadMaskB64, "蒙版图片"); + form.append("mask", new Blob([maskBytes], { type: "image/png" }), "mask.png"); } form.append("prompt", request.payload.prompt); form.append("model", imageModel); @@ -268,3 +288,43 @@ export async function buildImagesRequestBody( body: JSON.stringify(payload), }; } + +function assertPreparedPNGDataURL(value: string, label: string): string { + let parsed: { mimeType: string; b64: string }; + try { + parsed = assertImageDataURLWithinLimit(value, label); + } catch (error) { + throw new RemoteKernelError(String((error as any)?.message || error)); + } + if (parsed.mimeType !== "image/png" || detectImageMimeTypeFromBase64(parsed.b64) !== "image/png") { + throw new RemoteKernelError(`${label}必须是 PNG`); + } + return `data:image/png;base64,${parsed.b64}`; +} + +function assertPreparedPNGBase64(value: string, label: string): string { + let encoded: string; + try { + encoded = assertBase64WithinLimit(value, label); + } catch (error) { + throw new RemoteKernelError(String((error as any)?.message || error)); + } + if (detectImageMimeTypeFromBase64(encoded) !== "image/png") { + throw new RemoteKernelError(`${label}必须是 PNG`); + } + return encoded; +} + +function decodeCheckedBase64(value: string, label: string): ArrayBuffer { + const encoded = assertPreparedPNGBase64(value, label); + try { + const binary = atob(encoded); + const bytes = new Uint8Array(binary.length); + for (let index = 0; index < binary.length; index++) { + bytes[index] = binary.charCodeAt(index); + } + return bytes.buffer; + } catch { + throw new RemoteKernelError(`${label} base64 无效`); + } +} diff --git a/image-studio/frontend/src/platform/runtime/remote-kernel/types.ts b/image-studio/frontend/src/platform/runtime/remote-kernel/types.ts index 252b8b429..d4bc90acb 100644 --- a/image-studio/frontend/src/platform/runtime/remote-kernel/types.ts +++ b/image-studio/frontend/src/platform/runtime/remote-kernel/types.ts @@ -1,4 +1,5 @@ import type { RequestPolicy, UpstreamProvider } from "../../../types/domain"; +import type { PreparedMaskComposite } from "../../../lib/maskComposite.ts"; export type KernelImageSource = { path?: string; @@ -70,6 +71,7 @@ export type PartialImageCallback = (partial: { export type RemoteJobRequest = { payload: RemoteGeneratePayload; sourceImages?: KernelImageSource[]; + preparedMask?: PreparedMaskComposite | null; }; export type RemoteJobCallbacks = { diff --git a/image-studio/frontend/src/state/studioStore.images.ts b/image-studio/frontend/src/state/studioStore.images.ts index 4bd981231..0dcc6ff54 100644 --- a/image-studio/frontend/src/state/studioStore.images.ts +++ b/image-studio/frontend/src/state/studioStore.images.ts @@ -187,6 +187,13 @@ export function createImageActions(store: StateAdapter) { editSourceMode: "manual", currentImage: toPreviewOnlyHistoryItem(localItem), resultGridOpen: false, + maskDataURL: null, + maskTargetPath: null, + strokes: [], + annotations: [], + selectedAnnotationId: null, + undoStack: [], + redoStack: [], sources: alreadyIn ? existing : [...existing, { @@ -317,6 +324,13 @@ export function createImageActions(store: StateAdapter) { currentImage: ref ? { ...importedItem, previewOnly: true } : importedItem, batchResults: [], resultGridOpen: false, + maskDataURL: null, + maskTargetPath: null, + strokes: [], + annotations: [], + selectedAnnotationId: null, + undoStack: [], + redoStack: [], mode: "edit", editSourceMode: "manual", size: alreadyIn || existingSources.length > 0 ? store.getState().size : "auto", diff --git a/image-studio/frontend/src/state/studioStore.media.ts b/image-studio/frontend/src/state/studioStore.media.ts index 41a05593e..ef53e73c8 100644 --- a/image-studio/frontend/src/state/studioStore.media.ts +++ b/image-studio/frontend/src/state/studioStore.media.ts @@ -93,7 +93,11 @@ export function createMediaActions(store: StateAdapter) { resultGridOpen: false, compareB: null, maskDataURL: null, + maskTargetPath: null, + strokes: [], annotations: [], + undoStack: [], + redoStack: [], tool: "pan", workspaces: patchWorkspaceRuntime(state.workspaces, state.activeWorkspaceId, { currentImageId: full.id, diff --git a/image-studio/frontend/src/state/studioStore.shared.ts b/image-studio/frontend/src/state/studioStore.shared.ts index 765cad007..2dcac3f3d 100644 --- a/image-studio/frontend/src/state/studioStore.shared.ts +++ b/image-studio/frontend/src/state/studioStore.shared.ts @@ -5,7 +5,7 @@ import { RegisterTrustedOutputDir, } from "../platform/runtime/host"; import type { ThemeMode, HistoryItem, Annotation } from "../types/domain"; -import type { ModeConfig, Stroke } from "./studioStore.types"; +import type { ModeConfig } from "./studioStore.types"; import { isWindows } from "../platform"; import { ACTIVE_PROFILE_LS_KEY, PROFILES_LS_KEY, tryParseProfile } from "../lib/profiles"; export { loadStoredAIProfileId, persistAIProfileId } from "../lib/profiles"; @@ -149,45 +149,6 @@ export function stripDataURLPrefix(dataURL: string): string { return idx >= 0 ? dataURL.slice(idx + 1) : dataURL; } -export function buildMaskPNGDataURL(strokes: Stroke[], dims: { w: number; h: number } | null): string | null { - if (!dims || strokes.length === 0) return null; - const c = document.createElement("canvas"); - c.width = dims.w; - c.height = dims.h; - const ctx = c.getContext("2d"); - if (!ctx) return null; - // OpenAI masks use alpha: transparent pixels are editable, opaque pixels - // protect the source. Keep RGB black so only alpha carries mask semantics. - ctx.fillStyle = "rgba(0,0,0,1)"; - ctx.fillRect(0, 0, c.width, c.height); - ctx.lineCap = "round"; - ctx.lineJoin = "round"; - let hasPaint = false; - for (const s of strokes) { - ctx.globalCompositeOperation = s.erase ? "source-over" : "destination-out"; - ctx.strokeStyle = "#000"; - ctx.lineWidth = s.size; - ctx.beginPath(); - if (s.points.length === 2) { - ctx.arc(s.points[0], s.points[1], s.size / 2, 0, Math.PI * 2); - ctx.fillStyle = "#000"; - ctx.fill(); - if (!s.erase) hasPaint = true; - continue; - } - for (let i = 0; i < s.points.length; i += 2) { - const x = s.points[i]; - const y = s.points[i + 1]; - if (i === 0) ctx.moveTo(x, y); - else ctx.lineTo(x, y); - } - ctx.stroke(); - if (!s.erase) hasPaint = true; - } - ctx.globalCompositeOperation = "source-over"; - return hasPaint ? c.toDataURL("image/png") : null; -} - export async function registerTrustedOutputRoots(roots: string[]): Promise { for (const root of roots) { if (!root.trim()) continue; @@ -241,5 +202,5 @@ export function augmentPromptWithAnnotations( return `${vPart}${hPart}部`; }; const positions = rects.map(describe).join("、"); - return `${prompt}\n(请重点关注${positions}标注区域)`; + return `${prompt}\n(请重点关注第 1 张参考图的${positions}标注区域)`; } diff --git a/image-studio/frontend/src/state/studioStore.ts b/image-studio/frontend/src/state/studioStore.ts index c141d635e..8553b3348 100644 --- a/image-studio/frontend/src/state/studioStore.ts +++ b/image-studio/frontend/src/state/studioStore.ts @@ -76,7 +76,14 @@ import { import { normalizeAppUpdateInfo } from "../lib/appUpdate.ts"; import { appVersion } from "../lib/version.ts"; import { normalizeSavePromptRequest, type SavePromptRequest } from "../lib/savePromptState"; -import { detectImageMimeTypeFromBase64 } from "../lib/images"; +import { detectImageMimeTypeFromBase64, prioritizeImageSource } from "../lib/images"; +import { + createFullEditableMaskDataURL, + loadImageDimensionsFromSource, + normalizeImportedMaskDataURL, + renderMaskPNGDataURL, + type MaskDimensions, +} from "../lib/maskEditor"; import { readSavePromptSuppressed, writeSavePromptSuppressed, @@ -159,7 +166,6 @@ import { buildMacWorkspacePreview, buildWindowsRightRailPreview, readPreviewScen import { applyTheme, augmentPromptWithAnnotations, - buildMaskPNGDataURL, clearLegacyModeLocalStorage, genId, imageDims, @@ -685,6 +691,9 @@ export const useStudioStore = create((set, get) => ({ annotationColor: "#ff4d4d", selectedAnnotationId: null, maskDataURL: null, + maskTargetPath: null, + maskVisible: true, + maskOpacity: 0.45, strokes: [], annotations: [], undoStack: [], @@ -978,13 +987,24 @@ export const useStudioStore = create((set, get) => ({ : key === "loopGeneration" ? normalizeLoopGenerationConfig(value) : value; + const previousCurrentImageId = key === "currentImage" ? get().currentImage?.id ?? null : null; set({ [key]: normalizedValue } as any); if (key === "currentImage") { const item = normalizedValue as HistoryItem | null; + const currentImageChanged = previousCurrentImageId !== (item?.id ?? null); const workspace = get().workspaces.find((w) => w.id === get().activeWorkspaceId); set({ compareB: null, resultGridOpen: false, + ...(currentImageChanged ? { + maskDataURL: null, + maskTargetPath: null, + strokes: [], + annotations: [], + selectedAnnotationId: null, + undoStack: [], + redoStack: [], + } : {}), workspaces: patchWorkspaceRuntime(get().workspaces, get().activeWorkspaceId, { currentImageId: currentImageIdForWorkspaceSnapshot(item, get().streamPreview, get().streamPreviews, workspace?.currentImageId ?? null), resultGridOpen: false, @@ -1085,7 +1105,6 @@ export const useStudioStore = create((set, get) => ({ get().pushToast(message, "error", 6000); } finally { dispatchFullscreenResize(); - set((state) => ({ canvasViewResetTick: state.canvasViewResetTick + 1 })); } }, toggleFullscreen: async () => { @@ -1229,9 +1248,43 @@ export const useStudioStore = create((set, get) => ({ return; } } + let effectiveSources = s.sources; + let annotationDimensions: { w: number; h: number } | null = null; + if (s.mode === "edit" && !batchProcessEnabled && s.annotations.length > 0 && s.currentImage) { + let annotationTarget = await materializeHistoryItem(s.currentImage).catch(() => null); + annotationTarget = await ensureFullHistoryItem(annotationTarget).catch(() => annotationTarget); + const targetPath = annotationTarget?.savedPath?.trim() ?? ""; + if (!annotationTarget || !targetPath) { + set({ + errorMessage: "无法读取标注所在的画布图。请重新导入图片后再添加参考图。", + errorCanRetry: false, + errorRawPath: null, + }); + return; + } + const existingTarget = s.sources.find((source) => source.path === targetPath); + const targetSource: SourceImage = existingTarget ?? { + path: targetPath, + name: targetPath.split(/[\\/]/).pop() ?? "annotation-source.png", + size: 0, + imageBlob: annotationTarget.previewUrl ? null : (annotationTarget.previewBlob ?? annotationTarget.imageBlob ?? null), + imageB64: annotationTarget.previewUrl ? undefined : annotationTarget.imageB64, + previewUrl: annotationTarget.previewUrl, + previewWidth: annotationTarget.previewWidth, + previewHeight: annotationTarget.previewHeight, + }; + effectiveSources = prioritizeImageSource(s.sources, targetSource); + + const targetB64 = annotationTarget.imageB64 || await ReadImageAsBase64(targetPath).catch(() => ""); + annotationDimensions = await loadImageDims(targetB64); + if (!annotationDimensions) { + annotationDimensions = await loadImageDimensionsFromSource(annotationTarget.fullUrl || annotationTarget.previewUrl || ""); + } + } + let editSourcePaths: string[] = []; if (s.mode === "edit" && !batchProcessEnabled) { - editSourcePaths = s.sources.map((src) => src.path).filter(Boolean); + editSourcePaths = effectiveSources.map((src) => src.path).filter(Boolean); if (editSourcePaths.length === 0 && s.currentImage) { const materialized = await materializeHistoryItem(s.currentImage).catch(() => null); if (materialized?.savedPath) { @@ -1266,25 +1319,67 @@ export const useStudioStore = create((set, get) => ({ jobsCompleted: 0, runningJobs: [], }; - const importedMaskDataURL = s.maskDataURL && s.maskDataURL !== "__PENDING_MASK__" ? s.maskDataURL : null; - let maskTargetDims = s.currentImage?.imageB64 ? await loadImageDims(s.currentImage.imageB64) : null; - if (s.mode === "edit" && s.strokes.length > 0 && !maskTargetDims && editSourcePaths[0]) { - const sourceB64 = await ReadImageAsBase64(editSourcePaths[0]).catch(() => ""); - maskTargetDims = await loadImageDims(sourceB64); + const hasMaskDraft = !!s.maskDataURL || s.strokes.some((stroke) => !stroke.erase); + if (hasMaskDraft && (s.provider === "google" || s.provider === "grok")) { + set({ + errorMessage: `${s.provider === "google" ? "Google Interactions" : "Grok Imagine"} 不支持 OpenAI 蒙版参数，请清空蒙版或切换上游`, + errorCanRetry: false, + errorRawPath: null, + }); + return; } - if (s.mode === "edit" && s.strokes.length > 0 && !maskTargetDims) { + if (hasMaskDraft && s.mode !== "edit") { set({ - errorMessage: "无法读取源图真实尺寸，未提交蒙版任务。请重新导入源图后再试。", + errorMessage: "蒙版只能用于图生图。请重新进入蒙版工具，让当前画布图成为主参考图。", errorCanRetry: false, errorRawPath: null, }); return; } - const maskDataURL = s.mode === "edit" - ? (s.strokes.length > 0 - ? buildMaskPNGDataURL(s.strokes, maskTargetDims) - : importedMaskDataURL) - : null; + if (hasMaskDraft && batchProcessEnabled) { + set({ + errorMessage: "批量目录处理不支持复用同一张蒙版。请关闭批量处理后再提交。", + errorCanRetry: false, + errorRawPath: null, + }); + return; + } + + let maskDataURL: string | null = null; + if (hasMaskDraft) { + const maskTargetPath = s.maskTargetPath || editSourcePaths[0] || ""; + if (!maskTargetPath || maskTargetPath !== editSourcePaths[0]) { + set({ + errorMessage: "蒙版目标已不是第一张参考图。请重新进入蒙版工具确认目标后再提交。", + errorCanRetry: false, + errorRawPath: null, + }); + return; + } + const sourceB64 = await ReadImageAsBase64(maskTargetPath).catch(() => ""); + const maskTargetDims = await loadImageDims(sourceB64); + if (!maskTargetDims) { + set({ + errorMessage: "无法读取蒙版目标图的真实尺寸。请重新导入源图后再试。", + errorCanRetry: false, + errorRawPath: null, + }); + return; + } + maskDataURL = await renderMaskPNGDataURL({ + dimensions: maskTargetDims, + baseMaskDataURL: s.maskDataURL, + strokes: s.strokes, + }).catch(() => null); + if (!maskDataURL) { + set({ + errorMessage: "当前蒙版没有可编辑区域，请先涂抹需要修改的范围。", + errorCanRetry: false, + errorRawPath: null, + }); + return; + } + } const maskB64 = maskDataURL ? stripDataURLPrefix(maskDataURL) : ""; set({ ...runPatch, @@ -1293,9 +1388,7 @@ export const useStudioStore = create((set, get) => ({ resultGridOpen: requestedJobCount > 1, compareB: null, currentImage: clearCurrentForNewRun ? null : s.currentImage, - maskDataURL: null, annotations: [], - strokes: [], workspaces: patchWorkspaceRuntime(s.workspaces, workspaceId, { ...runPatch, currentImageId: clearCurrentForNewRun ? null : s.currentImage?.id ?? null, @@ -1303,7 +1396,11 @@ export const useStudioStore = create((set, get) => ({ resultGridOpen: requestedJobCount > 1, }), }); - let augmentedPrompt = augmentPromptWithAnnotations(s.prompt, s.annotations, s.currentImage?.imageB64 ? imageDims(s.currentImage.imageB64) : null); + let augmentedPrompt = augmentPromptWithAnnotations( + s.prompt, + s.annotations, + annotationDimensions ?? (s.currentImage?.imageB64 ? imageDims(s.currentImage.imageB64) : null), + ); // Append style chip suffix if the user picked one (other than "全部"). const styleSuffix = STYLE_SUFFIXES[s.styleTag]; if (styleSuffix) { @@ -1326,8 +1423,8 @@ export const useStudioStore = create((set, get) => ({ const effectiveEditAutoAspectResolution = normalizedEditAutoAspectResolution === "auto" ? "1k" : normalizedEditAutoAspectResolution; - const editReferenceDimensions = s.sources[0]?.previewWidth && s.sources[0]?.previewHeight - ? { width: s.sources[0].previewWidth, height: s.sources[0].previewHeight } + const editReferenceDimensions = effectiveSources[0]?.previewWidth && effectiveSources[0]?.previewHeight + ? { width: effectiveSources[0].previewWidth, height: effectiveSources[0].previewHeight } : s.currentImage?.previewWidth && s.currentImage?.previewHeight ? { width: s.currentImage.previewWidth, height: s.currentImage.previewHeight } : null; @@ -1403,11 +1500,13 @@ export const useStudioStore = create((set, get) => ({ } : undefined, autoRetryEnabled: batchProcessEnabled ? batchProcess.retryOnFailure : s.autoRetryEnabled, - disablePreview: s.partialImages === 0 || (loopEnabled && !loopGeneration.livePreview) || forceDisableStreamPreview, + // Partial frames are not locally composited with the mask and would show + // temporary drift in protected areas. Keep masked edits on final-only. + disablePreview: !!maskB64 || s.partialImages === 0 || (loopEnabled && !loopGeneration.livePreview) || forceDisableStreamPreview, }; const remotePayload: RuntimeGenerateOptions = { ...basePayload, - sourceImages: s.mode === "edit" && !batchProcessEnabled ? s.sources : undefined, + sourceImages: s.mode === "edit" && !batchProcessEnabled ? effectiveSources : undefined, }; const persistedPayload = basePayload; @@ -1438,7 +1537,7 @@ export const useStudioStore = create((set, get) => ({ size: resolvedSize, quality: resolvedQuality, outputFormat: s.outputFormat, - sources: s.sources, + sources: effectiveSources, currentImage: s.currentImage, styleTag: s.styleTag, loopGeneration, @@ -1990,15 +2089,75 @@ export const useStudioStore = create((set, get) => ({ void backfillHistoryPreviewRefs(items); }, + activateMaskTool: async () => { + const state = get(); + if (!state.currentImage) return false; + if (state.provider === "google" || state.provider === "grok") { + state.pushToast(`${state.provider === "google" ? "Google Interactions" : "Grok Imagine"} 不支持 OpenAI 蒙版参数`, "warn"); + return false; + } + if (state.batchProcess.enabled) { + state.pushToast("请先关闭批量目录处理，再编辑单张图片蒙版", "warn"); + return false; + } + + let targetItem = await materializeHistoryItem(state.currentImage).catch(() => null); + targetItem = await ensureFullHistoryItem(targetItem).catch(() => null); + if (!targetItem?.savedPath) { + state.pushToast("当前画布图无法作为蒙版目标，请先将图片保存或重新导入", "error"); + return false; + } + if (!targetItem.previewUrl && !targetItem.previewBlob && !targetItem.imageB64) { + const ref = await RegisterImportedImageAsset(targetItem.savedPath).catch(() => null); + if (ref) targetItem = withMediaAssetRef(targetItem, ref); + } + + const targetPath = targetItem.savedPath; + if (!targetPath) return false; + const existingSources = get().sources; + const existingTarget = existingSources.find((source) => source.path === targetPath); + const targetSource: SourceImage = existingTarget ?? { + path: targetPath, + name: targetPath.split(/[\\/]/).pop() ?? "mask-source.png", + size: 0, + imageBlob: targetItem.previewUrl ? null : (targetItem.previewBlob ?? targetItem.imageBlob ?? null), + imageB64: targetItem.previewUrl ? undefined : targetItem.imageB64, + previewUrl: targetItem.previewUrl, + previewWidth: targetItem.previewWidth, + previewHeight: targetItem.previewHeight, + }; + const targetChanged = !!get().maskTargetPath && get().maskTargetPath !== targetPath; + set({ + mode: "edit", + editSourceMode: "manual", + currentImage: targetItem, + resultGridOpen: false, + tool: "mask", + sources: [targetSource, ...existingSources.filter((source) => source.path !== targetPath)], + maskTargetPath: targetPath, + maskVisible: true, + ...(targetChanged ? { + maskDataURL: null, + strokes: [], + undoStack: [], + redoStack: [], + } : {}), + }); + if (targetChanged) get().pushToast("已切换蒙版目标并清空旧蒙版", "info"); + return true; + }, + importMaskImage: async () => { try { + if (!await get().activateMaskTool()) return; const res = await OpenMaskImageDialog(); if (!res?.path) return; const b64 = res.imageB64; if (!b64) throw new Error("未读取到图片内容"); const beforeStrokes = get().strokes; const beforeMaskDataURL = get().maskDataURL; - const nextMaskDataURL = tempDataURLFromB64(b64); + const nextMaskDataURL = await normalizeImportedMaskDataURL(tempDataURLFromB64(b64)); + if (!nextMaskDataURL) throw new Error("蒙版未包含可编辑区域"); const entry: UndoEntry = { label: "import-mask", undo: () => ({ strokes: beforeStrokes, maskDataURL: beforeMaskDataURL }), @@ -2008,6 +2167,7 @@ export const useStudioStore = create((set, get) => ({ tool: "mask", strokes: [], maskDataURL: nextMaskDataURL, + maskVisible: true, undoStack: [...get().undoStack, entry], redoStack: [], errorMessage: null, @@ -2032,23 +2192,87 @@ export const useStudioStore = create((set, get) => ({ }; set({ strokes: after, + maskTargetPath: get().maskTargetPath || get().currentImage?.savedPath || get().sources[0]?.path || null, + maskVisible: true, undoStack: [...get().undoStack, entry], redoStack: [], }); }, + fillMask: async () => { + if (!await get().activateMaskTool()) return; + const dimensions = await resolveMaskTargetDimensions(get()); + if (!dimensions) { + get().pushToast("无法读取蒙版目标图尺寸", "error"); + return; + } + const beforeStrokes = get().strokes; + const beforeMaskDataURL = get().maskDataURL; + const nextMaskDataURL = await createFullEditableMaskDataURL(dimensions); + const entry: UndoEntry = { + label: "fill-mask", + undo: () => ({ strokes: beforeStrokes, maskDataURL: beforeMaskDataURL }), + redo: () => ({ strokes: [], maskDataURL: nextMaskDataURL }), + }; + set({ + strokes: [], + maskDataURL: nextMaskDataURL, + maskVisible: true, + undoStack: [...get().undoStack, entry], + redoStack: [], + }); + get().pushToast("已全选可编辑区域", "success"); + }, + + invertMask: async () => { + if (!await get().activateMaskTool()) return; + const dimensions = await resolveMaskTargetDimensions(get()); + if (!dimensions) { + get().pushToast("无法读取蒙版目标图尺寸", "error"); + return; + } + const beforeStrokes = get().strokes; + const beforeMaskDataURL = get().maskDataURL; + const nextMaskDataURL = await renderMaskPNGDataURL({ + dimensions, + baseMaskDataURL: beforeMaskDataURL, + strokes: beforeStrokes, + invert: true, + allowEmpty: true, + }); + if (!nextMaskDataURL) { + get().pushToast("无法反选当前蒙版", "error"); + return; + } + const entry: UndoEntry = { + label: "invert-mask", + undo: () => ({ strokes: beforeStrokes, maskDataURL: beforeMaskDataURL }), + redo: () => ({ strokes: [], maskDataURL: nextMaskDataURL }), + }; + set({ + strokes: [], + maskDataURL: nextMaskDataURL, + maskVisible: true, + undoStack: [...get().undoStack, entry], + redoStack: [], + }); + get().pushToast("已反选蒙版", "success"); + }, + resetMask: () => { const before = get().strokes; const beforeMaskDataURL = get().maskDataURL; + const beforeMaskTargetPath = get().maskTargetPath; if (before.length === 0 && !beforeMaskDataURL) return; const entry: UndoEntry = { label: "clear-mask", - undo: () => ({ strokes: before, maskDataURL: beforeMaskDataURL }), - redo: () => ({ strokes: [], maskDataURL: null }), + undo: () => ({ strokes: before, maskDataURL: beforeMaskDataURL, maskTargetPath: beforeMaskTargetPath }), + redo: () => ({ strokes: [], maskDataURL: null, maskTargetPath: null }), }; set({ strokes: [], maskDataURL: null, + maskTargetPath: null, undoStack: [...get().undoStack, entry], redoStack: [], }); @@ -2832,6 +3056,37 @@ useStudioStore.subscribe((state) => { scheduleCompatibilityExport(state); }); +async function resolveMaskTargetDimensions(state: StudioState): Promise { + const targetPath = state.maskTargetPath || state.sources[0]?.path || state.currentImage?.savedPath || ""; + if (targetPath) { + const sourceB64 = await ReadImageAsBase64(targetPath).catch(() => ""); + const dimensions = await loadImageDims(sourceB64); + if (dimensions) return dimensions; + } + + const current = state.currentImage; + if (!current) return null; + if (current.imageB64) { + const dimensions = await loadImageDims(current.imageB64); + if (dimensions) return dimensions; + } + if (current.imageBlob) { + const objectURL = URL.createObjectURL(current.imageBlob); + try { + const dimensions = await loadImageDimensionsFromSource(objectURL); + if (dimensions) return dimensions; + } finally { + URL.revokeObjectURL(objectURL); + } + } + const dimensions = await loadImageDimensionsFromSource(current.fullUrl || current.previewUrl || ""); + if (dimensions) return dimensions; + if ((current.previewWidth ?? 0) > 0 && (current.previewHeight ?? 0) > 0) { + return { w: current.previewWidth!, h: current.previewHeight! }; + } + return null; +} + async function materializeHistoryItem(item: HistoryItem): Promise { return materializeHistoryItemRuntime(item, { setState: (fn) => useStudioStore.setState((state) => fn(state)), diff --git a/image-studio/frontend/src/state/studioStore.types.ts b/image-studio/frontend/src/state/studioStore.types.ts index 73912b814..c3a8f80f5 100644 --- a/image-studio/frontend/src/state/studioStore.types.ts +++ b/image-studio/frontend/src/state/studioStore.types.ts @@ -147,6 +147,9 @@ export interface StudioState { annotationColor: string; selectedAnnotationId: string | null; maskDataURL: string | null; + maskTargetPath: string | null; + maskVisible: boolean; + maskOpacity: number; strokes: Stroke[]; annotations: Annotation[]; compareB: HistoryItem | null; @@ -214,9 +217,12 @@ export interface StudioState { clearHistory: () => Promise; saveCurrentImageAs: () => Promise; bootstrap: () => Promise; + activateMaskTool: () => Promise; importMaskImage: () => Promise; setMaskDataURL: (v: string | null) => void; pushStroke: (s: Stroke) => void; + fillMask: () => Promise; + invertMask: () => Promise; resetMask: () => void; addAnnotation: (a: Annotation) => void; removeAnnotation: (id: string) => void; diff --git a/image-studio/frontend/src/state/studioStore.workspaces.ts b/image-studio/frontend/src/state/studioStore.workspaces.ts index 1b92f9f6f..c8973f02d 100644 --- a/image-studio/frontend/src/state/studioStore.workspaces.ts +++ b/image-studio/frontend/src/state/studioStore.workspaces.ts @@ -94,6 +94,9 @@ export function createWorkspaceActions(store: StateAdapter) { annotations: [], strokes: [], maskDataURL: null, + maskTargetPath: null, + undoStack: [], + redoStack: [], runningJobs: [], jobsTotal: 0, jobsCompleted: 0, @@ -151,6 +154,9 @@ export function createWorkspaceActions(store: StateAdapter) { annotations: [], strokes: [], maskDataURL: null, + maskTargetPath: null, + undoStack: [], + redoStack: [], runningJobs, jobsTotal: target.jobsTotal ?? 0, jobsCompleted: target.jobsCompleted ?? 0, @@ -219,6 +225,9 @@ export function createWorkspaceActions(store: StateAdapter) { annotations: [], strokes: [], maskDataURL: null, + maskTargetPath: null, + undoStack: [], + redoStack: [], runningJobs, jobsTotal: next.jobsTotal ?? 0, jobsCompleted: next.jobsCompleted ?? 0, diff --git a/image-studio/frontend/test/imageSources.test.mjs b/image-studio/frontend/test/imageSources.test.mjs index bf843d039..59f50640d 100644 --- a/image-studio/frontend/test/imageSources.test.mjs +++ b/image-studio/frontend/test/imageSources.test.mjs @@ -46,3 +46,16 @@ test("historyPreviewSrc remains preview-first for grids and thumbnails", () => { assert.equal(images.historyPreviewSrc(item, null), "/media/preview/img-3"); }); + +test("annotated canvas target stays first when another reference is added", () => { + const annotated = { path: "/tmp/annotated.png", name: "annotated.png", size: 1 }; + const added = { path: "/tmp/new-reference.png", name: "new-reference.png", size: 1 }; + assert.deepEqual( + images.prioritizeImageSource([added], annotated).map((source) => source.path), + ["/tmp/annotated.png", "/tmp/new-reference.png"], + ); + assert.deepEqual( + images.prioritizeImageSource([annotated, added], annotated).map((source) => source.path), + ["/tmp/annotated.png", "/tmp/new-reference.png"], + ); +}); diff --git a/image-studio/frontend/test/maskEditor.test.mjs b/image-studio/frontend/test/maskEditor.test.mjs new file mode 100644 index 000000000..9370a14d6 --- /dev/null +++ b/image-studio/frontend/test/maskEditor.test.mjs @@ -0,0 +1,59 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +const { + invertMaskPixels, + normalizeMaskPixels, +} = await import("../src/lib/maskEditor.ts"); + +test("opaque black and white masks normalize to alpha edit semantics", () => { + const pixels = new Uint8ClampedArray([ + 0, 0, 0, 255, + 255, 255, 255, 255, + ]); + + assert.equal(normalizeMaskPixels(pixels), true); + assert.deepEqual(Array.from(pixels), [ + 0, 0, 0, 255, + 0, 0, 0, 0, + ]); +}); + +test("alpha masks preserve alpha and discard irrelevant RGB channels", () => { + const pixels = new Uint8ClampedArray([ + 240, 10, 80, 255, + 12, 180, 90, 96, + ]); + + assert.equal(normalizeMaskPixels(pixels), true); + assert.deepEqual(Array.from(pixels), [ + 0, 0, 0, 255, + 0, 0, 0, 96, + ]); +}); + +test("mask inversion swaps protected and editable alpha", () => { + const pixels = new Uint8ClampedArray([ + 0, 0, 0, 255, + 0, 0, 0, 0, + 0, 0, 0, 64, + ]); + + assert.equal(invertMaskPixels(pixels), true); + assert.deepEqual(Array.from(pixels), [ + 0, 0, 0, 0, + 0, 0, 0, 255, + 0, 0, 0, 191, + ]); +}); + +test("inverting a fully editable mask reports an empty selection", () => { + const pixels = new Uint8ClampedArray([0, 0, 0, 0]); + assert.equal(invertMaskPixels(pixels), false); + assert.deepEqual(Array.from(pixels), [0, 0, 0, 255]); +}); + +test("fully protected masks report no editable pixels", () => { + const pixels = new Uint8ClampedArray([0, 0, 0, 255]); + assert.equal(normalizeMaskPixels(pixels), false); +}); diff --git a/image-studio/frontend/test/platform.test.mjs b/image-studio/frontend/test/platform.test.mjs index cee49a701..105ed332d 100644 --- a/image-studio/frontend/test/platform.test.mjs +++ b/image-studio/frontend/test/platform.test.mjs @@ -1,4 +1,5 @@ import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; import test from "node:test"; const realWindow = globalThis.window; @@ -135,3 +136,8 @@ test("macOS keeps the Apple UI family", async () => { assert.equal(state.usesAppleUI, true); }); }); + +test("Android responsive target changes do not remount the workspace shell", async () => { + const source = await readFile(new URL("../src/app/components/PlatformWorkspace.tsx", import.meta.url), "utf8"); + assert.doesNotMatch(source, /key=\{targetPlatform\}/); +}); diff --git a/image-studio/frontend/test/remoteKernel.test.mjs b/image-studio/frontend/test/remoteKernel.test.mjs index ec6cc7a15..367049262 100644 --- a/image-studio/frontend/test/remoteKernel.test.mjs +++ b/image-studio/frontend/test/remoteKernel.test.mjs @@ -369,6 +369,34 @@ test("remote payload builder uses Grok JSON edit contract", async () => { assert.deepEqual(body.image, { type: "image_url", url: "data:image/png;base64,YWJj" }); }); +test("remote Images edit uses image array fields for every multi-image source", async () => { + const payloads = await import(`../src/platform/runtime/remote-kernel/requestPayloads.ts?multi-image-fields=${Date.now()}-${Math.random()}`); + const built = await payloads.buildImagesRequestBody({ + payload: { + provider: "openai", + apiKey: "key", + mode: "edit", + prompt: "combine", + size: "1024x1024", + quality: "auto", + outputFormat: "png", + baseURL: "https://upstream.example", + imageModelID: "gpt-image-2", + requestPolicy: "openai", + negativePrompt: "", + maskB64: "", + }, + }, [ + "data:image/png;base64,YWJj", + "data:image/png;base64,ZGVm", + ]); + assert.equal(built.protocol, "openai-images"); + const imageFields = [...built.body.entries()] + .filter(([, value]) => value instanceof Blob) + .map(([key]) => key); + assert.deepEqual(imageFields, ["image[]", "image[]"]); +}); + test("runRemoteImageJob retries when Responses API only returns partial previews", async () => { let calls = 0; const partials = []; @@ -973,7 +1001,7 @@ test("runRemoteImageJob sends Responses API mask as input_image_mask data URL", }); }); -test("runRemoteImageJob sends Images API edit mask with image MIME type", async () => { +test("runRemoteImageJob uploads a canonical PNG source and mask pair for Images edits", async () => { let captured = null; await withPatchedGlobals(async () => { globalThis.fetch = async (url, init) => { @@ -999,7 +1027,7 @@ test("runRemoteImageJob sends Images API edit mask with image MIME type", async outputFormat: "png", imagePaths: [], imagePath: "", - maskB64: "iVBORw0KGgpmYWtl", + maskB64: "iVBORw0KGgptYXNr", seed: 0, negativePrompt: "", userIdentifier: "user-hash-123", @@ -1011,17 +1039,104 @@ test("runRemoteImageJob sends Images API edit mask with image MIME type", async noPromptRevision: false, }, sourceImages: [ - { imageB64: "iVBORw0KGgpzb3VyY2U=", name: "source.png", mimeType: "image/png" }, + { imageB64: "/9j/c291cmNl", name: "source.jpg", mimeType: "image/jpeg" }, ], + preparedMask: { + sourceDataURL: "data:image/jpeg;base64,/9j/c291cmNl", + sourceUploadDataURL: "data:image/png;base64,iVBORw0KGgpzb3VyY2U=", + maskB64: "iVBORw0KGgptYXNr", + }, }, { signal: new AbortController().signal }, ); assert.equal(captured.url, "https://upstream.example/v1/images/edits"); assert.ok(captured.body instanceof FormData); assert.equal(captured.body.get("user"), "user-hash-123"); + const source = captured.body.get("image"); + assert.ok(source instanceof Blob); + assert.equal(source.type, "image/png"); + assert.equal(source.name, "source-1.png"); const mask = captured.body.get("mask"); assert.ok(mask instanceof Blob); assert.equal(mask.type, "image/png"); + assert.equal(mask.name, "mask.png"); + }); +}); + +test("remote Images mask validation rejects invalid combinations before fetch", async () => { + let hits = 0; + await withPatchedGlobals(async () => { + globalThis.fetch = async () => { + hits++; + throw new Error("unexpected fetch"); + }; + }, async () => { + const payloads = await import(`../src/platform/runtime/remote-kernel/requestPayloads.ts?mask-validation=${Date.now()}-${Math.random()}`); + const basePayload = { + apiKey: "key", + prompt: "bird", + size: "1024x1024", + quality: "medium", + outputFormat: "png", + imagePaths: [], + imagePath: "", + seed: 0, + negativePrompt: "", + baseURL: "https://upstream.example", + textModelID: "", + imageModelID: "gpt-image-2", + apiMode: "images", + requestPolicy: "openai", + noPromptRevision: false, + }; + + await assert.rejects( + payloads.buildImagesRequestBody({ payload: { ...basePayload, mode: "generate", maskB64: "iVBORw0KGgptYXNr" } }, []), + /图生图模式/, + ); + await assert.rejects( + payloads.buildImagesRequestBody({ payload: { ...basePayload, mode: "edit", maskB64: "iVBORw0KGgptYXNr" } }, []), + /至少一张源图/, + ); + await assert.rejects( + payloads.buildImagesRequestBody( + { payload: { ...basePayload, mode: "edit", maskB64: "%%%" } }, + ["data:image/png;base64,iVBORw0KGgpzb3VyY2U="], + ), + /base64 无效/, + ); + assert.equal(hits, 0); + }); +}); + +test("remote Images unmasked JPEG edits preserve the original source MIME", async () => { + await withPatchedGlobals(async () => {}, async () => { + const payloads = await import(`../src/platform/runtime/remote-kernel/requestPayloads.ts?unmasked-jpeg=${Date.now()}-${Math.random()}`); + const built = await payloads.buildImagesRequestBody({ + payload: { + apiKey: "key", + mode: "edit", + prompt: "bird", + size: "1024x1024", + quality: "medium", + outputFormat: "png", + imagePaths: [], + imagePath: "", + maskB64: "", + seed: 0, + negativePrompt: "", + baseURL: "https://upstream.example", + textModelID: "", + imageModelID: "gpt-image-2", + apiMode: "images", + requestPolicy: "openai", + noPromptRevision: false, + }, + }, ["data:image/jpeg;base64,/9j/c291cmNl"]); + const source = built.body.get("image"); + assert.ok(source instanceof Blob); + assert.equal(source.type, "image/jpeg"); + assert.equal(source.name, "source-1.jpg"); }); }); diff --git a/image-studio/frontend/test/requestModel.test.mjs b/image-studio/frontend/test/requestModel.test.mjs index 097b80b32..53a13ee59 100644 --- a/image-studio/frontend/test/requestModel.test.mjs +++ b/image-studio/frontend/test/requestModel.test.mjs @@ -145,3 +145,22 @@ test("repairSizeForOpenAI snaps invalid sizes to nearest legal 16-aligned value" prompt: "cat", }); }); + +test("Responses mask requires edit mode and a first source image", () => { + const base = { + prompt: "edit", + mode: "generate", + maskB64: "iVBORw0KGgptYXNr", + imageModelID: "gpt-image-2", + textModelID: "gpt-5.5", + }; + assert.throws(() => buildResponsesPayload(base, []), /图生图模式/); + assert.throws(() => buildResponsesPayload({ ...base, mode: "edit" }, []), /至少一张源图/); + assert.throws( + () => buildResponsesPayload( + { ...base, mode: "edit", maskB64: "%%%" }, + ["data:image/png;base64,iVBORw0KGgpzb3VyY2U="], + ), + /base64 无效/, + ); +}); diff --git a/shared/kernel/requestModel.js b/shared/kernel/requestModel.js index 52cd16a12..6695be3b1 100644 --- a/shared/kernel/requestModel.js +++ b/shared/kernel/requestModel.js @@ -401,14 +401,32 @@ export function buildResponsesImageTool(payload, sourceDataURLs, options = {}) { if (compatExtensions && negativePrompt) tool.negative_prompt = negativePrompt; const maskMimeType = String(options.maskMimeType || "image/png").trim() || "image/png"; - if (payload.maskB64) { + const maskB64 = validateMaskRequest(payload, sourceDataURLs); + if (maskB64) { tool.input_image_mask = { - image_url: dataURLFromBase64Image(payload.maskB64, maskMimeType), + image_url: dataURLFromBase64Image(maskB64, maskMimeType), }; } return tool; } +function validateMaskRequest(payload, sourceDataURLs) { + const maskB64 = String(payload?.maskB64 || "").replace(/\s+/g, ""); + if (!maskB64) return ""; + if (payload?.mode !== "edit") throw new Error("蒙版仅支持图生图模式"); + if (!Array.isArray(sourceDataURLs) || sourceDataURLs.length === 0) { + throw new Error("蒙版任务需要至少一张源图"); + } + if (!/^[A-Za-z0-9+/]*={0,2}$/.test(maskB64) || maskB64.length % 4 === 1) { + throw new Error("蒙版图片 base64 无效"); + } + const padding = maskB64.endsWith("==") ? 2 : maskB64.endsWith("=") ? 1 : 0; + const decodedBytes = Math.floor((maskB64.length * 3) / 4) - padding; + if (decodedBytes <= 0) throw new Error("蒙版图片为空"); + if (decodedBytes > 50 * 1024 * 1024) throw new Error("蒙版图片超过 50MB 上限"); + return maskB64; +} + export function buildResponsesPayload(payload, sourceDataURLs, options = {}) { const content = buildResponsesInputContent(payload.prompt, sourceDataURLs); const userIdentifier = normalizeUserIdentifier(payload.userIdentifier); From 613d49105de37c9ad52d21796d79238603f44ea2 Mon Sep 17 00:00:00 2001 From: lin <1609924163@qq.com> Date: Tue, 6 Oct 2026 13:38:58 +0800 Subject: [PATCH 4/6] Fix history retention, failure alerts, batch queues and native drag Audit all 12 open upstream issues and document their implementation and verification status. Preserve complete history across pagination, import/export and clients; add deduplicated failure feedback; keep queued jobs alive between waves; start native file drag from mouse gestures and guard Windows capture cleanup. Validated 203 frontend tests, 5 Worker tests, all four Go modules, 15 Android JVM tests, Android APK build/signature/manifest/assets, five frontend targets, Windows amd64/arm64 backend compilation, macOS universal packaging/runtime probe, local upstream smoke and issue tooling. Windows visual and OneCommander/capture-release checks remain explicitly documented for target-machine verification. --- README.md | 1 + docs/issue-close-comments.md | 61 ++- docs/issue-fixes-20261006.md | 86 ++++ docs/issue-progress.md | 418 +----------------- docs/manual-verification.md | 17 + docs/verification/issue-audit-20261006.json | 290 ++++++++++++ gio-client/internal/compat/state.go | 5 +- gio-client/internal/compat/state_test.go | 21 + .../internal/ui/completion_notification.go | 17 + .../ui/completion_notification_test.go | 40 ++ gio-client/internal/ui/run_job.go | 18 +- gio-client/internal/ui/state.go | 1 + image-studio/backend/native_drag_policy.go | 23 + .../backend/native_drag_policy_test.go | 28 ++ image-studio/backend/native_drag_windows.go | 62 ++- image-studio/frontend/package-lock.json | 253 ++++++----- image-studio/frontend/package.json | 1 + .../components/canvas/DragExportHandle.tsx | 20 +- .../src/components/common/SavePromptModal.tsx | 24 +- .../components/common/useNativeFileDrag.ts | 33 ++ .../history/HistoryPromptGroupModal.tsx | 15 +- .../src/components/history/HistoryTile.tsx | 15 +- .../history/HistoryTimelineModal.tsx | 2 +- .../components/panel/ResultDetailDrawer.tsx | 18 +- .../src/components/panel/SettingsPanel.tsx | 6 +- image-studio/frontend/src/lib/compatState.ts | 15 +- .../frontend/src/lib/completionSound.ts | 10 +- image-studio/frontend/src/lib/dragExport.ts | 15 +- image-studio/frontend/src/lib/failureAlert.ts | 12 + image-studio/frontend/src/lib/history.ts | 11 + image-studio/frontend/src/lib/storage.ts | 89 ++-- .../android/settings/AndroidSettingsPanel.tsx | 6 +- .../platform/runtime/nativeFileDragGesture.ts | 40 ++ .../frontend/src/state/historyCleanup.ts | 4 +- .../frontend/src/state/studioStore.images.ts | 2 +- .../frontend/src/state/studioStore.media.ts | 72 ++- .../frontend/src/state/studioStore.shared.ts | 13 - .../frontend/src/state/studioStore.ts | 76 ++-- .../frontend/src/state/studioStore.types.ts | 3 +- .../frontend/src/state/workspaceRuntime.ts | 22 +- .../frontend/test/compatState.test.mjs | 26 ++ .../frontend/test/completionSound.test.mjs | 9 + .../frontend/test/dragExport.test.mjs | 20 + .../frontend/test/failureAlert.test.mjs | 19 + .../frontend/test/historyCleanup.test.mjs | 4 +- .../frontend/test/historyStorage.test.mjs | 86 ++++ .../test/nativeFileDragGesture.test.mjs | 44 ++ .../test/studioStoreWorkflow.test.mjs | 154 +++++++ .../frontend/test/workspaceRuntime.test.mjs | 14 + scripts/issue-close-data.json | 34 +- scripts/render-issue-close-comments.mjs | 8 +- 51 files changed, 1526 insertions(+), 757 deletions(-) create mode 100644 docs/issue-fixes-20261006.md create mode 100644 docs/verification/issue-audit-20261006.json create mode 100644 image-studio/backend/native_drag_policy.go create mode 100644 image-studio/backend/native_drag_policy_test.go create mode 100644 image-studio/frontend/src/components/common/useNativeFileDrag.ts create mode 100644 image-studio/frontend/src/lib/failureAlert.ts create mode 100644 image-studio/frontend/src/lib/history.ts create mode 100644 image-studio/frontend/src/platform/runtime/nativeFileDragGesture.ts create mode 100644 image-studio/frontend/test/failureAlert.test.mjs create mode 100644 image-studio/frontend/test/historyStorage.test.mjs create mode 100644 image-studio/frontend/test/nativeFileDragGesture.test.mjs create mode 100644 image-studio/frontend/test/studioStoreWorkflow.test.mjs diff --git a/README.md b/README.md index fa652cdc3..64fc8f502 100644 --- a/README.md +++ b/README.md @@ -58,6 +58,7 @@ Image Studio 面向 OpenAI 兼容图像上游，重点解决长时间图像推 | 功能清单、平台能力、快捷键 | [docs/features.md](./docs/features.md) | | 图生图批处理的入口、流程、输入输出规则 | [docs/batch-img2img/README.md](./docs/batch-img2img/README.md) | | 当前 issue 处理进展与待验证项 | [docs/issue-progress.md](./docs/issue-progress.md) | +| 本轮修复根因、维护约束和防回归检查 | [docs/issue-fixes-20261006.md](./docs/issue-fixes-20261006.md) | | 可直接复用的 issue 关单评论模板 | [docs/issue-close-comments.md](./docs/issue-close-comments.md) | | 源码构建、验证脚本、CI 产物链路 | [docs/build.md](./docs/build.md) | | 真机 / 真实上游手工验证矩阵 | [docs/manual-verification.md](./docs/manual-verification.md) | diff --git a/docs/issue-close-comments.md b/docs/issue-close-comments.md index 22f329981..6189621c8 100644 --- a/docs/issue-close-comments.md +++ b/docs/issue-close-comments.md @@ -1,6 +1,6 @@ # Issue 关单评论模板 -更新时间：2026-07-10 +更新时间：2026-10-06 本文档只覆盖当前 **GitHub 仍 open，但当前仓库代码与本地验证已经覆盖** 的 issue。 @@ -12,14 +12,17 @@ - `#50` - `#51` - `#53` +- `#57` +- `#58` 统一验证基线： -- 2026-07-10 已重新跑通本地平台总链： - - `npm test; go test ./... (image-studio, go-cli, gio-client, shared/compat-go); npm run build:windows; npm run build:android; npm run build:macos; ./gradlew :app:testDebugUnitTest; git diff --check` - - `platform-kernel-summary.json`：`status = passed` +- 2026-10-06 已重新完成本机验证： + - `npm test (frontend/worker); go test ./... (4 modules); 5 frontend builds; Android JVM/APK; macOS universal/runtime probe; Windows amd64/arm64 backend compile; local live/smoke; issue tooling; git diff --check` + - `docs/verification/issue-audit-20261006.json`：`status = passed` - 结构化结果： -- 前端测试当前为 `162/162` 通过。 + - `docs/verification/issue-audit-20261006.json` +- 前端测试当前为 `203/203` 通过。 建议用法： @@ -44,13 +47,20 @@ node scripts/render-issue-close-comments.mjs --write - `README.md` - `plan.json` - `plan.md` -- `issue-24.md` ... `issue-42.md` +- `issue-24.md` +- `issue-44.md` +- `issue-49.md` +- `issue-50.md` +- `issue-51.md` +- `issue-53.md` +- `issue-57.md` +- `issue-58.md` 如果只是想先确认“当前哪些 issue 会被处理、处理方式是什么”，优先用： ```bash node scripts/issue-close-helper.mjs plan -node scripts/issue-close-helper.mjs plan 24 25 --comment-only +node scripts/issue-close-helper.mjs plan 24 44 --comment-only ``` `apply` 默认不会执行。只有显式提供 `--execute`，并且环境里存在 `GITHUB_TOKEN` 或 `GH_TOKEN` 时，才会真正对 GitHub 发评论或关闭 issue。例如： @@ -84,7 +94,7 @@ Issue: [#24](https://github.com/RoseKhlifa/Image-Studio/issues/24) - `image-studio/frontend/src/components/history/HistoryRail.tsx` - `image-studio/frontend/src/components/history/WindowsHistoryRail.tsx` -最终前端全量测试为 162/162，这个 issue 可以关闭。 +2026-10-06 重新完成本机回归；单条与日期删除也会清理旧库副本，避免迁移恢复记录。验证依据见 docs/verification/issue-audit-20261006.json。 ``` ## `#44` [Feature]: 添加批处理功能 @@ -111,6 +121,8 @@ Issue: [#44](https://github.com/RoseKhlifa/Image-Studio/issues/44) - `docs/batch-img2img/README.md` 批处理路径已有前后端测试覆盖，这个 issue 可以关闭；更复杂的命名模板可以单独作为增强需求。 + +2026-10-06 补充修复：队列两轮之间不会因 runningJobs 暂时为空而提前清零；提交失败按任务 ID 正确结算，重复结算不增加计数。真实 store 多轮事件链及 650 次结算回归已覆盖。 ``` ## `#49` [Feature]: 模板/历史里面的历史条数混乱，不够清晰的看到每一条的内容 @@ -206,8 +218,39 @@ Issue: [#53](https://github.com/RoseKhlifa/Image-Studio/issues/53) 前端、Go、Gio 与 Android 单测均通过，这个 issue 可以关闭。官方上游若要求 PNG、同尺寸或 alpha 通道，仍需提供符合模型约束的蒙版。 ``` +## `#57` [Feature]: 声音提示 + +Issue: [#57](https://github.com/RoseKhlifa/Image-Studio/issues/57) + +```md +生成失败反馈已补齐。 + +- 提交失败、生成错误和结果处理错误均显示可见错误提示。 +- 按现有任务提示音开关播放声音，后台且通知已授权时发失败系统通知。 +- 同次提交的并发错误只提醒一次；新提交可以再次提醒，主动取消保持静默。 +- Gio 同步支持失败声音和通知。 + +真实 store 提交/事件链测试覆盖并发失败、提交拒绝和取消；音频拒绝播放时可见提示仍保留。验证记录：docs/verification/issue-audit-20261006.json。 +``` + +## `#58` [Feature]: 生图历史似乎不能显示所有的图片 + +Issue: [#58](https://github.com/RoseKhlifa/Image-Studio/issues/58) + +```md +历史保留与分页已修复。 + +- 移除前端/Gio 的 120 条截断；生成和导入不再按可见列表删除数据库。 +- 按 createdAt + id 分页，支持同一天数百张与同毫秒结果，完整历史仍可复用提示词。 +- 导出与共享配置包含未加载记录；导入旧客户端部分快照不会删除本地档案。 +- 按日期清理覆盖未加载记录，并同步删除对应全图和旧库副本。 + +650 条 IndexedDB 分页、完整图保留及配置导入/导出回归通过。旧版本已经删除的提示词不能从图片文件自动恢复。维护约束与验证记录见 docs/issue-fixes-20261006.md、docs/verification/issue-audit-20261006.json。 +``` + ## 暂不建议关闭的 open issue - `#30`：原生 Wails light/dark 标题栏颜色与 CSS token 已有自动化一致性验证，但仍缺 Windows 真机视觉确认。 -- `#52`：五个拖出入口已统一走 Windows 原生 OLE，IDataObject 也显式提供 Unicode CF_HDROP；仍缺 Windows + OneCommander 实机确认。 +- `#52`：Unicode CF_HDROP 保留，五个入口统一为避免 HTML/native 重叠的手势路径；文件格式与内部 MIME 均有测试覆盖；仍需 Windows + OneCommander 实机验收。 +- `#55`：五处入口改为鼠标手势启动原生拖动，处理 mouseup/Esc/失焦/卸载；Windows 添加真实按钮检查、定时退出与捕获释放，单测和双架构编译通过；仍需 Windows 上验证其他应用拖拽恢复。 - `#14`：这是已回答的产品问答；当前没有独立在线 Web 版，也没有可据此实现的功能验收标准。 diff --git a/docs/issue-fixes-20261006.md b/docs/issue-fixes-20261006.md new file mode 100644 index 000000000..e148a455b --- /dev/null +++ b/docs/issue-fixes-20261006.md @@ -0,0 +1,86 @@ +# 2026-10-06 修复与防回归说明 + +审核覆盖 React/TypeScript 前端、Wails Go 后端、共享请求模型、Go CLI、Gio、Cloudflare Worker、Android 壳层及构建/issue 工具。逐项结果见 [issue-progress.md](./issue-progress.md)。 + +## 历史保留与分页（#58、#24） + +旧代码同时限制前端和 Gio 历史为 120 条。前端生成和导入后还调用 `persistTrimmedHistory`，将当前加载列表作为唯一保留清单删除 IndexedDB。即使没有达到 120 条，未加载的其他页也可能被删除。按天分页还会一次读取同一天的全部图片，再经过截断跳过该天剩余记录。 + +`lib/history.ts` 现在只合并、去重、排序。`storage.ts` 用 `(createdAt, id)` 作为游标，每页有固定上限，并读取下一条确认是否还有更多。生成、导入、Gio 和共享配置均保留完整档案。共享配置导出查询全部持久化记录，导入旧客户端部分快照时合并历史。分页返回后使用最新 store 状态合并，避免覆盖等待期间新生成的结果。 + +按日期清理通过数据库索引删除指定范围及其全图，覆盖未加载记录；单条删除和日期清理同时移除旧 keyval 副本。全部删除仍使用同一事务清空当前 metadata/full stores，并清理各工作区、详情、对比及待保存队列。 + +维护约束： + +- 视图页、缩略图缓存和归档历史分别管理。可见条数不代表允许删除数据库记录。 +- 时间戳不是唯一键；分页、去重和结算均需要记录/任务 ID。 +- 生成和配置导入不能自动清理档案；删除只能来自用户明确的单条、日期或全部删除操作。 +- 导入/导出必须覆盖未加载记录；共享配置中缺少一条记录不能解释为用户删除它。 +- 不恢复 `MAX_HISTORY_ITEMS`、`trimHistory` 或按可见列表运行的 `pruneHistoryStorage`。 + +回归测试：`historyStorage.test.mjs`、`compatState.test.mjs`、`storageClear.test.mjs`、`historyCleanup.test.mjs` 和 Gio `TestMergeHistoryRetainsLargeBatchAndDeduplicates`。旧版本已删除的提示词没有可恢复来源；修复防止继续丢失，不声称能从图片反推出原记录。 + +## 失败反馈与队列（#57、#44） + +以前只有成功事件触发完成声音，错误事件仅更新工作区错误字段。现在所有生成失败路径经过每次提交独立的 `failureAlert`，首次失败显示 toast、按设置播放任务提示音，并在后台且通知已授权时显示失败通知。取消不发失败提醒。Gio 同步支持，首次错误用原子比较交换去重。音频被宿主拒绝时返回 `false`，可见错误提示仍保留。 + +`runningJobs` 暂时为空可能只是下一轮任务尚未启动。旧代码在这个间隙将 `jobsTotal` 归零，队列会提前结束；提交拒绝也没有正确递增完成数。`completeWorkspaceJob` 现在按任务 ID 结算、拒绝重复计数，在所有预定任务结束后才清零，并在队列间隙保持忙碌。失败停止循环时仅保留已经启动的任务计数，避免卡住忙碌状态。 + +维护约束： + +- 提交失败、生成错误、结果处理错误都需要反馈，不能只检查成功分支。 +- 一个提交共享同一失败提醒 gate；新提交创建新 gate。 +- 取消不冒充失败；后台系统通知遵循用户设置和宿主权限。 +- 空运行列表不能单独证明队列完成；成功、错误和提交拒绝都结算一次。 + +回归测试：`failureAlert.test.mjs`、`completionSound.test.mjs`、`workspaceRuntime.test.mjs`、`studioStoreWorkflow.test.mjs` 及 Gio 失败声音/通知测试。store 集成测试调用真实提交逻辑和 runtime 事件，只替换宿主服务，不请求收费上游。 + +## 原生拖出与浏览器回退（#55、#52） + +发现的风险是：五处 UI 从 HTML `dragstart` 取消浏览器拖动，再异步调用 Go `DoDragDrop`，WebView 原生拖动/鼠标捕获退出可能与新 OLE 循环重叠。这与 issue 描述相符，但本机是 macOS，未将推断写成 Windows 实机已证实的根因。 + +五处入口统一使用 `useNativeFileDrag`：本机文件不再启用 HTML draggable；左键移动超过 5px 后，在当前输入事件结束后启动原生拖动。mouseup、Esc、失焦、卸载都取消尚未启动的手势；错误显示 toast。虚拟路径和 URL 不交给本机文件拖动。 + +Windows 保留 Unicode `CF_HDROP`、完整路径和双 NUL 文件列表。进入 OLE 前检查真实左键状态，拒绝晚到的手势；退出释放当前线程鼠标捕获。定时器在 30 秒期限到达后向当前 OLE 捕获窗口投递 Esc，退出检查不会只依赖用户再次移动鼠标；没有发送全局键盘输入。此保护不能强制中断任意卡死的第三方 COM 调用。 + +浏览器回退统一调用 `writeHistoryItemFileDragData`：先写文件格式/清除浏览器默认内容，再写内部历史 MIME。旧顺序会在 `clearData` 时丢掉历史元数据。 + +维护约束： + +- 不从 HTML `dragstart` 调用原生 OLE/NSDragging；五个入口共享同一手势处理。 +- 不把 `memory://`、URL 或相对路径当作本机文件。 +- 不移除 `CF_HDROP`、双 NUL、退出检查和定时退出；取消只针对当前拖动线程。 +- 文件格式写入后保留内部 MIME，避免应用内拖放退化。 +- 交叉编译/单测不能替代 OneCommander 和 Windows 全局拖拽实机验收。 + +微软依据：[DoDragDrop](https://learn.microsoft.com/en-us/windows/win32/api/ole2/nf-ole2-dodragdrop)、[QueryContinueDrag](https://learn.microsoft.com/en-us/windows/win32/api/oleidl/nf-oleidl-idropsource-querycontinuedrag)、[ReleaseCapture](https://learn.microsoft.com/en-us/windows/win32/api/winuser/nf-winuser-releasecapture)、[SetTimer](https://learn.microsoft.com/en-us/windows/win32/api/winuser/nf-winuser-settimer)。 + +回归测试：`nativeFileDragGesture.test.mjs`、`dragExport.test.mjs`、`native_drag_policy_test.go`、`native_drag_hdrop_test.go`，并编译 Windows amd64/arm64 后端。实机步骤见 [manual-verification.md](./manual-verification.md)。 + +## 既有功能的重新核对 + +蒙版、URL 图片结果、Prompt 历史编号、批处理输出命名和 Windows 标题栏 token 已有实现。本次保留请求格式与现有测试，重新测试/构建，不依据旧进展记录跳过验证。 + +Nano Banana 2 的路由按[官方图像生成说明](https://ai.google.dev/gemini-api/docs/image-generation)和 [Interactions API](https://ai.google.dev/api/interactions-api)核对。Google 原生请求与标准 Images relay 分支分开验证，错误不静默回退。没有用 fixture 成功代替真实 key/模型权限证明。 + +前端锁文件的 7 项依赖审计问题已更新到现有主版本范围内的修复版。新增测试依赖 `fake-indexeddb`，重新运行依赖审计和五种目标平台构建。 + +## 本机复验 + +在仓库根目录执行以下命令。Go 缓存目录可复用现有配置；HTTP mock 需要允许监听 `127.0.0.1`。 + +```bash +(cd image-studio/frontend && npm ci && npm test) +(cd cloudflare-worker && npm test) +(cd image-studio && go test ./...) +(cd go-cli && go test ./...) +(cd gio-client && go test ./...) +(cd shared/compat-go && go test ./...) +(cd image-studio/frontend && npm run build:windows && npm run build:macos && npm run build:linux && npm run build:android-pad && npm run build:android) +node scripts/verify-local-live-verify.mjs +node scripts/local-smoke-check.mjs +node scripts/verify-issue-close-tooling.mjs +git diff --check +``` + +Android JVM/APK 和 macOS universal 包按 [build.md](./build.md) 的工具链执行。本轮命令、结果和未执行项见 [verification/issue-audit-20261006.json](./verification/issue-audit-20261006.json)，避免把编译、fixture、真机和真实上游混为同一种验证。 diff --git a/docs/issue-progress.md b/docs/issue-progress.md index 47e6167fe..bd8382795 100644 --- a/docs/issue-progress.md +++ b/docs/issue-progress.md @@ -1,400 +1,22 @@ # Issue 进展记录 -更新时间：2026-07-10 - -GitHub issue 状态已按 2026-07-10 当天的 GitHub API 重新核对。 - -## 2026-07-10 当前 open issue 补充 - -本轮重新核对到当前仍 open 的 issue 为 `#14` `#24` `#30` `#44` `#49` `#50` `#51` `#52` `#53`。 - -| 编号 | 标题 | 当前仓库状态 | 下一步 | -|---|---|---|---| -| `#53` | 支持发送蒙版的 API 参数 | Wails、Android 与 Gio 都有蒙版导入/清除入口；提交时复用 `maskB64`，Responses 按 `input_image_mask`、Images edits 按 multipart `mask` 文件发送，并校验真实图片 MIME | 代码与自动化测试已完成，可关闭；真实上游 smoke 作为发布验证 | -| `#52` | OneCommander 拖拽结果图片另存为异常 | 五个结果拖出入口已统一走 Windows 原生 OLE；`IDataObject` 显式提供 Unicode `CF_HDROP`、完整路径和双 NUL 文件列表，不再回退成 URL | 保持 open，等待 Windows + OneCommander 实机拖拽验证 | -| `#51` | 兼容 Google nano banana2 | 对 Google 官方主机 + `gemini-3.1-flash-image` 窄路由到官方 `POST /v1beta/interactions`，按 `x-goog-api-key`、`input`、`response_format`、`store=false` 请求并解析 `steps[].content[].data/uri`；第三方 relay 仍走标准 Images API，不假定其支持 Google 原生协议 | 代码与官方 fixture 测试已完成，可关闭；真实可用性仍取决于 key / relay 模型权限 | -| `#50` | Images API 返回 URL 生图兼容 | 已支持 `data[0].url` 下载并转为 base64，桌面 Go client、浏览器远程内核与 Android 原生二进制下载路径都覆盖 50MB 上限和图片类型检查 | 代码与自动化测试已完成，可关闭；真实 relay smoke 作为发布验证 | -| `#49` | 模板/历史里的历史条数混乱 | Prompt 历史改为带 `01 / 02 / ...` 序号和明确分隔线的有序列表，Android 与 Gio 使用同一编号语义 | 可关闭 | -| `#44` | 添加批处理功能 | 批处理已有基础实现；本轮补可编辑输出文件名前缀，保留原文件名并继续自动去重 | 可关闭；更复杂命名模板可作为新增强 issue | -| `#30` | 标题背景与标题栏颜色不一致 | 原生 Wails light/dark 标题栏色与 CSS token 已有自动化一致性验证 | 保持 open，等待 Windows 真机视觉确认 | -| `#24` | 支持全部删除 | `history` / `historyFull` 原子 bulk clear，并同步清理旧库、分页、画布、详情、待保存队列和全部 workspace 引用 | 可关闭 | -| `#14` | 有 Web 端吗 | 无独立在线 Web 版；这是已回答的产品问答，没有可从当前描述推导出的功能验收标准 | 保持搁置或评论说明后关闭 | - -## 全部 issue 总表 - -下表保留 2026-06-07 历史快照，只统计 GitHub 上真正的 issue，不包含 PR 编号。当前 open issue 以本页顶部 2026-07-05 补充为准。 - -| 编号 | 标题 | GitHub 状态 | 当前仓库状态 | 下一步 | -|---|---|---|---|---| -| `#1` | Releases v0.1.0 里没有包含 `image-studio-windows-amd64.exe` | closed | 已关闭 | 无 | -| `#2` | 比例增加 auto / 自定义分辨率 | closed | 已关闭 | 无 | -| `#3` | 无法识别透明背景参考图 | closed | 已关闭 | 无 | -| `#4` | 无法并发生图 | closed | 已关闭 | 无 | -| `#9` | 多张出图只会保存一张 | closed | 已关闭 | 无 | -| `#10` | 无法右击已生成的图片 | closed | 已关闭 | 无 | -| `#11` | 点击全屏按钮会黑屏 | closed | 已关闭 | 无 | -| `#12` | `gpt-image-2` 也会优化提示词且无法关闭 | closed | 已关闭 | 无 | -| `#13` | `/v1/images/edits` 的 mask MIME 类型问题 | closed | 已关闭 | 无 | -| `#14` | 有 web 端吗 | open | 已明确“无独立在线 Web 版”，当前目标也已搁置 | 保持搁置，不继续推进 | -| `#15` | 点击历史图后提示词保留可复制修改 | closed | 已关闭 | 无 | -| `#18` | 点了“比例/分辨率自动”后无法再更改 | closed | 已关闭 | 无 | -| `#20` | macOS 版本无法删除配置 | closed | 已关闭 | 无 | -| `#21` | PC 端 log 文件自动清除 | closed | 已关闭 | 无 | -| `#22` | relay 已支持 `b64_json` 但程序提示不支持 | closed | 已关闭 | 无 | -| `#23` | 不生图 | open | 按当前任务约定视为上游问题，不纳入本轮收口 | 保持不处理 | -| `#24` | 支持全部删除 | open | 已有代码覆盖 | 如允许维护 issue，可直接关闭或补评论 | -| `#25` | 上游配置升级后的同步问题 | open | 已有代码覆盖 | 如允许维护 issue，可直接关闭或补评论 | -| `#26` | 失败重试开关 | open | 已有代码覆盖 | 如允许维护 issue，可直接关闭或补评论 | -| `#27` | 上游失败重试路由功能 | open | 已有代码覆盖 | 如允许维护 issue，可直接关闭或补评论 | -| `#28` | 保存用户自定义提示词模板 | open | 已有代码覆盖 | 如允许维护 issue，可直接关闭或补评论 | -| `#29` | 生图完成时触发提示音 | closed | 已关闭 | 无 | -| `#30` | 标题背景与标题栏颜色不一致 | open | 代码已修，但仍缺 Windows 真机视觉确认 | 需要 Windows 真机手工验证 | -| `#32` | 拖结果到文件复制 | open | 已有代码覆盖 | 如允许维护 issue，可直接关闭或补评论 | -| `#33` | 点击参考图缩略图无法查看大图 | closed | 已关闭 | 无 | -| `#34` | 应用参数未包含参考图 | closed | 已关闭 | 无 | -| `#35` | 工作区滚动方向改为横向 | open | 已有代码覆盖 | 如允许维护 issue，可直接关闭或补评论 | -| `#36` | 多图并发时模糊 / 只出轮廓 | open | 核心修复已落地，但仍缺真实上游与真机长链路证明 | 需要 Android / Windows 真机 + 真实高并发上游复核 | -| `#37` | 结果多图希望能左右键浏览 | open | 已有代码覆盖 | 如允许维护 issue，可直接关闭或补评论 | -| `#40` | 多图张数按钮选择 | open | 已有代码覆盖 | 如允许维护 issue，可直接关闭或补评论 | -| `#42` | 一些图生图 / 模板 / 对比功能需求 | open | 已有代码覆盖 | 如允许维护 issue，可直接关闭或补评论 | - -这份文档只记录当前仓库代码与本地验证状态，不代表 GitHub issue 已关闭。 - -## 按状态分组 - -### 已关闭的 issue - -`#1` `#2` `#3` `#4` `#9` `#10` `#11` `#12` `#13` `#15` `#18` `#20` `#21` `#22` `#29` `#33` `#34` - -这些 issue 当前 GitHub 状态已经是 `closed`，本地不需要再补额外动作。 - -### GitHub 仍 open，但当前代码已覆盖 - -`#24` `#25` `#26` `#27` `#28` `#32` `#35` `#37` `#40` `#42` - -这批 issue 的共同特点是： - -- 当前仓库已经有对应实现。 -- 本地验证链已经覆盖到主要行为。 -- 没有新的外部前置条件阻塞。 - -如果允许直接维护 GitHub issue，优先建议先清这一批，避免后续重复盘点。 - -可直接复用的评论模板见 [issue-close-comments.md](./issue-close-comments.md)。 - -当前这批 open issue 还额外有一层本地工具链保证： - -- 数据源：`scripts/issue-close-data.json` -- helper：`scripts/issue-close-helper.mjs` -- 独立校验：`scripts/verify-issue-close-tooling.mjs` - -截至 2026-06-07 的最新本地实证结果： - -- `closable = 10` -- `holdOpen = 2` -- `deferred = 2` -- `unexpectedOpen = 0` -- 最新总链结果目录： - - `/private/tmp/verify-results-issue-close-final4/platform-kernel-summary.json` - - `/private/tmp/verify-results-issue-close-final4/issue-close-tooling.json` - - `/private/tmp/verify-results-issue-close-final4/issue-close-export-bundle/manifest.json` - -另外，本地 issue 收口工具链已经在 mock GitHub 环境里补了两类真实执行证明： - -- `comment-only` 会发评论，但不会关闭 issue。 -- `comment-and-close` 会发评论，并发送关闭 issue 的 patch。 -- 无 `GITHUB_TOKEN` / `GH_TOKEN` 时，独立校验会默认切到离线模式，不再因为 GitHub rate limit 打断本地验证。 - -也就是说： - -- 当前建议关闭的 open issue 仍然就是这 10 个。 -- 需要继续保留 open 的仍然只有 `#30` 和 `#36`。 -- 目前没有冒出新的、未被本地分组覆盖的 open issue。 - -## 已有代码覆盖 - -### `#42` 一些功能需求 - -当前代码已覆盖： - -- 参考图存在时，图生图默认切到 `Auto` 比例。 -- 会根据参考图比例推导最接近的 2K / 4K 分辨率。 -- 支持自定义提示词模板与快捷填入。 -- 结果图与参考图对比查看。 - -主要落点： - -- `image-studio/frontend/src/components/panel/sizeCapabilities.ts` -- `image-studio/frontend/src/components/panel/PromptTemplateManagerModal.tsx` -- `image-studio/frontend/src/components/canvas/CompareOverlay.tsx` - -### `#40` 多图张数按钮选择 - -Android 端多图张数已从滑杆改为带边界的按钮选择。 - -主要落点： - -- `image-studio/frontend/src/platform/android/AndroidPhoneComposePanel.tsx` -- `image-studio/frontend/src/platform/android/AndroidPadComposePanel.tsx` - -### `#37` 结果里生成的多张图片查看问题 - -当前代码已支持在画板里对同批结果做左右切换浏览。 - -主要落点： - -- `image-studio/frontend/src/components/canvas/useCanvasShortcuts.ts` -- `image-studio/frontend/src/state/studioStore.media.ts` - -### `#35` 工作区滚轮改横向 - -工作区滚轮方向已改为横向滚动。 - -主要落点： - -- `image-studio/frontend/src/components/layout/WorkspaceBar.tsx` - -### `#32` 拖结果到文件复制 - -桌面端结果图已支持拖出到文件管理器复制。 - -主要落点： - -- `image-studio/backend/dialogs.go` -- `image-studio/frontend/src/lib/dragExport.ts` - -### `#29` 生图完成时触发提示音 - -已支持： - -- 完成提示音开关。 -- 默认 / 自定义提示音切换。 -- 预览提示音。 -- 整批任务完成时只播放一次。 - -主要落点： - -- `image-studio/frontend/src/lib/completionSound.ts` -- `image-studio/frontend/src/components/panel/SettingsPanel.tsx` -- `image-studio/frontend/src/platform/android/settings/AndroidSettingsPanel.tsx` - -### `#28` 提示词模板 - -已支持添加、删除、修改、保存、点击填充提示词模板。 - -主要落点： - -- `image-studio/frontend/src/lib/promptTemplates.ts` -- `image-studio/frontend/src/components/panel/PromptTemplateManagerModal.tsx` - -### `#27` 上游失败重试路由 - -已支持把失败后的重试路由到用户指定的备用上游。 - -主要落点： - -- `go-cli/pkg/client/retry.go` -- `image-studio/frontend/src/components/panel/UpstreamProfileEditor.tsx` - -### `#26` 失败重试开关 - -已支持全局失败自动重试开关。 - -主要落点： - -- `image-studio/frontend/src/components/panel/SettingsPanel.tsx` -- `image-studio/frontend/src/platform/android/settings/AndroidSettingsPanel.tsx` - -### `#25` 上游配置升级后的同步问题 - -已支持上游配置导入 / 导出；兼容状态也会落到宿主侧共享路径。 - -主要落点： - -- `image-studio/frontend/src/lib/upstreamConfigTransfer.ts` -- `image-studio/backend/compatibility.go` - -### `#24` 支持全部删除 - -历史结果已支持全部删除。 - -主要落点： - -- `image-studio/frontend/src/components/history/HistoryRail.tsx` - -## 已部分解决，但还缺更强实机证明 - -### `#36` 多张并发时模糊 / 只出轮廓 - -当前已做： - -- 不再把 `partial_image` 预览当成最终成功结果。 -- “只拿到流式预览，没有最终图”会被判定为不完整并进入重试。 -- Android native HTTP 桥接做了长超时、精简进度、原始响应落盘、宿主侧提取最终图。 -- 新增“流式预览保护”开关： - - Android：并发 `>=2` 自动关闭流式预览。 - - Android：2K / 4K 大尺寸自动关闭流式预览。 - - Desktop：并发 `>=8` 自动关闭流式预览。 -- “生成后保存提示”与“完成提示音”都已有设置入口，并补了回归测试，确认关掉后仍能重新打开。 - -当前证据： - -- 前端测试：`114/114` 通过。 -- `go-cli/pkg/client`、`image-studio/backend`、`shared/compat-go` 测试通过。 -- 2026-06-07 本地平台总链再次通过： - - `IMAGE_STUDIO_INCLUDE_ISSUE_CLOSE_VERIFY=1 IMAGE_STUDIO_VERIFY_RESULTS_DIR=/private/tmp/verify-results-issue-close-final4 node scripts/verify-local-platform-kernel.mjs` - - `platform-kernel-summary.json` 状态为 `passed`，共 `11` 个步骤。 -- 总的平台验证链已显式覆盖： - - 本地 `live verify` smoke - - `image-studio` - - `shared/compat-go` - - `go-cli/pkg/client` -- `live-verify.mjs` 已有本地 smoke 证明： - - `scripts/verify-local-live-verify.mjs` - - 会启动仓库内 mock upstream，再跑 direct vs worker parity，确认 `live verify` 脚本本身可执行。 -- Android 本地校验脚本可通过： - - `scripts/verify-local-android-shell.mjs` - - 会先跑 Android JVM 单测，再检查 debug APK 的 `versionName` / `versionCode`、APK 签名和前端资源注入。 - - 若本机接入设备或模拟器，还可用 `IMAGE_STUDIO_ANDROID_DEVICE_SMOKE=1` 继续做 APK 安装 / 启动 smoke。 -- 2026-06-07 的结构化子结果也都已通过： - - `/private/tmp/verify-results-issue-close-final4/live-verify.json` - - `/private/tmp/verify-results-issue-close-final4/local-smoke.json` - - `/private/tmp/verify-results-issue-close-final4/android-shell.json` - - `/private/tmp/verify-results-issue-close-final4/macos-release.json` - - `/private/tmp/verify-results-issue-close-final4/issue-close-tooling.json` -- `issue-close-tooling.json` 还会额外沉淀： - - mock GitHub 下的 `comment-only` / `comment-and-close` 执行结果 - - 导出评论包摘要 - - 是否跳过 GitHub 实时同步 -- 本地 smoke / local live verify / Android shell / macOS release 这几条子验证脚本在中途失败时，也会尽量把环境与错误原因写进各自 JSON 结果，避免只剩控制台日志。 - -这轮还额外确认了两个与 `#36` 相邻的关键点： - -- 设置里新增的“流式预览保护”开关已经在桌面端和 Android 端落地，不再只是策略硬编码。 -- 编译后的 macOS App 已完成实际探针联调，确认同 semver core 的 CI 版本不会再误报更新： - - `runtimeUpdateProbe.hasUpdate = false` - - `runtimeUpdateProbe.shouldShowUpdate = false` - - `runtimeUpdateProbe.appUpdateModalOpen = false` - -仍缺： - -- 真实高并发上游环境下的长链路实机证明。 -- Windows 真机 / Android 真机上更大批量、更大尺寸任务的稳定性复核。 - -## 已修代码，但缺目标平台外观实机确认 - -### `#30` 窗口标题背景与标题栏颜色不一致 - -Windows Fluent 标题区和工作区上沿的颜色对齐代码已调整。 - -主要落点： - -- `image-studio/frontend/src/styles/fluent/_windows-theme.css` -- `image-studio/frontend/src/components/layout/WorkspaceBar.tsx` -- `image-studio/frontend/test/windowsThemeParity.test.mjs` - -当前额外保证： - -- 前端标题区与 workspace 条带现在共用 `--window-titlebar-bg` token。 -- 自动化测试会校验这个 token 与 `main.go` 里 Wails 原生标题栏颜色一致，避免后续再漂移。 - -仍缺： - -- Windows 真机视觉确认。 - -## 按当前范围暂不继续推进 - -### `#23` 不生图 - -按当前任务约定，视为上游问题，本轮不继续处理。 - -### `#14` 有 web 端吗 - -当前仓库已经明确说明： - -- 没有独立在线 Web / SaaS 版。 -- 浏览器形态主要用于调试和目标平台预览。 - -但“独立在线 Web 版”已经被明确后移，当前暂不继续推进。 - -## 建议的下一步 - -1. 如果有 Windows 真机，先补 `#30` 外观确认，再决定是否需要继续微调标题区样式。 -2. 如果有稳定可复现的真实上游并发环境，重点补 `#36` 的 Windows / Android 实机长链路证明。 - 手工验证步骤可直接按 [manual-verification.md](./manual-verification.md) 执行。 -3. 如果允许维护 issue 状态，可以把上述“代码已覆盖”的 issue 补评论或直接关闭，避免后续重复盘点。 -4. `#14` 等独立在线 Web 版需求，继续保持搁置，直到桌面 / Android 路径完全收口。 - -## 关单顺序建议 - -如果下一步允许直接维护 GitHub issue，建议按这个顺序处理： - -1. 先关闭“代码已覆盖且不依赖外部条件”的 open issue： - `#24` `#25` `#26` `#27` `#28` `#32` `#35` `#37` `#40` `#42` -2. 保留 `#30` 与 `#36` 为 open，直到真机 / 真实上游证据补齐。 -3. 保持 `#23` 不处理、`#14` 搁置，避免范围重新发散。 - -## 剩余执行计划 - -### 阶段 1：清理已完成但未关单的 issue - -目标： - -- 先把“代码已经落地、当前本地验证已覆盖、且不依赖外部条件”的 open issue 清掉。 - -建议顺序： - -1. `#24` 全部删除 -2. `#25` 上游配置导入导出 / 共享 -3. `#26` 自动重试开关 -4. `#27` 失败重试路由 -5. `#28` 提示词模板 -6. `#32` 拖结果到文件复制 -7. `#35` 工作区横向滚动 -8. `#37` 画板内左右切换同批结果 -9. `#40` Android 多图张数按钮 -10. `#42` 图生图相关需求集合 - -执行方式： - -- 每个 issue 至少补一条评论，附上对应功能点与主要落点。 -- 若仓库维护方式允许，评论后可直接关闭。 - -### 阶段 2：完成 `#30` Windows 真机确认 - -目标： - -- 确认原生标题栏、应用标题区、workspace 条带在 Windows 真机上亮/暗色都无明显断层。 - -执行方式： - -1. 按 [manual-verification.md](./manual-verification.md) 初始化 `30-windows` 手工验证目录。 -2. 使用当前构建版本在 Windows 真机截图取证。 -3. 通过后给 `#30` 补图和结论，再关闭 issue。 - -### 阶段 3：完成 `#36` 真实环境长链路验证 - -目标： - -- 验证高并发与大尺寸场景下，“流式预览保护”策略和“最终图完整性优先”策略在真实环境里成立。 - -执行方式： - -1. Android 真机 / 模拟器执行 `36-android` 验证矩阵。 -2. Windows 真机执行 `36-windows` 验证矩阵。 -3. 尽量使用真实高并发上游，覆盖： - - 并发 1 - - 并发 2 / 8 - - 2K / 4K - - 保护开关开 / 关对照 -4. 保存截图、raw 响应、日志和结论。 -5. 证据充分后再决定是否关闭 `#36`。 - -### 阶段 4：保持范围稳定 - -当前明确不进入本轮收口的内容： - -- `#23`：按上游问题处理。 -- `#14`：独立在线 Web 版继续搁置。 - -如果后续重新启动这两项，应作为新阶段单独规划，不与当前桌面 / Android 收尾混在一起。 +更新时间：2026-10-06（Asia/Shanghai）。本次重新读取上游全部 open issue 及评论，共 12 项。旧记录里的“可关闭”结论已重新核对，GitHub 上的 open 状态不等于当前分支缺少实现。 + +| Issue | 本次核对与处理 | 验证与剩余条件 | +|---|---|---| +| [#58 历史图片不完整](https://github.com/RoseKhlifa/Image-Studio/issues/58) | 移除前端和 Gio 的 120 条截断；取消生成/导入时按可见列表删除数据库；按时间与 ID 分页；导出完整档案，合并旧客户端的部分快照；修复筛选加载及未加载记录的日期清理 | IndexedDB 650 条分页、同时间戳、旧全图保留、共享快照导入/导出测试；已被旧版本删除的提示词不能由图片文件自动恢复 | +| [#57 失败声音提示](https://github.com/RoseKhlifa/Image-Studio/issues/57) | 提交失败、生成失败、处理结果失败均触发可见错误提示；复用任务提示音开关；后台且授权时发失败系统通知；同次提交的并发错误只提醒一次，取消保持静默；Gio 同步支持 | 真实 store 事件链与声音/通知 fixture 测试，覆盖并发失败、提交拒绝、取消；实际扬声器与通知权限依赖宿主环境 | +| [#55 拖出复制导致其他应用无法拖动](https://github.com/RoseKhlifa/Image-Studio/issues/55) | 五个入口统一鼠标手势启动原生拖动，取消 HTML/native 双循环；处理 mouseup、Esc、失焦、卸载；Windows 拒绝已结束的手势，添加 OLE 定时退出与捕获释放；失败显示 toast | 手势事件、退出策略单测与 Windows amd64/arm64 编译；仍需 Windows 上确认 Explorer 和其他应用拖拽恢复 | +| [#53 可选 API 蒙版](https://github.com/RoseKhlifa/Image-Studio/issues/53) | 现有桌面、Android、Gio 导入/清除入口已覆盖；Responses 发送 `input_image_mask`，Images edits 发送 multipart `mask` 并保留真实 MIME | 共享请求模型、Go、远程内核及 Android 测试；真实上游的 PNG/尺寸/透明通道约束仍适用 | +| [#52 OneCommander 导出成 .url](https://github.com/RoseKhlifa/Image-Studio/issues/52) | 保留原生 OLE 与 Unicode `CF_HDROP`，统一五个入口；浏览器回退在写文件格式后保留内部历史 MIME，避免元数据被 `clearData` 清掉 | `CF_HDROP` 双 NUL/Unicode、拖拽格式测试与 Windows 双架构编译；OneCommander 实机验收仍待完成 | +| [#51 Nano Banana 2](https://github.com/RoseKhlifa/Image-Studio/issues/51) | 复核官方 Interactions 文档与现有路由；官方主机/Google provider 使用 Interactions，标准兼容 relay 保持 Images 协议 | Go、共享模型与远程内核请求/响应 fixture；未调用收费真实上游，实际模型权限需发布时验证 | +| [#50 URL 图片结果](https://github.com/RoseKhlifa/Image-Studio/issues/50) | 现有 URL 下载与 base64 转换可用，检查 Go、浏览器和 Android 原生二进制路径及 50MB/图片类型限制 | Go、前端、Android fixture 和本机 mock upstream | +| [#49 Prompt 历史区分不清](https://github.com/RoseKhlifa/Image-Studio/issues/49) | 现有序号、分隔线和跨平台编号语义已覆盖 | Prompt 模板/历史测试及 Gio 测试 | +| [#44 文件夹批处理](https://github.com/RoseKhlifa/Image-Studio/issues/44) | 保留输入目录/多文件、并发、Auto 比例、输出目录、原名/前缀/去重；修复队列两轮之间错误清零与提交失败计数 | 真实 store 多轮队列、650 次结算、输出路径及客户端测试 | +| [#30 Windows 标题栏颜色](https://github.com/RoseKhlifa/Image-Studio/issues/30) | 现有 CSS token 与原生 Wails 亮/暗色配置匹配，主题切换路径仍受测试保护 | 自动化颜色一致性；Windows 原生标题栏视觉验收仍待完成 | +| [#24 全部删除](https://github.com/RoseKhlifa/Image-Studio/issues/24) | 现有原子清空与全工作区引用清理有效；本次补单条/按日期删除的旧库副本清理，避免再次迁移恢复 | 当前/旧库、全图、未加载记录、待保存队列及工作区回归测试 | +| [#14 Web 端问答](https://github.com/RoseKhlifa/Image-Studio/issues/14) | 已核对问题与“未来会有”的回复；README 已说明没有独立在线 Web 版，浏览器仅用于前端预览 | 产品问答已记录；当前描述没有独立 Web 产品的验收要求，不据此新增线上服务 | + +根因、维护约束和复现命令见 [issue-fixes-20261006.md](./issue-fixes-20261006.md)。本机验证结果见 [verification/issue-audit-20261006.json](./verification/issue-audit-20261006.json)。 + +`scripts/issue-close-data.json` 和 [issue-close-comments.md](./issue-close-comments.md) 同步本次清单：本地验证覆盖的 8 项、待 Windows 实机的 3 项、产品问答 1 项。这些文件是维护记录与待使用的评论草稿；本次没有向上游发评论或关闭 issue。 diff --git a/docs/manual-verification.md b/docs/manual-verification.md index e33cdcaf7..3fcc5fb36 100644 --- a/docs/manual-verification.md +++ b/docs/manual-verification.md @@ -4,6 +4,10 @@ - `#36` 多张并发时模糊 / 只出轮廓 - `#30` Windows 标题栏与标题区颜色一致性真机确认 +- `#52` OneCommander 拖出真实图片文件 +- `#55` Windows 原生拖动结束后，其他应用仍可正常拖拽 + +2026-10-06 的本机自动验证与编译结果见 [issue-fixes-20261006.md](./issue-fixes-20261006.md) 和 [验证记录](./verification/issue-audit-20261006.json)。真机验收应引用实际测试版本与证据，不能只引用旧关单评论中的测试数量。 在开始手工验证前，建议先跑一遍当前仓库内可自动完成的验证链： @@ -167,6 +171,19 @@ IMAGE_STUDIO_ANDROID_DEVICE_SMOKE=1 node scripts/verify-local-android-shell.mjs ## 结果回填建议 +### `#52/#55` Windows 拖出与捕获释放 + +1. 使用本分支构建的 Windows 应用，记录 Windows、WebView2 和 OneCommander 版本。 +2. 在画布“拖出复制”、历史缩略图、同提示词历史组、另存提示、结果详情五个入口分别拖一张已保存的图片到 Explorer 和 OneCommander。 +3. 确认得到 PNG/JPEG/WebP 图片文件，能直接打开，文件内容与原图一致；不能得到 `.url`。 +4. 每次正常松开、Esc 取消、快速按下/松开后，立即在桌面、Explorer 和另一个程序内拖动文件/内容，确认没有遗留鼠标捕获。 +5. 保持鼠标不移动超过 30 秒，确认当前拖动取消；之后其他应用仍可拖动。此超时保护针对正常 OLE 消息循环，不代替对卡死第三方 COM 目标的实际检查。 +6. 重复 20 次。至少保存成功拖出、取消后其他程序正常拖动的录屏，以及遇到错误时的 toast/日志。 + +通过标准：五处入口均输出真实图片，正常结束和取消后均恢复其他程序的拖拽；连续使用无挂起。检查完成前，`#52/#55` 保持待实机验证。 + +### 其他回填 + 如果允许继续维护 issue 状态，建议在 GitHub issue 回填： - 平台 diff --git a/docs/verification/issue-audit-20261006.json b/docs/verification/issue-audit-20261006.json new file mode 100644 index 000000000..1c9522de3 --- /dev/null +++ b/docs/verification/issue-audit-20261006.json @@ -0,0 +1,290 @@ +{ + "date": "2026-10-06", + "timezone": "Asia/Shanghai", + "recordedAt": "2026-10-06T13:37:25.146269+08:00", + "status": "passed", + "scope": "Local automated verification only; manual checks are listed separately.", + "baseCommit": "c49519cfb1a230b6fbcacd451f2b6717f2200af0", + "branch": "codex/issue-audit-20261006", + "environment": { + "os": "macOS", + "architecture": "arm64", + "node": "v25.8.1", + "go": "go1.26.3", + "androidExecution": "local Docker with SDK 34; JVM target 17; runtime JDK 21" + }, + "checks": [ + { + "name": "frontend", + "status": "passed", + "command": "cd image-studio/frontend && npm test", + "detail": { + "tests": 203, + "passed": 203, + "failed": 0 + }, + "logSha256": "efa4814ada05ab64cc1689b519eb94cbe2b25c0a9fdb04543e42925edbf6fb1d" + }, + { + "name": "worker", + "status": "passed", + "command": "cd cloudflare-worker && npm test", + "detail": { + "tests": 5, + "passed": 5, + "failed": 0 + }, + "logSha256": "c4e10511516407b5d39d48f7dacd7bfb91aad2c2249b1cac3ca7085c420b5ba6" + }, + { + "name": "desktop-go", + "status": "passed", + "command": "cd image-studio && go test ./...", + "detail": { + "allPackagesPassed": true + }, + "logSha256": "94a00f17329d4eb9b5c528f6e4739e5fe504f5fb9ec5f08cea2ec1a409b70e28" + }, + { + "name": "client-go", + "status": "passed", + "command": "cd go-cli && go test ./...", + "detail": { + "allPackagesPassed": true + }, + "logSha256": "dcd3d10878ef71e35dcfe340541512037083d7408de917efe226622bd74009c9" + }, + { + "name": "gio-go", + "status": "passed", + "command": "cd gio-client && go test ./...", + "detail": { + "allPackagesPassed": true + }, + "logSha256": "dcf9c55da718ee63d3ed569b6cf3ea1d798393dc77050758aa4636709d9f2aff" + }, + { + "name": "compat-go", + "status": "passed", + "command": "cd shared/compat-go && go test ./...", + "detail": { + "allPackagesPassed": true + } + }, + { + "name": "frontend-builds", + "status": "passed", + "command": "npm run build: (image-studio/frontend)", + "detail": { + "platforms": [ + "windows", + "macos", + "linux", + "android-pad", + "android" + ], + "typecheckPassed": true + } + }, + { + "name": "windows-backend-compile", + "status": "passed", + "command": "GOOS=windows GOARCH= CGO_ENABLED=0 go test -c ./backend", + "detail": { + "architectures": [ + "amd64", + "arm64" + ], + "executionOnWindows": false + } + }, + { + "name": "live-verify", + "status": "passed", + "command": "node scripts/verify-local-live-verify.mjs", + "detail": { + "mockUpstream": true, + "completedAt": "2026-10-06T05:20:23.650Z" + } + }, + { + "name": "local-smoke", + "status": "passed", + "command": "node scripts/local-smoke-check.mjs", + "detail": { + "mockUpstream": true, + "completedAt": "2026-10-06T05:20:28.476Z" + } + }, + { + "name": "android-jvm-tests", + "status": "passed", + "command": "Docker SDK 34: ./gradlew --offline --no-daemon :app:testDebugUnitTest --rerun", + "detail": { + "tests": 15, + "suites": [ + { + "name": "top.gptcodex.imagestudio.android.AndroidStreamEventParserTest", + "tests": 9, + "failures": 0, + "errors": 0 + }, + { + "name": "top.gptcodex.imagestudio.android.ActiveTaskRegistryTest", + "tests": 3, + "failures": 0, + "errors": 0 + }, + { + "name": "top.gptcodex.imagestudio.android.AndroidBinaryResponseTest", + "tests": 3, + "failures": 0, + "errors": 0 + } + ], + "container": "ghcr.io/cirruslabs/android-sdk:34", + "jdk": "21.0.5", + "forcedTestExecution": true + }, + "logSha256": "7735e627edcbf7937149546f76096f6dbe43e022cdb311b2be7b23e97ad1838a" + }, + { + "name": "android-apk-build", + "status": "passed", + "command": "Docker SDK 34: ./gradlew --offline --no-daemon :app:testDebugUnitTest :app:assembleDebug", + "detail": { + "prebuiltFrontend": true, + "artifact": "android-shell/app/build/outputs/apk/debug/app-debug.apk" + }, + "logSha256": "fcca8c60bd6d387ad02b48d7d144d6c31c6d4f86f9922f79664b142683f7277e" + }, + { + "name": "android-apk-validation", + "status": "passed", + "command": "apksigner verify; binary AndroidManifest.xml and ZIP assets inspection", + "detail": { + "applicationId": "top.gptcodex.imagestudio.android.debug", + "versionCode": 1050001, + "versionName": "0.1.5-dev-debug", + "signatureV2": true, + "frontendHtml": "assets/index.html", + "assetEntries": 18, + "apkSha256": "67ca4c2ca20ea6e971f99472d3fd56b4c6708ae9bc0615c531b080f6da022270", + "badgingMethod": "binary manifest decoded locally; SDK aapt unavailable due amd64 loader mismatch in arm64 container" + } + }, + { + "name": "macos-release", + "status": "passed", + "command": "node scripts/verify-local-macos-release.mjs", + "detail": { + "architectures": [ + "x86_64", + "arm64" + ], + "signature": "adhoc", + "plistVerified": true, + "runtimeProbeExecuted": true, + "fixtureVersion": "1.1.13-ci.local+verify", + "completedAt": "2026-10-06T05:31:29.354Z" + } + }, + { + "name": "issue-tooling", + "status": "passed", + "command": "IMAGE_STUDIO_SKIP_ISSUE_CLOSE_GITHUB_SYNC=1 node scripts/verify-issue-close-tooling.mjs", + "detail": { + "checks": 23, + "realGitHubMutations": false, + "exportedComments": 8 + } + }, + { + "name": "issue-live-read", + "status": "passed", + "command": "node scripts/issue-close-helper.mjs verify-open --json", + "detail": { + "closable": 8, + "holdOpen": 3, + "deferred": 1, + "unexpectedOpen": 0, + "mutations": false + } + }, + { + "name": "dependencies", + "status": "passed", + "command": "cd image-studio/frontend && npm audit --json", + "detail": { + "knownVulnerabilities": 0 + } + }, + { + "name": "diff-check", + "status": "passed", + "command": "git diff --check", + "detail": { + "passed": true + } + } + ], + "issues": { + "upstream": "RoseKhlifa/Image-Studio", + "openAtAudit": [ + 58, + 57, + 55, + 53, + 52, + 51, + 50, + 49, + 44, + 30, + 24, + 14 + ], + "localCoverage": [ + 24, + 44, + 49, + 50, + 51, + 53, + 57, + 58 + ], + "windowsManualPending": [ + 30, + 52, + 55 + ], + "productQuestion": [ + 14 + ] + }, + "notExecuted": [ + { + "check": "Windows real-machine OneCommander/Explorer/capture release", + "issues": [ + 52, + 55 + ] + }, + { + "check": "Windows native titlebar visual confirmation", + "issues": [ + 30 + ] + }, + { + "check": "Paid real upstream and Android device install/runtime smoke", + "reason": "No supplied upstream credentials or connected target device were used." + } + ], + "limitations": [ + "Prior history metadata already deleted by older versions cannot be reconstructed from image files.", + "Cross-compiling Windows backends does not validate Windows OLE runtime behavior.", + "macOS linker deployment-version warnings were present; builds and runtime probe passed on this host." + ] +} diff --git a/gio-client/internal/compat/state.go b/gio-client/internal/compat/state.go index 799cc2102..71b29db72 100644 --- a/gio-client/internal/compat/state.go +++ b/gio-client/internal/compat/state.go @@ -459,7 +459,7 @@ func WriteAPIKey(profileID, value string) error { } func mergeHistory(item shared.HistoryItem, items []shared.HistoryItem) []shared.HistoryItem { - out := make([]shared.HistoryItem, 0, min(len(items)+1, 120)) + out := make([]shared.HistoryItem, 0, len(items)+1) seen := map[string]struct{}{item.ID: {}} out = append(out, item) for _, existing := range items { @@ -475,9 +475,6 @@ func mergeHistory(item shared.HistoryItem, items []shared.HistoryItem) []shared. sort.SliceStable(out, func(i, j int) bool { return out[i].CreatedAt > out[j].CreatedAt }) - if len(out) > 120 { - out = out[:120] - } return out } diff --git a/gio-client/internal/compat/state_test.go b/gio-client/internal/compat/state_test.go index e2113193a..d21e3fbd5 100644 --- a/gio-client/internal/compat/state_test.go +++ b/gio-client/internal/compat/state_test.go @@ -282,6 +282,27 @@ func TestSaveConfigAndHistorySerializesConcurrentHistoryUpdates(t *testing.T) { } } +func TestMergeHistoryRetainsLargeBatchAndDeduplicates(t *testing.T) { + items := make([]shared.HistoryItem, 650) + for i := range items { + items[i] = shared.HistoryItem{ID: fmt.Sprintf("image-%04d", i), Prompt: fmt.Sprintf("prompt-%d", i), CreatedAt: int64(i)} + } + updated := items[10] + updated.Prompt = "updated prompt" + merged := mergeHistory(updated, items) + if len(merged) != len(items) { + t.Fatalf("history len=%d want %d", len(merged), len(items)) + } + if merged[0].ID != "image-0649" || merged[len(merged)-1].ID != "image-0000" { + t.Fatal("history should preserve the entire archive in descending date order") + } + for _, item := range merged { + if item.ID == updated.ID && item.Prompt != updated.Prompt { + t.Fatal("the newest version of a duplicate should replace the old record") + } + } +} + func TestSaveStatePrunesHistoryFullForNonPreviewOnlyHistory(t *testing.T) { root := t.TempDir() origStable := StableDataRootForTest() diff --git a/gio-client/internal/ui/completion_notification.go b/gio-client/internal/ui/completion_notification.go index 9dcc4f86e..9728a8bc5 100644 --- a/gio-client/internal/ui/completion_notification.go +++ b/gio-client/internal/ui/completion_notification.go @@ -183,6 +183,23 @@ func (a *App) maybeSendCompletionNotification(item sharedCompat.HistoryItem, com }() } +func (a *App) notifyRunFailure(message string) { + a.maybePlayCompletionSound(1, 1) + a.mu.Lock() + enabled := a.completionNotification.Enabled + permission := a.completionNotificationPermission + focused := a.windowFocused + a.mu.Unlock() + if !enabled || focused || permission != systemNotificationPermissionGranted { + return + } + go func() { + if err := showSystemNotificationFunc("Image Studio · 生成失败", message, notificationOpenResultAction{}); err != nil { + a.appendLog("发送失败通知失败: " + err.Error()) + } + }() +} + func showSystemNotification(title string, body string, action notificationOpenResultAction) error { title = strings.TrimSpace(title) body = strings.TrimSpace(body) diff --git a/gio-client/internal/ui/completion_notification_test.go b/gio-client/internal/ui/completion_notification_test.go index c6e54cbee..36325453d 100644 --- a/gio-client/internal/ui/completion_notification_test.go +++ b/gio-client/internal/ui/completion_notification_test.go @@ -1,12 +1,52 @@ package ui import ( + "errors" "testing" "time" sharedCompat "image-studio/shared/compat" ) +func TestFailedRunNotifiesAndPlaysConfiguredSound(t *testing.T) { + app := &App{ + completionSound: sharedCompat.CompletionSoundSettings{Enabled: true, Mode: "default"}, + completionNotification: sharedCompat.CompletionNotificationSettings{Enabled: true}, + completionNotificationPermission: systemNotificationPermissionGranted, + } + sounds := make(chan bool, 1) + notifications := make(chan string, 1) + origSound, origNotification := playCompletionSoundFunc, showSystemNotificationFunc + t.Cleanup(func() { + playCompletionSoundFunc, showSystemNotificationFunc = origSound, origNotification + }) + playCompletionSoundFunc = func(config sharedCompat.CompletionSoundSettings, force bool) error { + sounds <- force + return nil + } + showSystemNotificationFunc = func(title, body string, _ notificationOpenResultAction) error { + notifications <- title + ":" + body + return nil + } + app.finishWithError(errors.New("HTTP 503"), "") + select { + case force := <-sounds: + if force { + t.Fatal("failure should honor the configured sound setting") + } + case <-time.After(time.Second): + t.Fatal("failure did not trigger sound") + } + select { + case got := <-notifications: + if got != "Image Studio · 生成失败:HTTP 503" { + t.Fatalf("notification=%q", got) + } + case <-time.After(time.Second): + t.Fatal("failure did not trigger background notification") + } +} + func TestShouldSendCompletionNotificationOnlyOnFinalBackgroundResult(t *testing.T) { config := sharedCompat.CompletionNotificationSettings{Enabled: true} if shouldSendCompletionNotification(config, 1, 3, false) { diff --git a/gio-client/internal/ui/run_job.go b/gio-client/internal/ui/run_job.go index 7b56fdf66..b6c0ba83b 100644 --- a/gio-client/internal/ui/run_job.go +++ b/gio-client/internal/ui/run_job.go @@ -305,6 +305,7 @@ func (a *App) startRunWithConfig(cfg kernel.Config, total int, workflowWorkspace var once sync.Once var firstErr atomic.Pointer[error] var jobsDone atomic.Int32 + var jobsFailed atomic.Int32 concurrency := runConcurrency if concurrency < 1 { concurrency = 1 @@ -396,6 +397,11 @@ func (a *App) startRunWithConfig(cfg kernel.Config, total int, workflowWorkspace return } if batchMode { + jobsFailed.Add(1) + errCopy := err + if firstErr.CompareAndSwap(nil, &errCopy) { + a.notifyRunFailure(err.Error()) + } a.appendLog(fmt.Sprintf("[%s] 失败并跳过: %v", jobLabel, err)) completed := int(jobsDone.Add(1)) a.mu.Lock() @@ -405,15 +411,14 @@ func (a *App) startRunWithConfig(cfg kernel.Config, total int, workflowWorkspace a.cancel = nil a.lastRunConcurrency = 0 a.clearBatchPreviewItemsLocked() - a.status = fmt.Sprintf("完成 - %.1fs", time.Since(batchStarted).Seconds()) + a.status = fmt.Sprintf("完成，%d 项失败 - %.1fs", jobsFailed.Load(), time.Since(batchStarted).Seconds()) } a.mu.Unlock() a.invalidateNow() return } - if firstErr.Load() == nil { - errCopy := err - firstErr.Store(&errCopy) + errCopy := err + if firstErr.CompareAndSwap(nil, &errCopy) { a.finishWithError(err, res.RawPath) } cancelAll() @@ -538,6 +543,9 @@ func (a *App) startRunWithConfig(cfg kernel.Config, total int, workflowWorkspace a.lastRunConcurrency = 0 a.clearBatchPreviewItemsLocked() a.status = fmt.Sprintf("完成 - %.1fs", time.Since(batchStarted).Seconds()) + if failed := jobsFailed.Load(); failed > 0 { + a.status = fmt.Sprintf("完成，%d 项失败 - %.1fs", failed, time.Since(batchStarted).Seconds()) + } } a.mu.Unlock() if openSavePromptAfterUnlock { @@ -546,7 +554,7 @@ func (a *App) startRunWithConfig(cfg kernel.Config, total int, workflowWorkspace if openBatchSavePromptAfterUnlock { a.openBatchSavePrompt(batchSaveItems) } - if completed == total { + if completed == total && firstErr.Load() == nil { a.maybePlayCompletionSound(completed, total) a.maybeSendCompletionNotification(displayItem, completed, total) } diff --git a/gio-client/internal/ui/state.go b/gio-client/internal/ui/state.go index aad4e0d99..293ae2702 100644 --- a/gio-client/internal/ui/state.go +++ b/gio-client/internal/ui/state.go @@ -69,6 +69,7 @@ func (a *App) finishWithError(err error, rawPath string) { } a.appendLogLocked("失败: " + err.Error()) a.mu.Unlock() + a.notifyRunFailure(err.Error()) a.invalidateNow() } diff --git a/image-studio/backend/native_drag_policy.go b/image-studio/backend/native_drag_policy.go new file mode 100644 index 000000000..33a15f7c9 --- /dev/null +++ b/image-studio/backend/native_drag_policy.go @@ -0,0 +1,23 @@ +package backend + +import "time" + +const nativeDragMaxDuration = 30 * time.Second + +type nativeDragAction uint8 + +const ( + nativeDragContinue nativeDragAction = iota + nativeDragDrop + nativeDragCancel +) + +func nativeDragNextAction(escape, leftButtonDown, asyncLeftButtonDown bool, elapsed time.Duration) nativeDragAction { + if escape || elapsed >= nativeDragMaxDuration { + return nativeDragCancel + } + if !leftButtonDown || !asyncLeftButtonDown { + return nativeDragDrop + } + return nativeDragContinue +} diff --git a/image-studio/backend/native_drag_policy_test.go b/image-studio/backend/native_drag_policy_test.go new file mode 100644 index 000000000..3fd03d009 --- /dev/null +++ b/image-studio/backend/native_drag_policy_test.go @@ -0,0 +1,28 @@ +package backend + +import ( + "testing" + "time" +) + +func TestNativeDragStopsOnReleaseEscapeAndDeadline(t *testing.T) { + for _, tc := range []struct { + name string + escape, leftDown, asyncDown bool + elapsed time.Duration + want nativeDragAction + }{ + {"held", false, true, true, time.Second, nativeDragContinue}, + {"released", false, false, true, time.Second, nativeDragDrop}, + {"missed mouseup", false, true, false, time.Second, nativeDragDrop}, + {"escape", true, true, true, time.Second, nativeDragCancel}, + {"deadline", false, true, true, nativeDragMaxDuration, nativeDragCancel}, + {"expired release", false, false, false, nativeDragMaxDuration, nativeDragCancel}, + } { + t.Run(tc.name, func(t *testing.T) { + if got := nativeDragNextAction(tc.escape, tc.leftDown, tc.asyncDown, tc.elapsed); got != tc.want { + t.Fatalf("drag action=%v want %v", got, tc.want) + } + }) + } +} diff --git a/image-studio/backend/native_drag_windows.go b/image-studio/backend/native_drag_windows.go index 06de40636..c14054c81 100644 --- a/image-studio/backend/native_drag_windows.go +++ b/image-studio/backend/native_drag_windows.go @@ -6,6 +6,7 @@ import ( "fmt" "runtime" "strings" + "sync" "sync/atomic" "syscall" "time" @@ -25,7 +26,8 @@ const ( mouseKeyStateLeftButtonDown uintptr = 0x00000001 globalAllocMoveableZeroInit uintptr = 0x00000042 virtualKeyLeftButton uintptr = 0x01 - nativeDragMaxDuration = 30 * time.Second + virtualKeyEscape uintptr = 0x1b + wmKeyDown uintptr = 0x0100 cfHDrop uint16 = 15 dvAspectContent uint32 = 1 tymedHGlobal uint32 = 1 @@ -47,6 +49,13 @@ var ( procGlobalUnlock = kernel32.NewProc("GlobalUnlock") procGlobalFree = kernel32.NewProc("GlobalFree") procGetAsyncKeyState = user32.NewProc("GetAsyncKeyState") + procGetCapture = user32.NewProc("GetCapture") + procReleaseCapture = user32.NewProc("ReleaseCapture") + procSetTimer = user32.NewProc("SetTimer") + procKillTimer = user32.NewProc("KillTimer") + procPostMessage = user32.NewProc("PostMessageW") + nativeDragTimers sync.Map + nativeDragTimerProc = windows.NewCallback(nativeDragTimerTick) iidIUnknown = windows.GUID{Data1: 0x00000000, Data2: 0x0000, Data3: 0x0000, Data4: [8]byte{0xc0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x46}} iidIDataObj = windows.GUID{Data1: 0x0000010e, Data2: 0x0000, Data3: 0x0000, Data4: [8]byte{0xc0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x46}} @@ -87,6 +96,13 @@ func beginNativeFileDrag(path string) error { runtime.LockOSThread() defer runtime.UnlockOSThread() + // An asynchronous WebView binding can arrive after mouseup. Never acquire + // native capture for an already finished gesture. + asyncState, _, _ := procGetAsyncKeyState.Call(virtualKeyLeftButton) + if uint16(asyncState)&0x8000 == 0 { + return nil + } + defer procReleaseCapture.Call() hr, _, _ := procOleInitialize.Call(0) if failedHRESULT(hr) { @@ -121,7 +137,16 @@ func beginNativeFileDrag(path string) error { } source := &dropSource{lpVtbl: &nativeDropSourceVtbl, refs: 1, startedUnix: time.Now().UnixNano()} - var effect uintptr + timer, _, timerErr := procSetTimer.Call(0, 0, 100, nativeDragTimerProc) + if timer == 0 { + return fmt.Errorf("SetTimer for native drag failed: %v", timerErr) + } + nativeDragTimers.Store(timer, source) + defer func() { + procKillTimer.Call(0, timer) + nativeDragTimers.Delete(timer) + }() + var effect uint32 hr, _, _ = procDoDragDrop.Call( dataObject, uintptr(unsafe.Pointer(source)), @@ -241,25 +266,38 @@ func dropSourceRelease(this uintptr) uintptr { } func dropSourceQueryContinueDrag(this uintptr, escapePressed uintptr, keyState uintptr) uintptr { - if escapePressed != 0 { - return dragDropSCancel - } + var elapsed time.Duration if this != 0 { source := (*dropSource)(unsafe.Pointer(this)) - if source.startedUnix > 0 && time.Since(time.Unix(0, source.startedUnix)) >= nativeDragMaxDuration { - return dragDropSCancel - } + elapsed = time.Since(time.Unix(0, source.startedUnix)) } asyncState, _, _ := procGetAsyncKeyState.Call(virtualKeyLeftButton) - if keyState&mouseKeyStateLeftButtonDown == 0 || uint16(asyncState)&0x8000 == 0 { + switch nativeDragNextAction(escapePressed != 0, keyState&mouseKeyStateLeftButtonDown != 0, uint16(asyncState)&0x8000 != 0, elapsed) { + case nativeDragCancel: + return dragDropSCancel + case nativeDragDrop: return dragDropSDrop } - // Some third-party file managers poll IDropSource extremely aggressively. - // Yield briefly so a stalled target cannot pin a CPU core while hovering. - time.Sleep(time.Millisecond) return sOK } +func nativeDragTimerTick(_ uintptr, _ uintptr, timer uintptr, _ uintptr) uintptr { + value, ok := nativeDragTimers.Load(timer) + if !ok { + return 0 + } + source := value.(*dropSource) + if time.Since(time.Unix(0, source.startedUnix)) >= nativeDragMaxDuration { + // QueryContinueDrag is input-driven. Wake the owning OLE loop even if + // the user has stopped moving; never synthesize global keyboard input. + capture, _, _ := procGetCapture.Call() + if capture != 0 { + procPostMessage.Call(capture, wmKeyDown, virtualKeyEscape, 1) + } + } + return 0 +} + func dropSourceGiveFeedback(_ uintptr, _ uintptr) uintptr { return dragDropSUseDefaultCursors } diff --git a/image-studio/frontend/package-lock.json b/image-studio/frontend/package-lock.json index 27e0420b1..636dfdbc8 100644 --- a/image-studio/frontend/package-lock.json +++ b/image-studio/frontend/package-lock.json @@ -7,6 +7,7 @@ "": { "name": "frontend", "version": "0.1.5", + "license": "AGPL-3.0-only", "dependencies": { "idb-keyval": "^6.2.2", "konva": "^9.3.22", @@ -21,19 +22,20 @@ "@types/react": "^18.3.0", "@types/react-dom": "^18.3.0", "@vitejs/plugin-react": "^4.3.0", + "fake-indexeddb": "^6.2.5", "tailwindcss": "^4.0.0", "typescript": "^5.7.0", "vite": "^6.0.0" } }, "node_modules/@babel/code-frame": { - "version": "7.29.0", - "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.29.0.tgz", - "integrity": "sha512-9NhCeYjq9+3uxgdtp20LSiJXJvN0FeCtNGpJxuMFZ1Kv3cWUNb6DOhJwUvcVCzKGR66cw4njwM6hrJLqgOwbcw==", + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.29.7.tgz", + "integrity": "sha512-Aup7aUOfpbAUg2ROOJN6Iw5f9DMBlzu0mIkm/malLQFN/YQgO48wCj0Kxa3sEHJvPVFg7siR+qRInwXd2qhQKw==", "dev": true, "license": "MIT", "dependencies": { - "@babel/helper-validator-identifier": "^7.28.5", + "@babel/helper-validator-identifier": "^7.29.7", "js-tokens": "^4.0.0", "picocolors": "^1.1.1" }, @@ -42,9 +44,9 @@ } }, "node_modules/@babel/compat-data": { - "version": "7.29.3", - "resolved": "https://registry.npmjs.org/@babel/compat-data/-/compat-data-7.29.3.tgz", - "integrity": "sha512-LIVqM46zQWZhj17qA8wb4nW/ixr2y1Nw+r1etiAWgRM6U1IqP+LNhL1yg440jYZR72jCWcWbLWzIosH+uP1fqg==", + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/compat-data/-/compat-data-7.29.7.tgz", + "integrity": "sha512-locTkQyKvwIEgBzVrn8693ebc97F2U8ZHjbXwDXJ5Fn2TCpNwTlKcaKLkdHop5c/icOFE7qt7Q9JC5hnKNa6Gg==", "dev": true, "license": "MIT", "engines": { @@ -52,21 +54,21 @@ } }, "node_modules/@babel/core": { - "version": "7.29.0", - "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.29.0.tgz", - "integrity": "sha512-CGOfOJqWjg2qW/Mb6zNsDm+u5vFQ8DxXfbM09z69p5Z6+mE1ikP2jUXw+j42Pf1XTYED2Rni5f95npYeuwMDQA==", + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.29.7.tgz", + "integrity": "sha512-RgHBCvtjbOK2gXSNBNIkNoEc9qoVEtau3hj8gEqKQuL3HZAibKarWFEI3Lfm6EYKkLalOh8eSrj9b+ch9H/VBA==", "dev": true, "license": "MIT", "dependencies": { - "@babel/code-frame": "^7.29.0", - "@babel/generator": "^7.29.0", - "@babel/helper-compilation-targets": "^7.28.6", - "@babel/helper-module-transforms": "^7.28.6", - "@babel/helpers": "^7.28.6", - "@babel/parser": "^7.29.0", - "@babel/template": "^7.28.6", - "@babel/traverse": "^7.29.0", - "@babel/types": "^7.29.0", + "@babel/code-frame": "^7.29.7", + "@babel/generator": "^7.29.7", + "@babel/helper-compilation-targets": "^7.29.7", + "@babel/helper-module-transforms": "^7.29.7", + "@babel/helpers": "^7.29.7", + "@babel/parser": "^7.29.7", + "@babel/template": "^7.29.7", + "@babel/traverse": "^7.29.7", + "@babel/types": "^7.29.7", "@jridgewell/remapping": "^2.3.5", "convert-source-map": "^2.0.0", "debug": "^4.1.0", @@ -83,14 +85,14 @@ } }, "node_modules/@babel/generator": { - "version": "7.29.1", - "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.29.1.tgz", - "integrity": "sha512-qsaF+9Qcm2Qv8SRIMMscAvG4O3lJ0F1GuMo5HR/Bp02LopNgnZBC/EkbevHFeGs4ls/oPz9v+Bsmzbkbe+0dUw==", + "version": "7.29.8", + "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.29.8.tgz", + "integrity": "sha512-gZbepsdh3WDtgZKWL+vTPh71LSBrm/Y4/QDZBVCcYfmeTEEuoOYwlSy+G1StfJg+/Zy550u/3TATbm7qDbbMtg==", "dev": true, "license": "MIT", "dependencies": { - "@babel/parser": "^7.29.0", - "@babel/types": "^7.29.0", + "@babel/parser": "^7.29.8", + "@babel/types": "^7.29.8", "@jridgewell/gen-mapping": "^0.3.12", "@jridgewell/trace-mapping": "^0.3.28", "jsesc": "^3.0.2" @@ -100,14 +102,14 @@ } }, "node_modules/@babel/helper-compilation-targets": { - "version": "7.28.6", - "resolved": "https://registry.npmjs.org/@babel/helper-compilation-targets/-/helper-compilation-targets-7.28.6.tgz", - "integrity": "sha512-JYtls3hqi15fcx5GaSNL7SCTJ2MNmjrkHXg4FSpOA/grxK8KwyZ5bubHsCq8FXCkua6xhuaaBit+3b7+VZRfcA==", + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-compilation-targets/-/helper-compilation-targets-7.29.7.tgz", + "integrity": "sha512-wem6WaBj4NaVYVdNhLPPVacES6ZJ+KBBfSkTMD3YZxbP3rm3Di85tJU5ljaUNhaOynt+Aj0xruhYuzQBt8n71g==", "dev": true, "license": "MIT", "dependencies": { - "@babel/compat-data": "^7.28.6", - "@babel/helper-validator-option": "^7.27.1", + "@babel/compat-data": "^7.29.7", + "@babel/helper-validator-option": "^7.29.7", "browserslist": "^4.24.0", "lru-cache": "^5.1.1", "semver": "^6.3.1" @@ -117,9 +119,9 @@ } }, "node_modules/@babel/helper-globals": { - "version": "7.28.0", - "resolved": "https://registry.npmjs.org/@babel/helper-globals/-/helper-globals-7.28.0.tgz", - "integrity": "sha512-+W6cISkXFa1jXsDEdYA8HeevQT/FULhxzR99pxphltZcVaugps53THCeiWA8SguxxpSp3gKPiuYfSWopkLQ4hw==", + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-globals/-/helper-globals-7.29.7.tgz", + "integrity": "sha512-3nQVUAtvkKH9zahfWgw96Jc/uFOmjACE1kQz82E2lqWmHBgjzbNlsC22nuQTfahmWeQtTq5nQ/4Nnd2A1wj4zA==", "dev": true, "license": "MIT", "engines": { @@ -127,29 +129,29 @@ } }, "node_modules/@babel/helper-module-imports": { - "version": "7.28.6", - "resolved": "https://registry.npmjs.org/@babel/helper-module-imports/-/helper-module-imports-7.28.6.tgz", - "integrity": "sha512-l5XkZK7r7wa9LucGw9LwZyyCUscb4x37JWTPz7swwFE/0FMQAGpiWUZn8u9DzkSBWEcK25jmvubfpw2dnAMdbw==", + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-module-imports/-/helper-module-imports-7.29.7.tgz", + "integrity": "sha512-ejHwrQQYcm9xnTivShn2IDOlIzInN34AXskvq9QicvCtEzq1Vzclu/tKF8Jq1Cg8JG2GL6/EmjgsCT7lXepE3g==", "dev": true, "license": "MIT", "dependencies": { - "@babel/traverse": "^7.28.6", - "@babel/types": "^7.28.6" + "@babel/traverse": "^7.29.7", + "@babel/types": "^7.29.7" }, "engines": { "node": ">=6.9.0" } }, "node_modules/@babel/helper-module-transforms": { - "version": "7.28.6", - "resolved": "https://registry.npmjs.org/@babel/helper-module-transforms/-/helper-module-transforms-7.28.6.tgz", - "integrity": "sha512-67oXFAYr2cDLDVGLXTEABjdBJZ6drElUSI7WKp70NrpyISso3plG9SAGEF6y7zbha/wOzUByWWTJvEDVNIUGcA==", + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-module-transforms/-/helper-module-transforms-7.29.7.tgz", + "integrity": "sha512-UPUVSyXbOh627KiCIGQSgwWzGeBKLkaJ9PJEdrngIwMSzxLR4jS4+f1f1jb7VzBbg8nFLaYotvVPFCTqdrmTAg==", "dev": true, "license": "MIT", "dependencies": { - "@babel/helper-module-imports": "^7.28.6", - "@babel/helper-validator-identifier": "^7.28.5", - "@babel/traverse": "^7.28.6" + "@babel/helper-module-imports": "^7.29.7", + "@babel/helper-validator-identifier": "^7.29.7", + "@babel/traverse": "^7.29.7" }, "engines": { "node": ">=6.9.0" @@ -169,9 +171,9 @@ } }, "node_modules/@babel/helper-string-parser": { - "version": "7.27.1", - "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.27.1.tgz", - "integrity": "sha512-qMlSxKbpRlAridDExk92nSobyDdpPijUq2DW6oDnUqd0iOGxmQjyqhMIihI9+zv4LPyZdRje2cavWPbCbWm3eA==", + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.29.7.tgz", + "integrity": "sha512-Pb5ijPrZ89GDH8223L4UP8i6QApWxs04RbPQJTeWDV0/keR2E36MeKnyr6LYmUUvqRRI+Iv87SuF1W6ErINzYw==", "dev": true, "license": "MIT", "engines": { @@ -179,9 +181,9 @@ } }, "node_modules/@babel/helper-validator-identifier": { - "version": "7.28.5", - "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.28.5.tgz", - "integrity": "sha512-qSs4ifwzKJSV39ucNjsvc6WVHs6b7S03sOh2OcHF9UHfVPqWWALUsNUVzhSBiItjRZoLHx7nIarVjqKVusUZ1Q==", + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.29.7.tgz", + "integrity": "sha512-qehxGkRj55h/ff8EMaJ+cYhyaKlHIxqYDn682wQD7RNp9UujOQsHog2uS0r2vzr4pW+sXf90NeeayjcNaX3fFg==", "dev": true, "license": "MIT", "engines": { @@ -189,9 +191,9 @@ } }, "node_modules/@babel/helper-validator-option": { - "version": "7.27.1", - "resolved": "https://registry.npmjs.org/@babel/helper-validator-option/-/helper-validator-option-7.27.1.tgz", - "integrity": "sha512-YvjJow9FxbhFFKDSuFnVCe2WxXk1zWc22fFePVNEaWJEu8IrZVlda6N0uHwzZrUM1il7NC9Mlp4MaJYbYd9JSg==", + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-option/-/helper-validator-option-7.29.7.tgz", + "integrity": "sha512-N9ZErrD+yW5geCDtBqnOoxmR8+tNKiGuxKlDpuJxfsqpa2dFcexaziGAE/qoHLiDDreVNMupxGmSoNlyvsA3gw==", "dev": true, "license": "MIT", "engines": { @@ -199,27 +201,27 @@ } }, "node_modules/@babel/helpers": { - "version": "7.29.2", - "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.29.2.tgz", - "integrity": "sha512-HoGuUs4sCZNezVEKdVcwqmZN8GoHirLUcLaYVNBK2J0DadGtdcqgr3BCbvH8+XUo4NGjNl3VOtSjEKNzqfFgKw==", + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.29.7.tgz", + "integrity": "sha512-1k2lAGRMfHTcwuNYcCNUmaUffmQv8KWMfh2iJUUeRlwlwH4FdNG7mfPI10NPfLHJFThE4Tyr4mv7kTNZOiPuBg==", "dev": true, "license": "MIT", "dependencies": { - "@babel/template": "^7.28.6", - "@babel/types": "^7.29.0" + "@babel/template": "^7.29.7", + "@babel/types": "^7.29.7" }, "engines": { "node": ">=6.9.0" } }, "node_modules/@babel/parser": { - "version": "7.29.3", - "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.29.3.tgz", - "integrity": "sha512-b3ctpQwp+PROvU/cttc4OYl4MzfJUWy6FZg+PMXfzmt/+39iHVF0sDfqay8TQM3JA2EUOyKcFZt75jWriQijsA==", + "version": "7.29.9", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.29.9.tgz", + "integrity": "sha512-CjXrNHTnvqBVqHgdBysY3vk2T8tpJHb5/RMeHJBTyVa9xgugCB0CJTx/3oO8RV2QRQP391RWpB7D6hLjm8V9uA==", "dev": true, "license": "MIT", "dependencies": { - "@babel/types": "^7.29.0" + "@babel/types": "^7.29.8" }, "bin": { "parser": "bin/babel-parser.js" @@ -261,33 +263,33 @@ } }, "node_modules/@babel/template": { - "version": "7.28.6", - "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.28.6.tgz", - "integrity": "sha512-YA6Ma2KsCdGb+WC6UpBVFJGXL58MDA6oyONbjyF/+5sBgxY/dwkhLogbMT2GXXyU84/IhRw/2D1Os1B/giz+BQ==", + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.29.7.tgz", + "integrity": "sha512-puq+Gf35oI24FeN11LkoUQFqv9uwNeWpxXZi/Ji3rRIoKAzKnxRaZ+Gkj0vKS9ZCiTESfng1N9LyOyXvo+m+Gg==", "dev": true, "license": "MIT", "dependencies": { - "@babel/code-frame": "^7.28.6", - "@babel/parser": "^7.28.6", - "@babel/types": "^7.28.6" + "@babel/code-frame": "^7.29.7", + "@babel/parser": "^7.29.7", + "@babel/types": "^7.29.7" }, "engines": { "node": ">=6.9.0" } }, "node_modules/@babel/traverse": { - "version": "7.29.0", - "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.29.0.tgz", - "integrity": "sha512-4HPiQr0X7+waHfyXPZpWPfWL/J7dcN1mx9gL6WdQVMbPnF3+ZhSMs8tCxN7oHddJE9fhNE7+lxdnlyemKfJRuA==", + "version": "7.29.8", + "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.29.8.tgz", + "integrity": "sha512-I5z7H3bf/41ktsNVLtpN0wAa336HkqIHQ5BuPLEhTkt1jVSyZpeNKIzTgEWmlxjdg81R0IgUCcaE+Ok3NvrfZg==", "dev": true, "license": "MIT", "dependencies": { - "@babel/code-frame": "^7.29.0", - "@babel/generator": "^7.29.0", - "@babel/helper-globals": "^7.28.0", - "@babel/parser": "^7.29.0", - "@babel/template": "^7.28.6", - "@babel/types": "^7.29.0", + "@babel/code-frame": "^7.29.7", + "@babel/generator": "^7.29.8", + "@babel/helper-globals": "^7.29.7", + "@babel/parser": "^7.29.8", + "@babel/template": "^7.29.7", + "@babel/types": "^7.29.8", "debug": "^4.3.1" }, "engines": { @@ -295,14 +297,14 @@ } }, "node_modules/@babel/types": { - "version": "7.29.0", - "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.29.0.tgz", - "integrity": "sha512-LwdZHpScM4Qz8Xw2iKSzS+cfglZzJGvofQICy7W7v4caru4EaAmyUuO6BGrbyQ2mYV11W0U8j5mBhd14dd3B0A==", + "version": "7.29.8", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.29.8.tgz", + "integrity": "sha512-Vj1jF3cPfxg7OAfoI7QnVKLoILlm2JF9pnVHrX8qx7AHMiYWT+NDAA7jChlNgRS4WTLc/fD1lXLmPixluj+3Gg==", "dev": true, "license": "MIT", "dependencies": { - "@babel/helper-string-parser": "^7.27.1", - "@babel/helper-validator-identifier": "^7.28.5" + "@babel/helper-string-parser": "^7.29.7", + "@babel/helper-validator-identifier": "^7.29.7" }, "engines": { "node": ">=6.9.0" @@ -1589,9 +1591,9 @@ } }, "node_modules/baseline-browser-mapping": { - "version": "2.10.31", - "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.10.31.tgz", - "integrity": "sha512-MujYO3eP72uvmSE0i4wltsodRfIpZATP3jvzRNRGGxgzId7aVocVJJV3nf01qnzzKFGxQVC9bpWxl5cjxTr/7Q==", + "version": "2.11.27", + "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.11.27.tgz", + "integrity": "sha512-ElY12DaROGuan+lMmZ8Cvo/ZUbXPe7Enc/9VU/b1T3Kp4dwytRcNdR8DoSJN5SNJT/CuvcCA0DHDVmMOCePdRQ==", "dev": true, "license": "Apache-2.0", "bin": { @@ -1602,9 +1604,9 @@ } }, "node_modules/browserslist": { - "version": "4.28.2", - "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.28.2.tgz", - "integrity": "sha512-48xSriZYYg+8qXna9kwqjIVzuQxi+KYWp2+5nCYnYKPTr0LvD89Jqk2Or5ogxz0NUMfIjhh2lIUX/LyX9B4oIg==", + "version": "4.29.3", + "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.29.3.tgz", + "integrity": "sha512-1R4kiYKXGViqEN0CnoDrXc1StD9niAwu+j2dukWzrD4bJgsD4lDmEp0CRbc6E/vYJIfTHwPmwyaKtVSudICdPA==", "dev": true, "funding": [ { @@ -1622,11 +1624,11 @@ ], "license": "MIT", "dependencies": { - "baseline-browser-mapping": "^2.10.12", - "caniuse-lite": "^1.0.30001782", - "electron-to-chromium": "^1.5.328", - "node-releases": "^2.0.36", - "update-browserslist-db": "^1.2.3" + "baseline-browser-mapping": "^2.11.26", + "caniuse-lite": "^1.0.30001813", + "electron-to-chromium": "^1.5.439", + "node-releases": "^2.0.57", + "update-browserslist-db": "^1.3.3" }, "bin": { "browserslist": "cli.js" @@ -1636,9 +1638,9 @@ } }, "node_modules/caniuse-lite": { - "version": "1.0.30001793", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001793.tgz", - "integrity": "sha512-iwSsYWaCOoh26cV8NwNRViHlrfUvYsHDfRVcbtmw0Kg6PJIZZXwMkj1442FYLBGkeUf1juAsU3DTfxW579mrPA==", + "version": "1.0.30001814", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001814.tgz", + "integrity": "sha512-/Uaf1lAzr59XcMpW0o96WoEfr+VXK2OX4U9AgFoiSHsVJ4HppnIFUjtYzsyDH2+tgANaQb2/oxYGwCPapN1FpA==", "dev": true, "funding": [ { @@ -1698,9 +1700,9 @@ } }, "node_modules/electron-to-chromium": { - "version": "1.5.360", - "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.360.tgz", - "integrity": "sha512-GkcBt6YYAw9SxFWn+xVar4cLVGlXVuswwtRLBozi2zp0GjXs4ZnOrqV4zbXzg35n7w81hCkyJNYicgXlVHAmBA==", + "version": "1.5.445", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.445.tgz", + "integrity": "sha512-iX7o1nF+qNb3Bv5mLYgqfgF9Znh9e/8MKXdmbb8Gv2Q47OKIzTwEPD1oda1HrgEwYatTLJj0wQjHvIFt8Db53Q==", "dev": true, "license": "ISC" }, @@ -1770,6 +1772,16 @@ "node": ">=6" } }, + "node_modules/fake-indexeddb": { + "version": "6.2.5", + "resolved": "https://registry.npmjs.org/fake-indexeddb/-/fake-indexeddb-6.2.5.tgz", + "integrity": "sha512-CGnyrvbhPlWYMngksqrSSUT1BAVP49dZocrHuK0SvtR0D5TMs5wP0o3j7jexDJW01KSadjBp1M/71o/KR3nD1w==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=18" + } + }, "node_modules/fdir": { "version": "6.5.0", "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.5.0.tgz", @@ -2222,9 +2234,9 @@ "license": "MIT" }, "node_modules/nanoid": { - "version": "3.3.12", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.12.tgz", - "integrity": "sha512-ZB9RH/39qpq5Vu6Y+NmUaFhQR6pp+M2Xt76XBnEwDaGcVAqhlvxrl3B2bKS5D3NH3QR76v3aSrKaF/Kiy7lEtQ==", + "version": "3.3.20", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.20.tgz", + "integrity": "sha512-uKdg2G3GNCKQn9byYOpxbGqrT2fGO5KRt5J/8b3pok8rT6qxGWF6hxMyJiEYtAf+FVyYuD9hRaDqX5uPFYJ4ZQ==", "dev": true, "funding": [ { @@ -2241,11 +2253,14 @@ } }, "node_modules/node-releases": { - "version": "2.0.44", - "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.44.tgz", - "integrity": "sha512-5WUyunoPMsvvEhS8AxHtRzP+oA8UCkJ7YRxatWKjngndhDGLiqEVAQKWjFAiAiuL8zMRGzGSJxFnLetoa43qGQ==", + "version": "2.0.57", + "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.57.tgz", + "integrity": "sha512-kQK9LGGFiHtrWiNhZtA7Qbw17AQz+dmsEKODRIVTXA9+e5MS/2gZEBhYJt13GrAz5/IOZKddH/0Z3TP/Zgo+yw==", "dev": true, - "license": "MIT" + "license": "MIT", + "engines": { + "node": ">=18" + } }, "node_modules/picocolors": { "version": "1.1.1", @@ -2268,9 +2283,9 @@ } }, "node_modules/postcss": { - "version": "8.5.15", - "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.15.tgz", - "integrity": "sha512-FfR8sjd4em2T6fb3I2MwAJU7HWVMr9zba+enmQeeWFfCbm+UOC/0X4DS8XtpUTMwWMGbjKYP7xjfNekzyGmB3A==", + "version": "8.5.29", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.29.tgz", + "integrity": "sha512-49cGhUbXj8Qenv0iTMxA1cFBzxXoctpC9Ujd77t1WcbJIr6nF/eI7g/8MgxrYldFRuAXvja7xQRwavoW7kgrxQ==", "dev": true, "funding": [ { @@ -2288,9 +2303,9 @@ ], "license": "MIT", "dependencies": { - "nanoid": "^3.3.12", + "nanoid": "^3.3.19", "picocolors": "^1.1.1", - "source-map-js": "^1.2.1" + "source-map-js": "^1.2.2" }, "engines": { "node": "^10 || ^12 || >=14" @@ -2443,9 +2458,9 @@ } }, "node_modules/source-map-js": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", - "integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==", + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.2.tgz", + "integrity": "sha512-KGj/8Y43x35aZVDtt+J4mK1hoLGHULMYfSkODJNQjNDC3oW1PqPoxMwo0pLUsWM/UEGzON/NxeHywEfNXNP3Vw==", "dev": true, "license": "BSD-3-Clause", "engines": { @@ -2505,9 +2520,9 @@ } }, "node_modules/update-browserslist-db": { - "version": "1.2.3", - "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.2.3.tgz", - "integrity": "sha512-Js0m9cx+qOgDxo0eMiFGEueWztz+d4+M3rGlmKPT+T4IS/jP4ylw3Nwpu6cpTTP8R1MAC1kF4VbdLt3ARf209w==", + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.3.3.tgz", + "integrity": "sha512-pJ2sYawQS0R/WI928Gj5GlPhTGzbMelq0+4INtSYNDV9ErKJcX6xjGWkoG/VnB3dpUm00zALaqkrUD77pO5TDQ==", "dev": true, "funding": [ { @@ -2545,9 +2560,9 @@ } }, "node_modules/vite": { - "version": "6.4.2", - "resolved": "https://registry.npmjs.org/vite/-/vite-6.4.2.tgz", - "integrity": "sha512-2N/55r4JDJ4gdrCvGgINMy+HH3iRpNIz8K6SFwVsA+JbQScLiC+clmAxBgwiSPgcG9U15QmvqCGWzMbqda5zGQ==", + "version": "6.4.3", + "resolved": "https://registry.npmjs.org/vite/-/vite-6.4.3.tgz", + "integrity": "sha512-NTKlcQjlAK7MlQoyb6LgaqHc8sso/pVyUJYWMws3jg21uTJw/LddqIFPcPqP6PzpgbIcZyKI85sFE4HBrQDA8A==", "dev": true, "license": "MIT", "dependencies": { diff --git a/image-studio/frontend/package.json b/image-studio/frontend/package.json index c6ee02715..7f0952785 100644 --- a/image-studio/frontend/package.json +++ b/image-studio/frontend/package.json @@ -35,6 +35,7 @@ "@types/react": "^18.3.0", "@types/react-dom": "^18.3.0", "@vitejs/plugin-react": "^4.3.0", + "fake-indexeddb": "^6.2.5", "tailwindcss": "^4.0.0", "typescript": "^5.7.0", "vite": "^6.0.0" diff --git a/image-studio/frontend/src/components/canvas/DragExportHandle.tsx b/image-studio/frontend/src/components/canvas/DragExportHandle.tsx index bbea504f6..d80482f4f 100644 --- a/image-studio/frontend/src/components/canvas/DragExportHandle.tsx +++ b/image-studio/frontend/src/components/canvas/DragExportHandle.tsx @@ -2,10 +2,9 @@ import type { HistoryItem } from "../../types/domain"; import { buildHistoryItemDragExport, shouldUseNativeFileDrag, - writeImageFileDragData, - writeInternalHistoryItemDragData, + writeHistoryItemFileDragData, } from "../../lib/dragExport.ts"; -import { BeginNativeFileDrag } from "../../platform/runtime/host"; +import { useNativeFileDrag } from "../common/useNativeFileDrag"; import { usePlatform } from "../../platform/context"; export function DragExportHandle({ @@ -18,6 +17,7 @@ export function DragExportHandle({ sourceURL?: string | null; }) { const { targetPlatform } = usePlatform(); + const nativeDrag = useNativeFileDrag(item.savedPath); const spec = buildHistoryItemDragExport(item, sourceURL); if (!spec) return null; @@ -28,11 +28,14 @@ export function DragExportHandle({ event.stopPropagation()} + onMouseDown={(event) => { + event.stopPropagation(); + nativeDrag.onMouseDown(event); + }} onClick={(event) => { event.preventDefault(); event.stopPropagation(); @@ -41,15 +44,10 @@ export function DragExportHandle({ event.stopPropagation(); if (shouldUseNativeFileDrag(targetPlatform, item.savedPath)) { event.preventDefault(); - console.debug("[drag-export] native-file-drag", item.savedPath); - void BeginNativeFileDrag(item.savedPath).catch((error) => { - console.error("[drag-export] native-file-drag failed", error); - }); return; } event.dataTransfer.effectAllowed = "copy"; - writeInternalHistoryItemDragData(event.dataTransfer, item); - writeImageFileDragData(event.dataTransfer, spec); + writeHistoryItemFileDragData(event.dataTransfer, item, spec); }} > 拖出复制 diff --git a/image-studio/frontend/src/components/common/SavePromptModal.tsx b/image-studio/frontend/src/components/common/SavePromptModal.tsx index 63ac6fde5..1d810813d 100644 --- a/image-studio/frontend/src/components/common/SavePromptModal.tsx +++ b/image-studio/frontend/src/components/common/SavePromptModal.tsx @@ -6,12 +6,12 @@ import { historyPreviewSrc, useBlobURL } from "../../lib/images"; import { buildHistoryItemDragExport, shouldUseNativeFileDrag, - writeImageFileDragData, - writeInternalHistoryItemDragData, + writeHistoryItemFileDragData, } from "../../lib/dragExport.ts"; import { saveHistoryItemAs, saveHistoryItemsToDirectory } from "../../lib/saveResultImage"; import { androidSaveHint, androidTarget } from "../../platform/android/bridge"; -import { BeginNativeFileDrag, getHostCapabilities, ChooseDirectory } from "../../platform/runtime/host"; +import { getHostCapabilities, ChooseDirectory } from "../../platform/runtime/host"; +import { useNativeFileDrag } from "./useNativeFileDrag"; import { usePlatform } from "../../platform/context"; import { Modal } from "./Modal"; @@ -39,11 +39,12 @@ export function SavePromptModal() { [batchItems, selectedIds], ); - if (!request) return null; - const isBatch = request.kind === "batch"; - const singleItem = request.kind === "single" ? request.item : null; - const batchRequest = request.kind === "batch" ? request : null; + const isBatch = request?.kind === "batch"; + const singleItem = request?.kind === "single" ? request.item : null; + const batchRequest = request?.kind === "batch" ? request : null; const singlePreviewURL = useBlobURL(singleItem?.previewBlob ?? singleItem?.imageBlob ?? null, singleItem?.imageB64 ?? null); + const nativeDrag = useNativeFileDrag(singleItem?.savedPath); + if (!request) return null; async function saveSingleAs() { if (saving || !singleItem) return; @@ -98,14 +99,10 @@ export function SavePromptModal() { event.stopPropagation(); if (shouldUseNativeFileDrag(targetPlatform, singleItem.savedPath)) { event.preventDefault(); - void BeginNativeFileDrag(singleItem.savedPath).catch((error) => { - console.error("[drag-export] native-file-drag failed", error); - }); return; } event.dataTransfer.effectAllowed = "copy"; - writeInternalHistoryItemDragData(event.dataTransfer, singleItem); - writeImageFileDragData(event.dataTransfer, dragSpec); + writeHistoryItemFileDragData(event.dataTransfer, singleItem, dragSpec); } return ( @@ -113,7 +110,8 @@ export function SavePromptModal() {
    void>(() => {}); + const pendingRef = useRef(false); + useEffect(() => () => cleanupRef.current(), []); + + const onMouseDown = (event: MouseEvent) => { + if (!native || !savedPath || event.button !== 0) return; + event.preventDefault(); + event.stopPropagation(); + if (pendingRef.current) return; + cleanupRef.current(); + cleanupRef.current = trackNativeFileDragGesture(window, event, () => { + pendingRef.current = true; + void BeginNativeFileDrag(savedPath).catch((error: unknown) => { + useStudioStore.getState().pushToast( + `拖出复制失败:${error instanceof Error ? error.message : String(error)}`, + "error", + 6000, + ); + }).finally(() => { pendingRef.current = false; }); + }); + }; + return { native, onMouseDown }; +} diff --git a/image-studio/frontend/src/components/history/HistoryPromptGroupModal.tsx b/image-studio/frontend/src/components/history/HistoryPromptGroupModal.tsx index 27b193a1f..e47658f72 100644 --- a/image-studio/frontend/src/components/history/HistoryPromptGroupModal.tsx +++ b/image-studio/frontend/src/components/history/HistoryPromptGroupModal.tsx @@ -5,10 +5,9 @@ import { historyPreviewSrc, useBlobURL } from "../../lib/images"; import { buildHistoryItemDragExport, shouldUseNativeFileDrag, - writeImageFileDragData, - writeInternalHistoryItemDragData, + writeHistoryItemFileDragData, } from "../../lib/dragExport.ts"; -import { BeginNativeFileDrag } from "../../platform/runtime/host"; +import { useNativeFileDrag } from "../common/useNativeFileDrag"; import type { HistoryItem } from "../../types/domain"; import { usePlatform } from "../../platform/context"; import { HistoryModeBadge } from "./HistoryModeBadge"; @@ -128,6 +127,7 @@ function HistoryPromptModalThumbnail({ const displayIndex = typeof item.batchIndex === "number" ? item.batchIndex + 1 : index + 1; const dragSpec = buildHistoryItemDragExport(item); const { targetPlatform } = usePlatform(); + const nativeDrag = useNativeFileDrag(item.savedPath); function openMenu(event: React.MouseEvent) { event.preventDefault(); @@ -152,14 +152,10 @@ function HistoryPromptModalThumbnail({ event.stopPropagation(); if (shouldUseNativeFileDrag(targetPlatform, item.savedPath)) { event.preventDefault(); - void BeginNativeFileDrag(item.savedPath).catch((error) => { - console.error("[drag-export] native-file-drag failed", error); - }); return; } event.dataTransfer.effectAllowed = "copy"; - writeInternalHistoryItemDragData(event.dataTransfer, item); - writeImageFileDragData(event.dataTransfer, dragSpec); + writeHistoryItemFileDragData(event.dataTransfer, item, dragSpec); } return ( @@ -167,7 +163,8 @@ function HistoryPromptModalThumbnail({ type="button" className={`history-prompt-modal-thumb ${isCurrent ? "active" : ""} ${isCompare ? "compare" : ""}`} title={item.prompt} - draggable={!!dragSpec} + draggable={!!dragSpec && !nativeDrag.native} + onMouseDown={nativeDrag.onMouseDown} onClick={handleSelect} onDoubleClick={() => void onReuse(item)} onContextMenu={openMenu} diff --git a/image-studio/frontend/src/components/history/HistoryTile.tsx b/image-studio/frontend/src/components/history/HistoryTile.tsx index 03cec8e99..459508016 100644 --- a/image-studio/frontend/src/components/history/HistoryTile.tsx +++ b/image-studio/frontend/src/components/history/HistoryTile.tsx @@ -3,11 +3,10 @@ import type React from "react"; import { buildHistoryItemDragExport, shouldUseNativeFileDrag, - writeImageFileDragData, - writeInternalHistoryItemDragData, + writeHistoryItemFileDragData, } from "../../lib/dragExport.ts"; import { historyPreviewSrc, useBlobURL, useImageLoadState } from "../../lib/images"; -import { BeginNativeFileDrag } from "../../platform/runtime/host"; +import { useNativeFileDrag } from "../common/useNativeFileDrag"; import { usePlatform } from "../../platform/context"; import type { HistoryItem } from "../../types/domain"; import { HistoryMetaBadges } from "./HistoryMetaBadges"; @@ -36,6 +35,7 @@ export function HistoryTile({ variant?: "default" | "phone" | "phoneFeature" | "windowsFeature" | "windowsList"; }) { const { isMac, targetPlatform, usesFluentUI } = usePlatform(); + const nativeDrag = useNativeFileDrag(item.savedPath); const previewURL = useBlobURL(item.previewBlob ?? item.imageBlob ?? null, item.imageB64 ?? null); const imageSrc = historyPreviewSrc(item, previewURL); const imageLoadState = useImageLoadState(imageSrc || null); @@ -68,20 +68,15 @@ export function HistoryTile({ e.stopPropagation(); if (shouldUseNativeFileDrag(targetPlatform, item.savedPath)) { e.preventDefault(); - console.debug("[drag-export] native-file-drag", item.savedPath); - void BeginNativeFileDrag(item.savedPath).catch((error) => { - console.error("[drag-export] native-file-drag failed", error); - }); return; } e.dataTransfer.effectAllowed = "copy"; - writeInternalHistoryItemDragData(e.dataTransfer, item); - writeImageFileDragData(e.dataTransfer, dragSpec); + writeHistoryItemFileDragData(e.dataTransfer, item, dragSpec); } function renderDragImageNode(wrapperClassName?: string, imageClassName?: string) { return ( -
    +
    {imageLoadState === "ready" ? ( s.setField); const pushToast = useStudioStore((s) => s.pushToast); const { usesFluentUI, targetPlatform } = usePlatform(); + const nativeDrag = useNativeFileDrag(item?.savedPath); + const previewURL = useBlobURL(item?.previewBlob ?? item?.imageBlob ?? null, item?.imageB64 ?? null); if (!item) return null; const detail = item; const created = new Date(detail.createdAt).toLocaleString(); - const previewURL = useBlobURL(detail.previewBlob ?? detail.imageBlob ?? null, detail.imageB64 ?? null); const imageSrc = historyPreviewSrc(detail, previewURL); const dragSpec = buildHistoryItemDragExport(detail); @@ -38,14 +39,10 @@ export function ResultDetailDrawer() { event.stopPropagation(); if (shouldUseNativeFileDrag(targetPlatform, detail.savedPath)) { event.preventDefault(); - void BeginNativeFileDrag(detail.savedPath).catch((error) => { - console.error("[drag-export] native-file-drag failed", error); - }); return; } event.dataTransfer.effectAllowed = "copy"; - writeInternalHistoryItemDragData(event.dataTransfer, detail); - writeImageFileDragData(event.dataTransfer, dragSpec); + writeHistoryItemFileDragData(event.dataTransfer, detail, dragSpec); } function copy(text: string, label: string) { @@ -77,7 +74,8 @@ export function ResultDetailDrawer() {
    id="settings-alerts" ref={setDesktopSectionRef("alerts")} title="通知与提示" - description="完成整批生成时的声音和系统级通知统一放在这里。" + description="整批完成或首次生成失败时的声音与系统通知。" > - +
    { setCompletionSoundEnabled(true); @@ -669,7 +669,7 @@ export function SettingsPanel({ open, onClose }: { open: boolean; onClose: () =>

    - 整批生成全部完成后只播放一次。当前 {completionSound.mode === "custom" && completionSound.customName ? `使用 ${completionSound.customName}` : "使用内置默认音"}。 + 整批完成或首次失败时播放；主动取消时不播放。当前 {completionSound.mode === "custom" && completionSound.customName ? `使用 ${completionSound.customName}` : "使用内置默认音"}。

    diff --git a/image-studio/frontend/src/lib/compatState.ts b/image-studio/frontend/src/lib/compatState.ts index 95784af2c..5f42e09b6 100644 --- a/image-studio/frontend/src/lib/compatState.ts +++ b/image-studio/frontend/src/lib/compatState.ts @@ -28,10 +28,11 @@ import { } from "./customAspectRatios.ts"; import { loadTrustedOutputRoots, + loadAllHistory, persistHistoryFullImages, persistHistoryItems, - pruneHistoryStorage, } from "./storage.ts"; +import { mergeHistoryItems } from "./history.ts"; import { normalizeCompletionSoundConfig, persistCompletionSoundConfig, @@ -151,7 +152,10 @@ export function scheduleCompatibilityExport(input: CompatibilityExportInput): vo } export async function exportCompatibilityStateNow(input: CompatibilityExportInput): Promise { - const state = buildCompatibilityState(input); + const state = buildCompatibilityState({ + ...input, + history: mergeHistoryItems([...input.history, ...await loadAllHistory()]), + }); await SaveCompatibilityState(state as unknown as Record); writeLocalMarker(state.updatedAt); } @@ -294,15 +298,16 @@ function applyCompatibilityLocalStorage(state: CompatibilityState): void { async function persistCompatibilityHistory(state: CompatibilityState): Promise { const items = state.history.map(toSerializableHistoryItem).filter((item): item is HistoryItem => item !== null); - await persistHistoryItems(items).catch(() => undefined); + await persistHistoryItems(items); const fullImages = [ ...items .filter((item) => typeof item.imageB64 === "string" && item.imageB64.trim()) .map((item) => ({ id: item.id, imageB64: item.imageB64 as string })), ...(state.historyFull ?? []).filter((item) => item?.id && item.imageB64?.trim()), ]; - await persistHistoryFullImages(fullImages).catch(() => undefined); - await pruneHistoryStorage(items.map((item) => item.id)).catch(() => undefined); + await persistHistoryFullImages(fullImages); + // Older clients may export only their visible page. Merge that snapshot; + // absence from a configuration import is not an explicit history deletion. } function normalizeCompatibilityState(raw: unknown): CompatibilityState | null { diff --git a/image-studio/frontend/src/lib/completionSound.ts b/image-studio/frontend/src/lib/completionSound.ts index a1b9cd849..f0ea708b6 100644 --- a/image-studio/frontend/src/lib/completionSound.ts +++ b/image-studio/frontend/src/lib/completionSound.ts @@ -102,9 +102,13 @@ export async function playCompletionSound( }); const audio = createAudio(src); if (!audio) return false; - audio.currentTime = 0; - await Promise.resolve(audio.play()).catch(() => undefined); - return true; + try { + audio.currentTime = 0; + await audio.play(); + return true; + } catch { + return false; + } } function normalizeAudioMimeType(type: string | null | undefined, name: string | null | undefined): string | null { diff --git a/image-studio/frontend/src/lib/dragExport.ts b/image-studio/frontend/src/lib/dragExport.ts index 0bdd7b9fa..13e6de0b5 100644 --- a/image-studio/frontend/src/lib/dragExport.ts +++ b/image-studio/frontend/src/lib/dragExport.ts @@ -15,8 +15,9 @@ export function shouldUseNativeFileDrag( targetPlatform: string, savedPath?: string | null, ): savedPath is string { + const path = savedPath?.trim() ?? ""; return (targetPlatform === "macos" || targetPlatform === "windows") - && !!savedPath?.trim(); + && (/^[a-zA-Z]:[\\/]/.test(path) || path.startsWith("/") || path.startsWith("\\\\")); } type DragExportHistoryItem = Pick< @@ -73,6 +74,7 @@ function isWindowsDrivePath(path: string): boolean { function fileURLFromPath(path?: string): string { const trimmed = path?.trim() || ""; if (!trimmed) return ""; + if (trimmed.startsWith("memory://")) return ""; if (trimmed.startsWith("file://")) return trimmed; if (/^[a-zA-Z][a-zA-Z\d+\-.]*:/.test(trimmed) && !isWindowsDrivePath(trimmed)) return trimmed; const normalized = trimmed.replace(/\\/g, "/"); @@ -196,6 +198,17 @@ export function writeInternalHistoryItemDragData( } } +export function writeHistoryItemFileDragData( + dataTransfer: Pick | null | undefined, + item: InternalHistoryDragItem, + spec: DragExportSpec, +): void { + // File formats clear the browser's defaults. Add the app payload afterwards + // so dropping the same image inside the studio still retains its metadata. + writeImageFileDragData(dataTransfer, spec); + writeInternalHistoryItemDragData(dataTransfer, item); +} + export function readInternalHistoryItemDragData( dataTransfer: Pick | null | undefined, ): HistoryItem | null { diff --git a/image-studio/frontend/src/lib/failureAlert.ts b/image-studio/frontend/src/lib/failureAlert.ts new file mode 100644 index 000000000..a2816a3cb --- /dev/null +++ b/image-studio/frontend/src/lib/failureAlert.ts @@ -0,0 +1,12 @@ +// Alert on the first failure of a submission, including a loop or folder batch. +// A new submission gets a new gate, so later failures are never permanently muted. +export function createFailureAlertGate(): (message: string) => boolean { + let alerted = false; + return (message) => { + if (alerted || /\b(?:context canceled|cancelled|canceled|AbortError)\b|已取消|用户取消/i.test(message)) { + return false; + } + alerted = true; + return true; + }; +} diff --git a/image-studio/frontend/src/lib/history.ts b/image-studio/frontend/src/lib/history.ts new file mode 100644 index 000000000..cb856b3d1 --- /dev/null +++ b/image-studio/frontend/src/lib/history.ts @@ -0,0 +1,11 @@ +import type { HistoryItem } from "../types/domain"; + +// A loaded page is only a view of the archive. Never use it as a retention list. +export function mergeHistoryItems(items: HistoryItem[]): HistoryItem[] { + const seen = new Set(); + return items.filter((item) => { + if (!item.id || seen.has(item.id)) return false; + seen.add(item.id); + return true; + }).sort((a, b) => b.createdAt - a.createdAt); +} diff --git a/image-studio/frontend/src/lib/storage.ts b/image-studio/frontend/src/lib/storage.ts index 9167c9ce8..420e8e467 100644 --- a/image-studio/frontend/src/lib/storage.ts +++ b/image-studio/frontend/src/lib/storage.ts @@ -15,7 +15,8 @@ const LEGACY_SHARED_API_KEY = "gptcodex.apiKey"; type HistoryRecord = HistoryItem & { searchText: string; searchTokens: string[] }; type FullRecord = { id: string; image: Blob }; export type HistoryPageCursor = { - beforeDayStart: number; + createdAt: number; + id: string; }; export type HistoryPageResult = { items: HistoryItem[]; @@ -202,12 +203,6 @@ function stripHistoryRecord(record: HistoryRecord): HistoryItem { return item; } -function startOfLocalDay(createdAt: number): number { - const d = new Date(createdAt); - d.setHours(0, 0, 0, 0); - return d.getTime(); -} - export async function persistHistoryItems(items: HistoryItem[]): Promise { if (items.length === 0) return; const { store, tx } = await openHistoryTx("readwrite"); @@ -242,24 +237,34 @@ export async function loadHistoryFullImage(id: string): Promise { return rec?.image ? blobToBase64(rec.image) : ""; } -export async function pruneHistoryStorage(keepIDs: string[]): Promise { - const keep = new Set(keepIDs); - const { store: historyStore, tx: historyTx } = await openHistoryTx("readwrite"); - const historyKeys = await reqAsPromise(historyStore.getAllKeys()); - for (const id of historyKeys) { - if (typeof id === "string" && !keep.has(id)) historyStore.delete(id); - } - await txDone(historyTx); - - const { store: fullStore, tx: fullTx } = await openFullTx("readwrite"); - const fullKeys = await reqAsPromise(fullStore.getAllKeys()); - for (const id of fullKeys) { - if (typeof id === "string" && !keep.has(id)) fullStore.delete(id); - } - await txDone(fullTx); +export async function removeHistoryOlderThan(cutoff: number): Promise { + await migrateLegacyHistoryIfNeeded(); + const db = await openDB(); + const tx = db.transaction([HISTORY_STORE, HISTORY_FULL_STORE], "readwrite"); + const donePromise = txDone(tx); + const historyStore = tx.objectStore(HISTORY_STORE); + const fullStore = tx.objectStore(HISTORY_FULL_STORE); + const ids = await new Promise((resolve, reject) => { + const removed: string[] = []; + const req = historyStore.index("createdAt").openCursor(IDBKeyRange.upperBound(cutoff, true)); + req.onsuccess = () => { + const cursor = req.result; + if (!cursor) { resolve(removed); return; } + const id = (cursor.value as HistoryRecord).id; + removed.push(id); + cursor.delete(); + fullStore.delete(id); + cursor.continue(); + }; + req.onerror = () => reject(req.error); + }); + await donePromise; + await removeLegacyHistoryItems(ids); + return ids; } export async function removeHistoryItem(id: string): Promise { + await removeLegacyHistoryItems([id]); const { store: historyStore, tx: historyTx } = await openHistoryTx("readwrite"); historyStore.delete(id); await txDone(historyTx); @@ -269,6 +274,24 @@ export async function removeHistoryItem(id: string): Promise { await txDone(fullTx); } +async function removeLegacyHistoryItems(ids: string[]): Promise { + if (ids.length === 0) return; + const legacy = await openLegacyDB(); + try { + if (!legacy.objectStoreNames.contains(LEGACY_STORE_NAME)) return; + const tx = legacy.transaction(LEGACY_STORE_NAME, "readwrite"); + const donePromise = txDone(tx); + const store = tx.objectStore(LEGACY_STORE_NAME); + for (const id of ids) { + store.delete(`history:${id}`); + store.delete(`history-full:${id}`); + } + await donePromise; + } finally { + legacy.close(); + } +} + async function clearLegacyHistoryStorage(): Promise { const legacy = await openLegacyDB(); if (!legacy.objectStoreNames.contains(LEGACY_STORE_NAME)) return []; @@ -401,13 +424,10 @@ export async function loadHistoryPage(opts?: { const limit = typeof opts?.limit === "number" && Number.isFinite(opts.limit) && opts.limit > 0 ? Math.floor(opts.limit) : 24; - const beforeDayStart = opts?.cursor?.beforeDayStart ?? 0; - const range = beforeDayStart > 0 - ? IDBKeyRange.upperBound(beforeDayStart, true) - : null; + const boundary = opts?.cursor; + const range = boundary ? IDBKeyRange.upperBound(boundary.createdAt) : null; const result = await new Promise<{ records: HistoryRecord[]; nextCursor: HistoryPageCursor | null }>((resolve, reject) => { const out: HistoryRecord[] = []; - let currentDayStart: number | null = null; const req = store.index("createdAt").openCursor(range, "prev"); req.onsuccess = () => { const cursor = req.result; @@ -416,26 +436,19 @@ export async function loadHistoryPage(opts?: { return; } const value = cursor.value as HistoryRecord; - const dayStart = startOfLocalDay(value.createdAt); - if (currentDayStart === null) { - currentDayStart = dayStart; - out.push(value); - cursor.continue(); - return; - } - if (dayStart === currentDayStart) { - out.push(value); + // createdAt is not unique: concurrent results often share a millisecond. + if (boundary && value.createdAt === boundary.createdAt + && indexedDB.cmp(cursor.primaryKey, boundary.id) >= 0) { cursor.continue(); return; } if (out.length >= limit) { resolve({ records: out, - nextCursor: { beforeDayStart: currentDayStart }, + nextCursor: { createdAt: out[out.length - 1].createdAt, id: out[out.length - 1].id }, }); return; } - currentDayStart = dayStart; out.push(value); cursor.continue(); }; diff --git a/image-studio/frontend/src/platform/android/settings/AndroidSettingsPanel.tsx b/image-studio/frontend/src/platform/android/settings/AndroidSettingsPanel.tsx index 85c1594e5..00a120abd 100644 --- a/image-studio/frontend/src/platform/android/settings/AndroidSettingsPanel.tsx +++ b/image-studio/frontend/src/platform/android/settings/AndroidSettingsPanel.tsx @@ -399,16 +399,16 @@ export function AndroidSettingsPanel({
    - 完成提示音 + 任务提示音 {completionSound.enabled ? (completionSound.mode === "custom" && completionSound.customName ? `当前使用 ${completionSound.customName}` : "当前使用内置默认音") - : "生成完成时不播放提示音。"} + : "整批完成或生成失败时不播放提示音。"}
    -
    +
    - {compareB ? ( + {compareItemId ? ( @@ -411,8 +423,8 @@ export function HistoryRail() { {rawPath && } setActivePromptGroup(null)} onSelect={(item) => void selectCurrent(item)} onReuse={reuseAsSource} @@ -584,7 +596,7 @@ export function HistoryRail() { )}
    - {compareB && ( + {compareItemId && (
    - {compareB ? ( + {compareItemId ? ( @@ -224,8 +224,8 @@ export function WindowsHistoryRail({
    setCompareB(next)} onReuse={reuseAsSource} @@ -253,8 +253,8 @@ export function WindowsHistoryRail({ void }) { - const { fullscreen, theme, setTheme, pushToast, workspaces, newWorkspace, openStarPrompt } = useStudioStore(); + const { fullscreen, theme, setTheme, pushToast, newWorkspace, openStarPrompt } = useStudioFields([ + "fullscreen", "theme", "setTheme", "pushToast", + "newWorkspace", "openStarPrompt", + ]); + const workspaceCount = useStudioStore((state) => state.workspaces.length); const { isAndroid, isAndroidPhone, isAndroidPad, isMac, usesFluentUI, usesAndroidUI, usesAppleUI } = usePlatform(); if (fullscreen) return null; @@ -35,12 +40,12 @@ export function AppHeader({ onOpenSettings }: { onOpenSettings: () => void }) {
    {!isAndroid && newWorkspace()} - title={workspaces.length > 1 ? `${workspaces.length} 个标签 · 新建` : "新建标签"} + title={workspaceCount > 1 ? `${workspaceCount} 个标签 · 新建` : "新建标签"} > - {workspaces.length > 1 && ( + {workspaceCount > 1 && ( - {workspaces.length} + {workspaceCount} )} } diff --git a/image-studio/frontend/src/components/layout/FooterBar.tsx b/image-studio/frontend/src/components/layout/FooterBar.tsx index bf0c50acd..b16f3f2ba 100644 --- a/image-studio/frontend/src/components/layout/FooterBar.tsx +++ b/image-studio/frontend/src/components/layout/FooterBar.tsx @@ -1,5 +1,6 @@ import { Folder, Github, MessageSquare } from "lucide-react"; import { useStudioStore } from "../../state/studioStore"; +import { useStudioFields } from "../../state/useStudioFields"; import { OpenExternalURL, OpenOutputDir } from "../../platform/runtime/host"; import { androidTarget, openExternalURLForPlatform, openOutputLocationForPlatform } from "../../platform/android/bridge"; import { appVersion } from "../../lib/version"; @@ -9,12 +10,15 @@ const REPO_URL = "https://github.com/RoseKhlifa/Image-Studio"; const ISSUES_URL = "https://github.com/RoseKhlifa/Image-Studio/issues"; export function FooterBar() { - const { fullscreen, history, runningJobs, isRunning, workspaces, pushToast } = useStudioStore(); + const { fullscreen, history, runningJobs, isRunning, pushToast } = useStudioFields([ + "fullscreen", "history", "runningJobs", "isRunning", + "pushToast", + ]); + const totalRunning = useStudioStore((state) => state.workspaces.reduce((sum, w) => sum + (w.runningJobIds?.length ?? 0), 0)); const { isAndroid, isMac, isWindows, usesFluentUI, usesAppleUI } = usePlatform(); if (fullscreen) return null; if (isAndroid) return null; if (isMac) return null; - const totalRunning = workspaces.reduce((sum, w) => sum + (w.runningJobIds?.length ?? 0), 0); const activeRunning = isRunning; const anyRunning = activeRunning || totalRunning > 0; diff --git a/image-studio/frontend/src/components/panel/ControlPanel.tsx b/image-studio/frontend/src/components/panel/ControlPanel.tsx index a1a017700..19b71abe1 100644 --- a/image-studio/frontend/src/components/panel/ControlPanel.tsx +++ b/image-studio/frontend/src/components/panel/ControlPanel.tsx @@ -1,5 +1,6 @@ import { useState } from "react"; import { useStudioStore } from "../../state/studioStore"; +import { useStudioFields } from "../../state/useStudioFields"; import { SizeValue, QualityValue, Mode } from "../../types/domain"; import { usePlatform } from "../../platform/context"; import { ChooseDirectory } from "../../platform/runtime/host"; @@ -55,7 +56,23 @@ export function ControlPanel({ openCustomSizeModal, openUpstreamConfig, submit, cancel, retryLast, optimizePrompt, inferPromptFromCanvas, - } = useStudioStore(); + } = useStudioFields([ + "apiKey", "mode", "prompt", "background", + "imageStyle", "inputFidelity", "moderation", "negativePrompt", + "outputCompression", "size", "quality", "seed", + "styleTag", "userIdentifier", "partialImages", "outputFormat", + "batchCount", "editSourceMode", "editAutoAspectResolution", "batchProcess", + "loopGeneration", "sources", "currentImage", "errorMessage", + "errorCanRetry", "errorRawPath", "isRunning", "lastPayload", + "isTestingKey", "isOptimizingPrompt", "isInferringPrompt", "apiMode", + "requestPolicy", "baseURL", "profiles", "aiProfileId", + "imageModelID", "customAspectRatios", "setField", "clearError", + "pushToast", "selectSourceImage", "chooseBatchInputDir", "chooseBatchInputFiles", + "refreshBatchInputDir", "removeSource", "clearSources", "viewSourceOnCanvas", + "compareSourceOnCanvas", "openCustomAspectRatioModal", "openCustomSizeModal", "openUpstreamConfig", + "submit", "cancel", "retryLast", "optimizePrompt", + "inferPromptFromCanvas", + ]); const [advancedOpen, setAdvancedOpen] = useState(false); const [promptPopover, setPromptPopover] = useState(false); const [macComposeOpen, setMacComposeOpen] = useState(false); diff --git a/image-studio/frontend/src/lib/compatState.ts b/image-studio/frontend/src/lib/compatState.ts index 5f42e09b6..2a9e2500d 100644 --- a/image-studio/frontend/src/lib/compatState.ts +++ b/image-studio/frontend/src/lib/compatState.ts @@ -128,6 +128,44 @@ export type CompatibilityExportInput = { }; let exportTimer: ReturnType | null = null; +let pendingExportInput: CompatibilityExportInput | null = null; +let exportInFlight = false; + +const exportFields: Record = { + history: true, profiles: true, activeProfileId: true, aiProfileId: true, + proxyMode: true, proxyURL: true, theme: true, fontScale: true, outputFormat: true, + background: true, outputCompression: true, inputFidelity: true, imageStyle: true, + moderation: true, userIdentifier: true, partialImages: true, protectStreamPreview: true, + autoRetryEnabled: true, autoRetryCount: true, promptTemplates: true, promptHistory: true, + presets: true, customAspectRatios: true, kernelRuntimeMode: true, keepLogs: true, + cleanupPreviewCacheOnExit: true, ignoredReleaseTag: true, completionSound: true, + completionNotification: true, +}; +const exportKeys = Object.keys(exportFields) as Array; + +export function createCompatibilityExportChangeDetector(): (input: CompatibilityExportInput) => boolean { + let previousValues: unknown[] | null = null; + let previousPreferences = ""; + let previousFingerprint = ""; + return (input) => { + const values = exportKeys.map((key) => input[key]); + const preferences = JSON.stringify([ + readLocalStorageString(advancedFloatingPanelStorageKey()), + readLocalStorageString("gptcodex.outputDir"), + readLocalStorageString("gptcodex.trustedOutputRoots"), + readLocalStorageString("gptcodex.savePromptSuppressed"), + ]); + if (previousValues && preferences === previousPreferences + && values.every((value, index) => Object.is(value, previousValues![index]))) return false; + + const fingerprint = compatibilityExportFingerprint(input); + previousValues = values; + previousPreferences = preferences; + if (fingerprint === previousFingerprint) return false; + previousFingerprint = fingerprint; + return true; + }; +} export async function importCompatibilityStateIfNewer(): Promise { const state = normalizeCompatibilityState(await LoadCompatibilityState()); @@ -142,15 +180,30 @@ export async function importCompatibilityStateIfNewer(): Promise { export function scheduleCompatibilityExport(input: CompatibilityExportInput): void { if (exportTimer) clearTimeout(exportTimer); - const snapshot = cloneExportInput(input); + // Store actions replace arrays/objects. Keep only their immutable references + // until the debounce fires; superseded events must not clone the full archive. + pendingExportInput = Object.fromEntries(exportKeys.map((key) => [key, input[key]])) as CompatibilityExportInput; exportTimer = setTimeout(() => { exportTimer = null; - void exportCompatibilityStateNow(snapshot).catch((error) => { - if (typeof console !== "undefined") console.warn("compat export failed", error); - }); + void flushCompatibilityExport(); }, 250); } +async function flushCompatibilityExport(): Promise { + if (exportInFlight || !pendingExportInput) return; + const snapshot = pendingExportInput; + pendingExportInput = null; + exportInFlight = true; + try { + await exportCompatibilityStateNow(cloneExportInput(snapshot)); + } catch (error) { + if (typeof console !== "undefined") console.warn("compat export failed", error); + } finally { + exportInFlight = false; + if (pendingExportInput && !exportTimer) void flushCompatibilityExport(); + } +} + export async function exportCompatibilityStateNow(input: CompatibilityExportInput): Promise { const state = buildCompatibilityState({ ...input, diff --git a/image-studio/frontend/src/lib/storage.ts b/image-studio/frontend/src/lib/storage.ts index 420e8e467..8725ef040 100644 --- a/image-studio/frontend/src/lib/storage.ts +++ b/image-studio/frontend/src/lib/storage.ts @@ -24,6 +24,7 @@ export type HistoryPageResult = { }; let dbPromise: Promise | null = null; +let legacyMigrationPromise: Promise | null = null; function openDB(): Promise { if (!dbPromise) { @@ -43,8 +44,16 @@ function openDB(): Promise { db.createObjectStore(HISTORY_FULL_STORE, { keyPath: "id" }); } }; - req.onsuccess = () => resolve(req.result); - req.onerror = () => reject(req.error); + req.onsuccess = () => { + const db = req.result; + db.onversionchange = () => { + db.close(); + dbPromise = null; + legacyMigrationPromise = null; + }; + resolve(db); + }; + req.onerror = () => { dbPromise = null; reject(req.error); }; }); } return dbPromise; @@ -344,17 +353,31 @@ export async function clearHistoryStorage(): Promise { } async function migrateLegacyHistoryIfNeeded(): Promise { + if (!legacyMigrationPromise) { + legacyMigrationPromise = migrateLegacyHistory().catch(() => { + // Keep startup usable after a transient failure, but retry on the next + // read instead of marking a failed migration permanently complete. + legacyMigrationPromise = null; + }); + } + await legacyMigrationPromise; +} + +async function migrateLegacyHistory(): Promise { const [historyCount, fullCount] = await Promise.all([withHistoryCount(), withFullCount()]); if (historyCount > 0 && fullCount > 0) return; + const legacy = await openLegacyDB(); try { - const legacy = await openLegacyDB(); + if (!legacy.objectStoreNames.contains(LEGACY_STORE_NAME)) return; const tx = legacy.transaction(LEGACY_STORE_NAME, "readonly"); + const done = txDone(tx); const store = tx.objectStore(LEGACY_STORE_NAME); if (!store.getAll || !store.getAllKeys) return; const [keys, values] = await Promise.all([ reqAsPromise(store.getAllKeys()), reqAsPromise(store.getAll()), + done, ]); const records = keys.map((k, i) => ({ key: k, value: values[i] })); const historyItems = records.filter(({ key }) => typeof key === "string" && key.startsWith("history:")); @@ -379,8 +402,8 @@ async function migrateLegacyHistoryIfNeeded(): Promise { } await txDone(fullTx); } - } catch { - // ignore migration failures; app can still run with empty new db + } finally { + legacy.close(); } } @@ -436,11 +459,18 @@ export async function loadHistoryPage(opts?: { return; } const value = cursor.value as HistoryRecord; - // createdAt is not unique: concurrent results often share a millisecond. - if (boundary && value.createdAt === boundary.createdAt - && indexedDB.cmp(cursor.primaryKey, boundary.id) >= 0) { - cursor.continue(); - return; + if (boundary && value.createdAt === boundary.createdAt) { + const order = indexedDB.cmp(cursor.primaryKey, boundary.id); + if (order >= 0) { + // Seek over timestamp ties instead of materializing each earlier + // result. Keep a correct fallback for older embedded webviews. + if (order > 0 && typeof cursor.continuePrimaryKey === "function") { + cursor.continuePrimaryKey(boundary.createdAt, boundary.id); + } else { + cursor.continue(); + } + return; + } } if (out.length >= limit) { resolve({ diff --git a/image-studio/frontend/src/platform/android/AndroidPadComposePanel.tsx b/image-studio/frontend/src/platform/android/AndroidPadComposePanel.tsx index 8f82cda03..adf31b372 100644 --- a/image-studio/frontend/src/platform/android/AndroidPadComposePanel.tsx +++ b/image-studio/frontend/src/platform/android/AndroidPadComposePanel.tsx @@ -3,6 +3,7 @@ import { CheckCircle2, ImagePlus, ListPlus, ScanSearch, Sparkles, } from "lucide-react"; import { useStudioStore } from "../../state/studioStore"; +import { useStudioFields } from "../../state/useStudioFields"; import { availableQualityOptions, normalizeQualitySelection, STYLE_CHIPS } from "../../components/panel/panelOptions"; import type { Mode, SizeValue } from "../../types/domain"; import { AndroidModeSwitch } from "./AndroidModeSwitch"; @@ -49,7 +50,19 @@ export function AndroidPadComposePanel({ compareSourceOnCanvas, openCustomAspectRatioModal, openCustomSizeModal, openUpstreamConfig, submit, cancel, optimizePrompt, inferPromptFromCanvas, - } = useStudioStore(); + } = useStudioFields([ + "apiKey", "mode", "prompt", "background", + "imageStyle", "inputFidelity", "moderation", "negativePrompt", + "outputCompression", "size", "quality", "seed", + "styleTag", "outputFormat", "userIdentifier", "partialImages", + "batchCount", "editAutoAspectResolution", "loopGeneration", "sources", + "currentImage", "isRunning", "isOptimizingPrompt", "isInferringPrompt", + "apiMode", "requestPolicy", "baseURL", "imageModelID", + "profiles", "aiProfileId", "customAspectRatios", "setField", + "selectSourceImage", "removeSource", "clearSources", "viewSourceOnCanvas", + "compareSourceOnCanvas", "openCustomAspectRatioModal", "openCustomSizeModal", "openUpstreamConfig", + "submit", "cancel", "optimizePrompt", "inferPromptFromCanvas", + ]); const [templateOpen, setTemplateOpen] = useState(false); const [templateManagerOpen, setTemplateManagerOpen] = useState(false); const [advancedOpen, setAdvancedOpen] = useState(false); diff --git a/image-studio/frontend/src/platform/android/AndroidPhoneComposePanel.tsx b/image-studio/frontend/src/platform/android/AndroidPhoneComposePanel.tsx index 2ca518253..0b0258b4b 100644 --- a/image-studio/frontend/src/platform/android/AndroidPhoneComposePanel.tsx +++ b/image-studio/frontend/src/platform/android/AndroidPhoneComposePanel.tsx @@ -3,6 +3,7 @@ import { FileText, ListPlus, RotateCw, ScanSearch, Settings, Sparkles, X, } from "lucide-react"; import { useStudioStore } from "../../state/studioStore"; +import { useStudioFields } from "../../state/useStudioFields"; import { OpenFile } from "../runtime/host"; import { Mode, SizeValue } from "../../types/domain"; import { availableQualityOptions, normalizeQualitySelection, STYLE_CHIPS } from "../../components/panel/panelOptions"; @@ -50,7 +51,21 @@ export function AndroidPhoneComposePanel({ setField, clearError, pushToast, selectSourceImage, removeSource, clearSources, viewSourceOnCanvas, openCustomAspectRatioModal, openCustomSizeModal, openUpstreamConfig, submit, cancel, retryLast, optimizePrompt, inferPromptFromCanvas, compareSourceOnCanvas, - } = useStudioStore(); + } = useStudioFields([ + "apiKey", "mode", "prompt", "background", + "imageStyle", "inputFidelity", "moderation", "negativePrompt", + "outputCompression", "size", "quality", "seed", + "styleTag", "userIdentifier", "partialImages", "outputFormat", + "batchCount", "editAutoAspectResolution", "loopGeneration", "sources", + "currentImage", "errorMessage", "errorCanRetry", "errorRawPath", + "isRunning", "lastPayload", "isOptimizingPrompt", "isInferringPrompt", + "apiMode", "requestPolicy", "baseURL", "profiles", + "aiProfileId", "imageModelID", "customAspectRatios", "setField", + "clearError", "pushToast", "selectSourceImage", "removeSource", + "clearSources", "viewSourceOnCanvas", "openCustomAspectRatioModal", "openCustomSizeModal", + "openUpstreamConfig", "submit", "cancel", "retryLast", + "optimizePrompt", "inferPromptFromCanvas", "compareSourceOnCanvas", + ]); const [templateOpen, setTemplateOpen] = useState(false); const [templateManagerOpen, setTemplateManagerOpen] = useState(false); const [parametersOpen, setParametersOpen] = useState(false); diff --git a/image-studio/frontend/src/platform/android/canvas/AndroidCanvasStage.tsx b/image-studio/frontend/src/platform/android/canvas/AndroidCanvasStage.tsx index 73d22fe76..76c2fab44 100644 --- a/image-studio/frontend/src/platform/android/canvas/AndroidCanvasStage.tsx +++ b/image-studio/frontend/src/platform/android/canvas/AndroidCanvasStage.tsx @@ -1,7 +1,8 @@ -import { useCallback, useEffect, useLayoutEffect, useRef, useState } from "react"; +import { memo, useCallback, useEffect, useLayoutEffect, useMemo, useRef, useState } from "react"; import { Arrow, Image as KonvaImage, Layer, Line, Rect, Stage } from "react-konva"; import Konva from "konva"; import { useStudioStore } from "../../../state/studioStore"; +import { useStudioFields } from "../../../state/useStudioFields"; import type { HistoryItem } from "../../../types/domain"; import type { Stroke } from "../../../state/studioStore.types"; import { AnnotationShape } from "../../../components/canvas/AnnotationShape"; @@ -26,7 +27,7 @@ const MIN_VIEW_SCALE = 0.05; const MAX_VIEW_SCALE = 8; const ZOOM_STEP = 1.2; -export function AndroidCanvasStage() { +function AndroidCanvasStageContent() { const { currentImage, tool, brushSize, brushMode, annotationKind, annotationColor, @@ -50,18 +51,31 @@ export function AndroidCanvasStage() { batchResults, resultGridOpen, selectBatchResult, closeResultGrid, canvasViewResetTick, stepBatchResult, - } = useStudioStore(); + mode, prompt, size, quality, outputFormat, + } = useStudioFields([ + "currentImage", "tool", "brushSize", "brushMode", + "annotationKind", "annotationColor", "selectedAnnotationId", "annotations", + "addAnnotation", "removeAnnotation", "maskDataURL", "maskVisible", + "maskOpacity", "activateMaskTool", "strokes", "pushStroke", + "undo", "redo", "compareB", "compareSplit", + "setCompareSplit", "setCompareB", "isRunning", "cancel", + "errorMessage", "setField", "streamPreview", "streamPreviews", + "runningJobs", "jobsTotal", "jobsCompleted", "activeWorkspaceId", + "fullscreen", "toggleFullscreen", "history", "batchResults", + "resultGridOpen", "selectBatchResult", "closeResultGrid", "canvasViewResetTick", + "stepBatchResult", + "mode", "prompt", "size", "quality", "outputFormat", + ]); const streamPreviewItems = streamPreviewItemsFromPreviews(streamPreviews, { workspaceId: activeWorkspaceId, - mode: useStudioStore.getState().mode, - prompt: useStudioStore.getState().prompt, - size: useStudioStore.getState().size, - quality: useStudioStore.getState().quality, - outputFormat: useStudioStore.getState().outputFormat, + mode, prompt, size, quality, outputFormat, currentImage, }); - const orderedBatchResults = sortHistoryItemsByCreatedAtAsc(batchResults); - const navigationItems = orderedNavigationItemsForCurrent(currentImage?.id, history, batchResults); + const orderedBatchResults = useMemo(() => sortHistoryItemsByCreatedAtAsc(batchResults), [batchResults]); + const navigationItems = useMemo( + () => orderedNavigationItemsForCurrent(currentImage?.id, history, batchResults), + [currentImage?.id, history, batchResults], + ); const liveBatchSlotCount = runningJobs.length; const liveBatchSlots: BatchGridSlot[] = Array.from({ length: liveBatchSlotCount }, (_, index) => ({ type: "pending", id: `pending-${index}` })); for (const item of [...orderedBatchResults].reverse()) { @@ -676,6 +690,8 @@ export function AndroidCanvasStage() { ); } +export const AndroidCanvasStage = memo(AndroidCanvasStageContent); + export function androidCanvasModeLabel(item: HistoryItem | null) { if (!item) return "空画布"; return item.mode === "edit" ? "编辑结果" : "生成结果"; diff --git a/image-studio/frontend/src/platform/android/canvas/AndroidCanvasWorkspace.tsx b/image-studio/frontend/src/platform/android/canvas/AndroidCanvasWorkspace.tsx index bd3bc4709..b6ac4f201 100644 --- a/image-studio/frontend/src/platform/android/canvas/AndroidCanvasWorkspace.tsx +++ b/image-studio/frontend/src/platform/android/canvas/AndroidCanvasWorkspace.tsx @@ -36,6 +36,7 @@ import type { ReactNode } from "react"; import { useEffect, useState } from "react"; import { ANNOTATION_COLORS, type AnnotationKind, type HistoryItem, type SourceImage } from "../../../types/domain"; import { useStudioStore } from "../../../state/studioStore"; +import { useStudioFields } from "../../../state/useStudioFields"; import { base64ToBlob, useBlobURL } from "../../../lib/images"; import { qualityLabel, sizeLabel } from "../../../components/history/historyLabels"; import { StreamPreviewBadge } from "../../../components/canvas/StreamPreviewBadge"; @@ -104,7 +105,22 @@ export function AndroidCanvasWorkspace() { reuseAsSource, pushToast, toggleFullscreen, - } = useStudioStore(); + } = useStudioFields([ + "currentImage", "mode", "sources", "provider", + "batchProcess", "isRunning", "progress", "streamPreview", + "runningJobs", "jobsCompleted", "jobsTotal", "tool", + "brushMode", "brushSize", "maskDataURL", "maskVisible", + "maskOpacity", "strokes", "annotationKind", "annotationColor", + "annotations", "selectedAnnotationId", "fullscreen", "viewZoom", + "batchResults", "resultGridOpen", "undoStack", "redoStack", + "setField", "undo", "redo", "activateMaskTool", + "importMaskImage", "fillMask", "invertMask", "resetMask", + "clearAnnotations", "saveCurrentImageAs", "rotateCurrent", "flipCurrent", + "cropToRect", "openResultGrid", "closeResultGrid", "openResultDetail", + "selectSourceImage", "viewSourceOnCanvas", "compareSourceOnCanvas", "removeSource", + "reorderSources", "clearSources", "reuseAsSource", "pushToast", + "toggleFullscreen", + ]); const { isAndroidPad, androidOrientation } = usePlatform(); const [sourceOpen, setSourceOpen] = useState(true); const hasImage = !!currentImage; diff --git a/image-studio/frontend/src/state/README.md b/image-studio/frontend/src/state/README.md index 06ed910b8..8022f2d1e 100644 --- a/image-studio/frontend/src/state/README.md +++ b/image-studio/frontend/src/state/README.md @@ -16,6 +16,8 @@ - store 可以调用 `platform/runtime/*` 暴露的统一接口。 - store 不应重新实现宿主桥接。 - `workspaceRuntime.ts` 优先保持纯函数化，方便测试与复用。 +- 常驻界面用 `useStudioFields` 或单字段 selector 订阅实际读取的字段，避免进度/日志刷新导致整页重绘。渲染期间读取的派生值也必须有对应订阅；事件处理器里的 `getState()` 用于获取最新值。 +- store 中的数组和对象必须以新引用更新。兼容配置变化检测及画布的缓存计算依赖这一约束。 ## 改动建议 diff --git a/image-studio/frontend/src/state/studioStore.ts b/image-studio/frontend/src/state/studioStore.ts index c7a1d0b75..2d1e900ba 100644 --- a/image-studio/frontend/src/state/studioStore.ts +++ b/image-studio/frontend/src/state/studioStore.ts @@ -69,7 +69,7 @@ import { loadHistoryPage, } from "../lib/storage"; import { - compatibilityExportFingerprint, + createCompatibilityExportChangeDetector, importCompatibilityStateIfNewer, readIgnoredReleaseTag, scheduleCompatibilityExport, @@ -3065,20 +3065,18 @@ async function launchOneJob( export { tempDataURLFromB64, writeBase64ToTempFile }; let compatibilityExportEnabled = false; -let compatibilityFingerprint = ""; +const compatibilityExportChanged = createCompatibilityExportChangeDetector(); function enableCompatibilityExport() { const state = useStudioStore.getState(); compatibilityExportEnabled = true; - compatibilityFingerprint = compatibilityExportFingerprint(state); + compatibilityExportChanged(state); scheduleCompatibilityExport(state); } useStudioStore.subscribe((state) => { if (!compatibilityExportEnabled) return; - const next = compatibilityExportFingerprint(state); - if (next === compatibilityFingerprint) return; - compatibilityFingerprint = next; + if (!compatibilityExportChanged(state)) return; scheduleCompatibilityExport(state); }); diff --git a/image-studio/frontend/src/state/useStudioFields.ts b/image-studio/frontend/src/state/useStudioFields.ts new file mode 100644 index 000000000..a1a754516 --- /dev/null +++ b/image-studio/frontend/src/state/useStudioFields.ts @@ -0,0 +1,13 @@ +import { useShallow } from "zustand/react/shallow"; +import { useStudioStore } from "./studioStore"; +import type { StudioState } from "./studioStore.types"; + +// Component subscriptions must include every field read during rendering. +// Actions and getState() calls made inside event handlers can stay stable. +export function useStudioFields(keys: readonly Key[]): Pick { + return useStudioStore(useShallow((state) => { + const selected = {} as Pick; + for (const key of keys) selected[key] = state[key]; + return selected; + })); +} diff --git a/image-studio/frontend/test/compatState.test.mjs b/image-studio/frontend/test/compatState.test.mjs index 6342ea995..d755b2991 100644 --- a/image-studio/frontend/test/compatState.test.mjs +++ b/image-studio/frontend/test/compatState.test.mjs @@ -1,5 +1,6 @@ import assert from "node:assert/strict"; import test from "node:test"; +import { setTimeout as delay } from "node:timers/promises"; import "fake-indexeddb/auto"; import { clearHistoryStorage, loadAllHistory, persistHistoryItems } from "../src/lib/storage.ts"; @@ -182,3 +183,76 @@ test("compat fingerprint changes when previewPath or sourcePaths change", async assert.notEqual(fingerprintA, fingerprintB); assert.notEqual(fingerprintA, fingerprintC); }); + +function exportInput(overrides = {}) { + return { + history: [], profiles: [], activeProfileId: "", aiProfileId: "", proxyMode: "system", proxyURL: "", theme: "system", fontScale: 1, + outputFormat: "png", background: "auto", outputCompression: 100, inputFidelity: "auto", imageStyle: "default", moderation: "low", + userIdentifier: "", partialImages: 1, protectStreamPreview: true, autoRetryEnabled: true, autoRetryCount: 5, + promptTemplates: [], promptHistory: [], presets: [], customAspectRatios: [], kernelRuntimeMode: "auto", keepLogs: false, + cleanupPreviewCacheOnExit: false, ignoredReleaseTag: "", completionSound: { enabled: true, mode: "default", customName: "", customDataURL: "" }, + completionNotification: { enabled: true }, ...overrides, + }; +} + +test("progress and preview updates do not serialize history; persisted preferences still trigger export", async () => { + const preferences = installStorage(); + installService(); + const compat = await import("../src/lib/compatState.ts"); + let imageReads = 0; + const image = { id: "image", prompt: "fixture", createdAt: 1000, get imageB64() { imageReads += 1; return "AAAA"; } }; + const input = exportInput({ history: [image] }); + const changed = compat.createCompatibilityExportChangeDetector(); + assert.equal(changed(input), true); + assert.equal(imageReads, 1); + for (let i = 0; i < 200; i++) { + assert.equal(changed({ ...input, progress: { bytes: i }, viewZoom: i, currentImage: { imageB64: `preview-${i}` } }), false); + } + assert.equal(imageReads, 1, "unrelated events must not inspect full images"); + preferences.set("gptcodex.outputDir", "/tmp/new-output"); + assert.equal(changed(input), true); + assert.equal(imageReads, 2); + assert.equal(changed(input), false); + assert.equal(changed({ ...input, history: [{ ...image, prompt: "changed prompt" }] }), true); +}); + +test("scheduled exports clone only the latest snapshot and serialize writes without losing pending state", async () => { + installStorage(); + let releaseFirst; + let firstStarted; + const started = new Promise((resolve) => { firstStarted = resolve; }); + const held = new Promise((resolve) => { releaseFirst = resolve; }); + const saved = []; + let active = 0; + let peak = 0; + installService({ async SaveCompatibilityState(state) { + active += 1; + peak = Math.max(peak, active); + saved.push(state); + if (saved.length === 1) { firstStarted(); await held; } + active -= 1; + } }); + const compat = await import(`../src/lib/compatState.ts?scheduled=${Date.now()}`); + let reads = 0; + const image = { + id: "scheduled", prompt: "fixture", mode: "generate", size: "1024x1024", quality: "high", createdAt: 1000, + get imageB64() { reads += 1; return "AAAA"; }, + }; + const input = exportInput({ history: [image] }); + for (let i = 0; i < 50; i++) compat.scheduleCompatibilityExport({ ...input, theme: i === 49 ? "light" : "system" }); + assert.equal(reads, 0, "superseded schedules must not copy the archive"); + await Promise.race([started, delay(2000).then(() => { throw new Error("scheduled export did not start"); })]); + compat.scheduleCompatibilityExport({ ...input, theme: "system" }); + compat.scheduleCompatibilityExport({ ...input, theme: "dark" }); + await delay(300); + assert.equal(saved.length, 1, "a pending write must wait for the first one"); + releaseFirst(); + const deadline = Date.now() + 2000; + while (saved.length < 2 || active > 0) { + if (Date.now() > deadline) throw new Error("latest snapshot was not persisted"); + await delay(5); + } + assert.equal(peak, 1); + assert.deepEqual(saved.map((state) => state.settings.theme), ["light", "dark"]); + assert.equal(saved[1].history[0].id, "scheduled"); +}); diff --git a/image-studio/frontend/test/historyStorage.test.mjs b/image-studio/frontend/test/historyStorage.test.mjs index 25e588fc5..305f1a0f1 100644 --- a/image-studio/frontend/test/historyStorage.test.mjs +++ b/image-studio/frontend/test/historyStorage.test.mjs @@ -1,6 +1,7 @@ import assert from "node:assert/strict"; import test from "node:test"; import "fake-indexeddb/auto"; +import { IDBCursor, IDBFactory, IDBIndex, IDBObjectStore } from "fake-indexeddb"; import * as storage from "../src/lib/storage.ts"; import { mergeHistoryItems } from "../src/lib/history.ts"; @@ -62,6 +63,8 @@ test("date cleanup deletes unloaded old records and their full images while reta }); test("single history deletion also removes legacy copies so migration cannot restore deleted records", async () => { + // A fresh module models restarting after an older client wrote its archive. + const storage = await import(`../src/lib/storage.ts?legacy-restart=${Date.now()}`); const old = item("legacy", 1000); await new Promise((resolve, reject) => { const tx = legacy.transaction("keyval", "readwrite"); @@ -84,3 +87,116 @@ test("history merge keeps every batch result beyond the former 120-item limit", assert.equal(merged.at(-1).id, "image-0"); assert.equal(merged.find((i) => i.id === "image-1").prompt, "prompt image-1"); }); + +test("paging never rereads earlier timestamp ties and migrates the legacy database once", async () => { + const current = await import(`../src/lib/storage.ts?bounded-reads=${Date.now()}`); + const items = Array.from({ length: 300 }, (_, i) => item(`tie-${String(i).padStart(4, "0")}`, 1000)); + await current.persistHistoryItems(items); + const open = indexedDB.open; + const openCursor = IDBIndex.prototype.openCursor; + let legacyOpens = 0; + let rows = 0; + indexedDB.open = function (name, ...args) { + if (name === "keyval-store") legacyOpens += 1; + return open.call(this, name, ...args); + }; + IDBIndex.prototype.openCursor = function (...args) { + const request = openCursor.apply(this, args); + request.addEventListener("success", () => { if (request.result) rows += 1; }); + return request; + }; + try { + let cursor = null; + let loaded = 0; + do { + rows = 0; + const page = await current.loadHistoryPage({ cursor, limit: 18 }); + assert.ok(rows <= 21, `a page inspected ${rows} records`); + loaded += page.items.length; + cursor = page.nextCursor; + } while (cursor); + assert.equal(loaded, 300); + assert.equal(legacyOpens, 1); + } finally { + indexedDB.open = open; + IDBIndex.prototype.openCursor = openCursor; + } +}); + +test("a version-2 archive stays readable with unchanged metadata and full images", async () => { + const original = globalThis.indexedDB; + const factory = new IDBFactory(); + globalThis.indexedDB = factory; + try { + const old = await new Promise((resolve, reject) => { + const request = factory.open("image-studio", 2); + request.onupgradeneeded = () => { + request.result.createObjectStore("history", { keyPath: "id" }).createIndex("createdAt", "createdAt"); + request.result.createObjectStore("historyFull", { keyPath: "id" }); + }; + request.onsuccess = () => resolve(request.result); + request.onerror = () => reject(request.error); + }); + await new Promise((resolve, reject) => { + const tx = old.transaction(["history", "historyFull"], "readwrite"); + tx.objectStore("history").put(item("before-upgrade", 1000)); + tx.objectStore("historyFull").put({ id: "before-upgrade", image: new Blob([new Uint8Array([1, 2, 3])]) }); + tx.oncomplete = resolve; + tx.onerror = () => reject(tx.error); + }); + old.close(); + const upgraded = await import(`../src/lib/storage.ts?upgrade-v2=${Date.now()}`); + const page = await upgraded.loadHistoryPage({ limit: 18 }); + assert.equal(page.items[0].id, "before-upgrade"); + assert.equal(page.items[0].prompt, "prompt before-upgrade"); + assert.equal(await upgraded.loadHistoryFullImage("before-upgrade"), "AQID"); + await new Promise((resolve, reject) => { + const request = factory.deleteDatabase("image-studio"); + request.onsuccess = resolve; + request.onerror = () => reject(request.error); + }); + } finally { + globalThis.indexedDB = original; + } +}); + +test("older webviews without primary-key seeking still return complete timestamp-tied pages", async () => { + await storage.persistHistoryItems(Array.from({ length: 60 }, (_, i) => item(`fallback-${i}`, 1000))); + const original = IDBCursor.prototype.continuePrimaryKey; + IDBCursor.prototype.continuePrimaryKey = undefined; + try { + const ids = new Set(); + let cursor = null; + do { + const page = await storage.loadHistoryPage({ cursor, limit: 18 }); + for (const item of page.items) { assert.equal(ids.has(item.id), false); ids.add(item.id); } + cursor = page.nextCursor; + } while (cursor); + assert.equal(ids.size, 60); + } finally { + IDBCursor.prototype.continuePrimaryKey = original; + } +}); + +test("a transient legacy read failure is retried instead of caching an empty archive", async () => { + await new Promise((resolve, reject) => { + const tx = legacy.transaction("keyval", "readwrite"); + tx.objectStore("keyval").put(item("retry-legacy", 1000), "history:retry-legacy"); + tx.oncomplete = resolve; + tx.onerror = () => reject(tx.error); + }); + const fresh = await import(`../src/lib/storage.ts?migration-retry=${Date.now()}`); + const original = IDBObjectStore.prototype.getAll; + let failed = false; + IDBObjectStore.prototype.getAll = function (...args) { + if (this.name === "keyval" && !failed) { failed = true; throw new Error("temporary read failure"); } + return original.apply(this, args); + }; + try { + assert.deepEqual(await fresh.loadAllHistory(), []); + assert.equal(failed, true); + assert.equal((await fresh.loadAllHistory())[0].id, "retry-legacy"); + } finally { + IDBObjectStore.prototype.getAll = original; + } +}); diff --git a/image-studio/frontend/test/studioFields.test.mjs b/image-studio/frontend/test/studioFields.test.mjs new file mode 100644 index 000000000..506ad72a0 --- /dev/null +++ b/image-studio/frontend/test/studioFields.test.mjs @@ -0,0 +1,60 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { build } from "esbuild"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { fileURLToPath, pathToFileURL } from "node:url"; + +test("static UI subscriptions skip 200 progress/preview updates and still render changed history", async () => { + const dir = await mkdtemp(`${tmpdir()}/image-studio-field-render-`); + const storageDescriptor = Object.getOwnPropertyDescriptor(globalThis, "localStorage"); + Object.defineProperty(globalThis, "localStorage", { configurable: true, value: { getItem: () => null } }); + let renderer; + try { + await build({ + stdin: { + contents: `export { useStudioStore } from "./src/state/studioStore.ts"; + export { useStudioFields } from "./src/state/useStudioFields.ts"; + export { createElement, memo } from "react"; + export { default as renderer } from "react-test-renderer";`, + resolveDir: fileURLToPath(new URL("..", import.meta.url)), + }, + outfile: `${dir}/fields.mjs`, bundle: true, platform: "node", format: "esm", + define: { "import.meta.env": JSON.stringify({ VITE_TARGET_PLATFORM: "macos", PACKAGE_VERSION: "test" }) }, + }); + const { useStudioStore: store, useStudioFields, createElement, memo, renderer: { create, act } } = await import(pathToFileURL(`${dir}/fields.mjs`)); + store.setState({ currentImage: { id: "same-image" } }); + let fieldRenders = 0; + let fullRenders = 0; + const StaticRegion = memo(() => { + const { history } = useStudioFields(["history", "historyHasMore"]); + const id = store((state) => state.currentImage?.id); + fieldRenders += 1; + return createElement("span", null, `${history.length}:${id}`); + }); + function WholeStoreRegion() { + store(); + fullRenders += 1; + return createElement(StaticRegion); + } + // These updates are synchronous; async act in a bundled Node runtime + // otherwise falls back to browser MessageChannels that keep ports alive. + act(() => { renderer = create(createElement(WholeStoreRegion)); }); + for (let i = 0; i < 200; i++) { + act(() => { store.setState({ progress: { bytes: i }, currentImage: { id: "same-image", imageB64: `frame-${i}` } }); }); + } + assert.equal(fullRenders, 201); + assert.equal(fieldRenders, 1); + act(() => { store.setState({ history: [{ id: "new", createdAt: 1000 }] }); }); + assert.equal(fieldRenders, 2); + assert.equal(renderer.toJSON().children[0], "1:same-image"); + act(() => { store.setState({ currentImage: { id: "selected-image" } }); }); + assert.equal(fieldRenders, 3); + assert.equal(renderer.toJSON().children[0], "1:selected-image"); + } finally { + renderer?.unmount(); + if (storageDescriptor) Object.defineProperty(globalThis, "localStorage", storageDescriptor); + else delete globalThis.localStorage; + await rm(dir, { recursive: true, force: true }); + } +}); diff --git a/image-studio/frontend/vite.config.ts b/image-studio/frontend/vite.config.ts index 93a1ff26c..91a18bd79 100644 --- a/image-studio/frontend/vite.config.ts +++ b/image-studio/frontend/vite.config.ts @@ -10,7 +10,6 @@ const explicitOutDir = (process.env.VITE_OUT_DIR ?? "").trim(); function manualChunks(id: string) { if (id.includes("/wailsjs/")) return "wails-runtime"; - if (id.includes("/src/platform/android/") || id.includes("/src/platform/desktop/")) return "platform-ui"; if (id.includes("/node_modules/")) { if (id.includes("/react-konva/") || id.includes("/konva/")) return "canvas-vendor"; if (id.includes("/react/") || id.includes("/react-dom/") || id.includes("/scheduler/")) return "react-vendor"; From 95f3e9b835220aff261def2fcbb12ea2fa32b2fa Mon Sep 17 00:00:00 2001 From: lin <1609924163@qq.com> Date: Tue, 6 Oct 2026 16:57:58 +0800 Subject: [PATCH 6/6] Align platform verification runtime with TypeScript tests Use Node 24 for the platform verification workflow and local .nvmrc. Reproduce the Node 20 TypeScript loader failure and verify all 210 frontend and 5 Worker tests under Node 24 before upstream submission. --- .github/workflows/verify-platform-kernel.yml | 2 +- .nvmrc | 1 + docs/build.md | 2 +- docs/verification/pr-preflight-20261006.json | 30 ++++++++++++++++++++ 4 files changed, 33 insertions(+), 2 deletions(-) create mode 100644 .nvmrc create mode 100644 docs/verification/pr-preflight-20261006.json diff --git a/.github/workflows/verify-platform-kernel.yml b/.github/workflows/verify-platform-kernel.yml index 02a1cd411..fb75cf367 100644 --- a/.github/workflows/verify-platform-kernel.yml +++ b/.github/workflows/verify-platform-kernel.yml @@ -25,7 +25,7 @@ jobs: - name: Set up Node uses: actions/setup-node@v4 with: - node-version: "20" + node-version: "24" cache: npm cache-dependency-path: | image-studio/frontend/package-lock.json diff --git a/.nvmrc b/.nvmrc new file mode 100644 index 000000000..a45fd52cc --- /dev/null +++ b/.nvmrc @@ -0,0 +1 @@ +24 diff --git a/docs/build.md b/docs/build.md index 7ffd1972f..2da604d53 100644 --- a/docs/build.md +++ b/docs/build.md @@ -5,7 +5,7 @@ ## 环境要求 - Go 1.25.x。当前 `go.mod` 使用 `go 1.25.5` 与 `toolchain go1.26.3`。 -- Node.js 20 或更新版本。 +- Node.js 24 或更新版本。前端测试直接加载 TypeScript，平台验证 CI 使用 Node 24；仓库 `.nvmrc` 与此一致。 - Wails CLI v2.12.0。非 macOS release workflow 使用 `go install github.com/wailsapp/wails/v2/cmd/wails@v2.12.0`。 - Android 构建需要 JDK 17、Android SDK 34、Build Tools 34.0.0、Gradle 8.7。 diff --git a/docs/verification/pr-preflight-20261006.json b/docs/verification/pr-preflight-20261006.json new file mode 100644 index 000000000..395d747a0 --- /dev/null +++ b/docs/verification/pr-preflight-20261006.json @@ -0,0 +1,30 @@ +{ + "date": "2026-10-06", + "timezone": "Asia/Shanghai", + "recordedAt": "2026-10-06T16:57:58.105616+08:00", + "target": "RoseKhlifa/Image-Studio:main", + "upstreamCommit": "4c221a5", + "branch": "DR-lin-eng:codex/issue-audit-20261006", + "upstreamSyncChanges": [ + "README.md" + ], + "runtimeCompatibility": { + "before": { + "node": "20.20.2", + "fixture": "test/completionSound.test.mjs", + "result": "ERR_UNKNOWN_FILE_EXTENSION for src/lib/completionSound.ts" + }, + "after": { + "node": "24.19.0", + "frontendTests": 210, + "workerTests": 5, + "failures": 0 + } + }, + "workflow": "verify-platform-kernel.yml: node-version 24", + "localVersionFile": ".nvmrc: 24", + "applicationVerificationRecords": [ + "issue-audit-20261006.json", + "performance-20261006.json" + ] +}