diff --git a/.github/ISSUE_TEMPLATE/00-bug_report_zh.yml b/.github/ISSUE_TEMPLATE/00-bug_report_zh.yml index 7a642bfcc..ec18a650b 100644 --- a/.github/ISSUE_TEMPLATE/00-bug_report_zh.yml +++ b/.github/ISSUE_TEMPLATE/00-bug_report_zh.yml @@ -13,7 +13,7 @@ body: attributes: label: 请确认以下事项 description: | - 您必须勾选以下内容，否则您的问题可能会被直接关闭。 + 您必须阅读、检查、确认、同意以下内容，否则您的问题一定会被直接关闭。 或者您可以去[讨论区](https://github.com/OpenListTeam/OpenList/discussions)。 options: - label: | @@ -34,7 +34,8 @@ body: 我认为此问题必须由`OpenList`处理，而非第三方。 - label: | 我已确认这个问题在最新版本中没有被修复。 - + - label: | + 我没有阅读这个清单，只是闭眼选中了所有的复选框，请关闭这个 Issue 。 - type: input id: version attributes: @@ -60,19 +61,21 @@ body: validations: required: true - type: textarea - id: config + id: logs attributes: - label: 配置文件内容（必填） + label: 日志（必填） description: | - 请提供您的`OpenList`应用的配置文件，并截图相关存储配置。(可隐藏隐私字段) + 请复制粘贴错误日志，或者截图。(可隐藏隐私字段) [查看方法](https://doc.oplist.org/faq/howto#%E5%A6%82%E4%BD%95%E5%BF%AB%E9%80%9F%E5%AE%9A%E4%BD%8Dbug) validations: required: true - type: textarea - id: logs + id: config attributes: - label: 日志（可选） + label: 配置文件内容（必填） description: | - 请复制粘贴错误日志，或者截图。(可隐藏隐私字段) [查看方法](https://doc.oplist.org/faq/howto#%E5%A6%82%E4%BD%95%E5%BF%AB%E9%80%9F%E5%AE%9A%E4%BD%8Dbug) + 请提供您的`OpenList`应用的配置文件，并截图相关存储配置。(可隐藏隐私字段) + validations: + required: true - type: textarea id: reproduction attributes: diff --git a/.github/ISSUE_TEMPLATE/01-bug_report_en.yml b/.github/ISSUE_TEMPLATE/01-bug_report_en.yml index 322bbf968..0085d4de5 100644 --- a/.github/ISSUE_TEMPLATE/01-bug_report_en.yml +++ b/.github/ISSUE_TEMPLATE/01-bug_report_en.yml @@ -13,7 +13,7 @@ body: attributes: label: Please confirm the following description: | - You must check all the following, otherwise your issue may be closed directly. + You must read, check, confirm, and agree to all the following, otherwise your issue will definitely be closed directly. Or you can go to the [discussions](https://github.com/OpenListTeam/OpenList/discussions). options: - label: | @@ -34,7 +34,8 @@ body: I believe this issue must be handled by `OpenList` and not by a third party. - label: | I confirm this issue is not fixed in the latest version. - + - label: | + I have not read these checkboxes and therefore I just ticked them all, Please close this issue. - type: input id: version attributes: @@ -60,19 +61,21 @@ body: validations: required: true - type: textarea - id: config + id: logs attributes: - label: Configuration File Content (required) + label: Logs (required) description: | - Please provide your `OpenList` application's configuration file and a screenshot of the relevant storage configuration. (You may mask sensitive fields) + Please copy and paste any relevant log output or screenshots. (You may mask sensitive fields) [Guide](https://doc.oplist.org/faq/howto#how-to-quickly-locate-bugs) validations: required: true - type: textarea - id: logs + id: config attributes: - label: Logs (optional) + label: Configuration File Content (required) description: | - Please copy and paste any relevant log output or screenshots. (You may mask sensitive fields) [Guide](https://doc.oplist.org/faq/howto#how-to-quickly-locate-bugs) + Please provide your `OpenList` application's configuration file and a screenshot of the relevant storage configuration. (You may mask sensitive fields) + validations: + required: true - type: textarea id: reproduction attributes: diff --git a/.github/ISSUE_TEMPLATE/02-feature_request_zh.yml b/.github/ISSUE_TEMPLATE/02-feature_request_zh.yml index 76ef66d32..907240261 100644 --- a/.github/ISSUE_TEMPLATE/02-feature_request_zh.yml +++ b/.github/ISSUE_TEMPLATE/02-feature_request_zh.yml @@ -7,7 +7,7 @@ body: attributes: label: 请确认以下事项 description: | - 您必须勾选以下内容，否则您的问题可能会被直接关闭。 + 您必须阅读、检查、确认、同意以下内容，否则您的问题可能会被直接关闭。 或者您可以去[讨论区](https://github.com/OpenListTeam/OpenList/discussions)。 options: - label: | @@ -28,6 +28,8 @@ body: 我已确认此功能尚未被实现。 - label: | 我已确认此功能是合理的，且有普遍需求，并非我个人需要。 + - label: | + 我没有阅读这个清单，只是闭眼选中了所有的复选框，请关闭这个 Issue 。 - type: textarea id: feature-description attributes: diff --git a/.github/ISSUE_TEMPLATE/03-feature_request_en.yml b/.github/ISSUE_TEMPLATE/03-feature_request_en.yml index 521a4a2fc..393118592 100644 --- a/.github/ISSUE_TEMPLATE/03-feature_request_en.yml +++ b/.github/ISSUE_TEMPLATE/03-feature_request_en.yml @@ -1,13 +1,13 @@ name: "Feature Request" description: Feature Request / Enhancement -title: "[Feature] Please change the title to your feature name" +title: "[Feature] Please modify the title to your feature name" labels: [enhancement] body: - type: checkboxes attributes: label: Please confirm the following description: | - You must check all the following, otherwise your request may be closed directly. + You must read, check, confirm, and agree to all the following, otherwise your request may be closed directly. Or you can go to the [discussions](https://github.com/OpenListTeam/OpenList/discussions). options: - label: | @@ -28,6 +28,8 @@ body: I confirm this feature has not been implemented yet. - label: | I confirm this feature is reasonable and has general demand, not just my personal need. + - label: | + I have not read these checkboxes and therefore I just ticked them all, Please close this issue. - type: textarea id: feature-description attributes: diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 000000000..58e122837 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,114 @@ + + +## Summary / 摘要 + + + + + +- [ ] This PR has breaking changes. + / 此 PR 包含破坏性变更。 +- [ ] This PR changes public API, config, storage format, or migration behavior. + / 此 PR 修改了公开 API、配置、存储格式或迁移行为。 +- [ ] This PR requires corresponding changes in related repositories. + / 此 PR 需要关联仓库同步修改。 + +Related repository PRs / 关联仓库 PR: + +- OpenList-Frontend: +- OpenList-Docs: + +## Related Issues / 关联 Issue + + + +## Testing / 测试 + + + +- [ ] `go test ./...` +- [ ] Manual test / 手动测试: + +## Checklist / 检查清单 + +- [ ] I have read [CONTRIBUTING](https://github.com/OpenListTeam/OpenList/blob/main/CONTRIBUTING.md). + / 我已阅读 [CONTRIBUTING](https://github.com/OpenListTeam/OpenList/blob/main/CONTRIBUTING.md)。 +- [ ] I confirm this contribution follows the repository license, contribution policy, and code of conduct. + / 我确认此贡献符合仓库许可证、贡献规范和行为准则。 +- [ ] I have formatted the changed code with `gofmt`, `go fmt`, or `prettier` where applicable. + / 我已按适用情况使用 `gofmt`、`go fmt` 或 `prettier` 格式化变更代码。 +- [ ] I have requested review from relevant maintainers or code owners where applicable. + / 我已在适用情况下请求相关维护者或代码所有者审查。 + +## AI Disclosure / AI 使用声明 + + + +- [ ] This PR includes AI-assisted content. + / 此 PR 包含 AI 辅助内容。 + +Tools used / 使用工具: + +- [ ] ChatGPT +- [ ] Codex +- [ ] GitHub Copilot +- [ ] Claude +- [ ] Gemini +- [ ] Other (please specify) / 其他（请注明）: + +Usage scope / 使用范围: + +- [ ] Code generation / 代码生成 +- [ ] Refactoring / 重构 +- [ ] Documentation / 文档 +- [ ] Tests / 测试 +- [ ] Translation / 翻译 +- [ ] Review assistance / 审查辅助 + +- [ ] I have reviewed and validated all AI-assisted content included in this PR. + / 我已审核并验证此 PR 中的所有 AI 辅助内容。 +- [ ] I have ensured that all AI-assisted commits include `Co-Authored-By` attribution. + / 我已确保所有 AI 辅助提交都包含 `Co-Authored-By` 归属信息。 +- [ ] I can reproduce all AI-assisted content included in this PR without any AI tools. + / 我可以在没有任何 AI 工具的情况下重现此 PR 中包含的所有 AI 辅助内容。 diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f8e46b8ee..f443d6896 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -20,15 +20,15 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v7 - - uses: benjlevesque/short-sha@v3.0 + - uses: benjlevesque/short-sha@dbe07338b37c456ce06d23409b35a56a7815eef4 # v4.0 id: short-sha - name: Setup Go - uses: actions/setup-go@v5 + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7 with: - go-version: "1.24.5" + go-version: "1.26.4" - name: Setup web run: bash build.sh dev web @@ -37,7 +37,7 @@ jobs: FRONTEND_REPO: ${{ vars.FRONTEND_REPO }} - name: Build - uses: OpenListTeam/cgo-actions@v1.2.2 + uses: OpenListTeam/cgo-actions@v1.2.5 with: targets: ${{ matrix.target }} flags: ${{ contains(matrix.target, '-musl') && '-ldflags=-linkmode external -extldflags ''-static -fpic''' || '-ldflags=' }} @@ -64,7 +64,7 @@ jobs: fi - name: Upload artifact - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: openlist_${{ steps.short-sha.outputs.sha }}_${{ matrix.target }} path: build/* diff --git a/.github/workflows/changelog.yml b/.github/workflows/changelog.yml index 8d458314a..8fa3117a0 100644 --- a/.github/workflows/changelog.yml +++ b/.github/workflows/changelog.yml @@ -14,7 +14,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@v7 with: fetch-depth: 0 diff --git a/.github/workflows/clean_untagged_docker.yml b/.github/workflows/clean_untagged_docker.yml new file mode 100644 index 000000000..571b2a3e8 --- /dev/null +++ b/.github/workflows/clean_untagged_docker.yml @@ -0,0 +1,215 @@ +name: Container Registry Cleanup + +permissions: + contents: read + packages: write + +on: + workflow_dispatch: + inputs: + package: + description: 'The name of the package to clean up' + required: true + default: 'openlist-git' + owner: + description: 'The owner of the package (user or organization)' + required: false + default: 'openlistteam' + older_than: + description: 'Delete untagged images older than this many days' + required: false + default: '-1' + untagged_timestamp_tolerance: + description: 'Tolerance in milliseconds for untagged images close to tagged ones' + required: false + # We do this because multi-arch docker containers will be pushed as separate untagged images + # to the container registry and GitHub cannot recognize them as part of a tagged image. + # Setting this option large enough prevents accidental deletion of one of these images. + # Here we set it to 10 seconds + default: '10000' + token: + description: 'GitHub token with permissions to delete packages (leave empty to use default GITHUB_TOKEN)' + required: false + default: '' + +jobs: + cleanup: + runs-on: ubuntu-slim + + steps: + - name: Check inputs and tools + id: setup + shell: bash + run: | + PACKAGE="${{ github.event.inputs.package || 'openlist-git' }}" + OLDER="${{ github.event.inputs.older_than || '-1' }}" + TOLERANCE_VAL="${{ github.event.inputs.untagged_timestamp_tolerance || '10000' }}" + TOKEN_INPUT="${{ github.event.inputs.token || '' }}" + OWNER="${{ github.event.inputs.owner || 'openlistteam' }}" + + echo "package_name=$PACKAGE" >> "$GITHUB_OUTPUT" + echo "older_than=$OLDER" >> "$GITHUB_OUTPUT" + echo "tolerance=$TOLERANCE_VAL" >> "$GITHUB_OUTPUT" + echo "owner=$OWNER" >> "$GITHUB_OUTPUT" + + if [[ -z "$TOKEN_INPUT" ]]; then + echo "::notice:: No token provided, using GITHUB_TOKEN instead." + echo "token=${{ secrets.GITHUB_TOKEN }}" >> "$GITHUB_OUTPUT" + else + echo "::add-mask::$TOKEN_INPUT" + echo "token=$TOKEN_INPUT" >> "$GITHUB_OUTPUT" + fi + + if [[ -z "$PACKAGE" || -z "$OWNER" || ( -z "$TOKEN_INPUT" && -z "${{ secrets.GITHUB_TOKEN }}" ) ]]; then + echo "::error:: Missing required inputs (package, token, or owner context)." + exit 1 + fi + + command -v curl >/dev/null 2>&1 || { echo "::error::curl is required but not installed."; exit 1; } + command -v jq >/dev/null 2>&1 || { echo "::error::jq is required but not installed."; exit 1; } + + - name: Determine owner type (user or organization) + id: owner_type + shell: bash + run: | + OWNER="${{ steps.setup.outputs.owner }}" + TOKEN="${{ steps.setup.outputs.token }}" + + TYPE=$(curl -s -H "Authorization: token $TOKEN" \ + -H "Accept: application/vnd.github.v3+json" \ + "https://api.github.com/users/${OWNER}" | jq -r '.type') + + if [[ "$TYPE" == "Organization" ]]; then + OWNER_PATH="orgs" + else + OWNER_PATH="users" + fi + + echo "owner_path=$OWNER_PATH" >> "$GITHUB_OUTPUT" + echo "Detected owner type: $TYPE (using /${OWNER_PATH}/)" + + - name: Access all versions from GitHub + id: fetch + shell: bash + run: | + PACKAGE_NAME="${{ steps.setup.outputs.package_name }}" + OWNER="${{ steps.setup.outputs.owner }}" + TOKEN="${{ steps.setup.outputs.token }}" + OWNER_PATH="${{ steps.owner_type.outputs.owner_path }}" + + BASE_URL="https://api.github.com/${OWNER_PATH}/${OWNER}/packages/container/${PACKAGE_NAME}/versions" + URL="${BASE_URL}?per_page=100" + TEMP_FILE="/tmp/all_versions.ndjson" + + > "$TEMP_FILE" + PAGE=1 + while [[ -n "$URL" && $PAGE -le 10 ]]; do + echo "Fetching: $URL" + + curl -s -H "Authorization: token $TOKEN" \ + -H "Accept: application/vnd.github.v3+json" \ + -D headers.txt \ + "$URL" > page.json + + jq -c '.[]' page.json >> "$TEMP_FILE" + + URL=$(grep -i '^link:' headers.txt | grep -o '<[^>]*>;\s*rel="next"' | sed 's/<\(.*\)>.*/\1/' || true) + PAGE=$((PAGE + 1)) + done + rm -f headers.txt page.json + + echo "temp_file=$TEMP_FILE" >> "$GITHUB_OUTPUT" + + - name: Filter untagged versions + id: filter + shell: bash + run: | + OLDER_THAN="${{ steps.setup.outputs.older_than }}" + TOLERANCE="${{ steps.setup.outputs.tolerance }}" + TEMP_FILE="${{ steps.fetch.outputs.temp_file }}" + + TO_DELETE=$(jq -c -s \ + --arg older_than "$OLDER_THAN" \ + --arg tolerance "$TOLERANCE" \ + ' + def abs: if . < 0 then -. else . end; + def to_ms: sub("\\.[0-9]+Z$"; "Z") | strptime("%Y-%m-%dT%H:%M:%SZ") | mktime * 1000; + + . as $all | + ($older_than | tonumber) as $older_than_val | + ($tolerance | tonumber) as $tolerance_val | + now as $now | + (if $older_than_val > 0 then ($now - ($older_than_val * 86400)) * 1000 else 0 end) as $cutoff_ms | + + [ $all[] | select((.metadata.container.tags // []) | length > 0) | .created_at | to_ms ] as $tagged_times | + + $all[] | + select((.metadata.container.tags // []) | length == 0) | + . as $item | + (.created_at | to_ms) as $created_ms | + + (if $older_than_val > 0 then $created_ms < $cutoff_ms else true end) as $pass_age | + + (if $tolerance_val <= 0 then true + else [ $tagged_times[] | select(($created_ms - .) | abs < $tolerance_val) ] | length == 0 + end) as $pass_tolerance | + + select($pass_age and $pass_tolerance) | + {id, name} + ' "$TEMP_FILE") + + rm -f "$TEMP_FILE" + + if [[ -z "$TO_DELETE" ]]; then + DELETE_COUNT=0 + else + DELETE_COUNT=$(echo "$TO_DELETE" | wc -l | tr -d ' ') + fi + + echo "Found $DELETE_COUNT untagged images no longer necessary" + + { + echo 'to_delete<> "$GITHUB_OUTPUT" + + echo "delete_count=$DELETE_COUNT" >> "$GITHUB_OUTPUT" + + - name: Delete untagged versions + id: delete + shell: bash + run: | + PACKAGE_NAME="${{ steps.setup.outputs.package_name }}" + OWNER="${{ steps.setup.outputs.owner }}" + TOKEN="${{ steps.setup.outputs.token }}" + OWNER_PATH="${{ steps.owner_type.outputs.owner_path }}" + DELETE_COUNT="${{ steps.filter.outputs.delete_count }}" + TO_DELETE='${{ steps.filter.outputs.to_delete }}' + + if [[ "$DELETE_COUNT" -gt 0 && -n "$TO_DELETE" ]]; then + while IFS= read -r item; do + [[ -z "$item" ]] && continue + + # $item 是完整的单行 JSON，如 {"id":123,"name":"sha256:..."} + ID=$(echo "$item" | jq -r '.id') + NAME=$(echo "$item" | jq -r '.name') + + DELETE_URL="https://api.github.com/${OWNER_PATH}/${OWNER}/packages/container/${PACKAGE_NAME}/versions/${ID}" + + HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" -X DELETE \ + -H "Authorization: token $TOKEN" \ + -H "Accept: application/vnd.github.v3+json" \ + "$DELETE_URL") + + if [[ "$HTTP_CODE" == "204" ]]; then + echo "Deleted untagged container image '${NAME}'" + else + echo "::warning:: Failed to delete ${NAME} (ID: ${ID}). HTTP Status: ${HTTP_CODE}" + fi + done <<< "$TO_DELETE" + else + echo "No images to delete." + fi + + echo "Cleanup completed." diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 39ab7dbca..783a9abcb 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -44,14 +44,15 @@ jobs: swap-storage: true - name: Setup Go - uses: actions/setup-go@v5 + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7 with: - go-version: '1.24' + go-version: '1.26.4' - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v7 with: - fetch-depth: 0 + fetch-depth: 1 + fetch-tags: true - name: Install dependencies if: matrix.target-platform == '' @@ -69,8 +70,9 @@ jobs: FRONTEND_REPO: ${{ vars.FRONTEND_REPO }} - name: Upload assets - uses: softprops/action-gh-release@v2 - with: - files: build/compress/* - prerelease: false - + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + for file in build/compress/*; do + gh release upload "${{ github.event.release.tag_name }}" "$file" --clobber + done diff --git a/.github/workflows/trigger-makefile-update.yml b/.github/workflows/trigger-makefile-update.yml index 58b53b485..b8c4491cb 100644 --- a/.github/workflows/trigger-makefile-update.yml +++ b/.github/workflows/trigger-makefile-update.yml @@ -16,20 +16,33 @@ jobs: runs-on: ubuntu-latest steps: - name: Trigger Makefile hash update - uses: peter-evans/repository-dispatch@v3 + uses: actions/github-script@v9 with: - token: ${{ secrets.EXTERNAL_REPO_TOKEN_LUCI_APP_OPENLIST }} - repository: ${{ vars.HOOK_REPO || 'OpenListTeam/OpenList-OpenWRT' }} - event-type: update-hashes - client-payload: | - { - "source_repository": "${{ github.repository }}", - "release_tag": "${{ inputs.tag || github.ref_name }}", - "release_name": "${{ inputs.tag || github.ref_name }}", - "release_url": "${{ github.server_url }}/${{ github.repository }}/releases/tag/${{ inputs.tag || github.ref_name }}", - "triggered_by": "${{ github.actor }}", - "trigger_reason": "${{ github.event_name }}" - } + github-token: ${{ secrets.EXTERNAL_REPO_TOKEN_LUCI_APP_OPENLIST }} + script: | + const repository = process.env.REPOSITORY; + const [owner, repo] = repository.split('/'); + + await github.rest.repos.createDispatchEvent({ + owner: owner, + repo: repo, + event_type: 'update-hashes', + client_payload: { + source_repository: process.env.SOURCE_REPOSITORY, + release_tag: process.env.RELEASE_TAG, + release_name: process.env.RELEASE_TAG, + release_url: process.env.RELEASE_URL, + triggered_by: process.env.ACTOR, + trigger_reason: process.env.EVENT_NAME + } + }); + env: + REPOSITORY: ${{ vars.HOOK_REPO || 'OpenListTeam/OpenList-OpenWRT' }} + SOURCE_REPOSITORY: ${{ github.repository }} + RELEASE_TAG: ${{ inputs.tag || github.ref_name }} + RELEASE_URL: ${{ github.server_url }}/${{ github.repository }}/releases/tag/${{ inputs.tag || github.ref_name }} + ACTOR: ${{ github.actor }} + EVENT_NAME: ${{ github.event_name }} - name: Log trigger information run: | diff --git a/.gitignore b/.gitignore index 7922287bb..e572f24ad 100644 --- a/.gitignore +++ b/.gitignore @@ -22,6 +22,7 @@ output/ # vendor/ /bin/* *.json +!renovate.json /build /data/ /tmp/ @@ -29,7 +30,8 @@ output/ /lang/ /daemon/ /public/dist/* -/!public/dist/README.md +!/public/dist/README.md .VSCodeCounter /.worktrees/ +*.syso diff --git a/build.sh b/build.sh index ff769ce1a..f1079f756 100644 --- a/build.sh +++ b/build.sh @@ -31,7 +31,7 @@ else # webVersion=$(eval "curl -fsSL --max-time 2 $githubAuthArgs \"https://api.github.com/repos/$frontendRepo/releases/latest\"" | grep "tag_name" | head -n 1 | awk -F ":" '{print $2}' | sed 's/\"//g;s/,//g;s/ //g') fi -webVersion=4.2.3 +webVersion=4.2.5 echo "backend version: $version" echo "frontend version: $webVersion" diff --git a/cmd/crypt.go b/cmd/crypt.go index 12f51f5cd..7cdb29258 100644 --- a/cmd/crypt.go +++ b/cmd/crypt.go @@ -23,7 +23,7 @@ type options struct { pwd string //de/encrypt password salt string - filenameEncryption string //reference drivers\crypt\meta.go Addtion + filenameEncryption string //reference drivers\crypt\meta.go Addition dirnameEncryption string filenameEncode string suffix string diff --git a/drivers/139/types.go b/drivers/139/types.go index 8c23cdcbb..a7401a3aa 100644 --- a/drivers/139/types.go +++ b/drivers/139/types.go @@ -9,6 +9,7 @@ const ( MetaFamily string = "family" MetaGroup string = "group" MetaPersonalNew string = "personal_new" + MetaShare string = "share" ) type BaseResp struct { @@ -285,6 +286,56 @@ type PersonalUploadUrlResp struct { } } +type ShareCatalog struct { + CaID string `json:"caId"` + CaName string `json:"caName"` + UdTime string `json:"udTime"` +} + +type ShareContent struct { + CoID string `json:"coId"` + CoName string `json:"coName"` + CoSize int64 `json:"coSize"` + CoType int `json:"coType"` + UdTime string `json:"udTime"` + CoPath string `json:"coPath"` + PresentURL string `json:"presentURL"` + DownloadURL string `json:"downloadURL"` +} + +type ShareListResp struct { + BaseResp + Data struct { + LKName string `json:"lkName"` + Passwd string `json:"password"` + CaLst []ShareCatalog `json:"caLst"` + CoLst []ShareContent `json:"coLst"` + } `json:"data"` +} + +type ShareContentInfo struct { + PresentURL string `json:"presentURL"` + DownloadURL string `json:"cdnDownLoadUrl"` +} + +type ShareDownloadResp struct { + BaseResp + Data struct { + DownloadURL string `json:"downloadURL"` + RedrURL string `json:"redrUrl"` + ExtInfo struct { + CDNDownloadURL string `json:"cdnDownloadUrl"` + } `json:"extInfo"` + } `json:"data"` +} + +type ShareContentInfoResp struct { + BaseResp + Data struct { + ContentInfo ShareContentInfo `json:"contentInfo"` + } `json:"data"` +} + type QueryRoutePolicyResp struct { Success bool `json:"success"` Code string `json:"code"` diff --git a/drivers/189/driver.go b/drivers/189/driver.go index 813cee0b0..4ed2d57c6 100644 --- a/drivers/189/driver.go +++ b/drivers/189/driver.go @@ -254,5 +254,6 @@ func (d *Cloud189) GetDetails(ctx context.Context) (*model.StorageDetails, error }, nil } + var _ driver.Driver = (*Cloud189)(nil) var _ driver.PutResult = (*Cloud189)(nil) diff --git a/drivers/189/util.go b/drivers/189/util.go index 2211f2669..6a66df5a5 100644 --- a/drivers/189/util.go +++ b/drivers/189/util.go @@ -238,7 +238,7 @@ func (d *Cloud189) oldUpload(dstDir model.Obj, file model.FileStreamer) error { if utils.Json.Get(res.Body(), "MD5").ToString() != "" { return nil } - log.Debugf(res.String()) + log.Debugf("%s", res.String()) return errors.New(res.String()) } diff --git a/drivers/all.go b/drivers/all.go index ddf86130c..0234a7abf 100644 --- a/drivers/all.go +++ b/drivers/all.go @@ -23,6 +23,7 @@ import ( _ "github.com/OpenListTeam/OpenList/v4/drivers/azure_blob" _ "github.com/OpenListTeam/OpenList/v4/drivers/baidu_netdisk" _ "github.com/OpenListTeam/OpenList/v4/drivers/baidu_photo" + _ "github.com/OpenListTeam/OpenList/v4/drivers/bunny_storage" _ "github.com/OpenListTeam/OpenList/v4/drivers/chaoxing" _ "github.com/OpenListTeam/OpenList/v4/drivers/chunk" _ "github.com/OpenListTeam/OpenList/v4/drivers/cloudflare_imgbed" diff --git a/drivers/bunny_storage/driver.go b/drivers/bunny_storage/driver.go new file mode 100644 index 000000000..c097da80d --- /dev/null +++ b/drivers/bunny_storage/driver.go @@ -0,0 +1,226 @@ +package bunny_storage + +import ( + "bytes" + "context" + "fmt" + "net/http" + "net/url" + stdpath "path" + "strings" + "time" + + "github.com/OpenListTeam/OpenList/v4/drivers/base" + "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/errs" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/go-resty/resty/v2" +) + +type BunnyStorage struct { + model.Storage + Addition + client *resty.Client + endpoint *url.URL + cdnBase *url.URL +} + +func (d *BunnyStorage) Config() driver.Config { + cfg := config + if d.StorageZoneName != "" && d.CDNBaseURL == "" { + cfg.OnlyProxy = true + cfg.PreferProxy = true + } + if d.CDNTokenKey != "" && d.CDNTokenIncludeIP { + cfg.LinkCacheMode = driver.LinkCacheIP + } + return cfg +} + +func (d *BunnyStorage) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *BunnyStorage) Init(ctx context.Context) error { + if d.RootFolderPath == "" { + d.RootFolderPath = "/" + } + if d.Endpoint == "" { + d.Endpoint = defaultEndpoint + } + if d.SignURLExpire <= 0 { + d.SignURLExpire = 4 + } + if d.CDNTokenMethod == "" { + d.CDNTokenMethod = cdnTokenMethodSHA256 + } + endpoint, err := normalizeBaseURL(d.Endpoint, defaultEndpoint) + if err != nil { + return fmt.Errorf("invalid endpoint: %w", err) + } + d.endpoint = endpoint + if d.CDNBaseURL != "" { + cdnBase, err := normalizeBaseURL(d.CDNBaseURL, "") + if err != nil { + return fmt.Errorf("invalid cdn_base_url: %w", err) + } + d.cdnBase = cdnBase + } + d.client = base.RestyClient + if d.client == nil { + d.client = base.NewRestyClient() + } + return nil +} + +func (d *BunnyStorage) Drop(ctx context.Context) error { + return nil +} + +func (d *BunnyStorage) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + var items []bunnyObject + resp, err := d.authRequest(). + SetContext(ctx). + SetResult(&items). + Get(d.storageURL(dir.GetPath(), true)) + if err != nil { + return nil, err + } + if err := d.handleResponseError(resp); err != nil { + return nil, err + } + result := make([]model.Obj, 0, len(items)) + placeholder := d.placeholderName() + for _, item := range items { + if item.ObjectName == "" { + continue + } + if !args.S3ShowPlaceholder && !item.IsDirectory && item.ObjectName == placeholder { + continue + } + result = append(result, d.toObj(dir.GetPath(), item)) + } + return result, nil +} + +func (d *BunnyStorage) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + if file.IsDir() { + return nil, errs.NotFile + } + cacheTTL := time.Duration(0) + if d.cdnBase != nil { + linkURL := d.cdnURL(d.cdnObjectPath(file.GetPath())) + link := &model.Link{ + URL: linkURL, + ContentLength: file.GetSize(), + Expiration: &cacheTTL, + } + if d.CDNTokenKey != "" { + signedURL, _, err := d.signCDNURL(linkURL, args.IP) + if err != nil { + return nil, err + } + link.URL = signedURL + } + return link, nil + } + return &model.Link{ + URL: d.storageURL(file.GetPath(), false), + Header: http.Header{"AccessKey": []string{d.AccessKey}}, + ContentLength: file.GetSize(), + Expiration: &cacheTTL, + }, nil +} + +func (d *BunnyStorage) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { + dirPath := stdpath.Join(parentDir.GetPath(), dirName) + placeholderPath := stdpath.Join(dirPath, d.placeholderName()) + if err := d.putReader(ctx, placeholderPath, bytes.NewReader(nil), 0, "application/octet-stream", nil); err != nil { + return nil, err + } + now := time.Now() + return &model.Object{ + Path: dirPath, + Name: dirName, + Modified: now, + Ctime: now, + IsFolder: true, + }, nil +} + +func (d *BunnyStorage) Remove(ctx context.Context, obj model.Obj) error { + resp, err := d.authRequest(). + SetContext(ctx). + Delete(d.storageURL(obj.GetPath(), obj.IsDir())) + if err != nil { + return err + } + return d.handleResponseError(resp) +} + +func (d *BunnyStorage) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { + if up == nil { + up = func(float64) {} + } + dstPath := stdpath.Join(dstDir.GetPath(), file.GetName()) + err := d.putReader(ctx, dstPath, driver.NewLimitedUploadStream(ctx, &driver.ReaderUpdatingProgress{ + Reader: file, + UpdateProgress: up, + }), file.GetSize(), file.GetMimetype(), nil) + if err != nil { + return nil, err + } + now := time.Now() + return &model.Object{ + Path: dstPath, + Name: file.GetName(), + Size: file.GetSize(), + Modified: now, + Ctime: now, + }, nil +} + +func (d *BunnyStorage) putReader(ctx context.Context, path string, body any, size int64, contentType string, extraHeaders http.Header) error { + if contentType == "" { + contentType = "application/octet-stream" + } + req := d.authRequest(). + SetContext(ctx). + SetBody(body). + SetHeader("Content-Type", contentType) + if size >= 0 { + req.SetHeader("Content-Length", fmt.Sprint(size)) + } + for key, values := range extraHeaders { + for _, value := range values { + req.SetHeader(key, value) + } + } + resp, err := req.Put(d.storageURL(path, false)) + if err != nil { + return err + } + return d.handleResponseError(resp) +} + +func (d *BunnyStorage) Get(ctx context.Context, path string) (model.Obj, error) { + fullPath := stdpath.Join(d.GetRootPath(), path) + parentPath, name := stdpath.Split(fullPath) + parentPath = strings.TrimSuffix(parentPath, "/") + if parentPath == "" { + parentPath = "/" + } + objs, err := d.List(ctx, &model.Object{Path: parentPath, IsFolder: true}, model.ListArgs{S3ShowPlaceholder: true}) + if err != nil { + return nil, err + } + for _, obj := range objs { + if obj.GetName() == name { + return obj, nil + } + } + return nil, errs.ObjectNotFound +} + +var _ driver.Driver = (*BunnyStorage)(nil) +var _ driver.Getter = (*BunnyStorage)(nil) diff --git a/drivers/bunny_storage/meta.go b/drivers/bunny_storage/meta.go new file mode 100644 index 000000000..46e135903 --- /dev/null +++ b/drivers/bunny_storage/meta.go @@ -0,0 +1,32 @@ +package bunny_storage + +import ( + "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/op" +) + +type Addition struct { + driver.RootPath + StorageZoneName string `json:"storage_zone_name" required:"true"` + AccessKey string `json:"access_key" required:"true"` + Endpoint string `json:"endpoint" required:"true" default:"storage.bunnycdn.com"` + CDNBaseURL string `json:"cdn_base_url"` + CDNTokenKey string `json:"cdn_token_key"` + CDNTokenMethod string `json:"cdn_token_method" type:"select" options:"sha256,hmac_sha256" default:"sha256"` + CDNTokenIncludeIP bool `json:"cdn_token_include_ip" default:"false"` + SignURLExpire int `json:"sign_url_expire" type:"number" default:"4"` + Placeholder string `json:"placeholder" default:".openlist"` +} + +var config = driver.Config{ + Name: "Bunny Storage", + LocalSort: true, + DefaultRoot: "/", + CheckStatus: true, +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &BunnyStorage{} + }) +} diff --git a/drivers/bunny_storage/types.go b/drivers/bunny_storage/types.go new file mode 100644 index 000000000..0a4d1016a --- /dev/null +++ b/drivers/bunny_storage/types.go @@ -0,0 +1,27 @@ +package bunny_storage + +import "time" + +type bunnyObject struct { + Guid string `json:"Guid"` + StorageZoneName string `json:"StorageZoneName"` + Path string `json:"Path"` + ObjectName string `json:"ObjectName"` + Length int64 `json:"Length"` + LastChanged string `json:"LastChanged"` + IsDirectory bool `json:"IsDirectory"` + ServerID int `json:"ServerId"` + UserID string `json:"UserId"` + DateCreated string `json:"DateCreated"` + StorageZoneID int64 `json:"StorageZoneId"` +} + +type apiError struct { + HttpCode int `json:"HttpCode"` + Message string `json:"Message"` +} + +type parsedTimes struct { + modified time.Time + created time.Time +} diff --git a/drivers/bunny_storage/util.go b/drivers/bunny_storage/util.go new file mode 100644 index 000000000..dd8b5da64 --- /dev/null +++ b/drivers/bunny_storage/util.go @@ -0,0 +1,275 @@ +package bunny_storage + +import ( + "crypto/hmac" + "crypto/sha256" + "encoding/base64" + "encoding/json" + "fmt" + "net/http" + "net/url" + stdpath "path" + "sort" + "strconv" + "strings" + "time" + + "github.com/OpenListTeam/OpenList/v4/internal/errs" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/go-resty/resty/v2" +) + +const ( + defaultEndpoint = "storage.bunnycdn.com" + defaultPlaceholder = ".openlist" + + cdnTokenMethodSHA256 = "sha256" + cdnTokenMethodHMACSHA256 = "hmac_sha256" +) + +func normalizeBaseURL(raw string, fallback string) (*url.URL, error) { + raw = strings.TrimSpace(raw) + if raw == "" { + raw = fallback + } + if raw == "" { + return nil, fmt.Errorf("empty url") + } + if !strings.Contains(raw, "://") { + raw = "https://" + raw + } + u, err := url.Parse(raw) + if err != nil { + return nil, err + } + if u.Host == "" { + return nil, fmt.Errorf("invalid url: %s", raw) + } + u.Path = strings.TrimRight(u.Path, "/") + return u, nil +} + +func cleanObjectPath(path string) string { + if path == "" { + return "/" + } + return stdpath.Clean("/" + strings.TrimPrefix(path, "/")) +} + +func stripObjectPathPrefix(path string, prefix string) (string, bool) { + path = cleanObjectPath(path) + prefix = cleanObjectPath(prefix) + if prefix == "/" { + return path, false + } + if path == prefix { + return "/", true + } + if strings.HasPrefix(path, prefix+"/") { + return cleanObjectPath(strings.TrimPrefix(path, prefix)), true + } + return path, false +} + +func isObjectPathOrChild(path string, parent string) bool { + path = cleanObjectPath(path) + parent = cleanObjectPath(parent) + return path == parent || strings.HasPrefix(path, parent+"/") +} + +func trimCDNBasePath(path string, mountPath string) string { + path = cleanObjectPath(path) + if path == "/" { + return "" + } + if stripped, ok := stripObjectPathPrefix(path, mountPath); ok { + path = stripped + } + if path == "/" { + return "" + } + return strings.TrimRight(path, "/") +} + +func (d *BunnyStorage) cdnObjectPath(path string) string { + objectPath := cleanObjectPath(path) + if stripped, ok := stripObjectPathPrefix(objectPath, d.GetStorage().MountPath); ok { + objectPath = stripped + } + rootPath := cleanObjectPath(d.GetRootPath()) + if rootPath != "/" && !isObjectPathOrChild(objectPath, rootPath) { + objectPath = cleanObjectPath(stdpath.Join(rootPath, objectPath)) + } + return objectPath +} + +func (d *BunnyStorage) placeholderName() string { + if d.Placeholder == "" { + return defaultPlaceholder + } + return d.Placeholder +} + +func (d *BunnyStorage) storageURL(path string, dir bool) string { + u := *d.endpoint + cleanPath := cleanObjectPath(path) + zone := strings.Trim(d.StorageZoneName, "/") + if cleanPath == "/" { + u.Path = "/" + zone + "/" + return u.String() + } + u.Path = "/" + zone + "/" + strings.TrimPrefix(cleanPath, "/") + if dir && !strings.HasSuffix(u.Path, "/") { + u.Path += "/" + } + return u.String() +} + +func (d *BunnyStorage) cdnURL(path string) string { + u := *d.cdnBase + cleanPath := cleanObjectPath(path) + basePath := trimCDNBasePath(u.Path, d.GetStorage().MountPath) + if cleanPath == "/" { + if basePath == "" { + u.Path = "/" + } else { + u.Path = basePath + "/" + } + return u.String() + } + u.Path = basePath + "/" + strings.TrimPrefix(cleanPath, "/") + return u.String() +} + +func (d *BunnyStorage) authRequest() *resty.Request { + return d.client.R().SetHeader("AccessKey", d.AccessKey) +} + +func (d *BunnyStorage) handleResponseError(resp *resty.Response) error { + if resp == nil { + return fmt.Errorf("empty response") + } + if resp.StatusCode() >= http.StatusOK && resp.StatusCode() < http.StatusMultipleChoices { + return nil + } + message := strings.TrimSpace(resp.String()) + var apiErrors []apiError + if err := json.Unmarshal(resp.Body(), &apiErrors); err == nil && len(apiErrors) > 0 && apiErrors[0].Message != "" { + message = apiErrors[0].Message + } + switch resp.StatusCode() { + case http.StatusUnauthorized, http.StatusForbidden: + return errs.NewErr(errs.PermissionDenied, "bunny storage request failed: %s", message) + case http.StatusNotFound: + return errs.NewErr(errs.ObjectNotFound, "bunny storage request failed: %s", message) + default: + return fmt.Errorf("bunny storage request failed: %s: %s", resp.Status(), message) + } +} + +func (d *BunnyStorage) parseTimes(item bunnyObject) parsedTimes { + return parsedTimes{ + modified: parseBunnyTime(item.LastChanged, d.Modified), + created: parseBunnyTime(item.DateCreated, time.Time{}), + } +} + +func parseBunnyTime(value string, fallback time.Time) time.Time { + if value == "" { + return fallback + } + if t, err := time.Parse(time.RFC3339Nano, value); err == nil { + return t + } + if t, err := time.Parse("2006-01-02T15:04:05.999999999", value); err == nil { + return t + } + if t, err := time.Parse("2006-01-02T15:04:05", value); err == nil { + return t + } + return fallback +} + +func (d *BunnyStorage) toObj(parentPath string, item bunnyObject) model.Obj { + times := d.parseTimes(item) + return &model.Object{ + ID: item.Guid, + Path: stdpath.Join(parentPath, item.ObjectName), + Name: item.ObjectName, + Size: item.Length, + Modified: times.modified, + Ctime: times.created, + IsFolder: item.IsDirectory, + } +} + +func canonicalQuery(values url.Values) (string, error) { + keys := make([]string, 0, len(values)) + for key := range values { + if key == "token" || key == "expires" { + continue + } + keys = append(keys, key) + } + sort.Strings(keys) + parts := make([]string, 0, len(keys)) + for _, key := range keys { + vals := values[key] + if len(vals) > 1 { + return "", fmt.Errorf("duplicate query parameter %q is not supported", key) + } + value := "" + if len(vals) == 1 { + value = vals[0] + } + parts = append(parts, key+"="+value) + } + return strings.Join(parts, "&"), nil +} + +func (d *BunnyStorage) signCDNURL(rawURL string, clientIP string) (string, time.Duration, error) { + return d.signCDNURLAt(rawURL, clientIP, time.Now()) +} + +func (d *BunnyStorage) signCDNURLAt(rawURL string, clientIP string, now time.Time) (string, time.Duration, error) { + expire := time.Hour * time.Duration(d.SignURLExpire) + if expire <= 0 { + expire = 4 * time.Hour + } + expires := now.Add(expire).Unix() + u, err := url.Parse(rawURL) + if err != nil { + return "", 0, err + } + query := u.Query() + parameterData, err := canonicalQuery(query) + if err != nil { + return "", 0, err + } + signaturePath, err := url.PathUnescape(u.EscapedPath()) + if err != nil { + signaturePath = u.Path + } + if !d.CDNTokenIncludeIP { + clientIP = "" + } + token := d.signCDNToken(signaturePath, strconv.FormatInt(expires, 10), parameterData, clientIP) + query.Set("token", token) + query.Set("expires", strconv.FormatInt(expires, 10)) + u.RawQuery = query.Encode() + return u.String(), expire, nil +} + +func (d *BunnyStorage) signCDNToken(signaturePath string, expires string, parameterData string, clientIP string) string { + switch strings.ToLower(strings.TrimSpace(d.CDNTokenMethod)) { + case cdnTokenMethodHMACSHA256: + message := signaturePath + expires + parameterData + clientIP + mac := hmac.New(sha256.New, []byte(d.CDNTokenKey)) + _, _ = mac.Write([]byte(message)) + return "HS256-" + base64.RawURLEncoding.EncodeToString(mac.Sum(nil)) + default: + hashableBase := d.CDNTokenKey + signaturePath + expires + parameterData + clientIP + sum := sha256.Sum256([]byte(hashableBase)) + return base64.RawURLEncoding.EncodeToString(sum[:]) + } +} diff --git a/drivers/bunny_storage/util_test.go b/drivers/bunny_storage/util_test.go new file mode 100644 index 000000000..ba9fe0b72 --- /dev/null +++ b/drivers/bunny_storage/util_test.go @@ -0,0 +1,255 @@ +package bunny_storage + +import ( + "context" + "net/url" + "testing" + "time" + + "github.com/OpenListTeam/OpenList/v4/internal/model" +) + +func TestStorageURL(t *testing.T) { + endpoint, err := normalizeBaseURL("ny.storage.bunnycdn.com", defaultEndpoint) + if err != nil { + t.Fatal(err) + } + driver := &BunnyStorage{ + Addition: Addition{ + StorageZoneName: "my-zone", + }, + endpoint: endpoint, + } + if got, want := driver.storageURL("/", true), "https://ny.storage.bunnycdn.com/my-zone/"; got != want { + t.Fatalf("root list url = %q, want %q", got, want) + } + if got, want := driver.storageURL("/dir/a file.txt", false), "https://ny.storage.bunnycdn.com/my-zone/dir/a%20file.txt"; got != want { + t.Fatalf("file url = %q, want %q", got, want) + } + if got, want := driver.storageURL("/dir", true), "https://ny.storage.bunnycdn.com/my-zone/dir/"; got != want { + t.Fatalf("dir url = %q, want %q", got, want) + } +} + +func TestCDNURLWithBasePath(t *testing.T) { + cdnBase, err := normalizeBaseURL("https://cdn.example.com/prefix/", "") + if err != nil { + t.Fatal(err) + } + driver := &BunnyStorage{cdnBase: cdnBase} + if got, want := driver.cdnURL("/dir/a file.txt"), "https://cdn.example.com/prefix/dir/a%20file.txt"; got != want { + t.Fatalf("cdn url = %q, want %q", got, want) + } +} + +func TestCDNURLUsesObjectPathWithoutMountPath(t *testing.T) { + cdnBase, err := normalizeBaseURL("https://cdn.firmant.me", "") + if err != nil { + t.Fatal(err) + } + driver := &BunnyStorage{ + Storage: model.Storage{MountPath: "/BS"}, + cdnBase: cdnBase, + } + if got, want := driver.cdnURL(driver.cdnObjectPath("/BS/Video")), "https://cdn.firmant.me/Video"; got != want { + t.Fatalf("cdn url = %q, want %q", got, want) + } +} + +func TestCDNURLDropsMountPathFromBaseURL(t *testing.T) { + cdnBase, err := normalizeBaseURL("https://cdn.firmant.me/BS/", "") + if err != nil { + t.Fatal(err) + } + driver := &BunnyStorage{ + Storage: model.Storage{MountPath: "/BS"}, + cdnBase: cdnBase, + } + if got, want := driver.cdnURL(driver.cdnObjectPath("/Video")), "https://cdn.firmant.me/Video"; got != want { + t.Fatalf("cdn url = %q, want %q", got, want) + } +} + +func TestCDNObjectPathKeepsRootFolderPath(t *testing.T) { + driver := &BunnyStorage{} + driver.MountPath = "/BS" + driver.RootFolderPath = "/library" + if got, want := driver.cdnObjectPath("/BS/Video"), "/library/Video"; got != want { + t.Fatalf("cdn object path = %q, want %q", got, want) + } + if got, want := driver.cdnObjectPath("/library/Video"), "/library/Video"; got != want { + t.Fatalf("cdn object path = %q, want %q", got, want) + } +} + +func TestLinkDisablesLongLivedCache(t *testing.T) { + cdnBase, err := normalizeBaseURL("https://cdn.example.com", "") + if err != nil { + t.Fatal(err) + } + driver := &BunnyStorage{cdnBase: cdnBase} + link, err := driver.Link(context.Background(), &model.Object{ + Path: "/video.mp4", + Name: "video.mp4", + Size: 123, + }, model.LinkArgs{}) + if err != nil { + t.Fatal(err) + } + if link.Expiration == nil || *link.Expiration != 0 { + t.Fatalf("link expiration = %v, want immediate cache expiration", link.Expiration) + } +} + +func TestSignCDNURL(t *testing.T) { + driver := &BunnyStorage{ + Addition: Addition{ + CDNTokenKey: "secret", + CDNTokenIncludeIP: true, + SignURLExpire: 1, + }, + } + signed, expire, err := driver.signCDNURLAt("https://zone.b-cdn.net/video.mp4?quality=high", "192.0.2.1", time.Unix(1700000000, 0)) + if err != nil { + t.Fatal(err) + } + if expire <= 0 { + t.Fatal("expected positive expiration") + } + parsed, err := url.Parse(signed) + if err != nil { + t.Fatal(err) + } + token := parsed.Query().Get("token") + if want := "FxSpFem88zFo6uHFziwTuoMQTgDaD2PEn5n1zTMBUBI"; token != want { + t.Fatalf("token = %q, want %q", token, want) + } + if parsed.Query().Get("expires") != "1700003600" { + t.Fatalf("expires = %q, want 1700003600", parsed.Query().Get("expires")) + } + if parsed.Query().Get("quality") != "high" { + t.Fatal("expected existing query parameters to be preserved") + } +} + +func TestSignCDNURLSupportsHMACSHA256(t *testing.T) { + driver := &BunnyStorage{ + Addition: Addition{ + CDNTokenKey: "secret", + CDNTokenMethod: cdnTokenMethodHMACSHA256, + CDNTokenIncludeIP: true, + SignURLExpire: 1, + }, + } + signed, _, err := driver.signCDNURLAt("https://zone.b-cdn.net/video.mp4?quality=high", "192.0.2.1", time.Unix(1700000000, 0)) + if err != nil { + t.Fatal(err) + } + parsed, err := url.Parse(signed) + if err != nil { + t.Fatal(err) + } + if got, want := parsed.Query().Get("token"), "HS256-sdrSSJE2JVwhSk2AoDUrmTV1muH6R5UHpZVcVfHeNxg"; got != want { + t.Fatalf("token = %q, want %q", got, want) + } +} + +func TestSignCDNURLUsesDecodedPathForSHA256(t *testing.T) { + driver := &BunnyStorage{ + Addition: Addition{ + CDNTokenKey: "secret", + SignURLExpire: 1, + }, + } + signed, _, err := driver.signCDNURLAt("https://zone.b-cdn.net/%E8%A7%86%E9%A2%91/%5Ba%20b%5D.mp4", "", time.Unix(1700000000, 0)) + if err != nil { + t.Fatal(err) + } + parsed, err := url.Parse(signed) + if err != nil { + t.Fatal(err) + } + if got, want := parsed.Query().Get("token"), "yq1evD7klw0e3DjCbv8dJptbW4S4JwVW3GKLnxfeKGM"; got != want { + t.Fatalf("token = %q, want %q", got, want) + } +} + +func TestSignCDNURLTreatsPlusAsLiteralPathCharacter(t *testing.T) { + driver := &BunnyStorage{ + Addition: Addition{ + CDNTokenKey: "secret", + SignURLExpire: 1, + }, + } + now := time.Unix(1700000000, 0) + literal, _, err := driver.signCDNURLAt("https://zone.b-cdn.net/a+b.mp4", "", now) + if err != nil { + t.Fatal(err) + } + encoded, _, err := driver.signCDNURLAt("https://zone.b-cdn.net/a%2Bb.mp4", "", now) + if err != nil { + t.Fatal(err) + } + space, _, err := driver.signCDNURLAt("https://zone.b-cdn.net/a%20b.mp4", "", now) + if err != nil { + t.Fatal(err) + } + literalURL, err := url.Parse(literal) + if err != nil { + t.Fatal(err) + } + encodedURL, err := url.Parse(encoded) + if err != nil { + t.Fatal(err) + } + spaceURL, err := url.Parse(space) + if err != nil { + t.Fatal(err) + } + if got, want := literalURL.Query().Get("token"), encodedURL.Query().Get("token"); got != want { + t.Fatalf("literal plus token = %q, encoded plus token = %q", got, want) + } + if got, notWant := literalURL.Query().Get("token"), spaceURL.Query().Get("token"); got == notWant { + t.Fatalf("literal plus token = %q, space token should differ", got) + } +} + +func TestSignCDNURLRejectsDuplicateQueryParameters(t *testing.T) { + driver := &BunnyStorage{ + Addition: Addition{ + CDNTokenKey: "secret", + SignURLExpire: 1, + }, + } + _, _, err := driver.signCDNURLAt("https://zone.b-cdn.net/video.mp4?quality=high&quality=low", "", time.Unix(1700000000, 0)) + if err == nil { + t.Fatal("expected duplicate query parameters to be rejected") + } + if got, want := err.Error(), `duplicate query parameter "quality" is not supported`; got != want { + t.Fatalf("error = %q, want %q", got, want) + } +} + +func TestParseBunnyTimeSupportsFractionalSecondsWithoutTimezone(t *testing.T) { + fallback := time.Unix(1, 0) + got := parseBunnyTime("2023-03-21T13:38:31.693", fallback) + want := time.Date(2023, 3, 21, 13, 38, 31, 693000000, time.UTC) + if !got.Equal(want) { + t.Fatalf("parsed time = %s, want %s", got.Format(time.RFC3339Nano), want.Format(time.RFC3339Nano)) + } +} + +func TestConfigProxyMode(t *testing.T) { + registeredConfig := (&BunnyStorage{}).Config() + if registeredConfig.OnlyProxy { + t.Fatal("driver registration config should allow users to choose proxy policy") + } + withoutCDN := (&BunnyStorage{Addition: Addition{StorageZoneName: "my-zone"}}).Config() + if !withoutCDN.OnlyProxy { + t.Fatal("storage API links require AccessKey headers and should be proxied without CDN") + } + withCDN := (&BunnyStorage{Addition: Addition{StorageZoneName: "my-zone", CDNBaseURL: "https://zone.b-cdn.net"}}).Config() + if withCDN.OnlyProxy { + t.Fatal("CDN links should be allowed to redirect directly") + } +} diff --git a/drivers/chaoxing/driver.go b/drivers/chaoxing/driver.go index dfd25d195..54345ff12 100644 --- a/drivers/chaoxing/driver.go +++ b/drivers/chaoxing/driver.go @@ -55,13 +55,13 @@ func (d *ChaoXing) refreshCookie() error { func (d *ChaoXing) Init(ctx context.Context) error { err := d.refreshCookie() if err != nil { - log.Errorf(ctx, err.Error()) + log.Errorf(ctx, "%s", err.Error()) } d.cron = cron.NewCron(time.Hour * 12) d.cron.Do(func() { err = d.refreshCookie() if err != nil { - log.Errorf(ctx, err.Error()) + log.Errorf(ctx, "%s", err.Error()) } }) return nil diff --git a/drivers/crypt/driver.go b/drivers/crypt/driver.go index 9e8b5c5c7..7dca84283 100644 --- a/drivers/crypt/driver.go +++ b/drivers/crypt/driver.go @@ -251,7 +251,7 @@ func (d *Crypt) Link(ctx context.Context, file model.Obj, _ model.LinkArgs) (*mo rrf, err := stream.GetRangeReaderFromLink(remoteSize, remoteLink) if err != nil { _ = remoteLink.Close() - return nil, fmt.Errorf("the remote storage driver need to be enhanced to support encrytion") + return nil, fmt.Errorf("the remote storage driver need to be enhanced to support encryption") } mu := &sync.Mutex{} diff --git a/drivers/github_releases/driver.go b/drivers/github_releases/driver.go index 92d94b4dc..e19f7998a 100644 --- a/drivers/github_releases/driver.go +++ b/drivers/github_releases/driver.go @@ -45,134 +45,126 @@ func (d *GithubReleases) List(ctx context.Context, dir model.Obj, args model.Lis for i := range d.points { point := &d.points[i] - if !d.Addition.ShowAllVersion { // latest - err := point.RequestRelease(d.GetRequest, args.Refresh) + if !d.Addition.ShowAllVersion { + // latest version mode + release, err := d.getLatestRelease(point.Repo) if err != nil { log.Warnf("failed to request release for %s: %v", point.Repo, err) + continue + } + if release == nil { + continue } - if point.Point == path { // 与仓库路径相同 - if point.Release == nil { - if err != nil { - return nil, fmt.Errorf("failed to get release for %s: %w", point.Repo, err) - } - return nil, fmt.Errorf("failed to get release for %s: unknown error", point.Repo) - } - files = append(files, point.GetLatestRelease()...) + if point.Point == path { + // 当前目录就是仓库挂载点 + files = append(files, releaseToFiles(point.Point, release)...) if d.Addition.ShowReadme { - otherFiles, err := point.GetOtherFile(d.GetRequest, args.Refresh) - if err != nil { - return nil, fmt.Errorf("failed to get other files for %s: %w", point.Repo, err) + other, err := d.fetchRepoFiles(point.Repo) + if err == nil { + files = append(files, otherFiles(point.Point, other)...) + } else { + log.Warnf("failed to get other files for %s: %v", point.Repo, err) } - files = append(files, otherFiles...) } if d.Addition.ShowSourceCode { - files = append(files, point.GetSourceCode()...) + files = append(files, sourceCodeFiles(point.Point, release)...) } - } else if strings.HasPrefix(point.Point, path) { // 仓库目录的父目录 + } else if strings.HasPrefix(point.Point, path) { + // 仓库目录的父目录，需要聚合显示 nextDir := GetNextDir(point.Point, path) if nextDir == "" { continue } - if err != nil { - return nil, fmt.Errorf("failed to get release for %s: %w", point.Repo, err) - } hasSameDir := false for index := range files { if files[index].GetName() == nextDir { hasSameDir = true - files[index].Size += point.GetLatestSize() + files[index].Size += releaseSize(release) break } } if !hasSameDir { - var updateAt, createAt string - if point.Release != nil { - updateAt = point.Release.PublishedAt - createAt = point.Release.CreatedAt - } files = append(files, File{ Path: stdpath.Join(path, nextDir), FileName: nextDir, - Size: point.GetLatestSize(), - UpdateAt: updateAt, - CreateAt: createAt, + Size: releaseSize(release), + UpdateAt: release.PublishedAt, + CreateAt: release.CreatedAt, Type: "dir", Url: "", }) } } - } else { // all version - err := point.RequestReleases(d.GetRequest, args.Refresh) + } else { + // all versions mode + releases, err := d.getAllReleases(point.Repo) if err != nil { log.Warnf("failed to request releases for %s: %v", point.Repo, err) + continue } - - if point.Point == path { // 与仓库路径相同 - if point.Releases == nil { - if err != nil { - return nil, fmt.Errorf("failed to get releases for %s: %w", point.Repo, err) + if len(releases) == 0 { + // no releases but may still have repo files (e.g. README) + if point.Point == path && d.Addition.ShowReadme { + other, err := d.fetchRepoFiles(point.Repo) + if err == nil { + files = append(files, otherFiles(point.Point, other)...) + } else { + log.Warnf("failed to get other files for %s: %v", point.Repo, err) } - return nil, fmt.Errorf("failed to get releases for %s: unknown error", point.Repo) } - files = append(files, point.GetAllVersion()...) + continue + } + + if point.Point == path { + // 当前目录就是仓库挂载点 + files = append(files, releasesToVersionDirs(point.Point, releases)...) if d.Addition.ShowReadme { - otherFiles, err := point.GetOtherFile(d.GetRequest, args.Refresh) - if err != nil { - return nil, fmt.Errorf("failed to get other files for %s: %w", point.Repo, err) + other, err := d.fetchRepoFiles(point.Repo) + if err == nil { + files = append(files, otherFiles(point.Point, other)...) + } else { + log.Warnf("failed to get other files for %s: %v", point.Repo, err) } - files = append(files, otherFiles...) } - } else if strings.HasPrefix(point.Point, path) { // 仓库目录的父目录 + } else if strings.HasPrefix(point.Point, path) { + // 仓库目录的父目录 nextDir := GetNextDir(point.Point, path) if nextDir == "" { continue } - if err != nil { - return nil, fmt.Errorf("failed to get releases for %s: %w", point.Repo, err) - } hasSameDir := false for index := range files { if files[index].GetName() == nextDir { hasSameDir = true - files[index].Size += point.GetAllVersionSize() + files[index].Size += releasesTotalSize(releases) break } } if !hasSameDir { - var updateAt, createAt string - if point.Releases != nil && len(*point.Releases) > 0 { - updateAt = (*point.Releases)[0].PublishedAt - createAt = (*point.Releases)[0].CreatedAt - } files = append(files, File{ FileName: nextDir, Path: stdpath.Join(path, nextDir), - Size: point.GetAllVersionSize(), - UpdateAt: updateAt, - CreateAt: createAt, + Size: releasesTotalSize(releases), + UpdateAt: releases[0].PublishedAt, + CreateAt: releases[0].CreatedAt, Type: "dir", Url: "", }) } - } else if strings.HasPrefix(path, point.Point) { // 仓库目录的子目录 + } else if strings.HasPrefix(path, point.Point) { + // 仓库目录的子目录（某个版本） tagName := GetNextDir(path, point.Point) if tagName == "" { continue } - if point.Releases == nil { - if err != nil { - return nil, fmt.Errorf("failed to get releases for %s: %w", point.Repo, err) - } - return nil, fmt.Errorf("failed to get releases for %s: unknown error", point.Repo) - } - files = append(files, point.GetReleaseByTagName(tagName)...) + files = append(files, releaseAssetsByTag(point.Point, tagName, releases)...) if d.Addition.ShowSourceCode { - files = append(files, point.GetSourceCodeByTagName(tagName)...) + files = append(files, sourceCodeFilesByTag(point.Point, releases, tagName)...) } } } diff --git a/drivers/github_releases/meta.go b/drivers/github_releases/meta.go index fc5fad695..36f8b3cf8 100644 --- a/drivers/github_releases/meta.go +++ b/drivers/github_releases/meta.go @@ -12,7 +12,9 @@ type Addition struct { Token string `json:"token" type:"string" required:"false" help:"GitHub token, if you want to access private repositories or increase the rate limit"` ShowSourceCode bool `json:"show_source_code" type:"bool" default:"false" help:"show Source code (zip/tar.gz)"` ShowAllVersion bool `json:"show_all_version" type:"bool" default:"false" help:"show all versions"` - GitHubProxy string `json:"gh_proxy" type:"string" default:"" help:"GitHub proxy, e.g. https://ghproxy.net/github.com or https://gh-proxy.com/github.com "` + PerPage int `json:"per_page" type:"number" default:"30" help:"releases per page (max 100), only works when show all versions"` + MaxPage int `json:"max_page" type:"number" default:"0" help:"max pages to fetch (0 = unlimited), only works when show all versions"` + GitHubProxy string `json:"gh_proxy" type:"string" default:"" help:"GitHub proxy, e.g. https://ghproxy.net/https://github.com or https://gh-proxy.com/https://github.com"` } var config = driver.Config{ diff --git a/drivers/github_releases/types.go b/drivers/github_releases/types.go index 38db702ac..3d418f517 100644 --- a/drivers/github_releases/types.go +++ b/drivers/github_releases/types.go @@ -1,76 +1,28 @@ package github_releases import ( - "encoding/json" "path" "strings" "time" "github.com/OpenListTeam/OpenList/v4/pkg/utils" - "github.com/go-resty/resty/v2" ) +// MountPoint 表示一个仓库挂载点 type MountPoint struct { - Point string // 挂载点 - Repo string // 仓库名 owner/repo - Release *Release // Release 指针 latest - Releases *[]Release // []Release 指针 - OtherFile *[]FileInfo // 仓库根目录下的其他文件 + Point string // 挂载点路径 + Repo string // 仓库名 owner/repo } -// 请求最新版本 -func (m *MountPoint) RequestRelease(get func(url string) (*resty.Response, error), refresh bool) error { - if m.Repo == "" { +// Release 转为 File 列表 +func releaseToFiles(point string, release *Release) []File { + if release == nil { return nil } - - if m.Release == nil || refresh { - resp, err := get("https://api.github.com/repos/" + m.Repo + "/releases/latest") - if err != nil { - m.Release = nil - return err - } - release := new(Release) - if err := json.Unmarshal(resp.Body(), release); err != nil { - m.Release = nil - return err - } - m.Release = release - } - return nil -} - -// 请求所有版本 -func (m *MountPoint) RequestReleases(get func(url string) (*resty.Response, error), refresh bool) error { - if m.Repo == "" { - return nil - } - - if m.Releases == nil || refresh { - resp, err := get("https://api.github.com/repos/" + m.Repo + "/releases") - if err != nil { - m.Releases = nil - return err - } - releases := new([]Release) - if err := json.Unmarshal(resp.Body(), releases); err != nil { - m.Releases = nil - return err - } - m.Releases = releases - } - return nil -} - -// 获取最新版本 -func (m *MountPoint) GetLatestRelease() []File { - if m.Release == nil { - return nil - } - files := make([]File, 0, len(m.Release.Assets)) - for _, asset := range m.Release.Assets { + files := make([]File, 0, len(release.Assets)) + for _, asset := range release.Assets { files = append(files, File{ - Path: path.Join(m.Point, asset.Name), + Path: path.Join(point, asset.Name), FileName: asset.Name, Size: asset.Size, Type: "file", @@ -82,53 +34,43 @@ func (m *MountPoint) GetLatestRelease() []File { return files } -// 获取最新版本大小 -func (m *MountPoint) GetLatestSize() int64 { - if m.Release == nil { +// 计算 release 的 asset 总大小 +func releaseSize(release *Release) int64 { + if release == nil { return 0 } size := int64(0) - for _, asset := range m.Release.Assets { + for _, asset := range release.Assets { size += asset.Size } return size } -// 获取所有版本 -func (m *MountPoint) GetAllVersion() []File { - if m.Releases == nil { - return nil - } - files := make([]File, 0) - for _, release := range *m.Releases { - file := File{ - Path: path.Join(m.Point, release.TagName), +// Releases 列表转为版本目录 File 列表 +func releasesToVersionDirs(point string, releases []Release) []File { + files := make([]File, 0, len(releases)) + for _, release := range releases { + files = append(files, File{ + Path: path.Join(point, release.TagName), FileName: release.TagName, - Size: m.GetSizeByTagName(release.TagName), + Size: releaseSize(&release), Type: "dir", UpdateAt: release.PublishedAt, CreateAt: release.CreatedAt, Url: release.HtmlUrl, - } - for _, asset := range release.Assets { - file.Size += asset.Size - } - files = append(files, file) + }) } return files } -// 根据版本号获取版本 -func (m *MountPoint) GetReleaseByTagName(tagName string) []File { - if m.Releases == nil { - return nil - } - for _, item := range *m.Releases { +// 根据 tagName 查找 release 的 asset 文件列表 +func releaseAssetsByTag(point, tagName string, releases []Release) []File { + for _, item := range releases { if item.TagName == tagName { - files := make([]File, 0) + files := make([]File, 0, len(item.Assets)) for _, asset := range item.Assets { files = append(files, File{ - Path: path.Join(m.Point, tagName, asset.Name), + Path: path.Join(point, tagName, asset.Name), FileName: asset.Name, Size: asset.Size, Type: "file", @@ -143,118 +85,74 @@ func (m *MountPoint) GetReleaseByTagName(tagName string) []File { return nil } -// 根据版本号获取版本大小 -func (m *MountPoint) GetSizeByTagName(tagName string) int64 { - if m.Releases == nil { - return 0 - } - for _, item := range *m.Releases { +// 根据 tagName 计算 asset 总大小 +func releasesSizeByTag(releases []Release, tagName string) int64 { + for _, item := range releases { if item.TagName == tagName { - size := int64(0) - for _, asset := range item.Assets { - size += asset.Size - } - return size + return releaseSize(&item) } } return 0 } -// 获取所有版本大小 -func (m *MountPoint) GetAllVersionSize() int64 { - if m.Releases == nil { - return 0 - } +// 计算所有 releases 的 asset 总大小 +func releasesTotalSize(releases []Release) int64 { size := int64(0) - for _, release := range *m.Releases { - for _, asset := range release.Assets { - size += asset.Size - } + for _, release := range releases { + size += releaseSize(&release) } return size } -func (m *MountPoint) GetSourceCode() []File { - if m.Release == nil { +// Source code 文件 +func sourceCodeFiles(point string, release *Release) []File { + if release == nil { return nil } - files := make([]File, 0) - - // 无法获取文件大小，此处设为 1 - files = append(files, File{ - Path: path.Join(m.Point, "Source code (zip)"), - FileName: "Source code (zip)", - Size: 1, - Type: "file", - UpdateAt: m.Release.CreatedAt, - CreateAt: m.Release.CreatedAt, - Url: m.Release.ZipballUrl, - }) - files = append(files, File{ - Path: path.Join(m.Point, "Source code (tar.gz)"), - FileName: "Source code (tar.gz)", - Size: 1, - Type: "file", - UpdateAt: m.Release.CreatedAt, - CreateAt: m.Release.CreatedAt, - Url: m.Release.TarballUrl, - }) - - return files + return []File{ + { + Path: path.Join(point, "Source code (zip)"), + FileName: "Source code (zip)", + Size: 1, + Type: "file", + UpdateAt: release.CreatedAt, + CreateAt: release.CreatedAt, + Url: release.ZipballUrl, + }, + { + Path: path.Join(point, "Source code (tar.gz)"), + FileName: "Source code (tar.gz)", + Size: 1, + Type: "file", + UpdateAt: release.CreatedAt, + CreateAt: release.CreatedAt, + Url: release.TarballUrl, + }, + } } -func (m *MountPoint) GetSourceCodeByTagName(tagName string) []File { - if m.Releases == nil { - return nil - } - for _, item := range *m.Releases { +// 根据 tagName 获取 Source Code 文件 +func sourceCodeFilesByTag(point string, releases []Release, tagName string) []File { + for _, item := range releases { if item.TagName == tagName { - files := make([]File, 0) - files = append(files, File{ - Path: path.Join(m.Point, "Source code (zip)"), - FileName: "Source code (zip)", - Size: 1, - Type: "file", - UpdateAt: item.CreatedAt, - CreateAt: item.CreatedAt, - Url: item.ZipballUrl, - }) - files = append(files, File{ - Path: path.Join(m.Point, "Source code (tar.gz)"), - FileName: "Source code (tar.gz)", - Size: 1, - Type: "file", - UpdateAt: item.CreatedAt, - CreateAt: item.CreatedAt, - Url: item.TarballUrl, - }) - return files + return sourceCodeFiles(point, &item) } } return nil } -func (m *MountPoint) GetOtherFile(get func(url string) (*resty.Response, error), refresh bool) ([]File, error) { - if m.OtherFile == nil || refresh { - resp, err := get("https://api.github.com/repos/" + m.Repo + "/contents") - if err != nil { - m.OtherFile = nil - return nil, err - } - otherFile := new([]FileInfo) - if err := json.Unmarshal(resp.Body(), otherFile); err != nil { - m.OtherFile = nil - return nil, err - } - m.OtherFile = otherFile - } - +// 仓库根目录下的 README/LICENSE 文件 +func otherFiles(point string, fileInfos []FileInfo) []File { files := make([]File, 0) defaultTime := "1970-01-01T00:00:00Z" - for _, file := range *m.OtherFile { - if strings.HasSuffix(file.Name, ".md") || strings.HasPrefix(file.Name, "LICENSE") { + for _, file := range fileInfos { + if file.Type == "dir" { + continue + } + name := file.Name + if strings.EqualFold(name, "README.md") || strings.HasPrefix(name, "LICENSE") { files = append(files, File{ - Path: path.Join(m.Point, file.Name), + Path: path.Join(point, file.Name), FileName: file.Name, Size: file.Size, Type: "file", @@ -264,7 +162,7 @@ func (m *MountPoint) GetOtherFile(get func(url string) (*resty.Response, error), }) } } - return files, nil + return files } type File struct { @@ -289,10 +187,6 @@ func (f File) GetSize() int64 { return f.Size } -func (f File) GetDuration() int { - return 0 -} - func (f File) GetName() string { return f.FileName } @@ -314,3 +208,7 @@ func (f File) IsDir() bool { func (f File) GetID() string { return f.Url } + +func (f File) GetDuration() int { + return 0 +} diff --git a/drivers/github_releases/util.go b/drivers/github_releases/util.go index a6401e6cd..cbaafb08b 100644 --- a/drivers/github_releases/util.go +++ b/drivers/github_releases/util.go @@ -1,8 +1,8 @@ package github_releases import ( + "encoding/json" "fmt" - "path/filepath" "strings" "github.com/OpenListTeam/OpenList/v4/drivers/base" @@ -49,10 +49,8 @@ func (d *GithubReleases) ParseRepos(text string) ([]MountPoint, error) { } points = append(points, MountPoint{ - Point: path, - Repo: repo, - Release: nil, - Releases: nil, + Point: path, + Repo: repo, }) } d.points = points @@ -76,9 +74,79 @@ func GetNextDir(wholePath string, basePath string) string { return "" } -// 判断当前目录是否是目标目录的祖先目录 -func IsAncestorDir(parentDir string, targetDir string) bool { - absTargetDir, _ := filepath.Abs(targetDir) - absParentDir, _ := filepath.Abs(parentDir) - return strings.HasPrefix(absTargetDir, absParentDir) +// getLatestRelease 获取最新 release +func (d *GithubReleases) getLatestRelease(repo string) (*Release, error) { + resp, err := d.GetRequest("https://api.github.com/repos/" + repo + "/releases/latest") + if err != nil { + return nil, err + } + release := new(Release) + if err := json.Unmarshal(resp.Body(), release); err != nil { + return nil, err + } + return release, nil +} + +// getAllReleases 获取所有 releases（支持自动翻页） +func (d *GithubReleases) getAllReleases(repo string) ([]Release, error) { + perPage := d.Addition.PerPage + if perPage < 1 { + perPage = 30 + } else if perPage > 100 { + perPage = 100 + } + + maxPage := d.Addition.MaxPage + if maxPage < 0 { + maxPage = 0 + } + + allReleases := make([]Release, 0) + page := 1 + + for { + url := fmt.Sprintf("https://api.github.com/repos/%s/releases?per_page=%d&page=%d", repo, perPage, page) + resp, err := d.GetRequest(url) + if err != nil { + return nil, err + } + + releases := make([]Release, 0) + if err := json.Unmarshal(resp.Body(), &releases); err != nil { + return nil, err + } + + if len(releases) == 0 { + break + } + + allReleases = append(allReleases, releases...) + + // 达到最大页数限制 + if maxPage > 0 && page >= maxPage { + break + } + + // 如果返回数量小于 perPage，说明是最后一页 + if len(releases) < perPage { + break + } + + page++ + } + + return allReleases, nil +} + +// fetchRepoFiles 获取仓库根目录文件列表 +func (d *GithubReleases) fetchRepoFiles(repo string) ([]FileInfo, error) { + resp, err := d.GetRequest("https://api.github.com/repos/" + repo + "/contents") + if err != nil { + return nil, err + } + files := make([]FileInfo, 0) + if err := json.Unmarshal(resp.Body(), &files); err != nil { + return nil, err + } + return files, nil } diff --git a/drivers/google_drive/util.go b/drivers/google_drive/util.go index 042abafa4..e4f5f1a07 100644 --- a/drivers/google_drive/util.go +++ b/drivers/google_drive/util.go @@ -170,7 +170,7 @@ func (d *GoogleDrive) refreshToken() error { } log.Debug(res.String()) if e.Error != "" { - return fmt.Errorf(e.Error) + return fmt.Errorf("%s", e.Error) } d.AccessToken = resp.AccessToken return nil @@ -192,7 +192,7 @@ func (d *GoogleDrive) refreshToken() error { } log.Debug(res.String()) if e.Error != "" { - return fmt.Errorf(e.Error) + return fmt.Errorf("%s", e.Error) } d.AccessToken = resp.AccessToken return nil diff --git a/drivers/google_photo/util.go b/drivers/google_photo/util.go index 3a9b66ab2..81d05b492 100644 --- a/drivers/google_photo/util.go +++ b/drivers/google_photo/util.go @@ -32,7 +32,7 @@ func (d *GooglePhoto) refreshToken() error { return err } if e.Error != "" { - return fmt.Errorf(e.Error) + return fmt.Errorf("%s", e.Error) } d.AccessToken = resp.AccessToken return nil diff --git a/drivers/lanzou/help.go b/drivers/lanzou/help.go index 930c72486..f2599b283 100644 --- a/drivers/lanzou/help.go +++ b/drivers/lanzou/help.go @@ -304,7 +304,7 @@ var findFromReg = regexp.MustCompile(`data : '(.+?)'`) // 查找from字符串 func htmlFormToMap(html string) (map[string]string, error) { forms := findFromReg.FindStringSubmatch(html) if len(forms) != 2 { - return nil, fmt.Errorf("not find file sgin") + return nil, fmt.Errorf("not find file sign") } return formToMap(forms[1]), nil } diff --git a/drivers/lanzou/types.go b/drivers/lanzou/types.go index 27763eb36..4f591b6f9 100644 --- a/drivers/lanzou/types.go +++ b/drivers/lanzou/types.go @@ -83,8 +83,7 @@ func (f *FileOrFolder) GetSize() int64 { } return *f.size } -func (f *FileOrFolder) GetDuration() int { return 0 } -func (f *FileOrFolder) IsDir() bool { return f.FolID != "" } +func (f *FileOrFolder) IsDir() bool { return f.FolID != "" } func (f *FileOrFolder) ModTime() time.Time { if f.time == nil { time := MustParseTime(f.Time) @@ -133,7 +132,7 @@ type FileOrFolderByShareUrl struct { //T int `json:"t"` // 文件夹特有 - IsFloder bool `json:"-"` + IsFolder bool `json:"-"` // Url string `json:"-"` @@ -163,8 +162,9 @@ func (f *FileOrFolderByShareUrl) GetSize() int64 { } return *f.size } +func (f *FileOrFolder) GetDuration() int { return 0 } func (f *FileOrFolderByShareUrl) GetDuration() int { return 0 } -func (f *FileOrFolderByShareUrl) IsDir() bool { return f.IsFloder } +func (f *FileOrFolderByShareUrl) IsDir() bool { return f.IsFolder } func (f *FileOrFolderByShareUrl) ModTime() time.Time { if f.time == nil { time := MustParseTime(f.Time) diff --git a/drivers/lanzou/util.go b/drivers/lanzou/util.go index e70154402..f20f5ae0b 100644 --- a/drivers/lanzou/util.go +++ b/drivers/lanzou/util.go @@ -90,7 +90,7 @@ func (d *LanZou) _post(url string, callback base.ReqCallback, resp interface{}, if info == "" { info = utils.Json.Get(data, "info").ToString() } - return data, fmt.Errorf(info) + return data, fmt.Errorf("%s", info) } } @@ -157,25 +157,43 @@ func (d *LanZou) request(url string, method string, callback base.ReqCallback, u } func (d *LanZou) Login() ([]*http.Cookie, error) { - resp, err := base.NewRestyClient().SetRedirectPolicy(resty.NoRedirectPolicy()). - R().SetFormData(map[string]string{ - "task": "3", - "uid": d.Account, - "pwd": d.Password, - "setSessionId": "", - "setSig": "", - "setScene": "", - "setTocen": "", - "formhash": "", - }).Post("https://up.woozooo.com/mlogin.php") - if err != nil { - return nil, err - } - if utils.Json.Get(resp.Body(), "zt").ToInt() != 1 { - return nil, fmt.Errorf("login err: %s", resp.Body()) + var vs string + for retry := 0; retry < 3; retry++ { + req := base.NewRestyClient().SetRedirectPolicy(resty.NoRedirectPolicy()).R() + + // 如果已计算出 acw_sc__v2，通过 cookie 携带 + if vs != "" { + req.SetHeader("cookie", "acw_sc__v2="+vs) + } + + resp, err := req.SetFormData(map[string]string{ + "task": "3", + "uid": d.Account, + "pwd": d.Password, + "setSessionId": "", + "setSig": "", + "setScene": "", + "setTocen": "", + "formhash": "", + }).Post("https://up.woozooo.com/mlogin.php") + if err != nil { + return nil, err + } + bodyStr := resp.String() + if strings.Contains(bodyStr, "acw_sc__v2") { + vs, err = CalcAcwScV2(bodyStr) + if err != nil { + return nil, err + } + continue + } + if utils.Json.Get(resp.Body(), "zt").ToInt() != 1 { + return nil, fmt.Errorf("login err: %s", resp.Body()) + } + d.Cookie = CookieToString(resp.Cookies()) + return resp.Cookies(), nil } - d.Cookie = CookieToString(resp.Cookies()) - return resp.Cookies(), nil + return nil, errors.New("acw_sc__v2 validation error") } /* @@ -572,14 +590,14 @@ func (d *LanZou) getFolderByShareUrl(pwd string, sharePageData string) ([]FileOr files := make([]FileOrFolderByShareUrl, 0) // vip获取文件夹 - floders := findSubFolderReg.FindAllStringSubmatch(sharePageData, -1) - for _, floder := range floders { - if len(floder) == 3 { + folders := findSubFolderReg.FindAllStringSubmatch(sharePageData, -1) + for _, folder := range folders { + if len(folder) == 3 { files = append(files, FileOrFolderByShareUrl{ // Pwd: pwd, // 子文件夹不加密 - ID: floder[1], - NameAll: floder[2], - IsFloder: true, + ID: folder[1], + NameAll: folder[2], + IsFolder: true, }) } } diff --git a/drivers/s3/driver.go b/drivers/s3/driver.go index 46d7f414a..711f46ab5 100644 --- a/drivers/s3/driver.go +++ b/drivers/s3/driver.go @@ -178,7 +178,7 @@ func (d *S3) Move(ctx context.Context, srcObj, dstDir model.Obj) error { } func (d *S3) Rename(ctx context.Context, srcObj model.Obj, newName string) error { - err := d.copy(ctx, srcObj.GetPath(), stdpath.Join(stdpath.Dir(srcObj.GetPath()), newName), srcObj.IsDir()) + err := d.copy(ctx, srcObj.GetPath(), stdpath.Join(stdpath.Dir(srcObj.GetPath()), newName), srcObj.GetSize(), srcObj.IsDir()) if err != nil { return err } @@ -186,7 +186,7 @@ func (d *S3) Rename(ctx context.Context, srcObj model.Obj, newName string) error } func (d *S3) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { - return d.copy(ctx, srcObj.GetPath(), stdpath.Join(dstDir.GetPath(), srcObj.GetName()), srcObj.IsDir()) + return d.copy(ctx, srcObj.GetPath(), stdpath.Join(dstDir.GetPath(), srcObj.GetName()), srcObj.GetSize(), srcObj.IsDir()) } func (d *S3) Remove(ctx context.Context, obj model.Obj) error { diff --git a/drivers/s3/util.go b/drivers/s3/util.go index 8500ed264..cba8698fa 100644 --- a/drivers/s3/util.go +++ b/drivers/s3/util.go @@ -3,6 +3,7 @@ package s3 import ( "context" "errors" + "fmt" "net/http" "net/url" "path" @@ -19,6 +20,13 @@ import ( log "github.com/sirupsen/logrus" ) +const ( + maxCopyObjectSize int64 = 5 * 1000 * 1000 * 1000 + defaultCopyPartSize int64 = 100 * 1024 * 1024 + maxCopyPartSize int64 = 5 * 1024 * 1024 * 1024 + maxCopyParts int64 = 10000 +) + // do others that not defined in Driver interface func (d *S3) initSession() error { @@ -212,17 +220,20 @@ func (d *S3) listV2(dirPath string, args model.ListArgs) ([]model.Obj, error) { return files, nil } -func (d *S3) copy(ctx context.Context, src string, dst string, isDir bool) error { +func (d *S3) copy(ctx context.Context, src string, dst string, size int64, isDir bool) error { if isDir { return d.copyDir(ctx, src, dst) } - return d.copyFile(ctx, src, dst) + return d.copyFile(ctx, src, dst, size) } -func (d *S3) copyFile(ctx context.Context, src string, dst string) error { +func (d *S3) copyFile(ctx context.Context, src string, dst string, size int64) error { srcKey := getKey(src, false) dstKey := getKey(dst, false) encodedKey := strings.ReplaceAll(url.PathEscape(d.Bucket+"/"+srcKey), "+", "%2B") + if size > maxCopyObjectSize { + return d.copyFileMultipart(ctx, srcKey, dstKey, encodedKey, size) + } input := &s3.CopyObjectInput{ Bucket: &d.Bucket, CopySource: aws.String(encodedKey), @@ -232,6 +243,106 @@ func (d *S3) copyFile(ctx context.Context, src string, dst string) error { return err } +func (d *S3) copyFileMultipart(ctx context.Context, srcKey, dstKey, encodedKey string, size int64) (err error) { + head, err := d.client.HeadObjectWithContext(ctx, &s3.HeadObjectInput{ + Bucket: &d.Bucket, + Key: &srcKey, + }) + if err != nil { + return err + } + if head.ContentLength != nil { + size = *head.ContentLength + } + partSize, err := getCopyPartSize(size) + if err != nil { + return err + } + createInput := &s3.CreateMultipartUploadInput{ + Bucket: &d.Bucket, + Key: &dstKey, + CacheControl: head.CacheControl, + ContentDisposition: head.ContentDisposition, + ContentEncoding: head.ContentEncoding, + ContentLanguage: head.ContentLanguage, + ContentType: head.ContentType, + Metadata: head.Metadata, + WebsiteRedirectLocation: head.WebsiteRedirectLocation, + } + if head.Expires != nil { + if expires, parseErr := http.ParseTime(*head.Expires); parseErr == nil { + createInput.Expires = &expires + } + } + created, err := d.client.CreateMultipartUploadWithContext(ctx, createInput) + if err != nil { + return err + } + uploadID := aws.StringValue(created.UploadId) + if uploadID == "" { + return errors.New("create multipart upload returned an empty upload ID") + } + completed := false + defer func() { + if completed { + return + } + _, abortErr := d.client.AbortMultipartUploadWithContext(context.WithoutCancel(ctx), &s3.AbortMultipartUploadInput{ + Bucket: &d.Bucket, + Key: &dstKey, + UploadId: &uploadID, + }) + if abortErr != nil { + err = errors.Join(err, fmt.Errorf("failed to abort multipart copy: %w", abortErr)) + } + }() + + parts := make([]*s3.CompletedPart, 0, (size+partSize-1)/partSize) + for start, partNumber := int64(0), int64(1); start < size; start, partNumber = start+partSize, partNumber+1 { + end := min(start+partSize, size) - 1 + copied, copyErr := d.client.UploadPartCopyWithContext(ctx, &s3.UploadPartCopyInput{ + Bucket: &d.Bucket, + CopySource: &encodedKey, + CopySourceRange: aws.String(fmt.Sprintf("bytes=%d-%d", start, end)), + Key: &dstKey, + PartNumber: &partNumber, + UploadId: &uploadID, + }) + if copyErr != nil { + return copyErr + } + if copied.CopyPartResult == nil || aws.StringValue(copied.CopyPartResult.ETag) == "" { + return fmt.Errorf("multipart copy part %d returned an empty ETag", partNumber) + } + parts = append(parts, &s3.CompletedPart{ + ETag: copied.CopyPartResult.ETag, + PartNumber: &partNumber, + }) + } + + _, err = d.client.CompleteMultipartUploadWithContext(ctx, &s3.CompleteMultipartUploadInput{ + Bucket: &d.Bucket, + Key: &dstKey, + UploadId: &uploadID, + MultipartUpload: &s3.CompletedMultipartUpload{ + Parts: parts, + }, + }) + if err != nil { + return err + } + completed = true + return nil +} + +func getCopyPartSize(size int64) (int64, error) { + partSize := max(defaultCopyPartSize, (size-1)/maxCopyParts+1) + if partSize > maxCopyPartSize { + return 0, fmt.Errorf("object size %d exceeds multipart copy limit", size) + } + return partSize, nil +} + func (d *S3) copyDir(ctx context.Context, src string, dst string) error { objs, err := op.List(ctx, d, src, model.ListArgs{S3ShowPlaceholder: true}) if err != nil { @@ -243,7 +354,7 @@ func (d *S3) copyDir(ctx context.Context, src string, dst string) error { if obj.IsDir() { err = d.copyDir(ctx, cSrc, cDst) } else { - err = d.copyFile(ctx, cSrc, cDst) + err = d.copyFile(ctx, cSrc, cDst, obj.GetSize()) } if err != nil { return err diff --git a/drivers/s3/util_test.go b/drivers/s3/util_test.go new file mode 100644 index 000000000..6c718a2f6 --- /dev/null +++ b/drivers/s3/util_test.go @@ -0,0 +1,215 @@ +package s3 + +import ( + "context" + "fmt" + "io" + "net/http" + "net/http/httptest" + "strconv" + "strings" + "testing" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/session" + awss3 "github.com/aws/aws-sdk-go/service/s3" +) + +func TestCopyFileUsesCopyObjectAtLimit(t *testing.T) { + copyRequests := 0 + d := newTestS3Driver(t, func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPut || r.URL.Query().Get("uploadId") != "" { + t.Errorf("unexpected request: %s %s", r.Method, r.URL.String()) + w.WriteHeader(http.StatusBadRequest) + return + } + copyRequests++ + writeTestXML(t, w, `"copy"`) + }) + + if err := d.copyFile(context.Background(), "source+file", "destination", maxCopyObjectSize); err != nil { + t.Fatalf("copyFile: %v", err) + } + if copyRequests != 1 { + t.Fatalf("copy requests = %d, want 1", copyRequests) + } +} + +func TestCopyFileUsesMultipartCopyAboveLimit(t *testing.T) { + size := maxCopyObjectSize + 1 + wantParts := int((size + defaultCopyPartSize - 1) / defaultCopyPartSize) + ranges := make(map[int]string, wantParts) + completed := false + aborted := false + + d := newTestS3Driver(t, func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodHead: + w.Header().Set("Content-Length", strconv.FormatInt(size, 10)) + w.Header().Set("Content-Type", "application/octet-stream") + w.Header().Set("Cache-Control", "max-age=60") + w.Header().Set("Content-Disposition", "attachment") + w.Header().Set("Expires", "Wed, 21 Oct 2015 07:28:00 GMT") + w.Header().Set("X-Amz-Meta-Source", "preserved") + w.Header().Set("X-Amz-Website-Redirect-Location", "/redirect") + w.WriteHeader(http.StatusOK) + case r.Method == http.MethodPost && r.URL.Query().Has("uploads"): + if got := r.Header.Get("Cache-Control"); got != "max-age=60" { + t.Errorf("Cache-Control = %q, want %q", got, "max-age=60") + } + if got := r.Header.Get("Content-Disposition"); got != "attachment" { + t.Errorf("Content-Disposition = %q, want %q", got, "attachment") + } + if got := r.Header.Get("Content-Type"); got != "application/octet-stream" { + t.Errorf("Content-Type = %q, want %q", got, "application/octet-stream") + } + if got := r.Header.Get("Expires"); got != "Wed, 21 Oct 2015 07:28:00 GMT" { + t.Errorf("Expires = %q, want an unchanged HTTP date", got) + } + if got := r.Header.Get("X-Amz-Meta-Source"); got != "preserved" { + t.Errorf("metadata = %q, want %q", got, "preserved") + } + if got := r.Header.Get("X-Amz-Website-Redirect-Location"); got != "/redirect" { + t.Errorf("website redirect = %q, want %q", got, "/redirect") + } + writeTestXML(t, w, `upload-id`) + case r.Method == http.MethodPut && r.URL.Query().Get("uploadId") == "upload-id": + partNumber, err := strconv.Atoi(r.URL.Query().Get("partNumber")) + if err != nil { + t.Errorf("invalid part number: %v", err) + w.WriteHeader(http.StatusBadRequest) + return + } + if got := r.Header.Get("X-Amz-Copy-Source"); !strings.Contains(got, "source%2Bfile") { + t.Errorf("copy source = %q, want encoded source key", got) + } + ranges[partNumber] = r.Header.Get("X-Amz-Copy-Source-Range") + writeTestXML(t, w, fmt.Sprintf(`"part-%d"`, partNumber)) + case r.Method == http.MethodPost && r.URL.Query().Get("uploadId") == "upload-id": + body, err := io.ReadAll(r.Body) + if err != nil { + t.Errorf("read complete body: %v", err) + } + if got := strings.Count(string(body), ""); got != wantParts { + t.Errorf("completed parts = %d, want %d", got, wantParts) + } + completed = true + writeTestXML(t, w, `"complete"`) + case r.Method == http.MethodDelete && r.URL.Query().Get("uploadId") == "upload-id": + aborted = true + w.WriteHeader(http.StatusNoContent) + default: + t.Errorf("unexpected request: %s %s", r.Method, r.URL.String()) + w.WriteHeader(http.StatusBadRequest) + } + }) + + if err := d.copyFile(context.Background(), "source+file", "destination", size); err != nil { + t.Fatalf("copyFile: %v", err) + } + if !completed { + t.Fatal("multipart upload was not completed") + } + if aborted { + t.Fatal("successful multipart upload was aborted") + } + if len(ranges) != wantParts { + t.Fatalf("copied parts = %d, want %d", len(ranges), wantParts) + } + if got := ranges[1]; got != fmt.Sprintf("bytes=0-%d", defaultCopyPartSize-1) { + t.Errorf("first range = %q", got) + } + lastStart := int64(wantParts-1) * defaultCopyPartSize + if got := ranges[wantParts]; got != fmt.Sprintf("bytes=%d-%d", lastStart, size-1) { + t.Errorf("last range = %q", got) + } +} + +func TestCopyFileMultipartAbortsOnPartFailure(t *testing.T) { + size := maxCopyObjectSize + 1 + aborted := false + completed := false + + d := newTestS3Driver(t, func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodHead: + w.Header().Set("Content-Length", strconv.FormatInt(size, 10)) + w.WriteHeader(http.StatusOK) + case r.Method == http.MethodPost && r.URL.Query().Has("uploads"): + writeTestXML(t, w, `upload-id`) + case r.Method == http.MethodPut && r.URL.Query().Get("uploadId") == "upload-id": + w.WriteHeader(http.StatusInternalServerError) + writeTestXML(t, w, `InternalErrorcopy failed`) + case r.Method == http.MethodDelete && r.URL.Query().Get("uploadId") == "upload-id": + aborted = true + w.WriteHeader(http.StatusNoContent) + case r.Method == http.MethodPost && r.URL.Query().Get("uploadId") == "upload-id": + completed = true + w.WriteHeader(http.StatusOK) + default: + t.Errorf("unexpected request: %s %s", r.Method, r.URL.String()) + w.WriteHeader(http.StatusBadRequest) + } + }) + + if err := d.copyFile(context.Background(), "source", "destination", size); err == nil { + t.Fatal("copyFile returned nil error") + } + if !aborted { + t.Fatal("failed multipart upload was not aborted") + } + if completed { + t.Fatal("failed multipart upload was completed") + } +} + +func TestGetCopyPartSize(t *testing.T) { + partSize, err := getCopyPartSize(defaultCopyPartSize * maxCopyParts) + if err != nil { + t.Fatalf("getCopyPartSize: %v", err) + } + if partSize != defaultCopyPartSize { + t.Fatalf("part size = %d, want %d", partSize, defaultCopyPartSize) + } + + partSize, err = getCopyPartSize(defaultCopyPartSize*maxCopyParts + 1) + if err != nil { + t.Fatalf("getCopyPartSize: %v", err) + } + if partSize != defaultCopyPartSize+1 { + t.Fatalf("grown part size = %d, want %d", partSize, defaultCopyPartSize+1) + } + + if _, err := getCopyPartSize(maxCopyPartSize*maxCopyParts + 1); err == nil { + t.Fatal("getCopyPartSize returned nil error for an oversized object") + } +} + +func newTestS3Driver(t *testing.T, handler http.HandlerFunc) *S3 { + t.Helper() + server := httptest.NewServer(handler) + t.Cleanup(server.Close) + sess, err := session.NewSession(&aws.Config{ + Credentials: credentials.NewStaticCredentials("access-key", "secret-key", ""), + Endpoint: aws.String(server.URL), + Region: aws.String("us-east-1"), + S3ForcePathStyle: aws.Bool(true), + MaxRetries: aws.Int(0), + }) + if err != nil { + t.Fatalf("create AWS session: %v", err) + } + return &S3{ + Addition: Addition{Bucket: "bucket"}, + client: awss3.New(sess), + } +} + +func writeTestXML(t *testing.T, w http.ResponseWriter, body string) { + t.Helper() + w.Header().Set("Content-Type", "application/xml") + if _, err := io.WriteString(w, body); err != nil { + t.Errorf("write response: %v", err) + } +} diff --git a/drivers/strm/types.go b/drivers/strm/types.go deleted file mode 100644 index dfa427d42..000000000 --- a/drivers/strm/types.go +++ /dev/null @@ -1,36 +0,0 @@ -package strm - -func supportSuffix() map[string]struct{} { - return map[string]struct{}{ - // video - "mp4": {}, - "mkv": {}, - "flv": {}, - "avi": {}, - "wmv": {}, - "ts": {}, - "rmvb": {}, - "webm": {}, - // audio - "mp3": {}, - "flac": {}, - "aac": {}, - "wav": {}, - "ogg": {}, - "m4a": {}, - "wma": {}, - "alac": {}, - } -} - -func downloadSuffix() map[string]struct{} { - return map[string]struct{}{ - // strm - "strm": {}, - // subtitles - "ass": {}, - "srt": {}, - "vtt": {}, - "sub": {}, - } -} diff --git a/drivers/thunder_browser/driver.go b/drivers/thunder_browser/driver.go index d105f86bb..a6ff2d9fe 100644 --- a/drivers/thunder_browser/driver.go +++ b/drivers/thunder_browser/driver.go @@ -897,7 +897,7 @@ func (xc *XunLeiBrowserCommon) OfflineList(ctx context.Context, nextPageToken st func (xc *XunLeiBrowserCommon) DeleteOfflineTasks(ctx context.Context, taskIDs []string) error { queryParams := map[string]string{ "task_ids": strings.Join(taskIDs, ","), - "_t": fmt.Sprintf("%d", time.Now().UnixMilli()), + "_t": strconv.FormatInt(time.Now().UnixMilli(), 10), } if xc.UseFluentPlay { queryParams["space"] = ThunderBrowserDriveFluentPlayFolderType diff --git a/drivers/wps/util.go b/drivers/wps/util.go index 10f104c93..cb6e84d66 100644 --- a/drivers/wps/util.go +++ b/drivers/wps/util.go @@ -188,19 +188,29 @@ func (d *Wps) getGroups(ctx context.Context) ([]Group, error) { func (d *Wps) getFiles(ctx context.Context, groupID, parentID int64) ([]FileInfo, error) { var resp filesResp - url := fmt.Sprintf("%s/api/v5/groups/%d/files", d.driveHost()+d.drivePrefix(), groupID) - r, err := d.request(ctx). - SetQueryParam("parentid", strconv.FormatInt(parentID, 10)). - SetResult(&resp). - SetError(&resp). - Get(url) - if err != nil { - return nil, err - } - if r != nil && r.IsError() { - return nil, fmt.Errorf("http error: %d", r.StatusCode()) + var files []FileInfo + next_offset := 0 + for range 50 { + url := fmt.Sprintf("%s/api/v5/groups/%d/files", d.driveHost()+d.drivePrefix(), groupID) + r, err := d.request(ctx). + SetQueryParam("parentid", strconv.FormatInt(parentID, 10)). + SetQueryParam("offset", fmt.Sprint(next_offset)). + SetResult(&resp). + SetError(&resp). + Get(url) + if err != nil { + return nil, err + } + if r != nil && r.IsError() { + return nil, fmt.Errorf("http error: %d", r.StatusCode()) + } + files = append(files, resp.Files...) + if resp.NextOffset == -1 { + break + } + next_offset = resp.NextOffset } - return resp.Files, nil + return files, nil } func parseTime(v int64) time.Time { diff --git a/go.mod b/go.mod index ed7bd092d..cfd170c65 100644 --- a/go.mod +++ b/go.mod @@ -75,7 +75,6 @@ require ( github.com/tchap/go-patricia/v2 v2.3.3 github.com/u2takey/ffmpeg-go v0.5.0 github.com/upyun/go-sdk/v3 v3.0.4 - github.com/winfsp/cgofuse v1.6.0 github.com/zzzhr1990/go-common-entity v0.0.0-20250202070650-1a200048f0d3 golang.org/x/crypto v0.51.0 golang.org/x/image v0.29.0 diff --git a/go.sum b/go.sum index 02c1ae6a9..7c4a9cb25 100644 --- a/go.sum +++ b/go.sum @@ -672,8 +672,6 @@ github.com/unknwon/goconfig v1.0.0 h1:rS7O+CmUdli1T+oDm7fYj1MwqNWtEJfNj+FqcUHML8 github.com/unknwon/goconfig v1.0.0/go.mod h1:qu2ZQ/wcC/if2u32263HTVC39PeOQRSmidQk3DuDFQ8= github.com/upyun/go-sdk/v3 v3.0.4 h1:2DCJa/Yi7/3ZybT9UCPATSzvU3wpPPxhXinNlb1Hi8Q= github.com/upyun/go-sdk/v3 v3.0.4/go.mod h1:P/SnuuwhrIgAVRd/ZpzDWqCsBAf/oHg7UggbAxyZa0E= -github.com/winfsp/cgofuse v1.6.0 h1:re3W+HTd0hj4fISPBqfsrwyvPFpzqhDu8doJ9nOPDB0= -github.com/winfsp/cgofuse v1.6.0/go.mod h1:uxjoF2jEYT3+x+vC2KJddEGdk/LU8pRowXmyVMHSV5I= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no= diff --git a/internal/bootstrap/config.go b/internal/bootstrap/config.go index c913a961d..a3ed9b22c 100644 --- a/internal/bootstrap/config.go +++ b/internal/bootstrap/config.go @@ -54,6 +54,7 @@ func InitConfig() { } } configPath = filepath.Clean(configPath) + conf.ConfigPath = configPath log.Infof("reading config file: %s", configPath) if !utils.Exists(configPath) { log.Infof("config file not exists, creating default config file") diff --git a/internal/bootstrap/data/setting.go b/internal/bootstrap/data/setting.go index 48d21684a..27ff64e52 100644 --- a/internal/bootstrap/data/setting.go +++ b/internal/bootstrap/data/setting.go @@ -243,6 +243,7 @@ func InitialSettings() []model.SettingItem { {Key: conf.TaskOfflineDownloadTransferThreadsNum, Value: strconv.Itoa(conf.Conf.Tasks.Transfer.Workers), Type: conf.TypeNumber, Group: model.TRAFFIC, Flag: model.PRIVATE}, {Key: conf.TaskUploadThreadsNum, Value: strconv.Itoa(conf.Conf.Tasks.Upload.Workers), Type: conf.TypeNumber, Group: model.TRAFFIC, Flag: model.PRIVATE}, {Key: conf.TaskCopyThreadsNum, Value: strconv.Itoa(conf.Conf.Tasks.Copy.Workers), Type: conf.TypeNumber, Group: model.TRAFFIC, Flag: model.PRIVATE}, + {Key: conf.TaskMoveThreadsNum, Value: strconv.Itoa(conf.Conf.Tasks.Move.Workers), Type: conf.TypeNumber, Group: model.TRAFFIC, Flag: model.PRIVATE}, {Key: conf.TaskDecompressDownloadThreadsNum, Value: strconv.Itoa(conf.Conf.Tasks.Decompress.Workers), Type: conf.TypeNumber, Group: model.TRAFFIC, Flag: model.PRIVATE}, {Key: conf.TaskDecompressUploadThreadsNum, Value: strconv.Itoa(conf.Conf.Tasks.DecompressUpload.Workers), Type: conf.TypeNumber, Group: model.TRAFFIC, Flag: model.PRIVATE}, {Key: conf.StreamMaxClientDownloadSpeed, Value: "-1", Type: conf.TypeNumber, Group: model.TRAFFIC, Flag: model.PRIVATE}, @@ -282,6 +283,8 @@ func InitialSettings() []model.SettingItem { {Key: "THUNDER_id", Value: "0", Type: conf.TypeNumber, Group: model.SINGLE, Flag: model.PRIVATE}, {Key: "OPEN115_id", Value: "0", Type: conf.TypeNumber, Group: model.SINGLE, Flag: model.PRIVATE}, {Key: "BAIDU_id", Value: "0", Type: conf.TypeNumber, Group: model.SINGLE, Flag: model.PRIVATE}, + {Key: conf.MultipartEnabled, Value: "true", Type: conf.TypeBool, Group: model.TRAFFIC, Flag: model.PUBLIC}, + {Key: conf.MultipartChunkSize, Value: "10", Type: conf.TypeNumber, Group: model.TRAFFIC, Flag: model.PUBLIC, Help: `chunk size of multipart upload in MB (positive integer), keep it under your CDN's request body limit; each active session buffers up to 8 chunks on the server's disk`}, } additionalSettingItems := tool.Tools.Items() // 固定顺序 diff --git a/internal/bootstrap/patch/all.go b/internal/bootstrap/patch/all.go index c4a72a966..5d4c814dc 100644 --- a/internal/bootstrap/patch/all.go +++ b/internal/bootstrap/patch/all.go @@ -44,6 +44,7 @@ var UpgradePatches = []VersionPatches{ Version: "v4.1.9", Patches: []func(){ v4_1_9.EnableWebDavProxy, + v4_1_9.ResetSkipTlsVerify, }, }, } diff --git a/internal/bootstrap/patch/v3_all/rename.go b/internal/bootstrap/patch/v3_all/rename.go deleted file mode 100644 index 323b7391f..000000000 --- a/internal/bootstrap/patch/v3_all/rename.go +++ /dev/null @@ -1,33 +0,0 @@ -package v3_all - -import ( - "github.com/OpenListTeam/OpenList/v4/internal/db" - "github.com/OpenListTeam/OpenList/v4/pkg/utils" -) - -// Rename Alist V3 driver to OpenList -func RenameAlistV3Driver() { - storages, _, err := db.GetStorages(1, -1) - if err != nil { - utils.Log.Errorf("[RenameAlistV3Driver] failed to get storages: %s", err.Error()) - return - } - - updatedCount := 0 - for _, s := range storages { - if s.Driver == "AList V3" { - utils.Log.Warnf("[RenameAlistV3Driver] rename storage [%d]%s from Alist V3 to OpenList", s.ID, s.MountPath) - s.Driver = "OpenList" - err = db.UpdateStorage(&s) - if err != nil { - utils.Log.Errorf("[RenameAlistV3Driver] failed to update storage [%d]%s: %s", s.ID, s.MountPath, err.Error()) - } else { - updatedCount++ - } - } - } - - if updatedCount > 0 { - utils.Log.Infof("[RenameAlistV3Driver] updated %d storages from Alist V3 to OpenList", updatedCount) - } -} diff --git a/internal/bootstrap/patch/v4_1_9/skip_tls.go b/internal/bootstrap/patch/v4_1_9/skip_tls.go new file mode 100644 index 000000000..1d9858d95 --- /dev/null +++ b/internal/bootstrap/patch/v4_1_9/skip_tls.go @@ -0,0 +1,32 @@ +package v4_1_9 + +import ( + "os" + "strings" + + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" +) + +func ResetSkipTlsVerify() { + if !conf.Conf.TlsInsecureSkipVerify { + return + } + if !strings.HasPrefix(conf.Version, "v") { + return + } + + conf.Conf.TlsInsecureSkipVerify = false + + confBody, err := utils.Json.MarshalIndent(conf.Conf, "", " ") + if err != nil { + utils.Log.Errorf("[ResetSkipTlsVerify] failed to rewrite config: marshal config error: %+v", err) + return + } + err = os.WriteFile(conf.ConfigPath, confBody, 0o777) + if err != nil { + utils.Log.Errorf("[ResetSkipTlsVerify] failed to rewrite config: update config struct error: %+v", err) + return + } + utils.Log.Infof("[ResetSkipTlsVerify] succeeded to set tls_insecure_skip_verify to false") +} diff --git a/internal/conf/const.go b/internal/conf/const.go index 5fa23cd1a..cf0539974 100644 --- a/internal/conf/const.go +++ b/internal/conf/const.go @@ -200,6 +200,8 @@ const ( StreamMaxClientUploadSpeed = "max_client_upload_speed" StreamMaxServerDownloadSpeed = "max_server_download_speed" StreamMaxServerUploadSpeed = "max_server_upload_speed" + MultipartEnabled = "multipart_enabled" + MultipartChunkSize = "multipart_chunk_size" ) const ( diff --git a/internal/conf/var.go b/internal/conf/var.go index 6e0d02f1c..149ec3a84 100644 --- a/internal/conf/var.go +++ b/internal/conf/var.go @@ -12,6 +12,7 @@ var ( GitCommit string = "unknown" Version string = "dev" WebVersion string = "rolling" + ConfigPath string ) var ( @@ -20,6 +21,17 @@ var ( ) var SlicesMap = make(map[string][]string) + +var ( + // 在HybridCache中使用[]byte缓存数据流的限制，内存为Go自动管理，直到GC + AutoMemoryLimit uint64 = 4 * 1024 * 1024 + // 最小空闲内存，当内存不足时，HybridCache会回退到文件缓存。 + // 如果为0，HybridCache会使用文件缓存，不占用内存。 + MinFreeMemory uint64 = 16 * 1024 * 1024 + // 限制HybridCache手动管理内存单次的扩容大小，超过该阈值将分多次扩容。 + // MinFreeMemory大于0时，也限制 Downloader 的PartSize + MaxBlockLimit uint64 = 16 * 1024 * 1024 +) var FilenameCharMap = make(map[string]string) var PrivacyReg []*regexp.Regexp diff --git a/internal/fuse/fs.go b/internal/fuse/fs.go deleted file mode 100644 index 7783b169f..000000000 --- a/internal/fuse/fs.go +++ /dev/null @@ -1,170 +0,0 @@ -package fuse - -import "github.com/winfsp/cgofuse/fuse" - -type Fs struct { - RootFolder string - fuse.FileSystemBase -} - -func (fs *Fs) Init() { - //TODO implement me - panic("implement me") -} - -func (fs *Fs) Destroy() { - //TODO implement me - panic("implement me") -} - -func (fs *Fs) Statfs(path string, stat *fuse.Statfs_t) int { - //TODO implement me - panic("implement me") -} - -func (fs *Fs) Mknod(path string, mode uint32, dev uint64) int { - //TODO implement me - panic("implement me") -} - -func (fs *Fs) Mkdir(path string, mode uint32) int { - //TODO implement me - panic("implement me") -} - -func (fs *Fs) Unlink(path string) int { - //TODO implement me - panic("implement me") -} - -func (fs *Fs) Rmdir(path string) int { - //TODO implement me - panic("implement me") -} - -func (fs *Fs) Link(oldpath string, newpath string) int { - //TODO implement me - panic("implement me") -} - -func (fs *Fs) Symlink(target string, newpath string) int { - //TODO implement me - panic("implement me") -} - -func (fs *Fs) Readlink(path string) (int, string) { - //TODO implement me - panic("implement me") -} - -func (fs *Fs) Rename(oldpath string, newpath string) int { - //TODO implement me - panic("implement me") -} - -func (fs *Fs) Chmod(path string, mode uint32) int { - //TODO implement me - panic("implement me") -} - -func (fs *Fs) Chown(path string, uid uint32, gid uint32) int { - //TODO implement me - panic("implement me") -} - -func (fs *Fs) Utimens(path string, tmsp []fuse.Timespec) int { - //TODO implement me - panic("implement me") -} - -func (fs *Fs) Access(path string, mask uint32) int { - //TODO implement me - panic("implement me") -} - -func (fs *Fs) Create(path string, flags int, mode uint32) (int, uint64) { - //TODO implement me - panic("implement me") -} - -func (fs *Fs) Open(path string, flags int) (int, uint64) { - //TODO implement me - panic("implement me") -} - -func (fs *Fs) Getattr(path string, stat *fuse.Stat_t, fh uint64) int { - //TODO implement me - panic("implement me") -} - -func (fs *Fs) Truncate(path string, size int64, fh uint64) int { - //TODO implement me - panic("implement me") -} - -func (fs *Fs) Read(path string, buff []byte, ofst int64, fh uint64) int { - //TODO implement me - panic("implement me") -} - -func (fs *Fs) Write(path string, buff []byte, ofst int64, fh uint64) int { - //TODO implement me - panic("implement me") -} - -func (fs *Fs) Flush(path string, fh uint64) int { - //TODO implement me - panic("implement me") -} - -func (fs *Fs) Release(path string, fh uint64) int { - //TODO implement me - panic("implement me") -} - -func (fs *Fs) Fsync(path string, datasync bool, fh uint64) int { - //TODO implement me - panic("implement me") -} - -func (fs *Fs) Opendir(path string) (int, uint64) { - //TODO implement me - panic("implement me") -} - -func (fs *Fs) Readdir(path string, fill func(name string, stat *fuse.Stat_t, ofst int64) bool, ofst int64, fh uint64) int { - //TODO implement me - panic("implement me") -} - -func (fs *Fs) Releasedir(path string, fh uint64) int { - //TODO implement me - panic("implement me") -} - -func (fs *Fs) Fsyncdir(path string, datasync bool, fh uint64) int { - //TODO implement me - panic("implement me") -} - -func (fs *Fs) Setxattr(path string, name string, value []byte, flags int) int { - //TODO implement me - panic("implement me") -} - -func (fs *Fs) Getxattr(path string, name string) (int, []byte) { - //TODO implement me - panic("implement me") -} - -func (fs *Fs) Removexattr(path string, name string) int { - //TODO implement me - panic("implement me") -} - -func (fs *Fs) Listxattr(path string, fill func(name string) bool) int { - //TODO implement me - panic("implement me") -} - -var _ fuse.FileSystemInterface = (*Fs)(nil) diff --git a/internal/fuse/mount.go b/internal/fuse/mount.go deleted file mode 100644 index 30e3d5464..000000000 --- a/internal/fuse/mount.go +++ /dev/null @@ -1,9 +0,0 @@ -package fuse - -import "github.com/winfsp/cgofuse/fuse" - -func Mount(mountSrc, mountDst string, opts []string) { - fs := &Fs{RootFolder: mountSrc} - host := fuse.NewFileSystemHost(fs) - go host.Mount(mountDst, opts) -} diff --git a/internal/hybrid_cache/buffer.go b/internal/hybrid_cache/buffer.go new file mode 100644 index 000000000..0023996e6 --- /dev/null +++ b/internal/hybrid_cache/buffer.go @@ -0,0 +1,96 @@ +package hybrid_cache + +import ( + "fmt" + "io" +) + +type BufferStore struct { + blocks [][]byte + size int64 +} + +func (m *BufferStore) Size() int64 { + return m.size +} + +// 用于存储不复用的[]byte +func (m *BufferStore) Append(buf []byte) { + m.size += int64(len(buf)) + m.blocks = append(m.blocks, buf) +} + +func (m *BufferStore) Close() error { + if len(m.blocks) > 0 { + clear(m.blocks) + m.blocks = m.blocks[:0] + m.size = 0 + } + return nil +} + +func (m *BufferStore) ReadAt(p []byte, off int64) (int, error) { + if len(p) == 0 { + return 0, nil + } + if off < 0 || off >= m.size { + return 0, io.EOF + } + + var n int + for _, buf := range m.blocks { + if off >= int64(len(buf)) { + off -= int64(len(buf)) + continue + } + nn := copy(p[n:], buf[off:]) + n += nn + if n == len(p) { + return n, nil + } + off = 0 + } + + return n, io.EOF +} + +func (m *BufferStore) WriteAt(p []byte, off int64) (int, error) { + if len(p) == 0 { + return 0, nil + } + if off < 0 || off >= m.size { + return 0, io.ErrShortWrite + } + + var n int + for _, b := range m.blocks { + if off >= int64(len(b)) { + off -= int64(len(b)) + continue + } + nn := copy(b[off:], p[n:]) + n += nn + if n == len(p) { + return n, nil + } + off = 0 + } + + return n, io.ErrShortWrite +} + +func (m *BufferStore) GrowTo(size int64) (err error) { + if size <= m.size { + return nil + } + defer func() { + if r := recover(); r != nil { + err = fmt.Errorf("recovered in %v", r) + } + }() + m.blocks = append(m.blocks, make([]byte, size-m.size)) + m.size = size + return nil +} + +var _ BackingStore = (*BufferStore)(nil) diff --git a/internal/hybrid_cache/buffer_test.go b/internal/hybrid_cache/buffer_test.go new file mode 100644 index 000000000..439d7cb78 --- /dev/null +++ b/internal/hybrid_cache/buffer_test.go @@ -0,0 +1,100 @@ +package hybrid_cache_test + +import ( + "errors" + "io" + "testing" + + "github.com/OpenListTeam/OpenList/v4/internal/hybrid_cache" +) + +func TestBufferStore(t *testing.T) { + type args struct { + p []byte + off int64 + } + bs := &hybrid_cache.BufferStore{} + bs.Append([]byte("github.com")) + bs.Append([]byte("/OpenList")) + bs.Append([]byte("Team/?")) + b := []byte("OpenList") + off := bs.Size() - 1 + _ = bs.GrowTo(off + int64(len(b))) + _, _ = bs.WriteAt(b, off) + + tests := []struct { + name string + b *hybrid_cache.BufferStore + args args + check func(a args, n int, err error) error + }{ + { + name: "readAt len 10 offset 0", + b: bs, + args: args{ + p: make([]byte, 10), + off: 0, + }, + check: func(a args, n int, err error) error { + if n != len(a.p) { + return errors.New("read length not match") + } + if string(a.p) != "github.com" { + return errors.New("read content not match") + } + if err != nil { + return err + } + return nil + }, + }, + { + name: "readAt len 12 offset 11", + b: bs, + args: args{ + p: make([]byte, 12), + off: 11, + }, + check: func(a args, n int, err error) error { + if n != len(a.p) { + return errors.New("read length not match") + } + if string(a.p) != "OpenListTeam" { + return errors.New("read content not match") + } + if err != nil { + return err + } + return nil + }, + }, + { + name: "readAt len 50 offset 24", + b: bs, + args: args{ + p: make([]byte, 50), + off: 24, + }, + check: func(a args, n int, err error) error { + if n != int(bs.Size()-a.off) { + return errors.New("read length not match") + } + if string(a.p[:n]) != "OpenList" { + return errors.New("read content not match") + } + if err != io.EOF { + return errors.New("expect eof") + } + return nil + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := tt.b.ReadAt(tt.args.p, tt.args.off) + if err := tt.check(tt.args, got, err); err != nil { + t.Errorf("BufferStore.ReadAt() error = %v", err) + } + }) + } +} diff --git a/internal/hybrid_cache/file.go b/internal/hybrid_cache/file.go new file mode 100644 index 000000000..bca197dad --- /dev/null +++ b/internal/hybrid_cache/file.go @@ -0,0 +1,179 @@ +package hybrid_cache + +import ( + "errors" + "io" + "os" + + "github.com/OpenListTeam/OpenList/v4/internal/conf" +) + +type singleFileStore struct { + *os.File + size int64 +} + +func (s *singleFileStore) Size() int64 { + return s.size +} + +func (s *singleFileStore) GrowTo(size int64) error { + if size <= s.size { + return nil + } + err := s.File.Truncate(size) + if err == nil { + s.size = size + } + return err +} + +func (s *singleFileStore) Close() error { + err := s.File.Close() + _ = os.Remove(s.File.Name()) + return err +} + +type fileBlock struct { + file *os.File + size int64 + written int64 +} + +type MultiFileStore struct { + blocks []*fileBlock + size int64 +} + +func (s *MultiFileStore) Size() int64 { + return s.size +} + +func (m *MultiFileStore) Close() error { + var errs []error + for _, c := range m.blocks { + if err := c.file.Close(); err != nil { + errs = append(errs, err) + } + _ = os.Remove(c.file.Name()) + } + clear(m.blocks) + m.blocks = m.blocks[:0] + return errors.Join(errs...) +} + +func (m *MultiFileStore) GrowTo(size int64) error { + if size <= m.size { + return nil + } + f, err := os.CreateTemp(conf.Conf.TempDir, "file-*") + if err != nil { + return err + } + m.blocks = append(m.blocks, &fileBlock{file: f, size: size - m.size}) + m.size = size + return nil +} + +func (m *MultiFileStore) ReadAt(p []byte, off int64) (n int, err error) { + if len(p) == 0 { + return 0, nil + } + if off < 0 || off >= m.size { + return 0, io.EOF + } + + for _, c := range m.blocks { + if off >= c.size { + off -= c.size + continue + } + + canRead := min(len(p)-n, int(c.size-off)) + if canRead <= 0 { + break + } + + filled := 0 + + if off < c.written { + fileReadable := min(canRead, int(c.written-off)) + nn, fileErr := c.file.ReadAt(p[n:n+fileReadable], off) + n += nn + filled = nn + if fileErr != nil && !errors.Is(fileErr, io.EOF) { + return n, fileErr + } + } + + if n == len(p) { + return n, nil + } + + if zeroFill := canRead - filled; zeroFill > 0 { + clear(p[n : n+zeroFill]) + n += zeroFill + } + + if n == len(p) { + return n, nil + } + off = 0 + } + + return n, io.EOF +} + +func (m *MultiFileStore) WriteAt(p []byte, off int64) (n int, err error) { + if len(p) == 0 { + return 0, nil + } + if off < 0 || off >= m.size { + return 0, io.ErrShortWrite + } + + for _, b := range m.blocks { + if off >= b.size { + off -= b.size + continue + } + + canWrite := min(len(p)-n, int(b.size-off)) + if canWrite <= 0 { + break + } + + nn, fileErr := b.file.WriteAt(p[n:n+canWrite], off) + if end := off + int64(nn); end > b.written { + b.written = end + } + n += nn + if fileErr != nil { + return n, fileErr + } + if nn < canWrite { + return n, io.ErrShortWrite + } + if n == len(p) { + return n, nil + } + off = 0 + } + + return n, io.ErrShortWrite +} + +func NewFileStore(blockSize int64) (BackingStore, error) { + f, err := os.CreateTemp(conf.Conf.TempDir, "file-*") + if err != nil { + return nil, err + } + err = f.Truncate(blockSize) + if err == nil { + return &singleFileStore{File: f, size: blockSize}, nil + } + return &MultiFileStore{ + blocks: []*fileBlock{{file: f, size: blockSize}}, + size: blockSize, + }, nil +} diff --git a/internal/hybrid_cache/file_test.go b/internal/hybrid_cache/file_test.go new file mode 100644 index 000000000..7f83625db --- /dev/null +++ b/internal/hybrid_cache/file_test.go @@ -0,0 +1,101 @@ +package hybrid_cache_test + +import ( + "bytes" + "errors" + "io" + "os" + "reflect" + "testing" + + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/hybrid_cache" +) + +func TestFile(t *testing.T) { + f, err := os.CreateTemp("", "writeat-*") + if err != nil { + t.Error(err) + return + } + defer os.Remove(f.Name()) + defer f.Close() + t.Run("ReadAt", func(t *testing.T) { + _, err := f.ReadAt(make([]byte, 1), 20) + if err != nil && !errors.Is(err, io.EOF) { + t.Error(err) + } + }) + t.Run("WriteAt", func(t *testing.T) { + n, err := f.WriteAt([]byte("abc"), 20) + if err != nil { + t.Errorf("write n=%d err=%v", n, err) + return + } + stat, err := f.Stat() + if err != nil { + t.Errorf("stat err=%v", err) + return + } + if stat.Size() != 23 { + t.Fatalf("unexpected size: got %d want 23", stat.Size()) + } + + b := make([]byte, stat.Size()) + rn, rerr := f.ReadAt(b, 0) + if rn != len(b) || rerr != nil { + t.Fatalf("read n=%d err=%v", rn, rerr) + } + want := append(make([]byte, 20), []byte("abc")...) + if !reflect.DeepEqual(b, want) { + t.Fatalf("unexpected content: got %v want %v", b, want) + } + }) +} + +func TestMultiFileCache(t *testing.T) { + prevConf := conf.Conf + t.Cleanup(func() { + conf.Conf = prevConf + }) + conf.Conf = &conf.Config{} + f := hybrid_cache.MultiFileStore{} + defer f.Close() + t.Run("ReadAt", func(t *testing.T) { + _, err := f.ReadAt(make([]byte, 1), 20) + if err != nil && !errors.Is(err, io.EOF) { + t.Error(err) + } + }) + t.Run("WriteAt", func(t *testing.T) { + err := f.GrowTo(15) + if err != nil { + t.Errorf("truncate err=%v", err) + return + } + n, err := f.WriteAt([]byte("abc"), 10) + if err != nil { + t.Errorf("write n=%d err=%v", n, err) + return + } + + err = f.GrowTo(30) + if err != nil { + t.Errorf("truncate err=%v", err) + return + } + _, _ = f.WriteAt([]byte("123"), 15) + + b := append(make([]byte, 17), []byte("def")...) + b[0] = 'a' + rn, rerr := f.ReadAt(b, 8) + if rn != len(b) || rerr != nil { + t.Fatalf("read n=%d err=%v", rn, rerr) + } + want := []byte{0, 0, 'a', 'b', 'c', 0, 0, '1', '2', '3'} + want = append(want, make([]byte, 10)...) + if !bytes.Equal(b, want) { + t.Fatalf("unexpected content: got %v want %v", b, want) + } + }) +} diff --git a/internal/hybrid_cache/hybrid_cache.go b/internal/hybrid_cache/hybrid_cache.go new file mode 100644 index 000000000..c69147937 --- /dev/null +++ b/internal/hybrid_cache/hybrid_cache.go @@ -0,0 +1,239 @@ +package hybrid_cache + +import ( + "errors" + "io" + "runtime" + + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/mem" + "github.com/OpenListTeam/OpenList/v4/pkg/buffer" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" +) + +// 线程不安全，单线程使用，或者外部加锁保护 +type HybridCache struct { + blockSize uint64 + memoryStore mem.LinearMemory + memoryOffset uint64 + backingStore BackingStore + backingOffset uint64 + cleanup runtime.Cleanup +} + +// HybridCache本身是一个大的Block，支持分块成多个小的Block + +// 分配一个新的Block，支持读写，大小为size +func (hc *HybridCache) AllocBlock(size uint64) (buffer.Block, error) { +retry: + if hc.backingStore != nil { + if err := hc.backingStore.GrowTo(int64(hc.backingOffset + size)); err != nil { + return nil, err + } + base := hc.backingOffset + hc.backingOffset += size + fs := buffer.NewBlockAdapter( + io.NewOffsetWriter(hc.backingStore, int64(base)), + io.NewSectionReader(hc.backingStore, int64(base), int64(size)), + ) + return fs, nil + } + all, err := hc.memoryStore.Reallocate(hc.memoryOffset + size) + if err == nil { + start := hc.memoryOffset + hc.memoryOffset += size + return buffer.NewByteBlock(all[start : start+size]), nil + } + if err2 := hc.initFileCache(); err2 != nil { + return nil, errors.Join(err, err2) + } + goto retry +} + +func (hc *HybridCache) allocWriteAtSeeker(size uint64) (buffer.WriteAtSeeker, error) { +retry: + if hc.backingStore != nil { + if err := hc.backingStore.GrowTo(int64(hc.backingOffset + size)); err != nil { + return nil, err + } + base := hc.backingOffset + hc.backingOffset += size + return io.NewOffsetWriter(hc.backingStore, int64(base)), nil + } + all, err := hc.memoryStore.Reallocate(hc.memoryOffset + size) + if err == nil { + start := hc.memoryOffset + hc.memoryOffset += size + return io.NewOffsetWriter(buffer.NewByteBlock(all[start:start+size]), 0), nil + } + if err2 := hc.initFileCache(); err2 != nil { + return nil, errors.Join(err, err2) + } + goto retry +} + +func (hc *HybridCache) NextBlock() (buffer.Block, error) { + return hc.AllocBlock(hc.blockSize) +} + +func (hc *HybridCache) RewindBySize(size uint64) { + if hc.backingOffset >= size { + hc.backingOffset -= size + return + } + size -= hc.backingOffset + hc.backingOffset = 0 + if hc.memoryOffset >= size { + hc.memoryOffset -= size + return + } + size -= hc.memoryOffset + hc.memoryOffset = 0 +} + +func (hc *HybridCache) RewindOneBlock() { + hc.RewindBySize(hc.blockSize) +} + +func (hc *HybridCache) initFileCache() error { + file, err := NewFileStore(int64(hc.blockSize)) + if err != nil { + return err + } + hc.cleanup = runtime.AddCleanup(hc, func(file BackingStore) { + _ = file.Close() + }, file) + hc.backingStore = file + return nil +} + +func (hc *HybridCache) Close() error { + hc.cleanup.Stop() + var err error + if hc.memoryStore != nil { + err = hc.memoryStore.Free() + hc.memoryStore = nil + hc.memoryOffset = 0 + } + if hc.backingStore != nil { + err = errors.Join(err, hc.backingStore.Close()) + hc.backingStore = nil + hc.backingOffset = 0 + } + return err +} + +func (hc *HybridCache) Size() int64 { + return int64(hc.memoryOffset + hc.backingOffset) +} + +func (hc *HybridCache) ReadAt(p []byte, off int64) (n int, err error) { + if len(p) == 0 { + return 0, nil + } + if off < 0 || off >= hc.Size() { + return 0, io.EOF + } + + if off < int64(hc.memoryOffset) { + all, err := hc.memoryStore.Reallocate(min(hc.memoryOffset, uint64(off)+uint64(len(p)))) + if err != nil { + // 不可能失败 + panic(err) + } + n = copy(p, all[off:]) + if n == len(p) { + return n, nil + } + p = p[n:] + } + + off += int64(n) - int64(hc.memoryOffset) + canRead := int64(hc.backingOffset) - off + if canRead <= 0 { + return n, io.EOF + } + nn, err := hc.backingStore.ReadAt(p[:min(len(p), int(canRead))], off) + return n + nn, err +} + +func (hc *HybridCache) WriteAt(p []byte, off int64) (n int, err error) { + if len(p) == 0 { + return 0, nil + } + if off < 0 || off >= hc.Size() { + return 0, io.ErrShortWrite + } + + if off < int64(hc.memoryOffset) { + all, err := hc.memoryStore.Reallocate(min(hc.memoryOffset, uint64(off)+uint64(len(p)))) + if err != nil { + // 不可能失败 + panic(err) + } + n = copy(all[off:], p) + if n == len(p) { + return n, nil + } + p = p[n:] + } + + off += int64(n) - int64(hc.memoryOffset) + canWrite := int64(hc.backingOffset) - off + if canWrite <= 0 { + return n, io.ErrShortWrite + } + nn, err := hc.backingStore.WriteAt(p[:min(len(p), int(canWrite))], off) + return n + nn, err +} + +func (hc *HybridCache) CopyFromN(src io.Reader, n int64) (written int64, err error) { + limit := n + for limit > 0 { + blockSize := limit + if hc.backingStore == nil && blockSize > int64(conf.MaxBlockLimit) { + blockSize = int64(conf.MaxBlockLimit) + } + b, err := hc.allocWriteAtSeeker(uint64(blockSize)) + if err != nil { + return written, err + } + nn, err := utils.CopyWithBufferN(b, src, blockSize) + written += nn + if nn != blockSize { + return written, err + } + limit -= nn + } + return written, nil +} + +// HybridCache 线程不安全，单线程使用，或者外部加锁保护 +func NewHybridCache(blockSize, maxMemorySize uint64) (hc *HybridCache, err error) { + if conf.MinFreeMemory > 0 { + // 策略1: Go自动内存管理 + if maxMemorySize <= conf.AutoMemoryLimit { + return &HybridCache{backingStore: &BufferStore{}, blockSize: blockSize}, nil + } + + // 策略2: 手动内存管理 + if maxMemorySize >= blockSize { + var m mem.LinearMemory + // 手动管理内存，Uinx Mmap 或者 Windows VirtualAlloc + if m, err = mem.NewGuardedMemory(blockSize, maxMemorySize); err == nil { + hc = &HybridCache{memoryStore: m, blockSize: blockSize} + } + } + } + // 策略3: 文件后备 + if hc == nil { + hc = &HybridCache{blockSize: blockSize} + // 文件 + if err2 := hc.initFileCache(); err2 != nil { + return nil, errors.Join(err, err2) + } + } + return hc, nil +} + +var _ buffer.Block = (*HybridCache)(nil) diff --git a/internal/hybrid_cache/type.go b/internal/hybrid_cache/type.go new file mode 100644 index 000000000..edc6d3deb --- /dev/null +++ b/internal/hybrid_cache/type.go @@ -0,0 +1,13 @@ +package hybrid_cache + +import ( + "io" + + "github.com/OpenListTeam/OpenList/v4/pkg/buffer" +) + +type BackingStore interface { + buffer.Block + io.Closer + GrowTo(size int64) error +} diff --git a/internal/index115/model.go b/internal/index115/model.go index 8d8c885c5..287a13b81 100644 --- a/internal/index115/model.go +++ b/internal/index115/model.go @@ -10,6 +10,9 @@ type ShareSummary struct { FileCount int64 DirCount int64 UpdatedAt int64 + // Visible: search-only (bulk) shares are excluded from homepage/group + // listings; search and direct browse keep working for them. + Visible bool } // GroupInfo is one virtual directory rendered on the homepage. ID maps to the diff --git a/internal/index115/service.go b/internal/index115/service.go index 5d270a855..8294473c8 100644 --- a/internal/index115/service.go +++ b/internal/index115/service.go @@ -112,7 +112,7 @@ func (s *Service) listGroupMembers(ctx context.Context, gid int64) ([]FileItem, } items := make([]FileItem, 0) for _, share := range shares { - if share.GroupID != gid { + if share.GroupID != gid || !share.Visible { continue } items = append(items, newShareDirItem(share)) @@ -123,7 +123,7 @@ func (s *Service) listGroupMembers(ctx context.Context, gid int64) ([]FileItem, func looseShareItems(shares []ShareSummary) []FileItem { var items []FileItem for _, share := range shares { - if share.GroupID != 0 { + if share.GroupID != 0 || !share.Visible { continue } items = append(items, newShareDirItem(share)) diff --git a/internal/index115/service_test.go b/internal/index115/service_test.go index 3bb6ec284..a4575b991 100644 --- a/internal/index115/service_test.go +++ b/internal/index115/service_test.go @@ -9,7 +9,7 @@ import ( func TestServiceBrowseRootReturnsShares(t *testing.T) { svc := &Service{ store: stubStore{ - shares: []ShareSummary{{ShareCode: "sw1", ShareTitle: "S1", ReceiveCode: "rc1"}}, + shares: []ShareSummary{{ShareCode: "sw1", ShareTitle: "S1", ReceiveCode: "rc1", Visible: true}}, }, } @@ -132,8 +132,8 @@ func TestServiceBrowseRootListsGroupsThenLooseShares(t *testing.T) { store: stubStore{ groups: []GroupInfo{{ID: 1, Name: "欧美剧"}, {ID: 2, Name: "纪录片"}}, shares: []ShareSummary{ - {ShareCode: "swG", ShareTitle: "Grouped", GroupID: 1}, - {ShareCode: "swL", ShareTitle: "Loose", GroupID: 0}, + {ShareCode: "swG", ShareTitle: "Grouped", GroupID: 1, Visible: true}, + {ShareCode: "swL", ShareTitle: "Loose", GroupID: 0, Visible: true}, }, }, } @@ -160,9 +160,9 @@ func TestServiceBrowseGroupSentinelListsMembersOnly(t *testing.T) { svc := &Service{ store: stubStore{ shares: []ShareSummary{ - {ShareCode: "swG1", ShareTitle: "M1", GroupID: 1}, - {ShareCode: "swG2", ShareTitle: "M2", GroupID: 1}, - {ShareCode: "swO", ShareTitle: "Other", GroupID: 2}, + {ShareCode: "swG1", ShareTitle: "M1", GroupID: 1, Visible: true}, + {ShareCode: "swG2", ShareTitle: "M2", GroupID: 1, Visible: true}, + {ShareCode: "swO", ShareTitle: "Other", GroupID: 2, Visible: true}, }, }, } diff --git a/internal/index115/store.go b/internal/index115/store.go index 81a51b6e9..74d6132e7 100644 --- a/internal/index115/store.go +++ b/internal/index115/store.go @@ -17,6 +17,9 @@ type shareMeta struct { Status string LastCrawledAt int64 ID int64 + // Visible: search-only (bulk) shares have visible=0 and are excluded from + // homepage/group listings but stay reachable via search and direct browse. + Visible bool } type Store struct { @@ -43,9 +46,21 @@ func (s *Store) Close() error { } func (s *Store) RefreshShares(ctx context.Context) error { - rows, err := s.db.QueryContext(ctx, ` - SELECT id, share_code, COALESCE(receive_code, ''), COALESCE(share_title, ''), status, COALESCE(last_crawled_at, 0), COALESCE(group_id, 0) - FROM share`) + // The visible column ships with bulk-share (search-only) indexes. Older + // indexes lack it; treat every share there as browsable so a consumer + // upgrade can safely roll out before the index does. + hasVisible, err := s.shareHasVisibleColumn(ctx) + if err != nil { + return err + } + query := ` + SELECT id, share_code, COALESCE(receive_code, ''), COALESCE(share_title, ''), status, COALESCE(last_crawled_at, 0), COALESCE(group_id, 0)` + if hasVisible { + query += `, COALESCE(visible, 1)` + } + query += ` + FROM share` + rows, err := s.db.QueryContext(ctx, query) if err != nil { return err } @@ -54,9 +69,15 @@ func (s *Store) RefreshShares(ctx context.Context) error { shares := map[string]shareMeta{} for rows.Next() { var meta shareMeta - if err := rows.Scan(&meta.ID, &meta.ShareCode, &meta.ReceiveCode, &meta.ShareTitle, &meta.Status, &meta.LastCrawledAt, &meta.GroupID); err != nil { + var visible int64 = 1 + if hasVisible { + if err := rows.Scan(&meta.ID, &meta.ShareCode, &meta.ReceiveCode, &meta.ShareTitle, &meta.Status, &meta.LastCrawledAt, &meta.GroupID, &visible); err != nil { + return err + } + } else if err := rows.Scan(&meta.ID, &meta.ShareCode, &meta.ReceiveCode, &meta.ShareTitle, &meta.Status, &meta.LastCrawledAt, &meta.GroupID); err != nil { return err } + meta.Visible = visible != 0 current, ok := shares[meta.ShareCode] if !ok || preferShareMeta(meta, current) { shares[meta.ShareCode] = meta @@ -128,6 +149,17 @@ func (s *Store) RefreshShares(ctx context.Context) error { return nil } +// shareHasVisibleColumn reports whether the share table carries the visible +// column added by the bulk-shares indexer release. +func (s *Store) shareHasVisibleColumn(ctx context.Context) (bool, error) { + var n int + if err := s.db.QueryRowContext(ctx, + `SELECT count(*) FROM pragma_table_info('share') WHERE name = 'visible'`).Scan(&n); err != nil { + return false, err + } + return n == 1, nil +} + func preferShareMeta(next, current shareMeta) bool { if next.Status == "ACTIVE" && current.Status != "ACTIVE" { return true @@ -164,6 +196,9 @@ func (s *Store) ListShares(ctx context.Context) ([]ShareSummary, error) { item.GroupID = meta.GroupID item.ReceiveCode = meta.ReceiveCode item.ShareTitle = meta.ShareTitle + // A share with file rows but no share row is index corruption; default + // to browsable rather than silently hiding it. + item.Visible = meta.ShareCode == "" || meta.Visible if item.ShareTitle == "" { item.ShareTitle = item.ShareCode } diff --git a/internal/index115/visible_test.go b/internal/index115/visible_test.go new file mode 100644 index 000000000..740090030 --- /dev/null +++ b/internal/index115/visible_test.go @@ -0,0 +1,114 @@ +package index115 + +import ( + "context" + "path/filepath" + "testing" +) + +// TestRefreshSharesReadsVisibleColumn: an index that ships the visible column +// marks bulk (search-only) shares Visible=false while curated shares stay +// browsable. +func TestRefreshSharesReadsVisibleColumn(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "index.db") + store := openTestStore(t, dbPath) + if _, err := store.db.Exec(`ALTER TABLE share ADD COLUMN visible INTEGER NOT NULL DEFAULT 1`); err != nil { + t.Fatalf("add visible column: %v", err) + } + + insertTestShare(t, store.db, testShareRow{ShareCode: "swCur", ReceiveCode: "r1", ShareTitle: "Curated", Status: "ACTIVE"}) + insertTestShare(t, store.db, testShareRow{ShareCode: "swBulk", ReceiveCode: "r2", ShareTitle: "Bulk", Status: "ACTIVE"}) + if _, err := store.db.Exec(`UPDATE share SET visible = 0 WHERE share_code = 'swBulk'`); err != nil { + t.Fatal(err) + } + insertTestFile(t, store.db, testFileRow{FileID: "f1", ShareCode: "swCur", ParentID: "0", Name: "a.mkv", UpdatedAt: 1}) + insertTestFile(t, store.db, testFileRow{FileID: "f2", ShareCode: "swBulk", ParentID: "0", Name: "b.mkv", UpdatedAt: 2}) + + if err := store.RefreshShares(context.Background()); err != nil { + t.Fatalf("RefreshShares() error = %v", err) + } + if !store.shares["swCur"].Visible { + t.Fatalf("swCur.Visible = false, want true") + } + if store.shares["swBulk"].Visible { + t.Fatalf("swBulk.Visible = true, want false") + } + + summaries, err := store.ListShares(context.Background()) + if err != nil { + t.Fatalf("ListShares() error = %v", err) + } + for _, s := range summaries { + switch s.ShareCode { + case "swCur": + if !s.Visible { + t.Fatalf("ListShares swCur.Visible = false, want true") + } + case "swBulk": + if s.Visible { + t.Fatalf("ListShares swBulk.Visible = true, want false") + } + } + } + + // direct browse into the search-only share still works + items, err := store.ListChildren(context.Background(), "swBulk", "0") + if err != nil { + t.Fatalf("ListChildren(swBulk) error = %v", err) + } + if len(items) != 1 || items[0].Name != "b.mkv" { + t.Fatalf("swBulk children = %+v, want b.mkv (search-only share stays browsable directly)", items) + } +} + +// TestRefreshSharesDefaultsVisibleWithoutColumn: an older index without the +// visible column keeps every share browsable. +func TestRefreshSharesDefaultsVisibleWithoutColumn(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "index.db") + store := openTestStore(t, dbPath) // schema has no visible column + + insertTestShare(t, store.db, testShareRow{ShareCode: "sw1", ReceiveCode: "r1", ShareTitle: "S", Status: "ACTIVE"}) + insertTestFile(t, store.db, testFileRow{FileID: "f1", ShareCode: "sw1", ParentID: "0", Name: "a.mkv", UpdatedAt: 1}) + + if err := store.RefreshShares(context.Background()); err != nil { + t.Fatalf("RefreshShares() error = %v", err) + } + if !store.shares["sw1"].Visible { + t.Fatalf("sw1.Visible = false, want true fallback without column") + } +} + +// TestServiceBrowseHidesSearchOnlyShares: homepage and group listings exclude +// Visible=false shares. +func TestServiceBrowseHidesSearchOnlyShares(t *testing.T) { + svc := &Service{ + store: stubStore{ + groups: []GroupInfo{{ID: 1, Name: "欧美剧"}}, + shares: []ShareSummary{ + {ShareCode: "swBulkLoose", ShareTitle: "Hidden", GroupID: 0, Visible: false}, + {ShareCode: "swL", ShareTitle: "Loose", GroupID: 0, Visible: true}, + {ShareCode: "swBulkInGroup", ShareTitle: "HiddenMember", GroupID: 1, Visible: false}, + {ShareCode: "swG", ShareTitle: "Member", GroupID: 1, Visible: true}, + }, + }, + } + + items, err := svc.Browse(context.Background(), BrowseRequest{}) + if err != nil { + t.Fatalf("Browse() error = %v", err) + } + if len(items) != 2 { // group dir + swL + t.Fatalf("homepage items = %+v, want group + swL only", items) + } + if items[1].ShareCode != "swL" { + t.Fatalf("homepage loose item = %+v, want swL", items[1]) + } + + members, err := svc.Browse(context.Background(), BrowseRequest{ShareCode: "grp1"}) + if err != nil { + t.Fatalf("Browse(grp1) error = %v", err) + } + if len(members) != 1 || members[0].ShareCode != "swG" { + t.Fatalf("grp1 members = %+v, want [swG] only", members) + } +} diff --git a/internal/mem/mem_other.go b/internal/mem/mem_other.go new file mode 100644 index 000000000..2535c4570 --- /dev/null +++ b/internal/mem/mem_other.go @@ -0,0 +1,25 @@ +//go:build !unix && !windows + +package mem + +func NewMemory(cap, max uint64) (LinearMemory, error) { + return &sliceMemory{buf: make([]byte, 0, cap)}, nil +} + +type sliceMemory struct { + buf []byte +} + +func (b *sliceMemory) Free() error { + b.buf = nil + return nil +} + +func (b *sliceMemory) Reallocate(size uint64) ([]byte, error) { + if cap := uint64(cap(b.buf)); size > cap { + b.buf = append(b.buf[:cap], make([]byte, size-cap)...) + } else { + b.buf = b.buf[:size] + } + return b.buf, nil +} diff --git a/internal/mem/mem_unix.go b/internal/mem/mem_unix.go new file mode 100644 index 000000000..bd1979298 --- /dev/null +++ b/internal/mem/mem_unix.go @@ -0,0 +1,92 @@ +//go:build unix + +package mem + +import ( + "math" + + "golang.org/x/sys/unix" +) + +func NewMemory(cap, max uint64) (LinearMemory, error) { + // Round up to the page size. + rnd := uint64(unix.Getpagesize() - 1) + res := (max + rnd) &^ rnd + + if res > math.MaxInt { + // This ensures int(res) overflows to a negative value, + // and unix.Mmap returns EINVAL. + res = math.MaxUint64 + } + + com := res + prot := unix.PROT_READ | unix.PROT_WRITE + if cap < max { // Commit memory only if cap=max. + com = 0 + prot = unix.PROT_NONE + } + + // Reserve res bytes of address space, to ensure we won't need to move it. + // A protected, private, anonymous mapping should not commit memory. + b, err := unix.Mmap(-1, 0, int(res), prot, unix.MAP_PRIVATE|unix.MAP_ANON) + if err != nil { + return nil, err + } + return &mmappedMemory{buf: b[:com]}, nil +} + +// The slice covers the entire mmapped memory: +// - len(buf) is the already committed memory, +// - cap(buf) is the reserved address space. +type mmappedMemory struct { + buf []byte + growCheck GrowCheck +} + +func (m *mmappedMemory) SetGrowCheck(c GrowCheck) { + m.growCheck = c +} + +func (m *mmappedMemory) Reallocate(size uint64) ([]byte, error) { + com := uint64(len(m.buf)) + res := uint64(cap(m.buf)) + if com < size { + if size <= res { + // Grow geometrically, round up to the page size. + rnd := uint64(unix.Getpagesize() - 1) + new := com + com>>3 + new = min(max(size, new), res) + new = (new + rnd) &^ rnd + + if m.growCheck != nil { + if err := m.growCheck(new - com); err != nil { + return nil, err + } + } + + // Commit additional memory up to new bytes. + err := unix.Mprotect(m.buf[com:new], unix.PROT_READ|unix.PROT_WRITE) + if err != nil { + return nil, err + } + + m.buf = m.buf[:new] // Update committed memory. + } else { + return nil, ErrNotEnoughMemory + } + } + // Limit returned capacity because bytes beyond + // len(m.buf) have not yet been committed. + return m.buf[:size:len(m.buf)], nil +} + +func (m *mmappedMemory) Free() error { + if m.buf != nil { + err := unix.Munmap(m.buf[:cap(m.buf)]) + if err != nil { + return err + } + m.buf = nil + } + return nil +} diff --git a/internal/mem/mem_windows.go b/internal/mem/mem_windows.go new file mode 100644 index 000000000..e7a4bb27e --- /dev/null +++ b/internal/mem/mem_windows.go @@ -0,0 +1,94 @@ +package mem + +import ( + "math" + "unsafe" + + "golang.org/x/sys/windows" +) + +func NewMemory(cap, max uint64) (LinearMemory, error) { + // Round up to the page size. + rnd := uint64(windows.Getpagesize() - 1) + res := (max + rnd) &^ rnd + + if res > math.MaxInt { + // This ensures uintptr(res) overflows to a large value, + // and windows.VirtualAlloc returns an error. + res = math.MaxUint64 + } + + com := res + kind := windows.MEM_COMMIT + if cap < max { // Commit memory only if cap=max. + com = 0 + kind = windows.MEM_RESERVE + } + + // Reserve res bytes of address space, to ensure we won't need to move it. + r, err := windows.VirtualAlloc(0, uintptr(res), uint32(kind), windows.PAGE_READWRITE) + if err != nil { + return nil, err + } + + buf := unsafe.Slice((*byte)(unsafe.Pointer(r)), int(res)) + return &virtualMemory{addr: r, buf: buf[:com]}, nil +} + +// The slice covers the entire mmapped memory: +// - len(buf) is the already committed memory, +// - cap(buf) is the reserved address space. +type virtualMemory struct { + buf []byte + addr uintptr + growCheck GrowCheck +} + +func (m *virtualMemory) SetGrowCheck(c GrowCheck) { + m.growCheck = c +} + +func (m *virtualMemory) Reallocate(size uint64) ([]byte, error) { + com := uint64(len(m.buf)) + res := uint64(cap(m.buf)) + if com < size { + if size <= res { + // Grow geometrically, round up to the page size. + rnd := uint64(windows.Getpagesize() - 1) + new := com + com>>3 + new = min(max(size, new), res) + new = (new + rnd) &^ rnd + + if m.growCheck != nil { + if err := m.growCheck(new - com); err != nil { + return nil, err + } + } + + // Commit additional memory up to new bytes. + _, err := windows.VirtualAlloc(m.addr, uintptr(new), windows.MEM_COMMIT, windows.PAGE_READWRITE) + if err != nil { + return nil, err + } + + m.buf = m.buf[:new] // Update committed memory. + } else { + return nil, ErrNotEnoughMemory + } + } + // Limit returned capacity because bytes beyond + // len(m.buf) have not yet been committed. + return m.buf[:size:len(m.buf)], nil +} + +func (m *virtualMemory) Free() error { + if m.addr != 0 { + err := windows.VirtualFree(m.addr, 0, windows.MEM_RELEASE) + if err != nil { + return err + } + m.addr = 0 + m.buf = nil + } + return nil +} diff --git a/internal/mem/type.go b/internal/mem/type.go new file mode 100644 index 000000000..3ba8e35ca --- /dev/null +++ b/internal/mem/type.go @@ -0,0 +1,9 @@ +package mem + +type LinearMemory interface { + // 线程不安全 + Reallocate(size uint64) (all []byte, err error) + Free() error +} + +type GrowCheck func(growSize uint64) error diff --git a/internal/mem/utils.go b/internal/mem/utils.go new file mode 100644 index 000000000..ef82f9958 --- /dev/null +++ b/internal/mem/utils.go @@ -0,0 +1,87 @@ +package mem + +import ( + "errors" + "fmt" + "runtime" + "sync/atomic" + + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/pkg/singleflight" + "github.com/shirou/gopsutil/v4/mem" +) + +var ErrNotEnoughMemory = errors.New("not enough memory") + +func MemoryGrowCheck(growSize uint64) error { + if conf.MinFreeMemory == 0 { + return ErrNotEnoughMemory + } + r, err, _ := singleflight.AnyGroup.Do("MemoryGrowCheck", func() (any, error) { + m, err := mem.VirtualMemory() + if err != nil { + return nil, err + } + if m.Available < conf.MinFreeMemory { + return nil, ErrNotEnoughMemory + } + var res atomic.Uint64 + res.Store(m.Available) + return &res, nil + }) + if err != nil { + return err + } + res := r.(*atomic.Uint64) + for { + available := res.Load() + if available < growSize || available-growSize < conf.MinFreeMemory { + return ErrNotEnoughMemory + } + if res.CompareAndSwap(available, available-growSize) { + return nil + } + } +} + +func NewGuardedMemory(cap, max uint64) (m LinearMemory, err error) { + if err := MemoryGrowCheck(cap); err != nil { + return nil, err + } + defer func() { + if r := recover(); r != nil { + err = fmt.Errorf("%w: %v", ErrNotEnoughMemory, r) + } + }() + m, err = NewMemory(cap, max) + if err != nil { + return nil, err + } + if s, ok := m.(interface{ SetGrowCheck(GrowCheck) }); ok { + s.SetGrowCheck(MemoryGrowCheck) + } + gm := &guardedMemory{LinearMemory: m} + gm.cleanup = runtime.AddCleanup(gm, func(m LinearMemory) { + m.Free() + }, m) + return gm, nil +} + +type guardedMemory struct { + LinearMemory + cleanup runtime.Cleanup +} + +func (s *guardedMemory) Reallocate(size uint64) (all []byte, err error) { + defer func() { + if r := recover(); r != nil { + err = fmt.Errorf("%w: %v", ErrNotEnoughMemory, r) + } + }() + return s.LinearMemory.Reallocate(size) +} + +func (s *guardedMemory) Free() error { + s.cleanup.Stop() + return s.LinearMemory.Free() +} diff --git a/internal/model/file.go b/internal/model/file.go index 4ca7201e1..d6697cd0e 100644 --- a/internal/model/file.go +++ b/internal/model/file.go @@ -7,24 +7,26 @@ import ( // File is basic file level accessing interface type File interface { - io.Reader io.ReaderAt - io.Seeker + io.ReadSeeker +} +type FileWriter interface { + io.WriterAt + io.WriteSeeker } type FileCloser struct { File io.Closer } -func (f *FileCloser) Close() error { - var errs []error +func (f *FileCloser) Close() (err error) { if clr, ok := f.File.(io.Closer); ok { - errs = append(errs, clr.Close()) + err = clr.Close() } if f.Closer != nil { - errs = append(errs, f.Closer.Close()) + return errors.Join(err, f.Closer.Close()) } - return errors.Join(errs...) + return } // FileRangeReader 是对 RangeReaderIF 的轻量包装，表明由 RangeReaderIF.RangeRead diff --git a/internal/model/user.go b/internal/model/user.go index a9d766e00..2dacd752e 100644 --- a/internal/model/user.go +++ b/internal/model/user.go @@ -20,7 +20,15 @@ const ( ADMIN ) -const StaticHashSalt = "https://github.com/alist-org/alist" +const ( + StaticHashSalt = "https://github.com/alist-org/alist" + + InvalidUsernameOrPassword = "Invalid username or password" + Invalid2FACode = "Invalid 2FA code" + TooManyAttempts = "Too many unsuccessful sign-in attempts have been made using an incorrect username or password, Try again later." + GuestCannotUpdateProfile = "Guest user can not update profile" + GuestCannotGenerate2FA = "Guest user can not generate 2FA code" +) var LoginCache = cache.NewMemCache[int]() diff --git a/internal/multipart/session.go b/internal/multipart/session.go new file mode 100644 index 000000000..e25d1d1c7 --- /dev/null +++ b/internal/multipart/session.go @@ -0,0 +1,607 @@ +package multipart + +import ( + "context" + "errors" + "fmt" + "io" + "math" + "os" + stdpath "path" + "path/filepath" + "sync" + "sync/atomic" + "time" + + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/errs" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/op" + "github.com/OpenListTeam/OpenList/v4/internal/stream" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" + "github.com/google/uuid" +) + +type State string + +const ( + StateReceiving State = "receiving" + StateCompleted State = "completed" + StateFailedRetriable State = "failed_retriable" + StateFailedPermanent State = "failed_permanent" + StateAborted State = "aborted" +) + +// WindowSlots bounds the per-session disk footprint to WindowSlots*ChunkSize. +var WindowSlots = 8 + +const ( + // defaultSessionTTL is the sliding inactivity timeout; it also serves as the + // grace period during which finished sessions remain queryable. + defaultSessionTTL = 30 * time.Minute + gcInterval = time.Minute +) + +var ( + ErrSessionNotFound = errors.New("multipart upload session not found") + ErrNotOwner = errors.New("multipart upload session belongs to another user") + // errAborted wraps context.Canceled so a driver blocked on the stream sees + // the abort as a canceled request and runs its cancellation cleanup. + errAborted = fmt.Errorf("multipart upload aborted: %w", context.Canceled) +) + +// putFile is the pipeline tail: resolve the storage and run the regular upload +// path. It mirrors the checks of fs.putDirectly (internal/fs/put.go) but calls +// op.Put directly so the driver's progress callback can be observed. +// It is a variable so session tests can stub the storage layer out. +var putFile = func(ctx context.Context, dstDirPath string, fs *stream.FileStream, up driver.UpdateProgress) error { + storage, dstDirActualPath, err := op.GetStorageAndActualPath(dstDirPath) + if err != nil { + return err + } + if storage.Config().NoUpload { + return errs.UploadNotSupported + } + return op.Put(ctx, storage, dstDirActualPath, fs, up) +} + +// Session is one multipart upload: metadata survives pipeline attempts, the +// window (chunk data) does not. +type Session struct { + ID string + Path string // full destination path (dir + name), already user-joined + DstDir string + Name string + Size int64 + ChunkSize int64 + Total int + Mimetype string + Modified time.Time + Hashes map[*utils.HashType]string + Creator *model.User + + mu sync.Mutex + state State + err error + attempt int + win *Window + done chan struct{} + cancel context.CancelFunc + prevCRCs []uint32 + prevSet []bool + + storagePct atomic.Uint64 // math.Float64bits of the driver progress (0-100) + lastActive atomic.Int64 // unix nano +} + +func (s *Session) touch() { s.lastActive.Store(time.Now().UnixNano()) } + +func (s *Session) setStoragePct(p float64) { s.storagePct.Store(math.Float64bits(p)) } + +// Snapshot is the wire representation of a session used by all endpoints. +type SessionSnapshot struct { + ID string `json:"upload_id"` + State State `json:"state"` + Attempt int `json:"attempt"` + Path string `json:"path"` + Size int64 `json:"size"` + ChunkSize int64 `json:"chunk_size"` + TotalChunks int `json:"total_chunks"` + Received [][2]int `json:"received"` + ReceivedBytes int64 `json:"received_bytes"` + Frontier int `json:"frontier"` + StorageProgress float64 `json:"storage_progress"` + Error string `json:"error,omitempty"` +} + +func (s *Session) snapshotLocked() SessionSnapshot { + snap := SessionSnapshot{ + ID: s.ID, + State: s.state, + Attempt: s.attempt, + Path: s.Path, + Size: s.Size, + ChunkSize: s.ChunkSize, + TotalChunks: s.Total, + Received: [][2]int{}, + StorageProgress: math.Float64frombits(s.storagePct.Load()), + } + if s.err != nil { + snap.Error = s.err.Error() + } + switch { + case s.state == StateCompleted: + snap.Received = [][2]int{{0, s.Total - 1}} + snap.ReceivedBytes = s.Size + snap.Frontier = s.Total + snap.StorageProgress = 100 + case s.win != nil: + ws := s.win.Snapshot() + snap.Received = ws.Received + snap.ReceivedBytes = ws.ReceivedBytes + snap.Frontier = ws.Frontier + } + return snap +} + +func (s *Session) Snapshot() SessionSnapshot { + s.mu.Lock() + defer s.mu.Unlock() + return s.snapshotLocked() +} + +// InitReq carries everything the handler parsed from the init request. +type InitReq struct { + User *model.User + Path string // full destination path, already user-joined + Size int64 + ChunkSize int64 // final chunk size in bytes, already clamped by the handler + Mimetype string + Modified time.Time + Hashes map[*utils.HashType]string +} + +// Manager owns all live sessions. Sessions are in-memory only (aligned with +// upload tasks not being persisted); a restart drops them and the ring files +// are swept on the next start. +type Manager struct { + mu sync.Mutex + byID map[string]*Session + byKey map[string]string + gcOnce sync.Once + ttl time.Duration // 0 means defaultSessionTTL; tests shrink it per instance +} + +func (m *Manager) sessionTTL() time.Duration { + if m.ttl > 0 { + return m.ttl + } + return defaultSessionTTL +} + +var DefaultManager = &Manager{ + byID: make(map[string]*Session), + byKey: make(map[string]string), +} + +func (m *Manager) dir() string { + return filepath.Join(conf.Conf.TempDir, "multipart") +} + +func sessionKey(userID uint, path string, size int64) string { + return fmt.Sprintf("%d|%s|%d", userID, path, size) +} + +// hashesQualifyResume reports whether two hash sets prove the client is +// re-uploading the same file: they must share at least one hash type and +// agree on every shared one. Path+size alone is NOT enough to resume into a +// receiving session — buffered chunks of a different same-sized file would be +// silently mixed into the result. +func hashesQualifyResume(old, new map[*utils.HashType]string) bool { + shared := false + for t, ov := range old { + if nv, ok := new[t]; ok { + if ov != nv { + return false + } + shared = true + } + } + return shared +} + +// StartGC sweeps ring files orphaned by a previous run and starts the expiry +// loop. It is called at server startup so orphans are reclaimed even if no +// multipart upload ever happens again; Init also calls it, so embedders that +// skip the server wiring still get GC lazily. +func (m *Manager) StartGC() { + m.ensureGC() +} + +func (m *Manager) ensureGC() { + m.gcOnce.Do(func() { + // sweep ring files orphaned by a previous run; bootstrap's CleanTempDir + // only runs when no transfer tasks are pending, so do not rely on it + _ = os.RemoveAll(m.dir()) + go func() { + ticker := time.NewTicker(gcInterval) + for range ticker.C { + m.gc() + } + }() + }) +} + +func (m *Manager) gc() { + deadline := time.Now().Add(-m.sessionTTL()).UnixNano() + m.mu.Lock() + var expired []*Session + for _, s := range m.byID { + if s.lastActive.Load() < deadline { + expired = append(expired, s) + } + } + m.mu.Unlock() + for _, s := range expired { + m.terminate(s, errors.New("multipart upload session expired")) + } +} + +// terminate aborts a session (if still receiving) and drops it from the maps. +func (m *Manager) terminate(s *Session, cause error) { + s.mu.Lock() + if s.state == StateReceiving { + s.state = StateAborted + s.err = cause + } + s.killAttemptLocked() + s.mu.Unlock() + m.remove(s) +} + +// killAttemptLocked stops the running pipeline attempt: the context cancel +// interrupts drivers blocked on network I/O, and closing the window wakes a +// driver blocked in Read (context cancellation cannot interrupt cond.Wait). +// The caller must hold s.mu and must have set the final state first. +func (s *Session) killAttemptLocked() { + if s.cancel != nil { + s.cancel() + } + if s.win != nil { + _ = s.win.CloseWithError(errAborted) + } +} + +func (m *Manager) remove(s *Session) { + m.mu.Lock() + defer m.mu.Unlock() + delete(m.byID, s.ID) + key := sessionKey(s.Creator.ID, s.Path, s.Size) + if m.byKey[key] == s.ID { + delete(m.byKey, key) + } +} + +func (m *Manager) get(user *model.User, id string) (*Session, error) { + m.mu.Lock() + s, ok := m.byID[id] + m.mu.Unlock() + if !ok { + return nil, ErrSessionNotFound + } + if s.Creator.ID != user.ID { + return nil, ErrNotOwner + } + return s, nil +} + +// Init creates a session and starts its pipeline, or returns the live session +// for the same (user, path, size) so an interrupted client resumes implicitly. +func (m *Manager) Init(req InitReq) (SessionSnapshot, bool, error) { + m.ensureGC() + if req.Size <= 0 { + return SessionSnapshot{}, false, fmt.Errorf("multipart upload requires a positive X-File-Size, got %d", req.Size) + } + if req.ChunkSize <= 0 { + return SessionSnapshot{}, false, fmt.Errorf("invalid chunk size %d", req.ChunkSize) + } + key := sessionKey(req.User.ID, req.Path, req.Size) + + m.mu.Lock() + if id, ok := m.byKey[key]; ok { + if s, ok := m.byID[id]; ok { + s.mu.Lock() + st := s.state + s.mu.Unlock() + // failed_retriable resumes unconditionally: nothing of the old + // attempt survives except CRCs, and the re-fill CRC check catches + // a changed file. A receiving session still holds data, so it only + // resumes when hashes prove it is the same file. + if st == StateFailedRetriable || + (st == StateReceiving && hashesQualifyResume(s.Hashes, req.Hashes)) { + m.mu.Unlock() + s.touch() + return s.Snapshot(), true, nil + } + // finished session, or same path+size without proof of identity: + // drop the old session and start fresh + m.mu.Unlock() + m.terminate(s, errors.New("superseded by a new upload of the same path and size")) + m.mu.Lock() + } + } + m.mu.Unlock() + + dstDir, name := stdpath.Split(req.Path) + s := &Session{ + ID: uuid.NewString(), + Path: req.Path, + DstDir: dstDir, + Name: name, + Size: req.Size, + ChunkSize: req.ChunkSize, + Mimetype: req.Mimetype, + Modified: req.Modified, + Hashes: req.Hashes, + Creator: req.User, + state: StateReceiving, + } + s.touch() + + s.mu.Lock() + if err := m.startAttemptLocked(s); err != nil { + s.mu.Unlock() + return SessionSnapshot{}, false, err + } + s.Total = s.win.TotalChunks() + snap := s.snapshotLocked() + s.mu.Unlock() + + m.mu.Lock() + m.byID[s.ID] = s + m.byKey[key] = s.ID + m.mu.Unlock() + return snap, false, nil +} + +// startAttemptLocked builds a fresh window and spawns the pipeline goroutine. +// The caller must hold s.mu. +func (m *Manager) startAttemptLocked(s *Session) error { + win, err := NewWindow(m.dir(), fmt.Sprintf("%s.%d", s.ID, s.attempt), s.ChunkSize, s.Size, WindowSlots) + if err != nil { + return err + } + ctx, cancel := context.WithCancel(context.WithValue(context.Background(), conf.UserKey, s.Creator)) + done := make(chan struct{}) + s.win = win + s.cancel = cancel + s.done = done + s.state = StateReceiving + s.err = nil + s.setStoragePct(0) + + fileStream := &stream.FileStream{ + Obj: &model.Object{ + Name: s.Name, + Size: s.Size, + Modified: s.Modified, + HashInfo: utils.NewHashInfoByMap(s.Hashes), + }, + Reader: win, + Mimetype: s.Mimetype, + } + fileStream.Add(win) + + dstDir := s.DstDir + put := putFile // capture: the seam must not be read after spawn + go func() { + err := put(ctx, dstDir, fileStream, s.setStoragePct) + s.finishAttempt(win, err) + close(done) + }() + return nil +} + +// finishAttempt records the pipeline outcome and harvests the CRC table for +// re-fill verification. The window's data is gone at this point (op.Put closed +// it); only metadata survives. +func (s *Session) finishAttempt(win *Window, err error) { + crcs, set := win.CRCs() + _ = win.Close() // op.Put already closed it; make sure the ring file is gone anyway + + s.mu.Lock() + defer s.mu.Unlock() + s.touch() + if s.win == win { + s.win = nil + } + s.prevCRCs, s.prevSet = crcs, set + switch { + case err == nil: + s.state = StateCompleted + s.err = nil + s.setStoragePct(100) + case s.state != StateReceiving: + // Abort/expiry already labeled this attempt; keep that state. + if s.err == nil { + s.err = err + } + case isPermanentPutError(err): + s.state = StateFailedPermanent + s.err = err + default: + s.state = StateFailedRetriable + s.err = err + s.attempt++ + } +} + +func isPermanentPutError(err error) bool { + if errors.Is(err, context.Canceled) { + return false + } + for _, target := range []error{ + errs.UploadNotSupported, + errs.PermissionDenied, + errs.StorageNotFound, + errs.ObjectAlreadyExists, + errs.RelativePath, + errs.IgnoredSystemFile, + } { + if errors.Is(err, target) { + return true + } + } + return false +} + +// Chunk feeds one chunk into the session. Re-sending chunk 0 to a +// failed_retriable session re-fills it: a fresh window and pipeline attempt. +func (m *Manager) Chunk(user *model.User, id string, idx int, body io.Reader) (SessionSnapshot, error) { + s, err := m.get(user, id) + if err != nil { + return SessionSnapshot{}, err + } + s.touch() + + s.mu.Lock() + switch s.state { + case StateReceiving: + case StateFailedRetriable: + if idx != 0 { + snap := s.snapshotLocked() + s.mu.Unlock() + return snap, fmt.Errorf("upload attempt failed, resend from chunk 0 to retry: %w", s.err) + } + if err := m.startAttemptLocked(s); err != nil { + snap := s.snapshotLocked() + s.mu.Unlock() + return snap, err + } + case StateCompleted: + snap := s.snapshotLocked() + s.mu.Unlock() + return snap, nil // idempotent: stragglers after rapid-upload/finish succeed + default: + snap := s.snapshotLocked() + s.mu.Unlock() + return snap, fmt.Errorf("session is %s: %w", s.state, s.err) + } + win := s.win + var prevCRC uint32 + hasPrev := false + if idx < len(s.prevSet) && s.prevSet[idx] { + prevCRC, hasPrev = s.prevCRCs[idx], true + } + s.mu.Unlock() + + crc, err := win.WriteChunk(idx, body) + if err != nil { + // A closed window means the pipeline ended while this chunk was in + // flight — rapid upload makes this the NORMAL case: the driver + // succeeds off the hash alone with chunks still arriving. The window + // closes (op.Put's defer) moments before the verdict is recorded, so + // wait for the verdict instead of racing it, then absorb the chunk + // idempotently if the upload in fact succeeded. + if errors.Is(err, ErrClosed) || errors.Is(err, context.Canceled) { + s.mu.Lock() + done := s.done + s.mu.Unlock() + select { + case <-done: + case <-time.After(10 * time.Second): // pipeline teardown is µs-scale; never expected + } + } + s.mu.Lock() + completed := s.state == StateCompleted + snap := s.snapshotLocked() + s.mu.Unlock() + if completed { + return snap, nil + } + return snap, err + } + if hasPrev && crc != prevCRC { + err := fmt.Errorf("chunk %d content changed between attempts, aborting", idx) + s.mu.Lock() + s.state = StateFailedPermanent + s.err = err + s.killAttemptLocked() + s.mu.Unlock() + return s.Snapshot(), err + } + return s.Snapshot(), nil +} + +// Complete waits for the pipeline outcome. It refuses to block while chunks +// are still missing, so a buggy client cannot park a connection for the TTL. +func (m *Manager) Complete(ctx context.Context, user *model.User, id string) (SessionSnapshot, error) { + s, err := m.get(user, id) + if err != nil { + return SessionSnapshot{}, err + } + s.touch() + for { + s.mu.Lock() + st := s.state + done := s.done + if st == StateReceiving && s.win != nil { + if ws := s.win.Snapshot(); ws.ReceivedBytes < s.Size { + snap := s.snapshotLocked() + s.mu.Unlock() + return snap, fmt.Errorf("cannot complete: %d of %d bytes received", ws.ReceivedBytes, s.Size) + } + } + s.mu.Unlock() + if st != StateReceiving { + break + } + select { + case <-done: + case <-ctx.Done(): + return s.Snapshot(), ctx.Err() + } + } + s.touch() + s.mu.Lock() + st, serr := s.state, s.err + snap := s.snapshotLocked() + s.mu.Unlock() + if st == StateCompleted { + m.remove(s) // served its purpose; frees the key for future uploads + return snap, nil + } + return snap, fmt.Errorf("upload failed (%s): %w", st, serr) +} + +// Status looks a session up by id. +func (m *Manager) Status(user *model.User, id string) (SessionSnapshot, error) { + s, err := m.get(user, id) + if err != nil { + return SessionSnapshot{}, err + } + s.touch() + return s.Snapshot(), nil +} + +// Find looks a live session up by destination path and size, for resume discovery. +func (m *Manager) Find(user *model.User, path string, size int64) (SessionSnapshot, error) { + m.mu.Lock() + id, ok := m.byKey[sessionKey(user.ID, path, size)] + m.mu.Unlock() + if !ok { + return SessionSnapshot{}, ErrSessionNotFound + } + return m.Status(user, id) +} + +// Abort cancels the pipeline and forgets the session immediately. +func (m *Manager) Abort(user *model.User, id string) error { + s, err := m.get(user, id) + if err != nil { + return err + } + m.terminate(s, errors.New("multipart upload aborted by client")) + return nil +} diff --git a/internal/multipart/session_test.go b/internal/multipart/session_test.go new file mode 100644 index 000000000..282d6a488 --- /dev/null +++ b/internal/multipart/session_test.go @@ -0,0 +1,650 @@ +package multipart + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "os" + "sync" + "testing" + "time" + + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/errs" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/stream" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" +) + +// Tests in this file share the putFile seam and must not run in parallel. + +func setupSessionTest(t *testing.T) *Manager { + t.Helper() + oldConf := conf.Conf + conf.Conf = conf.DefaultConfig(t.TempDir()) + t.Cleanup(func() { conf.Conf = oldConf }) + return &Manager{byID: make(map[string]*Session), byKey: make(map[string]string)} +} + +func stubPut(t *testing.T, fn func(ctx context.Context, dst string, fs *stream.FileStream, up driver.UpdateProgress) error) { + t.Helper() + orig := putFile + putFile = fn + t.Cleanup(func() { putFile = orig }) +} + +func testUser() *model.User { return &model.User{ID: 7, Username: "tester"} } + +func initReq(user *model.User, size, chunkSize int64) InitReq { + return InitReq{ + User: user, + Path: "/local/test.bin", + Size: size, + ChunkSize: chunkSize, + Mimetype: "application/octet-stream", + Modified: time.Unix(1700000000, 0), + } +} + +func sendChunk(t *testing.T, m *Manager, user *model.User, id string, data []byte, idx int, chunkSize int64) SessionSnapshot { + t.Helper() + snap, err := m.Chunk(user, id, idx, bytes.NewReader(chunkOf(data, idx, chunkSize))) + if err != nil { + t.Fatalf("Chunk(%d): %v", idx, err) + } + return snap +} + +func waitState(t *testing.T, m *Manager, user *model.User, id string, want State) SessionSnapshot { + t.Helper() + deadline := time.Now().Add(10 * time.Second) + for { + snap, err := m.Status(user, id) + if err != nil { + t.Fatalf("Status: %v", err) + } + if snap.State == want { + return snap + } + if time.Now().After(deadline) { + t.Fatalf("session state = %s, want %s (err: %s)", snap.State, want, snap.Error) + } + time.Sleep(2 * time.Millisecond) + } +} + +func TestSessionHappyPath(t *testing.T) { + m := setupSessionTest(t) + const chunkSize = 1024 + totalSize := int64(3*chunkSize + 300) + data := genData(totalSize) + user := testUser() + + got := make(chan []byte, 1) + stubPut(t, func(ctx context.Context, dst string, fs *stream.FileStream, up driver.UpdateProgress) error { + defer fs.Close() + if dst != "/local/" { + return fmt.Errorf("unexpected dst dir %q", dst) + } + if fs.GetName() != "test.bin" || fs.GetSize() != totalSize { + return fmt.Errorf("unexpected stream meta %s/%d", fs.GetName(), fs.GetSize()) + } + b, err := io.ReadAll(fs) + if err != nil { + return err + } + up(100) + got <- b + return nil + }) + + snap, resumed, err := m.Init(initReq(user, totalSize, chunkSize)) + if err != nil || resumed { + t.Fatalf("Init = (resumed=%v, err=%v)", resumed, err) + } + if snap.TotalChunks != 4 || snap.State != StateReceiving { + t.Fatalf("init snapshot = %+v", snap) + } + for _, idx := range []int{1, 0, 3, 2} { // out of order on purpose + sendChunk(t, m, user, snap.ID, data, idx, chunkSize) + } + final, err := m.Complete(context.Background(), user, snap.ID) + if err != nil { + t.Fatalf("Complete: %v", err) + } + if final.State != StateCompleted || final.StorageProgress != 100 { + t.Fatalf("final snapshot = %+v", final) + } + if !bytes.Equal(<-got, data) { + t.Fatal("driver received different bytes") + } + if _, err := m.Status(user, snap.ID); !errors.Is(err, ErrSessionNotFound) { + t.Fatalf("session should be removed after Complete, got %v", err) + } +} + +func TestSessionRetriableRefill(t *testing.T) { + m := setupSessionTest(t) + const chunkSize = 1024 + totalSize := int64(2 * chunkSize) + data := genData(totalSize) + user := testUser() + + attempts := 0 + got := make(chan []byte, 1) + stubPut(t, func(ctx context.Context, dst string, fs *stream.FileStream, up driver.UpdateProgress) error { + defer fs.Close() + attempts++ + if attempts == 1 { + buf := make([]byte, chunkSize) + if _, err := io.ReadFull(fs, buf); err != nil { + return err + } + return errors.New("transient storage hiccup") + } + b, err := io.ReadAll(fs) + if err != nil { + return err + } + got <- b + return nil + }) + + snap, _, err := m.Init(initReq(user, totalSize, chunkSize)) + if err != nil { + t.Fatalf("Init: %v", err) + } + sendChunk(t, m, user, snap.ID, data, 0, chunkSize) + failed := waitState(t, m, user, snap.ID, StateFailedRetriable) + if failed.Attempt != 1 { + t.Fatalf("attempt = %d, want 1", failed.Attempt) + } + if len(failed.Received) != 0 { + t.Fatalf("failed session must report nothing received, got %v", failed.Received) + } + + // chunks other than 0 are rejected until the client restarts the fill + if _, err := m.Chunk(user, snap.ID, 1, bytes.NewReader(chunkOf(data, 1, chunkSize))); err == nil { + t.Fatal("chunk 1 on failed_retriable session: expected error") + } + // re-fill from chunk 0 respawns the pipeline + sendChunk(t, m, user, snap.ID, data, 0, chunkSize) + sendChunk(t, m, user, snap.ID, data, 1, chunkSize) + final, err := m.Complete(context.Background(), user, snap.ID) + if err != nil { + t.Fatalf("Complete after refill: %v", err) + } + if final.State != StateCompleted || attempts != 2 { + t.Fatalf("state=%s attempts=%d, want completed/2", final.State, attempts) + } + if !bytes.Equal(<-got, data) { + t.Fatal("driver received different bytes after refill") + } +} + +func TestSessionPermanentFailure(t *testing.T) { + m := setupSessionTest(t) + const chunkSize = 1024 + data := genData(chunkSize) + user := testUser() + + stubPut(t, func(ctx context.Context, dst string, fs *stream.FileStream, up driver.UpdateProgress) error { + defer fs.Close() + return fmt.Errorf("denied: %w", errs.PermissionDenied) + }) + + snap, _, err := m.Init(initReq(user, chunkSize, chunkSize)) + if err != nil { + t.Fatalf("Init: %v", err) + } + waitState(t, m, user, snap.ID, StateFailedPermanent) + if _, err := m.Chunk(user, snap.ID, 0, bytes.NewReader(data)); err == nil { + t.Fatal("chunk on failed_permanent session: expected error") + } + if _, err := m.Complete(context.Background(), user, snap.ID); err == nil { + t.Fatal("Complete on failed_permanent session: expected error") + } +} + +func TestRefillCRCMismatch(t *testing.T) { + m := setupSessionTest(t) + const chunkSize = 1024 + totalSize := int64(2 * chunkSize) + data := genData(totalSize) + user := testUser() + + attempts := 0 + stubPut(t, func(ctx context.Context, dst string, fs *stream.FileStream, up driver.UpdateProgress) error { + defer fs.Close() + attempts++ + if attempts == 1 { + buf := make([]byte, chunkSize) + if _, err := io.ReadFull(fs, buf); err != nil { + return err + } + return errors.New("transient") + } + _, err := io.ReadAll(fs) + return err + }) + + snap, _, err := m.Init(initReq(user, totalSize, chunkSize)) + if err != nil { + t.Fatalf("Init: %v", err) + } + sendChunk(t, m, user, snap.ID, data, 0, chunkSize) + waitState(t, m, user, snap.ID, StateFailedRetriable) + + tampered := genData(chunkSize + 5)[:chunkSize] // different content, same length + if _, err := m.Chunk(user, snap.ID, 0, bytes.NewReader(tampered)); err == nil { + t.Fatal("re-fill with changed content: expected error") + } + waitState(t, m, user, snap.ID, StateFailedPermanent) +} + +func TestCompleteRefusesIncomplete(t *testing.T) { + m := setupSessionTest(t) + const chunkSize = 1024 + totalSize := int64(3 * chunkSize) + data := genData(totalSize) + user := testUser() + + stubPut(t, func(ctx context.Context, dst string, fs *stream.FileStream, up driver.UpdateProgress) error { + defer fs.Close() + _, err := io.ReadAll(fs) + return err + }) + + snap, _, err := m.Init(initReq(user, totalSize, chunkSize)) + if err != nil { + t.Fatalf("Init: %v", err) + } + sendChunk(t, m, user, snap.ID, data, 0, chunkSize) + if _, err := m.Complete(context.Background(), user, snap.ID); err == nil { + t.Fatal("Complete with missing chunks: expected error") + } + // unblock the pipeline goroutine before the test tears down + if err := m.Abort(user, snap.ID); err != nil { + t.Fatalf("Abort: %v", err) + } +} + +func TestAbortAndOwnership(t *testing.T) { + m := setupSessionTest(t) + const chunkSize = 1024 + totalSize := int64(4 * chunkSize) + data := genData(totalSize) + user := testUser() + + stubPut(t, func(ctx context.Context, dst string, fs *stream.FileStream, up driver.UpdateProgress) error { + defer fs.Close() + _, err := io.ReadAll(fs) + return err + }) + + snap, _, err := m.Init(initReq(user, totalSize, chunkSize)) + if err != nil { + t.Fatalf("Init: %v", err) + } + sendChunk(t, m, user, snap.ID, data, 0, chunkSize) + + stranger := &model.User{ID: 99, Username: "stranger"} + if _, err := m.Status(stranger, snap.ID); !errors.Is(err, ErrNotOwner) { + t.Fatalf("stranger Status err = %v, want ErrNotOwner", err) + } + if err := m.Abort(stranger, snap.ID); !errors.Is(err, ErrNotOwner) { + t.Fatalf("stranger Abort err = %v, want ErrNotOwner", err) + } + if err := m.Abort(user, snap.ID); err != nil { + t.Fatalf("Abort: %v", err) + } + if _, err := m.Status(user, snap.ID); !errors.Is(err, ErrSessionNotFound) { + t.Fatalf("Status after Abort err = %v, want ErrSessionNotFound", err) + } +} + +func TestExpiry(t *testing.T) { + m := setupSessionTest(t) + m.ttl = 30 * time.Millisecond + + const chunkSize = 1024 + user := testUser() + stubPut(t, func(ctx context.Context, dst string, fs *stream.FileStream, up driver.UpdateProgress) error { + defer fs.Close() + _, err := io.ReadAll(fs) + return err + }) + + snap, _, err := m.Init(initReq(user, 4*chunkSize, chunkSize)) + if err != nil { + t.Fatalf("Init: %v", err) + } + time.Sleep(60 * time.Millisecond) + m.gc() + if _, err := m.Status(user, snap.ID); !errors.Is(err, ErrSessionNotFound) { + t.Fatalf("Status after expiry err = %v, want ErrSessionNotFound", err) + } + // the terminated pipeline must release the ring file + deadline := time.Now().Add(5 * time.Second) + for { + entries, _ := os.ReadDir(m.dir()) + if len(entries) == 0 { + break + } + if time.Now().After(deadline) { + names := make([]string, 0, len(entries)) + for _, e := range entries { + names = append(names, e.Name()) + } + t.Fatalf("ring files not cleaned up after expiry: %v", names) + } + time.Sleep(5 * time.Millisecond) + } +} + +func TestInitResume(t *testing.T) { + m := setupSessionTest(t) + const chunkSize = 1024 + totalSize := int64(4 * chunkSize) + data := genData(totalSize) + user := testUser() + + stubPut(t, func(ctx context.Context, dst string, fs *stream.FileStream, up driver.UpdateProgress) error { + defer fs.Close() + _, err := io.ReadAll(fs) + return err + }) + + hashed := initReq(user, totalSize, chunkSize) + hashed.Hashes = map[*utils.HashType]string{utils.MD5: "0123456789abcdef0123456789abcdef"} + snap, _, err := m.Init(hashed) + if err != nil { + t.Fatalf("Init: %v", err) + } + sendChunk(t, m, user, snap.ID, data, 2, chunkSize) + + // same hashes prove the same file: resume with buffered chunks skippable + again, resumed, err := m.Init(hashed) + if err != nil || !resumed { + t.Fatalf("hashed re-Init = (resumed=%v, err=%v), want resumed", resumed, err) + } + if again.ID != snap.ID { + t.Fatalf("resumed session id = %s, want %s", again.ID, snap.ID) + } + if len(again.Received) != 1 || again.Received[0] != [2]int{2, 2} { + t.Fatalf("resumed received = %v, want [[2,2]]", again.Received) + } + + // a different size is a different upload + other, resumed, err := m.Init(initReq(user, totalSize+1, chunkSize)) + if err != nil || resumed || other.ID == snap.ID { + t.Fatalf("different-size Init = (id=%s, resumed=%v, err=%v)", other.ID, resumed, err) + } + + if _, err := m.Find(user, "/local/test.bin", totalSize); err != nil { + t.Fatalf("Find: %v", err) + } + if _, err := m.Find(user, "/local/nope.bin", totalSize); !errors.Is(err, ErrSessionNotFound) { + t.Fatalf("Find miss err = %v, want ErrSessionNotFound", err) + } + + // without hashes, path+size cannot prove identity against a receiving + // session holding buffered data — the old session must be dropped, or a + // same-sized different file would be silently mixed into the result + bare := initReq(user, totalSize, chunkSize) + fresh, resumed, err := m.Init(bare) + if err != nil || resumed || fresh.ID == snap.ID { + t.Fatalf("bare re-Init = (id=%s, resumed=%v, err=%v), want a fresh session", fresh.ID, resumed, err) + } + if _, err := m.Status(user, snap.ID); !errors.Is(err, ErrSessionNotFound) { + t.Fatalf("superseded session err = %v, want ErrSessionNotFound", err) + } +} + +func TestResumeFailedRetriableWithoutHash(t *testing.T) { + m := setupSessionTest(t) + const chunkSize = 1024 + totalSize := int64(2 * chunkSize) + data := genData(totalSize) + user := testUser() + + attempts := 0 + stubPut(t, func(ctx context.Context, dst string, fs *stream.FileStream, up driver.UpdateProgress) error { + defer fs.Close() + attempts++ + if attempts == 1 { + buf := make([]byte, chunkSize) + if _, err := io.ReadFull(fs, buf); err != nil { + return err + } + return errors.New("transient") + } + _, err := io.ReadAll(fs) + return err + }) + + snap, _, err := m.Init(initReq(user, totalSize, chunkSize)) + if err != nil { + t.Fatalf("Init: %v", err) + } + sendChunk(t, m, user, snap.ID, data, 0, chunkSize) + waitState(t, m, user, snap.ID, StateFailedRetriable) + + // a failed_retriable session keeps no chunk data, so identity proof is not + // required to resume: the retry-button flow works without rapid hashing, + // and the re-fill CRC check still catches a changed file + again, resumed, err := m.Init(initReq(user, totalSize, chunkSize)) + if err != nil || !resumed || again.ID != snap.ID { + t.Fatalf("re-Init on failed_retriable = (id=%s, resumed=%v, err=%v), want resumed same session", again.ID, resumed, err) + } + sendChunk(t, m, user, snap.ID, data, 0, chunkSize) + sendChunk(t, m, user, snap.ID, data, 1, chunkSize) + if _, err := m.Complete(context.Background(), user, snap.ID); err != nil { + t.Fatalf("Complete after hashless refill: %v", err) + } +} + +func TestRapidUploadShortCircuit(t *testing.T) { + m := setupSessionTest(t) + const chunkSize = 1024 + totalSize := int64(6 * chunkSize) + data := genData(totalSize) + user := testUser() + + wantMD5 := "0123456789abcdef0123456789abcdef" + wantSHA1 := "da39a3ee5e6b4b0d3255bfef95601890afd80709" + stubPut(t, func(ctx context.Context, dst string, fs *stream.FileStream, up driver.UpdateProgress) error { + defer fs.Close() + // the hash provided at init must reach the driver through the stream, + // that is what lets PutRapid-style drivers skip the transfer entirely + if got := fs.GetHash().GetHash(utils.MD5); got != wantMD5 { + return fmt.Errorf("md5 not propagated: %q", got) + } + if got := fs.GetHash().GetHash(utils.SHA1); got != wantSHA1 { + return fmt.Errorf("sha1 not propagated: %q", got) + } + time.Sleep(50 * time.Millisecond) // simulated rapid-upload API round trip + up(100) + return nil // rapid upload hit: succeed without reading the stream + }) + + req := initReq(user, totalSize, chunkSize) + req.Hashes = map[*utils.HashType]string{utils.MD5: wantMD5, utils.SHA1: wantSHA1} + snap, _, err := m.Init(req) + if err != nil { + t.Fatalf("Init: %v", err) + } + + // spam chunks across the completion moment: some land while receiving, + // some race the window close, some arrive after completion — with the + // completed-session absorption none of them may surface an error + var wg sync.WaitGroup + errCh := make(chan error, 64) + for w := 0; w < 4; w++ { + wg.Add(1) + go func(w int) { + defer wg.Done() + for round := 0; round < 8; round++ { + idx := (w*8 + round) % 6 + _, err := m.Chunk(user, snap.ID, idx, bytes.NewReader(chunkOf(data, idx, chunkSize))) + // flow-control signals are part of the protocol, not failures + if err != nil && !errors.Is(err, ErrChunkInFlight) && !errors.Is(err, ErrOutOfWindow) { + errCh <- fmt.Errorf("worker %d chunk %d: %w", w, idx, err) + return + } + time.Sleep(5 * time.Millisecond) + } + }(w) + } + wg.Wait() + close(errCh) + for err := range errCh { + t.Error(err) + } + + final, err := m.Complete(context.Background(), user, snap.ID) + if err != nil { + t.Fatalf("Complete: %v", err) + } + if final.State != StateCompleted || final.StorageProgress != 100 { + t.Fatalf("final snapshot = %+v", final) + } +} + +// TestChunkRacesRapidCompletion pins the exact race the completed-session +// absorption exists for: a chunk request grabs the live window, stalls while +// receiving its body, the pipeline completes off the hash alone (rapid +// upload) and closes the window — the stalled chunk must then succeed +// idempotently instead of surfacing "window closed" to a client whose upload +// in fact just finished. +func TestChunkRacesRapidCompletion(t *testing.T) { + m := setupSessionTest(t) + const chunkSize = 1024 + totalSize := int64(2 * chunkSize) + data := genData(totalSize) + user := testUser() + + proceed := make(chan struct{}) + stubPut(t, func(ctx context.Context, dst string, fs *stream.FileStream, up driver.UpdateProgress) error { + defer fs.Close() + <-proceed // the rapid-upload verdict arrives when the test says so + return nil + }) + + snap, _, err := m.Init(initReq(user, totalSize, chunkSize)) + if err != nil { + t.Fatalf("Init: %v", err) + } + + gate := make(chan struct{}) + type result struct { + snap SessionSnapshot + err error + } + resCh := make(chan result, 1) + go func() { + s, e := m.Chunk(user, snap.ID, 0, + &gatedReader{release: gate, inner: bytes.NewReader(chunkOf(data, 0, chunkSize))}) + resCh <- result{s, e} + }() + + // wait until the chunk writer holds slot 0 in the filling state + m.mu.Lock() + sess := m.byID[snap.ID] + m.mu.Unlock() + sess.mu.Lock() + win := sess.win + sess.mu.Unlock() + deadline := time.Now().Add(5 * time.Second) + for { + win.mu.Lock() + filling := win.slotState[0] == slotFilling + win.mu.Unlock() + if filling { + break + } + if time.Now().After(deadline) { + t.Fatal("chunk writer never reached the filling state") + } + time.Sleep(time.Millisecond) + } + + close(proceed) // rapid upload succeeds, window closes + waitState(t, m, user, snap.ID, StateCompleted) // completion recorded + close(gate) // stalled chunk finishes against the closed window + + res := <-resCh + if res.err != nil { + t.Fatalf("in-flight chunk across rapid completion must be absorbed, got: %v", res.err) + } + if res.snap.State != StateCompleted { + t.Fatalf("absorbed chunk snapshot state = %s, want completed", res.snap.State) + } +} + +// TestChunkDuringCompletionGap covers the moment between op.Put closing the +// window (its defer) and the verdict being recorded: a chunk hitting the +// closed window inside that gap must wait for the verdict and be absorbed, +// not bounce a "window closed" error at a client whose upload just succeeded. +func TestChunkDuringCompletionGap(t *testing.T) { + m := setupSessionTest(t) + const chunkSize = 1024 + totalSize := int64(2 * chunkSize) + data := genData(totalSize) + user := testUser() + + windowClosed := make(chan struct{}) + allowVerdict := make(chan struct{}) + stubPut(t, func(ctx context.Context, dst string, fs *stream.FileStream, up driver.UpdateProgress) error { + _ = fs.Close() // what op.Put's defer does before Put returns + close(windowClosed) + <-allowVerdict // hold the pipeline return open: this IS the gap + return nil + }) + + snap, _, err := m.Init(initReq(user, totalSize, chunkSize)) + if err != nil { + t.Fatalf("Init: %v", err) + } + <-windowClosed + + type result struct { + snap SessionSnapshot + err error + } + resCh := make(chan result, 1) + go func() { + s, e := m.Chunk(user, snap.ID, 0, bytes.NewReader(chunkOf(data, 0, chunkSize))) + resCh <- result{s, e} + }() + + select { + case r := <-resCh: + t.Fatalf("chunk inside the gap returned early with (%s, %v); it must wait for the verdict", r.snap.State, r.err) + case <-time.After(150 * time.Millisecond): + // still waiting on the verdict, as designed + } + + close(allowVerdict) + select { + case r := <-resCh: + if r.err != nil { + t.Fatalf("gap chunk must be absorbed after completion, got: %v", r.err) + } + if r.snap.State != StateCompleted { + t.Fatalf("gap chunk snapshot state = %s, want completed", r.snap.State) + } + case <-time.After(5 * time.Second): + t.Fatal("gap chunk never returned after the verdict") + } +} + +func TestInitRejectsBadSize(t *testing.T) { + m := setupSessionTest(t) + if _, _, err := m.Init(initReq(testUser(), 0, 1024)); err == nil { + t.Fatal("Init with size 0: expected error") + } +} diff --git a/internal/multipart/window.go b/internal/multipart/window.go new file mode 100644 index 000000000..01e726701 --- /dev/null +++ b/internal/multipart/window.go @@ -0,0 +1,377 @@ +package multipart + +import ( + "errors" + "fmt" + "hash/crc32" + "io" + "os" + "path/filepath" + "sync" + "time" + + "github.com/OpenListTeam/OpenList/v4/pkg/utils" +) + +const ( + slotFree uint8 = iota + slotFilling + slotReady +) + +var ( + // ErrClosed is the sticky error after Close; all pending and future reads/writes fail with it. + ErrClosed = errors.New("multipart upload window closed") + // ErrChunkInFlight means another request is uploading the same chunk right now. + ErrChunkInFlight = errors.New("chunk is being uploaded by another request") + // ErrOutOfWindow means the chunk is still too far ahead of the consumption + // frontier after waiting WindowWaitTimeout; the client should back off and + // resend it later (flow control, not a failure). + ErrOutOfWindow = errors.New("chunk is out of the receiving window") +) + +// WindowWaitTimeout bounds how long WriteChunk blocks waiting for its slot. +// Browsers cannot reliably read responses sent before the request body is +// consumed (they report a network error), so under backpressure it is far +// better to hold the request until a slot frees — the wait must just stay +// well below CDN request deadlines (Cloudflare: ~100s). Tests shrink this. +var WindowWaitTimeout = 10 * time.Second + +// Window reassembles concurrently uploaded chunks into a sequential stream. +// Chunks land in a ring file of slots*chunkSize bytes (chunk i -> slot i%slots), +// and Read serves bytes in order, blocking until the next needed chunk arrives. +// A chunk slot is released as soon as the reader crosses its boundary, so the +// disk footprint is bounded by slots*chunkSize regardless of the file size. +// +// WriteChunk is safe for concurrent use; Read must be called from a single +// goroutine (the same contract as the FileStreamer it backs). +type Window struct { + mu sync.Mutex + cond *sync.Cond + + f *os.File + path string + + chunkSize int64 + totalSize int64 + total int + slots int + + slotState []uint8 + slotChunk []int + readPos int64 + + crcs []uint32 + crcSet []bool + + err error +} + +// Snapshot describes the receiving state, used for status responses and resume. +type Snapshot struct { + // Frontier is the next chunk index to be consumed (== TotalChunks when the stream is fully consumed). + Frontier int + // ReadPos is the number of bytes already consumed by the pipeline. + ReadPos int64 + // ReceivedBytes is the number of payload bytes received from the client (consumed + buffered). + ReceivedBytes int64 + // Received holds inclusive ranges of chunk indexes the client does not need to resend. + Received [][2]int +} + +func NewWindow(dir, id string, chunkSize, totalSize int64, slots int) (*Window, error) { + if chunkSize <= 0 || totalSize <= 0 || slots <= 0 { + return nil, fmt.Errorf("invalid window params: chunkSize=%d totalSize=%d slots=%d", chunkSize, totalSize, slots) + } + if err := os.MkdirAll(dir, 0o700); err != nil { + return nil, err + } + path := filepath.Join(dir, id+".ring") + f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0o600) + if err != nil { + return nil, err + } + total := int((totalSize + chunkSize - 1) / chunkSize) + w := &Window{ + f: f, + path: path, + chunkSize: chunkSize, + totalSize: totalSize, + total: total, + slots: slots, + slotState: make([]uint8, slots), + slotChunk: make([]int, slots), + crcs: make([]uint32, total), + crcSet: make([]bool, total), + } + for i := range w.slotChunk { + w.slotChunk[i] = -1 + } + w.cond = sync.NewCond(&w.mu) + return w, nil +} + +func (w *Window) TotalChunks() int { return w.total } + +// ChunkLen returns the payload length of chunk idx (the last chunk may be short). +func (w *Window) ChunkLen(idx int) int64 { + if idx == w.total-1 { + return w.totalSize - int64(idx)*w.chunkSize + } + return w.chunkSize +} + +func stopTimer(t *time.Timer) { + if t != nil { + t.Stop() + } +} + +// frontier returns the next chunk index to be consumed. Callers must hold mu. +func (w *Window) frontier() int { + if w.readPos >= w.totalSize { + return w.total + } + return int(w.readPos / w.chunkSize) +} + +// WriteChunk reads exactly the chunk payload from r into the ring and returns its CRC32 (IEEE). +// Re-sending an already buffered or consumed chunk succeeds immediately without touching data. +func (w *Window) WriteChunk(idx int, r io.Reader) (uint32, error) { + w.mu.Lock() + if w.err != nil { + w.mu.Unlock() + return 0, w.err + } + if idx < 0 || idx >= w.total { + w.mu.Unlock() + return 0, fmt.Errorf("chunk index %d out of range [0,%d)", idx, w.total) + } + length := w.ChunkLen(idx) + slot := idx % w.slots + // Admission control with a bounded wait: instead of bouncing a chunk the + // moment its slot is busy, park the request until the reader frees the + // slot. Rejecting fast would answer before the request body is read, which + // browsers surface as a network error — so under backpressure, waiting IS + // the flow control. sync.Cond has no timed wait; a timer broadcast wakes + // the loop at the deadline. + deadline := time.Now().Add(WindowWaitTimeout) + var timer *time.Timer + for { + if w.err != nil { + w.mu.Unlock() + stopTimer(timer) + return 0, w.err + } + if int64(idx)*w.chunkSize+length <= w.readPos { + // already fully consumed + crc := w.crcs[idx] + w.mu.Unlock() + stopTimer(timer) + return crc, nil + } + if w.slotChunk[slot] == idx { + if w.slotState[slot] == slotReady { + crc := w.crcs[idx] + w.mu.Unlock() + stopTimer(timer) + return crc, nil + } + if w.slotState[slot] == slotFilling { + w.mu.Unlock() + stopTimer(timer) + return 0, ErrChunkInFlight + } + } + if w.slotState[slot] == slotFree && idx < w.frontier()+w.slots { + break // admissible + } + if !time.Now().Before(deadline) { + w.mu.Unlock() + stopTimer(timer) + return 0, ErrOutOfWindow + } + if timer == nil { + timer = time.AfterFunc(time.Until(deadline), func() { + w.mu.Lock() + w.cond.Broadcast() + w.mu.Unlock() + }) + } + w.cond.Wait() + } + stopTimer(timer) + w.slotState[slot] = slotFilling + w.slotChunk[slot] = idx + f := w.f + w.mu.Unlock() + + h := crc32.NewIEEE() + n, err := utils.CopyWithBufferN(io.NewOffsetWriter(f, int64(slot)*w.chunkSize), io.TeeReader(r, h), length) + if err == nil { + // the body must contain exactly one chunk + var b [1]byte + if m, _ := io.ReadFull(r, b[:]); m > 0 { + err = fmt.Errorf("chunk %d larger than expected %d bytes", idx, length) + } + } else { + err = fmt.Errorf("incomplete chunk %d: got %d of %d bytes: %w", idx, n, length, err) + } + + w.mu.Lock() + defer w.mu.Unlock() + if w.slotState[slot] != slotFilling || w.slotChunk[slot] != idx { + // the window was closed and reset the slot while we were writing + if w.err != nil { + return 0, w.err + } + return 0, ErrClosed + } + if err == nil && w.err != nil { + err = w.err + } + if err != nil { + w.slotState[slot] = slotFree + w.slotChunk[slot] = -1 + return 0, err + } + w.slotState[slot] = slotReady + w.crcs[idx] = h.Sum32() + w.crcSet[idx] = true + w.cond.Broadcast() + return w.crcs[idx], nil +} + +// Read serves the reassembled stream in order, blocking until the next chunk +// is available, the window is closed, or the stream ends (io.EOF). +func (w *Window) Read(p []byte) (int, error) { + if len(p) == 0 { + return 0, nil + } + w.mu.Lock() + for { + if w.err != nil { + w.mu.Unlock() + return 0, w.err + } + if w.readPos >= w.totalSize { + w.mu.Unlock() + return 0, io.EOF + } + cur := int(w.readPos / w.chunkSize) + slot := cur % w.slots + if w.slotState[slot] == slotReady && w.slotChunk[slot] == cur { + chunkStart := int64(cur) * w.chunkSize + chunkEnd := chunkStart + w.ChunkLen(cur) + n := int64(len(p)) + if avail := chunkEnd - w.readPos; n > avail { + n = avail + } + off := int64(slot)*w.chunkSize + (w.readPos - chunkStart) + f := w.f + w.mu.Unlock() + + read, err := f.ReadAt(p[:n], off) + + w.mu.Lock() + if read > 0 { + w.readPos += int64(read) + if w.readPos >= chunkEnd { + w.slotState[slot] = slotFree + w.slotChunk[slot] = -1 + w.cond.Broadcast() // writers may be parked waiting for this slot + } + } + sticky := w.err + w.mu.Unlock() + if read > 0 { + return read, nil + } + if sticky != nil { + return 0, sticky + } + if err == nil { + err = io.ErrUnexpectedEOF + } + return 0, err + } + w.cond.Wait() + } +} + +// Close makes all pending and future operations fail with ErrClosed and removes +// the ring file. It is invoked by op.Put via FileStream.Closers when the +// pipeline ends, and is safe to call multiple times. +func (w *Window) Close() error { + return w.CloseWithError(ErrClosed) +} + +// CloseWithError is Close with a caller-chosen sticky error. The session's +// abort path passes an error wrapping context.Canceled so that a driver woken +// up from a blocked Read treats the abort exactly like a canceled request +// (e.g. the local driver only removes partially written files in that case). +func (w *Window) CloseWithError(sticky error) error { + w.mu.Lock() + if w.err == nil { + w.err = sticky + } + f := w.f + w.f = nil + for i := range w.slotState { + w.slotState[i] = slotFree + w.slotChunk[i] = -1 + } + w.cond.Broadcast() + w.mu.Unlock() + if f == nil { + return nil + } + err := f.Close() + if rmErr := os.Remove(w.path); rmErr != nil && err == nil { + err = rmErr + } + return err +} + +// Snapshot reports the receiving state for status responses and resume discovery. +func (w *Window) Snapshot() Snapshot { + w.mu.Lock() + defer w.mu.Unlock() + snap := Snapshot{Frontier: w.frontier(), ReadPos: w.readPos, ReceivedBytes: w.readPos} + var ranges [][2]int + if snap.Frontier > 0 { + ranges = append(ranges, [2]int{0, snap.Frontier - 1}) + } + for idx := snap.Frontier; idx < snap.Frontier+w.slots && idx < w.total; idx++ { + slot := idx % w.slots + if w.slotState[slot] != slotReady || w.slotChunk[slot] != idx { + continue + } + start := int64(idx) * w.chunkSize + var consumed int64 + if w.readPos > start { + consumed = w.readPos - start + } + snap.ReceivedBytes += w.ChunkLen(idx) - consumed + if len(ranges) > 0 && ranges[len(ranges)-1][1] == idx-1 { + ranges[len(ranges)-1][1] = idx + } else { + ranges = append(ranges, [2]int{idx, idx}) + } + } + snap.Received = ranges + return snap +} + +// CRCs returns a copy of the per-chunk CRC32 table and which entries are set. +// It remains readable after Close, so the session can compare re-filled chunks +// against a previous attempt. +func (w *Window) CRCs() ([]uint32, []bool) { + w.mu.Lock() + defer w.mu.Unlock() + crcs := make([]uint32, len(w.crcs)) + set := make([]bool, len(w.crcSet)) + copy(crcs, w.crcs) + copy(set, w.crcSet) + return crcs, set +} diff --git a/internal/multipart/window_test.go b/internal/multipart/window_test.go new file mode 100644 index 000000000..d368672c9 --- /dev/null +++ b/internal/multipart/window_test.go @@ -0,0 +1,527 @@ +package multipart + +import ( + "bytes" + "crypto/sha256" + "errors" + "hash/crc32" + "io" + "math/rand" + "sync" + "sync/atomic" + "testing" + "time" +) + +func genData(size int64) []byte { + data := make([]byte, size) + rnd := rand.New(rand.NewSource(size*7919 + 13)) + rnd.Read(data) + return data +} + +func newTestWindow(t *testing.T, chunkSize, totalSize int64, slots int) *Window { + t.Helper() + w, err := NewWindow(t.TempDir(), "test", chunkSize, totalSize, slots) + if err != nil { + t.Fatalf("NewWindow: %v", err) + } + t.Cleanup(func() { _ = w.Close() }) + return w +} + +func chunkOf(data []byte, idx int, chunkSize int64) []byte { + start := int64(idx) * chunkSize + end := start + chunkSize + if end > int64(len(data)) { + end = int64(len(data)) + } + return data[start:end] +} + +func writeChunkOK(t *testing.T, w *Window, data []byte, idx int) uint32 { + t.Helper() + crc, err := w.WriteChunk(idx, bytes.NewReader(chunkOf(data, idx, w.chunkSize))) + if err != nil { + t.Fatalf("WriteChunk(%d): %v", idx, err) + } + if want := crc32.ChecksumIEEE(chunkOf(data, idx, w.chunkSize)); crc != want { + t.Fatalf("WriteChunk(%d) crc = %08x, want %08x", idx, crc, want) + } + return crc +} + +// readAllWithin reads the whole stream in a goroutine and fails the test on timeout, +// so a reassembly bug cannot hang the suite. +func readAllWithin(t *testing.T, w *Window, timeout time.Duration) []byte { + t.Helper() + type result struct { + data []byte + err error + } + ch := make(chan result, 1) + go func() { + data, err := io.ReadAll(w) + ch <- result{data, err} + }() + select { + case res := <-ch: + if res.err != nil { + t.Fatalf("ReadAll: %v", res.err) + } + return res.data + case <-time.After(timeout): + t.Fatal("ReadAll timed out") + return nil + } +} + +func TestSequentialReadWrite(t *testing.T) { + const chunkSize = 64 * 1024 + totalSize := int64(4*chunkSize + 32*1024) // last chunk is short + data := genData(totalSize) + w := newTestWindow(t, chunkSize, totalSize, 8) + + go func() { + for i := 0; i < w.TotalChunks(); i++ { + if _, err := w.WriteChunk(i, bytes.NewReader(chunkOf(data, i, chunkSize))); err != nil { + t.Errorf("WriteChunk(%d): %v", i, err) + return + } + time.Sleep(time.Millisecond) + } + }() + + got := readAllWithin(t, w, 10*time.Second) + if !bytes.Equal(got, data) { + t.Fatalf("reassembled stream differs: got %d bytes, want %d", len(got), len(data)) + } +} + +func TestOutOfOrderWrites(t *testing.T) { + const chunkSize = 16 * 1024 + totalSize := int64(5*chunkSize - 100) + data := genData(totalSize) + w := newTestWindow(t, chunkSize, totalSize, 8) + + for _, idx := range []int{3, 0, 4, 2, 1} { + writeChunkOK(t, w, data, idx) + } + got := readAllWithin(t, w, 10*time.Second) + if !bytes.Equal(got, data) { + t.Fatal("reassembled stream differs after out-of-order writes") + } +} + +func TestConcurrentWriters(t *testing.T) { + const chunkSize = 32 * 1024 + totalSize := int64(8 * chunkSize) + data := genData(totalSize) + w := newTestWindow(t, chunkSize, totalSize, 8) + + var wg sync.WaitGroup + for i := 0; i < w.TotalChunks(); i++ { + wg.Add(1) + go func(idx int) { + defer wg.Done() + if _, err := w.WriteChunk(idx, bytes.NewReader(chunkOf(data, idx, chunkSize))); err != nil { + t.Errorf("WriteChunk(%d): %v", idx, err) + } + }(i) + } + got := readAllWithin(t, w, 10*time.Second) + wg.Wait() + if !bytes.Equal(got, data) { + t.Fatal("reassembled stream differs after concurrent writes") + } +} + +func setWaitTimeout(t *testing.T, d time.Duration) { + t.Helper() + old := WindowWaitTimeout + WindowWaitTimeout = d + t.Cleanup(func() { WindowWaitTimeout = old }) +} + +func TestBackpressure(t *testing.T) { + setWaitTimeout(t, 50*time.Millisecond) // assert the post-deadline rejection + const chunkSize = 1024 + totalSize := int64(5 * chunkSize) + data := genData(totalSize) + w := newTestWindow(t, chunkSize, totalSize, 2) + + if _, err := w.WriteChunk(2, bytes.NewReader(chunkOf(data, 2, chunkSize))); !errors.Is(err, ErrOutOfWindow) { + t.Fatalf("chunk 2 with frontier 0: err = %v, want ErrOutOfWindow", err) + } + writeChunkOK(t, w, data, 0) + writeChunkOK(t, w, data, 1) + // both slots occupied, frontier still 0 + if _, err := w.WriteChunk(2, bytes.NewReader(chunkOf(data, 2, chunkSize))); !errors.Is(err, ErrOutOfWindow) { + t.Fatalf("chunk 2 with full window: err = %v, want ErrOutOfWindow", err) + } + // consume chunk 0 -> slot released, frontier advances + buf := make([]byte, chunkSize) + if _, err := io.ReadFull(w, buf); err != nil { + t.Fatalf("ReadFull chunk 0: %v", err) + } + if !bytes.Equal(buf, chunkOf(data, 0, chunkSize)) { + t.Fatal("chunk 0 content differs") + } + writeChunkOK(t, w, data, 2) + // chunk 3 maps to the slot still holding buffered chunk 1 + if _, err := w.WriteChunk(3, bytes.NewReader(chunkOf(data, 3, chunkSize))); !errors.Is(err, ErrOutOfWindow) { + t.Fatalf("chunk 3 with occupied slot: err = %v, want ErrOutOfWindow", err) + } +} + +// TestWriteChunkWaitsForSlot pins the browser-friendly flow control: a chunk +// whose slot is occupied parks until the reader frees it instead of bouncing +// with an immediate rejection (early responses read as network errors in +// browsers). +func TestWriteChunkWaitsForSlot(t *testing.T) { + setWaitTimeout(t, 5*time.Second) + const chunkSize = 1024 + totalSize := int64(5 * chunkSize) + data := genData(totalSize) + w := newTestWindow(t, chunkSize, totalSize, 2) + + writeChunkOK(t, w, data, 0) + writeChunkOK(t, w, data, 1) + + done := make(chan error, 1) + go func() { + _, err := w.WriteChunk(2, bytes.NewReader(chunkOf(data, 2, chunkSize))) + done <- err + }() + select { + case err := <-done: + t.Fatalf("chunk 2 should be parked while the window is full, returned %v", err) + case <-time.After(100 * time.Millisecond): + // parked, as designed + } + + // consuming chunk 0 frees its slot and must wake the parked writer + buf := make([]byte, chunkSize) + if _, err := io.ReadFull(w, buf); err != nil { + t.Fatalf("ReadFull: %v", err) + } + select { + case err := <-done: + if err != nil { + t.Fatalf("parked chunk after slot freed: %v", err) + } + case <-time.After(5 * time.Second): + t.Fatal("parked chunk never admitted after its slot freed") + } +} + +func TestIdempotentResend(t *testing.T) { + const chunkSize = 1024 + totalSize := int64(2 * chunkSize) + data := genData(totalSize) + w := newTestWindow(t, chunkSize, totalSize, 4) + + first := writeChunkOK(t, w, data, 0) + again := writeChunkOK(t, w, data, 0) // buffered, not yet consumed + if first != again { + t.Fatalf("resend crc = %08x, want %08x", again, first) + } + buf := make([]byte, chunkSize) + if _, err := io.ReadFull(w, buf); err != nil { + t.Fatalf("ReadFull: %v", err) + } + consumed := writeChunkOK(t, w, data, 0) // already consumed + if consumed != first { + t.Fatalf("post-consume resend crc = %08x, want %08x", consumed, first) + } +} + +// gatedReader blocks the first Read until released, to hold a chunk in the filling state. +type gatedReader struct { + release <-chan struct{} + inner io.Reader +} + +func (g *gatedReader) Read(p []byte) (int, error) { + <-g.release + return g.inner.Read(p) +} + +func TestInFlightConflict(t *testing.T) { + const chunkSize = 1024 + totalSize := int64(2 * chunkSize) + data := genData(totalSize) + w := newTestWindow(t, chunkSize, totalSize, 4) + + release := make(chan struct{}) + done := make(chan error, 1) + go func() { + _, err := w.WriteChunk(0, &gatedReader{release: release, inner: bytes.NewReader(chunkOf(data, 0, chunkSize))}) + done <- err + }() + + // wait until the writer marked the slot as filling + deadline := time.Now().Add(5 * time.Second) + for { + w.mu.Lock() + filling := w.slotState[0] == slotFilling + w.mu.Unlock() + if filling { + break + } + if time.Now().After(deadline) { + t.Fatal("writer never reached filling state") + } + time.Sleep(time.Millisecond) + } + + if _, err := w.WriteChunk(0, bytes.NewReader(chunkOf(data, 0, chunkSize))); !errors.Is(err, ErrChunkInFlight) { + t.Fatalf("concurrent same-chunk write: err = %v, want ErrChunkInFlight", err) + } + close(release) + if err := <-done; err != nil { + t.Fatalf("gated WriteChunk: %v", err) + } + writeChunkOK(t, w, data, 0) // idempotent after settle +} + +func TestShortBodyRecovers(t *testing.T) { + const chunkSize = 1024 + totalSize := int64(2 * chunkSize) + data := genData(totalSize) + w := newTestWindow(t, chunkSize, totalSize, 4) + + if _, err := w.WriteChunk(0, bytes.NewReader(chunkOf(data, 0, chunkSize)[:100])); err == nil { + t.Fatal("short body: expected error") + } + writeChunkOK(t, w, data, 0) // slot must have been recycled +} + +func TestOversizeBodyRejected(t *testing.T) { + const chunkSize = 1024 + totalSize := int64(2 * chunkSize) + data := genData(totalSize) + w := newTestWindow(t, chunkSize, totalSize, 4) + + oversize := append(append([]byte{}, chunkOf(data, 0, chunkSize)...), 0xFF) + if _, err := w.WriteChunk(0, bytes.NewReader(oversize)); err == nil { + t.Fatal("oversize body: expected error") + } + writeChunkOK(t, w, data, 0) + + // the short last chunk must also reject a full-size body + last := w.TotalChunks() - 1 + if last == 0 { + t.Fatal("test needs at least 2 chunks") + } +} + +func TestLastChunkShortStrict(t *testing.T) { + const chunkSize = 1024 + totalSize := int64(chunkSize + 100) + data := genData(totalSize) + w := newTestWindow(t, chunkSize, totalSize, 4) + + if _, err := w.WriteChunk(1, bytes.NewReader(genData(chunkSize))); err == nil { + t.Fatal("full-size body for short last chunk: expected error") + } + writeChunkOK(t, w, data, 1) + writeChunkOK(t, w, data, 0) + got := readAllWithin(t, w, 10*time.Second) + if !bytes.Equal(got, data) { + t.Fatal("reassembled stream differs") + } +} + +func TestCloseUnblocksReader(t *testing.T) { + w := newTestWindow(t, 1024, 4096, 4) + errCh := make(chan error, 1) + go func() { + buf := make([]byte, 16) + _, err := w.Read(buf) + errCh <- err + }() + time.Sleep(20 * time.Millisecond) + _ = w.Close() + select { + case err := <-errCh: + if !errors.Is(err, ErrClosed) { + t.Fatalf("blocked Read after Close: err = %v, want ErrClosed", err) + } + case <-time.After(5 * time.Second): + t.Fatal("Read still blocked after Close") + } +} + +func TestCloseWithErrorPropagatesSticky(t *testing.T) { + w := newTestWindow(t, 1024, 4096, 4) + cause := errors.New("aborted for a reason") + errCh := make(chan error, 1) + go func() { + buf := make([]byte, 16) + _, err := w.Read(buf) + errCh <- err + }() + time.Sleep(20 * time.Millisecond) + _ = w.CloseWithError(cause) + select { + case err := <-errCh: + if !errors.Is(err, cause) { + t.Fatalf("blocked Read after CloseWithError: err = %v, want %v", err, cause) + } + case <-time.After(5 * time.Second): + t.Fatal("Read still blocked after CloseWithError") + } + if _, err := w.WriteChunk(0, bytes.NewReader(make([]byte, 1024))); !errors.Is(err, cause) { + t.Fatalf("WriteChunk after CloseWithError: err = %v, want %v", err, cause) + } +} + +func TestWriteAfterClose(t *testing.T) { + w := newTestWindow(t, 1024, 4096, 4) + _ = w.Close() + if _, err := w.WriteChunk(0, bytes.NewReader(make([]byte, 1024))); !errors.Is(err, ErrClosed) { + t.Fatalf("WriteChunk after Close: err = %v, want ErrClosed", err) + } +} + +func TestCloseUnblocksInFlightWriter(t *testing.T) { + const chunkSize = 1024 + data := genData(2 * chunkSize) + w := newTestWindow(t, chunkSize, 2*chunkSize, 4) + + release := make(chan struct{}) + done := make(chan error, 1) + go func() { + _, err := w.WriteChunk(0, &gatedReader{release: release, inner: bytes.NewReader(chunkOf(data, 0, chunkSize))}) + done <- err + }() + time.Sleep(20 * time.Millisecond) + _ = w.Close() + close(release) + if err := <-done; !errors.Is(err, ErrClosed) { + t.Fatalf("in-flight WriteChunk across Close: err = %v, want ErrClosed", err) + } +} + +func TestEOFExact(t *testing.T) { + const chunkSize = 1024 + totalSize := int64(chunkSize + 5) + data := genData(totalSize) + w := newTestWindow(t, chunkSize, totalSize, 4) + writeChunkOK(t, w, data, 0) + writeChunkOK(t, w, data, 1) + + got := readAllWithin(t, w, 10*time.Second) + if !bytes.Equal(got, data) { + t.Fatal("reassembled stream differs") + } + buf := make([]byte, 1) + if n, err := w.Read(buf); n != 0 || err != io.EOF { + t.Fatalf("Read at EOF = (%d, %v), want (0, io.EOF)", n, err) + } +} + +func TestManyLapsSmallWindow(t *testing.T) { + const chunkSize = 8 * 1024 + const chunks = 64 + totalSize := int64(chunks*chunkSize - 777) + data := genData(totalSize) + w := newTestWindow(t, chunkSize, totalSize, 3) + + var next atomic.Int64 + var wg sync.WaitGroup + for i := 0; i < 3; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for { + idx := int(next.Add(1) - 1) + if idx >= w.TotalChunks() { + return + } + for { + _, err := w.WriteChunk(idx, bytes.NewReader(chunkOf(data, idx, chunkSize))) + if err == nil { + break + } + if errors.Is(err, ErrOutOfWindow) || errors.Is(err, ErrChunkInFlight) { + time.Sleep(200 * time.Microsecond) + continue + } + t.Errorf("WriteChunk(%d): %v", idx, err) + return + } + } + }() + } + + got := readAllWithin(t, w, 30*time.Second) + wg.Wait() + if wantSum, gotSum := sha256.Sum256(data), sha256.Sum256(got); wantSum != gotSum { + t.Fatalf("reassembled stream differs: got %d bytes, want %d", len(got), len(data)) + } +} + +func TestSnapshot(t *testing.T) { + const chunkSize = 1024 + totalSize := int64(5*chunkSize + 512) + data := genData(totalSize) + w := newTestWindow(t, chunkSize, totalSize, 4) + + writeChunkOK(t, w, data, 0) + writeChunkOK(t, w, data, 1) + writeChunkOK(t, w, data, 3) + + snap := w.Snapshot() + if snap.Frontier != 0 || snap.ReadPos != 0 { + t.Fatalf("snapshot frontier/readPos = %d/%d, want 0/0", snap.Frontier, snap.ReadPos) + } + wantRanges := [][2]int{{0, 1}, {3, 3}} + if len(snap.Received) != len(wantRanges) || snap.Received[0] != wantRanges[0] || snap.Received[1] != wantRanges[1] { + t.Fatalf("snapshot received = %v, want %v", snap.Received, wantRanges) + } + if snap.ReceivedBytes != 3*chunkSize { + t.Fatalf("snapshot receivedBytes = %d, want %d", snap.ReceivedBytes, 3*chunkSize) + } + + buf := make([]byte, chunkSize) + if _, err := io.ReadFull(w, buf); err != nil { + t.Fatalf("ReadFull: %v", err) + } + snap = w.Snapshot() + if snap.Frontier != 1 || snap.ReadPos != chunkSize { + t.Fatalf("snapshot frontier/readPos = %d/%d, want 1/%d", snap.Frontier, snap.ReadPos, chunkSize) + } + if len(snap.Received) != 2 || snap.Received[0] != [2]int{0, 1} || snap.Received[1] != [2]int{3, 3} { + t.Fatalf("snapshot received = %v, want [[0,1],[3,3]]", snap.Received) + } + if snap.ReceivedBytes != 3*chunkSize { + t.Fatalf("snapshot receivedBytes = %d, want %d", snap.ReceivedBytes, 3*chunkSize) + } +} + +func TestIndexOutOfRange(t *testing.T) { + w := newTestWindow(t, 1024, 4096, 4) + if _, err := w.WriteChunk(-1, bytes.NewReader(nil)); err == nil { + t.Fatal("negative index: expected error") + } + if _, err := w.WriteChunk(4, bytes.NewReader(nil)); err == nil { + t.Fatal("index == total: expected error") + } +} + +func TestCRCsSurviveClose(t *testing.T) { + const chunkSize = 1024 + data := genData(2 * chunkSize) + w := newTestWindow(t, chunkSize, 2*chunkSize, 4) + want := writeChunkOK(t, w, data, 0) + _ = w.Close() + crcs, set := w.CRCs() + if !set[0] || crcs[0] != want { + t.Fatalf("CRCs after Close = (%08x, %v), want (%08x, true)", crcs[0], set[0], want) + } + if set[1] { + t.Fatal("chunk 1 crc should not be set") + } +} diff --git a/internal/net/request_test.go b/internal/net/request_test.go index da16a3165..f7a5a27b6 100644 --- a/internal/net/request_test.go +++ b/internal/net/request_test.go @@ -11,13 +11,12 @@ import ( "net/http" "sync" "testing" + "time" "github.com/OpenListTeam/OpenList/v4/pkg/http_range" "github.com/sirupsen/logrus" ) -var buf22MB = make([]byte, 1024*1024*22) - func containsString(slice []string, val string) bool { for _, item := range slice { if item == val { @@ -27,18 +26,6 @@ func containsString(slice []string, val string) bool { return false } -func dummyHttpRequest(data []byte, p http_range.Range) io.ReadCloser { - - end := p.Start + p.Length - 1 - - if end >= int64(len(data)) { - end = int64(len(data)) - } - - bodyBytes := data[p.Start:end] - return io.NopCloser(bytes.NewReader(bodyBytes)) -} - func TestDownloadOrder(t *testing.T) { buff := []byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15} downloader, invocations, ranges := newDownloadRangeClient(buff) @@ -67,8 +54,8 @@ func TestDownloadOrder(t *testing.T) { if err != nil { t.Fatalf("expect no error, got %v", err) } - if exp, a := int(length), len(resultBuf); exp != a { - t.Errorf("expect buffer length=%d, got %d", exp, a) + if exp, a := buff[start:start+length2], resultBuf; !bytes.Equal(exp, a) { + t.Errorf("expect buffer %v, got %v", exp, a) } chunkSize := int(length+int64(partSize)-1) / partSize if e, a := chunkSize, *invocations; e != a { @@ -84,7 +71,100 @@ func TestDownloadOrder(t *testing.T) { if e, a := expectRngs, *ranges; len(e) != len(a) { t.Errorf("expect %v ranges, got %v", e, a) } + if err := readCloser.Close(); err != nil { + t.Errorf("expect no error on close, got %v", err) + } } + +func TestDownloadInterrupt(t *testing.T) { + buff := []byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15} + buff = append(buff, buff...) + downloader, _, _ := newDownloadRangeClient(buff) + con, partSize := 6, 3 + d := NewDownloader(func(d *Downloader) { + d.Concurrency = con + d.PartSize = partSize + d.HttpClient = downloader.HttpRequest + d.ConcurrencyLimit = &ConcurrencyLimit{ + Limit: 5, + } + }) + + var start, length int64 = 0, int64(len(buff)) + req := &HttpRequestParams{ + Range: http_range.Range{Start: start, Length: length}, + Size: int64(len(buff)), + } + ctx, cancel := context.WithCancel(context.Background()) + readCloser, err := d.Download(ctx, req) + + if err != nil { + t.Fatalf("expect no error, got %v", err) + } + _, err = io.CopyN(io.Discard, readCloser, 8) + if err != nil { + t.Fatalf("expect no error, got %v", err) + } + cancel() + if err := readCloser.Close(); err != nil { + t.Errorf("expect no error on close, got %v", err) + } +} + +func TestHighConcurrency(t *testing.T) { + buff := make([]byte, 8<<10) + for i := range len(buff) { + buff[i] = byte(i % 256) + } + downloader, invocations, _ := newDownloadRangeClient(buff) + con, partSize := 64, 100 + concurrencyLimit := 32 + d := NewDownloader(func(d *Downloader) { + d.Concurrency = con + d.PartSize = partSize + d.HttpClient = downloader.HttpRequest + d.ConcurrencyLimit = &ConcurrencyLimit{ + Limit: concurrencyLimit, + } + }) + + var start, length int64 = 2, 7 << 10 + length2 := length + if length2 == -1 { + length2 = int64(len(buff)) - start + } + req := &HttpRequestParams{ + Range: http_range.Range{Start: start, Length: length}, + Size: int64(len(buff)), + } + readCloser, err := d.Download(context.Background(), req) + + if err != nil { + t.Fatalf("expect no error, got %v", err) + } + resultBuf, err := io.ReadAll(readCloser) + if err != nil { + t.Fatalf("expect no error, got %v", err) + } + if !bytes.Equal(buff[start:start+length2], resultBuf) { + t.Error("expect buffer content matches, but got mismatch") + } + chunkSize := int(length+int64(partSize)-1) / partSize + if e, a := chunkSize, *invocations; e != a { + t.Errorf("expect %v API calls, got %v", e, a) + } + if err := readCloser.Close(); err != nil { + t.Errorf("expect no error on close, got %v", err) + } + for range 100 { + time.Sleep(10 * time.Millisecond) + if d.ConcurrencyLimit.Limit == concurrencyLimit { + return + } + } + t.Errorf("expect concurrency limit to be %v, got %v", concurrencyLimit, d.ConcurrencyLimit.Limit) +} + func init() { Formatter := new(logrus.TextFormatter) Formatter.TimestampFormat = "2006-01-02T15:04:05.999999999" @@ -136,6 +216,9 @@ func TestDownloadSingle(t *testing.T) { if e, a := expectRngs, *ranges; len(e) != len(a) { t.Errorf("expect %v ranges, got %v", e, a) } + if err := readCloser.Close(); err != nil { + t.Errorf("expect no error on close, got %v", err) + } } type downloadCaptureClient struct { diff --git a/internal/offline_download/115/client.go b/internal/offline_download/115/client.go index d31971d7c..c026ee7ee 100644 --- a/internal/offline_download/115/client.go +++ b/internal/offline_download/115/client.go @@ -125,7 +125,7 @@ func (p *Cloud115) Status(task *tool.DownloadTask) (*tool.Status, error) { s.Completed = t.IsDone() s.TotalBytes = t.Size if t.IsFailed() { - s.Err = fmt.Errorf(t.GetStatus()) + s.Err = fmt.Errorf("%s", t.GetStatus()) } return s, nil } diff --git a/internal/offline_download/115_open/client.go b/internal/offline_download/115_open/client.go index 6f4c7fd04..dd179f319 100644 --- a/internal/offline_download/115_open/client.go +++ b/internal/offline_download/115_open/client.go @@ -122,7 +122,7 @@ func (o *Open115) Status(task *tool.DownloadTask) (*tool.Status, error) { s.Completed = t.IsDone() s.TotalBytes = t.Size if t.IsFailed() { - s.Err = fmt.Errorf(t.GetStatus()) + s.Err = fmt.Errorf("%s", t.GetStatus()) } return s, nil } diff --git a/internal/offline_download/pikpak/pikpak.go b/internal/offline_download/pikpak/pikpak.go index 8a6693091..e223937e7 100644 --- a/internal/offline_download/pikpak/pikpak.go +++ b/internal/offline_download/pikpak/pikpak.go @@ -127,7 +127,7 @@ func (p *PikPak) Status(task *tool.DownloadTask) (*tool.Status, error) { s.TotalBytes = 0 } if t.Phase == "PHASE_TYPE_ERROR" { - s.Err = fmt.Errorf(t.Message) + s.Err = fmt.Errorf("%s", t.Message) } return s, nil } diff --git a/internal/offline_download/tool/add.go b/internal/offline_download/tool/add.go index 33128ccc2..e147d159e 100644 --- a/internal/offline_download/tool/add.go +++ b/internal/offline_download/tool/add.go @@ -12,6 +12,7 @@ import ( _115_open "github.com/OpenListTeam/OpenList/v4/drivers/115_open" _123 "github.com/OpenListTeam/OpenList/v4/drivers/123" _123_open "github.com/OpenListTeam/OpenList/v4/drivers/123_open" + "github.com/OpenListTeam/OpenList/v4/drivers/guangyapan" "github.com/OpenListTeam/OpenList/v4/drivers/pikpak" "github.com/OpenListTeam/OpenList/v4/drivers/thunder" "github.com/OpenListTeam/OpenList/v4/drivers/thunder_browser" @@ -162,6 +163,16 @@ func AddURL(ctx context.Context, args *AddURLArgs) (task.TaskExtensionInfo, erro } else { tempDir = filepath.Join(setting.GetStr(conf.ThunderXTempDir), uid) } + case "GuangYaPan": + if _, ok := storage.(*guangyapan.GuangYaPan); ok { + tempDir = args.DstDirPath + } else { + tempBase := setting.GetStr(conf.GuangYaPanTempDir) + if tempBase == "" { + return nil, errors.New("GuangYaPan temp dir is not set") + } + tempDir = filepath.Join(tempBase, uid) + } } taskCreator, _ := ctx.Value(conf.UserKey).(*model.User) // taskCreator is nil when convert failed diff --git a/internal/offline_download/tool/download.go b/internal/offline_download/tool/download.go index b7f7a1a9d..ca8776839 100644 --- a/internal/offline_download/tool/download.go +++ b/internal/offline_download/tool/download.go @@ -98,6 +98,9 @@ outer: if t.tool.Name() == "ThunderX" { return nil } + if t.tool.Name() == "GuangYaPan" { + return nil + } if t.tool.Name() == "115 Cloud" { // hack for 115 <-time.After(time.Second * 1) @@ -176,7 +179,7 @@ func (t *DownloadTask) Update() (bool, error) { func (t *DownloadTask) Transfer() error { toolName := t.tool.Name() - if toolName == "115 Cloud" || toolName == "115 Open" || toolName == "123 Open" || toolName == "123Pan" || toolName == "PikPak" || toolName == "Thunder" || toolName == "ThunderX" || toolName == "ThunderBrowser" { + if toolName == "115 Cloud" || toolName == "115 Open" || toolName == "123 Open" || toolName == "123Pan" || toolName == "PikPak" || toolName == "Thunder" || toolName == "ThunderX" || toolName == "ThunderBrowser" || toolName == "GuangYaPan" { // 如果不是直接下载到目标路径，则进行转存 if t.TempDir != t.DstDirPath { return transferObj(t.Ctx(), t.TempDir, t.DstDirPath, t.DeletePolicy) diff --git a/internal/op/hook.go b/internal/op/hook.go index 5cf01730d..3d8530f93 100644 --- a/internal/op/hook.go +++ b/internal/op/hook.go @@ -2,7 +2,9 @@ package op import ( "context" + "fmt" "regexp" + "strconv" "strings" "github.com/OpenListTeam/OpenList/v4/internal/conf" @@ -83,6 +85,17 @@ var settingItemHooks = map[string]SettingItemHook{ conf.SlicesMap[conf.IgnoreDirectLinkParams] = strings.Split(item.Value, ",") return nil }, + conf.MultipartChunkSize: func(item *model.SettingItem) error { + size, err := strconv.Atoi(strings.TrimSpace(item.Value)) + if err != nil || size < 1 { + // deliberately a plain error: SaveSettings formats hook errors + // with %+v, which would dump a full stack trace into the UI + // notification for stack-carrying errors + return fmt.Errorf("multipart chunk size must be a positive integer (MB), got %q", item.Value) + } + item.Value = strconv.Itoa(size) + return nil + }, } func RegisterSettingItemHook(key string, hook SettingItemHook) { diff --git a/internal/search/bleve/search.go b/internal/search/bleve/search.go index e95e82dfc..ed1518711 100644 --- a/internal/search/bleve/search.go +++ b/internal/search/bleve/search.go @@ -26,18 +26,9 @@ func (b *Bleve) Config() searcher.Config { } func (b *Bleve) Search(ctx context.Context, req model.SearchReq) ([]model.SearchNode, int64, error) { - var queries []query2.Query - query := bleve.NewMatchQuery(req.Keywords) - query.SetField("name") - queries = append(queries, query) - if req.Scope != 0 { - isDir := req.Scope == 1 - isDirQuery := bleve.NewBoolFieldQuery(isDir) - queries = append(queries, isDirQuery) - } - reqQuery := bleve.NewConjunctionQuery(queries...) + reqQuery := buildQuery(req) search := bleve.NewSearchRequest(reqQuery) - search.SortBy([]string{"name"}) + search.SortBy([]string{"name", "_id"}) search.From = (req.Page - 1) * req.PerPage search.Size = req.PerPage search.Fields = []string{"*"} @@ -47,14 +38,74 @@ func (b *Bleve) Search(ctx context.Context, req model.SearchReq) ([]model.Search return nil, 0, err } res, err := utils.SliceConvert(searchResults.Hits, func(src *search2.DocumentMatch) (model.SearchNode, error) { - return model.SearchNode{ - Parent: src.Fields["parent"].(string), - Name: src.Fields["name"].(string), - IsDir: src.Fields["is_dir"].(bool), - Size: int64(src.Fields["size"].(float64)), - }, nil + return searchNodeFromHit(src), nil }) - return res, int64(searchResults.Total), nil + return res, int64(searchResults.Total), err +} + +const searchBatchSize = 1000 + +func (b *Bleve) SearchFiltered(ctx context.Context, req model.SearchReq, filter searcher.Filter) ([]model.SearchNode, int64, error) { + reqQuery := buildQuery(req) + from := int64(req.Page-1) * int64(req.PerPage) + to := from + int64(req.PerPage) + var ( + result []model.SearchNode + total int64 + searchAfter []string + ) + for { + search := bleve.NewSearchRequest(reqQuery) + search.SortBy([]string{"name", "_id"}) + search.Size = searchBatchSize + search.Fields = []string{"*"} + if searchAfter != nil { + search.SetSearchAfter(searchAfter) + } + searchResults, err := b.BIndex.Search(search) + if err != nil { + log.Errorf("search error: %+v", err) + return nil, 0, err + } + for _, hit := range searchResults.Hits { + node := searchNodeFromHit(hit) + if !utils.IsSubPath(req.Parent, node.Parent) || filter != nil && !filter(node) { + continue + } + if total >= from && total < to { + result = append(result, node) + } + total++ + } + if len(searchResults.Hits) < searchBatchSize { + break + } + last := searchResults.Hits[len(searchResults.Hits)-1] + searchAfter = append(searchAfter[:0], last.Sort...) + } + return result, total, nil +} + +func buildQuery(req model.SearchReq) query2.Query { + var queries []query2.Query + query := bleve.NewMatchQuery(req.Keywords) + query.SetField("name") + queries = append(queries, query) + if req.Scope != 0 { + isDir := req.Scope == 1 + isDirQuery := bleve.NewBoolFieldQuery(isDir) + queries = append(queries, isDirQuery) + } + return bleve.NewConjunctionQuery(queries...) +} + +func searchNodeFromHit(src *search2.DocumentMatch) model.SearchNode { + return model.SearchNode{ + Parent: src.Fields["parent"].(string), + Name: src.Fields["name"].(string), + IsDir: src.Fields["is_dir"].(bool), + Size: int64(src.Fields["size"].(float64)), + } } func (b *Bleve) Index(ctx context.Context, node model.SearchNode) error { @@ -103,3 +154,4 @@ func (b *Bleve) Clear(ctx context.Context) error { } var _ searcher.Searcher = (*Bleve)(nil) +var _ searcher.FilteredSearcher = (*Bleve)(nil) diff --git a/internal/search/bleve/search_test.go b/internal/search/bleve/search_test.go new file mode 100644 index 000000000..5b4ca7037 --- /dev/null +++ b/internal/search/bleve/search_test.go @@ -0,0 +1,52 @@ +package bleve + +import ( + "context" + "fmt" + "testing" + + "github.com/OpenListTeam/OpenList/v4/internal/model" + blevelib "github.com/blevesearch/bleve/v2" +) + +func TestSearchFilteredKeepsDuplicateSortValuesAcrossBatches(t *testing.T) { + indexMapping := blevelib.NewIndexMapping() + searchNodeMapping := blevelib.NewDocumentMapping() + searchNodeMapping.AddFieldMappingsAt("is_dir", blevelib.NewBooleanFieldMapping()) + searchNodeMapping.AddFieldMappingsAt("parent", blevelib.NewTextFieldMapping()) + searchNodeMapping.AddFieldMappingsAt("name", blevelib.NewKeywordFieldMapping()) + indexMapping.AddDocumentMapping("SearchNode", searchNodeMapping) + index, err := blevelib.NewMemOnly(indexMapping) + if err != nil { + t.Fatalf("NewMemOnly() error = %v", err) + } + t.Cleanup(func() { _ = index.Close() }) + + batch := index.NewBatch() + for i := 0; i < searchBatchSize+1; i++ { + batch.Index(fmt.Sprintf("allowed-%04d", i), model.SearchNode{ + Parent: "/base", + Name: "duplicate", + }) + } + batch.Index("denied", model.SearchNode{Parent: "/base2", Name: "duplicate"}) + if err := index.Batch(batch); err != nil { + t.Fatalf("Batch() error = %v", err) + } + + b := &Bleve{BIndex: index} + nodes, total, err := b.SearchFiltered(context.Background(), model.SearchReq{ + Parent: "/base", + Keywords: "duplicate", + PageReq: model.PageReq{Page: 1, PerPage: searchBatchSize + 1}, + }, nil) + if err != nil { + t.Fatalf("SearchFiltered() error = %v", err) + } + if total != searchBatchSize+1 { + t.Fatalf("SearchFiltered() total = %d, want %d", total, searchBatchSize+1) + } + if len(nodes) != searchBatchSize+1 { + t.Fatalf("SearchFiltered() returned %d nodes, want %d", len(nodes), searchBatchSize+1) + } +} diff --git a/internal/search/build_test.go b/internal/search/build_test.go new file mode 100644 index 000000000..fed2e155e --- /dev/null +++ b/internal/search/build_test.go @@ -0,0 +1,58 @@ +package search + +import ( + "testing" + "time" +) + +func TestLockUpdateSerializesSameParent(t *testing.T) { + unlockFirst := lockUpdate("/same-parent") + secondStarted := make(chan struct{}) + secondAcquired := make(chan struct{}) + secondReleased := make(chan struct{}) + go func() { + close(secondStarted) + unlockSecond := lockUpdate("/same-parent") + close(secondAcquired) + unlockSecond() + close(secondReleased) + }() + <-secondStarted + + select { + case <-secondAcquired: + t.Fatal("second update acquired the same parent lock") + case <-time.After(20 * time.Millisecond): + } + + unlockFirst() + select { + case <-secondReleased: + case <-time.After(time.Second): + t.Fatal("second update did not acquire the released parent lock") + } + + updateLocksMu.Lock() + defer updateLocksMu.Unlock() + if len(updateLocks) != 0 { + t.Fatalf("update locks were not cleaned up: %d", len(updateLocks)) + } +} + +func TestLockUpdateAllowsDifferentParents(t *testing.T) { + unlockFirst := lockUpdate("/first-parent") + defer unlockFirst() + + secondAcquired := make(chan struct{}) + go func() { + unlockSecond := lockUpdate("/second-parent") + unlockSecond() + close(secondAcquired) + }() + + select { + case <-secondAcquired: + case <-time.After(time.Second): + t.Fatal("update for a different parent was blocked") + } +} diff --git a/internal/search/search.go b/internal/search/search.go index 85be84951..0ca1e2417 100644 --- a/internal/search/search.go +++ b/internal/search/search.go @@ -51,6 +51,46 @@ func Search(ctx context.Context, req model.SearchReq) ([]model.SearchNode, int64 return instance.Search(ctx, req) } +const searchBatchSize = 1000 + +func SearchFiltered(ctx context.Context, req model.SearchReq, filter searcher.Filter) ([]model.SearchNode, int64, error) { + if filteredSearcher, ok := instance.(searcher.FilteredSearcher); ok { + return filteredSearcher.SearchFiltered(ctx, req, filter) + } + + batchReq := req + batchReq.Page = 1 + batchReq.PerPage = searchBatchSize + from := int64(req.Page-1) * int64(req.PerPage) + to := from + int64(req.PerPage) + var ( + result []model.SearchNode + filteredTotal int64 + processedTotal int64 + ) + for { + nodes, total, err := instance.Search(ctx, batchReq) + if err != nil { + return nil, 0, err + } + for _, node := range nodes { + if filter != nil && !filter(node) { + continue + } + if filteredTotal >= from && filteredTotal < to { + result = append(result, node) + } + filteredTotal++ + } + processedTotal += int64(len(nodes)) + if len(nodes) == 0 || processedTotal >= total { + break + } + batchReq.Page++ + } + return result, filteredTotal, nil +} + func Index(ctx context.Context, parent string, obj model.Obj) error { if instance == nil { return errs.SearchNotAvailable diff --git a/internal/search/search_test.go b/internal/search/search_test.go new file mode 100644 index 000000000..0b1d75d44 --- /dev/null +++ b/internal/search/search_test.go @@ -0,0 +1,61 @@ +package search + +import ( + "context" + "testing" + + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/search/searcher" +) + +type filteredSearchStub struct { + nodes []model.SearchNode +} + +func (s *filteredSearchStub) Config() searcher.Config { + return searcher.Config{Name: "stub"} +} + +func (s *filteredSearchStub) Search(_ context.Context, req model.SearchReq) ([]model.SearchNode, int64, error) { + from := (req.Page - 1) * req.PerPage + if from >= len(s.nodes) { + return nil, int64(len(s.nodes)), nil + } + to := min(from+req.PerPage, len(s.nodes)) + return s.nodes[from:to], int64(len(s.nodes)), nil +} + +func (s *filteredSearchStub) Index(context.Context, model.SearchNode) error { return nil } +func (s *filteredSearchStub) BatchIndex(context.Context, []model.SearchNode) error { return nil } +func (s *filteredSearchStub) Get(context.Context, string) ([]model.SearchNode, error) { + return nil, nil +} +func (s *filteredSearchStub) Del(context.Context, string) error { return nil } +func (s *filteredSearchStub) Release(context.Context) error { return nil } +func (s *filteredSearchStub) Clear(context.Context) error { return nil } + +func TestSearchFilteredFiltersBeforePagination(t *testing.T) { + previous := instance + instance = &filteredSearchStub{nodes: []model.SearchNode{ + {Name: "denied-1"}, + {Name: "allowed-1"}, + {Name: "denied-2"}, + {Name: "allowed-2"}, + }} + t.Cleanup(func() { instance = previous }) + + nodes, total, err := SearchFiltered(context.Background(), model.SearchReq{ + PageReq: model.PageReq{Page: 2, PerPage: 1}, + }, func(node model.SearchNode) bool { + return node.Name == "allowed-1" || node.Name == "allowed-2" + }) + if err != nil { + t.Fatalf("SearchFiltered() error = %v", err) + } + if total != 2 { + t.Fatalf("SearchFiltered() total = %d, want 2", total) + } + if len(nodes) != 1 || nodes[0].Name != "allowed-2" { + t.Fatalf("SearchFiltered() nodes = %#v, want allowed-2", nodes) + } +} diff --git a/internal/search/searcher/searcher.go b/internal/search/searcher/searcher.go index e7b7ba1e3..aaecdf53c 100644 --- a/internal/search/searcher/searcher.go +++ b/internal/search/searcher/searcher.go @@ -11,6 +11,13 @@ type Config struct { AutoUpdate bool } +type Filter func(node model.SearchNode) bool + +// FilteredSearcher applies filtering before calculating pagination and totals. +type FilteredSearcher interface { + SearchFiltered(ctx context.Context, req model.SearchReq, filter Filter) ([]model.SearchNode, int64, error) +} + type Searcher interface { // Config of the searcher Config() Config diff --git a/internal/search/update_lock.go b/internal/search/update_lock.go new file mode 100644 index 000000000..7ca8d28dd --- /dev/null +++ b/internal/search/update_lock.go @@ -0,0 +1,38 @@ +package search + +import "sync" + +var ( + updateLocksMu sync.Mutex + updateLocks = make(map[string]*updateLock) +) + +type updateLock struct { + mu sync.Mutex + refs uint +} + +// lockUpdate serializes index updates for the same parent while allowing +// unrelated directories to update concurrently. +func lockUpdate(parent string) func() { + updateLocksMu.Lock() + lock, ok := updateLocks[parent] + if !ok { + lock = &updateLock{} + updateLocks[parent] = lock + } + lock.refs++ + updateLocksMu.Unlock() + + lock.mu.Lock() + return func() { + lock.mu.Unlock() + + updateLocksMu.Lock() + lock.refs-- + if lock.refs == 0 { + delete(updateLocks, parent) + } + updateLocksMu.Unlock() + } +} diff --git a/internal/stream/stream.go b/internal/stream/stream.go index 4c8238100..6d3b36072 100644 --- a/internal/stream/stream.go +++ b/internal/stream/stream.go @@ -580,7 +580,7 @@ func (r *RangeReadReadAtSeeker) Seek(offset int64, whence int) (int64, error) { default: return 0, errors.New("Seek: invalid whence") } - if offset < 0 || offset > r.ss.GetSize() { + if offset < 0 { return 0, errors.New("Seek: invalid offset") } r.masterOff = offset diff --git a/pkg/buffer/buffer.go b/pkg/buffer/buffer.go new file mode 100644 index 000000000..54d13c56d --- /dev/null +++ b/pkg/buffer/buffer.go @@ -0,0 +1,41 @@ +package buffer + +import ( + "io" +) + +type byteBlock struct { + buf []byte +} + +func NewByteBlock(buf []byte) Block { + return &byteBlock{buf: buf} +} + +func (b *byteBlock) Size() int64 { + return int64(len(b.buf)) +} + +func (b *byteBlock) ReadAt(p []byte, off int64) (n int, err error) { + if len(b.buf) == 0 || off < 0 || off >= b.Size() { + return 0, io.EOF + } + n = copy(p, b.buf[off:]) + if n < len(p) { + err = io.EOF + } + return +} + +func (b *byteBlock) WriteAt(p []byte, off int64) (n int, err error) { + if len(b.buf) == 0 || off < 0 || off >= b.Size() { + return 0, io.ErrShortWrite + } + n = copy(b.buf[off:], p) + if n < len(p) { + err = io.ErrShortWrite + } + return +} + +var _ Block = (*byteBlock)(nil) diff --git a/pkg/buffer/pipe.go b/pkg/buffer/pipe.go new file mode 100644 index 000000000..05e13e243 --- /dev/null +++ b/pkg/buffer/pipe.go @@ -0,0 +1,164 @@ +package buffer + +import ( + "context" + "fmt" + "io" + "sync" +) + +type PipeBuffer struct { + limit int //expected size + ctx context.Context + offR int + offW int + rw sync.Mutex + ioWg sync.WaitGroup + block Block + + readSignal chan struct{} + readPending bool +} + +// NewPipeBuffer is a buffer that can have 1 read & 1 write at the same time. +// when read is faster write, immediately feed data to read after written +func NewPipeBuffer(ctx context.Context, block Block) *PipeBuffer { + br := &PipeBuffer{ + ctx: ctx, + limit: int(block.Size()), + readSignal: make(chan struct{}, 1), + block: block, + } + return br +} + +func (br *PipeBuffer) Read(p []byte) (int, error) { + if err := br.ctx.Err(); err != nil { + return 0, err + } + if len(p) == 0 { + return 0, nil + } + if br.offR >= br.limit { + return 0, io.EOF + } + + for { + br.rw.Lock() + if br.block == nil { + br.rw.Unlock() + return 0, io.ErrClosedPipe + } + + if br.offW == br.offR { + br.readPending = true + br.rw.Unlock() + select { + case <-br.ctx.Done(): + return 0, br.ctx.Err() + case _, ok := <-br.readSignal: + if !ok { + return 0, io.ErrClosedPipe + } + continue + } + } + break + } + + canRead := br.offW - br.offR + if canRead < 0 { + br.rw.Unlock() + return 0, io.ErrUnexpectedEOF + } + + off := br.offR + block := br.block + br.ioWg.Add(1) + defer br.ioWg.Done() + br.rw.Unlock() + + n, err := block.ReadAt(p[:min(len(p), canRead)], int64(off)) + + br.rw.Lock() + br.offR += n + br.rw.Unlock() + + if n < len(p) && br.offR >= br.limit { + return n, io.EOF + } + return n, err +} + +func (br *PipeBuffer) Write(p []byte) (int, error) { + if err := br.ctx.Err(); err != nil { + return 0, err + } + if len(p) == 0 { + return 0, nil + } + + br.rw.Lock() + if br.block == nil { + br.rw.Unlock() + return 0, io.ErrClosedPipe + } + + canWrite := br.limit - br.offW + if canWrite <= 0 { + br.rw.Unlock() + return 0, io.ErrShortWrite + } + + off := br.offW + block := br.block + br.ioWg.Add(1) + defer br.ioWg.Done() + br.rw.Unlock() + + n, err := block.WriteAt(p[:min(canWrite, len(p))], int64(off)) + + br.rw.Lock() + br.offW += n + if br.readPending { + br.readPending = false + select { + case br.readSignal <- struct{}{}: + default: + } + } + br.rw.Unlock() + + if n < len(p) && err == nil { + return n, io.ErrShortWrite + } + return n, err +} + +func (br *PipeBuffer) Reset(limit int) error { + br.ioWg.Wait() + br.rw.Lock() + defer br.rw.Unlock() + if br.block == nil { + return io.ErrClosedPipe + } + if int64(limit) > br.block.Size() { + return fmt.Errorf("reset limit %d exceeds max size %d", limit, br.block.Size()) + } + br.limit = limit + br.offR = 0 + br.offW = 0 + return nil +} + +func (br *PipeBuffer) Close() error { + br.rw.Lock() + if br.block != nil { + br.block = nil + br.readPending = false + close(br.readSignal) + } + br.rw.Unlock() + br.ioWg.Wait() + return nil +} diff --git a/pkg/buffer/pipe_test.go b/pkg/buffer/pipe_test.go new file mode 100644 index 000000000..2c44d8b4a --- /dev/null +++ b/pkg/buffer/pipe_test.go @@ -0,0 +1,131 @@ +package buffer + +import ( + "context" + "errors" + "io" + "testing" + "time" +) + +type blockingBlock struct { + data []byte + blockOn string + started chan struct{} + release chan struct{} +} + +func newBlockingBlock(blockOn string) *blockingBlock { + return &blockingBlock{ + data: make([]byte, 1), + blockOn: blockOn, + started: make(chan struct{}), + release: make(chan struct{}), + } +} + +func (b *blockingBlock) Size() int64 { + return int64(len(b.data)) +} + +func (b *blockingBlock) ReadAt(p []byte, off int64) (int, error) { + if b.blockOn == "read" { + close(b.started) + <-b.release + } + n := copy(p, b.data[off:]) + if n < len(p) { + return n, io.EOF + } + return n, nil +} + +func (b *blockingBlock) WriteAt(p []byte, off int64) (int, error) { + if b.blockOn == "write" { + close(b.started) + <-b.release + } + n := copy(b.data[off:], p) + if n < len(p) { + return n, io.ErrShortWrite + } + return n, nil +} + +func TestPipeBufferCloseWaitsForActiveIO(t *testing.T) { + for _, operation := range []string{"read", "write"} { + t.Run(operation, func(t *testing.T) { + block := newBlockingBlock(operation) + buf := NewPipeBuffer(context.Background(), block) + if operation == "read" { + if _, err := buf.Write([]byte{1}); err != nil { + t.Fatalf("prepare read: %v", err) + } + } + + ioDone := make(chan error, 1) + go func() { + var err error + if operation == "read" { + _, err = buf.Read(make([]byte, 1)) + } else { + _, err = buf.Write([]byte{1}) + } + ioDone <- err + }() + + select { + case <-block.started: + case <-time.After(time.Second): + t.Fatal("I/O did not start") + } + + closeDone := make(chan error, 1) + go func() { + closeDone <- buf.Close() + }() + + deadline := time.Now().Add(time.Second) + for { + buf.rw.Lock() + closed := buf.block == nil + buf.rw.Unlock() + if closed { + break + } + if time.Now().After(deadline) { + t.Fatal("buffer did not enter the closed state") + } + time.Sleep(time.Millisecond) + } + + select { + case err := <-closeDone: + t.Fatalf("Close returned before active %s completed: %v", operation, err) + default: + } + + close(block.release) + select { + case err := <-ioDone: + if err != nil { + t.Fatalf("active %s failed: %v", operation, err) + } + case <-time.After(time.Second): + t.Fatalf("active %s did not complete", operation) + } + select { + case err := <-closeDone: + if err != nil { + t.Fatalf("Close failed: %v", err) + } + case <-time.After(time.Second): + t.Fatal("Close did not wait for active I/O") + } + + if _, err := buf.Write([]byte{1}); !errors.Is(err, io.ErrClosedPipe) { + t.Fatalf("write after Close error = %v, want %v", err, io.ErrClosedPipe) + } + }) + } +} diff --git a/pkg/buffer/type.go b/pkg/buffer/type.go new file mode 100644 index 000000000..ce0d78b2e --- /dev/null +++ b/pkg/buffer/type.go @@ -0,0 +1,24 @@ +package buffer + +import ( + "io" + + "github.com/OpenListTeam/OpenList/v4/internal/model" +) + +type Block interface { + io.ReaderAt + io.WriterAt + Size() int64 +} + +type WriteAtSeeker = model.FileWriter +type WriteAtSeekerProvider interface{ GetWriteAtSeeker() WriteAtSeeker } + +type ReadAtSeeker = model.File +type ReadAtSeekerProvider interface{ GetReadAtSeeker() ReadAtSeeker } + +type SizedReadAtSeeker interface { + ReadAtSeeker + Size() int64 +} diff --git a/pkg/buffer/utils.go b/pkg/buffer/utils.go new file mode 100644 index 000000000..cf494ce13 --- /dev/null +++ b/pkg/buffer/utils.go @@ -0,0 +1,93 @@ +package buffer + +import ( + "errors" + "io" +) + +func WriteAtSeekerOf(b Block) WriteAtSeeker { + if p, ok := b.(WriteAtSeekerProvider); ok { + return p.GetWriteAtSeeker() + } + return io.NewOffsetWriter(b, 0) +} + +// 将一个Block包装为ReadAtSeeker。 +// 固定大小：当前Block的Size()。 +func ReadAtSeekerOf(b Block) ReadAtSeeker { + if p, ok := b.(ReadAtSeekerProvider); ok { + return p.GetReadAtSeeker() + } + return io.NewSectionReader(b, 0, b.Size()) +} + +type blockAdapter struct { + WriteAtSeeker + SizedReadAtSeeker +} + +func (b *blockAdapter) GetWriteAtSeeker() WriteAtSeeker { + return b.WriteAtSeeker +} + +func (b *blockAdapter) GetReadAtSeeker() ReadAtSeeker { + return b.SizedReadAtSeeker +} +func NewBlockAdapter(w WriteAtSeeker, r SizedReadAtSeeker) Block { + return &blockAdapter{ + WriteAtSeeker: w, + SizedReadAtSeeker: r, + } +} + +var _ Block = (*blockAdapter)(nil) + +// 将一个Block包装为ReadAtSeeker。 +// 动态大小：Size() 是动态跟随底层 Block。 +type DynamicReadAtSeeker struct { + block Block + offset int64 +} + +func (r *DynamicReadAtSeeker) ReadAt(p []byte, off int64) (n int, err error) { + return r.block.ReadAt(p, off) +} + +func (r *DynamicReadAtSeeker) Read(p []byte) (n int, err error) { + n, err = r.block.ReadAt(p, r.offset) + if n > 0 { + r.offset += int64(n) + } + return n, err +} + +func (r *DynamicReadAtSeeker) Size() int64 { + return r.block.Size() +} + +func (r *DynamicReadAtSeeker) Seek(offset int64, whence int) (int64, error) { + switch whence { + case io.SeekStart: + case io.SeekCurrent: + if offset == 0 { + return r.offset, nil + } + offset = r.offset + offset + case io.SeekEnd: + offset = r.block.Size() + offset + default: + return 0, errors.New("Seek: invalid whence") + } + + if offset < 0 { + return 0, errors.New("Seek: invalid offset") + } + r.offset = offset + return offset, nil +} + +func NewDynamicReadAtSeeker(block Block) *DynamicReadAtSeeker { + return &DynamicReadAtSeeker{ + block: block, + } +} diff --git a/pkg/pool/pool.go b/pkg/pool/pool.go index ce92cd1fc..01cd736d3 100644 --- a/pkg/pool/pool.go +++ b/pkg/pool/pool.go @@ -3,9 +3,7 @@ package pool import "sync" type Pool[T any] struct { - New func() T - MaxCap int - + New func() T cache []T mu sync.Mutex } @@ -24,9 +22,7 @@ func (p *Pool[T]) Get() T { func (p *Pool[T]) Put(item T) { p.mu.Lock() defer p.mu.Unlock() - if p.MaxCap == 0 || len(p.cache) < int(p.MaxCap) { - p.cache = append(p.cache, item) - } + p.cache = append(p.cache, item) } func (p *Pool[T]) Reset() { @@ -35,3 +31,8 @@ func (p *Pool[T]) Reset() { clear(p.cache) p.cache = nil } + +func (p *Pool[T]) Close() error { + p.Reset() + return nil +} diff --git a/server/common/proxy.go b/server/common/proxy.go index e66a5f342..be1a8a933 100644 --- a/server/common/proxy.go +++ b/server/common/proxy.go @@ -76,6 +76,7 @@ func Proxy(w http.ResponseWriter, r *http.Request, link *model.Link, file model. defer res.Body.Close() maps.Copy(w.Header(), res.Header) + w.Header().Set("Content-Disposition", utils.GenerateContentDisposition(file.GetName())) w.WriteHeader(res.StatusCode) if r.Method == http.MethodHead { return nil diff --git a/server/handles/auth.go b/server/handles/auth.go index 35776ba6a..780069091 100644 --- a/server/handles/auth.go +++ b/server/handles/auth.go @@ -45,27 +45,28 @@ func loginHash(c *gin.Context, req *LoginReq) { ip := c.ClientIP() count, ok := model.LoginCache.Get(ip) if ok && count >= model.DefaultMaxAuthRetries { - common.ErrorStrResp(c, "Too many unsuccessful sign-in attempts have been made using an incorrect username or password, Try again later.", 429) + common.ErrorStrResp(c, model.TooManyAttempts, 429) model.LoginCache.Expire(ip, model.DefaultLockDuration) return } // check username user, err := op.GetUserByName(req.Username) if err != nil { - common.ErrorResp(c, err, 400) + common.ErrorStrResp(c, model.InvalidUsernameOrPassword, 401) model.LoginCache.Set(ip, count+1) return } // validate password hash if err := user.ValidatePwdStaticHash(req.Password); err != nil { - common.ErrorResp(c, err, 400) + common.ErrorStrResp(c, model.InvalidUsernameOrPassword, 401) model.LoginCache.Set(ip, count+1) return } // check 2FA if user.OtpSecret != "" { if !totp.Validate(req.OtpCode, user.OtpSecret) { - common.ErrorStrResp(c, "Invalid 2FA code", 402) + // 402 - need opt + common.ErrorStrResp(c, model.Invalid2FACode, 402) model.LoginCache.Set(ip, count+1) return } @@ -73,7 +74,7 @@ func loginHash(c *gin.Context, req *LoginReq) { // generate token token, err := common.GenerateToken(user) if err != nil { - common.ErrorResp(c, err, 400, true) + common.ErrorResp(c, err, 500, true) return } common.SuccessResp(c, gin.H{"token": token}) @@ -107,7 +108,7 @@ func UpdateCurrent(c *gin.Context) { } user := c.Request.Context().Value(conf.UserKey).(*model.User) if user.IsGuest() { - common.ErrorStrResp(c, "Guest user can not update profile", 403) + common.ErrorStrResp(c, model.GuestCannotUpdateProfile, 403) return } user.Username = req.Username @@ -125,7 +126,7 @@ func UpdateCurrent(c *gin.Context) { func Generate2FA(c *gin.Context) { user := c.Request.Context().Value(conf.UserKey).(*model.User) if user.IsGuest() { - common.ErrorStrResp(c, "Guest user can not generate 2FA code", 403) + common.ErrorStrResp(c, model.GuestCannotGenerate2FA, 403) return } key, err := totp.Generate(totp.GenerateOpts{ @@ -164,11 +165,11 @@ func Verify2FA(c *gin.Context) { } user := c.Request.Context().Value(conf.UserKey).(*model.User) if user.IsGuest() { - common.ErrorStrResp(c, "Guest user can not generate 2FA code", 403) + common.ErrorStrResp(c, model.GuestCannotGenerate2FA, 403) return } if !totp.Validate(req.Code, req.Secret) { - common.ErrorStrResp(c, "Invalid 2FA code", 400) + common.ErrorStrResp(c, model.Invalid2FACode, 400) return } user.OtpSecret = req.Secret diff --git a/server/handles/fsbatch.go b/server/handles/fsbatch.go index e672165cd..e0f98284a 100644 --- a/server/handles/fsbatch.go +++ b/server/handles/fsbatch.go @@ -188,6 +188,11 @@ func FsBatchRename(c *gin.Context) { if renameObject.SrcName == "" || renameObject.NewName == "" { continue } + err = checkRelativePath(renameObject.SrcName) + if err != nil { + common.ErrorResp(c, err, 403) + return + } err = checkRelativePath(renameObject.NewName) if err != nil { common.ErrorResp(c, err, 403) diff --git a/server/handles/index115_test.go b/server/handles/index115_test.go index 06aab1169..d114723f0 100644 --- a/server/handles/index115_test.go +++ b/server/handles/index115_test.go @@ -165,3 +165,7 @@ func (s stubIndex115HTTPService) Search(ctx context.Context, req index115.Search func (s stubIndex115HTTPService) Link(ctx context.Context, req index115.LinkRequest) (index115.ResolvedLink, error) { return s.link, s.err } + +func (s stubIndex115HTTPService) Detail(ctx context.Context, fileID string) (index115.FileItem, bool, error) { + return index115.FileItem{}, false, s.err +} diff --git a/server/handles/multipart.go b/server/handles/multipart.go new file mode 100644 index 000000000..991a102bc --- /dev/null +++ b/server/handles/multipart.go @@ -0,0 +1,232 @@ +package handles + +import ( + "errors" + "io" + "net/url" + stdpath "path" + "strconv" + + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/errs" + "github.com/OpenListTeam/OpenList/v4/internal/fs" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/multipart" + "github.com/OpenListTeam/OpenList/v4/internal/op" + "github.com/OpenListTeam/OpenList/v4/internal/setting" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" + "github.com/OpenListTeam/OpenList/v4/server/common" + "github.com/gin-gonic/gin" +) + +const multipartMinChunkSize = int64(1) << 20 // 1MB + +// multipartChunkSize resolves the effective chunk size. The admin setting is +// the ceiling: a client may suggest a smaller chunk via X-Chunk-Size but never +// a larger one — the server buffers a window of several chunks per session, so +// an unbounded client suggestion would translate directly into server-side +// disk usage. +func multipartChunkSize(requested int64) int64 { + ceiling := int64(setting.GetInt(conf.MultipartChunkSize, 10)) << 20 + if ceiling < multipartMinChunkSize { + ceiling = multipartMinChunkSize + } + size := ceiling + if requested > 0 && requested < ceiling { + size = max(requested, multipartMinChunkSize) + } + return size +} + +type MultipartInitResp struct { + multipart.SessionSnapshot + Resumed bool `json:"resumed"` +} + +// MultipartInit creates (or resumes) a multipart upload session and starts its +// upload pipeline. Headers mirror FsStream (fsup.go). +func MultipartInit(c *gin.Context) { + if !setting.GetBool(conf.MultipartEnabled) { + common.ErrorStrResp(c, "multipart upload is disabled", 403) + return + } + path := c.GetHeader("File-Path") + path, err := url.PathUnescape(path) + if err != nil { + common.ErrorResp(c, err, 400) + return + } + user := c.Request.Context().Value(conf.UserKey).(*model.User) + path, err = user.JoinPath(path) + if err != nil { + common.ErrorResp(c, err, 403) + return + } + size, err := strconv.ParseInt(c.GetHeader("X-File-Size"), 10, 64) + if err != nil { + common.ErrorStrResp(c, "multipart upload requires a valid X-File-Size header", 400) + return + } + if size <= 0 { + common.ErrorStrResp(c, "multipart upload requires a positive X-File-Size; upload empty files via /fs/put", 400) + return + } + var requestedChunkSize int64 + if v := c.GetHeader("X-Chunk-Size"); v != "" { + requestedChunkSize, err = strconv.ParseInt(v, 10, 64) + if err != nil { + common.ErrorResp(c, err, 400) + return + } + } + overwrite := c.GetHeader("Overwrite") != "false" + if !overwrite { + if res, _ := fs.Get(c.Request.Context(), path, &fs.GetArgs{NoLog: true}); res != nil { + common.ErrorStrResp(c, "file exists", 403) + return + } + } + dir, name := stdpath.Split(path) + if shouldIgnoreSystemFile(name) { + common.ErrorStrResp(c, errs.IgnoredSystemFile.Error(), 403) + return + } + // fail fast on unusable destinations instead of letting the pipeline discover it + storage, _, err := op.GetStorageAndActualPath(dir) + if err != nil { + common.ErrorResp(c, err, 500) + return + } + if storage.Config().NoUpload { + common.ErrorResp(c, errs.UploadNotSupported, 405) + return + } + h := make(map[*utils.HashType]string) + if md5 := c.GetHeader("X-File-Md5"); md5 != "" { + h[utils.MD5] = md5 + } + if sha1 := c.GetHeader("X-File-Sha1"); sha1 != "" { + h[utils.SHA1] = sha1 + } + if sha256 := c.GetHeader("X-File-Sha256"); sha256 != "" { + h[utils.SHA256] = sha256 + } + mimetype := c.GetHeader("Content-Type") + if len(mimetype) == 0 { + mimetype = utils.GetMimeType(name) + } + snap, resumed, err := multipart.DefaultManager.Init(multipart.InitReq{ + User: user, + Path: path, + Size: size, + ChunkSize: multipartChunkSize(requestedChunkSize), + Mimetype: mimetype, + Modified: getLastModified(c), + Hashes: h, + }) + if err != nil { + common.ErrorResp(c, err, 500) + return + } + common.SuccessResp(c, MultipartInitResp{SessionSnapshot: snap, Resumed: resumed}) +} + +// MultipartChunk ingests one chunk. Chunks are idempotent and may be sent +// concurrently and out of order within the receiving window. +// code 429 = window full (flow control, retry after a short delay), +// code 409 = the same chunk is already in flight on another connection. +func MultipartChunk(c *gin.Context) { + user := c.Request.Context().Value(conf.UserKey).(*model.User) + id := c.GetHeader("X-Upload-Id") + idx, err := strconv.Atoi(c.GetHeader("X-Chunk-Index")) + if err != nil { + common.ErrorStrResp(c, "invalid X-Chunk-Index header", 400) + return + } + snap, err := multipart.DefaultManager.Chunk(user, id, idx, c.Request.Body) + // Answer only after the request body is consumed — on EVERY path. Flow + // control (429), absorbed chunks and validation errors would otherwise + // respond while the browser is still streaming the body, which it reports + // as a network error and which poisons its connection pool. A rejected + // chunk gets resent anyway, so draining costs no extra round trip. The + // drain is bounded so a malformed request cannot pin the handler. + limit := multipartChunkSize(0) + 64*1024 + if snap.ChunkSize > 0 { + limit = snap.ChunkSize + 64*1024 + } + _, _ = utils.CopyWithBuffer(io.Discard, io.LimitReader(c.Request.Body, limit)) + if err != nil { + common.ErrorWithDataResp(c, err, multipartErrCode(err), snap) + return + } + common.SuccessResp(c, snap) +} + +// MultipartComplete waits for the pipeline outcome and reports it, mirroring +// how /fs/put only responds once the driver upload finished. +func MultipartComplete(c *gin.Context) { + user := c.Request.Context().Value(conf.UserKey).(*model.User) + id := c.GetHeader("X-Upload-Id") + snap, err := multipart.DefaultManager.Complete(c.Request.Context(), user, id) + if err != nil { + common.ErrorWithDataResp(c, err, multipartErrCode(err), snap) + return + } + common.SuccessResp(c, snap) +} + +// MultipartStatus looks a session up by upload_id, or by path+size so an +// interrupted client can discover a resumable session. +func MultipartStatus(c *gin.Context) { + user := c.Request.Context().Value(conf.UserKey).(*model.User) + if id := c.Query("upload_id"); id != "" { + snap, err := multipart.DefaultManager.Status(user, id) + if err != nil { + common.ErrorResp(c, err, multipartErrCode(err)) + return + } + common.SuccessResp(c, snap) + return + } + path, err := user.JoinPath(c.Query("path")) + if err != nil { + common.ErrorResp(c, err, 403) + return + } + size, err := strconv.ParseInt(c.Query("size"), 10, 64) + if err != nil { + common.ErrorStrResp(c, "status lookup requires upload_id, or path and size", 400) + return + } + snap, err := multipart.DefaultManager.Find(user, path, size) + if err != nil { + common.ErrorResp(c, err, multipartErrCode(err)) + return + } + common.SuccessResp(c, snap) +} + +// MultipartAbort cancels the pipeline and discards the session. +func MultipartAbort(c *gin.Context) { + user := c.Request.Context().Value(conf.UserKey).(*model.User) + if err := multipart.DefaultManager.Abort(user, c.GetHeader("X-Upload-Id")); err != nil { + common.ErrorResp(c, err, multipartErrCode(err)) + return + } + common.SuccessResp(c) +} + +func multipartErrCode(err error) int { + switch { + case errors.Is(err, multipart.ErrOutOfWindow): + return 429 + case errors.Is(err, multipart.ErrChunkInFlight): + return 409 + case errors.Is(err, multipart.ErrSessionNotFound): + return 404 + case errors.Is(err, multipart.ErrNotOwner): + return 403 + default: + return 400 + } +} diff --git a/server/handles/offline_download.go b/server/handles/offline_download.go index b0cc2cc7b..339ad71c8 100644 --- a/server/handles/offline_download.go +++ b/server/handles/offline_download.go @@ -516,7 +516,6 @@ func SetThunderBrowser(c *gin.Context) { } common.SuccessResp(c, "ok") } - func OfflineDownloadTools(c *gin.Context) { tools := tool.Tools.Names() common.SuccessResp(c, tools) diff --git a/server/handles/webauthn.go b/server/handles/webauthn.go index 6848020fd..dee944965 100644 --- a/server/handles/webauthn.go +++ b/server/handles/webauthn.go @@ -95,7 +95,7 @@ func FinishAuthnLogin(c *gin.Context) { } else { // client-side discoverable login _, err = authnInstance.FinishDiscoverableLogin(func(_, userHandle []byte) (webauthn.User, error) { // first param `rawID` in this callback function is equal to ID in webauthn.Credential, - // but it's unnnecessary to check it. + // but it's unnecessary to check it. // userHandle param is equal to (User).WebAuthnID(). userID := uint(binary.LittleEndian.Uint64(userHandle)) user, err = db.GetUserById(userID) diff --git a/server/s3/pager.go b/server/s3/pager.go index 268242971..a607b1db5 100644 --- a/server/s3/pager.go +++ b/server/s3/pager.go @@ -8,7 +8,7 @@ import ( "github.com/itsHenry35/gofakes3" ) -// pager splits the object list into smulitply pages. +// pager splits the object list into multiple pages. func (db *s3Backend) pager(list *gofakes3.ObjectList, page gofakes3.ListBucketPage) (*gofakes3.ObjectList, error) { // sort by alphabet sort.Slice(list.CommonPrefixes, func(i, j int) bool {