From 9dc92ac3c906b7ea914c387f35f84f165ecdba89 Mon Sep 17 00:00:00 2001 From: LancerComet Date: Wed, 15 Apr 2026 17:26:42 +0800 Subject: [PATCH 1/2] + Add qiniu prefetch feature. --- QiniuUploader/README.md | 24 ++++- QiniuUploader/index.mjs | 184 ++++++++++++++++++++++++++++++++++++- QiniuUploader/package.json | 2 +- 3 files changed, 203 insertions(+), 7 deletions(-) diff --git a/QiniuUploader/README.md b/QiniuUploader/README.md index b256b61..201cdc8 100644 --- a/QiniuUploader/README.md +++ b/QiniuUploader/README.md @@ -13,7 +13,29 @@ npx qiniu-uploader --upload-folder dist \ --qiniu-base-path some-folder-on-qiniu/sub-directory \ --qiniu-access-key *** \ --qiniu-secret-key *** \ - --qiniu-bucket some-bucket + --qiniu-bucket some-bucket \ + --prefetch \ + --prefetch-domain https://cdn.example.com ``` `qiniu-access-key`, `qiniu-secret-key`, `qiniu-bucket` can be replaced with the system environments `QINIU_ACCESS_KEY`, `QINIU_SECRET_KEY`, `QINIU_BUCKET`. + +## Prefetch + +When `--prefetch` is enabled, all successfully uploaded file URLs will be submitted to Qiniu DCDN for prefetching (cache warming) after the upload is completed. + +### Options + +| Option | Description | Required | +|---|---|---| +| `--prefetch` | Enable DCDN prefetch after upload. | No | +| `--prefetch-domain ` | The CDN domain for constructing prefetch URLs, e.g. `https://cdn.example.com`. | Yes (when `--prefetch` is enabled) | +| `--prefetch-product ` | The prefetch product type, defaults to `dcdn`. | No | + +`prefetch-domain` can also be set via the environment variable `QINIU_PREFETCH_DOMAIN`. + +### How it works + +- URLs are submitted in batches of 60 (Qiniu API limit). +- A 1-second interval is applied between batches to respect the rate limit. +- On QPS throttling (403024) or server errors (500000), the batch will be retried up to 3 times with exponential backoff. diff --git a/QiniuUploader/index.mjs b/QiniuUploader/index.mjs index acfd0d4..1a459c0 100644 --- a/QiniuUploader/index.mjs +++ b/QiniuUploader/index.mjs @@ -2,6 +2,8 @@ import fs from 'fs' import path from 'path' +import crypto from 'crypto' +import https from 'https' import qiniu from 'qiniu' import { program } from 'commander' import assert from 'assert' @@ -15,6 +17,9 @@ program .option('--qiniu-bucket ', '七牛的 Bucket 名称.') .option('--qiniu-access-key ', '七牛 Access Key.') .option('--qiniu-secret-key ', '七牛 Secret Key.') + .option('--prefetch', '上传完成后执行 DCDN 预取.') + .option('--prefetch-domain ', '预取使用的 CDN 域名, 如 https://cdn.example.com.') + .option('--prefetch-product ', '预取产品类型, 默认 dcdn.', 'dcdn') program.parse() @@ -24,6 +29,9 @@ const qiniuBucket = options.qiniuBucket || process.env.QINIU_BUCKET const qiniuAccessKey = options.qiniuAccessKey || process.env.QINIU_ACCESS_KEY const qiniuSecretKey = options.qiniuSecretKey || process.env.QINIU_SECRET_KEY const qiniuBasePath = options.qiniuBasePath +const prefetchEnabled = !!options.prefetch +const prefetchDomain = options.prefetchDomain || process.env.QINIU_PREFETCH_DOMAIN +const prefetchProduct = options.prefetchProduct assert(distDirPath, '--upload-folder 未指定.') assert(qiniuBucket, '--qiniu-bucket 未指定.') @@ -31,8 +39,18 @@ assert(qiniuAccessKey, '--qiniu-access-key 未指定.') assert(qiniuSecretKey, '--qiniu-secret-key 未指定.') assert(qiniuBasePath, '--qiniu-base-path 未指定.') +if (prefetchEnabled) { + assert(prefetchDomain, '--prefetch-domain 未指定, 启用 --prefetch 时必须提供.') +} + distDirPath = path.resolve(process.cwd(), distDirPath) +// ============================================================ +// Upload. +// ============================================================ + +const uploadedKeys = [] + const walkDir = async (dirPath) => { for (const filename of fs.readdirSync(dirPath)) { const filePath = path.resolve(dirPath, filename) @@ -64,22 +82,178 @@ const walkDir = async (dirPath) => { ) const uploadToken = putPolicy.uploadToken(mac) const putExtra = new qiniu.form_up.PutExtra() - const config = new qiniu.conf.Config({ + const uploadConfig = new qiniu.conf.Config({ zone: qiniu.zone.Zone_z0 }) - const formUploader = new qiniu.form_up.FormUploader(config) - const uploadResult= await formUploader.putStream(uploadToken, key, stream, putExtra, () => { + const formUploader = new qiniu.form_up.FormUploader(uploadConfig) + const uploadResult = await formUploader.putStream(uploadToken, key, stream, putExtra, () => { // ... }) const uploadedKey = uploadResult.data.key if (uploadedKey) { console.log('Upload done, key:', uploadedKey) + uploadedKeys.push(uploadedKey) } else { console.error('No key was returned from Qiniu, upload may failed.') } } } -walkDir(distDirPath) - .catch(console.error) +// ============================================================ +// Prefetch. +// ============================================================ + +/** + * Generate QBox access token. + * QBox signing only signs the request path, not the body. + * + * @param {string} apiPath - The API path, e.g. "/v2/tune/prefetch". + * @param {string} accessKey + * @param {string} secretKey + * @returns {string} The full Authorization header value. + */ +const generateQBoxToken = (apiPath, accessKey, secretKey) => { + const signingStr = apiPath + '\n' + const sign = crypto + .createHmac('sha1', secretKey) + .update(signingStr) + .digest('base64') + const safeSign = sign + .replace(/\+/g, '-') + .replace(/\//g, '_') + return `QBox ${accessKey}:${safeSign}` +} + +/** + * Send a prefetch request to Qiniu DCDN. + * + * @param {string[]} urls - URLs to prefetch, max 60. + * @param {{accessKey: string, secretKey: string, product: string}} param + * @returns {Promise<{code: number, error: string, requestId: string, invalidUrls: string[]|null, quotaDay: number, surplusDay: number}>} + */ +const sendPrefetchRequest = (urls, { accessKey, secretKey, product }) => { + const apiPath = '/v2/tune/prefetch' + const token = generateQBoxToken(apiPath, accessKey, secretKey) + const body = JSON.stringify({ urls, product }) + + return new Promise((resolve, reject) => { + const req = https.request({ + hostname: 'fusion.qiniuapi.com', + port: 443, + path: apiPath, + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'Content-Length': Buffer.byteLength(body), + Authorization: token + } + }, (res) => { + let data = '' + res.on('data', (chunk) => { data += chunk }) + res.on('end', () => { + try { + resolve(JSON.parse(data)) + } catch { + reject(new Error(`Failed to parse prefetch response: ${data}`)) + } + }) + }) + + req.on('error', reject) + req.write(body) + req.end() + }) +} + +const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)) + +/** + * Prefetch all URLs in batches. + * Each batch contains at most 60 URLs, with 1 second interval between batches. + * Retries up to 3 times on QPS limit (403024) or server error (500000). + * + * @param {string[]} allUrls + * @param {{accessKey: string, secretKey: string, product: string}} param + */ +const prefetchAllUrls = async (allUrls, { accessKey, secretKey, product }) => { + const batchSize = 60 + const maxRetries = 3 + const batches = [] + + for (let i = 0; i < allUrls.length; i += batchSize) { + batches.push(allUrls.slice(i, i + batchSize)) + } + + console.log(`Prefetching ${allUrls.length} URL(s) in ${batches.length} batch(es) ...`) + + for (let i = 0; i < batches.length; i++) { + const batch = batches[i] + let retries = 0 + let success = false + while (retries <= maxRetries && !success) { + if (retries > 0) { + const delay = retries * 2000 + console.log(` Retry ${retries}/${maxRetries} after ${delay}ms ...`) + await sleep(delay) + } + + const result = await sendPrefetchRequest(batch, { accessKey, secretKey, product }) + + if (result.code === 200) { + console.log( + ` Batch ${i + 1}/${batches.length}: ${batch.length} URL(s) prefetched.` + + ` RequestId: ${result.requestId}.` + + ` Quota: ${result.surplusDay}/${result.quotaDay}.` + ) + success = true + } else if (result.code === 400033) { + console.error(` Batch ${i + 1}: Daily prefetch quota exceeded. Stopping.`) + return + } else if ((result.code === 403024 || result.code === 500000) && retries < maxRetries) { + console.warn(` Batch ${i + 1}: [${result.code}] ${result.error}, will retry.`) + retries++ + } else { + console.error(` Batch ${i + 1}: Failed. [${result.code}] ${result.error}`) + if (result.invalidUrls && result.invalidUrls.length > 0) { + console.error(` Invalid URLs: ${result.invalidUrls.join(', ')}`) + } + success = true // Move on to next batch. + } + } + + // Rate limit: wait 1 second between batches to stay within 60 URL/s. + if (i < batches.length - 1) { + await sleep(1000) + } + } +} + +// ============================================================ +// Main. +// ============================================================ + +const main = async () => { + await walkDir(distDirPath) + + if (prefetchEnabled) { + if (uploadedKeys.length < 1) { + console.log('\nNo files were uploaded, skipping prefetch.') + return + } + + const domain = prefetchDomain.replace(/\/+$/, '') + const urls = uploadedKeys.map((key) => `${domain}/${key}`) + + console.log(`\nUpload completed. Starting prefetch for ${urls.length} URL(s) ...`) + await prefetchAllUrls(urls, { + accessKey: qiniuAccessKey, + secretKey: qiniuSecretKey, + product: prefetchProduct + }) + console.log('Prefetch completed.') + } +} + +main() + .catch(console.error) diff --git a/QiniuUploader/package.json b/QiniuUploader/package.json index 873f67a..3cab469 100644 --- a/QiniuUploader/package.json +++ b/QiniuUploader/package.json @@ -1,6 +1,6 @@ { "name": "@lancercomet/qiniu-uploader", - "version": "0.1.2", + "version": "0.2.0-test", "description": "This package is used for uploading your assets to the Qiniu cloud.", "main": "index.mjs", "scripts": {}, From 8f4c1465af68cdb3cee3e6854124852aa7be85cc Mon Sep 17 00:00:00 2001 From: LancerComet Date: Wed, 15 Apr 2026 18:05:48 +0800 Subject: [PATCH 2/2] + Update API endpoint. --- QiniuUploader/README.md | 8 ++++---- QiniuUploader/index.mjs | 29 +++++++++++++++++------------ QiniuUploader/package-lock.json | 4 ++-- QiniuUploader/package.json | 2 +- 4 files changed, 24 insertions(+), 19 deletions(-) diff --git a/QiniuUploader/README.md b/QiniuUploader/README.md index 201cdc8..76d41e1 100644 --- a/QiniuUploader/README.md +++ b/QiniuUploader/README.md @@ -22,20 +22,20 @@ npx qiniu-uploader --upload-folder dist \ ## Prefetch -When `--prefetch` is enabled, all successfully uploaded file URLs will be submitted to Qiniu DCDN for prefetching (cache warming) after the upload is completed. +When `--prefetch` is enabled, all successfully uploaded file URLs will be submitted to Qiniu Fusion CDN for prefetching (cache warming) after the upload is completed. ### Options | Option | Description | Required | |---|---|---| -| `--prefetch` | Enable DCDN prefetch after upload. | No | +| `--prefetch` | Enable CDN prefetch after upload. | No | | `--prefetch-domain ` | The CDN domain for constructing prefetch URLs, e.g. `https://cdn.example.com`. | Yes (when `--prefetch` is enabled) | -| `--prefetch-product ` | The prefetch product type, defaults to `dcdn`. | No | +| `--prefetch-product ` | The prefetch product type (e.g. `dcdn`). Not required for Fusion CDN. | No | `prefetch-domain` can also be set via the environment variable `QINIU_PREFETCH_DOMAIN`. ### How it works -- URLs are submitted in batches of 60 (Qiniu API limit). +- URLs are submitted in batches of 20 (Qiniu Fusion CDN API limit). - A 1-second interval is applied between batches to respect the rate limit. - On QPS throttling (403024) or server errors (500000), the batch will be retried up to 3 times with exponential backoff. diff --git a/QiniuUploader/index.mjs b/QiniuUploader/index.mjs index 1a459c0..3a1494d 100644 --- a/QiniuUploader/index.mjs +++ b/QiniuUploader/index.mjs @@ -17,9 +17,9 @@ program .option('--qiniu-bucket ', '七牛的 Bucket 名称.') .option('--qiniu-access-key ', '七牛 Access Key.') .option('--qiniu-secret-key ', '七牛 Secret Key.') - .option('--prefetch', '上传完成后执行 DCDN 预取.') + .option('--prefetch', '上传完成后执行预取.') .option('--prefetch-domain ', '预取使用的 CDN 域名, 如 https://cdn.example.com.') - .option('--prefetch-product ', '预取产品类型, 默认 dcdn.', 'dcdn') + .option('--prefetch-product ', '预取产品类型, 如 dcdn. 不指定则为融合 CDN.') program.parse() @@ -125,16 +125,20 @@ const generateQBoxToken = (apiPath, accessKey, secretKey) => { } /** - * Send a prefetch request to Qiniu DCDN. + * Send a prefetch request to Qiniu Fusion CDN. * - * @param {string[]} urls - URLs to prefetch, max 60. - * @param {{accessKey: string, secretKey: string, product: string}} param - * @returns {Promise<{code: number, error: string, requestId: string, invalidUrls: string[]|null, quotaDay: number, surplusDay: number}>} + * @param {string[]} urls - URLs to prefetch, max 20. + * @param {{accessKey: string, secretKey: string, product?: string}} param + * @returns {Promise<{code: number, error: string, requestId: string, taskIds: Object|null, invalidUrls: string[]|null, quotaDay: number, surplusDay: number}>} */ const sendPrefetchRequest = (urls, { accessKey, secretKey, product }) => { const apiPath = '/v2/tune/prefetch' const token = generateQBoxToken(apiPath, accessKey, secretKey) - const body = JSON.stringify({ urls, product }) + const payload = { urls } + if (product) { + payload.product = product + } + const body = JSON.stringify(payload) return new Promise((resolve, reject) => { const req = https.request({ @@ -169,14 +173,14 @@ const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)) /** * Prefetch all URLs in batches. - * Each batch contains at most 60 URLs, with 1 second interval between batches. + * Each batch contains at most 20 URLs, with 1 second interval between batches. * Retries up to 3 times on QPS limit (403024) or server error (500000). * * @param {string[]} allUrls - * @param {{accessKey: string, secretKey: string, product: string}} param + * @param {{accessKey: string, secretKey: string, product?: string}} param */ const prefetchAllUrls = async (allUrls, { accessKey, secretKey, product }) => { - const batchSize = 60 + const batchSize = 20 const maxRetries = 3 const batches = [] @@ -201,8 +205,9 @@ const prefetchAllUrls = async (allUrls, { accessKey, secretKey, product }) => { const result = await sendPrefetchRequest(batch, { accessKey, secretKey, product }) if (result.code === 200) { + const taskCount = result.taskIds ? Object.keys(result.taskIds).length : 0 console.log( - ` Batch ${i + 1}/${batches.length}: ${batch.length} URL(s) prefetched.` + + ` Batch ${i + 1}/${batches.length}: ${batch.length} URL(s) prefetched (${taskCount} task(s)).` + ` RequestId: ${result.requestId}.` + ` Quota: ${result.surplusDay}/${result.quotaDay}.` ) @@ -222,7 +227,7 @@ const prefetchAllUrls = async (allUrls, { accessKey, secretKey, product }) => { } } - // Rate limit: wait 1 second between batches to stay within 60 URL/s. + // Rate limit: wait 1 second between batches to stay within QPS limit. if (i < batches.length - 1) { await sleep(1000) } diff --git a/QiniuUploader/package-lock.json b/QiniuUploader/package-lock.json index 7c919c6..9ed0230 100644 --- a/QiniuUploader/package-lock.json +++ b/QiniuUploader/package-lock.json @@ -1,12 +1,12 @@ { "name": "@lancercomet/qiniu-uploader", - "version": "0.1.2", + "version": "0.2.1-test", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@lancercomet/qiniu-uploader", - "version": "0.1.2", + "version": "0.2.1-test", "license": "Apache-2.0", "dependencies": { "commander": "^11.1.0", diff --git a/QiniuUploader/package.json b/QiniuUploader/package.json index 3cab469..280a605 100644 --- a/QiniuUploader/package.json +++ b/QiniuUploader/package.json @@ -1,6 +1,6 @@ { "name": "@lancercomet/qiniu-uploader", - "version": "0.2.0-test", + "version": "0.2.1-test", "description": "This package is used for uploading your assets to the Qiniu cloud.", "main": "index.mjs", "scripts": {},