diff --git a/QiniuUploader/README.md b/QiniuUploader/README.md index b256b61..76d41e1 100644 --- a/QiniuUploader/README.md +++ b/QiniuUploader/README.md @@ -13,7 +13,29 @@ npx qiniu-uploader --upload-folder dist \ --qiniu-base-path some-folder-on-qiniu/sub-directory \ --qiniu-access-key *** \ --qiniu-secret-key *** \ - --qiniu-bucket some-bucket + --qiniu-bucket some-bucket \ + --prefetch \ + --prefetch-domain https://cdn.example.com ``` `qiniu-access-key`, `qiniu-secret-key`, `qiniu-bucket` can be replaced with the system environments `QINIU_ACCESS_KEY`, `QINIU_SECRET_KEY`, `QINIU_BUCKET`. + +## Prefetch + +When `--prefetch` is enabled, all successfully uploaded file URLs will be submitted to Qiniu Fusion CDN for prefetching (cache warming) after the upload is completed. + +### Options + +| Option | Description | Required | +|---|---|---| +| `--prefetch` | Enable CDN prefetch after upload. | No | +| `--prefetch-domain ` | The CDN domain for constructing prefetch URLs, e.g. `https://cdn.example.com`. | Yes (when `--prefetch` is enabled) | +| `--prefetch-product ` | The prefetch product type (e.g. `dcdn`). Not required for Fusion CDN. | No | + +`prefetch-domain` can also be set via the environment variable `QINIU_PREFETCH_DOMAIN`. + +### How it works + +- URLs are submitted in batches of 20 (Qiniu Fusion CDN API limit). +- A 1-second interval is applied between batches to respect the rate limit. +- On QPS throttling (403024) or server errors (500000), the batch will be retried up to 3 times with exponential backoff. diff --git a/QiniuUploader/index.mjs b/QiniuUploader/index.mjs index acfd0d4..3a1494d 100644 --- a/QiniuUploader/index.mjs +++ b/QiniuUploader/index.mjs @@ -2,6 +2,8 @@ import fs from 'fs' import path from 'path' +import crypto from 'crypto' +import https from 'https' import qiniu from 'qiniu' import { program } from 'commander' import assert from 'assert' @@ -15,6 +17,9 @@ program .option('--qiniu-bucket ', '七牛的 Bucket 名称.') .option('--qiniu-access-key ', '七牛 Access Key.') .option('--qiniu-secret-key ', '七牛 Secret Key.') + .option('--prefetch', '上传完成后执行预取.') + .option('--prefetch-domain ', '预取使用的 CDN 域名, 如 https://cdn.example.com.') + .option('--prefetch-product ', '预取产品类型, 如 dcdn. 不指定则为融合 CDN.') program.parse() @@ -24,6 +29,9 @@ const qiniuBucket = options.qiniuBucket || process.env.QINIU_BUCKET const qiniuAccessKey = options.qiniuAccessKey || process.env.QINIU_ACCESS_KEY const qiniuSecretKey = options.qiniuSecretKey || process.env.QINIU_SECRET_KEY const qiniuBasePath = options.qiniuBasePath +const prefetchEnabled = !!options.prefetch +const prefetchDomain = options.prefetchDomain || process.env.QINIU_PREFETCH_DOMAIN +const prefetchProduct = options.prefetchProduct assert(distDirPath, '--upload-folder 未指定.') assert(qiniuBucket, '--qiniu-bucket 未指定.') @@ -31,8 +39,18 @@ assert(qiniuAccessKey, '--qiniu-access-key 未指定.') assert(qiniuSecretKey, '--qiniu-secret-key 未指定.') assert(qiniuBasePath, '--qiniu-base-path 未指定.') +if (prefetchEnabled) { + assert(prefetchDomain, '--prefetch-domain 未指定, 启用 --prefetch 时必须提供.') +} + distDirPath = path.resolve(process.cwd(), distDirPath) +// ============================================================ +// Upload. +// ============================================================ + +const uploadedKeys = [] + const walkDir = async (dirPath) => { for (const filename of fs.readdirSync(dirPath)) { const filePath = path.resolve(dirPath, filename) @@ -64,22 +82,183 @@ const walkDir = async (dirPath) => { ) const uploadToken = putPolicy.uploadToken(mac) const putExtra = new qiniu.form_up.PutExtra() - const config = new qiniu.conf.Config({ + const uploadConfig = new qiniu.conf.Config({ zone: qiniu.zone.Zone_z0 }) - const formUploader = new qiniu.form_up.FormUploader(config) - const uploadResult= await formUploader.putStream(uploadToken, key, stream, putExtra, () => { + const formUploader = new qiniu.form_up.FormUploader(uploadConfig) + const uploadResult = await formUploader.putStream(uploadToken, key, stream, putExtra, () => { // ... }) const uploadedKey = uploadResult.data.key if (uploadedKey) { console.log('Upload done, key:', uploadedKey) + uploadedKeys.push(uploadedKey) } else { console.error('No key was returned from Qiniu, upload may failed.') } } } -walkDir(distDirPath) - .catch(console.error) +// ============================================================ +// Prefetch. +// ============================================================ + +/** + * Generate QBox access token. + * QBox signing only signs the request path, not the body. + * + * @param {string} apiPath - The API path, e.g. "/v2/tune/prefetch". + * @param {string} accessKey + * @param {string} secretKey + * @returns {string} The full Authorization header value. + */ +const generateQBoxToken = (apiPath, accessKey, secretKey) => { + const signingStr = apiPath + '\n' + const sign = crypto + .createHmac('sha1', secretKey) + .update(signingStr) + .digest('base64') + const safeSign = sign + .replace(/\+/g, '-') + .replace(/\//g, '_') + return `QBox ${accessKey}:${safeSign}` +} + +/** + * Send a prefetch request to Qiniu Fusion CDN. + * + * @param {string[]} urls - URLs to prefetch, max 20. + * @param {{accessKey: string, secretKey: string, product?: string}} param + * @returns {Promise<{code: number, error: string, requestId: string, taskIds: Object|null, invalidUrls: string[]|null, quotaDay: number, surplusDay: number}>} + */ +const sendPrefetchRequest = (urls, { accessKey, secretKey, product }) => { + const apiPath = '/v2/tune/prefetch' + const token = generateQBoxToken(apiPath, accessKey, secretKey) + const payload = { urls } + if (product) { + payload.product = product + } + const body = JSON.stringify(payload) + + return new Promise((resolve, reject) => { + const req = https.request({ + hostname: 'fusion.qiniuapi.com', + port: 443, + path: apiPath, + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'Content-Length': Buffer.byteLength(body), + Authorization: token + } + }, (res) => { + let data = '' + res.on('data', (chunk) => { data += chunk }) + res.on('end', () => { + try { + resolve(JSON.parse(data)) + } catch { + reject(new Error(`Failed to parse prefetch response: ${data}`)) + } + }) + }) + + req.on('error', reject) + req.write(body) + req.end() + }) +} + +const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)) + +/** + * Prefetch all URLs in batches. + * Each batch contains at most 20 URLs, with 1 second interval between batches. + * Retries up to 3 times on QPS limit (403024) or server error (500000). + * + * @param {string[]} allUrls + * @param {{accessKey: string, secretKey: string, product?: string}} param + */ +const prefetchAllUrls = async (allUrls, { accessKey, secretKey, product }) => { + const batchSize = 20 + const maxRetries = 3 + const batches = [] + + for (let i = 0; i < allUrls.length; i += batchSize) { + batches.push(allUrls.slice(i, i + batchSize)) + } + + console.log(`Prefetching ${allUrls.length} URL(s) in ${batches.length} batch(es) ...`) + + for (let i = 0; i < batches.length; i++) { + const batch = batches[i] + let retries = 0 + let success = false + while (retries <= maxRetries && !success) { + if (retries > 0) { + const delay = retries * 2000 + console.log(` Retry ${retries}/${maxRetries} after ${delay}ms ...`) + await sleep(delay) + } + + const result = await sendPrefetchRequest(batch, { accessKey, secretKey, product }) + + if (result.code === 200) { + const taskCount = result.taskIds ? Object.keys(result.taskIds).length : 0 + console.log( + ` Batch ${i + 1}/${batches.length}: ${batch.length} URL(s) prefetched (${taskCount} task(s)).` + + ` RequestId: ${result.requestId}.` + + ` Quota: ${result.surplusDay}/${result.quotaDay}.` + ) + success = true + } else if (result.code === 400033) { + console.error(` Batch ${i + 1}: Daily prefetch quota exceeded. Stopping.`) + return + } else if ((result.code === 403024 || result.code === 500000) && retries < maxRetries) { + console.warn(` Batch ${i + 1}: [${result.code}] ${result.error}, will retry.`) + retries++ + } else { + console.error(` Batch ${i + 1}: Failed. [${result.code}] ${result.error}`) + if (result.invalidUrls && result.invalidUrls.length > 0) { + console.error(` Invalid URLs: ${result.invalidUrls.join(', ')}`) + } + success = true // Move on to next batch. + } + } + + // Rate limit: wait 1 second between batches to stay within QPS limit. + if (i < batches.length - 1) { + await sleep(1000) + } + } +} + +// ============================================================ +// Main. +// ============================================================ + +const main = async () => { + await walkDir(distDirPath) + + if (prefetchEnabled) { + if (uploadedKeys.length < 1) { + console.log('\nNo files were uploaded, skipping prefetch.') + return + } + + const domain = prefetchDomain.replace(/\/+$/, '') + const urls = uploadedKeys.map((key) => `${domain}/${key}`) + + console.log(`\nUpload completed. Starting prefetch for ${urls.length} URL(s) ...`) + await prefetchAllUrls(urls, { + accessKey: qiniuAccessKey, + secretKey: qiniuSecretKey, + product: prefetchProduct + }) + console.log('Prefetch completed.') + } +} + +main() + .catch(console.error) diff --git a/QiniuUploader/package-lock.json b/QiniuUploader/package-lock.json index 7c919c6..9ed0230 100644 --- a/QiniuUploader/package-lock.json +++ b/QiniuUploader/package-lock.json @@ -1,12 +1,12 @@ { "name": "@lancercomet/qiniu-uploader", - "version": "0.1.2", + "version": "0.2.1-test", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@lancercomet/qiniu-uploader", - "version": "0.1.2", + "version": "0.2.1-test", "license": "Apache-2.0", "dependencies": { "commander": "^11.1.0", diff --git a/QiniuUploader/package.json b/QiniuUploader/package.json index 873f67a..280a605 100644 --- a/QiniuUploader/package.json +++ b/QiniuUploader/package.json @@ -1,6 +1,6 @@ { "name": "@lancercomet/qiniu-uploader", - "version": "0.1.2", + "version": "0.2.1-test", "description": "This package is used for uploading your assets to the Qiniu cloud.", "main": "index.mjs", "scripts": {},