Feature/qiniu prefetch

QiniuUploader/README.md

@@ -13,7 +13,29 @@ npx qiniu-uploader --upload-folder dist \
   --qiniu-base-path some-folder-on-qiniu/sub-directory \
   --qiniu-access-key *** \
   --qiniu-secret-key *** \
-  --qiniu-bucket some-bucket
+  --qiniu-bucket some-bucket \
+  --prefetch \
+  --prefetch-domain https://cdn.example.com
 ```
 
 `qiniu-access-key`, `qiniu-secret-key`, `qiniu-bucket` can be replaced with the system environments `QINIU_ACCESS_KEY`, `QINIU_SECRET_KEY`, `QINIU_BUCKET`.
+
+## Prefetch
+
+When `--prefetch` is enabled, all successfully uploaded file URLs will be submitted to Qiniu Fusion CDN for prefetching (cache warming) after the upload is completed.
+
+### Options
+
+| Option | Description | Required |
+|---|---|---|
+| `--prefetch` | Enable CDN prefetch after upload. | No |
+| `--prefetch-domain <url>` | The CDN domain for constructing prefetch URLs, e.g. `https://cdn.example.com`. | Yes (when `--prefetch` is enabled) |
+| `--prefetch-product <str>` | The prefetch product type (e.g. `dcdn`). Not required for Fusion CDN. | No |
+
+`prefetch-domain` can also be set via the environment variable `QINIU_PREFETCH_DOMAIN`.
+
+### How it works
+
+- URLs are submitted in batches of 20 (Qiniu Fusion CDN API limit).
+- A 1-second interval is applied between batches to respect the rate limit.
+- On QPS throttling (403024) or server errors (500000), the batch will be retried up to 3 times with exponential backoff.

QiniuUploader/index.mjs

@@ -2,6 +2,8 @@
 
 import fs from 'fs'
 import path from 'path'
+import crypto from 'crypto'
+import https from 'https'
 import qiniu from 'qiniu'
 import { program } from 'commander'
 import assert from 'assert'
@@ -15,6 +17,9 @@ program
   .option('--qiniu-bucket <char>', '七牛的 Bucket 名称.')
   .option('--qiniu-access-key <char>', '七牛 Access Key.')
   .option('--qiniu-secret-key <char>', '七牛 Secret Key.')
+  .option('--prefetch', '上传完成后执行预取.')
+  .option('--prefetch-domain <char>', '预取使用的 CDN 域名, 如 https://cdn.example.com.')
+  .option('--prefetch-product <char>', '预取产品类型, 如 dcdn. 不指定则为融合 CDN.')
 
 program.parse()
 
@@ -24,15 +29,28 @@ const qiniuBucket = options.qiniuBucket || process.env.QINIU_BUCKET
 const qiniuAccessKey = options.qiniuAccessKey || process.env.QINIU_ACCESS_KEY
 const qiniuSecretKey = options.qiniuSecretKey || process.env.QINIU_SECRET_KEY
 const qiniuBasePath = options.qiniuBasePath
+const prefetchEnabled = !!options.prefetch
+const prefetchDomain = options.prefetchDomain || process.env.QINIU_PREFETCH_DOMAIN
+const prefetchProduct = options.prefetchProduct
 
 assert(distDirPath, '--upload-folder 未指定.')
 assert(qiniuBucket, '--qiniu-bucket 未指定.')
 assert(qiniuAccessKey, '--qiniu-access-key 未指定.')
 assert(qiniuSecretKey, '--qiniu-secret-key 未指定.')
 assert(qiniuBasePath, '--qiniu-base-path 未指定.')
 
+if (prefetchEnabled) {
+  assert(prefetchDomain, '--prefetch-domain 未指定, 启用 --prefetch 时必须提供.')
+}
+
 distDirPath = path.resolve(process.cwd(), distDirPath)
 
+// ============================================================
+// Upload.
+// ============================================================
+
+const uploadedKeys = []
+
 const walkDir = async (dirPath) => {
   for (const filename of fs.readdirSync(dirPath)) {
     const filePath = path.resolve(dirPath, filename)
@@ -64,22 +82,183 @@ const walkDir = async (dirPath) => {
     )
     const uploadToken = putPolicy.uploadToken(mac)
     const putExtra = new qiniu.form_up.PutExtra()
-    const config = new qiniu.conf.Config({
+    const uploadConfig = new qiniu.conf.Config({
       zone: qiniu.zone.Zone_z0
     })
-    const formUploader = new qiniu.form_up.FormUploader(config)
-    const uploadResult= await formUploader.putStream(uploadToken, key, stream, putExtra, () => {
+    const formUploader = new qiniu.form_up.FormUploader(uploadConfig)
+    const uploadResult = await formUploader.putStream(uploadToken, key, stream, putExtra, () => {
       // ...
     })
     const uploadedKey = uploadResult.data.key
     if (uploadedKey) {
       console.log('Upload done, key:', uploadedKey)
+      uploadedKeys.push(uploadedKey)
     } else {
       console.error('No key was returned from Qiniu, upload may failed.')
     }
   }
 }
 
-walkDir(distDirPath)
-  .catch(console.error)
+// ============================================================
+// Prefetch.
+// ============================================================
+
+/**
+ * Generate QBox access token.
+ * QBox signing only signs the request path, not the body.
+ *
+ * @param {string} apiPath - The API path, e.g. "/v2/tune/prefetch".
+ * @param {string} accessKey
+ * @param {string} secretKey
+ * @returns {string} The full Authorization header value.
+ */
+const generateQBoxToken = (apiPath, accessKey, secretKey) => {
+  const signingStr = apiPath + '\n'
+  const sign = crypto
+    .createHmac('sha1', secretKey)
+    .update(signingStr)
+    .digest('base64')
+  const safeSign = sign
+    .replace(/\+/g, '-')
+    .replace(/\//g, '_')
+  return `QBox ${accessKey}:${safeSign}`
+}
+
+/**
+ * Send a prefetch request to Qiniu Fusion CDN.
+ *
+ * @param {string[]} urls - URLs to prefetch, max 20.
+ * @param {{accessKey: string, secretKey: string, product?: string}} param
+ * @returns {Promise<{code: number, error: string, requestId: string, taskIds: Object<string, string>|null, invalidUrls: string[]|null, quotaDay: number, surplusDay: number}>}
+ */
+const sendPrefetchRequest = (urls, { accessKey, secretKey, product }) => {
+  const apiPath = '/v2/tune/prefetch'
+  const token = generateQBoxToken(apiPath, accessKey, secretKey)
+  const payload = { urls }
+  if (product) {
+    payload.product = product
+  }
+  const body = JSON.stringify(payload)
+
+  return new Promise((resolve, reject) => {
+    const req = https.request({
+      hostname: 'fusion.qiniuapi.com',
+      port: 443,
+      path: apiPath,
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+        'Content-Length': Buffer.byteLength(body),
+        Authorization: token
+      }
+    }, (res) => {
+      let data = ''
+      res.on('data', (chunk) => { data += chunk })
+      res.on('end', () => {
+        try {
+          resolve(JSON.parse(data))
+        } catch {
+          reject(new Error(`Failed to parse prefetch response: ${data}`))
+        }
+      })
+    })
+
+    req.on('error', reject)
+    req.write(body)
+    req.end()
+  })
+}
+
+const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms))
+
+/**
+ * Prefetch all URLs in batches.
+ * Each batch contains at most 20 URLs, with 1 second interval between batches.
+ * Retries up to 3 times on QPS limit (403024) or server error (500000).
+ *
+ * @param {string[]} allUrls
+ * @param {{accessKey: string, secretKey: string, product?: string}} param
+ */
+const prefetchAllUrls = async (allUrls, { accessKey, secretKey, product }) => {
+  const batchSize = 20
+  const maxRetries = 3
+  const batches = []
+
+  for (let i = 0; i < allUrls.length; i += batchSize) {
+    batches.push(allUrls.slice(i, i + batchSize))
+  }
+
+  console.log(`Prefetching ${allUrls.length} URL(s) in ${batches.length} batch(es) ...`)
+
+  for (let i = 0; i < batches.length; i++) {
+    const batch = batches[i]
+    let retries = 0
+    let success = false
 
+    while (retries <= maxRetries && !success) {
+      if (retries > 0) {
+        const delay = retries * 2000
+        console.log(`  Retry ${retries}/${maxRetries} after ${delay}ms ...`)
+        await sleep(delay)
+      }
+
+      const result = await sendPrefetchRequest(batch, { accessKey, secretKey, product })
+
+      if (result.code === 200) {
+        const taskCount = result.taskIds ? Object.keys(result.taskIds).length : 0
+        console.log(
+          `  Batch ${i + 1}/${batches.length}: ${batch.length} URL(s) prefetched (${taskCount} task(s)).` +
+          ` RequestId: ${result.requestId}.` +
+          ` Quota: ${result.surplusDay}/${result.quotaDay}.`
+        )
+        success = true
+      } else if (result.code === 400033) {
+        console.error(`  Batch ${i + 1}: Daily prefetch quota exceeded. Stopping.`)
+        return
+      } else if ((result.code === 403024 || result.code === 500000) && retries < maxRetries) {
+        console.warn(`  Batch ${i + 1}: [${result.code}] ${result.error}, will retry.`)
+        retries++
+      } else {
+        console.error(`  Batch ${i + 1}: Failed. [${result.code}] ${result.error}`)
+        if (result.invalidUrls && result.invalidUrls.length > 0) {
+          console.error(`  Invalid URLs: ${result.invalidUrls.join(', ')}`)
+        }
+        success = true // Move on to next batch.
+      }
+    }
+
+    // Rate limit: wait 1 second between batches to stay within QPS limit.
+    if (i < batches.length - 1) {
+      await sleep(1000)
+    }
+  }
+}
+
+// ============================================================
+// Main.
+// ============================================================
+
+const main = async () => {
+  await walkDir(distDirPath)
+
+  if (prefetchEnabled) {
+    if (uploadedKeys.length < 1) {
+      console.log('\nNo files were uploaded, skipping prefetch.')
+      return
+    }
+
+    const domain = prefetchDomain.replace(/\/+$/, '')
+    const urls = uploadedKeys.map((key) => `${domain}/${key}`)
+
+    console.log(`\nUpload completed. Starting prefetch for ${urls.length} URL(s) ...`)
+    await prefetchAllUrls(urls, {
+      accessKey: qiniuAccessKey,
+      secretKey: qiniuSecretKey,
+      product: prefetchProduct
+    })
+    console.log('Prefetch completed.')
+  }
+}
+
+main()
+  .catch(console.error)

QiniuUploader/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lancercomet/qiniu-uploader",
-  "version": "0.1.2",
+  "version": "0.2.1-test",
   "description": "This package is used for uploading your assets to the Qiniu cloud.",
   "main": "index.mjs",
   "scripts": {},