diff --git a/.github/workflows/npm_publish_bq_scripts.yml b/.github/workflows/npm_publish_bq_scripts.yml index 2958e80572..abc3624552 100644 --- a/.github/workflows/npm_publish_bq_scripts.yml +++ b/.github/workflows/npm_publish_bq_scripts.yml @@ -18,7 +18,7 @@ jobs: - name: Setup node uses: actions/setup-node@v4 with: - node-version: 20 + node-version: 24 - name: NPM install run: npm install - name: Publish BigQuery Schema Views diff --git a/.github/workflows/scripts/npm_publish.sh b/.github/workflows/scripts/npm_publish.sh index da230f5dc2..8cd8120346 100755 --- a/.github/workflows/scripts/npm_publish.sh +++ b/.github/workflows/scripts/npm_publish.sh @@ -63,4 +63,6 @@ if npm_package_version_exists "$NPM_PACKAGE_NAME" "$NPM_PACKAGE_VERSION"; then fi echo "Attempting to publish $NPM_PACKAGE_NAME version $NPM_PACKAGE_VERSION..." -npm publish +npm --version +node --version +npm publish --provenance diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 59a6807011..89832a64ae 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - node: ["20"] + node: ["22"] name: node.js_${{ matrix.node }}_test steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index 96504aeb37..3e6db004a6 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -13,7 +13,7 @@ jobs: - name: Setup node uses: actions/setup-node@v3 with: - node-version: 20 + node-version: 22 - name: NPM install run: SKIP_POSTINSTALL=yes npm i - name: Prettier Lint Check diff --git a/firestore-bigquery-export/CHANGELOG.md b/firestore-bigquery-export/CHANGELOG.md index 969e82fff3..d856610c44 100644 --- a/firestore-bigquery-export/CHANGELOG.md +++ b/firestore-bigquery-export/CHANGELOG.md @@ -1,3 +1,13 @@ +## Version 0.3.0 + +breaking change: reject invalid partitioning configuration combinations at startup with explicit error messages + +fix: normalize `NONE` / `omit` partitioning sentinels before mapping to change-tracker 2.x partitioning strategy + +## Version 0.2.11 + +chore: bump firestore-bigquery-change-tracker dependency to v2 in functions package + ## Version 0.2.10 chore: add overrides for http-proxy-agent and inquirer to resolve npm audit vulnerabilities diff --git a/firestore-bigquery-export/extension.yaml b/firestore-bigquery-export/extension.yaml index cba62c04d3..11115bfdec 100644 --- a/firestore-bigquery-export/extension.yaml +++ b/firestore-bigquery-export/extension.yaml @@ -13,7 +13,7 @@ # limitations under the License. name: firestore-bigquery-export -version: 0.2.10 +version: 0.3.0 specVersion: v1beta displayName: Stream Firestore to BigQuery diff --git a/firestore-bigquery-export/firestore-bigquery-change-tracker/package-lock.json b/firestore-bigquery-export/firestore-bigquery-change-tracker/package-lock.json index 04bea730d6..b3eb4b9dcc 100644 --- a/firestore-bigquery-export/firestore-bigquery-change-tracker/package-lock.json +++ b/firestore-bigquery-export/firestore-bigquery-change-tracker/package-lock.json @@ -1,21 +1,18 @@ { "name": "@firebaseextensions/firestore-bigquery-change-tracker", - "version": "2.0.0", + "version": "2.0.2", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@firebaseextensions/firestore-bigquery-change-tracker", - "version": "2.0.0", + "version": "2.0.2", "license": "Apache-2.0", "dependencies": { "@google-cloud/bigquery": "^7.6.0", "@google-cloud/resource-manager": "^5.1.0", "firebase-admin": "^13.2.0", "firebase-functions": "^6.3.2", - "generate-schema": "^2.6.0", - "inquirer": "^6.4.0", - "lodash": "^4.17.14", "node-fetch": "^2.6.1", "sql-formatter": "^2.3.3", "traverse": "^0.6.6" @@ -1698,14 +1695,6 @@ "node": ">=8" } }, - "node_modules/ansi-escapes": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-3.2.0.tgz", - "integrity": "sha512-cBhpre4ma+U0T1oM5fXg7Dy1Jw7zzwv7lt/GoCpr+hDQJoYnKVPLL4dCvSEFMmQurOQvSrwT7SL/DAlhBI97RQ==", - "engines": { - "node": ">=4" - } - }, "node_modules/ansi-regex": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", @@ -2257,11 +2246,6 @@ "node": ">=10" } }, - "node_modules/chardet": { - "version": "0.7.0", - "resolved": "https://registry.npmjs.org/chardet/-/chardet-0.7.0.tgz", - "integrity": "sha512-mT8iDcrh03qDGRRmoA2hmBJnxpllMR+0/0qlzjqZES6NdiWDcZkCNAk4rPFZ9Q85r27unkiNNg8ZOiwZXBHwcA==" - }, "node_modules/check-error": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/check-error/-/check-error-1.0.2.tgz", @@ -2310,22 +2294,6 @@ "node": ">=6" } }, - "node_modules/cli-cursor": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/cli-cursor/-/cli-cursor-2.1.0.tgz", - "integrity": "sha512-8lgKz8LmCRYZZQDpRyT2m5rKJ08TnU4tR9FFFW2rxpxR1FzWi4PQ/NfyODchAatHaUgnSPVcx/R5w6NuTBzFiw==", - "dependencies": { - "restore-cursor": "^2.0.0" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/cli-width": { - "version": "2.2.1", - "resolved": "https://registry.npmjs.org/cli-width/-/cli-width-2.2.1.tgz", - "integrity": "sha512-GRMWDxpOB6Dgk2E5Uo+3eEBvtOOlimMmpbFiKuLFnQzYDavtLFY3K5ona41jgN/WdRZtG7utuVSVTL4HbZHGkw==" - }, "node_modules/cliui": { "version": "8.0.1", "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", @@ -2382,11 +2350,6 @@ "node": ">= 0.8" } }, - "node_modules/commander": { - "version": "2.20.3", - "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz", - "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==" - }, "node_modules/commondir": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/commondir/-/commondir-1.0.1.tgz", @@ -3053,19 +3016,6 @@ "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==" }, - "node_modules/external-editor": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/external-editor/-/external-editor-3.1.0.tgz", - "integrity": "sha512-hMQ4CX1p1izmuLYyZqLMO/qGNw10wSv9QDCPfzXfyFrOaCSSoRfqE1Kf1s5an66J5JZC62NewG+mK49jOCtQew==", - "dependencies": { - "chardet": "^0.7.0", - "iconv-lite": "^0.4.24", - "tmp": "^0.0.33" - }, - "engines": { - "node": ">=4" - } - }, "node_modules/farmhash-modern": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/farmhash-modern/-/farmhash-modern-1.1.0.tgz", @@ -3128,25 +3078,6 @@ "bser": "2.1.1" } }, - "node_modules/figures": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/figures/-/figures-2.0.0.tgz", - "integrity": "sha512-Oa2M9atig69ZkfwiApY8F2Yy+tzMbazyvqv21R0NsSC8floSOC09BbT1ITWAdoMGQvJ/aZnR1KMwdx9tvHnTNA==", - "dependencies": { - "escape-string-regexp": "^1.0.5" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/figures/node_modules/escape-string-regexp": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", - "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==", - "engines": { - "node": ">=0.8.0" - } - }, "node_modules/fill-range": { "version": "7.1.1", "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", @@ -3463,18 +3394,6 @@ "node": ">=14" } }, - "node_modules/generate-schema": { - "version": "2.6.0", - "resolved": "https://registry.npmjs.org/generate-schema/-/generate-schema-2.6.0.tgz", - "integrity": "sha512-EUBKfJNzT8f91xUk5X5gKtnbdejZeE065UAJ3BCzE8VEbvwKI9Pm5jaWmqVeK1MYc1g5weAVFDTSJzN7ymtTqA==", - "dependencies": { - "commander": "^2.9.0", - "type-of-is": "^3.4.0" - }, - "bin": { - "generate-schema": "bin/generate-schema" - } - }, "node_modules/gensync": { "version": "1.0.0-beta.2", "resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz", @@ -3906,151 +3825,6 @@ "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" }, - "node_modules/inquirer": { - "version": "6.5.2", - "resolved": "https://registry.npmjs.org/inquirer/-/inquirer-6.5.2.tgz", - "integrity": "sha512-cntlB5ghuB0iuO65Ovoi8ogLHiWGs/5yNrtUcKjFhSSiVeAIVpD7koaSU9RM8mpXw5YDi9RdYXGQMaOURB7ycQ==", - "dependencies": { - "ansi-escapes": "^3.2.0", - "chalk": "^2.4.2", - "cli-cursor": "^2.1.0", - "cli-width": "^2.0.0", - "external-editor": "^3.0.3", - "figures": "^2.0.0", - "lodash": "^4.17.12", - "mute-stream": "0.0.7", - "run-async": "^2.2.0", - "rxjs": "^6.4.0", - "string-width": "^2.1.0", - "strip-ansi": "^5.1.0", - "through": "^2.3.6" - }, - "engines": { - "node": ">=6.0.0" - } - }, - "node_modules/inquirer/node_modules/ansi-regex": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-3.0.1.tgz", - "integrity": "sha512-+O9Jct8wf++lXxxFc4hc8LsjaSq0HFzzL7cVsw8pRDIPdjKD2mT4ytDZlLuSBZ4cLKZFXIrMGO7DbQCtMJJMKw==", - "engines": { - "node": ">=4" - } - }, - "node_modules/inquirer/node_modules/ansi-styles": { - "version": "3.2.1", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", - "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", - "dependencies": { - "color-convert": "^1.9.0" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/inquirer/node_modules/chalk": { - "version": "2.4.2", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", - "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", - "dependencies": { - "ansi-styles": "^3.2.1", - "escape-string-regexp": "^1.0.5", - "supports-color": "^5.3.0" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/inquirer/node_modules/color-convert": { - "version": "1.9.3", - "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", - "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", - "dependencies": { - "color-name": "1.1.3" - } - }, - "node_modules/inquirer/node_modules/color-name": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", - "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==" - }, - "node_modules/inquirer/node_modules/escape-string-regexp": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", - "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==", - "engines": { - "node": ">=0.8.0" - } - }, - "node_modules/inquirer/node_modules/has-flag": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", - "integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==", - "engines": { - "node": ">=4" - } - }, - "node_modules/inquirer/node_modules/is-fullwidth-code-point": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz", - "integrity": "sha512-VHskAKYM8RfSFXwee5t5cbN5PZeq1Wrh6qd5bkyiXIf6UQcN6w/A0eXM9r6t8d+GYOh+o6ZhiEnb88LN/Y8m2w==", - "engines": { - "node": ">=4" - } - }, - "node_modules/inquirer/node_modules/string-width": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-2.1.1.tgz", - "integrity": "sha512-nOqH59deCq9SRHlxq1Aw85Jnt4w6KvLKqWVik6oA9ZklXLNIOlqg4F2yrT1MVaTjAqvVwdfeZ7w7aCvJD7ugkw==", - "dependencies": { - "is-fullwidth-code-point": "^2.0.0", - "strip-ansi": "^4.0.0" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/inquirer/node_modules/string-width/node_modules/strip-ansi": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-4.0.0.tgz", - "integrity": "sha512-4XaJ2zQdCzROZDivEVIDPkcQn8LMFSa8kj8Gxb/Lnwzv9A8VctNZ+lfivC/sV3ivW8ElJTERXZoPBRrZKkNKow==", - "dependencies": { - "ansi-regex": "^3.0.0" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/inquirer/node_modules/strip-ansi": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-5.2.0.tgz", - "integrity": "sha512-DuRs1gKbBqsMKIZlrffwlug8MHkcnpjs5VPmL1PAh+mA30U0DTotfDZ0d2UUsXpPmPmMMJ6W773MaA3J+lbiWA==", - "dependencies": { - "ansi-regex": "^4.1.0" - }, - "engines": { - "node": ">=6" - } - }, - "node_modules/inquirer/node_modules/strip-ansi/node_modules/ansi-regex": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.1.tgz", - "integrity": "sha512-ILlv4k/3f6vfQ4OoP2AGvirOktlQ98ZEL1k9FaQjxa3L1abBgbuTDAdPOpvbGncC0BTVQrl+OM8xZGK6tWXt7g==", - "engines": { - "node": ">=6" - } - }, - "node_modules/inquirer/node_modules/supports-color": { - "version": "5.5.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", - "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", - "dependencies": { - "has-flag": "^3.0.0" - }, - "engines": { - "node": ">=4" - } - }, "node_modules/ipaddr.js": { "version": "1.9.1", "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", @@ -5406,14 +5180,6 @@ "node": ">= 0.6" } }, - "node_modules/mimic-fn": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-1.2.0.tgz", - "integrity": "sha512-jf84uxzwiuiIVKiOLpfYk7N46TSy8ubTonmneY9vrpHNAnp0QBt2BxWV9dO3/j+BoVAb+a5G6YDPW3M5HOdMWQ==", - "engines": { - "node": ">=4" - } - }, "node_modules/minimatch": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", @@ -5463,11 +5229,6 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" }, - "node_modules/mute-stream": { - "version": "0.0.7", - "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-0.0.7.tgz", - "integrity": "sha512-r65nCZhrbXXb6dXOACihYApHw2Q6pV0M3V0PSxd74N0+D8nzAdEAITq2oAjA1jVnKI+tGvEBUpqiMh0+rW6zDQ==" - }, "node_modules/natural-compare": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", @@ -5725,25 +5486,6 @@ "wrappy": "1" } }, - "node_modules/onetime": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/onetime/-/onetime-2.0.1.tgz", - "integrity": "sha512-oyyPpiMaKARvvcgip+JV+7zci5L8D1W9RZIz2l1o08AM3pfspitVWnPt3mzHcBPp12oYMTy0pqrFs/C+m3EwsQ==", - "dependencies": { - "mimic-fn": "^1.0.0" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/os-tmpdir": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/os-tmpdir/-/os-tmpdir-1.0.2.tgz", - "integrity": "sha512-D2FR03Vir7FIu45XBY20mTb+/ZSWB00sjU9jdQXt83gDrI4Ztz5Fs7/yy74g2N5SVQY4xY1qDr4rNddwYRVX0g==", - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/p-limit": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", @@ -6183,18 +5925,6 @@ "node": ">=10" } }, - "node_modules/restore-cursor": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/restore-cursor/-/restore-cursor-2.0.0.tgz", - "integrity": "sha512-6IzJLuGi4+R14vwagDHX+JrXmPVtPpn4mffDJ1UdR7/Edm87fl6yi8mMBIVvFtJaNTUvjughmW4hwLhRG7gC1Q==", - "dependencies": { - "onetime": "^2.0.0", - "signal-exit": "^3.0.2" - }, - "engines": { - "node": ">=4" - } - }, "node_modules/retry": { "version": "0.13.1", "resolved": "https://registry.npmjs.org/retry/-/retry-0.13.1.tgz", @@ -6230,30 +5960,6 @@ "rimraf": "bin.js" } }, - "node_modules/run-async": { - "version": "2.4.1", - "resolved": "https://registry.npmjs.org/run-async/-/run-async-2.4.1.tgz", - "integrity": "sha512-tvVnVv01b8c1RrA6Ep7JkStj85Guv/YrMcwqYQnwjsAS2cTmmPGBBjAjpCW7RrSodNSoE2/qg9O4bceNvUuDgQ==", - "engines": { - "node": ">=0.12.0" - } - }, - "node_modules/rxjs": { - "version": "6.6.7", - "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-6.6.7.tgz", - "integrity": "sha512-hTdwr+7yYNIT5n4AMYp85KA6yw2Va0FLa3Rguvbpa4W3I5xynaBZo41cM3XM+4Q6fRMj3sBYIR1VAmZMXYJvRQ==", - "dependencies": { - "tslib": "^1.9.0" - }, - "engines": { - "npm": ">=2.0.0" - } - }, - "node_modules/rxjs/node_modules/tslib": { - "version": "1.14.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz", - "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==" - }, "node_modules/safe-buffer": { "version": "5.1.2", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", @@ -6475,7 +6181,8 @@ "node_modules/signal-exit": { "version": "3.0.7", "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", - "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==" + "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==", + "dev": true }, "node_modules/sisteransi": { "version": "1.0.5", @@ -6775,23 +6482,6 @@ "node": ">=8" } }, - "node_modules/through": { - "version": "2.3.8", - "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", - "integrity": "sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg==" - }, - "node_modules/tmp": { - "version": "0.0.33", - "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.0.33.tgz", - "integrity": "sha512-jRCJlojKnZ3addtTOjdIqoRuPEKBvNXcGYqzO6zWZX8KfKEpnGY5jfggJQ3EjKuu8D4bJRr0y+cYJFmYbImXGw==", - "license": "MIT", - "dependencies": { - "os-tmpdir": "~1.0.2" - }, - "engines": { - "node": ">=0.6.0" - } - }, "node_modules/tmpl": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/tmpl/-/tmpl-1.0.5.tgz", @@ -6913,14 +6603,6 @@ "node": ">= 0.6" } }, - "node_modules/type-of-is": { - "version": "3.5.1", - "resolved": "https://registry.npmjs.org/type-of-is/-/type-of-is-3.5.1.tgz", - "integrity": "sha512-SOnx8xygcAh8lvDU2exnK2bomASfNjzB3Qz71s2tw9QnX8fkAo7aC+D0H7FV0HjRKj94CKV2Hi71kVkkO6nOxg==", - "engines": { - "node": ">=0.10.5" - } - }, "node_modules/typedarray-to-buffer": { "version": "3.1.5", "resolved": "https://registry.npmjs.org/typedarray-to-buffer/-/typedarray-to-buffer-3.1.5.tgz", @@ -8520,11 +8202,6 @@ "indent-string": "^4.0.0" } }, - "ansi-escapes": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-3.2.0.tgz", - "integrity": "sha512-cBhpre4ma+U0T1oM5fXg7Dy1Jw7zzwv7lt/GoCpr+hDQJoYnKVPLL4dCvSEFMmQurOQvSrwT7SL/DAlhBI97RQ==" - }, "ansi-regex": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", @@ -8914,11 +8591,6 @@ "integrity": "sha512-kWWXztvZ5SBQV+eRgKFeh8q5sLuZY2+8WUIzlxWVTg+oGwY14qylx1KbKzHd8P6ZYkAg0xyIDU9JMHhyJMZ1jw==", "dev": true }, - "chardet": { - "version": "0.7.0", - "resolved": "https://registry.npmjs.org/chardet/-/chardet-0.7.0.tgz", - "integrity": "sha512-mT8iDcrh03qDGRRmoA2hmBJnxpllMR+0/0qlzjqZES6NdiWDcZkCNAk4rPFZ9Q85r27unkiNNg8ZOiwZXBHwcA==" - }, "check-error": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/check-error/-/check-error-1.0.2.tgz", @@ -8949,19 +8621,6 @@ "integrity": "sha512-4diC9HaTE+KRAMWhDhrGOECgWZxoevMc5TlkObMqNSsVU62PYzXZ/SMTjzyGAFF1YusgxGcSWTEXBhp0CPwQ1A==", "dev": true }, - "cli-cursor": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/cli-cursor/-/cli-cursor-2.1.0.tgz", - "integrity": "sha512-8lgKz8LmCRYZZQDpRyT2m5rKJ08TnU4tR9FFFW2rxpxR1FzWi4PQ/NfyODchAatHaUgnSPVcx/R5w6NuTBzFiw==", - "requires": { - "restore-cursor": "^2.0.0" - } - }, - "cli-width": { - "version": "2.2.1", - "resolved": "https://registry.npmjs.org/cli-width/-/cli-width-2.2.1.tgz", - "integrity": "sha512-GRMWDxpOB6Dgk2E5Uo+3eEBvtOOlimMmpbFiKuLFnQzYDavtLFY3K5ona41jgN/WdRZtG7utuVSVTL4HbZHGkw==" - }, "cliui": { "version": "8.0.1", "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", @@ -9005,11 +8664,6 @@ "delayed-stream": "~1.0.0" } }, - "commander": { - "version": "2.20.3", - "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz", - "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==" - }, "commondir": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/commondir/-/commondir-1.0.1.tgz", @@ -9490,16 +9144,6 @@ "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==" }, - "external-editor": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/external-editor/-/external-editor-3.1.0.tgz", - "integrity": "sha512-hMQ4CX1p1izmuLYyZqLMO/qGNw10wSv9QDCPfzXfyFrOaCSSoRfqE1Kf1s5an66J5JZC62NewG+mK49jOCtQew==", - "requires": { - "chardet": "^0.7.0", - "iconv-lite": "^0.4.24", - "tmp": "^0.0.33" - } - }, "farmhash-modern": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/farmhash-modern/-/farmhash-modern-1.1.0.tgz", @@ -9543,21 +9187,6 @@ "bser": "2.1.1" } }, - "figures": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/figures/-/figures-2.0.0.tgz", - "integrity": "sha512-Oa2M9atig69ZkfwiApY8F2Yy+tzMbazyvqv21R0NsSC8floSOC09BbT1ITWAdoMGQvJ/aZnR1KMwdx9tvHnTNA==", - "requires": { - "escape-string-regexp": "^1.0.5" - }, - "dependencies": { - "escape-string-regexp": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", - "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==" - } - } - }, "fill-range": { "version": "7.1.1", "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", @@ -9769,15 +9398,6 @@ "json-bigint": "^1.0.0" } }, - "generate-schema": { - "version": "2.6.0", - "resolved": "https://registry.npmjs.org/generate-schema/-/generate-schema-2.6.0.tgz", - "integrity": "sha512-EUBKfJNzT8f91xUk5X5gKtnbdejZeE065UAJ3BCzE8VEbvwKI9Pm5jaWmqVeK1MYc1g5weAVFDTSJzN7ymtTqA==", - "requires": { - "commander": "^2.9.0", - "type-of-is": "^3.4.0" - } - }, "gensync": { "version": "1.0.0-beta.2", "resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz", @@ -10075,121 +9695,6 @@ "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" }, - "inquirer": { - "version": "6.5.2", - "resolved": "https://registry.npmjs.org/inquirer/-/inquirer-6.5.2.tgz", - "integrity": "sha512-cntlB5ghuB0iuO65Ovoi8ogLHiWGs/5yNrtUcKjFhSSiVeAIVpD7koaSU9RM8mpXw5YDi9RdYXGQMaOURB7ycQ==", - "requires": { - "ansi-escapes": "^3.2.0", - "chalk": "^2.4.2", - "cli-cursor": "^2.1.0", - "cli-width": "^2.0.0", - "external-editor": "^3.0.3", - "figures": "^2.0.0", - "lodash": "^4.17.12", - "mute-stream": "0.0.7", - "run-async": "^2.2.0", - "rxjs": "^6.4.0", - "string-width": "^2.1.0", - "strip-ansi": "^5.1.0", - "through": "^2.3.6" - }, - "dependencies": { - "ansi-regex": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-3.0.1.tgz", - "integrity": "sha512-+O9Jct8wf++lXxxFc4hc8LsjaSq0HFzzL7cVsw8pRDIPdjKD2mT4ytDZlLuSBZ4cLKZFXIrMGO7DbQCtMJJMKw==" - }, - "ansi-styles": { - "version": "3.2.1", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", - "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", - "requires": { - "color-convert": "^1.9.0" - } - }, - "chalk": { - "version": "2.4.2", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", - "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", - "requires": { - "ansi-styles": "^3.2.1", - "escape-string-regexp": "^1.0.5", - "supports-color": "^5.3.0" - } - }, - "color-convert": { - "version": "1.9.3", - "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", - "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", - "requires": { - "color-name": "1.1.3" - } - }, - "color-name": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", - "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==" - }, - "escape-string-regexp": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", - "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==" - }, - "has-flag": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", - "integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==" - }, - "is-fullwidth-code-point": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz", - "integrity": "sha512-VHskAKYM8RfSFXwee5t5cbN5PZeq1Wrh6qd5bkyiXIf6UQcN6w/A0eXM9r6t8d+GYOh+o6ZhiEnb88LN/Y8m2w==" - }, - "string-width": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-2.1.1.tgz", - "integrity": "sha512-nOqH59deCq9SRHlxq1Aw85Jnt4w6KvLKqWVik6oA9ZklXLNIOlqg4F2yrT1MVaTjAqvVwdfeZ7w7aCvJD7ugkw==", - "requires": { - "is-fullwidth-code-point": "^2.0.0", - "strip-ansi": "^4.0.0" - }, - "dependencies": { - "strip-ansi": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-4.0.0.tgz", - "integrity": "sha512-4XaJ2zQdCzROZDivEVIDPkcQn8LMFSa8kj8Gxb/Lnwzv9A8VctNZ+lfivC/sV3ivW8ElJTERXZoPBRrZKkNKow==", - "requires": { - "ansi-regex": "^3.0.0" - } - } - } - }, - "strip-ansi": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-5.2.0.tgz", - "integrity": "sha512-DuRs1gKbBqsMKIZlrffwlug8MHkcnpjs5VPmL1PAh+mA30U0DTotfDZ0d2UUsXpPmPmMMJ6W773MaA3J+lbiWA==", - "requires": { - "ansi-regex": "^4.1.0" - }, - "dependencies": { - "ansi-regex": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.1.tgz", - "integrity": "sha512-ILlv4k/3f6vfQ4OoP2AGvirOktlQ98ZEL1k9FaQjxa3L1abBgbuTDAdPOpvbGncC0BTVQrl+OM8xZGK6tWXt7g==" - } - } - }, - "supports-color": { - "version": "5.5.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", - "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", - "requires": { - "has-flag": "^3.0.0" - } - } - } - }, "ipaddr.js": { "version": "1.9.1", "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", @@ -11241,11 +10746,6 @@ "mime-db": "1.52.0" } }, - "mimic-fn": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-1.2.0.tgz", - "integrity": "sha512-jf84uxzwiuiIVKiOLpfYk7N46TSy8ubTonmneY9vrpHNAnp0QBt2BxWV9dO3/j+BoVAb+a5G6YDPW3M5HOdMWQ==" - }, "minimatch": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", @@ -11283,11 +10783,6 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" }, - "mute-stream": { - "version": "0.0.7", - "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-0.0.7.tgz", - "integrity": "sha512-r65nCZhrbXXb6dXOACihYApHw2Q6pV0M3V0PSxd74N0+D8nzAdEAITq2oAjA1jVnKI+tGvEBUpqiMh0+rW6zDQ==" - }, "natural-compare": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", @@ -11482,19 +10977,6 @@ "wrappy": "1" } }, - "onetime": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/onetime/-/onetime-2.0.1.tgz", - "integrity": "sha512-oyyPpiMaKARvvcgip+JV+7zci5L8D1W9RZIz2l1o08AM3pfspitVWnPt3mzHcBPp12oYMTy0pqrFs/C+m3EwsQ==", - "requires": { - "mimic-fn": "^1.0.0" - } - }, - "os-tmpdir": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/os-tmpdir/-/os-tmpdir-1.0.2.tgz", - "integrity": "sha512-D2FR03Vir7FIu45XBY20mTb+/ZSWB00sjU9jdQXt83gDrI4Ztz5Fs7/yy74g2N5SVQY4xY1qDr4rNddwYRVX0g==" - }, "p-limit": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", @@ -11805,15 +11287,6 @@ "integrity": "sha512-X2UW6Nw3n/aMgDVy+0rSqgHlv39WZAlZrXCdnbyEiKm17DSqHX4MmQMaST3FbeWR5FTuRcUwYAziZajji0Y7mg==", "dev": true }, - "restore-cursor": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/restore-cursor/-/restore-cursor-2.0.0.tgz", - "integrity": "sha512-6IzJLuGi4+R14vwagDHX+JrXmPVtPpn4mffDJ1UdR7/Edm87fl6yi8mMBIVvFtJaNTUvjughmW4hwLhRG7gC1Q==", - "requires": { - "onetime": "^2.0.0", - "signal-exit": "^3.0.2" - } - }, "retry": { "version": "0.13.1", "resolved": "https://registry.npmjs.org/retry/-/retry-0.13.1.tgz", @@ -11839,26 +11312,6 @@ "glob": "^7.1.3" } }, - "run-async": { - "version": "2.4.1", - "resolved": "https://registry.npmjs.org/run-async/-/run-async-2.4.1.tgz", - "integrity": "sha512-tvVnVv01b8c1RrA6Ep7JkStj85Guv/YrMcwqYQnwjsAS2cTmmPGBBjAjpCW7RrSodNSoE2/qg9O4bceNvUuDgQ==" - }, - "rxjs": { - "version": "6.6.7", - "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-6.6.7.tgz", - "integrity": "sha512-hTdwr+7yYNIT5n4AMYp85KA6yw2Va0FLa3Rguvbpa4W3I5xynaBZo41cM3XM+4Q6fRMj3sBYIR1VAmZMXYJvRQ==", - "requires": { - "tslib": "^1.9.0" - }, - "dependencies": { - "tslib": { - "version": "1.14.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz", - "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==" - } - } - }, "safe-buffer": { "version": "5.1.2", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", @@ -12028,7 +11481,8 @@ "signal-exit": { "version": "3.0.7", "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", - "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==" + "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==", + "dev": true }, "sisteransi": { "version": "1.0.5", @@ -12251,19 +11705,6 @@ "minimatch": "^3.0.4" } }, - "through": { - "version": "2.3.8", - "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", - "integrity": "sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg==" - }, - "tmp": { - "version": "0.0.33", - "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.0.33.tgz", - "integrity": "sha512-jRCJlojKnZ3addtTOjdIqoRuPEKBvNXcGYqzO6zWZX8KfKEpnGY5jfggJQ3EjKuu8D4bJRr0y+cYJFmYbImXGw==", - "requires": { - "os-tmpdir": "~1.0.2" - } - }, "tmpl": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/tmpl/-/tmpl-1.0.5.tgz", @@ -12336,11 +11777,6 @@ "mime-types": "~2.1.24" } }, - "type-of-is": { - "version": "3.5.1", - "resolved": "https://registry.npmjs.org/type-of-is/-/type-of-is-3.5.1.tgz", - "integrity": "sha512-SOnx8xygcAh8lvDU2exnK2bomASfNjzB3Qz71s2tw9QnX8fkAo7aC+D0H7FV0HjRKj94CKV2Hi71kVkkO6nOxg==" - }, "typedarray-to-buffer": { "version": "3.1.5", "resolved": "https://registry.npmjs.org/typedarray-to-buffer/-/typedarray-to-buffer-3.1.5.tgz", diff --git a/firestore-bigquery-export/firestore-bigquery-change-tracker/package.json b/firestore-bigquery-export/firestore-bigquery-change-tracker/package.json index b9a778ca7b..5f259429fd 100644 --- a/firestore-bigquery-export/firestore-bigquery-change-tracker/package.json +++ b/firestore-bigquery-export/firestore-bigquery-change-tracker/package.json @@ -2,10 +2,10 @@ "name": "@firebaseextensions/firestore-bigquery-change-tracker", "repository": { "type": "git", - "url": "github.com/firebase/extensions.git", + "url": "git+https://github.com/firebase/extensions.git", "directory": "firestore-bigquery-export/firestore-bigquery-change-tracker" }, - "version": "2.0.0", + "version": "2.0.2", "description": "Core change-tracker library for Cloud Firestore Collection BigQuery Exports", "main": "./lib/index.js", "scripts": { @@ -17,10 +17,8 @@ "generate-stresstest-table": "bq query --project_id=extensions-testing --use_legacy_sql=false < ./src/__tests__/fixtures/sql/generateSnapshotStresstestTable.sql" }, "files": [ - "lib/*.js", - "lib/bigquery/*.js", - "lib/*.d.ts", - "lib/bigquery/*.d.ts" + "lib/**/*.js", + "lib/**/*.d.ts" ], "author": "Jan Wyszynski ", "license": "Apache-2.0", @@ -29,9 +27,6 @@ "@google-cloud/resource-manager": "^5.1.0", "firebase-admin": "^13.2.0", "firebase-functions": "^6.3.2", - "generate-schema": "^2.6.0", - "inquirer": "^6.4.0", - "lodash": "^4.17.14", "node-fetch": "^2.6.1", "sql-formatter": "^2.3.3", "traverse": "^0.6.6" diff --git a/firestore-bigquery-export/firestore-bigquery-change-tracker/src/__tests__/bigquery/failedTransaction.test.ts b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/__tests__/bigquery/failedTransaction.test.ts index 92bf3586bd..c1b7c9e948 100644 --- a/firestore-bigquery-export/firestore-bigquery-change-tracker/src/__tests__/bigquery/failedTransaction.test.ts +++ b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/__tests__/bigquery/failedTransaction.test.ts @@ -1,5 +1,5 @@ import * as admin from "firebase-admin"; -import { Config } from "../../bigquery/types"; +import { ChangeTrackerConfig } from "../../bigquery/types"; import handleFailedTransactions from "../../bigquery/handleFailedTransactions"; @@ -15,7 +15,7 @@ describe("handleFailedTransactions", () => { const collectionName = "testing"; const doc = db.collection("testing").doc("600"); - const config: Config = { + const config: ChangeTrackerConfig = { backupTableId: collectionName, datasetId: "", tableId: "", diff --git a/firestore-bigquery-export/firestore-bigquery-change-tracker/src/__tests__/bigquery/materializedViews/initializeLatestView.test.ts b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/__tests__/bigquery/materializedViews/initializeLatestView.test.ts index ccf5e0dcbe..a73c48389c 100644 --- a/firestore-bigquery-export/firestore-bigquery-change-tracker/src/__tests__/bigquery/materializedViews/initializeLatestView.test.ts +++ b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/__tests__/bigquery/materializedViews/initializeLatestView.test.ts @@ -1,6 +1,6 @@ import { initializeLatestView } from "../../../bigquery/initializeLatestView"; import { initializeLatestMaterializedView } from "../../../bigquery/initializeLatestMaterializedView"; -import { Config } from "../../../bigquery/types"; +import { ChangeTrackerConfig } from "../../../bigquery/types"; jest.mock("../../../bigquery/initializeLatestMaterializedView"); @@ -12,7 +12,7 @@ describe("initializeLatestView", () => { create: jest.fn(), }; - const mockConfig: Config = { + const mockConfig: ChangeTrackerConfig = { datasetId: "test_dataset", tableId: "test_raw_table", datasetLocation: "US", diff --git a/firestore-bigquery-export/firestore-bigquery-change-tracker/src/__tests__/bigquery/partitioning.test.ts b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/__tests__/bigquery/partitioning.test.ts index 88c217f223..0fb8e975f8 100644 --- a/firestore-bigquery-export/firestore-bigquery-change-tracker/src/__tests__/bigquery/partitioning.test.ts +++ b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/__tests__/bigquery/partitioning.test.ts @@ -3,7 +3,7 @@ import * as admin from "firebase-admin"; import { BigQuery, Dataset, Table } from "@google-cloud/bigquery"; import { ChangeType, FirestoreDocumentChangeEvent } from "../.."; -import { Config } from "../../bigquery/types"; +import { ChangeTrackerConfig } from "../../bigquery/types"; import { Partitioning } from "../../bigquery/partitioning"; import { PartitioningConfig } from "../../bigquery/partitioning/config"; import { deleteTable } from "../fixtures/clearTables"; @@ -15,8 +15,12 @@ let dataset: Dataset; let table: Table; let randomID: string; let datasetId: string; +const describeIfBigQueryIntegration = + process.env.RUN_BIGQUERY_INTEGRATION_TESTS === "true" + ? describe + : describe.skip; -describe("processing partitions on a new table", () => { +describeIfBigQueryIntegration("processing partitions on a new table", () => { beforeAll(async () => { jest.spyOn(logger, "debug").mockImplementation(() => {}); jest.spyOn(logger, "info").mockImplementation(() => {}); @@ -50,7 +54,7 @@ describe("processing partitions on a new table", () => { }); describe("addPartitioningToSchema", () => { test("adds a custom TIMESTAMP to a schema", async () => { - const config: Config = { + const config: ChangeTrackerConfig = { datasetId: "dataset", tableId: "table", datasetLocation: "US", @@ -84,7 +88,7 @@ describe("processing partitions on a new table", () => { }); test("adds a custom DATETIME to a schema", async () => { - const config: Config = { + const config: ChangeTrackerConfig = { datasetId: "dataset", tableId: "table", datasetLocation: "US", @@ -118,7 +122,7 @@ describe("processing partitions on a new table", () => { }); test("does not add an invalid time partition type to a schema", async () => { - const config: Config = { + const config: ChangeTrackerConfig = { datasetId: "dataset", tableId: "table", datasetLocation: "US", @@ -147,7 +151,7 @@ describe("processing partitions on a new table", () => { }); test("does not add partitioning without a valid timePartitioning value ", async () => { - const config: Config = { + const config: ChangeTrackerConfig = { datasetId: "", tableId: "", datasetLocation: "", @@ -173,7 +177,7 @@ describe("processing partitions on a new table", () => { }); test("does not add partitioning without a timePartitioningFirestoreField", async () => { - const config: Config = { + const config: ChangeTrackerConfig = { datasetId: "", tableId: "", datasetLocation: "", @@ -201,7 +205,7 @@ describe("processing partitions on a new table", () => { describe("getPartitionValue", () => { test("returns a value when timePartitioningField and timePartitioningFirestoreField string value has been defined", async () => { - const config: Config = { + const config: ChangeTrackerConfig = { datasetId: "", tableId: "", datasetLocation: "", @@ -235,7 +239,7 @@ describe("processing partitions on a new table", () => { }); test("returns a value when timePartitioningField and timePartitioningFirestoreField string value has been defined, with a timestamp-like value", async () => { - const config: Config = { + const config: ChangeTrackerConfig = { datasetId: "", tableId: "", datasetLocation: "", @@ -273,7 +277,7 @@ describe("processing partitions on a new table", () => { }); test("returns an empty object when _seconds or _nanoseconds is not a number", async () => { - const config: Config = { + const config: ChangeTrackerConfig = { datasetId: "", tableId: "", datasetLocation: "", @@ -310,7 +314,7 @@ describe("processing partitions on a new table", () => { }); test("returns a value when timePartitioningField and timePartitioningFirestoreField string value has been defined, and is timestamp-like", async () => { - const config: Config = { + const config: ChangeTrackerConfig = { datasetId: "", tableId: "", datasetLocation: "", @@ -347,7 +351,7 @@ describe("processing partitions on a new table", () => { }); test("returns an empty object if timePartitioningFirestoreField has not been provided", async () => { - const config: Config = { + const config: ChangeTrackerConfig = { datasetId: "", tableId: "", datasetLocation: "", @@ -380,7 +384,7 @@ describe("processing partitions on a new table", () => { expect(value).toEqual({}); }); test("returns an empty object if timePartitioningFirestoreField has not been provided", async () => { - const config: Config = { + const config: ChangeTrackerConfig = { datasetId: "", tableId: "", datasetLocation: "", @@ -414,7 +418,7 @@ describe("processing partitions on a new table", () => { }); test("returns an empty object if timePartitioningFirestoreField timePartitioningField", async () => { - const config: Config = { + const config: ChangeTrackerConfig = { datasetId: "", tableId: "", datasetLocation: "", @@ -448,7 +452,7 @@ describe("processing partitions on a new table", () => { }); test("returns an empty object if no event data has been provided", async () => { - const config: Config = { + const config: ChangeTrackerConfig = { datasetId: "", tableId: "", datasetLocation: "", @@ -480,7 +484,7 @@ describe("processing partitions on a new table", () => { }); test("returns an empty object if a non string or Timestamp value is synced from Firestore", async () => { - const config: Config = { + const config: ChangeTrackerConfig = { datasetId: "", tableId: "", datasetLocation: "", @@ -516,7 +520,7 @@ describe("processing partitions on a new table", () => { describe("isTablePartitioned", () => { test("partition return false if table is not provided", async () => { - const config: Config = { + const config: ChangeTrackerConfig = { datasetId: "", tableId: "", datasetLocation: "", @@ -541,7 +545,7 @@ describe("processing partitions on a new table", () => { }); test("partition return false if table is not provided", async () => { - const config: Config = { + const config: ChangeTrackerConfig = { datasetId: "", tableId: "", datasetLocation: "", @@ -576,7 +580,7 @@ describe("processing partitions on a new table", () => { }); }); -describe("updateTableMetadata", () => { +describeIfBigQueryIntegration("updateTableMetadata", () => { let testTable: Table; let testDataset: Dataset; @@ -596,7 +600,7 @@ describe("updateTableMetadata", () => { }); test("updates the table metadata with the timestamp field", async () => { - const config: Config = { + const config: ChangeTrackerConfig = { datasetId: "", tableId: "", datasetLocation: "", @@ -623,7 +627,7 @@ describe("updateTableMetadata", () => { }); }); test("Should not update if there is a custom option with the timestamp option", async () => { - const config: Config = { + const config: ChangeTrackerConfig = { datasetId: "", tableId: "", datasetLocation: "", @@ -728,103 +732,106 @@ describe("updateTableMetadata", () => { }); }); -describe("getPartitionValue with DELETE operations", () => { - let testTable: Table; - let testDataset: Dataset; - - beforeAll(async () => { - const randomID = (Math.random() + 1).toString(36).substring(7); - const testDatasetId = `bq_delete_${randomID}`; - [testDataset] = await bq.createDataset(testDatasetId, { - location: "europe-west2", - }); - [testTable] = await testDataset.createTable(`bq_delete_${randomID}`, {}); - }); - - afterAll(async () => { - await deleteTable({ - datasetId: testDataset.id, +describeIfBigQueryIntegration( + "getPartitionValue with DELETE operations", + () => { + let testTable: Table; + let testDataset: Dataset; + + beforeAll(async () => { + const randomID = (Math.random() + 1).toString(36).substring(7); + const testDatasetId = `bq_delete_${randomID}`; + [testDataset] = await bq.createDataset(testDatasetId, { + location: "europe-west2", + }); + [testTable] = await testDataset.createTable(`bq_delete_${randomID}`, {}); }); - }); - test("uses oldData for DELETE operations", () => { - const partitioningConfig = new PartitioningConfig({ - granularity: "DAY", - bigqueryColumnName: "end_date", - bigqueryColumnType: "TIMESTAMP", - firestoreFieldName: "endDate", + afterAll(async () => { + await deleteTable({ + datasetId: testDataset.id, + }); }); - const oldDate = admin.firestore.Timestamp.fromDate( - new Date("2024-01-15T10:00:00Z") - ); + test("uses oldData for DELETE operations", () => { + const partitioningConfig = new PartitioningConfig({ + granularity: "DAY", + bigqueryColumnName: "end_date", + bigqueryColumnType: "TIMESTAMP", + firestoreFieldName: "endDate", + }); - const event: FirestoreDocumentChangeEvent = { - timestamp: "", - operation: ChangeType.DELETE, - documentName: "test/doc", - eventId: "event1", - documentId: "doc", - data: null, - oldData: { endDate: oldDate }, - }; + const oldDate = admin.firestore.Timestamp.fromDate( + new Date("2024-01-15T10:00:00Z") + ); - const partitioning = new Partitioning(partitioningConfig, testTable); - const value = partitioning.getPartitionValue(event); + const event: FirestoreDocumentChangeEvent = { + timestamp: "", + operation: ChangeType.DELETE, + documentName: "test/doc", + eventId: "event1", + documentId: "doc", + data: null, + oldData: { endDate: oldDate }, + }; - expect(value.end_date).toBeDefined(); - }); + const partitioning = new Partitioning(partitioningConfig, testTable); + const value = partitioning.getPartitionValue(event); - test("returns empty object for DELETE when oldData is null", () => { - const partitioningConfig = new PartitioningConfig({ - granularity: "DAY", - bigqueryColumnName: "end_date", - bigqueryColumnType: "TIMESTAMP", - firestoreFieldName: "endDate", + expect(value.end_date).toBeDefined(); }); - const event: FirestoreDocumentChangeEvent = { - timestamp: "", - operation: ChangeType.DELETE, - documentName: "test/doc", - eventId: "event1", - documentId: "doc", - data: null, - oldData: null, - }; + test("returns empty object for DELETE when oldData is null", () => { + const partitioningConfig = new PartitioningConfig({ + granularity: "DAY", + bigqueryColumnName: "end_date", + bigqueryColumnType: "TIMESTAMP", + firestoreFieldName: "endDate", + }); - const partitioning = new Partitioning(partitioningConfig, testTable); - const value = partitioning.getPartitionValue(event); + const event: FirestoreDocumentChangeEvent = { + timestamp: "", + operation: ChangeType.DELETE, + documentName: "test/doc", + eventId: "event1", + documentId: "doc", + data: null, + oldData: null, + }; - expect(value).toEqual({}); - }); + const partitioning = new Partitioning(partitioningConfig, testTable); + const value = partitioning.getPartitionValue(event); - test("returns empty object for DELETE when oldData lacks the field", () => { - const partitioningConfig = new PartitioningConfig({ - granularity: "DAY", - bigqueryColumnName: "end_date", - bigqueryColumnType: "TIMESTAMP", - firestoreFieldName: "endDate", + expect(value).toEqual({}); }); - const event: FirestoreDocumentChangeEvent = { - timestamp: "", - operation: ChangeType.DELETE, - documentName: "test/doc", - eventId: "event1", - documentId: "doc", - data: null, - oldData: { otherField: "value" }, - }; + test("returns empty object for DELETE when oldData lacks the field", () => { + const partitioningConfig = new PartitioningConfig({ + granularity: "DAY", + bigqueryColumnName: "end_date", + bigqueryColumnType: "TIMESTAMP", + firestoreFieldName: "endDate", + }); - const partitioning = new Partitioning(partitioningConfig, testTable); - const value = partitioning.getPartitionValue(event); + const event: FirestoreDocumentChangeEvent = { + timestamp: "", + operation: ChangeType.DELETE, + documentName: "test/doc", + eventId: "event1", + documentId: "doc", + data: null, + oldData: { otherField: "value" }, + }; - expect(value).toEqual({}); - }); -}); + const partitioning = new Partitioning(partitioningConfig, testTable); + const value = partitioning.getPartitionValue(event); + + expect(value).toEqual({}); + }); + } +); -describe("isValidPartitionForExistingTable", () => { +describeIfBigQueryIntegration("isValidPartitionForExistingTable", () => { let testTable: Table; let testDataset: Dataset; let partitionedTable: Table; @@ -886,7 +893,7 @@ describe("isValidPartitionForExistingTable", () => { }); }); -describe("addPartitioningToSchema with DATE type", () => { +describeIfBigQueryIntegration("addPartitioningToSchema with DATE type", () => { let testTable: Table; let testDataset: Dataset; diff --git a/firestore-bigquery-export/firestore-bigquery-change-tracker/src/__tests__/bigquery/partitioning/config.test.ts b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/__tests__/bigquery/partitioning/config.test.ts index db4b5446a6..ee4428ff9a 100644 --- a/firestore-bigquery-export/firestore-bigquery-change-tracker/src/__tests__/bigquery/partitioning/config.test.ts +++ b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/__tests__/bigquery/partitioning/config.test.ts @@ -87,65 +87,30 @@ describe("PartitioningConfig", () => { }); }); - describe("static factory methods", () => { - test("none() creates a NONE config", () => { - const config = PartitioningConfig.none(); - expect(config.getType()).toBe(PartitioningType.NONE); - expect(config.getGranularity()).toBe("NONE"); - }); - - test("ingestionTime() creates an INGESTION_TIME config", () => { - const config = PartitioningConfig.ingestionTime("MONTH"); - expect(config.getType()).toBe(PartitioningType.INGESTION_TIME); - expect(config.getGranularity()).toBe("MONTH"); - }); - - test("firestoreTimestamp() creates a FIRESTORE_TIMESTAMP config", () => { - const config = PartitioningConfig.firestoreTimestamp("DAY"); - expect(config.getType()).toBe(PartitioningType.FIRESTORE_TIMESTAMP); - expect(config.getBigQueryColumnName()).toBe("timestamp"); - expect(config.getBigQueryColumnType()).toBeUndefined(); - }); - - test("firestoreTimestamp() with columnType creates correct config", () => { - const config = PartitioningConfig.firestoreTimestamp("DAY", "DATE"); - expect(config.getType()).toBe(PartitioningType.FIRESTORE_TIMESTAMP); - expect(config.getBigQueryColumnType()).toBe("DATE"); - }); - - test("firestoreField() creates a FIRESTORE_FIELD config", () => { - const config = PartitioningConfig.firestoreField( - "YEAR", - "event_date", - "DATE", - "eventDate" - ); - expect(config.getType()).toBe(PartitioningType.FIRESTORE_FIELD); - expect(config.getGranularity()).toBe("YEAR"); - expect(config.getBigQueryColumnName()).toBe("event_date"); - expect(config.getBigQueryColumnType()).toBe("DATE"); - expect(config.getFirestoreFieldName()).toBe("eventDate"); - }); - }); - describe("getter methods", () => { - test("getStrategy() returns the original config", () => { - const originalConfig = { + test("getStrategy() returns the original strategy", () => { + const strategy = { granularity: "DAY" as const, bigqueryColumnName: "test", bigqueryColumnType: "TIMESTAMP" as const, firestoreFieldName: "test", }; - const config = new PartitioningConfig(originalConfig); - expect(config.getStrategy()).toEqual(originalConfig); + const config = new PartitioningConfig(strategy); + expect(config.getStrategy()).toEqual(strategy); }); test("all boolean methods return false for non-matching types", () => { - const config = PartitioningConfig.ingestionTime("DAY"); + const config = new PartitioningConfig({ granularity: "DAY" }); expect(config.isNoPartitioning()).toBe(false); expect(config.isFirestoreTimestampPartitioning()).toBe(false); expect(config.isFirestoreFieldPartitioning()).toBe(false); expect(config.isIngestionTimePartitioning()).toBe(true); }); + + test("constructor defaults to no partitioning when undefined", () => { + const config = new PartitioningConfig(); + expect(config.getType()).toBe(PartitioningType.NONE); + expect(config.getStrategy()).toEqual({ granularity: "NONE" }); + }); }); }); diff --git a/firestore-bigquery-export/firestore-bigquery-change-tracker/src/__tests__/fixtures/changeTracker.ts b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/__tests__/fixtures/changeTracker.ts index 92ab3b82e9..dab94e1789 100644 --- a/firestore-bigquery-export/firestore-bigquery-change-tracker/src/__tests__/fixtures/changeTracker.ts +++ b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/__tests__/fixtures/changeTracker.ts @@ -5,7 +5,7 @@ import { FirestoreDocumentChangeEvent, } from "../.."; import { LogLevel } from "../../logger"; -import { Config } from "../../bigquery/types"; +import { ChangeTrackerConfig } from "../../bigquery/types"; export const changeTracker = ({ datasetId = "", @@ -24,7 +24,7 @@ export const changeTracker = ({ maxStaleness = undefined, refreshIntervalMinutes = undefined, logLevel = LogLevel.DEBUG, -}: Partial): FirestoreBigQueryEventHistoryTracker => { +}: Partial): FirestoreBigQueryEventHistoryTracker => { return new FirestoreBigQueryEventHistoryTracker({ datasetId, tableId, diff --git a/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/checkUpdates.ts b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/checkUpdates.ts index bc6520611e..6313f32cc5 100644 --- a/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/checkUpdates.ts +++ b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/checkUpdates.ts @@ -2,11 +2,11 @@ import { Table, TableMetadata } from "@google-cloud/bigquery/build/src/table"; import { Partitioning } from "./partitioning"; import { PartitioningConfig } from "./partitioning/config"; -import { Config } from "."; +import { ChangeTrackerConfig } from "."; interface TableRequiresUpdateOptions { table: Table; - config: Config; + config: ChangeTrackerConfig; documentIdColExists: boolean; pathParamsColExists: boolean; oldDataColExists: boolean; @@ -49,7 +49,7 @@ export async function tableRequiresUpdate({ interface ViewRequiresUpdateOptions { metadata?: TableMetadata; - config: Config; + config: ChangeTrackerConfig; documentIdColExists: boolean; pathParamsColExists: boolean; oldDataColExists: boolean; diff --git a/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/clustering.ts b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/clustering.ts index abe63e0680..9e518af78d 100644 --- a/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/clustering.ts +++ b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/clustering.ts @@ -1,4 +1,4 @@ -import { Config } from "."; +import { ChangeTrackerConfig } from "."; import * as logs from "../logs"; import * as bigquery from "@google-cloud/bigquery"; @@ -25,11 +25,15 @@ interface InvalidFieldType { } export class Clustering { - public config: Config; + public config: ChangeTrackerConfig; public table: bigquery.Table; public schema: object; - constructor(config: Config, table?: bigquery.Table, schema?: object) { + constructor( + config: ChangeTrackerConfig, + table?: bigquery.Table, + schema?: object + ) { this.config = config; this.table = table; this.schema = schema; diff --git a/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/handleFailedTransactions.ts b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/handleFailedTransactions.ts index 68436a2d3a..4addaeec9d 100644 --- a/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/handleFailedTransactions.ts +++ b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/handleFailedTransactions.ts @@ -1,13 +1,17 @@ import * as admin from "firebase-admin"; import { initializeApp } from "firebase-admin/app"; import { getFirestore } from "firebase-admin/firestore"; -import { Config } from "."; +import { ChangeTrackerConfig } from "."; if (!admin.apps.length) { initializeApp(); } -export default async (rows: any[], config: Config, e: Error): Promise => { +export default async ( + rows: any[], + config: ChangeTrackerConfig, + e: Error +): Promise => { const db = getFirestore(config.firestoreInstanceId!); db.settings({ ignoreUndefinedProperties: true, diff --git a/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/index.ts b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/index.ts index 7cd34c04a1..7b6f8a15e0 100644 --- a/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/index.ts +++ b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/index.ts @@ -44,9 +44,9 @@ import { initializeLatestView } from "./initializeLatestView"; import { logger, LogLevel } from "../logger"; export { RawChangelogSchema, RawChangelogViewSchema } from "./schema"; -import type { Config } from "./types"; +import type { ChangeTrackerConfig } from "./types"; import { PartitioningConfig } from "./partitioning/config"; -export type { Config } from "./types"; +export type { ChangeTrackerConfig } from "./types"; /** * An FirestoreEventHistoryTracker that exports data to BigQuery. @@ -65,7 +65,7 @@ export class FirestoreBigQueryEventHistoryTracker _initialized: boolean = false; partitioningConfig: PartitioningConfig; - constructor(public config: Config) { + constructor(public config: ChangeTrackerConfig) { this.bq = new bigquery.BigQuery(); this.bq.projectId = config.bqProjectId || process.env.PROJECT_ID; diff --git a/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/initializeLatestMaterializedView.ts b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/initializeLatestMaterializedView.ts index ae2a943dde..67cc1db642 100644 --- a/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/initializeLatestMaterializedView.ts +++ b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/initializeLatestMaterializedView.ts @@ -1,5 +1,5 @@ import { BigQuery, Table, TableMetadata } from "@google-cloud/bigquery"; -import { Config } from "."; +import { ChangeTrackerConfig } from "."; import * as logs from "../logs"; import { buildMaterializedViewQuery, @@ -10,7 +10,7 @@ import * as sqlFormatter from "sql-formatter"; interface InitializeLatestMaterializedViewOptions { bq: BigQuery; - changeTrackerConfig: Config; + changeTrackerConfig: ChangeTrackerConfig; view: Table; viewExists: boolean; rawChangeLogTableName: string; @@ -20,7 +20,7 @@ interface InitializeLatestMaterializedViewOptions { export async function shouldRecreateMaterializedView( view: Table, - config: Config, + config: ChangeTrackerConfig, source: string ): Promise { const [viewMetadata] = await view.getMetadata(); diff --git a/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/initializeLatestView.ts b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/initializeLatestView.ts index 88299ed6fe..9f9b59eb1e 100644 --- a/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/initializeLatestView.ts +++ b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/initializeLatestView.ts @@ -9,7 +9,7 @@ import { documentPathParams, RawChangelogViewSchema, } from "./schema"; -import { Config } from "."; +import { ChangeTrackerConfig } from "."; import * as logs from "../logs"; import { latestConsistentSnapshotView } from "./snapshot"; import { viewRequiresUpdate } from "./checkUpdates"; @@ -17,7 +17,7 @@ import { initializeLatestMaterializedView } from "./initializeLatestMaterialized interface InitializeLatestViewOptions { bq: BigQuery; - changeTrackerConfig: Config; + changeTrackerConfig: ChangeTrackerConfig; dataset: Dataset; view: Table; viewExists: boolean; diff --git a/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/partitioning/config.ts b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/partitioning/config.ts index 84567b31bd..48165b2fcc 100644 --- a/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/partitioning/config.ts +++ b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/partitioning/config.ts @@ -1,42 +1,49 @@ +/** Granularity for BigQuery time-based partitioning. */ export type TimePartitioningGranularity = "HOUR" | "DAY" | "MONTH" | "YEAR"; +/** BigQuery column type for a custom partitioning field. */ export type PartitioningFieldType = "TIMESTAMP" | "DATE" | "DATETIME"; -export interface BasePartitioningConfig { - granularity?: TimePartitioningGranularity | "NONE" | null; - bigqueryColumnName?: string; - bigqueryColumnType?: PartitioningFieldType; - firestoreFieldName?: string; -} - -export interface NoPartitioning extends BasePartitioningConfig { +/** Disables partitioning. */ +export interface NoPartitioning { granularity?: "NONE" | null | undefined; bigqueryColumnName?: undefined; bigqueryColumnType?: undefined; firestoreFieldName?: undefined; } -export interface IngestionTimePartitioning extends BasePartitioningConfig { +/** Partitions by BigQuery ingestion time. */ +export interface IngestionTimePartitioning { granularity: TimePartitioningGranularity; bigqueryColumnName?: undefined; bigqueryColumnType?: undefined; firestoreFieldName?: undefined; } -export interface FirestoreTimestampPartitioning extends BasePartitioningConfig { +/** Partitions by the built-in changelog `timestamp` column. */ +export interface FirestoreTimestampPartitioning { granularity: TimePartitioningGranularity; bigqueryColumnName: "timestamp"; bigqueryColumnType?: PartitioningFieldType; firestoreFieldName?: undefined; } -export interface FirestoreFieldPartitioning extends BasePartitioningConfig { +/** Partitions by an arbitrary Firestore document field. */ +export interface FirestoreFieldPartitioning { granularity: TimePartitioningGranularity; bigqueryColumnName: string; bigqueryColumnType: PartitioningFieldType; firestoreFieldName: string; } +/** + * Discriminated union describing how a BigQuery table should be partitioned. + * + * - {@link NoPartitioning} — no partitioning. + * - {@link IngestionTimePartitioning} — partition by ingestion time. + * - {@link FirestoreTimestampPartitioning} — partition by the changelog `timestamp` column. + * - {@link FirestoreFieldPartitioning} — partition by a custom Firestore field. + */ export type PartitioningStrategy = | NoPartitioning | IngestionTimePartitioning @@ -50,39 +57,45 @@ export enum PartitioningType { FIRESTORE_FIELD = "FIRESTORE_FIELD", } +/** + * Wraps a {@link PartitioningStrategy} and determines the effective + * {@link PartitioningType}. + */ export class PartitioningConfig { private strategy: PartitioningStrategy; private type: PartitioningType; - constructor(config: PartitioningStrategy) { - this.strategy = config; - this.type = this.determineType(config); + constructor(strategy?: PartitioningStrategy) { + this.strategy = strategy ?? { granularity: "NONE" }; + this.type = this.determineType(this.strategy); } - private determineType(config: PartitioningStrategy): PartitioningType { - if (!config.granularity || config.granularity === "NONE") { + private determineType(strategy: PartitioningStrategy): PartitioningType { + if (!strategy.granularity || strategy.granularity === "NONE") { return PartitioningType.NONE; } - if (!config.bigqueryColumnName && !config.firestoreFieldName) { + if (!strategy.bigqueryColumnName && !strategy.firestoreFieldName) { return PartitioningType.INGESTION_TIME; } if ( - config.bigqueryColumnName === "timestamp" && - !config.firestoreFieldName + strategy.bigqueryColumnName === "timestamp" && + !strategy.firestoreFieldName ) { return PartitioningType.FIRESTORE_TIMESTAMP; } if ( - config.bigqueryColumnName && - config.bigqueryColumnType && - config.firestoreFieldName + strategy.bigqueryColumnName && + strategy.bigqueryColumnType && + strategy.firestoreFieldName ) { return PartitioningType.FIRESTORE_FIELD; } + // TODO: throw an error here instead of silently falling back to NONE + // when we next do a major version bump. return PartitioningType.NONE; } @@ -125,39 +138,4 @@ export class PartitioningConfig { getFirestoreFieldName(): string | undefined { return this.strategy.firestoreFieldName; } - - static none(): PartitioningConfig { - return new PartitioningConfig({ granularity: "NONE" }); - } - - static ingestionTime( - granularity: TimePartitioningGranularity - ): PartitioningConfig { - return new PartitioningConfig({ granularity }); - } - - static firestoreTimestamp( - granularity: TimePartitioningGranularity, - columnType?: PartitioningFieldType - ): PartitioningConfig { - return new PartitioningConfig({ - granularity, - bigqueryColumnName: "timestamp", - bigqueryColumnType: columnType, - }); - } - - static firestoreField( - granularity: TimePartitioningGranularity, - bigqueryColumnName: string, - bigqueryColumnType: PartitioningFieldType, - firestoreFieldName: string - ): PartitioningConfig { - return new PartitioningConfig({ - granularity, - bigqueryColumnName, - bigqueryColumnType, - firestoreFieldName, - }); - } } diff --git a/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/types.ts b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/types.ts index 9a21aef6c3..0f7087e3da 100644 --- a/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/types.ts +++ b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/types.ts @@ -1,10 +1,9 @@ import { PartitioningStrategy } from "./partitioning/config"; /** - * Base configuration for all variants. Includes all parameters - * that are not dependent on the partitioning strategy. + * Configuration for {@link FirestoreBigQueryEventHistoryTracker}. */ -export interface Config { +export interface ChangeTrackerConfig { datasetId: string; tableId: string; firestoreInstanceId?: string; diff --git a/firestore-bigquery-export/firestore-bigquery-change-tracker/src/index.ts b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/index.ts index 858c256ab2..3573653d24 100644 --- a/firestore-bigquery-export/firestore-bigquery-change-tracker/src/index.ts +++ b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/index.ts @@ -25,3 +25,13 @@ export { FirestoreEventHistoryTracker, } from "./tracker"; export { LogLevel, Logger } from "./logger"; +export type { + ChangeTrackerConfig, + /** @deprecated Use {@link ChangeTrackerConfig} instead. */ + ChangeTrackerConfig as Config, +} from "./bigquery/types"; +export type { + PartitioningStrategy, + PartitioningFieldType, + TimePartitioningGranularity, +} from "./bigquery/partitioning/config"; diff --git a/firestore-bigquery-export/functions/__tests__/__snapshots__/config.test.ts.snap b/firestore-bigquery-export/functions/__tests__/__snapshots__/config.test.ts.snap index 58ae2dab95..e0cd3bbf2c 100644 --- a/firestore-bigquery-export/functions/__tests__/__snapshots__/config.test.ts.snap +++ b/firestore-bigquery-export/functions/__tests__/__snapshots__/config.test.ts.snap @@ -26,6 +26,9 @@ Object { "maxDispatchesPerSecond": 10, "maxEnqueueAttempts": 3, "maxStaleness": undefined, + "partitioning": Object { + "granularity": "NONE", + }, "projectId": undefined, "refreshIntervalMinutes": undefined, "tableId": "my_table", diff --git a/firestore-bigquery-export/functions/__tests__/config.test.ts b/firestore-bigquery-export/functions/__tests__/config.test.ts index a7189867ca..b398c03c37 100644 --- a/firestore-bigquery-export/functions/__tests__/config.test.ts +++ b/firestore-bigquery-export/functions/__tests__/config.test.ts @@ -5,7 +5,7 @@ import { resolve as pathResolve } from "path"; import * as yaml from "js-yaml"; import mockedEnv from "mocked-env"; -import { clustering } from "../src/config"; +import { buildPartitioningConfig, clustering } from "../src/config"; let restoreEnv; let extensionYaml; @@ -150,4 +150,395 @@ describe("extension config", () => { }); }); }); + + describe("buildPartitioningConfig", () => { + describe("no partitioning (TABLE_PARTITIONING is null)", () => { + test.each([ + { + description: "all undefined", + field: undefined, + fieldType: undefined, + firestoreField: undefined, + }, + { + description: "all empty strings", + field: "", + fieldType: "", + firestoreField: "", + }, + { + description: "all whitespace-only", + field: " ", + fieldType: " ", + firestoreField: " ", + }, + { + description: "field and firestoreField are NONE, fieldType is omit", + field: "NONE", + fieldType: "omit", + firestoreField: "NONE", + }, + { + description: "all omit", + field: "omit", + fieldType: "omit", + firestoreField: "omit", + }, + ])( + "returns NONE when $description", + ({ field, fieldType, firestoreField }) => { + expect( + buildPartitioningConfig({ + timePartitioning: null, + timePartitioningField: field, + timePartitioningFieldType: fieldType, + timePartitioningFirestoreField: firestoreField, + }) + ).toEqual({ granularity: "NONE" }); + } + ); + + test("returns NONE with mixed sentinels across fields", () => { + expect( + buildPartitioningConfig({ + timePartitioning: null, + timePartitioningField: "", + timePartitioningFieldType: "omit", + timePartitioningFirestoreField: "NONE", + }) + ).toEqual({ granularity: "NONE" }); + }); + + test("throws when real field values are provided without partitioning", () => { + expect(() => + buildPartitioningConfig({ + timePartitioning: null, + timePartitioningField: "created_at", + timePartitioningFieldType: "TIMESTAMP", + timePartitioningFirestoreField: "createdAt", + }) + ).toThrow(/TABLE_PARTITIONING is NONE/); + }); + + test("throws when only fieldName is provided without partitioning", () => { + expect(() => + buildPartitioningConfig({ + timePartitioning: null, + timePartitioningField: "my_field", + timePartitioningFieldType: undefined, + timePartitioningFirestoreField: undefined, + }) + ).toThrow(/TABLE_PARTITIONING is NONE/); + }); + + test("throws when only fieldType is a real value without partitioning", () => { + expect(() => + buildPartitioningConfig({ + timePartitioning: null, + timePartitioningField: undefined, + timePartitioningFieldType: "TIMESTAMP", + timePartitioningFirestoreField: undefined, + }) + ).toThrow(/TABLE_PARTITIONING is NONE/); + }); + }); + + describe("ingestion-time partitioning", () => { + test.each(["HOUR", "DAY", "MONTH", "YEAR"] as const)( + "uses ingestion-time with %s granularity", + (granularity) => { + expect( + buildPartitioningConfig({ + timePartitioning: granularity, + timePartitioningField: undefined, + timePartitioningFieldType: undefined, + timePartitioningFirestoreField: undefined, + }) + ).toEqual({ granularity }); + } + ); + + test("normalizes NONE/omit sentinels in optional fields to ingestion-time", () => { + expect( + buildPartitioningConfig({ + timePartitioning: "DAY", + timePartitioningField: "NONE", + timePartitioningFieldType: "omit", + timePartitioningFirestoreField: "NONE", + }) + ).toEqual({ granularity: "DAY" }); + }); + + test("normalizes empty strings in optional fields to ingestion-time", () => { + expect( + buildPartitioningConfig({ + timePartitioning: "DAY", + timePartitioningField: "", + timePartitioningFieldType: "", + timePartitioningFirestoreField: "", + }) + ).toEqual({ granularity: "DAY" }); + }); + }); + + describe("timestamp field partitioning", () => { + test("supports timestamp field with TIMESTAMP type", () => { + expect( + buildPartitioningConfig({ + timePartitioning: "DAY", + timePartitioningField: "timestamp", + timePartitioningFieldType: "TIMESTAMP", + timePartitioningFirestoreField: undefined, + }) + ).toEqual({ + granularity: "DAY", + bigqueryColumnName: "timestamp", + bigqueryColumnType: "TIMESTAMP", + }); + }); + + test("supports timestamp field with DATE type", () => { + expect( + buildPartitioningConfig({ + timePartitioning: "MONTH", + timePartitioningField: "timestamp", + timePartitioningFieldType: "DATE", + timePartitioningFirestoreField: undefined, + }) + ).toEqual({ + granularity: "MONTH", + bigqueryColumnName: "timestamp", + bigqueryColumnType: "DATE", + }); + }); + + test("supports timestamp field with DATETIME type", () => { + expect( + buildPartitioningConfig({ + timePartitioning: "MONTH", + timePartitioningField: "timestamp", + timePartitioningFieldType: "DATETIME", + timePartitioningFirestoreField: undefined, + }) + ).toEqual({ + granularity: "MONTH", + bigqueryColumnName: "timestamp", + bigqueryColumnType: "DATETIME", + }); + }); + + test("supports timestamp field with no field type", () => { + expect( + buildPartitioningConfig({ + timePartitioning: "DAY", + timePartitioningField: "timestamp", + timePartitioningFieldType: undefined, + timePartitioningFirestoreField: undefined, + }) + ).toEqual({ + granularity: "DAY", + bigqueryColumnName: "timestamp", + }); + }); + + test("treats omit fieldType as no field type for timestamp field", () => { + expect( + buildPartitioningConfig({ + timePartitioning: "DAY", + timePartitioningField: "timestamp", + timePartitioningFieldType: "omit", + timePartitioningFirestoreField: undefined, + }) + ).toEqual({ + granularity: "DAY", + bigqueryColumnName: "timestamp", + }); + }); + + test("treats NONE fieldType as no field type for timestamp field", () => { + expect( + buildPartitioningConfig({ + timePartitioning: "HOUR", + timePartitioningField: "timestamp", + timePartitioningFieldType: "NONE", + timePartitioningFirestoreField: undefined, + }) + ).toEqual({ + granularity: "HOUR", + bigqueryColumnName: "timestamp", + }); + }); + + test("supports timestamp field with NONE firestoreField sentinel", () => { + expect( + buildPartitioningConfig({ + timePartitioning: "DAY", + timePartitioningField: "timestamp", + timePartitioningFieldType: "TIMESTAMP", + timePartitioningFirestoreField: "NONE", + }) + ).toEqual({ + granularity: "DAY", + bigqueryColumnName: "timestamp", + bigqueryColumnType: "TIMESTAMP", + }); + }); + }); + + describe("custom field partitioning", () => { + test("returns custom config with TIMESTAMP type", () => { + expect( + buildPartitioningConfig({ + timePartitioning: "HOUR", + timePartitioningField: "partition_column", + timePartitioningFieldType: "TIMESTAMP", + timePartitioningFirestoreField: "time", + }) + ).toEqual({ + granularity: "HOUR", + bigqueryColumnName: "partition_column", + bigqueryColumnType: "TIMESTAMP", + firestoreFieldName: "time", + }); + }); + + test("returns custom config with DATE type", () => { + expect( + buildPartitioningConfig({ + timePartitioning: "DAY", + timePartitioningField: "date_col", + timePartitioningFieldType: "DATE", + timePartitioningFirestoreField: "eventDate", + }) + ).toEqual({ + granularity: "DAY", + bigqueryColumnName: "date_col", + bigqueryColumnType: "DATE", + firestoreFieldName: "eventDate", + }); + }); + + test("returns custom config with DATETIME type", () => { + expect( + buildPartitioningConfig({ + timePartitioning: "MONTH", + timePartitioningField: "dt_col", + timePartitioningFieldType: "DATETIME", + timePartitioningFirestoreField: "createdAt", + }) + ).toEqual({ + granularity: "MONTH", + bigqueryColumnName: "dt_col", + bigqueryColumnType: "DATETIME", + firestoreFieldName: "createdAt", + }); + }); + + test("trims whitespace from field values", () => { + expect( + buildPartitioningConfig({ + timePartitioning: "DAY", + timePartitioningField: " partition_column ", + timePartitioningFieldType: " TIMESTAMP ", + timePartitioningFirestoreField: " time ", + }) + ).toEqual({ + granularity: "DAY", + bigqueryColumnName: "partition_column", + bigqueryColumnType: "TIMESTAMP", + firestoreFieldName: "time", + }); + }); + }); + + describe("error cases", () => { + test("throws when only fieldName is provided with partitioning enabled", () => { + expect(() => + buildPartitioningConfig({ + timePartitioning: "HOUR", + timePartitioningField: "partition_column", + timePartitioningFieldType: undefined, + timePartitioningFirestoreField: undefined, + }) + ).toThrow(/Invalid partitioning configuration/); + }); + + test("throws when fieldName and fieldType are provided but firestoreField is missing", () => { + expect(() => + buildPartitioningConfig({ + timePartitioning: "HOUR", + timePartitioningField: "partition_column", + timePartitioningFieldType: "TIMESTAMP", + timePartitioningFirestoreField: undefined, + }) + ).toThrow(/Invalid partitioning configuration/); + }); + + test("throws when only firestoreField is provided", () => { + expect(() => + buildPartitioningConfig({ + timePartitioning: "HOUR", + timePartitioningField: undefined, + timePartitioningFieldType: undefined, + timePartitioningFirestoreField: "time", + }) + ).toThrow(/Invalid partitioning configuration/); + }); + + test("throws when fieldType and firestoreField are provided but fieldName is missing", () => { + expect(() => + buildPartitioningConfig({ + timePartitioning: "HOUR", + timePartitioningField: undefined, + timePartitioningFieldType: "TIMESTAMP", + timePartitioningFirestoreField: "time", + }) + ).toThrow(/Valid combinations are/); + }); + + test("throws when fieldType is omit sentinel with real fieldName", () => { + expect(() => + buildPartitioningConfig({ + timePartitioning: "HOUR", + timePartitioningField: "partition_column", + timePartitioningFieldType: "omit", + timePartitioningFirestoreField: undefined, + }) + ).toThrow(/Invalid partitioning configuration/); + }); + + test("throws with invalid fieldType string for custom config", () => { + expect(() => + buildPartitioningConfig({ + timePartitioning: "DAY", + timePartitioningField: "partition_column", + timePartitioningFieldType: "INTEGER", + timePartitioningFirestoreField: "createdAt", + }) + ).toThrow(/Invalid partitioning configuration/); + }); + + test("error message includes received parameter values", () => { + expect(() => + buildPartitioningConfig({ + timePartitioning: "HOUR", + timePartitioningField: "my_field", + timePartitioningFieldType: undefined, + timePartitioningFirestoreField: undefined, + }) + ).toThrow(/Received TABLE_PARTITIONING/); + }); + + test("error message lists valid combinations", () => { + expect(() => + buildPartitioningConfig({ + timePartitioning: "HOUR", + timePartitioningField: "my_field", + timePartitioningFieldType: undefined, + timePartitioningFirestoreField: undefined, + }) + ).toThrow(/Valid combinations are/); + }); + }); + }); }); diff --git a/firestore-bigquery-export/functions/package-lock.json b/firestore-bigquery-export/functions/package-lock.json index 1854099fd1..8d6cbc41cb 100644 --- a/firestore-bigquery-export/functions/package-lock.json +++ b/firestore-bigquery-export/functions/package-lock.json @@ -7,7 +7,7 @@ "name": "firestore-bigquery-export", "license": "Apache-2.0", "dependencies": { - "@firebaseextensions/firestore-bigquery-change-tracker": "^1.1.42", + "@firebaseextensions/firestore-bigquery-change-tracker": "^2.0.2", "@google-cloud/bigquery": "^7.6.0", "@types/express-serve-static-core": "4.17.30", "firebase-admin": "^13.2.0", @@ -686,18 +686,15 @@ } }, "node_modules/@firebaseextensions/firestore-bigquery-change-tracker": { - "version": "1.1.42", - "resolved": "https://registry.npmjs.org/@firebaseextensions/firestore-bigquery-change-tracker/-/firestore-bigquery-change-tracker-1.1.42.tgz", - "integrity": "sha512-IdGKcVoLGZOZNlGMGye3AndiobqEK9by/3TgOCy/AUs+YbcCQOFPopkt7Q0jHbj0VHeFHfGV0vg5ngI0tG2TMg==", + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/@firebaseextensions/firestore-bigquery-change-tracker/-/firestore-bigquery-change-tracker-2.0.2.tgz", + "integrity": "sha512-nWmlXNMjp3BqXtk3U8ZOoNRM/8OaKVl7LFanDNeNkC3oXb7cnE/4fQUwEpqBjf9rggzbPbvI+LrfJtoDUyF4jA==", "license": "Apache-2.0", "dependencies": { "@google-cloud/bigquery": "^7.6.0", "@google-cloud/resource-manager": "^5.1.0", "firebase-admin": "^13.2.0", "firebase-functions": "^6.3.2", - "generate-schema": "^2.6.0", - "inquirer": "^6.4.0", - "lodash": "^4.17.14", "node-fetch": "^2.6.1", "sql-formatter": "^2.3.3", "traverse": "^0.6.6" @@ -900,356 +897,6 @@ "node": ">=6" } }, - "node_modules/@inquirer/ansi": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/@inquirer/ansi/-/ansi-1.0.2.tgz", - "integrity": "sha512-S8qNSZiYzFd0wAcyG5AXCvUHC5Sr7xpZ9wZ2py9XR88jUz8wooStVx5M6dRzczbBWjic9NP7+rY0Xi7qqK/aMQ==", - "license": "MIT", - "engines": { - "node": ">=18" - } - }, - "node_modules/@inquirer/checkbox": { - "version": "4.3.2", - "resolved": "https://registry.npmjs.org/@inquirer/checkbox/-/checkbox-4.3.2.tgz", - "integrity": "sha512-VXukHf0RR1doGe6Sm4F0Em7SWYLTHSsbGfJdS9Ja2bX5/D5uwVOEjr07cncLROdBvmnvCATYEWlHqYmXv2IlQA==", - "license": "MIT", - "dependencies": { - "@inquirer/ansi": "^1.0.2", - "@inquirer/core": "^10.3.2", - "@inquirer/figures": "^1.0.15", - "@inquirer/type": "^3.0.10", - "yoctocolors-cjs": "^2.1.3" - }, - "engines": { - "node": ">=18" - }, - "peerDependencies": { - "@types/node": ">=18" - }, - "peerDependenciesMeta": { - "@types/node": { - "optional": true - } - } - }, - "node_modules/@inquirer/confirm": { - "version": "5.1.21", - "resolved": "https://registry.npmjs.org/@inquirer/confirm/-/confirm-5.1.21.tgz", - "integrity": "sha512-KR8edRkIsUayMXV+o3Gv+q4jlhENF9nMYUZs9PA2HzrXeHI8M5uDag70U7RJn9yyiMZSbtF5/UexBtAVtZGSbQ==", - "license": "MIT", - "dependencies": { - "@inquirer/core": "^10.3.2", - "@inquirer/type": "^3.0.10" - }, - "engines": { - "node": ">=18" - }, - "peerDependencies": { - "@types/node": ">=18" - }, - "peerDependenciesMeta": { - "@types/node": { - "optional": true - } - } - }, - "node_modules/@inquirer/core": { - "version": "10.3.2", - "resolved": "https://registry.npmjs.org/@inquirer/core/-/core-10.3.2.tgz", - "integrity": "sha512-43RTuEbfP8MbKzedNqBrlhhNKVwoK//vUFNW3Q3vZ88BLcrs4kYpGg+B2mm5p2K/HfygoCxuKwJJiv8PbGmE0A==", - "license": "MIT", - "dependencies": { - "@inquirer/ansi": "^1.0.2", - "@inquirer/figures": "^1.0.15", - "@inquirer/type": "^3.0.10", - "cli-width": "^4.1.0", - "mute-stream": "^2.0.0", - "signal-exit": "^4.1.0", - "wrap-ansi": "^6.2.0", - "yoctocolors-cjs": "^2.1.3" - }, - "engines": { - "node": ">=18" - }, - "peerDependencies": { - "@types/node": ">=18" - }, - "peerDependenciesMeta": { - "@types/node": { - "optional": true - } - } - }, - "node_modules/@inquirer/editor": { - "version": "4.2.23", - "resolved": "https://registry.npmjs.org/@inquirer/editor/-/editor-4.2.23.tgz", - "integrity": "sha512-aLSROkEwirotxZ1pBaP8tugXRFCxW94gwrQLxXfrZsKkfjOYC1aRvAZuhpJOb5cu4IBTJdsCigUlf2iCOu4ZDQ==", - "license": "MIT", - "dependencies": { - "@inquirer/core": "^10.3.2", - "@inquirer/external-editor": "^1.0.3", - "@inquirer/type": "^3.0.10" - }, - "engines": { - "node": ">=18" - }, - "peerDependencies": { - "@types/node": ">=18" - }, - "peerDependenciesMeta": { - "@types/node": { - "optional": true - } - } - }, - "node_modules/@inquirer/expand": { - "version": "4.0.23", - "resolved": "https://registry.npmjs.org/@inquirer/expand/-/expand-4.0.23.tgz", - "integrity": "sha512-nRzdOyFYnpeYTTR2qFwEVmIWypzdAx/sIkCMeTNTcflFOovfqUk+HcFhQQVBftAh9gmGrpFj6QcGEqrDMDOiew==", - "license": "MIT", - "dependencies": { - "@inquirer/core": "^10.3.2", - "@inquirer/type": "^3.0.10", - "yoctocolors-cjs": "^2.1.3" - }, - "engines": { - "node": ">=18" - }, - "peerDependencies": { - "@types/node": ">=18" - }, - "peerDependenciesMeta": { - "@types/node": { - "optional": true - } - } - }, - "node_modules/@inquirer/external-editor": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/@inquirer/external-editor/-/external-editor-1.0.3.tgz", - "integrity": "sha512-RWbSrDiYmO4LbejWY7ttpxczuwQyZLBUyygsA9Nsv95hpzUWwnNTVQmAq3xuh7vNwCp07UTmE5i11XAEExx4RA==", - "license": "MIT", - "dependencies": { - "chardet": "^2.1.1", - "iconv-lite": "^0.7.0" - }, - "engines": { - "node": ">=18" - }, - "peerDependencies": { - "@types/node": ">=18" - }, - "peerDependenciesMeta": { - "@types/node": { - "optional": true - } - } - }, - "node_modules/@inquirer/external-editor/node_modules/iconv-lite": { - "version": "0.7.2", - "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.2.tgz", - "integrity": "sha512-im9DjEDQ55s9fL4EYzOAv0yMqmMBSZp6G0VvFyTMPKWxiSBHUj9NW/qqLmXUwXrrM7AvqSlTCfvqRb0cM8yYqw==", - "license": "MIT", - "dependencies": { - "safer-buffer": ">= 2.1.2 < 3.0.0" - }, - "engines": { - "node": ">=0.10.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" - } - }, - "node_modules/@inquirer/figures": { - "version": "1.0.15", - "resolved": "https://registry.npmjs.org/@inquirer/figures/-/figures-1.0.15.tgz", - "integrity": "sha512-t2IEY+unGHOzAaVM5Xx6DEWKeXlDDcNPeDyUpsRc6CUhBfU3VQOEl+Vssh7VNp1dR8MdUJBWhuObjXCsVpjN5g==", - "license": "MIT", - "engines": { - "node": ">=18" - } - }, - "node_modules/@inquirer/input": { - "version": "4.3.1", - "resolved": "https://registry.npmjs.org/@inquirer/input/-/input-4.3.1.tgz", - "integrity": "sha512-kN0pAM4yPrLjJ1XJBjDxyfDduXOuQHrBB8aLDMueuwUGn+vNpF7Gq7TvyVxx8u4SHlFFj4trmj+a2cbpG4Jn1g==", - "license": "MIT", - "dependencies": { - "@inquirer/core": "^10.3.2", - "@inquirer/type": "^3.0.10" - }, - "engines": { - "node": ">=18" - }, - "peerDependencies": { - "@types/node": ">=18" - }, - "peerDependenciesMeta": { - "@types/node": { - "optional": true - } - } - }, - "node_modules/@inquirer/number": { - "version": "3.0.23", - "resolved": "https://registry.npmjs.org/@inquirer/number/-/number-3.0.23.tgz", - "integrity": "sha512-5Smv0OK7K0KUzUfYUXDXQc9jrf8OHo4ktlEayFlelCjwMXz0299Y8OrI+lj7i4gCBY15UObk76q0QtxjzFcFcg==", - "license": "MIT", - "dependencies": { - "@inquirer/core": "^10.3.2", - "@inquirer/type": "^3.0.10" - }, - "engines": { - "node": ">=18" - }, - "peerDependencies": { - "@types/node": ">=18" - }, - "peerDependenciesMeta": { - "@types/node": { - "optional": true - } - } - }, - "node_modules/@inquirer/password": { - "version": "4.0.23", - "resolved": "https://registry.npmjs.org/@inquirer/password/-/password-4.0.23.tgz", - "integrity": "sha512-zREJHjhT5vJBMZX/IUbyI9zVtVfOLiTO66MrF/3GFZYZ7T4YILW5MSkEYHceSii/KtRk+4i3RE7E1CUXA2jHcA==", - "license": "MIT", - "dependencies": { - "@inquirer/ansi": "^1.0.2", - "@inquirer/core": "^10.3.2", - "@inquirer/type": "^3.0.10" - }, - "engines": { - "node": ">=18" - }, - "peerDependencies": { - "@types/node": ">=18" - }, - "peerDependenciesMeta": { - "@types/node": { - "optional": true - } - } - }, - "node_modules/@inquirer/prompts": { - "version": "7.10.1", - "resolved": "https://registry.npmjs.org/@inquirer/prompts/-/prompts-7.10.1.tgz", - "integrity": "sha512-Dx/y9bCQcXLI5ooQ5KyvA4FTgeo2jYj/7plWfV5Ak5wDPKQZgudKez2ixyfz7tKXzcJciTxqLeK7R9HItwiByg==", - "license": "MIT", - "dependencies": { - "@inquirer/checkbox": "^4.3.2", - "@inquirer/confirm": "^5.1.21", - "@inquirer/editor": "^4.2.23", - "@inquirer/expand": "^4.0.23", - "@inquirer/input": "^4.3.1", - "@inquirer/number": "^3.0.23", - "@inquirer/password": "^4.0.23", - "@inquirer/rawlist": "^4.1.11", - "@inquirer/search": "^3.2.2", - "@inquirer/select": "^4.4.2" - }, - "engines": { - "node": ">=18" - }, - "peerDependencies": { - "@types/node": ">=18" - }, - "peerDependenciesMeta": { - "@types/node": { - "optional": true - } - } - }, - "node_modules/@inquirer/rawlist": { - "version": "4.1.11", - "resolved": "https://registry.npmjs.org/@inquirer/rawlist/-/rawlist-4.1.11.tgz", - "integrity": "sha512-+LLQB8XGr3I5LZN/GuAHo+GpDJegQwuPARLChlMICNdwW7OwV2izlCSCxN6cqpL0sMXmbKbFcItJgdQq5EBXTw==", - "license": "MIT", - "dependencies": { - "@inquirer/core": "^10.3.2", - "@inquirer/type": "^3.0.10", - "yoctocolors-cjs": "^2.1.3" - }, - "engines": { - "node": ">=18" - }, - "peerDependencies": { - "@types/node": ">=18" - }, - "peerDependenciesMeta": { - "@types/node": { - "optional": true - } - } - }, - "node_modules/@inquirer/search": { - "version": "3.2.2", - "resolved": "https://registry.npmjs.org/@inquirer/search/-/search-3.2.2.tgz", - "integrity": "sha512-p2bvRfENXCZdWF/U2BXvnSI9h+tuA8iNqtUKb9UWbmLYCRQxd8WkvwWvYn+3NgYaNwdUkHytJMGG4MMLucI1kA==", - "license": "MIT", - "dependencies": { - "@inquirer/core": "^10.3.2", - "@inquirer/figures": "^1.0.15", - "@inquirer/type": "^3.0.10", - "yoctocolors-cjs": "^2.1.3" - }, - "engines": { - "node": ">=18" - }, - "peerDependencies": { - "@types/node": ">=18" - }, - "peerDependenciesMeta": { - "@types/node": { - "optional": true - } - } - }, - "node_modules/@inquirer/select": { - "version": "4.4.2", - "resolved": "https://registry.npmjs.org/@inquirer/select/-/select-4.4.2.tgz", - "integrity": "sha512-l4xMuJo55MAe+N7Qr4rX90vypFwCajSakx59qe/tMaC1aEHWLyw68wF4o0A4SLAY4E0nd+Vt+EyskeDIqu1M6w==", - "license": "MIT", - "dependencies": { - "@inquirer/ansi": "^1.0.2", - "@inquirer/core": "^10.3.2", - "@inquirer/figures": "^1.0.15", - "@inquirer/type": "^3.0.10", - "yoctocolors-cjs": "^2.1.3" - }, - "engines": { - "node": ">=18" - }, - "peerDependencies": { - "@types/node": ">=18" - }, - "peerDependenciesMeta": { - "@types/node": { - "optional": true - } - } - }, - "node_modules/@inquirer/type": { - "version": "3.0.10", - "resolved": "https://registry.npmjs.org/@inquirer/type/-/type-3.0.10.tgz", - "integrity": "sha512-BvziSRxfz5Ov8ch0z/n3oijRSEcEsHnhggm4xFZe93DHcUCTlutlq9Ox4SVENAfcRD22UQq7T/atg9Wr3k09eA==", - "license": "MIT", - "engines": { - "node": ">=18" - }, - "peerDependencies": { - "@types/node": ">=18" - }, - "peerDependenciesMeta": { - "@types/node": { - "optional": true - } - } - }, "node_modules/@isaacs/cliui": { "version": "8.0.2", "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz", @@ -2875,12 +2522,6 @@ "node": ">=10" } }, - "node_modules/chardet": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/chardet/-/chardet-2.1.1.tgz", - "integrity": "sha512-PsezH1rqdV9VvyNhxxOW32/d75r01NY7TQCmOqomRo15ZSOKbpTFVsfjghxo6JloQUCGnH4k1LGu0R4yCLlWQQ==", - "license": "MIT" - }, "node_modules/check-error": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/check-error/-/check-error-1.0.3.tgz", @@ -2937,15 +2578,6 @@ "node": ">=6" } }, - "node_modules/cli-width": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/cli-width/-/cli-width-4.1.0.tgz", - "integrity": "sha512-ouuZd4/dm2Sw5Gmqy6bGyNNNe1qt9RpmxveLSO7KcgsTnU7RXfsw+/bukWGo1abgBiMAic068rclZsO4IWmmxQ==", - "license": "ISC", - "engines": { - "node": ">= 12" - } - }, "node_modules/cliui": { "version": "8.0.1", "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", @@ -4904,32 +4536,6 @@ "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", "license": "ISC" }, - "node_modules/inquirer": { - "version": "12.11.1", - "resolved": "https://registry.npmjs.org/inquirer/-/inquirer-12.11.1.tgz", - "integrity": "sha512-9VF7mrY+3OmsAfjH3yKz/pLbJ5z22E23hENKw3/LNSaA/sAt3v49bDRY+Ygct1xwuKT+U+cBfTzjCPySna69Qw==", - "license": "MIT", - "dependencies": { - "@inquirer/ansi": "^1.0.2", - "@inquirer/core": "^10.3.2", - "@inquirer/prompts": "^7.10.1", - "@inquirer/type": "^3.0.10", - "mute-stream": "^2.0.0", - "run-async": "^4.0.6", - "rxjs": "^7.8.2" - }, - "engines": { - "node": ">=18" - }, - "peerDependencies": { - "@types/node": ">=18" - }, - "peerDependenciesMeta": { - "@types/node": { - "optional": true - } - } - }, "node_modules/internal-slot": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/internal-slot/-/internal-slot-1.1.0.tgz", @@ -6711,15 +6317,6 @@ "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", "license": "MIT" }, - "node_modules/mute-stream": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-2.0.0.tgz", - "integrity": "sha512-WWdIxpyjEn+FhQJQQv9aQAYlHoNVdzIzUySNV1gHUPDSdZJ3yZn7pAAbQcV7B56Mvu881q9FZV+0Vx2xC44VWA==", - "license": "ISC", - "engines": { - "node": "^18.17.0 || >=20.5.0" - } - }, "node_modules/natural-compare": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", @@ -7778,24 +7375,6 @@ "url": "https://github.com/sponsors/isaacs" } }, - "node_modules/run-async": { - "version": "4.0.6", - "resolved": "https://registry.npmjs.org/run-async/-/run-async-4.0.6.tgz", - "integrity": "sha512-IoDlSLTs3Yq593mb3ZoKWKXMNu3UpObxhgA/Xuid5p4bbfi2jdY1Hj0m1K+0/tEuQTxIGMhQDqGjKb7RuxGpAQ==", - "license": "MIT", - "engines": { - "node": ">=0.12.0" - } - }, - "node_modules/rxjs": { - "version": "7.8.2", - "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-7.8.2.tgz", - "integrity": "sha512-dhKf903U/PQZY6boNNtAGdWbG85WAbjT/1xYoZIC7FAY0yWapOBQVsVrDl58W86//e1VpMNBtRV4MaXfdMySFA==", - "license": "Apache-2.0", - "dependencies": { - "tslib": "^2.1.0" - } - }, "node_modules/safe-array-concat": { "version": "1.1.3", "resolved": "https://registry.npmjs.org/safe-array-concat/-/safe-array-concat-1.1.3.tgz", @@ -9164,6 +8743,7 @@ "version": "6.2.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-6.2.0.tgz", "integrity": "sha512-r6lPcBGxZXlIcymEu7InxDMhdW0KDxpLgoFLcguasxCaJ/SOIZwINatK9KY/tf+ZrlywOKU0UDj3ATXUBfxJXA==", + "dev": true, "license": "MIT", "dependencies": { "ansi-styles": "^4.0.0", @@ -9284,18 +8864,6 @@ "funding": { "url": "https://github.com/sponsors/sindresorhus" } - }, - "node_modules/yoctocolors-cjs": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/yoctocolors-cjs/-/yoctocolors-cjs-2.1.3.tgz", - "integrity": "sha512-U/PBtDf35ff0D8X8D0jfdzHYEPFxAI7jJlxZXwCSez5M3190m+QobIfh+sWDWSHMCWWJN2AWamkegn6vr6YBTw==", - "license": "MIT", - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } } } } diff --git a/firestore-bigquery-export/functions/package.json b/firestore-bigquery-export/functions/package.json index 7e1f88c3bc..9070b2cbd2 100644 --- a/firestore-bigquery-export/functions/package.json +++ b/firestore-bigquery-export/functions/package.json @@ -13,7 +13,7 @@ "author": "Jan Wyszynski ", "license": "Apache-2.0", "dependencies": { - "@firebaseextensions/firestore-bigquery-change-tracker": "^1.1.42", + "@firebaseextensions/firestore-bigquery-change-tracker": "^2.0.2", "@google-cloud/bigquery": "^7.6.0", "@types/express-serve-static-core": "4.17.30", "firebase-admin": "^13.2.0", diff --git a/firestore-bigquery-export/functions/src/config.ts b/firestore-bigquery-export/functions/src/config.ts index f58d43cf38..afced8680d 100644 --- a/firestore-bigquery-export/functions/src/config.ts +++ b/firestore-bigquery-export/functions/src/config.ts @@ -14,8 +14,16 @@ * limitations under the License. */ import { LogLevel } from "@firebaseextensions/firestore-bigquery-change-tracker"; +import type { + ChangeTrackerConfig, + PartitioningFieldType, + TimePartitioningGranularity, +} from "@firebaseextensions/firestore-bigquery-change-tracker"; +type TrackerLogLevel = "debug" | "info" | "warn" | "error" | "silent"; -function timePartitioning(type) { +function timePartitioning( + type: string | undefined +): TimePartitioningGranularity | null { if ( type === "HOUR" || type === "DAY" || @@ -32,6 +40,127 @@ export function clustering(clusters: string | undefined) { return clusters ? clusters.split(",").slice(0, 4) : null; } +function normalizeOptionalPartitionValue( + value: string | undefined +): string | undefined { + const normalized = value?.trim(); + + if (!normalized || normalized === "NONE" || normalized === "omit") { + return undefined; + } + + return normalized; +} + +function normalizePartitionFieldType( + value: string | undefined +): PartitioningFieldType | undefined { + const normalized = normalizeOptionalPartitionValue(value); + if ( + normalized === "TIMESTAMP" || + normalized === "DATE" || + normalized === "DATETIME" + ) { + return normalized; + } + return undefined; +} + +export function buildPartitioningConfig(params: { + timePartitioning: TimePartitioningGranularity | null; + timePartitioningField: string | undefined; + timePartitioningFieldType: string | undefined; + timePartitioningFirestoreField: string | undefined; +}): ChangeTrackerConfig["partitioning"] { + const { timePartitioning } = params; + const rawFieldName = params.timePartitioningField?.trim(); + const rawFieldType = params.timePartitioningFieldType?.trim(); + const rawFirestoreField = params.timePartitioningFirestoreField?.trim(); + + const formatValue = (value: string | undefined): string => + value && value.length > 0 ? `"${value}"` : "(empty)"; + + const throwInvalidPartitioningConfig = (detail: string): never => { + throw new Error( + [ + "Invalid partitioning configuration for firestore-bigquery-export.", + detail, + `Received TABLE_PARTITIONING=${formatValue( + timePartitioning ?? undefined + )},`, + `TIME_PARTITIONING_FIELD=${formatValue(rawFieldName)},`, + `TIME_PARTITIONING_FIRESTORE_FIELD=${formatValue(rawFirestoreField)},`, + `TIME_PARTITIONING_FIELD_TYPE=${formatValue(rawFieldType)}.`, + "Valid combinations are:", + "1) Ingestion-time: TABLE_PARTITIONING set and all TIME_PARTITIONING_* values empty/NONE/omit.", + "2) Timestamp field: TABLE_PARTITIONING set, TIME_PARTITIONING_FIELD=timestamp, TIME_PARTITIONING_FIRESTORE_FIELD empty.", + "3) Custom field: TABLE_PARTITIONING set, and TIME_PARTITIONING_FIELD + TIME_PARTITIONING_FIRESTORE_FIELD + TIME_PARTITIONING_FIELD_TYPE all provided.", + ].join(" ") + ); + }; + + const fieldName = normalizeOptionalPartitionValue( + params.timePartitioningField + ); + const fieldType = normalizePartitionFieldType( + params.timePartitioningFieldType + ); + const firestoreField = normalizeOptionalPartitionValue( + params.timePartitioningFirestoreField + ); + + if (!timePartitioning) { + if (fieldName || fieldType || firestoreField) { + return throwInvalidPartitioningConfig( + "Partition-specific fields cannot be provided when TABLE_PARTITIONING is NONE." + ); + } + return { granularity: "NONE" }; + } + + if (!fieldName && !firestoreField) { + return { granularity: timePartitioning }; + } + + if (fieldName === "timestamp" && !firestoreField) { + return { + granularity: timePartitioning, + bigqueryColumnName: "timestamp", + ...(fieldType ? { bigqueryColumnType: fieldType } : {}), + }; + } + + if (fieldName && firestoreField && fieldType) { + return { + granularity: timePartitioning, + bigqueryColumnName: fieldName, + bigqueryColumnType: fieldType, + firestoreFieldName: firestoreField, + }; + } + + return throwInvalidPartitioningConfig( + "When TABLE_PARTITIONING is set, partitioning fields are either incomplete or invalid." + ); +} + +function normalizeLogLevel(level: string | undefined): TrackerLogLevel { + switch ((level || "").toLowerCase()) { + case "debug": + return "debug"; + case "info": + return "info"; + case "warn": + return "warn"; + case "error": + return "error"; + case "silent": + return "silent"; + default: + return LogLevel.INFO; + } +} + export default { bqProjectId: process.env.BIGQUERY_PROJECT_ID, projectId: process.env.PROJECT_ID, @@ -53,6 +182,13 @@ export default { ? process.env.TIME_PARTITIONING_FIELD_TYPE : undefined, timePartitioningFirestoreField: process.env.TIME_PARTITIONING_FIRESTORE_FIELD, + partitioning: buildPartitioningConfig({ + timePartitioning: timePartitioning(process.env.TABLE_PARTITIONING), + timePartitioningField: process.env.TIME_PARTITIONING_FIELD, + timePartitioningFieldType: process.env.TIME_PARTITIONING_FIELD_TYPE, + timePartitioningFirestoreField: + process.env.TIME_PARTITIONING_FIRESTORE_FIELD, + }), clustering: clustering(process.env.CLUSTERING), wildcardIds: process.env.WILDCARD_IDS === "true", useNewSnapshotQuerySyntax: @@ -76,5 +212,5 @@ export default { backupBucketName: process.env.BACKUP_GCS_BUCKET || `${process.env.PROJECT_ID}.appspot.com`, backupDir: `_${process.env.INSTANCE_ID || "firestore-bigquery-export"}`, - logLevel: process.env.LOG_LEVEL || LogLevel.INFO, + logLevel: normalizeLogLevel(process.env.LOG_LEVEL), }; diff --git a/firestore-bigquery-export/functions/src/index.ts b/firestore-bigquery-export/functions/src/index.ts index 9225572436..8a80286ffb 100644 --- a/firestore-bigquery-export/functions/src/index.ts +++ b/firestore-bigquery-export/functions/src/index.ts @@ -42,10 +42,7 @@ const eventTrackerConfig = { datasetLocation: config.datasetLocation, backupTableId: config.backupCollectionId, transformFunction: config.transformFunction, - timePartitioning: config.timePartitioning, - timePartitioningField: config.timePartitioningField, - timePartitioningFieldType: config.timePartitioningFieldType, - timePartitioningFirestoreField: config.timePartitioningFirestoreField, + partitioning: config.partitioning, // Database related configurations databaseId: config.databaseId, clustering: config.clustering, diff --git a/firestore-send-email/CHANGELOG.md b/firestore-send-email/CHANGELOG.md index faaf0b561a..340d3e54d6 100644 --- a/firestore-send-email/CHANGELOG.md +++ b/firestore-send-email/CHANGELOG.md @@ -1,3 +1,7 @@ +## Version 0.2.7 + +feat: add `customArgs` and `ipPoolName` fields to SendGrid options (contributed by @michalpechnet) + ## Version 0.2.6 chore: audit and audit fix packages diff --git a/firestore-send-email/POSTINSTALL.md b/firestore-send-email/POSTINSTALL.md index 5224adb988..89b7a371ac 100644 --- a/firestore-send-email/POSTINSTALL.md +++ b/firestore-send-email/POSTINSTALL.md @@ -82,6 +82,8 @@ When using SendGrid, you can use SendGrid Dynamic Templates to create and send t Add this document to the Firestore mail collection to send an email using a SendGrid Dynamic Template. The `templateId` is required and should be your SendGrid Dynamic Template ID (always starts with 'd-'). The `dynamicTemplateData` object contains the variables that will be used in your template. +You can also include optional SendGrid fields in the same `sendGrid` object, such as `customArgs` (string key/value metadata) and `ipPoolName` (IP pool selection). + For more details, see the [SendGrid Dynamic Templates documentation](https://docs.sendgrid.com/ui/sending-email/how-to-send-an-email-with-dynamic-templates). #### Understanding SendGrid Email IDs diff --git a/firestore-send-email/extension.yaml b/firestore-send-email/extension.yaml index 354a45a3c6..4d8572fb9f 100644 --- a/firestore-send-email/extension.yaml +++ b/firestore-send-email/extension.yaml @@ -13,7 +13,7 @@ # limitations under the License. name: firestore-send-email -version: 0.2.6 +version: 0.2.7 specVersion: v1beta displayName: Trigger Email from Firestore diff --git a/firestore-send-email/functions/__tests__/nodemailer-sendgrid/index.test.ts b/firestore-send-email/functions/__tests__/nodemailer-sendgrid/index.test.ts index 3ec2c77536..bab3e3445c 100644 --- a/firestore-send-email/functions/__tests__/nodemailer-sendgrid/index.test.ts +++ b/firestore-send-email/functions/__tests__/nodemailer-sendgrid/index.test.ts @@ -408,6 +408,62 @@ describe("SendGridTransport", () => { }); }); + test("send: forwards customArgs object", async () => { + const transport = new SendGridTransport({ apiKey: "KEY" }); + const fakeMail: any = { + normalize: (cb: any) => + cb(null, { + from: { address: "a@x.com" }, + to: [{ address: "b@x.com" }], + subject: "Custom args test", + customArgs: { campaign: "welcome", source: "signup" }, + }), + }; + const cb = jest.fn(); + + transport.send(fakeMail, cb); + await new Promise((r) => setImmediate(r)); + + const sent = (sgMail.send as jest.Mock).mock.calls[0][0]; + expect(sent.customArgs).toEqual({ campaign: "welcome", source: "signup" }); + expect(cb).toHaveBeenCalledWith(null, { + messageId: null, + queueId: "test-message-id", + accepted: ["b@x.com"], + rejected: [], + pending: [], + response: "status=202", + }); + }); + + test("send: forwards ipPoolName string", async () => { + const transport = new SendGridTransport({ apiKey: "KEY" }); + const fakeMail: any = { + normalize: (cb: any) => + cb(null, { + from: { address: "a@x.com" }, + to: [{ address: "b@x.com" }], + subject: "IP pool test", + ipPoolName: "transactional", + }), + }; + const cb = jest.fn(); + + transport.send(fakeMail, cb); + await new Promise((r) => setImmediate(r)); + + const sent = (sgMail.send as jest.Mock).mock.calls[0][0]; + expect(sent.ipPoolName).toEqual("transactional"); + expect(cb).toHaveBeenCalledWith(null, { + messageId: null, + queueId: "test-message-id", + accepted: ["b@x.com"], + rejected: [], + pending: [], + response: "status=202", + }); + }); + test("send: deduplicates and normalizes email addresses", async () => { const transport = new SendGridTransport(); const source = { diff --git a/firestore-send-email/functions/__tests__/validation.test.ts b/firestore-send-email/functions/__tests__/validation.test.ts index 1d74fca27c..e54990b3cc 100644 --- a/firestore-send-email/functions/__tests__/validation.test.ts +++ b/firestore-send-email/functions/__tests__/validation.test.ts @@ -212,6 +212,28 @@ describe("validatePayload", () => { }); }); + it("should validate a SendGrid payload with customArgs", () => { + const validPayload = { + to: "test@example.com", + sendGrid: { + templateId: "d-template-id", + customArgs: { campaign: "welcome", source: "signup" }, + }, + }; + expect(() => validatePayload(validPayload)).not.toThrow(); + }); + + it("should validate a SendGrid payload with ipPoolName", () => { + const validPayload = { + to: "test@example.com", + sendGrid: { + templateId: "d-template-id", + ipPoolName: "transactional", + }, + }; + expect(() => validatePayload(validPayload)).not.toThrow(); + }); + it("should validate a SendGrid payload with only mailSettings", () => { const validPayload = { to: "test@example.com", @@ -255,6 +277,26 @@ describe("validatePayload", () => { ); }); + it("should throw ValidationError for SendGrid customArgs with non-string values", () => { + const invalidPayload = { + to: "test@example.com", + sendGrid: { + customArgs: { campaign: 123 }, + }, + }; + expect(() => validatePayload(invalidPayload)).toThrow(ValidationError); + }); + + it("should throw ValidationError for SendGrid ipPoolName with non-string value", () => { + const invalidPayload = { + to: "test@example.com", + sendGrid: { + ipPoolName: 123, + }, + }; + expect(() => validatePayload(invalidPayload)).toThrow(ValidationError); + }); + it("should throw ValidationError for custom template without name", () => { const invalidPayload = { to: "test@example.com", diff --git a/firestore-send-email/functions/src/index.ts b/firestore-send-email/functions/src/index.ts index 1611a051e7..97cc48e091 100644 --- a/firestore-send-email/functions/src/index.ts +++ b/firestore-send-email/functions/src/index.ts @@ -179,6 +179,8 @@ async function deliver(ref: DocumentReference): Promise { templateId: payload.sendGrid?.templateId, dynamicTemplateData: payload.sendGrid?.dynamicTemplateData, mailSettings: payload.sendGrid?.mailSettings, + customArgs: payload.sendGrid?.customArgs, + ipPoolName: payload.sendGrid?.ipPoolName, }; logs.info("Sending via transport.sendMail()", { mailOptions }); diff --git a/firestore-send-email/functions/src/nodemailer-sendgrid/index.ts b/firestore-send-email/functions/src/nodemailer-sendgrid/index.ts index 9d7d24c78d..e02807aa35 100644 --- a/firestore-send-email/functions/src/nodemailer-sendgrid/index.ts +++ b/firestore-send-email/functions/src/nodemailer-sendgrid/index.ts @@ -152,6 +152,12 @@ export class SendGridTransport { case "mailSettings": msg.mailSettings = source.mailSettings; break; + case "customArgs": + msg.customArgs = source.customArgs; + break; + case "ipPoolName": + msg.ipPoolName = source.ipPoolName; + break; default: msg[key] = source[key]; diff --git a/firestore-send-email/functions/src/nodemailer-sendgrid/types.ts b/firestore-send-email/functions/src/nodemailer-sendgrid/types.ts index f8e6e626c8..3f7ed10c5c 100644 --- a/firestore-send-email/functions/src/nodemailer-sendgrid/types.ts +++ b/firestore-send-email/functions/src/nodemailer-sendgrid/types.ts @@ -46,6 +46,8 @@ export interface MailSource { templateId?: string; dynamicTemplateData?: Record; mailSettings?: Record; + customArgs?: Record; + ipPoolName?: string; [key: string]: unknown; } @@ -91,5 +93,7 @@ export interface SendGridMessage { templateId?: string; dynamicTemplateData?: Record; mailSettings?: Record; + customArgs?: Record; + ipPoolName?: string; [key: string]: unknown; } diff --git a/firestore-send-email/functions/src/types.ts b/firestore-send-email/functions/src/types.ts index e03e74e745..1b476fb234 100644 --- a/firestore-send-email/functions/src/types.ts +++ b/firestore-send-email/functions/src/types.ts @@ -86,6 +86,8 @@ export interface QueuePayload { templateId?: string; dynamicTemplateData?: { [key: string]: any }; mailSettings?: { [key: string]: any }; + customArgs?: Record; + ipPoolName?: string; }; to: string[]; toUids?: string[]; @@ -127,4 +129,6 @@ export interface ExtendedSendMailOptions extends nodemailer.SendMailOptions { templateId?: string; dynamicTemplateData?: Record; mailSettings?: Record; + customArgs?: Record; + ipPoolName?: string; } diff --git a/firestore-send-email/functions/src/validation.ts b/firestore-send-email/functions/src/validation.ts index 0d13af7075..98f4e4ca7c 100644 --- a/firestore-send-email/functions/src/validation.ts +++ b/firestore-send-email/functions/src/validation.ts @@ -93,6 +93,8 @@ const sendGridSchema = z templateId: z.string().optional(), dynamicTemplateData: z.record(z.any()).optional(), mailSettings: z.record(z.any()).optional(), + customArgs: z.record(z.string()).optional(), + ipPoolName: z.string().optional(), }) .refine( (data) => {