From 5c0c6daf5502894302fb3a90f810f0bb5f0242ca Mon Sep 17 00:00:00 2001 From: vilenarios Date: Wed, 5 Aug 2026 01:00:05 -0400 Subject: [PATCH 1/5] chore(deps): bump ardrive-core-js 4.0.0 -> 4.3.0 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pull in the just-published ardrive-core-js@4.3.0: public-file pinning (pinPublicFile) and the AES-256-CTR private-file decrypt fix (plus 4.1/4.2's parallel folder downloads + gateway hardening + GraphQL page-size tuning). Additive minor bump — no CLI code changes needed. Verified: yarn build (tsc) 0 errors, yarn lint 0 errors, full nyc mocha suite 87 passing / 0 failing on Node 18. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01T1adK4kjWUY8PU5YBd1LcX --- package.json | 2 +- yarn.lock | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/package.json b/package.json index 005b4610..8f19123a 100644 --- a/package.json +++ b/package.json @@ -8,7 +8,7 @@ }, "types": "./lib/index.d.ts", "dependencies": { - "ardrive-core-js": "4.0.0", + "ardrive-core-js": "4.3.0", "arweave": "1.15.7", "axios": "^0.21.1", "commander": "^8.2.0", diff --git a/yarn.lock b/yarn.lock index a9f80be0..b43f6fc6 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2259,7 +2259,7 @@ __metadata: "@types/source-map-support": ^0 "@typescript-eslint/eslint-plugin": ^6.2.1 "@typescript-eslint/parser": ^6.2.1 - ardrive-core-js: 4.0.0 + ardrive-core-js: 4.3.0 arweave: 1.15.7 axios: ^0.21.1 chai: ^4.3.4 @@ -2286,9 +2286,9 @@ __metadata: languageName: unknown linkType: soft -"ardrive-core-js@npm:4.0.0": - version: 4.0.0 - resolution: "ardrive-core-js@npm:4.0.0" +"ardrive-core-js@npm:4.3.0": + version: 4.3.0 + resolution: "ardrive-core-js@npm:4.3.0" dependencies: "@ardrive/ardrive-promise-cache": ^1.1.4 "@ardrive/turbo-sdk": ^1.0.1 @@ -2310,7 +2310,7 @@ __metadata: smartweave: ^0.4.49 utf8: ^3.0.0 uuid: ^9.0.1 - checksum: 0daaf67eded0c5a09bebda0df132cd8e949e4545a14e62ddfd91d231560e9b506cd983a8f0d03be1ab9f8a6c819564ea0d037d1ffe7cc4e7f8f89d817ffffd36 + checksum: 77d242524d0d5b60f209774672c9e346d0de7c639dd62262592642214302b0e32262a2f7ced385f8817fd72b365637e771f08e31848165733d9179c5dd2e6eac languageName: node linkType: hard From 9c17a1a23f12ec9b40f7cb5d4fbe1262ccc46f25 Mon Sep 17 00:00:00 2001 From: vilenarios Date: Wed, 5 Aug 2026 01:28:08 -0400 Subject: [PATCH 2/5] feat(pin): add pin-file command for core-js pinPublicFile Exposes ARDrive.pinPublicFile (new in ardrive-core-js 4.3.0): reference an existing Arweave data transaction as a new public ArFS file entity without re-uploading. Public drives only; core-js's clear "Pinning is only supported for public drives" error propagates through the same catch-and-print handling every other command relies on. Reuses existing parameters (--parent-folder-id, --tx-id, --dest-file-name, --drive-id, --skip, wallet/gateway/turbo/boost/dry-run) rather than inventing new flags, following the retry-tx precedent for repurposing --tx-id via TxID() validation. --drive-id is optional and only asserted against the folder's resolved drive, matching core-js's own resolution. Adds a colocated unit test suite (mocked ARDrive.pinPublicFile) covering flag wiring, --drive-id/--skip mapping, invalid tx-id rejection, and the private-drive error path. Adds README docs + examples under "Pinning a File" and lists pin-file in the command index. --- README.md | 26 ++++++ src/commands/index.ts | 1 + src/commands/pin_file.test.ts | 147 ++++++++++++++++++++++++++++++++++ src/commands/pin_file.ts | 94 ++++++++++++++++++++++ 4 files changed, 268 insertions(+) create mode 100644 src/commands/pin_file.test.ts create mode 100644 src/commands/pin_file.ts diff --git a/README.md b/README.md index cc9b9bc2..10d414c5 100644 --- a/README.md +++ b/README.md @@ -139,6 +139,7 @@ ardrive upload-file --wallet-file /path/to/my/wallet.json --parent-folder-id "f0 19. [Uploading a Custom Manifest](#uploading-a-custom-manifest) 20. [Uploading Files with Custom MetaData](#uploading-files-with-custom-metadata) 21. [Applying Unique Custom MetaData During Bulk Workflows](#applying-unique-custom-metadata-during-bulk-workflows) + 22. [Pinning a File](#pinning-a-file) 8. [Other Utility Operations](#other-utility-operations) 1. [Monitoring Transactions](#monitoring-transactions) 2. [Dealing With Network Congestion](#dealing-with-network-congestion) @@ -1330,6 +1331,30 @@ ardrive upload-file -F f0c58c11-430c-4383-8e54-4d864cc7e927 --local-path "../upl done ``` +### Pinning a File + +Pinning lets you reference an **existing** Arweave data transaction as a new file entity in one of your PUBLIC drives, without re-uploading any data. This is useful for adopting data that already lives permanently on Arweave (e.g. a transaction uploaded outside of ArDrive, or one belonging to someone else) into your drive's folder structure, so it shows up alongside your other files with its own name, metadata, and location. + +Because a pinned file's metadata transaction only references the existing `--data-tx-id` (it doesn't touch the underlying data bytes), pinning a small file costs the same tiny metadata-only fee as any other file operation -- there is no data-upload cost, regardless of the size of the original file. + +Some important constraints: + +- **Public drives only.** Pinning writes a plaintext ArFS metadata transaction that points at the referenced data. Private drives are not supported -- targeting a private `--parent-folder-id` fails with a clear error. +- **The referenced transaction is never re-uploaded or modified.** Only a new file metadata entity is created; `--data-tx-id` is reused as-is as the new file's data transaction. +- **Name conflicts throw by default.** If `--dest-file-name` already exists in the destination folder, the command fails unless `--skip` is provided, in which case the command exits successfully having made no changes. + +```shell +ardrive pin-file --parent-folder-id "a2c8a0cb-0ca7-4dbb-8bf8-93f75f308e63" --tx-id "Y7GFF8r9y0MEU_oi1aZeD87vrmai97JdRQ2L0cbGJ68" --dest-file-name "hello_world.txt" -w "/path/to/wallet" +``` + +`--drive-id` is optional -- the destination drive is normally resolved automatically from `--parent-folder-id`. Supply it only if you want the command to assert that the folder belongs to the drive you expect (the command fails if it doesn't): + +```shell +ardrive pin-file --parent-folder-id "a2c8a0cb-0ca7-4dbb-8bf8-93f75f308e63" --drive-id "bc9af866-6421-40f1-ac89-202bddb5c487" --tx-id "Y7GFF8r9y0MEU_oi1aZeD87vrmai97JdRQ2L0cbGJ68" --dest-file-name "hello_world.txt" -w "/path/to/wallet" +``` + +Like other write commands, `pin-file` supports `--dry-run`, `--boost`, `--turbo`/`--turbo-url`, and `--gateway`. See `ardrive pin-file --help` for the full flag list. + ## Other Utility Operations ### Monitoring Transactions @@ -1498,6 +1523,7 @@ create-drive create-folder upload-file create-manifest +pin-file move-file move-folder diff --git a/src/commands/index.ts b/src/commands/index.ts index e7aeeac4..92728b18 100644 --- a/src/commands/index.ts +++ b/src/commands/index.ts @@ -23,6 +23,7 @@ import './list_drive'; import './list_folder'; import './move_file'; import './move_folder'; +import './pin_file'; import './rename_drive'; import './rename_file'; import './rename_folder'; diff --git a/src/commands/pin_file.test.ts b/src/commands/pin_file.test.ts new file mode 100644 index 00000000..fd17bd4b --- /dev/null +++ b/src/commands/pin_file.test.ts @@ -0,0 +1,147 @@ +import { expect } from 'chai'; +import sinon from 'sinon'; +import { CLICommand, CommandDescriptor } from '../CLICommand/cli_command'; +import { ParametersHelper } from '../CLICommand/parameters_helper'; +import { ERROR_EXIT_CODE, SUCCESS_EXIT_CODE } from '../CLICommand/error_codes'; +import * as cliIndex from '..'; +import { skipOnConflicts, Wallet } from 'ardrive-core-js'; + +// Importing the real command module registers 'pin-file' (with the actual commander program +// singleton) as a side effect -- mirrors how the CLI itself discovers commands via `./commands`. +import './pin_file'; + +const VALID_FOLDER_ID = 'a2c8a0cb-0ca7-4dbb-8bf8-93f75f308e63'; +const VALID_DRIVE_ID = 'bc9af866-6421-40f1-ac89-202bddb5c487'; +const VALID_TX_ID = 'a'.repeat(43); +const DEST_FILE_NAME = 'hello_world.txt'; + +const fakeWallet = ({ getAddress: async () => 'fake-address' } as unknown) as Wallet; + +function getPinFileDescriptor(): CommandDescriptor { + const descriptor = CLICommand.getAllCommandDescriptors().find((cmd) => cmd.name === 'pin-file'); + if (!descriptor) { + throw new Error(`'pin-file' command was not registered`); + } + return descriptor; +} + +describe('pin-file command', () => { + afterEach(() => { + sinon.restore(); + }); + + it('is discoverable via the command registry', () => { + const descriptor = getPinFileDescriptor(); + const parameterNames = descriptor.parameters.map((param) => (typeof param === 'string' ? param : param.name)); + + expect(parameterNames).to.include.members([ + 'parentFolderId', + 'txId', + 'destFileName', + 'driveId', + 'skip', + 'boost', + 'dryRun', + 'walletFile', + 'seedPhrase', + 'gateway', + 'turbo', + 'turboUrl' + ]); + }); + + it('calls ARDrive.pinPublicFile with the correct params and prints the result', async () => { + const fakeResult = { + created: [ + { + type: 'file', + metadataTxId: 'meta-tx-id-000000000000000000000000000', + dataTxId: VALID_TX_ID, + entityId: 'fake-file-id', + entityName: DEST_FILE_NAME + } + ], + tips: [], + fees: {} + }; + const pinPublicFileStub = sinon.stub().resolves(fakeResult); + sinon.stub(cliIndex, 'cliArDriveFactory').returns({ pinPublicFile: pinPublicFileStub } as never); + sinon.stub(ParametersHelper.prototype, 'getRequiredWallet').resolves(fakeWallet); + const consoleLogStub = sinon.stub(console, 'log'); + + const descriptor = getPinFileDescriptor(); + const exitCode = await descriptor.action.trigger({ + parentFolderId: VALID_FOLDER_ID, + txId: VALID_TX_ID, + destFileName: DEST_FILE_NAME + }); + + expect(exitCode).to.equal(SUCCESS_EXIT_CODE); + expect(pinPublicFileStub.calledOnce).to.be.true; + + const callArgs = pinPublicFileStub.firstCall.args[0]; + expect(String(callArgs.parentFolderId)).to.equal(VALID_FOLDER_ID); + expect(String(callArgs.dataTxId)).to.equal(VALID_TX_ID); + expect(callArgs.pinnedFileName).to.equal(DEST_FILE_NAME); + expect(callArgs.driveId).to.be.undefined; + expect(callArgs.conflictResolution).to.be.undefined; + + expect(consoleLogStub.calledWithMatch(sinon.match(/"entityName": "hello_world.txt"/))).to.be.true; + }); + + it('passes the optional --drive-id through as an assertion and maps --skip to skipOnConflicts', async () => { + const pinPublicFileStub = sinon.stub().resolves({ created: [], tips: [], fees: {} }); + sinon.stub(cliIndex, 'cliArDriveFactory').returns({ pinPublicFile: pinPublicFileStub } as never); + sinon.stub(ParametersHelper.prototype, 'getRequiredWallet').resolves(fakeWallet); + sinon.stub(console, 'log'); + + const descriptor = getPinFileDescriptor(); + const exitCode = await descriptor.action.trigger({ + parentFolderId: VALID_FOLDER_ID, + txId: VALID_TX_ID, + destFileName: DEST_FILE_NAME, + driveId: VALID_DRIVE_ID, + skip: true + }); + + expect(exitCode).to.equal(SUCCESS_EXIT_CODE); + const callArgs = pinPublicFileStub.firstCall.args[0]; + expect(String(callArgs.driveId)).to.equal(VALID_DRIVE_ID); + expect(callArgs.conflictResolution).to.equal(skipOnConflicts); + }); + + it('errors clearly, without calling pinPublicFile, when --tx-id is not a valid 43-character transaction id', async () => { + const pinPublicFileStub = sinon.stub().resolves({ created: [], tips: [], fees: {} }); + sinon.stub(cliIndex, 'cliArDriveFactory').returns({ pinPublicFile: pinPublicFileStub } as never); + sinon.stub(ParametersHelper.prototype, 'getRequiredWallet').resolves(fakeWallet); + const consoleLogStub = sinon.stub(console, 'log'); + + const descriptor = getPinFileDescriptor(); + const exitCode = await descriptor.action.trigger({ + parentFolderId: VALID_FOLDER_ID, + txId: 'not-a-valid-tx-id', + destFileName: DEST_FILE_NAME + }); + + expect(exitCode).to.equal(ERROR_EXIT_CODE); + expect(pinPublicFileStub.called).to.be.false; + expect(consoleLogStub.calledWithMatch(sinon.match(/43-character/))).to.be.true; + }); + + it('surfaces the core-js "public drives only" error clearly for a private-drive target, not a raw throw', async () => { + const pinPublicFileStub = sinon.stub().rejects(new Error('Pinning is only supported for public drives')); + sinon.stub(cliIndex, 'cliArDriveFactory').returns({ pinPublicFile: pinPublicFileStub } as never); + sinon.stub(ParametersHelper.prototype, 'getRequiredWallet').resolves(fakeWallet); + const consoleLogStub = sinon.stub(console, 'log'); + + const descriptor = getPinFileDescriptor(); + const exitCode = await descriptor.action.trigger({ + parentFolderId: VALID_FOLDER_ID, + txId: VALID_TX_ID, + destFileName: DEST_FILE_NAME + }); + + expect(exitCode).to.equal(ERROR_EXIT_CODE); + expect(consoleLogStub.calledWithMatch(sinon.match(/Pinning is only supported for public drives/))).to.be.true; + }); +}); diff --git a/src/commands/pin_file.ts b/src/commands/pin_file.ts new file mode 100644 index 00000000..3cf2f29f --- /dev/null +++ b/src/commands/pin_file.ts @@ -0,0 +1,94 @@ +import { CLICommand, ParametersHelper } from '../CLICommand'; +import { + BoostParameter, + DestinationFileNameParameter, + DriveIdParameter, + DryRunParameter, + GatewayParameter, + ParentFolderIdParameter, + SeedPhraseParameter, + ShouldTurboParameter, + SkipParameter, + TransactionIdParameter, + TurboUrlParameter, + WalletFileParameter +} from '../parameter_declarations'; +import { cliArDriveFactory } from '..'; +import { SUCCESS_EXIT_CODE } from '../CLICommand/error_codes'; +import { CLIAction } from '../CLICommand/action'; +import { EID, skipOnConflicts, TxID, Wallet } from 'ardrive-core-js'; +import { getArweaveFromURL } from '../utils/get_arweave_for_url'; + +new CLICommand({ + name: 'pin-file', + parameters: [ + ParentFolderIdParameter, + { + name: TransactionIdParameter, + description: `the transaction ID of the EXISTING Arweave data transaction to pin into your drive +\t\t\t\t\t\t\t• The referenced data is reused as-is and is NOT re-uploaded (free)` + }, + { + name: DestinationFileNameParameter, + required: true, + description: `the name to give the newly pinned file entity within your ArDrive drive` + }, + { + name: DriveIdParameter, + required: false, + aliases: ['--drive-id'], + description: `(OPTIONAL) the ArFS entity ID of the destination drive +\t\t\t\t\t\t\t• When provided, must match the drive that actually owns --parent-folder-id, or the command will fail +\t\t\t\t\t\t\t• When omitted, the destination drive is resolved automatically from --parent-folder-id` + }, + SkipParameter, + BoostParameter, + DryRunParameter, + WalletFileParameter, + SeedPhraseParameter, + GatewayParameter, + ShouldTurboParameter, + TurboUrlParameter + ], + action: new CLIAction(async function action(options) { + const parameters = new ParametersHelper(options); + const arweave = getArweaveFromURL(parameters.getGateway()); + const useTurbo = !!parameters.getParameterValue(ShouldTurboParameter); + const turboUrl = parameters.getTurbo(); + + const wallet: Wallet = await parameters.getRequiredWallet(); + const arDrive = cliArDriveFactory({ + wallet, + feeMultiple: parameters.getOptionalBoostSetting(), + dryRun: parameters.isDryRun(), + arweave, + turboSettings: useTurbo ? { turboUrl } : undefined + }); + + const parentFolderId = parameters.getRequiredParameterValue(ParentFolderIdParameter, EID); + const dataTxId = parameters.getRequiredParameterValue(TransactionIdParameter, TxID); + const pinnedFileName = parameters.getRequiredParameterValue(DestinationFileNameParameter); + const driveId = parameters.getParameterValue(DriveIdParameter, EID); + + // Pinning only distinguishes "skip on name conflict" from the default (throw on name conflict) -- + // there is no replace/upsert/interactive-ask behavior for pinned files (mirrors ArDrive.pinPublicFile, + // which only special-cases skipOnConflicts and otherwise throws on any destination name collision). + const conflictResolution = parameters.getParameterValue(SkipParameter) ? skipOnConflicts : undefined; + + // NOTE: ArDrive.pinPublicFile throws a clear 'Pinning is only supported for public drives' error when + // --parent-folder-id resolves to a private drive. That Error propagates unmodified up through this + // action and is caught/printed cleanly by CLIAction (same handling every other command relies on) -- + // no raw/uncaught stack trace reaches the user. + const result = await arDrive.pinPublicFile({ + parentFolderId, + dataTxId, + pinnedFileName, + driveId, + conflictResolution + }); + + console.log(JSON.stringify(result, null, 4)); + + return SUCCESS_EXIT_CODE; + }) +}); From dad2712ea8c9544a5ea50fb2520ea620b42d2d15 Mon Sep 17 00:00:00 2001 From: vilenarios Date: Wed, 5 Aug 2026 02:08:41 -0400 Subject: [PATCH 3/5] feat(snapshot): add create-snapshot command [core-js 4.3.0] Adds a genuine (never-shipped) create-snapshot command, adapted from the old WIP branch history to core-js 4.3.0's current APIs. A snapshot is a standalone Arweave data transaction tagged Entity-Type=snapshot/Drive-Id/Block-Start/ Block-End (per ArFS), whose JSON body indexes a drive's entity metadata history so it round-trips through core-js's own parseSnapshotData / snapshotEntityFromGQLNode / buildSnapshotQuery consumption path. - src/utils/snapshots/create_snapshot.ts: queries a drive's owner-scoped entity metadata tx history (excluding prior snapshot txs of the same drive), fetches each entity's metadata JSON, and builds the { txSnapshots: [...] } body core-js's parser expects. - src/commands/create_snapshot.ts: resolves --drive-id, builds the snapshot body + tags (reusing core-js's SnapshotTagName/SNAPSHOT_ENTITY_TYPE/ SNAPSHOT_CONTENT_TYPE constants), estimates AR cost via ARDataPriceNetworkEstimator, asserts wallet balance before posting, and posts via ArFSDAO.prepareArFSObjectTransaction + sendTransactionsAsChunks (AR path) or ArFSDAO.prepareArFSDataItem + Turbo.sendDataItem (--turbo path) -- the same primitives the rest of the CLI posts through. excludedTagNames: ['ArFS'] keeps the tx a bare tagged data tx (Content-Type + the snapshot tags only), not a wrapped ArFS file entity. --dry-run signs locally but skips the actual post; public drives only for now (private snapshots need base64(ciphertext) bodies -- follow-up). - Registered in src/commands/index.ts; README gets a "Creating a Snapshot" section + TOC entry + ASCII command-index entry. - Tests (src/commands/create_snapshot.test.ts, Node 18, sinon-mocked network/posting, never broadcasts): command discoverability; a valid run's posted tx round-trips through core-js's real parseSnapshotData and is tagged with its constants, excluding prior snapshots; --dry-run doesn't post (AR and Turbo); insufficient balance refuses to post; bad --drive-id and empty-history error cleanly. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01T1adK4kjWUY8PU5YBd1LcX --- README.md | 25 ++ src/commands/create_snapshot.test.ts | 325 +++++++++++++++++++++++++ src/commands/create_snapshot.ts | 204 ++++++++++++++++ src/commands/index.ts | 1 + src/utils/snapshots/create_snapshot.ts | 131 ++++++++++ 5 files changed, 686 insertions(+) create mode 100644 src/commands/create_snapshot.test.ts create mode 100644 src/commands/create_snapshot.ts create mode 100644 src/utils/snapshots/create_snapshot.ts diff --git a/README.md b/README.md index 10d414c5..f72c6499 100644 --- a/README.md +++ b/README.md @@ -140,6 +140,7 @@ ardrive upload-file --wallet-file /path/to/my/wallet.json --parent-folder-id "f0 20. [Uploading Files with Custom MetaData](#uploading-files-with-custom-metadata) 21. [Applying Unique Custom MetaData During Bulk Workflows](#applying-unique-custom-metadata-during-bulk-workflows) 22. [Pinning a File](#pinning-a-file) + 23. [Creating a Snapshot](#creating-a-snapshot) 8. [Other Utility Operations](#other-utility-operations) 1. [Monitoring Transactions](#monitoring-transactions) 2. [Dealing With Network Congestion](#dealing-with-network-congestion) @@ -1355,6 +1356,29 @@ ardrive pin-file --parent-folder-id "a2c8a0cb-0ca7-4dbb-8bf8-93f75f308e63" --dri Like other write commands, `pin-file` supports `--dry-run`, `--boost`, `--turbo`/`--turbo-url`, and `--gateway`. See `ardrive pin-file --help` for the full flag list. +### Creating a Snapshot + +A **snapshot** is a single Arweave transaction, tagged `Entity-Type: snapshot`, `Drive-Id`, `Block-Start`, and `Block-End`, whose body is a JSON index of every ArFS entity metadata transaction (drive, folder, and file revisions) mined for that drive across the block range it covers. It exists purely as a read-path optimization: a client that wants to list a drive's full entity history can read the snapshot's JSON body directly instead of paginating through and re-fetching every individual metadata transaction the drive has ever produced. `create-snapshot` builds this snapshot for you and posts it to Arweave. + +Some important things to know: + +- **Costs to post, like any other data transaction.** For a drive with a long entity history the snapshot body can be large, so posting it is not free -- `create-snapshot` estimates the cost up front, asserts your wallet can cover it, and prints the cost before sending. +- **When to use it.** Snapshotting is most useful for drives with a large number of files/folders/revisions, where clients that support snapshot-accelerated listing would otherwise have to replay a long transaction history on every listing. It's a maintenance operation you run occasionally (e.g. periodically, or before publishing a drive expected to see heavy read traffic) -- not something every drive needs. +- **Public drives only (for now).** Private drive snapshots are not yet supported. +- **Idempotent-ish, not automatic.** Each run creates a NEW snapshot transaction covering the drive's entity history at that point in time; it does not update or replace a previous snapshot. + +```shell +ardrive create-snapshot --drive-id "bc9af866-6421-40f1-ac89-202bddb5c487" -w "/path/to/wallet" +``` + +Use `--dry-run` to see the block range, entity count, byte size, and estimated cost without posting anything: + +```shell +ardrive create-snapshot --drive-id "bc9af866-6421-40f1-ac89-202bddb5c487" -w "/path/to/wallet" --dry-run +``` + +Like other write commands, `create-snapshot` supports `--boost`, `--turbo`/`--turbo-url`, and `--gateway`. See `ardrive create-snapshot --help` for the full flag list. + ## Other Utility Operations ### Monitoring Transactions @@ -1524,6 +1548,7 @@ create-folder upload-file create-manifest pin-file +create-snapshot move-file move-folder diff --git a/src/commands/create_snapshot.test.ts b/src/commands/create_snapshot.test.ts new file mode 100644 index 00000000..5e36c74a --- /dev/null +++ b/src/commands/create_snapshot.test.ts @@ -0,0 +1,325 @@ +import { expect } from 'chai'; +import sinon from 'sinon'; +import * as fs from 'fs'; +import { CLICommand, CommandDescriptor } from '../CLICommand/cli_command'; +import { ParametersHelper } from '../CLICommand/parameters_helper'; +import { ERROR_EXIT_CODE, SUCCESS_EXIT_CODE } from '../CLICommand/error_codes'; +import { + ArFSDAO, + GatewayAPI, + GQLEdgeInterface, + GQLNodeInterface, + JWKWallet, + parseSnapshotData, + SNAPSHOT_CONTENT_TYPE, + SnapshotTagName, + WalletDAO, + Winston +} from 'ardrive-core-js'; +import { ARDataPriceNetworkEstimator } from 'ardrive-core-js/lib/pricing/ar_data_price_network_estimator'; +import { Turbo } from 'ardrive-core-js/lib/arfs/turbo'; + +// Importing the real command module registers 'create-snapshot' (with the actual commander program +// singleton) as a side effect -- mirrors how the CLI itself discovers commands via `./commands`. +import './create_snapshot'; + +const VALID_DRIVE_ID = 'bc9af866-6421-40f1-ac89-202bddb5c487'; +const DRIVE_METADATA_TX_ID = 'a'.repeat(43); +const FILE_METADATA_TX_ID = 'b'.repeat(43); +const PRIOR_SNAPSHOT_TX_ID = 'c'.repeat(43); + +// A long-established, funds-free test fixture already used by this repo's other test suites +// (see src/CLICommand/parameters_helper.test.ts) to exercise REAL wallet signing offline. +// Using a real JWKWallet here (rather than a `{ getAddress: async () => ... }` stub, as pin-file's +// tests use) is required because this command signs a real Arweave transaction / data item locally +// -- exactly what we want to assert against (tags, round-trippable body), never a network post. +const testWallet = new JWKWallet(JSON.parse(fs.readFileSync('./test_wallet.json', { encoding: 'utf8' }))); + +function makeNode( + overrides: Partial & Pick +): GQLNodeInterface { + return { + anchor: '', + signature: '', + recipient: '', + owner: { address: 'fake-owner-address', key: '' }, + fee: { winston: '0', ar: '0' }, + quantity: { winston: '0', ar: '0' }, + data: { size: 0, type: 'application/json' }, + parent: { id: '' }, + ...overrides + }; +} + +function getCreateSnapshotDescriptor(): CommandDescriptor { + const descriptor = CLICommand.getAllCommandDescriptors().find((cmd) => cmd.name === 'create-snapshot'); + if (!descriptor) { + throw new Error(`'create-snapshot' command was not registered`); + } + return descriptor; +} + +/** + * Stubs the GQL query + tx-data-fetch legs of `constructSnapshotData` with a fixed, deterministic + * drive history: one drive-metadata tx, one file-metadata tx, and a PRIOR snapshot tx (which must + * be excluded from the new snapshot's body). Never touches the network. + */ +function stubDriveHistory() { + const driveMetadata = { name: 'My Drive', rootFolderId: VALID_DRIVE_ID }; + const fileMetadata = { name: 'hello.txt', size: 5, dataTxId: 'd'.repeat(43), dataContentType: 'text/plain' }; + + const driveNode = makeNode({ + id: DRIVE_METADATA_TX_ID, + tags: [ + { name: 'Entity-Type', value: 'drive' }, + { name: SnapshotTagName.driveId, value: VALID_DRIVE_ID }, + { name: 'Content-Type', value: 'application/json' } + ], + block: { id: 'block-a', timestamp: 1000, height: 100, previous: '' } + }); + const fileNode = makeNode({ + id: FILE_METADATA_TX_ID, + tags: [ + { name: 'Entity-Type', value: 'file' }, + { name: SnapshotTagName.driveId, value: VALID_DRIVE_ID }, + { name: 'Content-Type', value: 'application/json' } + ], + block: { id: 'block-b', timestamp: 2000, height: 150, previous: '' } + }); + // A previous snapshot of the SAME drive -- carries the Drive-Id tag too (per REQUIRED_SNAPSHOT_TAG_NAMES), + // so it would be picked up by a naive owner+Drive-Id query. It must never be indexed into a new snapshot. + const priorSnapshotNode = makeNode({ + id: PRIOR_SNAPSHOT_TX_ID, + tags: [ + { name: SnapshotTagName.entityType, value: 'snapshot' }, + { name: SnapshotTagName.driveId, value: VALID_DRIVE_ID }, + { name: SnapshotTagName.blockStart, value: '1' }, + { name: SnapshotTagName.blockEnd, value: '99' }, + { name: SnapshotTagName.contentType, value: SNAPSHOT_CONTENT_TYPE } + ], + block: { id: 'block-0', timestamp: 500, height: 99, previous: '' } + }); + + const edges: GQLEdgeInterface[] = [ + { cursor: 'cursor-1', node: fileNode }, + { cursor: 'cursor-2', node: driveNode }, + { cursor: 'cursor-3', node: priorSnapshotNode } + ]; + + const gqlRequestStub = sinon + .stub(GatewayAPI.prototype, 'gqlRequest') + .resolves({ pageInfo: { hasNextPage: false }, edges }); + + const metadataByTxId: Record = { + [DRIVE_METADATA_TX_ID]: Buffer.from(JSON.stringify(driveMetadata)), + [FILE_METADATA_TX_ID]: Buffer.from(JSON.stringify(fileMetadata)) + }; + const getTxDataStub = sinon.stub(GatewayAPI.prototype, 'getTxData').callsFake(async (txId) => { + const data = metadataByTxId[`${txId}`]; + if (!data) { + throw new Error(`Test setup error: unexpected getTxData call for ${txId}`); + } + return data; + }); + + return { gqlRequestStub, getTxDataStub, driveMetadata, fileMetadata }; +} + +describe('create-snapshot command', () => { + afterEach(() => { + sinon.restore(); + }); + + it('is discoverable via the command registry with the expected params', () => { + const descriptor = getCreateSnapshotDescriptor(); + const parameterNames = descriptor.parameters.map((param) => (typeof param === 'string' ? param : param.name)); + + expect(parameterNames).to.include.members([ + 'driveId', + 'boost', + 'dryRun', + 'walletFile', + 'seedPhrase', + 'gateway', + 'turbo', + 'turboUrl' + ]); + }); + + describe('the AR (layer 1) path', () => { + beforeEach(() => { + sinon.stub(ParametersHelper.prototype, 'getRequiredWallet').resolves(testWallet); + sinon.stub(WalletDAO.prototype, 'walletHasBalance').resolves(true); + sinon + .stub(ARDataPriceNetworkEstimator.prototype, 'getBaseWinstonPriceForByteCount') + .resolves(new Winston('100')); + }); + + it('builds a snapshot body that round-trips through parseSnapshotData, tags the tx with the core-js snapshot constants, excludes prior snapshots, and posts it', async () => { + const { driveMetadata, fileMetadata } = stubDriveHistory(); + const sendTransactionsAsChunksStub = sinon.stub(ArFSDAO.prototype, 'sendTransactionsAsChunks').resolves(); + const consoleLogStub = sinon.stub(console, 'log'); + + const descriptor = getCreateSnapshotDescriptor(); + const exitCode = await descriptor.action.trigger({ driveId: VALID_DRIVE_ID }); + + expect(exitCode).to.equal(SUCCESS_EXIT_CODE); + expect(sendTransactionsAsChunksStub.calledOnce).to.be.true; + + const [postedTransactions] = sendTransactionsAsChunksStub.firstCall.args; + expect(postedTransactions).to.have.lengthOf(1); + const transaction = postedTransactions[0]; + + // Tag correctness: exactly the ArFS snapshot tag set (core-js's own constants), decoded back + // from the real, locally-signed Arweave transaction. + const decodedTags = transaction.tags.map((tag: { get: (f: string, o: unknown) => string }) => ({ + name: tag.get('name', { decode: true, string: true }), + value: tag.get('value', { decode: true, string: true }) + })); + const tagValue = (name: string) => decodedTags.find((t: { name: string }) => t.name === name)?.value; + + expect(tagValue(SnapshotTagName.entityType)).to.equal('snapshot'); + expect(tagValue(SnapshotTagName.driveId)).to.equal(VALID_DRIVE_ID); + expect(tagValue(SnapshotTagName.blockStart)).to.equal('100'); + expect(tagValue(SnapshotTagName.blockEnd)).to.equal('150'); + expect(tagValue(SnapshotTagName.contentType)).to.equal(SNAPSHOT_CONTENT_TYPE); + + // Round-trip: the exact body this command posted must be readable by core-js's OWN parser. + const parsed = parseSnapshotData(Buffer.from(transaction.data)); + expect(parsed.txSnapshots).to.have.lengthOf(2); // excludes the prior snapshot tx + + const byId = (id: string) => parsed.txSnapshots.find((tx) => tx.gqlNode.id === id); + expect(byId(DRIVE_METADATA_TX_ID)?.jsonMetadata).to.equal(JSON.stringify(driveMetadata)); + expect(byId(FILE_METADATA_TX_ID)?.jsonMetadata).to.equal(JSON.stringify(fileMetadata)); + expect(byId(PRIOR_SNAPSHOT_TX_ID)).to.be.undefined; + + // Printed result surfaces the same facts + const printedJson = consoleLogStub + .getCalls() + .map((call) => call.args[0]) + .join('\n'); + expect(printedJson).to.include('"entityCount": 2'); + expect(printedJson).to.include('"posted": true'); + expect(printedJson).to.include('"dryRun": false'); + }); + + it('--dry-run signs the transaction locally but does NOT post it', async () => { + stubDriveHistory(); + const sendTransactionsAsChunksStub = sinon.stub(ArFSDAO.prototype, 'sendTransactionsAsChunks').resolves(); + const consoleLogStub = sinon.stub(console, 'log'); + + const descriptor = getCreateSnapshotDescriptor(); + const exitCode = await descriptor.action.trigger({ driveId: VALID_DRIVE_ID, dryRun: true }); + + expect(exitCode).to.equal(SUCCESS_EXIT_CODE); + expect(sendTransactionsAsChunksStub.called).to.be.false; + + const printedJson = consoleLogStub + .getCalls() + .map((call) => call.args[0]) + .join('\n'); + expect(printedJson).to.include('"posted": false'); + expect(printedJson).to.include('"dryRun": true'); + expect(printedJson).to.match(/"snapshotId": "[\w-]{43}"/); // still resolves a real (unposted) tx id + }); + + it('refuses to post when the wallet balance cannot cover the estimated cost, without ever posting', async () => { + stubDriveHistory(); + (WalletDAO.prototype.walletHasBalance as sinon.SinonStub).resolves(false); + const sendTransactionsAsChunksStub = sinon.stub(ArFSDAO.prototype, 'sendTransactionsAsChunks').resolves(); + const consoleLogStub = sinon.stub(console, 'log'); + + const descriptor = getCreateSnapshotDescriptor(); + const exitCode = await descriptor.action.trigger({ driveId: VALID_DRIVE_ID }); + + expect(exitCode).to.equal(ERROR_EXIT_CODE); + expect(sendTransactionsAsChunksStub.called).to.be.false; + expect(consoleLogStub.calledWithMatch(sinon.match(/Insufficient wallet balance/))).to.be.true; + }); + + it('errors cleanly, without querying or posting, when --drive-id is not a valid entity ID', async () => { + const gqlRequestStub = sinon.stub(GatewayAPI.prototype, 'gqlRequest'); + const sendTransactionsAsChunksStub = sinon.stub(ArFSDAO.prototype, 'sendTransactionsAsChunks').resolves(); + const consoleLogStub = sinon.stub(console, 'log'); + + const descriptor = getCreateSnapshotDescriptor(); + const exitCode = await descriptor.action.trigger({ driveId: 'not-a-drive-id' }); + + expect(exitCode).to.equal(ERROR_EXIT_CODE); + expect(gqlRequestStub.called).to.be.false; + expect(sendTransactionsAsChunksStub.called).to.be.false; + expect(consoleLogStub.calledWithMatch(sinon.match(/Invalid entity ID/))).to.be.true; + }); + + it('errors cleanly when the drive has no entity history to snapshot', async () => { + sinon.stub(GatewayAPI.prototype, 'gqlRequest').resolves({ pageInfo: { hasNextPage: false }, edges: [] }); + const sendTransactionsAsChunksStub = sinon.stub(ArFSDAO.prototype, 'sendTransactionsAsChunks').resolves(); + const consoleLogStub = sinon.stub(console, 'log'); + + const descriptor = getCreateSnapshotDescriptor(); + const exitCode = await descriptor.action.trigger({ driveId: VALID_DRIVE_ID }); + + expect(exitCode).to.equal(ERROR_EXIT_CODE); + expect(sendTransactionsAsChunksStub.called).to.be.false; + expect(consoleLogStub.calledWithMatch(sinon.match(/nothing to snapshot yet/))).to.be.true; + }); + }); + + describe('the --turbo path', () => { + beforeEach(() => { + sinon.stub(ParametersHelper.prototype, 'getRequiredWallet').resolves(testWallet); + }); + + it('prepares a data item tagged with the core-js snapshot constants and posts it via Turbo', async () => { + stubDriveHistory(); + const sendDataItemStub = sinon.stub(Turbo.prototype, 'sendDataItem').resolves({ + id: 'turbo-response-id', + owner: 'owner', + dataCaches: [], + fastFinalityIndexes: [], + winc: '0' + }); + const consoleLogStub = sinon.stub(console, 'log'); + + const descriptor = getCreateSnapshotDescriptor(); + const exitCode = await descriptor.action.trigger({ driveId: VALID_DRIVE_ID, turbo: true }); + + expect(exitCode).to.equal(SUCCESS_EXIT_CODE); + expect(sendDataItemStub.calledOnce).to.be.true; + + const [postedDataItem] = sendDataItemStub.firstCall.args; + const tagValue = (name: string) => + postedDataItem.tags.find((t: { name: string }) => t.name === name)?.value; + + expect(tagValue(SnapshotTagName.entityType)).to.equal('snapshot'); + expect(tagValue(SnapshotTagName.driveId)).to.equal(VALID_DRIVE_ID); + expect(tagValue(SnapshotTagName.blockStart)).to.equal('100'); + expect(tagValue(SnapshotTagName.blockEnd)).to.equal('150'); + + const printedJson = consoleLogStub + .getCalls() + .map((call) => call.args[0]) + .join('\n'); + expect(printedJson).to.include('"posted": true'); + }); + + it('--dry-run does NOT post the data item to Turbo', async () => { + stubDriveHistory(); + const sendDataItemStub = sinon.stub(Turbo.prototype, 'sendDataItem').resolves(); + const consoleLogStub = sinon.stub(console, 'log'); + + const descriptor = getCreateSnapshotDescriptor(); + const exitCode = await descriptor.action.trigger({ driveId: VALID_DRIVE_ID, turbo: true, dryRun: true }); + + expect(exitCode).to.equal(SUCCESS_EXIT_CODE); + expect(sendDataItemStub.called).to.be.false; + + const printedJson = consoleLogStub + .getCalls() + .map((call) => call.args[0]) + .join('\n'); + expect(printedJson).to.include('"posted": false'); + expect(printedJson).to.include('"dryRun": true'); + }); + }); +}); diff --git a/src/commands/create_snapshot.ts b/src/commands/create_snapshot.ts new file mode 100644 index 00000000..24d739c7 --- /dev/null +++ b/src/commands/create_snapshot.ts @@ -0,0 +1,204 @@ +import { CLICommand, ParametersHelper } from '../CLICommand'; +import { + BoostParameter, + DriveIdParameter, + DryRunParameter, + GatewayParameter, + SeedPhraseParameter, + ShouldTurboParameter, + TurboUrlParameter, + WalletFileParameter +} from '../parameter_declarations'; +import { SUCCESS_EXIT_CODE } from '../CLICommand/error_codes'; +import { CLIAction } from '../CLICommand/action'; +import { + AR, + ArFSDAO, + ArFSPublicFileDataPrototype, + ArFSPublicFileDataTransactionData, + ByteCount, + EID, + GatewayAPI, + GatewayOracle, + gatewayUrlForArweave, + RewardSettings, + SNAPSHOT_CONTENT_TYPE, + SNAPSHOT_ENTITY_TYPE, + SnapshotTagName, + Wallet, + WalletDAO +} from 'ardrive-core-js'; +import { ArFSTagSettings } from 'ardrive-core-js/lib/arfs/arfs_tag_settings'; +// Not part of the public `exports.ts` surface (same as ArFSTagSettings above) -- deep-imported +// directly from their source modules, mirroring the existing pattern in `src/index.ts`. +import { ARDataPriceNetworkEstimator } from 'ardrive-core-js/lib/pricing/ar_data_price_network_estimator'; +import { Turbo } from 'ardrive-core-js/lib/arfs/turbo'; +import { CLI_APP_NAME, CLI_APP_VERSION } from '..'; +import { getArweaveFromURL } from '../utils/get_arweave_for_url'; +import { constructSnapshotData, snapshotDataToBuffer } from '../utils/snapshots/create_snapshot'; + +new CLICommand({ + name: 'create-snapshot', + parameters: [ + { + name: DriveIdParameter, + required: true, + description: `the ArFS entity ID of the PUBLIC drive to snapshot +\t\t\t\t\t\t\t• Private drive snapshots are not yet supported` + }, + BoostParameter, + DryRunParameter, + WalletFileParameter, + SeedPhraseParameter, + GatewayParameter, + ShouldTurboParameter, + TurboUrlParameter + ], + action: new CLIAction(async function action(options) { + const parameters = new ParametersHelper(options); + + const driveId = parameters.getRequiredParameterValue(DriveIdParameter, EID); + const dryRun = parameters.isDryRun(); + const boost = parameters.getOptionalBoostSetting(); + const useTurbo = !!parameters.getParameterValue(ShouldTurboParameter); + const turboUrl = parameters.getTurbo(); + + const arweave = getArweaveFromURL(parameters.getGateway()); + const wallet: Wallet = await parameters.getRequiredWallet(); + const owner = await wallet.getAddress(); + + // A single GatewayAPI instance is shared by the entity-history query AND the DAO used to + // post the resulting snapshot transaction, so both honor the same --gateway selection. + const gatewayApi = new GatewayAPI({ gatewayUrl: gatewayUrlForArweave(arweave) }); + + // 1. Gather the drive's entity metadata history and build the snapshot body. The body shape + // (`{ txSnapshots: [...] }`) is dictated by core-js's own `parseSnapshotData`, so a snapshot + // written here round-trips through core-js's snapshot-accelerated listing path. + const { data, blockStart, blockEnd, entityCount } = await constructSnapshotData({ + owner, + driveId, + gatewayApi + }); + const snapshotBody = snapshotDataToBuffer(data); + + console.error( + `Snapshotting ${entityCount} entity revision(s) of drive '${driveId}' spanning blocks ${blockStart}-${blockEnd} (${snapshotBody.byteLength} bytes)...` + ); + + // A snapshot is a standalone data transaction tagged Entity-Type/Drive-Id/Block-Start/Block-End + // (per ArFS), NOT a child file entity -- so it is built as a bare ArFSPublicFileDataPrototype + // (Content-Type + these custom tags only) rather than going through the normal file-upload path, + // which would additionally attach File-Id/Parent-Folder-Id/Name ArFS entity tags. + const objectData = new ArFSPublicFileDataTransactionData(snapshotBody); + const dataPrototype = new ArFSPublicFileDataPrototype(objectData, SNAPSHOT_CONTENT_TYPE, { + [SnapshotTagName.entityType]: SNAPSHOT_ENTITY_TYPE, + [SnapshotTagName.driveId]: `${driveId}`, + [SnapshotTagName.blockStart]: `${blockStart}`, + [SnapshotTagName.blockEnd]: `${blockEnd}` + }); + + const arFSTagSettings = new ArFSTagSettings({ appName: CLI_APP_NAME, appVersion: CLI_APP_VERSION }); + const arFsDao = new ArFSDAO( + wallet, + arweave, + dryRun, + CLI_APP_NAME, + CLI_APP_VERSION, + arFSTagSettings, + undefined, + gatewayApi + ); + + if (useTurbo) { + // excludedTagNames: ['ArFS'] routes through the raw file-DATA tag assembly (Content-Type + + // our custom tags + App-Name/App-Version), skipping the ArFS entity-metadata tag set. + const dataItem = await arFsDao.prepareArFSDataItem({ + objectMetaData: dataPrototype, + excludedTagNames: ['ArFS'] + }); + + let turboResult: Awaited> | undefined; + if (!dryRun) { + const turbo = new Turbo({ turboUploadUrl: turboUrl, isDryRun: dryRun }); + console.error(`Uploading snapshot data item '${dataItem.id}' to Turbo...`); + turboResult = await turbo.sendDataItem(dataItem); + } else { + console.error(`DRY RUN: would upload snapshot data item '${dataItem.id}' to Turbo`); + } + + console.log( + JSON.stringify( + { + snapshotId: dataItem.id, + driveId: `${driveId}`, + blockStart, + blockEnd, + entityCount, + dataSize: snapshotBody.byteLength, + turbo: turboResult, + dryRun, + posted: !dryRun + }, + null, + 4 + ) + ); + + return SUCCESS_EXIT_CODE; + } + + // AR (layer 1) path -- estimate cost, then require the wallet actually have the funds before + // signing/sending, mirroring the balance assertion every other costed command relies on. + const priceEstimator = new ARDataPriceNetworkEstimator(new GatewayOracle(gatewayUrlForArweave(arweave))); + const baseReward = await priceEstimator.getBaseWinstonPriceForByteCount(new ByteCount(snapshotBody.byteLength)); + const boostedReward = boost?.wouldBoostReward() ? boost.boostedWinstonReward(baseReward) : baseReward; + + console.error(`Estimated cost: ${new AR(boostedReward).toString()} AR (${boostedReward.toString()} Winston)`); + + const walletDAO = new WalletDAO(arweave, CLI_APP_NAME, CLI_APP_VERSION); + const hasBalance = await walletDAO.walletHasBalance(wallet, boostedReward); + if (!hasBalance) { + throw new Error( + `Insufficient wallet balance to post this snapshot. Estimated cost is ${new AR( + boostedReward + ).toString()} AR, sent from wallet address '${owner}'.` + ); + } + + const rewardSettings: RewardSettings = boost + ? { reward: baseReward, feeMultiple: boost } + : { reward: baseReward }; + const transaction = await arFsDao.prepareArFSObjectTransaction({ + objectMetaData: dataPrototype, + rewardSettings, + excludedTagNames: ['ArFS'] + }); + + if (!dryRun) { + console.error(`Posting snapshot transaction '${transaction.id}'...`); + await arFsDao.sendTransactionsAsChunks([transaction]); + } else { + console.error(`DRY RUN: would post snapshot transaction '${transaction.id}'`); + } + + console.log( + JSON.stringify( + { + snapshotId: transaction.id, + driveId: `${driveId}`, + blockStart, + blockEnd, + entityCount, + dataSize: snapshotBody.byteLength, + reward: boostedReward.toString(), + dryRun, + posted: !dryRun + }, + null, + 4 + ) + ); + + return SUCCESS_EXIT_CODE; + }) +}); diff --git a/src/commands/index.ts b/src/commands/index.ts index 92728b18..14e48174 100644 --- a/src/commands/index.ts +++ b/src/commands/index.ts @@ -3,6 +3,7 @@ import './base_reward'; import './create_drive'; import './create_folder'; import './create_manifest'; +import './create_snapshot'; import './create_tx'; import './download_drive'; import './download_file'; diff --git a/src/utils/snapshots/create_snapshot.ts b/src/utils/snapshots/create_snapshot.ts new file mode 100644 index 00000000..b72bb080 --- /dev/null +++ b/src/utils/snapshots/create_snapshot.ts @@ -0,0 +1,131 @@ +import { + ArweaveAddress, + DESCENDING_ORDER, + DriveID, + GatewayAPI, + GQLEdgeInterface, + SnapshotData, + SnapshotTagName, + SNAPSHOT_ENTITY_TYPE, + TxSnapshot, + TxID, + buildQuery +} from 'ardrive-core-js'; + +export interface ConstructSnapshotDataParams { + /** The drive owner -- snapshots (and the entities they index) are only ever queried owner-scoped */ + owner: ArweaveAddress; + driveId: DriveID; + gatewayApi: GatewayAPI; +} + +export interface ConstructSnapshotDataResult { + /** The parsed snapshot body -- shape MUST stay in lockstep with core-js's own `parseSnapshotData` */ + data: SnapshotData; + /** Lowest block height among the indexed entity transactions (inclusive) */ + blockStart: number; + /** Highest block height among the indexed entity transactions (inclusive) */ + blockEnd: number; + /** Number of entity transaction revisions captured in this snapshot */ + entityCount: number; +} + +/** + * Queries every ArFS entity metadata transaction (drive/folder/file, all revisions) belonging to + * the given drive, owner-scoped and sorted newest-first -- the same owner + `Drive-Id` scoping + * core-js's own `buildSnapshotQuery` uses for reading snapshots back. + * + * Transactions that are themselves PRIOR SNAPSHOTS of this drive are excluded client-side (a + * snapshot must index entity revisions, never another snapshot -- there is no server-side "NOT" + * tag filter available to exclude them at the GQL layer). + */ +export async function queryAllDriveEntityTxs({ + owner, + driveId, + gatewayApi +}: ConstructSnapshotDataParams): Promise { + const edges: GQLEdgeInterface[] = []; + let cursor: string | undefined = undefined; + let hasNextPage = true; + + while (hasNextPage) { + const query = buildQuery({ + owner, + sort: DESCENDING_ORDER, + tags: [{ name: SnapshotTagName.driveId, value: `${driveId}` }], + cursor + }); + + const { edges: pageEdges, pageInfo } = await gatewayApi.gqlRequest(query); + + for (const edge of pageEdges) { + const entityTypeTag = edge.node.tags.find((tag) => tag.name === SnapshotTagName.entityType); + if (entityTypeTag?.value === SNAPSHOT_ENTITY_TYPE) { + // A previous snapshot of this drive -- never index a snapshot inside another snapshot + continue; + } + edges.push(edge); + } + + hasNextPage = pageInfo.hasNextPage; + cursor = pageEdges.length ? pageEdges[pageEdges.length - 1].cursor : undefined; + if (!cursor) { + break; + } + } + + return edges; +} + +/** + * Builds the JSON-serializable {@link SnapshotData} body for a drive snapshot: the drive's full + * entity metadata history (every drive/folder/file revision), paired with the block-height range + * it spans. + * + * The output shape (`{ txSnapshots: [{ gqlNode, jsonMetadata }, ...] }`) is dictated by core-js's + * own `parseSnapshotData`/`SnapshotData` types -- this is what makes a snapshot written by this + * command consumable by core-js's own snapshot-accelerated listing path. + */ +export async function constructSnapshotData({ + owner, + driveId, + gatewayApi +}: ConstructSnapshotDataParams): Promise { + const edges = await queryAllDriveEntityTxs({ owner, driveId, gatewayApi }); + + if (edges.length === 0) { + throw new Error( + `No entity metadata transactions were found for drive '${driveId}' owned by '${owner}' -- there is nothing to snapshot yet.` + ); + } + + const txSnapshots: TxSnapshot[] = await Promise.all( + edges.map( + async ({ node }): Promise => { + const jsonMetadata = (await gatewayApi.getTxData(TxID(node.id))).toString(); + return { gqlNode: node, jsonMetadata }; + } + ) + ); + + const blockHeights = edges + .map((edge) => edge.node.block?.height) + .filter((height): height is number => typeof height === 'number'); + + if (blockHeights.length === 0) { + throw new Error( + `None of the ${edges.length} entity transaction(s) found for drive '${driveId}' have been mined yet -- wait for them to confirm before creating a snapshot.` + ); + } + + const data: SnapshotData = { txSnapshots }; + const blockStart = Math.min(...blockHeights); + const blockEnd = Math.max(...blockHeights); + + return { data, blockStart, blockEnd, entityCount: txSnapshots.length }; +} + +/** Serializes a {@link SnapshotData} body exactly as core-js's `parseSnapshotData` expects to read it back */ +export function snapshotDataToBuffer(data: SnapshotData): Buffer { + return Buffer.from(JSON.stringify(data)); +} From 7cbbdfe6565ba06e7fa139dd0092c619011d7437 Mon Sep 17 00:00:00 2001 From: vilenarios Date: Wed, 5 Aug 2026 02:21:10 -0400 Subject: [PATCH 4/5] feat(hide): add hide-file/unhide-file/hide-folder/unhide-folder commands [core-js 4.3.0] Adds four new CLI commands (hide-file, unhide-file, hide-folder, unhide-folder) modeled directly on rename-file/rename-folder, calling core-js 4.3.0's hide/unhidePublic*/Private* methods instead of rename. Each supports both public and private drives (via --drive-key or wallet+password), plus the usual --boost/--turbo/--dry-run/--gateway flags. Registered in src/commands/index.ts and documented in README.md. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01T1adK4kjWUY8PU5YBd1LcX --- README.md | 35 ++++++ src/commands/hide_file.ts | 69 +++++++++++ src/commands/hide_folder.ts | 70 +++++++++++ src/commands/hide_unhide.test.ts | 205 +++++++++++++++++++++++++++++++ src/commands/index.ts | 4 + src/commands/unhide_file.ts | 69 +++++++++++ src/commands/unhide_folder.ts | 70 +++++++++++ 7 files changed, 522 insertions(+) create mode 100644 src/commands/hide_file.ts create mode 100644 src/commands/hide_folder.ts create mode 100644 src/commands/hide_unhide.test.ts create mode 100644 src/commands/unhide_file.ts create mode 100644 src/commands/unhide_folder.ts diff --git a/README.md b/README.md index f72c6499..8c6eea1f 100644 --- a/README.md +++ b/README.md @@ -141,6 +141,7 @@ ardrive upload-file --wallet-file /path/to/my/wallet.json --parent-folder-id "f0 21. [Applying Unique Custom MetaData During Bulk Workflows](#applying-unique-custom-metadata-during-bulk-workflows) 22. [Pinning a File](#pinning-a-file) 23. [Creating a Snapshot](#creating-a-snapshot) + 24. [Hiding and Unhiding a File or Folder](#hiding-and-unhiding-a-file-or-folder) 8. [Other Utility Operations](#other-utility-operations) 1. [Monitoring Transactions](#monitoring-transactions) 2. [Dealing With Network Congestion](#dealing-with-network-congestion) @@ -1379,6 +1380,36 @@ ardrive create-snapshot --drive-id "bc9af866-6421-40f1-ac89-202bddb5c487" -w "/p Like other write commands, `create-snapshot` supports `--boost`, `--turbo`/`--turbo-url`, and `--gateway`. See `ardrive create-snapshot --help` for the full flag list. +### Hiding and Unhiding a File or Folder + +The `hide-file`, `unhide-file`, `hide-folder`, and `unhide-folder` commands let you toggle whether a file or folder entity is flagged as hidden, without touching its data or metadata otherwise. Hiding writes a new metadata revision with an `isHidden` flag set to `true`; clients that respect this flag (e.g. the ArDrive web/desktop apps) omit the entity from their normal drive listings, while it remains fully present on-chain. Unhiding writes another revision flipping the flag back to `false`. + +Some important things to know: + +- **Reversible.** Hiding never deletes or re-uploads data -- it's a metadata-only toggle, and `unhide-file`/`unhide-folder` fully restores visibility at any time. +- **Works on both public and private entities.** Pass `--drive-key` or (`--wallet-file`/`--seed-phrase` plus `--unsafe-drive-password`) to target a private file/folder; omit them to target a public one, exactly like `rename-file`/`rename-folder`. +- **Costs a small metadata fee.** Like a rename, hiding/unhiding writes a new metadata revision to Arweave, so it isn't free, but it's the same tiny metadata-only cost as any other rename/move operation -- no file data is re-uploaded. +- **Not recursive.** Hiding a folder flags only that folder's own metadata; it does not walk its contents and hide child files/folders individually. + +```shell +# Hide a public file +ardrive hide-file --file-id "290a3f9a-37b2-4f0f-a899-6fac983833b3" -w "/path/to/wallet.json" + +# Unhide it again +ardrive unhide-file --file-id "290a3f9a-37b2-4f0f-a899-6fac983833b3" -w "/path/to/wallet.json" + +# Hide a private file (drive key derived from wallet + password) +ardrive hide-file --file-id "290a3f9a-37b2-4f0f-a899-6fac983833b3" -w "/path/to/wallet.json" --unsafe-drive-password "p4ssw0rd" + +# Hide a public folder +ardrive hide-folder --folder-id "568d5eba-dbf3-4a49-8129-1c58f7fd35bc" -w "/path/to/wallet.json" + +# Unhide a private folder using a raw drive key +ardrive unhide-folder --folder-id "568d5eba-dbf3-4a49-8129-1c58f7fd35bc" -w "/path/to/wallet.json" --drive-key "base64EncodedDriveKey" +``` + +Like other write commands, `hide-file`/`unhide-file`/`hide-folder`/`unhide-folder` support `--dry-run`, `--boost`, `--turbo`/`--turbo-url`, and `--gateway`. See `ardrive hide-file --help` (and `unhide-file`/`hide-folder`/`unhide-folder --help`) for the full flag list. + ## Other Utility Operations ### Monitoring Transactions @@ -1549,6 +1580,10 @@ upload-file create-manifest pin-file create-snapshot +hide-file +unhide-file +hide-folder +unhide-folder move-file move-folder diff --git a/src/commands/hide_file.ts b/src/commands/hide_file.ts new file mode 100644 index 00000000..fa97d17b --- /dev/null +++ b/src/commands/hide_file.ts @@ -0,0 +1,69 @@ +import { CLICommand, ParametersHelper } from '../CLICommand'; +import { + BoostParameter, + DryRunParameter, + FileIdParameter, + DrivePrivacyParameters, + GatewayParameter, + ShouldTurboParameter, + TurboUrlParameter +} from '../parameter_declarations'; +import { cliArDriveFactory } from '..'; +import { SUCCESS_EXIT_CODE } from '../CLICommand/error_codes'; +import { CLIAction } from '../CLICommand/action'; +import { EID, Wallet } from 'ardrive-core-js'; +import { getArweaveFromURL } from '../utils/get_arweave_for_url'; + +new CLICommand({ + name: 'hide-file', + parameters: [ + FileIdParameter, + BoostParameter, + ShouldTurboParameter, + TurboUrlParameter, + DryRunParameter, + ...DrivePrivacyParameters, + GatewayParameter + ], + action: new CLIAction(async function action(options) { + const parameters = new ParametersHelper(options); + const arweave = getArweaveFromURL(parameters.getGateway()); + + const dryRun = parameters.isDryRun(); + const fileId = parameters.getRequiredParameterValue(FileIdParameter, EID); + const shouldUseTurbo = !!parameters.getParameterValue(ShouldTurboParameter); + const turboUrl = parameters.getTurbo(); + + const wallet: Wallet = await parameters.getRequiredWallet(); + const ardrive = cliArDriveFactory({ + wallet: wallet, + feeMultiple: parameters.getOptionalBoostSetting(), + dryRun, + turboSettings: shouldUseTurbo ? { turboUrl } : undefined, + arweave + }); + + const result = await (async function () { + if (await parameters.getIsPrivate()) { + const driveId = await ardrive.getDriveIdForFileId(fileId); + const driveKey = await parameters.getDriveKey({ + driveId, + arDrive: ardrive, + owner: await wallet.getAddress() + }); + + return ardrive.hidePrivateFile({ + fileId, + driveKey + }); + } else { + return ardrive.hidePublicFile({ + fileId + }); + } + })(); + + console.log(JSON.stringify(result, null, 4)); + return SUCCESS_EXIT_CODE; + }) +}); diff --git a/src/commands/hide_folder.ts b/src/commands/hide_folder.ts new file mode 100644 index 00000000..08cc95ed --- /dev/null +++ b/src/commands/hide_folder.ts @@ -0,0 +1,70 @@ +import { CLICommand, ParametersHelper } from '../CLICommand'; +import { + BoostParameter, + DryRunParameter, + FolderIdParameter, + DrivePrivacyParameters, + GatewayParameter, + ShouldTurboParameter, + TurboUrlParameter +} from '../parameter_declarations'; +import { cliArDriveFactory } from '..'; +import { SUCCESS_EXIT_CODE } from '../CLICommand/error_codes'; +import { CLIAction } from '../CLICommand/action'; +import { EID, Wallet } from 'ardrive-core-js'; +import { getArweaveFromURL } from '../utils/get_arweave_for_url'; + +new CLICommand({ + name: 'hide-folder', + parameters: [ + FolderIdParameter, + BoostParameter, + DryRunParameter, + ShouldTurboParameter, + TurboUrlParameter, + ...DrivePrivacyParameters, + GatewayParameter + ], + action: new CLIAction(async function action(options) { + const parameters = new ParametersHelper(options); + + const arweave = getArweaveFromURL(parameters.getGateway()); + + const dryRun = parameters.isDryRun(); + const folderId = parameters.getRequiredParameterValue(FolderIdParameter, EID); + const shouldUseTurbo = !!parameters.getParameterValue(ShouldTurboParameter); + const turboUrl = parameters.getTurbo(); + + const wallet: Wallet = await parameters.getRequiredWallet(); + const ardrive = cliArDriveFactory({ + wallet: wallet, + feeMultiple: parameters.getOptionalBoostSetting(), + turboSettings: shouldUseTurbo ? { turboUrl } : undefined, + dryRun, + arweave + }); + + const result = await (async function () { + if (await parameters.getIsPrivate()) { + const driveId = await ardrive.getDriveIdForFolderId(folderId); + const driveKey = await parameters.getDriveKey({ + driveId, + arDrive: ardrive, + owner: await wallet.getAddress() + }); + + return ardrive.hidePrivateFolder({ + folderId, + driveKey + }); + } else { + return ardrive.hidePublicFolder({ + folderId + }); + } + })(); + + console.log(JSON.stringify(result, null, 4)); + return SUCCESS_EXIT_CODE; + }) +}); diff --git a/src/commands/hide_unhide.test.ts b/src/commands/hide_unhide.test.ts new file mode 100644 index 00000000..091ed5b4 --- /dev/null +++ b/src/commands/hide_unhide.test.ts @@ -0,0 +1,205 @@ +import { expect } from 'chai'; +import sinon from 'sinon'; +import { CLICommand, CommandDescriptor } from '../CLICommand/cli_command'; +import { ParametersHelper } from '../CLICommand/parameters_helper'; +import { SUCCESS_EXIT_CODE } from '../CLICommand/error_codes'; +import * as cliIndex from '..'; +import { DriveKey, Wallet } from 'ardrive-core-js'; + +// Importing the real command modules registers 'hide-file'/'unhide-file'/'hide-folder'/'unhide-folder' +// (with the actual commander program singleton) as a side effect -- mirrors how the CLI itself +// discovers commands via `./commands`. +import './hide_file'; +import './unhide_file'; +import './hide_folder'; +import './unhide_folder'; + +const VALID_FILE_ID = '290a3f9a-37b2-4f0f-a899-6fac983833b3'; +const VALID_FOLDER_ID = '568d5eba-dbf3-4a49-8129-1c58f7fd35bc'; +const VALID_DRIVE_ID = 'bc9af866-6421-40f1-ac89-202bddb5c487'; +const FAKE_DRIVE_KEY = ({} as unknown) as DriveKey; + +const fakeWallet = ({ getAddress: async () => 'fake-address' } as unknown) as Wallet; + +function getDescriptor(name: string): CommandDescriptor { + const descriptor = CLICommand.getAllCommandDescriptors().find((cmd) => cmd.name === name); + if (!descriptor) { + throw new Error(`'${name}' command was not registered`); + } + return descriptor; +} + +interface HideCommandSpec { + commandName: string; + idParamName: 'fileId' | 'folderId'; + validId: string; + publicMethod: string; + privateMethod: string; + getDriveIdMethod: string; +} + +const specs: HideCommandSpec[] = [ + { + commandName: 'hide-file', + idParamName: 'fileId', + validId: VALID_FILE_ID, + publicMethod: 'hidePublicFile', + privateMethod: 'hidePrivateFile', + getDriveIdMethod: 'getDriveIdForFileId' + }, + { + commandName: 'unhide-file', + idParamName: 'fileId', + validId: VALID_FILE_ID, + publicMethod: 'unhidePublicFile', + privateMethod: 'unhidePrivateFile', + getDriveIdMethod: 'getDriveIdForFileId' + }, + { + commandName: 'hide-folder', + idParamName: 'folderId', + validId: VALID_FOLDER_ID, + publicMethod: 'hidePublicFolder', + privateMethod: 'hidePrivateFolder', + getDriveIdMethod: 'getDriveIdForFolderId' + }, + { + commandName: 'unhide-folder', + idParamName: 'folderId', + validId: VALID_FOLDER_ID, + publicMethod: 'unhidePublicFolder', + privateMethod: 'unhidePrivateFolder', + getDriveIdMethod: 'getDriveIdForFolderId' + } +]; + +describe('hide-file / unhide-file / hide-folder / unhide-folder commands', () => { + afterEach(() => { + sinon.restore(); + }); + + for (const spec of specs) { + const { commandName, idParamName, validId, publicMethod, privateMethod, getDriveIdMethod } = spec; + + describe(`${commandName} command`, () => { + it('is discoverable via the command registry with the expected params', () => { + const descriptor = getDescriptor(commandName); + const parameterNames = descriptor.parameters.map((param) => + typeof param === 'string' ? param : param.name + ); + + expect(parameterNames).to.include.members([ + idParamName, + 'boost', + 'dryRun', + 'turbo', + 'turboUrl', + 'driveKey', + 'walletFile', + 'seedPhrase', + 'private', + 'unsafeDrivePassword', + 'gateway' + ]); + + // hide/unhide never take a "new name" -- unlike rename-file/rename-folder + expect(parameterNames).to.not.include.members(['fileName', 'folderName']); + }); + + it(`(public) calls ArDrive.${publicMethod} with just the ${idParamName} and prints the result, without touching drive-key resolution`, async () => { + const fakeResult = { created: [{ type: 'file', entityId: validId }], tips: [], fees: {} }; + const publicMethodStub = sinon.stub().resolves(fakeResult); + const getDriveIdStub = sinon.stub(); + const getDriveKeyStub = sinon.stub(ParametersHelper.prototype, 'getDriveKey'); + + sinon.stub(cliIndex, 'cliArDriveFactory').returns({ + [publicMethod]: publicMethodStub, + [getDriveIdMethod]: getDriveIdStub + } as never); + sinon.stub(ParametersHelper.prototype, 'getRequiredWallet').resolves(fakeWallet); + const consoleLogStub = sinon.stub(console, 'log'); + + const descriptor = getDescriptor(commandName); + const exitCode = await descriptor.action.trigger({ [idParamName]: validId }); + + expect(exitCode).to.equal(SUCCESS_EXIT_CODE); + expect(publicMethodStub.calledOnce).to.be.true; + + const callArgs = publicMethodStub.firstCall.args[0]; + expect(String(callArgs[idParamName])).to.equal(validId); + expect(Object.keys(callArgs)).to.deep.equal([idParamName]); + + // Public path never resolves a drive ID or drive key + expect(getDriveIdStub.called).to.be.false; + expect(getDriveKeyStub.called).to.be.false; + + expect(consoleLogStub.calledWithMatch(sinon.match(JSON.stringify(fakeResult, null, 4)))).to.be.true; + }); + + it(`(private, --drive-key) calls ArDrive.${privateMethod} with the ${idParamName} and resolved driveKey, after resolving the driveId via ${getDriveIdMethod}`, async () => { + const fakeResult = { created: [{ type: 'file', entityId: validId }], tips: [], fees: {} }; + const privateMethodStub = sinon.stub().resolves(fakeResult); + const getDriveIdStub = sinon.stub().resolves(VALID_DRIVE_ID); + const publicMethodStub = sinon.stub(); + + sinon.stub(cliIndex, 'cliArDriveFactory').returns({ + [privateMethod]: privateMethodStub, + [publicMethod]: publicMethodStub, + [getDriveIdMethod]: getDriveIdStub + } as never); + sinon.stub(ParametersHelper.prototype, 'getRequiredWallet').resolves(fakeWallet); + const getDriveKeyStub = sinon.stub(ParametersHelper.prototype, 'getDriveKey').resolves(FAKE_DRIVE_KEY); + const consoleLogStub = sinon.stub(console, 'log'); + + const descriptor = getDescriptor(commandName); + // A non-empty --drive-key is what flips ParametersHelper#getIsPrivate() to true; the + // actual key material resolution is fully stubbed out via getDriveKey above, so no real + // drive-key derivation or network call happens here. + const exitCode = await descriptor.action.trigger({ + [idParamName]: validId, + driveKey: 'ZmFrZS1kcml2ZS1rZXk=' + }); + + expect(exitCode).to.equal(SUCCESS_EXIT_CODE); + expect(publicMethodStub.called).to.be.false; + expect(getDriveIdStub.calledOnceWith(sinon.match((id) => String(id) === validId))).to.be.true; + expect(getDriveKeyStub.calledOnce).to.be.true; + expect(getDriveKeyStub.firstCall.args[0]).to.deep.include({ driveId: VALID_DRIVE_ID }); + + expect(privateMethodStub.calledOnce).to.be.true; + const callArgs = privateMethodStub.firstCall.args[0]; + expect(String(callArgs[idParamName])).to.equal(validId); + expect(callArgs.driveKey).to.equal(FAKE_DRIVE_KEY); + expect(Object.keys(callArgs).sort()).to.deep.equal([idParamName, 'driveKey'].sort()); + + expect(consoleLogStub.calledWithMatch(sinon.match(JSON.stringify(fakeResult, null, 4)))).to.be.true; + }); + + it('(private, --unsafe-drive-password) also routes to the private method', async () => { + const fakeResult = { created: [], tips: [], fees: {} }; + const privateMethodStub = sinon.stub().resolves(fakeResult); + const publicMethodStub = sinon.stub(); + const getDriveIdStub = sinon.stub().resolves(VALID_DRIVE_ID); + + sinon.stub(cliIndex, 'cliArDriveFactory').returns({ + [privateMethod]: privateMethodStub, + [publicMethod]: publicMethodStub, + [getDriveIdMethod]: getDriveIdStub + } as never); + sinon.stub(ParametersHelper.prototype, 'getRequiredWallet').resolves(fakeWallet); + sinon.stub(ParametersHelper.prototype, 'getDriveKey').resolves(FAKE_DRIVE_KEY); + sinon.stub(console, 'log'); + + const descriptor = getDescriptor(commandName); + const exitCode = await descriptor.action.trigger({ + [idParamName]: validId, + unsafeDrivePassword: 'super-secret-password' + }); + + expect(exitCode).to.equal(SUCCESS_EXIT_CODE); + expect(publicMethodStub.called).to.be.false; + expect(privateMethodStub.calledOnce).to.be.true; + }); + }); + } +}); diff --git a/src/commands/index.ts b/src/commands/index.ts index 14e48174..48080120 100644 --- a/src/commands/index.ts +++ b/src/commands/index.ts @@ -18,6 +18,8 @@ import './get_balance'; import './get_drive_key'; import './get_file_key'; import './get_mempool'; +import './hide_file'; +import './hide_folder'; import './last_tx'; import './list_all_drives'; import './list_drive'; @@ -32,6 +34,8 @@ import './retry_tx'; import './send_ar'; import './send_tx'; import './tx_status'; +import './unhide_file'; +import './unhide_folder'; import './upload_file'; // Please keep this list in alphabetical order. Thank you <3 diff --git a/src/commands/unhide_file.ts b/src/commands/unhide_file.ts new file mode 100644 index 00000000..14bd9be1 --- /dev/null +++ b/src/commands/unhide_file.ts @@ -0,0 +1,69 @@ +import { CLICommand, ParametersHelper } from '../CLICommand'; +import { + BoostParameter, + DryRunParameter, + FileIdParameter, + DrivePrivacyParameters, + GatewayParameter, + ShouldTurboParameter, + TurboUrlParameter +} from '../parameter_declarations'; +import { cliArDriveFactory } from '..'; +import { SUCCESS_EXIT_CODE } from '../CLICommand/error_codes'; +import { CLIAction } from '../CLICommand/action'; +import { EID, Wallet } from 'ardrive-core-js'; +import { getArweaveFromURL } from '../utils/get_arweave_for_url'; + +new CLICommand({ + name: 'unhide-file', + parameters: [ + FileIdParameter, + BoostParameter, + ShouldTurboParameter, + TurboUrlParameter, + DryRunParameter, + ...DrivePrivacyParameters, + GatewayParameter + ], + action: new CLIAction(async function action(options) { + const parameters = new ParametersHelper(options); + const arweave = getArweaveFromURL(parameters.getGateway()); + + const dryRun = parameters.isDryRun(); + const fileId = parameters.getRequiredParameterValue(FileIdParameter, EID); + const shouldUseTurbo = !!parameters.getParameterValue(ShouldTurboParameter); + const turboUrl = parameters.getTurbo(); + + const wallet: Wallet = await parameters.getRequiredWallet(); + const ardrive = cliArDriveFactory({ + wallet: wallet, + feeMultiple: parameters.getOptionalBoostSetting(), + dryRun, + turboSettings: shouldUseTurbo ? { turboUrl } : undefined, + arweave + }); + + const result = await (async function () { + if (await parameters.getIsPrivate()) { + const driveId = await ardrive.getDriveIdForFileId(fileId); + const driveKey = await parameters.getDriveKey({ + driveId, + arDrive: ardrive, + owner: await wallet.getAddress() + }); + + return ardrive.unhidePrivateFile({ + fileId, + driveKey + }); + } else { + return ardrive.unhidePublicFile({ + fileId + }); + } + })(); + + console.log(JSON.stringify(result, null, 4)); + return SUCCESS_EXIT_CODE; + }) +}); diff --git a/src/commands/unhide_folder.ts b/src/commands/unhide_folder.ts new file mode 100644 index 00000000..ec56e09a --- /dev/null +++ b/src/commands/unhide_folder.ts @@ -0,0 +1,70 @@ +import { CLICommand, ParametersHelper } from '../CLICommand'; +import { + BoostParameter, + DryRunParameter, + FolderIdParameter, + DrivePrivacyParameters, + GatewayParameter, + ShouldTurboParameter, + TurboUrlParameter +} from '../parameter_declarations'; +import { cliArDriveFactory } from '..'; +import { SUCCESS_EXIT_CODE } from '../CLICommand/error_codes'; +import { CLIAction } from '../CLICommand/action'; +import { EID, Wallet } from 'ardrive-core-js'; +import { getArweaveFromURL } from '../utils/get_arweave_for_url'; + +new CLICommand({ + name: 'unhide-folder', + parameters: [ + FolderIdParameter, + BoostParameter, + DryRunParameter, + ShouldTurboParameter, + TurboUrlParameter, + ...DrivePrivacyParameters, + GatewayParameter + ], + action: new CLIAction(async function action(options) { + const parameters = new ParametersHelper(options); + + const arweave = getArweaveFromURL(parameters.getGateway()); + + const dryRun = parameters.isDryRun(); + const folderId = parameters.getRequiredParameterValue(FolderIdParameter, EID); + const shouldUseTurbo = !!parameters.getParameterValue(ShouldTurboParameter); + const turboUrl = parameters.getTurbo(); + + const wallet: Wallet = await parameters.getRequiredWallet(); + const ardrive = cliArDriveFactory({ + wallet: wallet, + feeMultiple: parameters.getOptionalBoostSetting(), + turboSettings: shouldUseTurbo ? { turboUrl } : undefined, + dryRun, + arweave + }); + + const result = await (async function () { + if (await parameters.getIsPrivate()) { + const driveId = await ardrive.getDriveIdForFolderId(folderId); + const driveKey = await parameters.getDriveKey({ + driveId, + arDrive: ardrive, + owner: await wallet.getAddress() + }); + + return ardrive.unhidePrivateFolder({ + folderId, + driveKey + }); + } else { + return ardrive.unhidePublicFolder({ + folderId + }); + } + })(); + + console.log(JSON.stringify(result, null, 4)); + return SUCCESS_EXIT_CODE; + }) +}); From 575858cb4907e5cb2785f5842090d50ee50ba89c Mon Sep 17 00:00:00 2001 From: vilenarios Date: Wed, 5 Aug 2026 02:38:17 -0400 Subject: [PATCH 5/5] fix(cli): address CodeRabbit review on #377 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - create-snapshot: only snapshot MINED revisions — filter out unmined (no-block-height) edges before building the body, block bounds, and count, so Block-Start/Block-End can't claim to cover state they don't (data-integrity). - create-snapshot: bound the entity-metadata fetch to a fixed worker pool (SNAPSHOT_TX_FETCH_CONCURRENCY=8) instead of an unbounded Promise.all, so a large drive can't open thousands of simultaneous gateway requests (stability). - pin-file: mark --tx-id (TransactionIdParameter) required in the declaration so a missing flag fails at parse + shows in --help, not only at action time. - README: correct pin-file prose flag name --data-tx-id -> --tx-id. - hide/unhide test: replace `.not.include.members([a,b])` (only asserts 'not ALL') with two single-member `.not.include` checks (asserts NEITHER present). - Add src/utils/snapshots/create_snapshot.test.ts: mined-filter exclusion, none-mined error, concurrency cap (<=8, still concurrent), input-order preserved. Node 18.17.0: typecheck 0, lint 0, test 119 passing (only the known env-only deriveIpfsCid network-timeout fails locally; passes in CI). Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01T1adK4kjWUY8PU5YBd1LcX --- README.md | 4 +- src/commands/hide_unhide.test.ts | 8 +- src/commands/pin_file.ts | 1 + src/utils/snapshots/create_snapshot.test.ts | 134 ++++++++++++++++++++ src/utils/snapshots/create_snapshot.ts | 65 +++++++--- 5 files changed, 192 insertions(+), 20 deletions(-) create mode 100644 src/utils/snapshots/create_snapshot.test.ts diff --git a/README.md b/README.md index 8c6eea1f..a1f107c6 100644 --- a/README.md +++ b/README.md @@ -1337,12 +1337,12 @@ done Pinning lets you reference an **existing** Arweave data transaction as a new file entity in one of your PUBLIC drives, without re-uploading any data. This is useful for adopting data that already lives permanently on Arweave (e.g. a transaction uploaded outside of ArDrive, or one belonging to someone else) into your drive's folder structure, so it shows up alongside your other files with its own name, metadata, and location. -Because a pinned file's metadata transaction only references the existing `--data-tx-id` (it doesn't touch the underlying data bytes), pinning a small file costs the same tiny metadata-only fee as any other file operation -- there is no data-upload cost, regardless of the size of the original file. +Because a pinned file's metadata transaction only references the existing `--tx-id` (it doesn't touch the underlying data bytes), pinning a small file costs the same tiny metadata-only fee as any other file operation -- there is no data-upload cost, regardless of the size of the original file. Some important constraints: - **Public drives only.** Pinning writes a plaintext ArFS metadata transaction that points at the referenced data. Private drives are not supported -- targeting a private `--parent-folder-id` fails with a clear error. -- **The referenced transaction is never re-uploaded or modified.** Only a new file metadata entity is created; `--data-tx-id` is reused as-is as the new file's data transaction. +- **The referenced transaction is never re-uploaded or modified.** Only a new file metadata entity is created; `--tx-id` is reused as-is as the new file's data transaction. - **Name conflicts throw by default.** If `--dest-file-name` already exists in the destination folder, the command fails unless `--skip` is provided, in which case the command exits successfully having made no changes. ```shell diff --git a/src/commands/hide_unhide.test.ts b/src/commands/hide_unhide.test.ts index 091ed5b4..23c4f65c 100644 --- a/src/commands/hide_unhide.test.ts +++ b/src/commands/hide_unhide.test.ts @@ -102,8 +102,12 @@ describe('hide-file / unhide-file / hide-folder / unhide-folder commands', () => 'gateway' ]); - // hide/unhide never take a "new name" -- unlike rename-file/rename-folder - expect(parameterNames).to.not.include.members(['fileName', 'folderName']); + // hide/unhide never take a "new name" -- unlike rename-file/rename-folder. Assert absence of + // EACH name param separately: `.not.include.members([a, b])` only asserts "not ALL of a and + // b", so it would still pass if one of them crept back in. Two single-member `.not.include` + // checks assert NEITHER is present. + expect(parameterNames).to.not.include('fileName'); + expect(parameterNames).to.not.include('folderName'); }); it(`(public) calls ArDrive.${publicMethod} with just the ${idParamName} and prints the result, without touching drive-key resolution`, async () => { diff --git a/src/commands/pin_file.ts b/src/commands/pin_file.ts index 3cf2f29f..f1dbd5ab 100644 --- a/src/commands/pin_file.ts +++ b/src/commands/pin_file.ts @@ -25,6 +25,7 @@ new CLICommand({ ParentFolderIdParameter, { name: TransactionIdParameter, + required: true, description: `the transaction ID of the EXISTING Arweave data transaction to pin into your drive \t\t\t\t\t\t\t• The referenced data is reused as-is and is NOT re-uploaded (free)` }, diff --git a/src/utils/snapshots/create_snapshot.test.ts b/src/utils/snapshots/create_snapshot.test.ts new file mode 100644 index 00000000..9ff9049b --- /dev/null +++ b/src/utils/snapshots/create_snapshot.test.ts @@ -0,0 +1,134 @@ +import { expect } from 'chai'; +import sinon from 'sinon'; +import { ArweaveAddress, EID, GatewayAPI, GQLEdgeInterface, GQLNodeInterface, SnapshotTagName } from 'ardrive-core-js'; +import { constructSnapshotData, SNAPSHOT_TX_FETCH_CONCURRENCY } from './create_snapshot'; + +const VALID_DRIVE_ID = 'bc9af866-6421-40f1-ac89-202bddb5c487'; +const OWNER = new ArweaveAddress('a'.repeat(43)); + +// A minimal file-entity GQL node. `height === undefined` models an UNMINED (pending) revision: +// no block, so `constructSnapshotData` must exclude it from the body, the block bounds, and the count. +function fileNode(id: string, height: number | undefined): GQLNodeInterface { + return { + id, + anchor: '', + signature: '', + recipient: '', + owner: { address: 'fake-owner', key: '' }, + fee: { winston: '0', ar: '0' }, + quantity: { winston: '0', ar: '0' }, + data: { size: 0, type: 'application/json' }, + tags: [ + { name: 'Entity-Type', value: 'file' }, + { name: SnapshotTagName.driveId, value: VALID_DRIVE_ID } + ], + block: + height === undefined + ? ((undefined as unknown) as GQLNodeInterface['block']) + : { id: `block-${height}`, timestamp: height, height, previous: '' }, + parent: { id: '' } + }; +} + +function stubGql(edges: GQLEdgeInterface[]): void { + sinon.stub(GatewayAPI.prototype, 'gqlRequest').resolves({ pageInfo: { hasNextPage: false }, edges }); +} + +function makeGatewayApi(): GatewayAPI { + return new GatewayAPI({ gatewayUrl: new URL('https://example.invalid') }); +} + +describe('constructSnapshotData (unit): mined filter + bounded fetch', () => { + afterEach(() => sinon.restore()); + + it('excludes unmined (no block height) revisions from the body, block bounds, and entity count, and never fetches their data', async () => { + const edges: GQLEdgeInterface[] = [ + { cursor: '1', node: fileNode('a'.repeat(43), 200) }, + { cursor: '2', node: fileNode('e'.repeat(43), undefined) } // unmined -- must be dropped + ]; + stubGql(edges); + const getTxDataStub = sinon.stub(GatewayAPI.prototype, 'getTxData').resolves(Buffer.from('{}')); + + const { data, blockStart, blockEnd, entityCount } = await constructSnapshotData({ + owner: OWNER, + driveId: EID(VALID_DRIVE_ID), + gatewayApi: makeGatewayApi() + }); + + expect(entityCount).to.equal(1); + expect(blockStart).to.equal(200); + expect(blockEnd).to.equal(200); + expect(data.txSnapshots).to.have.lengthOf(1); + expect(data.txSnapshots[0].gqlNode.id).to.equal('a'.repeat(43)); + expect(getTxDataStub.calledOnce).to.be.true; // only the mined tx was fetched + }); + + it('throws a "have been mined yet" error when the drive has entities but none are mined', async () => { + stubGql([{ cursor: '1', node: fileNode('a'.repeat(43), undefined) }]); + sinon.stub(GatewayAPI.prototype, 'getTxData').resolves(Buffer.from('{}')); + + let error: Error | undefined; + try { + await constructSnapshotData({ owner: OWNER, driveId: EID(VALID_DRIVE_ID), gatewayApi: makeGatewayApi() }); + } catch (e) { + error = e as Error; + } + expect(error?.message ?? '').to.match(/have been mined yet/); + }); + + it(`never exceeds SNAPSHOT_TX_FETCH_CONCURRENCY (${SNAPSHOT_TX_FETCH_CONCURRENCY}) concurrent getTxData requests, yet still runs them concurrently`, async () => { + const count = SNAPSHOT_TX_FETCH_CONCURRENCY * 3; + const edges: GQLEdgeInterface[] = Array.from({ length: count }, (_unused, i) => ({ + cursor: `c${i}`, + node: fileNode(`${i}`.padStart(43, '0'), 100 + i) + })); + stubGql(edges); + + let inFlight = 0; + let maxInFlight = 0; + sinon.stub(GatewayAPI.prototype, 'getTxData').callsFake(async () => { + inFlight += 1; + maxInFlight = Math.max(maxInFlight, inFlight); + await new Promise((resolve) => setTimeout(resolve, 5)); + inFlight -= 1; + return Buffer.from('{}'); + }); + + const { entityCount } = await constructSnapshotData({ + owner: OWNER, + driveId: EID(VALID_DRIVE_ID), + gatewayApi: makeGatewayApi() + }); + + expect(entityCount).to.equal(count); + expect(maxInFlight).to.be.at.most(SNAPSHOT_TX_FETCH_CONCURRENCY); // the cap holds + expect(maxInFlight).to.be.greaterThan(1); // ...but it is a real pool, not serial + }); + + it('preserves input order in the snapshot body even when fetches complete out of order', async () => { + const edges: GQLEdgeInterface[] = [ + { cursor: '1', node: fileNode('1'.repeat(43), 100) }, + { cursor: '2', node: fileNode('2'.repeat(43), 101) }, + { cursor: '3', node: fileNode('3'.repeat(43), 102) } + ]; + stubGql(edges); + // Resolve the FIRST edge slowest so completion order != input order. + sinon.stub(GatewayAPI.prototype, 'getTxData').callsFake(async (txId) => { + const id = `${txId}`; + await new Promise((resolve) => setTimeout(resolve, id.startsWith('1') ? 15 : 1)); + return Buffer.from(JSON.stringify({ id })); + }); + + const { data } = await constructSnapshotData({ + owner: OWNER, + driveId: EID(VALID_DRIVE_ID), + gatewayApi: makeGatewayApi() + }); + + expect(data.txSnapshots.map((t) => t.gqlNode.id)).to.deep.equal([ + '1'.repeat(43), + '2'.repeat(43), + '3'.repeat(43) + ]); + }); +}); diff --git a/src/utils/snapshots/create_snapshot.ts b/src/utils/snapshots/create_snapshot.ts index b72bb080..614110bc 100644 --- a/src/utils/snapshots/create_snapshot.ts +++ b/src/utils/snapshots/create_snapshot.ts @@ -78,9 +78,43 @@ export async function queryAllDriveEntityTxs({ } /** - * Builds the JSON-serializable {@link SnapshotData} body for a drive snapshot: the drive's full - * entity metadata history (every drive/folder/file revision), paired with the block-height range - * it spans. + * How many entity-metadata transactions to fetch from the gateway concurrently when building a + * snapshot body. A large drive can have thousands of entity revisions; fetching them all with an + * unbounded `Promise.all` opens one gateway request per revision at once, which can exhaust memory + * or sockets and trip gateway rate limits before the snapshot completes. A small fixed worker pool + * keeps snapshot creation bounded and gateway-friendly. + */ +export const SNAPSHOT_TX_FETCH_CONCURRENCY = 8; + +/** + * Fetches each edge's `jsonMetadata` with at most {@link SNAPSHOT_TX_FETCH_CONCURRENCY} requests in + * flight at once (a shared-cursor worker pool), preserving input order in the returned array. + */ +async function fetchTxSnapshotsBounded( + edges: GQLEdgeInterface[], + gatewayApi: GatewayAPI, + concurrency: number = SNAPSHOT_TX_FETCH_CONCURRENCY +): Promise { + const txSnapshots: TxSnapshot[] = new Array(edges.length); + let nextIndex = 0; + + async function worker(): Promise { + for (let i = nextIndex++; i < edges.length; i = nextIndex++) { + const { node } = edges[i]; + const jsonMetadata = (await gatewayApi.getTxData(TxID(node.id))).toString(); + txSnapshots[i] = { gqlNode: node, jsonMetadata }; + } + } + + const workerCount = Math.min(Math.max(concurrency, 1), edges.length); + await Promise.all(Array.from({ length: workerCount }, () => worker())); + return txSnapshots; +} + +/** + * Builds the JSON-serializable {@link SnapshotData} body for a drive snapshot: the drive's mined + * entity metadata history (every mined drive/folder/file revision), paired with the block-height + * range it spans. * * The output shape (`{ txSnapshots: [{ gqlNode, jsonMetadata }, ...] }`) is dictated by core-js's * own `parseSnapshotData`/`SnapshotData` types -- this is what makes a snapshot written by this @@ -99,25 +133,24 @@ export async function constructSnapshotData({ ); } - const txSnapshots: TxSnapshot[] = await Promise.all( - edges.map( - async ({ node }): Promise => { - const jsonMetadata = (await gatewayApi.getTxData(TxID(node.id))).toString(); - return { gqlNode: node, jsonMetadata }; - } - ) - ); - - const blockHeights = edges - .map((edge) => edge.node.block?.height) - .filter((height): height is number => typeof height === 'number'); + // Only MINED revisions belong in a snapshot. The snapshot's Block-Start/Block-End tags describe a + // closed block range; an unmined (pending, no block height) revision would put body content into + // the snapshot that those tags cannot represent. Filter first, then use the SAME mined set for the + // body, the block bounds, AND the entity count so all three stay mutually consistent. + const minedEdges = edges.filter((edge) => typeof edge.node.block?.height === 'number'); - if (blockHeights.length === 0) { + if (minedEdges.length === 0) { throw new Error( `None of the ${edges.length} entity transaction(s) found for drive '${driveId}' have been mined yet -- wait for them to confirm before creating a snapshot.` ); } + const txSnapshots = await fetchTxSnapshotsBounded(minedEdges, gatewayApi); + + const blockHeights = minedEdges + .map((edge) => edge.node.block?.height) + .filter((height): height is number => typeof height === 'number'); + const data: SnapshotData = { txSnapshots }; const blockStart = Math.min(...blockHeights); const blockEnd = Math.max(...blockHeights);