Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions lib/composer/composer/autoload_classmap.php
Original file line number Diff line number Diff line change
Expand Up @@ -596,6 +596,8 @@
'OCP\\Files\\ObjectStore\\IObjectStore' => $baseDir . '/lib/public/Files/ObjectStore/IObjectStore.php',
'OCP\\Files\\ObjectStore\\IObjectStoreMetaData' => $baseDir . '/lib/public/Files/ObjectStore/IObjectStoreMetaData.php',
'OCP\\Files\\ObjectStore\\IObjectStoreMultiPartUpload' => $baseDir . '/lib/public/Files/ObjectStore/IObjectStoreMultiPartUpload.php',
'OCP\\Files\\ObjectStore\\IObjectStorePresignedUpload' => $baseDir . '/lib/public/Files/ObjectStore/IObjectStorePresignedUpload.php',
'OCP\\Files\\ObjectStore\\IObjectStoreStorage' => $baseDir . '/lib/public/Files/ObjectStore/IObjectStoreStorage.php',
'OCP\\Files\\ReservedWordException' => $baseDir . '/lib/public/Files/ReservedWordException.php',
'OCP\\Files\\Search\\ISearchBinaryOperator' => $baseDir . '/lib/public/Files/Search/ISearchBinaryOperator.php',
'OCP\\Files\\Search\\ISearchComparison' => $baseDir . '/lib/public/Files/Search/ISearchComparison.php',
Expand Down
2 changes: 2 additions & 0 deletions lib/composer/composer/autoload_static.php
Original file line number Diff line number Diff line change
Expand Up @@ -637,6 +637,8 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2
'OCP\\Files\\ObjectStore\\IObjectStore' => __DIR__ . '/../../..' . '/lib/public/Files/ObjectStore/IObjectStore.php',
'OCP\\Files\\ObjectStore\\IObjectStoreMetaData' => __DIR__ . '/../../..' . '/lib/public/Files/ObjectStore/IObjectStoreMetaData.php',
'OCP\\Files\\ObjectStore\\IObjectStoreMultiPartUpload' => __DIR__ . '/../../..' . '/lib/public/Files/ObjectStore/IObjectStoreMultiPartUpload.php',
'OCP\\Files\\ObjectStore\\IObjectStorePresignedUpload' => __DIR__ . '/../../..' . '/lib/public/Files/ObjectStore/IObjectStorePresignedUpload.php',
'OCP\\Files\\ObjectStore\\IObjectStoreStorage' => __DIR__ . '/../../..' . '/lib/public/Files/ObjectStore/IObjectStoreStorage.php',
'OCP\\Files\\ReservedWordException' => __DIR__ . '/../../..' . '/lib/public/Files/ReservedWordException.php',
'OCP\\Files\\Search\\ISearchBinaryOperator' => __DIR__ . '/../../..' . '/lib/public/Files/Search/ISearchBinaryOperator.php',
'OCP\\Files\\Search\\ISearchComparison' => __DIR__ . '/../../..' . '/lib/public/Files/Search/ISearchComparison.php',
Expand Down
5 changes: 4 additions & 1 deletion lib/private/Files/ObjectStore/ObjectStoreStorage.php
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
use OCP\Files\ObjectStore\IObjectStore;
use OCP\Files\ObjectStore\IObjectStoreMetaData;
use OCP\Files\ObjectStore\IObjectStoreMultiPartUpload;
use OCP\Files\ObjectStore\IObjectStoreStorage;
use OCP\Files\Storage\IChunkedFileWrite;
use OCP\Files\Storage\IStorage;
use OCP\IDBConnection;
Expand All @@ -38,7 +39,7 @@
use Override;
use Psr\Log\LoggerInterface;

class ObjectStoreStorage extends Common implements IChunkedFileWrite {
class ObjectStoreStorage extends Common implements IChunkedFileWrite, IObjectStoreStorage {
use CopyDirectory;

protected IObjectStore $objectStore;
Expand Down Expand Up @@ -270,6 +271,7 @@ public function getPermissions(string $path): int {
*
* @return string the unified resource name used to identify the object
*/
#[\Override]
public function getURN(int $fileId): string {
return $this->objectPrefix . $fileId;
}
Expand Down Expand Up @@ -604,6 +606,7 @@ public function writeStream(string $path, $stream, ?int $size = null): int {
return $totalWritten;
}

#[\Override]
public function getObjectStore(): IObjectStore {
return $this->objectStore;
}
Expand Down
92 changes: 91 additions & 1 deletion lib/private/Files/ObjectStore/S3.php
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,13 @@
use OCP\Files\ObjectStore\IObjectStore;
use OCP\Files\ObjectStore\IObjectStoreMetaData;
use OCP\Files\ObjectStore\IObjectStoreMultiPartUpload;
use OCP\Files\ObjectStore\IObjectStorePresignedUpload;

class S3 implements IObjectStore, IObjectStoreMultiPartUpload, IObjectStorePresignedUpload, IObjectStoreMetaData {
/** S3's limits for one part of a multipart upload. */
private const MAX_PART_NUMBER = 10000;
private const MAX_PART_SIZE = 5 * 1024 * 1024 * 1024;

class S3 implements IObjectStore, IObjectStoreMultiPartUpload, IObjectStoreMetaData {
use S3ConnectionTrait;
use S3ObjectTrait;

Expand Down Expand Up @@ -102,6 +107,91 @@ public function abortMultipartUpload($urn, $uploadId): void {
]);
}

#[\Override]
public function canPreSignUploads(): bool {
// A customer-provided key would have to be sent with every part, and
// legacy (V2) signatures can't sign a checksum.
return $this->getSSECKey() === null && empty($this->params['legacy_auth']);
}

#[\Override]
public function initiatePreSignedMultipartUpload(string $urn, ?string $mimetype = null): string {
if (!$this->canPreSignUploads()) {
throw new Exception('This object store configuration can\'t take presigned uploads');
}
$args = [
'Bucket' => $this->bucket,
'Key' => $urn,
'ChecksumAlgorithm' => 'SHA256',
'StorageClass' => $this->storageClass,
] + $this->getServerSideEncryptionParameters();
if ($mimetype !== null && $mimetype !== '') {
$args['ContentType'] = $mimetype;
}
$uploadId = $this->getConnection()->createMultipartUpload($args)->get('UploadId');
if (!is_string($uploadId) || $uploadId === '') {
throw new Exception('No upload id returned');
}
return $uploadId;
}

#[\Override]
public function preSignUploadPart(string $urn, string $uploadId, int $partNumber, int $size, string $sha256, \DateTimeInterface $expiration): array {
if ($partNumber < 1 || $partNumber > self::MAX_PART_NUMBER) {
throw new \InvalidArgumentException('The part number must be between 1 and ' . self::MAX_PART_NUMBER);
}
if ($size < 0 || $size > self::MAX_PART_SIZE) {
throw new \InvalidArgumentException('A part has at most ' . self::MAX_PART_SIZE . ' bytes');
}
if (preg_match('/^[0-9a-f]{64}$/', $sha256) !== 1) {
throw new \InvalidArgumentException('The SHA-256 must be lowercase hex');
}
if (!$this->canPreSignUploads()) {
throw new Exception('This object store configuration can\'t take presigned uploads');
}
$command = $this->getConnection()->getCommand('UploadPart', [
'Bucket' => $this->bucket,
'Key' => $urn,
'UploadId' => $uploadId,
'PartNumber' => $partNumber,
'ContentLength' => $size,
'ChecksumSHA256' => base64_encode((string)hex2bin($sha256)),
]);
$request = $this->getConnection()->createPresignedRequest($command, $expiration);

// The client has to send exactly the headers the signature covers;
// its HTTP library sets the host from the URL.
parse_str($request->getUri()->getQuery(), $query);
$signed = is_string($query['X-Amz-SignedHeaders'] ?? null) ? explode(';', $query['X-Amz-SignedHeaders']) : [];
$headers = [];
foreach ($signed as $name) {
if ($name !== 'host' && $request->hasHeader($name)) {
$headers[$name] = $request->getHeaderLine($name);
}
}
return ['method' => 'PUT', 'url' => (string)$request->getUri(), 'headers' => $headers];
}

#[\Override]
public function getPreSignedUploadParts(string $urn, string $uploadId): array {
$parts = [];
foreach ($this->getMultipartUploads($urn, $uploadId) as $part) {
$checksum = $part['ChecksumSHA256'] ?? null;
$sha256 = null;
if (is_string($checksum) && $checksum !== '') {
$raw = base64_decode($checksum, true);
$sha256 = ($raw !== false && strlen($raw) === 32) ? bin2hex($raw) : null;
}
$parts[] = [
'number' => (int)$part['PartNumber'],
'size' => (int)$part['Size'],
'sha256' => $sha256,
];
}
usort($parts, static fn (array $a, array $b): int => $a['number'] <=> $b['number']);
return $parts;
}

private function parseS3Metadata(array $metadata): array {
$result = [];
foreach ($metadata as $key => $value) {
Expand Down
72 changes: 72 additions & 0 deletions lib/public/Files/ObjectStore/IObjectStorePresignedUpload.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
<?php

declare(strict_types=1);

/**
* SPDX-FileCopyrightText: 2026 Nextcloud GmbH and Nextcloud contributors
* SPDX-License-Identifier: AGPL-3.0-or-later
*/

namespace OCP\Files\ObjectStore;

/**
* An object store whose multipart uploads a client can feed directly: the
* server starts the upload and hands out a presigned request per part, the
* client sends the part's bytes straight to the store, and the server lists
* and completes the upload with {@see IObjectStoreMultiPartUpload}. The
* bytes never pass through the server, and the store's credentials never
* leave it.
*
* Every part carries the SHA-256 it was presigned for; the store refuses a
* part whose bytes don't match it.
*
* @since 36.0.0
*/
interface IObjectStorePresignedUpload extends IObjectStoreMultiPartUpload {
/**
* Whether presigned uploads are possible with this store's
* configuration. They are not when a request would need a secret the
* client must not have, such as a customer-provided encryption key, or
* when the store's signature scheme can't sign a checksum.
*
* @since 36.0.0
*/
public function canPreSignUploads(): bool;

/**
* Starts a multipart upload of $urn whose parts carry SHA-256 checksums.
* The object appears only when the upload is completed.
*
* @return string the upload's ID
* @throws \Exception when the store refuses
* @since 36.0.0
*/
public function initiatePreSignedMultipartUpload(string $urn, ?string $mimetype = null): string;

/**
* A presigned request that uploads part $partNumber (1 to 10,000) of the
* upload $uploadId of $urn: exactly $size bytes whose SHA-256 is
* $sha256. The client sends the request with the given method and
* headers, unchanged, and the bytes as its body.
*
* The URL is a secret until it expires: whoever has it can upload this
* one part.
*
* @param string $sha256 lowercase hex SHA-256 of the part's bytes
* @return array{method: 'PUT', url: string, headers: array<string, string>}
* @throws \InvalidArgumentException for a part number, size or hash out of range
* @throws \Exception when the request can't be signed
* @since 36.0.0
*/
public function preSignUploadPart(string $urn, string $uploadId, int $partNumber, int $size, string $sha256, \DateTimeInterface $expiration): array;

/**
* The parts the store holds for the upload, ordered by number, with the
* SHA-256 it checked (null when the store kept none).
*
* @return list<array{number: int, size: int, sha256: ?string}>
* @throws \Exception when the upload doesn't exist
* @since 36.0.0
*/
public function getPreSignedUploadParts(string $urn, string $uploadId): array;
}
35 changes: 35 additions & 0 deletions lib/public/Files/ObjectStore/IObjectStoreStorage.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
<?php

declare(strict_types=1);

/**
* SPDX-FileCopyrightText: 2026 Nextcloud GmbH and Nextcloud contributors
* SPDX-License-Identifier: AGPL-3.0-or-later
*/

namespace OCP\Files\ObjectStore;

use OCP\Files\Storage\IStorage;

/**
* A storage that keeps its files' content in an object store, one object
* per file. Storage wrappers pass these calls on; check with
* {@see IStorage::instanceOfStorage()}.
*
* @since 36.0.0
*/
interface IObjectStoreStorage extends IStorage {
/**
* The object store that holds the content.
*
* @since 36.0.0
*/
public function getObjectStore(): IObjectStore;

/**
* The name of the object that holds the content of the file $fileId.
*
* @since 36.0.0
*/
public function getURN(int $fileId): string;
}
96 changes: 96 additions & 0 deletions tests/lib/Files/ObjectStore/S3PresignedUploadTest.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,96 @@
<?php

declare(strict_types=1);

/**
* SPDX-FileCopyrightText: 2026 Nextcloud GmbH and Nextcloud contributors
* SPDX-License-Identifier: AGPL-3.0-or-later
*/

namespace Test\Files\ObjectStore;

use OC\Files\ObjectStore\ObjectStoreStorage;
use OC\Files\ObjectStore\S3;
use OC\Files\ObjectStore\StorageObjectStore;
use OC\Files\Storage\Temporary;
use OCP\Files\ObjectStore\IObjectStoreStorage;
use Test\TestCase;

/**
* Signing presigned part uploads happens locally, so it needs no S3 server.
*/
class S3PresignedUploadTest extends TestCase {
private const SHA256 = 'a29968fad2e782aa9f2040a35f05adb97ed8979eb1f572c8c8ea78637e275f3c';

private function store(array $extra = []): S3 {
return new S3($extra + [
'bucket' => 'nextcloud',
'hostname' => 's3.example.com',
'port' => 443,
'region' => 'eu-central-1',
'key' => 'AKIDEXAMPLE',
'secret' => 'wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY',
'use_path_style' => true,
'verify_bucket_exists' => false,
]);
}

public function testPresignsOnePartWithItsChecksum(): void {
$expires = new \DateTimeImmutable('+15 minutes');
$request = $this->store()->preSignUploadPart('urn:oid:42', 'upload-1', 3, 5242880, self::SHA256, $expires);

$this->assertSame('PUT', $request['method']);
$this->assertSame(['x-amz-checksum-sha256' => base64_encode(hex2bin(self::SHA256))], $request['headers']);
$url = parse_url($request['url']);
$this->assertSame('https', $url['scheme']);
$this->assertSame('s3.example.com', $url['host']);
$this->assertSame('/nextcloud/urn%3Aoid%3A42', $url['path']);
parse_str($url['query'], $query);
$this->assertSame('upload-1', $query['uploadId']);
$this->assertSame('3', $query['partNumber']);
$this->assertSame('900', $query['X-Amz-Expires']);
$this->assertContains('x-amz-checksum-sha256', explode(';', $query['X-Amz-SignedHeaders']));
$this->assertMatchesRegularExpression('/^[0-9a-f]{64}$/', $query['X-Amz-Signature']);
$this->assertStringNotContainsString('wJalrXUtnFEMI', $request['url']);
}

public static function invalidParts(): array {
return [
'part 0' => [0, 1, self::SHA256],
'part 10001' => [10001, 1, self::SHA256],
'negative size' => [1, -1, self::SHA256],
'larger than 5 GiB' => [1, 5 * 1024 * 1024 * 1024 + 1, self::SHA256],
'uppercase hash' => [1, 1, strtoupper(self::SHA256)],
'short hash' => [1, 1, 'abc'],
];
}

#[\PHPUnit\Framework\Attributes\DataProvider('invalidParts')]
public function testRefusesPartsOutOfRange(int $number, int $size, string $sha256): void {
$this->expectException(\InvalidArgumentException::class);
$this->store()->preSignUploadPart('urn:oid:42', 'upload-1', $number, $size, $sha256, new \DateTimeImmutable('+15 minutes'));
}

public function testCanPreSignOnlyWithoutCustomerKeysAndWithV4Signatures(): void {
$this->assertTrue($this->store()->canPreSignUploads());
$this->assertTrue($this->store(['sse_kms_enabled' => true])->canPreSignUploads());
$this->assertFalse($this->store(['sse_c_key' => base64_encode(random_bytes(32))])->canPreSignUploads());
$this->assertFalse($this->store(['legacy_auth' => true])->canPreSignUploads());
}

public function testRefusesToPresignWithACustomerKey(): void {
$this->expectException(\Exception::class);
$this->store(['sse_c_key' => base64_encode(random_bytes(32))])
->preSignUploadPart('urn:oid:42', 'upload-1', 1, 1, self::SHA256, new \DateTimeImmutable('+15 minutes'));
}

public function testObjectStoreStorageNamesItsObjects(): void {
$objectStore = new StorageObjectStore(new Temporary([]));
$storage = new ObjectStoreStorage(['objectstore' => $objectStore, 'objectPrefix' => 'urn:test:']);

$this->assertInstanceOf(IObjectStoreStorage::class, $storage);
$this->assertTrue($storage->instanceOfStorage(IObjectStoreStorage::class));
$this->assertSame($objectStore, $storage->getObjectStore());
$this->assertSame('urn:test:7', $storage->getURN(7));
}
}
Loading
Loading