diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/ParallelPresignedUrlMultipartDownloaderSubscriber.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/ParallelPresignedUrlMultipartDownloaderSubscriber.java index 3497aa3745e6..435811492acd 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/ParallelPresignedUrlMultipartDownloaderSubscriber.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/ParallelPresignedUrlMultipartDownloaderSubscriber.java @@ -148,8 +148,7 @@ private void sendFirstRequest(AsyncResponseTransformer CompletableFuture downloadObject( doMultipartDownload(presignedRequest, asyncResponseTransformer) .whenComplete((result, error) -> { Throwable cause = error instanceof CompletionException ? error.getCause() : error; - // Parallel path wraps it as EmptyObjectRangeNotSatisfiableException; - // serial path (toBytes, custom transformers) surfaces raw S3Exception. - if (cause instanceof EmptyObjectRangeNotSatisfiableException - || isRangeNotSatisfiable(cause)) { + if (isRangeNotSatisfiable(cause)) { log.debug(() -> "Received 416 on first request, falling back to non-range GET for empty object"); asyncPresignedUrlExtension.getObject(presignedRequest, asyncResponseTransformer) .whenComplete((r, e) -> { @@ -242,14 +239,4 @@ static boolean isRangeNotSatisfiable(Throwable error) { Throwable cause = error instanceof CompletionException ? error.getCause() : error; return cause instanceof S3Exception && ((S3Exception) cause).statusCode() == 416; } - - /** - * Marker exception wrapping a 416 on the first range request, signaling an empty object. - * Used to distinguish from 416 errors on subsequent requests which should propagate as failures. - */ - static class EmptyObjectRangeNotSatisfiableException extends RuntimeException { - EmptyObjectRangeNotSatisfiableException(Throwable cause) { - super("Object is empty (416 on first range request)", cause); - } - } } \ No newline at end of file diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/PresignedUrlMultipartDownloaderSubscriber.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/PresignedUrlMultipartDownloaderSubscriber.java index 86c4c76308ca..5d672d549278 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/PresignedUrlMultipartDownloaderSubscriber.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/PresignedUrlMultipartDownloaderSubscriber.java @@ -137,8 +137,7 @@ private void makeRangeRequest(long partIndex, if (error != null) { if (partIndex == 0 && PresignedUrlDownloadHelper.isRangeNotSatisfiable(error)) { log.debug(() -> "Received 416 on first range request, object is empty"); - resultFuture.completeExceptionally( - new PresignedUrlDownloadHelper.EmptyObjectRangeNotSatisfiableException(error)); + resultFuture.completeExceptionally(error); synchronized (lock) { subscription.cancel(); } diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/presignedurl/model/PresignedUrlDownloadRequest.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/presignedurl/model/PresignedUrlDownloadRequest.java index 6c19159251fc..1c33ce4a51c5 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/presignedurl/model/PresignedUrlDownloadRequest.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/presignedurl/model/PresignedUrlDownloadRequest.java @@ -25,6 +25,13 @@ /** * Request object for performing download operations using a presigned URL. + * + *

Presigned URLs with additional signed headers (beyond {@code Range}, {@code If-Match}, and + * {@code x-amz-checksum-mode}) are not supported. If the presigned URL was signed with {@code Range} or + * {@code If-Match}, those values must be set on this request via {@link Builder#range(String)} or + * {@link Builder#ifMatch(String)} — otherwise S3 will return a {@code SignatureDoesNotMatch} error. + * The {@code x-amz-checksum-mode} header is replayed automatically by the SDK and requires no + * additional configuration. */ @SdkPublicApi public final class PresignedUrlDownloadRequest implements ToCopyableBuilder c.minimumPartSizeInBytes(8L * MIB)) + .build(); + + presigner = S3Presigner.builder() + .region(REGION).credentialsProvider(CREDENTIALS_PROVIDER_CHAIN).build(); + + for (ContentSize size : ContentSize.values()) { + size.upload(); + } + } + + @AfterAll + static void cleanup() { + for (ContentSize size : ContentSize.values()) { + size.delete(); + } + if (singlePartClient != null) singlePartClient.close(); + if (multipartClient != null) multipartClient.close(); + if (presigner != null) presigner.close(); + } + @ParameterizedTest + @MethodSource("downloadConfigs") + void download_withPresignedUrl_contentMatchesUpload(DownloadConfig config) throws Exception { + + S3AsyncClient client = config.clientType == ClientType.SINGLE_PART ? singlePartClient : multipartClient; + ObjectWithCRC testObj = config.contentSize.object(); + + PresignedUrlDownloadRequest request = presignAndBuildRequest(testObj.key, config.range); + DownloadResult result = download(client, request, config.transformerType); + + String downloadedCrc32 = crc32(result.bytes); + if (config.range == null) { + assertThat(downloadedCrc32) + .as("CRC32 mismatch: %s — downloaded content differs from uploaded", config) + .isEqualTo(testObj.crc32); + + if (config.clientType == ClientType.SINGLE_PART + && config.contentSize != ContentSize.MPU + && result.responseCrc32 != null) { + assertThat(downloadedCrc32) + .as("CRC32 mismatch with S3 header: %s", config) + .isEqualTo(result.responseCrc32); + } + } else { + int expectedSize = parseRangeSize(config.range); + assertThat(result.bytes.length) + .as("Size mismatch for range request: %s", config) + .isEqualTo(expectedSize); + } + + } + private static List downloadConfigs() { + List configs = new ArrayList<>(); + for (ClientType clientType : ClientType.values()) { + configs.add(new DownloadConfig(clientType, ContentSize.EMPTY, TransformerType.BYTES, null)); + configs.add(new DownloadConfig(clientType, ContentSize.SMALL, TransformerType.BYTES, null)); + configs.add(new DownloadConfig(clientType, ContentSize.SMALL, TransformerType.FILE, null)); + configs.add(new DownloadConfig(clientType, ContentSize.SMALL, TransformerType.CUSTOM, null)); + configs.add(new DownloadConfig(clientType, ContentSize.LARGE_27MB, TransformerType.FILE, null)); + configs.add(new DownloadConfig(clientType, ContentSize.MPU, TransformerType.BYTES, null)); + } + configs.add(new DownloadConfig(ClientType.MULTIPART, ContentSize.LARGE_27MB, TransformerType.FILE, "bytes=5242880-10485759")); + return configs; + } + enum ClientType { + SINGLE_PART, MULTIPART + } + + enum TransformerType { + BYTES, FILE, CUSTOM + } + + enum ContentSize { + EMPTY(0, false), + SMALL(16 * KIB, false), + LARGE_27MB(27 * MIB, false), + MPU(16 * MIB, true); + + private final int size; + private final boolean multipartUpload; + private ObjectWithCRC obj; + + ContentSize(int size, boolean multipartUpload) { + this.size = size; + this.multipartUpload = multipartUpload; + } + + ObjectWithCRC object() { + return obj; + } + + void upload() { + if (multipartUpload) { + obj = doMpuUpload(name().toLowerCase(), size); + } else { + obj = doSingleUpload(name().toLowerCase(), size); + } + } + + void delete() { + if (obj != null) { + try { + s3.deleteObject(r -> r.bucket(bucketName).key(obj.key)); + } catch (Exception e) { + // ignore cleanup failures + } + } + } + } + static class DownloadConfig { + final ClientType clientType; + final ContentSize contentSize; + final TransformerType transformerType; + final String range; + + DownloadConfig(ClientType clientType, ContentSize contentSize, TransformerType transformerType, String range) { + this.clientType = clientType; + this.contentSize = contentSize; + this.transformerType = transformerType; + this.range = range; + } + + @Override + public String toString() { + return ToString.builder("DownloadConfig") + .add("client", clientType) + .add("size", contentSize) + .add("transformer", transformerType) + .add("range", range) + .build(); + } + } + + private static class ObjectWithCRC { + final String key; + final String crc32; + + ObjectWithCRC(String key, String crc32) { + this.key = key; + this.crc32 = crc32; + } + } + + private static class DownloadResult { + final byte[] bytes; + final String responseCrc32; + + DownloadResult(byte[] bytes, String responseCrc32) { + this.bytes = bytes; + this.responseCrc32 = responseCrc32; + } + } + private static ObjectWithCRC doSingleUpload(String label, int sizeBytes) { + String key = String.format("presigned-dl-%s-%s.dat", label, UUID.randomUUID()); + byte[] data = new byte[sizeBytes]; + if (sizeBytes > 0) { + new Random().nextBytes(data); + } + String dataCrc32 = crc32(data); + s3.putObject(r -> r.bucket(bucketName).key(key), RequestBody.fromBytes(data)); + return new ObjectWithCRC(key, dataCrc32); + } + + private static ObjectWithCRC doMpuUpload(String label, int sizeBytes) { + String key = String.format("presigned-dl-%s-%s.dat", label, UUID.randomUUID()); + byte[] fullData = new byte[sizeBytes]; + new Random().nextBytes(fullData); + String dataCrc32 = crc32(fullData); + + CreateMultipartUploadResponse createResp = s3.createMultipartUpload( + b -> b.bucket(bucketName).key(key).checksumAlgorithm(ChecksumAlgorithm.CRC32)); + String uploadId = createResp.uploadId(); + List parts = new ArrayList<>(); + int partSize = 8 * MIB; + int numParts = (sizeBytes + partSize - 1) / partSize; + + for (int i = 0; i < numParts; i++) { + int start = i * partSize; + int end = Math.min(start + partSize, sizeBytes); + byte[] partData = Arrays.copyOfRange(fullData, start, end); + final int partNum = i + 1; + UploadPartResponse uploadResp = s3.uploadPart( + b -> b.bucket(bucketName).key(key).uploadId(uploadId).partNumber(partNum) + .checksumAlgorithm(ChecksumAlgorithm.CRC32), + RequestBody.fromBytes(partData)); + parts.add(CompletedPart.builder() + .partNumber(partNum).eTag(uploadResp.eTag()) + .checksumCRC32(uploadResp.checksumCRC32()).build()); + } + + s3.completeMultipartUpload(b -> b.bucket(bucketName).key(key).uploadId(uploadId) + .multipartUpload(CompletedMultipartUpload.builder().parts(parts).build())); + return new ObjectWithCRC(key, dataCrc32); + } + private PresignedUrlDownloadRequest presignAndBuildRequest(String key, String range) { + PresignedGetObjectRequest presigned = presigner.presignGetObject(r -> r + .getObjectRequest(req -> { + req.bucket(bucketName).key(key).checksumMode(ChecksumMode.ENABLED); + if (range != null) req.range(range); + }) + .signatureDuration(Duration.ofMinutes(10))); + + PresignedUrlDownloadRequest.Builder reqBuilder = PresignedUrlDownloadRequest.builder() + .presignedUrl(presigned.url()); + if (range != null) reqBuilder.range(range); + return reqBuilder.build(); + } + + private DownloadResult download(S3AsyncClient client, PresignedUrlDownloadRequest request, + TransformerType transformerType) throws Exception { + switch (transformerType) { + case BYTES: { + ResponseBytes resp = client.presignedUrlExtension() + .getObject(request, AsyncResponseTransformer.toBytes()).get(5, TimeUnit.MINUTES); + return new DownloadResult(resp.asByteArray(), resp.response().checksumCRC32()); + } + case FILE: { + Path file = tempDir.resolve("dl-" + UUID.randomUUID() + ".bin"); + GetObjectResponse resp = client.presignedUrlExtension() + .getObject(request, file).get(5, TimeUnit.MINUTES); + return new DownloadResult(Files.readAllBytes(file), resp.checksumCRC32()); + } + case CUSTOM: + return downloadWithCustomTransformer(client, request); + default: + throw new IllegalArgumentException("Unknown transformer: " + transformerType); + } + } + + private DownloadResult downloadWithCustomTransformer(S3AsyncClient client, PresignedUrlDownloadRequest request) + throws Exception { + AsyncResponseTransformer transformer = + new AsyncResponseTransformer() { + private CompletableFuture future; + private GetObjectResponse response; + private ByteArrayOutputStream baos; + + @Override + public CompletableFuture prepare() { + future = new CompletableFuture<>(); + baos = new ByteArrayOutputStream(); + return future; + } + + @Override public void onResponse(GetObjectResponse r) { this.response = r; } + + @Override + public void onStream(SdkPublisher publisher) { + publisher.subscribe(new Subscriber() { + @Override public void onSubscribe(Subscription s) { s.request(Long.MAX_VALUE); } + @Override public void onNext(ByteBuffer b) { + byte[] bytes = new byte[b.remaining()]; + b.get(bytes); + baos.write(bytes, 0, bytes.length); + } + @Override public void onError(Throwable t) { future.completeExceptionally(t); } + @Override public void onComplete() { + future.complete(new DownloadResult(baos.toByteArray(), + response != null ? response.checksumCRC32() : null)); + } + }); + } + + @Override public void exceptionOccurred(Throwable error) { future.completeExceptionally(error); } + }; + + return client.presignedUrlExtension().getObject(request, transformer).get(5, TimeUnit.MINUTES); + } + + private static int parseRangeSize(String range) { + String[] parts = range.replace("bytes=", "").split("-"); + return Integer.parseInt(parts[1]) - Integer.parseInt(parts[0]) + 1; + } +}