chore: limit output file size#509
Merged
Merged
Conversation
in case someone requests an unbounded image `fit-in/0x0` the image handler currently delivers the image in its original size ... This PR changes this by imposing an upper limit to the horizontal and/or vertical size which will hopefully keep the image small enough so we do not hit the lambda payload limit. # Conflicts: # source/image-handler/src/thumbor-mapper.ts diff --git c/README.md i/README.md index f7b3782..7cbdf45 100644 --- c/README.md +++ i/README.md @@ -56,12 +56,15 @@ For more details see the [Usage](docs/Usage.md) documentation. The following environment variables are used by the image-handler: -| Name | Description | -|---------------------------|-------------------------------------------------| -| `AUTO_WEBP` | Flag if the AUTO WEBP feature should be enabled | -| `CORS_ENABLED` | Flag if CORS should be enabled | -| `CORS_ORIGIN` | CORS origin. | -| `SOURCE_BUCKETS` | S3 Bucket with source images | +| Name | Description | Default | +|----------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------| +| `AUTO_WEBP` | Flag if the AUTO WEBP feature should be enabled | – | +| `CORS_ENABLED` | Flag if CORS should be enabled | – | +| `CORS_ORIGIN` | CORS origin. | – | +| `SOURCE_BUCKETS` | S3 Bucket with source images | – | +| `UNBOUNDED_FIT_IN_MAX_DIMENSION` | Max longest side (px) for unbounded `fit-in/0x0` requests. Without a bound the image is returned at full native resolution and can exceed the 6 MB Lambda payload limit (`TooLargeImageException`). Only downscales; smaller images pass through untouched. | `4000` | +| `MAX_ANIMATED_PIXELS` | Pixel budget for animated images (GIF/animated WebP). The number of decoded frames is capped to `MAX_ANIMATED_PIXELS / (frameWidth × frameHeight)` so high-frame-count animations stay under the payload limit. Excess frames are dropped (animation truncated). | `5000000` | +| `MAX_ANIMATED_FRAMES` | Absolute upper bound on decoded frames for animated images, applied alongside `MAX_ANIMATED_PIXELS`. | `100` | ### Building diff --git c/source/image-handler/src/image-handler.ts i/source/image-handler/src/image-handler.ts index e2bd3e4..c6ee0f8 100644 --- c/source/image-handler/src/image-handler.ts +++ i/source/image-handler/src/image-handler.ts @@ -3,12 +3,22 @@ import sharp, { OverlayOptions, SharpOptions } from 'sharp'; -import { ContentTypes, ImageEdits, ImageFitTypes, ImageFormatTypes, ImageHandlerError, ImageRequestInfo, StatusCodes, } from './lib'; +import { + ContentTypes, + ImageEdits, + ImageFitTypes, + ImageFormatTypes, + ImageHandlerError, + ImageRequestInfo, + StatusCodes, +} from './lib'; import { S3 } from '@aws-sdk/client-s3'; import { rgbaToThumbHash } from './lib/thumbhash'; export class ImageHandler { private readonly LAMBDA_PAYLOAD_LIMIT = 6 * 1024 * 1024; + private readonly MAX_ANIMATED_PIXELS = parseInt(process.env.MAX_ANIMATED_PIXELS) || 5_000_000; + private readonly MAX_ANIMATED_FRAMES = parseInt(process.env.MAX_ANIMATED_FRAMES) || 100; constructor(private readonly s3Client: S3) {} @@ -97,6 +107,16 @@ export class ImageHandler { if (!metadata.pages || metadata.pages <= 1) { options.animated = false; image = await this.instantiateSharpImage(originalImage, edits, options); + } else { + const perFramePixels = (metadata.width ?? 1) * (metadata.pageHeight ?? metadata.height ?? 1); + const maxFrames = Math.min( + this.MAX_ANIMATED_FRAMES, + Math.max(1, Math.floor(this.MAX_ANIMATED_PIXELS / perFramePixels)), + ); + if (metadata.pages > maxFrames) { + options.pages = maxFrames; + image = await this.instantiateSharpImage(originalImage, edits, options); + } } } diff --git c/source/image-handler/src/thumbor-mapper.ts i/source/image-handler/src/thumbor-mapper.ts index 4c798d1..13344f4 100644 --- c/source/image-handler/src/thumbor-mapper.ts +++ i/source/image-handler/src/thumbor-mapper.ts @@ -8,14 +8,7 @@ import { ImageEdits, ImageFitTypes, ImageFormatTypes } from './lib'; export class ThumborMapper { private static readonly EMPTY_IMAGE_EDITS: ImageEdits = {}; - - // Upper bound (px, longest side) applied to otherwise-unbounded `0x0` resize requests - // (e.g. `fit-in/0x0`). Without a bound, sharp returns the image at its full native (cropped) - // resolution; for multi-megapixel originals the base64-encoded result exceeds the 6 MB Lambda - // payload limit and fails with `TooLargeImageException`. Override via the - // `UNBOUNDED_FIT_IN_MAX_DIMENSION` environment variable. private static readonly UNBOUNDED_FIT_IN_MAX_DIMENSION = parseInt(process.env.UNBOUNDED_FIT_IN_MAX_DIMENSION) || 4000; - /** * Initializer function for creating a new Thumbor mapping, used by the image * handler to perform image modifications based on legacy URL path requests. @@ -458,12 +451,7 @@ export class ThumborMapper { if (width === 0 || height === 0) { resizeEdit.resize.fit = ImageFitTypes.INSIDE; } - if (width === 0 && height === 0) { - // `0x0` (e.g. `fit-in/0x0`) requests no resize bound, which would return the image at - // its full native (cropped) resolution and can blow past the 6 MB Lambda payload limit. - // Clamp the longest side so large images are downscaled to fit; `withoutEnlargement` - // leaves images already smaller than the bound untouched (no upscaling). resizeEdit.resize.width = ThumborMapper.UNBOUNDED_FIT_IN_MAX_DIMENSION; resizeEdit.resize.height = ThumborMapper.UNBOUNDED_FIT_IN_MAX_DIMENSION; resizeEdit.resize.withoutEnlargement = true; diff --git c/source/image-handler/terraform/main.tf i/source/image-handler/terraform/main.tf index 7f54074..45d4005 100644 --- c/source/image-handler/terraform/main.tf +++ i/source/image-handler/terraform/main.tf @@ -27,10 +27,13 @@ module "lambda" { environment = { variables = { - AUTO_WEBP = "Yes" - CORS_ENABLED = "Yes" - CORS_ORIGIN = "*" - SOURCE_BUCKETS = "master-images-${var.account_id}-${var.region}" + AUTO_WEBP = "Yes" + CORS_ENABLED = "Yes" + CORS_ORIGIN = "*" + SOURCE_BUCKETS = "master-images-${var.account_id}-${var.region}" + UNBOUNDED_FIT_IN_MAX_DIMENSION = "4000" + MAX_ANIMATED_PIXELS = "5000000" + MAX_ANIMATED_FRAMES = "100" } } diff --git c/source/image-handler/test/image-handler/animated.spec.ts i/source/image-handler/test/image-handler/animated.spec.ts index 81fb90b..275529a 100644 --- c/source/image-handler/test/image-handler/animated.spec.ts +++ i/source/image-handler/test/image-handler/animated.spec.ts @@ -153,6 +153,34 @@ describe('animated', () => { }); }); + it('Should cap decoded frames when the animation exceeds the frame guard', async () => { + // Arrange — force the cap to 1 frame so the 2-page fixture trips the guard + process.env.MAX_ANIMATED_FRAMES = '1'; + const request: ImageRequestInfo = { + requestType: RequestTypes.DEFAULT, + contentType: ContentTypes.GIF, + bucket: 'sample-bucket', + key: 'sample-image-001.gif', + edits: { grayscale: true }, + originalImage: gifImage, + }; + + // Act + const imageHandler = new ImageHandler(s3Client); + const instantiateSpy = jest.spyOn<any, 'instantiateSharpImage'>(imageHandler, 'instantiateSharpImage'); + await imageHandler.process(request); + + // Assert — re-instantiated with an explicit page cap, animation semantics preserved + expect(instantiateSpy).toHaveBeenCalledTimes(2); + expect(instantiateSpy).toHaveBeenLastCalledWith(request.originalImage, request.edits, { + failOn: 'none', + animated: true, + pages: 1, + }); + + delete process.env.MAX_ANIMATED_FRAMES; + }); + it('Should attempt to create animated image if animated edit is set to true, regardless of original image and content type', async () => { // Arrange const request: ImageRequestInfo = {
thisismana
force-pushed
the
limit-file-size
branch
from
05a4212 to
b0967f2
Compare
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
2 participants
Add this suggestion to a batch that can be applied as a single commit.This suggestion is invalid because no changes were made to the code.Suggestions cannot be applied while the pull request is closed.Suggestions cannot be applied while viewing a subset of changes.Only one suggestion per line can be applied in a batch.Add this suggestion to a batch that can be applied as a single commit.Applying suggestions on deleted lines is not supported.You must change the existing code in this line in order to create a valid suggestion.Outdated suggestions cannot be applied.This suggestion has been applied or marked resolved.Suggestions cannot be applied from pending reviews.Suggestions cannot be applied on multi-line comments.Suggestions cannot be applied while the pull request is queued to merge.Suggestion cannot be applied right now. Please check back later.
in case someone requests an unbounded image
fit-in/0x0the image handler currently delivers the image in its original size ...This PR changes this by imposing an upper limit to the horizontal and/or vertical size which will hopefully keep the image small enough so we do not hit the lambda payload limit.