Speed up and stabilize integration tests (parallelism + deploy retry)

Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/xunit.runner.json

@@ -1,6 +1,6 @@
 {
   "$schema": "https://xunit.net/schema/current/xunit.runner.schema.json",
-  "parallelizeTestCollections": false,
+  "parallelizeTestCollections": true,
   "parallelizeAssembly": false,
-  "maxParallelThreads": 1
+  "maxParallelThreads": 4
 }

Libraries/test/Amazon.Lambda.DurableExecution.Tests/CallbackOperationTests.cs

@@ -163,7 +163,7 @@ public async Task GetResultAsync_FreshExecution_SuspendsExecution()
 
         // GetResultAsync should signal termination and return a never-completing task.
         var resultTask = callback.GetResultAsync();
-        await Task.Delay(10);
+        await tm.WaitForTerminationAsync();
 
         Assert.True(tm.IsTerminated);
         Assert.False(resultTask.IsCompleted);
@@ -193,7 +193,7 @@ public async Task ReplayStarted_DoesNotReFlushStart_AndSuspendsOnGetResult()
         Assert.False(tm.IsTerminated);
 
         var resultTask = callback.GetResultAsync();
-        await Task.Delay(10);
+        await tm.WaitForTerminationAsync();
 
         Assert.True(tm.IsTerminated);
         Assert.False(resultTask.IsCompleted);

Libraries/test/Amazon.Lambda.DurableExecution.Tests/ChildContextOperationTests.cs

@@ -322,7 +322,7 @@ public async Task RunInChildContextAsync_ChildSuspendsOnWait_TerminatesWithWaitS
             },
             name: "phase");
 
-        await Task.Delay(50);
+        await tm.WaitForTerminationAsync();
 
         Assert.True(tm.IsTerminated);
         Assert.False(task.IsCompleted);

Libraries/test/Amazon.Lambda.DurableExecution.Tests/DurableContextTests.cs

@@ -379,7 +379,7 @@ public async Task WaitAsync_NewExecution_SignalsTermination()
         var waitTask = context.WaitAsync(TimeSpan.FromSeconds(30), name: "my_wait");
 
         // Give it a moment to execute
-        await Task.Delay(10);
+        await tm.WaitForTerminationAsync();
 
         Assert.True(tm.IsTerminated);
         Assert.False(waitTask.IsCompleted);
@@ -433,7 +433,7 @@ public async Task WaitAsync_StartedButNotExpired_ResuspendsWithoutNewCheckpoint(
 
         var waitTask = context.WaitAsync(TimeSpan.FromSeconds(30), name: "pending_wait");
 
-        await Task.Delay(10);
+        await tm.WaitForTerminationAsync();
 
         Assert.True(tm.IsTerminated);
         Assert.False(waitTask.IsCompleted);

Libraries/test/Amazon.Lambda.DurableExecution.Tests/InvokeOperationTests.cs

@@ -74,7 +74,7 @@ public async Task InvokeAsync_PreservesUnqualifiedArn_AndPassesItThrough()
             payload: "x",
             name: "noversion");
 
-        await Task.Delay(20);
+        await tm.WaitForTerminationAsync();
         Assert.True(tm.IsTerminated);
         Assert.False(task.IsCompleted);
 
@@ -100,7 +100,7 @@ public async Task InvokeAsync_FreshExecution_CheckpointsStartAndSuspends()
 
         // Service-side suspend mechanics: TerminationManager fires before the
         // user task completes; the task itself never resolves on the fresh path.
-        await Task.Delay(20);
+        await tm.WaitForTerminationAsync();
         Assert.True(tm.IsTerminated);
         Assert.False(task.IsCompleted);
 
@@ -130,7 +130,7 @@ public async Task InvokeAsync_FreshExecution_NoTenantId_OmitsTenantId()
 
         var task = context.InvokeAsync<string, string>(FunctionArn, "payload", name: "no_tenant");
 
-        await Task.Delay(20);
+        await tm.WaitForTerminationAsync();
         Assert.True(tm.IsTerminated);
         Assert.False(task.IsCompleted);
 
@@ -154,7 +154,7 @@ public async Task InvokeAsync_FreshExecution_StartIsSyncFlushed()
         var (context, recorder, tm, _) = CreateContext();
 
         var task = context.InvokeAsync<string, string>(FunctionArn, "x", name: "sync_flush");
-        await Task.Delay(20);
+        await tm.WaitForTerminationAsync();
 
         Assert.True(tm.IsTerminated);
         Assert.False(task.IsCompleted);
@@ -350,7 +350,7 @@ public async Task InvokeAsync_ReplayStarted_ResuspendsWithoutRecheckpoint()
         });
 
         var task = context.InvokeAsync<string, string>(FunctionArn, "x", name: "still_running");
-        await Task.Delay(20);
+        await tm.WaitForTerminationAsync();
 
         Assert.True(tm.IsTerminated);
         Assert.False(task.IsCompleted);
@@ -377,7 +377,7 @@ public async Task InvokeAsync_ReplayPending_ResuspendsWithoutRecheckpoint()
         });
 
         var task = context.InvokeAsync<string, string>(FunctionArn, "x", name: "pending");
-        await Task.Delay(20);
+        await tm.WaitForTerminationAsync();
 
         Assert.True(tm.IsTerminated);
         Assert.False(task.IsCompleted);

Libraries/test/Amazon.Lambda.DurableExecution.Tests/TerminationTestHelpers.cs

@@ -0,0 +1,23 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+using Amazon.Lambda.DurableExecution.Internal;
+
+namespace Amazon.Lambda.DurableExecution.Tests;
+
+/// <summary>
+/// Shared helpers for tests that exercise the suspend/terminate path.
+/// </summary>
+internal static class TerminationTestHelpers
+{
+    /// <summary>
+    /// Waits for the suspend signal deterministically instead of a fixed delay, which races under
+    /// CI thread-pool pressure (the original <c>Task.Delay</c> assumed the suspend happened within a
+    /// fixed window, which isn't guaranteed). The suspend path trips
+    /// <see cref="TerminationManager.Terminate"/>, which completes
+    /// <see cref="TerminationManager.TerminationTask"/>. Bounded by a timeout so a genuine
+    /// non-suspension fails fast at the following assert instead of hanging.
+    /// </summary>
+    public static Task WaitForTerminationAsync(this TerminationManager tm, int timeoutSeconds = 10) =>
+        Task.WhenAny(tm.TerminationTask, Task.Delay(TimeSpan.FromSeconds(timeoutSeconds)));
+}

Libraries/test/Amazon.Lambda.DurableExecution.Tests/WaitForConditionOperationTests.cs

@@ -89,7 +89,7 @@ public async Task FreshExecution_StrategyContinues_EmitsRetryAndSuspends()
             },
             name: "poll");
 
-        await Task.Delay(50);
+        await tm.WaitForTerminationAsync();
 
         Assert.True(tm.IsTerminated);
         Assert.False(task.IsCompleted);
@@ -818,7 +818,7 @@ public async Task FreshExecution_FlushesStartBeforeSuspending()
             },
             name: "poll");
 
-        await Task.Delay(50);
+        await tm.WaitForTerminationAsync();
 
         Assert.True(tm.IsTerminated);
         Assert.False(task.IsCompleted);

Libraries/test/Amazon.Lambda.RuntimeSupport.Tests/Amazon.Lambda.RuntimeSupport.UnitTests/TestHelpers/TestFileStream.cs

@@ -1,7 +1,5 @@
 using System;
-using System.Collections.Generic;
 using System.IO;
-using System.Linq;
 
 namespace Amazon.Lambda.RuntimeSupport.UnitTests.TestHelpers
 {
@@ -19,13 +17,14 @@ public TestFileStream(Action<byte[], int, int> writeAction)
 
         public override void Write(byte[] buffer, int offset, int count)
         {
-            WriteAction(TrimTrailingNullBytes(buffer).Take(count).ToArray(), offset, count);
-        }
-
-        private static IEnumerable<byte> TrimTrailingNullBytes(IEnumerable<byte> buffer)
-        {
-            // Trim trailing null bytes to make testing assertions easier
-            return buffer.Reverse().SkipWhile(x => x == 0).Reverse();
+            // Capture exactly the bytes that were written: [offset, offset + count).
+            // The previous implementation trimmed trailing null bytes from the buffer, which was
+            // flaky: a log header ends with an 8-byte big-endian microsecond timestamp, and roughly
+            // 1 in 256 timestamps ends in a 0x00 byte. Trimming that legitimate byte made the
+            // captured header 15 bytes instead of 16 and failed MaxSizeProducesOneLogFrame.
+            var written = new byte[count];
+            Array.Copy(buffer, offset, written, 0, count);
+            WriteAction(written, offset, count);
         }
     }
 }

Libraries/test/IntegrationTests.Helpers/LambdaHelper.cs

@@ -2,39 +2,50 @@
 // SPDX-License-Identifier: Apache-2.0
 
 using System.Collections.Generic;
+using System.Linq;
 using System.Threading.Tasks;
+using Amazon.CloudFormation;
+using Amazon.CloudFormation.Model;
 using Amazon.Lambda;
 using Amazon.Lambda.Model;
 
 namespace IntegrationTests.Helpers
 {
     public class LambdaHelper
     {
+        // Resource type that SAM AWS::Serverless::Function resources are transformed into in the deployed stack.
+        private const string LambdaFunctionResourceType = "AWS::Lambda::Function";
+
         private readonly IAmazonLambda _lambdaClient;
+        private readonly IAmazonCloudFormation _cloudFormationClient;
 
-        public LambdaHelper(IAmazonLambda lambdaClient)
+        public LambdaHelper(IAmazonLambda lambdaClient, IAmazonCloudFormation cloudFormationClient)
         {
             _lambdaClient = lambdaClient;
+            _cloudFormationClient = cloudFormationClient;
         }
 
+        /// <summary>
+        /// Returns the Lambda functions belonging to a CloudFormation stack by listing the stack's
+        /// resources directly. This is O(stack size) and independent of how many functions exist in
+        /// the account, unlike scanning every function and reading its tags one at a time, which is
+        /// slow and prone to throttling in a shared test account.
+        /// </summary>
         public async Task<List<LambdaFunction>> FilterByCloudFormationStackAsync(string stackName)
         {
-            const string stackNameKey = "aws:cloudformation:stack-name";
-            const string logicalIdKey = "aws:cloudformation:logical-id";
             var lambdaFunctions = new List<LambdaFunction>();
-            var paginator = _lambdaClient.Paginators.ListFunctions(new ListFunctionsRequest());
+            var paginator = _cloudFormationClient.Paginators.ListStackResources(
+                new ListStackResourcesRequest { StackName = stackName });
 
-            await foreach (var function in paginator.Functions)
+            await foreach (var resource in paginator.StackResourceSummaries)
             {
-                var tags = (await _lambdaClient.ListTagsAsync(new ListTagsRequest { Resource = function.FunctionArn })).Tags;
-                if (tags.ContainsKey(stackNameKey) && string.Equals(tags[stackNameKey], stackName))
+                if (string.Equals(resource.ResourceType, LambdaFunctionResourceType))
                 {
-                    var lambdaFunction = new LambdaFunction
+                    lambdaFunctions.Add(new LambdaFunction
                     {
-                        LogicalId = tags[logicalIdKey],
-                        Name = function.FunctionName
-                    };
-                    lambdaFunctions.Add(lambdaFunction);
+                        LogicalId = resource.LogicalResourceId,
+                        Name = resource.PhysicalResourceId
+                    });
                 }
             }
 

Libraries/test/TestCustomAuthorizerApp.IntegrationTests/DeploymentScript.ps1

@@ -42,7 +42,32 @@ try
     $json =  Get-Content .\aws-lambda-tools-defaults.json | Out-String | ConvertFrom-Json
     $region = $json.region
 
-    dotnet tool install -g Amazon.Lambda.Tools
+    # Install Amazon.Lambda.Tools idempotently. The integration test projects deploy in parallel,
+    # so several DeploymentScript.ps1 processes may run "dotnet tool install -g" at the same time and
+    # collide on the global tool store ("a file or directory with the same name already exists").
+    # Skip if already present, and tolerate the concurrent-install race by treating an
+    # already-installed/already-exists result as success, with a short retry for the transient case.
+    if (dotnet tool list -g | Select-String -SimpleMatch 'amazon.lambda.tools')
+    {
+        Write-Host "Amazon.Lambda.Tools already installed."
+    }
+    else
+    {
+        for ($i = 1; $i -le 5; $i++)
+        {
+            $output = dotnet tool install -g Amazon.Lambda.Tools 2>&1 | Out-String
+            Write-Host $output
+            if ($LASTEXITCODE -eq 0 -or $output -match 'already installed' -or $output -match 'already exists')
+            {
+                break
+            }
+            if ($i -eq 5)
+            {
+                throw "Failed to install Amazon.Lambda.Tools after $i attempts."
+            }
+            Start-Sleep -Seconds ($i * 3)
+        }
+    }
     Write-Host "Creating S3 Bucket $identifier"
 
     if(![string]::IsNullOrEmpty($region))
@@ -59,11 +84,51 @@ try
         throw "Failed to create the following bucket: $identifier"
     }
     dotnet restore
-    Write-Host "Creating CloudFormation Stack $identifier, Architecture $arch"
-    dotnet lambda deploy-serverless
-    if (!$?)
+
+    # Deploy with retries. The stack contains many Lambda functions that each reference
+    # an IAM role created in the same stack. CloudFormation occasionally calls Lambda
+    # CreateFunction before the role's trust policy has propagated through IAM, producing
+    # "The role defined for the function cannot be assumed by Lambda" and rolling the whole
+    # stack back. This is a transient eventual-consistency race, so retry the deployment.
+    $maxAttempts = 3
+    $deploySucceeded = $false
+    for ($attempt = 1; $attempt -le $maxAttempts; $attempt++)
+    {
+        Write-Host "Creating CloudFormation Stack $identifier, Architecture $arch (attempt $attempt of $maxAttempts)"
+        dotnet lambda deploy-serverless
+        if ($?)
+        {
+            $deploySucceeded = $true
+            break
+        }
+
+        Write-Host "Deployment attempt $attempt failed. Fetching CloudFormation stack events for debugging..."
+        try {
+            $events = aws cloudformation describe-stack-events --stack-name $identifier --query "StackEvents[?ResourceStatus=='CREATE_FAILED' || ResourceStatus=='UPDATE_FAILED' || ResourceStatus=='DELETE_FAILED']" --output json 2>&1
+            if ($events) {
+                Write-Host "CloudFormation failed events:"
+                Write-Host $events
+            }
+        }
+        catch {
+            Write-Host "Could not fetch CloudFormation events: $_"
+        }
+
+        if ($attempt -lt $maxAttempts)
+        {
+            # A failed create leaves the stack in ROLLBACK_COMPLETE, which cannot be updated
+            # or re-created. Delete it (and wait for the delete to finish) before retrying.
+            Write-Host "Deleting rolled-back stack $identifier before retrying..."
+            aws cloudformation delete-stack --stack-name $identifier
+            aws cloudformation wait stack-delete-complete --stack-name $identifier
+            # Brief pause to give IAM additional time to settle before the next attempt.
+            Start-Sleep -Seconds 15
+        }
+    }
+
+    if (!$deploySucceeded)
     {
-        throw "Failed to create the following CloudFormation stack: $identifier"
+        throw "Failed to create the following CloudFormation stack after $maxAttempts attempts: $identifier"
     }
 }
 finally

Libraries/test/TestCustomAuthorizerApp.IntegrationTests/HealthCheckTests.cs

@@ -1,13 +1,13 @@
 using System.Net;
 using Xunit;
+using Xunit.Extensions.AssemblyFixture;
 
 namespace TestCustomAuthorizerApp.IntegrationTests;
 
 /// <summary>
 /// Tests for the health check endpoint which does not require authorization.
 /// </summary>
-[Collection("Integration Tests")]
-public class HealthCheckTests
+public class HealthCheckTests : IAssemblyFixture<IntegrationTestContextFixture>
 {
     private readonly IntegrationTestContextFixture _fixture;
 

Libraries/test/TestCustomAuthorizerApp.IntegrationTests/HttpApiV1Tests.cs

@@ -2,6 +2,7 @@
 using System.Net.Http.Headers;
 using Newtonsoft.Json.Linq;
 using Xunit;
+using Xunit.Extensions.AssemblyFixture;
 
 namespace TestCustomAuthorizerApp.IntegrationTests;
 
@@ -12,8 +13,7 @@ namespace TestCustomAuthorizerApp.IntegrationTests;
 /// These tests verify that the source-generated Lambda handler correctly extracts
 /// values from the authorizer context using [FromCustomAuthorizer] attributes.
 /// </summary>
-[Collection("Integration Tests")]
-public class HttpApiV1Tests
+public class HttpApiV1Tests : IAssemblyFixture<IntegrationTestContextFixture>
 {
     private readonly IntegrationTestContextFixture _fixture;
 

Libraries/test/TestCustomAuthorizerApp.IntegrationTests/HttpApiV2Tests.cs

@@ -2,6 +2,7 @@
 using System.Net.Http.Headers;
 using Newtonsoft.Json.Linq;
 using Xunit;
+using Xunit.Extensions.AssemblyFixture;
 
 namespace TestCustomAuthorizerApp.IntegrationTests;
 
@@ -12,8 +13,7 @@ namespace TestCustomAuthorizerApp.IntegrationTests;
 /// These tests verify that the source-generated Lambda handler correctly extracts
 /// values from the authorizer context using [FromCustomAuthorizer] attributes.
 /// </summary>
-[Collection("Integration Tests")]
-public class HttpApiV2Tests
+public class HttpApiV2Tests : IAssemblyFixture<IntegrationTestContextFixture>
 {
     private readonly IntegrationTestContextFixture _fixture;
 

Libraries/test/TestCustomAuthorizerApp.IntegrationTests/IntegrationTestContextFixture.cs

@@ -39,9 +39,10 @@ public class IntegrationTestContextFixture : IAsyncLifetime
 
     public IntegrationTestContextFixture()
     {
-        _cloudFormationHelper = new CloudFormationHelper(new AmazonCloudFormationClient(Amazon.RegionEndpoint.USWest2));
+        var cloudFormationClient = new AmazonCloudFormationClient(Amazon.RegionEndpoint.USWest2);
+        _cloudFormationHelper = new CloudFormationHelper(cloudFormationClient);
         _s3Helper = new S3Helper(new AmazonS3Client(Amazon.RegionEndpoint.USWest2));
-        LambdaHelper = new LambdaHelper(new AmazonLambdaClient(Amazon.RegionEndpoint.USWest2));
+        LambdaHelper = new LambdaHelper(new AmazonLambdaClient(Amazon.RegionEndpoint.USWest2), cloudFormationClient);
         CloudWatchHelper = new CloudWatchHelper(new AmazonCloudWatchLogsClient(Amazon.RegionEndpoint.USWest2));
         HttpClient = new HttpClient();
     }