diff --git a/packages/durabletask-js/src/worker/orchestration-executor.ts b/packages/durabletask-js/src/worker/orchestration-executor.ts index b43a939..e69a0dc 100644 --- a/packages/durabletask-js/src/worker/orchestration-executor.ts +++ b/packages/durabletask-js/src/worker/orchestration-executor.ts @@ -413,6 +413,17 @@ export class OrchestrationExecutor { // Event names are case-insensitive const eventName = event.getEventraised()?.getName()?.toLowerCase(); + // On the classic (Azure Storage / DurableTask.Core) backend, an entity call's response is + // delivered as an EVENTRAISED whose name equals the entity call's requestId (a lowercase GUID) + // and whose input is a DTFx ResponseMessage JSON, rather than as an ENTITYOPERATIONCOMPLETED. + // Route it back to the pending entity call so the callEntity task resolves/rejects. On DTS this + // branch never fires (responses arrive as ENTITYOPERATIONCOMPLETED and the pending call is + // removed by handleEntityOperationCompleted before any EVENTRAISED could match). + if (eventName && ctx._entityFeature.pendingEntityCalls.has(eventName)) { + await this.handleEntityResponseFromEventRaised(ctx, eventName, event); + return; + } + if (!ctx._isReplaying) { WorkerLogs.orchestrationEventRaised(this._logger, ctx._instanceId, eventName!); } @@ -462,16 +473,147 @@ export class OrchestrationExecutor { } } + /** + * Handles an entity operation response that arrived as an EVENTRAISED event on the classic + * (Azure Storage / DurableTask.Core) backend. In that code path the Durable Functions gRPC shim + * wraps entity responses in a DTFx ResponseMessage JSON keyed by the call's requestId, rather than + * emitting an ENTITYOPERATIONCOMPLETED proto event. This decodes the wrapper and completes (or + * fails) the pending callEntity task. Mirrors the Java SDK's handleEntityResponseFromEventRaised. + */ + private async handleEntityResponseFromEventRaised( + ctx: RuntimeOrchestrationContext, + requestId: string, + event: pb.HistoryEvent, + ): Promise { + const pendingCall = ctx._entityFeature.pendingEntityCalls.get(requestId); + if (!pendingCall) { + return; + } + + // Remove from pending calls and recover any critical-section lock before completing. + ctx._entityFeature.pendingEntityCalls.delete(requestId); + ctx._entityFeature.recoverLockAfterCall(pendingCall.entityId); + + const decoded = this.decodeEntityResponseMessage(event.getEventraised()?.getInput()?.getValue()); + + if (decoded.isFailure) { + const exception = new EntityOperationFailedException(pendingCall.entityId, pendingCall.operationName, { + errorType: decoded.errorType, + errorMessage: decoded.errorMessage, + stackTrace: decoded.stackTrace, + }); + pendingCall.task.failWithError(exception); + } else { + pendingCall.task.complete(decoded.result); + } + + await ctx.resume(); + } + + /** + * Decodes a DTFx ResponseMessage JSON wrapper (as delivered via EVENTRAISED on the classic + * backend) into either a success result or a failure. Mirrors the Java SDK's decoding, with one + * deliberate enhancement (see below). + * + * The wrapper shape is: + * - `result` — the serialized operation result (double-encoded string; may be null/absent). + * - `exceptionType` — present ⇒ failure. Misleading name: it carries the error message string + * (the C# property is ErrorMessage but its [DataMember(Name = "exceptionType")] + * overrides the JSON key). Omitted when null. + * - `failureDetails` — optional structured `{ ErrorType, ErrorMessage, StackTrace, ... }` (PascalCase). + * + * NOTE (intentionally beyond the Java SDK): the Java SDK builds `new FailureDetails(errorType, + * errorMessage, null, false)`, dropping the stack trace. Over gRPC the classic backend uses + * DurableTask.Core native entities, whose ResponseMessage.FailureDetails carries a real StackTrace, + * so we propagate it into `EntityOperationFailedException.failureDetails.stackTrace`. Do not "fix" + * this back to null for Java parity — the extra fidelity is the point. + */ + private decodeEntityResponseMessage( + rawInput: string | undefined, + ): + | { isFailure: false; result: any } + | { isFailure: true; errorType: string; errorMessage: string; stackTrace?: string } { + if (rawInput === undefined || rawInput === "") { + return { isFailure: false, result: undefined }; + } + + let parsed: any; + try { + parsed = JSON.parse(rawInput); + } catch { + // Not JSON at all — treat the raw string as the result value. + return { isFailure: false, result: rawInput }; + } + + const isObject = parsed !== null && typeof parsed === "object" && !Array.isArray(parsed); + if (!isObject || !Object.prototype.hasOwnProperty.call(parsed, "result")) { + // Not a recognized ResponseMessage wrapper — the parsed value itself is the result. + // (Mirrors Java's fallback to direct deserialization for a possible future raw-result format.) + return { isFailure: false, result: parsed }; + } + + const exceptionType = parsed.exceptionType; + const failureDetails = parsed.failureDetails; + const hasExceptionType = exceptionType !== undefined && exceptionType !== null; + const hasFailureDetails = failureDetails !== undefined && failureDetails !== null; + + if (hasExceptionType || hasFailureDetails) { + // The "exceptionType" JSON field actually carries the error message (misleading name). + let errorMessage = hasExceptionType ? String(exceptionType) : "Entity operation failed"; + let errorType = "unknown"; + let stackTrace: string | undefined; + + if (hasFailureDetails && typeof failureDetails === "object") { + if (failureDetails.ErrorType !== undefined && failureDetails.ErrorType !== null) { + errorType = String(failureDetails.ErrorType); + } + if (failureDetails.ErrorMessage !== undefined && failureDetails.ErrorMessage !== null) { + errorMessage = String(failureDetails.ErrorMessage); + } + if (failureDetails.StackTrace !== undefined && failureDetails.StackTrace !== null) { + stackTrace = String(failureDetails.StackTrace); + } + } + + return { isFailure: true, errorType, errorMessage, stackTrace }; + } + + // Success — extract the inner (double-encoded) result value. + const resultNode = parsed.result; + if (resultNode === null || resultNode === undefined) { + return { isFailure: false, result: undefined }; + } + + const innerResult = typeof resultNode === "string" ? resultNode : JSON.stringify(resultNode); + try { + return { isFailure: false, result: JSON.parse(innerResult) }; + } catch { + // Defensive: if the inner payload isn't valid JSON, use the raw string. + return { isFailure: false, result: innerResult }; + } + } + private async handleEventSent(ctx: RuntimeOrchestrationContext, event: pb.HistoryEvent): Promise { // This history event confirms that a sendEvent action was successfully processed by the sidecar. const eventId = event.getEventid(); const action = ctx._pendingActions[eventId]; - const isSendEventAction = action?.hasSendevent(); - if (!action) { throw getNonDeterminismError(eventId, getName(ctx.sendEvent)); - } else if (!isSendEventAction) { + } + + // On the classic (Azure Storage / DurableTask.Core) backend, entity operations do not use + // the DTS entity protocol. The Durable Functions gRPC shim translates a sendEntityMessage + // action (call / signal / lock / unlock) into a classic send-event, so its confirmation + // arrives here as an EVENTSENT rather than as ENTITYOPERATIONCALLED/SIGNALED. Treat that as + // a valid confirmation: remove the pending action so it isn't re-sent. The matching entity + // response (for a call) is delivered later as an EVENTRAISED and routed by handleEventRaised. + if (action.hasSendentitymessage()) { + delete ctx._pendingActions[eventId]; + return; + } + + if (!action.hasSendevent()) { const expectedMethodName = getName(ctx.sendEvent); throw new NonDeterminismError( `A previous execution called ${expectedMethodName} with ID=${eventId}, but the current execution is instead trying to call a different method as part of rebuilding its history.`, diff --git a/packages/durabletask-js/test/orchestration_executor.spec.ts b/packages/durabletask-js/test/orchestration_executor.spec.ts index 576f57d..95452da 100644 --- a/packages/durabletask-js/test/orchestration_executor.spec.ts +++ b/packages/durabletask-js/test/orchestration_executor.spec.ts @@ -31,6 +31,8 @@ import { CompletableTask } from "../src/task/completable-task"; import { Task } from "../src/task/task"; import { getName, whenAll, whenAny } from "../src/task"; import { RuntimeOrchestrationContext } from "../src/worker/runtime-orchestration-context"; +import { EntityInstanceId } from "../src/entities/entity-instance-id"; +import { EntityOperationFailedException } from "../src/entities/entity-operation-failed-exception"; // Use NoOpLogger to suppress log output during tests const testLogger = new NoOpLogger(); @@ -2369,6 +2371,237 @@ describe("EventSent Handler", () => { }); }); +describe("Entity call on classic (Azure Storage) backend — EVENTSENT/EVENTRAISED", () => { + const ENTITY_INSTANCE_ID = new EntityInstanceId("counter", "my-counter").toString(); + + // On the classic backend the Durable Functions gRPC shim turns a callEntity into a classic + // send-event: the request confirmation replays as EVENTSENT and the result as EVENTRAISED + // (name = requestId, input = a DTFx ResponseMessage JSON). These tests exercise that path. + + it("accepts EVENTSENT confirming a callEntity CALL action without throwing (removes it from pendingActions)", async () => { + let callResult: number | undefined; + const orchestrator: TOrchestrator = async function* (ctx: OrchestrationContext): any { + const entityId = new EntityInstanceId("counter", "my-counter"); + const result: number = yield ctx.entities.callEntity(entityId, "get"); + callResult = result; + return result; + }; + + const registry = new Registry(); + registry.addNamedOrchestrator("TestOrchestrator", orchestrator); + const executor = new OrchestrationExecutor(registry); + + const startEvents = [ + newOrchestratorStartedEvent(new Date()), + newExecutionStartedEvent("TestOrchestrator", "test-instance"), + ]; + + // Phase 1: capture the entity call action and its sequence id / requestId. + const result1 = await executor.execute("test-instance", [], startEvents); + const callAction = result1.actions[0]; + expect(callAction.hasSendentitymessage()).toBe(true); + expect(callAction.getSendentitymessage()!.hasEntityoperationcalled()).toBe(true); + const seqId = callAction.getId(); + + // Phase 2: replay with only the EVENTSENT confirmation (no response yet). This must NOT throw + // a NonDeterminismError, and — since the result hasn't arrived — the call stays pending. + const oldEvents2 = [...startEvents, newEventSentEvent(seqId, ENTITY_INSTANCE_ID, "get")]; + const newEvents2 = [newOrchestratorStartedEvent(new Date())]; + const result2 = await executor.execute("test-instance", oldEvents2, newEvents2); + + const failed = result2.actions.find( + (a) => + a.hasCompleteorchestration() && + a.getCompleteorchestration()!.getOrchestrationstatus() === + pb.OrchestrationStatus.ORCHESTRATION_STATUS_FAILED, + ); + expect(failed).toBeUndefined(); + // No result delivered → orchestrator is still awaiting the entity, so it hasn't completed. + expect(callResult).toBeUndefined(); + expect(result2.actions.find((a) => a.hasCompleteorchestration())).toBeUndefined(); + expect(result2.actions.find((a) => a.hasSendentitymessage())).toBeUndefined(); + }); + + it("accepts EVENTSENT confirming a signalEntity SIGNAL action without throwing", async () => { + const orchestrator: TOrchestrator = async function* (ctx: OrchestrationContext): any { + const entityId = new EntityInstanceId("counter", "my-counter"); + ctx.entities.signalEntity(entityId, "add", 5); + // Suspend on a timer so replay has to reconcile the (unconfirmed) timer and the signal. + yield ctx.createTimer(1); + }; + + const registry = new Registry(); + registry.addNamedOrchestrator("TestOrchestrator", orchestrator); + const executor = new OrchestrationExecutor(registry); + + const startEvents = [ + newOrchestratorStartedEvent(new Date()), + newExecutionStartedEvent("TestOrchestrator", "test-instance"), + ]; + + const result1 = await executor.execute("test-instance", [], startEvents); + const signalAction = result1.actions.find((a) => a.hasSendentitymessage()); + expect(signalAction).toBeDefined(); + expect(signalAction!.getSendentitymessage()!.hasEntityoperationsignaled()).toBe(true); + const seqId = signalAction!.getId(); + + // Replay with the EVENTSENT confirming the signal. Must not throw NonDeterminismError. + const oldEvents2 = [...startEvents, newEventSentEvent(seqId, ENTITY_INSTANCE_ID, "add")]; + const newEvents2 = [newOrchestratorStartedEvent(new Date())]; + const result2 = await executor.execute("test-instance", oldEvents2, newEvents2); + + const failed = result2.actions.find( + (a) => + a.hasCompleteorchestration() && + a.getCompleteorchestration()!.getOrchestrationstatus() === + pb.OrchestrationStatus.ORCHESTRATION_STATUS_FAILED, + ); + expect(result2.actions.find((a) => a.hasSendentitymessage())).toBeUndefined(); + expect(failed).toBeUndefined(); + }); + + it("resolves callEntity when confirmed via EVENTSENT then result delivered via EVENTRAISED", async () => { + let callResult: number | undefined; + const orchestrator: TOrchestrator = async function* (ctx: OrchestrationContext): any { + const entityId = new EntityInstanceId("counter", "my-counter"); + const result: number = yield ctx.entities.callEntity(entityId, "get"); + callResult = result; + return result; + }; + + const registry = new Registry(); + registry.addNamedOrchestrator("TestOrchestrator", orchestrator); + const executor = new OrchestrationExecutor(registry); + + const startEvents = [ + newOrchestratorStartedEvent(new Date()), + newExecutionStartedEvent("TestOrchestrator", "test-instance"), + ]; + + const result1 = await executor.execute("test-instance", [], startEvents); + const callAction = result1.actions[0]; + const requestId = callAction.getSendentitymessage()!.getEntityoperationcalled()!.getRequestid(); + const seqId = callAction.getId(); + + // Classic backend: EVENTSENT confirms the send, then EVENTRAISED (name = requestId) delivers + // the DTFx ResponseMessage wrapper. The result value is double-encoded (42 -> "42"). + const oldEvents2 = [...startEvents]; + const newEvents2 = [ + newOrchestratorStartedEvent(new Date()), + newEventSentEvent(seqId, ENTITY_INSTANCE_ID, "get"), + newEventRaisedEvent(requestId, JSON.stringify({ result: "42" })), + ]; + const result2 = await executor.execute("test-instance", oldEvents2, newEvents2); + + expect(callResult).toBe(42); + const completeAction = result2.actions.find((a) => a.hasCompleteorchestration()); + expect(completeAction).toBeDefined(); + expect(completeAction!.getCompleteorchestration()!.getOrchestrationstatus()).toEqual( + pb.OrchestrationStatus.ORCHESTRATION_STATUS_COMPLETED, + ); + expect(completeAction!.getCompleteorchestration()!.getResult()?.getValue()).toEqual(JSON.stringify(42)); + }); + + it("rejects callEntity with EntityOperationFailedException when EVENTRAISED carries a failureDetails ResponseMessage", async () => { + let caughtError: Error | undefined; + const orchestrator: TOrchestrator = async function* (ctx: OrchestrationContext): any { + const entityId = new EntityInstanceId("counter", "my-counter"); + try { + yield ctx.entities.callEntity(entityId, "get"); + } catch (e) { + caughtError = e as Error; + } + return "handled"; + }; + + const registry = new Registry(); + registry.addNamedOrchestrator("TestOrchestrator", orchestrator); + const executor = new OrchestrationExecutor(registry); + + const startEvents = [ + newOrchestratorStartedEvent(new Date()), + newExecutionStartedEvent("TestOrchestrator", "test-instance"), + ]; + + const result1 = await executor.execute("test-instance", [], startEvents); + const callAction = result1.actions[0]; + const requestId = callAction.getSendentitymessage()!.getEntityoperationcalled()!.getRequestid(); + const seqId = callAction.getId(); + + // Failure ResponseMessage: "result" is present (null) alongside structured failureDetails, + // including a StackTrace (the classic backend uses DurableTask.Core native entities over gRPC, + // whose FailureDetails carries a real stack trace). + const responseJson = JSON.stringify({ + result: null, + failureDetails: { + ErrorType: "InvalidOperationException", + ErrorMessage: "boom", + StackTrace: " at Counter.Get() in Counter.cs:line 42", + }, + }); + const oldEvents2 = [...startEvents]; + const newEvents2 = [ + newOrchestratorStartedEvent(new Date()), + newEventSentEvent(seqId, ENTITY_INSTANCE_ID, "get"), + newEventRaisedEvent(requestId, responseJson), + ]; + await executor.execute("test-instance", oldEvents2, newEvents2); + + expect(caughtError).toBeInstanceOf(EntityOperationFailedException); + const entityError = caughtError as EntityOperationFailedException; + expect(entityError.operationName).toBe("get"); + expect(entityError.entityId.name).toBe("counter"); + expect(entityError.entityId.key).toBe("my-counter"); + expect(entityError.failureDetails.errorType).toBe("InvalidOperationException"); + expect(entityError.failureDetails.errorMessage).toBe("boom"); + expect(entityError.failureDetails.stackTrace).toBe(" at Counter.Get() in Counter.cs:line 42"); + }); + + it("rejects callEntity with EntityOperationFailedException when EVENTRAISED carries an exceptionType-only ResponseMessage", async () => { + let caughtError: Error | undefined; + const orchestrator: TOrchestrator = async function* (ctx: OrchestrationContext): any { + const entityId = new EntityInstanceId("counter", "my-counter"); + try { + yield ctx.entities.callEntity(entityId, "get"); + } catch (e) { + caughtError = e as Error; + } + return "handled"; + }; + + const registry = new Registry(); + registry.addNamedOrchestrator("TestOrchestrator", orchestrator); + const executor = new OrchestrationExecutor(registry); + + const startEvents = [ + newOrchestratorStartedEvent(new Date()), + newExecutionStartedEvent("TestOrchestrator", "test-instance"), + ]; + + const result1 = await executor.execute("test-instance", [], startEvents); + const callAction = result1.actions[0]; + const requestId = callAction.getSendentitymessage()!.getEntityoperationcalled()!.getRequestid(); + const seqId = callAction.getId(); + + // WebJobs-variant failure: "exceptionType" present (carries the error message), no failureDetails. + const responseJson = JSON.stringify({ result: "serialized-exception", exceptionType: "Something failed" }); + const oldEvents2 = [...startEvents]; + const newEvents2 = [ + newOrchestratorStartedEvent(new Date()), + newEventSentEvent(seqId, ENTITY_INSTANCE_ID, "get"), + newEventRaisedEvent(requestId, responseJson), + ]; + await executor.execute("test-instance", oldEvents2, newEvents2); + + expect(caughtError).toBeInstanceOf(EntityOperationFailedException); + const entityError = caughtError as EntityOperationFailedException; + expect(entityError.failureDetails.errorType).toBe("unknown"); + expect(entityError.failureDetails.errorMessage).toBe("Something failed"); + // WebJobs-variant path carries no structured failureDetails, so there is no stack trace. + expect(entityError.failureDetails.stackTrace).toBeUndefined(); + }); +}); + function getAndValidateSingleCompleteOrchestrationAction( result: OrchestrationExecutionResult, ): CompleteOrchestrationAction | undefined {