From 73d1189bf938f5a7bf72d882766c2d9f03a9f594 Mon Sep 17 00:00:00 2001 From: Vipin Gupta Date: Wed, 2 Sep 2026 13:31:55 +0000 Subject: [PATCH 1/2] feat: Emit worker pool size as a DEBUG log event during init on Lambda Managed Instances --- .../RELEASE.CHANGELOG.md | 4 +++ .../lambda/runtime/api/client/AWSLambda.java | 16 +++++++++ .../client/logging/AbstractLambdaLogger.java | 6 ++++ .../api/client/logging/JsonLogFormatter.java | 12 +++++-- .../api/client/logging/LogFormatter.java | 4 +++ .../client/logging/StructuredLogMessage.java | 2 +- .../runtime/api/client/AWSLambdaTest.java | 34 +++++++++++++++++++ 7 files changed, 75 insertions(+), 3 deletions(-) diff --git a/aws-lambda-java-runtime-interface-client/RELEASE.CHANGELOG.md b/aws-lambda-java-runtime-interface-client/RELEASE.CHANGELOG.md index 97d177034..9ca18e745 100644 --- a/aws-lambda-java-runtime-interface-client/RELEASE.CHANGELOG.md +++ b/aws-lambda-java-runtime-interface-client/RELEASE.CHANGELOG.md @@ -1,3 +1,7 @@ +### September 2, 2026 +`2.12.1` +- Emit a structured `runtime_worker_pool_initializing` DEBUG log event once during INIT in multi-concurrent (Lambda Managed Instances) mode, reporting the worker pool size (`workerCount`) and the maximum concurrency the execution environment supports (`executionEnvironmentMaxConcurrency`). Only visible when the function log level is DEBUG or lower; not emitted for standard on-demand functions. + ### July 17, 2026 `2.12.0` - Add `Lambda-Runtime-Invocation-Id` header support for cross-wiring protection. The RIC now echoes the invocation ID received from RAPID on `/next` back on `/response` and `/error`, enabling RAPID to detect and reject stale responses from timed-out invocations. diff --git a/aws-lambda-java-runtime-interface-client/src/main/java/com/amazonaws/services/lambda/runtime/api/client/AWSLambda.java b/aws-lambda-java-runtime-interface-client/src/main/java/com/amazonaws/services/lambda/runtime/api/client/AWSLambda.java index b9aa0fd11..b89fcf504 100644 --- a/aws-lambda-java-runtime-interface-client/src/main/java/com/amazonaws/services/lambda/runtime/api/client/AWSLambda.java +++ b/aws-lambda-java-runtime-interface-client/src/main/java/com/amazonaws/services/lambda/runtime/api/client/AWSLambda.java @@ -251,6 +251,11 @@ protected static void startRuntimeLoops(LambdaRequestHandler lambdaRequestHandle if (concurrencyConfig.isMultiConcurrent()) { lambdaLogger.log(concurrencyConfig.getConcurrencyConfigMessage(), lambdaLogger.getLogFormat() == LogFormat.JSON ? LogLevel.INFO : LogLevel.UNDEFINED); ExecutorService platformThreadExecutor = Executors.newFixedThreadPool(concurrencyConfig.getNumberOfPlatformThreads()); + lambdaLogger.logStructuredEvent( + new WorkerPoolInitializedEvent( + concurrencyConfig.getNumberOfPlatformThreads(), + concurrencyConfig.getNumberOfPlatformThreads()), + LogLevel.DEBUG); try { for (int i = 0; i < concurrencyConfig.getNumberOfPlatformThreads(); i++) { startRuntimeLoopWithExecutor(lambdaRequestHandler, lambdaLogger, platformThreadExecutor, runtimeClient); @@ -373,4 +378,15 @@ private static void logExceptionCloudWatch(LambdaContextLogger lambdaLogger, Exc protected static URLClassLoader getCustomerClassLoader() { return customerClassLoader; } + + static class WorkerPoolInitializedEvent { + final String event = "runtime_worker_pool_initializing"; + final int workerCount; + final int executionEnvironmentMaxConcurrency; + + WorkerPoolInitializedEvent(int workerCount, int executionEnvironmentMaxConcurrency) { + this.workerCount = workerCount; + this.executionEnvironmentMaxConcurrency = executionEnvironmentMaxConcurrency; + } + } } diff --git a/aws-lambda-java-runtime-interface-client/src/main/java/com/amazonaws/services/lambda/runtime/api/client/logging/AbstractLambdaLogger.java b/aws-lambda-java-runtime-interface-client/src/main/java/com/amazonaws/services/lambda/runtime/api/client/logging/AbstractLambdaLogger.java index f987b0bdb..69b0d3a03 100644 --- a/aws-lambda-java-runtime-interface-client/src/main/java/com/amazonaws/services/lambda/runtime/api/client/logging/AbstractLambdaLogger.java +++ b/aws-lambda-java-runtime-interface-client/src/main/java/com/amazonaws/services/lambda/runtime/api/client/logging/AbstractLambdaLogger.java @@ -64,6 +64,12 @@ public void log(byte[] message) { this.log(message, LogLevel.UNDEFINED); } + public void logStructuredEvent(Object event, LogLevel logLevel) { + if (logFiltering.isEnabled(logLevel)) { + this.logMessage(logFormatter.format(event, logLevel), logLevel); + } + } + public void setLambdaContext(LambdaContext lambdaContext) { this.logFormatter.setLambdaContext(lambdaContext); } diff --git a/aws-lambda-java-runtime-interface-client/src/main/java/com/amazonaws/services/lambda/runtime/api/client/logging/JsonLogFormatter.java b/aws-lambda-java-runtime-interface-client/src/main/java/com/amazonaws/services/lambda/runtime/api/client/logging/JsonLogFormatter.java index f1051a216..f05c3317f 100644 --- a/aws-lambda-java-runtime-interface-client/src/main/java/com/amazonaws/services/lambda/runtime/api/client/logging/JsonLogFormatter.java +++ b/aws-lambda-java-runtime-interface-client/src/main/java/com/amazonaws/services/lambda/runtime/api/client/logging/JsonLogFormatter.java @@ -26,14 +26,22 @@ public class JsonLogFormatter implements LogFormatter { @Override public String format(String message, LogLevel logLevel) { + return serialize(createLogMessage(message, logLevel)); + } + + @Override + public String format(Object message, LogLevel logLevel) { + return serialize(createLogMessage(message, logLevel)); + } + + private String serialize(StructuredLogMessage msg) { ByteArrayOutputStream stream = new ByteArrayOutputStream(); - StructuredLogMessage msg = createLogMessage(message, logLevel); serializer.toJson(msg, stream); stream.write('\n'); return new String(stream.toByteArray(), StandardCharsets.UTF_8); } - private StructuredLogMessage createLogMessage(String message, LogLevel logLevel) { + private StructuredLogMessage createLogMessage(Object message, LogLevel logLevel) { StructuredLogMessage msg = new StructuredLogMessage(); msg.timestamp = dateFormatter.format(LocalDateTime.now()); msg.message = message; diff --git a/aws-lambda-java-runtime-interface-client/src/main/java/com/amazonaws/services/lambda/runtime/api/client/logging/LogFormatter.java b/aws-lambda-java-runtime-interface-client/src/main/java/com/amazonaws/services/lambda/runtime/api/client/logging/LogFormatter.java index 283b52289..d4f1a1e89 100644 --- a/aws-lambda-java-runtime-interface-client/src/main/java/com/amazonaws/services/lambda/runtime/api/client/logging/LogFormatter.java +++ b/aws-lambda-java-runtime-interface-client/src/main/java/com/amazonaws/services/lambda/runtime/api/client/logging/LogFormatter.java @@ -11,6 +11,10 @@ public interface LogFormatter { String format(String message, LogLevel logLevel); + default String format(Object message, LogLevel logLevel) { + return format(String.valueOf(message), logLevel); + } + default void setLambdaContext(LambdaContext context) { } } diff --git a/aws-lambda-java-runtime-interface-client/src/main/java/com/amazonaws/services/lambda/runtime/api/client/logging/StructuredLogMessage.java b/aws-lambda-java-runtime-interface-client/src/main/java/com/amazonaws/services/lambda/runtime/api/client/logging/StructuredLogMessage.java index 0ae19961f..68cf4d8e0 100644 --- a/aws-lambda-java-runtime-interface-client/src/main/java/com/amazonaws/services/lambda/runtime/api/client/logging/StructuredLogMessage.java +++ b/aws-lambda-java-runtime-interface-client/src/main/java/com/amazonaws/services/lambda/runtime/api/client/logging/StructuredLogMessage.java @@ -9,7 +9,7 @@ class StructuredLogMessage { public String timestamp; - public String message; + public Object message; public LogLevel level; public String AWSRequestId; public String tenantId; diff --git a/aws-lambda-java-runtime-interface-client/src/test/java/com/amazonaws/services/lambda/runtime/api/client/AWSLambdaTest.java b/aws-lambda-java-runtime-interface-client/src/test/java/com/amazonaws/services/lambda/runtime/api/client/AWSLambdaTest.java index 100465531..b5c522e71 100644 --- a/aws-lambda-java-runtime-interface-client/src/test/java/com/amazonaws/services/lambda/runtime/api/client/AWSLambdaTest.java +++ b/aws-lambda-java-runtime-interface-client/src/test/java/com/amazonaws/services/lambda/runtime/api/client/AWSLambdaTest.java @@ -576,6 +576,40 @@ void testSequentialWithVirtualMachineErrorStopsLoop() throws Throwable { assertEquals(2 * SampleHandler.nOfIterations, SampleHandler.globalCounter.get()); } + @Test + @Timeout(value = 1, unit = TimeUnit.MINUTES) + void testWorkerPoolInitializedEventEmittedOnceInMultiConcurrentMode() throws Throwable { + when(concurrencyConfig.isMultiConcurrent()).thenReturn(true); + when(concurrencyConfig.getNumberOfPlatformThreads()).thenReturn(4); + + when(runtimeClient.nextInvocationWithExponentialBackoff(lambdaLogger)) + .thenThrow(fakelambdaRuntimeClientMaxRetriesExceededException); + + AWSLambda.startRuntimeLoops(lambdaRequestHandler, lambdaLogger, concurrencyConfig, runtimeClient); + + org.mockito.ArgumentCaptor eventCaptor = org.mockito.ArgumentCaptor.forClass(Object.class); + verify(lambdaLogger, times(1)).logStructuredEvent(eventCaptor.capture(), eq(LogLevel.DEBUG)); + + AWSLambda.WorkerPoolInitializedEvent event = (AWSLambda.WorkerPoolInitializedEvent) eventCaptor.getValue(); + assertEquals("runtime_worker_pool_initializing", event.event); + assertEquals(4, event.workerCount); + assertEquals(4, event.executionEnvironmentMaxConcurrency); + } + + @Test + @Timeout(value = 1, unit = TimeUnit.MINUTES) + void testWorkerPoolInitializedEventNotEmittedInSequentialMode() throws Throwable { + when(concurrencyConfig.isMultiConcurrent()).thenReturn(false); + + InvocationRequest fatalRequest = mock(InvocationRequest.class); + when(fatalRequest.getId()).thenThrow(UserFault.makeUserFault(new IOError(new Throwable()), true)).thenReturn("fatal"); + when(runtimeClient.nextInvocation()).thenReturn(fatalRequest); + + AWSLambda.startRuntimeLoops(lambdaRequestHandler, lambdaLogger, concurrencyConfig, runtimeClient); + + verify(lambdaLogger, never()).logStructuredEvent(any(), any()); + } + @Test @Timeout(value = 1, unit = TimeUnit.MINUTES) void testInvocationIdIsPassedToReportSuccess() throws Throwable { From 5dd520e4995049c35cd1dfcbe234c4cd9c0030f2 Mon Sep 17 00:00:00 2001 From: Vipin Gupta Date: Thu, 3 Sep 2026 16:56:07 +0000 Subject: [PATCH 2/2] test: Gson serialization behavior for Object-typed log messages --- .../runtime/api/client/AWSLambdaTest.java | 23 ++++ .../client/logging/JsonLogFormatterTest.java | 116 ++++++++++++++++++ 2 files changed, 139 insertions(+) diff --git a/aws-lambda-java-runtime-interface-client/src/test/java/com/amazonaws/services/lambda/runtime/api/client/AWSLambdaTest.java b/aws-lambda-java-runtime-interface-client/src/test/java/com/amazonaws/services/lambda/runtime/api/client/AWSLambdaTest.java index b5c522e71..2f1348c17 100644 --- a/aws-lambda-java-runtime-interface-client/src/test/java/com/amazonaws/services/lambda/runtime/api/client/AWSLambdaTest.java +++ b/aws-lambda-java-runtime-interface-client/src/test/java/com/amazonaws/services/lambda/runtime/api/client/AWSLambdaTest.java @@ -610,6 +610,29 @@ void testWorkerPoolInitializedEventNotEmittedInSequentialMode() throws Throwable verify(lambdaLogger, never()).logStructuredEvent(any(), any()); } + /* + * Pins the exact wire format of the event through the real JSON formatter (Gson), + * proving the Object-typed StructuredLogMessage.message serializes the event as a + * nested JSON object with exactly the documented schema. + */ + @Test + void testWorkerPoolInitializedEventJsonWireFormat() { + com.amazonaws.services.lambda.runtime.api.client.logging.JsonLogFormatter formatter = + new com.amazonaws.services.lambda.runtime.api.client.logging.JsonLogFormatter(); + String output = formatter.format(new AWSLambda.WorkerPoolInitializedEvent(16, 16), LogLevel.DEBUG); + + com.amazonaws.lambda.thirdparty.org.json.JSONObject parsed = + new com.amazonaws.lambda.thirdparty.org.json.JSONObject(output); + assertEquals("DEBUG", parsed.getString("level")); + org.junit.jupiter.api.Assertions.assertNotNull(parsed.getString("timestamp")); + + com.amazonaws.lambda.thirdparty.org.json.JSONObject message = parsed.getJSONObject("message"); + assertEquals("runtime_worker_pool_initializing", message.getString("event")); + assertEquals(16, message.getInt("workerCount")); + assertEquals(16, message.getInt("executionEnvironmentMaxConcurrency")); + assertEquals(3, message.length()); + } + @Test @Timeout(value = 1, unit = TimeUnit.MINUTES) void testInvocationIdIsPassedToReportSuccess() throws Throwable { diff --git a/aws-lambda-java-runtime-interface-client/src/test/java/com/amazonaws/services/lambda/runtime/api/client/logging/JsonLogFormatterTest.java b/aws-lambda-java-runtime-interface-client/src/test/java/com/amazonaws/services/lambda/runtime/api/client/logging/JsonLogFormatterTest.java index 91ce9d2a3..26ab20ce7 100644 --- a/aws-lambda-java-runtime-interface-client/src/test/java/com/amazonaws/services/lambda/runtime/api/client/logging/JsonLogFormatterTest.java +++ b/aws-lambda-java-runtime-interface-client/src/test/java/com/amazonaws/services/lambda/runtime/api/client/logging/JsonLogFormatterTest.java @@ -1,17 +1,133 @@ package com.amazonaws.services.lambda.runtime.api.client.logging; +import com.amazonaws.lambda.thirdparty.org.json.JSONObject; import com.amazonaws.services.lambda.runtime.api.client.api.LambdaContext; import com.amazonaws.services.lambda.runtime.serialization.PojoSerializer; import com.amazonaws.services.lambda.runtime.serialization.factories.GsonFactory; import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import com.amazonaws.services.lambda.runtime.logging.LogLevel; public class JsonLogFormatterTest { + /* + * Gson serialization-neutrality tests for the String -> Object type change of + * StructuredLogMessage.message. These pin the exact serialized behavior so any + * future regression (e.g. from a Gson upgrade changing runtime-type adapter + * resolution) is caught here rather than in customers' log pipelines. + */ + + @Test + void testStringMessageStillSerializesAsJsonStringWithObjectDeclaredField() { + JsonLogFormatter formatter = new JsonLogFormatter(); + String output = formatter.format("test log", LogLevel.INFO); + + JSONObject parsed = new JSONObject(output); + // message must be a JSON string value, not an object or anything else + assertTrue(parsed.get("message") instanceof String); + assertEquals("test log", parsed.getString("message")); + assertEquals("INFO", parsed.getString("level")); + } + + @Test + void testStringMessageEscapingUnchangedWithObjectDeclaredField() { + // quotes, backslashes, newlines, unicode and HTML-sensitive chars must + // round-trip exactly as before the field type change + String tricky = "quote\" backslash\\ newline\n tab\t unicode\u00e9 html<>&"; + JsonLogFormatter formatter = new JsonLogFormatter(); + String output = formatter.format(tricky, LogLevel.WARN); + + JSONObject parsed = new JSONObject(output); + assertEquals(tricky, parsed.getString("message")); + } + + @Test + void testNullStringMessageOmittedExactlyAsBefore() { + // serializeNulls(false) omitted a null String message before the change; + // a null Object message must behave identically + JsonLogFormatter formatter = new JsonLogFormatter(); + String output = formatter.format((String) null, LogLevel.INFO); + + JSONObject parsed = new JSONObject(output); + assertFalse(parsed.has("message")); + assertNotNull(parsed.getString("timestamp")); + } + + static class SampleStructuredEvent { + final String event = "sample_event"; + final int intValue; + final String textValue; + + SampleStructuredEvent(int intValue, String textValue) { + this.intValue = intValue; + this.textValue = textValue; + } + } + + @Test + void testObjectMessageSerializesAsNestedJsonObject() { + JsonLogFormatter formatter = new JsonLogFormatter(); + String output = formatter.format(new SampleStructuredEvent(42, "abc"), LogLevel.DEBUG); + + JSONObject parsed = new JSONObject(output); + JSONObject message = parsed.getJSONObject("message"); + assertEquals("sample_event", message.getString("event")); + assertEquals(42, message.getInt("intValue")); + assertEquals("abc", message.getString("textValue")); + assertEquals(3, message.length()); + assertEquals("DEBUG", parsed.getString("level")); + assertNotNull(parsed.getString("timestamp")); + } + + @Test + void testObjectMessageWithLambdaContextKeepsEnvelopeFields() { + JsonLogFormatter formatter = new JsonLogFormatter(); + formatter.setLambdaContext(new LambdaContext( + 0, 0, "request-id", null, null, "function-name", + null, null, "function-arn", "tenant-id", null, null)); + String output = formatter.format(new SampleStructuredEvent(1, "x"), LogLevel.DEBUG); + + JSONObject parsed = new JSONObject(output); + assertEquals("request-id", parsed.getString("AWSRequestId")); + assertEquals("tenant-id", parsed.getString("tenantId")); + assertEquals(1, parsed.getJSONObject("message").getInt("intValue")); + } + + @Test + void testReflectiveStringAccessToMessageFieldStillWorks() throws Exception { + // StructuredLogMessage is internal, but be conservative about reflective + // consumers: setting and reading a String through the field must not break. + StructuredLogMessage msg = new StructuredLogMessage(); + java.lang.reflect.Field field = StructuredLogMessage.class.getDeclaredField("message"); + field.set(msg, "reflective string"); + assertEquals("reflective string", (String) field.get(msg)); + + PojoSerializer serializer = + GsonFactory.getInstance().getSerializer(StructuredLogMessage.class); + java.io.ByteArrayOutputStream stream = new java.io.ByteArrayOutputStream(); + serializer.toJson(msg, stream); + JSONObject parsed = new JSONObject(stream.toString("UTF-8")); + assertEquals("reflective string", parsed.getString("message")); + } + + @Test + void testStringMessageDeserializationRoundTripUnchanged() { + // fromJson of a string-message log line must still yield a String in the field + JsonLogFormatter formatter = new JsonLogFormatter(); + String output = formatter.format("round trip", LogLevel.INFO); + + PojoSerializer serializer = + GsonFactory.getInstance().getSerializer(StructuredLogMessage.class); + StructuredLogMessage result = serializer.fromJson(output); + assertTrue(result.message instanceof String); + assertEquals("round trip", result.message); + } + @Test void testFormattingWithoutLambdaContext() { assertFormatsString("test log", LogLevel.WARN, null);