From d47e011e90fa30245a0da8341b51f7d1861be5db Mon Sep 17 00:00:00 2001 From: Zakaria Khchiche <45357915+Zakariakhchiche@users.noreply.github.com> Date: Thu, 15 Oct 2026 13:17:57 +0200 Subject: [PATCH 1/4] Python: TypeSafe: support inline images and Box questions (Jev 1.14) --- .../agent_framework_typesafe/_chat_client.py | 45 ++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/python/packages/typesafe/agent_framework_typesafe/_chat_client.py b/python/packages/typesafe/agent_framework_typesafe/_chat_client.py index 51237c00cdb..c09fd53e142 100644 --- a/python/packages/typesafe/agent_framework_typesafe/_chat_client.py +++ b/python/packages/typesafe/agent_framework_typesafe/_chat_client.py @@ -38,6 +38,7 @@ from typesafe_sdk import ( AsyncTypeSafeClient, ChoiceAnswer, + Image, Questions, ScoreAnswer, SystemOneResponse, @@ -57,6 +58,9 @@ _TYPESAFE_DEFAULT_BASE_URL = "https://api.typesafe.ai" _TYPESAFE_SYSTEM_ONE_PATH = "/v1/systemone" +# TypeSafe gates image assets and Box questions behind this opt-in header. +# See https://docs.typesafe.ai/go/images and https://docs.typesafe.ai/go/box. +_TYPESAFE_IMAGES_HEADER = "TypeSafe-Danger-Zone-Images" class TypeSafeSettings(TypedDict, total=False): @@ -244,12 +248,19 @@ async def _get_response() -> ChatResponse[Any]: instructions=normalized_options.get("instructions"), ) + extra_headers = ( + {_TYPESAFE_IMAGES_HEADER: "true"} + if self._state_has_images(state) or self._has_box_questions(questions) + else None + ) + try: response = await self.client.system_one( # pyright: ignore[reportUnknownMemberType] state=state, questions=questions, model=model, response_model=SystemOneResponse, + extra_headers=extra_headers, ) except TypeSafeError as exc: self._raise_sdk_error(exc) @@ -389,10 +400,12 @@ def _build_state(messages: Sequence[Message], *, instructions: Any) -> dict[str, content.type for content in message.contents if content.type not in {"text", "text_reasoning", "function_call", "function_result"} + and not RawTypeSafeChatClient._is_inline_image(content) }) if unsupported_content_types: raise ChatClientInvalidRequestException( - f"TypeSafe only supports text, reasoning summaries, and function call/result content; " + f"TypeSafe only supports text, inline images, reasoning summaries, and function call/result " + f"content; " f"message {index} contains: " f"{', '.join(unsupported_content_types)}." ) @@ -412,6 +425,8 @@ def _build_state(messages: Sequence[Message], *, instructions: Any) -> dict[str, }) elif content.type == "function_result": contents.append(RawTypeSafeChatClient._serialize_function_result(content)) + elif RawTypeSafeChatClient._is_inline_image(content): + contents.append({"type": "image", "image": RawTypeSafeChatClient._to_image(content)}) if contents: state_messages.append({ "role": str(message.role), @@ -426,6 +441,34 @@ def _build_state(messages: Sequence[Message], *, instructions: Any) -> dict[str, state["instructions"] = instructions return state + @staticmethod + def _is_inline_image(content: Content) -> bool: + return content.type == "data" and (content.media_type or "").startswith("image/") + + @staticmethod + def _to_image(content: Content) -> Image: + uri = content.uri or "" + header, separator, payload = uri.partition(",") + if not separator or not header.endswith(";base64"): + raise ChatClientInvalidRequestException("TypeSafe image content must be a base64 data URI.") + try: + return Image.from_base64(payload) + except Exception as exc: + raise ChatClientInvalidRequestException(f"TypeSafe could not read image content: {exc}") from exc + + @staticmethod + def _state_has_images(state: Mapping[str, Any]) -> bool: + return any( + item.get("type") == "image" for message in state.get("messages", ()) for item in message.get("contents", ()) + ) + + @staticmethod + def _has_box_questions(questions: Questions) -> bool: + return any( + (question.get("type") if isinstance(question, Mapping) else getattr(question, "type", None)) == "box" + for question in questions.values() + ) + @staticmethod def _serialize_function_result(content: Content) -> dict[str, Any]: items: list[dict[str, str]] = [] From 15f21948cf1f2203ac52081da400c874ccb86191 Mon Sep 17 00:00:00 2001 From: Zakaria Khchiche <45357915+Zakariakhchiche@users.noreply.github.com> Date: Thu, 15 Oct 2026 13:18:19 +0200 Subject: [PATCH 2/4] Python: TypeSafe: add tests for inline images and Box questions --- .../tests/typesafe/test_chat_client.py | 107 +++++++++++++++++- 1 file changed, 106 insertions(+), 1 deletion(-) diff --git a/python/packages/typesafe/tests/typesafe/test_chat_client.py b/python/packages/typesafe/tests/typesafe/test_chat_client.py index 0b58cdbc9ba..98d70b1614d 100644 --- a/python/packages/typesafe/tests/typesafe/test_chat_client.py +++ b/python/packages/typesafe/tests/typesafe/test_chat_client.py @@ -22,7 +22,9 @@ from pydantic import BaseModel from typesafe_sdk import ( AsyncTypeSafeClient, + Box, Choice, + Image, Noul, Questions, SystemOneResponse, @@ -396,7 +398,109 @@ async def test_non_text_content_is_rejected() -> None: client = make_client() message = Message("user", [Content.from_uri("https://example.com/image.png", media_type="image/png")]) - with pytest.raises(ChatClientInvalidRequestException, match="only supports text, reasoning summaries"): + with pytest.raises(ChatClientInvalidRequestException, match="only supports text, inline images"): + await client.get_response([message], options={"response_format": questions()}) + + +_PNG_1X1 = ( + b"\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x01\x00\x00\x00\x01\x08\x06\x00\x00\x00\x1f\x15\xc4\x89" + b"\x00\x00\x00\rIDATx\x9cc\xf8\x0f\x00\x00\x01\x01\x00\x05\x18\xd8N\x00\x00\x00\x00IEND\xaeB`\x82" +) + + +async def test_inline_image_content_is_sent_as_image_asset_with_opt_in_header() -> None: + stub = StubTypeSafeClient() + client = make_client(stub) + message = Message( + "user", + ["Is the form signed?", Content.from_data(data=_PNG_1X1, media_type="image/png")], + ) + + await client.get_response([message], options={"response_format": questions()}) + + contents = stub.calls[0]["state"]["messages"][0]["contents"] + assert contents[0] == {"type": "text", "text": "Is the form signed?"} + assert contents[1]["type"] == "image" + assert isinstance(contents[1]["image"], Image) + assert stub.calls[0]["extra_headers"] == {"TypeSafe-Danger-Zone-Images": "true"} + + +async def test_box_question_sends_opt_in_header_and_returns_box_answer() -> None: + stub = StubTypeSafeClient( + response=make_response({ + "signature": { + "type": "box", + "box": {"top": 0.7, "bottom": 0.9, "left": 0.5, "right": 0.95}, + "confidence": 0.88, + } + }) + ) + client = make_client(stub, model="jev-1.14.0") + message = Message("user", [Content.from_data(data=_PNG_1X1, media_type="image/png")]) + + response = await client.get_response( + [message], + options={"response_format": {"signature": Box(instructions="the signature on the form")}}, + ) + + assert stub.calls[0]["extra_headers"] == {"TypeSafe-Danger-Zone-Images": "true"} + assert isinstance(response.value, SystemOneResponse) + answer = response.value.answers["signature"] + assert answer.type == "box" + assert answer.box.right == pytest.approx(0.95) + + +async def test_text_only_requests_do_not_send_images_header() -> None: + stub = StubTypeSafeClient() + client = make_client(stub) + + await client.get_response([Message("user", ["hello"])], options={"response_format": questions()}) + + assert stub.calls[0]["extra_headers"] is None + + +async def test_owned_client_sends_images_header_and_image_part() -> None: + received: list[tuple[dict[str, str], bytes]] = [] + body = make_response().model_dump_json().encode() + + async def handle_request(reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: + await reader.readline() + headers: dict[str, str] = {} + while (line := await reader.readline()) not in (b"\r\n", b""): + name, _, value = line.decode("latin-1").partition(":") + headers[name.strip().lower()] = value.strip() + payload = await reader.readexactly(int(headers.get("content-length", "0"))) + received.append((headers, payload)) + writer.write( + b"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nx-typesafe-request-id: request-123\r\n" + + f"Content-Length: {len(body)}\r\nConnection: close\r\n\r\n".encode() + + body + ) + await writer.drain() + writer.close() + await writer.wait_closed() + + server = await asyncio.start_server(handle_request, "127.0.0.1", 0) + port = server.sockets[0].getsockname()[1] + async with ( + server, + TypeSafeChatClient(api_key="configured-api-key", base_url=f"http://127.0.0.1:{port}") as client, + ): + await client.get_response( + [Message("user", ["Is the form signed?", Content.from_data(data=_PNG_1X1, media_type="image/png")])], + options={"response_format": questions()}, + ) + + headers, payload = received[0] + assert headers.get("typesafe-danger-zone-images") == "true" + assert _PNG_1X1 in payload + + +async def test_remote_image_uri_is_rejected() -> None: + client = make_client() + message = Message("user", [Content.from_uri("https://example.com/image.png", media_type="image/png")]) + + with pytest.raises(ChatClientInvalidRequestException, match="inline images"): await client.get_response([message], options={"response_format": questions()}) @@ -559,6 +663,7 @@ async def test_request_and_response_mapping() -> None: "questions": questions(), "model": "jev-preview", "response_model": SystemOneResponse, + "extra_headers": None, } ] assert response.response_id == "request-123" From d2a6b66a6e781a4a7e4100dac59a15e02f49c107 Mon Sep 17 00:00:00 2001 From: Zakaria Khchiche <45357915+Zakariakhchiche@users.noreply.github.com> Date: Thu, 15 Oct 2026 13:18:39 +0200 Subject: [PATCH 3/4] Python: TypeSafe: document inline images and Box questions --- python/packages/typesafe/AGENTS.md | 6 +++-- python/packages/typesafe/README.md | 39 +++++++++++++++++++++++++++--- 2 files changed, 40 insertions(+), 5 deletions(-) diff --git a/python/packages/typesafe/AGENTS.md b/python/packages/typesafe/AGENTS.md index 3b8f7a96c32..ce531432008 100644 --- a/python/packages/typesafe/AGENTS.md +++ b/python/packages/typesafe/AGENTS.md @@ -12,8 +12,10 @@ Integration with TypeSafe AI System One models, including Jev. ## Behavioral Contract - Calls always return structured `SystemOneResponse` data. This provider does not generate free-form chat text. -- Every call requires `response_format` to contain at least one TypeSafe `Noul`, `Choice`, or `Score` question. -- Streaming and non-text message content are not supported. +- Every call requires `response_format` to contain at least one TypeSafe `Noul`, `Choice`, `Score`, or `Box` question. +- Streaming is not supported. Non-text message content is rejected, except inline image data content, which is + sent as TypeSafe image assets. Requests with an image or a `Box` question send the + `TypeSafe-Danger-Zone-Images: true` opt-in header. Remote image URIs are rejected. - Tool calling defaults to one call per run; callers can opt into sequential round trips with `FunctionInvocationConfiguration.max_function_calls`. Closed-set schemas support constants, enums/Literals, booleans, and arrays of enums/Literals. Optional supported arguments use a presence question so omitted values diff --git a/python/packages/typesafe/README.md b/python/packages/typesafe/README.md index f6a1d52202e..597325c5476 100644 --- a/python/packages/typesafe/README.md +++ b/python/packages/typesafe/README.md @@ -74,12 +74,45 @@ honored. | Option | Description | | --- | --- | -| `response_format` | Required non-empty TypeSafe `Questions` mapping containing `Noul`, `Choice`, or `Score` questions. | +| `response_format` | Required non-empty TypeSafe `Questions` mapping containing `Noul`, `Choice`, `Score`, or `Box` questions. | | `model` | Optional per-call model override. | | `instructions` | Agent instructions included in the structured state sent to TypeSafe. | -Streaming, non-text message content, and generative settings such as `temperature` -are rejected. +Streaming, non-text message content other than inline images, and generative +settings such as `temperature` are rejected. + +## Images and Box questions (experimental) + +Jev 1.14 adds image inputs and the `Box` question type, which locates one target +in an image and returns its bounding box. TypeSafe marks both as experimental +("Danger Zone") features that are not yet recommended for production. See +[Images](https://docs.typesafe.ai/go/images) and +[Box](https://docs.typesafe.ai/go/box). + +Pass images as inline data content. The connector sends them as TypeSafe image +assets and adds the required `TypeSafe-Danger-Zone-Images: true` header to any +request that contains an image or a `Box` question. Remote image URLs are +rejected; download the image and pass its bytes instead. + +```python +from agent_framework import Content, Message +from typesafe_sdk import Box, Noul + +response = await client.get_response( + [Message("user", ["The scanned form:", Content.from_data(data=png_bytes, media_type="image/png")])], + options={ + "model": "jev-1.14.0", + "response_format": { + "signed": Noul(instructions="Is the form signed?"), + "signature": Box(instructions="the signature on the form"), + }, + }, +) +signature = response.value.answers["signature"] # BoxAnswer with top/bottom/left/right fractions +``` + +A `Box` answer always contains a box, even when the target is absent. Pair it +with a `Noul` question, as above, and ignore the box when that answer is low. `TypeSafeChatClient` is the recommended client and layers function invocation, middleware, and telemetry over `RawTypeSafeChatClient`. Use the raw client only From 2d072f2f0d08ba1bf1e64e0ec16aa260509c8efd Mon Sep 17 00:00:00 2001 From: Zakaria Khchiche <45357915+Zakariakhchiche@users.noreply.github.com> Date: Thu, 15 Oct 2026 13:49:10 +0200 Subject: [PATCH 4/4] Python: TypeSafe: use a valid PNG fixture and cover Box-only header detection --- .../tests/typesafe/test_chat_client.py | 56 ++++++++++++++++++- 1 file changed, 55 insertions(+), 1 deletion(-) diff --git a/python/packages/typesafe/tests/typesafe/test_chat_client.py b/python/packages/typesafe/tests/typesafe/test_chat_client.py index 98d70b1614d..3fe8849d2f0 100644 --- a/python/packages/typesafe/tests/typesafe/test_chat_client.py +++ b/python/packages/typesafe/tests/typesafe/test_chat_client.py @@ -404,10 +404,31 @@ async def test_non_text_content_is_rejected() -> None: _PNG_1X1 = ( b"\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x01\x00\x00\x00\x01\x08\x06\x00\x00\x00\x1f\x15\xc4\x89" - b"\x00\x00\x00\rIDATx\x9cc\xf8\x0f\x00\x00\x01\x01\x00\x05\x18\xd8N\x00\x00\x00\x00IEND\xaeB`\x82" + b"\x00\x00\x00\x0bIDATx\x9cc`\x00\x02\x00\x00\x05\x00\x01z^\xab?\x00\x00\x00\x00IEND\xaeB`\x82" ) +def test_png_fixture_is_a_decodable_image() -> None: + import struct + import zlib + + assert _PNG_1X1.startswith(b"\x89PNG\r\n\x1a\n") + offset = 8 + chunk_types: list[bytes] = [] + while offset < len(_PNG_1X1): + (length,) = struct.unpack(">I", _PNG_1X1[offset : offset + 4]) + chunk_type = _PNG_1X1[offset + 4 : offset + 8] + data = _PNG_1X1[offset + 8 : offset + 8 + length] + (crc,) = struct.unpack(">I", _PNG_1X1[offset + 8 + length : offset + 12 + length]) + assert zlib.crc32(chunk_type + data) & 0xFFFFFFFF == crc + if chunk_type == b"IDAT": + assert zlib.decompress(data) == b"\x00\x00\x00\x00\x00" + chunk_types.append(chunk_type) + offset += 12 + length + assert offset == len(_PNG_1X1) + assert chunk_types == [b"IHDR", b"IDAT", b"IEND"] + + async def test_inline_image_content_is_sent_as_image_asset_with_opt_in_header() -> None: stub = StubTypeSafeClient() client = make_client(stub) @@ -425,6 +446,39 @@ async def test_inline_image_content_is_sent_as_image_asset_with_opt_in_header() assert stub.calls[0]["extra_headers"] == {"TypeSafe-Danger-Zone-Images": "true"} +@pytest.mark.parametrize( + ("box_question", "with_image"), + [ + (Box(instructions="the signature"), False), + ({"type": "box", "instructions": "the signature"}, False), + (Box(instructions="the signature"), True), + ({"type": "box", "instructions": "the signature"}, True), + ], + ids=["sdk-object-text-only", "mapping-text-only", "sdk-object-with-image", "mapping-with-image"], +) +async def test_box_question_alone_triggers_opt_in_header(box_question: Any, with_image: bool) -> None: + stub = StubTypeSafeClient( + response=make_response({ + "signature": { + "type": "box", + "box": {"top": 0.1, "bottom": 0.2, "left": 0.3, "right": 0.4}, + "confidence": 0.5, + } + }) + ) + client = make_client(stub) + contents: list[Any] = ["Locate the signature."] + if with_image: + contents.append(Content.from_data(data=_PNG_1X1, media_type="image/png")) + + await client.get_response( + [Message("user", contents)], + options={"response_format": {"signature": box_question}}, + ) + + assert stub.calls[0]["extra_headers"] == {"TypeSafe-Danger-Zone-Images": "true"} + + async def test_box_question_sends_opt_in_header_and_returns_box_answer() -> None: stub = StubTypeSafeClient( response=make_response({