Python: TypeSafe: support inline images and Box questions (Jev 1.14)

python/packages/typesafe/AGENTS.md

@@ -12,8 +12,10 @@ Integration with TypeSafe AI System One models, including Jev.
 ## Behavioral Contract
 
 - Calls always return structured `SystemOneResponse` data. This provider does not generate free-form chat text.
-- Every call requires `response_format` to contain at least one TypeSafe `Noul`, `Choice`, or `Score` question.
-- Streaming and non-text message content are not supported.
+- Every call requires `response_format` to contain at least one TypeSafe `Noul`, `Choice`, `Score`, or `Box` question.
+- Streaming is not supported. Non-text message content is rejected, except inline image data content, which is
+  sent as TypeSafe image assets. Requests with an image or a `Box` question send the
+  `TypeSafe-Danger-Zone-Images: true` opt-in header. Remote image URIs are rejected.
 - Tool calling defaults to one call per run; callers can opt into sequential round trips with
   `FunctionInvocationConfiguration.max_function_calls`. Closed-set schemas support constants, enums/Literals,
   booleans, and arrays of enums/Literals. Optional supported arguments use a presence question so omitted values

python/packages/typesafe/README.md

@@ -74,12 +74,45 @@ honored.
 
 | Option | Description |
 | --- | --- |
-| `response_format` | Required non-empty TypeSafe `Questions` mapping containing `Noul`, `Choice`, or `Score` questions. |
+| `response_format` | Required non-empty TypeSafe `Questions` mapping containing `Noul`, `Choice`, `Score`, or `Box` questions. |
 | `model` | Optional per-call model override. |
 | `instructions` | Agent instructions included in the structured state sent to TypeSafe. |
 
-Streaming, non-text message content, and generative settings such as `temperature`
-are rejected.
+Streaming, non-text message content other than inline images, and generative
+settings such as `temperature` are rejected.
+
+## Images and Box questions (experimental)
+
+Jev 1.14 adds image inputs and the `Box` question type, which locates one target
+in an image and returns its bounding box. TypeSafe marks both as experimental
+("Danger Zone") features that are not yet recommended for production. See
+[Images](https://docs.typesafe.ai/go/images) and
+[Box](https://docs.typesafe.ai/go/box).
+
+Pass images as inline data content. The connector sends them as TypeSafe image
+assets and adds the required `TypeSafe-Danger-Zone-Images: true` header to any
+request that contains an image or a `Box` question. Remote image URLs are
+rejected; download the image and pass its bytes instead.
+
+```python
+from agent_framework import Content, Message
+from typesafe_sdk import Box, Noul
+
+response = await client.get_response(
+    [Message("user", ["The scanned form:", Content.from_data(data=png_bytes, media_type="image/png")])],
+    options={
+        "model": "jev-1.14.0",
+        "response_format": {
+            "signed": Noul(instructions="Is the form signed?"),
+            "signature": Box(instructions="the signature on the form"),
+        },
+    },
+)
+signature = response.value.answers["signature"]  # BoxAnswer with top/bottom/left/right fractions
+```
+
+A `Box` answer always contains a box, even when the target is absent. Pair it
+with a `Noul` question, as above, and ignore the box when that answer is low.
 
 `TypeSafeChatClient` is the recommended client and layers function invocation,
 middleware, and telemetry over `RawTypeSafeChatClient`. Use the raw client only

python/packages/typesafe/agent_framework_typesafe/_chat_client.py

@@ -38,6 +38,7 @@
 from typesafe_sdk import (
     AsyncTypeSafeClient,
     ChoiceAnswer,
+    Image,
     Questions,
     ScoreAnswer,
     SystemOneResponse,
@@ -57,6 +58,9 @@
 
 _TYPESAFE_DEFAULT_BASE_URL = "https://api.typesafe.ai"
 _TYPESAFE_SYSTEM_ONE_PATH = "/v1/systemone"
+# TypeSafe gates image assets and Box questions behind this opt-in header.
+# See https://docs.typesafe.ai/go/images and https://docs.typesafe.ai/go/box.
+_TYPESAFE_IMAGES_HEADER = "TypeSafe-Danger-Zone-Images"
 
 
 class TypeSafeSettings(TypedDict, total=False):
@@ -244,12 +248,19 @@ async def _get_response() -> ChatResponse[Any]:
                 instructions=normalized_options.get("instructions"),
             )
 
+            extra_headers = (
+                {_TYPESAFE_IMAGES_HEADER: "true"}
+                if self._state_has_images(state) or self._has_box_questions(questions)
+                else None
+            )
+
             try:
                 response = await self.client.system_one(  # pyright: ignore[reportUnknownMemberType]
                     state=state,
                     questions=questions,
                     model=model,
                     response_model=SystemOneResponse,
+                    extra_headers=extra_headers,
                 )
             except TypeSafeError as exc:
                 self._raise_sdk_error(exc)
@@ -389,10 +400,12 @@ def _build_state(messages: Sequence[Message], *, instructions: Any) -> dict[str,
                 content.type
                 for content in message.contents
                 if content.type not in {"text", "text_reasoning", "function_call", "function_result"}
+                and not RawTypeSafeChatClient._is_inline_image(content)
             })
             if unsupported_content_types:
                 raise ChatClientInvalidRequestException(
-                    f"TypeSafe only supports text, reasoning summaries, and function call/result content; "
+                    f"TypeSafe only supports text, inline images, reasoning summaries, and function call/result "
+                    f"content; "
                     f"message {index} contains: "
                     f"{', '.join(unsupported_content_types)}."
                 )
@@ -412,6 +425,8 @@ def _build_state(messages: Sequence[Message], *, instructions: Any) -> dict[str,
                     })
                 elif content.type == "function_result":
                     contents.append(RawTypeSafeChatClient._serialize_function_result(content))
+                elif RawTypeSafeChatClient._is_inline_image(content):
+                    contents.append({"type": "image", "image": RawTypeSafeChatClient._to_image(content)})
             if contents:
                 state_messages.append({
                     "role": str(message.role),
@@ -426,6 +441,34 @@ def _build_state(messages: Sequence[Message], *, instructions: Any) -> dict[str,
             state["instructions"] = instructions
         return state
 
+    @staticmethod
+    def _is_inline_image(content: Content) -> bool:
+        return content.type == "data" and (content.media_type or "").startswith("image/")
+
+    @staticmethod
+    def _to_image(content: Content) -> Image:
+        uri = content.uri or ""
+        header, separator, payload = uri.partition(",")
+        if not separator or not header.endswith(";base64"):
+            raise ChatClientInvalidRequestException("TypeSafe image content must be a base64 data URI.")
+        try:
+            return Image.from_base64(payload)
+        except Exception as exc:
+            raise ChatClientInvalidRequestException(f"TypeSafe could not read image content: {exc}") from exc
+
+    @staticmethod
+    def _state_has_images(state: Mapping[str, Any]) -> bool:
+        return any(
+            item.get("type") == "image" for message in state.get("messages", ()) for item in message.get("contents", ())
+        )
+
+    @staticmethod
+    def _has_box_questions(questions: Questions) -> bool:
+        return any(
+            (question.get("type") if isinstance(question, Mapping) else getattr(question, "type", None)) == "box"
+            for question in questions.values()
+        )
+
     @staticmethod
     def _serialize_function_result(content: Content) -> dict[str, Any]:
         items: list[dict[str, str]] = []

python/packages/typesafe/tests/typesafe/test_chat_client.py

@@ -22,7 +22,9 @@
 from pydantic import BaseModel
 from typesafe_sdk import (
     AsyncTypeSafeClient,
+    Box,
     Choice,
+    Image,
     Noul,
     Questions,
     SystemOneResponse,
@@ -396,7 +398,163 @@ async def test_non_text_content_is_rejected() -> None:
     client = make_client()
     message = Message("user", [Content.from_uri("https://example.com/image.png", media_type="image/png")])
 
-    with pytest.raises(ChatClientInvalidRequestException, match="only supports text, reasoning summaries"):
+    with pytest.raises(ChatClientInvalidRequestException, match="only supports text, inline images"):
+        await client.get_response([message], options={"response_format": questions()})
+
+
+_PNG_1X1 = (
+    b"\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x01\x00\x00\x00\x01\x08\x06\x00\x00\x00\x1f\x15\xc4\x89"
+    b"\x00\x00\x00\x0bIDATx\x9cc`\x00\x02\x00\x00\x05\x00\x01z^\xab?\x00\x00\x00\x00IEND\xaeB`\x82"
+)
+
+
+def test_png_fixture_is_a_decodable_image() -> None:
+    import struct
+    import zlib
+
+    assert _PNG_1X1.startswith(b"\x89PNG\r\n\x1a\n")
+    offset = 8
+    chunk_types: list[bytes] = []
+    while offset < len(_PNG_1X1):
+        (length,) = struct.unpack(">I", _PNG_1X1[offset : offset + 4])
+        chunk_type = _PNG_1X1[offset + 4 : offset + 8]
+        data = _PNG_1X1[offset + 8 : offset + 8 + length]
+        (crc,) = struct.unpack(">I", _PNG_1X1[offset + 8 + length : offset + 12 + length])
+        assert zlib.crc32(chunk_type + data) & 0xFFFFFFFF == crc
+        if chunk_type == b"IDAT":
+            assert zlib.decompress(data) == b"\x00\x00\x00\x00\x00"
+        chunk_types.append(chunk_type)
+        offset += 12 + length
+    assert offset == len(_PNG_1X1)
+    assert chunk_types == [b"IHDR", b"IDAT", b"IEND"]
+
+
+async def test_inline_image_content_is_sent_as_image_asset_with_opt_in_header() -> None:
+    stub = StubTypeSafeClient()
+    client = make_client(stub)
+    message = Message(
+        "user",
+        ["Is the form signed?", Content.from_data(data=_PNG_1X1, media_type="image/png")],
+    )
+
+    await client.get_response([message], options={"response_format": questions()})
+
+    contents = stub.calls[0]["state"]["messages"][0]["contents"]
+    assert contents[0] == {"type": "text", "text": "Is the form signed?"}
+    assert contents[1]["type"] == "image"
+    assert isinstance(contents[1]["image"], Image)
+    assert stub.calls[0]["extra_headers"] == {"TypeSafe-Danger-Zone-Images": "true"}
+
+
+@pytest.mark.parametrize(
+    ("box_question", "with_image"),
+    [
+        (Box(instructions="the signature"), False),
+        ({"type": "box", "instructions": "the signature"}, False),
+        (Box(instructions="the signature"), True),
+        ({"type": "box", "instructions": "the signature"}, True),
+    ],
+    ids=["sdk-object-text-only", "mapping-text-only", "sdk-object-with-image", "mapping-with-image"],
+)
+async def test_box_question_alone_triggers_opt_in_header(box_question: Any, with_image: bool) -> None:
+    stub = StubTypeSafeClient(
+        response=make_response({
+            "signature": {
+                "type": "box",
+                "box": {"top": 0.1, "bottom": 0.2, "left": 0.3, "right": 0.4},
+                "confidence": 0.5,
+            }
+        })
+    )
+    client = make_client(stub)
+    contents: list[Any] = ["Locate the signature."]
+    if with_image:
+        contents.append(Content.from_data(data=_PNG_1X1, media_type="image/png"))
+
+    await client.get_response(
+        [Message("user", contents)],
+        options={"response_format": {"signature": box_question}},
+    )
+
+    assert stub.calls[0]["extra_headers"] == {"TypeSafe-Danger-Zone-Images": "true"}
+
+
+async def test_box_question_sends_opt_in_header_and_returns_box_answer() -> None:
+    stub = StubTypeSafeClient(
+        response=make_response({
+            "signature": {
+                "type": "box",
+                "box": {"top": 0.7, "bottom": 0.9, "left": 0.5, "right": 0.95},
+                "confidence": 0.88,
+            }
+        })
+    )
+    client = make_client(stub, model="jev-1.14.0")
+    message = Message("user", [Content.from_data(data=_PNG_1X1, media_type="image/png")])
+
+    response = await client.get_response(
+        [message],
+        options={"response_format": {"signature": Box(instructions="the signature on the form")}},
+    )
+
+    assert stub.calls[0]["extra_headers"] == {"TypeSafe-Danger-Zone-Images": "true"}
+    assert isinstance(response.value, SystemOneResponse)
+    answer = response.value.answers["signature"]
+    assert answer.type == "box"
+    assert answer.box.right == pytest.approx(0.95)
+
+
+async def test_text_only_requests_do_not_send_images_header() -> None:
+    stub = StubTypeSafeClient()
+    client = make_client(stub)
+
+    await client.get_response([Message("user", ["hello"])], options={"response_format": questions()})
+
+    assert stub.calls[0]["extra_headers"] is None
+
+
+async def test_owned_client_sends_images_header_and_image_part() -> None:
+    received: list[tuple[dict[str, str], bytes]] = []
+    body = make_response().model_dump_json().encode()
+
+    async def handle_request(reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None:
+        await reader.readline()
+        headers: dict[str, str] = {}
+        while (line := await reader.readline()) not in (b"\r\n", b""):
+            name, _, value = line.decode("latin-1").partition(":")
+            headers[name.strip().lower()] = value.strip()
+        payload = await reader.readexactly(int(headers.get("content-length", "0")))
+        received.append((headers, payload))
+        writer.write(
+            b"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nx-typesafe-request-id: request-123\r\n"
+            + f"Content-Length: {len(body)}\r\nConnection: close\r\n\r\n".encode()
+            + body
+        )
+        await writer.drain()
+        writer.close()
+        await writer.wait_closed()
+
+    server = await asyncio.start_server(handle_request, "127.0.0.1", 0)
+    port = server.sockets[0].getsockname()[1]
+    async with (
+        server,
+        TypeSafeChatClient(api_key="configured-api-key", base_url=f"http://127.0.0.1:{port}") as client,
+    ):
+        await client.get_response(
+            [Message("user", ["Is the form signed?", Content.from_data(data=_PNG_1X1, media_type="image/png")])],
+            options={"response_format": questions()},
+        )
+
+    headers, payload = received[0]
+    assert headers.get("typesafe-danger-zone-images") == "true"
+    assert _PNG_1X1 in payload
+
+
+async def test_remote_image_uri_is_rejected() -> None:
+    client = make_client()
+    message = Message("user", [Content.from_uri("https://example.com/image.png", media_type="image/png")])
+
+    with pytest.raises(ChatClientInvalidRequestException, match="inline images"):
         await client.get_response([message], options={"response_format": questions()})
 
 
@@ -559,6 +717,7 @@ async def test_request_and_response_mapping() -> None:
             "questions": questions(),
             "model": "jev-preview",
             "response_model": SystemOneResponse,
+            "extra_headers": None,
         }
     ]
     assert response.response_id == "request-123"