Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion docs/02_concepts/11_timeouts.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ A `timedelta` longer than `timeout_max` is capped at `timeout_max`, and the clie

## Waiting for a job to finish

Methods with a `wait_for_finish` parameter, such as <ApiLink to="class/ActorClient#start">`ActorClient.start`</ApiLink> and <ApiLink to="class/ActorClient#build">`ActorClient.build`</ApiLink>, ask the API to hold the response until the run or build finishes, for at most 60 seconds. When `timeout` is a tier, the client adds `wait_for_finish` (capped at 60 seconds) to the tier's value, so the request doesn't time out while the API still holds it. A `timedelta` and `'no_timeout'` are used as given.
Methods with a `wait_for_finish` parameter, such as <ApiLink to="class/ActorClient#start">`ActorClient.start`</ApiLink>, <ApiLink to="class/ActorClient#build">`ActorClient.build`</ApiLink>, <ApiLink to="class/RunClient#get">`RunClient.get`</ApiLink>, and <ApiLink to="class/BuildClient#get">`BuildClient.get`</ApiLink>, ask the API to hold the response until the run or build finishes, for at most 60 seconds. When `timeout` is a tier, the client adds `wait_for_finish` (capped at 60 seconds) to the tier's value, so the request doesn't time out while the API still holds it. A `timedelta` and `'no_timeout'` are used as given.

## Interaction with retries

Expand Down
8 changes: 4 additions & 4 deletions src/apify_client/_resource_clients/_resource_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -219,7 +219,7 @@ def __init__(
api_base_url=api_base_url,
)

def _get(self, *, timeout: Timeout) -> dict | None:
def _get(self, *, timeout: Timeout, **kwargs: QueryParamValue) -> dict | None:
"""Perform a GET request for this resource, returning the parsed response or None if not found.

404s collapse to `None` only for ID-identified clients. Chained clients without a `resource_id`
Expand All @@ -229,7 +229,7 @@ def _get(self, *, timeout: Timeout) -> dict | None:
response = self._http_client.call(
url=self._build_url(),
method='GET',
params=self._build_params(),
params=self._build_params(**kwargs),
timeout=timeout,
)
return response_to_dict(response)
Expand Down Expand Up @@ -414,7 +414,7 @@ def __init__(
api_base_url=api_base_url,
)

async def _get(self, *, timeout: Timeout) -> dict | None:
async def _get(self, *, timeout: Timeout, **kwargs: QueryParamValue) -> dict | None:
"""Perform a GET request for this resource, returning the parsed response or None if not found.

404s collapse to `None` only for ID-identified clients. Chained clients without a `resource_id`
Expand All @@ -424,7 +424,7 @@ async def _get(self, *, timeout: Timeout) -> dict | None:
response = await self._http_client.call(
url=self._build_url(),
method='GET',
params=self._build_params(),
params=self._build_params(**kwargs),
timeout=timeout,
)
return response_to_dict(response)
Expand Down
29 changes: 23 additions & 6 deletions src/apify_client/_resource_clients/build.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
from apify_client._models import Build, BuildResponse
from apify_client._resource_clients._resource_client import ResourceClient, ResourceClientAsync
from apify_client._utils.http import response_to_dict
from apify_client._utils.time import to_seconds

if TYPE_CHECKING:
from datetime import timedelta
Expand Down Expand Up @@ -35,18 +36,26 @@ def __init__(
**kwargs,
)

def get(self, *, timeout: Timeout = 'short') -> Build | None:
def get(self, *, wait_for_finish: timedelta | None = None, timeout: Timeout = 'short') -> Build | None:
"""Return information about the Actor build.

https://docs.apify.com/api/v2#/reference/actor-builds/build-object/get-build

Args:
timeout: Timeout for the API HTTP request.
wait_for_finish: The maximum duration to wait for the build to finish before returning the
(unfinished) build object. The API holds the response for at most 60 seconds, so a longer
duration has no additional effect.
timeout: Timeout for the API HTTP request. A timeout tier is extended by `wait_for_finish`, so the
request outlasts the time the API holds the response.

Returns:
The retrieved Actor build data.
"""
result = self._get(timeout=timeout)
wait_for_finish_secs = to_seconds(wait_for_finish, as_int=True)
result = self._get(
timeout=self._timeout_for_wait_for_finish(timeout, wait_for_finish_secs),
waitForFinish=wait_for_finish_secs,
)
if result is None:
return None
return BuildResponse.model_validate(result).data
Expand Down Expand Up @@ -155,18 +164,26 @@ def __init__(
**kwargs,
)

async def get(self, *, timeout: Timeout = 'short') -> Build | None:
async def get(self, *, wait_for_finish: timedelta | None = None, timeout: Timeout = 'short') -> Build | None:
"""Return information about the Actor build.

https://docs.apify.com/api/v2#/reference/actor-builds/build-object/get-build

Args:
timeout: Timeout for the API HTTP request.
wait_for_finish: The maximum duration to wait for the build to finish before returning the
(unfinished) build object. The API holds the response for at most 60 seconds, so a longer
duration has no additional effect.
timeout: Timeout for the API HTTP request. A timeout tier is extended by `wait_for_finish`, so the
request outlasts the time the API holds the response.

Returns:
The retrieved Actor build data.
"""
result = await self._get(timeout=timeout)
wait_for_finish_secs = to_seconds(wait_for_finish, as_int=True)
result = await self._get(
timeout=self._timeout_for_wait_for_finish(timeout, wait_for_finish_secs),
waitForFinish=wait_for_finish_secs,
)
if result is None:
return None
return BuildResponse.model_validate(result).data
Expand Down
28 changes: 22 additions & 6 deletions src/apify_client/_resource_clients/run.py
Original file line number Diff line number Diff line change
Expand Up @@ -59,18 +59,26 @@ def __init__(
**kwargs,
)

def get(self, *, timeout: Timeout = 'short') -> Run | None:
def get(self, *, wait_for_finish: timedelta | None = None, timeout: Timeout = 'short') -> Run | None:
"""Return information about the Actor run.

https://docs.apify.com/api/v2#/reference/actor-runs/run-object/get-run

Args:
timeout: Timeout for the API HTTP request.
wait_for_finish: The maximum duration to wait for the run to finish before returning the (unfinished)
run object. The API holds the response for at most 60 seconds, so a longer duration has no
additional effect.
timeout: Timeout for the API HTTP request. A timeout tier is extended by `wait_for_finish`, so the
request outlasts the time the API holds the response.

Returns:
The retrieved Actor run data.
"""
result = self._get(timeout=timeout)
wait_for_finish_secs = to_seconds(wait_for_finish, as_int=True)
result = self._get(
timeout=self._timeout_for_wait_for_finish(timeout, wait_for_finish_secs),
waitForFinish=wait_for_finish_secs,
)
if result is None:
return None
return RunResponse.model_validate(result).data
Expand Down Expand Up @@ -616,18 +624,26 @@ def __init__(
**kwargs,
)

async def get(self, *, timeout: Timeout = 'short') -> Run | None:
async def get(self, *, wait_for_finish: timedelta | None = None, timeout: Timeout = 'short') -> Run | None:
"""Return information about the Actor run.

https://docs.apify.com/api/v2#/reference/actor-runs/run-object/get-run

Args:
timeout: Timeout for the API HTTP request.
wait_for_finish: The maximum duration to wait for the run to finish before returning the (unfinished)
run object. The API holds the response for at most 60 seconds, so a longer duration has no
additional effect.
timeout: Timeout for the API HTTP request. A timeout tier is extended by `wait_for_finish`, so the
request outlasts the time the API holds the response.

Returns:
The retrieved Actor run data.
"""
result = await self._get(timeout=timeout)
wait_for_finish_secs = to_seconds(wait_for_finish, as_int=True)
result = await self._get(
timeout=self._timeout_for_wait_for_finish(timeout, wait_for_finish_secs),
waitForFinish=wait_for_finish_secs,
)
if result is None:
return None
return RunResponse.model_validate(result).data
Expand Down
95 changes: 94 additions & 1 deletion tests/unit/test_wait_for_finish_timeout.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
from unittest.mock import Mock

import pytest
from werkzeug import Response
from werkzeug import Request, Response

from apify_client import ApifyClient, ApifyClientAsync

Expand All @@ -23,6 +23,8 @@

ACTOR_ID = 'test_actor_id'
TASK_ID = 'test_task_id'
RUN_ID = 'test_run_id'
BUILD_ID = 'test_build_id'

RUN_RESPONSE = {
'data': {
Expand Down Expand Up @@ -57,6 +59,14 @@
}


def _as_wait_for_finish_timedelta(kwargs: dict[str, Any]) -> dict[str, Any]:
"""Convert the `wait_for_finish` seconds used by the other call sites to the `timedelta` `get` expects."""
wait_for_finish = kwargs.get('wait_for_finish')
if wait_for_finish is None:
return kwargs
return {**kwargs, 'wait_for_finish': timedelta(seconds=wait_for_finish)}


@dataclass(frozen=True)
class CallSite:
"""A client method that sends `waitForFinish`, with the endpoint it calls and its default timeout tier."""
Expand Down Expand Up @@ -109,6 +119,41 @@ class CallSite:
),
id='task start',
),
pytest.param(
CallSite(
invoke=lambda client, **kwargs: client.run(RUN_ID).get(**_as_wait_for_finish_timedelta(kwargs)),
method='GET',
path=f'/v2/actor-runs/{RUN_ID}',
response_body=RUN_RESPONSE,
tier_secs=5,
),
id='run get',
),
pytest.param(
CallSite(
invoke=lambda client, **kwargs: client.build(BUILD_ID).get(**_as_wait_for_finish_timedelta(kwargs)),
method='GET',
path=f'/v2/actor-builds/{BUILD_ID}',
response_body=BUILD_RESPONSE,
tier_secs=5,
),
id='build get',
),
]

GET_CALL_SITES = [
pytest.param(
lambda client, **kwargs: client.run(RUN_ID).get(**kwargs),
f'/v2/actor-runs/{RUN_ID}',
RUN_RESPONSE,
id='run get',
),
pytest.param(
lambda client, **kwargs: client.build(BUILD_ID).get(**kwargs),
f'/v2/actor-builds/{BUILD_ID}',
BUILD_RESPONSE,
id='build get',
),
]


Expand Down Expand Up @@ -194,3 +239,51 @@ def test_timeout_for_wait_for_finish(
actor_client = client.actor(ACTOR_ID)

assert actor_client._timeout_for_wait_for_finish(timeout, wait_for_finish) == expected


@pytest.mark.parametrize(('invoke', 'path', 'response_body'), GET_CALL_SITES)
async def test_get_sends_wait_for_finish_as_whole_seconds(
client: ApifyClient | ApifyClientAsync,
httpserver: HTTPServer,
invoke: Callable[..., Any],
path: str,
response_body: dict,
) -> None:
"""`get` sends `wait_for_finish` as the `waitForFinish` query parameter, rounded down to whole seconds."""
captured: list[Request] = []

def handler(request: Request) -> Response:
captured.append(request)
return Response(json.dumps(response_body), status=200, mimetype='application/json')

httpserver.expect_request(path, method='GET').respond_with_handler(handler)

result = invoke(client, wait_for_finish=timedelta(seconds=45, milliseconds=500))
if inspect.isawaitable(result):
await result

assert captured[0].args.get('waitForFinish') == '45'


@pytest.mark.parametrize(('invoke', 'path', 'response_body'), GET_CALL_SITES)
async def test_get_omits_wait_for_finish_by_default(
client: ApifyClient | ApifyClientAsync,
httpserver: HTTPServer,
invoke: Callable[..., Any],
path: str,
response_body: dict,
) -> None:
"""`get` omits the `waitForFinish` query parameter when `wait_for_finish` is not given."""
captured: list[Request] = []

def handler(request: Request) -> Response:
captured.append(request)
return Response(json.dumps(response_body), status=200, mimetype='application/json')

httpserver.expect_request(path, method='GET').respond_with_handler(handler)

result = invoke(client)
if inspect.isawaitable(result):
await result

assert 'waitForFinish' not in captured[0].args
Loading