diff --git a/.github/workflows/php-integration-tests.yml b/.github/workflows/php-integration-tests.yml index a9df912..e2b7876 100644 --- a/.github/workflows/php-integration-tests.yml +++ b/.github/workflows/php-integration-tests.yml @@ -66,7 +66,8 @@ jobs: uses: shivammathur/setup-php@v2 with: php-version: '8.1' - extensions: mbstring, json, curl, zlib + # intl is optional for users but loaded here so the punycode host normalization test runs. + extensions: mbstring, json, curl, zlib, intl coverage: none tools: composer:v2 diff --git a/CHANGELOG.md b/CHANGELOG.md index 971f68d..dbec268 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,46 @@ # Changelog +## 0.12.0 + +- Synced with Apify OpenAPI spec `v2-2026-11-01T214220Z`. Response models return the new fields as + received (`toArray()`/`get()`), including the `RunStandby` deployment of Server Actor runs and a + `null` schedule `uiState`. +- Added a `deployment` filter to `RunListOptions` (Actor Standby deployments; honoured by `runs()` and + `actor($id)->runs()`) and an `includeInputSchema` option to `StoreListOptions`. +- `UserClient::monthlyUsage()`, `limits()` and `updateLimits()` now work for any user ID, not only `me()`, + as the spec documents the by-ID endpoints; they no longer throw `LogicException`. +- Added `RunClient::iterateDatasetItems()` with `RunIterateDatasetItemsOptions`: lazily reads a run's + default dataset while the run is still producing items and returns once the run has finished and the + dataset is drained. +- **Breaking:** the `short` and `medium` timeout tiers now default to 5 and 30 seconds, as in the + reference client, instead of the overall `timeoutSecs`. The `long` tier and `timeoutMaxSecs` still + default to `timeoutSecs` (360 seconds). Configure `timeoutShortSecs`/`timeoutMediumSecs` or pass a + per-call `$timeoutSecs` for a call that needs longer. +- A request that asks the API to hold its response with `waitForFinish` (`RunClient::get()`, + `BuildClient::get()`, `ActorClient::defaultBuild()` and the `waitForFinish` option of `ActorClient::start()`/ + `build()` and `TaskClient::start()`) now gets that hold, at most 60 seconds, on top of its tier timeout. +- **Breaking:** the URL fields of response models that the spec declares as `format: uri` are returned in the + WHATWG URL serialization (trailing slash for an empty path, lowercased scheme and host, default port + dropped, punycode host with the optional `intl` extension, percent-encoded unsafe characters), e.g. + `ActorRun::getContainerUrl()` returns `https://abc.runs.apify.net/`. +- **Breaking:** `ScheduleClient::getLog()` returns the list of invocation entries (`message`, `level`, + `createdAt`) the endpoint responds with, instead of the raw JSON text, and uses the `medium` tier. +- **Breaking:** `DatasetClient::createItemsPublicUrl()` and `KeyValueStoreClient::createKeysPublicUrl()` throw + `InvalidArgumentException` for an `expiresInSecs` of zero or less before sending any request; leave it out + for a URL that never expires. `createItemsPublicUrl()`, `createKeysPublicUrl()` and `getRecordPublicUrl()` + take a trailing `$timeoutSecs` for the request that fetches the storage (default `long` tier). +- `DatasetClient::pushItems()` no longer retries a request that timed out, since the API may already have stored + the items. +- Aligned the default timeout tier of individual methods with the reference client: `short` for + `ActorClient::validateInput()`, `BuildClient::abort()`, the Actor version and env-var collection calls, the + storage collections' `getOrCreate()`, the schedule and webhook `create()`, and the request-queue `listHead()`, + `addRequest()`, `batchDeleteRequests()` and `deleteRequestLock()`; `long` for the key-value store `update()` + and `recordExists()`; `medium` for `DatasetClient::pushItems()`. +- `RunClient::waitForFinish()`, `BuildClient::waitForFinish()`, and `call()` on Actors and tasks take an + optional `$timeoutSecs`; polls and the start request of `call()` have no request timeout by default. +- Documented that `KeyValueStoreClient::getRecord()` never parses a record and that `setRecord()` sends its + value as given, also under `application/json`. + ## 0.11.1 - Synced with Apify OpenAPI spec `v2-2026-10-26T112451Z`. The spec now documents the full Actor diff --git a/README.md b/README.md index 560aaf3..e13d6ae 100644 --- a/README.md +++ b/README.md @@ -67,9 +67,11 @@ $configured = new ApifyClient( | `maxRetries` | `8` | Maximum retries for failed requests. | | `minDelayBetweenRetriesMillis` | `500` | Minimum delay between retries (exponential backoff). | | `maxDelayBetweenRetriesMillis` | `timeoutSecs × 1000` (360000) | Upper bound (milliseconds) on the growing inter-retry delay; defaults to the request timeout expressed in milliseconds. | -| `timeoutSecs` | `360` | Overall per-request timeout, and the default duration of every timeout tier below that is left unset. | -| `timeoutShortSecs` / `timeoutMediumSecs` / `timeoutLongSecs` | `timeoutSecs` | Duration (seconds) of the named timeout tier — see [Timeout tiers](#timeout-tiers). | -| `timeoutMaxSecs` | `timeoutSecs` | Upper bound on any single request's timeout, including a per-call override. | +| `timeoutSecs` | `360` | Overall per-request timeout: the default duration of the `long` tier and of `timeoutMaxSecs`. | +| `timeoutShortSecs` | `5` | Duration (seconds) of the `short` timeout tier — see [Timeout tiers](#timeout-tiers). | +| `timeoutMediumSecs` | `30` | Duration (seconds) of the `medium` timeout tier. | +| `timeoutLongSecs` | `timeoutSecs` | Duration (seconds) of the `long` timeout tier. | +| `timeoutMaxSecs` | `timeoutSecs` | Upper bound on any single request attempt's timeout, including a per-call override and the growth across retries. | | `compression` | `null` (brotli, falling back to gzip) | Request-body compression: `'brotli'`, `'gzip'`, or a custom `HttpCompressorInterface` — see [HTTP compression](#http-compression). | | `userAgentSuffix` | `null` | Custom suffix appended to the `User-Agent` header. | | `httpClient` | Guzzle | The replaceable transport (`Apify\Client\Http\HttpClientInterface`). | @@ -84,9 +86,12 @@ not thrown — `get()` returns `null` and `delete()` is treated as a successful Every method that sends a request is assigned a timeout tier: `short` (metadata reads/writes, e.g. `get()`/`update()`/`delete()`), `medium` (listing/batch/trigger calls), or `long` -(downloads/uploads/streaming). Each tier's duration defaults to the client's single `timeoutSecs`, so -a client constructed without the tier options behaves exactly as before they existed; set -`timeoutShortSecs`/`timeoutMediumSecs`/`timeoutLongSecs` to give a tier its own duration instead. +(downloads/uploads/streaming). The tiers last 5 seconds (`short`), 30 seconds (`medium`) and the client's +`timeoutSecs`, 360 seconds by default (`long`); set `timeoutShortSecs`/`timeoutMediumSecs`/ +`timeoutLongSecs` to give a tier another duration. A call that asks the API to hold its response with a +`waitForFinish` value (`RunClient::get()`, `BuildClient::get()`, `ActorClient::defaultBuild()`, and the +`waitForFinish` option of `ActorClient::start()`/`build()` and `TaskClient::start()`) gets that hold, +at most 60 seconds, on top of its tier, so the client never aborts a request the API is still holding. Every method that sends a request additionally takes an optional trailing `$timeoutSecs`, overriding the tier for that one call — a number of seconds, a tier name (`'short'`/`'medium'`/`'long'`), or @@ -99,7 +104,8 @@ $client->requestQueue('my-queue')->unlockRequests(timeoutSecs: 'noTimeout'); // ``` `timeoutMaxSecs` caps every tier and every per-call override alike (defaulting to `timeoutSecs`), so -raise it whenever a call legitimately needs longer than the default 360 seconds. A request-queue +raise it whenever a call legitimately needs longer than the default 360 seconds. A failed attempt is +retried with a timeout doubled up to that cap. A request-queue client's own `timeoutSecs` (`requestQueue(id, options)`) caps every tier and per-call override of calls made through that client specifically, on top of `timeoutMaxSecs`. diff --git a/composer.json b/composer.json index 1ae64be..47d3213 100644 --- a/composer.json +++ b/composer.json @@ -36,6 +36,9 @@ "friendsofphp/php-cs-fixer": "^3.52", "phpstan/phpstan": "^1.11" }, + "suggest": { + "ext-intl": "Converts an internationalized host of a URL field in a response to punycode, as the reference client does." + }, "autoload": { "psr-4": { "Apify\\Client\\": "src/" diff --git a/docs/README.md b/docs/README.md index eba8456..54f5223 100644 --- a/docs/README.md +++ b/docs/README.md @@ -19,7 +19,7 @@ Every class is under the `Apify\Client\` PSR-4 root. Use these when writing `use |---|---|---| | `Apify\Client\` | The entry point and version constants. | `ApifyClient`, `Version` | | `Apify\Client\Model\` | Response models returned by the API. | `RequestQueueRequest`, `ActorEnvVar`, `Dataset`, `ActorRun`, `PaginationList` | -| `Apify\Client\Options\` | Option objects (all the `*Options` classes) **and** enums. | e.g. `ActorListOptions`, `ActorStartOptions`, `TaskStartOptions`, `RunListOptions`, `RunResurrectOptions`, `StorageListOptions`, `StoreListOptions`, `DatasetListItemsOptions`, `ListKeysOptions`, `GetRecordOptions`, `ListRequestsOptions`, `BatchAddRequestsOptions`, `PaginateRequestsOptions`, `LogOptions`, `DownloadItemsFormat` — see [options reference](options.md) for the full list | +| `Apify\Client\Options\` | Option objects (all the `*Options` classes) **and** enums. | e.g. `ActorListOptions`, `ActorStartOptions`, `TaskStartOptions`, `RunListOptions`, `RunIterateDatasetItemsOptions`, `RunResurrectOptions`, `StorageListOptions`, `StoreListOptions`, `DatasetListItemsOptions`, `ListKeysOptions`, `GetRecordOptions`, `ListRequestsOptions`, `BatchAddRequestsOptions`, `PaginateRequestsOptions`, `LogOptions`, `DownloadItemsFormat` — see [options reference](options.md) for the full list | | `Apify\Client\Http\` | The replaceable transport, its adapters, and compressors. | `HttpClientInterface`, `GuzzleHttpClient`, `Psr18HttpClient`, `HttpCompressorInterface`, `BrotliHttpCompressor`, `GzipHttpCompressor` | | `Apify\Client\Exception\` | Exceptions thrown by the client. | `ApifyApiException` and its status-specific subclasses (`NotFoundException`, `RateLimitException`, ...), `TransportException` | @@ -97,7 +97,7 @@ A few methods return data whose shape is not modelled and is instead returned as JSON value — typically an associative array, though `getInput()` is typed `mixed` and returns whatever JSON value was stored (or accept an arbitrary value serialized to JSON): -- Read: `me()->monthlyUsage(...)`, `me()->limits()`, `task($id)->getInput()`, +- Read: `me()->monthlyUsage(...)`, `me()->limits()`, `schedule($id)->getLog()`, `task($id)->getInput()`, `build($id)->getOpenApiDefinition()`, `dataset($id)->getStatistics()`. Note that `deleteRequestLock` returns `void` (it releases a lock and has no meaningful body), so it is not in this list. The request-queue lock/list/unlock/batch-delete operations diff --git a/docs/actors.md b/docs/actors.md index 91dd2fb..0dcc8f9 100644 --- a/docs/actors.md +++ b/docs/actors.md @@ -35,7 +35,7 @@ Addressed by ID or `username~name` (a `username/name` is accepted too). - `update(mixed $newFields): Actor` - `delete(): void` - `start(mixed $input = null, ?ActorStartOptions $options = null): ActorRun` — start and return immediately. -- `call(mixed $input = null, ?ActorStartOptions $options = null, ?int $waitSecs = null): ActorRun` — start and wait. +- `call(mixed $input = null, ?ActorStartOptions $options = null, ?int $waitSecs = null, int|float|string|null $timeoutSecs = null): ActorRun` — start and wait. `$timeoutSecs` is the timeout of the start request and of every poll (default `'noTimeout'`). - `validateInput(mixed $input = null, ?ValidateInputOptions $options = null): bool` `$input` is a JSON-serializable array (the normal case), or a raw `string` of bytes sent exactly as diff --git a/docs/builds.md b/docs/builds.md index 799d6e9..e085221 100644 --- a/docs/builds.md +++ b/docs/builds.md @@ -22,7 +22,7 @@ An Actor's builds are available at `$client->actor($id)->builds()`. - `get(?int $waitForFinishSecs = null): ?Build` — fetch, optionally waiting server-side (max 60s). - `abort(): Build` - `delete(): void` -- `waitForFinish(?int $waitSecs = null): Build` — poll until terminal (`null` waits indefinitely). +- `waitForFinish(?int $waitSecs = null, int|float|string|null $timeoutSecs = null): Build` — poll until terminal (`null` waits indefinitely); `$timeoutSecs` is the timeout of each poll request (default `'noTimeout'`). - `getOpenApiDefinition(): ?array` — the build's generated OpenAPI document. Throws if the build itself does not exist (a 404 here cannot mean anything else). - `log(): LogClient` — `get()` throws instead of resolving to `null` on a 404, since there is no ID diff --git a/docs/misc.md b/docs/misc.md index 1636766..73dfd63 100644 --- a/docs/misc.md +++ b/docs/misc.md @@ -23,8 +23,8 @@ foreach ($client->store()->iterate(new StoreListOptions(search: 'scraper'), 50) ## Users — `$client->me()` / `$client->user($id)` - `get(): ?User` — for `me()`, private account details are available via `toArray()`. -- `monthlyUsage(?string $date = null): array` — current-account monthly usage (only for `me()`). `$date` is an ISO date in `YYYY-MM-DD` format; the report covers the monthly usage cycle containing that date. Omit it (or pass `null`) to report the current month. -- `limits(): array`, `updateLimits(mixed $newLimits): void` — account limits (only for `me()`). +- `monthlyUsage(?string $date = null): array` — the user's monthly usage. `$date` is an ISO date in `YYYY-MM-DD` format; the report covers the monthly usage cycle containing that date. Omit it (or pass `null`) to report the current month. +- `limits(): array`, `updateLimits(mixed $newLimits): void` — the user's account limits. `monthlyUsage()`, `limits()` and `updateLimits()` work for `me()` and for any user ID the token may access. ```php $me = $client->me()->get(); diff --git a/docs/models.md b/docs/models.md index 69b35a2..5dfa214 100644 --- a/docs/models.md +++ b/docs/models.md @@ -8,7 +8,17 @@ returns. Each model exposes the commonly-used fields as typed getters; every mod even fields without a dedicated getter). - `get(string $key): mixed` — a single raw field by its API name. -Getters return `null` when the field is absent from the API response. The `RequestQueueRequest` +Getters return `null` when the field is absent from the API response. + +URL fields the OpenAPI specification declares as `format: uri` — a run's `consoleUrl`, `containerUrl` and +`output` links, the `consoleUrl` and public URLs of storages, a key's `recordPublicUrl`, a task's +`standbyUrl`, a webhook's `requestUrl`, a Store Actor's `url` and `userPictureUrl`, and a user profile's +`pictureUrl` and `websiteUrl` — are returned in the WHATWG URL serialization, as the reference client +does, by the getters, `get()` and `toArray()` alike: an empty path gets a trailing slash +(`https://abc.runs.apify.net` becomes `https://abc.runs.apify.net/`), the scheme and host are lowercased, +a default port is dropped, an internationalized host becomes punycode (needs the optional `intl` +extension) and unsafe characters are percent-encoded. A value that is not an absolute http(s)/ws(s)/ftp +URL is kept as received, so a value compared against a stored URL has to be in that form too. The `RequestQueueRequest` model is also used as an input object and therefore additionally exposes setters and a constructor (see [Input models](#input-models)). diff --git a/docs/options.md b/docs/options.md index 8fa8013..5fff686 100644 --- a/docs/options.md +++ b/docs/options.md @@ -74,6 +74,7 @@ Extra filters for `runs()->list()`, combined with a `ListOptions`. | `status` | `list\|null` | Filter by one or more run statuses (e.g. `SUCCEEDED`, `RUNNING`); sent comma-separated. | | `startedAfter` | `?string` | Only runs started after this ISO-8601 timestamp. | | `startedBefore` | `?string` | Only runs started before this ISO-8601 timestamp. | +| `deployment` | `list\|null` | Only Actor Standby runs of these deployments (`SINGLE_TENANT`, `MULTI_TENANT_FREE_USERS`, `MULTI_TENANT_PAYING_USERS`, `MULTI_TENANT_OWNER_PAID`); sent comma-separated. The API requires `status` to hold only non-terminal statuses (`READY`, `RUNNING`, `TIMING-OUT`, `ABORTING`) when it is used. Honoured by `runs()` and `actor($id)->runs()`. | ### `StoreListOptions` For `store()->list()` / `store()->iterate()`. @@ -89,9 +90,24 @@ For `store()->list()` / `store()->iterate()`. | `includeUnrunnableActors` | `?bool` | Include Actors the current user cannot run. | | `allowsAgenticUsers` | `?bool` | Filter to Actors that allow agentic users. | | `responseFormat` | `?string` | The response format (`full`, `agent`). | +| `includeInputSchema` | `?bool` | Include each Actor's compact input schema (only returned when its default build succeeded and defines input properties). | Supports [`withPagination($offset, $limit)`](#manual-offset-paging-with-withpagination). +### `RunIterateDatasetItemsOptions` +For `run($id)->iterateDatasetItems()`. +| Field | Type | Description | +|---|---|---| +| `offset` | `?int` | Number of dataset rows to skip from the beginning (default 0). | +| `limit` | `?int` | Maximum number of dataset rows to scan (default: no limit). Fewer items are yielded when filters drop some, more when `unwind` splits a row into several. | +| `chunkSize` | `?int` | Maximum number of dataset rows requested per API call (default 1000). | +| `clean` | `?bool` | Yield only non-empty items and skip hidden fields. | +| `fields` / `omit` | `list\|null` | Field names to include / exclude. | +| `skipEmpty` | `?bool` | Skip empty items. | +| `skipHidden` | `?bool` | Skip hidden fields (those starting with `#`). | +| `unwind` | `list\|null` | Field names to unwind: each array value becomes a separate item. | +| `pollIntervalSecs` | `?int` | How long to wait for the run to finish between polls, in seconds (default 5). | + ### `LastRunOptions` For `actor()->lastRun()` / `task()->lastRun()`. | Field | Type | Description | diff --git a/docs/runs.md b/docs/runs.md index 14fdbd7..7302922 100644 --- a/docs/runs.md +++ b/docs/runs.md @@ -27,10 +27,11 @@ An Actor's or task's runs are available at `$client->actor($id)->runs()` / `$cli - `reboot(): ActorRun` - `resurrect(?RunResurrectOptions $options = null): ActorRun` - `charge(RunChargeOptions $options): void` — for pay-per-event Actors. -- `waitForFinish(?int $waitSecs = null): ActorRun` +- `waitForFinish(?int $waitSecs = null, int|float|string|null $timeoutSecs = null): ActorRun` — `$timeoutSecs` is the timeout of each poll request (default `'noTimeout'`). - `dataset(): DatasetClient`, `keyValueStore(): KeyValueStoreClient`, `requestQueue(): RequestQueueClient` — `get()`/`delete()` on these throw instead of resolving to `null`/no-op on a 404, since there is no ID to pin the 404 to the run or the sub-resource. +- `iterateDatasetItems(?RunIterateDatasetItemsOptions $options = null, int|float|string|null $timeoutSecs = null): Generator` — lazily iterates the items of the run's default dataset *while the run is still producing them*, and returns once the run has finished and the dataset is drained. `$timeoutSecs` is the timeout of each request (default `'long'` tier), also used for the polls between pages. See [below](#reading-a-dataset-while-the-run-is-running). - `log(): LogClient`, `getStreamedLog(): StreamInterface` — `log()->get()` throws on a 404 for the same reason. ```php @@ -39,6 +40,24 @@ $client->run('RUN_ID')->charge(new RunChargeOptions(eventName: 'result', count: $items = $client->run('RUN_ID')->dataset()->listItems(); ``` +### Reading a dataset while the run is running + +`iterateDatasetItems()` yields the run's dataset items as the run pushes them, polling the dataset's +`itemCount` and waiting up to `pollIntervalSecs` (default 5) for the run to finish between polls. It +returns after the run has finished and the remaining rows are read, which also covers the few seconds +the dataset's `itemCount` lags behind the pushed items. Each page is requested with a `limit` that ends at +`itemCount`, so offsets stay exact even when `clean`, `skipEmpty` or `unwind` change how many items a page +returns. On a last-run client (`actor($id)->lastRun()`), the iteration sticks to the run its first request +resolved to. + +```php +$run = $client->actor('my-actor')->start(); + +foreach ($client->run($run->getId())->iterateDatasetItems(new RunIterateDatasetItemsOptions(chunkSize: 100)) as $item) { + echo json_encode($item) . PHP_EOL; +} +``` + ### `waitForFinish` — two distinct meanings `waitForFinish` appears in two different roles; do not confuse them: @@ -51,4 +70,5 @@ $items = $client->run('RUN_ID')->dataset()->listItems(); up and throws a `NotFoundException` instead of polling for the rest of `$waitSecs`. - **The server-side `waitForFinish` parameter** — `get(?int $waitForFinishSecs = null)` (and `defaultBuild()`) and the `waitForFinish` field on `*Options` (e.g. `ActorStartOptions`). This is a - single API-side wait and the server caps it at 60 seconds, so the client clamps larger values. + single API-side wait and the server caps it at 60 seconds, so the client clamps larger values. The + request's timeout is that wait, at most 60 seconds, on top of the method's [timeout tier](../README.md#timeout-tiers). diff --git a/docs/schedules.md b/docs/schedules.md index 96824c1..edd262d 100644 --- a/docs/schedules.md +++ b/docs/schedules.md @@ -25,10 +25,13 @@ foreach ($client->schedules()->iterate(new ListOptions(), 50) as $s) { ## A single schedule — `$client->schedule($id)` - `get(): ?Schedule`, `update(mixed $newFields): Schedule`, `delete(): void` -- `getLog(): string` — the schedule's invocation log (an empty string if there is no log yet). Throws +- `getLog(): array` — the schedule's invocation log: a list with one entry per invocation, each an + array with `message`, `level` and `createdAt` (an empty list if the schedule has not run yet). Throws if the schedule itself does not exist (a 404 here cannot mean anything else). ```php $updated = $client->schedule('SCHEDULE_ID')->update(['cronExpression' => '0 12 * * *']); -$log = $client->schedule('SCHEDULE_ID')->getLog(); +foreach ($client->schedule('SCHEDULE_ID')->getLog() as $entry) { + echo $entry['createdAt'] . ' ' . $entry['message'] . PHP_EOL; +} ``` diff --git a/docs/storages.md b/docs/storages.md index df462c3..a758672 100644 --- a/docs/storages.md +++ b/docs/storages.md @@ -25,7 +25,7 @@ Single — `$client->dataset($id)`: - `pushItems(mixed $items): void` - `getStatistics(): ?array` — throws if the dataset itself does not exist (a 404 here cannot mean anything else). -- `createItemsPublicUrl(?DatasetListItemsOptions $options = null, ?int $expiresInSecs = null, ?DownloadItemsFormat $format = null): string` — builds a shareable URL for downloading this dataset's items (forwarding the given item filters); for a private dataset it appends an access signature, optionally bounded to `$expiresInSecs`. `$format` selects the served output format (defaults to `json`). +- `createItemsPublicUrl(?DatasetListItemsOptions $options = null, ?int $expiresInSecs = null, ?DownloadItemsFormat $format = null, int|float|string|null $timeoutSecs = null): string` — builds a shareable URL for downloading this dataset's items (forwarding the given item filters); for a private dataset it appends an access signature, optionally bounded to `$expiresInSecs`, which must be positive (an `InvalidArgumentException` is thrown otherwise, before any request; omit it for a URL that never expires). `$timeoutSecs` applies to the request that fetches the dataset (default: the `long` tier). `$format` selects the served output format (defaults to `json`). ```php $dataset = $client->datasets()->getOrCreate('my-dataset'); @@ -61,12 +61,12 @@ Single — `$client->keyValueStore($id)`: - `listKeys(?ListKeysOptions $options = null): KeyValueStoreKeysPage` - `iterateKeys(?ListKeysOptions $options = null, ?int $chunkSize = null): iterable` — lazily iterate all keys, following cursor pagination (`exclusiveStartKey`/`nextExclusiveStartKey`). The options' `limit` caps the total number of keys yielded across all pages (unset = all); `$chunkSize` is the per-page size (unset = the remaining total cap, or the server default when unbounded). - `recordExists(string $key): bool` — reports whether a record with the given key exists, without downloading its value (a `HEAD` request). -- `getRecord(string $key, ?GetRecordOptions $options = null): ?KeyValueStoreRecord` -- `setRecord(string $key, string $value, string $contentType, ?SetRecordOptions $options = null, ?string $contentEncoding = null): void` — `$contentEncoding`, if given, is sent as the `Content-Encoding` header, for a `$value` that is already compressed (gzip, br, ...); the client's own best-effort request compression always backs off when `Content-Encoding` is already set, so the bytes reach the API exactly as given. +- `getRecord(string $key, ?GetRecordOptions $options = null): ?KeyValueStoreRecord` — the record value is never parsed: `getValue()` returns the exact bytes the API sent, whatever the content type, so a JSON record is decoded by the caller. +- `setRecord(string $key, string $value, string $contentType, ?SetRecordOptions $options = null, ?string $contentEncoding = null): void` — `$value` is sent exactly as given, also under `application/json`, so it has to be valid JSON already (use `setRecordJson()` to have an array encoded). `$contentEncoding`, if given, is sent as the `Content-Encoding` header, for a `$value` that is already compressed (gzip, br, ...); the client's own best-effort request compression always backs off when `Content-Encoding` is already set, so the bytes reach the API exactly as given. - `setRecordJson(string $key, mixed $value): void` — convenience over `setRecord()` that JSON-encodes `$value` and stores it with a JSON content type. - `deleteRecord(string $key): void` — permanently removes the record with the given key. -- `getRecordPublicUrl(string $key): string` — builds a shareable URL for downloading a single record; for a private store it appends an access signature so the URL works without an API token. -- `createKeysPublicUrl(?ListKeysOptions $options = null, ?int $expiresInSecs = null): string` — builds a shareable URL for listing this store's keys (forwarding the given key filters); for a private store it appends an access signature, optionally bounded to `$expiresInSecs`. +- `getRecordPublicUrl(string $key, int|float|string|null $timeoutSecs = null): string` — builds a shareable URL for downloading a single record; for a private store it appends an access signature so the URL works without an API token. +- `createKeysPublicUrl(?ListKeysOptions $options = null, ?int $expiresInSecs = null, int|float|string|null $timeoutSecs = null): string` — builds a shareable URL for listing this store's keys (forwarding the given key filters); for a private store it appends an access signature, optionally bounded to `$expiresInSecs`, which must be positive (an `InvalidArgumentException` is thrown otherwise, before any request; omit it for a URL that never expires). `$timeoutSecs` applies to the request that fetches the store (default: the `long` tier). ```php $store = $client->keyValueStores()->getOrCreate('my-store'); diff --git a/docs/tasks.md b/docs/tasks.md index 5c50de8..3492ab2 100644 --- a/docs/tasks.md +++ b/docs/tasks.md @@ -33,7 +33,7 @@ foreach ($client->tasks()->iterate(new ListOptions(), 50) as $t) { - `start(mixed $input = null, ?TaskStartOptions $options = null): ActorRun` — `$options->waitForResources` retries a start the API rejects for lack of account resources; see the Actor docs' `start()` for the full behavior. -- `call(mixed $input = null, ?TaskStartOptions $options = null, ?int $waitSecs = null): ActorRun` +- `call(mixed $input = null, ?TaskStartOptions $options = null, ?int $waitSecs = null, int|float|string|null $timeoutSecs = null): ActorRun` - `getInput(): mixed` (throws if the task itself does not exist — a 404 here cannot mean anything else), `updateInput(mixed $input): mixed` - `lastRun(?LastRunOptions $options = null): RunClient` diff --git a/src/ApifyClient.php b/src/ApifyClient.php index 08a3ab3..12af463 100644 --- a/src/ApifyClient.php +++ b/src/ApifyClient.php @@ -90,19 +90,18 @@ final class ApifyClient * @param int|null $maxDelayBetweenRetriesMillis upper bound for the growing inter-retry delay * (defaults to the request timeout) * @param int $timeoutSecs overall per-request timeout in seconds (default - * 360); also the default duration of every timeout - * tier below that is left unset + * 360); the default of the {@code long} tier and of + * {@code $timeoutMaxSecs} * @param int|null $timeoutShortSecs duration (seconds) of the {@code short} timeout - * tier (metadata reads/writes); defaults to - * {@code $timeoutSecs} + * tier (metadata reads/writes); defaults to 5 * @param int|null $timeoutMediumSecs duration (seconds) of the {@code medium} timeout - * tier (listing/batch/trigger calls); defaults to - * {@code $timeoutSecs} + * tier (listing/batch/trigger calls); defaults to 30 * @param int|null $timeoutLongSecs duration (seconds) of the {@code long} timeout * tier (downloads/uploads/streaming); defaults to * {@code $timeoutSecs} - * @param int|null $timeoutMaxSecs upper bound on any single request's timeout, - * including a per-call override; defaults to + * @param int|null $timeoutMaxSecs upper bound on any single request attempt's + * timeout, including a per-call override and the + * growth across retries; defaults to * {@code $timeoutSecs} * @param string|null $userAgentSuffix custom suffix appended to the User-Agent header * @param HttpClientInterface|null $httpClient replaces the default transport (Guzzle) @@ -217,7 +216,7 @@ public function runs(): RunCollectionClient /** A client for a specific Actor run. */ public function run(string $id): RunClient { - return new RunClient($this->http, $this->baseUrl, 'actor-runs', $id); + return new RunClient($this, $this->http, $this->baseUrl, 'actor-runs', $id); } // ----- Dataset accessors --------------------------------------------------- diff --git a/src/Internal/DatasetItemsPage.php b/src/Internal/DatasetItemsPage.php new file mode 100644 index 0000000..9fc1318 --- /dev/null +++ b/src/Internal/DatasetItemsPage.php @@ -0,0 +1,62 @@ +,1:?int} + */ + public static function fetch(HttpClientCore $http, ResourceContext $ctx, DatasetListItemsOptions $options, ?float $timeoutSecs = null): array + { + $params = new QueryParams(); + $options->appendTo($params); + $url = $ctx->mergedParams($params)->applyToUrl($ctx->subUrl('items')); + $response = $http->call('GET', $url, timeoutSecs: $timeoutSecs); + + $items = Json::decode((string) $response->getBody()); + $items = is_array($items) ? array_values($items) : []; + $count = count($items); + + $page = PaginationList::fromItems( + $items, + $response->getHeaderLine('X-Apify-Pagination-Total') === '' ? $count : (int) $response->getHeaderLine('X-Apify-Pagination-Total'), + $response->getHeaderLine('X-Apify-Pagination-Offset') === '' ? 0 : (int) $response->getHeaderLine('X-Apify-Pagination-Offset'), + $response->getHeaderLine('X-Apify-Pagination-Limit') === '' ? $count : (int) $response->getHeaderLine('X-Apify-Pagination-Limit'), + $count, + // Prefer the server-reported X-Apify-Pagination-Desc header (matches the reference JS + // client's `_createPaginationList`); fall back to the requested option when the header + // is absent, e.g. against an older API version that predates it. + self::headerBool($response->getHeaderLine('X-Apify-Pagination-Desc'), $options->desc ?? false), + ); + + $scannedHeader = $response->getHeaderLine('X-Apify-Pagination-Count'); + $scanned = $scannedHeader === '' ? null : (int) $scannedHeader; + + return [$page, $scanned]; + } + + private static function headerBool(string $value, bool $fallback): bool + { + return $value === '' ? $fallback : strtolower($value) === 'true'; + } +} diff --git a/src/Internal/HttpClientCore.php b/src/Internal/HttpClientCore.php index b66ddef..4c6c358 100644 --- a/src/Internal/HttpClientCore.php +++ b/src/Internal/HttpClientCore.php @@ -67,12 +67,6 @@ public function userAgent(): string return $this->userAgent; } - /** The configured overall per-request timeout budget, in seconds. */ - public function requestTimeoutSecs(): float - { - return $this->retry->timeoutSecs; - } - /** The configured timeout tiers (short/medium/long/max). See {@see TimeoutTiers}. */ public function timeoutTiers(): TimeoutTiers { diff --git a/src/Internal/ResourceContext.php b/src/Internal/ResourceContext.php index 2bcf74f..3e108fd 100644 --- a/src/Internal/ResourceContext.php +++ b/src/Internal/ResourceContext.php @@ -30,8 +30,9 @@ final class ResourceContext private const WAIT_REQUEST_SECS = 60; /** - * Safety margin subtracted from the configured per-request timeout when choosing the server-side + * Largest safety margin subtracted from the per-request timeout when choosing the server-side * {@code waitForFinish} value, so the server responds before the client's socket timeout fires. + * The margin never exceeds half of the timeout, so a short timeout still gets a positive hold. */ private const WAIT_TIMEOUT_MARGIN_SECS = 5; @@ -88,12 +89,6 @@ public function withTimeout(?float $timeoutSecs): self return $this; } - /** The per-context request timeout (seconds), or {@code null} to use the client-wide default. */ - public function requestTimeoutSecs(): ?float - { - return $this->requestTimeoutSecs; - } - /** Creates a context for a collection endpoint: {@code {base}/{resourcePath}}. */ public static function collection(HttpClientCore $http, string $baseUrl, string $resourcePath): self { @@ -234,6 +229,28 @@ public function resolveTimeout(int|float|string|null $timeoutSecs, string $defau return $capped === INF ? TimeoutTiers::INFINITE_SECS : $capped; } + /** + * Picks the timeout of a request that asks the API to hold its response with {@code waitForFinish}: + * the hold the caller asked for (the API holds a response for at most {@see WAIT_REQUEST_SECS} + * seconds) plus the round trip the method's tier allows, so the client does not abort a request the + * API is still holding, which for {@code start()} and {@code build()} would retry a call that + * creates a resource. An explicit per-call {@code $timeoutSecs} is used as given. + * + * @param int|float|string|null $timeoutSecs + */ + public function resolveWaitTimeout(int|float|string|null $timeoutSecs, string $defaultTier, ?int $waitForFinishSecs): float + { + if ($timeoutSecs !== null || $waitForFinishSecs === null) { + return $this->resolveTimeout($timeoutSecs, $defaultTier); + } + $holdSecs = min($waitForFinishSecs, self::WAIT_REQUEST_SECS); + if ($holdSecs <= 0) { + return $this->resolveTimeout(null, $defaultTier); + } + $tierSecs = $this->http->timeoutTiers()->secondsFor($defaultTier); + return $this->resolveTimeout($holdSecs + $tierSecs, $defaultTier); + } + /** * GET a paginated listing and build a {@see PaginationList} with each item hydrated. * @@ -514,28 +531,32 @@ public function putRaw( // ---- Wait-for-finish ------------------------------------------------------ /** - * The largest server-side {@code waitForFinish} value that is safe to send: below the configured - * per-request timeout by a safety margin (or the API's 60s cap when no finite timeout is set). + * The largest server-side {@code waitForFinish} value that is safe to send for a request whose + * timeout is {@code $requestTimeoutSecs}: below it by a safety margin, and never above the 60 seconds + * the API holds a response for. {@see TimeoutTiers::INFINITE_SECS} (no request timeout) allows the + * full 60 seconds. */ - private function serverWaitCapSecs(): int + private static function serverWaitCapSecs(float $requestTimeoutSecs): int { - $configured = (int) $this->http->requestTimeoutSecs(); - return $configured > 0 - ? max(0, $configured - self::WAIT_TIMEOUT_MARGIN_SECS) - : self::WAIT_REQUEST_SECS; + if ($requestTimeoutSecs <= TimeoutTiers::INFINITE_SECS) { + return self::WAIT_REQUEST_SECS; + } + $margin = min((float) self::WAIT_TIMEOUT_MARGIN_SECS, $requestTimeoutSecs / 2); + return min(self::WAIT_REQUEST_SECS, max(0, (int) floor($requestTimeoutSecs - $margin))); } /** - * Clamps a caller-supplied server-side {@code waitForFinish} value (seconds) to the server wait - * cap, so a synchronous get/wait never asks the server to hold the connection longer than the - * client's own per-request timeout. Returns {@code null} for a {@code null} input. + * Clamps a caller-supplied server-side {@code waitForFinish} value (seconds) so a synchronous get + * never asks the API to hold the connection for longer than a request can last: the value is capped + * at 60 seconds, and below the {@code timeoutMaxSecs} cap of a single attempt by a safety margin. + * Returns {@code null} for a {@code null} input. */ public function clampServerWait(?int $waitForFinishSecs): ?int { if ($waitForFinishSecs === null) { return null; } - return min(max(0, $waitForFinishSecs), $this->serverWaitCapSecs()); + return min(max(0, $waitForFinishSecs), self::serverWaitCapSecs($this->http->timeoutTiers()->maxSecs)); } /** @@ -548,17 +569,22 @@ public function clampServerWait(?int $waitForFinishSecs): ?int * wait gives up and rethrows that 404 instead of polling for the rest of {@code $waitSecs} (which * defaults to effectively indefinite). A successful poll in between resets this grace window. * + * Each poll's request timeout is {@code $timeoutSecs} (a number of seconds, a tier name or + * {@code 'noTimeout'}; default {@code 'noTimeout'}), and the server-side hold of a poll stays below it. + * * @param callable(array):bool $isTerminal + * @param int|float|string|null $timeoutSecs * @return array */ - public function waitForFinish(?int $waitSecs, string $resourceName, callable $isTerminal): array + public function waitForFinish(?int $waitSecs, string $resourceName, callable $isTerminal, int|float|string|null $timeoutSecs = null): array { $effectiveWaitSecs = $waitSecs !== null ? min(max($waitSecs, 0), self::MAX_WAIT_FOR_FINISH_SECS) : self::MAX_WAIT_FOR_FINISH_SECS; $budgetMillis = $effectiveWaitSecs * 1000; $start = self::nowMillis(); - $serverWaitCap = $this->serverWaitCapSecs(); + $pollTimeoutSecs = $this->resolveTimeout($timeoutSecs ?? TimeoutTiers::NO_TIMEOUT, TimeoutTiers::TIER_SHORT); + $serverWaitCap = self::serverWaitCapSecs($pollTimeoutSecs); $resource = null; $present = false; @@ -574,7 +600,7 @@ public function waitForFinish(?int $waitSecs, string $resourceName, callable $is $params->addInt('waitForFinish', $requestSecs); try { - $data = $this->getResourceRequired('', $params); + $data = $this->getResourceRequired('', $params, $pollTimeoutSecs); if (is_array($data)) { $resource = $data; $present = true; diff --git a/src/Internal/Signatures.php b/src/Internal/Signatures.php index c29b0ef..51b1576 100644 --- a/src/Internal/Signatures.php +++ b/src/Internal/Signatures.php @@ -4,6 +4,8 @@ namespace Apify\Client\Internal; +use InvalidArgumentException; + /** * Apify storage-content URL signing, byte-for-byte compatible with the platform's * {@code @apify/utilities} implementation that the reference clients rely on. @@ -37,6 +39,14 @@ public static function createHmacSignature(string $secretKey, string $message): return self::hexToBase62($truncated); } + /** Rejects a zero or negative expiry, which the signer would read as "never expires" or "already expired". */ + public static function requirePositiveExpiry(?int $expiresInSecs): void + { + if ($expiresInSecs !== null && $expiresInSecs <= 0) { + throw new InvalidArgumentException('expiresInSecs must be positive (omit it for a URL that never expires)'); + } + } + /** * Builds a storage-content signature for a resource's public URL, byte-for-byte compatible with * upstream {@code createStorageContentSignature}. diff --git a/src/Internal/TimeoutTiers.php b/src/Internal/TimeoutTiers.php index acc5cd0..d8f8ccf 100644 --- a/src/Internal/TimeoutTiers.php +++ b/src/Internal/TimeoutTiers.php @@ -12,13 +12,10 @@ * 'long'} for downloads/uploads/streaming. {@code 'noTimeout'} (accepted per call, not a configured * duration) opts a single call out of any request timeout. * - * Unlike the reference client — which lowers the default duration of the {@code short}/{@code medium} - * tiers (5s/30s) relative to v2's single 360s budget, a deliberate breaking change documented in its - * v3 upgrade guide — every tier here defaults to this client's existing single {@code $timeoutSecs} - * budget. A caller who never touches the new {@code timeout*Secs} constructor options therefore gets - * byte-for-byte the same effective timeout on every call as before this feature existed; tightening a - * tier is strictly opt-in, consistent with {@code client_requirements.md}'s rule against breaking the - * pre-existing public interface for anything short of a bugfix. + * Defaults match the reference client: {@code short} 5s, {@code medium} 30s, {@code long} and the + * per-attempt cap {@code max} the client's overall {@code $timeoutSecs} (360s by default). Every + * tier is configurable on the {@see \Apify\Client\ApifyClient} constructor; a tier or per-call + * duration above the cap is capped at it. * * @internal */ @@ -39,6 +36,12 @@ final class TimeoutTiers */ public const INFINITE_SECS = 0.0; + /** Default duration of the {@code short} tier, in seconds (metadata reads and writes). */ + public const DEFAULT_SHORT_SECS = 5.0; + + /** Default duration of the {@code medium} tier, in seconds (listing, batch and trigger calls). */ + public const DEFAULT_MEDIUM_SECS = 30.0; + private function __construct( public readonly float $shortSecs, public readonly float $mediumSecs, @@ -48,9 +51,9 @@ private function __construct( } /** - * Builds the tier configuration. Every tier argument left {@code null} falls back to - * {@code $defaultSecs} (this client's single overall {@code $timeoutSecs}), so a client configured - * without any of the new options behaves exactly as it did before tiers existed. + * Builds the tier configuration. A tier argument left {@code null} falls back to its default: + * {@see DEFAULT_SHORT_SECS}, {@see DEFAULT_MEDIUM_SECS}, and, for the {@code long} tier and the + * per-attempt cap, {@code $defaultSecs} (this client's overall {@code $timeoutSecs}). */ public static function create( float $defaultSecs, @@ -60,8 +63,8 @@ public static function create( ?float $maxSecs, ): self { return new self( - $shortSecs ?? $defaultSecs, - $mediumSecs ?? $defaultSecs, + $shortSecs ?? self::DEFAULT_SHORT_SECS, + $mediumSecs ?? self::DEFAULT_MEDIUM_SECS, $longSecs ?? $defaultSecs, $maxSecs ?? $defaultSecs, ); diff --git a/src/Internal/UrlNormalizer.php b/src/Internal/UrlNormalizer.php new file mode 100644 index 0000000..9367a91 --- /dev/null +++ b/src/Internal/UrlNormalizer.php @@ -0,0 +1,274 @@ + 80, 'https' => 443, 'ws' => 80, 'wss' => 443, 'ftp' => 21]; + + /** Characters (besides controls, space and non-ASCII) percent-encoded in a path. */ + private const PATH_UNSAFE = '"#<>?`{}'; + + /** Characters (besides controls, space and non-ASCII) percent-encoded in a query. */ + private const QUERY_UNSAFE = '"#<>\''; + + /** Characters (besides controls, space and non-ASCII) percent-encoded in a fragment. */ + private const FRAGMENT_UNSAFE = '"<>`'; + + /** Characters (besides controls, space and non-ASCII) percent-encoded in the user name and password. */ + private const USERINFO_UNSAFE = '"#<>?`{}/:;=@[\\]^|'; + + /** An IPv6 literal in a host: the brackets around hexadecimal digits, colons and dots. */ + private const IPV6_LITERAL_PATTERN = '/^\[[0-9a-f:.]+\]$/'; + + /** + * The IDNA error bits that WHATWG URL parsing ignores (it turns {@code CheckHyphens} and + * {@code VerifyDnsLength} off): empty label, label too long, domain too long, leading hyphen, + * trailing hyphen and hyphens in positions 3 and 4. + */ + private const IGNORED_IDNA_ERRORS = IDNA_ERROR_EMPTY_LABEL | IDNA_ERROR_LABEL_TOO_LONG | IDNA_ERROR_DOMAIN_NAME_TOO_LONG + | IDNA_ERROR_LEADING_HYPHEN | IDNA_ERROR_TRAILING_HYPHEN | IDNA_ERROR_HYPHEN_3_4; + + /** The highest valid port number. */ + private const MAX_PORT = 65535; + + /** Code points that may not appear in a decoded host (the WHATWG forbidden domain code points). */ + private const FORBIDDEN_HOST_PATTERN = '/[\x00-\x20#\/:<>?@\[\\\\\]^|%\x7F]/'; + + /** The highest code point of the space and control range, which is always percent-encoded. */ + private const LAST_CONTROL_OR_SPACE = 0x20; + + /** The highest printable ASCII code point; DEL and non-ASCII bytes above it are always encoded. */ + private const LAST_PRINTABLE = 0x7E; + + private function __construct() + { + } + + /** + * Returns the normalized form of {@code $url}, or {@code $url} itself when it is not an absolute + * special-scheme URL. + */ + public static function normalize(string $url): string + { + $cleaned = preg_replace('/[\t\n\r]/', '', trim($url, "\x00..\x20")); + if ($cleaned === null || preg_match('/^([A-Za-z][A-Za-z0-9+.\-]*):(.*)$/s', $cleaned, $m) !== 1) { + return $url; + } + $scheme = strtolower($m[1]); + if (!isset(self::DEFAULT_PORTS[$scheme])) { + return $url; + } + + // A special URL tolerates any number of slashes (or backslashes) before the authority. + $rest = ltrim($m[2], '/\\'); + $authorityEnd = strcspn($rest, '/\\?#'); + $authority = substr($rest, 0, $authorityEnd); + $remainder = substr($rest, $authorityEnd); + + $host = self::normalizeAuthority($authority, $scheme); + if ($host === null) { + return $url; + } + + $fragment = ''; + $hashAt = strpos($remainder, '#'); + if ($hashAt !== false) { + $fragment = '#' . self::encode(substr($remainder, $hashAt + 1), self::FRAGMENT_UNSAFE); + $remainder = substr($remainder, 0, $hashAt); + } + $query = ''; + $queryAt = strpos($remainder, '?'); + if ($queryAt !== false) { + $query = '?' . self::encode(substr($remainder, $queryAt + 1), self::QUERY_UNSAFE); + $remainder = substr($remainder, 0, $queryAt); + } + + return $scheme . '://' . $host . self::normalizePath($remainder) . $query . $fragment; + } + + /** + * Normalizes every URL field named by {@code $paths} in a decoded response object. A path is a + * dot-separated list of keys; the key {@code *} matches every entry of an object (for a map of + * URLs). Only string values are touched. + * + * @param array $data + * @param list $paths + * @return array + */ + public static function normalizeFields(array $data, array $paths): array + { + foreach ($paths as $path) { + $data = self::normalizeNode($data, explode('.', $path)); + } + return $data; + } + + /** + * @param array $node + * @param list $keys + * @return array + */ + private static function normalizeNode(array $node, array $keys): array + { + $key = array_shift($keys); + if ($key === null) { + return $node; + } + $targets = $key === '*' ? array_keys($node) : (array_key_exists($key, $node) ? [$key] : []); + foreach ($targets as $target) { + $value = $node[$target]; + if ($keys === []) { + if (is_string($value)) { + $node[$target] = self::normalize($value); + } + } elseif (is_array($value)) { + $node[$target] = self::normalizeNode($value, $keys); + } + } + return $node; + } + + /** Normalizes {@code [userinfo@]host[:port]}; {@code null} when it has no usable host. */ + private static function normalizeAuthority(string $authority, string $scheme): ?string + { + $userinfo = ''; + $at = strrpos($authority, '@'); + if ($at !== false) { + $credentials = substr($authority, 0, $at); + $authority = substr($authority, $at + 1); + $colon = strpos($credentials, ':'); + $user = $colon === false ? $credentials : substr($credentials, 0, $colon); + $userinfo = self::encode($user, self::USERINFO_UNSAFE); + if ($colon !== false) { + $userinfo .= ':' . self::encode(substr($credentials, $colon + 1), self::USERINFO_UNSAFE); + } + $userinfo .= '@'; + } + + $port = ''; + if (str_starts_with($authority, '[')) { + $close = strpos($authority, ']'); + if ($close !== false) { + $tail = substr($authority, $close + 1); + $authority = substr($authority, 0, $close + 1); + if ($tail !== '' && !str_starts_with($tail, ':')) { + return null; + } + $port = $tail === '' ? '' : substr($tail, 1); + } + } else { + $colon = strrpos($authority, ':'); + if ($colon !== false) { + $port = substr($authority, $colon + 1); + $authority = substr($authority, 0, $colon); + } + } + if ($port !== '' && (!ctype_digit($port) || (int) $port > self::MAX_PORT)) { + return null; + } + if ($authority === '') { + return null; + } + + if (str_starts_with($authority, '[')) { + $host = strtolower($authority); + if (preg_match(self::IPV6_LITERAL_PATTERN, $host) !== 1) { + return null; + } + } else { + $host = self::normalizeHost($authority); + } + if ($host === '') { + return null; + } + if ($port !== '' && (int) $port !== self::DEFAULT_PORTS[$scheme]) { + $host .= ':' . (int) $port; + } + return $userinfo . $host; + } + + /** Lowercases a domain host and converts it to punycode when {@code intl} can; {@code ''} when it is not a valid host. */ + private static function normalizeHost(string $host): string + { + $decoded = rawurldecode($host); + $lower = mb_strtolower($decoded, 'UTF-8'); + if (preg_match(self::FORBIDDEN_HOST_PATTERN, $lower) === 1) { + return ''; + } + if (preg_match('/[^\x20-\x7E]/', $lower) === 1 && function_exists('idn_to_ascii')) { + $info = []; + // IDNA_CHECK_BIDI is off: Node's `new URL()` accepts hosts that ICU's RFC 5893 check rejects (such as + // `١.com`), and ICU flags both cases with the same error, so a host that Node rejects is punycoded + // instead, the smaller divergence. + $flags = IDNA_NONTRANSITIONAL_TO_ASCII | IDNA_CHECK_CONTEXTJ; + $ascii = idn_to_ascii($lower, $flags, INTL_IDNA_VARIANT_UTS46, $info); + if ($ascii === false && (($info['errors'] ?? -1) & ~self::IGNORED_IDNA_ERRORS) === 0) { + $ascii = $info['result'] ?? false; + } + // The UTS #46 mapping turns fullwidth look-alikes into the forbidden ASCII characters. + if (!is_string($ascii) || $ascii === '' || preg_match(self::FORBIDDEN_HOST_PATTERN, $ascii) === 1) { + return ''; + } + return $ascii; + } + return $lower; + } + + /** Resolves dot segments of a path and percent-encodes it; an empty path becomes {@code /}. */ + private static function normalizePath(string $path): string + { + if ($path === '') { + return '/'; + } + $segments = explode('/', str_replace('\\', '/', $path)); + array_shift($segments); + $resolved = []; + $lastIndex = count($segments) - 1; + foreach ($segments as $index => $segment) { + $lower = strtolower($segment); + $isParent = in_array($lower, ['..', '.%2e', '%2e.', '%2e%2e'], true); + $isCurrent = in_array($lower, ['.', '%2e'], true); + if ($isParent) { + array_pop($resolved); + } elseif (!$isCurrent) { + $resolved[] = self::encode($segment, self::PATH_UNSAFE); + } + if (($isParent || $isCurrent) && $index === $lastIndex) { + $resolved[] = ''; + } + } + return '/' . implode('/', $resolved); + } + + /** + * Percent-encodes every byte that is a control, a space, outside printable ASCII, or in + * {@code $unsafe}, leaving existing {@code %XX} escapes as they are. + */ + private static function encode(string $value, string $unsafe): string + { + $encoded = ''; + $length = strlen($value); + for ($i = 0; $i < $length; $i++) { + $byte = ord($value[$i]); + $needsEncoding = $byte <= self::LAST_CONTROL_OR_SPACE || $byte > self::LAST_PRINTABLE || str_contains($unsafe, $value[$i]); + $encoded .= $needsEncoding ? sprintf('%%%02X', $byte) : $value[$i]; + } + return $encoded; + } +} diff --git a/src/Model/ActorRun.php b/src/Model/ActorRun.php index 90c1c5d..32ad666 100644 --- a/src/Model/ActorRun.php +++ b/src/Model/ActorRun.php @@ -9,6 +9,8 @@ /** A single execution of an Actor. */ final class ActorRun extends ApifyResource { + protected const URL_FIELDS = ['consoleUrl', 'containerUrl', 'output.*']; + /** The unique run ID. */ public function getId(): ?string { diff --git a/src/Model/ActorStoreListItem.php b/src/Model/ActorStoreListItem.php index 47faf29..cdf0a0d 100644 --- a/src/Model/ActorStoreListItem.php +++ b/src/Model/ActorStoreListItem.php @@ -7,6 +7,8 @@ /** An Actor as listed in the Apify Store. */ final class ActorStoreListItem extends ApifyResource { + protected const URL_FIELDS = ['userPictureUrl', 'url']; + /** The unique Actor ID. */ public function getId(): ?string { diff --git a/src/Model/ApifyResource.php b/src/Model/ApifyResource.php index b5b34c1..8fd3330 100644 --- a/src/Model/ApifyResource.php +++ b/src/Model/ApifyResource.php @@ -4,20 +4,37 @@ namespace Apify\Client\Model; +use Apify\Client\Internal\UrlNormalizer; + /** * Base class for API resource models. * * Each model wraps the raw decoded JSON object and exposes commonly-used fields as typed getters. * The full payload — including any field the API adds that is not modelled here — is always * available via {@see toArray()} and {@see get()}, so additive API changes never lose data. + * + * The fields a model lists in {@see URL_FIELDS} (those the OpenAPI specification declares as + * {@code format: uri}) are normalized to the WHATWG URL serialization when the model is built, the same + * way the reference client does: a trailing slash is added to an empty path, the scheme and host are + * lowercased, a default port is dropped and unsafe characters are percent-encoded. A value that is not + * an absolute http(s)/ws(s)/ftp URL is kept as received. */ abstract class ApifyResource { + /** + * The URL fields of this resource: dot-separated key paths into the raw object, where {@code *} + * stands for every entry of an object. + * + * @var list + */ + protected const URL_FIELDS = []; + /** * @param array $data the raw decoded resource object */ public function __construct(protected array $data) { + $this->data = UrlNormalizer::normalizeFields($data, static::URL_FIELDS); } /** diff --git a/src/Model/Dataset.php b/src/Model/Dataset.php index 5459974..200e5cc 100644 --- a/src/Model/Dataset.php +++ b/src/Model/Dataset.php @@ -7,6 +7,8 @@ /** A dataset stores structured results from Actor runs. */ final class Dataset extends ApifyResource { + protected const URL_FIELDS = ['consoleUrl', 'itemsPublicUrl']; + /** The unique dataset ID. */ public function getId(): ?string { diff --git a/src/Model/KeyValueStore.php b/src/Model/KeyValueStore.php index 50e7e3d..3a8d645 100644 --- a/src/Model/KeyValueStore.php +++ b/src/Model/KeyValueStore.php @@ -7,6 +7,8 @@ /** A key-value store holds arbitrary data records. */ final class KeyValueStore extends ApifyResource { + protected const URL_FIELDS = ['consoleUrl', 'keysPublicUrl', 'recordsPublicUrl']; + /** The unique store ID. */ public function getId(): ?string { diff --git a/src/Model/KeyValueStoreKey.php b/src/Model/KeyValueStoreKey.php index a9d47bd..143b988 100644 --- a/src/Model/KeyValueStoreKey.php +++ b/src/Model/KeyValueStoreKey.php @@ -7,6 +7,8 @@ /** A single key listed from a key-value store. */ final class KeyValueStoreKey extends ApifyResource { + protected const URL_FIELDS = ['recordPublicUrl']; + /** The record key. */ public function getKey(): ?string { diff --git a/src/Model/RequestQueue.php b/src/Model/RequestQueue.php index cb146ed..8898c74 100644 --- a/src/Model/RequestQueue.php +++ b/src/Model/RequestQueue.php @@ -7,6 +7,8 @@ /** A request queue stores URLs to be crawled. */ final class RequestQueue extends ApifyResource { + protected const URL_FIELDS = ['consoleUrl']; + /** The unique queue ID. */ public function getId(): ?string { diff --git a/src/Model/Task.php b/src/Model/Task.php index c4e784b..8a4488f 100644 --- a/src/Model/Task.php +++ b/src/Model/Task.php @@ -7,6 +7,8 @@ /** A pre-configured Actor run (an Actor task). */ final class Task extends ApifyResource { + protected const URL_FIELDS = ['standbyUrl']; + /** The unique task ID. */ public function getId(): ?string { diff --git a/src/Model/User.php b/src/Model/User.php index d62a18d..8472403 100644 --- a/src/Model/User.php +++ b/src/Model/User.php @@ -10,6 +10,8 @@ */ final class User extends ApifyResource { + protected const URL_FIELDS = ['profile.pictureUrl', 'profile.websiteUrl']; + /** The unique user ID. */ public function getId(): ?string { diff --git a/src/Model/Webhook.php b/src/Model/Webhook.php index 9b4b4ae..30e08c0 100644 --- a/src/Model/Webhook.php +++ b/src/Model/Webhook.php @@ -7,6 +7,8 @@ /** A webhook notifies an external service when specific events occur. */ final class Webhook extends ApifyResource { + protected const URL_FIELDS = ['requestUrl']; + /** The unique webhook ID. */ public function getId(): ?string { diff --git a/src/Model/WebhookDispatch.php b/src/Model/WebhookDispatch.php index dcfb31d..2f2a4ff 100644 --- a/src/Model/WebhookDispatch.php +++ b/src/Model/WebhookDispatch.php @@ -7,6 +7,8 @@ /** A single invocation of a webhook. */ final class WebhookDispatch extends ApifyResource { + protected const URL_FIELDS = ['webhook.requestUrl']; + /** The unique dispatch ID. */ public function getId(): ?string { diff --git a/src/Options/RunIterateDatasetItemsOptions.php b/src/Options/RunIterateDatasetItemsOptions.php new file mode 100644 index 0000000..e040d38 --- /dev/null +++ b/src/Options/RunIterateDatasetItemsOptions.php @@ -0,0 +1,44 @@ +|null $fields field names to include in the items (all others are omitted) + * @param list|null $omit field names to exclude from the items + * @param list|null $unwind field names to unwind: each array value becomes a separate item + */ + public function __construct( + /** Number of dataset rows to skip from the beginning (default 0). */ + public readonly ?int $offset = null, + /** + * Maximum number of dataset rows to scan (default: no limit). Fewer items are yielded when + * filters drop some, more when {@code unwind} splits a row into several. + */ + public readonly ?int $limit = null, + /** Maximum number of dataset rows requested per API call (default 1000). */ + public readonly ?int $chunkSize = null, + /** If {@code true}, yields only non-empty items and skips hidden fields. */ + public readonly ?bool $clean = null, + public readonly ?array $fields = null, + public readonly ?array $omit = null, + /** If {@code true}, skips empty items. */ + public readonly ?bool $skipEmpty = null, + /** If {@code true}, skips hidden fields (fields starting with {@code #}). */ + public readonly ?bool $skipHidden = null, + public readonly ?array $unwind = null, + /** How long to wait for the run to finish between polls, in seconds (default 5). */ + public readonly ?int $pollIntervalSecs = null, + ) { + } +} diff --git a/src/Options/RunListOptions.php b/src/Options/RunListOptions.php index 175dfa1..35d8dc7 100644 --- a/src/Options/RunListOptions.php +++ b/src/Options/RunListOptions.php @@ -22,6 +22,17 @@ public function __construct( public readonly ?string $startedAfter = null, /** Filter to runs started before this ISO-8601 timestamp. */ public readonly ?string $startedBefore = null, + /** + * Filter to runs of the given Actor Standby deployments ({@code SINGLE_TENANT}, + * {@code MULTI_TENANT_FREE_USERS}, {@code MULTI_TENANT_PAYING_USERS}, + * {@code MULTI_TENANT_OWNER_PAID}); sent as a comma-separated list. Honoured by the Actor-scoped + * and account-wide run collections. The API requires {@code status} to be set to non-terminal + * statuses only ({@code READY}, {@code RUNNING}, {@code TIMING-OUT}, {@code ABORTING}) when this + * filter is used. + * + * @var list|null + */ + public readonly ?array $deployment = null, ) { } @@ -30,6 +41,7 @@ public function appendTo(QueryParams $q): void { $q->addCsv('status', $this->status !== null ? array_values($this->status) : null) ->addString('startedAfter', $this->startedAfter) - ->addString('startedBefore', $this->startedBefore); + ->addString('startedBefore', $this->startedBefore) + ->addCsv('deployment', $this->deployment !== null ? array_values($this->deployment) : null); } } diff --git a/src/Options/StoreListOptions.php b/src/Options/StoreListOptions.php index aaa58a8..7c2eb6e 100644 --- a/src/Options/StoreListOptions.php +++ b/src/Options/StoreListOptions.php @@ -36,6 +36,11 @@ public function __construct( public readonly ?bool $allowsAgenticUsers = null, /** The response format ({@code full}, {@code agent}). */ public readonly ?string $responseFormat = null, + /** + * Include the compact input schema of each Actor's default build in the listing (only + * returned when the default build succeeded and defines input properties). + */ + public readonly ?bool $includeInputSchema = null, ) { } @@ -56,6 +61,7 @@ public function withPagination(?int $offset, ?int $limit): self $this->includeUnrunnableActors, $this->allowsAgenticUsers, $this->responseFormat, + $this->includeInputSchema, ); } @@ -71,6 +77,7 @@ public function appendTo(QueryParams $q): void ->addString('pricingModel', $this->pricingModel) ->addBool('includeUnrunnableActors', $this->includeUnrunnableActors) ->addBool('allowsAgenticUsers', $this->allowsAgenticUsers) - ->addString('responseFormat', $this->responseFormat); + ->addString('responseFormat', $this->responseFormat) + ->addBool('includeInputSchema', $this->includeInputSchema); } } diff --git a/src/Resource/ActorClient.php b/src/Resource/ActorClient.php index 3097d38..cd93f95 100644 --- a/src/Resource/ActorClient.php +++ b/src/Resource/ActorClient.php @@ -90,7 +90,7 @@ public function start( $params, $body, $options->contentTypeOrDefault(), - $this->ctx->resolveTimeout($timeoutSecs, TimeoutTiers::TIER_MEDIUM) + $this->ctx->resolveWaitTimeout($timeoutSecs, TimeoutTiers::TIER_MEDIUM, $options->waitForFinish) )), $options->waitForResources, ); @@ -103,11 +103,15 @@ public function start( * string of bytes sent exactly as given — see {@see start()} * @param int|null $waitSecs bounds the wait; {@code null} waits indefinitely. Time spent retrying a * {@code $options->waitForResources} start does not count toward it. + * @param int|float|string|null $timeoutSecs timeout of the start request and of every poll (a number of + * seconds, a tier name or {@code 'noTimeout'}); default {@code 'noTimeout'} */ - public function call(mixed $input = null, ?ActorStartOptions $options = null, ?int $waitSecs = null): ActorRun + public function call(mixed $input = null, ?ActorStartOptions $options = null, ?int $waitSecs = null, int|float|string|null $timeoutSecs = null): ActorRun { - $run = $this->start($input, $options); - return $this->root->run((string) $run->getId())->waitForFinish($waitSecs); + // Like the reference client, the start request and every poll have no request timeout by default. + $timeoutSecs ??= TimeoutTiers::NO_TIMEOUT; + $run = $this->start($input, $options, $timeoutSecs); + return $this->root->run((string) $run->getId())->waitForFinish($waitSecs, $timeoutSecs); } /** @@ -132,7 +136,7 @@ public function validateInput( $params, $body, $options->contentTypeOrDefault(), - $this->ctx->resolveTimeout($timeoutSecs, TimeoutTiers::TIER_MEDIUM) + $this->ctx->resolveTimeout($timeoutSecs, TimeoutTiers::TIER_SHORT) ); return is_array($result) && ($result['valid'] ?? false) === true; } @@ -152,13 +156,14 @@ public function build( } $params = new QueryParams(); $params->addString('version', $versionNumber); - ($options ?? new ActorBuildOptions())->appendTo($params); + $options ??= new ActorBuildOptions(); + $options->appendTo($params); return new Build($this->ctx->postWithBody( 'builds', $params, null, ResourceContext::CONTENT_TYPE_JSON, - $this->ctx->resolveTimeout($timeoutSecs, TimeoutTiers::TIER_MEDIUM) + $this->ctx->resolveWaitTimeout($timeoutSecs, TimeoutTiers::TIER_MEDIUM, $options->waitForFinish) )); } @@ -169,9 +174,9 @@ public function build( public function defaultBuild(?int $waitForFinish = null, int|float|string|null $timeoutSecs = null): BuildClient { $params = new QueryParams(); - // Clamp the server-side wait below the per-request timeout, consistent with run/build get(). + // Keep the server-side wait below the cap of a single attempt, consistent with run/build get(). $params->addInt('waitForFinish', $this->ctx->clampServerWait($waitForFinish)); - $data = $this->ctx->getResourceRequired('builds/default', $params, $this->ctx->resolveTimeout($timeoutSecs, TimeoutTiers::TIER_SHORT)); + $data = $this->ctx->getResourceRequired('builds/default', $params, $this->ctx->resolveWaitTimeout($timeoutSecs, TimeoutTiers::TIER_SHORT, $waitForFinish)); $build = new Build(is_array($data) ? $data : []); return new BuildClient($this->http, $this->baseUrl, (string) $build->getId()); } @@ -179,7 +184,7 @@ public function defaultBuild(?int $waitForFinish = null, int|float|string|null $ /** Returns a client for the last run of this Actor, optionally filtered by status and/or origin. */ public function lastRun(?LastRunOptions $options = null): RunClient { - $client = new RunClient($this->http, $this->ctx->subUrl(''), 'runs', 'last'); + $client = new RunClient($this->root, $this->http, $this->ctx->subUrl(''), 'runs', 'last'); $client->setLastRunParams($options ?? new LastRunOptions()); return $client; } diff --git a/src/Resource/ActorEnvVarCollectionClient.php b/src/Resource/ActorEnvVarCollectionClient.php index b655040..1ac312e 100644 --- a/src/Resource/ActorEnvVarCollectionClient.php +++ b/src/Resource/ActorEnvVarCollectionClient.php @@ -37,7 +37,7 @@ public function list(int|float|string|null $timeoutSecs = null): PaginationList '', new QueryParams(), static fn (array $d) => ActorEnvVar::fromArray($d), - $this->ctx->resolveTimeout($timeoutSecs, TimeoutTiers::TIER_MEDIUM) + $this->ctx->resolveTimeout($timeoutSecs, TimeoutTiers::TIER_SHORT) ); } @@ -60,7 +60,7 @@ public function iterate(int|float|string|null $timeoutSecs = null): Generator public function create(ActorEnvVar $envVar, int|float|string|null $timeoutSecs = null): ActorEnvVar { return ActorEnvVar::fromArray( - $this->ctx->createResource(new QueryParams(), $envVar->toArray(), $this->ctx->resolveTimeout($timeoutSecs, TimeoutTiers::TIER_MEDIUM)) + $this->ctx->createResource(new QueryParams(), $envVar->toArray(), $this->ctx->resolveTimeout($timeoutSecs, TimeoutTiers::TIER_SHORT)) ); } } diff --git a/src/Resource/ActorVersionCollectionClient.php b/src/Resource/ActorVersionCollectionClient.php index 9896af4..96779d2 100644 --- a/src/Resource/ActorVersionCollectionClient.php +++ b/src/Resource/ActorVersionCollectionClient.php @@ -31,7 +31,7 @@ public function __construct(HttpClientCore $http, string $actorUrl) */ public function list(int|float|string|null $timeoutSecs = null): PaginationList { - return $this->ctx->listResource('', new QueryParams(), static fn (array $d) => new ActorVersion($d), $this->ctx->resolveTimeout($timeoutSecs, TimeoutTiers::TIER_MEDIUM)); + return $this->ctx->listResource('', new QueryParams(), static fn (array $d) => new ActorVersion($d), $this->ctx->resolveTimeout($timeoutSecs, TimeoutTiers::TIER_SHORT)); } /** @@ -56,6 +56,6 @@ public function iterate(int|float|string|null $timeoutSecs = null): Generator */ public function create(mixed $version, int|float|string|null $timeoutSecs = null): ActorVersion { - return new ActorVersion($this->ctx->createResource(new QueryParams(), $version, $this->ctx->resolveTimeout($timeoutSecs, TimeoutTiers::TIER_MEDIUM))); + return new ActorVersion($this->ctx->createResource(new QueryParams(), $version, $this->ctx->resolveTimeout($timeoutSecs, TimeoutTiers::TIER_SHORT))); } } diff --git a/src/Resource/BuildClient.php b/src/Resource/BuildClient.php index d381382..798e0c5 100644 --- a/src/Resource/BuildClient.php +++ b/src/Resource/BuildClient.php @@ -24,17 +24,17 @@ public function __construct(private HttpClientCore $http, string $baseUrl, strin /** * Fetches the build, optionally asking the API to wait up to {@code $waitForFinishSecs} seconds - * for the build to finish before responding. The value is clamped client-side to the per-request - * timeout budget (minus a safety margin) so the server is never asked to hold the connection - * longer than the client will wait; the server additionally caps the wait at 60s. Returns - * {@code null} if it does not exist. + * for the build to finish before responding. The API caps the wait at 60s, and the client + * additionally keeps it below the {@code timeoutMaxSecs} cap of a single attempt. Unless + * {@code $timeoutSecs} is given, the request timeout is that wait on top of the {@code short} tier. + * Returns {@code null} if it does not exist. */ public function get(?int $waitForFinishSecs = null, int|float|string|null $timeoutSecs = null): ?Build { $params = new QueryParams(); - // Clamp to the client's per-request timeout so a short custom timeout doesn't abort the call. + // Keep the server-side wait below the cap of a single attempt, so the API never holds longer than a request can last. $params->addInt('waitForFinish', $this->ctx->clampServerWait($waitForFinishSecs)); - $data = $this->ctx->getResource('', $params, $this->ctx->resolveTimeout($timeoutSecs, TimeoutTiers::TIER_SHORT)); + $data = $this->ctx->getResource('', $params, $this->ctx->resolveWaitTimeout($timeoutSecs, TimeoutTiers::TIER_SHORT, $waitForFinishSecs)); return is_array($data) ? new Build($data) : null; } @@ -42,7 +42,7 @@ public function get(?int $waitForFinishSecs = null, int|float|string|null $timeo public function abort(int|float|string|null $timeoutSecs = null): Build { return new Build( - $this->ctx->postWithBody('abort', new QueryParams(), null, '', $this->ctx->resolveTimeout($timeoutSecs, TimeoutTiers::TIER_MEDIUM)) + $this->ctx->postWithBody('abort', new QueryParams(), null, '', $this->ctx->resolveTimeout($timeoutSecs, TimeoutTiers::TIER_SHORT)) ); } @@ -54,14 +54,16 @@ public function delete(int|float|string|null $timeoutSecs = null): void /** * Polls until the build reaches a terminal state or {@code $waitSecs} elapses ({@code null} waits - * indefinitely). Returns the latest build. + * indefinitely). Returns the latest build. {@code $timeoutSecs} is the timeout of each poll request (a + * number of seconds, a tier name or {@code 'noTimeout'}, the default). */ - public function waitForFinish(?int $waitSecs = null): Build + public function waitForFinish(?int $waitSecs = null, int|float|string|null $timeoutSecs = null): Build { $data = $this->ctx->waitForFinish( $waitSecs, 'build', - static fn (array $d): bool => (new Build($d))->isTerminal() + static fn (array $d): bool => (new Build($d))->isTerminal(), + $timeoutSecs ); return new Build($data); } diff --git a/src/Resource/DatasetClient.php b/src/Resource/DatasetClient.php index e83e869..d7e909e 100644 --- a/src/Resource/DatasetClient.php +++ b/src/Resource/DatasetClient.php @@ -4,6 +4,7 @@ namespace Apify\Client\Resource; +use Apify\Client\Internal\DatasetItemsPage; use Apify\Client\Internal\HttpClientCore; use Apify\Client\Internal\Json; use Apify\Client\Internal\QueryParams; @@ -16,7 +17,7 @@ use Apify\Client\Options\DatasetListItemsOptions; use Apify\Client\Options\DownloadItemsFormat; use Generator; -use Psr\Http\Message\ResponseInterface; +use InvalidArgumentException; use Psr\Http\Message\StreamInterface; /** A client for a specific dataset (and run-nested variants). */ @@ -33,16 +34,11 @@ public static function forId(HttpClientCore $http, string $baseUrl, string $id): } /** - * Creates a dataset client for a run's default dataset (nested path only, no ID). Any - * {@code $inheritedParams} (e.g. the {@code status}/{@code origin} filters pinned by a last-run - * accessor) become base params so every request resolves the correct run's dataset. @internal + * Creates a dataset client over a ready context, such as the one of a run's default dataset (nested + * path only, no ID). @internal */ - public static function nested(HttpClientCore $http, string $base, string $subPath, ?QueryParams $inheritedParams = null): self + public static function fromContext(HttpClientCore $http, ResourceContext $ctx): self { - $ctx = ResourceContext::collection($http, $base, $subPath); - if ($inheritedParams !== null) { - $ctx->baseParams = $inheritedParams->copy(); - } return new self($http, $ctx); } @@ -95,7 +91,7 @@ public function delete(int|float|string|null $timeoutSecs = null): void public function listItems(?DatasetListItemsOptions $options = null, int|float|string|null $timeoutSecs = null): PaginationList { $resolved = $this->ctx->resolveTimeout($timeoutSecs, TimeoutTiers::TIER_LONG); - return $this->fetchItemsPage($options ?? new DatasetListItemsOptions(), $resolved)[0]; + return DatasetItemsPage::fetch($this->http, $this->ctx, $options ?? new DatasetListItemsOptions(), $resolved)[0]; } /** @@ -136,7 +132,7 @@ public function iterateItems(?DatasetListItemsOptions $options = null, ?int $chu $resolved = $this->ctx->resolveTimeout($timeoutSecs, TimeoutTiers::TIER_LONG); $firstPageLimit = ResourceContext::minLimit($limit, $chunkSize); - [$page, $scanned] = $this->fetchItemsPage($options->withPagination($startOffset, $firstPageLimit), $resolved); + [$page, $scanned] = DatasetItemsPage::fetch($this->http, $this->ctx, $options->withPagination($startOffset, $firstPageLimit), $resolved); foreach ($page->getItems() as $item) { yield $item; } @@ -150,7 +146,7 @@ public function iterateItems(?DatasetListItemsOptions $options = null, ?int $chu // $remaining having already reached 0 (the caller's $limit is satisfied) — either ends the loop. while ($pageScanned > 0 && ($remaining === null || $remaining > 0)) { $pageLimit = ResourceContext::minLimit($remaining, $chunkSize); - [$page, $scanned] = $this->fetchItemsPage($options->withPagination($currentOffset, $pageLimit), $resolved); + [$page, $scanned] = DatasetItemsPage::fetch($this->http, $this->ctx, $options->withPagination($currentOffset, $pageLimit), $resolved); foreach ($page->getItems() as $item) { yield $item; } @@ -186,42 +182,6 @@ private static function scannedRows(PaginationList $page, ?int $scannedHeader, ? return $requestedLimit !== null ? min($scanned, $requestedLimit) : $scanned; } - /** - * Fetches one page of dataset items and parses the pagination headers. Returns the page together - * with the {@code X-Apify-Pagination-Count} "scanned rows" count ({@code null} if the response - * omits the header), which {@see iterateItems()} uses to advance — see its doc comment. - * - * @return array{0:PaginationList,1:?int} - */ - private function fetchItemsPage(DatasetListItemsOptions $options, ?float $timeoutSecs = null): array - { - $params = new QueryParams(); - $options->appendTo($params); - $url = $this->ctx->mergedParams($params)->applyToUrl($this->ctx->subUrl('items')); - $response = $this->http->call('GET', $url, timeoutSecs: $timeoutSecs); - - $items = Json::decode((string) $response->getBody()); - $items = is_array($items) ? array_values($items) : []; - $count = count($items); - - $page = PaginationList::fromItems( - $items, - $this->headerInt($response, 'X-Apify-Pagination-Total', $count), - $this->headerInt($response, 'X-Apify-Pagination-Offset', 0), - $this->headerInt($response, 'X-Apify-Pagination-Limit', $count), - $count, - // Prefer the server-reported X-Apify-Pagination-Desc header (matches the reference JS - // client's `_createPaginationList`); fall back to the requested option when the header - // is absent, e.g. against an older API version that predates it. - $this->headerBool($response, 'X-Apify-Pagination-Desc', $options->desc ?? false), - ); - - $scannedHeader = $response->getHeaderLine('X-Apify-Pagination-Count'); - $scanned = $scannedHeader === '' ? null : (int) $scannedHeader; - - return [$page, $scanned]; - } - /** * Downloads dataset items serialized in the given format, returning the raw bytes as a string. * Unlike {@see listItems()} (parsed items), this returns the items already serialized to JSON, @@ -277,7 +237,9 @@ public function pushItems(mixed $items, int|float|string|null $timeoutSecs = nul $url, Json::encode($items), ResourceContext::CONTENT_TYPE_JSON_CHARSET, - $this->ctx->resolveTimeout($timeoutSecs, TimeoutTiers::TIER_LONG) + $this->ctx->resolveTimeout($timeoutSecs, TimeoutTiers::TIER_MEDIUM), + // A push that timed out may still have been stored, so retrying it could duplicate the items. + doNotRetryTimeouts: true, ); } @@ -304,14 +266,20 @@ public function getStatistics(int|float|string|null $timeoutSecs = null): ?array * It fetches the dataset, and if the dataset exposes a URL-signing secret key (i.e. it is * private), appends an HMAC-SHA256 signature so the URL grants access without an API token. * {@code $expiresInSecs} optionally bounds the validity of a signed URL ({@code null} for - * non-expiring). {@code $format} selects the output format of the items served by the URL + * non-expiring); it must be positive, otherwise an {@see InvalidArgumentException} is thrown before + * any request is sent. {@code $format} selects the output format of the items served by the URL * (defaults to {@code json} when omitted). The URL is built from the configured public base URL. + * {@code $timeoutSecs} applies to the request that fetches the dataset (default: the {@code long} tier). + * + * @throws InvalidArgumentException if {@code $expiresInSecs} is zero or negative */ public function createItemsPublicUrl( ?DatasetListItemsOptions $options = null, ?int $expiresInSecs = null, ?DownloadItemsFormat $format = null, + int|float|string|null $timeoutSecs = null, ): string { + Signatures::requirePositiveExpiry($expiresInSecs); $options ??= new DatasetListItemsOptions(); $params = new QueryParams(); $options->appendTo($params); @@ -321,7 +289,7 @@ public function createItemsPublicUrl( // Only compute a signature when the caller did not already supply one, otherwise the URL // would carry two conflicting `signature` query params (mirrors KeyValueStoreClient::createKeysPublicUrl). if ($options->signature === null) { - $dataset = $this->get(); + $dataset = $this->get($timeoutSecs ?? TimeoutTiers::TIER_LONG); if ($dataset !== null) { $secret = $dataset->get('urlSigningSecretKey'); if (is_string($secret)) { @@ -332,16 +300,4 @@ public function createItemsPublicUrl( } return $params->applyToUrl($this->ctx->publicUrl('items')); } - - private function headerInt(ResponseInterface $response, string $name, int $fallback): int - { - $value = $response->getHeaderLine($name); - return $value === '' ? $fallback : (int) $value; - } - - private function headerBool(ResponseInterface $response, string $name, bool $fallback): bool - { - $value = $response->getHeaderLine($name); - return $value === '' ? $fallback : $value === 'true'; - } } diff --git a/src/Resource/DatasetCollectionClient.php b/src/Resource/DatasetCollectionClient.php index 99f31af..20112bb 100644 --- a/src/Resource/DatasetCollectionClient.php +++ b/src/Resource/DatasetCollectionClient.php @@ -63,6 +63,6 @@ public function iterate(?StorageListOptions $options = null, ?int $chunkSize = n */ public function getOrCreate(?string $name = null, ?array $schema = null, int|float|string|null $timeoutSecs = null): Dataset { - return new Dataset($this->ctx->getOrCreateNamed($name, $schema, $this->ctx->resolveTimeout($timeoutSecs, TimeoutTiers::TIER_MEDIUM))); + return new Dataset($this->ctx->getOrCreateNamed($name, $schema, $this->ctx->resolveTimeout($timeoutSecs, TimeoutTiers::TIER_SHORT))); } } diff --git a/src/Resource/KeyValueStoreClient.php b/src/Resource/KeyValueStoreClient.php index cf5d833..7c9a193 100644 --- a/src/Resource/KeyValueStoreClient.php +++ b/src/Resource/KeyValueStoreClient.php @@ -18,6 +18,7 @@ use Apify\Client\Options\ListKeysOptions; use Apify\Client\Options\SetRecordOptions; use Generator; +use InvalidArgumentException; /** A client for a specific key-value store (and run-nested variants). */ final class KeyValueStoreClient @@ -72,7 +73,7 @@ public function get(int|float|string|null $timeoutSecs = null): ?KeyValueStore */ public function update(mixed $newFields, int|float|string|null $timeoutSecs = null): KeyValueStore { - return new KeyValueStore($this->ctx->updateResource('', $newFields, $this->ctx->resolveTimeout($timeoutSecs, TimeoutTiers::TIER_SHORT))); + return new KeyValueStore($this->ctx->updateResource('', $newFields, $this->ctx->resolveTimeout($timeoutSecs, TimeoutTiers::TIER_LONG))); } /** @@ -153,13 +154,15 @@ public function recordExists(string $key, int|float|string|null $timeoutSecs = n return $this->ctx->headExists( 'records/' . ResourceContext::encodePathSegment($key), new QueryParams(), - $this->ctx->resolveTimeout($timeoutSecs, TimeoutTiers::TIER_SHORT) + $this->ctx->resolveTimeout($timeoutSecs, TimeoutTiers::TIER_LONG) ); } /** * Fetches a record by key, or {@code null} if it does not exist. Like the reference client, it - * requests the record as an attachment so the API returns the raw bytes directly. + * requests the record as an attachment so the API returns the raw bytes directly. The value is never + * parsed: {@see KeyValueStoreRecord::getValue()} holds the exact bytes the API sent whatever the + * content type, so a JSON record is decoded by the caller. */ public function getRecord(string $key, ?GetRecordOptions $options = null, int|float|string|null $timeoutSecs = null): ?KeyValueStoreRecord { @@ -182,7 +185,9 @@ public function getRecord(string $key, ?GetRecordOptions $options = null, int|fl /** * Stores a record with raw bytes and the given content type, honoring the given write options - * ({@code timeoutSecs}, {@code doNotRetryTimeouts}). + * ({@code timeoutSecs}, {@code doNotRetryTimeouts}). {@code $value} is sent exactly as given, also + * under {@code application/json}, so it has to be valid JSON already; use {@see setRecordJson()} to + * have a value encoded. * * {@code $contentEncoding}, if given, is sent as the request's {@code Content-Encoding} header, for * a {@code $value} that is already compressed (gzip, br, deflate, ...): the client's own best-effort @@ -238,10 +243,10 @@ public function deleteRecord(string $key, int|float|string|null $timeoutSecs = n * exposes a URL-signing secret key (i.e. it is private), appends an HMAC-SHA256 signature so the * URL grants access without an API token. The URL is built from the configured public base URL. */ - public function getRecordPublicUrl(string $key): string + public function getRecordPublicUrl(string $key, int|float|string|null $timeoutSecs = null): string { $params = new QueryParams(); - $store = $this->get(); + $store = $this->get($timeoutSecs ?? TimeoutTiers::TIER_LONG); if ($store !== null) { $secret = $store->get('urlSigningSecretKey'); if (is_string($secret)) { @@ -254,15 +259,21 @@ public function getRecordPublicUrl(string $key): string /** * Builds a public URL for listing this store's keys, forwarding the given key-listing filters * into the URL. As with {@see getRecordPublicUrl()}, a signature is appended for private stores - * unless the caller already supplied one. {@code $expiresInSecs} optionally bounds a signed URL. + * unless the caller already supplied one. {@code $expiresInSecs} optionally bounds a signed URL; it + * must be positive, otherwise an {@see InvalidArgumentException} is thrown before any request is + * sent. {@code $timeoutSecs} applies to the request that fetches the store (default: the {@code long} + * tier). + * + * @throws InvalidArgumentException if {@code $expiresInSecs} is zero or negative */ - public function createKeysPublicUrl(?ListKeysOptions $options = null, ?int $expiresInSecs = null): string + public function createKeysPublicUrl(?ListKeysOptions $options = null, ?int $expiresInSecs = null, int|float|string|null $timeoutSecs = null): string { + Signatures::requirePositiveExpiry($expiresInSecs); $options ??= new ListKeysOptions(); $params = new QueryParams(); $options->appendTo($params); if ($options->signature === null) { - $store = $this->get(); + $store = $this->get($timeoutSecs ?? TimeoutTiers::TIER_LONG); if ($store !== null) { $secret = $store->get('urlSigningSecretKey'); if (is_string($secret)) { diff --git a/src/Resource/KeyValueStoreCollectionClient.php b/src/Resource/KeyValueStoreCollectionClient.php index 68e07af..bc44294 100644 --- a/src/Resource/KeyValueStoreCollectionClient.php +++ b/src/Resource/KeyValueStoreCollectionClient.php @@ -63,6 +63,6 @@ public function iterate(?StorageListOptions $options = null, ?int $chunkSize = n */ public function getOrCreate(?string $name = null, ?array $schema = null, int|float|string|null $timeoutSecs = null): KeyValueStore { - return new KeyValueStore($this->ctx->getOrCreateNamed($name, $schema, $this->ctx->resolveTimeout($timeoutSecs, TimeoutTiers::TIER_MEDIUM))); + return new KeyValueStore($this->ctx->getOrCreateNamed($name, $schema, $this->ctx->resolveTimeout($timeoutSecs, TimeoutTiers::TIER_SHORT))); } } diff --git a/src/Resource/RequestQueueClient.php b/src/Resource/RequestQueueClient.php index 482573f..2b16c0e 100644 --- a/src/Resource/RequestQueueClient.php +++ b/src/Resource/RequestQueueClient.php @@ -144,7 +144,7 @@ public function listHead(?int $limit = null, int|float|string|null $timeoutSecs $params->addInt('limit', $limit); $this->applyClientKey($params); return RequestQueueHead::fromData( - $this->ctx->getResourceRequired('head', $params, $this->ctx->resolveTimeout($timeoutSecs, TimeoutTiers::TIER_MEDIUM)) + $this->ctx->getResourceRequired('head', $params, $this->ctx->resolveTimeout($timeoutSecs, TimeoutTiers::TIER_SHORT)) ); } @@ -159,7 +159,7 @@ public function addRequest(RequestQueueRequest $request, bool $forefront = false $params, Json::encode($request->toArray()), ResourceContext::CONTENT_TYPE_JSON, - $this->ctx->resolveTimeout($timeoutSecs, TimeoutTiers::TIER_MEDIUM) + $this->ctx->resolveTimeout($timeoutSecs, TimeoutTiers::TIER_SHORT) ); return new RequestQueueOperationInfo($data); } @@ -482,7 +482,7 @@ public function batchDeleteRequests(array $requests, int|float|string|null $time $params = $this->applyClientKey(new QueryParams()); $payload = array_map(static fn (RequestQueueRequest $r) => $r->toArray(), $requests); return BatchDeleteResult::fromData( - $this->ctx->deleteWithBody('requests/batch', $params, $payload, $this->ctx->resolveTimeout($timeoutSecs, TimeoutTiers::TIER_MEDIUM)) + $this->ctx->deleteWithBody('requests/batch', $params, $payload, $this->ctx->resolveTimeout($timeoutSecs, TimeoutTiers::TIER_SHORT)) ); } @@ -532,7 +532,7 @@ public function deleteRequestLock(string $id, bool $forefront = false, int|float $url = $this->ctx->mergedParams($params) ->applyToUrl($this->ctx->subUrl('requests/' . ResourceContext::encodePathSegment($id) . '/lock')); try { - $this->http->call('DELETE', $url, timeoutSecs: $this->ctx->resolveTimeout($timeoutSecs, TimeoutTiers::TIER_MEDIUM)); + $this->http->call('DELETE', $url, timeoutSecs: $this->ctx->resolveTimeout($timeoutSecs, TimeoutTiers::TIER_SHORT)); } catch (ApifyApiException $e) { if (!HttpClientCore::isNotFound($e)) { throw $e; diff --git a/src/Resource/RequestQueueCollectionClient.php b/src/Resource/RequestQueueCollectionClient.php index 3c89a1b..2d6e780 100644 --- a/src/Resource/RequestQueueCollectionClient.php +++ b/src/Resource/RequestQueueCollectionClient.php @@ -60,6 +60,6 @@ public function iterate(?StorageListOptions $options = null, ?int $chunkSize = n */ public function getOrCreate(?string $name = null, int|float|string|null $timeoutSecs = null): RequestQueue { - return new RequestQueue($this->ctx->getOrCreateNamed($name, null, $this->ctx->resolveTimeout($timeoutSecs, TimeoutTiers::TIER_MEDIUM))); + return new RequestQueue($this->ctx->getOrCreateNamed($name, null, $this->ctx->resolveTimeout($timeoutSecs, TimeoutTiers::TIER_SHORT))); } } diff --git a/src/Resource/RunClient.php b/src/Resource/RunClient.php index 562ca17..a860c5f 100644 --- a/src/Resource/RunClient.php +++ b/src/Resource/RunClient.php @@ -4,17 +4,22 @@ namespace Apify\Client\Resource; +use Apify\Client\ApifyClient; +use Apify\Client\Internal\DatasetItemsPage; use Apify\Client\Internal\HttpClientCore; use Apify\Client\Internal\Json; use Apify\Client\Internal\QueryParams; use Apify\Client\Internal\ResourceContext; use Apify\Client\Internal\TimeoutTiers; use Apify\Client\Model\ActorRun; +use Apify\Client\Options\DatasetListItemsOptions; use Apify\Client\Options\LastRunOptions; use Apify\Client\Options\LogOptions; use Apify\Client\Options\MetamorphOptions; use Apify\Client\Options\RunChargeOptions; +use Apify\Client\Options\RunIterateDatasetItemsOptions; use Apify\Client\Options\RunResurrectOptions; +use Generator; use InvalidArgumentException; use Psr\Http\Message\StreamInterface; @@ -29,10 +34,17 @@ final class RunClient /** Header the API uses to deduplicate charge requests. */ private const CHARGE_IDEMPOTENCY_HEADER = 'idempotency-key'; + /** Default number of dataset rows requested per API call by {@see iterateDatasetItems()}. */ + private const ITERATE_DEFAULT_CHUNK_SIZE = 1000; + + /** Default time, in seconds, {@see iterateDatasetItems()} waits for the run to finish between polls. */ + private const ITERATE_DEFAULT_POLL_INTERVAL_SECS = 5; + private ResourceContext $ctx; /** @internal */ public function __construct( + private ApifyClient $root, private HttpClientCore $http, string $baseUrl, string $resourcePath, @@ -59,16 +71,16 @@ public function setLastRunParams(LastRunOptions $options): void /** * Fetches the run, optionally asking the API to wait up to {@code $waitForFinishSecs} seconds - * for the run to reach a terminal state. The value is clamped client-side to the per-request - * timeout budget (minus a safety margin) so the server is never asked to hold the connection - * longer than the client will wait; the server additionally caps the wait at 60s. Returns - * {@code null} if it does not exist. + * for the run to reach a terminal state. The API caps the wait at 60s, and the client additionally + * keeps it below the {@code timeoutMaxSecs} cap of a single attempt. Unless {@code $timeoutSecs} is + * given, the request timeout is that wait on top of the {@code short} tier. Returns {@code null} + * if it does not exist. */ public function get(?int $waitForFinishSecs = null, int|float|string|null $timeoutSecs = null): ?ActorRun { $params = new QueryParams(); $params->addInt('waitForFinish', $this->ctx->clampServerWait($waitForFinishSecs)); - $data = $this->ctx->getResource('', $params, $this->ctx->resolveTimeout($timeoutSecs, TimeoutTiers::TIER_SHORT)); + $data = $this->ctx->getResource('', $params, $this->ctx->resolveWaitTimeout($timeoutSecs, TimeoutTiers::TIER_SHORT, $waitForFinishSecs)); return is_array($data) ? new ActorRun($data) : null; } @@ -192,14 +204,16 @@ private function generateIdempotencyKey(string $eventName): string /** * Polls until the run reaches a terminal state or {@code $waitSecs} elapses ({@code null} waits - * indefinitely). Returns the latest run. + * indefinitely). Returns the latest run. {@code $timeoutSecs} is the timeout of each poll request (a + * number of seconds, a tier name or {@code 'noTimeout'}, the default). */ - public function waitForFinish(?int $waitSecs = null): ActorRun + public function waitForFinish(?int $waitSecs = null, int|float|string|null $timeoutSecs = null): ActorRun { $data = $this->ctx->waitForFinish( $waitSecs, 'run', - static fn (array $d): bool => (new ActorRun($d))->isTerminal() + static fn (array $d): bool => (new ActorRun($d))->isTerminal(), + $timeoutSecs ); return new ActorRun($data); } @@ -215,7 +229,7 @@ public function waitForFinish(?int $waitSecs = null): ActorRun */ public function dataset(): DatasetClient { - return DatasetClient::nested($this->http, $this->ctx->subUrl(''), 'dataset', $this->ctx->baseParams); + return DatasetClient::fromContext($this->http, $this->datasetContext()); } /** @@ -269,4 +283,119 @@ public function getStreamedLog(): StreamInterface { return $this->log()->stream(new LogOptions(raw: true)); } + + /** + * Lazily iterates over the items of the run's default dataset while the run is still producing + * them, and returns once the run has finished and the dataset is drained. + * + * While the run has not finished, each poll yields the rows below the dataset's {@code itemCount} + * and then waits up to {@code pollIntervalSecs} for the run to finish, so the last rows are read as + * soon as it does. Each page is requested with a {@code limit} that ends at {@code itemCount}, so it + * covers exactly the rows it asks for, whatever the filters or {@code unwind} do to the items. + * {@code itemCount} lags a few seconds behind the pushed items, so once the run reaches a terminal + * status, the rows past it are read a page at a time until none are left. On a last-run client, the + * iteration sticks to the run its first request resolves to. + * + * @param int|float|string|null $timeoutSecs timeout of each API request (default: the {@code long} + * tier) + * @return Generator the dataset items, each decoded like {@see DatasetClient::listItems()} + */ + public function iterateDatasetItems(?RunIterateDatasetItemsOptions $options = null, int|float|string|null $timeoutSecs = null): Generator + { + $options ??= new RunIterateDatasetItemsOptions(); + $timeoutSecs ??= TimeoutTiers::TIER_LONG; + $pageSize = $options->chunkSize !== null && $options->chunkSize > 0 ? $options->chunkSize : self::ITERATE_DEFAULT_CHUNK_SIZE; + $pollIntervalSecs = max(0, $options->pollIntervalSecs ?? self::ITERATE_DEFAULT_POLL_INTERVAL_SECS); + $position = $options->offset ?? 0; + $end = $options->limit !== null && $options->limit > 0 ? $position + $options->limit : null; + + $run = $this->get(null, $timeoutSecs); + // A last-run client resolves `runs/last` on each request, so a newer run would swap the dataset + // mid-iteration: pin the iteration to the run the first request resolved to. + $runClient = $run !== null && $run->getId() !== null && $run->getId() !== $this->id ? $this->root->run($run->getId()) : $this; + // One context serves both the dataset client and the page reads, so they cannot drift apart. + $itemsCtx = $runClient->datasetContext(); + $datasetClient = DatasetClient::fromContext($this->http, $itemsCtx); + $requestTimeoutSecs = $this->ctx->resolveTimeout($timeoutSecs, TimeoutTiers::TIER_LONG); + + while (true) { + $isFinished = $run === null || $run->isTerminal(); + $itemCount = $datasetClient->get($timeoutSecs)?->getItemCount() ?? 0; + if ($end !== null) { + $itemCount = min($itemCount, $end); + } + + while ($position < $itemCount) { + $pageLimit = min($pageSize, $itemCount - $position); + [$page] = DatasetItemsPage::fetch($this->http, $itemsCtx, self::pageOptions($options, $position, $pageLimit), $requestTimeoutSecs); + foreach ($page->getItems() as $item) { + yield $item; + } + $position += $pageLimit; + } + + if ($end !== null && $position >= $end) { + return; + } + if ($isFinished) { + break; + } + $run = $runClient->waitForFinish($pollIntervalSecs, $timeoutSecs); + } + + // The run has finished but `itemCount` lags the pushed items: read on, a page at a time. + while (true) { + $pageLimit = $end !== null ? min($pageSize, $end - $position) : $pageSize; + [$page, $scanned] = DatasetItemsPage::fetch($this->http, $itemsCtx, self::pageOptions($options, $position, $pageLimit), $requestTimeoutSecs); + foreach ($page->getItems() as $item) { + yield $item; + } + // Only an empty page marks the end, as filters can shorten a full one. A page that `clean`, + // `skipEmpty` or `unwind` emptied past a lagging `itemCount` reports no scanned rows either, + // so a plain, unfiltered one-row read confirms the dataset really has nothing left. + $isEmpty = $page->getItems() === [] && ($scanned ?? 0) === 0; + if ($isEmpty && (!self::hasRowChangingFilter($options) || $this->hasNoItemsAt($datasetClient, $position, $timeoutSecs))) { + return; + } + $position += $pageLimit; + if ($end !== null && $position >= $end) { + return; + } + } + } + + /** The dataset options reading one page of a live iteration: its item filters at the given row window. */ + private static function pageOptions(RunIterateDatasetItemsOptions $options, int $offset, int $limit): DatasetListItemsOptions + { + return new DatasetListItemsOptions( + offset: $offset, + limit: $limit, + fields: $options->fields, + omit: $options->omit, + skipEmpty: $options->skipEmpty, + skipHidden: $options->skipHidden, + clean: $options->clean, + unwind: $options->unwind, + ); + } + + /** Whether a filter can drop whole rows or multiply them, so that a page's item count is not its row count. */ + private static function hasRowChangingFilter(RunIterateDatasetItemsOptions $options): bool + { + return $options->clean === true || $options->skipEmpty === true || ($options->unwind !== null && $options->unwind !== []); + } + + /** The context of this run's default dataset, inheriting the last-run filters pinned on this client. */ + private function datasetContext(): ResourceContext + { + $context = ResourceContext::collection($this->http, $this->ctx->subUrl(''), 'dataset'); + $context->baseParams = $this->ctx->baseParams->copy(); + return $context; + } + + /** Whether an unfiltered read of one row at {@code $offset} finds nothing. */ + private function hasNoItemsAt(DatasetClient $datasetClient, int $offset, int|float|string $timeoutSecs): bool + { + return $datasetClient->listItems(new DatasetListItemsOptions(offset: $offset, limit: 1), $timeoutSecs)->getItems() === []; + } } diff --git a/src/Resource/ScheduleClient.php b/src/Resource/ScheduleClient.php index 577dfcc..ce6dcce 100644 --- a/src/Resource/ScheduleClient.php +++ b/src/Resource/ScheduleClient.php @@ -45,14 +45,19 @@ public function delete(int|float|string|null $timeoutSecs = null): void } /** - * Fetches the schedule's invocation log as text (an empty string if there is no log yet). + * Fetches the schedule's invocation log: one entry per invocation, each with {@code message}, + * {@code level} and {@code createdAt}. The list is empty when the schedule has not run yet. * * Unlike {@see get()}, a 404 here is not swallowed: it always throws instead, since the only way - * this fixed sub-path 404s is the schedule itself being gone (matching the reference client). The - * return type is therefore never {@code null}. + * this fixed sub-path 404s is the schedule itself being gone (matching the reference client). + * + * @return list> */ - public function getLog(int|float|string|null $timeoutSecs = null): string + public function getLog(int|float|string|null $timeoutSecs = null): array { - return (string) $this->ctx->getRawRequired('log', new QueryParams(), $this->ctx->resolveTimeout($timeoutSecs, TimeoutTiers::TIER_LONG))->getBody(); + $entries = $this->ctx->getResourceRequired('log', new QueryParams(), $this->ctx->resolveTimeout($timeoutSecs, TimeoutTiers::TIER_MEDIUM)); + /** @var list> $log */ + $log = is_array($entries) ? array_values($entries) : []; + return $log; } } diff --git a/src/Resource/ScheduleCollectionClient.php b/src/Resource/ScheduleCollectionClient.php index df90ceb..d7cf4d7 100644 --- a/src/Resource/ScheduleCollectionClient.php +++ b/src/Resource/ScheduleCollectionClient.php @@ -61,6 +61,6 @@ public function iterate(?ListOptions $options = null, ?int $chunkSize = null, in */ public function create(mixed $schedule, int|float|string|null $timeoutSecs = null): Schedule { - return new Schedule($this->ctx->createResource(new QueryParams(), $schedule, $this->ctx->resolveTimeout($timeoutSecs, TimeoutTiers::TIER_MEDIUM))); + return new Schedule($this->ctx->createResource(new QueryParams(), $schedule, $this->ctx->resolveTimeout($timeoutSecs, TimeoutTiers::TIER_SHORT))); } } diff --git a/src/Resource/TaskClient.php b/src/Resource/TaskClient.php index a9b3b17..c32d2f0 100644 --- a/src/Resource/TaskClient.php +++ b/src/Resource/TaskClient.php @@ -108,7 +108,7 @@ public function start( $params, $body, ResourceContext::CONTENT_TYPE_JSON, - $this->ctx->resolveTimeout($timeoutSecs, TimeoutTiers::TIER_MEDIUM) + $this->ctx->resolveWaitTimeout($timeoutSecs, TimeoutTiers::TIER_MEDIUM, $options->waitForFinish) )), $options->waitForResources, ); @@ -120,11 +120,15 @@ public function start( * @param mixed $input optionally overrides the task's stored input * @param int|null $waitSecs bounds the wait; {@code null} waits indefinitely. Time spent retrying a * {@code $options->waitForResources} start does not count toward it. + * @param int|float|string|null $timeoutSecs timeout of the start request and of every poll (a number of + * seconds, a tier name or {@code 'noTimeout'}); default {@code 'noTimeout'} */ - public function call(mixed $input = null, ?TaskStartOptions $options = null, ?int $waitSecs = null): ActorRun + public function call(mixed $input = null, ?TaskStartOptions $options = null, ?int $waitSecs = null, int|float|string|null $timeoutSecs = null): ActorRun { - $run = $this->start($input, $options); - return $this->root->run((string) $run->getId())->waitForFinish($waitSecs); + // Like the reference client, the start request and every poll have no request timeout by default. + $timeoutSecs ??= TimeoutTiers::NO_TIMEOUT; + $run = $this->start($input, $options, $timeoutSecs); + return $this->root->run((string) $run->getId())->waitForFinish($waitSecs, $timeoutSecs); } /** @@ -164,7 +168,7 @@ public function updateInput(mixed $input, int|float|string|null $timeoutSecs = n /** Returns a client for the last run of this task, optionally filtered by status and/or origin. */ public function lastRun(?LastRunOptions $options = null): RunClient { - $client = new RunClient($this->http, $this->ctx->subUrl(''), 'runs', 'last'); + $client = new RunClient($this->root, $this->http, $this->ctx->subUrl(''), 'runs', 'last'); $client->setLastRunParams($options ?? new LastRunOptions()); return $client; } diff --git a/src/Resource/UserClient.php b/src/Resource/UserClient.php index 8eccc9e..bb0ae35 100644 --- a/src/Resource/UserClient.php +++ b/src/Resource/UserClient.php @@ -10,26 +10,21 @@ use Apify\Client\Internal\ResourceContext; use Apify\Client\Internal\TimeoutTiers; use Apify\Client\Model\User; -use LogicException; /** * A client for accessing user data ({@code /v2/users/{userId}} or {@code /v2/users/me}). * - * For the current user ({@code me}), it also exposes account usage and limits. Those endpoints only - * exist for {@code me} and throw {@see LogicException} if called on another user's client. + * It also exposes the user's monthly usage and account limits, addressed by the same user ID (or + * {@code me}). */ final class UserClient { - private const ME = 'me'; - private ResourceContext $ctx; - private bool $isMe; /** @internal */ public function __construct(private HttpClientCore $http, string $baseUrl, string $id) { $this->ctx = ResourceContext::single($http, $baseUrl, 'users', $id); - $this->isMe = $id === self::ME; } /** @@ -43,15 +38,13 @@ public function get(int|float|string|null $timeoutSecs = null): ?User } /** - * Fetches the current account's monthly usage for the month containing the given date (formatted - * as {@code YYYY-MM-DD}). An empty/{@code null} date reports the current month. Only available - * for {@code me}. + * Fetches the user's monthly usage for the month containing the given date (formatted as + * {@code YYYY-MM-DD}). An empty/{@code null} date reports the current month. * * @return array */ public function monthlyUsage(?string $date = null, int|float|string|null $timeoutSecs = null): array { - $this->requireMe(); $params = new QueryParams(); if ($date !== null && $date !== '') { $params->addString('date', $date); @@ -61,25 +54,23 @@ public function monthlyUsage(?string $date = null, int|float|string|null $timeou } /** - * Fetches the current account's resource limits. Only available for {@code me}. + * Fetches the user's resource limits. * * @return array */ public function limits(int|float|string|null $timeoutSecs = null): array { - $this->requireMe(); $data = $this->ctx->getResourceRequired('limits', new QueryParams(), $this->ctx->resolveTimeout($timeoutSecs, TimeoutTiers::TIER_SHORT)); return is_array($data) ? $data : []; } /** - * Updates the current account's resource limits. Only available for {@code me}. + * Updates the user's resource limits. * * @param mixed $newLimits any JSON-serializable limits object */ public function updateLimits(mixed $newLimits, int|float|string|null $timeoutSecs = null): void { - $this->requireMe(); $this->http->call( 'PUT', $this->ctx->subUrl('limits'), @@ -88,11 +79,4 @@ public function updateLimits(mixed $newLimits, int|float|string|null $timeoutSec $this->ctx->resolveTimeout($timeoutSecs, TimeoutTiers::TIER_SHORT) ); } - - private function requireMe(): void - { - if (!$this->isMe) { - throw new LogicException('this operation is only available for the current user (use me())'); - } - } } diff --git a/src/Resource/WebhookCollectionClient.php b/src/Resource/WebhookCollectionClient.php index 5512e76..2a4fdce 100644 --- a/src/Resource/WebhookCollectionClient.php +++ b/src/Resource/WebhookCollectionClient.php @@ -22,6 +22,6 @@ final class WebhookCollectionClient extends AbstractWebhookCollectionClient */ public function create(mixed $webhook, int|float|string|null $timeoutSecs = null): Webhook { - return new Webhook($this->ctx->createResource(new QueryParams(), $webhook, $this->ctx->resolveTimeout($timeoutSecs, TimeoutTiers::TIER_MEDIUM))); + return new Webhook($this->ctx->createResource(new QueryParams(), $webhook, $this->ctx->resolveTimeout($timeoutSecs, TimeoutTiers::TIER_SHORT))); } } diff --git a/src/Version.php b/src/Version.php index 3ebbd67..f968a2b 100644 --- a/src/Version.php +++ b/src/Version.php @@ -17,13 +17,13 @@ final class Version * The semantic version of this client library (see https://semver.org/). * Changes to the public interface other than additive ones are considered breaking changes. */ - public const CLIENT_VERSION = '0.11.1'; + public const CLIENT_VERSION = '0.12.0'; /** * The version of the Apify OpenAPI specification this client was generated and verified * against. Corresponds to the {@code info.version} field of the Apify OpenAPI document. */ - public const API_SPEC_VERSION = 'v2-2026-10-26T112451Z'; + public const API_SPEC_VERSION = 'v2-2026-11-01T214220Z'; private function __construct() { diff --git a/tests/Examples/DocSnippetsTest.php b/tests/Examples/DocSnippetsTest.php index dec1a2c..a1fab47 100644 --- a/tests/Examples/DocSnippetsTest.php +++ b/tests/Examples/DocSnippetsTest.php @@ -100,6 +100,7 @@ private function imports(): string 'Apify\Client\Options\MetamorphOptions', 'Apify\Client\Options\RunResurrectOptions', 'Apify\Client\Options\RunChargeOptions', + 'Apify\Client\Options\RunIterateDatasetItemsOptions', 'Apify\Client\Options\ValidateInputOptions', 'Apify\Client\Options\BatchAddRequestsOptions', ]; diff --git a/tests/Integration/ActorIntegrationTest.php b/tests/Integration/ActorIntegrationTest.php index 8aa7c8a..ef41eaa 100644 --- a/tests/Integration/ActorIntegrationTest.php +++ b/tests/Integration/ActorIntegrationTest.php @@ -48,6 +48,22 @@ public function testActorCrudFlow(): void } } + public function testActorDefaultBuildAndWebhookList(): void + { + $client = $this->requireClient(); + + $defaultBuild = $client->actor('apify/hello-world')->defaultBuild(); + self::assertNotNull($defaultBuild->get()); + + $created = $client->actors()->create(self::minimalActor(self::uniqueName('hooks'))); + try { + $page = $client->actor((string) $created->getId())->webhooks()->list(new ListOptions()); + self::assertSame(0, $page->getCount()); + } finally { + $client->actor((string) $created->getId())->delete(); + } + } + public function testActorVersionCrudFlow(): void { $client = $this->requireClient(); diff --git a/tests/Integration/ActorRunIntegrationTest.php b/tests/Integration/ActorRunIntegrationTest.php index 5ff8182..0709ce9 100644 --- a/tests/Integration/ActorRunIntegrationTest.php +++ b/tests/Integration/ActorRunIntegrationTest.php @@ -4,10 +4,14 @@ namespace Apify\Client\Tests\Integration; +use Apify\Client\ApifyClient; +use Apify\Client\Internal\UrlNormalizer; use Apify\Client\Options\DatasetListItemsOptions; use Apify\Client\Options\LastRunOptions; use Apify\Client\Options\ListOptions; +use Apify\Client\Options\RunIterateDatasetItemsOptions; use Apify\Client\Options\RunListOptions; +use Apify\Client\Resource\RunCollectionClient; final class ActorRunIntegrationTest extends IntegrationTestCase { @@ -53,6 +57,167 @@ public function testRunActorAndReadOutputs(): void $client->run((string) $run->getId())->keyValueStore()->getRecord('OUTPUT'); } + public function testReadRunStoragesAndLogById(): void + { + $client = $this->requireClient(); + $run = $client->actor('apify/hello-world')->call(null, null, 120); + $runClient = $client->run((string) $run->getId()); + + $dataset = $runClient->dataset()->get(); + self::assertNotNull($dataset); + self::assertSame($run->getDefaultDatasetId(), $dataset->getId()); + self::assertNotNull($runClient->dataset()->getStatistics()); + + $store = $runClient->keyValueStore()->get(); + self::assertNotNull($store); + self::assertSame($run->getDefaultKeyValueStoreId(), $store->getId()); + + $queue = $runClient->requestQueue()->get(); + self::assertNotNull($queue); + self::assertSame($run->getDefaultRequestQueueId(), $queue->getId()); + + $log = $client->log((string) $run->getId())->get(); + self::assertNotNull($log); + self::assertNotSame('', $log); + } + + public function testIterateDatasetItemsOfAFinishedRunYieldsEveryItem(): void + { + $client = $this->requireClient(); + $run = $client->actor('apify/hello-world')->call(null, null, 120); + self::assertSame('SUCCEEDED', $run->getStatus()); + $runClient = $client->run((string) $run->getId()); + + $expected = iterator_to_array($runClient->dataset()->iterateItems(), false); + $live = iterator_to_array($runClient->iterateDatasetItems(new RunIterateDatasetItemsOptions(chunkSize: 1, pollIntervalSecs: 1)), false); + + self::assertSame($expected, $live); + } + + public function testIterateDatasetItemsReadsItemsWhileTheRunIsStillRunning(): void + { + $client = $this->requireClient(); + // An Actor that pushes one item every ten seconds, three times, straight through the API + // (the Actor runtime provides the token, the dataset ID and the public API base URL). + $script = <<<'JS' + const base = (process.env.APIFY_API_PUBLIC_BASE_URL || 'https://api.apify.com').replace(/\/$/, ''); + const url = `${base}/v2/datasets/${process.env.APIFY_DEFAULT_DATASET_ID}/items?token=${process.env.APIFY_TOKEN}`; + const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); + (async () => { + for (let n = 1; n <= 3; n++) { + const response = await fetch(url, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ n }), + }); + if (!response.ok) throw new Error(`push failed: ${response.status}`); + await sleep(10000); + } + })(); + JS; + $actorId = $this->createBuiltActor($client, 'live', $script); + try { + $actorClient = $client->actor($actorId); + $run = $actorClient->start(); + $runClient = $client->run((string) $run->getId()); + $statusWhenReceived = []; + $items = []; + foreach ($runClient->iterateDatasetItems(new RunIterateDatasetItemsOptions(pollIntervalSecs: 1)) as $item) { + $items[] = $item['n']; + $statusWhenReceived[] = $runClient->get()?->getStatus(); + } + + self::assertSame([1, 2, 3], $items); + // The first item reached the caller while the Actor was still sleeping before its last push. + self::assertContains($statusWhenReceived[0], ['READY', 'RUNNING']); + self::assertTrue($runClient->get()?->isTerminal()); + } finally { + $client->actor($actorId)->delete(); + } + } + + /** + * Creates an Actor whose {@code main.js} is the given script and waits for its build to succeed; + * returns the Actor ID. The caller deletes the Actor. + */ + private function createBuiltActor(ApifyClient $client, string $label, string $mainJs): string + { + $definition = self::minimalActor(self::uniqueName($label)); + $definition['versions'][0]['sourceFiles'][1]['content'] = $mainJs; + $actorId = (string) $client->actors()->create($definition)->getId(); + try { + $build = $client->actor($actorId)->build('0.0'); + self::assertSame('SUCCEEDED', $client->build((string) $build->getId())->waitForFinish(300)->getStatus()); + } catch (\Throwable $e) { + $client->actor($actorId)->delete(); + throw $e; + } + return $actorId; + } + + public function testRunsListAcceptsTheDeploymentFilter(): void + { + $client = $this->requireClient(); + // A test-owned Actor whose run keeps running, so the listing is isolated from other tests. The run is + // aborted at the end, so it can be given a lifetime that outlasts the retries below. + $actorId = $this->createBuiltActor($client, 'deploy', 'setTimeout(() => {}, 600000);'); + try { + $actor = $client->actor($actorId); + $runId = (string) $actor->start()->getId(); + $running = new RunListOptions(status: ['RUNNING']); + $standbyOnly = new RunListOptions(status: ['RUNNING'], deployment: ['SINGLE_TENANT']); + // Newest first, so the 50 runs read account-wide include this one. + $newestFirst = new ListOptions(limit: 50, desc: true); + + // Run listings lag: wait until the running run shows up without the filter, then check that the same + // listing with the deployment filter leaves it out, as it is not a Standby run. + self::assertEventuallyIterated( + [$runId], + fn (): array => self::listedRunIds($actor->runs(), new ListOptions(), $running), + 'running run', + ); + self::assertArrayNotHasKey($runId, self::listedRunIds($actor->runs(), new ListOptions(), $standbyOnly)); + + self::assertEventuallyIterated( + [$runId], + fn (): array => self::listedRunIds($client->runs(), $newestFirst, $running), + 'running run (account-wide)', + ); + self::assertArrayNotHasKey($runId, self::listedRunIds($client->runs(), $newestFirst, $standbyOnly)); + + $client->run($runId)->abort(); + } finally { + $client->actor($actorId)->delete(); + } + } + + /** + * The IDs of the runs a listing yields, as a set (ID => true). + * + * @return array + */ + private static function listedRunIds(RunCollectionClient $runs, ListOptions $options, RunListOptions $filter): array + { + $ids = []; + foreach ($runs->iterate($options, $filter) as $listed) { + $ids[(string) $listed->getId()] = true; + } + return $ids; + } + + public function testRunUrlFieldsAreNormalized(): void + { + $client = $this->requireClient(); + $run = $client->actor('apify/hello-world')->call(null, null, 120); + + foreach (['consoleUrl', 'containerUrl'] as $field) { + $url = $run->get($field); + if (is_string($url)) { + self::assertSame(UrlNormalizer::normalize($url), $url, "$field must be returned normalized"); + } + } + } + public function testLastRunAccess(): void { $client = $this->requireClient(); diff --git a/tests/Integration/IntegrationTestCase.php b/tests/Integration/IntegrationTestCase.php index 1659cbb..3ea6a68 100644 --- a/tests/Integration/IntegrationTestCase.php +++ b/tests/Integration/IntegrationTestCase.php @@ -39,6 +39,15 @@ protected static function resolveBaseUrl(?string $apiUrl): string return $trimmed; } + /** + * How many times, and how many seconds apart, a listing is re-read while a resource the test just + * created is missing from it. The list endpoints read from database secondaries that may lag by up + * to 90 seconds, so 30 attempts 3 seconds apart cover that window; a pass returns as soon as every + * expected ID shows up. + */ + private const LISTING_MAX_ATTEMPTS = 30; + private const LISTING_RETRY_DELAY_SECS = 3; + /** Returns a configured client, or skips the test if {@code APIFY_TOKEN} is unset. */ protected function requireClient(): ApifyClient { @@ -69,7 +78,7 @@ protected static function uniqueName(string $prefix): string * Apify's list/pagination endpoints are eventually consistent, and this suite is designed to run * concurrently with other test runs (including other language clients) against the same shared * test account; under that load, a resource created immediately before an {@code iterate()} call - * can take a few seconds to be reflected in a collection listing. Retrying the whole pass (rather + * can take a while to be reflected in a collection listing. Retrying the whole pass (rather * than looping forever or giving up after one try) keeps the test genuinely exercising * {@code iterate()} while tolerating that lag: it still fails, with the same diagnostic message, * if a resource never appears within the bounded timeout. @@ -80,17 +89,14 @@ protected static function uniqueName(string $prefix): string */ protected static function assertEventuallyIterated(array $expectedIds, callable $collect, string $label): void { - $maxAttempts = 8; - $delaySecs = 0.5; $seen = []; - for ($attempt = 1; $attempt <= $maxAttempts; $attempt++) { + for ($attempt = 1; $attempt <= self::LISTING_MAX_ATTEMPTS; $attempt++) { $seen = $collect(); $missing = array_filter($expectedIds, static fn (string $id): bool => !isset($seen[$id])); - if ($missing === [] || $attempt === $maxAttempts) { + if ($missing === [] || $attempt === self::LISTING_MAX_ATTEMPTS) { break; } - usleep((int) ($delaySecs * 1_000_000)); - $delaySecs = min($delaySecs * 1.6, 5.0); + sleep(self::LISTING_RETRY_DELAY_SECS); } // Always assert (even on a first-attempt success) so the test genuinely records assertions // instead of relying on an early return, which PHPUnit flags as a "risky" no-assertion test. diff --git a/tests/Integration/ScheduleIntegrationTest.php b/tests/Integration/ScheduleIntegrationTest.php index ea2310c..179f0fb 100644 --- a/tests/Integration/ScheduleIntegrationTest.php +++ b/tests/Integration/ScheduleIntegrationTest.php @@ -76,9 +76,12 @@ public function testScheduleCrudFlow(): void self::assertNotNull($schedule->get()); $updated = $schedule->update(['cronExpression' => '0 12 * * *']); self::assertSame('0 12 * * *', $updated->getCronExpression()); - // A fresh schedule may have no invocations yet (an empty string), which is a valid result - // — we only assert the call itself succeeds. - $schedule->getLog(); + // A fresh schedule may have no invocations yet (an empty list), which is a valid result; + // any entry it does have carries the documented fields. + foreach ($schedule->getLog() as $entry) { + self::assertArrayHasKey('message', $entry); + self::assertArrayHasKey('createdAt', $entry); + } } finally { $client->schedule((string) $sch->getId())->delete(); } diff --git a/tests/Integration/StoreIntegrationTest.php b/tests/Integration/StoreIntegrationTest.php index bb42d2d..5bfd455 100644 --- a/tests/Integration/StoreIntegrationTest.php +++ b/tests/Integration/StoreIntegrationTest.php @@ -15,6 +15,22 @@ public function testListStore(): void self::assertLessThanOrEqual(5, count($page->getItems())); } + public function testListStoreWithInputSchema(): void + { + $client = $this->requireClient(); + $withSchema = $client->store()->list(new StoreListOptions(search: 'website content crawler', limit: 10, includeInputSchema: true)); + self::assertNotEmpty($withSchema->getItems()); + $withoutSchema = $client->store()->list(new StoreListOptions(search: 'website content crawler', limit: 10)); + foreach ($withoutSchema->getItems() as $item) { + self::assertNull($item->get('inputSchema'), 'the input schema is only returned on request'); + } + $schemas = array_filter( + array_map(static fn ($item) => $item->get('inputSchema'), $withSchema->getItems()), + static fn ($schema) => $schema !== null, + ); + self::assertNotEmpty($schemas, 'at least one Store Actor carries its input schema'); + } + public function testIterateStore(): void { $client = $this->requireClient(); diff --git a/tests/Integration/TaskIntegrationTest.php b/tests/Integration/TaskIntegrationTest.php index 6f6c330..b199b3b 100644 --- a/tests/Integration/TaskIntegrationTest.php +++ b/tests/Integration/TaskIntegrationTest.php @@ -97,6 +97,23 @@ public function testPublishUnpublish(): void } } + public function testTaskLastRun(): void + { + $client = $this->requireClient(); + $task = $client->tasks()->create(self::taskDef(self::uniqueName('task-last'))); + try { + $tc = $client->task((string) $task->getId()); + $run = $tc->call(null, null, 120); + self::assertSame('SUCCEEDED', $run->getStatus()); + + $lastRun = $tc->lastRun()->get(); + self::assertNotNull($lastRun); + self::assertSame($run->getId(), $lastRun->getId()); + } finally { + $client->task((string) $task->getId())->delete(); + } + } + public function testTaskCrudFlow(): void { $client = $this->requireClient(); @@ -114,6 +131,7 @@ public function testTaskCrudFlow(): void self::assertSame('Updated Title', $updated->getTitle()); self::assertSame('Updated description', $updated->getDescription()); $tc->runs()->list(new ListOptions(), new RunListOptions()); + self::assertSame(0, $tc->webhooks()->list(new ListOptions())->getCount()); } finally { $client->task((string) $task->getId())->delete(); } diff --git a/tests/Integration/UserIntegrationTest.php b/tests/Integration/UserIntegrationTest.php index ebe8803..fd6d5e4 100644 --- a/tests/Integration/UserIntegrationTest.php +++ b/tests/Integration/UserIntegrationTest.php @@ -27,6 +27,15 @@ public function testGetMonthlyUsageForDate(): void self::assertNotEmpty($client->me()->monthlyUsage('2026-06-01')); } + public function testGetMonthlyUsageAndLimitsByUserId(): void + { + $client = $this->requireClient(); + $id = $client->me()->get()?->getId(); + self::assertNotNull($id); + self::assertNotEmpty($client->user($id)->monthlyUsage()); + self::assertNotEmpty($client->user($id)->limits()); + } + public function testGetLimits(): void { $client = $this->requireClient(); diff --git a/tests/Unit/MethodTierTest.php b/tests/Unit/MethodTierTest.php new file mode 100644 index 0000000..2e4ec84 --- /dev/null +++ b/tests/Unit/MethodTierTest.php @@ -0,0 +1,110 @@ + */ + public static function tierCases(): iterable + { + yield 'Actor validateInput' => [static fn (ApifyClient $c) => $c->actor('a')->validateInput(), self::SHORT, '{"valid":true}']; + yield 'Build abort' => [static fn (ApifyClient $c) => $c->build('b')->abort(), self::SHORT, self::ITEM_BODY]; + yield 'Actor env var list' => [static fn (ApifyClient $c) => $c->actor('a')->version('0.1')->envVars()->list(), self::SHORT, self::PAGE_BODY]; + yield 'Actor env var create' => [static fn (ApifyClient $c) => $c->actor('a')->version('0.1')->envVars()->create(new ActorEnvVar('N', 'v')), self::SHORT, self::ITEM_BODY]; + yield 'Actor version list' => [static fn (ApifyClient $c) => $c->actor('a')->versions()->list(), self::SHORT, self::PAGE_BODY]; + yield 'Actor version create' => [static fn (ApifyClient $c) => $c->actor('a')->versions()->create(['versionNumber' => '0.1']), self::SHORT, self::ITEM_BODY]; + yield 'Dataset getOrCreate' => [static fn (ApifyClient $c) => $c->datasets()->getOrCreate('d'), self::SHORT, self::ITEM_BODY]; + yield 'Key-value store getOrCreate' => [static fn (ApifyClient $c) => $c->keyValueStores()->getOrCreate('s'), self::SHORT, self::ITEM_BODY]; + yield 'Request queue getOrCreate' => [static fn (ApifyClient $c) => $c->requestQueues()->getOrCreate('q'), self::SHORT, self::ITEM_BODY]; + yield 'Schedule create' => [static fn (ApifyClient $c) => $c->schedules()->create(['name' => 's']), self::SHORT, self::ITEM_BODY]; + yield 'Webhook create' => [static fn (ApifyClient $c) => $c->webhooks()->create(['requestUrl' => 'https://example.com']), self::SHORT, self::ITEM_BODY]; + yield 'Request queue listHead' => [static fn (ApifyClient $c) => $c->requestQueue('q')->listHead(), self::SHORT, '{"data":{"items":[]}}']; + yield 'Request queue addRequest' => [static fn (ApifyClient $c) => $c->requestQueue('q')->addRequest(new RequestQueueRequest('https://example.com', 'k')), self::SHORT, self::ITEM_BODY]; + yield 'Request queue batchDeleteRequests' => [ + static fn (ApifyClient $c) => $c->requestQueue('q')->batchDeleteRequests([new RequestQueueRequest('https://example.com', 'k', ['id' => 'r1'])]), + self::SHORT, + '{"data":{"processedRequests":[],"unprocessedRequests":[]}}', + ]; + yield 'Request queue deleteRequestLock' => [static fn (ApifyClient $c) => $c->requestQueue('q')->deleteRequestLock('r1'), self::SHORT, '']; + yield 'Key-value store update' => [static fn (ApifyClient $c) => $c->keyValueStore('s')->update(['title' => 't']), self::LONG, self::ITEM_BODY]; + yield 'Key-value store recordExists' => [static fn (ApifyClient $c) => $c->keyValueStore('s')->recordExists('k'), self::LONG, '']; + yield 'Key-value store getRecordPublicUrl' => [static fn (ApifyClient $c) => $c->keyValueStore('s')->getRecordPublicUrl('k'), self::LONG, self::ITEM_BODY]; + yield 'Key-value store createKeysPublicUrl' => [static fn (ApifyClient $c) => $c->keyValueStore('s')->createKeysPublicUrl(), self::LONG, self::ITEM_BODY]; + yield 'Dataset createItemsPublicUrl' => [static fn (ApifyClient $c) => $c->dataset('d')->createItemsPublicUrl(), self::LONG, self::ITEM_BODY]; + yield 'Dataset pushItems' => [static fn (ApifyClient $c) => $c->dataset('d')->pushItems([['a' => 1]]), self::MEDIUM, '']; + yield 'Schedule getLog' => [static fn (ApifyClient $c) => $c->schedule('s')->getLog(), self::MEDIUM, '{"data":[]}']; + } + + /** + * @param callable(ApifyClient):mixed $call + */ + #[DataProvider('tierCases')] + public function testDefaultTierMatchesTheReferenceClient(callable $call, float $expectedTimeout, string $body): void + { + $transport = (new MockTransport())->queueResponse(200, $body); + + $call(new ApifyClient(token: 't', httpClient: $transport)); + + // The call's first (and for the public-URL builders, only) request carries the tier's timeout. + self::assertSame($expectedTimeout, $transport->timeouts[0]); + } + + public function testPublicUrlBuildersAcceptATimeoutForTheStorageRequest(): void + { + $transport = (new MockTransport()) + ->queueResponse(200, self::ITEM_BODY) + ->queueResponse(200, self::ITEM_BODY) + ->queueResponse(200, self::ITEM_BODY); + $client = new ApifyClient(token: 't', httpClient: $transport); + + $client->keyValueStore('s')->getRecordPublicUrl('k', 7); + $client->keyValueStore('s')->createKeysPublicUrl(null, null, 8); + $client->dataset('d')->createItemsPublicUrl(null, null, null, 9); + + self::assertSame([7.0, 8.0, 9.0], $transport->timeouts); + } + + public function testPushItemsDoesNotRetryATimeout(): void + { + $transport = (new MockTransport())->queueError(timeout: true)->queueResponse(200, ''); + $client = new ApifyClient(token: 't', minDelayBetweenRetriesMillis: 1, httpClient: $transport); + + try { + $client->dataset('d')->pushItems([['a' => 1]]); + self::fail('a timed-out push must not be retried'); + } catch (\Apify\Client\Exception\TransportException) { + self::assertSame(1, $transport->callCount()); + } + } + + public function testOtherRequestsRetryATimeout(): void + { + $transport = (new MockTransport())->queueError(timeout: true)->queueResponse(200, Json::encode(['data' => ['id' => 'a']])); + $client = new ApifyClient(token: 't', minDelayBetweenRetriesMillis: 1, httpClient: $transport); + + $client->actor('a')->get(); + + self::assertSame(2, $transport->callCount()); + } +} diff --git a/tests/Unit/PublicUrlExpiryTest.php b/tests/Unit/PublicUrlExpiryTest.php new file mode 100644 index 0000000..aad9f62 --- /dev/null +++ b/tests/Unit/PublicUrlExpiryTest.php @@ -0,0 +1,76 @@ + */ + public static function nonPositiveExpiries(): iterable + { + yield 'zero' => [0]; + yield 'negative' => [-60]; + } + + #[DataProvider('nonPositiveExpiries')] + public function testItemsUrlRejectsANonPositiveExpiryWithoutARequest(int $expiresInSecs): void + { + $transport = new MockTransport(); + $client = new ApifyClient(token: 't', httpClient: $transport); + + try { + $client->dataset('d')->createItemsPublicUrl(null, $expiresInSecs); + self::fail('expected an InvalidArgumentException'); + } catch (InvalidArgumentException $e) { + self::assertStringContainsString('expiresInSecs', $e->getMessage()); + } + self::assertSame(0, $transport->callCount()); + } + + #[DataProvider('nonPositiveExpiries')] + public function testKeysUrlRejectsANonPositiveExpiryWithoutARequest(int $expiresInSecs): void + { + $transport = new MockTransport(); + $client = new ApifyClient(token: 't', httpClient: $transport); + + try { + $client->keyValueStore('s')->createKeysPublicUrl(null, $expiresInSecs); + self::fail('expected an InvalidArgumentException'); + } catch (InvalidArgumentException $e) { + self::assertStringContainsString('expiresInSecs', $e->getMessage()); + } + self::assertSame(0, $transport->callCount()); + } + + public function testAPositiveExpiryStillSignsTheUrl(): void + { + $store = Json::encode(['data' => ['id' => 'd1', 'urlSigningSecretKey' => 'secret']]); + $transport = (new MockTransport())->queueResponse(200, $store)->queueResponse(200, $store); + $client = new ApifyClient(token: 't', httpClient: $transport); + + $items = $client->dataset('d1')->createItemsPublicUrl(null, 60); + $keys = $client->keyValueStore('d1')->createKeysPublicUrl(null, 60); + + self::assertStringContainsString('signature=', $items); + self::assertStringContainsString('signature=', $keys); + } + + public function testOmittingTheExpiryStillGivesANonExpiringSignedUrl(): void + { + $transport = (new MockTransport())->queueResponse(200, Json::encode(['data' => ['id' => 'd1', 'urlSigningSecretKey' => 'secret']])); + $client = new ApifyClient(token: 't', httpClient: $transport); + + self::assertStringContainsString('signature=', $client->dataset('d1')->createItemsPublicUrl()); + } +} diff --git a/tests/Unit/RequestShapeTest.php b/tests/Unit/RequestShapeTest.php index ac4723e..50757aa 100644 --- a/tests/Unit/RequestShapeTest.php +++ b/tests/Unit/RequestShapeTest.php @@ -155,7 +155,7 @@ public function testDefaultBuildClampsWaitToPerRequestTimeout(): void $client = new ApifyClient(token: 't', minDelayBetweenRetriesMillis: 1, timeoutSecs: 5, httpClient: $transport); $client->actor('me~a')->defaultBuild(120); - self::assertStringContainsString('waitForFinish=0', (string) $transport->lastRequest()->getUri()); + self::assertStringContainsString('waitForFinish=2', (string) $transport->lastRequest()->getUri()); } public function testRequestQueueOptionsApplyClientKey(): void diff --git a/tests/Unit/RunIterateDatasetItemsTest.php b/tests/Unit/RunIterateDatasetItemsTest.php new file mode 100644 index 0000000..bd83a52 --- /dev/null +++ b/tests/Unit/RunIterateDatasetItemsTest.php @@ -0,0 +1,206 @@ + ['id' => $id, 'status' => $status]]); + } + + private static function dataset(int $itemCount): string + { + return Json::encode(['data' => ['id' => 'ds1', 'itemCount' => $itemCount]]); + } + + /** @param list $ids */ + private static function items(array $ids): string + { + return Json::encode(array_map(static fn (int $id): array => ['id' => $id], $ids)); + } + + /** @return array */ + private static function scanned(int $count): array + { + return ['X-Apify-Pagination-Count' => (string) $count]; + } + + /** @return list> the offset/limit query of every request, in order */ + private static function queries(MockTransport $transport): array + { + $queries = []; + foreach ($transport->received as $request) { + parse_str($request->getUri()->getQuery(), $query); + $queries[] = $query + ['path' => $request->getUri()->getPath()]; + } + return $queries; + } + + /** + * @param iterable $items + * @return list + */ + private static function ids(iterable $items): array + { + $ids = []; + foreach ($items as $item) { + $ids[] = $item['id']; + } + return $ids; + } + + public function testFinishedRunIsReadThroughItsLaggingItemCountToTheEnd(): void + { + $transport = (new MockTransport()) + ->queueResponse(200, self::runBody('SUCCEEDED')) + ->queueResponse(200, self::dataset(3)) + ->queueResponse(200, self::items([1, 2, 3]), self::scanned(3)) + // `itemCount` (3) lags: a fourth row was pushed. The post-run read finds it... + ->queueResponse(200, self::items([4]), self::scanned(1)) + // ...and the next page is empty, which ends the iteration. + ->queueResponse(200, self::items([]), self::scanned(0)); + + $ids = self::ids($this->client($transport)->run('run1')->iterateDatasetItems()); + + self::assertSame([1, 2, 3, 4], $ids); + $queries = self::queries($transport); + self::assertCount(5, $queries); + self::assertStringEndsWith('/actor-runs/run1/dataset/items', $queries[2]['path']); + self::assertSame(['0', '3'], [$queries[2]['offset'], $queries[2]['limit']]); + self::assertSame(['3', '1000'], [$queries[3]['offset'], $queries[3]['limit']]); + // A non-empty page advances the offset by the page limit (rows requested), not the items returned. + self::assertSame(['1003', '1000'], [$queries[4]['offset'], $queries[4]['limit']]); + } + + public function testRunningRunIsPolledUntilItFinishes(): void + { + $transport = (new MockTransport()) + ->queueResponse(200, self::runBody('RUNNING')) + ->queueResponse(200, self::dataset(2)) + ->queueResponse(200, self::items([1, 2]), self::scanned(2)) + // waitForFinish() poll: the run is done now. + ->queueResponse(200, self::runBody('SUCCEEDED')) + ->queueResponse(200, self::dataset(4)) + ->queueResponse(200, self::items([3, 4]), self::scanned(2)) + ->queueResponse(200, self::items([]), self::scanned(0)); + + $ids = self::ids($this->client($transport)->run('run1')->iterateDatasetItems(new RunIterateDatasetItemsOptions(pollIntervalSecs: 0))); + + self::assertSame([1, 2, 3, 4], $ids); + $queries = self::queries($transport); + self::assertSame(['2', '2'], [$queries[5]['offset'], $queries[5]['limit']]); + } + + public function testChunkSizeSplitsPagesAndLimitStopsTheIteration(): void + { + $transport = (new MockTransport()) + ->queueResponse(200, self::runBody('SUCCEEDED')) + ->queueResponse(200, self::dataset(10)) + ->queueResponse(200, self::items([11, 12]), self::scanned(2)) + ->queueResponse(200, self::items([13, 14]), self::scanned(2)) + ->queueResponse(200, self::items([15]), self::scanned(1)); + + $options = new RunIterateDatasetItemsOptions(offset: 10, limit: 5, chunkSize: 2); + $ids = self::ids($this->client($transport)->run('run1')->iterateDatasetItems($options)); + + // `limit` caps the rows scanned, and the dataset's itemCount (10) is below the offset. + self::assertSame([11, 12, 13, 14, 15], $ids); + $queries = self::queries($transport); + self::assertSame(['10', '2'], [$queries[2]['offset'], $queries[2]['limit']]); + self::assertSame(['14', '1'], [$queries[4]['offset'], $queries[4]['limit']]); + self::assertCount(5, $queries); + } + + public function testFilteredEmptyPageIsConfirmedByAnUnfilteredRead(): void + { + $transport = (new MockTransport()) + ->queueResponse(200, self::runBody('SUCCEEDED')) + ->queueResponse(200, self::dataset(0)) + // The first post-run page is empty after `clean`, but a row sits there: it scanned nothing + // as far as the header says, so the unfiltered one-row read must find it and keep going. + ->queueResponse(200, self::items([]), self::scanned(0)) + ->queueResponse(200, self::items([7])) + ->queueResponse(200, self::items([8]), self::scanned(1)) + ->queueResponse(200, self::items([]), self::scanned(0)) + ->queueResponse(200, self::items([])); + + $ids = self::ids($this->client($transport)->run('run1')->iterateDatasetItems(new RunIterateDatasetItemsOptions(clean: true))); + + self::assertSame([8], $ids); + $queries = self::queries($transport); + // The confirmation read is unfiltered: one row, no `clean`. + self::assertSame('1', $queries[3]['limit']); + self::assertArrayNotHasKey('clean', $queries[3]); + self::assertSame('1', $queries[2]['clean']); + self::assertSame('1000', $queries[4]['limit']); + self::assertCount(7, $queries); + } + + public function testFilteredIterationEndsWhenTheUnfilteredReadFindsNothing(): void + { + $transport = (new MockTransport()) + ->queueResponse(200, self::runBody('SUCCEEDED')) + ->queueResponse(200, self::dataset(0)) + ->queueResponse(200, self::items([]), self::scanned(0)) + ->queueResponse(200, self::items([])); + + $ids = self::ids($this->client($transport)->run('run1')->iterateDatasetItems(new RunIterateDatasetItemsOptions(skipEmpty: true))); + + self::assertSame([], $ids); + self::assertSame(4, $transport->callCount()); + } + + public function testLastRunIterationIsPinnedToTheRunItResolvedTo(): void + { + $transport = (new MockTransport()) + ->queueResponse(200, self::runBody('SUCCEEDED', 'resolvedRun')) + ->queueResponse(200, self::dataset(1)) + ->queueResponse(200, self::items([1]), self::scanned(1)) + ->queueResponse(200, self::items([]), self::scanned(0)); + + $ids = self::ids($this->client($transport)->actor('user~actor')->lastRun()->iterateDatasetItems()); + + self::assertSame([1], $ids); + $queries = self::queries($transport); + self::assertStringEndsWith('/actors/user~actor/runs/last', $queries[0]['path']); + // Every later request addresses the resolved run by ID, not `runs/last`. + self::assertStringEndsWith('/actor-runs/resolvedRun/dataset', $queries[1]['path']); + self::assertStringEndsWith('/actor-runs/resolvedRun/dataset/items', $queries[2]['path']); + } + + public function testRequestsUseTheLongTierByDefaultAndTheCallerTimeoutWhenGiven(): void + { + $transport = (new MockTransport()) + ->queueResponse(200, self::runBody('SUCCEEDED')) + ->queueResponse(200, self::dataset(0)) + ->queueResponse(200, self::items([]), self::scanned(0)); + $client = new ApifyClient(token: 't', timeoutLongSecs: 100, httpClient: $transport); + iterator_to_array($client->run('run1')->iterateDatasetItems()); + self::assertSame([100.0, 100.0, 100.0], $transport->timeouts); + + $transport = (new MockTransport()) + ->queueResponse(200, self::runBody('SUCCEEDED')) + ->queueResponse(200, self::dataset(0)) + ->queueResponse(200, self::items([]), self::scanned(0)); + $client = new ApifyClient(token: 't', httpClient: $transport); + iterator_to_array($client->run('run1')->iterateDatasetItems(null, 7)); + self::assertSame([7.0, 7.0, 7.0], $transport->timeouts); + } +} diff --git a/tests/Unit/SpecSyncTest.php b/tests/Unit/SpecSyncTest.php new file mode 100644 index 0000000..622ac34 --- /dev/null +++ b/tests/Unit/SpecSyncTest.php @@ -0,0 +1,160 @@ + */ + private static function queryOf(MockTransport $transport): array + { + parse_str($transport->lastRequest()->getUri()->getQuery(), $query); + return $query; + } + + public function testRunListsSendTheDeploymentFilterAsACommaSeparatedList(): void + { + $page = Json::encode(['data' => ['items' => [], 'total' => 0, 'offset' => 0, 'limit' => 10, 'count' => 0, 'desc' => false]]); + $options = new RunListOptions(status: ['RUNNING'], deployment: ['SINGLE_TENANT', 'MULTI_TENANT_FREE_USERS']); + + $transport = (new MockTransport())->queueResponse(200, $page); + $this->client($transport)->runs()->list(null, $options); + self::assertStringContainsString('/v2/actor-runs?', (string) $transport->lastRequest()->getUri()); + self::assertSame('SINGLE_TENANT,MULTI_TENANT_FREE_USERS', self::queryOf($transport)['deployment']); + + $transport = (new MockTransport())->queueResponse(200, $page); + $this->client($transport)->actor('a1')->runs()->list(null, $options); + self::assertStringContainsString('/v2/actors/a1/runs?', (string) $transport->lastRequest()->getUri()); + self::assertSame('SINGLE_TENANT,MULTI_TENANT_FREE_USERS', self::queryOf($transport)['deployment']); + } + + public function testDeploymentFilterIsOmittedByDefault(): void + { + $transport = (new MockTransport())->queueResponse(200, Json::encode(['data' => ['items' => []]])); + $this->client($transport)->runs()->list(); + + self::assertArrayNotHasKey('deployment', self::queryOf($transport)); + } + + public function testStoreListSendsIncludeInputSchema(): void + { + $transport = (new MockTransport())->queueResponse(200, Json::encode(['data' => ['items' => []]])); + $this->client($transport)->store()->list(new StoreListOptions(includeInputSchema: true)); + + self::assertSame('1', self::queryOf($transport)['includeInputSchema']); + } + + public function testStoreIterationKeepsIncludeInputSchemaOnEveryPage(): void + { + $options = (new StoreListOptions(includeInputSchema: true))->withPagination(10, 5); + + $transport = (new MockTransport())->queueResponse(200, Json::encode(['data' => ['items' => []]])); + $this->client($transport)->store()->list($options); + + self::assertSame('1', self::queryOf($transport)['includeInputSchema']); + self::assertSame('10', self::queryOf($transport)['offset']); + } + + public function testUserLimitsAndMonthlyUsageAreAvailableForAnyUser(): void + { + $transport = (new MockTransport()) + ->queueResponse(200, Json::encode(['data' => ['limits' => ['maxConcurrentActorJobs' => 5]]])) + ->queueResponse(200, Json::encode(['data' => ['usageCycle' => []]])) + ->queueResponse(200, ''); + $client = $this->client($transport); + + self::assertSame(['limits' => ['maxConcurrentActorJobs' => 5]], $client->user('someone')->limits()); + $client->user('someone')->monthlyUsage('2026-10-01'); + $client->user('someone')->updateLimits(['maxMonthlyUsageUsd' => 10]); + + $uris = array_map(static fn ($request): string => (string) $request->getUri(), $transport->received); + self::assertStringEndsWith('/v2/users/someone/limits', $uris[0]); + self::assertStringEndsWith('/v2/users/someone/usage/monthly?date=2026-10-01', $uris[1]); + self::assertStringEndsWith('/v2/users/someone/limits', $uris[2]); + self::assertSame('PUT', $transport->lastRequest()->getMethod()); + } + + public function testScheduleLogReturnsTheInvocationEntries(): void + { + $entries = [ + ['message' => 'Schedule invoked', 'level' => 'INFO', 'createdAt' => '2026-10-01T10:00:00.000Z'], + ['message' => 'Cannot start Actor task', 'level' => 'ERROR', 'createdAt' => '2026-10-01T11:00:00.000Z'], + ]; + $transport = (new MockTransport()) + ->queueResponse(200, Json::encode(['data' => $entries])) + ->queueResponse(200, Json::encode(['data' => []])); + $client = new ApifyClient(token: 't', httpClient: $transport); + + self::assertSame($entries, $client->schedule('s1')->getLog()); + self::assertSame([], $client->schedule('s1')->getLog()); + self::assertStringEndsWith('/v2/schedules/s1/log', (string) $transport->lastRequest()->getUri()); + // The schedule log is read with the medium tier (30s by default), not the long download tier. + self::assertSame([30.0, 30.0], $transport->timeouts); + } + + /** @return iterable */ + public static function emptyScheduleLogBodies(): iterable + { + yield 'null data' => ['{"data":null}']; + yield 'missing data' => ['{}']; + } + + #[DataProvider('emptyScheduleLogBodies')] + public function testScheduleLogWithoutEntriesIsAnEmptyList(string $body): void + { + $transport = (new MockTransport())->queueResponse(200, $body); + + self::assertSame([], $this->client($transport)->schedule('s1')->getLog()); + } + + public function testScheduleWithANullUiStateIsReadAsIs(): void + { + // The API returns `uiState: null` once an API update changed the cron expression or timezone. + $transport = (new MockTransport())->queueResponse(200, Json::encode(['data' => [ + 'id' => 's1', + 'cronExpression' => '0 12 * * *', + 'uiState' => null, + ]])); + + $schedule = $this->client($transport)->schedule('s1')->get(); + + self::assertNotNull($schedule); + self::assertSame('0 12 * * *', $schedule->getCronExpression()); + self::assertNull($schedule->get('uiState')); + self::assertArrayHasKey('uiState', $schedule->toArray()); + } + + public function testPendingStoredRequestWithANullHandledAtIsReadAsIs(): void + { + $transport = (new MockTransport())->queueResponse(200, Json::encode(['data' => [ + 'id' => 'r1', + 'uniqueKey' => 'https://example.com', + 'url' => 'https://example.com', + 'handledAt' => null, + ]])); + + $request = $this->client($transport)->requestQueue('q1')->getRequest('r1'); + + self::assertNotNull($request); + self::assertSame('r1', $request->getId()); + self::assertNull($request->get('handledAt')); + } +} diff --git a/tests/Unit/TimeoutTiersTest.php b/tests/Unit/TimeoutTiersTest.php index bb9b318..4557611 100644 --- a/tests/Unit/TimeoutTiersTest.php +++ b/tests/Unit/TimeoutTiersTest.php @@ -6,28 +6,79 @@ use Apify\Client\ApifyClient; use Apify\Client\Internal\Json; +use Apify\Client\Options\ActorBuildOptions; +use Apify\Client\Options\ActorStartOptions; use Apify\Client\Options\RequestQueueClientOptions; +use Apify\Client\Options\TaskStartOptions; use PHPUnit\Framework\TestCase; /** - * Covers the timeout-tier configuration (matches the reference client's `#1046`): every tier defaults - * to the client's single legacy {@code timeoutSecs} unless explicitly configured, so a client built - * without the new options sends every call with exactly the timeout it always has (no breaking - * behavior change from introducing tiers); a per-call override replaces the method's tier for that - * call alone; and {@code timeoutMaxSecs} caps both. + * Covers the timeout-tier configuration (matches the reference client's `#1046`): the {@code short} and + * {@code medium} tiers default to 5s and 30s, the {@code long} tier and {@code timeoutMaxSecs} to the + * client's overall {@code timeoutSecs}; a per-call override replaces the method's tier for that call + * alone; {@code timeoutMaxSecs} caps both; and a {@code waitForFinish} request gets the API's hold on + * top of its tier. */ final class TimeoutTiersTest extends TestCase { - public function testEveryTierDefaultsToTheLegacyTimeoutSecsWhenUnconfigured(): void + public function testTiersUseTheReferenceDefaultsWhenUnconfigured(): void { - // No timeoutShortSecs/timeoutMediumSecs/timeoutLongSecs/timeoutMaxSecs given: every call must - // use exactly $timeoutSecs, byte-for-byte the pre-tiers behavior. - $transport = (new MockTransport())->queueResponse(200, Json::encode(['data' => ['id' => 'a1']])); + // No timeout*Secs tier option given: short 5s, medium 30s, long and the cap the overall timeoutSecs. + $transport = (new MockTransport()) + ->queueResponse(200, Json::encode(['data' => ['id' => 'a1']])) + ->queueResponse(200, Json::encode(['data' => ['items' => []]])) + ->queueResponse(200, Json::encode(['data' => ['id' => 'a1']])); $client = new ApifyClient(token: 't', timeoutSecs: 123, httpClient: $transport); $client->actor('a1')->get(); + $client->actors()->list(); + $client->actor('a1')->get(timeoutSecs: 'long'); - self::assertSame([123.0], $transport->timeouts); + self::assertSame([5.0, 30.0, 123.0], $transport->timeouts); + } + + public function testDefaultTimeoutSecsIsTheLongTierAndTheCap(): void + { + $transport = (new MockTransport())->queueResponse(200, Json::encode(['data' => ['id' => 'a1']])); + $client = new ApifyClient(token: 't', httpClient: $transport); + + $client->actor('a1')->get(timeoutSecs: 'long'); + + self::assertSame([360.0], $transport->timeouts); + } + + public function testWaitForFinishRequestGetsTheApiHoldOnTopOfItsTier(): void + { + $transport = (new MockTransport()) + ->queueResponse(200, Json::encode(['data' => ['id' => 'run1', 'status' => 'RUNNING']])) + ->queueResponse(200, Json::encode(['data' => ['id' => 'run1', 'status' => 'RUNNING']])) + ->queueResponse(200, Json::encode(['data' => ['id' => 'run1', 'status' => 'RUNNING']])) + ->queueResponse(200, Json::encode(['data' => ['id' => 'run1', 'status' => 'RUNNING']])); + $client = new ApifyClient(token: 't', httpClient: $transport); + + $client->run('run1')->get(); + $client->run('run1')->get(30); + // The API holds a response for at most 60s, however long the parameter asks for. + $client->run('run1')->get(600); + // An explicit per-call timeout is used as given. + $client->run('run1')->get(30, 9); + + self::assertSame([5.0, 35.0, 65.0, 9.0], $transport->timeouts); + } + + public function testStartAndBuildWaitForFinishOptionExtendTheMediumTier(): void + { + $transport = (new MockTransport()) + ->queueResponse(200, Json::encode(['data' => ['id' => 'run1', 'status' => 'READY']])) + ->queueResponse(200, Json::encode(['data' => ['id' => 'run1', 'status' => 'READY']])) + ->queueResponse(200, Json::encode(['data' => ['id' => 'b1', 'status' => 'READY']])); + $client = new ApifyClient(token: 't', httpClient: $transport); + + $client->actor('a1')->start(null, new ActorStartOptions(waitForFinish: 20)); + $client->task('t1')->start(null, new TaskStartOptions(waitForFinish: 60)); + $client->actor('a1')->build('0.1', new ActorBuildOptions(waitForFinish: 5)); + + self::assertSame([50.0, 90.0, 35.0], $transport->timeouts); } public function testConfiguredShortTierAppliesToGet(): void @@ -138,4 +189,76 @@ public function testUpdateAndDeleteAcceptThePerCallOverrideToo(): void self::assertSame([11.0, 22.0], $transport->timeouts); } + + private static function runBody(string $status): string + { + return Json::encode(['data' => ['id' => 'run1', 'status' => $status]]); + } + + /** @return array */ + private static function queryOf(MockTransport $transport, int $index): array + { + parse_str($transport->received[$index]->getUri()->getQuery(), $query); + return $query; + } + + public function testWaitForFinishPollsHaveNoRequestTimeoutByDefault(): void + { + // A raised cap must not make the poll hold the API (60s) under a shorter timeout. + $transport = (new MockTransport())->queueResponse(200, self::runBody('SUCCEEDED')); + $client = new ApifyClient(token: 't', timeoutSecs: 30, timeoutMaxSecs: 600, httpClient: $transport); + + $client->run('run1')->waitForFinish(); + + self::assertSame([0.0], $transport->timeouts); + self::assertSame('60', self::queryOf($transport, 0)['waitForFinish']); + } + + public function testWaitForFinishHonoursAnExplicitPollTimeoutAndKeepsTheHoldBelowIt(): void + { + $transport = (new MockTransport()) + ->queueResponse(200, self::runBody('SUCCEEDED')) + ->queueResponse(200, Json::encode(['data' => ['id' => 'b1', 'status' => 'SUCCEEDED']])); + $client = new ApifyClient(token: 't', httpClient: $transport); + + $client->run('run1')->waitForFinish(120, 20); + $client->build('b1')->waitForFinish(120, 20); + + self::assertSame([20.0, 20.0], $transport->timeouts); + self::assertSame('15', self::queryOf($transport, 0)['waitForFinish']); + self::assertSame('15', self::queryOf($transport, 1)['waitForFinish']); + } + + public function testCallStartsAndPollsWithoutARequestTimeoutByDefault(): void + { + $transport = (new MockTransport()) + ->queueResponse(200, self::runBody('READY')) + ->queueResponse(200, self::runBody('SUCCEEDED')) + ->queueResponse(200, self::runBody('READY')) + ->queueResponse(200, self::runBody('SUCCEEDED')); + $client = new ApifyClient(token: 't', httpClient: $transport); + + $client->actor('a1')->call(); + $client->task('t1')->call(null, null, null, 40); + + self::assertSame([0.0, 0.0, 40.0, 40.0], $transport->timeouts); + } + + public function testLiveDatasetIterationUsesItsTimeoutForThePollsToo(): void + { + $transport = (new MockTransport()) + ->queueResponse(200, self::runBody('RUNNING')) + ->queueResponse(200, Json::encode(['data' => ['id' => 'ds1', 'itemCount' => 0]])) + ->queueResponse(200, self::runBody('SUCCEEDED')) + ->queueResponse(200, Json::encode(['data' => ['id' => 'ds1', 'itemCount' => 0]])) + ->queueResponse(200, '[]', ['X-Apify-Pagination-Count' => '0']); + $client = new ApifyClient(token: 't', httpClient: $transport); + + iterator_to_array($client->run('run1')->iterateDatasetItems(null, 12), false); + + // Every request, the wait-for-finish poll included, carries the iteration's timeout. + self::assertSame([12.0, 12.0, 12.0, 12.0, 12.0], $transport->timeouts); + // The poll's hold (5s poll interval) stays below its 12s timeout. + self::assertSame('5', self::queryOf($transport, 2)['waitForFinish']); + } } diff --git a/tests/Unit/UrlNormalizerTest.php b/tests/Unit/UrlNormalizerTest.php new file mode 100644 index 0000000..c4374ac --- /dev/null +++ b/tests/Unit/UrlNormalizerTest.php @@ -0,0 +1,160 @@ + */ + public static function normalizationCases(): iterable + { + yield 'already normalized' => ['https://api.apify.com/v2/datasets/abc/items', 'https://api.apify.com/v2/datasets/abc/items']; + yield 'empty path gets a slash' => ['https://abc.runs.apify.net', 'https://abc.runs.apify.net/']; + yield 'scheme and host are lowercased, default port dropped' => ['HTTPS://Example.COM:443/Path', 'https://example.com/Path']; + yield 'default port of http' => ['http://example.com:80/a', 'http://example.com/a']; + yield 'other port kept' => ['http://example.com:8080', 'http://example.com:8080/']; + yield 'leading zeros of a port' => ['https://example.com:0080/', 'https://example.com:80/']; + yield 'dot segments' => ['http://example.com:80/a/./b/../c', 'http://example.com/a/c']; + yield 'trailing parent segment' => ['https://example.com/a/..', 'https://example.com/']; + yield 'trailing current segment' => ['https://example.com/a/.', 'https://example.com/a/']; + yield 'unsafe characters' => [ + 'https://example.com/a b?x=a b&y="q"#frag ment', + 'https://example.com/a%20b?x=a%20b&y=%22q%22#frag%20ment', + ]; + yield 'existing escapes and non-ASCII' => [ + 'https://example.com/a%20b/ü?ü=1#ü', + 'https://example.com/a%20b/%C3%BC?%C3%BC=1#%C3%BC', + ]; + yield 'path characters' => ['https://example.com/{x}|`<>"', 'https://example.com/%7Bx%7D|%60%3C%3E%22']; + yield 'user info' => ['https://user:p@ss@example.com/x', 'https://user:p%40ss@example.com/x']; + yield 'user without password' => ['https://user@example.com', 'https://user@example.com/']; + yield 'ipv6 host' => ['http://[::1]:3000/x', 'http://[::1]:3000/x']; + yield 'query without path' => ['https://example.com?q=1', 'https://example.com/?q=1']; + yield 'fragment without path' => ['https://example.com#f', 'https://example.com/#f']; + yield 'empty segments are kept' => ['https://example.com//a//b', 'https://example.com//a//b']; + yield 'backslash is a slash' => ['https://example.com/a\\b', 'https://example.com/a/b']; + yield 'websocket scheme' => ['wss://Example.com:443', 'wss://example.com/']; + } + + #[DataProvider('normalizationCases')] + public function testNormalizesLikeTheWhatwgUrlSerialization(string $input, string $expected): void + { + self::assertSame($expected, UrlNormalizer::normalize($input)); + // Normalizing is idempotent. + self::assertSame($expected, UrlNormalizer::normalize($expected)); + } + + public function testInternationalizedHostBecomesPunycode(): void + { + if (!function_exists('idn_to_ascii')) { + self::markTestSkipped('The intl extension is needed to convert an internationalized host to punycode.'); + } + self::assertSame('https://xn--bcher-kva.example/stra%C3%9Fe', UrlNormalizer::normalize('https://bücher.example/straße')); + } + + /** @return iterable */ + public static function internationalizedHostCases(): iterable + { + yield 'hyphens' => ["https://AB--c.b\u{00FC}cher.example/x", 'https://ab--c.xn--bcher-kva.example/x']; + yield 'leading hyphen' => ["https://-a.b\u{00FC}cher.example/x", 'https://-a.xn--bcher-kva.example/x']; + yield 'empty label' => ["https://a..b\u{00FC}cher.example/x", 'https://a..xn--bcher-kva.example/x']; + yield 'right-to-left digit label' => ["https://\u{0661}.com/", 'https://xn--9hb.com/']; + yield 'right-to-left label after an ASCII one' => ["https://\u{05D0}.1a.com/", 'https://xn--4db.1a.com/']; + yield 'replacement character' => ["https://example.com\u{FFFD}/", "https://example.com\u{FFFD}/"]; + yield 'zero width joiner' => ["https://a\u{200D}b\u{00FC}.com/", "https://a\u{200D}b\u{00FC}.com/"]; + } + + #[DataProvider('internationalizedHostCases')] + public function testInternationalizedHostsFollowWhatwg(string $input, string $expected): void + { + if (!function_exists('idn_to_ascii')) { + self::markTestSkipped('The intl extension is needed to convert an internationalized host to punycode.'); + } + self::assertSame($expected, UrlNormalizer::normalize($input)); + } + + /** @return iterable */ + public static function untouchedValues(): iterable + { + yield 'relative' => ['/v2/datasets']; + yield 'no host' => ['https:///']; + yield 'bad port' => ['https://example.com:port/x']; + yield 'non-special scheme' => ['mailto:Someone@Example.com']; + yield 'empty' => ['']; + yield 'encoded slash in host' => ['https://ex%2Fample.com/hook']; + yield 'encoded space in host' => ['https://exa%20mple.com/']; + yield 'fullwidth at sign in host' => ["https://evil.com\u{FF20}good.com/hook"]; + yield 'fullwidth slash in host' => ["https://ex\u{FF0F}ample.com/a"]; + yield 'fullwidth percent in host' => ["https://ex\u{FF05}ample.com/a"]; + yield 'text after the IPv6 literal' => ['https://[::1]x/y']; + yield 'not an IP literal in brackets' => ['https://[nothost]/x']; + yield 'port out of range' => ['https://example.com:70000/x']; + } + + #[DataProvider('untouchedValues')] + public function testLeavesAValueThatIsNotAnAbsoluteWebUrlAsReceived(string $value): void + { + self::assertSame($value, UrlNormalizer::normalize($value)); + } + + public function testNormalizeFieldsFollowsPathsAndWildcards(): void + { + $data = [ + 'a' => 'https://Example.com', + 'nested' => ['b' => 'http://X.org:80', 'untouched' => 'http://X.org:80'], + 'map' => ['one' => 'https://One.example', 'two' => 'https://Two.example:443'], + 'notAString' => 5, + ]; + + $normalized = UrlNormalizer::normalizeFields($data, ['a', 'nested.b', 'map.*', 'notAString', 'missing.key']); + + self::assertSame('https://example.com/', $normalized['a']); + self::assertSame('http://x.org/', $normalized['nested']['b']); + self::assertSame('http://X.org:80', $normalized['nested']['untouched']); + self::assertSame(['one' => 'https://one.example/', 'two' => 'https://two.example/'], $normalized['map']); + self::assertSame(5, $normalized['notAString']); + } + + public function testModelsNormalizeTheirUrlFields(): void + { + $run = new ActorRun([ + 'containerUrl' => 'https://abc.runs.apify.net', + 'consoleUrl' => 'https://console.apify.com/actors/runs/r1', + 'output' => ['results' => 'https://API.apify.com:443/v2/datasets/d1/items'], + ]); + self::assertSame('https://abc.runs.apify.net/', $run->getContainerUrl()); + self::assertSame('https://abc.runs.apify.net/', $run->get('containerUrl')); + self::assertSame('https://api.apify.com/v2/datasets/d1/items', $run->toArray()['output']['results']); + + self::assertSame('https://x.example/', (new Dataset(['itemsPublicUrl' => 'https://X.example']))->get('itemsPublicUrl')); + self::assertSame('https://x.example/', (new KeyValueStore(['keysPublicUrl' => 'https://X.example']))->get('keysPublicUrl')); + self::assertSame('https://x.example/', (new KeyValueStore(['recordsPublicUrl' => 'https://X.example']))->get('recordsPublicUrl')); + self::assertSame('https://x.example/', (new KeyValueStoreKey(['recordPublicUrl' => 'https://X.example']))->get('recordPublicUrl')); + self::assertSame('https://x.example/', (new Task(['standbyUrl' => 'https://X.example']))->get('standbyUrl')); + self::assertSame('https://x.example/hook', (new Webhook(['requestUrl' => 'https://X.example:443/hook']))->getRequestUrl()); + self::assertSame( + 'https://x.example/hook', + (new WebhookDispatch(['webhook' => ['requestUrl' => 'https://X.example/hook']]))->toArray()['webhook']['requestUrl'], + ); + $user = new User(['profile' => ['pictureUrl' => 'https://X.example', 'websiteUrl' => 'https://Y.example', 'name' => 'Not A URL']]); + self::assertSame(['https://x.example/', 'https://y.example/', 'Not A URL'], array_values($user->toArray()['profile'])); + } +} diff --git a/tests/Unit/WaitForFinishTest.php b/tests/Unit/WaitForFinishTest.php index 3906631..9618440 100644 --- a/tests/Unit/WaitForFinishTest.php +++ b/tests/Unit/WaitForFinishTest.php @@ -84,4 +84,13 @@ public function testNonNotFoundErrorPropagatesImmediately(): void $this->expectException(\Apify\Client\Exception\ForbiddenException::class); $this->client($transport)->run('run1')->waitForFinish(30); } + + public function testShortPollTimeoutStillAsksForAPositiveServerHold(): void + { + $transport = (new MockTransport())->queueResponse(200, self::runBody('SUCCEEDED')); + + $this->client($transport)->run('run1')->waitForFinish(30, 5); + + self::assertStringContainsString('waitForFinish=2', (string) $transport->lastRequest()->getUri()); + } }