Skip to content

Commit 8e628a7

Browse files
mrkaye97mrkaye97
authored
Docs: Durable execution, part I (#3387)
* docs: initial work rewriting durable documentation fix: reword a bit fix: rename page fix: rewrite a bit more fix: more wording, sdk specifics fix: more wording fix: rm guides chore: add tsbuildinfo to gitignore fix: rm unhelpful entry from table fix: more rework * fix: finish rewriting top level durable tasks page * fix: actually finish this time * fix: finish rewriting dags page * fix: remove mixing patterns section, fix links, move important stuff elsewhere * fix: revert changes to next batch of pages * fix: dead links * chore: lint * fix: rm superseded quality test * fix: links, redirects * fix: another link * fix: rename * fix: add callout in durable section * fix: add redirect from mixing-patterns page * fix: start reworking child spawning page, remove a bunch of unnecessary stuff * docs: add callout * fix: start moving durable tasks around * fix: start moving around dags page, eviction page, etc. * docs: rewrite task eviction docs * docs: rewrite durable sleep docs * fix: consolidate events docs * fix: clean up events pages * docs: finish up event waits page, write some more examples * fix: remove on failure page, it's not helpful * docs: initial dags page cleanup * fix: remove more ai cruft * fix: more dags work * chore: lint * fix: more dag cleanup * fix: search quality tests, redirects * chore: delete a bunch of unused pages * docs: rewrite a bunch of the concurrency page * docs: rewrite the rest of the concurrency doc * fix: reorder durable section to put child spawning second * fix: typo * fix: semantics
1 parent 0d9e180 commit 8e628a7

48 files changed

Lines changed: 754 additions & 2034 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎examples/python/durable/worker.py‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -249,7 +249,8 @@ async def durable_sleep_event_spawn(
249249

250250

251251
class EventLookbackInput(BaseModel):
252-
scope: str
252+
scope: str | None = None
253+
user_id: int | None = None
253254

254255

255256
class LookbackEventPayload(BaseModel):
@@ -275,9 +276,12 @@ async def wait_for_event_lookback(
275276
input: EventLookbackInput, ctx: DurableContext
276277
) -> EventLookbackResultWithEvent:
277278
start = time.time()
279+
280+
# > Wait for event with lookback
278281
event = await ctx.aio_wait_for_event(
279-
EVENT_KEY,
280-
scope=input.scope,
282+
key="user:create",
283+
expression=f"input.user_id == {input.user_id}",
284+
scope=f"user_id:{input.user_id}",
281285
lookback_window=timedelta(minutes=1),
282286
payload_validator=LookbackEventPayload,
283287
)

‎examples/typescript/durable/workflow.ts‎

Lines changed: 15 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
import { z } from 'zod';
12
import { Or, SleepCondition, UserEventCondition } from '@hatchet-dev/typescript-sdk/v1/conditions';
23
import { NonDeterminismError } from '@hatchet-dev/typescript-sdk/util/errors/non-determinism-error';
34
import sleep from '@hatchet-dev/typescript-sdk/util/sleep';
@@ -248,18 +249,26 @@ export const durableReplayReset = hatchet.durableTask({
248249

249250
export const LOOKBACK_WINDOW = '1m' as const;
250251

252+
const lookbackEventPayloadSchema = z.object({
253+
order: z.string(),
254+
user_id: z.number(),
255+
});
256+
251257
export const waitForEventLookback = hatchet.durableTask({
252258
name: 'wait-for-event-lookback',
253259
executionTimeout: '10m',
254-
fn: async (input: { scope: string }, ctx) => {
260+
fn: async (input: { userId: number }, ctx) => {
255261
const start = Date.now();
262+
263+
// > Wait for event with lookback
256264
const event = await ctx.waitForEvent(
257-
EVENT_KEY,
258-
undefined,
259-
undefined,
260-
input.scope,
261-
LOOKBACK_WINDOW
265+
'user:create',
266+
`input.user_id == ${input.userId}`,
267+
lookbackEventPayloadSchema,
268+
`user_id:${input.userId}`,
269+
'1m'
262270
);
271+
263272
return {
264273
elapsed: (Date.now() - start) / 1000,
265274
event,

‎examples/typescript/durable_event/workflow.ts‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import { hatchet } from '../hatchet-client';
22

33
export const EVENT_KEY = 'user:update';
4+
export const SCOPE = 'user-1234';
45

56
// > Durable Event
67
export const durableEvent = hatchet.durableTask({
@@ -31,3 +32,18 @@ export const durableEventWithFilter = hatchet.durableTask({
3132
};
3233
},
3334
});
35+
36+
// > Durable Event With Lookback
37+
export const durableEventWithLookback = hatchet.durableTask({
38+
name: 'durable-event-with-lookback',
39+
executionTimeout: '10m',
40+
fn: async (_, ctx) => {
41+
const res = await ctx.waitForEvent(EVENT_KEY, undefined, undefined, SCOPE, '1m');
42+
43+
console.log('res', res);
44+
45+
return {
46+
Value: 'done',
47+
};
48+
},
49+
});

‎frontend/docs/.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,3 +4,4 @@ public/llms.txt
44
public/llms-full.txt
55
public/llms/
66
public/llms-search-index.json
7+
tsconfig.tsbuildinfo

‎frontend/docs/next.config.mjs‎

Lines changed: 20 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -48,14 +48,25 @@ const nextConfig = {
4848
{ source: '/reference/typescript/feature-clients/', destination: '/reference/typescript/feature-clients/crons', permanent: false, basePath: false },
4949
{ source: '/reference/python/feature-clients', destination: '/reference/python/feature-clients/cron', permanent: false, basePath: false },
5050
{ source: '/reference/python/feature-clients/', destination: '/reference/python/feature-clients/cron', permanent: false, basePath: false },
51+
// --- Deleted v1 pages that were consolidated/renamed ---
52+
{ source: '/v1/conditions', destination: '/v1/directed-acyclic-graphs#branching-with-parent-conditions', permanent: true, basePath: false },
53+
{ source: '/v1/on-failure', destination: '/v1/error-handling', permanent: true, basePath: false },
54+
{ source: '/v1/sleep', destination: '/v1/durable-sleep', permanent: true, basePath: false },
55+
{ source: '/v1/external-events', destination: '/v1/events', permanent: true, basePath: false },
56+
{ source: '/v1/external-events/:path*', destination: '/v1/events', permanent: true, basePath: false },
57+
{ source: '/v1/patterns/directed-acyclic-graphs', destination: '/v1/directed-acyclic-graphs', permanent: true, basePath: false },
58+
{ source: '/v1/patterns/durable-task-execution', destination: '/v1/durable-tasks', permanent: true, basePath: false },
59+
{ source: '/v1/patterns/mixing-patterns', destination: '/v1/durable-execution', permanent: true, basePath: false },
60+
{ source: '/v1/patterns', destination: '/v1/durable-execution', permanent: true, basePath: false },
61+
{ source: '/v1/patterns/:path*', destination: '/v1/durable-execution', permanent: true, basePath: false },
5162
// --- Old main: /home/* → /v1/* (only paths that existed on main) ---
52-
{ source: '/home/conditional-workflows', destination: '/v1/conditions', permanent: true, basePath: false },
53-
{ source: '/home/on-failure-tasks', destination: '/v1/on-failure', permanent: true, basePath: false },
54-
{ source: '/home/durable-execution', destination: '/v1/patterns/durable-task-execution', permanent: true, basePath: false },
55-
{ source: '/home/:slug(dags|orchestration)', destination: '/v1/patterns/directed-acyclic-graphs', permanent: true, basePath: false },
56-
{ source: '/home/durable-sleep', destination: '/v1/sleep', permanent: true, basePath: false },
63+
{ source: '/home/conditional-workflows', destination: '/v1/directed-acyclic-graphs#branching-with-parent-conditions', permanent: true, basePath: false },
64+
{ source: '/home/on-failure-tasks', destination: '/v1/error-handling', permanent: true, basePath: false },
65+
{ source: '/home/durable-execution', destination: '/v1/durable-execution', permanent: true, basePath: false },
66+
{ source: '/home/:slug(dags|orchestration)', destination: '/v1/directed-acyclic-graphs', permanent: true, basePath: false },
67+
{ source: '/home/durable-sleep', destination: '/v1/durable-sleep', permanent: true, basePath: false },
5768
{ source: '/home/durable-events', destination: '/v1/events', permanent: true, basePath: false },
58-
{ source: '/home/durable-best-practices', destination: '/v1/patterns/mixing-patterns', permanent: true, basePath: false },
69+
{ source: '/home/durable-best-practices', destination: '/v1/durable-execution', permanent: true, basePath: false },
5970
{ source: '/home/architecture', destination: '/v1/architecture-and-guarantees', permanent: true, basePath: false },
6071
{ source: '/home/your-first-task', destination: '/v1/tasks', permanent: true, basePath: false },
6172
{ source: '/home/running-tasks', destination: '/v1/tasks', permanent: true, basePath: false },
@@ -75,11 +86,11 @@ const nextConfig = {
7586
{ source: '/home/basics/(steps|workflows)', destination: '/v1/tasks', permanent: true, basePath: false },
7687
{ source: '/home/basics/environments', destination: '/v1/setup/advanced/environments', permanent: true, basePath: false },
7788
{ source: '/home/features/concurrency/:path*', destination: '/v1/concurrency', permanent: true, basePath: false },
78-
{ source: '/home/features/durable-execution', destination: '/v1/patterns/durable-task-execution', permanent: true, basePath: false },
89+
{ source: '/home/features/durable-execution', destination: '/v1/durable-execution', permanent: true, basePath: false },
7990
{ source: '/home/features/retries/:path*', destination: '/v1/retry-policies', permanent: true, basePath: false },
8091
{ source: '/home/features/errors-and-logging', destination: '/v1/logging', permanent: true, basePath: false },
81-
{ source: '/home/features/on-failure-step', destination: '/v1/on-failure', permanent: true, basePath: false },
82-
{ source: '/home/features/triggering-runs/event-trigger', destination: '/v1/external-events/run-on-event', permanent: true, basePath: false },
92+
{ source: '/home/features/on-failure-step', destination: '/v1/error-handling', permanent: true, basePath: false },
93+
{ source: '/home/features/triggering-runs/event-trigger', destination: '/v1/events', permanent: true, basePath: false },
8394
{ source: '/home/features/triggering-runs/cron-trigger', destination: '/v1/cron-runs', permanent: true, basePath: false },
8495
{ source: '/home/features/triggering-runs/schedule-trigger', destination: '/v1/scheduled-runs', permanent: true, basePath: false },
8596
{ source: '/home/features/rate-limits', destination: '/v1/rate-limits', permanent: true, basePath: false },

‎frontend/docs/pages/reference/python/context.mdx‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ Methods:
1313

1414
| Name | Description |
1515
| ---------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
16-
| `was_skipped` | Check if a given task was skipped. You can read about skipping in [the docs](/v1/conditions#checking-if-a-task-was-skipped). |
16+
| `was_skipped` | Check if a given task was skipped. You can read about skipping in [the docs](/v1/directed-acyclic-graphs#checking-if-a-parent-was-skipped). |
1717
| `task_output` | Get the output of a parent task in a DAG. |
1818
| `cancel` | Cancel the current task run. This will call the Hatchet API to cancel the step run and set the exit flag to True. |
1919
| `aio_cancel` | Cancel the current task run. This will call the Hatchet API to cancel the step run and set the exit flag to True. |
@@ -146,7 +146,7 @@ Returns:
146146

147147
#### `was_skipped`
148148

149-
Check if a given task was skipped. You can read about skipping in [the docs](/v1/conditions#checking-if-a-task-was-skipped).
149+
Check if a given task was skipped. You can read about skipping in [the docs](/v1/directed-acyclic-graphs#checking-if-a-parent-was-skipped).
150150

151151
Parameters:
152152

‎frontend/docs/pages/v1/_meta.js‎

Lines changed: 8 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -20,12 +20,9 @@ export default {
2020
},
2121
"scheduled-runs": "Scheduled Runs",
2222
"cron-runs": "Cron Runs",
23-
"bulk-run": "Bulk Runs",
23+
"bulk-run": "Bulk Triggering",
24+
events: "Events",
2425
webhooks: "Webhooks",
25-
"external-events": {
26-
title: "Event Triggers",
27-
theme: { collapsed: true },
28-
},
2926
"inter-service-triggering": "Inter-Service Triggering",
3027
"--reliability": {
3128
title: "Reliability",
@@ -43,19 +40,15 @@ export default {
4340
"rate-limits": "Rate Limits",
4441
priority: "Priority",
4542
"--durable-workflows-section": {
46-
title: "Durable Workflows",
43+
title: "Durable Execution",
4744
type: "separator",
4845
},
49-
patterns: {
50-
title: "Patterns",
51-
theme: { collapsed: true },
52-
},
46+
"durable-tasks": "Durable Tasks",
5347
"child-spawning": "Child Spawning",
54-
sleep: "Sleep & Delays",
55-
events: "Wait For Events",
56-
conditions: "Conditions & Branching",
57-
"on-failure": "Error Handling",
58-
"task-eviction": "Resource Management",
48+
"durable-sleep": "Sleeps",
49+
"durable-event-waits": "Event Waits",
50+
"task-eviction": "Task Eviction",
51+
"directed-acyclic-graphs": "DAGs as Durable Workflows",
5952
"--workers-section": {
6053
title: "Workers",
6154
type: "separator",
@@ -122,7 +115,4 @@ export default {
122115
"error-handling": {
123116
display: "hidden",
124117
},
125-
"flow-control": {
126-
display: "hidden",
127-
},
128118
};

‎frontend/docs/pages/v1/bulk-run.mdx‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ import UniversalTabs from "@/components/UniversalTabs";
33
import { Snippet } from "@/components/code";
44
import { snippets } from "@/lib/generated/snippets";
55

6-
# Bulk Run Many Tasks
6+
# Running Tasks in Bulk
77

88
Often you may want to run a task multiple times with different inputs. There is significant overhead (i.e. network roundtrips) to write the task, so if you're running multiple tasks, it's best to use the bulk run methods.
99

0 commit comments

Comments
 (0)