Skip to content

Commit c79ff1a

Browse files
authored
fix(aws): round-trip typed DynamoDB cursors and allow percentile-only CloudWatch statistics (#9967)
* fix(aws): round-trip typed DynamoDB cursors and allow percentile-only CloudWatch statistics Encode large-number and binary keys in lastEvaluatedKey as {N}/{B} and decode them in exclusiveStartKey, so pagination resumes on tables whose keys a JS number or string cannot carry. Item output is unchanged. Let get_metric_statistics take extendedStatistics alone: statistics is optional and the contract requires one of the two. * fix(dynamodb): keep inexact decimal keys exact in pagination cursors Hold every N value a JS number cannot represent exactly as a NumberValue so the cursor encodes it as {N}; item output keeps the same number/string rule. * fix(dynamodb): treat a number as exact only when its string form round-trips A decimal like 1.0000000000000000001 parses to a safe integer, so the safe-integer shortcut dropped its digits. Return a JS number only when String(parsed) equals the DynamoDB value (and it is not an unsafe integer); item output is unchanged.
1 parent 140b92c commit c79ff1a

8 files changed

Lines changed: 208 additions & 40 deletions

File tree

‎apps/docs/content/docs/integrations/cloudwatch.mdx‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -239,7 +239,7 @@ Get statistics for a CloudWatch metric over a time range
239239
| `startTime` | number | Yes | Start time as Unix epoch seconds |
240240
| `endTime` | number | Yes | End time as Unix epoch seconds |
241241
| `period` | number | Yes | Granularity in seconds \(e.g., 60, 300, 3600\) |
242-
| `statistics` | array | Yes | Statistics to retrieve \(Average, Sum, Minimum, Maximum, SampleCount\) |
242+
| `statistics` | array | No | Statistics to retrieve \(Average, Sum, Minimum, Maximum, SampleCount\). Required unless extendedStatistics is set, which takes precedence |
243243
| `dimensions` | string | No | Dimensions as JSON \(e.g., \{"InstanceId": "i-1234"\}\) |
244244
| `extendedStatistics` | array | No | Percentile statistics to retrieve \(e.g., \["p99", "p95"\]\). AWS accepts either Statistics or ExtendedStatistics, so when set these are requested instead of statistics |
245245
| `unit` | string | No | Only return data reported with this unit \(e.g., Percent, Milliseconds\). Omit to return all units |

‎apps/sim/lib/api/contracts/tools/aws/cloudwatch-get-metric-statistics.ts‎

Lines changed: 27 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -7,28 +7,33 @@ import type {
77
import { defineRouteContract } from '@/lib/api/contracts/types'
88
import { validateAwsRegion } from '@/lib/core/security/input-validation'
99

10-
const GetMetricStatisticsSchema = z.object({
11-
region: z
12-
.string()
13-
.min(1, 'AWS region is required')
14-
.refine((v) => validateAwsRegion(v).isValid, {
15-
message: 'Invalid AWS region format (e.g., us-east-1, eu-west-2)',
16-
}),
17-
accessKeyId: z.string().min(1, 'AWS access key ID is required'),
18-
secretAccessKey: z.string().min(1, 'AWS secret access key is required'),
19-
namespace: z.string().min(1, 'Namespace is required'),
20-
metricName: z.string().min(1, 'Metric name is required'),
21-
startTime: z.coerce.number().int(),
22-
endTime: z.coerce.number().int(),
23-
period: z.coerce.number().int().min(1),
24-
statistics: z
25-
.array(z.enum(['Average', 'Sum', 'Minimum', 'Maximum', 'SampleCount']))
26-
.min(1, 'At least one statistic is required')
27-
.max(5, 'At most 5 statistics are allowed per GetMetricStatistics call'),
28-
dimensions: z.string().optional(),
29-
extendedStatistics: z.array(z.string().min(1)).max(10).optional(),
30-
unit: z.string().optional(),
31-
})
10+
const GetMetricStatisticsSchema = z
11+
.object({
12+
region: z
13+
.string()
14+
.min(1, 'AWS region is required')
15+
.refine((v) => validateAwsRegion(v).isValid, {
16+
message: 'Invalid AWS region format (e.g., us-east-1, eu-west-2)',
17+
}),
18+
accessKeyId: z.string().min(1, 'AWS access key ID is required'),
19+
secretAccessKey: z.string().min(1, 'AWS secret access key is required'),
20+
namespace: z.string().min(1, 'Namespace is required'),
21+
metricName: z.string().min(1, 'Metric name is required'),
22+
startTime: z.coerce.number().int(),
23+
endTime: z.coerce.number().int(),
24+
period: z.coerce.number().int().min(1),
25+
statistics: z
26+
.array(z.enum(['Average', 'Sum', 'Minimum', 'Maximum', 'SampleCount']))
27+
.max(5, 'At most 5 statistics are allowed per GetMetricStatistics call')
28+
.optional(),
29+
dimensions: z.string().optional(),
30+
extendedStatistics: z.array(z.string().min(1)).max(10).optional(),
31+
unit: z.string().optional(),
32+
})
33+
.refine((body) => Boolean(body.statistics?.length || body.extendedStatistics?.length), {
34+
message: 'Provide statistics or extendedStatistics',
35+
path: ['statistics'],
36+
})
3237

3338
const GetMetricStatisticsResponseSchema = z.object({
3439
success: z.literal(true),

‎apps/sim/lib/internal/dynamodb/client.test.ts‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,23 @@ function clientReturning(output: Record<string, unknown>) {
1414
return client
1515
}
1616

17+
/** A client that records the marshalled request input DynamoDB would receive. */
18+
function clientCapturing(onInput: (input: Record<string, unknown>) => void) {
19+
const client = createDynamoDBClient({
20+
region: 'us-east-1',
21+
accessKeyId: 'AKIAEXAMPLE',
22+
secretAccessKey: 'secret',
23+
})
24+
client.middlewareStack.add(
25+
() => async (args) => {
26+
onInput(args.input as Record<string, unknown>)
27+
return { output: { $metadata: {}, Items: [], Count: 0 }, response: {} }
28+
},
29+
{ step: 'build' }
30+
)
31+
return client
32+
}
33+
1734
describe('DynamoDB item unmarshalling', () => {
1835
it('returns JSON-serializable items for sets, binary and integers beyond 2^53', async () => {
1936
const client = clientReturning({
@@ -27,6 +44,7 @@ describe('DynamoDB item unmarshalling', () => {
2744
avatar: { B: new Uint8Array([104, 105]) },
2845
snowflake: { N: '1234567890123456789' },
2946
age: { N: '42' },
47+
ratio: { N: '0.12345678901234567890123' },
3048
},
3149
],
3250
})
@@ -44,11 +62,59 @@ describe('DynamoDB item unmarshalling', () => {
4462
avatar: 'aGk=',
4563
snowflake: '1234567890123456789',
4664
age: 42,
65+
ratio: 0.12345678901234568,
4766
},
4867
],
4968
count: 1,
5069
scannedCount: 3,
5170
})
5271
client.destroy()
5372
})
73+
74+
it('round-trips large-number and binary keys through the pagination cursor', async () => {
75+
const pageClient = clientReturning({
76+
Count: 0,
77+
Items: [],
78+
LastEvaluatedKey: {
79+
pk: { S: 'USER#1' },
80+
sk: { N: '1234567890123456789' },
81+
bin: { B: new Uint8Array([104, 105]) },
82+
age: { N: '42' },
83+
ratio: { N: '0.12345678901234567890123' },
84+
nearOne: { N: '1.0000000000000000001' },
85+
},
86+
})
87+
const firstPage = await queryItems(pageClient, 'Users', 'pk = :pk', {
88+
expressionAttributeValues: { ':pk': 'USER#1' },
89+
})
90+
const cursor = await Response.json(firstPage.lastEvaluatedKey).json()
91+
expect(cursor).toEqual({
92+
pk: 'USER#1',
93+
sk: { N: '1234567890123456789' },
94+
bin: { B: 'aGk=' },
95+
age: 42,
96+
ratio: { N: '0.12345678901234567890123' },
97+
nearOne: { N: '1.0000000000000000001' },
98+
})
99+
100+
let sentStartKey: unknown
101+
const client = clientCapturing((input) => {
102+
sentStartKey = input.ExclusiveStartKey
103+
})
104+
await queryItems(client, 'Users', 'pk = :pk', {
105+
expressionAttributeValues: { ':pk': 'USER#1' },
106+
exclusiveStartKey: cursor,
107+
})
108+
109+
expect(sentStartKey).toEqual({
110+
pk: { S: 'USER#1' },
111+
sk: { N: '1234567890123456789' },
112+
bin: { B: new Uint8Array([104, 105]) },
113+
age: { N: '42' },
114+
ratio: { N: '0.12345678901234567890123' },
115+
nearOne: { N: '1.0000000000000000001' },
116+
})
117+
pageClient.destroy()
118+
client.destroy()
119+
})
54120
})

‎apps/sim/lib/internal/dynamodb/client.ts‎

Lines changed: 66 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -3,34 +3,52 @@ import {
33
DeleteCommand,
44
DynamoDBDocumentClient,
55
GetCommand,
6+
NumberValue,
67
PutCommand,
78
QueryCommand,
89
ScanCommand,
910
UpdateCommand,
1011
} from '@aws-sdk/lib-dynamodb'
12+
import { isRecordLike } from '@sim/utils/object'
1113
import { awsClientConfig } from '@/lib/internal/aws/credentials'
1214
import type { DynamoDBConnectionConfig, DynamoDBTableSchema } from '@/tools/dynamodb/types'
1315

1416
type DynamoDBReturnValues = 'NONE' | 'ALL_OLD' | 'UPDATED_OLD' | 'ALL_NEW' | 'UPDATED_NEW'
1517

1618
/**
17-
* Keeps integers beyond `Number.MAX_SAFE_INTEGER` (and other values a JS number cannot hold)
18-
* as their exact DynamoDB string, since the SDK default turns them into a `BigInt` that the
19-
* JSON response cannot serialize.
19+
* Keeps any DynamoDB number a JS number cannot hold exactly (integers beyond
20+
* `Number.MAX_SAFE_INTEGER`, high-precision decimals, non-finite values) as a `NumberValue`
21+
* with the exact string, so pagination cursors can round-trip it. The SDK default would turn
22+
* large integers into a `BigInt` that the JSON response cannot serialize.
2023
*/
21-
function unwrapNumber(value: string): number | string {
24+
function unwrapNumber(value: string): number | NumberValue {
2225
const parsed = Number(value)
23-
if (!Number.isFinite(parsed)) return value
24-
if (Number.isInteger(parsed) && !Number.isSafeInteger(parsed)) return value
26+
const exact = Number.isFinite(parsed) && String(parsed) === value
27+
if (exact && (!Number.isInteger(parsed) || Number.isSafeInteger(parsed))) return parsed
28+
return NumberValue.from(value)
29+
}
30+
31+
/**
32+
* Item output for a number kept as a `NumberValue`: the exact string for integers beyond
33+
* `Number.MAX_SAFE_INTEGER` and non-finite values, otherwise the nearest JS number.
34+
*/
35+
function numberValueToJson(value: NumberValue): number | string {
36+
const exact = value.toString()
37+
const parsed = Number(exact)
38+
if (!Number.isFinite(parsed) || (Number.isInteger(parsed) && !Number.isSafeInteger(parsed))) {
39+
return exact
40+
}
2541
return parsed
2642
}
2743

2844
/**
2945
* Converts unmarshalled DynamoDB values into JSON-serializable shapes: string, number and
30-
* binary sets become arrays, and binary values become base64 strings. Without this a set
31-
* serializes as `{}` and a binary value as an object of byte indexes.
46+
* binary sets become arrays, binary values become base64 strings, and numbers kept as a
47+
* `NumberValue` become a JS number or, when one cannot hold them, their exact string. Without this a set serializes as `{}` and a binary
48+
* value as an object of byte indexes.
3249
*/
3350
function toJsonSafe(value: unknown): unknown {
51+
if (value instanceof NumberValue) return numberValueToJson(value)
3452
if (value instanceof Set) return Array.from(value, toJsonSafe)
3553
if (value instanceof Uint8Array) return Buffer.from(value).toString('base64')
3654
if (Array.isArray(value)) return value.map(toJsonSafe)
@@ -46,6 +64,42 @@ function toJsonSafeRecord(
4664
return value ? (toJsonSafe(value) as Record<string, unknown>) : undefined
4765
}
4866

67+
/**
68+
* Encodes a `LastEvaluatedKey` so it can be passed back as `exclusiveStartKey`. Key
69+
* attributes are always scalar (S, N or B), so strings and exact numbers stay as-is while a
70+
* number a JS number cannot hold exactly becomes `{ N }` and a binary key becomes `{ B }` (base64):
71+
* plain strings would otherwise be sent back as S and fail the key type check.
72+
*/
73+
function toCursor(key: Record<string, unknown> | undefined): Record<string, unknown> | undefined {
74+
if (!key) return undefined
75+
return Object.fromEntries(
76+
Object.entries(key).map(([name, value]) => {
77+
if (value instanceof NumberValue) return [name, { N: value.toString() }]
78+
if (value instanceof Uint8Array) return [name, { B: toJsonSafe(value) }]
79+
return [name, value]
80+
})
81+
)
82+
}
83+
84+
const DYNAMODB_NUMBER = /^-?\d+(\.\d+)?([eE][+-]?\d+)?$/
85+
86+
/** Restores the `{ N }` and `{ B }` cursor values written by {@link toCursor}. */
87+
function fromCursor(key: Record<string, unknown>): Record<string, unknown> {
88+
return Object.fromEntries(
89+
Object.entries(key).map(([name, value]) => {
90+
if (isRecordLike(value) && Object.keys(value).length === 1) {
91+
if (typeof value.N === 'string' && DYNAMODB_NUMBER.test(value.N)) {
92+
return [name, NumberValue.from(value.N)]
93+
}
94+
if (typeof value.B === 'string') {
95+
return [name, new Uint8Array(Buffer.from(value.B, 'base64'))]
96+
}
97+
}
98+
return [name, value]
99+
})
100+
)
101+
}
102+
49103
export function createDynamoDBClient(config: DynamoDBConnectionConfig): DynamoDBDocumentClient {
50104
const client = new DynamoDBClient({
51105
...awsClientConfig(config.region, config.accessKeyId, config.secretAccessKey),
@@ -152,7 +206,7 @@ export async function queryItems(
152206
}),
153207
...(options?.indexName && { IndexName: options.indexName }),
154208
...(options?.limit && { Limit: options.limit }),
155-
...(options?.exclusiveStartKey && { ExclusiveStartKey: options.exclusiveStartKey }),
209+
...(options?.exclusiveStartKey && { ExclusiveStartKey: fromCursor(options.exclusiveStartKey) }),
156210
...(options?.scanIndexForward !== undefined && { ScanIndexForward: options.scanIndexForward }),
157211
...(options?.projectionExpression && { ProjectionExpression: options.projectionExpression }),
158212
...(options?.consistentRead !== undefined && { ConsistentRead: options.consistentRead }),
@@ -163,7 +217,7 @@ export async function queryItems(
163217
items: (response.Items ?? []).map((item) => toJsonSafeRecord(item) ?? {}),
164218
count: response.Count || 0,
165219
scannedCount: response.ScannedCount ?? null,
166-
lastEvaluatedKey: toJsonSafeRecord(response.LastEvaluatedKey),
220+
lastEvaluatedKey: toCursor(response.LastEvaluatedKey),
167221
}
168222
}
169223

@@ -200,7 +254,7 @@ export async function scanItems(
200254
ExpressionAttributeValues: options.expressionAttributeValues,
201255
}),
202256
...(options?.limit && { Limit: options.limit }),
203-
...(options?.exclusiveStartKey && { ExclusiveStartKey: options.exclusiveStartKey }),
257+
...(options?.exclusiveStartKey && { ExclusiveStartKey: fromCursor(options.exclusiveStartKey) }),
204258
...(options?.indexName && { IndexName: options.indexName }),
205259
...(options?.consistentRead !== undefined && { ConsistentRead: options.consistentRead }),
206260
...(options?.segment !== undefined && { Segment: options.segment }),
@@ -212,7 +266,7 @@ export async function scanItems(
212266
items: (response.Items ?? []).map((item) => toJsonSafeRecord(item) ?? {}),
213267
count: response.Count || 0,
214268
scannedCount: response.ScannedCount ?? null,
215-
lastEvaluatedKey: toJsonSafeRecord(response.LastEvaluatedKey),
269+
lastEvaluatedKey: toCursor(response.LastEvaluatedKey),
216270
}
217271
}
218272

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
import { describe, expect, it } from 'vitest'
2+
import { awsCloudwatchGetMetricStatisticsContract } from '@/lib/api/contracts/tools/aws/cloudwatch-get-metric-statistics'
3+
import { getMetricStatisticsTool } from '@/tools/cloudwatch/get_metric_statistics'
4+
import { validateRequiredParametersAfterMerge } from '@/tools/utils'
5+
6+
const PARAMS = {
7+
awsRegion: 'us-east-1',
8+
awsAccessKeyId: 'AKIAEXAMPLE',
9+
awsSecretAccessKey: 'secret',
10+
namespace: 'AWS/EC2',
11+
metricName: 'CPUUtilization',
12+
startTime: 1700000000,
13+
endTime: 1700003600,
14+
period: 300,
15+
}
16+
17+
function parseRequest(params: Record<string, unknown>) {
18+
const body = getMetricStatisticsTool.operation.input(params as never)
19+
return awsCloudwatchGetMetricStatisticsContract.body?.safeParse(body)
20+
}
21+
22+
describe('cloudwatch_get_metric_statistics', () => {
23+
it('accepts percentile statistics without a standard statistic', () => {
24+
const params = { ...PARAMS, extendedStatistics: ['p99', 'p95'] }
25+
expect(() =>
26+
validateRequiredParametersAfterMerge(
27+
'cloudwatch_get_metric_statistics',
28+
getMetricStatisticsTool,
29+
params
30+
)
31+
).not.toThrow()
32+
expect(parseRequest(params)?.success).toBe(true)
33+
})
34+
35+
it('rejects a request with neither standard nor percentile statistics', () => {
36+
const result = parseRequest(PARAMS)
37+
expect(result?.success).toBe(false)
38+
expect(result?.error?.issues.map((issue) => issue.message)).toEqual([
39+
'Provide statistics or extendedStatistics',
40+
])
41+
})
42+
})

‎apps/sim/tools/cloudwatch/get_metric_statistics.ts‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -66,9 +66,10 @@ export const getMetricStatisticsTool: InternalToolConfig<
6666
},
6767
statistics: {
6868
type: 'array',
69-
required: true,
69+
required: false,
7070
visibility: 'user-or-llm',
71-
description: 'Statistics to retrieve (Average, Sum, Minimum, Maximum, SampleCount)',
71+
description:
72+
'Statistics to retrieve (Average, Sum, Minimum, Maximum, SampleCount). Required unless extendedStatistics is set, which takes precedence',
7273
},
7374
dimensions: {
7475
type: 'string',
@@ -102,7 +103,7 @@ export const getMetricStatisticsTool: InternalToolConfig<
102103
startTime: params.startTime,
103104
endTime: params.endTime,
104105
period: params.period,
105-
statistics: params.statistics,
106+
...(params.statistics && params.statistics.length > 0 && { statistics: params.statistics }),
106107
...(params.dimensions && { dimensions: params.dimensions }),
107108
...(params.extendedStatistics &&
108109
params.extendedStatistics.length > 0 && {

‎apps/sim/tools/cloudwatch/types.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -106,7 +106,7 @@ export interface CloudWatchGetMetricStatisticsParams extends CloudWatchConnectio
106106
startTime: number
107107
endTime: number
108108
period: number
109-
statistics: string[]
109+
statistics?: string[]
110110
dimensions?: string
111111
extendedStatistics?: string[]
112112
unit?: string

‎apps/sim/tools/generated/tool-metadata.ts‎

Lines changed: 1 addition & 1 deletion
Large diffs are not rendered by default.

0 commit comments

Comments
 (0)