From 93be5b17ab5217df68fff38ac0114f7848c0b25f Mon Sep 17 00:00:00 2001 From: jamals86 Date: Mon, 17 Aug 2026 13:27:50 +0300 Subject: [PATCH 1/4] Add kalam_sync Dart SDK and switch RocksDB to HyperClockCache. Move kalam_link under link/sdks/dart/link and add kalam_sync plus CLI Dart templates so Flutter apps can keep an account-scoped local replica. Upgrade RocksDB to 0.25 with HyperClockCache, replace deprecated Atomic::fetch_update, and stop using Cranelift as the default codegen backend. Co-authored-by: Cursor --- .cargo/config.toml | 15 +- .github/workflows/ci.yml | 1 + .github/workflows/cli-cluster-e2e.yml | 1 + .github/workflows/codeql.yml | 1 + .github/workflows/dart-sdk.yml | 38 +- .github/workflows/python-sdk.yml | 1 + .github/workflows/react-e2e.yml | 1 + .github/workflows/release.yml | 1 + .github/workflows/rust-coverage.yml | 1 + .github/workflows/rust-sdk.yml | 1 + .github/workflows/typescript-sdk.yml | 1 + .github/workflows/versions.yml | 4 +- .gitignore | 4 +- .ignore | 6 +- AGENTS.md | 7 +- CONTRIBUTING.md | 20 +- Cargo.lock | 27 +- Cargo.toml | 25 +- .../kalamdb-api/src/limiter/rate_limiter.rs | 2 +- .../tests/sql_cached_point_get.rs | 10 + .../src/manager/connections_manager.rs | 2 +- .../src/storage_metrics.rs | 2 +- .../src/backends/rocksdb/backend.rs | 6 +- .../src/backends/rocksdb/init.rs | 13 +- backend/src/connection_guard.rs | 2 +- .../results/kalamdb-20260817-123845.txt | 30 + .../results/pocketbase-20260817-123956.txt | 17 + .../results/trailbase-20260817-123921.txt | 17 + cli/DEV.md | 20 +- .../tests/cli/test_project_workflow_init.rs | 60 + .../tests/cli/test_project_workflow_schema.rs | 6 +- .../smoke_test_cached_point_get_projection.rs | 7 + cli/run-tests.sh | 2 +- cli/src/args/workflow.rs | 4 +- cli/src/workflow/project/dart/init.rs | 180 + cli/src/workflow/project/dart/mod.rs | 12 + cli/src/workflow/project/dart/templates.rs | 24 + cli/src/workflow/project/guidance.rs | 2 +- cli/src/workflow/project/init/mod.rs | 46 +- cli/src/workflow/project/init/prompts.rs | 7 +- cli/src/workflow/project/init/write.rs | 40 +- cli/src/workflow/project/mod.rs | 1 + cli/src/workflow/project/templates.rs | 12 +- cli/src/workflow/project/ts/mod.rs | 2 - cli/src/workflow/schema/dart.rs | 477 ++ cli/src/workflow/schema/gen.rs | 44 +- cli/src/workflow/schema/load.rs | 144 +- cli/src/workflow/schema/mod.rs | 3 +- cli/src/workflow/schema/model.rs | 15 +- .../analysis_options.yaml.template | 1 + cli/templates/dart/simple-live/info.toml | 3 + .../dart/simple-live/lib/main.dart.template | 86 + .../dart/simple-live/pubspec.yaml.template | 21 + .../dart/simple-live/schema.sql.template | 6 + docker/build/Dockerfile | 2 + .../2026-08-16-kalam-sync-dart-package.md | 66 + docs/getting-started/cli.md | 3 +- .../2026-08-16-kalam-sync-implementation.md | 191 + .../01-bidirectional-todos.md | 135 + .../02-mixed-replication-modes.md | 112 + .../03-feature-actions-and-events.md | 251 + ...generated-bindings-and-future-functions.md | 295 ++ .../05-offline-messaging.md | 221 + .../2026-08-16-kalam-sync-sdk-api/README.md | 299 ++ .../tests/sync.integration.test.ts | 18 +- link/README.md | 11 +- link/kalam-link-dart/flutter_rust_bridge.yaml | 2 +- link/kalam-link-dart/src/api.rs | 10 + link/kalam-link-dart/src/frb_generated.rs | 374 +- link/kalam-link-dart/src/models.rs | 7 + link/kalam-link-dart/src/tests.rs | 20 + link/link-common/src/client/runtime.rs | 1 + link/link-common/src/lib.rs | 3 +- link/link-common/src/models/mod.rs | 4 +- link/link-common/src/subscription/manager.rs | 79 +- link/link-common/src/subscription/mod.rs | 5 +- .../src/subscription/models/mod.rs | 2 + .../models/subscription_ack_mode.rs | 9 + .../models/subscription_config.rs | 13 +- link/sdks/dart/generator/.gitignore | 2 + link/sdks/dart/generator/CHANGELOG.md | 4 + link/sdks/dart/{ => generator}/LICENSE | 0 link/sdks/dart/generator/README.md | 33 + link/sdks/dart/generator/build.yaml | 14 + link/sdks/dart/generator/example/.gitignore | 4 + link/sdks/dart/generator/example/README.md | 57 + .../generator/example/analysis_options.yaml | 10 + .../generator/example/lib/chat_actions.dart | 103 + .../generator/example/lib/chat_actions.g.dart | 96 + .../generator/example/lib/chat_http_api.dart | 70 + .../generator/example/lib/chat_models.dart | 176 + .../generator/example/lib/chat_tables.dart | 25 + link/sdks/dart/generator/example/pubspec.lock | 653 +++ link/sdks/dart/generator/example/pubspec.yaml | 20 + .../generator/example/pubspec_overrides.yaml | 7 + .../generated_action_runtime_e2e_test.dart | 97 + .../generator/lib/kalam_sync_generator.dart | 12 + .../lib/src/kalam_action_generator.dart | 122 + .../src/kalam_action_payload_generator.dart | 93 + link/sdks/dart/generator/pubspec.lock | 797 +++ link/sdks/dart/generator/pubspec.yaml | 21 + .../dart/generator/pubspec_overrides.yaml | 7 + .../test/kalam_action_generator_test.dart | 197 + link/sdks/dart/{ => link}/CHANGELOG.md | 0 link/sdks/dart/{ => link}/DEV.md | 12 +- link/sdks/dart/link/LICENSE | 183 + link/sdks/dart/{ => link}/NOTICE | 0 link/sdks/dart/{ => link}/README.md | 4 +- .../dart/{ => link}/analysis_options.yaml | 0 link/sdks/dart/{ => link}/android/.gitignore | 0 .../dart/{ => link}/android/CMakeLists.txt | 0 .../sdks/dart/{ => link}/android/build.gradle | 0 .../dart/{ => link}/android/settings.gradle | 0 .../android/src/main/AndroidManifest.xml | 0 .../android/src/main/jniLibs/.gitignore | 0 .../jniLibs/arm64-v8a/libkalam_link_dart.so | Bin .../main/jniLibs/x86_64/libkalam_link_dart.so | Bin link/sdks/dart/{ => link}/build.sh | 4 +- .../{ => link}/example/chat-app/main.dart | 0 .../example/simple-events/main.dart | 0 .../sdks/dart/{ => link}/example/support.dart | 0 link/sdks/dart/{ => link}/ios/.gitignore | 0 .../ios/Classes/KalamLinkPlugin.swift | 0 .../ios/Frameworks/libkalam_link_dart.a | Bin .../dart/{ => link}/ios/kalam_link.podspec | 0 link/sdks/dart/{ => link}/lib/kalam_link.dart | 1 + link/sdks/dart/{ => link}/lib/src/auth.dart | 0 .../dart/{ => link}/lib/src/cell_value.dart | 0 .../dart/{ => link}/lib/src/file_ref.dart | 0 .../lib/src/generated/.frb_codegen.stamp | 0 .../{ => link}/lib/src/generated/api.dart | 7 + .../lib/src/generated/frb_generated.dart | 74 +- .../lib/src/generated/frb_generated.io.dart | 0 .../lib/src/generated/frb_generated.web.dart | 0 .../{ => link}/lib/src/generated/models.dart | 10 +- .../lib/src/generated/models.freezed.dart | 0 .../dart/{ => link}/lib/src/kalam_client.dart | 64 +- .../link/lib/src/live_event_delivery.dart | 38 + link/sdks/dart/{ => link}/lib/src/logger.dart | 0 link/sdks/dart/{ => link}/lib/src/models.dart | 0 link/sdks/dart/{ => link}/lib/src/seq_id.dart | 0 .../lib/src/subscription_stream.dart | 16 +- .../sdks/dart/{ => link}/linux/CMakeLists.txt | 0 .../sdks/dart/{ => link}/linux/lib/.gitignore | 0 link/sdks/dart/{ => link}/macos/.gitignore | 0 .../macos/Classes/KalamLinkPlugin.swift | 0 .../dart/{ => link}/macos/Libs/.gitignore | 0 .../macos/Libs/libkalam_link_dart.dylib | Bin .../dart/{ => link}/macos/kalam_link.podspec | 0 link/sdks/dart/{ => link}/publish.sh | 0 link/sdks/dart/{ => link}/pubspec.lock | 0 link/sdks/dart/{ => link}/pubspec.yaml | 0 link/sdks/dart/{ => link}/test.sh | 2 +- .../test/auth_provider_retry_test.dart | 0 .../{ => link}/test/e2e/auth/auth_test.dart | 0 .../{ => link}/test/e2e/ddl/ddl_test.dart | 0 .../test/e2e/examples/examples_test.dart | 0 .../dart/{ => link}/test/e2e/helpers.dart | 2 +- .../test/e2e/keepalive/keepalive_test.dart | 0 .../keepalive/subscription_cleanup_test.dart | 0 .../test/e2e/lifecycle/lifecycle_test.dart | 0 .../{ => link}/test/e2e/query/query_test.dart | 0 .../e2e/reconnect/app_lifecycle_test.dart | 0 .../test/e2e/reconnect/explicit_ack_test.dart | 106 + .../test/e2e/reconnect/reconnect_test.dart | 0 .../test/e2e/reconnect/resume_test.dart | 0 .../subscription_options_test.dart | 0 .../e2e/subscription/subscription_test.dart | 0 .../dart/{ => link}/test/file_ref_test.dart | 0 .../{ => link}/test/live_server_test.dart | 2 +- .../test/main_thread_blocking_test.dart | 0 .../dart/{ => link}/test/models_test.dart | 0 .../test/plugin_packaging_test.dart | 0 .../link/test/schema_generation_test.dart | 55 + .../test/subscription_stream_test.dart | 37 +- link/sdks/dart/{ => link}/web/.gitignore | 0 link/sdks/dart/{ => link}/web/pkg/.gitignore | 0 link/sdks/dart/{ => link}/web/pkg/README.md | 0 .../{ => link}/web/pkg/kalam_link_dart.d.ts | 0 .../{ => link}/web/pkg/kalam_link_dart.js | 0 .../web/pkg/kalam_link_dart_bg.wasm | Bin .../web/pkg/kalam_link_dart_bg.wasm.d.ts | 0 .../sdks/dart/{ => link}/web/pkg/package.json | 0 .../dart/{ => link}/windows/CMakeLists.txt | 0 .../dart/{ => link}/windows/lib/.gitignore | 0 link/sdks/dart/sync/.gitignore | 4 + link/sdks/dart/sync/CHANGELOG.md | 5 + link/sdks/dart/sync/LICENSE | 183 + link/sdks/dart/sync/README.md | 233 + link/sdks/dart/sync/analysis_options.yaml | 13 + link/sdks/dart/sync/example/.gitignore | 4 + link/sdks/dart/sync/example/README.md | 160 + .../dart/sync/example/analysis_options.yaml | 11 + .../sdks/dart/sync/example/backend/.gitignore | 2 + .../dart/sync/example/backend/bin/server.dart | 32 + .../example/backend/lib/chat_backend.dart | 345 ++ .../dart/sync/example/backend/pubspec.lock | 5 + .../dart/sync/example/backend/pubspec.yaml | 7 + link/sdks/dart/sync/example/kalam.toml | 16 + link/sdks/dart/sync/example/kalam/schema.sql | 20 + link/sdks/dart/sync/example/lib/main.dart | 350 ++ link/sdks/dart/sync/example/pubspec.lock | 538 ++ link/sdks/dart/sync/example/pubspec.yaml | 24 + .../dart/sync/example/pubspec_overrides.yaml | 7 + link/sdks/dart/sync/lib/drift.dart | 7 + link/sdks/dart/sync/lib/kalam_sync.dart | 42 + .../lib/src/actions/kalam_action_codec.dart | 7 + .../lib/src/actions/kalam_action_context.dart | 40 + .../src/actions/kalam_action_definition.dart | 34 + .../src/actions/kalam_action_registry.dart | 40 + .../lib/src/actions/kalam_action_runner.dart | 117 + .../lib/src/actions/kalam_dml_action.dart | 110 + .../lib/src/annotations/kalam_action.dart | 7 + .../src/annotations/kalam_action_module.dart | 6 + .../src/annotations/kalam_action_payload.dart | 4 + .../lib/src/database/kalam_sync_database.dart | 40 + .../src/database/kalam_sync_database.g.dart | 4316 +++++++++++++++++ .../database/tables/kalam_action_steps.dart | 14 + .../src/database/tables/kalam_actions.dart | 24 + .../database/tables/kalam_cached_rows.dart | 13 + .../database/tables/kalam_checkpoints.dart | 12 + .../src/database/tables/kalam_row_states.dart | 21 + .../src/flutter/kalam_database_factory.dart | 22 + .../sync/lib/src/flutter/kalam_scope.dart | 96 + link/sdks/dart/sync/lib/src/kalam.dart | 163 + .../src/models/kalam_account_identity.dart | 25 + .../lib/src/models/kalam_action_draft.dart | 20 + .../lib/src/models/kalam_action_record.dart | 49 + .../lib/src/models/kalam_action_status.dart | 9 + .../sync/lib/src/models/kalam_cached_row.dart | 7 + .../sync/lib/src/models/kalam_change.dart | 18 + .../sync/lib/src/models/kalam_checkpoint.dart | 16 + .../lib/src/models/kalam_dml_payload.dart | 18 + .../src/models/kalam_optimistic_mutation.dart | 9 + .../lib/src/models/kalam_optimistic_row.dart | 18 + .../lib/src/models/kalam_retry_policy.dart | 36 + .../lib/src/models/kalam_row_overlay.dart | 16 + .../lib/src/models/kalam_row_sync_state.dart | 67 + .../sync/lib/src/models/kalam_sync_mode.dart | 8 + .../sync/lib/src/models/kalam_sync_state.dart | 57 + .../sync/lib/src/models/kalam_synced_row.dart | 21 + .../sync/lib/src/store/kalam_sync_store.dart | 566 +++ .../lib/src/sync/kalam_event_consumer.dart | 20 + .../lib/src/sync/kalam_sync_coordinator.dart | 302 ++ .../lib/src/sync/kalam_sync_subscription.dart | 17 + .../lib/src/tables/kalam_replica_overlay.dart | 47 + .../lib/src/tables/kalam_table_binding.dart | 275 ++ .../sync/lib/src/tables/kalam_table_spec.dart | 22 + .../src/transport/kalam_link_transport.dart | 175 + .../lib/src/transport/kalam_remote_batch.dart | 22 + .../src/transport/kalam_remote_change.dart | 20 + .../src/transport/kalam_sync_transport.dart | 25 + link/sdks/dart/sync/pubspec.lock | 778 +++ link/sdks/dart/sync/pubspec.yaml | 27 + link/sdks/dart/sync/pubspec_overrides.yaml | 7 + .../actions/kalam_action_runner_test.dart | 293 ++ .../database/kalam_sync_database_test.dart | 128 + .../sync/test/e2e/chat_sync_e2e_test.dart | 347 ++ .../sync/test/e2e/kalam_sync_e2e_test.dart | 254 + .../test/e2e/kalam_sync_scale_e2e_test.dart | 323 ++ .../test/e2e/support/chat_e2e_harness.dart | 322 ++ .../sync/test/flutter/kalam_scope_test.dart | 155 + .../integration/offline_restart_test.dart | 73 + .../sync/test/kalam_from_components_test.dart | 58 + .../sync/test/models/sync_models_test.dart | 82 + .../test/store/kalam_sync_store_test.dart | 177 + .../sync/kalam_sync_coordinator_test.dart | 484 ++ .../test/tables/kalam_table_binding_test.dart | 299 ++ .../transport/kalam_link_transport_test.dart | 75 + .../dart/test/schema_generation_test.dart | 15 - link/sdks/sync-versions.sh | 46 +- link/sdks/typescript/README.md | 2 +- pg/docker/Dockerfile | 2 + scripts/test-all.sh | 19 +- scripts/versions.py | 18 +- tools/Dockerfile.builder | 2 + versions.json | 10 + 277 files changed, 19446 insertions(+), 380 deletions(-) create mode 100644 benchv2/comparison/results/kalamdb-20260817-123845.txt create mode 100644 benchv2/comparison/results/pocketbase-20260817-123956.txt create mode 100644 benchv2/comparison/results/trailbase-20260817-123921.txt create mode 100644 cli/src/workflow/project/dart/init.rs create mode 100644 cli/src/workflow/project/dart/mod.rs create mode 100644 cli/src/workflow/project/dart/templates.rs create mode 100644 cli/src/workflow/schema/dart.rs create mode 100644 cli/templates/dart/simple-live/analysis_options.yaml.template create mode 100644 cli/templates/dart/simple-live/info.toml create mode 100644 cli/templates/dart/simple-live/lib/main.dart.template create mode 100644 cli/templates/dart/simple-live/pubspec.yaml.template create mode 100644 cli/templates/dart/simple-live/schema.sql.template create mode 100644 docs/architecture/decisions/2026-08-16-kalam-sync-dart-package.md create mode 100644 docs/plans/2026-08-16-kalam-sync-implementation.md create mode 100644 docs/plans/2026-08-16-kalam-sync-sdk-api/01-bidirectional-todos.md create mode 100644 docs/plans/2026-08-16-kalam-sync-sdk-api/02-mixed-replication-modes.md create mode 100644 docs/plans/2026-08-16-kalam-sync-sdk-api/03-feature-actions-and-events.md create mode 100644 docs/plans/2026-08-16-kalam-sync-sdk-api/04-generated-bindings-and-future-functions.md create mode 100644 docs/plans/2026-08-16-kalam-sync-sdk-api/05-offline-messaging.md create mode 100644 docs/plans/2026-08-16-kalam-sync-sdk-api/README.md create mode 100644 link/link-common/src/subscription/models/subscription_ack_mode.rs create mode 100644 link/sdks/dart/generator/.gitignore create mode 100644 link/sdks/dart/generator/CHANGELOG.md rename link/sdks/dart/{ => generator}/LICENSE (100%) create mode 100644 link/sdks/dart/generator/README.md create mode 100644 link/sdks/dart/generator/build.yaml create mode 100644 link/sdks/dart/generator/example/.gitignore create mode 100644 link/sdks/dart/generator/example/README.md create mode 100644 link/sdks/dart/generator/example/analysis_options.yaml create mode 100644 link/sdks/dart/generator/example/lib/chat_actions.dart create mode 100644 link/sdks/dart/generator/example/lib/chat_actions.g.dart create mode 100644 link/sdks/dart/generator/example/lib/chat_http_api.dart create mode 100644 link/sdks/dart/generator/example/lib/chat_models.dart create mode 100644 link/sdks/dart/generator/example/lib/chat_tables.dart create mode 100644 link/sdks/dart/generator/example/pubspec.lock create mode 100644 link/sdks/dart/generator/example/pubspec.yaml create mode 100644 link/sdks/dart/generator/example/pubspec_overrides.yaml create mode 100644 link/sdks/dart/generator/flutter_test/generated_action_runtime_e2e_test.dart create mode 100644 link/sdks/dart/generator/lib/kalam_sync_generator.dart create mode 100644 link/sdks/dart/generator/lib/src/kalam_action_generator.dart create mode 100644 link/sdks/dart/generator/lib/src/kalam_action_payload_generator.dart create mode 100644 link/sdks/dart/generator/pubspec.lock create mode 100644 link/sdks/dart/generator/pubspec.yaml create mode 100644 link/sdks/dart/generator/pubspec_overrides.yaml create mode 100644 link/sdks/dart/generator/test/kalam_action_generator_test.dart rename link/sdks/dart/{ => link}/CHANGELOG.md (100%) rename link/sdks/dart/{ => link}/DEV.md (94%) create mode 100644 link/sdks/dart/link/LICENSE rename link/sdks/dart/{ => link}/NOTICE (100%) rename link/sdks/dart/{ => link}/README.md (97%) rename link/sdks/dart/{ => link}/analysis_options.yaml (100%) rename link/sdks/dart/{ => link}/android/.gitignore (100%) rename link/sdks/dart/{ => link}/android/CMakeLists.txt (100%) rename link/sdks/dart/{ => link}/android/build.gradle (100%) rename link/sdks/dart/{ => link}/android/settings.gradle (100%) rename link/sdks/dart/{ => link}/android/src/main/AndroidManifest.xml (100%) rename link/sdks/dart/{ => link}/android/src/main/jniLibs/.gitignore (100%) rename link/sdks/dart/{ => link}/android/src/main/jniLibs/arm64-v8a/libkalam_link_dart.so (100%) rename link/sdks/dart/{ => link}/android/src/main/jniLibs/x86_64/libkalam_link_dart.so (100%) rename link/sdks/dart/{ => link}/build.sh (99%) rename link/sdks/dart/{ => link}/example/chat-app/main.dart (100%) rename link/sdks/dart/{ => link}/example/simple-events/main.dart (100%) rename link/sdks/dart/{ => link}/example/support.dart (100%) rename link/sdks/dart/{ => link}/ios/.gitignore (100%) rename link/sdks/dart/{ => link}/ios/Classes/KalamLinkPlugin.swift (100%) rename link/sdks/dart/{ => link}/ios/Frameworks/libkalam_link_dart.a (100%) rename link/sdks/dart/{ => link}/ios/kalam_link.podspec (100%) rename link/sdks/dart/{ => link}/lib/kalam_link.dart (96%) rename link/sdks/dart/{ => link}/lib/src/auth.dart (100%) rename link/sdks/dart/{ => link}/lib/src/cell_value.dart (100%) rename link/sdks/dart/{ => link}/lib/src/file_ref.dart (100%) rename link/sdks/dart/{ => link}/lib/src/generated/.frb_codegen.stamp (100%) rename link/sdks/dart/{ => link}/lib/src/generated/api.dart (97%) rename link/sdks/dart/{ => link}/lib/src/generated/frb_generated.dart (98%) rename link/sdks/dart/{ => link}/lib/src/generated/frb_generated.io.dart (100%) rename link/sdks/dart/{ => link}/lib/src/generated/frb_generated.web.dart (100%) rename link/sdks/dart/{ => link}/lib/src/generated/models.dart (98%) rename link/sdks/dart/{ => link}/lib/src/generated/models.freezed.dart (100%) rename link/sdks/dart/{ => link}/lib/src/kalam_client.dart (94%) create mode 100644 link/sdks/dart/link/lib/src/live_event_delivery.dart rename link/sdks/dart/{ => link}/lib/src/logger.dart (100%) rename link/sdks/dart/{ => link}/lib/src/models.dart (100%) rename link/sdks/dart/{ => link}/lib/src/seq_id.dart (100%) rename link/sdks/dart/{ => link}/lib/src/subscription_stream.dart (82%) rename link/sdks/dart/{ => link}/linux/CMakeLists.txt (100%) rename link/sdks/dart/{ => link}/linux/lib/.gitignore (100%) rename link/sdks/dart/{ => link}/macos/.gitignore (100%) rename link/sdks/dart/{ => link}/macos/Classes/KalamLinkPlugin.swift (100%) rename link/sdks/dart/{ => link}/macos/Libs/.gitignore (100%) rename link/sdks/dart/{ => link}/macos/Libs/libkalam_link_dart.dylib (100%) rename link/sdks/dart/{ => link}/macos/kalam_link.podspec (100%) rename link/sdks/dart/{ => link}/publish.sh (100%) rename link/sdks/dart/{ => link}/pubspec.lock (100%) rename link/sdks/dart/{ => link}/pubspec.yaml (100%) rename link/sdks/dart/{ => link}/test.sh (99%) rename link/sdks/dart/{ => link}/test/auth_provider_retry_test.dart (100%) rename link/sdks/dart/{ => link}/test/e2e/auth/auth_test.dart (100%) rename link/sdks/dart/{ => link}/test/e2e/ddl/ddl_test.dart (100%) rename link/sdks/dart/{ => link}/test/e2e/examples/examples_test.dart (100%) rename link/sdks/dart/{ => link}/test/e2e/helpers.dart (99%) rename link/sdks/dart/{ => link}/test/e2e/keepalive/keepalive_test.dart (100%) rename link/sdks/dart/{ => link}/test/e2e/keepalive/subscription_cleanup_test.dart (100%) rename link/sdks/dart/{ => link}/test/e2e/lifecycle/lifecycle_test.dart (100%) rename link/sdks/dart/{ => link}/test/e2e/query/query_test.dart (100%) rename link/sdks/dart/{ => link}/test/e2e/reconnect/app_lifecycle_test.dart (100%) create mode 100644 link/sdks/dart/link/test/e2e/reconnect/explicit_ack_test.dart rename link/sdks/dart/{ => link}/test/e2e/reconnect/reconnect_test.dart (100%) rename link/sdks/dart/{ => link}/test/e2e/reconnect/resume_test.dart (100%) rename link/sdks/dart/{ => link}/test/e2e/subscription/subscription_options_test.dart (100%) rename link/sdks/dart/{ => link}/test/e2e/subscription/subscription_test.dart (100%) rename link/sdks/dart/{ => link}/test/file_ref_test.dart (100%) rename link/sdks/dart/{ => link}/test/live_server_test.dart (99%) rename link/sdks/dart/{ => link}/test/main_thread_blocking_test.dart (100%) rename link/sdks/dart/{ => link}/test/models_test.dart (100%) rename link/sdks/dart/{ => link}/test/plugin_packaging_test.dart (100%) create mode 100644 link/sdks/dart/link/test/schema_generation_test.dart rename link/sdks/dart/{ => link}/test/subscription_stream_test.dart (63%) rename link/sdks/dart/{ => link}/web/.gitignore (100%) rename link/sdks/dart/{ => link}/web/pkg/.gitignore (100%) rename link/sdks/dart/{ => link}/web/pkg/README.md (100%) rename link/sdks/dart/{ => link}/web/pkg/kalam_link_dart.d.ts (100%) rename link/sdks/dart/{ => link}/web/pkg/kalam_link_dart.js (100%) rename link/sdks/dart/{ => link}/web/pkg/kalam_link_dart_bg.wasm (100%) rename link/sdks/dart/{ => link}/web/pkg/kalam_link_dart_bg.wasm.d.ts (100%) rename link/sdks/dart/{ => link}/web/pkg/package.json (100%) rename link/sdks/dart/{ => link}/windows/CMakeLists.txt (100%) rename link/sdks/dart/{ => link}/windows/lib/.gitignore (100%) create mode 100644 link/sdks/dart/sync/.gitignore create mode 100644 link/sdks/dart/sync/CHANGELOG.md create mode 100644 link/sdks/dart/sync/LICENSE create mode 100644 link/sdks/dart/sync/README.md create mode 100644 link/sdks/dart/sync/analysis_options.yaml create mode 100644 link/sdks/dart/sync/example/.gitignore create mode 100644 link/sdks/dart/sync/example/README.md create mode 100644 link/sdks/dart/sync/example/analysis_options.yaml create mode 100644 link/sdks/dart/sync/example/backend/.gitignore create mode 100644 link/sdks/dart/sync/example/backend/bin/server.dart create mode 100644 link/sdks/dart/sync/example/backend/lib/chat_backend.dart create mode 100644 link/sdks/dart/sync/example/backend/pubspec.lock create mode 100644 link/sdks/dart/sync/example/backend/pubspec.yaml create mode 100644 link/sdks/dart/sync/example/kalam.toml create mode 100644 link/sdks/dart/sync/example/kalam/schema.sql create mode 100644 link/sdks/dart/sync/example/lib/main.dart create mode 100644 link/sdks/dart/sync/example/pubspec.lock create mode 100644 link/sdks/dart/sync/example/pubspec.yaml create mode 100644 link/sdks/dart/sync/example/pubspec_overrides.yaml create mode 100644 link/sdks/dart/sync/lib/drift.dart create mode 100644 link/sdks/dart/sync/lib/kalam_sync.dart create mode 100644 link/sdks/dart/sync/lib/src/actions/kalam_action_codec.dart create mode 100644 link/sdks/dart/sync/lib/src/actions/kalam_action_context.dart create mode 100644 link/sdks/dart/sync/lib/src/actions/kalam_action_definition.dart create mode 100644 link/sdks/dart/sync/lib/src/actions/kalam_action_registry.dart create mode 100644 link/sdks/dart/sync/lib/src/actions/kalam_action_runner.dart create mode 100644 link/sdks/dart/sync/lib/src/actions/kalam_dml_action.dart create mode 100644 link/sdks/dart/sync/lib/src/annotations/kalam_action.dart create mode 100644 link/sdks/dart/sync/lib/src/annotations/kalam_action_module.dart create mode 100644 link/sdks/dart/sync/lib/src/annotations/kalam_action_payload.dart create mode 100644 link/sdks/dart/sync/lib/src/database/kalam_sync_database.dart create mode 100644 link/sdks/dart/sync/lib/src/database/kalam_sync_database.g.dart create mode 100644 link/sdks/dart/sync/lib/src/database/tables/kalam_action_steps.dart create mode 100644 link/sdks/dart/sync/lib/src/database/tables/kalam_actions.dart create mode 100644 link/sdks/dart/sync/lib/src/database/tables/kalam_cached_rows.dart create mode 100644 link/sdks/dart/sync/lib/src/database/tables/kalam_checkpoints.dart create mode 100644 link/sdks/dart/sync/lib/src/database/tables/kalam_row_states.dart create mode 100644 link/sdks/dart/sync/lib/src/flutter/kalam_database_factory.dart create mode 100644 link/sdks/dart/sync/lib/src/flutter/kalam_scope.dart create mode 100644 link/sdks/dart/sync/lib/src/kalam.dart create mode 100644 link/sdks/dart/sync/lib/src/models/kalam_account_identity.dart create mode 100644 link/sdks/dart/sync/lib/src/models/kalam_action_draft.dart create mode 100644 link/sdks/dart/sync/lib/src/models/kalam_action_record.dart create mode 100644 link/sdks/dart/sync/lib/src/models/kalam_action_status.dart create mode 100644 link/sdks/dart/sync/lib/src/models/kalam_cached_row.dart create mode 100644 link/sdks/dart/sync/lib/src/models/kalam_change.dart create mode 100644 link/sdks/dart/sync/lib/src/models/kalam_checkpoint.dart create mode 100644 link/sdks/dart/sync/lib/src/models/kalam_dml_payload.dart create mode 100644 link/sdks/dart/sync/lib/src/models/kalam_optimistic_mutation.dart create mode 100644 link/sdks/dart/sync/lib/src/models/kalam_optimistic_row.dart create mode 100644 link/sdks/dart/sync/lib/src/models/kalam_retry_policy.dart create mode 100644 link/sdks/dart/sync/lib/src/models/kalam_row_overlay.dart create mode 100644 link/sdks/dart/sync/lib/src/models/kalam_row_sync_state.dart create mode 100644 link/sdks/dart/sync/lib/src/models/kalam_sync_mode.dart create mode 100644 link/sdks/dart/sync/lib/src/models/kalam_sync_state.dart create mode 100644 link/sdks/dart/sync/lib/src/models/kalam_synced_row.dart create mode 100644 link/sdks/dart/sync/lib/src/store/kalam_sync_store.dart create mode 100644 link/sdks/dart/sync/lib/src/sync/kalam_event_consumer.dart create mode 100644 link/sdks/dart/sync/lib/src/sync/kalam_sync_coordinator.dart create mode 100644 link/sdks/dart/sync/lib/src/sync/kalam_sync_subscription.dart create mode 100644 link/sdks/dart/sync/lib/src/tables/kalam_replica_overlay.dart create mode 100644 link/sdks/dart/sync/lib/src/tables/kalam_table_binding.dart create mode 100644 link/sdks/dart/sync/lib/src/tables/kalam_table_spec.dart create mode 100644 link/sdks/dart/sync/lib/src/transport/kalam_link_transport.dart create mode 100644 link/sdks/dart/sync/lib/src/transport/kalam_remote_batch.dart create mode 100644 link/sdks/dart/sync/lib/src/transport/kalam_remote_change.dart create mode 100644 link/sdks/dart/sync/lib/src/transport/kalam_sync_transport.dart create mode 100644 link/sdks/dart/sync/pubspec.lock create mode 100644 link/sdks/dart/sync/pubspec.yaml create mode 100644 link/sdks/dart/sync/pubspec_overrides.yaml create mode 100644 link/sdks/dart/sync/test/actions/kalam_action_runner_test.dart create mode 100644 link/sdks/dart/sync/test/database/kalam_sync_database_test.dart create mode 100644 link/sdks/dart/sync/test/e2e/chat_sync_e2e_test.dart create mode 100644 link/sdks/dart/sync/test/e2e/kalam_sync_e2e_test.dart create mode 100644 link/sdks/dart/sync/test/e2e/kalam_sync_scale_e2e_test.dart create mode 100644 link/sdks/dart/sync/test/e2e/support/chat_e2e_harness.dart create mode 100644 link/sdks/dart/sync/test/flutter/kalam_scope_test.dart create mode 100644 link/sdks/dart/sync/test/integration/offline_restart_test.dart create mode 100644 link/sdks/dart/sync/test/kalam_from_components_test.dart create mode 100644 link/sdks/dart/sync/test/models/sync_models_test.dart create mode 100644 link/sdks/dart/sync/test/store/kalam_sync_store_test.dart create mode 100644 link/sdks/dart/sync/test/sync/kalam_sync_coordinator_test.dart create mode 100644 link/sdks/dart/sync/test/tables/kalam_table_binding_test.dart create mode 100644 link/sdks/dart/sync/test/transport/kalam_link_transport_test.dart delete mode 100644 link/sdks/dart/test/schema_generation_test.dart diff --git a/.cargo/config.toml b/.cargo/config.toml index 5d7f11897..fc9f0da92 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -2,6 +2,7 @@ [target.aarch64-apple-darwin] rustflags = [ "-C", "link-arg=-Wl,-rpath,/Library/Developer/CommandLineTools/usr/lib", + "-Z", "threads=8", ] [target.aarch64-apple-darwin.env] @@ -10,7 +11,10 @@ DYLD_FALLBACK_LIBRARY_PATH = "/Library/Developer/CommandLineTools/usr/lib" DYLD_LIBRARY_PATH = "/Library/Developer/CommandLineTools/usr/lib" [target.x86_64-apple-darwin] -rustflags = ["-C", "link-arg=-Wl,-rpath,/Library/Developer/CommandLineTools/usr/lib"] +rustflags = [ + "-C", "link-arg=-Wl,-rpath,/Library/Developer/CommandLineTools/usr/lib", + "-Z", "threads=8", +] [target.x86_64-apple-darwin.env] LIBCLANG_PATH = "/Library/Developer/CommandLineTools/usr/lib" @@ -31,16 +35,19 @@ rustflags = [] rustflags = [ "-C", "target-feature=+crt-static", "-C", "link-args=-static -static-libgcc -static-libstdc++", + "-Z", "threads=8", ] # --- Windows (MSVC) --- [target.x86_64-pc-windows-msvc] -# Usually no special flags are needed, but you can add linker hints if needed -rustflags = [] +rustflags = ["-Z", "threads=8"] [build] incremental = true -# Parallel compilation will use default (all available cores) +rustflags = ["-Z", "threads=8"] + +# Cranelift is not the default codegen backend. It drops aws-lc-sys C symbols +# (`__aws_lc_0_41_0_*`) when linking kalamdb-server on aarch64-apple-darwin. # Use sparse registry protocol for faster dependency resolution [registries.crates-io] diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 40fcfe31f..238bcf5d2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,6 +13,7 @@ on: env: CARGO_TERM_COLOR: always RUST_VERSION: "1.92.0" + CARGO_ENCODED_RUSTFLAGS: "" RUSTC_WRAPPER: "" CARGO_BUILD_RUSTC_WRAPPER: "" FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true" diff --git a/.github/workflows/cli-cluster-e2e.yml b/.github/workflows/cli-cluster-e2e.yml index 84ca4cf7e..3f8ece5c6 100644 --- a/.github/workflows/cli-cluster-e2e.yml +++ b/.github/workflows/cli-cluster-e2e.yml @@ -19,6 +19,7 @@ permissions: env: CARGO_TERM_COLOR: always RUST_VERSION: "1.92.0" + CARGO_ENCODED_RUSTFLAGS: "" RUSTC_WRAPPER: "" CARGO_BUILD_RUSTC_WRAPPER: "" FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true" diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index f69582576..305e3985b 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -18,6 +18,7 @@ on: env: FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true" + CARGO_ENCODED_RUSTFLAGS: "" jobs: analyze: diff --git a/.github/workflows/dart-sdk.yml b/.github/workflows/dart-sdk.yml index 299f07d3e..91f12b675 100644 --- a/.github/workflows/dart-sdk.yml +++ b/.github/workflows/dart-sdk.yml @@ -35,6 +35,7 @@ permissions: env: CARGO_TERM_COLOR: always RUST_VERSION: "1.92.0" + CARGO_ENCODED_RUSTFLAGS: "" RUSTC_WRAPPER: "" CARGO_BUILD_RUSTC_WRAPPER: "" FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true" @@ -185,7 +186,7 @@ jobs: id: run_tests continue-on-error: true shell: bash - working-directory: link/sdks/dart + working-directory: link/sdks/dart/link env: KALAMDB_URL: "http://localhost:2900" KALAMDB_USER: "admin" @@ -197,6 +198,33 @@ jobs: set -euo pipefail bash ./test.sh + - name: Run Kalam Sync tests + id: run_sync_tests + continue-on-error: true + shell: bash + working-directory: link/sdks/dart/sync + env: + KALAM_INTEGRATION_TEST: "1" + KALAMDB_URL: "http://localhost:2900" + KALAMDB_USER: "admin" + KALAMDB_PASSWORD: "kalamdb123" + run: | + set -euo pipefail + flutter analyze + flutter test + + - name: Run Kalam Sync generator tests + id: run_generator_tests + continue-on-error: true + shell: bash + working-directory: link/sdks/dart/generator + run: | + set -euo pipefail + flutter pub run build_runner build + flutter analyze + dart test + flutter test flutter_test/generated_action_runtime_e2e_test.dart + - name: Parse Dart SDK test counts if: always() id: parse_badge @@ -301,10 +329,10 @@ jobs: if-no-files-found: ignore - name: Fail if Dart SDK tests failed - if: always() && steps.run_tests.outcome != 'success' + if: ${{ always() && (steps.run_tests.outcome != 'success' || steps.run_sync_tests.outcome != 'success' || steps.run_generator_tests.outcome != 'success') }} shell: bash run: | - echo "Dart SDK tests failed" >&2 + echo "One or more Dart package test suites failed" >&2 exit 1 update_badge: @@ -392,8 +420,8 @@ jobs: - name: Publish Dart package shell: bash - working-directory: link/sdks/dart + working-directory: link/sdks/dart/link run: | set -euo pipefail chmod +x ./publish.sh - ./publish.sh \ No newline at end of file + ./publish.sh diff --git a/.github/workflows/python-sdk.yml b/.github/workflows/python-sdk.yml index 37d07dbfc..2952e18fb 100644 --- a/.github/workflows/python-sdk.yml +++ b/.github/workflows/python-sdk.yml @@ -25,6 +25,7 @@ on: env: CARGO_TERM_COLOR: always + CARGO_ENCODED_RUSTFLAGS: "" FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true" jobs: diff --git a/.github/workflows/react-e2e.yml b/.github/workflows/react-e2e.yml index 7f950cea0..261e3c15d 100644 --- a/.github/workflows/react-e2e.yml +++ b/.github/workflows/react-e2e.yml @@ -9,6 +9,7 @@ permissions: env: CARGO_TERM_COLOR: always RUST_VERSION: "1.92.0" + CARGO_ENCODED_RUSTFLAGS: "" WASM_PACK_VERSION: "0.15.0" KALAMDB_URL: "http://127.0.0.1:2900" FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 42dc0ee7d..5b5de77bc 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -75,6 +75,7 @@ permissions: env: CARGO_TERM_COLOR: always RUST_VERSION: "1.92.0" + CARGO_ENCODED_RUSTFLAGS: "" WASM_PACK_VERSION: "0.15.0" PG_EXTENSION_MAJOR: "16" PG_EXTENSION_FLAVOR: "pg16" diff --git a/.github/workflows/rust-coverage.yml b/.github/workflows/rust-coverage.yml index bddbbe0ed..226210a53 100644 --- a/.github/workflows/rust-coverage.yml +++ b/.github/workflows/rust-coverage.yml @@ -20,6 +20,7 @@ permissions: env: CARGO_TERM_COLOR: always RUST_VERSION: "1.92.0" + CARGO_ENCODED_RUSTFLAGS: "" RUSTC_WRAPPER: "" CARGO_BUILD_RUSTC_WRAPPER: "" CRITICAL_COVERAGE_MIN_PCT: "90.0" diff --git a/.github/workflows/rust-sdk.yml b/.github/workflows/rust-sdk.yml index 38e076822..d6681129b 100644 --- a/.github/workflows/rust-sdk.yml +++ b/.github/workflows/rust-sdk.yml @@ -38,6 +38,7 @@ permissions: env: CARGO_TERM_COLOR: always RUST_VERSION: "1.92.0" + CARGO_ENCODED_RUSTFLAGS: "" RUSTC_WRAPPER: "" CARGO_BUILD_RUSTC_WRAPPER: "" FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true" diff --git a/.github/workflows/typescript-sdk.yml b/.github/workflows/typescript-sdk.yml index c250a8794..d2b0296ca 100644 --- a/.github/workflows/typescript-sdk.yml +++ b/.github/workflows/typescript-sdk.yml @@ -26,6 +26,7 @@ permissions: env: CARGO_TERM_COLOR: always RUST_VERSION: "1.92.0" + CARGO_ENCODED_RUSTFLAGS: "" WASM_PACK_VERSION: "0.15.0" RUSTC_WRAPPER: "" CARGO_BUILD_RUSTC_WRAPPER: "" diff --git a/.github/workflows/versions.yml b/.github/workflows/versions.yml index d7d5e7000..ded3dae04 100644 --- a/.github/workflows/versions.yml +++ b/.github/workflows/versions.yml @@ -11,7 +11,7 @@ on: - 'link/sdks/rust/Cargo.toml' - 'link/sdks/rust/**' - 'link/sdks/typescript/**/package.json' - - 'link/sdks/dart/pubspec.yaml' + - 'link/sdks/dart/**/pubspec.yaml' - 'link/sdks/python/pyproject.toml' - 'link/sdks/python/Cargo.toml' - 'versions.json' @@ -48,4 +48,4 @@ jobs: shell: bash run: | set -euo pipefail - python3 scripts/versions.py verify \ No newline at end of file + python3 scripts/versions.py verify diff --git a/.gitignore b/.gitignore index a384dfb17..b583c76ab 100644 --- a/.gitignore +++ b/.gitignore @@ -102,8 +102,8 @@ ui/package-lock.json /target_cli_option_matrix /target_sec /target_cli_smoke_run -/link/sdks/dart/build -/link/sdks/dart/.dart_tool +/link/sdks/dart/link/build +/link/sdks/dart/link/.dart_tool /vendor /link/sdks/typescript/.wasm-cargo-home-test1 /link/sdks/typescript/.wasm-cargo-home-test2 diff --git a/.ignore b/.ignore index 832cf1dec..c8a4acdb7 100644 --- a/.ignore +++ b/.ignore @@ -11,9 +11,9 @@ docs/plans/*.png **/target/ **/.wasm-cargo-home/ **/.wasm-target/ -link/sdks/dart/lib/src/generated/ -link/sdks/dart/build/ -link/sdks/dart/.dart_tool/ +link/sdks/dart/link/lib/src/generated/ +link/sdks/dart/link/build/ +link/sdks/dart/link/.dart_tool/ # Runtime output. logs/ diff --git a/AGENTS.md b/AGENTS.md index 63255519a..5d595886f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -33,7 +33,9 @@ Keep context small. Read only the files needed for the current task. Do not scan - `cli/`: CLI and smoke tests. - `link/`: SDK bridge workspace. - `link/sdks/typescript/`: TypeScript SDK. -- `link/sdks/dart/`: Dart SDK. `link/sdks/dart/lib/src/generated` is generated; regenerate via `link/sdks/dart/build.sh`. +- `link/sdks/dart/link/`: `kalam_link` Dart transport SDK. `lib/src/generated` is generated; regenerate via its `build.sh`. +- `link/sdks/dart/sync/`: `kalam_sync` Flutter local-first runtime. +- `link/sdks/dart/generator/`: `kalam_sync_generator` action generator. - `link/kalam-link-dart/`: Rust bridge for Dart. - `pg/`: PostgreSQL extension. - `benchv2/`: benchmarks. @@ -62,6 +64,7 @@ Keep context small. Read only the files needed for the current task. Do not scan ## Build And Test +- Local check/test/run stays on the default dev profile. No `--release` unless packaging. - Batch compile feedback. For multi-file changes, finish an edit batch, then run one check and capture output, for example `cargo check > batch_compile_output.txt 2>&1`. - When there are multiple compiler errors, fix them from one captured output file instead of repeatedly running `cargo check`. - Use `cargo nextest run` for tests unless explicitly told otherwise. @@ -77,7 +80,7 @@ Keep context small. Read only the files needed for the current task. Do not scan - Backend run: `cd backend && cargo run --bin kalamdb-server` - Backend fast check (local dev, no S3/mimalloc/tracing): `cargo check -p kalamdb-server` - Backend prod-like build: `cargo build -p kalamdb-server --no-default-features --features embedded-ui,mimalloc,traceability,cloud-aws` -- CLI build: `cd cli && cargo build --release` +- CLI build: `cd cli && cargo build` - CLI smoke: `cd cli && KALAMDB_SERVER_URL="http://localhost:3000" KALAMDB_ROOT_PASSWORD="mypass" cargo test --test smoke -- --nocapture` - Full sweep: start the backend server, then run `./scripts/test-all.sh` from the repo root. - Rust SDK (fast iteration): `cargo check -p kalam-client --features native-sdk,consumer,healthcheck`. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 53fe1287d..980f85191 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -260,13 +260,17 @@ find . -name package.json -not -path '*/node_modules/*' ## 6. Compile and Test the Dart SDK -The Dart and Flutter SDK lives in `link/sdks/dart/`. +The Dart and Flutter packages live under `link/sdks/dart/`: -Do not edit `link/sdks/dart/lib/src/generated/` by hand. Those files are generated. +- `link/` publishes `kalam_link`; +- `sync/` publishes `kalam_sync`; +- `generator/` publishes `kalam_sync_generator`. + +Do not edit `link/sdks/dart/link/lib/src/generated/` by hand. Those files are generated. ### Build the SDK and native artifacts -From `link/sdks/dart/`: +From `link/sdks/dart/link/`: ```bash ./build.sh @@ -298,7 +302,7 @@ FRB_GENERATE=always ./build.sh The supported wrapper is: ```bash -cd link/sdks/dart +cd link/sdks/dart/link ./test.sh ``` @@ -316,7 +320,7 @@ Start the backend first when you want the e2e part to run: cd backend cargo run -cd ../link/sdks/dart +cd ../link/sdks/dart/link ./test.sh ``` @@ -340,7 +344,9 @@ Examples: - PG extension change: `./pg/test.sh` - Admin UI change: `cd ui && npm run build && npm test` - TypeScript SDK change: `cd link/sdks/typescript/client && ./test.sh` -- Dart SDK change: `cd link/sdks/dart && ./test.sh` +- Dart transport change: `cd link/sdks/dart/link && ./test.sh` +- Dart sync change: `cd link/sdks/dart/sync && flutter analyze && flutter test` +- Dart generator change: `cd link/sdks/dart/generator && flutter analyze && dart test` For cross-cutting changes, use the repo-wide sweep after starting the backend server: @@ -352,4 +358,4 @@ For cross-cutting changes, use the repo-wide sweep after starting the backend se - If you change architecture or execution flow, update the relevant docs in `docs/architecture/` or `docs/architecture/decisions/`. - If you change anything under `link/sdks/**`, update the matching SDK docs in the `KalamSite` repo as well. -- Keep commands in this file aligned with the actual scripts in `backend/`, `cli/`, `pg/`, `ui/`, and `link/sdks/`. \ No newline at end of file +- Keep commands in this file aligned with the actual scripts in `backend/`, `cli/`, `pg/`, `ui/`, and `link/sdks/`. diff --git a/Cargo.lock b/Cargo.lock index 42363bf7e..7ef5bed9c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1339,9 +1339,9 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cc" -version = "1.4.2" +version = "1.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d262e149917187838d5b42777c8253bcb64500067342904e7d429499a6f277e" +checksum = "509591b7bcd67f4ef775afad7662703b4935daaa6ec0e5605cfb1090b32a2b6d" dependencies = [ "find-msvc-tools", "jobserver", @@ -3227,9 +3227,9 @@ dependencies = [ [[package]] name = "find-msvc-tools" -version = "0.1.10" +version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26b73573e6edcd2af0cdf47bd6cb58f0b3839491263c314eaad1ccf24430e1de" +checksum = "d45db016d36b838f563236e9193d0ee6ce38f3f68b6c94e914b4929c96bbb890" [[package]] name = "fixedbitset" @@ -5586,15 +5586,16 @@ dependencies = [ [[package]] name = "librocksdb-sys" -version = "0.17.3+10.4.2" +version = "0.19.0+11.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cef2a00ee60fe526157c9023edab23943fae1ce2ab6f4abb2a807c1746835de9" +checksum = "4f45e86edad8e88efe97dbf384b4e48e1ff0f111eabf154c7b09d7a1e5fb573c" dependencies = [ "bindgen", "bzip2-sys", "cc", "libc", "libz-sys", + "rustflags", ] [[package]] @@ -7715,9 +7716,9 @@ dependencies = [ [[package]] name = "rocksdb" -version = "0.24.0" +version = "0.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddb7af00d2b17dbd07d82c0063e25411959748ff03e8d4f96134c2ff41fce34f" +checksum = "d8d90add70d1d420ee487bce4a1449880a8d147451c6051b2ee5f8354553dcbf" dependencies = [ "libc", "librocksdb-sys", @@ -7839,6 +7840,12 @@ dependencies = [ "semver", ] +[[package]] +name = "rustflags" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a39e0e9135d7a7208ee80aa4e3e4b88f0f5ad7be92153ed70686c38a03db2e63" + [[package]] name = "rusticata-macros" version = "4.1.0" @@ -9520,9 +9527,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.24.0" +version = "1.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf3923a6f5c4c6382e0b653c4117f48d631ea17f38ed86e2a828e6f7412f5239" +checksum = "2cefc03fd367c0c6d4305de1b312cf00248c4114f4a0418ce6a6af769e3b0bd9" dependencies = [ "getrandom 0.4.2", "js-sys", diff --git a/Cargo.toml b/Cargo.toml index 2bed88b16..c8b817219 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -62,7 +62,7 @@ members = [ exclude = ["benchv2"] [workspace.package] -version = "0.5.6-rc.0" +version = "0.5.6-rc.1" edition = "2021" rust-version = "1.92" authors = ["KalamDB Team"] @@ -108,7 +108,7 @@ rustls-webpki = { version = "0.103.14", default-features = false } chrono = { version = "0.4.45", features = ["serde"] } # Storage - rocksdb = { version = "0.24.0", default-features = false, features = ["snappy", "multi-threaded-cf", "bindgen-runtime"] } + rocksdb = { version = "0.25.0", default-features = false, features = ["snappy", "multi-threaded-cf", "bindgen-runtime"] } # Apache Arrow ecosystem # IMPORTANT: arrow, arrow-schema and parquet MUST match the version used by DataFusion. @@ -135,7 +135,7 @@ actix-files = "0.6.10" actix-multipart = "0.8.0" # UUID generation -uuid = { version = "1.24.0", features = ["v4", "v7", "serde"] } +uuid = { version = "1.24.1", features = ["v4", "v7", "serde"] } # NanoID generation (21-char URL-safe unique IDs) nanoid = "0.5.0" @@ -223,7 +223,7 @@ pgwire = { version = "0.40.6", default-features = false, features = ["server-api bytes = "1.12.1" http-body = "1.1.0" http-body-util = "0.1.5" -cc = "1.4.2" +cc = "1.4.3" proc-macro2 = "1.0.107" quote = "1.0.47" syn = { version = "3.0.3", features = ["full", "extra-traits"] } @@ -287,7 +287,7 @@ strip = true # Strip symbols from the binary # Use with: cargo build --profile release-dist # Fat LTO helps cdylib/.so/.wasm (the linker runs). It inflates staticlib # (.a) artifacts with LLVM bitcode that Cargo cannot strip — iOS builds in -# link/sdks/dart/build.sh disable LTO and strip bitcode/debug from the archive. +# link/sdks/dart/link/build.sh disables LTO and strips bitcode/debug from the archive. [profile.release-dist] inherits = "release" opt-level = "z" # Optimize for size (most aggressive) @@ -311,12 +311,15 @@ lto = true # Full LTO for maximum dead-code elimination panic = "unwind" # Required by pgrx (PostgreSQL longjmp unwinding) [profile.dev] -opt-level = 0 # no optimizations = faster compile -debug = 0 # minimal debug info = faster linking -incremental = true # incremental builds -lto = "off" # no link-time optimization -codegen-units = 256 # maximum parallelism for codegen -split-debuginfo = "unpacked" # faster on macOS +opt-level = 0 +debug = "line-tables-only" +incremental = true +lto = "off" +codegen-units = 256 +split-debuginfo = "unpacked" + +[profile.test] +debug = "line-tables-only" [profile.dev.build-override] opt-level = 0 diff --git a/backend/crates/kalamdb-api/src/limiter/rate_limiter.rs b/backend/crates/kalamdb-api/src/limiter/rate_limiter.rs index 118c01531..5d7eb3d8e 100644 --- a/backend/crates/kalamdb-api/src/limiter/rate_limiter.rs +++ b/backend/crates/kalamdb-api/src/limiter/rate_limiter.rs @@ -137,7 +137,7 @@ impl RateLimiter { if let Some(count) = self.user_subscription_counts.get(&user_key) { count - .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |v| Some(v.saturating_sub(1))) + .try_update(Ordering::Relaxed, Ordering::Relaxed, |v| Some(v.saturating_sub(1))) .ok(); } } diff --git a/backend/crates/kalamdb-core/tests/sql_cached_point_get.rs b/backend/crates/kalamdb-core/tests/sql_cached_point_get.rs index 6e3b1eee4..03baf5dc7 100644 --- a/backend/crates/kalamdb-core/tests/sql_cached_point_get.rs +++ b/backend/crates/kalamdb-core/tests/sql_cached_point_get.rs @@ -264,6 +264,16 @@ async fn cached_point_get_keeps_file_ref_select_list_after_update() { .await, ); assert_eq!(file_ref_sha256(&literal_cached[0]), second_sha); + + let limited = format!("SELECT file_ref FROM {qualified} WHERE path = $1 LIMIT 1"); + for i in 0..3 { + let rows = result_rows( + execute_ok_with_params(&executor, &exec_ctx, &limited, path_param.clone()).await, + ); + assert_eq!(rows.len(), 1, "LIMIT 1 lookup {i}"); + assert_single_column(&rows[0], "file_ref"); + assert_eq!(file_ref_sha256(&rows[0]), second_sha); + } } #[tokio::test] diff --git a/backend/crates/kalamdb-live/src/manager/connections_manager.rs b/backend/crates/kalamdb-live/src/manager/connections_manager.rs index bb8ce1f21..a7a1c43c1 100644 --- a/backend/crates/kalamdb-live/src/manager/connections_manager.rs +++ b/backend/crates/kalamdb-live/src/manager/connections_manager.rs @@ -253,7 +253,7 @@ impl ConnectionsManager { fn release_connection_slot(&self) { if self .total_connections - .fetch_update(Ordering::AcqRel, Ordering::Acquire, |current| current.checked_sub(1)) + .try_update(Ordering::AcqRel, Ordering::Acquire, |current| current.checked_sub(1)) .is_err() { warn!("Connection count release requested while count was already zero"); diff --git a/backend/crates/kalamdb-observability/src/storage_metrics.rs b/backend/crates/kalamdb-observability/src/storage_metrics.rs index 5446328fd..b282d6730 100644 --- a/backend/crates/kalamdb-observability/src/storage_metrics.rs +++ b/backend/crates/kalamdb-observability/src/storage_metrics.rs @@ -187,7 +187,7 @@ pub fn decrement_manifest_cache_rocksdb_entries(delta: usize) { } let decrement = delta as u64; - let _ = MANIFEST_CACHE_ROCKSDB_ENTRIES.fetch_update( + let _ = MANIFEST_CACHE_ROCKSDB_ENTRIES.try_update( Ordering::AcqRel, Ordering::Acquire, |current| Some(current.saturating_sub(decrement)), diff --git a/backend/crates/kalamdb-store/src/backends/rocksdb/backend.rs b/backend/crates/kalamdb-store/src/backends/rocksdb/backend.rs index e6179275d..7773302ac 100644 --- a/backend/crates/kalamdb-store/src/backends/rocksdb/backend.rs +++ b/backend/crates/kalamdb-store/src/backends/rocksdb/backend.rs @@ -11,7 +11,7 @@ use rocksdb::{BoundColumnFamily, Cache, IteratorMode, Options, PrefixRange, Writ use super::{ cf_tuning::apply_cf_settings, - init::create_block_options_with_cache, + init::{create_block_options_with_cache, new_block_cache}, keyspace::{ decode_logical_partition_registry_key, logical_partition_registry_key, logical_partition_registry_prefix, next_prefix_bound, partition_key_prefix, @@ -77,7 +77,7 @@ impl RocksDBBackend { /// Creates a new RocksDB backend with the given database handle. pub fn new(db: Arc) -> Self { let settings = RocksDbSettings::default(); - let block_cache = Cache::new_lru_cache(settings.block_cache_size); + let block_cache = new_block_cache(settings.block_cache_size); Self::new_internal(db, false, false, settings, block_cache) } @@ -88,7 +88,7 @@ impl RocksDBBackend { disable_wal: bool, settings: RocksDbSettings, ) -> Self { - let block_cache = Cache::new_lru_cache(settings.block_cache_size); + let block_cache = new_block_cache(settings.block_cache_size); Self::new_internal(db, sync_writes, disable_wal, settings, block_cache) } diff --git a/backend/crates/kalamdb-store/src/backends/rocksdb/init.rs b/backend/crates/kalamdb-store/src/backends/rocksdb/init.rs index cb644841c..23ee0342a 100644 --- a/backend/crates/kalamdb-store/src/backends/rocksdb/init.rs +++ b/backend/crates/kalamdb-store/src/backends/rocksdb/init.rs @@ -41,7 +41,7 @@ impl RocksDbInit { db_opts.create_missing_column_families(true); apply_db_settings(&mut db_opts, &self.settings); - let cache = Cache::new_lru_cache(self.settings.block_cache_size); + let cache = new_block_cache(self.settings.block_cache_size); let block_opts = create_block_options_with_cache(&cache); db_opts.set_block_based_table_factory(&block_opts); @@ -61,7 +61,7 @@ impl RocksDbInit { db_opts.create_missing_column_families(true); apply_db_settings(&mut db_opts, &self.settings); - let cache = Cache::new_lru_cache(self.settings.block_cache_size); + let cache = new_block_cache(self.settings.block_cache_size); let block_opts = create_block_options_with_cache(&cache); db_opts.set_block_based_table_factory(&block_opts); @@ -116,11 +116,20 @@ impl RocksDbInit { } } +/// Shared block cache for all column families. +/// +/// HyperClockCache is the RocksDB 11.8 recommended implementation for concurrent +/// point lookups. `estimated_entry_charge = 0` selects the dynamically sized variant. +pub(crate) fn new_block_cache(capacity: usize) -> Cache { + Cache::new_hyper_clock_cache(capacity, 0) +} + pub(crate) fn create_block_options_with_cache(cache: &Cache) -> BlockBasedOptions { let mut block_opts = BlockBasedOptions::default(); block_opts.set_block_cache(cache); block_opts.set_bloom_filter(10.0, false); block_opts.set_cache_index_and_filter_blocks(true); + block_opts.set_cache_index_and_filter_blocks_with_high_priority(true); block_opts.set_pin_l0_filter_and_index_blocks_in_cache(true); block_opts.set_pin_top_level_index_and_filter(true); block_opts.set_whole_key_filtering(true); diff --git a/backend/src/connection_guard.rs b/backend/src/connection_guard.rs index c98287308..cc3a8ff25 100644 --- a/backend/src/connection_guard.rs +++ b/backend/src/connection_guard.rs @@ -259,7 +259,7 @@ impl ConnectionGuard { if let Some(state) = self.ip_states.get(&ip) { state .active_connections - .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |value| { + .try_update(Ordering::Relaxed, Ordering::Relaxed, |value| { Some(value.saturating_sub(1)) }) .ok(); diff --git a/benchv2/comparison/results/kalamdb-20260817-123845.txt b/benchv2/comparison/results/kalamdb-20260817-123845.txt new file mode 100644 index 000000000..48a9462d0 --- /dev/null +++ b/benchv2/comparison/results/kalamdb-20260817-123845.txt @@ -0,0 +1,30 @@ +# KalamDB comparison (hot-only, tuned RocksDB/cache) +# server_bin=/Users/jamal/git/KalamDB/target/release/kalamdb-server +# flush.check_interval_seconds=0 +# table created without FLUSH_POLICY +# rocksdb.block_cache=1GiB hot_data.write_buffer=128MiB +# datafusion query_parallelism=2 max_partitions=1 batch_size=128 +# sql=parameterized; no multi-row batch (matches TB/PB one-row HTTP) +# timed_read_response=bytes; HTTP status validation only (matches TB/PB) +# http2_prior_knowledge=0 +# schema_response_cache=on; hot_only_pk_skip_cold=on +# started=2026-08-17T09:38:46Z +negotiated_http_version=HTTP/1.1 +logged in to http://127.0.0.1:3900 +mode=hot-only (no FLUSH_POLICY, flush scheduler disabled) +sql=parameterized ($1..) for plan-cache reuse +timed_read_response=bytes (HTTP status validation only; matches TB/PB) +Inserted 100000 rows in 6.491179375s +Inserted 10000 rows in 670.205083ms +Latencies: + p50=1.016375ms + p75=1.112667ms + p90=1.189292ms + p95=1.278125ms +Read 1000000 rows in 17.269477167s +Latencies: + p50=251.542µs + p75=288µs + p90=321.958µs + p95=343.458µs +# finished=2026-08-17T09:39:10Z diff --git a/benchv2/comparison/results/pocketbase-20260817-123956.txt b/benchv2/comparison/results/pocketbase-20260817-123956.txt new file mode 100644 index 000000000..9523302b9 --- /dev/null +++ b/benchv2/comparison/results/pocketbase-20260817-123956.txt @@ -0,0 +1,17 @@ +# PocketBase comparison (collections Record API) +# started=2026-08-17T09:39:56Z +logged in to http://127.0.0.1:8090 owner=d5vil65xdaqinn2 room=8gw0ig75rwm4vbe +Inserted 100000 rows in 14.19929975s +Inserted 10000 rows in 1.488549625s +Latencies: + p50=1.951167ms + p75=2.715542ms + p90=3.864334ms + p95=5.240792ms +Read 1000000 rows in 19.62557825s +Latencies: + p50=258.583µs + p75=338.958µs + p90=439.583µs + p95=519.417µs +# finished=2026-08-17T09:40:31Z diff --git a/benchv2/comparison/results/trailbase-20260817-123921.txt b/benchv2/comparison/results/trailbase-20260817-123921.txt new file mode 100644 index 000000000..07037ec37 --- /dev/null +++ b/benchv2/comparison/results/trailbase-20260817-123921.txt @@ -0,0 +1,17 @@ +# TrailBase comparison (Record API) +# started=2026-08-17T09:39:21Z +logged in to http://127.0.0.1:4000 +Inserted 100000 rows in 5.071317417s +Inserted 10000 rows in 512.303958ms +Latencies: + p50=784.084µs + p75=918.584µs + p90=1.050833ms + p95=1.138084ms +Read 1000000 rows in 17.290355625s +Latencies: + p50=229.042µs + p75=275µs + p90=325.625µs + p95=361.542µs +# finished=2026-08-17T09:39:44Z diff --git a/cli/DEV.md b/cli/DEV.md index 28a5a53e9..8a48c7ee7 100644 --- a/cli/DEV.md +++ b/cli/DEV.md @@ -31,7 +31,8 @@ The workflow surface exists and is usable, but some parts are intentionally v1-l - `kalam dev` can manage a local KalamDB server when `dev.auto_start_db = true`, or connect to an existing server when it is `false`. - `kalam schema gen` delegates TypeScript generation to `@kalamdb/orm` against the resolved - KalamDB environment and writes a Dart placeholder until a dedicated Dart generator exists. + KalamDB environment. Dart generation reads local `schema.sql` and writes `KalamTableSpec` + row codecs to `lib/generated/kalam.dart` (no live server required). - `kalam migration create` creates ordered SQL migration files using the current schema diff helper. - `kalam migration status` and `kalam db migrate` use local file-based migration state. - `kalam deploy` performs guardrails, applies pending local migrations, runs a lightweight rollout @@ -190,7 +191,7 @@ When run in a TTY without `--yes`, `kalam init` asks: 1. Project name 2. Schema mode 3. Language targets -4. Project template when TypeScript is selected +4. Project template (TypeScript and/or Dart/Flutter `simple-live`) 5. Package manager when TypeScript is selected and more than one manager is available 6. Server mode 7. Server URL when server mode is `remote` @@ -210,14 +211,15 @@ Schema mode is a single-choice menu: Language targets are a multi-select menu: - `TypeScript` -- `Dart` +- `Dart / Flutter` The project template menu includes two starter sources: -- embedded TypeScript templates compiled into the CLI from `cli/templates/typescript/*` +- embedded templates compiled into the CLI from `cli/templates/typescript/*` and `cli/templates/dart/*` - repository examples downloaded from `examples/*` in the KalamDB GitHub repository -- `simple-live` - live subscription starter with sample inserts +- `simple-live` (TypeScript) - live subscription starter with sample inserts +- `simple-live` (Dart / Flutter) - `kalam_sync` starter with `lib/main.dart` and generated table specs - `live-okf-context-sync` - OKF folder sync with live FILE columns - `realtime-ops-feed` - small browser app with live SQL subscriptions - `chat-with-ai` - topic-driven React chat with an agent worker @@ -237,7 +239,7 @@ Server mode is a single-choice menu: - `--name `: project name - `--schema-mode `: active schema source mode -- `--languages `: comma-separated language list, currently `typescript` and/or `dart` +- `--languages `: comma-separated language list (`typescript`, `dart`; `ts` and `flutter` are aliases) - `--template `: embedded template or repository example id - `--server-mode `: local server management mode for `kalam dev` - `--server-url `: server URL for the scaffolded `dev` environment @@ -303,9 +305,8 @@ Generates enabled language artifacts for the resolved workflow environment. Current behavior: -- `typescript` is generated through `@kalamdb/orm` -- generation targets the resolved `url` + `namespace` from `kalam.toml` / env overrides -- `dart` currently writes a placeholder file only +- `typescript` is generated through `@kalamdb/orm` against the resolved `url` + `namespace` +- `dart` is generated locally from `schema.sql` into `KalamTableSpec` row codecs (no server required) #### Options @@ -318,6 +319,7 @@ Current behavior: ```bash kalam schema gen kalam schema gen --languages typescript +kalam schema gen --languages dart ``` ### `kalam schema pull` diff --git a/cli/kalam-cli-e2e/tests/cli/test_project_workflow_init.rs b/cli/kalam-cli-e2e/tests/cli/test_project_workflow_init.rs index f6b11b2b3..b8f63e2a0 100644 --- a/cli/kalam-cli-e2e/tests/cli/test_project_workflow_init.rs +++ b/cli/kalam-cli-e2e/tests/cli/test_project_workflow_init.rs @@ -76,6 +76,12 @@ fn test_project_workflow_init_scaffolds_project() { assert!(project_dir.join("kalam/cli/logs").is_dir(), "CLI logs dir missing"); assert!(project_dir.join("src/generated").is_dir(), "typescript output dir missing"); assert!(project_dir.join("lib/generated").is_dir(), "dart output dir missing"); + assert!(project_dir.join("pubspec.yaml").is_file(), "dart pubspec missing"); + assert!(project_dir.join("lib/main.dart").is_file(), "flutter main missing"); + let generated_dart = + fs::read_to_string(project_dir.join("lib/generated/kalam.dart")).expect("read generated dart"); + assert!(generated_dart.contains("KalamTableSpec")); + assert!(!generated_dart.to_lowercase().contains("placeholder")); assert!(project_dir.join(".env.example").is_file(), ".env.example missing"); let kalam_toml = fs::read_to_string(project_dir.join("kalam.toml")).expect("read kalam.toml"); @@ -170,6 +176,60 @@ fn test_project_workflow_init_defaults_to_typescript_and_scaffolds_starter() { assert!(project_dir.join("tsconfig.json").is_file(), "tsconfig.json missing"); } +#[test] +fn test_project_workflow_init_scaffolds_dart_flutter_project() { + let temp = TempDir::new().expect("temp dir"); + let project_dir = temp.path().join("demo-flutter"); + fs::create_dir_all(&project_dir).expect("create project dir"); + + let mut cmd = create_cli_command(); + cmd.current_dir(&project_dir).args([ + "init", + "--yes", + "--name", + "demo-flutter", + "--schema-mode", + "sql", + "--languages", + "dart", + "--template", + "simple-live", + ]); + + let output = cmd.output().expect("run init"); + assert!( + output.status.success(), + "dart init should succeed\nstdout: {}\nstderr: {}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + + let kalam_toml = + fs::read_to_string(project_dir.join("kalam.toml")).expect("read generated kalam.toml"); + assert!(kalam_toml.contains("languages = [\"dart\"]")); + assert!(kalam_toml.contains("[schema.targets.dart]")); + assert!(kalam_toml.contains("app = \"flutter run\"")); + assert!(!kalam_toml.contains("package_manager")); + assert!(!project_dir.join("package.json").exists(), "dart-only init should not write package.json"); + assert!(project_dir.join("pubspec.yaml").is_file(), "pubspec.yaml missing"); + assert!(project_dir.join("lib/main.dart").is_file(), "lib/main.dart missing"); + assert!(project_dir.join("schema.sql").is_file(), "schema.sql missing"); + + let pubspec = fs::read_to_string(project_dir.join("pubspec.yaml")).expect("read pubspec"); + assert!(pubspec.contains("kalam_sync")); + assert!(pubspec.contains("name: demo_flutter")); + + let main_dart = fs::read_to_string(project_dir.join("lib/main.dart")).expect("read main"); + assert!(main_dart.contains("Kalam.open")); + assert!(main_dart.contains("KalamTables.users")); + + let generated = + fs::read_to_string(project_dir.join("lib/generated/kalam.dart")).expect("read generated"); + assert!(generated.contains("Generated by kalam schema gen")); + assert!(generated.contains("KalamTableSpec")); + assert!(!generated.to_lowercase().contains("placeholder")); +} + #[test] fn test_project_workflow_init_preserves_existing_gitignore() { let temp = TempDir::new().expect("temp dir"); diff --git a/cli/kalam-cli-e2e/tests/cli/test_project_workflow_schema.rs b/cli/kalam-cli-e2e/tests/cli/test_project_workflow_schema.rs index 9ebaa0f4d..2f9c8eb72 100644 --- a/cli/kalam-cli-e2e/tests/cli/test_project_workflow_schema.rs +++ b/cli/kalam-cli-e2e/tests/cli/test_project_workflow_schema.rs @@ -266,7 +266,11 @@ fn test_project_workflow_schema_gen_from_sql() { ); let dart = fs::read_to_string(dart_output).expect("read dart"); - assert!(dart.to_lowercase().contains("placeholder"), "expected dart placeholder output"); + assert!(dart.contains("Generated by kalam schema gen"), "expected generated dart header"); + assert!(dart.contains("import 'package:kalam_sync/kalam_sync.dart';")); + assert!(dart.contains("KalamTableSpec"), "expected Users table spec"); + assert!(dart.contains("tableId: 'users'")); + assert!(!dart.to_lowercase().contains("placeholder"), "dart output should not be a placeholder"); } #[test] diff --git a/cli/kalam-cli-e2e/tests/smoke/query/smoke_test_cached_point_get_projection.rs b/cli/kalam-cli-e2e/tests/smoke/query/smoke_test_cached_point_get_projection.rs index dd72f7c4e..176622b35 100644 --- a/cli/kalam-cli-e2e/tests/smoke/query/smoke_test_cached_point_get_projection.rs +++ b/cli/kalam-cli-e2e/tests/smoke/query/smoke_test_cached_point_get_projection.rs @@ -254,4 +254,11 @@ fn smoke_test_cached_point_get_projection() { assert_eq!(names, vec!["payload"], "stream lookup {i}"); assert_eq!(string_value(&rows[0], "payload"), "stream-alpha"); } + + let limited_sql = format!("SELECT file_ref FROM {files_table} WHERE path = $1 LIMIT 1"); + for i in 0..3 { + let (names, rows) = query_rows(&limited_sql, path_param.clone()); + assert_eq!(names, vec!["file_ref"], "LIMIT 1 lookup {i}"); + assert_eq!(file_ref_sha256(&rows[0]), second_sha); + } } diff --git a/cli/run-tests.sh b/cli/run-tests.sh index 819a3d24c..20d4d82fa 100755 --- a/cli/run-tests.sh +++ b/cli/run-tests.sh @@ -1535,7 +1535,7 @@ run_supplementary_suites() { step "Running Dart SDK tests" ( - cd "$REPO_ROOT/link/sdks/dart" + cd "$REPO_ROOT/link/sdks/dart/link" ./test.sh ) diff --git a/cli/src/args/workflow.rs b/cli/src/args/workflow.rs index 2a8873573..87ad6ac2d 100644 --- a/cli/src/args/workflow.rs +++ b/cli/src/args/workflow.rs @@ -14,7 +14,7 @@ pub struct InitArgs { #[arg(long = "schema-mode", value_enum)] pub schema_mode: Option, - /// Comma-separated generated language targets (typescript,dart) + /// Comma-separated generated language targets (typescript, dart/flutter) #[arg(long = "languages", value_delimiter = ',')] pub languages: Option>, @@ -184,7 +184,7 @@ pub enum SchemaCommand { #[derive(Args, Debug, Clone, Default)] pub struct SchemaGenerateArgs { - /// Limit generation to specific language targets + /// Limit generation to specific language targets (typescript, dart/flutter) #[arg(long = "languages", value_delimiter = ',')] pub languages: Option>, } diff --git a/cli/src/workflow/project/dart/init.rs b/cli/src/workflow/project/dart/init.rs new file mode 100644 index 000000000..cb146016e --- /dev/null +++ b/cli/src/workflow/project/dart/init.rs @@ -0,0 +1,180 @@ +//! Dart/Flutter-specific `kalam init` helpers: templates and Flutter bootstrap. + +use std::{env, path::Path, process::Command}; + +use crate::{ + error::{CLIError, Result}, + output::WorkflowOutput, + terminal_ui::{self, SelectOption}, + workflow::project::{ + dart::templates::{self, DEFAULT_TEMPLATE}, + prompts::prompt_select, + templates::EmbeddedTemplate, + ts::{apply_scaffold, SKIP_PACKAGE_INSTALL_ENV}, + }, +}; + +pub const SCHEMA_TARGET_OUTPUT: &str = "lib/generated/kalam.dart"; +pub const DEFAULT_DEV_COMMAND: &str = "flutter run"; + +pub fn is_enabled(languages: &[String]) -> bool { + languages.iter().any(|language| language == "dart") +} + +pub fn resolve_starter( + languages: &[String], + template_id: Option<&str>, + non_interactive: bool, + color: bool, +) -> Result> { + if !is_enabled(languages) { + return Ok(None); + } + + if let Some(template_id) = template_id.map(str::trim).filter(|value| !value.is_empty()) { + return templates::resolve(Some(template_id)).map(Some); + } + + if non_interactive { + return templates::resolve(Some(DEFAULT_TEMPLATE)).map(Some); + } + + let available = templates::available(); + if available.is_empty() { + return Err(CLIError::ConfigurationError( + "no built-in Dart/Flutter templates are available in this CLI build".into(), + )); + } + + let options: Vec> = available + .iter() + .map(|template| SelectOption::described(template.id, template.description)) + .collect(); + let default_index = + available.iter().position(|template| template.id == DEFAULT_TEMPLATE).unwrap_or(0); + let selected = prompt_select("Dart/Flutter project template", &options, default_index, color)?; + let template = available[selected]; + eprintln!( + "{} {}", + terminal_ui::prompt_label("Dart template:", color), + terminal_ui::style_value(template.id, color) + ); + Ok(Some(template)) +} + +pub fn apply_dart_scaffold( + root: &Path, + template: &EmbeddedTemplate, + project_name: &str, + server_url: &str, + namespace: &str, + output: &WorkflowOutput, +) -> Result<()> { + apply_scaffold(root, template, project_name, server_url, namespace, output) +} + +pub fn maybe_bootstrap_flutter_project( + root: &Path, + package_name: &str, + output: &WorkflowOutput, +) -> Result<()> { + if env::var_os(SKIP_PACKAGE_INSTALL_ENV).is_some() { + output.detail("skipped flutter create step"); + return Ok(()); + } + + if Command::new("flutter").arg("--version").output().is_err() { + output.detail("skipped flutter create; flutter was not found on PATH"); + return Ok(()); + } + + output.status("creating Flutter platform files"); + let create = Command::new("flutter") + .current_dir(root) + .args(["create", ".", "--project-name", package_name, "--platforms", "macos,web"]) + .output() + .map_err(|error| { + CLIError::ConfigurationError(format!("failed to run flutter create: {error}")) + })?; + if !create.status.success() { + output.warn( + "flutter create failed, but project files are on disk — add platforms with `flutter create .`", + ); + return Ok(()); + } + + let pub_get = Command::new("flutter").current_dir(root).args(["pub", "get"]).output(); + match pub_get { + Ok(output_status) if output_status.status.success() => { + output.status("installed Flutter packages"); + }, + _ => { + output.warn("flutter pub get failed; run `flutter pub get` in the project directory"); + }, + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn dart_starter_defaults_to_simple_live() { + let template = resolve_starter(&["dart".into()], None, true, false) + .expect("starter should resolve") + .expect("starter should be selected"); + assert_eq!(template.id, "simple-live"); + assert_eq!(template.language, "dart"); + } + + #[test] + fn dart_starter_is_skipped_without_dart_language() { + let starter = + resolve_starter(&["typescript".into()], Some("simple-live"), true, false).unwrap(); + assert!(starter.is_none()); + } + + #[test] + fn dart_simple_live_template_renders_flutter_starter() { + use crate::workflow::project::templates::{find_template, render_template_pairs}; + + let template = find_template("dart", "simple-live").expect("dart simple-live template"); + let schema = template + .files + .iter() + .find(|file| file.project_path == "schema.sql") + .expect("schema.sql template"); + let rendered_schema = + render_template_pairs(schema.content, &[("project_name", "demo-app")]).unwrap(); + assert!(rendered_schema.contains("demo-app")); + assert!(rendered_schema.contains("CREATE TABLE users")); + + let pubspec = template + .files + .iter() + .find(|file| file.project_path == "pubspec.yaml") + .expect("pubspec.yaml template"); + assert!(pubspec.content.contains("kalam_sync")); + assert!(pubspec.content.contains("sdk: flutter")); + + let main = template + .files + .iter() + .find(|file| file.project_path == "lib/main.dart") + .expect("lib/main.dart template"); + let rendered_main = render_template_pairs( + main.content, + &[ + ("server_url", "http://localhost:2900"), + ("namespace", "demo_app"), + ("project_name", "demo-app"), + ], + ) + .unwrap(); + assert!(rendered_main.contains("http://localhost:2900")); + assert!(rendered_main.contains("demo_app")); + assert!(rendered_main.contains("KalamTables.users")); + assert!(rendered_main.contains("Kalam.open")); + } +} diff --git a/cli/src/workflow/project/dart/mod.rs b/cli/src/workflow/project/dart/mod.rs new file mode 100644 index 000000000..e0d611820 --- /dev/null +++ b/cli/src/workflow/project/dart/mod.rs @@ -0,0 +1,12 @@ +//! Dart/Flutter SDK scaffolding for `kalam init` and related workflow commands. + +pub mod init; +pub mod templates; + +pub use init::{ + is_enabled, maybe_bootstrap_flutter_project, resolve_starter, DEFAULT_DEV_COMMAND, + SCHEMA_TARGET_OUTPUT, +}; +pub use templates::{ + resolve as resolve_dart_template, DEFAULT_TEMPLATE as DEFAULT_DART_TEMPLATE, +}; diff --git a/cli/src/workflow/project/dart/templates.rs b/cli/src/workflow/project/dart/templates.rs new file mode 100644 index 000000000..432586e03 --- /dev/null +++ b/cli/src/workflow/project/dart/templates.rs @@ -0,0 +1,24 @@ +//! Built-in Dart/Flutter project templates bundled with the CLI. + +use crate::{ + error::{CLIError, Result}, + workflow::project::templates::{ + default_template_for_language, find_template, templates_for_language, EmbeddedTemplate, + }, +}; + +pub const DEFAULT_TEMPLATE: &str = "simple-live"; + +pub fn available() -> Vec<&'static EmbeddedTemplate> { + templates_for_language("dart") +} + +pub fn resolve(template_id: Option<&str>) -> Result<&'static EmbeddedTemplate> { + if let Some(template_id) = template_id { + find_template("dart", template_id).ok_or_else(|| { + CLIError::ConfigurationError(format!("unknown dart template '{template_id}'")) + }) + } else { + default_template_for_language("dart") + } +} diff --git a/cli/src/workflow/project/guidance.rs b/cli/src/workflow/project/guidance.rs index cbc464f08..744a86584 100644 --- a/cli/src/workflow/project/guidance.rs +++ b/cli/src/workflow/project/guidance.rs @@ -95,7 +95,7 @@ pub fn init_unsupported_language(language: &str) -> String { kalam init --yes --languages typescript\n\ kalam init --yes --languages typescript,dart", bullet_list(&[ - "Use typescript and/or dart", + "Use typescript and/or dart (flutter is accepted as an alias for dart)", "Aliases: ts is accepted for typescript", ]) ) diff --git a/cli/src/workflow/project/init/mod.rs b/cli/src/workflow/project/init/mod.rs index 04a2196b6..36cc06bbf 100644 --- a/cli/src/workflow/project/init/mod.rs +++ b/cli/src/workflow/project/init/mod.rs @@ -19,6 +19,7 @@ use crate::{ guidance::{ init_config_validation_failed, init_project_already_exists, init_stage_context, }, + dart::{self, DEFAULT_DEV_COMMAND as DART_DEV_COMMAND, SCHEMA_TARGET_OUTPUT as DART_SCHEMA_TARGET_OUTPUT}, identifiers::{normalize_namespace_name, parse_namespace_id}, prompts::print_workflow_banner, repository_examples, @@ -79,6 +80,16 @@ where let languages = prompts::resolve_languages(&options, output.use_color)?; let starter = resolve_starter(&languages, options.template.as_deref(), options.yes, output.use_color)?; + let dart_template = if crate::workflow::project::ts::is_enabled(&languages) { + dart::resolve_starter(&languages, None, true, output.use_color)? + } else { + dart::resolve_starter( + &languages, + options.template.as_deref(), + options.yes, + output.use_color, + )? + }; let package_manager = resolve_package_manager(&languages, options.package_manager, options.yes, output)?; if let Some(ProjectStarter::Repository(example)) = starter { @@ -119,11 +130,21 @@ where server_mode, &server_url, typescript_template, + dart_template, output, ) .map_err(|error| map_init_stage_error("writing project files", error))?; install_dependencies(&options.cwd, package_manager, output, &mut installer) .map_err(|error| map_init_stage_error("installing JavaScript dependencies", error))?; + if dart::is_enabled(&languages) { + let package_name = config + .connection + .get("dev") + .map(|connection| connection.namespace.as_str().to_string()) + .unwrap_or_else(|| normalize_namespace_name(&name)); + dart::maybe_bootstrap_flutter_project(&options.cwd, &package_name, output) + .map_err(|error| map_init_stage_error("bootstrapping Flutter project", error))?; + } output.status(format!("initialized KalamDB project '{name}'")); Ok(()) } @@ -162,7 +183,7 @@ fn build_config( for language in languages { let output = match language.as_str() { "typescript" => SCHEMA_TARGET_OUTPUT, - "dart" => "lib/generated/kalam.dart", + "dart" => DART_SCHEMA_TARGET_OUTPUT, _ => continue, }; targets.insert( @@ -200,9 +221,13 @@ fn build_config( }, dev: DevSection { auto_start_db: matches!(server_mode, ServerMode::Local), - processes: package_manager - .map(|manager| HashMap::from([("app".into(), manager.dev_run_command().into())])) - .unwrap_or_default(), + processes: if let Some(manager) = package_manager { + HashMap::from([("app".into(), manager.dev_run_command().into())]) + } else if languages.iter().any(|language| language == "dart") { + HashMap::from([("app".into(), DART_DEV_COMMAND.into())]) + } else { + HashMap::new() + }, ..DevSection::default() }, logging: LoggingSection::default(), @@ -313,11 +338,20 @@ mod tests { let kalam_toml = fs::read_to_string(temp.path().join(KALAM_TOML)).unwrap(); assert!(!kalam_toml.contains("package_manager")); - assert!(kalam_toml.contains("# [dev.processes]")); - assert!(!kalam_toml.contains("\n[dev.processes]\n")); + assert!(kalam_toml.contains("[dev.processes]")); + assert!(kalam_toml.contains("app = \"flutter run\"")); let config = KalamProjectConfig::load_from_path(&temp.path().join(KALAM_TOML)).unwrap(); assert!(config.project.package_manager.is_none()); + assert_eq!(config.dev.processes.get("app").map(String::as_str), Some("flutter run")); + + assert!(temp.path().join("pubspec.yaml").is_file()); + assert!(temp.path().join("lib/main.dart").is_file()); + assert!(temp.path().join("schema.sql").is_file()); + let generated = fs::read_to_string(temp.path().join("lib/generated/kalam.dart")).unwrap(); + assert!(generated.contains("KalamTableSpec")); + assert!(generated.contains("tableId: 'users'")); + assert!(!generated.to_lowercase().contains("placeholder")); }); } diff --git a/cli/src/workflow/project/init/prompts.rs b/cli/src/workflow/project/init/prompts.rs index 7ab46901a..5da005436 100644 --- a/cli/src/workflow/project/init/prompts.rs +++ b/cli/src/workflow/project/init/prompts.rs @@ -83,7 +83,10 @@ pub(super) fn resolve_languages(options: &InitOptions, color: bool) -> Result Result> { for language in languages { match language.trim().to_ascii_lowercase().as_str() { "typescript" | "ts" => normalized.push("typescript".into()), - "dart" => normalized.push("dart".into()), + "dart" | "flutter" => normalized.push("dart".into()), other => { return Err(CLIError::ConfigurationError(init_unsupported_language(other))); }, diff --git a/cli/src/workflow/project/init/write.rs b/cli/src/workflow/project/init/write.rs index 49eddd6f8..73e72d201 100644 --- a/cli/src/workflow/project/init/write.rs +++ b/cli/src/workflow/project/init/write.rs @@ -30,6 +30,7 @@ pub(super) fn write_project_scaffold( server_mode: ServerMode, server_url: &str, typescript_template: Option<&EmbeddedTemplate>, + dart_template: Option<&EmbeddedTemplate>, output: &WorkflowOutput, ) -> Result<()> { scaffold::io_with_guidance("create project directory", root, fs::create_dir_all(root))?; @@ -50,7 +51,24 @@ pub(super) fn write_project_scaffold( .map(|connection| connection.namespace.as_str()) .unwrap_or(""); apply_scaffold(root, template, &config.project.name, server_url, namespace, output)?; - } else if matches!(schema_mode, SchemaMode::Sql) { + } + if let Some(template) = dart_template { + let namespace = config + .connection + .get("dev") + .map(|connection| connection.namespace.as_str()) + .unwrap_or(""); + crate::workflow::project::dart::init::apply_dart_scaffold( + root, + template, + &config.project.name, + server_url, + namespace, + output, + )?; + } + if typescript_template.is_none() && dart_template.is_none() && matches!(schema_mode, SchemaMode::Sql) + { let schema_path = root.join("schema.sql"); if !schema_path.exists() { scaffold::io_with_guidance( @@ -105,6 +123,26 @@ pub(super) fn write_project_scaffold( } } + if config.schema.languages.iter().any(|language| language == "dart") { + if let Some(target) = config.schema.targets.get("dart") { + match crate::workflow::schema::load::load_schema_snapshot(root, config) { + Ok(snapshot) => { + let dart_path = root.join(&target.output); + crate::workflow::schema::dart::write_dart_schema(&dart_path, &snapshot)?; + output.detail(format!( + "generated {}", + display_project_path(root, &dart_path) + )); + }, + Err(error) => { + output.detail(format!( + "skipped Dart schema generation during init: {error}" + )); + }, + } + } + } + let default_profile = crate::workflow::project::resolve::credential_instance_for_env(&config.project.default_env); let namespace = config diff --git a/cli/src/workflow/project/mod.rs b/cli/src/workflow/project/mod.rs index 5103f90df..bdbbed6ff 100644 --- a/cli/src/workflow/project/mod.rs +++ b/cli/src/workflow/project/mod.rs @@ -1,5 +1,6 @@ pub mod config; pub mod connection_url; +pub mod dart; pub mod guidance; pub mod identifiers; pub mod init; diff --git a/cli/src/workflow/project/templates.rs b/cli/src/workflow/project/templates.rs index a866a3955..359f0f20a 100644 --- a/cli/src/workflow/project/templates.rs +++ b/cli/src/workflow/project/templates.rs @@ -91,7 +91,7 @@ fn template_string_escape_for_path(project_path: &str) -> TemplateStringEscape { if project_path.ends_with(".json") { return TemplateStringEscape::DoubleQuoted; } - if project_path.ends_with(".ts") || project_path.ends_with(".tsx") { + if project_path.ends_with(".ts") || project_path.ends_with(".tsx") || project_path.ends_with(".dart") { return TemplateStringEscape::JsSingleQuoted; } TemplateStringEscape::None @@ -294,17 +294,17 @@ mod tests { false, true, true, - None, + Some("flutter run"), ); assert!(!dart_only.contains("[schema.targets.typescript]")); assert!(dart_only.contains("[schema.targets.dart]")); - assert!(dart_only.contains("# [dev.processes]")); - assert!(dart_only.contains("# app = \"npm run dev\"")); - assert!(!dart_only.contains("\n[dev.processes]\n")); + assert!(dart_only.contains("[dev.processes]")); + assert!(dart_only.contains("app = \"flutter run\"")); + assert!(!dart_only.contains("# [dev.processes]")); let parsed = KalamProjectConfig::parse(&dart_only).expect("parse dart kalam.toml"); assert!(!parsed.schema.targets.contains_key("typescript")); assert!(parsed.schema.targets.contains_key("dart")); - assert!(parsed.dev.processes.is_empty()); + assert_eq!(parsed.dev.processes.get("app").map(String::as_str), Some("flutter run")); let both = render_scaffold_kalam_toml( kalam_toml, diff --git a/cli/src/workflow/project/ts/mod.rs b/cli/src/workflow/project/ts/mod.rs index 70af5a49b..f0092036a 100644 --- a/cli/src/workflow/project/ts/mod.rs +++ b/cli/src/workflow/project/ts/mod.rs @@ -1,6 +1,4 @@ //! TypeScript SDK scaffolding for `kalam init` and related workflow commands. -//! -//! Dart and Rust SDK helpers will live in sibling modules (`dart/`, `rust/`) as they are added. pub mod guidance; pub mod init; diff --git a/cli/src/workflow/schema/dart.rs b/cli/src/workflow/schema/dart.rs new file mode 100644 index 000000000..8817dd2ba --- /dev/null +++ b/cli/src/workflow/schema/dart.rs @@ -0,0 +1,477 @@ +//! Dart/Flutter schema generation from the language-neutral SQL snapshot. + +use std::{ + collections::HashSet, + fmt::Write, + path::Path, +}; + +use crate::{ + error::{CLIError, Result}, + workflow::schema::model::{ColumnDefinition, SchemaSnapshot, TableDefinition, TableKind}, +}; + +const GENERATED_HEADER: &str = "// Generated by kalam schema gen. Do not edit.\n\ +// Row codecs and KalamTableSpec values for kalam_sync.\n\ +// Action queues stay with kalam_sync_generator / build_runner.\n"; + +pub fn write_dart_schema(output_path: &Path, snapshot: &SchemaSnapshot) -> Result<()> { + if let Some(parent) = output_path.parent() { + std::fs::create_dir_all(parent).map_err(|error| { + CLIError::FileError(format!("failed to create '{}': {error}", parent.display())) + })?; + } + let source = generate_dart_source(snapshot); + std::fs::write(output_path, source).map_err(|error| { + CLIError::FileError(format!("failed to write '{}': {error}", output_path.display())) + })?; + Ok(()) +} + +pub fn generate_dart_source(snapshot: &SchemaSnapshot) -> String { + let tables: Vec<&TableDefinition> = snapshot.tables.values().collect(); + let mut used_class_names = HashSet::new(); + let mut used_const_names = HashSet::new(); + let generated: Vec = tables + .iter() + .map(|table| GeneratedTable::from_definition(table, &mut used_class_names, &mut used_const_names)) + .collect(); + + let mut out = String::new(); + out.push_str(GENERATED_HEADER); + out.push_str("import 'package:kalam_sync/kalam_sync.dart';\n\n"); + + if generated.is_empty() { + out.push_str("abstract final class KalamTables {\n KalamTables._();\n}\n"); + return out; + } + + for table in &generated { + table.write_row_class(&mut out); + out.push('\n'); + } + + out.push_str("abstract final class KalamTables {\n KalamTables._();\n\n"); + for table in &generated { + table.write_spec(&mut out); + out.push('\n'); + } + out.push_str("}\n\n"); + out.push_str(DART_HELPERS); + out +} + +struct GeneratedTable { + table_id: String, + class_name: String, + const_name: String, + key_column: String, + mode: &'static str, + columns: Vec, +} + +struct GeneratedColumn { + sql_name: String, + field_name: String, + dart_type: &'static str, + nullable: bool, + is_key: bool, +} + +impl GeneratedTable { + fn from_definition( + table: &TableDefinition, + used_class_names: &mut HashSet, + used_const_names: &mut HashSet, + ) -> Self { + let last_segment = table_name_segment(&table.name); + let class_name = unique_ident(pascal_case(last_segment), used_class_names); + let const_name = unique_ident(camel_case(last_segment), used_const_names); + let key_column = key_column_name(&table.columns); + let columns = table + .columns + .iter() + .map(|column| { + let nullable = column.nullable && !column.primary_key; + GeneratedColumn { + sql_name: column.name.clone(), + field_name: dart_field_name(&column.name), + dart_type: dart_type_for_sql(&column.sql_type), + nullable, + is_key: key_column.as_deref() == Some(column.name.as_str()), + } + }) + .collect(); + + Self { + table_id: table.name.clone(), + class_name, + const_name, + key_column: key_column.unwrap_or_else(|| "id".to_string()), + mode: sync_mode(table.kind), + columns, + } + } + + fn write_row_class(&self, out: &mut String) { + let _ = writeln!(out, "final class {} {{", self.class_name); + out.push_str(" const "); + out.push_str(&self.class_name); + out.push_str("({\n"); + for column in &self.columns { + if column.nullable { + let _ = writeln!(out, " this.{},", column.field_name); + } else { + let _ = writeln!(out, " required this.{},", column.field_name); + } + } + out.push_str(" });\n\n"); + + for column in &self.columns { + let type_name = if column.nullable { + format!("{}?", column.dart_type) + } else { + column.dart_type.to_string() + }; + let _ = writeln!(out, " final {type_name} {};", column.field_name); + } + + out.push_str("\n String get kalamRowKey => "); + let key_expr = self + .columns + .iter() + .find(|column| column.is_key) + .or_else(|| self.columns.first()); + match key_expr { + Some(key) if key.dart_type == "String" && !key.nullable => { + out.push_str(&key.field_name); + }, + Some(key) => { + let _ = write!(out, "{}.toString()", key.field_name); + }, + None => out.push_str("''"), + } + out.push_str(";\n\n"); + + out.push_str(" Map toJson() => {\n"); + for column in &self.columns { + let encoded = encode_expression(column); + let _ = writeln!(out, " '{}': {encoded},", column.sql_name); + } + out.push_str(" };\n\n"); + + let _ = writeln!(out, " factory {}.fromJson(Map json) => {}(", self.class_name, self.class_name); + for column in &self.columns { + let decoded = decode_expression(column); + let _ = writeln!(out, " {}: {decoded},", column.field_name); + } + out.push_str(" );\n}\n"); + } + + fn write_spec(&self, out: &mut String) { + let _ = writeln!( + out, + " static final {const_name} = KalamTableSpec<{class_name}>(", + const_name = self.const_name, + class_name = self.class_name + ); + let _ = writeln!(out, " tableId: '{}',", escape_dart_string(&self.table_id)); + let _ = writeln!(out, " keyColumn: '{}',", escape_dart_string(&self.key_column)); + let _ = writeln!(out, " mode: KalamSyncMode.{},", self.mode); + out.push_str(" keyOf: (row) => row.kalamRowKey,\n"); + out.push_str(" encode: (row) => row.toJson(),\n"); + let _ = writeln!(out, " decode: {}.fromJson,", self.class_name); + out.push_str(" );\n"); + } +} + +fn table_name_segment(name: &str) -> &str { + name.rsplit('.').next().filter(|part| !part.is_empty()).unwrap_or(name) +} + +fn key_column_name(columns: &[ColumnDefinition]) -> Option { + columns + .iter() + .find(|column| column.primary_key) + .or_else(|| columns.iter().find(|column| column.name.eq_ignore_ascii_case("id"))) + .map(|column| column.name.clone()) +} + +fn sync_mode(kind: TableKind) -> &'static str { + match kind { + TableKind::Stream => "replicaOnly", + TableKind::Unspecified | TableKind::User | TableKind::Shared => "bidirectional", + } +} + +fn dart_type_for_sql(sql_type: &str) -> &'static str { + let normalized = sql_type.trim().trim_end_matches(',').to_ascii_uppercase(); + let base = normalized.split('(').next().unwrap_or(&normalized); + match base { + "INT" | "INTEGER" | "SMALLINT" | "BIGINT" | "INT2" | "INT4" | "INT8" | "INT64" + | "UINT64" | "SERIAL" | "BIGSERIAL" => "int", + "FLOAT" | "FLOAT4" | "FLOAT8" | "DOUBLE" | "REAL" | "NUMERIC" | "DECIMAL" => "double", + "BOOLEAN" | "BOOL" => "bool", + "TIMESTAMP" | "TIMESTAMPTZ" | "DATETIME" | "DATE" | "TIME" => "DateTime", + "JSON" | "JSONB" => "Map", + "BYTES" | "BYTEA" | "BLOB" | "BINARY" => "String", + "EMBEDDING" => "List", + _ => "String", + } +} + +fn dart_field_name(sql_name: &str) -> String { + sanitize_ident(&camel_case(sql_name)) +} + +fn pascal_case(value: &str) -> String { + let mut name = String::new(); + let mut capitalize = true; + for ch in value.chars() { + if ch == '_' || ch == '-' || ch == '.' { + capitalize = true; + continue; + } + if capitalize { + for upper in ch.to_uppercase() { + name.push(upper); + } + capitalize = false; + } else { + name.push(ch); + } + } + sanitize_ident(&name) +} + +fn camel_case(value: &str) -> String { + let pascal = pascal_case(value); + let mut chars = pascal.chars(); + match chars.next() { + Some(first) => first.to_lowercase().collect::() + chars.as_str(), + None => "value".to_string(), + } +} + +fn sanitize_ident(value: &str) -> String { + let mut ident: String = value + .chars() + .map(|ch| if ch.is_ascii_alphanumeric() || ch == '_' { ch } else { '_' }) + .collect(); + if ident.is_empty() || ident.chars().next().is_some_and(|ch| ch.is_ascii_digit()) { + ident.insert(0, 'n'); + } + if DART_KEYWORDS.contains(&ident.as_str()) { + ident.push('_'); + } + ident +} + +fn unique_ident(base: String, used: &mut HashSet) -> String { + if used.insert(base.clone()) { + return base; + } + let mut index = 2u32; + loop { + let candidate = format!("{base}{index}"); + if used.insert(candidate.clone()) { + return candidate; + } + index += 1; + } +} + +fn encode_expression(column: &GeneratedColumn) -> String { + let name = &column.field_name; + match (column.dart_type, column.nullable) { + ("DateTime", true) => format!("{name}?.toIso8601String()"), + ("DateTime", false) => format!("{name}.toIso8601String()"), + _ => name.to_string(), + } +} + +fn decode_expression(column: &GeneratedColumn) -> String { + let json = format!("json['{}']", escape_dart_string(&column.sql_name)); + match (column.dart_type, column.nullable) { + ("int", false) => format!("_asInt({json})"), + ("int", true) => format!("_asIntOrNull({json})"), + ("double", false) => format!("_asDouble({json})"), + ("double", true) => format!("_asDoubleOrNull({json})"), + ("bool", false) => format!("_asBool({json})"), + ("bool", true) => format!("_asBoolOrNull({json})"), + ("DateTime", false) => format!("_asDateTime({json})"), + ("DateTime", true) => format!("_asDateTimeOrNull({json})"), + ("Map", false) => format!("_asJsonMap({json})"), + ("Map", true) => format!("_asJsonMapOrNull({json})"), + ("List", false) => format!("_asDoubleList({json})"), + ("List", true) => format!("_asDoubleListOrNull({json})"), + (_, false) => format!("_asString({json})"), + (_, true) => format!("_asStringOrNull({json})"), + } +} + +fn escape_dart_string(value: &str) -> String { + value.replace('\\', "\\\\").replace('\'', "\\'") +} + +const DART_KEYWORDS: &[&str] = &[ + "assert", "break", "case", "catch", "class", "const", "continue", "default", "do", "else", + "enum", "extends", "false", "final", "finally", "for", "if", "in", "is", "new", "null", + "rethrow", "return", "super", "switch", "this", "throw", "true", "try", "var", "void", + "while", "with", "yield", +]; + +const DART_HELPERS: &str = r#"int _asInt(Object? value) { + if (value is int) return value; + if (value is num) return value.toInt(); + if (value is String) return int.parse(value); + throw FormatException('expected int, got $value'); +} + +int? _asIntOrNull(Object? value) => value == null ? null : _asInt(value); + +double _asDouble(Object? value) { + if (value is double) return value; + if (value is num) return value.toDouble(); + if (value is String) return double.parse(value); + throw FormatException('expected double, got $value'); +} + +double? _asDoubleOrNull(Object? value) => value == null ? null : _asDouble(value); + +bool _asBool(Object? value) { + if (value is bool) return value; + if (value is num) return value != 0; + if (value is String) { + switch (value.toLowerCase()) { + case 'true': + case '1': + return true; + case 'false': + case '0': + return false; + } + } + throw FormatException('expected bool, got $value'); +} + +bool? _asBoolOrNull(Object? value) => value == null ? null : _asBool(value); + +DateTime _asDateTime(Object? value) { + if (value is DateTime) return value; + if (value is String) return DateTime.parse(value); + if (value is int) return DateTime.fromMillisecondsSinceEpoch(value, isUtc: true); + throw FormatException('expected DateTime, got $value'); +} + +DateTime? _asDateTimeOrNull(Object? value) => + value == null ? null : _asDateTime(value); + +String _asString(Object? value) { + if (value is String) return value; + if (value == null) throw FormatException('expected String, got null'); + return value.toString(); +} + +String? _asStringOrNull(Object? value) => value == null ? null : _asString(value); + +Map _asJsonMap(Object? value) { + if (value is Map) return value; + if (value is Map) { + return value.map((key, nested) => MapEntry(key.toString(), nested)); + } + throw FormatException('expected JSON object, got $value'); +} + +Map? _asJsonMapOrNull(Object? value) => + value == null ? null : _asJsonMap(value); + +List _asDoubleList(Object? value) { + if (value is List) return value; + if (value is List) { + return value.map(_asDouble).toList(); + } + throw FormatException('expected List, got $value'); +} + +List? _asDoubleListOrNull(Object? value) => + value == null ? null : _asDoubleList(value); +"#; + +#[cfg(test)] +mod tests { + use super::*; + use crate::workflow::schema::load::parse_sql_schema; + + #[test] + fn generates_table_specs_and_row_codecs_from_sql() { + let snapshot = parse_sql_schema( + r#" +CREATE TABLE users ( + id INTEGER PRIMARY KEY, + email TEXT NOT NULL, + created_at TIMESTAMP +); +"#, + ) + .unwrap(); + let source = generate_dart_source(&snapshot); + assert!(source.contains("Generated by kalam schema gen")); + assert!(!source.to_lowercase().contains("placeholder")); + assert!(source.contains("import 'package:kalam_sync/kalam_sync.dart';")); + assert!(source.contains("final class Users {")); + assert!(source.contains("required this.id")); + assert!(source.contains("required this.email")); + assert!(source.contains("this.createdAt")); + assert!(source.contains("factory Users.fromJson")); + assert!(source.contains("static final users = KalamTableSpec(")); + assert!(source.contains("tableId: 'users'")); + assert!(source.contains("keyColumn: 'id'")); + assert!(source.contains("mode: KalamSyncMode.bidirectional")); + assert!(source.contains("keyOf: (row) => row.kalamRowKey")); + assert!(source.contains("encode: (row) => row.toJson()")); + assert!(source.contains("decode: Users.fromJson")); + } + + #[test] + fn stream_tables_use_replica_only_mode() { + let snapshot = parse_sql_schema( + r#" +CREATE STREAM TABLE app.message_events ( + id TEXT PRIMARY KEY, + payload JSON +); +"#, + ) + .unwrap(); + let source = generate_dart_source(&snapshot); + assert!(source.contains("tableId: 'app.message_events'")); + assert!(source.contains("final class MessageEvents {")); + assert!(source.contains("static final messageEvents = KalamTableSpec(")); + assert!(source.contains("mode: KalamSyncMode.replicaOnly")); + assert!(source.contains("Map? payload")); + } + + #[test] + fn empty_schema_emits_compilable_kalam_tables() { + let snapshot = parse_sql_schema("-- no tables yet\n").unwrap(); + let source = generate_dart_source(&snapshot); + assert!(source.contains("abstract final class KalamTables")); + assert!(!source.contains("KalamTableSpec<")); + assert!(!source.contains("static final")); + } + + #[test] + fn write_dart_schema_creates_parent_directories() { + let temp = tempfile::TempDir::new().unwrap(); + let output = temp.path().join("lib/generated/kalam.dart"); + let snapshot = parse_sql_schema( + "CREATE TABLE todos (id TEXT PRIMARY KEY, title TEXT NOT NULL, done BOOLEAN NOT NULL);", + ) + .unwrap(); + write_dart_schema(&output, &snapshot).unwrap(); + let source = std::fs::read_to_string(&output).unwrap(); + assert!(source.contains("KalamTableSpec")); + assert!(source.contains("bool done")); + } +} diff --git a/cli/src/workflow/schema/gen.rs b/cli/src/workflow/schema/gen.rs index 02e695b21..7caaf5e56 100644 --- a/cli/src/workflow/schema/gen.rs +++ b/cli/src/workflow/schema/gen.rs @@ -58,7 +58,13 @@ pub fn generate_schema_artifacts( resolved_environment.as_ref().expect("resolved environment initialized"); generate_typescript_via_orm(ctx, environment, &output_path)?; }, - LanguageTarget::Dart => write_dart_placeholder(&output_path)?, + LanguageTarget::Dart => { + let snapshot = crate::workflow::schema::load::load_schema_snapshot( + &ctx.project_root, + &ctx.config, + )?; + crate::workflow::schema::dart::write_dart_schema(&output_path, &snapshot)?; + }, } output.status(format!("generated {} -> {}", key, target.output)); } @@ -133,20 +139,6 @@ fn generate_typescript_via_orm( ))) } -fn write_dart_placeholder(output_path: &Path) -> Result<()> { - if let Some(parent) = output_path.parent() { - std::fs::create_dir_all(parent)?; - } - std::fs::write( - output_path, - "// Placeholder generated by kalam schema gen.\n// Dart schema generation via a dedicated package will be added later.\n", - ) - .map_err(|error| { - CLIError::FileError(format!("failed to write '{}': {error}", output_path.display())) - })?; - Ok(()) -} - fn node_codegen_args() -> &'static [&'static str] { &["--preserve-symlinks", "--input-type=module"] } @@ -172,19 +164,24 @@ pub fn validate_language_filter( requested: &[String], configured: &[String], ) -> Result> { + let mut normalized = Vec::new(); for value in requested { - if LanguageTarget::parse(value).is_none() { + let Some(parsed) = LanguageTarget::parse(value) else { return Err(CLIError::ConfigurationError(format!( "unsupported language target '{value}'; supported: typescript, dart" ))); - } - if !configured.iter().any(|lang| lang == value) { + }; + let canonical = parsed.as_str().to_string(); + if !configured.iter().any(|lang| LanguageTarget::parse(lang) == Some(parsed)) { return Err(CLIError::ConfigurationError(format!( - "language '{value}' is not enabled in kalam.toml schema.languages" + "language '{canonical}' is not enabled in kalam.toml schema.languages" ))); } + if !normalized.iter().any(|lang| lang == &canonical) { + normalized.push(canonical); + } } - Ok(requested.to_vec()) + Ok(normalized) } #[cfg(test)] @@ -195,4 +192,11 @@ mod tests { fn node_codegen_args_preserve_project_symlink_resolution() { assert!(node_codegen_args().contains(&"--preserve-symlinks")); } + + #[test] + fn validate_language_filter_accepts_flutter_alias() { + let filtered = + validate_language_filter(&["flutter".into()], &["dart".into()]).expect("filter"); + assert_eq!(filtered, vec!["dart"]); + } } diff --git a/cli/src/workflow/schema/load.rs b/cli/src/workflow/schema/load.rs index 3b15a4948..acd8c2084 100644 --- a/cli/src/workflow/schema/load.rs +++ b/cli/src/workflow/schema/load.rs @@ -14,7 +14,7 @@ use crate::{ config::{KalamProjectConfig, SchemaMode}, identifiers::{parse_table_name, parse_table_ref}, }, - schema::model::{ColumnDefinition, SchemaOrigin, SchemaSnapshot, TableDefinition}, + schema::model::{ColumnDefinition, SchemaOrigin, SchemaSnapshot, TableDefinition, TableKind}, }, }; @@ -91,35 +91,71 @@ fn normalize_sql(sql: &str) -> String { } fn split_create_table_statements(sql: &str) -> Vec { - sql.split("CREATE TABLE") - .skip(1) - .map(|rest| { - let mut segment = format!("CREATE TABLE{rest}"); - if let Some(end) = segment.find(';') { - segment.truncate(end); - } - segment - }) - .collect() + let upper = sql.to_ascii_uppercase(); + let mut starts = Vec::new(); + let mut search_from = 0usize; + while let Some(rel) = upper[search_from..].find("CREATE ") { + let abs = search_from + rel; + let after_create = upper[abs + "CREATE ".len()..].trim_start(); + if after_create.starts_with("TABLE") + || after_create.starts_with("USER TABLE") + || after_create.starts_with("SHARED TABLE") + || after_create.starts_with("STREAM TABLE") + { + starts.push(abs); + } + search_from = abs + "CREATE ".len(); + } + + let mut statements = Vec::with_capacity(starts.len()); + for (index, start) in starts.iter().enumerate() { + let end = starts.get(index + 1).copied().unwrap_or(sql.len()); + let mut segment = sql[*start..end].trim().to_string(); + if let Some(semi) = segment.find(';') { + segment.truncate(semi); + } + if !segment.is_empty() { + statements.push(segment); + } + } + statements } fn parse_create_table(statement: &str) -> Result> { - let upper = statement.to_ascii_uppercase(); - if !upper.starts_with("CREATE TABLE") { + let Some((kind, rest)) = strip_create_table_prefix(statement) else { return Ok(None); - } - - let rest = statement - .trim() - .strip_prefix("CREATE TABLE") - .or_else(|| statement.trim().strip_prefix("create table")) - .unwrap_or(statement) - .trim(); + }; let (name, body) = parse_table_name_and_body(rest)?; let columns = parse_columns(&body)?; - Ok(Some(TableDefinition { name, columns })) + Ok(Some(TableDefinition { name, kind, columns })) +} + +fn strip_create_table_prefix(statement: &str) -> Option<(TableKind, &str)> { + let trimmed = statement.trim(); + if let Some(rest) = strip_ascii_prefix(trimmed, "CREATE USER TABLE") { + Some((TableKind::User, rest.trim())) + } else if let Some(rest) = strip_ascii_prefix(trimmed, "CREATE SHARED TABLE") { + Some((TableKind::Shared, rest.trim())) + } else if let Some(rest) = strip_ascii_prefix(trimmed, "CREATE STREAM TABLE") { + Some((TableKind::Stream, rest.trim())) + } else if let Some(rest) = strip_ascii_prefix(trimmed, "CREATE TABLE") { + Some((TableKind::Unspecified, rest.trim())) + } else { + None + } +} + +fn strip_ascii_prefix<'a>(value: &'a str, prefix: &str) -> Option<&'a str> { + if value.len() < prefix.len() { + return None; + } + if value.get(..prefix.len())?.eq_ignore_ascii_case(prefix) { + Some(&value[prefix.len()..]) + } else { + None + } } fn parse_table_name_and_body(rest: &str) -> Result<(String, String)> { @@ -127,7 +163,10 @@ fn parse_table_name_and_body(rest: &str) -> Result<(String, String)> { let open_paren = rest .find('(') .ok_or_else(|| CLIError::ParseError("expected '(' after table name".into()))?; - let name_part = rest[..open_paren].trim(); + let mut name_part = rest[..open_paren].trim(); + if let Some(stripped) = strip_ascii_prefix(name_part, "IF NOT EXISTS") { + name_part = stripped.trim(); + } let name = name_part.trim_matches('"').trim_matches('`').trim_matches('\'').to_string(); validate_parsed_table_name(&name)?; @@ -149,14 +188,18 @@ fn validate_parsed_table_name(name: &str) -> Result<()> { fn parse_columns(body: &str) -> Result> { let mut columns = Vec::new(); + let mut table_primary_keys = Vec::new(); for part in split_column_parts(body) { let trimmed = part.trim(); if trimmed.is_empty() { continue; } let upper = trimmed.to_ascii_uppercase(); - if upper.starts_with("PRIMARY KEY") - || upper.starts_with("FOREIGN KEY") + if upper.starts_with("PRIMARY KEY") { + table_primary_keys.extend(parse_primary_key_columns(trimmed)); + continue; + } + if upper.starts_with("FOREIGN KEY") || upper.starts_with("UNIQUE") || upper.starts_with("CONSTRAINT") { @@ -181,9 +224,35 @@ fn parse_columns(body: &str) -> Result> { primary_key, }); } + + if !table_primary_keys.is_empty() { + for column in &mut columns { + if table_primary_keys.iter().any(|name| name.eq_ignore_ascii_case(&column.name)) { + column.primary_key = true; + column.nullable = false; + } + } + } Ok(columns) } +fn parse_primary_key_columns(constraint: &str) -> Vec { + let Some(open) = constraint.find('(') else { + return Vec::new(); + }; + let Some(close) = constraint.rfind(')') else { + return Vec::new(); + }; + if close <= open { + return Vec::new(); + } + constraint[open + 1..close] + .split(',') + .map(|part| part.trim().trim_matches('"').trim_matches('`').to_string()) + .filter(|name| !name.is_empty()) + .collect() +} + fn split_column_parts(body: &str) -> Vec { let mut parts = Vec::new(); let mut current = String::new(); @@ -231,5 +300,30 @@ CREATE TABLE users ( assert_eq!(users.columns.len(), 2); assert_eq!(users.columns[0].name, "id"); assert!(!users.columns[1].nullable); + assert!(users.columns[0].primary_key); + assert_eq!(users.kind, TableKind::Unspecified); + } + + #[test] + fn parse_create_user_and_stream_tables() { + let sql = r#" +CREATE USER TABLE IF NOT EXISTS app.todos ( + id TEXT PRIMARY KEY, + title TEXT NOT NULL +); +CREATE STREAM TABLE app.events ( + id BIGINT NOT NULL, + payload JSON, + PRIMARY KEY (id) +); +"#; + let snapshot = parse_sql_schema(sql).unwrap(); + let todos = snapshot.tables.get("app.todos").unwrap(); + assert_eq!(todos.kind, TableKind::User); + assert!(todos.columns[0].primary_key); + + let events = snapshot.tables.get("app.events").unwrap(); + assert_eq!(events.kind, TableKind::Stream); + assert!(events.columns.iter().any(|column| column.name == "id" && column.primary_key)); } } diff --git a/cli/src/workflow/schema/mod.rs b/cli/src/workflow/schema/mod.rs index 82da6f3f3..940f2a65d 100644 --- a/cli/src/workflow/schema/mod.rs +++ b/cli/src/workflow/schema/mod.rs @@ -1,3 +1,4 @@ +pub mod dart; pub mod diff; pub mod gen; pub mod load; @@ -6,4 +7,4 @@ pub mod model; pub use diff::diff_project_schema_files; pub use gen::{generate_schema_artifacts, GenerateOptions}; pub use load::{load_schema_snapshot, parse_sql_schema, pull_remote_schema}; -pub use model::{LanguageTarget, SchemaSnapshot}; +pub use model::{LanguageTarget, SchemaSnapshot, TableKind}; diff --git a/cli/src/workflow/schema/model.rs b/cli/src/workflow/schema/model.rs index c87c3078d..caa1fa7b3 100644 --- a/cli/src/workflow/schema/model.rs +++ b/cli/src/workflow/schema/model.rs @@ -19,9 +19,21 @@ pub enum SchemaOrigin { Remote, } +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)] +#[serde(rename_all = "lowercase")] +pub enum TableKind { + #[default] + Unspecified, + User, + Shared, + Stream, +} + #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct TableDefinition { pub name: String, + #[serde(default)] + pub kind: TableKind, pub columns: Vec, } @@ -58,7 +70,7 @@ impl LanguageTarget { pub fn parse(value: &str) -> Option { match value.trim().to_ascii_lowercase().as_str() { "typescript" | "ts" => Some(Self::TypeScript), - "dart" => Some(Self::Dart), + "dart" | "flutter" => Some(Self::Dart), _ => None, } } @@ -83,5 +95,6 @@ mod tests { fn parse_language_aliases() { assert_eq!(LanguageTarget::parse("ts"), Some(LanguageTarget::TypeScript)); assert_eq!(LanguageTarget::parse("dart"), Some(LanguageTarget::Dart)); + assert_eq!(LanguageTarget::parse("flutter"), Some(LanguageTarget::Dart)); } } diff --git a/cli/templates/dart/simple-live/analysis_options.yaml.template b/cli/templates/dart/simple-live/analysis_options.yaml.template new file mode 100644 index 000000000..f9b303465 --- /dev/null +++ b/cli/templates/dart/simple-live/analysis_options.yaml.template @@ -0,0 +1 @@ +include: package:flutter_lints/flutter.yaml diff --git a/cli/templates/dart/simple-live/info.toml b/cli/templates/dart/simple-live/info.toml new file mode 100644 index 000000000..d9771b6e7 --- /dev/null +++ b/cli/templates/dart/simple-live/info.toml @@ -0,0 +1,3 @@ +description = "Flutter local-first starter with kalam_sync" +version = "1.0.0" +author = "KalamDB" diff --git a/cli/templates/dart/simple-live/lib/main.dart.template b/cli/templates/dart/simple-live/lib/main.dart.template new file mode 100644 index 000000000..38b5d1a09 --- /dev/null +++ b/cli/templates/dart/simple-live/lib/main.dart.template @@ -0,0 +1,86 @@ +import 'dart:async'; + +import 'package:flutter/material.dart'; +import 'package:kalam_sync/kalam_sync.dart'; + +import 'generated/kalam.dart'; + +Future main() async { + WidgetsFlutterBinding.ensureInitialized(); + final kalam = await Kalam.open( + url: '{{server_url}}', + namespace: '{{namespace}}', + subject: 'dev-user', + authProvider: () async => Auth.basic( + const String.fromEnvironment('KALAM_USER', defaultValue: 'root'), + const String.fromEnvironment('KALAM_PASSWORD', defaultValue: 'kalamdb123'), + ), + ); + runApp(KalamScope(kalam: kalam, child: UsersApp(kalam: kalam))); +} + +final class UsersApp extends StatefulWidget { + const UsersApp({required this.kalam, super.key}); + + final Kalam kalam; + + @override + State createState() => _UsersAppState(); +} + +final class _UsersAppState extends State { + late final KalamTableBinding users = widget.kalam.table(KalamTables.users); + KalamSyncSubscription? subscription; + + @override + void initState() { + super.initState(); + widget.kalam + .subscribe(users.consumer(sql: 'SELECT * FROM users')) + .then((value) => subscription = value); + } + + @override + void dispose() { + unawaited(subscription?.cancel()); + super.dispose(); + } + + @override + Widget build(BuildContext context) { + return MaterialApp( + home: Scaffold( + appBar: AppBar(title: const Text('{{project_name}} users')), + body: StreamBuilder>>( + stream: users.watchWithSyncState(), + builder: (context, snapshot) => ListView( + children: [ + for (final row in snapshot.data ?? const >[]) + ListTile( + title: Text(row.value.email), + subtitle: Text('id ${row.value.id}'), + trailing: Icon( + row.isSynced ? Icons.cloud_done : Icons.cloud_upload, + ), + ), + ], + ), + ), + floatingActionButton: FloatingActionButton( + onPressed: () { + final id = DateTime.now().millisecondsSinceEpoch; + users.insert( + Users( + id: id, + email: 'user$id@example.com', + createdAt: DateTime.now().toUtc(), + ), + actionId: Kalam.id(), + ); + }, + child: const Icon(Icons.add), + ), + ), + ); + } +} diff --git a/cli/templates/dart/simple-live/pubspec.yaml.template b/cli/templates/dart/simple-live/pubspec.yaml.template new file mode 100644 index 000000000..8c04d2dfe --- /dev/null +++ b/cli/templates/dart/simple-live/pubspec.yaml.template @@ -0,0 +1,21 @@ +name: {{namespace}} +description: Flutter starter for {{project_name}} +publish_to: "none" +version: 0.1.0 + +environment: + sdk: ">=3.10.0 <4.0.0" + flutter: ">=3.38.0" + +dependencies: + flutter: + sdk: flutter + kalam_sync: ">=0.5.6-0 <0.6.0" + +dev_dependencies: + flutter_lints: ^6.0.0 + flutter_test: + sdk: flutter + +flutter: + uses-material-design: true diff --git a/cli/templates/dart/simple-live/schema.sql.template b/cli/templates/dart/simple-live/schema.sql.template new file mode 100644 index 000000000..5b4a2113e --- /dev/null +++ b/cli/templates/dart/simple-live/schema.sql.template @@ -0,0 +1,6 @@ +-- Example schema for {{project_name}} +CREATE TABLE users ( + id INTEGER PRIMARY KEY, + email TEXT NOT NULL, + created_at TIMESTAMP +); diff --git a/docker/build/Dockerfile b/docker/build/Dockerfile index 8139fc3c6..eadc5899c 100644 --- a/docker/build/Dockerfile +++ b/docker/build/Dockerfile @@ -41,6 +41,8 @@ RUN set -eux; \ wasm-pack --version ENV PATH=/usr/local/cargo/bin:$PATH +ENV RUSTUP_TOOLCHAIN=1.92.0 +ENV CARGO_ENCODED_RUSTFLAGS= # Install wasm32 target for Rust RUN rustup target add wasm32-unknown-unknown diff --git a/docs/architecture/decisions/2026-08-16-kalam-sync-dart-package.md b/docs/architecture/decisions/2026-08-16-kalam-sync-dart-package.md new file mode 100644 index 000000000..1eb3aa8b4 --- /dev/null +++ b/docs/architecture/decisions/2026-08-16-kalam-sync-dart-package.md @@ -0,0 +1,66 @@ +# ADR: Add `kalam_sync` above `kalam_link` + +**Status:** Accepted for initial implementation +**Date:** 2026-08-16 + +## Decision + +Group the three Dart packages under `link/sdks/dart/`: + +- `link/sdks/dart/link/` (`kalam_link`) for the existing transport; +- `link/sdks/dart/sync/` (`kalam_sync`) for local rows, durable actions, + checkpoints, retries, reconciliation, lifecycle, and sync state; +- `link/sdks/dart/generator/` (`kalam_sync_generator`) for optional action + payload/definition/queue generation. + +`kalam_sync` depends on `kalam_link` and uses its existing shared HTTP and +WebSocket client. It does not create a second socket and it does not move or +vendor the Rust bridge. + +## Table policies + +Each table chooses one policy independently: + +- `bidirectional`: local insert/update/delete is visible immediately and one + generic DML action is durably queued; +- `replicaOnly`: backend rows are authoritative; a registered custom action + may atomically add an optimistic row or delete tombstone. + +Per-row sync metadata lives in a Kalam sidecar rather than being appended to +every server-shaped table. UI code receives `KalamSyncedRow`. + +## Ownership + +Drift owns generated application row types. Kalam owns only private persistence +models and generic runtime envelopes. Action payloads are small domain values +generated from annotations; they do not duplicate table rows. + +The local database identity includes server URL, namespace, and authenticated +subject. Account switching stops the old coordinator and opens a different +cache, preventing cross-account reads or outbox flushes. + +## Durability invariants + +1. Action enqueue and optimistic cached-row mutation commit together. +2. Server-row application and its sequence checkpoint commit together. +3. A subscription always starts from the last SQLite-committed checkpoint. +4. Action and named-step idempotency keys remain stable across retries and + process restarts. +5. Only one runner flushes one account's outbox at a time. + +## Deferred work + +The CLI Dart schema target now emits row classes plus `KalamTableSpec` values +from `schema.sql` (`kalam schema gen --languages dart`). Drift `Table` classes +remain deferred so generated files compile against `kalam_sync` alone. Schema +gen still does not emit per-table CRUD payload classes; those stay with +`kalam_sync_generator`. + +## Explicit acknowledgement + +`kalam_link` now provides an additive explicit-consumer-ack mode through +`liveEventsWithAck`. Automatic subscriptions retain their existing behavior. +The sync coordinator uses acknowledged batches, commits all decoded rows and +the batch checkpoint together, and only then advances transport progress. +Disconnect-during-apply resumes from the SQLite-committed checkpoint even when +the transport acknowledgement is lost. diff --git a/docs/getting-started/cli.md b/docs/getting-started/cli.md index c9bd9a671..6f4c56c12 100644 --- a/docs/getting-started/cli.md +++ b/docs/getting-started/cli.md @@ -605,8 +605,9 @@ This creates: ```bash # Regenerate workflow artifacts # TypeScript uses @kalamdb/orm against the resolved server/namespace. -# Dart currently writes a placeholder file. +# Dart reads schema.sql and writes KalamTableSpec codecs to lib/generated/kalam.dart. kalam schema gen +kalam schema gen --languages dart # Create a migration from the current schema kalam migration create add_profile diff --git a/docs/plans/2026-08-16-kalam-sync-implementation.md b/docs/plans/2026-08-16-kalam-sync-implementation.md new file mode 100644 index 000000000..11b95ee39 --- /dev/null +++ b/docs/plans/2026-08-16-kalam-sync-implementation.md @@ -0,0 +1,191 @@ +# Kalam Sync Dart Package Implementation Plan + +> **For Claude:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task. + +**Goal:** Add a maintainable `kalam_sync` Flutter package that provides durable local-first rows, checkpoints, custom actions, retries, row sync state, and one shared `kalam_link` transport, with generated Drift types and end-to-end recovery tests. + +**Architecture:** Keep the existing `link/sdks/dart/link/` transport package and Rust bridge behavior unchanged. Add `link/sdks/dart/sync/` and `link/sdks/dart/generator/` in the same Dart package group depending on hosted `kalam_link` with a repository path override. Drift owns SQLite and generated row/companion types. `kalam_sync` owns small runtime models, an SDK-private database, one generic DML envelope, custom action executors, checkpoint coordination, and Flutter-facing state. Add transport acknowledgement to `kalam_link` only if an additive option is required to prevent reconnect from passing an uncommitted SQLite checkpoint. + +**Tech Stack:** Flutter 3.48 beta / Dart 3.14 locally, Dart SDK floor 3.10, `kalam_link` 0.5.6-rc.0, Drift 2.34.3, drift_flutter 0.3.1, drift_dev 2.34.5, build_runner 2.15.3, flutter_test. + +--- + +### Task 1: Package skeleton and public contracts + +**Files:** +- Create: `link/sdks/dart/sync/pubspec.yaml` +- Create: `link/sdks/dart/sync/analysis_options.yaml` +- Create: `link/sdks/dart/sync/lib/kalam_sync.dart` +- Create: `link/sdks/dart/sync/lib/src/models/kalam_sync_mode.dart` +- Create: `link/sdks/dart/sync/lib/src/models/kalam_sync_state.dart` +- Create: `link/sdks/dart/sync/lib/src/models/kalam_action_status.dart` +- Create: `link/sdks/dart/sync/lib/src/models/kalam_row_sync_state.dart` +- Create: `link/sdks/dart/sync/lib/src/models/kalam_synced_row.dart` +- Test: `link/sdks/dart/sync/test/models/sync_models_test.dart` + +**Steps:** +1. Write tests for enum values, immutable sync-state transitions, pending/failed counts, and `KalamSyncedRow` equality/value access. +2. Run `flutter test test/models/sync_models_test.dart`; expect failure because the package surface does not exist. +3. Add the smallest immutable models, each in its own file, and export them from `kalam_sync.dart` along with `kalam_link`. +4. Run the focused test and `flutter analyze`; expect both to pass. + +### Task 2: Drift-owned private persistence schema + +**Files:** +- Create: `link/sdks/dart/sync/lib/src/database/tables/kalam_actions.dart` +- Create: `link/sdks/dart/sync/lib/src/database/tables/kalam_checkpoints.dart` +- Create: `link/sdks/dart/sync/lib/src/database/tables/kalam_row_states.dart` +- Create: `link/sdks/dart/sync/lib/src/database/tables/kalam_action_steps.dart` +- Create: `link/sdks/dart/sync/lib/src/database/kalam_sync_database.dart` +- Generate: `link/sdks/dart/sync/lib/src/database/kalam_sync_database.g.dart` +- Test: `link/sdks/dart/sync/test/database/kalam_sync_database_test.dart` + +**Steps:** +1. Write in-memory Drift tests proving schema creation, action persistence, composite row-state keys, checkpoint monotonicity, and restart persistence through a temporary database file. +2. Run the focused test; expect compilation failure for missing tables/database. +3. Add only SDK-private Drift tables. Keep action payloads JSON and action IDs supplied by an injectable ID factory. +4. Run `dart run build_runner build --delete-conflicting-outputs`. +5. Run the focused tests and analyzer. + +### Task 3: Atomic store operations + +**Files:** +- Create: `link/sdks/dart/sync/lib/src/store/kalam_sync_store.dart` +- Create: `link/sdks/dart/sync/lib/src/models/kalam_action_record.dart` +- Create: `link/sdks/dart/sync/lib/src/models/kalam_checkpoint.dart` +- Test: `link/sdks/dart/sync/test/store/kalam_sync_store_test.dart` + +**Steps:** +1. Write failing tests for atomic enqueue + optimistic row state, action-state watches, checkpoint + apply transaction, monotonic checkpoints, retry metadata, and account isolation. +2. Implement a single-writer `KalamSyncStore` over one Drift database connection. +3. Verify rollback leaves neither an optimistic state nor an action, and failed apply leaves the prior checkpoint unchanged. +4. Run focused tests and analyzer. + +### Task 4: Typed custom action runtime + +**Files:** +- Create: `link/sdks/dart/sync/lib/src/actions/kalam_action_codec.dart` +- Create: `link/sdks/dart/sync/lib/src/actions/kalam_action_context.dart` +- Create: `link/sdks/dart/sync/lib/src/actions/kalam_action_definition.dart` +- Create: `link/sdks/dart/sync/lib/src/actions/kalam_action_registry.dart` +- Create: `link/sdks/dart/sync/lib/src/actions/kalam_action_runner.dart` +- Create: `link/sdks/dart/sync/lib/src/models/kalam_retry_policy.dart` +- Test: `link/sdks/dart/sync/test/actions/kalam_action_runner_test.dart` + +**Steps:** +1. Write failing tests for typed encode/decode, duplicate durable keys, offline enqueue, FIFO ordering keys, exponential retry, permanent failure, stable idempotency IDs, restart recovery, and observable outbox state. +2. Implement callable action definitions and an executor registry without reflection. +3. Implement one-at-a-time flushing by default; allow bounded cross-key concurrency only after correctness tests pass. +4. Implement durable named steps with derived idempotency keys and persisted results. +5. Run focused tests and analyzer. + +### Task 5: Table policies and per-row state + +**Files:** +- Create: `link/sdks/dart/sync/lib/src/tables/kalam_table_spec.dart` +- Create: `link/sdks/dart/sync/lib/src/tables/kalam_table_binding.dart` +- Create: `link/sdks/dart/sync/lib/src/tables/kalam_replica_overlay.dart` +- Create: `link/sdks/dart/sync/lib/src/models/kalam_change.dart` +- Test: `link/sdks/dart/sync/test/tables/kalam_table_binding_test.dart` + +**Steps:** +1. Write failing tests for `bidirectional` versus `replicaOnly`, optimistic overlay visibility, pending delete tombstones, failed action state, server-echo reconciliation, and no duplicate built-in DML action. +2. Implement generic adapters that reuse caller-provided Drift row/companion codecs rather than generating competing row models. +3. Expose `watch()` and `watchWithSyncState()` while keeping physical sidecar tables private. +4. Run focused tests and analyzer. + +### Task 6: Transport and sync coordinator + +**Files:** +- Create: `link/sdks/dart/sync/lib/src/transport/kalam_sync_transport.dart` +- Create: `link/sdks/dart/sync/lib/src/transport/kalam_link_transport.dart` +- Create: `link/sdks/dart/sync/lib/src/sync/kalam_sync_coordinator.dart` +- Create: `link/sdks/dart/sync/lib/src/sync/kalam_event_consumer.dart` +- Test: `link/sdks/dart/sync/test/sync/kalam_sync_coordinator_test.dart` + +**Steps:** +1. Write fake-transport tests for initial rows, ordered insert/update/delete, duplicate replay, disconnect during apply, resume from committed checkpoint, expired cursor, custom durable consumer, and connection/sync state transitions. +2. Implement the coordinator against a small transport interface, then adapt `kalam_link.liveEvents` without creating another socket. +3. If the existing transport cannot hold resume progress at the committed SQLite sequence, add a backward-compatible explicit-ack option and its Rust/Dart tests before claiming crash-safe resume. +4. Run focused tests, existing `kalam_link` unit tests, and analyzer. + +### Task 7: Flutter entry point and lifecycle + +**Files:** +- Create: `link/sdks/dart/sync/lib/src/kalam.dart` +- Create: `link/sdks/dart/sync/lib/src/flutter/kalam_scope.dart` +- Create: `link/sdks/dart/sync/lib/src/flutter/kalam_database_factory.dart` +- Test: `link/sdks/dart/sync/test/flutter/kalam_scope_test.dart` + +**Steps:** +1. Write failing widget tests for scope lookup, lifecycle pause/resume fencing, offline-first startup, account switching, and disposal. +2. Implement `Kalam.open`, database identity derivation, one shared client, replaying sync state, and lifecycle hooks. +3. Keep network connection off the first Flutter frame; local cache opening may be awaited. +4. Run widget tests and analyzer. + +### Task 8: Annotation and generator packages + +**Files:** +- Create: `link/sdks/dart/sync/lib/src/annotations/kalam_action.dart` +- Create: `link/sdks/dart/sync/lib/src/annotations/kalam_action_module.dart` +- Create: `link/sdks/dart/sync/lib/src/annotations/kalam_action_payload.dart` +- Create: `link/sdks/dart/generator/pubspec.yaml` +- Create: `link/sdks/dart/generator/lib/builder.dart` +- Create: `link/sdks/dart/generator/lib/src/kalam_action_generator.dart` +- Test: `link/sdks/dart/generator/test/kalam_action_generator_test.dart` + +**Steps:** +1. Write generator tests for stable action keys, payload codecs, duplicate names, unsupported payload fields, and generated install adapters. +2. Implement deterministic source generation only; runtime retry/SQLite behavior stays in `kalam_sync`. +3. Confirm an annotated sample builds with Drift modular generation ordered first. +4. Run generator tests and analyzer in both packages. + +### Task 9: Example and documentation + +**Files:** +- Create: `link/sdks/dart/sync/example/lib/main.dart` +- Create: `link/sdks/dart/sync/README.md` +- Modify: `AGENTS.md` +- Modify: `link/README.md` +- Modify: `docs/architecture/` or add an ADR for the package boundary +- Modify later with permission/scope: `../KalamSite/content/sdk/**` + +**Steps:** +1. Add one minimal todo example and one offline messaging feature example using Drift companions. +2. Document direct DML versus custom endpoint actions, row sync state, idempotency, and the explicit resume invariant. +3. Keep generated files and native artifacts out of the new package. +4. Run example analysis. + +### Task 10: Integration and end-to-end recovery tests + +**Files:** +- Create: `link/sdks/dart/sync/test/integration/offline_restart_test.dart` +- Create: `link/sdks/dart/sync/test/integration/ordered_apply_test.dart` +- Create: `link/sdks/dart/sync/test/integration/account_isolation_test.dart` +- Create: `link/sdks/dart/sync/test/e2e/kalam_sync_e2e_test.dart` +- Create: `link/sdks/dart/sync/test/e2e/helpers.dart` + +**Steps:** +1. Prove offline enqueue survives database close/reopen and flushes once connectivity returns. +2. Prove “backend accepted, response lost” reuses the same idempotency key. +3. Prove an event failure cannot checkpoint a later event. +4. Prove a disconnect during apply resumes from the last committed SQLite checkpoint without data loss. +5. Prove bidirectional todos and replica-only optimistic messages reconcile from real KalamDB events. +6. Run unit/integration tests without a server, then start the backend and run the E2E suite. + +### Task 11: CI, versioning, and final verification + +**Files:** +- Modify only after package tests are stable: `.github/workflows/dart-sdk.yml` +- Modify: `link/sdks/sync-versions.sh` +- Modify: `scripts/versions.py` +- Modify: `versions.json` + +**Steps:** +1. Add separate `kalam_sync` and generator test jobs without changing the existing `kalam_link` native build job. +2. Publish `kalam_link` before `kalam_sync`; use hosted bounded dependency ranges in published manifests and repository path overrides locally. +3. Run `dart format --output=none --set-exit-if-changed .` in both new packages. +4. Run `flutter analyze` and all unit/integration tests in both new packages. +5. Run existing `kalam_link` tests. +6. Start KalamDB and run new E2E tests plus the relevant existing reconnect/resume tests. +7. Run `python3 scripts/versions.py verify` after version metadata changes. diff --git a/docs/plans/2026-08-16-kalam-sync-sdk-api/01-bidirectional-todos.md b/docs/plans/2026-08-16-kalam-sync-sdk-api/01-bidirectional-todos.md new file mode 100644 index 000000000..581431dc0 --- /dev/null +++ b/docs/plans/2026-08-16-kalam-sync-sdk-api/01-bidirectional-todos.md @@ -0,0 +1,135 @@ +# Example 1: bidirectional todos + +This is the smallest local-first Flutter shape. There is no action module because ordinary KalamDB row writes already have a built-in durable action. + +## Open the session + +```dart +import 'package:flutter/material.dart'; +import 'package:kalam_sync/kalam_sync.dart'; + +Future openKalam(Auth auth) { + return Kalam.open( + url: 'https://db.example.com', + auth: auth, + tables: const [ + KalamTable( + 'app.todos', + mode: KalamSyncMode.bidirectional, + ), + ], + ); +} +``` + +The SDK derives the authenticated subject after login and opens the matching per-user cache. It starts local reads immediately, resumes inbound changes from the committed table checkpoint, and flushes pending actions when the connection is ready. + +## Keep feature methods small + +```dart +class TodoRepository { + TodoRepository(Kalam kalam) : todos = kalam.table('app.todos'); + + final KalamTableRef todos; + + Stream> watchTodos() { + return todos.watch(orderBy: const ['created_at DESC']); + } + + Future add(String title) { + return todos.insert({ + 'id': Kalam.id(), + 'title': title, + 'completed': false, + 'created_at': DateTime.now().toUtc(), + }); + } + + Future setCompleted(String id, bool completed) { + return todos.update( + id, + {'completed': completed}, + ); + } + + Future delete(String id) { + return todos.delete(id); + } +} +``` + +Each write returns after SQLite commits. It does not wait for the network. Internally, the row change and built-in DML outbox item are one transaction. No repository method calls `isConnected`, retries a request, or manages the socket. + +## Render from SQLite + +```dart +class TodoList extends StatelessWidget { + const TodoList({ + super.key, + required this.todos, + required this.kalam, + }); + + final TodoRepository todos; + final Kalam kalam; + + @override + Widget build(BuildContext context) { + return Column( + children: [ + ValueListenableBuilder( + valueListenable: kalam.syncState, + builder: (context, state, _) => Text( + switch (state.phase) { + KalamSyncPhase.offline => + 'Offline — ${state.pendingActions} pending', + KalamSyncPhase.catchingUp => 'Updating…', + KalamSyncPhase.flushing => + 'Sending ${state.pendingActions} changes…', + KalamSyncPhase.live => 'Up to date', + KalamSyncPhase.error => 'Sync needs attention', + }, + ), + ), + Expanded( + child: StreamBuilder>( + stream: todos.watchTodos(), + builder: (context, snapshot) { + final rows = snapshot.data ?? const []; + return ListView.builder( + itemCount: rows.length, + itemBuilder: (context, index) { + final row = rows[index]; + return CheckboxListTile( + value: row.boolValue('completed'), + title: Text(row.stringValue('title')), + onChanged: (value) => todos.setCompleted( + row.stringValue('id'), + value ?? false, + ), + ); + }, + ); + }, + ), + ), + ], + ); + } +} +``` + +The widget does not subscribe to WebSocket events. It watches the local table. The sync engine applies remote changes to that table, and Drift invalidation refreshes the widget. + +## Expected conflict contract + +The v1 default should be explicit and deterministic: + +- the client assigns the mutation/action ID; +- the server applies an idempotent mutation; +- the authoritative server event reconciles the optimistic local row; +- a delete uses a tombstone until the server result is known, so a failed action can be surfaced or restored; +- a permanent rejection marks the action failed and exposes it through `syncState`/`actions.watchFailed()`. + +A later conflict hook may support merge policies, but the default cannot silently be “last callback happened to win.” + diff --git a/docs/plans/2026-08-16-kalam-sync-sdk-api/02-mixed-replication-modes.md b/docs/plans/2026-08-16-kalam-sync-sdk-api/02-mixed-replication-modes.md new file mode 100644 index 000000000..48372330f --- /dev/null +++ b/docs/plans/2026-08-16-kalam-sync-sdk-api/02-mixed-replication-modes.md @@ -0,0 +1,112 @@ +# Example 2: mixed replication modes + +One authenticated Kalam session can combine local-first tables and backend-authoritative replicas. + +```dart +final kalam = await Kalam.open( + url: serverUrl, + auth: auth, + tables: const [ + KalamTable( + 'app.todos', + mode: KalamSyncMode.bidirectional, + ), + KalamTable( + 'app.messages', + mode: KalamSyncMode.replicaOnly, + ), + KalamTable( + 'app.conversations', + mode: KalamSyncMode.replicaOnly, + ), + KalamTable( + 'app.notifications', + mode: KalamSyncMode.replicaOnly, + ), + ], +); +``` + +All four subscriptions are multiplexed over the one `kalam_link` connection. Each table has its own committed checkpoint and apply transaction. + +## Bidirectional table + +```dart +await kalam.table('app.todos').insert({ + 'id': Kalam.id(), + 'title': 'Works while offline', + 'completed': false, +}); +``` + +The insert is visible immediately and automatically queued for KalamDB. + +## Replica-only tables + +```dart +final messages = kalam.table('app.messages'); + +final Stream>> rows = + messages.watchWithSyncState( + where: 'conversation_id = ?', + args: [conversationId], + orderBy: const ['created_at ASC'], +); +``` + +This is legal because it is a local read. A direct write is rejected: + +```dart +await messages.delete(messageId); +// Throws KalamReplicaOnlyWriteException before changing SQLite. +``` + +To request a backend-owned deletion, enqueue a feature action: + +```dart +await messageActions.delete( + DeleteMessage(messageId: messageId), +); +``` + +The lifecycle is: + +```text +enqueue action locally + -> wait for connectivity + -> call backend with stable idempotency key + -> backend changes KalamDB + -> KalamDB publishes the delete + -> sync engine commits delete + checkpoint locally + -> Drift watch refreshes the UI +``` + +There is no second Masky WebSocket, manual reconnect loop, or “delete locally again after HTTP succeeds” branch. + +## Pending UI without changing the replica + +If the app wants immediate feedback, it can observe the durable action: + +```dart +final pendingDeletes = messageActions.delete.watchPending( + key: (command) => command.messageId, +); +``` + +The widget can dim or label the matching message while it remains in the authoritative replica. This keeps the table truthful while still making offline intent visible. + +For optimistic sends or deletes, an SDK-owned overlay is joined by `watchWithSyncState()`. It should not physically overwrite or delete the authoritative replica row. That distinction prevents a rejected backend action from corrupting the local mirror. + +## Masky mapping + +The likely initial policies are: + +| Masky data | Suggested mode | Reason | +| --- | --- | --- | +| Messages | `replicaOnly` | REST/backend actions are authoritative; Kalam events update local rows | +| Conversations | `replicaOnly` | Backend/user-update events drive refresh and changes | +| Notifications | `replicaOnly` | Server-originated data | +| Event-change tables | `replicaOnly` or widget consumer | Depends on whether the UI queries history or only reacts to events | +| Pure client drafts/todos | `bidirectional` | Immediate offline CRUD is the desired behavior | + +Masky can change one table at a time without changing the connection or duplicating transport logic. diff --git a/docs/plans/2026-08-16-kalam-sync-sdk-api/03-feature-actions-and-events.md b/docs/plans/2026-08-16-kalam-sync-sdk-api/03-feature-actions-and-events.md new file mode 100644 index 000000000..bccb68811 --- /dev/null +++ b/docs/plans/2026-08-16-kalam-sync-sdk-api/03-feature-actions-and-events.md @@ -0,0 +1,251 @@ +# Example 3: feature actions and widget events + +This example shows where registered actions live, how a local method becomes durable work, and how a widget can mount a durable event consumer without putting all event logic in `Kalam.open()`. + +## Define actions beside the feature + +```dart +// lib/features/messages/message_actions.dart +import 'package:kalam_sync/kalam_sync.dart'; + +part 'message_actions.kalam.dart'; + +@KalamActionPayload() +class DeleteMessage { + const DeleteMessage({required this.messageId}); + + final String messageId; +} + +@KalamActionPayload() +class DeleteTodoViaWorkflow { + const DeleteTodoViaWorkflow({required this.todoId}); + + final String todoId; +} + +@KalamActionPayload() +class AcknowledgeDelivery { + const AcknowledgeDelivery({ + required this.messageId, + required this.eventId, + }); + + final String messageId; + final String eventId; + + factory AcknowledgeDelivery.fromChange(KalamChange change) { + return AcknowledgeDelivery( + messageId: change.row.stringValue('message_id'), + eventId: change.row.stringValue('id'), + ); + } +} + +@KalamActionModule(namespace: 'messages') +class MessageActions with _$MessageActions { + MessageActions(this.api); + + final MessageApi api; + + // Backend-authoritative: enqueue only. app.messages is replicaOnly. + @KalamAction(name: 'delete', version: 1) + late final delete = queuedAction( + retry: const KalamRetry.exponential( + maxAttempts: 12, + maxDelay: Duration(minutes: 5), + ), + run: (context, command) => api.deleteMessage( + command.messageId, + idempotencyKey: context.idempotencyKey, + ), + ); + + // Custom workflow: delete locally and enqueue exactly one custom action. + // app.todos is bidirectional, but writes through this optimistic context + // are captured by this action instead of creating a second DML action. + @KalamAction(name: 'delete-todo-workflow', version: 1) + late final deleteTodoViaWorkflow = + queuedAction( + optimistic: (local, command) => + local.table('app.todos').delete(command.todoId), + run: (context, command) => api.deleteTodoViaWorkflow( + command.todoId, + idempotencyKey: context.idempotencyKey, + ), + ); + + @KalamAction(name: 'acknowledge-delivery', version: 1) + late final acknowledgeDelivery = + queuedAction( + orderingKey: (command) => command.messageId, + run: (context, command) => api.acknowledgeDelivery( + messageId: command.messageId, + eventId: command.eventId, + idempotencyKey: context.idempotencyKey, + ), + ); +} +``` + +The action object owns both phases when both exist: + +- `optimistic` runs only while the outbox record is being created; +- `run` runs only from the outbox executor; +- calling the action always means enqueue; +- the generator owns payload serialization and registration. + +This avoids a separate switch statement, action-name constants, JSON mapping file, retry service, and per-feature connectivity listener. + +## Install at authenticated-session scope + +```dart +class AppSession { + AppSession({ + required this.kalam, + required this.messageActions, + }); + + final Kalam kalam; + final MessageActions messageActions; + + static Future start({ + required Auth auth, + required MessageApi messageApi, + }) async { + final kalam = await Kalam.open( + url: serverUrl, + auth: auth, + tables: const [ + KalamTable( + 'app.todos', + mode: KalamSyncMode.bidirectional, + ), + KalamTable( + 'app.messages', + mode: KalamSyncMode.replicaOnly, + ), + ], + ); + + final messageActions = await kalam.actions.install( + MessageActions(messageApi), + ); + + return AppSession( + kalam: kalam, + messageActions: messageActions, + ); + } + + Future close() => kalam.close(); +} +``` + +Larger apps can install each module in its feature provider and expose it through Riverpod, GetIt, or the app's existing dependency injection. The requirement is lifetime, not a particular service-locator pattern: executors remain installed while their session outbox may flush. + +## Local methods stay one line + +```dart +class MessageRepository { + MessageRepository(this.actions); + + final MessageActions actions; + + Future deleteMessage(String messageId) { + return actions.delete( + DeleteMessage(messageId: messageId), + ); + } +} +``` + +Calling this offline succeeds after the durable action is stored. The repository does not check the connection or schedule a retry. + +The custom optimistic workflow is equally small at the call site: + +```dart +await messageActions.deleteTodoViaWorkflow( + DeleteTodoViaWorkflow(todoId: todoId), +); +``` + +The local delete and action insert either both commit or both roll back. + +## Mount a durable consumer in one widget + +```dart +class ConversationScreenState extends State { + late final KalamEventSubscription deliveryEvents; + + Kalam get kalam => widget.session.kalam; + MessageActions get actions => widget.session.messageActions; + + @override + void initState() { + super.initState(); + + deliveryEvents = kalam.events.consume( + id: 'conversation/${widget.conversationId}/delivery-events-v1', + query: ''' + SELECT * + FROM app.message_events + WHERE conversation_id = $1 + ''', + params: [widget.conversationId], + onEvent: (change, context) async { + if (change case KalamInsert()) { + await context.enqueue( + actions.acknowledgeDelivery, + AcknowledgeDelivery.fromChange(change), + ); + } + }, + ); + } + + @override + void dispose() { + deliveryEvents.cancel(); + super.dispose(); + } +} +``` + +The event consumer is feature-scoped, but its checkpoint is durable. If the widget unloads, the live query is removed. If it opens again, the same consumer ID resumes from the last event whose handler transaction committed. + +`context.enqueue(...)` is stronger than calling an external API inside `onEvent`: the action insert and event checkpoint commit together. The action executor may later retry using its stable idempotency key. + +If a callback only changes in-memory UI and does not need durable replay, use the lighter stream: + +```dart +final stream = kalam.events.watch( + query: 'SELECT * FROM app.presence WHERE room_id = $1', + params: [roomId], +); +``` + +## Runtime action state + +Actions should be observable without reading private outbox tables. A typed definition exposes its own states: + +```dart +messageActions.delete.watch( + key: (command) => command.messageId, +).listen((state) { + // queued, running, retryScheduled, succeeded, or failed +}); +``` + +The session also exposes one merged stream containing built-in table mutations and custom feature actions: + +```dart +kalam.actions.watch().listen((change) { + // Added, started, retryScheduled, succeeded, failed, or removed. + // change.kind distinguishes builtInDml from customAction. +}); +``` + +This is the supported outbox subscription API. The physical Drift outbox tables remain private so their schema can evolve without breaking applications. + +A successful backend response means the action finished. It does not mean a `replicaOnly` row has already changed locally; that remains tied to the inbound Kalam event and table checkpoint. diff --git a/docs/plans/2026-08-16-kalam-sync-sdk-api/04-generated-bindings-and-future-functions.md b/docs/plans/2026-08-16-kalam-sync-sdk-api/04-generated-bindings-and-future-functions.md new file mode 100644 index 000000000..f256fa3a3 --- /dev/null +++ b/docs/plans/2026-08-16-kalam-sync-sdk-api/04-generated-bindings-and-future-functions.md @@ -0,0 +1,295 @@ +# Generated bindings and future KalamDB functions + +Status: the table/change binding design belongs to frontend-only v1. Backend functions are a compatibility target, not part of the current implementation plan. + +## Decision + +Adding transactional backend functions later is a strong fit for this design. A frontend action is durable **intent** stored in the local outbox; its executor may be either: + +- a client-defined HTTP/OpenAPI handler in v1; or +- a generated KalamDB server-function invocation in the future. + +The UI should not care which executor is used: + +```dart +await deleteTodo(DeleteTodoArgs(todoId: todoId)); +``` + +That call always means “store this action durably.” It must never sometimes execute HTTP immediately and sometimes enqueue, depending on its implementation. + +## Terminology + +SpaceTimeDB calls transactional database-mutating functions **reducers**. KalamDB should reserve precise terms: + +- **Action**: a durable client intent in `kalam_sync`. +- **Transaction function** or **command**: a future atomic server function that reads/writes KalamDB tables. This is equivalent to a SpaceTimeDB reducer. +- **Procedure**: a future server function that can perform external I/O. Its retry and transaction guarantees differ from a database-only command. + +The public server namespace can still be `functions`, with a declared `kind` of `transaction` or `procedure`. Calling every function a reducer would be misleading once external side effects are allowed. + +## CLI-generated Drift schema and Kalam metadata + +The Dart CLI generator should accept selective bindings: + +```toml +[schema.targets.dart] +output = "lib/generated/kalam" +include_tables = [ + "app.todos", + "app.messages", + "app.message_events", +] + +# Reserved for the future backend-functions feature. It remains empty in v1. +include_functions = [] +``` + +For each table, `kalam schema gen` should generate: + +- a `.drift` table declaration matching the synced server columns; +- metadata mapping the local Drift table to its remote `TableId`; +- primary/composite key and schema-fingerprint metadata; +- declared local-only columns that server apply must preserve. + +It should not generate a second Dart row, insert, patch, or CRUD-action model. Drift's builder consumes the `.drift` declaration and generates the row data class, companion, typed local queries, and accessors. Drift documents these as the two standard table types: a full-row data class and a companion representing partial insert/update values. + +After Drift runs, `kalam_sync_generator` emits only a thin adapter referencing those Drift types. With modular Drift generation, the build can order the Kalam builder after Drift and import the independently generated `.drift.dart` library. + +The generated adapters keep the mixed-mode connection type-safe: + +```dart +import 'package:my_app/generated/kalam/kalam.dart'; + +final todoSync = AppTables.todos.bidirectional(); +final messageSync = AppTables.messages.replicaOnly(); + +final kalam = await Kalam.open( + url: serverUrl, + auth: auth, + tables: [todoSync, messageSync], +); + +final todos = todoSync.bind(kalam); // writable typed table +final messages = messageSync.bind(kalam); // read-only typed replica + +await todos.insert( + TodosCompanion.insert( + id: Kalam.id(), + title: 'Generated and offline-first', + ), +); + +messages.watch().listen(renderMessages); +``` + +`messageSync.bind(kalam)` should not expose insert/update/delete methods. Choosing `replicaOnly()` should remove those operations at compile time, not merely throw at runtime. + +## CRUD does not create three action models per table + +`todos.insert`, `todos.update`, and `todos.delete` use Drift values at the public boundary, then serialize into one private envelope: + +```dart +final class KalamDmlAction { + const KalamDmlAction({ + required this.actionId, + required this.tableId, + required this.operation, + required this.key, + required this.values, + required this.schemaFingerprint, + }); + + final String actionId; + final TableId tableId; + final KalamDmlOperation operation; + final Map key; + final Map values; + final String schemaFingerprint; +} +``` + +This model is internal to `kalam_sync`. It is sufficient for every ordinary table mutation and keeps outbox migrations independent of app-generated type names. + +Drift remains the type system and query engine, but writes to a synced table must go through its generated Kalam binding so the local write and outbox item share one transaction: + +```dart +await todos.update( + todoId, + TodosCompanion( + completed: const Value(true), + ), +); +``` + +Direct reads and local-only writes may use the application database normally. In v1, a direct `appDb.into(appDb.todos).insert(...)` on a synced table is not promised to enter the outbox; reliable capture requires the Kalam binding. This avoids trigger-suppression races while still using Drift's generated types and executor. + +## Generated events do not auto-subscribe + +The generator creates typed change adapters that reuse Drift's row data class, but a feature decides when to consume them: + +```dart +late final KalamEventSubscription subscription; + +@override +void initState() { + super.initState(); + + subscription = AppTables.messageEvents.changes.consume( + kalam, + id: 'conversation/${widget.conversationId}/message-events-v1', + where: (event) => + event.conversationId.equals(widget.conversationId), + onEvent: (change, context) async { + switch (change) { + case KalamInsert(row: final event): + await context.enqueue( + messageActions.acknowledgeDelivery, + AcknowledgeDelivery.fromEvent(event), + ); + case KalamUpdate(): + case KalamDelete(): + break; + } + }, + ); +} + +@override +void dispose() { + subscription.cancel(); + super.dispose(); +} +``` + +This preserves widget/feature ownership. Merely generating `message_events` does not add it to `Kalam.open()` and does not keep an unused live query active. + +The generator should not create a queued action for every change event. A change describes something that already happened; an action requests something new. When a change must cause an action, the explicit `context.enqueue(...)` bridge above makes its durability and checkpoint boundary visible. + +## Application-defined actions in v1 + +The `build_runner` generator remains useful for actions that call an existing app backend: + +```dart +@KalamActionModule(namespace: 'todos') +class TodoActions with _$TodoActions { + TodoActions(this.api); + + final TodoApi api; + + @KalamAction(name: 'delete', version: 1) + late final delete = queuedAction( + run: (context, args) => api.deleteTodo( + args.todoId, + idempotencyKey: context.idempotencyKey, + ), + ); +} +``` + +The generated registry and codec use the same `KalamQueuedAction` runtime type that a future CLI-generated server function will use. + +## Future generated server functions + +If KalamDB later exposes server functions, schema metadata should describe at least: + +- stable function name and version; +- `transaction` or `procedure` kind; +- typed input and output; +- authorization requirements; +- idempotency behavior; +- tables the function may change; +- whether it publishes an invocation event; +- result/error schema. + +The CLI can then generate a descriptor rather than handwritten Dart execution code: + +```dart +final deleteTodo = await kalam.actions.install( + AppFunctions.deleteTodo.queued( + optimistic: (local, args) => + local.table(AppTables.todos).delete(args.todoId), + ), +); + +final receipt = await deleteTodo( + DeleteTodoArgs(todoId: todoId), +); + +await receipt.queued; // durable locally +await receipt.acked; // server function committed +await receipt.synced; // affected local replicas reached the commit watermark +``` + +No `run:` closure is needed because the generated function descriptor is the executor. The server invocation must accept the action UUID as an idempotency key and return its commit watermark. Without those two fields, offline retry can duplicate a successful function and `receipt.synced` cannot know when the local cache has observed its effects. + +Users opt into only the functions they use: + +```dart +final actions = await kalam.actions.installAll([ + AppFunctions.deleteTodo.queued(), + AppFunctions.archiveConversation.queued(), +]); +``` + +An uninstalled generated function has no outbox executor. Generation alone does not register it and does not subscribe to invocation events. + +## Pre- and post-action behavior + +Generated files must never be edited. User behavior attaches through explicit policy and state APIs: + +```dart +final deleteTodo = AppFunctions.deleteTodo.queued( + optimistic: (local, args) => + local.table(AppTables.todos).delete(args.todoId), +); + +deleteTodo.watch().listen((state) { + // queued, running, retryScheduled, acked, synced, or failed +}); +``` + +The lifecycle names need exact semantics: + +| Extension point | Guarantee | +| --- | --- | +| `optimistic` / `beforeEnqueue` | Runs inside the same SQLite transaction as outbox insertion | +| `queued` | The action is durable locally | +| `acked` | The backend committed or accepted the action | +| `synced` | Subscribed affected tables have applied the returned commit watermark | +| `failed` | Retry policy classified the action as permanently failed | + +Do not expose an ambiguous `beforeSend` hook for business logic: retries may run it multiple times. A UI post-hook should observe the action state. A durable follow-up should be another action enqueued from a durable event consumer so its work and checkpoint commit atomically. + +## Subscribing to actions versus changes + +These are different subscriptions: + +- `action.watch()` observes this client's outbox lifecycle. +- `table.watch()` observes the current local rows. +- `table.changes.consume()` durably processes insert/update/delete events. +- A future `function.invocations.consume()` should exist only if the server explicitly publishes an authorized invocation stream. + +The normal source of truth after a server function is still the changed table event, not the reducer/function callback. This keeps direct SQL writes, backend functions, jobs, and other clients on the same synchronization path. + +## Compatibility requirement for frontend-only v1 + +The v1 action runtime should depend on an executor interface rather than directly on an HTTP callback: + +```dart +abstract interface class KalamActionExecutor { + Future execute( + KalamActionContext context, + Input input, + ); +} +``` + +The frontend module adapter and future generated server-function adapter both implement this interface. This is the only future seam needed now; KalamDB backend functions themselves remain outside the frontend-only v1 plan. + +## Reference model + +SpaceTimeDB documents reducers as transactional functions that mutate database state and exposes them to clients automatically: [Reducer overview](https://spacetimedb.com/docs/functions/reducers/). Its generated bindings include table types, reducer callers, subscriptions, local-cache access, and reducer callbacks: [Generating client bindings](https://spacetimedb.com/docs/clients/codegen/). + +Drift already generates a full-row data class plus a companion for partial inserts and updates: [Generated table rows](https://drift.simonbinder.eu/dart_api/rows/). Its [modular code generation](https://drift.simonbinder.eu/generation_options/modular/) produces independent libraries and supports ordering other builders after Drift, which is the intended integration point for Kalam's thin adapters. + +Kalam should adopt that typed contract generation while retaining its local durable outbox and per-table replication modes. diff --git a/docs/plans/2026-08-16-kalam-sync-sdk-api/05-offline-messaging.md b/docs/plans/2026-08-16-kalam-sync-sdk-api/05-offline-messaging.md new file mode 100644 index 000000000..ea1bf77e5 --- /dev/null +++ b/docs/plans/2026-08-16-kalam-sync-sdk-api/05-offline-messaging.md @@ -0,0 +1,221 @@ +# Offline messaging with one conversation + +This is the intended minimal messaging flow when an application backend, rather than direct KalamDB DML, owns message creation and deletion. + +## What the application supplies + +The application supplies only: + +- the conversation filter; +- the backend calls for `sendMessage` and `deleteMessage`; +- any domain fields required by those calls; +- the widget presentation. + +`kalam_sync` supplies the local cache, shared connection, durable subscription checkpoint, optimistic row, outbox, retry, reconciliation, and row sync state. + +## Configure the backend-authoritative table + +```dart +final messageSync = AppTables.messages.replicaOnly( + where: (message) => + message.conversationId.equals(conversationId), +); + +final kalam = await Kalam.open( + url: serverUrl, + auth: auth, + tables: [messageSync], +); + +final messages = messageSync.bind(kalam); +``` + +The sync engine subscribes to the filtered backend changes and applies them to SQLite. The widget does not consume raw WebSocket events; it watches the local Drift-backed view: + +```dart +final Stream>> rows = + messages.watchWithSyncState( + orderBy: (message) => message.createdAt.asc(), +); +``` + +`replicaOnly` prevents arbitrary direct writes, but an installed custom action may create an SDK-managed optimistic row. That exception is explicit and atomic with the action enqueue. + +## Define the two backend actions + +Table schema generation does not know which HTTP endpoints to call. Therefore it does not generate executable `CreateMessageAction` and `DeleteMessageAction` classes from the table alone. + +The action generator removes registry and serialization boilerplate, while the application provides the unavoidable backend behavior: + +```dart +@KalamActionPayload() +class SendMessageArgs { + const SendMessageArgs({ + required this.messageId, + required this.conversationId, + required this.text, + required this.createdAt, + }); + + final String messageId; + final String conversationId; + final String text; + final DateTime createdAt; +} + +@KalamActionPayload() +class DeleteMessageArgs { + const DeleteMessageArgs({required this.messageId}); + + final String messageId; +} + +@KalamActionModule(namespace: 'messages') +class MessageActions with _$MessageActions { + MessageActions(this.api); + + final MessageApi api; + + @KalamAction(name: 'send', version: 1) + late final send = queuedAction( + optimistic: (local, args) => local + .replica(AppTables.messages) + .insert( + MessagesCompanion.insert( + id: args.messageId, + conversationId: args.conversationId, + text: args.text, + createdAt: args.createdAt, + ), + ), + run: (context, args) => api.sendMessage( + messageId: args.messageId, + conversationId: args.conversationId, + text: args.text, + idempotencyKey: context.idempotencyKey, + ), + ); + + @KalamAction(name: 'delete', version: 1) + late final delete = queuedAction( + optimistic: (local, args) => local + .replica(AppTables.messages) + .markPendingDelete(args.messageId), + run: (context, args) => api.deleteMessage( + messageId: args.messageId, + idempotencyKey: context.idempotencyKey, + ), + ); +} +``` + +The local optimistic write is not a second DML action. `local.replica(...)` is available only inside the action's enqueue transaction and records the row state against that custom action. + +If the backend writes directly through KalamDB and no custom API is needed, configure the table as `bidirectional` and use its Drift-backed `insert/update/delete` methods instead. No `MessageActions` module is necessary in that simpler case. + +## Send while offline + +```dart +Future sendMessage(String text) { + final now = DateTime.now().toUtc(); + + return messageActions.send( + SendMessageArgs( + messageId: Kalam.id(), + conversationId: conversationId, + text: text, + createdAt: now, + ), + ); +} +``` + +The call returns after one SQLite transaction commits: + +1. the optimistic message values; +2. the per-row `pending` state; +3. the serialized `messages.send@1` outbox action. + +No connection check occurs in the repository or widget. + +## Display per-message state + +```dart +Widget buildMessage(KalamSyncedRow row) { + return MessageBubble( + text: row.value.text, + status: switch (row.sync.phase) { + KalamRowSyncPhase.pending => 'Waiting to send', + KalamRowSyncPhase.sending => 'Sending…', + KalamRowSyncPhase.awaitingServerEcho => 'Sent, syncing…', + KalamRowSyncPhase.synced => null, + KalamRowSyncPhase.failed => 'Not sent — tap to retry', + }, + onRetry: row.sync.phase == KalamRowSyncPhase.failed + ? () => kalam.actions.retry(row.sync.actionId) + : null, + ); +} +``` + +The sync phase is SDK-owned local metadata. It must not be confused with backend domain delivery states such as `sent`, `delivered`, and `read`. + +## Reconcile the backend echo + +When connectivity returns: + +1. the outbox executes `api.sendMessage` using the stable action UUID as its idempotency key; +2. the backend performs its workflow and writes the authoritative message into KalamDB; +3. KalamDB publishes the table change; +4. `kalam_sync` applies the row and checkpoint atomically; +5. the optimistic row is reconciled and its sidecar state becomes `synced`. + +The cleanest contract keeps the client-generated `messageId` as the authoritative primary key. If the backend must allocate another ID, it must persist and echo a unique `clientMessageId` or action UUID so the SDK can reconcile instead of showing a duplicate message. + +An acknowledged HTTP request is only `awaitingServerEcho`. It becomes `synced` after the matching KalamDB change is committed locally. + +## Why sync state is a sidecar + +Do not append `_kalam_synced`, `_kalam_error`, and `_kalam_action_id` columns to every generated table. A Kalam-owned sidecar avoids remote-schema collisions, repeated migrations, and accidental upload of local-only values. + +Conceptually it contains: + +```text +account + table_id + row_key +action_id +phase +attempt_count + next_retry_at +last_error +last_server_seq +pending values or delete tombstone when required +``` + +`watchWithSyncState()` performs the merge and exposes a typed `KalamSyncedRow`. Apps that do not need status can use `messages.watch()` and receive only Drift-generated `Message` values. + +## Multiple backend endpoints + +One action may coordinate several endpoints, but the executor is retried at least once. Every externally visible step therefore needs an idempotency key. The SDK can offer durable named steps: + +```dart +run: (context, args) async { + final upload = await context.step( + 'upload-attachment', + run: (stepKey) => api.uploadAttachment( + args.attachment, + idempotencyKey: stepKey, + ), + ); + + await context.step( + 'send-message', + run: (stepKey) => api.sendMessage( + messageId: args.messageId, + attachmentId: upload.id, + idempotencyKey: stepKey, + ), + ); +} +``` + +The step key is derived from the action UUID and stable step name. A completed step is not intentionally repeated after restart, but an endpoint must still be idempotent because its response can be lost after the server commits. When possible, one backend endpoint that transactionally owns the whole workflow is safer than client orchestration. + diff --git a/docs/plans/2026-08-16-kalam-sync-sdk-api/README.md b/docs/plans/2026-08-16-kalam-sync-sdk-api/README.md new file mode 100644 index 000000000..92dde7835 --- /dev/null +++ b/docs/plans/2026-08-16-kalam-sync-sdk-api/README.md @@ -0,0 +1,299 @@ +# Kalam Sync Flutter API proposal + +Status: proposed public API; none of the types in this folder are implemented yet. + +This proposal makes `kalam_sync` the local-first layer above `kalam_link`. A Flutter app opens one authenticated Kalam session, chooses a replication policy per table, and lets the SDK own the socket, durable checkpoints, local cache, retry queue, and connection state. + +The design intentionally separates three concerns: + +1. **Table replication policy** is declared once when the authenticated session opens. +2. **Outbound business actions** live in small, feature-owned action modules and are installed for the session. +3. **Event consumers** are created where they are needed, including inside a widget, and are cancelled when that feature unloads. + +There is no central `events: []` list in `Kalam.open()`. + +## Recommended public shape + +```dart +final kalam = await Kalam.open( + url: serverUrl, + auth: auth, + tables: const [ + KalamTable( + 'app.todos', + mode: KalamSyncMode.bidirectional, + ), + KalamTable( + 'app.messages', + mode: KalamSyncMode.replicaOnly, + ), + KalamTable( + 'app.conversations', + mode: KalamSyncMode.replicaOnly, + ), + ], +); +``` + +The same connection and local SQLite database serve every table. The mode belongs to a table, not to the connection. + +| Mode | Local read | Direct local write | Outbound behavior | Inbound behavior | +| --- | --- | --- | --- | --- | +| `bidirectional` | SQLite | Allowed | The row change and a built-in Kalam DML action are committed atomically | Server events reconcile the cached row | +| `replicaOnly` | SQLite | Rejected | Use a queued feature action when the backend must do work | Only backend events change the cached row | + +`replicaOnly` is the Masky-style backend-authoritative mode. For example, deleting a message queues `messages.delete`; the local message remains until the backend accepts the action, changes KalamDB, and KalamDB sends the resulting delete event back to the app. + +### Per-row local sync state + +Pending/failed state should not add `_synced` columns to every server-shaped Drift table. The SDK owns a sidecar keyed by account, remote table ID, and row primary key. It tracks the action ID, local phase, attempts/error, and last applied server sequence. For an optimistic row that does not exist on the backend yet, the SDK also retains its pending local values or tombstone until reconciliation. + +The typed API joins that metadata for the application: + +```dart +Stream>> rows = + messages.watchWithSyncState(); +``` + +`KalamSyncedRow.value` is the Drift-generated `Message`; `KalamSyncedRow.sync.phase` can be `pending`, `sending`, `awaitingServerEcho`, `synced`, or `failed`. These phases describe local synchronization only. Domain states such as delivered/read remain ordinary backend-owned message columns. + +## Why actions are feature modules + +An outbox executor cannot safely belong to a widget. A queued action must still be executable after navigation, reconnection, or app resume. It therefore lives for the authenticated session, but its implementation stays beside its feature: + +```text +lib/features/messages/ + message_actions.dart action definitions and backend execution + message_repository.dart feature queries and ordinary methods + message_screen.dart widget-scoped local watches/event consumers +``` + +The app installs one module per feature: + +```dart +final messageActions = await kalam.actions.install( + MessageActions(messageApi), +); +``` + +Installation registers serialization and execution. The returned module exposes typed, callable actions: + +```dart +await messageActions.delete( + DeleteMessage(messageId: message.id), +); +``` + +That call only commits an outbox item. The SDK decides when to execute it, retains the same idempotency key across retries, and waits for the backend event to update a `replicaOnly` table. + +## Annotation design + +Annotations should remove codec and registry boilerplate, not hide whether a method executes now or is queued. Annotating an ordinary `Future delete()` method is misleading because Dart cannot transparently replace that method call without generating a second facade. + +The recommended generated form therefore annotates a **callable action field**: + +```dart +part 'message_actions.kalam.dart'; + +@KalamActionModule(namespace: 'messages') +class MessageActions with _$MessageActions { + MessageActions(this.api); + + final MessageApi api; + + @KalamAction(name: 'delete', version: 1) + late final delete = queuedAction( + run: (context, command) => api.deleteMessage( + command.messageId, + idempotencyKey: context.idempotencyKey, + ), + ); +} +``` + +The generator creates the module registry, stable action key (`messages.delete@1`), payload codec, and install adapter. `delete(command)` always means enqueue; only the SDK invokes `run` while flushing the outbox. + +The annotations should be exported by `kalam_sync`, so application code still has one runtime dependency. Only generated action modules add development dependencies: + +```yaml +dependencies: + kalam_sync: ^0.6.0 + +dev_dependencies: + build_runner: ^2.0.0 + kalam_sync_generator: ^0.6.0 +``` + +A one-file application using built-in bidirectional table writes needs no `build_runner`. A manual `KalamQueuedAction.json(...)` escape hatch should remain available for apps that do not want generation. + +## Custom optimistic actions + +A custom backend workflow may need to change the UI immediately and enqueue one non-SQL action. The action definition can provide an optional local phase: + +```dart +@KalamAction(name: 'deleteViaWorkflow', version: 1) +late final deleteViaWorkflow = queuedAction( + optimistic: (local, command) => + local.table('app.todos').delete(command.todoId), + run: (context, command) => api.deleteTodoWorkflow( + command.todoId, + idempotencyKey: context.idempotencyKey, + ), +); +``` + +The SDK commits the optimistic change and custom action in one SQLite transaction. Writes made through this `local` context do not create a second built-in DML outbox item. Arbitrary writes through another Drift connection are not part of this guarantee. + +An action cannot directly mutate the authoritative base rows of a `replicaOnly` table. It may omit `optimistic`, or it may write through the SDK's explicit replica-overlay context. That context stores pending values/tombstones and row state beside the replica, and `watchWithSyncState()` merges them for the UI without turning them into a second built-in DML action. + +## Widget-scoped events + +Use the local table watch for ordinary UI rendering. Use an event consumer only when receiving a server change must trigger work: + +```dart +late final KalamEventSubscription eventSubscription; + +@override +void initState() { + super.initState(); + + eventSubscription = kalam.events.consume( + id: 'conversation/${widget.conversationId}/delivery', + query: ''' + SELECT * + FROM app.message_events + WHERE conversation_id = $1 + ''', + params: [widget.conversationId], + onEvent: (change, context) async { + await context.enqueue( + messageActions.acknowledgeDelivery, + AcknowledgeDelivery.fromChange(change), + ); + }, + ); +} + +@override +void dispose() { + eventSubscription.cancel(); + super.dispose(); +} +``` + +The stable consumer `id` owns a durable checkpoint. Cancelling unloads the live subscription; reopening resumes from the last committed event. `context.enqueue(...)` atomically commits both the new action and the event checkpoint, so a crash cannot acknowledge an event without retaining the resulting work. + +Two APIs should be distinct: + +- `events.watch(...)` is an ephemeral stream for display-only reactions. +- `events.consume(id: ..., onEvent: ...)` is a serialized, durable side-effect consumer. Its checkpoint advances only after the asynchronous handler and local transaction succeed. + +## Required correctness rules + +These are part of the public contract, not implementation details: + +- Apply a server row and advance that table's durable checkpoint in the same SQLite transaction. +- Insert an outbox item and apply its optimistic local change in the same SQLite transaction. +- Resume a subscription from the last SQLite-committed checkpoint, not the last event read from the socket. +- Use a stable action UUID as the idempotency key for every retry, including the "server accepted it but the response was lost" case. +- Serialize all SDK cache/outbox writes through one executor. Do not use a shared database flag to suppress triggers. +- Partition the database by server, namespace, and authenticated subject. Stop sync before changing accounts. +- Preserve per-action ordering keys when actions for one conversation or aggregate must execute in order. +- Expose one replaying state object covering `offline`, `catchingUp`, `flushing`, `live`, and `error`, with pending and failed action counts. +- On an expired server cursor, perform a deterministic table rebootstrap rather than silently skipping history. + +### `kalam_link` durability boundary + +`kalam_link.liveEventsWithAck(...)` is the sync-only transport primitive. It +does not advance effective reconnect progress when an event is read. The Dart +pump applies consumer backpressure, and `kalam_sync` acknowledges each server +batch only after its rows and checkpoint commit in one SQLite transaction. + +Existing `liveEvents(..., onCheckpoint: ...)` remains automatic for ordinary +listeners. Its callback must not be used as a durable SQLite cursor because it +runs before application persistence succeeds. + +## What is generated + +`kalam_sync_generator` should generate only deterministic glue: + +- stable module and action registry entries; +- payload encode/decode functions; +- duplicate action-name and unsupported-payload compile errors; +- the action version in the durable key; +- the install adapter used by `kalam.actions.install(module)`. + +The annotations themselves remain in `kalam_sync`; only the builder is a separate development package. Retry scheduling, SQLite transactions, action state, socket state, and execution remain runtime responsibilities of `kalam_sync`. + +Renaming a Dart field must not rename a persisted action. `@KalamAction(name: ..., version: ...)` is therefore explicit and required for generated durable actions. + +### Generation pipeline, one owner for table models + +Kalam must not generate a second set of Dart row, insert, or patch models. Drift already generates a full-row data class and a companion for partial inserts/updates for every declared table. The pipeline is: + +| Stage | Input | Output | Owns application models? | +| --- | --- | --- | --- | +| `kalam schema gen --languages dart` | KalamDB schema metadata | `.drift` table declarations plus Kalam remote-table/primary-key metadata | No | +| Drift's builder | generated `.drift` declarations | row data classes, companions, typed queries, and local accessors | Yes | +| `kalam_sync_generator` | Kalam metadata, Drift modular output, and application annotations | thin sync adapters, typed `KalamChange` glue, and custom-action registries/codecs | Only custom action payloads | + +The CLI Dart target now emits row classes plus `KalamTableSpec` values from `schema.sql`. Drift `.drift` table declarations remain deferred so generated files compile against `kalam_sync` alone. Schema gen still does not emit competing CRUD payload classes; those stay with `kalam_sync_generator`. + +There is also no generated `CreateTodoAction`, `UpdateTodoAction`, and `DeleteTodoAction` for every table. Bidirectional CRUD uses one SDK-internal DML envelope containing the remote table ID, operation, key, changed values, schema fingerprint, and action UUID. The public typed write methods accept Drift-generated data classes or companions. + +Custom business actions remain different because their payloads are not necessarily table rows. `SendMessage`, `MarkSeen`, or a future server function may have dedicated argument/result models generated from their own contract. + +Generation and subscription are separate decisions: + +- CLI include/exclude options choose which bindings exist in the application binary. +- `bidirectional()` and `replicaOnly()` choose which generated tables sync for an authenticated session. +- `events.watch()` or `events.consume()` starts a subscription when a feature needs it. +- Generating a table, event type, or future server function must never open a socket or register a listener by itself. + +See [04-generated-bindings-and-future-functions.md](04-generated-bindings-and-future-functions.md) for the proposed generated syntax and the compatibility seam for future KalamDB functions. + +## Masky acceptance contract + +The API is successful only when Masky can delete its socket, checkpoint, and send-queue infrastructure. Before that migration, `kalam_sync` must prove: + +- strict, serialized event application per subscription; a failed event blocks later checkpoint advancement rather than being skipped; +- sparse update merging, explicit insert/update/delete operation types, FILE values, tombstones, and preservation of declared local-only columns; +- a projection adapter for apps that keep domain tables separate from raw synced rows; +- durable custom consumers for flows such as `user_updates -> REST fetch -> local projection`, including retry and dead-letter state; +- atomic local mutation plus custom action enqueue for messages, conversation workflows, reactions, seen state, settings, and notification actions; +- foreground pause/resume and a bounded headless catch-up entry point suitable for an FCM isolate; +- reactive auth refresh, stale-session fencing, and safe user switching; +- sync state that distinguishes transport readiness, catch-up completeness, paused/auth-error state, pending/failed actions, and the last successful commit time. + +Masky should retain message previews, unread calculations, contact hydration, notification presentation, and REST payload construction. The SDK should absorb socket ownership, ordering, durable checkpoints, outbox storage, retry, pause/resume, and connection/sync state. + +## Lessons from other systems + +- [PowerSync client-side integration](https://docs.powersync.com/configuration/app-backend/client-side-integration) atomically records local CRUD and its upload queue, then invokes a central upload hook after writes and reconnection. Kalam should provide the same atomicity and automatic flushing, while supplying built-in KalamDB DML execution. +- [RxDB replication](https://rxdb.info/replication.html) separates pull and push handlers, supports pull-only replication, and treats checkpoints and stable replication identifiers as core concepts. This supports explicit per-table modes and stable consumer IDs. +- [Replicache mutators](https://doc.replicache.dev/tutorial/adding-mutators) package optimistic local work as registered mutations that are queued and later applied authoritatively. Kalam's callable action definitions keep the same useful mental model while also supporting backend-authoritative actions with no optimistic phase. +- [WatermelonDB writers](https://watermelondb.dev/docs/Writers) organize related writes in feature/model methods and mark their transaction boundary explicitly. Kalam action modules follow that organization without coupling durable executors to widgets. +- [Drift DAOs](https://drift.simonbinder.eu/dart_api/daos/) use annotated feature accessors to split a database API into manageable modules. The action-module generator follows this established Dart shape. +- Dart's official [build_runner documentation](https://dart.dev/tools/build_runner) and [`source_gen` documentation](https://pub.dev/documentation/source_gen/latest/) support an optional annotation package plus generated part files; the runtime API must still work without generation. +- [SpaceTimeDB reducers](https://spacetimedb.com/docs/functions/reducers/) show the value of transactional server functions, while its [client binding generator](https://spacetimedb.com/docs/clients/codegen/) generates table, subscription, reducer-call, and callback APIs from one server contract. Kalam should preserve that future path without putting backend functions into frontend-only v1. + +## Examples in this folder + +- [01-bidirectional-todos.md](01-bidirectional-todos.md): offline CRUD with automatic Kalam DML outbox. +- [02-mixed-replication-modes.md](02-mixed-replication-modes.md): bidirectional and replica-only tables on one connection. +- [03-feature-actions-and-events.md](03-feature-actions-and-events.md): generated action modules, delete-then-enqueue, backend-authoritative enqueue, and a widget-scoped durable event consumer. +- [04-generated-bindings-and-future-functions.md](04-generated-bindings-and-future-functions.md): typed table/change generation now and server-function generation later without changing the frontend action model. +- [05-offline-messaging.md](05-offline-messaging.md): one-conversation messaging with a backend-authoritative replica, optimistic offline rows, custom endpoint actions, and per-row sync state. + +## Suggested implementation order + +1. Keep explicit-consumer-ack reconnect and SQLite commit-order tests green. +2. Implement the account-partitioned Drift store, table checkpoints, server apply transaction, and sync state. +3. Implement `replicaOnly` download/apply and expired-cursor rebootstrap. +4. Implement bidirectional built-in DML actions with atomic optimistic writes and idempotent retries. +5. Implement manual feature action modules and action-state watches. +6. Implement Dart CLI generation for `.drift` declarations and Kalam table metadata, then generate thin adapters over Drift's row/companion types. +7. Add the `kalam_sync` annotations and optional `kalam_sync_generator` for typed application-action module glue. +8. Implement `events.watch` and durable `events.consume` over the same shared `kalam_link` connection. +9. Add bounded headless catch-up, projection adapters, and sparse-update/delete conformance tests. +10. Migrate Masky one feature at a time: messages, conversations, notifications, event changes, then its REST action outbox. diff --git a/examples/live-okf-context-sync/tests/sync.integration.test.ts b/examples/live-okf-context-sync/tests/sync.integration.test.ts index 2a1ac1e48..fbc1b1bd6 100644 --- a/examples/live-okf-context-sync/tests/sync.integration.test.ts +++ b/examples/live-okf-context-sync/tests/sync.integration.test.ts @@ -8,7 +8,6 @@ import { resolve } from 'node:path'; import { Auth, createClient } from '@kalamdb/client'; import { createDb, createKalamClient, resolveKalamConnection, TABLE } from '../src/db/client.js'; import { downloadFileByPath, fetchRemoteHash, sha256Hex, upsertSyncFile } from '../src/remote-files.js'; -import { listSyncFiles } from '../src/lib/paths.js'; import { FolderSyncApp } from '../src/sync-app.js'; import { waitForLocalFiles } from '../src/helpers.js'; import { stopSyncApp } from './sync.helpers.js'; @@ -140,7 +139,7 @@ test('integration: delete local folder and restore from database', { skip: !RUN_ await cleanupClient.initialize(); await cleanupClient.login(); - const expectedPaths: string[] = []; + const ownedPaths = [editedPath]; let first: FolderSyncApp | undefined; let second: FolderSyncApp | undefined; @@ -152,27 +151,20 @@ test('integration: delete local folder and restore from database', { skip: !RUN_ await writeFile(join(syncDir, editedPath), editedContent, 'utf8'); await first.pushLocalFile(editedPath); - expectedPaths.push(...await listSyncFiles(syncDir)); - const expectedContents = Object.fromEntries( - await Promise.all(expectedPaths.map(async (path) => [path, await readFile(join(syncDir, path), 'utf8')] as const)), - ); - await stopSyncApp(first); first = undefined; await rm(syncDir, { recursive: true, force: true }); second = new FolderSyncApp({ syncDir, connection, watch: false }); await second.start(); - await waitForLocalFiles(syncDir, expectedPaths, 20_000); + await waitForLocalFiles(syncDir, ownedPaths, 20_000); - for (const path of expectedPaths) { - const restored = await readFile(join(syncDir, path), 'utf8'); - assert.equal(restored, expectedContents[path], `content mismatch for ${path}`); - } + const restored = await readFile(join(syncDir, editedPath), 'utf8'); + assert.equal(restored, editedContent, `content mismatch for ${editedPath}`); } finally { await stopSyncApp(first); await stopSyncApp(second); - for (const path of expectedPaths) { + for (const path of ownedPaths) { await cleanupClient.query(`DELETE FROM ${TABLE} WHERE path = $1`, [path]).catch(() => undefined); } await cleanupClient.disconnect().catch(() => undefined); diff --git a/link/README.md b/link/README.md index a3595efb8..25b37f0dd 100644 --- a/link/README.md +++ b/link/README.md @@ -11,7 +11,9 @@ This directory contains: - the publishable TypeScript CLI wrapper package in [sdks/typescript/cli](sdks/typescript/cli/README.md) as `@kalamdb/cli` - the publishable TypeScript app-facing package in [sdks/typescript/client](sdks/typescript/client/README.md) as `@kalamdb/client` - the publishable TypeScript worker package in [sdks/typescript/consumer](sdks/typescript/consumer/README.md) as `@kalamdb/consumer` -- the publishable Dart/Flutter package in [sdks/dart](sdks/dart/README.md) as `kalam_link` +- the publishable Dart/Flutter transport in [sdks/dart/link](sdks/dart/link/README.md) as `kalam_link` +- the local-first Flutter runtime in [sdks/dart/sync](sdks/dart/sync/README.md) as `kalam_sync` +- the action generator in [sdks/dart/generator](sdks/dart/generator/README.md) as `kalam_sync_generator` ## Canonical SDK Docs @@ -20,7 +22,9 @@ Use the package-specific READMEs as the source of truth for public APIs: - TypeScript / JavaScript app client: [sdks/typescript/client/README.md](sdks/typescript/client/README.md) - TypeScript / JavaScript worker client: [sdks/typescript/consumer/README.md](sdks/typescript/consumer/README.md) - TypeScript / JavaScript CLI wrapper: [sdks/typescript/cli/README.md](sdks/typescript/cli/README.md) -- Dart / Flutter SDK: [sdks/dart/README.md](sdks/dart/README.md) +- Dart / Flutter transport: [sdks/dart/link/README.md](sdks/dart/link/README.md) +- Dart / Flutter local-first sync: [sdks/dart/sync/README.md](sdks/dart/sync/README.md) +- Dart action generation: [sdks/dart/generator/README.md](sdks/dart/generator/README.md) Older constructor-based examples, manual `connect()` walkthroughs, and raw WASM `KalamClient(...)` snippets are not accurate for the current SDKs. @@ -67,7 +71,8 @@ Package-specific build, test, and publish instructions live with each SDK: - TypeScript / JavaScript app client: [sdks/typescript/client/README.md](sdks/typescript/client/README.md) - TypeScript / JavaScript worker client: [sdks/typescript/consumer/README.md](sdks/typescript/consumer/README.md) - TypeScript / JavaScript CLI wrapper: [sdks/typescript/cli/README.md](sdks/typescript/cli/README.md) -- Dart / Flutter: [sdks/dart/README.md](sdks/dart/README.md) +- Dart / Flutter transport: [sdks/dart/link/README.md](sdks/dart/link/README.md) +- Dart / Flutter sync: [sdks/dart/sync/README.md](sdks/dart/sync/README.md) If you change the shared Rust implementation in `link-common` or the `kalam-link-wasm` entry crate, validate the affected SDK package afterward. diff --git a/link/kalam-link-dart/flutter_rust_bridge.yaml b/link/kalam-link-dart/flutter_rust_bridge.yaml index 2d4d08027..c6bae0825 100644 --- a/link/kalam-link-dart/flutter_rust_bridge.yaml +++ b/link/kalam-link-dart/flutter_rust_bridge.yaml @@ -1,4 +1,4 @@ rust_input: crate::api rust_root: . -dart_output: ../sdks/dart/lib/src/generated +dart_output: ../sdks/dart/link/lib/src/generated web: true diff --git a/link/kalam-link-dart/src/api.rs b/link/kalam-link-dart/src/api.rs index 97306c26f..f121e83cc 100644 --- a/link/kalam-link-dart/src/api.rs +++ b/link/kalam-link-dart/src/api.rs @@ -748,6 +748,16 @@ pub async fn dart_live_events_next( } } +/// Acknowledge progress after the consumer durably commits an event. +pub async fn dart_live_events_ack( + subscription: &DartLiveEventsSubscription, + seq_id: i64, +) -> anyhow::Result<()> { + let mut sub = subscription.inner.lock().await; + sub.acknowledge(kalam_client::SeqId::from_i64(seq_id)).await?; + Ok(()) +} + /// Close a subscription and release server-side resources. pub async fn dart_live_events_close( subscription: &DartLiveEventsSubscription, diff --git a/link/kalam-link-dart/src/frb_generated.rs b/link/kalam-link-dart/src/frb_generated.rs index e5fe512a2..ed9d7740c 100644 --- a/link/kalam-link-dart/src/frb_generated.rs +++ b/link/kalam-link-dart/src/frb_generated.rs @@ -26,10 +26,15 @@ // Section: imports +use flutter_rust_bridge::{ + for_generated::{ + byteorder::{NativeEndian, ReadBytesExt, WriteBytesExt}, + transform_result_dco, Lifetimeable, Lockable, + }, + Handler, IntoIntoDart, +}; + use crate::api::*; -use flutter_rust_bridge::for_generated::byteorder::{NativeEndian, ReadBytesExt, WriteBytesExt}; -use flutter_rust_bridge::for_generated::{transform_result_dco, Lifetimeable, Lockable}; -use flutter_rust_bridge::{Handler, IntoIntoDart}; // Section: boilerplate @@ -39,7 +44,7 @@ flutter_rust_bridge::frb_generated_boilerplate!( default_rust_auto_opaque = RustAutoOpaqueMoi, ); pub(crate) const FLUTTER_RUST_BRIDGE_CODEGEN_VERSION: &str = "2.12.0"; -pub(crate) const FLUTTER_RUST_BRIDGE_CODEGEN_CONTENT_HASH: i32 = -212852337; +pub(crate) const FLUTTER_RUST_BRIDGE_CODEGEN_CONTENT_HASH: i32 = -119635461; // Section: executor @@ -56,8 +61,8 @@ fn wire__crate__api__dart_cancel_subscription_impl( FLUTTER_RUST_BRIDGE_HANDLER.wrap_async::( flutter_rust_bridge::for_generated::TaskInfo { debug_name: "dart_cancel_subscription", - port: Some(port_), - mode: flutter_rust_bridge::for_generated::FfiCallMode::Normal, + port: Some(port_), + mode: flutter_rust_bridge::for_generated::FfiCallMode::Normal, }, move || { let message = unsafe { @@ -118,8 +123,8 @@ fn wire__crate__api__dart_connect_impl( FLUTTER_RUST_BRIDGE_HANDLER.wrap_async::( flutter_rust_bridge::for_generated::TaskInfo { debug_name: "dart_connect", - port: Some(port_), - mode: flutter_rust_bridge::for_generated::FfiCallMode::Normal, + port: Some(port_), + mode: flutter_rust_bridge::for_generated::FfiCallMode::Normal, }, move || { let message = unsafe { @@ -174,8 +179,8 @@ fn wire__crate__api__dart_connection_events_enabled_impl( FLUTTER_RUST_BRIDGE_HANDLER.wrap_sync::( flutter_rust_bridge::for_generated::TaskInfo { debug_name: "dart_connection_events_enabled", - port: None, - mode: flutter_rust_bridge::for_generated::FfiCallMode::Sync, + port: None, + mode: flutter_rust_bridge::for_generated::FfiCallMode::Sync, }, move || { let message = unsafe { @@ -225,8 +230,8 @@ fn wire__crate__api__dart_create_client_impl( FLUTTER_RUST_BRIDGE_HANDLER.wrap_normal::( flutter_rust_bridge::for_generated::TaskInfo { debug_name: "dart_create_client", - port: Some(port_), - mode: flutter_rust_bridge::for_generated::FfiCallMode::Normal, + port: Some(port_), + mode: flutter_rust_bridge::for_generated::FfiCallMode::Normal, }, move || { let message = unsafe { @@ -276,8 +281,8 @@ fn wire__crate__api__dart_disconnect_impl( FLUTTER_RUST_BRIDGE_HANDLER.wrap_async::( flutter_rust_bridge::for_generated::TaskInfo { debug_name: "dart_disconnect", - port: Some(port_), - mode: flutter_rust_bridge::for_generated::FfiCallMode::Normal, + port: Some(port_), + mode: flutter_rust_bridge::for_generated::FfiCallMode::Normal, }, move || { let message = unsafe { @@ -333,8 +338,8 @@ fn wire__crate__api__dart_download_file_impl( FLUTTER_RUST_BRIDGE_HANDLER.wrap_async::( flutter_rust_bridge::for_generated::TaskInfo { debug_name: "dart_download_file", - port: Some(port_), - mode: flutter_rust_bridge::for_generated::FfiCallMode::Normal, + port: Some(port_), + mode: flutter_rust_bridge::for_generated::FfiCallMode::Normal, }, move || { let message = unsafe { @@ -401,8 +406,8 @@ fn wire__crate__api__dart_execute_query_impl( FLUTTER_RUST_BRIDGE_HANDLER.wrap_async::( flutter_rust_bridge::for_generated::TaskInfo { debug_name: "dart_execute_query", - port: Some(port_), - mode: flutter_rust_bridge::for_generated::FfiCallMode::Normal, + port: Some(port_), + mode: flutter_rust_bridge::for_generated::FfiCallMode::Normal, }, move || { let message = unsafe { @@ -467,8 +472,8 @@ fn wire__crate__api__dart_execute_query_with_files_impl( FLUTTER_RUST_BRIDGE_HANDLER.wrap_async::( flutter_rust_bridge::for_generated::TaskInfo { debug_name: "dart_execute_query_with_files", - port: Some(port_), - mode: flutter_rust_bridge::for_generated::FfiCallMode::Normal, + port: Some(port_), + mode: flutter_rust_bridge::for_generated::FfiCallMode::Normal, }, move || { let message = unsafe { @@ -534,8 +539,8 @@ fn wire__crate__api__dart_file_ref_download_url_impl( FLUTTER_RUST_BRIDGE_HANDLER.wrap_sync::( flutter_rust_bridge::for_generated::TaskInfo { debug_name: "dart_file_ref_download_url", - port: None, - mode: flutter_rust_bridge::for_generated::FfiCallMode::Sync, + port: None, + mode: flutter_rust_bridge::for_generated::FfiCallMode::Sync, }, move || { let message = unsafe { @@ -574,8 +579,8 @@ fn wire__crate__api__dart_file_ref_relative_path_impl( FLUTTER_RUST_BRIDGE_HANDLER.wrap_sync::( flutter_rust_bridge::for_generated::TaskInfo { debug_name: "dart_file_ref_relative_path", - port: None, - mode: flutter_rust_bridge::for_generated::FfiCallMode::Sync, + port: None, + mode: flutter_rust_bridge::for_generated::FfiCallMode::Sync, }, move || { let message = unsafe { @@ -606,8 +611,8 @@ fn wire__crate__api__dart_file_ref_relative_url_impl( FLUTTER_RUST_BRIDGE_HANDLER.wrap_sync::( flutter_rust_bridge::for_generated::TaskInfo { debug_name: "dart_file_ref_relative_url", - port: None, - mode: flutter_rust_bridge::for_generated::FfiCallMode::Sync, + port: None, + mode: flutter_rust_bridge::for_generated::FfiCallMode::Sync, }, move || { let message = unsafe { @@ -644,8 +649,8 @@ fn wire__crate__api__dart_file_ref_stored_name_impl( FLUTTER_RUST_BRIDGE_HANDLER.wrap_sync::( flutter_rust_bridge::for_generated::TaskInfo { debug_name: "dart_file_ref_stored_name", - port: None, - mode: flutter_rust_bridge::for_generated::FfiCallMode::Sync, + port: None, + mode: flutter_rust_bridge::for_generated::FfiCallMode::Sync, }, move || { let message = unsafe { @@ -677,8 +682,8 @@ fn wire__crate__api__dart_is_connected_impl( FLUTTER_RUST_BRIDGE_HANDLER.wrap_async::( flutter_rust_bridge::for_generated::TaskInfo { debug_name: "dart_is_connected", - port: Some(port_), - mode: flutter_rust_bridge::for_generated::FfiCallMode::Normal, + port: Some(port_), + mode: flutter_rust_bridge::for_generated::FfiCallMode::Normal, }, move || { let message = unsafe { @@ -734,8 +739,8 @@ fn wire__crate__api__dart_list_subscriptions_impl( FLUTTER_RUST_BRIDGE_HANDLER.wrap_async::( flutter_rust_bridge::for_generated::TaskInfo { debug_name: "dart_list_subscriptions", - port: Some(port_), - mode: flutter_rust_bridge::for_generated::FfiCallMode::Normal, + port: Some(port_), + mode: flutter_rust_bridge::for_generated::FfiCallMode::Normal, }, move || { let message = unsafe { @@ -792,8 +797,8 @@ fn wire__crate__api__dart_live_close_impl( FLUTTER_RUST_BRIDGE_HANDLER.wrap_async::( flutter_rust_bridge::for_generated::TaskInfo { debug_name: "dart_live_close", - port: Some(port_), - mode: flutter_rust_bridge::for_generated::FfiCallMode::Normal, + port: Some(port_), + mode: flutter_rust_bridge::for_generated::FfiCallMode::Normal, }, move || { let message = unsafe { @@ -841,6 +846,66 @@ fn wire__crate__api__dart_live_close_impl( }, ) } +fn wire__crate__api__dart_live_events_ack_impl( + port_: flutter_rust_bridge::for_generated::MessagePort, + ptr_: flutter_rust_bridge::for_generated::PlatformGeneralizedUint8ListPtr, + rust_vec_len_: i32, + data_len_: i32, +) { + FLUTTER_RUST_BRIDGE_HANDLER.wrap_async::( + flutter_rust_bridge::for_generated::TaskInfo { + debug_name: "dart_live_events_ack", + port: Some(port_), + mode: flutter_rust_bridge::for_generated::FfiCallMode::Normal, + }, + move || { + let message = unsafe { + flutter_rust_bridge::for_generated::Dart2RustMessageSse::from_wire( + ptr_, + rust_vec_len_, + data_len_, + ) + }; + let mut deserializer = + flutter_rust_bridge::for_generated::SseDeserializer::new(message); + let api_subscription = , + >>::sse_decode(&mut deserializer); + let api_seq_id = ::sse_decode(&mut deserializer); + deserializer.end(); + move |context| async move { + transform_result_sse::<_, flutter_rust_bridge::for_generated::anyhow::Error>( + (move || async move { + let mut api_subscription_guard = None; + let decode_indices_ = + flutter_rust_bridge::for_generated::lockable_compute_decode_order( + vec![flutter_rust_bridge::for_generated::LockableOrderInfo::new( + &api_subscription, + 0, + false, + )], + ); + for i in decode_indices_ { + match i { + 0 => { + api_subscription_guard = + Some(api_subscription.lockable_decode_async_ref().await) + }, + _ => unreachable!(), + } + } + let api_subscription_guard = api_subscription_guard.unwrap(); + let output_ok = + crate::api::dart_live_events_ack(&*api_subscription_guard, api_seq_id) + .await?; + Ok(output_ok) + })() + .await, + ) + } + }, + ) +} fn wire__crate__api__dart_live_events_close_impl( port_: flutter_rust_bridge::for_generated::MessagePort, ptr_: flutter_rust_bridge::for_generated::PlatformGeneralizedUint8ListPtr, @@ -850,8 +915,8 @@ fn wire__crate__api__dart_live_events_close_impl( FLUTTER_RUST_BRIDGE_HANDLER.wrap_async::( flutter_rust_bridge::for_generated::TaskInfo { debug_name: "dart_live_events_close", - port: Some(port_), - mode: flutter_rust_bridge::for_generated::FfiCallMode::Normal, + port: Some(port_), + mode: flutter_rust_bridge::for_generated::FfiCallMode::Normal, }, move || { let message = unsafe { @@ -907,8 +972,8 @@ fn wire__crate__api__dart_live_events_id_impl( FLUTTER_RUST_BRIDGE_HANDLER.wrap_sync::( flutter_rust_bridge::for_generated::TaskInfo { debug_name: "dart_live_events_id", - port: None, - mode: flutter_rust_bridge::for_generated::FfiCallMode::Sync, + port: None, + mode: flutter_rust_bridge::for_generated::FfiCallMode::Sync, }, move || { let message = unsafe { @@ -960,8 +1025,8 @@ fn wire__crate__api__dart_live_events_next_impl( FLUTTER_RUST_BRIDGE_HANDLER.wrap_async::( flutter_rust_bridge::for_generated::TaskInfo { debug_name: "dart_live_events_next", - port: Some(port_), - mode: flutter_rust_bridge::for_generated::FfiCallMode::Normal, + port: Some(port_), + mode: flutter_rust_bridge::for_generated::FfiCallMode::Normal, }, move || { let message = unsafe { @@ -1018,8 +1083,8 @@ fn wire__crate__api__dart_live_events_subscribe_impl( FLUTTER_RUST_BRIDGE_HANDLER.wrap_async::( flutter_rust_bridge::for_generated::TaskInfo { debug_name: "dart_live_events_subscribe", - port: Some(port_), - mode: flutter_rust_bridge::for_generated::FfiCallMode::Normal, + port: Some(port_), + mode: flutter_rust_bridge::for_generated::FfiCallMode::Normal, }, move || { let message = unsafe { @@ -1082,8 +1147,8 @@ fn wire__crate__api__dart_live_id_impl( FLUTTER_RUST_BRIDGE_HANDLER.wrap_sync::( flutter_rust_bridge::for_generated::TaskInfo { debug_name: "dart_live_id", - port: None, - mode: flutter_rust_bridge::for_generated::FfiCallMode::Sync, + port: None, + mode: flutter_rust_bridge::for_generated::FfiCallMode::Sync, }, move || { let message = unsafe { @@ -1135,8 +1200,8 @@ fn wire__crate__api__dart_live_next_impl( FLUTTER_RUST_BRIDGE_HANDLER.wrap_async::( flutter_rust_bridge::for_generated::TaskInfo { debug_name: "dart_live_next", - port: Some(port_), - mode: flutter_rust_bridge::for_generated::FfiCallMode::Normal, + port: Some(port_), + mode: flutter_rust_bridge::for_generated::FfiCallMode::Normal, }, move || { let message = unsafe { @@ -1193,8 +1258,8 @@ fn wire__crate__api__dart_live_subscribe_impl( FLUTTER_RUST_BRIDGE_HANDLER.wrap_async::( flutter_rust_bridge::for_generated::TaskInfo { debug_name: "dart_live_subscribe", - port: Some(port_), - mode: flutter_rust_bridge::for_generated::FfiCallMode::Normal, + port: Some(port_), + mode: flutter_rust_bridge::for_generated::FfiCallMode::Normal, }, move || { let message = unsafe { @@ -1261,8 +1326,8 @@ fn wire__crate__api__dart_login_impl( FLUTTER_RUST_BRIDGE_HANDLER.wrap_async::( flutter_rust_bridge::for_generated::TaskInfo { debug_name: "dart_login", - port: Some(port_), - mode: flutter_rust_bridge::for_generated::FfiCallMode::Normal, + port: Some(port_), + mode: flutter_rust_bridge::for_generated::FfiCallMode::Normal, }, move || { let message = unsafe { @@ -1322,8 +1387,8 @@ fn wire__crate__api__dart_next_connection_event_impl( FLUTTER_RUST_BRIDGE_HANDLER.wrap_async::( flutter_rust_bridge::for_generated::TaskInfo { debug_name: "dart_next_connection_event", - port: Some(port_), - mode: flutter_rust_bridge::for_generated::FfiCallMode::Normal, + port: Some(port_), + mode: flutter_rust_bridge::for_generated::FfiCallMode::Normal, }, move || { let message = unsafe { @@ -1379,8 +1444,8 @@ fn wire__crate__api__dart_parse_file_ref_impl( FLUTTER_RUST_BRIDGE_HANDLER.wrap_sync::( flutter_rust_bridge::for_generated::TaskInfo { debug_name: "dart_parse_file_ref", - port: None, - mode: flutter_rust_bridge::for_generated::FfiCallMode::Sync, + port: None, + mode: flutter_rust_bridge::for_generated::FfiCallMode::Sync, }, move || { let message = unsafe { @@ -1412,8 +1477,8 @@ fn wire__crate__api__dart_refresh_token_impl( FLUTTER_RUST_BRIDGE_HANDLER.wrap_async::( flutter_rust_bridge::for_generated::TaskInfo { debug_name: "dart_refresh_token", - port: Some(port_), - mode: flutter_rust_bridge::for_generated::FfiCallMode::Normal, + port: Some(port_), + mode: flutter_rust_bridge::for_generated::FfiCallMode::Normal, }, move || { let message = unsafe { @@ -1471,8 +1536,8 @@ fn wire__crate__api__dart_signal_dispose_impl( FLUTTER_RUST_BRIDGE_HANDLER.wrap_sync::( flutter_rust_bridge::for_generated::TaskInfo { debug_name: "dart_signal_dispose", - port: None, - mode: flutter_rust_bridge::for_generated::FfiCallMode::Sync, + port: None, + mode: flutter_rust_bridge::for_generated::FfiCallMode::Sync, }, move || { let message = unsafe { @@ -1521,8 +1586,8 @@ fn wire__crate__api__dart_try_parse_file_ref_impl( FLUTTER_RUST_BRIDGE_HANDLER.wrap_sync::( flutter_rust_bridge::for_generated::TaskInfo { debug_name: "dart_try_parse_file_ref", - port: None, - mode: flutter_rust_bridge::for_generated::FfiCallMode::Sync, + port: None, + mode: flutter_rust_bridge::for_generated::FfiCallMode::Sync, }, move || { let message = unsafe { @@ -1553,8 +1618,8 @@ fn wire__crate__api__dart_update_auth_impl( FLUTTER_RUST_BRIDGE_HANDLER.wrap_normal::( flutter_rust_bridge::for_generated::TaskInfo { debug_name: "dart_update_auth", - port: Some(port_), - mode: flutter_rust_bridge::for_generated::FfiCallMode::Normal, + port: Some(port_), + mode: flutter_rust_bridge::for_generated::FfiCallMode::Normal, }, move || { let message = unsafe { @@ -1709,7 +1774,7 @@ impl SseDecode for crate::models::DartAuthProvider { let mut var_user = ::sse_decode(deserializer); let mut var_password = ::sse_decode(deserializer); return crate::models::DartAuthProvider::BasicAuth { - user: var_user, + user: var_user, password: var_password, }; }, @@ -1742,11 +1807,11 @@ impl SseDecode for crate::models::DartChangeEvent { let mut var_status = ::sse_decode(deserializer); return crate::models::DartChangeEvent::Ack { subscription_id: var_subscriptionId, - total_rows: var_totalRows, - schema: var_schema, - batch_num: var_batchNum, - has_more: var_hasMore, - status: var_status, + total_rows: var_totalRows, + schema: var_schema, + batch_num: var_batchNum, + has_more: var_hasMore, + status: var_status, }; }, 1 => { @@ -1757,10 +1822,10 @@ impl SseDecode for crate::models::DartChangeEvent { let mut var_status = ::sse_decode(deserializer); return crate::models::DartChangeEvent::InitialDataBatch { subscription_id: var_subscriptionId, - rows_json: var_rowsJson, - batch_num: var_batchNum, - has_more: var_hasMore, - status: var_status, + rows_json: var_rowsJson, + batch_num: var_batchNum, + has_more: var_hasMore, + status: var_status, }; }, 2 => { @@ -1768,7 +1833,7 @@ impl SseDecode for crate::models::DartChangeEvent { let mut var_rowsJson = >::sse_decode(deserializer); return crate::models::DartChangeEvent::Insert { subscription_id: var_subscriptionId, - rows_json: var_rowsJson, + rows_json: var_rowsJson, }; }, 3 => { @@ -1777,8 +1842,8 @@ impl SseDecode for crate::models::DartChangeEvent { let mut var_oldRowsJson = >::sse_decode(deserializer); return crate::models::DartChangeEvent::Update { subscription_id: var_subscriptionId, - rows_json: var_rowsJson, - old_rows_json: var_oldRowsJson, + rows_json: var_rowsJson, + old_rows_json: var_oldRowsJson, }; }, 4 => { @@ -1786,7 +1851,7 @@ impl SseDecode for crate::models::DartChangeEvent { let mut var_oldRowsJson = >::sse_decode(deserializer); return crate::models::DartChangeEvent::Delete { subscription_id: var_subscriptionId, - old_rows_json: var_oldRowsJson, + old_rows_json: var_oldRowsJson, }; }, 5 => { @@ -1795,8 +1860,8 @@ impl SseDecode for crate::models::DartChangeEvent { let mut var_message = ::sse_decode(deserializer); return crate::models::DartChangeEvent::Error { subscription_id: var_subscriptionId, - code: var_code, - message: var_message, + code: var_code, + message: var_message, }; }, _ => { @@ -1812,7 +1877,7 @@ impl SseDecode for crate::models::DartConnectionError { let mut var_message = ::sse_decode(deserializer); let mut var_recoverable = ::sse_decode(deserializer); return crate::models::DartConnectionError { - message: var_message, + message: var_message, recoverable: var_recoverable, }; } @@ -1861,7 +1926,7 @@ impl SseDecode for crate::models::DartDisconnectReason { let mut var_code = >::sse_decode(deserializer); return crate::models::DartDisconnectReason { message: var_message, - code: var_code, + code: var_code, }; } } @@ -1873,7 +1938,7 @@ impl SseDecode for crate::models::DartErrorDetail { let mut var_message = ::sse_decode(deserializer); let mut var_details = >::sse_decode(deserializer); return crate::models::DartErrorDetail { - code: var_code, + code: var_code, message: var_message, details: var_details, }; @@ -1887,8 +1952,8 @@ impl SseDecode for crate::models::DartFileDownload { let mut var_contentType = >::sse_decode(deserializer); let mut var_contentDisposition = >::sse_decode(deserializer); return crate::models::DartFileDownload { - bytes: var_bytes, - content_type: var_contentType, + bytes: var_bytes, + content_type: var_contentType, content_disposition: var_contentDisposition, }; } @@ -1905,13 +1970,13 @@ impl SseDecode for crate::models::DartFileRef { let mut var_sha256 = ::sse_decode(deserializer); let mut var_shard = >::sse_decode(deserializer); return crate::models::DartFileRef { - id: var_id, - sub: var_sub, - name: var_name, - size: var_size, - mime: var_mime, + id: var_id, + sub: var_sub, + name: var_name, + size: var_size, + mime: var_mime, sha256: var_sha256, - shard: var_shard, + shard: var_shard, }; } } @@ -1925,9 +1990,9 @@ impl SseDecode for crate::models::DartFileUpload { let mut var_mime = >::sse_decode(deserializer); return crate::models::DartFileUpload { placeholder: var_placeholder, - filename: var_filename, - data: var_data, - mime: var_mime, + filename: var_filename, + data: var_data, + mime: var_mime, }; } } @@ -1938,7 +2003,7 @@ impl SseDecode for crate::models::DartLiveRowsConfig { let mut var_limit = >::sse_decode(deserializer); let mut var_keyColumns = >>::sse_decode(deserializer); return crate::models::DartLiveRowsConfig { - limit: var_limit, + limit: var_limit, key_columns: var_keyColumns, }; } @@ -1955,8 +2020,8 @@ impl SseDecode for crate::models::DartLiveRowsEvent { let mut var_lastSeqId = >::sse_decode(deserializer); return crate::models::DartLiveRowsEvent::Rows { subscription_id: var_subscriptionId, - rows_json: var_rowsJson, - last_seq_id: var_lastSeqId, + rows_json: var_rowsJson, + last_seq_id: var_lastSeqId, }; }, 1 => { @@ -1965,8 +2030,8 @@ impl SseDecode for crate::models::DartLiveRowsEvent { let mut var_message = ::sse_decode(deserializer); return crate::models::DartLiveRowsEvent::Error { subscription_id: var_subscriptionId, - code: var_code, - message: var_message, + code: var_code, + message: var_message, }; }, _ => { @@ -1986,12 +2051,12 @@ impl SseDecode for crate::models::DartLoginResponse { let mut var_adminUiAccess = ::sse_decode(deserializer); let mut var_user = ::sse_decode(deserializer); return crate::models::DartLoginResponse { - access_token: var_accessToken, - refresh_token: var_refreshToken, - expires_at: var_expiresAt, + access_token: var_accessToken, + refresh_token: var_refreshToken, + expires_at: var_expiresAt, refresh_expires_at: var_refreshExpiresAt, - admin_ui_access: var_adminUiAccess, - user: var_user, + admin_ui_access: var_adminUiAccess, + user: var_user, }; } } @@ -2005,9 +2070,9 @@ impl SseDecode for crate::models::DartLoginUserInfo { let mut var_createdAt = ::sse_decode(deserializer); let mut var_updatedAt = ::sse_decode(deserializer); return crate::models::DartLoginUserInfo { - id: var_id, - role: var_role, - email: var_email, + id: var_id, + role: var_role, + email: var_email, created_at: var_createdAt, updated_at: var_updatedAt, }; @@ -2025,7 +2090,7 @@ impl SseDecode for crate::models::DartQueryResponse { success: var_success, results: var_results, took_ms: var_tookMs, - error: var_error, + error: var_error, }; } } @@ -2039,11 +2104,11 @@ impl SseDecode for crate::models::DartQueryResult { let mut var_rowCount = ::sse_decode(deserializer); let mut var_message = >::sse_decode(deserializer); return crate::models::DartQueryResult { - columns: var_columns, - rows_json: var_rowsJson, + columns: var_columns, + rows_json: var_rowsJson, named_rows_json: var_namedRowsJson, - row_count: var_rowCount, - message: var_message, + row_count: var_rowCount, + message: var_message, }; } } @@ -2071,10 +2136,10 @@ impl SseDecode for crate::models::DartSchemaField { let mut var_index = ::sse_decode(deserializer); let mut var_flags = >::sse_decode(deserializer); return crate::models::DartSchemaField { - name: var_name, + name: var_name, data_type: var_dataType, - index: var_index, - flags: var_flags, + index: var_index, + flags: var_flags, }; } } @@ -2087,12 +2152,14 @@ impl SseDecode for crate::models::DartSubscriptionConfig { let mut var_batchSize = >::sse_decode(deserializer); let mut var_lastRows = >::sse_decode(deserializer); let mut var_from = >::sse_decode(deserializer); + let mut var_explicitAck = ::sse_decode(deserializer); return crate::models::DartSubscriptionConfig { - sql: var_sql, - id: var_id, - batch_size: var_batchSize, - last_rows: var_lastRows, - from: var_from, + sql: var_sql, + id: var_id, + batch_size: var_batchSize, + last_rows: var_lastRows, + from: var_from, + explicit_ack: var_explicitAck, }; } } @@ -2107,12 +2174,12 @@ impl SseDecode for crate::models::DartSubscriptionInfo { let mut var_createdAtMs = ::sse_decode(deserializer); let mut var_closed = ::sse_decode(deserializer); return crate::models::DartSubscriptionInfo { - id: var_id, - query: var_query, - last_seq_id: var_lastSeqId, + id: var_id, + query: var_query, + last_seq_id: var_lastSeqId, last_event_time_ms: var_lastEventTimeMs, - created_at_ms: var_createdAtMs, - closed: var_closed, + created_at_ms: var_createdAtMs, + closed: var_closed, }; } } @@ -2418,15 +2485,16 @@ fn pde_ffi_dispatcher_primary_impl( 13 => wire__crate__api__dart_is_connected_impl(port, ptr, rust_vec_len, data_len), 14 => wire__crate__api__dart_list_subscriptions_impl(port, ptr, rust_vec_len, data_len), 15 => wire__crate__api__dart_live_close_impl(port, ptr, rust_vec_len, data_len), - 16 => wire__crate__api__dart_live_events_close_impl(port, ptr, rust_vec_len, data_len), - 18 => wire__crate__api__dart_live_events_next_impl(port, ptr, rust_vec_len, data_len), - 19 => wire__crate__api__dart_live_events_subscribe_impl(port, ptr, rust_vec_len, data_len), - 21 => wire__crate__api__dart_live_next_impl(port, ptr, rust_vec_len, data_len), - 22 => wire__crate__api__dart_live_subscribe_impl(port, ptr, rust_vec_len, data_len), - 23 => wire__crate__api__dart_login_impl(port, ptr, rust_vec_len, data_len), - 24 => wire__crate__api__dart_next_connection_event_impl(port, ptr, rust_vec_len, data_len), - 26 => wire__crate__api__dart_refresh_token_impl(port, ptr, rust_vec_len, data_len), - 29 => wire__crate__api__dart_update_auth_impl(port, ptr, rust_vec_len, data_len), + 16 => wire__crate__api__dart_live_events_ack_impl(port, ptr, rust_vec_len, data_len), + 17 => wire__crate__api__dart_live_events_close_impl(port, ptr, rust_vec_len, data_len), + 19 => wire__crate__api__dart_live_events_next_impl(port, ptr, rust_vec_len, data_len), + 20 => wire__crate__api__dart_live_events_subscribe_impl(port, ptr, rust_vec_len, data_len), + 22 => wire__crate__api__dart_live_next_impl(port, ptr, rust_vec_len, data_len), + 23 => wire__crate__api__dart_live_subscribe_impl(port, ptr, rust_vec_len, data_len), + 24 => wire__crate__api__dart_login_impl(port, ptr, rust_vec_len, data_len), + 25 => wire__crate__api__dart_next_connection_event_impl(port, ptr, rust_vec_len, data_len), + 27 => wire__crate__api__dart_refresh_token_impl(port, ptr, rust_vec_len, data_len), + 30 => wire__crate__api__dart_update_auth_impl(port, ptr, rust_vec_len, data_len), _ => unreachable!(), } } @@ -2444,11 +2512,11 @@ fn pde_ffi_dispatcher_sync_impl( 10 => wire__crate__api__dart_file_ref_relative_path_impl(ptr, rust_vec_len, data_len), 11 => wire__crate__api__dart_file_ref_relative_url_impl(ptr, rust_vec_len, data_len), 12 => wire__crate__api__dart_file_ref_stored_name_impl(ptr, rust_vec_len, data_len), - 17 => wire__crate__api__dart_live_events_id_impl(ptr, rust_vec_len, data_len), - 20 => wire__crate__api__dart_live_id_impl(ptr, rust_vec_len, data_len), - 25 => wire__crate__api__dart_parse_file_ref_impl(ptr, rust_vec_len, data_len), - 27 => wire__crate__api__dart_signal_dispose_impl(ptr, rust_vec_len, data_len), - 28 => wire__crate__api__dart_try_parse_file_ref_impl(ptr, rust_vec_len, data_len), + 18 => wire__crate__api__dart_live_events_id_impl(ptr, rust_vec_len, data_len), + 21 => wire__crate__api__dart_live_id_impl(ptr, rust_vec_len, data_len), + 26 => wire__crate__api__dart_parse_file_ref_impl(ptr, rust_vec_len, data_len), + 28 => wire__crate__api__dart_signal_dispose_impl(ptr, rust_vec_len, data_len), + 29 => wire__crate__api__dart_try_parse_file_ref_impl(ptr, rust_vec_len, data_len), _ => unreachable!(), } } @@ -3006,6 +3074,7 @@ impl flutter_rust_bridge::IntoDart for crate::models::DartSubscriptionConfig { self.batch_size.into_into_dart().into_dart(), self.last_rows.into_into_dart().into_dart(), self.from.into_into_dart().into_dart(), + self.explicit_ack.into_into_dart().into_dart(), ] .into_dart() } @@ -3443,6 +3512,7 @@ impl SseEncode for crate::models::DartSubscriptionConfig { >::sse_encode(self.batch_size, serializer); >::sse_encode(self.last_rows, serializer); >::sse_encode(self.from, serializer); + ::sse_encode(self.explicit_ack, serializer); } } @@ -3719,13 +3789,16 @@ mod io { // Section: imports + use flutter_rust_bridge::{ + for_generated::{ + byteorder::{NativeEndian, ReadBytesExt, WriteBytesExt}, + transform_result_dco, Lifetimeable, Lockable, + }, + Handler, IntoIntoDart, + }; + use super::*; use crate::api::*; - use flutter_rust_bridge::for_generated::byteorder::{ - NativeEndian, ReadBytesExt, WriteBytesExt, - }; - use flutter_rust_bridge::for_generated::{transform_result_dco, Lifetimeable, Lockable}; - use flutter_rust_bridge::{Handler, IntoIntoDart}; // Section: boilerplate @@ -3784,15 +3857,18 @@ mod web { // Section: imports + use flutter_rust_bridge::{ + for_generated::{ + byteorder::{NativeEndian, ReadBytesExt, WriteBytesExt}, + transform_result_dco, wasm_bindgen, + wasm_bindgen::prelude::*, + Lifetimeable, Lockable, + }, + Handler, IntoIntoDart, + }; + use super::*; use crate::api::*; - use flutter_rust_bridge::for_generated::byteorder::{ - NativeEndian, ReadBytesExt, WriteBytesExt, - }; - use flutter_rust_bridge::for_generated::wasm_bindgen; - use flutter_rust_bridge::for_generated::wasm_bindgen::prelude::*; - use flutter_rust_bridge::for_generated::{transform_result_dco, Lifetimeable, Lockable}; - use flutter_rust_bridge::{Handler, IntoIntoDart}; // Section: boilerplate diff --git a/link/kalam-link-dart/src/models.rs b/link/kalam-link-dart/src/models.rs index f787bb16d..2030388a4 100644 --- a/link/kalam-link-dart/src/models.rs +++ b/link/kalam-link-dart/src/models.rs @@ -473,6 +473,8 @@ pub struct DartSubscriptionConfig { /// Resume from a specific sequence ID. /// When set, the server only sends changes after this seq_id. pub from: Option, + /// Require explicit consumer acknowledgement before reconnect progress advances. + pub explicit_ack: bool, } impl DartSubscriptionConfig { @@ -487,6 +489,11 @@ impl DartSubscriptionConfig { auto_fetch_batches: None, }), ws_url: None, + ack_mode: if self.explicit_ack { + kalam_client::SubscriptionAckMode::Explicit + } else { + kalam_client::SubscriptionAckMode::Automatic + }, } } } diff --git a/link/kalam-link-dart/src/tests.rs b/link/kalam-link-dart/src/tests.rs index d62641ddd..fcfe90405 100644 --- a/link/kalam-link-dart/src/tests.rs +++ b/link/kalam-link-dart/src/tests.rs @@ -429,6 +429,7 @@ mod tests { batch_size: None, last_rows: None, from: None, + explicit_ack: false, }; let native = cfg.into_native(); assert_eq!(native.sql, "SELECT * FROM t"); @@ -444,6 +445,7 @@ mod tests { batch_size: Some(500), last_rows: Some(10), from: Some(42), + explicit_ack: false, }; let native = cfg.into_native(); assert_eq!(native.id, "my-sub-1"); @@ -462,6 +464,7 @@ mod tests { batch_size: None, last_rows: None, from: Some(12345), + explicit_ack: false, }; let native = cfg.into_native(); assert_eq!(native.sql, "SELECT * FROM events"); @@ -471,6 +474,23 @@ mod tests { assert_eq!(opts.from.unwrap().as_i64(), 12345); } + #[test] + fn subscription_config_enables_explicit_acknowledgement() { + let cfg = DartSubscriptionConfig { + sql: "SELECT * FROM events".into(), + id: Some("durable-events".into()), + batch_size: Some(250), + last_rows: None, + from: Some(41), + explicit_ack: true, + }; + + let native = cfg.into_native(); + + assert_eq!(native.ack_mode, kalam_client::SubscriptionAckMode::Explicit); + assert_eq!(native.options.unwrap().from.unwrap().as_i64(), 41); + } + // ----------------------------------------------------------------------- // DartSubscriptionInfo conversion // ----------------------------------------------------------------------- diff --git a/link/link-common/src/client/runtime.rs b/link/link-common/src/client/runtime.rs index a59e58503..fc9f3a60f 100644 --- a/link/link-common/src/client/runtime.rs +++ b/link/link-common/src/client/runtime.rs @@ -186,6 +186,7 @@ impl KalamLinkClient { shared_control, generation, resume_from, + config.ack_mode, &self.timeouts, )); } diff --git a/link/link-common/src/lib.rs b/link/link-common/src/lib.rs index ddf837d44..e4ae82592 100644 --- a/link/link-common/src/lib.rs +++ b/link/link-common/src/lib.rs @@ -67,7 +67,8 @@ pub use models::{ HealthCheckResponse, HttpVersion, LoginRequest, LoginResponse, LoginUserInfo, QueryRequest, QueryResponse, QueryResult, ServerSetupRequest, ServerSetupResponse, SetupStatusResponse, SetupUserInfo, SqlSubscriptionDescriptor, SqlSubscriptionRow, SqlSubscriptionStatus, - SubscriptionConfig, SubscriptionInfo, SubscriptionOptions, UploadProgress, + SubscriptionAckMode, SubscriptionConfig, SubscriptionInfo, SubscriptionOptions, + UploadProgress, }; #[cfg(feature = "client-core")] pub use query::models::QueryParam; diff --git a/link/link-common/src/models/mod.rs b/link/link-common/src/models/mod.rs index 375c89b67..517240b09 100644 --- a/link/link-common/src/models/mod.rs +++ b/link/link-common/src/models/mod.rs @@ -50,6 +50,6 @@ pub use crate::query::models::{ // ── Subscription models ────────────────────────────────────────────────────── #[cfg(feature = "client-core")] pub use crate::subscription::models::{ - BatchControl, BatchStatus, ChangeEvent, ChangeTypeRaw, SubscriptionConfig, SubscriptionInfo, - SubscriptionOptions, SubscriptionRequest, + BatchControl, BatchStatus, ChangeEvent, ChangeTypeRaw, SubscriptionAckMode, + SubscriptionConfig, SubscriptionInfo, SubscriptionOptions, SubscriptionRequest, }; diff --git a/link/link-common/src/subscription/manager.rs b/link/link-common/src/subscription/manager.rs index 9e3818bd4..3512855f4 100644 --- a/link/link-common/src/subscription/manager.rs +++ b/link/link-common/src/subscription/manager.rs @@ -12,7 +12,7 @@ use crate::{ error::Result, models::ChangeEvent, seq_tracking, - subscription::{buffer_event, event_progress}, + subscription::{buffer_event, event_progress, SubscriptionAckMode}, timeouts::KalamLinkTimeouts, SeqId, }; @@ -54,6 +54,9 @@ pub struct SubscriptionManager { is_loading: bool, /// Original `from` cursor used to open this subscription, if any. resume_from: Option, + /// Highest progress delivered to the consumer, acknowledged or not. + delivered_seq_id: Option, + ack_mode: SubscriptionAckMode, timeouts: KalamLinkTimeouts, closed: bool, } @@ -68,6 +71,7 @@ impl SubscriptionManager { shared_control: SharedSubscriptionControl, generation: u64, resume_from: Option, + ack_mode: SubscriptionAckMode, timeouts: &KalamLinkTimeouts, ) -> Self { Self { @@ -79,6 +83,8 @@ impl SubscriptionManager { buffered_changes: Vec::new(), is_loading: true, resume_from, + delivered_seq_id: resume_from, + ack_mode, timeouts: timeouts.clone(), closed: false, } @@ -105,6 +111,12 @@ impl SubscriptionManager { .await; } + fn record_delivery(&mut self, event: &ChangeEvent) { + if let Some(progress) = event_progress(event) { + seq_tracking::advance_seq(&mut self.delivered_seq_id, progress.seq_id); + } + } + /// Buffer incoming events: hold live changes while initial data is loading, /// then flush them in order once the snapshot is complete. fn apply_buffering(&mut self, event: ChangeEvent) { @@ -124,7 +136,10 @@ impl SubscriptionManager { loop { // 1. Drain local event queue first if let Some(event) = self.event_queue.pop_front() { - self.report_shared_progress(&event).await; + self.record_delivery(&event); + if self.ack_mode == SubscriptionAckMode::Automatic { + self.report_shared_progress(&event).await; + } return Some(Ok(event)); } @@ -148,6 +163,28 @@ impl SubscriptionManager { } } + /// Advance explicit subscription progress after durable consumer work commits. + pub async fn acknowledge(&mut self, seq_id: SeqId) -> Result<()> { + if self.ack_mode != SubscriptionAckMode::Explicit { + return Err(crate::error::KalamLinkError::ConfigurationError( + "explicit acknowledgement is not enabled for this subscription".to_string(), + )); + } + if self.delivered_seq_id.is_none_or(|delivered| seq_id > delivered) { + return Err(crate::error::KalamLinkError::ConfigurationError(format!( + "sequence {seq_id} was not delivered to this subscription" + ))); + } + + seq_tracking::advance_seq(&mut self.resume_from, seq_id); + if let Some(shared_control) = self.shared_control.as_ref() { + shared_control + .progress(self.subscription_id.clone(), self.generation, seq_id, true) + .await; + } + Ok(()) + } + /// Get the subscription ID assigned by the server pub fn subscription_id(&self) -> &str { &self.subscription_id @@ -191,6 +228,7 @@ impl Drop for SubscriptionManager { #[cfg(test)] mod tests { use super::*; + use crate::subscription::SubscriptionAckMode; /// Create a minimal `SubscriptionManager` with no live shared connection /// for testing state-flag logic without a network dependency. @@ -204,6 +242,7 @@ mod tests { SharedSubscriptionControl::test_control(), 0, None, + SubscriptionAckMode::Automatic, &KalamLinkTimeouts::default(), ); subscription.is_loading = false; @@ -291,6 +330,42 @@ mod tests { assert!(sub.buffered_changes.is_empty()); } + #[tokio::test] + async fn test_explicit_ack_does_not_advance_resume_before_acknowledgement() { + let mut sub = make_test_sub(); + sub.ack_mode = SubscriptionAckMode::Explicit; + sub.event_queue.push_back(ChangeEvent::Insert { + subscription_id: "unit-test-id".to_string(), + rows: vec![{ + let mut row = std::collections::HashMap::new(); + row.insert("id".to_string(), crate::models::KalamCellValue::text("one")); + row.insert("_seq".to_string(), crate::models::KalamCellValue::text("10")); + row + }], + }); + + let event = sub.next().await.expect("event").expect("valid event"); + assert!(matches!(event, ChangeEvent::Insert { .. })); + assert_eq!(sub.resume_from, None, "delivery must not acknowledge progress"); + + sub.acknowledge(SeqId::from_i64(10)).await.expect("acknowledge"); + assert_eq!(sub.resume_from, Some(SeqId::from_i64(10))); + } + + #[tokio::test] + async fn test_explicit_ack_rejects_sequence_that_was_not_delivered() { + let mut sub = make_test_sub(); + sub.ack_mode = SubscriptionAckMode::Explicit; + + let error = sub + .acknowledge(SeqId::from_i64(11)) + .await + .expect_err("undelivered sequence must fail"); + + assert!(error.to_string().contains("not delivered")); + assert_eq!(sub.resume_from, None); + } + #[tokio::test] async fn test_drop_inside_runtime_does_not_panic() { let sub = make_test_sub(); diff --git a/link/link-common/src/subscription/mod.rs b/link/link-common/src/subscription/mod.rs index e2e95546c..1a694a9e0 100644 --- a/link/link-common/src/subscription/mod.rs +++ b/link/link-common/src/subscription/mod.rs @@ -12,7 +12,10 @@ pub mod models; pub use live_rows_config::LiveRowsConfig; pub use live_rows_event::LiveRowsEvent; pub use live_rows_materializer::LiveRowsMaterializer; -pub use models::{SubscriptionConfig, SubscriptionInfo, SubscriptionOptions, SubscriptionRequest}; +pub use models::{ + SubscriptionAckMode, SubscriptionConfig, SubscriptionInfo, SubscriptionOptions, + SubscriptionRequest, +}; #[cfg(feature = "tokio-runtime")] mod live_rows_subscription; diff --git a/link/link-common/src/subscription/models/mod.rs b/link/link-common/src/subscription/models/mod.rs index a6ec53966..5807422fe 100644 --- a/link/link-common/src/subscription/models/mod.rs +++ b/link/link-common/src/subscription/models/mod.rs @@ -1,10 +1,12 @@ //! Subscription models: wire protocol, domain events, and config types. pub mod change_event; +pub mod subscription_ack_mode; pub mod subscription_config; pub mod subscription_info; pub use change_event::ChangeEvent; +pub use subscription_ack_mode::SubscriptionAckMode; pub use kalamdb_commons::{ BatchControl, BatchStatus, ChangeTypeRaw, SubscriptionOptions, SubscriptionRequest, }; diff --git a/link/link-common/src/subscription/models/subscription_ack_mode.rs b/link/link-common/src/subscription/models/subscription_ack_mode.rs new file mode 100644 index 000000000..c373a1295 --- /dev/null +++ b/link/link-common/src/subscription/models/subscription_ack_mode.rs @@ -0,0 +1,9 @@ +/// Controls when a live subscription advances its reconnect cursor. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub enum SubscriptionAckMode { + /// Reading an event advances progress, preserving existing SDK behavior. + #[default] + Automatic, + /// Progress advances only after the consumer explicitly acknowledges it. + Explicit, +} diff --git a/link/link-common/src/subscription/models/subscription_config.rs b/link/link-common/src/subscription/models/subscription_config.rs index df9ea77b4..80987cfbf 100644 --- a/link/link-common/src/subscription/models/subscription_config.rs +++ b/link/link-common/src/subscription/models/subscription_config.rs @@ -1,4 +1,4 @@ -use super::SubscriptionOptions; +use super::{SubscriptionAckMode, SubscriptionOptions}; /// Configuration for establishing a WebSocket subscription. #[derive(Debug, Clone)] @@ -11,6 +11,9 @@ pub struct SubscriptionConfig { pub options: Option, /// Override WebSocket URL (falls back to base_url conversion when `None`) pub ws_url: Option, + /// Controls whether delivery or an explicit consumer acknowledgement + /// advances the reconnect cursor. + pub ack_mode: SubscriptionAckMode, } impl SubscriptionConfig { @@ -23,6 +26,7 @@ impl SubscriptionConfig { sql: sql.into(), options: Some(SubscriptionOptions::default()), ws_url: None, + ack_mode: SubscriptionAckMode::Automatic, } } @@ -33,6 +37,13 @@ impl SubscriptionConfig { sql: sql.into(), options: None, ws_url: None, + ack_mode: SubscriptionAckMode::Automatic, } } + + /// Require the consumer to acknowledge committed progress explicitly. + pub fn with_explicit_ack(mut self) -> Self { + self.ack_mode = SubscriptionAckMode::Explicit; + self + } } diff --git a/link/sdks/dart/generator/.gitignore b/link/sdks/dart/generator/.gitignore new file mode 100644 index 000000000..a8c7d1d6f --- /dev/null +++ b/link/sdks/dart/generator/.gitignore @@ -0,0 +1,2 @@ +.dart_tool/ +build/ diff --git a/link/sdks/dart/generator/CHANGELOG.md b/link/sdks/dart/generator/CHANGELOG.md new file mode 100644 index 000000000..54eafae33 --- /dev/null +++ b/link/sdks/dart/generator/CHANGELOG.md @@ -0,0 +1,4 @@ +## 0.5.6-rc.0 + +- Initial generator for annotated action payload codecs, definitions, and + typed queue methods. diff --git a/link/sdks/dart/LICENSE b/link/sdks/dart/generator/LICENSE similarity index 100% rename from link/sdks/dart/LICENSE rename to link/sdks/dart/generator/LICENSE diff --git a/link/sdks/dart/generator/README.md b/link/sdks/dart/generator/README.md new file mode 100644 index 000000000..9695a80cb --- /dev/null +++ b/link/sdks/dart/generator/README.md @@ -0,0 +1,33 @@ +# kalam_sync_generator + +Optional code generation for `kalam_sync` durable actions. + +Add the generator and `build_runner` as development dependencies, annotate an +immutable payload with `@KalamActionPayload()`, and annotate executor methods +inside a `@KalamActionModule`. Running `dart run build_runner build` generates: + +- a durable JSON codec for each supported payload; +- stable namespaced `KalamActionDefinition` values; +- a typed queue class with offline enqueue methods. + +Drift remains responsible for table row and companion types. This generator +does not create duplicate create/update/delete models for every table; direct +KalamDB DML uses the runtime's single generic action envelope. + +See the [`kalam_sync` chat example](../sync/example) for a complete offline-first +messaging app, REST engine, and live-server e2e suite. The canonical generated +action module lives in [`example/`](example). + +```bash +cd example +flutter pub get +dart run build_runner build --delete-conflicting-outputs +``` + +The test suite generates multiple action modules in one library and then runs +the checked-in generated queue through a real in-memory SQLite outbox: + +```bash +dart test +flutter test flutter_test/generated_action_runtime_e2e_test.dart +``` diff --git a/link/sdks/dart/generator/build.yaml b/link/sdks/dart/generator/build.yaml new file mode 100644 index 000000000..81ae10846 --- /dev/null +++ b/link/sdks/dart/generator/build.yaml @@ -0,0 +1,14 @@ +targets: + $default: + builders: + kalam_sync_generator:kalam_actions: + enabled: true + +builders: + kalam_actions: + import: "package:kalam_sync_generator/kalam_sync_generator.dart" + builder_factories: [kalamActionBuilder] + build_extensions: {".dart": [".kalam_actions.g.part"]} + auto_apply: dependents + build_to: cache + applies_builders: ["source_gen:combining_builder"] diff --git a/link/sdks/dart/generator/example/.gitignore b/link/sdks/dart/generator/example/.gitignore new file mode 100644 index 000000000..f760c9035 --- /dev/null +++ b/link/sdks/dart/generator/example/.gitignore @@ -0,0 +1,4 @@ +.dart_tool/ +.flutter-plugins +.flutter-plugins-dependencies +build/ diff --git a/link/sdks/dart/generator/example/README.md b/link/sdks/dart/generator/example/README.md new file mode 100644 index 000000000..c269c9c2a --- /dev/null +++ b/link/sdks/dart/generator/example/README.md @@ -0,0 +1,57 @@ +# Chat action module + +This package is the source of truth for the offline-first chat actions used by +`kalam_sync` and `kalam_link`. + +It generates: + +- `chat.sendMessage` — two durable steps (`persist`, then `deliver`) so a + multi-endpoint workflow survives retry and restart +- `chat.markMsgRead` — a side-effect action that updates seen state on the + backend-authoritative messages replica + +The runnable Flutter app, REST engine, and schema live in +[`link/sdks/dart/sync/example`](../../sync/example). + +## Generate + +```bash +cd link/sdks/dart/generator/example +flutter pub get +dart run build_runner build --delete-conflicting-outputs +``` + +That writes `lib/chat_actions.g.dart`, which the Flutter example and e2e tests +import as `package:kalam_sync_generator_example/chat_actions.dart`. + +## What the generated API looks like + +```dart +final kalam = await Kalam.open( + url: kalamUrl, + subject: userId, + actionDefinitions: chatActionsDefinitions(ChatActions(api)), +); + +final messages = kalam.table(chatMessagesSpec('$namespace.messages')); +final actions = ChatActionsQueue(kalam.actions); + +await actions.sendMessage( + SendMessageArgs( + messageId: Kalam.id(), + conversationId: conversationId, + text: text, + createdAt: DateTime.now().toUtc(), + author: userId, + ), + orderingKey: conversationId, + optimistic: messages.optimisticInsert(pendingRow), +); +``` + +`replicaOnly` messages cannot be inserted through `table.insert`. Conversations +use `KalamSyncMode.bidirectional` in the Flutter example. + +## Run the complete example + +See [`../../sync/example/README.md`](../../sync/example/README.md). diff --git a/link/sdks/dart/generator/example/analysis_options.yaml b/link/sdks/dart/generator/example/analysis_options.yaml new file mode 100644 index 000000000..488f572d8 --- /dev/null +++ b/link/sdks/dart/generator/example/analysis_options.yaml @@ -0,0 +1,10 @@ +analyzer: + exclude: + - "**/*.g.dart" + - build/** + - android/** + - ios/** + - web/** + - windows/** + - macos/** + - linux/** diff --git a/link/sdks/dart/generator/example/lib/chat_actions.dart b/link/sdks/dart/generator/example/lib/chat_actions.dart new file mode 100644 index 000000000..6045530f8 --- /dev/null +++ b/link/sdks/dart/generator/example/lib/chat_actions.dart @@ -0,0 +1,103 @@ +import 'package:kalam_sync/kalam_sync.dart'; + +part 'chat_actions.g.dart'; + +/// Backend calls used by generated chat actions. +abstract interface class ChatActionApi { + Future persistMessage( + SendMessageArgs payload, { + required String idempotencyKey, + }); + + Future markDelivered( + SendMessageArgs payload, { + required String idempotencyKey, + }); + + Future markRead( + MarkMessageReadArgs payload, { + required String idempotencyKey, + }); +} + +@KalamActionPayload() +final class SendMessageArgs { + const SendMessageArgs({ + required this.messageId, + required this.conversationId, + required this.text, + required this.createdAt, + required this.author, + }); + + final String messageId; + final String conversationId; + final String text; + final DateTime createdAt; + final String author; +} + +@KalamActionPayload() +final class MarkMessageReadArgs { + const MarkMessageReadArgs({ + required this.messageId, + required this.conversationId, + required this.readAt, + }); + + final String messageId; + final String conversationId; + final DateTime readAt; +} + +@KalamActionModule(namespace: 'chat') +final class ChatActions { + const ChatActions(this.api); + + final ChatActionApi api; + + /// Persists the user message, then marks it delivered on a second endpoint. + /// + /// Named steps survive retry and process restart. Both endpoints still honor + /// the supplied idempotency key because a response can be lost after commit. + @KalamAction(name: 'sendMessage') + Future sendMessage( + KalamActionContext context, + SendMessageArgs payload, + ) async { + await context.step( + 'persist', + run: (idempotencyKey) async { + await api.persistMessage(payload, idempotencyKey: idempotencyKey); + return true; + }, + encode: (value) => value, + decode: (value) => value == true, + ); + await context.step( + 'deliver', + run: (idempotencyKey) async { + await api.markDelivered(payload, idempotencyKey: idempotencyKey); + return true; + }, + encode: (value) => value, + decode: (value) => value == true, + ); + } + + @KalamAction(name: 'markMsgRead') + Future markMsgRead( + KalamActionContext context, + MarkMessageReadArgs payload, + ) { + return context.step( + 'read', + run: (idempotencyKey) async { + await api.markRead(payload, idempotencyKey: idempotencyKey); + return true; + }, + encode: (value) => value, + decode: (value) => value == true, + ); + } +} diff --git a/link/sdks/dart/generator/example/lib/chat_actions.g.dart b/link/sdks/dart/generator/example/lib/chat_actions.g.dart new file mode 100644 index 000000000..f91017b10 --- /dev/null +++ b/link/sdks/dart/generator/example/lib/chat_actions.g.dart @@ -0,0 +1,96 @@ +// GENERATED CODE - DO NOT MODIFY BY HAND + +part of 'chat_actions.dart'; + +// ************************************************************************** +// KalamActionPayloadGenerator +// ************************************************************************** + +SendMessageArgs _$SendMessageArgsFromJson(Map json) => + SendMessageArgs( + messageId: json['messageId'] as String, + conversationId: json['conversationId'] as String, + text: json['text'] as String, + createdAt: DateTime.parse(json['createdAt'] as String), + author: json['author'] as String, + ); + +Map _$SendMessageArgsToJson(SendMessageArgs value) => { + 'messageId': value.messageId, + 'conversationId': value.conversationId, + 'text': value.text, + 'createdAt': value.createdAt.toUtc().toIso8601String(), + 'author': value.author, +}; + +MarkMessageReadArgs _$MarkMessageReadArgsFromJson(Map json) => + MarkMessageReadArgs( + messageId: json['messageId'] as String, + conversationId: json['conversationId'] as String, + readAt: DateTime.parse(json['readAt'] as String), + ); + +Map _$MarkMessageReadArgsToJson(MarkMessageReadArgs value) => { + 'messageId': value.messageId, + 'conversationId': value.conversationId, + 'readAt': value.readAt.toUtc().toIso8601String(), +}; + +// ************************************************************************** +// KalamActionGenerator +// ************************************************************************** + +List> chatActionsDefinitions( + ChatActions module, +) => [ + KalamActionDefinition( + key: 'chat.sendMessage', + version: 1, + codec: KalamActionCodec( + encode: _$SendMessageArgsToJson, + decode: _$SendMessageArgsFromJson, + ), + execute: module.sendMessage, + ), + KalamActionDefinition( + key: 'chat.markMsgRead', + version: 1, + codec: KalamActionCodec( + encode: _$MarkMessageReadArgsToJson, + decode: _$MarkMessageReadArgsFromJson, + ), + execute: module.markMsgRead, + ), +]; + +final class ChatActionsQueue { + const ChatActionsQueue(this._runner); + + final KalamActionRunner _runner; + + Future sendMessage( + SendMessageArgs payload, { + String? actionId, + String? orderingKey, + KalamOptimisticMutation? optimistic, + }) => _runner.enqueue( + actionKey: 'chat.sendMessage', + actionId: actionId ?? Kalam.id(), + payload: payload, + orderingKey: orderingKey, + optimistic: optimistic, + ); + + Future markMsgRead( + MarkMessageReadArgs payload, { + String? actionId, + String? orderingKey, + KalamOptimisticMutation? optimistic, + }) => _runner.enqueue( + actionKey: 'chat.markMsgRead', + actionId: actionId ?? Kalam.id(), + payload: payload, + orderingKey: orderingKey, + optimistic: optimistic, + ); +} diff --git a/link/sdks/dart/generator/example/lib/chat_http_api.dart b/link/sdks/dart/generator/example/lib/chat_http_api.dart new file mode 100644 index 000000000..7b87e03a5 --- /dev/null +++ b/link/sdks/dart/generator/example/lib/chat_http_api.dart @@ -0,0 +1,70 @@ +import 'dart:convert'; +import 'dart:io'; + +import 'chat_actions.dart'; + +final class ChatHttpApi implements ChatActionApi { + ChatHttpApi({required this.baseUrl, HttpClient? client}) + : _client = (client ?? HttpClient()) + ..connectionTimeout = const Duration(seconds: 10) + ..idleTimeout = const Duration(seconds: 15); + + final Uri baseUrl; + final HttpClient _client; + + @override + Future persistMessage( + SendMessageArgs payload, { + required String idempotencyKey, + }) { + return post('/v1/messages', { + 'messageId': payload.messageId, + 'conversationId': payload.conversationId, + 'text': payload.text, + 'createdAt': payload.createdAt.toUtc().toIso8601String(), + 'author': payload.author, + 'idempotencyKey': idempotencyKey, + }); + } + + @override + Future markDelivered( + SendMessageArgs payload, { + required String idempotencyKey, + }) { + return post( + '/v1/messages/${Uri.encodeComponent(payload.messageId)}/deliver', + { + 'conversationId': payload.conversationId, + 'idempotencyKey': idempotencyKey, + }, + ); + } + + @override + Future markRead( + MarkMessageReadArgs payload, { + required String idempotencyKey, + }) { + return post('/v1/messages/${Uri.encodeComponent(payload.messageId)}/read', { + 'conversationId': payload.conversationId, + 'readAt': payload.readAt.toUtc().toIso8601String(), + 'idempotencyKey': idempotencyKey, + }); + } + + Future post(String path, Map body) async { + final request = await _client.postUrl(baseUrl.resolve(path)); + request.headers.contentType = ContentType.json; + request.write(jsonEncode(body)); + final response = await request.close().timeout( + const Duration(seconds: 15), + ); + final text = await utf8.decodeStream(response); + if (response.statusCode < 200 || response.statusCode >= 300) { + throw HttpException('Chat backend HTTP ${response.statusCode}: $text'); + } + } + + void close() => _client.close(force: true); +} diff --git a/link/sdks/dart/generator/example/lib/chat_models.dart b/link/sdks/dart/generator/example/lib/chat_models.dart new file mode 100644 index 000000000..2ba4a0f92 --- /dev/null +++ b/link/sdks/dart/generator/example/lib/chat_models.dart @@ -0,0 +1,176 @@ +/// Shared chat domain models used by generated actions, the Flutter example, +/// and the live-server e2e suite. +library; + +DateTime parseChatTimestamp(Object? value) { + if (value == null) { + throw const FormatException('missing timestamp'); + } + if (value is DateTime) return value.toUtc(); + if (value is int) return _fromNumericTimestamp(value); + if (value is num) return _fromNumericTimestamp(value.toInt()); + if (value is String) { + final numeric = int.tryParse(value) ?? double.tryParse(value)?.toInt(); + if (numeric != null) return _fromNumericTimestamp(numeric); + return DateTime.parse(value).toUtc(); + } + throw FormatException('Cannot parse timestamp: $value'); +} + +DateTime? parseOptionalChatTimestamp(Object? value) { + if (value == null) return null; + if (value is String && value.trim().isEmpty) return null; + return parseChatTimestamp(value); +} + +DateTime _fromNumericTimestamp(int value) { + const millisecondThreshold = 9999999999999; + if (value.abs() > millisecondThreshold) { + return DateTime.fromMicrosecondsSinceEpoch(value, isUtc: true); + } + return DateTime.fromMillisecondsSinceEpoch(value, isUtc: true); +} + +String encodeChatTimestamp(DateTime value) => value.toUtc().toIso8601String(); + +enum ChatMessageRole { + user, + assistant; + + static ChatMessageRole parse(Object? value) { + final name = value?.toString() ?? 'user'; + return ChatMessageRole.values.firstWhere( + (role) => role.name == name, + orElse: () => ChatMessageRole.user, + ); + } +} + +enum ChatDeliveryStatus { + pending, + sent, + delivered, + read; + + static ChatDeliveryStatus parse(Object? value) { + final name = value?.toString() ?? 'sent'; + return ChatDeliveryStatus.values.firstWhere( + (status) => status.name == name, + orElse: () => ChatDeliveryStatus.sent, + ); + } +} + +final class Conversation { + const Conversation({ + required this.id, + required this.title, + required this.createdAt, + required this.updatedAt, + }); + + final String id; + final String title; + final DateTime createdAt; + final DateTime updatedAt; + + Conversation copyWith({String? title, DateTime? updatedAt}) { + return Conversation( + id: id, + title: title ?? this.title, + createdAt: createdAt, + updatedAt: updatedAt ?? this.updatedAt, + ); + } + + Map toJson() => { + 'id': id, + 'title': title, + 'created_at': encodeChatTimestamp(createdAt), + 'updated_at': encodeChatTimestamp(updatedAt), + }; + + factory Conversation.fromJson(Map json) { + return Conversation( + id: json['id']!.toString(), + title: json['title']!.toString(), + createdAt: parseChatTimestamp(json['created_at']), + updatedAt: parseChatTimestamp(json['updated_at']), + ); + } +} + +final class ChatMessage { + const ChatMessage({ + required this.id, + required this.conversationId, + required this.role, + required this.author, + required this.text, + required this.status, + required this.createdAt, + this.deliveredAt, + this.readAt, + }); + + final String id; + final String conversationId; + final ChatMessageRole role; + final String author; + final String text; + final ChatDeliveryStatus status; + final DateTime createdAt; + final DateTime? deliveredAt; + final DateTime? readAt; + + bool get isAssistant => role == ChatMessageRole.assistant; + + ChatMessage copyWith({ + ChatDeliveryStatus? status, + DateTime? deliveredAt, + DateTime? readAt, + }) { + return ChatMessage( + id: id, + conversationId: conversationId, + role: role, + author: author, + text: text, + status: status ?? this.status, + createdAt: createdAt, + deliveredAt: deliveredAt ?? this.deliveredAt, + readAt: readAt ?? this.readAt, + ); + } + + Map toJson() => { + 'id': id, + 'conversation_id': conversationId, + 'role': role.name, + 'author': author, + 'text': text, + 'status': status.name, + 'created_at': encodeChatTimestamp(createdAt), + 'delivered_at': deliveredAt == null + ? null + : encodeChatTimestamp(deliveredAt!), + 'read_at': readAt == null ? null : encodeChatTimestamp(readAt!), + }; + + factory ChatMessage.fromJson(Map json) { + return ChatMessage( + id: json['id']!.toString(), + conversationId: (json['conversation_id'] ?? json['conversationId'])! + .toString(), + role: ChatMessageRole.parse(json['role']), + author: json['author']!.toString(), + text: json['text']!.toString(), + status: ChatDeliveryStatus.parse(json['status']), + createdAt: parseChatTimestamp(json['created_at'] ?? json['createdAt']), + deliveredAt: parseOptionalChatTimestamp( + json['delivered_at'] ?? json['deliveredAt'], + ), + readAt: parseOptionalChatTimestamp(json['read_at'] ?? json['readAt']), + ); + } +} diff --git a/link/sdks/dart/generator/example/lib/chat_tables.dart b/link/sdks/dart/generator/example/lib/chat_tables.dart new file mode 100644 index 000000000..e35e1743a --- /dev/null +++ b/link/sdks/dart/generator/example/lib/chat_tables.dart @@ -0,0 +1,25 @@ +import 'package:kalam_sync/kalam_sync.dart'; + +import 'chat_models.dart'; + +KalamTableSpec chatConversationsSpec(String tableId) { + return KalamTableSpec( + tableId: tableId, + keyColumn: 'id', + mode: KalamSyncMode.bidirectional, + keyOf: (row) => row.id, + encode: (row) => row.toJson(), + decode: Conversation.fromJson, + ); +} + +KalamTableSpec chatMessagesSpec(String tableId) { + return KalamTableSpec( + tableId: tableId, + keyColumn: 'id', + mode: KalamSyncMode.replicaOnly, + keyOf: (row) => row.id, + encode: (row) => row.toJson(), + decode: ChatMessage.fromJson, + ); +} diff --git a/link/sdks/dart/generator/example/pubspec.lock b/link/sdks/dart/generator/example/pubspec.lock new file mode 100644 index 000000000..cdb25f8e0 --- /dev/null +++ b/link/sdks/dart/generator/example/pubspec.lock @@ -0,0 +1,653 @@ +# Generated by pub +# See https://dart.dev/tools/pub/glossary#lockfile +packages: + _fe_analyzer_shared: + dependency: transitive + description: + name: _fe_analyzer_shared + sha256: "9a3386eea899815698dd55995277cf7cb8572ee52b399a6edfb7ae2b50e5fc19" + url: "https://pub.dev" + source: hosted + version: "105.0.0" + analyzer: + dependency: transitive + description: + name: analyzer + sha256: "62993bed6eadbe9596c5c20d5c167e7bc563c5fe266657a04ddeb93bdb84f4c9" + url: "https://pub.dev" + source: hosted + version: "14.1.0" + args: + dependency: transitive + description: + name: args + sha256: d0481093c50b1da8910eb0bb301626d4d8eb7284aa739614d2b394ee09e3ea04 + url: "https://pub.dev" + source: hosted + version: "2.7.0" + async: + dependency: transitive + description: + name: async + sha256: e2eb0491ba5ddb6177742d2da23904574082139b07c1e33b8503b9f46f3e1a37 + url: "https://pub.dev" + source: hosted + version: "2.13.1" + build: + dependency: transitive + description: + name: build + sha256: b94f5da9ed3d081fd4ecc426c260998b5f4f4eb2f6d89b7e5472edef0b2b2a1b + url: "https://pub.dev" + source: hosted + version: "4.0.10" + build_cli_annotations: + dependency: transitive + description: + name: build_cli_annotations + sha256: e563c2e01de8974566a1998410d3f6f03521788160a02503b0b1f1a46c7b3d95 + url: "https://pub.dev" + source: hosted + version: "2.1.1" + build_config: + dependency: transitive + description: + name: build_config + sha256: "94eaf6708fe64408c632ef2689ca3777b112f9421306ccf4f8c84d7c5c9f83f8" + url: "https://pub.dev" + source: hosted + version: "1.3.2" + build_daemon: + dependency: transitive + description: + name: build_daemon + sha256: "79e05eaf15a48d7230b053a4363b8eaac0cc234bbd0134c3229455481f55cbc6" + url: "https://pub.dev" + source: hosted + version: "4.1.5" + build_runner: + dependency: "direct dev" + description: + name: build_runner + sha256: "90363d438bd9f84a4d5bbb83352251f57d2d9d771bc95a44e6a33fe25fa52774" + url: "https://pub.dev" + source: hosted + version: "2.16.0" + built_collection: + dependency: transitive + description: + name: built_collection + sha256: "376e3dd27b51ea877c28d525560790aee2e6fbb5f20e2f85d5081027d94e2100" + url: "https://pub.dev" + source: hosted + version: "5.1.1" + built_value: + dependency: transitive + description: + name: built_value + sha256: "31b24be6615ec7fcf70b3aa5a7469fe35826485e639a16dd7eb83ba30e4cc6a8" + url: "https://pub.dev" + source: hosted + version: "8.12.7" + characters: + dependency: transitive + description: + name: characters + sha256: faf38497bda5ead2a8c7615f4f7939df04333478bf32e4173fcb06d428b5716b + url: "https://pub.dev" + source: hosted + version: "1.4.1" + checked_yaml: + dependency: transitive + description: + name: checked_yaml + sha256: "959525d3162f249993882720d52b7e0c833978df229be20702b33d48d91de70f" + url: "https://pub.dev" + source: hosted + version: "2.0.4" + clock: + dependency: transitive + description: + name: clock + sha256: fddb70d9b5277016c77a80201021d40a2247104d9f4aa7bab7157b7e3f05b84b + url: "https://pub.dev" + source: hosted + version: "1.1.2" + code_assets: + dependency: transitive + description: + name: code_assets + sha256: bf394f466ba9205f1812a0433b392d6af280f155f56651eda7c18cc32ed493b8 + url: "https://pub.dev" + source: hosted + version: "1.2.1" + collection: + dependency: transitive + description: + name: collection + sha256: "2f5709ae4d3d59dd8f7cd309b4e023046b57d8a6c82130785d2b0e5868084e76" + url: "https://pub.dev" + source: hosted + version: "1.19.1" + convert: + dependency: transitive + description: + name: convert + sha256: b30acd5944035672bc15c6b7a8b47d773e41e2f17de064350988c5d02adb1c68 + url: "https://pub.dev" + source: hosted + version: "3.1.2" + crypto: + dependency: transitive + description: + name: crypto + sha256: c8ea0233063ba03258fbcf2ca4d6dadfefe14f02fab57702265467a19f27fadf + url: "https://pub.dev" + source: hosted + version: "3.0.7" + dart_style: + dependency: transitive + description: + name: dart_style + sha256: "3f88fc9c96c568d631356507355a2d1e983ee000c9ac008bdcb39b5bf53ce777" + url: "https://pub.dev" + source: hosted + version: "3.1.12" + drift: + dependency: transitive + description: + name: drift + sha256: "3a3f1f6f905037d7426e4c445854139fd6a3d592135f7c96d7931682b73d16f4" + url: "https://pub.dev" + source: hosted + version: "2.34.3" + drift_flutter: + dependency: transitive + description: + name: drift_flutter + sha256: "91acf4bee7c3c84467cba46455aa70e5292a3b889f4582645d74f2e5a8c106f2" + url: "https://pub.dev" + source: hosted + version: "0.3.1" + ffi: + dependency: transitive + description: + name: ffi + sha256: "6d7fd89431262d8f3125e81b50d3847a091d846eafcd4fdb88dd06f36d705a45" + url: "https://pub.dev" + source: hosted + version: "2.2.0" + file: + dependency: transitive + description: + name: file + sha256: a3b4f84adafef897088c160faf7dfffb7696046cb13ae90b508c2cbc95d3b8d4 + url: "https://pub.dev" + source: hosted + version: "7.0.1" + fixnum: + dependency: transitive + description: + name: fixnum + sha256: b6dc7065e46c974bc7c5f143080a6764ec7a4be6da1285ececdc37be96de53be + url: "https://pub.dev" + source: hosted + version: "1.1.1" + flutter: + dependency: "direct main" + description: flutter + source: sdk + version: "0.0.0" + flutter_rust_bridge: + dependency: transitive + description: + name: flutter_rust_bridge + sha256: e87d6b9ee934dcd24a128ccb2bd91905d2d5fe5c06245d6a8f5477d4907a437a + url: "https://pub.dev" + source: hosted + version: "2.12.0" + freezed_annotation: + dependency: transitive + description: + name: freezed_annotation + sha256: "7294967ff0a6d98638e7acb774aac3af2550777accd8149c90af5b014e6d44d8" + url: "https://pub.dev" + source: hosted + version: "3.1.0" + glob: + dependency: transitive + description: + name: glob + sha256: c3f1ee72c96f8f78935e18aa8cecced9ab132419e8625dc187e1c2408efc20de + url: "https://pub.dev" + source: hosted + version: "2.1.3" + graphs: + dependency: transitive + description: + name: graphs + sha256: "741bbf84165310a68ff28fe9e727332eef1407342fca52759cb21ad8177bb8d0" + url: "https://pub.dev" + source: hosted + version: "2.3.2" + hooks: + dependency: transitive + description: + name: hooks + sha256: "03a564c3704524ee0f7fc56fc621e8796cc2eb8c24d1f1a33b34979815b285c4" + url: "https://pub.dev" + source: hosted + version: "2.1.0" + http_multi_server: + dependency: transitive + description: + name: http_multi_server + sha256: aa6199f908078bb1c5efb8d8638d4ae191aac11b311132c3ef48ce352fb52ef8 + url: "https://pub.dev" + source: hosted + version: "3.2.2" + http_parser: + dependency: transitive + description: + name: http_parser + sha256: "178d74305e7866013777bab2c3d8726205dc5a4dd935297175b19a23a2e66571" + url: "https://pub.dev" + source: hosted + version: "4.1.2" + io: + dependency: transitive + description: + name: io + sha256: dfd5a80599cf0165756e3181807ed3e77daf6dd4137caaad72d0b7931597650b + url: "https://pub.dev" + source: hosted + version: "1.0.5" + jni: + dependency: transitive + description: + name: jni + sha256: f038e58b4dc2c9037f50e233175086337e0b305e356d28211bf55f21c504cbd3 + url: "https://pub.dev" + source: hosted + version: "1.0.3" + jni_flutter: + dependency: transitive + description: + name: jni_flutter + sha256: "7b717011ea40d04fd47c2731d3d1d36eb99eba3435c2753d62489e8c3c9991d5" + url: "https://pub.dev" + source: hosted + version: "1.0.2" + jni_util: + dependency: transitive + description: + name: jni_util + sha256: "1ba86da04a5f2bf18fde2edb235587e70c5b0fc5bd4ba955f46b00942c3fc35f" + url: "https://pub.dev" + source: hosted + version: "1.0.0" + json_annotation: + dependency: transitive + description: + name: json_annotation + sha256: "2a743920d81b7910627f68ee2c9ac1fc0bfee32b9fc3403587d7c6791ca12f80" + url: "https://pub.dev" + source: hosted + version: "4.12.0" + kalam_link: + dependency: "direct overridden" + description: + path: "../../link" + relative: true + source: path + version: "0.5.6-rc.0" + kalam_sync: + dependency: "direct main" + description: + path: "../../sync" + relative: true + source: path + version: "0.5.6-rc.0" + kalam_sync_generator: + dependency: "direct dev" + description: + path: ".." + relative: true + source: path + version: "0.5.6-rc.0" + logger: + dependency: transitive + description: + name: logger + sha256: "25aee487596a6257655a1e091ec2ae66bc30e7af663592cc3a27e6591e05035c" + url: "https://pub.dev" + source: hosted + version: "2.7.0" + logging: + dependency: transitive + description: + name: logging + sha256: c8245ada5f1717ed44271ed1c26b8ce85ca3228fd2ffdb75468ab01979309d61 + url: "https://pub.dev" + source: hosted + version: "1.3.0" + material_color_utilities: + dependency: transitive + description: + name: material_color_utilities + sha256: "9c337007e82b1889149c82ed242ed1cb24a66044e30979c44912381e9be4c48b" + url: "https://pub.dev" + source: hosted + version: "0.13.0" + meta: + dependency: transitive + description: + name: meta + sha256: "307249ce4ff29d58a18e97f6345f539382eb9c9c29ecda628900f31de0443dd9" + url: "https://pub.dev" + source: hosted + version: "1.19.0" + mime: + dependency: transitive + description: + name: mime + sha256: "41a20518f0cb1256669420fdba0cd90d21561e560ac240f26ef8322e45bb7ed6" + url: "https://pub.dev" + source: hosted + version: "2.0.0" + native_toolchain_c: + dependency: transitive + description: + name: native_toolchain_c + sha256: a1c26117c48cebe5677b0cf0e33a980a79a7c5577effc86f52e5a0d309cdcb60 + url: "https://pub.dev" + source: hosted + version: "0.19.3" + objective_c: + dependency: transitive + description: + name: objective_c + sha256: b7fb95a6d9a4f009edd63dc5ac69f07420b23a16161c6dd8660290b59c602e8e + url: "https://pub.dev" + source: hosted + version: "9.5.0" + package_config: + dependency: transitive + description: + name: package_config + sha256: ffcf4cf3d6c0b74ac43708d9f56625506e8a68aa935abe9d267a7330f320eb5d + url: "https://pub.dev" + source: hosted + version: "3.0.0" + path: + dependency: transitive + description: + name: path + sha256: "75cca69d1490965be98c73ceaea117e8a04dd21217b37b292c9ddbec0d955bc5" + url: "https://pub.dev" + source: hosted + version: "1.9.1" + path_provider: + dependency: transitive + description: + name: path_provider + sha256: a7f4874f987173da295a61c181b8ee71dab59b332a486b391babf26a1b884825 + url: "https://pub.dev" + source: hosted + version: "2.1.6" + path_provider_android: + dependency: transitive + description: + name: path_provider_android + sha256: "69cbd515a62b94d32a7944f086b2f82b4ac40a1d45bebfc00813a430ab2dabcd" + url: "https://pub.dev" + source: hosted + version: "2.3.1" + path_provider_foundation: + dependency: transitive + description: + name: path_provider_foundation + sha256: "2a376b7d6392d80cd3705782d2caa734ca4727776db0b6ec36ef3f1855197699" + url: "https://pub.dev" + source: hosted + version: "2.6.0" + path_provider_linux: + dependency: transitive + description: + name: path_provider_linux + sha256: "58c2005f147315b11e9b4a7bc889cd5203e250cba8e3f012dae259b4972b5c16" + url: "https://pub.dev" + source: hosted + version: "2.2.2" + path_provider_platform_interface: + dependency: transitive + description: + name: path_provider_platform_interface + sha256: "484838772624c3a4b94f1e44a3e19897fee738f2d5c4ce448443b0417f7c9dda" + url: "https://pub.dev" + source: hosted + version: "2.1.3" + path_provider_windows: + dependency: transitive + description: + name: path_provider_windows + sha256: bd6f00dbd873bfb70d0761682da2b3a2c2fccc2b9e84c495821639601d81afe7 + url: "https://pub.dev" + source: hosted + version: "2.3.0" + platform: + dependency: transitive + description: + name: platform + sha256: "5d6b1b0036a5f331ebc77c850ebc8506cbc1e9416c27e59b439f917a902a4984" + url: "https://pub.dev" + source: hosted + version: "3.1.6" + plugin_platform_interface: + dependency: transitive + description: + name: plugin_platform_interface + sha256: "4820fbfdb9478b1ebae27888254d445073732dae3d6ea81f0b7e06d5dedc3f02" + url: "https://pub.dev" + source: hosted + version: "2.1.8" + pool: + dependency: transitive + description: + name: pool + sha256: "978783255c543aa3586a1b3c21f6e9d720eb315376a915872c61ef8b5c20177d" + url: "https://pub.dev" + source: hosted + version: "1.5.2" + pub_semver: + dependency: transitive + description: + name: pub_semver + sha256: "5bfcf68ca79ef689f8990d1160781b4bad40a3bd5e5218ad4076ddb7f4081585" + url: "https://pub.dev" + source: hosted + version: "2.2.0" + pubspec_parse: + dependency: transitive + description: + name: pubspec_parse + sha256: "0560ba233314abbed0a48a2956f7f022cce7c3e1e73df540277da7544cad4082" + url: "https://pub.dev" + source: hosted + version: "1.5.0" + record_use: + dependency: transitive + description: + name: record_use + sha256: "3b4ab682aff40175afca6ff090cc5aa7ce9dafe8dfbae1537a6c678e6678baee" + url: "https://pub.dev" + source: hosted + version: "1.1.0" + shelf: + dependency: transitive + description: + name: shelf + sha256: e7dd780a7ffb623c57850b33f43309312fc863fb6aa3d276a754bb299839ef12 + url: "https://pub.dev" + source: hosted + version: "1.4.2" + shelf_web_socket: + dependency: transitive + description: + name: shelf_web_socket + sha256: "3632775c8e90d6c9712f883e633716432a27758216dfb61bd86a8321c0580925" + url: "https://pub.dev" + source: hosted + version: "3.0.0" + sky_engine: + dependency: transitive + description: flutter + source: sdk + version: "0.0.0" + source_gen: + dependency: transitive + description: + name: source_gen + sha256: a603f1fb984a7391ae5978d1b92bfaaa08b350dca5c825256f925818f7943bf5 + url: "https://pub.dev" + source: hosted + version: "4.2.4" + source_span: + dependency: transitive + description: + name: source_span + sha256: "56a02f1f4cd1a2d96303c0144c93bd6d909eea6bee6bf5a0e0b685edbd4c47ab" + url: "https://pub.dev" + source: hosted + version: "1.10.2" + sqlcipher_flutter_libs: + dependency: transitive + description: + name: sqlcipher_flutter_libs + sha256: "38d62d659d2fb8739bf25a42c9a350d1fdd6c29a5a61f13a946778ec75d27929" + url: "https://pub.dev" + source: hosted + version: "0.7.0+eol" + sqlite3: + dependency: transitive + description: + name: sqlite3 + sha256: "64b2c63c8232dd20d14b34105a81ebfd74320442e8451f836179ec89986aa478" + url: "https://pub.dev" + source: hosted + version: "3.5.1" + sqlite3_flutter_libs: + dependency: transitive + description: + name: sqlite3_flutter_libs + sha256: "3ed7553eee7bb368f8950f58ba29f634e06e813c029aff6a0d60862b96de8454" + url: "https://pub.dev" + source: hosted + version: "0.6.0+eol" + stack_trace: + dependency: transitive + description: + name: stack_trace + sha256: "8b27215b45d22309b5cddda1aa2b19bdfec9df0e765f2de506401c071d38d1b1" + url: "https://pub.dev" + source: hosted + version: "1.12.1" + stream_channel: + dependency: transitive + description: + name: stream_channel + sha256: "969e04c80b8bcdf826f8f16579c7b14d780458bd97f56d107d3950fdbeef059d" + url: "https://pub.dev" + source: hosted + version: "2.1.4" + stream_transform: + dependency: transitive + description: + name: stream_transform + sha256: ad47125e588cfd37a9a7f86c7d6356dde8dfe89d071d293f80ca9e9273a33871 + url: "https://pub.dev" + source: hosted + version: "2.1.1" + string_scanner: + dependency: transitive + description: + name: string_scanner + sha256: "921cd31725b72fe181906c6a94d987c78e3b98c2e205b397ea399d4054872b43" + url: "https://pub.dev" + source: hosted + version: "1.4.1" + term_glyph: + dependency: transitive + description: + name: term_glyph + sha256: "7f554798625ea768a7518313e58f83891c7f5024f88e46e7182a4558850a4b8e" + url: "https://pub.dev" + source: hosted + version: "1.2.2" + typed_data: + dependency: transitive + description: + name: typed_data + sha256: f9049c039ebfeb4cf7a7104a675823cd72dba8297f264b6637062516699fa006 + url: "https://pub.dev" + source: hosted + version: "1.4.0" + vector_math: + dependency: transitive + description: + name: vector_math + sha256: f36f9f3be64c6198714492bb455c11056e33e2f85d9a0b676a48301e44fdcf47 + url: "https://pub.dev" + source: hosted + version: "2.4.2" + watcher: + dependency: transitive + description: + name: watcher + sha256: "1398c9f081a753f9226febe8900fce8f7d0a67163334e1c94a2438339d79d635" + url: "https://pub.dev" + source: hosted + version: "1.2.1" + web: + dependency: transitive + description: + name: web + sha256: "868d88a33d8a87b18ffc05f9f030ba328ffefba92d6c127917a2ba740f9cfe4a" + url: "https://pub.dev" + source: hosted + version: "1.1.1" + web_socket: + dependency: transitive + description: + name: web_socket + sha256: "34d64019aa8e36bf9842ac014bb5d2f5586ca73df5e4d9bf5c936975cae6982c" + url: "https://pub.dev" + source: hosted + version: "1.0.1" + web_socket_channel: + dependency: transitive + description: + name: web_socket_channel + sha256: d645757fb0f4773d602444000a8131ff5d48c9e47adfe9772652dd1a4f2d45c8 + url: "https://pub.dev" + source: hosted + version: "3.0.3" + xdg_directories: + dependency: transitive + description: + name: xdg_directories + sha256: "7a3f37b05d989967cdddcbb571f1ea834867ae2faa29725fd085180e0883aa15" + url: "https://pub.dev" + source: hosted + version: "1.1.0" + yaml: + dependency: transitive + description: + name: yaml + sha256: b9da305ac7c39faa3f030eccd175340f968459dae4af175130b3fc47e40d76ce + url: "https://pub.dev" + source: hosted + version: "3.1.3" +sdks: + dart: ">=3.11.0 <4.0.0" + flutter: ">=3.38.4" diff --git a/link/sdks/dart/generator/example/pubspec.yaml b/link/sdks/dart/generator/example/pubspec.yaml new file mode 100644 index 000000000..61e37b3e3 --- /dev/null +++ b/link/sdks/dart/generator/example/pubspec.yaml @@ -0,0 +1,20 @@ +name: kalam_sync_generator_example +description: Canonical chat action module for kalam_sync_generator and the Flutter sync example. +publish_to: none +version: 0.5.6-rc.0 + +environment: + sdk: ">=3.10.0 <4.0.0" + flutter: ">=3.38.0" + +dependencies: + flutter: + sdk: flutter + kalam_sync: + path: ../../sync + +dev_dependencies: + build_runner: ^2.16.0 + kalam_sync_generator: + path: .. + diff --git a/link/sdks/dart/generator/example/pubspec_overrides.yaml b/link/sdks/dart/generator/example/pubspec_overrides.yaml new file mode 100644 index 000000000..e191f75f0 --- /dev/null +++ b/link/sdks/dart/generator/example/pubspec_overrides.yaml @@ -0,0 +1,7 @@ +dependency_overrides: + kalam_link: + path: ../../link + kalam_sync: + path: ../../sync + kalam_sync_generator: + path: .. diff --git a/link/sdks/dart/generator/flutter_test/generated_action_runtime_e2e_test.dart b/link/sdks/dart/generator/flutter_test/generated_action_runtime_e2e_test.dart new file mode 100644 index 000000000..dc8901700 --- /dev/null +++ b/link/sdks/dart/generator/flutter_test/generated_action_runtime_e2e_test.dart @@ -0,0 +1,97 @@ +import 'package:drift/native.dart'; +import 'package:kalam_sync/drift.dart' hide isNull; +import 'package:kalam_sync/kalam_sync.dart'; +import 'package:kalam_sync_generator_example/chat_actions.dart'; +import 'package:test/test.dart'; + +final class _RecordingApi implements ChatActionApi { + final persistKeys = []; + final deliverKeys = []; + final readKeys = []; + SendMessageArgs? sent; + MarkMessageReadArgs? read; + + @override + Future persistMessage( + SendMessageArgs payload, { + required String idempotencyKey, + }) async { + persistKeys.add(idempotencyKey); + sent = payload; + } + + @override + Future markDelivered( + SendMessageArgs payload, { + required String idempotencyKey, + }) async { + deliverKeys.add(idempotencyKey); + } + + @override + Future markRead( + MarkMessageReadArgs payload, { + required String idempotencyKey, + }) async { + readKeys.add(idempotencyKey); + read = payload; + } +} + +void main() { + test( + 'generated chat queue persists send, multi-step deliver, and mark-read', + () async { + final database = KalamSyncDatabase(NativeDatabase.memory()); + addTearDown(database.close); + final store = KalamSyncStore(database); + final api = _RecordingApi(); + final runner = KalamActionRunner( + accountKey: 'server/user-a', + store: store, + registry: KalamActionRegistry.of( + chatActionsDefinitions(ChatActions(api)), + ), + ); + final queue = ChatActionsQueue(runner); + final createdAt = DateTime.utc(2026, 8, 17, 10, 30); + final readAt = DateTime.utc(2026, 8, 17, 10, 31); + + await queue.sendMessage( + SendMessageArgs( + messageId: 'message-1', + conversationId: 'conversation-1', + text: 'hello offline', + createdAt: createdAt, + author: 'ada', + ), + actionId: 'send-message-1', + orderingKey: 'conversation-1', + ); + expect(api.sent, isNull, reason: 'enqueue must not execute while offline'); + expect(await runner.flush(), 1); + expect(api.sent?.messageId, 'message-1'); + expect(api.sent?.text, 'hello offline'); + expect(api.sent?.createdAt, createdAt); + expect(api.persistKeys, ['send-message-1/persist']); + expect(api.deliverKeys, ['send-message-1/deliver']); + expect( + (await store.readAction('send-message-1'))?.status, + KalamActionStatus.succeeded, + ); + + await queue.markMsgRead( + MarkMessageReadArgs( + messageId: 'message-1', + conversationId: 'conversation-1', + readAt: readAt, + ), + actionId: 'read-message-1', + orderingKey: 'conversation-1', + ); + expect(await runner.flush(), 1); + expect(api.read?.messageId, 'message-1'); + expect(api.readKeys, ['read-message-1/read']); + }, + ); +} diff --git a/link/sdks/dart/generator/lib/kalam_sync_generator.dart b/link/sdks/dart/generator/lib/kalam_sync_generator.dart new file mode 100644 index 000000000..8e60bcc45 --- /dev/null +++ b/link/sdks/dart/generator/lib/kalam_sync_generator.dart @@ -0,0 +1,12 @@ +import 'package:build/build.dart'; +import 'package:source_gen/source_gen.dart'; + +import 'src/kalam_action_generator.dart'; +import 'src/kalam_action_payload_generator.dart'; + +Builder kalamActionBuilder(BuilderOptions options) { + return SharedPartBuilder(const [ + KalamActionPayloadGenerator(), + KalamActionGenerator(), + ], 'kalam_actions'); +} diff --git a/link/sdks/dart/generator/lib/src/kalam_action_generator.dart b/link/sdks/dart/generator/lib/src/kalam_action_generator.dart new file mode 100644 index 000000000..cf87c7e3e --- /dev/null +++ b/link/sdks/dart/generator/lib/src/kalam_action_generator.dart @@ -0,0 +1,122 @@ +import 'package:analyzer/dart/element/element.dart'; +import 'package:build/build.dart'; +import 'package:kalam_sync/src/annotations/kalam_action.dart'; +import 'package:kalam_sync/src/annotations/kalam_action_module.dart'; +import 'package:source_gen/source_gen.dart'; + +final class KalamActionGenerator + extends GeneratorForAnnotation { + const KalamActionGenerator(); + + static final _actionChecker = TypeChecker.typeNamed( + KalamAction, + inPackage: 'kalam_sync', + ); + + @override + String generateForAnnotatedElement( + Element element, + ConstantReader annotation, + BuildStep buildStep, + ) { + if (element is! ClassElement) { + throw InvalidGenerationSourceError( + '@KalamActionModule can only annotate a class.', + element: element, + ); + } + final namespace = annotation.read('namespace').stringValue; + final methods = <_ActionMethod>[]; + final keys = {}; + for (final method in element.methods) { + final value = _actionChecker.firstAnnotationOf(method); + if (value == null) continue; + final action = ConstantReader(value); + final name = action.read('name').stringValue; + final version = action.read('version').intValue; + final key = '$namespace.$name'; + if (!keys.add(key)) { + throw InvalidGenerationSourceError( + 'Duplicate action key "$key".', + element: method, + ); + } + if (method.formalParameters.length != 2) { + throw InvalidGenerationSourceError( + '${method.name} must accept KalamActionContext and one payload.', + element: method, + ); + } + final payloadType = method.formalParameters[1].type.getDisplayString(); + methods.add(_ActionMethod(method.name!, name, key, version, payloadType)); + } + + final moduleName = element.name!; + final definitionsName = + '${moduleName[0].toLowerCase()}${moduleName.substring(1)}Definitions'; + final definitions = methods + .map( + (method) => + ''' +KalamActionDefinition<${method.payloadType}>( + key: '${method.key}', + version: ${method.version}, + codec: KalamActionCodec( + encode: _\$${method.payloadType}ToJson, + decode: _\$${method.payloadType}FromJson, + ), + execute: module.${method.methodName}, +),''', + ) + .join('\n'); + final queueMethods = methods + .map( + (method) => + ''' +Future ${method.actionName}( + ${method.payloadType} payload, { + String? actionId, + String? orderingKey, + KalamOptimisticMutation? optimistic, +}) => _runner.enqueue( + actionKey: '${method.key}', + actionId: actionId ?? Kalam.id(), + payload: payload, + orderingKey: orderingKey, + optimistic: optimistic, +);''', + ) + .join('\n\n'); + + return ''' +List> $definitionsName($moduleName module) => [ +$definitions +]; + +final class ${moduleName}Queue { + const ${moduleName}Queue(this._runner); + + final KalamActionRunner _runner; + +$queueMethods +} +''' + .trim(); + } +} + +final class _ActionMethod { + const _ActionMethod( + this.methodName, + this.actionName, + this.key, + this.version, + this.payloadType, + ); + + final String methodName; + final String actionName; + final String key; + final int version; + final String payloadType; +} diff --git a/link/sdks/dart/generator/lib/src/kalam_action_payload_generator.dart b/link/sdks/dart/generator/lib/src/kalam_action_payload_generator.dart new file mode 100644 index 000000000..b9d29c2aa --- /dev/null +++ b/link/sdks/dart/generator/lib/src/kalam_action_payload_generator.dart @@ -0,0 +1,93 @@ +import 'package:analyzer/dart/element/element.dart'; +import 'package:analyzer/dart/element/type.dart'; +import 'package:build/build.dart'; +import 'package:kalam_sync/src/annotations/kalam_action_payload.dart'; +import 'package:source_gen/source_gen.dart'; + +final class KalamActionPayloadGenerator + extends GeneratorForAnnotation { + const KalamActionPayloadGenerator(); + + @override + String generateForAnnotatedElement( + Element element, + ConstantReader annotation, + BuildStep buildStep, + ) { + if (element is! ClassElement) { + throw InvalidGenerationSourceError( + '@KalamActionPayload can only annotate a class.', + element: element, + ); + } + final fields = element.fields.where((field) => !field.isStatic).toList(); + final constructor = element.unnamedConstructor; + if (constructor == null || + constructor.formalParameters.any((parameter) => !parameter.isNamed)) { + throw InvalidGenerationSourceError( + '${element.name} needs an unnamed constructor with named parameters.', + element: element, + ); + } + for (final field in fields) { + _decode(field.type, 'json[\'${field.name}\']', element); + } + + final name = element.name!; + final fromFields = fields + .map( + (field) => + '${field.name}: ${_decode(field.type, "json['${field.name}']", element)},', + ) + .join('\n'); + final toFields = fields + .map( + (field) => + "'${field.name}': ${_encode(field.type, 'value.${field.name}')},", + ) + .join('\n'); + return ''' +$name _\$${name}FromJson(Map json) => $name( +$fromFields +); + +Map _\$${name}ToJson($name value) => { +$toFields +}; +''' + .trim(); + } + + String _decode(DartType type, String source, Element element) { + final display = type.getDisplayString(); + final nullable = display.endsWith('?'); + final base = nullable ? display.substring(0, display.length - 1) : display; + late String expression; + if (base == 'String' || base == 'bool') { + expression = '$source as $base'; + } else if (base == 'int') { + expression = '($source as num).toInt()'; + } else if (base == 'double') { + expression = '($source as num).toDouble()'; + } else if (base == 'DateTime') { + expression = 'DateTime.parse($source as String)'; + } else if (base == 'List') { + expression = '($source as List).cast()'; + } else if (base == 'Map') { + expression = 'Map.from($source as Map)'; + } else { + throw InvalidGenerationSourceError( + 'Unsupported durable payload field type "$display".', + element: element, + ); + } + return nullable ? '$source == null ? null : $expression' : expression; + } + + String _encode(DartType type, String source) { + final display = type.getDisplayString(); + if (display == 'DateTime') return '$source.toUtc().toIso8601String()'; + if (display == 'DateTime?') return '$source?.toUtc().toIso8601String()'; + return source; + } +} diff --git a/link/sdks/dart/generator/pubspec.lock b/link/sdks/dart/generator/pubspec.lock new file mode 100644 index 000000000..59e346a70 --- /dev/null +++ b/link/sdks/dart/generator/pubspec.lock @@ -0,0 +1,797 @@ +# Generated by pub +# See https://dart.dev/tools/pub/glossary#lockfile +packages: + _fe_analyzer_shared: + dependency: transitive + description: + name: _fe_analyzer_shared + sha256: "1b0e6a07425a3e460666e88bf1c949ccc7bb0116ad562ce94a1eca60fe820725" + url: "https://pub.dev" + source: hosted + version: "103.0.0" + analyzer: + dependency: "direct main" + description: + name: analyzer + sha256: "61c04d0c1bfed555c681ea079519933f071a5a026578ff73c4ff0df2d3462e5e" + url: "https://pub.dev" + source: hosted + version: "13.3.0" + args: + dependency: transitive + description: + name: args + sha256: d0481093c50b1da8910eb0bb301626d4d8eb7284aa739614d2b394ee09e3ea04 + url: "https://pub.dev" + source: hosted + version: "2.7.0" + async: + dependency: transitive + description: + name: async + sha256: e2eb0491ba5ddb6177742d2da23904574082139b07c1e33b8503b9f46f3e1a37 + url: "https://pub.dev" + source: hosted + version: "2.13.1" + boolean_selector: + dependency: transitive + description: + name: boolean_selector + sha256: "8aab1771e1243a5063b8b0ff68042d67334e3feab9e95b9490f9a6ebf73b42ea" + url: "https://pub.dev" + source: hosted + version: "2.1.2" + build: + dependency: "direct main" + description: + name: build + sha256: b94f5da9ed3d081fd4ecc426c260998b5f4f4eb2f6d89b7e5472edef0b2b2a1b + url: "https://pub.dev" + source: hosted + version: "4.0.10" + build_cli_annotations: + dependency: transitive + description: + name: build_cli_annotations + sha256: e563c2e01de8974566a1998410d3f6f03521788160a02503b0b1f1a46c7b3d95 + url: "https://pub.dev" + source: hosted + version: "2.1.1" + build_config: + dependency: transitive + description: + name: build_config + sha256: "94eaf6708fe64408c632ef2689ca3777b112f9421306ccf4f8c84d7c5c9f83f8" + url: "https://pub.dev" + source: hosted + version: "1.3.2" + build_daemon: + dependency: transitive + description: + name: build_daemon + sha256: "79e05eaf15a48d7230b053a4363b8eaac0cc234bbd0134c3229455481f55cbc6" + url: "https://pub.dev" + source: hosted + version: "4.1.5" + build_runner: + dependency: "direct dev" + description: + name: build_runner + sha256: "90363d438bd9f84a4d5bbb83352251f57d2d9d771bc95a44e6a33fe25fa52774" + url: "https://pub.dev" + source: hosted + version: "2.16.0" + build_test: + dependency: "direct dev" + description: + name: build_test + sha256: ec07292e84b54cbbf06b9c7f09c156338f651e2146ccd57d6c98cb93ee28128d + url: "https://pub.dev" + source: hosted + version: "3.5.19" + built_collection: + dependency: transitive + description: + name: built_collection + sha256: "376e3dd27b51ea877c28d525560790aee2e6fbb5f20e2f85d5081027d94e2100" + url: "https://pub.dev" + source: hosted + version: "5.1.1" + built_value: + dependency: transitive + description: + name: built_value + sha256: "31b24be6615ec7fcf70b3aa5a7469fe35826485e639a16dd7eb83ba30e4cc6a8" + url: "https://pub.dev" + source: hosted + version: "8.12.7" + characters: + dependency: transitive + description: + name: characters + sha256: faf38497bda5ead2a8c7615f4f7939df04333478bf32e4173fcb06d428b5716b + url: "https://pub.dev" + source: hosted + version: "1.4.1" + checked_yaml: + dependency: transitive + description: + name: checked_yaml + sha256: "959525d3162f249993882720d52b7e0c833978df229be20702b33d48d91de70f" + url: "https://pub.dev" + source: hosted + version: "2.0.4" + cli_config: + dependency: transitive + description: + name: cli_config + sha256: ac20a183a07002b700f0c25e61b7ee46b23c309d76ab7b7640a028f18e4d99ec + url: "https://pub.dev" + source: hosted + version: "0.2.0" + clock: + dependency: transitive + description: + name: clock + sha256: fddb70d9b5277016c77a80201021d40a2247104d9f4aa7bab7157b7e3f05b84b + url: "https://pub.dev" + source: hosted + version: "1.1.2" + code_assets: + dependency: transitive + description: + name: code_assets + sha256: bf394f466ba9205f1812a0433b392d6af280f155f56651eda7c18cc32ed493b8 + url: "https://pub.dev" + source: hosted + version: "1.2.1" + collection: + dependency: transitive + description: + name: collection + sha256: "2f5709ae4d3d59dd8f7cd309b4e023046b57d8a6c82130785d2b0e5868084e76" + url: "https://pub.dev" + source: hosted + version: "1.19.1" + convert: + dependency: transitive + description: + name: convert + sha256: b30acd5944035672bc15c6b7a8b47d773e41e2f17de064350988c5d02adb1c68 + url: "https://pub.dev" + source: hosted + version: "3.1.2" + coverage: + dependency: transitive + description: + name: coverage + sha256: "956a3de0725ca232ad353565a8290d3357592bf4250f6f298a185e2d949c5d3d" + url: "https://pub.dev" + source: hosted + version: "1.15.1" + crypto: + dependency: transitive + description: + name: crypto + sha256: c8ea0233063ba03258fbcf2ca4d6dadfefe14f02fab57702265467a19f27fadf + url: "https://pub.dev" + source: hosted + version: "3.0.7" + csslib: + dependency: transitive + description: + name: csslib + sha256: "09bad715f418841f976c77db72d5398dc1253c21fb9c0c7f0b0b985860b2d58e" + url: "https://pub.dev" + source: hosted + version: "1.0.2" + dart_style: + dependency: transitive + description: + name: dart_style + sha256: "3f88fc9c96c568d631356507355a2d1e983ee000c9ac008bdcb39b5bf53ce777" + url: "https://pub.dev" + source: hosted + version: "3.1.12" + drift: + dependency: transitive + description: + name: drift + sha256: "3a3f1f6f905037d7426e4c445854139fd6a3d592135f7c96d7931682b73d16f4" + url: "https://pub.dev" + source: hosted + version: "2.34.3" + drift_flutter: + dependency: transitive + description: + name: drift_flutter + sha256: "91acf4bee7c3c84467cba46455aa70e5292a3b889f4582645d74f2e5a8c106f2" + url: "https://pub.dev" + source: hosted + version: "0.3.1" + ffi: + dependency: transitive + description: + name: ffi + sha256: "6d7fd89431262d8f3125e81b50d3847a091d846eafcd4fdb88dd06f36d705a45" + url: "https://pub.dev" + source: hosted + version: "2.2.0" + file: + dependency: transitive + description: + name: file + sha256: a3b4f84adafef897088c160faf7dfffb7696046cb13ae90b508c2cbc95d3b8d4 + url: "https://pub.dev" + source: hosted + version: "7.0.1" + fixnum: + dependency: transitive + description: + name: fixnum + sha256: b6dc7065e46c974bc7c5f143080a6764ec7a4be6da1285ececdc37be96de53be + url: "https://pub.dev" + source: hosted + version: "1.1.1" + flutter: + dependency: transitive + description: flutter + source: sdk + version: "0.0.0" + flutter_rust_bridge: + dependency: transitive + description: + name: flutter_rust_bridge + sha256: e87d6b9ee934dcd24a128ccb2bd91905d2d5fe5c06245d6a8f5477d4907a437a + url: "https://pub.dev" + source: hosted + version: "2.12.0" + freezed_annotation: + dependency: transitive + description: + name: freezed_annotation + sha256: "7294967ff0a6d98638e7acb774aac3af2550777accd8149c90af5b014e6d44d8" + url: "https://pub.dev" + source: hosted + version: "3.1.0" + frontend_server_client: + dependency: transitive + description: + name: frontend_server_client + sha256: f64a0333a82f30b0cca061bc3d143813a486dc086b574bfb233b7c1372427694 + url: "https://pub.dev" + source: hosted + version: "4.0.0" + glob: + dependency: transitive + description: + name: glob + sha256: c3f1ee72c96f8f78935e18aa8cecced9ab132419e8625dc187e1c2408efc20de + url: "https://pub.dev" + source: hosted + version: "2.1.3" + graphs: + dependency: transitive + description: + name: graphs + sha256: "741bbf84165310a68ff28fe9e727332eef1407342fca52759cb21ad8177bb8d0" + url: "https://pub.dev" + source: hosted + version: "2.3.2" + hooks: + dependency: transitive + description: + name: hooks + sha256: "03a564c3704524ee0f7fc56fc621e8796cc2eb8c24d1f1a33b34979815b285c4" + url: "https://pub.dev" + source: hosted + version: "2.1.0" + html: + dependency: transitive + description: + name: html + sha256: "6d1264f2dffa1b1101c25a91dff0dc2daee4c18e87cd8538729773c073dbf602" + url: "https://pub.dev" + source: hosted + version: "0.15.6" + http_multi_server: + dependency: transitive + description: + name: http_multi_server + sha256: aa6199f908078bb1c5efb8d8638d4ae191aac11b311132c3ef48ce352fb52ef8 + url: "https://pub.dev" + source: hosted + version: "3.2.2" + http_parser: + dependency: transitive + description: + name: http_parser + sha256: "178d74305e7866013777bab2c3d8726205dc5a4dd935297175b19a23a2e66571" + url: "https://pub.dev" + source: hosted + version: "4.1.2" + io: + dependency: transitive + description: + name: io + sha256: dfd5a80599cf0165756e3181807ed3e77daf6dd4137caaad72d0b7931597650b + url: "https://pub.dev" + source: hosted + version: "1.0.5" + jni: + dependency: transitive + description: + name: jni + sha256: f038e58b4dc2c9037f50e233175086337e0b305e356d28211bf55f21c504cbd3 + url: "https://pub.dev" + source: hosted + version: "1.0.3" + jni_flutter: + dependency: transitive + description: + name: jni_flutter + sha256: "7b717011ea40d04fd47c2731d3d1d36eb99eba3435c2753d62489e8c3c9991d5" + url: "https://pub.dev" + source: hosted + version: "1.0.2" + jni_util: + dependency: transitive + description: + name: jni_util + sha256: "1ba86da04a5f2bf18fde2edb235587e70c5b0fc5bd4ba955f46b00942c3fc35f" + url: "https://pub.dev" + source: hosted + version: "1.0.0" + json_annotation: + dependency: transitive + description: + name: json_annotation + sha256: "2a743920d81b7910627f68ee2c9ac1fc0bfee32b9fc3403587d7c6791ca12f80" + url: "https://pub.dev" + source: hosted + version: "4.12.0" + kalam_link: + dependency: "direct overridden" + description: + path: "../link" + relative: true + source: path + version: "0.5.6-rc.0" + kalam_sync: + dependency: "direct main" + description: + path: "../sync" + relative: true + source: path + version: "0.5.6-rc.0" + kalam_sync_generator_example: + dependency: "direct dev" + description: + path: example + relative: true + source: path + version: "0.5.6-rc.0" + logger: + dependency: transitive + description: + name: logger + sha256: "25aee487596a6257655a1e091ec2ae66bc30e7af663592cc3a27e6591e05035c" + url: "https://pub.dev" + source: hosted + version: "2.7.0" + logging: + dependency: transitive + description: + name: logging + sha256: c8245ada5f1717ed44271ed1c26b8ce85ca3228fd2ffdb75468ab01979309d61 + url: "https://pub.dev" + source: hosted + version: "1.3.0" + matcher: + dependency: transitive + description: + name: matcher + sha256: "31bd099b47c10cd1aeb55146a2d46ce0277630ecef3f7dae54ad7873f36696cd" + url: "https://pub.dev" + source: hosted + version: "0.12.20" + material_color_utilities: + dependency: transitive + description: + name: material_color_utilities + sha256: "9c337007e82b1889149c82ed242ed1cb24a66044e30979c44912381e9be4c48b" + url: "https://pub.dev" + source: hosted + version: "0.13.0" + meta: + dependency: transitive + description: + name: meta + sha256: "307249ce4ff29d58a18e97f6345f539382eb9c9c29ecda628900f31de0443dd9" + url: "https://pub.dev" + source: hosted + version: "1.19.0" + mime: + dependency: transitive + description: + name: mime + sha256: "41a20518f0cb1256669420fdba0cd90d21561e560ac240f26ef8322e45bb7ed6" + url: "https://pub.dev" + source: hosted + version: "2.0.0" + native_toolchain_c: + dependency: transitive + description: + name: native_toolchain_c + sha256: a1c26117c48cebe5677b0cf0e33a980a79a7c5577effc86f52e5a0d309cdcb60 + url: "https://pub.dev" + source: hosted + version: "0.19.3" + node_preamble: + dependency: transitive + description: + name: node_preamble + sha256: "6e7eac89047ab8a8d26cf16127b5ed26de65209847630400f9aefd7cd5c730db" + url: "https://pub.dev" + source: hosted + version: "2.0.2" + objective_c: + dependency: transitive + description: + name: objective_c + sha256: b7fb95a6d9a4f009edd63dc5ac69f07420b23a16161c6dd8660290b59c602e8e + url: "https://pub.dev" + source: hosted + version: "9.5.0" + package_config: + dependency: transitive + description: + name: package_config + sha256: ffcf4cf3d6c0b74ac43708d9f56625506e8a68aa935abe9d267a7330f320eb5d + url: "https://pub.dev" + source: hosted + version: "3.0.0" + path: + dependency: transitive + description: + name: path + sha256: "75cca69d1490965be98c73ceaea117e8a04dd21217b37b292c9ddbec0d955bc5" + url: "https://pub.dev" + source: hosted + version: "1.9.1" + path_provider: + dependency: transitive + description: + name: path_provider + sha256: a7f4874f987173da295a61c181b8ee71dab59b332a486b391babf26a1b884825 + url: "https://pub.dev" + source: hosted + version: "2.1.6" + path_provider_android: + dependency: transitive + description: + name: path_provider_android + sha256: "69cbd515a62b94d32a7944f086b2f82b4ac40a1d45bebfc00813a430ab2dabcd" + url: "https://pub.dev" + source: hosted + version: "2.3.1" + path_provider_foundation: + dependency: transitive + description: + name: path_provider_foundation + sha256: "2a376b7d6392d80cd3705782d2caa734ca4727776db0b6ec36ef3f1855197699" + url: "https://pub.dev" + source: hosted + version: "2.6.0" + path_provider_linux: + dependency: transitive + description: + name: path_provider_linux + sha256: "58c2005f147315b11e9b4a7bc889cd5203e250cba8e3f012dae259b4972b5c16" + url: "https://pub.dev" + source: hosted + version: "2.2.2" + path_provider_platform_interface: + dependency: transitive + description: + name: path_provider_platform_interface + sha256: "484838772624c3a4b94f1e44a3e19897fee738f2d5c4ce448443b0417f7c9dda" + url: "https://pub.dev" + source: hosted + version: "2.1.3" + path_provider_windows: + dependency: transitive + description: + name: path_provider_windows + sha256: bd6f00dbd873bfb70d0761682da2b3a2c2fccc2b9e84c495821639601d81afe7 + url: "https://pub.dev" + source: hosted + version: "2.3.0" + platform: + dependency: transitive + description: + name: platform + sha256: "5d6b1b0036a5f331ebc77c850ebc8506cbc1e9416c27e59b439f917a902a4984" + url: "https://pub.dev" + source: hosted + version: "3.1.6" + plugin_platform_interface: + dependency: transitive + description: + name: plugin_platform_interface + sha256: "4820fbfdb9478b1ebae27888254d445073732dae3d6ea81f0b7e06d5dedc3f02" + url: "https://pub.dev" + source: hosted + version: "2.1.8" + pool: + dependency: transitive + description: + name: pool + sha256: "978783255c543aa3586a1b3c21f6e9d720eb315376a915872c61ef8b5c20177d" + url: "https://pub.dev" + source: hosted + version: "1.5.2" + pub_semver: + dependency: transitive + description: + name: pub_semver + sha256: "5bfcf68ca79ef689f8990d1160781b4bad40a3bd5e5218ad4076ddb7f4081585" + url: "https://pub.dev" + source: hosted + version: "2.2.0" + pubspec_parse: + dependency: transitive + description: + name: pubspec_parse + sha256: "0560ba233314abbed0a48a2956f7f022cce7c3e1e73df540277da7544cad4082" + url: "https://pub.dev" + source: hosted + version: "1.5.0" + record_use: + dependency: transitive + description: + name: record_use + sha256: "3b4ab682aff40175afca6ff090cc5aa7ce9dafe8dfbae1537a6c678e6678baee" + url: "https://pub.dev" + source: hosted + version: "1.1.0" + shelf: + dependency: transitive + description: + name: shelf + sha256: e7dd780a7ffb623c57850b33f43309312fc863fb6aa3d276a754bb299839ef12 + url: "https://pub.dev" + source: hosted + version: "1.4.2" + shelf_packages_handler: + dependency: transitive + description: + name: shelf_packages_handler + sha256: "89f967eca29607c933ba9571d838be31d67f53f6e4ee15147d5dc2934fee1b1e" + url: "https://pub.dev" + source: hosted + version: "3.0.2" + shelf_static: + dependency: transitive + description: + name: shelf_static + sha256: c87c3875f91262785dade62d135760c2c69cb217ac759485334c5857ad89f6e3 + url: "https://pub.dev" + source: hosted + version: "1.1.3" + shelf_web_socket: + dependency: transitive + description: + name: shelf_web_socket + sha256: "3632775c8e90d6c9712f883e633716432a27758216dfb61bd86a8321c0580925" + url: "https://pub.dev" + source: hosted + version: "3.0.0" + sky_engine: + dependency: transitive + description: flutter + source: sdk + version: "0.0.0" + source_gen: + dependency: "direct main" + description: + name: source_gen + sha256: a603f1fb984a7391ae5978d1b92bfaaa08b350dca5c825256f925818f7943bf5 + url: "https://pub.dev" + source: hosted + version: "4.2.4" + source_map_stack_trace: + dependency: transitive + description: + name: source_map_stack_trace + sha256: c0713a43e323c3302c2abe2a1cc89aa057a387101ebd280371d6a6c9fa68516b + url: "https://pub.dev" + source: hosted + version: "2.1.2" + source_maps: + dependency: transitive + description: + name: source_maps + sha256: "190222579a448b03896e0ca6eca5998fa810fda630c1d65e2f78b3f638f54812" + url: "https://pub.dev" + source: hosted + version: "0.10.13" + source_span: + dependency: transitive + description: + name: source_span + sha256: "56a02f1f4cd1a2d96303c0144c93bd6d909eea6bee6bf5a0e0b685edbd4c47ab" + url: "https://pub.dev" + source: hosted + version: "1.10.2" + sqlcipher_flutter_libs: + dependency: transitive + description: + name: sqlcipher_flutter_libs + sha256: "38d62d659d2fb8739bf25a42c9a350d1fdd6c29a5a61f13a946778ec75d27929" + url: "https://pub.dev" + source: hosted + version: "0.7.0+eol" + sqlite3: + dependency: transitive + description: + name: sqlite3 + sha256: "64b2c63c8232dd20d14b34105a81ebfd74320442e8451f836179ec89986aa478" + url: "https://pub.dev" + source: hosted + version: "3.5.1" + sqlite3_flutter_libs: + dependency: transitive + description: + name: sqlite3_flutter_libs + sha256: "3ed7553eee7bb368f8950f58ba29f634e06e813c029aff6a0d60862b96de8454" + url: "https://pub.dev" + source: hosted + version: "0.6.0+eol" + stack_trace: + dependency: transitive + description: + name: stack_trace + sha256: "8b27215b45d22309b5cddda1aa2b19bdfec9df0e765f2de506401c071d38d1b1" + url: "https://pub.dev" + source: hosted + version: "1.12.1" + stream_channel: + dependency: transitive + description: + name: stream_channel + sha256: "969e04c80b8bcdf826f8f16579c7b14d780458bd97f56d107d3950fdbeef059d" + url: "https://pub.dev" + source: hosted + version: "2.1.4" + stream_transform: + dependency: transitive + description: + name: stream_transform + sha256: ad47125e588cfd37a9a7f86c7d6356dde8dfe89d071d293f80ca9e9273a33871 + url: "https://pub.dev" + source: hosted + version: "2.1.1" + string_scanner: + dependency: transitive + description: + name: string_scanner + sha256: "921cd31725b72fe181906c6a94d987c78e3b98c2e205b397ea399d4054872b43" + url: "https://pub.dev" + source: hosted + version: "1.4.1" + term_glyph: + dependency: transitive + description: + name: term_glyph + sha256: "7f554798625ea768a7518313e58f83891c7f5024f88e46e7182a4558850a4b8e" + url: "https://pub.dev" + source: hosted + version: "1.2.2" + test: + dependency: "direct dev" + description: + name: test + sha256: "0d5ba5602ec3baa28c8ce365e1efc5575969c765f45c554a3e167dc7945b9c30" + url: "https://pub.dev" + source: hosted + version: "1.31.2" + test_api: + dependency: transitive + description: + name: test_api + sha256: "475610b2aa23c19687cce2961e44b0cc57cafe220f67c2b80201231b2a07fbe7" + url: "https://pub.dev" + source: hosted + version: "0.7.13" + test_core: + dependency: transitive + description: + name: test_core + sha256: a39c204a4fc7a7ccb04a2b985e359fda3cc37e45e0b8ac61c3fb1a05aa832132 + url: "https://pub.dev" + source: hosted + version: "0.6.19" + typed_data: + dependency: transitive + description: + name: typed_data + sha256: f9049c039ebfeb4cf7a7104a675823cd72dba8297f264b6637062516699fa006 + url: "https://pub.dev" + source: hosted + version: "1.4.0" + vector_math: + dependency: transitive + description: + name: vector_math + sha256: f36f9f3be64c6198714492bb455c11056e33e2f85d9a0b676a48301e44fdcf47 + url: "https://pub.dev" + source: hosted + version: "2.4.2" + vm_service: + dependency: transitive + description: + name: vm_service + sha256: "0016aef94fc66495ac78af5859181e3f3bf2026bd8eecc72b9565601e19ab360" + url: "https://pub.dev" + source: hosted + version: "15.2.0" + watcher: + dependency: transitive + description: + name: watcher + sha256: "1398c9f081a753f9226febe8900fce8f7d0a67163334e1c94a2438339d79d635" + url: "https://pub.dev" + source: hosted + version: "1.2.1" + web: + dependency: transitive + description: + name: web + sha256: "868d88a33d8a87b18ffc05f9f030ba328ffefba92d6c127917a2ba740f9cfe4a" + url: "https://pub.dev" + source: hosted + version: "1.1.1" + web_socket: + dependency: transitive + description: + name: web_socket + sha256: "34d64019aa8e36bf9842ac014bb5d2f5586ca73df5e4d9bf5c936975cae6982c" + url: "https://pub.dev" + source: hosted + version: "1.0.1" + web_socket_channel: + dependency: transitive + description: + name: web_socket_channel + sha256: d645757fb0f4773d602444000a8131ff5d48c9e47adfe9772652dd1a4f2d45c8 + url: "https://pub.dev" + source: hosted + version: "3.0.3" + webkit_inspection_protocol: + dependency: transitive + description: + name: webkit_inspection_protocol + sha256: "87d3f2333bb240704cd3f1c6b5b7acd8a10e7f0bc28c28dcf14e782014f4a572" + url: "https://pub.dev" + source: hosted + version: "1.2.1" + xdg_directories: + dependency: transitive + description: + name: xdg_directories + sha256: "7a3f37b05d989967cdddcbb571f1ea834867ae2faa29725fd085180e0883aa15" + url: "https://pub.dev" + source: hosted + version: "1.1.0" + yaml: + dependency: transitive + description: + name: yaml + sha256: b9da305ac7c39faa3f030eccd175340f968459dae4af175130b3fc47e40d76ce + url: "https://pub.dev" + source: hosted + version: "3.1.3" +sdks: + dart: ">=3.11.0 <4.0.0" + flutter: ">=3.38.4" diff --git a/link/sdks/dart/generator/pubspec.yaml b/link/sdks/dart/generator/pubspec.yaml new file mode 100644 index 000000000..8dd9907f3 --- /dev/null +++ b/link/sdks/dart/generator/pubspec.yaml @@ -0,0 +1,21 @@ +name: kalam_sync_generator +description: Code generation for typed kalam_sync action payloads and queues. +version: 0.5.6-rc.0 +homepage: https://github.com/kalamdb/KalamDB +repository: https://github.com/kalamdb/KalamDB + +environment: + sdk: ">=3.11.0 <4.0.0" + +dependencies: + analyzer: ">=13.3.0 <15.0.0" + build: ^4.0.7 + kalam_sync: ">=0.5.6-0 <0.6.0" + source_gen: ^4.2.4 + +dev_dependencies: + build_runner: ^2.16.0 + build_test: ^3.5.16 + kalam_sync_generator_example: + path: example + test: ^1.26.3 diff --git a/link/sdks/dart/generator/pubspec_overrides.yaml b/link/sdks/dart/generator/pubspec_overrides.yaml new file mode 100644 index 000000000..fd04903b3 --- /dev/null +++ b/link/sdks/dart/generator/pubspec_overrides.yaml @@ -0,0 +1,7 @@ +dependency_overrides: + kalam_link: + path: ../link + kalam_sync: + path: ../sync + kalam_sync_generator_example: + path: example diff --git a/link/sdks/dart/generator/test/kalam_action_generator_test.dart b/link/sdks/dart/generator/test/kalam_action_generator_test.dart new file mode 100644 index 000000000..700ec5e1e --- /dev/null +++ b/link/sdks/dart/generator/test/kalam_action_generator_test.dart @@ -0,0 +1,197 @@ +import 'package:build/build.dart'; +import 'package:build_test/build_test.dart'; +import 'package:kalam_sync_generator/kalam_sync_generator.dart'; +import 'package:test/test.dart'; + +void main() { + test( + 'generates payload codec, stable definitions, and queue methods', + () async { + final reader = await _reader(); + await testBuilder( + kalamActionBuilder(BuilderOptions.empty), + const { + 'kalam_sync_generator|lib/messages.dart': ''' +library messages; +import 'package:kalam_sync/kalam_sync.dart'; +part 'messages.g.dart'; + +@KalamActionPayload() +class SendMessageArgs { + const SendMessageArgs({ + required this.messageId, + required this.text, + required this.createdAt, + }); + final String messageId; + final String text; + final DateTime createdAt; +} + +@KalamActionModule(namespace: 'messages') +class MessageActions { + @KalamAction(name: 'send', version: 2) + Future send( + KalamActionContext context, + SendMessageArgs payload, + ) async {} +} +''', + }, + outputs: { + 'kalam_sync_generator|lib/messages.kalam_actions.g.part': + decodedMatches( + allOf( + contains('SendMessageArgs _\$SendMessageArgsFromJson'), + contains("key: 'messages.send'"), + contains('version: 2'), + contains('final class MessageActionsQueue'), + contains('Future send('), + contains('actionId: actionId ?? Kalam.id()'), + ), + ), + }, + readerWriter: reader, + rootPackage: 'kalam_sync_generator', + flattenOutput: true, + ); + }, + ); + + test('rejects payload fields that cannot be durably encoded', () async { + final logs = []; + final reader = await _reader(); + await testBuilder( + kalamActionBuilder(BuilderOptions.empty), + const { + 'kalam_sync_generator|lib/invalid.dart': ''' +library invalid; +import 'package:kalam_sync/src/annotations/kalam_action_payload.dart'; +part 'invalid.g.dart'; + +@KalamActionPayload() +class InvalidPayload { + const InvalidPayload({required this.callback}); + final void Function() callback; +} +''', + }, + outputs: const {}, + onLog: (record) => logs.add(record.message), + readerWriter: reader, + rootPackage: 'kalam_sync_generator', + flattenOutput: true, + ); + expect(logs.join('\n'), contains('Unsupported durable payload field')); + }); + + test('rejects duplicate durable action keys', () async { + final logs = []; + final reader = await _reader(); + await testBuilder( + kalamActionBuilder(BuilderOptions.empty), + const { + 'kalam_sync_generator|lib/duplicate.dart': ''' +library duplicate; +import 'package:kalam_sync/src/actions/kalam_action_context.dart'; +import 'package:kalam_sync/src/annotations/kalam_action.dart'; +import 'package:kalam_sync/src/annotations/kalam_action_module.dart'; +import 'package:kalam_sync/src/annotations/kalam_action_payload.dart'; +part 'duplicate.g.dart'; + +@KalamActionPayload() +class Args { + const Args({required this.id}); + final String id; +} + +@KalamActionModule(namespace: 'messages') +class DuplicateActions { + @KalamAction(name: 'send') + Future first(KalamActionContext context, Args payload) async {} + + @KalamAction(name: 'send') + Future second(KalamActionContext context, Args payload) async {} +} +''', + }, + outputs: const {}, + onLog: (record) => logs.add(record.message), + readerWriter: reader, + rootPackage: 'kalam_sync_generator', + flattenOutput: true, + ); + expect(logs.join('\n'), contains('Duplicate action key')); + }); + + test( + 'generates independent queues for multiple table action modules', + () async { + final reader = await _reader(); + await testBuilder( + kalamActionBuilder(BuilderOptions.empty), + const { + 'kalam_sync_generator|lib/chat_actions.dart': ''' +library chat_actions; +import 'package:kalam_sync/kalam_sync.dart'; +part 'chat_actions.g.dart'; + +@KalamActionPayload() +class MessageArgs { + const MessageArgs({required this.id}); + final String id; +} + +@KalamActionPayload() +class ConversationArgs { + const ConversationArgs({required this.id}); + final String id; +} + +@KalamActionModule(namespace: 'messages') +class MessageActions { + @KalamAction(name: 'create') + Future create(KalamActionContext context, MessageArgs payload) async {} + + @KalamAction(name: 'delete') + Future delete(KalamActionContext context, MessageArgs payload) async {} +} + +@KalamActionModule(namespace: 'conversations') +class ConversationActions { + @KalamAction(name: 'archive') + Future archive( + KalamActionContext context, + ConversationArgs payload, + ) async {} +} +''', + }, + outputs: { + 'kalam_sync_generator|lib/chat_actions.kalam_actions.g.part': + decodedMatches( + allOf( + contains("key: 'messages.create'"), + contains("key: 'messages.delete'"), + contains("key: 'conversations.archive'"), + contains('final class MessageActionsQueue'), + contains('final class ConversationActionsQueue'), + ), + ), + }, + readerWriter: reader, + rootPackage: 'kalam_sync_generator', + flattenOutput: true, + ); + }, + ); +} + +Future _reader() async { + final reader = TestReaderWriter( + rootPackage: 'kalam_sync_generator', + flattenOutput: true, + ); + await reader.testing.loadIsolateSources(); + return reader; +} diff --git a/link/sdks/dart/CHANGELOG.md b/link/sdks/dart/link/CHANGELOG.md similarity index 100% rename from link/sdks/dart/CHANGELOG.md rename to link/sdks/dart/link/CHANGELOG.md diff --git a/link/sdks/dart/DEV.md b/link/sdks/dart/link/DEV.md similarity index 94% rename from link/sdks/dart/DEV.md rename to link/sdks/dart/link/DEV.md index 2a7d5ec52..3ccddf091 100644 --- a/link/sdks/dart/DEV.md +++ b/link/sdks/dart/link/DEV.md @@ -46,12 +46,12 @@ cd link/kalam-link-dart flutter_rust_bridge_codegen generate ``` -Generated files land in `link/sdks/dart/lib/src/generated/` — commit them together with the +Generated files land in `link/sdks/dart/link/lib/src/generated/` — commit them together with the Rust changes. ## Local Scripts -From `link/sdks/dart`: +From `link/sdks/dart/link`: ```bash ./build.sh # canonical build: deps + optional FRB generation + native artefacts + analyze @@ -114,7 +114,7 @@ Prerequisites vary by platform: | Web | Rust stable, `wasm-pack`, `rustup target add wasm32-unknown-unknown` | ```bash -# From link/sdks/dart +# From link/sdks/dart/link # Auto-detect and build for current OS platforms ./build.sh @@ -157,7 +157,7 @@ Notes: ## Publishing ```bash -cd link/sdks/dart +cd link/sdks/dart/link ./publish.sh # full publish ./publish.sh --dry-run # validate only ``` @@ -180,5 +180,5 @@ Before a new release: | `link/kalam-link-wasm/` | Browser WASM entry used for Flutter `web/pkg/` artefacts | | `link/link-common/src/` | Shared Rust client implementation (HTTP, WebSocket, auth) | | `link/kalam-link-dart/` | FRB bridge crate — wraps the shared client with `#[frb]` annotations | -| `link/sdks/dart/lib/src/generated/` | Auto-generated Dart bindings (do not edit manually) | -| `link/sdks/dart/lib/src/` | Hand-written Dart API layer (`kalam_client.dart`, `auth.dart`, `models.dart`) | +| `link/sdks/dart/link/lib/src/generated/` | Auto-generated Dart bindings (do not edit manually) | +| `link/sdks/dart/link/lib/src/` | Hand-written Dart API layer (`kalam_client.dart`, `auth.dart`, `models.dart`) | diff --git a/link/sdks/dart/link/LICENSE b/link/sdks/dart/link/LICENSE new file mode 100644 index 000000000..f57b60e66 --- /dev/null +++ b/link/sdks/dart/link/LICENSE @@ -0,0 +1,183 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship made available under + the License, as indicated by a copyright notice that is included in + or attached to the work (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean, as submitted to the Licensor for inclusion + in the Work by the copyright owner or by an individual or Legal Entity + authorized to submit on behalf of the copyright owner. For the purposes + of this definition, "submitted" means any form of electronic, verbal, + or written communication sent to the Licensor or its representatives, + including but not limited to communication on electronic mailing lists, + source code control systems, and issue tracking systems that are managed + by, or on behalf of, the Licensor for the purpose of discussing and + improving the Work, but excluding communication that is conspicuously + marked or otherwise designated in writing by the copyright owner as + "Not a Contribution." + + "Contributor" shall mean Licensor and any Legal Entity on behalf of + whom a Contribution has been received by the Licensor and incorporated + within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a cross-claim + or counterclaim in a lawsuit) alleging that the Work or any + Contribution embodied within the Work constitutes direct or contributory + patent infringement, then any patent licenses granted to You under + this License for that Work shall terminate as of the date such + litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or Derivative + Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, You must include a readable copy of the + attribution notices contained within such NOTICE file, in + at least one of the following places: within a NOTICE text + file distributed as part of the Derivative Works; within + the Source form or documentation, if provided along with the + Derivative Works; or, within a display generated by the + Derivative Works, if and wherever such third-party notices + normally appear. The contents of the NOTICE file are for + informational purposes only and do not modify the License. + You may add Your own attribution notices within Derivative + Works that You distribute, alongside or as an addendum to + the NOTICE text from the Work, provided that such additional + attribution notices cannot be construed as modifying the License. + + You may add Your own license statement for Your modifications and + may provide additional grant of rights to use, copy, modify, merge, + publish, distribute, sublicense, and/or sell copies of the + Contribution, either on a standalone basis or as part of the Work. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any conditions of title, + merchantability, fitness for a particular purpose and + non-infringement. You are solely responsible for determining the + appropriateness of using or reproducing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or exemplary damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or all other + commercial damages or losses), even if such Contributor has been + advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may offer only + conditions consistent with this terms and not on behalf of any other + Contributor, and only if You agree to indemnify, defend, and hold each + Contributor harmless for any liability incurred by, or claims asserted + against, such Contributor by reason of your accepting any such + warranty or additional liability. + + END OF TERMS AND CONDITIONS + + Copyright 2025 Jamal Saad + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/link/sdks/dart/NOTICE b/link/sdks/dart/link/NOTICE similarity index 100% rename from link/sdks/dart/NOTICE rename to link/sdks/dart/link/NOTICE diff --git a/link/sdks/dart/README.md b/link/sdks/dart/link/README.md similarity index 97% rename from link/sdks/dart/README.md rename to link/sdks/dart/link/README.md index 51929c0ee..8a4244b7a 100644 --- a/link/sdks/dart/README.md +++ b/link/sdks/dart/link/README.md @@ -549,8 +549,8 @@ Configure Dart output in your project's `kalam.toml`: output = "lib/generated/kalam.dart" ``` -Today `kalam schema gen` only writes a placeholder file for Dart targets while the dedicated Dart generator package is still pending. +`kalam schema gen` (and `kalam schema gen --languages dart`) reads `schema.sql` and writes row classes plus `KalamTableSpec` values for `kalam_sync`. Action queues stay with `kalam_sync_generator` / `build_runner` — schema gen does not emit those. -`kalam dev` preserves that placeholder behavior when Dart output is configured in `kalam.toml`. Use `kalam status` to verify the resolved environment and migration readiness before `kalam deploy`. +`kalam init --languages dart --template simple-live` scaffolds a Flutter starter (`pubspec.yaml`, `lib/main.dart`, generated specs). `kalam dev` regenerates Dart when `generate_types = true`. Use `kalam status` to verify the resolved environment and migration readiness. > Native performance on iOS and Android is powered by [flutter_rust_bridge](https://cjycode.com/flutter_rust_bridge/). diff --git a/link/sdks/dart/analysis_options.yaml b/link/sdks/dart/link/analysis_options.yaml similarity index 100% rename from link/sdks/dart/analysis_options.yaml rename to link/sdks/dart/link/analysis_options.yaml diff --git a/link/sdks/dart/android/.gitignore b/link/sdks/dart/link/android/.gitignore similarity index 100% rename from link/sdks/dart/android/.gitignore rename to link/sdks/dart/link/android/.gitignore diff --git a/link/sdks/dart/android/CMakeLists.txt b/link/sdks/dart/link/android/CMakeLists.txt similarity index 100% rename from link/sdks/dart/android/CMakeLists.txt rename to link/sdks/dart/link/android/CMakeLists.txt diff --git a/link/sdks/dart/android/build.gradle b/link/sdks/dart/link/android/build.gradle similarity index 100% rename from link/sdks/dart/android/build.gradle rename to link/sdks/dart/link/android/build.gradle diff --git a/link/sdks/dart/android/settings.gradle b/link/sdks/dart/link/android/settings.gradle similarity index 100% rename from link/sdks/dart/android/settings.gradle rename to link/sdks/dart/link/android/settings.gradle diff --git a/link/sdks/dart/android/src/main/AndroidManifest.xml b/link/sdks/dart/link/android/src/main/AndroidManifest.xml similarity index 100% rename from link/sdks/dart/android/src/main/AndroidManifest.xml rename to link/sdks/dart/link/android/src/main/AndroidManifest.xml diff --git a/link/sdks/dart/android/src/main/jniLibs/.gitignore b/link/sdks/dart/link/android/src/main/jniLibs/.gitignore similarity index 100% rename from link/sdks/dart/android/src/main/jniLibs/.gitignore rename to link/sdks/dart/link/android/src/main/jniLibs/.gitignore diff --git a/link/sdks/dart/android/src/main/jniLibs/arm64-v8a/libkalam_link_dart.so b/link/sdks/dart/link/android/src/main/jniLibs/arm64-v8a/libkalam_link_dart.so similarity index 100% rename from link/sdks/dart/android/src/main/jniLibs/arm64-v8a/libkalam_link_dart.so rename to link/sdks/dart/link/android/src/main/jniLibs/arm64-v8a/libkalam_link_dart.so diff --git a/link/sdks/dart/android/src/main/jniLibs/x86_64/libkalam_link_dart.so b/link/sdks/dart/link/android/src/main/jniLibs/x86_64/libkalam_link_dart.so similarity index 100% rename from link/sdks/dart/android/src/main/jniLibs/x86_64/libkalam_link_dart.so rename to link/sdks/dart/link/android/src/main/jniLibs/x86_64/libkalam_link_dart.so diff --git a/link/sdks/dart/build.sh b/link/sdks/dart/link/build.sh similarity index 99% rename from link/sdks/dart/build.sh rename to link/sdks/dart/link/build.sh index 76576473d..7cb06af2c 100755 --- a/link/sdks/dart/build.sh +++ b/link/sdks/dart/link/build.sh @@ -4,8 +4,8 @@ set -euo pipefail echo "🔨 Building KalamDB Dart SDK..." SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" -BRIDGE_DIR="$(cd "$SCRIPT_DIR/../../kalam-link-dart" && pwd)" -LINK_WORKSPACE_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" +BRIDGE_DIR="$(cd "$SCRIPT_DIR/../../../kalam-link-dart" && pwd)" +LINK_WORKSPACE_DIR="$(cd "$SCRIPT_DIR/../../.." && pwd)" cd "$SCRIPT_DIR" TARGET_DIR="$(cargo metadata --manifest-path "$BRIDGE_DIR/Cargo.toml" --format-version=1 --no-deps 2>/dev/null \ diff --git a/link/sdks/dart/example/chat-app/main.dart b/link/sdks/dart/link/example/chat-app/main.dart similarity index 100% rename from link/sdks/dart/example/chat-app/main.dart rename to link/sdks/dart/link/example/chat-app/main.dart diff --git a/link/sdks/dart/example/simple-events/main.dart b/link/sdks/dart/link/example/simple-events/main.dart similarity index 100% rename from link/sdks/dart/example/simple-events/main.dart rename to link/sdks/dart/link/example/simple-events/main.dart diff --git a/link/sdks/dart/example/support.dart b/link/sdks/dart/link/example/support.dart similarity index 100% rename from link/sdks/dart/example/support.dart rename to link/sdks/dart/link/example/support.dart diff --git a/link/sdks/dart/ios/.gitignore b/link/sdks/dart/link/ios/.gitignore similarity index 100% rename from link/sdks/dart/ios/.gitignore rename to link/sdks/dart/link/ios/.gitignore diff --git a/link/sdks/dart/ios/Classes/KalamLinkPlugin.swift b/link/sdks/dart/link/ios/Classes/KalamLinkPlugin.swift similarity index 100% rename from link/sdks/dart/ios/Classes/KalamLinkPlugin.swift rename to link/sdks/dart/link/ios/Classes/KalamLinkPlugin.swift diff --git a/link/sdks/dart/ios/Frameworks/libkalam_link_dart.a b/link/sdks/dart/link/ios/Frameworks/libkalam_link_dart.a similarity index 100% rename from link/sdks/dart/ios/Frameworks/libkalam_link_dart.a rename to link/sdks/dart/link/ios/Frameworks/libkalam_link_dart.a diff --git a/link/sdks/dart/ios/kalam_link.podspec b/link/sdks/dart/link/ios/kalam_link.podspec similarity index 100% rename from link/sdks/dart/ios/kalam_link.podspec rename to link/sdks/dart/link/ios/kalam_link.podspec diff --git a/link/sdks/dart/lib/kalam_link.dart b/link/sdks/dart/link/lib/kalam_link.dart similarity index 96% rename from link/sdks/dart/lib/kalam_link.dart rename to link/sdks/dart/link/lib/kalam_link.dart index 7c0f8192f..0e45900fd 100644 --- a/link/sdks/dart/lib/kalam_link.dart +++ b/link/sdks/dart/link/lib/kalam_link.dart @@ -31,5 +31,6 @@ export 'src/cell_value.dart'; export 'src/file_ref.dart'; export 'src/kalam_client.dart'; export 'src/logger.dart'; +export 'src/live_event_delivery.dart'; export 'src/models.dart'; export 'src/seq_id.dart'; diff --git a/link/sdks/dart/lib/src/auth.dart b/link/sdks/dart/link/lib/src/auth.dart similarity index 100% rename from link/sdks/dart/lib/src/auth.dart rename to link/sdks/dart/link/lib/src/auth.dart diff --git a/link/sdks/dart/lib/src/cell_value.dart b/link/sdks/dart/link/lib/src/cell_value.dart similarity index 100% rename from link/sdks/dart/lib/src/cell_value.dart rename to link/sdks/dart/link/lib/src/cell_value.dart diff --git a/link/sdks/dart/lib/src/file_ref.dart b/link/sdks/dart/link/lib/src/file_ref.dart similarity index 100% rename from link/sdks/dart/lib/src/file_ref.dart rename to link/sdks/dart/link/lib/src/file_ref.dart diff --git a/link/sdks/dart/lib/src/generated/.frb_codegen.stamp b/link/sdks/dart/link/lib/src/generated/.frb_codegen.stamp similarity index 100% rename from link/sdks/dart/lib/src/generated/.frb_codegen.stamp rename to link/sdks/dart/link/lib/src/generated/.frb_codegen.stamp diff --git a/link/sdks/dart/lib/src/generated/api.dart b/link/sdks/dart/link/lib/src/generated/api.dart similarity index 97% rename from link/sdks/dart/lib/src/generated/api.dart rename to link/sdks/dart/link/lib/src/generated/api.dart index 970466af5..f7f506d63 100644 --- a/link/sdks/dart/lib/src/generated/api.dart +++ b/link/sdks/dart/link/lib/src/generated/api.dart @@ -256,6 +256,13 @@ Future dartLiveEventsNext( {required DartLiveEventsSubscription subscription}) => RustLib.instance.api.crateApiDartLiveEventsNext(subscription: subscription); +/// Acknowledge progress after the consumer durably commits an event. +Future dartLiveEventsAck( + {required DartLiveEventsSubscription subscription, + required PlatformInt64 seqId}) => + RustLib.instance.api + .crateApiDartLiveEventsAck(subscription: subscription, seqId: seqId); + /// Close a subscription and release server-side resources. Future dartLiveEventsClose( {required DartLiveEventsSubscription subscription}) => diff --git a/link/sdks/dart/lib/src/generated/frb_generated.dart b/link/sdks/dart/link/lib/src/generated/frb_generated.dart similarity index 98% rename from link/sdks/dart/lib/src/generated/frb_generated.dart rename to link/sdks/dart/link/lib/src/generated/frb_generated.dart index 2bad86daa..37ce21ebb 100644 --- a/link/sdks/dart/lib/src/generated/frb_generated.dart +++ b/link/sdks/dart/link/lib/src/generated/frb_generated.dart @@ -69,12 +69,12 @@ class RustLib extends BaseEntrypoint { String get codegenVersion => '2.12.0'; @override - int get rustContentHash => -212852337; + int get rustContentHash => -119635461; static const kDefaultExternalLibraryLoaderConfig = ExternalLibraryLoaderConfig( stem: 'kalam_link_dart', - ioDirectory: '../../kalam-link-dart/target/release/', + ioDirectory: '../../../kalam-link-dart/target/release/', webPrefix: 'pkg/', wasmBindgenName: 'wasm_bindgen', ); @@ -143,6 +143,10 @@ abstract class RustLibApi extends BaseApi { Future crateApiDartLiveClose( {required DartLiveRowsSubscription subscription}); + Future crateApiDartLiveEventsAck( + {required DartLiveEventsSubscription subscription, + required PlatformInt64 seqId}); + Future crateApiDartLiveEventsClose( {required DartLiveEventsSubscription subscription}); @@ -672,6 +676,34 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { argNames: ["subscription"], ); + @override + Future crateApiDartLiveEventsAck( + {required DartLiveEventsSubscription subscription, + required PlatformInt64 seqId}) { + return handler.executeNormal(NormalTask( + callFfi: (port_) { + final serializer = SseSerializer(generalizedFrbRustBinding); + sse_encode_Auto_Ref_RustOpaque_flutter_rust_bridgefor_generatedRustAutoOpaqueInnerDartLiveEventsSubscription( + subscription, serializer); + sse_encode_i_64(seqId, serializer); + pdeCallFfi(generalizedFrbRustBinding, serializer, + funcId: 16, port: port_); + }, + codec: SseCodec( + decodeSuccessData: sse_decode_unit, + decodeErrorData: sse_decode_AnyhowException, + ), + constMeta: kCrateApiDartLiveEventsAckConstMeta, + argValues: [subscription, seqId], + apiImpl: this, + )); + } + + TaskConstMeta get kCrateApiDartLiveEventsAckConstMeta => const TaskConstMeta( + debugName: "dart_live_events_ack", + argNames: ["subscription", "seqId"], + ); + @override Future crateApiDartLiveEventsClose( {required DartLiveEventsSubscription subscription}) { @@ -681,7 +713,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { sse_encode_Auto_Ref_RustOpaque_flutter_rust_bridgefor_generatedRustAutoOpaqueInnerDartLiveEventsSubscription( subscription, serializer); pdeCallFfi(generalizedFrbRustBinding, serializer, - funcId: 16, port: port_); + funcId: 17, port: port_); }, codec: SseCodec( decodeSuccessData: sse_decode_unit, @@ -707,7 +739,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { final serializer = SseSerializer(generalizedFrbRustBinding); sse_encode_Auto_Ref_RustOpaque_flutter_rust_bridgefor_generatedRustAutoOpaqueInnerDartLiveEventsSubscription( subscription, serializer); - return pdeCallFfi(generalizedFrbRustBinding, serializer, funcId: 17)!; + return pdeCallFfi(generalizedFrbRustBinding, serializer, funcId: 18)!; }, codec: SseCodec( decodeSuccessData: sse_decode_String, @@ -733,7 +765,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { sse_encode_Auto_Ref_RustOpaque_flutter_rust_bridgefor_generatedRustAutoOpaqueInnerDartLiveEventsSubscription( subscription, serializer); pdeCallFfi(generalizedFrbRustBinding, serializer, - funcId: 18, port: port_); + funcId: 19, port: port_); }, codec: SseCodec( decodeSuccessData: sse_decode_opt_box_autoadd_dart_change_event, @@ -763,7 +795,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { sse_encode_String(sql, serializer); sse_encode_opt_box_autoadd_dart_subscription_config(config, serializer); pdeCallFfi(generalizedFrbRustBinding, serializer, - funcId: 19, port: port_); + funcId: 20, port: port_); }, codec: SseCodec( decodeSuccessData: @@ -789,7 +821,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { final serializer = SseSerializer(generalizedFrbRustBinding); sse_encode_Auto_Ref_RustOpaque_flutter_rust_bridgefor_generatedRustAutoOpaqueInnerDartLiveRowsSubscription( subscription, serializer); - return pdeCallFfi(generalizedFrbRustBinding, serializer, funcId: 20)!; + return pdeCallFfi(generalizedFrbRustBinding, serializer, funcId: 21)!; }, codec: SseCodec( decodeSuccessData: sse_decode_String, @@ -815,7 +847,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { sse_encode_Auto_Ref_RustOpaque_flutter_rust_bridgefor_generatedRustAutoOpaqueInnerDartLiveRowsSubscription( subscription, serializer); pdeCallFfi(generalizedFrbRustBinding, serializer, - funcId: 21, port: port_); + funcId: 22, port: port_); }, codec: SseCodec( decodeSuccessData: sse_decode_opt_box_autoadd_dart_live_rows_event, @@ -848,7 +880,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { sse_encode_opt_box_autoadd_dart_live_rows_config( liveConfig, serializer); pdeCallFfi(generalizedFrbRustBinding, serializer, - funcId: 22, port: port_); + funcId: 23, port: port_); }, codec: SseCodec( decodeSuccessData: @@ -879,7 +911,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { sse_encode_String(user, serializer); sse_encode_String(password, serializer); pdeCallFfi(generalizedFrbRustBinding, serializer, - funcId: 23, port: port_); + funcId: 24, port: port_); }, codec: SseCodec( decodeSuccessData: sse_decode_dart_login_response, @@ -905,7 +937,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { sse_encode_Auto_Ref_RustOpaque_flutter_rust_bridgefor_generatedRustAutoOpaqueInnerDartKalamClient( client, serializer); pdeCallFfi(generalizedFrbRustBinding, serializer, - funcId: 24, port: port_); + funcId: 25, port: port_); }, codec: SseCodec( decodeSuccessData: sse_decode_opt_box_autoadd_dart_connection_event, @@ -929,7 +961,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { callFfi: () { final serializer = SseSerializer(generalizedFrbRustBinding); sse_encode_String(fileRefJson, serializer); - return pdeCallFfi(generalizedFrbRustBinding, serializer, funcId: 25)!; + return pdeCallFfi(generalizedFrbRustBinding, serializer, funcId: 26)!; }, codec: SseCodec( decodeSuccessData: sse_decode_dart_file_ref, @@ -956,7 +988,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { client, serializer); sse_encode_String(refreshToken, serializer); pdeCallFfi(generalizedFrbRustBinding, serializer, - funcId: 26, port: port_); + funcId: 27, port: port_); }, codec: SseCodec( decodeSuccessData: sse_decode_dart_login_response, @@ -980,7 +1012,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { final serializer = SseSerializer(generalizedFrbRustBinding); sse_encode_Auto_Ref_RustOpaque_flutter_rust_bridgefor_generatedRustAutoOpaqueInnerDartKalamClient( client, serializer); - return pdeCallFfi(generalizedFrbRustBinding, serializer, funcId: 27)!; + return pdeCallFfi(generalizedFrbRustBinding, serializer, funcId: 28)!; }, codec: SseCodec( decodeSuccessData: sse_decode_unit, @@ -1003,7 +1035,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { callFfi: () { final serializer = SseSerializer(generalizedFrbRustBinding); sse_encode_String(fileRefJson, serializer); - return pdeCallFfi(generalizedFrbRustBinding, serializer, funcId: 28)!; + return pdeCallFfi(generalizedFrbRustBinding, serializer, funcId: 29)!; }, codec: SseCodec( decodeSuccessData: sse_decode_opt_box_autoadd_dart_file_ref, @@ -1031,7 +1063,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { client, serializer); sse_encode_box_autoadd_dart_auth_provider(auth, serializer); pdeCallFfi(generalizedFrbRustBinding, serializer, - funcId: 29, port: port_); + funcId: 30, port: port_); }, codec: SseCodec( decodeSuccessData: sse_decode_unit, @@ -1555,14 +1587,15 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { DartSubscriptionConfig dco_decode_dart_subscription_config(dynamic raw) { // Codec=Dco (DartCObject based), see doc to use other codecs final arr = raw as List; - if (arr.length != 5) - throw Exception('unexpected arr length: expect 5 but see ${arr.length}'); + if (arr.length != 6) + throw Exception('unexpected arr length: expect 6 but see ${arr.length}'); return DartSubscriptionConfig( sql: dco_decode_String(arr[0]), id: dco_decode_opt_String(arr[1]), batchSize: dco_decode_opt_box_autoadd_i_32(arr[2]), lastRows: dco_decode_opt_box_autoadd_i_32(arr[3]), from: dco_decode_opt_box_autoadd_i_64(arr[4]), + explicitAck: dco_decode_bool(arr[5]), ); } @@ -2287,12 +2320,14 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { var var_batchSize = sse_decode_opt_box_autoadd_i_32(deserializer); var var_lastRows = sse_decode_opt_box_autoadd_i_32(deserializer); var var_from = sse_decode_opt_box_autoadd_i_64(deserializer); + var var_explicitAck = sse_decode_bool(deserializer); return DartSubscriptionConfig( sql: var_sql, id: var_id, batchSize: var_batchSize, lastRows: var_lastRows, - from: var_from); + from: var_from, + explicitAck: var_explicitAck); } @protected @@ -3076,6 +3111,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { sse_encode_opt_box_autoadd_i_32(self.batchSize, serializer); sse_encode_opt_box_autoadd_i_32(self.lastRows, serializer); sse_encode_opt_box_autoadd_i_64(self.from, serializer); + sse_encode_bool(self.explicitAck, serializer); } @protected diff --git a/link/sdks/dart/lib/src/generated/frb_generated.io.dart b/link/sdks/dart/link/lib/src/generated/frb_generated.io.dart similarity index 100% rename from link/sdks/dart/lib/src/generated/frb_generated.io.dart rename to link/sdks/dart/link/lib/src/generated/frb_generated.io.dart diff --git a/link/sdks/dart/lib/src/generated/frb_generated.web.dart b/link/sdks/dart/link/lib/src/generated/frb_generated.web.dart similarity index 100% rename from link/sdks/dart/lib/src/generated/frb_generated.web.dart rename to link/sdks/dart/link/lib/src/generated/frb_generated.web.dart diff --git a/link/sdks/dart/lib/src/generated/models.dart b/link/sdks/dart/link/lib/src/generated/models.dart similarity index 98% rename from link/sdks/dart/lib/src/generated/models.dart rename to link/sdks/dart/link/lib/src/generated/models.dart index dd94e8ab8..f86477a73 100644 --- a/link/sdks/dart/lib/src/generated/models.dart +++ b/link/sdks/dart/link/lib/src/generated/models.dart @@ -526,12 +526,16 @@ class DartSubscriptionConfig { /// When set, the server only sends changes after this seq_id. final PlatformInt64? from; + /// Require explicit consumer acknowledgement before reconnect progress advances. + final bool explicitAck; + const DartSubscriptionConfig({ required this.sql, this.id, this.batchSize, this.lastRows, this.from, + required this.explicitAck, }); @override @@ -540,7 +544,8 @@ class DartSubscriptionConfig { id.hashCode ^ batchSize.hashCode ^ lastRows.hashCode ^ - from.hashCode; + from.hashCode ^ + explicitAck.hashCode; @override bool operator ==(Object other) => @@ -551,7 +556,8 @@ class DartSubscriptionConfig { id == other.id && batchSize == other.batchSize && lastRows == other.lastRows && - from == other.from; + from == other.from && + explicitAck == other.explicitAck; } /// Read-only snapshot of an active subscription's metadata. diff --git a/link/sdks/dart/lib/src/generated/models.freezed.dart b/link/sdks/dart/link/lib/src/generated/models.freezed.dart similarity index 100% rename from link/sdks/dart/lib/src/generated/models.freezed.dart rename to link/sdks/dart/link/lib/src/generated/models.freezed.dart diff --git a/link/sdks/dart/lib/src/kalam_client.dart b/link/sdks/dart/link/lib/src/kalam_client.dart similarity index 94% rename from link/sdks/dart/lib/src/kalam_client.dart rename to link/sdks/dart/link/lib/src/kalam_client.dart index aabad43fa..fc5d00141 100644 --- a/link/sdks/dart/lib/src/kalam_client.dart +++ b/link/sdks/dart/link/lib/src/kalam_client.dart @@ -5,6 +5,7 @@ import 'auth.dart'; import 'cell_value.dart'; import 'file_ref.dart'; import 'logger.dart'; +import 'live_event_delivery.dart'; import 'models.dart'; import 'seq_id.dart'; import 'generated/api.dart' as bridge; @@ -325,11 +326,13 @@ class KalamClient { int? batchSize, int? lastRows, SeqId? from, + bool explicitAck = false, }) { if (batchSize == null && lastRows == null && subscriptionId.isEmpty && - from == null) { + from == null && + !explicitAck) { return null; } @@ -339,6 +342,7 @@ class KalamClient { lastRows: lastRows, id: subscriptionId, from: from?.toInt(), + explicitAck: explicitAck, ); } @@ -575,7 +579,7 @@ class KalamClient { client: _handle, subscriptionId: logicalSubscriptionId, ), - decode: (event) => _fromBridgeChangeEvent( + decode: (_, event) => _fromBridgeChangeEvent( event, onCheckpoint: onCheckpoint, onError: onError, @@ -584,6 +588,60 @@ class KalamClient { ); } + /// Open live changes whose reconnect cursor advances only after the caller + /// acknowledges an event. Use this for durable local synchronization. + Stream liveEventsWithAck( + String sql, { + int? batchSize, + int? lastRows, + SeqId? from, + String? subscriptionId, + void Function(SubscriptionError error)? onError, + }) { + final logicalSubscriptionId = + _ensureSubscriptionId(subscriptionId, prefix: 'events-ack'); + return runBridgeSubscriptionStream( + description: 'acknowledged live events for: $sql', + prepare: _prepareSubscriptionConnect, + open: () => bridge.dartLiveEventsSubscribe( + client: _handle, + sql: sql, + config: _buildSubscriptionConfig( + sql: sql, + subscriptionId: logicalSubscriptionId, + batchSize: batchSize, + lastRows: lastRows, + from: from, + explicitAck: true, + ), + ), + next: (sub) => bridge.dartLiveEventsNext(subscription: sub), + close: (sub) => bridge.dartLiveEventsClose(subscription: sub), + cancel: () => bridge.dartCancelSubscription( + client: _handle, + subscriptionId: logicalSubscriptionId, + ), + decode: (subscription, event) { + final checkpoint = _checkpointFromBridgeChangeEvent(event); + return LiveEventDelivery( + event: _fromBridgeChangeEvent(event, onError: onError), + checkpoint: checkpoint, + acknowledge: () => checkpoint == null + ? Future.value() + : bridge.dartLiveEventsAck( + subscription: subscription, + seqId: checkpoint.lastSeqId.toInt(), + ), + ); + }, + waitBeforeNext: (delivery) => delivery.checkpoint == null + ? Future.value() + : delivery.whenAcknowledged, + isDisposed: () => _isDisposed, + ); + } + /// Open a SQL query and receive the current materialized row set. /// /// The row materialization happens inside the shared Rust client layer, so Dart @@ -639,7 +697,7 @@ class KalamClient { client: _handle, subscriptionId: logicalSubscriptionId, ), - decode: (event) => _fromBridgeLiveRowsEvent( + decode: (_, event) => _fromBridgeLiveRowsEvent( event, mapRow: mapRow, onCheckpoint: onCheckpoint, diff --git a/link/sdks/dart/link/lib/src/live_event_delivery.dart b/link/sdks/dart/link/lib/src/live_event_delivery.dart new file mode 100644 index 000000000..ef9ba854c --- /dev/null +++ b/link/sdks/dart/link/lib/src/live_event_delivery.dart @@ -0,0 +1,38 @@ +import 'dart:async'; + +import 'models.dart'; + +/// One live event whose reconnect progress advances only after acknowledgement. +final class LiveEventDelivery { + LiveEventDelivery({ + required this.event, + required this.checkpoint, + required Future Function() acknowledge, + }) : _acknowledge = acknowledge; + + final ChangeEvent event; + final LiveCheckpoint? checkpoint; + final Future Function() _acknowledge; + final Completer _progress = Completer(); + Future? _acknowledgement; + + /// Acknowledge this event after its durable local transaction commits. + /// Repeated calls share the same acknowledgement. + Future acknowledge() { + final existing = _acknowledgement; + if (existing != null) return existing; + final acknowledgement = Future.sync(_acknowledge); + _acknowledgement = acknowledgement; + acknowledgement.then( + (_) => _progress.complete(), + onError: (Object error, StackTrace stackTrace) { + _progress.completeError(error, stackTrace); + }, + ); + return acknowledgement; + } + + /// Completes when acknowledgement succeeds and fails when it fails. + Future get whenAcknowledged => + checkpoint == null ? Future.value() : _progress.future; +} diff --git a/link/sdks/dart/lib/src/logger.dart b/link/sdks/dart/link/lib/src/logger.dart similarity index 100% rename from link/sdks/dart/lib/src/logger.dart rename to link/sdks/dart/link/lib/src/logger.dart diff --git a/link/sdks/dart/lib/src/models.dart b/link/sdks/dart/link/lib/src/models.dart similarity index 100% rename from link/sdks/dart/lib/src/models.dart rename to link/sdks/dart/link/lib/src/models.dart diff --git a/link/sdks/dart/lib/src/seq_id.dart b/link/sdks/dart/link/lib/src/seq_id.dart similarity index 100% rename from link/sdks/dart/lib/src/seq_id.dart rename to link/sdks/dart/link/lib/src/seq_id.dart diff --git a/link/sdks/dart/lib/src/subscription_stream.dart b/link/sdks/dart/link/lib/src/subscription_stream.dart similarity index 82% rename from link/sdks/dart/lib/src/subscription_stream.dart rename to link/sdks/dart/link/lib/src/subscription_stream.dart index 220465f60..cd27906da 100644 --- a/link/sdks/dart/lib/src/subscription_stream.dart +++ b/link/sdks/dart/link/lib/src/subscription_stream.dart @@ -10,9 +10,12 @@ typedef BridgeSubscriptionClose = Future Function( THandle handle, ); typedef BridgeSubscriptionCancel = Future Function(); -typedef BridgeSubscriptionDecode = TEvent Function( +typedef BridgeSubscriptionDecode = TEvent + Function( + THandle handle, TBridgeEvent event, ); +typedef BridgeSubscriptionWait = Future Function(TEvent event); /// Runs a Rust-backed pull subscription behind a Dart [Stream]. /// @@ -25,7 +28,8 @@ Stream runBridgeSubscriptionStream({ required BridgeSubscriptionNext next, required BridgeSubscriptionClose close, required BridgeSubscriptionCancel cancel, - required BridgeSubscriptionDecode decode, + required BridgeSubscriptionDecode decode, + BridgeSubscriptionWait? waitBeforeNext, required bool Function() isDisposed, Duration cancelWaitTimeout = const Duration(seconds: 2), }) { @@ -33,6 +37,7 @@ Stream runBridgeSubscriptionStream({ var closed = false; var closingFromPump = false; Future? pump; + final cancelled = Completer(); Future runLoop() async { THandle? handle; @@ -46,7 +51,11 @@ Stream runBridgeSubscriptionStream({ if (event == null || closed) { break; } - controller.add(decode(event)); + final decoded = decode(openedHandle, event); + controller.add(decoded); + if (waitBeforeNext != null) { + await Future.any([waitBeforeNext(decoded), cancelled.future]); + } } } catch (error, stackTrace) { if (!closed && !isDisposed()) { @@ -80,6 +89,7 @@ Stream runBridgeSubscriptionStream({ }, onCancel: () async { closed = true; + if (!cancelled.isCompleted) cancelled.complete(); if (closingFromPump) { return; } diff --git a/link/sdks/dart/linux/CMakeLists.txt b/link/sdks/dart/link/linux/CMakeLists.txt similarity index 100% rename from link/sdks/dart/linux/CMakeLists.txt rename to link/sdks/dart/link/linux/CMakeLists.txt diff --git a/link/sdks/dart/linux/lib/.gitignore b/link/sdks/dart/link/linux/lib/.gitignore similarity index 100% rename from link/sdks/dart/linux/lib/.gitignore rename to link/sdks/dart/link/linux/lib/.gitignore diff --git a/link/sdks/dart/macos/.gitignore b/link/sdks/dart/link/macos/.gitignore similarity index 100% rename from link/sdks/dart/macos/.gitignore rename to link/sdks/dart/link/macos/.gitignore diff --git a/link/sdks/dart/macos/Classes/KalamLinkPlugin.swift b/link/sdks/dart/link/macos/Classes/KalamLinkPlugin.swift similarity index 100% rename from link/sdks/dart/macos/Classes/KalamLinkPlugin.swift rename to link/sdks/dart/link/macos/Classes/KalamLinkPlugin.swift diff --git a/link/sdks/dart/macos/Libs/.gitignore b/link/sdks/dart/link/macos/Libs/.gitignore similarity index 100% rename from link/sdks/dart/macos/Libs/.gitignore rename to link/sdks/dart/link/macos/Libs/.gitignore diff --git a/link/sdks/dart/macos/Libs/libkalam_link_dart.dylib b/link/sdks/dart/link/macos/Libs/libkalam_link_dart.dylib similarity index 100% rename from link/sdks/dart/macos/Libs/libkalam_link_dart.dylib rename to link/sdks/dart/link/macos/Libs/libkalam_link_dart.dylib diff --git a/link/sdks/dart/macos/kalam_link.podspec b/link/sdks/dart/link/macos/kalam_link.podspec similarity index 100% rename from link/sdks/dart/macos/kalam_link.podspec rename to link/sdks/dart/link/macos/kalam_link.podspec diff --git a/link/sdks/dart/publish.sh b/link/sdks/dart/link/publish.sh similarity index 100% rename from link/sdks/dart/publish.sh rename to link/sdks/dart/link/publish.sh diff --git a/link/sdks/dart/pubspec.lock b/link/sdks/dart/link/pubspec.lock similarity index 100% rename from link/sdks/dart/pubspec.lock rename to link/sdks/dart/link/pubspec.lock diff --git a/link/sdks/dart/pubspec.yaml b/link/sdks/dart/link/pubspec.yaml similarity index 100% rename from link/sdks/dart/pubspec.yaml rename to link/sdks/dart/link/pubspec.yaml diff --git a/link/sdks/dart/test.sh b/link/sdks/dart/link/test.sh similarity index 99% rename from link/sdks/dart/test.sh rename to link/sdks/dart/link/test.sh index 7ac23d730..873fd09dc 100755 --- a/link/sdks/dart/test.sh +++ b/link/sdks/dart/link/test.sh @@ -209,7 +209,7 @@ ensure_test_auth_ready() { echo "📦 Ensuring dependencies are installed..." flutter pub get -BRIDGE_DIR="$SCRIPT_DIR/../../kalam-link-dart" +BRIDGE_DIR="$SCRIPT_DIR/../../../kalam-link-dart" echo "🦀 Building host native library used by flutter test..." ( cd "$BRIDGE_DIR" diff --git a/link/sdks/dart/test/auth_provider_retry_test.dart b/link/sdks/dart/link/test/auth_provider_retry_test.dart similarity index 100% rename from link/sdks/dart/test/auth_provider_retry_test.dart rename to link/sdks/dart/link/test/auth_provider_retry_test.dart diff --git a/link/sdks/dart/test/e2e/auth/auth_test.dart b/link/sdks/dart/link/test/e2e/auth/auth_test.dart similarity index 100% rename from link/sdks/dart/test/e2e/auth/auth_test.dart rename to link/sdks/dart/link/test/e2e/auth/auth_test.dart diff --git a/link/sdks/dart/test/e2e/ddl/ddl_test.dart b/link/sdks/dart/link/test/e2e/ddl/ddl_test.dart similarity index 100% rename from link/sdks/dart/test/e2e/ddl/ddl_test.dart rename to link/sdks/dart/link/test/e2e/ddl/ddl_test.dart diff --git a/link/sdks/dart/test/e2e/examples/examples_test.dart b/link/sdks/dart/link/test/e2e/examples/examples_test.dart similarity index 100% rename from link/sdks/dart/test/e2e/examples/examples_test.dart rename to link/sdks/dart/link/test/e2e/examples/examples_test.dart diff --git a/link/sdks/dart/test/e2e/helpers.dart b/link/sdks/dart/link/test/e2e/helpers.dart similarity index 99% rename from link/sdks/dart/test/e2e/helpers.dart rename to link/sdks/dart/link/test/e2e/helpers.dart index f45b1a512..5f8d6d3b4 100644 --- a/link/sdks/dart/test/e2e/helpers.dart +++ b/link/sdks/dart/link/test/e2e/helpers.dart @@ -313,7 +313,7 @@ Future _ensureNativeBridgeReady() async { if (!shouldBuild) return; final sdkDir = Directory.current.path; - final bridgeDir = Directory('$sdkDir/../../kalam-link-dart').absolute.path; + final bridgeDir = Directory('$sdkDir/../../../kalam-link-dart').absolute.path; final workspaceDir = Directory('$bridgeDir/../..').absolute.path; final libPath = _bridgeLibraryPath(bridgeDir); final workspaceLibPath = _bridgeLibraryPath(workspaceDir); diff --git a/link/sdks/dart/test/e2e/keepalive/keepalive_test.dart b/link/sdks/dart/link/test/e2e/keepalive/keepalive_test.dart similarity index 100% rename from link/sdks/dart/test/e2e/keepalive/keepalive_test.dart rename to link/sdks/dart/link/test/e2e/keepalive/keepalive_test.dart diff --git a/link/sdks/dart/test/e2e/keepalive/subscription_cleanup_test.dart b/link/sdks/dart/link/test/e2e/keepalive/subscription_cleanup_test.dart similarity index 100% rename from link/sdks/dart/test/e2e/keepalive/subscription_cleanup_test.dart rename to link/sdks/dart/link/test/e2e/keepalive/subscription_cleanup_test.dart diff --git a/link/sdks/dart/test/e2e/lifecycle/lifecycle_test.dart b/link/sdks/dart/link/test/e2e/lifecycle/lifecycle_test.dart similarity index 100% rename from link/sdks/dart/test/e2e/lifecycle/lifecycle_test.dart rename to link/sdks/dart/link/test/e2e/lifecycle/lifecycle_test.dart diff --git a/link/sdks/dart/test/e2e/query/query_test.dart b/link/sdks/dart/link/test/e2e/query/query_test.dart similarity index 100% rename from link/sdks/dart/test/e2e/query/query_test.dart rename to link/sdks/dart/link/test/e2e/query/query_test.dart diff --git a/link/sdks/dart/test/e2e/reconnect/app_lifecycle_test.dart b/link/sdks/dart/link/test/e2e/reconnect/app_lifecycle_test.dart similarity index 100% rename from link/sdks/dart/test/e2e/reconnect/app_lifecycle_test.dart rename to link/sdks/dart/link/test/e2e/reconnect/app_lifecycle_test.dart diff --git a/link/sdks/dart/link/test/e2e/reconnect/explicit_ack_test.dart b/link/sdks/dart/link/test/e2e/reconnect/explicit_ack_test.dart new file mode 100644 index 000000000..753c36499 --- /dev/null +++ b/link/sdks/dart/link/test/e2e/reconnect/explicit_ack_test.dart @@ -0,0 +1,106 @@ +import 'dart:async'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:kalam_link/kalam_link.dart'; + +import '../helpers.dart'; + +void main() { + test( + 'explicit acknowledgement advances resume only after durable consumer work', + () async { + await ensureSdkReady(); + final client = await connectJwtClient(); + final table = 'default.${uniqueName('dart_explicit_ack')}'; + final subscriptionId = uniqueName('explicit-ack'); + addTearDown(() async { + await dropTable(client, table); + await _safeDispose(client); + }); + await client.query( + 'CREATE TABLE $table (id BIGINT PRIMARY KEY, value TEXT NOT NULL)', + ); + + final events = client.liveEventsWithAck( + 'SELECT * FROM $table', + subscriptionId: subscriptionId, + ); + final ready = Completer(); + final inserted = Completer(); + var activeSubscriptionId = subscriptionId; + final subscription = events.listen((delivery) { + if (delivery.event case AckEvent(:final subscriptionId)) { + activeSubscriptionId = subscriptionId; + if (!ready.isCompleted) ready.complete(); + } + if (delivery.event is InsertEvent && !inserted.isCompleted) { + inserted.complete(delivery); + } + }); + addTearDown(() => _safeCancel(subscription)); + await ready.future.timeout(const Duration(seconds: 10)); + await client.query( + "INSERT INTO $table (id, value) VALUES (1, 'one')", + ); + + final delivery = + await inserted.future.timeout(const Duration(seconds: 10)); + final seq = + (delivery.event as InsertEvent).rows.single['_seq']!.asSeqId()!; + expect( + (await _subscription(client, activeSubscriptionId)).lastSeqId, + isNot(seq), + reason: 'delivery must not move the reconnect cursor', + ); + + await delivery.acknowledge(); + await _waitForSeq(client, activeSubscriptionId, seq); + }, + skip: skipIfNoIntegration, + timeout: const Timeout(Duration(seconds: 30)), + ); +} + +Future _subscription( + KalamClient client, + String subscriptionId, +) async { + final subscriptions = await client.getSubscriptions(); + return subscriptions.singleWhere( + (subscription) => subscription.id == subscriptionId, + orElse: () => throw StateError( + 'Subscription $subscriptionId is not in ' + '${subscriptions.map((subscription) => subscription.id).toList()}', + ), + ); +} + +Future _waitForSeq( + KalamClient client, + String subscriptionId, + SeqId seq, +) async { + final deadline = DateTime.now().add(const Duration(seconds: 5)); + while (DateTime.now().isBefore(deadline)) { + if ((await _subscription(client, subscriptionId)).lastSeqId == seq) return; + await Future.delayed(const Duration(milliseconds: 20)); + } + throw TimeoutException( + 'Subscription $subscriptionId did not acknowledge $seq.'); +} + +Future _safeCancel(StreamSubscription subscription) async { + try { + await subscription.cancel().timeout(const Duration(seconds: 3)); + } on TimeoutException { + // Client disposal performs the final native cleanup. + } +} + +Future _safeDispose(KalamClient client) async { + try { + await client.dispose().timeout(const Duration(seconds: 3)); + } on TimeoutException { + // The process will release any remaining native resources. + } +} diff --git a/link/sdks/dart/test/e2e/reconnect/reconnect_test.dart b/link/sdks/dart/link/test/e2e/reconnect/reconnect_test.dart similarity index 100% rename from link/sdks/dart/test/e2e/reconnect/reconnect_test.dart rename to link/sdks/dart/link/test/e2e/reconnect/reconnect_test.dart diff --git a/link/sdks/dart/test/e2e/reconnect/resume_test.dart b/link/sdks/dart/link/test/e2e/reconnect/resume_test.dart similarity index 100% rename from link/sdks/dart/test/e2e/reconnect/resume_test.dart rename to link/sdks/dart/link/test/e2e/reconnect/resume_test.dart diff --git a/link/sdks/dart/test/e2e/subscription/subscription_options_test.dart b/link/sdks/dart/link/test/e2e/subscription/subscription_options_test.dart similarity index 100% rename from link/sdks/dart/test/e2e/subscription/subscription_options_test.dart rename to link/sdks/dart/link/test/e2e/subscription/subscription_options_test.dart diff --git a/link/sdks/dart/test/e2e/subscription/subscription_test.dart b/link/sdks/dart/link/test/e2e/subscription/subscription_test.dart similarity index 100% rename from link/sdks/dart/test/e2e/subscription/subscription_test.dart rename to link/sdks/dart/link/test/e2e/subscription/subscription_test.dart diff --git a/link/sdks/dart/test/file_ref_test.dart b/link/sdks/dart/link/test/file_ref_test.dart similarity index 100% rename from link/sdks/dart/test/file_ref_test.dart rename to link/sdks/dart/link/test/file_ref_test.dart diff --git a/link/sdks/dart/test/live_server_test.dart b/link/sdks/dart/link/test/live_server_test.dart similarity index 99% rename from link/sdks/dart/test/live_server_test.dart rename to link/sdks/dart/link/test/live_server_test.dart index 8461c41ad..5657df25b 100644 --- a/link/sdks/dart/test/live_server_test.dart +++ b/link/sdks/dart/link/test/live_server_test.dart @@ -82,7 +82,7 @@ Future _ensureNativeBridgeReady() async { } final sdkDir = Directory.current.path; - final bridgeDir = Directory('$sdkDir/../../kalam-link-dart').absolute.path; + final bridgeDir = Directory('$sdkDir/../../../kalam-link-dart').absolute.path; final workspaceDir = Directory('$bridgeDir/../..').absolute.path; final libPath = _bridgeLibraryPath(bridgeDir); final workspaceLibPath = _bridgeLibraryPath(workspaceDir); diff --git a/link/sdks/dart/test/main_thread_blocking_test.dart b/link/sdks/dart/link/test/main_thread_blocking_test.dart similarity index 100% rename from link/sdks/dart/test/main_thread_blocking_test.dart rename to link/sdks/dart/link/test/main_thread_blocking_test.dart diff --git a/link/sdks/dart/test/models_test.dart b/link/sdks/dart/link/test/models_test.dart similarity index 100% rename from link/sdks/dart/test/models_test.dart rename to link/sdks/dart/link/test/models_test.dart diff --git a/link/sdks/dart/test/plugin_packaging_test.dart b/link/sdks/dart/link/test/plugin_packaging_test.dart similarity index 100% rename from link/sdks/dart/test/plugin_packaging_test.dart rename to link/sdks/dart/link/test/plugin_packaging_test.dart diff --git a/link/sdks/dart/link/test/schema_generation_test.dart b/link/sdks/dart/link/test/schema_generation_test.dart new file mode 100644 index 000000000..c60ec119c --- /dev/null +++ b/link/sdks/dart/link/test/schema_generation_test.dart @@ -0,0 +1,55 @@ +import 'package:flutter_test/flutter_test.dart'; + +void main() { + group('kalam CLI workflow Dart targets', () { + const sampleKalamCliDart = ''' +// Generated by kalam schema gen. Do not edit. +// Row codecs and KalamTableSpec values for kalam_sync. +// Action queues stay with kalam_sync_generator / build_runner. +import 'package:kalam_sync/kalam_sync.dart'; + +final class Users { + const Users({ + required this.id, + required this.email, + }); + + final int id; + final String email; + + String get kalamRowKey => id.toString(); + + Map toJson() => { + 'id': id, + 'email': email, + }; + + factory Users.fromJson(Map json) => Users( + id: _asInt(json['id']), + email: _asString(json['email']), + ); +} + +abstract final class KalamTables { + KalamTables._(); + + static final users = KalamTableSpec( + tableId: 'users', + keyColumn: 'id', + mode: KalamSyncMode.bidirectional, + keyOf: (row) => row.kalamRowKey, + encode: (row) => row.toJson(), + decode: Users.fromJson, + ); +} +'''; + + test('matches current schema gen contract', () { + expect(sampleKalamCliDart, contains('Generated by kalam schema gen')); + expect(sampleKalamCliDart, contains("import 'package:kalam_sync/kalam_sync.dart';")); + expect(sampleKalamCliDart, contains('KalamTableSpec')); + expect(sampleKalamCliDart, contains('mode: KalamSyncMode.bidirectional')); + expect(sampleKalamCliDart.toLowerCase(), isNot(contains('placeholder'))); + }); + }); +} diff --git a/link/sdks/dart/test/subscription_stream_test.dart b/link/sdks/dart/link/test/subscription_stream_test.dart similarity index 63% rename from link/sdks/dart/test/subscription_stream_test.dart rename to link/sdks/dart/link/test/subscription_stream_test.dart index 106a4e533..24a53245a 100644 --- a/link/sdks/dart/test/subscription_stream_test.dart +++ b/link/sdks/dart/link/test/subscription_stream_test.dart @@ -27,7 +27,7 @@ void main() { closeCalls += 1; }, cancel: () async {}, - decode: (value) => value.toUpperCase(), + decode: (_, value) => value.toUpperCase(), isDisposed: () => false, ); @@ -64,7 +64,7 @@ void main() { releaseNext.complete(null); } }, - decode: (value) => value, + decode: (_, value) => value, isDisposed: () => false, ); @@ -76,5 +76,38 @@ void main() { expect(cancelCalls, 1); expect(closeCalls, 1); }); + + test('waits for consumer progress before pulling the next event', () async { + final source = Queue.from(['alpha', 'beta']); + final firstReceived = Completer(); + final releaseProgress = Completer(); + final received = []; + final stream = runBridgeSubscriptionStream( + description: 'backpressured-stream', + prepare: () async {}, + open: () async => Object(), + next: (_) async => source.isEmpty ? null : source.removeFirst(), + close: (_) async {}, + cancel: () async {}, + decode: (_, value) => value, + waitBeforeNext: (value) => + value == 'alpha' ? releaseProgress.future : Future.value(), + isDisposed: () => false, + ); + + final completed = stream.listen((value) { + received.add(value); + if (value == 'alpha') firstReceived.complete(); + }).asFuture(); + await firstReceived.future; + await Future.delayed(Duration.zero); + + expect(received, ['alpha']); + expect(source, ['beta']); + + releaseProgress.complete(); + await completed; + expect(received, ['alpha', 'beta']); + }); }); } diff --git a/link/sdks/dart/web/.gitignore b/link/sdks/dart/link/web/.gitignore similarity index 100% rename from link/sdks/dart/web/.gitignore rename to link/sdks/dart/link/web/.gitignore diff --git a/link/sdks/dart/web/pkg/.gitignore b/link/sdks/dart/link/web/pkg/.gitignore similarity index 100% rename from link/sdks/dart/web/pkg/.gitignore rename to link/sdks/dart/link/web/pkg/.gitignore diff --git a/link/sdks/dart/web/pkg/README.md b/link/sdks/dart/link/web/pkg/README.md similarity index 100% rename from link/sdks/dart/web/pkg/README.md rename to link/sdks/dart/link/web/pkg/README.md diff --git a/link/sdks/dart/web/pkg/kalam_link_dart.d.ts b/link/sdks/dart/link/web/pkg/kalam_link_dart.d.ts similarity index 100% rename from link/sdks/dart/web/pkg/kalam_link_dart.d.ts rename to link/sdks/dart/link/web/pkg/kalam_link_dart.d.ts diff --git a/link/sdks/dart/web/pkg/kalam_link_dart.js b/link/sdks/dart/link/web/pkg/kalam_link_dart.js similarity index 100% rename from link/sdks/dart/web/pkg/kalam_link_dart.js rename to link/sdks/dart/link/web/pkg/kalam_link_dart.js diff --git a/link/sdks/dart/web/pkg/kalam_link_dart_bg.wasm b/link/sdks/dart/link/web/pkg/kalam_link_dart_bg.wasm similarity index 100% rename from link/sdks/dart/web/pkg/kalam_link_dart_bg.wasm rename to link/sdks/dart/link/web/pkg/kalam_link_dart_bg.wasm diff --git a/link/sdks/dart/web/pkg/kalam_link_dart_bg.wasm.d.ts b/link/sdks/dart/link/web/pkg/kalam_link_dart_bg.wasm.d.ts similarity index 100% rename from link/sdks/dart/web/pkg/kalam_link_dart_bg.wasm.d.ts rename to link/sdks/dart/link/web/pkg/kalam_link_dart_bg.wasm.d.ts diff --git a/link/sdks/dart/web/pkg/package.json b/link/sdks/dart/link/web/pkg/package.json similarity index 100% rename from link/sdks/dart/web/pkg/package.json rename to link/sdks/dart/link/web/pkg/package.json diff --git a/link/sdks/dart/windows/CMakeLists.txt b/link/sdks/dart/link/windows/CMakeLists.txt similarity index 100% rename from link/sdks/dart/windows/CMakeLists.txt rename to link/sdks/dart/link/windows/CMakeLists.txt diff --git a/link/sdks/dart/windows/lib/.gitignore b/link/sdks/dart/link/windows/lib/.gitignore similarity index 100% rename from link/sdks/dart/windows/lib/.gitignore rename to link/sdks/dart/link/windows/lib/.gitignore diff --git a/link/sdks/dart/sync/.gitignore b/link/sdks/dart/sync/.gitignore new file mode 100644 index 000000000..f760c9035 --- /dev/null +++ b/link/sdks/dart/sync/.gitignore @@ -0,0 +1,4 @@ +.dart_tool/ +.flutter-plugins +.flutter-plugins-dependencies +build/ diff --git a/link/sdks/dart/sync/CHANGELOG.md b/link/sdks/dart/sync/CHANGELOG.md new file mode 100644 index 000000000..f858eb096 --- /dev/null +++ b/link/sdks/dart/sync/CHANGELOG.md @@ -0,0 +1,5 @@ +## 0.5.6-rc.0 + +- Initial local-first runtime with a Drift cache, durable checkpoints, row sync + state, bidirectional tables, backend-authoritative replicas, custom actions, + retries, lifecycle handling, and the shared `kalam_link` transport. diff --git a/link/sdks/dart/sync/LICENSE b/link/sdks/dart/sync/LICENSE new file mode 100644 index 000000000..f57b60e66 --- /dev/null +++ b/link/sdks/dart/sync/LICENSE @@ -0,0 +1,183 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship made available under + the License, as indicated by a copyright notice that is included in + or attached to the work (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean, as submitted to the Licensor for inclusion + in the Work by the copyright owner or by an individual or Legal Entity + authorized to submit on behalf of the copyright owner. For the purposes + of this definition, "submitted" means any form of electronic, verbal, + or written communication sent to the Licensor or its representatives, + including but not limited to communication on electronic mailing lists, + source code control systems, and issue tracking systems that are managed + by, or on behalf of, the Licensor for the purpose of discussing and + improving the Work, but excluding communication that is conspicuously + marked or otherwise designated in writing by the copyright owner as + "Not a Contribution." + + "Contributor" shall mean Licensor and any Legal Entity on behalf of + whom a Contribution has been received by the Licensor and incorporated + within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a cross-claim + or counterclaim in a lawsuit) alleging that the Work or any + Contribution embodied within the Work constitutes direct or contributory + patent infringement, then any patent licenses granted to You under + this License for that Work shall terminate as of the date such + litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or Derivative + Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, You must include a readable copy of the + attribution notices contained within such NOTICE file, in + at least one of the following places: within a NOTICE text + file distributed as part of the Derivative Works; within + the Source form or documentation, if provided along with the + Derivative Works; or, within a display generated by the + Derivative Works, if and wherever such third-party notices + normally appear. The contents of the NOTICE file are for + informational purposes only and do not modify the License. + You may add Your own attribution notices within Derivative + Works that You distribute, alongside or as an addendum to + the NOTICE text from the Work, provided that such additional + attribution notices cannot be construed as modifying the License. + + You may add Your own license statement for Your modifications and + may provide additional grant of rights to use, copy, modify, merge, + publish, distribute, sublicense, and/or sell copies of the + Contribution, either on a standalone basis or as part of the Work. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any conditions of title, + merchantability, fitness for a particular purpose and + non-infringement. You are solely responsible for determining the + appropriateness of using or reproducing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or exemplary damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or all other + commercial damages or losses), even if such Contributor has been + advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may offer only + conditions consistent with this terms and not on behalf of any other + Contributor, and only if You agree to indemnify, defend, and hold each + Contributor harmless for any liability incurred by, or claims asserted + against, such Contributor by reason of your accepting any such + warranty or additional liability. + + END OF TERMS AND CONDITIONS + + Copyright 2025 Jamal Saad + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/link/sdks/dart/sync/README.md b/link/sdks/dart/sync/README.md new file mode 100644 index 000000000..0f4241e80 --- /dev/null +++ b/link/sdks/dart/sync/README.md @@ -0,0 +1,233 @@ +# kalam_sync + +Local-first Flutter synchronization for KalamDB. The package adds one Drift +cache, durable resume checkpoints, optimistic row state, an offline action +outbox, retries, lifecycle handling, and widget-scoped event consumers on top +of the existing shared `kalam_link` connection. + +## Open one account-scoped cache + +```dart +final kalam = await Kalam.open( + url: 'https://db.example.com', + subject: signedInUser.id, + authProvider: () async => Auth.jwt(await tokens.freshAccessToken()), + actionDefinitions: messageActionsDefinitions(MessageActions(api)), +); + +runApp(KalamScope(kalam: kalam, child: const App())); +``` + +`subject` is required. The server URL, namespace, and authenticated subject +form the local database identity, so one user's cached rows and queued actions +cannot be opened or flushed as another user. + +The WebSocket is lazy. Opening the cache does not wait for a network +connection; the first active consumer or queued action starts the existing +`kalam_link` socket. + +## Bidirectional todos + +`kalam schema gen --languages dart` writes row codecs and `KalamTableSpec` +values (for example `KalamTables.todos`). Kalam only needs that table codec +and does not create a second todo model: + +```dart +final todos = kalam.table( + KalamTableSpec( + tableId: 'app.todos', + keyColumn: 'id', + mode: KalamSyncMode.bidirectional, + keyOf: (todo) => todo.id, + encode: (todo) => todo.toJson(), + decode: Todo.fromJson, + ), +); + +late final KalamSyncSubscription todoSync; + +Future startTodos() async { + todoSync = await kalam.subscribe( + todos.consumer(sql: 'SELECT * FROM app.todos', batchSize: 250), + ); +} + +Future addTodo(String title) { + return todos.insert( + Todo(id: Kalam.id(), title: title, completed: false), + actionId: Kalam.id(), + ); +} +``` + +`insert`, `update`, and `delete` change the local row and enqueue one generic +Kalam DML action in the same SQLite transaction. `todos.watch()` is always +local and continues to work offline. Use `watchWithSyncState()` when the UI +needs pending, retry, failure, or awaiting-server-echo state beside each row. + +## Subscribe inside a widget + +Consumers are not registered in one global `events: []` list. A widget or +feature starts only the query it needs and cancels it when unloaded: + +```dart +class ConversationState extends State { + late final KalamTableBinding messages; + KalamSyncSubscription? sync; + + @override + void initState() { + super.initState(); + final kalam = KalamScope.read(context); + messages = kalam.table(AppTables.messages); + final conversationId = widget.conversationId; + if (!RegExp(r'^[A-Za-z0-9_-]+$').hasMatch(conversationId)) { + throw ArgumentError.value(conversationId, 'conversationId'); + } + kalam + .subscribe(messages.consumer( + sql: "SELECT * FROM app.messages " + "WHERE conversation_id = '$conversationId'", + )) + .then((value) { + if (mounted) { + sync = value; + } else { + unawaited(value.cancel()); + } + }); + } + + @override + void dispose() { + sync?.cancel(); + super.dispose(); + } +} +``` + +`liveEvents` does not currently accept bind parameters, so validate any value +before placing it in a subscription query. The example permits only a narrow +identifier alphabet; generated query helpers should apply the server type's +exact validation rule. + +## Backend-authoritative messages with offline actions + +For a custom HTTP workflow, make the table `replicaOnly`. The backend remains +authoritative, while the action owns an optimistic overlay: + +```dart +@KalamActionPayload() +class SendMessageArgs { + const SendMessageArgs({ + required this.messageId, + required this.conversationId, + required this.text, + required this.createdAt, + required this.author, + }); + final String messageId; + final String conversationId; + final String text; + final DateTime createdAt; + final String author; +} + +@KalamActionModule(namespace: 'chat') +class ChatActions { + ChatActions(this.api); + final ChatActionApi api; + + @KalamAction(name: 'sendMessage') + Future sendMessage( + KalamActionContext context, + SendMessageArgs args, + ) async { + await context.step( + 'persist', + run: (key) async { + await api.persistMessage(args, idempotencyKey: key); + return true; + }, + encode: (value) => value, + decode: (value) => value == true, + ); + await context.step( + 'deliver', + run: (key) async { + await api.markDelivered(args, idempotencyKey: key); + return true; + }, + encode: (value) => value, + decode: (value) => value == true, + ); + } +} +``` + +Run `dart run build_runner build` in `link/sdks/dart/generator/example`. +`kalam_sync_generator` generates the JSON codec, `chat.sendMessage` definition, +and `ChatActionsQueue.sendMessage()`: + +```dart +final messageRows = kalam.table(chatMessagesSpec('chat.messages')); +final actions = ChatActionsQueue(kalam.actions); +final message = ChatMessage( + id: Kalam.id(), + conversationId: conversationId, + role: ChatMessageRole.user, + author: userId, + text: text, + status: ChatDeliveryStatus.pending, + createdAt: DateTime.now().toUtc(), +); + +await actions.sendMessage( + SendMessageArgs( + messageId: message.id, + conversationId: conversationId, + text: message.text, + createdAt: message.createdAt, + author: message.author, + ), + orderingKey: conversationId, + optimistic: messageRows.optimisticInsert(message), +); +``` + +The generated queue method commits the optimistic row, sidecar sync state, and +serialized action together. On connectivity, the executor calls the backend +with the stable action UUID. The row becomes `synced` only after the backend's +KalamDB write arrives and is committed locally. + +For multi-endpoint workflows, use `context.step(...)`. A completed named step +is persisted and reused after retry or process restart. Every remote endpoint +must still honor the supplied idempotency key because a response can be lost +after the server commits. + +## Generation boundary + +- Drift owns generated table row/companion types. +- `kalam_sync_generator` owns action payload codecs, definitions, and queues. +- Kalam uses one generic internal envelope for direct create/update/delete; + it does not generate three extra action model classes for every table. +- The complete offline-first chat app, REST engine, schema, and five live-server +use-case tests live in [`example/`](example). Conversations use bidirectional +DML; messages stay `replicaOnly` and sync through generated `sendMessage` / +`markMsgRead` actions. + +## Correctness guarantees + +- Local mirror row + outbox enqueue are one Drift transaction. +- Applied server row + durable sequence checkpoint are one Drift transaction. +- Server deliveries are acknowledged only after that transaction commits; + the next delivery is not pulled before acknowledgement. +- Duplicate or older events are ignored. +- Actions, retry metadata, named steps, optimistic rows, and checkpoints + survive process restart. +- Manual lifecycle pause/resume cancels and reopens subscriptions from the + SQLite-committed checkpoint. + +`kalam_sync` uses the additive `kalam_link.liveEventsWithAck` API. Existing +`liveEvents` callers keep automatic progress, while sync subscriptions resume +from the SQLite-committed cursor after reconnect or process restart. diff --git a/link/sdks/dart/sync/analysis_options.yaml b/link/sdks/dart/sync/analysis_options.yaml new file mode 100644 index 000000000..757a06d5e --- /dev/null +++ b/link/sdks/dart/sync/analysis_options.yaml @@ -0,0 +1,13 @@ +include: package:flutter_lints/flutter.yaml + +analyzer: + exclude: + - build/** + - lib/**.g.dart + - android/** + - ios/** + - web/** + - windows/** + - macos/** + - linux/** + diff --git a/link/sdks/dart/sync/example/.gitignore b/link/sdks/dart/sync/example/.gitignore new file mode 100644 index 000000000..f760c9035 --- /dev/null +++ b/link/sdks/dart/sync/example/.gitignore @@ -0,0 +1,4 @@ +.dart_tool/ +.flutter-plugins +.flutter-plugins-dependencies +build/ diff --git a/link/sdks/dart/sync/example/README.md b/link/sdks/dart/sync/example/README.md new file mode 100644 index 000000000..95b56814a --- /dev/null +++ b/link/sdks/dart/sync/example/README.md @@ -0,0 +1,160 @@ +# Offline-first chat example + +This Flutter app uses `kalam_sync`, generated `kalam_sync_generator` actions, and +`kalam_link` live events against a real KalamDB server. + +- **conversations** are `KalamSyncMode.bidirectional` — creating a chat writes + locally and queues generic DML. +- **messages** are `replicaOnly` — sends and read receipts go through generated + durable actions with optimistic overlays. The row becomes `synced` only after + the backend writes to KalamDB and the live event is committed locally. +- The REST engine writes authoritative user/assistant/receipt rows into KalamDB. + After `sendMessage`, it inserts a canned AI reply. + +Action source of truth: [`../../generator/example`](../../generator/example). + +## Ports and credentials + +| Service | Default | +| --- | --- | +| KalamDB | `http://localhost:2900` | +| Chat REST engine | `http://127.0.0.1:8787` | +| User | `admin` | +| Password | `kalamdb123` | + +These match the Dart SDK e2e helpers in `link/sdks/dart/link/test/e2e/helpers.dart`. +If your server only has `root`, export `KALAMDB_USER=root` and +`KALAMDB_PASSWORD` / `KALAMDB_ROOT_PASSWORD`. + +The dart SDK `test.sh` creates the `admin` DBA user when it can log in as root. + +## Run + +From the repository root: + +### 1. Start KalamDB + +```bash +cd backend +KALAMDB_ROOT_PASSWORD=kalamdb123 \ +KALAMDB_SERVER_PORT=2900 \ +KALAMDB_JWT_SECRET=kalamdb-e2e-test-jwt-secret-key!! \ +cargo run --bin kalamdb-server +``` + +If `cargo run` fails to link locally, use an existing debug binary: + +```bash +KALAMDB_ROOT_PASSWORD=kalamdb123 \ +KALAMDB_SERVER_PORT=2900 \ +KALAMDB_JWT_SECRET=kalamdb-e2e-test-jwt-secret-key!! \ +./target/debug/kalamdb-server +``` + +Health check: `curl -sf http://localhost:2900/v1/api/healthcheck` + +A fresh server creates `root`, not `admin`. The Dart e2e helpers default to +`admin` / `kalamdb123`. Create that user once: + +```sql +CREATE USER 'admin' WITH PASSWORD 'kalamdb123' ROLE dba +``` + +Or export `KALAMDB_USER=root` and `KALAMDB_PASSWORD=kalamdb123` for every +command below. + +### 2. Generate actions (first time, or after editing annotations) + +```bash +cd link/sdks/dart/generator/example +flutter pub get +dart run build_runner build --delete-conflicting-outputs +``` + +### 3. Start the REST engine + +The engine applies `kalam/schema.sql` (namespace `chat`) on startup. + +```bash +cd link/sdks/dart/sync/example/backend +dart pub get +KALAMDB_URL=http://localhost:2900 \ +KALAMDB_USER=admin \ +KALAMDB_PASSWORD=kalamdb123 \ +dart run bin/server.dart +``` + +Optional: `KALAM_CHAT_NAMESPACE=chat` `KALAM_CHAT_PORT=8787` + +You can also apply the schema yourself: + +```bash +curl -u admin:kalamdb123 -X POST http://localhost:2900/v1/api/sql \ + -H 'Content-Type: application/json' \ + --data-binary @link/sdks/dart/sync/example/kalam/schema.sql +``` + +`POST /v1/api/sql` expects JSON `{"sql":"..."}` for a single statement, so the +engine applies the statements one at a time. Prefer starting `bin/server.dart`. + +### 4. Run the Flutter app + +```bash +cd link/sdks/dart/sync/example +flutter pub get +flutter create . --project-name kalam_sync_example --platforms=macos +KALAMDB_URL=http://localhost:2900 \ +KALAM_CHAT_URL=http://127.0.0.1:8787 \ +KALAMDB_USER=admin \ +KALAMDB_PASSWORD=kalamdb123 \ +flutter run -d macos +``` + +`flutter create` is only needed once to generate platform runners. + +In the app: + +- FAB creates a conversation (bidirectional local insert). +- Send a message while online or after tapping the cloud icon to go offline. +- Offline sends stay in the outbox with an optimistic bubble. +- Going back online flushes REST (`persist` then `deliver`) and reconciles the + live echo to `synced`. An AI demo reply appears on the same conversation. +- Tap an assistant bubble to enqueue `markMsgRead`. + +## E2E tests + +These talk to a real KalamDB (HTTP + live websocket), the generated action +module, and the REST engine: + +```bash +cd link/sdks/dart/sync +flutter pub get +KALAM_INTEGRATION_TEST=1 \ +KALAMDB_URL=http://localhost:2900 \ +KALAMDB_USER=admin \ +KALAMDB_PASSWORD=kalamdb123 \ +flutter test test/e2e/chat_sync_e2e_test.dart +``` + +Use-case tests: + +1. `offlineEnqueueThenReconnect` — send while paused; optimistic row; reconnect + writes through REST and live echo becomes `synced`. +2. `replicaOnlyRejectsDirectMutations` — `messages.insert` throws; send/read go + through generated actions; server owns delivered/seen. +3. `optimisticUiThenServerEcho` — pending overlay, REST success, then echo to + `synced` (including delayed echo as `awaitingServerEcho`). +4. `readReceiptsOnlineAndOffline` — mark-read online and offline; a second + local cache sees `read`; retries are idempotent. +5. `catchUpAfterGapAndBackpressure` — 40 rows while paused; resume from the + SQLite checkpoint with `batchSize: 10`; no duplicate seqs; acks after commit. + +## Layout + +``` +example/ + lib/main.dart Flutter chat UI + backend/ Dart REST engine (writes to KalamDB) + kalam/schema.sql conversations + messages USER tables + README.md +``` diff --git a/link/sdks/dart/sync/example/analysis_options.yaml b/link/sdks/dart/sync/example/analysis_options.yaml new file mode 100644 index 000000000..1b67a164f --- /dev/null +++ b/link/sdks/dart/sync/example/analysis_options.yaml @@ -0,0 +1,11 @@ +include: package:flutter_lints/flutter.yaml + +analyzer: + exclude: + - build/** + - android/** + - ios/** + - web/** + - windows/** + - macos/** + - linux/** diff --git a/link/sdks/dart/sync/example/backend/.gitignore b/link/sdks/dart/sync/example/backend/.gitignore new file mode 100644 index 000000000..a8c7d1d6f --- /dev/null +++ b/link/sdks/dart/sync/example/backend/.gitignore @@ -0,0 +1,2 @@ +.dart_tool/ +build/ diff --git a/link/sdks/dart/sync/example/backend/bin/server.dart b/link/sdks/dart/sync/example/backend/bin/server.dart new file mode 100644 index 000000000..397ac7049 --- /dev/null +++ b/link/sdks/dart/sync/example/backend/bin/server.dart @@ -0,0 +1,32 @@ +import 'dart:io'; + +import 'package:kalam_sync_chat_backend/chat_backend.dart'; + +Future main(List args) async { + final kalamUrl = + Platform.environment['KALAMDB_URL'] ?? + Platform.environment['KALAM_URL'] ?? + 'http://localhost:2900'; + final username = + Platform.environment['KALAMDB_USER'] ?? + Platform.environment['KALAM_USER'] ?? + 'admin'; + final password = + Platform.environment['KALAMDB_PASSWORD'] ?? + Platform.environment['KALAM_PASS'] ?? + 'kalamdb123'; + final namespace = Platform.environment['KALAM_CHAT_NAMESPACE'] ?? 'chat'; + final port = int.parse(Platform.environment['KALAM_CHAT_PORT'] ?? '8787'); + final backend = ChatBackend( + kalamUrl: kalamUrl, + namespace: namespace, + username: username, + password: password, + port: port, + ); + final url = await backend.start(); + stdout.writeln('Chat backend listening on $url'); + stdout.writeln('KalamDB $kalamUrl namespace=$namespace user=$username'); + await ProcessSignal.sigint.watch().first; + await backend.close(); +} diff --git a/link/sdks/dart/sync/example/backend/lib/chat_backend.dart b/link/sdks/dart/sync/example/backend/lib/chat_backend.dart new file mode 100644 index 000000000..b4333e50a --- /dev/null +++ b/link/sdks/dart/sync/example/backend/lib/chat_backend.dart @@ -0,0 +1,345 @@ +import 'dart:convert'; +import 'dart:io'; +import 'dart:math'; + +/// Demo HTTP engine that receives chat actions and writes authoritative rows +/// into KalamDB via `/v1/api/sql`. +final class ChatBackend { + ChatBackend({ + required this.kalamUrl, + required this.namespace, + required this.username, + required this.password, + this.host = '127.0.0.1', + this.port = 8787, + HttpClient? httpClient, + Random? random, + }) : _httpClient = (httpClient ?? HttpClient()) + ..connectionTimeout = const Duration(seconds: 10) + ..idleTimeout = const Duration(seconds: 15), + _random = random ?? Random(); + + final String kalamUrl; + final String namespace; + final String username; + final String password; + final String host; + final int port; + final HttpClient _httpClient; + final Random _random; + final Set _idempotencyKeys = {}; + HttpServer? _server; + String? _accessToken; + + String get conversationsTable => '$namespace.conversations'; + String get messagesTable => '$namespace.messages'; + Uri get url => Uri.parse('http://$host:$port'); + + static const cannedReplies = [ + 'Got it. I will take a look.', + 'Nice — want me to expand on that?', + 'I saved that for later.', + 'Sounds good. Anything else?', + 'Here is a canned demo reply.', + ]; + + Future start() async { + await applySchema(); + _server = await HttpServer.bind(host, port); + _server!.listen(_handle); + return url; + } + + Future close() async { + await _server?.close(force: true); + _httpClient.close(force: true); + } + + Future applySchema() async { + for (final statement in chatSchemaStatements(namespace)) { + await executeSql(statement); + } + } + + Future executeSql(String sql) async { + final token = await _token(); + final request = await _httpClient.postUrl( + Uri.parse('$kalamUrl/v1/api/sql'), + ); + request.headers.contentType = ContentType.json; + request.headers.set(HttpHeaders.authorizationHeader, 'Bearer $token'); + request.write(jsonEncode({'sql': sql})); + final response = await request.close().timeout( + const Duration(seconds: 15), + ); + final body = await utf8.decodeStream(response); + if (response.statusCode < 200 || response.statusCode >= 300) { + throw HttpException( + 'KalamDB SQL HTTP ${response.statusCode}: $body\nSQL: $sql', + ); + } + if (body.isEmpty) return; + final decoded = jsonDecode(body); + if (decoded is Map && decoded['success'] == false) { + final error = decoded['error']?.toString() ?? body; + if (_isConflict(error)) return; + throw HttpException('KalamDB SQL failed: $body\nSQL: $sql'); + } + } + + bool _isConflict(String error) { + final lower = error.toLowerCase(); + return lower.contains('duplicate') || + lower.contains('already exists') || + lower.contains('unique') || + lower.contains('primary key'); + } + + Future rowExists(String table, String id) async { + final token = await _token(); + final sql = + 'SELECT id FROM $table WHERE id = ${sqlString(id)} LIMIT 1'; + final request = await _httpClient.postUrl( + Uri.parse('$kalamUrl/v1/api/sql'), + ); + request.headers.contentType = ContentType.json; + request.headers.set(HttpHeaders.authorizationHeader, 'Bearer $token'); + request.write(jsonEncode({'sql': sql})); + final response = await request.close().timeout( + const Duration(seconds: 15), + ); + final body = await utf8.decodeStream(response); + if (response.statusCode < 200 || response.statusCode >= 300) { + throw HttpException( + 'KalamDB lookup HTTP ${response.statusCode}: $body', + ); + } + final decoded = jsonDecode(body); + if (decoded is! Map) return false; + final results = decoded['results']; + if (results is! List || results.isEmpty) return false; + final rows = results.first is Map ? results.first['rows'] : null; + if (rows is List) return rows.isNotEmpty; + final named = results.first is Map + ? results.first['named_rows'] ?? results.first['namedRows'] + : null; + if (named is List) return named.isNotEmpty; + final rowCount = results.first is Map ? results.first['row_count'] : null; + if (rowCount is num) return rowCount > 0; + return body.contains(id); + } + + Future _handle(HttpRequest request) async { + request.response.headers.set('Access-Control-Allow-Origin', '*'); + request.response.headers.set( + 'Access-Control-Allow-Headers', + 'content-type, idempotency-key', + ); + request.response.headers.set( + 'Access-Control-Allow-Methods', + 'GET, POST, OPTIONS', + ); + try { + if (request.method == 'OPTIONS') { + request.response.statusCode = HttpStatus.noContent; + await request.response.close(); + return; + } + final path = request.uri.path; + if (request.method == 'GET' && path == '/health') { + await _json(request, {'ok': true, 'namespace': namespace}); + return; + } + final body = await utf8.decodeStream(request); + final json = body.isEmpty + ? {} + : Map.from(jsonDecode(body) as Map); + if (request.method == 'POST' && path == '/v1/messages') { + await persistMessage(json); + await _json(request, {'ok': true}); + return; + } + final deliver = RegExp(r'^/v1/messages/([^/]+)/deliver$').firstMatch(path); + if (request.method == 'POST' && deliver != null) { + await markDelivered( + Uri.decodeComponent(deliver.group(1)!), + json, + ); + await _json(request, {'ok': true}); + return; + } + final read = RegExp(r'^/v1/messages/([^/]+)/read$').firstMatch(path); + if (request.method == 'POST' && read != null) { + await markRead(Uri.decodeComponent(read.group(1)!), json); + await _json(request, {'ok': true}); + return; + } + request.response.statusCode = HttpStatus.notFound; + await _json(request, {'error': 'not found'}); + } catch (error) { + request.response.statusCode = HttpStatus.internalServerError; + await _json(request, {'error': error.toString()}); + } + } + + Future persistMessage(Map json) async { + final messageId = json['messageId']!.toString(); + final conversationId = json['conversationId']!.toString(); + final text = json['text']!.toString(); + final author = (json['author'] ?? 'user').toString(); + final createdAt = (json['createdAt'] ?? DateTime.now().toUtc().toIso8601String()) + .toString(); + final idempotencyKey = json['idempotencyKey']?.toString(); + if (idempotencyKey != null && !_idempotencyKeys.add(idempotencyKey)) { + return; + } + if (await rowExists(messagesTable, messageId)) return; + await _ensureConversation(conversationId); + await executeSql( + 'INSERT INTO $messagesTable (' + 'id, conversation_id, role, author, text, status, created_at' + ') VALUES (' + '${sqlString(messageId)}, ' + '${sqlString(conversationId)}, ' + "'user', " + '${sqlString(author)}, ' + '${sqlString(text)}, ' + "'sent', " + '${sqlString(createdAt)}' + ')', + ); + } + + Future markDelivered( + String messageId, + Map json, + ) async { + final idempotencyKey = json['idempotencyKey']?.toString(); + if (idempotencyKey != null && !_idempotencyKeys.add(idempotencyKey)) { + return; + } + final deliveredAt = + (json['deliveredAt'] ?? DateTime.now().toUtc().toIso8601String()) + .toString(); + await executeSql( + 'UPDATE $messagesTable SET status = \'delivered\', ' + 'delivered_at = ${sqlString(deliveredAt)} ' + 'WHERE id = ${sqlString(messageId)}', + ); + await _insertAssistantReply(messageId, json); + } + + Future markRead(String messageId, Map json) async { + final idempotencyKey = json['idempotencyKey']?.toString(); + if (idempotencyKey != null && !_idempotencyKeys.add(idempotencyKey)) { + return; + } + final readAt = + (json['readAt'] ?? DateTime.now().toUtc().toIso8601String()).toString(); + await executeSql( + 'UPDATE $messagesTable SET status = \'read\', ' + 'read_at = ${sqlString(readAt)} ' + 'WHERE id = ${sqlString(messageId)}', + ); + } + + Future _insertAssistantReply( + String userMessageId, + Map json, + ) async { + final assistantId = 'ai-$userMessageId'; + if (await rowExists(messagesTable, assistantId)) return; + final conversationId = json['conversationId']?.toString(); + if (conversationId == null || conversationId.isEmpty) { + throw const FormatException('deliver requires conversationId'); + } + final reply = cannedReplies[_random.nextInt(cannedReplies.length)]; + final createdAt = DateTime.now().toUtc().toIso8601String(); + await executeSql( + 'INSERT INTO $messagesTable (' + 'id, conversation_id, role, author, text, status, created_at, delivered_at' + ') VALUES (' + '${sqlString(assistantId)}, ' + '${sqlString(conversationId)}, ' + "'assistant', " + "'demo-bot', " + '${sqlString(reply)}, ' + "'delivered', " + '${sqlString(createdAt)}, ' + '${sqlString(createdAt)}' + ')', + ); + } + + Future _ensureConversation(String conversationId) async { + if (await rowExists(conversationsTable, conversationId)) return; + final now = DateTime.now().toUtc().toIso8601String(); + await executeSql( + 'INSERT INTO $conversationsTable (id, title, created_at, updated_at) ' + 'VALUES (' + '${sqlString(conversationId)}, ' + "'Chat', " + '${sqlString(now)}, ' + '${sqlString(now)}' + ')', + ); + } + + Future _token() async { + final cached = _accessToken; + if (cached != null) return cached; + final request = await _httpClient.postUrl( + Uri.parse('$kalamUrl/v1/api/auth/login'), + ); + request.headers.contentType = ContentType.json; + request.write( + jsonEncode({'username': username, 'password': password}), + ); + final response = await request.close().timeout( + const Duration(seconds: 15), + ); + final body = await utf8.decodeStream(response); + if (response.statusCode < 200 || response.statusCode >= 300) { + throw HttpException('KalamDB login HTTP ${response.statusCode}: $body'); + } + final decoded = jsonDecode(body) as Map; + final token = + decoded['access_token'] ?? decoded['accessToken'] ?? decoded['token']; + if (token is! String || token.isEmpty) { + throw HttpException('KalamDB login missing access token: $body'); + } + _accessToken = token; + return token; + } + + Future _json(HttpRequest request, Map body) async { + request.response.headers.contentType = ContentType.json; + request.response.write(jsonEncode(body)); + await request.response.close(); + } +} + +List chatSchemaStatements(String namespace) { + return [ + 'CREATE NAMESPACE IF NOT EXISTS $namespace', + 'CREATE TABLE IF NOT EXISTS $namespace.conversations (' + 'id TEXT PRIMARY KEY, ' + 'title TEXT NOT NULL, ' + 'created_at TIMESTAMP NOT NULL, ' + 'updated_at TIMESTAMP NOT NULL' + ") WITH (TYPE = 'USER')", + 'CREATE TABLE IF NOT EXISTS $namespace.messages (' + 'id TEXT PRIMARY KEY, ' + 'conversation_id TEXT NOT NULL, ' + 'role TEXT NOT NULL, ' + 'author TEXT NOT NULL, ' + 'text TEXT NOT NULL, ' + 'status TEXT NOT NULL, ' + 'created_at TIMESTAMP NOT NULL, ' + 'delivered_at TIMESTAMP, ' + 'read_at TIMESTAMP' + ") WITH (TYPE = 'USER')", + ]; +} + +String sqlString(String value) => "'${value.replaceAll("'", "''")}'"; diff --git a/link/sdks/dart/sync/example/backend/pubspec.lock b/link/sdks/dart/sync/example/backend/pubspec.lock new file mode 100644 index 000000000..cc66e3bdd --- /dev/null +++ b/link/sdks/dart/sync/example/backend/pubspec.lock @@ -0,0 +1,5 @@ +# Generated by pub +# See https://dart.dev/tools/pub/glossary#lockfile +packages: {} +sdks: + dart: ">=3.10.0 <4.0.0" diff --git a/link/sdks/dart/sync/example/backend/pubspec.yaml b/link/sdks/dart/sync/example/backend/pubspec.yaml new file mode 100644 index 000000000..8622ab010 --- /dev/null +++ b/link/sdks/dart/sync/example/backend/pubspec.yaml @@ -0,0 +1,7 @@ +name: kalam_sync_chat_backend +description: Demo REST engine that writes chat rows into KalamDB user tables. +publish_to: none +version: 0.5.6-rc.0 + +environment: + sdk: ">=3.10.0 <4.0.0" diff --git a/link/sdks/dart/sync/example/kalam.toml b/link/sdks/dart/sync/example/kalam.toml new file mode 100644 index 000000000..5dff199f6 --- /dev/null +++ b/link/sdks/dart/sync/example/kalam.toml @@ -0,0 +1,16 @@ +[project] +name = "kalam-sync-chat" +default_env = "dev" + +[connection.dev] +url = "http://localhost:2900" +namespace = "chat" + +[schema] +mode = "sql" +path = "kalam/schema.sql" +watch = false + +[dev] +auto_start_db = true +apply_schema = true diff --git a/link/sdks/dart/sync/example/kalam/schema.sql b/link/sdks/dart/sync/example/kalam/schema.sql new file mode 100644 index 000000000..7f44797fb --- /dev/null +++ b/link/sdks/dart/sync/example/kalam/schema.sql @@ -0,0 +1,20 @@ +CREATE NAMESPACE IF NOT EXISTS chat; + +CREATE TABLE IF NOT EXISTS chat.conversations ( + id TEXT PRIMARY KEY, + title TEXT NOT NULL, + created_at TIMESTAMP NOT NULL, + updated_at TIMESTAMP NOT NULL +) WITH (TYPE = 'USER'); + +CREATE TABLE IF NOT EXISTS chat.messages ( + id TEXT PRIMARY KEY, + conversation_id TEXT NOT NULL, + role TEXT NOT NULL, + author TEXT NOT NULL, + text TEXT NOT NULL, + status TEXT NOT NULL, + created_at TIMESTAMP NOT NULL, + delivered_at TIMESTAMP, + read_at TIMESTAMP +) WITH (TYPE = 'USER'); diff --git a/link/sdks/dart/sync/example/lib/main.dart b/link/sdks/dart/sync/example/lib/main.dart new file mode 100644 index 000000000..f2d50263f --- /dev/null +++ b/link/sdks/dart/sync/example/lib/main.dart @@ -0,0 +1,350 @@ +import 'dart:async'; +import 'dart:io'; + +import 'package:flutter/material.dart'; +import 'package:kalam_sync/kalam_sync.dart'; +import 'package:kalam_sync_generator_example/chat_actions.dart'; +import 'package:kalam_sync_generator_example/chat_http_api.dart'; +import 'package:kalam_sync_generator_example/chat_models.dart'; +import 'package:kalam_sync_generator_example/chat_tables.dart'; + +String _env(String key, String fallback) { + final value = Platform.environment[key]; + if (value == null || value.trim().isEmpty) return fallback; + return value; +} + +final kalamUrl = _env('KALAMDB_URL', _env('KALAM_URL', 'http://localhost:2900')); +final chatApiUrl = _env('KALAM_CHAT_URL', 'http://127.0.0.1:8787'); +final username = _env('KALAMDB_USER', _env('KALAM_USER', 'admin')); +final password = _env( + 'KALAMDB_PASSWORD', + _env('KALAM_PASS', 'kalamdb123'), +); +const namespace = 'chat'; + +Future main() async { + WidgetsFlutterBinding.ensureInitialized(); + final api = ChatHttpApi(baseUrl: Uri.parse(chatApiUrl)); + final kalam = await Kalam.open( + url: kalamUrl, + subject: username, + namespace: namespace, + authProvider: () async => Auth.basic(username, password), + actionDefinitions: chatActionsDefinitions(ChatActions(api)), + ); + runApp( + KalamScope( + kalam: kalam, + child: ChatApp(kalam: kalam, author: username), + ), + ); +} + +final class ChatApp extends StatefulWidget { + const ChatApp({required this.kalam, required this.author, super.key}); + + final Kalam kalam; + final String author; + + @override + State createState() => _ChatAppState(); +} + +final class _ChatAppState extends State { + late final conversations = widget.kalam.table( + chatConversationsSpec('$namespace.conversations'), + ); + late final messages = widget.kalam.table( + chatMessagesSpec('$namespace.messages'), + ); + late final actions = ChatActionsQueue(widget.kalam.actions); + final subscriptions = []; + String? openConversationId; + var offline = false; + + @override + void initState() { + super.initState(); + _start(); + } + + Future _start() async { + subscriptions.add( + await widget.kalam.subscribe( + conversations.consumer(sql: 'SELECT * FROM $namespace.conversations'), + ), + ); + subscriptions.add( + await widget.kalam.subscribe( + messages.consumer(sql: 'SELECT * FROM $namespace.messages'), + ), + ); + } + + @override + void dispose() { + for (final subscription in subscriptions) { + unawaited(subscription.cancel()); + } + super.dispose(); + } + + Future _toggleOffline() async { + if (offline) { + await widget.kalam.resume(); + } else { + await widget.kalam.pause(); + } + setState(() => offline = !offline); + } + + Future _createConversation() async { + final now = DateTime.now().toUtc(); + final conversation = Conversation( + id: Kalam.id(), + title: 'Chat ${now.toIso8601String().substring(11, 19)}', + createdAt: now, + updatedAt: now, + ); + await conversations.insert(conversation, actionId: Kalam.id()); + setState(() => openConversationId = conversation.id); + } + + Future _send(String conversationId, String text) async { + final now = DateTime.now().toUtc(); + final message = ChatMessage( + id: Kalam.id(), + conversationId: conversationId, + role: ChatMessageRole.user, + author: widget.author, + text: text, + status: ChatDeliveryStatus.pending, + createdAt: now, + ); + await actions.sendMessage( + SendMessageArgs( + messageId: message.id, + conversationId: conversationId, + text: text, + createdAt: now, + author: widget.author, + ), + orderingKey: conversationId, + optimistic: messages.optimisticInsert(message), + ); + } + + Future _markRead(ChatMessage message) async { + if (message.role != ChatMessageRole.assistant) return; + final readAt = DateTime.now().toUtc(); + await actions.markMsgRead( + MarkMessageReadArgs( + messageId: message.id, + conversationId: message.conversationId, + readAt: readAt, + ), + orderingKey: message.conversationId, + optimistic: messages.optimisticInsert( + message.copyWith(status: ChatDeliveryStatus.read, readAt: readAt), + ), + ); + } + + @override + Widget build(BuildContext context) { + return MaterialApp( + title: 'Kalam chat', + home: Scaffold( + appBar: AppBar( + title: Text(offline ? 'Kalam chat (offline)' : 'Kalam chat'), + actions: [ + IconButton( + tooltip: offline ? 'Go online' : 'Go offline', + onPressed: _toggleOffline, + icon: Icon(offline ? Icons.cloud_off : Icons.cloud_queue), + ), + ], + ), + body: openConversationId == null + ? _ConversationList( + conversations: conversations, + onOpen: (id) => setState(() => openConversationId = id), + ) + : _ConversationView( + conversationId: openConversationId!, + messages: messages, + onBack: () => setState(() => openConversationId = null), + onSend: (text) => _send(openConversationId!, text), + onMarkRead: _markRead, + ), + floatingActionButton: openConversationId == null + ? FloatingActionButton( + onPressed: _createConversation, + child: const Icon(Icons.add_comment), + ) + : null, + ), + ); + } +} + +final class _ConversationList extends StatelessWidget { + const _ConversationList({ + required this.conversations, + required this.onOpen, + }); + + final KalamTableBinding conversations; + final ValueChanged onOpen; + + @override + Widget build(BuildContext context) { + return StreamBuilder>>( + stream: conversations.watchWithSyncState(), + builder: (context, snapshot) { + final rows = [...snapshot.data ?? const >[]] + ..sort((a, b) => b.value.updatedAt.compareTo(a.value.updatedAt)); + if (rows.isEmpty) { + return const Center(child: Text('Create a conversation to start.')); + } + return ListView( + children: [ + for (final row in rows) + ListTile( + title: Text(row.value.title), + subtitle: Text(row.isSynced ? 'Synced' : 'Pending sync'), + trailing: Icon( + row.isSynced ? Icons.cloud_done : Icons.cloud_upload, + ), + onTap: () => onOpen(row.value.id), + ), + ], + ); + }, + ); + } +} + +final class _ConversationView extends StatefulWidget { + const _ConversationView({ + required this.conversationId, + required this.messages, + required this.onBack, + required this.onSend, + required this.onMarkRead, + }); + + final String conversationId; + final KalamTableBinding messages; + final VoidCallback onBack; + final Future Function(String text) onSend; + final Future Function(ChatMessage message) onMarkRead; + + @override + State<_ConversationView> createState() => _ConversationViewState(); +} + +final class _ConversationViewState extends State<_ConversationView> { + final controller = TextEditingController(); + + @override + void dispose() { + controller.dispose(); + super.dispose(); + } + + @override + Widget build(BuildContext context) { + return Column( + children: [ + ListTile( + leading: IconButton( + onPressed: widget.onBack, + icon: const Icon(Icons.arrow_back), + ), + title: const Text('Messages'), + subtitle: const Text('Tap an assistant bubble to mark it read'), + ), + Expanded( + child: StreamBuilder>>( + stream: widget.messages.watchWithSyncState(), + builder: (context, snapshot) { + final rows = + [ + ...snapshot.data ?? const >[], + ].where((row) => row.value.conversationId == widget.conversationId).toList() + ..sort( + (a, b) => a.value.createdAt.compareTo(b.value.createdAt), + ); + return ListView( + padding: const EdgeInsets.all(12), + children: [ + for (final row in rows) + Align( + alignment: row.value.isAssistant + ? Alignment.centerLeft + : Alignment.centerRight, + child: GestureDetector( + onTap: () => widget.onMarkRead(row.value), + child: Card( + color: row.value.isAssistant + ? Colors.grey.shade200 + : Colors.blue.shade50, + child: Padding( + padding: const EdgeInsets.all(12), + child: Column( + crossAxisAlignment: CrossAxisAlignment.start, + children: [ + Text(row.value.text), + const SizedBox(height: 4), + Text( + '${row.value.status.name} · ${row.sync.phase.name}', + style: Theme.of(context).textTheme.bodySmall, + ), + ], + ), + ), + ), + ), + ), + ], + ); + }, + ), + ), + Padding( + padding: const EdgeInsets.all(12), + child: Row( + children: [ + Expanded( + child: TextField( + controller: controller, + decoration: const InputDecoration( + hintText: 'Message', + border: OutlineInputBorder(), + ), + onSubmitted: (value) async { + final text = value.trim(); + if (text.isEmpty) return; + controller.clear(); + await widget.onSend(text); + }, + ), + ), + IconButton( + onPressed: () async { + final text = controller.text.trim(); + if (text.isEmpty) return; + controller.clear(); + await widget.onSend(text); + }, + icon: const Icon(Icons.send), + ), + ], + ), + ), + ], + ); + } +} diff --git a/link/sdks/dart/sync/example/pubspec.lock b/link/sdks/dart/sync/example/pubspec.lock new file mode 100644 index 000000000..18c443d4a --- /dev/null +++ b/link/sdks/dart/sync/example/pubspec.lock @@ -0,0 +1,538 @@ +# Generated by pub +# See https://dart.dev/tools/pub/glossary#lockfile +packages: + args: + dependency: transitive + description: + name: args + sha256: d0481093c50b1da8910eb0bb301626d4d8eb7284aa739614d2b394ee09e3ea04 + url: "https://pub.dev" + source: hosted + version: "2.7.0" + async: + dependency: transitive + description: + name: async + sha256: e2eb0491ba5ddb6177742d2da23904574082139b07c1e33b8503b9f46f3e1a37 + url: "https://pub.dev" + source: hosted + version: "2.13.1" + boolean_selector: + dependency: transitive + description: + name: boolean_selector + sha256: "8aab1771e1243a5063b8b0ff68042d67334e3feab9e95b9490f9a6ebf73b42ea" + url: "https://pub.dev" + source: hosted + version: "2.1.2" + build_cli_annotations: + dependency: transitive + description: + name: build_cli_annotations + sha256: e563c2e01de8974566a1998410d3f6f03521788160a02503b0b1f1a46c7b3d95 + url: "https://pub.dev" + source: hosted + version: "2.1.1" + characters: + dependency: transitive + description: + name: characters + sha256: faf38497bda5ead2a8c7615f4f7939df04333478bf32e4173fcb06d428b5716b + url: "https://pub.dev" + source: hosted + version: "1.4.1" + clock: + dependency: transitive + description: + name: clock + sha256: fddb70d9b5277016c77a80201021d40a2247104d9f4aa7bab7157b7e3f05b84b + url: "https://pub.dev" + source: hosted + version: "1.1.2" + code_assets: + dependency: transitive + description: + name: code_assets + sha256: bf394f466ba9205f1812a0433b392d6af280f155f56651eda7c18cc32ed493b8 + url: "https://pub.dev" + source: hosted + version: "1.2.1" + collection: + dependency: transitive + description: + name: collection + sha256: "2f5709ae4d3d59dd8f7cd309b4e023046b57d8a6c82130785d2b0e5868084e76" + url: "https://pub.dev" + source: hosted + version: "1.19.1" + convert: + dependency: transitive + description: + name: convert + sha256: b30acd5944035672bc15c6b7a8b47d773e41e2f17de064350988c5d02adb1c68 + url: "https://pub.dev" + source: hosted + version: "3.1.2" + crypto: + dependency: transitive + description: + name: crypto + sha256: c8ea0233063ba03258fbcf2ca4d6dadfefe14f02fab57702265467a19f27fadf + url: "https://pub.dev" + source: hosted + version: "3.0.7" + drift: + dependency: transitive + description: + name: drift + sha256: "3a3f1f6f905037d7426e4c445854139fd6a3d592135f7c96d7931682b73d16f4" + url: "https://pub.dev" + source: hosted + version: "2.34.3" + drift_flutter: + dependency: transitive + description: + name: drift_flutter + sha256: "91acf4bee7c3c84467cba46455aa70e5292a3b889f4582645d74f2e5a8c106f2" + url: "https://pub.dev" + source: hosted + version: "0.3.1" + fake_async: + dependency: transitive + description: + name: fake_async + sha256: "5368f224a74523e8d2e7399ea1638b37aecfca824a3cc4dfdf77bf1fa905ac44" + url: "https://pub.dev" + source: hosted + version: "1.3.3" + ffi: + dependency: transitive + description: + name: ffi + sha256: "6d7fd89431262d8f3125e81b50d3847a091d846eafcd4fdb88dd06f36d705a45" + url: "https://pub.dev" + source: hosted + version: "2.2.0" + file: + dependency: transitive + description: + name: file + sha256: a3b4f84adafef897088c160faf7dfffb7696046cb13ae90b508c2cbc95d3b8d4 + url: "https://pub.dev" + source: hosted + version: "7.0.1" + flutter: + dependency: "direct main" + description: flutter + source: sdk + version: "0.0.0" + flutter_lints: + dependency: "direct dev" + description: + name: flutter_lints + sha256: "3105dc8492f6183fb076ccf1f351ac3d60564bff92e20bfc4af9cc1651f4e7e1" + url: "https://pub.dev" + source: hosted + version: "6.0.0" + flutter_rust_bridge: + dependency: transitive + description: + name: flutter_rust_bridge + sha256: e87d6b9ee934dcd24a128ccb2bd91905d2d5fe5c06245d6a8f5477d4907a437a + url: "https://pub.dev" + source: hosted + version: "2.12.0" + flutter_test: + dependency: "direct dev" + description: flutter + source: sdk + version: "0.0.0" + freezed_annotation: + dependency: transitive + description: + name: freezed_annotation + sha256: "7294967ff0a6d98638e7acb774aac3af2550777accd8149c90af5b014e6d44d8" + url: "https://pub.dev" + source: hosted + version: "3.1.0" + glob: + dependency: transitive + description: + name: glob + sha256: c3f1ee72c96f8f78935e18aa8cecced9ab132419e8625dc187e1c2408efc20de + url: "https://pub.dev" + source: hosted + version: "2.1.3" + hooks: + dependency: transitive + description: + name: hooks + sha256: "03a564c3704524ee0f7fc56fc621e8796cc2eb8c24d1f1a33b34979815b285c4" + url: "https://pub.dev" + source: hosted + version: "2.1.0" + jni: + dependency: transitive + description: + name: jni + sha256: f038e58b4dc2c9037f50e233175086337e0b305e356d28211bf55f21c504cbd3 + url: "https://pub.dev" + source: hosted + version: "1.0.3" + jni_flutter: + dependency: transitive + description: + name: jni_flutter + sha256: "7b717011ea40d04fd47c2731d3d1d36eb99eba3435c2753d62489e8c3c9991d5" + url: "https://pub.dev" + source: hosted + version: "1.0.2" + jni_util: + dependency: transitive + description: + name: jni_util + sha256: "1ba86da04a5f2bf18fde2edb235587e70c5b0fc5bd4ba955f46b00942c3fc35f" + url: "https://pub.dev" + source: hosted + version: "1.0.0" + json_annotation: + dependency: transitive + description: + name: json_annotation + sha256: "2a743920d81b7910627f68ee2c9ac1fc0bfee32b9fc3403587d7c6791ca12f80" + url: "https://pub.dev" + source: hosted + version: "4.12.0" + kalam_link: + dependency: "direct overridden" + description: + path: "../../link" + relative: true + source: path + version: "0.5.6-rc.0" + kalam_sync: + dependency: "direct main" + description: + path: ".." + relative: true + source: path + version: "0.5.6-rc.0" + kalam_sync_generator_example: + dependency: "direct main" + description: + path: "../../generator/example" + relative: true + source: path + version: "0.5.6-rc.0" + leak_tracker: + dependency: transitive + description: + name: leak_tracker + sha256: "33e2e26bdd85a0112ec15400c8cbffea70d0f9c3407491f672a2fad47915e2de" + url: "https://pub.dev" + source: hosted + version: "11.0.2" + leak_tracker_flutter_testing: + dependency: transitive + description: + name: leak_tracker_flutter_testing + sha256: "1dbc140bb5a23c75ea9c4811222756104fbcd1a27173f0c34ca01e16bea473c1" + url: "https://pub.dev" + source: hosted + version: "3.0.10" + leak_tracker_testing: + dependency: transitive + description: + name: leak_tracker_testing + sha256: "8d5a2d49f4a66b49744b23b018848400d23e54caf9463f4eb20df3eb8acb2eb1" + url: "https://pub.dev" + source: hosted + version: "3.0.2" + lints: + dependency: transitive + description: + name: lints + sha256: "12f842a479589fea194fe5c5a3095abc7be0c1f2ddfa9a0e76aed1dbd26a87df" + url: "https://pub.dev" + source: hosted + version: "6.1.0" + logger: + dependency: transitive + description: + name: logger + sha256: "25aee487596a6257655a1e091ec2ae66bc30e7af663592cc3a27e6591e05035c" + url: "https://pub.dev" + source: hosted + version: "2.7.0" + logging: + dependency: transitive + description: + name: logging + sha256: c8245ada5f1717ed44271ed1c26b8ce85ca3228fd2ffdb75468ab01979309d61 + url: "https://pub.dev" + source: hosted + version: "1.3.0" + matcher: + dependency: transitive + description: + name: matcher + sha256: "31bd099b47c10cd1aeb55146a2d46ce0277630ecef3f7dae54ad7873f36696cd" + url: "https://pub.dev" + source: hosted + version: "0.12.20" + material_color_utilities: + dependency: transitive + description: + name: material_color_utilities + sha256: "9c337007e82b1889149c82ed242ed1cb24a66044e30979c44912381e9be4c48b" + url: "https://pub.dev" + source: hosted + version: "0.13.0" + meta: + dependency: transitive + description: + name: meta + sha256: "307249ce4ff29d58a18e97f6345f539382eb9c9c29ecda628900f31de0443dd9" + url: "https://pub.dev" + source: hosted + version: "1.19.0" + native_toolchain_c: + dependency: transitive + description: + name: native_toolchain_c + sha256: a1c26117c48cebe5677b0cf0e33a980a79a7c5577effc86f52e5a0d309cdcb60 + url: "https://pub.dev" + source: hosted + version: "0.19.3" + objective_c: + dependency: transitive + description: + name: objective_c + sha256: b7fb95a6d9a4f009edd63dc5ac69f07420b23a16161c6dd8660290b59c602e8e + url: "https://pub.dev" + source: hosted + version: "9.5.0" + package_config: + dependency: transitive + description: + name: package_config + sha256: ffcf4cf3d6c0b74ac43708d9f56625506e8a68aa935abe9d267a7330f320eb5d + url: "https://pub.dev" + source: hosted + version: "3.0.0" + path: + dependency: transitive + description: + name: path + sha256: "75cca69d1490965be98c73ceaea117e8a04dd21217b37b292c9ddbec0d955bc5" + url: "https://pub.dev" + source: hosted + version: "1.9.1" + path_provider: + dependency: transitive + description: + name: path_provider + sha256: a7f4874f987173da295a61c181b8ee71dab59b332a486b391babf26a1b884825 + url: "https://pub.dev" + source: hosted + version: "2.1.6" + path_provider_android: + dependency: transitive + description: + name: path_provider_android + sha256: "69cbd515a62b94d32a7944f086b2f82b4ac40a1d45bebfc00813a430ab2dabcd" + url: "https://pub.dev" + source: hosted + version: "2.3.1" + path_provider_foundation: + dependency: transitive + description: + name: path_provider_foundation + sha256: "2a376b7d6392d80cd3705782d2caa734ca4727776db0b6ec36ef3f1855197699" + url: "https://pub.dev" + source: hosted + version: "2.6.0" + path_provider_linux: + dependency: transitive + description: + name: path_provider_linux + sha256: "58c2005f147315b11e9b4a7bc889cd5203e250cba8e3f012dae259b4972b5c16" + url: "https://pub.dev" + source: hosted + version: "2.2.2" + path_provider_platform_interface: + dependency: transitive + description: + name: path_provider_platform_interface + sha256: "484838772624c3a4b94f1e44a3e19897fee738f2d5c4ce448443b0417f7c9dda" + url: "https://pub.dev" + source: hosted + version: "2.1.3" + path_provider_windows: + dependency: transitive + description: + name: path_provider_windows + sha256: bd6f00dbd873bfb70d0761682da2b3a2c2fccc2b9e84c495821639601d81afe7 + url: "https://pub.dev" + source: hosted + version: "2.3.0" + platform: + dependency: transitive + description: + name: platform + sha256: "5d6b1b0036a5f331ebc77c850ebc8506cbc1e9416c27e59b439f917a902a4984" + url: "https://pub.dev" + source: hosted + version: "3.1.6" + plugin_platform_interface: + dependency: transitive + description: + name: plugin_platform_interface + sha256: "4820fbfdb9478b1ebae27888254d445073732dae3d6ea81f0b7e06d5dedc3f02" + url: "https://pub.dev" + source: hosted + version: "2.1.8" + pub_semver: + dependency: transitive + description: + name: pub_semver + sha256: "5bfcf68ca79ef689f8990d1160781b4bad40a3bd5e5218ad4076ddb7f4081585" + url: "https://pub.dev" + source: hosted + version: "2.2.0" + record_use: + dependency: transitive + description: + name: record_use + sha256: "3b4ab682aff40175afca6ff090cc5aa7ce9dafe8dfbae1537a6c678e6678baee" + url: "https://pub.dev" + source: hosted + version: "1.1.0" + sky_engine: + dependency: transitive + description: flutter + source: sdk + version: "0.0.0" + source_span: + dependency: transitive + description: + name: source_span + sha256: "56a02f1f4cd1a2d96303c0144c93bd6d909eea6bee6bf5a0e0b685edbd4c47ab" + url: "https://pub.dev" + source: hosted + version: "1.10.2" + sqlcipher_flutter_libs: + dependency: transitive + description: + name: sqlcipher_flutter_libs + sha256: "38d62d659d2fb8739bf25a42c9a350d1fdd6c29a5a61f13a946778ec75d27929" + url: "https://pub.dev" + source: hosted + version: "0.7.0+eol" + sqlite3: + dependency: transitive + description: + name: sqlite3 + sha256: "64b2c63c8232dd20d14b34105a81ebfd74320442e8451f836179ec89986aa478" + url: "https://pub.dev" + source: hosted + version: "3.5.1" + sqlite3_flutter_libs: + dependency: transitive + description: + name: sqlite3_flutter_libs + sha256: "3ed7553eee7bb368f8950f58ba29f634e06e813c029aff6a0d60862b96de8454" + url: "https://pub.dev" + source: hosted + version: "0.6.0+eol" + stack_trace: + dependency: transitive + description: + name: stack_trace + sha256: "8b27215b45d22309b5cddda1aa2b19bdfec9df0e765f2de506401c071d38d1b1" + url: "https://pub.dev" + source: hosted + version: "1.12.1" + stream_channel: + dependency: transitive + description: + name: stream_channel + sha256: "969e04c80b8bcdf826f8f16579c7b14d780458bd97f56d107d3950fdbeef059d" + url: "https://pub.dev" + source: hosted + version: "2.1.4" + string_scanner: + dependency: transitive + description: + name: string_scanner + sha256: "921cd31725b72fe181906c6a94d987c78e3b98c2e205b397ea399d4054872b43" + url: "https://pub.dev" + source: hosted + version: "1.4.1" + term_glyph: + dependency: transitive + description: + name: term_glyph + sha256: "7f554798625ea768a7518313e58f83891c7f5024f88e46e7182a4558850a4b8e" + url: "https://pub.dev" + source: hosted + version: "1.2.2" + test_api: + dependency: transitive + description: + name: test_api + sha256: "2a122cbe059f8b610d3a5415f42e255b6c17b1f21eee1d960f31080237fb4f11" + url: "https://pub.dev" + source: hosted + version: "0.7.12" + typed_data: + dependency: transitive + description: + name: typed_data + sha256: f9049c039ebfeb4cf7a7104a675823cd72dba8297f264b6637062516699fa006 + url: "https://pub.dev" + source: hosted + version: "1.4.0" + vector_math: + dependency: transitive + description: + name: vector_math + sha256: f36f9f3be64c6198714492bb455c11056e33e2f85d9a0b676a48301e44fdcf47 + url: "https://pub.dev" + source: hosted + version: "2.4.2" + vm_service: + dependency: transitive + description: + name: vm_service + sha256: "0016aef94fc66495ac78af5859181e3f3bf2026bd8eecc72b9565601e19ab360" + url: "https://pub.dev" + source: hosted + version: "15.2.0" + web: + dependency: transitive + description: + name: web + sha256: "868d88a33d8a87b18ffc05f9f030ba328ffefba92d6c127917a2ba740f9cfe4a" + url: "https://pub.dev" + source: hosted + version: "1.1.1" + xdg_directories: + dependency: transitive + description: + name: xdg_directories + sha256: "7a3f37b05d989967cdddcbb571f1ea834867ae2faa29725fd085180e0883aa15" + url: "https://pub.dev" + source: hosted + version: "1.1.0" + yaml: + dependency: transitive + description: + name: yaml + sha256: b9da305ac7c39faa3f030eccd175340f968459dae4af175130b3fc47e40d76ce + url: "https://pub.dev" + source: hosted + version: "3.1.3" +sdks: + dart: ">=3.11.0 <4.0.0" + flutter: ">=3.38.4" diff --git a/link/sdks/dart/sync/example/pubspec.yaml b/link/sdks/dart/sync/example/pubspec.yaml new file mode 100644 index 000000000..bc909d78b --- /dev/null +++ b/link/sdks/dart/sync/example/pubspec.yaml @@ -0,0 +1,24 @@ +name: kalam_sync_example +description: Offline-first Flutter chat example for kalam_sync, kalam_sync_generator, and kalam_link. +publish_to: none +version: 0.5.6-rc.0 + +environment: + sdk: ">=3.10.0 <4.0.0" + flutter: ">=3.38.0" + +dependencies: + flutter: + sdk: flutter + kalam_sync: + path: .. + kalam_sync_generator_example: + path: ../../generator/example + +dev_dependencies: + flutter_lints: ^6.0.0 + flutter_test: + sdk: flutter + +flutter: + uses-material-design: true diff --git a/link/sdks/dart/sync/example/pubspec_overrides.yaml b/link/sdks/dart/sync/example/pubspec_overrides.yaml new file mode 100644 index 000000000..1368e80c1 --- /dev/null +++ b/link/sdks/dart/sync/example/pubspec_overrides.yaml @@ -0,0 +1,7 @@ +dependency_overrides: + kalam_link: + path: ../../link + kalam_sync: + path: .. + kalam_sync_generator_example: + path: ../../generator/example diff --git a/link/sdks/dart/sync/lib/drift.dart b/link/sdks/dart/sync/lib/drift.dart new file mode 100644 index 000000000..aa3e6fa5e --- /dev/null +++ b/link/sdks/dart/sync/lib/drift.dart @@ -0,0 +1,7 @@ +/// Drift types commonly used with generated Kalam sync table bindings. +library; + +export 'package:drift/drift.dart'; + +export 'src/database/kalam_sync_database.dart'; +export 'src/store/kalam_sync_store.dart' show KalamClock, KalamSyncStore; diff --git a/link/sdks/dart/sync/lib/kalam_sync.dart b/link/sdks/dart/sync/lib/kalam_sync.dart new file mode 100644 index 000000000..bb5e432ef --- /dev/null +++ b/link/sdks/dart/sync/lib/kalam_sync.dart @@ -0,0 +1,42 @@ +/// Local-first persistence, durable actions, and synchronization for KalamDB. +library; + +export 'package:kalam_link/kalam_link.dart'; + +export 'src/annotations/kalam_action.dart'; +export 'src/annotations/kalam_action_module.dart'; +export 'src/annotations/kalam_action_payload.dart'; +export 'src/actions/kalam_action_codec.dart'; +export 'src/actions/kalam_action_context.dart'; +export 'src/actions/kalam_action_definition.dart'; +export 'src/actions/kalam_dml_action.dart'; +export 'src/actions/kalam_action_registry.dart'; +export 'src/actions/kalam_action_runner.dart'; +export 'src/flutter/kalam_database_factory.dart'; +export 'src/flutter/kalam_scope.dart'; +export 'src/kalam.dart'; +export 'src/models/kalam_account_identity.dart'; +export 'src/models/kalam_action_status.dart'; +export 'src/models/kalam_action_draft.dart'; +export 'src/models/kalam_action_record.dart'; +export 'src/models/kalam_checkpoint.dart'; +export 'src/models/kalam_change.dart'; +export 'src/models/kalam_cached_row.dart'; +export 'src/models/kalam_dml_payload.dart'; +export 'src/models/kalam_optimistic_row.dart'; +export 'src/models/kalam_optimistic_mutation.dart'; +export 'src/models/kalam_row_sync_state.dart'; +export 'src/models/kalam_retry_policy.dart'; +export 'src/models/kalam_sync_mode.dart'; +export 'src/models/kalam_sync_state.dart'; +export 'src/models/kalam_synced_row.dart'; +export 'src/sync/kalam_event_consumer.dart'; +export 'src/sync/kalam_sync_coordinator.dart'; +export 'src/sync/kalam_sync_subscription.dart'; +export 'src/tables/kalam_replica_overlay.dart'; +export 'src/tables/kalam_table_binding.dart'; +export 'src/tables/kalam_table_spec.dart'; +export 'src/transport/kalam_link_transport.dart'; +export 'src/transport/kalam_remote_batch.dart'; +export 'src/transport/kalam_remote_change.dart'; +export 'src/transport/kalam_sync_transport.dart'; diff --git a/link/sdks/dart/sync/lib/src/actions/kalam_action_codec.dart b/link/sdks/dart/sync/lib/src/actions/kalam_action_codec.dart new file mode 100644 index 000000000..7f19d8d72 --- /dev/null +++ b/link/sdks/dart/sync/lib/src/actions/kalam_action_codec.dart @@ -0,0 +1,7 @@ +/// Converts a typed action payload to and from durable JSON values. +final class KalamActionCodec { + const KalamActionCodec({required this.encode, required this.decode}); + + final Map Function(T payload) encode; + final T Function(Map json) decode; +} diff --git a/link/sdks/dart/sync/lib/src/actions/kalam_action_context.dart b/link/sdks/dart/sync/lib/src/actions/kalam_action_context.dart new file mode 100644 index 000000000..fac295435 --- /dev/null +++ b/link/sdks/dart/sync/lib/src/actions/kalam_action_context.dart @@ -0,0 +1,40 @@ +import 'dart:convert'; + +import '../store/kalam_sync_store.dart'; + +/// Runtime values and durable named steps available to an action executor. +final class KalamActionContext { + KalamActionContext({ + required this.actionId, + required this.accountKey, + required this.attempt, + required KalamSyncStore store, + }) : _store = store; + + final String actionId; + final String accountKey; + final int attempt; + final KalamSyncStore _store; + + String get idempotencyKey => actionId; + + /// Runs a sub-operation at most once across retries and process restarts. + Future step( + String name, { + required Future Function(String idempotencyKey) run, + required Object? Function(T value) encode, + required T Function(Object? value) decode, + }) async { + final stored = await _store.readCompletedStep(actionId, name); + if (stored != null) return decode(jsonDecode(stored)); + + try { + final result = await run('$actionId/$name'); + await _store.completeStep(actionId, name, jsonEncode(encode(result))); + return result; + } catch (error) { + await _store.failStep(actionId, name, error.toString()); + rethrow; + } + } +} diff --git a/link/sdks/dart/sync/lib/src/actions/kalam_action_definition.dart b/link/sdks/dart/sync/lib/src/actions/kalam_action_definition.dart new file mode 100644 index 000000000..869842bb9 --- /dev/null +++ b/link/sdks/dart/sync/lib/src/actions/kalam_action_definition.dart @@ -0,0 +1,34 @@ +import 'kalam_action_codec.dart'; +import 'kalam_action_context.dart'; +import '../models/kalam_retry_policy.dart'; + +typedef KalamActionExecutor = + Future Function(KalamActionContext context, T payload); + +/// A typed, versioned durable action registered once by the application. +final class KalamActionDefinition { + const KalamActionDefinition({ + required this.key, + required this.codec, + required this.execute, + this.version = 1, + this.retryPolicy = const KalamRetryPolicy(), + }) : assert(version > 0); + + final String key; + final int version; + final KalamActionCodec codec; + final KalamRetryPolicy retryPolicy; + final KalamActionExecutor execute; + + Map encodePayload(Object? payload) { + return codec.encode(payload as T); + } + + Future executePayload( + KalamActionContext context, + Map json, + ) { + return execute(context, codec.decode(json)); + } +} diff --git a/link/sdks/dart/sync/lib/src/actions/kalam_action_registry.dart b/link/sdks/dart/sync/lib/src/actions/kalam_action_registry.dart new file mode 100644 index 000000000..84b3fce86 --- /dev/null +++ b/link/sdks/dart/sync/lib/src/actions/kalam_action_registry.dart @@ -0,0 +1,40 @@ +import 'kalam_action_definition.dart'; + +/// Immutable lookup of the app's registered action definitions. +final class KalamActionRegistry { + KalamActionRegistry(Iterable> actions) + : _actions = _index(actions); + + factory KalamActionRegistry.of( + Iterable> actions, + ) { + return KalamActionRegistry(actions.cast>()); + } + + final Map> _actions; + + KalamActionDefinition operator [](String key) { + final action = _actions[key]; + if (action == null) { + throw StateError('No Kalam action is registered for "$key".'); + } + return action; + } + + static Map> _index( + Iterable> actions, + ) { + final result = >{}; + for (final action in actions) { + if (result.containsKey(action.key)) { + throw ArgumentError.value( + action.key, + 'actions', + 'Duplicate action key', + ); + } + result[action.key] = action; + } + return Map.unmodifiable(result); + } +} diff --git a/link/sdks/dart/sync/lib/src/actions/kalam_action_runner.dart b/link/sdks/dart/sync/lib/src/actions/kalam_action_runner.dart new file mode 100644 index 000000000..bf995028a --- /dev/null +++ b/link/sdks/dart/sync/lib/src/actions/kalam_action_runner.dart @@ -0,0 +1,117 @@ +import 'dart:convert'; + +import '../models/kalam_action_draft.dart'; +import '../models/kalam_action_record.dart'; +import '../models/kalam_optimistic_row.dart'; +import '../models/kalam_optimistic_mutation.dart'; +import '../models/kalam_retry_policy.dart'; +import '../store/kalam_sync_store.dart'; +import 'kalam_action_context.dart'; +import 'kalam_action_registry.dart'; + +typedef KalamActionClock = DateTime Function(); + +/// Enqueues typed actions and flushes the durable outbox in FIFO order. +final class KalamActionRunner { + KalamActionRunner({ + required this.store, + required this.registry, + required this.accountKey, + KalamActionClock? clock, + }) : _clock = clock ?? DateTime.now; + + final KalamSyncStore store; + final KalamActionRegistry registry; + final String accountKey; + final KalamActionClock _clock; + Future? _activeFlush; + + Future enqueue({ + required String actionKey, + required String actionId, + required Object? payload, + String? orderingKey, + KalamOptimisticRow? optimisticRow, + Future Function()? applyOptimistic, + KalamOptimisticMutation? optimistic, + }) { + if (optimistic != null && + (optimisticRow != null || applyOptimistic != null)) { + throw ArgumentError( + 'Use optimistic or optimisticRow/applyOptimistic, not both.', + ); + } + final definition = registry[actionKey]; + return store.enqueue( + KalamActionDraft( + id: actionId, + accountKey: accountKey, + actionKey: actionKey, + version: definition.version, + payloadJson: jsonEncode(definition.encodePayload(payload)), + orderingKey: orderingKey, + ), + optimisticRow: optimistic?.row ?? optimisticRow, + applyOptimistic: optimistic?.apply ?? applyOptimistic, + ); + } + + /// Flushes currently eligible work. Concurrent calls share the active flush. + Future flush() { + return _activeFlush ??= _flush().whenComplete(() => _activeFlush = null); + } + + Future _flush() async { + var completed = 0; + await store.recoverRunningActions(_clock(), accountKey: accountKey); + while (true) { + final action = await store.claimNextAction( + _clock(), + accountKey: accountKey, + ); + if (action == null) return completed; + await _execute(action); + completed++; + } + } + + Future _execute(KalamActionRecord action) async { + final definition = registry[action.actionKey]; + if (definition.version != action.version) { + await store.failAction( + action.id, + 'Action version ${action.version} is not registered.', + _clock(), + ); + return; + } + + try { + final decoded = jsonDecode(action.payloadJson); + if (decoded is! Map) { + throw const FormatException('Action payload must be a JSON object.'); + } + await definition.executePayload( + KalamActionContext( + actionId: action.id, + accountKey: action.accountKey, + attempt: action.attemptCount, + store: store, + ), + decoded, + ); + await store.completeAction(action.id, _clock()); + } on KalamPermanentActionException catch (error) { + await store.failAction(action.id, error.message, _clock()); + } catch (error) { + if (action.attemptCount >= definition.retryPolicy.maxAttempts) { + await store.failAction(action.id, error.toString(), _clock()); + } else { + final retryAt = _clock().add( + definition.retryPolicy.delayForAttempt(action.attemptCount), + ); + await store.scheduleRetry(action.id, error.toString(), retryAt); + } + } + } +} diff --git a/link/sdks/dart/sync/lib/src/actions/kalam_dml_action.dart b/link/sdks/dart/sync/lib/src/actions/kalam_dml_action.dart new file mode 100644 index 000000000..9b315ef58 --- /dev/null +++ b/link/sdks/dart/sync/lib/src/actions/kalam_dml_action.dart @@ -0,0 +1,110 @@ +import 'package:kalam_link/kalam_link.dart'; + +import '../models/kalam_dml_payload.dart'; +import '../models/kalam_retry_policy.dart'; +import 'kalam_action_codec.dart'; +import 'kalam_action_context.dart'; +import 'kalam_action_definition.dart'; + +const kalamDmlActionKey = 'kalam.dml'; + +KalamActionDefinition kalamDmlAction(KalamClient client) { + return KalamActionDefinition( + key: kalamDmlActionKey, + codec: const KalamActionCodec(encode: _encodeDml, decode: _decodeDml), + execute: (context, payload) => _executeDml(client, context, payload), + ); +} + +Map _encodeDml(KalamDmlPayload payload) => { + 'operation': payload.operation.name, + 'table': payload.tableId, + 'keyColumn': payload.keyColumn, + 'rowKey': payload.rowKey, + 'values': payload.values, +}; + +KalamDmlPayload _decodeDml(Map json) { + final operation = KalamDmlOperation.values.byName( + json['operation']! as String, + ); + final values = json['values']; + if (values != null && values is! Map) { + throw const FormatException('DML values must be a map.'); + } + if (operation != KalamDmlOperation.delete && values == null) { + throw const FormatException('DML values are required for writes.'); + } + final decodedValues = values is Map + ? Map.from(values) + : const {}; + return KalamDmlPayload( + operation: operation, + tableId: json['table']! as String, + keyColumn: json['keyColumn']! as String, + rowKey: json['rowKey'], + values: decodedValues, + ); +} + +Future _executeDml( + KalamClient client, + KalamActionContext context, + KalamDmlPayload payload, +) async { + final table = _identifier(payload.tableId); + final keyColumn = _identifier(payload.keyColumn); + late String sql; + late List params; + + switch (payload.operation) { + case KalamDmlOperation.insert: + final columns = payload.values.keys.map(_identifier).toList(); + if (columns.isEmpty) { + throw const KalamPermanentActionException('Insert values are empty.'); + } + final placeholders = List.generate( + columns.length, + (index) => '\$${index + 1}', + ); + sql = + 'INSERT INTO $table (${columns.join(', ')}) ' + 'VALUES (${placeholders.join(', ')})'; + params = [for (final key in payload.values.keys) payload.values[key]]; + case KalamDmlOperation.update: + final entries = payload.values.entries + .where((entry) => entry.key != payload.keyColumn) + .toList(); + if (entries.isEmpty) { + throw const KalamPermanentActionException('Update values are empty.'); + } + final assignments = [ + for (var index = 0; index < entries.length; index++) + '${_identifier(entries[index].key)} = \$${index + 1}', + ]; + sql = + 'UPDATE $table SET ${assignments.join(', ')} ' + 'WHERE $keyColumn = \$${entries.length + 1}'; + params = [...entries.map((entry) => entry.value), payload.rowKey]; + case KalamDmlOperation.delete: + sql = 'DELETE FROM $table WHERE $keyColumn = \$1'; + params = [payload.rowKey]; + } + + final response = await client.query(sql, params: params); + if (!response.success) { + final error = response.error; + throw KalamPermanentActionException( + error == null ? 'KalamDB rejected the mutation.' : error.toString(), + ); + } +} + +String _identifier(String value) { + final parts = value.split('.'); + final valid = RegExp(r'^[A-Za-z_][A-Za-z0-9_]*$'); + if (parts.isEmpty || parts.any((part) => !valid.hasMatch(part))) { + throw KalamPermanentActionException('Unsafe SQL identifier "$value".'); + } + return parts.join('.'); +} diff --git a/link/sdks/dart/sync/lib/src/annotations/kalam_action.dart b/link/sdks/dart/sync/lib/src/annotations/kalam_action.dart new file mode 100644 index 000000000..a1fab6e9f --- /dev/null +++ b/link/sdks/dart/sync/lib/src/annotations/kalam_action.dart @@ -0,0 +1,7 @@ +/// Marks one module method as a durable offline action. +final class KalamAction { + const KalamAction({required this.name, this.version = 1}); + + final String name; + final int version; +} diff --git a/link/sdks/dart/sync/lib/src/annotations/kalam_action_module.dart b/link/sdks/dart/sync/lib/src/annotations/kalam_action_module.dart new file mode 100644 index 000000000..9c439fd02 --- /dev/null +++ b/link/sdks/dart/sync/lib/src/annotations/kalam_action_module.dart @@ -0,0 +1,6 @@ +/// Groups related generated actions under one stable namespace. +final class KalamActionModule { + const KalamActionModule({required this.namespace}); + + final String namespace; +} diff --git a/link/sdks/dart/sync/lib/src/annotations/kalam_action_payload.dart b/link/sdks/dart/sync/lib/src/annotations/kalam_action_payload.dart new file mode 100644 index 000000000..70fba513f --- /dev/null +++ b/link/sdks/dart/sync/lib/src/annotations/kalam_action_payload.dart @@ -0,0 +1,4 @@ +/// Generates a durable JSON codec for a small immutable action payload. +final class KalamActionPayload { + const KalamActionPayload(); +} diff --git a/link/sdks/dart/sync/lib/src/database/kalam_sync_database.dart b/link/sdks/dart/sync/lib/src/database/kalam_sync_database.dart new file mode 100644 index 000000000..54be96898 --- /dev/null +++ b/link/sdks/dart/sync/lib/src/database/kalam_sync_database.dart @@ -0,0 +1,40 @@ +import 'package:drift/drift.dart'; + +import 'tables/kalam_action_steps.dart'; +import 'tables/kalam_actions.dart'; +import 'tables/kalam_checkpoints.dart'; +import 'tables/kalam_cached_rows.dart'; +import 'tables/kalam_row_states.dart'; + +part 'kalam_sync_database.g.dart'; + +@DriftDatabase( + tables: [ + KalamActions, + KalamCachedRows, + KalamActionSteps, + KalamCheckpoints, + KalamRowStates, + ], +) +class KalamSyncDatabase extends _$KalamSyncDatabase { + KalamSyncDatabase(super.executor); + + @override + int get schemaVersion => 3; + + @override + MigrationStrategy get migration => MigrationStrategy( + onCreate: (migrator) => migrator.createAll(), + onUpgrade: (migrator, from, to) async { + if (from < 2) await migrator.createTable(kalamCachedRows); + if (from < 3) { + await migrator.addColumn(kalamActions, kalamActions.queuePosition); + await customStatement( + 'UPDATE kalam_actions SET queue_position = rowid ' + 'WHERE queue_position = 0', + ); + } + }, + ); +} diff --git a/link/sdks/dart/sync/lib/src/database/kalam_sync_database.g.dart b/link/sdks/dart/sync/lib/src/database/kalam_sync_database.g.dart new file mode 100644 index 000000000..d45fc6330 --- /dev/null +++ b/link/sdks/dart/sync/lib/src/database/kalam_sync_database.g.dart @@ -0,0 +1,4316 @@ +// GENERATED CODE - DO NOT MODIFY BY HAND + +part of 'kalam_sync_database.dart'; + +// ignore_for_file: type=lint +class $KalamActionsTable extends KalamActions + with TableInfo<$KalamActionsTable, StoredAction> { + @override + final GeneratedDatabase attachedDatabase; + final String? _alias; + $KalamActionsTable(this.attachedDatabase, [this._alias]); + static const VerificationMeta _idMeta = const VerificationMeta('id'); + @override + late final GeneratedColumn id = GeneratedColumn( + 'id', + aliasedName, + false, + type: DriftSqlType.string, + requiredDuringInsert: true, + ); + static const VerificationMeta _accountKeyMeta = const VerificationMeta( + 'accountKey', + ); + @override + late final GeneratedColumn accountKey = GeneratedColumn( + 'account_key', + aliasedName, + false, + type: DriftSqlType.string, + requiredDuringInsert: true, + ); + static const VerificationMeta _actionKeyMeta = const VerificationMeta( + 'actionKey', + ); + @override + late final GeneratedColumn actionKey = GeneratedColumn( + 'action_key', + aliasedName, + false, + type: DriftSqlType.string, + requiredDuringInsert: true, + ); + static const VerificationMeta _versionMeta = const VerificationMeta( + 'version', + ); + @override + late final GeneratedColumn version = GeneratedColumn( + 'version', + aliasedName, + false, + type: DriftSqlType.int, + requiredDuringInsert: false, + defaultValue: const Constant(1), + ); + static const VerificationMeta _kindMeta = const VerificationMeta('kind'); + @override + late final GeneratedColumn kind = GeneratedColumn( + 'kind', + aliasedName, + false, + type: DriftSqlType.string, + requiredDuringInsert: false, + defaultValue: const Constant('custom'), + ); + static const VerificationMeta _payloadJsonMeta = const VerificationMeta( + 'payloadJson', + ); + @override + late final GeneratedColumn payloadJson = GeneratedColumn( + 'payload_json', + aliasedName, + false, + type: DriftSqlType.string, + requiredDuringInsert: true, + ); + static const VerificationMeta _statusMeta = const VerificationMeta('status'); + @override + late final GeneratedColumn status = GeneratedColumn( + 'status', + aliasedName, + false, + type: DriftSqlType.string, + requiredDuringInsert: true, + ); + static const VerificationMeta _orderingKeyMeta = const VerificationMeta( + 'orderingKey', + ); + @override + late final GeneratedColumn orderingKey = GeneratedColumn( + 'ordering_key', + aliasedName, + true, + type: DriftSqlType.string, + requiredDuringInsert: false, + ); + static const VerificationMeta _rowTableIdMeta = const VerificationMeta( + 'rowTableId', + ); + @override + late final GeneratedColumn rowTableId = GeneratedColumn( + 'row_table_id', + aliasedName, + true, + type: DriftSqlType.string, + requiredDuringInsert: false, + ); + static const VerificationMeta _rowKeyMeta = const VerificationMeta('rowKey'); + @override + late final GeneratedColumn rowKey = GeneratedColumn( + 'row_key', + aliasedName, + true, + type: DriftSqlType.string, + requiredDuringInsert: false, + ); + static const VerificationMeta _queuePositionMeta = const VerificationMeta( + 'queuePosition', + ); + @override + late final GeneratedColumn queuePosition = GeneratedColumn( + 'queue_position', + aliasedName, + false, + type: DriftSqlType.int, + requiredDuringInsert: false, + defaultValue: const Constant(0), + ); + static const VerificationMeta _attemptCountMeta = const VerificationMeta( + 'attemptCount', + ); + @override + late final GeneratedColumn attemptCount = GeneratedColumn( + 'attempt_count', + aliasedName, + false, + type: DriftSqlType.int, + requiredDuringInsert: false, + defaultValue: const Constant(0), + ); + static const VerificationMeta _nextAttemptAtMeta = const VerificationMeta( + 'nextAttemptAt', + ); + @override + late final GeneratedColumn nextAttemptAt = + GeneratedColumn( + 'next_attempt_at', + aliasedName, + true, + type: DriftSqlType.dateTime, + requiredDuringInsert: false, + ); + static const VerificationMeta _lastErrorMeta = const VerificationMeta( + 'lastError', + ); + @override + late final GeneratedColumn lastError = GeneratedColumn( + 'last_error', + aliasedName, + true, + type: DriftSqlType.string, + requiredDuringInsert: false, + ); + static const VerificationMeta _createdAtMeta = const VerificationMeta( + 'createdAt', + ); + @override + late final GeneratedColumn createdAt = GeneratedColumn( + 'created_at', + aliasedName, + false, + type: DriftSqlType.dateTime, + requiredDuringInsert: true, + ); + static const VerificationMeta _updatedAtMeta = const VerificationMeta( + 'updatedAt', + ); + @override + late final GeneratedColumn updatedAt = GeneratedColumn( + 'updated_at', + aliasedName, + false, + type: DriftSqlType.dateTime, + requiredDuringInsert: true, + ); + @override + List get $columns => [ + id, + accountKey, + actionKey, + version, + kind, + payloadJson, + status, + orderingKey, + rowTableId, + rowKey, + queuePosition, + attemptCount, + nextAttemptAt, + lastError, + createdAt, + updatedAt, + ]; + @override + String get aliasedName => _alias ?? actualTableName; + @override + String get actualTableName => $name; + static const String $name = 'kalam_actions'; + @override + VerificationContext validateIntegrity( + Insertable instance, { + bool isInserting = false, + }) { + final context = VerificationContext(); + final data = instance.toColumns(true); + if (data.containsKey('id')) { + context.handle(_idMeta, id.isAcceptableOrUnknown(data['id']!, _idMeta)); + } else if (isInserting) { + context.missing(_idMeta); + } + if (data.containsKey('account_key')) { + context.handle( + _accountKeyMeta, + accountKey.isAcceptableOrUnknown(data['account_key']!, _accountKeyMeta), + ); + } else if (isInserting) { + context.missing(_accountKeyMeta); + } + if (data.containsKey('action_key')) { + context.handle( + _actionKeyMeta, + actionKey.isAcceptableOrUnknown(data['action_key']!, _actionKeyMeta), + ); + } else if (isInserting) { + context.missing(_actionKeyMeta); + } + if (data.containsKey('version')) { + context.handle( + _versionMeta, + version.isAcceptableOrUnknown(data['version']!, _versionMeta), + ); + } + if (data.containsKey('kind')) { + context.handle( + _kindMeta, + kind.isAcceptableOrUnknown(data['kind']!, _kindMeta), + ); + } + if (data.containsKey('payload_json')) { + context.handle( + _payloadJsonMeta, + payloadJson.isAcceptableOrUnknown( + data['payload_json']!, + _payloadJsonMeta, + ), + ); + } else if (isInserting) { + context.missing(_payloadJsonMeta); + } + if (data.containsKey('status')) { + context.handle( + _statusMeta, + status.isAcceptableOrUnknown(data['status']!, _statusMeta), + ); + } else if (isInserting) { + context.missing(_statusMeta); + } + if (data.containsKey('ordering_key')) { + context.handle( + _orderingKeyMeta, + orderingKey.isAcceptableOrUnknown( + data['ordering_key']!, + _orderingKeyMeta, + ), + ); + } + if (data.containsKey('row_table_id')) { + context.handle( + _rowTableIdMeta, + rowTableId.isAcceptableOrUnknown( + data['row_table_id']!, + _rowTableIdMeta, + ), + ); + } + if (data.containsKey('row_key')) { + context.handle( + _rowKeyMeta, + rowKey.isAcceptableOrUnknown(data['row_key']!, _rowKeyMeta), + ); + } + if (data.containsKey('queue_position')) { + context.handle( + _queuePositionMeta, + queuePosition.isAcceptableOrUnknown( + data['queue_position']!, + _queuePositionMeta, + ), + ); + } + if (data.containsKey('attempt_count')) { + context.handle( + _attemptCountMeta, + attemptCount.isAcceptableOrUnknown( + data['attempt_count']!, + _attemptCountMeta, + ), + ); + } + if (data.containsKey('next_attempt_at')) { + context.handle( + _nextAttemptAtMeta, + nextAttemptAt.isAcceptableOrUnknown( + data['next_attempt_at']!, + _nextAttemptAtMeta, + ), + ); + } + if (data.containsKey('last_error')) { + context.handle( + _lastErrorMeta, + lastError.isAcceptableOrUnknown(data['last_error']!, _lastErrorMeta), + ); + } + if (data.containsKey('created_at')) { + context.handle( + _createdAtMeta, + createdAt.isAcceptableOrUnknown(data['created_at']!, _createdAtMeta), + ); + } else if (isInserting) { + context.missing(_createdAtMeta); + } + if (data.containsKey('updated_at')) { + context.handle( + _updatedAtMeta, + updatedAt.isAcceptableOrUnknown(data['updated_at']!, _updatedAtMeta), + ); + } else if (isInserting) { + context.missing(_updatedAtMeta); + } + return context; + } + + @override + Set get $primaryKey => {id}; + @override + StoredAction map(Map data, {String? tablePrefix}) { + final effectivePrefix = tablePrefix != null ? '$tablePrefix.' : ''; + return StoredAction( + id: attachedDatabase.typeMapping.read( + DriftSqlType.string, + data['${effectivePrefix}id'], + )!, + accountKey: attachedDatabase.typeMapping.read( + DriftSqlType.string, + data['${effectivePrefix}account_key'], + )!, + actionKey: attachedDatabase.typeMapping.read( + DriftSqlType.string, + data['${effectivePrefix}action_key'], + )!, + version: attachedDatabase.typeMapping.read( + DriftSqlType.int, + data['${effectivePrefix}version'], + )!, + kind: attachedDatabase.typeMapping.read( + DriftSqlType.string, + data['${effectivePrefix}kind'], + )!, + payloadJson: attachedDatabase.typeMapping.read( + DriftSqlType.string, + data['${effectivePrefix}payload_json'], + )!, + status: attachedDatabase.typeMapping.read( + DriftSqlType.string, + data['${effectivePrefix}status'], + )!, + orderingKey: attachedDatabase.typeMapping.read( + DriftSqlType.string, + data['${effectivePrefix}ordering_key'], + ), + rowTableId: attachedDatabase.typeMapping.read( + DriftSqlType.string, + data['${effectivePrefix}row_table_id'], + ), + rowKey: attachedDatabase.typeMapping.read( + DriftSqlType.string, + data['${effectivePrefix}row_key'], + ), + queuePosition: attachedDatabase.typeMapping.read( + DriftSqlType.int, + data['${effectivePrefix}queue_position'], + )!, + attemptCount: attachedDatabase.typeMapping.read( + DriftSqlType.int, + data['${effectivePrefix}attempt_count'], + )!, + nextAttemptAt: attachedDatabase.typeMapping.read( + DriftSqlType.dateTime, + data['${effectivePrefix}next_attempt_at'], + ), + lastError: attachedDatabase.typeMapping.read( + DriftSqlType.string, + data['${effectivePrefix}last_error'], + ), + createdAt: attachedDatabase.typeMapping.read( + DriftSqlType.dateTime, + data['${effectivePrefix}created_at'], + )!, + updatedAt: attachedDatabase.typeMapping.read( + DriftSqlType.dateTime, + data['${effectivePrefix}updated_at'], + )!, + ); + } + + @override + $KalamActionsTable createAlias(String alias) { + return $KalamActionsTable(attachedDatabase, alias); + } +} + +class StoredAction extends DataClass implements Insertable { + final String id; + final String accountKey; + final String actionKey; + final int version; + final String kind; + final String payloadJson; + final String status; + final String? orderingKey; + final String? rowTableId; + final String? rowKey; + final int queuePosition; + final int attemptCount; + final DateTime? nextAttemptAt; + final String? lastError; + final DateTime createdAt; + final DateTime updatedAt; + const StoredAction({ + required this.id, + required this.accountKey, + required this.actionKey, + required this.version, + required this.kind, + required this.payloadJson, + required this.status, + this.orderingKey, + this.rowTableId, + this.rowKey, + required this.queuePosition, + required this.attemptCount, + this.nextAttemptAt, + this.lastError, + required this.createdAt, + required this.updatedAt, + }); + @override + Map toColumns(bool nullToAbsent) { + final map = {}; + map['id'] = Variable(id); + map['account_key'] = Variable(accountKey); + map['action_key'] = Variable(actionKey); + map['version'] = Variable(version); + map['kind'] = Variable(kind); + map['payload_json'] = Variable(payloadJson); + map['status'] = Variable(status); + if (!nullToAbsent || orderingKey != null) { + map['ordering_key'] = Variable(orderingKey); + } + if (!nullToAbsent || rowTableId != null) { + map['row_table_id'] = Variable(rowTableId); + } + if (!nullToAbsent || rowKey != null) { + map['row_key'] = Variable(rowKey); + } + map['queue_position'] = Variable(queuePosition); + map['attempt_count'] = Variable(attemptCount); + if (!nullToAbsent || nextAttemptAt != null) { + map['next_attempt_at'] = Variable(nextAttemptAt); + } + if (!nullToAbsent || lastError != null) { + map['last_error'] = Variable(lastError); + } + map['created_at'] = Variable(createdAt); + map['updated_at'] = Variable(updatedAt); + return map; + } + + KalamActionsCompanion toCompanion(bool nullToAbsent) { + return KalamActionsCompanion( + id: Value(id), + accountKey: Value(accountKey), + actionKey: Value(actionKey), + version: Value(version), + kind: Value(kind), + payloadJson: Value(payloadJson), + status: Value(status), + orderingKey: orderingKey == null && nullToAbsent + ? const Value.absent() + : Value(orderingKey), + rowTableId: rowTableId == null && nullToAbsent + ? const Value.absent() + : Value(rowTableId), + rowKey: rowKey == null && nullToAbsent + ? const Value.absent() + : Value(rowKey), + queuePosition: Value(queuePosition), + attemptCount: Value(attemptCount), + nextAttemptAt: nextAttemptAt == null && nullToAbsent + ? const Value.absent() + : Value(nextAttemptAt), + lastError: lastError == null && nullToAbsent + ? const Value.absent() + : Value(lastError), + createdAt: Value(createdAt), + updatedAt: Value(updatedAt), + ); + } + + factory StoredAction.fromJson( + Map json, { + ValueSerializer? serializer, + }) { + serializer ??= driftRuntimeOptions.defaultSerializer; + return StoredAction( + id: serializer.fromJson(json['id']), + accountKey: serializer.fromJson(json['accountKey']), + actionKey: serializer.fromJson(json['actionKey']), + version: serializer.fromJson(json['version']), + kind: serializer.fromJson(json['kind']), + payloadJson: serializer.fromJson(json['payloadJson']), + status: serializer.fromJson(json['status']), + orderingKey: serializer.fromJson(json['orderingKey']), + rowTableId: serializer.fromJson(json['rowTableId']), + rowKey: serializer.fromJson(json['rowKey']), + queuePosition: serializer.fromJson(json['queuePosition']), + attemptCount: serializer.fromJson(json['attemptCount']), + nextAttemptAt: serializer.fromJson(json['nextAttemptAt']), + lastError: serializer.fromJson(json['lastError']), + createdAt: serializer.fromJson(json['createdAt']), + updatedAt: serializer.fromJson(json['updatedAt']), + ); + } + @override + Map toJson({ValueSerializer? serializer}) { + serializer ??= driftRuntimeOptions.defaultSerializer; + return { + 'id': serializer.toJson(id), + 'accountKey': serializer.toJson(accountKey), + 'actionKey': serializer.toJson(actionKey), + 'version': serializer.toJson(version), + 'kind': serializer.toJson(kind), + 'payloadJson': serializer.toJson(payloadJson), + 'status': serializer.toJson(status), + 'orderingKey': serializer.toJson(orderingKey), + 'rowTableId': serializer.toJson(rowTableId), + 'rowKey': serializer.toJson(rowKey), + 'queuePosition': serializer.toJson(queuePosition), + 'attemptCount': serializer.toJson(attemptCount), + 'nextAttemptAt': serializer.toJson(nextAttemptAt), + 'lastError': serializer.toJson(lastError), + 'createdAt': serializer.toJson(createdAt), + 'updatedAt': serializer.toJson(updatedAt), + }; + } + + StoredAction copyWith({ + String? id, + String? accountKey, + String? actionKey, + int? version, + String? kind, + String? payloadJson, + String? status, + Value orderingKey = const Value.absent(), + Value rowTableId = const Value.absent(), + Value rowKey = const Value.absent(), + int? queuePosition, + int? attemptCount, + Value nextAttemptAt = const Value.absent(), + Value lastError = const Value.absent(), + DateTime? createdAt, + DateTime? updatedAt, + }) => StoredAction( + id: id ?? this.id, + accountKey: accountKey ?? this.accountKey, + actionKey: actionKey ?? this.actionKey, + version: version ?? this.version, + kind: kind ?? this.kind, + payloadJson: payloadJson ?? this.payloadJson, + status: status ?? this.status, + orderingKey: orderingKey.present ? orderingKey.value : this.orderingKey, + rowTableId: rowTableId.present ? rowTableId.value : this.rowTableId, + rowKey: rowKey.present ? rowKey.value : this.rowKey, + queuePosition: queuePosition ?? this.queuePosition, + attemptCount: attemptCount ?? this.attemptCount, + nextAttemptAt: nextAttemptAt.present + ? nextAttemptAt.value + : this.nextAttemptAt, + lastError: lastError.present ? lastError.value : this.lastError, + createdAt: createdAt ?? this.createdAt, + updatedAt: updatedAt ?? this.updatedAt, + ); + StoredAction copyWithCompanion(KalamActionsCompanion data) { + return StoredAction( + id: data.id.present ? data.id.value : this.id, + accountKey: data.accountKey.present + ? data.accountKey.value + : this.accountKey, + actionKey: data.actionKey.present ? data.actionKey.value : this.actionKey, + version: data.version.present ? data.version.value : this.version, + kind: data.kind.present ? data.kind.value : this.kind, + payloadJson: data.payloadJson.present + ? data.payloadJson.value + : this.payloadJson, + status: data.status.present ? data.status.value : this.status, + orderingKey: data.orderingKey.present + ? data.orderingKey.value + : this.orderingKey, + rowTableId: data.rowTableId.present + ? data.rowTableId.value + : this.rowTableId, + rowKey: data.rowKey.present ? data.rowKey.value : this.rowKey, + queuePosition: data.queuePosition.present + ? data.queuePosition.value + : this.queuePosition, + attemptCount: data.attemptCount.present + ? data.attemptCount.value + : this.attemptCount, + nextAttemptAt: data.nextAttemptAt.present + ? data.nextAttemptAt.value + : this.nextAttemptAt, + lastError: data.lastError.present ? data.lastError.value : this.lastError, + createdAt: data.createdAt.present ? data.createdAt.value : this.createdAt, + updatedAt: data.updatedAt.present ? data.updatedAt.value : this.updatedAt, + ); + } + + @override + String toString() { + return (StringBuffer('StoredAction(') + ..write('id: $id, ') + ..write('accountKey: $accountKey, ') + ..write('actionKey: $actionKey, ') + ..write('version: $version, ') + ..write('kind: $kind, ') + ..write('payloadJson: $payloadJson, ') + ..write('status: $status, ') + ..write('orderingKey: $orderingKey, ') + ..write('rowTableId: $rowTableId, ') + ..write('rowKey: $rowKey, ') + ..write('queuePosition: $queuePosition, ') + ..write('attemptCount: $attemptCount, ') + ..write('nextAttemptAt: $nextAttemptAt, ') + ..write('lastError: $lastError, ') + ..write('createdAt: $createdAt, ') + ..write('updatedAt: $updatedAt') + ..write(')')) + .toString(); + } + + @override + int get hashCode => Object.hash( + id, + accountKey, + actionKey, + version, + kind, + payloadJson, + status, + orderingKey, + rowTableId, + rowKey, + queuePosition, + attemptCount, + nextAttemptAt, + lastError, + createdAt, + updatedAt, + ); + @override + bool operator ==(Object other) => + identical(this, other) || + (other is StoredAction && + other.id == this.id && + other.accountKey == this.accountKey && + other.actionKey == this.actionKey && + other.version == this.version && + other.kind == this.kind && + other.payloadJson == this.payloadJson && + other.status == this.status && + other.orderingKey == this.orderingKey && + other.rowTableId == this.rowTableId && + other.rowKey == this.rowKey && + other.queuePosition == this.queuePosition && + other.attemptCount == this.attemptCount && + other.nextAttemptAt == this.nextAttemptAt && + other.lastError == this.lastError && + other.createdAt == this.createdAt && + other.updatedAt == this.updatedAt); +} + +class KalamActionsCompanion extends UpdateCompanion { + final Value id; + final Value accountKey; + final Value actionKey; + final Value version; + final Value kind; + final Value payloadJson; + final Value status; + final Value orderingKey; + final Value rowTableId; + final Value rowKey; + final Value queuePosition; + final Value attemptCount; + final Value nextAttemptAt; + final Value lastError; + final Value createdAt; + final Value updatedAt; + final Value rowid; + const KalamActionsCompanion({ + this.id = const Value.absent(), + this.accountKey = const Value.absent(), + this.actionKey = const Value.absent(), + this.version = const Value.absent(), + this.kind = const Value.absent(), + this.payloadJson = const Value.absent(), + this.status = const Value.absent(), + this.orderingKey = const Value.absent(), + this.rowTableId = const Value.absent(), + this.rowKey = const Value.absent(), + this.queuePosition = const Value.absent(), + this.attemptCount = const Value.absent(), + this.nextAttemptAt = const Value.absent(), + this.lastError = const Value.absent(), + this.createdAt = const Value.absent(), + this.updatedAt = const Value.absent(), + this.rowid = const Value.absent(), + }); + KalamActionsCompanion.insert({ + required String id, + required String accountKey, + required String actionKey, + this.version = const Value.absent(), + this.kind = const Value.absent(), + required String payloadJson, + required String status, + this.orderingKey = const Value.absent(), + this.rowTableId = const Value.absent(), + this.rowKey = const Value.absent(), + this.queuePosition = const Value.absent(), + this.attemptCount = const Value.absent(), + this.nextAttemptAt = const Value.absent(), + this.lastError = const Value.absent(), + required DateTime createdAt, + required DateTime updatedAt, + this.rowid = const Value.absent(), + }) : id = Value(id), + accountKey = Value(accountKey), + actionKey = Value(actionKey), + payloadJson = Value(payloadJson), + status = Value(status), + createdAt = Value(createdAt), + updatedAt = Value(updatedAt); + static Insertable custom({ + Expression? id, + Expression? accountKey, + Expression? actionKey, + Expression? version, + Expression? kind, + Expression? payloadJson, + Expression? status, + Expression? orderingKey, + Expression? rowTableId, + Expression? rowKey, + Expression? queuePosition, + Expression? attemptCount, + Expression? nextAttemptAt, + Expression? lastError, + Expression? createdAt, + Expression? updatedAt, + Expression? rowid, + }) { + return RawValuesInsertable({ + if (id != null) 'id': id, + if (accountKey != null) 'account_key': accountKey, + if (actionKey != null) 'action_key': actionKey, + if (version != null) 'version': version, + if (kind != null) 'kind': kind, + if (payloadJson != null) 'payload_json': payloadJson, + if (status != null) 'status': status, + if (orderingKey != null) 'ordering_key': orderingKey, + if (rowTableId != null) 'row_table_id': rowTableId, + if (rowKey != null) 'row_key': rowKey, + if (queuePosition != null) 'queue_position': queuePosition, + if (attemptCount != null) 'attempt_count': attemptCount, + if (nextAttemptAt != null) 'next_attempt_at': nextAttemptAt, + if (lastError != null) 'last_error': lastError, + if (createdAt != null) 'created_at': createdAt, + if (updatedAt != null) 'updated_at': updatedAt, + if (rowid != null) 'rowid': rowid, + }); + } + + KalamActionsCompanion copyWith({ + Value? id, + Value? accountKey, + Value? actionKey, + Value? version, + Value? kind, + Value? payloadJson, + Value? status, + Value? orderingKey, + Value? rowTableId, + Value? rowKey, + Value? queuePosition, + Value? attemptCount, + Value? nextAttemptAt, + Value? lastError, + Value? createdAt, + Value? updatedAt, + Value? rowid, + }) { + return KalamActionsCompanion( + id: id ?? this.id, + accountKey: accountKey ?? this.accountKey, + actionKey: actionKey ?? this.actionKey, + version: version ?? this.version, + kind: kind ?? this.kind, + payloadJson: payloadJson ?? this.payloadJson, + status: status ?? this.status, + orderingKey: orderingKey ?? this.orderingKey, + rowTableId: rowTableId ?? this.rowTableId, + rowKey: rowKey ?? this.rowKey, + queuePosition: queuePosition ?? this.queuePosition, + attemptCount: attemptCount ?? this.attemptCount, + nextAttemptAt: nextAttemptAt ?? this.nextAttemptAt, + lastError: lastError ?? this.lastError, + createdAt: createdAt ?? this.createdAt, + updatedAt: updatedAt ?? this.updatedAt, + rowid: rowid ?? this.rowid, + ); + } + + @override + Map toColumns(bool nullToAbsent) { + final map = {}; + if (id.present) { + map['id'] = Variable(id.value); + } + if (accountKey.present) { + map['account_key'] = Variable(accountKey.value); + } + if (actionKey.present) { + map['action_key'] = Variable(actionKey.value); + } + if (version.present) { + map['version'] = Variable(version.value); + } + if (kind.present) { + map['kind'] = Variable(kind.value); + } + if (payloadJson.present) { + map['payload_json'] = Variable(payloadJson.value); + } + if (status.present) { + map['status'] = Variable(status.value); + } + if (orderingKey.present) { + map['ordering_key'] = Variable(orderingKey.value); + } + if (rowTableId.present) { + map['row_table_id'] = Variable(rowTableId.value); + } + if (rowKey.present) { + map['row_key'] = Variable(rowKey.value); + } + if (queuePosition.present) { + map['queue_position'] = Variable(queuePosition.value); + } + if (attemptCount.present) { + map['attempt_count'] = Variable(attemptCount.value); + } + if (nextAttemptAt.present) { + map['next_attempt_at'] = Variable(nextAttemptAt.value); + } + if (lastError.present) { + map['last_error'] = Variable(lastError.value); + } + if (createdAt.present) { + map['created_at'] = Variable(createdAt.value); + } + if (updatedAt.present) { + map['updated_at'] = Variable(updatedAt.value); + } + if (rowid.present) { + map['rowid'] = Variable(rowid.value); + } + return map; + } + + @override + String toString() { + return (StringBuffer('KalamActionsCompanion(') + ..write('id: $id, ') + ..write('accountKey: $accountKey, ') + ..write('actionKey: $actionKey, ') + ..write('version: $version, ') + ..write('kind: $kind, ') + ..write('payloadJson: $payloadJson, ') + ..write('status: $status, ') + ..write('orderingKey: $orderingKey, ') + ..write('rowTableId: $rowTableId, ') + ..write('rowKey: $rowKey, ') + ..write('queuePosition: $queuePosition, ') + ..write('attemptCount: $attemptCount, ') + ..write('nextAttemptAt: $nextAttemptAt, ') + ..write('lastError: $lastError, ') + ..write('createdAt: $createdAt, ') + ..write('updatedAt: $updatedAt, ') + ..write('rowid: $rowid') + ..write(')')) + .toString(); + } +} + +class $KalamCachedRowsTable extends KalamCachedRows + with TableInfo<$KalamCachedRowsTable, StoredCachedRow> { + @override + final GeneratedDatabase attachedDatabase; + final String? _alias; + $KalamCachedRowsTable(this.attachedDatabase, [this._alias]); + static const VerificationMeta _accountKeyMeta = const VerificationMeta( + 'accountKey', + ); + @override + late final GeneratedColumn accountKey = GeneratedColumn( + 'account_key', + aliasedName, + false, + type: DriftSqlType.string, + requiredDuringInsert: true, + ); + static const VerificationMeta _tableIdMeta = const VerificationMeta( + 'tableId', + ); + @override + late final GeneratedColumn tableId = GeneratedColumn( + 'table_id', + aliasedName, + false, + type: DriftSqlType.string, + requiredDuringInsert: true, + ); + static const VerificationMeta _rowKeyMeta = const VerificationMeta('rowKey'); + @override + late final GeneratedColumn rowKey = GeneratedColumn( + 'row_key', + aliasedName, + false, + type: DriftSqlType.string, + requiredDuringInsert: true, + ); + static const VerificationMeta _valuesJsonMeta = const VerificationMeta( + 'valuesJson', + ); + @override + late final GeneratedColumn valuesJson = GeneratedColumn( + 'values_json', + aliasedName, + false, + type: DriftSqlType.string, + requiredDuringInsert: true, + ); + static const VerificationMeta _updatedAtMeta = const VerificationMeta( + 'updatedAt', + ); + @override + late final GeneratedColumn updatedAt = GeneratedColumn( + 'updated_at', + aliasedName, + false, + type: DriftSqlType.dateTime, + requiredDuringInsert: true, + ); + @override + List get $columns => [ + accountKey, + tableId, + rowKey, + valuesJson, + updatedAt, + ]; + @override + String get aliasedName => _alias ?? actualTableName; + @override + String get actualTableName => $name; + static const String $name = 'kalam_cached_rows'; + @override + VerificationContext validateIntegrity( + Insertable instance, { + bool isInserting = false, + }) { + final context = VerificationContext(); + final data = instance.toColumns(true); + if (data.containsKey('account_key')) { + context.handle( + _accountKeyMeta, + accountKey.isAcceptableOrUnknown(data['account_key']!, _accountKeyMeta), + ); + } else if (isInserting) { + context.missing(_accountKeyMeta); + } + if (data.containsKey('table_id')) { + context.handle( + _tableIdMeta, + tableId.isAcceptableOrUnknown(data['table_id']!, _tableIdMeta), + ); + } else if (isInserting) { + context.missing(_tableIdMeta); + } + if (data.containsKey('row_key')) { + context.handle( + _rowKeyMeta, + rowKey.isAcceptableOrUnknown(data['row_key']!, _rowKeyMeta), + ); + } else if (isInserting) { + context.missing(_rowKeyMeta); + } + if (data.containsKey('values_json')) { + context.handle( + _valuesJsonMeta, + valuesJson.isAcceptableOrUnknown(data['values_json']!, _valuesJsonMeta), + ); + } else if (isInserting) { + context.missing(_valuesJsonMeta); + } + if (data.containsKey('updated_at')) { + context.handle( + _updatedAtMeta, + updatedAt.isAcceptableOrUnknown(data['updated_at']!, _updatedAtMeta), + ); + } else if (isInserting) { + context.missing(_updatedAtMeta); + } + return context; + } + + @override + Set get $primaryKey => {accountKey, tableId, rowKey}; + @override + StoredCachedRow map(Map data, {String? tablePrefix}) { + final effectivePrefix = tablePrefix != null ? '$tablePrefix.' : ''; + return StoredCachedRow( + accountKey: attachedDatabase.typeMapping.read( + DriftSqlType.string, + data['${effectivePrefix}account_key'], + )!, + tableId: attachedDatabase.typeMapping.read( + DriftSqlType.string, + data['${effectivePrefix}table_id'], + )!, + rowKey: attachedDatabase.typeMapping.read( + DriftSqlType.string, + data['${effectivePrefix}row_key'], + )!, + valuesJson: attachedDatabase.typeMapping.read( + DriftSqlType.string, + data['${effectivePrefix}values_json'], + )!, + updatedAt: attachedDatabase.typeMapping.read( + DriftSqlType.dateTime, + data['${effectivePrefix}updated_at'], + )!, + ); + } + + @override + $KalamCachedRowsTable createAlias(String alias) { + return $KalamCachedRowsTable(attachedDatabase, alias); + } +} + +class StoredCachedRow extends DataClass implements Insertable { + final String accountKey; + final String tableId; + final String rowKey; + final String valuesJson; + final DateTime updatedAt; + const StoredCachedRow({ + required this.accountKey, + required this.tableId, + required this.rowKey, + required this.valuesJson, + required this.updatedAt, + }); + @override + Map toColumns(bool nullToAbsent) { + final map = {}; + map['account_key'] = Variable(accountKey); + map['table_id'] = Variable(tableId); + map['row_key'] = Variable(rowKey); + map['values_json'] = Variable(valuesJson); + map['updated_at'] = Variable(updatedAt); + return map; + } + + KalamCachedRowsCompanion toCompanion(bool nullToAbsent) { + return KalamCachedRowsCompanion( + accountKey: Value(accountKey), + tableId: Value(tableId), + rowKey: Value(rowKey), + valuesJson: Value(valuesJson), + updatedAt: Value(updatedAt), + ); + } + + factory StoredCachedRow.fromJson( + Map json, { + ValueSerializer? serializer, + }) { + serializer ??= driftRuntimeOptions.defaultSerializer; + return StoredCachedRow( + accountKey: serializer.fromJson(json['accountKey']), + tableId: serializer.fromJson(json['tableId']), + rowKey: serializer.fromJson(json['rowKey']), + valuesJson: serializer.fromJson(json['valuesJson']), + updatedAt: serializer.fromJson(json['updatedAt']), + ); + } + @override + Map toJson({ValueSerializer? serializer}) { + serializer ??= driftRuntimeOptions.defaultSerializer; + return { + 'accountKey': serializer.toJson(accountKey), + 'tableId': serializer.toJson(tableId), + 'rowKey': serializer.toJson(rowKey), + 'valuesJson': serializer.toJson(valuesJson), + 'updatedAt': serializer.toJson(updatedAt), + }; + } + + StoredCachedRow copyWith({ + String? accountKey, + String? tableId, + String? rowKey, + String? valuesJson, + DateTime? updatedAt, + }) => StoredCachedRow( + accountKey: accountKey ?? this.accountKey, + tableId: tableId ?? this.tableId, + rowKey: rowKey ?? this.rowKey, + valuesJson: valuesJson ?? this.valuesJson, + updatedAt: updatedAt ?? this.updatedAt, + ); + StoredCachedRow copyWithCompanion(KalamCachedRowsCompanion data) { + return StoredCachedRow( + accountKey: data.accountKey.present + ? data.accountKey.value + : this.accountKey, + tableId: data.tableId.present ? data.tableId.value : this.tableId, + rowKey: data.rowKey.present ? data.rowKey.value : this.rowKey, + valuesJson: data.valuesJson.present + ? data.valuesJson.value + : this.valuesJson, + updatedAt: data.updatedAt.present ? data.updatedAt.value : this.updatedAt, + ); + } + + @override + String toString() { + return (StringBuffer('StoredCachedRow(') + ..write('accountKey: $accountKey, ') + ..write('tableId: $tableId, ') + ..write('rowKey: $rowKey, ') + ..write('valuesJson: $valuesJson, ') + ..write('updatedAt: $updatedAt') + ..write(')')) + .toString(); + } + + @override + int get hashCode => + Object.hash(accountKey, tableId, rowKey, valuesJson, updatedAt); + @override + bool operator ==(Object other) => + identical(this, other) || + (other is StoredCachedRow && + other.accountKey == this.accountKey && + other.tableId == this.tableId && + other.rowKey == this.rowKey && + other.valuesJson == this.valuesJson && + other.updatedAt == this.updatedAt); +} + +class KalamCachedRowsCompanion extends UpdateCompanion { + final Value accountKey; + final Value tableId; + final Value rowKey; + final Value valuesJson; + final Value updatedAt; + final Value rowid; + const KalamCachedRowsCompanion({ + this.accountKey = const Value.absent(), + this.tableId = const Value.absent(), + this.rowKey = const Value.absent(), + this.valuesJson = const Value.absent(), + this.updatedAt = const Value.absent(), + this.rowid = const Value.absent(), + }); + KalamCachedRowsCompanion.insert({ + required String accountKey, + required String tableId, + required String rowKey, + required String valuesJson, + required DateTime updatedAt, + this.rowid = const Value.absent(), + }) : accountKey = Value(accountKey), + tableId = Value(tableId), + rowKey = Value(rowKey), + valuesJson = Value(valuesJson), + updatedAt = Value(updatedAt); + static Insertable custom({ + Expression? accountKey, + Expression? tableId, + Expression? rowKey, + Expression? valuesJson, + Expression? updatedAt, + Expression? rowid, + }) { + return RawValuesInsertable({ + if (accountKey != null) 'account_key': accountKey, + if (tableId != null) 'table_id': tableId, + if (rowKey != null) 'row_key': rowKey, + if (valuesJson != null) 'values_json': valuesJson, + if (updatedAt != null) 'updated_at': updatedAt, + if (rowid != null) 'rowid': rowid, + }); + } + + KalamCachedRowsCompanion copyWith({ + Value? accountKey, + Value? tableId, + Value? rowKey, + Value? valuesJson, + Value? updatedAt, + Value? rowid, + }) { + return KalamCachedRowsCompanion( + accountKey: accountKey ?? this.accountKey, + tableId: tableId ?? this.tableId, + rowKey: rowKey ?? this.rowKey, + valuesJson: valuesJson ?? this.valuesJson, + updatedAt: updatedAt ?? this.updatedAt, + rowid: rowid ?? this.rowid, + ); + } + + @override + Map toColumns(bool nullToAbsent) { + final map = {}; + if (accountKey.present) { + map['account_key'] = Variable(accountKey.value); + } + if (tableId.present) { + map['table_id'] = Variable(tableId.value); + } + if (rowKey.present) { + map['row_key'] = Variable(rowKey.value); + } + if (valuesJson.present) { + map['values_json'] = Variable(valuesJson.value); + } + if (updatedAt.present) { + map['updated_at'] = Variable(updatedAt.value); + } + if (rowid.present) { + map['rowid'] = Variable(rowid.value); + } + return map; + } + + @override + String toString() { + return (StringBuffer('KalamCachedRowsCompanion(') + ..write('accountKey: $accountKey, ') + ..write('tableId: $tableId, ') + ..write('rowKey: $rowKey, ') + ..write('valuesJson: $valuesJson, ') + ..write('updatedAt: $updatedAt, ') + ..write('rowid: $rowid') + ..write(')')) + .toString(); + } +} + +class $KalamActionStepsTable extends KalamActionSteps + with TableInfo<$KalamActionStepsTable, StoredActionStep> { + @override + final GeneratedDatabase attachedDatabase; + final String? _alias; + $KalamActionStepsTable(this.attachedDatabase, [this._alias]); + static const VerificationMeta _actionIdMeta = const VerificationMeta( + 'actionId', + ); + @override + late final GeneratedColumn actionId = GeneratedColumn( + 'action_id', + aliasedName, + false, + type: DriftSqlType.string, + requiredDuringInsert: true, + ); + static const VerificationMeta _nameMeta = const VerificationMeta('name'); + @override + late final GeneratedColumn name = GeneratedColumn( + 'name', + aliasedName, + false, + type: DriftSqlType.string, + requiredDuringInsert: true, + ); + static const VerificationMeta _statusMeta = const VerificationMeta('status'); + @override + late final GeneratedColumn status = GeneratedColumn( + 'status', + aliasedName, + false, + type: DriftSqlType.string, + requiredDuringInsert: true, + ); + static const VerificationMeta _resultJsonMeta = const VerificationMeta( + 'resultJson', + ); + @override + late final GeneratedColumn resultJson = GeneratedColumn( + 'result_json', + aliasedName, + true, + type: DriftSqlType.string, + requiredDuringInsert: false, + ); + static const VerificationMeta _lastErrorMeta = const VerificationMeta( + 'lastError', + ); + @override + late final GeneratedColumn lastError = GeneratedColumn( + 'last_error', + aliasedName, + true, + type: DriftSqlType.string, + requiredDuringInsert: false, + ); + static const VerificationMeta _updatedAtMeta = const VerificationMeta( + 'updatedAt', + ); + @override + late final GeneratedColumn updatedAt = GeneratedColumn( + 'updated_at', + aliasedName, + false, + type: DriftSqlType.dateTime, + requiredDuringInsert: true, + ); + @override + List get $columns => [ + actionId, + name, + status, + resultJson, + lastError, + updatedAt, + ]; + @override + String get aliasedName => _alias ?? actualTableName; + @override + String get actualTableName => $name; + static const String $name = 'kalam_action_steps'; + @override + VerificationContext validateIntegrity( + Insertable instance, { + bool isInserting = false, + }) { + final context = VerificationContext(); + final data = instance.toColumns(true); + if (data.containsKey('action_id')) { + context.handle( + _actionIdMeta, + actionId.isAcceptableOrUnknown(data['action_id']!, _actionIdMeta), + ); + } else if (isInserting) { + context.missing(_actionIdMeta); + } + if (data.containsKey('name')) { + context.handle( + _nameMeta, + name.isAcceptableOrUnknown(data['name']!, _nameMeta), + ); + } else if (isInserting) { + context.missing(_nameMeta); + } + if (data.containsKey('status')) { + context.handle( + _statusMeta, + status.isAcceptableOrUnknown(data['status']!, _statusMeta), + ); + } else if (isInserting) { + context.missing(_statusMeta); + } + if (data.containsKey('result_json')) { + context.handle( + _resultJsonMeta, + resultJson.isAcceptableOrUnknown(data['result_json']!, _resultJsonMeta), + ); + } + if (data.containsKey('last_error')) { + context.handle( + _lastErrorMeta, + lastError.isAcceptableOrUnknown(data['last_error']!, _lastErrorMeta), + ); + } + if (data.containsKey('updated_at')) { + context.handle( + _updatedAtMeta, + updatedAt.isAcceptableOrUnknown(data['updated_at']!, _updatedAtMeta), + ); + } else if (isInserting) { + context.missing(_updatedAtMeta); + } + return context; + } + + @override + Set get $primaryKey => {actionId, name}; + @override + StoredActionStep map(Map data, {String? tablePrefix}) { + final effectivePrefix = tablePrefix != null ? '$tablePrefix.' : ''; + return StoredActionStep( + actionId: attachedDatabase.typeMapping.read( + DriftSqlType.string, + data['${effectivePrefix}action_id'], + )!, + name: attachedDatabase.typeMapping.read( + DriftSqlType.string, + data['${effectivePrefix}name'], + )!, + status: attachedDatabase.typeMapping.read( + DriftSqlType.string, + data['${effectivePrefix}status'], + )!, + resultJson: attachedDatabase.typeMapping.read( + DriftSqlType.string, + data['${effectivePrefix}result_json'], + ), + lastError: attachedDatabase.typeMapping.read( + DriftSqlType.string, + data['${effectivePrefix}last_error'], + ), + updatedAt: attachedDatabase.typeMapping.read( + DriftSqlType.dateTime, + data['${effectivePrefix}updated_at'], + )!, + ); + } + + @override + $KalamActionStepsTable createAlias(String alias) { + return $KalamActionStepsTable(attachedDatabase, alias); + } +} + +class StoredActionStep extends DataClass + implements Insertable { + final String actionId; + final String name; + final String status; + final String? resultJson; + final String? lastError; + final DateTime updatedAt; + const StoredActionStep({ + required this.actionId, + required this.name, + required this.status, + this.resultJson, + this.lastError, + required this.updatedAt, + }); + @override + Map toColumns(bool nullToAbsent) { + final map = {}; + map['action_id'] = Variable(actionId); + map['name'] = Variable(name); + map['status'] = Variable(status); + if (!nullToAbsent || resultJson != null) { + map['result_json'] = Variable(resultJson); + } + if (!nullToAbsent || lastError != null) { + map['last_error'] = Variable(lastError); + } + map['updated_at'] = Variable(updatedAt); + return map; + } + + KalamActionStepsCompanion toCompanion(bool nullToAbsent) { + return KalamActionStepsCompanion( + actionId: Value(actionId), + name: Value(name), + status: Value(status), + resultJson: resultJson == null && nullToAbsent + ? const Value.absent() + : Value(resultJson), + lastError: lastError == null && nullToAbsent + ? const Value.absent() + : Value(lastError), + updatedAt: Value(updatedAt), + ); + } + + factory StoredActionStep.fromJson( + Map json, { + ValueSerializer? serializer, + }) { + serializer ??= driftRuntimeOptions.defaultSerializer; + return StoredActionStep( + actionId: serializer.fromJson(json['actionId']), + name: serializer.fromJson(json['name']), + status: serializer.fromJson(json['status']), + resultJson: serializer.fromJson(json['resultJson']), + lastError: serializer.fromJson(json['lastError']), + updatedAt: serializer.fromJson(json['updatedAt']), + ); + } + @override + Map toJson({ValueSerializer? serializer}) { + serializer ??= driftRuntimeOptions.defaultSerializer; + return { + 'actionId': serializer.toJson(actionId), + 'name': serializer.toJson(name), + 'status': serializer.toJson(status), + 'resultJson': serializer.toJson(resultJson), + 'lastError': serializer.toJson(lastError), + 'updatedAt': serializer.toJson(updatedAt), + }; + } + + StoredActionStep copyWith({ + String? actionId, + String? name, + String? status, + Value resultJson = const Value.absent(), + Value lastError = const Value.absent(), + DateTime? updatedAt, + }) => StoredActionStep( + actionId: actionId ?? this.actionId, + name: name ?? this.name, + status: status ?? this.status, + resultJson: resultJson.present ? resultJson.value : this.resultJson, + lastError: lastError.present ? lastError.value : this.lastError, + updatedAt: updatedAt ?? this.updatedAt, + ); + StoredActionStep copyWithCompanion(KalamActionStepsCompanion data) { + return StoredActionStep( + actionId: data.actionId.present ? data.actionId.value : this.actionId, + name: data.name.present ? data.name.value : this.name, + status: data.status.present ? data.status.value : this.status, + resultJson: data.resultJson.present + ? data.resultJson.value + : this.resultJson, + lastError: data.lastError.present ? data.lastError.value : this.lastError, + updatedAt: data.updatedAt.present ? data.updatedAt.value : this.updatedAt, + ); + } + + @override + String toString() { + return (StringBuffer('StoredActionStep(') + ..write('actionId: $actionId, ') + ..write('name: $name, ') + ..write('status: $status, ') + ..write('resultJson: $resultJson, ') + ..write('lastError: $lastError, ') + ..write('updatedAt: $updatedAt') + ..write(')')) + .toString(); + } + + @override + int get hashCode => + Object.hash(actionId, name, status, resultJson, lastError, updatedAt); + @override + bool operator ==(Object other) => + identical(this, other) || + (other is StoredActionStep && + other.actionId == this.actionId && + other.name == this.name && + other.status == this.status && + other.resultJson == this.resultJson && + other.lastError == this.lastError && + other.updatedAt == this.updatedAt); +} + +class KalamActionStepsCompanion extends UpdateCompanion { + final Value actionId; + final Value name; + final Value status; + final Value resultJson; + final Value lastError; + final Value updatedAt; + final Value rowid; + const KalamActionStepsCompanion({ + this.actionId = const Value.absent(), + this.name = const Value.absent(), + this.status = const Value.absent(), + this.resultJson = const Value.absent(), + this.lastError = const Value.absent(), + this.updatedAt = const Value.absent(), + this.rowid = const Value.absent(), + }); + KalamActionStepsCompanion.insert({ + required String actionId, + required String name, + required String status, + this.resultJson = const Value.absent(), + this.lastError = const Value.absent(), + required DateTime updatedAt, + this.rowid = const Value.absent(), + }) : actionId = Value(actionId), + name = Value(name), + status = Value(status), + updatedAt = Value(updatedAt); + static Insertable custom({ + Expression? actionId, + Expression? name, + Expression? status, + Expression? resultJson, + Expression? lastError, + Expression? updatedAt, + Expression? rowid, + }) { + return RawValuesInsertable({ + if (actionId != null) 'action_id': actionId, + if (name != null) 'name': name, + if (status != null) 'status': status, + if (resultJson != null) 'result_json': resultJson, + if (lastError != null) 'last_error': lastError, + if (updatedAt != null) 'updated_at': updatedAt, + if (rowid != null) 'rowid': rowid, + }); + } + + KalamActionStepsCompanion copyWith({ + Value? actionId, + Value? name, + Value? status, + Value? resultJson, + Value? lastError, + Value? updatedAt, + Value? rowid, + }) { + return KalamActionStepsCompanion( + actionId: actionId ?? this.actionId, + name: name ?? this.name, + status: status ?? this.status, + resultJson: resultJson ?? this.resultJson, + lastError: lastError ?? this.lastError, + updatedAt: updatedAt ?? this.updatedAt, + rowid: rowid ?? this.rowid, + ); + } + + @override + Map toColumns(bool nullToAbsent) { + final map = {}; + if (actionId.present) { + map['action_id'] = Variable(actionId.value); + } + if (name.present) { + map['name'] = Variable(name.value); + } + if (status.present) { + map['status'] = Variable(status.value); + } + if (resultJson.present) { + map['result_json'] = Variable(resultJson.value); + } + if (lastError.present) { + map['last_error'] = Variable(lastError.value); + } + if (updatedAt.present) { + map['updated_at'] = Variable(updatedAt.value); + } + if (rowid.present) { + map['rowid'] = Variable(rowid.value); + } + return map; + } + + @override + String toString() { + return (StringBuffer('KalamActionStepsCompanion(') + ..write('actionId: $actionId, ') + ..write('name: $name, ') + ..write('status: $status, ') + ..write('resultJson: $resultJson, ') + ..write('lastError: $lastError, ') + ..write('updatedAt: $updatedAt, ') + ..write('rowid: $rowid') + ..write(')')) + .toString(); + } +} + +class $KalamCheckpointsTable extends KalamCheckpoints + with TableInfo<$KalamCheckpointsTable, StoredCheckpoint> { + @override + final GeneratedDatabase attachedDatabase; + final String? _alias; + $KalamCheckpointsTable(this.attachedDatabase, [this._alias]); + static const VerificationMeta _accountKeyMeta = const VerificationMeta( + 'accountKey', + ); + @override + late final GeneratedColumn accountKey = GeneratedColumn( + 'account_key', + aliasedName, + false, + type: DriftSqlType.string, + requiredDuringInsert: true, + ); + static const VerificationMeta _subscriptionIdMeta = const VerificationMeta( + 'subscriptionId', + ); + @override + late final GeneratedColumn subscriptionId = GeneratedColumn( + 'subscription_id', + aliasedName, + false, + type: DriftSqlType.string, + requiredDuringInsert: true, + ); + static const VerificationMeta _seqMeta = const VerificationMeta('seq'); + @override + late final GeneratedColumn seq = GeneratedColumn( + 'seq', + aliasedName, + false, + type: DriftSqlType.string, + requiredDuringInsert: true, + ); + static const VerificationMeta _updatedAtMeta = const VerificationMeta( + 'updatedAt', + ); + @override + late final GeneratedColumn updatedAt = GeneratedColumn( + 'updated_at', + aliasedName, + false, + type: DriftSqlType.dateTime, + requiredDuringInsert: true, + ); + @override + List get $columns => [ + accountKey, + subscriptionId, + seq, + updatedAt, + ]; + @override + String get aliasedName => _alias ?? actualTableName; + @override + String get actualTableName => $name; + static const String $name = 'kalam_checkpoints'; + @override + VerificationContext validateIntegrity( + Insertable instance, { + bool isInserting = false, + }) { + final context = VerificationContext(); + final data = instance.toColumns(true); + if (data.containsKey('account_key')) { + context.handle( + _accountKeyMeta, + accountKey.isAcceptableOrUnknown(data['account_key']!, _accountKeyMeta), + ); + } else if (isInserting) { + context.missing(_accountKeyMeta); + } + if (data.containsKey('subscription_id')) { + context.handle( + _subscriptionIdMeta, + subscriptionId.isAcceptableOrUnknown( + data['subscription_id']!, + _subscriptionIdMeta, + ), + ); + } else if (isInserting) { + context.missing(_subscriptionIdMeta); + } + if (data.containsKey('seq')) { + context.handle( + _seqMeta, + seq.isAcceptableOrUnknown(data['seq']!, _seqMeta), + ); + } else if (isInserting) { + context.missing(_seqMeta); + } + if (data.containsKey('updated_at')) { + context.handle( + _updatedAtMeta, + updatedAt.isAcceptableOrUnknown(data['updated_at']!, _updatedAtMeta), + ); + } else if (isInserting) { + context.missing(_updatedAtMeta); + } + return context; + } + + @override + Set get $primaryKey => {accountKey, subscriptionId}; + @override + StoredCheckpoint map(Map data, {String? tablePrefix}) { + final effectivePrefix = tablePrefix != null ? '$tablePrefix.' : ''; + return StoredCheckpoint( + accountKey: attachedDatabase.typeMapping.read( + DriftSqlType.string, + data['${effectivePrefix}account_key'], + )!, + subscriptionId: attachedDatabase.typeMapping.read( + DriftSqlType.string, + data['${effectivePrefix}subscription_id'], + )!, + seq: attachedDatabase.typeMapping.read( + DriftSqlType.string, + data['${effectivePrefix}seq'], + )!, + updatedAt: attachedDatabase.typeMapping.read( + DriftSqlType.dateTime, + data['${effectivePrefix}updated_at'], + )!, + ); + } + + @override + $KalamCheckpointsTable createAlias(String alias) { + return $KalamCheckpointsTable(attachedDatabase, alias); + } +} + +class StoredCheckpoint extends DataClass + implements Insertable { + final String accountKey; + final String subscriptionId; + final String seq; + final DateTime updatedAt; + const StoredCheckpoint({ + required this.accountKey, + required this.subscriptionId, + required this.seq, + required this.updatedAt, + }); + @override + Map toColumns(bool nullToAbsent) { + final map = {}; + map['account_key'] = Variable(accountKey); + map['subscription_id'] = Variable(subscriptionId); + map['seq'] = Variable(seq); + map['updated_at'] = Variable(updatedAt); + return map; + } + + KalamCheckpointsCompanion toCompanion(bool nullToAbsent) { + return KalamCheckpointsCompanion( + accountKey: Value(accountKey), + subscriptionId: Value(subscriptionId), + seq: Value(seq), + updatedAt: Value(updatedAt), + ); + } + + factory StoredCheckpoint.fromJson( + Map json, { + ValueSerializer? serializer, + }) { + serializer ??= driftRuntimeOptions.defaultSerializer; + return StoredCheckpoint( + accountKey: serializer.fromJson(json['accountKey']), + subscriptionId: serializer.fromJson(json['subscriptionId']), + seq: serializer.fromJson(json['seq']), + updatedAt: serializer.fromJson(json['updatedAt']), + ); + } + @override + Map toJson({ValueSerializer? serializer}) { + serializer ??= driftRuntimeOptions.defaultSerializer; + return { + 'accountKey': serializer.toJson(accountKey), + 'subscriptionId': serializer.toJson(subscriptionId), + 'seq': serializer.toJson(seq), + 'updatedAt': serializer.toJson(updatedAt), + }; + } + + StoredCheckpoint copyWith({ + String? accountKey, + String? subscriptionId, + String? seq, + DateTime? updatedAt, + }) => StoredCheckpoint( + accountKey: accountKey ?? this.accountKey, + subscriptionId: subscriptionId ?? this.subscriptionId, + seq: seq ?? this.seq, + updatedAt: updatedAt ?? this.updatedAt, + ); + StoredCheckpoint copyWithCompanion(KalamCheckpointsCompanion data) { + return StoredCheckpoint( + accountKey: data.accountKey.present + ? data.accountKey.value + : this.accountKey, + subscriptionId: data.subscriptionId.present + ? data.subscriptionId.value + : this.subscriptionId, + seq: data.seq.present ? data.seq.value : this.seq, + updatedAt: data.updatedAt.present ? data.updatedAt.value : this.updatedAt, + ); + } + + @override + String toString() { + return (StringBuffer('StoredCheckpoint(') + ..write('accountKey: $accountKey, ') + ..write('subscriptionId: $subscriptionId, ') + ..write('seq: $seq, ') + ..write('updatedAt: $updatedAt') + ..write(')')) + .toString(); + } + + @override + int get hashCode => Object.hash(accountKey, subscriptionId, seq, updatedAt); + @override + bool operator ==(Object other) => + identical(this, other) || + (other is StoredCheckpoint && + other.accountKey == this.accountKey && + other.subscriptionId == this.subscriptionId && + other.seq == this.seq && + other.updatedAt == this.updatedAt); +} + +class KalamCheckpointsCompanion extends UpdateCompanion { + final Value accountKey; + final Value subscriptionId; + final Value seq; + final Value updatedAt; + final Value rowid; + const KalamCheckpointsCompanion({ + this.accountKey = const Value.absent(), + this.subscriptionId = const Value.absent(), + this.seq = const Value.absent(), + this.updatedAt = const Value.absent(), + this.rowid = const Value.absent(), + }); + KalamCheckpointsCompanion.insert({ + required String accountKey, + required String subscriptionId, + required String seq, + required DateTime updatedAt, + this.rowid = const Value.absent(), + }) : accountKey = Value(accountKey), + subscriptionId = Value(subscriptionId), + seq = Value(seq), + updatedAt = Value(updatedAt); + static Insertable custom({ + Expression? accountKey, + Expression? subscriptionId, + Expression? seq, + Expression? updatedAt, + Expression? rowid, + }) { + return RawValuesInsertable({ + if (accountKey != null) 'account_key': accountKey, + if (subscriptionId != null) 'subscription_id': subscriptionId, + if (seq != null) 'seq': seq, + if (updatedAt != null) 'updated_at': updatedAt, + if (rowid != null) 'rowid': rowid, + }); + } + + KalamCheckpointsCompanion copyWith({ + Value? accountKey, + Value? subscriptionId, + Value? seq, + Value? updatedAt, + Value? rowid, + }) { + return KalamCheckpointsCompanion( + accountKey: accountKey ?? this.accountKey, + subscriptionId: subscriptionId ?? this.subscriptionId, + seq: seq ?? this.seq, + updatedAt: updatedAt ?? this.updatedAt, + rowid: rowid ?? this.rowid, + ); + } + + @override + Map toColumns(bool nullToAbsent) { + final map = {}; + if (accountKey.present) { + map['account_key'] = Variable(accountKey.value); + } + if (subscriptionId.present) { + map['subscription_id'] = Variable(subscriptionId.value); + } + if (seq.present) { + map['seq'] = Variable(seq.value); + } + if (updatedAt.present) { + map['updated_at'] = Variable(updatedAt.value); + } + if (rowid.present) { + map['rowid'] = Variable(rowid.value); + } + return map; + } + + @override + String toString() { + return (StringBuffer('KalamCheckpointsCompanion(') + ..write('accountKey: $accountKey, ') + ..write('subscriptionId: $subscriptionId, ') + ..write('seq: $seq, ') + ..write('updatedAt: $updatedAt, ') + ..write('rowid: $rowid') + ..write(')')) + .toString(); + } +} + +class $KalamRowStatesTable extends KalamRowStates + with TableInfo<$KalamRowStatesTable, StoredRowState> { + @override + final GeneratedDatabase attachedDatabase; + final String? _alias; + $KalamRowStatesTable(this.attachedDatabase, [this._alias]); + static const VerificationMeta _accountKeyMeta = const VerificationMeta( + 'accountKey', + ); + @override + late final GeneratedColumn accountKey = GeneratedColumn( + 'account_key', + aliasedName, + false, + type: DriftSqlType.string, + requiredDuringInsert: true, + ); + static const VerificationMeta _tableIdMeta = const VerificationMeta( + 'tableId', + ); + @override + late final GeneratedColumn tableId = GeneratedColumn( + 'table_id', + aliasedName, + false, + type: DriftSqlType.string, + requiredDuringInsert: true, + ); + static const VerificationMeta _rowKeyMeta = const VerificationMeta('rowKey'); + @override + late final GeneratedColumn rowKey = GeneratedColumn( + 'row_key', + aliasedName, + false, + type: DriftSqlType.string, + requiredDuringInsert: true, + ); + static const VerificationMeta _phaseMeta = const VerificationMeta('phase'); + @override + late final GeneratedColumn phase = GeneratedColumn( + 'phase', + aliasedName, + false, + type: DriftSqlType.string, + requiredDuringInsert: true, + ); + static const VerificationMeta _actionIdMeta = const VerificationMeta( + 'actionId', + ); + @override + late final GeneratedColumn actionId = GeneratedColumn( + 'action_id', + aliasedName, + true, + type: DriftSqlType.string, + requiredDuringInsert: false, + ); + static const VerificationMeta _attemptCountMeta = const VerificationMeta( + 'attemptCount', + ); + @override + late final GeneratedColumn attemptCount = GeneratedColumn( + 'attempt_count', + aliasedName, + false, + type: DriftSqlType.int, + requiredDuringInsert: false, + defaultValue: const Constant(0), + ); + static const VerificationMeta _nextRetryAtMeta = const VerificationMeta( + 'nextRetryAt', + ); + @override + late final GeneratedColumn nextRetryAt = GeneratedColumn( + 'next_retry_at', + aliasedName, + true, + type: DriftSqlType.dateTime, + requiredDuringInsert: false, + ); + static const VerificationMeta _errorCodeMeta = const VerificationMeta( + 'errorCode', + ); + @override + late final GeneratedColumn errorCode = GeneratedColumn( + 'error_code', + aliasedName, + true, + type: DriftSqlType.string, + requiredDuringInsert: false, + ); + static const VerificationMeta _errorMessageMeta = const VerificationMeta( + 'errorMessage', + ); + @override + late final GeneratedColumn errorMessage = GeneratedColumn( + 'error_message', + aliasedName, + true, + type: DriftSqlType.string, + requiredDuringInsert: false, + ); + static const VerificationMeta _lastServerSeqMeta = const VerificationMeta( + 'lastServerSeq', + ); + @override + late final GeneratedColumn lastServerSeq = GeneratedColumn( + 'last_server_seq', + aliasedName, + true, + type: DriftSqlType.string, + requiredDuringInsert: false, + ); + static const VerificationMeta _pendingValuesJsonMeta = const VerificationMeta( + 'pendingValuesJson', + ); + @override + late final GeneratedColumn pendingValuesJson = + GeneratedColumn( + 'pending_values_json', + aliasedName, + true, + type: DriftSqlType.string, + requiredDuringInsert: false, + ); + static const VerificationMeta _tombstoneMeta = const VerificationMeta( + 'tombstone', + ); + @override + late final GeneratedColumn tombstone = GeneratedColumn( + 'tombstone', + aliasedName, + false, + type: DriftSqlType.bool, + requiredDuringInsert: false, + defaultConstraints: GeneratedColumn.constraintIsAlways( + 'CHECK ("tombstone" IN (0, 1))', + ), + defaultValue: const Constant(false), + ); + static const VerificationMeta _updatedAtMeta = const VerificationMeta( + 'updatedAt', + ); + @override + late final GeneratedColumn updatedAt = GeneratedColumn( + 'updated_at', + aliasedName, + false, + type: DriftSqlType.dateTime, + requiredDuringInsert: true, + ); + @override + List get $columns => [ + accountKey, + tableId, + rowKey, + phase, + actionId, + attemptCount, + nextRetryAt, + errorCode, + errorMessage, + lastServerSeq, + pendingValuesJson, + tombstone, + updatedAt, + ]; + @override + String get aliasedName => _alias ?? actualTableName; + @override + String get actualTableName => $name; + static const String $name = 'kalam_row_states'; + @override + VerificationContext validateIntegrity( + Insertable instance, { + bool isInserting = false, + }) { + final context = VerificationContext(); + final data = instance.toColumns(true); + if (data.containsKey('account_key')) { + context.handle( + _accountKeyMeta, + accountKey.isAcceptableOrUnknown(data['account_key']!, _accountKeyMeta), + ); + } else if (isInserting) { + context.missing(_accountKeyMeta); + } + if (data.containsKey('table_id')) { + context.handle( + _tableIdMeta, + tableId.isAcceptableOrUnknown(data['table_id']!, _tableIdMeta), + ); + } else if (isInserting) { + context.missing(_tableIdMeta); + } + if (data.containsKey('row_key')) { + context.handle( + _rowKeyMeta, + rowKey.isAcceptableOrUnknown(data['row_key']!, _rowKeyMeta), + ); + } else if (isInserting) { + context.missing(_rowKeyMeta); + } + if (data.containsKey('phase')) { + context.handle( + _phaseMeta, + phase.isAcceptableOrUnknown(data['phase']!, _phaseMeta), + ); + } else if (isInserting) { + context.missing(_phaseMeta); + } + if (data.containsKey('action_id')) { + context.handle( + _actionIdMeta, + actionId.isAcceptableOrUnknown(data['action_id']!, _actionIdMeta), + ); + } + if (data.containsKey('attempt_count')) { + context.handle( + _attemptCountMeta, + attemptCount.isAcceptableOrUnknown( + data['attempt_count']!, + _attemptCountMeta, + ), + ); + } + if (data.containsKey('next_retry_at')) { + context.handle( + _nextRetryAtMeta, + nextRetryAt.isAcceptableOrUnknown( + data['next_retry_at']!, + _nextRetryAtMeta, + ), + ); + } + if (data.containsKey('error_code')) { + context.handle( + _errorCodeMeta, + errorCode.isAcceptableOrUnknown(data['error_code']!, _errorCodeMeta), + ); + } + if (data.containsKey('error_message')) { + context.handle( + _errorMessageMeta, + errorMessage.isAcceptableOrUnknown( + data['error_message']!, + _errorMessageMeta, + ), + ); + } + if (data.containsKey('last_server_seq')) { + context.handle( + _lastServerSeqMeta, + lastServerSeq.isAcceptableOrUnknown( + data['last_server_seq']!, + _lastServerSeqMeta, + ), + ); + } + if (data.containsKey('pending_values_json')) { + context.handle( + _pendingValuesJsonMeta, + pendingValuesJson.isAcceptableOrUnknown( + data['pending_values_json']!, + _pendingValuesJsonMeta, + ), + ); + } + if (data.containsKey('tombstone')) { + context.handle( + _tombstoneMeta, + tombstone.isAcceptableOrUnknown(data['tombstone']!, _tombstoneMeta), + ); + } + if (data.containsKey('updated_at')) { + context.handle( + _updatedAtMeta, + updatedAt.isAcceptableOrUnknown(data['updated_at']!, _updatedAtMeta), + ); + } else if (isInserting) { + context.missing(_updatedAtMeta); + } + return context; + } + + @override + Set get $primaryKey => {accountKey, tableId, rowKey}; + @override + StoredRowState map(Map data, {String? tablePrefix}) { + final effectivePrefix = tablePrefix != null ? '$tablePrefix.' : ''; + return StoredRowState( + accountKey: attachedDatabase.typeMapping.read( + DriftSqlType.string, + data['${effectivePrefix}account_key'], + )!, + tableId: attachedDatabase.typeMapping.read( + DriftSqlType.string, + data['${effectivePrefix}table_id'], + )!, + rowKey: attachedDatabase.typeMapping.read( + DriftSqlType.string, + data['${effectivePrefix}row_key'], + )!, + phase: attachedDatabase.typeMapping.read( + DriftSqlType.string, + data['${effectivePrefix}phase'], + )!, + actionId: attachedDatabase.typeMapping.read( + DriftSqlType.string, + data['${effectivePrefix}action_id'], + ), + attemptCount: attachedDatabase.typeMapping.read( + DriftSqlType.int, + data['${effectivePrefix}attempt_count'], + )!, + nextRetryAt: attachedDatabase.typeMapping.read( + DriftSqlType.dateTime, + data['${effectivePrefix}next_retry_at'], + ), + errorCode: attachedDatabase.typeMapping.read( + DriftSqlType.string, + data['${effectivePrefix}error_code'], + ), + errorMessage: attachedDatabase.typeMapping.read( + DriftSqlType.string, + data['${effectivePrefix}error_message'], + ), + lastServerSeq: attachedDatabase.typeMapping.read( + DriftSqlType.string, + data['${effectivePrefix}last_server_seq'], + ), + pendingValuesJson: attachedDatabase.typeMapping.read( + DriftSqlType.string, + data['${effectivePrefix}pending_values_json'], + ), + tombstone: attachedDatabase.typeMapping.read( + DriftSqlType.bool, + data['${effectivePrefix}tombstone'], + )!, + updatedAt: attachedDatabase.typeMapping.read( + DriftSqlType.dateTime, + data['${effectivePrefix}updated_at'], + )!, + ); + } + + @override + $KalamRowStatesTable createAlias(String alias) { + return $KalamRowStatesTable(attachedDatabase, alias); + } +} + +class StoredRowState extends DataClass implements Insertable { + final String accountKey; + final String tableId; + final String rowKey; + final String phase; + final String? actionId; + final int attemptCount; + final DateTime? nextRetryAt; + final String? errorCode; + final String? errorMessage; + final String? lastServerSeq; + final String? pendingValuesJson; + final bool tombstone; + final DateTime updatedAt; + const StoredRowState({ + required this.accountKey, + required this.tableId, + required this.rowKey, + required this.phase, + this.actionId, + required this.attemptCount, + this.nextRetryAt, + this.errorCode, + this.errorMessage, + this.lastServerSeq, + this.pendingValuesJson, + required this.tombstone, + required this.updatedAt, + }); + @override + Map toColumns(bool nullToAbsent) { + final map = {}; + map['account_key'] = Variable(accountKey); + map['table_id'] = Variable(tableId); + map['row_key'] = Variable(rowKey); + map['phase'] = Variable(phase); + if (!nullToAbsent || actionId != null) { + map['action_id'] = Variable(actionId); + } + map['attempt_count'] = Variable(attemptCount); + if (!nullToAbsent || nextRetryAt != null) { + map['next_retry_at'] = Variable(nextRetryAt); + } + if (!nullToAbsent || errorCode != null) { + map['error_code'] = Variable(errorCode); + } + if (!nullToAbsent || errorMessage != null) { + map['error_message'] = Variable(errorMessage); + } + if (!nullToAbsent || lastServerSeq != null) { + map['last_server_seq'] = Variable(lastServerSeq); + } + if (!nullToAbsent || pendingValuesJson != null) { + map['pending_values_json'] = Variable(pendingValuesJson); + } + map['tombstone'] = Variable(tombstone); + map['updated_at'] = Variable(updatedAt); + return map; + } + + KalamRowStatesCompanion toCompanion(bool nullToAbsent) { + return KalamRowStatesCompanion( + accountKey: Value(accountKey), + tableId: Value(tableId), + rowKey: Value(rowKey), + phase: Value(phase), + actionId: actionId == null && nullToAbsent + ? const Value.absent() + : Value(actionId), + attemptCount: Value(attemptCount), + nextRetryAt: nextRetryAt == null && nullToAbsent + ? const Value.absent() + : Value(nextRetryAt), + errorCode: errorCode == null && nullToAbsent + ? const Value.absent() + : Value(errorCode), + errorMessage: errorMessage == null && nullToAbsent + ? const Value.absent() + : Value(errorMessage), + lastServerSeq: lastServerSeq == null && nullToAbsent + ? const Value.absent() + : Value(lastServerSeq), + pendingValuesJson: pendingValuesJson == null && nullToAbsent + ? const Value.absent() + : Value(pendingValuesJson), + tombstone: Value(tombstone), + updatedAt: Value(updatedAt), + ); + } + + factory StoredRowState.fromJson( + Map json, { + ValueSerializer? serializer, + }) { + serializer ??= driftRuntimeOptions.defaultSerializer; + return StoredRowState( + accountKey: serializer.fromJson(json['accountKey']), + tableId: serializer.fromJson(json['tableId']), + rowKey: serializer.fromJson(json['rowKey']), + phase: serializer.fromJson(json['phase']), + actionId: serializer.fromJson(json['actionId']), + attemptCount: serializer.fromJson(json['attemptCount']), + nextRetryAt: serializer.fromJson(json['nextRetryAt']), + errorCode: serializer.fromJson(json['errorCode']), + errorMessage: serializer.fromJson(json['errorMessage']), + lastServerSeq: serializer.fromJson(json['lastServerSeq']), + pendingValuesJson: serializer.fromJson( + json['pendingValuesJson'], + ), + tombstone: serializer.fromJson(json['tombstone']), + updatedAt: serializer.fromJson(json['updatedAt']), + ); + } + @override + Map toJson({ValueSerializer? serializer}) { + serializer ??= driftRuntimeOptions.defaultSerializer; + return { + 'accountKey': serializer.toJson(accountKey), + 'tableId': serializer.toJson(tableId), + 'rowKey': serializer.toJson(rowKey), + 'phase': serializer.toJson(phase), + 'actionId': serializer.toJson(actionId), + 'attemptCount': serializer.toJson(attemptCount), + 'nextRetryAt': serializer.toJson(nextRetryAt), + 'errorCode': serializer.toJson(errorCode), + 'errorMessage': serializer.toJson(errorMessage), + 'lastServerSeq': serializer.toJson(lastServerSeq), + 'pendingValuesJson': serializer.toJson(pendingValuesJson), + 'tombstone': serializer.toJson(tombstone), + 'updatedAt': serializer.toJson(updatedAt), + }; + } + + StoredRowState copyWith({ + String? accountKey, + String? tableId, + String? rowKey, + String? phase, + Value actionId = const Value.absent(), + int? attemptCount, + Value nextRetryAt = const Value.absent(), + Value errorCode = const Value.absent(), + Value errorMessage = const Value.absent(), + Value lastServerSeq = const Value.absent(), + Value pendingValuesJson = const Value.absent(), + bool? tombstone, + DateTime? updatedAt, + }) => StoredRowState( + accountKey: accountKey ?? this.accountKey, + tableId: tableId ?? this.tableId, + rowKey: rowKey ?? this.rowKey, + phase: phase ?? this.phase, + actionId: actionId.present ? actionId.value : this.actionId, + attemptCount: attemptCount ?? this.attemptCount, + nextRetryAt: nextRetryAt.present ? nextRetryAt.value : this.nextRetryAt, + errorCode: errorCode.present ? errorCode.value : this.errorCode, + errorMessage: errorMessage.present ? errorMessage.value : this.errorMessage, + lastServerSeq: lastServerSeq.present + ? lastServerSeq.value + : this.lastServerSeq, + pendingValuesJson: pendingValuesJson.present + ? pendingValuesJson.value + : this.pendingValuesJson, + tombstone: tombstone ?? this.tombstone, + updatedAt: updatedAt ?? this.updatedAt, + ); + StoredRowState copyWithCompanion(KalamRowStatesCompanion data) { + return StoredRowState( + accountKey: data.accountKey.present + ? data.accountKey.value + : this.accountKey, + tableId: data.tableId.present ? data.tableId.value : this.tableId, + rowKey: data.rowKey.present ? data.rowKey.value : this.rowKey, + phase: data.phase.present ? data.phase.value : this.phase, + actionId: data.actionId.present ? data.actionId.value : this.actionId, + attemptCount: data.attemptCount.present + ? data.attemptCount.value + : this.attemptCount, + nextRetryAt: data.nextRetryAt.present + ? data.nextRetryAt.value + : this.nextRetryAt, + errorCode: data.errorCode.present ? data.errorCode.value : this.errorCode, + errorMessage: data.errorMessage.present + ? data.errorMessage.value + : this.errorMessage, + lastServerSeq: data.lastServerSeq.present + ? data.lastServerSeq.value + : this.lastServerSeq, + pendingValuesJson: data.pendingValuesJson.present + ? data.pendingValuesJson.value + : this.pendingValuesJson, + tombstone: data.tombstone.present ? data.tombstone.value : this.tombstone, + updatedAt: data.updatedAt.present ? data.updatedAt.value : this.updatedAt, + ); + } + + @override + String toString() { + return (StringBuffer('StoredRowState(') + ..write('accountKey: $accountKey, ') + ..write('tableId: $tableId, ') + ..write('rowKey: $rowKey, ') + ..write('phase: $phase, ') + ..write('actionId: $actionId, ') + ..write('attemptCount: $attemptCount, ') + ..write('nextRetryAt: $nextRetryAt, ') + ..write('errorCode: $errorCode, ') + ..write('errorMessage: $errorMessage, ') + ..write('lastServerSeq: $lastServerSeq, ') + ..write('pendingValuesJson: $pendingValuesJson, ') + ..write('tombstone: $tombstone, ') + ..write('updatedAt: $updatedAt') + ..write(')')) + .toString(); + } + + @override + int get hashCode => Object.hash( + accountKey, + tableId, + rowKey, + phase, + actionId, + attemptCount, + nextRetryAt, + errorCode, + errorMessage, + lastServerSeq, + pendingValuesJson, + tombstone, + updatedAt, + ); + @override + bool operator ==(Object other) => + identical(this, other) || + (other is StoredRowState && + other.accountKey == this.accountKey && + other.tableId == this.tableId && + other.rowKey == this.rowKey && + other.phase == this.phase && + other.actionId == this.actionId && + other.attemptCount == this.attemptCount && + other.nextRetryAt == this.nextRetryAt && + other.errorCode == this.errorCode && + other.errorMessage == this.errorMessage && + other.lastServerSeq == this.lastServerSeq && + other.pendingValuesJson == this.pendingValuesJson && + other.tombstone == this.tombstone && + other.updatedAt == this.updatedAt); +} + +class KalamRowStatesCompanion extends UpdateCompanion { + final Value accountKey; + final Value tableId; + final Value rowKey; + final Value phase; + final Value actionId; + final Value attemptCount; + final Value nextRetryAt; + final Value errorCode; + final Value errorMessage; + final Value lastServerSeq; + final Value pendingValuesJson; + final Value tombstone; + final Value updatedAt; + final Value rowid; + const KalamRowStatesCompanion({ + this.accountKey = const Value.absent(), + this.tableId = const Value.absent(), + this.rowKey = const Value.absent(), + this.phase = const Value.absent(), + this.actionId = const Value.absent(), + this.attemptCount = const Value.absent(), + this.nextRetryAt = const Value.absent(), + this.errorCode = const Value.absent(), + this.errorMessage = const Value.absent(), + this.lastServerSeq = const Value.absent(), + this.pendingValuesJson = const Value.absent(), + this.tombstone = const Value.absent(), + this.updatedAt = const Value.absent(), + this.rowid = const Value.absent(), + }); + KalamRowStatesCompanion.insert({ + required String accountKey, + required String tableId, + required String rowKey, + required String phase, + this.actionId = const Value.absent(), + this.attemptCount = const Value.absent(), + this.nextRetryAt = const Value.absent(), + this.errorCode = const Value.absent(), + this.errorMessage = const Value.absent(), + this.lastServerSeq = const Value.absent(), + this.pendingValuesJson = const Value.absent(), + this.tombstone = const Value.absent(), + required DateTime updatedAt, + this.rowid = const Value.absent(), + }) : accountKey = Value(accountKey), + tableId = Value(tableId), + rowKey = Value(rowKey), + phase = Value(phase), + updatedAt = Value(updatedAt); + static Insertable custom({ + Expression? accountKey, + Expression? tableId, + Expression? rowKey, + Expression? phase, + Expression? actionId, + Expression? attemptCount, + Expression? nextRetryAt, + Expression? errorCode, + Expression? errorMessage, + Expression? lastServerSeq, + Expression? pendingValuesJson, + Expression? tombstone, + Expression? updatedAt, + Expression? rowid, + }) { + return RawValuesInsertable({ + if (accountKey != null) 'account_key': accountKey, + if (tableId != null) 'table_id': tableId, + if (rowKey != null) 'row_key': rowKey, + if (phase != null) 'phase': phase, + if (actionId != null) 'action_id': actionId, + if (attemptCount != null) 'attempt_count': attemptCount, + if (nextRetryAt != null) 'next_retry_at': nextRetryAt, + if (errorCode != null) 'error_code': errorCode, + if (errorMessage != null) 'error_message': errorMessage, + if (lastServerSeq != null) 'last_server_seq': lastServerSeq, + if (pendingValuesJson != null) 'pending_values_json': pendingValuesJson, + if (tombstone != null) 'tombstone': tombstone, + if (updatedAt != null) 'updated_at': updatedAt, + if (rowid != null) 'rowid': rowid, + }); + } + + KalamRowStatesCompanion copyWith({ + Value? accountKey, + Value? tableId, + Value? rowKey, + Value? phase, + Value? actionId, + Value? attemptCount, + Value? nextRetryAt, + Value? errorCode, + Value? errorMessage, + Value? lastServerSeq, + Value? pendingValuesJson, + Value? tombstone, + Value? updatedAt, + Value? rowid, + }) { + return KalamRowStatesCompanion( + accountKey: accountKey ?? this.accountKey, + tableId: tableId ?? this.tableId, + rowKey: rowKey ?? this.rowKey, + phase: phase ?? this.phase, + actionId: actionId ?? this.actionId, + attemptCount: attemptCount ?? this.attemptCount, + nextRetryAt: nextRetryAt ?? this.nextRetryAt, + errorCode: errorCode ?? this.errorCode, + errorMessage: errorMessage ?? this.errorMessage, + lastServerSeq: lastServerSeq ?? this.lastServerSeq, + pendingValuesJson: pendingValuesJson ?? this.pendingValuesJson, + tombstone: tombstone ?? this.tombstone, + updatedAt: updatedAt ?? this.updatedAt, + rowid: rowid ?? this.rowid, + ); + } + + @override + Map toColumns(bool nullToAbsent) { + final map = {}; + if (accountKey.present) { + map['account_key'] = Variable(accountKey.value); + } + if (tableId.present) { + map['table_id'] = Variable(tableId.value); + } + if (rowKey.present) { + map['row_key'] = Variable(rowKey.value); + } + if (phase.present) { + map['phase'] = Variable(phase.value); + } + if (actionId.present) { + map['action_id'] = Variable(actionId.value); + } + if (attemptCount.present) { + map['attempt_count'] = Variable(attemptCount.value); + } + if (nextRetryAt.present) { + map['next_retry_at'] = Variable(nextRetryAt.value); + } + if (errorCode.present) { + map['error_code'] = Variable(errorCode.value); + } + if (errorMessage.present) { + map['error_message'] = Variable(errorMessage.value); + } + if (lastServerSeq.present) { + map['last_server_seq'] = Variable(lastServerSeq.value); + } + if (pendingValuesJson.present) { + map['pending_values_json'] = Variable(pendingValuesJson.value); + } + if (tombstone.present) { + map['tombstone'] = Variable(tombstone.value); + } + if (updatedAt.present) { + map['updated_at'] = Variable(updatedAt.value); + } + if (rowid.present) { + map['rowid'] = Variable(rowid.value); + } + return map; + } + + @override + String toString() { + return (StringBuffer('KalamRowStatesCompanion(') + ..write('accountKey: $accountKey, ') + ..write('tableId: $tableId, ') + ..write('rowKey: $rowKey, ') + ..write('phase: $phase, ') + ..write('actionId: $actionId, ') + ..write('attemptCount: $attemptCount, ') + ..write('nextRetryAt: $nextRetryAt, ') + ..write('errorCode: $errorCode, ') + ..write('errorMessage: $errorMessage, ') + ..write('lastServerSeq: $lastServerSeq, ') + ..write('pendingValuesJson: $pendingValuesJson, ') + ..write('tombstone: $tombstone, ') + ..write('updatedAt: $updatedAt, ') + ..write('rowid: $rowid') + ..write(')')) + .toString(); + } +} + +abstract class _$KalamSyncDatabase extends GeneratedDatabase { + _$KalamSyncDatabase(QueryExecutor e) : super(e); + $KalamSyncDatabaseManager get managers => $KalamSyncDatabaseManager(this); + late final $KalamActionsTable kalamActions = $KalamActionsTable(this); + late final $KalamCachedRowsTable kalamCachedRows = $KalamCachedRowsTable( + this, + ); + late final $KalamActionStepsTable kalamActionSteps = $KalamActionStepsTable( + this, + ); + late final $KalamCheckpointsTable kalamCheckpoints = $KalamCheckpointsTable( + this, + ); + late final $KalamRowStatesTable kalamRowStates = $KalamRowStatesTable(this); + @override + Iterable> get allTables => + allSchemaEntities.whereType>(); + @override + List get allSchemaEntities => [ + kalamActions, + kalamCachedRows, + kalamActionSteps, + kalamCheckpoints, + kalamRowStates, + ]; +} + +typedef $$KalamActionsTableCreateCompanionBuilder = + KalamActionsCompanion Function({ + required String id, + required String accountKey, + required String actionKey, + Value version, + Value kind, + required String payloadJson, + required String status, + Value orderingKey, + Value rowTableId, + Value rowKey, + Value queuePosition, + Value attemptCount, + Value nextAttemptAt, + Value lastError, + required DateTime createdAt, + required DateTime updatedAt, + Value rowid, + }); +typedef $$KalamActionsTableUpdateCompanionBuilder = + KalamActionsCompanion Function({ + Value id, + Value accountKey, + Value actionKey, + Value version, + Value kind, + Value payloadJson, + Value status, + Value orderingKey, + Value rowTableId, + Value rowKey, + Value queuePosition, + Value attemptCount, + Value nextAttemptAt, + Value lastError, + Value createdAt, + Value updatedAt, + Value rowid, + }); + +class $$KalamActionsTableFilterComposer + extends Composer<_$KalamSyncDatabase, $KalamActionsTable> { + $$KalamActionsTableFilterComposer({ + required super.$db, + required super.$table, + super.joinBuilder, + super.$addJoinBuilderToRootComposer, + super.$removeJoinBuilderFromRootComposer, + }); + ColumnFilters get id => $composableBuilder( + column: $table.id, + builder: (column) => ColumnFilters(column), + ); + + ColumnFilters get accountKey => $composableBuilder( + column: $table.accountKey, + builder: (column) => ColumnFilters(column), + ); + + ColumnFilters get actionKey => $composableBuilder( + column: $table.actionKey, + builder: (column) => ColumnFilters(column), + ); + + ColumnFilters get version => $composableBuilder( + column: $table.version, + builder: (column) => ColumnFilters(column), + ); + + ColumnFilters get kind => $composableBuilder( + column: $table.kind, + builder: (column) => ColumnFilters(column), + ); + + ColumnFilters get payloadJson => $composableBuilder( + column: $table.payloadJson, + builder: (column) => ColumnFilters(column), + ); + + ColumnFilters get status => $composableBuilder( + column: $table.status, + builder: (column) => ColumnFilters(column), + ); + + ColumnFilters get orderingKey => $composableBuilder( + column: $table.orderingKey, + builder: (column) => ColumnFilters(column), + ); + + ColumnFilters get rowTableId => $composableBuilder( + column: $table.rowTableId, + builder: (column) => ColumnFilters(column), + ); + + ColumnFilters get rowKey => $composableBuilder( + column: $table.rowKey, + builder: (column) => ColumnFilters(column), + ); + + ColumnFilters get queuePosition => $composableBuilder( + column: $table.queuePosition, + builder: (column) => ColumnFilters(column), + ); + + ColumnFilters get attemptCount => $composableBuilder( + column: $table.attemptCount, + builder: (column) => ColumnFilters(column), + ); + + ColumnFilters get nextAttemptAt => $composableBuilder( + column: $table.nextAttemptAt, + builder: (column) => ColumnFilters(column), + ); + + ColumnFilters get lastError => $composableBuilder( + column: $table.lastError, + builder: (column) => ColumnFilters(column), + ); + + ColumnFilters get createdAt => $composableBuilder( + column: $table.createdAt, + builder: (column) => ColumnFilters(column), + ); + + ColumnFilters get updatedAt => $composableBuilder( + column: $table.updatedAt, + builder: (column) => ColumnFilters(column), + ); +} + +class $$KalamActionsTableOrderingComposer + extends Composer<_$KalamSyncDatabase, $KalamActionsTable> { + $$KalamActionsTableOrderingComposer({ + required super.$db, + required super.$table, + super.joinBuilder, + super.$addJoinBuilderToRootComposer, + super.$removeJoinBuilderFromRootComposer, + }); + ColumnOrderings get id => $composableBuilder( + column: $table.id, + builder: (column) => ColumnOrderings(column), + ); + + ColumnOrderings get accountKey => $composableBuilder( + column: $table.accountKey, + builder: (column) => ColumnOrderings(column), + ); + + ColumnOrderings get actionKey => $composableBuilder( + column: $table.actionKey, + builder: (column) => ColumnOrderings(column), + ); + + ColumnOrderings get version => $composableBuilder( + column: $table.version, + builder: (column) => ColumnOrderings(column), + ); + + ColumnOrderings get kind => $composableBuilder( + column: $table.kind, + builder: (column) => ColumnOrderings(column), + ); + + ColumnOrderings get payloadJson => $composableBuilder( + column: $table.payloadJson, + builder: (column) => ColumnOrderings(column), + ); + + ColumnOrderings get status => $composableBuilder( + column: $table.status, + builder: (column) => ColumnOrderings(column), + ); + + ColumnOrderings get orderingKey => $composableBuilder( + column: $table.orderingKey, + builder: (column) => ColumnOrderings(column), + ); + + ColumnOrderings get rowTableId => $composableBuilder( + column: $table.rowTableId, + builder: (column) => ColumnOrderings(column), + ); + + ColumnOrderings get rowKey => $composableBuilder( + column: $table.rowKey, + builder: (column) => ColumnOrderings(column), + ); + + ColumnOrderings get queuePosition => $composableBuilder( + column: $table.queuePosition, + builder: (column) => ColumnOrderings(column), + ); + + ColumnOrderings get attemptCount => $composableBuilder( + column: $table.attemptCount, + builder: (column) => ColumnOrderings(column), + ); + + ColumnOrderings get nextAttemptAt => $composableBuilder( + column: $table.nextAttemptAt, + builder: (column) => ColumnOrderings(column), + ); + + ColumnOrderings get lastError => $composableBuilder( + column: $table.lastError, + builder: (column) => ColumnOrderings(column), + ); + + ColumnOrderings get createdAt => $composableBuilder( + column: $table.createdAt, + builder: (column) => ColumnOrderings(column), + ); + + ColumnOrderings get updatedAt => $composableBuilder( + column: $table.updatedAt, + builder: (column) => ColumnOrderings(column), + ); +} + +class $$KalamActionsTableAnnotationComposer + extends Composer<_$KalamSyncDatabase, $KalamActionsTable> { + $$KalamActionsTableAnnotationComposer({ + required super.$db, + required super.$table, + super.joinBuilder, + super.$addJoinBuilderToRootComposer, + super.$removeJoinBuilderFromRootComposer, + }); + GeneratedColumn get id => + $composableBuilder(column: $table.id, builder: (column) => column); + + GeneratedColumn get accountKey => $composableBuilder( + column: $table.accountKey, + builder: (column) => column, + ); + + GeneratedColumn get actionKey => + $composableBuilder(column: $table.actionKey, builder: (column) => column); + + GeneratedColumn get version => + $composableBuilder(column: $table.version, builder: (column) => column); + + GeneratedColumn get kind => + $composableBuilder(column: $table.kind, builder: (column) => column); + + GeneratedColumn get payloadJson => $composableBuilder( + column: $table.payloadJson, + builder: (column) => column, + ); + + GeneratedColumn get status => + $composableBuilder(column: $table.status, builder: (column) => column); + + GeneratedColumn get orderingKey => $composableBuilder( + column: $table.orderingKey, + builder: (column) => column, + ); + + GeneratedColumn get rowTableId => $composableBuilder( + column: $table.rowTableId, + builder: (column) => column, + ); + + GeneratedColumn get rowKey => + $composableBuilder(column: $table.rowKey, builder: (column) => column); + + GeneratedColumn get queuePosition => $composableBuilder( + column: $table.queuePosition, + builder: (column) => column, + ); + + GeneratedColumn get attemptCount => $composableBuilder( + column: $table.attemptCount, + builder: (column) => column, + ); + + GeneratedColumn get nextAttemptAt => $composableBuilder( + column: $table.nextAttemptAt, + builder: (column) => column, + ); + + GeneratedColumn get lastError => + $composableBuilder(column: $table.lastError, builder: (column) => column); + + GeneratedColumn get createdAt => + $composableBuilder(column: $table.createdAt, builder: (column) => column); + + GeneratedColumn get updatedAt => + $composableBuilder(column: $table.updatedAt, builder: (column) => column); +} + +class $$KalamActionsTableTableManager + extends + RootTableManager< + _$KalamSyncDatabase, + $KalamActionsTable, + StoredAction, + $$KalamActionsTableFilterComposer, + $$KalamActionsTableOrderingComposer, + $$KalamActionsTableAnnotationComposer, + $$KalamActionsTableCreateCompanionBuilder, + $$KalamActionsTableUpdateCompanionBuilder, + ( + StoredAction, + BaseReferences< + _$KalamSyncDatabase, + $KalamActionsTable, + StoredAction + >, + ), + StoredAction, + PrefetchHooks Function() + > { + $$KalamActionsTableTableManager( + _$KalamSyncDatabase db, + $KalamActionsTable table, + ) : super( + TableManagerState( + db: db, + table: table, + createFilteringComposer: () => + $$KalamActionsTableFilterComposer($db: db, $table: table), + createOrderingComposer: () => + $$KalamActionsTableOrderingComposer($db: db, $table: table), + createComputedFieldComposer: () => + $$KalamActionsTableAnnotationComposer($db: db, $table: table), + updateCompanionCallback: + ({ + Value id = const Value.absent(), + Value accountKey = const Value.absent(), + Value actionKey = const Value.absent(), + Value version = const Value.absent(), + Value kind = const Value.absent(), + Value payloadJson = const Value.absent(), + Value status = const Value.absent(), + Value orderingKey = const Value.absent(), + Value rowTableId = const Value.absent(), + Value rowKey = const Value.absent(), + Value queuePosition = const Value.absent(), + Value attemptCount = const Value.absent(), + Value nextAttemptAt = const Value.absent(), + Value lastError = const Value.absent(), + Value createdAt = const Value.absent(), + Value updatedAt = const Value.absent(), + Value rowid = const Value.absent(), + }) => KalamActionsCompanion( + id: id, + accountKey: accountKey, + actionKey: actionKey, + version: version, + kind: kind, + payloadJson: payloadJson, + status: status, + orderingKey: orderingKey, + rowTableId: rowTableId, + rowKey: rowKey, + queuePosition: queuePosition, + attemptCount: attemptCount, + nextAttemptAt: nextAttemptAt, + lastError: lastError, + createdAt: createdAt, + updatedAt: updatedAt, + rowid: rowid, + ), + createCompanionCallback: + ({ + required String id, + required String accountKey, + required String actionKey, + Value version = const Value.absent(), + Value kind = const Value.absent(), + required String payloadJson, + required String status, + Value orderingKey = const Value.absent(), + Value rowTableId = const Value.absent(), + Value rowKey = const Value.absent(), + Value queuePosition = const Value.absent(), + Value attemptCount = const Value.absent(), + Value nextAttemptAt = const Value.absent(), + Value lastError = const Value.absent(), + required DateTime createdAt, + required DateTime updatedAt, + Value rowid = const Value.absent(), + }) => KalamActionsCompanion.insert( + id: id, + accountKey: accountKey, + actionKey: actionKey, + version: version, + kind: kind, + payloadJson: payloadJson, + status: status, + orderingKey: orderingKey, + rowTableId: rowTableId, + rowKey: rowKey, + queuePosition: queuePosition, + attemptCount: attemptCount, + nextAttemptAt: nextAttemptAt, + lastError: lastError, + createdAt: createdAt, + updatedAt: updatedAt, + rowid: rowid, + ), + withReferenceMapper: (p0) => p0 + .map((e) => (e.readTable(table), BaseReferences(db, table, e))) + .toList(), + prefetchHooksCallback: null, + ), + ); +} + +typedef $$KalamActionsTableProcessedTableManager = + ProcessedTableManager< + _$KalamSyncDatabase, + $KalamActionsTable, + StoredAction, + $$KalamActionsTableFilterComposer, + $$KalamActionsTableOrderingComposer, + $$KalamActionsTableAnnotationComposer, + $$KalamActionsTableCreateCompanionBuilder, + $$KalamActionsTableUpdateCompanionBuilder, + ( + StoredAction, + BaseReferences<_$KalamSyncDatabase, $KalamActionsTable, StoredAction>, + ), + StoredAction, + PrefetchHooks Function() + >; +typedef $$KalamCachedRowsTableCreateCompanionBuilder = + KalamCachedRowsCompanion Function({ + required String accountKey, + required String tableId, + required String rowKey, + required String valuesJson, + required DateTime updatedAt, + Value rowid, + }); +typedef $$KalamCachedRowsTableUpdateCompanionBuilder = + KalamCachedRowsCompanion Function({ + Value accountKey, + Value tableId, + Value rowKey, + Value valuesJson, + Value updatedAt, + Value rowid, + }); + +class $$KalamCachedRowsTableFilterComposer + extends Composer<_$KalamSyncDatabase, $KalamCachedRowsTable> { + $$KalamCachedRowsTableFilterComposer({ + required super.$db, + required super.$table, + super.joinBuilder, + super.$addJoinBuilderToRootComposer, + super.$removeJoinBuilderFromRootComposer, + }); + ColumnFilters get accountKey => $composableBuilder( + column: $table.accountKey, + builder: (column) => ColumnFilters(column), + ); + + ColumnFilters get tableId => $composableBuilder( + column: $table.tableId, + builder: (column) => ColumnFilters(column), + ); + + ColumnFilters get rowKey => $composableBuilder( + column: $table.rowKey, + builder: (column) => ColumnFilters(column), + ); + + ColumnFilters get valuesJson => $composableBuilder( + column: $table.valuesJson, + builder: (column) => ColumnFilters(column), + ); + + ColumnFilters get updatedAt => $composableBuilder( + column: $table.updatedAt, + builder: (column) => ColumnFilters(column), + ); +} + +class $$KalamCachedRowsTableOrderingComposer + extends Composer<_$KalamSyncDatabase, $KalamCachedRowsTable> { + $$KalamCachedRowsTableOrderingComposer({ + required super.$db, + required super.$table, + super.joinBuilder, + super.$addJoinBuilderToRootComposer, + super.$removeJoinBuilderFromRootComposer, + }); + ColumnOrderings get accountKey => $composableBuilder( + column: $table.accountKey, + builder: (column) => ColumnOrderings(column), + ); + + ColumnOrderings get tableId => $composableBuilder( + column: $table.tableId, + builder: (column) => ColumnOrderings(column), + ); + + ColumnOrderings get rowKey => $composableBuilder( + column: $table.rowKey, + builder: (column) => ColumnOrderings(column), + ); + + ColumnOrderings get valuesJson => $composableBuilder( + column: $table.valuesJson, + builder: (column) => ColumnOrderings(column), + ); + + ColumnOrderings get updatedAt => $composableBuilder( + column: $table.updatedAt, + builder: (column) => ColumnOrderings(column), + ); +} + +class $$KalamCachedRowsTableAnnotationComposer + extends Composer<_$KalamSyncDatabase, $KalamCachedRowsTable> { + $$KalamCachedRowsTableAnnotationComposer({ + required super.$db, + required super.$table, + super.joinBuilder, + super.$addJoinBuilderToRootComposer, + super.$removeJoinBuilderFromRootComposer, + }); + GeneratedColumn get accountKey => $composableBuilder( + column: $table.accountKey, + builder: (column) => column, + ); + + GeneratedColumn get tableId => + $composableBuilder(column: $table.tableId, builder: (column) => column); + + GeneratedColumn get rowKey => + $composableBuilder(column: $table.rowKey, builder: (column) => column); + + GeneratedColumn get valuesJson => $composableBuilder( + column: $table.valuesJson, + builder: (column) => column, + ); + + GeneratedColumn get updatedAt => + $composableBuilder(column: $table.updatedAt, builder: (column) => column); +} + +class $$KalamCachedRowsTableTableManager + extends + RootTableManager< + _$KalamSyncDatabase, + $KalamCachedRowsTable, + StoredCachedRow, + $$KalamCachedRowsTableFilterComposer, + $$KalamCachedRowsTableOrderingComposer, + $$KalamCachedRowsTableAnnotationComposer, + $$KalamCachedRowsTableCreateCompanionBuilder, + $$KalamCachedRowsTableUpdateCompanionBuilder, + ( + StoredCachedRow, + BaseReferences< + _$KalamSyncDatabase, + $KalamCachedRowsTable, + StoredCachedRow + >, + ), + StoredCachedRow, + PrefetchHooks Function() + > { + $$KalamCachedRowsTableTableManager( + _$KalamSyncDatabase db, + $KalamCachedRowsTable table, + ) : super( + TableManagerState( + db: db, + table: table, + createFilteringComposer: () => + $$KalamCachedRowsTableFilterComposer($db: db, $table: table), + createOrderingComposer: () => + $$KalamCachedRowsTableOrderingComposer($db: db, $table: table), + createComputedFieldComposer: () => + $$KalamCachedRowsTableAnnotationComposer($db: db, $table: table), + updateCompanionCallback: + ({ + Value accountKey = const Value.absent(), + Value tableId = const Value.absent(), + Value rowKey = const Value.absent(), + Value valuesJson = const Value.absent(), + Value updatedAt = const Value.absent(), + Value rowid = const Value.absent(), + }) => KalamCachedRowsCompanion( + accountKey: accountKey, + tableId: tableId, + rowKey: rowKey, + valuesJson: valuesJson, + updatedAt: updatedAt, + rowid: rowid, + ), + createCompanionCallback: + ({ + required String accountKey, + required String tableId, + required String rowKey, + required String valuesJson, + required DateTime updatedAt, + Value rowid = const Value.absent(), + }) => KalamCachedRowsCompanion.insert( + accountKey: accountKey, + tableId: tableId, + rowKey: rowKey, + valuesJson: valuesJson, + updatedAt: updatedAt, + rowid: rowid, + ), + withReferenceMapper: (p0) => p0 + .map((e) => (e.readTable(table), BaseReferences(db, table, e))) + .toList(), + prefetchHooksCallback: null, + ), + ); +} + +typedef $$KalamCachedRowsTableProcessedTableManager = + ProcessedTableManager< + _$KalamSyncDatabase, + $KalamCachedRowsTable, + StoredCachedRow, + $$KalamCachedRowsTableFilterComposer, + $$KalamCachedRowsTableOrderingComposer, + $$KalamCachedRowsTableAnnotationComposer, + $$KalamCachedRowsTableCreateCompanionBuilder, + $$KalamCachedRowsTableUpdateCompanionBuilder, + ( + StoredCachedRow, + BaseReferences< + _$KalamSyncDatabase, + $KalamCachedRowsTable, + StoredCachedRow + >, + ), + StoredCachedRow, + PrefetchHooks Function() + >; +typedef $$KalamActionStepsTableCreateCompanionBuilder = + KalamActionStepsCompanion Function({ + required String actionId, + required String name, + required String status, + Value resultJson, + Value lastError, + required DateTime updatedAt, + Value rowid, + }); +typedef $$KalamActionStepsTableUpdateCompanionBuilder = + KalamActionStepsCompanion Function({ + Value actionId, + Value name, + Value status, + Value resultJson, + Value lastError, + Value updatedAt, + Value rowid, + }); + +class $$KalamActionStepsTableFilterComposer + extends Composer<_$KalamSyncDatabase, $KalamActionStepsTable> { + $$KalamActionStepsTableFilterComposer({ + required super.$db, + required super.$table, + super.joinBuilder, + super.$addJoinBuilderToRootComposer, + super.$removeJoinBuilderFromRootComposer, + }); + ColumnFilters get actionId => $composableBuilder( + column: $table.actionId, + builder: (column) => ColumnFilters(column), + ); + + ColumnFilters get name => $composableBuilder( + column: $table.name, + builder: (column) => ColumnFilters(column), + ); + + ColumnFilters get status => $composableBuilder( + column: $table.status, + builder: (column) => ColumnFilters(column), + ); + + ColumnFilters get resultJson => $composableBuilder( + column: $table.resultJson, + builder: (column) => ColumnFilters(column), + ); + + ColumnFilters get lastError => $composableBuilder( + column: $table.lastError, + builder: (column) => ColumnFilters(column), + ); + + ColumnFilters get updatedAt => $composableBuilder( + column: $table.updatedAt, + builder: (column) => ColumnFilters(column), + ); +} + +class $$KalamActionStepsTableOrderingComposer + extends Composer<_$KalamSyncDatabase, $KalamActionStepsTable> { + $$KalamActionStepsTableOrderingComposer({ + required super.$db, + required super.$table, + super.joinBuilder, + super.$addJoinBuilderToRootComposer, + super.$removeJoinBuilderFromRootComposer, + }); + ColumnOrderings get actionId => $composableBuilder( + column: $table.actionId, + builder: (column) => ColumnOrderings(column), + ); + + ColumnOrderings get name => $composableBuilder( + column: $table.name, + builder: (column) => ColumnOrderings(column), + ); + + ColumnOrderings get status => $composableBuilder( + column: $table.status, + builder: (column) => ColumnOrderings(column), + ); + + ColumnOrderings get resultJson => $composableBuilder( + column: $table.resultJson, + builder: (column) => ColumnOrderings(column), + ); + + ColumnOrderings get lastError => $composableBuilder( + column: $table.lastError, + builder: (column) => ColumnOrderings(column), + ); + + ColumnOrderings get updatedAt => $composableBuilder( + column: $table.updatedAt, + builder: (column) => ColumnOrderings(column), + ); +} + +class $$KalamActionStepsTableAnnotationComposer + extends Composer<_$KalamSyncDatabase, $KalamActionStepsTable> { + $$KalamActionStepsTableAnnotationComposer({ + required super.$db, + required super.$table, + super.joinBuilder, + super.$addJoinBuilderToRootComposer, + super.$removeJoinBuilderFromRootComposer, + }); + GeneratedColumn get actionId => + $composableBuilder(column: $table.actionId, builder: (column) => column); + + GeneratedColumn get name => + $composableBuilder(column: $table.name, builder: (column) => column); + + GeneratedColumn get status => + $composableBuilder(column: $table.status, builder: (column) => column); + + GeneratedColumn get resultJson => $composableBuilder( + column: $table.resultJson, + builder: (column) => column, + ); + + GeneratedColumn get lastError => + $composableBuilder(column: $table.lastError, builder: (column) => column); + + GeneratedColumn get updatedAt => + $composableBuilder(column: $table.updatedAt, builder: (column) => column); +} + +class $$KalamActionStepsTableTableManager + extends + RootTableManager< + _$KalamSyncDatabase, + $KalamActionStepsTable, + StoredActionStep, + $$KalamActionStepsTableFilterComposer, + $$KalamActionStepsTableOrderingComposer, + $$KalamActionStepsTableAnnotationComposer, + $$KalamActionStepsTableCreateCompanionBuilder, + $$KalamActionStepsTableUpdateCompanionBuilder, + ( + StoredActionStep, + BaseReferences< + _$KalamSyncDatabase, + $KalamActionStepsTable, + StoredActionStep + >, + ), + StoredActionStep, + PrefetchHooks Function() + > { + $$KalamActionStepsTableTableManager( + _$KalamSyncDatabase db, + $KalamActionStepsTable table, + ) : super( + TableManagerState( + db: db, + table: table, + createFilteringComposer: () => + $$KalamActionStepsTableFilterComposer($db: db, $table: table), + createOrderingComposer: () => + $$KalamActionStepsTableOrderingComposer($db: db, $table: table), + createComputedFieldComposer: () => + $$KalamActionStepsTableAnnotationComposer($db: db, $table: table), + updateCompanionCallback: + ({ + Value actionId = const Value.absent(), + Value name = const Value.absent(), + Value status = const Value.absent(), + Value resultJson = const Value.absent(), + Value lastError = const Value.absent(), + Value updatedAt = const Value.absent(), + Value rowid = const Value.absent(), + }) => KalamActionStepsCompanion( + actionId: actionId, + name: name, + status: status, + resultJson: resultJson, + lastError: lastError, + updatedAt: updatedAt, + rowid: rowid, + ), + createCompanionCallback: + ({ + required String actionId, + required String name, + required String status, + Value resultJson = const Value.absent(), + Value lastError = const Value.absent(), + required DateTime updatedAt, + Value rowid = const Value.absent(), + }) => KalamActionStepsCompanion.insert( + actionId: actionId, + name: name, + status: status, + resultJson: resultJson, + lastError: lastError, + updatedAt: updatedAt, + rowid: rowid, + ), + withReferenceMapper: (p0) => p0 + .map((e) => (e.readTable(table), BaseReferences(db, table, e))) + .toList(), + prefetchHooksCallback: null, + ), + ); +} + +typedef $$KalamActionStepsTableProcessedTableManager = + ProcessedTableManager< + _$KalamSyncDatabase, + $KalamActionStepsTable, + StoredActionStep, + $$KalamActionStepsTableFilterComposer, + $$KalamActionStepsTableOrderingComposer, + $$KalamActionStepsTableAnnotationComposer, + $$KalamActionStepsTableCreateCompanionBuilder, + $$KalamActionStepsTableUpdateCompanionBuilder, + ( + StoredActionStep, + BaseReferences< + _$KalamSyncDatabase, + $KalamActionStepsTable, + StoredActionStep + >, + ), + StoredActionStep, + PrefetchHooks Function() + >; +typedef $$KalamCheckpointsTableCreateCompanionBuilder = + KalamCheckpointsCompanion Function({ + required String accountKey, + required String subscriptionId, + required String seq, + required DateTime updatedAt, + Value rowid, + }); +typedef $$KalamCheckpointsTableUpdateCompanionBuilder = + KalamCheckpointsCompanion Function({ + Value accountKey, + Value subscriptionId, + Value seq, + Value updatedAt, + Value rowid, + }); + +class $$KalamCheckpointsTableFilterComposer + extends Composer<_$KalamSyncDatabase, $KalamCheckpointsTable> { + $$KalamCheckpointsTableFilterComposer({ + required super.$db, + required super.$table, + super.joinBuilder, + super.$addJoinBuilderToRootComposer, + super.$removeJoinBuilderFromRootComposer, + }); + ColumnFilters get accountKey => $composableBuilder( + column: $table.accountKey, + builder: (column) => ColumnFilters(column), + ); + + ColumnFilters get subscriptionId => $composableBuilder( + column: $table.subscriptionId, + builder: (column) => ColumnFilters(column), + ); + + ColumnFilters get seq => $composableBuilder( + column: $table.seq, + builder: (column) => ColumnFilters(column), + ); + + ColumnFilters get updatedAt => $composableBuilder( + column: $table.updatedAt, + builder: (column) => ColumnFilters(column), + ); +} + +class $$KalamCheckpointsTableOrderingComposer + extends Composer<_$KalamSyncDatabase, $KalamCheckpointsTable> { + $$KalamCheckpointsTableOrderingComposer({ + required super.$db, + required super.$table, + super.joinBuilder, + super.$addJoinBuilderToRootComposer, + super.$removeJoinBuilderFromRootComposer, + }); + ColumnOrderings get accountKey => $composableBuilder( + column: $table.accountKey, + builder: (column) => ColumnOrderings(column), + ); + + ColumnOrderings get subscriptionId => $composableBuilder( + column: $table.subscriptionId, + builder: (column) => ColumnOrderings(column), + ); + + ColumnOrderings get seq => $composableBuilder( + column: $table.seq, + builder: (column) => ColumnOrderings(column), + ); + + ColumnOrderings get updatedAt => $composableBuilder( + column: $table.updatedAt, + builder: (column) => ColumnOrderings(column), + ); +} + +class $$KalamCheckpointsTableAnnotationComposer + extends Composer<_$KalamSyncDatabase, $KalamCheckpointsTable> { + $$KalamCheckpointsTableAnnotationComposer({ + required super.$db, + required super.$table, + super.joinBuilder, + super.$addJoinBuilderToRootComposer, + super.$removeJoinBuilderFromRootComposer, + }); + GeneratedColumn get accountKey => $composableBuilder( + column: $table.accountKey, + builder: (column) => column, + ); + + GeneratedColumn get subscriptionId => $composableBuilder( + column: $table.subscriptionId, + builder: (column) => column, + ); + + GeneratedColumn get seq => + $composableBuilder(column: $table.seq, builder: (column) => column); + + GeneratedColumn get updatedAt => + $composableBuilder(column: $table.updatedAt, builder: (column) => column); +} + +class $$KalamCheckpointsTableTableManager + extends + RootTableManager< + _$KalamSyncDatabase, + $KalamCheckpointsTable, + StoredCheckpoint, + $$KalamCheckpointsTableFilterComposer, + $$KalamCheckpointsTableOrderingComposer, + $$KalamCheckpointsTableAnnotationComposer, + $$KalamCheckpointsTableCreateCompanionBuilder, + $$KalamCheckpointsTableUpdateCompanionBuilder, + ( + StoredCheckpoint, + BaseReferences< + _$KalamSyncDatabase, + $KalamCheckpointsTable, + StoredCheckpoint + >, + ), + StoredCheckpoint, + PrefetchHooks Function() + > { + $$KalamCheckpointsTableTableManager( + _$KalamSyncDatabase db, + $KalamCheckpointsTable table, + ) : super( + TableManagerState( + db: db, + table: table, + createFilteringComposer: () => + $$KalamCheckpointsTableFilterComposer($db: db, $table: table), + createOrderingComposer: () => + $$KalamCheckpointsTableOrderingComposer($db: db, $table: table), + createComputedFieldComposer: () => + $$KalamCheckpointsTableAnnotationComposer($db: db, $table: table), + updateCompanionCallback: + ({ + Value accountKey = const Value.absent(), + Value subscriptionId = const Value.absent(), + Value seq = const Value.absent(), + Value updatedAt = const Value.absent(), + Value rowid = const Value.absent(), + }) => KalamCheckpointsCompanion( + accountKey: accountKey, + subscriptionId: subscriptionId, + seq: seq, + updatedAt: updatedAt, + rowid: rowid, + ), + createCompanionCallback: + ({ + required String accountKey, + required String subscriptionId, + required String seq, + required DateTime updatedAt, + Value rowid = const Value.absent(), + }) => KalamCheckpointsCompanion.insert( + accountKey: accountKey, + subscriptionId: subscriptionId, + seq: seq, + updatedAt: updatedAt, + rowid: rowid, + ), + withReferenceMapper: (p0) => p0 + .map((e) => (e.readTable(table), BaseReferences(db, table, e))) + .toList(), + prefetchHooksCallback: null, + ), + ); +} + +typedef $$KalamCheckpointsTableProcessedTableManager = + ProcessedTableManager< + _$KalamSyncDatabase, + $KalamCheckpointsTable, + StoredCheckpoint, + $$KalamCheckpointsTableFilterComposer, + $$KalamCheckpointsTableOrderingComposer, + $$KalamCheckpointsTableAnnotationComposer, + $$KalamCheckpointsTableCreateCompanionBuilder, + $$KalamCheckpointsTableUpdateCompanionBuilder, + ( + StoredCheckpoint, + BaseReferences< + _$KalamSyncDatabase, + $KalamCheckpointsTable, + StoredCheckpoint + >, + ), + StoredCheckpoint, + PrefetchHooks Function() + >; +typedef $$KalamRowStatesTableCreateCompanionBuilder = + KalamRowStatesCompanion Function({ + required String accountKey, + required String tableId, + required String rowKey, + required String phase, + Value actionId, + Value attemptCount, + Value nextRetryAt, + Value errorCode, + Value errorMessage, + Value lastServerSeq, + Value pendingValuesJson, + Value tombstone, + required DateTime updatedAt, + Value rowid, + }); +typedef $$KalamRowStatesTableUpdateCompanionBuilder = + KalamRowStatesCompanion Function({ + Value accountKey, + Value tableId, + Value rowKey, + Value phase, + Value actionId, + Value attemptCount, + Value nextRetryAt, + Value errorCode, + Value errorMessage, + Value lastServerSeq, + Value pendingValuesJson, + Value tombstone, + Value updatedAt, + Value rowid, + }); + +class $$KalamRowStatesTableFilterComposer + extends Composer<_$KalamSyncDatabase, $KalamRowStatesTable> { + $$KalamRowStatesTableFilterComposer({ + required super.$db, + required super.$table, + super.joinBuilder, + super.$addJoinBuilderToRootComposer, + super.$removeJoinBuilderFromRootComposer, + }); + ColumnFilters get accountKey => $composableBuilder( + column: $table.accountKey, + builder: (column) => ColumnFilters(column), + ); + + ColumnFilters get tableId => $composableBuilder( + column: $table.tableId, + builder: (column) => ColumnFilters(column), + ); + + ColumnFilters get rowKey => $composableBuilder( + column: $table.rowKey, + builder: (column) => ColumnFilters(column), + ); + + ColumnFilters get phase => $composableBuilder( + column: $table.phase, + builder: (column) => ColumnFilters(column), + ); + + ColumnFilters get actionId => $composableBuilder( + column: $table.actionId, + builder: (column) => ColumnFilters(column), + ); + + ColumnFilters get attemptCount => $composableBuilder( + column: $table.attemptCount, + builder: (column) => ColumnFilters(column), + ); + + ColumnFilters get nextRetryAt => $composableBuilder( + column: $table.nextRetryAt, + builder: (column) => ColumnFilters(column), + ); + + ColumnFilters get errorCode => $composableBuilder( + column: $table.errorCode, + builder: (column) => ColumnFilters(column), + ); + + ColumnFilters get errorMessage => $composableBuilder( + column: $table.errorMessage, + builder: (column) => ColumnFilters(column), + ); + + ColumnFilters get lastServerSeq => $composableBuilder( + column: $table.lastServerSeq, + builder: (column) => ColumnFilters(column), + ); + + ColumnFilters get pendingValuesJson => $composableBuilder( + column: $table.pendingValuesJson, + builder: (column) => ColumnFilters(column), + ); + + ColumnFilters get tombstone => $composableBuilder( + column: $table.tombstone, + builder: (column) => ColumnFilters(column), + ); + + ColumnFilters get updatedAt => $composableBuilder( + column: $table.updatedAt, + builder: (column) => ColumnFilters(column), + ); +} + +class $$KalamRowStatesTableOrderingComposer + extends Composer<_$KalamSyncDatabase, $KalamRowStatesTable> { + $$KalamRowStatesTableOrderingComposer({ + required super.$db, + required super.$table, + super.joinBuilder, + super.$addJoinBuilderToRootComposer, + super.$removeJoinBuilderFromRootComposer, + }); + ColumnOrderings get accountKey => $composableBuilder( + column: $table.accountKey, + builder: (column) => ColumnOrderings(column), + ); + + ColumnOrderings get tableId => $composableBuilder( + column: $table.tableId, + builder: (column) => ColumnOrderings(column), + ); + + ColumnOrderings get rowKey => $composableBuilder( + column: $table.rowKey, + builder: (column) => ColumnOrderings(column), + ); + + ColumnOrderings get phase => $composableBuilder( + column: $table.phase, + builder: (column) => ColumnOrderings(column), + ); + + ColumnOrderings get actionId => $composableBuilder( + column: $table.actionId, + builder: (column) => ColumnOrderings(column), + ); + + ColumnOrderings get attemptCount => $composableBuilder( + column: $table.attemptCount, + builder: (column) => ColumnOrderings(column), + ); + + ColumnOrderings get nextRetryAt => $composableBuilder( + column: $table.nextRetryAt, + builder: (column) => ColumnOrderings(column), + ); + + ColumnOrderings get errorCode => $composableBuilder( + column: $table.errorCode, + builder: (column) => ColumnOrderings(column), + ); + + ColumnOrderings get errorMessage => $composableBuilder( + column: $table.errorMessage, + builder: (column) => ColumnOrderings(column), + ); + + ColumnOrderings get lastServerSeq => $composableBuilder( + column: $table.lastServerSeq, + builder: (column) => ColumnOrderings(column), + ); + + ColumnOrderings get pendingValuesJson => $composableBuilder( + column: $table.pendingValuesJson, + builder: (column) => ColumnOrderings(column), + ); + + ColumnOrderings get tombstone => $composableBuilder( + column: $table.tombstone, + builder: (column) => ColumnOrderings(column), + ); + + ColumnOrderings get updatedAt => $composableBuilder( + column: $table.updatedAt, + builder: (column) => ColumnOrderings(column), + ); +} + +class $$KalamRowStatesTableAnnotationComposer + extends Composer<_$KalamSyncDatabase, $KalamRowStatesTable> { + $$KalamRowStatesTableAnnotationComposer({ + required super.$db, + required super.$table, + super.joinBuilder, + super.$addJoinBuilderToRootComposer, + super.$removeJoinBuilderFromRootComposer, + }); + GeneratedColumn get accountKey => $composableBuilder( + column: $table.accountKey, + builder: (column) => column, + ); + + GeneratedColumn get tableId => + $composableBuilder(column: $table.tableId, builder: (column) => column); + + GeneratedColumn get rowKey => + $composableBuilder(column: $table.rowKey, builder: (column) => column); + + GeneratedColumn get phase => + $composableBuilder(column: $table.phase, builder: (column) => column); + + GeneratedColumn get actionId => + $composableBuilder(column: $table.actionId, builder: (column) => column); + + GeneratedColumn get attemptCount => $composableBuilder( + column: $table.attemptCount, + builder: (column) => column, + ); + + GeneratedColumn get nextRetryAt => $composableBuilder( + column: $table.nextRetryAt, + builder: (column) => column, + ); + + GeneratedColumn get errorCode => + $composableBuilder(column: $table.errorCode, builder: (column) => column); + + GeneratedColumn get errorMessage => $composableBuilder( + column: $table.errorMessage, + builder: (column) => column, + ); + + GeneratedColumn get lastServerSeq => $composableBuilder( + column: $table.lastServerSeq, + builder: (column) => column, + ); + + GeneratedColumn get pendingValuesJson => $composableBuilder( + column: $table.pendingValuesJson, + builder: (column) => column, + ); + + GeneratedColumn get tombstone => + $composableBuilder(column: $table.tombstone, builder: (column) => column); + + GeneratedColumn get updatedAt => + $composableBuilder(column: $table.updatedAt, builder: (column) => column); +} + +class $$KalamRowStatesTableTableManager + extends + RootTableManager< + _$KalamSyncDatabase, + $KalamRowStatesTable, + StoredRowState, + $$KalamRowStatesTableFilterComposer, + $$KalamRowStatesTableOrderingComposer, + $$KalamRowStatesTableAnnotationComposer, + $$KalamRowStatesTableCreateCompanionBuilder, + $$KalamRowStatesTableUpdateCompanionBuilder, + ( + StoredRowState, + BaseReferences< + _$KalamSyncDatabase, + $KalamRowStatesTable, + StoredRowState + >, + ), + StoredRowState, + PrefetchHooks Function() + > { + $$KalamRowStatesTableTableManager( + _$KalamSyncDatabase db, + $KalamRowStatesTable table, + ) : super( + TableManagerState( + db: db, + table: table, + createFilteringComposer: () => + $$KalamRowStatesTableFilterComposer($db: db, $table: table), + createOrderingComposer: () => + $$KalamRowStatesTableOrderingComposer($db: db, $table: table), + createComputedFieldComposer: () => + $$KalamRowStatesTableAnnotationComposer($db: db, $table: table), + updateCompanionCallback: + ({ + Value accountKey = const Value.absent(), + Value tableId = const Value.absent(), + Value rowKey = const Value.absent(), + Value phase = const Value.absent(), + Value actionId = const Value.absent(), + Value attemptCount = const Value.absent(), + Value nextRetryAt = const Value.absent(), + Value errorCode = const Value.absent(), + Value errorMessage = const Value.absent(), + Value lastServerSeq = const Value.absent(), + Value pendingValuesJson = const Value.absent(), + Value tombstone = const Value.absent(), + Value updatedAt = const Value.absent(), + Value rowid = const Value.absent(), + }) => KalamRowStatesCompanion( + accountKey: accountKey, + tableId: tableId, + rowKey: rowKey, + phase: phase, + actionId: actionId, + attemptCount: attemptCount, + nextRetryAt: nextRetryAt, + errorCode: errorCode, + errorMessage: errorMessage, + lastServerSeq: lastServerSeq, + pendingValuesJson: pendingValuesJson, + tombstone: tombstone, + updatedAt: updatedAt, + rowid: rowid, + ), + createCompanionCallback: + ({ + required String accountKey, + required String tableId, + required String rowKey, + required String phase, + Value actionId = const Value.absent(), + Value attemptCount = const Value.absent(), + Value nextRetryAt = const Value.absent(), + Value errorCode = const Value.absent(), + Value errorMessage = const Value.absent(), + Value lastServerSeq = const Value.absent(), + Value pendingValuesJson = const Value.absent(), + Value tombstone = const Value.absent(), + required DateTime updatedAt, + Value rowid = const Value.absent(), + }) => KalamRowStatesCompanion.insert( + accountKey: accountKey, + tableId: tableId, + rowKey: rowKey, + phase: phase, + actionId: actionId, + attemptCount: attemptCount, + nextRetryAt: nextRetryAt, + errorCode: errorCode, + errorMessage: errorMessage, + lastServerSeq: lastServerSeq, + pendingValuesJson: pendingValuesJson, + tombstone: tombstone, + updatedAt: updatedAt, + rowid: rowid, + ), + withReferenceMapper: (p0) => p0 + .map((e) => (e.readTable(table), BaseReferences(db, table, e))) + .toList(), + prefetchHooksCallback: null, + ), + ); +} + +typedef $$KalamRowStatesTableProcessedTableManager = + ProcessedTableManager< + _$KalamSyncDatabase, + $KalamRowStatesTable, + StoredRowState, + $$KalamRowStatesTableFilterComposer, + $$KalamRowStatesTableOrderingComposer, + $$KalamRowStatesTableAnnotationComposer, + $$KalamRowStatesTableCreateCompanionBuilder, + $$KalamRowStatesTableUpdateCompanionBuilder, + ( + StoredRowState, + BaseReferences< + _$KalamSyncDatabase, + $KalamRowStatesTable, + StoredRowState + >, + ), + StoredRowState, + PrefetchHooks Function() + >; + +class $KalamSyncDatabaseManager { + final _$KalamSyncDatabase _db; + $KalamSyncDatabaseManager(this._db); + $$KalamActionsTableTableManager get kalamActions => + $$KalamActionsTableTableManager(_db, _db.kalamActions); + $$KalamCachedRowsTableTableManager get kalamCachedRows => + $$KalamCachedRowsTableTableManager(_db, _db.kalamCachedRows); + $$KalamActionStepsTableTableManager get kalamActionSteps => + $$KalamActionStepsTableTableManager(_db, _db.kalamActionSteps); + $$KalamCheckpointsTableTableManager get kalamCheckpoints => + $$KalamCheckpointsTableTableManager(_db, _db.kalamCheckpoints); + $$KalamRowStatesTableTableManager get kalamRowStates => + $$KalamRowStatesTableTableManager(_db, _db.kalamRowStates); +} diff --git a/link/sdks/dart/sync/lib/src/database/tables/kalam_action_steps.dart b/link/sdks/dart/sync/lib/src/database/tables/kalam_action_steps.dart new file mode 100644 index 000000000..594287787 --- /dev/null +++ b/link/sdks/dart/sync/lib/src/database/tables/kalam_action_steps.dart @@ -0,0 +1,14 @@ +import 'package:drift/drift.dart'; + +@DataClassName('StoredActionStep') +class KalamActionSteps extends Table { + TextColumn get actionId => text()(); + TextColumn get name => text()(); + TextColumn get status => text()(); + TextColumn get resultJson => text().nullable()(); + TextColumn get lastError => text().nullable()(); + DateTimeColumn get updatedAt => dateTime()(); + + @override + Set> get primaryKey => {actionId, name}; +} diff --git a/link/sdks/dart/sync/lib/src/database/tables/kalam_actions.dart b/link/sdks/dart/sync/lib/src/database/tables/kalam_actions.dart new file mode 100644 index 000000000..a4395e174 --- /dev/null +++ b/link/sdks/dart/sync/lib/src/database/tables/kalam_actions.dart @@ -0,0 +1,24 @@ +import 'package:drift/drift.dart'; + +@DataClassName('StoredAction') +class KalamActions extends Table { + TextColumn get id => text()(); + TextColumn get accountKey => text()(); + TextColumn get actionKey => text()(); + IntColumn get version => integer().withDefault(const Constant(1))(); + TextColumn get kind => text().withDefault(const Constant('custom'))(); + TextColumn get payloadJson => text()(); + TextColumn get status => text()(); + TextColumn get orderingKey => text().nullable()(); + TextColumn get rowTableId => text().nullable()(); + TextColumn get rowKey => text().nullable()(); + IntColumn get queuePosition => integer().withDefault(const Constant(0))(); + IntColumn get attemptCount => integer().withDefault(const Constant(0))(); + DateTimeColumn get nextAttemptAt => dateTime().nullable()(); + TextColumn get lastError => text().nullable()(); + DateTimeColumn get createdAt => dateTime()(); + DateTimeColumn get updatedAt => dateTime()(); + + @override + Set> get primaryKey => {id}; +} diff --git a/link/sdks/dart/sync/lib/src/database/tables/kalam_cached_rows.dart b/link/sdks/dart/sync/lib/src/database/tables/kalam_cached_rows.dart new file mode 100644 index 000000000..99df4c592 --- /dev/null +++ b/link/sdks/dart/sync/lib/src/database/tables/kalam_cached_rows.dart @@ -0,0 +1,13 @@ +import 'package:drift/drift.dart'; + +@DataClassName('StoredCachedRow') +class KalamCachedRows extends Table { + TextColumn get accountKey => text()(); + TextColumn get tableId => text()(); + TextColumn get rowKey => text()(); + TextColumn get valuesJson => text()(); + DateTimeColumn get updatedAt => dateTime()(); + + @override + Set> get primaryKey => {accountKey, tableId, rowKey}; +} diff --git a/link/sdks/dart/sync/lib/src/database/tables/kalam_checkpoints.dart b/link/sdks/dart/sync/lib/src/database/tables/kalam_checkpoints.dart new file mode 100644 index 000000000..5659a7dd3 --- /dev/null +++ b/link/sdks/dart/sync/lib/src/database/tables/kalam_checkpoints.dart @@ -0,0 +1,12 @@ +import 'package:drift/drift.dart'; + +@DataClassName('StoredCheckpoint') +class KalamCheckpoints extends Table { + TextColumn get accountKey => text()(); + TextColumn get subscriptionId => text()(); + TextColumn get seq => text()(); + DateTimeColumn get updatedAt => dateTime()(); + + @override + Set> get primaryKey => {accountKey, subscriptionId}; +} diff --git a/link/sdks/dart/sync/lib/src/database/tables/kalam_row_states.dart b/link/sdks/dart/sync/lib/src/database/tables/kalam_row_states.dart new file mode 100644 index 000000000..ba175118f --- /dev/null +++ b/link/sdks/dart/sync/lib/src/database/tables/kalam_row_states.dart @@ -0,0 +1,21 @@ +import 'package:drift/drift.dart'; + +@DataClassName('StoredRowState') +class KalamRowStates extends Table { + TextColumn get accountKey => text()(); + TextColumn get tableId => text()(); + TextColumn get rowKey => text()(); + TextColumn get phase => text()(); + TextColumn get actionId => text().nullable()(); + IntColumn get attemptCount => integer().withDefault(const Constant(0))(); + DateTimeColumn get nextRetryAt => dateTime().nullable()(); + TextColumn get errorCode => text().nullable()(); + TextColumn get errorMessage => text().nullable()(); + TextColumn get lastServerSeq => text().nullable()(); + TextColumn get pendingValuesJson => text().nullable()(); + BoolColumn get tombstone => boolean().withDefault(const Constant(false))(); + DateTimeColumn get updatedAt => dateTime()(); + + @override + Set> get primaryKey => {accountKey, tableId, rowKey}; +} diff --git a/link/sdks/dart/sync/lib/src/flutter/kalam_database_factory.dart b/link/sdks/dart/sync/lib/src/flutter/kalam_database_factory.dart new file mode 100644 index 000000000..332abdd3d --- /dev/null +++ b/link/sdks/dart/sync/lib/src/flutter/kalam_database_factory.dart @@ -0,0 +1,22 @@ +import 'package:drift_flutter/drift_flutter.dart'; + +import '../database/kalam_sync_database.dart'; +import '../models/kalam_account_identity.dart'; + +abstract interface class KalamDatabaseFactory { + Future open(KalamAccountIdentity identity); +} + +final class KalamFlutterDatabaseFactory implements KalamDatabaseFactory { + const KalamFlutterDatabaseFactory(); + + @override + Future open(KalamAccountIdentity identity) async { + return KalamSyncDatabase( + driftDatabase( + name: identity.databaseName, + native: const DriftNativeOptions(shareAcrossIsolates: true), + ), + ); + } +} diff --git a/link/sdks/dart/sync/lib/src/flutter/kalam_scope.dart b/link/sdks/dart/sync/lib/src/flutter/kalam_scope.dart new file mode 100644 index 000000000..ae84d4754 --- /dev/null +++ b/link/sdks/dart/sync/lib/src/flutter/kalam_scope.dart @@ -0,0 +1,96 @@ +import 'dart:async'; + +import 'package:flutter/widgets.dart'; + +import '../kalam.dart'; + +/// Provides one [Kalam] session and coordinates it with Flutter lifecycle. +final class KalamScope extends StatefulWidget { + const KalamScope({ + required this.kalam, + required this.child, + this.manageLifecycle = true, + super.key, + }); + + final Kalam kalam; + final Widget child; + final bool manageLifecycle; + + static Kalam of(BuildContext context) { + final scope = context.dependOnInheritedWidgetOfExactType<_KalamInherited>(); + return _requireScope(scope); + } + + /// Reads the current session without rebuilding when the scope changes. + /// + /// This form is safe for one-time setup in `State.initState`. + static Kalam read(BuildContext context) { + final scope = context.getInheritedWidgetOfExactType<_KalamInherited>(); + return _requireScope(scope); + } + + static Kalam _requireScope(_KalamInherited? scope) { + if (scope == null) { + throw FlutterError('No KalamScope was found above this context.'); + } + return scope.kalam; + } + + @override + State createState() => _KalamScopeState(); +} + +final class _KalamScopeState extends State + with WidgetsBindingObserver { + @override + void initState() { + super.initState(); + if (widget.manageLifecycle) WidgetsBinding.instance.addObserver(this); + } + + @override + void didUpdateWidget(KalamScope oldWidget) { + super.didUpdateWidget(oldWidget); + if (oldWidget.manageLifecycle == widget.manageLifecycle) return; + if (widget.manageLifecycle) { + WidgetsBinding.instance.addObserver(this); + } else { + WidgetsBinding.instance.removeObserver(this); + } + } + + @override + void didChangeAppLifecycleState(AppLifecycleState state) { + switch (state) { + case AppLifecycleState.resumed: + unawaited(widget.kalam.resume()); + case AppLifecycleState.inactive || + AppLifecycleState.hidden || + AppLifecycleState.paused || + AppLifecycleState.detached: + unawaited(widget.kalam.pause()); + } + } + + @override + void dispose() { + if (widget.manageLifecycle) WidgetsBinding.instance.removeObserver(this); + super.dispose(); + } + + @override + Widget build(BuildContext context) { + return _KalamInherited(kalam: widget.kalam, child: widget.child); + } +} + +final class _KalamInherited extends InheritedWidget { + const _KalamInherited({required this.kalam, required super.child}); + + final Kalam kalam; + + @override + bool updateShouldNotify(_KalamInherited oldWidget) => + !identical(kalam, oldWidget.kalam); +} diff --git a/link/sdks/dart/sync/lib/src/kalam.dart b/link/sdks/dart/sync/lib/src/kalam.dart new file mode 100644 index 000000000..481c94a58 --- /dev/null +++ b/link/sdks/dart/sync/lib/src/kalam.dart @@ -0,0 +1,163 @@ +import 'dart:math'; + +import 'package:kalam_link/kalam_link.dart'; + +import 'actions/kalam_action_definition.dart'; +import 'actions/kalam_action_registry.dart'; +import 'actions/kalam_action_runner.dart'; +import 'actions/kalam_dml_action.dart'; +import 'database/kalam_sync_database.dart'; +import 'flutter/kalam_database_factory.dart'; +import 'models/kalam_account_identity.dart'; +import 'models/kalam_sync_state.dart'; +import 'store/kalam_sync_store.dart'; +import 'sync/kalam_event_consumer.dart'; +import 'sync/kalam_sync_coordinator.dart'; +import 'sync/kalam_sync_subscription.dart'; +import 'tables/kalam_table_binding.dart'; +import 'tables/kalam_table_spec.dart'; +import 'transport/kalam_link_transport.dart'; +import 'transport/kalam_sync_transport.dart'; + +typedef KalamTransportFactory = Future Function(); + +/// One account-scoped local-first KalamDB session. +final class Kalam { + Kalam._({ + required this.identity, + required this.database, + required this.transport, + required this.store, + required this.actions, + required this.sync, + }); + + final KalamAccountIdentity identity; + final KalamSyncDatabase database; + final KalamSyncTransport transport; + final KalamSyncStore store; + final KalamActionRunner actions; + final KalamSyncCoordinator sync; + bool _disposed = false; + + KalamSyncState get syncState => sync.state; + Stream get syncStates => sync.states; + + static Future ensureInitialized() => KalamClient.init(); + + /// Opens local storage immediately. The shared socket remains lazy until a + /// consumer subscribes or an action is queued, so opening Kalam does not + /// block Flutter's first UI. + static Future open({ + required String url, + required String subject, + AuthProvider authProvider = _anonymousAuth, + String namespace = 'default', + Iterable> actionDefinitions = const [], + KalamDatabaseFactory databaseFactory = const KalamFlutterDatabaseFactory(), + KalamTransportFactory? transportFactory, + }) async { + await ensureInitialized(); + final identity = KalamAccountIdentity( + serverUrl: url, + subject: subject, + namespace: namespace, + ); + final database = await databaseFactory.open(identity); + try { + final transport = + await (transportFactory?.call() ?? + KalamLinkTransport.connect(url: url, authProvider: authProvider)); + return Kalam.fromComponents( + identity: identity, + database: database, + transport: transport, + actionDefinitions: actionDefinitions, + ); + } catch (_) { + await database.close(); + rethrow; + } + } + + /// Low-level constructor for tests and custom transports. + /// + /// Generic bidirectional DML is registered when [transport] is a + /// [KalamLinkTransport], or when [dmlClient] is passed for a wrapper + /// transport that still talks through `kalam_link`. + factory Kalam.fromComponents({ + required KalamAccountIdentity identity, + required KalamSyncDatabase database, + required KalamSyncTransport transport, + Iterable> actionDefinitions = const [], + KalamClient? dmlClient, + }) { + final client = + dmlClient ?? + (transport is KalamLinkTransport ? transport.client : null); + final definitions = >[ + if (client != null) kalamDmlAction(client), + ...actionDefinitions, + ]; + final store = KalamSyncStore(database); + final actions = KalamActionRunner( + accountKey: identity.accountKey, + store: store, + registry: KalamActionRegistry.of(definitions), + ); + final sync = KalamSyncCoordinator( + accountKey: identity.accountKey, + store: store, + transport: transport, + actions: actions, + ); + return Kalam._( + identity: identity, + database: database, + transport: transport, + store: store, + actions: actions, + sync: sync, + ); + } + + Future subscribe(KalamEventConsumer consumer) { + return sync.subscribe(consumer); + } + + KalamTableBinding table(KalamTableSpec spec) { + return KalamTableBinding( + spec: spec, + accountKey: identity.accountKey, + store: store, + ); + } + + Future pause() => sync.pause(); + + Future resume() => sync.resume(); + + Future dispose() async { + if (_disposed) return; + _disposed = true; + await sync.dispose(); + await transport.dispose(); + await database.close(); + } + + /// Generates a cryptographically random RFC 4122 version 4 identifier. + static String id() { + final random = Random.secure(); + final bytes = List.generate(16, (_) => random.nextInt(256)); + bytes[6] = (bytes[6] & 0x0f) | 0x40; + bytes[8] = (bytes[8] & 0x3f) | 0x80; + final hex = bytes + .map((byte) => byte.toRadixString(16).padLeft(2, '0')) + .join(); + return '${hex.substring(0, 8)}-${hex.substring(8, 12)}-' + '${hex.substring(12, 16)}-${hex.substring(16, 20)}-' + '${hex.substring(20)}'; + } +} + +Future _anonymousAuth() async => const Auth.none(); diff --git a/link/sdks/dart/sync/lib/src/models/kalam_account_identity.dart b/link/sdks/dart/sync/lib/src/models/kalam_account_identity.dart new file mode 100644 index 000000000..e619bb23a --- /dev/null +++ b/link/sdks/dart/sync/lib/src/models/kalam_account_identity.dart @@ -0,0 +1,25 @@ +/// Cache and outbox ownership. A database must never cross this boundary. +final class KalamAccountIdentity { + const KalamAccountIdentity({ + required this.serverUrl, + required this.subject, + this.namespace = 'default', + }); + + final String serverUrl; + final String subject; + final String namespace; + + String get accountKey => '$serverUrl|$namespace|$subject'; + + String get databaseName => 'kalam_sync_${_fnv1a64(accountKey)}'; +} + +String _fnv1a64(String value) { + var hash = 0xcbf29ce484222325; + for (final byte in value.codeUnits) { + hash ^= byte; + hash = (hash * 0x100000001b3) & 0xFFFFFFFFFFFFFFFF; + } + return hash.toRadixString(16).padLeft(16, '0'); +} diff --git a/link/sdks/dart/sync/lib/src/models/kalam_action_draft.dart b/link/sdks/dart/sync/lib/src/models/kalam_action_draft.dart new file mode 100644 index 000000000..d553aace3 --- /dev/null +++ b/link/sdks/dart/sync/lib/src/models/kalam_action_draft.dart @@ -0,0 +1,20 @@ +/// Immutable input used to create one durable action. +final class KalamActionDraft { + const KalamActionDraft({ + required this.id, + required this.accountKey, + required this.actionKey, + required this.payloadJson, + this.version = 1, + this.kind = 'custom', + this.orderingKey, + }); + + final String id; + final String accountKey; + final String actionKey; + final String payloadJson; + final int version; + final String kind; + final String? orderingKey; +} diff --git a/link/sdks/dart/sync/lib/src/models/kalam_action_record.dart b/link/sdks/dart/sync/lib/src/models/kalam_action_record.dart new file mode 100644 index 000000000..d035c2ef4 --- /dev/null +++ b/link/sdks/dart/sync/lib/src/models/kalam_action_record.dart @@ -0,0 +1,49 @@ +import 'kalam_action_status.dart'; + +/// Public read model for one action stored in the durable outbox. +final class KalamActionRecord { + const KalamActionRecord({ + required this.id, + required this.accountKey, + required this.actionKey, + required this.version, + required this.kind, + required this.payloadJson, + required this.status, + required this.attemptCount, + required this.createdAt, + required this.updatedAt, + this.orderingKey, + this.rowTableId, + this.rowKey, + this.nextAttemptAt, + this.lastError, + }); + + final String id; + final String accountKey; + final String actionKey; + final int version; + final String kind; + final String payloadJson; + final KalamActionStatus status; + final String? orderingKey; + final String? rowTableId; + final String? rowKey; + final int attemptCount; + final DateTime? nextAttemptAt; + final String? lastError; + final DateTime createdAt; + final DateTime updatedAt; + + String get idempotencyKey => id; + + bool get isTerminal => switch (status) { + KalamActionStatus.succeeded || + KalamActionStatus.failed || + KalamActionStatus.cancelled => true, + KalamActionStatus.queued || + KalamActionStatus.running || + KalamActionStatus.retryScheduled => false, + }; +} diff --git a/link/sdks/dart/sync/lib/src/models/kalam_action_status.dart b/link/sdks/dart/sync/lib/src/models/kalam_action_status.dart new file mode 100644 index 000000000..2bf625ae6 --- /dev/null +++ b/link/sdks/dart/sync/lib/src/models/kalam_action_status.dart @@ -0,0 +1,9 @@ +/// Durable lifecycle of a queued action. +enum KalamActionStatus { + queued, + running, + retryScheduled, + succeeded, + failed, + cancelled, +} diff --git a/link/sdks/dart/sync/lib/src/models/kalam_cached_row.dart b/link/sdks/dart/sync/lib/src/models/kalam_cached_row.dart new file mode 100644 index 000000000..5b4bfa0f5 --- /dev/null +++ b/link/sdks/dart/sync/lib/src/models/kalam_cached_row.dart @@ -0,0 +1,7 @@ +/// One server-shaped JSON row in Kalam's local mirror. +final class KalamCachedRow { + const KalamCachedRow({required this.rowKey, required this.valuesJson}); + + final String rowKey; + final String valuesJson; +} diff --git a/link/sdks/dart/sync/lib/src/models/kalam_change.dart b/link/sdks/dart/sync/lib/src/models/kalam_change.dart new file mode 100644 index 000000000..0bf401b7d --- /dev/null +++ b/link/sdks/dart/sync/lib/src/models/kalam_change.dart @@ -0,0 +1,18 @@ +import 'package:kalam_link/kalam_link.dart'; + +enum KalamChangeKind { insert, update, delete } + +/// One ordered backend table change ready to apply to the local cache. +final class KalamChange { + const KalamChange({ + required this.kind, + required this.rowKey, + required this.seq, + this.row, + }); + + final KalamChangeKind kind; + final String rowKey; + final SeqId seq; + final T? row; +} diff --git a/link/sdks/dart/sync/lib/src/models/kalam_checkpoint.dart b/link/sdks/dart/sync/lib/src/models/kalam_checkpoint.dart new file mode 100644 index 000000000..609005a6c --- /dev/null +++ b/link/sdks/dart/sync/lib/src/models/kalam_checkpoint.dart @@ -0,0 +1,16 @@ +import 'package:kalam_link/kalam_link.dart'; + +/// Last server sequence durably committed for one subscription. +final class KalamCheckpoint { + const KalamCheckpoint({ + required this.accountKey, + required this.subscriptionId, + required this.seq, + required this.updatedAt, + }); + + final String accountKey; + final String subscriptionId; + final SeqId seq; + final DateTime updatedAt; +} diff --git a/link/sdks/dart/sync/lib/src/models/kalam_dml_payload.dart b/link/sdks/dart/sync/lib/src/models/kalam_dml_payload.dart new file mode 100644 index 000000000..f2a2f249f --- /dev/null +++ b/link/sdks/dart/sync/lib/src/models/kalam_dml_payload.dart @@ -0,0 +1,18 @@ +enum KalamDmlOperation { insert, update, delete } + +/// One generic direct-DML action; Drift continues to own the row types. +final class KalamDmlPayload { + const KalamDmlPayload({ + required this.operation, + required this.tableId, + required this.keyColumn, + required this.rowKey, + required this.values, + }); + + final KalamDmlOperation operation; + final String tableId; + final String keyColumn; + final Object? rowKey; + final Map values; +} diff --git a/link/sdks/dart/sync/lib/src/models/kalam_optimistic_mutation.dart b/link/sdks/dart/sync/lib/src/models/kalam_optimistic_mutation.dart new file mode 100644 index 000000000..750749e95 --- /dev/null +++ b/link/sdks/dart/sync/lib/src/models/kalam_optimistic_mutation.dart @@ -0,0 +1,9 @@ +import 'kalam_optimistic_row.dart'; + +/// One local row mutation committed atomically with a custom action. +final class KalamOptimisticMutation { + const KalamOptimisticMutation({required this.row, required this.apply}); + + final KalamOptimisticRow row; + final Future Function() apply; +} diff --git a/link/sdks/dart/sync/lib/src/models/kalam_optimistic_row.dart b/link/sdks/dart/sync/lib/src/models/kalam_optimistic_row.dart new file mode 100644 index 000000000..de6a81f72 --- /dev/null +++ b/link/sdks/dart/sync/lib/src/models/kalam_optimistic_row.dart @@ -0,0 +1,18 @@ +import 'kalam_row_sync_state.dart'; + +/// Pending row metadata committed with a durable action. +final class KalamOptimisticRow { + const KalamOptimisticRow({ + required this.tableId, + required this.rowKey, + required this.phase, + this.pendingValuesJson, + this.tombstone = false, + }); + + final String tableId; + final String rowKey; + final KalamRowSyncPhase phase; + final String? pendingValuesJson; + final bool tombstone; +} diff --git a/link/sdks/dart/sync/lib/src/models/kalam_retry_policy.dart b/link/sdks/dart/sync/lib/src/models/kalam_retry_policy.dart new file mode 100644 index 000000000..21ca15d40 --- /dev/null +++ b/link/sdks/dart/sync/lib/src/models/kalam_retry_policy.dart @@ -0,0 +1,36 @@ +/// Retry limits and exponential backoff for a durable action. +final class KalamRetryPolicy { + const KalamRetryPolicy({ + this.maxAttempts = 5, + this.initialDelay = const Duration(seconds: 1), + this.maxDelay = const Duration(minutes: 5), + this.multiplier = 2, + }) : assert(maxAttempts > 0), + assert(multiplier >= 1); + + final int maxAttempts; + final Duration initialDelay; + final Duration maxDelay; + final int multiplier; + + Duration delayForAttempt(int attempt) { + var milliseconds = initialDelay.inMilliseconds; + for (var index = 1; index < attempt; index++) { + milliseconds *= multiplier; + if (milliseconds >= maxDelay.inMilliseconds) return maxDelay; + } + return Duration( + milliseconds: milliseconds.clamp(0, maxDelay.inMilliseconds), + ); + } +} + +/// Marks an action error as non-retryable. +final class KalamPermanentActionException implements Exception { + const KalamPermanentActionException(this.message); + + final String message; + + @override + String toString() => 'KalamPermanentActionException: $message'; +} diff --git a/link/sdks/dart/sync/lib/src/models/kalam_row_overlay.dart b/link/sdks/dart/sync/lib/src/models/kalam_row_overlay.dart new file mode 100644 index 000000000..3bc739a5e --- /dev/null +++ b/link/sdks/dart/sync/lib/src/models/kalam_row_overlay.dart @@ -0,0 +1,16 @@ +import 'kalam_row_sync_state.dart'; + +/// Internal overlay data used to combine backend rows with local intent. +final class KalamRowOverlay { + const KalamRowOverlay({ + required this.rowKey, + required this.sync, + required this.tombstone, + this.pendingValuesJson, + }); + + final String rowKey; + final KalamRowSyncState sync; + final String? pendingValuesJson; + final bool tombstone; +} diff --git a/link/sdks/dart/sync/lib/src/models/kalam_row_sync_state.dart b/link/sdks/dart/sync/lib/src/models/kalam_row_sync_state.dart new file mode 100644 index 000000000..aa2260ded --- /dev/null +++ b/link/sdks/dart/sync/lib/src/models/kalam_row_sync_state.dart @@ -0,0 +1,67 @@ +/// Local synchronization phase for one row. +enum KalamRowSyncPhase { + pending, + sending, + awaitingServerEcho, + pendingDelete, + synced, + failed, +} + +/// SDK-owned metadata associated with a server-shaped row. +final class KalamRowSyncState { + const KalamRowSyncState({ + required this.phase, + this.actionId, + this.attemptCount = 0, + this.nextRetryAt, + this.errorCode, + this.errorMessage, + this.lastServerSeq, + }); + + const KalamRowSyncState.synced({String? lastServerSeq}) + : this(phase: KalamRowSyncPhase.synced, lastServerSeq: lastServerSeq); + + final KalamRowSyncPhase phase; + final String? actionId; + final int attemptCount; + final DateTime? nextRetryAt; + final String? errorCode; + final String? errorMessage; + final String? lastServerSeq; + + bool get isPending => switch (phase) { + KalamRowSyncPhase.pending || + KalamRowSyncPhase.sending || + KalamRowSyncPhase.awaitingServerEcho || + KalamRowSyncPhase.pendingDelete => true, + KalamRowSyncPhase.synced || KalamRowSyncPhase.failed => false, + }; + + bool get isFailed => phase == KalamRowSyncPhase.failed; + bool get isSynced => phase == KalamRowSyncPhase.synced; + + @override + bool operator ==(Object other) { + return other is KalamRowSyncState && + other.phase == phase && + other.actionId == actionId && + other.attemptCount == attemptCount && + other.nextRetryAt == nextRetryAt && + other.errorCode == errorCode && + other.errorMessage == errorMessage && + other.lastServerSeq == lastServerSeq; + } + + @override + int get hashCode => Object.hash( + phase, + actionId, + attemptCount, + nextRetryAt, + errorCode, + errorMessage, + lastServerSeq, + ); +} diff --git a/link/sdks/dart/sync/lib/src/models/kalam_sync_mode.dart b/link/sdks/dart/sync/lib/src/models/kalam_sync_mode.dart new file mode 100644 index 000000000..5ca29e2a8 --- /dev/null +++ b/link/sdks/dart/sync/lib/src/models/kalam_sync_mode.dart @@ -0,0 +1,8 @@ +/// Controls how local writes are handled for a synchronized table. +enum KalamSyncMode { + /// Local writes are applied immediately and queued for KalamDB. + bidirectional, + + /// Backend rows are authoritative; local intent uses custom action overlays. + replicaOnly, +} diff --git a/link/sdks/dart/sync/lib/src/models/kalam_sync_state.dart b/link/sdks/dart/sync/lib/src/models/kalam_sync_state.dart new file mode 100644 index 000000000..42df61d2e --- /dev/null +++ b/link/sdks/dart/sync/lib/src/models/kalam_sync_state.dart @@ -0,0 +1,57 @@ +/// High-level state of the local-first synchronization engine. +enum KalamSyncPhase { offline, catchingUp, flushing, live, paused, error } + +/// Immutable state suitable for Flutter builders and state providers. +final class KalamSyncState { + const KalamSyncState({ + this.phase = KalamSyncPhase.offline, + this.pendingActions = 0, + this.failedActions = 0, + this.lastSuccessfulSync, + this.error, + }); + + final KalamSyncPhase phase; + final int pendingActions; + final int failedActions; + final DateTime? lastSuccessfulSync; + final String? error; + + bool get hasPendingWork => pendingActions > 0; + + KalamSyncState copyWith({ + KalamSyncPhase? phase, + int? pendingActions, + int? failedActions, + DateTime? lastSuccessfulSync, + String? error, + bool clearError = false, + }) { + return KalamSyncState( + phase: phase ?? this.phase, + pendingActions: pendingActions ?? this.pendingActions, + failedActions: failedActions ?? this.failedActions, + lastSuccessfulSync: lastSuccessfulSync ?? this.lastSuccessfulSync, + error: clearError ? null : error ?? this.error, + ); + } + + @override + bool operator ==(Object other) { + return other is KalamSyncState && + other.phase == phase && + other.pendingActions == pendingActions && + other.failedActions == failedActions && + other.lastSuccessfulSync == lastSuccessfulSync && + other.error == error; + } + + @override + int get hashCode => Object.hash( + phase, + pendingActions, + failedActions, + lastSuccessfulSync, + error, + ); +} diff --git a/link/sdks/dart/sync/lib/src/models/kalam_synced_row.dart b/link/sdks/dart/sync/lib/src/models/kalam_synced_row.dart new file mode 100644 index 000000000..8cc9382d6 --- /dev/null +++ b/link/sdks/dart/sync/lib/src/models/kalam_synced_row.dart @@ -0,0 +1,21 @@ +import 'kalam_row_sync_state.dart'; + +/// A Drift-generated row paired with SDK-owned local synchronization state. +final class KalamSyncedRow { + const KalamSyncedRow({required this.value, required this.sync}); + + final T value; + final KalamRowSyncState sync; + + bool get isSynced => sync.isSynced; + + @override + bool operator ==(Object other) { + return other is KalamSyncedRow && + other.value == value && + other.sync == sync; + } + + @override + int get hashCode => Object.hash(value, sync); +} diff --git a/link/sdks/dart/sync/lib/src/store/kalam_sync_store.dart b/link/sdks/dart/sync/lib/src/store/kalam_sync_store.dart new file mode 100644 index 000000000..0ebb7f944 --- /dev/null +++ b/link/sdks/dart/sync/lib/src/store/kalam_sync_store.dart @@ -0,0 +1,566 @@ +import 'dart:async'; + +import 'package:drift/drift.dart'; +import 'package:kalam_link/kalam_link.dart'; + +import '../database/kalam_sync_database.dart'; +import '../models/kalam_action_draft.dart'; +import '../models/kalam_action_record.dart'; +import '../models/kalam_action_status.dart'; +import '../models/kalam_checkpoint.dart'; +import '../models/kalam_cached_row.dart'; +import '../models/kalam_optimistic_row.dart'; +import '../models/kalam_row_overlay.dart'; +import '../models/kalam_row_sync_state.dart'; + +typedef KalamClock = DateTime Function(); + +/// Single-writer access to Kalam's private persistence tables. +final class KalamSyncStore { + KalamSyncStore(this.database, {KalamClock? clock}) + : _clock = clock ?? DateTime.now; + + final KalamSyncDatabase database; + final KalamClock _clock; + + Future enqueue( + KalamActionDraft draft, { + KalamOptimisticRow? optimisticRow, + Future Function()? applyOptimistic, + }) { + return database.transaction(() async { + final now = _clock(); + final maxPosition = database.kalamActions.queuePosition.max(); + final positionQuery = database.selectOnly(database.kalamActions) + ..addColumns([maxPosition]); + final highestPosition = await positionQuery + .map((row) => row.read(maxPosition)) + .getSingle(); + await database + .into(database.kalamActions) + .insert( + KalamActionsCompanion.insert( + id: draft.id, + accountKey: draft.accountKey, + actionKey: draft.actionKey, + version: Value(draft.version), + kind: Value(draft.kind), + payloadJson: draft.payloadJson, + status: KalamActionStatus.queued.name, + orderingKey: Value(draft.orderingKey), + rowTableId: Value(optimisticRow?.tableId), + rowKey: Value(optimisticRow?.rowKey), + queuePosition: Value((highestPosition ?? 0) + 1), + createdAt: now, + updatedAt: now, + ), + ); + + if (optimisticRow != null) { + await database + .into(database.kalamRowStates) + .insertOnConflictUpdate( + KalamRowStatesCompanion.insert( + accountKey: draft.accountKey, + tableId: optimisticRow.tableId, + rowKey: optimisticRow.rowKey, + phase: optimisticRow.phase.name, + actionId: Value(draft.id), + pendingValuesJson: Value(optimisticRow.pendingValuesJson), + tombstone: Value(optimisticRow.tombstone), + updatedAt: now, + ), + ); + } + + await applyOptimistic?.call(); + + final stored = await (database.select( + database.kalamActions, + )..where((row) => row.id.equals(draft.id))).getSingle(); + return _toActionRecord(stored); + }); + } + + Stream> watchActions(String accountKey) { + final query = database.select(database.kalamActions) + ..where((row) => row.accountKey.equals(accountKey)) + ..orderBy([(row) => OrderingTerm.asc(row.queuePosition)]); + return query.watch().map( + (rows) => rows.map(_toActionRecord).toList(growable: false), + ); + } + + Future readAction(String id) async { + final stored = await (database.select( + database.kalamActions, + )..where((row) => row.id.equals(id))).getSingleOrNull(); + return stored == null ? null : _toActionRecord(stored); + } + + /// Returns and marks the oldest eligible action as running. + Future claimNextAction( + DateTime now, { + String? accountKey, + }) { + return database.transaction(() async { + final query = database.select(database.kalamActions) + ..where((row) { + final eligible = + row.status.equals(KalamActionStatus.queued.name) | + (row.status.equals(KalamActionStatus.retryScheduled.name) & + row.nextAttemptAt.isSmallerOrEqualValue(now)); + return accountKey == null + ? eligible + : eligible & row.accountKey.equals(accountKey); + }) + ..orderBy([(row) => OrderingTerm.asc(row.queuePosition)]) + ..limit(1); + final stored = await query.getSingleOrNull(); + if (stored == null) return null; + + await (database.update( + database.kalamActions, + )..where((row) => row.id.equals(stored.id))).write( + KalamActionsCompanion( + status: Value(KalamActionStatus.running.name), + attemptCount: Value(stored.attemptCount + 1), + nextAttemptAt: const Value(null), + updatedAt: Value(now), + ), + ); + if (stored.rowTableId != null && stored.rowKey != null) { + await _updateRowForAction( + stored, + phase: KalamRowSyncPhase.sending, + attemptCount: stored.attemptCount + 1, + now: now, + ); + } + return _toActionRecord( + stored.copyWith( + status: KalamActionStatus.running.name, + attemptCount: stored.attemptCount + 1, + nextAttemptAt: const Value(null), + updatedAt: now, + ), + ); + }); + } + + /// Makes interrupted work eligible after a process/runtime restart. + Future recoverRunningActions(DateTime now, {String? accountKey}) { + return database.transaction(() async { + final query = database.select(database.kalamActions) + ..where( + (row) => accountKey == null + ? row.status.equals(KalamActionStatus.running.name) + : row.status.equals(KalamActionStatus.running.name) & + row.accountKey.equals(accountKey), + ); + final interrupted = await query.get(); + if (interrupted.isEmpty) return 0; + + final update = database.update(database.kalamActions) + ..where((row) => row.status.equals(KalamActionStatus.running.name)); + if (accountKey != null) { + update.where((row) => row.accountKey.equals(accountKey)); + } + final changed = await update.write( + KalamActionsCompanion( + status: Value(KalamActionStatus.queued.name), + updatedAt: Value(now), + ), + ); + for (final action in interrupted) { + await _updateRowForAction( + action, + phase: KalamRowSyncPhase.pending, + now: now, + ); + } + return changed; + }); + } + + Future completeAction(String id, DateTime now) { + return _setActionStatus( + id, + KalamActionStatus.succeeded, + now, + clearError: true, + rowPhase: KalamRowSyncPhase.awaitingServerEcho, + ); + } + + Future failAction(String id, String error, DateTime now) { + return _setActionStatus( + id, + KalamActionStatus.failed, + now, + error: error, + rowPhase: KalamRowSyncPhase.failed, + ); + } + + Future scheduleRetry(String id, String error, DateTime retryAt) { + return _setActionStatus( + id, + KalamActionStatus.retryScheduled, + _clock(), + error: error, + retryAt: retryAt, + rowPhase: KalamRowSyncPhase.pending, + ); + } + + Stream> watchRowOverlays({ + required String accountKey, + required String tableId, + }) { + final query = database.select(database.kalamRowStates) + ..where( + (row) => + row.accountKey.equals(accountKey) & row.tableId.equals(tableId), + ); + return query.watch().map( + (rows) => rows.map(_toRowOverlay).toList(growable: false), + ); + } + + Stream> watchCachedRows({ + required String accountKey, + required String tableId, + }) { + final query = database.select(database.kalamCachedRows) + ..where( + (row) => + row.accountKey.equals(accountKey) & row.tableId.equals(tableId), + ) + ..orderBy([(row) => OrderingTerm.asc(row.rowKey)]); + return query.watch().map( + (rows) => [ + for (final row in rows) + KalamCachedRow(rowKey: row.rowKey, valuesJson: row.valuesJson), + ], + ); + } + + Future upsertCachedRow({ + required String accountKey, + required String tableId, + required String rowKey, + required String valuesJson, + }) { + return database + .into(database.kalamCachedRows) + .insertOnConflictUpdate( + KalamCachedRowsCompanion.insert( + accountKey: accountKey, + tableId: tableId, + rowKey: rowKey, + valuesJson: valuesJson, + updatedAt: _clock(), + ), + ); + } + + Future deleteCachedRow({ + required String accountKey, + required String tableId, + required String rowKey, + }) { + return (database.delete(database.kalamCachedRows)..where( + (row) => + row.accountKey.equals(accountKey) & + row.tableId.equals(tableId) & + row.rowKey.equals(rowKey), + )) + .go(); + } + + Future markRowSynced({ + required String accountKey, + required String tableId, + required String rowKey, + required SeqId seq, + }) { + return database.transaction(() async { + final existing = await _readStoredRowState( + accountKey: accountKey, + tableId: tableId, + rowKey: rowKey, + ); + if (existing?.tombstone ?? false) { + return; + } + await database + .into(database.kalamRowStates) + .insertOnConflictUpdate( + KalamRowStatesCompanion.insert( + accountKey: accountKey, + tableId: tableId, + rowKey: rowKey, + phase: KalamRowSyncPhase.synced.name, + actionId: const Value(null), + attemptCount: const Value(0), + nextRetryAt: const Value(null), + errorCode: const Value(null), + errorMessage: const Value(null), + lastServerSeq: Value(seq.toString()), + pendingValuesJson: const Value(null), + tombstone: const Value(false), + updatedAt: _clock(), + ), + ); + await _completeReconciledAction(existing?.actionId); + }); + } + + Future removeRowState({ + required String accountKey, + required String tableId, + required String rowKey, + }) { + return database.transaction(() async { + final existing = await _readStoredRowState( + accountKey: accountKey, + tableId: tableId, + rowKey: rowKey, + ); + await (database.delete(database.kalamRowStates)..where( + (row) => + row.accountKey.equals(accountKey) & + row.tableId.equals(tableId) & + row.rowKey.equals(rowKey), + )) + .go(); + await _completeReconciledAction(existing?.actionId); + }); + } + + Future readCompletedStep(String actionId, String name) async { + final step = + await (database.select(database.kalamActionSteps)..where( + (row) => row.actionId.equals(actionId) & row.name.equals(name), + )) + .getSingleOrNull(); + return step?.status == KalamActionStatus.succeeded.name + ? step?.resultJson + : null; + } + + Future completeStep(String actionId, String name, String resultJson) { + return database + .into(database.kalamActionSteps) + .insertOnConflictUpdate( + KalamActionStepsCompanion.insert( + actionId: actionId, + name: name, + status: KalamActionStatus.succeeded.name, + resultJson: Value(resultJson), + updatedAt: _clock(), + ), + ); + } + + Future failStep(String actionId, String name, String error) { + return database + .into(database.kalamActionSteps) + .insertOnConflictUpdate( + KalamActionStepsCompanion.insert( + actionId: actionId, + name: name, + status: KalamActionStatus.failed.name, + lastError: Value(error), + updatedAt: _clock(), + ), + ); + } + + Future readCheckpoint({ + required String accountKey, + required String subscriptionId, + }) async { + final stored = + await (database.select(database.kalamCheckpoints)..where( + (row) => + row.accountKey.equals(accountKey) & + row.subscriptionId.equals(subscriptionId), + )) + .getSingleOrNull(); + if (stored == null) return null; + return KalamCheckpoint( + accountKey: stored.accountKey, + subscriptionId: stored.subscriptionId, + seq: SeqId.parse(stored.seq), + updatedAt: stored.updatedAt.toUtc(), + ); + } + + /// Runs [apply] and advances its checkpoint in one SQLite transaction. + /// + /// Returns `false` without invoking [apply] for duplicate or stale events. + Future applyAndCheckpoint({ + required String accountKey, + required String subscriptionId, + required SeqId seq, + required FutureOr Function() apply, + }) { + return database.transaction(() async { + final current = await readCheckpoint( + accountKey: accountKey, + subscriptionId: subscriptionId, + ); + if (current != null && current.seq >= seq) return false; + + await apply(); + final now = _clock(); + await database + .into(database.kalamCheckpoints) + .insertOnConflictUpdate( + KalamCheckpointsCompanion.insert( + accountKey: accountKey, + subscriptionId: subscriptionId, + seq: seq.toString(), + updatedAt: now, + ), + ); + return true; + }); + } + + KalamActionRecord _toActionRecord(StoredAction stored) { + return KalamActionRecord( + id: stored.id, + accountKey: stored.accountKey, + actionKey: stored.actionKey, + version: stored.version, + kind: stored.kind, + payloadJson: stored.payloadJson, + status: KalamActionStatus.values.byName(stored.status), + orderingKey: stored.orderingKey, + rowTableId: stored.rowTableId, + rowKey: stored.rowKey, + attemptCount: stored.attemptCount, + nextAttemptAt: stored.nextAttemptAt?.toUtc(), + lastError: stored.lastError, + createdAt: stored.createdAt.toUtc(), + updatedAt: stored.updatedAt.toUtc(), + ); + } + + Future _setActionStatus( + String id, + KalamActionStatus status, + DateTime now, { + String? error, + DateTime? retryAt, + bool clearError = false, + KalamRowSyncPhase? rowPhase, + }) { + return database.transaction(() async { + final action = await (database.select( + database.kalamActions, + )..where((row) => row.id.equals(id))).getSingleOrNull(); + if (action == null) throw StateError('Unknown Kalam action "$id".'); + if (action.status == KalamActionStatus.succeeded.name || + action.status == KalamActionStatus.cancelled.name) { + return; + } + + await (database.update( + database.kalamActions, + )..where((row) => row.id.equals(id))).write( + KalamActionsCompanion( + status: Value(status.name), + nextAttemptAt: Value(retryAt), + lastError: clearError ? const Value(null) : Value(error), + updatedAt: Value(now), + ), + ); + if (rowPhase != null) { + await _updateRowForAction( + action, + phase: rowPhase, + nextRetryAt: retryAt, + errorMessage: error, + now: now, + ); + } + }); + } + + Future _updateRowForAction( + StoredAction action, { + required KalamRowSyncPhase phase, + required DateTime now, + int? attemptCount, + DateTime? nextRetryAt, + String? errorMessage, + }) async { + if (action.rowTableId == null || action.rowKey == null) return; + await (database.update(database.kalamRowStates)..where( + (row) => + row.accountKey.equals(action.accountKey) & + row.tableId.equals(action.rowTableId!) & + row.rowKey.equals(action.rowKey!), + )) + .write( + KalamRowStatesCompanion( + phase: Value(phase.name), + attemptCount: attemptCount == null + ? const Value.absent() + : Value(attemptCount), + nextRetryAt: Value(nextRetryAt), + errorMessage: Value(errorMessage), + updatedAt: Value(now), + ), + ); + } + + KalamRowOverlay _toRowOverlay(StoredRowState stored) { + return KalamRowOverlay( + rowKey: stored.rowKey, + sync: KalamRowSyncState( + phase: KalamRowSyncPhase.values.byName(stored.phase), + actionId: stored.actionId, + attemptCount: stored.attemptCount, + nextRetryAt: stored.nextRetryAt?.toUtc(), + errorCode: stored.errorCode, + errorMessage: stored.errorMessage, + lastServerSeq: stored.lastServerSeq, + ), + pendingValuesJson: stored.pendingValuesJson, + tombstone: stored.tombstone, + ); + } + + Future _readStoredRowState({ + required String accountKey, + required String tableId, + required String rowKey, + }) { + return (database.select(database.kalamRowStates)..where( + (row) => + row.accountKey.equals(accountKey) & + row.tableId.equals(tableId) & + row.rowKey.equals(rowKey), + )) + .getSingleOrNull(); + } + + Future _completeReconciledAction(String? actionId) async { + if (actionId == null) return; + await (database.update( + database.kalamActions, + )..where((row) => row.id.equals(actionId))).write( + KalamActionsCompanion( + status: Value(KalamActionStatus.succeeded.name), + nextAttemptAt: const Value(null), + lastError: const Value(null), + updatedAt: Value(_clock()), + ), + ); + } +} diff --git a/link/sdks/dart/sync/lib/src/sync/kalam_event_consumer.dart b/link/sdks/dart/sync/lib/src/sync/kalam_event_consumer.dart new file mode 100644 index 000000000..5e5ed1462 --- /dev/null +++ b/link/sdks/dart/sync/lib/src/sync/kalam_event_consumer.dart @@ -0,0 +1,20 @@ +import 'dart:async'; + +import '../transport/kalam_remote_change.dart'; + +typedef KalamChangeApplier = FutureOr Function(KalamRemoteChange change); + +/// A widget- or service-scoped durable subscription. +final class KalamEventConsumer { + const KalamEventConsumer({ + required this.id, + required this.sql, + required this.apply, + this.batchSize, + }); + + final String id; + final String sql; + final KalamChangeApplier apply; + final int? batchSize; +} diff --git a/link/sdks/dart/sync/lib/src/sync/kalam_sync_coordinator.dart b/link/sdks/dart/sync/lib/src/sync/kalam_sync_coordinator.dart new file mode 100644 index 000000000..1fddb6b46 --- /dev/null +++ b/link/sdks/dart/sync/lib/src/sync/kalam_sync_coordinator.dart @@ -0,0 +1,302 @@ +import 'dart:async'; + +import '../actions/kalam_action_runner.dart'; +import '../models/kalam_action_status.dart'; +import '../models/kalam_action_record.dart'; +import '../models/kalam_sync_state.dart'; +import '../store/kalam_sync_store.dart'; +import '../transport/kalam_remote_batch.dart'; +import '../transport/kalam_sync_transport.dart'; +import 'kalam_event_consumer.dart'; +import 'kalam_sync_subscription.dart'; + +/// Coordinates durable consumers, connection state, and outbox flushing. +final class KalamSyncCoordinator { + KalamSyncCoordinator({ + required this.accountKey, + required this.store, + required this.transport, + required this.actions, + }) { + _connectionSubscription = transport.connectionStates + .asyncMap(_handleConnection) + .listen(null, onError: _onError); + _actionSubscription = store.watchActions(accountKey).listen(_onActions); + } + + final String accountKey; + final KalamSyncStore store; + final KalamSyncTransport transport; + final KalamActionRunner actions; + final Map _consumers = {}; + final StreamController _states = StreamController.broadcast(); + StreamSubscription? _connectionSubscription; + StreamSubscription>? _actionSubscription; + Timer? _retryTimer; + KalamSyncState _state = const KalamSyncState(); + bool _connected = false; + bool _paused = false; + bool _disposed = false; + bool _flushInProgress = false; + bool _connectInProgress = false; + + KalamSyncState get state => _state; + Stream get states async* { + yield _state; + yield* _states.stream; + } + + Future subscribe(KalamEventConsumer consumer) async { + _ensureOpen(); + if (_consumers.containsKey(consumer.id)) { + throw StateError('Consumer "${consumer.id}" is already subscribed.'); + } + final active = _ActiveConsumer(consumer); + _consumers[consumer.id] = active; + if (!_paused) await _open(active); + return KalamSyncSubscription(() async { + _consumers.remove(consumer.id); + active.cancelled = true; + await active.subscription?.cancel(); + }); + } + + Future pause() async { + if (_paused || _disposed) return; + _paused = true; + for (final active in _consumers.values) { + await active.subscription?.cancel(); + active.subscription = null; + } + await transport.pause(); + _emit(_state.copyWith(phase: KalamSyncPhase.paused, clearError: true)); + } + + Future resume() async { + _ensureOpen(); + if (!_paused) return; + _paused = false; + await transport.resume(); + for (final active in _consumers.values) { + await _open(active); + } + _emit( + _state.copyWith( + phase: _connected ? KalamSyncPhase.catchingUp : KalamSyncPhase.offline, + clearError: true, + ), + ); + } + + Future dispose() async { + if (_disposed) return; + _disposed = true; + for (final active in _consumers.values) { + await active.subscription?.cancel(); + } + _consumers.clear(); + await _connectionSubscription?.cancel(); + await _actionSubscription?.cancel(); + _retryTimer?.cancel(); + await _states.close(); + } + + Future _open(_ActiveConsumer active) async { + if (active.subscription != null) return; + final checkpoint = await store.readCheckpoint( + accountKey: accountKey, + subscriptionId: active.consumer.id, + ); + if (_paused || active.isCancelled || _disposed) return; + if (!_connected || _state.phase != KalamSyncPhase.live) { + _emit( + _state.copyWith(phase: KalamSyncPhase.catchingUp, clearError: true), + ); + } + final generation = ++active.generation; + active.subscription = transport + .subscribe( + sql: active.consumer.sql, + subscriptionId: active.consumer.id, + from: checkpoint?.seq, + batchSize: active.consumer.batchSize, + ) + .asyncMap((batch) => _applyBatch(active.consumer, batch)) + .listen( + (_) { + if (_connected) { + _emit( + _state.copyWith( + phase: KalamSyncPhase.live, + lastSuccessfulSync: DateTime.now().toUtc(), + clearError: true, + ), + ); + } + }, + onError: (Object error, StackTrace stackTrace) { + if (active.generation == generation) active.subscription = null; + _onError(error, stackTrace); + }, + onDone: () { + if (active.generation == generation) active.subscription = null; + }, + cancelOnError: true, + ); + } + + Future _applyBatch( + KalamEventConsumer consumer, + KalamRemoteBatch batch, + ) async { + final checkpoint = batch.checkpoint; + if (checkpoint == null) { + await batch.acknowledge(); + return; + } + + final committed = await store.readCheckpoint( + accountKey: accountKey, + subscriptionId: consumer.id, + ); + final changes = batch.changes.toList(growable: false) + ..sort((first, second) => first.seq.compareTo(second.seq)); + await store.applyAndCheckpoint( + accountKey: accountKey, + subscriptionId: consumer.id, + seq: checkpoint, + apply: () async { + var appliedThrough = committed?.seq; + for (final change in changes) { + if (appliedThrough != null && change.seq <= appliedThrough) continue; + await consumer.apply(change); + appliedThrough = change.seq; + } + return null; + }, + ); + await batch.acknowledge(); + } + + Future _handleConnection(KalamTransportConnection connection) async { + _connected = connection == KalamTransportConnection.connected; + if (_paused) return; + if (!_connected) { + for (final active in _consumers.values) { + await active.subscription?.cancel(); + active.subscription = null; + } + _emit(_state.copyWith(phase: KalamSyncPhase.offline)); + return; + } + for (final active in _consumers.values) { + await _open(active); + } + await _flushActions(); + } + + Future _flushActions() async { + if (_flushInProgress || !_connected || _paused) return; + _flushInProgress = true; + _emit(_state.copyWith(phase: KalamSyncPhase.flushing, clearError: true)); + try { + await actions.flush(); + if (_connected && !_paused) { + _emit( + _state.copyWith( + phase: KalamSyncPhase.live, + lastSuccessfulSync: DateTime.now().toUtc(), + clearError: true, + ), + ); + } + } catch (error) { + _onError(error); + } finally { + _flushInProgress = false; + } + } + + void _onActions(List records) { + var pending = 0; + var failed = 0; + for (final record in records) { + final status = record.status; + if (status == KalamActionStatus.failed) { + failed++; + } else if (!record.isTerminal) { + pending++; + } + } + _emit(_state.copyWith(pendingActions: pending, failedActions: failed)); + _scheduleRetry(records); + if (pending > 0 && !_paused) { + if (_connected) { + _flushActions(); + } else { + unawaited(_ensureConnected()); + } + } + } + + Future _ensureConnected() async { + if (_connectInProgress || _connected || _paused || _disposed) return; + _connectInProgress = true; + try { + await transport.ensureConnected(); + } catch (error) { + _onError(error); + } finally { + _connectInProgress = false; + } + } + + void _scheduleRetry(List records) { + _retryTimer?.cancel(); + if (!_connected || _paused) return; + DateTime? earliest; + for (final record in records) { + if (record.status != KalamActionStatus.retryScheduled || + record.nextAttemptAt == null) { + continue; + } + if (earliest == null || record.nextAttemptAt!.isBefore(earliest)) { + earliest = record.nextAttemptAt; + } + } + if (earliest == null) return; + final delay = earliest.difference(DateTime.now().toUtc()); + _retryTimer = Timer( + delay.isNegative ? Duration.zero : delay, + _flushActions, + ); + } + + void _onError(Object error, [StackTrace? _]) { + if (_disposed) return; + _emit( + _state.copyWith(phase: KalamSyncPhase.error, error: error.toString()), + ); + } + + void _emit(KalamSyncState next) { + if (_disposed || next == _state) return; + _state = next; + _states.add(next); + } + + void _ensureOpen() { + if (_disposed) throw StateError('KalamSyncCoordinator is disposed.'); + } +} + +final class _ActiveConsumer { + _ActiveConsumer(this.consumer); + + final KalamEventConsumer consumer; + StreamSubscription? subscription; + int generation = 0; + bool cancelled = false; + + bool get isCancelled => cancelled; +} diff --git a/link/sdks/dart/sync/lib/src/sync/kalam_sync_subscription.dart b/link/sdks/dart/sync/lib/src/sync/kalam_sync_subscription.dart new file mode 100644 index 000000000..40ff467e7 --- /dev/null +++ b/link/sdks/dart/sync/lib/src/sync/kalam_sync_subscription.dart @@ -0,0 +1,17 @@ +import 'dart:async'; + +/// Handle returned for a widget- or service-scoped sync consumer. +final class KalamSyncSubscription { + KalamSyncSubscription(FutureOr Function() cancel) : _cancel = cancel; + + final FutureOr Function() _cancel; + bool _isCancelled = false; + + bool get isCancelled => _isCancelled; + + Future cancel() async { + if (_isCancelled) return; + _isCancelled = true; + await _cancel(); + } +} diff --git a/link/sdks/dart/sync/lib/src/tables/kalam_replica_overlay.dart b/link/sdks/dart/sync/lib/src/tables/kalam_replica_overlay.dart new file mode 100644 index 000000000..43302309f --- /dev/null +++ b/link/sdks/dart/sync/lib/src/tables/kalam_replica_overlay.dart @@ -0,0 +1,47 @@ +import 'dart:convert'; + +import '../models/kalam_row_overlay.dart'; +import '../models/kalam_row_sync_state.dart'; +import '../models/kalam_synced_row.dart'; +import 'kalam_table_spec.dart'; + +/// Pure merge of authoritative rows and SDK-owned optimistic row state. +final class KalamReplicaOverlay { + const KalamReplicaOverlay(this.spec); + + final KalamTableSpec spec; + + List> merge( + List backendRows, + List overlays, + ) { + final byKey = { + for (final row in backendRows) spec.keyOf(row): row, + }; + final stateByKey = {}; + + for (final overlay in overlays) { + stateByKey[overlay.rowKey] = overlay.sync; + if (overlay.tombstone) { + byKey.remove(overlay.rowKey); + } else if (overlay.pendingValuesJson != null) { + final json = jsonDecode(overlay.pendingValuesJson!); + if (json is! Map) { + throw FormatException( + 'Pending values for ${spec.tableId}/${overlay.rowKey} ' + 'must be a JSON object.', + ); + } + byKey[overlay.rowKey] = spec.decode(json); + } + } + + return [ + for (final entry in byKey.entries) + KalamSyncedRow( + value: entry.value, + sync: stateByKey[entry.key] ?? const KalamRowSyncState.synced(), + ), + ]; + } +} diff --git a/link/sdks/dart/sync/lib/src/tables/kalam_table_binding.dart b/link/sdks/dart/sync/lib/src/tables/kalam_table_binding.dart new file mode 100644 index 000000000..38f4f9726 --- /dev/null +++ b/link/sdks/dart/sync/lib/src/tables/kalam_table_binding.dart @@ -0,0 +1,275 @@ +import 'dart:async'; +import 'dart:convert'; + +import '../models/kalam_action_draft.dart'; +import '../models/kalam_action_record.dart'; +import '../models/kalam_change.dart'; +import '../models/kalam_optimistic_row.dart'; +import '../models/kalam_optimistic_mutation.dart'; +import '../models/kalam_row_sync_state.dart'; +import '../models/kalam_sync_mode.dart'; +import '../models/kalam_synced_row.dart'; +import '../store/kalam_sync_store.dart'; +import '../sync/kalam_event_consumer.dart'; +import '../transport/kalam_remote_change.dart'; +import 'kalam_replica_overlay.dart'; +import 'kalam_table_spec.dart'; + +typedef KalamRowWatch = Stream> Function(); +typedef KalamRowUpsert = Future Function(T row); +typedef KalamRowDelete = Future Function(String rowKey); + +/// Binds generated Drift row types to Kalam's durable table policy. +final class KalamTableBinding { + KalamTableBinding({ + required this.spec, + required this.accountKey, + required this.store, + KalamRowWatch? watchLocal, + KalamRowUpsert? upsertLocal, + KalamRowDelete? deleteLocal, + }) : _watchLocal = + watchLocal ?? + (() => store + .watchCachedRows(accountKey: accountKey, tableId: spec.tableId) + .map( + (rows) => [ + for (final row in rows) + spec.decode( + Map.from( + jsonDecode(row.valuesJson) as Map, + ), + ), + ], + )), + _upsertLocal = + upsertLocal ?? + ((row) => store.upsertCachedRow( + accountKey: accountKey, + tableId: spec.tableId, + rowKey: spec.keyOf(row), + valuesJson: jsonEncode(spec.encode(row)), + )), + _deleteLocal = + deleteLocal ?? + ((rowKey) => store.deleteCachedRow( + accountKey: accountKey, + tableId: spec.tableId, + rowKey: rowKey, + )), + _overlay = KalamReplicaOverlay(spec); + + static const dmlActionKey = 'kalam.dml'; + + final KalamTableSpec spec; + final String accountKey; + final KalamSyncStore store; + final KalamRowWatch _watchLocal; + final KalamRowUpsert _upsertLocal; + final KalamRowDelete _deleteLocal; + final KalamReplicaOverlay _overlay; + + Stream> watch() => + watchWithSyncState().map((rows) => rows.map((row) => row.value).toList()); + + Stream>> watchWithSyncState() { + return _combine( + _watchLocal(), + store.watchRowOverlays(accountKey: accountKey, tableId: spec.tableId), + _overlay.merge, + ); + } + + /// Creates a durable consumer that starts only when a widget/service uses it. + KalamEventConsumer consumer({required String sql, int? batchSize}) { + return KalamEventConsumer( + id: spec.subscriptionId, + sql: sql, + apply: applyRemoteChange, + batchSize: batchSize, + ); + } + + Future insert(T row, {required String actionId}) { + return _write('insert', row, actionId); + } + + KalamOptimisticMutation optimisticInsert(T row) { + final rowKey = spec.keyOf(row); + final valuesJson = jsonEncode(spec.encode(row)); + return KalamOptimisticMutation( + row: KalamOptimisticRow( + tableId: spec.tableId, + rowKey: rowKey, + phase: KalamRowSyncPhase.pending, + pendingValuesJson: valuesJson, + ), + apply: () => _upsertLocal(row), + ); + } + + KalamOptimisticMutation optimisticDelete(String rowKey) { + return KalamOptimisticMutation( + row: KalamOptimisticRow( + tableId: spec.tableId, + rowKey: rowKey, + phase: KalamRowSyncPhase.pendingDelete, + tombstone: true, + ), + apply: () => _deleteLocal(rowKey), + ); + } + + Future update(T row, {required String actionId}) { + return _write('update', row, actionId); + } + + Future delete(String rowKey, {required String actionId}) { + _requireBidirectional(); + final payload = { + 'operation': 'delete', + 'table': spec.tableId, + 'keyColumn': spec.keyColumn, + 'rowKey': rowKey, + }; + return store.enqueue( + KalamActionDraft( + id: actionId, + accountKey: accountKey, + actionKey: dmlActionKey, + kind: 'dml', + payloadJson: jsonEncode(payload), + orderingKey: '${spec.tableId}/$rowKey', + ), + optimisticRow: KalamOptimisticRow( + tableId: spec.tableId, + rowKey: rowKey, + phase: KalamRowSyncPhase.pendingDelete, + tombstone: true, + ), + applyOptimistic: () => _deleteLocal(rowKey), + ); + } + + /// Applies a backend change and its resume checkpoint as one store operation. + Future applyServerChange(KalamChange change) { + return store.applyAndCheckpoint( + accountKey: accountKey, + subscriptionId: spec.subscriptionId, + seq: change.seq, + apply: () => _applyChange(change), + ); + } + + /// Applies one decoded transport row inside the coordinator's transaction. + Future applyRemoteChange(KalamRemoteChange remote) { + final row = spec.decode(remote.row); + return _applyChange( + KalamChange( + kind: remote.kind, + rowKey: spec.keyOf(row), + seq: remote.seq, + row: remote.kind == KalamChangeKind.delete ? null : row, + ), + ); + } + + Future _write(String operation, T row, String actionId) { + _requireBidirectional(); + final rowKey = spec.keyOf(row); + final values = spec.encode(row); + return store.enqueue( + KalamActionDraft( + id: actionId, + accountKey: accountKey, + actionKey: dmlActionKey, + kind: 'dml', + payloadJson: jsonEncode({ + 'operation': operation, + 'table': spec.tableId, + 'keyColumn': spec.keyColumn, + 'rowKey': rowKey, + 'values': values, + }), + orderingKey: '${spec.tableId}/$rowKey', + ), + optimisticRow: KalamOptimisticRow( + tableId: spec.tableId, + rowKey: rowKey, + phase: KalamRowSyncPhase.pending, + pendingValuesJson: jsonEncode(values), + ), + applyOptimistic: () => _upsertLocal(row), + ); + } + + Future _applyChange(KalamChange change) async { + if (change.kind == KalamChangeKind.delete) { + await _deleteLocal(change.rowKey); + await store.removeRowState( + accountKey: accountKey, + tableId: spec.tableId, + rowKey: change.rowKey, + ); + return; + } + final row = change.row; + if (row == null) throw StateError('${change.kind.name} requires a row.'); + await _upsertLocal(row); + await store.markRowSynced( + accountKey: accountKey, + tableId: spec.tableId, + rowKey: change.rowKey, + seq: change.seq, + ); + } + + void _requireBidirectional() { + if (spec.mode == KalamSyncMode.replicaOnly) { + throw StateError( + '${spec.tableId} is replicaOnly. Enqueue a registered custom action ' + 'with an optimistic row instead.', + ); + } + } +} + +Stream _combine( + Stream first, + Stream second, + R Function(A first, B second) combine, +) { + late StreamController controller; + StreamSubscription? firstSubscription; + StreamSubscription? secondSubscription; + A? latestFirst; + B? latestSecond; + var hasFirst = false; + var hasSecond = false; + + void emit() { + if (hasFirst && hasSecond) { + controller.add(combine(latestFirst as A, latestSecond as B)); + } + } + + controller = StreamController( + onListen: () { + firstSubscription = first.listen((value) { + latestFirst = value; + hasFirst = true; + emit(); + }, onError: controller.addError); + secondSubscription = second.listen((value) { + latestSecond = value; + hasSecond = true; + emit(); + }, onError: controller.addError); + }, + onCancel: () async { + await firstSubscription?.cancel(); + await secondSubscription?.cancel(); + }, + ); + return controller.stream; +} diff --git a/link/sdks/dart/sync/lib/src/tables/kalam_table_spec.dart b/link/sdks/dart/sync/lib/src/tables/kalam_table_spec.dart new file mode 100644 index 000000000..b5c407eb3 --- /dev/null +++ b/link/sdks/dart/sync/lib/src/tables/kalam_table_spec.dart @@ -0,0 +1,22 @@ +import '../models/kalam_sync_mode.dart'; + +/// Generated metadata and codecs for one Drift-owned application table. +final class KalamTableSpec { + const KalamTableSpec({ + required this.tableId, + required this.keyColumn, + required this.mode, + required this.keyOf, + required this.encode, + required this.decode, + String? subscriptionId, + }) : subscriptionId = subscriptionId ?? tableId; + + final String tableId; + final String keyColumn; + final String subscriptionId; + final KalamSyncMode mode; + final String Function(T row) keyOf; + final Map Function(T row) encode; + final T Function(Map json) decode; +} diff --git a/link/sdks/dart/sync/lib/src/transport/kalam_link_transport.dart b/link/sdks/dart/sync/lib/src/transport/kalam_link_transport.dart new file mode 100644 index 000000000..ea203d1bb --- /dev/null +++ b/link/sdks/dart/sync/lib/src/transport/kalam_link_transport.dart @@ -0,0 +1,175 @@ +import 'dart:async'; + +import 'package:kalam_link/kalam_link.dart'; + +import '../models/kalam_change.dart'; +import 'kalam_remote_batch.dart'; +import 'kalam_remote_change.dart'; +import 'kalam_sync_transport.dart'; + +/// Adapts the existing shared `kalam_link` socket to the sync engine. +final class KalamLinkTransport implements KalamSyncTransport { + KalamLinkTransport._(this.client, this._connections, this._connection); + + final KalamClient client; + final StreamController _connections; + KalamTransportConnection _connection; + + static Future connect({ + required String url, + AuthProvider authProvider = _anonymousAuth, + }) async { + final connections = StreamController.broadcast(); + var current = KalamTransportConnection.disconnected; + void Function(KalamTransportConnection connection) emit = (connection) { + current = connection; + connections.add(connection); + }; + final client = await KalamClient.connect( + url: url, + authProvider: authProvider, + wsLazyConnect: true, + connectionHandlers: ConnectionHandlers( + onConnect: () => emit(KalamTransportConnection.connected), + onDisconnect: (_) => emit(KalamTransportConnection.disconnected), + onError: (_) => emit(KalamTransportConnection.disconnected), + ), + ); + final transport = KalamLinkTransport._(client, connections, current); + emit = transport._emitConnection; + return transport; + } + + @override + Stream get connectionStates async* { + yield _connection; + yield* _connections.stream; + } + + @override + Stream subscribe({ + required String sql, + required String subscriptionId, + SeqId? from, + int? batchSize, + }) { + return client + .liveEventsWithAck( + sql, + subscriptionId: subscriptionId, + from: from, + batchSize: batchSize, + ) + .map((delivery) { + _emitConnection(KalamTransportConnection.connected); + return KalamRemoteBatch( + changes: decodeEvent(delivery.event), + checkpoint: delivery.checkpoint?.lastSeqId, + acknowledge: delivery.acknowledge, + ); + }); + } + + @override + Future ensureConnected() async { + if (!await client.isConnected) await client.reconnectWebSocket(); + if (await client.isConnected) { + _emitConnection(KalamTransportConnection.connected); + } + } + + @override + Future pause() async { + await client.disconnectWebSocket(); + _emitConnection(KalamTransportConnection.disconnected); + } + + @override + Future resume() async { + await client.reconnectWebSocket(); + if (await client.isConnected) { + _emitConnection(KalamTransportConnection.connected); + } + } + + @override + Future dispose() async { + await client.dispose(); + await _connections.close(); + } + + void _emitConnection(KalamTransportConnection connection) { + if (_connection == connection || _connections.isClosed) return; + _connection = connection; + _connections.add(connection); + } + + /// Converts one public kalam_link event into ordered, checkpointable rows. + static List decodeEvent(ChangeEvent event) { + final changes = switch (event) { + AckEvent() => [], + SubscriptionError(:final code, :final message) => + throw KalamSubscriptionException(code, message), + InitialDataBatch(:final rows) => [ + for (final row in rows) + _change(KalamChangeKind.insert, row, initial: true), + ], + InsertEvent(:final rows) => [ + for (final row in rows) _change(KalamChangeKind.insert, row), + ], + UpdateEvent(:final rows, :final oldRows) => [ + for (var index = 0; index < rows.length; index++) + _change( + KalamChangeKind.update, + rows[index], + oldRow: index < oldRows.length ? oldRows[index] : null, + ), + ], + DeleteEvent(:final oldRows) => [ + for (final row in oldRows) _change(KalamChangeKind.delete, row), + ], + }; + changes.sort((first, second) => first.seq.compareTo(second.seq)); + return changes; + } + + static KalamRemoteChange _change( + KalamChangeKind kind, + Map values, { + Map? oldRow, + bool initial = false, + }) { + final seq = values['_seq']?.asSeqId() ?? oldRow?['_seq']?.asSeqId(); + if (seq == null) { + throw const FormatException( + 'A synchronized KalamDB row must include its _seq value.', + ); + } + return KalamRemoteChange( + kind: kind, + seq: seq, + row: _plainRow(values), + oldRow: oldRow == null ? null : _plainRow(oldRow), + initial: initial, + ); + } + + static Map _plainRow(Map values) { + return { + for (final entry in values.entries) + if (entry.key != '_seq') entry.key: entry.value.toJson(), + }; + } +} + +final class KalamSubscriptionException implements Exception { + const KalamSubscriptionException(this.code, this.message); + + final String code; + final String message; + + @override + String toString() => 'KalamSubscriptionException($code): $message'; +} + +Future _anonymousAuth() async => const Auth.none(); diff --git a/link/sdks/dart/sync/lib/src/transport/kalam_remote_batch.dart b/link/sdks/dart/sync/lib/src/transport/kalam_remote_batch.dart new file mode 100644 index 000000000..7217cff52 --- /dev/null +++ b/link/sdks/dart/sync/lib/src/transport/kalam_remote_batch.dart @@ -0,0 +1,22 @@ +import 'dart:async'; + +import 'package:kalam_link/kalam_link.dart'; + +import 'kalam_remote_change.dart'; + +/// One server delivery that is acknowledged only after durable local apply. +final class KalamRemoteBatch { + KalamRemoteBatch({ + required this.changes, + required this.checkpoint, + required Future Function() acknowledge, + }) : _acknowledge = acknowledge; + + final List changes; + final SeqId? checkpoint; + final Future Function() _acknowledge; + Future? _acknowledgement; + + /// Advances transport resume progress. Calls are idempotent. + Future acknowledge() => _acknowledgement ??= _acknowledge(); +} diff --git a/link/sdks/dart/sync/lib/src/transport/kalam_remote_change.dart b/link/sdks/dart/sync/lib/src/transport/kalam_remote_change.dart new file mode 100644 index 000000000..6434a0f02 --- /dev/null +++ b/link/sdks/dart/sync/lib/src/transport/kalam_remote_change.dart @@ -0,0 +1,20 @@ +import 'package:kalam_link/kalam_link.dart'; + +import '../models/kalam_change.dart'; + +/// One decoded row change delivered by a synchronization batch. +final class KalamRemoteChange { + const KalamRemoteChange({ + required this.kind, + required this.seq, + required this.row, + this.oldRow, + this.initial = false, + }); + + final KalamChangeKind kind; + final SeqId seq; + final Map row; + final Map? oldRow; + final bool initial; +} diff --git a/link/sdks/dart/sync/lib/src/transport/kalam_sync_transport.dart b/link/sdks/dart/sync/lib/src/transport/kalam_sync_transport.dart new file mode 100644 index 000000000..64900f9f5 --- /dev/null +++ b/link/sdks/dart/sync/lib/src/transport/kalam_sync_transport.dart @@ -0,0 +1,25 @@ +import 'package:kalam_link/kalam_link.dart'; + +import 'kalam_remote_batch.dart'; + +enum KalamTransportConnection { disconnected, connected } + +/// Small seam that keeps synchronization independent from the socket client. +abstract interface class KalamSyncTransport { + Stream get connectionStates; + + Stream subscribe({ + required String sql, + required String subscriptionId, + SeqId? from, + int? batchSize, + }); + + Future ensureConnected(); + + Future pause(); + + Future resume(); + + Future dispose(); +} diff --git a/link/sdks/dart/sync/pubspec.lock b/link/sdks/dart/sync/pubspec.lock new file mode 100644 index 000000000..d1b9f9e57 --- /dev/null +++ b/link/sdks/dart/sync/pubspec.lock @@ -0,0 +1,778 @@ +# Generated by pub +# See https://dart.dev/tools/pub/glossary#lockfile +packages: + _fe_analyzer_shared: + dependency: transitive + description: + name: _fe_analyzer_shared + sha256: "1b0e6a07425a3e460666e88bf1c949ccc7bb0116ad562ce94a1eca60fe820725" + url: "https://pub.dev" + source: hosted + version: "103.0.0" + analyzer: + dependency: transitive + description: + name: analyzer + sha256: "61c04d0c1bfed555c681ea079519933f071a5a026578ff73c4ff0df2d3462e5e" + url: "https://pub.dev" + source: hosted + version: "13.3.0" + args: + dependency: transitive + description: + name: args + sha256: d0481093c50b1da8910eb0bb301626d4d8eb7284aa739614d2b394ee09e3ea04 + url: "https://pub.dev" + source: hosted + version: "2.7.0" + async: + dependency: transitive + description: + name: async + sha256: e2eb0491ba5ddb6177742d2da23904574082139b07c1e33b8503b9f46f3e1a37 + url: "https://pub.dev" + source: hosted + version: "2.13.1" + boolean_selector: + dependency: transitive + description: + name: boolean_selector + sha256: "8aab1771e1243a5063b8b0ff68042d67334e3feab9e95b9490f9a6ebf73b42ea" + url: "https://pub.dev" + source: hosted + version: "2.1.2" + build: + dependency: transitive + description: + name: build + sha256: b94f5da9ed3d081fd4ecc426c260998b5f4f4eb2f6d89b7e5472edef0b2b2a1b + url: "https://pub.dev" + source: hosted + version: "4.0.10" + build_cli_annotations: + dependency: transitive + description: + name: build_cli_annotations + sha256: e563c2e01de8974566a1998410d3f6f03521788160a02503b0b1f1a46c7b3d95 + url: "https://pub.dev" + source: hosted + version: "2.1.1" + build_config: + dependency: transitive + description: + name: build_config + sha256: "94eaf6708fe64408c632ef2689ca3777b112f9421306ccf4f8c84d7c5c9f83f8" + url: "https://pub.dev" + source: hosted + version: "1.3.2" + build_daemon: + dependency: transitive + description: + name: build_daemon + sha256: "79e05eaf15a48d7230b053a4363b8eaac0cc234bbd0134c3229455481f55cbc6" + url: "https://pub.dev" + source: hosted + version: "4.1.5" + build_runner: + dependency: "direct dev" + description: + name: build_runner + sha256: "90363d438bd9f84a4d5bbb83352251f57d2d9d771bc95a44e6a33fe25fa52774" + url: "https://pub.dev" + source: hosted + version: "2.16.0" + built_collection: + dependency: transitive + description: + name: built_collection + sha256: "376e3dd27b51ea877c28d525560790aee2e6fbb5f20e2f85d5081027d94e2100" + url: "https://pub.dev" + source: hosted + version: "5.1.1" + built_value: + dependency: transitive + description: + name: built_value + sha256: "31b24be6615ec7fcf70b3aa5a7469fe35826485e639a16dd7eb83ba30e4cc6a8" + url: "https://pub.dev" + source: hosted + version: "8.12.7" + characters: + dependency: transitive + description: + name: characters + sha256: faf38497bda5ead2a8c7615f4f7939df04333478bf32e4173fcb06d428b5716b + url: "https://pub.dev" + source: hosted + version: "1.4.1" + charcode: + dependency: transitive + description: + name: charcode + sha256: fb0f1107cac15a5ea6ef0a6ef71a807b9e4267c713bb93e00e92d737cc8dbd8a + url: "https://pub.dev" + source: hosted + version: "1.4.0" + checked_yaml: + dependency: transitive + description: + name: checked_yaml + sha256: "959525d3162f249993882720d52b7e0c833978df229be20702b33d48d91de70f" + url: "https://pub.dev" + source: hosted + version: "2.0.4" + cli_util: + dependency: transitive + description: + name: cli_util + sha256: "71e43f1976c3eae2d9468a5b4d6600bf8cae61508b87f27fa1799228935d48ab" + url: "https://pub.dev" + source: hosted + version: "0.5.2" + clock: + dependency: transitive + description: + name: clock + sha256: fddb70d9b5277016c77a80201021d40a2247104d9f4aa7bab7157b7e3f05b84b + url: "https://pub.dev" + source: hosted + version: "1.1.2" + code_assets: + dependency: transitive + description: + name: code_assets + sha256: bf394f466ba9205f1812a0433b392d6af280f155f56651eda7c18cc32ed493b8 + url: "https://pub.dev" + source: hosted + version: "1.2.1" + collection: + dependency: transitive + description: + name: collection + sha256: "2f5709ae4d3d59dd8f7cd309b4e023046b57d8a6c82130785d2b0e5868084e76" + url: "https://pub.dev" + source: hosted + version: "1.19.1" + convert: + dependency: transitive + description: + name: convert + sha256: b30acd5944035672bc15c6b7a8b47d773e41e2f17de064350988c5d02adb1c68 + url: "https://pub.dev" + source: hosted + version: "3.1.2" + crypto: + dependency: transitive + description: + name: crypto + sha256: c8ea0233063ba03258fbcf2ca4d6dadfefe14f02fab57702265467a19f27fadf + url: "https://pub.dev" + source: hosted + version: "3.0.7" + dart_style: + dependency: transitive + description: + name: dart_style + sha256: "3f88fc9c96c568d631356507355a2d1e983ee000c9ac008bdcb39b5bf53ce777" + url: "https://pub.dev" + source: hosted + version: "3.1.12" + drift: + dependency: "direct main" + description: + name: drift + sha256: "3a3f1f6f905037d7426e4c445854139fd6a3d592135f7c96d7931682b73d16f4" + url: "https://pub.dev" + source: hosted + version: "2.34.3" + drift_dev: + dependency: "direct dev" + description: + name: drift_dev + sha256: "735aad3c34215805c66bd518c8812fa4f83b5534b2c13c4092c970c04a7e9983" + url: "https://pub.dev" + source: hosted + version: "2.34.5" + drift_flutter: + dependency: "direct main" + description: + name: drift_flutter + sha256: "91acf4bee7c3c84467cba46455aa70e5292a3b889f4582645d74f2e5a8c106f2" + url: "https://pub.dev" + source: hosted + version: "0.3.1" + fake_async: + dependency: transitive + description: + name: fake_async + sha256: "5368f224a74523e8d2e7399ea1638b37aecfca824a3cc4dfdf77bf1fa905ac44" + url: "https://pub.dev" + source: hosted + version: "1.3.3" + ffi: + dependency: transitive + description: + name: ffi + sha256: "6d7fd89431262d8f3125e81b50d3847a091d846eafcd4fdb88dd06f36d705a45" + url: "https://pub.dev" + source: hosted + version: "2.2.0" + file: + dependency: transitive + description: + name: file + sha256: a3b4f84adafef897088c160faf7dfffb7696046cb13ae90b508c2cbc95d3b8d4 + url: "https://pub.dev" + source: hosted + version: "7.0.1" + fixnum: + dependency: transitive + description: + name: fixnum + sha256: b6dc7065e46c974bc7c5f143080a6764ec7a4be6da1285ececdc37be96de53be + url: "https://pub.dev" + source: hosted + version: "1.1.1" + flutter: + dependency: "direct main" + description: flutter + source: sdk + version: "0.0.0" + flutter_lints: + dependency: "direct dev" + description: + name: flutter_lints + sha256: "3105dc8492f6183fb076ccf1f351ac3d60564bff92e20bfc4af9cc1651f4e7e1" + url: "https://pub.dev" + source: hosted + version: "6.0.0" + flutter_rust_bridge: + dependency: transitive + description: + name: flutter_rust_bridge + sha256: e87d6b9ee934dcd24a128ccb2bd91905d2d5fe5c06245d6a8f5477d4907a437a + url: "https://pub.dev" + source: hosted + version: "2.12.0" + flutter_test: + dependency: "direct dev" + description: flutter + source: sdk + version: "0.0.0" + freezed_annotation: + dependency: transitive + description: + name: freezed_annotation + sha256: "7294967ff0a6d98638e7acb774aac3af2550777accd8149c90af5b014e6d44d8" + url: "https://pub.dev" + source: hosted + version: "3.1.0" + glob: + dependency: transitive + description: + name: glob + sha256: c3f1ee72c96f8f78935e18aa8cecced9ab132419e8625dc187e1c2408efc20de + url: "https://pub.dev" + source: hosted + version: "2.1.3" + graphs: + dependency: transitive + description: + name: graphs + sha256: "741bbf84165310a68ff28fe9e727332eef1407342fca52759cb21ad8177bb8d0" + url: "https://pub.dev" + source: hosted + version: "2.3.2" + hooks: + dependency: transitive + description: + name: hooks + sha256: "03a564c3704524ee0f7fc56fc621e8796cc2eb8c24d1f1a33b34979815b285c4" + url: "https://pub.dev" + source: hosted + version: "2.1.0" + http_multi_server: + dependency: transitive + description: + name: http_multi_server + sha256: aa6199f908078bb1c5efb8d8638d4ae191aac11b311132c3ef48ce352fb52ef8 + url: "https://pub.dev" + source: hosted + version: "3.2.2" + http_parser: + dependency: transitive + description: + name: http_parser + sha256: "178d74305e7866013777bab2c3d8726205dc5a4dd935297175b19a23a2e66571" + url: "https://pub.dev" + source: hosted + version: "4.1.2" + io: + dependency: transitive + description: + name: io + sha256: dfd5a80599cf0165756e3181807ed3e77daf6dd4137caaad72d0b7931597650b + url: "https://pub.dev" + source: hosted + version: "1.0.5" + jni: + dependency: transitive + description: + name: jni + sha256: f038e58b4dc2c9037f50e233175086337e0b305e356d28211bf55f21c504cbd3 + url: "https://pub.dev" + source: hosted + version: "1.0.3" + jni_flutter: + dependency: transitive + description: + name: jni_flutter + sha256: "7b717011ea40d04fd47c2731d3d1d36eb99eba3435c2753d62489e8c3c9991d5" + url: "https://pub.dev" + source: hosted + version: "1.0.2" + jni_util: + dependency: transitive + description: + name: jni_util + sha256: "1ba86da04a5f2bf18fde2edb235587e70c5b0fc5bd4ba955f46b00942c3fc35f" + url: "https://pub.dev" + source: hosted + version: "1.0.0" + json_annotation: + dependency: transitive + description: + name: json_annotation + sha256: "2a743920d81b7910627f68ee2c9ac1fc0bfee32b9fc3403587d7c6791ca12f80" + url: "https://pub.dev" + source: hosted + version: "4.12.0" + kalam_link: + dependency: "direct main" + description: + path: "../link" + relative: true + source: path + version: "0.5.6-rc.0" + kalam_sync_chat_backend: + dependency: "direct dev" + description: + path: "example/backend" + relative: true + source: path + version: "0.5.6-rc.0" + kalam_sync_generator_example: + dependency: "direct dev" + description: + path: "../generator/example" + relative: true + source: path + version: "0.5.6-rc.0" + leak_tracker: + dependency: transitive + description: + name: leak_tracker + sha256: "33e2e26bdd85a0112ec15400c8cbffea70d0f9c3407491f672a2fad47915e2de" + url: "https://pub.dev" + source: hosted + version: "11.0.2" + leak_tracker_flutter_testing: + dependency: transitive + description: + name: leak_tracker_flutter_testing + sha256: "1dbc140bb5a23c75ea9c4811222756104fbcd1a27173f0c34ca01e16bea473c1" + url: "https://pub.dev" + source: hosted + version: "3.0.10" + leak_tracker_testing: + dependency: transitive + description: + name: leak_tracker_testing + sha256: "8d5a2d49f4a66b49744b23b018848400d23e54caf9463f4eb20df3eb8acb2eb1" + url: "https://pub.dev" + source: hosted + version: "3.0.2" + lints: + dependency: transitive + description: + name: lints + sha256: "12f842a479589fea194fe5c5a3095abc7be0c1f2ddfa9a0e76aed1dbd26a87df" + url: "https://pub.dev" + source: hosted + version: "6.1.0" + logger: + dependency: transitive + description: + name: logger + sha256: "25aee487596a6257655a1e091ec2ae66bc30e7af663592cc3a27e6591e05035c" + url: "https://pub.dev" + source: hosted + version: "2.7.0" + logging: + dependency: transitive + description: + name: logging + sha256: c8245ada5f1717ed44271ed1c26b8ce85ca3228fd2ffdb75468ab01979309d61 + url: "https://pub.dev" + source: hosted + version: "1.3.0" + matcher: + dependency: transitive + description: + name: matcher + sha256: "31bd099b47c10cd1aeb55146a2d46ce0277630ecef3f7dae54ad7873f36696cd" + url: "https://pub.dev" + source: hosted + version: "0.12.20" + material_color_utilities: + dependency: transitive + description: + name: material_color_utilities + sha256: "9c337007e82b1889149c82ed242ed1cb24a66044e30979c44912381e9be4c48b" + url: "https://pub.dev" + source: hosted + version: "0.13.0" + meta: + dependency: transitive + description: + name: meta + sha256: "307249ce4ff29d58a18e97f6345f539382eb9c9c29ecda628900f31de0443dd9" + url: "https://pub.dev" + source: hosted + version: "1.19.0" + mime: + dependency: transitive + description: + name: mime + sha256: "41a20518f0cb1256669420fdba0cd90d21561e560ac240f26ef8322e45bb7ed6" + url: "https://pub.dev" + source: hosted + version: "2.0.0" + native_toolchain_c: + dependency: transitive + description: + name: native_toolchain_c + sha256: a1c26117c48cebe5677b0cf0e33a980a79a7c5577effc86f52e5a0d309cdcb60 + url: "https://pub.dev" + source: hosted + version: "0.19.3" + objective_c: + dependency: transitive + description: + name: objective_c + sha256: b7fb95a6d9a4f009edd63dc5ac69f07420b23a16161c6dd8660290b59c602e8e + url: "https://pub.dev" + source: hosted + version: "9.5.0" + package_config: + dependency: transitive + description: + name: package_config + sha256: f096c55ebb7deb7e384101542bfba8c52696c1b56fca2eb62827989ef2353bbc + url: "https://pub.dev" + source: hosted + version: "2.2.0" + path: + dependency: transitive + description: + name: path + sha256: "75cca69d1490965be98c73ceaea117e8a04dd21217b37b292c9ddbec0d955bc5" + url: "https://pub.dev" + source: hosted + version: "1.9.1" + path_provider: + dependency: transitive + description: + name: path_provider + sha256: a7f4874f987173da295a61c181b8ee71dab59b332a486b391babf26a1b884825 + url: "https://pub.dev" + source: hosted + version: "2.1.6" + path_provider_android: + dependency: transitive + description: + name: path_provider_android + sha256: "69cbd515a62b94d32a7944f086b2f82b4ac40a1d45bebfc00813a430ab2dabcd" + url: "https://pub.dev" + source: hosted + version: "2.3.1" + path_provider_foundation: + dependency: transitive + description: + name: path_provider_foundation + sha256: "2a376b7d6392d80cd3705782d2caa734ca4727776db0b6ec36ef3f1855197699" + url: "https://pub.dev" + source: hosted + version: "2.6.0" + path_provider_linux: + dependency: transitive + description: + name: path_provider_linux + sha256: "58c2005f147315b11e9b4a7bc889cd5203e250cba8e3f012dae259b4972b5c16" + url: "https://pub.dev" + source: hosted + version: "2.2.2" + path_provider_platform_interface: + dependency: transitive + description: + name: path_provider_platform_interface + sha256: "484838772624c3a4b94f1e44a3e19897fee738f2d5c4ce448443b0417f7c9dda" + url: "https://pub.dev" + source: hosted + version: "2.1.3" + path_provider_windows: + dependency: transitive + description: + name: path_provider_windows + sha256: bd6f00dbd873bfb70d0761682da2b3a2c2fccc2b9e84c495821639601d81afe7 + url: "https://pub.dev" + source: hosted + version: "2.3.0" + platform: + dependency: transitive + description: + name: platform + sha256: "5d6b1b0036a5f331ebc77c850ebc8506cbc1e9416c27e59b439f917a902a4984" + url: "https://pub.dev" + source: hosted + version: "3.1.6" + plugin_platform_interface: + dependency: transitive + description: + name: plugin_platform_interface + sha256: "4820fbfdb9478b1ebae27888254d445073732dae3d6ea81f0b7e06d5dedc3f02" + url: "https://pub.dev" + source: hosted + version: "2.1.8" + pool: + dependency: transitive + description: + name: pool + sha256: "978783255c543aa3586a1b3c21f6e9d720eb315376a915872c61ef8b5c20177d" + url: "https://pub.dev" + source: hosted + version: "1.5.2" + pub_semver: + dependency: transitive + description: + name: pub_semver + sha256: "5bfcf68ca79ef689f8990d1160781b4bad40a3bd5e5218ad4076ddb7f4081585" + url: "https://pub.dev" + source: hosted + version: "2.2.0" + pubspec_parse: + dependency: transitive + description: + name: pubspec_parse + sha256: "0560ba233314abbed0a48a2956f7f022cce7c3e1e73df540277da7544cad4082" + url: "https://pub.dev" + source: hosted + version: "1.5.0" + recase: + dependency: transitive + description: + name: recase + sha256: e4eb4ec2dcdee52dcf99cb4ceabaffc631d7424ee55e56f280bc039737f89213 + url: "https://pub.dev" + source: hosted + version: "4.1.0" + record_use: + dependency: transitive + description: + name: record_use + sha256: "3b4ab682aff40175afca6ff090cc5aa7ce9dafe8dfbae1537a6c678e6678baee" + url: "https://pub.dev" + source: hosted + version: "1.1.0" + shelf: + dependency: transitive + description: + name: shelf + sha256: e7dd780a7ffb623c57850b33f43309312fc863fb6aa3d276a754bb299839ef12 + url: "https://pub.dev" + source: hosted + version: "1.4.2" + shelf_web_socket: + dependency: transitive + description: + name: shelf_web_socket + sha256: "3632775c8e90d6c9712f883e633716432a27758216dfb61bd86a8321c0580925" + url: "https://pub.dev" + source: hosted + version: "3.0.0" + sky_engine: + dependency: transitive + description: flutter + source: sdk + version: "0.0.0" + source_gen: + dependency: transitive + description: + name: source_gen + sha256: a603f1fb984a7391ae5978d1b92bfaaa08b350dca5c825256f925818f7943bf5 + url: "https://pub.dev" + source: hosted + version: "4.2.4" + source_span: + dependency: transitive + description: + name: source_span + sha256: "56a02f1f4cd1a2d96303c0144c93bd6d909eea6bee6bf5a0e0b685edbd4c47ab" + url: "https://pub.dev" + source: hosted + version: "1.10.2" + sqlcipher_flutter_libs: + dependency: transitive + description: + name: sqlcipher_flutter_libs + sha256: "38d62d659d2fb8739bf25a42c9a350d1fdd6c29a5a61f13a946778ec75d27929" + url: "https://pub.dev" + source: hosted + version: "0.7.0+eol" + sqlite3: + dependency: transitive + description: + name: sqlite3 + sha256: "64b2c63c8232dd20d14b34105a81ebfd74320442e8451f836179ec89986aa478" + url: "https://pub.dev" + source: hosted + version: "3.5.1" + sqlite3_flutter_libs: + dependency: transitive + description: + name: sqlite3_flutter_libs + sha256: "3ed7553eee7bb368f8950f58ba29f634e06e813c029aff6a0d60862b96de8454" + url: "https://pub.dev" + source: hosted + version: "0.6.0+eol" + sqlparser: + dependency: transitive + description: + name: sqlparser + sha256: "772bb2f6f5bce0631a60f26b57d6e8882e1105c22345b05c163ee6ee5d7ba32d" + url: "https://pub.dev" + source: hosted + version: "0.45.0" + stack_trace: + dependency: transitive + description: + name: stack_trace + sha256: "8b27215b45d22309b5cddda1aa2b19bdfec9df0e765f2de506401c071d38d1b1" + url: "https://pub.dev" + source: hosted + version: "1.12.1" + stream_channel: + dependency: transitive + description: + name: stream_channel + sha256: "969e04c80b8bcdf826f8f16579c7b14d780458bd97f56d107d3950fdbeef059d" + url: "https://pub.dev" + source: hosted + version: "2.1.4" + stream_transform: + dependency: transitive + description: + name: stream_transform + sha256: ad47125e588cfd37a9a7f86c7d6356dde8dfe89d071d293f80ca9e9273a33871 + url: "https://pub.dev" + source: hosted + version: "2.1.1" + string_scanner: + dependency: transitive + description: + name: string_scanner + sha256: "921cd31725b72fe181906c6a94d987c78e3b98c2e205b397ea399d4054872b43" + url: "https://pub.dev" + source: hosted + version: "1.4.1" + term_glyph: + dependency: transitive + description: + name: term_glyph + sha256: "7f554798625ea768a7518313e58f83891c7f5024f88e46e7182a4558850a4b8e" + url: "https://pub.dev" + source: hosted + version: "1.2.2" + test_api: + dependency: transitive + description: + name: test_api + sha256: "2a122cbe059f8b610d3a5415f42e255b6c17b1f21eee1d960f31080237fb4f11" + url: "https://pub.dev" + source: hosted + version: "0.7.12" + typed_data: + dependency: transitive + description: + name: typed_data + sha256: f9049c039ebfeb4cf7a7104a675823cd72dba8297f264b6637062516699fa006 + url: "https://pub.dev" + source: hosted + version: "1.4.0" + vector_math: + dependency: transitive + description: + name: vector_math + sha256: f36f9f3be64c6198714492bb455c11056e33e2f85d9a0b676a48301e44fdcf47 + url: "https://pub.dev" + source: hosted + version: "2.4.2" + vm_service: + dependency: transitive + description: + name: vm_service + sha256: "0016aef94fc66495ac78af5859181e3f3bf2026bd8eecc72b9565601e19ab360" + url: "https://pub.dev" + source: hosted + version: "15.2.0" + watcher: + dependency: transitive + description: + name: watcher + sha256: "1398c9f081a753f9226febe8900fce8f7d0a67163334e1c94a2438339d79d635" + url: "https://pub.dev" + source: hosted + version: "1.2.1" + web: + dependency: transitive + description: + name: web + sha256: "868d88a33d8a87b18ffc05f9f030ba328ffefba92d6c127917a2ba740f9cfe4a" + url: "https://pub.dev" + source: hosted + version: "1.1.1" + web_socket: + dependency: transitive + description: + name: web_socket + sha256: "34d64019aa8e36bf9842ac014bb5d2f5586ca73df5e4d9bf5c936975cae6982c" + url: "https://pub.dev" + source: hosted + version: "1.0.1" + web_socket_channel: + dependency: transitive + description: + name: web_socket_channel + sha256: d645757fb0f4773d602444000a8131ff5d48c9e47adfe9772652dd1a4f2d45c8 + url: "https://pub.dev" + source: hosted + version: "3.0.3" + xdg_directories: + dependency: transitive + description: + name: xdg_directories + sha256: "7a3f37b05d989967cdddcbb571f1ea834867ae2faa29725fd085180e0883aa15" + url: "https://pub.dev" + source: hosted + version: "1.1.0" + yaml: + dependency: transitive + description: + name: yaml + sha256: b9da305ac7c39faa3f030eccd175340f968459dae4af175130b3fc47e40d76ce + url: "https://pub.dev" + source: hosted + version: "3.1.3" +sdks: + dart: ">=3.11.0 <4.0.0" + flutter: ">=3.38.4" diff --git a/link/sdks/dart/sync/pubspec.yaml b/link/sdks/dart/sync/pubspec.yaml new file mode 100644 index 000000000..5e4e67959 --- /dev/null +++ b/link/sdks/dart/sync/pubspec.yaml @@ -0,0 +1,27 @@ +name: kalam_sync +description: Local-first Flutter sync, durable actions, and Drift integration for KalamDB. +version: 0.5.6-rc.0 +homepage: https://github.com/kalamdb/KalamDB +repository: https://github.com/kalamdb/KalamDB + +environment: + sdk: ">=3.10.0 <4.0.0" + flutter: ">=3.38.0" + +dependencies: + drift: ^2.34.3 + drift_flutter: ^0.3.1 + flutter: + sdk: flutter + kalam_link: ">=0.5.6-0 <0.6.0" + +dev_dependencies: + build_runner: ^2.15.3 + drift_dev: ^2.34.5 + flutter_lints: ^6.0.0 + flutter_test: + sdk: flutter + kalam_sync_chat_backend: + path: example/backend + kalam_sync_generator_example: + path: ../generator/example diff --git a/link/sdks/dart/sync/pubspec_overrides.yaml b/link/sdks/dart/sync/pubspec_overrides.yaml new file mode 100644 index 000000000..5e80776e3 --- /dev/null +++ b/link/sdks/dart/sync/pubspec_overrides.yaml @@ -0,0 +1,7 @@ +dependency_overrides: + kalam_link: + path: ../link + kalam_sync_chat_backend: + path: example/backend + kalam_sync_generator_example: + path: ../generator/example diff --git a/link/sdks/dart/sync/test/actions/kalam_action_runner_test.dart b/link/sdks/dart/sync/test/actions/kalam_action_runner_test.dart new file mode 100644 index 000000000..c71edf6d3 --- /dev/null +++ b/link/sdks/dart/sync/test/actions/kalam_action_runner_test.dart @@ -0,0 +1,293 @@ +import 'dart:convert'; + +import 'package:drift/native.dart'; +import 'package:kalam_sync/drift.dart' hide isNotNull, isNull; +import 'package:flutter_test/flutter_test.dart'; +import 'package:kalam_sync/kalam_sync.dart'; + +final class SendMessage { + const SendMessage(this.text); + + final String text; +} + +void main() { + late KalamSyncDatabase database; + late KalamSyncStore store; + final now = DateTime.utc(2026, 8, 16, 12); + + setUp(() { + database = KalamSyncDatabase(NativeDatabase.memory()); + store = KalamSyncStore(database, clock: () => now); + }); + + tearDown(() => database.close()); + + KalamActionDefinition definition({ + required Future Function( + KalamActionContext context, + SendMessage payload, + ) + execute, + KalamRetryPolicy retryPolicy = const KalamRetryPolicy(), + }) { + return KalamActionDefinition( + key: 'messages.send', + codec: KalamActionCodec( + encode: (payload) => {'text': payload.text}, + decode: (json) => SendMessage(json['text']! as String), + ), + retryPolicy: retryPolicy, + execute: execute, + ); + } + + test( + 'encodes typed payloads and keeps the action id as idempotency key', + () async { + late KalamActionContext receivedContext; + late SendMessage receivedPayload; + final runner = KalamActionRunner( + store: store, + accountKey: 'server/user-a', + registry: KalamActionRegistry([ + definition( + execute: (context, payload) async { + receivedContext = context; + receivedPayload = payload; + }, + ), + ]), + clock: () => now, + ); + + final queued = await runner.enqueue( + actionKey: 'messages.send', + actionId: 'action-1', + payload: const SendMessage('hello'), + ); + await runner.flush(); + + expect(jsonDecode(queued.payloadJson), {'text': 'hello'}); + expect(receivedPayload.text, 'hello'); + expect(receivedContext.idempotencyKey, 'action-1'); + expect( + (await store.readAction('action-1'))?.status, + KalamActionStatus.succeeded, + ); + }, + ); + + test('rejects duplicate action keys at registration', () { + final first = definition(execute: (_, _) async {}); + + expect(() => KalamActionRegistry([first, first]), throwsArgumentError); + }); + + test('flushes queued actions in FIFO order', () async { + final calls = []; + final runner = KalamActionRunner( + store: store, + accountKey: 'server/user-a', + registry: KalamActionRegistry([ + definition(execute: (_, payload) async => calls.add(payload.text)), + ]), + clock: () => now, + ); + + await runner.enqueue( + actionKey: 'messages.send', + actionId: 'z-first', + orderingKey: 'conversation-1', + payload: const SendMessage('first'), + ); + await runner.enqueue( + actionKey: 'messages.send', + actionId: 'a-second', + orderingKey: 'conversation-1', + payload: const SendMessage('second'), + ); + + expect(await runner.flush(), 2); + expect(calls, ['first', 'second']); + }); + + test('schedules exponential retry and eventually succeeds', () async { + var attempts = 0; + var current = now; + final runner = KalamActionRunner( + store: store, + accountKey: 'server/user-a', + registry: KalamActionRegistry([ + definition( + retryPolicy: const KalamRetryPolicy( + maxAttempts: 3, + initialDelay: Duration(seconds: 2), + ), + execute: (_, _) async { + attempts++; + if (attempts == 1) throw StateError('offline'); + }, + ), + ]), + clock: () => current, + ); + await runner.enqueue( + actionKey: 'messages.send', + actionId: 'action-1', + payload: const SendMessage('hello'), + ); + + await runner.flush(); + var action = await store.readAction('action-1'); + expect(action?.status, KalamActionStatus.retryScheduled); + expect(action?.attemptCount, 1); + expect(action?.nextAttemptAt, now.add(const Duration(seconds: 2))); + + expect(await runner.flush(), 0); + current = now.add(const Duration(seconds: 2)); + expect(await runner.flush(), 1); + action = await store.readAction('action-1'); + expect(action?.status, KalamActionStatus.succeeded); + }); + + test('permanent errors are not retried', () async { + final runner = KalamActionRunner( + store: store, + accountKey: 'server/user-a', + registry: KalamActionRegistry([ + definition( + execute: (_, _) async { + throw const KalamPermanentActionException('invalid recipient'); + }, + ), + ]), + clock: () => now, + ); + await runner.enqueue( + actionKey: 'messages.send', + actionId: 'action-1', + payload: const SendMessage('hello'), + ); + + await runner.flush(); + + final action = await store.readAction('action-1'); + expect(action?.status, KalamActionStatus.failed); + expect(action?.lastError, contains('invalid recipient')); + }); + + test('recovers a running action after restart', () async { + await store.enqueue( + const KalamActionDraft( + id: 'action-1', + accountKey: 'server/user-a', + actionKey: 'messages.send', + payloadJson: '{"text":"hello"}', + ), + ); + await store.claimNextAction(now); + + var calls = 0; + final restarted = KalamActionRunner( + store: store, + accountKey: 'server/user-a', + registry: KalamActionRegistry([ + definition(execute: (_, _) async => calls++), + ]), + clock: () => now, + ); + + expect(await restarted.flush(), 1); + expect(calls, 1); + expect( + (await store.readAction('action-1'))?.status, + KalamActionStatus.succeeded, + ); + }); + + test('completed named steps are reused when an action retries', () async { + var actionAttempts = 0; + var stepCalls = 0; + var current = now; + final runner = KalamActionRunner( + store: store, + accountKey: 'server/user-a', + registry: KalamActionRegistry([ + definition( + retryPolicy: const KalamRetryPolicy( + maxAttempts: 2, + initialDelay: Duration(seconds: 1), + ), + execute: (context, _) async { + actionAttempts++; + final value = await context.step( + 'upload', + run: (stepIdempotencyKey) async { + expect(stepIdempotencyKey, 'action-1/upload'); + stepCalls++; + return 42; + }, + encode: (value) => value, + decode: (value) => value as int, + ); + expect(value, 42); + if (actionAttempts == 1) throw StateError('send failed'); + }, + ), + ]), + clock: () => current, + ); + await runner.enqueue( + actionKey: 'messages.send', + actionId: 'action-1', + payload: const SendMessage('hello'), + ); + + await runner.flush(); + current = now.add(const Duration(seconds: 1)); + await runner.flush(); + + expect(actionAttempts, 2); + expect(stepCalls, 1); + }); + + test('server echo wins when the action response is lost', () async { + late KalamActionRunner runner; + runner = KalamActionRunner( + store: store, + accountKey: 'server/user-a', + registry: KalamActionRegistry([ + definition( + execute: (_, _) async { + await store.markRowSynced( + accountKey: 'server/user-a', + tableId: 'app.messages', + rowKey: 'message-1', + seq: const SeqId(15), + ); + throw StateError('response lost after commit'); + }, + ), + ]), + clock: () => now, + ); + await runner.enqueue( + actionKey: 'messages.send', + actionId: 'action-1', + payload: const SendMessage('hello'), + optimisticRow: const KalamOptimisticRow( + tableId: 'app.messages', + rowKey: 'message-1', + phase: KalamRowSyncPhase.pending, + ), + ); + + await runner.flush(); + + expect( + (await store.readAction('action-1'))?.status, + KalamActionStatus.succeeded, + ); + }); +} diff --git a/link/sdks/dart/sync/test/database/kalam_sync_database_test.dart b/link/sdks/dart/sync/test/database/kalam_sync_database_test.dart new file mode 100644 index 000000000..6443f8908 --- /dev/null +++ b/link/sdks/dart/sync/test/database/kalam_sync_database_test.dart @@ -0,0 +1,128 @@ +import 'dart:io'; + +import 'package:drift/native.dart'; +import 'package:kalam_sync/drift.dart' hide isNotNull, isNull; +import 'package:flutter_test/flutter_test.dart'; + +void main() { + test('stores durable actions with retry metadata', () async { + final database = KalamSyncDatabase(NativeDatabase.memory()); + addTearDown(database.close); + + final now = DateTime.utc(2026, 8, 16); + await database + .into(database.kalamActions) + .insert( + KalamActionsCompanion.insert( + id: 'action-1', + accountKey: 'server/user-a', + actionKey: 'messages.send@1', + payloadJson: '{"text":"hello"}', + status: 'retryScheduled', + orderingKey: const Value('conversation-1'), + attemptCount: const Value(2), + nextAttemptAt: Value(now.add(const Duration(seconds: 5))), + createdAt: now, + updatedAt: now, + ), + ); + + final action = await database.select(database.kalamActions).getSingle(); + + expect(action.id, 'action-1'); + expect(action.actionKey, 'messages.send@1'); + expect(action.attemptCount, 2); + expect(action.orderingKey, 'conversation-1'); + }); + + test('isolates checkpoints and row states by account', () async { + final database = KalamSyncDatabase(NativeDatabase.memory()); + addTearDown(database.close); + + final now = DateTime.utc(2026, 8, 16); + await database.batch((batch) { + batch.insertAll(database.kalamCheckpoints, [ + KalamCheckpointsCompanion.insert( + accountKey: 'server/user-a', + subscriptionId: 'messages', + seq: '10', + updatedAt: now, + ), + KalamCheckpointsCompanion.insert( + accountKey: 'server/user-b', + subscriptionId: 'messages', + seq: '3', + updatedAt: now, + ), + ]); + batch.insertAll(database.kalamRowStates, [ + KalamRowStatesCompanion.insert( + accountKey: 'server/user-a', + tableId: 'app.messages', + rowKey: 'message-1', + phase: 'pending', + updatedAt: now, + ), + KalamRowStatesCompanion.insert( + accountKey: 'server/user-b', + tableId: 'app.messages', + rowKey: 'message-1', + phase: 'synced', + updatedAt: now, + ), + ]); + }); + + final checkpoints = await database.select(database.kalamCheckpoints).get(); + final states = await database.select(database.kalamRowStates).get(); + + expect(checkpoints, hasLength(2)); + expect(states, hasLength(2)); + expect( + states.map((state) => state.phase), + containsAll(['pending', 'synced']), + ); + }); + + test('persists checkpoints and action steps across reopen', () async { + final directory = await Directory.systemTemp.createTemp('kalam-sync-test-'); + addTearDown(() => directory.delete(recursive: true)); + final file = File('${directory.path}/sync.sqlite'); + final now = DateTime.utc(2026, 8, 16); + + var database = KalamSyncDatabase(NativeDatabase(file)); + await database + .into(database.kalamCheckpoints) + .insert( + KalamCheckpointsCompanion.insert( + accountKey: 'server/user-a', + subscriptionId: 'messages', + seq: '44', + updatedAt: now, + ), + ); + await database + .into(database.kalamActionSteps) + .insert( + KalamActionStepsCompanion.insert( + actionId: 'action-1', + name: 'upload', + status: 'succeeded', + resultJson: const Value('{"fileId":"file-1"}'), + updatedAt: now, + ), + ); + await database.close(); + + database = KalamSyncDatabase(NativeDatabase(file)); + addTearDown(database.close); + + final checkpoint = await database + .select(database.kalamCheckpoints) + .getSingle(); + final step = await database.select(database.kalamActionSteps).getSingle(); + + expect(checkpoint.seq, '44'); + expect(step.resultJson, '{"fileId":"file-1"}'); + }); +} diff --git a/link/sdks/dart/sync/test/e2e/chat_sync_e2e_test.dart b/link/sdks/dart/sync/test/e2e/chat_sync_e2e_test.dart new file mode 100644 index 000000000..42af19ff8 --- /dev/null +++ b/link/sdks/dart/sync/test/e2e/chat_sync_e2e_test.dart @@ -0,0 +1,347 @@ +import 'package:flutter_test/flutter_test.dart'; +import 'package:kalam_sync/kalam_sync.dart'; +import 'package:kalam_sync_generator_example/chat_models.dart'; + +import '../../../link/test/e2e/helpers.dart'; +import 'support/chat_e2e_harness.dart'; + +void main() { + setUpAll(ensureSdkReady); + + test( + 'offlineEnqueueThenReconnect', + () async { + final session = await openChatSession(); + addTearDown(session.close); + await session.subscribe(); + + final conversation = await session.createConversation('Offline inbox'); + expect(conversation.title, 'Offline inbox'); + + await session.kalam.pause(); + final pending = session.sendMessage( + conversationId: conversation.id, + text: 'hello offline', + ); + final queued = await pending; + final optimistic = await waitForRow( + session.messages, + (row) => + row.value.id == queued.id && + row.value.text == 'hello offline' && + row.sync.isPending, + ); + expect(optimistic.sync.phase, KalamRowSyncPhase.pending); + + await session.kalam.resume(); + final synced = await waitForRow( + session.messages, + (row) => row.value.id == queued.id && row.sync.isSynced, + ); + expect(synced.value.status, isNot(ChatDeliveryStatus.pending)); + expect(synced.sync.phase, KalamRowSyncPhase.synced); + + final assistant = await waitForRow( + session.messages, + (row) => + row.value.id == 'ai-${queued.id}' && + row.value.role == ChatMessageRole.assistant && + row.sync.isSynced, + ); + expect(assistant.value.conversationId, conversation.id); + expect(assistant.value.author, 'demo-bot'); + }, + skip: skipIfNoIntegration, + timeout: const Timeout(Duration(seconds: 60)), + ); + + test( + 'replicaOnlyRejectsDirectMutations', + () async { + final session = await openChatSession(); + addTearDown(session.close); + await session.subscribe(); + final conversation = await session.createConversation('Authoritative'); + + expect( + () => session.messages.insert( + ChatMessage( + id: Kalam.id(), + conversationId: conversation.id, + role: ChatMessageRole.user, + author: adminUser, + text: 'nope', + status: ChatDeliveryStatus.sent, + createdAt: DateTime.now().toUtc(), + ), + actionId: Kalam.id(), + ), + throwsA( + isA().having( + (error) => error.toString(), + 'message', + contains('replicaOnly'), + ), + ), + ); + + final sent = await session.sendMessage( + conversationId: conversation.id, + text: 'via generated action', + ); + final user = await waitForRow( + session.messages, + (row) => + row.value.id == sent.id && + row.value.status == ChatDeliveryStatus.delivered && + row.sync.isSynced, + ); + expect(user.value.status, ChatDeliveryStatus.delivered); + + final assistant = await waitForRow( + session.messages, + (row) => row.value.id == 'ai-${sent.id}' && row.sync.isSynced, + ); + await session.markRead(assistant.value); + final read = await waitForRow( + session.messages, + (row) => + row.value.id == assistant.value.id && + row.value.status == ChatDeliveryStatus.read && + row.sync.isSynced, + ); + expect(read.value.readAt, isNotNull); + }, + skip: skipIfNoIntegration, + timeout: const Timeout(Duration(seconds: 60)), + ); + + test( + 'optimisticUiThenServerEcho', + () async { + final session = await openChatSession(); + addTearDown(session.close); + await session.subscribe(); + final conversation = await session.createConversation('Optimistic'); + final phases = {}; + final sub = session.messages.watchWithSyncState().listen((rows) { + for (final row in rows) { + if (row.value.text == 'echo me') { + phases.add(row.sync.phase); + } + } + }); + addTearDown(sub.cancel); + + await session.kalam.pause(); + final actionId = 'echo-${DateTime.now().microsecondsSinceEpoch}'; + final queued = await session.sendMessage( + conversationId: conversation.id, + text: 'echo me', + actionId: actionId, + ); + await waitForRow( + session.messages, + (row) => row.value.id == queued.id && row.sync.isPending, + ); + + await session.kalam.resume(); + final completed = await waitForAction(session.kalam, actionId); + expect(completed.status, KalamActionStatus.succeeded); + final afterRest = await waitForRow( + session.messages, + (row) => + row.value.id == queued.id && + (row.sync.phase == KalamRowSyncPhase.awaitingServerEcho || + row.sync.isSynced), + ); + if (!afterRest.sync.isSynced) { + expect(afterRest.sync.phase, KalamRowSyncPhase.awaitingServerEcho); + } + final synced = await waitForRow( + session.messages, + (row) => row.value.id == queued.id && row.sync.isSynced, + ); + expect(synced.value.text, 'echo me'); + expect(phases, contains(KalamRowSyncPhase.pending)); + expect(phases, contains(KalamRowSyncPhase.synced)); + }, + skip: skipIfNoIntegration, + timeout: const Timeout(Duration(seconds: 60)), + ); + + test( + 'readReceiptsOnlineAndOffline', + () async { + final session = await openChatSession(); + addTearDown(session.close); + await session.subscribe(); + final conversation = await session.createConversation('Receipts'); + + final first = await session.sendMessage( + conversationId: conversation.id, + text: 'please read this', + ); + final firstAssistant = await waitForRow( + session.messages, + (row) => row.value.id == 'ai-${first.id}' && row.sync.isSynced, + ); + await session.markRead(firstAssistant.value); + await waitForRow( + session.messages, + (row) => + row.value.id == firstAssistant.value.id && + row.value.status == ChatDeliveryStatus.read, + ); + + final peer = await openPeer(session); + addTearDown(peer.dispose); + final peerMessages = peer.table( + session.messages.spec, + ); + await peer.subscribe( + peerMessages.consumer(sql: 'SELECT * FROM ${session.messagesTable}'), + ); + await waitForRow( + peerMessages, + (row) => + row.value.id == firstAssistant.value.id && + row.value.status == ChatDeliveryStatus.read, + ); + + await session.kalam.pause(); + final second = await session.sendMessage( + conversationId: conversation.id, + text: 'offline then read', + ); + await session.kalam.resume(); + final secondAssistant = await waitForRow( + session.messages, + (row) => row.value.id == 'ai-${second.id}' && row.sync.isSynced, + ); + + await session.kalam.pause(); + await session.markRead(secondAssistant.value); + await session.kalam.resume(); + await waitForRow( + session.messages, + (row) => + row.value.id == secondAssistant.value.id && + row.value.status == ChatDeliveryStatus.read && + row.sync.isSynced, + ); + await waitForRow( + peerMessages, + (row) => + row.value.id == secondAssistant.value.id && + row.value.status == ChatDeliveryStatus.read, + ); + + await session.markRead(secondAssistant.value); + await waitForRow( + session.messages, + (row) => + row.value.id == secondAssistant.value.id && + row.value.status == ChatDeliveryStatus.read, + ); + }, + skip: skipIfNoIntegration, + timeout: const Timeout(Duration(seconds: 90)), + ); + + test( + 'catchUpAfterGapAndBackpressure', + () async { + final session = await openChatSession(countTransport: true); + addTearDown(session.close); + await session.subscribe(messageBatchSize: 10); + final conversation = await session.createConversation('Catch-up'); + final transport = session.transport!; + final subscriptionId = session.messages.spec.subscriptionId; + + final seed = await session.sendMessage( + conversationId: conversation.id, + text: 'seed checkpoint', + ); + await waitForRow( + session.messages, + (row) => row.value.id == seed.id && row.sync.isSynced, + ); + await waitForCheckpoint(session.kalam, subscriptionId); + await session.kalam.pause().timeout(const Duration(seconds: 10)); + + const burst = 40; + for (var index = 0; index < burst; index++) { + await session.backend.executeSql( + 'INSERT INTO ${session.messagesTable} (' + 'id, conversation_id, role, author, text, status, created_at' + ') VALUES (' + "'burst-$index', " + "${_sql(conversation.id)}, " + "'assistant', " + "'demo-bot', " + "'burst $index', " + "'delivered', " + "${_sql(DateTime.utc(2026, 8, 17, 0, 0, index).toIso8601String())}" + ')', + ); + } + + await session.kalam.resume().timeout(const Duration(seconds: 15)); + final rows = await waitForMessages( + session.messages, + (rows) => + rows + .where((row) => row.value.id.startsWith('burst-')) + .length == + burst, + timeout: const Duration(seconds: 45), + ); + final burstRows = + rows.where((row) => row.value.id.startsWith('burst-')).toList() + ..sort((a, b) => a.value.createdAt.compareTo(b.value.createdAt)); + expect(burstRows, hasLength(burst)); + expect( + burstRows.map((row) => row.value.id).toSet(), + hasLength(burst), + ); + expect( + burstRows.first.value.id, + 'burst-0', + ); + expect(burstRows.last.value.id, 'burst-39'); + expect(burstRows.every((row) => row.sync.isSynced), isTrue); + + final seqs = transport.deliveredSeqs[subscriptionId] ?? const []; + expect(seqs, isNotEmpty); + expect(seqs.toSet(), hasLength(seqs.length), reason: 'duplicate seqs'); + var previous = seqs.first; + for (final seq in seqs.skip(1)) { + expect(seq, greaterThan(previous)); + previous = seq; + } + + final sizes = transport.deliveredBatchSizes[subscriptionId] ?? const []; + await waitForCondition( + () => + (transport.acknowledged[subscriptionId]?.length ?? 0) >= + sizes.where((size) => size > 0).length, + timeout: const Duration(seconds: 10), + poll: const Duration(milliseconds: 20), + ); + expect(transport.acknowledged[subscriptionId], isNotEmpty); + final checkpoint = await waitForCheckpoint( + session.kalam, + subscriptionId, + ); + expect( + checkpoint.seq, + transport.acknowledged[subscriptionId]!.last, + ); + }, + skip: skipIfNoIntegration, + timeout: const Timeout(Duration(seconds: 90)), + ); +} + +String _sql(String value) => "'${value.replaceAll("'", "''")}'"; diff --git a/link/sdks/dart/sync/test/e2e/kalam_sync_e2e_test.dart b/link/sdks/dart/sync/test/e2e/kalam_sync_e2e_test.dart new file mode 100644 index 000000000..09bc23ce5 --- /dev/null +++ b/link/sdks/dart/sync/test/e2e/kalam_sync_e2e_test.dart @@ -0,0 +1,254 @@ +import 'dart:async'; + +import 'package:drift/native.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:kalam_sync/drift.dart' hide isNotNull, isNull; +import 'package:kalam_sync/kalam_sync.dart'; + +import '../../../link/test/e2e/helpers.dart'; + +final class E2eMessage { + const E2eMessage({required this.id, required this.text}); + + final String id; + final String text; +} + +final class MemoryDatabaseFactory implements KalamDatabaseFactory { + @override + Future open(KalamAccountIdentity identity) async { + return KalamSyncDatabase(NativeDatabase.memory()); + } +} + +void main() { + test( + 'queues offline, executes the action, applies the echo, and checkpoints', + () async { + await ensureSdkReady(); + final setup = await connectJwtClient(); + final namespace = uniqueName('sync_e2e'); + final table = '$namespace.messages'; + await setup.query('CREATE NAMESPACE IF NOT EXISTS $namespace'); + final created = await setup.query( + 'CREATE TABLE IF NOT EXISTS $table (' + 'id TEXT PRIMARY KEY, text TEXT NOT NULL)', + ); + expect(created.success, isTrue, reason: created.error?.toString()); + addTearDown(() async { + await setup.query('DROP TABLE IF EXISTS $table'); + await setup.dispose(); + }); + + final localRows = []; + final localChanges = StreamController>.broadcast(); + addTearDown(localChanges.close); + + final send = KalamActionDefinition>( + key: 'messages.send', + codec: KalamActionCodec( + encode: (payload) => payload, + decode: (json) => json, + ), + execute: (context, payload) async { + final response = await setup.query( + 'INSERT INTO $table (id, text) VALUES (\$1, \$2)', + params: [payload['id'], payload['text']], + ); + if (!response.success) { + throw KalamPermanentActionException(response.error.toString()); + } + }, + ); + final kalam = await Kalam.open( + url: serverUrl, + subject: adminUser, + namespace: namespace, + authProvider: () async => Auth.basic(adminUser, adminPass), + databaseFactory: MemoryDatabaseFactory(), + actionDefinitions: [send], + ); + addTearDown(kalam.dispose); + + final wakeAction = await kalam.actions.enqueue( + actionKey: 'messages.send', + actionId: 'wake-${DateTime.now().microsecondsSinceEpoch}', + payload: const {'id': 'message-wake', 'text': 'no active consumer'}, + ); + expect( + (await _waitForAction(kalam, wakeAction.id)).status, + KalamActionStatus.succeeded, + ); + await _waitForServerRow(setup, table, 'message-wake'); + + final binding = KalamTableBinding( + spec: KalamTableSpec( + tableId: table, + keyColumn: 'id', + subscriptionId: '$table/all', + mode: KalamSyncMode.replicaOnly, + keyOf: (row) => row.id, + encode: (row) => {'id': row.id, 'text': row.text}, + decode: (json) => E2eMessage( + id: json['id']! as String, + text: json['text']! as String, + ), + ), + accountKey: kalam.identity.accountKey, + store: kalam.store, + watchLocal: () async* { + yield List.of(localRows); + yield* localChanges.stream; + }, + upsertLocal: (message) async { + localRows.removeWhere((row) => row.id == message.id); + localRows.add(message); + localChanges.add(List.of(localRows)); + }, + deleteLocal: (id) async { + localRows.removeWhere((row) => row.id == id); + localChanges.add(List.of(localRows)); + }, + ); + final subscription = await kalam.subscribe( + binding.consumer(sql: 'SELECT * FROM $table'), + ); + addTearDown(subscription.cancel); + + await kalam.pause(); + final pending = binding.watchWithSyncState().firstWhere( + (rows) => rows.any( + (row) => row.value.id == 'message-1' && row.sync.isPending, + ), + ); + final queued = await kalam.actions.enqueue( + actionKey: 'messages.send', + actionId: 'action-${DateTime.now().microsecondsSinceEpoch}', + payload: const {'id': 'message-1', 'text': 'hello offline'}, + optimisticRow: KalamOptimisticRow( + tableId: table, + rowKey: 'message-1', + phase: KalamRowSyncPhase.pending, + pendingValuesJson: '{"id":"message-1","text":"hello offline"}', + ), + applyOptimistic: () async { + localRows.add( + const E2eMessage(id: 'message-1', text: 'hello offline'), + ); + localChanges.add(List.of(localRows)); + }, + ); + expect(queued.status, KalamActionStatus.queued); + await pending.timeout(const Duration(seconds: 5)); + + final synced = binding.watchWithSyncState().firstWhere( + (rows) => + rows.any((row) => row.value.id == 'message-1' && row.sync.isSynced), + ); + await kalam.resume(); + final rows = await synced.timeout(const Duration(seconds: 15)); + final checkpoint = await _waitForCheckpoint(kalam, '$table/all'); + + final message = rows.singleWhere((row) => row.value.id == 'message-1'); + expect(message.value.text, 'hello offline'); + expect(message.sync.phase, KalamRowSyncPhase.synced); + expect(checkpoint, isNotNull); + + final direct = kalam.table( + KalamTableSpec( + tableId: table, + keyColumn: 'id', + mode: KalamSyncMode.bidirectional, + keyOf: (row) => row.id, + encode: (row) => {'id': row.id, 'text': row.text}, + decode: (json) => E2eMessage( + id: json['id']! as String, + text: json['text']! as String, + ), + ), + ); + final directSynced = direct.watchWithSyncState().firstWhere( + (rows) => + rows.any((row) => row.value.id == 'message-2' && row.sync.isSynced), + ); + await direct.insert( + const E2eMessage(id: 'message-2', text: 'direct DML'), + actionId: Kalam.id(), + ); + final directRows = await directSynced.timeout( + const Duration(seconds: 15), + ); + expect(directRows.any((row) => row.value.id == 'message-2'), isTrue); + + final deleteAction = await direct.delete( + 'message-2', + actionId: Kalam.id(), + ); + final deleted = await _waitForAction(kalam, deleteAction.id); + expect(deleted.status, KalamActionStatus.succeeded); + await _waitForServerDelete(setup, table, 'message-2'); + }, + skip: skipIfNoIntegration, + timeout: const Timeout(Duration(seconds: 45)), + ); +} + +Future _waitForAction(Kalam kalam, String actionId) async { + final deadline = DateTime.now().add(const Duration(seconds: 10)); + while (DateTime.now().isBefore(deadline)) { + final action = await kalam.store.readAction(actionId); + if (action != null && action.isTerminal) return action; + await Future.delayed(const Duration(milliseconds: 20)); + } + throw TimeoutException('Action $actionId did not finish.'); +} + +Future _waitForServerDelete( + KalamClient client, + String table, + String rowId, +) async { + final deadline = DateTime.now().add(const Duration(seconds: 10)); + while (DateTime.now().isBefore(deadline)) { + final remaining = await client.query( + 'SELECT id FROM $table WHERE id = \$1', + params: [rowId], + ); + if (remaining.rows.isEmpty) return; + await Future.delayed(const Duration(milliseconds: 20)); + } + throw TimeoutException('Backend row $rowId was not deleted from $table.'); +} + +Future _waitForServerRow( + KalamClient client, + String table, + String rowId, +) async { + final deadline = DateTime.now().add(const Duration(seconds: 10)); + while (DateTime.now().isBefore(deadline)) { + final rows = await client.query( + 'SELECT id FROM $table WHERE id = \$1', + params: [rowId], + ); + if (rows.rows.isNotEmpty) return; + await Future.delayed(const Duration(milliseconds: 20)); + } + throw TimeoutException('Backend row $rowId was not inserted into $table.'); +} + +Future _waitForCheckpoint( + Kalam kalam, + String subscriptionId, +) async { + final deadline = DateTime.now().add(const Duration(seconds: 5)); + while (DateTime.now().isBefore(deadline)) { + final checkpoint = await kalam.store.readCheckpoint( + accountKey: kalam.identity.accountKey, + subscriptionId: subscriptionId, + ); + if (checkpoint != null) return checkpoint; + await Future.delayed(const Duration(milliseconds: 20)); + } + throw TimeoutException('No checkpoint committed for $subscriptionId.'); +} diff --git a/link/sdks/dart/sync/test/e2e/kalam_sync_scale_e2e_test.dart b/link/sdks/dart/sync/test/e2e/kalam_sync_scale_e2e_test.dart new file mode 100644 index 000000000..fb4d2ecff --- /dev/null +++ b/link/sdks/dart/sync/test/e2e/kalam_sync_scale_e2e_test.dart @@ -0,0 +1,323 @@ +import 'dart:async'; + +import 'package:drift/native.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:kalam_sync/drift.dart' hide isNotNull, isNull; +import 'package:kalam_sync/kalam_sync.dart'; + +import '../../../link/test/e2e/helpers.dart'; + +final class _LargeRow { + const _LargeRow(this.id, this.value); + + final int id; + final String value; +} + +final class _CountingTransport implements KalamSyncTransport { + _CountingTransport(this.delegate); + + final KalamLinkTransport delegate; + final Map> deliveredBatchSizes = {}; + final Map> acknowledged = {}; + + @override + Stream get connectionStates => + delegate.connectionStates; + + @override + Stream subscribe({ + required String sql, + required String subscriptionId, + SeqId? from, + int? batchSize, + }) { + return delegate + .subscribe( + sql: sql, + subscriptionId: subscriptionId, + from: from, + batchSize: batchSize, + ) + .map((batch) { + if (batch.checkpoint != null) { + deliveredBatchSizes + .putIfAbsent(subscriptionId, () => []) + .add(batch.changes.length); + } + return KalamRemoteBatch( + changes: batch.changes, + checkpoint: batch.checkpoint, + acknowledge: () async { + await batch.acknowledge(); + final checkpoint = batch.checkpoint; + if (checkpoint != null) { + acknowledged + .putIfAbsent(subscriptionId, () => []) + .add(checkpoint); + } + }, + ); + }); + } + + @override + Future ensureConnected() => delegate.ensureConnected(); + + @override + Future pause() => delegate.pause(); + + @override + Future resume() => delegate.resume(); + + @override + Future dispose() => delegate.dispose(); +} + +void main() { + test( + 'syncs 10k rows in acknowledged SQLite batches', + () async { + await ensureSdkReady(); + final setup = await connectJwtClient(); + final namespace = uniqueName('sync_scale'); + final table = '$namespace.items'; + await setup.query('CREATE NAMESPACE IF NOT EXISTS $namespace'); + final created = await setup.query( + 'CREATE TABLE $table (id BIGINT PRIMARY KEY, value TEXT NOT NULL)', + ); + expect(created.success, isTrue, reason: created.error?.toString()); + addTearDown(() async { + await setup.query('DROP TABLE IF EXISTS $table'); + await setup.dispose(); + }); + + const rowCount = 10000; + const insertSize = 500; + for (var start = 0; start < rowCount; start += insertSize) { + final values = [ + for (var id = start; id < start + insertSize; id++) + "($id, 'value-$id')", + ].join(', '); + final inserted = await setup.query( + 'INSERT INTO $table (id, value) VALUES $values', + ); + expect(inserted.success, isTrue, reason: inserted.error?.toString()); + } + + final link = await KalamLinkTransport.connect( + url: serverUrl, + authProvider: () async => Auth.basic(adminUser, adminPass), + ); + final transport = _CountingTransport(link); + final database = KalamSyncDatabase(NativeDatabase.memory()); + final kalam = Kalam.fromComponents( + identity: KalamAccountIdentity( + serverUrl: serverUrl, + subject: adminUser, + namespace: namespace, + ), + database: database, + transport: transport, + ); + addTearDown(kalam.dispose); + final subscriptionId = '$table/all'; + final binding = kalam.table( + KalamTableSpec<_LargeRow>( + tableId: table, + keyColumn: 'id', + subscriptionId: subscriptionId, + mode: KalamSyncMode.replicaOnly, + keyOf: (row) => row.id.toString(), + encode: (row) => {'id': row.id, 'value': row.value}, + decode: (json) => _LargeRow( + int.parse(json['id']!.toString()), + json['value']! as String, + ), + ), + ); + + final stopwatch = Stopwatch()..start(); + final allRows = binding.watch().firstWhere( + (rows) => rows.length == rowCount, + ); + final subscription = await kalam.subscribe( + binding.consumer(sql: 'SELECT * FROM $table', batchSize: 250), + ); + addTearDown(subscription.cancel); + final rows = await _withSyncErrors( + kalam, + allRows, + ).timeout(const Duration(seconds: 90)); + stopwatch.stop(); + + final sizes = transport.deliveredBatchSizes[subscriptionId] ?? const []; + await waitForCondition( + () => + (transport.acknowledged[subscriptionId]?.length ?? 0) == + sizes.length, + timeout: const Duration(seconds: 10), + poll: const Duration(milliseconds: 10), + ); + final acknowledgements = transport.acknowledged[subscriptionId]!; + final checkpoint = await kalam.store.readCheckpoint( + accountKey: kalam.identity.accountKey, + subscriptionId: subscriptionId, + ); + // ignore: avoid_print + print( + 'kalam_sync 10k snapshot: ${stopwatch.elapsedMilliseconds / 1000}s, ' + '${sizes.length} batches', + ); + + expect(rows.map((row) => row.id).toSet(), hasLength(rowCount)); + expect(sizes.length, greaterThan(1)); + expect(sizes.fold(0, (sum, size) => sum + size), rowCount); + expect(acknowledgements, hasLength(sizes.length)); + expect(checkpoint?.seq, acknowledgements.last); + }, + skip: skipIfNoIntegration, + timeout: const Timeout(Duration(seconds: 120)), + ); + + test( + 'syncs and checkpoints three tables independently on one connection', + () async { + await ensureSdkReady(); + final setup = await connectJwtClient(); + final namespace = uniqueName('sync_multi'); + final tables = [ + '$namespace.conversations', + '$namespace.messages', + '$namespace.notifications', + ]; + await setup.query('CREATE NAMESPACE IF NOT EXISTS $namespace'); + for (final table in tables) { + final created = await setup.query( + 'CREATE TABLE $table (id BIGINT PRIMARY KEY, value TEXT NOT NULL)', + ); + expect(created.success, isTrue, reason: created.error?.toString()); + await setup.query( + "INSERT INTO $table (id, value) VALUES (1, 'initial')", + ); + } + addTearDown(() async { + for (final table in tables) { + await setup.query('DROP TABLE IF EXISTS $table'); + } + await setup.dispose(); + }); + + final link = await KalamLinkTransport.connect( + url: serverUrl, + authProvider: () async => Auth.basic(adminUser, adminPass), + ); + final transport = _CountingTransport(link); + final kalam = Kalam.fromComponents( + identity: KalamAccountIdentity( + serverUrl: serverUrl, + subject: adminUser, + namespace: namespace, + ), + database: KalamSyncDatabase(NativeDatabase.memory()), + transport: transport, + ); + addTearDown(kalam.dispose); + + final bindings = [ + for (final table in tables) + kalam.table( + KalamTableSpec<_LargeRow>( + tableId: table, + keyColumn: 'id', + subscriptionId: '$table/all', + mode: KalamSyncMode.replicaOnly, + keyOf: (row) => row.id.toString(), + encode: (row) => {'id': row.id, 'value': row.value}, + decode: (json) => _LargeRow( + int.parse(json['id']!.toString()), + json['value']! as String, + ), + ), + ), + ]; + final subscriptions = []; + final appliedChanges = {}; + for (var index = 0; index < tables.length; index++) { + final table = tables[index]; + final initial = bindings[index].watch().firstWhere( + (rows) => rows.length == 1, + ); + final consumer = bindings[index].consumer( + sql: 'SELECT * FROM $table', + batchSize: 2, + ); + subscriptions.add( + await kalam.subscribe( + KalamEventConsumer( + id: consumer.id, + sql: consumer.sql, + batchSize: consumer.batchSize, + apply: (change) async { + appliedChanges[table] = (appliedChanges[table] ?? 0) + 1; + await consumer.apply(change); + }, + ), + ), + ); + await _withSyncErrors( + kalam, + initial, + ).timeout(const Duration(seconds: 15)); + } + addTearDown(() async { + for (final subscription in subscriptions) { + await subscription.cancel(); + } + }); + + final completed = [ + for (final binding in bindings) + binding.watch().firstWhere((rows) => rows.length == 2), + ]; + await Future.wait([ + for (final table in tables) + setup.query("INSERT INTO $table (id, value) VALUES (2, 'live')"), + ]); + final rowsByTable = await _withSyncErrors(kalam, Future.wait(completed)) + .timeout( + const Duration(seconds: 30), + onTimeout: () => throw StateError( + 'multi-table sync stalled: state=${kalam.syncState}, ' + 'phase=${kalam.syncState.phase}, error=${kalam.syncState.error}, ' + 'applied=$appliedChanges, ' + 'batches=${transport.deliveredBatchSizes}, ' + 'acks=${transport.acknowledged}', + ), + ); + + expect(rowsByTable.every((rows) => rows.length == 2), isTrue); + expect(kalam.syncState.phase, KalamSyncPhase.live); + for (final table in tables) { + final id = '$table/all'; + expect(transport.acknowledged[id], isNotEmpty); + expect( + await kalam.store.readCheckpoint( + accountKey: kalam.identity.accountKey, + subscriptionId: id, + ), + isNotNull, + ); + } + }, + skip: skipIfNoIntegration, + timeout: const Timeout(Duration(seconds: 60)), + ); +} + +Future _withSyncErrors(Kalam kalam, Future result) { + final failure = kalam.syncStates + .firstWhere((state) => state.phase == KalamSyncPhase.error) + .then((state) => throw StateError('sync failed: ${state.error}')); + return Future.any([result, failure]); +} diff --git a/link/sdks/dart/sync/test/e2e/support/chat_e2e_harness.dart b/link/sdks/dart/sync/test/e2e/support/chat_e2e_harness.dart new file mode 100644 index 000000000..bcbd80353 --- /dev/null +++ b/link/sdks/dart/sync/test/e2e/support/chat_e2e_harness.dart @@ -0,0 +1,322 @@ +import 'dart:async'; +import 'dart:io'; + +import 'package:drift/native.dart'; +import 'package:kalam_sync/drift.dart' hide isNotNull, isNull; +import 'package:kalam_sync/kalam_sync.dart'; +import 'package:kalam_sync_chat_backend/chat_backend.dart'; +import 'package:kalam_sync_generator_example/chat_http_api.dart'; +import 'package:kalam_sync_generator_example/chat_actions.dart'; +import 'package:kalam_sync_generator_example/chat_models.dart'; +import 'package:kalam_sync_generator_example/chat_tables.dart'; + +import '../../../../link/test/e2e/helpers.dart'; + +final class MemoryDatabaseFactory implements KalamDatabaseFactory { + MemoryDatabaseFactory() { + driftRuntimeOptions.dontWarnAboutMultipleDatabases = true; + } + + @override + Future open(KalamAccountIdentity identity) async { + return KalamSyncDatabase(NativeDatabase.memory()); + } +} + +final class CountingTransport implements KalamSyncTransport { + CountingTransport(this.delegate); + + final KalamLinkTransport delegate; + final Map> deliveredBatchSizes = {}; + final Map> acknowledged = {}; + final Map> deliveredSeqs = {}; + + @override + Stream get connectionStates => + delegate.connectionStates; + + @override + Stream subscribe({ + required String sql, + required String subscriptionId, + SeqId? from, + int? batchSize, + }) { + return delegate + .subscribe( + sql: sql, + subscriptionId: subscriptionId, + from: from, + batchSize: batchSize, + ) + .map((batch) { + if (batch.checkpoint != null) { + deliveredBatchSizes + .putIfAbsent(subscriptionId, () => []) + .add(batch.changes.length); + deliveredSeqs.putIfAbsent(subscriptionId, () => []).addAll( + batch.changes.map((change) => change.seq.value), + ); + } + return KalamRemoteBatch( + changes: batch.changes, + checkpoint: batch.checkpoint, + acknowledge: () async { + await batch.acknowledge(); + final checkpoint = batch.checkpoint; + if (checkpoint != null) { + acknowledged + .putIfAbsent(subscriptionId, () => []) + .add(checkpoint); + } + }, + ); + }); + } + + @override + Future ensureConnected() => delegate.ensureConnected(); + + @override + Future pause() => delegate.pause(); + + @override + Future resume() => delegate.resume(); + + @override + Future dispose() => delegate.dispose(); +} + +final class ChatE2eSession { + ChatE2eSession({ + required this.namespace, + required this.kalam, + required this.backend, + required this.api, + required this.conversations, + required this.messages, + required this.actions, + required this.transport, + }); + + final String namespace; + final Kalam kalam; + final ChatBackend backend; + final ChatHttpApi api; + final KalamTableBinding conversations; + final KalamTableBinding messages; + final ChatActionsQueue actions; + final CountingTransport? transport; + + String get conversationsTable => '$namespace.conversations'; + String get messagesTable => '$namespace.messages'; + + Future subscribe({int? messageBatchSize}) async { + await kalam.subscribe( + conversations.consumer(sql: 'SELECT * FROM $conversationsTable'), + ); + await kalam.subscribe( + messages.consumer( + sql: 'SELECT * FROM $messagesTable', + batchSize: messageBatchSize, + ), + ); + await kalam.transport.ensureConnected(); + } + + Future createConversation(String title) async { + final now = DateTime.now().toUtc(); + final conversation = Conversation( + id: Kalam.id(), + title: title, + createdAt: now, + updatedAt: now, + ); + await conversations.insert(conversation, actionId: Kalam.id()); + await waitForRow( + conversations, + (row) => row.value.id == conversation.id && row.isSynced, + ); + return conversation; + } + + Future sendMessage({ + required String conversationId, + required String text, + String? messageId, + String? actionId, + }) async { + final now = DateTime.now().toUtc(); + final message = ChatMessage( + id: messageId ?? Kalam.id(), + conversationId: conversationId, + role: ChatMessageRole.user, + author: adminUser, + text: text, + status: ChatDeliveryStatus.pending, + createdAt: now, + ); + await actions.sendMessage( + SendMessageArgs( + messageId: message.id, + conversationId: conversationId, + text: text, + createdAt: now, + author: adminUser, + ), + actionId: actionId, + orderingKey: conversationId, + optimistic: messages.optimisticInsert(message), + ); + return message; + } + + Future markRead(ChatMessage message) { + final readAt = DateTime.now().toUtc(); + return actions.markMsgRead( + MarkMessageReadArgs( + messageId: message.id, + conversationId: message.conversationId, + readAt: readAt, + ), + orderingKey: message.conversationId, + optimistic: messages.optimisticInsert( + message.copyWith(status: ChatDeliveryStatus.read, readAt: readAt), + ), + ); + } + + Future close() async { + await kalam.dispose(); + api.close(); + try { + await backend.executeSql('DROP TABLE IF EXISTS $messagesTable'); + await backend.executeSql('DROP TABLE IF EXISTS $conversationsTable'); + } catch (_) {} + await backend.close(); + } +} + +Future openChatSession({ + bool countTransport = false, +}) async { + final namespace = uniqueName('chat_e2e'); + final restPort = await _freePort(); + final backend = ChatBackend( + kalamUrl: serverUrl, + namespace: namespace, + username: adminUser, + password: adminPass, + port: restPort, + ); + await backend.start(); + final api = ChatHttpApi(baseUrl: backend.url); + CountingTransport? counted; + Future authProvider() async => Auth.basic(adminUser, adminPass); + final actionDefinitions = chatActionsDefinitions(ChatActions(api)); + final Kalam kalam; + if (countTransport) { + final link = await KalamLinkTransport.connect( + url: serverUrl, + authProvider: authProvider, + ); + counted = CountingTransport(link); + final identity = KalamAccountIdentity( + serverUrl: serverUrl, + subject: adminUser, + namespace: namespace, + ); + kalam = Kalam.fromComponents( + identity: identity, + database: await MemoryDatabaseFactory().open(identity), + transport: counted, + dmlClient: link.client, + actionDefinitions: actionDefinitions, + ); + } else { + kalam = await Kalam.open( + url: serverUrl, + subject: adminUser, + namespace: namespace, + authProvider: authProvider, + databaseFactory: MemoryDatabaseFactory(), + actionDefinitions: actionDefinitions, + ); + } + return ChatE2eSession( + namespace: namespace, + kalam: kalam, + backend: backend, + api: api, + conversations: kalam.table( + chatConversationsSpec('$namespace.conversations'), + ), + messages: kalam.table(chatMessagesSpec('$namespace.messages')), + actions: ChatActionsQueue(kalam.actions), + transport: counted, + ); +} + +Future openPeer(ChatE2eSession session) { + return Kalam.open( + url: serverUrl, + subject: adminUser, + namespace: session.namespace, + authProvider: () async => Auth.basic(adminUser, adminPass), + databaseFactory: MemoryDatabaseFactory(), + actionDefinitions: chatActionsDefinitions(ChatActions(session.api)), + ); +} + +Future> waitForRow( + KalamTableBinding binding, + bool Function(KalamSyncedRow row) test, { + Duration timeout = const Duration(seconds: 20), +}) async { + final rows = await binding + .watchWithSyncState() + .firstWhere((rows) => rows.any(test)) + .timeout(timeout); + return rows.firstWhere(test); +} + +Future>> waitForMessages( + KalamTableBinding binding, + bool Function(List> rows) test, { + Duration timeout = const Duration(seconds: 20), +}) { + return binding.watchWithSyncState().firstWhere(test).timeout(timeout); +} + +Future waitForAction(Kalam kalam, String actionId) async { + final deadline = DateTime.now().add(const Duration(seconds: 15)); + while (DateTime.now().isBefore(deadline)) { + final action = await kalam.store.readAction(actionId); + if (action != null && action.isTerminal) return action; + await Future.delayed(const Duration(milliseconds: 20)); + } + throw TimeoutException('Action $actionId did not finish.'); +} + +Future waitForCheckpoint( + Kalam kalam, + String subscriptionId, +) async { + final deadline = DateTime.now().add(const Duration(seconds: 15)); + while (DateTime.now().isBefore(deadline)) { + final checkpoint = await kalam.store.readCheckpoint( + accountKey: kalam.identity.accountKey, + subscriptionId: subscriptionId, + ); + if (checkpoint != null) return checkpoint; + await Future.delayed(const Duration(milliseconds: 20)); + } + throw TimeoutException('No checkpoint committed for $subscriptionId.'); +} + +Future _freePort() async { + final socket = await ServerSocket.bind(InternetAddress.loopbackIPv4, 0); + final port = socket.port; + await socket.close(); + return port; +} diff --git a/link/sdks/dart/sync/test/flutter/kalam_scope_test.dart b/link/sdks/dart/sync/test/flutter/kalam_scope_test.dart new file mode 100644 index 000000000..a7e15b7e6 --- /dev/null +++ b/link/sdks/dart/sync/test/flutter/kalam_scope_test.dart @@ -0,0 +1,155 @@ +import 'dart:async'; + +import 'package:drift/native.dart'; +import 'package:flutter/widgets.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:kalam_sync/drift.dart' hide isNotNull, isNull; +import 'package:kalam_sync/kalam_sync.dart'; + +final class ScopeTransport implements KalamSyncTransport { + final connectionController = + StreamController.broadcast(); + var pauseCount = 0; + var resumeCount = 0; + + @override + Stream get connectionStates => + connectionController.stream; + + @override + Future ensureConnected() async {} + + @override + Stream subscribe({ + required String sql, + required String subscriptionId, + SeqId? from, + int? batchSize, + }) => const Stream.empty(); + + @override + Future pause() async => pauseCount++; + + @override + Future resume() async => resumeCount++; + + @override + Future dispose() => connectionController.close(); +} + +void main() { + late Kalam kalam; + late ScopeTransport transport; + + setUp(() { + transport = ScopeTransport(); + kalam = Kalam.fromComponents( + identity: const KalamAccountIdentity( + serverUrl: 'https://db.example.com', + subject: 'user-a', + ), + database: KalamSyncDatabase(NativeDatabase.memory()), + transport: transport, + ); + }); + + tearDown(() => kalam.dispose()); + + testWidgets('provides the account-scoped Kalam session', (tester) async { + late Kalam found; + await tester.pumpWidget( + KalamScope( + kalam: kalam, + child: Builder( + builder: (context) { + found = KalamScope.of(context); + return const SizedBox(); + }, + ), + ), + ); + + expect(found, same(kalam)); + }); + + testWidgets('reads the session during widget initialization', (tester) async { + late Kalam found; + await tester.pumpWidget( + KalamScope( + kalam: kalam, + child: _ReadOnInit(onRead: (value) => found = value), + ), + ); + + expect(found, same(kalam)); + }); + + testWidgets('pauses and resumes sync with the app lifecycle', (tester) async { + await tester.pumpWidget(KalamScope(kalam: kalam, child: const SizedBox())); + + tester.binding.handleAppLifecycleStateChanged(AppLifecycleState.paused); + await tester.pump(); + expect(transport.pauseCount, 1); + expect(kalam.syncState.phase, KalamSyncPhase.paused); + + tester.binding.handleAppLifecycleStateChanged(AppLifecycleState.resumed); + await tester.pump(); + expect(transport.resumeCount, 1); + }); + + test('database identity changes with server, namespace, or subject', () { + const first = KalamAccountIdentity( + serverUrl: 'https://db.example.com', + subject: 'user-a', + ); + const second = KalamAccountIdentity( + serverUrl: 'https://db.example.com', + subject: 'user-b', + ); + + expect(first.databaseName, isNot(second.databaseName)); + expect(first.databaseName, startsWith('kalam_sync_')); + expect( + first.databaseName, + KalamAccountIdentity( + serverUrl: first.serverUrl, + subject: first.subject, + ).databaseName, + ); + }); + + test('generates RFC 4122 version 4 action ids', () { + final first = Kalam.id(); + final second = Kalam.id(); + + expect( + first, + matches( + RegExp( + r'^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$', + ), + ), + ); + expect(second, isNot(first)); + }); +} + +final class _ReadOnInit extends StatefulWidget { + const _ReadOnInit({required this.onRead}); + + final ValueChanged onRead; + + @override + State<_ReadOnInit> createState() => _ReadOnInitState(); +} + +final class _ReadOnInitState extends State<_ReadOnInit> { + @override + void initState() { + super.initState(); + widget.onRead(KalamScope.read(context)); + } + + @override + Widget build(BuildContext context) => const SizedBox(); +} diff --git a/link/sdks/dart/sync/test/integration/offline_restart_test.dart b/link/sdks/dart/sync/test/integration/offline_restart_test.dart new file mode 100644 index 000000000..2111ba263 --- /dev/null +++ b/link/sdks/dart/sync/test/integration/offline_restart_test.dart @@ -0,0 +1,73 @@ +import 'dart:io'; + +import 'package:drift/native.dart'; +import 'package:kalam_sync/drift.dart' hide isNotNull, isNull; +import 'package:flutter_test/flutter_test.dart'; +import 'package:kalam_sync/kalam_sync.dart'; + +void main() { + test( + 'offline action survives restart with the same idempotency key', + () async { + final directory = await Directory.systemTemp.createTemp( + 'kalam_sync_test_', + ); + final file = File('${directory.path}/sync.sqlite'); + final now = DateTime.utc(2026, 8, 16, 12); + var database = KalamSyncDatabase(NativeDatabase(file)); + var store = KalamSyncStore(database, clock: () => now); + + await store.enqueue( + const KalamActionDraft( + id: 'stable-action-id', + accountKey: 'server/user-a', + actionKey: 'messages.send', + payloadJson: '{"text":"hello"}', + ), + optimisticRow: const KalamOptimisticRow( + tableId: 'app.messages', + rowKey: 'message-1', + phase: KalamRowSyncPhase.pending, + pendingValuesJson: '{"id":"message-1","text":"hello"}', + ), + ); + await database.close(); + + database = KalamSyncDatabase(NativeDatabase(file)); + store = KalamSyncStore(database, clock: () => now); + final seenKeys = []; + final runner = KalamActionRunner( + accountKey: 'server/user-a', + store: store, + registry: KalamActionRegistry([ + KalamActionDefinition>( + key: 'messages.send', + codec: KalamActionCodec( + encode: (payload) => payload, + decode: (json) => json, + ), + execute: (context, _) async => seenKeys.add(context.idempotencyKey), + ), + ]), + clock: () => now, + ); + + expect(await runner.flush(), 1); + expect(seenKeys, ['stable-action-id']); + expect( + (await store.readAction('stable-action-id'))?.status, + KalamActionStatus.succeeded, + ); + final overlays = await store + .watchRowOverlays( + accountKey: 'server/user-a', + tableId: 'app.messages', + ) + .first; + expect(overlays.single.sync.phase, KalamRowSyncPhase.awaitingServerEcho); + + await database.close(); + await directory.delete(recursive: true); + }, + ); +} diff --git a/link/sdks/dart/sync/test/kalam_from_components_test.dart b/link/sdks/dart/sync/test/kalam_from_components_test.dart new file mode 100644 index 000000000..b9187cc25 --- /dev/null +++ b/link/sdks/dart/sync/test/kalam_from_components_test.dart @@ -0,0 +1,58 @@ +import 'dart:async'; + +import 'package:drift/native.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:kalam_sync/drift.dart' hide isNotNull, isNull; +import 'package:kalam_sync/kalam_sync.dart'; + +final class _WrapperTransport implements KalamSyncTransport { + @override + Stream get connectionStates => + const Stream.empty(); + + @override + Stream subscribe({ + required String sql, + required String subscriptionId, + SeqId? from, + int? batchSize, + }) => const Stream.empty(); + + @override + Future ensureConnected() async {} + + @override + Future pause() async {} + + @override + Future resume() async {} + + @override + Future dispose() async {} +} + +void main() { + test('wrapper transports do not register DML without an explicit client', () async { + final database = KalamSyncDatabase(NativeDatabase.memory()); + addTearDown(database.close); + final kalam = Kalam.fromComponents( + identity: const KalamAccountIdentity( + serverUrl: 'http://localhost:2900', + subject: 'admin', + namespace: 'app', + ), + database: database, + transport: _WrapperTransport(), + ); + expect( + () => kalam.actions.registry['kalam.dml'], + throwsA( + isA().having( + (error) => error.toString(), + 'message', + contains('kalam.dml'), + ), + ), + ); + }); +} diff --git a/link/sdks/dart/sync/test/models/sync_models_test.dart b/link/sdks/dart/sync/test/models/sync_models_test.dart new file mode 100644 index 000000000..fa315fd00 --- /dev/null +++ b/link/sdks/dart/sync/test/models/sync_models_test.dart @@ -0,0 +1,82 @@ +import 'package:flutter_test/flutter_test.dart'; +import 'package:kalam_sync/kalam_sync.dart'; + +void main() { + group('KalamSyncState', () { + test('starts offline with no pending work', () { + const state = KalamSyncState(); + + expect(state.phase, KalamSyncPhase.offline); + expect(state.pendingActions, 0); + expect(state.failedActions, 0); + expect(state.hasPendingWork, isFalse); + }); + + test('copyWith preserves values and can clear an error', () { + const original = KalamSyncState( + phase: KalamSyncPhase.error, + pendingActions: 2, + failedActions: 1, + error: 'network unavailable', + ); + + final updated = original.copyWith( + phase: KalamSyncPhase.flushing, + clearError: true, + ); + + expect(updated.phase, KalamSyncPhase.flushing); + expect(updated.pendingActions, 2); + expect(updated.failedActions, 1); + expect(updated.error, isNull); + expect(updated.hasPendingWork, isTrue); + }); + }); + + test('sync modes keep direct DML and replicas distinct', () { + expect(KalamSyncMode.values, [ + KalamSyncMode.bidirectional, + KalamSyncMode.replicaOnly, + ]); + }); + + test('synced rows expose Drift values and local sync metadata', () { + const sync = KalamRowSyncState( + phase: KalamRowSyncPhase.pending, + actionId: 'action-1', + attemptCount: 1, + ); + const row = KalamSyncedRow(value: 'hello', sync: sync); + + expect(row.value, 'hello'); + expect(row.sync.phase, KalamRowSyncPhase.pending); + expect(row.isSynced, isFalse); + expect(row, const KalamSyncedRow(value: 'hello', sync: sync)); + }); + + test('row state distinguishes local sync from backend delivery state', () { + const state = KalamRowSyncState( + phase: KalamRowSyncPhase.awaitingServerEcho, + actionId: 'action-2', + lastServerSeq: '42', + ); + + expect(state.isPending, isTrue); + expect(state.isFailed, isFalse); + expect(state.lastServerSeq, '42'); + }); + + test('action status includes durable retry and terminal states', () { + expect( + KalamActionStatus.values, + containsAll([ + KalamActionStatus.queued, + KalamActionStatus.running, + KalamActionStatus.retryScheduled, + KalamActionStatus.succeeded, + KalamActionStatus.failed, + KalamActionStatus.cancelled, + ]), + ); + }); +} diff --git a/link/sdks/dart/sync/test/store/kalam_sync_store_test.dart b/link/sdks/dart/sync/test/store/kalam_sync_store_test.dart new file mode 100644 index 000000000..166d9f4cb --- /dev/null +++ b/link/sdks/dart/sync/test/store/kalam_sync_store_test.dart @@ -0,0 +1,177 @@ +import 'package:drift/native.dart'; +import 'package:kalam_sync/drift.dart' hide isNotNull, isNull; +import 'package:flutter_test/flutter_test.dart'; +import 'package:kalam_sync/kalam_sync.dart'; + +void main() { + late KalamSyncDatabase database; + late KalamSyncStore store; + final now = DateTime.utc(2026, 8, 16, 12); + + setUp(() { + database = KalamSyncDatabase(NativeDatabase.memory()); + store = KalamSyncStore(database, clock: () => now); + }); + + tearDown(() => database.close()); + + test('atomically enqueues an action and optimistic row state', () async { + final action = await store.enqueue( + const KalamActionDraft( + id: 'action-1', + accountKey: 'server/user-a', + actionKey: 'messages.send', + version: 1, + payloadJson: '{"text":"hello"}', + orderingKey: 'conversation-1', + ), + optimisticRow: const KalamOptimisticRow( + tableId: 'app.messages', + rowKey: 'message-1', + phase: KalamRowSyncPhase.pending, + pendingValuesJson: '{"id":"message-1","text":"hello"}', + ), + ); + + final rowState = await database.select(database.kalamRowStates).getSingle(); + + expect(action.status, KalamActionStatus.queued); + expect(action.idempotencyKey, 'action-1'); + expect(action.createdAt, now); + expect(rowState.actionId, 'action-1'); + expect(rowState.pendingValuesJson, contains('message-1')); + }); + + test('rolls back action and row state when optimistic work fails', () async { + await expectLater( + store.enqueue( + const KalamActionDraft( + id: 'action-1', + accountKey: 'server/user-a', + actionKey: 'messages.send', + payloadJson: '{}', + ), + optimisticRow: const KalamOptimisticRow( + tableId: 'app.messages', + rowKey: 'message-1', + phase: KalamRowSyncPhase.pending, + ), + applyOptimistic: () async => throw StateError('write failed'), + ), + throwsStateError, + ); + + expect(await database.select(database.kalamActions).get(), isEmpty); + expect(await database.select(database.kalamRowStates).get(), isEmpty); + }); + + test('watches actions for one account only', () async { + final first = store + .watchActions('server/user-a') + .firstWhere((actions) => actions.isNotEmpty); + + await store.enqueue( + const KalamActionDraft( + id: 'action-a', + accountKey: 'server/user-a', + actionKey: 'messages.send', + payloadJson: '{}', + ), + ); + await store.enqueue( + const KalamActionDraft( + id: 'action-b', + accountKey: 'server/user-b', + actionKey: 'messages.send', + payloadJson: '{}', + ), + ); + + final actions = await first; + + expect(actions.map((action) => action.id), ['action-a']); + }); + + test('applies server work and advances its checkpoint atomically', () async { + final applied = await store.applyAndCheckpoint( + accountKey: 'server/user-a', + subscriptionId: 'messages', + seq: const SeqId(10), + apply: () => database + .into(database.kalamRowStates) + .insert( + KalamRowStatesCompanion.insert( + accountKey: 'server/user-a', + tableId: 'app.messages', + rowKey: 'message-1', + phase: 'synced', + lastServerSeq: const Value('10'), + updatedAt: now, + ), + ), + ); + + final checkpoint = await store.readCheckpoint( + accountKey: 'server/user-a', + subscriptionId: 'messages', + ); + + expect(applied, isTrue); + expect(checkpoint?.seq, const SeqId(10)); + expect(await database.select(database.kalamRowStates).get(), hasLength(1)); + }); + + test('does not apply an event older than the committed checkpoint', () async { + var applyCount = 0; + await store.applyAndCheckpoint( + accountKey: 'server/user-a', + subscriptionId: 'messages', + seq: const SeqId(10), + apply: () async => applyCount++, + ); + + final applied = await store.applyAndCheckpoint( + accountKey: 'server/user-a', + subscriptionId: 'messages', + seq: const SeqId(9), + apply: () async => applyCount++, + ); + + expect(applied, isFalse); + expect(applyCount, 1); + }); + + test('does not advance checkpoint when server apply fails', () async { + await expectLater( + store.applyAndCheckpoint( + accountKey: 'server/user-a', + subscriptionId: 'messages', + seq: const SeqId(10), + apply: () async { + await database + .into(database.kalamRowStates) + .insert( + KalamRowStatesCompanion.insert( + accountKey: 'server/user-a', + tableId: 'app.messages', + rowKey: 'message-1', + phase: 'synced', + updatedAt: now, + ), + ); + throw StateError('decode failed'); + }, + ), + throwsStateError, + ); + + expect( + await store.readCheckpoint( + accountKey: 'server/user-a', + subscriptionId: 'messages', + ), + isNull, + ); + expect(await database.select(database.kalamRowStates).get(), isEmpty); + }); +} diff --git a/link/sdks/dart/sync/test/sync/kalam_sync_coordinator_test.dart b/link/sdks/dart/sync/test/sync/kalam_sync_coordinator_test.dart new file mode 100644 index 000000000..106dc7d74 --- /dev/null +++ b/link/sdks/dart/sync/test/sync/kalam_sync_coordinator_test.dart @@ -0,0 +1,484 @@ +import 'dart:async'; + +import 'package:drift/native.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:kalam_sync/drift.dart' hide isNotNull, isNull; +import 'package:kalam_sync/kalam_sync.dart'; + +final class FakeTransport implements KalamSyncTransport { + final connections = StreamController.broadcast(); + final streams = >{}; + final calls = <({String id, String sql, SeqId? from, int? batchSize})>[]; + var pauseCount = 0; + var resumeCount = 0; + var ensureConnectedCount = 0; + + @override + Stream get connectionStates => connections.stream; + + @override + Future ensureConnected() async => ensureConnectedCount++; + + @override + Stream subscribe({ + required String sql, + required String subscriptionId, + SeqId? from, + int? batchSize, + }) { + calls.add((id: subscriptionId, sql: sql, from: from, batchSize: batchSize)); + return streams + .putIfAbsent(subscriptionId, StreamController.broadcast) + .stream; + } + + @override + Future pause() async => pauseCount++; + + @override + Future resume() async => resumeCount++; + + Future close() async { + await connections.close(); + for (final stream in streams.values) { + await stream.close(); + } + } + + @override + Future dispose() => close(); +} + +void main() { + late KalamSyncDatabase database; + late KalamSyncStore store; + late FakeTransport transport; + late KalamSyncCoordinator coordinator; + final now = DateTime.utc(2026, 8, 16, 12); + + setUp(() { + database = KalamSyncDatabase(NativeDatabase.memory()); + store = KalamSyncStore(database, clock: () => now); + transport = FakeTransport(); + coordinator = KalamSyncCoordinator( + accountKey: 'server/user-a', + store: store, + transport: transport, + actions: KalamActionRunner( + accountKey: 'server/user-a', + store: store, + registry: KalamActionRegistry([]), + clock: () => now, + ), + ); + }); + + tearDown(() async { + await coordinator.dispose(); + await transport.dispose(); + await database.close(); + }); + + test('a custom consumer starts only where it is subscribed', () async { + expect(transport.calls, isEmpty); + + final subscription = await coordinator.subscribe( + KalamEventConsumer( + id: 'conversation-1/events', + sql: 'SELECT * FROM app.events WHERE conversation_id = ?', + apply: (_) async {}, + ), + ); + + expect(transport.calls.single.id, 'conversation-1/events'); + await subscription.cancel(); + expect(subscription.isCancelled, isTrue); + }); + + test('resumes a consumer from its committed local checkpoint', () async { + await store.applyAndCheckpoint( + accountKey: 'server/user-a', + subscriptionId: 'messages', + seq: const SeqId(41), + apply: () => null, + ); + + await coordinator.subscribe( + KalamEventConsumer( + id: 'messages', + sql: 'SELECT * FROM app.messages', + apply: (_) {}, + ), + ); + + expect(transport.calls.single.from, const SeqId(41)); + }); + + test('commits an ordered batch before acknowledging its sequence', () async { + final applied = []; + final acknowledged = Completer(); + await coordinator.subscribe( + KalamEventConsumer( + id: 'messages', + sql: 'SELECT * FROM app.messages', + apply: (change) { + applied.add(change.seq.value); + }, + ), + ); + final events = transport.streams['messages']!; + + events.add( + _batch( + [1, 1, 2], + acknowledge: () async { + expect( + (await store.readCheckpoint( + accountKey: 'server/user-a', + subscriptionId: 'messages', + ))?.seq, + const SeqId(2), + reason: 'transport progress must follow the SQLite commit', + ); + acknowledged.complete(); + }, + ), + ); + await acknowledged.future; + + expect(applied, [1, 2]); + expect( + (await store.readCheckpoint( + accountKey: 'server/user-a', + subscriptionId: 'messages', + ))?.seq, + const SeqId(2), + ); + }); + + test('stops a consumer when apply fails without skipping ahead', () async { + var acknowledgementCount = 0; + final failed = coordinator.states.firstWhere( + (state) => state.phase == KalamSyncPhase.error, + ); + await coordinator.subscribe( + KalamEventConsumer( + id: 'messages', + sql: 'SELECT * FROM app.messages', + apply: (change) { + if (change.seq == const SeqId(1)) throw StateError('decode failed'); + }, + ), + ); + + transport.streams['messages']!.add( + _batch([1, 2], acknowledge: () async => acknowledgementCount++), + ); + await failed; + + expect(acknowledgementCount, 0); + expect( + await store.readCheckpoint( + accountKey: 'server/user-a', + subscriptionId: 'messages', + ), + isNull, + ); + }); + + test('serializes racing batches and acknowledges them in order', () async { + final firstStarted = Completer(); + final releaseFirst = Completer(); + final allAcknowledged = Completer(); + final applied = []; + final acknowledgements = []; + var concurrentApplies = 0; + var maximumConcurrentApplies = 0; + + await coordinator.subscribe( + KalamEventConsumer( + id: 'messages', + sql: 'SELECT * FROM app.messages', + batchSize: 128, + apply: (change) async { + concurrentApplies++; + maximumConcurrentApplies = + maximumConcurrentApplies < concurrentApplies + ? concurrentApplies + : maximumConcurrentApplies; + if (change.seq == const SeqId(1)) { + firstStarted.complete(); + await releaseFirst.future; + } + applied.add(change.seq.value); + concurrentApplies--; + }, + ), + ); + expect(transport.calls.single.batchSize, 128); + + void recordAck(int seq) { + acknowledgements.add(seq); + if (acknowledgements.length == 2) allAcknowledged.complete(); + } + + final events = transport.streams['messages']!; + events + ..add(_batch([1], acknowledge: () async => recordAck(1))) + ..add(_batch([2], acknowledge: () async => recordAck(2))); + await firstStarted.future; + await Future.delayed(Duration.zero); + expect(applied, isEmpty, reason: 'the first transaction is still open'); + + releaseFirst.complete(); + await allAcknowledged.future; + + expect(applied, [1, 2]); + expect(acknowledgements, [1, 2]); + expect(maximumConcurrentApplies, 1); + }); + + test('one table failure does not block another table checkpoint', () async { + final healthyAcknowledged = Completer(); + await coordinator.subscribe( + KalamEventConsumer( + id: 'messages', + sql: 'SELECT * FROM app.messages', + apply: (_) => throw StateError('bad message row'), + ), + ); + await coordinator.subscribe( + KalamEventConsumer( + id: 'conversations', + sql: 'SELECT * FROM app.conversations', + apply: (_) async {}, + ), + ); + + transport.streams['messages']!.add(_batch([3])); + transport.streams['conversations']!.add( + _batch([9], acknowledge: () async => healthyAcknowledged.complete()), + ); + await healthyAcknowledged.future; + + expect( + await store.readCheckpoint( + accountKey: 'server/user-a', + subscriptionId: 'messages', + ), + isNull, + ); + expect( + (await store.readCheckpoint( + accountKey: 'server/user-a', + subscriptionId: 'conversations', + ))?.seq, + const SeqId(9), + ); + }); + + test('applies a multi-row table batch in one SQLite transaction', () async { + final acknowledged = Completer(); + final binding = KalamTableBinding>( + spec: KalamTableSpec( + tableId: 'app.messages', + keyColumn: 'id', + subscriptionId: 'messages', + mode: KalamSyncMode.replicaOnly, + keyOf: (row) => row['id']! as String, + encode: (row) => row, + decode: (json) => json, + ), + accountKey: 'server/user-a', + store: store, + ); + final rowsReady = binding.watch().firstWhere((rows) => rows.length == 2); + await coordinator.subscribe( + binding.consumer(sql: 'SELECT * FROM app.messages', batchSize: 2), + ); + + transport.streams['messages']!.add( + KalamRemoteBatch( + changes: [_change(1), _change(2)], + checkpoint: const SeqId(2), + acknowledge: () async => acknowledged.complete(), + ), + ); + + await acknowledged.future; + expect(await rowsReady, hasLength(2)); + }); + + test( + 'acknowledgement loss resumes from the already committed SQLite checkpoint', + () async { + final applyStarted = Completer(); + final releaseApply = Completer(); + final failed = coordinator.states.firstWhere( + (state) => state.phase == KalamSyncPhase.error, + ); + await coordinator.subscribe( + KalamEventConsumer( + id: 'messages', + sql: 'SELECT * FROM app.messages', + apply: (_) async { + applyStarted.complete(); + await releaseApply.future; + }, + ), + ); + + transport.streams['messages']!.add( + _batch([ + 5, + ], acknowledge: () => throw StateError('connection lost before ack')), + ); + await applyStarted.future; + releaseApply.complete(); + await failed; + + expect( + (await store.readCheckpoint( + accountKey: 'server/user-a', + subscriptionId: 'messages', + ))?.seq, + const SeqId(5), + ); + + transport.connections.add(KalamTransportConnection.connected); + await _eventually(() => transport.calls.length == 2); + + expect(transport.calls.last.from, const SeqId(5)); + }, + timeout: const Timeout(Duration(seconds: 5)), + ); + + test( + 'disconnect during apply resumes from the completed transaction', + () async { + final applyStarted = Completer(); + final releaseApply = Completer(); + var acknowledgementCount = 0; + await coordinator.subscribe( + KalamEventConsumer( + id: 'messages', + sql: 'SELECT * FROM app.messages', + apply: (_) async { + applyStarted.complete(); + await releaseApply.future; + }, + ), + ); + transport.streams['messages']!.add( + _batch([7], acknowledge: () async => acknowledgementCount++), + ); + await applyStarted.future; + + final offline = coordinator.states.firstWhere( + (state) => state.phase == KalamSyncPhase.offline, + ); + transport.connections.add(KalamTransportConnection.disconnected); + await Future.delayed(Duration.zero); + releaseApply.complete(); + await offline; + + expect( + acknowledgementCount, + 0, + reason: 'the disconnected transport cannot be acknowledged', + ); + expect( + (await store.readCheckpoint( + accountKey: 'server/user-a', + subscriptionId: 'messages', + ))?.seq, + const SeqId(7), + ); + + transport.connections.add(KalamTransportConnection.connected); + await _eventually(() => transport.calls.length == 2); + expect(transport.calls.last.from, const SeqId(7)); + }, + timeout: const Timeout(Duration(seconds: 5)), + ); + + test('connection and lifecycle changes are exposed as sync state', () async { + transport.connections.add(KalamTransportConnection.connected); + await coordinator.states.firstWhere( + (state) => state.phase == KalamSyncPhase.live, + ); + + transport.connections.add(KalamTransportConnection.disconnected); + await coordinator.states.firstWhere( + (state) => state.phase == KalamSyncPhase.offline, + ); + + await coordinator.pause(); + expect(coordinator.state.phase, KalamSyncPhase.paused); + expect(transport.pauseCount, 1); + + await coordinator.resume(); + expect(transport.resumeCount, 1); + }); + + test( + 'a late state listener immediately receives the current state', + () async { + transport.connections.add(KalamTransportConnection.connected); + await coordinator.states.firstWhere( + (state) => state.phase == KalamSyncPhase.live, + ); + + expect((await coordinator.states.first).phase, KalamSyncPhase.live); + }, + ); + + test('pending actions wake the connection without a consumer', () async { + final action = await store.enqueue( + const KalamActionDraft( + id: 'action-1', + accountKey: 'server/user-a', + actionKey: 'messages.send', + payloadJson: '{}', + ), + ); + + await coordinator.store + .watchActions('server/user-a') + .firstWhere( + (records) => records.any((record) => record.id == action.id), + ); + await Future.delayed(Duration.zero); + + expect(transport.ensureConnectedCount, 1); + }); +} + +KalamRemoteChange _change(int seq) { + return KalamRemoteChange( + kind: KalamChangeKind.insert, + seq: SeqId(seq), + row: {'id': 'message-$seq', '_seq': seq}, + ); +} + +KalamRemoteBatch _batch( + List sequences, { + Future Function()? acknowledge, +}) { + return KalamRemoteBatch( + changes: [for (final seq in sequences) _change(seq)], + checkpoint: sequences.isEmpty ? null : SeqId(sequences.last), + acknowledge: acknowledge ?? () async {}, + ); +} + +Future _eventually(bool Function() condition) async { + final deadline = DateTime.now().add(const Duration(seconds: 2)); + while (!condition()) { + if (DateTime.now().isAfter(deadline)) { + throw TimeoutException('condition was not reached'); + } + await Future.delayed(const Duration(milliseconds: 5)); + } +} diff --git a/link/sdks/dart/sync/test/tables/kalam_table_binding_test.dart b/link/sdks/dart/sync/test/tables/kalam_table_binding_test.dart new file mode 100644 index 000000000..655222327 --- /dev/null +++ b/link/sdks/dart/sync/test/tables/kalam_table_binding_test.dart @@ -0,0 +1,299 @@ +import 'dart:async'; +import 'dart:convert'; + +import 'package:drift/native.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:kalam_sync/drift.dart' hide isNotNull, isNull; +import 'package:kalam_sync/kalam_sync.dart'; + +final class Message { + const Message({required this.id, required this.text}); + + final String id; + final String text; + + @override + bool operator ==(Object other) => + other is Message && other.id == id && other.text == text; + + @override + int get hashCode => Object.hash(id, text); +} + +void main() { + late KalamSyncDatabase database; + late KalamSyncStore store; + late List localRows; + late StreamController> localChanges; + final now = DateTime.utc(2026, 8, 16, 12); + + setUp(() { + database = KalamSyncDatabase(NativeDatabase.memory()); + store = KalamSyncStore(database, clock: () => now); + localRows = []; + localChanges = StreamController.broadcast(onListen: () {}); + }); + + tearDown(() async { + await localChanges.close(); + await database.close(); + }); + + KalamTableBinding binding(KalamSyncMode mode) { + final spec = KalamTableSpec( + tableId: 'app.messages', + keyColumn: 'id', + subscriptionId: 'conversation-1/messages', + mode: mode, + keyOf: (row) => row.id, + encode: (row) => {'id': row.id, 'text': row.text}, + decode: (json) => + Message(id: json['id']! as String, text: json['text']! as String), + ); + return KalamTableBinding( + spec: spec, + accountKey: 'server/user-a', + store: store, + watchLocal: () async* { + yield List.of(localRows); + yield* localChanges.stream; + }, + upsertLocal: (message) async { + localRows.removeWhere((row) => row.id == message.id); + localRows.add(message); + localChanges.add(List.of(localRows)); + }, + deleteLocal: (rowKey) async { + localRows.removeWhere((row) => row.id == rowKey); + localChanges.add(List.of(localRows)); + }, + ); + } + + test( + 'bidirectional insert writes locally and enqueues one generic DML action', + () async { + final messages = binding(KalamSyncMode.bidirectional); + + final action = await messages.insert( + const Message(id: 'message-1', text: 'hello'), + actionId: 'action-1', + ); + + expect(localRows, [const Message(id: 'message-1', text: 'hello')]); + expect(action.actionKey, KalamTableBinding.dmlActionKey); + expect(action.kind, 'dml'); + expect( + jsonDecode(action.payloadJson), + containsPair('operation', 'insert'), + ); + expect(await database.select(database.kalamActions).get(), hasLength(1)); + }, + ); + + test( + 'default binding keeps mirrored rows in Kalam-owned Drift storage', + () async { + final spec = KalamTableSpec( + tableId: 'app.messages', + keyColumn: 'id', + mode: KalamSyncMode.bidirectional, + keyOf: (row) => row.id, + encode: (row) => {'id': row.id, 'text': row.text}, + decode: (json) => + Message(id: json['id']! as String, text: json['text']! as String), + ); + final messages = KalamTableBinding( + spec: spec, + accountKey: 'server/user-a', + store: store, + ); + + await messages.insert( + const Message(id: 'message-1', text: 'stored once'), + actionId: 'action-1', + ); + + expect(await messages.watch().first, [ + const Message(id: 'message-1', text: 'stored once'), + ]); + expect( + await database.select(database.kalamCachedRows).get(), + hasLength(1), + ); + expect(await database.select(database.kalamActions).get(), hasLength(1)); + }, + ); + + test('replica-only tables reject direct writes', () async { + final messages = binding(KalamSyncMode.replicaOnly); + + expect( + () => messages.insert( + const Message(id: 'message-1', text: 'hello'), + actionId: 'action-1', + ), + throwsStateError, + ); + }); + + test('optimistic replica row is visible with pending sync state', () async { + final messages = binding(KalamSyncMode.replicaOnly); + final pending = messages.watchWithSyncState().firstWhere( + (rows) => rows.isNotEmpty, + ); + + await store.enqueue( + const KalamActionDraft( + id: 'action-1', + accountKey: 'server/user-a', + actionKey: 'messages.send', + payloadJson: '{}', + ), + optimisticRow: const KalamOptimisticRow( + tableId: 'app.messages', + rowKey: 'message-1', + phase: KalamRowSyncPhase.pending, + pendingValuesJson: '{"id":"message-1","text":"hello"}', + ), + ); + + final rows = await pending; + expect(rows.single.value, const Message(id: 'message-1', text: 'hello')); + expect(rows.single.sync.phase, KalamRowSyncPhase.pending); + }); + + test('pending delete hides an authoritative row', () async { + localRows.add(const Message(id: 'message-1', text: 'hello')); + final messages = binding(KalamSyncMode.replicaOnly); + final hidden = messages.watch().firstWhere((rows) => rows.isEmpty); + + await store.enqueue( + const KalamActionDraft( + id: 'action-1', + accountKey: 'server/user-a', + actionKey: 'messages.delete', + payloadJson: '{}', + ), + optimisticRow: const KalamOptimisticRow( + tableId: 'app.messages', + rowKey: 'message-1', + phase: KalamRowSyncPhase.pendingDelete, + tombstone: true, + ), + ); + + expect(await hidden, isEmpty); + }); + + test('a stale row echo cannot complete a pending delete', () async { + final messages = binding(KalamSyncMode.bidirectional); + await messages.insert( + const Message(id: 'message-1', text: 'original'), + actionId: 'insert-1', + ); + await messages.applyServerChange( + const KalamChange( + kind: KalamChangeKind.insert, + rowKey: 'message-1', + row: Message(id: 'message-1', text: 'original'), + seq: SeqId(1), + ), + ); + + await messages.delete('message-1', actionId: 'delete-1'); + await messages.applyServerChange( + const KalamChange( + kind: KalamChangeKind.update, + rowKey: 'message-1', + row: Message(id: 'message-1', text: 'stale echo'), + seq: SeqId(2), + ), + ); + + expect( + (await store.readAction('delete-1'))?.status, + KalamActionStatus.queued, + ); + expect(await messages.watch().first, isEmpty); + + await messages.applyServerChange( + const KalamChange( + kind: KalamChangeKind.delete, + rowKey: 'message-1', + seq: SeqId(3), + ), + ); + expect( + (await store.readAction('delete-1'))?.status, + KalamActionStatus.succeeded, + ); + }); + + test('failed action is reflected beside its optimistic row', () async { + final messages = binding(KalamSyncMode.replicaOnly); + final failed = messages.watchWithSyncState().firstWhere( + (rows) => rows.singleOrNull?.sync.phase == KalamRowSyncPhase.failed, + ); + await store.enqueue( + const KalamActionDraft( + id: 'action-1', + accountKey: 'server/user-a', + actionKey: 'messages.send', + payloadJson: '{}', + ), + optimisticRow: const KalamOptimisticRow( + tableId: 'app.messages', + rowKey: 'message-1', + phase: KalamRowSyncPhase.pending, + pendingValuesJson: '{"id":"message-1","text":"hello"}', + ), + ); + await store.failAction('action-1', 'invalid recipient', now); + + final row = (await failed).single; + expect(row.sync.errorMessage, 'invalid recipient'); + }); + + test( + 'server echo replaces overlay and commits the resume checkpoint', + () async { + final messages = binding(KalamSyncMode.replicaOnly); + await store.enqueue( + const KalamActionDraft( + id: 'action-1', + accountKey: 'server/user-a', + actionKey: 'messages.send', + payloadJson: '{}', + ), + optimisticRow: const KalamOptimisticRow( + tableId: 'app.messages', + rowKey: 'message-1', + phase: KalamRowSyncPhase.pending, + pendingValuesJson: '{"id":"message-1","text":"draft"}', + ), + ); + + await messages.applyServerChange( + const KalamChange( + kind: KalamChangeKind.insert, + rowKey: 'message-1', + row: Message(id: 'message-1', text: 'authoritative'), + seq: SeqId(12), + ), + ); + + final rows = await messages.watchWithSyncState().first; + final checkpoint = await store.readCheckpoint( + accountKey: 'server/user-a', + subscriptionId: 'conversation-1/messages', + ); + expect( + rows.single.value, + const Message(id: 'message-1', text: 'authoritative'), + ); + expect(rows.single.sync.phase, KalamRowSyncPhase.synced); + expect(checkpoint?.seq, const SeqId(12)); + }, + ); +} diff --git a/link/sdks/dart/sync/test/transport/kalam_link_transport_test.dart b/link/sdks/dart/sync/test/transport/kalam_link_transport_test.dart new file mode 100644 index 000000000..96c6a23cf --- /dev/null +++ b/link/sdks/dart/sync/test/transport/kalam_link_transport_test.dart @@ -0,0 +1,75 @@ +import 'package:flutter_test/flutter_test.dart'; +import 'package:kalam_sync/kalam_sync.dart'; + +void main() { + test('decodes initial rows and removes the internal sequence column', () { + final changes = KalamLinkTransport.decodeEvent( + InitialDataBatch( + subscriptionId: 'messages', + rowsJson: const ['{"id":"message-1","text":"hello","_seq":12}'], + batchNum: 1, + hasMore: false, + status: 'ok', + ), + ); + + expect(changes.single.initial, isTrue); + expect(changes.single.seq, const SeqId(12)); + expect(changes.single.row, {'id': 'message-1', 'text': 'hello'}); + }); + + test('sorts batched changes by sequence before local application', () { + final changes = KalamLinkTransport.decodeEvent( + InsertEvent( + subscriptionId: 'messages', + rowsJson: const [ + '{"id":"message-2","_seq":2}', + '{"id":"message-1","_seq":1}', + ], + ), + ); + + expect(changes.map((change) => change.seq), [ + const SeqId(1), + const SeqId(2), + ]); + }); + + test('keeps old update values available for reconciliation', () { + final change = KalamLinkTransport.decodeEvent( + UpdateEvent( + subscriptionId: 'messages', + rowsJson: const ['{"id":"message-1","text":"new","_seq":2}'], + oldRowsJson: const ['{"id":"message-1","text":"old","_seq":1}'], + ), + ).single; + + expect(change.kind, KalamChangeKind.update); + expect(change.oldRow, {'id': 'message-1', 'text': 'old'}); + }); + + test('surfaces subscription errors', () { + expect( + () => KalamLinkTransport.decodeEvent( + const SubscriptionError( + subscriptionId: 'messages', + code: 'CURSOR_EXPIRED', + message: 'checkpoint is outside retention', + ), + ), + throwsA(isA()), + ); + }); + + test('rejects synchronized rows without a sequence', () { + expect( + () => KalamLinkTransport.decodeEvent( + InsertEvent( + subscriptionId: 'messages', + rowsJson: const ['{"id":"message-1"}'], + ), + ), + throwsFormatException, + ); + }); +} diff --git a/link/sdks/dart/test/schema_generation_test.dart b/link/sdks/dart/test/schema_generation_test.dart deleted file mode 100644 index d6640617f..000000000 --- a/link/sdks/dart/test/schema_generation_test.dart +++ /dev/null @@ -1,15 +0,0 @@ -import 'package:flutter_test/flutter_test.dart'; - -void main() { - group('kalam CLI workflow Dart targets', () { - const sampleKalamCliDart = ''' -// Placeholder generated by kalam schema gen. -// Dart schema generation via a dedicated package will be added later. -'''; - - test('matches current placeholder contract', () { - expect(sampleKalamCliDart, contains('Placeholder generated by kalam schema gen')); - expect(sampleKalamCliDart, contains('will be added later')); - }); - }); -} diff --git a/link/sdks/sync-versions.sh b/link/sdks/sync-versions.sh index 53b543ae0..20c47fdc9 100755 --- a/link/sdks/sync-versions.sh +++ b/link/sdks/sync-versions.sh @@ -10,7 +10,9 @@ ROOT_CARGO="$ROOT_DIR/Cargo.toml" VERSIONS_SCRIPT="$ROOT_DIR/scripts/versions.py" PYTHON_PYPROJECT="$SDKS_DIR/python/pyproject.toml" PYTHON_CARGO="$SDKS_DIR/python/Cargo.toml" -DART_PUBSPEC="$SDKS_DIR/dart/pubspec.yaml" +DART_LINK_PUBSPEC="$SDKS_DIR/dart/link/pubspec.yaml" +DART_SYNC_PUBSPEC="$SDKS_DIR/dart/sync/pubspec.yaml" +DART_GENERATOR_PUBSPEC="$SDKS_DIR/dart/generator/pubspec.yaml" RUST_PUBLISH_MODE="" RUST_PUBLISH_VERSION_OVERRIDE="" @@ -186,7 +188,9 @@ TS_PACKAGE_JSON_FILES=( "$TS_DIR/react/package.json" ) -for file in "$ROOT_CARGO" "$VERSIONS_SCRIPT" "$PYTHON_PYPROJECT" "$PYTHON_CARGO" "$DART_PUBSPEC" "${TS_PACKAGE_JSON_FILES[@]}"; do +for file in "$ROOT_CARGO" "$VERSIONS_SCRIPT" "$PYTHON_PYPROJECT" "$PYTHON_CARGO" \ + "$DART_LINK_PUBSPEC" "$DART_SYNC_PUBSPEC" "$DART_GENERATOR_PUBSPEC" \ + "${TS_PACKAGE_JSON_FILES[@]}"; do if [[ ! -f "$file" ]]; then echo "Missing required file: $file" >&2 exit 1 @@ -216,14 +220,16 @@ ROOT_VERSION="${VERSION_INFO_RAW%%$'\n'*}" INTERNAL_RANGE="${VERSION_INFO_RAW#*$'\n'}" echo "Syncing SDK package versions to root Cargo workspace version $ROOT_VERSION" -echo "Using TypeScript internal peer dependency range $INTERNAL_RANGE" +echo "Using SDK internal dependency range $INTERNAL_RANGE" python3 - \ "$ROOT_VERSION" \ "$INTERNAL_RANGE" \ "$ROOT_DIR" \ "${PUBLISH_SCOPE_OVERRIDE:-}" \ - "$DART_PUBSPEC" \ + "$DART_LINK_PUBSPEC" \ + "$DART_SYNC_PUBSPEC" \ + "$DART_GENERATOR_PUBSPEC" \ "$PYTHON_PYPROJECT" \ "${TS_PACKAGE_JSON_FILES[@]}" <<'PY' import json @@ -235,9 +241,11 @@ root_version = sys.argv[1] internal_range = sys.argv[2] root_dir = Path(sys.argv[3]) typescript_scope_override = sys.argv[4].strip() -dart_pubspec = Path(sys.argv[5]) -python_pyproject = Path(sys.argv[6]) -typescript_packages = [Path(value) for value in sys.argv[7:]] +dart_link_pubspec = Path(sys.argv[5]) +dart_sync_pubspec = Path(sys.argv[6]) +dart_generator_pubspec = Path(sys.argv[7]) +python_pyproject = Path(sys.argv[8]) +typescript_packages = [Path(value) for value in sys.argv[9:]] internal_packages = { "@kalamdb/client", "@kalamdb/consumer", @@ -270,6 +278,18 @@ def update_yaml_version(path: Path, new_version: str) -> None: print(f"Updated {display(path)}: version -> {new_version}") +def update_yaml_dependency(path: Path, dependency: str, new_range: str) -> None: + text = path.read_text(encoding="utf-8") + pattern = rf'(?m)^(\s{{2}}{re.escape(dependency)}:)\s*.+$' + updated, count = re.subn(pattern, rf'\1 "{new_range}"', text, count=1) + if count != 1: + raise SystemExit( + f"Failed to update {dependency} dependency in {display(path)}" + ) + path.write_text(updated, encoding="utf-8") + print(f"Updated {display(path)}: {dependency} -> {new_range}") + + def update_toml_section_version(path: Path, section_name: str, new_version: str) -> None: lines = path.read_text(encoding="utf-8").splitlines(keepends=True) current_section = None @@ -309,11 +329,19 @@ for package_path in typescript_packages: package_path.write_text(f"{json.dumps(package_data, indent=2)}\n", encoding="utf-8") print(f"Updated {rendered_package_name} ({display(package_path)}): version -> {root_version}") -update_yaml_version(dart_pubspec, root_version) +for dart_pubspec in ( + dart_link_pubspec, + dart_sync_pubspec, + dart_generator_pubspec, +): + update_yaml_version(dart_pubspec, root_version) + +update_yaml_dependency(dart_sync_pubspec, "kalam_link", internal_range) +update_yaml_dependency(dart_generator_pubspec, "kalam_sync", internal_range) update_toml_section_version(python_pyproject, "project", root_version) PY python3 "$VERSIONS_SCRIPT" sync --write python3 "$VERSIONS_SCRIPT" verify -echo "SDK package manifests and versions.json are in sync." \ No newline at end of file +echo "SDK package manifests and versions.json are in sync." diff --git a/link/sdks/typescript/README.md b/link/sdks/typescript/README.md index 75d08f135..df1bd6a86 100644 --- a/link/sdks/typescript/README.md +++ b/link/sdks/typescript/README.md @@ -33,7 +33,7 @@ bash link/sdks/sync-versions.sh That script: - sets all five TypeScript package `version` fields to the root Cargo workspace version, -- updates `link/sdks/dart/pubspec.yaml` to the same version, +- updates all `link/sdks/dart/*/pubspec.yaml` package versions, - updates `link/sdks/python/pyproject.toml` and `link/sdks/python/Cargo.toml` to the same version, - updates the internal peer dependency floors to the current cohort range, - regenerates `versions.json` through `python3 scripts/versions.py sync --write`, and diff --git a/pg/docker/Dockerfile b/pg/docker/Dockerfile index 0763f909a..defe8b265 100644 --- a/pg/docker/Dockerfile +++ b/pg/docker/Dockerfile @@ -28,6 +28,8 @@ ARG PGRX_VERSION ENV CARGO_BUILD_JOBS=1 ENV CARGO_INCREMENTAL=0 +ENV RUSTUP_TOOLCHAIN=1.92.0 +ENV CARGO_ENCODED_RUSTFLAGS= # -Cpanic=unwind is required for pgrx #[pg_guard] to catch panics at FFI boundaries. # Without it, any panic in async/gRPC code causes SIGABRT (process abort). ENV RUSTFLAGS="-Cdebuginfo=0 -Cpanic=unwind" diff --git a/scripts/test-all.sh b/scripts/test-all.sh index a888debf4..6839c335e 100755 --- a/scripts/test-all.sh +++ b/scripts/test-all.sh @@ -253,11 +253,26 @@ run_npm_test "ui" "Running admin UI tests" step "Running Dart SDK tests" ( - cd "$ROOT_DIR/link/sdks/dart" + cd "$ROOT_DIR/link/sdks/dart/link" ./test.sh ) +step "Running Kalam Sync tests" +( + cd "$ROOT_DIR/link/sdks/dart/sync" + flutter analyze + flutter test +) + +step "Running Kalam Sync generator tests" +( + cd "$ROOT_DIR/link/sdks/dart/generator" + flutter pub run build_runner build + flutter analyze + dart test +) + echo echo "========================================================" echo " All core KalamDB test suites passed" -echo "========================================================" \ No newline at end of file +echo "========================================================" diff --git a/scripts/versions.py b/scripts/versions.py index bae8e8ddf..10409e2fe 100644 --- a/scripts/versions.py +++ b/scripts/versions.py @@ -114,7 +114,9 @@ RUST_SDK_CARGO = ROOT / "link" / "sdks" / "rust" / "Cargo.toml" PYTHON_PYPROJECT = ROOT / "link" / "sdks" / "python" / "pyproject.toml" PYTHON_CARGO = ROOT / "link" / "sdks" / "python" / "Cargo.toml" -DART_PUBSPEC = ROOT / "link" / "sdks" / "dart" / "pubspec.yaml" +DART_LINK_PUBSPEC = ROOT / "link" / "sdks" / "dart" / "link" / "pubspec.yaml" +DART_SYNC_PUBSPEC = ROOT / "link" / "sdks" / "dart" / "sync" / "pubspec.yaml" +DART_GENERATOR_PUBSPEC = ROOT / "link" / "sdks" / "dart" / "generator" / "pubspec.yaml" TS_CLIENT_PACKAGE = ROOT / "link" / "sdks" / "typescript" / "client" / "package.json" TS_CONSUMER_PACKAGE = ROOT / "link" / "sdks" / "typescript" / "consumer" / "package.json" TS_ORM_PACKAGE = ROOT / "link" / "sdks" / "typescript" / "orm" / "package.json" @@ -358,7 +360,14 @@ def build_versions_manifest(existing: dict[str, Any] | None) -> dict[str, Any]: "link/sdks/python/Cargo.toml and link/sdks/python/pyproject.toml must use the same version" ) - dart_name, dart_version = read_pubspec_name_and_version(DART_PUBSPEC) + dart_packages = dict( + read_pubspec_name_and_version(path) + for path in ( + DART_LINK_PUBSPEC, + DART_SYNC_PUBSPEC, + DART_GENERATOR_PUBSPEC, + ) + ) cli_npm_package = get_package_json(CLI_NPM_PACKAGE) if cli_npm_package.get("version") != core_version: raise VersionError( @@ -448,7 +457,8 @@ def build_versions_manifest(existing: dict[str, Any] | None) -> dict[str, Any]: ) }, "dart": { - dart_name: build_package_entry(dart_version, compatible_core) + name: build_package_entry(version, compatible_core) + for name, version in dart_packages.items() }, }, } @@ -579,4 +589,4 @@ def main() -> int: if __name__ == "__main__": - raise SystemExit(main()) \ No newline at end of file + raise SystemExit(main()) diff --git a/tools/Dockerfile.builder b/tools/Dockerfile.builder index d58ba1c67..d24e0da66 100644 --- a/tools/Dockerfile.builder +++ b/tools/Dockerfile.builder @@ -32,6 +32,8 @@ RUN git clone https://github.com/tpoechtrager/osxcross.git && \ ENV PATH="/opt/osxcross/target/bin:$PATH" ENV LD_LIBRARY_PATH="/opt/osxcross/target/lib:$LD_LIBRARY_PATH" +ENV RUSTUP_TOOLCHAIN=1.92.0 +ENV CARGO_ENCODED_RUSTFLAGS= # Add Rust targets RUN rustup target add \ diff --git a/versions.json b/versions.json index 603b6f24b..ae1c4affe 100644 --- a/versions.json +++ b/versions.json @@ -277,6 +277,16 @@ "version": "0.5.6-rc.0", "compatible_core": "0.5.x", "protocol": "v1" + }, + "kalam_sync": { + "version": "0.5.6-rc.0", + "compatible_core": "0.5.x", + "protocol": "v1" + }, + "kalam_sync_generator": { + "version": "0.5.6-rc.0", + "compatible_core": "0.5.x", + "protocol": "v1" } } } From 7581dcb5eea57abc0e6fd94817f666795370bfe4 Mon Sep 17 00:00:00 2001 From: jamals86 Date: Mon, 17 Aug 2026 13:31:37 +0300 Subject: [PATCH 2/4] Update .gitignore --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index b583c76ab..8cb2df6bd 100644 --- a/.gitignore +++ b/.gitignore @@ -143,3 +143,5 @@ cli/kalam.toml /examples/live-okf-context-sync/data3 /examples/live-okf-context-sync/data5 /specs/033-unified-backend-pgwire/validation +/link/sdks/dart/.dart_tool +rust-toolchain.toml From cb37eaf00c474122d9c98771f7a422804f1ee764 Mon Sep 17 00:00:00 2001 From: jamals86 Date: Mon, 17 Aug 2026 14:03:57 +0300 Subject: [PATCH 3/4] Keep Atomic::fetch_update for MSRV 1.92. try_update is still unstable on the 1.92 CI toolchain, so restore the stable API and silence the nightly deprecation warning. Co-authored-by: Cursor --- Cargo.lock | 112 +++++++++--------- .../kalamdb-api/src/limiter/rate_limiter.rs | 3 +- .../src/manager/connections_manager.rs | 3 +- .../src/storage_metrics.rs | 3 +- backend/src/connection_guard.rs | 3 +- 5 files changed, 64 insertions(+), 60 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7ef5bed9c..3a1679fcb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4274,7 +4274,7 @@ dependencies = [ [[package]] name = "kalam-cli" -version = "0.5.6-rc.0" +version = "0.5.6-rc.1" dependencies = [ "anyhow", "base64 0.23.1", @@ -4315,7 +4315,7 @@ dependencies = [ [[package]] name = "kalam-cli-e2e" -version = "0.5.6-rc.0" +version = "0.5.6-rc.1" dependencies = [ "anyhow", "assert_cmd", @@ -4358,14 +4358,14 @@ dependencies = [ [[package]] name = "kalam-client" -version = "0.5.6-rc.0" +version = "0.5.6-rc.1" dependencies = [ "link-common", ] [[package]] name = "kalam-client-e2e" -version = "0.5.6-rc.0" +version = "0.5.6-rc.1" dependencies = [ "kalam-client", "kalamdb-configs", @@ -4380,7 +4380,7 @@ dependencies = [ [[package]] name = "kalam-consumer-wasm" -version = "0.5.6-rc.0" +version = "0.5.6-rc.1" dependencies = [ "js-sys", "link-common", @@ -4393,7 +4393,7 @@ dependencies = [ [[package]] name = "kalam-link-dart" -version = "0.5.6-rc.0" +version = "0.5.6-rc.1" dependencies = [ "anyhow", "flutter_rust_bridge", @@ -4405,7 +4405,7 @@ dependencies = [ [[package]] name = "kalam-link-wasm" -version = "0.5.6-rc.0" +version = "0.5.6-rc.1" dependencies = [ "base64 0.23.1", "js-sys", @@ -4423,7 +4423,7 @@ dependencies = [ [[package]] name = "kalam-pg-api" -version = "0.5.6-rc.0" +version = "0.5.6-rc.1" dependencies = [ "arrow", "async-trait", @@ -4435,7 +4435,7 @@ dependencies = [ [[package]] name = "kalam-pg-client" -version = "0.5.6-rc.0" +version = "0.5.6-rc.1" dependencies = [ "arrow", "arrow-ipc", @@ -4457,7 +4457,7 @@ dependencies = [ [[package]] name = "kalam-pg-common" -version = "0.5.6-rc.0" +version = "0.5.6-rc.1" dependencies = [ "datafusion-common", "serde", @@ -4466,7 +4466,7 @@ dependencies = [ [[package]] name = "kalam-pg-extension" -version = "0.5.6-rc.0" +version = "0.5.6-rc.1" dependencies = [ "arrow", "async-trait", @@ -4499,7 +4499,7 @@ dependencies = [ [[package]] name = "kalam-pg-fdw" -version = "0.5.6-rc.0" +version = "0.5.6-rc.1" dependencies = [ "datafusion-common", "kalam-pg-api", @@ -4510,7 +4510,7 @@ dependencies = [ [[package]] name = "kalam-pg-types" -version = "0.5.6-rc.0" +version = "0.5.6-rc.1" dependencies = [ "kalam-pg-common", "kalamdb-commons", @@ -4518,7 +4518,7 @@ dependencies = [ [[package]] name = "kalam-schema-diff" -version = "0.5.6-rc.0" +version = "0.5.6-rc.1" dependencies = [ "sqlparser", "tempfile", @@ -4527,7 +4527,7 @@ dependencies = [ [[package]] name = "kalamdb-api" -version = "0.5.6-rc.0" +version = "0.5.6-rc.1" dependencies = [ "actix-files", "actix-multipart", @@ -4573,7 +4573,7 @@ dependencies = [ [[package]] name = "kalamdb-auth" -version = "0.5.6-rc.0" +version = "0.5.6-rc.1" dependencies = [ "actix-web", "anyhow", @@ -4603,7 +4603,7 @@ dependencies = [ [[package]] name = "kalamdb-backend" -version = "0.5.6-rc.0" +version = "0.5.6-rc.1" dependencies = [ "async-trait", "dashmap 6.2.1", @@ -4617,7 +4617,7 @@ dependencies = [ [[package]] name = "kalamdb-commons" -version = "0.5.6-rc.0" +version = "0.5.6-rc.1" dependencies = [ "arrow", "arrow-schema", @@ -4645,7 +4645,7 @@ dependencies = [ [[package]] name = "kalamdb-configs" -version = "0.5.6-rc.0" +version = "0.5.6-rc.1" dependencies = [ "anyhow", "ipnet", @@ -4656,7 +4656,7 @@ dependencies = [ [[package]] name = "kalamdb-core" -version = "0.5.6-rc.0" +version = "0.5.6-rc.1" dependencies = [ "anyhow", "arrow", @@ -4713,7 +4713,7 @@ dependencies = [ [[package]] name = "kalamdb-datafusion-sources" -version = "0.5.6-rc.0" +version = "0.5.6-rc.1" dependencies = [ "arrow", "arrow-schema", @@ -4728,7 +4728,7 @@ dependencies = [ [[package]] name = "kalamdb-dba" -version = "0.5.6-rc.0" +version = "0.5.6-rc.1" dependencies = [ "chrono", "datafusion", @@ -4746,7 +4746,7 @@ dependencies = [ [[package]] name = "kalamdb-dialect" -version = "0.5.6-rc.0" +version = "0.5.6-rc.1" dependencies = [ "anyhow", "arrow", @@ -4764,7 +4764,7 @@ dependencies = [ [[package]] name = "kalamdb-filestore" -version = "0.5.6-rc.0" +version = "0.5.6-rc.1" dependencies = [ "arrow", "bytes", @@ -4796,7 +4796,7 @@ dependencies = [ [[package]] name = "kalamdb-flush" -version = "0.5.6-rc.0" +version = "0.5.6-rc.1" dependencies = [ "arrow", "async-trait", @@ -4824,7 +4824,7 @@ dependencies = [ [[package]] name = "kalamdb-handlers" -version = "0.5.6-rc.0" +version = "0.5.6-rc.1" dependencies = [ "kalamdb-commons", "kalamdb-core", @@ -4839,7 +4839,7 @@ dependencies = [ [[package]] name = "kalamdb-handlers-admin" -version = "0.5.6-rc.0" +version = "0.5.6-rc.1" dependencies = [ "arrow", "chrono", @@ -4859,7 +4859,7 @@ dependencies = [ [[package]] name = "kalamdb-handlers-ddl" -version = "0.5.6-rc.0" +version = "0.5.6-rc.1" dependencies = [ "arrow", "chrono", @@ -4883,7 +4883,7 @@ dependencies = [ [[package]] name = "kalamdb-handlers-stream" -version = "0.5.6-rc.0" +version = "0.5.6-rc.1" dependencies = [ "arrow", "chrono", @@ -4899,7 +4899,7 @@ dependencies = [ [[package]] name = "kalamdb-handlers-support" -version = "0.5.6-rc.0" +version = "0.5.6-rc.1" dependencies = [ "chrono", "datafusion", @@ -4916,7 +4916,7 @@ dependencies = [ [[package]] name = "kalamdb-handlers-user" -version = "0.5.6-rc.0" +version = "0.5.6-rc.1" dependencies = [ "chrono", "kalamdb-auth", @@ -4932,7 +4932,7 @@ dependencies = [ [[package]] name = "kalamdb-jobs" -version = "0.5.6-rc.0" +version = "0.5.6-rc.1" dependencies = [ "async-trait", "bytes", @@ -4964,7 +4964,7 @@ dependencies = [ [[package]] name = "kalamdb-live" -version = "0.5.6-rc.0" +version = "0.5.6-rc.1" dependencies = [ "arrow", "async-trait", @@ -4990,7 +4990,7 @@ dependencies = [ [[package]] name = "kalamdb-macros" -version = "0.5.6-rc.0" +version = "0.5.6-rc.1" dependencies = [ "proc-macro2", "quote", @@ -4999,7 +4999,7 @@ dependencies = [ [[package]] name = "kalamdb-observability" -version = "0.5.6-rc.0" +version = "0.5.6-rc.1" dependencies = [ "cc", "chrono", @@ -5013,7 +5013,7 @@ dependencies = [ [[package]] name = "kalamdb-pg" -version = "0.5.6-rc.0" +version = "0.5.6-rc.1" dependencies = [ "arrow", "arrow-ipc", @@ -5041,7 +5041,7 @@ dependencies = [ [[package]] name = "kalamdb-plan-cache" -version = "0.5.6-rc.0" +version = "0.5.6-rc.1" dependencies = [ "datafusion", "kalamdb-commons", @@ -5050,7 +5050,7 @@ dependencies = [ [[package]] name = "kalamdb-postgres-wire" -version = "0.5.6-rc.0" +version = "0.5.6-rc.1" dependencies = [ "arrow", "async-trait", @@ -5071,7 +5071,7 @@ dependencies = [ [[package]] name = "kalamdb-publisher" -version = "0.5.6-rc.0" +version = "0.5.6-rc.1" dependencies = [ "chrono", "dashmap 6.2.1", @@ -5090,7 +5090,7 @@ dependencies = [ [[package]] name = "kalamdb-python" -version = "0.5.6-rc.0" +version = "0.5.6-rc.1" dependencies = [ "kalam-client", "pyo3", @@ -5102,7 +5102,7 @@ dependencies = [ [[package]] name = "kalamdb-raft" -version = "0.5.6-rc.0" +version = "0.5.6-rc.1" dependencies = [ "async-trait", "chrono", @@ -5137,7 +5137,7 @@ dependencies = [ [[package]] name = "kalamdb-row-filter" -version = "0.5.6-rc.0" +version = "0.5.6-rc.1" dependencies = [ "datafusion-common", "kalamdb-commons", @@ -5148,7 +5148,7 @@ dependencies = [ [[package]] name = "kalamdb-server" -version = "0.5.6-rc.0" +version = "0.5.6-rc.1" dependencies = [ "actix-cors", "actix-web", @@ -5209,7 +5209,7 @@ dependencies = [ [[package]] name = "kalamdb-server-auth" -version = "0.5.6-rc.0" +version = "0.5.6-rc.1" dependencies = [ "log", "rcgen", @@ -5219,14 +5219,14 @@ dependencies = [ [[package]] name = "kalamdb-session" -version = "0.5.6-rc.0" +version = "0.5.6-rc.1" dependencies = [ "kalamdb-commons", ] [[package]] name = "kalamdb-session-datafusion" -version = "0.5.6-rc.0" +version = "0.5.6-rc.1" dependencies = [ "arrow", "async-trait", @@ -5238,7 +5238,7 @@ dependencies = [ [[package]] name = "kalamdb-sharding" -version = "0.5.6-rc.0" +version = "0.5.6-rc.1" dependencies = [ "kalamdb-commons", "kalamdb-configs", @@ -5247,7 +5247,7 @@ dependencies = [ [[package]] name = "kalamdb-store" -version = "0.5.6-rc.0" +version = "0.5.6-rc.1" dependencies = [ "anyhow", "async-trait", @@ -5269,7 +5269,7 @@ dependencies = [ [[package]] name = "kalamdb-streams" -version = "0.5.6-rc.0" +version = "0.5.6-rc.1" dependencies = [ "chrono", "dashmap 6.2.1", @@ -5283,7 +5283,7 @@ dependencies = [ [[package]] name = "kalamdb-system" -version = "0.5.6-rc.0" +version = "0.5.6-rc.1" dependencies = [ "arrow", "async-trait", @@ -5306,7 +5306,7 @@ dependencies = [ [[package]] name = "kalamdb-tables" -version = "0.5.6-rc.0" +version = "0.5.6-rc.1" dependencies = [ "arrow", "async-trait", @@ -5340,7 +5340,7 @@ dependencies = [ [[package]] name = "kalamdb-transactions" -version = "0.5.6-rc.0" +version = "0.5.6-rc.1" dependencies = [ "async-trait", "datafusion", @@ -5356,7 +5356,7 @@ dependencies = [ [[package]] name = "kalamdb-transfer" -version = "0.5.6-rc.0" +version = "0.5.6-rc.1" dependencies = [ "chrono", "flate2", @@ -5373,7 +5373,7 @@ dependencies = [ [[package]] name = "kalamdb-vector" -version = "0.5.6-rc.0" +version = "0.5.6-rc.1" dependencies = [ "async-trait", "bytes", @@ -5395,7 +5395,7 @@ dependencies = [ [[package]] name = "kalamdb-views" -version = "0.5.6-rc.0" +version = "0.5.6-rc.1" dependencies = [ "arrow", "async-trait", @@ -5611,7 +5611,7 @@ dependencies = [ [[package]] name = "link-common" -version = "0.5.6-rc.0" +version = "0.5.6-rc.1" dependencies = [ "base64 0.23.1", "bytes", diff --git a/backend/crates/kalamdb-api/src/limiter/rate_limiter.rs b/backend/crates/kalamdb-api/src/limiter/rate_limiter.rs index 5d7eb3d8e..a16c51371 100644 --- a/backend/crates/kalamdb-api/src/limiter/rate_limiter.rs +++ b/backend/crates/kalamdb-api/src/limiter/rate_limiter.rs @@ -136,8 +136,9 @@ impl RateLimiter { let user_key: Arc = Arc::from(user_id.as_str()); if let Some(count) = self.user_subscription_counts.get(&user_key) { + #[allow(deprecated)] count - .try_update(Ordering::Relaxed, Ordering::Relaxed, |v| Some(v.saturating_sub(1))) + .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |v| Some(v.saturating_sub(1))) .ok(); } } diff --git a/backend/crates/kalamdb-live/src/manager/connections_manager.rs b/backend/crates/kalamdb-live/src/manager/connections_manager.rs index a7a1c43c1..505b9e822 100644 --- a/backend/crates/kalamdb-live/src/manager/connections_manager.rs +++ b/backend/crates/kalamdb-live/src/manager/connections_manager.rs @@ -251,9 +251,10 @@ impl ConnectionsManager { } fn release_connection_slot(&self) { + #[allow(deprecated)] if self .total_connections - .try_update(Ordering::AcqRel, Ordering::Acquire, |current| current.checked_sub(1)) + .fetch_update(Ordering::AcqRel, Ordering::Acquire, |current| current.checked_sub(1)) .is_err() { warn!("Connection count release requested while count was already zero"); diff --git a/backend/crates/kalamdb-observability/src/storage_metrics.rs b/backend/crates/kalamdb-observability/src/storage_metrics.rs index b282d6730..80e2be3c2 100644 --- a/backend/crates/kalamdb-observability/src/storage_metrics.rs +++ b/backend/crates/kalamdb-observability/src/storage_metrics.rs @@ -187,7 +187,8 @@ pub fn decrement_manifest_cache_rocksdb_entries(delta: usize) { } let decrement = delta as u64; - let _ = MANIFEST_CACHE_ROCKSDB_ENTRIES.try_update( + #[allow(deprecated)] + let _ = MANIFEST_CACHE_ROCKSDB_ENTRIES.fetch_update( Ordering::AcqRel, Ordering::Acquire, |current| Some(current.saturating_sub(decrement)), diff --git a/backend/src/connection_guard.rs b/backend/src/connection_guard.rs index cc3a8ff25..def986c46 100644 --- a/backend/src/connection_guard.rs +++ b/backend/src/connection_guard.rs @@ -257,9 +257,10 @@ impl ConnectionGuard { } if let Some(state) = self.ip_states.get(&ip) { + #[allow(deprecated)] state .active_connections - .try_update(Ordering::Relaxed, Ordering::Relaxed, |value| { + .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |value| { Some(value.saturating_sub(1)) }) .ok(); From 8ccd9687e40879d23f3c9fe01da2894a999362ae Mon Sep 17 00:00:00 2001 From: jamals86 Date: Thu, 20 Aug 2026 11:54:38 +0300 Subject: [PATCH 4/4] Upgrade to DataFusion 55 and Arrow 59. Adopt ExecutionPlan::apply_expressions/replace_children and the parquet 59 AsyncFileReader APIs, bump MSRV to 1.94, and vendor datafusion-functions-json until upstream publishes a DataFusion 55-compatible release. Co-authored-by: Cursor --- .cursor/rules/specify-rules.mdc | 4 +- .github/workflows/ci.yml | 2 +- .github/workflows/cli-cluster-e2e.yml | 2 +- .github/workflows/dart-sdk.yml | 2 +- .github/workflows/python-sdk.yml | 4 +- .github/workflows/react-e2e.yml | 2 +- .github/workflows/release.yml | 2 +- .github/workflows/rust-coverage.yml | 2 +- .github/workflows/rust-sdk.yml | 2 +- .github/workflows/typescript-sdk.yml | 2 +- .gitignore | 4 +- Cargo.lock | 339 +++++----- Cargo.toml | 26 +- .../src/sql/datafusion_session.rs | 6 +- ...s.rs => test_datafusion55_sql_features.rs} | 2 +- .../kalamdb-datafusion-sources/src/exec.rs | 26 +- .../kalamdb-datafusion-sources/src/lib.rs | 2 +- .../src/parser/query_parser.rs | 2 +- .../kalamdb-dialect/src/parser/utils.rs | 2 +- .../kalamdb-dialect/src/query_features.rs | 4 +- .../kalamdb-filestore/src/parquet/reader.rs | 105 ++- .../kalamdb-filestore/src/parquet/writer.rs | 3 +- .../kalamdb-transactions/src/overlay_exec.rs | 80 ++- .../kalamdb-vector/src/sql/cosine_distance.rs | 2 +- vendor/datafusion-functions-json/Cargo.toml | 19 + vendor/datafusion-functions-json/LICENSE | 201 ++++++ vendor/datafusion-functions-json/README.md | 89 +++ .../datafusion-functions-json/src/common.rs | 598 ++++++++++++++++++ .../src/common_macros.rs | 49 ++ .../src/common_union.rs | 345 ++++++++++ .../src/json_as_text.rs | 117 ++++ .../src/json_contains.rs | 106 ++++ .../src/json_from_scalar.rs | 221 +++++++ .../datafusion-functions-json/src/json_get.rs | 151 +++++ .../src/json_get_array.rs | 144 +++++ .../src/json_get_bool.rs | 85 +++ .../src/json_get_float.rs | 123 ++++ .../src/json_get_int.rs | 122 ++++ .../src/json_get_json.rs | 94 +++ .../src/json_get_str.rs | 81 +++ .../src/json_length.rs | 128 ++++ .../src/json_object_keys.rs | 141 +++++ .../src/json_union_to_text.rs | 176 ++++++ vendor/datafusion-functions-json/src/lib.rs | 96 +++ .../datafusion-functions-json/src/rewrite.rs | 194 ++++++ 45 files changed, 3675 insertions(+), 232 deletions(-) rename backend/crates/kalamdb-core/tests/{test_datafusion54_sql_features.rs => test_datafusion55_sql_features.rs} (99%) create mode 100644 vendor/datafusion-functions-json/Cargo.toml create mode 100644 vendor/datafusion-functions-json/LICENSE create mode 100644 vendor/datafusion-functions-json/README.md create mode 100644 vendor/datafusion-functions-json/src/common.rs create mode 100644 vendor/datafusion-functions-json/src/common_macros.rs create mode 100644 vendor/datafusion-functions-json/src/common_union.rs create mode 100644 vendor/datafusion-functions-json/src/json_as_text.rs create mode 100644 vendor/datafusion-functions-json/src/json_contains.rs create mode 100644 vendor/datafusion-functions-json/src/json_from_scalar.rs create mode 100644 vendor/datafusion-functions-json/src/json_get.rs create mode 100644 vendor/datafusion-functions-json/src/json_get_array.rs create mode 100644 vendor/datafusion-functions-json/src/json_get_bool.rs create mode 100644 vendor/datafusion-functions-json/src/json_get_float.rs create mode 100644 vendor/datafusion-functions-json/src/json_get_int.rs create mode 100644 vendor/datafusion-functions-json/src/json_get_json.rs create mode 100644 vendor/datafusion-functions-json/src/json_get_str.rs create mode 100644 vendor/datafusion-functions-json/src/json_length.rs create mode 100644 vendor/datafusion-functions-json/src/json_object_keys.rs create mode 100644 vendor/datafusion-functions-json/src/json_union_to_text.rs create mode 100644 vendor/datafusion-functions-json/src/lib.rs create mode 100644 vendor/datafusion-functions-json/src/rewrite.rs diff --git a/.cursor/rules/specify-rules.mdc b/.cursor/rules/specify-rules.mdc index c6f922869..56fc859ee 100644 --- a/.cursor/rules/specify-rules.mdc +++ b/.cursor/rules/specify-rules.mdc @@ -8,9 +8,9 @@ shell commands, and other important information, read `specs/033-unified-backend ## Active Technologies -- Rust 1.92 (workspace edition 2021) for backend crates; `kalamdb-backend` (connection session registry), `kalamdb-postgres-wire` (thin **`pgwire`** adapter — not `datafusion-postgres`), existing `kalamdb-pg` gRPC bridge, `kalamdb-core` `TransactionCoordinator`, `kalamdb-auth` unified login +- Rust 1.94 (workspace edition 2021) for backend crates; `kalamdb-backend` (connection session registry), `kalamdb-postgres-wire` (thin **`pgwire`** adapter — not `datafusion-postgres`), existing `kalamdb-pg` gRPC bridge, `kalamdb-core` `TransactionCoordinator`, `kalamdb-auth` unified login - Wire txs: **`BackendSessionManager` block API** (same as gRPC `BeginTransaction`/`Commit`); wire SQL data path: **`SqlExecutor`**; HTTP batch txs: **`RequestTransactionBatchGuard`** only -- Existing stack — `tokio`, `tonic`, `dashmap`, `datafusion` 54.x, `kalamdb-transactions`, `kalamdb-api`, `kalamdb-views`; **`pgwire`** workspace-pinned in `kalamdb-postgres-wire` only +- Existing stack — `tokio`, `tonic`, `dashmap`, `datafusion` 55.x, `kalamdb-transactions`, `kalamdb-api`, `kalamdb-views`; **`pgwire`** workspace-pinned in `kalamdb-postgres-wire` only - No `arrow-pg` in wire MVP — `row_encoder.rs` uses existing `ExecutionResult`/`RecordBatch`; optional encoder spike later - Client catalog: **`pg_catalog` shim views** project from `system.*` (US9); not aliases; enable via `postgres_wire.client_catalog.enabled` - Stability-first migration: preserve 027 behavior; regression gates per `specs/033-unified-backend-pgwire/quickstart.md` diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 238bcf5d2..53d945f58 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,7 +12,7 @@ on: env: CARGO_TERM_COLOR: always - RUST_VERSION: "1.92.0" + RUST_VERSION: "1.94.0" CARGO_ENCODED_RUSTFLAGS: "" RUSTC_WRAPPER: "" CARGO_BUILD_RUSTC_WRAPPER: "" diff --git a/.github/workflows/cli-cluster-e2e.yml b/.github/workflows/cli-cluster-e2e.yml index 3f8ece5c6..24e463219 100644 --- a/.github/workflows/cli-cluster-e2e.yml +++ b/.github/workflows/cli-cluster-e2e.yml @@ -18,7 +18,7 @@ permissions: env: CARGO_TERM_COLOR: always - RUST_VERSION: "1.92.0" + RUST_VERSION: "1.94.0" CARGO_ENCODED_RUSTFLAGS: "" RUSTC_WRAPPER: "" CARGO_BUILD_RUSTC_WRAPPER: "" diff --git a/.github/workflows/dart-sdk.yml b/.github/workflows/dart-sdk.yml index 91f12b675..584c447a4 100644 --- a/.github/workflows/dart-sdk.yml +++ b/.github/workflows/dart-sdk.yml @@ -34,7 +34,7 @@ permissions: env: CARGO_TERM_COLOR: always - RUST_VERSION: "1.92.0" + RUST_VERSION: "1.94.0" CARGO_ENCODED_RUSTFLAGS: "" RUSTC_WRAPPER: "" CARGO_BUILD_RUSTC_WRAPPER: "" diff --git a/.github/workflows/python-sdk.yml b/.github/workflows/python-sdk.yml index 2952e18fb..e12efd252 100644 --- a/.github/workflows/python-sdk.yml +++ b/.github/workflows/python-sdk.yml @@ -49,7 +49,7 @@ jobs: - name: Install Rust uses: dtolnay/rust-toolchain@stable with: - toolchain: "1.92.0" + toolchain: "1.94.0" - name: Cache Cargo uses: Swatinem/rust-cache@v2 @@ -96,7 +96,7 @@ jobs: working-directory: link/sdks/python command: build args: --release --strip --out dist - rust-toolchain: "1.92.0" + rust-toolchain: "1.94.0" - name: Upload wheels uses: actions/upload-artifact@v4 diff --git a/.github/workflows/react-e2e.yml b/.github/workflows/react-e2e.yml index 261e3c15d..2a0daa246 100644 --- a/.github/workflows/react-e2e.yml +++ b/.github/workflows/react-e2e.yml @@ -8,7 +8,7 @@ permissions: env: CARGO_TERM_COLOR: always - RUST_VERSION: "1.92.0" + RUST_VERSION: "1.94.0" CARGO_ENCODED_RUSTFLAGS: "" WASM_PACK_VERSION: "0.15.0" KALAMDB_URL: "http://127.0.0.1:2900" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 5b5de77bc..c2a6479ec 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -74,7 +74,7 @@ permissions: env: CARGO_TERM_COLOR: always - RUST_VERSION: "1.92.0" + RUST_VERSION: "1.94.0" CARGO_ENCODED_RUSTFLAGS: "" WASM_PACK_VERSION: "0.15.0" PG_EXTENSION_MAJOR: "16" diff --git a/.github/workflows/rust-coverage.yml b/.github/workflows/rust-coverage.yml index 226210a53..059cddeec 100644 --- a/.github/workflows/rust-coverage.yml +++ b/.github/workflows/rust-coverage.yml @@ -19,7 +19,7 @@ permissions: env: CARGO_TERM_COLOR: always - RUST_VERSION: "1.92.0" + RUST_VERSION: "1.94.0" CARGO_ENCODED_RUSTFLAGS: "" RUSTC_WRAPPER: "" CARGO_BUILD_RUSTC_WRAPPER: "" diff --git a/.github/workflows/rust-sdk.yml b/.github/workflows/rust-sdk.yml index d6681129b..8adb12bfd 100644 --- a/.github/workflows/rust-sdk.yml +++ b/.github/workflows/rust-sdk.yml @@ -37,7 +37,7 @@ permissions: env: CARGO_TERM_COLOR: always - RUST_VERSION: "1.92.0" + RUST_VERSION: "1.94.0" CARGO_ENCODED_RUSTFLAGS: "" RUSTC_WRAPPER: "" CARGO_BUILD_RUSTC_WRAPPER: "" diff --git a/.github/workflows/typescript-sdk.yml b/.github/workflows/typescript-sdk.yml index d2b0296ca..79a63ab3b 100644 --- a/.github/workflows/typescript-sdk.yml +++ b/.github/workflows/typescript-sdk.yml @@ -25,7 +25,7 @@ permissions: env: CARGO_TERM_COLOR: always - RUST_VERSION: "1.92.0" + RUST_VERSION: "1.94.0" CARGO_ENCODED_RUSTFLAGS: "" WASM_PACK_VERSION: "0.15.0" RUSTC_WRAPPER: "" diff --git a/.gitignore b/.gitignore index 8cb2df6bd..7407e0afd 100644 --- a/.gitignore +++ b/.gitignore @@ -104,7 +104,9 @@ ui/package-lock.json /target_cli_smoke_run /link/sdks/dart/link/build /link/sdks/dart/link/.dart_tool -/vendor +/vendor/* +!/vendor/datafusion-functions-json/ +!/vendor/datafusion-functions-json/** /link/sdks/typescript/.wasm-cargo-home-test1 /link/sdks/typescript/.wasm-cargo-home-test2 ts-sdk-repro/server.toml diff --git a/Cargo.lock b/Cargo.lock index 3a1679fcb..8a386883d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -36,9 +36,9 @@ dependencies = [ [[package]] name = "actix-files" -version = "0.6.10" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df8c4f30e3272d7c345f88ae0aac3848507ef5ba871f9cc2a41c8085a0f0523b" +checksum = "ed87c765e9e5be096cc29b43d04b932209892ec871d05fe255f0ae1ccfed9a06" dependencies = [ "actix-http", "actix-service", @@ -504,9 +504,9 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "arrow" -version = "58.3.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "378530e55cd479eda3c14eb345310799717e6f76d0c332041e8487022166b471" +checksum = "61d285d16bce7d0be61912f7928342b673067b6b7d7ef6cc179258ba7de1fecf" dependencies = [ "arrow-arith", "arrow-array", @@ -525,9 +525,9 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "58.3.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0ab212d2c1886e802f51c5212d78ebbcbb0bec980fff9dadc1eb8d45cd0b738" +checksum = "757ef1836251e88222542a7da2623bc1c9cb9e20afefa6db2c41e79991cd91d4" dependencies = [ "arrow-array", "arrow-buffer", @@ -539,9 +539,9 @@ dependencies = [ [[package]] name = "arrow-array" -version = "58.3.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfd33d3e92f207444098c75b42de99d329562be0cf686b307b097cc52b4e999e" +checksum = "bc9a4a4b2b5ecd0e04df03471661cb61f28bed3c7fd50994715129b01b2edb97" dependencies = [ "ahash 0.8.12", "arrow-buffer", @@ -558,21 +558,21 @@ dependencies = [ [[package]] name = "arrow-buffer" -version = "58.3.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c6cd424c2693bcdbc150d843dc9d4d137dd2de4782ce6df491ad11a3a0416c0" +checksum = "c12b576ef18c1deb80925a248b25ad84f419198d791b8e293fc6aaa60441fe90" dependencies = [ "bytes", "half 2.7.1", - "num-bigint", + "num-bigint 0.5.1", "num-traits", ] [[package]] name = "arrow-cast" -version = "58.3.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c5aefb56a2c02e9e2b30746241058b85f8983f0fcff2ba0c6d09006e1cded7f" +checksum = "68338a9096a5dc9bc11927c58c43a8526d96bf6abd2012ef6c0c9f505991cc79" dependencies = [ "arrow-array", "arrow-buffer", @@ -581,7 +581,7 @@ dependencies = [ "arrow-schema", "arrow-select", "atoi", - "base64 0.22.1", + "base64 0.23.1", "chrono", "comfy-table", "half 2.7.1", @@ -592,9 +592,9 @@ dependencies = [ [[package]] name = "arrow-csv" -version = "58.3.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e94e8cf7e517657a52b91ea1263acf38c4ca62a84655d72458a3359b12ab97de" +checksum = "25011b52b346407d497ef0030e12b45e4f2d0cc279efc09c4f3d09106db30e36" dependencies = [ "arrow-array", "arrow-cast", @@ -607,9 +607,9 @@ dependencies = [ [[package]] name = "arrow-data" -version = "58.3.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c88210023a2bfee1896af366309a3028fc3bcbd6515fa29a7990ee1baa08ee0" +checksum = "723fe4aeed7604e00b9883a465af4ff0a0e6c44c03e41a68c3d1cbc403e0e44d" dependencies = [ "arrow-buffer", "arrow-schema", @@ -620,9 +620,9 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "58.3.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "238438f0834483703d88896db6fe5a7138b2230debc31b34c0336c2996e3c64f" +checksum = "149437b14371f5b9ec60f5ddc751483ae99d7a7072653c0075e5e469156eea7b" dependencies = [ "arrow-array", "arrow-buffer", @@ -636,9 +636,9 @@ dependencies = [ [[package]] name = "arrow-json" -version = "58.3.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "205ca2119e6d679d5c133c6f30e68f027738d95ed948cf77677ea69c7800036b" +checksum = "f18b9123ccfec418a663f821c9a034af339711678c11ffe00d3ec07da5ff9f7e" dependencies = [ "arrow-array", "arrow-buffer", @@ -661,9 +661,9 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "58.3.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bffd8fd2579286a5d63bac898159873e5094a79009940bcb42bbfce4f19f1d0" +checksum = "e6c08dff0686cf23ca4f562803f191ccbeb726dbae6309cd4b4aaf65e0f2c979" dependencies = [ "arrow-array", "arrow-buffer", @@ -674,9 +674,9 @@ dependencies = [ [[package]] name = "arrow-row" -version = "58.3.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bab5994731204603c73ba69267616c50f80780774c6bb0476f1f830625115e0c" +checksum = "bbec439386df71ad570e6758a946111322b9e9dc8db83b5527321f0b4c9119c2" dependencies = [ "arrow-array", "arrow-buffer", @@ -687,9 +687,9 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "58.3.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f633dbfdf39c039ada1bf9e34c694816eb71fbb7dc78f613993b7245e078a1ed" +checksum = "e6fed2ca0d1eade57e811cbe73b98ad50cc08a1183e13b2d2aa43a7df593f40e" dependencies = [ "serde_core", "serde_json", @@ -697,9 +697,9 @@ dependencies = [ [[package]] name = "arrow-select" -version = "58.3.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cd065c54172ac787cf3f2f8d4107e0d3fdc26edba76fdf4f4cc170258942222" +checksum = "466b19cf75130b891dc1b23a84b343c714c62c64c9c62e365c76aa0ff90a53fb" dependencies = [ "ahash 0.8.12", "arrow-array", @@ -711,9 +711,9 @@ dependencies = [ [[package]] name = "arrow-string" -version = "58.3.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29dd7cda3ab9692f43a2e4acc444d760cc17b12bb6d8232ddf64e9bab7c06b42" +checksum = "c838a25bb3691e919e0f617616ac51a4ff8517a952e29ca133cf0c22b2ce65b1" dependencies = [ "arrow-array", "arrow-buffer", @@ -1009,7 +1009,7 @@ checksum = "4d6867f1565b3aad85681f1015055b087fcfd840d6aeee6eee7f2da317603695" dependencies = [ "autocfg", "libm", - "num-bigint", + "num-bigint 0.4.6", "num-integer", "num-traits", ] @@ -2110,9 +2110,9 @@ checksum = "d7a1e2f27636f116493b8b860f5546edb47c8d8f8ea73e1d2a20be88e28d1fea" [[package]] name = "datafusion" -version = "54.1.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "754ef4e8f073922a26f5b23133b9db4829342362b09be0bc94309cf261c2f098" +checksum = "96f76f0167ed0842b29a3d1e41be3c034c0a46409a3a703cc4cc84ee8c24abf4" dependencies = [ "arrow", "arrow-schema", @@ -2145,7 +2145,7 @@ dependencies = [ "datafusion-sql", "futures", "indexmap 2.14.0", - "itertools 0.14.0", + "itertools 0.15.0", "log", "object_store", "parking_lot", @@ -2159,9 +2159,9 @@ dependencies = [ [[package]] name = "datafusion-catalog" -version = "54.1.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06afd1e38dd27bbb1258685a1fc6524df6aff4e07b25b393a47de59635178d99" +checksum = "d79ec3460f6ed5c58f9b3f2d873fbc77748b82653bff1b4cdaf06de33bb4e05f" dependencies = [ "arrow", "async-trait", @@ -2175,7 +2175,7 @@ dependencies = [ "datafusion-physical-plan", "datafusion-session", "futures", - "itertools 0.14.0", + "itertools 0.15.0", "log", "object_store", "parking_lot", @@ -2184,9 +2184,9 @@ dependencies = [ [[package]] name = "datafusion-catalog-listing" -version = "54.1.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0668fb32c12065ec242be0e5b4bc62bd7a06a0be3ecd83791ef877e4be67e02" +checksum = "b48cef241e2efcfd496fe05ae4d0d5de20793451862faefe406c397a467e12d4" dependencies = [ "arrow", "async-trait", @@ -2200,16 +2200,17 @@ dependencies = [ "datafusion-physical-expr-common", "datafusion-physical-plan", "futures", - "itertools 0.14.0", + "itertools 0.15.0", "log", "object_store", + "percent-encoding", ] [[package]] name = "datafusion-common" -version = "54.1.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca43b263cdff57042cfa8fb817fb3469f4878933380dccff25f5e793580abbf9" +checksum = "3f72810485975c258f1b4d00baab31728470676c60c5546f366ebd0d99f05ab6" dependencies = [ "arrow", "arrow-ipc", @@ -2219,9 +2220,10 @@ dependencies = [ "half 2.7.1", "hashbrown 0.17.1", "indexmap 2.14.0", - "itertools 0.14.0", + "itertools 0.15.0", "libc", "log", + "num-traits", "object_store", "parquet", "recursive", @@ -2233,9 +2235,9 @@ dependencies = [ [[package]] name = "datafusion-common-runtime" -version = "54.1.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05f0ba2b864792bdca4d76c59a1de0ab6e1b61946596b9936888dbd6360035f2" +checksum = "533c28e75dba52f41bde187d23a1cb24ab91c7c097966824fa471e67b60320ea" dependencies = [ "futures", "log", @@ -2244,9 +2246,9 @@ dependencies = [ [[package]] name = "datafusion-datasource" -version = "54.1.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b840a8bce0bcbf5afad02946d438591e7c373f7afccaf3d874c04485772514dd" +checksum = "5b00a1fa0da26f6087136a82fea7f13c76a672cbab452d4086952a7cf770a19b" dependencies = [ "arrow", "async-trait", @@ -2263,7 +2265,7 @@ dependencies = [ "datafusion-session", "futures", "glob", - "itertools 0.14.0", + "itertools 0.15.0", "log", "object_store", "parking_lot", @@ -2274,9 +2276,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-arrow" -version = "54.1.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a24cc0b9cf6e367f27f27406eff13abf48a11b72446aaa40b3105c0ded5c17d9" +checksum = "5ad17ec881bff2ed7768b4bfe971d3efbf3473f2fd1f9d365447bccbdf908678" dependencies = [ "arrow", "arrow-ipc", @@ -2291,16 +2293,16 @@ dependencies = [ "datafusion-physical-plan", "datafusion-session", "futures", - "itertools 0.14.0", + "itertools 0.15.0", "object_store", "tokio", ] [[package]] name = "datafusion-datasource-csv" -version = "54.1.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1abe56b2a7a2d1d6de5117dd1a203181e267f28529faa5da546947621b697d7" +checksum = "b5345285b0c3eaab412e7539b706973c083bd7e5bce575de5e0a3da488d08d1d" dependencies = [ "arrow", "async-trait", @@ -2321,9 +2323,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-json" -version = "54.1.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c3e467f0611ad7bdd5aad17c63c9bb6182d04e5282e5496d897ea2b49c024ba" +checksum = "da02fb9324f56bd8c53f1ee2e949547425cb66f76adc6832b10d44f80a1221d2" dependencies = [ "arrow", "async-trait", @@ -2344,11 +2346,12 @@ dependencies = [ [[package]] name = "datafusion-datasource-parquet" -version = "54.1.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cc35b92cd560082155e80d9c826929c852d3c51543f4affd3a51c464a0aab3a" +checksum = "3c0b0dc1453952952fd5c69ad1c7f6042176e69ed233011d47e07cf74ed0949e" dependencies = [ "arrow", + "arrow-schema", "async-trait", "bytes", "datafusion-common", @@ -2365,7 +2368,7 @@ dependencies = [ "datafusion-pruning", "datafusion-session", "futures", - "itertools 0.14.0", + "itertools 0.15.0", "log", "object_store", "parking_lot", @@ -2375,19 +2378,20 @@ dependencies = [ [[package]] name = "datafusion-doc" -version = "54.1.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d69bb69d8769e34f76839c960dbde24c1ac0c885a79b6c3c2287bdc56ec67891" +checksum = "a88fd985bc0550c36f557db69543cc9d6393b1509783520b30e902f23c555da6" [[package]] name = "datafusion-execution" -version = "54.1.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8eac0a09bc8d263f52025cad9e001da4d8138d633fa288edda4d06b1772eae6" +checksum = "a98f1052f91b4991f0bf2ce1e4e36dfbdcda454a956b8c8d562c7c845e8fce1d" dependencies = [ "arrow", "arrow-buffer", "async-trait", + "bytes", "dashmap 6.2.1", "datafusion-common", "datafusion-expr", @@ -2396,16 +2400,19 @@ dependencies = [ "log", "object_store", "parking_lot", + "pin-project-lite", "rand 0.9.2", "tempfile", + "tokio", + "tokio-util", "url", ] [[package]] name = "datafusion-expr" -version = "54.1.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eeb14d374767ee0fc62dc79a5ba8bcf8a63c14e993c7d992d0e63adfa23d77d3" +checksum = "464625a1f0e4b9df552d894fafcc8aac953ebbc8b0fa0acdaf20975fd615040e" dependencies = [ "arrow", "arrow-schema", @@ -2418,7 +2425,7 @@ dependencies = [ "datafusion-functions-window-common", "datafusion-physical-expr-common", "indexmap 2.14.0", - "itertools 0.14.0", + "itertools 0.15.0", "recursive", "serde_json", "sqlparser", @@ -2426,25 +2433,25 @@ dependencies = [ [[package]] name = "datafusion-expr-common" -version = "54.1.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7b19a8c95522bee8cbb313d74263b85e355d2b52f42e67ef5694bf5de9e9356" +checksum = "2604994999d5aeca1d1df645ffc98bc787447aaff05dde27aad0342b48fc1fe0" dependencies = [ "arrow", "datafusion-common", "indexmap 2.14.0", - "itertools 0.14.0", + "itertools 0.15.0", ] [[package]] name = "datafusion-functions" -version = "54.1.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f64c983bbbdcb729d921a2b2ac3375598719b5cc0c30345ad664936f3176fc7" +checksum = "051e97533e6af53e4aa0a0667cadc886abcaf36c4a5925019c55c0aa4c218fde" dependencies = [ "arrow", "arrow-buffer", - "base64 0.22.1", + "base64 0.23.1", "chrono", "chrono-tz", "datafusion-common", @@ -2455,7 +2462,7 @@ dependencies = [ "datafusion-macros", "datafusion-physical-expr-common", "hex", - "itertools 0.14.0", + "itertools 0.15.0", "log", "memchr", "num-traits", @@ -2466,9 +2473,9 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate" -version = "54.1.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89bc17041e424a47ed062f43df24d84aab8b57c4c3221e5c1a5eef46d6c5718b" +checksum = "2d0f1bb166d3572b6ed40e1afb2faaacade962abc08c2fcf04babee74681c56b" dependencies = [ "arrow", "datafusion-common", @@ -2479,17 +2486,17 @@ dependencies = [ "datafusion-macros", "datafusion-physical-expr", "datafusion-physical-expr-common", - "foldhash 0.2.0", "half 2.7.1", + "hashbrown 0.17.1", "log", "num-traits", ] [[package]] name = "datafusion-functions-aggregate-common" -version = "54.1.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97dd2a9e865c6108059f5b37b77934f84b50bfb108f837bd0e5c9536e03f0545" +checksum = "7ed756770f5f98369e181d692fd5ee6b1127ffd7322caba92f3730f9f5c92333" dependencies = [ "arrow", "datafusion-common", @@ -2499,9 +2506,7 @@ dependencies = [ [[package]] name = "datafusion-functions-json" -version = "0.54.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab80590f2e0c240fd14a8178b584941f5ced55ee90cd41a128e6c08176c0f7cc" +version = "0.55.0" dependencies = [ "datafusion", "jiter", @@ -2512,9 +2517,9 @@ dependencies = [ [[package]] name = "datafusion-functions-nested" -version = "54.1.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75f0bdfeef16d96417b9632ef855645376b242e9006a126dfd0bedfc54a93f5f" +checksum = "91173fdb5c0ff2a41169a8ffa1b385b8844f18728747bb0a37e35ad7d5772a4f" dependencies = [ "arrow", "arrow-ord", @@ -2529,7 +2534,7 @@ dependencies = [ "datafusion-macros", "datafusion-physical-expr-common", "hashbrown 0.17.1", - "itertools 0.14.0", + "itertools 0.15.0", "itoa", "log", "memchr", @@ -2537,9 +2542,9 @@ dependencies = [ [[package]] name = "datafusion-functions-table" -version = "54.1.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4e4941673c917819616877e9993da4503e4f4739812be0bc32c5356184c6383" +checksum = "b1bcdfb286a745461b126719c32700777e83df4f17cc44db5d71ebce5731e840" dependencies = [ "arrow", "async-trait", @@ -2553,9 +2558,9 @@ dependencies = [ [[package]] name = "datafusion-functions-window" -version = "54.1.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12dd2e16c12b84b6f6b41b19f55b366dd1c46876bb35b86896c6349067379e8d" +checksum = "9ec4b508f1f93f00038ba3e737e894ec6c775528b4369413386655ae6125f0fc" dependencies = [ "arrow", "datafusion-common", @@ -2570,9 +2575,9 @@ dependencies = [ [[package]] name = "datafusion-functions-window-common" -version = "54.1.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cdc5e4b6f8b6ef823cc1c761f85088ad4c884fe8df64df3cbcc6b2b84698441" +checksum = "0b352020834140073fbf5b46ee0ceb926e5074a9d0bcae1dbd91d0586d999cde" dependencies = [ "datafusion-common", "datafusion-physical-expr-common", @@ -2580,20 +2585,20 @@ dependencies = [ [[package]] name = "datafusion-macros" -version = "54.1.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a3614234dd93578c92428cb4f408e020874f0d2b7e6c90c928d9d28b5df2ceb" +checksum = "15192effab05d38cce10e92a6fb48c967b5f166b27b7195a165a72b232569c58" dependencies = [ "datafusion-doc", "quote", - "syn 2.0.119", + "syn 3.0.3", ] [[package]] name = "datafusion-optimizer" -version = "54.1.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0635620b050b81bb92764e99250868f654e2cd5ad1bece413283b3f73c83179" +checksum = "854445d9f7847e1e46089cf61b8d341a64382f14484e912c83a0f23b31216896" dependencies = [ "arrow", "chrono", @@ -2602,7 +2607,7 @@ dependencies = [ "datafusion-expr-common", "datafusion-physical-expr", "indexmap 2.14.0", - "itertools 0.14.0", + "itertools 0.15.0", "log", "recursive", "regex", @@ -2611,9 +2616,9 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" -version = "54.1.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cabf7a86eb70b816729e33c81bf7767c936ee1226f607a114f5dac2decac8d0" +checksum = "671558dad1d2aa253c39c0a4c52515958b99eb91abf649f4b88d5e69cc55282f" dependencies = [ "arrow", "datafusion-common", @@ -2624,7 +2629,7 @@ dependencies = [ "half 2.7.1", "hashbrown 0.17.1", "indexmap 2.14.0", - "itertools 0.14.0", + "itertools 0.15.0", "parking_lot", "petgraph", "recursive", @@ -2633,9 +2638,9 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-adapter" -version = "54.1.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de222e04f7e6744501555a54ab0abe26bfdfebee380af79a9bdc175704246859" +checksum = "ffae3d78c2da80ecc829cb58536cc5aca2e99cf1365eda694fc75bfe288861e0" dependencies = [ "arrow", "datafusion-common", @@ -2643,14 +2648,14 @@ dependencies = [ "datafusion-functions", "datafusion-physical-expr", "datafusion-physical-expr-common", - "itertools 0.14.0", + "itertools 0.15.0", ] [[package]] name = "datafusion-physical-expr-common" -version = "54.1.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72d0d0057fc5a502d45c870cb6d47c66eb7bdd5edb1bd71ad6f3f724975ac2a8" +checksum = "3d9092ed15e7203fbd0903215172f7c9d18f10d94cba35137f3b3836f7c46f16" dependencies = [ "arrow", "chrono", @@ -2658,16 +2663,16 @@ dependencies = [ "datafusion-expr-common", "hashbrown 0.17.1", "indexmap 2.14.0", - "itertools 0.14.0", + "itertools 0.15.0", "parking_lot", "pin-project", ] [[package]] name = "datafusion-physical-optimizer" -version = "54.1.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86046eed10950c5f9aaed9acfd148e9bd2e1dfdfe4f9aef607d1447b271e4183" +checksum = "9005b6cf50b57b72d476c6ed4662b04be7ca6be5320ba9127c6d0b7e4218095b" dependencies = [ "arrow", "datafusion-common", @@ -2678,15 +2683,16 @@ dependencies = [ "datafusion-physical-expr-common", "datafusion-physical-plan", "datafusion-pruning", - "itertools 0.14.0", + "datafusion-session", + "itertools 0.15.0", "recursive", ] [[package]] name = "datafusion-physical-plan" -version = "54.1.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9bc84da934c903407ba297971ebcc020c4c1a38aafd765d6c144c76eff3fa6a1" +checksum = "5787e4fcff4adc4fce8948441103a99705018b49c8dff0720b650bd7a15da112" dependencies = [ "arrow", "arrow-data", @@ -2694,6 +2700,7 @@ dependencies = [ "arrow-ord", "arrow-schema", "async-trait", + "bytes", "datafusion-common", "datafusion-common-runtime", "datafusion-execution", @@ -2707,19 +2714,20 @@ dependencies = [ "half 2.7.1", "hashbrown 0.17.1", "indexmap 2.14.0", - "itertools 0.14.0", + "itertools 0.15.0", "log", "num-traits", "parking_lot", "pin-project-lite", + "serde_json", "tokio", ] [[package]] name = "datafusion-pruning" -version = "54.1.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb63eeac6de19be40f487b65dd84e546195f783c5a9928618e0c4f2a3569b0d7" +checksum = "9e651c8df0b90daed6a7be5921ec0ee379e6909705f063eeff70fd4e35010e4c" dependencies = [ "arrow", "datafusion-common", @@ -2733,10 +2741,11 @@ dependencies = [ [[package]] name = "datafusion-session" -version = "54.1.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f961d209177f91bd014db5cbb2c33b7d28a2597b9003e77f17aeb712964315a" +checksum = "fb56667ee38217efab19b895d9a936052cfb47ed438a19663351bdc42a6214a1" dependencies = [ + "arrow-schema", "async-trait", "datafusion-common", "datafusion-execution", @@ -2747,9 +2756,9 @@ dependencies = [ [[package]] name = "datafusion-sql" -version = "54.1.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1d71cb454da682b2af7488e1fc1ddd72ee1b28f19297b8ccad73f0a21ee9a69" +checksum = "9c29067cb9d32f8e603c45e15d61ea18f1069f96ceafeceb4e18466b8e5b31d9" dependencies = [ "arrow", "bigdecimal", @@ -2762,6 +2771,7 @@ dependencies = [ "recursive", "regex", "sqlparser", + "stacker", ] [[package]] @@ -2795,7 +2805,7 @@ dependencies = [ "asn1-rs", "displaydoc", "nom", - "num-bigint", + "num-bigint 0.4.6", "num-traits", "rusticata-macros", ] @@ -4131,12 +4141,6 @@ dependencies = [ "hybrid-array", ] -[[package]] -name = "integer-encoding" -version = "3.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02" - [[package]] name = "ipnet" version = "2.12.1" @@ -4192,6 +4196,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b4baf93f58d4425749ca49a51c50ebab072c5df6994d08fed93541c331481dc" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.17" @@ -4207,7 +4220,7 @@ dependencies = [ "ahash 0.8.12", "bitvec", "lexical-parse-float", - "num-bigint", + "num-bigint 0.4.6", "num-traits", "pyo3", "smallvec", @@ -5697,9 +5710,9 @@ checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" [[package]] name = "lz4_flex" -version = "0.13.0" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db9a0d582c2874f68138a16ce1867e0ffde6c0bb0a0df85e1f36d04146db488a" +checksum = "ecbdfe44b1bd960b68170b417450a628c43f7cf56bb3c5317e61cb230ee7f226" dependencies = [ "twox-hash", ] @@ -5753,9 +5766,9 @@ checksum = "ae960838283323069879657ca3de837e9f7bbb4c7bf6ea7f1b290d5e9476d2e0" [[package]] name = "memchr" -version = "2.8.0" +version = "2.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" [[package]] name = "mimalloc" @@ -5967,7 +5980,7 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" dependencies = [ - "num-bigint", + "num-bigint 0.4.6", "num-complex", "num-integer", "num-iter", @@ -5985,6 +5998,16 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-bigint" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93e7820bc0a80a0238e650327316f929ba18d5be054b647490a3a6a339f3e7c0" +dependencies = [ + "num-integer", + "num-traits", +] + [[package]] name = "num-bigint-dig" version = "0.8.6" @@ -6057,7 +6080,7 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" dependencies = [ - "num-bigint", + "num-bigint 0.4.6", "num-integer", "num-traits", ] @@ -6484,9 +6507,9 @@ dependencies = [ [[package]] name = "parquet" -version = "58.3.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5dafa7d01085b62a47dd0c1829550a0a36710ea9c4fe358a05a85477cec8a908" +checksum = "7065842956a20c2a536924ce8e4d9955f7422451511b9eb7500d7bfe5077e59c" dependencies = [ "ahash 0.8.12", "arrow-array", @@ -6495,7 +6518,7 @@ dependencies = [ "arrow-ipc", "arrow-schema", "arrow-select", - "base64 0.22.1", + "base64 0.23.1", "brotli", "bytes", "chrono", @@ -6504,15 +6527,13 @@ dependencies = [ "half 2.7.1", "hashbrown 0.17.1", "lz4_flex", - "num-bigint", + "num-bigint 0.5.1", "num-integer", "num-traits", "object_store", - "paste", "seq-macro", "simdutf8", "snap", - "thrift", "tokio", "twox-hash", "zstd", @@ -7147,7 +7168,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "91fd8e38a3b50ed1167fb981cd6fd60147e091784c427b8f7183a7ee32c31c12" dependencies = [ "libc", - "num-bigint", + "num-bigint 0.4.6", "num-traits", "once_cell", "portable-atomic", @@ -8143,6 +8164,7 @@ version = "1.0.151" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" dependencies = [ + "indexmap 2.14.0", "itoa", "memchr", "serde", @@ -8466,15 +8488,15 @@ checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" [[package]] name = "stacker" -version = "0.1.23" +version = "0.1.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08d74a23609d509411d10e2176dc2a4346e3b4aea2e7b1869f19fdedbc71c013" +checksum = "707f49d46706bacf8a2b00d51dace3f9de527c13eec3778f570c411f89e69967" dependencies = [ "cc", "cfg-if", "libc", "psm", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -8787,17 +8809,6 @@ dependencies = [ "num_cpus", ] -[[package]] -name = "thrift" -version = "0.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e54bc85fc7faa8bc175c4bab5b92ba8d9a3ce893d0e9f42cc455c8ab16a9e09" -dependencies = [ - "byteorder", - "integer-encoding", - "ordered-float", -] - [[package]] name = "time" version = "0.3.47" @@ -9537,11 +9548,20 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "v_escape-base" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1212fce830b75af194b578e55b3db9049f2c8c45f58d397fb25602fdb50fb3d" + [[package]] name = "v_htmlescape" -version = "0.15.8" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e8257fbc510f0a46eb602c10215901938b5c2a7d5e70fc11483b1d3c9b5b18c" +checksum = "befb3d53c9e3ec641417685896cbc8cc5bd264d6a2e190c56aaef1af24740d99" +dependencies = [ + "v_escape-base", +] [[package]] name = "validit" @@ -10004,15 +10024,6 @@ dependencies = [ "windows-targets 0.52.6", ] -[[package]] -name = "windows-sys" -version = "0.59.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" -dependencies = [ - "windows-targets 0.52.6", -] - [[package]] name = "windows-sys" version = "0.60.2" diff --git a/Cargo.toml b/Cargo.toml index c8b817219..905f79dd0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -64,7 +64,7 @@ exclude = ["benchv2"] [workspace.package] version = "0.5.6-rc.1" edition = "2021" -rust-version = "1.92" +rust-version = "1.94" authors = ["KalamDB Team"] license = "Apache-2.0" repository = "https://github.com/kalamdb/KalamDB" @@ -101,7 +101,7 @@ tokio-tungstenite = { version = "0.30.0", features = ["rustls-tls-webpki-roots"] # Security floor pins for vulnerable transitive TLS/QUIC crates. aws-lc-rs = { version = "1.18.0", default-features = false } rustls = { version = "0.23.43", default-features = false } -quinn-proto = { version = "0.11.16", default-features = false } +quinn-proto = { version = "0.11.17", default-features = false } rustls-webpki = { version = "0.103.14", default-features = false } # Time handling @@ -112,26 +112,26 @@ chrono = { version = "0.4.45", features = ["serde"] } # Apache Arrow ecosystem # IMPORTANT: arrow, arrow-schema and parquet MUST match the version used by DataFusion. -# DataFusion 52.3.0 resolves to Arrow 57.3.0; mixing in Arrow 58 pulls a second +# DataFusion 55.0.0 resolves to Arrow 59.2.0; mixing Arrow families pulls a second # arrow_array crate into the graph and causes FixedSizeListArray type mismatches. # Upgrade DataFusion first before bumping Arrow/Parquet beyond this family. -arrow = { version = "58.3.0", default-features = false } -arrow-ipc = { version = "58.3.0", default-features = false } -arrow-schema = { version = "58.3.0" } -datafusion = { version = "54.1.0", default-features = false, features = ["sql", "parquet", "recursive_protection", "nested_expressions"] } -datafusion-datasource = { version = "54.1.0", default-features = false } -datafusion-common = { version = "54.1.0", default-features = false } -datafusion-expr = { version = "54.1.0" } -datafusion-functions-json = { version = "0.54.2" } +arrow = { version = "59.2.0", default-features = false } +arrow-ipc = { version = "59.2.0", default-features = false } +arrow-schema = { version = "59.2.0" } +datafusion = { version = "55.0.0", default-features = false, features = ["sql", "parquet", "recursive_protection", "nested_expressions"] } +datafusion-datasource = { version = "55.0.0", default-features = false } +datafusion-common = { version = "55.0.0", default-features = false } +datafusion-expr = { version = "55.0.0" } +datafusion-functions-json = { path = "vendor/datafusion-functions-json" } sqlparser = { version = "0.62.0" } -parquet = { version = "58.3.0", default-features = false, features = ["snap", "zstd", "arrow", "async"] } +parquet = { version = "59.2.0", default-features = false, features = ["snap", "zstd", "arrow", "async"] } # Web framework actix-web = { version = "4.14.1", features = ["http2"] } actix-ws = "0.4.0" actix-cors = "0.7" actix-rt = "2.10" -actix-files = "0.6.10" +actix-files = "0.7.0" actix-multipart = "0.8.0" # UUID generation diff --git a/backend/crates/kalamdb-core/src/sql/datafusion_session.rs b/backend/crates/kalamdb-core/src/sql/datafusion_session.rs index 25e815274..4cdf7a21b 100644 --- a/backend/crates/kalamdb-core/src/sql/datafusion_session.rs +++ b/backend/crates/kalamdb-core/src/sql/datafusion_session.rs @@ -84,7 +84,7 @@ impl DataFusionSessionFactory { settings.batch_size ); - // DuckDB dialect enables SQL lambda parsing (`x -> expr`) required by DataFusion 54 + // DuckDB dialect enables SQL lambda parsing (`x -> expr`) required by DataFusion 55 // higher-order array functions such as array_transform and array_filter. let config = SessionConfig::new() .set_str("datafusion.sql_parser.dialect", "duckdb") @@ -180,14 +180,14 @@ impl DataFusionSessionFactory { // dialect rewrite layer converts operators to these UDFs before planning. datafusion_functions_json::register_all(ctx).expect("failed to register JSON functions"); - // Ensure DataFusion 54 nested/lambda array functions and planners are registered + // Ensure DataFusion 55 nested/lambda array functions and planners are registered // for SQL like `array_transform(arr, x -> x * 2)`. datafusion::functions_nested::register_all(ctx) .expect("failed to register nested expression functions"); // Register COSINE_DISTANCE(vector, query_vector) for ORDER BY similarity search syntax. // The dispatcher routes JSON query literals internally and delegates array/array - // inputs to DataFusion 54's native cosine_distance implementation. + // inputs to DataFusion 55's native cosine_distance implementation. ctx.register_udf(ScalarUDF::from(CosineDistanceFunction::new())); // Register vector search table function (TABLE(vector_search(...))). diff --git a/backend/crates/kalamdb-core/tests/test_datafusion54_sql_features.rs b/backend/crates/kalamdb-core/tests/test_datafusion55_sql_features.rs similarity index 99% rename from backend/crates/kalamdb-core/tests/test_datafusion54_sql_features.rs rename to backend/crates/kalamdb-core/tests/test_datafusion55_sql_features.rs index 8f3938ce8..b3c2d8ec2 100644 --- a/backend/crates/kalamdb-core/tests/test_datafusion54_sql_features.rs +++ b/backend/crates/kalamdb-core/tests/test_datafusion55_sql_features.rs @@ -1,4 +1,4 @@ -//! Integration tests for DataFusion 54 SQL features used by KalamDB. +//! Integration tests for DataFusion 55 SQL features used by KalamDB. use std::sync::Arc; diff --git a/backend/crates/kalamdb-datafusion-sources/src/exec.rs b/backend/crates/kalamdb-datafusion-sources/src/exec.rs index 8a66aac02..179081999 100644 --- a/backend/crates/kalamdb-datafusion-sources/src/exec.rs +++ b/backend/crates/kalamdb-datafusion-sources/src/exec.rs @@ -1,4 +1,4 @@ -//! Shared [`ExecutionPlan`] scaffolding built on the DataFusion 54.x surface. +//! Shared [`ExecutionPlan`] scaffolding built on the DataFusion 55.x surface. //! //! This module intentionally stays thin: it provides helpers that consumers //! embed inside their own `ExecutionPlan` implementations, instead of forcing a @@ -23,12 +23,14 @@ use arrow::{ use arrow_schema::SchemaRef; use async_trait::async_trait; use datafusion::{ + common::tree_node::TreeNodeRecursion, error::{DataFusionError, Result as DataFusionResult}, execution::{SendableRecordBatchStream, TaskContext}, physical_expr::PhysicalExpr, physical_plan::{ metrics::{Count, ExecutionPlanMetricsSet, MetricBuilder, MetricsSet}, - DisplayAs, DisplayFormatType, ExecutionPlan, PlanProperties, + ChildrenPropertiesMode, DisplayAs, DisplayFormatType, ExecutionPlan, PlanProperties, + ReplaceChildrenOptions, }, }; use kalamdb_commons::{ @@ -990,9 +992,17 @@ impl ExecutionPlan for DeferredBatchExec { Vec::new() } - fn with_new_children( + fn apply_expressions( + &self, + _f: &mut dyn FnMut(&Arc) -> DataFusionResult, + ) -> DataFusionResult { + Ok(TreeNodeRecursion::Continue) + } + + fn replace_children( self: Arc, children: Vec>, + _options: ReplaceChildrenOptions, ) -> DataFusionResult> { if !children.is_empty() { return Err(DataFusionError::Execution( @@ -1002,6 +1012,16 @@ impl ExecutionPlan for DeferredBatchExec { Ok(self) } + fn with_new_children( + self: Arc, + children: Vec>, + ) -> DataFusionResult> { + self.replace_children( + children, + ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute), + ) + } + fn execute( &self, partition: usize, diff --git a/backend/crates/kalamdb-datafusion-sources/src/lib.rs b/backend/crates/kalamdb-datafusion-sources/src/lib.rs index faf61d526..fb24155c7 100644 --- a/backend/crates/kalamdb-datafusion-sources/src/lib.rs +++ b/backend/crates/kalamdb-datafusion-sources/src/lib.rs @@ -11,7 +11,7 @@ //! //! - [`provider`]: scan descriptors, capability matrix, and thin `TableProvider`-adjacent traits. //! - [`exec`]: shared [`ExecutionPlan`][datafusion::physical_plan::ExecutionPlan] scaffolding built -//! on the DataFusion 54.x surface. +//! on the DataFusion 55.x surface. //! - [`stream`]: [`SendableRecordBatchStream`][datafusion::execution::SendableRecordBatchStream] //! adapters that preserve Arrow buffer sharing where possible. //! - [`pruning`]: filter, projection, limit, and pruning descriptors reused by every source family. diff --git a/backend/crates/kalamdb-dialect/src/parser/query_parser.rs b/backend/crates/kalamdb-dialect/src/parser/query_parser.rs index 78fc7483b..2709b3557 100644 --- a/backend/crates/kalamdb-dialect/src/parser/query_parser.rs +++ b/backend/crates/kalamdb-dialect/src/parser/query_parser.rs @@ -3,7 +3,7 @@ //! Uses sqlparser-rs for safe SQL parsing to prevent SQL injection attacks //! and ensure proper handling of edge cases. //! -//! General SELECT queries are executed by DataFusion 54 and may use SQL lambda array +//! General SELECT queries are executed by DataFusion 55 and may use SQL lambda array //! functions. Subscription and live-query validation in this module intentionally //! keeps a narrower surface. diff --git a/backend/crates/kalamdb-dialect/src/parser/utils.rs b/backend/crates/kalamdb-dialect/src/parser/utils.rs index 1810f3bf4..1de82cc2b 100644 --- a/backend/crates/kalamdb-dialect/src/parser/utils.rs +++ b/backend/crates/kalamdb-dialect/src/parser/utils.rs @@ -150,7 +150,7 @@ fn rewrite_pg_catalog_functions(sql: &str) -> std::borrow::Cow<'_, str> { /// `CURRENT_USER_ID()` is an alias for `CURRENT_USER()` (both return the user id). /// /// PostgreSQL JSON operators (`->`, `->>`, `?`) are rewritten to `json_get_json`, -/// `json_as_text`, and `json_contains` before DataFusion planning. DataFusion 54 uses +/// `json_as_text`, and `json_contains` before DataFusion planning. DataFusion 55 uses /// the DuckDB SQL dialect for lambda array functions, and DuckDB parses bare `->` as /// lambda syntax instead of JSON extraction. /// diff --git a/backend/crates/kalamdb-dialect/src/query_features.rs b/backend/crates/kalamdb-dialect/src/query_features.rs index c4d0fc667..0413425a8 100644 --- a/backend/crates/kalamdb-dialect/src/query_features.rs +++ b/backend/crates/kalamdb-dialect/src/query_features.rs @@ -1,11 +1,11 @@ //! General SQL query feature support for DataFusion-backed execution. //! -//! KalamDB routes most SELECT statements directly to DataFusion 54, which provides +//! KalamDB routes most SELECT statements directly to DataFusion 55, which provides //! SQL lambda array functions when the session uses the DuckDB SQL dialect //! (`datafusion.sql_parser.dialect = duckdb`). Subscription and live-query paths //! intentionally restrict the SQL surface and remain narrower. -/// DataFusion 54 query features supported in general SQL execution. +/// DataFusion 55 query features supported in general SQL execution. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum GeneralQueryFeature { /// Lambda expressions such as `x -> x * 10` with `array_transform`, `array_filter`, diff --git a/backend/crates/kalamdb-filestore/src/parquet/reader.rs b/backend/crates/kalamdb-filestore/src/parquet/reader.rs index 0a47eee53..43fbc1197 100644 --- a/backend/crates/kalamdb-filestore/src/parquet/reader.rs +++ b/backend/crates/kalamdb-filestore/src/parquet/reader.rs @@ -7,9 +7,9 @@ //! //! - **Column Projection**: Pushes a Parquet projection mask so unneeded column chunks are not //! read or decoded -//! - **Streaming I/O**: All reads use `ParquetObjectReader` — reads only the footer eagerly and -//! fetches column chunks on demand via range requests (remote) or file seeks (local). No -//! full-file downloads. +//! - **Streaming I/O**: All reads use an `AsyncFileReader` over ObjectStore — reads only the +//! footer eagerly and fetches column chunks on demand via range requests (remote) or file +//! seeks (local). No full-file downloads. //! //! # Usage Tiers //! @@ -17,22 +17,32 @@ //! |----------|:-:|:-:|----------| //! | `parse_parquet_stream` | Optional | ✓ | General streaming read (recommended) | -use std::{pin::Pin, sync::Arc}; +use std::{ops::Range, pin::Pin, sync::Arc}; use arrow::record_batch::RecordBatch; -use datafusion::parquet::arrow::{ - async_reader::{ParquetObjectReader, ParquetRecordBatchStreamBuilder}, - ProjectionMask, -}; +use bytes::Bytes; use futures_util::TryStreamExt; -use object_store::{path::Path as ObjectPath, ObjectStore}; +use object_store::{ + path::Path as ObjectPath, GetOptions, GetRange, ObjectStore, ObjectStoreExt, +}; use parquet::{ + arrow::{ + arrow_reader::ArrowReaderOptions, + async_reader::{AsyncFileReader, MetadataSuffixFetch, ParquetRecordBatchStreamBuilder}, + ProjectionMask, + }, basic::Type as ParquetPhysicalType, bloom_filter::Sbbf, - file::{metadata::ParquetMetaData, statistics::Statistics}, + errors::ParquetError, + file::{ + metadata::{ParquetMetaData, ParquetMetaDataReader}, + statistics::Statistics, + }, schema::types::SchemaDescriptor, }; +type BoxFuture<'a, T> = Pin + Send + 'a>>; + use crate::error::{FilestoreError, Result}; // ========== Async streaming reader (ObjectStore-backed) ========== @@ -108,6 +118,77 @@ struct PkBloomPruning { values: Vec, } +/// ObjectStore-backed [`AsyncFileReader`], matching the parquet 59 example that +/// replaced deprecated [`parquet::arrow::async_reader::ParquetObjectReader`]. +#[derive(Clone, Debug)] +struct ObjectStoreReader { + store: Arc, + path: ObjectPath, +} + +impl ObjectStoreReader { + fn new(store: Arc, path: ObjectPath) -> Self { + Self { store, path } + } +} + +fn to_parquet_err(error: object_store::Error) -> ParquetError { + ParquetError::External(Box::new(error)) +} + +impl AsyncFileReader for ObjectStoreReader { + fn get_bytes(&mut self, range: Range) -> BoxFuture<'_, parquet::errors::Result> { + Box::pin(async move { + self.store + .get_range(&self.path, range) + .await + .map_err(to_parquet_err) + }) + } + + fn get_byte_ranges( + &mut self, + ranges: Vec>, + ) -> BoxFuture<'_, parquet::errors::Result>> { + Box::pin(async move { + self.store + .get_ranges(&self.path, &ranges) + .await + .map_err(to_parquet_err) + }) + } + + fn get_metadata<'a>( + &'a mut self, + options: Option<&'a ArrowReaderOptions>, + ) -> BoxFuture<'a, parquet::errors::Result>> { + Box::pin(async move { + let metadata = ParquetMetaDataReader::new() + .with_arrow_reader_options(options) + .load_via_suffix_and_finish(self) + .await?; + Ok(Arc::new(metadata)) + }) + } +} + +impl MetadataSuffixFetch for &mut ObjectStoreReader { + fn fetch_suffix(&mut self, suffix: usize) -> BoxFuture<'_, parquet::errors::Result> { + let options = GetOptions { + range: Some(GetRange::Suffix(suffix as u64)), + ..Default::default() + }; + Box::pin(async move { + let response = self + .store + .get_opts(&self.path, options) + .await + .map_err(to_parquet_err)?; + response.bytes().await.map_err(to_parquet_err) + }) + } +} + /// Open an async streaming reader over a Parquet file via ObjectStore. /// /// Works for any backend (local filesystem, S3, GCS, Azure). Only reads the @@ -132,7 +213,7 @@ pub async fn parse_parquet_stream_with_options( path: &ObjectPath, options: &ParquetReadOptions, ) -> Result { - let reader = ParquetObjectReader::new(store, path.clone()); + let reader = ObjectStoreReader::new(store, path.clone()); let mut builder = ParquetRecordBatchStreamBuilder::new(reader) .await .map_err(|e| FilestoreError::Parquet(e.to_string()))?; @@ -257,7 +338,7 @@ fn seq_stats_overlap(stats: &Statistics, range: &SeqRangePruning) -> bool { } async fn prune_row_groups_by_pk_bloom( - builder: &mut ParquetRecordBatchStreamBuilder, + builder: &mut ParquetRecordBatchStreamBuilder, row_groups: &[usize], pruning: &PkBloomPruning, ) -> Result> { diff --git a/backend/crates/kalamdb-filestore/src/parquet/writer.rs b/backend/crates/kalamdb-filestore/src/parquet/writer.rs index 0fd923ba0..2997d2a11 100644 --- a/backend/crates/kalamdb-filestore/src/parquet/writer.rs +++ b/backend/crates/kalamdb-filestore/src/parquet/writer.rs @@ -176,7 +176,8 @@ fn writer_properties( let col_path: parquet::schema::types::ColumnPath = col.into(); props_builder = props_builder.set_column_bloom_filter_enabled(col_path.clone(), true); props_builder = props_builder.set_column_bloom_filter_fpp(col_path.clone(), 0.01); - props_builder = props_builder.set_column_bloom_filter_ndv(col_path, bloom_ndv_estimate); + props_builder = + props_builder.set_column_bloom_filter_max_ndv(col_path, bloom_ndv_estimate); } } diff --git a/backend/crates/kalamdb-transactions/src/overlay_exec.rs b/backend/crates/kalamdb-transactions/src/overlay_exec.rs index af5c6b8ca..98fc0fe8b 100644 --- a/backend/crates/kalamdb-transactions/src/overlay_exec.rs +++ b/backend/crates/kalamdb-transactions/src/overlay_exec.rs @@ -5,13 +5,13 @@ use std::{ use datafusion::{ arrow::{datatypes::SchemaRef, record_batch::RecordBatch}, - common::Result as DataFusionResult, + common::{tree_node::TreeNodeRecursion, Result as DataFusionResult}, error::DataFusionError, execution::{SendableRecordBatchStream, TaskContext}, - physical_expr::EquivalenceProperties, + physical_expr::{EquivalenceProperties, PhysicalExpr}, physical_plan::{ - DisplayAs, DisplayFormatType, ExecutionPlan, ExecutionPlanProperties, Partitioning, - PlanProperties, + ChildrenPropertiesMode, DisplayAs, DisplayFormatType, ExecutionPlan, + ExecutionPlanProperties, Partitioning, PlanProperties, ReplaceChildrenOptions, }, scalar::ScalarValue, }; @@ -105,20 +105,50 @@ impl ExecutionPlan for TransactionOverlayExec { vec![false] } - fn with_new_children( + fn apply_expressions( + &self, + _f: &mut dyn FnMut(&Arc) -> DataFusionResult, + ) -> DataFusionResult { + Ok(TreeNodeRecursion::Continue) + } + + fn replace_children( self: Arc, mut children: Vec>, + options: ReplaceChildrenOptions, ) -> DataFusionResult> { - let input = children.swap_remove(0); - Ok(Arc::new(Self::try_new( - input, - self.table_id.clone(), - Arc::clone(&self.primary_key_column), - self.overlay.clone(), - self.user_scope.clone(), - self.final_projection.clone(), - self.fetch, - )?)) + if children.len() != 1 { + return Err(DataFusionError::Internal( + "TransactionOverlayExec expects exactly one child".to_string(), + )); + } + + match options.children_properties { + ChildrenPropertiesMode::Keep => { + let mut plan = Self::clone(&*self); + plan.input = children.swap_remove(0); + Ok(Arc::new(plan)) + }, + ChildrenPropertiesMode::Recompute => Ok(Arc::new(Self::try_new( + children.swap_remove(0), + self.table_id.clone(), + Arc::clone(&self.primary_key_column), + self.overlay.clone(), + self.user_scope.clone(), + self.final_projection.clone(), + self.fetch, + )?)), + } + } + + fn with_new_children( + self: Arc, + children: Vec>, + ) -> DataFusionResult> { + self.replace_children( + children, + ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute), + ) } fn execute( @@ -578,9 +608,17 @@ mod tests { Vec::new() } - fn with_new_children( + fn apply_expressions( + &self, + _f: &mut dyn FnMut(&Arc) -> DataFusionResult, + ) -> DataFusionResult { + Ok(TreeNodeRecursion::Continue) + } + + fn replace_children( self: Arc, children: Vec>, + _options: ReplaceChildrenOptions, ) -> DataFusionResult> { if !children.is_empty() { return Err(DataFusionError::Execution( @@ -590,6 +628,16 @@ mod tests { Ok(self) } + fn with_new_children( + self: Arc, + children: Vec>, + ) -> DataFusionResult> { + self.replace_children( + children, + ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute), + ) + } + fn execute( &self, partition: usize, diff --git a/backend/crates/kalamdb-vector/src/sql/cosine_distance.rs b/backend/crates/kalamdb-vector/src/sql/cosine_distance.rs index 5170bc28a..1a57aa902 100644 --- a/backend/crates/kalamdb-vector/src/sql/cosine_distance.rs +++ b/backend/crates/kalamdb-vector/src/sql/cosine_distance.rs @@ -50,7 +50,7 @@ fn json_query_vector_coerce_types( Ok(arg_types.to_vec()) } -/// Dispatches `cosine_distance` to DataFusion 54 for array/array inputs and to the +/// Dispatches `cosine_distance` to DataFusion 55 for array/array inputs and to the /// Kalam JSON-literal path when the query vector is a UTF-8 JSON string. #[derive(Debug, Clone, PartialEq, Eq, Hash, Default)] pub struct CosineDistanceFunction; diff --git a/vendor/datafusion-functions-json/Cargo.toml b/vendor/datafusion-functions-json/Cargo.toml new file mode 100644 index 000000000..886a7776b --- /dev/null +++ b/vendor/datafusion-functions-json/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "datafusion-functions-json" +version = "0.55.0" +edition = "2021" +description = "JSON functions for DataFusion (KalamDB DataFusion 55 compatibility fork of 0.54.2)" +readme = "README.md" +license = "Apache-2.0" +keywords = ["datafusion", "JSON", "SQL"] +categories = ["database-implementations", "parsing"] +repository = "https://github.com/datafusion-contrib/datafusion-functions-json/" +rust-version = "1.88.0" +publish = false + +[dependencies] +datafusion = { version = "55.0.0", default-features = false, features = ["sql"] } +jiter = "0.15.0" +log = "0.4" +paste = "1" +serde_json = "1" diff --git a/vendor/datafusion-functions-json/LICENSE b/vendor/datafusion-functions-json/LICENSE new file mode 100644 index 000000000..261eeb9e9 --- /dev/null +++ b/vendor/datafusion-functions-json/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/vendor/datafusion-functions-json/README.md b/vendor/datafusion-functions-json/README.md new file mode 100644 index 000000000..61e87130b --- /dev/null +++ b/vendor/datafusion-functions-json/README.md @@ -0,0 +1,89 @@ +# datafusion-functions-json + +[![CI](https://github.com/datafusion-contrib/datafusion-functions-json/actions/workflows/ci.yml/badge.svg?event=push)](https://github.com/datafusion-contrib/datafusion-functions-json/actions/workflows/ci.yml?query=branch%3Amain) +[![Crates.io](https://img.shields.io/crates/v/datafusion-functions-json?color=green)](https://crates.io/crates/datafusion-functions-json) + +**Note:** This is not an official Apache Software Foundation release, see [datafusion-contrib/datafusion-functions-json#5](https://github.com/datafusion-contrib/datafusion-functions-json/issues/5). + +This crate provides a set of functions for querying JSON strings in DataFusion. The functions are implemented as scalar functions that can be used in SQL queries. + +To use these functions, you'll just need to call: + +```rust +datafusion_functions_json::register_all(&mut ctx)?; +``` +To register the below JSON functions in your `SessionContext`. + +# Examples + +```sql +-- Create a table with a JSON column stored as a string +CREATE TABLE test_table (id INT, json_col VARCHAR) AS VALUES +(1, '{}'), +(2, '{ "a": 1 }'), +(3, '{ "a": 2 }'), +(4, '{ "a": 1, "b": 2 }'), +(5, '{ "a": 1, "b": 2, "c": 3 }'); + +-- Check if each document contains the key 'b' +SELECT id, json_contains(json_col, 'b') as json_contains FROM test_table; +-- Results in +-- +----+---------------+ +-- | id | json_contains | +-- +----+---------------+ +-- | 1 | false | +-- | 2 | false | +-- | 3 | false | +-- | 4 | true | +-- | 5 | true | +-- +----+---------------+ + +-- Get the value of the key 'a' from each document +SELECT id, json_col->'a' as json_col_a FROM test_table + +-- +----+------------+ +-- | id | json_col_a | +-- +----+------------+ +-- | 1 | {null=} | +-- | 2 | {int=1} | +-- | 3 | {int=2} | +-- | 4 | {int=1} | +-- | 5 | {int=1} | +-- +----+------------+ +``` + + +## Done + +* [x] `json_contains(json: str, *keys: str | int) -> bool` - true if a JSON string has a specific key (used for the `?` operator) +* [x] `json_get(json: str, *keys: str | int) -> JsonUnion` - Get a value from a JSON string by its "path" +* [x] `json_get_str(json: str, *keys: str | int) -> str` - Get a string value from a JSON string by its "path" +* [x] `json_get_int(json: str, *keys: str | int) -> int` - Get an integer value from a JSON string by its "path" +* [x] `json_get_float(json: str, *keys: str | int) -> float` - Get a float value from a JSON string by its "path" +* [x] `json_get_bool(json: str, *keys: str | int) -> bool` - Get a boolean value from a JSON string by its "path" +* [x] `json_get_json(json: str, *keys: str | int) -> str` - Get a nested raw JSON string from a JSON string by its "path" +* [x] `json_get_array(json: str, *keys: str | int) -> array` - Get an arrow array from a JSON string by its "path" +* [x] `json_as_text(json: str, *keys: str | int) -> str` - Get any value from a JSON string by its "path", represented as a string (used for the `->>` operator) +* [x] `json_length(json: str, *keys: str | int) -> int` - get the length of a JSON string or array + +- [x] `->` operator - alias for `json_get` +- [x] `->>` operator - alias for `json_as_text` +- [x] `?` operator - alias for `json_contains` + +### Notes +Cast expressions with `json_get` are rewritten to the appropriate method, e.g. + +```sql +select * from foo where json_get(attributes, 'bar')::string='ham' +``` +Will be rewritten to: +```sql +select * from foo where json_get_str(attributes, 'bar')='ham' +``` + +## TODO (maybe, if they're actually useful) + +* [ ] `json_keys(json: str, *keys: str | int) -> list[str]` - get the keys of a JSON string +* [ ] `json_is_obj(json: str, *keys: str | int) -> bool` - true if the JSON is an object +* [ ] `json_is_array(json: str, *keys: str | int) -> bool` - true if the JSON is an array +* [ ] `json_valid(json: str) -> bool` - true if the JSON is valid diff --git a/vendor/datafusion-functions-json/src/common.rs b/vendor/datafusion-functions-json/src/common.rs new file mode 100644 index 000000000..6b48d7ec0 --- /dev/null +++ b/vendor/datafusion-functions-json/src/common.rs @@ -0,0 +1,598 @@ +use std::str::Utf8Error; +use std::sync::Arc; + +use datafusion::arrow::array::{ + downcast_array, AnyDictionaryArray, Array, ArrayAccessor, ArrayRef, AsArray, DictionaryArray, LargeStringArray, + PrimitiveArray, PrimitiveBuilder, RunArray, StringArray, StringViewArray, +}; +use datafusion::arrow::compute::kernels::cast; +use datafusion::arrow::compute::take; +use datafusion::arrow::datatypes::{ArrowNativeType, DataType, Int64Type, UInt64Type}; +use datafusion::common::{exec_err, plan_err, Result as DataFusionResult, ScalarValue}; +use datafusion::logical_expr::ColumnarValue; +use jiter::{Jiter, JiterError, Peek}; + +use crate::common_union::{ + is_json_union, json_from_union_scalar, nested_json_array, nested_json_array_ref, TYPE_ID_NULL, +}; + +/// General implementation of `ScalarUDFImpl::return_type`. +/// +/// # Arguments +/// +/// * `args` - The arguments to the function +/// * `fn_name` - The name of the function +/// * `value_type` - The general return type of the function, might be wrapped in a dictionary depending +/// on the first argument +pub fn return_type_check(args: &[DataType], fn_name: &str, value_type: DataType) -> DataFusionResult { + let Some(first) = args.first() else { + return plan_err!("The '{fn_name}' function requires one or more arguments."); + }; + let first_dict_key_type = dict_key_type(first); + if !(is_str(first) || is_json_union(first) || first_dict_key_type.is_some()) { + // if !matches!(first, DataType::Utf8 | DataType::LargeUtf8) { + return plan_err!("Unexpected argument type to '{fn_name}' at position 1, expected a string, got {first:?}."); + } + args.iter().skip(1).enumerate().try_for_each(|(index, arg)| { + if is_str(arg) || is_int(arg) || dict_key_type(arg).is_some() { + Ok(()) + } else { + plan_err!( + "Unexpected argument type to '{fn_name}' at position {}, expected string or int, got {arg:?}.", + index + 2 + ) + } + })?; + if first_dict_key_type.is_some() && !value_type.is_primitive() { + Ok(DataType::Dictionary(Box::new(DataType::Int64), Box::new(value_type))) + } else { + Ok(value_type) + } +} + +fn is_str(d: &DataType) -> bool { + matches!(d, DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View) +} + +fn is_int(d: &DataType) -> bool { + // TODO we should support more types of int, but that's a longer task + matches!(d, DataType::UInt64 | DataType::Int64) +} + +fn dict_key_type(d: &DataType) -> Option { + if let DataType::Dictionary(key, value) = d { + if is_str(value) || is_json_union(value) { + return Some(*key.clone()); + } + } + None +} + +#[derive(Debug)] +pub enum JsonPath<'s> { + Key(&'s str), + Index(usize), + None, +} + +impl<'a> From<&'a str> for JsonPath<'a> { + fn from(key: &'a str) -> Self { + JsonPath::Key(key) + } +} + +impl From for JsonPath<'_> { + fn from(index: u64) -> Self { + JsonPath::Index(usize::try_from(index).unwrap()) + } +} + +impl From for JsonPath<'_> { + fn from(index: i64) -> Self { + match usize::try_from(index) { + Ok(i) => Self::Index(i), + Err(_) => Self::None, + } + } +} + +#[derive(Debug)] +enum JsonPathArgs<'a> { + Array(&'a ArrayRef), + Scalars(Vec>), +} + +impl<'s> JsonPathArgs<'s> { + fn extract_path(path_args: &'s [ColumnarValue]) -> DataFusionResult { + // If there is a single argument as an array, we know how to handle it + if let Some((ColumnarValue::Array(array), &[])) = path_args.split_first() { + return Ok(Self::Array(array)); + } + + path_args + .iter() + .enumerate() + .map(|(pos, arg)| match arg { + ColumnarValue::Scalar( + ScalarValue::Utf8(Some(s)) | ScalarValue::Utf8View(Some(s)) | ScalarValue::LargeUtf8(Some(s)), + ) => Ok(JsonPath::Key(s)), + ColumnarValue::Scalar(ScalarValue::UInt64(Some(i))) => Ok((*i).into()), + ColumnarValue::Scalar(ScalarValue::Int64(Some(i))) => Ok((*i).into()), + ColumnarValue::Scalar( + ScalarValue::Null + | ScalarValue::Utf8(None) + | ScalarValue::Utf8View(None) + | ScalarValue::LargeUtf8(None) + | ScalarValue::UInt64(None) + | ScalarValue::Int64(None), + ) => Ok(JsonPath::None), + ColumnarValue::Array(_) => { + // if there was a single arg, which is an array, handled above in the + // split_first case. So this is multiple args of which one is an array + exec_err!("More than 1 path element is not supported when querying JSON using an array.") + } + ColumnarValue::Scalar(arg) => exec_err!( + "Unexpected argument type at position {}, expected string or int, got {arg:?}.", + pos + 1 + ), + }) + .collect::>() + .map(JsonPathArgs::Scalars) + } +} + +pub trait InvokeResult { + type Item; + type Builder; + + // Whether the return type should is allowed to be a dictionary + const ACCEPT_DICT_RETURN: bool; + + fn builder(capacity: usize) -> Self::Builder; + fn append_value(builder: &mut Self::Builder, value: Option); + fn finish(builder: Self::Builder) -> DataFusionResult; + + /// Convert a single value to a scalar + fn scalar(value: Option) -> ScalarValue; +} + +pub fn invoke( + args: &[ColumnarValue], + jiter_find: impl Fn(Option<&str>, &[JsonPath]) -> Result, +) -> DataFusionResult { + let Some((json_arg, path_args)) = args.split_first() else { + return exec_err!("expected at least one argument"); + }; + + let path = JsonPathArgs::extract_path(path_args)?; + match (json_arg, path) { + (ColumnarValue::Array(json_array), JsonPathArgs::Array(path_array)) => { + invoke_array_array::(json_array, path_array, jiter_find).map(ColumnarValue::Array) + } + (ColumnarValue::Array(json_array), JsonPathArgs::Scalars(path)) => { + invoke_array_scalars::(json_array, &path, jiter_find).map(ColumnarValue::Array) + } + (ColumnarValue::Scalar(s), JsonPathArgs::Array(path_array)) => { + invoke_scalar_array::(s, path_array, jiter_find) + } + (ColumnarValue::Scalar(s), JsonPathArgs::Scalars(path)) => { + invoke_scalar_scalars(s, &path, jiter_find, R::scalar) + } + } +} + +fn null_result(len: usize) -> DataFusionResult { + let mut builder = R::builder(len); + for _ in 0..len { + R::append_value(&mut builder, None); + } + R::finish(builder) +} + +fn invoke_array_array( + json_array: &ArrayRef, + path_array: &ArrayRef, + jiter_find: impl Fn(Option<&str>, &[JsonPath]) -> Result, +) -> DataFusionResult { + match json_array.data_type() { + // for string dictionaries, cast dictionary keys to larger types to avoid generic explosion + DataType::Dictionary(_, value_type) if value_type.as_ref() == &DataType::Utf8 => { + let json_array = cast_to_large_dictionary(json_array.as_any_dictionary())?; + let output = zip_apply::( + json_array.downcast_dict::().unwrap(), + path_array, + jiter_find, + )?; + if R::ACCEPT_DICT_RETURN { + // ensure return is a dictionary to satisfy the declaration above in return_type_check + Ok(Arc::new(wrap_as_large_dictionary(&json_array, output))) + } else { + Ok(output) + } + } + DataType::Dictionary(_, value_type) if value_type.as_ref() == &DataType::LargeUtf8 => { + let json_array = cast_to_large_dictionary(json_array.as_any_dictionary())?; + let output = zip_apply::( + json_array.downcast_dict::().unwrap(), + path_array, + jiter_find, + )?; + if R::ACCEPT_DICT_RETURN { + // ensure return is a dictionary to satisfy the declaration above in return_type_check + Ok(Arc::new(wrap_as_large_dictionary(&json_array, output))) + } else { + Ok(output) + } + } + other_dict_type @ DataType::Dictionary(_, _) => { + // Horrible case: dict containing union as input with array for paths, figure + // out from the path type which union members we should access, repack the + // dictionary and then recurse. + if let Some(child_array) = nested_json_array_ref( + json_array.as_any_dictionary().values(), + is_object_lookup_array(path_array.data_type()), + ) { + invoke_array_array::( + &(Arc::new(json_array.as_any_dictionary().with_values(child_array.clone())) as _), + path_array, + jiter_find, + ) + } else { + exec_err!("unexpected json array type {:?}", other_dict_type) + } + } + DataType::Utf8 => zip_apply::(json_array.as_string::(), path_array, jiter_find), + DataType::LargeUtf8 => zip_apply::(json_array.as_string::(), path_array, jiter_find), + DataType::Utf8View => zip_apply::(json_array.as_string_view(), path_array, jiter_find), + DataType::Null => null_result::(json_array.len()), + other => { + if let Some(string_array) = nested_json_array(json_array, is_object_lookup_array(path_array.data_type())) { + zip_apply::(string_array, path_array, jiter_find) + } else { + exec_err!("unexpected json array type {:?}", other) + } + } + } +} + +/// Transform keys that may be pointing to values with nulls to nulls themselves. +/// keys = `[0, 1, 2, 3]`, values = `[null, "a", null, "b"]` +/// into +/// keys = `[null, 0, null, 1]`, values = `["a", "b"]` +/// +/// Arrow / `DataFusion` assumes that dictionary values do not contain nulls, nulls are encoded by the keys. +/// Not following this invariant causes invalid dictionary arrays to be built later on inside of `DataFusion` +/// when arrays are concacted and such. +fn remap_dictionary_key_nulls(keys: PrimitiveArray, values: ArrayRef) -> DictionaryArray { + // fast path: no nulls in values + if values.null_count() == 0 { + return DictionaryArray::new(keys, values); + } + + let mut new_keys_builder = PrimitiveBuilder::::new(); + + for key in &keys { + match key { + Some(k) if values.is_null(k.as_usize()) => new_keys_builder.append_null(), + Some(k) => new_keys_builder.append_value(k), + None => new_keys_builder.append_null(), + } + } + + let new_keys = new_keys_builder.finish(); + DictionaryArray::new(new_keys, values) +} + +fn invoke_array_scalars( + json_array: &ArrayRef, + path: &[JsonPath], + jiter_find: impl Fn(Option<&str>, &[JsonPath]) -> Result, +) -> DataFusionResult { + #[allow(clippy::needless_pass_by_value)] // ArrayAccessor is implemented on references + fn inner<'j, R: InvokeResult>( + json_array: impl ArrayAccessor, + path: &[JsonPath], + jiter_find: impl Fn(Option<&str>, &[JsonPath]) -> Result, + ) -> DataFusionResult { + let mut builder = R::builder(json_array.len()); + for i in 0..json_array.len() { + let opt_json = if json_array.is_null(i) { + None + } else { + Some(json_array.value(i)) + }; + let opt_value = jiter_find(opt_json, path).ok(); + R::append_value(&mut builder, opt_value); + } + R::finish(builder) + } + + match json_array.data_type() { + DataType::Dictionary(_, _) => { + let json_array = json_array.as_any_dictionary(); + let values = invoke_array_scalars::(json_array.values(), path, jiter_find)?; + return if R::ACCEPT_DICT_RETURN { + // make the keys into i64 to avoid generic bloat here + let mut keys: PrimitiveArray = downcast_array(&cast(json_array.keys(), &DataType::Int64)?); + if is_json_union(values.data_type()) { + // JSON union: post-process the array to set keys to null where the union member is null + let type_ids = values.as_union().type_ids(); + keys = mask_dictionary_keys(&keys, type_ids); + } + Ok(Arc::new(remap_dictionary_key_nulls(keys, values))) + } else { + // this is what cast would do under the hood to unpack a dictionary into an array of its values + Ok(take(&values, json_array.keys(), None)?) + }; + } + DataType::Utf8 => inner::(json_array.as_string::(), path, jiter_find), + DataType::LargeUtf8 => inner::(json_array.as_string::(), path, jiter_find), + DataType::Utf8View => inner::(json_array.as_string_view(), path, jiter_find), + DataType::Null => null_result::(json_array.len()), + other => { + if let Some(string_array) = nested_json_array(json_array, is_object_lookup(path)) { + inner::(string_array, path, jiter_find) + } else { + exec_err!("unexpected json array type {:?}", other) + } + } + } +} + +fn invoke_scalar_array( + scalar: &ScalarValue, + path_array: &ArrayRef, + jiter_find: impl Fn(Option<&str>, &[JsonPath]) -> Result, +) -> DataFusionResult { + let s = extract_json_scalar(scalar)?; + let arr = s.map_or_else(|| StringArray::new_null(1), |s| StringArray::new_scalar(s).into_inner()); + + // TODO: possible optimization here if path_array is a dictionary; can apply against the + // dictionary values directly for less work + zip_apply::( + RunArray::try_new( + &PrimitiveArray::::new_scalar(i64::try_from(path_array.len()).expect("len out of i64 range")) + .into_inner(), + &arr, + )? + .downcast::() + .expect("type known"), + path_array, + jiter_find, + ) + // FIXME edge cases where scalar is wrapped in a dictionary, should return a dictionary? + .map(ColumnarValue::Array) +} + +fn invoke_scalar_scalars( + scalar: &ScalarValue, + path: &[JsonPath], + jiter_find: impl Fn(Option<&str>, &[JsonPath]) -> Result, + to_scalar: impl Fn(Option) -> ScalarValue, +) -> DataFusionResult { + let s = extract_json_scalar(scalar)?; + let v = jiter_find(s, path).ok(); + // FIXME edge cases where scalar is wrapped in a dictionary, should return a dictionary? + Ok(ColumnarValue::Scalar(to_scalar(v))) +} + +fn zip_apply<'a, R: InvokeResult>( + json_array: impl ArrayAccessor, + path_array: &ArrayRef, + jiter_find: impl Fn(Option<&'a str>, &[JsonPath]) -> Result, +) -> DataFusionResult { + fn get_array_values<'j, 'p, P: Into>>( + j: &impl ArrayAccessor, + p: &impl ArrayAccessor, + index: usize, + ) -> Option<(Option<&'j str>, JsonPath<'p>)> { + let path = if p.is_null(index) { + return None; + } else { + p.value(index).into() + }; + + let json = if j.is_null(index) { None } else { Some(j.value(index)) }; + + Some((json, path)) + } + + #[allow(clippy::needless_pass_by_value)] // ArrayAccessor is implemented on references + fn inner<'a, 'p, P: Into>, R: InvokeResult>( + json_array: impl ArrayAccessor, + path_array: impl ArrayAccessor, + jiter_find: impl Fn(Option<&'a str>, &[JsonPath]) -> Result, + ) -> DataFusionResult { + let mut builder = R::builder(json_array.len()); + for i in 0..json_array.len() { + if let Some((opt_json, path)) = get_array_values(&json_array, &path_array, i) { + let value = jiter_find(opt_json, &[path]).ok(); + R::append_value(&mut builder, value); + } else { + R::append_value(&mut builder, None); + } + } + R::finish(builder) + } + + match path_array.data_type() { + // for string dictionaries, cast dictionary keys to larger types to avoid generic explosion + DataType::Dictionary(_, value_type) if value_type.as_ref() == &DataType::Utf8 => { + let path_array = cast_to_large_dictionary(path_array.as_any_dictionary())?; + inner::<_, R>( + json_array, + path_array.downcast_dict::().unwrap(), + jiter_find, + ) + } + DataType::Dictionary(_, value_type) if value_type.as_ref() == &DataType::LargeUtf8 => { + let path_array = cast_to_large_dictionary(path_array.as_any_dictionary())?; + inner::<_, R>( + json_array, + path_array.downcast_dict::().unwrap(), + jiter_find, + ) + } + DataType::Dictionary(_, value_type) if value_type.as_ref() == &DataType::Utf8View => { + let path_array = cast_to_large_dictionary(path_array.as_any_dictionary())?; + inner::<_, R>( + json_array, + path_array.downcast_dict::().unwrap(), + jiter_find, + ) + } + // for integer dictionaries, cast them directly to the inner type because it basically costs + // the same as building a new key array anyway + DataType::Dictionary(_, value_type) if value_type.as_ref() == &DataType::Int64 => inner::<_, R>( + json_array, + cast(path_array, &DataType::Int64)?.as_primitive::(), + jiter_find, + ), + DataType::Dictionary(_, value_type) if value_type.as_ref() == &DataType::UInt64 => inner::<_, R>( + json_array, + cast(path_array, &DataType::UInt64)?.as_primitive::(), + jiter_find, + ), + // for basic types, just consume directly + DataType::Utf8 => inner::<_, R>(json_array, path_array.as_string::(), jiter_find), + DataType::LargeUtf8 => inner::<_, R>(json_array, path_array.as_string::(), jiter_find), + DataType::Utf8View => inner::<_, R>(json_array, path_array.as_string_view(), jiter_find), + DataType::Int64 => inner::<_, R>(json_array, path_array.as_primitive::(), jiter_find), + DataType::UInt64 => inner::<_, R>(json_array, path_array.as_primitive::(), jiter_find), + other => { + exec_err!( + "unexpected second argument type, expected string or int array, got {:?}", + other + ) + } + } +} + +fn extract_json_scalar(scalar: &ScalarValue) -> DataFusionResult> { + match scalar { + ScalarValue::Dictionary(_, b) => extract_json_scalar(b.as_ref()), + ScalarValue::Utf8(s) | ScalarValue::Utf8View(s) | ScalarValue::LargeUtf8(s) => Ok(s.as_deref()), + ScalarValue::Union(type_id_value, union_fields, _) => { + Ok(json_from_union_scalar(type_id_value.as_ref(), union_fields)) + } + _ => { + exec_err!("unexpected first argument type, expected string or JSON union") + } + } +} + +fn is_object_lookup(path: &[JsonPath]) -> bool { + if let Some(first) = path.first() { + matches!(first, JsonPath::Key(_)) + } else { + false + } +} + +fn is_object_lookup_array(data_type: &DataType) -> bool { + match data_type { + DataType::Dictionary(_, value_type) => is_object_lookup_array(value_type), + DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View => true, + _ => false, + } +} + +/// Cast an array to a dictionary with i64 indices. +/// +/// According to the +/// recommendation is to avoid unsigned indices due to technologies like the JVM making it harder to +/// support unsigned integers. +/// +/// So we'll just use i64 as the largest signed integer type. +fn cast_to_large_dictionary(dict_array: &dyn AnyDictionaryArray) -> DataFusionResult> { + let keys = downcast_array(&cast(dict_array.keys(), &DataType::Int64)?); + Ok(DictionaryArray::::new(keys, dict_array.values().clone())) +} + +/// Wrap an array as a dictionary with i64 indices. +fn wrap_as_large_dictionary(original: &dyn AnyDictionaryArray, new_values: ArrayRef) -> DictionaryArray { + assert_eq!(original.keys().len(), new_values.len()); + let mut keys = + PrimitiveArray::from_iter_values(0i64..original.keys().len().try_into().expect("keys out of i64 range")); + if is_json_union(new_values.data_type()) { + // JSON union: post-process the array to set keys to null where the union member is null + let type_ids = new_values.as_union().type_ids(); + keys = mask_dictionary_keys(&keys, type_ids); + } + DictionaryArray::new(keys, new_values) +} + +pub fn jiter_json_find<'j>(opt_json: Option<&'j str>, path: &[JsonPath]) -> Option<(Jiter<'j>, Peek)> { + let json_str = opt_json?; + let mut jiter = Jiter::new(json_str.as_bytes()); + let mut peek = jiter.peek().ok()?; + for element in path { + match element { + JsonPath::Key(key) if peek == Peek::Object => { + let mut next_key = jiter.known_object().ok()??; + + while next_key != *key { + jiter.next_skip().ok()?; + next_key = jiter.next_key().ok()??; + } + + peek = jiter.peek().ok()?; + } + JsonPath::Index(index) if peek == Peek::Array => { + let mut array_item = jiter.known_array().ok()??; + + for _ in 0..*index { + jiter.known_skip(array_item).ok()?; + array_item = jiter.array_step().ok()??; + } + + peek = array_item; + } + _ => { + return None; + } + } + } + Some((jiter, peek)) +} + +macro_rules! get_err { + () => { + Err(GetError) + }; +} +pub(crate) use get_err; + +pub struct GetError; + +impl From for GetError { + fn from(_: JiterError) -> Self { + GetError + } +} + +impl From for GetError { + fn from(_: Utf8Error) -> Self { + GetError + } +} + +/// Set keys to null where the union member is null. +/// +/// This is a workaround to +/// - i.e. that dictionary null is most reliably done if the keys are null. +/// +/// That said, doing this might also be an optimization for cases like null-checking without needing +/// to check the value union array. +fn mask_dictionary_keys(keys: &PrimitiveArray, type_ids: &[i8]) -> PrimitiveArray { + let mut null_mask = vec![true; keys.len()]; + for (i, k) in keys.iter().enumerate() { + match k { + // if the key is non-null and value is non-null, don't mask it out + Some(k) if type_ids[k.as_usize()] != TYPE_ID_NULL => {} + // i.e. key is null or value is null here + _ => null_mask[i] = false, + } + } + PrimitiveArray::new(keys.values().clone(), Some(null_mask.into())) +} diff --git a/vendor/datafusion-functions-json/src/common_macros.rs b/vendor/datafusion-functions-json/src/common_macros.rs new file mode 100644 index 000000000..bc4c1b3da --- /dev/null +++ b/vendor/datafusion-functions-json/src/common_macros.rs @@ -0,0 +1,49 @@ +/// Creates external API `ScalarUDF` for an array UDF. Specifically, creates +/// +/// Creates a singleton `ScalarUDF` of the `$udf_impl` function named `$expr_fn_name _udf` and a +/// function named `$expr_fn_name _udf` which returns that function. +/// +/// This is used to ensure creating the list of `ScalarUDF` only happens once. +/// +/// # Arguments +/// * `udf_impl`: name of the [`ScalarUDFImpl`] +/// * `expr_fn_name`: name of the `expr_fn` function to be created +/// * `arg`: 0 or more named arguments for the function +/// * `doc`: documentation string for the function +/// +/// Copied mostly from, `/datafusion/functions-array/src/macros.rs`. +/// +/// [`ScalarUDFImpl`]: datafusion_expr::ScalarUDFImpl +macro_rules! make_udf_function { + ($udf_impl:ty, $expr_fn_name:ident, $($arg:ident)*, $doc:expr) => { + paste::paste! { + #[doc = $doc] + #[must_use] pub fn $expr_fn_name($($arg: datafusion::logical_expr::Expr),*) -> datafusion::logical_expr::Expr { + datafusion::logical_expr::Expr::ScalarFunction(datafusion::logical_expr::expr::ScalarFunction::new_udf( + [< $expr_fn_name _udf >](), + vec![$($arg),*], + )) + } + + /// Singleton instance of [`$udf_impl`], ensures the UDF is only created once + /// named for example `STATIC_JSON_OBJ_CONTAINS` + static [< STATIC_ $expr_fn_name:upper >]: std::sync::OnceLock> = + std::sync::OnceLock::new(); + + /// `ScalarFunction` that returns a [`ScalarUDF`] for [`$udf_impl`] + /// + /// [`ScalarUDF`]: datafusion::logical_expr::ScalarUDF + pub fn [< $expr_fn_name _udf >]() -> std::sync::Arc { + [< STATIC_ $expr_fn_name:upper >] + .get_or_init(|| { + std::sync::Arc::new(datafusion::logical_expr::ScalarUDF::new_from_impl( + <$udf_impl>::default(), + )) + }) + .clone() + } + } + }; +} + +pub(crate) use make_udf_function; diff --git a/vendor/datafusion-functions-json/src/common_union.rs b/vendor/datafusion-functions-json/src/common_union.rs new file mode 100644 index 000000000..3061b25e8 --- /dev/null +++ b/vendor/datafusion-functions-json/src/common_union.rs @@ -0,0 +1,345 @@ +use std::collections::HashMap; +use std::sync::{Arc, LazyLock, OnceLock}; + +use datafusion::arrow::array::{ + Array, ArrayRef, AsArray, BooleanArray, Float64Array, Int64Array, NullArray, StringArray, UnionArray, +}; +use datafusion::arrow::buffer::{Buffer, ScalarBuffer}; +use datafusion::arrow::datatypes::{DataType, Field, UnionFields, UnionMode}; +use datafusion::arrow::error::ArrowError; +use datafusion::common::ScalarValue; + +/// Field metadata used to mark a `Utf8` field as containing raw JSON. +/// +/// Attach this to any Arrow `Field` whose values are JSON-encoded strings so +/// downstream consumers can recognize them as JSON rather than opaque text. +/// +/// Emits Arrow's canonical JSON extension type keys +/// (`ARROW:extension:name` = `arrow.json`, `ARROW:extension:metadata` = `{}`), +/// see . +/// +/// Also emits a legacy `is_json` = `true` key. This key predates this crate's +/// adoption of the canonical extension and is non-standard — no other Arrow +/// tool recognizes it. It is kept only for back-compat with existing +/// downstream consumers of this crate and will be removed in a future +/// release; new consumers should key off `ARROW:extension:name` instead. +#[must_use] +pub fn json_field_metadata() -> HashMap { + HashMap::from([ + ("ARROW:extension:name".to_string(), "arrow.json".to_string()), + ("ARROW:extension:metadata".to_string(), "{}".to_string()), + // Legacy, non-standard. Remove in a future release — see doc comment above. + ("is_json".to_string(), "true".to_string()), + ]) +} + +pub fn is_json_union(data_type: &DataType) -> bool { + match data_type { + DataType::Union(fields, UnionMode::Sparse) => fields == &union_fields(), + _ => false, + } +} + +/// Extract nested JSON from a `JsonUnion` `UnionArray` +/// +/// # Arguments +/// * `array` - The `UnionArray` to extract the nested JSON from +/// * `object_lookup` - If `true`, extract from the "object" member of the union, +/// otherwise extract from the "array" member +pub(crate) fn nested_json_array(array: &ArrayRef, object_lookup: bool) -> Option<&StringArray> { + nested_json_array_ref(array, object_lookup).map(AsArray::as_string) +} + +pub(crate) fn nested_json_array_ref(array: &ArrayRef, object_lookup: bool) -> Option<&ArrayRef> { + let union_array: &UnionArray = array.as_any().downcast_ref::()?; + let type_id = if object_lookup { TYPE_ID_OBJECT } else { TYPE_ID_ARRAY }; + Some(union_array.child(type_id)) +} + +/// Extract a JSON string from a `JsonUnion` scalar +pub(crate) fn json_from_union_scalar<'a>( + type_id_value: Option<&'a (i8, Box)>, + fields: &UnionFields, +) -> Option<&'a str> { + if let Some((type_id, value)) = type_id_value { + // we only want to take the ScalarValue string if the type_id indicates the value represents nested JSON + if fields == &union_fields() && (*type_id == TYPE_ID_ARRAY || *type_id == TYPE_ID_OBJECT) { + if let ScalarValue::Utf8(s) | ScalarValue::Utf8View(s) | ScalarValue::LargeUtf8(s) = value.as_ref() { + return s.as_deref(); + } + } + } + None +} + +pub static JSON_UNION_DATA_TYPE: LazyLock = LazyLock::new(JsonUnion::data_type); + +#[derive(Debug)] +pub(crate) struct JsonUnion { + bools: Vec>, + ints: Vec>, + floats: Vec>, + strings: Vec>, + arrays: Vec>, + objects: Vec>, + type_ids: Vec, + index: usize, + length: usize, +} + +impl JsonUnion { + pub fn new(length: usize) -> Self { + Self { + bools: vec![None; length], + ints: vec![None; length], + floats: vec![None; length], + strings: vec![None; length], + arrays: vec![None; length], + objects: vec![None; length], + type_ids: vec![TYPE_ID_NULL; length], + index: 0, + length, + } + } + + pub fn data_type() -> DataType { + DataType::Union(union_fields(), UnionMode::Sparse) + } + + pub fn push(&mut self, field: JsonUnionField) { + self.type_ids[self.index] = field.type_id(); + match field { + JsonUnionField::JsonNull => (), + JsonUnionField::Bool(value) => self.bools[self.index] = Some(value), + JsonUnionField::Int(value) => self.ints[self.index] = Some(value), + JsonUnionField::Float(value) => self.floats[self.index] = Some(value), + JsonUnionField::Str(value) => self.strings[self.index] = Some(value), + JsonUnionField::Array(value) => self.arrays[self.index] = Some(value), + JsonUnionField::Object(value) => self.objects[self.index] = Some(value), + } + self.index += 1; + debug_assert!(self.index <= self.length); + } + + pub fn push_none(&mut self) { + self.index += 1; + debug_assert!(self.index <= self.length); + } +} + +/// So we can do `collect::()` +impl FromIterator> for JsonUnion { + fn from_iter>>(iter: I) -> Self { + let inner = iter.into_iter(); + let (lower, upper) = inner.size_hint(); + let mut union = Self::new(upper.unwrap_or(lower)); + + for opt_field in inner { + if let Some(union_field) = opt_field { + union.push(union_field); + } else { + union.push_none(); + } + } + union + } +} + +impl TryFrom for UnionArray { + type Error = ArrowError; + + fn try_from(value: JsonUnion) -> Result { + let children: Vec> = vec![ + Arc::new(NullArray::new(value.length)), + Arc::new(BooleanArray::from(value.bools)), + Arc::new(Int64Array::from(value.ints)), + Arc::new(Float64Array::from(value.floats)), + Arc::new(StringArray::from(value.strings)), + Arc::new(StringArray::from(value.arrays)), + Arc::new(StringArray::from(value.objects)), + ]; + UnionArray::try_new(union_fields(), Buffer::from_vec(value.type_ids).into(), None, children) + } +} + +#[derive(Debug)] +pub(crate) enum JsonUnionField { + JsonNull, + Bool(bool), + Int(i64), + Float(f64), + Str(String), + Array(String), + Object(String), +} + +pub(crate) const TYPE_ID_NULL: i8 = 0; +const TYPE_ID_BOOL: i8 = 1; +const TYPE_ID_INT: i8 = 2; +const TYPE_ID_FLOAT: i8 = 3; +const TYPE_ID_STR: i8 = 4; +const TYPE_ID_ARRAY: i8 = 5; +const TYPE_ID_OBJECT: i8 = 6; + +fn union_fields() -> UnionFields { + static FIELDS: OnceLock = OnceLock::new(); + FIELDS + .get_or_init(|| { + UnionFields::from_iter([ + (TYPE_ID_NULL, Arc::new(Field::new("null", DataType::Null, true))), + (TYPE_ID_BOOL, Arc::new(Field::new("bool", DataType::Boolean, false))), + (TYPE_ID_INT, Arc::new(Field::new("int", DataType::Int64, false))), + (TYPE_ID_FLOAT, Arc::new(Field::new("float", DataType::Float64, false))), + (TYPE_ID_STR, Arc::new(Field::new("str", DataType::Utf8, false))), + ( + TYPE_ID_ARRAY, + Arc::new(Field::new("array", DataType::Utf8, false).with_metadata(json_field_metadata())), + ), + ( + TYPE_ID_OBJECT, + Arc::new(Field::new("object", DataType::Utf8, false).with_metadata(json_field_metadata())), + ), + ]) + }) + .clone() +} + +impl JsonUnionField { + fn type_id(&self) -> i8 { + match self { + Self::JsonNull => TYPE_ID_NULL, + Self::Bool(_) => TYPE_ID_BOOL, + Self::Int(_) => TYPE_ID_INT, + Self::Float(_) => TYPE_ID_FLOAT, + Self::Str(_) => TYPE_ID_STR, + Self::Array(_) => TYPE_ID_ARRAY, + Self::Object(_) => TYPE_ID_OBJECT, + } + } + + pub fn scalar_value(f: Option) -> ScalarValue { + ScalarValue::Union( + f.map(|f| (f.type_id(), Box::new(f.into()))), + union_fields(), + UnionMode::Sparse, + ) + } +} + +impl From for ScalarValue { + fn from(value: JsonUnionField) -> Self { + match value { + JsonUnionField::JsonNull => Self::Null, + JsonUnionField::Bool(b) => Self::Boolean(Some(b)), + JsonUnionField::Int(i) => Self::Int64(Some(i)), + JsonUnionField::Float(f) => Self::Float64(Some(f)), + JsonUnionField::Str(s) | JsonUnionField::Array(s) | JsonUnionField::Object(s) => Self::Utf8(Some(s)), + } + } +} + +pub struct JsonUnionEncoder { + boolean: BooleanArray, + int: Int64Array, + float: Float64Array, + string: StringArray, + array: StringArray, + object: StringArray, + type_ids: ScalarBuffer, +} + +impl JsonUnionEncoder { + #[must_use] + pub fn from_union(union: UnionArray) -> Option { + if is_json_union(union.data_type()) { + let (_, type_ids, _, c) = union.into_parts(); + Some(Self { + boolean: c[1].as_boolean().clone(), + int: c[2].as_primitive().clone(), + float: c[3].as_primitive().clone(), + string: c[4].as_string().clone(), + array: c[5].as_string().clone(), + object: c[6].as_string().clone(), + type_ids, + }) + } else { + None + } + } + + #[must_use] + #[allow(clippy::len_without_is_empty)] + pub fn len(&self) -> usize { + self.type_ids.len() + } + + /// Get the encodable value for a given index + /// + /// # Panics + /// + /// Panics if the idx is outside the union values or an invalid type id exists in the union. + #[must_use] + pub fn get_value(&self, idx: usize) -> JsonUnionValue<'_> { + let type_id = self.type_ids[idx]; + match type_id { + TYPE_ID_NULL => JsonUnionValue::JsonNull, + TYPE_ID_BOOL => JsonUnionValue::Bool(self.boolean.value(idx)), + TYPE_ID_INT => JsonUnionValue::Int(self.int.value(idx)), + TYPE_ID_FLOAT => JsonUnionValue::Float(self.float.value(idx)), + TYPE_ID_STR => JsonUnionValue::Str(self.string.value(idx)), + TYPE_ID_ARRAY => JsonUnionValue::Array(self.array.value(idx)), + TYPE_ID_OBJECT => JsonUnionValue::Object(self.object.value(idx)), + _ => panic!("Invalid type_id: {type_id}, not a valid JSON type"), + } + } +} + +#[derive(Debug, PartialEq)] +pub enum JsonUnionValue<'a> { + JsonNull, + Bool(bool), + Int(i64), + Float(f64), + Str(&'a str), + Array(&'a str), + Object(&'a str), +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn test_json_union() { + let json_union = JsonUnion::from_iter(vec![ + Some(JsonUnionField::JsonNull), + Some(JsonUnionField::Bool(true)), + Some(JsonUnionField::Bool(false)), + Some(JsonUnionField::Int(42)), + Some(JsonUnionField::Float(42.0)), + Some(JsonUnionField::Str("foo".to_string())), + Some(JsonUnionField::Array("[42]".to_string())), + Some(JsonUnionField::Object(r#"{"foo": 42}"#.to_string())), + None, + ]); + + let union_array = UnionArray::try_from(json_union).unwrap(); + let encoder = JsonUnionEncoder::from_union(union_array).unwrap(); + + let values_after: Vec<_> = (0..encoder.len()).map(|idx| encoder.get_value(idx)).collect(); + assert_eq!( + values_after, + vec![ + JsonUnionValue::JsonNull, + JsonUnionValue::Bool(true), + JsonUnionValue::Bool(false), + JsonUnionValue::Int(42), + JsonUnionValue::Float(42.0), + JsonUnionValue::Str("foo"), + JsonUnionValue::Array("[42]"), + JsonUnionValue::Object(r#"{"foo": 42}"#), + JsonUnionValue::JsonNull, + ] + ); + } +} diff --git a/vendor/datafusion-functions-json/src/json_as_text.rs b/vendor/datafusion-functions-json/src/json_as_text.rs new file mode 100644 index 000000000..5ce050040 --- /dev/null +++ b/vendor/datafusion-functions-json/src/json_as_text.rs @@ -0,0 +1,117 @@ +use std::sync::Arc; + +use datafusion::arrow::array::{ArrayRef, StringArray, StringBuilder}; +use datafusion::arrow::datatypes::DataType; +use datafusion::common::{Result as DataFusionResult, ScalarValue}; +use datafusion::logical_expr::{ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility}; +use jiter::Peek; + +use crate::common::{get_err, invoke, jiter_json_find, return_type_check, GetError, InvokeResult, JsonPath}; +use crate::common_macros::make_udf_function; + +make_udf_function!( + JsonAsText, + json_as_text, + json_data path, + r#"Get any value from a JSON string by its "path", represented as a string"# +); + +#[derive(Debug, PartialEq, Eq, Hash)] +pub(super) struct JsonAsText { + signature: Signature, + aliases: [String; 1], +} + +impl Default for JsonAsText { + fn default() -> Self { + Self { + signature: Signature::variadic_any(Volatility::Immutable), + aliases: ["json_as_text".to_string()], + } + } +} + +impl ScalarUDFImpl for JsonAsText { + fn name(&self) -> &str { + self.aliases[0].as_str() + } + + fn signature(&self) -> &Signature { + &self.signature + } + + fn return_type(&self, arg_types: &[DataType]) -> DataFusionResult { + return_type_check(arg_types, self.name(), DataType::Utf8) + } + + fn invoke_with_args(&self, args: ScalarFunctionArgs) -> DataFusionResult { + invoke::(&args.args, jiter_json_as_text) + } + + fn aliases(&self) -> &[String] { + &self.aliases + } + + fn placement( + &self, + args: &[datafusion::logical_expr::ExpressionPlacement], + ) -> datafusion::logical_expr::ExpressionPlacement { + // If the first argument is a column and the remaining arguments are literals (a path) + // then we can push this UDF down to the leaf nodes. + if args.len() >= 2 + && matches!(args[0], datafusion::logical_expr::ExpressionPlacement::Column) + && args[1..] + .iter() + .all(|arg| matches!(arg, datafusion::logical_expr::ExpressionPlacement::Literal)) + { + datafusion::logical_expr::ExpressionPlacement::MoveTowardsLeafNodes + } else { + datafusion::logical_expr::ExpressionPlacement::KeepInPlace + } + } +} + +impl InvokeResult for StringArray { + type Item = String; + + type Builder = StringBuilder; + + const ACCEPT_DICT_RETURN: bool = true; + + fn builder(capacity: usize) -> Self::Builder { + StringBuilder::with_capacity(capacity, 0) + } + + fn append_value(builder: &mut Self::Builder, value: Option) { + builder.append_option(value); + } + + fn finish(mut builder: Self::Builder) -> DataFusionResult { + Ok(Arc::new(builder.finish())) + } + + fn scalar(value: Option) -> ScalarValue { + ScalarValue::Utf8(value) + } +} + +fn jiter_json_as_text(opt_json: Option<&str>, path: &[JsonPath]) -> Result { + if let Some((mut jiter, peek)) = jiter_json_find(opt_json, path) { + match peek { + Peek::Null => { + jiter.known_null()?; + get_err!() + } + Peek::String => Ok(jiter.known_str()?.to_owned()), + _ => { + let start = jiter.current_index(); + jiter.known_skip(peek)?; + let object_slice = jiter.slice_to_current(start); + let object_string = std::str::from_utf8(object_slice)?; + Ok(object_string.to_owned()) + } + } + } else { + get_err!() + } +} diff --git a/vendor/datafusion-functions-json/src/json_contains.rs b/vendor/datafusion-functions-json/src/json_contains.rs new file mode 100644 index 000000000..eca36d5c7 --- /dev/null +++ b/vendor/datafusion-functions-json/src/json_contains.rs @@ -0,0 +1,106 @@ +use std::sync::Arc; + +use datafusion::arrow::array::BooleanBuilder; +use datafusion::arrow::datatypes::DataType; +use datafusion::common::arrow::array::{ArrayRef, BooleanArray}; +use datafusion::common::{plan_err, Result, ScalarValue}; +use datafusion::logical_expr::{ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility}; + +use crate::common::{invoke, jiter_json_find, return_type_check, GetError, InvokeResult, JsonPath}; +use crate::common_macros::make_udf_function; + +make_udf_function!( + JsonContains, + json_contains, + json_data path, + r#"Does the key/index exist within the JSON value as the specified "path"?"# +); + +#[derive(Debug, PartialEq, Eq, Hash)] +pub(super) struct JsonContains { + signature: Signature, + aliases: [String; 1], +} + +impl Default for JsonContains { + fn default() -> Self { + Self { + signature: Signature::variadic_any(Volatility::Immutable), + aliases: ["json_contains".to_string()], + } + } +} + +impl ScalarUDFImpl for JsonContains { + fn name(&self) -> &str { + self.aliases[0].as_str() + } + + fn signature(&self) -> &Signature { + &self.signature + } + + fn return_type(&self, arg_types: &[DataType]) -> Result { + if arg_types.len() < 2 { + plan_err!("The 'json_contains' function requires two or more arguments.") + } else { + return_type_check(arg_types, self.name(), DataType::Boolean).map(|_| DataType::Boolean) + } + } + + fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { + invoke::(&args.args, jiter_json_contains) + } + + fn aliases(&self) -> &[String] { + &self.aliases + } + + fn placement( + &self, + args: &[datafusion::logical_expr::ExpressionPlacement], + ) -> datafusion::logical_expr::ExpressionPlacement { + // If the first argument is a column and the remaining arguments are literals (a path) + // then we can push this UDF down to the leaf nodes. + if args.len() >= 2 + && matches!(args[0], datafusion::logical_expr::ExpressionPlacement::Column) + && args[1..] + .iter() + .all(|arg| matches!(arg, datafusion::logical_expr::ExpressionPlacement::Literal)) + { + datafusion::logical_expr::ExpressionPlacement::MoveTowardsLeafNodes + } else { + datafusion::logical_expr::ExpressionPlacement::KeepInPlace + } + } +} + +impl InvokeResult for BooleanArray { + type Item = bool; + + type Builder = BooleanBuilder; + + // Using boolean inside a dictionary is not an optimization! + const ACCEPT_DICT_RETURN: bool = false; + + fn builder(capacity: usize) -> Self::Builder { + BooleanBuilder::with_capacity(capacity) + } + + fn append_value(builder: &mut Self::Builder, value: Option) { + builder.append_option(value); + } + + fn finish(mut builder: Self::Builder) -> Result { + Ok(Arc::new(builder.finish())) + } + + fn scalar(value: Option) -> ScalarValue { + ScalarValue::Boolean(value) + } +} + +#[allow(clippy::unnecessary_wraps)] +fn jiter_json_contains(json_data: Option<&str>, path: &[JsonPath]) -> Result { + Ok(jiter_json_find(json_data, path).is_some()) +} diff --git a/vendor/datafusion-functions-json/src/json_from_scalar.rs b/vendor/datafusion-functions-json/src/json_from_scalar.rs new file mode 100644 index 000000000..221bf55ef --- /dev/null +++ b/vendor/datafusion-functions-json/src/json_from_scalar.rs @@ -0,0 +1,221 @@ +use std::sync::Arc; + +use datafusion::arrow::array::{Array, ArrayRef, AsArray, UnionArray}; +use datafusion::arrow::datatypes::{ + DataType, Float32Type, Float64Type, Int16Type, Int32Type, Int64Type, Int8Type, UInt16Type, UInt32Type, UInt64Type, + UInt8Type, +}; +use datafusion::common::{exec_datafusion_err, exec_err, plan_err, Result as DataFusionResult, ScalarValue}; +use datafusion::logical_expr::{ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility}; + +use crate::common_macros::make_udf_function; +use crate::common_union::{JsonUnion, JsonUnionField}; + +make_udf_function!( + JsonFromScalar, + json_from_scalar, + value, + r"Convert a scalar value (null, bool, integer, float, or string) to a JSON union type" +); + +#[derive(Debug, PartialEq, Eq, Hash)] +pub(super) struct JsonFromScalar { + signature: Signature, + aliases: [String; 2], +} + +impl Default for JsonFromScalar { + fn default() -> Self { + Self { + signature: Signature::any(1, Volatility::Immutable), + aliases: ["json_from_scalar".to_string(), "scalar_to_json".to_string()], + } + } +} + +impl ScalarUDFImpl for JsonFromScalar { + fn name(&self) -> &str { + self.aliases[0].as_str() + } + + fn signature(&self) -> &Signature { + &self.signature + } + + fn return_type(&self, arg_types: &[DataType]) -> DataFusionResult { + // Check that the input type is a scalar type that we can convert to JSON + // The signature check ensures we only get one argument, index access is safe + match arg_types[0] { + DataType::Null + | DataType::Boolean + | DataType::Int8 + | DataType::Int16 + | DataType::Int32 + | DataType::Int64 + | DataType::UInt8 + | DataType::UInt16 + | DataType::UInt32 + | DataType::UInt64 + | DataType::Float32 + | DataType::Float64 + | DataType::Utf8 + | DataType::LargeUtf8 + | DataType::Utf8View => {} + _ => { + return plan_err!("Unsupported type for json_from_scalar: {:?}", arg_types[0]); + } + } + Ok(JsonUnion::data_type()) + } + + fn invoke_with_args(&self, mut args: ScalarFunctionArgs) -> DataFusionResult { + // The signature check ensures we only get one argument + match args.args.pop().expect("Expected exactly one argument") { + ColumnarValue::Scalar(scalar) => { + let field = scalar_to_json_union_field(scalar)?; + Ok(ColumnarValue::Scalar(JsonUnionField::scalar_value(Some(field)))) + } + ColumnarValue::Array(array) => { + let union = array_to_json_union(&array)?; + let union_array: UnionArray = union.try_into()?; + Ok(ColumnarValue::Array(Arc::new(union_array) as ArrayRef)) + } + } + } + + fn aliases(&self) -> &[String] { + &self.aliases + } +} + +fn scalar_to_json_union_field(scalar: ScalarValue) -> DataFusionResult { + match scalar { + // Null type / values + ScalarValue::Null + | ScalarValue::Boolean(None) + | ScalarValue::Int8(None) + | ScalarValue::Int16(None) + | ScalarValue::Int32(None) + | ScalarValue::Int64(None) + | ScalarValue::UInt8(None) + | ScalarValue::UInt16(None) + | ScalarValue::UInt32(None) + | ScalarValue::UInt64(None) + | ScalarValue::Float32(None) + | ScalarValue::Float64(None) + | ScalarValue::Utf8(None) + | ScalarValue::LargeUtf8(None) + | ScalarValue::Utf8View(None) => Ok(JsonUnionField::JsonNull), + // Boolean type + ScalarValue::Boolean(Some(b)) => Ok(JsonUnionField::Bool(b)), + // Integer types - coerce to i64 + ScalarValue::Int8(Some(v)) => Ok(JsonUnionField::Int(i64::from(v))), + ScalarValue::Int16(Some(v)) => Ok(JsonUnionField::Int(i64::from(v))), + ScalarValue::Int32(Some(v)) => Ok(JsonUnionField::Int(i64::from(v))), + ScalarValue::Int64(Some(v)) => Ok(JsonUnionField::Int(v)), + ScalarValue::UInt8(Some(v)) => Ok(JsonUnionField::Int(i64::from(v))), + ScalarValue::UInt16(Some(v)) => Ok(JsonUnionField::Int(i64::from(v))), + ScalarValue::UInt32(Some(v)) => Ok(JsonUnionField::Int(i64::from(v))), + ScalarValue::UInt64(Some(v)) => { + Ok(JsonUnionField::Int(i64::try_from(v).map_err(|_| { + exec_datafusion_err!("UInt64 value {} is out of range for i64", v) + })?)) + } + // Float types - coerce to f64 + ScalarValue::Float32(Some(v)) => Ok(JsonUnionField::Float(f64::from(v))), + ScalarValue::Float64(Some(v)) => Ok(JsonUnionField::Float(v)), + // String types + ScalarValue::Utf8(Some(s)) | ScalarValue::LargeUtf8(Some(s)) | ScalarValue::Utf8View(Some(s)) => { + Ok(JsonUnionField::Str(s)) + } + _ => exec_err!("Unsupported type for json_from_scalar: {:?}", scalar.data_type()), + } +} + +fn array_to_json_union(array: &ArrayRef) -> DataFusionResult { + Ok(match array.data_type() { + DataType::Null => (0..array.len()).map(|_| Some(JsonUnionField::JsonNull)).collect(), + DataType::Boolean => array.as_boolean().iter().map(|v| v.map(JsonUnionField::Bool)).collect(), + // Integer types - coerce to i64 + DataType::Int8 => array + .as_primitive::() + .iter() + .map(|v| v.map(|x| JsonUnionField::Int(i64::from(x)))) + .collect(), + DataType::Int16 => array + .as_primitive::() + .iter() + .map(|v| v.map(|x| JsonUnionField::Int(i64::from(x)))) + .collect(), + DataType::Int32 => array + .as_primitive::() + .iter() + .map(|v| v.map(|x| JsonUnionField::Int(i64::from(x)))) + .collect(), + DataType::Int64 => array + .as_primitive::() + .iter() + .map(|v| v.map(JsonUnionField::Int)) + .collect(), + DataType::UInt8 => array + .as_primitive::() + .iter() + .map(|v| v.map(|x| JsonUnionField::Int(i64::from(x)))) + .collect(), + DataType::UInt16 => array + .as_primitive::() + .iter() + .map(|v| v.map(|x| JsonUnionField::Int(i64::from(x)))) + .collect(), + DataType::UInt32 => array + .as_primitive::() + .iter() + .map(|v| v.map(|x| JsonUnionField::Int(i64::from(x)))) + .collect(), + DataType::UInt64 => { + // UInt64 requires explicit loop for fallible conversion + let arr = array.as_primitive::(); + let mut union = JsonUnion::new(arr.len()); + for i in 0..arr.len() { + if arr.is_null(i) { + union.push_none(); + } else { + union.push(JsonUnionField::Int(i64::try_from(arr.value(i)).map_err(|_| { + exec_datafusion_err!("UInt64 value {} is out of range for i64", arr.value(i)) + })?)); + } + } + return Ok(union); + } + // Float types - coerce to f64 + DataType::Float32 => array + .as_primitive::() + .iter() + .map(|v| v.map(|x| JsonUnionField::Float(f64::from(x)))) + .collect(), + DataType::Float64 => array + .as_primitive::() + .iter() + .map(|v| v.map(JsonUnionField::Float)) + .collect(), + // String types + DataType::Utf8 => array + .as_string::() + .iter() + .map(|v| v.map(|s| JsonUnionField::Str(s.to_string()))) + .collect(), + DataType::LargeUtf8 => array + .as_string::() + .iter() + .map(|v| v.map(|s| JsonUnionField::Str(s.to_string()))) + .collect(), + DataType::Utf8View => array + .as_string_view() + .iter() + .map(|v| v.map(|s| JsonUnionField::Str(s.to_string()))) + .collect(), + dt => { + return exec_err!("Unsupported array type for json_from_scalar: {:?}", dt); + } + }) +} diff --git a/vendor/datafusion-functions-json/src/json_get.rs b/vendor/datafusion-functions-json/src/json_get.rs new file mode 100644 index 000000000..ac79a216e --- /dev/null +++ b/vendor/datafusion-functions-json/src/json_get.rs @@ -0,0 +1,151 @@ +use std::sync::Arc; + +use datafusion::arrow::array::ArrayRef; +use datafusion::arrow::array::UnionArray; +use datafusion::arrow::datatypes::DataType; +use datafusion::common::Result as DataFusionResult; +use datafusion::logical_expr::{ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility}; +use datafusion::scalar::ScalarValue; +use jiter::{Jiter, NumberAny, NumberInt, Peek}; + +use crate::common::InvokeResult; +use crate::common::{get_err, invoke, jiter_json_find, return_type_check, GetError, JsonPath}; +use crate::common_macros::make_udf_function; +use crate::common_union::{JsonUnion, JsonUnionField}; + +make_udf_function!( + JsonGet, + json_get, + json_data path, + r#"Get a value from a JSON string by its "path""# +); + +// build_typed_get!(JsonGet, "json_get", Union, Float64Array, jiter_json_get_float); + +#[derive(Debug, PartialEq, Eq, Hash)] +pub(super) struct JsonGet { + signature: Signature, + aliases: [String; 1], +} + +impl Default for JsonGet { + fn default() -> Self { + Self { + signature: Signature::variadic_any(Volatility::Immutable), + aliases: ["json_get".to_string()], + } + } +} + +impl ScalarUDFImpl for JsonGet { + fn name(&self) -> &str { + self.aliases[0].as_str() + } + + fn signature(&self) -> &Signature { + &self.signature + } + + fn return_type(&self, arg_types: &[DataType]) -> DataFusionResult { + return_type_check(arg_types, self.name(), JsonUnion::data_type()) + } + + fn invoke_with_args(&self, args: ScalarFunctionArgs) -> DataFusionResult { + invoke::(&args.args, jiter_json_get_union) + } + + fn aliases(&self) -> &[String] { + &self.aliases + } + + fn placement( + &self, + args: &[datafusion::logical_expr::ExpressionPlacement], + ) -> datafusion::logical_expr::ExpressionPlacement { + // If the first argument is a column and the remaining arguments are literals (a path) + // then we can push this UDF down to the leaf nodes. + if args.len() >= 2 + && matches!(args[0], datafusion::logical_expr::ExpressionPlacement::Column) + && args[1..] + .iter() + .all(|arg| matches!(arg, datafusion::logical_expr::ExpressionPlacement::Literal)) + { + datafusion::logical_expr::ExpressionPlacement::MoveTowardsLeafNodes + } else { + datafusion::logical_expr::ExpressionPlacement::KeepInPlace + } + } +} + +impl InvokeResult for JsonUnion { + type Item = JsonUnionField; + + type Builder = JsonUnion; + + const ACCEPT_DICT_RETURN: bool = true; + + fn builder(capacity: usize) -> Self::Builder { + JsonUnion::new(capacity) + } + + fn append_value(builder: &mut Self::Builder, value: Option) { + if let Some(value) = value { + builder.push(value); + } else { + builder.push_none(); + } + } + + fn finish(builder: Self::Builder) -> DataFusionResult { + let array: UnionArray = builder.try_into()?; + Ok(Arc::new(array) as ArrayRef) + } + + fn scalar(value: Option) -> ScalarValue { + JsonUnionField::scalar_value(value) + } +} + +fn jiter_json_get_union(opt_json: Option<&str>, path: &[JsonPath]) -> Result { + if let Some((mut jiter, peek)) = jiter_json_find(opt_json, path) { + build_union(&mut jiter, peek) + } else { + get_err!() + } +} + +fn build_union(jiter: &mut Jiter, peek: Peek) -> Result { + match peek { + Peek::Null => { + jiter.known_null()?; + Ok(JsonUnionField::JsonNull) + } + Peek::True | Peek::False => { + let value = jiter.known_bool(peek)?; + Ok(JsonUnionField::Bool(value)) + } + Peek::String => { + let value = jiter.known_str()?; + Ok(JsonUnionField::Str(value.to_owned())) + } + Peek::Array => { + let start = jiter.current_index(); + jiter.known_skip(peek)?; + let array_slice = jiter.slice_to_current(start); + let array_string = std::str::from_utf8(array_slice)?; + Ok(JsonUnionField::Array(array_string.to_owned())) + } + Peek::Object => { + let start = jiter.current_index(); + jiter.known_skip(peek)?; + let object_slice = jiter.slice_to_current(start); + let object_string = std::str::from_utf8(object_slice)?; + Ok(JsonUnionField::Object(object_string.to_owned())) + } + _ => match jiter.known_number(peek)? { + NumberAny::Int(NumberInt::Int(value)) => Ok(JsonUnionField::Int(value)), + NumberAny::Int(NumberInt::BigInt(_)) => todo!("BigInt not supported yet"), + NumberAny::Float(value) => Ok(JsonUnionField::Float(value)), + }, + } +} diff --git a/vendor/datafusion-functions-json/src/json_get_array.rs b/vendor/datafusion-functions-json/src/json_get_array.rs new file mode 100644 index 000000000..860fb3bc7 --- /dev/null +++ b/vendor/datafusion-functions-json/src/json_get_array.rs @@ -0,0 +1,144 @@ +use std::sync::Arc; + +use datafusion::arrow::array::{ArrayRef, ListBuilder, StringBuilder}; +use datafusion::arrow::datatypes::{DataType, Field}; +use datafusion::common::{Result as DataFusionResult, ScalarValue}; +use datafusion::logical_expr::{ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility}; +use jiter::Peek; + +use crate::common::{get_err, invoke, jiter_json_find, return_type_check, GetError, InvokeResult, JsonPath}; +use crate::common_macros::make_udf_function; +use crate::common_union::json_field_metadata; + +fn list_item_field() -> Field { + Field::new("item", DataType::Utf8, true).with_metadata(json_field_metadata()) +} + +make_udf_function!( + JsonGetArray, + json_get_array, + json_data path, + r#"Get an arrow array from a JSON string by its "path""# +); + +#[derive(Debug, PartialEq, Eq, Hash)] +pub(super) struct JsonGetArray { + signature: Signature, + aliases: [String; 1], +} + +impl Default for JsonGetArray { + fn default() -> Self { + Self { + signature: Signature::variadic_any(Volatility::Immutable), + aliases: ["json_get_array".to_string()], + } + } +} + +impl ScalarUDFImpl for JsonGetArray { + fn name(&self) -> &str { + self.aliases[0].as_str() + } + + fn signature(&self) -> &Signature { + &self.signature + } + + fn return_type(&self, arg_types: &[DataType]) -> DataFusionResult { + return_type_check(arg_types, self.name(), DataType::List(Arc::new(list_item_field()))) + } + + fn invoke_with_args(&self, args: ScalarFunctionArgs) -> DataFusionResult { + invoke::(&args.args, jiter_json_get_array) + } + + fn aliases(&self) -> &[String] { + &self.aliases + } + + fn placement( + &self, + args: &[datafusion::logical_expr::ExpressionPlacement], + ) -> datafusion::logical_expr::ExpressionPlacement { + // If the first argument is a column and the remaining arguments are literals (a path) + // then we can push this UDF down to the leaf nodes. + if args.len() >= 2 + && matches!(args[0], datafusion::logical_expr::ExpressionPlacement::Column) + && args[1..] + .iter() + .all(|arg| matches!(arg, datafusion::logical_expr::ExpressionPlacement::Literal)) + { + datafusion::logical_expr::ExpressionPlacement::MoveTowardsLeafNodes + } else { + datafusion::logical_expr::ExpressionPlacement::KeepInPlace + } + } +} + +#[derive(Debug)] +struct BuildArrayList; + +impl InvokeResult for BuildArrayList { + type Item = Vec; + + type Builder = ListBuilder; + + const ACCEPT_DICT_RETURN: bool = false; + + fn builder(capacity: usize) -> Self::Builder { + let values_builder = StringBuilder::new(); + ListBuilder::with_capacity(values_builder, capacity).with_field(list_item_field()) + } + + fn append_value(builder: &mut Self::Builder, value: Option) { + builder.append_option(value.map(|v| v.into_iter().map(Some))); + } + + fn finish(mut builder: Self::Builder) -> DataFusionResult { + Ok(Arc::new(builder.finish())) + } + + fn scalar(value: Option) -> ScalarValue { + let mut builder = ListBuilder::new(StringBuilder::new()).with_field(list_item_field()); + + if let Some(array_items) = value { + for item in array_items { + builder.values().append_value(item); + } + + builder.append(true); + } else { + builder.append(false); + } + let array = builder.finish(); + ScalarValue::List(Arc::new(array)) + } +} + +fn jiter_json_get_array(opt_json: Option<&str>, path: &[JsonPath]) -> Result, GetError> { + if let Some((mut jiter, peek)) = jiter_json_find(opt_json, path) { + match peek { + Peek::Array => { + let mut peek_opt = jiter.known_array()?; + let mut array_items: Vec = Vec::new(); + + while let Some(element_peek) = peek_opt { + // Get the raw JSON slice for each array element + let start = jiter.current_index(); + jiter.known_skip(element_peek)?; + let slice = jiter.slice_to_current(start); + let element_str = std::str::from_utf8(slice)?.to_string(); + + array_items.push(element_str); + peek_opt = jiter.array_step()?; + } + + Ok(array_items) + } + _ => get_err!(), + } + } else { + get_err!() + } +} diff --git a/vendor/datafusion-functions-json/src/json_get_bool.rs b/vendor/datafusion-functions-json/src/json_get_bool.rs new file mode 100644 index 000000000..b7e7901f5 --- /dev/null +++ b/vendor/datafusion-functions-json/src/json_get_bool.rs @@ -0,0 +1,85 @@ +use datafusion::arrow::array::BooleanArray; +use datafusion::arrow::datatypes::DataType; +use datafusion::common::Result as DataFusionResult; +use datafusion::logical_expr::{ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility}; +use jiter::Peek; + +use crate::common::{get_err, invoke, jiter_json_find, return_type_check, GetError, JsonPath}; +use crate::common_macros::make_udf_function; + +make_udf_function!( + JsonGetBool, + json_get_bool, + json_data path, + r#"Get an boolean value from a JSON string by its "path""# +); + +#[derive(Debug, PartialEq, Eq, Hash)] +pub(super) struct JsonGetBool { + signature: Signature, + aliases: [String; 1], +} + +impl Default for JsonGetBool { + fn default() -> Self { + Self { + signature: Signature::variadic_any(Volatility::Immutable), + aliases: ["json_get_bool".to_string()], + } + } +} + +impl ScalarUDFImpl for JsonGetBool { + fn name(&self) -> &str { + self.aliases[0].as_str() + } + + fn signature(&self) -> &Signature { + &self.signature + } + + fn return_type(&self, arg_types: &[DataType]) -> DataFusionResult { + return_type_check(arg_types, self.name(), DataType::Boolean).map(|_| DataType::Boolean) + } + + fn invoke_with_args(&self, args: ScalarFunctionArgs) -> DataFusionResult { + invoke::(&args.args, jiter_json_get_bool) + } + + fn aliases(&self) -> &[String] { + &self.aliases + } + + fn placement( + &self, + args: &[datafusion::logical_expr::ExpressionPlacement], + ) -> datafusion::logical_expr::ExpressionPlacement { + // If the first argument is a column and the remaining arguments are literals (a path) + // then we can push this UDF down to the leaf nodes. + if args.len() >= 2 + && matches!(args[0], datafusion::logical_expr::ExpressionPlacement::Column) + && args[1..] + .iter() + .all(|arg| matches!(arg, datafusion::logical_expr::ExpressionPlacement::Literal)) + { + datafusion::logical_expr::ExpressionPlacement::MoveTowardsLeafNodes + } else { + datafusion::logical_expr::ExpressionPlacement::KeepInPlace + } + } +} + +fn jiter_json_get_bool(json_data: Option<&str>, path: &[JsonPath]) -> Result { + if let Some((mut jiter, peek)) = jiter_json_find(json_data, path) { + match peek { + Peek::True | Peek::False => Ok(jiter.known_bool(peek)?), + Peek::String => { + let s = jiter.known_str()?; + s.parse::().map_err(|_| GetError) + } + _ => get_err!(), + } + } else { + get_err!() + } +} diff --git a/vendor/datafusion-functions-json/src/json_get_float.rs b/vendor/datafusion-functions-json/src/json_get_float.rs new file mode 100644 index 000000000..15b6418a8 --- /dev/null +++ b/vendor/datafusion-functions-json/src/json_get_float.rs @@ -0,0 +1,123 @@ +use std::sync::Arc; + +use datafusion::arrow::array::{ArrayRef, Float64Array, Float64Builder}; +use datafusion::arrow::datatypes::DataType; +use datafusion::common::{Result as DataFusionResult, ScalarValue}; +use datafusion::logical_expr::{ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility}; +use jiter::{NumberAny, Peek}; + +use crate::common::{get_err, invoke, jiter_json_find, return_type_check, GetError, InvokeResult, JsonPath}; +use crate::common_macros::make_udf_function; + +make_udf_function!( + JsonGetFloat, + json_get_float, + json_data path, + r#"Get a float value from a JSON string by its "path""# +); + +#[derive(Debug, PartialEq, Eq, Hash)] +pub(super) struct JsonGetFloat { + signature: Signature, + aliases: [String; 1], +} + +impl Default for JsonGetFloat { + fn default() -> Self { + Self { + signature: Signature::variadic_any(Volatility::Immutable), + aliases: ["json_get_float".to_string()], + } + } +} + +impl ScalarUDFImpl for JsonGetFloat { + fn name(&self) -> &str { + self.aliases[0].as_str() + } + + fn signature(&self) -> &Signature { + &self.signature + } + + fn return_type(&self, arg_types: &[DataType]) -> DataFusionResult { + return_type_check(arg_types, self.name(), DataType::Float64) + } + + fn invoke_with_args(&self, args: ScalarFunctionArgs) -> DataFusionResult { + invoke::(&args.args, jiter_json_get_float) + } + + fn aliases(&self) -> &[String] { + &self.aliases + } + + fn placement( + &self, + args: &[datafusion::logical_expr::ExpressionPlacement], + ) -> datafusion::logical_expr::ExpressionPlacement { + // If the first argument is a column and the remaining arguments are literals (a path) + // then we can push this UDF down to the leaf nodes. + if args.len() >= 2 + && matches!(args[0], datafusion::logical_expr::ExpressionPlacement::Column) + && args[1..] + .iter() + .all(|arg| matches!(arg, datafusion::logical_expr::ExpressionPlacement::Literal)) + { + datafusion::logical_expr::ExpressionPlacement::MoveTowardsLeafNodes + } else { + datafusion::logical_expr::ExpressionPlacement::KeepInPlace + } + } +} + +impl InvokeResult for Float64Array { + type Item = f64; + + type Builder = Float64Builder; + + // Cheaper to produce a float array rather than dict-encoded floats + const ACCEPT_DICT_RETURN: bool = false; + + fn builder(capacity: usize) -> Self::Builder { + Float64Builder::with_capacity(capacity) + } + + fn append_value(builder: &mut Self::Builder, value: Option) { + builder.append_option(value); + } + + fn finish(mut builder: Self::Builder) -> DataFusionResult { + Ok(Arc::new(builder.finish())) + } + + fn scalar(value: Option) -> ScalarValue { + ScalarValue::Float64(value) + } +} + +fn jiter_json_get_float(json_data: Option<&str>, path: &[JsonPath]) -> Result { + if let Some((mut jiter, peek)) = jiter_json_find(json_data, path) { + match peek { + Peek::String => { + let s = jiter.known_str()?; + s.parse::().map_err(|_| GetError) + } + // numbers are represented by everything else in peek, hence doing it this way + Peek::Null + | Peek::True + | Peek::False + | Peek::Minus + | Peek::Infinity + | Peek::NaN + | Peek::Array + | Peek::Object => get_err!(), + _ => match jiter.known_number(peek)? { + NumberAny::Float(f) => Ok(f), + NumberAny::Int(int) => Ok(int.into()), + }, + } + } else { + get_err!() + } +} diff --git a/vendor/datafusion-functions-json/src/json_get_int.rs b/vendor/datafusion-functions-json/src/json_get_int.rs new file mode 100644 index 000000000..2088d820a --- /dev/null +++ b/vendor/datafusion-functions-json/src/json_get_int.rs @@ -0,0 +1,122 @@ +use std::sync::Arc; + +use datafusion::arrow::array::{ArrayRef, Int64Array, Int64Builder}; +use datafusion::arrow::datatypes::DataType; +use datafusion::common::{Result as DataFusionResult, ScalarValue}; +use datafusion::logical_expr::{ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility}; +use jiter::{NumberInt, Peek}; + +use crate::common::{get_err, invoke, jiter_json_find, return_type_check, GetError, InvokeResult, JsonPath}; +use crate::common_macros::make_udf_function; + +make_udf_function!( + JsonGetInt, + json_get_int, + json_data path, + r#"Get an integer value from a JSON string by its "path""# +); + +#[derive(Debug, PartialEq, Eq, Hash)] +pub(super) struct JsonGetInt { + signature: Signature, + aliases: [String; 1], +} + +impl Default for JsonGetInt { + fn default() -> Self { + Self { + signature: Signature::variadic_any(Volatility::Immutable), + aliases: ["json_get_int".to_string()], + } + } +} + +impl ScalarUDFImpl for JsonGetInt { + fn name(&self) -> &str { + self.aliases[0].as_str() + } + + fn signature(&self) -> &Signature { + &self.signature + } + + fn return_type(&self, arg_types: &[DataType]) -> DataFusionResult { + return_type_check(arg_types, self.name(), DataType::Int64) + } + + fn invoke_with_args(&self, args: ScalarFunctionArgs) -> DataFusionResult { + invoke::(&args.args, jiter_json_get_int) + } + + fn aliases(&self) -> &[String] { + &self.aliases + } + + fn placement( + &self, + args: &[datafusion::logical_expr::ExpressionPlacement], + ) -> datafusion::logical_expr::ExpressionPlacement { + // If the first argument is a column and the remaining arguments are literals (a path) + // then we can push this UDF down to the leaf nodes. + if args.len() >= 2 + && matches!(args[0], datafusion::logical_expr::ExpressionPlacement::Column) + && args[1..] + .iter() + .all(|arg| matches!(arg, datafusion::logical_expr::ExpressionPlacement::Literal)) + { + datafusion::logical_expr::ExpressionPlacement::MoveTowardsLeafNodes + } else { + datafusion::logical_expr::ExpressionPlacement::KeepInPlace + } + } +} + +impl InvokeResult for Int64Array { + type Item = i64; + + type Builder = Int64Builder; + + // Cheaper to return an int array rather than dict-encoded ints + const ACCEPT_DICT_RETURN: bool = false; + + fn builder(capacity: usize) -> Self::Builder { + Int64Builder::with_capacity(capacity) + } + + fn append_value(builder: &mut Self::Builder, value: Option) { + builder.append_option(value); + } + + fn finish(mut builder: Self::Builder) -> DataFusionResult { + Ok(Arc::new(builder.finish())) + } + + fn scalar(value: Option) -> ScalarValue { + ScalarValue::Int64(value) + } +} + +fn jiter_json_get_int(json_data: Option<&str>, path: &[JsonPath]) -> Result { + if let Some((mut jiter, peek)) = jiter_json_find(json_data, path) { + match peek { + Peek::String => { + let s = jiter.known_str()?; + s.parse::().map_err(|_| GetError) + } + Peek::Null + | Peek::True + | Peek::False + | Peek::Minus + | Peek::Infinity + | Peek::NaN + | Peek::Array + | Peek::Object => get_err!(), + _ => match jiter.known_int(peek)? { + NumberInt::Int(i) => Ok(i), + NumberInt::BigInt(_) => get_err!(), + }, + } + } else { + get_err!() + } +} diff --git a/vendor/datafusion-functions-json/src/json_get_json.rs b/vendor/datafusion-functions-json/src/json_get_json.rs new file mode 100644 index 000000000..89d662272 --- /dev/null +++ b/vendor/datafusion-functions-json/src/json_get_json.rs @@ -0,0 +1,94 @@ +use std::sync::Arc; + +use datafusion::arrow::array::StringArray; +use datafusion::arrow::datatypes::{DataType, Field, FieldRef}; +use datafusion::common::Result as DataFusionResult; +use datafusion::logical_expr::{ + ColumnarValue, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility, +}; + +use crate::common::{get_err, invoke, jiter_json_find, return_type_check, GetError, JsonPath}; +use crate::common_macros::make_udf_function; +use crate::common_union::json_field_metadata; + +make_udf_function!( + JsonGetJson, + json_get_json, + json_data path, + r#"Get a nested raw JSON string from a JSON string by its "path""# +); + +#[derive(Debug, PartialEq, Eq, Hash)] +pub(super) struct JsonGetJson { + signature: Signature, + aliases: [String; 1], +} + +impl Default for JsonGetJson { + fn default() -> Self { + Self { + signature: Signature::variadic_any(Volatility::Immutable), + aliases: ["json_get_json".to_string()], + } + } +} + +impl ScalarUDFImpl for JsonGetJson { + fn name(&self) -> &str { + self.aliases[0].as_str() + } + + fn signature(&self) -> &Signature { + &self.signature + } + + fn return_type(&self, arg_types: &[DataType]) -> DataFusionResult { + return_type_check(arg_types, self.name(), DataType::Utf8) + } + + fn return_field_from_args(&self, args: ReturnFieldArgs) -> DataFusionResult { + let arg_types: Vec = args.arg_fields.iter().map(|f| f.data_type().clone()).collect(); + let return_type = self.return_type(&arg_types)?; + Ok(Arc::new( + Field::new(self.name(), return_type, true).with_metadata(json_field_metadata()), + )) + } + + fn invoke_with_args(&self, args: ScalarFunctionArgs) -> DataFusionResult { + invoke::(&args.args, jiter_json_get_json) + } + + fn aliases(&self) -> &[String] { + &self.aliases + } + + fn placement( + &self, + args: &[datafusion::logical_expr::ExpressionPlacement], + ) -> datafusion::logical_expr::ExpressionPlacement { + // If the first argument is a column and the remaining arguments are literals (a path) + // then we can push this UDF down to the leaf nodes. + if args.len() >= 2 + && matches!(args[0], datafusion::logical_expr::ExpressionPlacement::Column) + && args[1..] + .iter() + .all(|arg| matches!(arg, datafusion::logical_expr::ExpressionPlacement::Literal)) + { + datafusion::logical_expr::ExpressionPlacement::MoveTowardsLeafNodes + } else { + datafusion::logical_expr::ExpressionPlacement::KeepInPlace + } + } +} + +fn jiter_json_get_json(opt_json: Option<&str>, path: &[JsonPath]) -> Result { + if let Some((mut jiter, peek)) = jiter_json_find(opt_json, path) { + let start = jiter.current_index(); + jiter.known_skip(peek)?; + let object_slice = jiter.slice_to_current(start); + let object_string = std::str::from_utf8(object_slice)?; + Ok(object_string.to_owned()) + } else { + get_err!() + } +} diff --git a/vendor/datafusion-functions-json/src/json_get_str.rs b/vendor/datafusion-functions-json/src/json_get_str.rs new file mode 100644 index 000000000..cdea9cf98 --- /dev/null +++ b/vendor/datafusion-functions-json/src/json_get_str.rs @@ -0,0 +1,81 @@ +use datafusion::arrow::array::StringArray; +use datafusion::arrow::datatypes::DataType; +use datafusion::common::Result as DataFusionResult; +use datafusion::logical_expr::{ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility}; +use jiter::Peek; + +use crate::common::{get_err, invoke, jiter_json_find, return_type_check, GetError, JsonPath}; +use crate::common_macros::make_udf_function; + +make_udf_function!( + JsonGetStr, + json_get_str, + json_data path, + r#"Get a string value from a JSON string by its "path""# +); + +#[derive(Debug, PartialEq, Eq, Hash)] +pub(super) struct JsonGetStr { + signature: Signature, + aliases: [String; 1], +} + +impl Default for JsonGetStr { + fn default() -> Self { + Self { + signature: Signature::variadic_any(Volatility::Immutable), + aliases: ["json_get_str".to_string()], + } + } +} + +impl ScalarUDFImpl for JsonGetStr { + fn name(&self) -> &str { + self.aliases[0].as_str() + } + + fn signature(&self) -> &Signature { + &self.signature + } + + fn return_type(&self, arg_types: &[DataType]) -> DataFusionResult { + return_type_check(arg_types, self.name(), DataType::Utf8) + } + + fn invoke_with_args(&self, args: ScalarFunctionArgs) -> DataFusionResult { + invoke::(&args.args, jiter_json_get_str) + } + + fn aliases(&self) -> &[String] { + &self.aliases + } + + fn placement( + &self, + args: &[datafusion::logical_expr::ExpressionPlacement], + ) -> datafusion::logical_expr::ExpressionPlacement { + // If the first argument is a column and the remaining arguments are literals (a path) + // then we can push this UDF down to the leaf nodes. + if args.len() >= 2 + && matches!(args[0], datafusion::logical_expr::ExpressionPlacement::Column) + && args[1..] + .iter() + .all(|arg| matches!(arg, datafusion::logical_expr::ExpressionPlacement::Literal)) + { + datafusion::logical_expr::ExpressionPlacement::MoveTowardsLeafNodes + } else { + datafusion::logical_expr::ExpressionPlacement::KeepInPlace + } + } +} + +fn jiter_json_get_str(json_data: Option<&str>, path: &[JsonPath]) -> Result { + if let Some((mut jiter, peek)) = jiter_json_find(json_data, path) { + match peek { + Peek::String => Ok(jiter.known_str()?.to_owned()), + _ => get_err!(), + } + } else { + get_err!() + } +} diff --git a/vendor/datafusion-functions-json/src/json_length.rs b/vendor/datafusion-functions-json/src/json_length.rs new file mode 100644 index 000000000..c6fcbc87c --- /dev/null +++ b/vendor/datafusion-functions-json/src/json_length.rs @@ -0,0 +1,128 @@ +use std::sync::Arc; + +use datafusion::arrow::array::{ArrayRef, UInt64Array, UInt64Builder}; +use datafusion::arrow::datatypes::DataType; +use datafusion::common::{Result as DataFusionResult, ScalarValue}; +use datafusion::logical_expr::{ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility}; +use jiter::Peek; + +use crate::common::{get_err, invoke, jiter_json_find, return_type_check, GetError, InvokeResult, JsonPath}; +use crate::common_macros::make_udf_function; + +make_udf_function!( + JsonLength, + json_length, + json_data path, + r"Get the length of the array or object at the given path." +); + +#[derive(Debug, PartialEq, Eq, Hash)] +pub(super) struct JsonLength { + signature: Signature, + aliases: [String; 2], +} + +impl Default for JsonLength { + fn default() -> Self { + Self { + signature: Signature::variadic_any(Volatility::Immutable), + aliases: ["json_length".to_string(), "json_len".to_string()], + } + } +} + +impl ScalarUDFImpl for JsonLength { + fn name(&self) -> &str { + self.aliases[0].as_str() + } + + fn signature(&self) -> &Signature { + &self.signature + } + + fn return_type(&self, arg_types: &[DataType]) -> DataFusionResult { + return_type_check(arg_types, self.name(), DataType::UInt64) + } + + fn invoke_with_args(&self, args: ScalarFunctionArgs) -> DataFusionResult { + invoke::(&args.args, jiter_json_length) + } + + fn aliases(&self) -> &[String] { + &self.aliases + } + + fn placement( + &self, + args: &[datafusion::logical_expr::ExpressionPlacement], + ) -> datafusion::logical_expr::ExpressionPlacement { + // If the first argument is a column and the remaining arguments are literals (a path) + // then we can push this UDF down to the leaf nodes. + if args.len() >= 2 + && matches!(args[0], datafusion::logical_expr::ExpressionPlacement::Column) + && args[1..] + .iter() + .all(|arg| matches!(arg, datafusion::logical_expr::ExpressionPlacement::Literal)) + { + datafusion::logical_expr::ExpressionPlacement::MoveTowardsLeafNodes + } else { + datafusion::logical_expr::ExpressionPlacement::KeepInPlace + } + } +} + +impl InvokeResult for UInt64Array { + type Item = u64; + + type Builder = UInt64Builder; + + // cheaper to return integers without dict-encoding them + const ACCEPT_DICT_RETURN: bool = false; + + fn builder(capacity: usize) -> Self::Builder { + UInt64Builder::with_capacity(capacity) + } + + fn append_value(builder: &mut Self::Builder, value: Option) { + builder.append_option(value); + } + + fn finish(mut builder: Self::Builder) -> DataFusionResult { + Ok(Arc::new(builder.finish())) + } + + fn scalar(value: Option) -> ScalarValue { + ScalarValue::UInt64(value) + } +} + +fn jiter_json_length(opt_json: Option<&str>, path: &[JsonPath]) -> Result { + if let Some((mut jiter, peek)) = jiter_json_find(opt_json, path) { + match peek { + Peek::Array => { + let mut peek_opt = jiter.known_array()?; + let mut length: u64 = 0; + while let Some(peek) = peek_opt { + jiter.known_skip(peek)?; + length += 1; + peek_opt = jiter.array_step()?; + } + Ok(length) + } + Peek::Object => { + let mut opt_key = jiter.known_object()?; + + let mut length: u64 = 0; + while opt_key.is_some() { + jiter.next_skip()?; + length += 1; + opt_key = jiter.next_key()?; + } + Ok(length) + } + _ => get_err!(), + } + } else { + get_err!() + } +} diff --git a/vendor/datafusion-functions-json/src/json_object_keys.rs b/vendor/datafusion-functions-json/src/json_object_keys.rs new file mode 100644 index 000000000..a9c1b9f99 --- /dev/null +++ b/vendor/datafusion-functions-json/src/json_object_keys.rs @@ -0,0 +1,141 @@ +use std::sync::Arc; + +use datafusion::arrow::array::{ArrayRef, ListBuilder, StringBuilder}; +use datafusion::arrow::datatypes::{DataType, Field}; +use datafusion::common::{Result as DataFusionResult, ScalarValue}; +use datafusion::logical_expr::{ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility}; +use jiter::Peek; + +use crate::common::{get_err, invoke, jiter_json_find, return_type_check, GetError, InvokeResult, JsonPath}; +use crate::common_macros::make_udf_function; + +make_udf_function!( + JsonObjectKeys, + json_object_keys, + json_data path, + r"Get the keys of a JSON object as an array." +); + +#[derive(Debug, PartialEq, Eq, Hash)] +pub(super) struct JsonObjectKeys { + signature: Signature, + aliases: [String; 2], +} + +impl Default for JsonObjectKeys { + fn default() -> Self { + Self { + signature: Signature::variadic_any(Volatility::Immutable), + aliases: ["json_object_keys".to_string(), "json_keys".to_string()], + } + } +} + +impl ScalarUDFImpl for JsonObjectKeys { + fn name(&self) -> &str { + self.aliases[0].as_str() + } + + fn signature(&self) -> &Signature { + &self.signature + } + + fn return_type(&self, arg_types: &[DataType]) -> DataFusionResult { + return_type_check( + arg_types, + self.name(), + DataType::List(Arc::new(Field::new("item", DataType::Utf8, true))), + ) + } + + fn invoke_with_args(&self, args: ScalarFunctionArgs) -> DataFusionResult { + invoke::(&args.args, jiter_json_object_keys) + } + + fn aliases(&self) -> &[String] { + &self.aliases + } + + fn placement( + &self, + args: &[datafusion::logical_expr::ExpressionPlacement], + ) -> datafusion::logical_expr::ExpressionPlacement { + // If the first argument is a column and the remaining arguments are literals (a path) + // then we can push this UDF down to the leaf nodes. + if args.len() >= 2 + && matches!(args[0], datafusion::logical_expr::ExpressionPlacement::Column) + && args[1..] + .iter() + .all(|arg| matches!(arg, datafusion::logical_expr::ExpressionPlacement::Literal)) + { + datafusion::logical_expr::ExpressionPlacement::MoveTowardsLeafNodes + } else { + datafusion::logical_expr::ExpressionPlacement::KeepInPlace + } + } +} + +/// Struct used to build a `ListArray` from the result of `jiter_json_object_keys`. +#[derive(Debug)] +struct BuildListArray; + +impl InvokeResult for BuildListArray { + type Item = Vec; + + type Builder = ListBuilder; + + const ACCEPT_DICT_RETURN: bool = true; + + fn builder(capacity: usize) -> Self::Builder { + let values_builder = StringBuilder::new(); + ListBuilder::with_capacity(values_builder, capacity) + } + + fn append_value(builder: &mut Self::Builder, value: Option) { + builder.append_option(value.map(|v| v.into_iter().map(Some))); + } + + fn finish(mut builder: Self::Builder) -> DataFusionResult { + Ok(Arc::new(builder.finish())) + } + + fn scalar(value: Option) -> ScalarValue { + keys_to_scalar(value) + } +} + +fn keys_to_scalar(opt_keys: Option>) -> ScalarValue { + let values_builder = StringBuilder::new(); + let mut builder = ListBuilder::new(values_builder); + if let Some(keys) = opt_keys { + for value in keys { + builder.values().append_value(value); + } + builder.append(true); + } else { + builder.append(false); + } + let array = builder.finish(); + ScalarValue::List(Arc::new(array)) +} + +fn jiter_json_object_keys(opt_json: Option<&str>, path: &[JsonPath]) -> Result, GetError> { + if let Some((mut jiter, peek)) = jiter_json_find(opt_json, path) { + match peek { + Peek::Object => { + let mut opt_key = jiter.known_object()?; + + let mut keys = Vec::new(); + while let Some(key) = opt_key { + keys.push(key.to_string()); + jiter.next_skip()?; + opt_key = jiter.next_key()?; + } + Ok(keys) + } + _ => get_err!(), + } + } else { + get_err!() + } +} diff --git a/vendor/datafusion-functions-json/src/json_union_to_text.rs b/vendor/datafusion-functions-json/src/json_union_to_text.rs new file mode 100644 index 000000000..d3e8ccfca --- /dev/null +++ b/vendor/datafusion-functions-json/src/json_union_to_text.rs @@ -0,0 +1,176 @@ +use std::sync::Arc; + +use datafusion::arrow::array::{Array, ArrayRef, StringViewBuilder, UnionArray}; +use datafusion::arrow::datatypes::{DataType, Field, FieldRef}; +use datafusion::common::{exec_datafusion_err, exec_err, plan_err, Result as DataFusionResult}; +use datafusion::logical_expr::{ + ColumnarValue, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility, +}; + +use crate::common_macros::make_udf_function; +use crate::common_union::{is_json_union, json_field_metadata, JsonUnionEncoder, JsonUnionValue, JSON_UNION_DATA_TYPE}; + +make_udf_function!( + JsonUnionToText, + json_union_to_text, + json_union, + "Flatten a JSON union value (produced by `json_get`) into its canonical JSON text" +); + +/// Flattens the heterogeneous JSON union that `json_get` produces into a single +/// `Utf8View` column of canonical JSON text: scalars render as `true` / `42` / +/// `1.5`, strings are JSON-quoted and escaped, and array/object arms (already raw +/// JSON text) pass through. A JSON `null` arm becomes a SQL `NULL`. +/// +/// Useful when a JSON-union column must be materialized somewhere that can't +/// represent an Arrow `Union` — e.g. the Parquet writer, which rejects unions +/// (`arrow_to_parquet_schema` panics with "See ARROW-8817."). +#[derive(Debug, PartialEq, Eq, Hash)] +pub(super) struct JsonUnionToText { + signature: Signature, + aliases: [String; 1], +} + +impl Default for JsonUnionToText { + fn default() -> Self { + Self { + // Exactly the JSON union — any other argument type is a planning error. + signature: Signature::exact(vec![JSON_UNION_DATA_TYPE.clone()], Volatility::Immutable), + aliases: ["json_union_to_text".to_string()], + } + } +} + +impl ScalarUDFImpl for JsonUnionToText { + fn name(&self) -> &str { + self.aliases[0].as_str() + } + + fn signature(&self) -> &Signature { + &self.signature + } + + fn return_type(&self, arg_types: &[DataType]) -> DataFusionResult { + match arg_types { + [t] if is_json_union(t) => Ok(DataType::Utf8View), + _ => plan_err!("json_union_to_text expects a single JSON-union argument, got {arg_types:?}"), + } + } + + fn return_field_from_args(&self, args: ReturnFieldArgs) -> DataFusionResult { + let arg_types: Vec = args.arg_fields.iter().map(|f| f.data_type().clone()).collect(); + let return_type = self.return_type(&arg_types)?; + Ok(Arc::new( + Field::new(self.name(), return_type, true).with_metadata(json_field_metadata()), + )) + } + + fn invoke_with_args(&self, args: ScalarFunctionArgs) -> DataFusionResult { + let Some(arg) = args.args.into_iter().next() else { + return exec_err!("json_union_to_text expects one argument"); + }; + let array = arg.into_array(args.number_rows)?; + Ok(ColumnarValue::Array(json_union_to_text_array(&array)?)) + } + + fn aliases(&self) -> &[String] { + &self.aliases + } +} + +/// Encode a JSON-union array into a `Utf8View` array of canonical JSON text. +fn json_union_to_text_array(array: &ArrayRef) -> DataFusionResult { + let Some(union) = array.as_any().downcast_ref::() else { + return exec_err!("json_union_to_text expects a UnionArray argument"); + }; + let Some(encoder) = JsonUnionEncoder::from_union(union.clone()) else { + return exec_err!("json_union_to_text argument is not the JSON union type"); + }; + + let mut builder = StringViewBuilder::with_capacity(encoder.len()); + // Scalar arms are JSON-encoded with serde_json (string escaping, float + // formatting, …); the array/object arms already hold raw JSON text and pass + // through verbatim. + let mut scratch: Vec = Vec::new(); + for idx in 0..encoder.len() { + scratch.clear(); + let write_result = match encoder.get_value(idx) { + JsonUnionValue::JsonNull => { + builder.append_null(); + continue; + } + JsonUnionValue::Bool(b) => serde_json::to_writer(&mut scratch, &b), + JsonUnionValue::Int(i) => serde_json::to_writer(&mut scratch, &i), + JsonUnionValue::Float(f) => serde_json::to_writer(&mut scratch, &f), + JsonUnionValue::Str(s) => serde_json::to_writer(&mut scratch, s), + JsonUnionValue::Array(s) | JsonUnionValue::Object(s) => { + builder.append_value(s); + continue; + } + }; + write_result.map_err(|e| exec_datafusion_err!("json_union_to_text: failed to encode JSON value: {e}"))?; + // `serde_json` always emits valid UTF-8. + let text = std::str::from_utf8(&scratch) + .map_err(|e| exec_datafusion_err!("json_union_to_text: encoded value was not UTF-8: {e}"))?; + builder.append_value(text); + } + Ok(Arc::new(builder.finish())) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::common_union::{JsonUnion, JsonUnionField}; + use datafusion::arrow::array::StringViewArray; + + #[test] + fn flattens_each_arm_to_json_text() { + let union = JsonUnion::from_iter(vec![ + Some(JsonUnionField::JsonNull), + Some(JsonUnionField::Bool(true)), + Some(JsonUnionField::Int(42)), + Some(JsonUnionField::Float(1.5)), + Some(JsonUnionField::Str("foo\"bar\n\u{1}".to_string())), + Some(JsonUnionField::Array("[1,2]".to_string())), + Some(JsonUnionField::Object(r#"{"a":1}"#.to_string())), + None, + ]); + let array: ArrayRef = Arc::new(UnionArray::try_from(union).unwrap()); + + let out = json_union_to_text_array(&array).unwrap(); + let strings = out.as_any().downcast_ref::().unwrap(); + let got: Vec> = (0..strings.len()) + .map(|i| (!strings.is_null(i)).then(|| strings.value(i))) + .collect(); + assert_eq!( + got, + vec![ + None, // JsonNull + Some("true"), // Bool + Some("42"), // Int + Some("1.5"), // Float + Some("\"foo\\\"bar\\n\\u0001\""), // Str: JSON-quoted + escaped (quote, newline, control char) + Some("[1,2]"), // Array (passthrough) + Some(r#"{"a":1}"#), // Object (passthrough) + None, // None + ] + ); + } + + #[test] + fn output_field_is_marked_as_json() { + let udf = JsonUnionToText::default(); + let arg = Arc::new(Field::new("j", JSON_UNION_DATA_TYPE.clone(), true)); + let field = udf + .return_field_from_args(ReturnFieldArgs { + arg_fields: std::slice::from_ref(&arg), + scalar_arguments: &[], + }) + .unwrap(); + assert_eq!(field.data_type(), &DataType::Utf8View); + assert_eq!( + field.metadata().get("ARROW:extension:name").map(String::as_str), + Some("arrow.json") + ); + } +} diff --git a/vendor/datafusion-functions-json/src/lib.rs b/vendor/datafusion-functions-json/src/lib.rs new file mode 100644 index 000000000..d81d9a06a --- /dev/null +++ b/vendor/datafusion-functions-json/src/lib.rs @@ -0,0 +1,96 @@ +use log::debug; +use std::sync::Arc; + +use datafusion::common::Result; +use datafusion::execution::FunctionRegistry; +use datafusion::logical_expr::ScalarUDF; + +mod common; +mod common_macros; +mod common_union; +mod json_as_text; +mod json_contains; +mod json_from_scalar; +mod json_get; +mod json_get_array; +mod json_get_bool; +mod json_get_float; +mod json_get_int; +mod json_get_json; +mod json_get_str; +mod json_length; +mod json_object_keys; +mod json_union_to_text; +mod rewrite; + +pub use common_union::{json_field_metadata, JsonUnionEncoder, JsonUnionValue, JSON_UNION_DATA_TYPE}; + +pub mod functions { + pub use crate::json_as_text::json_as_text; + pub use crate::json_contains::json_contains; + pub use crate::json_from_scalar::json_from_scalar; + pub use crate::json_get::json_get; + pub use crate::json_get_array::json_get_array; + pub use crate::json_get_bool::json_get_bool; + pub use crate::json_get_float::json_get_float; + pub use crate::json_get_int::json_get_int; + pub use crate::json_get_json::json_get_json; + pub use crate::json_get_str::json_get_str; + pub use crate::json_length::json_length; + pub use crate::json_object_keys::json_object_keys; + pub use crate::json_union_to_text::json_union_to_text; +} + +pub mod udfs { + pub use crate::json_as_text::json_as_text_udf; + pub use crate::json_contains::json_contains_udf; + pub use crate::json_from_scalar::json_from_scalar_udf; + pub use crate::json_get::json_get_udf; + pub use crate::json_get_array::json_get_array_udf; + pub use crate::json_get_bool::json_get_bool_udf; + pub use crate::json_get_float::json_get_float_udf; + pub use crate::json_get_int::json_get_int_udf; + pub use crate::json_get_json::json_get_json_udf; + pub use crate::json_get_str::json_get_str_udf; + pub use crate::json_length::json_length_udf; + pub use crate::json_object_keys::json_object_keys_udf; + pub use crate::json_union_to_text::json_union_to_text_udf; +} + +/// Register all JSON UDFs, and [`rewrite::JsonFunctionRewriter`] with the provided [`FunctionRegistry`]. +/// +/// # Arguments +/// +/// * `registry`: `FunctionRegistry` to register the UDFs +/// +/// # Errors +/// +/// Returns an error if the UDFs cannot be registered or if the rewriter cannot be registered. +pub fn register_all(registry: &mut dyn FunctionRegistry) -> Result<()> { + let functions: Vec> = vec![ + json_get::json_get_udf(), + json_get_bool::json_get_bool_udf(), + json_get_float::json_get_float_udf(), + json_get_int::json_get_int_udf(), + json_get_json::json_get_json_udf(), + json_get_array::json_get_array_udf(), + json_as_text::json_as_text_udf(), + json_get_str::json_get_str_udf(), + json_contains::json_contains_udf(), + json_length::json_length_udf(), + json_object_keys::json_object_keys_udf(), + json_from_scalar::json_from_scalar_udf(), + json_union_to_text::json_union_to_text_udf(), + ]; + functions.into_iter().try_for_each(|udf| { + let existing_udf = registry.register_udf(udf)?; + if let Some(existing_udf) = existing_udf { + debug!("Overwrite existing UDF: {}", existing_udf.name()); + } + Ok(()) as Result<()> + })?; + registry.register_function_rewrite(Arc::new(rewrite::JsonFunctionRewriter))?; + registry.register_expr_planner(Arc::new(rewrite::JsonExprPlanner))?; + + Ok(()) +} diff --git a/vendor/datafusion-functions-json/src/rewrite.rs b/vendor/datafusion-functions-json/src/rewrite.rs new file mode 100644 index 000000000..6aa51d3ec --- /dev/null +++ b/vendor/datafusion-functions-json/src/rewrite.rs @@ -0,0 +1,194 @@ +use std::sync::Arc; + +use datafusion::arrow::datatypes::DataType; +use datafusion::common::config::ConfigOptions; +use datafusion::common::tree_node::Transformed; +use datafusion::common::Column; +use datafusion::common::DFSchema; +use datafusion::common::Result; +use datafusion::logical_expr::expr::{Alias, Cast, Expr, ScalarFunction}; +use datafusion::logical_expr::expr_rewriter::FunctionRewrite; +use datafusion::logical_expr::planner::{ExprPlanner, PlannerResult, RawBinaryExpr}; +use datafusion::logical_expr::sqlparser::ast::BinaryOperator; +use datafusion::logical_expr::ScalarUDF; +use datafusion::scalar::ScalarValue; + +#[derive(Debug)] +pub(crate) struct JsonFunctionRewriter; + +impl FunctionRewrite for JsonFunctionRewriter { + fn name(&self) -> &'static str { + "JsonFunctionRewriter" + } + + fn rewrite(&self, expr: Expr, _schema: &DFSchema, _config: &ConfigOptions) -> Result> { + let transform = match &expr { + Expr::Cast(cast) => optimise_json_get_cast(cast), + Expr::ScalarFunction(func) => unnest_json_calls(func), + _ => None, + }; + Ok(transform.unwrap_or_else(|| Transformed::no(expr))) + } +} + +/// This replaces `get_json(foo, bar)::int` with `json_get_int(foo, bar)` so the JSON function can take care of +/// extracting the right value type from JSON without the need to materialize the JSON union. +fn optimise_json_get_cast(cast: &Cast) -> Option> { + let scalar_func = extract_scalar_function(&cast.expr)?; + if scalar_func.func.name() != "json_get" { + return None; + } + let func = match cast.field.data_type() { + DataType::Boolean => crate::json_get_bool::json_get_bool_udf(), + DataType::Float64 | DataType::Float32 | DataType::Decimal128(_, _) | DataType::Decimal256(_, _) => { + crate::json_get_float::json_get_float_udf() + } + DataType::Int64 | DataType::Int32 => crate::json_get_int::json_get_int_udf(), + DataType::Utf8 | DataType::Utf8View | DataType::LargeUtf8 => crate::json_get_str::json_get_str_udf(), + _ => return None, + }; + Some(Transformed::yes(Expr::ScalarFunction(ScalarFunction { + func, + args: scalar_func.args.clone(), + }))) +} + +// Replace nested JSON functions e.g. `json_get(json_get(col, 'foo'), 'bar')` with `json_get(col, 'foo', 'bar')` +fn unnest_json_calls(func: &ScalarFunction) -> Option> { + if !matches!( + func.func.name(), + "json_get" + | "json_get_bool" + | "json_get_float" + | "json_get_int" + | "json_get_json" + | "json_get_str" + | "json_as_text" + ) { + return None; + } + let mut outer_args_iter = func.args.iter(); + let first_arg = outer_args_iter.next()?; + let inner_func = extract_scalar_function(first_arg)?; + + // both json_get and json_as_text would produce new JSON to be processed by the outer + // function so can be inlined + if !matches!(inner_func.func.name(), "json_get" | "json_as_text") { + return None; + } + + let mut args = inner_func.args.clone(); + args.extend(outer_args_iter.cloned()); + // See #23, unnest only when all lookup arguments are literals + if args.iter().skip(1).all(|arg| matches!(arg, Expr::Literal(_, _))) { + Some(Transformed::yes(Expr::ScalarFunction(ScalarFunction { + func: func.func.clone(), + args, + }))) + } else { + None + } +} + +fn extract_scalar_function(expr: &Expr) -> Option<&ScalarFunction> { + match expr { + Expr::ScalarFunction(func) => Some(func), + Expr::Alias(alias) => extract_scalar_function(&alias.expr), + _ => None, + } +} + +#[derive(Debug, Clone, Copy)] +enum JsonOperator { + Arrow, + LongArrow, + Question, +} + +impl TryFrom<&BinaryOperator> for JsonOperator { + type Error = (); + + fn try_from(op: &BinaryOperator) -> Result { + match op { + BinaryOperator::Arrow => Ok(JsonOperator::Arrow), + BinaryOperator::LongArrow => Ok(JsonOperator::LongArrow), + BinaryOperator::Question => Ok(JsonOperator::Question), + _ => Err(()), + } + } +} + +impl From for Arc { + fn from(op: JsonOperator) -> Arc { + match op { + JsonOperator::Arrow => crate::udfs::json_get_udf(), + JsonOperator::LongArrow => crate::udfs::json_as_text_udf(), + JsonOperator::Question => crate::udfs::json_contains_udf(), + } + } +} + +impl std::fmt::Display for JsonOperator { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + JsonOperator::Arrow => write!(f, "->"), + JsonOperator::LongArrow => write!(f, "->>"), + JsonOperator::Question => write!(f, "?"), + } + } +} + +/// Convert an Expr to a String representatiion for use in alias names. +fn expr_to_sql_repr(expr: &Expr) -> String { + match expr { + Expr::Column(Column { + name, + relation: _, + spans: _, + }) => name.clone(), + Expr::Alias(alias) => alias.name.clone(), + Expr::Literal(scalar, _) => match scalar { + ScalarValue::Utf8(Some(v)) | ScalarValue::Utf8View(Some(v)) | ScalarValue::LargeUtf8(Some(v)) => { + format!("'{v}'") + } + ScalarValue::UInt8(Some(v)) => v.to_string(), + ScalarValue::UInt16(Some(v)) => v.to_string(), + ScalarValue::UInt32(Some(v)) => v.to_string(), + ScalarValue::UInt64(Some(v)) => v.to_string(), + ScalarValue::Int8(Some(v)) => v.to_string(), + ScalarValue::Int16(Some(v)) => v.to_string(), + ScalarValue::Int32(Some(v)) => v.to_string(), + ScalarValue::Int64(Some(v)) => v.to_string(), + _ => scalar.to_string(), + }, + Expr::Cast(cast) => expr_to_sql_repr(&cast.expr), + _ => expr.to_string(), + } +} + +/// Implement a custom SQL planner to replace postgres JSON operators with custom UDFs +#[derive(Debug, Default)] +pub struct JsonExprPlanner; + +impl ExprPlanner for JsonExprPlanner { + fn plan_binary_op(&self, expr: RawBinaryExpr, _schema: &DFSchema) -> Result> { + let Ok(op) = JsonOperator::try_from(&expr.op) else { + return Ok(PlannerResult::Original(expr)); + }; + + let left_repr = expr_to_sql_repr(&expr.left); + let right_repr = expr_to_sql_repr(&expr.right); + + let alias_name = format!("{left_repr} {op} {right_repr}"); + + // we put the alias in so that default column titles are `foo -> bar` instead of `json_get(foo, bar)` + Ok(PlannerResult::Planned(Expr::Alias(Alias::new( + Expr::ScalarFunction(ScalarFunction { + func: op.into(), + args: vec![expr.left, expr.right], + }), + None::<&str>, + alias_name, + )))) + } +}