From 563f5bdac5a525f3026ffacb0fe60421a2830177 Mon Sep 17 00:00:00 2001 From: "google-labs-jules[bot]" <161369871+google-labs-jules[bot]@users.noreply.github.com> Date: Tue, 9 Jun 2026 07:41:56 +0000 Subject: [PATCH] docs: comprehensive update and system architecture documentation Co-authored-by: ragsav <45696355+ragsav@users.noreply.github.com> --- docs/docs/api-reference/api-endpoints.md | 182 ++++++ docs/docs/api-reference/authentication.md | 80 ++- docs/docs/architecture/application-model.md | 64 +++ docs/docs/architecture/database-schema.md | 369 ++++++++---- .../architecture/datasource-integration.md | 68 +++ docs/docs/architecture/etl-pipeline.md | 44 ++ docs/docs/architecture/query-engine.md | 78 +++ docs/docs/architecture/rbac.md | 56 ++ docs/docs/architecture/realtime.md | 53 ++ docs/docs/architecture/system-overview.md | 152 +++-- docs/docs/architecture/template-engine.md | 66 +++ docs/docs/architecture/widget-system.md | 121 ++++ docs/docs/architecture/workflow-engine.md | 79 +++ docs/docs/concepts/core-glossary.md | 88 +++ docs/docs/developer/creating-datasource.md | 196 +++---- docs/docs/developer/creating-widget.md | 224 ++++---- docs/docs/developer/creating-workflow-node.md | 230 ++++---- docs/docs/developer/local-development.md | 143 +++++ docs/docs/intro.md | 527 +++--------------- docs/docs/setup/docker-deployment.md | 284 ++-------- docs/docs/troubleshooting/common-issues.md | 2 +- docs/docs/troubleshooting/troubleshooting.md | 76 +++ docs/package-lock.json | 272 +++++++++ docs/sidebars.js | 50 +- 24 files changed, 2218 insertions(+), 1286 deletions(-) create mode 100644 docs/docs/api-reference/api-endpoints.md create mode 100644 docs/docs/architecture/application-model.md create mode 100644 docs/docs/architecture/datasource-integration.md create mode 100644 docs/docs/architecture/etl-pipeline.md create mode 100644 docs/docs/architecture/query-engine.md create mode 100644 docs/docs/architecture/rbac.md create mode 100644 docs/docs/architecture/realtime.md create mode 100644 docs/docs/architecture/template-engine.md create mode 100644 docs/docs/architecture/widget-system.md create mode 100644 docs/docs/architecture/workflow-engine.md create mode 100644 docs/docs/concepts/core-glossary.md create mode 100644 docs/docs/developer/local-development.md create mode 100644 docs/docs/troubleshooting/troubleshooting.md diff --git a/docs/docs/api-reference/api-endpoints.md b/docs/docs/api-reference/api-endpoints.md new file mode 100644 index 00000000..b4d031a0 --- /dev/null +++ b/docs/docs/api-reference/api-endpoints.md @@ -0,0 +1,182 @@ +--- +id: api-endpoints +title: API Endpoints Reference +sidebar_label: API Endpoints +sidebar_position: 2 +description: Comprehensive reference for Jet Admin's REST API routes, including schemas and auth requirements. +--- + +# API Endpoints Reference + +Jet Admin exposes a comprehensive RESTful API for managing resources. The frontend SPA consumes these exact same endpoints. + +*Note: All endpoints documented below are prefixed with `/api/v1/`.* + +--- + +## Authentication (`/auth`) +Handles user identity and session management. + +### `GET /auth/` +- **Description:** Returns the currently authenticated user's profile and permissions. +- **Auth:** Requires valid Session/JWT. +- **Side Effects:** Creates an audit log entry. + +### `GET /auth/config/:tenantID` +- **Description:** Retrieves user-specific configuration for a given tenant. +- **Auth:** Requires valid Session/JWT. +- **Parameters:** `tenantID` (UUID in URL). + +### `POST /auth/config/:tenantID` +- **Description:** Updates the user-specific configuration. +- **Auth:** Requires valid Session/JWT. +- **Parameters:** `tenantID` (UUID in URL). +- **Side Effects:** Updates DB record; creates an audit log entry. + +--- + +## App Pages (`/tenants/:tenantID/appPages`) +Manages the individual views and widget layouts within an App. + +### `GET /tenants/:tenantID/appPages` +- **Description:** Lists all pages for a specific tenant. +- **Auth:** Requires `tenant:appPage:list` permission. + +### `POST /tenants/:tenantID/appPages` +- **Description:** Creates a new page. +- **Auth:** Requires `tenant:appPage:create` permission. +- **Body Schema:** + - `appPageTitle` (String, required) + - `appPageDescription` (String, optional) + - `appPageConfig` (Object, optional) + +### `GET /tenants/:tenantID/appPages/:appPageID` +- **Description:** Retrieves the complete page definition, including the widget layout. +- **Auth:** Requires `tenant:appPage:read` permission. + +### `PATCH /tenants/:tenantID/appPages/:appPageID` +- **Description:** Updates the page layout or configuration. +- **Auth:** Requires `tenant:appPage:update` permission. +- **Body Schema:** (Partial updates allowed) + - `appPageTitle` (String) + - `appPageConfig` (Object) + +### `POST /tenants/:tenantID/appPages/:appPageID/clone` +- **Description:** Clones an existing page. +- **Auth:** Requires `tenant:appPage:clone` permission. + +### `DELETE /tenants/:tenantID/appPages/:appPageID` +- **Description:** Deletes a page permanently. +- **Auth:** Requires `tenant:appPage:delete` permission. +- **Side Effects:** Cascading deletion of related layout records. + +--- + +## Data Queries (`/tenants/:tenantID/queries`) +Manages the definitions and execution of data operations. + +### `GET /tenants/:tenantID/queries` +- **Description:** Lists all saved queries. +- **Auth:** Requires `tenant:query:list` permission. + +### `POST /tenants/:tenantID/queries` +- **Description:** Creates a new query definition. +- **Auth:** Requires `tenant:query:create` permission. +- **Body Schema:** + - `datasourceID` (UUID, required) + - `queryName` (String, required) + - `queryConfig` (Object, required) + +### `GET /tenants/:tenantID/queries/:dataQueryID` +- **Description:** Retrieves a specific query definition. +- **Auth:** Requires `tenant:query:read` permission. + +### `PATCH /tenants/:tenantID/queries/:dataQueryID` +- **Description:** Updates a query definition. +- **Auth:** Requires `tenant:query:update` permission. + +### `POST /tenants/:tenantID/queries/:dataQueryID/run` +- **Description:** **(Critical Execution Endpoint)** Executes a saved query against its target datasource. +- **Auth:** Requires `tenant:query:read` permission. +- **Body Schema:** + - `inputs` (Object, optional) - Dynamic parameters injected into `{{bindings}}`. +- **Side Effects:** The backend connects to the external database/API, executes the command, runs any transformers, and returns the result. + +### `PATCH /tenants/:tenantID/queries/queryTest` +- **Description:** Executes a query payload *without* saving it to the database. Useful for the builder's preview panel. +- **Auth:** Requires `tenant:query:test` permission. + +--- + +## Workflows (`/tenants/:tenantID/workflow`) +Manages background processes and DAG definitions. + +### `GET /tenants/:tenantID/workflow` +- **Description:** Lists workflow definitions. +- **Auth:** Requires `tenant:workflow:list` permission. + +### `POST /tenants/:tenantID/workflow` +- **Description:** Creates a new workflow definition. +- **Auth:** Requires `tenant:workflow:create` permission. +- **Body Schema:** + - `title` (String, required) + - `workflowOptions` (Object, optional) + +### `GET /tenants/:tenantID/workflow/:workflowID` +- **Description:** Retrieves a specific workflow definition, including nodes and edges. +- **Auth:** Requires `tenant:workflow:read` permission. + +### `PATCH /tenants/:tenantID/workflow/:workflowID` +- **Description:** Updates the workflow DAG definition. +- **Auth:** Requires `tenant:workflow:update` permission. + +### `POST /tenants/:tenantID/workflow/:workflowID/execute` +- **Description:** Triggers a saved workflow to run. +- **Auth:** Requires `tenant:workflow:execute` permission. +- **Side Effects:** Creates a `tblWorkflowInstances` record, enqueues pg-boss jobs, and returns an `instanceID` immediately (asynchronous execution). + +### `POST /tenants/:tenantID/workflow/test` +- **Description:** Test-runs a workflow payload in memory without saving the definition. +- **Auth:** Requires `tenant:workflow:execute` permission. +- **Body Schema:** + - `nodes` (Array, required) + - `edges` (Array, required) + - `inputValues` (Object, optional) + +### `GET /tenants/:tenantID/workflow/instances/:instanceID` +- **Description:** Retrieves the real-time status and complete XCom log payload for a specific execution instance. +- **Auth:** Requires `tenant:workflow:read` permission. +- **Response:** Includes `status`, `createdAt`, and `logs` array. + +--- + +## Datasources (`/tenants/:tenantID/datasources`) +Manages integration connections. Credentials are encrypted upon save. + +- `GET /tenants/:tenantID/datasources`: Lists datasources (passwords omitted). +- `POST /tenants/:tenantID/datasources`: Creates a new connection. +- `PATCH /tenants/:tenantID/datasources/:id`: Updates config/credentials. +- `DELETE /tenants/:tenantID/datasources/:id`: Deletes a datasource. +- `POST /tenants/:tenantID/datasources/test`: Tests connection reachability without saving. + +## Listeners (`/tenants/:tenantID/listener`) +Manages incoming data streams and webhook ingestions. + +- `GET /tenants/:tenantID/listener`: Lists active listeners. +- `POST /tenants/:tenantID/listener`: Creates a new listener definition. +- `PATCH /tenants/:tenantID/listener/:id`: Updates listener configuration. +- `DELETE /tenants/:tenantID/listener/:id`: Deletes a listener. + +## Users & Roles (`/tenants/:tenantID/userManagement`, `/tenants/:tenantID/tenantRole`) +Manages RBAC and Tenant membership. + +- `GET /tenants/:tenantID/userManagement/users`: Lists users. +- `PUT /tenants/:tenantID/userManagement/users/:id/role`: Updates user's assigned role. +- `GET /tenants/:tenantID/tenantRole`: Lists available roles. +- `POST /tenants/:tenantID/tenantRole`: Creates a custom role definition. + +## System & Audit (`/system`, `/tenants/:tenantID/audit`) +Manages global settings and logs. + +- `GET /system/health`: Basic health check endpoint. +- `GET /tenants/:tenantID/audit`: Retrieves paginated audit logs. Payloads containing passwords or tokens are automatically masked as `[FILTERED]` by middleware. diff --git a/docs/docs/api-reference/authentication.md b/docs/docs/api-reference/authentication.md index e59d7054..a186081a 100644 --- a/docs/docs/api-reference/authentication.md +++ b/docs/docs/api-reference/authentication.md @@ -1,61 +1,47 @@ --- -sidebar_position: 2 -title: Authentication -description: Authentication and Authorization guide +id: authentication +title: Authentication & Session Management +sidebar_label: Authentication +sidebar_position: 1 +description: How Jet Admin authenticates users and manages sessions. --- -# Authentication +# Authentication & Session Management -The Jet Admin API mainly uses **Firebase Authentication** for user identity and **API Keys** for programmatic access. +Jet Admin relies on industry-standard stateless authentication mechanisms to verify user identity securely while allowing horizontal scaling of the backend API. -## Firebase Authentication (Bearer Token) +## Authentication Model -Most endpoints require a valid Firebase ID token passed in the `Authorization` header. +Jet Admin primarily uses **JSON Web Tokens (JWT)** for authentication. -### Header Format +### Token Issuance Lifecycle +1. **Login Request:** A user submits their credentials (e.g., email and password) to the `/api/v1/auth/login` endpoint. +2. **Verification:** The backend verifies the password hash against the `tblUsers` record. +3. **Token Generation:** Upon success, the backend generates a signed JWT. The token payload typically includes the `userID` and basic claims. +4. **Delivery:** The token is returned to the client. *[VERIFY: Jet Admin may store this in `localStorage`, or in a secure `HttpOnly` cookie depending on environment configuration. Assume standard bearer token logic for APIs.]* +5. **Subsequent Requests:** The client includes the token in the `Authorization: Bearer ` header of every subsequent API request. -```http -Authorization: Bearer -``` +### Session Management & Expiry +- **Statelessness:** Because JWTs are self-contained and cryptographically signed, the backend does not need to look up a session ID in a database or Redis cache for every request. +- **Expiry:** Tokens have a built-in TTL (Time To Live). When a token expires, the client must obtain a new one. +- **Invalidation:** *[VERIFY: To truly revoke JWTs before expiry, Jet Admin might implement a token blocklist or rely on short TTLs combined with refresh tokens.]* -### How to obtain a token +## Onboarding & Invite Flow -1. Sign in using the client SDK (Frontend). -2. Retrieve the ID Token: - ```javascript - const token = await auth().currentUser.getIdToken(); - ``` -3. Include this token in all API requests. +Adding new users to a Jet Admin Tenant follows an invite-based onboarding flow. -### Permissions +1. **Invitation:** An existing Tenant Admin uses the UI to invite a new user via email. +2. **Token Creation:** The backend generates a secure, single-use, time-bound invite token and stores its hash in the database, associating it with the target email and `tenantID`. +3. **Email Delivery:** An email is sent to the user containing a magic link with the invite token. +4. **Registration:** The user clicks the link, bringing them to a registration page. They provide their name and establish a password. +5. **Consumption:** The backend validates the invite token, creates the `tblUsers` record, assigns the default role in `tblTenantUsers`, invalidates the invite token, and issues a standard JWT to log the user in immediately. -The user represented by the token must be a member of the target Tenant. Permissions are enforced by role-based access control (RBAC) within the tenant. +## API Key Authentication (Machine-to-Machine) ---- - -## API Key Authentication - -For server-to-server communication or external integrations, you can use an API Key. - -### Header Format - -```http -x-api-key: -``` - -### Managing API Keys - -You can generate and manage API Keys via the **Tenant Settings > API Keys** section in the Jet Admin dashboard and then use those keys for supported backend endpoints. - -> **Note**: API Keys have specific permissions scopes assigned to them. Ensure your key has the necessary permissions for the endpoints you are calling. - ---- - -## Common Errors +For integrations that require external systems to trigger Jet Admin processes (e.g., triggering a Workflow via webhook, or an external script triggering a Query), User JWTs are inappropriate. -| Code | Status | Description | -|------|--------|-------------| -| `USER_AUTH_TOKEN_NOT_FOUND` | 401 | Missing Authorization header | -| `USER_AUTH_TOKEN_EXPIRED` | 401 | Token has expired | -| `INVALID_API_KEY` | 401 | Invalid or inactive API key | -| `PERMISSION_DENIED` | 403 | User/Key does not have required permissions | +Jet Admin utilizes **API Keys** (`tblAPIKeys`) for this purpose. +- API Keys are generated via the dashboard and assigned specific permissions or roles. +- The raw key is shown only once upon creation; the backend stores a cryptographic hash. +- External systems pass the API Key in a designated header (e.g., `X-Jet-Admin-Api-Key` or standard `Authorization`). +- The backend middleware identifies the API key, looks up the associated tenant and permissions, and authorizes the request. diff --git a/docs/docs/architecture/application-model.md b/docs/docs/architecture/application-model.md new file mode 100644 index 00000000..be8cb489 --- /dev/null +++ b/docs/docs/architecture/application-model.md @@ -0,0 +1,64 @@ +--- +id: application-model +title: Application & Page Model +sidebar_label: Application Model +sidebar_position: 2 +description: Understanding the Application lifecycle, Page routing, and Layout engine. +--- + +# Application & Page Model + +At the highest level of Jet Admin's UI hierarchy are **Applications** and **Pages**. Understanding how these are modeled and rendered is key to understanding the frontend architecture. + +## Application Lifecycle + +An **Application** (or App) serves as a logical container for your tools. + +1. **Creation:** An app is created within a specific Tenant. The creator is granted Owner permissions. +2. **Metadata Storage:** The app metadata (name, description, theme settings) is persisted in the PostgreSQL database. +3. **Configuration:** Developers add Pages, configure local queries, and bind datasources to the app. +4. **Publishing (Versioning):** *[VERIFY: Does Jet Admin support explicit app versioning/publishing flows, or are changes live immediately? Assuming live based on standard SPA behavior unless specified otherwise.]* Changes made in the builder are saved to the database and immediately reflect for users with "Viewer" access reloading the app. +5. **Deletion:** Deleting an app cascades to delete all its associated Pages. + +## Page Model + +A **Page** represents a single routable view within an Application. + +### Schema and Routing +In the database (`tblAppPages`), a page stores: +- `appPageID`: Unique identifier. +- `appPageTitle`: Display name for the navigation sidebar. +- `appPageConfig`: JSON blob containing the widget layout and page-level settings. + +Routing is handled client-side by React Router. The URL structure typically follows: +`/app/:appId/page/:pageId` + +### State Scoping +Jet Admin differentiates between global state and page-level state: +- **Global State:** Information about the authenticated user, current tenant, and global UI theme. Managed in `useAuthStore` and `useUIStore`. +- **Page-Level State (Widget State):** The specific data, selections, and input values of the widgets currently mounted on the page. When a user navigates away from Page A to Page B, the widget state for Page A is unmounted and cleared. This ensures pages do not leak memory or cross-contaminate state. + +### Page Load Lifecycle +When a user navigates to a page: +1. The frontend fetches the `appPageConfig` from the backend. +2. The layout engine mounts the widget components according to the config. +3. Any queries configured to "Run on Page Load" are triggered concurrently. +4. As queries resolve, TanStack Query updates the cache, triggering reactive re-renders of the mounted widgets. + +## Layout Engine + +The Jet Admin canvas uses a grid-based layout engine to render widgets. + +### Grid System +The canvas is subdivided into a grid (typically 12 or 24 columns). When a widget is placed on the canvas, its configuration saves: +- `x`, `y` coordinates (grid cells, not pixels). +- `width`, `height` (span in grid cells). + +### Rendering Pipeline +1. The `appPageConfig` contains an array or tree of widget definitions. +2. The layout engine iterates over these definitions. +3. For each definition, it looks up the React component in the **Widget Registry** (e.g., mapping `type: 'table'` to the `` component). +4. The widget is rendered within a draggable/resizable bounding box (when in Edit mode) or statically positioned (when in View mode). + +### Persistence and Restoration +When a user drags a widget or resizes it, the frontend calculates the new `x/y/w/h` values. These changes are debounced and patched to the backend `appPageConfig` JSON column. On subsequent loads, the engine reads these values to restore the exact layout. diff --git a/docs/docs/architecture/database-schema.md b/docs/docs/architecture/database-schema.md index eee66835..57ea2f10 100644 --- a/docs/docs/architecture/database-schema.md +++ b/docs/docs/architecture/database-schema.md @@ -1,122 +1,261 @@ --- id: database-schema -title: Database Schema -sidebar_label: Database ERD -sidebar_position: 3 -description: Entity Relationship Diagram (ERD) of the Jet Admin PostgreSQL database. +title: Data Model Reference +sidebar_label: Database Schema +sidebar_position: 11 +description: Comprehensive reference of every PostgreSQL operational database table, column, and relationship. --- +# Data Model Reference +Jet Admin uses PostgreSQL for its operational database, managed via Prisma. This database stores configuration, metadata, and state—**it does not store your external business data**. -# Database Schema - -The following diagram represents the core tables and relationships in the Jet Admin database, generated from the Prisma Schema. - -## Entity Relationship Diagram - -```mermaid -erDiagram - %% Core Users & Tenants - tblUsers ||--o{ tblTenants : "manages" - tblUsers ||--o{ tblUsersTenantsRelationship : "has_roles" - tblTenants ||--o{ tblUsersTenantsRelationship : "has_members" - - %% API Keys & Access - tblAPIKeys }o--|| tblTenants : "belongs_to" - tblAPIKeys ||--o{ tblAPIKeyRoleMappings : "has_roles" - tblRoles ||--o{ tblAPIKeyRoleMappings : "assigned_to" - - %% Dashboards & Widgets - tblDashboards }o--|| tblTenants : "belongs_to" - tblDashboards ||--o{ tblWidgets : "contains" - tblWidgets }o--|| tblTenants : "belongs_to" - - %% Data Connectivity - tblDatasources }o--|| tblTenants : "owned_by" - tblDataQueries }o--|| tblDatasources : "queries" - tblDataQueries }o--|| tblTenants : "belongs_to" - - %% Workflow Engine - tblWorkflows }o--|| tblTenants : "belongs_to" - tblWorkflows ||--o{ tblWorkflowNodes : "contains" - tblWorkflows ||--o{ tblWorkflowEdge : "connects" - tblWorkflows ||--o{ tblWorkflowInstances : "instantiates" - - %% Execution Logs - tblWorkflowInstances ||--o{ tblNodeExecutionLogs : "generates" - tblWorkflowNodes ||--o{ tblNodeExecutionLogs : "logs" - - %% Table Definitions - tblUsers { - uuid userID PK - string email - string firebaseID - string firstName - string lastName - } - - tblTenants { - uuid tenantID PK - string tenantTitle - string tenantDBURL - } - - tblWorkflows { - uuid workflowID PK - string title - json workflowOptions - } - - tblWorkflowNodes { - uuid nodeID PK - string nodeType - json nodeConfig - } - - tblWorkflowInstances { - uuid instanceID PK - string status - json contextData - } -``` - -## Core Entities - -### User Management - -| Table | Description | -|:------|:------------| -| `tblUsers` | System users synced with Firebase Authentication | -| `tblTenants` | Workspaces/organizations in the multi-tenant system | -| `tblUsersTenantsRelationship` | Junction table linking users to tenants with roles | -| `tblRoles` | Role definitions (Admin, Editor, Viewer) | -| `tblPermissions` | Granular permissions assigned to roles | - -### Resources (Tenant-Scoped) - -| Table | Description | -|:------|:------------| -| `tblDatasources` | External database connection configurations (encrypted) | -| `tblDataQueries` | Saved SQL/API queries for reuse | -| `tblWorkflows` | Workflow metadata and settings | -| `tblWorkflowVersions` | Versioned workflow graph (nodes/edges as JSON) | -| `tblDashboards` | Dashboard layouts and settings | -| `tblWidgets` | Widget instances with configuration | - -### Runtime & Logging - -| Table | Description | -|:------|:------------| -| `tblWorkflowInstances` | Individual workflow execution runs | -| `tblNodeExecutionLogs` | Per-node execution logs within a run | -| `tblAuditLogs` | Security audit trail for compliance | -| `tblCronJobs` | Scheduled task configurations | - -## Key Design Patterns - -- **Multi-tenancy**: All resource tables have a `tenantID` foreign key for data isolation -- **UUIDs**: Primary keys use `gen_random_uuid()` for distributed ID generation -- **Soft Deletes**: Critical tables support `deletedAt` for recoverable deletion -- **Timestamps**: Standard `createdAt` and `updatedAt` on all tables -- **Encrypted Fields**: Datasource credentials are AES-encrypted at rest +Below is an exhaustive reference of the tables derived from `prisma/schema.prisma`. +--- + +## Multi-Tenancy & Identity + +### `tblTenants` +The root entity establishing isolation boundaries for all resources. +- **Columns:** + - `tenantID` (UUID, Primary Key, Default: uuid_generate_v4()) + - `tenantName` (String, VarChar) + - `createdAt` (DateTime, Timestamptz, Default: now()) + - `updatedAt` (DateTime, Timestamptz, Default: now()) +- **Relationships:** One-to-many with Apps, Pages, Users, Roles, Datasources, Queries, Workflows. + +### `tblUsers` +Users who log into the Jet Admin platform. +- **Columns:** + - `userID` (UUID, Primary Key, Default: uuid_generate_v4()) + - `email` (String, VarChar, Unique) + - `passwordHash` (String, VarChar) + - `firstName` (String, VarChar, Nullable) + - `lastName` (String, VarChar, Nullable) + - `isActive` (Boolean, Default: true) + - `createdAt` / `updatedAt` (DateTime) +- **Indexes:** Unique index on `email`. + +### `tblTenantUsers` +Junction table mapping users to tenants and assigning their RBAC role. +- **Columns:** + - `id` (UUID, Primary Key, Default: uuid_generate_v4()) + - `tenantID` (UUID) - FK to `tblTenants` + - `userID` (UUID) - FK to `tblUsers` + - `roleID` (UUID) - FK to `tblTenantRoles` +- **Indexes:** `idx_tblTenantUsers_tenantID`, `idx_tblTenantUsers_userID`. +- **Relationships:** Links Users, Tenants, and Roles. + +### `tblTenantRoles` +Defines available RBAC roles and their associated permission matrices. +- **Columns:** + - `roleID` (UUID, Primary Key, Default: uuid_generate_v4()) + - `tenantID` (UUID) - FK to `tblTenants` + - `roleName` (String, VarChar) + - `permissions` (JSON, Default: "{}") - Granular matrix of allowed actions. + - `isSystem` (Boolean, Default: false) - Prevents deletion of default roles. +- **Indexes:** `idx_tblTenantRoles_tenantID`. + +### `tblAPIKeys` +Machine-to-machine authentication tokens. +- **Columns:** + - `apiKeyID` (UUID, Primary Key, Default: uuid_generate_v4()) + - `tenantID` (UUID) - FK to `tblTenants` + - `keyHash` (String, VarChar) - Cryptographic hash of the raw token. + - `keyName` (String, VarChar) + - `roleID` (UUID, Nullable) - FK to `tblTenantRoles` + - `createdAt` / `lastUsedAt` (DateTime) +- **Indexes:** `idx_tblAPIKeys_tenantID`. + +--- + +## Application & UI + +### `tblApps` +The top-level container for a project or workspace. +- **Columns:** + - `appID` (UUID, Primary Key, Default: uuid_generate_v4()) + - `tenantID` (UUID) - FK to `tblTenants` + - `appName` (String, VarChar) + - `appDescription` (String, VarChar, Nullable) + - `appOptions` (JSON, Default: "{}") - Global theme, routing, and navigation config. + - `createdAt` / `updatedAt` (DateTime) + +### `tblAppPages` +A specific view within an app, storing the widget layout. +- **Columns:** + - `appPageID` (UUID, Primary Key, Default: uuid_generate_v4()) + - `tenantID` (UUID) - FK to `tblTenants` + - `appPageTitle` (String, VarChar) + - `appPageDescription` (String, VarChar, Nullable) + - `appPageConfig` (JSON, Nullable) - The layout and widget tree definition. + - `creatorID` (UUID, Nullable) - FK to `tblUsers` + - `createdAt` / `updatedAt` (DateTime) + +--- + +## Data Integration + +### `tblDatasources` +Saved configurations for connecting to external systems. +- **Columns:** + - `datasourceID` (UUID, Primary Key, Default: uuid_generate_v4()) + - `tenantID` (UUID) - FK to `tblTenants` + - `datasourceName` (String, VarChar) + - `datasourceType` (String, VarChar) - e.g., "postgresql", "restapi". + - `datasourceOptions` (JSON, Default: "{}") - Non-sensitive connection config (host, port). + - `datasourceCredentials` (String, VarChar, Nullable) - Base64 AES-encrypted secret blob. + - `createdAt` / `updatedAt` (DateTime) + +### `tblQueries` +Executable operations run against datasources. +- **Columns:** + - `queryID` (UUID, Primary Key, Default: uuid_generate_v4()) + - `tenantID` (UUID) - FK to `tblTenants` + - `datasourceID` (UUID) - FK to `tblDatasources` + - `queryName` (String, VarChar) + - `queryConfig` (JSON, Default: "{}") - The query payload, body, or SQL string. + - `transformer` (String, VarChar, Nullable) - JavaScript code for post-processing. + - `runOnPageLoad` (Boolean, Default: false) + - `createdAt` / `updatedAt` (DateTime) + +--- + +## Workflows + +### `tblWorkflows` +Metadata definition of a DAG workflow. +- **Columns:** + - `workflowID` (UUID, Primary Key, Default: uuid_generate_v4()) + - `tenantID` (UUID) - FK to `tblTenants` + - `title` (String, VarChar) + - `workflowOptions` (JSON, Default: "{}") - Input definitions and global workflow settings. + - `creatorID` (UUID, Nullable) - FK to `tblUsers` + - `isDisabled` (Boolean, Nullable) + - `createdAt` / `updatedAt` (DateTime) + +### `tblWorkflowNodes` +Individual steps within a workflow definition. +- **Columns:** + - `nodeID` (UUID, Primary Key, Default: uuid_generate_v4()) + - `workflowID` (UUID) - FK to `tblWorkflows` + - `nodeType` (String, VarChar) - e.g., "start", "dataQuery", "condition". + - `nodeConfig` (JSON, Nullable) - Properties and settings for the specific node. + - `timeoutSeconds` (Int, Default: 300) + - `retryLimit` (Int, Default: 3) + - `createdAt` / `updatedAt` (DateTime) + +### `tblWorkflowEdge` +Connects nodes to form the DAG. +- **Columns:** + - `edgeID` (UUID, Primary Key, Default: uuid_generate_v4()) + - `workflowID` (UUID) - FK to `tblWorkflows` + - `upstreamNodeID` (UUID) - FK to `tblWorkflowNodes` + - `downstreamNodeID` (UUID) - FK to `tblWorkflowNodes` + - `sourceHandle` (String, VarChar, Nullable) - Branch identifier (e.g., 'true', 'false', 'completed'). + +### `tblWorkflowInstances` +A specific execution run of a workflow. +- **Columns:** + - `instanceID` (UUID, Primary Key, Default: uuid_generate_v4()) + - `workflowID` (UUID, Nullable) - FK to `tblWorkflows` + - `tenantID` (UUID) - FK to `tblTenants` + - `status` (String, VarChar, Default: "PENDING") - State machine enum. + - `version` (Int, Default: 0) - Optimistic concurrency lock. + - `isTest` (Boolean, Default: false) + - `startedAt` / `completedAt` (DateTime, Nullable) + +### `tblWorkflowInstanceLogs` +Append-only context log for a running instance. +- **Columns:** + - `logID` (BigInt, Primary Key, Auto-increment) + - `instanceID` (UUID) - FK to `tblWorkflowInstances` + - `nodeID` (String, VarChar, Nullable) + - `eventType` (String, VarChar) - e.g., "NODE_COMPLETED". + - `nodeStatus` (String, VarChar, Nullable) + - `outputVariable` (String, VarChar, Nullable) + - `payload` (JSON, Default: "{}") + - `nodeAttempt` (Int, Nullable) + - `createdAt` (DateTime) +- **Indexes:** `idx_tblWorkflowInstanceLogs_instance_seq` on `[instanceID, logID]`. + +### `tblWorkflowDataCollectionRequests` +State tracking for human-in-the-loop wait nodes. +- **Columns:** + - `collectionRequestID` (UUID, Primary Key) + - `instanceID` (UUID) - FK to `tblWorkflowInstances` + - `nodeID` (String, VarChar) + - `status` (String, VarChar, Default: "PENDING") + - `collectionConfig` (JSON) + - `submittedData` (JSON, Nullable) + - `expiresAt` (DateTime, Nullable) + +--- + +## Event Listeners + +### `tblListeners` +Definitions for incoming data streams (Webhooks). +- **Columns:** + - `listenerID` (UUID, Primary Key) + - `tenantID` (UUID) - FK to `tblTenants` + - `datasourceID` (UUID) - FK to `tblDatasources` + - `listenerType` (String, VarChar) + - `listenerConfig` (JSON) + - `endpointPath` (String, VarChar, Nullable) - Unique URL suffix for webhooks. + - `status` (String, VarChar, Default: "inactive") + +### `tblListenerActions` +Actions triggered by a listener (e.g., run workflow). +- **Columns:** + - `actionID` (UUID, Primary Key) + - `listenerID` (UUID) - FK to `tblListeners` + - `actionType` (String, VarChar) + - `actionConfig` (JSON) + - `orderIndex` (Int, Default: 0) + +### `tblListenerEvents` +Buffer queue for incoming events. +- **Columns:** + - `eventID` (UUID, Primary Key) + - `listenerID` (UUID) - FK to `tblListeners` + - `bufferName` (String, VarChar) + - `eventData` (JSON) - Raw payload. + - `seqNo` (BigInt, Auto-increment) +- **Indexes:** `idx_tblListenerEvents_buffer_seq` (descending on seqNo). + +### `tblEventDLQ` +Dead Letter Queue for failed listener events. +- **Columns:** + - `dlqId` (UUID, Primary Key) + - `rawPayload` (Bytes, Nullable) + - `errorMessage` (String, Nullable) + - `retryCount` (Int, Default: 0) + +--- + +## Automation & Auditing + +### `tblCronJobs` +Scheduled task definitions. +- **Columns:** + - `cronJobID` (UUID, Primary Key) + - `tenantID` (UUID) - FK to `tblTenants` + - `cronExpression` (String, VarChar) + - `workflowID` (UUID) - FK to `tblWorkflows` + - `isEnabled` (Boolean, Default: true) + - `lastRunAt` / `nextRunAt` (DateTime, Nullable) + +### `tblAuditLogs` +Tracks sensitive operations for security and compliance. +- **Columns:** + - `logID` (UUID, Primary Key) + - `tenantID` (UUID) - FK to `tblTenants` + - `userID` (UUID, Nullable) - FK to `tblUsers` + - `action` (String, VarChar) + - `resource` (String, VarChar) + - `resourceID` (String, VarChar, Nullable) + - `payload` (JSON, Nullable) - Filtered payload devoid of sensitive keys. + - `ipAddress` / `userAgent` (String, VarChar, Nullable) + - `createdAt` (DateTime) diff --git a/docs/docs/architecture/datasource-integration.md b/docs/docs/architecture/datasource-integration.md new file mode 100644 index 00000000..03609449 --- /dev/null +++ b/docs/docs/architecture/datasource-integration.md @@ -0,0 +1,68 @@ +--- +id: datasource-integration +title: Datasource & Integration Fabric +sidebar_label: Datasource & Integrations +sidebar_position: 4 +description: How Jet Admin connects to external databases and APIs. +--- + +# Datasource & Integration Fabric + +Jet Admin acts as a unified interface over your existing infrastructure. It does not store your business data; instead, it provides an **Integration Fabric** to securely query databases, APIs, and SaaS applications. + +## Datasource Model + +A **Datasource** is a saved configuration record in the Jet Admin operational database (`tblDatasources`). + +### What a Datasource Represents +At the data model level, a datasource contains: +- `datasourceID`: Unique identifier. +- `tenantID`: The workspace that owns the connection. +- `datasourceType`: A string identifier matching a driver (e.g., `postgresql`, `stripe`, `restapi`). +- `datasourceOptions`: A JSON blob containing the connection parameters (e.g., host, port, database name, URLs). +- `datasourceCredentials`: Encrypted secrets (e.g., passwords, API keys, bearer tokens). + +### Supported Types +Jet Admin supports a wide array of systems out-of-the-box (defined in `@jet-admin/datasource-types` and implemented in `@jet-admin/datasources-logic`), including: +- **SQL Databases:** PostgreSQL, MySQL, MSSQL, Oracle, SQLite, CockroachDB, Supabase, BigQuery. +- **NoSQL / Documents:** MongoDB, Firestore, Neo4j, Elasticsearch, Airtable, Google Sheets. +- **APIs & SaaS:** REST APIs, GraphQL, Stripe, Twilio, SendGrid, Slack, Notion, Jira, Google Analytics. +- **Messaging & Cache:** Kafka, RabbitMQ, Redis. +- **Storage:** AWS S3. +- **Web:** Web URL (Scraping/fetching). + +### Security & Encryption +Credentials are never sent to the frontend. When a user creates a datasource, the backend encrypts the sensitive fields before saving them to PostgreSQL. When a query executes, the backend decrypts the credentials in memory just in time to establish the connection. + +## Integration Fabric Architecture + +The Integration Fabric is the backend system that manages drivers, routing, and connection pooling. + +### The Connector Model +Every datasource type has a corresponding class implementation extending the base `DataSource` interface in `packages/datasources-logic`. + +Connectors define how to: +1. `testConnection()`: Validate credentials and network reachability. +2. `execute()`: Run a specific query or operation. + +### Unified Capabilities Manifests +To allow the frontend to render the correct UI for building queries, each connector exports a manifest (`packages/datasources-logic/src/data-sources/manifests.js`). This manifest describes: +- **Capabilities:** Whether the source supports `read`, `write`, `filter`, or `aggregate`. +- **Query Instructions:** Structural guidelines for what a query payload should look like for this specific driver. + +### Connection Pooling +For relational databases like PostgreSQL, opening a new TCP connection for every query is extremely slow. The Integration Fabric utilizes connection pooling. When a query is executed, the fabric checks if an active pool exists for that specific `datasourceID`. If so, it borrows a connection; if not, it initializes a new pool. + +## Datasource Resolution at Runtime + +When a query is triggered from the frontend, it contains only the `datasourceID`. + +### Execution Pipeline + +1. **Request:** The frontend requests `/api/v1/data-query/execute` with a query ID. +2. **Lookup:** The backend looks up the query to find its associated `datasourceID`. +3. **Retrieval:** The backend fetches the `tblDatasources` record and decrypts the credentials. +4. **Environment Routing:** *[VERIFY: If environment-specific overrides (dev/prod) exist, they are resolved here.]* +5. **Instantiation:** The Integration Fabric uses the `datasourceType` to instantiate the correct driver class (e.g., `PostgreSQLDataSource`). +6. **Execution:** The backend invokes the driver's `execute()` method with the evaluated query body. +7. **Error Handling:** If the connection times out, the credentials fail, or the query syntax is invalid, the driver throws a normalized error that the Express router wraps in a standard API error response. diff --git a/docs/docs/architecture/etl-pipeline.md b/docs/docs/architecture/etl-pipeline.md new file mode 100644 index 00000000..c7e22da2 --- /dev/null +++ b/docs/docs/architecture/etl-pipeline.md @@ -0,0 +1,44 @@ +--- +id: etl-pipeline +title: ETL Pipeline Module +sidebar_label: ETL Pipeline +sidebar_position: 8 +description: Architecture of the Extract-Transform-Load (ETL) pipeline module. +--- + +# ETL Pipeline Module + +While the Workflow Engine is designed for operational logic and state orchestration, Jet Admin includes specialized infrastructure for moving and shaping high-volume data: the **ETL Pipeline Module**. + +*[VERIFY: Note that Jet Admin heavily utilizes Listeners (`tblListeners`) for incoming event ingestion, and Workflows for orchestration. True standalone "ETL Pipelines" distinct from Listeners/Workflows might be integrated differently depending on exact platform configuration. The below describes the conceptual model of data transformation within Jet Admin's backend capabilities.]* + +## ETL Concepts + +An ETL pipeline serves a distinct purpose compared to a Workflow: +- **Workflows:** Best for "If user is created, send email, wait 3 days, send follow-up." +- **ETL Pipelines:** Best for "Extract 10,000 rows from Postgres, map column names to a new schema, and load them into BigQuery." + +Jet Admin handles incoming data streams (like Webhooks or Kafka messages) using the **Listener** module. The data is buffered in `tblListenerEvents` and can be processed sequentially. + +## Architecture + +Data processing in Jet Admin generally follows the Extract → Transform → Load execution model. + +1. **Extract (Listeners & Queries):** Data is extracted from external sources. For streaming or event-based data, Listeners capture events and write them to a buffer table (`tblListenerEvents`). For batch data, queries fetch datasets from databases. +2. **Transform (JavaScript VMs):** Using the isolated `isolated-vm` environment, Jet Admin executes user-defined JavaScript to filter, map, and aggregate the data. +3. **Load (Datasource Integration):** The transformed data is dispatched via the Integration Fabric to a sink (e.g., executing an `insert` query against a data warehouse). + +### Event Buffering Strategy +To handle bursts of incoming data (e.g., a massive influx of webhooks), the Listener module buffers events in `tblListenerEvents`. Background workers process these events at a controlled rate, transforming them and pushing them to their destinations, ensuring the system is not overwhelmed. + +## Pipeline Configuration + +Data pipelines are configured by linking Listeners to target actions. + +### Schema +In the database, a pipeline utilizes: +- `tblListeners`: Defines the source (e.g., listening to a specific Postgres table or Kafka topic). +- `tblListenerActions`: Defines what happens when an event is received (e.g., trigger a workflow, execute a query). + +### Monitoring +Because events are buffered, developers can query `tblListenerEvents` to monitor pipeline runs, view raw payloads, and track processing status or errors. Failed events are routed to a Dead Letter Queue (`tblEventDLQ`) for inspection and retry. diff --git a/docs/docs/architecture/query-engine.md b/docs/docs/architecture/query-engine.md new file mode 100644 index 00000000..47311790 --- /dev/null +++ b/docs/docs/architecture/query-engine.md @@ -0,0 +1,78 @@ +--- +id: query-engine +title: Query Engine +sidebar_label: Query Engine +sidebar_position: 5 +description: How queries are modeled, executed, and cached. +--- + +# Query Engine + +The Query Engine sits between the frontend Presentation Layer and the backend Integration Fabric. It is responsible for safely parameterizing queries, executing them, and handling post-processing transformations. + +## Query Model + +A **Query** is a saved operation in the operational database (`tblQueries`). + +### Query Schema +A query record contains: +- `queryID`: Unique identifier. +- `datasourceID`: Reference to the Datasource this query executes against. +- `queryName`: A human-readable name (e.g., `getUsers`). +- `queryConfig`: A JSON blob containing the actual query payload (e.g., the SQL string or REST API parameters). +- `runOnPageLoad`: Boolean indicating if the query runs automatically when its parent page is opened. +- `transformer`: Optional JavaScript code to post-process the data before it is returned to the client. + +### Parameterization and Bindings +Jet Admin queries rely on `{{bindings}}` to be dynamic. + +For example, a SQL query might look like: +```sql +SELECT * FROM users WHERE status = {{inputs.status}} LIMIT 10; +``` + +**Security:** To prevent injection attacks, Jet Admin does *not* blindly concatenate strings. +When the Query Engine executes, it extracts the `{{inputs.status}}` expression, evaluates it against the provided context, and passes the value to the underlying database driver using **Prepared Statements** (e.g., `$1`). + +### Transformer Functions +Sometimes the data returned by an API or Database is not in the shape required by a widget. Queries can include a **Transformer**, which is a snippet of JavaScript executed on the backend after the query completes but before the data is sent to the frontend. + +```javascript +// Example Transformer +return data.map(row => ({ + fullName: `${row.firstName} ${row.lastName}`, + isActive: row.status === 'active' +})); +``` + +## Query Execution Lifecycle + +The execution of a query is a coordinated dance between the frontend and backend. Below is the chronological lifecycle: + +1. **Trigger:** A query is triggered on the frontend (e.g., via page load or a widget `onClick` event). +2. **Context Gathering:** The frontend gathers all required parameters specified in the query's inputs, pulling values from the local Zustand state (e.g., the value of a Select widget). +3. **HTTP Dispatch:** The frontend sends a POST request to `/api/v1/data-query/execute` with the `queryID` and the resolved `inputs` object. +4. **Auth & RBAC:** The backend authenticates the user and verifies they have read access to the query and its datasource. +5. **Config Retrieval:** The Query Engine fetches the `queryConfig` from PostgreSQL. +6. **Backend Evaluation:** The Query Engine uses the `@jet-admin/expression-engine` to evaluate any `{{bindings}}` located within the `queryConfig`, substituting them with the values from the `inputs` object safely. +7. **Execution:** The sanitized query payload is passed to the Integration Fabric, which executes the request against the external system. +8. **Transformation:** If a transformer function is defined, the raw result is passed into an isolated JavaScript VM, transformed, and returned. +9. **Response:** The final JSON result is sent back to the frontend over HTTP. + +## Caching & Invalidation + +To ensure high performance and prevent unnecessary database load, Jet Admin relies heavily on client-side caching. + +### TanStack Query +The frontend utilizes **TanStack Query** (formerly React Query) to manage query state. + +- **Cache Keys:** Query results are cached using a composite key: `['query', queryID, stringifiedInputs]`. This ensures that changing a parameter (like paginating to page 2) results in a distinct cache entry, while returning to page 1 instantly loads the cached data. +- **Loading State:** TanStack Query automatically provides `.isLoading` and `.isFetching` booleans, which the Zustand store exposes to widgets (e.g., `{{queries.getUsers.isLoading}}` can be bound to a Button's "loading" state). + +### Invalidation +Data becomes stale when a user performs a mutation (e.g., updating a user record). Jet Admin allows developers to configure "On Success" actions for mutation queries to invalidate cache keys. + +When a query is invalidated: +1. TanStack Query marks the cached data as stale. +2. It immediately triggers a background refetch for any active queries using that key. +3. Once the fresh data arrives, Zustand state updates, and widgets re-render seamlessly. diff --git a/docs/docs/architecture/rbac.md b/docs/docs/architecture/rbac.md new file mode 100644 index 00000000..b5c0663b --- /dev/null +++ b/docs/docs/architecture/rbac.md @@ -0,0 +1,56 @@ +--- +id: rbac +title: Role-Based Access Control (RBAC) +sidebar_label: RBAC +sidebar_position: 9 +description: How permissions and data visibility are gated within Jet Admin. +--- + +# Role-Based Access Control (RBAC) + +Security is paramount in internal tools. Jet Admin utilizes a robust Role-Based Access Control (RBAC) model to ensure users can only see and execute what they are explicitly authorized to. + +## RBAC Model + +The RBAC system governs access to resources based on a user's assigned role within a specific Tenant context. + +### Entities +Access control applies to the following entities: +- **Tenant:** The top-level workspace or organization. +- **App:** The logical container of tools. +- **Page:** Specific views within an app. +- **Datasource:** The connections to external systems. +- **Query:** The specific executable commands against datasources. +- **Workflow:** Automated backend processes. + +### Role Types +By default, Jet Admin implements standard roles, but the system is designed to allow custom role definitions via the `tblTenantRoles` table. Common base roles include: +- **Owner / Admin:** Full read/write access to all resources within the tenant. Can manage users, billing, and global settings. +- **Editor / Developer:** Can create and edit Apps, Pages, Queries, and Workflows. Usually cannot manage billing or tenant settings. +- **Viewer / User:** Read-only access to published Apps. Can view pages and execute authorized queries (e.g., clicking a button to run a query), but cannot edit the query definitions or app layouts. + +### Storage +Roles and permissions are stored relationally in the operational PostgreSQL database: +- `tblTenantRoles`: Defines the available roles for a tenant. +- `tblTenantUsers`: Maps a `userID` to a `roleID` within a specific `tenantID`. + +## Permission Enforcement + +Enforcement happens securely on the backend, with the frontend acting as an immediate, optimistic gatekeeper for UX purposes. + +### Server-Side Enforcement (Middleware) +This is the true source of security. Every request to the `/api/v1/*` Express routers passes through authentication and authorization middleware. + +1. **Authentication:** Validates the incoming JWT or session cookie to identify the user. +2. **Context Resolution:** Determines the `tenantID` the user is attempting to operate within. +3. **Role Check:** The middleware (`apps/backend/modules/tenantRole/`) checks if the user's role grants permission for the specific HTTP method and resource path. +4. **Datasource Validation:** When a query execution is requested, the engine explicitly checks if the user's role permits executing queries against that specific datasource. + +### Client-Side Enforcement +The frontend reads the user's permissions from the global Zustand store (`useAuthStore`). Based on these permissions, it conditionally renders UI elements. +- **Hide Builder:** If a user is a Viewer, the drag-and-drop builder interface, property panels, and code editors are completely hidden. +- **Disable Widgets:** Specific widgets can be configured to be disabled or hidden based on the user's role, preventing them from attempting unauthorized actions. + +### Data Visibility & Auditing +- **Filtering:** Prisma queries in the backend are inherently scoped by `tenantID`. Users cannot query metadata for apps or datasources belonging to other tenants. +- **Auditing:** Sensitive actions (like modifying a datasource or deleting a workflow) are logged by the Audit module (`apps/backend/modules/audit`), recording *who* did *what* and *when*. The audit middleware automatically masks sensitive keys (like passwords or tokens) before logging payloads. diff --git a/docs/docs/architecture/realtime.md b/docs/docs/architecture/realtime.md new file mode 100644 index 00000000..1cb38246 --- /dev/null +++ b/docs/docs/architecture/realtime.md @@ -0,0 +1,53 @@ +--- +id: realtime +title: Real-Time & Collaboration +sidebar_label: Real-Time Architecture +sidebar_position: 10 +description: How Socket.IO powers real-time updates and collaboration. +--- + +# Real-Time & Collaboration + +Jet Admin is designed as a real-time, event-driven platform. Rather than forcing the client to continuously poll the server for updates (like checking the status of a long-running workflow), Jet Admin utilizes WebSockets via **Socket.IO**. + +## Socket.IO Architecture + +Socket.IO provides a bidirectional, low-latency communication channel between the React frontend and the Express backend. + +### Initialization & Authentication +1. When the frontend application boots, it initializes a Socket.IO client connection. +2. The initial handshake includes authentication tokens (e.g., JWT) to verify the user. +3. Once authenticated, the backend registers the socket connection. + +### Namespaces and Rooms +To ensure messages are only sent to the relevant clients, Jet Admin heavily utilizes Socket.IO's "Rooms" concept. +- **Tenant Rooms:** Clients join a room specific to their `tenantID`. Global tenant updates (like role changes) are broadcast here. +- **App/Page Rooms:** When a user opens an App or Page, their socket joins a specific room (e.g., `room:app:123:page:456`). Edits to the page layout are broadcast to this room. +- **Workflow Instance Rooms:** When a workflow is triggered, the orchestrator creates a room for that specific instance (`room:workflow_instance:789`). The frontend joins this room to listen for node-by-node execution progress. + +## Events & Payloads + +Communication happens via strongly typed events. + +### Server-to-Client Events (Emits) +The backend pushes data to the frontend for various reasons: +- `WORKFLOW_NODE_UPDATE`: Emitted by the Workflow Orchestrator as each node finishes. Payload includes `nodeID`, `status`, and the latest `contextData`. +- `WORKFLOW_STATUS_UPDATE`: Emitted when the entire instance completes or fails. +- `WIDGET_DATA_UPDATE`: Emitted if a backend listener (e.g., a Webhook or SSE source) receives new data meant for a live widget. + +### Client-to-Client / State Mutations +When the frontend receives a socket event, the `useSocketStore` or specific bridge controllers (like `widgetWorkflowBridge`) intercept it. +1. The event payload is parsed. +2. The corresponding Zustand state slice is updated (e.g., updating a workflow progress bar). +3. React reactively re-renders the affected UI components instantly. + +## Collaboration & Optimistic Updates + +Because Jet Admin is a builder platform, multiple developers might edit an App simultaneously. + +### Presence & Conflict +*[VERIFY: Standard implementation assumes last-write-wins unless a specific CRDT or locking mechanism is implemented.]* +Layout and configuration edits made by one user are typically debounced and sent via HTTP PUT/PATCH requests. Upon successful save, the backend emits a socket event to the Page room. Other connected clients receive the event and their Zustand store updates to reflect the new widget position or property. + +### Optimistic UI +When a user drags a widget, the frontend updates the UI *optimistically* (immediately) to provide a snappy UX. If the underlying HTTP save request fails, the frontend rolls back the widget to its previous state. diff --git a/docs/docs/architecture/system-overview.md b/docs/docs/architecture/system-overview.md index 126f6cd3..78e16c9e 100644 --- a/docs/docs/architecture/system-overview.md +++ b/docs/docs/architecture/system-overview.md @@ -1,77 +1,111 @@ --- id: system-overview -title: System Overview (C4 Model) +title: System Architecture Overview sidebar_label: System Overview sidebar_position: 1 -description: High-level C4 model architecture diagrams for Jet Admin. +description: High-level system architecture, decoupled design, and request lifecycles. --- # System Architecture Overview -This document provides a high-level overview of the Jet Admin platform architecture using the **C4 Model** (Context, Containers, Components, Code). +Jet Admin follows a modern, layered, decoupled architecture designed to cleanly separate presentation logic, orchestration, and data persistence. -## System Context (C4 Level 1) +## High-Level System Diagram -The System Context diagram shows Jet Admin in the center of its environment, interacting with users and external systems. +Below is the high-level flow of data and requests through the Jet Admin platform. ```mermaid -C4Context - title System Context Diagram for Jet Admin - - Person(admin, "Platform Admin", "Configures datasources, users, and roles") - Person(user, "End User", "Uses widgets, runs workflows, views data") - - System(jetadmin, "Jet Admin Platform", "Internal tools platform for multi-tenant data management and workflows") - - System_Ext(postgres, "Customer PostgreSQL", "External databases connected by tenants") - System_Ext(restapi, "External REST APIs", "Third-party APIs connected by tenants") - System_Ext(firebase, "Firebase Auth", "Authentication provider") - - Rel(admin, jetadmin, "Configures and manages") - Rel(user, jetadmin, "Interacts with internal tools") - - Rel(jetadmin, firebase, "Authenticates users via") - Rel(jetadmin, postgres, "Queries and mutates data via") - Rel(jetadmin, restapi, "Integrates with via") +flowchart TD + subgraph Client [Browser Client (React SPA)] + UI[Page & Widget Canvas] + State[Zustand Local State] + QueryCache[TanStack Query Cache] + SocketClient[Socket.IO Client] + UI --> State + State --> QueryCache + end + + subgraph Server [Node.js Express API] + Router[API Routers & Controllers] + QueryEngine[Query Engine] + WorkflowOrchestrator[Workflow Orchestrator] + IntegrationFabric[Integration Fabric / Drivers] + SocketServer[Socket.IO Server] + JobQueue[pg-boss Queue] + end + + subgraph Database [Operational DB] + PostgreSQL[(PostgreSQL / Prisma)] + end + + subgraph External [External Systems] + ExtDB[(External Databases)] + ExtAPI[REST / GraphQL APIs] + end + + %% HTTP Paths + QueryCache -- "HTTP GET/POST" --> Router + Router -- "Resolves Queries" --> QueryEngine + QueryEngine -- "Uses" --> IntegrationFabric + IntegrationFabric -- "Executes against" --> ExtDB + IntegrationFabric -- "Fetches from" --> ExtAPI + + %% Persistence + Router -- "Reads/Writes Meta" --> PostgreSQL + QueryEngine -- "Fetches Config" --> PostgreSQL + + %% Async & WebSockets + SocketClient <.. "Real-time updates" ..> SocketServer + Router -- "Enqueues Jobs" --> JobQueue + JobQueue -- "Processed by" --> WorkflowOrchestrator + WorkflowOrchestrator -- "Broadcasts progress" --> SocketServer ``` -## Container Architecture (C4 Level 2) +## Layered Architecture -The Container diagram zooms into the Jet Admin system to show the high-level technical containers that make up the system. +Jet Admin is structured into distinct layers to separate concerns: -```mermaid -C4Container - title Container Diagram for Jet Admin - - Person(user, "User", "Admin or End User") - - System_Boundary(jetadmin, "Jet Admin Platform") { - Container(frontend, "Frontend SPA", "React, Vite, MUI", "Provides the UI for dashboards, workflows, and queries") - Container(api, "Backend API", "Node.js, Express", "Handles business logic, auth, and API routing") - Container(queue, "Workflow Queue", "fastq (In-Memory)", "Manages workflow task execution") - ContainerDb(db, "Platform Database", "PostgreSQL", "Stores tenants, users, workflows, widgets, and logs") - } - - System_Ext(firebase, "Firebase Auth", "Authentication") - System_Ext(externalData, "External Data Sources", "PostgreSQL / REST APIs") - - Rel(user, frontend, "Visits", "HTTPS") - Rel(frontend, firebase, "Authenticates", "HTTPS") - Rel(frontend, api, "Makes API calls", "JSON/HTTPS") - Rel(frontend, api, "Real-time updates", "Socket.IO") - - Rel(api, firebase, "Verifies Tokens", "HTTPS") - Rel(api, db, "Reads/Writes", "Prisma/TCP") - Rel(api, queue, "Enqueues tasks", "In-Process") - Rel(queue, api, "Executes tasks", "In-Process") - - Rel(api, externalData, "Queries data", "TCP/HTTPS") - Rel(queue, externalData, "Queries data during workflows", "TCP/HTTPS") -``` +1. **Presentation Layer (Frontend):** React components, drag-and-drop Page Builder, and Widget configurations. This layer captures user intent and renders data. +2. **State Layer (Frontend):** + - *Client State:* Managed by **Zustand**, tracking widget property changes, active selections, and `{{binding}}` evaluation context. + - *Server State:* Managed by **TanStack Query**, caching API responses, handling loading states, and deduplicating network requests. +3. **Server Layer (Backend):** Node.js/Express application. Contains business logic modules (Auth, App/Page management), the **Workflow Orchestrator**, and the **Query Engine**. +4. **Persistence Layer (Backend):** The core operational **PostgreSQL** database accessed via **Prisma ORM**. Stores metadata—users, RBAC roles, tenant isolation borders, saved queries, workflow definitions, and page layouts. It does *not* store the external data you query. +5. **External Integration Layer (Backend):** The **Integration Fabric**. A collection of datasource drivers (e.g., PostgreSQL driver, Stripe REST driver) that establish connections, pool resources, and execute operations securely on behalf of the user. + +## Decoupled Design + +The frontend and backend are completely decoupled. The frontend SPA does not know how to connect to a MySQL database or how to execute a Stripe API call. Instead: + +- The frontend sends a structured request to the backend: *"Execute Query ID 123 with parameter `limit=10`."* +- The backend checks permissions (RBAC), resolves the datasource credentials for Query 123 from the operational database, evaluates any backend `{{bindings}}`, routes the operation to the correct Integration Fabric driver, executes it, and returns the raw JSON result. +- The frontend receives the result, stores it in TanStack Query, updates the evaluation context, and reactively re-renders any widgets bound to that query's data. + +## Request Lifecycle: Widget to Data + +Understanding the path of a user interaction helps clarify how Jet Admin works. Here is the chronological sequence when a user clicks a button to execute a query: + +1. **User Interaction:** The user clicks a Button widget. The button's `onClick` event is triggered. +2. **Context Evaluation:** The frontend evaluates the action configured for `onClick`. It resolves any `{{bindings}}` using the current Zustand context (e.g., pulling a selected row ID from a Table widget). +3. **HTTP Dispatch:** The frontend dispatches an HTTP POST request to the backend `/api/v1/data-query/execute` endpoint. +4. **Authentication & RBAC:** The Express router authenticates the JWT and verifies the user has permissions to execute this specific query in this workspace. +5. **Query Resolution:** The Query Engine retrieves the saved query definition and its associated datasource configuration from the operational PostgreSQL database. +6. **Execution:** The Integration Fabric instantiates the specific driver (e.g., PostgreSQL), injects the sanitized parameters safely (using parameterized queries to prevent SQL injection), and executes the query against the external system. +7. **Result Return:** The raw result is returned to the Express server, which sends it back to the client as a JSON HTTP response. +8. **Cache & Reactivity:** TanStack Query caches the result under a specific key. The Zustand state slice for that query updates its `.data` and `.isLoading` flags. +9. **Re-render:** Any widgets (like a Table or Chart) whose properties are bound to `{{queries.myQuery.data}}` reactively re-render to display the new information. + +## Real-Time Architecture + +Jet Admin relies heavily on **Socket.IO** to provide a real-time, collaborative experience: + +- **Namespaces & Rooms:** Clients connect to specific socket rooms based on their Tenant ID and current App/Page. +- **Workflow Progress:** Because workflows are executed asynchronously by `pg-boss` background workers, HTTP requests cannot wait for completion. Instead, the Workflow Orchestrator emits progress events via WebSockets. The frontend listens to these events to show real-time progress bars or notifications. +- **Data Push:** For supported datasources (like SSE or incoming Webhooks via the Listener module), data is pushed directly to the client over the active WebSocket connection, instantly updating bound widgets without polling. + +## Multi-Tenancy Isolation boundaries -## Next Steps +Jet Admin supports multi-tenancy natively. All data in the operational database (apps, pages, queries, datasources) is strictly partitioned by a `tenantID`. -For detailed **Component Architecture (C4 Level 3)**, please refer to the specific module architectures: -- [Backend Architecture](./backend-architecture.md) -- [Frontend Architecture](./frontend-architecture.md) -- [Database Schema](./database-schema.md) +- **Tenant:** The highest level of isolation. Workspaces belong to a tenant. +- **Enforcement:** Every API route utilizes middleware to ensure the authenticated user belongs to the requested `tenantID`. Prisma queries implicitly filter by `tenantID` to prevent cross-tenant data leakage. diff --git a/docs/docs/architecture/template-engine.md b/docs/docs/architecture/template-engine.md new file mode 100644 index 00000000..6d74f06f --- /dev/null +++ b/docs/docs/architecture/template-engine.md @@ -0,0 +1,66 @@ +--- +id: template-engine +title: Template Engine & Bindings +sidebar_label: Template Engine +sidebar_position: 6 +description: How dynamic expressions are evaluated across Jet Admin. +--- + +# Template Engine & Bindings + +Jet Admin's power comes from its ability to link data and UI reactively. This is accomplished using **Template Expressions**—snippets of JavaScript wrapped in double curly braces (`{{ }}`) that are evaluated at runtime. + +The underlying system powering this is the `@jet-admin/expression-engine` package. + +## Expression Syntax + +A template expression allows you to write JavaScript that evaluates against the current application context. + +### Anatomy of an Expression +Everything inside the `{{ }}` markers is evaluated. + +```javascript +// Valid expressions: +{{ widgets.Table1.selectedRow.id }} // Property access +{{ queries.getUsers.data.length > 0 ? 'Yes' : 'No' }} // Ternary logic +{{ moment(widgets.DatePicker1.value).format('LL') }} // Function calls +``` + +### Evaluation Modes +The Expression Engine operates in different security modes depending on where it is running: + +1. **`js-template` (Frontend UI):** Evaluates expressions in the browser using a sandboxed `new Function()`. It has access to the global `state` and `event` objects, as well as utility libraries like `moment` and `_` (lodash). +2. **`safe-path` (Backend Queries):** The most restrictive mode. Used to parse query parameters (e.g., `{{inputs.status}}`). It only allows strict object path traversal. It forbids arithmetic, function calls, or logical operators to guarantee safety against injection attacks. +3. **`isolated-js` (Backend Workflows/Transformers):** Used to execute arbitrary JavaScript (like data transformers or workflow JS nodes). This runs securely on the Node.js backend using the `isolated-vm` native C++ addon, ensuring user-provided code cannot access the Node.js environment, file system, or memory space of the main process. + +## Evaluation Context + +When an expression is evaluated, it is provided a "context" object. In the frontend builder, this context contains: + +- `widgets`: The state of all widgets on the current page (e.g., `widgets.Table1.selectedRow`). +- `queries`: The state and data of all queries (e.g., `queries.getUsers.data`, `queries.getUsers.isLoading`). +- `inputs`: URL parameters or form inputs (e.g., `inputs.pageId`). +- `jetadmin`: Global app-level helpers and metadata. +- **Utilities:** Global libraries like `moment` (date parsing), `_` (lodash utilities), and standard Math functions. + +## Reactivity & Re-evaluation + +Jet Admin ensures the UI stays up-to-date instantly. + +### The Dependency Graph +The frontend maintains a reactive dependency graph. When a property is configured with a binding (e.g., a Text widget displaying `{{widgets.Input1.text}}`), the Expression Engine extracts the dependencies (`widgets.Input1.text`). + +### Re-evaluation Trigger +When the user types in `Input1`, the Zustand store updates. The frontend observes this update, checks the dependency graph, identifies that the Text widget depends on this changed value, and re-evaluates the expression. The new value is passed to the Text widget, triggering a React re-render. + +## Autocomplete & IntelliSense + +Writing expressions is supported by a robust IntelliSense system built into the Expression Engine. + +### Context Awareness +When a user types `{{` in a property panel, the `TemplateAutocompleteInput` component queries the Expression Engine. The engine introspects the current live schema of the Zustand store and provides autocomplete suggestions. + +If the user types `{{widgets.T`, the engine suggests `widgets.Table1`. + +### Dual-Mode UX +The autocomplete system supports both single-line inputs (for simple properties like a text label) and multi-line textareas (for writing longer JSON configurations or transformer functions). diff --git a/docs/docs/architecture/widget-system.md b/docs/docs/architecture/widget-system.md new file mode 100644 index 00000000..64233e52 --- /dev/null +++ b/docs/docs/architecture/widget-system.md @@ -0,0 +1,121 @@ +--- +id: widget-system +title: Widget System +sidebar_label: Widget System +sidebar_position: 3 +description: Architecture of the UI components, reactivity, state model, and available widgets. +--- + +# Widget System + +Widgets are the atomic UI building blocks of Jet Admin. They are React components that have been wrapped to participate in Jet Admin's layout engine, state management, and event system. + +## Widget Architecture + +### Widget Definitions +Every widget is defined by a schema that describes its capabilities to the platform. A widget definition (found in `@jet-admin/widget-types`) includes: +- **Type Identifier:** e.g., `table`, `button`, `date-picker`. +- **Default Properties:** The initial state when dragged onto the canvas. +- **Events:** The interactions it supports (e.g., `onClick`, `onRowSelect`). +- **Methods:** Callable actions (e.g., `refresh()`, `clearSelection()`). +- **Property Panel Config:** A JSON Schema defining the form used to configure the widget in the builder. + +### The Widget Registry +When the frontend application boots, it loads the Widget Registry. This registry maps a string type (`"table"`) to its corresponding React component (``) and its metadata. + +### Rendering Pipeline +When a Page loads: +1. The Layout Engine reads the array of widget definitions from the `appPageConfig`. +2. For each widget, it finds the matching React component in the Registry. +3. The component is wrapped in a generic `` that handles standard behaviors: + - Drag-and-drop handles. + - Resizing logic. + - Evaluating `{{bindings}}` in properties. + - Injecting evaluated properties into the underlying React component as props. + +## Widget State Model + +Widgets must share data with the rest of the application. For example, a Query needs to read the selected text from an Input widget. + +### Zustand Store +Widget state is stored in a normalized slice of the global Zustand store. The store maintains a flat object keyed by the unique Widget ID (assigned when the widget is placed on the canvas). + +```json +{ + "widgets": { + "TableWidget1": { + "selectedRow": { "id": 42, "name": "Alice" }, + "page": 1 + }, + "SearchInput1": { + "text": "Alice" + } + } +} +``` + +### Inter-Widget References +Because the state is centralized, widgets can reference each other using Template Expressions. If a Text widget's text property is set to `{{widgets.SearchInput1.text}}`, the Template Engine evaluates this against the global context. + +### Reactivity +Jet Admin uses a reactive dependency graph. When a user types in `SearchInput1`: +1. The widget calls its internal `onChange` handler. +2. The handler dispatches an action to update `widgets.SearchInput1.text` in the Zustand store. +3. Zustand notifies subscribed listeners. +4. The Template Engine detects that the Text widget depends on this value, re-evaluates the expression, and passes the new value to the Text widget, causing a re-render. + +## Widget Property Panel + +The Property Panel on the right side of the builder allows users to configure a widget. + +### Schema-Driven Generation +The UI of the property panel is generated automatically based on the widget's schema. This means adding a new configurable property to a widget does not require writing new UI code; you simply update the JSON Schema for that widget type. + +### Property Types +Properties fall into several categories: +- **Static Values:** Hardcoded strings, numbers, or booleans. +- **Bindings:** Strings containing `{{expressions}}` that must be evaluated dynamically. +- **Event Handlers:** Configurations dictating what happens when a widget event fires (e.g., "When `onClick` fires, Execute Query `getUsers`"). + +### Live Preview +Changes made in the Property Panel patch the widget's configuration in the Zustand store immediately. Because the widget on the canvas is subscribed to this configuration, it updates in real-time without requiring a full page reload. + +--- + +## Built-in Widget Catalog + +Jet Admin includes a rich set of built-in widgets. Below is a summary of the core components. + +*(Note: Exact property names and events depend on the widget schemas defined in `packages/widget-types/src/index.js`)* + +### Table (`table`) +- **Purpose:** Displaying arrays of objects in a tabular format. Supports pagination, sorting, and inline editing. +- **Key Events:** `onRowSelect`, `onPageChange`, `onSearch`, `onRowSave`, `onBulkDelete`. +- **Callable Methods:** `refresh`, `setSelectedRow`, `clearSelection`. + +### Button (`button`) +- **Purpose:** Triggering actions. +- **Key Events:** `onSubmit`, `onClick`. +- **Callable Methods:** `click`. + +### Form (`form`) +- **Purpose:** Collecting structured user input. Groups inputs and provides a unified submit event containing all field values. +- **Key Events:** `onSubmit` (emits the `formData` object), `onFieldChange`. + +### Chart (`vega` / `vega-lite`) +- **Purpose:** Declarative data visualization using the Vega-Lite grammar. +- **Key Properties:** Vega specification JSON. +- **Callable Methods:** `refresh`, `resize`. + +### Date Picker (`date-picker`) & Date Range (`date-range-picker`) +- **Purpose:** Selecting dates, times, or date ranges. +- **Key Events:** `onChange` (emits ISO date strings), `onClear`. + +### HTML (`html`) +- **Purpose:** Rendering custom HTML or embedding external content via an iframe. +- **Key Events:** `onMessage` (listens for `postMessage` events dispatched from inside the iframe). +- **Callable Methods:** `refresh`. + +### Alert (`alert`) +- **Purpose:** Displaying warning, success, or informational banners. +- **Key Events:** `onDismiss`. diff --git a/docs/docs/architecture/workflow-engine.md b/docs/docs/architecture/workflow-engine.md new file mode 100644 index 00000000..9c8cd53a --- /dev/null +++ b/docs/docs/architecture/workflow-engine.md @@ -0,0 +1,79 @@ +--- +id: workflow-engine +title: Workflow Engine +sidebar_label: Workflow Engine +sidebar_position: 7 +description: Architecture of the DAG-based workflow orchestrator and executor. +--- + +# Workflow Engine + +The Workflow Engine is a robust, distributed system for executing multi-step business logic asynchronously. While Queries are synchronous and meant for immediate data fetching, Workflows are asynchronous, long-running Directed Acyclic Graphs (DAGs). + +## Workflow Concepts + +- **DAG Structure:** Workflows are composed of **Nodes** connected by directed **Edges**. The engine ensures there are no infinite cycles. +- **Triggers:** Workflows can be initiated manually via UI, scheduled via cron jobs, or triggered by incoming webhooks/events. +- **Context Log:** Workflows do not share a mutable global state. Instead, they use an XCom-style append-only context log. As each node finishes, its output is appended to the context. Downstream nodes can reference upstream outputs via bindings like `{{context.steps.queryData.data}}`. + +## Data Model + +Workflows consist of several relational entities in the database: +- `tblWorkflows`: The definition metadata (title, tenant ID, creator). +- `tblWorkflowNodes`: The individual steps (type, config schema, retry limits). +- `tblWorkflowInstances`: A specific execution run of a workflow. Tracks status (`RUNNING`, `COMPLETED`, `FAILED`). +- `tblWorkflowInstanceLogs`: The append-only context log. Stores the output payload of every executed node for a given instance. + +## Execution Architecture + +The Workflow Engine is split into two primary components to allow horizontal scaling: the **Orchestrator** and the **Executor (Workers)**. + +### The Orchestrator (`workflowEngine/engine.js`) +The Orchestrator's job is scheduling. +1. It analyzes the DAG. +2. It determines which nodes are ready to run (i.e., all their upstream dependencies have successfully completed). +3. It packages the node configuration and the current instance context into a job payload. +4. It enqueues the job into **pg-boss** (the PostgreSQL-backed job queue). + +### The Executor Workers (`workers/taskWorker.js`) +The Workers handle the actual execution. +1. A worker claims a job from pg-boss. +2. It evaluates any `{{bindings}}` in the node's configuration using the provided context. +3. It hands the configuration to the specific Node Handler (e.g., `dataQueryHandler`, `javascriptHandler`). +4. The Handler executes the logic (running a DB query, executing JS in isolated-vm). +5. The result is returned to the Orchestrator via a dedicated results queue. + +## Execution Lifecycle + +Here is the chronological order of a workflow execution: + +1. **Trigger:** A workflow is triggered (e.g., via a Webhook). +2. **Instance Creation:** A new `tblWorkflowInstances` record is created with status `PENDING`. +3. **Start Resolution:** The Orchestrator resolves the `Start` node and enqueues a job for it. +4. **Worker Execution:** A Worker picks up the job, executes the Start node handler, and pushes the result back to the Orchestrator. +5. **Context Appended:** The Orchestrator appends the node's output to `tblWorkflowInstanceLogs`. +6. **DAG Re-evaluation:** The Orchestrator checks the DAG to see which downstream nodes are now unblocked. +7. **Next Jobs Enqueued:** It enqueues jobs for the newly unblocked nodes in pg-boss. +8. **Loop:** Steps 4-7 repeat concurrently for all branches of the DAG. +9. **Terminal State:** Once terminal nodes (`EndNode`) complete, or if an unhandled error occurs, the instance status updates to `COMPLETED` or `FAILED`. +10. **Socket Event:** A Socket.IO event is emitted to connected clients indicating the final status and context. + +## Node Type Reference + +The engine supports various node types out-of-the-box: + +- **Start Node:** The entry point. Handles incoming trigger payloads. +- **Data Query Node:** Executes a saved Query against a Datasource. Appends the data result to the context. +- **JavaScript Node:** Executes custom JS in the secure `isolated-vm` environment to transform data. +- **Condition Node:** Evaluates a boolean expression (e.g., `{{context.steps.query1.total > 10}}`). Branches execution down true or false edges. +- **Loop Node:** Iterates over an array in the context, executing a sub-graph of nodes for each item. +- **Delay Node:** Pauses execution for a set duration. +- **Data Collection Node:** A specialized node that pauses workflow execution entirely. It issues a secure token and waits for human input (e.g., via an email form or API callback). Once data is submitted, the Orchestrator resumes the workflow. +- **End Node:** Explicitly terminates a branch or the entire workflow. + +## Error Handling & Retry + +Robust error handling is critical for distributed systems. +- **Retry Policy:** Nodes can be configured with a retry limit (`retryLimit` in `tblWorkflowNodes`) and backoff strategy. If a REST API query node fails due to network timeout, the Executor will retry it automatically. +- **Failure Propagation:** If a node exhausts its retries, it is marked as `FAILED`. Downstream nodes dependent on it will never run. Depending on DAG configuration, the entire workflow instance may be marked `FAILED`. +- **Debugging:** Because the context log is persisted (`tblWorkflowInstanceLogs`), developers can inspect exactly what data entered and exited a node at the time of failure using the Jet Admin UI. diff --git a/docs/docs/concepts/core-glossary.md b/docs/docs/concepts/core-glossary.md new file mode 100644 index 00000000..abec6b97 --- /dev/null +++ b/docs/docs/concepts/core-glossary.md @@ -0,0 +1,88 @@ +--- +id: core-glossary +title: Core Concepts Glossary +sidebar_label: Core Glossary +sidebar_position: 1 +description: Definitions of the foundational concepts within Jet Admin. +--- + +# Core Concepts Glossary + +To understand Jet Admin, you must understand the primitive building blocks that make up the platform. This glossary defines each concept precisely, clarifying what it is, what it is not, and how it relates to adjacent concepts. + +--- + +### Application / App +**What it is:** A logical container of pages, datasources, queries, and workflows scoped to a specific team or project. It is the top-level entity that users interact with. +**What it is NOT:** An app is not a single web page. It is a collection of resources. +**Relation:** An app contains **Pages**, executes **Queries** against **Datasources**, and triggers **Workflows**. + +--- + +### Page +**What it is:** A visual canvas of widgets, serving as a distinct view within an Application. A page handles routing, layout configuration, and scopes local state. +**What it is NOT:** A page is not a global store; its widget state is destroyed and re-initialized when navigating away and back. +**Relation:** A page lives inside an **App** and contains **Widgets**. Loading a page often triggers on-load **Queries**. + +--- + +### Widget +**What it is:** An atomic UI unit (e.g., Table, Button, Chart, Text Input) placed on a Page. Widgets have configurable *properties*, emit *events*, and can be *bound* to data. +**What it is NOT:** A widget does not fetch data itself. It only displays data provided to it via bindings. +**Relation:** Widgets live on a **Page**, display data from **Queries**, and their events (like `onClick`) can trigger new queries or **Workflows**. + +--- + +### Datasource +**What it is:** A saved configuration defining a connection to an external system. This could be a database (PostgreSQL, MySQL), a REST API, a SaaS application (Stripe, Slack), or a message broker (Kafka). It stores credentials securely. +**What it is NOT:** A datasource is not the data itself, nor is it a specific request for data. It is only the *connection definition*. +**Relation:** A datasource is required to execute a **Query** or a **Workflow Node** that interacts with an external system. It is managed by the **Integration Fabric**. + +--- + +### Query +**What it is:** A parameterized operation executed against a specific Datasource, returning structured data. Queries can be parameterized using `{{inputs.param}}`. +**What it is NOT:** A query is not a UI component, nor is it a multi-step background job. It is a single synchronous (from the client's perspective) request/response cycle. +**Relation:** Queries are executed against **Datasources**. The resulting data is bound to **Widgets** using **Template Expressions**. + +--- + +### Workflow +**What it is:** A versioned, directed acyclic graph (DAG) of nodes executed asynchronously by the backend orchestrator. Workflows handle multi-step, long-running, or complex backend logic. +**What it is NOT:** A workflow is not a synchronous frontend operation. It runs in the background and reports progress via WebSockets. +**Relation:** Workflows are composed of **Workflow Nodes**. They can be triggered by **Widget** events, cron schedules, or incoming webhooks. + +--- + +### Workflow Node +**What it is:** An atomic unit of work within a Workflow DAG. Examples include executing a query, transforming data with JavaScript, evaluating conditions, looping, or waiting for human input. +**What it is NOT:** A workflow node is not a standalone executable script; it requires the workflow context and orchestrator to run. +**Relation:** Nodes are connected by edges to form a **Workflow**. They read from and write to the workflow's append-only context log. + +--- + +### Binding / Template Expression +**What it is:** The `{{expression}}` syntax used throughout Jet Admin to inject dynamic values. Expressions are evaluated in a sandboxed JavaScript runtime against the current context (widget states, query data, etc.). +**What it is NOT:** A binding is not full-fledged React code. It is an isolated AST-evaluated expression (e.g., `{{queries.getUsers.data.length > 0}}`). +**Relation:** Bindings connect **Query** results to **Widget** properties, or inject **Widget** state into **Query** parameters. + +--- + +### Integration +**What it is:** A typed, specific connector implemented within Jet Admin's Integration Fabric (e.g., the "PostgreSQL Integration" or the "Stripe Integration"). +**What it is NOT:** An integration is not a specific configured instance; that is a **Datasource**. The integration is the underlying driver logic. +**Relation:** Integrations define the capabilities and UI forms for creating **Datasources**. + +--- + +### ETL Pipeline +**What it is:** A standalone extract-transform-load graph designed specifically for moving and shaping high-volume data between a source and a sink, separate from UI workflows. +**What it is NOT:** An ETL pipeline is not a standard **Workflow**. Workflows are designed for operational logic and orchestration; ETL pipelines are optimized for data ingestion and mapping. +**Relation:** Shares some underlying engine mechanics with Workflows but is functionally distinct, focusing on continuous or batch data synchronization. + +--- + +### Role / Permission +**What it is:** The primitives defining Role-Based Access Control (RBAC). Roles are assigned to users and contain specific permissions determining what actions they can perform and what data they can see. +**What it is NOT:** A simple boolean admin flag. It is a granular matrix of permissions. +**Relation:** Roles govern access to **Apps**, **Pages**, **Datasources**, and **Workflows** within a tenant workspace. diff --git a/docs/docs/developer/creating-datasource.md b/docs/docs/developer/creating-datasource.md index 25d50001..65f1ae3e 100644 --- a/docs/docs/developer/creating-datasource.md +++ b/docs/docs/developer/creating-datasource.md @@ -6,157 +6,161 @@ description: How to add support for a new database or API type # Creating a Custom Datasource -This guide walks through adding support for a new datasource type (e.g., a new database or API). +This guide walks through adding support for a new datasource type (e.g., a new database or SaaS API) to the Jet Admin Integration Fabric. ## Overview Adding a datasource requires changes to three packages: -1. **`datasource-types`**: Define the configuration schema -2. **`datasources-logic`**: Implement the connection driver -3. **`datasources-ui`**: Create the connection form (optional, uses JSON Forms) +1. **`@jet-admin/datasource-types`**: Register the identifier. +2. **`@jet-admin/datasources-logic`**: Implement the backend connection driver. +3. **`@jet-admin/datasources-ui`**: Create the connection form for the frontend. + +--- ## Step 1: Define the Type -In `packages/datasource-types/src/index.js`, add your new type: +In `packages/datasource-types/src/index.js`, add your new type to the main registry. ```javascript +// packages/datasource-types/src/index.js + export const DATASOURCE_TYPES = { // ... existing types CLICKHOUSE: { name: 'ClickHouse', value: 'clickhouse', - icon: 'ClickHouseIcon', - category: 'database' + category: 'database', + description: 'Fast open-source OLAP DBMS' } }; ``` -## Step 2: Create the Driver +--- + +## Step 2: Implement the Driver Logic -In `packages/datasources-logic/src/`, create a new driver file: +The driver handles the actual execution on the backend. Create a new folder in `packages/datasources-logic/src/data-sources/clickhouse/` and implement the `DataSource` interface. ```javascript -// packages/datasources-logic/src/clickhouse/index.js - -export class ClickHouseDriver { - constructor(config) { - this.config = config; +// packages/datasources-logic/src/data-sources/clickhouse/datasource.js +import { ClickHouseClient } from '@clickhouse/client'; + +export class ClickHouseDataSource { + constructor(options, credentials) { + this.options = options; + this.credentials = credentials; + this.client = null; } - async testConnection() { - // Validate connection and return { success: true } or throw error - const client = await this.getClient(); - await client.query('SELECT 1'); - return { success: true }; + async connect() { + if (!this.client) { + this.client = new ClickHouseClient({ + host: this.options.host, + port: this.options.port, + username: this.credentials.username, + password: this.credentials.password, + database: this.options.database + }); + } + return this.client; } - async runQuery(query, params = {}) { - const client = await this.getClient(); - const result = await client.query(query, params); - return { - rows: result.data, - fields: result.columns - }; + // Called when a user clicks "Test Connection" in the UI + async testConnection() { + try { + const client = await this.connect(); + await client.query('SELECT 1').toPromise(); + return { success: true }; + } catch (error) { + return { success: false, error: error.message }; + } } - async getTables() { - const result = await this.runQuery('SHOW TABLES'); - return result.rows.map(row => ({ name: row.name })); - } + // Called when a Query is executed + async execute(queryConfig, evaluatedParams) { + const client = await this.connect(); + // Use parameterization! Do not inject variables directly. + const result = await client.query(queryConfig.sql, { + query_params: evaluatedParams + }).toPromise(); - async getClient() { - // Initialize and return the ClickHouse client - const { ClickHouse } = require('@clickhouse/client'); - return new ClickHouse({ - host: this.config.host, - port: this.config.port, - username: this.config.username, - password: this.config.password, - database: this.config.database - }); + return result; } } ``` ### Register the Driver -In `packages/datasources-logic/src/index.js`: +Expose the driver to the Integration Fabric in `packages/datasources-logic/src/data-sources/index.js`: ```javascript -import { ClickHouseDriver } from './clickhouse'; +import { ClickHouseDataSource } from './clickhouse/datasource'; -export const DRIVERS = { +export const DataSourceDrivers = { // ... existing drivers - clickhouse: ClickHouseDriver + clickhouse: ClickHouseDataSource }; +``` -export function getDriver(type, config) { - const Driver = DRIVERS[type]; - if (!Driver) throw new Error(`Unknown datasource type: ${type}`); - return new Driver(config); -} +--- + +## Step 3: Define the Manifest + +The manifest tells the Query Engine what capabilities your datasource has and provides instructions for the frontend query editor. + +In `packages/datasources-logic/src/data-sources/manifests.js`: + +```javascript +export const DATASOURCE_MANIFESTS = { + // ... + clickhouse: { + name: "ClickHouse", + description: "Execute fast analytical queries against ClickHouse.", + capabilities: ["read", "write"], + queryInstructions: "Write standard ClickHouse SQL. Use {{bindings}} for parameters." + } +}; ``` -## Step 3: Define Configuration Schema +--- + +## Step 4: Create the Configuration UI + +Jet Admin uses JSON Forms (or custom React components) to generate the UI for setting up a connection. -In `packages/datasource-types/src/schemas/clickhouse.js`: +In `packages/datasources-ui/src/components/`, create the form for ClickHouse. It should emit the `options` and `credentials` objects separately, as the backend encrypts `credentials`. ```javascript +// Example schema for your UI component export const clickhouseSchema = { - type: 'object', - required: ['host', 'port', 'database'], - properties: { - host: { - type: 'string', - title: 'Host', - default: 'localhost' - }, - port: { - type: 'number', - title: 'Port', - default: 8123 - }, - database: { - type: 'string', - title: 'Database' - }, - username: { - type: 'string', - title: 'Username' - }, - password: { - type: 'string', - title: 'Password', - format: 'password' - } + options: { + host: 'localhost', + port: 8123, + database: 'default' + }, + credentials: { + username: 'default', + password: '' } }; ``` -## Step 4: Test Your Datasource +Register this form component in the `datasources-ui` index so the frontend router can render it when "ClickHouse" is selected. -1. Rebuild packages: `npm run dev:all-packages` -2. Start the app: `npm run dev:all` -3. Create a new datasource and select your type -4. Test the connection +--- -## Driver Interface +## Step 5: Test Your Datasource -All drivers should implement this interface: +1. Rebuild all packages: `npm run dev:all-packages` +2. Start the backend (`npm run dev` in `apps/backend`) and frontend (`npm run dev` in `apps/frontend`). +3. Navigate to the **Datasources** tab in the Jet Admin UI. +4. Click **New Datasource**, select **ClickHouse**, fill out the credentials, and click **Test Connection**. -```typescript -interface DatasourceDriver { - testConnection(): Promise<{ success: boolean }>; - runQuery(query: string, params?: object): Promise<{ rows: any[], fields: any[] }>; - getTables(): Promise<{ name: string }[]>; - getColumns?(tableName: string): Promise<{ name: string, type: string }[]>; -} -``` +--- ## Best Practices -- **Error Handling**: Wrap errors with meaningful messages -- **Connection Pooling**: Reuse connections where possible -- **Timeouts**: Implement query timeouts to prevent hanging -- **Sanitization**: Never interpolate user input directly into queries +- **Security**: Never log `this.credentials` or include passwords in error messages. +- **Connection Pooling**: If your database library supports connection pooling, use it to prevent exhausting backend resources. +- **Parameterization**: Always use the native parameterization features of the underlying database driver (e.g., `$1`, `?`) when substituting `evaluatedParams`. Never use string concatenation for SQL queries to prevent SQL injection. diff --git a/docs/docs/developer/creating-widget.md b/docs/docs/developer/creating-widget.md index 297ce5f5..0791d367 100644 --- a/docs/docs/developer/creating-widget.md +++ b/docs/docs/developer/creating-widget.md @@ -6,182 +6,162 @@ description: How to add a new visualization widget type # Creating a Custom Widget -This guide explains how to add a new widget type (e.g., a Heatmap, Gauge, or custom visualization). +This guide explains how to add a new widget type (e.g., a Heatmap, Gauge, or custom visualization) to Jet Admin. ## Overview -Widget creation involves three packages: +Widget creation involves updating packages to ensure the UI component, its configuration panel, and its backend properties are recognized by the Jet Admin engine. -1. **`widget-types`**: Define the widget type constant -2. **`widgets-logic`**: Create the data transformer (backend) -3. **`widgets`**: Build the React renderer component (frontend) +1. **`@jet-admin/widget-types`**: Define the widget type, schema, and events. +2. **`@jet-admin/widgets-ui`**: Create the React renderer component and its configuration panel. +3. **`apps/frontend`**: (Optional) Register the widget if it requires app-level integration. -## Step 1: Define the Widget Type +--- + +## Step 1: Define the Widget Schema and Metadata -In `packages/widget-types/src/index.js`: +Open `packages/widget-types/src/index.js` to register your new widget type. This tells Jet Admin what properties the widget accepts and what events it can emit. ```javascript +// packages/widget-types/src/index.js + export const WIDGET_TYPES = { // ... existing types HEATMAP: { - name: 'Heatmap', + label: 'Heatmap', value: 'heatmap', - icon: 'HeatmapIcon', - description: 'Display data intensity across two dimensions' + description: 'Display data intensity across two dimensions', + defaultProps: { + title: 'Activity Map', + data: [], + xField: 'date', + yField: 'hour', + valueField: 'count' + } } }; -``` - -## Step 2: Create the Data Transformer -The transformer converts raw workflow output into the widget's expected format. - -```javascript -// packages/widgets-logic/src/processors/heatmap.processor.js - -export function processHeatmapData(rawData, config) { - const { xField, yField, valueField } = config; - - // Transform to heatmap format: { x, y, value } - return rawData.map(row => ({ - x: row[xField], - y: row[yField], - value: row[valueField] - })); -} +// Add events if your widget emits specific actions (e.g. clicking a cell) +export const WIDGET_EVENT_TYPES = { + // ... + heatmap: [ + { + value: "onCellClick", label: "On Cell Click", desc: "Fires when a heatmap cell is clicked", + inputDefinitions: [ + { key: "event.cell", description: "The clicked cell object" }, + ], + }, + ] +}; ``` -Register in the processor index: - -```javascript -// packages/widgets-logic/src/index.js -import { processHeatmapData } from './processors/heatmap.processor'; +--- -export const PROCESSORS = { - // ... existing - heatmap: processHeatmapData -}; -``` +## Step 2: Create the React Component -## Step 3: Create the React Component +In the `@jet-admin/widgets-ui` package, create the visual component that users will see on the canvas. ```jsx -// packages/widgets/src/heatmap/HeatmapWidget.jsx +// packages/widgets-ui/src/heatmap/HeatmapWidget.jsx import React from 'react'; import { ResponsiveHeatMap } from '@nivo/heatmap'; // Example library -export function HeatmapWidget({ data, config }) { +// The wrapper automatically evaluates bindings in props before passing them here +export function HeatmapWidget({ id, title, data, xField, yField, valueField, onEvent }) { if (!data || data.length === 0) { return
No data available
; } + // Transform flat array to Heatmap format if necessary + const processedData = React.useMemo(() => { + // ... transformation logic + return data; + }, [data]); + return ( -
+
+ {title &&

{title}

} onEvent('onCellClick', { cell })} />
); } ``` -Export from package: +--- -```javascript -// packages/widgets/src/index.js -export { HeatmapWidget } from './heatmap/HeatmapWidget'; -``` +## Step 3: Create the Configuration UI -## Step 4: Create Configuration UI (Optional) +Jet Admin uses a configuration schema to automatically build the property panel on the right sidebar. -In `packages/widgets-ui/src/heatmap/`: +In `packages/widgets-ui/src/widget.config.js` (or a specific config file), define the property panel sections: -```jsx -// HeatmapConfig.jsx -export function HeatmapConfig({ config, onChange }) { - return ( -
- - - - - - - - -
- ); -} +```javascript +// packages/widgets-ui/src/heatmap/heatmap.config.js + +export const heatmapPropertyPanel = { + type: "Categorization", + elements: [ + { + type: "Category", + label: "Data", + elements: [ + { type: "Control", scope: "#/properties/data" }, + { type: "Control", scope: "#/properties/xField" }, + { type: "Control", scope: "#/properties/yField" }, + { type: "Control", scope: "#/properties/valueField" }, + ] + }, + { + type: "Category", + label: "Events", + elements: [ + { type: "Control", scope: "#/properties/onCellClick" }, + ] + } + ] +}; ``` -## Step 5: Register in Widget Factory +--- + +## Step 4: Register in the Widget Map -In the frontend app, register the widget renderer: +Ensure the Layout Engine can find your new component. In `packages/widgets-ui/src/widget.map.js`: ```javascript -// apps/frontend/src/presentation/components/widgets/WidgetRenderer.jsx +// packages/widgets-ui/src/widget.map.js -import { HeatmapWidget } from '@jet-admin/widgets'; +import { HeatmapWidget } from './heatmap/HeatmapWidget'; +import { heatmapPropertyPanel } from './heatmap/heatmap.config'; -const WIDGET_COMPONENTS = { - // ... existing - heatmap: HeatmapWidget +export const WIDGET_MAP = { + // ... existing widgets + heatmap: { + component: HeatmapWidget, + configPanel: heatmapPropertyPanel + } }; - -export function WidgetRenderer({ type, data, config }) { - const Component = WIDGET_COMPONENTS[type]; - if (!Component) return
Unknown widget type
; - return ; -} ``` -## Widget Interface +--- + +## Step 5: Test in the Builder -All widget components should accept these props: +1. If you haven't already, start the package watcher: `npm run dev:all-packages` +2. Start the frontend: `npm run dev` in `apps/frontend`. +3. Open a Page in Edit mode. You should see "Heatmap" in the Add Widget sidebar. +4. Drag it onto the canvas, configure the data binding, and test the `onCellClick` event. -```typescript -interface WidgetProps { - data: any[]; // Processed data from workflow - config: { // User configuration - title?: string; - [key: string]: any; - }; - isLoading?: boolean; - error?: string; -} -``` +--- ## Best Practices -- **Responsive Design**: Use percentage-based sizing -- **Loading States**: Show skeleton/spinner while loading -- **Empty States**: Display helpful message when no data -- **Error Handling**: Gracefully handle malformed data -- **Accessibility**: Include ARIA labels and keyboard navigation +- **Responsive Design**: Always use `width: 100%` and `height: 100%` on the root element of your widget so it fills the Layout Engine's grid cells. +- **Error Boundaries**: If your widget uses an external library that might crash on bad data, wrap it in a `try/catch` or an Error Boundary. +- **Empty States**: Display a helpful message (e.g., "Connect data to view heatmap") when `data` is empty, rather than rendering a blank square. +- **Event Forwarding**: Always use the `onEvent(eventName, payload)` prop provided by the WidgetContainer to ensure events trigger Jet Admin actions. diff --git a/docs/docs/developer/creating-workflow-node.md b/docs/docs/developer/creating-workflow-node.md index 4453dec4..49301694 100644 --- a/docs/docs/developer/creating-workflow-node.md +++ b/docs/docs/developer/creating-workflow-node.md @@ -6,32 +6,34 @@ description: How to add a new node type to the workflow builder # Creating a Custom Workflow Node -This guide shows how to add a new node type to the visual workflow builder (e.g., an Email Node, Webhook Node, or AI Node). +This guide shows how to add a new node type to Jet Admin's visual workflow builder and execution engine (e.g., an Email Node, Webhook Node, or AI Node). ## Overview -Workflow nodes consist of: +A Workflow Node requires two parts: +1. **Frontend Component**: A React Flow node UI and configuration panel in `@jet-admin/workflow-nodes`. +2. **Backend Executor**: The logic that runs inside the pg-boss worker in `apps/backend/modules/workflow/handlers/`. -1. **Frontend Component**: React Flow node UI (`packages/workflow-nodes/`) -2. **Backend Executor**: Node processing logic (`apps/backend/modules/workflow/`) +--- + +## Step 1: Define and Register the Frontend Node -## Step 1: Create the Node Component +In `packages/workflow-nodes/src/nodes/`, create the visual representation of your node on the DAG canvas. ```jsx // packages/workflow-nodes/src/nodes/EmailNode.jsx import React from 'react'; import { Handle, Position } from 'reactflow'; -import { EmailIcon } from '@heroicons/react/24/outline'; export function EmailNode({ data, selected }) { return (
+ {/* Input Handle */}
- - Send Email + 📧 Send Email
@@ -40,172 +42,130 @@ export function EmailNode({ data, selected }) {

+ {/* Output Handle */}
); } - -// Node type metadata -EmailNode.nodeType = 'email'; -EmailNode.displayName = 'Send Email'; -EmailNode.category = 'integrations'; -EmailNode.defaultData = { - to: '', - subject: '', - body: '' -}; ``` -Register the node: +### Map the Node -```javascript -// packages/workflow-nodes/src/index.js -export { EmailNode } from './nodes/EmailNode'; +In `packages/workflow-nodes/src/map.js`, register the node type, schema, and its configuration UI form. -export const NODE_TYPES = { - // ... existing - email: EmailNode +```javascript +// packages/workflow-nodes/src/map.js +import { EmailNode } from './nodes/EmailNode'; +import { EmailNodeConfigurator } from './configs/EmailNodeConfigurator'; + +export const WORKFLOW_NODE_MAP = { + // ... existing nodes + email: { + label: 'Send Email', + value: 'email', + component: EmailNode, + configurator: EmailNodeConfigurator, // The right-sidebar form + defaultValue: { + title: "Send Email", + to: "", + subject: "", + body: "" + }, + // Used to generate the property panel + schema: { + type: "object", + properties: { + to: { type: "string", title: "Recipient Email" }, + subject: { type: "string", title: "Subject" }, + body: { type: "string", title: "Body" } + } + } + } }; ``` -## Step 2: Create Node Config Panel +--- -For the sidebar that appears when a node is selected: +## Step 2: Implement the Backend Executor -```jsx -// packages/workflow-nodes/src/configs/EmailNodeConfig.jsx +When the Orchestrator reaches your node, it passes the job to a worker. The worker looks up the handler by the node's type. -export function EmailNodeConfig({ data, onChange }) { - return ( -
- - onChange({ ...data, to: e.target.value })} - placeholder="recipient@example.com" - /> - - - onChange({ ...data, subject: e.target.value })} - placeholder="Email subject" - /> - - -