feat(core): add FILE instrument kind and storage schemas

.env.template

@@ -19,6 +19,8 @@ GATEWAY_PORT=3500
 RELEASE_CHANNEL=latest
 # The MongoDB version to use
 MONGODB_VERSION=7.0
+# The RustFS (S3-compatible storage) version to use
+RUSTFS_VERSION=latest
 
 ## ---------------------------------
 ## PRODUCTION + DEVELOPMENT
@@ -66,6 +68,30 @@ LOGIN_REQUEST_THROTTLER_LIMIT=
 # the duration in milliseconds to enforce LOGIN_REQUEST_THROTTLER_LIMIT (default: 60,000)
 LOGIN_REQUEST_THROTTLER_TTL=
 
+# Enable S3-compatible object storage, required for file instruments. When set to true, all
+# of STORAGE_ACCESS_KEY, STORAGE_SECRET_KEY, STORAGE_BUCKET and STORAGE_ENDPOINT must be set.
+# When false, file instruments are unavailable and the variables below are ignored.
+STORAGE_ENABLED=true
+# Storage service internal endpoint used by the backend/server for S3-compatible API calls
+STORAGE_ENDPOINT=http://localhost:9000
+# Public endpoint used in generated download/file URLs returned to users (the browser
+# uploads/downloads directly to these presigned URLs).
+# In the production compose stack this may be left blank: the api service defaults it to
+# http://localhost:${APP_PORT}/storage, which Caddy proxies to rustfs (see Caddyfile).
+# Set it explicitly to the externally accessible storage URL for a real deployment
+# (e.g. https://your-domain.com/storage).
+STORAGE_PUBLIC_ENDPOINT=
+# Access key for the S3 service (generated by scripts/generate-env.sh; also used as the
+# rustfs admin access key in the production compose stack)
+STORAGE_ACCESS_KEY=
+# Secret key for the S3 service (generated by scripts/generate-env.sh; also used as the
+# rustfs admin secret key in the production compose stack)
+STORAGE_SECRET_KEY=
+# Default bucket/container name where files are stored
+STORAGE_BUCKET=open-data-capture
+# Storage region name (required, can be set to anything if using self-hosted S3)
+STORAGE_REGION=us-east-1
+
 # Disable iteration for password hashing (not recommended for production)
 # See https://pages.nist.gov/800-63-3/sp800-63b.html
 # DANGEROUSLY_DISABLE_PBKDF2_ITERATION=

.gitignore

@@ -61,6 +61,7 @@ apps/outreach/src/content/docs/en/runtime-core-docs/
 
 # docker dbs
 mongo/
+rustfs/
 sqlite/
 
 # runtime core intermediate build

Caddyfile

@@ -5,6 +5,12 @@
 		reverse_proxy api
 	}
 
+	handle_path /storage/* {
+		reverse_proxy rustfs:9000 {
+			header_up Host rustfs:9000
+		}
+	}
+
 	handle {
 		reverse_proxy web
 	}

apps/api/Dockerfile

@@ -6,7 +6,7 @@ ENV PNPM_HOME="/pnpm"
 ENV PATH="$PNPM_HOME:$PNPM_HOME/bin:$PATH"
 ENV NODE_OPTIONS="--max-old-space-size=8192"
 RUN corepack enable
-RUN pnpm install -g turbo@latest
+RUN pnpm install -g turbo@2.9.16
 
 # PRUNE WORKSPACE
 # Note: Here we cannot use --docker, as is recommended, since the generated

apps/api/package.json

@@ -15,6 +15,8 @@
     "test": "env-cmd -f ../../.env vitest"
   },
   "dependencies": {
+    "@aws-sdk/client-s3": "^3.1044.0",
+    "@aws-sdk/s3-request-presigner": "^3.1044.0",
     "@casl/ability": "^6.7.5",
     "@casl/prisma": "^1.5.1",
     "@douglasneuroinformatics/libcrypto": "catalog:",

apps/api/prisma/schema.prisma

@@ -103,53 +103,72 @@ type GroupSettings {
 }
 
 model Group {
-  createdAt               DateTime           @default(now()) @db.Date
-  updatedAt               DateTime           @updatedAt @db.Date
-  id                      String             @id @default(auto()) @map("_id") @db.ObjectId
+  createdAt               DateTime               @default(now()) @db.Date
+  updatedAt               DateTime               @updatedAt @db.Date
+  id                      String                 @id @default(auto()) @map("_id") @db.ObjectId
   accessibleInstrumentIds String[]
-  accessibleInstruments   Instrument[]       @relation(fields: [accessibleInstrumentIds], references: [id])
+  accessibleInstruments   Instrument[]           @relation(fields: [accessibleInstrumentIds], references: [id])
   assignments             Assignment[]
   auditLogs               AuditLog[]
   instrumentRecords       InstrumentRecord[]
-  name                    String             @unique
+  instrumentRecordFiles   InstrumentRecordFile[]
+  name                    String                 @unique
   settings                GroupSettings
   sessions                Session[]
-  subjects                Subject[]          @relation(fields: [subjectIds], references: [id])
+  subjects                Subject[]              @relation(fields: [subjectIds], references: [id])
   subjectIds              String[]
   type                    GroupType
-  userIds                 String[]           @db.ObjectId
-  users                   User[]             @relation(fields: [userIds], references: [id])
+  userIds                 String[]               @db.ObjectId
+  users                   User[]                 @relation(fields: [userIds], references: [id])
 
   @@map("GroupModel")
 }
 
 /// Instrument Records
 
 model InstrumentRecord {
-  createdAt        DateTime    @default(now()) @db.Date
-  updatedAt        DateTime    @updatedAt @db.Date
-  id               String      @id @default(auto()) @map("_id") @db.ObjectId
+  createdAt        DateTime               @default(now()) @db.Date
+  updatedAt        DateTime               @updatedAt @db.Date
+  id               String                 @id @default(auto()) @map("_id") @db.ObjectId
   /// [ComputedMeasures]
   computedMeasures Json?
   data             Json?
-  date             DateTime    @db.Date
-  group            Group?      @relation(fields: [groupId], references: [id])
-  groupId          String?     @db.ObjectId
-  subject          Subject     @relation(fields: [subjectId], references: [id])
+  date             DateTime               @db.Date
+  files            InstrumentRecordFile[]
+  group            Group?                 @relation(fields: [groupId], references: [id])
+  groupId          String?                @db.ObjectId
+  subject          Subject                @relation(fields: [subjectId], references: [id])
   subjectId        String
-  instrument       Instrument  @relation(fields: [instrumentId], references: [id])
+  instrument       Instrument             @relation(fields: [instrumentId], references: [id])
   instrumentId     String
-  assignment       Assignment? @relation(fields: [assignmentId], references: [id])
-  assignmentId     String?     @unique
-  session          Session     @relation(fields: [sessionId], references: [id])
-  sessionId        String      @db.ObjectId
+  assignment       Assignment?            @relation(fields: [assignmentId], references: [id])
+  assignmentId     String?                @unique
+  session          Session                @relation(fields: [sessionId], references: [id])
+  sessionId        String                 @db.ObjectId
+  pending          Boolean?
 
   @@map("InstrumentRecordModel")
 }
 
-// Instruments 
+model InstrumentRecordFile {
+  basename  String
+  createdAt DateTime         @default(now()) @db.Date
+  group     Group?           @relation(fields: [groupId], references: [id])
+  groupId   String?          @db.ObjectId
+  id        String           @id @default(auto()) @map("_id") @db.ObjectId
+  index     Int
+  name      String
+  record    InstrumentRecord @relation(fields: [recordId], references: [id])
+  recordId  String           @db.ObjectId
+  size      Int
+
+  @@map("InstrumentRecordFileModel")
+}
+
+// Instruments
 
 enum InstrumentKind {
+  FILE
   FORM
   INTERACTIVE
   SERIES

apps/api/src/auth/__tests__/ability.factory.test.ts

@@ -66,4 +66,41 @@ describe('AbilityFactory', () => {
     expect(ability.can('read', subject('User', { id: 'user-1' }) as any)).toBe(true);
     expect(ability.can('read', subject('User', { id: 'user-2' }) as any)).toBe(false);
   });
+
+  it('should scope standard user instrument record file uploads to their groups', () => {
+    const payload = {
+      additionalPermissions: undefined,
+      basePermissionLevel: 'STANDARD',
+      firstName: 'Test',
+      groups: [{ id: 'group-1' }],
+      id: 'user-1',
+      lastName: 'User',
+      permissions: [] as any,
+      username: 'standard-user'
+    };
+
+    const ability = abilityFactory.createForPayload(payload as any);
+
+    expect(ability.can('create', subject('InstrumentRecordFile', { groupId: 'group-1' }) as any)).toBe(true);
+    expect(ability.can('create', subject('InstrumentRecordFile', { groupId: 'group-2' }) as any)).toBe(false);
+    expect(ability.can('read', subject('InstrumentRecordFile', { groupId: 'group-1' }) as any)).toBe(false);
+  });
+
+  it('should scope group manager instrument record file uploads to their groups', () => {
+    const payload = {
+      additionalPermissions: undefined,
+      basePermissionLevel: 'GROUP_MANAGER',
+      firstName: 'Test',
+      groups: [{ id: 'group-1' }],
+      id: 'user-1',
+      lastName: 'User',
+      permissions: [] as any,
+      username: 'manager-user'
+    };
+
+    const ability = abilityFactory.createForPayload(payload as any);
+
+    expect(ability.can('create', subject('InstrumentRecordFile', { groupId: 'group-1' }) as any)).toBe(true);
+    expect(ability.can('create', subject('InstrumentRecordFile', { groupId: 'group-2' }) as any)).toBe(false);
+  });
 });

apps/api/src/auth/__tests__/ability.utils.test.ts

@@ -1,6 +1,6 @@
 import { describe, expect, it, vi } from 'vitest';
 
-import { accessibleQuery } from '../ability.utils';
+import { accessibleQuery, createAppAbility, forcedAppSubject } from '../ability.utils';
 
 const accessibleBy = vi.hoisted(() => vi.fn());
 
@@ -22,3 +22,22 @@ describe('accessibleQuery', () => {
     expect(accessibleBy).toHaveBeenCalledExactlyOnceWith(ability, 'manage');
   });
 });
+
+describe('forcedAppSubject', () => {
+  it('should limit access by groupId when ability is scoped to a group', () => {
+    const ability = createAppAbility([
+      { action: 'create', conditions: { groupId: 'group-1' }, subject: 'InstrumentRecordFile' }
+    ]);
+
+    expect(ability.can('create', forcedAppSubject('InstrumentRecordFile', { groupId: 'group-1' }))).toBe(true);
+    expect(ability.can('create', forcedAppSubject('InstrumentRecordFile', { groupId: 'group-2' }))).toBe(false);
+  });
+
+  it('should deny access when no groupId is provided and ability requires one', () => {
+    const ability = createAppAbility([
+      { action: 'create', conditions: { groupId: 'group-1' }, subject: 'InstrumentRecordFile' }
+    ]);
+
+    expect(ability.can('create', forcedAppSubject('InstrumentRecordFile', {}))).toBe(false);
+  });
+});

apps/api/src/auth/ability.factory.ts

@@ -28,7 +28,9 @@ export class AbilityFactory {
         ability.can('manage', 'Group', { id: { in: groupIds } });
         ability.can('read', 'Instrument');
         ability.can('create', 'InstrumentRecord');
+        ability.can('create', 'InstrumentRecordFile', { groupId: { in: groupIds } });
         ability.can('read', 'InstrumentRecord', { groupId: { in: groupIds } });
+        ability.can('read', 'InstrumentRecordFile', { groupId: { in: groupIds } });
         ability.can('create', 'Session');
         ability.can('read', 'Session', { groupId: { in: groupIds } });
         ability.can('create', 'Subject');
@@ -39,6 +41,7 @@ export class AbilityFactory {
         ability.can('read', 'Group', { id: { in: groupIds } });
         ability.can('read', 'Instrument');
         ability.can('create', 'InstrumentRecord');
+        ability.can('create', 'InstrumentRecordFile', { groupId: { in: groupIds } });
         ability.can('read', 'Session', { groupId: { in: groupIds } });
         ability.can('create', 'Session');
         ability.can('create', 'Subject');

apps/api/src/auth/ability.utils.ts

@@ -1,12 +1,12 @@
-import { detectSubjectType } from '@casl/ability';
+import { detectSubjectType, subject } from '@casl/ability';
 import { createPrismaAbility } from '@casl/prisma';
 import type { PrismaQuery } from '@casl/prisma';
 import { createAccessibleByFactory } from '@casl/prisma/runtime';
 import type { AppSubject, Prisma } from '@prisma/client';
 
 import type { PrismaModelWhereInputMap } from '@/core/prisma';
 
-import type { AppAbilities, AppAbility, AppAction, Permission } from './auth.types';
+import type { AppAbilities, AppAbility, AppAction, AppSubjectModels, AppSubjectName, Permission } from './auth.types';
 
 const accessibleBy = createAccessibleByFactory<PrismaModelWhereInputMap, PrismaQuery>();
 
@@ -17,6 +17,16 @@ export function detectAppSubject(obj: { [key: string]: any }) {
   return detectSubjectType(obj) as AppSubject;
 }
 
+export function forcedAppSubject<TSubjectName extends Exclude<AppSubjectName, 'all'>>(
+  name: TSubjectName,
+  obj: Partial<AppSubjectModels[TSubjectName]>
+) {
+  return subject(name, {
+    __modelName: name,
+    ...obj
+  } as unknown as AppSubjectModels[TSubjectName]);
+}
+
 export function createAppAbility(permissions: Permission[]): AppAbility {
   return createPrismaAbility<AppAbilities>(permissions, {
     detectSubjectType: detectAppSubject

apps/api/src/auth/auth.types.ts

@@ -6,18 +6,20 @@ import type { DefaultSelection } from '@prisma/client/runtime/library';
 
 type AppAction = 'create' | 'delete' | 'manage' | 'read' | 'update';
 
-type AppSubjects =
-  | 'all'
-  | Subjects<{
-      [K in keyof Prisma.TypeMap['model']]: DefaultSelection<Prisma.TypeMap['model'][K]['payload']>;
-    }>;
+type AppSubjectModels = {
+  [K in keyof Prisma.TypeMap['model']]: DefaultSelection<Prisma.TypeMap['model'][K]['payload']>;
+};
+
+type AppSubjects = 'all' | Subjects<AppSubjectModels>;
 
 type AppSubjectName = Extract<AppSubjects, string>;
 
+type AppSubjectModel = Extract<AppSubjects, object>;
+
 type AppAbilities = [AppAction, AppSubjects];
 
 type AppAbility = PureAbility<AppAbilities, PrismaQuery>;
 
 type Permission = RawRuleOf<PureAbility<[AppAction, AppSubjectName], PrismaQuery>>;
 
-export type { AppAbilities, AppAbility, AppAction, AppSubjectName, Permission };
+export type { AppAbilities, AppAbility, AppAction, AppSubjectModel, AppSubjectModels, AppSubjectName, Permission };

apps/api/src/core/schemas/env.schema.ts

@@ -13,7 +13,14 @@ export const $Env = $BaseEnv
     GATEWAY_ENABLED: $BooleanLike,
     GATEWAY_INTERNAL_NETWORK_URL: $UrlLike.optional(),
     GATEWAY_REFRESH_INTERVAL: $NumberLike.pipe(z.number().positive().int()),
-    GATEWAY_SITE_ADDRESS: $UrlLike.optional()
+    GATEWAY_SITE_ADDRESS: $UrlLike.optional(),
+    STORAGE_ACCESS_KEY: z.string().min(1).optional(),
+    STORAGE_BUCKET: z.string().min(1).optional(),
+    STORAGE_ENABLED: $BooleanLike.optional(),
+    STORAGE_ENDPOINT: z.url().optional(),
+    STORAGE_PUBLIC_ENDPOINT: z.url().optional(),
+    STORAGE_REGION: z.string().optional(),
+    STORAGE_SECRET_KEY: z.string().min(1).optional()
   })
   .transform((env, ctx) => {
     if (env.NODE_ENV === 'production') {
@@ -31,5 +38,16 @@ export const $Env = $BaseEnv
         });
       }
     }
+    if (env.STORAGE_ENABLED) {
+      for (const key of ['STORAGE_ACCESS_KEY', 'STORAGE_BUCKET', 'STORAGE_ENDPOINT', 'STORAGE_SECRET_KEY'] as const) {
+        if (!env[key]) {
+          ctx.addIssue({
+            code: z.ZodIssueCode.custom,
+            message: `${key} must be defined when STORAGE_ENABLED is true`,
+            path: [key]
+          });
+        }
+      }
+    }
     return { ...env, API_PORT: env.API_DEV_SERVER_PORT ?? 80 };
   });

apps/api/src/demo/demo.service.ts

@@ -3,6 +3,8 @@ import { InjectPrismaClient, LoggingService } from '@douglasneuroinformatics/lib
 import { faker } from '@faker-js/faker';
 import { Injectable, InternalServerErrorException } from '@nestjs/common';
 import { DEMO_GROUPS, DEMO_USERS } from '@opendatacapture/demo';
+import arbitrarySingleFileInstrument from '@opendatacapture/instrument-library/file/ARBITRARY_SINGLE_FILE.js';
+import mriScanSessionInstrument from '@opendatacapture/instrument-library/file/MRI_SCAN_SESSION.js';
 import enhancedDemographicsQuestionnaire from '@opendatacapture/instrument-library/forms/DNP_ENHANCED_DEMOGRAPHICS_QUESTIONNAIRE.js';
 import generalConsentForm from '@opendatacapture/instrument-library/forms/DNP_GENERAL_CONSENT_FORM.js';
 import happinessQuestionnaire from '@opendatacapture/instrument-library/forms/DNP_HAPPINESS_QUESTIONNAIRE.js';
@@ -72,6 +74,12 @@ export class DemoService {
       await this.instrumentsService.create({ bundle: happinessQuestionnaireWithConsent });
       this.loggingService.debug('Done creating series instruments');
 
+      await Promise.all([
+        this.instrumentsService.create({ bundle: arbitrarySingleFileInstrument }),
+        this.instrumentsService.create({ bundle: mriScanSessionInstrument })
+      ]);
+      this.loggingService.debug('Done creating file instruments');
+
       const groups: (Group & { dummyIdPrefix?: string })[] = [];
       for (const group of DEMO_GROUPS) {
         const { dummyIdPrefix, ...createGroupData } = group;

apps/api/src/instrument-records/__tests__/instrument-records.service.spec.ts

@@ -6,6 +6,7 @@ import { ForbiddenException, NotFoundException } from '@nestjs/common';
 import { Test } from '@nestjs/testing';
 import { beforeEach, describe, expect, it } from 'vitest';
 
+import { StorageService } from '@/storage/storage.service';
 import { UsersService } from '@/users/users.service';
 
 import { GroupsService } from '../../groups/groups.service';
@@ -35,6 +36,7 @@ describe('InstrumentRecordsService', () => {
         MockFactory.createForService(InstrumentMeasuresService),
         MockFactory.createForService(InstrumentsService),
         MockFactory.createForService(SessionsService),
+        MockFactory.createForService(StorageService),
         MockFactory.createForService(SubjectsService)
       ]
     }).compile();